From 1394dcf579849e5d8103c31556e9af0216a875d2 Mon Sep 17 00:00:00 2001 From: jiangpengfei Date: Wed, 19 Aug 2020 17:15:51 +0800 Subject: [PATCH 13/16] agent: support setting multi queues of interface reason: support setting multi queues of a interface when runtime passing Queue in the request. Signed-off-by: jiangpengfei --- network.go | 12 +++++- pkg/net/ethtool.go | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++ pkg/types/types.pb.go | 82 ++++++++++++++++++++++++++----------- pkg/types/types.proto | 1 + 4 files changed, 181 insertions(+), 24 deletions(-) create mode 100644 pkg/net/ethtool.go diff --git a/network.go b/network.go index 7046cf8..1baaa2e 100644 --- a/network.go +++ b/network.go @@ -17,7 +17,7 @@ import ( "syscall" "golang.org/x/sys/unix" - + agentNet "github.com/kata-containers/agent/pkg/net" "github.com/kata-containers/agent/pkg/types" pb "github.com/kata-containers/agent/protocols/grpc" "github.com/sirupsen/logrus" @@ -273,6 +273,16 @@ func (s *sandbox) updateInterface(netHandle *netlink.Handle, iface *types.Interf if err == nil { err = retErr } + + // if link is up, then set the multi queue to it, + // the kernel of newer version may set multi queue by itself, + // but we can not rely on it. + if err == nil && iface.Queues > 0 { + if ethErr := agentNet.GetEthtool().SetChannel(iface.Name, iface.Queues); ethErr != nil { + err = grpcStatus.Errorf(codes.Internal, "Could not set multi queue %d for interface %v: %v", + iface.Queues, link, ethErr) + } + } }() fieldLogger.WithField("link", fmt.Sprintf("%+v", link)).Info("Link found") diff --git a/pkg/net/ethtool.go b/pkg/net/ethtool.go new file mode 100644 index 0000000..56a1ece --- /dev/null +++ b/pkg/net/ethtool.go @@ -0,0 +1,110 @@ +/* +Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +Description: common functions +Author: fengshaobao +Create: 2019-05-28 +*/ + +package net + +import ( + "fmt" + "sync" + "syscall" + "unsafe" + + "github.com/sirupsen/logrus" +) + +const ( + ethtoolGChannels = 0x0000003c /* Get no of channels */ + ethtoolSChannels = 0x0000003d /* Set no of channels */ + ifNameSize = 16 + siocEthtool = 0x8946 +) + +var ( + t *Ethtool + once sync.Once +) + +// Ethtool to set multiqueue of a network interface, +// with the public method "SetChannel" to set queues of an interface. +type Ethtool struct { + fd int +} + +type ifReq struct { + ifrName [ifNameSize]byte + ifrData uintptr +} + +type ethtoolChannels struct { + // ETHTOOL_{G,S}CHANNELS + cmd uint32 + // Read only. Maximum number of receive channel the driver support. + maxRx uint32 + // Read only. Maximum number of transmit channel the driver support + maxTx uint32 + // Read only. Maximum number of other channel the driver support + maxOther uint32 + //Read only. Maximum number of combined channel the driver support. Set of queues RX, TX or other + maxCombined uint32 + // Valid values are in the range 1 to the max_rx + rxCount uint32 + // Valid values are in the range 1 to the max_tx + txCount uint32 + // Valid values are in the range 1 to the max_other + otherCount uint32 + // Valid values are in the range 1 to the max_combined + combinedCount uint32 +} + +// GetEthtool to config multiqueue of a network interface +func GetEthtool() *Ethtool { + once.Do(func() { + var err error + if t, err = newEthtool(); err != nil { + panic("can not init a socket fd for ethtool") + } + }) + return t +} + +func newEthtool() (*Ethtool, error) { + fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP) + if err != nil || fd < 0 { + logrus.Warningf("Can not get socket of inet") + var newErr error + fd, newErr = syscall.Socket(syscall.AF_NETLINK, syscall.SOCK_RAW, syscall.NETLINK_GENERIC) + if newErr != nil || fd < 0 { + return nil, fmt.Errorf("create inet socket with error: %v, and create netlink socket with error: %v", err, newErr) + } + } + + return &Ethtool{ + fd: int(fd), + }, nil +} + +// SetChannel Set the queues of a network interface. +// @devName: the network interface name, e.g. eth0. +// @queues: queues for the network interface. +func (e *Ethtool) SetChannel(devName string, queues uint32) error { + c := ðtoolChannels{ + cmd: ethtoolSChannels, + combinedCount: queues, + } + var ifName [ifNameSize]byte + copy(ifName[:], []byte(devName)) + ifr := ifReq{ + ifrName: ifName, + ifrData: uintptr(unsafe.Pointer(&c)), + } + _, _, ep := syscall.Syscall(syscall.SYS_IOCTL, uintptr(e.fd), siocEthtool, uintptr(unsafe.Pointer(&ifr))) + if ep != 0 { + return syscall.Errno(ep) + } + return nil +} diff --git a/pkg/types/types.pb.go b/pkg/types/types.pb.go index 7ea63e3..8b7e2a5 100644 --- a/pkg/types/types.pb.go +++ b/pkg/types/types.pb.go @@ -100,6 +100,7 @@ type Interface struct { // list: "veth", "macvtap", "vlan", "macvlan", "tap", ... Type string `protobuf:"bytes,7,opt,name=type,proto3" json:"type,omitempty"` RawFlags uint32 `protobuf:"varint,8,opt,name=raw_flags,json=rawFlags,proto3" json:"raw_flags,omitempty"` + Queues uint32 `protobuf:"varint,9,opt,name=Queues,proto3" json:"Queues,omitempty"` } func (m *Interface) Reset() { *m = Interface{} } @@ -163,6 +164,13 @@ func (m *Interface) GetRawFlags() uint32 { return 0 } +func (m *Interface) GetQueues() uint32 { + if m != nil { + return m.Queues + } + return 0 +} + type Route struct { Dest string `protobuf:"bytes,1,opt,name=dest,proto3" json:"dest,omitempty"` Gateway string `protobuf:"bytes,2,opt,name=gateway,proto3" json:"gateway,omitempty"` @@ -319,6 +327,11 @@ func (m *Interface) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintTypes(dAtA, i, uint64(m.RawFlags)) } + if m.Queues != 0 { + dAtA[i] = 0x48 + i++ + i = encodeVarintTypes(dAtA, i, uint64(m.Queues)) + } return i, nil } @@ -430,6 +443,9 @@ func (m *Interface) Size() (n int) { if m.RawFlags != 0 { n += 1 + sovTypes(uint64(m.RawFlags)) } + if m.Queues != 0 { + n += 1 + sovTypes(uint64(m.Queues)) + } return n } @@ -841,6 +857,25 @@ func (m *Interface) Unmarshal(dAtA []byte) error { break } } + case 9: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Queues", wireType) + } + m.Queues = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Queues |= (uint32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipTypes(dAtA[iNdEx:]) @@ -1155,27 +1190,28 @@ var ( func init() { proto.RegisterFile("pkg/types/types.proto", fileDescriptorTypes) } var fileDescriptorTypes = []byte{ - // 345 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x91, 0xd1, 0x6a, 0xfa, 0x30, - 0x14, 0xc6, 0xff, 0xb1, 0xb6, 0xda, 0xe3, 0xdf, 0xad, 0x84, 0x4d, 0xc2, 0x06, 0x52, 0xbc, 0x59, - 0xd9, 0x85, 0x03, 0x37, 0x76, 0xbf, 0x5d, 0x08, 0xde, 0x49, 0x5e, 0x60, 0xc4, 0x36, 0x3a, 0xd1, - 0xda, 0xd2, 0x44, 0x8b, 0xec, 0x05, 0x77, 0xb9, 0x47, 0x10, 0x9f, 0x64, 0xe4, 0x24, 0x8a, 0xbb, - 0xd1, 0xef, 0x97, 0x93, 0xd3, 0xef, 0x7c, 0x27, 0x70, 0x5b, 0xae, 0x16, 0x4f, 0x7a, 0x5f, 0x4a, - 0x65, 0x7f, 0x87, 0x65, 0x55, 0xe8, 0x82, 0xfa, 0x08, 0x83, 0x19, 0x84, 0x93, 0xe9, 0x5b, 0x96, - 0x55, 0x52, 0x29, 0xfa, 0x00, 0xc1, 0x5c, 0xe4, 0xcb, 0xf5, 0x9e, 0x91, 0x98, 0x24, 0x57, 0xa3, - 0xeb, 0xa1, 0xed, 0x98, 0x4c, 0xc7, 0x78, 0xcc, 0x5d, 0x99, 0x32, 0x68, 0x09, 0xdb, 0xc3, 0x1a, - 0x31, 0x49, 0x42, 0x7e, 0x42, 0x4a, 0xa1, 0x99, 0x0b, 0xb5, 0x62, 0x1e, 0x1e, 0xa3, 0x1e, 0x1c, - 0x08, 0x84, 0x93, 0x8d, 0x96, 0xd5, 0x5c, 0xa4, 0x92, 0xf6, 0x20, 0xc8, 0xe4, 0x6e, 0x99, 0x4a, - 0x34, 0x09, 0xb9, 0x23, 0xd3, 0xb9, 0x11, 0xb9, 0x74, 0x1f, 0x44, 0x4d, 0x47, 0xd0, 0x39, 0x4f, - 0x27, 0x15, 0xf3, 0x62, 0x2f, 0xe9, 0x8c, 0xa2, 0xf3, 0x54, 0xae, 0xc2, 0x2f, 0x2f, 0xd1, 0x08, - 0xbc, 0x5c, 0x6f, 0x59, 0x33, 0x26, 0x49, 0x93, 0x1b, 0x69, 0x1c, 0x3f, 0x6b, 0x73, 0x81, 0xf9, - 0xd6, 0xd1, 0x92, 0x49, 0x51, 0xa6, 0x4b, 0x2c, 0x04, 0x36, 0x85, 0x43, 0x33, 0x8b, 0xf1, 0x60, - 0x2d, 0x3b, 0x8b, 0xd1, 0xf4, 0x1e, 0xc2, 0x4a, 0xd4, 0x1f, 0xf3, 0xb5, 0x58, 0x28, 0xd6, 0x8e, - 0x49, 0xd2, 0xe5, 0xed, 0x4a, 0xd4, 0x63, 0xc3, 0x83, 0x2f, 0xf0, 0x79, 0xb1, 0xd5, 0x98, 0x22, - 0x93, 0x4a, 0xbb, 0x6c, 0xa8, 0x8d, 0xcf, 0x42, 0x68, 0x59, 0x8b, 0xfd, 0x69, 0x5b, 0x0e, 0x2f, - 0x76, 0xe1, 0xfd, 0xd9, 0x45, 0x0f, 0x02, 0x55, 0x6c, 0xab, 0x54, 0x62, 0x8c, 0x90, 0x3b, 0xa2, - 0x37, 0xe0, 0xab, 0xb4, 0x28, 0x25, 0x06, 0xe9, 0x72, 0x0b, 0x8f, 0x77, 0xd0, 0x3e, 0xbd, 0x10, - 0x0d, 0xa0, 0xb1, 0x7b, 0x89, 0xfe, 0xe1, 0xff, 0x6b, 0x44, 0xde, 0xff, 0x7f, 0x1f, 0xfb, 0xe4, - 0xe7, 0xd8, 0x27, 0x87, 0x63, 0x9f, 0xcc, 0x02, 0x7c, 0xfb, 0xe7, 0xdf, 0x00, 0x00, 0x00, 0xff, - 0xff, 0xb5, 0x52, 0x37, 0xf2, 0x14, 0x02, 0x00, 0x00, + // 356 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x92, 0xc1, 0x4e, 0xc2, 0x40, + 0x10, 0x86, 0x5d, 0x4a, 0x0b, 0x1d, 0x44, 0x9b, 0x8d, 0x92, 0x8d, 0x26, 0xa4, 0xe1, 0x62, 0xe3, + 0x01, 0x13, 0x34, 0xde, 0xf5, 0x40, 0xc2, 0x0d, 0xf7, 0x05, 0xcc, 0xd2, 0x2e, 0x48, 0xa0, 0xb4, + 0xe9, 0xb6, 0x34, 0xc4, 0x17, 0xf4, 0xe8, 0x23, 0x18, 0x9e, 0xc2, 0xa3, 0xd9, 0xd9, 0x85, 0xe0, + 0x05, 0xfe, 0x6f, 0x67, 0xa7, 0xff, 0xfc, 0xd3, 0xc2, 0x75, 0xbe, 0x5a, 0x3c, 0x94, 0xbb, 0x5c, + 0x2a, 0xf3, 0x3b, 0xcc, 0x8b, 0xac, 0xcc, 0xa8, 0x8b, 0x30, 0x98, 0x81, 0x3f, 0x99, 0xbe, 0x24, + 0x49, 0x21, 0x95, 0xa2, 0x77, 0xe0, 0xcd, 0x45, 0xba, 0x5c, 0xef, 0x18, 0x09, 0x49, 0x74, 0x31, + 0xba, 0x1c, 0x9a, 0x8e, 0xc9, 0x74, 0x8c, 0xc7, 0xdc, 0x96, 0x29, 0x83, 0x96, 0x30, 0x3d, 0xac, + 0x11, 0x92, 0xc8, 0xe7, 0x07, 0xa4, 0x14, 0x9a, 0xa9, 0x50, 0x2b, 0xe6, 0xe0, 0x31, 0xea, 0xc1, + 0x2f, 0x01, 0x7f, 0xb2, 0x29, 0x65, 0x31, 0x17, 0xb1, 0xa4, 0x3d, 0xf0, 0x12, 0xb9, 0x5d, 0xc6, + 0x12, 0x4d, 0x7c, 0x6e, 0x49, 0x77, 0x6e, 0x44, 0x2a, 0xed, 0x03, 0x51, 0xd3, 0x11, 0x74, 0x8e, + 0xd3, 0x49, 0xc5, 0x9c, 0xd0, 0x89, 0x3a, 0xa3, 0xe0, 0x38, 0x95, 0xad, 0xf0, 0xd3, 0x4b, 0x34, + 0x00, 0x27, 0x2d, 0x2b, 0xd6, 0x0c, 0x49, 0xd4, 0xe4, 0x5a, 0x6a, 0xc7, 0x8f, 0x5a, 0x5f, 0x60, + 0xae, 0x71, 0x34, 0xa4, 0x53, 0xe4, 0xf1, 0x12, 0x0b, 0x9e, 0x49, 0x61, 0x51, 0xcf, 0xa2, 0x3d, + 0x58, 0xcb, 0xcc, 0xa2, 0x35, 0xbd, 0x05, 0xbf, 0x10, 0xf5, 0xfb, 0x7c, 0x2d, 0x16, 0x8a, 0xb5, + 0x43, 0x12, 0x75, 0x79, 0xbb, 0x10, 0xf5, 0x58, 0xb3, 0xb6, 0x78, 0xab, 0x64, 0x25, 0x15, 0xf3, + 0xb1, 0x62, 0x69, 0xf0, 0x09, 0x2e, 0xcf, 0xaa, 0x12, 0xd3, 0x25, 0x52, 0x95, 0x36, 0x33, 0x6a, + 0xed, 0xbf, 0x10, 0xa5, 0xac, 0xc5, 0xee, 0xb0, 0x45, 0x8b, 0x27, 0x3b, 0x72, 0xfe, 0xed, 0xa8, + 0x07, 0x9e, 0xca, 0xaa, 0x22, 0x96, 0x18, 0xcf, 0xe7, 0x96, 0xe8, 0x15, 0xb8, 0x2a, 0xce, 0x72, + 0x89, 0x01, 0xbb, 0xdc, 0xc0, 0xfd, 0x0d, 0xb4, 0x0f, 0x6f, 0x8e, 0x7a, 0xd0, 0xd8, 0x3e, 0x05, + 0x67, 0xf8, 0xff, 0x1c, 0x90, 0xd7, 0xf3, 0xaf, 0x7d, 0x9f, 0x7c, 0xef, 0xfb, 0xe4, 0x67, 0xdf, + 0x27, 0x33, 0x0f, 0xbf, 0x89, 0xc7, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa5, 0x57, 0x70, 0xc8, + 0x2c, 0x02, 0x00, 0x00, } diff --git a/pkg/types/types.proto b/pkg/types/types.proto index f6856e1..149df13 100644 --- a/pkg/types/types.proto +++ b/pkg/types/types.proto @@ -37,6 +37,7 @@ message Interface { // list: "veth", "macvtap", "vlan", "macvlan", "tap", ... string type = 7; uint32 raw_flags = 8; + uint32 Queues = 9; } message Route { -- 2.14.3 (Apple Git-98)