From e861f426c9e6702e820348ddc61b18013c853402 Mon Sep 17 00:00:00 2001 From: jiangpengfei Date: Thu, 13 Aug 2020 11:24:58 +0800 Subject: [PATCH 24/50] kata-runtime: support hotplug tap interface into kata VM reason: support hotplug exist tap interface or a new created tap interface into kata VM. Signed-off-by: jiangpengfei --- cli/network.go | 45 ++++++++++++++--- virtcontainers/kata_agent.go | 12 ++++- virtcontainers/network.go | 100 +++++++++++++++++++++++--------------- virtcontainers/pkg/types/types.go | 16 +++--- virtcontainers/qemu.go | 11 +++-- virtcontainers/sandbox.go | 24 +++++++-- virtcontainers/tap_endpoint.go | 23 +++++++-- 7 files changed, 161 insertions(+), 70 deletions(-) diff --git a/cli/network.go b/cli/network.go index 881a2358..7e7791f1 100644 --- a/cli/network.go +++ b/cli/network.go @@ -26,6 +26,8 @@ const ( routeType ) +const defaultLinkType = "tap" + var kataNetworkCLICommand = cli.Command{ Name: "kata-network", Usage: "manage interfaces and routes for container", @@ -42,10 +44,22 @@ var kataNetworkCLICommand = cli.Command{ } var addIfaceCommand = cli.Command{ - Name: "add-iface", - Usage: "add an interface to a container", - ArgsUsage: `add-iface file or - for stdin`, - Flags: []cli.Flag{}, + Name: "add-iface", + Usage: "add an interface to a container", + ArgsUsage: `add-iface file or - for stdin + file or stdin for example: + { + "device":"", + "name":"", + "IPAddresses":[{"address":"","mask":""}], + "mtu":, + "hwAddr":"", + "linkType":"tap", + "vhostUserSocket":"" + } + device,name,mtu,hwAddr are required, IPAddresses and vhostUserSocket are optional. + `, + Flags: []cli.Flag{}, Action: func(context *cli.Context) error { ctx, err := cliContextToContext(context) if err != nil { @@ -57,10 +71,20 @@ var addIfaceCommand = cli.Command{ } var delIfaceCommand = cli.Command{ - Name: "del-iface", - Usage: "delete an interface from a container", - ArgsUsage: `del-iface file or - for stdin`, - Flags: []cli.Flag{}, + Name: "del-iface", + Usage: "delete an interface from a container", + ArgsUsage: `del-iface file or - for stdin + file or stdin for example: + { + "device":"", + "name":"", + "IPAddresses":[], + "mtu":0, + "hwAddr":"" + } + Only the "name" field is required. + `, + Flags: []cli.Flag{}, Action: func(context *cli.Context) error { ctx, err := cliContextToContext(context) if err != nil { @@ -156,6 +180,11 @@ func networkModifyCommand(ctx context.Context, containerID, input string, opType if err = json.NewDecoder(f).Decode(&inf); err != nil { return err } + + if len(inf.LinkType) == 0 { + inf.LinkType = defaultLinkType + } + if add { resultingInf, err = vci.AddInterface(ctx, sandboxID, inf) if err != nil { diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index 8e073339..dfdd263b 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -600,8 +600,16 @@ func (k *kataAgent) updateInterface(ifc *vcTypes.Interface) (*vcTypes.Interface, "resulting-interface": fmt.Sprintf("%+v", resultingInterface), }).WithError(err).Error("update interface request failed") } - if resultInterface, ok := resultingInterface.(*vcTypes.Interface); ok { - return resultInterface, err + if resultInterface, ok := resultingInterface.(*aTypes.Interface); ok { + iface := &vcTypes.Interface{ + Device: resultInterface.Device, + Name: resultInterface.Name, + IPAddresses: k.convertToIPAddresses(resultInterface.IPAddresses), + Mtu: resultInterface.Mtu, + HwAddr: resultInterface.HwAddr, + PciAddr: resultInterface.PciAddr, + } + return iface, err } return nil, err } diff --git a/virtcontainers/network.go b/virtcontainers/network.go index d70c5360..e909a822 100644 --- a/virtcontainers/network.go +++ b/virtcontainers/network.go @@ -117,6 +117,7 @@ type NetlinkIface struct { // NetworkInfo gathers all information related to a network interface. // It can be used to store the description of the underlying network. type NetworkInfo struct { + Device string Iface NetlinkIface Addrs []netlink.Addr Routes []netlink.Route @@ -303,6 +304,20 @@ func createLink(netHandle *netlink.Handle, name string, expectedLink netlink.Lin var newLink netlink.Link var fds []*os.File + // check if tapname exists, if true, use existing one instead of create new one + if name != "" { + retLink, err := netlink.LinkByName(name) + // link exist, use it + if err == nil { + networkLogger().Debugf("exist tap device is found, instead of creating new one") + tuntapLink, ok := retLink.(*netlink.Tuntap) + if ok { + fds = tuntapLink.Fds + } + return retLink, nil, nil + } + } + switch expectedLink.Type() { case (&netlink.Tuntap{}).Type(): flags := netlink.TUNTAP_VNET_HDR @@ -1156,7 +1171,7 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li // Check if interface is a physical interface. Do not create // tap interface/bridge if it is. - isPhysical, err := isPhysicalIface(netInfo.Iface.Name) + isPhysical, err := isPhysicalIface(netInfo.Device) if err != nil { return nil, err } @@ -1164,49 +1179,54 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li if isPhysical { networkLogger().WithField("interface", netInfo.Iface.Name).Info("Physical network interface found") endpoint, err = createPhysicalEndpoint(netInfo) - } else { - var socketPath string + return endpoint, err + } - // Check if this is a dummy interface which has a vhost-user socket associated with it - socketPath, err = vhostUserSocketPath(netInfo) - if err != nil { - return nil, err - } + // Check if this is a dummy interface which has a vhost-user socket associated with it + socketPath, err := vhostUserSocketPath(netInfo) + if err != nil { + return nil, err + } - if socketPath != "" { - networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found") - endpoint, err = createVhostUserEndpoint(netInfo, socketPath) - } else if netInfo.Iface.Type == "macvlan" { - networkLogger().Infof("macvlan interface found") - endpoint, err = createBridgedMacvlanNetworkEndpoint(idx, netInfo.Iface.Name, model) - } else if netInfo.Iface.Type == "macvtap" { - networkLogger().Infof("macvtap interface found") - endpoint, err = createMacvtapNetworkEndpoint(netInfo) - } else if netInfo.Iface.Type == "tap" { - networkLogger().Info("tap interface found") - endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name) - } else if netInfo.Iface.Type == "tuntap" { - if link != nil { - switch link.(*netlink.Tuntap).Mode { - case 0: - // mount /sys/class/net to get links - return nil, fmt.Errorf("Network device mode not determined correctly. Mount sysfs in caller") - case 1: - return nil, fmt.Errorf("tun networking device not yet supported") - case 2: - networkLogger().Info("tuntap tap interface found") - endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, model) - default: - return nil, fmt.Errorf("tuntap network %v mode unsupported", link.(*netlink.Tuntap).Mode) - } + if socketPath != "" { + networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found") + endpoint, err = createVhostUserEndpoint(netInfo, socketPath) + return endpoint, err + } + + // We should create tap interface/bridge of other interface type. + networkLogger().Infof("%s interface found", netInfo.Iface.Type) + switch netInfo.Iface.Type { + case "macvlan": + networkLogger().Infof("macvlan interface found") + endpoint, err = createBridgedMacvlanNetworkEndpoint(idx, netInfo.Iface.Name, model) + case "macvtap": + networkLogger().Infof("macvtap interface found") + endpoint, err = createMacvtapNetworkEndpoint(netInfo) + case "tap": + networkLogger().Info("tap interface found") + endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Device) + case "tuntap": + if link != nil { + switch link.(*netlink.Tuntap).Mode { + case 0: + // mount /sys/class/net to get links + return nil, fmt.Errorf("Network device mode not determined correctly. Mount sysfs in caller") + case 1: + return nil, fmt.Errorf("tun networking device not yet supported") + case 2: + networkLogger().Info("tuntap tap interface found") + endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, model) + default: + return nil, fmt.Errorf("tuntap network %v mode unsupported", link.(*netlink.Tuntap).Mode) } - } else if netInfo.Iface.Type == "veth" { - endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model) - } else if netInfo.Iface.Type == "ipvlan" { - endpoint, err = createIPVlanNetworkEndpoint(idx, netInfo.Iface.Name) - } else { - return nil, fmt.Errorf("Unsupported network interface: %s", netInfo.Iface.Type) } + case "veth": + endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model) + case "ipvlan": + endpoint, err = createIPVlanNetworkEndpoint(idx, netInfo.Iface.Name) + default: + err = fmt.Errorf("Unsupported network interface, %s", netInfo.Iface.Type) } return endpoint, err diff --git a/virtcontainers/pkg/types/types.go b/virtcontainers/pkg/types/types.go index 0d4a9cfa..fcc63d84 100644 --- a/virtcontainers/pkg/types/types.go +++ b/virtcontainers/pkg/types/types.go @@ -14,21 +14,21 @@ type IPAddress struct { // Interface describes a network interface. type Interface struct { - Device string - Name string - IPAddresses []*IPAddress - Mtu uint64 - RawFlags uint32 - HwAddr string + Device string `json:"device,omitempty"` + Name string `json:"name,omitempty"` + IPAddresses []*IPAddress `json:"IPAddresses,omitempty"` + Mtu uint64 `json:"mtu,omitempty"` + RawFlags uint32 `json:"rawFlags,omitempty"` + HwAddr string `json:"hwAddr,omitempty"` // pciAddr is the PCI address in the format "bridgeAddr/deviceAddr". // Here, bridgeAddr is the address at which the bridge is attached on the root bus, // while deviceAddr is the address at which the network device is attached on the bridge. - PciAddr string + PciAddr string `json:"pciAddr,omitempty"` // LinkType defines the type of interface described by this structure. // The expected values are the one that are defined by the netlink // library, regarding each type of link. Here is a non exhaustive // list: "veth", "macvtap", "vlan", "macvlan", "tap", ... - LinkType string + LinkType string `json:"linkType,omitempty"` } // Route describes a network route. diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index 7bae3278..bb83b1bb 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -1361,7 +1361,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro return nil } -func (q *qemu) hotAddNetDevice(name, hardAddr string, VMFds, VhostFds []*os.File) error { +func (q *qemu) hotAddNetDevice(deviceName, name, hardAddr string, VMFds, VhostFds []*os.File) error { var ( VMFdNames []string VhostFdNames []string @@ -1381,7 +1381,12 @@ func (q *qemu) hotAddNetDevice(name, hardAddr string, VMFds, VhostFds []*os.File VhostFd.Close() VhostFdNames = append(VhostFdNames, fdName) } - return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", name, VMFdNames, VhostFdNames) + + if len(VMFdNames) != 0 || len(VhostFdNames) != 0 { + return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", name, VMFdNames, VhostFdNames) + } + + return q.qmpMonitorCh.qmp.ExecuteNetdevAdd(q.qmpMonitorCh.ctx, "tap", name, deviceName, "no", "no", 0) } func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) { @@ -1404,7 +1409,7 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) { devID := "virtio-" + tap.ID if op == addDevice { - if err = q.hotAddNetDevice(tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds); err != nil { + if err = q.hotAddNetDevice(tap.TAPIface.Name, tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds); err != nil { return err } diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index ba704249..c8981a41 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -937,6 +937,7 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) { } return NetworkInfo{ + Device: inf.Device, Iface: NetlinkIface{ LinkAttrs: netlink.LinkAttrs{ Name: inf.Name, @@ -950,7 +951,7 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) { } // AddInterface adds new nic to the sandbox. -func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (*vcTypes.Interface, error) { +func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (grpcIf *vcTypes.Interface, err error) { netInfo, err := s.generateNetInfo(inf) if err != nil { return nil, err @@ -962,28 +963,41 @@ func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (*vcTypes.Interface, erro } endpoint.SetProperties(netInfo) - if err := doNetNS(s.networkNS.NetNsPath, func(_ ns.NetNS) error { + if err = doNetNS(s.networkNS.NetNsPath, func(_ ns.NetNS) error { s.Logger().WithField("endpoint-type", endpoint.Type()).Info("Hot attaching endpoint") return endpoint.HotAttach(s.hypervisor) }); err != nil { return nil, err } + defer func() { + if err != nil { + if errDetach := endpoint.HotDetach(s.hypervisor, s.networkNS.NetNsCreated, s.networkNS.NetNsPath); errDetach != nil { + s.Logger().WithField("endpoint-type", endpoint.Type()).Errorf("rollback hot attach endpoint failed") + } + } + }() + // Update the sandbox storage s.networkNS.Endpoints = append(s.networkNS.Endpoints, endpoint) - if err := s.Save(); err != nil { + if err = s.Save(); err != nil { return nil, err } // Add network for vm inf.PciAddr = endpoint.PciAddr() - return s.agent.updateInterface(inf) + grpcIf, err = s.agent.updateInterface(inf) + if err != nil { + return nil, err + } + + return } // RemoveInterface removes a nic of the sandbox. func (s *Sandbox) RemoveInterface(inf *vcTypes.Interface) (*vcTypes.Interface, error) { for i, endpoint := range s.networkNS.Endpoints { - if endpoint.HardwareAddr() == inf.HwAddr { + if endpoint.HardwareAddr() == inf.HwAddr || endpoint.Name() == inf.Name { s.Logger().WithField("endpoint-type", endpoint.Type()).Info("Hot detaching endpoint") if err := endpoint.HotDetach(s.hypervisor, s.networkNS.NetNsCreated, s.networkNS.NetNsPath); err != nil { return inf, err diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go index cb441b87..7d33d5a2 100644 --- a/virtcontainers/tap_endpoint.go +++ b/virtcontainers/tap_endpoint.go @@ -7,6 +7,7 @@ package virtcontainers import ( "fmt" + "os" "github.com/containernetworking/plugins/pkg/ns" "github.com/vishvananda/netlink" @@ -111,7 +112,7 @@ func (endpoint *TapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPat return nil } -func createTapNetworkEndpoint(idx int, ifName string) (*TapEndpoint, error) { +func createTapNetworkEndpoint(idx int, ifName string, tapIfName string) (*TapEndpoint, error) { if idx < 0 { return &TapEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx) } @@ -131,6 +132,10 @@ func createTapNetworkEndpoint(idx int, ifName string) (*TapEndpoint, error) { endpoint.TapInterface.Name = ifName } + if tapIfName != "" { + endpoint.TapInterface.TAPIface.Name = tapIfName + } + return endpoint, nil } @@ -145,9 +150,19 @@ func tapNetwork(endpoint *TapEndpoint, numCPUs uint32, disableVhostNet bool) err if err != nil { return fmt.Errorf("Could not create TAP interface: %s", err) } + + defer func() { + if err != nil { + if errDel := netHandle.LinkDel(tapLink); errDel != nil { + networkLogger().WithError(errDel).Error("tapNetwork fail to rollback del link") + } + } + }() + endpoint.TapInterface.VMFds = fds if !disableVhostNet { - vhostFds, err := createVhostFds(int(numCPUs)) + var vhostFds []*os.File + vhostFds, err = createVhostFds(int(numCPUs)) if err != nil { return fmt.Errorf("Could not setup vhost fds %s : %s", endpoint.TapInterface.Name, err) } @@ -161,10 +176,10 @@ func tapNetwork(endpoint *TapEndpoint, numCPUs uint32, disableVhostNet bool) err // bridge created by the network plugin on the host actually expects // to see traffic from this MAC address and not another one. endpoint.TapInterface.TAPIface.HardAddr = linkAttrs.HardwareAddr.String() - if err := netHandle.LinkSetMTU(tapLink, linkAttrs.MTU); err != nil { + if err = netHandle.LinkSetMTU(tapLink, linkAttrs.MTU); err != nil { return fmt.Errorf("Could not set TAP MTU %d: %s", linkAttrs.MTU, err) } - if err := netHandle.LinkSetUp(tapLink); err != nil { + if err = netHandle.LinkSetUp(tapLink); err != nil { return fmt.Errorf("Could not enable TAP %s: %s", endpoint.TapInterface.Name, err) } return nil -- 2.14.3 (Apple Git-98)