From babe7b3028d601a9b00aedda673e6a472f29ad07 Mon Sep 17 00:00:00 2001 From: yangfeiyu Date: Mon, 28 Sep 2020 21:01:00 +0800 Subject: [PATCH] kata-runtime: retry inserting of CNI interface reason: when netmon is enable, a interface is inserted into the netns created by the sandbox, it should wait for the generating of IP of the new interface, and get the ipv4 address Signed-off-by: yangfeiyu --- netmon/netmon.go | 128 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 112 insertions(+), 16 deletions(-) diff --git a/netmon/netmon.go b/netmon/netmon.go index 94305ce2..57beacfb 100644 --- a/netmon/netmon.go +++ b/netmon/netmon.go @@ -52,7 +52,7 @@ var ( version = "unknown" // For simplicity the code will only focus on IPv4 addresses for now. - netlinkFamily = netlink.FAMILY_ALL + netlinkFamily = netlink.FAMILY_V4 storageParentPath = "/var/run/kata-containers/netmon/sbs" ) @@ -70,7 +70,9 @@ type netmon struct { storagePath string sharedFile string - netIfaces map[int]vcTypes.Interface + netIfaces map[int]vcTypes.Interface + plugedIfaces map[string]vcTypes.Interface + pendingRoutes map[string][]vcTypes.Route linkUpdateCh chan netlink.LinkUpdate linkDoneCh chan struct{} @@ -148,15 +150,17 @@ func newNetmon(params netmonParams) (*netmon, error) { } n := &netmon{ - netmonParams: params, - storagePath: filepath.Join(storageParentPath, params.sandboxID), - sharedFile: filepath.Join(storageParentPath, params.sandboxID, sharedFile), - netIfaces: make(map[int]vcTypes.Interface), - linkUpdateCh: make(chan netlink.LinkUpdate), - linkDoneCh: make(chan struct{}), - rtUpdateCh: make(chan netlink.RouteUpdate), - rtDoneCh: make(chan struct{}), - netHandler: handler, + netmonParams: params, + storagePath: filepath.Join(storageParentPath, params.sandboxID), + sharedFile: filepath.Join(storageParentPath, params.sandboxID, sharedFile), + netIfaces: make(map[int]vcTypes.Interface), + plugedIfaces: make(map[string]vcTypes.Interface), + pendingRoutes: make(map[string][]vcTypes.Route), + linkUpdateCh: make(chan netlink.LinkUpdate), + linkDoneCh: make(chan struct{}), + rtUpdateCh: make(chan netlink.RouteUpdate), + rtDoneCh: make(chan struct{}), + netHandler: handler, } if err := os.MkdirAll(n.storagePath, storageDirPerm); err != nil { @@ -266,7 +270,6 @@ func convertInterface(linkAttrs *netlink.LinkAttrs, linkType string, addrs []net } var ipAddrs []*vcTypes.IPAddress - for _, addr := range addrs { if addr.IPNet == nil { continue @@ -450,10 +453,24 @@ func (n *netmon) updateRoutes() error { if err != nil { return err } + if len(netlinkRoutes) == 0 { + n.logger().Debug("get 0 routes") + return nil + } // Translate them into Route structures. routes := convertRoutes(netlinkRoutes) + // if the device of the routes have not be hotplug to guest, + // the update operation will be failed, pending them. + // For all routes are belong the same device, so we just need + // judge the device of the first route + if _, pluged := n.plugedIfaces[routes[0].Device]; !pluged { + n.pendingRoutes[routes[0].Device] = routes + n.logger().Infof("dev %s have not been added, pending:%v", routes[0].Device, routes) + return nil + } + // Update the routes through the Kata CLI. return n.updateRoutesCLI(routes) } @@ -489,6 +506,9 @@ func (n *netmon) handleRTMNewLink(ev netlink.LinkUpdate) error { return nil } + // the link is usually not ready, and `sleep 3` is an empirical value. + time.Sleep(3 * time.Second) + // Check if the interface exist in the internal list. if _, exist := n.netIfaces[int(ev.Index)]; exist { n.logger().Debugf("Ignoring interface %s because already exist", @@ -504,10 +524,26 @@ func (n *netmon) handleRTMNewLink(ev netlink.LinkUpdate) error { return nil } - // Get the list of IP addresses associated with this interface. - addrs, err := n.netHandler.AddrList(ev.Link, netlinkFamily) - if err != nil { - return err + // In some scenarios, the ip have not prepared,so we should do some retries. + var addrs []netlink.Addr + var err error + for i := 0; i < 5; i++ { + // Get the list of IP addresses associated with this interface. + addrs, err = n.netHandler.AddrList(ev.Link, netlinkFamily) + if err != nil { + return err + } + if len(addrs) > 0 { + break + } + time.Sleep(500 * time.Millisecond) + } + + // In some scenarios, the link reported by event can not found, do extras check here. + if n.checkLinkByHw(linkAttrs.HardwareAddr.String()) != true { + n.logger().Infof("Ignore %v because can not find link by HW %s", + linkAttrs.Name, linkAttrs.HardwareAddr.String()) + return nil } // Convert the interfaces in the appropriate structure format. @@ -520,6 +556,17 @@ func (n *netmon) handleRTMNewLink(ev netlink.LinkUpdate) error { // Add the interface to the internal list. n.netIfaces[linkAttrs.Index] = iface + n.plugedIfaces[iface.Name] = iface + + // The pending routes is preferentially selected. + if routes, ok := n.pendingRoutes[iface.Name]; ok { + n.logger().Infof("dev %s find pending routes:%v", iface.Name, routes) + err = n.updateRoutesCLI(routes) + if err != nil { + return err + } + delete(n.pendingRoutes, iface.Name) + } // Complete by updating the routes. return n.updateRoutes() @@ -556,6 +603,8 @@ func (n *netmon) handleRTMDelLink(ev netlink.LinkUpdate) error { // Delete the interface from the internal list. delete(n.netIfaces, linkAttrs.Index) + delete(n.plugedIfaces, iface.Name) + delete(n.pendingRoutes, iface.Name) // Complete by updating the routes. return n.updateRoutes() @@ -640,6 +689,53 @@ func (n *netmon) handleEvents() (err error) { } } +func linkByHwAddr(netHandle *netlink.Handle, hwAddr string) (netlink.Link, error) { + if netHandle == nil { + return nil, fmt.Errorf("no handler ") + } + + links, err := netHandle.LinkList() + if err != nil { + return nil, err + } + + for _, link := range links { + if link == nil { + continue + } + + lAttrs := link.Attrs() + if lAttrs == nil { + continue + } + + if lAttrs.HardwareAddr == nil { + continue + } + + if lAttrs.HardwareAddr.String() == hwAddr { + return link, nil + } + } + + return nil, fmt.Errorf("could not find the link corresponding to HwAddr %s", hwAddr) +} + +func (n *netmon) checkLinkByHw(hw string) bool { + netHandle, err := netlink.NewHandle(unix.NETLINK_ROUTE) + if err != nil { + return false + } + defer netHandle.Delete() + + link, err := linkByHwAddr(netHandle, hw) + if err != nil || link == nil { + return false + } + + return true +} + func main() { // Parse parameters. params := parseOptions() -- 2.23.0