kata-containers/runtime/patches/0061-kata-runtime-retry-inserting-of-CNI-interface.patch

From babe7b3028d601a9b00aedda673e6a472f29ad07 Mon Sep 17 00:00:00 2001
From: yangfeiyu <yangfeiyu2@huawei.com>
Date: Mon, 28 Sep 2020 21:01:00 +0800
Subject: [PATCH] kata-runtime:  retry inserting of CNI interface

reason: when netmon is enable, a interface is inserted into the
netns created by the sandbox, it should wait for the generating of
IP of the new interface, and get the ipv4 address

Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
---
 netmon/netmon.go | 128 +++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 112 insertions(+), 16 deletions(-)

diff --git a/netmon/netmon.go b/netmon/netmon.go
index 94305ce2..57beacfb 100644
--- a/netmon/netmon.go
+++ b/netmon/netmon.go
@@ -52,7 +52,7 @@ var (
 	version = "unknown"
 
 	// For simplicity the code will only focus on IPv4 addresses for now.
-	netlinkFamily = netlink.FAMILY_ALL
+	netlinkFamily = netlink.FAMILY_V4
 
 	storageParentPath = "/var/run/kata-containers/netmon/sbs"
 )
@@ -70,7 +70,9 @@ type netmon struct {
 	storagePath string
 	sharedFile  string
 
-	netIfaces map[int]vcTypes.Interface
+	netIfaces     map[int]vcTypes.Interface
+	plugedIfaces  map[string]vcTypes.Interface
+	pendingRoutes map[string][]vcTypes.Route
 
 	linkUpdateCh chan netlink.LinkUpdate
 	linkDoneCh   chan struct{}
@@ -148,15 +150,17 @@ func newNetmon(params netmonParams) (*netmon, error) {
 	}
 
 	n := &netmon{
-		netmonParams: params,
-		storagePath:  filepath.Join(storageParentPath, params.sandboxID),
-		sharedFile:   filepath.Join(storageParentPath, params.sandboxID, sharedFile),
-		netIfaces:    make(map[int]vcTypes.Interface),
-		linkUpdateCh: make(chan netlink.LinkUpdate),
-		linkDoneCh:   make(chan struct{}),
-		rtUpdateCh:   make(chan netlink.RouteUpdate),
-		rtDoneCh:     make(chan struct{}),
-		netHandler:   handler,
+		netmonParams:  params,
+		storagePath:   filepath.Join(storageParentPath, params.sandboxID),
+		sharedFile:    filepath.Join(storageParentPath, params.sandboxID, sharedFile),
+		netIfaces:     make(map[int]vcTypes.Interface),
+		plugedIfaces:  make(map[string]vcTypes.Interface),
+		pendingRoutes: make(map[string][]vcTypes.Route),
+		linkUpdateCh:  make(chan netlink.LinkUpdate),
+		linkDoneCh:    make(chan struct{}),
+		rtUpdateCh:    make(chan netlink.RouteUpdate),
+		rtDoneCh:      make(chan struct{}),
+		netHandler:    handler,
 	}
 
 	if err := os.MkdirAll(n.storagePath, storageDirPerm); err != nil {
@@ -266,7 +270,6 @@ func convertInterface(linkAttrs *netlink.LinkAttrs, linkType string, addrs []net
 	}
 
 	var ipAddrs []*vcTypes.IPAddress
-
 	for _, addr := range addrs {
 		if addr.IPNet == nil {
 			continue
@@ -450,10 +453,24 @@ func (n *netmon) updateRoutes() error {
 	if err != nil {
 		return err
 	}
+	if len(netlinkRoutes) == 0 {
+		n.logger().Debug("get 0 routes")
+		return nil
+	}
 
 	// Translate them into Route structures.
 	routes := convertRoutes(netlinkRoutes)
 
+	// if the device of the routes have not be hotplug to guest,
+	// the update operation will be failed, pending them.
+	// For all routes are belong the same device, so we just need
+	// judge the device of the first route
+	if _, pluged := n.plugedIfaces[routes[0].Device]; !pluged {
+		n.pendingRoutes[routes[0].Device] = routes
+		n.logger().Infof("dev %s have not been added, pending:%v", routes[0].Device, routes)
+		return nil
+	}
+
 	// Update the routes through the Kata CLI.
 	return n.updateRoutesCLI(routes)
 }
@@ -489,6 +506,9 @@ func (n *netmon) handleRTMNewLink(ev netlink.LinkUpdate) error {
 		return nil
 	}
 
+	// the link is usually not ready, and `sleep 3` is an empirical value.
+	time.Sleep(3 * time.Second)
+
 	// Check if the interface exist in the internal list.
 	if _, exist := n.netIfaces[int(ev.Index)]; exist {
 		n.logger().Debugf("Ignoring interface %s because already exist",
@@ -504,10 +524,26 @@ func (n *netmon) handleRTMNewLink(ev netlink.LinkUpdate) error {
 		return nil
 	}
 
-	// Get the list of IP addresses associated with this interface.
-	addrs, err := n.netHandler.AddrList(ev.Link, netlinkFamily)
-	if err != nil {
-		return err
+	// In some scenarios, the ip have not prepared,so we should do some retries.
+	var addrs []netlink.Addr
+	var err error
+	for i := 0; i < 5; i++ {
+		// Get the list of IP addresses associated with this interface.
+		addrs, err = n.netHandler.AddrList(ev.Link, netlinkFamily)
+		if err != nil {
+			return err
+		}
+		if len(addrs) > 0 {
+			break
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+
+	// In some scenarios, the link reported by event can not found, do extras check here.
+	if n.checkLinkByHw(linkAttrs.HardwareAddr.String()) != true {
+		n.logger().Infof("Ignore %v because can not find link by HW %s",
+			linkAttrs.Name, linkAttrs.HardwareAddr.String())
+		return nil
 	}
 
 	// Convert the interfaces in the appropriate structure format.
@@ -520,6 +556,17 @@ func (n *netmon) handleRTMNewLink(ev netlink.LinkUpdate) error {
 
 	// Add the interface to the internal list.
 	n.netIfaces[linkAttrs.Index] = iface
+	n.plugedIfaces[iface.Name] = iface
+
+	// The pending routes is preferentially selected.
+	if routes, ok := n.pendingRoutes[iface.Name]; ok {
+		n.logger().Infof("dev %s find pending routes:%v", iface.Name, routes)
+		err = n.updateRoutesCLI(routes)
+		if err != nil {
+			return err
+		}
+		delete(n.pendingRoutes, iface.Name)
+	}
 
 	// Complete by updating the routes.
 	return n.updateRoutes()
@@ -556,6 +603,8 @@ func (n *netmon) handleRTMDelLink(ev netlink.LinkUpdate) error {
 
 	// Delete the interface from the internal list.
 	delete(n.netIfaces, linkAttrs.Index)
+	delete(n.plugedIfaces, iface.Name)
+	delete(n.pendingRoutes, iface.Name)
 
 	// Complete by updating the routes.
 	return n.updateRoutes()
@@ -640,6 +689,53 @@ func (n *netmon) handleEvents() (err error) {
 	}
 }
 
+func linkByHwAddr(netHandle *netlink.Handle, hwAddr string) (netlink.Link, error) {
+	if netHandle == nil {
+		return nil, fmt.Errorf("no handler ")
+	}
+
+	links, err := netHandle.LinkList()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, link := range links {
+		if link == nil {
+			continue
+		}
+
+		lAttrs := link.Attrs()
+		if lAttrs == nil {
+			continue
+		}
+
+		if lAttrs.HardwareAddr == nil {
+			continue
+		}
+
+		if lAttrs.HardwareAddr.String() == hwAddr {
+			return link, nil
+		}
+	}
+
+	return nil, fmt.Errorf("could not find the link corresponding to HwAddr %s", hwAddr)
+}
+
+func (n *netmon) checkLinkByHw(hw string) bool {
+	netHandle, err := netlink.NewHandle(unix.NETLINK_ROUTE)
+	if err != nil {
+		return false
+	}
+	defer netHandle.Delete()
+
+	link, err := linkByHwAddr(netHandle, hw)
+	if err != nil || link == nil {
+		return false
+	}
+
+	return true
+}
+
 func main() {
 	// Parse parameters.
 	params := parseOptions()
-- 
2.23.0