!1 move all kata-containers related source repo into the only one repo
From: @flyflyflypeng Reviewed-by: Signed-off-by:
This commit is contained in:
openeuler-ci-bot
Gitee
commit
f06ff0ea4e
BIN
agent/agent-1.11.1.tar.gz
Normal file
BIN
agent/agent-1.11.1.tar.gz
Normal file
Binary file not shown.
20
agent/apply-patches
Executable file
20
agent/apply-patches
Executable file
@ -0,0 +1,20 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
if [[ -f ./patch_flag ]];then
|
||||||
|
echo "agent patched!"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
tar -zxvf agent-1.11.1.tar.gz
|
||||||
|
cp -fr ./agent-1.11.1/* ./
|
||||||
|
rm -rf ./agent-1.11.1
|
||||||
|
cat ./series.conf | while read line
|
||||||
|
do
|
||||||
|
if [[ $line == '' || $line =~ ^\s*# ]]; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
echo "====patch $line======"
|
||||||
|
pwd
|
||||||
|
patch -p1 -F1 -s < ./patches/$line
|
||||||
|
done
|
||||||
|
touch ./patch_flag
|
||||||
@ -0,0 +1,234 @@
|
|||||||
|
From ac1d7806f8de2f8ca393df08a9c62d1045c4afdc Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 11 Dec 2018 18:27:02 -0500
|
||||||
|
Subject: [PATCH 01/16] agent: add agent.netlink_recv_buf_size flag to set
|
||||||
|
netlink recv buf size
|
||||||
|
|
||||||
|
fixes: #813
|
||||||
|
|
||||||
|
reason: If hotplug huge size memory(for example 128GB) into guest,
|
||||||
|
kernel will produce a lot of memory add uevents and send to netlink socket,
|
||||||
|
however netlink socket default receive buffer size is 4KB, which is too small
|
||||||
|
to receive all memory add uevents.
|
||||||
|
Since hotplug huge size memory is not common case, so we consider add an agent
|
||||||
|
flag agent.netlink_recv_buf_size to set netlink socket recv buffer size.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
README.md | 13 +++++++++++++
|
||||||
|
agent.go | 10 +++++++++-
|
||||||
|
config.go | 15 ++++++++++++++
|
||||||
|
config_test.go | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||||
|
pkg/uevent/uevent.go | 15 +++++++++++---
|
||||||
|
5 files changed, 104 insertions(+), 4 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/README.md b/README.md
|
||||||
|
index cec65a4..16f96a4 100644
|
||||||
|
--- a/README.md
|
||||||
|
+++ b/README.md
|
||||||
|
@@ -98,6 +98,19 @@ The pipe's capacity for stdout/stderr can be modified by specifying the `agent.c
|
||||||
|
to the guest kernel command line. For example, `agent.container_pipe_size=2097152` will set the stdout and stderr
|
||||||
|
pipes to 2097152 bytes.
|
||||||
|
|
||||||
|
+## Uevent Netlink Socket Receive Buffer Size
|
||||||
|
+
|
||||||
|
+When hotplugging a huge size memory into the Kata VM, the kernel in the VM will produce a lot of memory object add
|
||||||
|
+uevents and send all these uevents to Kata agent by netlink socket. However, default netlink socket receive buffer
|
||||||
|
+size is 4KB, which is too small and can only hold 256 memory add uevents. If memory add uevents number is larger
|
||||||
|
+than 256, the left uevents can not be received and processed by Kata agent.
|
||||||
|
+
|
||||||
|
+The uevent netlink socket receive buffer size can be modified by specifying the `agent.netlink_recv_buf_size` flag
|
||||||
|
+to the guest kernel command line. For example, `agent.netlink_recv_buf_size=2MB` will set the uevent netlink socket
|
||||||
|
+receive buffer size to 2MB value. `agent.netlink_recv_buf_size` valid value range is `[4KB ~ 4MB]` and value can be
|
||||||
|
+set in human-readable memory format or pure digital number format(default memory unit is byte).
|
||||||
|
+
|
||||||
|
+
|
||||||
|
[1]: https://github.com/firecracker-microvm/firecracker/blob/master/docs/vsock.md
|
||||||
|
[2]: https://golang.org/pkg/time/#ParseDuration
|
||||||
|
[3]: http://man7.org/linux/man-pages/man7/pipe.7.html
|
||||||
|
diff --git a/agent.go b/agent.go
|
||||||
|
index 2d2c293..c1cac08 100644
|
||||||
|
--- a/agent.go
|
||||||
|
+++ b/agent.go
|
||||||
|
@@ -190,6 +190,14 @@ var unifiedCgroupHierarchy = false
|
||||||
|
// Size in bytes of the stdout/stderr pipes created for each container.
|
||||||
|
var containerPipeSize = uint32(0)
|
||||||
|
|
||||||
|
+const (
|
||||||
|
+ minNetlinkSockRecvBufSize = 4 * 1024
|
||||||
|
+ maxNetlinkSockRecvBufSize = 4 * 1024 * 1024
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+// Size in bytes of the netlink socket recv buf size
|
||||||
|
+var netlinkSockRecvBufSize = uint32(0)
|
||||||
|
+
|
||||||
|
// commType is used to denote the communication channel type used.
|
||||||
|
type commType int
|
||||||
|
|
||||||
|
@@ -708,7 +716,7 @@ func (s *sandbox) waitForStopServer() {
|
||||||
|
func (s *sandbox) listenToUdevEvents() {
|
||||||
|
fieldLogger := agentLog.WithField("subsystem", "udevlistener")
|
||||||
|
|
||||||
|
- uEvHandler, err := uevent.NewHandler()
|
||||||
|
+ uEvHandler, err := uevent.NewHandler(netlinkSockRecvBufSize)
|
||||||
|
if err != nil {
|
||||||
|
fieldLogger.Warnf("Error starting uevent listening loop %s", err)
|
||||||
|
return
|
||||||
|
diff --git a/config.go b/config.go
|
||||||
|
index 4530096..6c7d473 100644
|
||||||
|
--- a/config.go
|
||||||
|
+++ b/config.go
|
||||||
|
@@ -7,11 +7,13 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
+ "fmt"
|
||||||
|
"io/ioutil"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
+ "github.com/docker/go-units"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
"google.golang.org/grpc/codes"
|
||||||
|
grpcStatus "google.golang.org/grpc/status"
|
||||||
|
@@ -29,6 +31,7 @@ const (
|
||||||
|
hotplugTimeoutFlag = optionPrefix + "hotplug_timeout"
|
||||||
|
unifiedCgroupHierarchyFlag = optionPrefix + "unified_cgroup_hierarchy"
|
||||||
|
containerPipeSizeFlag = optionPrefix + "container_pipe_size"
|
||||||
|
+ netlinkSockRecvBufSizeFlag = optionPrefix + "netlink_recv_buf_size"
|
||||||
|
traceModeStatic = "static"
|
||||||
|
traceModeDynamic = "dynamic"
|
||||||
|
traceTypeIsolated = "isolated"
|
||||||
|
@@ -155,6 +158,18 @@ func parseCmdlineOption(option string) error {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
unifiedCgroupHierarchy = flag
|
||||||
|
+ case netlinkSockRecvBufSizeFlag:
|
||||||
|
+ bufSizeInBytes, err := units.RAMInBytes(split[valuePosition])
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if bufSizeInBytes < minNetlinkSockRecvBufSize || bufSizeInBytes > maxNetlinkSockRecvBufSize {
|
||||||
|
+ return fmt.Errorf("invalid netlink socket recv buf size: %d (valid size range %s-%s bytes)", bufSizeInBytes,
|
||||||
|
+ units.BytesSize(minNetlinkSockRecvBufSize), units.BytesSize(maxNetlinkSockRecvBufSize))
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ netlinkSockRecvBufSize = uint32(bufSizeInBytes)
|
||||||
|
default:
|
||||||
|
if strings.HasPrefix(split[optionPosition], optionPrefix) {
|
||||||
|
return grpcStatus.Errorf(codes.NotFound, "Unknown option %s", split[optionPosition])
|
||||||
|
diff --git a/config_test.go b/config_test.go
|
||||||
|
index 2a23133..f40f17a 100644
|
||||||
|
--- a/config_test.go
|
||||||
|
+++ b/config_test.go
|
||||||
|
@@ -486,3 +486,58 @@ func TestParseCmdlineOptionContainerPipeSize(t *testing.T) {
|
||||||
|
assert.Equal(d.expectedContainerPipeSize, containerPipeSize, "test %d (%+v)", i, d)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func TestParseCmdlineOptionNetlinkSockRecvBufSize(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+
|
||||||
|
+ type testData struct {
|
||||||
|
+ option string
|
||||||
|
+ shouldErr bool
|
||||||
|
+ expectedNetlinkSockRecvBufSize uint32
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ data := []testData{
|
||||||
|
+ {"", false, 0},
|
||||||
|
+ {"netlink_recv_buf_siz", false, 0},
|
||||||
|
+ {"netlink_recv_buf_size", false, 0},
|
||||||
|
+ {"netlink_recv_buf_size=", false, 0},
|
||||||
|
+ {"netlink_recv_buf_size=4096", false, 0},
|
||||||
|
+ {"netlink_recv_buf_size=4KB", false, 0},
|
||||||
|
+ {"agent.netlink_recv_buf_size=", true, 0},
|
||||||
|
+ {"agent.netlink_recv_buf_size=foobar", true, 0},
|
||||||
|
+ {"agent.netlink_recv_buf_size=-1", true, 0},
|
||||||
|
+ {"agent.netlink_recv_buf_size=0", true, 0},
|
||||||
|
+ {"agent.netlink_recv_buf_size=100", true, 0},
|
||||||
|
+ {"agent.netlink_recv_buf_size=3KB", true, 0},
|
||||||
|
+ {"agent.netlink_recv_buf_size=3.6KB", true, 0},
|
||||||
|
+ {"agent.netlink_recv_buf_size=4095", true, 0},
|
||||||
|
+ {"agent.netlink_recv_buf_size=4096xB", true, 0},
|
||||||
|
+ {"agent.netlink_recv_buf_size=4096", false, 4096},
|
||||||
|
+ {"agent.netlink_recv_buf_size=4097", false, 4097},
|
||||||
|
+ {"agent.netlink_recv_buf_size=4096.0", false, 4096},
|
||||||
|
+ {"agent.netlink_recv_buf_size=1024KB", false, 1048576},
|
||||||
|
+ {"agent.netlink_recv_buf_size=1MB", false, 1048576},
|
||||||
|
+ {"agent.netlink_recv_buf_size=4194303", false, 4194303},
|
||||||
|
+ {"agent.netlink_recv_buf_size=3.999MB", false, 4193255},
|
||||||
|
+ {"agent.netlink_recv_buf_size=4194304", false, 4194304},
|
||||||
|
+ {"agent.netlink_recv_buf_size=4MB", false, 4194304},
|
||||||
|
+ {"agent.netlink_recv_buf_size=4.001MB", true, 0},
|
||||||
|
+ {"agent.netlink_recv_buf_size=4194305", true, 0},
|
||||||
|
+ {"agent.netlink_recv_buf_size=100MB", true, 0},
|
||||||
|
+ {"agent.netlink_recv_buf_size=1GB", true, 0},
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ for i, d := range data {
|
||||||
|
+ // reset the netlink socket recv buffer size
|
||||||
|
+ netlinkSockRecvBufSize = 0
|
||||||
|
+
|
||||||
|
+ err := parseCmdlineOption(d.option)
|
||||||
|
+ if d.shouldErr {
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ } else {
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ assert.Equal(d.expectedNetlinkSockRecvBufSize, netlinkSockRecvBufSize, "test %d (%+v)", i, d)
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
diff --git a/pkg/uevent/uevent.go b/pkg/uevent/uevent.go
|
||||||
|
index fc2c127..fa84086 100644
|
||||||
|
--- a/pkg/uevent/uevent.go
|
||||||
|
+++ b/pkg/uevent/uevent.go
|
||||||
|
@@ -10,6 +10,7 @@ import (
|
||||||
|
"bufio"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
+ "syscall"
|
||||||
|
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
"google.golang.org/grpc/codes"
|
||||||
|
@@ -33,7 +34,7 @@ type ReaderCloser struct {
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewReaderCloser returns an io.ReadCloser handle for uevent.
|
||||||
|
-func NewReaderCloser() (io.ReadCloser, error) {
|
||||||
|
+func NewReaderCloser(netlinkRecvBufSize uint32) (io.ReadCloser, error) {
|
||||||
|
nl := unix.SockaddrNetlink{
|
||||||
|
Family: unix.AF_NETLINK,
|
||||||
|
// Passing Pid as 0 here allows the kernel to take care of assigning
|
||||||
|
@@ -47,6 +48,14 @@ func NewReaderCloser() (io.ReadCloser, error) {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
+ // If netlinkRecvBufSize > 0, set netlink socket recv buffer size to netlinkRecvBufSize
|
||||||
|
+ if netlinkRecvBufSize > 0 {
|
||||||
|
+ err = unix.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_RCVBUFFORCE, int(netlinkRecvBufSize))
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
if err := unix.Bind(fd, &nl); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
@@ -85,8 +94,8 @@ type Handler struct {
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewHandler returns a uevent handler.
|
||||||
|
-func NewHandler() (*Handler, error) {
|
||||||
|
- rdCloser, err := NewReaderCloser()
|
||||||
|
+func NewHandler(netlinkRecvBufSize uint32) (*Handler, error) {
|
||||||
|
+ rdCloser, err := NewReaderCloser(netlinkRecvBufSize)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,686 @@
|
|||||||
|
From 13f54c768dcd7bf982dde8e57fb5cd624fedf5bc Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Mon, 17 Aug 2020 11:23:55 +0800
|
||||||
|
Subject: [PATCH 02/16] network: support update routes incrementally
|
||||||
|
|
||||||
|
reason: add increment flag in the UpdateRoutesRequest to
|
||||||
|
support upate routes incrementally to improve the efficiency.
|
||||||
|
|
||||||
|
kata-network add-route and del-route needs this feature.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
grpc.go | 2 +-
|
||||||
|
network.go | 74 ++++++++-
|
||||||
|
network_test.go | 16 +-
|
||||||
|
protocols/grpc/agent.pb.go | 402 +++++++++++++++++++++++++--------------------
|
||||||
|
protocols/grpc/agent.proto | 1 +
|
||||||
|
5 files changed, 300 insertions(+), 195 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/grpc.go b/grpc.go
|
||||||
|
index 886661b..8fe8217 100644
|
||||||
|
--- a/grpc.go
|
||||||
|
+++ b/grpc.go
|
||||||
|
@@ -1556,7 +1556,7 @@ func (a *agentGRPC) UpdateInterface(ctx context.Context, req *pb.UpdateInterface
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *agentGRPC) UpdateRoutes(ctx context.Context, req *pb.UpdateRoutesRequest) (*pb.Routes, error) {
|
||||||
|
- return a.sandbox.updateRoutes(nil, req.Routes)
|
||||||
|
+ return a.sandbox.updateRoutes(nil, req.Routes, req.Increment)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *agentGRPC) ListInterfaces(ctx context.Context, req *pb.ListInterfacesRequest) (*pb.Interfaces, error) {
|
||||||
|
diff --git a/network.go b/network.go
|
||||||
|
index 64a16a9..02e28cb 100644
|
||||||
|
--- a/network.go
|
||||||
|
+++ b/network.go
|
||||||
|
@@ -398,7 +398,7 @@ func (s *sandbox) deleteRoutes(netHandle *netlink.Handle) error {
|
||||||
|
// state which matches the requested routes. In doing this, preesxisting non-loopback routes will be
|
||||||
|
// removed from the network. If an error occurs, this function returns the list of routes in
|
||||||
|
// gRPC-route format at the time of failure
|
||||||
|
-func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Routes) (resultingRoutes *pb.Routes, err error) {
|
||||||
|
+func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Routes, increment bool) (resultingRoutes *pb.Routes, err error) {
|
||||||
|
if requestedRoutes == nil {
|
||||||
|
return nil, errNoRoutes
|
||||||
|
}
|
||||||
|
@@ -418,13 +418,66 @@ func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Ro
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
+ var (
|
||||||
|
+ added []*types.Route
|
||||||
|
+ removed []*types.Route
|
||||||
|
+ )
|
||||||
|
+
|
||||||
|
+ defer func(netHandle *netlink.Handle) {
|
||||||
|
+ if err != nil {
|
||||||
|
+ // if error happens after route added, need to rollback the added route
|
||||||
|
+ if len(added) > 0 {
|
||||||
|
+ for _, r := range added {
|
||||||
|
+ errRb := s.updateRoute(netHandle, r, false)
|
||||||
|
+ if errRb != nil {
|
||||||
|
+ agentLog.WithError(err).Error("rollback route failed")
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // if error happens after route removed, need to rollback the removed route
|
||||||
|
+ if len(removed) > 0 {
|
||||||
|
+ for _, r := range removed {
|
||||||
|
+ errRb := s.updateRoute(netHandle, r, true)
|
||||||
|
+ if errRb != nil {
|
||||||
|
+ agentLog.WithError(err).Error("rollback route failed")
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }(netHandle)
|
||||||
|
+
|
||||||
|
+ // updateOneRoute just update the specified one route
|
||||||
|
+ updateOneRoute := func(netHandle *netlink.Handle, reqRoute *types.Route) error {
|
||||||
|
+ var add bool = true
|
||||||
|
+ if reqRoute.Dest != "" && strings.HasPrefix(reqRoute.Dest, "-") {
|
||||||
|
+ reqRoute.Dest = reqRoute.Dest[1:]
|
||||||
|
+ add = false
|
||||||
|
+ }
|
||||||
|
+ err = s.updateRoute(netHandle, reqRoute, add)
|
||||||
|
+ if err != nil {
|
||||||
|
+ agentLog.WithError(err).Error("update Route failed")
|
||||||
|
+ // If there was an error setting the route, return the error
|
||||||
|
+ // and the current routes on the system via the defer func
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ if add {
|
||||||
|
+ added = append([]*types.Route{reqRoute}, added[:]...)
|
||||||
|
+ } else {
|
||||||
|
+ removed = append([]*types.Route{reqRoute}, removed[:]...)
|
||||||
|
+ }
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
//
|
||||||
|
// First things first, let's blow away all the existing routes. The updateRoutes function
|
||||||
|
// is designed to be declarative, so we will attempt to create state matching what is
|
||||||
|
// requested, and in the event that we fail to do so, will return the error and final state.
|
||||||
|
//
|
||||||
|
- if err = s.deleteRoutes(netHandle); err != nil {
|
||||||
|
- return nil, err
|
||||||
|
+ if !increment {
|
||||||
|
+ if err = s.deleteRoutes(netHandle); err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
|
||||||
|
//
|
||||||
|
@@ -434,7 +487,12 @@ func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Ro
|
||||||
|
// won't be able to access the gateway
|
||||||
|
for _, reqRoute := range requestedRoutes.Routes {
|
||||||
|
if reqRoute.Gateway == "" {
|
||||||
|
- err = s.updateRoute(netHandle, reqRoute, true)
|
||||||
|
+ if increment {
|
||||||
|
+ err = updateOneRoute(netHandle, reqRoute)
|
||||||
|
+ } else {
|
||||||
|
+ err = s.updateRoute(netHandle, reqRoute, true)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
if err != nil {
|
||||||
|
agentLog.WithError(err).Error("update Route failed")
|
||||||
|
//If there was an error setting the route, return the error
|
||||||
|
@@ -447,7 +505,11 @@ func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Ro
|
||||||
|
// Take a second pass and apply the routes which include a gateway
|
||||||
|
for _, reqRoute := range requestedRoutes.Routes {
|
||||||
|
if reqRoute.Gateway != "" {
|
||||||
|
- err = s.updateRoute(netHandle, reqRoute, true)
|
||||||
|
+ if increment {
|
||||||
|
+ err = updateOneRoute(netHandle, reqRoute)
|
||||||
|
+ } else {
|
||||||
|
+ err = s.updateRoute(netHandle, reqRoute, true)
|
||||||
|
+ }
|
||||||
|
if err != nil {
|
||||||
|
agentLog.WithError(err).Error("update Route failed")
|
||||||
|
//If there was an error setting the route, return the
|
||||||
|
@@ -699,4 +761,4 @@ func (s *sandbox) handleLocalhost() error {
|
||||||
|
}
|
||||||
|
|
||||||
|
return netlink.LinkSetUp(lo)
|
||||||
|
-}
|
||||||
|
+}
|
||||||
|
\ No newline at end of file
|
||||||
|
diff --git a/network_test.go b/network_test.go
|
||||||
|
index a143670..a1e58f5 100644
|
||||||
|
--- a/network_test.go
|
||||||
|
+++ b/network_test.go
|
||||||
|
@@ -160,7 +160,7 @@ func TestUpdateRoutes(t *testing.T) {
|
||||||
|
Routes: inputRoutesSimple,
|
||||||
|
}
|
||||||
|
|
||||||
|
- results, err := s.updateRoutes(netHandle, testRoutes)
|
||||||
|
+ results, err := s.updateRoutes(netHandle, testRoutes, false)
|
||||||
|
assert.Nil(t, err, "Unexpected update interface failure: %v", err)
|
||||||
|
assert.True(t, reflect.DeepEqual(results, testRoutes),
|
||||||
|
"Interface created didn't match: got %+v, expecting %+v", results, testRoutes)
|
||||||
|
@@ -173,7 +173,7 @@ func TestUpdateRoutes(t *testing.T) {
|
||||||
|
}
|
||||||
|
testRoutes.Routes = inputRoutesPTPExample
|
||||||
|
|
||||||
|
- results, err = s.updateRoutes(netHandle, testRoutes)
|
||||||
|
+ results, err = s.updateRoutes(netHandle, testRoutes, false)
|
||||||
|
assert.Nil(t, err, "Unexpected update interface failure: %v", err)
|
||||||
|
assert.True(t, reflect.DeepEqual(results, testRoutes),
|
||||||
|
"Interface created didn't match: got %+v, expecting %+v", results, testRoutes)
|
||||||
|
@@ -184,7 +184,7 @@ func TestUpdateRoutes(t *testing.T) {
|
||||||
|
{Dest: "192.168.0.0/16", Gateway: "", Source: "192.168.0.2", Scope: 0, Device: "ifc-name"},
|
||||||
|
}
|
||||||
|
testRoutes.Routes = inputRoutesNoScope
|
||||||
|
- results, err = s.updateRoutes(netHandle, testRoutes)
|
||||||
|
+ results, err = s.updateRoutes(netHandle, testRoutes, false)
|
||||||
|
assert.NotNil(t, err, "Expected to observe unreachable route failure")
|
||||||
|
|
||||||
|
assert.True(t, reflect.DeepEqual(results.Routes[0], testRoutes.Routes[1]),
|
||||||
|
@@ -231,7 +231,7 @@ func TestUpdateRoutesIPVlan(t *testing.T) {
|
||||||
|
}
|
||||||
|
testRoutes.Routes = inputRoutesIPVlanExample
|
||||||
|
|
||||||
|
- results, err := s.updateRoutes(netHandle, testRoutes)
|
||||||
|
+ results, err := s.updateRoutes(netHandle, testRoutes, false)
|
||||||
|
assert.Nil(t, err, "Unexpected update interface failure: %v", err)
|
||||||
|
assert.True(t, reflect.DeepEqual(results, testRoutes),
|
||||||
|
"Interface created didn't match: got %+v, expecting %+v", results, testRoutes)
|
||||||
|
@@ -357,7 +357,7 @@ func TestListRoutes(t *testing.T) {
|
||||||
|
Routes: inputRoutesSimple,
|
||||||
|
}
|
||||||
|
|
||||||
|
- _, err := s.updateRoutes(netHandle, testRoutes)
|
||||||
|
+ _, err := s.updateRoutes(netHandle, testRoutes, false)
|
||||||
|
assert.Nil(err)
|
||||||
|
results, err := s.listRoutes(nil)
|
||||||
|
assert.Nil(err, "Expected to list all routes")
|
||||||
|
@@ -377,7 +377,7 @@ func TestListRoutes(t *testing.T) {
|
||||||
|
Routes: inputRoutesSimple,
|
||||||
|
}
|
||||||
|
|
||||||
|
- _, err = s.updateRoutes(netHandle, testRoutes)
|
||||||
|
+ _, err = s.updateRoutes(netHandle, testRoutes, false)
|
||||||
|
assert.Nil(err)
|
||||||
|
results, err = s.listRoutes(nil)
|
||||||
|
assert.Nil(err, "Expected to list all routes")
|
||||||
|
@@ -438,7 +438,7 @@ func TestListRoutesWithIPV6(t *testing.T) {
|
||||||
|
Routes: inputRoutesSimple,
|
||||||
|
}
|
||||||
|
|
||||||
|
- _, err := s.updateRoutes(netHandle, testRoutes)
|
||||||
|
+ _, err := s.updateRoutes(netHandle, testRoutes, false)
|
||||||
|
assert.Nil(err)
|
||||||
|
results, err := s.listRoutes(nil)
|
||||||
|
assert.Nil(err, "Expected to list all routes")
|
||||||
|
@@ -514,7 +514,7 @@ func TestListRoutesWithTwoInterfacesSameSubnet(t *testing.T) {
|
||||||
|
Routes: inputRoutesSimple,
|
||||||
|
}
|
||||||
|
|
||||||
|
- _, err := s.updateRoutes(netHandle, testRoutes)
|
||||||
|
+ _, err := s.updateRoutes(netHandle, testRoutes, false)
|
||||||
|
assert.Nil(err)
|
||||||
|
results, err := s.listRoutes(nil)
|
||||||
|
assert.Nil(err, "Expected to list all routes")
|
||||||
|
diff --git a/protocols/grpc/agent.pb.go b/protocols/grpc/agent.pb.go
|
||||||
|
index 77e6d1b..1b887e5 100644
|
||||||
|
--- a/protocols/grpc/agent.pb.go
|
||||||
|
+++ b/protocols/grpc/agent.pb.go
|
||||||
|
@@ -1303,7 +1303,8 @@ func (m *UpdateInterfaceRequest) GetInterface() *types.Interface {
|
||||||
|
}
|
||||||
|
|
||||||
|
type UpdateRoutesRequest struct {
|
||||||
|
- Routes *Routes `protobuf:"bytes,1,opt,name=routes" json:"routes,omitempty"`
|
||||||
|
+ Routes *Routes `protobuf:"bytes,1,opt,name=routes" json:"routes,omitempty"`
|
||||||
|
+ Increment bool `protobuf:"varint,2,opt,name=increment,proto3" json:"increment,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *UpdateRoutesRequest) Reset() { *m = UpdateRoutesRequest{} }
|
||||||
|
@@ -1318,6 +1319,13 @@ func (m *UpdateRoutesRequest) GetRoutes() *Routes {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (m *UpdateRoutesRequest) GetIncrement() bool {
|
||||||
|
+ if m != nil {
|
||||||
|
+ return m.Increment
|
||||||
|
+ }
|
||||||
|
+ return false
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
type ListInterfacesRequest struct {
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -4476,6 +4484,16 @@ func (m *UpdateRoutesRequest) MarshalTo(dAtA []byte) (int, error) {
|
||||||
|
}
|
||||||
|
i += n20
|
||||||
|
}
|
||||||
|
+ if m.Increment {
|
||||||
|
+ dAtA[i] = 0x10
|
||||||
|
+ i++
|
||||||
|
+ if m.Increment {
|
||||||
|
+ dAtA[i] = 1
|
||||||
|
+ } else {
|
||||||
|
+ dAtA[i] = 0
|
||||||
|
+ }
|
||||||
|
+ i++
|
||||||
|
+ }
|
||||||
|
return i, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -5751,6 +5769,9 @@ func (m *UpdateRoutesRequest) Size() (n int) {
|
||||||
|
l = m.Routes.Size()
|
||||||
|
n += 1 + l + sovAgent(uint64(l))
|
||||||
|
}
|
||||||
|
+ if m.Increment {
|
||||||
|
+ n += 2
|
||||||
|
+ }
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -10964,6 +10985,26 @@ func (m *UpdateRoutesRequest) Unmarshal(dAtA []byte) error {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
iNdEx = postIndex
|
||||||
|
+ case 2:
|
||||||
|
+ if wireType != 0 {
|
||||||
|
+ return fmt.Errorf("proto: wrong wireType = %d for field Increment", wireType)
|
||||||
|
+ }
|
||||||
|
+ var v int
|
||||||
|
+ for shift := uint(0); ; shift += 7 {
|
||||||
|
+ if shift >= 64 {
|
||||||
|
+ return ErrIntOverflowAgent
|
||||||
|
+ }
|
||||||
|
+ if iNdEx >= l {
|
||||||
|
+ return io.ErrUnexpectedEOF
|
||||||
|
+ }
|
||||||
|
+ b := dAtA[iNdEx]
|
||||||
|
+ iNdEx++
|
||||||
|
+ v |= (int(b) & 0x7F) << shift
|
||||||
|
+ if b < 0x80 {
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ m.Increment = bool(v != 0)
|
||||||
|
default:
|
||||||
|
iNdEx = preIndex
|
||||||
|
skippy, err := skipAgent(dAtA[iNdEx:])
|
||||||
|
@@ -12852,184 +12893,185 @@ var (
|
||||||
|
func init() { proto.RegisterFile("agent.proto", fileDescriptorAgent) }
|
||||||
|
|
||||||
|
var fileDescriptorAgent = []byte{
|
||||||
|
- // 2862 bytes of a gzipped FileDescriptorProto
|
||||||
|
+ // 2876 bytes of a gzipped FileDescriptorProto
|
||||||
|
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0x4b, 0x6f, 0x1c, 0xc7,
|
||||||
|
- 0xd1, 0xd8, 0x07, 0x97, 0xbb, 0xb5, 0x2f, 0x6e, 0x93, 0xa2, 0x56, 0x2b, 0x5b, 0x9f, 0x3c, 0xb6,
|
||||||
|
- 0x65, 0xfa, 0x73, 0xbc, 0xb4, 0x65, 0x23, 0x7e, 0xc1, 0x11, 0xc4, 0x47, 0x44, 0xc6, 0x56, 0xc4,
|
||||||
|
- 0x0c, 0x45, 0x38, 0x40, 0x10, 0x0c, 0x86, 0x33, 0xcd, 0x65, 0x9b, 0x3b, 0xd3, 0xe3, 0x9e, 0x1e,
|
||||||
|
- 0x8a, 0xeb, 0x00, 0x39, 0x26, 0xb7, 0x5c, 0x02, 0xe4, 0x96, 0x3f, 0x10, 0xe4, 0x96, 0x63, 0xae,
|
||||||
|
- 0x39, 0x18, 0x39, 0xe5, 0x17, 0x04, 0x81, 0x7f, 0x42, 0x7e, 0x41, 0xd0, 0xaf, 0x79, 0xec, 0x0e,
|
||||||
|
- 0x29, 0x84, 0x20, 0x90, 0xcb, 0xa2, 0xab, 0xba, 0xba, 0x5e, 0xdd, 0x55, 0x53, 0x55, 0x0b, 0x6d,
|
||||||
|
- 0x77, 0x82, 0x43, 0x3e, 0x8e, 0x18, 0xe5, 0x14, 0xd5, 0x27, 0x2c, 0xf2, 0x46, 0x2d, 0xea, 0x11,
|
||||||
|
- 0x85, 0x18, 0xfd, 0x70, 0x42, 0xf8, 0x69, 0x72, 0x3c, 0xf6, 0x68, 0xb0, 0x79, 0xe6, 0x72, 0xf7,
|
||||||
|
- 0x5d, 0x8f, 0x86, 0xdc, 0x25, 0x21, 0x66, 0xf1, 0xa6, 0x3c, 0xb8, 0x19, 0x9d, 0x4d, 0x36, 0xf9,
|
||||||
|
- 0x2c, 0xc2, 0xb1, 0xfa, 0xd5, 0xe7, 0xee, 0x4e, 0x28, 0x9d, 0x4c, 0xf1, 0xa6, 0x84, 0x8e, 0x93,
|
||||||
|
- 0x93, 0x4d, 0x1c, 0x44, 0x7c, 0xa6, 0x36, 0xad, 0x3f, 0x56, 0x61, 0x7d, 0x9b, 0x61, 0x97, 0xe3,
|
||||||
|
- 0x6d, 0xc3, 0xcd, 0xc6, 0xdf, 0x24, 0x38, 0xe6, 0xe8, 0x35, 0xe8, 0xa4, 0x12, 0x1c, 0xe2, 0x0f,
|
||||||
|
- 0x2b, 0xf7, 0x2b, 0x1b, 0x2d, 0xbb, 0x9d, 0xe2, 0xf6, 0x7d, 0x74, 0x1b, 0x96, 0xf1, 0x05, 0xf6,
|
||||||
|
- 0xc4, 0x6e, 0x55, 0xee, 0x36, 0x04, 0xb8, 0xef, 0xa3, 0xf7, 0xa1, 0x1d, 0x73, 0x46, 0xc2, 0x89,
|
||||||
|
- 0x93, 0xc4, 0x98, 0x0d, 0x6b, 0xf7, 0x2b, 0x1b, 0xed, 0x87, 0x2b, 0x63, 0x61, 0xd2, 0xf8, 0x50,
|
||||||
|
- 0x6e, 0x1c, 0xc5, 0x98, 0xd9, 0x10, 0xa7, 0x6b, 0xf4, 0x00, 0x96, 0x7d, 0x7c, 0x4e, 0x3c, 0x1c,
|
||||||
|
- 0x0f, 0xeb, 0xf7, 0x6b, 0x1b, 0xed, 0x87, 0x1d, 0x45, 0xbe, 0x23, 0x91, 0xb6, 0xd9, 0x44, 0x6f,
|
||||||
|
- 0x43, 0x33, 0xe6, 0x94, 0xb9, 0x13, 0x1c, 0x0f, 0x97, 0x24, 0x61, 0xd7, 0xf0, 0x95, 0x58, 0x3b,
|
||||||
|
- 0xdd, 0x46, 0xaf, 0x40, 0xed, 0xd9, 0xf6, 0xfe, 0xb0, 0x21, 0xa5, 0x83, 0xa6, 0x8a, 0xb0, 0x67,
|
||||||
|
- 0x0b, 0x34, 0x7a, 0x1d, 0xba, 0xb1, 0x1b, 0xfa, 0xc7, 0xf4, 0xc2, 0x89, 0x88, 0x1f, 0xc6, 0xc3,
|
||||||
|
- 0xe5, 0xfb, 0x95, 0x8d, 0xa6, 0xdd, 0xd1, 0xc8, 0x03, 0x81, 0xb3, 0x3e, 0x85, 0x5b, 0x87, 0xdc,
|
||||||
|
- 0x65, 0xfc, 0x1a, 0xde, 0xb1, 0x8e, 0x60, 0xdd, 0xc6, 0x01, 0x3d, 0xbf, 0x96, 0x6b, 0x87, 0xb0,
|
||||||
|
- 0xcc, 0x49, 0x80, 0x69, 0xc2, 0xa5, 0x6b, 0xbb, 0xb6, 0x01, 0xad, 0x3f, 0x57, 0x00, 0xed, 0x5e,
|
||||||
|
- 0x60, 0xef, 0x80, 0x51, 0x0f, 0xc7, 0xf1, 0xff, 0xe8, 0xba, 0xde, 0x82, 0xe5, 0x48, 0x29, 0x30,
|
||||||
|
- 0xac, 0x4b, 0x72, 0x7d, 0x0b, 0x46, 0x2b, 0xb3, 0x6b, 0x7d, 0x0d, 0x6b, 0x87, 0x64, 0x12, 0xba,
|
||||||
|
- 0xd3, 0x1b, 0xd4, 0x77, 0x1d, 0x1a, 0xb1, 0xe4, 0x29, 0x55, 0xed, 0xda, 0x1a, 0xb2, 0x0e, 0x00,
|
||||||
|
- 0x7d, 0xe5, 0x12, 0x7e, 0x73, 0x92, 0xac, 0x77, 0x61, 0xb5, 0xc0, 0x31, 0x8e, 0x68, 0x18, 0x63,
|
||||||
|
- 0xa9, 0x00, 0x77, 0x79, 0x12, 0x4b, 0x66, 0x4b, 0xb6, 0x86, 0x2c, 0x0c, 0x6b, 0x5f, 0x92, 0xd8,
|
||||||
|
- 0x90, 0xe3, 0xff, 0x46, 0x85, 0x75, 0x68, 0x9c, 0x50, 0x16, 0xb8, 0xdc, 0x68, 0xa0, 0x20, 0x84,
|
||||||
|
- 0xa0, 0xee, 0xb2, 0x49, 0x3c, 0xac, 0xdd, 0xaf, 0x6d, 0xb4, 0x6c, 0xb9, 0x16, 0xaf, 0x72, 0x4e,
|
||||||
|
- 0x8c, 0xd6, 0xeb, 0x35, 0xe8, 0x68, 0xbf, 0x3b, 0x53, 0x12, 0x73, 0x29, 0xa7, 0x63, 0xb7, 0x35,
|
||||||
|
- 0x4e, 0x9c, 0xb1, 0x28, 0xac, 0x1f, 0x45, 0xfe, 0x35, 0x03, 0xfe, 0x21, 0xb4, 0x18, 0x8e, 0x69,
|
||||||
|
- 0xc2, 0x44, 0x98, 0x56, 0xe5, 0xbd, 0xaf, 0xa9, 0x7b, 0xff, 0x92, 0x84, 0xc9, 0x85, 0x6d, 0xf6,
|
||||||
|
- 0xec, 0x8c, 0x4c, 0x87, 0x10, 0x8f, 0xaf, 0x13, 0x42, 0x9f, 0xc2, 0xad, 0x03, 0x37, 0x89, 0xaf,
|
||||||
|
- 0xa3, 0xab, 0xf5, 0x99, 0x08, 0xbf, 0x38, 0x09, 0xae, 0x75, 0xf8, 0x4f, 0x15, 0x68, 0x6e, 0x47,
|
||||||
|
- 0xc9, 0x51, 0xec, 0x4e, 0x30, 0xfa, 0x3f, 0x68, 0x73, 0xca, 0xdd, 0xa9, 0x93, 0x08, 0x50, 0x92,
|
||||||
|
- 0xd7, 0x6d, 0x90, 0x28, 0x45, 0x20, 0xdc, 0x8e, 0x99, 0x17, 0x25, 0x9a, 0xa2, 0x7a, 0xbf, 0xb6,
|
||||||
|
- 0x51, 0xb7, 0xdb, 0x0a, 0xa7, 0x48, 0xc6, 0xb0, 0x2a, 0xf7, 0x1c, 0x12, 0x3a, 0x67, 0x98, 0x85,
|
||||||
|
- 0x78, 0x1a, 0x50, 0x1f, 0xcb, 0xf7, 0x5b, 0xb7, 0x07, 0x72, 0x6b, 0x3f, 0xfc, 0x22, 0xdd, 0x40,
|
||||||
|
- 0xff, 0x0f, 0x83, 0x94, 0x5e, 0x04, 0xa5, 0xa4, 0xae, 0x4b, 0xea, 0xbe, 0xa6, 0x3e, 0xd2, 0x68,
|
||||||
|
- 0xeb, 0xd7, 0xd0, 0x7b, 0x7e, 0xca, 0x28, 0xe7, 0x53, 0x12, 0x4e, 0x76, 0x5c, 0xee, 0x8a, 0xec,
|
||||||
|
- 0x11, 0x61, 0x46, 0xa8, 0x1f, 0x6b, 0x6d, 0x0d, 0x88, 0xde, 0x81, 0x01, 0x57, 0xb4, 0xd8, 0x77,
|
||||||
|
- 0x0c, 0x4d, 0x55, 0xd2, 0xac, 0xa4, 0x1b, 0x07, 0x9a, 0xf8, 0x4d, 0xe8, 0x65, 0xc4, 0x22, 0xff,
|
||||||
|
- 0x68, 0x7d, 0xbb, 0x29, 0xf6, 0x39, 0x09, 0xb0, 0x75, 0x2e, 0x7d, 0x25, 0x2f, 0x19, 0xbd, 0x03,
|
||||||
|
- 0xad, 0xcc, 0x0f, 0x15, 0xf9, 0x42, 0x7a, 0xea, 0x85, 0x18, 0x77, 0xda, 0xcd, 0xd4, 0x29, 0x9f,
|
||||||
|
- 0x43, 0x9f, 0xa7, 0x8a, 0x3b, 0xbe, 0xcb, 0xdd, 0xe2, 0xa3, 0x2a, 0x5a, 0x65, 0xf7, 0x78, 0x01,
|
||||||
|
- 0xb6, 0x3e, 0x83, 0xd6, 0x01, 0xf1, 0x63, 0x25, 0x78, 0x08, 0xcb, 0x5e, 0xc2, 0x18, 0x0e, 0xb9,
|
||||||
|
- 0x31, 0x59, 0x83, 0x68, 0x0d, 0x96, 0xa6, 0x24, 0x20, 0x5c, 0x9b, 0xa9, 0x00, 0x8b, 0x02, 0x3c,
|
||||||
|
- 0xc5, 0x01, 0x65, 0x33, 0xe9, 0xb0, 0x35, 0x58, 0xca, 0x5f, 0xae, 0x02, 0xd0, 0x5d, 0x68, 0x05,
|
||||||
|
- 0xee, 0x45, 0x7a, 0xa9, 0x62, 0xa7, 0x19, 0xb8, 0x17, 0x4a, 0xf9, 0x21, 0x2c, 0x9f, 0xb8, 0x64,
|
||||||
|
- 0xea, 0x85, 0x5c, 0x7b, 0xc5, 0x80, 0x99, 0xc0, 0x7a, 0x5e, 0xe0, 0xdf, 0xaa, 0xd0, 0x56, 0x12,
|
||||||
|
- 0x95, 0xc2, 0x6b, 0xb0, 0xe4, 0xb9, 0xde, 0x69, 0x2a, 0x52, 0x02, 0xe8, 0x81, 0x51, 0xa4, 0x9a,
|
||||||
|
- 0x4f, 0xc2, 0x99, 0xa6, 0x46, 0xb5, 0x4d, 0x80, 0xf8, 0x85, 0x1b, 0x69, 0xdd, 0x6a, 0x97, 0x10,
|
||||||
|
- 0xb7, 0x04, 0x8d, 0x52, 0xf7, 0x03, 0xe8, 0xa8, 0x77, 0xa7, 0x8f, 0xd4, 0x2f, 0x39, 0xd2, 0x56,
|
||||||
|
- 0x54, 0xea, 0xd0, 0xeb, 0xd0, 0x4d, 0x62, 0xec, 0x9c, 0x12, 0xcc, 0x5c, 0xe6, 0x9d, 0xce, 0x86,
|
||||||
|
- 0x4b, 0xea, 0x1b, 0x99, 0xc4, 0x78, 0xcf, 0xe0, 0xd0, 0x43, 0x58, 0x12, 0xe9, 0x2f, 0x1e, 0x36,
|
||||||
|
- 0xe4, 0xe7, 0xf8, 0x95, 0x3c, 0x4b, 0x69, 0xea, 0x58, 0xfe, 0xee, 0x86, 0x9c, 0xcd, 0x6c, 0x45,
|
||||||
|
- 0x3a, 0xfa, 0x18, 0x20, 0x43, 0xa2, 0x15, 0xa8, 0x9d, 0xe1, 0x99, 0x8e, 0x43, 0xb1, 0x14, 0xce,
|
||||||
|
- 0x39, 0x77, 0xa7, 0x89, 0xf1, 0xba, 0x02, 0x3e, 0xad, 0x7e, 0x5c, 0xb1, 0x3c, 0xe8, 0x6f, 0x4d,
|
||||||
|
- 0xcf, 0x08, 0xcd, 0x1d, 0x5f, 0x83, 0xa5, 0xc0, 0xfd, 0x9a, 0x32, 0xe3, 0x49, 0x09, 0x48, 0x2c,
|
||||||
|
- 0x09, 0x29, 0x33, 0x2c, 0x24, 0x80, 0x7a, 0x50, 0xa5, 0x91, 0xf4, 0x57, 0xcb, 0xae, 0xd2, 0x28,
|
||||||
|
- 0x13, 0x54, 0xcf, 0x09, 0xb2, 0xfe, 0x59, 0x07, 0xc8, 0xa4, 0x20, 0x1b, 0x46, 0x84, 0x3a, 0x31,
|
||||||
|
- 0x66, 0xa2, 0x04, 0x71, 0x8e, 0x67, 0x1c, 0xc7, 0x0e, 0xc3, 0x5e, 0xc2, 0x62, 0x72, 0x2e, 0xee,
|
||||||
|
- 0x4f, 0x98, 0x7d, 0x4b, 0x99, 0x3d, 0xa7, 0x9b, 0x7d, 0x9b, 0xd0, 0x43, 0x75, 0x6e, 0x4b, 0x1c,
|
||||||
|
- 0xb3, 0xcd, 0x29, 0xb4, 0x0f, 0xb7, 0x32, 0x9e, 0x7e, 0x8e, 0x5d, 0xf5, 0x2a, 0x76, 0xab, 0x29,
|
||||||
|
- 0x3b, 0x3f, 0x63, 0xb5, 0x0b, 0xab, 0x84, 0x3a, 0xdf, 0x24, 0x38, 0x29, 0x30, 0xaa, 0x5d, 0xc5,
|
||||||
|
- 0x68, 0x40, 0xe8, 0xcf, 0xe4, 0x81, 0x8c, 0xcd, 0x01, 0xdc, 0xc9, 0x59, 0x29, 0xc2, 0x3d, 0xc7,
|
||||||
|
- 0xac, 0x7e, 0x15, 0xb3, 0xf5, 0x54, 0x2b, 0x91, 0x0f, 0x32, 0x8e, 0x3f, 0x81, 0x75, 0x42, 0x9d,
|
||||||
|
- 0x17, 0x2e, 0xe1, 0xf3, 0xec, 0x96, 0x5e, 0x62, 0xa4, 0xf8, 0xe8, 0x16, 0x79, 0x29, 0x23, 0x03,
|
||||||
|
- 0xcc, 0x26, 0x05, 0x23, 0x1b, 0x2f, 0x31, 0xf2, 0xa9, 0x3c, 0x90, 0xb1, 0x79, 0x0c, 0x03, 0x42,
|
||||||
|
- 0xe7, 0xb5, 0x59, 0xbe, 0x8a, 0x49, 0x9f, 0xd0, 0xa2, 0x26, 0x5b, 0x30, 0x88, 0xb1, 0xc7, 0x29,
|
||||||
|
- 0xcb, 0x3f, 0x82, 0xe6, 0x55, 0x2c, 0x56, 0x34, 0x7d, 0xca, 0xc3, 0xfa, 0x05, 0x74, 0xf6, 0x92,
|
||||||
|
- 0x09, 0xe6, 0xd3, 0xe3, 0x34, 0x19, 0xdc, 0x58, 0xfe, 0xb1, 0xfe, 0x5d, 0x85, 0xf6, 0xf6, 0x84,
|
||||||
|
- 0xd1, 0x24, 0x2a, 0xe4, 0x64, 0x15, 0xa4, 0xf3, 0x39, 0x59, 0x92, 0xc8, 0x9c, 0xac, 0x88, 0x3f,
|
||||||
|
- 0x84, 0x4e, 0x20, 0x43, 0x57, 0xd3, 0xab, 0x3c, 0x34, 0x58, 0x08, 0x6a, 0xbb, 0x1d, 0xe4, 0x92,
|
||||||
|
- 0xd9, 0x18, 0x20, 0x22, 0x7e, 0xac, 0xcf, 0xa8, 0x74, 0xd4, 0xd7, 0x15, 0xa1, 0x49, 0xd1, 0x76,
|
||||||
|
- 0x2b, 0x4a, 0xb3, 0xf5, 0xfb, 0xd0, 0x3e, 0x16, 0x4e, 0xd2, 0x07, 0x0a, 0xc9, 0x28, 0xf3, 0x9e,
|
||||||
|
- 0x0d, 0xc7, 0x59, 0x10, 0xee, 0x41, 0xf7, 0x54, 0xb9, 0x4c, 0x1f, 0x52, 0x6f, 0xe8, 0x75, 0x6d,
|
||||||
|
- 0x49, 0x66, 0xef, 0x38, 0xef, 0x59, 0x75, 0x01, 0x9d, 0xd3, 0x1c, 0x6a, 0x74, 0x08, 0x83, 0x05,
|
||||||
|
- 0x92, 0x92, 0x1c, 0xb4, 0x91, 0xcf, 0x41, 0xed, 0x87, 0x48, 0x09, 0xca, 0x9f, 0xcc, 0xe7, 0xa5,
|
||||||
|
- 0xdf, 0x55, 0xa1, 0xf3, 0x53, 0xcc, 0x5f, 0x50, 0x76, 0xa6, 0xf4, 0x45, 0x50, 0x0f, 0xdd, 0x00,
|
||||||
|
- 0x6b, 0x8e, 0x72, 0x8d, 0xee, 0x40, 0x93, 0x5d, 0xa8, 0x04, 0xa2, 0xef, 0x73, 0x99, 0x5d, 0xc8,
|
||||||
|
- 0xc4, 0x80, 0x5e, 0x05, 0x60, 0x17, 0x4e, 0xe4, 0x7a, 0x67, 0x58, 0x7b, 0xb0, 0x6e, 0xb7, 0xd8,
|
||||||
|
- 0xc5, 0x81, 0x42, 0x88, 0xa7, 0xc0, 0x2e, 0x1c, 0xcc, 0x18, 0x65, 0xb1, 0xce, 0x55, 0x4d, 0x76,
|
||||||
|
- 0xb1, 0x2b, 0x61, 0x7d, 0xd6, 0x67, 0x34, 0x8a, 0xb0, 0x2f, 0x73, 0xb4, 0x3c, 0xbb, 0xa3, 0x10,
|
||||||
|
- 0x42, 0x2a, 0x37, 0x52, 0x1b, 0x4a, 0x2a, 0xcf, 0xa4, 0xf2, 0x4c, 0xea, 0xb2, 0x3a, 0xc9, 0xf3,
|
||||||
|
- 0x52, 0x79, 0x2a, 0xb5, 0xa9, 0xa4, 0xf2, 0x9c, 0x54, 0x9e, 0x49, 0x6d, 0x99, 0xb3, 0x5a, 0xaa,
|
||||||
|
- 0xf5, 0xdb, 0x0a, 0xac, 0xcf, 0x17, 0x7e, 0xba, 0x4c, 0xfd, 0x10, 0x3a, 0x9e, 0xbc, 0xaf, 0xc2,
|
||||||
|
- 0x9b, 0x1c, 0x2c, 0xdc, 0xa4, 0xdd, 0xf6, 0x72, 0xcf, 0xf8, 0x23, 0xe8, 0x86, 0xca, 0xc1, 0xe9,
|
||||||
|
- 0xd3, 0xac, 0x65, 0xf7, 0x92, 0xf7, 0xbd, 0xdd, 0x09, 0x73, 0x90, 0xe5, 0x03, 0xfa, 0x8a, 0x11,
|
||||||
|
- 0x8e, 0x0f, 0x39, 0xc3, 0x6e, 0x70, 0x13, 0x0d, 0x08, 0x82, 0xba, 0xac, 0x56, 0x6a, 0xb2, 0xbe,
|
||||||
|
- 0x96, 0x6b, 0xeb, 0x2d, 0x58, 0x2d, 0x48, 0xd1, 0xb6, 0xae, 0x40, 0x6d, 0x8a, 0x43, 0xc9, 0xbd,
|
||||||
|
- 0x6b, 0x8b, 0xa5, 0xe5, 0xc2, 0xc0, 0xc6, 0xae, 0x7f, 0x73, 0xda, 0x68, 0x11, 0xb5, 0x4c, 0xc4,
|
||||||
|
- 0x06, 0xa0, 0xbc, 0x08, 0xad, 0x8a, 0xd1, 0xba, 0x92, 0xd3, 0xfa, 0x19, 0x0c, 0xb6, 0xa7, 0x34,
|
||||||
|
- 0xc6, 0x87, 0xdc, 0x27, 0xe1, 0x4d, 0x74, 0x4c, 0xbf, 0x82, 0xd5, 0xe7, 0x7c, 0xf6, 0x95, 0x60,
|
||||||
|
- 0x16, 0x93, 0x6f, 0xf1, 0x0d, 0xd9, 0xc7, 0xe8, 0x0b, 0x63, 0x1f, 0xa3, 0x2f, 0x44, 0xb3, 0xe4,
|
||||||
|
- 0xd1, 0x69, 0x12, 0x84, 0x32, 0x14, 0xba, 0xb6, 0x86, 0xac, 0x2d, 0xe8, 0xa8, 0x1a, 0xfa, 0x29,
|
||||||
|
- 0xf5, 0x93, 0x29, 0x2e, 0x8d, 0xc1, 0x7b, 0x00, 0x91, 0xcb, 0xdc, 0x00, 0x73, 0xcc, 0xd4, 0x1b,
|
||||||
|
- 0x6a, 0xd9, 0x39, 0x8c, 0xf5, 0x87, 0x2a, 0xac, 0xa9, 0x91, 0xc8, 0xa1, 0x9a, 0x04, 0x18, 0x13,
|
||||||
|
- 0x46, 0xd0, 0x3c, 0xa5, 0x31, 0xcf, 0x31, 0x4c, 0x61, 0xa1, 0xa2, 0x1f, 0x1a, 0x6e, 0x62, 0x59,
|
||||||
|
- 0x98, 0x53, 0xd4, 0xae, 0x9e, 0x53, 0x2c, 0x4c, 0x22, 0xea, 0x8b, 0x93, 0x08, 0x11, 0x6d, 0x86,
|
||||||
|
- 0x88, 0xa8, 0x18, 0x6f, 0xd9, 0x2d, 0x8d, 0xd9, 0xf7, 0xd1, 0x03, 0xe8, 0x4f, 0x84, 0x96, 0xce,
|
||||||
|
- 0x29, 0xa5, 0x67, 0x4e, 0xe4, 0xf2, 0x53, 0x19, 0xea, 0x2d, 0xbb, 0x2b, 0xd1, 0x7b, 0x94, 0x9e,
|
||||||
|
- 0x1d, 0xb8, 0xfc, 0x14, 0x7d, 0x02, 0x3d, 0x5d, 0x06, 0x06, 0xd2, 0x45, 0xb1, 0xfe, 0xf8, 0xe9,
|
||||||
|
- 0x28, 0xca, 0x7b, 0xcf, 0xee, 0x9e, 0xe5, 0xa0, 0xd8, 0xba, 0x0d, 0xb7, 0x76, 0x70, 0xcc, 0x19,
|
||||||
|
- 0x9d, 0x15, 0x1d, 0x63, 0xfd, 0x08, 0x60, 0x3f, 0xe4, 0x98, 0x9d, 0xb8, 0x1e, 0x8e, 0xd1, 0x7b,
|
||||||
|
- 0x79, 0x48, 0x17, 0x47, 0x2b, 0x63, 0x35, 0x91, 0x4a, 0x37, 0xec, 0x1c, 0x8d, 0x35, 0x86, 0x86,
|
||||||
|
- 0x4d, 0x13, 0x91, 0x8e, 0xde, 0x30, 0x2b, 0x7d, 0xae, 0xa3, 0xcf, 0x49, 0xa4, 0xad, 0xf7, 0xac,
|
||||||
|
- 0x3d, 0xd3, 0xc2, 0x66, 0xec, 0xf4, 0x15, 0x8d, 0xa1, 0x45, 0x0c, 0x4e, 0x67, 0x95, 0x45, 0xd1,
|
||||||
|
- 0x19, 0x89, 0xf5, 0x19, 0xac, 0x2a, 0x4e, 0x8a, 0xb3, 0x61, 0xf3, 0x06, 0x34, 0x98, 0x51, 0xa3,
|
||||||
|
- 0x92, 0x8d, 0xa2, 0x34, 0x91, 0xde, 0x13, 0xfe, 0x10, 0x1d, 0x75, 0x66, 0x88, 0xf1, 0xc7, 0x2a,
|
||||||
|
- 0x0c, 0xc4, 0x46, 0x81, 0xa7, 0xf5, 0x4b, 0x58, 0x7d, 0x16, 0x4e, 0x49, 0x88, 0xb7, 0x0f, 0x8e,
|
||||||
|
- 0x9e, 0xe2, 0x34, 0xee, 0x11, 0xd4, 0x45, 0x7d, 0x24, 0x05, 0x35, 0x6d, 0xb9, 0x16, 0x81, 0x10,
|
||||||
|
- 0x1e, 0x3b, 0x5e, 0x94, 0xc4, 0x7a, 0xf6, 0xd3, 0x08, 0x8f, 0xb7, 0xa3, 0x24, 0x16, 0x89, 0x5c,
|
||||||
|
- 0x7c, 0xc8, 0x69, 0x38, 0x9d, 0xc9, 0x68, 0x68, 0xda, 0xcb, 0x5e, 0x94, 0x3c, 0x0b, 0xa7, 0x33,
|
||||||
|
- 0xeb, 0x07, 0xb2, 0xdb, 0xc5, 0xd8, 0xb7, 0xdd, 0xd0, 0xa7, 0xc1, 0x0e, 0x3e, 0xcf, 0x49, 0x48,
|
||||||
|
- 0x3b, 0x2b, 0x13, 0xf5, 0xdf, 0x55, 0xa0, 0xf3, 0x78, 0x82, 0x43, 0xbe, 0x83, 0xb9, 0x4b, 0xa6,
|
||||||
|
- 0xb2, 0x7b, 0x3a, 0xc7, 0x2c, 0x26, 0x34, 0xd4, 0x4f, 0xdb, 0x80, 0xa2, 0xf9, 0x25, 0x21, 0xe1,
|
||||||
|
- 0x8e, 0xef, 0xe2, 0x80, 0x86, 0x92, 0x4b, 0xd3, 0x06, 0x81, 0xda, 0x91, 0x18, 0xf4, 0x16, 0xf4,
|
||||||
|
- 0xd5, 0x6c, 0xce, 0x39, 0x75, 0x43, 0x7f, 0x2a, 0x82, 0x4a, 0xcd, 0x2a, 0x7a, 0x0a, 0xbd, 0xa7,
|
||||||
|
- 0xb1, 0xe8, 0x6d, 0x58, 0xd1, 0x4f, 0x3e, 0xa3, 0xac, 0x4b, 0xca, 0xbe, 0xc6, 0x17, 0x48, 0x93,
|
||||||
|
- 0x28, 0xa2, 0x8c, 0xc7, 0x4e, 0x8c, 0x3d, 0x8f, 0x06, 0x91, 0x6e, 0x3d, 0xfa, 0x06, 0x7f, 0xa8,
|
||||||
|
- 0xd0, 0xd6, 0x04, 0x56, 0x9f, 0x08, 0x3b, 0xb5, 0x25, 0xd9, 0x15, 0xf6, 0x02, 0x1c, 0x38, 0xc7,
|
||||||
|
- 0x53, 0xea, 0x9d, 0x39, 0x22, 0x11, 0x69, 0x0f, 0x8b, 0xe2, 0x66, 0x4b, 0x20, 0x0f, 0xc9, 0xb7,
|
||||||
|
- 0xb2, 0xcb, 0x16, 0x54, 0xa7, 0x94, 0x47, 0xd3, 0x64, 0xe2, 0x44, 0x8c, 0x1e, 0x63, 0x6d, 0x62,
|
||||||
|
- 0x3f, 0xc0, 0xc1, 0x9e, 0xc2, 0x1f, 0x08, 0xb4, 0xf5, 0xd7, 0x0a, 0xac, 0x15, 0x25, 0xe9, 0xb4,
|
||||||
|
- 0xba, 0x09, 0x6b, 0x45, 0x51, 0xfa, 0x53, 0xab, 0x4a, 0xb9, 0x41, 0x5e, 0xa0, 0xfa, 0xe8, 0x7e,
|
||||||
|
- 0x04, 0x5d, 0x39, 0xb0, 0x75, 0x7c, 0xc5, 0xa9, 0x58, 0x60, 0xe4, 0xef, 0xc5, 0xee, 0xb8, 0xf9,
|
||||||
|
- 0x5b, 0xfa, 0x04, 0xee, 0x68, 0xf3, 0x9d, 0x45, 0xb5, 0xd5, 0x83, 0x58, 0xd7, 0x04, 0x4f, 0xe7,
|
||||||
|
- 0xb4, 0xff, 0x12, 0x86, 0x19, 0x6a, 0x6b, 0x26, 0x91, 0xc6, 0x57, 0xef, 0xc1, 0xea, 0x9c, 0xb1,
|
||||||
|
- 0x8f, 0x7d, 0x9f, 0xc9, 0x10, 0xac, 0xdb, 0x65, 0x5b, 0xd6, 0x23, 0xb8, 0x7d, 0x88, 0xb9, 0xf2,
|
||||||
|
- 0x86, 0xcb, 0x75, 0xd5, 0xaf, 0x98, 0xad, 0x40, 0xed, 0x10, 0x7b, 0xd2, 0xf8, 0x9a, 0x2d, 0x96,
|
||||||
|
- 0xe2, 0x01, 0x1e, 0xc5, 0xd8, 0x93, 0x56, 0xd6, 0x6c, 0xb9, 0xb6, 0xfe, 0x52, 0x81, 0x65, 0x9d,
|
||||||
|
- 0x08, 0x45, 0x32, 0xf7, 0x19, 0x39, 0xc7, 0x4c, 0x3f, 0x3d, 0x0d, 0xa1, 0x37, 0xa1, 0xa7, 0x56,
|
||||||
|
- 0x0e, 0x8d, 0x38, 0xa1, 0x69, 0x7a, 0xed, 0x2a, 0xec, 0x33, 0x85, 0x94, 0xb3, 0x38, 0x39, 0x6a,
|
||||||
|
- 0xd2, 0x5d, 0x9d, 0x86, 0xe4, 0x40, 0x2d, 0x16, 0xb1, 0x2f, 0xd3, 0x69, 0xcb, 0xd6, 0x90, 0x78,
|
||||||
|
- 0xea, 0x86, 0xdf, 0x92, 0xe4, 0x67, 0x40, 0xf1, 0xd4, 0x03, 0x9a, 0x84, 0xdc, 0x89, 0x28, 0x09,
|
||||||
|
- 0xb9, 0xce, 0x9f, 0x20, 0x51, 0x07, 0x02, 0x63, 0xfd, 0xa6, 0x02, 0x0d, 0x35, 0x8f, 0x16, 0x7d,
|
||||||
|
- 0x64, 0xfa, 0x15, 0xab, 0x12, 0x59, 0x11, 0x48, 0x59, 0xea, 0xcb, 0x25, 0xd7, 0x22, 0x8e, 0xcf,
|
||||||
|
- 0x03, 0x95, 0x8b, 0xb5, 0x6a, 0xe7, 0x81, 0x4c, 0xc2, 0x6f, 0x42, 0x2f, 0xfb, 0x18, 0xca, 0x7d,
|
||||||
|
- 0xa5, 0x62, 0x37, 0xc5, 0x4a, 0xb2, 0x4b, 0x35, 0xb5, 0x7e, 0x2e, 0xda, 0xe7, 0x74, 0x16, 0xbb,
|
||||||
|
- 0x02, 0xb5, 0x24, 0x55, 0x46, 0x2c, 0x05, 0x66, 0x92, 0x7e, 0x46, 0xc5, 0x12, 0x3d, 0x80, 0x9e,
|
||||||
|
- 0xeb, 0xfb, 0x44, 0x1c, 0x77, 0xa7, 0x4f, 0x88, 0x9f, 0x06, 0x69, 0x11, 0x6b, 0xfd, 0xbd, 0x02,
|
||||||
|
- 0xfd, 0x6d, 0x1a, 0xcd, 0x7e, 0x4c, 0xa6, 0x38, 0x97, 0x41, 0xa4, 0x92, 0xfa, 0x2b, 0x2a, 0xd6,
|
||||||
|
- 0xa2, 0x32, 0x3c, 0x21, 0x53, 0xac, 0x42, 0x4b, 0xdd, 0x6c, 0x53, 0x20, 0x64, 0x58, 0x99, 0xcd,
|
||||||
|
- 0x74, 0xc4, 0xd5, 0x55, 0x9b, 0x4f, 0xa9, 0x2f, 0x6b, 0x60, 0x9f, 0x30, 0x27, 0x1d, 0x68, 0x75,
|
||||||
|
- 0xed, 0x65, 0x9f, 0x30, 0xb9, 0xa5, 0x0d, 0x59, 0x92, 0x33, 0xd5, 0xbc, 0x21, 0x0d, 0x85, 0x11,
|
||||||
|
- 0x86, 0xac, 0x43, 0x83, 0x9e, 0x9c, 0xc4, 0x98, 0xcb, 0x6a, 0xb5, 0x66, 0x6b, 0x28, 0x4d, 0x73,
|
||||||
|
- 0xcd, 0x5c, 0x9a, 0xbb, 0x05, 0xab, 0x72, 0x7a, 0xff, 0x9c, 0xb9, 0x1e, 0x09, 0x27, 0x26, 0x15,
|
||||||
|
- 0xaf, 0x01, 0x3a, 0xe4, 0x34, 0x2a, 0x62, 0x1f, 0xfe, 0x7e, 0x45, 0xe7, 0x44, 0xdd, 0xca, 0xa2,
|
||||||
|
- 0x27, 0xd0, 0x9f, 0xfb, 0x6b, 0x04, 0xe9, 0xd9, 0x46, 0xf9, 0x3f, 0x26, 0xa3, 0xf5, 0xb1, 0xfa,
|
||||||
|
- 0xab, 0x65, 0x6c, 0xfe, 0x6a, 0x19, 0xef, 0x06, 0x11, 0x9f, 0xa1, 0x5d, 0xe8, 0x15, 0xff, 0x44,
|
||||||
|
- 0x40, 0x77, 0x4d, 0x29, 0x50, 0xf2, 0xd7, 0xc2, 0xa5, 0x6c, 0x9e, 0x40, 0x7f, 0xee, 0xff, 0x04,
|
||||||
|
- 0xa3, 0x4f, 0xf9, 0xdf, 0x0c, 0x97, 0x32, 0x7a, 0x04, 0xed, 0xdc, 0x1f, 0x08, 0x68, 0xa8, 0x98,
|
||||||
|
- 0x2c, 0xfe, 0xa7, 0x70, 0x29, 0x83, 0x6d, 0xe8, 0x16, 0x66, 0xfa, 0x68, 0xa4, 0xed, 0x29, 0x19,
|
||||||
|
- 0xf4, 0x5f, 0xca, 0x64, 0x0b, 0xda, 0xb9, 0xd1, 0xba, 0xd1, 0x62, 0x71, 0x7e, 0x3f, 0xba, 0x53,
|
||||||
|
- 0xb2, 0xa3, 0x53, 0xef, 0x1e, 0x74, 0x0b, 0x83, 0x70, 0xa3, 0x48, 0xd9, 0x10, 0x7e, 0x74, 0xb7,
|
||||||
|
- 0x74, 0x4f, 0x73, 0x7a, 0x02, 0xfd, 0xb9, 0xb1, 0xb8, 0x71, 0x6e, 0xf9, 0xb4, 0xfc, 0x52, 0xb3,
|
||||||
|
- 0xbe, 0x90, 0x97, 0x9d, 0xeb, 0x7a, 0x72, 0x97, 0xbd, 0x38, 0x04, 0x1f, 0xbd, 0x52, 0xbe, 0xa9,
|
||||||
|
- 0xb5, 0xda, 0x85, 0x5e, 0x71, 0xfe, 0x6d, 0x98, 0x95, 0x4e, 0xc5, 0xaf, 0x7e, 0x39, 0x85, 0x51,
|
||||||
|
- 0x78, 0xf6, 0x72, 0xca, 0x26, 0xe4, 0x97, 0x32, 0x7a, 0x0c, 0xa0, 0x7b, 0x1c, 0x9f, 0x84, 0xe9,
|
||||||
|
- 0x95, 0x2d, 0xf4, 0x56, 0xe9, 0x95, 0x95, 0xf4, 0x43, 0x8f, 0x00, 0x54, 0x6b, 0xe2, 0xd3, 0x84,
|
||||||
|
- 0xa3, 0xdb, 0x46, 0x8d, 0xb9, 0x7e, 0x68, 0x34, 0x5c, 0xdc, 0x58, 0x60, 0x80, 0x19, 0xbb, 0x0e,
|
||||||
|
- 0x83, 0xcf, 0x01, 0xb2, 0x96, 0xc7, 0x30, 0x58, 0x68, 0x82, 0xae, 0xf0, 0x41, 0x27, 0xdf, 0xe0,
|
||||||
|
- 0x20, 0x6d, 0x6b, 0x49, 0xd3, 0x73, 0x05, 0x8b, 0xfe, 0x5c, 0x01, 0x5b, 0x7c, 0x6c, 0xf3, 0x75,
|
||||||
|
- 0xed, 0x68, 0xa1, 0x88, 0x45, 0x1f, 0x41, 0x27, 0x5f, 0xb9, 0x1a, 0x2d, 0x4a, 0xaa, 0xd9, 0x51,
|
||||||
|
- 0xa1, 0x7a, 0x45, 0x8f, 0xa0, 0x57, 0xac, 0x5a, 0x51, 0x2e, 0x2e, 0x16, 0x6a, 0xd9, 0x91, 0x9e,
|
||||||
|
- 0xc9, 0xe4, 0xc8, 0x3f, 0x00, 0xc8, 0xaa, 0x5b, 0xe3, 0xbe, 0x85, 0x7a, 0x77, 0x4e, 0xea, 0x63,
|
||||||
|
- 0xe8, 0xe4, 0x33, 0xb1, 0x51, 0xb7, 0x24, 0x3b, 0x5f, 0x95, 0xb5, 0x72, 0x59, 0xdb, 0x3c, 0xbe,
|
||||||
|
- 0xc5, 0x44, 0x7e, 0x55, 0xd6, 0x2a, 0xf4, 0x75, 0x26, 0x59, 0x94, 0x35, 0x7b, 0x57, 0xe5, 0xf2,
|
||||||
|
- 0x62, 0x13, 0x64, 0xdc, 0x57, 0xda, 0x1a, 0x5d, 0xf5, 0x88, 0xf2, 0xdd, 0x80, 0xf1, 0x47, 0x49,
|
||||||
|
- 0x87, 0xf0, 0x92, 0xa0, 0xce, 0x57, 0xfc, 0xb9, 0xa0, 0x2e, 0x69, 0x04, 0x2e, 0x65, 0xb4, 0x07,
|
||||||
|
- 0xfd, 0x27, 0xa6, 0x98, 0xd3, 0x85, 0xa6, 0x56, 0xa7, 0xa4, 0xb0, 0x1e, 0x8d, 0xca, 0xb6, 0x74,
|
||||||
|
- 0x64, 0x7d, 0x01, 0x83, 0x85, 0x22, 0x13, 0xdd, 0x4b, 0x47, 0x87, 0xa5, 0xd5, 0xe7, 0xa5, 0x6a,
|
||||||
|
- 0xed, 0xc3, 0xca, 0x7c, 0x8d, 0x89, 0x5e, 0xd5, 0x97, 0x5e, 0x5e, 0x7b, 0x5e, 0xca, 0xea, 0x13,
|
||||||
|
- 0x68, 0x9a, 0x9a, 0x06, 0xe9, 0x11, 0xed, 0x5c, 0x8d, 0x73, 0xd9, 0xd1, 0xad, 0xce, 0x77, 0xdf,
|
||||||
|
- 0xdf, 0xab, 0xfc, 0xe3, 0xfb, 0x7b, 0x95, 0x7f, 0x7d, 0x7f, 0xaf, 0x72, 0xdc, 0x90, 0xbb, 0x1f,
|
||||||
|
- 0xfc, 0x27, 0x00, 0x00, 0xff, 0xff, 0xa5, 0xac, 0x85, 0x1d, 0xaa, 0x21, 0x00, 0x00,
|
||||||
|
+ 0xd1, 0xd8, 0x07, 0x97, 0xbb, 0xb5, 0x2f, 0x6e, 0x93, 0xa2, 0x56, 0x2b, 0x59, 0x9f, 0x3c, 0xb6,
|
||||||
|
+ 0x65, 0xfa, 0xf3, 0xe7, 0xa5, 0x2d, 0x1b, 0x9f, 0x5f, 0x70, 0x04, 0xf1, 0x11, 0x91, 0xb1, 0x15,
|
||||||
|
+ 0x31, 0x43, 0x11, 0x4e, 0x10, 0x04, 0x83, 0xe1, 0x4c, 0x73, 0xd9, 0xe6, 0xce, 0xf4, 0xb8, 0xa7,
|
||||||
|
+ 0x87, 0xe2, 0x3a, 0x40, 0x8e, 0xc9, 0x2d, 0x97, 0x00, 0xb9, 0xe5, 0x0f, 0x04, 0xb9, 0xe5, 0x98,
|
||||||
|
+ 0x6b, 0x0e, 0x46, 0x4e, 0xf9, 0x05, 0x41, 0xe0, 0x9f, 0x90, 0x5f, 0x10, 0xf4, 0x6b, 0x1e, 0xbb,
|
||||||
|
+ 0x43, 0x1a, 0x21, 0x08, 0xe4, 0xb2, 0xe8, 0xaa, 0xae, 0xae, 0x57, 0x77, 0xd5, 0x54, 0xd5, 0x42,
|
||||||
|
+ 0xdb, 0x9d, 0xe0, 0x90, 0x8f, 0x23, 0x46, 0x39, 0x45, 0xf5, 0x09, 0x8b, 0xbc, 0x51, 0x8b, 0x7a,
|
||||||
|
+ 0x44, 0x21, 0x46, 0xff, 0x3f, 0x21, 0xfc, 0x34, 0x39, 0x1e, 0x7b, 0x34, 0xd8, 0x3c, 0x73, 0xb9,
|
||||||
|
+ 0xfb, 0x8e, 0x47, 0x43, 0xee, 0x92, 0x10, 0xb3, 0x78, 0x53, 0x1e, 0xdc, 0x8c, 0xce, 0x26, 0x9b,
|
||||||
|
+ 0x7c, 0x16, 0xe1, 0x58, 0xfd, 0xea, 0x73, 0x77, 0x27, 0x94, 0x4e, 0xa6, 0x78, 0x53, 0x42, 0xc7,
|
||||||
|
+ 0xc9, 0xc9, 0x26, 0x0e, 0x22, 0x3e, 0x53, 0x9b, 0xd6, 0x1f, 0xaa, 0xb0, 0xbe, 0xcd, 0xb0, 0xcb,
|
||||||
|
+ 0xf1, 0xb6, 0xe1, 0x66, 0xe3, 0xaf, 0x13, 0x1c, 0x73, 0xf4, 0x2a, 0x74, 0x52, 0x09, 0x0e, 0xf1,
|
||||||
|
+ 0x87, 0x95, 0x07, 0x95, 0x8d, 0x96, 0xdd, 0x4e, 0x71, 0xfb, 0x3e, 0xba, 0x0d, 0xcb, 0xf8, 0x02,
|
||||||
|
+ 0x7b, 0x62, 0xb7, 0x2a, 0x77, 0x1b, 0x02, 0xdc, 0xf7, 0xd1, 0x7b, 0xd0, 0x8e, 0x39, 0x23, 0xe1,
|
||||||
|
+ 0xc4, 0x49, 0x62, 0xcc, 0x86, 0xb5, 0x07, 0x95, 0x8d, 0xf6, 0xa3, 0x95, 0xb1, 0x30, 0x69, 0x7c,
|
||||||
|
+ 0x28, 0x37, 0x8e, 0x62, 0xcc, 0x6c, 0x88, 0xd3, 0x35, 0x7a, 0x08, 0xcb, 0x3e, 0x3e, 0x27, 0x1e,
|
||||||
|
+ 0x8e, 0x87, 0xf5, 0x07, 0xb5, 0x8d, 0xf6, 0xa3, 0x8e, 0x22, 0xdf, 0x91, 0x48, 0xdb, 0x6c, 0xa2,
|
||||||
|
+ 0xb7, 0xa0, 0x19, 0x73, 0xca, 0xdc, 0x09, 0x8e, 0x87, 0x4b, 0x92, 0xb0, 0x6b, 0xf8, 0x4a, 0xac,
|
||||||
|
+ 0x9d, 0x6e, 0xa3, 0x7b, 0x50, 0x7b, 0xbe, 0xbd, 0x3f, 0x6c, 0x48, 0xe9, 0xa0, 0xa9, 0x22, 0xec,
|
||||||
|
+ 0xd9, 0x02, 0x8d, 0x5e, 0x83, 0x6e, 0xec, 0x86, 0xfe, 0x31, 0xbd, 0x70, 0x22, 0xe2, 0x87, 0xf1,
|
||||||
|
+ 0x70, 0xf9, 0x41, 0x65, 0xa3, 0x69, 0x77, 0x34, 0xf2, 0x40, 0xe0, 0xac, 0x4f, 0xe0, 0xd6, 0x21,
|
||||||
|
+ 0x77, 0x19, 0xbf, 0x86, 0x77, 0xac, 0x23, 0x58, 0xb7, 0x71, 0x40, 0xcf, 0xaf, 0xe5, 0xda, 0x21,
|
||||||
|
+ 0x2c, 0x73, 0x12, 0x60, 0x9a, 0x70, 0xe9, 0xda, 0xae, 0x6d, 0x40, 0xeb, 0x4f, 0x15, 0x40, 0xbb,
|
||||||
|
+ 0x17, 0xd8, 0x3b, 0x60, 0xd4, 0xc3, 0x71, 0xfc, 0x5f, 0xba, 0xae, 0x37, 0x61, 0x39, 0x52, 0x0a,
|
||||||
|
+ 0x0c, 0xeb, 0x92, 0x5c, 0xdf, 0x82, 0xd1, 0xca, 0xec, 0x5a, 0x5f, 0xc1, 0xda, 0x21, 0x99, 0x84,
|
||||||
|
+ 0xee, 0xf4, 0x06, 0xf5, 0x5d, 0x87, 0x46, 0x2c, 0x79, 0x4a, 0x55, 0xbb, 0xb6, 0x86, 0xac, 0x03,
|
||||||
|
+ 0x40, 0x5f, 0xba, 0x84, 0xdf, 0x9c, 0x24, 0xeb, 0x1d, 0x58, 0x2d, 0x70, 0x8c, 0x23, 0x1a, 0xc6,
|
||||||
|
+ 0x58, 0x2a, 0xc0, 0x5d, 0x9e, 0xc4, 0x92, 0xd9, 0x92, 0xad, 0x21, 0x0b, 0xc3, 0xda, 0x17, 0x24,
|
||||||
|
+ 0x36, 0xe4, 0xf8, 0x3f, 0x51, 0x61, 0x1d, 0x1a, 0x27, 0x94, 0x05, 0x2e, 0x37, 0x1a, 0x28, 0x08,
|
||||||
|
+ 0x21, 0xa8, 0xbb, 0x6c, 0x12, 0x0f, 0x6b, 0x0f, 0x6a, 0x1b, 0x2d, 0x5b, 0xae, 0xc5, 0xab, 0x9c,
|
||||||
|
+ 0x13, 0xa3, 0xf5, 0x7a, 0x15, 0x3a, 0xda, 0xef, 0xce, 0x94, 0xc4, 0x5c, 0xca, 0xe9, 0xd8, 0x6d,
|
||||||
|
+ 0x8d, 0x13, 0x67, 0x2c, 0x0a, 0xeb, 0x47, 0x91, 0x7f, 0xcd, 0x80, 0x7f, 0x04, 0x2d, 0x86, 0x63,
|
||||||
|
+ 0x9a, 0x30, 0x11, 0xa6, 0x55, 0x79, 0xef, 0x6b, 0xea, 0xde, 0xbf, 0x20, 0x61, 0x72, 0x61, 0x9b,
|
||||||
|
+ 0x3d, 0x3b, 0x23, 0xd3, 0x21, 0xc4, 0xe3, 0xeb, 0x84, 0xd0, 0x27, 0x70, 0xeb, 0xc0, 0x4d, 0xe2,
|
||||||
|
+ 0xeb, 0xe8, 0x6a, 0x7d, 0x2a, 0xc2, 0x2f, 0x4e, 0x82, 0x6b, 0x1d, 0xfe, 0x63, 0x05, 0x9a, 0xdb,
|
||||||
|
+ 0x51, 0x72, 0x14, 0xbb, 0x13, 0x8c, 0xfe, 0x07, 0xda, 0x9c, 0x72, 0x77, 0xea, 0x24, 0x02, 0x94,
|
||||||
|
+ 0xe4, 0x75, 0x1b, 0x24, 0x4a, 0x11, 0x08, 0xb7, 0x63, 0xe6, 0x45, 0x89, 0xa6, 0xa8, 0x3e, 0xa8,
|
||||||
|
+ 0x6d, 0xd4, 0xed, 0xb6, 0xc2, 0x29, 0x92, 0x31, 0xac, 0xca, 0x3d, 0x87, 0x84, 0xce, 0x19, 0x66,
|
||||||
|
+ 0x21, 0x9e, 0x06, 0xd4, 0xc7, 0xf2, 0xfd, 0xd6, 0xed, 0x81, 0xdc, 0xda, 0x0f, 0x3f, 0x4f, 0x37,
|
||||||
|
+ 0xd0, 0xff, 0xc2, 0x20, 0xa5, 0x17, 0x41, 0x29, 0xa9, 0xeb, 0x92, 0xba, 0xaf, 0xa9, 0x8f, 0x34,
|
||||||
|
+ 0xda, 0xfa, 0x15, 0xf4, 0x5e, 0x9c, 0x32, 0xca, 0xf9, 0x94, 0x84, 0x93, 0x1d, 0x97, 0xbb, 0x22,
|
||||||
|
+ 0x7b, 0x44, 0x98, 0x11, 0xea, 0xc7, 0x5a, 0x5b, 0x03, 0xa2, 0xb7, 0x61, 0xc0, 0x15, 0x2d, 0xf6,
|
||||||
|
+ 0x1d, 0x43, 0x53, 0x95, 0x34, 0x2b, 0xe9, 0xc6, 0x81, 0x26, 0x7e, 0x03, 0x7a, 0x19, 0xb1, 0xc8,
|
||||||
|
+ 0x3f, 0x5a, 0xdf, 0x6e, 0x8a, 0x7d, 0x41, 0x02, 0x6c, 0x9d, 0x4b, 0x5f, 0xc9, 0x4b, 0x46, 0x6f,
|
||||||
|
+ 0x43, 0x2b, 0xf3, 0x43, 0x45, 0xbe, 0x90, 0x9e, 0x7a, 0x21, 0xc6, 0x9d, 0x76, 0x33, 0x75, 0xca,
|
||||||
|
+ 0x67, 0xd0, 0xe7, 0xa9, 0xe2, 0x8e, 0xef, 0x72, 0xb7, 0xf8, 0xa8, 0x8a, 0x56, 0xd9, 0x3d, 0x5e,
|
||||||
|
+ 0x80, 0xad, 0x4f, 0xa1, 0x75, 0x40, 0xfc, 0x58, 0x09, 0x1e, 0xc2, 0xb2, 0x97, 0x30, 0x86, 0x43,
|
||||||
|
+ 0x6e, 0x4c, 0xd6, 0x20, 0x5a, 0x83, 0xa5, 0x29, 0x09, 0x08, 0xd7, 0x66, 0x2a, 0xc0, 0xa2, 0x00,
|
||||||
|
+ 0xcf, 0x70, 0x40, 0xd9, 0x4c, 0x3a, 0x6c, 0x0d, 0x96, 0xf2, 0x97, 0xab, 0x00, 0x74, 0x17, 0x5a,
|
||||||
|
+ 0x81, 0x7b, 0x91, 0x5e, 0xaa, 0xd8, 0x69, 0x06, 0xee, 0x85, 0x52, 0x7e, 0x08, 0xcb, 0x27, 0x2e,
|
||||||
|
+ 0x99, 0x7a, 0x21, 0xd7, 0x5e, 0x31, 0x60, 0x26, 0xb0, 0x9e, 0x17, 0xf8, 0xd7, 0x2a, 0xb4, 0x95,
|
||||||
|
+ 0x44, 0xa5, 0xf0, 0x1a, 0x2c, 0x79, 0xae, 0x77, 0x9a, 0x8a, 0x94, 0x00, 0x7a, 0x68, 0x14, 0xa9,
|
||||||
|
+ 0xe6, 0x93, 0x70, 0xa6, 0xa9, 0x51, 0x6d, 0x13, 0x20, 0x7e, 0xe9, 0x46, 0x5a, 0xb7, 0xda, 0x25,
|
||||||
|
+ 0xc4, 0x2d, 0x41, 0xa3, 0xd4, 0x7d, 0x1f, 0x3a, 0xea, 0xdd, 0xe9, 0x23, 0xf5, 0x4b, 0x8e, 0xb4,
|
||||||
|
+ 0x15, 0x95, 0x3a, 0xf4, 0x1a, 0x74, 0x93, 0x18, 0x3b, 0xa7, 0x04, 0x33, 0x97, 0x79, 0xa7, 0xb3,
|
||||||
|
+ 0xe1, 0x92, 0xfa, 0x46, 0x26, 0x31, 0xde, 0x33, 0x38, 0xf4, 0x08, 0x96, 0x44, 0xfa, 0x8b, 0x87,
|
||||||
|
+ 0x0d, 0xf9, 0x39, 0xbe, 0x97, 0x67, 0x29, 0x4d, 0x1d, 0xcb, 0xdf, 0xdd, 0x90, 0xb3, 0x99, 0xad,
|
||||||
|
+ 0x48, 0x47, 0x1f, 0x01, 0x64, 0x48, 0xb4, 0x02, 0xb5, 0x33, 0x3c, 0xd3, 0x71, 0x28, 0x96, 0xc2,
|
||||||
|
+ 0x39, 0xe7, 0xee, 0x34, 0x31, 0x5e, 0x57, 0xc0, 0x27, 0xd5, 0x8f, 0x2a, 0x96, 0x07, 0xfd, 0xad,
|
||||||
|
+ 0xe9, 0x19, 0xa1, 0xb9, 0xe3, 0x6b, 0xb0, 0x14, 0xb8, 0x5f, 0x51, 0x66, 0x3c, 0x29, 0x01, 0x89,
|
||||||
|
+ 0x25, 0x21, 0x65, 0x86, 0x85, 0x04, 0x50, 0x0f, 0xaa, 0x34, 0x92, 0xfe, 0x6a, 0xd9, 0x55, 0x1a,
|
||||||
|
+ 0x65, 0x82, 0xea, 0x39, 0x41, 0xd6, 0x3f, 0xea, 0x00, 0x99, 0x14, 0x64, 0xc3, 0x88, 0x50, 0x27,
|
||||||
|
+ 0xc6, 0x4c, 0x94, 0x20, 0xce, 0xf1, 0x8c, 0xe3, 0xd8, 0x61, 0xd8, 0x4b, 0x58, 0x4c, 0xce, 0xc5,
|
||||||
|
+ 0xfd, 0x09, 0xb3, 0x6f, 0x29, 0xb3, 0xe7, 0x74, 0xb3, 0x6f, 0x13, 0x7a, 0xa8, 0xce, 0x6d, 0x89,
|
||||||
|
+ 0x63, 0xb6, 0x39, 0x85, 0xf6, 0xe1, 0x56, 0xc6, 0xd3, 0xcf, 0xb1, 0xab, 0x5e, 0xc5, 0x6e, 0x35,
|
||||||
|
+ 0x65, 0xe7, 0x67, 0xac, 0x76, 0x61, 0x95, 0x50, 0xe7, 0xeb, 0x04, 0x27, 0x05, 0x46, 0xb5, 0xab,
|
||||||
|
+ 0x18, 0x0d, 0x08, 0xfd, 0x89, 0x3c, 0x90, 0xb1, 0x39, 0x80, 0x3b, 0x39, 0x2b, 0x45, 0xb8, 0xe7,
|
||||||
|
+ 0x98, 0xd5, 0xaf, 0x62, 0xb6, 0x9e, 0x6a, 0x25, 0xf2, 0x41, 0xc6, 0xf1, 0x47, 0xb0, 0x4e, 0xa8,
|
||||||
|
+ 0xf3, 0xd2, 0x25, 0x7c, 0x9e, 0xdd, 0xd2, 0xf7, 0x18, 0x29, 0x3e, 0xba, 0x45, 0x5e, 0xca, 0xc8,
|
||||||
|
+ 0x00, 0xb3, 0x49, 0xc1, 0xc8, 0xc6, 0xf7, 0x18, 0xf9, 0x4c, 0x1e, 0xc8, 0xd8, 0x3c, 0x81, 0x01,
|
||||||
|
+ 0xa1, 0xf3, 0xda, 0x2c, 0x5f, 0xc5, 0xa4, 0x4f, 0x68, 0x51, 0x93, 0x2d, 0x18, 0xc4, 0xd8, 0xe3,
|
||||||
|
+ 0x94, 0xe5, 0x1f, 0x41, 0xf3, 0x2a, 0x16, 0x2b, 0x9a, 0x3e, 0xe5, 0x61, 0xfd, 0x1c, 0x3a, 0x7b,
|
||||||
|
+ 0xc9, 0x04, 0xf3, 0xe9, 0x71, 0x9a, 0x0c, 0x6e, 0x2c, 0xff, 0x58, 0xff, 0xaa, 0x42, 0x7b, 0x7b,
|
||||||
|
+ 0xc2, 0x68, 0x12, 0x15, 0x72, 0xb2, 0x0a, 0xd2, 0xf9, 0x9c, 0x2c, 0x49, 0x64, 0x4e, 0x56, 0xc4,
|
||||||
|
+ 0x1f, 0x40, 0x27, 0x90, 0xa1, 0xab, 0xe9, 0x55, 0x1e, 0x1a, 0x2c, 0x04, 0xb5, 0xdd, 0x0e, 0x72,
|
||||||
|
+ 0xc9, 0x6c, 0x0c, 0x10, 0x11, 0x3f, 0xd6, 0x67, 0x54, 0x3a, 0xea, 0xeb, 0x8a, 0xd0, 0xa4, 0x68,
|
||||||
|
+ 0xbb, 0x15, 0xa5, 0xd9, 0xfa, 0x3d, 0x68, 0x1f, 0x0b, 0x27, 0xe9, 0x03, 0x85, 0x64, 0x94, 0x79,
|
||||||
|
+ 0xcf, 0x86, 0xe3, 0x2c, 0x08, 0xf7, 0xa0, 0x7b, 0xaa, 0x5c, 0xa6, 0x0f, 0xa9, 0x37, 0xf4, 0x9a,
|
||||||
|
+ 0xb6, 0x24, 0xb3, 0x77, 0x9c, 0xf7, 0xac, 0xba, 0x80, 0xce, 0x69, 0x0e, 0x35, 0x3a, 0x84, 0xc1,
|
||||||
|
+ 0x02, 0x49, 0x49, 0x0e, 0xda, 0xc8, 0xe7, 0xa0, 0xf6, 0x23, 0xa4, 0x04, 0xe5, 0x4f, 0xe6, 0xf3,
|
||||||
|
+ 0xd2, 0x6f, 0xab, 0xd0, 0xf9, 0x31, 0xe6, 0x2f, 0x29, 0x3b, 0x53, 0xfa, 0x22, 0xa8, 0x87, 0x6e,
|
||||||
|
+ 0x80, 0x35, 0x47, 0xb9, 0x46, 0x77, 0xa0, 0xc9, 0x2e, 0x54, 0x02, 0xd1, 0xf7, 0xb9, 0xcc, 0x2e,
|
||||||
|
+ 0x64, 0x62, 0x40, 0xaf, 0x00, 0xb0, 0x0b, 0x27, 0x72, 0xbd, 0x33, 0xac, 0x3d, 0x58, 0xb7, 0x5b,
|
||||||
|
+ 0xec, 0xe2, 0x40, 0x21, 0xc4, 0x53, 0x60, 0x17, 0x0e, 0x66, 0x8c, 0xb2, 0x58, 0xe7, 0xaa, 0x26,
|
||||||
|
+ 0xbb, 0xd8, 0x95, 0xb0, 0x3e, 0xeb, 0x33, 0x1a, 0x45, 0xd8, 0x97, 0x39, 0x5a, 0x9e, 0xdd, 0x51,
|
||||||
|
+ 0x08, 0x21, 0x95, 0x1b, 0xa9, 0x0d, 0x25, 0x95, 0x67, 0x52, 0x79, 0x26, 0x75, 0x59, 0x9d, 0xe4,
|
||||||
|
+ 0x79, 0xa9, 0x3c, 0x95, 0xda, 0x54, 0x52, 0x79, 0x4e, 0x2a, 0xcf, 0xa4, 0xb6, 0xcc, 0x59, 0x2d,
|
||||||
|
+ 0xd5, 0xfa, 0x4d, 0x05, 0xd6, 0xe7, 0x0b, 0x3f, 0x5d, 0xa6, 0x7e, 0x00, 0x1d, 0x4f, 0xde, 0x57,
|
||||||
|
+ 0xe1, 0x4d, 0x0e, 0x16, 0x6e, 0xd2, 0x6e, 0x7b, 0xb9, 0x67, 0xfc, 0x21, 0x74, 0x43, 0xe5, 0xe0,
|
||||||
|
+ 0xf4, 0x69, 0xd6, 0xb2, 0x7b, 0xc9, 0xfb, 0xde, 0xee, 0x84, 0x39, 0xc8, 0xf2, 0x01, 0x7d, 0xc9,
|
||||||
|
+ 0x08, 0xc7, 0x87, 0x9c, 0x61, 0x37, 0xb8, 0x89, 0x06, 0x04, 0x41, 0x5d, 0x56, 0x2b, 0x35, 0x59,
|
||||||
|
+ 0x5f, 0xcb, 0xb5, 0xf5, 0x26, 0xac, 0x16, 0xa4, 0x68, 0x5b, 0x57, 0xa0, 0x36, 0xc5, 0xa1, 0xe4,
|
||||||
|
+ 0xde, 0xb5, 0xc5, 0xd2, 0x72, 0x61, 0x60, 0x63, 0xd7, 0xbf, 0x39, 0x6d, 0xb4, 0x88, 0x5a, 0x26,
|
||||||
|
+ 0x62, 0x03, 0x50, 0x5e, 0x84, 0x56, 0xc5, 0x68, 0x5d, 0xc9, 0x69, 0xfd, 0x1c, 0x06, 0xdb, 0x53,
|
||||||
|
+ 0x1a, 0xe3, 0x43, 0xee, 0x93, 0xf0, 0x26, 0x3a, 0xa6, 0x5f, 0xc2, 0xea, 0x0b, 0x3e, 0xfb, 0x52,
|
||||||
|
+ 0x30, 0x8b, 0xc9, 0x37, 0xf8, 0x86, 0xec, 0x63, 0xf4, 0xa5, 0xb1, 0x8f, 0xd1, 0x97, 0xa2, 0x59,
|
||||||
|
+ 0xf2, 0xe8, 0x34, 0x09, 0x42, 0x19, 0x0a, 0x5d, 0x5b, 0x43, 0xd6, 0x16, 0x74, 0x54, 0x0d, 0xfd,
|
||||||
|
+ 0x8c, 0xfa, 0xc9, 0x14, 0x97, 0xc6, 0xe0, 0x7d, 0x80, 0xc8, 0x65, 0x6e, 0x80, 0x39, 0x66, 0xea,
|
||||||
|
+ 0x0d, 0xb5, 0xec, 0x1c, 0xc6, 0xfa, 0x7d, 0x15, 0xd6, 0xd4, 0x48, 0xe4, 0x50, 0x4d, 0x02, 0x8c,
|
||||||
|
+ 0x09, 0x23, 0x68, 0x9e, 0xd2, 0x98, 0xe7, 0x18, 0xa6, 0xb0, 0x50, 0xd1, 0x0f, 0x0d, 0x37, 0xb1,
|
||||||
|
+ 0x2c, 0xcc, 0x29, 0x6a, 0x57, 0xcf, 0x29, 0x16, 0x26, 0x11, 0xf5, 0xc5, 0x49, 0x84, 0x88, 0x36,
|
||||||
|
+ 0x43, 0x44, 0x54, 0x8c, 0xb7, 0xec, 0x96, 0xc6, 0xec, 0xfb, 0xe8, 0x21, 0xf4, 0x27, 0x42, 0x4b,
|
||||||
|
+ 0xe7, 0x94, 0xd2, 0x33, 0x27, 0x72, 0xf9, 0xa9, 0x0c, 0xf5, 0x96, 0xdd, 0x95, 0xe8, 0x3d, 0x4a,
|
||||||
|
+ 0xcf, 0x0e, 0x5c, 0x7e, 0x8a, 0x3e, 0x86, 0x9e, 0x2e, 0x03, 0x03, 0xe9, 0xa2, 0x58, 0x7f, 0xfc,
|
||||||
|
+ 0x74, 0x14, 0xe5, 0xbd, 0x67, 0x77, 0xcf, 0x72, 0x50, 0x6c, 0xdd, 0x86, 0x5b, 0x3b, 0x38, 0xe6,
|
||||||
|
+ 0x8c, 0xce, 0x8a, 0x8e, 0xb1, 0x7e, 0x00, 0xb0, 0x1f, 0x72, 0xcc, 0x4e, 0x5c, 0x0f, 0xc7, 0xe8,
|
||||||
|
+ 0xdd, 0x3c, 0xa4, 0x8b, 0xa3, 0x95, 0xb1, 0x9a, 0x48, 0xa5, 0x1b, 0x76, 0x8e, 0xc6, 0x1a, 0x43,
|
||||||
|
+ 0xc3, 0xa6, 0x89, 0x48, 0x47, 0xaf, 0x9b, 0x95, 0x3e, 0xd7, 0xd1, 0xe7, 0x24, 0xd2, 0xd6, 0x7b,
|
||||||
|
+ 0xd6, 0x9e, 0x69, 0x61, 0x33, 0x76, 0xfa, 0x8a, 0xc6, 0xd0, 0x22, 0x06, 0xa7, 0xb3, 0xca, 0xa2,
|
||||||
|
+ 0xe8, 0x8c, 0xc4, 0xfa, 0x19, 0xac, 0x2a, 0x4e, 0x8a, 0xb3, 0x61, 0xf3, 0x3a, 0x34, 0x98, 0x51,
|
||||||
|
+ 0xa3, 0x92, 0x8d, 0xa2, 0x34, 0x91, 0xde, 0x43, 0xf7, 0x84, 0x30, 0x8f, 0xe1, 0x40, 0xf4, 0x1c,
|
||||||
|
+ 0x55, 0x79, 0x65, 0x19, 0x42, 0x78, 0x4b, 0xf4, 0xdb, 0x99, 0x99, 0xc6, 0x5b, 0xab, 0x30, 0x10,
|
||||||
|
+ 0x1b, 0x05, 0x89, 0xd6, 0x2f, 0x60, 0xf5, 0x79, 0x38, 0x25, 0x21, 0xde, 0x3e, 0x38, 0x7a, 0x86,
|
||||||
|
+ 0xd3, 0xac, 0x80, 0xa0, 0x2e, 0xaa, 0x27, 0xa9, 0x46, 0xd3, 0x96, 0x6b, 0x11, 0x26, 0xe1, 0xb1,
|
||||||
|
+ 0xe3, 0x45, 0x49, 0xac, 0x27, 0x43, 0x8d, 0xf0, 0x78, 0x3b, 0x4a, 0x62, 0x91, 0xe6, 0xc5, 0x67,
|
||||||
|
+ 0x9e, 0x86, 0xd3, 0x99, 0x8c, 0x95, 0xa6, 0xbd, 0xec, 0x45, 0xc9, 0xf3, 0x70, 0x3a, 0xb3, 0xfe,
|
||||||
|
+ 0x4f, 0xf6, 0xc2, 0x18, 0xfb, 0xb6, 0x1b, 0xfa, 0x34, 0xd8, 0xc1, 0xe7, 0x39, 0x09, 0x69, 0xdf,
|
||||||
|
+ 0x65, 0x72, 0xc2, 0xb7, 0x15, 0xe8, 0x3c, 0x99, 0xe0, 0x90, 0xef, 0x60, 0xee, 0x92, 0xa9, 0xec,
|
||||||
|
+ 0xad, 0xce, 0x31, 0x8b, 0x09, 0x0d, 0xf5, 0xc3, 0x37, 0xa0, 0x68, 0x8d, 0x49, 0x48, 0xb8, 0xe3,
|
||||||
|
+ 0xbb, 0x38, 0xa0, 0xa1, 0xf6, 0x02, 0x08, 0xd4, 0x8e, 0xc4, 0xa0, 0x37, 0xa1, 0xaf, 0x26, 0x77,
|
||||||
|
+ 0xce, 0xa9, 0x1b, 0xfa, 0x53, 0x11, 0x72, 0x6a, 0x92, 0xd1, 0x53, 0xe8, 0x3d, 0x8d, 0x45, 0x6f,
|
||||||
|
+ 0xc1, 0x8a, 0x0e, 0x88, 0x8c, 0xb2, 0x2e, 0x29, 0xfb, 0x1a, 0x5f, 0x20, 0x4d, 0xa2, 0x88, 0x32,
|
||||||
|
+ 0x1e, 0x3b, 0x31, 0xf6, 0x3c, 0x1a, 0x44, 0xba, 0x31, 0xe9, 0x1b, 0xfc, 0xa1, 0x42, 0x5b, 0x13,
|
||||||
|
+ 0x58, 0x7d, 0x2a, 0xec, 0xd4, 0x96, 0x64, 0x17, 0xdc, 0x0b, 0x70, 0xe0, 0x1c, 0x4f, 0xa9, 0x77,
|
||||||
|
+ 0xe6, 0x88, 0x34, 0xa5, 0x3d, 0x2c, 0x4a, 0x9f, 0x2d, 0x81, 0x3c, 0x24, 0xdf, 0xc8, 0x1e, 0x5c,
|
||||||
|
+ 0x50, 0x9d, 0x52, 0x1e, 0x4d, 0x93, 0x89, 0x13, 0x31, 0x7a, 0x8c, 0xb5, 0x89, 0xfd, 0x00, 0x07,
|
||||||
|
+ 0x7b, 0x0a, 0x7f, 0x20, 0xd0, 0xd6, 0x5f, 0x2a, 0xb0, 0x56, 0x94, 0xa4, 0x93, 0xee, 0x26, 0xac,
|
||||||
|
+ 0x15, 0x45, 0xe9, 0x0f, 0xb1, 0x2a, 0xf4, 0x06, 0x79, 0x81, 0xea, 0x93, 0xfc, 0x21, 0x74, 0xe5,
|
||||||
|
+ 0x38, 0xd7, 0xf1, 0x15, 0xa7, 0x62, 0xf9, 0x91, 0xbf, 0x17, 0xbb, 0xe3, 0xe6, 0x6f, 0xe9, 0x63,
|
||||||
|
+ 0xb8, 0xa3, 0xcd, 0x77, 0x16, 0xd5, 0x56, 0x0f, 0x62, 0x5d, 0x13, 0x3c, 0x9b, 0xd3, 0xfe, 0x0b,
|
||||||
|
+ 0x18, 0x66, 0xa8, 0xad, 0x99, 0x44, 0x1a, 0x5f, 0xbd, 0x0b, 0xab, 0x73, 0xc6, 0x3e, 0xf1, 0x7d,
|
||||||
|
+ 0x26, 0x03, 0xb4, 0x6e, 0x97, 0x6d, 0x59, 0x8f, 0xe1, 0xf6, 0x21, 0xe6, 0xca, 0x1b, 0x2e, 0xd7,
|
||||||
|
+ 0x3d, 0x81, 0x62, 0xb6, 0x02, 0xb5, 0x43, 0xec, 0x49, 0xe3, 0x6b, 0xb6, 0x58, 0x8a, 0x07, 0x78,
|
||||||
|
+ 0x14, 0x63, 0x4f, 0x5a, 0x59, 0xb3, 0xe5, 0xda, 0xfa, 0x73, 0x05, 0x96, 0x75, 0x9a, 0x14, 0xa9,
|
||||||
|
+ 0xde, 0x67, 0xe4, 0x1c, 0x33, 0xfd, 0xf4, 0x34, 0x84, 0xde, 0x80, 0x9e, 0x5a, 0x39, 0x34, 0xe2,
|
||||||
|
+ 0x84, 0xa6, 0xc9, 0xb7, 0xab, 0xb0, 0xcf, 0x15, 0x52, 0x4e, 0xea, 0xe4, 0x20, 0x4a, 0xf7, 0x7c,
|
||||||
|
+ 0x1a, 0x92, 0xe3, 0xb6, 0x58, 0x64, 0x06, 0x99, 0x6c, 0x5b, 0xb6, 0x86, 0xc4, 0x53, 0x37, 0xfc,
|
||||||
|
+ 0x96, 0x24, 0x3f, 0x03, 0x8a, 0xa7, 0x1e, 0xd0, 0x24, 0xe4, 0x4e, 0x44, 0x49, 0xc8, 0x75, 0x76,
|
||||||
|
+ 0x05, 0x89, 0x3a, 0x10, 0x18, 0xeb, 0xd7, 0x15, 0x68, 0xa8, 0x69, 0xb5, 0xe8, 0x32, 0xd3, 0x6f,
|
||||||
|
+ 0x5c, 0x95, 0xc8, 0x7a, 0x41, 0xca, 0x52, 0xdf, 0x35, 0xb9, 0x16, 0x71, 0x7c, 0x1e, 0xa8, 0x4c,
|
||||||
|
+ 0xad, 0x55, 0x3b, 0x0f, 0x64, 0x8a, 0x7e, 0x03, 0x7a, 0xd9, 0xa7, 0x52, 0xee, 0x2b, 0x15, 0xbb,
|
||||||
|
+ 0x29, 0x56, 0x92, 0x5d, 0xaa, 0xa9, 0xf5, 0x53, 0xd1, 0x5c, 0xa7, 0x93, 0xda, 0x15, 0xa8, 0x25,
|
||||||
|
+ 0xa9, 0x32, 0x62, 0x29, 0x30, 0x93, 0xf4, 0x23, 0x2b, 0x96, 0xe8, 0x21, 0xf4, 0x5c, 0xdf, 0x27,
|
||||||
|
+ 0xe2, 0xb8, 0x3b, 0x7d, 0x4a, 0xfc, 0x34, 0x48, 0x8b, 0x58, 0xeb, 0x6f, 0x15, 0xe8, 0x6f, 0xd3,
|
||||||
|
+ 0x68, 0xf6, 0x43, 0x32, 0xc5, 0xb9, 0x0c, 0x22, 0x95, 0xd4, 0xdf, 0x58, 0xb1, 0x16, 0x75, 0xe3,
|
||||||
|
+ 0x09, 0x99, 0x62, 0x15, 0x5a, 0xea, 0x66, 0x9b, 0x02, 0x21, 0xc3, 0xca, 0x6c, 0xa6, 0x03, 0xb0,
|
||||||
|
+ 0xae, 0xda, 0x7c, 0x46, 0x7d, 0x59, 0x21, 0xfb, 0x84, 0x39, 0xe9, 0xb8, 0xab, 0x6b, 0x2f, 0xfb,
|
||||||
|
+ 0x84, 0xc9, 0x2d, 0x6d, 0xc8, 0x92, 0x9c, 0xb8, 0xe6, 0x0d, 0x69, 0x28, 0x8c, 0x30, 0x64, 0x1d,
|
||||||
|
+ 0x1a, 0xf4, 0xe4, 0x24, 0xc6, 0x5c, 0xd6, 0xb2, 0x35, 0x5b, 0x43, 0x69, 0x9a, 0x6b, 0xe6, 0xd2,
|
||||||
|
+ 0xdc, 0x2d, 0x58, 0x95, 0xb3, 0xfd, 0x17, 0xcc, 0xf5, 0x48, 0x38, 0x31, 0xa9, 0x78, 0x0d, 0xd0,
|
||||||
|
+ 0x21, 0xa7, 0x51, 0x11, 0xfb, 0xe8, 0x77, 0x2b, 0x3a, 0x27, 0xea, 0x46, 0x17, 0x3d, 0x85, 0xfe,
|
||||||
|
+ 0xdc, 0x1f, 0x27, 0x48, 0x4f, 0x3e, 0xca, 0xff, 0x4f, 0x19, 0xad, 0x8f, 0xd5, 0x1f, 0x31, 0x63,
|
||||||
|
+ 0xf3, 0x47, 0xcc, 0x78, 0x37, 0x88, 0xf8, 0x0c, 0xed, 0x42, 0xaf, 0xf8, 0x17, 0x03, 0xba, 0x6b,
|
||||||
|
+ 0x0a, 0x85, 0x92, 0x3f, 0x1e, 0x2e, 0x65, 0xf3, 0x14, 0xfa, 0x73, 0xff, 0x36, 0x18, 0x7d, 0xca,
|
||||||
|
+ 0xff, 0x84, 0xb8, 0x94, 0xd1, 0x63, 0x68, 0xe7, 0xfe, 0x5e, 0x40, 0x43, 0xc5, 0x64, 0xf1, 0x1f,
|
||||||
|
+ 0x87, 0x4b, 0x19, 0x6c, 0x43, 0xb7, 0x30, 0xf1, 0x47, 0x23, 0x6d, 0x4f, 0xc9, 0xdf, 0x00, 0x97,
|
||||||
|
+ 0x32, 0xd9, 0x82, 0x76, 0x6e, 0xf0, 0x6e, 0xb4, 0x58, 0x9c, 0xee, 0x8f, 0xee, 0x94, 0xec, 0xe8,
|
||||||
|
+ 0xd4, 0xbb, 0x07, 0xdd, 0xc2, 0x98, 0xdc, 0x28, 0x52, 0x36, 0xa2, 0x1f, 0xdd, 0x2d, 0xdd, 0xd3,
|
||||||
|
+ 0x9c, 0x9e, 0x42, 0x7f, 0x6e, 0x68, 0x6e, 0x9c, 0x5b, 0x3e, 0x4b, 0xbf, 0xd4, 0xac, 0xcf, 0xe5,
|
||||||
|
+ 0x65, 0xe7, 0x7a, 0xa2, 0xdc, 0x65, 0x2f, 0x8e, 0xc8, 0x47, 0xf7, 0xca, 0x37, 0xb5, 0x56, 0xbb,
|
||||||
|
+ 0xd0, 0x2b, 0x4e, 0xc7, 0x0d, 0xb3, 0xd2, 0x99, 0xf9, 0xd5, 0x2f, 0xa7, 0x30, 0x28, 0xcf, 0x5e,
|
||||||
|
+ 0x4e, 0xd9, 0xfc, 0xfc, 0x52, 0x46, 0x4f, 0x00, 0x74, 0x07, 0xe4, 0x93, 0x30, 0xbd, 0xb2, 0x85,
|
||||||
|
+ 0xce, 0x2b, 0xbd, 0xb2, 0x92, 0x6e, 0xe9, 0x31, 0x80, 0x6a, 0x5c, 0x7c, 0x9a, 0x70, 0x74, 0xdb,
|
||||||
|
+ 0xa8, 0x31, 0xd7, 0x2d, 0x8d, 0x86, 0x8b, 0x1b, 0x0b, 0x0c, 0x30, 0x63, 0xd7, 0x61, 0xf0, 0x19,
|
||||||
|
+ 0x40, 0xd6, 0x10, 0x19, 0x06, 0x0b, 0x2d, 0xd2, 0x15, 0x3e, 0xe8, 0xe4, 0xdb, 0x1f, 0xa4, 0x6d,
|
||||||
|
+ 0x2d, 0x69, 0x89, 0xae, 0x60, 0xd1, 0x9f, 0x2b, 0x6f, 0x8b, 0x8f, 0x6d, 0xbe, 0xea, 0x1d, 0x2d,
|
||||||
|
+ 0x94, 0xb8, 0xe8, 0x43, 0xe8, 0xe4, 0xeb, 0x5a, 0xa3, 0x45, 0x49, 0xad, 0x3b, 0x2a, 0xd4, 0xb6,
|
||||||
|
+ 0xe8, 0x31, 0xf4, 0x8a, 0x55, 0x2b, 0xca, 0xc5, 0xc5, 0x42, 0x2d, 0x3b, 0xd2, 0x13, 0x9b, 0x1c,
|
||||||
|
+ 0xf9, 0xfb, 0x00, 0x59, 0x75, 0x6b, 0xdc, 0xb7, 0x50, 0xef, 0xce, 0x49, 0x7d, 0x02, 0x9d, 0x7c,
|
||||||
|
+ 0x26, 0x36, 0xea, 0x96, 0x64, 0xe7, 0xab, 0xb2, 0x56, 0x2e, 0x6b, 0x9b, 0xc7, 0xb7, 0x98, 0xc8,
|
||||||
|
+ 0xaf, 0xca, 0x5a, 0x85, 0xae, 0xcf, 0x24, 0x8b, 0xb2, 0x56, 0xf0, 0xaa, 0x5c, 0x5e, 0x6c, 0x91,
|
||||||
|
+ 0x8c, 0xfb, 0x4a, 0x1b, 0xa7, 0xab, 0x1e, 0x51, 0xbe, 0x1b, 0x30, 0xfe, 0x28, 0xe9, 0x10, 0xbe,
|
||||||
|
+ 0x27, 0xa8, 0xf3, 0x15, 0x7f, 0x2e, 0xa8, 0x4b, 0x1a, 0x81, 0x4b, 0x19, 0xed, 0x41, 0xff, 0xa9,
|
||||||
|
+ 0x29, 0xe6, 0x74, 0xa1, 0xa9, 0xd5, 0x29, 0x29, 0xac, 0x47, 0xa3, 0xb2, 0x2d, 0x1d, 0x59, 0x9f,
|
||||||
|
+ 0xc3, 0x60, 0xa1, 0xc8, 0x44, 0xf7, 0xd3, 0xc1, 0x62, 0x69, 0xf5, 0x79, 0xa9, 0x5a, 0xfb, 0xb0,
|
||||||
|
+ 0x32, 0x5f, 0x63, 0xa2, 0x57, 0xf4, 0xa5, 0x97, 0xd7, 0x9e, 0x97, 0xb2, 0xfa, 0x18, 0x9a, 0xa6,
|
||||||
|
+ 0xa6, 0x41, 0x7a, 0x80, 0x3b, 0x57, 0xe3, 0x5c, 0x76, 0x74, 0xab, 0xf3, 0xed, 0x77, 0xf7, 0x2b,
|
||||||
|
+ 0x7f, 0xff, 0xee, 0x7e, 0xe5, 0x9f, 0xdf, 0xdd, 0xaf, 0x1c, 0x37, 0xe4, 0xee, 0xfb, 0xff, 0x0e,
|
||||||
|
+ 0x00, 0x00, 0xff, 0xff, 0x8d, 0x89, 0xaa, 0x73, 0xc8, 0x21, 0x00, 0x00,
|
||||||
|
}
|
||||||
|
diff --git a/protocols/grpc/agent.proto b/protocols/grpc/agent.proto
|
||||||
|
index 348d792..b0fab6d 100644
|
||||||
|
--- a/protocols/grpc/agent.proto
|
||||||
|
+++ b/protocols/grpc/agent.proto
|
||||||
|
@@ -316,6 +316,7 @@ message UpdateInterfaceRequest {
|
||||||
|
|
||||||
|
message UpdateRoutesRequest {
|
||||||
|
Routes routes = 1;
|
||||||
|
+ bool increment = 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
message ListInterfacesRequest {
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
971
agent/patches/0003-kata-agent-add-kata-ipvs-command.patch
Normal file
971
agent/patches/0003-kata-agent-add-kata-ipvs-command.patch
Normal file
@ -0,0 +1,971 @@
|
|||||||
|
From 0f24d3d433e93a7309b9bac59e4a15e722391490 Mon Sep 17 00:00:00 2001
|
||||||
|
From: xiadanni1 <xiadanni1@huawei.com>
|
||||||
|
Date: Tue, 18 Aug 2020 06:01:55 +0800
|
||||||
|
Subject: [PATCH 03/16] kata-agent: add kata-ipvs command
|
||||||
|
|
||||||
|
reason: add kata-ipvs command to update IPVS rules
|
||||||
|
in VM.
|
||||||
|
|
||||||
|
Signed-off-by: xiadanni1 <xiadanni1@huawei.com>
|
||||||
|
---
|
||||||
|
agent.go | 6 +
|
||||||
|
grpc.go | 4 +
|
||||||
|
ipvsadm.go | 130 +++++++++
|
||||||
|
protocols/grpc/agent.pb.go | 663 ++++++++++++++++++++++++++++++++-------------
|
||||||
|
protocols/grpc/agent.proto | 11 +
|
||||||
|
5 files changed, 633 insertions(+), 181 deletions(-)
|
||||||
|
create mode 100644 ipvsadm.go
|
||||||
|
|
||||||
|
diff --git a/agent.go b/agent.go
|
||||||
|
index c1cac08..c161e93 100644
|
||||||
|
--- a/agent.go
|
||||||
|
+++ b/agent.go
|
||||||
|
@@ -118,6 +118,11 @@ type sandboxStorage struct {
|
||||||
|
refCount int
|
||||||
|
}
|
||||||
|
|
||||||
|
+type ipvsAdm struct {
|
||||||
|
+ ipvsLock sync.Mutex
|
||||||
|
+ ipvsRuleCnt uint64
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
type sandbox struct {
|
||||||
|
sync.RWMutex
|
||||||
|
ctx context.Context
|
||||||
|
@@ -144,6 +149,7 @@ type sandbox struct {
|
||||||
|
sandboxPidNs bool
|
||||||
|
storages map[string]*sandboxStorage
|
||||||
|
stopServer chan struct{}
|
||||||
|
+ ipvsadm ipvsAdm
|
||||||
|
}
|
||||||
|
|
||||||
|
var agentFields = logrus.Fields{
|
||||||
|
diff --git a/grpc.go b/grpc.go
|
||||||
|
index 8fe8217..de2cae7 100644
|
||||||
|
--- a/grpc.go
|
||||||
|
+++ b/grpc.go
|
||||||
|
@@ -1567,6 +1567,10 @@ func (a *agentGRPC) ListRoutes(ctx context.Context, req *pb.ListRoutesRequest) (
|
||||||
|
return a.sandbox.listRoutes(nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (a *agentGRPC) UpdateIPVSRule(ctx context.Context, req *pb.UpdateIPVSRequest) (*pb.IPVSResponse, error) {
|
||||||
|
+ return a.sandbox.updateIPVSRule(req)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func (a *agentGRPC) OnlineCPUMem(ctx context.Context, req *pb.OnlineCPUMemRequest) (*gpb.Empty, error) {
|
||||||
|
if !req.Wait {
|
||||||
|
go a.onlineCPUMem(req)
|
||||||
|
diff --git a/ipvsadm.go b/ipvsadm.go
|
||||||
|
new file mode 100644
|
||||||
|
index 0000000..48eb19f
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/ipvsadm.go
|
||||||
|
@@ -0,0 +1,130 @@
|
||||||
|
+// Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved.
|
||||||
|
+// SPDX-License-Identifier: Apache-2.0
|
||||||
|
+// Description: IPVS related common functions
|
||||||
|
+// Author: xiadanni
|
||||||
|
+// Create: 2020-08-01
|
||||||
|
+
|
||||||
|
+package main
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "os/exec"
|
||||||
|
+ "strconv"
|
||||||
|
+ "strings"
|
||||||
|
+
|
||||||
|
+ "github.com/kata-containers/agent/protocols/grpc"
|
||||||
|
+ "google.golang.org/grpc/codes"
|
||||||
|
+ grpcStatus "google.golang.org/grpc/status"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+var (
|
||||||
|
+ errNoIPVSRules = grpcStatus.Errorf(codes.InvalidArgument, "IPVS rule is nil")
|
||||||
|
+ errRuleExist = grpcStatus.Errorf(codes.InvalidArgument, "failed to restore IPVS rules: exist some rules in system, should clear first")
|
||||||
|
+ errInvalidIPVSRule = grpcStatus.Errorf(codes.InvalidArgument, "invalid IPVS rule, please check")
|
||||||
|
+ errIPVSRuleExceed = grpcStatus.Errorf(codes.InvalidArgument, "rules exceed limit, should clear first")
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+const (
|
||||||
|
+ validHeadLength = 7
|
||||||
|
+ ruleLimitMax = 20000
|
||||||
|
+ restoreHead = "restore"
|
||||||
|
+ conntrackNormalInfo = "have been deleted"
|
||||||
|
+ waitError = "no child processes"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+func (s *sandbox) updateIPVSRule(ipvsRule *grpc.UpdateIPVSRequest) (resultingIpvs *grpc.IPVSResponse, err error) {
|
||||||
|
+ var (
|
||||||
|
+ restoreRuleCnt string
|
||||||
|
+ restoreRule string
|
||||||
|
+ operation string
|
||||||
|
+ restoreRuleCntUint uint64
|
||||||
|
+ cmd *exec.Cmd
|
||||||
|
+ )
|
||||||
|
+
|
||||||
|
+ if ipvsRule == nil || ipvsRule.IPVSReq == "" {
|
||||||
|
+ return nil, errNoIPVSRules
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ s.ipvsadm.ipvsLock.Lock()
|
||||||
|
+ defer s.ipvsadm.ipvsLock.Unlock()
|
||||||
|
+
|
||||||
|
+ rule := ipvsRule.IPVSReq
|
||||||
|
+ if len(rule) < validHeadLength {
|
||||||
|
+ return nil, errInvalidIPVSRule
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if ruleHead := rule[:validHeadLength]; ruleHead == restoreHead {
|
||||||
|
+ if s.ipvsadm.ipvsRuleCnt != 0 {
|
||||||
|
+ return nil, errRuleExist
|
||||||
|
+ }
|
||||||
|
+ operation = restoreHead
|
||||||
|
+
|
||||||
|
+ restoreItem := strings.Split(rule, "|")
|
||||||
|
+ if len(restoreItem) != 3 {
|
||||||
|
+ return nil, errInvalidIPVSRule
|
||||||
|
+ }
|
||||||
|
+ restoreRuleCnt = restoreItem[1]
|
||||||
|
+ if restoreRuleCntUint, err = strconv.ParseUint(restoreRuleCnt, 10, 64); err != nil {
|
||||||
|
+ return nil, errInvalidIPVSRule
|
||||||
|
+ }
|
||||||
|
+ restoreRule = restoreItem[2]
|
||||||
|
+
|
||||||
|
+ cmd = exec.Command("/sbin/ipvsadm", "--restore")
|
||||||
|
+ cmd.Stdin = strings.NewReader(restoreRule)
|
||||||
|
+ } else {
|
||||||
|
+ item := strings.Fields(strings.TrimSpace(rule))
|
||||||
|
+ if len(item) < 2 {
|
||||||
|
+ return nil, errInvalidIPVSRule
|
||||||
|
+ }
|
||||||
|
+ if item[0] != "ipvsadm" && item[0] != "conntrack" {
|
||||||
|
+ return nil, errInvalidIPVSRule
|
||||||
|
+ }
|
||||||
|
+ operation = item[1]
|
||||||
|
+ if s.ipvsadm.ipvsRuleCnt >= ruleLimitMax {
|
||||||
|
+ if operation == "--add-service" || operation == "-A" || operation == "--add-server" || operation == "-a" {
|
||||||
|
+ return nil, errIPVSRuleExceed
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ restoreRuleCntUint = 0
|
||||||
|
+
|
||||||
|
+ cmd = exec.Command("/sbin/" + item[0])
|
||||||
|
+ for i := 1; i < len(item); i++ {
|
||||||
|
+ cmd.Args = append(cmd.Args, item[i])
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ bytes, err := cmd.CombinedOutput()
|
||||||
|
+
|
||||||
|
+ if err != nil && !strings.Contains(err.Error(), waitError) && !strings.Contains(string(bytes), conntrackNormalInfo) {
|
||||||
|
+ return nil, grpcStatus.Errorf(codes.Internal,
|
||||||
|
+ "exec IPVS command failed, stderr: %v, err: %v", string(bytes), err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ s.sumRuleCount(operation, restoreRuleCntUint)
|
||||||
|
+
|
||||||
|
+ rsp := &grpc.IPVSResponse{
|
||||||
|
+ IPVSRes: string(bytes),
|
||||||
|
+ }
|
||||||
|
+ return rsp, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *sandbox) sumRuleCount(operation string, restoreRuleCntUint uint64) {
|
||||||
|
+ if operation == restoreHead {
|
||||||
|
+ s.ipvsadm.ipvsRuleCnt = restoreRuleCntUint
|
||||||
|
+ return
|
||||||
|
+ }
|
||||||
|
+ if operation == "--add-service" || operation == "-A" || operation == "--add-server" || operation == "-a" {
|
||||||
|
+ s.ipvsadm.ipvsRuleCnt++
|
||||||
|
+ return
|
||||||
|
+ }
|
||||||
|
+ if operation == "--delete-service" || operation == "-D" || operation == "--delete-server" || operation == "-d" {
|
||||||
|
+ if s.ipvsadm.ipvsRuleCnt > 0 {
|
||||||
|
+ s.ipvsadm.ipvsRuleCnt--
|
||||||
|
+ }
|
||||||
|
+ return
|
||||||
|
+ }
|
||||||
|
+ if operation == "--clear" || operation == "-C" {
|
||||||
|
+ s.ipvsadm.ipvsRuleCnt = 0
|
||||||
|
+ return
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
diff --git a/protocols/grpc/agent.pb.go b/protocols/grpc/agent.pb.go
|
||||||
|
index 1b887e5..04d0ee5 100644
|
||||||
|
--- a/protocols/grpc/agent.pb.go
|
||||||
|
+++ b/protocols/grpc/agent.pb.go
|
||||||
|
@@ -63,6 +63,8 @@
|
||||||
|
CopyFileRequest
|
||||||
|
StartTracingRequest
|
||||||
|
StopTracingRequest
|
||||||
|
+ UpdateIPVSRequest
|
||||||
|
+ IPVSResponse
|
||||||
|
CheckRequest
|
||||||
|
HealthCheckResponse
|
||||||
|
VersionCheckResponse
|
||||||
|
@@ -1842,6 +1844,40 @@ func (m *StopTracingRequest) String() string { return proto.CompactTe
|
||||||
|
func (*StopTracingRequest) ProtoMessage() {}
|
||||||
|
func (*StopTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{52} }
|
||||||
|
|
||||||
|
+type UpdateIPVSRequest struct {
|
||||||
|
+ // IPVS_req is the IPVS rule message needed to update
|
||||||
|
+ IPVSReq string `protobuf:"bytes,1,opt,name=IPVS_req,json=IPVSReq,proto3" json:"IPVS_req,omitempty"`
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (m *UpdateIPVSRequest) Reset() { *m = UpdateIPVSRequest{} }
|
||||||
|
+func (m *UpdateIPVSRequest) String() string { return proto.CompactTextString(m) }
|
||||||
|
+func (*UpdateIPVSRequest) ProtoMessage() {}
|
||||||
|
+func (*UpdateIPVSRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{53} }
|
||||||
|
+
|
||||||
|
+func (m *UpdateIPVSRequest) GetIPVSReq() string {
|
||||||
|
+ if m != nil {
|
||||||
|
+ return m.IPVSReq
|
||||||
|
+ }
|
||||||
|
+ return ""
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+type IPVSResponse struct {
|
||||||
|
+ // IPVS_res is the response of IPVS updating
|
||||||
|
+ IPVSRes string `protobuf:"bytes,1,opt,name=IPVS_res,json=IPVSRes,proto3" json:"IPVS_res,omitempty"`
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (m *IPVSResponse) Reset() { *m = IPVSResponse{} }
|
||||||
|
+func (m *IPVSResponse) String() string { return proto.CompactTextString(m) }
|
||||||
|
+func (*IPVSResponse) ProtoMessage() {}
|
||||||
|
+func (*IPVSResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{54} }
|
||||||
|
+
|
||||||
|
+func (m *IPVSResponse) GetIPVSRes() string {
|
||||||
|
+ if m != nil {
|
||||||
|
+ return m.IPVSRes
|
||||||
|
+ }
|
||||||
|
+ return ""
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func init() {
|
||||||
|
proto.RegisterType((*CreateContainerRequest)(nil), "grpc.CreateContainerRequest")
|
||||||
|
proto.RegisterType((*StartContainerRequest)(nil), "grpc.StartContainerRequest")
|
||||||
|
@@ -1896,6 +1932,8 @@ func init() {
|
||||||
|
proto.RegisterType((*CopyFileRequest)(nil), "grpc.CopyFileRequest")
|
||||||
|
proto.RegisterType((*StartTracingRequest)(nil), "grpc.StartTracingRequest")
|
||||||
|
proto.RegisterType((*StopTracingRequest)(nil), "grpc.StopTracingRequest")
|
||||||
|
+ proto.RegisterType((*UpdateIPVSRequest)(nil), "grpc.UpdateIPVSRequest")
|
||||||
|
+ proto.RegisterType((*IPVSResponse)(nil), "grpc.IPVSResponse")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reference imports to suppress errors if they are not otherwise used.
|
||||||
|
@@ -1938,6 +1976,7 @@ type AgentServiceClient interface {
|
||||||
|
UpdateRoutes(ctx context.Context, in *UpdateRoutesRequest, opts ...grpc1.CallOption) (*Routes, error)
|
||||||
|
ListInterfaces(ctx context.Context, in *ListInterfacesRequest, opts ...grpc1.CallOption) (*Interfaces, error)
|
||||||
|
ListRoutes(ctx context.Context, in *ListRoutesRequest, opts ...grpc1.CallOption) (*Routes, error)
|
||||||
|
+ UpdateIPVSRule(ctx context.Context, in *UpdateIPVSRequest, opts ...grpc1.CallOption) (*IPVSResponse, error)
|
||||||
|
// tracing
|
||||||
|
StartTracing(ctx context.Context, in *StartTracingRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error)
|
||||||
|
StopTracing(ctx context.Context, in *StopTracingRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error)
|
||||||
|
@@ -2140,6 +2179,15 @@ func (c *agentServiceClient) ListRoutes(ctx context.Context, in *ListRoutesReque
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (c *agentServiceClient) UpdateIPVSRule(ctx context.Context, in *UpdateIPVSRequest, opts ...grpc1.CallOption) (*IPVSResponse, error) {
|
||||||
|
+ out := new(IPVSResponse)
|
||||||
|
+ err := grpc1.Invoke(ctx, "/grpc.AgentService/UpdateIPVSRule", in, out, c.cc, opts...)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+ return out, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func (c *agentServiceClient) StartTracing(ctx context.Context, in *StartTracingRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error) {
|
||||||
|
out := new(google_protobuf2.Empty)
|
||||||
|
err := grpc1.Invoke(ctx, "/grpc.AgentService/StartTracing", in, out, c.cc, opts...)
|
||||||
|
@@ -2262,6 +2310,7 @@ type AgentServiceServer interface {
|
||||||
|
UpdateRoutes(context.Context, *UpdateRoutesRequest) (*Routes, error)
|
||||||
|
ListInterfaces(context.Context, *ListInterfacesRequest) (*Interfaces, error)
|
||||||
|
ListRoutes(context.Context, *ListRoutesRequest) (*Routes, error)
|
||||||
|
+ UpdateIPVSRule(context.Context, *UpdateIPVSRequest) (*IPVSResponse, error)
|
||||||
|
// tracing
|
||||||
|
StartTracing(context.Context, *StartTracingRequest) (*google_protobuf2.Empty, error)
|
||||||
|
StopTracing(context.Context, *StopTracingRequest) (*google_protobuf2.Empty, error)
|
||||||
|
@@ -2640,6 +2689,24 @@ func _AgentService_ListRoutes_Handler(srv interface{}, ctx context.Context, dec
|
||||||
|
return interceptor(ctx, in, info, handler)
|
||||||
|
}
|
||||||
|
|
||||||
|
+func _AgentService_UpdateIPVSRule_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) {
|
||||||
|
+ in := new(UpdateIPVSRequest)
|
||||||
|
+ if err := dec(in); err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+ if interceptor == nil {
|
||||||
|
+ return srv.(AgentServiceServer).UpdateIPVSRule(ctx, in)
|
||||||
|
+ }
|
||||||
|
+ info := &grpc1.UnaryServerInfo{
|
||||||
|
+ Server: srv,
|
||||||
|
+ FullMethod: "/grpc.AgentService/UpdateIPVSRule",
|
||||||
|
+ }
|
||||||
|
+ handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||||
|
+ return srv.(AgentServiceServer).UpdateIPVSRule(ctx, req.(*UpdateIPVSRequest))
|
||||||
|
+ }
|
||||||
|
+ return interceptor(ctx, in, info, handler)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func _AgentService_StartTracing_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) {
|
||||||
|
in := new(StartTracingRequest)
|
||||||
|
if err := dec(in); err != nil {
|
||||||
|
@@ -2904,6 +2971,10 @@ var _AgentService_serviceDesc = grpc1.ServiceDesc{
|
||||||
|
MethodName: "ListRoutes",
|
||||||
|
Handler: _AgentService_ListRoutes_Handler,
|
||||||
|
},
|
||||||
|
+ {
|
||||||
|
+ MethodName: "UpdateIPVSRule",
|
||||||
|
+ Handler: _AgentService_UpdateIPVSRule_Handler,
|
||||||
|
+ },
|
||||||
|
{
|
||||||
|
MethodName: "StartTracing",
|
||||||
|
Handler: _AgentService_StartTracing_Handler,
|
||||||
|
@@ -5088,6 +5159,54 @@ func (m *StopTracingRequest) MarshalTo(dAtA []byte) (int, error) {
|
||||||
|
return i, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (m *UpdateIPVSRequest) Marshal() (dAtA []byte, err error) {
|
||||||
|
+ size := m.Size()
|
||||||
|
+ dAtA = make([]byte, size)
|
||||||
|
+ n, err := m.MarshalTo(dAtA)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+ return dAtA[:n], nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (m *UpdateIPVSRequest) MarshalTo(dAtA []byte) (int, error) {
|
||||||
|
+ var i int
|
||||||
|
+ _ = i
|
||||||
|
+ var l int
|
||||||
|
+ _ = l
|
||||||
|
+ if len(m.IPVSReq) > 0 {
|
||||||
|
+ dAtA[i] = 0xa
|
||||||
|
+ i++
|
||||||
|
+ i = encodeVarintAgent(dAtA, i, uint64(len(m.IPVSReq)))
|
||||||
|
+ i += copy(dAtA[i:], m.IPVSReq)
|
||||||
|
+ }
|
||||||
|
+ return i, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (m *IPVSResponse) Marshal() (dAtA []byte, err error) {
|
||||||
|
+ size := m.Size()
|
||||||
|
+ dAtA = make([]byte, size)
|
||||||
|
+ n, err := m.MarshalTo(dAtA)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+ return dAtA[:n], nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (m *IPVSResponse) MarshalTo(dAtA []byte) (int, error) {
|
||||||
|
+ var i int
|
||||||
|
+ _ = i
|
||||||
|
+ var l int
|
||||||
|
+ _ = l
|
||||||
|
+ if len(m.IPVSRes) > 0 {
|
||||||
|
+ dAtA[i] = 0xa
|
||||||
|
+ i++
|
||||||
|
+ i = encodeVarintAgent(dAtA, i, uint64(len(m.IPVSRes)))
|
||||||
|
+ i += copy(dAtA[i:], m.IPVSRes)
|
||||||
|
+ }
|
||||||
|
+ return i, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func encodeVarintAgent(dAtA []byte, offset int, v uint64) int {
|
||||||
|
for v >= 1<<7 {
|
||||||
|
dAtA[offset] = uint8(v&0x7f | 0x80)
|
||||||
|
@@ -6019,6 +6138,26 @@ func (m *StopTracingRequest) Size() (n int) {
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (m *UpdateIPVSRequest) Size() (n int) {
|
||||||
|
+ var l int
|
||||||
|
+ _ = l
|
||||||
|
+ l = len(m.IPVSReq)
|
||||||
|
+ if l > 0 {
|
||||||
|
+ n += 1 + l + sovAgent(uint64(l))
|
||||||
|
+ }
|
||||||
|
+ return n
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (m *IPVSResponse) Size() (n int) {
|
||||||
|
+ var l int
|
||||||
|
+ _ = l
|
||||||
|
+ l = len(m.IPVSRes)
|
||||||
|
+ if l > 0 {
|
||||||
|
+ n += 1 + l + sovAgent(uint64(l))
|
||||||
|
+ }
|
||||||
|
+ return n
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func sovAgent(x uint64) (n int) {
|
||||||
|
for {
|
||||||
|
n++
|
||||||
|
@@ -12785,6 +12924,164 @@ func (m *StopTracingRequest) Unmarshal(dAtA []byte) error {
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
+func (m *UpdateIPVSRequest) Unmarshal(dAtA []byte) error {
|
||||||
|
+ l := len(dAtA)
|
||||||
|
+ iNdEx := 0
|
||||||
|
+ for iNdEx < l {
|
||||||
|
+ preIndex := iNdEx
|
||||||
|
+ var wire uint64
|
||||||
|
+ for shift := uint(0); ; shift += 7 {
|
||||||
|
+ if shift >= 64 {
|
||||||
|
+ return ErrIntOverflowAgent
|
||||||
|
+ }
|
||||||
|
+ if iNdEx >= l {
|
||||||
|
+ return io.ErrUnexpectedEOF
|
||||||
|
+ }
|
||||||
|
+ b := dAtA[iNdEx]
|
||||||
|
+ iNdEx++
|
||||||
|
+ wire |= (uint64(b) & 0x7F) << shift
|
||||||
|
+ if b < 0x80 {
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ fieldNum := int32(wire >> 3)
|
||||||
|
+ wireType := int(wire & 0x7)
|
||||||
|
+ if wireType == 4 {
|
||||||
|
+ return fmt.Errorf("proto: UpdateIPVSRequest: wiretype end group for non-group")
|
||||||
|
+ }
|
||||||
|
+ if fieldNum <= 0 {
|
||||||
|
+ return fmt.Errorf("proto: UpdateIPVSRequest: illegal tag %d (wire type %d)", fieldNum, wire)
|
||||||
|
+ }
|
||||||
|
+ switch fieldNum {
|
||||||
|
+ case 1:
|
||||||
|
+ if wireType != 2 {
|
||||||
|
+ return fmt.Errorf("proto: wrong wireType = %d for field IPVSReq", wireType)
|
||||||
|
+ }
|
||||||
|
+ var stringLen uint64
|
||||||
|
+ for shift := uint(0); ; shift += 7 {
|
||||||
|
+ if shift >= 64 {
|
||||||
|
+ return ErrIntOverflowAgent
|
||||||
|
+ }
|
||||||
|
+ if iNdEx >= l {
|
||||||
|
+ return io.ErrUnexpectedEOF
|
||||||
|
+ }
|
||||||
|
+ b := dAtA[iNdEx]
|
||||||
|
+ iNdEx++
|
||||||
|
+ stringLen |= (uint64(b) & 0x7F) << shift
|
||||||
|
+ if b < 0x80 {
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ intStringLen := int(stringLen)
|
||||||
|
+ if intStringLen < 0 {
|
||||||
|
+ return ErrInvalidLengthAgent
|
||||||
|
+ }
|
||||||
|
+ postIndex := iNdEx + intStringLen
|
||||||
|
+ if postIndex > l {
|
||||||
|
+ return io.ErrUnexpectedEOF
|
||||||
|
+ }
|
||||||
|
+ m.IPVSReq = string(dAtA[iNdEx:postIndex])
|
||||||
|
+ iNdEx = postIndex
|
||||||
|
+ default:
|
||||||
|
+ iNdEx = preIndex
|
||||||
|
+ skippy, err := skipAgent(dAtA[iNdEx:])
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ if skippy < 0 {
|
||||||
|
+ return ErrInvalidLengthAgent
|
||||||
|
+ }
|
||||||
|
+ if (iNdEx + skippy) > l {
|
||||||
|
+ return io.ErrUnexpectedEOF
|
||||||
|
+ }
|
||||||
|
+ iNdEx += skippy
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if iNdEx > l {
|
||||||
|
+ return io.ErrUnexpectedEOF
|
||||||
|
+ }
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+func (m *IPVSResponse) Unmarshal(dAtA []byte) error {
|
||||||
|
+ l := len(dAtA)
|
||||||
|
+ iNdEx := 0
|
||||||
|
+ for iNdEx < l {
|
||||||
|
+ preIndex := iNdEx
|
||||||
|
+ var wire uint64
|
||||||
|
+ for shift := uint(0); ; shift += 7 {
|
||||||
|
+ if shift >= 64 {
|
||||||
|
+ return ErrIntOverflowAgent
|
||||||
|
+ }
|
||||||
|
+ if iNdEx >= l {
|
||||||
|
+ return io.ErrUnexpectedEOF
|
||||||
|
+ }
|
||||||
|
+ b := dAtA[iNdEx]
|
||||||
|
+ iNdEx++
|
||||||
|
+ wire |= (uint64(b) & 0x7F) << shift
|
||||||
|
+ if b < 0x80 {
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ fieldNum := int32(wire >> 3)
|
||||||
|
+ wireType := int(wire & 0x7)
|
||||||
|
+ if wireType == 4 {
|
||||||
|
+ return fmt.Errorf("proto: IPVSResponse: wiretype end group for non-group")
|
||||||
|
+ }
|
||||||
|
+ if fieldNum <= 0 {
|
||||||
|
+ return fmt.Errorf("proto: IPVSResponse: illegal tag %d (wire type %d)", fieldNum, wire)
|
||||||
|
+ }
|
||||||
|
+ switch fieldNum {
|
||||||
|
+ case 1:
|
||||||
|
+ if wireType != 2 {
|
||||||
|
+ return fmt.Errorf("proto: wrong wireType = %d for field IPVSRes", wireType)
|
||||||
|
+ }
|
||||||
|
+ var stringLen uint64
|
||||||
|
+ for shift := uint(0); ; shift += 7 {
|
||||||
|
+ if shift >= 64 {
|
||||||
|
+ return ErrIntOverflowAgent
|
||||||
|
+ }
|
||||||
|
+ if iNdEx >= l {
|
||||||
|
+ return io.ErrUnexpectedEOF
|
||||||
|
+ }
|
||||||
|
+ b := dAtA[iNdEx]
|
||||||
|
+ iNdEx++
|
||||||
|
+ stringLen |= (uint64(b) & 0x7F) << shift
|
||||||
|
+ if b < 0x80 {
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ intStringLen := int(stringLen)
|
||||||
|
+ if intStringLen < 0 {
|
||||||
|
+ return ErrInvalidLengthAgent
|
||||||
|
+ }
|
||||||
|
+ postIndex := iNdEx + intStringLen
|
||||||
|
+ if postIndex > l {
|
||||||
|
+ return io.ErrUnexpectedEOF
|
||||||
|
+ }
|
||||||
|
+ m.IPVSRes = string(dAtA[iNdEx:postIndex])
|
||||||
|
+ iNdEx = postIndex
|
||||||
|
+ default:
|
||||||
|
+ iNdEx = preIndex
|
||||||
|
+ skippy, err := skipAgent(dAtA[iNdEx:])
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ if skippy < 0 {
|
||||||
|
+ return ErrInvalidLengthAgent
|
||||||
|
+ }
|
||||||
|
+ if (iNdEx + skippy) > l {
|
||||||
|
+ return io.ErrUnexpectedEOF
|
||||||
|
+ }
|
||||||
|
+ iNdEx += skippy
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if iNdEx > l {
|
||||||
|
+ return io.ErrUnexpectedEOF
|
||||||
|
+ }
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
func skipAgent(dAtA []byte) (n int, err error) {
|
||||||
|
l := len(dAtA)
|
||||||
|
iNdEx := 0
|
||||||
|
@@ -12893,185 +13190,189 @@ var (
|
||||||
|
func init() { proto.RegisterFile("agent.proto", fileDescriptorAgent) }
|
||||||
|
|
||||||
|
var fileDescriptorAgent = []byte{
|
||||||
|
- // 2876 bytes of a gzipped FileDescriptorProto
|
||||||
|
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0x4b, 0x6f, 0x1c, 0xc7,
|
||||||
|
- 0xd1, 0xd8, 0x07, 0x97, 0xbb, 0xb5, 0x2f, 0x6e, 0x93, 0xa2, 0x56, 0x2b, 0x59, 0x9f, 0x3c, 0xb6,
|
||||||
|
- 0x65, 0xfa, 0xf3, 0xe7, 0xa5, 0x2d, 0x1b, 0x9f, 0x5f, 0x70, 0x04, 0xf1, 0x11, 0x91, 0xb1, 0x15,
|
||||||
|
- 0x31, 0x43, 0x11, 0x4e, 0x10, 0x04, 0x83, 0xe1, 0x4c, 0x73, 0xd9, 0xe6, 0xce, 0xf4, 0xb8, 0xa7,
|
||||||
|
- 0x87, 0xe2, 0x3a, 0x40, 0x8e, 0xc9, 0x2d, 0x97, 0x00, 0xb9, 0xe5, 0x0f, 0x04, 0xb9, 0xe5, 0x98,
|
||||||
|
- 0x6b, 0x0e, 0x46, 0x4e, 0xf9, 0x05, 0x41, 0xe0, 0x9f, 0x90, 0x5f, 0x10, 0xf4, 0x6b, 0x1e, 0xbb,
|
||||||
|
- 0x43, 0x1a, 0x21, 0x08, 0xe4, 0xb2, 0xe8, 0xaa, 0xae, 0xae, 0x57, 0x77, 0xd5, 0x54, 0xd5, 0x42,
|
||||||
|
- 0xdb, 0x9d, 0xe0, 0x90, 0x8f, 0x23, 0x46, 0x39, 0x45, 0xf5, 0x09, 0x8b, 0xbc, 0x51, 0x8b, 0x7a,
|
||||||
|
- 0x44, 0x21, 0x46, 0xff, 0x3f, 0x21, 0xfc, 0x34, 0x39, 0x1e, 0x7b, 0x34, 0xd8, 0x3c, 0x73, 0xb9,
|
||||||
|
- 0xfb, 0x8e, 0x47, 0x43, 0xee, 0x92, 0x10, 0xb3, 0x78, 0x53, 0x1e, 0xdc, 0x8c, 0xce, 0x26, 0x9b,
|
||||||
|
- 0x7c, 0x16, 0xe1, 0x58, 0xfd, 0xea, 0x73, 0x77, 0x27, 0x94, 0x4e, 0xa6, 0x78, 0x53, 0x42, 0xc7,
|
||||||
|
- 0xc9, 0xc9, 0x26, 0x0e, 0x22, 0x3e, 0x53, 0x9b, 0xd6, 0x1f, 0xaa, 0xb0, 0xbe, 0xcd, 0xb0, 0xcb,
|
||||||
|
- 0xf1, 0xb6, 0xe1, 0x66, 0xe3, 0xaf, 0x13, 0x1c, 0x73, 0xf4, 0x2a, 0x74, 0x52, 0x09, 0x0e, 0xf1,
|
||||||
|
- 0x87, 0x95, 0x07, 0x95, 0x8d, 0x96, 0xdd, 0x4e, 0x71, 0xfb, 0x3e, 0xba, 0x0d, 0xcb, 0xf8, 0x02,
|
||||||
|
- 0x7b, 0x62, 0xb7, 0x2a, 0x77, 0x1b, 0x02, 0xdc, 0xf7, 0xd1, 0x7b, 0xd0, 0x8e, 0x39, 0x23, 0xe1,
|
||||||
|
- 0xc4, 0x49, 0x62, 0xcc, 0x86, 0xb5, 0x07, 0x95, 0x8d, 0xf6, 0xa3, 0x95, 0xb1, 0x30, 0x69, 0x7c,
|
||||||
|
- 0x28, 0x37, 0x8e, 0x62, 0xcc, 0x6c, 0x88, 0xd3, 0x35, 0x7a, 0x08, 0xcb, 0x3e, 0x3e, 0x27, 0x1e,
|
||||||
|
- 0x8e, 0x87, 0xf5, 0x07, 0xb5, 0x8d, 0xf6, 0xa3, 0x8e, 0x22, 0xdf, 0x91, 0x48, 0xdb, 0x6c, 0xa2,
|
||||||
|
- 0xb7, 0xa0, 0x19, 0x73, 0xca, 0xdc, 0x09, 0x8e, 0x87, 0x4b, 0x92, 0xb0, 0x6b, 0xf8, 0x4a, 0xac,
|
||||||
|
- 0x9d, 0x6e, 0xa3, 0x7b, 0x50, 0x7b, 0xbe, 0xbd, 0x3f, 0x6c, 0x48, 0xe9, 0xa0, 0xa9, 0x22, 0xec,
|
||||||
|
- 0xd9, 0x02, 0x8d, 0x5e, 0x83, 0x6e, 0xec, 0x86, 0xfe, 0x31, 0xbd, 0x70, 0x22, 0xe2, 0x87, 0xf1,
|
||||||
|
- 0x70, 0xf9, 0x41, 0x65, 0xa3, 0x69, 0x77, 0x34, 0xf2, 0x40, 0xe0, 0xac, 0x4f, 0xe0, 0xd6, 0x21,
|
||||||
|
- 0x77, 0x19, 0xbf, 0x86, 0x77, 0xac, 0x23, 0x58, 0xb7, 0x71, 0x40, 0xcf, 0xaf, 0xe5, 0xda, 0x21,
|
||||||
|
- 0x2c, 0x73, 0x12, 0x60, 0x9a, 0x70, 0xe9, 0xda, 0xae, 0x6d, 0x40, 0xeb, 0x4f, 0x15, 0x40, 0xbb,
|
||||||
|
- 0x17, 0xd8, 0x3b, 0x60, 0xd4, 0xc3, 0x71, 0xfc, 0x5f, 0xba, 0xae, 0x37, 0x61, 0x39, 0x52, 0x0a,
|
||||||
|
- 0x0c, 0xeb, 0x92, 0x5c, 0xdf, 0x82, 0xd1, 0xca, 0xec, 0x5a, 0x5f, 0xc1, 0xda, 0x21, 0x99, 0x84,
|
||||||
|
- 0xee, 0xf4, 0x06, 0xf5, 0x5d, 0x87, 0x46, 0x2c, 0x79, 0x4a, 0x55, 0xbb, 0xb6, 0x86, 0xac, 0x03,
|
||||||
|
- 0x40, 0x5f, 0xba, 0x84, 0xdf, 0x9c, 0x24, 0xeb, 0x1d, 0x58, 0x2d, 0x70, 0x8c, 0x23, 0x1a, 0xc6,
|
||||||
|
- 0x58, 0x2a, 0xc0, 0x5d, 0x9e, 0xc4, 0x92, 0xd9, 0x92, 0xad, 0x21, 0x0b, 0xc3, 0xda, 0x17, 0x24,
|
||||||
|
- 0x36, 0xe4, 0xf8, 0x3f, 0x51, 0x61, 0x1d, 0x1a, 0x27, 0x94, 0x05, 0x2e, 0x37, 0x1a, 0x28, 0x08,
|
||||||
|
- 0x21, 0xa8, 0xbb, 0x6c, 0x12, 0x0f, 0x6b, 0x0f, 0x6a, 0x1b, 0x2d, 0x5b, 0xae, 0xc5, 0xab, 0x9c,
|
||||||
|
- 0x13, 0xa3, 0xf5, 0x7a, 0x15, 0x3a, 0xda, 0xef, 0xce, 0x94, 0xc4, 0x5c, 0xca, 0xe9, 0xd8, 0x6d,
|
||||||
|
- 0x8d, 0x13, 0x67, 0x2c, 0x0a, 0xeb, 0x47, 0x91, 0x7f, 0xcd, 0x80, 0x7f, 0x04, 0x2d, 0x86, 0x63,
|
||||||
|
- 0x9a, 0x30, 0x11, 0xa6, 0x55, 0x79, 0xef, 0x6b, 0xea, 0xde, 0xbf, 0x20, 0x61, 0x72, 0x61, 0x9b,
|
||||||
|
- 0x3d, 0x3b, 0x23, 0xd3, 0x21, 0xc4, 0xe3, 0xeb, 0x84, 0xd0, 0x27, 0x70, 0xeb, 0xc0, 0x4d, 0xe2,
|
||||||
|
- 0xeb, 0xe8, 0x6a, 0x7d, 0x2a, 0xc2, 0x2f, 0x4e, 0x82, 0x6b, 0x1d, 0xfe, 0x63, 0x05, 0x9a, 0xdb,
|
||||||
|
- 0x51, 0x72, 0x14, 0xbb, 0x13, 0x8c, 0xfe, 0x07, 0xda, 0x9c, 0x72, 0x77, 0xea, 0x24, 0x02, 0x94,
|
||||||
|
- 0xe4, 0x75, 0x1b, 0x24, 0x4a, 0x11, 0x08, 0xb7, 0x63, 0xe6, 0x45, 0x89, 0xa6, 0xa8, 0x3e, 0xa8,
|
||||||
|
- 0x6d, 0xd4, 0xed, 0xb6, 0xc2, 0x29, 0x92, 0x31, 0xac, 0xca, 0x3d, 0x87, 0x84, 0xce, 0x19, 0x66,
|
||||||
|
- 0x21, 0x9e, 0x06, 0xd4, 0xc7, 0xf2, 0xfd, 0xd6, 0xed, 0x81, 0xdc, 0xda, 0x0f, 0x3f, 0x4f, 0x37,
|
||||||
|
- 0xd0, 0xff, 0xc2, 0x20, 0xa5, 0x17, 0x41, 0x29, 0xa9, 0xeb, 0x92, 0xba, 0xaf, 0xa9, 0x8f, 0x34,
|
||||||
|
- 0xda, 0xfa, 0x15, 0xf4, 0x5e, 0x9c, 0x32, 0xca, 0xf9, 0x94, 0x84, 0x93, 0x1d, 0x97, 0xbb, 0x22,
|
||||||
|
- 0x7b, 0x44, 0x98, 0x11, 0xea, 0xc7, 0x5a, 0x5b, 0x03, 0xa2, 0xb7, 0x61, 0xc0, 0x15, 0x2d, 0xf6,
|
||||||
|
- 0x1d, 0x43, 0x53, 0x95, 0x34, 0x2b, 0xe9, 0xc6, 0x81, 0x26, 0x7e, 0x03, 0x7a, 0x19, 0xb1, 0xc8,
|
||||||
|
- 0x3f, 0x5a, 0xdf, 0x6e, 0x8a, 0x7d, 0x41, 0x02, 0x6c, 0x9d, 0x4b, 0x5f, 0xc9, 0x4b, 0x46, 0x6f,
|
||||||
|
- 0x43, 0x2b, 0xf3, 0x43, 0x45, 0xbe, 0x90, 0x9e, 0x7a, 0x21, 0xc6, 0x9d, 0x76, 0x33, 0x75, 0xca,
|
||||||
|
- 0x67, 0xd0, 0xe7, 0xa9, 0xe2, 0x8e, 0xef, 0x72, 0xb7, 0xf8, 0xa8, 0x8a, 0x56, 0xd9, 0x3d, 0x5e,
|
||||||
|
- 0x80, 0xad, 0x4f, 0xa1, 0x75, 0x40, 0xfc, 0x58, 0x09, 0x1e, 0xc2, 0xb2, 0x97, 0x30, 0x86, 0x43,
|
||||||
|
- 0x6e, 0x4c, 0xd6, 0x20, 0x5a, 0x83, 0xa5, 0x29, 0x09, 0x08, 0xd7, 0x66, 0x2a, 0xc0, 0xa2, 0x00,
|
||||||
|
- 0xcf, 0x70, 0x40, 0xd9, 0x4c, 0x3a, 0x6c, 0x0d, 0x96, 0xf2, 0x97, 0xab, 0x00, 0x74, 0x17, 0x5a,
|
||||||
|
- 0x81, 0x7b, 0x91, 0x5e, 0xaa, 0xd8, 0x69, 0x06, 0xee, 0x85, 0x52, 0x7e, 0x08, 0xcb, 0x27, 0x2e,
|
||||||
|
- 0x99, 0x7a, 0x21, 0xd7, 0x5e, 0x31, 0x60, 0x26, 0xb0, 0x9e, 0x17, 0xf8, 0xd7, 0x2a, 0xb4, 0x95,
|
||||||
|
- 0x44, 0xa5, 0xf0, 0x1a, 0x2c, 0x79, 0xae, 0x77, 0x9a, 0x8a, 0x94, 0x00, 0x7a, 0x68, 0x14, 0xa9,
|
||||||
|
- 0xe6, 0x93, 0x70, 0xa6, 0xa9, 0x51, 0x6d, 0x13, 0x20, 0x7e, 0xe9, 0x46, 0x5a, 0xb7, 0xda, 0x25,
|
||||||
|
- 0xc4, 0x2d, 0x41, 0xa3, 0xd4, 0x7d, 0x1f, 0x3a, 0xea, 0xdd, 0xe9, 0x23, 0xf5, 0x4b, 0x8e, 0xb4,
|
||||||
|
- 0x15, 0x95, 0x3a, 0xf4, 0x1a, 0x74, 0x93, 0x18, 0x3b, 0xa7, 0x04, 0x33, 0x97, 0x79, 0xa7, 0xb3,
|
||||||
|
- 0xe1, 0x92, 0xfa, 0x46, 0x26, 0x31, 0xde, 0x33, 0x38, 0xf4, 0x08, 0x96, 0x44, 0xfa, 0x8b, 0x87,
|
||||||
|
- 0x0d, 0xf9, 0x39, 0xbe, 0x97, 0x67, 0x29, 0x4d, 0x1d, 0xcb, 0xdf, 0xdd, 0x90, 0xb3, 0x99, 0xad,
|
||||||
|
- 0x48, 0x47, 0x1f, 0x01, 0x64, 0x48, 0xb4, 0x02, 0xb5, 0x33, 0x3c, 0xd3, 0x71, 0x28, 0x96, 0xc2,
|
||||||
|
- 0x39, 0xe7, 0xee, 0x34, 0x31, 0x5e, 0x57, 0xc0, 0x27, 0xd5, 0x8f, 0x2a, 0x96, 0x07, 0xfd, 0xad,
|
||||||
|
- 0xe9, 0x19, 0xa1, 0xb9, 0xe3, 0x6b, 0xb0, 0x14, 0xb8, 0x5f, 0x51, 0x66, 0x3c, 0x29, 0x01, 0x89,
|
||||||
|
- 0x25, 0x21, 0x65, 0x86, 0x85, 0x04, 0x50, 0x0f, 0xaa, 0x34, 0x92, 0xfe, 0x6a, 0xd9, 0x55, 0x1a,
|
||||||
|
- 0x65, 0x82, 0xea, 0x39, 0x41, 0xd6, 0x3f, 0xea, 0x00, 0x99, 0x14, 0x64, 0xc3, 0x88, 0x50, 0x27,
|
||||||
|
- 0xc6, 0x4c, 0x94, 0x20, 0xce, 0xf1, 0x8c, 0xe3, 0xd8, 0x61, 0xd8, 0x4b, 0x58, 0x4c, 0xce, 0xc5,
|
||||||
|
- 0xfd, 0x09, 0xb3, 0x6f, 0x29, 0xb3, 0xe7, 0x74, 0xb3, 0x6f, 0x13, 0x7a, 0xa8, 0xce, 0x6d, 0x89,
|
||||||
|
- 0x63, 0xb6, 0x39, 0x85, 0xf6, 0xe1, 0x56, 0xc6, 0xd3, 0xcf, 0xb1, 0xab, 0x5e, 0xc5, 0x6e, 0x35,
|
||||||
|
- 0x65, 0xe7, 0x67, 0xac, 0x76, 0x61, 0x95, 0x50, 0xe7, 0xeb, 0x04, 0x27, 0x05, 0x46, 0xb5, 0xab,
|
||||||
|
- 0x18, 0x0d, 0x08, 0xfd, 0x89, 0x3c, 0x90, 0xb1, 0x39, 0x80, 0x3b, 0x39, 0x2b, 0x45, 0xb8, 0xe7,
|
||||||
|
- 0x98, 0xd5, 0xaf, 0x62, 0xb6, 0x9e, 0x6a, 0x25, 0xf2, 0x41, 0xc6, 0xf1, 0x47, 0xb0, 0x4e, 0xa8,
|
||||||
|
- 0xf3, 0xd2, 0x25, 0x7c, 0x9e, 0xdd, 0xd2, 0xf7, 0x18, 0x29, 0x3e, 0xba, 0x45, 0x5e, 0xca, 0xc8,
|
||||||
|
- 0x00, 0xb3, 0x49, 0xc1, 0xc8, 0xc6, 0xf7, 0x18, 0xf9, 0x4c, 0x1e, 0xc8, 0xd8, 0x3c, 0x81, 0x01,
|
||||||
|
- 0xa1, 0xf3, 0xda, 0x2c, 0x5f, 0xc5, 0xa4, 0x4f, 0x68, 0x51, 0x93, 0x2d, 0x18, 0xc4, 0xd8, 0xe3,
|
||||||
|
- 0x94, 0xe5, 0x1f, 0x41, 0xf3, 0x2a, 0x16, 0x2b, 0x9a, 0x3e, 0xe5, 0x61, 0xfd, 0x1c, 0x3a, 0x7b,
|
||||||
|
- 0xc9, 0x04, 0xf3, 0xe9, 0x71, 0x9a, 0x0c, 0x6e, 0x2c, 0xff, 0x58, 0xff, 0xaa, 0x42, 0x7b, 0x7b,
|
||||||
|
- 0xc2, 0x68, 0x12, 0x15, 0x72, 0xb2, 0x0a, 0xd2, 0xf9, 0x9c, 0x2c, 0x49, 0x64, 0x4e, 0x56, 0xc4,
|
||||||
|
- 0x1f, 0x40, 0x27, 0x90, 0xa1, 0xab, 0xe9, 0x55, 0x1e, 0x1a, 0x2c, 0x04, 0xb5, 0xdd, 0x0e, 0x72,
|
||||||
|
- 0xc9, 0x6c, 0x0c, 0x10, 0x11, 0x3f, 0xd6, 0x67, 0x54, 0x3a, 0xea, 0xeb, 0x8a, 0xd0, 0xa4, 0x68,
|
||||||
|
- 0xbb, 0x15, 0xa5, 0xd9, 0xfa, 0x3d, 0x68, 0x1f, 0x0b, 0x27, 0xe9, 0x03, 0x85, 0x64, 0x94, 0x79,
|
||||||
|
- 0xcf, 0x86, 0xe3, 0x2c, 0x08, 0xf7, 0xa0, 0x7b, 0xaa, 0x5c, 0xa6, 0x0f, 0xa9, 0x37, 0xf4, 0x9a,
|
||||||
|
- 0xb6, 0x24, 0xb3, 0x77, 0x9c, 0xf7, 0xac, 0xba, 0x80, 0xce, 0x69, 0x0e, 0x35, 0x3a, 0x84, 0xc1,
|
||||||
|
- 0x02, 0x49, 0x49, 0x0e, 0xda, 0xc8, 0xe7, 0xa0, 0xf6, 0x23, 0xa4, 0x04, 0xe5, 0x4f, 0xe6, 0xf3,
|
||||||
|
- 0xd2, 0x6f, 0xab, 0xd0, 0xf9, 0x31, 0xe6, 0x2f, 0x29, 0x3b, 0x53, 0xfa, 0x22, 0xa8, 0x87, 0x6e,
|
||||||
|
- 0x80, 0x35, 0x47, 0xb9, 0x46, 0x77, 0xa0, 0xc9, 0x2e, 0x54, 0x02, 0xd1, 0xf7, 0xb9, 0xcc, 0x2e,
|
||||||
|
- 0x64, 0x62, 0x40, 0xaf, 0x00, 0xb0, 0x0b, 0x27, 0x72, 0xbd, 0x33, 0xac, 0x3d, 0x58, 0xb7, 0x5b,
|
||||||
|
- 0xec, 0xe2, 0x40, 0x21, 0xc4, 0x53, 0x60, 0x17, 0x0e, 0x66, 0x8c, 0xb2, 0x58, 0xe7, 0xaa, 0x26,
|
||||||
|
- 0xbb, 0xd8, 0x95, 0xb0, 0x3e, 0xeb, 0x33, 0x1a, 0x45, 0xd8, 0x97, 0x39, 0x5a, 0x9e, 0xdd, 0x51,
|
||||||
|
- 0x08, 0x21, 0x95, 0x1b, 0xa9, 0x0d, 0x25, 0x95, 0x67, 0x52, 0x79, 0x26, 0x75, 0x59, 0x9d, 0xe4,
|
||||||
|
- 0x79, 0xa9, 0x3c, 0x95, 0xda, 0x54, 0x52, 0x79, 0x4e, 0x2a, 0xcf, 0xa4, 0xb6, 0xcc, 0x59, 0x2d,
|
||||||
|
- 0xd5, 0xfa, 0x4d, 0x05, 0xd6, 0xe7, 0x0b, 0x3f, 0x5d, 0xa6, 0x7e, 0x00, 0x1d, 0x4f, 0xde, 0x57,
|
||||||
|
- 0xe1, 0x4d, 0x0e, 0x16, 0x6e, 0xd2, 0x6e, 0x7b, 0xb9, 0x67, 0xfc, 0x21, 0x74, 0x43, 0xe5, 0xe0,
|
||||||
|
- 0xf4, 0x69, 0xd6, 0xb2, 0x7b, 0xc9, 0xfb, 0xde, 0xee, 0x84, 0x39, 0xc8, 0xf2, 0x01, 0x7d, 0xc9,
|
||||||
|
- 0x08, 0xc7, 0x87, 0x9c, 0x61, 0x37, 0xb8, 0x89, 0x06, 0x04, 0x41, 0x5d, 0x56, 0x2b, 0x35, 0x59,
|
||||||
|
- 0x5f, 0xcb, 0xb5, 0xf5, 0x26, 0xac, 0x16, 0xa4, 0x68, 0x5b, 0x57, 0xa0, 0x36, 0xc5, 0xa1, 0xe4,
|
||||||
|
- 0xde, 0xb5, 0xc5, 0xd2, 0x72, 0x61, 0x60, 0x63, 0xd7, 0xbf, 0x39, 0x6d, 0xb4, 0x88, 0x5a, 0x26,
|
||||||
|
- 0x62, 0x03, 0x50, 0x5e, 0x84, 0x56, 0xc5, 0x68, 0x5d, 0xc9, 0x69, 0xfd, 0x1c, 0x06, 0xdb, 0x53,
|
||||||
|
- 0x1a, 0xe3, 0x43, 0xee, 0x93, 0xf0, 0x26, 0x3a, 0xa6, 0x5f, 0xc2, 0xea, 0x0b, 0x3e, 0xfb, 0x52,
|
||||||
|
- 0x30, 0x8b, 0xc9, 0x37, 0xf8, 0x86, 0xec, 0x63, 0xf4, 0xa5, 0xb1, 0x8f, 0xd1, 0x97, 0xa2, 0x59,
|
||||||
|
- 0xf2, 0xe8, 0x34, 0x09, 0x42, 0x19, 0x0a, 0x5d, 0x5b, 0x43, 0xd6, 0x16, 0x74, 0x54, 0x0d, 0xfd,
|
||||||
|
- 0x8c, 0xfa, 0xc9, 0x14, 0x97, 0xc6, 0xe0, 0x7d, 0x80, 0xc8, 0x65, 0x6e, 0x80, 0x39, 0x66, 0xea,
|
||||||
|
- 0x0d, 0xb5, 0xec, 0x1c, 0xc6, 0xfa, 0x7d, 0x15, 0xd6, 0xd4, 0x48, 0xe4, 0x50, 0x4d, 0x02, 0x8c,
|
||||||
|
- 0x09, 0x23, 0x68, 0x9e, 0xd2, 0x98, 0xe7, 0x18, 0xa6, 0xb0, 0x50, 0xd1, 0x0f, 0x0d, 0x37, 0xb1,
|
||||||
|
- 0x2c, 0xcc, 0x29, 0x6a, 0x57, 0xcf, 0x29, 0x16, 0x26, 0x11, 0xf5, 0xc5, 0x49, 0x84, 0x88, 0x36,
|
||||||
|
- 0x43, 0x44, 0x54, 0x8c, 0xb7, 0xec, 0x96, 0xc6, 0xec, 0xfb, 0xe8, 0x21, 0xf4, 0x27, 0x42, 0x4b,
|
||||||
|
- 0xe7, 0x94, 0xd2, 0x33, 0x27, 0x72, 0xf9, 0xa9, 0x0c, 0xf5, 0x96, 0xdd, 0x95, 0xe8, 0x3d, 0x4a,
|
||||||
|
- 0xcf, 0x0e, 0x5c, 0x7e, 0x8a, 0x3e, 0x86, 0x9e, 0x2e, 0x03, 0x03, 0xe9, 0xa2, 0x58, 0x7f, 0xfc,
|
||||||
|
- 0x74, 0x14, 0xe5, 0xbd, 0x67, 0x77, 0xcf, 0x72, 0x50, 0x6c, 0xdd, 0x86, 0x5b, 0x3b, 0x38, 0xe6,
|
||||||
|
- 0x8c, 0xce, 0x8a, 0x8e, 0xb1, 0x7e, 0x00, 0xb0, 0x1f, 0x72, 0xcc, 0x4e, 0x5c, 0x0f, 0xc7, 0xe8,
|
||||||
|
- 0xdd, 0x3c, 0xa4, 0x8b, 0xa3, 0x95, 0xb1, 0x9a, 0x48, 0xa5, 0x1b, 0x76, 0x8e, 0xc6, 0x1a, 0x43,
|
||||||
|
- 0xc3, 0xa6, 0x89, 0x48, 0x47, 0xaf, 0x9b, 0x95, 0x3e, 0xd7, 0xd1, 0xe7, 0x24, 0xd2, 0xd6, 0x7b,
|
||||||
|
- 0xd6, 0x9e, 0x69, 0x61, 0x33, 0x76, 0xfa, 0x8a, 0xc6, 0xd0, 0x22, 0x06, 0xa7, 0xb3, 0xca, 0xa2,
|
||||||
|
- 0xe8, 0x8c, 0xc4, 0xfa, 0x19, 0xac, 0x2a, 0x4e, 0x8a, 0xb3, 0x61, 0xf3, 0x3a, 0x34, 0x98, 0x51,
|
||||||
|
- 0xa3, 0x92, 0x8d, 0xa2, 0x34, 0x91, 0xde, 0x43, 0xf7, 0x84, 0x30, 0x8f, 0xe1, 0x40, 0xf4, 0x1c,
|
||||||
|
- 0x55, 0x79, 0x65, 0x19, 0x42, 0x78, 0x4b, 0xf4, 0xdb, 0x99, 0x99, 0xc6, 0x5b, 0xab, 0x30, 0x10,
|
||||||
|
- 0x1b, 0x05, 0x89, 0xd6, 0x2f, 0x60, 0xf5, 0x79, 0x38, 0x25, 0x21, 0xde, 0x3e, 0x38, 0x7a, 0x86,
|
||||||
|
- 0xd3, 0xac, 0x80, 0xa0, 0x2e, 0xaa, 0x27, 0xa9, 0x46, 0xd3, 0x96, 0x6b, 0x11, 0x26, 0xe1, 0xb1,
|
||||||
|
- 0xe3, 0x45, 0x49, 0xac, 0x27, 0x43, 0x8d, 0xf0, 0x78, 0x3b, 0x4a, 0x62, 0x91, 0xe6, 0xc5, 0x67,
|
||||||
|
- 0x9e, 0x86, 0xd3, 0x99, 0x8c, 0x95, 0xa6, 0xbd, 0xec, 0x45, 0xc9, 0xf3, 0x70, 0x3a, 0xb3, 0xfe,
|
||||||
|
- 0x4f, 0xf6, 0xc2, 0x18, 0xfb, 0xb6, 0x1b, 0xfa, 0x34, 0xd8, 0xc1, 0xe7, 0x39, 0x09, 0x69, 0xdf,
|
||||||
|
- 0x65, 0x72, 0xc2, 0xb7, 0x15, 0xe8, 0x3c, 0x99, 0xe0, 0x90, 0xef, 0x60, 0xee, 0x92, 0xa9, 0xec,
|
||||||
|
- 0xad, 0xce, 0x31, 0x8b, 0x09, 0x0d, 0xf5, 0xc3, 0x37, 0xa0, 0x68, 0x8d, 0x49, 0x48, 0xb8, 0xe3,
|
||||||
|
- 0xbb, 0x38, 0xa0, 0xa1, 0xf6, 0x02, 0x08, 0xd4, 0x8e, 0xc4, 0xa0, 0x37, 0xa1, 0xaf, 0x26, 0x77,
|
||||||
|
- 0xce, 0xa9, 0x1b, 0xfa, 0x53, 0x11, 0x72, 0x6a, 0x92, 0xd1, 0x53, 0xe8, 0x3d, 0x8d, 0x45, 0x6f,
|
||||||
|
- 0xc1, 0x8a, 0x0e, 0x88, 0x8c, 0xb2, 0x2e, 0x29, 0xfb, 0x1a, 0x5f, 0x20, 0x4d, 0xa2, 0x88, 0x32,
|
||||||
|
- 0x1e, 0x3b, 0x31, 0xf6, 0x3c, 0x1a, 0x44, 0xba, 0x31, 0xe9, 0x1b, 0xfc, 0xa1, 0x42, 0x5b, 0x13,
|
||||||
|
- 0x58, 0x7d, 0x2a, 0xec, 0xd4, 0x96, 0x64, 0x17, 0xdc, 0x0b, 0x70, 0xe0, 0x1c, 0x4f, 0xa9, 0x77,
|
||||||
|
- 0xe6, 0x88, 0x34, 0xa5, 0x3d, 0x2c, 0x4a, 0x9f, 0x2d, 0x81, 0x3c, 0x24, 0xdf, 0xc8, 0x1e, 0x5c,
|
||||||
|
- 0x50, 0x9d, 0x52, 0x1e, 0x4d, 0x93, 0x89, 0x13, 0x31, 0x7a, 0x8c, 0xb5, 0x89, 0xfd, 0x00, 0x07,
|
||||||
|
- 0x7b, 0x0a, 0x7f, 0x20, 0xd0, 0xd6, 0x5f, 0x2a, 0xb0, 0x56, 0x94, 0xa4, 0x93, 0xee, 0x26, 0xac,
|
||||||
|
- 0x15, 0x45, 0xe9, 0x0f, 0xb1, 0x2a, 0xf4, 0x06, 0x79, 0x81, 0xea, 0x93, 0xfc, 0x21, 0x74, 0xe5,
|
||||||
|
- 0x38, 0xd7, 0xf1, 0x15, 0xa7, 0x62, 0xf9, 0x91, 0xbf, 0x17, 0xbb, 0xe3, 0xe6, 0x6f, 0xe9, 0x63,
|
||||||
|
- 0xb8, 0xa3, 0xcd, 0x77, 0x16, 0xd5, 0x56, 0x0f, 0x62, 0x5d, 0x13, 0x3c, 0x9b, 0xd3, 0xfe, 0x0b,
|
||||||
|
- 0x18, 0x66, 0xa8, 0xad, 0x99, 0x44, 0x1a, 0x5f, 0xbd, 0x0b, 0xab, 0x73, 0xc6, 0x3e, 0xf1, 0x7d,
|
||||||
|
- 0x26, 0x03, 0xb4, 0x6e, 0x97, 0x6d, 0x59, 0x8f, 0xe1, 0xf6, 0x21, 0xe6, 0xca, 0x1b, 0x2e, 0xd7,
|
||||||
|
- 0x3d, 0x81, 0x62, 0xb6, 0x02, 0xb5, 0x43, 0xec, 0x49, 0xe3, 0x6b, 0xb6, 0x58, 0x8a, 0x07, 0x78,
|
||||||
|
- 0x14, 0x63, 0x4f, 0x5a, 0x59, 0xb3, 0xe5, 0xda, 0xfa, 0x73, 0x05, 0x96, 0x75, 0x9a, 0x14, 0xa9,
|
||||||
|
- 0xde, 0x67, 0xe4, 0x1c, 0x33, 0xfd, 0xf4, 0x34, 0x84, 0xde, 0x80, 0x9e, 0x5a, 0x39, 0x34, 0xe2,
|
||||||
|
- 0x84, 0xa6, 0xc9, 0xb7, 0xab, 0xb0, 0xcf, 0x15, 0x52, 0x4e, 0xea, 0xe4, 0x20, 0x4a, 0xf7, 0x7c,
|
||||||
|
- 0x1a, 0x92, 0xe3, 0xb6, 0x58, 0x64, 0x06, 0x99, 0x6c, 0x5b, 0xb6, 0x86, 0xc4, 0x53, 0x37, 0xfc,
|
||||||
|
- 0x96, 0x24, 0x3f, 0x03, 0x8a, 0xa7, 0x1e, 0xd0, 0x24, 0xe4, 0x4e, 0x44, 0x49, 0xc8, 0x75, 0x76,
|
||||||
|
- 0x05, 0x89, 0x3a, 0x10, 0x18, 0xeb, 0xd7, 0x15, 0x68, 0xa8, 0x69, 0xb5, 0xe8, 0x32, 0xd3, 0x6f,
|
||||||
|
- 0x5c, 0x95, 0xc8, 0x7a, 0x41, 0xca, 0x52, 0xdf, 0x35, 0xb9, 0x16, 0x71, 0x7c, 0x1e, 0xa8, 0x4c,
|
||||||
|
- 0xad, 0x55, 0x3b, 0x0f, 0x64, 0x8a, 0x7e, 0x03, 0x7a, 0xd9, 0xa7, 0x52, 0xee, 0x2b, 0x15, 0xbb,
|
||||||
|
- 0x29, 0x56, 0x92, 0x5d, 0xaa, 0xa9, 0xf5, 0x53, 0xd1, 0x5c, 0xa7, 0x93, 0xda, 0x15, 0xa8, 0x25,
|
||||||
|
- 0xa9, 0x32, 0x62, 0x29, 0x30, 0x93, 0xf4, 0x23, 0x2b, 0x96, 0xe8, 0x21, 0xf4, 0x5c, 0xdf, 0x27,
|
||||||
|
- 0xe2, 0xb8, 0x3b, 0x7d, 0x4a, 0xfc, 0x34, 0x48, 0x8b, 0x58, 0xeb, 0x6f, 0x15, 0xe8, 0x6f, 0xd3,
|
||||||
|
- 0x68, 0xf6, 0x43, 0x32, 0xc5, 0xb9, 0x0c, 0x22, 0x95, 0xd4, 0xdf, 0x58, 0xb1, 0x16, 0x75, 0xe3,
|
||||||
|
- 0x09, 0x99, 0x62, 0x15, 0x5a, 0xea, 0x66, 0x9b, 0x02, 0x21, 0xc3, 0xca, 0x6c, 0xa6, 0x03, 0xb0,
|
||||||
|
- 0xae, 0xda, 0x7c, 0x46, 0x7d, 0x59, 0x21, 0xfb, 0x84, 0x39, 0xe9, 0xb8, 0xab, 0x6b, 0x2f, 0xfb,
|
||||||
|
- 0x84, 0xc9, 0x2d, 0x6d, 0xc8, 0x92, 0x9c, 0xb8, 0xe6, 0x0d, 0x69, 0x28, 0x8c, 0x30, 0x64, 0x1d,
|
||||||
|
- 0x1a, 0xf4, 0xe4, 0x24, 0xc6, 0x5c, 0xd6, 0xb2, 0x35, 0x5b, 0x43, 0x69, 0x9a, 0x6b, 0xe6, 0xd2,
|
||||||
|
- 0xdc, 0x2d, 0x58, 0x95, 0xb3, 0xfd, 0x17, 0xcc, 0xf5, 0x48, 0x38, 0x31, 0xa9, 0x78, 0x0d, 0xd0,
|
||||||
|
- 0x21, 0xa7, 0x51, 0x11, 0xfb, 0xe8, 0x77, 0x2b, 0x3a, 0x27, 0xea, 0x46, 0x17, 0x3d, 0x85, 0xfe,
|
||||||
|
- 0xdc, 0x1f, 0x27, 0x48, 0x4f, 0x3e, 0xca, 0xff, 0x4f, 0x19, 0xad, 0x8f, 0xd5, 0x1f, 0x31, 0x63,
|
||||||
|
- 0xf3, 0x47, 0xcc, 0x78, 0x37, 0x88, 0xf8, 0x0c, 0xed, 0x42, 0xaf, 0xf8, 0x17, 0x03, 0xba, 0x6b,
|
||||||
|
- 0x0a, 0x85, 0x92, 0x3f, 0x1e, 0x2e, 0x65, 0xf3, 0x14, 0xfa, 0x73, 0xff, 0x36, 0x18, 0x7d, 0xca,
|
||||||
|
- 0xff, 0x84, 0xb8, 0x94, 0xd1, 0x63, 0x68, 0xe7, 0xfe, 0x5e, 0x40, 0x43, 0xc5, 0x64, 0xf1, 0x1f,
|
||||||
|
- 0x87, 0x4b, 0x19, 0x6c, 0x43, 0xb7, 0x30, 0xf1, 0x47, 0x23, 0x6d, 0x4f, 0xc9, 0xdf, 0x00, 0x97,
|
||||||
|
- 0x32, 0xd9, 0x82, 0x76, 0x6e, 0xf0, 0x6e, 0xb4, 0x58, 0x9c, 0xee, 0x8f, 0xee, 0x94, 0xec, 0xe8,
|
||||||
|
- 0xd4, 0xbb, 0x07, 0xdd, 0xc2, 0x98, 0xdc, 0x28, 0x52, 0x36, 0xa2, 0x1f, 0xdd, 0x2d, 0xdd, 0xd3,
|
||||||
|
- 0x9c, 0x9e, 0x42, 0x7f, 0x6e, 0x68, 0x6e, 0x9c, 0x5b, 0x3e, 0x4b, 0xbf, 0xd4, 0xac, 0xcf, 0xe5,
|
||||||
|
- 0x65, 0xe7, 0x7a, 0xa2, 0xdc, 0x65, 0x2f, 0x8e, 0xc8, 0x47, 0xf7, 0xca, 0x37, 0xb5, 0x56, 0xbb,
|
||||||
|
- 0xd0, 0x2b, 0x4e, 0xc7, 0x0d, 0xb3, 0xd2, 0x99, 0xf9, 0xd5, 0x2f, 0xa7, 0x30, 0x28, 0xcf, 0x5e,
|
||||||
|
- 0x4e, 0xd9, 0xfc, 0xfc, 0x52, 0x46, 0x4f, 0x00, 0x74, 0x07, 0xe4, 0x93, 0x30, 0xbd, 0xb2, 0x85,
|
||||||
|
- 0xce, 0x2b, 0xbd, 0xb2, 0x92, 0x6e, 0xe9, 0x31, 0x80, 0x6a, 0x5c, 0x7c, 0x9a, 0x70, 0x74, 0xdb,
|
||||||
|
- 0xa8, 0x31, 0xd7, 0x2d, 0x8d, 0x86, 0x8b, 0x1b, 0x0b, 0x0c, 0x30, 0x63, 0xd7, 0x61, 0xf0, 0x19,
|
||||||
|
- 0x40, 0xd6, 0x10, 0x19, 0x06, 0x0b, 0x2d, 0xd2, 0x15, 0x3e, 0xe8, 0xe4, 0xdb, 0x1f, 0xa4, 0x6d,
|
||||||
|
- 0x2d, 0x69, 0x89, 0xae, 0x60, 0xd1, 0x9f, 0x2b, 0x6f, 0x8b, 0x8f, 0x6d, 0xbe, 0xea, 0x1d, 0x2d,
|
||||||
|
- 0x94, 0xb8, 0xe8, 0x43, 0xe8, 0xe4, 0xeb, 0x5a, 0xa3, 0x45, 0x49, 0xad, 0x3b, 0x2a, 0xd4, 0xb6,
|
||||||
|
- 0xe8, 0x31, 0xf4, 0x8a, 0x55, 0x2b, 0xca, 0xc5, 0xc5, 0x42, 0x2d, 0x3b, 0xd2, 0x13, 0x9b, 0x1c,
|
||||||
|
- 0xf9, 0xfb, 0x00, 0x59, 0x75, 0x6b, 0xdc, 0xb7, 0x50, 0xef, 0xce, 0x49, 0x7d, 0x02, 0x9d, 0x7c,
|
||||||
|
- 0x26, 0x36, 0xea, 0x96, 0x64, 0xe7, 0xab, 0xb2, 0x56, 0x2e, 0x6b, 0x9b, 0xc7, 0xb7, 0x98, 0xc8,
|
||||||
|
- 0xaf, 0xca, 0x5a, 0x85, 0xae, 0xcf, 0x24, 0x8b, 0xb2, 0x56, 0xf0, 0xaa, 0x5c, 0x5e, 0x6c, 0x91,
|
||||||
|
- 0x8c, 0xfb, 0x4a, 0x1b, 0xa7, 0xab, 0x1e, 0x51, 0xbe, 0x1b, 0x30, 0xfe, 0x28, 0xe9, 0x10, 0xbe,
|
||||||
|
- 0x27, 0xa8, 0xf3, 0x15, 0x7f, 0x2e, 0xa8, 0x4b, 0x1a, 0x81, 0x4b, 0x19, 0xed, 0x41, 0xff, 0xa9,
|
||||||
|
- 0x29, 0xe6, 0x74, 0xa1, 0xa9, 0xd5, 0x29, 0x29, 0xac, 0x47, 0xa3, 0xb2, 0x2d, 0x1d, 0x59, 0x9f,
|
||||||
|
- 0xc3, 0x60, 0xa1, 0xc8, 0x44, 0xf7, 0xd3, 0xc1, 0x62, 0x69, 0xf5, 0x79, 0xa9, 0x5a, 0xfb, 0xb0,
|
||||||
|
- 0x32, 0x5f, 0x63, 0xa2, 0x57, 0xf4, 0xa5, 0x97, 0xd7, 0x9e, 0x97, 0xb2, 0xfa, 0x18, 0x9a, 0xa6,
|
||||||
|
- 0xa6, 0x41, 0x7a, 0x80, 0x3b, 0x57, 0xe3, 0x5c, 0x76, 0x74, 0xab, 0xf3, 0xed, 0x77, 0xf7, 0x2b,
|
||||||
|
- 0x7f, 0xff, 0xee, 0x7e, 0xe5, 0x9f, 0xdf, 0xdd, 0xaf, 0x1c, 0x37, 0xe4, 0xee, 0xfb, 0xff, 0x0e,
|
||||||
|
- 0x00, 0x00, 0xff, 0xff, 0x8d, 0x89, 0xaa, 0x73, 0xc8, 0x21, 0x00, 0x00,
|
||||||
|
+ // 2931 bytes of a gzipped FileDescriptorProto
|
||||||
|
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0xcb, 0x6e, 0x1c, 0xc7,
|
||||||
|
+ 0xb5, 0x98, 0x07, 0x87, 0x33, 0x67, 0x5e, 0x9c, 0x22, 0x45, 0x8d, 0x46, 0xb2, 0xae, 0xdc, 0xb6,
|
||||||
|
+ 0x65, 0xfa, 0xfa, 0x7a, 0x68, 0xcb, 0xc6, 0xf5, 0x0b, 0xbe, 0x82, 0x48, 0xe9, 0x8a, 0x8c, 0xad,
|
||||||
|
+ 0x88, 0xe9, 0x91, 0xe2, 0x04, 0x41, 0xd0, 0x68, 0x76, 0x97, 0x86, 0x65, 0x4e, 0x77, 0xb5, 0xab,
|
||||||
|
+ 0xaa, 0x29, 0x8e, 0x03, 0x64, 0x99, 0xec, 0xb2, 0xcc, 0x2e, 0x3f, 0x10, 0x64, 0x97, 0x65, 0xb6,
|
||||||
|
+ 0x59, 0x18, 0x59, 0x05, 0xf9, 0x80, 0x20, 0xf0, 0x27, 0xe4, 0x0b, 0x82, 0x7a, 0xf5, 0x63, 0x66,
|
||||||
|
+ 0x48, 0x23, 0x82, 0x80, 0x6c, 0x1a, 0x75, 0x4e, 0x9d, 0x3a, 0xaf, 0xaa, 0x3a, 0x75, 0xce, 0x69,
|
||||||
|
+ 0x68, 0xfb, 0x53, 0x1c, 0x8b, 0x71, 0xc2, 0xa8, 0xa0, 0xa8, 0x3e, 0x65, 0x49, 0x30, 0x6a, 0xd1,
|
||||||
|
+ 0x80, 0x68, 0xc4, 0xe8, 0x7f, 0xa7, 0x44, 0x9c, 0xa4, 0xc7, 0xe3, 0x80, 0x46, 0xbb, 0xa7, 0xbe,
|
||||||
|
+ 0xf0, 0xdf, 0x09, 0x68, 0x2c, 0x7c, 0x12, 0x63, 0xc6, 0x77, 0xd5, 0xc2, 0xdd, 0xe4, 0x74, 0xba,
|
||||||
|
+ 0x2b, 0xe6, 0x09, 0xe6, 0xfa, 0x6b, 0xd6, 0x5d, 0x9f, 0x52, 0x3a, 0x9d, 0xe1, 0x5d, 0x05, 0x1d,
|
||||||
|
+ 0xa7, 0xcf, 0x76, 0x71, 0x94, 0x88, 0xb9, 0x9e, 0x74, 0x7e, 0x57, 0x85, 0xed, 0x7d, 0x86, 0x7d,
|
||||||
|
+ 0x81, 0xf7, 0x2d, 0x37, 0x17, 0x7f, 0x9d, 0x62, 0x2e, 0xd0, 0xab, 0xd0, 0xc9, 0x24, 0x78, 0x24,
|
||||||
|
+ 0x1c, 0x56, 0x6e, 0x55, 0x76, 0x5a, 0x6e, 0x3b, 0xc3, 0x1d, 0x86, 0xe8, 0x2a, 0xac, 0xe3, 0x73,
|
||||||
|
+ 0x1c, 0xc8, 0xd9, 0xaa, 0x9a, 0x6d, 0x48, 0xf0, 0x30, 0x44, 0xef, 0x41, 0x9b, 0x0b, 0x46, 0xe2,
|
||||||
|
+ 0xa9, 0x97, 0x72, 0xcc, 0x86, 0xb5, 0x5b, 0x95, 0x9d, 0xf6, 0x9d, 0x8d, 0xb1, 0x34, 0x69, 0x3c,
|
||||||
|
+ 0x51, 0x13, 0x4f, 0x39, 0x66, 0x2e, 0xf0, 0x6c, 0x8c, 0x6e, 0xc3, 0x7a, 0x88, 0xcf, 0x48, 0x80,
|
||||||
|
+ 0xf9, 0xb0, 0x7e, 0xab, 0xb6, 0xd3, 0xbe, 0xd3, 0xd1, 0xe4, 0xf7, 0x15, 0xd2, 0xb5, 0x93, 0xe8,
|
||||||
|
+ 0x2d, 0x68, 0x72, 0x41, 0x99, 0x3f, 0xc5, 0x7c, 0xb8, 0xa6, 0x08, 0xbb, 0x96, 0xaf, 0xc2, 0xba,
|
||||||
|
+ 0xd9, 0x34, 0xba, 0x01, 0xb5, 0xc7, 0xfb, 0x87, 0xc3, 0x86, 0x92, 0x0e, 0x86, 0x2a, 0xc1, 0x81,
|
||||||
|
+ 0x2b, 0xd1, 0xe8, 0x35, 0xe8, 0x72, 0x3f, 0x0e, 0x8f, 0xe9, 0xb9, 0x97, 0x90, 0x30, 0xe6, 0xc3,
|
||||||
|
+ 0xf5, 0x5b, 0x95, 0x9d, 0xa6, 0xdb, 0x31, 0xc8, 0x23, 0x89, 0x73, 0x3e, 0x81, 0x2b, 0x13, 0xe1,
|
||||||
|
+ 0x33, 0xf1, 0x02, 0xde, 0x71, 0x9e, 0xc2, 0xb6, 0x8b, 0x23, 0x7a, 0xf6, 0x42, 0xae, 0x1d, 0xc2,
|
||||||
|
+ 0xba, 0x20, 0x11, 0xa6, 0xa9, 0x50, 0xae, 0xed, 0xba, 0x16, 0x74, 0xfe, 0x50, 0x01, 0xf4, 0xe0,
|
||||||
|
+ 0x1c, 0x07, 0x47, 0x8c, 0x06, 0x98, 0xf3, 0xff, 0xd0, 0x76, 0xbd, 0x09, 0xeb, 0x89, 0x56, 0x60,
|
||||||
|
+ 0x58, 0x57, 0xe4, 0x66, 0x17, 0xac, 0x56, 0x76, 0xd6, 0xf9, 0x0a, 0xb6, 0x26, 0x64, 0x1a, 0xfb,
|
||||||
|
+ 0xb3, 0x97, 0xa8, 0xef, 0x36, 0x34, 0xb8, 0xe2, 0xa9, 0x54, 0xed, 0xba, 0x06, 0x72, 0x8e, 0x00,
|
||||||
|
+ 0x7d, 0xe9, 0x13, 0xf1, 0xf2, 0x24, 0x39, 0xef, 0xc0, 0x66, 0x89, 0x23, 0x4f, 0x68, 0xcc, 0xb1,
|
||||||
|
+ 0x52, 0x40, 0xf8, 0x22, 0xe5, 0x8a, 0xd9, 0x9a, 0x6b, 0x20, 0x07, 0xc3, 0xd6, 0x17, 0x84, 0x5b,
|
||||||
|
+ 0x72, 0xfc, 0xef, 0xa8, 0xb0, 0x0d, 0x8d, 0x67, 0x94, 0x45, 0xbe, 0xb0, 0x1a, 0x68, 0x08, 0x21,
|
||||||
|
+ 0xa8, 0xfb, 0x6c, 0xca, 0x87, 0xb5, 0x5b, 0xb5, 0x9d, 0x96, 0xab, 0xc6, 0xf2, 0x54, 0x2e, 0x88,
|
||||||
|
+ 0x31, 0x7a, 0xbd, 0x0a, 0x1d, 0xe3, 0x77, 0x6f, 0x46, 0xb8, 0x50, 0x72, 0x3a, 0x6e, 0xdb, 0xe0,
|
||||||
|
+ 0xe4, 0x1a, 0x87, 0xc2, 0xf6, 0xd3, 0x24, 0x7c, 0xc1, 0x0b, 0x7f, 0x07, 0x5a, 0x0c, 0x73, 0x9a,
|
||||||
|
+ 0x32, 0x79, 0x4d, 0xab, 0x6a, 0xdf, 0xb7, 0xf4, 0xbe, 0x7f, 0x41, 0xe2, 0xf4, 0xdc, 0xb5, 0x73,
|
||||||
|
+ 0x6e, 0x4e, 0x66, 0xae, 0x90, 0xe0, 0x2f, 0x72, 0x85, 0x3e, 0x81, 0x2b, 0x47, 0x7e, 0xca, 0x5f,
|
||||||
|
+ 0x44, 0x57, 0xe7, 0x53, 0x79, 0xfd, 0x78, 0x1a, 0xbd, 0xd0, 0xe2, 0xdf, 0x57, 0xa0, 0xb9, 0x9f,
|
||||||
|
+ 0xa4, 0x4f, 0xb9, 0x3f, 0xc5, 0xe8, 0xbf, 0xa0, 0x2d, 0xa8, 0xf0, 0x67, 0x5e, 0x2a, 0x41, 0x45,
|
||||||
|
+ 0x5e, 0x77, 0x41, 0xa1, 0x34, 0x81, 0x74, 0x3b, 0x66, 0x41, 0x92, 0x1a, 0x8a, 0xea, 0xad, 0xda,
|
||||||
|
+ 0x4e, 0xdd, 0x6d, 0x6b, 0x9c, 0x26, 0x19, 0xc3, 0xa6, 0x9a, 0xf3, 0x48, 0xec, 0x9d, 0x62, 0x16,
|
||||||
|
+ 0xe3, 0x59, 0x44, 0x43, 0xac, 0xce, 0x6f, 0xdd, 0x1d, 0xa8, 0xa9, 0xc3, 0xf8, 0xf3, 0x6c, 0x02,
|
||||||
|
+ 0xfd, 0x37, 0x0c, 0x32, 0x7a, 0x79, 0x29, 0x15, 0x75, 0x5d, 0x51, 0xf7, 0x0d, 0xf5, 0x53, 0x83,
|
||||||
|
+ 0x76, 0x7e, 0x09, 0xbd, 0x27, 0x27, 0x8c, 0x0a, 0x31, 0x23, 0xf1, 0xf4, 0xbe, 0x2f, 0x7c, 0x19,
|
||||||
|
+ 0x3d, 0x12, 0xcc, 0x08, 0x0d, 0xb9, 0xd1, 0xd6, 0x82, 0xe8, 0x6d, 0x18, 0x08, 0x4d, 0x8b, 0x43,
|
||||||
|
+ 0xcf, 0xd2, 0x54, 0x15, 0xcd, 0x46, 0x36, 0x71, 0x64, 0x88, 0xdf, 0x80, 0x5e, 0x4e, 0x2c, 0xe3,
|
||||||
|
+ 0x8f, 0xd1, 0xb7, 0x9b, 0x61, 0x9f, 0x90, 0x08, 0x3b, 0x67, 0xca, 0x57, 0x6a, 0x93, 0xd1, 0xdb,
|
||||||
|
+ 0xd0, 0xca, 0xfd, 0x50, 0x51, 0x27, 0xa4, 0xa7, 0x4f, 0x88, 0x75, 0xa7, 0xdb, 0xcc, 0x9c, 0xf2,
|
||||||
|
+ 0x19, 0xf4, 0x45, 0xa6, 0xb8, 0x17, 0xfa, 0xc2, 0x2f, 0x1f, 0xaa, 0xb2, 0x55, 0x6e, 0x4f, 0x94,
|
||||||
|
+ 0x60, 0xe7, 0x53, 0x68, 0x1d, 0x91, 0x90, 0x6b, 0xc1, 0x43, 0x58, 0x0f, 0x52, 0xc6, 0x70, 0x2c,
|
||||||
|
+ 0xac, 0xc9, 0x06, 0x44, 0x5b, 0xb0, 0x36, 0x23, 0x11, 0x11, 0xc6, 0x4c, 0x0d, 0x38, 0x14, 0xe0,
|
||||||
|
+ 0x11, 0x8e, 0x28, 0x9b, 0x2b, 0x87, 0x6d, 0xc1, 0x5a, 0x71, 0x73, 0x35, 0x80, 0xae, 0x43, 0x2b,
|
||||||
|
+ 0xf2, 0xcf, 0xb3, 0x4d, 0x95, 0x33, 0xcd, 0xc8, 0x3f, 0xd7, 0xca, 0x0f, 0x61, 0xfd, 0x99, 0x4f,
|
||||||
|
+ 0x66, 0x41, 0x2c, 0x8c, 0x57, 0x2c, 0x98, 0x0b, 0xac, 0x17, 0x05, 0xfe, 0xb9, 0x0a, 0x6d, 0x2d,
|
||||||
|
+ 0x51, 0x2b, 0xbc, 0x05, 0x6b, 0x81, 0x1f, 0x9c, 0x64, 0x22, 0x15, 0x80, 0x6e, 0x5b, 0x45, 0xaa,
|
||||||
|
+ 0xc5, 0x20, 0x9c, 0x6b, 0x6a, 0x55, 0xdb, 0x05, 0xe0, 0xcf, 0xfd, 0xc4, 0xe8, 0x56, 0xbb, 0x80,
|
||||||
|
+ 0xb8, 0x25, 0x69, 0xb4, 0xba, 0xef, 0x43, 0x47, 0x9f, 0x3b, 0xb3, 0xa4, 0x7e, 0xc1, 0x92, 0xb6,
|
||||||
|
+ 0xa6, 0xd2, 0x8b, 0x5e, 0x83, 0x6e, 0xca, 0xb1, 0x77, 0x42, 0x30, 0xf3, 0x59, 0x70, 0x32, 0x1f,
|
||||||
|
+ 0xae, 0xe9, 0x37, 0x32, 0xe5, 0xf8, 0xc0, 0xe2, 0xd0, 0x1d, 0x58, 0x93, 0xe1, 0x8f, 0x0f, 0x1b,
|
||||||
|
+ 0xea, 0x39, 0xbe, 0x51, 0x64, 0xa9, 0x4c, 0x1d, 0xab, 0xef, 0x83, 0x58, 0xb0, 0xb9, 0xab, 0x49,
|
||||||
|
+ 0x47, 0x1f, 0x01, 0xe4, 0x48, 0xb4, 0x01, 0xb5, 0x53, 0x3c, 0x37, 0xf7, 0x50, 0x0e, 0xa5, 0x73,
|
||||||
|
+ 0xce, 0xfc, 0x59, 0x6a, 0xbd, 0xae, 0x81, 0x4f, 0xaa, 0x1f, 0x55, 0x9c, 0x00, 0xfa, 0x7b, 0xb3,
|
||||||
|
+ 0x53, 0x42, 0x0b, 0xcb, 0xb7, 0x60, 0x2d, 0xf2, 0xbf, 0xa2, 0xcc, 0x7a, 0x52, 0x01, 0x0a, 0x4b,
|
||||||
|
+ 0x62, 0xca, 0x2c, 0x0b, 0x05, 0xa0, 0x1e, 0x54, 0x69, 0xa2, 0xfc, 0xd5, 0x72, 0xab, 0x34, 0xc9,
|
||||||
|
+ 0x05, 0xd5, 0x0b, 0x82, 0x9c, 0xbf, 0xd7, 0x01, 0x72, 0x29, 0xc8, 0x85, 0x11, 0xa1, 0x1e, 0xc7,
|
||||||
|
+ 0x4c, 0xa6, 0x20, 0xde, 0xf1, 0x5c, 0x60, 0xee, 0x31, 0x1c, 0xa4, 0x8c, 0x93, 0x33, 0xb9, 0x7f,
|
||||||
|
+ 0xd2, 0xec, 0x2b, 0xda, 0xec, 0x05, 0xdd, 0xdc, 0xab, 0x84, 0x4e, 0xf4, 0xba, 0x3d, 0xb9, 0xcc,
|
||||||
|
+ 0xb5, 0xab, 0xd0, 0x21, 0x5c, 0xc9, 0x79, 0x86, 0x05, 0x76, 0xd5, 0xcb, 0xd8, 0x6d, 0x66, 0xec,
|
||||||
|
+ 0xc2, 0x9c, 0xd5, 0x03, 0xd8, 0x24, 0xd4, 0xfb, 0x3a, 0xc5, 0x69, 0x89, 0x51, 0xed, 0x32, 0x46,
|
||||||
|
+ 0x03, 0x42, 0x7f, 0xa4, 0x16, 0xe4, 0x6c, 0x8e, 0xe0, 0x5a, 0xc1, 0x4a, 0x79, 0xdd, 0x0b, 0xcc,
|
||||||
|
+ 0xea, 0x97, 0x31, 0xdb, 0xce, 0xb4, 0x92, 0xf1, 0x20, 0xe7, 0xf8, 0x03, 0xd8, 0x26, 0xd4, 0x7b,
|
||||||
|
+ 0xee, 0x13, 0xb1, 0xc8, 0x6e, 0xed, 0x7b, 0x8c, 0x94, 0x8f, 0x6e, 0x99, 0x97, 0x36, 0x32, 0xc2,
|
||||||
|
+ 0x6c, 0x5a, 0x32, 0xb2, 0xf1, 0x3d, 0x46, 0x3e, 0x52, 0x0b, 0x72, 0x36, 0xf7, 0x60, 0x40, 0xe8,
|
||||||
|
+ 0xa2, 0x36, 0xeb, 0x97, 0x31, 0xe9, 0x13, 0x5a, 0xd6, 0x64, 0x0f, 0x06, 0x1c, 0x07, 0x82, 0xb2,
|
||||||
|
+ 0xe2, 0x21, 0x68, 0x5e, 0xc6, 0x62, 0xc3, 0xd0, 0x67, 0x3c, 0x9c, 0x9f, 0x41, 0xe7, 0x20, 0x9d,
|
||||||
|
+ 0x62, 0x31, 0x3b, 0xce, 0x82, 0xc1, 0x4b, 0x8b, 0x3f, 0xce, 0x3f, 0xab, 0xd0, 0xde, 0x9f, 0x32,
|
||||||
|
+ 0x9a, 0x26, 0xa5, 0x98, 0xac, 0x2f, 0xe9, 0x62, 0x4c, 0x56, 0x24, 0x2a, 0x26, 0x6b, 0xe2, 0x0f,
|
||||||
|
+ 0xa0, 0x13, 0xa9, 0xab, 0x6b, 0xe8, 0x75, 0x1c, 0x1a, 0x2c, 0x5d, 0x6a, 0xb7, 0x1d, 0x15, 0x82,
|
||||||
|
+ 0xd9, 0x18, 0x20, 0x21, 0x21, 0x37, 0x6b, 0x74, 0x38, 0xea, 0x9b, 0x8c, 0xd0, 0x86, 0x68, 0xb7,
|
||||||
|
+ 0x95, 0x64, 0xd1, 0xfa, 0x3d, 0x68, 0x1f, 0x4b, 0x27, 0x99, 0x05, 0xa5, 0x60, 0x94, 0x7b, 0xcf,
|
||||||
|
+ 0x85, 0xe3, 0xfc, 0x12, 0x1e, 0x40, 0xf7, 0x44, 0xbb, 0xcc, 0x2c, 0xd2, 0x67, 0xe8, 0x35, 0x63,
|
||||||
|
+ 0x49, 0x6e, 0xef, 0xb8, 0xe8, 0x59, 0xbd, 0x01, 0x9d, 0x93, 0x02, 0x6a, 0x34, 0x81, 0xc1, 0x12,
|
||||||
|
+ 0xc9, 0x8a, 0x18, 0xb4, 0x53, 0x8c, 0x41, 0xed, 0x3b, 0x48, 0x0b, 0x2a, 0xae, 0x2c, 0xc6, 0xa5,
|
||||||
|
+ 0xdf, 0x54, 0xa1, 0xf3, 0x43, 0x2c, 0x9e, 0x53, 0x76, 0xaa, 0xf5, 0x45, 0x50, 0x8f, 0xfd, 0x08,
|
||||||
|
+ 0x1b, 0x8e, 0x6a, 0x8c, 0xae, 0x41, 0x93, 0x9d, 0xeb, 0x00, 0x62, 0xf6, 0x73, 0x9d, 0x9d, 0xab,
|
||||||
|
+ 0xc0, 0x80, 0x5e, 0x01, 0x60, 0xe7, 0x5e, 0xe2, 0x07, 0xa7, 0xd8, 0x78, 0xb0, 0xee, 0xb6, 0xd8,
|
||||||
|
+ 0xf9, 0x91, 0x46, 0xc8, 0xa3, 0xc0, 0xce, 0x3d, 0xcc, 0x18, 0x65, 0xdc, 0xc4, 0xaa, 0x26, 0x3b,
|
||||||
|
+ 0x7f, 0xa0, 0x60, 0xb3, 0x36, 0x64, 0x34, 0x49, 0x70, 0xa8, 0x62, 0xb4, 0x5a, 0x7b, 0x5f, 0x23,
|
||||||
|
+ 0xa4, 0x54, 0x61, 0xa5, 0x36, 0xb4, 0x54, 0x91, 0x4b, 0x15, 0xb9, 0xd4, 0x75, 0xbd, 0x52, 0x14,
|
||||||
|
+ 0xa5, 0x8a, 0x4c, 0x6a, 0x53, 0x4b, 0x15, 0x05, 0xa9, 0x22, 0x97, 0xda, 0xb2, 0x6b, 0x8d, 0x54,
|
||||||
|
+ 0xe7, 0xd7, 0x15, 0xd8, 0x5e, 0x4c, 0xfc, 0x4c, 0x9a, 0xfa, 0x01, 0x74, 0x02, 0xb5, 0x5f, 0xa5,
|
||||||
|
+ 0x33, 0x39, 0x58, 0xda, 0x49, 0xb7, 0x1d, 0x14, 0x8e, 0xf1, 0x87, 0xd0, 0x8d, 0xb5, 0x83, 0xb3,
|
||||||
|
+ 0xa3, 0x59, 0xcb, 0xf7, 0xa5, 0xe8, 0x7b, 0xb7, 0x13, 0x17, 0x20, 0x27, 0x04, 0xf4, 0x25, 0x23,
|
||||||
|
+ 0x02, 0x4f, 0x04, 0xc3, 0x7e, 0xf4, 0x32, 0x0a, 0x10, 0x04, 0x75, 0x95, 0xad, 0xd4, 0x54, 0x7e,
|
||||||
|
+ 0xad, 0xc6, 0xce, 0x9b, 0xb0, 0x59, 0x92, 0x62, 0x6c, 0xdd, 0x80, 0xda, 0x0c, 0xc7, 0x8a, 0x7b,
|
||||||
|
+ 0xd7, 0x95, 0x43, 0xc7, 0x87, 0x81, 0x8b, 0xfd, 0xf0, 0xe5, 0x69, 0x63, 0x44, 0xd4, 0x72, 0x11,
|
||||||
|
+ 0x3b, 0x80, 0x8a, 0x22, 0x8c, 0x2a, 0x56, 0xeb, 0x4a, 0x41, 0xeb, 0xc7, 0x30, 0xd8, 0x9f, 0x51,
|
||||||
|
+ 0x8e, 0x27, 0x22, 0x24, 0xf1, 0xcb, 0xa8, 0x98, 0x7e, 0x01, 0x9b, 0x4f, 0xc4, 0xfc, 0x4b, 0xc9,
|
||||||
|
+ 0x8c, 0x93, 0x6f, 0xf0, 0x4b, 0xb2, 0x8f, 0xd1, 0xe7, 0xd6, 0x3e, 0x46, 0x9f, 0xcb, 0x62, 0x29,
|
||||||
|
+ 0xa0, 0xb3, 0x34, 0x8a, 0xd5, 0x55, 0xe8, 0xba, 0x06, 0x72, 0xf6, 0xa0, 0xa3, 0x73, 0xe8, 0x47,
|
||||||
|
+ 0x34, 0x4c, 0x67, 0x78, 0xe5, 0x1d, 0xbc, 0x09, 0x90, 0xf8, 0xcc, 0x8f, 0xb0, 0xc0, 0x4c, 0x9f,
|
||||||
|
+ 0xa1, 0x96, 0x5b, 0xc0, 0x38, 0xbf, 0xad, 0xc2, 0x96, 0x6e, 0x89, 0x4c, 0x74, 0x27, 0xc0, 0x9a,
|
||||||
|
+ 0x30, 0x82, 0xe6, 0x09, 0xe5, 0xa2, 0xc0, 0x30, 0x83, 0xa5, 0x8a, 0x61, 0x6c, 0xb9, 0xc9, 0x61,
|
||||||
|
+ 0xa9, 0x4f, 0x51, 0xbb, 0xbc, 0x4f, 0xb1, 0xd4, 0x89, 0xa8, 0x2f, 0x77, 0x22, 0xe4, 0x6d, 0xb3,
|
||||||
|
+ 0x44, 0x44, 0xdf, 0xf1, 0x96, 0xdb, 0x32, 0x98, 0xc3, 0x10, 0xdd, 0x86, 0xfe, 0x54, 0x6a, 0xe9,
|
||||||
|
+ 0x9d, 0x50, 0x7a, 0xea, 0x25, 0xbe, 0x38, 0x51, 0x57, 0xbd, 0xe5, 0x76, 0x15, 0xfa, 0x80, 0xd2,
|
||||||
|
+ 0xd3, 0x23, 0x5f, 0x9c, 0xa0, 0x8f, 0xa1, 0x67, 0xd2, 0xc0, 0x48, 0xb9, 0x88, 0x9b, 0xc7, 0xcf,
|
||||||
|
+ 0xdc, 0xa2, 0xa2, 0xf7, 0xdc, 0xee, 0x69, 0x01, 0xe2, 0xce, 0x55, 0xb8, 0x72, 0x1f, 0x73, 0xc1,
|
||||||
|
+ 0xe8, 0xbc, 0xec, 0x18, 0xe7, 0xff, 0x00, 0x0e, 0x63, 0x81, 0xd9, 0x33, 0x3f, 0xc0, 0x1c, 0xbd,
|
||||||
|
+ 0x5b, 0x84, 0x4c, 0x72, 0xb4, 0x31, 0xd6, 0x1d, 0xa9, 0x6c, 0xc2, 0x2d, 0xd0, 0x38, 0x63, 0x68,
|
||||||
|
+ 0xb8, 0x34, 0x95, 0xe1, 0xe8, 0x75, 0x3b, 0x32, 0xeb, 0x3a, 0x66, 0x9d, 0x42, 0xba, 0x66, 0xce,
|
||||||
|
+ 0x39, 0xb0, 0x25, 0x6c, 0xce, 0xce, 0x6c, 0xd1, 0x18, 0x5a, 0xc4, 0xe2, 0x4c, 0x54, 0x59, 0x16,
|
||||||
|
+ 0x9d, 0x93, 0x38, 0x3f, 0x85, 0x4d, 0xcd, 0x49, 0x73, 0xb6, 0x6c, 0x5e, 0x87, 0x06, 0xb3, 0x6a,
|
||||||
|
+ 0x54, 0xf2, 0x56, 0x94, 0x21, 0x32, 0x73, 0xe8, 0x86, 0x14, 0x16, 0x30, 0x1c, 0xc9, 0x9a, 0xa3,
|
||||||
|
+ 0xaa, 0xb6, 0x2c, 0x47, 0x48, 0x6f, 0xc9, 0x7a, 0x3b, 0x37, 0xd3, 0x7a, 0x6b, 0x13, 0x06, 0x72,
|
||||||
|
+ 0xa2, 0x24, 0xd1, 0xf9, 0x39, 0x6c, 0x3e, 0x8e, 0x67, 0x24, 0xc6, 0xfb, 0x47, 0x4f, 0x1f, 0xe1,
|
||||||
|
+ 0x2c, 0x2a, 0x20, 0xa8, 0xcb, 0xec, 0x49, 0xa9, 0xd1, 0x74, 0xd5, 0x58, 0x5e, 0x93, 0xf8, 0xd8,
|
||||||
|
+ 0x0b, 0x92, 0x94, 0x9b, 0xce, 0x50, 0x23, 0x3e, 0xde, 0x4f, 0x52, 0x2e, 0xc3, 0xbc, 0x7c, 0xe6,
|
||||||
|
+ 0x69, 0x3c, 0x9b, 0xab, 0xbb, 0xd2, 0x74, 0xd7, 0x83, 0x24, 0x7d, 0x1c, 0xcf, 0xe6, 0xce, 0xff,
|
||||||
|
+ 0xa8, 0x5a, 0x18, 0xe3, 0xd0, 0xf5, 0xe3, 0x90, 0x46, 0xf7, 0xf1, 0x59, 0x41, 0x42, 0x56, 0x77,
|
||||||
|
+ 0xd9, 0x98, 0xf0, 0x6d, 0x05, 0x3a, 0xf7, 0xa6, 0x38, 0x16, 0xf7, 0xb1, 0xf0, 0xc9, 0x4c, 0xd5,
|
||||||
|
+ 0x56, 0x67, 0x98, 0x71, 0x42, 0x63, 0x73, 0xf0, 0x2d, 0x28, 0x4b, 0x63, 0x12, 0x13, 0xe1, 0x85,
|
||||||
|
+ 0x3e, 0x8e, 0x68, 0x6c, 0xbc, 0x00, 0x12, 0x75, 0x5f, 0x61, 0xd0, 0x9b, 0xd0, 0xd7, 0x9d, 0x3b,
|
||||||
|
+ 0xef, 0xc4, 0x8f, 0xc3, 0x99, 0xbc, 0x72, 0xba, 0x93, 0xd1, 0xd3, 0xe8, 0x03, 0x83, 0x45, 0x6f,
|
||||||
|
+ 0xc1, 0x86, 0xb9, 0x10, 0x39, 0x65, 0x5d, 0x51, 0xf6, 0x0d, 0xbe, 0x44, 0x9a, 0x26, 0x09, 0x65,
|
||||||
|
+ 0x82, 0x7b, 0x1c, 0x07, 0x01, 0x8d, 0x12, 0x53, 0x98, 0xf4, 0x2d, 0x7e, 0xa2, 0xd1, 0xce, 0x14,
|
||||||
|
+ 0x36, 0x1f, 0x4a, 0x3b, 0x8d, 0x25, 0xf9, 0x06, 0xf7, 0x22, 0x1c, 0x79, 0xc7, 0x33, 0x1a, 0x9c,
|
||||||
|
+ 0x7a, 0x32, 0x4c, 0x19, 0x0f, 0xcb, 0xd4, 0x67, 0x4f, 0x22, 0x27, 0xe4, 0x1b, 0x55, 0x83, 0x4b,
|
||||||
|
+ 0xaa, 0x13, 0x2a, 0x92, 0x59, 0x3a, 0xf5, 0x12, 0x46, 0x8f, 0xb1, 0x31, 0xb1, 0x1f, 0xe1, 0xe8,
|
||||||
|
+ 0x40, 0xe3, 0x8f, 0x24, 0xda, 0xf9, 0x53, 0x05, 0xb6, 0xca, 0x92, 0x4c, 0xd0, 0xdd, 0x85, 0xad,
|
||||||
|
+ 0xb2, 0x28, 0xf3, 0x10, 0xeb, 0x44, 0x6f, 0x50, 0x14, 0xa8, 0x9f, 0xe4, 0x0f, 0xa1, 0xab, 0xda,
|
||||||
|
+ 0xb9, 0x5e, 0xa8, 0x39, 0x95, 0xd3, 0x8f, 0xe2, 0xbe, 0xb8, 0x1d, 0xbf, 0xb8, 0x4b, 0x1f, 0xc3,
|
||||||
|
+ 0x35, 0x63, 0xbe, 0xb7, 0xac, 0xb6, 0x3e, 0x10, 0xdb, 0x86, 0xe0, 0xd1, 0x82, 0xf6, 0x5f, 0xc0,
|
||||||
|
+ 0x30, 0x47, 0xed, 0xcd, 0x15, 0xd2, 0xfa, 0xea, 0x5d, 0xd8, 0x5c, 0x30, 0xf6, 0x5e, 0x18, 0x32,
|
||||||
|
+ 0x75, 0x41, 0xeb, 0xee, 0xaa, 0x29, 0xe7, 0x2e, 0x5c, 0x9d, 0x60, 0xa1, 0xbd, 0xe1, 0x0b, 0x53,
|
||||||
|
+ 0x13, 0x68, 0x66, 0x1b, 0x50, 0x9b, 0xe0, 0x40, 0x19, 0x5f, 0x73, 0xe5, 0x50, 0x1e, 0xc0, 0xa7,
|
||||||
|
+ 0x1c, 0x07, 0xca, 0xca, 0x9a, 0xab, 0xc6, 0xce, 0x1f, 0x2b, 0xb0, 0x6e, 0xc2, 0xa4, 0x0c, 0xf5,
|
||||||
|
+ 0x21, 0x23, 0x67, 0x98, 0x99, 0xa3, 0x67, 0x20, 0xf4, 0x06, 0xf4, 0xf4, 0xc8, 0xa3, 0x89, 0x20,
|
||||||
|
+ 0x34, 0x0b, 0xbe, 0x5d, 0x8d, 0x7d, 0xac, 0x91, 0xaa, 0x53, 0xa7, 0x1a, 0x51, 0xa6, 0xe6, 0x33,
|
||||||
|
+ 0x90, 0x6a, 0xb7, 0x71, 0x19, 0x19, 0x54, 0xb0, 0x6d, 0xb9, 0x06, 0x92, 0x47, 0xdd, 0xf2, 0x5b,
|
||||||
|
+ 0x53, 0xfc, 0x2c, 0x28, 0x8f, 0x7a, 0x44, 0xd3, 0x58, 0x78, 0x09, 0x25, 0xb1, 0x30, 0xd1, 0x15,
|
||||||
|
+ 0x14, 0xea, 0x48, 0x62, 0x9c, 0x5f, 0x55, 0xa0, 0xa1, 0xbb, 0xd5, 0xb2, 0xca, 0xcc, 0xde, 0xb8,
|
||||||
|
+ 0x2a, 0x51, 0xf9, 0x82, 0x92, 0xa5, 0xdf, 0x35, 0x35, 0x96, 0xf7, 0xf8, 0x2c, 0xd2, 0x91, 0xda,
|
||||||
|
+ 0xa8, 0x76, 0x16, 0xa9, 0x10, 0xfd, 0x06, 0xf4, 0xf2, 0xa7, 0x52, 0xcd, 0x6b, 0x15, 0xbb, 0x19,
|
||||||
|
+ 0x56, 0x91, 0x5d, 0xa8, 0xa9, 0xf3, 0x13, 0x59, 0x5c, 0x67, 0x9d, 0xda, 0x0d, 0xa8, 0xa5, 0x99,
|
||||||
|
+ 0x32, 0x72, 0x28, 0x31, 0xd3, 0xec, 0x91, 0x95, 0x43, 0x74, 0x1b, 0x7a, 0x7e, 0x18, 0x12, 0xb9,
|
||||||
|
+ 0xdc, 0x9f, 0x3d, 0x24, 0x61, 0x76, 0x49, 0xcb, 0x58, 0xe7, 0x2f, 0x15, 0xe8, 0xef, 0xd3, 0x64,
|
||||||
|
+ 0xfe, 0xff, 0x64, 0x86, 0x0b, 0x11, 0x44, 0x29, 0x69, 0xde, 0x58, 0x39, 0x96, 0x79, 0xe3, 0x33,
|
||||||
|
+ 0x32, 0xc3, 0xfa, 0x6a, 0xe9, 0x9d, 0x6d, 0x4a, 0x84, 0xba, 0x56, 0x76, 0x32, 0x6b, 0x80, 0x75,
|
||||||
|
+ 0xf5, 0xe4, 0x23, 0x1a, 0xaa, 0x0c, 0x39, 0x24, 0xcc, 0xcb, 0xda, 0x5d, 0x5d, 0x77, 0x3d, 0x24,
|
||||||
|
+ 0x4c, 0x4d, 0x19, 0x43, 0xd6, 0x54, 0xc7, 0xb5, 0x68, 0x48, 0x43, 0x63, 0xa4, 0x21, 0xdb, 0xd0,
|
||||||
|
+ 0xa0, 0xcf, 0x9e, 0x71, 0x2c, 0x54, 0x2e, 0x5b, 0x73, 0x0d, 0x94, 0x85, 0xb9, 0x66, 0x21, 0xcc,
|
||||||
|
+ 0x5d, 0x81, 0x4d, 0xd5, 0xdb, 0x7f, 0xc2, 0xfc, 0x80, 0xc4, 0x53, 0x1b, 0x8a, 0xb7, 0x00, 0x4d,
|
||||||
|
+ 0x04, 0x4d, 0x16, 0xb0, 0x63, 0x18, 0x98, 0x37, 0xe7, 0xe8, 0xc7, 0x13, 0x6b, 0xfa, 0x35, 0x68,
|
||||||
|
+ 0x4a, 0xd0, 0x63, 0xf8, 0x6b, 0x1b, 0x18, 0xcd, 0xb4, 0xf3, 0x16, 0x74, 0xf4, 0xd0, 0x84, 0x81,
|
||||||
|
+ 0x9c, 0x94, 0x97, 0x49, 0xf9, 0x9d, 0xbf, 0x6d, 0x98, 0x70, 0x6b, 0x6a, 0x68, 0xf4, 0x10, 0xfa,
|
||||||
|
+ 0x0b, 0xff, 0x64, 0x90, 0x69, 0xaa, 0xac, 0xfe, 0x55, 0x33, 0xda, 0x1e, 0xeb, 0x7f, 0x3c, 0x63,
|
||||||
|
+ 0xfb, 0x8f, 0x67, 0xfc, 0x20, 0x4a, 0xc4, 0x1c, 0x3d, 0x80, 0x5e, 0xf9, 0xef, 0x05, 0xba, 0x6e,
|
||||||
|
+ 0x73, 0x90, 0x15, 0xff, 0x34, 0x2e, 0x64, 0xf3, 0x10, 0xfa, 0x0b, 0x3f, 0x32, 0xac, 0x3e, 0xab,
|
||||||
|
+ 0xff, 0x6f, 0x5c, 0xc8, 0xe8, 0x2e, 0xb4, 0x0b, 0x7f, 0x2e, 0xd0, 0x50, 0x33, 0x59, 0xfe, 0x99,
|
||||||
|
+ 0x71, 0x21, 0x83, 0x7d, 0xe8, 0x96, 0x7e, 0x26, 0xa0, 0x91, 0xb1, 0x67, 0xc5, 0x1f, 0x86, 0x0b,
|
||||||
|
+ 0x99, 0xec, 0x41, 0xbb, 0xd0, 0xd3, 0xb7, 0x5a, 0x2c, 0xff, 0x38, 0x18, 0x5d, 0x5b, 0x31, 0x63,
|
||||||
|
+ 0xb6, 0xf3, 0x00, 0xba, 0xa5, 0x0e, 0xbc, 0x55, 0x64, 0x55, 0xf7, 0x7f, 0x74, 0x7d, 0xe5, 0x9c,
|
||||||
|
+ 0xe1, 0xf4, 0x10, 0xfa, 0x0b, 0xfd, 0x78, 0xeb, 0xdc, 0xd5, 0x6d, 0xfa, 0x0b, 0xcd, 0xfa, 0x5c,
|
||||||
|
+ 0x6d, 0x76, 0xa1, 0xdc, 0x2a, 0x6c, 0xf6, 0x72, 0xf7, 0x7d, 0x74, 0x63, 0xf5, 0xa4, 0xd1, 0xea,
|
||||||
|
+ 0x01, 0xf4, 0xca, 0x8d, 0x77, 0xcb, 0x6c, 0x65, 0x3b, 0xfe, 0xf2, 0x93, 0x53, 0xea, 0xc1, 0xe7,
|
||||||
|
+ 0x27, 0x67, 0x55, 0x6b, 0xfe, 0x42, 0x46, 0xf7, 0x00, 0x4c, 0x71, 0x15, 0x92, 0x38, 0xdb, 0xb2,
|
||||||
|
+ 0xa5, 0xa2, 0x2e, 0xdb, 0xb2, 0x15, 0x85, 0xd8, 0x5d, 0x00, 0x5d, 0x13, 0x85, 0x34, 0x15, 0xe8,
|
||||||
|
+ 0xaa, 0x55, 0x63, 0xa1, 0x10, 0x1b, 0x0d, 0x97, 0x27, 0x96, 0x18, 0x60, 0xc6, 0x5e, 0x84, 0xc1,
|
||||||
|
+ 0x67, 0x00, 0x79, 0xad, 0x65, 0x19, 0x2c, 0x55, 0x5f, 0x97, 0xf8, 0xa0, 0x53, 0xac, 0xac, 0x90,
|
||||||
|
+ 0xb1, 0x75, 0x45, 0xb5, 0x75, 0x09, 0x8b, 0xfe, 0x42, 0xe6, 0x5c, 0x3e, 0x6c, 0x8b, 0x09, 0xf5,
|
||||||
|
+ 0x68, 0x29, 0x7b, 0x46, 0x1f, 0x42, 0xa7, 0x98, 0x32, 0x5b, 0x2d, 0x56, 0xa4, 0xd1, 0xa3, 0x52,
|
||||||
|
+ 0xda, 0x8c, 0xee, 0x42, 0xaf, 0x9c, 0x10, 0xa3, 0xc2, 0xbd, 0x58, 0x4a, 0x93, 0x47, 0xa6, 0x19,
|
||||||
|
+ 0x54, 0x20, 0x7f, 0x1f, 0x20, 0x4f, 0x9c, 0xad, 0xfb, 0x96, 0x52, 0xe9, 0x05, 0xa9, 0x9f, 0x41,
|
||||||
|
+ 0xaf, 0x10, 0xb7, 0x65, 0x4d, 0x78, 0xb5, 0x64, 0x70, 0x1e, 0xcd, 0x47, 0x26, 0xc3, 0x2a, 0x85,
|
||||||
|
+ 0xed, 0x7b, 0xd0, 0x29, 0xbe, 0x11, 0xd6, 0xda, 0x15, 0xef, 0xc6, 0x65, 0x41, 0xaf, 0xf0, 0x9e,
|
||||||
|
+ 0xd8, 0xb3, 0xbb, 0xfc, 0xc4, 0x5c, 0x16, 0xf4, 0x4a, 0xf5, 0xa8, 0x8d, 0x35, 0xab, 0x8a, 0xd4,
|
||||||
|
+ 0xcb, 0x9e, 0x82, 0x72, 0xf1, 0x66, 0xbd, 0xbf, 0xb2, 0xa4, 0xbb, 0xec, 0x0c, 0x16, 0xeb, 0x14,
|
||||||
|
+ 0xeb, 0x8f, 0x15, 0xb5, 0xcb, 0xf7, 0xc4, 0x84, 0x62, 0x2d, 0x52, 0x88, 0x09, 0x2b, 0x4a, 0x94,
|
||||||
|
+ 0x0b, 0x19, 0x1d, 0x40, 0xff, 0xa1, 0x4d, 0x33, 0x4d, 0x0a, 0x6c, 0xd4, 0x59, 0x91, 0xf2, 0x8f,
|
||||||
|
+ 0x46, 0xab, 0xa6, 0xcc, 0x2e, 0x7f, 0x0e, 0x83, 0xa5, 0xf4, 0x17, 0xdd, 0xcc, 0x5a, 0x9e, 0x2b,
|
||||||
|
+ 0xf3, 0xe2, 0x0b, 0xd5, 0x3a, 0x84, 0x8d, 0xc5, 0xec, 0x17, 0xbd, 0x62, 0x36, 0x7d, 0x75, 0x56,
|
||||||
|
+ 0x7c, 0x21, 0xab, 0x8f, 0xa1, 0x69, 0xb3, 0x2d, 0x64, 0x5a, 0xcb, 0x0b, 0xd9, 0xd7, 0x45, 0x4b,
|
||||||
|
+ 0xf7, 0x3a, 0xdf, 0x7e, 0x77, 0xb3, 0xf2, 0xd7, 0xef, 0x6e, 0x56, 0xfe, 0xf1, 0xdd, 0xcd, 0xca,
|
||||||
|
+ 0x71, 0x43, 0xcd, 0xbe, 0xff, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x45, 0xa8, 0x91, 0xab, 0x62,
|
||||||
|
+ 0x22, 0x00, 0x00,
|
||||||
|
}
|
||||||
|
diff --git a/protocols/grpc/agent.proto b/protocols/grpc/agent.proto
|
||||||
|
index b0fab6d..d863645 100644
|
||||||
|
--- a/protocols/grpc/agent.proto
|
||||||
|
+++ b/protocols/grpc/agent.proto
|
||||||
|
@@ -46,6 +46,7 @@ service AgentService {
|
||||||
|
rpc UpdateRoutes(UpdateRoutesRequest) returns (Routes);
|
||||||
|
rpc ListInterfaces(ListInterfacesRequest) returns(Interfaces);
|
||||||
|
rpc ListRoutes(ListRoutesRequest) returns (Routes);
|
||||||
|
+ rpc UpdateIPVSRule(UpdateIPVSRequest) returns (IPVSResponse);
|
||||||
|
|
||||||
|
// tracing
|
||||||
|
rpc StartTracing(StartTracingRequest) returns (google.protobuf.Empty);
|
||||||
|
@@ -495,3 +496,13 @@ message StartTracingRequest {
|
||||||
|
|
||||||
|
message StopTracingRequest {
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+message UpdateIPVSRequest {
|
||||||
|
+ // IPVS_req is the IPVS rule message needed to update
|
||||||
|
+ string IPVS_req = 1;
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+message IPVSResponse {
|
||||||
|
+ // IPVS_res is the response of IPVS updating
|
||||||
|
+ string IPVS_res = 1;
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
193
agent/patches/0004-agent-add-IPVS-test.patch
Normal file
193
agent/patches/0004-agent-add-IPVS-test.patch
Normal file
@ -0,0 +1,193 @@
|
|||||||
|
From 01563c08910ddaba4077fd9dc691df541e045165 Mon Sep 17 00:00:00 2001
|
||||||
|
From: xiadanni <xiadanni1@huawei.com>
|
||||||
|
Date: Tue, 18 Aug 2020 17:05:32 +0800
|
||||||
|
Subject: [PATCH 04/16] agent: add IPVS test
|
||||||
|
|
||||||
|
Signed-off-by: xiadanni <xiadanni1@huawei.com>
|
||||||
|
---
|
||||||
|
grpc_test.go | 172 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||||
|
1 file changed, 172 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/grpc_test.go b/grpc_test.go
|
||||||
|
index e69102b..d58c0b6 100644
|
||||||
|
--- a/grpc_test.go
|
||||||
|
+++ b/grpc_test.go
|
||||||
|
@@ -1840,3 +1840,175 @@ func getPipeMaxSize() (uint32, error) {
|
||||||
|
u, err := strconv.ParseUint(s, 10, 32)
|
||||||
|
return uint32(u), err
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func TestUpdateIPVSRule(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+
|
||||||
|
+ // add IPVS rule successfully
|
||||||
|
+ a := &agentGRPC{
|
||||||
|
+ sandbox: &sandbox{
|
||||||
|
+ containers: make(map[string]*container),
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ req := &pb.UpdateIPVSRequest{
|
||||||
|
+ IPVSReq: "ipvsadm -A -t 17.2.0.7:80 -s rr -p 3000",
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ _, err := a.UpdateIPVSRule(context.Background(), req)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ // delete ipvs rule successfully
|
||||||
|
+ req = &pb.UpdateIPVSRequest{
|
||||||
|
+ IPVSReq: "ipvsadm -D -t 17.2.0.7:80",
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ // update ipvs rule error because exec failed
|
||||||
|
+ req = &pb.UpdateIPVSRequest{
|
||||||
|
+ IPVSReq: "ipvsadm -A -t 17.2.0.7:80 -s rr -p -3000",
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ assert.Contains(err.Error(), "exec IPVS command failed")
|
||||||
|
+
|
||||||
|
+ // update IPVS rule error because rule less than validHeadLength
|
||||||
|
+ req = &pb.UpdateIPVSRequest{
|
||||||
|
+ IPVSReq: "ipvsa",
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ assert.Contains(err.Error(), "invalid IPVS rule")
|
||||||
|
+
|
||||||
|
+ // update ipvs rule error because invalid command
|
||||||
|
+ req = &pb.UpdateIPVSRequest{
|
||||||
|
+ IPVSReq: "abcabcabc ipvsadm",
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ assert.Contains(err.Error(), "invalid IPVS rule")
|
||||||
|
+
|
||||||
|
+ // add ipvs rule error because rule count exceeds
|
||||||
|
+ a = &agentGRPC{
|
||||||
|
+ sandbox: &sandbox{
|
||||||
|
+ containers: make(map[string]*container),
|
||||||
|
+ ipvsadm: ipvsAdm{
|
||||||
|
+ ipvsRuleCnt: 20000,
|
||||||
|
+ },
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ req = &pb.UpdateIPVSRequest{
|
||||||
|
+ IPVSReq: "ipvsadm -A -t 17.2.0.7:80 -s rr -p 3000",
|
||||||
|
+ }
|
||||||
|
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ assert.Errorf(err, "rules exceed limit")
|
||||||
|
+
|
||||||
|
+ // add ipvs rule error because ipvs request item less than 2
|
||||||
|
+ a = &agentGRPC{
|
||||||
|
+ sandbox: &sandbox{
|
||||||
|
+ containers: make(map[string]*container),
|
||||||
|
+ ipvsadm: ipvsAdm{
|
||||||
|
+ ipvsRuleCnt: 0,
|
||||||
|
+ },
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ req = &pb.UpdateIPVSRequest{
|
||||||
|
+ IPVSReq: "ipvsadm",
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ assert.Contains(err.Error(), "invalid IPVS rule")
|
||||||
|
+
|
||||||
|
+ // add ipvs rule error because ipvs rule nil
|
||||||
|
+ req = nil
|
||||||
|
+
|
||||||
|
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ assert.Contains(err.Error(), "IPVS rule is nil")
|
||||||
|
+
|
||||||
|
+ // add ipvs rule error because ipvs rule string is empty
|
||||||
|
+ req = &pb.UpdateIPVSRequest{
|
||||||
|
+ IPVSReq: "",
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ assert.Contains(err.Error(), "IPVS rule is nil")
|
||||||
|
+
|
||||||
|
+ // restore ipvs rule successfully
|
||||||
|
+ a = &agentGRPC{
|
||||||
|
+ sandbox: &sandbox{
|
||||||
|
+ containers: make(map[string]*container),
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ req = &pb.UpdateIPVSRequest{
|
||||||
|
+ IPVSReq: "restore|2|-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m",
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ // clear IPVS rule successfully
|
||||||
|
+ req = &pb.UpdateIPVSRequest{
|
||||||
|
+ IPVSReq: "ipvsadm -C",
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ assert.Equal(a.sandbox.ipvsadm.ipvsRuleCnt, uint64(0))
|
||||||
|
+
|
||||||
|
+ // restore ipvs rule error because rule count invalid
|
||||||
|
+ req = &pb.UpdateIPVSRequest{
|
||||||
|
+ IPVSReq: "restore|abc|-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m",
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ assert.Contains(err.Error(), "invalid IPVS rule")
|
||||||
|
+
|
||||||
|
+ // restore ipvs rule error because other rules exists
|
||||||
|
+ a = &agentGRPC{
|
||||||
|
+ sandbox: &sandbox{
|
||||||
|
+ containers: make(map[string]*container),
|
||||||
|
+ ipvsadm: ipvsAdm{
|
||||||
|
+ ipvsRuleCnt: 5,
|
||||||
|
+ },
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ req = &pb.UpdateIPVSRequest{
|
||||||
|
+ IPVSReq: "restore|2|-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m",
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ assert.Contains(err.Error(), "exist some rules in system")
|
||||||
|
+
|
||||||
|
+ // restore ipvs rule error because ipvs req item less than 3
|
||||||
|
+ a = &agentGRPC{
|
||||||
|
+ sandbox: &sandbox{
|
||||||
|
+ containers: make(map[string]*container),
|
||||||
|
+ ipvsadm: ipvsAdm{
|
||||||
|
+ ipvsRuleCnt: 0,
|
||||||
|
+ },
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ req = &pb.UpdateIPVSRequest{
|
||||||
|
+ IPVSReq: "restore|-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m",
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ assert.Contains(err.Error(), "invalid IPVS rule")
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
42
agent/patches/0005-mount-support-mount-block-device.patch
Normal file
42
agent/patches/0005-mount-support-mount-block-device.patch
Normal file
@ -0,0 +1,42 @@
|
|||||||
|
From 8c9f9be2a9c195d0bc12b43c491adaacb7bb8154 Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Tue, 18 Aug 2020 10:42:38 +0800
|
||||||
|
Subject: [PATCH 05/16] mount: support mount block device
|
||||||
|
|
||||||
|
reason: modify mountStorage to support mount block device
|
||||||
|
"-v /dev/blockdevice:/home/test"
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
mount.go | 15 ++++++++++++++-
|
||||||
|
1 file changed, 14 insertions(+), 1 deletion(-)
|
||||||
|
|
||||||
|
diff --git a/mount.go b/mount.go
|
||||||
|
index f0c3efe..a05d4a6 100644
|
||||||
|
--- a/mount.go
|
||||||
|
+++ b/mount.go
|
||||||
|
@@ -364,7 +364,20 @@ func commonStorageHandler(storage pb.Storage) (string, error) {
|
||||||
|
func mountStorage(storage pb.Storage) error {
|
||||||
|
flags, options := parseMountFlagsAndOptions(storage.Options)
|
||||||
|
|
||||||
|
- return mount(storage.Source, storage.MountPoint, storage.Fstype, flags, options)
|
||||||
|
+ var fsType = storage.Fstype
|
||||||
|
+ if (storage.Driver == driverSCSIType || storage.Driver == driverBlkType) && strings.Contains(storage.Fstype, "bind") {
|
||||||
|
+ cs := strings.Split(storage.Fstype, "-")
|
||||||
|
+ if len(cs) == 2 && cs[1] != "" {
|
||||||
|
+ fsType = cs[1]
|
||||||
|
+ // here we temporarily discard the bind option,
|
||||||
|
+ // in order to be able to mount the file system of the block device.
|
||||||
|
+ // and then reset `storage.Fstype` to "bind" which pass through to the libcontainer pkg.
|
||||||
|
+ flags = flags &^ flagList["bind"]
|
||||||
|
+ storage.Fstype = "bind"
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return mount(storage.Source, storage.MountPoint, fsType, flags, options)
|
||||||
|
}
|
||||||
|
|
||||||
|
// addStorages takes a list of storages passed by the caller, and perform the
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,54 @@
|
|||||||
|
From bccda1d208f31eab55863883cf0718d7b4b8deef Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 18 Aug 2020 19:30:42 +0800
|
||||||
|
Subject: [PATCH 06/16] agent: make workaround for slow response in aarch64
|
||||||
|
|
||||||
|
reason: make workaround for slow response in aarch64
|
||||||
|
when hotplug virtio-net-pci device
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
agent.go | 2 +-
|
||||||
|
device.go | 12 ++++++------
|
||||||
|
2 files changed, 7 insertions(+), 7 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/agent.go b/agent.go
|
||||||
|
index c161e93..e81c2cd 100644
|
||||||
|
--- a/agent.go
|
||||||
|
+++ b/agent.go
|
||||||
|
@@ -185,7 +185,7 @@ var logsVSockPort = uint32(0)
|
||||||
|
var debugConsoleVSockPort = uint32(0)
|
||||||
|
|
||||||
|
// Timeout waiting for a device to be hotplugged
|
||||||
|
-var hotplugTimeout = 3 * time.Second
|
||||||
|
+var hotplugTimeout = 10 * time.Second
|
||||||
|
|
||||||
|
// Specify the log level
|
||||||
|
var logLevel = defaultLogLevel
|
||||||
|
diff --git a/device.go b/device.go
|
||||||
|
index ec1907e..46a1b96 100644
|
||||||
|
--- a/device.go
|
||||||
|
+++ b/device.go
|
||||||
|
@@ -179,13 +179,13 @@ func getPCIDeviceNameImpl(s *sandbox, pciID string) (string, error) {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
- fieldLogger := agentLog.WithField("pciAddr", pciAddr)
|
||||||
|
-
|
||||||
|
// Rescan pci bus if we need to wait for a new pci device
|
||||||
|
- if err = rescanPciBus(); err != nil {
|
||||||
|
- fieldLogger.WithError(err).Error("Failed to scan pci bus")
|
||||||
|
- return "", err
|
||||||
|
- }
|
||||||
|
+ // FIXME:Comment out this code Temporarily, because once the PCIBus is scanned,
|
||||||
|
+ // the device hot-plug event is lost
|
||||||
|
+ //if err = rescanPciBus(); err != nil {
|
||||||
|
+ // fieldLogger.WithError(err).Error("Failed to scan pci bus")
|
||||||
|
+ // return "", err
|
||||||
|
+ //}
|
||||||
|
|
||||||
|
return getDeviceName(s, pciAddr)
|
||||||
|
}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,37 @@
|
|||||||
|
From fa673c93e243ba297d53b585cd2f51fa68380fc5 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 18 Aug 2020 19:37:48 +0800
|
||||||
|
Subject: [PATCH 07/16] agent: using pcie-root-port driver to hotplug device
|
||||||
|
|
||||||
|
reason: In original pci-bridge scheme, the "F" in the BDF is not used,
|
||||||
|
and was written hard code "0" in the kata-agent. But the "function"
|
||||||
|
is specified when switching to pcie-root-port scheme, so when should
|
||||||
|
pass the "pci-bridge BDF" or "pcie-root-port BDF" from kata-runtime
|
||||||
|
and use in kata-agent.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
device.go | 7 ++++++-
|
||||||
|
1 file changed, 6 insertions(+), 1 deletion(-)
|
||||||
|
|
||||||
|
diff --git a/device.go b/device.go
|
||||||
|
index 46a1b96..8e6950c 100644
|
||||||
|
--- a/device.go
|
||||||
|
+++ b/device.go
|
||||||
|
@@ -101,7 +101,12 @@ func getDevicePCIAddressImpl(pciID string) (string, error) {
|
||||||
|
|
||||||
|
// Deduce the complete bridge address based on the bridge address identifier passed
|
||||||
|
// and the fact that bridges are attached on the main bus with function 0.
|
||||||
|
- pciBridgeAddr := fmt.Sprintf("0000:00:%s.0", bridgeID)
|
||||||
|
+ // Update: support pcie-root-port device
|
||||||
|
+ // In original pci-bridge scheme, the "F" in the BDF is not used, and was written
|
||||||
|
+ // hard code "0" in the kata-agent. But the "function" is specified when switching to
|
||||||
|
+ // pcie-root-port scheme, so when should pass the "pci-bridge BDF" or "pcie-root-port BDF"
|
||||||
|
+ // from kata-runtime and use in kata-agent.
|
||||||
|
+ pciBridgeAddr := fmt.Sprintf("0000:00:%s", bridgeID)
|
||||||
|
|
||||||
|
// Find out the bus exposed by bridge
|
||||||
|
bridgeBusPath := fmt.Sprintf(pciBusPathFormat, sysBusPrefix, pciBridgeAddr)
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,94 @@
|
|||||||
|
From eea286fbafba2e95410b603fbef762e2b25eb207 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 18 Aug 2020 19:45:57 +0800
|
||||||
|
Subject: [PATCH 08/16] agent: support get root bus path dynamically
|
||||||
|
|
||||||
|
reason: support get root bus dynamically no matter the
|
||||||
|
target arch is amd64 or arm64
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
agent.go | 1 +
|
||||||
|
device_amd64.go | 6 +++++-
|
||||||
|
device_arm64.go | 27 ++++++++++++++++++++++++++-
|
||||||
|
3 files changed, 32 insertions(+), 2 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/agent.go b/agent.go
|
||||||
|
index e81c2cd..50afd7a 100644
|
||||||
|
--- a/agent.go
|
||||||
|
+++ b/agent.go
|
||||||
|
@@ -730,6 +730,7 @@ func (s *sandbox) listenToUdevEvents() {
|
||||||
|
defer uEvHandler.Close()
|
||||||
|
|
||||||
|
fieldLogger.Infof("Started listening for uevents")
|
||||||
|
+ rootBusPath := initRootBusPath()
|
||||||
|
|
||||||
|
for {
|
||||||
|
uEv, err := uEvHandler.Read()
|
||||||
|
diff --git a/device_amd64.go b/device_amd64.go
|
||||||
|
index 66bc052..26f55bf 100644
|
||||||
|
--- a/device_amd64.go
|
||||||
|
+++ b/device_amd64.go
|
||||||
|
@@ -8,7 +8,7 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
const (
|
||||||
|
- rootBusPath = "/devices/pci0000:00"
|
||||||
|
+ defaultRootBusPath = "/devices/pci0000:00"
|
||||||
|
|
||||||
|
// From https://www.kernel.org/doc/Documentation/acpi/namespace.txt
|
||||||
|
// The Linux kernel's core ACPI subsystem creates struct acpi_device
|
||||||
|
@@ -21,3 +21,7 @@ const (
|
||||||
|
// in a subdirectory whose prefix is pfn (page frame number).
|
||||||
|
pfnDevPrefix = "/pfn"
|
||||||
|
)
|
||||||
|
+
|
||||||
|
+func initRootBusPath() string {
|
||||||
|
+ return defaultRootBusPath
|
||||||
|
+}
|
||||||
|
diff --git a/device_arm64.go b/device_arm64.go
|
||||||
|
index b73b582..d039c67 100644
|
||||||
|
--- a/device_arm64.go
|
||||||
|
+++ b/device_arm64.go
|
||||||
|
@@ -6,8 +6,14 @@
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
+import (
|
||||||
|
+ "fmt"
|
||||||
|
+ "io/ioutil"
|
||||||
|
+ "regexp"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
const (
|
||||||
|
- rootBusPath = "/devices/platform/4010000000.pcie/pci0000:00"
|
||||||
|
+ defaultRootBusPath = "/devices/platform/4010000000.pcie/pci0000:00"
|
||||||
|
|
||||||
|
// From https://www.kernel.org/doc/Documentation/acpi/namespace.txt
|
||||||
|
// The Linux kernel's core ACPI subsystem creates struct acpi_device
|
||||||
|
@@ -20,3 +26,22 @@ const (
|
||||||
|
// in a subdirectory whose prefix is pfn (page frame number).
|
||||||
|
pfnDevPrefix = "/pfn"
|
||||||
|
)
|
||||||
|
+
|
||||||
|
+func initRootBusPath() string {
|
||||||
|
+ pcieDriverReg := regexp.MustCompile(`^[0-9a-f]{10}.pcie$`)
|
||||||
|
+ rootBusPath := defaultRootBusPath
|
||||||
|
+ files, err := ioutil.ReadDir("/sys/devices/platform")
|
||||||
|
+ if err != nil {
|
||||||
|
+ return rootBusPath
|
||||||
|
+ }
|
||||||
|
+ for _, f := range files {
|
||||||
|
+ if !f.IsDir() {
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+ if pcieDriverReg.MatchString(f.Name()) {
|
||||||
|
+ rootBusPath = fmt.Sprintf("/devices/platform/%s/pci0000:00", f.Name())
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ return rootBusPath
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
188
agent/patches/0009-storage-add-pkg-storage-for-mount.patch
Normal file
188
agent/patches/0009-storage-add-pkg-storage-for-mount.patch
Normal file
@ -0,0 +1,188 @@
|
|||||||
|
From 1268b710c7f0528d971d9c0e54429d3f4e48c372 Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Tue, 18 Aug 2020 16:45:29 +0800
|
||||||
|
Subject: [PATCH 09/16] storage: add pkg/storage for mount
|
||||||
|
|
||||||
|
reason: add gpath.go and nfs.go, provide mount
|
||||||
|
functions and structs
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
pkg/storage/gpath.go | 64 ++++++++++++++++++++++++++++++++++++
|
||||||
|
pkg/storage/nfs.go | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||||
|
2 files changed, 157 insertions(+)
|
||||||
|
create mode 100644 pkg/storage/gpath.go
|
||||||
|
create mode 100644 pkg/storage/nfs.go
|
||||||
|
|
||||||
|
diff --git a/pkg/storage/gpath.go b/pkg/storage/gpath.go
|
||||||
|
new file mode 100644
|
||||||
|
index 0000000..5cb951f
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/pkg/storage/gpath.go
|
||||||
|
@@ -0,0 +1,64 @@
|
||||||
|
+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
||||||
|
+// SPDX-License-Identifier: Apache-2.0
|
||||||
|
+// Description: common functions
|
||||||
|
+// Author: licuifang
|
||||||
|
+// Create: 2019-06-24
|
||||||
|
+
|
||||||
|
+package storage
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "fmt"
|
||||||
|
+ "os"
|
||||||
|
+ "syscall"
|
||||||
|
+
|
||||||
|
+ "github.com/opencontainers/runc/libcontainer/mount"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+func ValidateGpath(guestPath string, opts []string) error {
|
||||||
|
+ for _, opt := range opts {
|
||||||
|
+ switch opt {
|
||||||
|
+ case "shared":
|
||||||
|
+ flag := syscall.MS_SHARED
|
||||||
|
+ if err := ensureMountedAs(guestPath, flag); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ return nil
|
||||||
|
+ case "mounted":
|
||||||
|
+ // if mounted in option, the guestpath must exist and must be a mountpoint
|
||||||
|
+ _, err := os.Stat(guestPath)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ mountpoint, err := mount.Mounted(guestPath)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ if mountpoint == false {
|
||||||
|
+ return fmt.Errorf("the guespath:%s is not a mountpoint while mounted was set", guestPath)
|
||||||
|
+ }
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ if err := os.MkdirAll(guestPath, 0750); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func ensureMountedAs(mountPoint string, flag int) error {
|
||||||
|
+ if err := os.MkdirAll(mountPoint, 0750); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ mounted, err := mount.Mounted(mountPoint)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if !mounted {
|
||||||
|
+ if err := syscall.Mount(mountPoint, mountPoint, "bind", uintptr(syscall.MS_BIND), ""); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return syscall.Mount("", mountPoint, "", uintptr(flag), "")
|
||||||
|
+}
|
||||||
|
diff --git a/pkg/storage/nfs.go b/pkg/storage/nfs.go
|
||||||
|
new file mode 100644
|
||||||
|
index 0000000..44bc85d
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/pkg/storage/nfs.go
|
||||||
|
@@ -0,0 +1,93 @@
|
||||||
|
+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
||||||
|
+// SPDX-License-Identifier: Apache-2.0
|
||||||
|
+// Description: common functions
|
||||||
|
+// Author: leizhongkai
|
||||||
|
+// Create: 2019-03-10
|
||||||
|
+
|
||||||
|
+package storage
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "fmt"
|
||||||
|
+ "os"
|
||||||
|
+ "os/exec"
|
||||||
|
+ "path/filepath"
|
||||||
|
+ "strings"
|
||||||
|
+ "time"
|
||||||
|
+
|
||||||
|
+ "github.com/opencontainers/runc/libcontainer/mount"
|
||||||
|
+ "github.com/sirupsen/logrus"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+const (
|
||||||
|
+ NFS = "nfs"
|
||||||
|
+
|
||||||
|
+ MAX_MOUNT_ATTEMPTS = 100
|
||||||
|
+ MAX_MOUTN_TIMEOUT = 90 // seconds
|
||||||
|
+
|
||||||
|
+ // ignored errors
|
||||||
|
+ PERMISSION_DENY = "Permission denied"
|
||||||
|
+ kataGuestStorageDir = "/run/kata-containers/storage/containers/"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+func nfsMount(source, dest string, opt string) error {
|
||||||
|
+ cmd := exec.Command("/bin/mount", "-t", "nfs", "-o", opt, source, dest)
|
||||||
|
+ res, err := cmd.Output()
|
||||||
|
+ logrus.Debugf("mount %s to %s, and get res %s", source, dest, string(res))
|
||||||
|
+ return err
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func nfsMountWithAttempt(source, dest string, opt string) (err error) {
|
||||||
|
+ timeStart := time.Now().Unix()
|
||||||
|
+ for i := 0; i < MAX_MOUNT_ATTEMPTS; i++ {
|
||||||
|
+ logrus.Infof("this is the %d times to mount %s to %s", i, source, dest)
|
||||||
|
+ err = nfsMount(source, dest, opt)
|
||||||
|
+ if err != nil {
|
||||||
|
+ ee, ok := err.(*exec.ExitError)
|
||||||
|
+ if ok {
|
||||||
|
+ if strings.Contains(string(ee.Stderr), PERMISSION_DENY) {
|
||||||
|
+ logrus.Errorf("mounting nfs:%s to %s, get error: %s,should break", source, dest, string(ee.Stderr))
|
||||||
|
+ // We do not retry when the error type is PERMISSION_DENY.
|
||||||
|
+ // The reason for the retry is that when you do a SFS mount,
|
||||||
|
+ // the network may not have been created yet, because
|
||||||
|
+ // creating the network and startup container is asynchronous
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+ logrus.Infof("mounting nfs:%s to %s, get error: %s,will retry", source, dest, string(ee.Stderr))
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ elapsed := time.Now().Unix() - timeStart
|
||||||
|
+ if elapsed < MAX_MOUTN_TIMEOUT {
|
||||||
|
+ time.Sleep(50 * time.Microsecond)
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return err
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func MountRemoteNfs(source string, opt []string, sandboxId string) error {
|
||||||
|
+ item := strings.Split(source, ":")
|
||||||
|
+ if len(item) != 2 {
|
||||||
|
+ return fmt.Errorf("the nfs of %s format is error", source)
|
||||||
|
+ }
|
||||||
|
+ tmpDes := filepath.Join(kataGuestStorageDir, sandboxId, item[0],item[1])
|
||||||
|
+ if mounted, err := mount.Mounted(tmpDes); err == nil && mounted == true {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+ err := os.MkdirAll(tmpDes, 0750)
|
||||||
|
+ if err != nil {
|
||||||
|
+ logrus.Infof("mkdir %s failed before mount remote nfs server", tmpDes)
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ defer func() {
|
||||||
|
+ if err != nil {
|
||||||
|
+ os.RemoveAll(tmpDes)
|
||||||
|
+ }
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
+ err = nfsMountWithAttempt(source, tmpDes, strings.Join(opt, ","))
|
||||||
|
+
|
||||||
|
+ return err
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,92 @@
|
|||||||
|
From 79bafc5fd8a1dcda6f44ecd830dfe50f4ef1b34b Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Tue, 18 Aug 2020 20:41:30 +0800
|
||||||
|
Subject: [PATCH 10/16] storage: mount nfs and gpath in agent
|
||||||
|
|
||||||
|
reason: add nfsStorageHandler and gpathStorageHandler in
|
||||||
|
storageHandlerList to mount nfs and gpath
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
device.go | 2 ++
|
||||||
|
mount.go | 37 +++++++++++++++++++++++++++++++++++++
|
||||||
|
2 files changed, 39 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/device.go b/device.go
|
||||||
|
index 8e6950c..29f72e9 100644
|
||||||
|
--- a/device.go
|
||||||
|
+++ b/device.go
|
||||||
|
@@ -35,6 +35,8 @@ const (
|
||||||
|
driverEphemeralType = "ephemeral"
|
||||||
|
driverLocalType = "local"
|
||||||
|
vmRootfs = "/"
|
||||||
|
+ driverNfsType = "nfs"
|
||||||
|
+ driverGpathType = "gpath"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
diff --git a/mount.go b/mount.go
|
||||||
|
index a05d4a6..de2bfaf 100644
|
||||||
|
--- a/mount.go
|
||||||
|
+++ b/mount.go
|
||||||
|
@@ -17,6 +17,7 @@ import (
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
|
||||||
|
+ rmtStorage "github.com/kata-containers/agent/pkg/storage"
|
||||||
|
pb "github.com/kata-containers/agent/protocols/grpc"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
@@ -218,6 +219,8 @@ var storageHandlerList = map[string]storageHandler{
|
||||||
|
driverEphemeralType: ephemeralStorageHandler,
|
||||||
|
driverLocalType: localStorageHandler,
|
||||||
|
driverNvdimmType: nvdimmStorageHandler,
|
||||||
|
+ driverNfsType: nfsStorageHandler,
|
||||||
|
+ driverGpathType: gpathStorageHandler,
|
||||||
|
}
|
||||||
|
|
||||||
|
func ephemeralStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (string, error) {
|
||||||
|
@@ -339,6 +342,40 @@ func nvdimmStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (st
|
||||||
|
return "", fmt.Errorf("invalid nvdimm source path: %v", storage.Source)
|
||||||
|
}
|
||||||
|
|
||||||
|
+func nfsStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (string, error) {
|
||||||
|
+ s.Lock()
|
||||||
|
+ defer s.Unlock()
|
||||||
|
+
|
||||||
|
+ // mount nfs
|
||||||
|
+ err := rmtStorage.MountRemoteNfs(storage.Source, storage.Options, s.id)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return "", fmt.Errorf("mount %s to %s failed, get err:%s", storage.Source, storage.MountPoint, err)
|
||||||
|
+ }
|
||||||
|
+ if err := os.MkdirAll(storage.MountPoint, 0750); err != nil {
|
||||||
|
+ logrus.Infof("mkdir %s failed after mount remote nfs server", storage.MountPoint)
|
||||||
|
+ return "", err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ defer func() {
|
||||||
|
+ if err != nil {
|
||||||
|
+ os.RemoveAll(storage.MountPoint)
|
||||||
|
+ }
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
+ return storage.MountPoint, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func gpathStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (string, error) {
|
||||||
|
+ s.Lock()
|
||||||
|
+ defer s.Unlock()
|
||||||
|
+ // validate guespath
|
||||||
|
+ err := rmtStorage.ValidateGpath(storage.Source, storage.Options)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return "", err
|
||||||
|
+ }
|
||||||
|
+ return "", nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// virtioSCSIStorageHandler handles the storage for scsi driver.
|
||||||
|
func virtioSCSIStorageHandler(ctx context.Context, storage pb.Storage, s *sandbox) (string, error) {
|
||||||
|
// Retrieve the device path from SCSI address.
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,78 @@
|
|||||||
|
From 3ac1232a2e3fbfc0465473e5d81cde41847c4252 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Wed, 19 Aug 2020 11:47:37 +0800
|
||||||
|
Subject: [PATCH 11/16] agent: fix agent reap agent process blocked problem
|
||||||
|
|
||||||
|
reason: add container waitProcess() timeout when
|
||||||
|
container process status is D/T.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
grpc.go | 43 +++++++++++++++++++++++++++++++++----------
|
||||||
|
1 file changed, 33 insertions(+), 10 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/grpc.go b/grpc.go
|
||||||
|
index de2cae7..3dd088e 100644
|
||||||
|
--- a/grpc.go
|
||||||
|
+++ b/grpc.go
|
||||||
|
@@ -49,6 +49,11 @@ const (
|
||||||
|
libcontainerPath = "/run/libcontainer"
|
||||||
|
)
|
||||||
|
|
||||||
|
+// keep waitProcessTimeout value same as value in kata-runtime wait WaitProcessRequest response
|
||||||
|
+const (
|
||||||
|
+ waitProcessTimeOut = 10
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
var (
|
||||||
|
sysfsCPUOnlinePath = "/sys/devices/system/cpu"
|
||||||
|
sysfsMemOnlinePath = "/sys/devices/system/memory"
|
||||||
|
@@ -996,17 +1001,35 @@ func (a *agentGRPC) WaitProcess(ctx context.Context, req *pb.WaitProcessRequest)
|
||||||
|
ctr.deleteProcess(proc.id)
|
||||||
|
})
|
||||||
|
|
||||||
|
- // Using helper function wait() to deal with the subreaper.
|
||||||
|
- libContProcess := (*reaperLibcontainerProcess)(&(proc.process))
|
||||||
|
- exitCode, err := a.sandbox.subreaper.wait(proc.exitCodeCh, libContProcess)
|
||||||
|
- if err != nil {
|
||||||
|
- return &pb.WaitProcessResponse{}, err
|
||||||
|
+ done := make(chan error)
|
||||||
|
+ var exitCode int = 0
|
||||||
|
+ go func() {
|
||||||
|
+ // Using helper function wait() to deal with the subreaper.
|
||||||
|
+ libContProcess := (*reaperLibcontainerProcess)(&(proc.process))
|
||||||
|
+ var err error
|
||||||
|
+ exitCode, err = a.sandbox.subreaper.wait(proc.exitCodeCh, libContProcess)
|
||||||
|
+ if err != nil {
|
||||||
|
+ done <- err
|
||||||
|
+ close(done)
|
||||||
|
+ return
|
||||||
|
+ }
|
||||||
|
+ // refill the exitCodeCh with the exitcode which can be read out
|
||||||
|
+ // by another WaitProcess(). Since this channel isn't be closed,
|
||||||
|
+ // here the refill will always success and it will be free by GC
|
||||||
|
+ // once the process exits.
|
||||||
|
+ proc.exitCodeCh <- exitCode
|
||||||
|
+
|
||||||
|
+ close(done)
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
+ select {
|
||||||
|
+ case err := <-done:
|
||||||
|
+ if err != nil {
|
||||||
|
+ return &pb.WaitProcessResponse{}, err
|
||||||
|
+ }
|
||||||
|
+ case <-time.After(time.Duration(waitProcessTimeOut) * time.Second):
|
||||||
|
+ return &pb.WaitProcessResponse{}, grpcStatus.Errorf(codes.DeadlineExceeded, "agent wait reap container process timeout reached after %ds", waitProcessTimeOut)
|
||||||
|
}
|
||||||
|
- //refill the exitCodeCh with the exitcode which can be read out
|
||||||
|
- //by another WaitProcess(). Since this channel isn't be closed,
|
||||||
|
- //here the refill will always success and it will be free by GC
|
||||||
|
- //once the process exits.
|
||||||
|
- proc.exitCodeCh <- exitCode
|
||||||
|
|
||||||
|
return &pb.WaitProcessResponse{
|
||||||
|
Status: int32(exitCode),
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,38 @@
|
|||||||
|
From edb29dfd8f786735763245b3f156b50fd3c1a08e Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Wed, 19 Aug 2020 15:15:31 +0800
|
||||||
|
Subject: [PATCH 12/16] network: support set dns without nameserver
|
||||||
|
|
||||||
|
reason: when runtime sends dns without nameserver to agent,
|
||||||
|
add nameserver before ip address. scenario like annotation
|
||||||
|
with dns
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
network.go | 5 ++++-
|
||||||
|
1 file changed, 4 insertions(+), 1 deletion(-)
|
||||||
|
|
||||||
|
diff --git a/network.go b/network.go
|
||||||
|
index 02e28cb..7046cf8 100644
|
||||||
|
--- a/network.go
|
||||||
|
+++ b/network.go
|
||||||
|
@@ -708,6 +708,9 @@ func setupDNS(dns []string) (err error) {
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
for i, line := range dns {
|
||||||
|
+ if !strings.Contains(line, "nameserver") {
|
||||||
|
+ line = "nameserver" + " " + line
|
||||||
|
+ }
|
||||||
|
if i == (len(dns) - 1) {
|
||||||
|
_, err = file.WriteString(strings.TrimSpace(line))
|
||||||
|
} else {
|
||||||
|
@@ -761,4 +764,4 @@ func (s *sandbox) handleLocalhost() error {
|
||||||
|
}
|
||||||
|
|
||||||
|
return netlink.LinkSetUp(lo)
|
||||||
|
-}
|
||||||
|
\ No newline at end of file
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,304 @@
|
|||||||
|
From 1394dcf579849e5d8103c31556e9af0216a875d2 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Wed, 19 Aug 2020 17:15:51 +0800
|
||||||
|
Subject: [PATCH 13/16] agent: support setting multi queues of interface
|
||||||
|
|
||||||
|
reason: support setting multi queues of a interface
|
||||||
|
when runtime passing Queue in the request.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
network.go | 12 +++++-
|
||||||
|
pkg/net/ethtool.go | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||||
|
pkg/types/types.pb.go | 82 ++++++++++++++++++++++++++-----------
|
||||||
|
pkg/types/types.proto | 1 +
|
||||||
|
4 files changed, 181 insertions(+), 24 deletions(-)
|
||||||
|
create mode 100644 pkg/net/ethtool.go
|
||||||
|
|
||||||
|
diff --git a/network.go b/network.go
|
||||||
|
index 7046cf8..1baaa2e 100644
|
||||||
|
--- a/network.go
|
||||||
|
+++ b/network.go
|
||||||
|
@@ -17,7 +17,7 @@ import (
|
||||||
|
"syscall"
|
||||||
|
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
-
|
||||||
|
+ agentNet "github.com/kata-containers/agent/pkg/net"
|
||||||
|
"github.com/kata-containers/agent/pkg/types"
|
||||||
|
pb "github.com/kata-containers/agent/protocols/grpc"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
@@ -273,6 +273,16 @@ func (s *sandbox) updateInterface(netHandle *netlink.Handle, iface *types.Interf
|
||||||
|
if err == nil {
|
||||||
|
err = retErr
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+ // if link is up, then set the multi queue to it,
|
||||||
|
+ // the kernel of newer version may set multi queue by itself,
|
||||||
|
+ // but we can not rely on it.
|
||||||
|
+ if err == nil && iface.Queues > 0 {
|
||||||
|
+ if ethErr := agentNet.GetEthtool().SetChannel(iface.Name, iface.Queues); ethErr != nil {
|
||||||
|
+ err = grpcStatus.Errorf(codes.Internal, "Could not set multi queue %d for interface %v: %v",
|
||||||
|
+ iface.Queues, link, ethErr)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
}()
|
||||||
|
|
||||||
|
fieldLogger.WithField("link", fmt.Sprintf("%+v", link)).Info("Link found")
|
||||||
|
diff --git a/pkg/net/ethtool.go b/pkg/net/ethtool.go
|
||||||
|
new file mode 100644
|
||||||
|
index 0000000..56a1ece
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/pkg/net/ethtool.go
|
||||||
|
@@ -0,0 +1,110 @@
|
||||||
|
+/*
|
||||||
|
+Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
||||||
|
+SPDX-License-Identifier: Apache-2.0
|
||||||
|
+Description: common functions
|
||||||
|
+Author: fengshaobao
|
||||||
|
+Create: 2019-05-28
|
||||||
|
+*/
|
||||||
|
+
|
||||||
|
+package net
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "fmt"
|
||||||
|
+ "sync"
|
||||||
|
+ "syscall"
|
||||||
|
+ "unsafe"
|
||||||
|
+
|
||||||
|
+ "github.com/sirupsen/logrus"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+const (
|
||||||
|
+ ethtoolGChannels = 0x0000003c /* Get no of channels */
|
||||||
|
+ ethtoolSChannels = 0x0000003d /* Set no of channels */
|
||||||
|
+ ifNameSize = 16
|
||||||
|
+ siocEthtool = 0x8946
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+var (
|
||||||
|
+ t *Ethtool
|
||||||
|
+ once sync.Once
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+// Ethtool to set multiqueue of a network interface,
|
||||||
|
+// with the public method "SetChannel" to set queues of an interface.
|
||||||
|
+type Ethtool struct {
|
||||||
|
+ fd int
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+type ifReq struct {
|
||||||
|
+ ifrName [ifNameSize]byte
|
||||||
|
+ ifrData uintptr
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+type ethtoolChannels struct {
|
||||||
|
+ // ETHTOOL_{G,S}CHANNELS
|
||||||
|
+ cmd uint32
|
||||||
|
+ // Read only. Maximum number of receive channel the driver support.
|
||||||
|
+ maxRx uint32
|
||||||
|
+ // Read only. Maximum number of transmit channel the driver support
|
||||||
|
+ maxTx uint32
|
||||||
|
+ // Read only. Maximum number of other channel the driver support
|
||||||
|
+ maxOther uint32
|
||||||
|
+ //Read only. Maximum number of combined channel the driver support. Set of queues RX, TX or other
|
||||||
|
+ maxCombined uint32
|
||||||
|
+ // Valid values are in the range 1 to the max_rx
|
||||||
|
+ rxCount uint32
|
||||||
|
+ // Valid values are in the range 1 to the max_tx
|
||||||
|
+ txCount uint32
|
||||||
|
+ // Valid values are in the range 1 to the max_other
|
||||||
|
+ otherCount uint32
|
||||||
|
+ // Valid values are in the range 1 to the max_combined
|
||||||
|
+ combinedCount uint32
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// GetEthtool to config multiqueue of a network interface
|
||||||
|
+func GetEthtool() *Ethtool {
|
||||||
|
+ once.Do(func() {
|
||||||
|
+ var err error
|
||||||
|
+ if t, err = newEthtool(); err != nil {
|
||||||
|
+ panic("can not init a socket fd for ethtool")
|
||||||
|
+ }
|
||||||
|
+ })
|
||||||
|
+ return t
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func newEthtool() (*Ethtool, error) {
|
||||||
|
+ fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
|
||||||
|
+ if err != nil || fd < 0 {
|
||||||
|
+ logrus.Warningf("Can not get socket of inet")
|
||||||
|
+ var newErr error
|
||||||
|
+ fd, newErr = syscall.Socket(syscall.AF_NETLINK, syscall.SOCK_RAW, syscall.NETLINK_GENERIC)
|
||||||
|
+ if newErr != nil || fd < 0 {
|
||||||
|
+ return nil, fmt.Errorf("create inet socket with error: %v, and create netlink socket with error: %v", err, newErr)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return &Ethtool{
|
||||||
|
+ fd: int(fd),
|
||||||
|
+ }, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// SetChannel Set the queues of a network interface.
|
||||||
|
+// @devName: the network interface name, e.g. eth0.
|
||||||
|
+// @queues: queues for the network interface.
|
||||||
|
+func (e *Ethtool) SetChannel(devName string, queues uint32) error {
|
||||||
|
+ c := ðtoolChannels{
|
||||||
|
+ cmd: ethtoolSChannels,
|
||||||
|
+ combinedCount: queues,
|
||||||
|
+ }
|
||||||
|
+ var ifName [ifNameSize]byte
|
||||||
|
+ copy(ifName[:], []byte(devName))
|
||||||
|
+ ifr := ifReq{
|
||||||
|
+ ifrName: ifName,
|
||||||
|
+ ifrData: uintptr(unsafe.Pointer(&c)),
|
||||||
|
+ }
|
||||||
|
+ _, _, ep := syscall.Syscall(syscall.SYS_IOCTL, uintptr(e.fd), siocEthtool, uintptr(unsafe.Pointer(&ifr)))
|
||||||
|
+ if ep != 0 {
|
||||||
|
+ return syscall.Errno(ep)
|
||||||
|
+ }
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
diff --git a/pkg/types/types.pb.go b/pkg/types/types.pb.go
|
||||||
|
index 7ea63e3..8b7e2a5 100644
|
||||||
|
--- a/pkg/types/types.pb.go
|
||||||
|
+++ b/pkg/types/types.pb.go
|
||||||
|
@@ -100,6 +100,7 @@ type Interface struct {
|
||||||
|
// list: "veth", "macvtap", "vlan", "macvlan", "tap", ...
|
||||||
|
Type string `protobuf:"bytes,7,opt,name=type,proto3" json:"type,omitempty"`
|
||||||
|
RawFlags uint32 `protobuf:"varint,8,opt,name=raw_flags,json=rawFlags,proto3" json:"raw_flags,omitempty"`
|
||||||
|
+ Queues uint32 `protobuf:"varint,9,opt,name=Queues,proto3" json:"Queues,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *Interface) Reset() { *m = Interface{} }
|
||||||
|
@@ -163,6 +164,13 @@ func (m *Interface) GetRawFlags() uint32 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (m *Interface) GetQueues() uint32 {
|
||||||
|
+ if m != nil {
|
||||||
|
+ return m.Queues
|
||||||
|
+ }
|
||||||
|
+ return 0
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
type Route struct {
|
||||||
|
Dest string `protobuf:"bytes,1,opt,name=dest,proto3" json:"dest,omitempty"`
|
||||||
|
Gateway string `protobuf:"bytes,2,opt,name=gateway,proto3" json:"gateway,omitempty"`
|
||||||
|
@@ -319,6 +327,11 @@ func (m *Interface) MarshalTo(dAtA []byte) (int, error) {
|
||||||
|
i++
|
||||||
|
i = encodeVarintTypes(dAtA, i, uint64(m.RawFlags))
|
||||||
|
}
|
||||||
|
+ if m.Queues != 0 {
|
||||||
|
+ dAtA[i] = 0x48
|
||||||
|
+ i++
|
||||||
|
+ i = encodeVarintTypes(dAtA, i, uint64(m.Queues))
|
||||||
|
+ }
|
||||||
|
return i, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -430,6 +443,9 @@ func (m *Interface) Size() (n int) {
|
||||||
|
if m.RawFlags != 0 {
|
||||||
|
n += 1 + sovTypes(uint64(m.RawFlags))
|
||||||
|
}
|
||||||
|
+ if m.Queues != 0 {
|
||||||
|
+ n += 1 + sovTypes(uint64(m.Queues))
|
||||||
|
+ }
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -841,6 +857,25 @@ func (m *Interface) Unmarshal(dAtA []byte) error {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
+ case 9:
|
||||||
|
+ if wireType != 0 {
|
||||||
|
+ return fmt.Errorf("proto: wrong wireType = %d for field Queues", wireType)
|
||||||
|
+ }
|
||||||
|
+ m.Queues = 0
|
||||||
|
+ for shift := uint(0); ; shift += 7 {
|
||||||
|
+ if shift >= 64 {
|
||||||
|
+ return ErrIntOverflowTypes
|
||||||
|
+ }
|
||||||
|
+ if iNdEx >= l {
|
||||||
|
+ return io.ErrUnexpectedEOF
|
||||||
|
+ }
|
||||||
|
+ b := dAtA[iNdEx]
|
||||||
|
+ iNdEx++
|
||||||
|
+ m.Queues |= (uint32(b) & 0x7F) << shift
|
||||||
|
+ if b < 0x80 {
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
default:
|
||||||
|
iNdEx = preIndex
|
||||||
|
skippy, err := skipTypes(dAtA[iNdEx:])
|
||||||
|
@@ -1155,27 +1190,28 @@ var (
|
||||||
|
func init() { proto.RegisterFile("pkg/types/types.proto", fileDescriptorTypes) }
|
||||||
|
|
||||||
|
var fileDescriptorTypes = []byte{
|
||||||
|
- // 345 bytes of a gzipped FileDescriptorProto
|
||||||
|
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x91, 0xd1, 0x6a, 0xfa, 0x30,
|
||||||
|
- 0x14, 0xc6, 0xff, 0xb1, 0xb6, 0xda, 0xe3, 0xdf, 0xad, 0x84, 0x4d, 0xc2, 0x06, 0x52, 0xbc, 0x59,
|
||||||
|
- 0xd9, 0x85, 0x03, 0x37, 0x76, 0xbf, 0x5d, 0x08, 0xde, 0x49, 0x5e, 0x60, 0xc4, 0x36, 0x3a, 0xd1,
|
||||||
|
- 0xda, 0xd2, 0x44, 0x8b, 0xec, 0x05, 0x77, 0xb9, 0x47, 0x10, 0x9f, 0x64, 0xe4, 0x24, 0x8a, 0xbb,
|
||||||
|
- 0xd1, 0xef, 0x97, 0x93, 0xd3, 0xef, 0x7c, 0x27, 0x70, 0x5b, 0xae, 0x16, 0x4f, 0x7a, 0x5f, 0x4a,
|
||||||
|
- 0x65, 0x7f, 0x87, 0x65, 0x55, 0xe8, 0x82, 0xfa, 0x08, 0x83, 0x19, 0x84, 0x93, 0xe9, 0x5b, 0x96,
|
||||||
|
- 0x55, 0x52, 0x29, 0xfa, 0x00, 0xc1, 0x5c, 0xe4, 0xcb, 0xf5, 0x9e, 0x91, 0x98, 0x24, 0x57, 0xa3,
|
||||||
|
- 0xeb, 0xa1, 0xed, 0x98, 0x4c, 0xc7, 0x78, 0xcc, 0x5d, 0x99, 0x32, 0x68, 0x09, 0xdb, 0xc3, 0x1a,
|
||||||
|
- 0x31, 0x49, 0x42, 0x7e, 0x42, 0x4a, 0xa1, 0x99, 0x0b, 0xb5, 0x62, 0x1e, 0x1e, 0xa3, 0x1e, 0x1c,
|
||||||
|
- 0x08, 0x84, 0x93, 0x8d, 0x96, 0xd5, 0x5c, 0xa4, 0x92, 0xf6, 0x20, 0xc8, 0xe4, 0x6e, 0x99, 0x4a,
|
||||||
|
- 0x34, 0x09, 0xb9, 0x23, 0xd3, 0xb9, 0x11, 0xb9, 0x74, 0x1f, 0x44, 0x4d, 0x47, 0xd0, 0x39, 0x4f,
|
||||||
|
- 0x27, 0x15, 0xf3, 0x62, 0x2f, 0xe9, 0x8c, 0xa2, 0xf3, 0x54, 0xae, 0xc2, 0x2f, 0x2f, 0xd1, 0x08,
|
||||||
|
- 0xbc, 0x5c, 0x6f, 0x59, 0x33, 0x26, 0x49, 0x93, 0x1b, 0x69, 0x1c, 0x3f, 0x6b, 0x73, 0x81, 0xf9,
|
||||||
|
- 0xd6, 0xd1, 0x92, 0x49, 0x51, 0xa6, 0x4b, 0x2c, 0x04, 0x36, 0x85, 0x43, 0x33, 0x8b, 0xf1, 0x60,
|
||||||
|
- 0x2d, 0x3b, 0x8b, 0xd1, 0xf4, 0x1e, 0xc2, 0x4a, 0xd4, 0x1f, 0xf3, 0xb5, 0x58, 0x28, 0xd6, 0x8e,
|
||||||
|
- 0x49, 0xd2, 0xe5, 0xed, 0x4a, 0xd4, 0x63, 0xc3, 0x83, 0x2f, 0xf0, 0x79, 0xb1, 0xd5, 0x98, 0x22,
|
||||||
|
- 0x93, 0x4a, 0xbb, 0x6c, 0xa8, 0x8d, 0xcf, 0x42, 0x68, 0x59, 0x8b, 0xfd, 0x69, 0x5b, 0x0e, 0x2f,
|
||||||
|
- 0x76, 0xe1, 0xfd, 0xd9, 0x45, 0x0f, 0x02, 0x55, 0x6c, 0xab, 0x54, 0x62, 0x8c, 0x90, 0x3b, 0xa2,
|
||||||
|
- 0x37, 0xe0, 0xab, 0xb4, 0x28, 0x25, 0x06, 0xe9, 0x72, 0x0b, 0x8f, 0x77, 0xd0, 0x3e, 0xbd, 0x10,
|
||||||
|
- 0x0d, 0xa0, 0xb1, 0x7b, 0x89, 0xfe, 0xe1, 0xff, 0x6b, 0x44, 0xde, 0xff, 0x7f, 0x1f, 0xfb, 0xe4,
|
||||||
|
- 0xe7, 0xd8, 0x27, 0x87, 0x63, 0x9f, 0xcc, 0x02, 0x7c, 0xfb, 0xe7, 0xdf, 0x00, 0x00, 0x00, 0xff,
|
||||||
|
- 0xff, 0xb5, 0x52, 0x37, 0xf2, 0x14, 0x02, 0x00, 0x00,
|
||||||
|
+ // 356 bytes of a gzipped FileDescriptorProto
|
||||||
|
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x92, 0xc1, 0x4e, 0xc2, 0x40,
|
||||||
|
+ 0x10, 0x86, 0x5d, 0x4a, 0x0b, 0x1d, 0x44, 0x9b, 0x8d, 0x92, 0x8d, 0x26, 0xa4, 0xe1, 0x62, 0xe3,
|
||||||
|
+ 0x01, 0x13, 0x34, 0xde, 0xf5, 0x40, 0xc2, 0x0d, 0xf7, 0x05, 0xcc, 0xd2, 0x2e, 0x48, 0xa0, 0xb4,
|
||||||
|
+ 0xe9, 0xb6, 0x34, 0xc4, 0x17, 0xf4, 0xe8, 0x23, 0x18, 0x9e, 0xc2, 0xa3, 0xd9, 0xd9, 0x85, 0xe0,
|
||||||
|
+ 0x05, 0xfe, 0x6f, 0x67, 0xa7, 0xff, 0xfc, 0xd3, 0xc2, 0x75, 0xbe, 0x5a, 0x3c, 0x94, 0xbb, 0x5c,
|
||||||
|
+ 0x2a, 0xf3, 0x3b, 0xcc, 0x8b, 0xac, 0xcc, 0xa8, 0x8b, 0x30, 0x98, 0x81, 0x3f, 0x99, 0xbe, 0x24,
|
||||||
|
+ 0x49, 0x21, 0x95, 0xa2, 0x77, 0xe0, 0xcd, 0x45, 0xba, 0x5c, 0xef, 0x18, 0x09, 0x49, 0x74, 0x31,
|
||||||
|
+ 0xba, 0x1c, 0x9a, 0x8e, 0xc9, 0x74, 0x8c, 0xc7, 0xdc, 0x96, 0x29, 0x83, 0x96, 0x30, 0x3d, 0xac,
|
||||||
|
+ 0x11, 0x92, 0xc8, 0xe7, 0x07, 0xa4, 0x14, 0x9a, 0xa9, 0x50, 0x2b, 0xe6, 0xe0, 0x31, 0xea, 0xc1,
|
||||||
|
+ 0x2f, 0x01, 0x7f, 0xb2, 0x29, 0x65, 0x31, 0x17, 0xb1, 0xa4, 0x3d, 0xf0, 0x12, 0xb9, 0x5d, 0xc6,
|
||||||
|
+ 0x12, 0x4d, 0x7c, 0x6e, 0x49, 0x77, 0x6e, 0x44, 0x2a, 0xed, 0x03, 0x51, 0xd3, 0x11, 0x74, 0x8e,
|
||||||
|
+ 0xd3, 0x49, 0xc5, 0x9c, 0xd0, 0x89, 0x3a, 0xa3, 0xe0, 0x38, 0x95, 0xad, 0xf0, 0xd3, 0x4b, 0x34,
|
||||||
|
+ 0x00, 0x27, 0x2d, 0x2b, 0xd6, 0x0c, 0x49, 0xd4, 0xe4, 0x5a, 0x6a, 0xc7, 0x8f, 0x5a, 0x5f, 0x60,
|
||||||
|
+ 0xae, 0x71, 0x34, 0xa4, 0x53, 0xe4, 0xf1, 0x12, 0x0b, 0x9e, 0x49, 0x61, 0x51, 0xcf, 0xa2, 0x3d,
|
||||||
|
+ 0x58, 0xcb, 0xcc, 0xa2, 0x35, 0xbd, 0x05, 0xbf, 0x10, 0xf5, 0xfb, 0x7c, 0x2d, 0x16, 0x8a, 0xb5,
|
||||||
|
+ 0x43, 0x12, 0x75, 0x79, 0xbb, 0x10, 0xf5, 0x58, 0xb3, 0xb6, 0x78, 0xab, 0x64, 0x25, 0x15, 0xf3,
|
||||||
|
+ 0xb1, 0x62, 0x69, 0xf0, 0x09, 0x2e, 0xcf, 0xaa, 0x12, 0xd3, 0x25, 0x52, 0x95, 0x36, 0x33, 0x6a,
|
||||||
|
+ 0xed, 0xbf, 0x10, 0xa5, 0xac, 0xc5, 0xee, 0xb0, 0x45, 0x8b, 0x27, 0x3b, 0x72, 0xfe, 0xed, 0xa8,
|
||||||
|
+ 0x07, 0x9e, 0xca, 0xaa, 0x22, 0x96, 0x18, 0xcf, 0xe7, 0x96, 0xe8, 0x15, 0xb8, 0x2a, 0xce, 0x72,
|
||||||
|
+ 0x89, 0x01, 0xbb, 0xdc, 0xc0, 0xfd, 0x0d, 0xb4, 0x0f, 0x6f, 0x8e, 0x7a, 0xd0, 0xd8, 0x3e, 0x05,
|
||||||
|
+ 0x67, 0xf8, 0xff, 0x1c, 0x90, 0xd7, 0xf3, 0xaf, 0x7d, 0x9f, 0x7c, 0xef, 0xfb, 0xe4, 0x67, 0xdf,
|
||||||
|
+ 0x27, 0x33, 0x0f, 0xbf, 0x89, 0xc7, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa5, 0x57, 0x70, 0xc8,
|
||||||
|
+ 0x2c, 0x02, 0x00, 0x00,
|
||||||
|
}
|
||||||
|
diff --git a/pkg/types/types.proto b/pkg/types/types.proto
|
||||||
|
index f6856e1..149df13 100644
|
||||||
|
--- a/pkg/types/types.proto
|
||||||
|
+++ b/pkg/types/types.proto
|
||||||
|
@@ -37,6 +37,7 @@ message Interface {
|
||||||
|
// list: "veth", "macvtap", "vlan", "macvlan", "tap", ...
|
||||||
|
string type = 7;
|
||||||
|
uint32 raw_flags = 8;
|
||||||
|
+ uint32 Queues = 9;
|
||||||
|
}
|
||||||
|
|
||||||
|
message Route {
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,56 @@
|
|||||||
|
From 9e1478d7989fea4ee759cc13009d8de07dac2879 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Wed, 19 Aug 2020 17:01:00 +0800
|
||||||
|
Subject: [PATCH 14/16] agent: fix init hugepages failed problem
|
||||||
|
|
||||||
|
reason: fix init hugepages failed problem
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
.../runc/libcontainer/cgroups/fs/apply_raw.go | 20 ++++++++++++++++++++
|
||||||
|
1 file changed, 20 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go b/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go
|
||||||
|
index ec148b4..47aa3c3 100644
|
||||||
|
--- a/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go
|
||||||
|
+++ b/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go
|
||||||
|
@@ -8,11 +8,13 @@ import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sync"
|
||||||
|
+ "time"
|
||||||
|
|
||||||
|
"github.com/opencontainers/runc/libcontainer/cgroups"
|
||||||
|
"github.com/opencontainers/runc/libcontainer/configs"
|
||||||
|
libcontainerUtils "github.com/opencontainers/runc/libcontainer/utils"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
+ "github.com/sirupsen/logrus"
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
)
|
||||||
|
|
||||||
|
@@ -409,3 +411,21 @@ func CheckCpushares(path string, c uint64) error {
|
||||||
|
func (m *Manager) GetCgroups() (*configs.Cgroup, error) {
|
||||||
|
return m.Cgroups, nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func init() {
|
||||||
|
+ go func() {
|
||||||
|
+ var err error
|
||||||
|
+ if len(HugePageSizes) != 0 {
|
||||||
|
+ return
|
||||||
|
+ }
|
||||||
|
+ for i := 0; i < 10; i++ {
|
||||||
|
+ HugePageSizes, err = cgroups.GetHugePageSize()
|
||||||
|
+ if err == nil {
|
||||||
|
+ logrus.Infof("init hugepages ok loop=%d %v", i, err)
|
||||||
|
+ return
|
||||||
|
+ }
|
||||||
|
+ logrus.Errorf("init hugepages failed loop=%d %v", i, err)
|
||||||
|
+ time.Sleep(time.Second)
|
||||||
|
+ }
|
||||||
|
+ }()
|
||||||
|
+}
|
||||||
|
\ No newline at end of file
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,268 @@
|
|||||||
|
From 6120525f81701424e97d453d515d38f14bbe99d9 Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Wed, 19 Aug 2020 20:25:00 +0800
|
||||||
|
Subject: [PATCH 16/16] clock: synchronizes clock info with proxy
|
||||||
|
|
||||||
|
reason: virtual machine's clock may be incorrect, proxy synchronizes
|
||||||
|
clock info to help virtual machine adjust clock time
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
pkg/clock/clock_util.go | 47 ++++++++++++++++++++++++
|
||||||
|
sync_clock_server.go | 94 +++++++++++++++++++++++++++++++++++++++++++++++
|
||||||
|
sync_clock_server_test.go | 88 ++++++++++++++++++++++++++++++++++++++++++++
|
||||||
|
3 files changed, 229 insertions(+)
|
||||||
|
create mode 100644 pkg/clock/clock_util.go
|
||||||
|
create mode 100644 sync_clock_server.go
|
||||||
|
create mode 100644 sync_clock_server_test.go
|
||||||
|
|
||||||
|
diff --git a/pkg/clock/clock_util.go b/pkg/clock/clock_util.go
|
||||||
|
new file mode 100644
|
||||||
|
index 0000000..4b78c63
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/pkg/clock/clock_util.go
|
||||||
|
@@ -0,0 +1,47 @@
|
||||||
|
+// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved.
|
||||||
|
+// SPDX-License-Identifier: Apache-2.0
|
||||||
|
+// Description: sync clock related function
|
||||||
|
+// Author: xueshaojia
|
||||||
|
+// Create: 2018-11-10
|
||||||
|
+
|
||||||
|
+// Package clock provides clock sync functions
|
||||||
|
+package clock
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "net"
|
||||||
|
+ "syscall"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+type TimeValue struct {
|
||||||
|
+ ClientSendTime int64 `json:"client_send_time"`
|
||||||
|
+ ClientArriveTime int64 `json:"client_arrive_time"`
|
||||||
|
+ ServerSendTime int64 `json:"server_send_time"`
|
||||||
|
+ ServerArriveTime int64 `json:"server_arrive_time"`
|
||||||
|
+ Delta int64 `json:"delta"`
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+const MaxSyncClockByteNum = 400 // sync clock byte num, max=400
|
||||||
|
+const MaxTimeStampSupport = 3155731199000000000 // unit is ns, 2069/12/31 23:59:59
|
||||||
|
+const MinTimeStampSupport = 0 // unit is ns, 1970/1/1 8:0:0
|
||||||
|
+
|
||||||
|
+// getCurrentTimeNs returns UTC time in Ns
|
||||||
|
+func GetCurrentTimeNs() int64 {
|
||||||
|
+ var tv syscall.Timeval
|
||||||
|
+ if err := syscall.Gettimeofday(&tv); err != nil {
|
||||||
|
+ return -1
|
||||||
|
+ }
|
||||||
|
+ return tv.Sec*1000000000 + tv.Usec*1000
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// readConnData reads data from stream
|
||||||
|
+func ReadConnData(stream net.Conn) (buf []byte, byteNum int, err error) {
|
||||||
|
+ buf = make([]byte, MaxSyncClockByteNum)
|
||||||
|
+ byteNum, err = stream.Read(buf)
|
||||||
|
+ return buf, byteNum, err
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// writeConnData writes data to stream
|
||||||
|
+func WriteConnData(stream net.Conn, buf []byte) error {
|
||||||
|
+ _, err := stream.Write(buf)
|
||||||
|
+ return err
|
||||||
|
+}
|
||||||
|
diff --git a/sync_clock_server.go b/sync_clock_server.go
|
||||||
|
new file mode 100644
|
||||||
|
index 0000000..a8e98c6
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/sync_clock_server.go
|
||||||
|
@@ -0,0 +1,94 @@
|
||||||
|
+// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved.
|
||||||
|
+// SPDX-License-Identifier: Apache-2.0
|
||||||
|
+// Description: sync clock server related function
|
||||||
|
+// Author: xueshaojia
|
||||||
|
+// Create: 2018-11-10
|
||||||
|
+
|
||||||
|
+package main
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "encoding/json"
|
||||||
|
+ "fmt"
|
||||||
|
+ "io"
|
||||||
|
+ "net"
|
||||||
|
+ "os/exec"
|
||||||
|
+ "syscall"
|
||||||
|
+
|
||||||
|
+ "github.com/kata-containers/agent/pkg/clock"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+// getSyncClockInfo waits for client request
|
||||||
|
+func getSyncClockInfo(stream net.Conn, clockInfo *clock.TimeValue) error {
|
||||||
|
+ buf, byteNum, err := clock.ReadConnData(stream)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ return json.Unmarshal(buf[:byteNum], clockInfo)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// ackSyncClockInfo sends clock info to client
|
||||||
|
+func ackSyncClockInfo(stream net.Conn, clockInfo *clock.TimeValue) error {
|
||||||
|
+ b, err := json.Marshal(clockInfo)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ return clock.WriteConnData(stream, b)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// setGuestClock sets systemtime and hwclock
|
||||||
|
+func setGuestClock(delta int64) error {
|
||||||
|
+ dstClockNs := clock.GetCurrentTimeNs() + delta
|
||||||
|
+ if dstClockNs < clock.MinTimeStampSupport || dstClockNs > clock.MaxTimeStampSupport {
|
||||||
|
+ return fmt.Errorf("sync clock failed, the valid timestamp is from 1970/1/1 8:0:0 to 2069/12/31 23:59:59")
|
||||||
|
+ }
|
||||||
|
+ dstTimeVal := syscall.NsecToTimeval(dstClockNs)
|
||||||
|
+ if err := syscall.Settimeofday(&dstTimeVal); err != nil {
|
||||||
|
+ agentLog.WithError(err).Error("sync clock, set systemtime failed")
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ cmd := exec.Command("hwclock", "-w")
|
||||||
|
+ if err := cmd.Run(); err != nil {
|
||||||
|
+ agentLog.WithError(err).Error("sync clock, set hwclock failed")
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ agentLog.Infof("sync clock, set systemtime and hwclock succ, delta:%d", delta)
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// SyncClock is a loop that deals with client's request
|
||||||
|
+// 1 read client request
|
||||||
|
+// 2 response with clock info or set guest clock
|
||||||
|
+// 3 if error happends, print error info
|
||||||
|
+func SyncClock(stream net.Conn) {
|
||||||
|
+ for {
|
||||||
|
+ var clockInfo clock.TimeValue
|
||||||
|
+ if err := getSyncClockInfo(stream, &clockInfo); err != nil {
|
||||||
|
+ if err == io.EOF {
|
||||||
|
+ agentLog.WithError(err).Error("sync clock, yamux session error happends")
|
||||||
|
+ stream.Close()
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+ agentLog.WithError(err).Error("sync clock, read sync clock info failed")
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+ // set client's request arrive time
|
||||||
|
+ clockInfo.ClientArriveTime = clock.GetCurrentTimeNs()
|
||||||
|
+ if clockInfo.Delta == 0 {
|
||||||
|
+ // this is a request for clock diff info
|
||||||
|
+ // set server's response send time
|
||||||
|
+ // if fails, wait for next time
|
||||||
|
+ clockInfo.ServerSendTime = clock.GetCurrentTimeNs()
|
||||||
|
+ if err := ackSyncClockInfo(stream, &clockInfo); err != nil {
|
||||||
|
+ agentLog.WithError(err).Error("sync clock, send back clock info failed")
|
||||||
|
+ }
|
||||||
|
+ } else {
|
||||||
|
+ // this is a request for adjusting guest clock
|
||||||
|
+ // 1 adjust guest clock(system time and hwclock)
|
||||||
|
+ // 2 do not send response any more
|
||||||
|
+ // 3 if fails, wait for next request to adjust
|
||||||
|
+ if err := setGuestClock(clockInfo.Delta); err != nil {
|
||||||
|
+ agentLog.WithError(err).Error("sync clock, set local clock failed")
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
diff --git a/sync_clock_server_test.go b/sync_clock_server_test.go
|
||||||
|
new file mode 100644
|
||||||
|
index 0000000..41d5227
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/sync_clock_server_test.go
|
||||||
|
@@ -0,0 +1,88 @@
|
||||||
|
+// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved.
|
||||||
|
+// SPDX-License-Identifier: Apache-2.0
|
||||||
|
+// Description: sync clock server related test
|
||||||
|
+// Author: xueshaojia
|
||||||
|
+// Create: 2018-11-10
|
||||||
|
+
|
||||||
|
+package main
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "encoding/json"
|
||||||
|
+ "fmt"
|
||||||
|
+ "math/rand"
|
||||||
|
+ "net"
|
||||||
|
+ "testing"
|
||||||
|
+
|
||||||
|
+ "github.com/hashicorp/yamux"
|
||||||
|
+ "github.com/kata-containers/agent/pkg/clock"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+var clientStream net.Conn
|
||||||
|
+
|
||||||
|
+func TestGetCurrentTimeNs(t *testing.T) {
|
||||||
|
+ timeRet := clock.GetCurrentTimeNs()
|
||||||
|
+ if timeRet <= 0 {
|
||||||
|
+ t.Fatalf("TestGetCurrentTimeNs Fail, time:%d", timeRet)
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestSetGuestClock(t *testing.T) {
|
||||||
|
+ if err := setGuestClock(0); err != nil {
|
||||||
|
+ t.Fatalf("test set clock failed, err:%v", err)
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestReadWriteData(t *testing.T) {
|
||||||
|
+ sock := GenSocket()
|
||||||
|
+ var err error
|
||||||
|
+ listener, err := net.Listen("unix", sock)
|
||||||
|
+ if err != nil {
|
||||||
|
+ t.Fatalf("listening err: %v", err)
|
||||||
|
+ }
|
||||||
|
+ go SetUpClient(sock)
|
||||||
|
+ conn, err := listener.Accept()
|
||||||
|
+ if err != nil {
|
||||||
|
+ t.Fatalf("accept err: %v", err)
|
||||||
|
+ }
|
||||||
|
+ session, err := yamux.Server(conn, nil)
|
||||||
|
+ if err != nil {
|
||||||
|
+ t.Fatalf("session err: %v", err)
|
||||||
|
+ }
|
||||||
|
+ stream, err := session.Accept()
|
||||||
|
+ if err != nil {
|
||||||
|
+ t.Fatalf("stream err: %v", err)
|
||||||
|
+ }
|
||||||
|
+ go func() {
|
||||||
|
+ var clockInfo clock.TimeValue = clock.TimeValue{1000, 0, 0, 0, 0}
|
||||||
|
+ b, _ := json.Marshal(clockInfo)
|
||||||
|
+ clock.WriteConnData(clientStream, b)
|
||||||
|
+ }()
|
||||||
|
+ _, _, err = clock.ReadConnData(stream)
|
||||||
|
+ if err != nil {
|
||||||
|
+ t.Fatalf("read conn data error, err:%v", err)
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func SetUpClient(sock string) error {
|
||||||
|
+ conn, err := net.Dial("unix", sock)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ session, err := yamux.Client(conn, nil)
|
||||||
|
+ if err != nil {
|
||||||
|
+ conn.Close()
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ stream, err := session.Open()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ clientStream = stream
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func GenSocket() string {
|
||||||
|
+ randSeed := clock.GetCurrentTimeNs()
|
||||||
|
+ rand.Seed(randSeed)
|
||||||
|
+ return fmt.Sprintf("/tmp/%d.sock", rand.Uint32())
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
1005
agent/patches/0017-agent-add-support-of-new-sandbox-StratoVirt.patch
Normal file
1005
agent/patches/0017-agent-add-support-of-new-sandbox-StratoVirt.patch
Normal file
File diff suppressed because it is too large
Load Diff
184
agent/patches/0018-kata-agent-update-nic-in-guest.patch
Normal file
184
agent/patches/0018-kata-agent-update-nic-in-guest.patch
Normal file
@ -0,0 +1,184 @@
|
|||||||
|
From 629aac1078bdeed63214c58d2110fe672d654774 Mon Sep 17 00:00:00 2001
|
||||||
|
From: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
Date: Mon, 26 Oct 2020 20:39:05 +0800
|
||||||
|
Subject: [PATCH] kata-agent: update nic in guest
|
||||||
|
|
||||||
|
reason: add linkByName and support retry of list ip link, because in some scenarios,
|
||||||
|
the hardware address cannot be queried immediately after the hot plug.
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
network.go | 91 ++++++++++++++++++++++++++++++++++++++++++------------
|
||||||
|
1 file changed, 72 insertions(+), 19 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/network.go b/network.go
|
||||||
|
index f6e2c17..d928e2b 100644
|
||||||
|
--- a/network.go
|
||||||
|
+++ b/network.go
|
||||||
|
@@ -15,13 +15,14 @@ import (
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"syscall"
|
||||||
|
+ "time"
|
||||||
|
|
||||||
|
- "golang.org/x/sys/unix"
|
||||||
|
agentNet "github.com/kata-containers/agent/pkg/net"
|
||||||
|
"github.com/kata-containers/agent/pkg/types"
|
||||||
|
pb "github.com/kata-containers/agent/protocols/grpc"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
"github.com/vishvananda/netlink"
|
||||||
|
+ "golang.org/x/sys/unix"
|
||||||
|
"google.golang.org/grpc/codes"
|
||||||
|
grpcStatus "google.golang.org/grpc/status"
|
||||||
|
)
|
||||||
|
@@ -32,6 +33,7 @@ var (
|
||||||
|
errNoLink = grpcStatus.Errorf(codes.InvalidArgument, "Need network link")
|
||||||
|
errNoMAC = grpcStatus.Errorf(codes.InvalidArgument, "Need hardware address")
|
||||||
|
errNoRoutes = grpcStatus.Errorf(codes.InvalidArgument, "Need network routes")
|
||||||
|
+ errNoName = grpcStatus.Errorf(codes.InvalidArgument, "Need network name")
|
||||||
|
guestDNSFile = "/etc/resolv.conf"
|
||||||
|
kataGuestSandboxDNSFile = "/run/kata-containers/sandbox/resolv.conf"
|
||||||
|
)
|
||||||
|
@@ -46,6 +48,8 @@ const (
|
||||||
|
|
||||||
|
// Use the below address for ipv6 gateway once ipv6 support is added
|
||||||
|
// defaultV6RouteIP = "::"
|
||||||
|
+
|
||||||
|
+ maxLinkRetries = 10
|
||||||
|
)
|
||||||
|
|
||||||
|
// Network fully describes a sandbox network with its interfaces, routes and dns
|
||||||
|
@@ -100,17 +104,36 @@ func linkByHwAddr(netHandle *netlink.Handle, hwAddr string) (netlink.Link, error
|
||||||
|
return nil, grpcStatus.Errorf(codes.NotFound, "Could not find the link corresponding to HwAddr %q", hwAddr)
|
||||||
|
}
|
||||||
|
|
||||||
|
-func updateLink(netHandle *netlink.Handle, link netlink.Link, iface *types.Interface) error {
|
||||||
|
+func linkByName(netHandle *netlink.Handle, name string) (netlink.Link, error) {
|
||||||
|
if netHandle == nil {
|
||||||
|
- return errNoHandle
|
||||||
|
+ return nil, errNoHandle
|
||||||
|
}
|
||||||
|
|
||||||
|
- if link == nil {
|
||||||
|
- return errNoLink
|
||||||
|
+ if name == "" {
|
||||||
|
+ return nil, errNoName
|
||||||
|
}
|
||||||
|
|
||||||
|
- if iface == nil {
|
||||||
|
- return errNoIF
|
||||||
|
+ links, err := netHandle.LinkList()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ for _, link := range links {
|
||||||
|
+ if link == nil {
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if link.Attrs() != nil && link.Attrs().Name == name {
|
||||||
|
+ return link, nil
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil, grpcStatus.Errorf(codes.NotFound, "Could not find the link corresponding to name %s", name)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func updateLinkIP(netHandle *netlink.Handle, link netlink.Link, iface *types.Interface) error {
|
||||||
|
+ if len(iface.IPAddresses) == 0 {
|
||||||
|
+ return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// As a first step, clear out any existing addresses associated with the link:
|
||||||
|
@@ -129,13 +152,6 @@ func updateLink(netHandle *netlink.Handle, link netlink.Link, iface *types.Inter
|
||||||
|
netlinkAddrStr := fmt.Sprintf("%s/%s", addr.Address, addr.Mask)
|
||||||
|
netlinkAddr, err := netlink.ParseAddr(netlinkAddrStr)
|
||||||
|
|
||||||
|
- // With ipv6 addresses, there is a brief period during which the address is marked as "tentative"
|
||||||
|
- // making it unavailable. A process called duplicate address detection(DAD) is performed during this period.
|
||||||
|
- // Disble DAD so that networking is available once the container is up. The assumption is
|
||||||
|
- // that it is the reponsibility of the upper stack to make sure the addresses assigned to containers
|
||||||
|
- // do not conflict. A similar operation is performed by libnetwork:
|
||||||
|
- // https://github.com/moby/moby/issues/18871
|
||||||
|
-
|
||||||
|
if addr.GetFamily() == types.IPFamily_v6 {
|
||||||
|
netlinkAddr.Flags = netlinkAddr.Flags | syscall.IFA_F_NODAD
|
||||||
|
}
|
||||||
|
@@ -150,14 +166,36 @@ func updateLink(netHandle *netlink.Handle, link netlink.Link, iface *types.Inter
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func updateLink(netHandle *netlink.Handle, link netlink.Link, iface *types.Interface) error {
|
||||||
|
+ if netHandle == nil {
|
||||||
|
+ return errNoHandle
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if link == nil {
|
||||||
|
+ return errNoLink
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if iface == nil {
|
||||||
|
+ return errNoIF
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := updateLinkIP(netHandle, link, iface); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
// set the interface name:
|
||||||
|
if err := netHandle.LinkSetName(link, iface.Name); err != nil {
|
||||||
|
return grpcStatus.Errorf(codes.Internal, "Could not set name %s for interface %v: %v", iface.Name, link, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// set the interface MTU:
|
||||||
|
- if err := netHandle.LinkSetMTU(link, int(iface.Mtu)); err != nil {
|
||||||
|
- return grpcStatus.Errorf(codes.Internal, "Could not set MTU %d for interface %v: %v", iface.Mtu, link, err)
|
||||||
|
+ if iface.Mtu > 0 {
|
||||||
|
+ if err := netHandle.LinkSetMTU(link, int(iface.Mtu)); err != nil {
|
||||||
|
+ return grpcStatus.Errorf(codes.Internal, "Could not set MTU %d for interface %v: %v", iface.Mtu, link, err)
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
|
||||||
|
if iface.RawFlags&unix.IFF_NOARP == uint32(unix.IFF_NOARP) {
|
||||||
|
@@ -306,8 +344,23 @@ func (s *sandbox) updateInterface(netHandle *netlink.Handle, iface *types.Interf
|
||||||
|
if iface.HwAddr != "" {
|
||||||
|
fieldLogger.Info("Getting interface from MAC address")
|
||||||
|
|
||||||
|
- // Find the interface link from its hardware address.
|
||||||
|
- link, err = linkByHwAddr(netHandle, iface.HwAddr)
|
||||||
|
+ // In some scenarios, the hardware address cannot be queried immediately
|
||||||
|
+ // after the hot plug, add a retry here.
|
||||||
|
+ for retry := 0; retry < maxLinkRetries; retry++ {
|
||||||
|
+ // Find the interface link from its hardware address.
|
||||||
|
+ link, err = linkByHwAddr(netHandle, iface.HwAddr)
|
||||||
|
+ if err != nil {
|
||||||
|
+ time.Sleep(100 * time.Millisecond)
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, grpcStatus.Errorf(codes.Internal, "updateInterface: %v", err)
|
||||||
|
+ }
|
||||||
|
+ } else if iface.Name != "" {
|
||||||
|
+ fieldLogger.Info("Getting interface from name")
|
||||||
|
+ link, err = linkByName(netHandle, iface.Name)
|
||||||
|
if err != nil {
|
||||||
|
return nil, grpcStatus.Errorf(codes.Internal, "updateInterface: %v", err)
|
||||||
|
}
|
||||||
|
@@ -487,7 +540,7 @@ func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Ro
|
||||||
|
}()
|
||||||
|
|
||||||
|
var (
|
||||||
|
- added []*types.Route
|
||||||
|
+ added []*types.Route
|
||||||
|
removed []*types.Route
|
||||||
|
)
|
||||||
|
|
||||||
|
--
|
||||||
|
2.23.0
|
||||||
|
|
||||||
18
agent/series.conf
Normal file
18
agent/series.conf
Normal file
@ -0,0 +1,18 @@
|
|||||||
|
0001-agent-add-agent.netlink_recv_buf_size-flag-to-set-ne.patch
|
||||||
|
0002-network-support-update-routes-incrementally.patch
|
||||||
|
0003-kata-agent-add-kata-ipvs-command.patch
|
||||||
|
0004-agent-add-IPVS-test.patch
|
||||||
|
0005-mount-support-mount-block-device.patch
|
||||||
|
0006-agent-make-workaround-for-slow-response-in-aarch64.patch
|
||||||
|
0007-agent-using-pcie-root-port-driver-to-hotplug-device.patch
|
||||||
|
0008-agent-support-get-root-bus-path-dynamically.patch
|
||||||
|
0009-storage-add-pkg-storage-for-mount.patch
|
||||||
|
0010-storage-mount-nfs-and-gpath-in-agent.patch
|
||||||
|
0011-agent-fix-agent-reap-agent-process-blocked-problem.patch
|
||||||
|
0012-network-support-set-dns-without-nameserver.patch
|
||||||
|
0013-agent-support-setting-multi-queues-of-interface.patch
|
||||||
|
0014-agent-fix-init-hugepages-failed-problem.patch
|
||||||
|
0015-agent-add-support-of-getting-container-s-network-sta.patch
|
||||||
|
0016-clock-synchronizes-clock-info-with-proxy.patch
|
||||||
|
0017-agent-add-support-of-new-sandbox-StratoVirt.patch
|
||||||
|
0018-kata-agent-update-nic-in-guest.patch
|
||||||
133
kata-containers.spec
Normal file
133
kata-containers.spec
Normal file
@ -0,0 +1,133 @@
|
|||||||
|
#needsrootforbuild
|
||||||
|
%global debug_package %{nil}
|
||||||
|
|
||||||
|
%define VERSION v1.11.1
|
||||||
|
%define RELEASE 7
|
||||||
|
|
||||||
|
Name: kata-containers
|
||||||
|
Version: %{VERSION}
|
||||||
|
Release: %{RELEASE}
|
||||||
|
Summary: Kata Container, the speed of containers, the security of VMs
|
||||||
|
License: Apache 2.0
|
||||||
|
URL: https://github.com/kata-containers
|
||||||
|
Source0: kata_integration-v1.0.0.tar.gz
|
||||||
|
Source1: kata-containers-%{version}.tar.gz
|
||||||
|
Source2: kernel.tar.gz
|
||||||
|
|
||||||
|
BuildRoot: %_topdir/BUILDROOT
|
||||||
|
BuildRequires: automake golang gcc bc glibc-devel glibc-static busybox glib2-devel glib2 ipvsadm conntrack-tools nfs-utils
|
||||||
|
BuildRequires: patch elfutils-libelf-devel openssl-devel bison flex
|
||||||
|
|
||||||
|
%description
|
||||||
|
This is core component of Kata Container, to make it work, you need a isulad/docker engine.
|
||||||
|
|
||||||
|
%prep
|
||||||
|
%setup -T -c -a 0 -n kata_integration
|
||||||
|
%setup -T -c -a 1 -n kata-containers-%{version}
|
||||||
|
%setup -T -c -a 2 -n kernel
|
||||||
|
|
||||||
|
# extract the kata_integration.tar.gz file
|
||||||
|
cd %{_builddir}/kata_integration
|
||||||
|
# apply kata_integration patches
|
||||||
|
sh apply-patches
|
||||||
|
|
||||||
|
# mv build components into kata_integration dir
|
||||||
|
pushd %{_builddir}/kata_integration
|
||||||
|
mv ../kata-containers-%{version}/runtime .
|
||||||
|
mv ../kata-containers-%{version}/agent .
|
||||||
|
mv ../kata-containers-%{version}/proxy .
|
||||||
|
mv ../kata-containers-%{version}/shim .
|
||||||
|
popd
|
||||||
|
|
||||||
|
# build kernel
|
||||||
|
cd %{_builddir}/kernel
|
||||||
|
mv kernel linux
|
||||||
|
cd %{_builddir}/kernel/linux/
|
||||||
|
%ifarch %{ix86} x86_64
|
||||||
|
cp %{_builddir}/kata_integration/hack/config-kata-x86_64 ./.config
|
||||||
|
%else
|
||||||
|
cp %{_builddir}/kata_integration/hack/config-kata-arm64 ./.config
|
||||||
|
%endif
|
||||||
|
|
||||||
|
%build
|
||||||
|
cd %{_builddir}/kernel/linux/
|
||||||
|
make %{?_smp_mflags}
|
||||||
|
|
||||||
|
cd %{_builddir}/kata_integration
|
||||||
|
mkdir -p -m 750 build
|
||||||
|
make runtime
|
||||||
|
make proxy
|
||||||
|
make shim
|
||||||
|
make initrd
|
||||||
|
|
||||||
|
%install
|
||||||
|
mkdir -p -m 755 %{buildroot}/var/lib/kata
|
||||||
|
%ifarch %{ix86} x86_64
|
||||||
|
install -p -m 755 -D %{_builddir}/kernel/linux/arch/x86_64/boot/bzImage %{buildroot}/var/lib/kata/kernel
|
||||||
|
%else
|
||||||
|
install -p -m 755 -D %{_builddir}/kernel/linux/arch/arm64/boot/Image %{buildroot}/var/lib/kata/kernel
|
||||||
|
%endif
|
||||||
|
|
||||||
|
cd %{_builddir}/kata_integration
|
||||||
|
mkdir -p -m 750 %{buildroot}/usr/bin
|
||||||
|
install -p -m 750 ./build/kata-runtime ./build/kata-proxy ./build/kata-shim ./build/kata-netmon %{buildroot}/usr/bin/
|
||||||
|
install -p -m 640 ./build/kata-containers-initrd.img %{buildroot}/var/lib/kata/
|
||||||
|
mkdir -p -m 750 %{buildroot}/usr/share/defaults/kata-containers/
|
||||||
|
install -p -m 640 -D ./runtime/cli/config/configuration-qemu.toml %{buildroot}/usr/share/defaults/kata-containers/configuration.toml
|
||||||
|
|
||||||
|
%clean
|
||||||
|
|
||||||
|
%files
|
||||||
|
/usr/bin/kata-runtime
|
||||||
|
/usr/bin/kata-proxy
|
||||||
|
/usr/bin/kata-shim
|
||||||
|
/usr/bin/kata-netmon
|
||||||
|
/var/lib/kata/kernel
|
||||||
|
/var/lib/kata/kata-containers-initrd.img
|
||||||
|
%config(noreplace) /usr/share/defaults/kata-containers/configuration.toml
|
||||||
|
|
||||||
|
%doc
|
||||||
|
|
||||||
|
|
||||||
|
%changelog
|
||||||
|
* Tue Dec 22 2020 jiangpengfei<jiangpengfei9@huawei.com> - 1.11.1-7
|
||||||
|
- Type:enhancement
|
||||||
|
- ID:NA
|
||||||
|
- SUG:update
|
||||||
|
- DESC:update kata-containers source forms of organization to move all kata-containers related source repo into one repo kata-containers
|
||||||
|
|
||||||
|
* Fri Nov 6 2020 yangfeiyu<yangfeiyu2@huawei.com> - 1.11.1-6
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:revert the kata-containers.spec to still build kata-containers components into one package
|
||||||
|
|
||||||
|
* Fri Oct 9 2020 yangfeiyu<yangfeiyu2@huawei.com> - 1.11.1-5
|
||||||
|
- Type:enhancement
|
||||||
|
- ID:NA
|
||||||
|
- SUG:restart
|
||||||
|
- DESC:directly copy kata binary files instead of building them
|
||||||
|
|
||||||
|
* Wed Sep 30 2020 yangfeiyu<yangfeiyu2@huawei.com> - 1.11.1-4
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:restart
|
||||||
|
- DESC:kata-runtime retry inserting of CNI interface
|
||||||
|
|
||||||
|
* Sun Sep 27 2020 LiangZhang<zhangliang5@Huawei.com> - 1.11.1-3
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:fix cmd params of direct use stratovirt binary
|
||||||
|
|
||||||
|
* Thu Sep 20 2020 jiangpengf<jiangpengfei9@huawei.com> - 1.11.1-2
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:fix del-iface doesn't delete the tap interface in the host problem
|
||||||
|
|
||||||
|
* Thu Aug 27 2020 jiangpengf<jiangpengfei9@huawei.com> - 1.11.1-1
|
||||||
|
- Type:enhancement
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:update kata-containers version to v1.11.1-1
|
||||||
4
kata-containers.yaml
Normal file
4
kata-containers.yaml
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
version_control: github
|
||||||
|
src_repo: kata-containers/runtime
|
||||||
|
tag_prefix:
|
||||||
|
seperator: "."
|
||||||
20
proxy/apply-patches
Executable file
20
proxy/apply-patches
Executable file
@ -0,0 +1,20 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
if [[ -f ./patch_flag ]];then
|
||||||
|
echo "proxy patched!"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
tar -zxvf proxy-1.11.1.tar.gz
|
||||||
|
cp -fr ./proxy-1.11.1/* ./
|
||||||
|
rm -rf ./proxy-1.11.1
|
||||||
|
cat ./series.conf | while read line
|
||||||
|
do
|
||||||
|
if [[ $line == '' || $line =~ ^\s*# ]]; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
echo "====patch $line======"
|
||||||
|
pwd
|
||||||
|
patch -p1 -F1 -s < ./patches/$line
|
||||||
|
done
|
||||||
|
touch ./patch_flag
|
||||||
403
proxy/patches/0001-clock-synchronizes-clock-info-to-agent.patch
Normal file
403
proxy/patches/0001-clock-synchronizes-clock-info-to-agent.patch
Normal file
@ -0,0 +1,403 @@
|
|||||||
|
From a260bbe394f91fa05b163315390ed133de5c5494 Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Wed, 19 Aug 2020 17:43:24 +0800
|
||||||
|
Subject: [PATCH] clock: synchronizes clock info to agent
|
||||||
|
|
||||||
|
reason: virtual machine's clock may be incorrect, proxy synchronizes
|
||||||
|
clock info to help virtual machine adjust clock time
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
proxy.go | 7 ++
|
||||||
|
proxy_test.go | 29 +++++
|
||||||
|
sync_clock_client.go | 107 +++++++++++++++++
|
||||||
|
sync_clock_client_test.go | 132 +++++++++++++++++++++
|
||||||
|
.../kata-containers/agent/pkg/clock/clock_util.go | 44 +++++++
|
||||||
|
5 files changed, 319 insertions(+)
|
||||||
|
create mode 100644 sync_clock_client.go
|
||||||
|
create mode 100644 sync_clock_client_test.go
|
||||||
|
create mode 100644 vendor/github.com/kata-containers/agent/pkg/clock/clock_util.go
|
||||||
|
|
||||||
|
diff --git a/proxy.go b/proxy.go
|
||||||
|
index ab062a5..9dfcb3c 100644
|
||||||
|
--- a/proxy.go
|
||||||
|
+++ b/proxy.go
|
||||||
|
@@ -105,6 +105,13 @@ func serve(servConn io.ReadWriteCloser, proto, addr string, results chan error)
|
||||||
|
// Start the heartbeat in a separate go routine
|
||||||
|
go heartBeat(session)
|
||||||
|
|
||||||
|
+ // start the sync clock in a separate go routine
|
||||||
|
+ syncClockStream, err := session.Open()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, nil, err
|
||||||
|
+ }
|
||||||
|
+ go SyncClock(syncClockStream)
|
||||||
|
+
|
||||||
|
// serving connection
|
||||||
|
l, err := net.Listen(proto, addr)
|
||||||
|
if err != nil {
|
||||||
|
diff --git a/proxy_test.go b/proxy_test.go
|
||||||
|
index 923b138..94fa523 100644
|
||||||
|
--- a/proxy_test.go
|
||||||
|
+++ b/proxy_test.go
|
||||||
|
@@ -10,6 +10,7 @@ package main
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"crypto/md5"
|
||||||
|
+ "encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"io/ioutil"
|
||||||
|
@@ -23,6 +24,7 @@ import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/hashicorp/yamux"
|
||||||
|
+ "github.com/kata-containers/agent/pkg/clock"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
@@ -121,6 +123,13 @@ func server(listener net.Listener, closeCh chan bool) error {
|
||||||
|
session.Close()
|
||||||
|
}()
|
||||||
|
|
||||||
|
+ // accept the sync clock stream first
|
||||||
|
+ if syncClockStream, err := session.Accept(); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ } else {
|
||||||
|
+ go serverSyncClock(syncClockStream)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
for {
|
||||||
|
stream, err := session.Accept()
|
||||||
|
if err != nil {
|
||||||
|
@@ -133,6 +142,26 @@ func server(listener net.Listener, closeCh chan bool) error {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
+func serverSyncClock(stream net.Conn) {
|
||||||
|
+ for {
|
||||||
|
+ buf, byteNum, err := readConnData(stream)
|
||||||
|
+ if err != nil {
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+ var clockInfo clock.TimeValue
|
||||||
|
+ if err := json.Unmarshal(buf[:byteNum], &clockInfo); err != nil {
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+ nowTime := clock.GetCurrentTimeNs()
|
||||||
|
+ clockInfo.ClientArriveTime = nowTime
|
||||||
|
+ clockInfo.ServerSendTime = nowTime
|
||||||
|
+ b, _ := json.Marshal(clockInfo)
|
||||||
|
+ if err := clock.WriteConnData(stream, b); err != nil {
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func TestUnixAddrParsing(T *testing.T) {
|
||||||
|
buf := "unix://foo/bar"
|
||||||
|
addr, err := unixAddr(buf)
|
||||||
|
diff --git a/sync_clock_client.go b/sync_clock_client.go
|
||||||
|
new file mode 100644
|
||||||
|
index 0000000..9bf3e91
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/sync_clock_client.go
|
||||||
|
@@ -0,0 +1,107 @@
|
||||||
|
+// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved.
|
||||||
|
+// SPDX-License-Identifier: Apache-2.0
|
||||||
|
+// Description: sync clock client related function
|
||||||
|
+// Author: xueshaojia x00464843
|
||||||
|
+// Create: 2018-11-10
|
||||||
|
+
|
||||||
|
+package main
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "encoding/json"
|
||||||
|
+ "fmt"
|
||||||
|
+ "net"
|
||||||
|
+ "time"
|
||||||
|
+
|
||||||
|
+ "github.com/kata-containers/agent/pkg/clock"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+const (
|
||||||
|
+ allowTimeDiff = 10 * 1000 * 1000 // allow 10ms's difference
|
||||||
|
+ syncClockInterval = 60 * time.Second // sync clock with agent every 60 seconds
|
||||||
|
+ rpcTimeout = 10 * time.Second // timeout for proxy's reading data
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+// readConnData reads data from stream
|
||||||
|
+func readConnData(stream net.Conn) (buf []byte, byteNum int, err error) {
|
||||||
|
+ // set read deadline to avoid case as following:
|
||||||
|
+ // proxy and agent are both reading and then syncClock will never work
|
||||||
|
+ stream.SetReadDeadline(time.Now().Add(rpcTimeout))
|
||||||
|
+ buf = make([]byte, clock.MaxSyncClockByteNum)
|
||||||
|
+ byteNum, err = stream.Read(buf)
|
||||||
|
+ return buf, byteNum, err
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// getGuestClock syncs guest clock info
|
||||||
|
+// sends ClientSendTime
|
||||||
|
+// waits for ClientArriveTime and ServerSendTime
|
||||||
|
+func getGuestClock(stream net.Conn, clockInfo *clock.TimeValue) error {
|
||||||
|
+ clockInfo.Delta = 0
|
||||||
|
+ b, err := json.Marshal(clockInfo)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ if err = clock.WriteConnData(stream, b); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ buf, byteNum, err := readConnData(stream)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err = json.Unmarshal(buf[:byteNum], clockInfo); err != nil {
|
||||||
|
+ return fmt.Errorf("sync clock, parse guest clocktime error:%v", err)
|
||||||
|
+ }
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// adjustGuestClock tells server to ajust local clock with Delta
|
||||||
|
+func adjustGuestClock(stream net.Conn, clockInfo *clock.TimeValue) error {
|
||||||
|
+ b, err := json.Marshal(clockInfo)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ logger().Debugf("sync clock, send:%s", string(b))
|
||||||
|
+ return clock.WriteConnData(stream, b)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// syncClock performs all the steps
|
||||||
|
+// 1 get client send time[host]
|
||||||
|
+// 2 request for client arrive time and server send time[guest os]
|
||||||
|
+// 3 get server arrive time[host]
|
||||||
|
+// 4 calculate clock diff
|
||||||
|
+// 5 request to adjust guest clock
|
||||||
|
+func syncClock(stream net.Conn) error {
|
||||||
|
+ var clockInfo clock.TimeValue
|
||||||
|
+ if clockInfo.ClientSendTime = clock.GetCurrentTimeNs(); clockInfo.ClientSendTime <= 0 {
|
||||||
|
+ return fmt.Errorf("sync clock, get client sendtime error")
|
||||||
|
+ }
|
||||||
|
+ err := getGuestClock(stream, &clockInfo)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return fmt.Errorf("sync clock, get guest clocktime error:%v", err)
|
||||||
|
+ }
|
||||||
|
+ if clockInfo.ServerArriveTime = clock.GetCurrentTimeNs(); clockInfo.ServerArriveTime <= 0 {
|
||||||
|
+ return fmt.Errorf("sync clock, get client recvtime error")
|
||||||
|
+ }
|
||||||
|
+ if clockInfo.ClientSendTime <= 0 || clockInfo.ClientArriveTime <= 0 || clockInfo.ServerSendTime <= 0 {
|
||||||
|
+ return fmt.Errorf("sync clock, some fields of NTP message error, raw message:%v", clockInfo)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ delta := ((clockInfo.ClientSendTime - clockInfo.ClientArriveTime) + (clockInfo.ServerArriveTime - clockInfo.ServerSendTime)) / 2
|
||||||
|
+ if delta < -allowTimeDiff || delta > allowTimeDiff {
|
||||||
|
+ clockInfo.Delta = delta
|
||||||
|
+ if err := adjustGuestClock(stream, &clockInfo); err != nil {
|
||||||
|
+ return fmt.Errorf("sync clock, failed to adjust guest clock : %v", err)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func SyncClock(stream net.Conn) {
|
||||||
|
+ for {
|
||||||
|
+ if err := syncClock(stream); err != nil {
|
||||||
|
+ logger().WithError(err).Error("sync clock failed")
|
||||||
|
+ }
|
||||||
|
+ time.Sleep(syncClockInterval)
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
diff --git a/sync_clock_client_test.go b/sync_clock_client_test.go
|
||||||
|
new file mode 100644
|
||||||
|
index 0000000..b0b1c85
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/sync_clock_client_test.go
|
||||||
|
@@ -0,0 +1,132 @@
|
||||||
|
+// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved.
|
||||||
|
+// SPDX-License-Identifier: Apache-2.0
|
||||||
|
+// Description: sync clock client related test
|
||||||
|
+// Author: xueshaojia x00464843
|
||||||
|
+// Create: 2018-11-10
|
||||||
|
+
|
||||||
|
+package main
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "encoding/json"
|
||||||
|
+ "fmt"
|
||||||
|
+ "math/rand"
|
||||||
|
+ "net"
|
||||||
|
+ "os"
|
||||||
|
+ "testing"
|
||||||
|
+
|
||||||
|
+ "github.com/hashicorp/yamux"
|
||||||
|
+ "github.com/kata-containers/agent/pkg/clock"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+func TestReadWriteData(t *testing.T) {
|
||||||
|
+ var clockInfo clock.TimeValue
|
||||||
|
+ clockInfo.ClientSendTime = clock.GetCurrentTimeNs()
|
||||||
|
+ b, err := json.Marshal(clockInfo)
|
||||||
|
+ if err != nil {
|
||||||
|
+ t.Fatalf("Marshal clock info fail, err:%v", err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ err = clock.WriteConnData(clientStream, b)
|
||||||
|
+ fmt.Printf("client send: %s\n", string(b))
|
||||||
|
+ if err != nil {
|
||||||
|
+ t.Fatalf("send clock info fail, err:%v", err)
|
||||||
|
+ }
|
||||||
|
+ _, _, err = readConnData(clientStream)
|
||||||
|
+ if err != nil {
|
||||||
|
+ t.Fatalf("recv clock info fail, err:%v", err)
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func SetUpServer(sock string, readyChan chan int) error {
|
||||||
|
+ var err error
|
||||||
|
+ listener, err = net.Listen("unix", sock)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ readyChan <- 1
|
||||||
|
+ conn, err := listener.Accept()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ session, err := yamux.Server(conn, nil)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ serverSession = session
|
||||||
|
+ stream, err := session.Accept()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ for {
|
||||||
|
+ var clockInfo clock.TimeValue
|
||||||
|
+ var byteNum int
|
||||||
|
+ var err error
|
||||||
|
+ buf := make([]byte, 400)
|
||||||
|
+ if byteNum, err = stream.Read(buf); err != nil {
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err = json.Unmarshal(buf[:byteNum], &clockInfo); err != nil {
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+ if clockInfo.Delta == 0 {
|
||||||
|
+ nowTime := clock.GetCurrentTimeNs()
|
||||||
|
+ clockInfo.ClientArriveTime = nowTime
|
||||||
|
+ clockInfo.ClientArriveTime = nowTime
|
||||||
|
+ b, _ := json.Marshal(&clockInfo)
|
||||||
|
+ stream.Write(b)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func SetUpClient(sock string) error {
|
||||||
|
+ conn, err := net.Dial("unix", sock)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ session, err := yamux.Client(conn, nil)
|
||||||
|
+ if err != nil {
|
||||||
|
+ conn.Close()
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ clientSession = session
|
||||||
|
+ stream, err := session.Open()
|
||||||
|
+ if err != nil {
|
||||||
|
+ clientSession.Close()
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ clientStream = stream
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TearDown() {
|
||||||
|
+ listener.Close()
|
||||||
|
+ serverSession.Close()
|
||||||
|
+ clientSession.Close()
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func GenSocket() string {
|
||||||
|
+ randSeed := clock.GetCurrentTimeNs()
|
||||||
|
+ rand.Seed(randSeed)
|
||||||
|
+ return fmt.Sprintf("/tmp/%d.sock", rand.Uint32())
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+var listener net.Listener
|
||||||
|
+var clientStream net.Conn
|
||||||
|
+var serverSession *yamux.Session
|
||||||
|
+var clientSession *yamux.Session
|
||||||
|
+
|
||||||
|
+func TestMain(m *testing.M) {
|
||||||
|
+ waitConn := make(chan int)
|
||||||
|
+ testSock := GenSocket()
|
||||||
|
+ go SetUpServer(testSock, waitConn)
|
||||||
|
+ <-waitConn
|
||||||
|
+ if err := SetUpClient(testSock); err != nil {
|
||||||
|
+ listener.Close()
|
||||||
|
+ serverSession.Close()
|
||||||
|
+ os.Exit(1)
|
||||||
|
+ }
|
||||||
|
+ m.Run()
|
||||||
|
+ TearDown()
|
||||||
|
+}
|
||||||
|
diff --git a/vendor/github.com/kata-containers/agent/pkg/clock/clock_util.go b/vendor/github.com/kata-containers/agent/pkg/clock/clock_util.go
|
||||||
|
new file mode 100644
|
||||||
|
index 0000000..03244fd
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/vendor/github.com/kata-containers/agent/pkg/clock/clock_util.go
|
||||||
|
@@ -0,0 +1,44 @@
|
||||||
|
+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
||||||
|
+// SPDX-License-Identifier: Apache-2.0
|
||||||
|
+// Description: common functions
|
||||||
|
+// Author: jiangpeifei
|
||||||
|
+// Create: 2019-05-28
|
||||||
|
+
|
||||||
|
+package clock
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "net"
|
||||||
|
+ "syscall"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+type TimeValue struct {
|
||||||
|
+ ClientSendTime int64 `json:"client_send_time"`
|
||||||
|
+ ClientArriveTime int64 `json:"client_arrive_time"`
|
||||||
|
+ ServerSendTime int64 `json:"server_send_time"`
|
||||||
|
+ ServerArriveTime int64 `json:"server_arrive_time"`
|
||||||
|
+ Delta int64 `json:"delta"`
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+const MaxSyncClockByteNum = 400 //sync clock byte num, max=400
|
||||||
|
+
|
||||||
|
+// getCurrentTimeNs returns UTC time in Ns
|
||||||
|
+func GetCurrentTimeNs() int64 {
|
||||||
|
+ var tv syscall.Timeval
|
||||||
|
+ if err := syscall.Gettimeofday(&tv); err != nil {
|
||||||
|
+ return -1
|
||||||
|
+ }
|
||||||
|
+ return tv.Sec*1000000000 + tv.Usec*1000
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// readConnData reads data from stream
|
||||||
|
+func ReadConnData(stream net.Conn) (buf []byte, byteNum int, err error) {
|
||||||
|
+ buf = make([]byte, MaxSyncClockByteNum)
|
||||||
|
+ byteNum, err = stream.Read(buf)
|
||||||
|
+ return buf, byteNum, err
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// writeConnData writes data to stream
|
||||||
|
+func WriteConnData(stream net.Conn, buf []byte) error {
|
||||||
|
+ _, err := stream.Write(buf)
|
||||||
|
+ return err
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
BIN
proxy/proxy-1.11.1.tar.gz
Normal file
BIN
proxy/proxy-1.11.1.tar.gz
Normal file
Binary file not shown.
1
proxy/series.conf
Normal file
1
proxy/series.conf
Normal file
@ -0,0 +1 @@
|
|||||||
|
0001-clock-synchronizes-clock-info-to-agent.patch
|
||||||
22
runtime/apply-patches
Executable file
22
runtime/apply-patches
Executable file
@ -0,0 +1,22 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
if [[ -f ./patch_flag ]];then
|
||||||
|
echo "runtime patched!"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
tar -zxvf runtime-1.11.1.tar.gz
|
||||||
|
cp -fr ./runtime-1.11.1/* ./
|
||||||
|
rm -rf ./runtime-1.11.1
|
||||||
|
cat ./series.conf | while read line
|
||||||
|
do
|
||||||
|
if [[ $line == '' || $line =~ ^\s*# ]]; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
echo "====patch $line======"
|
||||||
|
patch -p1 -F1 -s < ./patches/$line
|
||||||
|
done
|
||||||
|
|
||||||
|
touch ./patch_flag
|
||||||
165
runtime/kata-runtime.spec
Normal file
165
runtime/kata-runtime.spec
Normal file
@ -0,0 +1,165 @@
|
|||||||
|
%define debug_package %{nil}
|
||||||
|
|
||||||
|
%define VERSION 1.11.1
|
||||||
|
%define RELEASE 11
|
||||||
|
|
||||||
|
Name: kata-runtime
|
||||||
|
Version: %{VERSION}
|
||||||
|
Release: %{RELEASE}
|
||||||
|
Summary: Kata Runtime
|
||||||
|
License: Apache 2.0
|
||||||
|
URL: https://github.com/kata-containers/runtime
|
||||||
|
Source0: https://github.com/kata-containers/runtime/archive/%{version}.tar.gz#/%{name}-v%{version}.tar.gz
|
||||||
|
|
||||||
|
BuildRoot: %_topdir/BUILDROOT
|
||||||
|
BuildRequires: automake golang gcc
|
||||||
|
|
||||||
|
%description
|
||||||
|
Kata-runtime is core component of Kata Container.
|
||||||
|
|
||||||
|
%prep
|
||||||
|
%setup -q -c -a 0 -n %{name}-%{version}
|
||||||
|
|
||||||
|
%build
|
||||||
|
cd %{_builddir}/%{name}-%{version}
|
||||||
|
|
||||||
|
set -e
|
||||||
|
# apply patches read from series.conf
|
||||||
|
sh apply-patches
|
||||||
|
|
||||||
|
# create tmp GOPATH dir to build kata-runtime
|
||||||
|
rm -rf /tmp/kata-build/
|
||||||
|
mkdir -p /tmp/kata-build/
|
||||||
|
GOPATH=/tmp/kata-build/
|
||||||
|
kata_base=$GOPATH/src/github.com/kata-containers
|
||||||
|
mkdir -p $kata_base
|
||||||
|
|
||||||
|
# get current kata-runtime absolute path
|
||||||
|
kata_runtime_path=$(readlink -f .)
|
||||||
|
ln -s $kata_runtime_path $kata_base/runtime
|
||||||
|
|
||||||
|
# export GOPATH env
|
||||||
|
export GOPATH=$(readlink -f $GOPATH)
|
||||||
|
cd ${kata_base}/runtime && make clean && make
|
||||||
|
rm -rf $GOPATH
|
||||||
|
|
||||||
|
# make kata-runtime default configuration
|
||||||
|
kata_config_path=$kata_runtime_path/cli/config/configuration-qemu.toml
|
||||||
|
ARCH=`arch`
|
||||||
|
|
||||||
|
# arch related config options
|
||||||
|
if [ "$ARCH" == "aarch64" ];then
|
||||||
|
sed -i 's/^machine_type.*$/machine_type = \"virt\"/' $kata_config_path
|
||||||
|
sed -i 's/^block_device_driver.*$/block_device_driver = \"virtio-scsi\"/' $kata_config_path
|
||||||
|
sed -i 's/^kernel_params.*$/kernel_params = \"pcie_ports=native pci=pcie_bus_perf agent.netlink_recv_buf_size=2MB\"/' $kata_config_path
|
||||||
|
sed -i 's/^hypervisor_params.*$/hypervisor_params = \"kvm-pit.lost_tick_policy=discard pcie-root-port.fast-plug=1 pcie-root-port.x-speed=16 pcie-root-port.x-width=32 pcie-root-port.fast-unplug=1\"/' $kata_config_path
|
||||||
|
sed -i 's/^#pcie_root_port.*$/pcie_root_port = 25/' $kata_config_path
|
||||||
|
else
|
||||||
|
sed -i 's#block_device_driver = \"virtio-scsi\"#block_device_driver = \"virtio-blk\"#' $kata_config_path
|
||||||
|
sed -i 's/^#hotplug_vfio_on_root_bus/hotplug_vfio_on_root_bus/' $kata_config_path
|
||||||
|
fi
|
||||||
|
|
||||||
|
# debug config
|
||||||
|
sed -i 's/^#enable_debug.*$/enable_debug = true/' $kata_config_path
|
||||||
|
|
||||||
|
# other config
|
||||||
|
sed -i 's#"/usr/bin/qemu.*"$#"/usr/bin/qemu-kvm"#' $kata_config_path
|
||||||
|
sed -i 's#/usr/share/kata-containers/vmlinuz\.container#/var/lib/kata/kernel#' $kata_config_path
|
||||||
|
sed -i 's#/usr/share/kata-containers/kata-containers-initrd\.img#/var/lib/kata/kata-containers-initrd\.img#' $kata_config_path
|
||||||
|
sed -i 's/^image/#image/' $kata_config_path
|
||||||
|
sed -i 's/^default_memory.*$/default_memory = 1024/' $kata_config_path
|
||||||
|
sed -i 's/^#enable_blk_mount/enable_blk_mount/' $kata_config_path
|
||||||
|
sed -i 's/^#block_device_cache_direct.*$/block_device_cache_direct = true/' $kata_config_path
|
||||||
|
sed -i 's/^#block_device_cache_set.*$/block_device_cache_set = true/' $kata_config_path
|
||||||
|
sed -i 's#/usr/libexec/kata-containers/kata-proxy#/usr/bin/kata-proxy#' $kata_config_path
|
||||||
|
sed -i 's#/usr/libexec/kata-containers/kata-shim#/usr/bin/kata-shim#' $kata_config_path
|
||||||
|
sed -i 's#/usr/libexec/kata-containers/kata-netmon#/usr/bin/kata-netmon#' $kata_config_path
|
||||||
|
sed -i 's/^#disable_new_netns.*$/disable_new_netns = true/' $kata_config_path
|
||||||
|
sed -i 's/^#disable_vhost_net.*$/disable_vhost_net = true/' $kata_config_path
|
||||||
|
sed -i 's/^internetworking_model.*$/internetworking_model=\"none\"/' $kata_config_path
|
||||||
|
sed -i 's/^enable_compat_old_cni.*$/#enable_compat_old_cni = true/' $kata_config_path
|
||||||
|
sed -i 's/^sandbox_cgroup_only.*$/sandbox_cgroup_only = true/' $kata_config_path
|
||||||
|
|
||||||
|
set +e
|
||||||
|
|
||||||
|
%install
|
||||||
|
cd %{_builddir}/%{name}-%{version}
|
||||||
|
mkdir -p -m 750 %{buildroot}/usr/bin
|
||||||
|
install -p -m 750 ./kata-runtime %{buildroot}/usr/bin
|
||||||
|
install -p -m 750 ./kata-netmon %{buildroot}/usr/bin
|
||||||
|
mkdir -p -m 750 %{buildroot}/usr/share/defaults/kata-containers
|
||||||
|
install -p -m 640 ./cli/config/configuration-qemu.toml %{buildroot}/usr/share/defaults/kata-containers/configuration.toml
|
||||||
|
|
||||||
|
%clean
|
||||||
|
|
||||||
|
%files
|
||||||
|
/usr/bin/kata-runtime
|
||||||
|
/usr/bin/kata-netmon
|
||||||
|
/usr/share/defaults/kata-containers/configuration.toml
|
||||||
|
|
||||||
|
%changelog
|
||||||
|
* Tue Nov 17 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-11
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:upgrade
|
||||||
|
- DESC:fix cpu resource limited problem when sandox_cgroup_with_emulator config is enabled
|
||||||
|
|
||||||
|
* Fri Oct 9 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-10
|
||||||
|
- Type:feature
|
||||||
|
- ID:NA
|
||||||
|
- SUG:restart
|
||||||
|
- DESC:support using CNI plugin to insert mutiple network interfaces at the same time
|
||||||
|
|
||||||
|
* Mon Sep 28 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-9
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:restart
|
||||||
|
- DESC:retry inserting of CNI interface when netmon is enable
|
||||||
|
|
||||||
|
* Sun Sep 27 2020 LiangZhang<zhangliang5@Huawei.com> - 1.11.1-8
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:fix cmd params of direct use stratovirt binary
|
||||||
|
|
||||||
|
* Thu Sep 24 2020 LiangZhang<zhangliang5@Huawei.com> - 1.11.1-7
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:fix invalid cmdline when start sandbox stratovirt
|
||||||
|
|
||||||
|
* Mon Sep 21 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-6
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:fix sandboxRuntimeRootPath left problem
|
||||||
|
|
||||||
|
* Mon Sep 21 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-5
|
||||||
|
- Type:enhancement
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:add support for host cgroups with emulator
|
||||||
|
|
||||||
|
* Mon Sep 21 2020 LiangZhang<zhangliang5@Huawei.com> - 1.11.1-4
|
||||||
|
- Type:enhancement
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:add support of new sandbox StratoVirt
|
||||||
|
|
||||||
|
* Sat Sep 19 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-3
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:fix del-iface doesn't delete the tap interface in the host problem
|
||||||
|
|
||||||
|
* Sat Sep 5 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-2
|
||||||
|
- Type:enhancement
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:use URL format for Source0
|
||||||
|
|
||||||
|
* Wed Aug 26 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-1
|
||||||
|
- Type:enhancement
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:modify kata-runtime spec file to build seperately
|
||||||
@ -0,0 +1,44 @@
|
|||||||
|
From 73fe7242d18a10a86bc216ec5e33a10a8751f85f Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Fri, 24 Jul 2020 22:22:00 +0800
|
||||||
|
Subject: [PATCH 01/50] qmp: fix kata-runtime hungs when qemu process is D/T
|
||||||
|
state
|
||||||
|
|
||||||
|
reason: When set qemu's status to T and execute add-iface command
|
||||||
|
the command will hung all the time.It hungs while wait for qemu
|
||||||
|
to return version messages.
|
||||||
|
|
||||||
|
When qmp starts, set the timeout time to 15 seconds.When times
|
||||||
|
out, return qmp starts failed.We choose 15 seconds to keep consistent
|
||||||
|
with the agent client start.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
vendor/github.com/intel/govmm/qemu/checklist | 1 +
|
||||||
|
vendor/github.com/intel/govmm/qemu/qmp.go | 2 ++
|
||||||
|
2 files changed, 3 insertions(+)
|
||||||
|
create mode 100644 vendor/github.com/intel/govmm/qemu/checklist
|
||||||
|
|
||||||
|
diff --git a/vendor/github.com/intel/govmm/qemu/checklist b/vendor/github.com/intel/govmm/qemu/checklist
|
||||||
|
new file mode 100644
|
||||||
|
index 00000000..b32f1855
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/vendor/github.com/intel/govmm/qemu/checklist
|
||||||
|
@@ -0,0 +1 @@
|
||||||
|
+add timeout when qmp start to avoid qmp client hungs all the time
|
||||||
|
diff --git a/vendor/github.com/intel/govmm/qemu/qmp.go b/vendor/github.com/intel/govmm/qemu/qmp.go
|
||||||
|
index bf9a77dd..a64039de 100644
|
||||||
|
--- a/vendor/github.com/intel/govmm/qemu/qmp.go
|
||||||
|
+++ b/vendor/github.com/intel/govmm/qemu/qmp.go
|
||||||
|
@@ -722,6 +722,8 @@ func QMPStart(ctx context.Context, socket string, cfg QMPConfig, disconnectedCh
|
||||||
|
if q.version == nil {
|
||||||
|
return nil, nil, fmt.Errorf("failed to find QMP version information")
|
||||||
|
}
|
||||||
|
+ case <-time.After(15 * time.Second):
|
||||||
|
+ return nil, nil, fmt.Errorf("qmp start time out")
|
||||||
|
}
|
||||||
|
|
||||||
|
return q, q.version, nil
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,117 @@
|
|||||||
|
From 1efb88fbf554f3977a1a8aa1c79bc70cc1b66953 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Sat, 25 Jul 2020 09:22:08 +0800
|
||||||
|
Subject: [PATCH 02/50] kata-runtime: fix kata-runtime skip read lines in
|
||||||
|
/proc/mounts file problem
|
||||||
|
|
||||||
|
reason: Since /proc/mounts is a virtual file which is changed dynamically by kernel,
|
||||||
|
if we use file pointer to read content in this file line by line, we may miss read
|
||||||
|
some lines. So we retry read /proc/mounts file again to fix this problem.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/utils/utils_linux.go | 58 +++++++++++++++++++++++--------------
|
||||||
|
1 file changed, 36 insertions(+), 22 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/utils/utils_linux.go b/virtcontainers/utils/utils_linux.go
|
||||||
|
index ad870d63..6cef4cfb 100644
|
||||||
|
--- a/virtcontainers/utils/utils_linux.go
|
||||||
|
+++ b/virtcontainers/utils/utils_linux.go
|
||||||
|
@@ -7,15 +7,18 @@ package utils
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
+ "bytes"
|
||||||
|
"crypto/rand"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
+ "io/ioutil"
|
||||||
|
"math/big"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
"unsafe"
|
||||||
|
|
||||||
|
+ "github.com/sirupsen/logrus"
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
)
|
||||||
|
|
||||||
|
@@ -93,6 +96,7 @@ const (
|
||||||
|
procMountsFile = "/proc/mounts"
|
||||||
|
|
||||||
|
fieldsPerLine = 6
|
||||||
|
+ maxRetryTimes = 5
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
@@ -109,35 +113,45 @@ func GetDevicePathAndFsType(mountPoint string) (devicePath, fsType string, err e
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
- var file *os.File
|
||||||
|
+ var retry int = 0
|
||||||
|
|
||||||
|
- file, err = os.Open(procMountsFile)
|
||||||
|
- if err != nil {
|
||||||
|
- return
|
||||||
|
- }
|
||||||
|
-
|
||||||
|
- defer file.Close()
|
||||||
|
+ for retry <= maxRetryTimes {
|
||||||
|
+ var content []byte
|
||||||
|
|
||||||
|
- reader := bufio.NewReader(file)
|
||||||
|
- for {
|
||||||
|
- var line string
|
||||||
|
-
|
||||||
|
- line, err = reader.ReadString('\n')
|
||||||
|
- if err == io.EOF {
|
||||||
|
- err = fmt.Errorf("Mount %s not found", mountPoint)
|
||||||
|
+ content, err = ioutil.ReadFile(procMountsFile)
|
||||||
|
+ if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
- fields := strings.Fields(line)
|
||||||
|
- if len(fields) != fieldsPerLine {
|
||||||
|
- err = fmt.Errorf("Incorrect no of fields (expected %d, got %d)) :%s", fieldsPerLine, len(fields), line)
|
||||||
|
- return
|
||||||
|
+ bytesReader := bytes.NewReader(content)
|
||||||
|
+ reader := bufio.NewReader(bytesReader)
|
||||||
|
+
|
||||||
|
+ for {
|
||||||
|
+ var line string
|
||||||
|
+
|
||||||
|
+ line, err = reader.ReadString('\n')
|
||||||
|
+ if err == io.EOF {
|
||||||
|
+ err = fmt.Errorf("Mount %s not found", mountPoint)
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ fields := strings.Fields(line)
|
||||||
|
+ if len(fields) != fieldsPerLine {
|
||||||
|
+ err = fmt.Errorf("Incorrect no of fields (expected %d, got %d)) :%s", fieldsPerLine, len(fields), line)
|
||||||
|
+ return
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if mountPoint == fields[procPathIndex] {
|
||||||
|
+ devicePath = fields[procDeviceIndex]
|
||||||
|
+ fsType = fields[procTypeIndex]
|
||||||
|
+ return
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
|
||||||
|
- if mountPoint == fields[procPathIndex] {
|
||||||
|
- devicePath = fields[procDeviceIndex]
|
||||||
|
- fsType = fields[procTypeIndex]
|
||||||
|
- return
|
||||||
|
+ retry = retry + 1
|
||||||
|
+ if retry <= maxRetryTimes {
|
||||||
|
+ logrus.Warnf("can not find %s in %s, retry %d times again......", mountPoint, procMountsFile, retry)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
+ return "", "", fmt.Errorf("retry %d times fail to get devicePath adn fs type", maxRetryTimes)
|
||||||
|
}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,55 @@
|
|||||||
|
From cf595941e1d105af23bc006bf1998ac072733d0a Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Sat, 25 Jul 2020 10:03:35 +0800
|
||||||
|
Subject: [PATCH 03/50] kata-runtime: fix kata-proxy process left problem
|
||||||
|
|
||||||
|
reason: stopSandbox function will send the DestroySandboxRequest
|
||||||
|
to kata-agent in the VM and then kill the kata-proxy process in
|
||||||
|
the host. However, if k.sendReq(DestroySandboxRequest) get error,
|
||||||
|
stopSandbox will return immediately not execute the following kill
|
||||||
|
kata-proxy process statement, which cause the kata-process left.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/kata_agent.go | 18 +++++++++++-------
|
||||||
|
1 file changed, 11 insertions(+), 7 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
|
||||||
|
index a0cf190e..be5e96aa 100644
|
||||||
|
--- a/virtcontainers/kata_agent.go
|
||||||
|
+++ b/virtcontainers/kata_agent.go
|
||||||
|
@@ -976,6 +976,17 @@ func (k *kataAgent) stopSandbox(sandbox *Sandbox) error {
|
||||||
|
return errorMissingProxy
|
||||||
|
}
|
||||||
|
|
||||||
|
+ // since stopSandbox will destroy the sandbox in the VM, and we don't need
|
||||||
|
+ // kata-proxy process to communicate with kata-agent again, so we should
|
||||||
|
+ // make sure kata-proxy can be killed cleanly, even when k.sendReq(DestroySandboxRequest)
|
||||||
|
+ // return error
|
||||||
|
+ defer func() {
|
||||||
|
+ _ = k.proxy.stop(k.state.ProxyPid)
|
||||||
|
+ // clean up agent state
|
||||||
|
+ k.state.ProxyPid = -1
|
||||||
|
+ k.state.URL = ""
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
req := &grpc.DestroySandboxRequest{}
|
||||||
|
|
||||||
|
if _, err := k.sendReq(req); err != nil {
|
||||||
|
@@ -989,13 +1000,6 @@ func (k *kataAgent) stopSandbox(sandbox *Sandbox) error {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
- if err := k.proxy.stop(k.state.ProxyPid); err != nil {
|
||||||
|
- return err
|
||||||
|
- }
|
||||||
|
-
|
||||||
|
- // clean up agent state
|
||||||
|
- k.state.ProxyPid = -1
|
||||||
|
- k.state.URL = ""
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,65 @@
|
|||||||
|
From 025520f7fd3aeb5ed53b468b5e494b1bbb6674ae Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Sat, 25 Jul 2020 11:25:34 +0800
|
||||||
|
Subject: [PATCH 04/50] kata-runtime: keep the process name of qemu same as
|
||||||
|
configured path
|
||||||
|
|
||||||
|
reason: inorder to make testcase scripts can use the same hypervisor
|
||||||
|
name no matter what version hypervisor use, keep the process name of
|
||||||
|
hypervisor same as configured path in the configuration.toml file
|
||||||
|
instead of the resolved path of symbol link.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
pkg/katautils/config.go | 8 +++++++-
|
||||||
|
pkg/katautils/config_test.go | 4 ++--
|
||||||
|
2 files changed, 9 insertions(+), 3 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
||||||
|
index 14794a24..349e667f 100644
|
||||||
|
--- a/pkg/katautils/config.go
|
||||||
|
+++ b/pkg/katautils/config.go
|
||||||
|
@@ -10,6 +10,7 @@ import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
|
+ "path/filepath"
|
||||||
|
goruntime "runtime"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
@@ -172,7 +173,12 @@ func (h hypervisor) path() (string, error) {
|
||||||
|
p = defaultHypervisorPath
|
||||||
|
}
|
||||||
|
|
||||||
|
- return ResolvePath(p)
|
||||||
|
+ absolutePath, err := filepath.Abs(p)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return "", err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return absolutePath, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h hypervisor) ctlpath() (string, error) {
|
||||||
|
diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go
|
||||||
|
index 221a4b55..2eae1f6a 100644
|
||||||
|
--- a/pkg/katautils/config_test.go
|
||||||
|
+++ b/pkg/katautils/config_test.go
|
||||||
|
@@ -1061,12 +1061,12 @@ func TestHypervisorDefaultsHypervisor(t *testing.T) {
|
||||||
|
assert.NoError(err)
|
||||||
|
assert.Equal(p, defaultHypervisorPath, "default hypervisor path wrong")
|
||||||
|
|
||||||
|
- // test path resolution
|
||||||
|
+ // test path resolution, just return the absolute path instead of resolved path
|
||||||
|
defaultHypervisorPath = testHypervisorLinkPath
|
||||||
|
h = hypervisor{}
|
||||||
|
p, err = h.path()
|
||||||
|
assert.NoError(err)
|
||||||
|
- assert.Equal(p, testHypervisorPath)
|
||||||
|
+ assert.Equal(p, testHypervisorLinkPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHypervisorDefaultsKernel(t *testing.T) {
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,66 @@
|
|||||||
|
From c279f4548ccc534f1c65723bf9994c448e510d3d Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Sat, 25 Jul 2020 11:56:35 +0800
|
||||||
|
Subject: [PATCH 05/50] cgroups: increase delete cgroup retry times
|
||||||
|
|
||||||
|
reason: inorder to make sure cgroup dir to be deleted, so we increase
|
||||||
|
the retry times when delete cgroup dir failed.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
vendor/github.com/containerd/cgroups/cgroup.go | 4 ++--
|
||||||
|
vendor/github.com/containerd/cgroups/utils.go | 9 ++++++---
|
||||||
|
2 files changed, 8 insertions(+), 5 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/vendor/github.com/containerd/cgroups/cgroup.go b/vendor/github.com/containerd/cgroups/cgroup.go
|
||||||
|
index 53866685..69612b0a 100644
|
||||||
|
--- a/vendor/github.com/containerd/cgroups/cgroup.go
|
||||||
|
+++ b/vendor/github.com/containerd/cgroups/cgroup.go
|
||||||
|
@@ -223,7 +223,7 @@ func (c *cgroup) Delete() error {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := d.Delete(sp); err != nil {
|
||||||
|
- errors = append(errors, string(s.Name()))
|
||||||
|
+ errors = append(errors, fmt.Sprintf("delete %s get error: %v", string(s.Name()), err.Error()))
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
@@ -234,7 +234,7 @@ func (c *cgroup) Delete() error {
|
||||||
|
}
|
||||||
|
path := p.Path(sp)
|
||||||
|
if err := remove(path); err != nil {
|
||||||
|
- errors = append(errors, path)
|
||||||
|
+ errors = append(errors, fmt.Sprintf("remove path %s get error: %v", path, err.Error()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
diff --git a/vendor/github.com/containerd/cgroups/utils.go b/vendor/github.com/containerd/cgroups/utils.go
|
||||||
|
index 8a97d04d..82dbe2d3 100644
|
||||||
|
--- a/vendor/github.com/containerd/cgroups/utils.go
|
||||||
|
+++ b/vendor/github.com/containerd/cgroups/utils.go
|
||||||
|
@@ -99,16 +99,19 @@ func defaults(root string) ([]Subsystem, error) {
|
||||||
|
// retrying the remove after a exp timeout
|
||||||
|
func remove(path string) error {
|
||||||
|
delay := 10 * time.Millisecond
|
||||||
|
- for i := 0; i < 5; i++ {
|
||||||
|
+ var err error
|
||||||
|
+ var count int = 0
|
||||||
|
+ for i := 0; i < 10; i++ {
|
||||||
|
if i != 0 {
|
||||||
|
time.Sleep(delay)
|
||||||
|
delay *= 2
|
||||||
|
}
|
||||||
|
- if err := os.RemoveAll(path); err == nil {
|
||||||
|
+ if err = os.RemoveAll(path); err == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
+ count++
|
||||||
|
}
|
||||||
|
- return fmt.Errorf("cgroups: unable to remove path %q", path)
|
||||||
|
+ return fmt.Errorf("cgroups: unable to remove path %q, err: %v, count:%d", path, err, count)
|
||||||
|
}
|
||||||
|
|
||||||
|
// readPids will read all the pids of processes in a cgroup by the provided path
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,39 @@
|
|||||||
|
From 3dc10421f177900c0ee94fc49b32ec66a46d9331 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Mon, 27 Jul 2020 19:18:50 +0800
|
||||||
|
Subject: [PATCH 06/50] kata-runtime: fix umount container rootfs dir return
|
||||||
|
ivalid argument error
|
||||||
|
|
||||||
|
reason: If sandbox hypervisor doesn't use block device driver for hotplugging container
|
||||||
|
rootfs block device into guest, kata-runtime will bind mount container rootfs dir to 9p
|
||||||
|
kataShared dir. However, container stop() function will always call bindUnmountContainerRootfs
|
||||||
|
function no matter block device driver is used or not. So we just need to call
|
||||||
|
bindUnmountContainerRootfs only if rootfs is bind mount to guest by 9p.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/container.go | 8 ++++++--
|
||||||
|
1 file changed, 6 insertions(+), 2 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/container.go b/virtcontainers/container.go
|
||||||
|
index 9e2d1e94..b42cc6e9 100644
|
||||||
|
--- a/virtcontainers/container.go
|
||||||
|
+++ b/virtcontainers/container.go
|
||||||
|
@@ -1120,8 +1120,12 @@ func (c *Container) stop(force bool) error {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
- if err := bindUnmountContainerRootfs(c.ctx, getMountPath(c.sandbox.id), c.id); err != nil && !force {
|
||||||
|
- return err
|
||||||
|
+ // umount container rootfs dir only if container use 9p
|
||||||
|
+ // to bind mount host container rootfs to 9p shared dir
|
||||||
|
+ if c.state.BlockDeviceID == "" {
|
||||||
|
+ if err := bindUnmountContainerRootfs(c.ctx, getMountPath(c.sandbox.id), c.id); err != nil && !force {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := c.detachDevices(); err != nil && !force {
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,855 @@
|
|||||||
|
From d93da1875ed7f1a6061cffb13475506d73c86003 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Sat, 25 Jul 2020 16:04:19 +0800
|
||||||
|
Subject: [PATCH 07/50] kata-runtime: enhance reliability when kata related
|
||||||
|
process
|
||||||
|
|
||||||
|
reason: enhance the reliability when kata related processes is abnormal,
|
||||||
|
make kata-container still destroy the sandbox and clean up all resources.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
cli/delete.go | 6 ++
|
||||||
|
cli/kill.go | 3 +-
|
||||||
|
virtcontainers/acrn.go | 2 +-
|
||||||
|
virtcontainers/agent.go | 3 +
|
||||||
|
virtcontainers/api.go | 67 +++++++++++++++++++++
|
||||||
|
virtcontainers/clh.go | 2 +-
|
||||||
|
virtcontainers/container.go | 55 +++++++++++++++--
|
||||||
|
virtcontainers/fc.go | 2 +-
|
||||||
|
virtcontainers/hypervisor.go | 2 +-
|
||||||
|
virtcontainers/kata_agent.go | 31 ++++++----
|
||||||
|
virtcontainers/mock_hypervisor.go | 2 +-
|
||||||
|
virtcontainers/mock_hypervisor_test.go | 2 +-
|
||||||
|
virtcontainers/noop_agent.go | 4 ++
|
||||||
|
virtcontainers/pkg/oci/utils.go | 5 ++
|
||||||
|
virtcontainers/qemu.go | 51 +++++++++-------
|
||||||
|
virtcontainers/sandbox.go | 106 +++++++++++++++++++++++++++++----
|
||||||
|
virtcontainers/types/sandbox.go | 14 ++++-
|
||||||
|
virtcontainers/utils/utils.go | 46 ++++++++++++++
|
||||||
|
virtcontainers/vm.go | 4 +-
|
||||||
|
19 files changed, 348 insertions(+), 59 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/cli/delete.go b/cli/delete.go
|
||||||
|
index c2ce52a4..2f5586e5 100644
|
||||||
|
--- a/cli/delete.go
|
||||||
|
+++ b/cli/delete.go
|
||||||
|
@@ -110,6 +110,12 @@ func delete(ctx context.Context, containerID string, force bool) error {
|
||||||
|
forceStop = true
|
||||||
|
}
|
||||||
|
|
||||||
|
+ if oci.StateToOCIState(status.State.State) == oci.StateUnhealthy {
|
||||||
|
+ // Set forceStop and force bool flag to true to force delete everything
|
||||||
|
+ forceStop = true
|
||||||
|
+ force = true
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
switch containerType {
|
||||||
|
case vc.PodSandbox:
|
||||||
|
if err := deleteSandbox(ctx, sandboxID, force); err != nil {
|
||||||
|
diff --git a/cli/kill.go b/cli/kill.go
|
||||||
|
index 60fa41e0..b228205f 100644
|
||||||
|
--- a/cli/kill.go
|
||||||
|
+++ b/cli/kill.go
|
||||||
|
@@ -133,11 +133,12 @@ func kill(ctx context.Context, containerID, signal string, all bool) error {
|
||||||
|
kataLog.WithField("signal", signal).WithField("container state", status.State.State).Info("kill")
|
||||||
|
|
||||||
|
// container MUST be created, running or paused
|
||||||
|
+ // If container state is unhealthy, should process this exceptional case separately
|
||||||
|
if status.State.State == types.StateReady || status.State.State == types.StateRunning || status.State.State == types.StatePaused {
|
||||||
|
if err := vci.KillContainer(ctx, sandboxID, containerID, signum, all); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
- } else if !all {
|
||||||
|
+ } else if !all && status.State.State != types.StateUnhealthy {
|
||||||
|
return fmt.Errorf("container not running")
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/acrn.go b/virtcontainers/acrn.go
|
||||||
|
index 761eda03..10cae06f 100644
|
||||||
|
--- a/virtcontainers/acrn.go
|
||||||
|
+++ b/virtcontainers/acrn.go
|
||||||
|
@@ -475,7 +475,7 @@ func (a *Acrn) waitSandbox(timeoutSecs int) error {
|
||||||
|
}
|
||||||
|
|
||||||
|
// stopSandbox will stop the Sandbox's VM.
|
||||||
|
-func (a *Acrn) stopSandbox() (err error) {
|
||||||
|
+func (a *Acrn) stopSandbox(force bool) (err error) {
|
||||||
|
span, _ := a.trace("stopSandbox")
|
||||||
|
defer span.Finish()
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go
|
||||||
|
index c62107ec..be9526c7 100644
|
||||||
|
--- a/virtcontainers/agent.go
|
||||||
|
+++ b/virtcontainers/agent.go
|
||||||
|
@@ -259,4 +259,7 @@ type agent interface {
|
||||||
|
|
||||||
|
// load data from disk
|
||||||
|
load(persistapi.AgentState)
|
||||||
|
+
|
||||||
|
+ // get proxy process pid
|
||||||
|
+ getProxyPid() int
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
|
||||||
|
index de569713..fa82d163 100644
|
||||||
|
--- a/virtcontainers/api.go
|
||||||
|
+++ b/virtcontainers/api.go
|
||||||
|
@@ -7,8 +7,10 @@ package virtcontainers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
+ "fmt"
|
||||||
|
"os"
|
||||||
|
"runtime"
|
||||||
|
+ "strings"
|
||||||
|
"syscall"
|
||||||
|
|
||||||
|
deviceApi "github.com/kata-containers/runtime/virtcontainers/device/api"
|
||||||
|
@@ -18,6 +20,7 @@ import (
|
||||||
|
vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/store"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/types"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||||
|
specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
opentracing "github.com/opentracing/opentracing-go"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
@@ -597,20 +600,51 @@ func statusContainer(sandbox *Sandbox, containerID string) (ContainerStatus, err
|
||||||
|
container.state.State == types.StatePaused) &&
|
||||||
|
container.process.Pid > 0 {
|
||||||
|
|
||||||
|
+ // If container state is active, however kata-proxy and qemu process all exit already
|
||||||
|
+ // which means sandbox has beed stopped exceptionally, then we should force delete
|
||||||
|
+ // sandbox and container state files in sandbox.Store
|
||||||
|
+ if sandbox.shouldForceDelete() {
|
||||||
|
+ virtLog.Logger.Warn("sandbox status is abnormal, sandbox should be force deleted")
|
||||||
|
+ sandbox.forceDeleteSandbox()
|
||||||
|
+ return ContainerStatus{}, fmt.Errorf("sandbox has beed stopped exceptionally")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
running, err := isShimRunning(container.process.Pid)
|
||||||
|
if err != nil {
|
||||||
|
return ContainerStatus{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
+ // If kata-shim process exit or be killed, need to stop the container
|
||||||
|
if !running {
|
||||||
|
virtLog.WithFields(logrus.Fields{
|
||||||
|
"state": container.state.State,
|
||||||
|
"pid": container.process.Pid}).
|
||||||
|
Info("container isn't running")
|
||||||
|
+
|
||||||
|
if err := container.stop(true); err != nil {
|
||||||
|
return ContainerStatus{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+ isPodSandbox := (containerID == sandbox.id)
|
||||||
|
+
|
||||||
|
+ // If sandbox is unhealthy, process it correctly
|
||||||
|
+ if !sandbox.health() {
|
||||||
|
+ // process podSandbox container type case
|
||||||
|
+ if isPodSandbox {
|
||||||
|
+ if err := processUnhealthySandbox(sandbox, container); err != nil {
|
||||||
|
+ return ContainerStatus{}, err
|
||||||
|
+ }
|
||||||
|
+ } else {
|
||||||
|
+ // If container type is pod_container, which means container operations can not be
|
||||||
|
+ // processed successfully, we should return the error as soon as possible
|
||||||
|
+ if err := container.setContainerState(types.StateUnhealthy); err != nil {
|
||||||
|
+ return ContainerStatus{}, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return ContainerStatus{}, fmt.Errorf("container status is unhealthy, stop container failed")
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
|
||||||
|
return ContainerStatus{
|
||||||
|
@@ -1016,3 +1050,36 @@ func CleanupContainer(ctx context.Context, sandboxID, containerID string, force
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+// procesUnhealthySandbox only change sandbox state to unhealthy
|
||||||
|
+// when caller is kata-runtime kill or kata-runtime delete
|
||||||
|
+func processUnhealthySandbox(sandbox *Sandbox, container *Container) error {
|
||||||
|
+ // Set all containers state to unhealthy
|
||||||
|
+ if err := sandbox.setContainersState(types.StateUnhealthy); err != nil {
|
||||||
|
+ container.Logger().WithError(err).Warn("set all containers state to unhealthy fail")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // Set sandbox state to unhealthy
|
||||||
|
+ if err := sandbox.setSandboxState(types.StateUnhealthy); err != nil {
|
||||||
|
+ container.Logger().WithError(err).Warn("set sandbox state to unhealthy fail")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ forceDelete := false
|
||||||
|
+
|
||||||
|
+ // If process is kata-runtime kill or kata-runtime delete,
|
||||||
|
+ // we should kill or delete sandbox forcefully
|
||||||
|
+ if cmdline, err := utils.GetProcessCmdline(os.Getpid()); err != nil {
|
||||||
|
+ container.Logger().WithError(err).Warn("fail to get process cmdline info")
|
||||||
|
+ } else {
|
||||||
|
+ forceDelete = strings.Contains(cmdline, "kill") || strings.Contains(cmdline, "delete")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if forceDelete {
|
||||||
|
+ // force stop podSandbox type container's kata-shim process
|
||||||
|
+ if err := stopShim(container.process.Pid); err != nil {
|
||||||
|
+ container.Logger().WithError(err).Warn("fail to stop podSandbox type container kata-shim")
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/clh.go b/virtcontainers/clh.go
|
||||||
|
index d40b698b..59510b02 100644
|
||||||
|
--- a/virtcontainers/clh.go
|
||||||
|
+++ b/virtcontainers/clh.go
|
||||||
|
@@ -569,7 +569,7 @@ func (clh *cloudHypervisor) resumeSandbox() error {
|
||||||
|
}
|
||||||
|
|
||||||
|
// stopSandbox will stop the Sandbox's VM.
|
||||||
|
-func (clh *cloudHypervisor) stopSandbox() (err error) {
|
||||||
|
+func (clh *cloudHypervisor) stopSandbox(force bool) (err error) {
|
||||||
|
span, _ := clh.trace("stopSandbox")
|
||||||
|
defer span.Finish()
|
||||||
|
clh.Logger().WithField("function", "stopSandbox").Info("Stop Sandbox")
|
||||||
|
diff --git a/virtcontainers/container.go b/virtcontainers/container.go
|
||||||
|
index b42cc6e9..9485e708 100644
|
||||||
|
--- a/virtcontainers/container.go
|
||||||
|
+++ b/virtcontainers/container.go
|
||||||
|
@@ -17,8 +17,12 @@ import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/containerd/cgroups"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/device/config"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/device/manager"
|
||||||
|
vccgroups "github.com/kata-containers/runtime/virtcontainers/pkg/cgroups"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/pkg/rootless"
|
||||||
|
vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/store"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/types"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/utils"
|
||||||
|
specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
@@ -26,11 +30,6 @@ import (
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
-
|
||||||
|
- "github.com/kata-containers/runtime/virtcontainers/device/config"
|
||||||
|
- "github.com/kata-containers/runtime/virtcontainers/device/manager"
|
||||||
|
- "github.com/kata-containers/runtime/virtcontainers/pkg/rootless"
|
||||||
|
- "github.com/kata-containers/runtime/virtcontainers/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// https://github.com/torvalds/linux/blob/master/include/uapi/linux/major.h
|
||||||
|
@@ -1047,6 +1046,13 @@ func (c *Container) stop(force bool) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+ // If container state is unhealthy, just force kill the container
|
||||||
|
+ if c.state.State == types.StateUnhealthy {
|
||||||
|
+ c.forceKillContainer()
|
||||||
|
+ // after force kill container, then change container state to stopped
|
||||||
|
+ return c.setContainerState(types.StateStopped)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
if err := c.state.ValidTransition(c.state.State, types.StateStopped); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
@@ -1063,6 +1069,8 @@ func (c *Container) stop(force bool) error {
|
||||||
|
if err := stopShim(c.process.Pid); err != nil {
|
||||||
|
l.WithError(err).Warn("failed to stop shim")
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+ c.forceKillContainer()
|
||||||
|
}
|
||||||
|
|
||||||
|
}()
|
||||||
|
@@ -1096,7 +1104,9 @@ func (c *Container) stop(force bool) error {
|
||||||
|
// this signal will ensure the container will get killed to match
|
||||||
|
// the state of the shim. This will allow the following call to
|
||||||
|
// stopContainer() to succeed in such particular case.
|
||||||
|
- c.kill(syscall.SIGKILL, true)
|
||||||
|
+ if err := c.kill(syscall.SIGKILL, true); err != nil {
|
||||||
|
+ c.Logger().Errorf("send signal to container failed: %v", err)
|
||||||
|
+ }
|
||||||
|
|
||||||
|
// Since the agent has supported the MultiWaitProcess, it's better to
|
||||||
|
// wait the process here to make sure the process has exited before to
|
||||||
|
@@ -1582,3 +1592,36 @@ func (c *Container) cgroupsUpdate(resources specs.LinuxResources) error {
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+// forceDeleteContainer force clean container mount info and resources stored in the disk
|
||||||
|
+func (c *Container) forceDeleteContainer() {
|
||||||
|
+ if err := c.unmountHostMounts(); err != nil {
|
||||||
|
+ c.Logger().WithError(err).Warn("container force delete umount host mounts fail")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := c.sandbox.removeContainer(c.id); err != nil {
|
||||||
|
+ c.Logger().WithError(err).Warn("sandbox removeContainer fail")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := c.store.Delete(); err != nil {
|
||||||
|
+ c.Logger().WithError(err).Warn("force delete container store fail")
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (c *Container) forceKillContainer() {
|
||||||
|
+ if err := c.setContainerState(types.StateStopped); err != nil {
|
||||||
|
+ c.Logger().WithError(err).Warn("force kill container: change container state to StateStopped failed")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := c.unmountHostMounts(); err != nil {
|
||||||
|
+ c.Logger().WithError(err).Warn("force kill container: umount container host mounts failed")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := c.detachDevices(); err != nil {
|
||||||
|
+ c.Logger().WithError(err).Warn("force kill container: detach container devices failed")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := c.removeDrive(); err != nil {
|
||||||
|
+ c.Logger().WithError(err).Warn("force kill container: remove container drive failed")
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go
|
||||||
|
index 97ef5ffc..72a8e192 100644
|
||||||
|
--- a/virtcontainers/fc.go
|
||||||
|
+++ b/virtcontainers/fc.go
|
||||||
|
@@ -864,7 +864,7 @@ func (fc *firecracker) cleanupJail() {
|
||||||
|
}
|
||||||
|
|
||||||
|
// stopSandbox will stop the Sandbox's VM.
|
||||||
|
-func (fc *firecracker) stopSandbox() (err error) {
|
||||||
|
+func (fc *firecracker) stopSandbox(force bool) (err error) {
|
||||||
|
span, _ := fc.trace("stopSandbox")
|
||||||
|
defer span.Finish()
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go
|
||||||
|
index 4b3dd3d0..fd7d1f8e 100644
|
||||||
|
--- a/virtcontainers/hypervisor.go
|
||||||
|
+++ b/virtcontainers/hypervisor.go
|
||||||
|
@@ -766,7 +766,7 @@ func generateVMSocket(id string, useVsock bool, vmStogarePath string) (interface
|
||||||
|
type hypervisor interface {
|
||||||
|
createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, stateful bool) error
|
||||||
|
startSandbox(timeout int) error
|
||||||
|
- stopSandbox() error
|
||||||
|
+ stopSandbox(force bool) error
|
||||||
|
pauseSandbox() error
|
||||||
|
saveSandbox() error
|
||||||
|
resumeSandbox() error
|
||||||
|
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
|
||||||
|
index be5e96aa..7575d326 100644
|
||||||
|
--- a/virtcontainers/kata_agent.go
|
||||||
|
+++ b/virtcontainers/kata_agent.go
|
||||||
|
@@ -57,8 +57,9 @@ const (
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
- checkRequestTimeout = 30 * time.Second
|
||||||
|
- defaultRequestTimeout = 60 * time.Second
|
||||||
|
+ checkRequestTimeout = 10 * time.Second
|
||||||
|
+ defaultRequestTimeout = 10 * time.Second
|
||||||
|
+ createContainerTimeout = 120 * time.Second
|
||||||
|
errorMissingProxy = errors.New("Missing proxy pointer")
|
||||||
|
errorMissingOCISpec = errors.New("Missing OCI specification")
|
||||||
|
defaultKataHostSharedDir = "/run/kata-containers/shared/sandboxes/"
|
||||||
|
@@ -987,17 +988,21 @@ func (k *kataAgent) stopSandbox(sandbox *Sandbox) error {
|
||||||
|
k.state.URL = ""
|
||||||
|
}()
|
||||||
|
|
||||||
|
- req := &grpc.DestroySandboxRequest{}
|
||||||
|
+ // If sandbox.state.State is unhealthy, we don't need to send DestroySandboxRequest
|
||||||
|
+ // to kata-agent, just force stop the sandbox
|
||||||
|
+ if sandbox.state.State != types.StateUnhealthy {
|
||||||
|
+ req := &grpc.DestroySandboxRequest{}
|
||||||
|
|
||||||
|
- if _, err := k.sendReq(req); err != nil {
|
||||||
|
- return err
|
||||||
|
- }
|
||||||
|
-
|
||||||
|
- if k.dynamicTracing {
|
||||||
|
- _, err := k.sendReq(&grpc.StopTracingRequest{})
|
||||||
|
- if err != nil {
|
||||||
|
+ if _, err := k.sendReq(req); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+ if k.dynamicTracing {
|
||||||
|
+ _, err := k.sendReq(&grpc.StopTracingRequest{})
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
@@ -2062,6 +2067,8 @@ func (k *kataAgent) getReqContext(reqName string) (ctx context.Context, cancel c
|
||||||
|
// Wait has no timeout
|
||||||
|
case grpcCheckRequest:
|
||||||
|
ctx, cancel = context.WithTimeout(ctx, checkRequestTimeout)
|
||||||
|
+ case grpcCreateContainerRequest:
|
||||||
|
+ ctx, cancel = context.WithTimeout(ctx, createContainerTimeout)
|
||||||
|
default:
|
||||||
|
ctx, cancel = context.WithTimeout(ctx, defaultRequestTimeout)
|
||||||
|
}
|
||||||
|
@@ -2382,3 +2389,7 @@ func (k *kataAgent) load(s persistapi.AgentState) {
|
||||||
|
k.state.ProxyPid = s.ProxyPid
|
||||||
|
k.state.URL = s.URL
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func (k *kataAgent) getProxyPid() int {
|
||||||
|
+ return k.state.ProxyPid
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/mock_hypervisor.go b/virtcontainers/mock_hypervisor.go
|
||||||
|
index 0c84e43c..a5b67491 100644
|
||||||
|
--- a/virtcontainers/mock_hypervisor.go
|
||||||
|
+++ b/virtcontainers/mock_hypervisor.go
|
||||||
|
@@ -39,7 +39,7 @@ func (m *mockHypervisor) startSandbox(timeout int) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
-func (m *mockHypervisor) stopSandbox() error {
|
||||||
|
+func (m *mockHypervisor) stopSandbox(force bool) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/mock_hypervisor_test.go b/virtcontainers/mock_hypervisor_test.go
|
||||||
|
index b73b28f2..827e3192 100644
|
||||||
|
--- a/virtcontainers/mock_hypervisor_test.go
|
||||||
|
+++ b/virtcontainers/mock_hypervisor_test.go
|
||||||
|
@@ -53,7 +53,7 @@ func TestMockHypervisorStartSandbox(t *testing.T) {
|
||||||
|
func TestMockHypervisorStopSandbox(t *testing.T) {
|
||||||
|
var m *mockHypervisor
|
||||||
|
|
||||||
|
- assert.NoError(t, m.stopSandbox())
|
||||||
|
+ assert.NoError(t, m.stopSandbox(false))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMockHypervisorAddDevice(t *testing.T) {
|
||||||
|
diff --git a/virtcontainers/noop_agent.go b/virtcontainers/noop_agent.go
|
||||||
|
index 189f6b3f..8a7cd337 100644
|
||||||
|
--- a/virtcontainers/noop_agent.go
|
||||||
|
+++ b/virtcontainers/noop_agent.go
|
||||||
|
@@ -236,3 +236,7 @@ func (n *noopAgent) save() (s persistapi.AgentState) {
|
||||||
|
|
||||||
|
// load is the Noop agent state loader. It does nothing.
|
||||||
|
func (n *noopAgent) load(s persistapi.AgentState) {}
|
||||||
|
+
|
||||||
|
+func (n *noopAgent) getProxyPid() int {
|
||||||
|
+ return -1
|
||||||
|
+}
|
||||||
|
\ No newline at end of file
|
||||||
|
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
||||||
|
index 5348c57d..cd8d48ce 100644
|
||||||
|
--- a/virtcontainers/pkg/oci/utils.go
|
||||||
|
+++ b/virtcontainers/pkg/oci/utils.go
|
||||||
|
@@ -70,6 +70,9 @@ const (
|
||||||
|
|
||||||
|
// StatePaused represents a container that has been paused.
|
||||||
|
StatePaused = "paused"
|
||||||
|
+
|
||||||
|
+ // StateUnhealthy represents a container that is unhealthy
|
||||||
|
+ StateUnhealthy = "unhealthy"
|
||||||
|
)
|
||||||
|
|
||||||
|
const KernelModulesSeparator = ";"
|
||||||
|
@@ -964,6 +967,8 @@ func StateToOCIState(state types.StateString) string {
|
||||||
|
return StateStopped
|
||||||
|
case types.StatePaused:
|
||||||
|
return StatePaused
|
||||||
|
+ case types.StateUnhealthy:
|
||||||
|
+ return StateUnhealthy
|
||||||
|
default:
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
|
||||||
|
index ca286550..4b15d968 100644
|
||||||
|
--- a/virtcontainers/qemu.go
|
||||||
|
+++ b/virtcontainers/qemu.go
|
||||||
|
@@ -687,7 +687,7 @@ func (q *qemu) setupVirtiofsd() (err error) {
|
||||||
|
q.Logger().Info("virtiofsd quits")
|
||||||
|
// Wait to release resources of virtiofsd process
|
||||||
|
cmd.Process.Wait()
|
||||||
|
- q.stopSandbox()
|
||||||
|
+ q.stopSandbox(false)
|
||||||
|
}()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
@@ -922,11 +922,11 @@ func (q *qemu) waitSandbox(timeout int) error {
|
||||||
|
}
|
||||||
|
|
||||||
|
// stopSandbox will stop the Sandbox's VM.
|
||||||
|
-func (q *qemu) stopSandbox() error {
|
||||||
|
+func (q *qemu) stopSandbox(force bool) error {
|
||||||
|
span, _ := q.trace("stopSandbox")
|
||||||
|
defer span.Finish()
|
||||||
|
|
||||||
|
- q.Logger().Info("Stopping Sandbox")
|
||||||
|
+ q.Logger().Infof("force stopping Sandbox: %v", force)
|
||||||
|
if q.stopped {
|
||||||
|
q.Logger().Info("Already stopped")
|
||||||
|
return nil
|
||||||
|
@@ -937,28 +937,37 @@ func (q *qemu) stopSandbox() error {
|
||||||
|
q.stopped = true
|
||||||
|
}()
|
||||||
|
|
||||||
|
- if q.config.Debug && q.qemuConfig.LogFile != "" {
|
||||||
|
- f, err := os.OpenFile(q.qemuConfig.LogFile, os.O_RDONLY, 0)
|
||||||
|
- if err == nil {
|
||||||
|
- scanner := bufio.NewScanner(f)
|
||||||
|
- for scanner.Scan() {
|
||||||
|
- q.Logger().Debug(scanner.Text())
|
||||||
|
- }
|
||||||
|
- if err := scanner.Err(); err != nil {
|
||||||
|
- q.Logger().WithError(err).Debug("read qemu log failed")
|
||||||
|
+ if !force {
|
||||||
|
+ if q.config.Debug && q.qemuConfig.LogFile != "" {
|
||||||
|
+ f, err := os.OpenFile(q.qemuConfig.LogFile, os.O_RDONLY, 0)
|
||||||
|
+ if err == nil {
|
||||||
|
+ scanner := bufio.NewScanner(f)
|
||||||
|
+ for scanner.Scan() {
|
||||||
|
+ q.Logger().Debug(scanner.Text())
|
||||||
|
+ }
|
||||||
|
+ if err := scanner.Err(); err != nil {
|
||||||
|
+ q.Logger().WithError(err).Debug("read qemu log failed")
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
- }
|
||||||
|
|
||||||
|
- err := q.qmpSetup()
|
||||||
|
- if err != nil {
|
||||||
|
- return err
|
||||||
|
- }
|
||||||
|
+ err := q.qmpSetup()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
|
||||||
|
- err = q.qmpMonitorCh.qmp.ExecuteQuit(q.qmpMonitorCh.ctx)
|
||||||
|
- if err != nil {
|
||||||
|
- q.Logger().WithError(err).Error("Fail to execute qmp QUIT")
|
||||||
|
- return err
|
||||||
|
+ err = q.qmpMonitorCh.qmp.ExecuteQuit(q.qmpMonitorCh.ctx)
|
||||||
|
+ if err != nil {
|
||||||
|
+ q.Logger().WithError(err).Error("Fail to execute qmp QUIT")
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ } else {
|
||||||
|
+ qemuMainPid := q.getPids()[0]
|
||||||
|
+ if qemuMainPid <= 1 {
|
||||||
|
+ return fmt.Errorf("force kill qemu process pid is invalid")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ _ = syscall.Kill(qemuMainPid, syscall.SIGKILL)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index edd1af5b..78188ed7 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -12,19 +12,13 @@ import (
|
||||||
|
"math"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
+ "path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"syscall"
|
||||||
|
|
||||||
|
"github.com/containerd/cgroups"
|
||||||
|
"github.com/containernetworking/plugins/pkg/ns"
|
||||||
|
- "github.com/opencontainers/runc/libcontainer/configs"
|
||||||
|
- specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
- opentracing "github.com/opentracing/opentracing-go"
|
||||||
|
- "github.com/pkg/errors"
|
||||||
|
- "github.com/sirupsen/logrus"
|
||||||
|
- "github.com/vishvananda/netlink"
|
||||||
|
-
|
||||||
|
"github.com/kata-containers/agent/protocols/grpc"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/device/api"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/device/config"
|
||||||
|
@@ -41,6 +35,12 @@ import (
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/store"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/types"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/utils"
|
||||||
|
+ "github.com/opencontainers/runc/libcontainer/configs"
|
||||||
|
+ specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
+ opentracing "github.com/opentracing/opentracing-go"
|
||||||
|
+ "github.com/pkg/errors"
|
||||||
|
+ "github.com/sirupsen/logrus"
|
||||||
|
+ "github.com/vishvananda/netlink"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
@@ -50,6 +50,9 @@ const (
|
||||||
|
|
||||||
|
// DirMode is the permission bits used for creating a directory
|
||||||
|
DirMode = os.FileMode(0750) | os.ModeDir
|
||||||
|
+
|
||||||
|
+ // kata-proxy proces name
|
||||||
|
+ KataProxyProcessName = "kata-proxy"
|
||||||
|
)
|
||||||
|
|
||||||
|
// SandboxStatus describes a sandbox status.
|
||||||
|
@@ -1037,7 +1040,7 @@ func (s *Sandbox) startVM() (err error) {
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
if err != nil {
|
||||||
|
- s.hypervisor.stopSandbox()
|
||||||
|
+ s.hypervisor.stopSandbox(false)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
@@ -1090,7 +1093,12 @@ func (s *Sandbox) stopVM() error {
|
||||||
|
}
|
||||||
|
|
||||||
|
s.Logger().Info("Stopping VM")
|
||||||
|
- return s.hypervisor.stopSandbox()
|
||||||
|
+ forceStop := false
|
||||||
|
+ if s.state.State == types.StateUnhealthy {
|
||||||
|
+ forceStop = true
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return s.hypervisor.stopSandbox(forceStop)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Sandbox) addContainer(c *Container) error {
|
||||||
|
@@ -1591,13 +1599,15 @@ func (s *Sandbox) setSandboxState(state types.StateString) error {
|
||||||
|
return vcTypes.ErrNeedState
|
||||||
|
}
|
||||||
|
|
||||||
|
+ s.Logger().Debugf("Setting sandbox state from %v to %v", s.state.State, state)
|
||||||
|
// update in-memory state
|
||||||
|
s.state.State = state
|
||||||
|
|
||||||
|
if useOldStore(s.ctx) {
|
||||||
|
return s.store.Store(store.State, s.state)
|
||||||
|
+ } else {
|
||||||
|
+ return s.Save()
|
||||||
|
}
|
||||||
|
- return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const maxBlockIndex = 65535
|
||||||
|
@@ -2207,3 +2217,79 @@ func (s *Sandbox) GetPatchedOCISpec() *specs.Spec {
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+// health return current sandbox healthy or not
|
||||||
|
+// If qemu/kata-proxy/kata-agent process is abnormal,
|
||||||
|
+// s.agent.check() will return false
|
||||||
|
+func (s *Sandbox) health() bool {
|
||||||
|
+ err := s.agent.check()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return false
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return true
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// shouldForceDelete force delete the sandbox when kata-proxy and hypervisor process exit
|
||||||
|
+// already and current process is kata-runtime kill or kata-runtime delete
|
||||||
|
+func (s *Sandbox) shouldForceDelete() bool {
|
||||||
|
+ cmdline, err := utils.GetProcessCmdline(os.Getpid())
|
||||||
|
+ if err != nil {
|
||||||
|
+ s.Logger().Errorf("fail to get process cmdline: %v", err)
|
||||||
|
+ return false
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ proxyPid := s.agent.getProxyPid()
|
||||||
|
+ hypervisorPids := s.hypervisor.getPids()
|
||||||
|
+ if len(hypervisorPids) <= 0 {
|
||||||
|
+ s.Logger().Warnf("get hypervisor main pid fail")
|
||||||
|
+ return false
|
||||||
|
+ }
|
||||||
|
+ hypervisorMainPid := hypervisorPids[0]
|
||||||
|
+ hypervisorPath := s.hypervisor.hypervisorConfig().HypervisorPath
|
||||||
|
+ hypervisorName := filepath.Base(hypervisorPath)
|
||||||
|
+
|
||||||
|
+ if !utils.IsProcessRunning(proxyPid, KataProxyProcessName, s.id) && !utils.IsProcessRunning(hypervisorMainPid, hypervisorName, s.id) &&
|
||||||
|
+ strings.Contains(cmdline, "delete") && strings.Contains(cmdline, "force") {
|
||||||
|
+ return true
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return false
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *Sandbox) forceDeleteSandbox() {
|
||||||
|
+ for _, c := range s.containers {
|
||||||
|
+ // force delete all containers in the sandbox
|
||||||
|
+ c.forceDeleteContainer()
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ globalSandboxList.removeSandbox(s.id)
|
||||||
|
+
|
||||||
|
+ if s.monitor != nil {
|
||||||
|
+ s.monitor.stop()
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := s.hypervisor.cleanup(); err != nil {
|
||||||
|
+ s.Logger().WithError(err).Error("failed to force cleanup hypervisor resource")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ s.agent.cleanup(s)
|
||||||
|
+
|
||||||
|
+ if err := s.store.Delete(); err != nil {
|
||||||
|
+ s.Logger().WithError(err).Warn("sandbox force delete store failed")
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *Sandbox) setContainersState(state types.StateString) error {
|
||||||
|
+ if state == "" {
|
||||||
|
+ return vcTypes.ErrNeedState
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ for _, c := range s.containers {
|
||||||
|
+ if err := c.setContainerState(state); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/types/sandbox.go b/virtcontainers/types/sandbox.go
|
||||||
|
index 3b64b20a..5d586b21 100644
|
||||||
|
--- a/virtcontainers/types/sandbox.go
|
||||||
|
+++ b/virtcontainers/types/sandbox.go
|
||||||
|
@@ -28,6 +28,9 @@ const (
|
||||||
|
|
||||||
|
// StateStopped represents a sandbox/container that has been stopped.
|
||||||
|
StateStopped StateString = "stopped"
|
||||||
|
+
|
||||||
|
+ // StateUnhealthy represents a sandbox/container that's in abnormal state.
|
||||||
|
+ StateUnhealthy StateString = "unhealthy"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
@@ -90,17 +93,17 @@ func (state *StateString) validTransition(oldState StateString, newState StateSt
|
||||||
|
|
||||||
|
switch *state {
|
||||||
|
case StateReady:
|
||||||
|
- if newState == StateRunning || newState == StateStopped {
|
||||||
|
+ if newState == StateRunning || newState == StateStopped || newState == StateUnhealthy {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
case StateRunning:
|
||||||
|
- if newState == StatePaused || newState == StateStopped {
|
||||||
|
+ if newState == StatePaused || newState == StateStopped || newState == StateUnhealthy {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
case StatePaused:
|
||||||
|
- if newState == StateRunning || newState == StateStopped {
|
||||||
|
+ if newState == StateRunning || newState == StateStopped || newState == StateUnhealthy {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -108,6 +111,11 @@ func (state *StateString) validTransition(oldState StateString, newState StateSt
|
||||||
|
if newState == StateRunning {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+ case StateUnhealthy:
|
||||||
|
+ if newState == StateStopped {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Errorf("Can not move from %v to %v",
|
||||||
|
diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go
|
||||||
|
index 85c55489..2b555ebb 100644
|
||||||
|
--- a/virtcontainers/utils/utils.go
|
||||||
|
+++ b/virtcontainers/utils/utils.go
|
||||||
|
@@ -9,9 +9,13 @@ import (
|
||||||
|
"crypto/rand"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
+ "io/ioutil"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
+ "strconv"
|
||||||
|
+ "strings"
|
||||||
|
+ "syscall"
|
||||||
|
)
|
||||||
|
|
||||||
|
const cpBinaryName = "cp"
|
||||||
|
@@ -275,3 +279,45 @@ const (
|
||||||
|
MiB = KiB << 10
|
||||||
|
GiB = MiB << 10
|
||||||
|
)
|
||||||
|
+
|
||||||
|
+// Get process cmdline info by read /proc/<pid>/cmdline file
|
||||||
|
+func GetProcessCmdline(pid int) (cmdline string, err error) {
|
||||||
|
+ if pid <= 1 {
|
||||||
|
+ return "", fmt.Errorf("invalid pid number")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ bytes, err := ioutil.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "cmdline"))
|
||||||
|
+ if err != nil {
|
||||||
|
+ return "", err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return string(bytes), nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func IsProcessRunning(pid int, processName string, sandboxID string) bool {
|
||||||
|
+ if pid <= 0 {
|
||||||
|
+ return false
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ process, err := os.FindProcess(pid)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return false
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := process.Signal(syscall.Signal(0)); err != nil {
|
||||||
|
+ return false
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ cmdline, err := GetProcessCmdline(pid)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return false
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // If process's cmdline contains processName and sandboxID keyword,
|
||||||
|
+ // We think this process isn't be reused
|
||||||
|
+ if strings.Contains(cmdline, processName) && strings.Contains(cmdline, sandboxID) {
|
||||||
|
+ return true
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return false
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/vm.go b/virtcontainers/vm.go
|
||||||
|
index fcda1e97..8d27b1fe 100644
|
||||||
|
--- a/virtcontainers/vm.go
|
||||||
|
+++ b/virtcontainers/vm.go
|
||||||
|
@@ -191,7 +191,7 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) {
|
||||||
|
defer func() {
|
||||||
|
if err != nil {
|
||||||
|
virtLog.WithField("vm", id).WithError(err).Info("clean up vm")
|
||||||
|
- hypervisor.stopSandbox()
|
||||||
|
+ hypervisor.stopSandbox(false)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
@@ -333,7 +333,7 @@ func (v *VM) Disconnect() error {
|
||||||
|
func (v *VM) Stop() error {
|
||||||
|
v.logger().Info("stop vm")
|
||||||
|
|
||||||
|
- if err := v.hypervisor.stopSandbox(); err != nil {
|
||||||
|
+ if err := v.hypervisor.stopSandbox(false); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,177 @@
|
|||||||
|
From a1bf2e1c696b703935f4b81ca087a60cc2559464 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Mon, 27 Jul 2020 21:45:25 +0800
|
||||||
|
Subject: [PATCH 08/50] kata-runtime: fix kata-runtime resource left problem
|
||||||
|
|
||||||
|
reason: fix the following resource left problem
|
||||||
|
- sandbox stored files and directory are deleted before work container delete
|
||||||
|
- ignore StatusSandbox error, let stopSandbox function call be executed
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
cli/delete.go | 25 +++++++++++++++++++++++--
|
||||||
|
cli/delete_test.go | 15 +++++++++++++++
|
||||||
|
cli/state.go | 6 ++++++
|
||||||
|
virtcontainers/api.go | 4 +++-
|
||||||
|
4 files changed, 47 insertions(+), 3 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/cli/delete.go b/cli/delete.go
|
||||||
|
index 2f5586e5..871ac40d 100644
|
||||||
|
--- a/cli/delete.go
|
||||||
|
+++ b/cli/delete.go
|
||||||
|
@@ -10,6 +10,7 @@ import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
+ "strings"
|
||||||
|
|
||||||
|
"github.com/kata-containers/runtime/pkg/katautils"
|
||||||
|
vc "github.com/kata-containers/runtime/virtcontainers"
|
||||||
|
@@ -68,6 +69,12 @@ func delete(ctx context.Context, containerID string, force bool) error {
|
||||||
|
setExternalLoggers(ctx, kataLog)
|
||||||
|
span.SetTag("container", containerID)
|
||||||
|
|
||||||
|
+ // Remove the containerID to sandboxID mapping
|
||||||
|
+ // no matter what error return
|
||||||
|
+ defer func() {
|
||||||
|
+ _ = katautils.DelContainerIDMapping(ctx, containerID)
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
// Checks the MUST and MUST NOT from OCI runtime specification
|
||||||
|
status, sandboxID, err := getExistingContainerInfo(ctx, containerID)
|
||||||
|
if err != nil {
|
||||||
|
@@ -75,6 +82,15 @@ func delete(ctx context.Context, containerID string, force bool) error {
|
||||||
|
kataLog.Warnf("Failed to get container, force will not fail: %s", err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+ // If err info containers "no such file or directory, because pod_sandbox type
|
||||||
|
+ // container is deleted before pod_container type container, just return nil
|
||||||
|
+ // and let containerd delete container operations continue
|
||||||
|
+ if strings.Contains(err.Error(), "no such file or directory") {
|
||||||
|
+ kataLog.Warnf("pod_sandbox deleted before pod_container: %v", err)
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -123,7 +139,12 @@ func delete(ctx context.Context, containerID string, force bool) error {
|
||||||
|
}
|
||||||
|
case vc.PodContainer:
|
||||||
|
if err := deleteContainer(ctx, sandboxID, containerID, forceStop); err != nil {
|
||||||
|
- return err
|
||||||
|
+ // If err info containers "no such file or directory, because pod_sandbox type
|
||||||
|
+ // container is deleted before pod_container type container, just return nil
|
||||||
|
+ // and let containerd delete container operations continue
|
||||||
|
+ if !strings.Contains(err.Error(), "no such file or directory") {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("Invalid container type found")
|
||||||
|
@@ -134,7 +155,7 @@ func delete(ctx context.Context, containerID string, force bool) error {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
- return katautils.DelContainerIDMapping(ctx, containerID)
|
||||||
|
+ return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func deleteSandbox(ctx context.Context, sandboxID string, force bool) error {
|
||||||
|
diff --git a/cli/delete_test.go b/cli/delete_test.go
|
||||||
|
index a2455dee..ae421cd7 100644
|
||||||
|
--- a/cli/delete_test.go
|
||||||
|
+++ b/cli/delete_test.go
|
||||||
|
@@ -184,6 +184,9 @@ func TestDeleteSandbox(t *testing.T) {
|
||||||
|
assert.Error(err)
|
||||||
|
assert.True(vcmock.IsMockError(err))
|
||||||
|
|
||||||
|
+ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
testingImpl.StatusSandboxFunc = func(ctx context.Context, sandboxID string) (vc.SandboxStatus, error) {
|
||||||
|
return vc.SandboxStatus{
|
||||||
|
ID: sandbox.ID(),
|
||||||
|
@@ -201,6 +204,9 @@ func TestDeleteSandbox(t *testing.T) {
|
||||||
|
assert.Error(err)
|
||||||
|
assert.True(vcmock.IsMockError(err))
|
||||||
|
|
||||||
|
+ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
testingImpl.StopSandboxFunc = func(ctx context.Context, sandboxID string, force bool) (vc.VCSandbox, error) {
|
||||||
|
return sandbox, nil
|
||||||
|
}
|
||||||
|
@@ -213,6 +219,9 @@ func TestDeleteSandbox(t *testing.T) {
|
||||||
|
assert.Error(err)
|
||||||
|
assert.True(vcmock.IsMockError(err))
|
||||||
|
|
||||||
|
+ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
testingImpl.DeleteSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) {
|
||||||
|
return sandbox, nil
|
||||||
|
}
|
||||||
|
@@ -302,6 +311,9 @@ func TestDeleteSandboxRunning(t *testing.T) {
|
||||||
|
assert.Error(err)
|
||||||
|
assert.False(vcmock.IsMockError(err))
|
||||||
|
|
||||||
|
+ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
testingImpl.StatusSandboxFunc = func(ctx context.Context, sandboxID string) (vc.SandboxStatus, error) {
|
||||||
|
return vc.SandboxStatus{
|
||||||
|
ID: sandbox.ID(),
|
||||||
|
@@ -325,6 +337,9 @@ func TestDeleteSandboxRunning(t *testing.T) {
|
||||||
|
assert.Error(err)
|
||||||
|
assert.True(vcmock.IsMockError(err))
|
||||||
|
|
||||||
|
+ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
testingImpl.DeleteSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) {
|
||||||
|
return sandbox, nil
|
||||||
|
}
|
||||||
|
diff --git a/cli/state.go b/cli/state.go
|
||||||
|
index a2fcc12e..de843d34 100644
|
||||||
|
--- a/cli/state.go
|
||||||
|
+++ b/cli/state.go
|
||||||
|
@@ -11,6 +11,7 @@ import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
+ "strings"
|
||||||
|
|
||||||
|
"github.com/kata-containers/runtime/pkg/katautils"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
|
||||||
|
@@ -52,6 +53,11 @@ func state(ctx context.Context, containerID string) error {
|
||||||
|
// Checks the MUST and MUST NOT from OCI runtime specification
|
||||||
|
status, _, err := getExistingContainerInfo(ctx, containerID)
|
||||||
|
if err != nil {
|
||||||
|
+ // If err info containers "no such file or directory, because pod_sandbox type
|
||||||
|
+ // container is deleted before pod_container type container, just return nil
|
||||||
|
+ if strings.Contains(err.Error(), "no such file or directory") {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
|
||||||
|
index fa82d163..449a03e0 100644
|
||||||
|
--- a/virtcontainers/api.go
|
||||||
|
+++ b/virtcontainers/api.go
|
||||||
|
@@ -374,7 +374,9 @@ func StatusSandbox(ctx context.Context, sandboxID string) (SandboxStatus, error)
|
||||||
|
for _, container := range s.containers {
|
||||||
|
contStatus, err := statusContainer(s, container.id)
|
||||||
|
if err != nil {
|
||||||
|
- return SandboxStatus{}, err
|
||||||
|
+ // Since statusContainer may get error because of qemu process D or T,
|
||||||
|
+ // So just ignore this error and let StatusSandbox function return actually SandboxStatus
|
||||||
|
+ s.Logger().Warnf("Status container's status in sandbox get error: %v", contStatus)
|
||||||
|
}
|
||||||
|
|
||||||
|
contStatusList = append(contStatusList, contStatus)
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,132 @@
|
|||||||
|
From 508fd9b94b1b12be3167342b03a47f8f97245e9c Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 28 Jul 2020 10:53:30 +0800
|
||||||
|
Subject: [PATCH 09/50] kata-runtime: add kata-runtime global flag --debug
|
||||||
|
|
||||||
|
reason: add the same debug flag with runc to adapt to
|
||||||
|
containerd 1.2.0
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
cli/kata-env_test.go | 2 +-
|
||||||
|
cli/main.go | 6 +++++-
|
||||||
|
containerd-shim-v2/create.go | 2 +-
|
||||||
|
pkg/katautils/config.go | 4 ++--
|
||||||
|
pkg/katautils/config_test.go | 8 ++++----
|
||||||
|
5 files changed, 13 insertions(+), 9 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/cli/kata-env_test.go b/cli/kata-env_test.go
|
||||||
|
index b31b6cc2..75bb697f 100644
|
||||||
|
--- a/cli/kata-env_test.go
|
||||||
|
+++ b/cli/kata-env_test.go
|
||||||
|
@@ -178,7 +178,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
|
||||||
|
return "", oci.RuntimeConfig{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
- _, config, err = katautils.LoadConfiguration(configFile, true, false)
|
||||||
|
+ _, config, err = katautils.LoadConfiguration(configFile, true, false, false)
|
||||||
|
if err != nil {
|
||||||
|
return "", oci.RuntimeConfig{}, err
|
||||||
|
}
|
||||||
|
diff --git a/cli/main.go b/cli/main.go
|
||||||
|
index df16c5f3..1362a8fb 100644
|
||||||
|
--- a/cli/main.go
|
||||||
|
+++ b/cli/main.go
|
||||||
|
@@ -115,6 +115,10 @@ var runtimeFlags = []cli.Flag{
|
||||||
|
Name: "systemd-cgroup",
|
||||||
|
Usage: "enable systemd cgroup support, expects cgroupsPath to be of form \"slice:prefix:name\" for e.g. \"system.slice:runc:434234\"",
|
||||||
|
},
|
||||||
|
+ cli.BoolFlag{
|
||||||
|
+ Name: "debug",
|
||||||
|
+ Usage: "enable debug output for logging",
|
||||||
|
+ },
|
||||||
|
}
|
||||||
|
|
||||||
|
// runtimeCommands is the list of supported command-line (sub-)
|
||||||
|
@@ -334,7 +338,7 @@ func beforeSubcommands(c *cli.Context) error {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
- configFile, runtimeConfig, err = katautils.LoadConfiguration(c.GlobalString(configFilePathOption), ignoreConfigLogs, false)
|
||||||
|
+ configFile, runtimeConfig, err = katautils.LoadConfiguration(c.GlobalString(configFilePathOption), ignoreConfigLogs, false, c.GlobalBool("debug"))
|
||||||
|
if err != nil {
|
||||||
|
fatal(err)
|
||||||
|
}
|
||||||
|
diff --git a/containerd-shim-v2/create.go b/containerd-shim-v2/create.go
|
||||||
|
index affdbae2..9749073d 100644
|
||||||
|
--- a/containerd-shim-v2/create.go
|
||||||
|
+++ b/containerd-shim-v2/create.go
|
||||||
|
@@ -167,7 +167,7 @@ func loadRuntimeConfig(s *service, r *taskAPI.CreateTaskRequest, anno map[string
|
||||||
|
configPath = os.Getenv("KATA_CONF_FILE")
|
||||||
|
}
|
||||||
|
|
||||||
|
- _, runtimeConfig, err := katautils.LoadConfiguration(configPath, false, true)
|
||||||
|
+ _, runtimeConfig, err := katautils.LoadConfiguration(configPath, false, true, false)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
||||||
|
index 349e667f..448d23ac 100644
|
||||||
|
--- a/pkg/katautils/config.go
|
||||||
|
+++ b/pkg/katautils/config.go
|
||||||
|
@@ -1141,7 +1141,7 @@ func initConfig() (config oci.RuntimeConfig, err error) {
|
||||||
|
//
|
||||||
|
// All paths are resolved fully meaning if this function does not return an
|
||||||
|
// error, all paths are valid at the time of the call.
|
||||||
|
-func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolvedConfigPath string, config oci.RuntimeConfig, err error) {
|
||||||
|
+func LoadConfiguration(configPath string, ignoreLogging, builtIn bool, debugFlag bool) (resolvedConfigPath string, config oci.RuntimeConfig, err error) {
|
||||||
|
|
||||||
|
config, err = initConfig()
|
||||||
|
if err != nil {
|
||||||
|
@@ -1154,7 +1154,7 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved
|
||||||
|
}
|
||||||
|
|
||||||
|
config.Debug = tomlConf.Runtime.Debug
|
||||||
|
- if !tomlConf.Runtime.Debug {
|
||||||
|
+ if !tomlConf.Runtime.Debug && !debugFlag {
|
||||||
|
// If debug is not required, switch back to the original
|
||||||
|
// default log priority, otherwise continue in debug mode.
|
||||||
|
kataUtilsLogger.Logger.Level = originalLoggerLevel
|
||||||
|
diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go
|
||||||
|
index 2eae1f6a..31afcca6 100644
|
||||||
|
--- a/pkg/katautils/config_test.go
|
||||||
|
+++ b/pkg/katautils/config_test.go
|
||||||
|
@@ -264,7 +264,7 @@ func testLoadConfiguration(t *testing.T, dir string,
|
||||||
|
assert.NoError(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
- resolvedConfigPath, config, err := LoadConfiguration(file, ignoreLogging, false)
|
||||||
|
+ resolvedConfigPath, config, err := LoadConfiguration(file, ignoreLogging, false, false)
|
||||||
|
if expectFail {
|
||||||
|
assert.Error(t, err)
|
||||||
|
|
||||||
|
@@ -578,7 +578,7 @@ func TestMinimalRuntimeConfig(t *testing.T) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
- _, config, err := LoadConfiguration(configPath, false, false)
|
||||||
|
+ _, config, err := LoadConfiguration(configPath, false, false, false)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("Expected loadConfiguration to fail as shim path does not exist: %+v", config)
|
||||||
|
}
|
||||||
|
@@ -608,7 +608,7 @@ func TestMinimalRuntimeConfig(t *testing.T) {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
- _, config, err = LoadConfiguration(configPath, false, false)
|
||||||
|
+ _, config, err = LoadConfiguration(configPath, false, false, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
@@ -748,7 +748,7 @@ func TestMinimalRuntimeConfigWithVsock(t *testing.T) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
- _, config, err := LoadConfiguration(configPath, false, false)
|
||||||
|
+ _, config, err := LoadConfiguration(configPath, false, false, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,172 @@
|
|||||||
|
From 76cbca91608e94c1855705ad1a8d06ffa2273115 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 28 Jul 2020 18:18:54 +0800
|
||||||
|
Subject: [PATCH 10/50] kata-runtime: fix kata-shim pid reused problem
|
||||||
|
|
||||||
|
reason: If kata-shim process exit and it's pid reused by other process,
|
||||||
|
it may cause kill other proecss and cause some problem.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/api.go | 2 +-
|
||||||
|
virtcontainers/container.go | 6 +++---
|
||||||
|
virtcontainers/shim.go | 21 +++++++++++++++++----
|
||||||
|
virtcontainers/shim_test.go | 10 +++++-----
|
||||||
|
4 files changed, 26 insertions(+), 13 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
|
||||||
|
index 449a03e0..5e8c9c9e 100644
|
||||||
|
--- a/virtcontainers/api.go
|
||||||
|
+++ b/virtcontainers/api.go
|
||||||
|
@@ -611,7 +611,7 @@ func statusContainer(sandbox *Sandbox, containerID string) (ContainerStatus, err
|
||||||
|
return ContainerStatus{}, fmt.Errorf("sandbox has beed stopped exceptionally")
|
||||||
|
}
|
||||||
|
|
||||||
|
- running, err := isShimRunning(container.process.Pid)
|
||||||
|
+ running, err := isShimRunning(container.process.Pid, containerID)
|
||||||
|
if err != nil {
|
||||||
|
return ContainerStatus{}, err
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/container.go b/virtcontainers/container.go
|
||||||
|
index 9485e708..75f590eb 100644
|
||||||
|
--- a/virtcontainers/container.go
|
||||||
|
+++ b/virtcontainers/container.go
|
||||||
|
@@ -1063,7 +1063,7 @@ func (c *Container) stop(force bool) error {
|
||||||
|
|
||||||
|
// If shim is still running something went wrong
|
||||||
|
// Make sure we stop the shim process
|
||||||
|
- if running, _ := isShimRunning(c.process.Pid); running {
|
||||||
|
+ if running, _ := isShimRunning(c.process.Pid, c.id); running {
|
||||||
|
l := c.Logger()
|
||||||
|
l.Error("Failed to stop container so stopping dangling shim")
|
||||||
|
if err := stopShim(c.process.Pid); err != nil {
|
||||||
|
@@ -1081,7 +1081,7 @@ func (c *Container) stop(force bool) error {
|
||||||
|
// However, if the signal didn't reach its goal, the caller still
|
||||||
|
// expects this container to be stopped, that's why we should not
|
||||||
|
// return an error, but instead try to kill it forcefully.
|
||||||
|
- if err := waitForShim(c.process.Pid); err != nil {
|
||||||
|
+ if err := waitForShim(c.process.Pid, c.id); err != nil {
|
||||||
|
// Force the container to be killed.
|
||||||
|
if err := c.kill(syscall.SIGKILL, true); err != nil && !force {
|
||||||
|
return err
|
||||||
|
@@ -1091,7 +1091,7 @@ func (c *Container) stop(force bool) error {
|
||||||
|
// to succeed. Indeed, we have already given a second chance
|
||||||
|
// to the container by trying to kill it with SIGKILL, there
|
||||||
|
// is no reason to try to go further if we got an error.
|
||||||
|
- if err := waitForShim(c.process.Pid); err != nil && !force {
|
||||||
|
+ if err := waitForShim(c.process.Pid, c.id); err != nil && !force {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/shim.go b/virtcontainers/shim.go
|
||||||
|
index 8ec7458b..6f784a03 100644
|
||||||
|
--- a/virtcontainers/shim.go
|
||||||
|
+++ b/virtcontainers/shim.go
|
||||||
|
@@ -9,11 +9,13 @@ import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
+ "strings"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/types"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||||
|
"github.com/mitchellh/mapstructure"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
)
|
||||||
|
@@ -227,7 +229,7 @@ func startShim(args []string, params ShimParams) (int, error) {
|
||||||
|
return cmd.Process.Pid, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
-func isShimRunning(pid int) (bool, error) {
|
||||||
|
+func isShimRunning(pid int, containerID string) (bool, error) {
|
||||||
|
if pid <= 0 {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
@@ -241,19 +243,30 @@ func isShimRunning(pid int) (bool, error) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
- return true, nil
|
||||||
|
+ cmdline, err := utils.GetProcessCmdline(pid)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return false, nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // If process's cmdline contains kata-shim and containerID keyword, we think this process pid isn't be reused
|
||||||
|
+ if strings.Contains(cmdline, "kata-shim") && strings.Contains(cmdline, containerID) {
|
||||||
|
+ return true, nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ shimLogger().Errorf("%d process isn't a kata-shim process", pid)
|
||||||
|
+ return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitForShim waits for the end of the shim unless it reaches the timeout
|
||||||
|
// first, returning an error in that case.
|
||||||
|
-func waitForShim(pid int) error {
|
||||||
|
+func waitForShim(pid int, containerID string) error {
|
||||||
|
if pid <= 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
tInit := time.Now()
|
||||||
|
for {
|
||||||
|
- running, err := isShimRunning(pid)
|
||||||
|
+ running, err := isShimRunning(pid, containerID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/shim_test.go b/virtcontainers/shim_test.go
|
||||||
|
index e9bd027c..62471311 100644
|
||||||
|
--- a/virtcontainers/shim_test.go
|
||||||
|
+++ b/virtcontainers/shim_test.go
|
||||||
|
@@ -190,7 +190,7 @@ func TestStopShimSuccessfulProcessRunning(t *testing.T) {
|
||||||
|
|
||||||
|
func testIsShimRunning(t *testing.T, pid int, expected bool) {
|
||||||
|
assert := assert.New(t)
|
||||||
|
- running, err := isShimRunning(pid)
|
||||||
|
+ running, err := isShimRunning(pid, containerID)
|
||||||
|
assert.NoError(err)
|
||||||
|
assert.Equal(running, expected)
|
||||||
|
}
|
||||||
|
@@ -205,7 +205,7 @@ func TestIsShimRunningTrue(t *testing.T) {
|
||||||
|
cmd := testRunSleep999AndGetCmd(t)
|
||||||
|
assert := assert.New(t)
|
||||||
|
|
||||||
|
- testIsShimRunning(t, cmd.Process.Pid, true)
|
||||||
|
+ testIsShimRunning(t, cmd.Process.Pid, false)
|
||||||
|
|
||||||
|
err := syscall.Kill(cmd.Process.Pid, syscall.SIGKILL)
|
||||||
|
assert.NoError(err)
|
||||||
|
@@ -216,7 +216,7 @@ func TestWaitForShimInvalidPidSuccessful(t *testing.T) {
|
||||||
|
assert := assert.New(t)
|
||||||
|
|
||||||
|
for _, val := range wrongValuesList {
|
||||||
|
- err := waitForShim(val)
|
||||||
|
+ err := waitForShim(val, containerID)
|
||||||
|
assert.NoError(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
@@ -224,7 +224,7 @@ func TestWaitForShimInvalidPidSuccessful(t *testing.T) {
|
||||||
|
func TestWaitForShimNotRunningSuccessful(t *testing.T) {
|
||||||
|
pid := testRunSleep0AndGetPid(t)
|
||||||
|
assert := assert.New(t)
|
||||||
|
- assert.NoError(waitForShim(pid))
|
||||||
|
+ assert.NoError(waitForShim(pid, containerID))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWaitForShimRunningForTooLongFailure(t *testing.T) {
|
||||||
|
@@ -232,6 +232,6 @@ func TestWaitForShimRunningForTooLongFailure(t *testing.T) {
|
||||||
|
assert := assert.New(t)
|
||||||
|
|
||||||
|
waitForShimTimeout = 0.1
|
||||||
|
- assert.Error(waitForShim(cmd.Process.Pid))
|
||||||
|
+ assert.NoError(waitForShim(cmd.Process.Pid, containerID))
|
||||||
|
assert.NoError(syscall.Kill(cmd.Process.Pid, syscall.SIGKILL))
|
||||||
|
}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,119 @@
|
|||||||
|
From 0aeff2632eac58eefdc8ae438891303332831ec5 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 28 Jul 2020 20:48:24 +0800
|
||||||
|
Subject: [PATCH 11/50] kata-runtime: check the process info before send
|
||||||
|
SIGKILL
|
||||||
|
|
||||||
|
reason: In order to avoid the pid reuse problem, check the
|
||||||
|
process info before send SIGKILL signal to process.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/kata_proxy.go | 18 ++++++++++++++++++
|
||||||
|
virtcontainers/qemu.go | 5 +++++
|
||||||
|
virtcontainers/shim.go | 9 +++++++++
|
||||||
|
virtcontainers/shim_test.go | 8 ++++----
|
||||||
|
4 files changed, 36 insertions(+), 4 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/kata_proxy.go b/virtcontainers/kata_proxy.go
|
||||||
|
index e04b4cff..ed272bad 100644
|
||||||
|
--- a/virtcontainers/kata_proxy.go
|
||||||
|
+++ b/virtcontainers/kata_proxy.go
|
||||||
|
@@ -6,8 +6,12 @@
|
||||||
|
package virtcontainers
|
||||||
|
|
||||||
|
import (
|
||||||
|
+ "fmt"
|
||||||
|
"os/exec"
|
||||||
|
+ "strings"
|
||||||
|
"syscall"
|
||||||
|
+
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||||
|
)
|
||||||
|
|
||||||
|
// This is the Kata Containers implementation of the proxy interface.
|
||||||
|
@@ -61,6 +65,20 @@ func (p *kataProxy) start(params proxyParams) (int, string, error) {
|
||||||
|
|
||||||
|
// stop is kataProxy stop implementation for proxy interface.
|
||||||
|
func (p *kataProxy) stop(pid int) error {
|
||||||
|
+ if pid <= 1 {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // check process info before send SIGKILL signal
|
||||||
|
+ cmdline, err := utils.GetProcessCmdline(pid)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return fmt.Errorf("get kata-proxy %d cmdline error: %v", pid, err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if !strings.Contains(cmdline, KataProxyProcessName) {
|
||||||
|
+ return fmt.Errorf("%d is not kata-proxy process, don't kill wrong process", pid)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
// Signal the proxy with SIGTERM.
|
||||||
|
return syscall.Kill(pid, syscall.SIGTERM)
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
|
||||||
|
index 4b15d968..4789101d 100644
|
||||||
|
--- a/virtcontainers/qemu.go
|
||||||
|
+++ b/virtcontainers/qemu.go
|
||||||
|
@@ -967,6 +967,11 @@ func (q *qemu) stopSandbox(force bool) error {
|
||||||
|
return fmt.Errorf("force kill qemu process pid is invalid")
|
||||||
|
}
|
||||||
|
|
||||||
|
+ cmdline, _ := utils.GetProcessCmdline(qemuMainPid)
|
||||||
|
+ if !strings.Contains(cmdline, string(QemuHypervisor)) {
|
||||||
|
+ return fmt.Errorf("force kill %d process is not qemu process, don't kill wrong process", qemuMainPid)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
_ = syscall.Kill(qemuMainPid, syscall.SIGKILL)
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/shim.go b/virtcontainers/shim.go
|
||||||
|
index 6f784a03..b192b258 100644
|
||||||
|
--- a/virtcontainers/shim.go
|
||||||
|
+++ b/virtcontainers/shim.go
|
||||||
|
@@ -143,6 +143,15 @@ func stopShim(pid int) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+ cmdline, err := utils.GetProcessCmdline(pid)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if !strings.Contains(cmdline, "kata-shim") {
|
||||||
|
+ return fmt.Errorf("%d process is not kata-shim process, don't kill wrong process", pid)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
if err := signalShim(pid, syscall.SIGKILL); err != nil && err != syscall.ESRCH {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/shim_test.go b/virtcontainers/shim_test.go
|
||||||
|
index 62471311..dc15eab0 100644
|
||||||
|
--- a/virtcontainers/shim_test.go
|
||||||
|
+++ b/virtcontainers/shim_test.go
|
||||||
|
@@ -176,16 +176,16 @@ func testRunSleep999AndGetCmd(t *testing.T) *exec.Cmd {
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
-func TestStopShimSuccessfulProcessNotRunning(t *testing.T) {
|
||||||
|
+func TestStopShimFailProcessNotRunning(t *testing.T) {
|
||||||
|
assert := assert.New(t)
|
||||||
|
pid := testRunSleep0AndGetPid(t)
|
||||||
|
- assert.NoError(stopShim(pid))
|
||||||
|
+ assert.Error(stopShim(pid))
|
||||||
|
}
|
||||||
|
|
||||||
|
-func TestStopShimSuccessfulProcessRunning(t *testing.T) {
|
||||||
|
+func TestStopShimFailProcessRunning(t *testing.T) {
|
||||||
|
assert := assert.New(t)
|
||||||
|
cmd := testRunSleep999AndGetCmd(t)
|
||||||
|
- assert.NoError(stopShim(cmd.Process.Pid))
|
||||||
|
+ assert.Error(stopShim(cmd.Process.Pid))
|
||||||
|
}
|
||||||
|
|
||||||
|
func testIsShimRunning(t *testing.T, pid int, expected bool) {
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,37 @@
|
|||||||
|
From 441d80f55f4dc5efb4c92d91608a3c8db3d087cb Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 28 Jul 2020 21:43:15 +0800
|
||||||
|
Subject: [PATCH 12/50] kata-runtime: truncate the log.json file before
|
||||||
|
kata-runtime subcommand executed
|
||||||
|
|
||||||
|
reason: since we have redirect the kata-runtime log to /var/log/messages, and avoid the
|
||||||
|
path of log.json file to be large in the tmpfs, so we truncate the log.json file every
|
||||||
|
time before subcommand is executed.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
cli/main.go | 8 ++++++++
|
||||||
|
1 file changed, 8 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/cli/main.go b/cli/main.go
|
||||||
|
index 1362a8fb..5eb2fb19 100644
|
||||||
|
--- a/cli/main.go
|
||||||
|
+++ b/cli/main.go
|
||||||
|
@@ -300,6 +300,14 @@ func beforeSubcommands(c *cli.Context) error {
|
||||||
|
ignoreConfigLogs = true
|
||||||
|
} else {
|
||||||
|
if path := c.GlobalString("log"); path != "" {
|
||||||
|
+ // since we have redirect the kata-runtime log to /var/log/messages, and avoid the
|
||||||
|
+ // path of log.json file to be large in the tmpfs, so we truncate the log.json file
|
||||||
|
+ // every time before subcommand is executed.
|
||||||
|
+ if path != "/dev/null" && katautils.FileExists(path) {
|
||||||
|
+ if err := os.Truncate(path, 0); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND|os.O_SYNC, 0640)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,98 @@
|
|||||||
|
From fd63d26a5b0542f35d61b0c19c80795f052b4518 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 28 Jul 2020 22:05:44 +0800
|
||||||
|
Subject: [PATCH 13/50] kata-runtime: get container info by containerID prefix
|
||||||
|
|
||||||
|
reason: get container info by containerID prefix, so we just
|
||||||
|
need to input the prefix containerID when call kata-runtime
|
||||||
|
subcommand
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
cli/oci.go | 35 +++++++++++++++++++++++++++++++++++
|
||||||
|
pkg/katautils/oci.go | 5 +++++
|
||||||
|
2 files changed, 40 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/cli/oci.go b/cli/oci.go
|
||||||
|
index 8ffac2df..bf962d03 100644
|
||||||
|
--- a/cli/oci.go
|
||||||
|
+++ b/cli/oci.go
|
||||||
|
@@ -9,6 +9,7 @@ import (
|
||||||
|
"bufio"
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
+ "io/ioutil"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
@@ -25,6 +26,8 @@ const (
|
||||||
|
// Filesystem type corresponding to CGROUP_SUPER_MAGIC as listed
|
||||||
|
// here: http://man7.org/linux/man-pages/man2/statfs.2.html
|
||||||
|
cgroupFsType = 0x27e0eb
|
||||||
|
+
|
||||||
|
+ maxIDLength = 64
|
||||||
|
)
|
||||||
|
|
||||||
|
var cgroupsDirPath string
|
||||||
|
@@ -38,6 +41,14 @@ func getContainerInfo(ctx context.Context, containerID string) (vc.ContainerStat
|
||||||
|
return vc.ContainerStatus{}, "", fmt.Errorf("Missing container ID")
|
||||||
|
}
|
||||||
|
|
||||||
|
+ if len(containerID) < maxIDLength {
|
||||||
|
+ fullContainerID, err := getContainerIDbyPrefix(containerID)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return vc.ContainerStatus{}, "", err
|
||||||
|
+ }
|
||||||
|
+ containerID = fullContainerID
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
sandboxID, err := katautils.FetchContainerIDMapping(containerID)
|
||||||
|
if err != nil {
|
||||||
|
return vc.ContainerStatus{}, "", err
|
||||||
|
@@ -211,3 +222,27 @@ func getCgroupsDirPath(mountInfoFile string) (string, error) {
|
||||||
|
|
||||||
|
return cgroupRootPath, nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func getContainerIDbyPrefix(prefix string) (string, error) {
|
||||||
|
+ files, err := ioutil.ReadDir(katautils.GetCtrsMapTreePath())
|
||||||
|
+ if err != nil {
|
||||||
|
+ return "", err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ containers := []string{}
|
||||||
|
+ for _, file := range files {
|
||||||
|
+ if file.IsDir() && strings.HasPrefix(file.Name(), prefix) {
|
||||||
|
+ containers = append(containers, file.Name())
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if len(containers) == 0 {
|
||||||
|
+ return "", fmt.Errorf("no such container ID (%v)", prefix)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if len(containers) > 1 {
|
||||||
|
+ return "", fmt.Errorf("multiple containers found (%v)", prefix)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return containers[0], nil
|
||||||
|
+}
|
||||||
|
diff --git a/pkg/katautils/oci.go b/pkg/katautils/oci.go
|
||||||
|
index 6de8101e..1334af35 100644
|
||||||
|
--- a/pkg/katautils/oci.go
|
||||||
|
+++ b/pkg/katautils/oci.go
|
||||||
|
@@ -25,6 +25,11 @@ func SetCtrsMapTreePath(path string) {
|
||||||
|
ctrsMapTreePath = path
|
||||||
|
}
|
||||||
|
|
||||||
|
+// GetCtrsMapTreePath return the containerID to SandboxID mapping dir
|
||||||
|
+func GetCtrsMapTreePath() string {
|
||||||
|
+ return ctrsMapTreePath
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// doUpdatePath returns whether a ctrsMapTreePath needs to be updated with a rootless prefix
|
||||||
|
func doUpdatePath() bool {
|
||||||
|
return rootless.IsRootless() && !strings.HasPrefix(ctrsMapTreePath, rootless.GetRootlessDir())
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,76 @@
|
|||||||
|
From 1bd3cb85a1cf0e94b3280412d5fb47cecc4721fd Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Wed, 29 Jul 2020 10:42:49 +0800
|
||||||
|
Subject: [PATCH 14/50] kata-runtime: add self defined annotations framework
|
||||||
|
|
||||||
|
reason: add self defined annotations framework to pass some
|
||||||
|
self defined resource for sandbox
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/pkg/oci/utils.go | 38 ++++++++++++++++++++++++++++++++++++++
|
||||||
|
1 file changed, 38 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
||||||
|
index cd8d48ce..05181efd 100644
|
||||||
|
--- a/virtcontainers/pkg/oci/utils.go
|
||||||
|
+++ b/virtcontainers/pkg/oci/utils.go
|
||||||
|
@@ -33,6 +33,10 @@ type annotationContainerType struct {
|
||||||
|
containerType vc.ContainerType
|
||||||
|
}
|
||||||
|
|
||||||
|
+type annotationHandler func(value string) error
|
||||||
|
+
|
||||||
|
+var annotationHandlerList = map[string]annotationHandler{}
|
||||||
|
+
|
||||||
|
var (
|
||||||
|
// ErrNoLinux is an error for missing Linux sections in the OCI configuration file.
|
||||||
|
ErrNoLinux = errors.New("missing Linux section")
|
||||||
|
@@ -342,6 +346,10 @@ func addAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) error {
|
||||||
|
if err := addAgentConfigOverrides(ocispec, config); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+ if err := addOtherSandboxAnnotation(ocispec, config); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -1016,3 +1024,33 @@ func GetOCIConfig(status vc.ContainerStatus) (specs.Spec, error) {
|
||||||
|
|
||||||
|
return *status.Spec, nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+// validateOtherSandboxAnnotations validate the value of annotation
|
||||||
|
+func validateOtherSandboxAnnotations(annotation, value string) error {
|
||||||
|
+ validateHandler, ok := annotationHandlerList[annotation]
|
||||||
|
+ if !ok {
|
||||||
|
+ return fmt.Errorf("unsupport Sandbox annotation type")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return validateHandler(value)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// addOtherSandboxAnnotation add self defined annotation for sandbox
|
||||||
|
+func addOtherSandboxAnnotation(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error {
|
||||||
|
+ otherSandboxAnnotationsKey := []string{}
|
||||||
|
+
|
||||||
|
+ for _, a := range otherSandboxAnnotationsKey {
|
||||||
|
+ value, ok := ocispec.Annotations[a]
|
||||||
|
+ if !ok {
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := validateOtherSandboxAnnotations(a, value); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ sbConfig.Annotations[a] = value
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,143 @@
|
|||||||
|
From c0a33c4584e1fbd9b39ff1ca3ed632efe85de65e Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Sun, 2 Aug 2020 15:51:10 +0800
|
||||||
|
Subject: [PATCH 15/50] kata-runtime: add reuse hypervisor cpu and memory
|
||||||
|
feature
|
||||||
|
|
||||||
|
reason: If default hypervisor cpu and memory is set too large,
|
||||||
|
which may waste resource, so we add enable_reuse_cpu_memory
|
||||||
|
config in the configuration.toml file to choose share the
|
||||||
|
hypervisor cpu and memory with container or not.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
cli/config/configuration-qemu.toml.in | 5 +++++
|
||||||
|
pkg/katautils/config.go | 2 ++
|
||||||
|
virtcontainers/hypervisor.go | 3 +++
|
||||||
|
virtcontainers/persist.go | 2 ++
|
||||||
|
virtcontainers/persist/api/config.go | 3 +++
|
||||||
|
virtcontainers/sandbox.go | 22 ++++++++++++++++++----
|
||||||
|
6 files changed, 33 insertions(+), 4 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in
|
||||||
|
index 46ce7d9b..82461732 100644
|
||||||
|
--- a/cli/config/configuration-qemu.toml.in
|
||||||
|
+++ b/cli/config/configuration-qemu.toml.in
|
||||||
|
@@ -95,6 +95,11 @@ default_memory = @DEFMEMSZ@
|
||||||
|
# Default false
|
||||||
|
#enable_virtio_mem = true
|
||||||
|
|
||||||
|
+# Specifies share hypervisor default cpu and memory resource
|
||||||
|
+# between hypervisor and container process.
|
||||||
|
+# Default false
|
||||||
|
+#enable_reuse_cpu_memory = false
|
||||||
|
+
|
||||||
|
# Disable block device from being used for a container's rootfs.
|
||||||
|
# In case of a storage driver like devicemapper where a container's
|
||||||
|
# root file system is backed by a block device, the block device is passed
|
||||||
|
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
||||||
|
index 448d23ac..d1883c94 100644
|
||||||
|
--- a/pkg/katautils/config.go
|
||||||
|
+++ b/pkg/katautils/config.go
|
||||||
|
@@ -112,6 +112,7 @@ type hypervisor struct {
|
||||||
|
MemorySize uint32 `toml:"default_memory"`
|
||||||
|
MemSlots uint32 `toml:"memory_slots"`
|
||||||
|
MemOffset uint32 `toml:"memory_offset"`
|
||||||
|
+ EnableCPUMemoryReuse bool `toml:"enable_reuse_cpu_memory"`
|
||||||
|
DefaultBridges uint32 `toml:"default_bridges"`
|
||||||
|
Msize9p uint32 `toml:"msize_9p"`
|
||||||
|
PCIeRootPort uint32 `toml:"pcie_root_port"`
|
||||||
|
@@ -640,6 +641,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
||||||
|
MemorySize: h.defaultMemSz(),
|
||||||
|
MemSlots: h.defaultMemSlots(),
|
||||||
|
MemOffset: h.defaultMemOffset(),
|
||||||
|
+ EnableCPUMemoryReuse: h.EnableCPUMemoryReuse,
|
||||||
|
VirtioMem: h.VirtioMem,
|
||||||
|
EntropySource: h.GetEntropySource(),
|
||||||
|
DefaultBridges: h.defaultBridges(),
|
||||||
|
diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go
|
||||||
|
index fd7d1f8e..5f8d24f9 100644
|
||||||
|
--- a/virtcontainers/hypervisor.go
|
||||||
|
+++ b/virtcontainers/hypervisor.go
|
||||||
|
@@ -250,6 +250,9 @@ type HypervisorConfig struct {
|
||||||
|
// MemOffset specifies memory space for nvdimm device
|
||||||
|
MemOffset uint32
|
||||||
|
|
||||||
|
+ // Enable hypervisor cpu and memory resource share with container
|
||||||
|
+ EnableCPUMemoryReuse bool
|
||||||
|
+
|
||||||
|
// VirtioFSCacheSize is the DAX cache size in MiB
|
||||||
|
VirtioFSCacheSize uint32
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go
|
||||||
|
index d96a3890..aef4d18d 100644
|
||||||
|
--- a/virtcontainers/persist.go
|
||||||
|
+++ b/virtcontainers/persist.go
|
||||||
|
@@ -214,6 +214,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
|
||||||
|
Msize9p: sconfig.HypervisorConfig.Msize9p,
|
||||||
|
MemSlots: sconfig.HypervisorConfig.MemSlots,
|
||||||
|
MemOffset: sconfig.HypervisorConfig.MemOffset,
|
||||||
|
+ EnableCPUMemoryReuse: sconfig.HypervisorConfig.EnableCPUMemoryReuse,
|
||||||
|
VirtioMem: sconfig.HypervisorConfig.VirtioMem,
|
||||||
|
VirtioFSCacheSize: sconfig.HypervisorConfig.VirtioFSCacheSize,
|
||||||
|
KernelPath: sconfig.HypervisorConfig.KernelPath,
|
||||||
|
@@ -503,6 +504,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
|
||||||
|
Msize9p: hconf.Msize9p,
|
||||||
|
MemSlots: hconf.MemSlots,
|
||||||
|
MemOffset: hconf.MemOffset,
|
||||||
|
+ EnableCPUMemoryReuse: hconf.EnableCPUMemoryReuse,
|
||||||
|
VirtioMem: hconf.VirtioMem,
|
||||||
|
VirtioFSCacheSize: hconf.VirtioFSCacheSize,
|
||||||
|
KernelPath: hconf.KernelPath,
|
||||||
|
diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go
|
||||||
|
index 34a5fd0f..a3c6ec91 100644
|
||||||
|
--- a/virtcontainers/persist/api/config.go
|
||||||
|
+++ b/virtcontainers/persist/api/config.go
|
||||||
|
@@ -35,6 +35,9 @@ type HypervisorConfig struct {
|
||||||
|
// MemOffset specifies memory space for nvdimm device
|
||||||
|
MemOffset uint32
|
||||||
|
|
||||||
|
+ // Enable hypervisor cpu and memory resource share with container
|
||||||
|
+ EnableCPUMemoryReuse bool
|
||||||
|
+
|
||||||
|
// VirtioFSCacheSize is the DAX cache size in MiB
|
||||||
|
VirtioFSCacheSize uint32
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index 78188ed7..e766d1f7 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -1833,12 +1833,26 @@ func (s *Sandbox) updateResources() error {
|
||||||
|
}
|
||||||
|
|
||||||
|
sandboxVCPUs := s.calculateSandboxCPUs()
|
||||||
|
- // Add default vcpus for sandbox
|
||||||
|
- sandboxVCPUs += s.hypervisor.hypervisorConfig().NumVCPUs
|
||||||
|
+ // Share the hypervisor vcpu with container process
|
||||||
|
+ if s.config.HypervisorConfig.EnableCPUMemoryReuse {
|
||||||
|
+ if sandboxVCPUs <= s.config.HypervisorConfig.NumVCPUs {
|
||||||
|
+ sandboxVCPUs = s.config.HypervisorConfig.NumVCPUs
|
||||||
|
+ }
|
||||||
|
+ } else {
|
||||||
|
+ // Add default vcpus for sandbox
|
||||||
|
+ sandboxVCPUs += s.hypervisor.hypervisorConfig().NumVCPUs
|
||||||
|
+ }
|
||||||
|
|
||||||
|
sandboxMemoryByte := s.calculateSandboxMemory()
|
||||||
|
- // Add default / rsvd memory for sandbox.
|
||||||
|
- sandboxMemoryByte += int64(s.hypervisor.hypervisorConfig().MemorySize) << utils.MibToBytesShift
|
||||||
|
+ // Share the hypervisor memory with container process
|
||||||
|
+ if s.config.HypervisorConfig.EnableCPUMemoryReuse {
|
||||||
|
+ if sandboxMemoryByte <= (int64(s.config.HypervisorConfig.MemorySize) << utils.MibToBytesShift) {
|
||||||
|
+ sandboxMemoryByte = int64(s.config.HypervisorConfig.MemorySize) << utils.MibToBytesShift
|
||||||
|
+ }
|
||||||
|
+ } else {
|
||||||
|
+ // Add default / rsvd memory for sandbox.
|
||||||
|
+ sandboxMemoryByte += int64(s.hypervisor.hypervisorConfig().MemorySize) << utils.MibToBytesShift
|
||||||
|
+ }
|
||||||
|
|
||||||
|
// Update VCPUs
|
||||||
|
s.Logger().WithField("cpus-sandbox", sandboxVCPUs).Debugf("Request to hypervisor to update vCPUs")
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,288 @@
|
|||||||
|
From dc9de8bb181e2cec2f3e0a76d02833fef45b46af Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Thu, 6 Aug 2020 09:28:34 -0400
|
||||||
|
Subject: [PATCH 16/50] virtcontainers: fix hotplug huge size memory cause
|
||||||
|
agent hang bug
|
||||||
|
|
||||||
|
fixes: #2872
|
||||||
|
|
||||||
|
reason: If hotplug huge size memory into kata VM at once time,
|
||||||
|
guest kernel will allocate some extra memory for memory management,
|
||||||
|
which may cause kata-agent hang and out of responding.
|
||||||
|
And hotplug more memory into VM, more extra memory is needed.
|
||||||
|
|
||||||
|
Inorder to solve this problem, we divide hotplug huge memory into
|
||||||
|
two steps. First, hotplug the max allowed memory into VM and wait
|
||||||
|
all first step hotplugged memory online. Second Step, hotplug the
|
||||||
|
left memory into VM.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/acrn.go | 4 ++++
|
||||||
|
virtcontainers/agent.go | 3 ++-
|
||||||
|
virtcontainers/clh.go | 4 ++++
|
||||||
|
virtcontainers/fc.go | 4 ++++
|
||||||
|
virtcontainers/hypervisor.go | 3 +++
|
||||||
|
virtcontainers/kata_agent.go | 4 ++--
|
||||||
|
virtcontainers/kata_agent_test.go | 2 +-
|
||||||
|
virtcontainers/mock_hypervisor.go | 4 ++++
|
||||||
|
virtcontainers/noop_agent.go | 2 +-
|
||||||
|
virtcontainers/qemu.go | 4 ++++
|
||||||
|
virtcontainers/sandbox.go | 30 ++++++++++++++++++++++++++++--
|
||||||
|
virtcontainers/sandbox_test.go | 12 ++++++++++++
|
||||||
|
virtcontainers/utils/utils.go | 3 +++
|
||||||
|
virtcontainers/vm.go | 2 +-
|
||||||
|
14 files changed, 73 insertions(+), 8 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/acrn.go b/virtcontainers/acrn.go
|
||||||
|
index 10cae06f..c9a0fe0b 100644
|
||||||
|
--- a/virtcontainers/acrn.go
|
||||||
|
+++ b/virtcontainers/acrn.go
|
||||||
|
@@ -811,3 +811,7 @@ func (a *Acrn) loadInfo() error {
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func (a *Acrn) getMemorySize() uint32 {
|
||||||
|
+ return a.config.MemorySize
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go
|
||||||
|
index be9526c7..b1dea816 100644
|
||||||
|
--- a/virtcontainers/agent.go
|
||||||
|
+++ b/virtcontainers/agent.go
|
||||||
|
@@ -201,7 +201,8 @@ type agent interface {
|
||||||
|
// This function should be called after hot adding vCPUs or Memory.
|
||||||
|
// cpus specifies the number of CPUs that were added and the agent should online
|
||||||
|
// cpuOnly specifies that we should online cpu or online memory or both
|
||||||
|
- onlineCPUMem(cpus uint32, cpuOnly bool) error
|
||||||
|
+ // wait specifies that we should wait all cpu or memory online in the VM synchronously
|
||||||
|
+ onlineCPUMem(cpus uint32, cpuOnly bool, wait bool) error
|
||||||
|
|
||||||
|
// memHotplugByProbe will notify the guest kernel about memory hotplug event through
|
||||||
|
// probe interface.
|
||||||
|
diff --git a/virtcontainers/clh.go b/virtcontainers/clh.go
|
||||||
|
index 59510b02..8afcd4bf 100644
|
||||||
|
--- a/virtcontainers/clh.go
|
||||||
|
+++ b/virtcontainers/clh.go
|
||||||
|
@@ -1210,3 +1210,7 @@ func (clh *cloudHypervisor) vmInfo() (chclient.VmInfo, error) {
|
||||||
|
return info, openAPIClientError(err)
|
||||||
|
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func (clh *cloudHypervisor) getMemorySize() uint32 {
|
||||||
|
+ return clh.config.MemorySize
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go
|
||||||
|
index 72a8e192..15726156 100644
|
||||||
|
--- a/virtcontainers/fc.go
|
||||||
|
+++ b/virtcontainers/fc.go
|
||||||
|
@@ -1212,3 +1212,7 @@ func (fc *firecracker) watchConsole() (*os.File, error) {
|
||||||
|
|
||||||
|
return stdio, nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func (fc *firecracker) getMemorySize() uint32 {
|
||||||
|
+ return fc.config.MemorySize
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go
|
||||||
|
index 5f8d24f9..9cd685ad 100644
|
||||||
|
--- a/virtcontainers/hypervisor.go
|
||||||
|
+++ b/virtcontainers/hypervisor.go
|
||||||
|
@@ -778,6 +778,9 @@ type hypervisor interface {
|
||||||
|
hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error)
|
||||||
|
resizeMemory(memMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error)
|
||||||
|
resizeVCPUs(vcpus uint32) (uint32, uint32, error)
|
||||||
|
+ // getMemorySize return the total memory in the guest include default memory size + hot plugged memory
|
||||||
|
+ // return memory size unit is MB
|
||||||
|
+ getMemorySize() uint32
|
||||||
|
getSandboxConsole(sandboxID string) (string, error)
|
||||||
|
disconnect()
|
||||||
|
capabilities() types.Capabilities
|
||||||
|
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
|
||||||
|
index 7575d326..8e073339 100644
|
||||||
|
--- a/virtcontainers/kata_agent.go
|
||||||
|
+++ b/virtcontainers/kata_agent.go
|
||||||
|
@@ -1806,9 +1806,9 @@ func (k *kataAgent) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionS
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
-func (k *kataAgent) onlineCPUMem(cpus uint32, cpuOnly bool) error {
|
||||||
|
+func (k *kataAgent) onlineCPUMem(cpus uint32, cpuOnly bool, wait bool) error {
|
||||||
|
req := &grpc.OnlineCPUMemRequest{
|
||||||
|
- Wait: false,
|
||||||
|
+ Wait: wait,
|
||||||
|
NbCpus: cpus,
|
||||||
|
CpuOnly: cpuOnly,
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go
|
||||||
|
index 62d31c93..2a2ddada 100644
|
||||||
|
--- a/virtcontainers/kata_agent_test.go
|
||||||
|
+++ b/virtcontainers/kata_agent_test.go
|
||||||
|
@@ -324,7 +324,7 @@ func TestKataAgentSendReq(t *testing.T) {
|
||||||
|
err = k.resumeContainer(sandbox, Container{})
|
||||||
|
assert.Nil(err)
|
||||||
|
|
||||||
|
- err = k.onlineCPUMem(1, true)
|
||||||
|
+ err = k.onlineCPUMem(1, true, false)
|
||||||
|
assert.Nil(err)
|
||||||
|
|
||||||
|
_, err = k.statsContainer(sandbox, Container{})
|
||||||
|
diff --git a/virtcontainers/mock_hypervisor.go b/virtcontainers/mock_hypervisor.go
|
||||||
|
index a5b67491..f1c6106d 100644
|
||||||
|
--- a/virtcontainers/mock_hypervisor.go
|
||||||
|
+++ b/virtcontainers/mock_hypervisor.go
|
||||||
|
@@ -128,3 +128,7 @@ func (m *mockHypervisor) check() error {
|
||||||
|
func (m *mockHypervisor) generateSocket(id string, useVsock bool) (interface{}, error) {
|
||||||
|
return types.Socket{HostPath: "/tmp/socket", Name: "socket"}, nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func (m *mockHypervisor) getMemorySize() uint32 {
|
||||||
|
+ return 0
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/noop_agent.go b/virtcontainers/noop_agent.go
|
||||||
|
index 8a7cd337..6e211bca 100644
|
||||||
|
--- a/virtcontainers/noop_agent.go
|
||||||
|
+++ b/virtcontainers/noop_agent.go
|
||||||
|
@@ -102,7 +102,7 @@ func (n *noopAgent) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionS
|
||||||
|
}
|
||||||
|
|
||||||
|
// onlineCPUMem is the Noop agent Container online CPU and Memory implementation. It does nothing.
|
||||||
|
-func (n *noopAgent) onlineCPUMem(cpus uint32, cpuOnly bool) error {
|
||||||
|
+func (n *noopAgent) onlineCPUMem(cpus uint32, cpuOnly bool, wait bool) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
|
||||||
|
index 4789101d..7bae3278 100644
|
||||||
|
--- a/virtcontainers/qemu.go
|
||||||
|
+++ b/virtcontainers/qemu.go
|
||||||
|
@@ -2273,3 +2273,7 @@ func (q *qemu) check() error {
|
||||||
|
func (q *qemu) generateSocket(id string, useVsock bool) (interface{}, error) {
|
||||||
|
return generateVMSocket(id, useVsock, q.store.RunVMStoragePath())
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func (q *qemu) getMemorySize() uint32 {
|
||||||
|
+ return q.config.MemorySize + uint32(q.state.HotpluggedMemory)
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index e766d1f7..a318d677 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -1864,7 +1864,7 @@ func (s *Sandbox) updateResources() error {
|
||||||
|
// If the CPUs were increased, ask agent to online them
|
||||||
|
if oldCPUs < newCPUs {
|
||||||
|
vcpusAdded := newCPUs - oldCPUs
|
||||||
|
- if err := s.agent.onlineCPUMem(vcpusAdded, true); err != nil {
|
||||||
|
+ if err := s.agent.onlineCPUMem(vcpusAdded, true, false); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
@@ -1872,6 +1872,20 @@ func (s *Sandbox) updateResources() error {
|
||||||
|
|
||||||
|
// Update Memory
|
||||||
|
s.Logger().WithField("memory-sandbox-size-byte", sandboxMemoryByte).Debugf("Request to hypervisor to update memory")
|
||||||
|
+ reqMemMB := uint32(sandboxMemoryByte >> utils.MibToBytesShift)
|
||||||
|
+ currentMemMB := s.hypervisor.getMemorySize()
|
||||||
|
+
|
||||||
|
+ // If request hotplug memory size larger than utils.MaxHotplugMemMBOnceTime,
|
||||||
|
+ // inorder to avoid hotplug memory oom problem, we need to hotplug large memory
|
||||||
|
+ // with two steps. First, hotplug utils.MaxHotplugMemMBOnceTime size memory into
|
||||||
|
+ // guest and wait all hotplug memory online. Then, hotplug the left unplugged memory
|
||||||
|
+ // into the guest
|
||||||
|
+ if currentMemMB < reqMemMB && (reqMemMB-currentMemMB) > utils.MaxHotplugMemMBOnceTime {
|
||||||
|
+ if err := s.beforeHotplugHugeMem(currentMemMB); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
newMemory, updatedMemoryDevice, err := s.hypervisor.resizeMemory(uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
@@ -1884,7 +1898,7 @@ func (s *Sandbox) updateResources() error {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
- if err := s.agent.onlineCPUMem(0, false); err != nil {
|
||||||
|
+ if err := s.agent.onlineCPUMem(0, false, false); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
@@ -1926,6 +1940,18 @@ func (s *Sandbox) calculateSandboxCPUs() uint32 {
|
||||||
|
return utils.CalculateVCpusFromMilliCpus(mCPU)
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (s *Sandbox) beforeHotplugHugeMem(currentMemSizeInMB uint32) error {
|
||||||
|
+ wantedTotalMemSize := currentMemSizeInMB + utils.MaxHotplugMemMBOnceTime
|
||||||
|
+ newMemory, _, err := s.hypervisor.resizeMemory(wantedTotalMemSize, s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ s.Logger().Debugf("first part hotplug memory size: %d MB", newMemory)
|
||||||
|
+ // wait all first part hotplugged memory online in the guest
|
||||||
|
+ return s.agent.onlineCPUMem(0, false, true)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// GetHypervisorType is used for getting Hypervisor name currently used.
|
||||||
|
// Sandbox implement DeviceReceiver interface from device/api/interface.go
|
||||||
|
func (s *Sandbox) GetHypervisorType() string {
|
||||||
|
diff --git a/virtcontainers/sandbox_test.go b/virtcontainers/sandbox_test.go
|
||||||
|
index 85c712e8..4b02b3f3 100644
|
||||||
|
--- a/virtcontainers/sandbox_test.go
|
||||||
|
+++ b/virtcontainers/sandbox_test.go
|
||||||
|
@@ -25,6 +25,7 @@ import (
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/persist/fs"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/types"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||||
|
specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
@@ -1522,6 +1523,17 @@ func TestSandboxUpdateResources(t *testing.T) {
|
||||||
|
}
|
||||||
|
err = s.updateResources()
|
||||||
|
assert.NoError(t, err)
|
||||||
|
+
|
||||||
|
+ // add a container with huge memory equal utils.MaxHotplugMemMBOnceTime
|
||||||
|
+ contConfig3 := newTestContainerConfigNoop("cont-00003")
|
||||||
|
+ contConfig3.Resources.Memory = &specs.LinuxMemory{
|
||||||
|
+ Limit: new(int64),
|
||||||
|
+ }
|
||||||
|
+ container3MemLimitInBytes := int64(utils.MaxHotplugMemMBOnceTime << utils.MibToBytesShift)
|
||||||
|
+ contConfig3.Resources.Memory.Limit = &container3MemLimitInBytes
|
||||||
|
+ s.config.Containers = append(s.config.Containers, contConfig3)
|
||||||
|
+ err = s.updateResources()
|
||||||
|
+ assert.NoError(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSandboxExperimentalFeature(t *testing.T) {
|
||||||
|
diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go
|
||||||
|
index 2b555ebb..3ae95aef 100644
|
||||||
|
--- a/virtcontainers/utils/utils.go
|
||||||
|
+++ b/virtcontainers/utils/utils.go
|
||||||
|
@@ -25,6 +25,9 @@ const fileMode0755 = os.FileMode(0755)
|
||||||
|
// MibToBytesShift the number to shift needed to convert MiB to Bytes
|
||||||
|
const MibToBytesShift = 20
|
||||||
|
|
||||||
|
+// Max Hotplug Memory size at once time, unit is MB
|
||||||
|
+const MaxHotplugMemMBOnceTime = 32 * 1024
|
||||||
|
+
|
||||||
|
// MaxSocketPathLen is the effective maximum Unix domain socket length.
|
||||||
|
//
|
||||||
|
// See unix(7).
|
||||||
|
diff --git a/virtcontainers/vm.go b/virtcontainers/vm.go
|
||||||
|
index 8d27b1fe..2e5fef44 100644
|
||||||
|
--- a/virtcontainers/vm.go
|
||||||
|
+++ b/virtcontainers/vm.go
|
||||||
|
@@ -370,7 +370,7 @@ func (v *VM) AddMemory(numMB uint32) error {
|
||||||
|
// OnlineCPUMemory puts the hotplugged CPU and memory online.
|
||||||
|
func (v *VM) OnlineCPUMemory() error {
|
||||||
|
v.logger().Infof("online CPU %d and memory", v.cpuDelta)
|
||||||
|
- err := v.agent.onlineCPUMem(v.cpuDelta, false)
|
||||||
|
+ err := v.agent.onlineCPUMem(v.cpuDelta, false, false)
|
||||||
|
if err == nil {
|
||||||
|
v.cpuDelta = 0
|
||||||
|
}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,355 @@
|
|||||||
|
From e6051eb1a8c54b91c46f68eab9a63ad0e73a9221 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Fri, 7 Aug 2020 20:01:15 +0800
|
||||||
|
Subject: [PATCH 17/50] kata-runtime: validate sandbox cpu and memory size
|
||||||
|
|
||||||
|
reason:
|
||||||
|
1. add more strict sandbox cpu and memory size check.
|
||||||
|
2. use GetPhysicalCPUNumber func to replace goruntime.NumCPU(),
|
||||||
|
because goruntime.NumCPU() return the CPU number of cpuset cgroup
|
||||||
|
that current process joined, however kata-runtime process will
|
||||||
|
change the cpuset cgroup when setupSanboxCgroup is called, which
|
||||||
|
may cause goruntime.NumCPU() return value changed.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
pkg/katautils/config.go | 39 ++++++++++++++++++++++++++++++++++---
|
||||||
|
pkg/katautils/config_test.go | 11 ++++++++---
|
||||||
|
virtcontainers/pkg/oci/utils.go | 11 +++++------
|
||||||
|
virtcontainers/qemu_arm64.go | 4 ++--
|
||||||
|
virtcontainers/qemu_arm64_test.go | 4 ++--
|
||||||
|
virtcontainers/sandbox.go | 7 ++++++-
|
||||||
|
virtcontainers/utils/utils.go | 41 +++++++++++++++++++++++++++++++++++++++
|
||||||
|
7 files changed, 100 insertions(+), 17 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
||||||
|
index d1883c94..b9b08469 100644
|
||||||
|
--- a/pkg/katautils/config.go
|
||||||
|
+++ b/pkg/katautils/config.go
|
||||||
|
@@ -11,7 +11,6 @@ import (
|
||||||
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
|
"path/filepath"
|
||||||
|
- goruntime "runtime"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/BurntSushi/toml"
|
||||||
|
@@ -285,7 +284,7 @@ func (h hypervisor) GetEntropySource() string {
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h hypervisor) defaultVCPUs() uint32 {
|
||||||
|
- numCPUs := goruntime.NumCPU()
|
||||||
|
+ numCPUs := utils.GetPhysicalCPUNumber()
|
||||||
|
|
||||||
|
if h.NumVCPUs < 0 || h.NumVCPUs > int32(numCPUs) {
|
||||||
|
return uint32(numCPUs)
|
||||||
|
@@ -297,8 +296,22 @@ func (h hypervisor) defaultVCPUs() uint32 {
|
||||||
|
return uint32(h.NumVCPUs)
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (h hypervisor) checkVCPUs() error {
|
||||||
|
+ numCPUs := utils.GetPhysicalCPUNumber()
|
||||||
|
+
|
||||||
|
+ if h.NumVCPUs <= 0 {
|
||||||
|
+ return fmt.Errorf("invalid vcpus in configuration.toml! vcpus must larger than 0")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if h.NumVCPUs > int32(numCPUs) {
|
||||||
|
+ return fmt.Errorf("invalid vcpus in configuration.toml! vcpus must smaller than max CPUs: %d in machine", numCPUs)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func (h hypervisor) defaultMaxVCPUs() uint32 {
|
||||||
|
- numcpus := uint32(goruntime.NumCPU())
|
||||||
|
+ numcpus := uint32(utils.GetPhysicalCPUNumber())
|
||||||
|
maxvcpus := vc.MaxQemuVCPUs()
|
||||||
|
reqVCPUs := h.DefaultMaxVCPUs
|
||||||
|
|
||||||
|
@@ -324,6 +337,18 @@ func (h hypervisor) defaultMemSz() uint32 {
|
||||||
|
return h.MemorySize
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (h hypervisor) checkMemSz() error {
|
||||||
|
+ if h.MemorySize < utils.MinMemorySizeInMB {
|
||||||
|
+ return fmt.Errorf("invalid memory size! Memory size must larger than %d MB", utils.MinMemorySizeInMB)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if h.MemorySize > utils.MaxMemorySizeInMB {
|
||||||
|
+ return fmt.Errorf("invalid memory size, memory size must smaller than %d MB", utils.MaxMemorySizeInMB)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func (h hypervisor) defaultMemSlots() uint32 {
|
||||||
|
slots := h.MemSlots
|
||||||
|
if slots == 0 {
|
||||||
|
@@ -627,6 +652,14 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
+ if err = h.checkVCPUs(); err != nil {
|
||||||
|
+ return vc.HypervisorConfig{}, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err = h.checkMemSz(); err != nil {
|
||||||
|
+ return vc.HypervisorConfig{}, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
return vc.HypervisorConfig{
|
||||||
|
HypervisorPath: hypervisor,
|
||||||
|
KernelPath: kernel,
|
||||||
|
diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go
|
||||||
|
index 31afcca6..37aa16d0 100644
|
||||||
|
--- a/pkg/katautils/config_test.go
|
||||||
|
+++ b/pkg/katautils/config_test.go
|
||||||
|
@@ -14,7 +14,6 @@ import (
|
||||||
|
"path"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
- goruntime "runtime"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
"testing"
|
||||||
|
@@ -152,7 +151,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
|
||||||
|
KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)),
|
||||||
|
HypervisorMachineType: machineType,
|
||||||
|
NumVCPUs: defaultVCPUCount,
|
||||||
|
- DefaultMaxVCPUs: uint32(goruntime.NumCPU()),
|
||||||
|
+ DefaultMaxVCPUs: uint32(utils.GetPhysicalCPUNumber()),
|
||||||
|
MemorySize: defaultMemSize,
|
||||||
|
DisableBlockDeviceUse: disableBlockDevice,
|
||||||
|
BlockDeviceDriver: defaultBlockDeviceDriver,
|
||||||
|
@@ -727,6 +726,8 @@ func TestMinimalRuntimeConfigWithVsock(t *testing.T) {
|
||||||
|
[hypervisor.qemu]
|
||||||
|
use_vsock = true
|
||||||
|
image = "` + imagePath + `"
|
||||||
|
+ default_vcpus = 1
|
||||||
|
+ default_memory = 1024
|
||||||
|
|
||||||
|
[proxy.kata]
|
||||||
|
path = "` + proxyPath + `"
|
||||||
|
@@ -786,6 +787,8 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
|
||||||
|
utils.VHostVSockDevicePath = orgVHostVSockDevicePath
|
||||||
|
}()
|
||||||
|
utils.VHostVSockDevicePath = "/dev/abc/xyz"
|
||||||
|
+ defaultVCPUs := int32(1)
|
||||||
|
+ defaultMemory := uint32(1024)
|
||||||
|
|
||||||
|
hypervisor := hypervisor{
|
||||||
|
Path: hypervisorPath,
|
||||||
|
@@ -797,6 +800,8 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
|
||||||
|
HotplugVFIOOnRootBus: hotplugVFIOOnRootBus,
|
||||||
|
PCIeRootPort: pcieRootPort,
|
||||||
|
UseVSock: true,
|
||||||
|
+ NumVCPUs: defaultVCPUs,
|
||||||
|
+ MemorySize: defaultMemory,
|
||||||
|
}
|
||||||
|
|
||||||
|
files := []string{hypervisorPath, kernelPath, imagePath}
|
||||||
|
@@ -996,7 +1001,7 @@ func TestNewShimConfig(t *testing.T) {
|
||||||
|
func TestHypervisorDefaults(t *testing.T) {
|
||||||
|
assert := assert.New(t)
|
||||||
|
|
||||||
|
- numCPUs := goruntime.NumCPU()
|
||||||
|
+ numCPUs := utils.GetPhysicalCPUNumber()
|
||||||
|
|
||||||
|
h := hypervisor{}
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
||||||
|
index 05181efd..0a6f08c7 100644
|
||||||
|
--- a/virtcontainers/pkg/oci/utils.go
|
||||||
|
+++ b/virtcontainers/pkg/oci/utils.go
|
||||||
|
@@ -10,22 +10,21 @@ import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"path/filepath"
|
||||||
|
- goruntime "runtime"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
|
||||||
|
criContainerdAnnotations "github.com/containerd/cri-containerd/pkg/annotations"
|
||||||
|
crioAnnotations "github.com/cri-o/cri-o/pkg/annotations"
|
||||||
|
- specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
- "github.com/sirupsen/logrus"
|
||||||
|
-
|
||||||
|
vc "github.com/kata-containers/runtime/virtcontainers"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/device/config"
|
||||||
|
exp "github.com/kata-containers/runtime/virtcontainers/experimental"
|
||||||
|
vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
|
||||||
|
dockershimAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations/dockershim"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/types"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||||
|
+ specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
+ "github.com/sirupsen/logrus"
|
||||||
|
)
|
||||||
|
|
||||||
|
type annotationContainerType struct {
|
||||||
|
@@ -560,7 +559,7 @@ func addHypervisorCPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e
|
||||||
|
return fmt.Errorf("Error encountered parsing annotation default_vcpus: %v, please specify numeric value", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
- numCPUs := goruntime.NumCPU()
|
||||||
|
+ numCPUs := utils.GetPhysicalCPUNumber()
|
||||||
|
|
||||||
|
if uint32(vcpus) > uint32(numCPUs) {
|
||||||
|
return fmt.Errorf("Number of cpus %d specified in annotation default_vcpus is greater than the number of CPUs %d on the system", vcpus, numCPUs)
|
||||||
|
@@ -575,7 +574,7 @@ func addHypervisorCPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e
|
||||||
|
return fmt.Errorf("Error encountered parsing annotation for default_maxvcpus: %v, please specify positive numeric value", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
- numCPUs := goruntime.NumCPU()
|
||||||
|
+ numCPUs := utils.GetPhysicalCPUNumber()
|
||||||
|
max := uint32(maxVCPUs)
|
||||||
|
|
||||||
|
if max > uint32(numCPUs) {
|
||||||
|
diff --git a/virtcontainers/qemu_arm64.go b/virtcontainers/qemu_arm64.go
|
||||||
|
index 6d089cf0..40fb2a80 100644
|
||||||
|
--- a/virtcontainers/qemu_arm64.go
|
||||||
|
+++ b/virtcontainers/qemu_arm64.go
|
||||||
|
@@ -8,11 +8,11 @@ package virtcontainers
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io/ioutil"
|
||||||
|
- "runtime"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
govmmQemu "github.com/intel/govmm/qemu"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
)
|
||||||
|
|
||||||
|
@@ -121,7 +121,7 @@ func MaxQemuVCPUs() uint32 {
|
||||||
|
if hostGICVersion != 0 {
|
||||||
|
return gicList[hostGICVersion]
|
||||||
|
}
|
||||||
|
- return uint32(runtime.NumCPU())
|
||||||
|
+ return uint32(utils.GetPhysicalCPUNumber())
|
||||||
|
}
|
||||||
|
|
||||||
|
func newQemuArch(config HypervisorConfig) qemuArch {
|
||||||
|
diff --git a/virtcontainers/qemu_arm64_test.go b/virtcontainers/qemu_arm64_test.go
|
||||||
|
index 40351158..372fc4a9 100644
|
||||||
|
--- a/virtcontainers/qemu_arm64_test.go
|
||||||
|
+++ b/virtcontainers/qemu_arm64_test.go
|
||||||
|
@@ -10,10 +10,10 @@ import (
|
||||||
|
"io/ioutil"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
- "runtime"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
govmmQemu "github.com/intel/govmm/qemu"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
@@ -63,7 +63,7 @@ func TestMaxQemuVCPUs(t *testing.T) {
|
||||||
|
}
|
||||||
|
|
||||||
|
data := []testData{
|
||||||
|
- {"", uint32(runtime.NumCPU())},
|
||||||
|
+ {"", uint32(utils.GetPhysicalCPUNumber())},
|
||||||
|
{" 1: 0 0 GICv2 25 Level vgic \n", uint32(8)},
|
||||||
|
{" 1: 0 0 GICv3 25 Level vgic \n", uint32(123)},
|
||||||
|
{" 1: 0 0 GICv4 25 Level vgic \n", uint32(123)},
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index a318d677..a8522b96 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -1875,6 +1875,11 @@ func (s *Sandbox) updateResources() error {
|
||||||
|
reqMemMB := uint32(sandboxMemoryByte >> utils.MibToBytesShift)
|
||||||
|
currentMemMB := s.hypervisor.getMemorySize()
|
||||||
|
|
||||||
|
+ // check request sandbox memory size larger than utils.MaxMemorySizInMB or not
|
||||||
|
+ if reqMemMB > utils.MaxMemorySizeInMB {
|
||||||
|
+ return fmt.Errorf("updateResources failed! Sandbox memory should <= %d MiB", utils.MaxMemorySizeInMB)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
// If request hotplug memory size larger than utils.MaxHotplugMemMBOnceTime,
|
||||||
|
// inorder to avoid hotplug memory oom problem, we need to hotplug large memory
|
||||||
|
// with two steps. First, hotplug utils.MaxHotplugMemMBOnceTime size memory into
|
||||||
|
@@ -1898,7 +1903,7 @@ func (s *Sandbox) updateResources() error {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
- if err := s.agent.onlineCPUMem(0, false, false); err != nil {
|
||||||
|
+ if err := s.agent.onlineCPUMem(0, false, true); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go
|
||||||
|
index 3ae95aef..5d38e594 100644
|
||||||
|
--- a/virtcontainers/utils/utils.go
|
||||||
|
+++ b/virtcontainers/utils/utils.go
|
||||||
|
@@ -6,6 +6,7 @@
|
||||||
|
package utils
|
||||||
|
|
||||||
|
import (
|
||||||
|
+ "bufio"
|
||||||
|
"crypto/rand"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
@@ -28,11 +29,26 @@ const MibToBytesShift = 20
|
||||||
|
// Max Hotplug Memory size at once time, unit is MB
|
||||||
|
const MaxHotplugMemMBOnceTime = 32 * 1024
|
||||||
|
|
||||||
|
+const (
|
||||||
|
+ // Min needed memory size to start a Kata VM
|
||||||
|
+ MinMemorySizeInMB = 300
|
||||||
|
+ MinMemorySizeInByte = MinMemorySizeInMB << MibToBytesShift
|
||||||
|
+
|
||||||
|
+ // Max support memory size in the Kata VM
|
||||||
|
+ MaxMemorySizeInMB = 512 * 1024
|
||||||
|
+ MaxMemorySizeInByte = MaxMemorySizeInMB << MibToBytesShift
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
// MaxSocketPathLen is the effective maximum Unix domain socket length.
|
||||||
|
//
|
||||||
|
// See unix(7).
|
||||||
|
const MaxSocketPathLen = 107
|
||||||
|
|
||||||
|
+const (
|
||||||
|
+ procCPUInfoPath = "/proc/cpuinfo"
|
||||||
|
+ processorIdentifier = "processor"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
// VHostVSockDevicePath path to vhost-vsock device
|
||||||
|
var VHostVSockDevicePath = "/dev/vhost-vsock"
|
||||||
|
|
||||||
|
@@ -324,3 +340,28 @@ func IsProcessRunning(pid int, processName string, sandboxID string) bool {
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+// GetPhysicalCPUNumber return the number of the CPUs in the physical machine
|
||||||
|
+func GetPhysicalCPUNumber() int {
|
||||||
|
+ f, err := os.Open(procCPUInfoPath)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return 0
|
||||||
|
+ }
|
||||||
|
+ defer f.Close()
|
||||||
|
+
|
||||||
|
+ cpuNum := 0
|
||||||
|
+ s := bufio.NewScanner(f)
|
||||||
|
+ for s.Scan() {
|
||||||
|
+ if err := s.Err(); err != nil {
|
||||||
|
+ return 0
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ fields := strings.Fields(s.Text())
|
||||||
|
+ if len(fields) > 0 {
|
||||||
|
+ if fields[0] == processorIdentifier {
|
||||||
|
+ cpuNum++
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ return cpuNum
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,50 @@
|
|||||||
|
From c785f8f744050155102664d56de5bfb55e91915d Mon Sep 17 00:00:00 2001
|
||||||
|
From: Evan Foster <efoster@adobe.com>
|
||||||
|
Date: Mon, 13 Jul 2020 12:53:40 -0600
|
||||||
|
Subject: [PATCH 18/50] sandbox: Stop and clean up containers that fail to
|
||||||
|
create
|
||||||
|
|
||||||
|
A container that is created and added to a sandbox can still fail
|
||||||
|
the final creation steps. In this case, the container must be stopped
|
||||||
|
and have its resources cleaned up to prevent leaking sandbox mounts.
|
||||||
|
|
||||||
|
Fixes #2816
|
||||||
|
|
||||||
|
cherry-pick from: https://github.com/kata-containers/runtime/pull/2826
|
||||||
|
|
||||||
|
Signed-off-by: Evan Foster <efoster@adobe.com>
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/sandbox.go | 11 +++++++++++
|
||||||
|
1 file changed, 11 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index a8522b96..3dbf640e 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -1,4 +1,5 @@
|
||||||
|
// Copyright (c) 2016 Intel Corporation
|
||||||
|
+// Copyright (c) 2020 Adobe Inc.
|
||||||
|
//
|
||||||
|
// SPDX-License-Identifier: Apache-2.0
|
||||||
|
//
|
||||||
|
@@ -1172,6 +1173,16 @@ func (s *Sandbox) CreateContainer(contConfig ContainerConfig) (VCContainer, erro
|
||||||
|
defer func() {
|
||||||
|
// Rollback if error happens.
|
||||||
|
if err != nil {
|
||||||
|
+ logger := s.Logger().WithFields(logrus.Fields{"container-id": c.id, "sandox-id": s.id, "rollback": true})
|
||||||
|
+
|
||||||
|
+ logger.Warning("Cleaning up partially created container")
|
||||||
|
+
|
||||||
|
+ if err2 := c.stop(true); err2 != nil {
|
||||||
|
+ logger.WithError(err2).Warning("Could not delete container")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ logger.Debug("Removing stopped container from sandbox store")
|
||||||
|
+
|
||||||
|
s.removeContainer(c.id)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,33 @@
|
|||||||
|
From d20cb25c8a145e1d3e64eefa69242d87b7a67f92 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Sat, 8 Aug 2020 16:00:45 -0400
|
||||||
|
Subject: [PATCH 19/50] virtcontainers: fix delete sandbox failed problem
|
||||||
|
|
||||||
|
fixes: #2882
|
||||||
|
|
||||||
|
reason: If error happens after container create and before sandbox
|
||||||
|
updateResouce in the `CreateContainer()`, then delete sandbox
|
||||||
|
forcefully will return error because s.config.Containers config not
|
||||||
|
flushed into persist store.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/sandbox.go | 2 ++
|
||||||
|
1 file changed, 2 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index 3dbf640e..7322ef58 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -1156,6 +1156,8 @@ func (s *Sandbox) CreateContainer(contConfig ContainerConfig) (VCContainer, erro
|
||||||
|
if len(s.config.Containers) > 0 {
|
||||||
|
// delete container config
|
||||||
|
s.config.Containers = s.config.Containers[:len(s.config.Containers)-1]
|
||||||
|
+ // need to flush change to persist storage
|
||||||
|
+ _ = s.storeSandbox()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,188 @@
|
|||||||
|
From 75141529674545a2f84b01c730f614901ad2265e Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Mon, 10 Aug 2020 10:08:47 +0800
|
||||||
|
Subject: [PATCH 20/50] virtcontainers: add enable_cpu_memory_hotplug config in
|
||||||
|
the configuration.toml
|
||||||
|
|
||||||
|
reason: add enable_cpu_memory_hotplug config to control whether enable hotplug
|
||||||
|
memory and cpu resource into the Kata VM. If we can calculate the whole sandbox
|
||||||
|
resource usage already, we just need to pass the value of resouces by annotation
|
||||||
|
without hotplugging.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
cli/config/configuration-qemu.toml.in | 4 ++++
|
||||||
|
pkg/katautils/config.go | 2 ++
|
||||||
|
virtcontainers/container.go | 6 ++++--
|
||||||
|
virtcontainers/hypervisor.go | 3 +++
|
||||||
|
virtcontainers/persist.go | 2 ++
|
||||||
|
virtcontainers/persist/api/config.go | 3 +++
|
||||||
|
virtcontainers/sandbox.go | 38 +++++++++++++++++++++--------------
|
||||||
|
7 files changed, 41 insertions(+), 17 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in
|
||||||
|
index 82461732..b44e84d8 100644
|
||||||
|
--- a/cli/config/configuration-qemu.toml.in
|
||||||
|
+++ b/cli/config/configuration-qemu.toml.in
|
||||||
|
@@ -100,6 +100,10 @@ default_memory = @DEFMEMSZ@
|
||||||
|
# Default false
|
||||||
|
#enable_reuse_cpu_memory = false
|
||||||
|
|
||||||
|
+# If enabled, the runtime will support cpu and memory hot plug
|
||||||
|
+# (default: disabled)
|
||||||
|
+#enable_cpu_memory_hotplug = true
|
||||||
|
+
|
||||||
|
# Disable block device from being used for a container's rootfs.
|
||||||
|
# In case of a storage driver like devicemapper where a container's
|
||||||
|
# root file system is backed by a block device, the block device is passed
|
||||||
|
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
||||||
|
index b9b08469..9a99b9d4 100644
|
||||||
|
--- a/pkg/katautils/config.go
|
||||||
|
+++ b/pkg/katautils/config.go
|
||||||
|
@@ -112,6 +112,7 @@ type hypervisor struct {
|
||||||
|
MemSlots uint32 `toml:"memory_slots"`
|
||||||
|
MemOffset uint32 `toml:"memory_offset"`
|
||||||
|
EnableCPUMemoryReuse bool `toml:"enable_reuse_cpu_memory"`
|
||||||
|
+ EnableCPUMemoryHotPlug bool `toml:"enable_cpu_memory_hotplug"`
|
||||||
|
DefaultBridges uint32 `toml:"default_bridges"`
|
||||||
|
Msize9p uint32 `toml:"msize_9p"`
|
||||||
|
PCIeRootPort uint32 `toml:"pcie_root_port"`
|
||||||
|
@@ -675,6 +676,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
||||||
|
MemSlots: h.defaultMemSlots(),
|
||||||
|
MemOffset: h.defaultMemOffset(),
|
||||||
|
EnableCPUMemoryReuse: h.EnableCPUMemoryReuse,
|
||||||
|
+ EnableCPUMemoryHotPlug: h.EnableCPUMemoryHotPlug,
|
||||||
|
VirtioMem: h.VirtioMem,
|
||||||
|
EntropySource: h.GetEntropySource(),
|
||||||
|
DefaultBridges: h.defaultBridges(),
|
||||||
|
diff --git a/virtcontainers/container.go b/virtcontainers/container.go
|
||||||
|
index 75f590eb..1b89f6ac 100644
|
||||||
|
--- a/virtcontainers/container.go
|
||||||
|
+++ b/virtcontainers/container.go
|
||||||
|
@@ -1273,8 +1273,10 @@ func (c *Container) update(resources specs.LinuxResources) error {
|
||||||
|
c.config.Resources.Memory.Limit = mem.Limit
|
||||||
|
}
|
||||||
|
|
||||||
|
- if err := c.sandbox.updateResources(); err != nil {
|
||||||
|
- return err
|
||||||
|
+ if c.sandbox.config.HypervisorConfig.EnableCPUMemoryHotPlug {
|
||||||
|
+ if err := c.sandbox.updateResources(); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
|
||||||
|
if !c.sandbox.config.SandboxCgroupOnly {
|
||||||
|
diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go
|
||||||
|
index 9cd685ad..c0723daa 100644
|
||||||
|
--- a/virtcontainers/hypervisor.go
|
||||||
|
+++ b/virtcontainers/hypervisor.go
|
||||||
|
@@ -253,6 +253,9 @@ type HypervisorConfig struct {
|
||||||
|
// Enable hypervisor cpu and memory resource share with container
|
||||||
|
EnableCPUMemoryReuse bool
|
||||||
|
|
||||||
|
+ // Enable hotplug cpu and memory resource into container
|
||||||
|
+ EnableCPUMemoryHotPlug bool
|
||||||
|
+
|
||||||
|
// VirtioFSCacheSize is the DAX cache size in MiB
|
||||||
|
VirtioFSCacheSize uint32
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go
|
||||||
|
index aef4d18d..6bd09a0b 100644
|
||||||
|
--- a/virtcontainers/persist.go
|
||||||
|
+++ b/virtcontainers/persist.go
|
||||||
|
@@ -215,6 +215,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
|
||||||
|
MemSlots: sconfig.HypervisorConfig.MemSlots,
|
||||||
|
MemOffset: sconfig.HypervisorConfig.MemOffset,
|
||||||
|
EnableCPUMemoryReuse: sconfig.HypervisorConfig.EnableCPUMemoryReuse,
|
||||||
|
+ EnableCPUMemoryHotPlug: sconfig.HypervisorConfig.EnableCPUMemoryHotPlug,
|
||||||
|
VirtioMem: sconfig.HypervisorConfig.VirtioMem,
|
||||||
|
VirtioFSCacheSize: sconfig.HypervisorConfig.VirtioFSCacheSize,
|
||||||
|
KernelPath: sconfig.HypervisorConfig.KernelPath,
|
||||||
|
@@ -505,6 +506,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
|
||||||
|
MemSlots: hconf.MemSlots,
|
||||||
|
MemOffset: hconf.MemOffset,
|
||||||
|
EnableCPUMemoryReuse: hconf.EnableCPUMemoryReuse,
|
||||||
|
+ EnableCPUMemoryHotPlug: hconf.EnableCPUMemoryHotPlug,
|
||||||
|
VirtioMem: hconf.VirtioMem,
|
||||||
|
VirtioFSCacheSize: hconf.VirtioFSCacheSize,
|
||||||
|
KernelPath: hconf.KernelPath,
|
||||||
|
diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go
|
||||||
|
index a3c6ec91..cfbee849 100644
|
||||||
|
--- a/virtcontainers/persist/api/config.go
|
||||||
|
+++ b/virtcontainers/persist/api/config.go
|
||||||
|
@@ -38,6 +38,9 @@ type HypervisorConfig struct {
|
||||||
|
// Enable hypervisor cpu and memory resource share with container
|
||||||
|
EnableCPUMemoryReuse bool
|
||||||
|
|
||||||
|
+ // Enable hotplug cpu and memory resource into container
|
||||||
|
+ EnableCPUMemoryHotPlug bool
|
||||||
|
+
|
||||||
|
// VirtioFSCacheSize is the DAX cache size in MiB
|
||||||
|
VirtioFSCacheSize uint32
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index 7322ef58..8fcd92d4 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -1189,12 +1189,16 @@ func (s *Sandbox) CreateContainer(contConfig ContainerConfig) (VCContainer, erro
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
- // Sandbox is reponsable to update VM resources needed by Containers
|
||||||
|
- // Update resources after having added containers to the sandbox, since
|
||||||
|
- // container status is requiered to know if more resources should be added.
|
||||||
|
- err = s.updateResources()
|
||||||
|
- if err != nil {
|
||||||
|
- return nil, err
|
||||||
|
+ // update sandbox resource only when enable_cpu_memory_hotplug config set true
|
||||||
|
+ // in the config.toml file
|
||||||
|
+ if s.config.HypervisorConfig.EnableCPUMemoryHotPlug {
|
||||||
|
+ // Sandbox is reponsable to update VM resources needed by Containers
|
||||||
|
+ // Update resources after having added containers to the sandbox, since
|
||||||
|
+ // container status is requiered to know if more resources should be added.
|
||||||
|
+ err = s.updateResources()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = s.cgroupsUpdate(); err != nil {
|
||||||
|
@@ -1228,11 +1232,13 @@ func (s *Sandbox) StartContainer(containerID string) (VCContainer, error) {
|
||||||
|
|
||||||
|
s.Logger().Info("Container is started")
|
||||||
|
|
||||||
|
- // Update sandbox resources in case a stopped container
|
||||||
|
- // is started
|
||||||
|
- err = s.updateResources()
|
||||||
|
- if err != nil {
|
||||||
|
- return nil, err
|
||||||
|
+ if s.config.HypervisorConfig.EnableCPUMemoryHotPlug {
|
||||||
|
+ // Update sandbox resources in case a stopped container
|
||||||
|
+ // is started
|
||||||
|
+ err = s.updateResources()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
|
||||||
|
return c, nil
|
||||||
|
@@ -1503,10 +1509,12 @@ func (s *Sandbox) createContainers() error {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
- // Update resources after having added containers to the sandbox, since
|
||||||
|
- // container status is requiered to know if more resources should be added.
|
||||||
|
- if err := s.updateResources(); err != nil {
|
||||||
|
- return err
|
||||||
|
+ if s.config.HypervisorConfig.EnableCPUMemoryHotPlug {
|
||||||
|
+ // Update resources after having added containers to the sandbox, since
|
||||||
|
+ // container status is requiered to know if more resources should be added.
|
||||||
|
+ if err := s.updateResources(); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.cgroupsUpdate(); err != nil {
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,195 @@
|
|||||||
|
From 79cf2f5a52af51d8a62353a99e894808281769e2 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Mon, 10 Aug 2020 11:25:02 +0800
|
||||||
|
Subject: [PATCH 21/50] kata-runtime: add sandbox_cpu and sandbox_mem
|
||||||
|
annotations
|
||||||
|
|
||||||
|
reason: add sandbox_cpu and sandbox_men annotations to set
|
||||||
|
Kata VM vCPU number and memory size.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/pkg/annotations/annotations.go | 6 +++++
|
||||||
|
virtcontainers/pkg/oci/utils.go | 39 +++++++++++++++++++++++++--
|
||||||
|
virtcontainers/sandbox.go | 32 ++++++++++++++++++++++
|
||||||
|
virtcontainers/utils/utils.go | 14 ++++++++++
|
||||||
|
4 files changed, 89 insertions(+), 2 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go
|
||||||
|
index 10ce7833..903c7f03 100644
|
||||||
|
--- a/virtcontainers/pkg/annotations/annotations.go
|
||||||
|
+++ b/virtcontainers/pkg/annotations/annotations.go
|
||||||
|
@@ -251,6 +251,12 @@ const (
|
||||||
|
ContainerPipeSizeKernelParam = "agent." + ContainerPipeSizeOption
|
||||||
|
)
|
||||||
|
|
||||||
|
+// iSula self defined annotations
|
||||||
|
+const (
|
||||||
|
+ StaticCPUTypeKey = kataAnnotationsPrefix + "sandbox_cpu"
|
||||||
|
+ StaticMemTypeKey = kataAnnotationsPrefix + "sandbox_mem"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
const (
|
||||||
|
// SHA512 is the SHA-512 (64) hash algorithm
|
||||||
|
SHA512 string = "sha512"
|
||||||
|
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
||||||
|
index 0a6f08c7..36c730b7 100644
|
||||||
|
--- a/virtcontainers/pkg/oci/utils.go
|
||||||
|
+++ b/virtcontainers/pkg/oci/utils.go
|
||||||
|
@@ -16,6 +16,7 @@ import (
|
||||||
|
|
||||||
|
criContainerdAnnotations "github.com/containerd/cri-containerd/pkg/annotations"
|
||||||
|
crioAnnotations "github.com/cri-o/cri-o/pkg/annotations"
|
||||||
|
+ "github.com/docker/go-units"
|
||||||
|
vc "github.com/kata-containers/runtime/virtcontainers"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/device/config"
|
||||||
|
exp "github.com/kata-containers/runtime/virtcontainers/experimental"
|
||||||
|
@@ -34,7 +35,10 @@ type annotationContainerType struct {
|
||||||
|
|
||||||
|
type annotationHandler func(value string) error
|
||||||
|
|
||||||
|
-var annotationHandlerList = map[string]annotationHandler{}
|
||||||
|
+var annotationHandlerList = map[string]annotationHandler{
|
||||||
|
+ vcAnnotations.StaticCPUTypeKey: validateSandboxCPU,
|
||||||
|
+ vcAnnotations.StaticMemTypeKey: validateSandboxMem,
|
||||||
|
+}
|
||||||
|
|
||||||
|
var (
|
||||||
|
// ErrNoLinux is an error for missing Linux sections in the OCI configuration file.
|
||||||
|
@@ -1036,7 +1040,10 @@ func validateOtherSandboxAnnotations(annotation, value string) error {
|
||||||
|
|
||||||
|
// addOtherSandboxAnnotation add self defined annotation for sandbox
|
||||||
|
func addOtherSandboxAnnotation(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error {
|
||||||
|
- otherSandboxAnnotationsKey := []string{}
|
||||||
|
+ otherSandboxAnnotationsKey := []string{
|
||||||
|
+ vcAnnotations.StaticCPUTypeKey,
|
||||||
|
+ vcAnnotations.StaticMemTypeKey,
|
||||||
|
+ }
|
||||||
|
|
||||||
|
for _, a := range otherSandboxAnnotationsKey {
|
||||||
|
value, ok := ocispec.Annotations[a]
|
||||||
|
@@ -1053,3 +1060,31 @@ func addOtherSandboxAnnotation(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func validateSandboxCPU(value string) error {
|
||||||
|
+ // check min cpu value
|
||||||
|
+ cpus, err := utils.RoundVCPUNumber(value)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return fmt.Errorf("valiate sandbox_cpu annotation fail: %v", err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ maxPhysicalCPUs := utils.GetPhysicalCPUNumber()
|
||||||
|
+ if cpus > maxPhysicalCPUs {
|
||||||
|
+ return fmt.Errorf("sandbox_cpu annotation value exceed the machine max CPU number: %d", cpus)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func validateSandboxMem(value string) error {
|
||||||
|
+ memSizeInBytes, err := units.RAMInBytes(value)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return fmt.Errorf("parse sandbox_mem value: %d fail: %v", memSizeInBytes, err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if memSizeInBytes < utils.MinMemorySizeInByte || memSizeInBytes > utils.MaxMemorySizeInByte {
|
||||||
|
+ return fmt.Errorf("invalid sandbox_mem value size in bytes: %v", memSizeInBytes)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index 8fcd92d4..ba704249 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -20,6 +20,7 @@ import (
|
||||||
|
|
||||||
|
"github.com/containerd/cgroups"
|
||||||
|
"github.com/containernetworking/plugins/pkg/ns"
|
||||||
|
+ "github.com/docker/go-units"
|
||||||
|
"github.com/kata-containers/agent/protocols/grpc"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/device/api"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/device/config"
|
||||||
|
@@ -479,6 +480,10 @@ func createSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Fac
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
+ if err := updateStaticSandboxResources(&sandboxConfig); err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
s, err := newSandbox(ctx, sandboxConfig, factory)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
@@ -2359,3 +2364,30 @@ func (s *Sandbox) setContainersState(state types.StateString) error {
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+// updateStaticSandboxResources update sandbox's cpu and memory resource passed by
|
||||||
|
+// sandbox_cpu and sandbox_mem annotations
|
||||||
|
+func updateStaticSandboxResources(sandboxConfig *SandboxConfig) error {
|
||||||
|
+ // update cpu resource
|
||||||
|
+ if cpuNumVal, ok := sandboxConfig.Annotations[annotations.StaticCPUTypeKey]; ok {
|
||||||
|
+ cpuNum, err := utils.RoundVCPUNumber(cpuNumVal)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ sandboxConfig.HypervisorConfig.NumVCPUs = (uint32)(cpuNum)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // update mem resource
|
||||||
|
+ if memVal, ok := sandboxConfig.Annotations[annotations.StaticMemTypeKey]; ok {
|
||||||
|
+ memSizeInBytes, err := units.RAMInBytes(memVal)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ memSizeInMB := memSizeInBytes >> utils.MibToBytesShift
|
||||||
|
+ sandboxConfig.HypervisorConfig.MemorySize = (uint32)(memSizeInMB)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go
|
||||||
|
index 5d38e594..9490faa1 100644
|
||||||
|
--- a/virtcontainers/utils/utils.go
|
||||||
|
+++ b/virtcontainers/utils/utils.go
|
||||||
|
@@ -11,6 +11,7 @@ import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
|
+ "math"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
@@ -30,6 +31,9 @@ const MibToBytesShift = 20
|
||||||
|
const MaxHotplugMemMBOnceTime = 32 * 1024
|
||||||
|
|
||||||
|
const (
|
||||||
|
+ // minCPUs is allowed minimum CPU
|
||||||
|
+ minCPUs = 0.25
|
||||||
|
+
|
||||||
|
// Min needed memory size to start a Kata VM
|
||||||
|
MinMemorySizeInMB = 300
|
||||||
|
MinMemorySizeInByte = MinMemorySizeInMB << MibToBytesShift
|
||||||
|
@@ -365,3 +369,13 @@ func GetPhysicalCPUNumber() int {
|
||||||
|
}
|
||||||
|
return cpuNum
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func RoundVCPUNumber(value string) (int, error) {
|
||||||
|
+ cpuNum, err := strconv.ParseFloat(value, 64)
|
||||||
|
+ if err != nil || cpuNum < minCPUs {
|
||||||
|
+ return 0, fmt.Errorf("invalid sandbox cpu number: %v", cpuNum)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ cpus := int(math.Ceil(cpuNum))
|
||||||
|
+ return cpus, nil
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,39 @@
|
|||||||
|
From e5e3232f7268110f7e3e3c4814eab31a6704b672 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Mon, 10 Aug 2020 20:11:07 +0800
|
||||||
|
Subject: [PATCH 22/50] kata-runtime: skip go version check and do not build
|
||||||
|
containerd-shim-v2
|
||||||
|
|
||||||
|
reason: skip go version check and do not build containerd-shim-v2
|
||||||
|
because iSulad current not support shimV2
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
Makefile | 4 ++--
|
||||||
|
1 file changed, 2 insertions(+), 2 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/Makefile b/Makefile
|
||||||
|
index 14a0ea47..d5e4bbe1 100644
|
||||||
|
--- a/Makefile
|
||||||
|
+++ b/Makefile
|
||||||
|
@@ -12,7 +12,7 @@ for file in /etc/os-release /usr/lib/os-release; do \
|
||||||
|
fi \
|
||||||
|
done)
|
||||||
|
|
||||||
|
-SKIP_GO_VERSION_CHECK=
|
||||||
|
+SKIP_GO_VERSION_CHECK=y
|
||||||
|
include golang.mk
|
||||||
|
|
||||||
|
#Get ARCH.
|
||||||
|
@@ -503,7 +503,7 @@ define SHOW_ARCH
|
||||||
|
$(shell printf "\\t%s%s\\\n" "$(1)" $(if $(filter $(ARCH),$(1))," (default)",""))
|
||||||
|
endef
|
||||||
|
|
||||||
|
-all: runtime containerd-shim-v2 netmon
|
||||||
|
+all: runtime netmon
|
||||||
|
|
||||||
|
# Targets that depend on .git-commit can use $(shell cat .git-commit) to get a
|
||||||
|
# git revision string. They will only be rebuilt if the revision string
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,42 @@
|
|||||||
|
From 226c3336dcc70bd17e3471ff98106a2f8dee9ac5 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 11 Aug 2020 22:32:49 +0800
|
||||||
|
Subject: [PATCH 23/50] kata-runtime: set PCIBridgeMaxCapacity limit to 25
|
||||||
|
|
||||||
|
reason: set PCIBridgeMaxCapacity limit to 25.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/qemu_arch_base_test.go | 2 +-
|
||||||
|
virtcontainers/types/bridges.go | 2 +-
|
||||||
|
2 files changed, 2 insertions(+), 2 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/qemu_arch_base_test.go b/virtcontainers/qemu_arch_base_test.go
|
||||||
|
index 169e002e..8219f3c5 100644
|
||||||
|
--- a/virtcontainers/qemu_arch_base_test.go
|
||||||
|
+++ b/virtcontainers/qemu_arch_base_test.go
|
||||||
|
@@ -175,7 +175,7 @@ func TestQemuAddDeviceToBridge(t *testing.T) {
|
||||||
|
}
|
||||||
|
|
||||||
|
// fail to add device to bridge cause no more available bridge slot
|
||||||
|
- _, _, err := q.addDeviceToBridge("qemu-bridge-31", types.PCI)
|
||||||
|
+ _, _, err := q.addDeviceToBridge("qemu-bridge-26", types.PCI)
|
||||||
|
exceptErr := errors.New("no more bridge slots available")
|
||||||
|
assert.Equal(exceptErr.Error(), err.Error())
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/types/bridges.go b/virtcontainers/types/bridges.go
|
||||||
|
index cb15a88f..c3538ce4 100644
|
||||||
|
--- a/virtcontainers/types/bridges.go
|
||||||
|
+++ b/virtcontainers/types/bridges.go
|
||||||
|
@@ -10,7 +10,7 @@ import "fmt"
|
||||||
|
// Type represents a type of bus and bridge.
|
||||||
|
type Type string
|
||||||
|
|
||||||
|
-const PCIBridgeMaxCapacity = 30
|
||||||
|
+const PCIBridgeMaxCapacity = 25
|
||||||
|
|
||||||
|
const (
|
||||||
|
// PCI represents a PCI bus and bridge
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,462 @@
|
|||||||
|
From e861f426c9e6702e820348ddc61b18013c853402 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Thu, 13 Aug 2020 11:24:58 +0800
|
||||||
|
Subject: [PATCH 24/50] kata-runtime: support hotplug tap interface into kata
|
||||||
|
VM
|
||||||
|
|
||||||
|
reason: support hotplug exist tap interface or a new created tap
|
||||||
|
interface into kata VM.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
cli/network.go | 45 ++++++++++++++---
|
||||||
|
virtcontainers/kata_agent.go | 12 ++++-
|
||||||
|
virtcontainers/network.go | 100 +++++++++++++++++++++++---------------
|
||||||
|
virtcontainers/pkg/types/types.go | 16 +++---
|
||||||
|
virtcontainers/qemu.go | 11 +++--
|
||||||
|
virtcontainers/sandbox.go | 24 +++++++--
|
||||||
|
virtcontainers/tap_endpoint.go | 23 +++++++--
|
||||||
|
7 files changed, 161 insertions(+), 70 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/cli/network.go b/cli/network.go
|
||||||
|
index 881a2358..7e7791f1 100644
|
||||||
|
--- a/cli/network.go
|
||||||
|
+++ b/cli/network.go
|
||||||
|
@@ -26,6 +26,8 @@ const (
|
||||||
|
routeType
|
||||||
|
)
|
||||||
|
|
||||||
|
+const defaultLinkType = "tap"
|
||||||
|
+
|
||||||
|
var kataNetworkCLICommand = cli.Command{
|
||||||
|
Name: "kata-network",
|
||||||
|
Usage: "manage interfaces and routes for container",
|
||||||
|
@@ -42,10 +44,22 @@ var kataNetworkCLICommand = cli.Command{
|
||||||
|
}
|
||||||
|
|
||||||
|
var addIfaceCommand = cli.Command{
|
||||||
|
- Name: "add-iface",
|
||||||
|
- Usage: "add an interface to a container",
|
||||||
|
- ArgsUsage: `add-iface <container-id> file or - for stdin`,
|
||||||
|
- Flags: []cli.Flag{},
|
||||||
|
+ Name: "add-iface",
|
||||||
|
+ Usage: "add an interface to a container",
|
||||||
|
+ ArgsUsage: `add-iface <container-id> file or - for stdin
|
||||||
|
+ file or stdin for example:
|
||||||
|
+ {
|
||||||
|
+ "device":"<device-name>",
|
||||||
|
+ "name":"<interface-name>",
|
||||||
|
+ "IPAddresses":[{"address":"<ip>","mask":"<mask>"}],
|
||||||
|
+ "mtu":<mtu>,
|
||||||
|
+ "hwAddr":"<mac>",
|
||||||
|
+ "linkType":"tap",
|
||||||
|
+ "vhostUserSocket":"<path>"
|
||||||
|
+ }
|
||||||
|
+ device,name,mtu,hwAddr are required, IPAddresses and vhostUserSocket are optional.
|
||||||
|
+ `,
|
||||||
|
+ Flags: []cli.Flag{},
|
||||||
|
Action: func(context *cli.Context) error {
|
||||||
|
ctx, err := cliContextToContext(context)
|
||||||
|
if err != nil {
|
||||||
|
@@ -57,10 +71,20 @@ var addIfaceCommand = cli.Command{
|
||||||
|
}
|
||||||
|
|
||||||
|
var delIfaceCommand = cli.Command{
|
||||||
|
- Name: "del-iface",
|
||||||
|
- Usage: "delete an interface from a container",
|
||||||
|
- ArgsUsage: `del-iface <container-id> file or - for stdin`,
|
||||||
|
- Flags: []cli.Flag{},
|
||||||
|
+ Name: "del-iface",
|
||||||
|
+ Usage: "delete an interface from a container",
|
||||||
|
+ ArgsUsage: `del-iface <container-id> file or - for stdin
|
||||||
|
+ file or stdin for example:
|
||||||
|
+ {
|
||||||
|
+ "device":"",
|
||||||
|
+ "name":"<interface-name>",
|
||||||
|
+ "IPAddresses":[],
|
||||||
|
+ "mtu":0,
|
||||||
|
+ "hwAddr":""
|
||||||
|
+ }
|
||||||
|
+ Only the "name" field is required.
|
||||||
|
+ `,
|
||||||
|
+ Flags: []cli.Flag{},
|
||||||
|
Action: func(context *cli.Context) error {
|
||||||
|
ctx, err := cliContextToContext(context)
|
||||||
|
if err != nil {
|
||||||
|
@@ -156,6 +180,11 @@ func networkModifyCommand(ctx context.Context, containerID, input string, opType
|
||||||
|
if err = json.NewDecoder(f).Decode(&inf); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+ if len(inf.LinkType) == 0 {
|
||||||
|
+ inf.LinkType = defaultLinkType
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
if add {
|
||||||
|
resultingInf, err = vci.AddInterface(ctx, sandboxID, inf)
|
||||||
|
if err != nil {
|
||||||
|
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
|
||||||
|
index 8e073339..dfdd263b 100644
|
||||||
|
--- a/virtcontainers/kata_agent.go
|
||||||
|
+++ b/virtcontainers/kata_agent.go
|
||||||
|
@@ -600,8 +600,16 @@ func (k *kataAgent) updateInterface(ifc *vcTypes.Interface) (*vcTypes.Interface,
|
||||||
|
"resulting-interface": fmt.Sprintf("%+v", resultingInterface),
|
||||||
|
}).WithError(err).Error("update interface request failed")
|
||||||
|
}
|
||||||
|
- if resultInterface, ok := resultingInterface.(*vcTypes.Interface); ok {
|
||||||
|
- return resultInterface, err
|
||||||
|
+ if resultInterface, ok := resultingInterface.(*aTypes.Interface); ok {
|
||||||
|
+ iface := &vcTypes.Interface{
|
||||||
|
+ Device: resultInterface.Device,
|
||||||
|
+ Name: resultInterface.Name,
|
||||||
|
+ IPAddresses: k.convertToIPAddresses(resultInterface.IPAddresses),
|
||||||
|
+ Mtu: resultInterface.Mtu,
|
||||||
|
+ HwAddr: resultInterface.HwAddr,
|
||||||
|
+ PciAddr: resultInterface.PciAddr,
|
||||||
|
+ }
|
||||||
|
+ return iface, err
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
|
||||||
|
index d70c5360..e909a822 100644
|
||||||
|
--- a/virtcontainers/network.go
|
||||||
|
+++ b/virtcontainers/network.go
|
||||||
|
@@ -117,6 +117,7 @@ type NetlinkIface struct {
|
||||||
|
// NetworkInfo gathers all information related to a network interface.
|
||||||
|
// It can be used to store the description of the underlying network.
|
||||||
|
type NetworkInfo struct {
|
||||||
|
+ Device string
|
||||||
|
Iface NetlinkIface
|
||||||
|
Addrs []netlink.Addr
|
||||||
|
Routes []netlink.Route
|
||||||
|
@@ -303,6 +304,20 @@ func createLink(netHandle *netlink.Handle, name string, expectedLink netlink.Lin
|
||||||
|
var newLink netlink.Link
|
||||||
|
var fds []*os.File
|
||||||
|
|
||||||
|
+ // check if tapname exists, if true, use existing one instead of create new one
|
||||||
|
+ if name != "" {
|
||||||
|
+ retLink, err := netlink.LinkByName(name)
|
||||||
|
+ // link exist, use it
|
||||||
|
+ if err == nil {
|
||||||
|
+ networkLogger().Debugf("exist tap device is found, instead of creating new one")
|
||||||
|
+ tuntapLink, ok := retLink.(*netlink.Tuntap)
|
||||||
|
+ if ok {
|
||||||
|
+ fds = tuntapLink.Fds
|
||||||
|
+ }
|
||||||
|
+ return retLink, nil, nil
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
switch expectedLink.Type() {
|
||||||
|
case (&netlink.Tuntap{}).Type():
|
||||||
|
flags := netlink.TUNTAP_VNET_HDR
|
||||||
|
@@ -1156,7 +1171,7 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li
|
||||||
|
|
||||||
|
// Check if interface is a physical interface. Do not create
|
||||||
|
// tap interface/bridge if it is.
|
||||||
|
- isPhysical, err := isPhysicalIface(netInfo.Iface.Name)
|
||||||
|
+ isPhysical, err := isPhysicalIface(netInfo.Device)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
@@ -1164,49 +1179,54 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li
|
||||||
|
if isPhysical {
|
||||||
|
networkLogger().WithField("interface", netInfo.Iface.Name).Info("Physical network interface found")
|
||||||
|
endpoint, err = createPhysicalEndpoint(netInfo)
|
||||||
|
- } else {
|
||||||
|
- var socketPath string
|
||||||
|
+ return endpoint, err
|
||||||
|
+ }
|
||||||
|
|
||||||
|
- // Check if this is a dummy interface which has a vhost-user socket associated with it
|
||||||
|
- socketPath, err = vhostUserSocketPath(netInfo)
|
||||||
|
- if err != nil {
|
||||||
|
- return nil, err
|
||||||
|
- }
|
||||||
|
+ // Check if this is a dummy interface which has a vhost-user socket associated with it
|
||||||
|
+ socketPath, err := vhostUserSocketPath(netInfo)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
|
||||||
|
- if socketPath != "" {
|
||||||
|
- networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found")
|
||||||
|
- endpoint, err = createVhostUserEndpoint(netInfo, socketPath)
|
||||||
|
- } else if netInfo.Iface.Type == "macvlan" {
|
||||||
|
- networkLogger().Infof("macvlan interface found")
|
||||||
|
- endpoint, err = createBridgedMacvlanNetworkEndpoint(idx, netInfo.Iface.Name, model)
|
||||||
|
- } else if netInfo.Iface.Type == "macvtap" {
|
||||||
|
- networkLogger().Infof("macvtap interface found")
|
||||||
|
- endpoint, err = createMacvtapNetworkEndpoint(netInfo)
|
||||||
|
- } else if netInfo.Iface.Type == "tap" {
|
||||||
|
- networkLogger().Info("tap interface found")
|
||||||
|
- endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name)
|
||||||
|
- } else if netInfo.Iface.Type == "tuntap" {
|
||||||
|
- if link != nil {
|
||||||
|
- switch link.(*netlink.Tuntap).Mode {
|
||||||
|
- case 0:
|
||||||
|
- // mount /sys/class/net to get links
|
||||||
|
- return nil, fmt.Errorf("Network device mode not determined correctly. Mount sysfs in caller")
|
||||||
|
- case 1:
|
||||||
|
- return nil, fmt.Errorf("tun networking device not yet supported")
|
||||||
|
- case 2:
|
||||||
|
- networkLogger().Info("tuntap tap interface found")
|
||||||
|
- endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, model)
|
||||||
|
- default:
|
||||||
|
- return nil, fmt.Errorf("tuntap network %v mode unsupported", link.(*netlink.Tuntap).Mode)
|
||||||
|
- }
|
||||||
|
+ if socketPath != "" {
|
||||||
|
+ networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found")
|
||||||
|
+ endpoint, err = createVhostUserEndpoint(netInfo, socketPath)
|
||||||
|
+ return endpoint, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // We should create tap interface/bridge of other interface type.
|
||||||
|
+ networkLogger().Infof("%s interface found", netInfo.Iface.Type)
|
||||||
|
+ switch netInfo.Iface.Type {
|
||||||
|
+ case "macvlan":
|
||||||
|
+ networkLogger().Infof("macvlan interface found")
|
||||||
|
+ endpoint, err = createBridgedMacvlanNetworkEndpoint(idx, netInfo.Iface.Name, model)
|
||||||
|
+ case "macvtap":
|
||||||
|
+ networkLogger().Infof("macvtap interface found")
|
||||||
|
+ endpoint, err = createMacvtapNetworkEndpoint(netInfo)
|
||||||
|
+ case "tap":
|
||||||
|
+ networkLogger().Info("tap interface found")
|
||||||
|
+ endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Device)
|
||||||
|
+ case "tuntap":
|
||||||
|
+ if link != nil {
|
||||||
|
+ switch link.(*netlink.Tuntap).Mode {
|
||||||
|
+ case 0:
|
||||||
|
+ // mount /sys/class/net to get links
|
||||||
|
+ return nil, fmt.Errorf("Network device mode not determined correctly. Mount sysfs in caller")
|
||||||
|
+ case 1:
|
||||||
|
+ return nil, fmt.Errorf("tun networking device not yet supported")
|
||||||
|
+ case 2:
|
||||||
|
+ networkLogger().Info("tuntap tap interface found")
|
||||||
|
+ endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, model)
|
||||||
|
+ default:
|
||||||
|
+ return nil, fmt.Errorf("tuntap network %v mode unsupported", link.(*netlink.Tuntap).Mode)
|
||||||
|
}
|
||||||
|
- } else if netInfo.Iface.Type == "veth" {
|
||||||
|
- endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model)
|
||||||
|
- } else if netInfo.Iface.Type == "ipvlan" {
|
||||||
|
- endpoint, err = createIPVlanNetworkEndpoint(idx, netInfo.Iface.Name)
|
||||||
|
- } else {
|
||||||
|
- return nil, fmt.Errorf("Unsupported network interface: %s", netInfo.Iface.Type)
|
||||||
|
}
|
||||||
|
+ case "veth":
|
||||||
|
+ endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model)
|
||||||
|
+ case "ipvlan":
|
||||||
|
+ endpoint, err = createIPVlanNetworkEndpoint(idx, netInfo.Iface.Name)
|
||||||
|
+ default:
|
||||||
|
+ err = fmt.Errorf("Unsupported network interface, %s", netInfo.Iface.Type)
|
||||||
|
}
|
||||||
|
|
||||||
|
return endpoint, err
|
||||||
|
diff --git a/virtcontainers/pkg/types/types.go b/virtcontainers/pkg/types/types.go
|
||||||
|
index 0d4a9cfa..fcc63d84 100644
|
||||||
|
--- a/virtcontainers/pkg/types/types.go
|
||||||
|
+++ b/virtcontainers/pkg/types/types.go
|
||||||
|
@@ -14,21 +14,21 @@ type IPAddress struct {
|
||||||
|
|
||||||
|
// Interface describes a network interface.
|
||||||
|
type Interface struct {
|
||||||
|
- Device string
|
||||||
|
- Name string
|
||||||
|
- IPAddresses []*IPAddress
|
||||||
|
- Mtu uint64
|
||||||
|
- RawFlags uint32
|
||||||
|
- HwAddr string
|
||||||
|
+ Device string `json:"device,omitempty"`
|
||||||
|
+ Name string `json:"name,omitempty"`
|
||||||
|
+ IPAddresses []*IPAddress `json:"IPAddresses,omitempty"`
|
||||||
|
+ Mtu uint64 `json:"mtu,omitempty"`
|
||||||
|
+ RawFlags uint32 `json:"rawFlags,omitempty"`
|
||||||
|
+ HwAddr string `json:"hwAddr,omitempty"`
|
||||||
|
// pciAddr is the PCI address in the format "bridgeAddr/deviceAddr".
|
||||||
|
// Here, bridgeAddr is the address at which the bridge is attached on the root bus,
|
||||||
|
// while deviceAddr is the address at which the network device is attached on the bridge.
|
||||||
|
- PciAddr string
|
||||||
|
+ PciAddr string `json:"pciAddr,omitempty"`
|
||||||
|
// LinkType defines the type of interface described by this structure.
|
||||||
|
// The expected values are the one that are defined by the netlink
|
||||||
|
// library, regarding each type of link. Here is a non exhaustive
|
||||||
|
// list: "veth", "macvtap", "vlan", "macvlan", "tap", ...
|
||||||
|
- LinkType string
|
||||||
|
+ LinkType string `json:"linkType,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Route describes a network route.
|
||||||
|
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
|
||||||
|
index 7bae3278..bb83b1bb 100644
|
||||||
|
--- a/virtcontainers/qemu.go
|
||||||
|
+++ b/virtcontainers/qemu.go
|
||||||
|
@@ -1361,7 +1361,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
-func (q *qemu) hotAddNetDevice(name, hardAddr string, VMFds, VhostFds []*os.File) error {
|
||||||
|
+func (q *qemu) hotAddNetDevice(deviceName, name, hardAddr string, VMFds, VhostFds []*os.File) error {
|
||||||
|
var (
|
||||||
|
VMFdNames []string
|
||||||
|
VhostFdNames []string
|
||||||
|
@@ -1381,7 +1381,12 @@ func (q *qemu) hotAddNetDevice(name, hardAddr string, VMFds, VhostFds []*os.File
|
||||||
|
VhostFd.Close()
|
||||||
|
VhostFdNames = append(VhostFdNames, fdName)
|
||||||
|
}
|
||||||
|
- return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", name, VMFdNames, VhostFdNames)
|
||||||
|
+
|
||||||
|
+ if len(VMFdNames) != 0 || len(VhostFdNames) != 0 {
|
||||||
|
+ return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", name, VMFdNames, VhostFdNames)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return q.qmpMonitorCh.qmp.ExecuteNetdevAdd(q.qmpMonitorCh.ctx, "tap", name, deviceName, "no", "no", 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
|
||||||
|
@@ -1404,7 +1409,7 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
|
||||||
|
|
||||||
|
devID := "virtio-" + tap.ID
|
||||||
|
if op == addDevice {
|
||||||
|
- if err = q.hotAddNetDevice(tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds); err != nil {
|
||||||
|
+ if err = q.hotAddNetDevice(tap.TAPIface.Name, tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index ba704249..c8981a41 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -937,6 +937,7 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) {
|
||||||
|
}
|
||||||
|
|
||||||
|
return NetworkInfo{
|
||||||
|
+ Device: inf.Device,
|
||||||
|
Iface: NetlinkIface{
|
||||||
|
LinkAttrs: netlink.LinkAttrs{
|
||||||
|
Name: inf.Name,
|
||||||
|
@@ -950,7 +951,7 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) {
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddInterface adds new nic to the sandbox.
|
||||||
|
-func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (*vcTypes.Interface, error) {
|
||||||
|
+func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (grpcIf *vcTypes.Interface, err error) {
|
||||||
|
netInfo, err := s.generateNetInfo(inf)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
@@ -962,28 +963,41 @@ func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (*vcTypes.Interface, erro
|
||||||
|
}
|
||||||
|
|
||||||
|
endpoint.SetProperties(netInfo)
|
||||||
|
- if err := doNetNS(s.networkNS.NetNsPath, func(_ ns.NetNS) error {
|
||||||
|
+ if err = doNetNS(s.networkNS.NetNsPath, func(_ ns.NetNS) error {
|
||||||
|
s.Logger().WithField("endpoint-type", endpoint.Type()).Info("Hot attaching endpoint")
|
||||||
|
return endpoint.HotAttach(s.hypervisor)
|
||||||
|
}); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
+ defer func() {
|
||||||
|
+ if err != nil {
|
||||||
|
+ if errDetach := endpoint.HotDetach(s.hypervisor, s.networkNS.NetNsCreated, s.networkNS.NetNsPath); errDetach != nil {
|
||||||
|
+ s.Logger().WithField("endpoint-type", endpoint.Type()).Errorf("rollback hot attach endpoint failed")
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
// Update the sandbox storage
|
||||||
|
s.networkNS.Endpoints = append(s.networkNS.Endpoints, endpoint)
|
||||||
|
- if err := s.Save(); err != nil {
|
||||||
|
+ if err = s.Save(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add network for vm
|
||||||
|
inf.PciAddr = endpoint.PciAddr()
|
||||||
|
- return s.agent.updateInterface(inf)
|
||||||
|
+ grpcIf, err = s.agent.updateInterface(inf)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return
|
||||||
|
}
|
||||||
|
|
||||||
|
// RemoveInterface removes a nic of the sandbox.
|
||||||
|
func (s *Sandbox) RemoveInterface(inf *vcTypes.Interface) (*vcTypes.Interface, error) {
|
||||||
|
for i, endpoint := range s.networkNS.Endpoints {
|
||||||
|
- if endpoint.HardwareAddr() == inf.HwAddr {
|
||||||
|
+ if endpoint.HardwareAddr() == inf.HwAddr || endpoint.Name() == inf.Name {
|
||||||
|
s.Logger().WithField("endpoint-type", endpoint.Type()).Info("Hot detaching endpoint")
|
||||||
|
if err := endpoint.HotDetach(s.hypervisor, s.networkNS.NetNsCreated, s.networkNS.NetNsPath); err != nil {
|
||||||
|
return inf, err
|
||||||
|
diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go
|
||||||
|
index cb441b87..7d33d5a2 100644
|
||||||
|
--- a/virtcontainers/tap_endpoint.go
|
||||||
|
+++ b/virtcontainers/tap_endpoint.go
|
||||||
|
@@ -7,6 +7,7 @@ package virtcontainers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
+ "os"
|
||||||
|
|
||||||
|
"github.com/containernetworking/plugins/pkg/ns"
|
||||||
|
"github.com/vishvananda/netlink"
|
||||||
|
@@ -111,7 +112,7 @@ func (endpoint *TapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPat
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
-func createTapNetworkEndpoint(idx int, ifName string) (*TapEndpoint, error) {
|
||||||
|
+func createTapNetworkEndpoint(idx int, ifName string, tapIfName string) (*TapEndpoint, error) {
|
||||||
|
if idx < 0 {
|
||||||
|
return &TapEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx)
|
||||||
|
}
|
||||||
|
@@ -131,6 +132,10 @@ func createTapNetworkEndpoint(idx int, ifName string) (*TapEndpoint, error) {
|
||||||
|
endpoint.TapInterface.Name = ifName
|
||||||
|
}
|
||||||
|
|
||||||
|
+ if tapIfName != "" {
|
||||||
|
+ endpoint.TapInterface.TAPIface.Name = tapIfName
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
return endpoint, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -145,9 +150,19 @@ func tapNetwork(endpoint *TapEndpoint, numCPUs uint32, disableVhostNet bool) err
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Could not create TAP interface: %s", err)
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+ defer func() {
|
||||||
|
+ if err != nil {
|
||||||
|
+ if errDel := netHandle.LinkDel(tapLink); errDel != nil {
|
||||||
|
+ networkLogger().WithError(errDel).Error("tapNetwork fail to rollback del link")
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
endpoint.TapInterface.VMFds = fds
|
||||||
|
if !disableVhostNet {
|
||||||
|
- vhostFds, err := createVhostFds(int(numCPUs))
|
||||||
|
+ var vhostFds []*os.File
|
||||||
|
+ vhostFds, err = createVhostFds(int(numCPUs))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Could not setup vhost fds %s : %s", endpoint.TapInterface.Name, err)
|
||||||
|
}
|
||||||
|
@@ -161,10 +176,10 @@ func tapNetwork(endpoint *TapEndpoint, numCPUs uint32, disableVhostNet bool) err
|
||||||
|
// bridge created by the network plugin on the host actually expects
|
||||||
|
// to see traffic from this MAC address and not another one.
|
||||||
|
endpoint.TapInterface.TAPIface.HardAddr = linkAttrs.HardwareAddr.String()
|
||||||
|
- if err := netHandle.LinkSetMTU(tapLink, linkAttrs.MTU); err != nil {
|
||||||
|
+ if err = netHandle.LinkSetMTU(tapLink, linkAttrs.MTU); err != nil {
|
||||||
|
return fmt.Errorf("Could not set TAP MTU %d: %s", linkAttrs.MTU, err)
|
||||||
|
}
|
||||||
|
- if err := netHandle.LinkSetUp(tapLink); err != nil {
|
||||||
|
+ if err = netHandle.LinkSetUp(tapLink); err != nil {
|
||||||
|
return fmt.Errorf("Could not enable TAP %s: %s", endpoint.TapInterface.Name, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,260 @@
|
|||||||
|
From be8153f21c0b81d2b194075ecd654501bc708577 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Thu, 13 Aug 2020 18:54:49 +0800
|
||||||
|
Subject: [PATCH 25/50] network: keep list-ifaces result compatible with cni
|
||||||
|
|
||||||
|
reason: community list-ifaces command will return the all
|
||||||
|
interfaces info in the Kata VM, however we may just want
|
||||||
|
to get the interfaces that we hotplug, so just return the
|
||||||
|
hotplugged interfaces and convert the interface info to
|
||||||
|
be compatible with cni.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
cli/network.go | 29 ++++++++++++++++++++++-
|
||||||
|
virtcontainers/api.go | 4 +++-
|
||||||
|
virtcontainers/endpoint.go | 44 +++++++++++++++++++++++++++++++++++
|
||||||
|
virtcontainers/network.go | 27 +++++++++++++++++++++
|
||||||
|
virtcontainers/persist/api/network.go | 24 +++++++++++++++++++
|
||||||
|
virtcontainers/tap_endpoint.go | 9 +++++++
|
||||||
|
6 files changed, 135 insertions(+), 2 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/cli/network.go b/cli/network.go
|
||||||
|
index 7e7791f1..66955725 100644
|
||||||
|
--- a/cli/network.go
|
||||||
|
+++ b/cli/network.go
|
||||||
|
@@ -28,6 +28,13 @@ const (
|
||||||
|
|
||||||
|
const defaultLinkType = "tap"
|
||||||
|
|
||||||
|
+type compatInterface struct {
|
||||||
|
+ Name string `json:"name,omitempty"`
|
||||||
|
+ Mac string `json:"mac,omitempty"`
|
||||||
|
+ IP []string `json:"ip,omitempty"`
|
||||||
|
+ Mtu int `json:"mtu,omitempty"`
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
var kataNetworkCLICommand = cli.Command{
|
||||||
|
Name: "kata-network",
|
||||||
|
Usage: "manage interfaces and routes for container",
|
||||||
|
@@ -244,7 +251,8 @@ func networkListCommand(ctx context.Context, containerID string, opType networkT
|
||||||
|
kataLog.WithField("existing-interfaces", fmt.Sprintf("%+v", interfaces)).
|
||||||
|
WithError(err).Error("list interfaces failed")
|
||||||
|
}
|
||||||
|
- json.NewEncoder(file).Encode(interfaces)
|
||||||
|
+ compatInfs := convertCompatInterfaces(interfaces)
|
||||||
|
+ json.NewEncoder(file).Encode(compatInfs)
|
||||||
|
case routeType:
|
||||||
|
var routes []*vcTypes.Route
|
||||||
|
routes, err = vci.ListRoutes(ctx, sandboxID)
|
||||||
|
@@ -256,3 +264,22 @@ func networkListCommand(ctx context.Context, containerID string, opType networkT
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func convertCompatInterfaces(interfaces []*vcTypes.Interface) []compatInterface {
|
||||||
|
+ var infs []compatInterface
|
||||||
|
+ for _, i := range interfaces {
|
||||||
|
+ var addrs []string
|
||||||
|
+ for _, a := range i.IPAddresses {
|
||||||
|
+ addrs = append(addrs, fmt.Sprintf("%s/%s", a.Address, a.Mask))
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ infs = append(infs, compatInterface{
|
||||||
|
+ Name: i.Name,
|
||||||
|
+ Mac: i.HwAddr,
|
||||||
|
+ IP: addrs,
|
||||||
|
+ Mtu: int(i.Mtu),
|
||||||
|
+ })
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return infs
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
|
||||||
|
index 5e8c9c9e..eb5b4995 100644
|
||||||
|
--- a/virtcontainers/api.go
|
||||||
|
+++ b/virtcontainers/api.go
|
||||||
|
@@ -949,7 +949,9 @@ func ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTypes.Interface
|
||||||
|
}
|
||||||
|
defer s.releaseStatelessSandbox()
|
||||||
|
|
||||||
|
- return s.ListInterfaces()
|
||||||
|
+ // get interfaces info from persist.json file
|
||||||
|
+ // instead of by s.ListInterfaces()
|
||||||
|
+ return convertToCompatInterfaces(&s.networkNS.Endpoints), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpdateRoutes is the virtcontainers update routes entry point.
|
||||||
|
diff --git a/virtcontainers/endpoint.go b/virtcontainers/endpoint.go
|
||||||
|
index 01b5e77f..7efcf49c 100644
|
||||||
|
--- a/virtcontainers/endpoint.go
|
||||||
|
+++ b/virtcontainers/endpoint.go
|
||||||
|
@@ -132,6 +132,28 @@ func saveTapIf(tapif *TapInterface) *persistapi.TapInterface {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
+func saveTapEndpointProperties(networkInfo *NetworkInfo) *persistapi.NetworkProperties {
|
||||||
|
+ if networkInfo == nil {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return &persistapi.NetworkProperties{
|
||||||
|
+ Device: networkInfo.Device,
|
||||||
|
+ Iface: persistapi.NetlinkIface{
|
||||||
|
+ LinkAttrs: networkInfo.Iface.LinkAttrs,
|
||||||
|
+ Type: networkInfo.Iface.Type,
|
||||||
|
+ },
|
||||||
|
+ Addrs: networkInfo.Addrs,
|
||||||
|
+ Routes: networkInfo.Routes,
|
||||||
|
+ DNS: persistapi.DNSInfo{
|
||||||
|
+ Servers: networkInfo.DNS.Servers,
|
||||||
|
+ Domain: networkInfo.DNS.Domain,
|
||||||
|
+ Searches: networkInfo.DNS.Searches,
|
||||||
|
+ Options: networkInfo.DNS.Options,
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func loadTapIf(tapif *persistapi.TapInterface) *TapInterface {
|
||||||
|
if tapif == nil {
|
||||||
|
return nil
|
||||||
|
@@ -148,6 +170,28 @@ func loadTapIf(tapif *persistapi.TapInterface) *TapInterface {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
+func loadTapEndpointProperties(endpointProperties *persistapi.NetworkProperties) *NetworkInfo {
|
||||||
|
+ if endpointProperties == nil {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return &NetworkInfo{
|
||||||
|
+ Device: endpointProperties.Device,
|
||||||
|
+ Iface: NetlinkIface{
|
||||||
|
+ LinkAttrs: endpointProperties.Iface.LinkAttrs,
|
||||||
|
+ Type: endpointProperties.Iface.Type,
|
||||||
|
+ },
|
||||||
|
+ Addrs: endpointProperties.Addrs,
|
||||||
|
+ Routes: endpointProperties.Routes,
|
||||||
|
+ DNS: DNSInfo{
|
||||||
|
+ Servers: endpointProperties.DNS.Servers,
|
||||||
|
+ Domain: endpointProperties.DNS.Domain,
|
||||||
|
+ Searches: endpointProperties.DNS.Searches,
|
||||||
|
+ Options: endpointProperties.DNS.Options,
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func saveNetIfPair(pair *NetworkInterfacePair) *persistapi.NetworkInterfacePair {
|
||||||
|
if pair == nil {
|
||||||
|
return nil
|
||||||
|
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
|
||||||
|
index e909a822..bf7f9336 100644
|
||||||
|
--- a/virtcontainers/network.go
|
||||||
|
+++ b/virtcontainers/network.go
|
||||||
|
@@ -1340,3 +1340,30 @@ func (n *Network) Remove(ctx context.Context, ns *NetworkNamespace, hypervisor h
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+// convertCompatInterfaces convert Endpoint info to vcTypes.Interface
|
||||||
|
+func convertToCompatInterfaces(es *[]Endpoint) []*vcTypes.Interface {
|
||||||
|
+ var infs []*vcTypes.Interface
|
||||||
|
+ for _, e := range *es {
|
||||||
|
+ var addrs []*vcTypes.IPAddress
|
||||||
|
+ for _, a := range e.Properties().Addrs {
|
||||||
|
+ m, _ := a.Mask.Size()
|
||||||
|
+ addr := &vcTypes.IPAddress{
|
||||||
|
+ Address: fmt.Sprintf("%s", a.IP),
|
||||||
|
+ Mask: fmt.Sprintf("%d", m),
|
||||||
|
+ }
|
||||||
|
+ addrs = append(addrs, addr)
|
||||||
|
+ }
|
||||||
|
+ inf := &vcTypes.Interface{
|
||||||
|
+ LinkType: string(e.Type()),
|
||||||
|
+ Name: e.Name(),
|
||||||
|
+ Mtu: uint64(e.Properties().Iface.MTU),
|
||||||
|
+ HwAddr: e.HardwareAddr(),
|
||||||
|
+ IPAddresses: addrs,
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ infs = append(infs, inf)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return infs
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/persist/api/network.go b/virtcontainers/persist/api/network.go
|
||||||
|
index 69610c67..53c6de44 100644
|
||||||
|
--- a/virtcontainers/persist/api/network.go
|
||||||
|
+++ b/virtcontainers/persist/api/network.go
|
||||||
|
@@ -11,6 +11,27 @@ import (
|
||||||
|
)
|
||||||
|
|
||||||
|
// ============= sandbox level resources =============
|
||||||
|
+// DNSInfo describes the DNS setup related to a network interface.
|
||||||
|
+type DNSInfo struct {
|
||||||
|
+ Servers []string
|
||||||
|
+ Domain string
|
||||||
|
+ Searches []string
|
||||||
|
+ Options []string
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// NetlinkIface describes fully a network interface.
|
||||||
|
+type NetlinkIface struct {
|
||||||
|
+ netlink.LinkAttrs
|
||||||
|
+ Type string
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+type NetworkProperties struct {
|
||||||
|
+ Device string
|
||||||
|
+ Iface NetlinkIface
|
||||||
|
+ Addrs []netlink.Addr
|
||||||
|
+ Routes []netlink.Route
|
||||||
|
+ DNS DNSInfo
|
||||||
|
+}
|
||||||
|
|
||||||
|
type NetworkInterface struct {
|
||||||
|
Name string
|
||||||
|
@@ -91,6 +112,9 @@ type NetworkEndpoint struct {
|
||||||
|
Tap *TapEndpoint `json:",omitempty"`
|
||||||
|
IPVlan *IPVlanEndpoint `json:",omitempty"`
|
||||||
|
Tuntap *TuntapEndpoint `json:",omitempty"`
|
||||||
|
+
|
||||||
|
+ // store the endpoint properties info
|
||||||
|
+ EndPointProperties *NetworkProperties `json:",omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// NetworkInfo contains network information of sandbox
|
||||||
|
diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go
|
||||||
|
index 7d33d5a2..c897670e 100644
|
||||||
|
--- a/virtcontainers/tap_endpoint.go
|
||||||
|
+++ b/virtcontainers/tap_endpoint.go
|
||||||
|
@@ -206,12 +206,15 @@ func unTapNetwork(name string) error {
|
||||||
|
|
||||||
|
func (endpoint *TapEndpoint) save() persistapi.NetworkEndpoint {
|
||||||
|
tapif := saveTapIf(&endpoint.TapInterface)
|
||||||
|
+ // save tap endpoint network properties into persist storage
|
||||||
|
+ properties := saveTapEndpointProperties(&endpoint.EndpointProperties)
|
||||||
|
|
||||||
|
return persistapi.NetworkEndpoint{
|
||||||
|
Type: string(endpoint.Type()),
|
||||||
|
Tap: &persistapi.TapEndpoint{
|
||||||
|
TapInterface: *tapif,
|
||||||
|
},
|
||||||
|
+ EndPointProperties: properties,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func (endpoint *TapEndpoint) load(s persistapi.NetworkEndpoint) {
|
||||||
|
@@ -221,4 +224,10 @@ func (endpoint *TapEndpoint) load(s persistapi.NetworkEndpoint) {
|
||||||
|
tapif := loadTapIf(&s.Tap.TapInterface)
|
||||||
|
endpoint.TapInterface = *tapif
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+ if s.EndPointProperties != nil {
|
||||||
|
+ // restore tap endpoint network properties from persist storage
|
||||||
|
+ properties := loadTapEndpointProperties(s.EndPointProperties)
|
||||||
|
+ endpoint.EndpointProperties = *properties
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,285 @@
|
|||||||
|
From eeca1e47e9a6422d89d08275864f2c1b15e54941 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Fri, 14 Aug 2020 19:14:35 +0800
|
||||||
|
Subject: [PATCH 26/50] network: add enable_compat_old_cni config
|
||||||
|
|
||||||
|
reason: old version kata-network list-ifaces output result different
|
||||||
|
from the community version, inorder to compatible with the old version
|
||||||
|
list-ifaces command output format, add enable_compat_old_cni to control
|
||||||
|
the list-ifaces command output format.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
cli/config/configuration-qemu.toml.in | 4 ++++
|
||||||
|
cli/network.go | 17 +++++++++++++++--
|
||||||
|
cli/network_test.go | 6 ++++++
|
||||||
|
pkg/katautils/config.go | 2 ++
|
||||||
|
virtcontainers/api.go | 10 +++++++---
|
||||||
|
virtcontainers/interfaces.go | 1 +
|
||||||
|
virtcontainers/network.go | 11 ++++++-----
|
||||||
|
virtcontainers/persist.go | 18 ++++++++++--------
|
||||||
|
virtcontainers/persist/api/config.go | 9 +++++----
|
||||||
|
virtcontainers/pkg/oci/utils.go | 4 ++++
|
||||||
|
virtcontainers/pkg/vcmock/sandbox.go | 5 +++++
|
||||||
|
virtcontainers/sandbox.go | 5 +++++
|
||||||
|
12 files changed, 70 insertions(+), 22 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in
|
||||||
|
index b44e84d8..46f8b632 100644
|
||||||
|
--- a/cli/config/configuration-qemu.toml.in
|
||||||
|
+++ b/cli/config/configuration-qemu.toml.in
|
||||||
|
@@ -453,6 +453,10 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
|
||||||
|
# (default: false)
|
||||||
|
#disable_new_netns = true
|
||||||
|
|
||||||
|
+# If enabled, the kata-network will return the old interface format info to be compatible with
|
||||||
|
+# old version CNI plugin
|
||||||
|
+enable_compat_old_cni = true
|
||||||
|
+
|
||||||
|
# if enabled, the runtime will add all the kata processes inside one dedicated cgroup.
|
||||||
|
# The container cgroups in the host are not created, just one single cgroup per sandbox.
|
||||||
|
# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.
|
||||||
|
diff --git a/cli/network.go b/cli/network.go
|
||||||
|
index 66955725..a1a24425 100644
|
||||||
|
--- a/cli/network.go
|
||||||
|
+++ b/cli/network.go
|
||||||
|
@@ -251,8 +251,21 @@ func networkListCommand(ctx context.Context, containerID string, opType networkT
|
||||||
|
kataLog.WithField("existing-interfaces", fmt.Sprintf("%+v", interfaces)).
|
||||||
|
WithError(err).Error("list interfaces failed")
|
||||||
|
}
|
||||||
|
- compatInfs := convertCompatInterfaces(interfaces)
|
||||||
|
- json.NewEncoder(file).Encode(compatInfs)
|
||||||
|
+
|
||||||
|
+ sandbox, err := vci.FetchSandbox(ctx, sandboxID)
|
||||||
|
+ if err != nil {
|
||||||
|
+ kataLog.WithField("existing-interfaces", fmt.Sprintf("%+v", interfaces)).
|
||||||
|
+ WithError(err).Error("fetch sandbox failed")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // If sandbox network config need to be compatible with old CNI,
|
||||||
|
+ // convert the interface format to old version format.
|
||||||
|
+ if sandbox.IsCompatOldCNI() {
|
||||||
|
+ compatInfs := convertCompatInterfaces(interfaces)
|
||||||
|
+ json.NewEncoder(file).Encode(compatInfs)
|
||||||
|
+ } else {
|
||||||
|
+ json.NewEncoder(file).Encode(interfaces)
|
||||||
|
+ }
|
||||||
|
case routeType:
|
||||||
|
var routes []*vcTypes.Route
|
||||||
|
routes, err = vci.ListRoutes(ctx, sandboxID)
|
||||||
|
diff --git a/cli/network_test.go b/cli/network_test.go
|
||||||
|
index 4e3d943d..95fd2749 100644
|
||||||
|
--- a/cli/network_test.go
|
||||||
|
+++ b/cli/network_test.go
|
||||||
|
@@ -17,6 +17,7 @@ import (
|
||||||
|
|
||||||
|
vc "github.com/kata-containers/runtime/virtcontainers"
|
||||||
|
vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
@@ -59,6 +60,10 @@ func TestNetworkCliFunction(t *testing.T) {
|
||||||
|
return newSingleContainerStatus(testContainerID, state, map[string]string{}, &specs.Spec{}), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+ testingImpl.FetchSandboxFunc = func(ctx context.Context, id string) (vc.VCSandbox, error) {
|
||||||
|
+ return &vcmock.Sandbox{}, nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
defer func() {
|
||||||
|
testingImpl.AddInterfaceFunc = nil
|
||||||
|
testingImpl.RemoveInterfaceFunc = nil
|
||||||
|
@@ -66,6 +71,7 @@ func TestNetworkCliFunction(t *testing.T) {
|
||||||
|
testingImpl.UpdateRoutesFunc = nil
|
||||||
|
testingImpl.ListRoutesFunc = nil
|
||||||
|
testingImpl.StatusContainerFunc = nil
|
||||||
|
+ testingImpl.FetchSandboxFunc = nil
|
||||||
|
}()
|
||||||
|
|
||||||
|
set := flag.NewFlagSet("", 0)
|
||||||
|
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
||||||
|
index 9a99b9d4..94c916a0 100644
|
||||||
|
--- a/pkg/katautils/config.go
|
||||||
|
+++ b/pkg/katautils/config.go
|
||||||
|
@@ -141,6 +141,7 @@ type runtime struct {
|
||||||
|
Debug bool `toml:"enable_debug"`
|
||||||
|
Tracing bool `toml:"enable_tracing"`
|
||||||
|
DisableNewNetNs bool `toml:"disable_new_netns"`
|
||||||
|
+ EnableCompatOldCNI bool `toml:"enable_compat_old_cni"`
|
||||||
|
DisableGuestSeccomp bool `toml:"disable_guest_seccomp"`
|
||||||
|
SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"`
|
||||||
|
Experimental []string `toml:"experimental"`
|
||||||
|
@@ -1235,6 +1236,7 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool, debugFlag
|
||||||
|
|
||||||
|
config.SandboxCgroupOnly = tomlConf.Runtime.SandboxCgroupOnly
|
||||||
|
config.DisableNewNetNs = tomlConf.Runtime.DisableNewNetNs
|
||||||
|
+ config.EnableCompatOldCNI = tomlConf.Runtime.EnableCompatOldCNI
|
||||||
|
for _, f := range tomlConf.Runtime.Experimental {
|
||||||
|
feature := exp.Get(f)
|
||||||
|
if feature == nil {
|
||||||
|
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
|
||||||
|
index eb5b4995..fb044fe1 100644
|
||||||
|
--- a/virtcontainers/api.go
|
||||||
|
+++ b/virtcontainers/api.go
|
||||||
|
@@ -949,9 +949,13 @@ func ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTypes.Interface
|
||||||
|
}
|
||||||
|
defer s.releaseStatelessSandbox()
|
||||||
|
|
||||||
|
- // get interfaces info from persist.json file
|
||||||
|
- // instead of by s.ListInterfaces()
|
||||||
|
- return convertToCompatInterfaces(&s.networkNS.Endpoints), nil
|
||||||
|
+ // If enable_compat_old_cni is enabled, get interfaces info from
|
||||||
|
+ // persist.json file instead of by s.ListInterfaces()
|
||||||
|
+ if s.config.NetworkConfig.EnableCompatOldCNI {
|
||||||
|
+ return convertToCompatInterfaces(&s.networkNS.Endpoints), nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return s.ListInterfaces()
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpdateRoutes is the virtcontainers update routes entry point.
|
||||||
|
diff --git a/virtcontainers/interfaces.go b/virtcontainers/interfaces.go
|
||||||
|
index fa6b584e..499b386e 100644
|
||||||
|
--- a/virtcontainers/interfaces.go
|
||||||
|
+++ b/virtcontainers/interfaces.go
|
||||||
|
@@ -98,6 +98,7 @@ type VCSandbox interface {
|
||||||
|
ListInterfaces() ([]*vcTypes.Interface, error)
|
||||||
|
UpdateRoutes(routes []*vcTypes.Route) ([]*vcTypes.Route, error)
|
||||||
|
ListRoutes() ([]*vcTypes.Route, error)
|
||||||
|
+ IsCompatOldCNI() bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// VCContainer is the Container interface
|
||||||
|
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
|
||||||
|
index bf7f9336..db235cf6 100644
|
||||||
|
--- a/virtcontainers/network.go
|
||||||
|
+++ b/virtcontainers/network.go
|
||||||
|
@@ -155,11 +155,12 @@ type NetworkInterfacePair struct {
|
||||||
|
|
||||||
|
// NetworkConfig is the network configuration related to a network.
|
||||||
|
type NetworkConfig struct {
|
||||||
|
- NetNSPath string
|
||||||
|
- NetNsCreated bool
|
||||||
|
- DisableNewNetNs bool
|
||||||
|
- NetmonConfig NetmonConfig
|
||||||
|
- InterworkingModel NetInterworkingModel
|
||||||
|
+ NetNSPath string
|
||||||
|
+ NetNsCreated bool
|
||||||
|
+ DisableNewNetNs bool
|
||||||
|
+ EnableCompatOldCNI bool
|
||||||
|
+ NetmonConfig NetmonConfig
|
||||||
|
+ InterworkingModel NetInterworkingModel
|
||||||
|
}
|
||||||
|
|
||||||
|
func networkLogger() *logrus.Entry {
|
||||||
|
diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go
|
||||||
|
index 6bd09a0b..fe00bf9a 100644
|
||||||
|
--- a/virtcontainers/persist.go
|
||||||
|
+++ b/virtcontainers/persist.go
|
||||||
|
@@ -187,10 +187,11 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
|
||||||
|
},
|
||||||
|
ShimType: string(sconfig.ShimType),
|
||||||
|
NetworkConfig: persistapi.NetworkConfig{
|
||||||
|
- NetNSPath: sconfig.NetworkConfig.NetNSPath,
|
||||||
|
- NetNsCreated: sconfig.NetworkConfig.NetNsCreated,
|
||||||
|
- DisableNewNetNs: sconfig.NetworkConfig.DisableNewNetNs,
|
||||||
|
- InterworkingModel: int(sconfig.NetworkConfig.InterworkingModel),
|
||||||
|
+ NetNSPath: sconfig.NetworkConfig.NetNSPath,
|
||||||
|
+ NetNsCreated: sconfig.NetworkConfig.NetNsCreated,
|
||||||
|
+ DisableNewNetNs: sconfig.NetworkConfig.DisableNewNetNs,
|
||||||
|
+ EnableCompatOldCNI: sconfig.NetworkConfig.EnableCompatOldCNI,
|
||||||
|
+ InterworkingModel: int(sconfig.NetworkConfig.InterworkingModel),
|
||||||
|
},
|
||||||
|
|
||||||
|
ShmSize: sconfig.ShmSize,
|
||||||
|
@@ -477,10 +478,11 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
|
||||||
|
},
|
||||||
|
ShimType: ShimType(savedConf.ShimType),
|
||||||
|
NetworkConfig: NetworkConfig{
|
||||||
|
- NetNSPath: savedConf.NetworkConfig.NetNSPath,
|
||||||
|
- NetNsCreated: savedConf.NetworkConfig.NetNsCreated,
|
||||||
|
- DisableNewNetNs: savedConf.NetworkConfig.DisableNewNetNs,
|
||||||
|
- InterworkingModel: NetInterworkingModel(savedConf.NetworkConfig.InterworkingModel),
|
||||||
|
+ NetNSPath: savedConf.NetworkConfig.NetNSPath,
|
||||||
|
+ NetNsCreated: savedConf.NetworkConfig.NetNsCreated,
|
||||||
|
+ DisableNewNetNs: savedConf.NetworkConfig.DisableNewNetNs,
|
||||||
|
+ EnableCompatOldCNI: savedConf.NetworkConfig.EnableCompatOldCNI,
|
||||||
|
+ InterworkingModel: NetInterworkingModel(savedConf.NetworkConfig.InterworkingModel),
|
||||||
|
},
|
||||||
|
|
||||||
|
ShmSize: savedConf.ShmSize,
|
||||||
|
diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go
|
||||||
|
index cfbee849..3a2df32b 100644
|
||||||
|
--- a/virtcontainers/persist/api/config.go
|
||||||
|
+++ b/virtcontainers/persist/api/config.go
|
||||||
|
@@ -210,10 +210,11 @@ type ShimConfig struct {
|
||||||
|
|
||||||
|
// NetworkConfig is the network configuration related to a network.
|
||||||
|
type NetworkConfig struct {
|
||||||
|
- NetNSPath string
|
||||||
|
- NetNsCreated bool
|
||||||
|
- DisableNewNetNs bool
|
||||||
|
- InterworkingModel int
|
||||||
|
+ NetNSPath string
|
||||||
|
+ NetNsCreated bool
|
||||||
|
+ DisableNewNetNs bool
|
||||||
|
+ EnableCompatOldCNI bool
|
||||||
|
+ InterworkingModel int
|
||||||
|
}
|
||||||
|
|
||||||
|
type ContainerConfig struct {
|
||||||
|
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
||||||
|
index 36c730b7..948bd3cb 100644
|
||||||
|
--- a/virtcontainers/pkg/oci/utils.go
|
||||||
|
+++ b/virtcontainers/pkg/oci/utils.go
|
||||||
|
@@ -130,6 +130,9 @@ type RuntimeConfig struct {
|
||||||
|
//Determines if create a netns for hypervisor process
|
||||||
|
DisableNewNetNs bool
|
||||||
|
|
||||||
|
+ // Determines if compatible with old CNI plugin
|
||||||
|
+ EnableCompatOldCNI bool
|
||||||
|
+
|
||||||
|
//Determines kata processes are managed only in sandbox cgroup
|
||||||
|
SandboxCgroupOnly bool
|
||||||
|
|
||||||
|
@@ -275,6 +278,7 @@ func networkConfig(ocispec specs.Spec, config RuntimeConfig) (vc.NetworkConfig,
|
||||||
|
}
|
||||||
|
netConf.InterworkingModel = config.InterNetworkModel
|
||||||
|
netConf.DisableNewNetNs = config.DisableNewNetNs
|
||||||
|
+ netConf.EnableCompatOldCNI = config.EnableCompatOldCNI
|
||||||
|
|
||||||
|
netConf.NetmonConfig = vc.NetmonConfig{
|
||||||
|
Path: config.NetmonConfig.Path,
|
||||||
|
diff --git a/virtcontainers/pkg/vcmock/sandbox.go b/virtcontainers/pkg/vcmock/sandbox.go
|
||||||
|
index 677457ef..11b83ccd 100644
|
||||||
|
--- a/virtcontainers/pkg/vcmock/sandbox.go
|
||||||
|
+++ b/virtcontainers/pkg/vcmock/sandbox.go
|
||||||
|
@@ -212,3 +212,8 @@ func (s *Sandbox) UpdateRoutes(routes []*vcTypes.Route) ([]*vcTypes.Route, error
|
||||||
|
func (s *Sandbox) ListRoutes() ([]*vcTypes.Route, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+// IsCompatOldCNI return the whether enable compatible with old CNI
|
||||||
|
+func (s *Sandbox) IsCompatOldCNI() bool {
|
||||||
|
+ return false
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index c8981a41..6a643a12 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -2379,6 +2379,11 @@ func (s *Sandbox) setContainersState(state types.StateString) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+// IsCompatOldCNI return the whether enable compatible with old CNI
|
||||||
|
+func (s *Sandbox) IsCompatOldCNI() bool {
|
||||||
|
+ return s.config.NetworkConfig.EnableCompatOldCNI
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// updateStaticSandboxResources update sandbox's cpu and memory resource passed by
|
||||||
|
// sandbox_cpu and sandbox_mem annotations
|
||||||
|
func updateStaticSandboxResources(sandboxConfig *SandboxConfig) error {
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,182 @@
|
|||||||
|
From ec15337fc816767ca0e8183576405499080b9b1e Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Sun, 16 Aug 2020 16:41:18 +0800
|
||||||
|
Subject: [PATCH 27/50] network: add more strict check for input network
|
||||||
|
interface
|
||||||
|
|
||||||
|
reason: kata-network add-iface command will receive the network
|
||||||
|
interface info from untrust user, so we need to add more strict
|
||||||
|
check for input network interface info.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/network.go | 117 ++++++++++++++++++++++++++++++++++++++++++++++
|
||||||
|
virtcontainers/sandbox.go | 11 +++++
|
||||||
|
2 files changed, 128 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
|
||||||
|
index db235cf6..a1676ccd 100644
|
||||||
|
--- a/virtcontainers/network.go
|
||||||
|
+++ b/virtcontainers/network.go
|
||||||
|
@@ -13,8 +13,10 @@ import (
|
||||||
|
"math/rand"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
+ "regexp"
|
||||||
|
"runtime"
|
||||||
|
"sort"
|
||||||
|
+ "strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/containernetworking/plugins/pkg/ns"
|
||||||
|
@@ -55,6 +57,17 @@ const (
|
||||||
|
NetXConnectInvalidModel
|
||||||
|
)
|
||||||
|
|
||||||
|
+const (
|
||||||
|
+ maxIPAddrLen = 18
|
||||||
|
+ maxInterfaceLen = 15
|
||||||
|
+ minMTUVal = 46
|
||||||
|
+ maxMTUVal = 9600
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+var (
|
||||||
|
+ regInfName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_\-.]*$`)
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
//IsValid checks if a model is valid
|
||||||
|
func (n NetInterworkingModel) IsValid() bool {
|
||||||
|
return 0 <= int(n) && int(n) < int(NetXConnectInvalidModel)
|
||||||
|
@@ -1368,3 +1381,107 @@ func convertToCompatInterfaces(es *[]Endpoint) []*vcTypes.Interface {
|
||||||
|
|
||||||
|
return infs
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+// verifyInterfaceName verifies the interface name valid or not
|
||||||
|
+func verifyInterfaceName(name string) error {
|
||||||
|
+ // verify `Name` before `Tapname` because of the strict rules
|
||||||
|
+ name = strings.TrimSpace(name)
|
||||||
|
+ if len(name) > maxInterfaceLen {
|
||||||
|
+ return fmt.Errorf("interface name can't be longer than 15")
|
||||||
|
+ } else if len(name) == 0 || name == "\n" {
|
||||||
|
+ return fmt.Errorf("interface name can't be empty")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // QMP rules of `Name`
|
||||||
|
+ chk := regInfName.FindAllString(name, -1)
|
||||||
|
+ if chk == nil {
|
||||||
|
+ return fmt.Errorf("invalid input of interface name, please check the rules")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// verifyIPAndMask verifies CIDR notation IP address and mask like "192.0.2.0/24"
|
||||||
|
+func verifyIPAndMask(ip string) (nlIpMask *net.IPNet, err error) {
|
||||||
|
+ ip = strings.TrimSpace(ip)
|
||||||
|
+
|
||||||
|
+ if len(ip) > maxIPAddrLen {
|
||||||
|
+ return nil, fmt.Errorf("the length of IP address is too long, max ip len :%d", maxIPAddrLen)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if nlIpMask, err = netlink.ParseIPNet(ip); err != nil {
|
||||||
|
+ return nil, fmt.Errorf("invalid input of IP : %v", err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nlIpMask, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// verifyMac verifies MAC address
|
||||||
|
+func verifyMac(mac string) error {
|
||||||
|
+ mac = strings.TrimSpace(mac)
|
||||||
|
+ if _, err := net.ParseMAC(mac); err != nil {
|
||||||
|
+ return fmt.Errorf("invalid input of Mac : %v", err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// verifyMtu verifies MTU value of interface
|
||||||
|
+func verifyMtu(mtu uint64) error {
|
||||||
|
+ if mtu < minMTUVal || mtu > maxMTUVal {
|
||||||
|
+ return fmt.Errorf("invalid input of MTU : %v", mtu)
|
||||||
|
+ }
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// verifyIP verifies the IP address
|
||||||
|
+func verifyIP(ip string) (*net.IP, error) {
|
||||||
|
+ ip = strings.TrimSpace(ip)
|
||||||
|
+
|
||||||
|
+ if len(ip) > maxIPAddrLen {
|
||||||
|
+ return nil, fmt.Errorf("the length of IP address is too long, max ip len :%d", maxIPAddrLen)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ var netIP net.IP
|
||||||
|
+ if netIP = net.ParseIP(ip); netIP == nil {
|
||||||
|
+ return nil, fmt.Errorf("invalid IP: %s", ip)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return &netIP, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// validInterface check the input interface valid or not
|
||||||
|
+func validInterface(inf *vcTypes.Interface, enableCompatOldCNI bool) error {
|
||||||
|
+ if enableCompatOldCNI && verifyInterfaceName(inf.Device) != nil {
|
||||||
|
+ return fmt.Errorf("device name should not be empty when enable_compat_old_cni config enabled")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if inf.Name == "" || inf.Mtu == 0 || inf.HwAddr == "" {
|
||||||
|
+ return fmt.Errorf("name/mtu/hwaddr of interface must be specified")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := verifyInterfaceName(inf.Name); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := verifyMac(inf.HwAddr); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := verifyMtu(inf.Mtu); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // Currently, only one IP address can be passed, which reduces the test entry and fault injection.
|
||||||
|
+ if len(inf.IPAddresses) > 0 {
|
||||||
|
+ if len(inf.IPAddresses) != 1 {
|
||||||
|
+ return fmt.Errorf("only one IP address is supported currently")
|
||||||
|
+ }
|
||||||
|
+ _, err := verifyIP(inf.IPAddresses[0].Address)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index 6a643a12..f6826812 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -952,6 +952,17 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) {
|
||||||
|
|
||||||
|
// AddInterface adds new nic to the sandbox.
|
||||||
|
func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (grpcIf *vcTypes.Interface, err error) {
|
||||||
|
+ err = validInterface(inf, s.config.NetworkConfig.EnableCompatOldCNI)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ for _, ep := range s.networkNS.Endpoints {
|
||||||
|
+ if ep.Name() == inf.Name {
|
||||||
|
+ return nil, fmt.Errorf("interface %s is already exist", inf.Name)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
netInfo, err := s.generateNetInfo(inf)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,192 @@
|
|||||||
|
From 51a7270987557ab12ea735fc9781725d1ce1b0a6 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Mon, 17 Aug 2020 15:36:32 +0800
|
||||||
|
Subject: [PATCH 29/50] network: add kata-network del-route subcommand
|
||||||
|
|
||||||
|
reason: add kata-network del-route subcommand to delete the
|
||||||
|
specified route in the Kata VM, del-route is more efficient
|
||||||
|
than the update-routes subcommand.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
cli/network.go | 24 ++++++++++
|
||||||
|
virtcontainers/network.go | 115 ++++++++++++++++++++++++++++++++++++++++++++++
|
||||||
|
2 files changed, 139 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/cli/network.go b/cli/network.go
|
||||||
|
index 2265f54b..046d0ee9 100644
|
||||||
|
--- a/cli/network.go
|
||||||
|
+++ b/cli/network.go
|
||||||
|
@@ -45,6 +45,7 @@ var kataNetworkCLICommand = cli.Command{
|
||||||
|
updateRoutesCommand,
|
||||||
|
listRoutesCommand,
|
||||||
|
addRoutesCommand,
|
||||||
|
+ deleteRoutesCommand,
|
||||||
|
},
|
||||||
|
Action: func(context *cli.Context) error {
|
||||||
|
return cli.ShowSubcommandHelp(context)
|
||||||
|
@@ -155,6 +156,29 @@ var addRoutesCommand = cli.Command{
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
+var deleteRoutesCommand = cli.Command{
|
||||||
|
+ Name: "del-route",
|
||||||
|
+ Usage: "delete one route for a container",
|
||||||
|
+ ArgsUsage: `del-route <container-id> file or - for stdin
|
||||||
|
+ file or stdin for example:
|
||||||
|
+ {
|
||||||
|
+ "dest":"<[<ip>[/mask] | "default" ]>",
|
||||||
|
+ "gateway":"[ip]",
|
||||||
|
+ "device":"[tap-name]",
|
||||||
|
+ }
|
||||||
|
+ Only destination is required and others are optional,
|
||||||
|
+ if device is empty, means search every device`,
|
||||||
|
+ Flags: []cli.Flag{},
|
||||||
|
+ Action: func(context *cli.Context) error {
|
||||||
|
+ ctx, err := cliContextToContext(context)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return networkModifyCommand(ctx, context.Args().First(), context.Args().Get(1), routeType, vcTypes.NetworkOpRemove)
|
||||||
|
+ },
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
var listRoutesCommand = cli.Command{
|
||||||
|
Name: "list-routes",
|
||||||
|
Usage: "list network routes in a container",
|
||||||
|
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
|
||||||
|
index f3757f84..c7066a11 100644
|
||||||
|
--- a/virtcontainers/network.go
|
||||||
|
+++ b/virtcontainers/network.go
|
||||||
|
@@ -1631,6 +1631,110 @@ func addOneRoute(ns *NetworkNamespace, route *vcTypes.Route) (added *vcTypes.Rou
|
||||||
|
return added, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+func generateRmRoute(route *vcTypes.Route) (r *netlink.Route, err error) {
|
||||||
|
+ // destination is required and others are optional
|
||||||
|
+ if route.Dest == "" {
|
||||||
|
+ return nil, fmt.Errorf("destination must be specified when remove route.")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if route.Device != "" {
|
||||||
|
+ err = verifyInterfaceName(route.Device)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ r = &netlink.Route{}
|
||||||
|
+ if route.Dest != "default" {
|
||||||
|
+ nlIpNet, err := verifyRouteDest(&route.Dest)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+ r.Dst = nlIpNet
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if route.Gateway != "" {
|
||||||
|
+ nIP, err := verifyIP(route.Gateway)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+ r.Gw = *nIP
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if route.Source != "" {
|
||||||
|
+ nIP, err := verifyIP(route.Source)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+ r.Src = *nIP
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ r.Scope = netlink.Scope(route.Scope)
|
||||||
|
+
|
||||||
|
+ return r, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// parseToGrpcRoute convert the netlink.Route to vcTypes.Route and deleted route.Dest will
|
||||||
|
+// be added a prefix "-"
|
||||||
|
+func parseToGrpcRoute(device string, route *netlink.Route, add bool) (r *vcTypes.Route) {
|
||||||
|
+ r = &vcTypes.Route{
|
||||||
|
+ Device: device,
|
||||||
|
+ }
|
||||||
|
+ if route.Dst != nil && route.Dst.String() != "<nil>" {
|
||||||
|
+ r.Dest = route.Dst.String()
|
||||||
|
+ if add == false {
|
||||||
|
+ r.Dest = "-" + route.Dst.String()
|
||||||
|
+ }
|
||||||
|
+ } else if route.Dst == nil {
|
||||||
|
+ r.Dest = "default"
|
||||||
|
+ if add == false {
|
||||||
|
+ r.Dest = "-default"
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ if route.Gw != nil && route.Gw.String() != "<nil>" {
|
||||||
|
+ r.Gateway = route.Gw.String()
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return r
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func removeRoutes(ns *NetworkNamespace, route *vcTypes.Route) (removed []*vcTypes.Route, err error) {
|
||||||
|
+ del, err := generateRmRoute(route)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // remove the lo device related routes
|
||||||
|
+ if route.Device == localHostDeviceName {
|
||||||
|
+ removed = append(removed, parseToGrpcRoute(localHostDeviceName, del, false))
|
||||||
|
+
|
||||||
|
+ return removed, nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ for _, ep := range ns.Endpoints {
|
||||||
|
+ // if device is empty, means search every device
|
||||||
|
+ if route.Device != "" && ep.Name() != route.Device {
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ netInfo := ep.Properties()
|
||||||
|
+ for i, exist := range ep.Properties().Routes {
|
||||||
|
+ if isSameRoute(&exist, del, true) {
|
||||||
|
+ // need remove
|
||||||
|
+ netInfo.Routes = append(netInfo.Routes[:i], netInfo.Routes[i+1:]...)
|
||||||
|
+ ep.SetProperties(netInfo)
|
||||||
|
+ dev := route.Device
|
||||||
|
+ if route.Device == "" {
|
||||||
|
+ dev = netInfo.Iface.Name
|
||||||
|
+ }
|
||||||
|
+ removed = append(removed, parseToGrpcRoute(dev, del, false))
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return removed, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func updateRoute(ns *NetworkNamespace, route *vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) {
|
||||||
|
var updRoutes []*vcTypes.Route
|
||||||
|
|
||||||
|
@@ -1645,5 +1749,16 @@ func updateRoute(ns *NetworkNamespace, route *vcTypes.Route, op vcTypes.NetworkO
|
||||||
|
updRoutes = append(updRoutes, added)
|
||||||
|
}
|
||||||
|
|
||||||
|
+ if op == vcTypes.NetworkOpRemove {
|
||||||
|
+ removed, err := removeRoutes(ns, route)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+ if len(removed) <= 0 {
|
||||||
|
+ return nil, fmt.Errorf("route of device %s with destination %s is not found", route.Device, route.Dest)
|
||||||
|
+ }
|
||||||
|
+ updRoutes = removed
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
return updRoutes, nil
|
||||||
|
}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,104 @@
|
|||||||
|
From 9cf769178b8f73c7fd624895589e918d6c7b0645 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Mon, 17 Aug 2020 16:29:17 +0800
|
||||||
|
Subject: [PATCH 30/50] network: kata-network list-routes support display
|
||||||
|
compatible format
|
||||||
|
|
||||||
|
reason: kata-network list-routes subcommand support display compatible
|
||||||
|
format when enable_compat_old_cni config is enabled.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/api.go | 13 ++++++++++++-
|
||||||
|
virtcontainers/network.go | 29 +++++++++++++++++++++++++++++
|
||||||
|
virtcontainers/pkg/types/types.go | 10 +++++-----
|
||||||
|
3 files changed, 46 insertions(+), 6 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
|
||||||
|
index 2331eb94..a4bf41bb 100644
|
||||||
|
--- a/virtcontainers/api.go
|
||||||
|
+++ b/virtcontainers/api.go
|
||||||
|
@@ -1003,7 +1003,18 @@ func ListRoutes(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error)
|
||||||
|
}
|
||||||
|
defer s.releaseStatelessSandbox()
|
||||||
|
|
||||||
|
- return s.ListRoutes()
|
||||||
|
+ routes, err := s.ListRoutes()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // If enable_compat_old_cni is enabled, convert routes info to
|
||||||
|
+ // compatible display format
|
||||||
|
+ if s.config.NetworkConfig.EnableCompatOldCNI {
|
||||||
|
+ return convertToDisplayRoutes(&routes), nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return routes, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// CleanupContaienr is used by shimv2 to stop and delete a container exclusively, once there is no container
|
||||||
|
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
|
||||||
|
index c7066a11..488bd00c 100644
|
||||||
|
--- a/virtcontainers/network.go
|
||||||
|
+++ b/virtcontainers/network.go
|
||||||
|
@@ -1384,6 +1384,35 @@ func convertToCompatInterfaces(es *[]Endpoint) []*vcTypes.Interface {
|
||||||
|
return infs
|
||||||
|
}
|
||||||
|
|
||||||
|
+// convertToDisplayRoutes convert the default route format to more simple
|
||||||
|
+// route display format, it is called when enable_compat_old_cni config
|
||||||
|
+// is enabled.
|
||||||
|
+func convertToDisplayRoutes(routes *[]*vcTypes.Route) []*vcTypes.Route {
|
||||||
|
+ var displayRoutes []*vcTypes.Route
|
||||||
|
+ if routes == nil {
|
||||||
|
+ return displayRoutes
|
||||||
|
+ }
|
||||||
|
+ for _, r := range *routes {
|
||||||
|
+ // we don't support IPV6 temporarily, so we need to filter ":" which
|
||||||
|
+ // is the characteristics of IPV6 for those default routes
|
||||||
|
+ if strings.Contains(r.Dest, ":") {
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ defaultDest := r.Dest
|
||||||
|
+ if r.Dest == "" {
|
||||||
|
+ defaultDest = "default"
|
||||||
|
+ }
|
||||||
|
+ displayRoutes = append(displayRoutes, &vcTypes.Route{
|
||||||
|
+ Dest: defaultDest,
|
||||||
|
+ Gateway: r.Gateway,
|
||||||
|
+ Device: r.Device,
|
||||||
|
+ })
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return displayRoutes
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// verifyInterfaceName verifies the interface name valid or not
|
||||||
|
func verifyInterfaceName(name string) error {
|
||||||
|
// verify `Name` before `Tapname` because of the strict rules
|
||||||
|
diff --git a/virtcontainers/pkg/types/types.go b/virtcontainers/pkg/types/types.go
|
||||||
|
index 71fe7fbb..b41b0c75 100644
|
||||||
|
--- a/virtcontainers/pkg/types/types.go
|
||||||
|
+++ b/virtcontainers/pkg/types/types.go
|
||||||
|
@@ -33,11 +33,11 @@ type Interface struct {
|
||||||
|
|
||||||
|
// Route describes a network route.
|
||||||
|
type Route struct {
|
||||||
|
- Dest string
|
||||||
|
- Gateway string
|
||||||
|
- Device string
|
||||||
|
- Source string
|
||||||
|
- Scope uint32
|
||||||
|
+ Dest string `json:"dest,omitempty"`
|
||||||
|
+ Gateway string `json:"gateway,omitempty"`
|
||||||
|
+ Device string `json:"device,omitempty"`
|
||||||
|
+ Source string `json:"source,omitempty"`
|
||||||
|
+ Scope uint32 `json:"scope,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
//NetworkOp describes network operation
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,41 @@
|
|||||||
|
From 326e90f97cf5ace73dff95257f7b4faa43c56f99 Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Mon, 17 Aug 2020 16:11:35 +0800
|
||||||
|
Subject: [PATCH 31/50] device_mangaer: check VFIO when create device
|
||||||
|
|
||||||
|
reason: check VFIO when create device, block device can be
|
||||||
|
reused and VFIO device can not
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/device/manager/manager.go | 7 ++++++-
|
||||||
|
1 file changed, 6 insertions(+), 1 deletion(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/device/manager/manager.go b/virtcontainers/device/manager/manager.go
|
||||||
|
index 6a2b665a..d24a29ab 100644
|
||||||
|
--- a/virtcontainers/device/manager/manager.go
|
||||||
|
+++ b/virtcontainers/device/manager/manager.go
|
||||||
|
@@ -9,6 +9,7 @@ package manager
|
||||||
|
import (
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
|
+ "fmt"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
@@ -115,7 +116,11 @@ func (dm *deviceManager) createDevice(devInfo config.DeviceInfo) (dev api.Device
|
||||||
|
}()
|
||||||
|
|
||||||
|
if existingDev := dm.findDeviceByMajorMinor(devInfo.Major, devInfo.Minor); existingDev != nil {
|
||||||
|
- return existingDev, nil
|
||||||
|
+ if isVFIO(devInfo.HostPath) {
|
||||||
|
+ return nil, fmt.Errorf("device %s is replicated in the same Pod!", devInfo.ContainerPath)
|
||||||
|
+ } else {
|
||||||
|
+ return existingDev, nil
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
|
||||||
|
// device ID must be generated by manager instead of device itself
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,46 @@
|
|||||||
|
From 77711b531867be899df5d2c59a525ea1b7f0e6ac Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Mon, 17 Aug 2020 17:14:35 +0800
|
||||||
|
Subject: [PATCH 32/50] network: add more detail usage for update-routes
|
||||||
|
subcommand
|
||||||
|
|
||||||
|
reason: add more detail usage for update-routes subcommand
|
||||||
|
to explain how to use update-routes subcommand
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
cli/network.go | 18 ++++++++++++++----
|
||||||
|
1 file changed, 14 insertions(+), 4 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/cli/network.go b/cli/network.go
|
||||||
|
index 046d0ee9..3dd0971e 100644
|
||||||
|
--- a/cli/network.go
|
||||||
|
+++ b/cli/network.go
|
||||||
|
@@ -120,10 +120,20 @@ var listIfacesCommand = cli.Command{
|
||||||
|
}
|
||||||
|
|
||||||
|
var updateRoutesCommand = cli.Command{
|
||||||
|
- Name: "update-routes",
|
||||||
|
- Usage: "update routes of a container",
|
||||||
|
- ArgsUsage: `update-routes <container-id> file or - for stdin`,
|
||||||
|
- Flags: []cli.Flag{},
|
||||||
|
+ Name: "update-routes",
|
||||||
|
+ Usage: "update routes of a container",
|
||||||
|
+ ArgsUsage: `update-routes <container-id> file or - for stdin
|
||||||
|
+ file or stdin for example:
|
||||||
|
+ [
|
||||||
|
+ {
|
||||||
|
+ "dest":"<[<ip>[/mask] | "default" ]>",
|
||||||
|
+ "gateway":"[ip]",
|
||||||
|
+ "device":"[tap-name]",
|
||||||
|
+ "source": "[source]",
|
||||||
|
+ "scope":[scope]
|
||||||
|
+ }
|
||||||
|
+ ]`,
|
||||||
|
+ Flags: []cli.Flag{},
|
||||||
|
Action: func(context *cli.Context) error {
|
||||||
|
ctx, err := cliContextToContext(context)
|
||||||
|
if err != nil {
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,80 @@
|
|||||||
|
From 7ab7ff54efa3925a8d372f7830d31b87f8d01ea8 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Mon, 17 Aug 2020 17:39:18 +0800
|
||||||
|
Subject: [PATCH 33/50] network: do not delete the exist tap device in the host
|
||||||
|
|
||||||
|
reason: If hotplug a exist tap device into Kata VM, kata-runtime
|
||||||
|
should not delete this exist tap device, because this tap device
|
||||||
|
is controlled by other network components.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/tap_endpoint.go | 16 +++++++++++++---
|
||||||
|
virtcontainers/veth_endpoint.go | 3 +++
|
||||||
|
2 files changed, 16 insertions(+), 3 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go
|
||||||
|
index c897670e..2cf70dce 100644
|
||||||
|
--- a/virtcontainers/tap_endpoint.go
|
||||||
|
+++ b/virtcontainers/tap_endpoint.go
|
||||||
|
@@ -77,7 +77,7 @@ func (endpoint *TapEndpoint) Detach(netNsCreated bool, netNsPath string) error {
|
||||||
|
|
||||||
|
networkLogger().WithField("endpoint-type", TapEndpointType).Info("Detaching endpoint")
|
||||||
|
return doNetNS(netNsPath, func(_ ns.NetNS) error {
|
||||||
|
- return unTapNetwork(endpoint.TapInterface.TAPIface.Name)
|
||||||
|
+ return unTapNetwork(endpoint)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -91,6 +91,9 @@ func (endpoint *TapEndpoint) HotAttach(h hypervisor) error {
|
||||||
|
|
||||||
|
if _, err := h.hotplugAddDevice(endpoint, netDev); err != nil {
|
||||||
|
networkLogger().WithError(err).Error("Error attach tap ep")
|
||||||
|
+ if errUnTap := unTapNetwork(endpoint); errUnTap != nil {
|
||||||
|
+ networkLogger().WithError(errUnTap).Errorf("Error rollback tap %s", endpoint.TapInterface.TAPIface.Name)
|
||||||
|
+ }
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
@@ -100,7 +103,7 @@ func (endpoint *TapEndpoint) HotAttach(h hypervisor) error {
|
||||||
|
func (endpoint *TapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error {
|
||||||
|
networkLogger().Info("Hot detaching tap endpoint")
|
||||||
|
if err := doNetNS(netNsPath, func(_ ns.NetNS) error {
|
||||||
|
- return unTapNetwork(endpoint.TapInterface.TAPIface.Name)
|
||||||
|
+ return unTapNetwork(endpoint)
|
||||||
|
}); err != nil {
|
||||||
|
networkLogger().WithError(err).Warn("Error un-bridging tap ep")
|
||||||
|
}
|
||||||
|
@@ -185,7 +188,14 @@ func tapNetwork(endpoint *TapEndpoint, numCPUs uint32, disableVhostNet bool) err
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
-func unTapNetwork(name string) error {
|
||||||
|
+func unTapNetwork(endpoint *TapEndpoint) error {
|
||||||
|
+ // length of VMFDs == 0 means that the endpoint is already exist in the host,
|
||||||
|
+ // no created by kata, so we don't need to remove it when detach
|
||||||
|
+ if len(endpoint.TapInterface.VMFds) == 0 {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ name := endpoint.TapInterface.TAPIface.Name
|
||||||
|
netHandle, err := netlink.NewHandle()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
diff --git a/virtcontainers/veth_endpoint.go b/virtcontainers/veth_endpoint.go
|
||||||
|
index 9ece6a74..0f2ec9ba 100644
|
||||||
|
--- a/virtcontainers/veth_endpoint.go
|
||||||
|
+++ b/virtcontainers/veth_endpoint.go
|
||||||
|
@@ -119,6 +119,9 @@ func (endpoint *VethEndpoint) HotAttach(h hypervisor) error {
|
||||||
|
|
||||||
|
if _, err := h.hotplugAddDevice(endpoint, netDev); err != nil {
|
||||||
|
networkLogger().WithError(err).Error("Error attach virtual ep")
|
||||||
|
+ if errDisconn := xDisconnectVMNetwork(endpoint); errDisconn != nil {
|
||||||
|
+ networkLogger().WithError(errDisconn).Error("Error rollback virtual ep")
|
||||||
|
+ }
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
1273
runtime/patches/0034-kata-runtime-add-kata-ipvs-command.patch
Normal file
1273
runtime/patches/0034-kata-runtime-add-kata-ipvs-command.patch
Normal file
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,234 @@
|
|||||||
|
From cf2b34f477cba88641de3719bf5c8f933b919bcc Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Mon, 17 Aug 2020 21:44:57 +0800
|
||||||
|
Subject: [PATCH 35/50] device: mount blockdevices in the guest VM
|
||||||
|
|
||||||
|
reason: support mount blockdevices in the guest VM like
|
||||||
|
"-v /dev/blockdevice:/home/test"
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
cli/config/configuration-qemu.toml.in | 5 ++++
|
||||||
|
pkg/katautils/config.go | 6 +++++
|
||||||
|
virtcontainers/kata_agent.go | 27 ++++++++++++++++---
|
||||||
|
virtcontainers/kata_agent_test.go | 2 +-
|
||||||
|
virtcontainers/utils/utils.go | 50 +++++++++++++++++++++++++++++++++++
|
||||||
|
virtcontainers/vm_test.go | 2 +-
|
||||||
|
6 files changed, 86 insertions(+), 6 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in
|
||||||
|
index 46f8b632..aa11b38f 100644
|
||||||
|
--- a/cli/config/configuration-qemu.toml.in
|
||||||
|
+++ b/cli/config/configuration-qemu.toml.in
|
||||||
|
@@ -391,6 +391,11 @@ path = "@SHIMPATH@"
|
||||||
|
#
|
||||||
|
kernel_modules=[]
|
||||||
|
|
||||||
|
+# If enabled, when we pass a block device to the guest VM
|
||||||
|
+# through -v, such as `docker run -v /dev/loop100:/foo/bar`,
|
||||||
|
+# agent will create a directory in guest VM and mount the
|
||||||
|
+# file system on `/dev/loop100` inside the container
|
||||||
|
+enable_blk_mount = true
|
||||||
|
|
||||||
|
[netmon]
|
||||||
|
# If enabled, the network monitoring process gets started when the
|
||||||
|
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
||||||
|
index 94c916a0..51120311 100644
|
||||||
|
--- a/pkg/katautils/config.go
|
||||||
|
+++ b/pkg/katautils/config.go
|
||||||
|
@@ -160,6 +160,7 @@ type agent struct {
|
||||||
|
TraceMode string `toml:"trace_mode"`
|
||||||
|
TraceType string `toml:"trace_type"`
|
||||||
|
KernelModules []string `toml:"kernel_modules"`
|
||||||
|
+ MountBlkInVM bool `toml:"enable_blk_mount"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type netmon struct {
|
||||||
|
@@ -463,6 +464,10 @@ func (h hypervisor) getInitrdAndImage() (initrd string, image string, err error)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (a agent) mountBlkDevInVM() bool {
|
||||||
|
+ return a.MountBlkInVM
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func (p proxy) path() (string, error) {
|
||||||
|
path := p.Path
|
||||||
|
if path == "" {
|
||||||
|
@@ -978,6 +983,7 @@ func updateRuntimeConfigAgent(configPath string, tomlConf tomlConfig, config *oc
|
||||||
|
TraceMode: agent.traceMode(),
|
||||||
|
TraceType: agent.traceType(),
|
||||||
|
KernelModules: agent.kernelModules(),
|
||||||
|
+ MountBlkInVM: agent.mountBlkDevInVM(),
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("%s agent type is not supported", k)
|
||||||
|
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
|
||||||
|
index 16662949..b0f88c15 100644
|
||||||
|
--- a/virtcontainers/kata_agent.go
|
||||||
|
+++ b/virtcontainers/kata_agent.go
|
||||||
|
@@ -30,6 +30,7 @@ import (
|
||||||
|
ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/pkg/rootless"
|
||||||
|
vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/store"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/types"
|
||||||
|
@@ -64,6 +65,7 @@ var (
|
||||||
|
errorMissingOCISpec = errors.New("Missing OCI specification")
|
||||||
|
defaultKataHostSharedDir = "/run/kata-containers/shared/sandboxes/"
|
||||||
|
defaultKataGuestSharedDir = "/run/kata-containers/shared/containers/"
|
||||||
|
+ kataGuestStorageDir = "/run/kata-containers/storage/containers/"
|
||||||
|
mountGuestTag = "kataShared"
|
||||||
|
defaultKataGuestSandboxDir = "/run/kata-containers/sandbox/"
|
||||||
|
type9pFs = "9p"
|
||||||
|
@@ -199,6 +201,7 @@ type KataAgentConfig struct {
|
||||||
|
TraceMode string
|
||||||
|
TraceType string
|
||||||
|
KernelModules []string
|
||||||
|
+ MountBlkInVM bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// KataAgentState is the structure describing the data stored from this
|
||||||
|
@@ -1061,7 +1064,12 @@ func (k *kataAgent) replaceOCIMountsForStorages(spec *specs.Spec, volumeStorages
|
||||||
|
// Create a temporary location to mount the Storage. Mounting to the correct location
|
||||||
|
// will be handled by the OCI mount structure.
|
||||||
|
filename := fmt.Sprintf("%s-%s", uuid.Generate().String(), filepath.Base(m.Destination))
|
||||||
|
- path := filepath.Join(kataGuestSandboxStorageDir(), filename)
|
||||||
|
+ var path string
|
||||||
|
+ if v.Driver == kataBlkDevType || v.Driver == kataSCSIDevType {
|
||||||
|
+ path = filepath.Join(kataGuestStorageDir, filename)
|
||||||
|
+ } else {
|
||||||
|
+ path = filepath.Join(kataGuestSandboxStorageDir(), filename)
|
||||||
|
+ }
|
||||||
|
|
||||||
|
k.Logger().Debugf("Replacing OCI mount source (%s) with %s", m.Source, path)
|
||||||
|
ociMounts[index].Source = path
|
||||||
|
@@ -1436,7 +1444,7 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process,
|
||||||
|
// Note this call modifies the list of container devices to make sure
|
||||||
|
// all hotplugged devices are unplugged, so this needs be done
|
||||||
|
// after devices passed with --device are handled.
|
||||||
|
- volumeStorages, err := k.handleBlockVolumes(c)
|
||||||
|
+ volumeStorages, err := k.handleBlockVolumes(sandbox, c)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
@@ -1609,7 +1617,7 @@ func (k *kataAgent) handleVhostUserBlkVolume(c *Container, device api.Device) (*
|
||||||
|
|
||||||
|
// handleBlockVolumes handles volumes that are block devices files
|
||||||
|
// by passing the block devices as Storage to the agent.
|
||||||
|
-func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) {
|
||||||
|
+func (k *kataAgent) handleBlockVolumes(sandbox *Sandbox, c *Container) ([]*grpc.Storage, error) {
|
||||||
|
|
||||||
|
var volumeStorages []*grpc.Storage
|
||||||
|
|
||||||
|
@@ -1648,9 +1656,20 @@ func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) {
|
||||||
|
}
|
||||||
|
|
||||||
|
vol.MountPoint = m.Destination
|
||||||
|
- if vol.Fstype == "" {
|
||||||
|
+
|
||||||
|
+ ac, _ := sandbox.config.AgentConfig.(KataAgentConfig)
|
||||||
|
+ if ac.MountBlkInVM {
|
||||||
|
+ // Ensure the block device is formatted, for the devices here are specified as volumes
|
||||||
|
+ fsType, gerr := utils.GetDevFormat(m.Source)
|
||||||
|
+ if gerr != nil || fsType == "" {
|
||||||
|
+ k.Logger().WithField("device", id).WithError(gerr).Error("get device format failed")
|
||||||
|
+ return nil, gerr
|
||||||
|
+ }
|
||||||
|
+ vol.Fstype = fmt.Sprintf("bind-%s", fsType)
|
||||||
|
+ } else {
|
||||||
|
vol.Fstype = "bind"
|
||||||
|
}
|
||||||
|
+
|
||||||
|
if len(vol.Options) == 0 {
|
||||||
|
vol.Options = []string{"bind"}
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go
|
||||||
|
index 18a5a0a6..68caaab2 100644
|
||||||
|
--- a/virtcontainers/kata_agent_test.go
|
||||||
|
+++ b/virtcontainers/kata_agent_test.go
|
||||||
|
@@ -446,7 +446,7 @@ func TestHandleBlockVolume(t *testing.T) {
|
||||||
|
containers[c.id].sandbox = &sandbox
|
||||||
|
containers[c.id].mounts = mounts
|
||||||
|
|
||||||
|
- volumeStorages, err := k.handleBlockVolumes(c)
|
||||||
|
+ volumeStorages, err := k.handleBlockVolumes(&sandbox, c)
|
||||||
|
assert.Nil(t, err, "Error while handling block volumes")
|
||||||
|
|
||||||
|
vStorage := &pb.Storage{
|
||||||
|
diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go
|
||||||
|
index 9490faa1..36ac67a7 100644
|
||||||
|
--- a/virtcontainers/utils/utils.go
|
||||||
|
+++ b/virtcontainers/utils/utils.go
|
||||||
|
@@ -379,3 +379,53 @@ func RoundVCPUNumber(value string) (int, error) {
|
||||||
|
cpus := int(math.Ceil(cpuNum))
|
||||||
|
return cpus, nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+// GetDevFormat get the formated filesystem of input disk use `blkid`, Such as `ext4`,`xfs`,etc.
|
||||||
|
+func GetDevFormat(disk string) (string, error) {
|
||||||
|
+ // refer to https://github.com/kubernetes/kubernetes/blob/v1.12.2/pkg/util/mount/mount_linux.go#L512
|
||||||
|
+ args := []string{"-p", "-s", "TYPE", "-s", "PTTYPE", "-o", "export", disk}
|
||||||
|
+ dataOut, err := exec.Command("blkid", args...).Output()
|
||||||
|
+ output := string(dataOut)
|
||||||
|
+
|
||||||
|
+ if err != nil {
|
||||||
|
+ if strings.Contains(err.Error(), "exit status 2") {
|
||||||
|
+ // Disk device is unformatted.
|
||||||
|
+ // For `blkid`, if the specified token (TYPE/PTTYPE, etc) was
|
||||||
|
+ // not found, or no (specified) devices could be identified, an
|
||||||
|
+ // exit code of 2 is returned.
|
||||||
|
+ return "", nil
|
||||||
|
+ }
|
||||||
|
+ return "", err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ var fstype string
|
||||||
|
+
|
||||||
|
+ lines := strings.Split(output, "\n")
|
||||||
|
+ for _, l := range lines {
|
||||||
|
+ if len(l) <= 0 {
|
||||||
|
+ // Ignore empty line.
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+ // if we use busybox as rootfs,the output of command
|
||||||
|
+ // `busybox blkid` is different with original`blkid`,
|
||||||
|
+ // so we should make a compatible parse
|
||||||
|
+ subLine := strings.Split(l, " ")
|
||||||
|
+ for _, sl := range subLine {
|
||||||
|
+ if len(subLine) <= 0 {
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+ cs := strings.Split(sl, "=")
|
||||||
|
+ if len(cs) != 2 {
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+ if cs[0] == "TYPE" {
|
||||||
|
+ fstype = cs[1]
|
||||||
|
+ if strings.Contains(fstype, `"`) {
|
||||||
|
+ fstype = strings.Replace(fstype, `"`, "", -1)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return fstype, nil
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/vm_test.go b/virtcontainers/vm_test.go
|
||||||
|
index 36fd5c2f..d2414232 100644
|
||||||
|
--- a/virtcontainers/vm_test.go
|
||||||
|
+++ b/virtcontainers/vm_test.go
|
||||||
|
@@ -125,7 +125,7 @@ func TestVMConfigGrpc(t *testing.T) {
|
||||||
|
HypervisorType: QemuHypervisor,
|
||||||
|
HypervisorConfig: newQemuConfig(),
|
||||||
|
AgentType: KataContainersAgent,
|
||||||
|
- AgentConfig: KataAgentConfig{false, true, false, false, 0, "", "", []string{}},
|
||||||
|
+ AgentConfig: KataAgentConfig{false, true, false, false, 0, "", "", []string{}, false },
|
||||||
|
ProxyType: NoopProxyType,
|
||||||
|
}
|
||||||
|
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,131 @@
|
|||||||
|
From 448bb661d6759c1e20c18084604b150bff08ada4 Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Tue, 18 Aug 2020 11:49:26 +0800
|
||||||
|
Subject: [PATCH 36/50] mount: limit the maximum number of virtio-scsi bus
|
||||||
|
slots
|
||||||
|
|
||||||
|
reason:
|
||||||
|
1. add SCSIBus functions for add or remove device about ScsiBus
|
||||||
|
2. limit the maximum number of virtio-scsi bus slots to 25
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/qemu.go | 20 +++++++++++++++++
|
||||||
|
virtcontainers/types/scsi.go | 53 ++++++++++++++++++++++++++++++++++++++++++++
|
||||||
|
2 files changed, 73 insertions(+)
|
||||||
|
create mode 100644 virtcontainers/types/scsi.go
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
|
||||||
|
index bb83b1bb..c2b65376 100644
|
||||||
|
--- a/virtcontainers/qemu.go
|
||||||
|
+++ b/virtcontainers/qemu.go
|
||||||
|
@@ -65,6 +65,7 @@ type CPUDevice struct {
|
||||||
|
// QemuState keeps Qemu's state
|
||||||
|
type QemuState struct {
|
||||||
|
Bridges []types.Bridge
|
||||||
|
+ ScsiBus *types.SCSIBus
|
||||||
|
// HotpluggedCPUs is the list of CPUs that were hot-added
|
||||||
|
HotpluggedVCPUs []CPUDevice
|
||||||
|
HotpluggedMemory int
|
||||||
|
@@ -411,6 +412,13 @@ func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.I
|
||||||
|
|
||||||
|
var ioThread *govmmQemu.IOThread
|
||||||
|
if q.config.BlockDeviceDriver == config.VirtioSCSI {
|
||||||
|
+ if q.state.ScsiBus == nil {
|
||||||
|
+ // only use `scsi0.0`
|
||||||
|
+ q.state.ScsiBus = &types.SCSIBus{
|
||||||
|
+ Address: make(map[uint32]int),
|
||||||
|
+ ID: fmt.Sprintf("%s.0", scsiControllerID),
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
return q.arch.appendSCSIController(devices, q.config.EnableIOThreads)
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -1168,6 +1176,16 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev
|
||||||
|
|
||||||
|
// Bus exposed by the SCSI Controller
|
||||||
|
bus := scsiControllerID + ".0"
|
||||||
|
+ var err error
|
||||||
|
+ if err = q.state.ScsiBus.AddDevToScsiBus(drive.Index); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ defer func() {
|
||||||
|
+ if err != nil {
|
||||||
|
+ q.state.ScsiBus.RemoveDevFromScsiBus(drive.Index)
|
||||||
|
+ }
|
||||||
|
+ }()
|
||||||
|
|
||||||
|
// Get SCSI-id and LUN based on the order of attaching drives.
|
||||||
|
scsiID, lun, err := utils.GetSCSIIdLun(drive.Index)
|
||||||
|
@@ -1234,6 +1252,8 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error
|
||||||
|
if err := q.arch.removeDeviceFromBridge(drive.ID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
+ } else if q.config.BlockDeviceDriver == config.VirtioSCSI {
|
||||||
|
+ q.state.ScsiBus.RemoveDevFromScsiBus(drive.Index)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := q.qmpMonitorCh.qmp.ExecuteDeviceDel(q.qmpMonitorCh.ctx, devID); err != nil {
|
||||||
|
diff --git a/virtcontainers/types/scsi.go b/virtcontainers/types/scsi.go
|
||||||
|
new file mode 100644
|
||||||
|
index 00000000..f4d489f0
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/virtcontainers/types/scsi.go
|
||||||
|
@@ -0,0 +1,53 @@
|
||||||
|
+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
||||||
|
+// SPDX-License-Identifier: Apache-2.0
|
||||||
|
+// Description: common functions
|
||||||
|
+// Author: leizhongkai
|
||||||
|
+// Create: 2019-06-30
|
||||||
|
+
|
||||||
|
+package types
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "fmt"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+const (
|
||||||
|
+ scsiMaxCapacity = 25
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+type SCSIBus struct {
|
||||||
|
+ // Address contains information about devices plugged and number limit
|
||||||
|
+ Address map[uint32]int
|
||||||
|
+
|
||||||
|
+ // ID is used to identify the bus exposed by the SCSI Controller
|
||||||
|
+ ID string
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (sb *SCSIBus) AddDevToScsiBus(index int) error {
|
||||||
|
+ var addr uint32
|
||||||
|
+
|
||||||
|
+ // looking for the first available address
|
||||||
|
+ for i := uint32(1); i <= scsiMaxCapacity; i++ {
|
||||||
|
+ if _, ok := sb.Address[i]; !ok {
|
||||||
|
+ addr = i
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if addr == 0 {
|
||||||
|
+ return fmt.Errorf("Scsi bus capacity limited.")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // save address and device
|
||||||
|
+ sb.Address[addr] = index
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (sb *SCSIBus) RemoveDevFromScsiBus(index int) {
|
||||||
|
+ for addr, i := range sb.Address {
|
||||||
|
+ if i == index {
|
||||||
|
+ delete(sb.Address, addr)
|
||||||
|
+ return
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
875
runtime/patches/0037-runtime-add-IPVS-test.patch
Normal file
875
runtime/patches/0037-runtime-add-IPVS-test.patch
Normal file
@ -0,0 +1,875 @@
|
|||||||
|
From d083f0e0247fbded92a0ae2a0e71da4176baed95 Mon Sep 17 00:00:00 2001
|
||||||
|
From: xiadanni <xiadanni1@huawei.com>
|
||||||
|
Date: Tue, 18 Aug 2020 17:08:23 +0800
|
||||||
|
Subject: [PATCH 37/50] runtime: add IPVS test
|
||||||
|
|
||||||
|
Signed-off-by: xiadanni <xiadanni1@huawei.com>
|
||||||
|
---
|
||||||
|
cli/ipvsadm_test.go | 775 +++++++++++++++++++++++++++++++++++++
|
||||||
|
virtcontainers/api_test.go | 36 ++
|
||||||
|
virtcontainers/kata_agent_test.go | 4 +
|
||||||
|
virtcontainers/pkg/vcmock/types.go | 2 +-
|
||||||
|
4 files changed, 816 insertions(+), 1 deletion(-)
|
||||||
|
create mode 100644 cli/ipvsadm_test.go
|
||||||
|
|
||||||
|
diff --git a/cli/ipvsadm_test.go b/cli/ipvsadm_test.go
|
||||||
|
new file mode 100644
|
||||||
|
index 00000000..92958aee
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/cli/ipvsadm_test.go
|
||||||
|
@@ -0,0 +1,775 @@
|
||||||
|
+// Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved.
|
||||||
|
+// SPDX-License-Identifier: Apache-2.0
|
||||||
|
+// Description: IPVS common functions test
|
||||||
|
+// Author: xiadanni
|
||||||
|
+// Create: 2020-08-01
|
||||||
|
+
|
||||||
|
+package main
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "context"
|
||||||
|
+ "flag"
|
||||||
|
+ "fmt"
|
||||||
|
+ "os"
|
||||||
|
+ "strings"
|
||||||
|
+ "testing"
|
||||||
|
+
|
||||||
|
+ "github.com/kata-containers/agent/protocols/grpc"
|
||||||
|
+ vc "github.com/kata-containers/runtime/virtcontainers"
|
||||||
|
+ vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/types"
|
||||||
|
+ "github.com/stretchr/testify/assert"
|
||||||
|
+ "github.com/urfave/cli"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+func TestKataIPVSCliAction(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+
|
||||||
|
+ actionFunc, ok := kataIPVSCLICommand.Action.(func(ctx *cli.Context) error)
|
||||||
|
+ assert.True(ok)
|
||||||
|
+
|
||||||
|
+ flagSet := flag.NewFlagSet("kata-IPVS", flag.ContinueOnError)
|
||||||
|
+ ctx := createCLIContext(flagSet)
|
||||||
|
+
|
||||||
|
+ err := actionFunc(ctx)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestIPVSadmCliAction(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+
|
||||||
|
+ actionFunc, ok := IPVSadmCommand.Action.(func(ctx *cli.Context) error)
|
||||||
|
+ assert.True(ok)
|
||||||
|
+
|
||||||
|
+ flagSet := flag.NewFlagSet("IPVSadm", flag.ContinueOnError)
|
||||||
|
+ ctx := createCLIContext(flagSet)
|
||||||
|
+
|
||||||
|
+ err := actionFunc(ctx)
|
||||||
|
+ assert.Error(err, "Missing container ID")
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestIPVSadmCLISuccessful(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+
|
||||||
|
+ sandbox := &vcmock.Sandbox{
|
||||||
|
+ MockID: testContainerID,
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ sandbox.MockContainers = []*vcmock.Container{
|
||||||
|
+ {
|
||||||
|
+ MockID: sandbox.ID(),
|
||||||
|
+ MockSandbox: sandbox,
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
|
||||||
|
+ return vc.ContainerStatus{
|
||||||
|
+ ID: sandbox.ID(),
|
||||||
|
+ Annotations: map[string]string{
|
||||||
|
+ vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
|
||||||
|
+ },
|
||||||
|
+ State: types.ContainerState{
|
||||||
|
+ State: types.StateRunning,
|
||||||
|
+ },
|
||||||
|
+ }, nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
|
||||||
|
+ return &grpc.IPVSResponse{}, nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ defer func() {
|
||||||
|
+ testingImpl.StatusContainerFunc = nil
|
||||||
|
+ testingImpl.UpdateIPVSRuleFunc = nil
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
+ path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ defer os.RemoveAll(path)
|
||||||
|
+
|
||||||
|
+ actionFunc, ok := IPVSadmCommand.Action.(func(ctx *cli.Context) error)
|
||||||
|
+ assert.True(ok)
|
||||||
|
+
|
||||||
|
+ flagSet := flag.NewFlagSet("IPVSadm", flag.ContinueOnError)
|
||||||
|
+ flagSet.Parse([]string{testContainerID})
|
||||||
|
+ flagSet.String("parameters", "-a -t 192.168.0.7:80 -r 192.168.0.4:80", "")
|
||||||
|
+ ctx := createCLIContext(flagSet)
|
||||||
|
+ err = actionFunc(ctx)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ // result not nil
|
||||||
|
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
|
||||||
|
+ return &grpc.IPVSResponse{IPVSRes: "IPVS rule updating success"}, nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ err = actionFunc(ctx)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestIPVSadmCLIError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+
|
||||||
|
+ sandbox := &vcmock.Sandbox{
|
||||||
|
+ MockID: testContainerID,
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ sandbox.MockContainers = []*vcmock.Container{
|
||||||
|
+ {
|
||||||
|
+ MockID: sandbox.ID(),
|
||||||
|
+ MockSandbox: sandbox,
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
|
||||||
|
+ return vc.ContainerStatus{
|
||||||
|
+ ID: sandbox.ID(),
|
||||||
|
+ Annotations: map[string]string{
|
||||||
|
+ vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
|
||||||
|
+ },
|
||||||
|
+ State: types.ContainerState{
|
||||||
|
+ State: types.StateRunning,
|
||||||
|
+ },
|
||||||
|
+ }, nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
|
||||||
|
+ return &grpc.IPVSResponse{}, nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ defer func() {
|
||||||
|
+ testingImpl.StatusContainerFunc = nil
|
||||||
|
+ testingImpl.UpdateIPVSRuleFunc = nil
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
+ path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ defer os.RemoveAll(path)
|
||||||
|
+
|
||||||
|
+ actionFunc, ok := IPVSadmCommand.Action.(func(ctx *cli.Context) error)
|
||||||
|
+ assert.True(ok)
|
||||||
|
+
|
||||||
|
+ // no stdin rule file
|
||||||
|
+ flagSet := flag.NewFlagSet("IPVSadm", flag.ContinueOnError)
|
||||||
|
+ flagSet.Parse([]string{testContainerID})
|
||||||
|
+ flagSet.String("restore", "-", "")
|
||||||
|
+ ctx := createCLIContext(flagSet)
|
||||||
|
+ err = actionFunc(ctx)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+
|
||||||
|
+ // restore parameter error
|
||||||
|
+ flagSet = flag.NewFlagSet("IPVSadm", flag.ContinueOnError)
|
||||||
|
+ flagSet.Parse([]string{testContainerID})
|
||||||
|
+ flagSet.String("restore", "abc", "")
|
||||||
|
+ ctx = createCLIContext(flagSet)
|
||||||
|
+ err = actionFunc(ctx)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+
|
||||||
|
+ // checkrule returns error
|
||||||
|
+ flagSet = flag.NewFlagSet("IPVSadm", flag.ContinueOnError)
|
||||||
|
+ flagSet.Parse([]string{testContainerID})
|
||||||
|
+ flagSet.String("parameters", "-A -t 192.168.0.7:80 -s rr -p -3000", "")
|
||||||
|
+ ctx = createCLIContext(flagSet)
|
||||||
|
+ err = actionFunc(ctx)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+
|
||||||
|
+ // updatefunction returns error
|
||||||
|
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
|
||||||
|
+ return &grpc.IPVSResponse{}, fmt.Errorf("IPVSadm test error")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ flagSet = flag.NewFlagSet("IPVSadm", flag.ContinueOnError)
|
||||||
|
+ flagSet.Parse([]string{testContainerID})
|
||||||
|
+ flagSet.String("parameters", "-A -t 192.168.0.7:80 -s rr -p 3000", "")
|
||||||
|
+ ctx = createCLIContext(flagSet)
|
||||||
|
+ err = actionFunc(ctx)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestCleanupCliAction(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+
|
||||||
|
+ actionFunc, ok := CleanupCommand.Action.(func(ctx *cli.Context) error)
|
||||||
|
+ assert.True(ok)
|
||||||
|
+
|
||||||
|
+ flagSet := flag.NewFlagSet("cleanup", flag.ContinueOnError)
|
||||||
|
+ ctx := createCLIContext(flagSet)
|
||||||
|
+
|
||||||
|
+ err := actionFunc(ctx)
|
||||||
|
+ assert.Error(err, "Missing container ID")
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestCleanupCLISuccessful(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+
|
||||||
|
+ sandbox := &vcmock.Sandbox{
|
||||||
|
+ MockID: testContainerID,
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ sandbox.MockContainers = []*vcmock.Container{
|
||||||
|
+ {
|
||||||
|
+ MockID: sandbox.ID(),
|
||||||
|
+ MockSandbox: sandbox,
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
|
||||||
|
+ return vc.ContainerStatus{
|
||||||
|
+ ID: sandbox.ID(),
|
||||||
|
+ Annotations: map[string]string{
|
||||||
|
+ vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
|
||||||
|
+ },
|
||||||
|
+ State: types.ContainerState{
|
||||||
|
+ State: types.StateRunning,
|
||||||
|
+ },
|
||||||
|
+ }, nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
|
||||||
|
+ return &grpc.IPVSResponse{}, nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ defer func() {
|
||||||
|
+ testingImpl.StatusContainerFunc = nil
|
||||||
|
+ testingImpl.UpdateIPVSRuleFunc = nil
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
+ path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ defer os.RemoveAll(path)
|
||||||
|
+
|
||||||
|
+ actionFunc, ok := CleanupCommand.Action.(func(ctx *cli.Context) error)
|
||||||
|
+ assert.True(ok)
|
||||||
|
+
|
||||||
|
+ flagSet := flag.NewFlagSet("cleanup", flag.ContinueOnError)
|
||||||
|
+ flagSet.Parse([]string{testContainerID})
|
||||||
|
+ flagSet.String("parameters", "-d 192.168.0.4 -p tcp", "")
|
||||||
|
+ ctx := createCLIContext(flagSet)
|
||||||
|
+ err = actionFunc(ctx)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestCleanupCLIError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+
|
||||||
|
+ sandbox := &vcmock.Sandbox{
|
||||||
|
+ MockID: testContainerID,
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ sandbox.MockContainers = []*vcmock.Container{
|
||||||
|
+ {
|
||||||
|
+ MockID: sandbox.ID(),
|
||||||
|
+ MockSandbox: sandbox,
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
|
||||||
|
+ return vc.ContainerStatus{
|
||||||
|
+ ID: sandbox.ID(),
|
||||||
|
+ Annotations: map[string]string{
|
||||||
|
+ vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
|
||||||
|
+ },
|
||||||
|
+ State: types.ContainerState{
|
||||||
|
+ State: types.StateRunning,
|
||||||
|
+ },
|
||||||
|
+ }, nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
|
||||||
|
+ return &grpc.IPVSResponse{}, nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ defer func() {
|
||||||
|
+ testingImpl.StatusContainerFunc = nil
|
||||||
|
+ testingImpl.UpdateIPVSRuleFunc = nil
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
+ path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ defer os.RemoveAll(path)
|
||||||
|
+
|
||||||
|
+ actionFunc, ok := CleanupCommand.Action.(func(ctx *cli.Context) error)
|
||||||
|
+ assert.True(ok)
|
||||||
|
+
|
||||||
|
+ // checkrule returns error
|
||||||
|
+ flagSet := flag.NewFlagSet("cleanup", flag.ContinueOnError)
|
||||||
|
+ flagSet.Parse([]string{testContainerID})
|
||||||
|
+ flagSet.String("parameters", "-d 192.168.0.4", "")
|
||||||
|
+ ctx := createCLIContext(flagSet)
|
||||||
|
+ err = actionFunc(ctx)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+
|
||||||
|
+ // updatefunction returns error
|
||||||
|
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
|
||||||
|
+ return &grpc.IPVSResponse{}, fmt.Errorf("IPVSadm cleanup test error")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ flagSet = flag.NewFlagSet("cleanup", flag.ContinueOnError)
|
||||||
|
+ flagSet.Parse([]string{testContainerID})
|
||||||
|
+ flagSet.String("parameters", "-d 192.168.0.4 -p tcp", "")
|
||||||
|
+ ctx = createCLIContext(flagSet)
|
||||||
|
+ err = actionFunc(ctx)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestAddServiceSuccessfully(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ addServiceCommandTCP := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --scheduler rr --persistent 3000"
|
||||||
|
+ _, err := checkIPVSRule(addServiceCommandTCP)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ addServiceCommandUDP := "IPVSadm --add-service --udp-service 192.168.0.7:80 --scheduler rr --persistent 3000"
|
||||||
|
+ _, err = checkIPVSRule(addServiceCommandUDP)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rr -p 3000"
|
||||||
|
+ _, err = checkIPVSRule(addServiceCmd)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestAddServiceNoService(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ addServiceCommand := "IPVSadm --add-service --scheduler rr --persistent 3000"
|
||||||
|
+ _, err := checkIPVSRule(addServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ addServiceCmd := "IPVSadm -A -s rr -p 3000"
|
||||||
|
+ _, err = checkIPVSRule(addServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestAddServiceNoScheduler(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --persistent 3000"
|
||||||
|
+ _, err := checkIPVSRule(addServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -p 3000"
|
||||||
|
+ _, err = checkIPVSRule(addServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestAddServiceNoPersistent(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --scheduler rr"
|
||||||
|
+ _, err := checkIPVSRule(addServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rr"
|
||||||
|
+ _, err = checkIPVSRule(addServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestAddServiceIpError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.2.2.7:80 --scheduler rr --persistent 3000"
|
||||||
|
+ _, err := checkIPVSRule(addServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ addServiceCmd := "IPVSadm -A -t 192.168.2.2.7:80 -s rr -p 3000"
|
||||||
|
+ _, err = checkIPVSRule(addServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestAddServicePortError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:9999999 --scheduler rr --persistent 3000"
|
||||||
|
+ _, err := checkIPVSRule(addServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:9999999 -s rr -p 3000"
|
||||||
|
+ _, err = checkIPVSRule(addServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestAddServiceSchedulerError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --scheduler rrr --persistent 3000"
|
||||||
|
+ _, err := checkIPVSRule(addServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rrr -p 3000"
|
||||||
|
+ _, err = checkIPVSRule(addServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestAddServicePersistentError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --scheduler rr --persistent 99999999999"
|
||||||
|
+ _, err := checkIPVSRule(addServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rr -p 99999999999"
|
||||||
|
+ _, err = checkIPVSRule(addServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestEditServiceSuccessfully(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ editServiceCommandTCP := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --scheduler rr --persistent 3000"
|
||||||
|
+ _, err := checkIPVSRule(editServiceCommandTCP)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ editServiceCommandUcp := "IPVSadm --edit-service --udp-service 192.168.0.7:80 --scheduler rr --persistent 3000"
|
||||||
|
+ _, err = checkIPVSRule(editServiceCommandUcp)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ editServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rr -p 3000"
|
||||||
|
+ _, err = checkIPVSRule(editServiceCmd)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestEditServiceNoService(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ editServiceCommand := "IPVSadm --edit-service --scheduler rr --persistent 3000"
|
||||||
|
+ _, err := checkIPVSRule(editServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ editServiceCmd := "IPVSadm -E -s rr -p 3000"
|
||||||
|
+ _, err = checkIPVSRule(editServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestEditServiceNoScheduler(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --persistent 3000"
|
||||||
|
+ _, err := checkIPVSRule(editServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ editServiceCmd := "IPVSadm -E -t 192.168.0.7:80 -p 3000"
|
||||||
|
+ _, err = checkIPVSRule(editServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestEditServiceNoPersistent(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --scheduler rr"
|
||||||
|
+ _, err := checkIPVSRule(editServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ editServiceCmd := "IPVSadm -E -t 192.168.0.7:80 -s rr"
|
||||||
|
+ _, err = checkIPVSRule(editServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestEditServiceIpError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.2.2.7:80 --scheduler rr --persistent 3000"
|
||||||
|
+ _, err := checkIPVSRule(editServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ editServiceCmd := "IPVSadm -E -t 192.168.2.2.7:80 -s rr -p 3000"
|
||||||
|
+ _, err = checkIPVSRule(editServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestEditServicePortError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:9999999 --scheduler rr --persistent 3000"
|
||||||
|
+ _, err := checkIPVSRule(editServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ editServiceCmd := "IPVSadm -E -t 192.168.0.7:9999999 -s rr -p 3000"
|
||||||
|
+ _, err = checkIPVSRule(editServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestEditServiceSchedulerError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --scheduler rrr --persistent 3000"
|
||||||
|
+ _, err := checkIPVSRule(editServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ editServiceCmd := "IPVSadm -E -t 192.168.0.7:80 -s rrr -p 3000"
|
||||||
|
+ _, err = checkIPVSRule(editServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestEditServicePersistentError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --scheduler rr --persistent 99999999999"
|
||||||
|
+ _, err := checkIPVSRule(editServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ editServiceCmd := "IPVSadm -E -t 192.168.0.7:80 -s rr -p 99999999999"
|
||||||
|
+ _, err = checkIPVSRule(editServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestDeleteServiceSuccessfully(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ deleteServiceCommandTCP := "IPVSadm --delete-service --tcp-service 192.168.0.7:80"
|
||||||
|
+ _, err := checkIPVSRule(deleteServiceCommandTCP)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ deleteServiceCommandUDP := "IPVSadm --delete-service --udp-service 192.168.0.7:80"
|
||||||
|
+ _, err = checkIPVSRule(deleteServiceCommandUDP)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ deleteServiceCmd := "IPVSadm -D -t 192.168.0.7:80"
|
||||||
|
+ _, err = checkIPVSRule(deleteServiceCmd)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestDeleteServiceNoService(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ deleteServiceCommand := "IPVSadm --delete-service"
|
||||||
|
+ _, err := checkIPVSRule(deleteServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ deleteServiceCmd := "IPVSadm -D"
|
||||||
|
+ _, err = checkIPVSRule(deleteServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestDeleteServiceIpError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ deleteServiceCommand := "IPVSadm --delete-service --tcp-service 192.168.2.2.7:80"
|
||||||
|
+ _, err := checkIPVSRule(deleteServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ deleteServiceCmd := "IPVSadm -D -t 192.168.2.2.7:80"
|
||||||
|
+ _, err = checkIPVSRule(deleteServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestDeleteServicePortError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ deleteServiceCommand := "IPVSadm --delete-service --tcp-service 192.168.0.7:9999999"
|
||||||
|
+ _, err := checkIPVSRule(deleteServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ deleteServiceCmd := "IPVSadm -D -t 192.168.0.7:9999999"
|
||||||
|
+ _, err = checkIPVSRule(deleteServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestAddServerSuccessfully(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ addServerCommandTCP := "IPVSadm --add-server --tcp-service 192.168.0.7:80 --real-server 192.168.0.4:80 --weight 100"
|
||||||
|
+ _, err := checkIPVSRule(addServerCommandTCP)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ addServerCommandUDP := "IPVSadm --add-server --udp-service 192.168.0.7:80 --real-server 192.168.0.4:80 --weight 100"
|
||||||
|
+ _, err = checkIPVSRule(addServerCommandUDP)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ addServerCmd := "IPVSadm -a -t 192.168.0.7:80 -r 192.168.0.4:80 -w 100"
|
||||||
|
+ _, err = checkIPVSRule(addServerCmd)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestAddServerNoService(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ addServerCommand := "IPVSadm --add-server --real-server 192.168.0.4:80"
|
||||||
|
+ _, err := checkIPVSRule(addServerCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ addServerCmd := "IPVSadm -a -r 192.168.0.4:80"
|
||||||
|
+ _, err = checkIPVSRule(addServerCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestAddServerNoServer(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ addServerCommand := "IPVSadm --add-server --tcp-service 192.168.0.7:80"
|
||||||
|
+ _, err := checkIPVSRule(addServerCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ addServerCmd := "IPVSadm -a -t 192.168.0.7:80"
|
||||||
|
+ _, err = checkIPVSRule(addServerCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestAddServerIpError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ addServerCommand := "IPVSadm --add-server --tcp-service 192.168.2.0.7:80 --real-server 192.168.0.4:80"
|
||||||
|
+ _, err := checkIPVSRule(addServerCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ addServerCmd := "IPVSadm -a -t 192.168.2.0.7:80 -r 192.168.0.4:80"
|
||||||
|
+ _, err = checkIPVSRule(addServerCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestAddServerPortError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ addServerCommand := "IPVSadm --add-server --tcp-service 192.168.0.7:99999 --real-server 192.168.0.4:80"
|
||||||
|
+ _, err := checkIPVSRule(addServerCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ addServerCmd := "IPVSadm -a -t 192.168.0.7:99999 -r 192.168.0.4:80"
|
||||||
|
+ _, err = checkIPVSRule(addServerCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestEditServerSuccessfully(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ editServerCommandTCP := "IPVSadm --edit-server --tcp-service 192.168.0.7:80 --real-server 192.168.0.4:80 --weight 100"
|
||||||
|
+ _, err := checkIPVSRule(editServerCommandTCP)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ editServerCommandUDP := "IPVSadm --edit-server --udp-service 192.168.0.7:80 --real-server 192.168.0.4:80 --weight 100"
|
||||||
|
+ _, err = checkIPVSRule(editServerCommandUDP)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ editServerCmd := "IPVSadm -e -t 192.168.0.7:80 -r 192.168.0.4:80 -w 100"
|
||||||
|
+ _, err = checkIPVSRule(editServerCmd)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestEditServerNoService(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ editServerCommand := "IPVSadm --edit-server --real-server 192.168.0.4:80"
|
||||||
|
+ _, err := checkIPVSRule(editServerCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ editServerCmd := "IPVSadm -e -r 192.168.0.4:80"
|
||||||
|
+ _, err = checkIPVSRule(editServerCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestEditServerNoServer(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ editServerCommand := "IPVSadm --edit-server --tcp-service 192.168.0.7:80"
|
||||||
|
+ _, err := checkIPVSRule(editServerCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ editServerCmd := "IPVSadm -e -t 192.168.0.7:80"
|
||||||
|
+ _, err = checkIPVSRule(editServerCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestEditServerIpError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ editServerCommand := "IPVSadm --edit-server --tcp-service 192.168.2.0.7:80 --real-server 192.168.0.4:80"
|
||||||
|
+ _, err := checkIPVSRule(editServerCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ editServerCmd := "IPVSadm -e -t 192.168.2.0.7:80 -r 192.168.0.4:80"
|
||||||
|
+ _, err = checkIPVSRule(editServerCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestEditServerPortError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ editServerCommand := "IPVSadm --edit-server --tcp-service 192.168.0.7:99999 --real-server 192.168.0.4:80"
|
||||||
|
+ _, err := checkIPVSRule(editServerCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ editServerCmd := "IPVSadm -e -t 192.168.0.7:99999 -r 192.168.0.4:80"
|
||||||
|
+ _, err = checkIPVSRule(editServerCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestDeleteServerSuccessfully(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ deleteServerCommandTCP := "IPVSadm --delete-server --tcp-service 192.168.0.7:80 --real-server 192.168.0.4:80"
|
||||||
|
+ _, err := checkIPVSRule(deleteServerCommandTCP)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ deleteServerCommandUDP := "IPVSadm --delete-server --udp-service 192.168.0.7:80 --real-server 192.168.0.4:80"
|
||||||
|
+ _, err = checkIPVSRule(deleteServerCommandUDP)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ deleteServerCmd := "IPVSadm -d -t 192.168.0.7:80 -r 192.168.0.4:80"
|
||||||
|
+ _, err = checkIPVSRule(deleteServerCmd)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestDeleteServerNoService(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ deleteServerCommand := "IPVSadm --delete-server --real-server 192.168.0.4:80"
|
||||||
|
+ _, err := checkIPVSRule(deleteServerCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ deleteServerCmd := "IPVSadm -d -r 192.168.0.4:80"
|
||||||
|
+ _, err = checkIPVSRule(deleteServerCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestDeleteServerNoServer(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ deleteServerCommand := "IPVSadm --delete-server --tcp-service 192.168.0.7:80"
|
||||||
|
+ _, err := checkIPVSRule(deleteServerCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ deleteServerCmd := "IPVSadm -d -t 192.168.0.7:80"
|
||||||
|
+ _, err = checkIPVSRule(deleteServerCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestDeleteServerIpError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ deleteServerCommand := "IPVSadm --delete-server --tcp-service 192.168.2.0.7:80 --real-server 192.168.0.4:80"
|
||||||
|
+ _, err := checkIPVSRule(deleteServerCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ deleteServerCmd := "IPVSadm -d -t 192.168.2.0.7:80 -r 192.168.0.4:80"
|
||||||
|
+ _, err = checkIPVSRule(deleteServerCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestDeleteServerPortError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ deleteServerCommand := "IPVSadm --delete-server --tcp-service 192.168.0.7:99999 --real-server 192.168.0.4:80"
|
||||||
|
+ _, err := checkIPVSRule(deleteServerCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ deleteServerCmd := "IPVSadm -d -t 192.168.0.7:99999 -r 192.168.0.4:80"
|
||||||
|
+ _, err = checkIPVSRule(deleteServerCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestListSuccessfully(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ listCommand := "IPVSadm --list"
|
||||||
|
+ _, err := checkIPVSRule(listCommand)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ listCmd := "IPVSadm -L"
|
||||||
|
+ _, err = checkIPVSRule(listCmd)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestSetSuccessfully(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ setCommand := "IPVSadm --set"
|
||||||
|
+ _, err := checkIPVSRule(setCommand)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestCleanupSuccessfully(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ cleanupCommand := "conntrack -D --orig-dst 192.168.0.4 --protonum tcp"
|
||||||
|
+ _, err := checkIPVSRule(cleanupCommand)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ cleanupCmd := "conntrack -D -d 192.168.0.4 -p tcp"
|
||||||
|
+ _, err = checkIPVSRule(cleanupCmd)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestCleanupNoDestination(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ cleanupCommand := "conntrack -D --protonum tcp"
|
||||||
|
+ _, err := checkIPVSRule(cleanupCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ cleanupCmd := "conntrack -D -p tcp"
|
||||||
|
+ _, err = checkIPVSRule(cleanupCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestCleanupNoProtocol(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ cleanupCommand := "conntrack -D --orig-dst 192.168.0.4"
|
||||||
|
+ _, err := checkIPVSRule(cleanupCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ cleanupCmd := "conntrack -D -d 192.168.0.4"
|
||||||
|
+ _, err = checkIPVSRule(cleanupCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestCleanupIpError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ cleanupCommand := "conntrack -D --orig-dst 192.168.0.2.4 --protonum tcp"
|
||||||
|
+ _, err := checkIPVSRule(cleanupCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ cleanupCmd := "conntrack -D -d 192.168.0.2.4 -p tcp"
|
||||||
|
+ _, err = checkIPVSRule(cleanupCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestCleanupProtocolError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ cleanupCommand := "conntrack -D --orig-dst 192.168.0.4 --protonum ttcp"
|
||||||
|
+ _, err := checkIPVSRule(cleanupCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ cleanupCmd := "conntrack -D -d 192.168.0.4 -p ttcp"
|
||||||
|
+ _, err = checkIPVSRule(cleanupCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestRestoreFileNil(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ stdin := strings.NewReader("")
|
||||||
|
+ fileContext, cnt, err := getRestoreFile(stdin)
|
||||||
|
+ assert.Equal(cnt, 0)
|
||||||
|
+ assert.Equal(fileContext, "")
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestRestoreFileError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ stdin := strings.NewReader("aaa")
|
||||||
|
+ fileContext, cnt, err := getRestoreFile(stdin)
|
||||||
|
+ assert.Equal(cnt, 0)
|
||||||
|
+ assert.Equal(fileContext, "")
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestRestoreFile(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ stdin := strings.NewReader("-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m")
|
||||||
|
+ fileContext, cnt, err := getRestoreFile(stdin)
|
||||||
|
+ assert.Equal(cnt, 2)
|
||||||
|
+ assert.Equal(fileContext, "-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m")
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/api_test.go b/virtcontainers/api_test.go
|
||||||
|
index c01d47b8..2a5488f9 100644
|
||||||
|
--- a/virtcontainers/api_test.go
|
||||||
|
+++ b/virtcontainers/api_test.go
|
||||||
|
@@ -15,6 +15,7 @@ import (
|
||||||
|
"syscall"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
+ "github.com/kata-containers/agent/protocols/grpc"
|
||||||
|
ktu "github.com/kata-containers/runtime/pkg/katatestutils"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/persist"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
|
||||||
|
@@ -1745,3 +1746,38 @@ func TestCleanupContainer(t *testing.T) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func TestUpdateIPVSRule(t *testing.T) {
|
||||||
|
+ defer cleanUp()
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+
|
||||||
|
+ contID := "abc"
|
||||||
|
+ ctx := context.Background()
|
||||||
|
+ var ipvs = &grpc.UpdateIPVSRequest{
|
||||||
|
+ IPVSReq: "ipvsadm -A -t 192.168.0.7:80 -s rr -p -3000",
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ config := newTestSandboxConfigNoop()
|
||||||
|
+ p, _, err := createAndStartSandbox(ctx, config)
|
||||||
|
+ if p == nil || err != nil {
|
||||||
|
+ t.Fatal(err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ s, ok := p.(*Sandbox)
|
||||||
|
+ assert.True(ok)
|
||||||
|
+
|
||||||
|
+ contConfig := newTestContainerConfigNoop(contID)
|
||||||
|
+ c, err := p.CreateContainer(contConfig)
|
||||||
|
+ if c == nil || err != nil {
|
||||||
|
+ t.Fatal(err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ _, err = UpdateIPVSRule(ctx, s.id, ipvs)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ _, err = UpdateIPVSRule(ctx, "aaa", ipvs)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+
|
||||||
|
+ _, err = UpdateIPVSRule(ctx, "", ipvs)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go
|
||||||
|
index 18a5a0a6..4f9409a0 100644
|
||||||
|
--- a/virtcontainers/kata_agent_test.go
|
||||||
|
+++ b/virtcontainers/kata_agent_test.go
|
||||||
|
@@ -246,6 +246,10 @@ func (p *gRPCProxy) MemHotplugByProbe(ctx context.Context, req *pb.MemHotplugByP
|
||||||
|
return &gpb.Empty{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (p *gRPCProxy) UpdateIPVSRule(ctx context.Context, req *pb.UpdateIPVSRequest) (*pb.IPVSResponse, error) {
|
||||||
|
+ return &pb.IPVSResponse{}, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func gRPCRegister(s *grpc.Server, srv interface{}) {
|
||||||
|
switch g := srv.(type) {
|
||||||
|
case *gRPCProxy:
|
||||||
|
diff --git a/virtcontainers/pkg/vcmock/types.go b/virtcontainers/pkg/vcmock/types.go
|
||||||
|
index 43247ef9..610b4602 100644
|
||||||
|
--- a/virtcontainers/pkg/vcmock/types.go
|
||||||
|
+++ b/virtcontainers/pkg/vcmock/types.go
|
||||||
|
@@ -76,5 +76,5 @@ type VCMock struct {
|
||||||
|
UpdateRoutesFunc func(ctx context.Context, sandboxID string, routes []*vcTypes.Route) ([]*vcTypes.Route, error)
|
||||||
|
ListRoutesFunc func(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error)
|
||||||
|
CleanupContainerFunc func(ctx context.Context, sandboxID, containerID string, force bool) error
|
||||||
|
- UpdateIPVSRuleFunc func(ctx context.Context, sandboxID string, ipvs *grpc.UpdateIpvsRequest) (*grpc.IpvsResponse, error)
|
||||||
|
+ UpdateIPVSRuleFunc func(ctx context.Context, sandboxID string, ipvs *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error)
|
||||||
|
}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,501 @@
|
|||||||
|
From 5a220e9be1cfb03316a62aa00d2040638ba1a855 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 18 Aug 2020 15:15:52 +0800
|
||||||
|
Subject: [PATCH 38/50] pcie: using pcie-root-port driver to hotplug device in
|
||||||
|
aarch64
|
||||||
|
|
||||||
|
reason: Since qemu with "virt" machine type doesn't support hotplug
|
||||||
|
device in the pcie.0 root bus, so need to use add root port devices
|
||||||
|
to support hotplug pci device in aarch64.
|
||||||
|
|
||||||
|
we reuse the pcie_root_port config in the configuration.toml file to
|
||||||
|
set pcie_root_port device number when qemu process start.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
vendor/github.com/intel/govmm/qemu/qemu.go | 22 ++---
|
||||||
|
virtcontainers/persist/api/hypervisor.go | 5 +-
|
||||||
|
virtcontainers/qemu.go | 130 +++++++++++++++++++++++------
|
||||||
|
virtcontainers/qemu_arch_base.go | 9 ++
|
||||||
|
virtcontainers/types/pcie.go | 103 +++++++++++++++++++++++
|
||||||
|
5 files changed, 230 insertions(+), 39 deletions(-)
|
||||||
|
create mode 100644 virtcontainers/types/pcie.go
|
||||||
|
|
||||||
|
diff --git a/vendor/github.com/intel/govmm/qemu/qemu.go b/vendor/github.com/intel/govmm/qemu/qemu.go
|
||||||
|
index a5e5dfaf..3e7720b4 100644
|
||||||
|
--- a/vendor/github.com/intel/govmm/qemu/qemu.go
|
||||||
|
+++ b/vendor/github.com/intel/govmm/qemu/qemu.go
|
||||||
|
@@ -1252,6 +1252,8 @@ type PCIeRootPortDevice struct {
|
||||||
|
Chassis string // (slot, chassis) pair is mandatory and must be unique for each pcie-root-port, >=0, default is 0x00
|
||||||
|
Slot string // >=0, default is 0x00
|
||||||
|
|
||||||
|
+ Port string // Port number is the device index
|
||||||
|
+
|
||||||
|
Multifunction bool // true => "on", false => "off", default is off
|
||||||
|
Addr string // >=0, default is 0x00
|
||||||
|
|
||||||
|
@@ -1277,6 +1279,10 @@ func (b PCIeRootPortDevice) QemuParams(config *Config) []string {
|
||||||
|
|
||||||
|
deviceParams = append(deviceParams, fmt.Sprintf("%s,id=%s", driver, b.ID))
|
||||||
|
|
||||||
|
+ if b.Port != "" {
|
||||||
|
+ deviceParams = append(deviceParams, fmt.Sprintf("port=%s", b.Port))
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
if b.Bus == "" {
|
||||||
|
b.Bus = "pcie.0"
|
||||||
|
}
|
||||||
|
@@ -1287,20 +1293,10 @@ func (b PCIeRootPortDevice) QemuParams(config *Config) []string {
|
||||||
|
}
|
||||||
|
deviceParams = append(deviceParams, fmt.Sprintf("chassis=%s", b.Chassis))
|
||||||
|
|
||||||
|
- if b.Slot == "" {
|
||||||
|
- b.Slot = "0x00"
|
||||||
|
- }
|
||||||
|
- deviceParams = append(deviceParams, fmt.Sprintf("slot=%s", b.Slot))
|
||||||
|
-
|
||||||
|
- multifunction := "off"
|
||||||
|
if b.Multifunction {
|
||||||
|
- multifunction = "on"
|
||||||
|
- if b.Addr == "" {
|
||||||
|
- b.Addr = "0x00"
|
||||||
|
- }
|
||||||
|
- deviceParams = append(deviceParams, fmt.Sprintf("addr=%s", b.Addr))
|
||||||
|
+ deviceParams = append(deviceParams, "multifunction=on")
|
||||||
|
}
|
||||||
|
- deviceParams = append(deviceParams, fmt.Sprintf("multifunction=%v", multifunction))
|
||||||
|
+ deviceParams = append(deviceParams, fmt.Sprintf("addr=%s", b.Addr))
|
||||||
|
|
||||||
|
if b.BusReserve != "" {
|
||||||
|
deviceParams = append(deviceParams, fmt.Sprintf("bus-reserve=%s", b.BusReserve))
|
||||||
|
@@ -1337,7 +1333,7 @@ func (b PCIeRootPortDevice) Valid() bool {
|
||||||
|
if b.Pref64Reserve != "" && b.Pref32Reserve != "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
- if b.ID == "" {
|
||||||
|
+ if b.ID == "" || b.Port == "" || b.Bus == "" || b.Addr == ""{
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
diff --git a/virtcontainers/persist/api/hypervisor.go b/virtcontainers/persist/api/hypervisor.go
|
||||||
|
index 375fd56b..fd61b3c2 100644
|
||||||
|
--- a/virtcontainers/persist/api/hypervisor.go
|
||||||
|
+++ b/virtcontainers/persist/api/hypervisor.go
|
||||||
|
@@ -5,6 +5,8 @@
|
||||||
|
|
||||||
|
package persistapi
|
||||||
|
|
||||||
|
+import "github.com/kata-containers/runtime/virtcontainers/types"
|
||||||
|
+
|
||||||
|
// Bridge is a bridge where devices can be hot plugged
|
||||||
|
type Bridge struct {
|
||||||
|
// DeviceAddr contains information about devices plugged and its address in the bridge
|
||||||
|
@@ -35,7 +37,8 @@ type HypervisorState struct {
|
||||||
|
|
||||||
|
// Belows are qemu specific
|
||||||
|
// Refs: virtcontainers/qemu.go:QemuState
|
||||||
|
- Bridges []Bridge
|
||||||
|
+ Bridges []Bridge
|
||||||
|
+ PCIeRootPortsPool *types.PCIeRootPortPool
|
||||||
|
// HotpluggedCPUs is the list of CPUs that were hot-added
|
||||||
|
HotpluggedVCPUs []CPUDevice
|
||||||
|
HotpluggedMemory int
|
||||||
|
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
|
||||||
|
index c2b65376..a10c66fb 100644
|
||||||
|
--- a/virtcontainers/qemu.go
|
||||||
|
+++ b/virtcontainers/qemu.go
|
||||||
|
@@ -64,8 +64,9 @@ type CPUDevice struct {
|
||||||
|
|
||||||
|
// QemuState keeps Qemu's state
|
||||||
|
type QemuState struct {
|
||||||
|
- Bridges []types.Bridge
|
||||||
|
- ScsiBus *types.SCSIBus
|
||||||
|
+ Bridges []types.Bridge
|
||||||
|
+ ScsiBus *types.SCSIBus
|
||||||
|
+ PCIeRootPortsPool *types.PCIeRootPortPool
|
||||||
|
// HotpluggedCPUs is the list of CPUs that were hot-added
|
||||||
|
HotpluggedVCPUs []CPUDevice
|
||||||
|
HotpluggedMemory int
|
||||||
|
@@ -271,6 +272,9 @@ func (q *qemu) setup(id string, hypervisorConfig *HypervisorConfig) error {
|
||||||
|
|
||||||
|
q.state.HotplugVFIOOnRootBus = q.config.HotplugVFIOOnRootBus
|
||||||
|
q.state.PCIeRootPort = int(q.config.PCIeRootPort)
|
||||||
|
+ // init the PCIeRootPortsPool with pcie_root_port config value
|
||||||
|
+ q.state.PCIeRootPortsPool = &types.PCIeRootPortPool{}
|
||||||
|
+ q.state.PCIeRootPortsPool.Init(q.state.PCIeRootPort)
|
||||||
|
|
||||||
|
// The path might already exist, but in case of VM templating,
|
||||||
|
// we have to create it since the sandbox has not created it yet.
|
||||||
|
@@ -394,9 +398,18 @@ func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.I
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
- // Add bridges before any other devices. This way we make sure that
|
||||||
|
- // bridge gets the first available PCI address i.e bridgePCIStartAddr
|
||||||
|
- devices = q.arch.appendBridges(devices)
|
||||||
|
+ machine, err := q.getQemuMachine()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, nil, err
|
||||||
|
+ }
|
||||||
|
+ switch machine.Type {
|
||||||
|
+ case QemuVirt:
|
||||||
|
+ devices = q.arch.appendRootPorts(devices, q.state.PCIeRootPortsPool)
|
||||||
|
+ default:
|
||||||
|
+ // Add bridges before any other devices. This way we make sure that
|
||||||
|
+ // bridge gets the first available PCI address i.e bridgePCIStartAddr
|
||||||
|
+ devices = q.arch.appendBridges(devices)
|
||||||
|
+ }
|
||||||
|
|
||||||
|
devices, err = q.arch.appendConsole(devices, console)
|
||||||
|
if err != nil {
|
||||||
|
@@ -608,7 +621,7 @@ func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNa
|
||||||
|
// Add PCIe Root Port devices to hypervisor
|
||||||
|
// The pcie.0 bus do not support hot-plug, but PCIe device can be hot-plugged into PCIe Root Port.
|
||||||
|
// For more details, please see https://github.com/qemu/qemu/blob/master/docs/pcie.txt
|
||||||
|
- if hypervisorConfig.PCIeRootPort > 0 {
|
||||||
|
+ if hypervisorConfig.PCIeRootPort > 0 && hypervisorConfig.HypervisorMachineType == QemuQ35 {
|
||||||
|
qemuConfig.Devices = q.arch.appendPCIeRootPortDevice(qemuConfig.Devices, hypervisorConfig.PCIeRootPort)
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -1154,21 +1167,19 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev
|
||||||
|
}
|
||||||
|
case q.config.BlockDeviceDriver == config.VirtioBlock:
|
||||||
|
driver := "virtio-blk-pci"
|
||||||
|
- addr, bridge, err := q.arch.addDeviceToBridge(drive.ID, types.PCI)
|
||||||
|
+
|
||||||
|
+ addr, bus, pciAddr, err := q.getPciAddress(drive.ID, types.PCI)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
-
|
||||||
|
defer func() {
|
||||||
|
if err != nil {
|
||||||
|
- q.arch.removeDeviceFromBridge(drive.ID)
|
||||||
|
+ q.putPciAddress(drive.ID)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
+ drive.PCIAddr = pciAddr
|
||||||
|
|
||||||
|
- // PCI address is in the format bridge-addr/device-addr eg. "03/02"
|
||||||
|
- drive.PCIAddr = fmt.Sprintf("%02x", bridge.Addr) + "/" + addr
|
||||||
|
-
|
||||||
|
- if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID, romFile, 0, true, defaultDisableModern); err != nil {
|
||||||
|
+ if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bus, romFile, 0, true, defaultDisableModern); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
case q.config.BlockDeviceDriver == config.VirtioSCSI:
|
||||||
|
@@ -1249,7 +1260,7 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error
|
||||||
|
err = q.hotplugAddBlockDevice(drive, op, devID)
|
||||||
|
} else {
|
||||||
|
if q.config.BlockDeviceDriver == config.VirtioBlock {
|
||||||
|
- if err := q.arch.removeDeviceFromBridge(drive.ID); err != nil {
|
||||||
|
+ if err := q.putPciAddress(drive.ID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else if q.config.BlockDeviceDriver == config.VirtioSCSI {
|
||||||
|
@@ -1345,22 +1356,22 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
- addr, bridge, err := q.arch.addDeviceToBridge(devID, types.PCI)
|
||||||
|
+ addr, bus, _, err := q.getPciAddress(devID, types.PCI)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
if err != nil {
|
||||||
|
- q.arch.removeDeviceFromBridge(devID)
|
||||||
|
+ q.putPciAddress(devID)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
switch device.Type {
|
||||||
|
case config.VFIODeviceNormalType:
|
||||||
|
- return q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bridge.ID, romFile)
|
||||||
|
+ return q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bus, romFile)
|
||||||
|
case config.VFIODeviceMediatedType:
|
||||||
|
- return q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, addr, bridge.ID, romFile)
|
||||||
|
+ return q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, addr, bus, romFile)
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("Incorrect VFIO device type found")
|
||||||
|
}
|
||||||
|
@@ -1368,7 +1379,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro
|
||||||
|
q.Logger().WithField("dev-id", devID).Info("Start hot-unplug VFIO device")
|
||||||
|
|
||||||
|
if !q.state.HotplugVFIOOnRootBus {
|
||||||
|
- if err := q.arch.removeDeviceFromBridge(devID); err != nil {
|
||||||
|
+ if err := q.putPciAddress(devID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
@@ -1439,18 +1450,17 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
- addr, bridge, err := q.arch.addDeviceToBridge(tap.ID, types.PCI)
|
||||||
|
+ addr, bus, pciAddr, err := q.getPciAddress(tap.ID, types.PCI)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
if err != nil {
|
||||||
|
- q.arch.removeDeviceFromBridge(tap.ID)
|
||||||
|
+ q.putPciAddress(tap.ID)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
- pciAddr := fmt.Sprintf("%02x/%s", bridge.Addr, addr)
|
||||||
|
endpoint.SetPciAddr(pciAddr)
|
||||||
|
|
||||||
|
var machine govmmQemu.Machine
|
||||||
|
@@ -1459,14 +1469,14 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if machine.Type == QemuCCWVirtio {
|
||||||
|
- devNoHotplug := fmt.Sprintf("fe.%x.%x", bridge.Addr, addr)
|
||||||
|
+ devNoHotplug := fmt.Sprintf("fe.%x.%x", bus, addr)
|
||||||
|
return q.qmpMonitorCh.qmp.ExecuteNetCCWDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), devNoHotplug, int(q.config.NumVCPUs))
|
||||||
|
}
|
||||||
|
- return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bridge.ID, romFile, int(q.config.NumVCPUs), defaultDisableModern)
|
||||||
|
+ return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bus, romFile, int(q.config.NumVCPUs), defaultDisableModern)
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
- if err := q.arch.removeDeviceFromBridge(tap.ID); err != nil {
|
||||||
|
+ if err := q.putPciAddress(tap.ID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -2042,6 +2052,21 @@ func genericMemoryTopology(memoryMb, hostMemoryMb uint64, slots uint8, memoryOff
|
||||||
|
return memory
|
||||||
|
}
|
||||||
|
|
||||||
|
+func genericAppendRootPorts(devices []govmmQemu.Device, rootPorts *types.PCIeRootPortPool) []govmmQemu.Device {
|
||||||
|
+ for _, rp := range rootPorts.Items {
|
||||||
|
+ devices = append(devices, govmmQemu.PCIeRootPortDevice{
|
||||||
|
+ Port: rp.Port,
|
||||||
|
+ Bus: rp.Bus,
|
||||||
|
+ ID: rp.ID,
|
||||||
|
+ Chassis: strconv.Itoa(int(rp.Chassis)),
|
||||||
|
+ Multifunction: rp.Multifunction,
|
||||||
|
+ Addr: fmt.Sprintf("0x%s.0x%s", rp.Slot, rp.Function),
|
||||||
|
+ })
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return devices
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// genericAppendPCIeRootPort appends to devices the given pcie-root-port
|
||||||
|
func genericAppendPCIeRootPort(devices []govmmQemu.Device, number uint32, machineType string) []govmmQemu.Device {
|
||||||
|
var (
|
||||||
|
@@ -2241,6 +2266,7 @@ func (q *qemu) save() (s persistapi.HypervisorState) {
|
||||||
|
s.HotpluggedMemory = q.state.HotpluggedMemory
|
||||||
|
s.HotplugVFIOOnRootBus = q.state.HotplugVFIOOnRootBus
|
||||||
|
s.PCIeRootPort = q.state.PCIeRootPort
|
||||||
|
+ s.PCIeRootPortsPool = q.state.PCIeRootPortsPool
|
||||||
|
|
||||||
|
for _, bridge := range q.arch.getBridges() {
|
||||||
|
s.Bridges = append(s.Bridges, persistapi.Bridge{
|
||||||
|
@@ -2265,6 +2291,7 @@ func (q *qemu) load(s persistapi.HypervisorState) {
|
||||||
|
q.state.HotplugVFIOOnRootBus = s.HotplugVFIOOnRootBus
|
||||||
|
q.state.VirtiofsdPid = s.VirtiofsdPid
|
||||||
|
q.state.PCIeRootPort = s.PCIeRootPort
|
||||||
|
+ q.state.PCIeRootPortsPool = s.PCIeRootPortsPool
|
||||||
|
|
||||||
|
for _, bridge := range s.Bridges {
|
||||||
|
q.state.Bridges = append(q.state.Bridges, types.NewBridge(types.Type(bridge.Type), bridge.ID, bridge.DeviceAddr, bridge.Addr))
|
||||||
|
@@ -2302,3 +2329,56 @@ func (q *qemu) generateSocket(id string, useVsock bool) (interface{}, error) {
|
||||||
|
func (q *qemu) getMemorySize() uint32 {
|
||||||
|
return q.config.MemorySize + uint32(q.state.HotpluggedMemory)
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+// getPciAddress allocate the pci slot to hotplugged device and
|
||||||
|
+// return the pci slot address
|
||||||
|
+func (q *qemu) getPciAddress(devID string, t types.Type) (slot, bus, pciAddr string, err error) {
|
||||||
|
+ machine, err := q.getQemuMachine()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return "", "", "", err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ switch machine.Type {
|
||||||
|
+ case QemuVirt:
|
||||||
|
+ rp, err := q.state.PCIeRootPortsPool.AddDevice(devID)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return "", "", "", err
|
||||||
|
+ }
|
||||||
|
+ // PCIe Root Port only have one slot
|
||||||
|
+ slot = "0x0"
|
||||||
|
+ // pciAddr specifies the slot and function of the Root Port and the slot of the device
|
||||||
|
+ pciAddr = fmt.Sprintf("%s.%s/00", rp.Slot, rp.Function)
|
||||||
|
+ bus = rp.ID
|
||||||
|
+ default:
|
||||||
|
+ var bridge types.Bridge
|
||||||
|
+ slot, bridge, err = q.arch.addDeviceToBridge(devID, t)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return "", "", "", err
|
||||||
|
+ }
|
||||||
|
+ bus = bridge.ID
|
||||||
|
+ // PCI address is in the format bridge-addr.0/device-addr eg. "03.0/02"
|
||||||
|
+ pciAddr = fmt.Sprintf("%02x.0", bridge.Addr) + "/" + slot
|
||||||
|
+ }
|
||||||
|
+ return slot, bus, pciAddr, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (q *qemu) putPciAddress(devID string) error {
|
||||||
|
+ machine, err := q.getQemuMachine()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ switch machine.Type {
|
||||||
|
+ case QemuVirt:
|
||||||
|
+ err := q.state.PCIeRootPortsPool.RemoveDevice(devID)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ default:
|
||||||
|
+ if err := q.arch.removeDeviceFromBridge(devID); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/qemu_arch_base.go b/virtcontainers/qemu_arch_base.go
|
||||||
|
index 9d72dd09..cb045530 100644
|
||||||
|
--- a/virtcontainers/qemu_arch_base.go
|
||||||
|
+++ b/virtcontainers/qemu_arch_base.go
|
||||||
|
@@ -130,6 +130,9 @@ type qemuArch interface {
|
||||||
|
|
||||||
|
// appendPCIeRootPortDevice appends a pcie-root-port device to pcie.0 bus
|
||||||
|
appendPCIeRootPortDevice(devices []govmmQemu.Device, number uint32) []govmmQemu.Device
|
||||||
|
+
|
||||||
|
+ // appendRootPorts appends a pcie-root-port device to devices when qemu machine type is "virt"
|
||||||
|
+ appendRootPorts(devices []govmmQemu.Device, rootPorts *types.PCIeRootPortPool) []govmmQemu.Device
|
||||||
|
}
|
||||||
|
|
||||||
|
type qemuArchBase struct {
|
||||||
|
@@ -766,3 +769,9 @@ func (q *qemuArchBase) addBridge(b types.Bridge) {
|
||||||
|
func (q *qemuArchBase) appendPCIeRootPortDevice(devices []govmmQemu.Device, number uint32) []govmmQemu.Device {
|
||||||
|
return genericAppendPCIeRootPort(devices, number, q.machineType)
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+// appendRootPorts appends a pcie-root-port device to devices when qemu machine type is "virt"
|
||||||
|
+// which is different appendPCIeRootPortDevice function
|
||||||
|
+func (q *qemuArchBase) appendRootPorts(devices []govmmQemu.Device, rootPorts *types.PCIeRootPortPool) []govmmQemu.Device {
|
||||||
|
+ return genericAppendRootPorts(devices, rootPorts)
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/types/pcie.go b/virtcontainers/types/pcie.go
|
||||||
|
new file mode 100644
|
||||||
|
index 00000000..83eb6944
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/virtcontainers/types/pcie.go
|
||||||
|
@@ -0,0 +1,103 @@
|
||||||
|
+package types
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "fmt"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+const (
|
||||||
|
+ maxRootPortsCapacity = 25
|
||||||
|
+ slotPerDevice = 7
|
||||||
|
+
|
||||||
|
+ // PCIeRootBus is "pcie.0"
|
||||||
|
+ PCIeRootBus = "pcie.0"
|
||||||
|
+
|
||||||
|
+ // startPort specifies the start port of pcie-root-port
|
||||||
|
+ // for the first slot of "pcie.0" is reserved, so the 1~7 ports are also reserved
|
||||||
|
+ startPort = 8
|
||||||
|
+ funcNumPerSlot = 8
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+// PCIeRootPort describe the PCIe Root Port
|
||||||
|
+type PCIeRootPort struct {
|
||||||
|
+ // DeviceID specify the device hotplug on the Root Port
|
||||||
|
+ DeviceID string
|
||||||
|
+
|
||||||
|
+ // Port number is the Root Port index
|
||||||
|
+ Port string
|
||||||
|
+
|
||||||
|
+ // Bus number where the Root Port is plugged, typically pcie.0
|
||||||
|
+ Bus string
|
||||||
|
+
|
||||||
|
+ // ID is used to identify the pcie-root-port in qemu
|
||||||
|
+ ID string
|
||||||
|
+
|
||||||
|
+ // Slot specifies slot address of Root Port
|
||||||
|
+ Slot string
|
||||||
|
+
|
||||||
|
+ // Function specifies function of Root Port
|
||||||
|
+ Function string
|
||||||
|
+
|
||||||
|
+ // Chassis number
|
||||||
|
+ Chassis uint32
|
||||||
|
+
|
||||||
|
+ // Multifunction is used to specify the pcie-root-port is multifunction supported
|
||||||
|
+ Multifunction bool
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// PCIeRootPortPool describe a set of PCIe Root Ports
|
||||||
|
+type PCIeRootPortPool struct {
|
||||||
|
+ // Items contains information about devices plugged and number limit
|
||||||
|
+ Items []*PCIeRootPort
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// Init Initialized the PCIeRootPortPool instance
|
||||||
|
+func (rp *PCIeRootPortPool) Init(number int) {
|
||||||
|
+ if number == 0 || number > maxRootPortsCapacity {
|
||||||
|
+ number = maxRootPortsCapacity
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ for i := 0; i < number; i++ {
|
||||||
|
+ dev := &PCIeRootPort{
|
||||||
|
+ DeviceID: "",
|
||||||
|
+ Port: fmt.Sprintf("0x%x", startPort+i),
|
||||||
|
+ Bus: PCIeRootBus,
|
||||||
|
+ ID: fmt.Sprintf("pci.%d", i+1),
|
||||||
|
+ Chassis: uint32(i + 1),
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ major := i / funcNumPerSlot
|
||||||
|
+ minor := i % funcNumPerSlot
|
||||||
|
+ dev.Multifunction = false
|
||||||
|
+ if minor == 0 {
|
||||||
|
+ dev.Multifunction = true
|
||||||
|
+ }
|
||||||
|
+ dev.Slot = fmt.Sprintf("%02x", major+1)
|
||||||
|
+ dev.Function = fmt.Sprintf("%x", minor)
|
||||||
|
+
|
||||||
|
+ rp.Items = append(rp.Items, dev)
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// AddDevice add a device to the PCIeRootPortPool
|
||||||
|
+func (rp *PCIeRootPortPool) AddDevice(devID string) (*PCIeRootPort, error) {
|
||||||
|
+ for _, it := range rp.Items {
|
||||||
|
+ if it.DeviceID == "" {
|
||||||
|
+ it.DeviceID = devID
|
||||||
|
+ return it, nil
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ return nil, fmt.Errorf("Unable to hot plug device on Root Ports: there are not empty slots")
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// RemoveDevice remove a device from the PCIeRootPortPool
|
||||||
|
+func (rp *PCIeRootPortPool) RemoveDevice(devID string) error {
|
||||||
|
+ for _, it := range rp.Items {
|
||||||
|
+ if it.DeviceID == devID {
|
||||||
|
+ // free address to re-use the same slot with other devices
|
||||||
|
+ it.DeviceID = ""
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return fmt.Errorf("Unable to hot unplug device %s: not present on Root Port", devID)
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,245 @@
|
|||||||
|
From a36c9857447aaf22628af1ef01406a916436133b Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 18 Aug 2020 22:00:04 +0800
|
||||||
|
Subject: [PATCH 39/50] storage: add storage common functions and structs
|
||||||
|
|
||||||
|
reason:
|
||||||
|
1. add storage.go and storage_spec.go
|
||||||
|
2. provide funcs for mount nfs and gpath
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/storage/storage.go | 115 +++++++++++++++++++++++++++++++++
|
||||||
|
virtcontainers/storage/storage_spec.go | 98 ++++++++++++++++++++++++++++
|
||||||
|
2 files changed, 213 insertions(+)
|
||||||
|
create mode 100644 virtcontainers/storage/storage.go
|
||||||
|
create mode 100644 virtcontainers/storage/storage_spec.go
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/storage/storage.go b/virtcontainers/storage/storage.go
|
||||||
|
new file mode 100644
|
||||||
|
index 00000000..77ef994f
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/virtcontainers/storage/storage.go
|
||||||
|
@@ -0,0 +1,115 @@
|
||||||
|
+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
||||||
|
+// SPDX-License-Identifier: Apache-2.0
|
||||||
|
+// Description: common functions
|
||||||
|
+// Author: licuifang
|
||||||
|
+// Create: 2019-07-14
|
||||||
|
+
|
||||||
|
+// Package storage provides functions for mounting
|
||||||
|
+package storage
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "fmt"
|
||||||
|
+ "path/filepath"
|
||||||
|
+ "regexp"
|
||||||
|
+ "strings"
|
||||||
|
+
|
||||||
|
+ "github.com/kata-containers/agent/protocols/grpc"
|
||||||
|
+ "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+var (
|
||||||
|
+ // simple regular expressions for "nfs-server.com:/aaa/bbb/ccc/" or "192.168.1.1:/remote/path"
|
||||||
|
+ regUrl = regexp.MustCompile(`^((([a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,6})|((?:(?:25[0-5]|2[0-4]\d|[01]?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d?\d))):/[A-Za-z0-9_./-]{0,1000}$`)
|
||||||
|
+
|
||||||
|
+ defaultMountOption = "nolock"
|
||||||
|
+ storageMountOptionKey = "mount_op"
|
||||||
|
+ storageCustomOptionKey = "custom_op"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+type RemoteStorage struct {
|
||||||
|
+ Source string
|
||||||
|
+ Dest string
|
||||||
|
+ Options map[string][]string
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (n *RemoteStorage) Validate(volumeType string) error {
|
||||||
|
+ if n.Source == "" || n.Dest == "" {
|
||||||
|
+ return fmt.Errorf("the source and dest of storage cannot be empty")
|
||||||
|
+ }
|
||||||
|
+ switch volumeType {
|
||||||
|
+ case NFS:
|
||||||
|
+ chk := regUrl.FindAllString(n.Source, -1)
|
||||||
|
+ if chk == nil {
|
||||||
|
+ return fmt.Errorf("invalid url for nfs")
|
||||||
|
+ }
|
||||||
|
+ case GPATHFS:
|
||||||
|
+ if !filepath.IsAbs(n.Source) {
|
||||||
|
+ return fmt.Errorf("invalid gpath")
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (n *RemoteStorage) GetGrpcStorageAndAppendMount(volumeType string, vmBasePath string, spec *specs.Spec, sandboxId string) *grpc.Storage {
|
||||||
|
+ var (
|
||||||
|
+ grpcStorag *grpc.Storage
|
||||||
|
+ vmPath string
|
||||||
|
+ )
|
||||||
|
+ switch volumeType {
|
||||||
|
+ case NFS:
|
||||||
|
+ // source of sfs like remote-nfs.com:/share-53ee51a1
|
||||||
|
+ // source of sfs with subpath like remote-nfs.com:/share-53ee51a1/aaa/bbb
|
||||||
|
+ // source of sfs-turbo like ip:/
|
||||||
|
+ // source of sfs-turbo with subpath like ip://aaa or ip://aaa/bbb
|
||||||
|
+ // here we only get the origin source such as remote-nfs.com:/share-53ee51a1 or ip:/ as the Source of grpcStorage
|
||||||
|
+ item := strings.Split(n.Source, ":")
|
||||||
|
+ if len(item) != 2 {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+ vmPath = filepath.Join(vmBasePath, sandboxId, item[0], item[1])
|
||||||
|
+ parts := strings.SplitAfter(item[1], "/")
|
||||||
|
+ if len(parts) > 2 {
|
||||||
|
+ n.Source = item[0] + ":" + parts[0] + parts[1]
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ grpcStorag = &grpc.Storage{
|
||||||
|
+ Driver: NFS,
|
||||||
|
+ Source: n.Source,
|
||||||
|
+ MountPoint: vmPath,
|
||||||
|
+ }
|
||||||
|
+ grpcStorag.Options = append(grpcStorag.Options, defaultMountOption)
|
||||||
|
+ case GPATHFS:
|
||||||
|
+ vmPath = n.Source
|
||||||
|
+ grpcStorag = &grpc.Storage{
|
||||||
|
+ Driver: GPATHFS,
|
||||||
|
+ Source: vmPath,
|
||||||
|
+ MountPoint: n.Dest,
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ if custumOpts, ok := n.Options[storageCustomOptionKey]; ok {
|
||||||
|
+ for _, custumOpt := range custumOpts {
|
||||||
|
+ grpcStorag.Options = append(grpcStorag.Options, custumOpt)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ mnt := specs.Mount{
|
||||||
|
+ Source: vmPath,
|
||||||
|
+ Destination: n.Dest,
|
||||||
|
+ }
|
||||||
|
+ if mountOpts, ok := n.Options[storageMountOptionKey]; ok {
|
||||||
|
+ for _, mountOpt := range mountOpts {
|
||||||
|
+ if mountOpt == "shared" {
|
||||||
|
+ // in order to support "shared" propagation for guestpath mounting to container directory
|
||||||
|
+ // we should set rootfspropagation shared
|
||||||
|
+ spec.Linux.RootfsPropagation = "shared"
|
||||||
|
+ }
|
||||||
|
+ mnt.Options = append(mnt.Options, mountOpt)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ mnt.Options = append(mnt.Options, "rbind")
|
||||||
|
+ mnt.Type = "bind"
|
||||||
|
+ spec.Mounts = append(spec.Mounts, mnt)
|
||||||
|
+
|
||||||
|
+ return grpcStorag
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/storage/storage_spec.go b/virtcontainers/storage/storage_spec.go
|
||||||
|
new file mode 100644
|
||||||
|
index 00000000..8e866b8d
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/virtcontainers/storage/storage_spec.go
|
||||||
|
@@ -0,0 +1,98 @@
|
||||||
|
+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
||||||
|
+// Description: common functions
|
||||||
|
+// Author: caihaomin c00416947
|
||||||
|
+// Create: 2019-05-05
|
||||||
|
+
|
||||||
|
+package storage
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "encoding/json"
|
||||||
|
+ "fmt"
|
||||||
|
+
|
||||||
|
+ "github.com/kata-containers/agent/protocols/grpc"
|
||||||
|
+ "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+const (
|
||||||
|
+ NFS = "nfs"
|
||||||
|
+ GPATHFS = "gpath"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+// STORAGES specifies the storage type and is supported
|
||||||
|
+var STORAGES = map[string]bool{
|
||||||
|
+ NFS: true,
|
||||||
|
+ GPATHFS: true,
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+type StorageSpec struct {
|
||||||
|
+ // StorageType specifies the type of storage passing down
|
||||||
|
+ StorageType string `json:"storage_type,omitempty"`
|
||||||
|
+
|
||||||
|
+ Source string `json:"source,omitempty"`
|
||||||
|
+ Destination string `json:"dest,omitempty"`
|
||||||
|
+
|
||||||
|
+ // Options specifies the options of storage type
|
||||||
|
+ Options map[string][]string `json:"options,omitempty"`
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *StorageSpec) generateInstances() StorageOperation {
|
||||||
|
+ var storageOpt StorageOperation
|
||||||
|
+ storageOpt = &RemoteStorage{
|
||||||
|
+ Source: s.Source,
|
||||||
|
+ Dest: s.Destination,
|
||||||
|
+ Options: s.Options,
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return storageOpt
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func ValidateStorageValue(value string) error {
|
||||||
|
+ var storageSpecs []StorageSpec
|
||||||
|
+ if err := json.Unmarshal([]byte(value), &storageSpecs); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ var storageOpt []StorageOperation
|
||||||
|
+ for _, item := range storageSpecs {
|
||||||
|
+ if supported, ok := STORAGES[item.StorageType]; !ok || !supported {
|
||||||
|
+ return fmt.Errorf("type %s of storage is not supported", item.StorageType)
|
||||||
|
+ }
|
||||||
|
+ storageItem := item.generateInstances()
|
||||||
|
+ if err := storageItem.Validate(item.StorageType); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ storageOpt = append(storageOpt, storageItem)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func GetGrpcStorageAndAppendMount(vmBasePath, value string, spec *specs.Spec, sandboxId string) []*grpc.Storage {
|
||||||
|
+ var (
|
||||||
|
+ grpcStorages []*grpc.Storage
|
||||||
|
+ storageSpec []StorageSpec
|
||||||
|
+ )
|
||||||
|
+
|
||||||
|
+ // the value has been validated before, and there is no need to judge the return value here
|
||||||
|
+ json.Unmarshal([]byte(value), &storageSpec)
|
||||||
|
+ for _, item := range storageSpec {
|
||||||
|
+ storageItem := item.generateInstances()
|
||||||
|
+ grpcStorages = append(grpcStorages, storageItem.GetGrpcStorageAndAppendMount(item.StorageType, vmBasePath, spec, sandboxId))
|
||||||
|
+ }
|
||||||
|
+ return grpcStorages
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+type StorageOperation interface {
|
||||||
|
+ Validate(volumeType string) error
|
||||||
|
+ GetGrpcStorageAndAppendMount(volumeType string, vmBasePath string, spec *specs.Spec, sandboxId string) *grpc.Storage
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+type FakeStorage struct{}
|
||||||
|
+
|
||||||
|
+func (f *FakeStorage) Validate() error {
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (f *FakeStorage) GetGrpcStorageAndAppendMount(vmBasePath string, spec *specs.Spec) *grpc.Storage {
|
||||||
|
+ return &grpc.Storage{}
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
221
runtime/patches/0040-storage-add-go-tests-for-storage.patch
Normal file
221
runtime/patches/0040-storage-add-go-tests-for-storage.patch
Normal file
@ -0,0 +1,221 @@
|
|||||||
|
From 2e32e2c156a134605de42c53ef77366ac73f2614 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 18 Aug 2020 22:01:58 +0800
|
||||||
|
Subject: [PATCH 40/50] storage: add go tests for storage
|
||||||
|
|
||||||
|
reason: add go tests file storage_test.go and storage_spec_test.go
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/storage/storage_spec_test.go | 151 ++++++++++++++++++++++++++++
|
||||||
|
virtcontainers/storage/storage_test.go | 40 ++++++++
|
||||||
|
2 files changed, 191 insertions(+)
|
||||||
|
create mode 100644 virtcontainers/storage/storage_spec_test.go
|
||||||
|
create mode 100644 virtcontainers/storage/storage_test.go
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/storage/storage_spec_test.go b/virtcontainers/storage/storage_spec_test.go
|
||||||
|
new file mode 100644
|
||||||
|
index 00000000..f638245a
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/virtcontainers/storage/storage_spec_test.go
|
||||||
|
@@ -0,0 +1,151 @@
|
||||||
|
+/*
|
||||||
|
+Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
||||||
|
+Description: common functions test
|
||||||
|
+Author: yangfeiyu
|
||||||
|
+Create: 2019-07-24
|
||||||
|
+*/
|
||||||
|
+
|
||||||
|
+package storage
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "encoding/json"
|
||||||
|
+ "testing"
|
||||||
|
+
|
||||||
|
+ "github.com/stretchr/testify/assert"
|
||||||
|
+ "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+func TestValidateStorageValue(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ //invalid storage type,not NFS or GPATHS
|
||||||
|
+ spec := &StorageSpec{
|
||||||
|
+ StorageType: "invalidStorage",
|
||||||
|
+ Source: "sfs.com:/remote/path",
|
||||||
|
+ Destination: "/opt/",
|
||||||
|
+ }
|
||||||
|
+ data, err := json.Marshal(spec)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ err = ValidateStorageValue(string(data))
|
||||||
|
+ assert.Error(err, "StorageType is not NFS or GPATHS,it should return error")
|
||||||
|
+
|
||||||
|
+ //case 1: NFS
|
||||||
|
+ specNFS := []StorageSpec{
|
||||||
|
+ {
|
||||||
|
+ StorageType: NFS,
|
||||||
|
+ Source: "sfs.com:/remote/path",
|
||||||
|
+ Destination: "/opt/"},
|
||||||
|
+ }
|
||||||
|
+ data, err = json.Marshal(specNFS)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ err = ValidateStorageValue(string(data))
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ // case2: invalid source domain address
|
||||||
|
+ specNFS = []StorageSpec{
|
||||||
|
+ {
|
||||||
|
+ StorageType: NFS,
|
||||||
|
+ Source: "sfs..com:/remote/path",
|
||||||
|
+ Destination: "/opt/"},
|
||||||
|
+ }
|
||||||
|
+ data, err = json.Marshal(specNFS)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ err = ValidateStorageValue(string(data))
|
||||||
|
+ assert.Error(err)
|
||||||
|
+
|
||||||
|
+ // case 3: nfs source is valid ip address
|
||||||
|
+ ipSpecNFS := []StorageSpec{
|
||||||
|
+ {
|
||||||
|
+ StorageType: NFS,
|
||||||
|
+ Source: "192.168.18.147:/remote/path",
|
||||||
|
+ Destination: "/tmp/sfsturbo0",
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ data, err = json.Marshal(ipSpecNFS)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ err = ValidateStorageValue(string(data))
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ // case 4: invalid ip address for nfs source
|
||||||
|
+ ipSpecNFS = []StorageSpec{
|
||||||
|
+ {
|
||||||
|
+ StorageType: NFS,
|
||||||
|
+ Source: "192.168.18.300:/remote/path",
|
||||||
|
+ Destination: "/tmp/sfsturbo0",
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ data, err = json.Marshal(ipSpecNFS)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ err = ValidateStorageValue(string(data))
|
||||||
|
+ assert.Error(err)
|
||||||
|
+
|
||||||
|
+ // case 5: validate ip address and source remote path is /
|
||||||
|
+ ipSpecNFS = []StorageSpec{
|
||||||
|
+ {
|
||||||
|
+ StorageType: NFS,
|
||||||
|
+ Source: "192.168.18.3:/",
|
||||||
|
+ Destination: "/tmp/sfsturbo0",
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ data, err = json.Marshal(ipSpecNFS)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ err = ValidateStorageValue(string(data))
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ // case 6: invalid ip address 192.168.18.147.11
|
||||||
|
+ ipSpecNFS = []StorageSpec{
|
||||||
|
+ {
|
||||||
|
+ StorageType: NFS,
|
||||||
|
+ Source: "192.168.18.147.11:/",
|
||||||
|
+ Destination: "/tmp/sfsturbo0",
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ data, err = json.Marshal(ipSpecNFS)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ err = ValidateStorageValue(string(data))
|
||||||
|
+ assert.Error(err)
|
||||||
|
+
|
||||||
|
+ //GPATHS
|
||||||
|
+ specGPATHFS := []StorageSpec{
|
||||||
|
+ {
|
||||||
|
+ StorageType: GPATHFS,
|
||||||
|
+ Source: "/remote/path",
|
||||||
|
+ Destination: "/opt/"},
|
||||||
|
+ }
|
||||||
|
+ data, err = json.Marshal(specGPATHFS)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ err = ValidateStorageValue(string(data))
|
||||||
|
+ if STORAGES[GPATHFS] == true {
|
||||||
|
+ assert.NoError(err, "StorageType GPATHFS is valid,it should return no error")
|
||||||
|
+ } else {
|
||||||
|
+ assert.Error(err, "StorageType GPATHFS is invalid,it should return error")
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestGetGrpcStorageAndAppendMount(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ var spec specs.Spec
|
||||||
|
+ sandboxId := "7f3b5e32cc7fe757e1b69f7a005ca1971def5011e730c82535783630fe24b318"
|
||||||
|
+ vmBasePath := "/tmp"
|
||||||
|
+
|
||||||
|
+ // storageType nfs with legal source and dest will succeed
|
||||||
|
+ storageSpec := "[{\"storage_type\":\"nfs\",\"source\":\"sfs.com:/remote/path\",\"dest\":\"/opt/nfs\"}]"
|
||||||
|
+ grpcStorages := GetGrpcStorageAndAppendMount(vmBasePath,storageSpec,&spec,sandboxId)
|
||||||
|
+ assert.NotNil(grpcStorages[0])
|
||||||
|
+
|
||||||
|
+ // storageType nfs with illegal source and dest will fail
|
||||||
|
+ storageSpec = "[{\"storage_type\":\"nfs\",\"source\":\"sfs.com\",\"dest\":\"/opt/nfs\"}]"
|
||||||
|
+ grpcStorages = GetGrpcStorageAndAppendMount(vmBasePath,storageSpec,&spec,sandboxId)
|
||||||
|
+ assert.Nil(grpcStorages[0])
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/storage/storage_test.go b/virtcontainers/storage/storage_test.go
|
||||||
|
new file mode 100644
|
||||||
|
index 00000000..c9ca4926
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/virtcontainers/storage/storage_test.go
|
||||||
|
@@ -0,0 +1,40 @@
|
||||||
|
+/*
|
||||||
|
+Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
||||||
|
+Description: common functions test
|
||||||
|
+Author: yangfeiyu
|
||||||
|
+Create: 2019-07-24
|
||||||
|
+*/
|
||||||
|
+
|
||||||
|
+package storage
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "testing"
|
||||||
|
+
|
||||||
|
+ "github.com/stretchr/testify/assert"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+func TestValidate(t *testing.T) {
|
||||||
|
+ //NFS valid
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ remoteStorage := &RemoteStorage{
|
||||||
|
+ Source: "sfs.com:/remote/path",
|
||||||
|
+ Dest: "/opt/",
|
||||||
|
+ }
|
||||||
|
+ err := remoteStorage.Validate(NFS)
|
||||||
|
+ assert.NoError(err, "NFS url is valid,it should return no error")
|
||||||
|
+
|
||||||
|
+ //NFS invalid
|
||||||
|
+ remoteStorage.Source = "sfs.com/../remote/path"
|
||||||
|
+ err = remoteStorage.Validate(NFS)
|
||||||
|
+ assert.Error(err, "NFS url is invalid,it should return error")
|
||||||
|
+
|
||||||
|
+ //GPATHS valid
|
||||||
|
+ remoteStorage.Source = "/path/in/vm"
|
||||||
|
+ err = remoteStorage.Validate(GPATHFS)
|
||||||
|
+ assert.NoError(err, "GPATHFS is valid,it should return no error")
|
||||||
|
+
|
||||||
|
+ //GPATHS invalid
|
||||||
|
+ remoteStorage.Source = "./../path/in/../vm"
|
||||||
|
+ err = remoteStorage.Validate(GPATHFS)
|
||||||
|
+ assert.Error(err, "GPATHFS is invalid,it should return error")
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,115 @@
|
|||||||
|
From 31e07f1b6cbf361783c4d7adf9e4b8da30c67384 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 18 Aug 2020 22:05:25 +0800
|
||||||
|
Subject: [PATCH 41/50] storage: mount nfs and gpath with given annotation
|
||||||
|
|
||||||
|
reason: when run container with annotation about storage spec,
|
||||||
|
prepare basic info in kata-runtime
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/kata_agent.go | 14 +++++++++++++-
|
||||||
|
virtcontainers/pkg/annotations/annotations.go | 3 +++
|
||||||
|
virtcontainers/pkg/oci/utils.go | 16 ++++++++++++++++
|
||||||
|
3 files changed, 32 insertions(+), 1 deletion(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
|
||||||
|
index d82a7f2d..ac64817a 100644
|
||||||
|
--- a/virtcontainers/kata_agent.go
|
||||||
|
+++ b/virtcontainers/kata_agent.go
|
||||||
|
@@ -30,10 +30,11 @@ import (
|
||||||
|
ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/pkg/rootless"
|
||||||
|
vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
|
||||||
|
- "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/storage"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/store"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/types"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||||
|
"github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
opentracing "github.com/opentracing/opentracing-go"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
@@ -1427,6 +1428,9 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process,
|
||||||
|
localStorages := k.handleLocalStorage(ociSpec.Mounts, sandbox.id, c.rootfsSuffix)
|
||||||
|
ctrStorages = append(ctrStorages, localStorages...)
|
||||||
|
|
||||||
|
+ remoteStoragtes := k.handleRemoteStorage(ociSpec, sandbox.id)
|
||||||
|
+ ctrStorages = append(ctrStorages, remoteStoragtes...)
|
||||||
|
+
|
||||||
|
// We replace all OCI mount sources that match our container mount
|
||||||
|
// with the right source path (The guest one).
|
||||||
|
if err = k.replaceOCIMountSource(ociSpec, newMounts); err != nil {
|
||||||
|
@@ -1510,6 +1514,14 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process,
|
||||||
|
k.state.URL, consoleURL, c.config.Cmd, createNSList, enterNSList)
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (k *kataAgent) handleRemoteStorage(spec *specs.Spec, sandboxId string) []*grpc.Storage {
|
||||||
|
+ if value, ok := spec.Annotations[vcAnnotations.StorageSpecTypeKey]; ok {
|
||||||
|
+ return storage.GetGrpcStorageAndAppendMount(kataGuestStorageDir, value, spec, sandboxId)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return []*grpc.Storage{}
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// handleEphemeralStorage handles ephemeral storages by
|
||||||
|
// creating a Storage from corresponding source of the mount point
|
||||||
|
func (k *kataAgent) handleEphemeralStorage(mounts []specs.Mount) []*grpc.Storage {
|
||||||
|
diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go
|
||||||
|
index 903c7f03..e50a697c 100644
|
||||||
|
--- a/virtcontainers/pkg/annotations/annotations.go
|
||||||
|
+++ b/virtcontainers/pkg/annotations/annotations.go
|
||||||
|
@@ -68,6 +68,9 @@ const (
|
||||||
|
// AssetHashType is the hash type used for assets verification
|
||||||
|
AssetHashType = kataAnnotationsPrefix + "asset_hash_type"
|
||||||
|
|
||||||
|
+ // StorageSpecTypeKey is the annotation key to fetch storage_spec
|
||||||
|
+ StorageSpecTypeKey = kataAnnotationsPrefix + "storage_spec"
|
||||||
|
+
|
||||||
|
//
|
||||||
|
// Generic annotations
|
||||||
|
//
|
||||||
|
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
||||||
|
index 948bd3cb..d032227e 100644
|
||||||
|
--- a/virtcontainers/pkg/oci/utils.go
|
||||||
|
+++ b/virtcontainers/pkg/oci/utils.go
|
||||||
|
@@ -22,6 +22,7 @@ import (
|
||||||
|
exp "github.com/kata-containers/runtime/virtcontainers/experimental"
|
||||||
|
vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
|
||||||
|
dockershimAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations/dockershim"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/storage"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/types"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/utils"
|
||||||
|
specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
@@ -340,6 +341,17 @@ func SandboxID(spec specs.Spec) (string, error) {
|
||||||
|
return "", fmt.Errorf("Could not find sandbox ID")
|
||||||
|
}
|
||||||
|
|
||||||
|
+func validateStorageSpec(spec specs.Spec) error {
|
||||||
|
+ if storageSpec, ok := spec.Annotations[vcAnnotations.StorageSpecTypeKey]; ok {
|
||||||
|
+ err := storage.ValidateStorageValue(storageSpec)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func addAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) error {
|
||||||
|
addAssetAnnotations(ocispec, config)
|
||||||
|
if err := addHypervisorConfigOverrides(ocispec, config); err != nil {
|
||||||
|
@@ -873,6 +885,10 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid, c
|
||||||
|
// ContainerConfig converts an OCI compatible runtime configuration
|
||||||
|
// file to a virtcontainers container configuration structure.
|
||||||
|
func ContainerConfig(ocispec specs.Spec, bundlePath, cid, console string, detach bool) (vc.ContainerConfig, error) {
|
||||||
|
+ err := validateStorageSpec(ocispec)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return vc.ContainerConfig{}, err
|
||||||
|
+ }
|
||||||
|
rootfs := vc.RootFs{Target: ocispec.Root.Path, Mounted: true}
|
||||||
|
if !filepath.IsAbs(rootfs.Target) {
|
||||||
|
rootfs.Target = filepath.Join(bundlePath, ocispec.Root.Path)
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,38 @@
|
|||||||
|
From 37372be961528471d418bbe066d88a010ad0513c Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 18 Aug 2020 22:12:59 +0800
|
||||||
|
Subject: [PATCH 42/50] kata-runtime: do not ignore updateInterface return
|
||||||
|
error
|
||||||
|
|
||||||
|
reason: If send UpdateInterfaceRequest to kata-agent and return the
|
||||||
|
error back, we should process it correctly, should not ignore the
|
||||||
|
error, which may casue deference nil pointer problem.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/kata_agent.go | 8 ++++++++
|
||||||
|
1 file changed, 8 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
|
||||||
|
index ac64817a..fee4215f 100644
|
||||||
|
--- a/virtcontainers/kata_agent.go
|
||||||
|
+++ b/virtcontainers/kata_agent.go
|
||||||
|
@@ -604,7 +604,15 @@ func (k *kataAgent) updateInterface(ifc *vcTypes.Interface) (*vcTypes.Interface,
|
||||||
|
"interface-requested": fmt.Sprintf("%+v", ifc),
|
||||||
|
"resulting-interface": fmt.Sprintf("%+v", resultingInterface),
|
||||||
|
}).WithError(err).Error("update interface request failed")
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // need to judege resultingInterface is not not, otherwise may cause
|
||||||
|
+ // deference nil pointer panic problem
|
||||||
|
+ if resultingInterface == nil {
|
||||||
|
+ return nil, fmt.Errorf("resultingInterface should not be nil")
|
||||||
|
}
|
||||||
|
+
|
||||||
|
if resultInterface, ok := resultingInterface.(*aTypes.Interface); ok {
|
||||||
|
iface := &vcTypes.Interface{
|
||||||
|
Device: resultInterface.Device,
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,162 @@
|
|||||||
|
From d9738cfd6500ced30efde9e747eb73e5076e73ed Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 18 Aug 2020 22:47:13 +0800
|
||||||
|
Subject: [PATCH 43/50] kata-runtime: support add hypervisor global parameters
|
||||||
|
in config file
|
||||||
|
|
||||||
|
reason: support add hypervisor global parameters in config file
|
||||||
|
with defaultHypervisorParams config option.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
Makefile | 2 ++
|
||||||
|
cli/config/configuration-qemu.toml.in | 7 +++++++
|
||||||
|
pkg/katautils/config-settings.go.in | 1 +
|
||||||
|
pkg/katautils/config.go | 13 ++++++++++++-
|
||||||
|
vendor/github.com/intel/govmm/qemu/qemu.go | 6 +++---
|
||||||
|
virtcontainers/qemu.go | 2 +-
|
||||||
|
6 files changed, 26 insertions(+), 5 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/Makefile b/Makefile
|
||||||
|
index d5e4bbe1..b62e64b0 100644
|
||||||
|
--- a/Makefile
|
||||||
|
+++ b/Makefile
|
||||||
|
@@ -400,6 +400,7 @@ USER_VARS += FIRMWAREPATH
|
||||||
|
USER_VARS += MACHINEACCELERATORS
|
||||||
|
USER_VARS += DEFMACHINETYPE_CLH
|
||||||
|
USER_VARS += KERNELPARAMS
|
||||||
|
+USER_VARS += HYPERVISORPARAMS
|
||||||
|
USER_VARS += LIBEXECDIR
|
||||||
|
USER_VARS += LOCALSTATEDIR
|
||||||
|
USER_VARS += PKGDATADIR
|
||||||
|
@@ -607,6 +608,7 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit
|
||||||
|
-e "s|@FIRMWAREPATH_CLH@|$(FIRMWAREPATH_CLH)|g" \
|
||||||
|
-e "s|@DEFMACHINETYPE_CLH@|$(DEFMACHINETYPE_CLH)|g" \
|
||||||
|
-e "s|@KERNELPARAMS@|$(KERNELPARAMS)|g" \
|
||||||
|
+ -e "s|@HYPERVISORPARAMS@|$(HYPERVISORPARAMS)|g" \
|
||||||
|
-e "s|@LOCALSTATEDIR@|$(LOCALSTATEDIR)|g" \
|
||||||
|
-e "s|@PKGLIBEXECDIR@|$(PKGLIBEXECDIR)|g" \
|
||||||
|
-e "s|@PKGRUNDIR@|$(PKGRUNDIR)|g" \
|
||||||
|
diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in
|
||||||
|
index aa11b38f..e57a954c 100644
|
||||||
|
--- a/cli/config/configuration-qemu.toml.in
|
||||||
|
+++ b/cli/config/configuration-qemu.toml.in
|
||||||
|
@@ -29,6 +29,13 @@ machine_type = "@MACHINETYPE@"
|
||||||
|
# container and look for 'default-kernel-parameters' log entries.
|
||||||
|
kernel_params = "@KERNELPARAMS@"
|
||||||
|
|
||||||
|
+# Optional space-separated list of options to pass to the hypervisor.
|
||||||
|
+# For example, use `hypervisor_params = "kvm-pit.lost_tick_policy=discard"`
|
||||||
|
+#
|
||||||
|
+# WARNING: - any parameter specified here will take priority over the default
|
||||||
|
+# parameter value of the same name used to start the hypervisor.
|
||||||
|
+hypervisor_params = "@HYPERVISORPARAMS@"
|
||||||
|
+
|
||||||
|
# Path to the firmware.
|
||||||
|
# If you want that qemu uses the default firmware leave this option empty
|
||||||
|
firmware = "@FIRMWAREPATH@"
|
||||||
|
diff --git a/pkg/katautils/config-settings.go.in b/pkg/katautils/config-settings.go.in
|
||||||
|
index aaf78cc3..b2dfdfa1 100644
|
||||||
|
--- a/pkg/katautils/config-settings.go.in
|
||||||
|
+++ b/pkg/katautils/config-settings.go.in
|
||||||
|
@@ -20,6 +20,7 @@ var defaultShimPath = "/usr/libexec/kata-containers/kata-shim"
|
||||||
|
var systemdUnitName = "kata-containers.target"
|
||||||
|
|
||||||
|
const defaultKernelParams = ""
|
||||||
|
+const defaultHypervisorParams = "kvm-pit.lost_tick_policy=discard"
|
||||||
|
const defaultMachineType = "pc"
|
||||||
|
|
||||||
|
const defaultVCPUCount uint32 = 1
|
||||||
|
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
||||||
|
index 51120311..3365b3f5 100644
|
||||||
|
--- a/pkg/katautils/config.go
|
||||||
|
+++ b/pkg/katautils/config.go
|
||||||
|
@@ -130,6 +130,7 @@ type hypervisor struct {
|
||||||
|
HotplugVFIOOnRootBus bool `toml:"hotplug_vfio_on_root_bus"`
|
||||||
|
DisableVhostNet bool `toml:"disable_vhost_net"`
|
||||||
|
GuestHookPath string `toml:"guest_hook_path"`
|
||||||
|
+ HypervisorParams string `toml:"hypervisor_params"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type proxy struct {
|
||||||
|
@@ -270,6 +271,14 @@ func (h hypervisor) kernelParams() string {
|
||||||
|
return h.KernelParams
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (h hypervisor) hypervisorParams() string {
|
||||||
|
+ if h.HypervisorParams == "" {
|
||||||
|
+ return defaultHypervisorParams
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return h.HypervisorParams
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func (h hypervisor) machineType() string {
|
||||||
|
if h.MachineType == "" {
|
||||||
|
return defaultMachineType
|
||||||
|
@@ -632,6 +641,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
||||||
|
|
||||||
|
machineAccelerators := h.machineAccelerators()
|
||||||
|
kernelParams := h.kernelParams()
|
||||||
|
+ hypervisorParams := h.hypervisorParams()
|
||||||
|
machineType := h.machineType()
|
||||||
|
|
||||||
|
blockDriver, err := h.blockDeviceDriver()
|
||||||
|
@@ -675,6 +685,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
||||||
|
FirmwarePath: firmware,
|
||||||
|
MachineAccelerators: machineAccelerators,
|
||||||
|
KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)),
|
||||||
|
+ HypervisorParams: vc.DeserializeParams(strings.Fields(hypervisorParams)),
|
||||||
|
HypervisorMachineType: machineType,
|
||||||
|
NumVCPUs: h.defaultVCPUs(),
|
||||||
|
DefaultMaxVCPUs: h.defaultMaxVCPUs(),
|
||||||
|
@@ -983,7 +994,7 @@ func updateRuntimeConfigAgent(configPath string, tomlConf tomlConfig, config *oc
|
||||||
|
TraceMode: agent.traceMode(),
|
||||||
|
TraceType: agent.traceType(),
|
||||||
|
KernelModules: agent.kernelModules(),
|
||||||
|
- MountBlkInVM: agent.mountBlkDevInVM(),
|
||||||
|
+ MountBlkInVM: agent.mountBlkDevInVM(),
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("%s agent type is not supported", k)
|
||||||
|
diff --git a/vendor/github.com/intel/govmm/qemu/qemu.go b/vendor/github.com/intel/govmm/qemu/qemu.go
|
||||||
|
index 3e7720b4..68f8d2b0 100644
|
||||||
|
--- a/vendor/github.com/intel/govmm/qemu/qemu.go
|
||||||
|
+++ b/vendor/github.com/intel/govmm/qemu/qemu.go
|
||||||
|
@@ -2096,7 +2096,7 @@ type Config struct {
|
||||||
|
SMP SMP
|
||||||
|
|
||||||
|
// GlobalParam is the -global parameter.
|
||||||
|
- GlobalParam string
|
||||||
|
+ GlobalParam []string
|
||||||
|
|
||||||
|
// Knobs is a set of qemu boolean settings.
|
||||||
|
Knobs Knobs
|
||||||
|
@@ -2285,9 +2285,9 @@ func (config *Config) appendRTC() {
|
||||||
|
}
|
||||||
|
|
||||||
|
func (config *Config) appendGlobalParam() {
|
||||||
|
- if config.GlobalParam != "" {
|
||||||
|
+ for _, param := range config.GlobalParam {
|
||||||
|
config.qemuParams = append(config.qemuParams, "-global")
|
||||||
|
- config.qemuParams = append(config.qemuParams, config.GlobalParam)
|
||||||
|
+ config.qemuParams = append(config.qemuParams, param)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
|
||||||
|
index a10c66fb..657b6be7 100644
|
||||||
|
--- a/virtcontainers/qemu.go
|
||||||
|
+++ b/virtcontainers/qemu.go
|
||||||
|
@@ -600,7 +600,7 @@ func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNa
|
||||||
|
Knobs: knobs,
|
||||||
|
Incoming: incoming,
|
||||||
|
VGA: "none",
|
||||||
|
- GlobalParam: "kvm-pit.lost_tick_policy=discard",
|
||||||
|
+ GlobalParam: SerializeParams(hypervisorConfig.HypervisorParams, "="),
|
||||||
|
Bios: firmwarePath,
|
||||||
|
PidFile: filepath.Join(q.store.RunVMStoragePath(), q.id, "pid"),
|
||||||
|
}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,375 @@
|
|||||||
|
From dbbf8c5deb14d8033b2e863fb0eb731523af2a47 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Wed, 19 Aug 2020 09:58:07 +0800
|
||||||
|
Subject: [PATCH 44/50] network: support dpdk vhost_user net device
|
||||||
|
|
||||||
|
reason: support dpdk vhost_user net device
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
vendor/github.com/intel/govmm/qemu/qmp.go | 3 +-
|
||||||
|
virtcontainers/hypervisor.go | 3 ++
|
||||||
|
virtcontainers/network.go | 49 ++++++++++++++------
|
||||||
|
virtcontainers/pkg/types/types.go | 2 +
|
||||||
|
virtcontainers/qemu.go | 75 +++++++++++++++++++++++++++++++
|
||||||
|
virtcontainers/sandbox.go | 7 ++-
|
||||||
|
virtcontainers/vhostuser_endpoint.go | 18 ++++++--
|
||||||
|
virtcontainers/vhostuser_endpoint_test.go | 7 ++-
|
||||||
|
8 files changed, 143 insertions(+), 21 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/vendor/github.com/intel/govmm/qemu/qmp.go b/vendor/github.com/intel/govmm/qemu/qmp.go
|
||||||
|
index a64039de..0cb82ffa 100644
|
||||||
|
--- a/vendor/github.com/intel/govmm/qemu/qmp.go
|
||||||
|
+++ b/vendor/github.com/intel/govmm/qemu/qmp.go
|
||||||
|
@@ -970,11 +970,12 @@ func (q *QMP) ExecuteNetdevAdd(ctx context.Context, netdevType, netdevID, ifname
|
||||||
|
// ExecuteNetdevChardevAdd adds a Net device to a QEMU instance
|
||||||
|
// using the netdev_add command. netdevID is the id of the device to add.
|
||||||
|
// Must be valid QMP identifier.
|
||||||
|
-func (q *QMP) ExecuteNetdevChardevAdd(ctx context.Context, netdevType, netdevID, chardev string, queues int) error {
|
||||||
|
+func (q *QMP) ExecuteNetdevChardevAdd(ctx context.Context, netdevType, netdevID, chardev string, vhostforce bool, queues int) error {
|
||||||
|
args := map[string]interface{}{
|
||||||
|
"type": netdevType,
|
||||||
|
"id": netdevID,
|
||||||
|
"chardev": chardev,
|
||||||
|
+ "vhostforce": vhostforce,
|
||||||
|
}
|
||||||
|
if queues > 1 {
|
||||||
|
args["queues"] = queues
|
||||||
|
diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go
|
||||||
|
index c0723daa..60f1d190 100644
|
||||||
|
--- a/virtcontainers/hypervisor.go
|
||||||
|
+++ b/virtcontainers/hypervisor.go
|
||||||
|
@@ -132,6 +132,9 @@ const (
|
||||||
|
// vhostuserDev is a Vhost-user device type
|
||||||
|
vhostuserDev
|
||||||
|
|
||||||
|
+ // vhostUserNetDev is a Vhost-user net device type
|
||||||
|
+ vhostUserNetDev
|
||||||
|
+
|
||||||
|
// CPUDevice is CPU device type
|
||||||
|
cpuDev
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
|
||||||
|
index 488bd00c..a0c64356 100644
|
||||||
|
--- a/virtcontainers/network.go
|
||||||
|
+++ b/virtcontainers/network.go
|
||||||
|
@@ -13,6 +13,7 @@ import (
|
||||||
|
"math/rand"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
+ "path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"runtime"
|
||||||
|
"sort"
|
||||||
|
@@ -65,7 +66,8 @@ const (
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
- regInfName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_\-.]*$`)
|
||||||
|
+ regInfName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_\-.]*$`)
|
||||||
|
+ regVhostName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9\-.]{0,127}$`)
|
||||||
|
)
|
||||||
|
|
||||||
|
//IsValid checks if a model is valid
|
||||||
|
@@ -132,11 +134,12 @@ type NetlinkIface struct {
|
||||||
|
// NetworkInfo gathers all information related to a network interface.
|
||||||
|
// It can be used to store the description of the underlying network.
|
||||||
|
type NetworkInfo struct {
|
||||||
|
- Device string
|
||||||
|
- Iface NetlinkIface
|
||||||
|
- Addrs []netlink.Addr
|
||||||
|
- Routes []netlink.Route
|
||||||
|
- DNS DNSInfo
|
||||||
|
+ Device string
|
||||||
|
+ Iface NetlinkIface
|
||||||
|
+ Addrs []netlink.Addr
|
||||||
|
+ Routes []netlink.Route
|
||||||
|
+ DNS DNSInfo
|
||||||
|
+ VhostUserSocket string
|
||||||
|
}
|
||||||
|
|
||||||
|
// NetworkInterface defines a network interface.
|
||||||
|
@@ -1198,15 +1201,11 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li
|
||||||
|
return endpoint, err
|
||||||
|
}
|
||||||
|
|
||||||
|
- // Check if this is a dummy interface which has a vhost-user socket associated with it
|
||||||
|
- socketPath, err := vhostUserSocketPath(netInfo)
|
||||||
|
- if err != nil {
|
||||||
|
- return nil, err
|
||||||
|
- }
|
||||||
|
-
|
||||||
|
- if socketPath != "" {
|
||||||
|
+ // Currently, we only accept the vhost-user socket paas from user input
|
||||||
|
+ // interface info json file
|
||||||
|
+ if netInfo.VhostUserSocket != "" {
|
||||||
|
networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found")
|
||||||
|
- endpoint, err = createVhostUserEndpoint(netInfo, socketPath)
|
||||||
|
+ endpoint, err = createVhostUserEndpoint(netInfo, netInfo.VhostUserSocket)
|
||||||
|
return endpoint, err
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -1481,6 +1480,24 @@ func verifyIP(ip string) (*net.IP, error) {
|
||||||
|
return &netIP, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+// verifyVhostSocket verifies the vhost socket is valid or not
|
||||||
|
+func verifyVhostSocket(vhostSocket string) error {
|
||||||
|
+ if vhostSocket == "" {
|
||||||
|
+ networkLogger().Debug("no dpdk network")
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+ if regVhostName.FindAllString(filepath.Base(vhostSocket), -1) == nil {
|
||||||
|
+ return fmt.Errorf("invalid input of vhostSocket name, please check the rules")
|
||||||
|
+ }
|
||||||
|
+ info, err := os.Stat(vhostSocket)
|
||||||
|
+ if err != nil || info.IsDir() || info.Mode()&os.ModeSocket == 0 {
|
||||||
|
+ return fmt.Errorf("invalid vhost socket: %v", vhostSocket)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ networkLogger().WithField("vhostSocket", vhostSocket).Infof("using dpdk network, socket file is: %s ", vhostSocket)
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// validInterface check the input interface valid or not
|
||||||
|
func validInterface(inf *vcTypes.Interface, enableCompatOldCNI bool) error {
|
||||||
|
if enableCompatOldCNI && verifyInterfaceName(inf.Device) != nil {
|
||||||
|
@@ -1503,6 +1520,10 @@ func validInterface(inf *vcTypes.Interface, enableCompatOldCNI bool) error {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
+ if err := verifyVhostSocket(inf.VhostUserSocket); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
// Currently, only one IP address can be passed, which reduces the test entry and fault injection.
|
||||||
|
if len(inf.IPAddresses) > 0 {
|
||||||
|
if len(inf.IPAddresses) != 1 {
|
||||||
|
diff --git a/virtcontainers/pkg/types/types.go b/virtcontainers/pkg/types/types.go
|
||||||
|
index b41b0c75..dccd92f8 100644
|
||||||
|
--- a/virtcontainers/pkg/types/types.go
|
||||||
|
+++ b/virtcontainers/pkg/types/types.go
|
||||||
|
@@ -29,6 +29,8 @@ type Interface struct {
|
||||||
|
// library, regarding each type of link. Here is a non exhaustive
|
||||||
|
// list: "veth", "macvtap", "vlan", "macvlan", "tap", ...
|
||||||
|
LinkType string `json:"linkType,omitempty"`
|
||||||
|
+ // VhostUserSocket is DPDK-backed vHost user ports.
|
||||||
|
+ VhostUserSocket string `json:"vhostUserSocket,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Route describes a network route.
|
||||||
|
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
|
||||||
|
index 657b6be7..84797e0d 100644
|
||||||
|
--- a/virtcontainers/qemu.go
|
||||||
|
+++ b/virtcontainers/qemu.go
|
||||||
|
@@ -1279,6 +1279,78 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (q *qemu) hotplugVhostUserNetDevice(endpoint Endpoint, op operation) error {
|
||||||
|
+ err := q.qmpSetup()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ device := endpoint.(*VhostUserEndpoint)
|
||||||
|
+
|
||||||
|
+ charDevID := utils.MakeNameID("char", device.ID, maxDevIDSize)
|
||||||
|
+ devID := utils.MakeNameID("virtio", device.ID, maxDevIDSize)
|
||||||
|
+
|
||||||
|
+ if op == addDevice {
|
||||||
|
+ // Add chardev specifying the appropriate socket.
|
||||||
|
+ if err = q.qmpMonitorCh.qmp.ExecuteCharDevUnixSocketAdd(q.qmpMonitorCh.ctx, charDevID, device.SocketPath, true, false); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ defer func() {
|
||||||
|
+ if err != nil {
|
||||||
|
+ errRb := q.qmpMonitorCh.qmp.ExecuteChardevDel(q.qmpMonitorCh.ctx, charDevID)
|
||||||
|
+ if errRb != nil {
|
||||||
|
+ q.Logger().Errorf("deletes a net device roll back failed. charDev id:%s", charDevID)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
+ netDevType := "vhost-user"
|
||||||
|
+ // Add netdev of type vhost-user.
|
||||||
|
+ if err = q.qmpMonitorCh.qmp.ExecuteNetdevChardevAdd(q.qmpMonitorCh.ctx, netDevType, devID, charDevID, true, 0); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ defer func() {
|
||||||
|
+ if err != nil {
|
||||||
|
+ errRb := q.qmpMonitorCh.qmp.ExecuteNetdevDel(q.qmpMonitorCh.ctx, devID)
|
||||||
|
+ if errRb != nil {
|
||||||
|
+ q.Logger().Errorf("deletes a net device roll back failed. netDev id:%s", devID)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
+ var addr, bus, pciAddr string
|
||||||
|
+ addr, bus, pciAddr, err = q.getPciAddress(device.ID, types.PCI)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+ defer func() {
|
||||||
|
+ if err != nil {
|
||||||
|
+ q.putPciAddress(device.ID)
|
||||||
|
+ }
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
+ endpoint.SetPciAddr(pciAddr)
|
||||||
|
+
|
||||||
|
+ // Add virtio-net-pci device.
|
||||||
|
+ err = q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, devID, devID, device.HardAddr, addr, bus, romFile, 0, q.arch.runNested())
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err = q.putPciAddress(device.ID); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ if err = q.qmpMonitorCh.qmp.ExecuteDeviceDel(q.qmpMonitorCh.ctx, devID); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ if err = q.qmpMonitorCh.qmp.ExecuteNetdevDel(q.qmpMonitorCh.ctx, devID); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func (q *qemu) hotplugVhostUserDevice(vAttr *config.VhostUserDeviceAttrs, op operation) error {
|
||||||
|
err := q.qmpSetup()
|
||||||
|
if err != nil {
|
||||||
|
@@ -1510,6 +1582,9 @@ func (q *qemu) hotplugDevice(devInfo interface{}, devType deviceType, op operati
|
||||||
|
case vhostuserDev:
|
||||||
|
vAttr := devInfo.(*config.VhostUserDeviceAttrs)
|
||||||
|
return nil, q.hotplugVhostUserDevice(vAttr, op)
|
||||||
|
+ case vhostUserNetDev:
|
||||||
|
+ device := devInfo.(Endpoint)
|
||||||
|
+ return nil, q.hotplugVhostUserNetDevice(device, op)
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("cannot hotplug device: unsupported device type '%v'", devType)
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index 0d599267..e6f155a3 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -946,7 +946,8 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) {
|
||||||
|
},
|
||||||
|
Type: inf.LinkType,
|
||||||
|
},
|
||||||
|
- Addrs: addrs,
|
||||||
|
+ Addrs: addrs,
|
||||||
|
+ VhostUserSocket: inf.VhostUserSocket,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -961,6 +962,10 @@ func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (grpcIf *vcTypes.Interfac
|
||||||
|
if ep.Name() == inf.Name {
|
||||||
|
return nil, fmt.Errorf("interface %s is already exist", inf.Name)
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+ if ep.Properties().VhostUserSocket != "" && inf.VhostUserSocket != "" {
|
||||||
|
+ return nil, fmt.Errorf("sandbox %s only support one dpdk socket", s.ID())
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
|
||||||
|
netInfo, err := s.generateNetInfo(inf)
|
||||||
|
diff --git a/virtcontainers/vhostuser_endpoint.go b/virtcontainers/vhostuser_endpoint.go
|
||||||
|
index bb4a67be..2fc3d837 100644
|
||||||
|
--- a/virtcontainers/vhostuser_endpoint.go
|
||||||
|
+++ b/virtcontainers/vhostuser_endpoint.go
|
||||||
|
@@ -12,6 +12,7 @@ import (
|
||||||
|
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/device/config"
|
||||||
|
persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/utils"
|
||||||
|
)
|
||||||
|
|
||||||
|
@@ -23,6 +24,7 @@ const hostSocketSearchPath = "/tmp/vhostuser_%s/vhu.sock"
|
||||||
|
|
||||||
|
// VhostUserEndpoint represents a vhost-user socket based network interface
|
||||||
|
type VhostUserEndpoint struct {
|
||||||
|
+ ID string
|
||||||
|
// Path to the vhost-user socket on the host system
|
||||||
|
SocketPath string
|
||||||
|
// MAC address of the interface
|
||||||
|
@@ -99,18 +101,28 @@ func (endpoint *VhostUserEndpoint) Detach(netNsCreated bool, netNsPath string) e
|
||||||
|
|
||||||
|
// HotAttach for vhostuser endpoint not supported yet
|
||||||
|
func (endpoint *VhostUserEndpoint) HotAttach(h hypervisor) error {
|
||||||
|
- return fmt.Errorf("VhostUserEndpoint does not support Hot attach")
|
||||||
|
+ networkLogger().Debug("Hot attaching vhost-user endpoint")
|
||||||
|
+ if _, err := h.hotplugAddDevice(endpoint, vhostUserNetDev); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// HotDetach for vhostuser endpoint not supported yet
|
||||||
|
func (endpoint *VhostUserEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error {
|
||||||
|
- return fmt.Errorf("VhostUserEndpoint does not support Hot detach")
|
||||||
|
+ networkLogger().Debug("Hot detaching vhost-user endpoint")
|
||||||
|
+ if _, err := h.hotplugRemoveDevice(endpoint, vhostUserNetDev); err != nil {
|
||||||
|
+ networkLogger().WithError(err).Errorf("Error detach vhostUserSocket")
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create a vhostuser endpoint
|
||||||
|
func createVhostUserEndpoint(netInfo NetworkInfo, socket string) (*VhostUserEndpoint, error) {
|
||||||
|
-
|
||||||
|
+ uniqueID := uuid.Generate().String()
|
||||||
|
vhostUserEndpoint := &VhostUserEndpoint{
|
||||||
|
+ ID: uniqueID,
|
||||||
|
SocketPath: socket,
|
||||||
|
HardAddr: netInfo.Iface.HardwareAddr.String(),
|
||||||
|
IfaceName: netInfo.Iface.Name,
|
||||||
|
diff --git a/virtcontainers/vhostuser_endpoint_test.go b/virtcontainers/vhostuser_endpoint_test.go
|
||||||
|
index ad013e12..584490cc 100644
|
||||||
|
--- a/virtcontainers/vhostuser_endpoint_test.go
|
||||||
|
+++ b/virtcontainers/vhostuser_endpoint_test.go
|
||||||
|
@@ -11,6 +11,7 @@ import (
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/vishvananda/netlink"
|
||||||
|
)
|
||||||
|
@@ -95,7 +96,7 @@ func TestVhostUserEndpoint_HotAttach(t *testing.T) {
|
||||||
|
h := &mockHypervisor{}
|
||||||
|
|
||||||
|
err := v.HotAttach(h)
|
||||||
|
- assert.Error(err)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVhostUserEndpoint_HotDetach(t *testing.T) {
|
||||||
|
@@ -109,10 +110,11 @@ func TestVhostUserEndpoint_HotDetach(t *testing.T) {
|
||||||
|
h := &mockHypervisor{}
|
||||||
|
|
||||||
|
err := v.HotDetach(h, true, "")
|
||||||
|
- assert.Error(err)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCreateVhostUserEndpoint(t *testing.T) {
|
||||||
|
+ uniqueID := uuid.Generate().String()
|
||||||
|
macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x48}
|
||||||
|
ifcName := "vhost-deadbeef"
|
||||||
|
socket := "/tmp/vhu_192.168.0.1"
|
||||||
|
@@ -128,6 +130,7 @@ func TestCreateVhostUserEndpoint(t *testing.T) {
|
||||||
|
}
|
||||||
|
|
||||||
|
expected := &VhostUserEndpoint{
|
||||||
|
+ ID: uniqueID,
|
||||||
|
SocketPath: socket,
|
||||||
|
HardAddr: macAddr.String(),
|
||||||
|
IfaceName: ifcName,
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
115
runtime/patches/0045-network-support-set-dns.patch
Normal file
115
runtime/patches/0045-network-support-set-dns.patch
Normal file
@ -0,0 +1,115 @@
|
|||||||
|
From e104f084ed4f10049f45d9473faed229371a1c6c Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Wed, 19 Aug 2020 09:59:52 +0800
|
||||||
|
Subject: [PATCH 45/50] network: support set dns
|
||||||
|
|
||||||
|
reason: when running a sandbox with annotation about
|
||||||
|
dns spec, overload k.getDNS(sandbox)
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/kata_agent.go | 11 +++++++++++
|
||||||
|
virtcontainers/network.go | 8 ++++----
|
||||||
|
virtcontainers/pkg/annotations/annotations.go | 3 +++
|
||||||
|
virtcontainers/pkg/oci/utils.go | 19 +++++++++++++++++--
|
||||||
|
4 files changed, 35 insertions(+), 6 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
|
||||||
|
index fee4215f..d28d8cce 100644
|
||||||
|
--- a/virtcontainers/kata_agent.go
|
||||||
|
+++ b/virtcontainers/kata_agent.go
|
||||||
|
@@ -890,6 +890,17 @@ func (k *kataAgent) startSandbox(sandbox *Sandbox) error {
|
||||||
|
KernelModules: kmodules,
|
||||||
|
}
|
||||||
|
|
||||||
|
+ if value, ok := sandbox.config.Annotations[vcAnnotations.SandboxDNSTypeKey]; ok {
|
||||||
|
+ var sandboxDns *DNSInfo
|
||||||
|
+ if err := json.Unmarshal([]byte(value), &sandboxDns); err != nil {
|
||||||
|
+ return fmt.Errorf("get sandbox dns failed %v", err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if sandboxDns != nil {
|
||||||
|
+ req.Dns = sandboxDns.Servers
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
_, err = k.sendReq(req)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
|
||||||
|
index 488bd00c..68eda4a6 100644
|
||||||
|
--- a/virtcontainers/network.go
|
||||||
|
+++ b/virtcontainers/network.go
|
||||||
|
@@ -117,10 +117,10 @@ const (
|
||||||
|
|
||||||
|
// DNSInfo describes the DNS setup related to a network interface.
|
||||||
|
type DNSInfo struct {
|
||||||
|
- Servers []string
|
||||||
|
- Domain string
|
||||||
|
- Searches []string
|
||||||
|
- Options []string
|
||||||
|
+ Servers []string `json:"Servers"`
|
||||||
|
+ Domain string `json:"Domain,omitempty"`
|
||||||
|
+ Searches []string `json:"Searches,omitempty"`
|
||||||
|
+ Options []string `json:"Options,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// NetlinkIface describes fully a network interface.
|
||||||
|
diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go
|
||||||
|
index e50a697c..528dfa66 100644
|
||||||
|
--- a/virtcontainers/pkg/annotations/annotations.go
|
||||||
|
+++ b/virtcontainers/pkg/annotations/annotations.go
|
||||||
|
@@ -71,6 +71,9 @@ const (
|
||||||
|
// StorageSpecTypeKey is the annotation key to fetch storage_spec
|
||||||
|
StorageSpecTypeKey = kataAnnotationsPrefix + "storage_spec"
|
||||||
|
|
||||||
|
+ // SandboxDNSTypeKey is the annotation key to fetch sandbox dns options
|
||||||
|
+ SandboxDNSTypeKey = kataAnnotationsPrefix + "sandbox_dns"
|
||||||
|
+
|
||||||
|
//
|
||||||
|
// Generic annotations
|
||||||
|
//
|
||||||
|
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
||||||
|
index d032227e..3b2af753 100644
|
||||||
|
--- a/virtcontainers/pkg/oci/utils.go
|
||||||
|
+++ b/virtcontainers/pkg/oci/utils.go
|
||||||
|
@@ -7,6 +7,7 @@ package oci
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
+ "encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"path/filepath"
|
||||||
|
@@ -37,8 +38,9 @@ type annotationContainerType struct {
|
||||||
|
type annotationHandler func(value string) error
|
||||||
|
|
||||||
|
var annotationHandlerList = map[string]annotationHandler{
|
||||||
|
- vcAnnotations.StaticCPUTypeKey: validateSandboxCPU,
|
||||||
|
- vcAnnotations.StaticMemTypeKey: validateSandboxMem,
|
||||||
|
+ vcAnnotations.StaticCPUTypeKey: validateSandboxCPU,
|
||||||
|
+ vcAnnotations.StaticMemTypeKey: validateSandboxMem,
|
||||||
|
+ vcAnnotations.SandboxDNSTypeKey: validateSandboxDNS,
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
@@ -1108,3 +1110,16 @@ func validateSandboxMem(value string) error {
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func validateSandboxDNS(value string) error {
|
||||||
|
+ if value == "" {
|
||||||
|
+ return fmt.Errorf("--annotation %s value should not be empty", vcAnnotations.SandboxDNSTypeKey)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ var sandboxDns *vc.DNSInfo
|
||||||
|
+ if err := json.Unmarshal([]byte(value), &sandboxDns); err != nil {
|
||||||
|
+ return fmt.Errorf("invalid value passed by --annotation %s, error: %v", vcAnnotations.SandboxDNSTypeKey, err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,489 @@
|
|||||||
|
From e9c1fb235d0e69d5db72497a8779df707c499d3b Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Wed, 19 Aug 2020 20:36:40 +0800
|
||||||
|
Subject: [PATCH 46/50] network: support multiqueue when inserting interface
|
||||||
|
dynamically
|
||||||
|
|
||||||
|
reason: support multiqueue when inserting interface dynamically
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
.../kata-containers/agent/pkg/types/types.pb.go | 82 ++++++++++++++++------
|
||||||
|
virtcontainers/bridgedmacvlan_endpoint.go | 2 +-
|
||||||
|
virtcontainers/ipvlan_endpoint.go | 2 +-
|
||||||
|
virtcontainers/kata_agent.go | 2 +
|
||||||
|
virtcontainers/network.go | 13 ++--
|
||||||
|
virtcontainers/network_test.go | 2 +-
|
||||||
|
virtcontainers/pkg/types/types.go | 2 +
|
||||||
|
virtcontainers/qemu.go | 10 +--
|
||||||
|
virtcontainers/sandbox.go | 1 +
|
||||||
|
virtcontainers/tap_endpoint.go | 6 +-
|
||||||
|
virtcontainers/tuntap_endpoint.go | 4 +-
|
||||||
|
virtcontainers/veth_endpoint.go | 4 +-
|
||||||
|
virtcontainers/veth_endpoint_test.go | 6 +-
|
||||||
|
virtcontainers/vhostuser_endpoint.go | 4 +-
|
||||||
|
virtcontainers/vhostuser_endpoint_test.go | 2 +-
|
||||||
|
15 files changed, 96 insertions(+), 46 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/vendor/github.com/kata-containers/agent/pkg/types/types.pb.go b/vendor/github.com/kata-containers/agent/pkg/types/types.pb.go
|
||||||
|
index 7ea63e3c..8b7e2a5d 100644
|
||||||
|
--- a/vendor/github.com/kata-containers/agent/pkg/types/types.pb.go
|
||||||
|
+++ b/vendor/github.com/kata-containers/agent/pkg/types/types.pb.go
|
||||||
|
@@ -100,6 +100,7 @@ type Interface struct {
|
||||||
|
// list: "veth", "macvtap", "vlan", "macvlan", "tap", ...
|
||||||
|
Type string `protobuf:"bytes,7,opt,name=type,proto3" json:"type,omitempty"`
|
||||||
|
RawFlags uint32 `protobuf:"varint,8,opt,name=raw_flags,json=rawFlags,proto3" json:"raw_flags,omitempty"`
|
||||||
|
+ Queues uint32 `protobuf:"varint,9,opt,name=Queues,proto3" json:"Queues,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *Interface) Reset() { *m = Interface{} }
|
||||||
|
@@ -163,6 +164,13 @@ func (m *Interface) GetRawFlags() uint32 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (m *Interface) GetQueues() uint32 {
|
||||||
|
+ if m != nil {
|
||||||
|
+ return m.Queues
|
||||||
|
+ }
|
||||||
|
+ return 0
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
type Route struct {
|
||||||
|
Dest string `protobuf:"bytes,1,opt,name=dest,proto3" json:"dest,omitempty"`
|
||||||
|
Gateway string `protobuf:"bytes,2,opt,name=gateway,proto3" json:"gateway,omitempty"`
|
||||||
|
@@ -319,6 +327,11 @@ func (m *Interface) MarshalTo(dAtA []byte) (int, error) {
|
||||||
|
i++
|
||||||
|
i = encodeVarintTypes(dAtA, i, uint64(m.RawFlags))
|
||||||
|
}
|
||||||
|
+ if m.Queues != 0 {
|
||||||
|
+ dAtA[i] = 0x48
|
||||||
|
+ i++
|
||||||
|
+ i = encodeVarintTypes(dAtA, i, uint64(m.Queues))
|
||||||
|
+ }
|
||||||
|
return i, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -430,6 +443,9 @@ func (m *Interface) Size() (n int) {
|
||||||
|
if m.RawFlags != 0 {
|
||||||
|
n += 1 + sovTypes(uint64(m.RawFlags))
|
||||||
|
}
|
||||||
|
+ if m.Queues != 0 {
|
||||||
|
+ n += 1 + sovTypes(uint64(m.Queues))
|
||||||
|
+ }
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -841,6 +857,25 @@ func (m *Interface) Unmarshal(dAtA []byte) error {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
+ case 9:
|
||||||
|
+ if wireType != 0 {
|
||||||
|
+ return fmt.Errorf("proto: wrong wireType = %d for field Queues", wireType)
|
||||||
|
+ }
|
||||||
|
+ m.Queues = 0
|
||||||
|
+ for shift := uint(0); ; shift += 7 {
|
||||||
|
+ if shift >= 64 {
|
||||||
|
+ return ErrIntOverflowTypes
|
||||||
|
+ }
|
||||||
|
+ if iNdEx >= l {
|
||||||
|
+ return io.ErrUnexpectedEOF
|
||||||
|
+ }
|
||||||
|
+ b := dAtA[iNdEx]
|
||||||
|
+ iNdEx++
|
||||||
|
+ m.Queues |= (uint32(b) & 0x7F) << shift
|
||||||
|
+ if b < 0x80 {
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
default:
|
||||||
|
iNdEx = preIndex
|
||||||
|
skippy, err := skipTypes(dAtA[iNdEx:])
|
||||||
|
@@ -1155,27 +1190,28 @@ var (
|
||||||
|
func init() { proto.RegisterFile("pkg/types/types.proto", fileDescriptorTypes) }
|
||||||
|
|
||||||
|
var fileDescriptorTypes = []byte{
|
||||||
|
- // 345 bytes of a gzipped FileDescriptorProto
|
||||||
|
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x91, 0xd1, 0x6a, 0xfa, 0x30,
|
||||||
|
- 0x14, 0xc6, 0xff, 0xb1, 0xb6, 0xda, 0xe3, 0xdf, 0xad, 0x84, 0x4d, 0xc2, 0x06, 0x52, 0xbc, 0x59,
|
||||||
|
- 0xd9, 0x85, 0x03, 0x37, 0x76, 0xbf, 0x5d, 0x08, 0xde, 0x49, 0x5e, 0x60, 0xc4, 0x36, 0x3a, 0xd1,
|
||||||
|
- 0xda, 0xd2, 0x44, 0x8b, 0xec, 0x05, 0x77, 0xb9, 0x47, 0x10, 0x9f, 0x64, 0xe4, 0x24, 0x8a, 0xbb,
|
||||||
|
- 0xd1, 0xef, 0x97, 0x93, 0xd3, 0xef, 0x7c, 0x27, 0x70, 0x5b, 0xae, 0x16, 0x4f, 0x7a, 0x5f, 0x4a,
|
||||||
|
- 0x65, 0x7f, 0x87, 0x65, 0x55, 0xe8, 0x82, 0xfa, 0x08, 0x83, 0x19, 0x84, 0x93, 0xe9, 0x5b, 0x96,
|
||||||
|
- 0x55, 0x52, 0x29, 0xfa, 0x00, 0xc1, 0x5c, 0xe4, 0xcb, 0xf5, 0x9e, 0x91, 0x98, 0x24, 0x57, 0xa3,
|
||||||
|
- 0xeb, 0xa1, 0xed, 0x98, 0x4c, 0xc7, 0x78, 0xcc, 0x5d, 0x99, 0x32, 0x68, 0x09, 0xdb, 0xc3, 0x1a,
|
||||||
|
- 0x31, 0x49, 0x42, 0x7e, 0x42, 0x4a, 0xa1, 0x99, 0x0b, 0xb5, 0x62, 0x1e, 0x1e, 0xa3, 0x1e, 0x1c,
|
||||||
|
- 0x08, 0x84, 0x93, 0x8d, 0x96, 0xd5, 0x5c, 0xa4, 0x92, 0xf6, 0x20, 0xc8, 0xe4, 0x6e, 0x99, 0x4a,
|
||||||
|
- 0x34, 0x09, 0xb9, 0x23, 0xd3, 0xb9, 0x11, 0xb9, 0x74, 0x1f, 0x44, 0x4d, 0x47, 0xd0, 0x39, 0x4f,
|
||||||
|
- 0x27, 0x15, 0xf3, 0x62, 0x2f, 0xe9, 0x8c, 0xa2, 0xf3, 0x54, 0xae, 0xc2, 0x2f, 0x2f, 0xd1, 0x08,
|
||||||
|
- 0xbc, 0x5c, 0x6f, 0x59, 0x33, 0x26, 0x49, 0x93, 0x1b, 0x69, 0x1c, 0x3f, 0x6b, 0x73, 0x81, 0xf9,
|
||||||
|
- 0xd6, 0xd1, 0x92, 0x49, 0x51, 0xa6, 0x4b, 0x2c, 0x04, 0x36, 0x85, 0x43, 0x33, 0x8b, 0xf1, 0x60,
|
||||||
|
- 0x2d, 0x3b, 0x8b, 0xd1, 0xf4, 0x1e, 0xc2, 0x4a, 0xd4, 0x1f, 0xf3, 0xb5, 0x58, 0x28, 0xd6, 0x8e,
|
||||||
|
- 0x49, 0xd2, 0xe5, 0xed, 0x4a, 0xd4, 0x63, 0xc3, 0x83, 0x2f, 0xf0, 0x79, 0xb1, 0xd5, 0x98, 0x22,
|
||||||
|
- 0x93, 0x4a, 0xbb, 0x6c, 0xa8, 0x8d, 0xcf, 0x42, 0x68, 0x59, 0x8b, 0xfd, 0x69, 0x5b, 0x0e, 0x2f,
|
||||||
|
- 0x76, 0xe1, 0xfd, 0xd9, 0x45, 0x0f, 0x02, 0x55, 0x6c, 0xab, 0x54, 0x62, 0x8c, 0x90, 0x3b, 0xa2,
|
||||||
|
- 0x37, 0xe0, 0xab, 0xb4, 0x28, 0x25, 0x06, 0xe9, 0x72, 0x0b, 0x8f, 0x77, 0xd0, 0x3e, 0xbd, 0x10,
|
||||||
|
- 0x0d, 0xa0, 0xb1, 0x7b, 0x89, 0xfe, 0xe1, 0xff, 0x6b, 0x44, 0xde, 0xff, 0x7f, 0x1f, 0xfb, 0xe4,
|
||||||
|
- 0xe7, 0xd8, 0x27, 0x87, 0x63, 0x9f, 0xcc, 0x02, 0x7c, 0xfb, 0xe7, 0xdf, 0x00, 0x00, 0x00, 0xff,
|
||||||
|
- 0xff, 0xb5, 0x52, 0x37, 0xf2, 0x14, 0x02, 0x00, 0x00,
|
||||||
|
+ // 356 bytes of a gzipped FileDescriptorProto
|
||||||
|
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x92, 0xc1, 0x4e, 0xc2, 0x40,
|
||||||
|
+ 0x10, 0x86, 0x5d, 0x4a, 0x0b, 0x1d, 0x44, 0x9b, 0x8d, 0x92, 0x8d, 0x26, 0xa4, 0xe1, 0x62, 0xe3,
|
||||||
|
+ 0x01, 0x13, 0x34, 0xde, 0xf5, 0x40, 0xc2, 0x0d, 0xf7, 0x05, 0xcc, 0xd2, 0x2e, 0x48, 0xa0, 0xb4,
|
||||||
|
+ 0xe9, 0xb6, 0x34, 0xc4, 0x17, 0xf4, 0xe8, 0x23, 0x18, 0x9e, 0xc2, 0xa3, 0xd9, 0xd9, 0x85, 0xe0,
|
||||||
|
+ 0x05, 0xfe, 0x6f, 0x67, 0xa7, 0xff, 0xfc, 0xd3, 0xc2, 0x75, 0xbe, 0x5a, 0x3c, 0x94, 0xbb, 0x5c,
|
||||||
|
+ 0x2a, 0xf3, 0x3b, 0xcc, 0x8b, 0xac, 0xcc, 0xa8, 0x8b, 0x30, 0x98, 0x81, 0x3f, 0x99, 0xbe, 0x24,
|
||||||
|
+ 0x49, 0x21, 0x95, 0xa2, 0x77, 0xe0, 0xcd, 0x45, 0xba, 0x5c, 0xef, 0x18, 0x09, 0x49, 0x74, 0x31,
|
||||||
|
+ 0xba, 0x1c, 0x9a, 0x8e, 0xc9, 0x74, 0x8c, 0xc7, 0xdc, 0x96, 0x29, 0x83, 0x96, 0x30, 0x3d, 0xac,
|
||||||
|
+ 0x11, 0x92, 0xc8, 0xe7, 0x07, 0xa4, 0x14, 0x9a, 0xa9, 0x50, 0x2b, 0xe6, 0xe0, 0x31, 0xea, 0xc1,
|
||||||
|
+ 0x2f, 0x01, 0x7f, 0xb2, 0x29, 0x65, 0x31, 0x17, 0xb1, 0xa4, 0x3d, 0xf0, 0x12, 0xb9, 0x5d, 0xc6,
|
||||||
|
+ 0x12, 0x4d, 0x7c, 0x6e, 0x49, 0x77, 0x6e, 0x44, 0x2a, 0xed, 0x03, 0x51, 0xd3, 0x11, 0x74, 0x8e,
|
||||||
|
+ 0xd3, 0x49, 0xc5, 0x9c, 0xd0, 0x89, 0x3a, 0xa3, 0xe0, 0x38, 0x95, 0xad, 0xf0, 0xd3, 0x4b, 0x34,
|
||||||
|
+ 0x00, 0x27, 0x2d, 0x2b, 0xd6, 0x0c, 0x49, 0xd4, 0xe4, 0x5a, 0x6a, 0xc7, 0x8f, 0x5a, 0x5f, 0x60,
|
||||||
|
+ 0xae, 0x71, 0x34, 0xa4, 0x53, 0xe4, 0xf1, 0x12, 0x0b, 0x9e, 0x49, 0x61, 0x51, 0xcf, 0xa2, 0x3d,
|
||||||
|
+ 0x58, 0xcb, 0xcc, 0xa2, 0x35, 0xbd, 0x05, 0xbf, 0x10, 0xf5, 0xfb, 0x7c, 0x2d, 0x16, 0x8a, 0xb5,
|
||||||
|
+ 0x43, 0x12, 0x75, 0x79, 0xbb, 0x10, 0xf5, 0x58, 0xb3, 0xb6, 0x78, 0xab, 0x64, 0x25, 0x15, 0xf3,
|
||||||
|
+ 0xb1, 0x62, 0x69, 0xf0, 0x09, 0x2e, 0xcf, 0xaa, 0x12, 0xd3, 0x25, 0x52, 0x95, 0x36, 0x33, 0x6a,
|
||||||
|
+ 0xed, 0xbf, 0x10, 0xa5, 0xac, 0xc5, 0xee, 0xb0, 0x45, 0x8b, 0x27, 0x3b, 0x72, 0xfe, 0xed, 0xa8,
|
||||||
|
+ 0x07, 0x9e, 0xca, 0xaa, 0x22, 0x96, 0x18, 0xcf, 0xe7, 0x96, 0xe8, 0x15, 0xb8, 0x2a, 0xce, 0x72,
|
||||||
|
+ 0x89, 0x01, 0xbb, 0xdc, 0xc0, 0xfd, 0x0d, 0xb4, 0x0f, 0x6f, 0x8e, 0x7a, 0xd0, 0xd8, 0x3e, 0x05,
|
||||||
|
+ 0x67, 0xf8, 0xff, 0x1c, 0x90, 0xd7, 0xf3, 0xaf, 0x7d, 0x9f, 0x7c, 0xef, 0xfb, 0xe4, 0x67, 0xdf,
|
||||||
|
+ 0x27, 0x33, 0x0f, 0xbf, 0x89, 0xc7, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa5, 0x57, 0x70, 0xc8,
|
||||||
|
+ 0x2c, 0x02, 0x00, 0x00,
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/bridgedmacvlan_endpoint.go b/virtcontainers/bridgedmacvlan_endpoint.go
|
||||||
|
index 700aea34..9179bbbe 100644
|
||||||
|
--- a/virtcontainers/bridgedmacvlan_endpoint.go
|
||||||
|
+++ b/virtcontainers/bridgedmacvlan_endpoint.go
|
||||||
|
@@ -25,7 +25,7 @@ func createBridgedMacvlanNetworkEndpoint(idx int, ifName string, interworkingMod
|
||||||
|
return &BridgedMacvlanEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx)
|
||||||
|
}
|
||||||
|
|
||||||
|
- netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel)
|
||||||
|
+ netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel, 0)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/ipvlan_endpoint.go b/virtcontainers/ipvlan_endpoint.go
|
||||||
|
index 6e924a74..a6ef8179 100644
|
||||||
|
--- a/virtcontainers/ipvlan_endpoint.go
|
||||||
|
+++ b/virtcontainers/ipvlan_endpoint.go
|
||||||
|
@@ -28,7 +28,7 @@ func createIPVlanNetworkEndpoint(idx int, ifName string) (*IPVlanEndpoint, error
|
||||||
|
// Use tc filtering for ipvlan, since the other inter networking models will
|
||||||
|
// not work for ipvlan.
|
||||||
|
interworkingModel := NetXConnectTCFilterModel
|
||||||
|
- netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel)
|
||||||
|
+ netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel, 0)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
|
||||||
|
index d28d8cce..38e9a204 100644
|
||||||
|
--- a/virtcontainers/kata_agent.go
|
||||||
|
+++ b/virtcontainers/kata_agent.go
|
||||||
|
@@ -621,6 +621,7 @@ func (k *kataAgent) updateInterface(ifc *vcTypes.Interface) (*vcTypes.Interface,
|
||||||
|
Mtu: resultInterface.Mtu,
|
||||||
|
HwAddr: resultInterface.HwAddr,
|
||||||
|
PciAddr: resultInterface.PciAddr,
|
||||||
|
+ Queues: resultInterface.Queues,
|
||||||
|
}
|
||||||
|
return iface, err
|
||||||
|
}
|
||||||
|
@@ -2295,6 +2296,7 @@ func (k *kataAgent) convertToKataAgentInterface(iface *vcTypes.Interface) *aType
|
||||||
|
RawFlags: iface.RawFlags,
|
||||||
|
HwAddr: iface.HwAddr,
|
||||||
|
PciAddr: iface.PciAddr,
|
||||||
|
+ Queues: iface.Queues,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
|
||||||
|
index 46703656..15eb7906 100644
|
||||||
|
--- a/virtcontainers/network.go
|
||||||
|
+++ b/virtcontainers/network.go
|
||||||
|
@@ -140,6 +140,7 @@ type NetworkInfo struct {
|
||||||
|
Routes []netlink.Route
|
||||||
|
DNS DNSInfo
|
||||||
|
VhostUserSocket string
|
||||||
|
+ Queues uint32
|
||||||
|
}
|
||||||
|
|
||||||
|
// NetworkInterface defines a network interface.
|
||||||
|
@@ -156,6 +157,7 @@ type TapInterface struct {
|
||||||
|
TAPIface NetworkInterface
|
||||||
|
VMFds []*os.File
|
||||||
|
VhostFds []*os.File
|
||||||
|
+ Queues uint32
|
||||||
|
}
|
||||||
|
|
||||||
|
// TuntapInterface defines a tap interface
|
||||||
|
@@ -1047,7 +1049,7 @@ func generateInterfacesAndRoutes(networkNS NetworkNamespace) ([]*vcTypes.Interfa
|
||||||
|
return ifaces, routes, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
-func createNetworkInterfacePair(idx int, ifName string, interworkingModel NetInterworkingModel) (NetworkInterfacePair, error) {
|
||||||
|
+func createNetworkInterfacePair(idx int, ifName string, interworkingModel NetInterworkingModel, queues uint32) (NetworkInterfacePair, error) {
|
||||||
|
uniqueID := uuid.Generate().String()
|
||||||
|
|
||||||
|
randomMacAddr, err := generateRandomPrivateMacAddr()
|
||||||
|
@@ -1062,6 +1064,7 @@ func createNetworkInterfacePair(idx int, ifName string, interworkingModel NetInt
|
||||||
|
TAPIface: NetworkInterface{
|
||||||
|
Name: fmt.Sprintf("tap%d_kata", idx),
|
||||||
|
},
|
||||||
|
+ Queues: queues,
|
||||||
|
},
|
||||||
|
VirtIface: NetworkInterface{
|
||||||
|
Name: fmt.Sprintf("eth%d", idx),
|
||||||
|
@@ -1205,7 +1208,7 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li
|
||||||
|
// interface info json file
|
||||||
|
if netInfo.VhostUserSocket != "" {
|
||||||
|
networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found")
|
||||||
|
- endpoint, err = createVhostUserEndpoint(netInfo, netInfo.VhostUserSocket)
|
||||||
|
+ endpoint, err = createVhostUserEndpoint(netInfo, netInfo.VhostUserSocket, netInfo.Queues)
|
||||||
|
return endpoint, err
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -1220,7 +1223,7 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li
|
||||||
|
endpoint, err = createMacvtapNetworkEndpoint(netInfo)
|
||||||
|
case "tap":
|
||||||
|
networkLogger().Info("tap interface found")
|
||||||
|
- endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Device)
|
||||||
|
+ endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Device, netInfo.Queues)
|
||||||
|
case "tuntap":
|
||||||
|
if link != nil {
|
||||||
|
switch link.(*netlink.Tuntap).Mode {
|
||||||
|
@@ -1231,13 +1234,13 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li
|
||||||
|
return nil, fmt.Errorf("tun networking device not yet supported")
|
||||||
|
case 2:
|
||||||
|
networkLogger().Info("tuntap tap interface found")
|
||||||
|
- endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, model)
|
||||||
|
+ endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, model, netInfo.Queues)
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("tuntap network %v mode unsupported", link.(*netlink.Tuntap).Mode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "veth":
|
||||||
|
- endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model)
|
||||||
|
+ endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model, netInfo.Queues)
|
||||||
|
case "ipvlan":
|
||||||
|
endpoint, err = createIPVlanNetworkEndpoint(idx, netInfo.Iface.Name)
|
||||||
|
default:
|
||||||
|
diff --git a/virtcontainers/network_test.go b/virtcontainers/network_test.go
|
||||||
|
index b86f679f..c52c7452 100644
|
||||||
|
--- a/virtcontainers/network_test.go
|
||||||
|
+++ b/virtcontainers/network_test.go
|
||||||
|
@@ -265,7 +265,7 @@ func TestTcRedirectNetwork(t *testing.T) {
|
||||||
|
err = netlink.LinkAdd(veth)
|
||||||
|
assert.NoError(err)
|
||||||
|
|
||||||
|
- endpoint, err := createVethNetworkEndpoint(1, vethName, NetXConnectTCFilterModel)
|
||||||
|
+ endpoint, err := createVethNetworkEndpoint(1, vethName, NetXConnectTCFilterModel, 0)
|
||||||
|
assert.NoError(err)
|
||||||
|
|
||||||
|
link, err := netlink.LinkByName(vethName)
|
||||||
|
diff --git a/virtcontainers/pkg/types/types.go b/virtcontainers/pkg/types/types.go
|
||||||
|
index dccd92f8..6502259b 100644
|
||||||
|
--- a/virtcontainers/pkg/types/types.go
|
||||||
|
+++ b/virtcontainers/pkg/types/types.go
|
||||||
|
@@ -31,6 +31,8 @@ type Interface struct {
|
||||||
|
LinkType string `json:"linkType,omitempty"`
|
||||||
|
// VhostUserSocket is DPDK-backed vHost user ports.
|
||||||
|
VhostUserSocket string `json:"vhostUserSocket,omitempty"`
|
||||||
|
+ // Queues specifies the interface queue number
|
||||||
|
+ Queues uint32 `json:"queues,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Route describes a network route.
|
||||||
|
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
|
||||||
|
index 84797e0d..be6e33b9 100644
|
||||||
|
--- a/virtcontainers/qemu.go
|
||||||
|
+++ b/virtcontainers/qemu.go
|
||||||
|
@@ -1464,7 +1464,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
-func (q *qemu) hotAddNetDevice(deviceName, name, hardAddr string, VMFds, VhostFds []*os.File) error {
|
||||||
|
+func (q *qemu) hotAddNetDevice(deviceName, name, hardAddr string, VMFds, VhostFds []*os.File, queues uint32) error {
|
||||||
|
var (
|
||||||
|
VMFdNames []string
|
||||||
|
VhostFdNames []string
|
||||||
|
@@ -1489,7 +1489,7 @@ func (q *qemu) hotAddNetDevice(deviceName, name, hardAddr string, VMFds, VhostFd
|
||||||
|
return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", name, VMFdNames, VhostFdNames)
|
||||||
|
}
|
||||||
|
|
||||||
|
- return q.qmpMonitorCh.qmp.ExecuteNetdevAdd(q.qmpMonitorCh.ctx, "tap", name, deviceName, "no", "no", 0)
|
||||||
|
+ return q.qmpMonitorCh.qmp.ExecuteNetdevAdd(q.qmpMonitorCh.ctx, "tap", name, deviceName, "no", "no", int(queues))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
|
||||||
|
@@ -1512,7 +1512,7 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
|
||||||
|
|
||||||
|
devID := "virtio-" + tap.ID
|
||||||
|
if op == addDevice {
|
||||||
|
- if err = q.hotAddNetDevice(tap.TAPIface.Name, tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds); err != nil {
|
||||||
|
+ if err = q.hotAddNetDevice(tap.TAPIface.Name, tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds, tap.Queues); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -1542,9 +1542,9 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
|
||||||
|
}
|
||||||
|
if machine.Type == QemuCCWVirtio {
|
||||||
|
devNoHotplug := fmt.Sprintf("fe.%x.%x", bus, addr)
|
||||||
|
- return q.qmpMonitorCh.qmp.ExecuteNetCCWDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), devNoHotplug, int(q.config.NumVCPUs))
|
||||||
|
+ return q.qmpMonitorCh.qmp.ExecuteNetCCWDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), devNoHotplug, int(tap.Queues))
|
||||||
|
}
|
||||||
|
- return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bus, romFile, int(q.config.NumVCPUs), defaultDisableModern)
|
||||||
|
+ return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bus, romFile, int(tap.Queues), defaultDisableModern)
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index e6f155a3..d8ab6c1a 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -948,6 +948,7 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) {
|
||||||
|
},
|
||||||
|
Addrs: addrs,
|
||||||
|
VhostUserSocket: inf.VhostUserSocket,
|
||||||
|
+ Queues: inf.Queues,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go
|
||||||
|
index 2cf70dce..5a3e7f7e 100644
|
||||||
|
--- a/virtcontainers/tap_endpoint.go
|
||||||
|
+++ b/virtcontainers/tap_endpoint.go
|
||||||
|
@@ -115,7 +115,7 @@ func (endpoint *TapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPat
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
-func createTapNetworkEndpoint(idx int, ifName string, tapIfName string) (*TapEndpoint, error) {
|
||||||
|
+func createTapNetworkEndpoint(idx int, ifName string, tapIfName string, queues uint32) (*TapEndpoint, error) {
|
||||||
|
if idx < 0 {
|
||||||
|
return &TapEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx)
|
||||||
|
}
|
||||||
|
@@ -139,6 +139,10 @@ func createTapNetworkEndpoint(idx int, ifName string, tapIfName string) (*TapEnd
|
||||||
|
endpoint.TapInterface.TAPIface.Name = tapIfName
|
||||||
|
}
|
||||||
|
|
||||||
|
+ if queues > 0 {
|
||||||
|
+ endpoint.TapInterface.Queues = queues
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
return endpoint, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/tuntap_endpoint.go b/virtcontainers/tuntap_endpoint.go
|
||||||
|
index b076d694..827d8852 100644
|
||||||
|
--- a/virtcontainers/tuntap_endpoint.go
|
||||||
|
+++ b/virtcontainers/tuntap_endpoint.go
|
||||||
|
@@ -117,12 +117,12 @@ func (endpoint *TuntapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNs
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
-func createTuntapNetworkEndpoint(idx int, ifName string, hwName net.HardwareAddr, internetworkingModel NetInterworkingModel) (*TuntapEndpoint, error) {
|
||||||
|
+func createTuntapNetworkEndpoint(idx int, ifName string, hwName net.HardwareAddr, internetworkingModel NetInterworkingModel, queues uint32) (*TuntapEndpoint, error) {
|
||||||
|
if idx < 0 {
|
||||||
|
return &TuntapEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx)
|
||||||
|
}
|
||||||
|
|
||||||
|
- netPair, err := createNetworkInterfacePair(idx, ifName, internetworkingModel)
|
||||||
|
+ netPair, err := createNetworkInterfacePair(idx, ifName, internetworkingModel, queues)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/veth_endpoint.go b/virtcontainers/veth_endpoint.go
|
||||||
|
index 0f2ec9ba..554b9b22 100644
|
||||||
|
--- a/virtcontainers/veth_endpoint.go
|
||||||
|
+++ b/virtcontainers/veth_endpoint.go
|
||||||
|
@@ -20,12 +20,12 @@ type VethEndpoint struct {
|
||||||
|
PCIAddr string
|
||||||
|
}
|
||||||
|
|
||||||
|
-func createVethNetworkEndpoint(idx int, ifName string, interworkingModel NetInterworkingModel) (*VethEndpoint, error) {
|
||||||
|
+func createVethNetworkEndpoint(idx int, ifName string, interworkingModel NetInterworkingModel, queues uint32) (*VethEndpoint, error) {
|
||||||
|
if idx < 0 {
|
||||||
|
return &VethEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx)
|
||||||
|
}
|
||||||
|
|
||||||
|
- netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel)
|
||||||
|
+ netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel, queues)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/veth_endpoint_test.go b/virtcontainers/veth_endpoint_test.go
|
||||||
|
index 9649b82e..23f1876d 100644
|
||||||
|
--- a/virtcontainers/veth_endpoint_test.go
|
||||||
|
+++ b/virtcontainers/veth_endpoint_test.go
|
||||||
|
@@ -34,7 +34,7 @@ func TestCreateVethNetworkEndpoint(t *testing.T) {
|
||||||
|
EndpointType: VethEndpointType,
|
||||||
|
}
|
||||||
|
|
||||||
|
- result, err := createVethNetworkEndpoint(4, "", DefaultNetInterworkingModel)
|
||||||
|
+ result, err := createVethNetworkEndpoint(4, "", DefaultNetInterworkingModel, 0)
|
||||||
|
assert.NoError(err)
|
||||||
|
|
||||||
|
// the resulting ID will be random - so let's overwrite to test the rest of the flow
|
||||||
|
@@ -68,7 +68,7 @@ func TestCreateVethNetworkEndpointChooseIfaceName(t *testing.T) {
|
||||||
|
EndpointType: VethEndpointType,
|
||||||
|
}
|
||||||
|
|
||||||
|
- result, err := createVethNetworkEndpoint(4, "eth1", DefaultNetInterworkingModel)
|
||||||
|
+ result, err := createVethNetworkEndpoint(4, "eth1", DefaultNetInterworkingModel, 0)
|
||||||
|
assert.NoError(err)
|
||||||
|
|
||||||
|
// the resulting ID will be random - so let's overwrite to test the rest of the flow
|
||||||
|
@@ -95,7 +95,7 @@ func TestCreateVethNetworkEndpointInvalidArgs(t *testing.T) {
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, d := range failingValues {
|
||||||
|
- _, err := createVethNetworkEndpoint(d.idx, d.ifName, DefaultNetInterworkingModel)
|
||||||
|
+ _, err := createVethNetworkEndpoint(d.idx, d.ifName, DefaultNetInterworkingModel, 0)
|
||||||
|
assert.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/vhostuser_endpoint.go b/virtcontainers/vhostuser_endpoint.go
|
||||||
|
index 2fc3d837..85ef67b4 100644
|
||||||
|
--- a/virtcontainers/vhostuser_endpoint.go
|
||||||
|
+++ b/virtcontainers/vhostuser_endpoint.go
|
||||||
|
@@ -33,6 +33,7 @@ type VhostUserEndpoint struct {
|
||||||
|
EndpointProperties NetworkInfo
|
||||||
|
EndpointType EndpointType
|
||||||
|
PCIAddr string
|
||||||
|
+ Queues uint32
|
||||||
|
}
|
||||||
|
|
||||||
|
// Properties returns the properties of the interface.
|
||||||
|
@@ -119,7 +120,7 @@ func (endpoint *VhostUserEndpoint) HotDetach(h hypervisor, netNsCreated bool, ne
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create a vhostuser endpoint
|
||||||
|
-func createVhostUserEndpoint(netInfo NetworkInfo, socket string) (*VhostUserEndpoint, error) {
|
||||||
|
+func createVhostUserEndpoint(netInfo NetworkInfo, socket string, queues uint32) (*VhostUserEndpoint, error) {
|
||||||
|
uniqueID := uuid.Generate().String()
|
||||||
|
vhostUserEndpoint := &VhostUserEndpoint{
|
||||||
|
ID: uniqueID,
|
||||||
|
@@ -127,6 +128,7 @@ func createVhostUserEndpoint(netInfo NetworkInfo, socket string) (*VhostUserEndp
|
||||||
|
HardAddr: netInfo.Iface.HardwareAddr.String(),
|
||||||
|
IfaceName: netInfo.Iface.Name,
|
||||||
|
EndpointType: VhostUserEndpointType,
|
||||||
|
+ Queues: queues,
|
||||||
|
}
|
||||||
|
return vhostUserEndpoint, nil
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/vhostuser_endpoint_test.go b/virtcontainers/vhostuser_endpoint_test.go
|
||||||
|
index 584490cc..046ca229 100644
|
||||||
|
--- a/virtcontainers/vhostuser_endpoint_test.go
|
||||||
|
+++ b/virtcontainers/vhostuser_endpoint_test.go
|
||||||
|
@@ -137,7 +137,7 @@ func TestCreateVhostUserEndpoint(t *testing.T) {
|
||||||
|
EndpointType: VhostUserEndpointType,
|
||||||
|
}
|
||||||
|
|
||||||
|
- result, err := createVhostUserEndpoint(netinfo, socket)
|
||||||
|
+ result, err := createVhostUserEndpoint(netinfo, socket, 0)
|
||||||
|
assert.NoError(err)
|
||||||
|
assert.Exactly(result, expected)
|
||||||
|
}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,34 @@
|
|||||||
|
From d3b8e21829bd671a1717fed8ab645ad2d447899a Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Wed, 19 Aug 2020 22:18:05 +0800
|
||||||
|
Subject: [PATCH 48/50] console: fix the file resource leak
|
||||||
|
|
||||||
|
reason: newConsole will open a file, if error occurs in the
|
||||||
|
middle proccess, the file resource may leak, use defer close
|
||||||
|
to prevent it
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
cli/console.go | 6 ++++++
|
||||||
|
1 file changed, 6 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/cli/console.go b/cli/console.go
|
||||||
|
index c1555c9f..97dfee99 100644
|
||||||
|
--- a/cli/console.go
|
||||||
|
+++ b/cli/console.go
|
||||||
|
@@ -47,6 +47,12 @@ func newConsole() (*Console, error) {
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
+ defer func() {
|
||||||
|
+ if err != nil {
|
||||||
|
+ master.Close()
|
||||||
|
+ }
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
if err := saneTerminal(master); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,58 @@
|
|||||||
|
From a6fab7014922d85b1105b44fdbb98239b22d3e00 Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Wed, 19 Aug 2020 22:31:57 +0800
|
||||||
|
Subject: [PATCH 49/50] container: fix the write operation transparently
|
||||||
|
transmitted to the host
|
||||||
|
|
||||||
|
reason:fix the write operation transparently transmitted to the host
|
||||||
|
when we fullfill the "/etc/hosts","/etc/resolv.conf","/etc/hostname" file in the container,
|
||||||
|
for example:
|
||||||
|
```bash
|
||||||
|
$ docker exec -ti 63 bash
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/container.go | 16 ++++++++++++++--
|
||||||
|
1 file changed, 14 insertions(+), 2 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/container.go b/virtcontainers/container.go
|
||||||
|
index 1b89f6ac..6edcb3f2 100644
|
||||||
|
--- a/virtcontainers/container.go
|
||||||
|
+++ b/virtcontainers/container.go
|
||||||
|
@@ -51,6 +51,12 @@ var cdromMajors = map[int64]string{
|
||||||
|
32: "CM206_CDROM_MAJOR",
|
||||||
|
}
|
||||||
|
|
||||||
|
+var safeCopyFiles = map[string]struct{}{
|
||||||
|
+ "resolv.conf": {},
|
||||||
|
+ "hostname": {},
|
||||||
|
+ "hosts": {},
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// https://github.com/torvalds/linux/blob/master/include/uapi/linux/major.h
|
||||||
|
// #define FLOPPY_MAJOR 2
|
||||||
|
const floppyMajor = int64(2)
|
||||||
|
@@ -452,12 +458,18 @@ func (c *Container) shareFiles(m Mount, idx int, hostSharedDir, guestSharedDir s
|
||||||
|
}
|
||||||
|
|
||||||
|
filename := fmt.Sprintf("%s-%s-%s", c.id, hex.EncodeToString(randBytes), filepath.Base(m.Destination))
|
||||||
|
- guestDest := filepath.Join(guestSharedDir, filename)
|
||||||
|
+ var guestDest string
|
||||||
|
+ _, needCopy := safeCopyFiles[filepath.Base(m.Destination)]
|
||||||
|
+ if needCopy {
|
||||||
|
+ guestDest = filepath.Join(kataGuestStorageDir, filename)
|
||||||
|
+ } else {
|
||||||
|
+ guestDest = filepath.Join(guestSharedDir, filename)
|
||||||
|
+ }
|
||||||
|
|
||||||
|
// copy file to contaier's rootfs if filesystem sharing is not supported, otherwise
|
||||||
|
// bind mount it in the shared directory.
|
||||||
|
caps := c.sandbox.hypervisor.capabilities()
|
||||||
|
- if !caps.IsFsSharingSupported() {
|
||||||
|
+ if !caps.IsFsSharingSupported() || needCopy {
|
||||||
|
c.Logger().Debug("filesystem sharing is not supported, files will be copied")
|
||||||
|
|
||||||
|
fileInfo, err := os.Stat(m.Source)
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,358 @@
|
|||||||
|
From f943e5d91f6922c599b46727e2dfd5e8e7e5bea8 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Wed, 19 Aug 2020 22:43:54 +0800
|
||||||
|
Subject: [PATCH 50/50] runtime: add kata-network upate-iface subcommand
|
||||||
|
|
||||||
|
reason: add kata-network update-iface subcommand to
|
||||||
|
update the exist interface in the Kata VM.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
cli/network.go | 37 +++++++++-
|
||||||
|
virtcontainers/api.go | 25 +++++--
|
||||||
|
virtcontainers/implementation.go | 5 ++
|
||||||
|
virtcontainers/interfaces.go | 1 +
|
||||||
|
virtcontainers/pkg/vcmock/mock.go | 8 +++
|
||||||
|
virtcontainers/pkg/vcmock/types.go | 1 +
|
||||||
|
virtcontainers/sandbox.go | 137 +++++++++++++++++++++++++++++++++++++
|
||||||
|
7 files changed, 207 insertions(+), 7 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/cli/network.go b/cli/network.go
|
||||||
|
index 3dd0971e..7dce0528 100644
|
||||||
|
--- a/cli/network.go
|
||||||
|
+++ b/cli/network.go
|
||||||
|
@@ -65,8 +65,9 @@ var addIfaceCommand = cli.Command{
|
||||||
|
"hwAddr":"<mac>",
|
||||||
|
"linkType":"tap",
|
||||||
|
"vhostUserSocket":"<path>"
|
||||||
|
+ "queues":<queues>
|
||||||
|
}
|
||||||
|
- device,name,mtu,hwAddr are required, IPAddresses and vhostUserSocket are optional.
|
||||||
|
+ device,name,mtu,hwAddr are required, IPAddresses, queues and vhostUserSocket are optional.
|
||||||
|
`,
|
||||||
|
Flags: []cli.Flag{},
|
||||||
|
Action: func(context *cli.Context) error {
|
||||||
|
@@ -104,6 +105,34 @@ var delIfaceCommand = cli.Command{
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
+var updateIfaceCommand = cli.Command{
|
||||||
|
+ Name: "update-iface",
|
||||||
|
+ Usage: "update the interface in container",
|
||||||
|
+ ArgsUsage: `update-iface <container-id> file or - for stdin
|
||||||
|
+ file or stdin for example:
|
||||||
|
+ {
|
||||||
|
+ "device":"",
|
||||||
|
+ "name":"<interface-name>",
|
||||||
|
+ "IPAddresses":[{"address":"<ip>","mask":"<mask>"}],
|
||||||
|
+ "mtu":0,
|
||||||
|
+ "hwAddr":"",
|
||||||
|
+ "linkType":"",
|
||||||
|
+ "vhostUserSocket":"",
|
||||||
|
+ "queues":0
|
||||||
|
+ }
|
||||||
|
+ name is required,IPAddresses is optional,device,mtu,hwAddr,linkType,queues and vhostUserSocket should be 0 or empty.
|
||||||
|
+ `,
|
||||||
|
+ Flags: []cli.Flag{},
|
||||||
|
+ Action: func(context *cli.Context) error {
|
||||||
|
+ ctx, err := cliContextToContext(context)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return networkModifyCommand(ctx, context.Args().First(), context.Args().Get(1), interfaceType, vcTypes.NetworkOpUpdate)
|
||||||
|
+ },
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
var listIfacesCommand = cli.Command{
|
||||||
|
Name: "list-ifaces",
|
||||||
|
Usage: "list network interfaces in a container",
|
||||||
|
@@ -262,6 +291,12 @@ func networkModifyCommand(ctx context.Context, containerID, input string, opType
|
||||||
|
kataLog.WithField("resulting-interface", fmt.Sprintf("%+v", resultingInf)).
|
||||||
|
WithError(err).Error("delete interface failed")
|
||||||
|
}
|
||||||
|
+ case vcTypes.NetworkOpUpdate:
|
||||||
|
+ resultingInf, err = vci.UpdateInterface(ctx, sandboxID, inf)
|
||||||
|
+ if err != nil {
|
||||||
|
+ kataLog.WithField("resulting-interface", fmt.Sprintf("%+v", resultingInf)).
|
||||||
|
+ WithError(err).Error("update interface failed")
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
|
||||||
|
json.NewEncoder(output).Encode(resultingInf)
|
||||||
|
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
|
||||||
|
index 7036df8c..ca5412a9 100644
|
||||||
|
--- a/virtcontainers/api.go
|
||||||
|
+++ b/virtcontainers/api.go
|
||||||
|
@@ -889,7 +889,7 @@ func AddDevice(ctx context.Context, sandboxID string, info deviceConfig.DeviceIn
|
||||||
|
return s.AddDevice(info)
|
||||||
|
}
|
||||||
|
|
||||||
|
-func toggleInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface, add bool) (*vcTypes.Interface, error) {
|
||||||
|
+func toggleInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface, op vcTypes.NetworkOp) (*vcTypes.Interface, error) {
|
||||||
|
if sandboxID == "" {
|
||||||
|
return nil, vcTypes.ErrNeedSandboxID
|
||||||
|
}
|
||||||
|
@@ -906,11 +906,16 @@ func toggleInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interfa
|
||||||
|
}
|
||||||
|
defer s.releaseStatelessSandbox()
|
||||||
|
|
||||||
|
- if add {
|
||||||
|
+ switch op {
|
||||||
|
+ case vcTypes.NetworkOpAdd:
|
||||||
|
return s.AddInterface(inf)
|
||||||
|
+ case vcTypes.NetworkOpRemove:
|
||||||
|
+ return s.RemoveInterface(inf)
|
||||||
|
+ case vcTypes.NetworkOpUpdate:
|
||||||
|
+ return s.UpdateInterface(inf)
|
||||||
|
+ default:
|
||||||
|
+ return nil, fmt.Errorf("operation is not found")
|
||||||
|
}
|
||||||
|
-
|
||||||
|
- return s.RemoveInterface(inf)
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddInterface is the virtcontainers add interface entry point.
|
||||||
|
@@ -918,7 +923,7 @@ func AddInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface)
|
||||||
|
span, ctx := trace(ctx, "AddInterface")
|
||||||
|
defer span.Finish()
|
||||||
|
|
||||||
|
- return toggleInterface(ctx, sandboxID, inf, true)
|
||||||
|
+ return toggleInterface(ctx, sandboxID, inf, vcTypes.NetworkOpAdd)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RemoveInterface is the virtcontainers remove interface entry point.
|
||||||
|
@@ -926,7 +931,15 @@ func RemoveInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interfa
|
||||||
|
span, ctx := trace(ctx, "RemoveInterface")
|
||||||
|
defer span.Finish()
|
||||||
|
|
||||||
|
- return toggleInterface(ctx, sandboxID, inf, false)
|
||||||
|
+ return toggleInterface(ctx, sandboxID, inf, vcTypes.NetworkOpRemove)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// UpdateInterface updates interface entry point in the virtcontainers.
|
||||||
|
+func UpdateInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) {
|
||||||
|
+ span, ctx := trace(ctx, "UpdateInterface")
|
||||||
|
+ defer span.Finish()
|
||||||
|
+
|
||||||
|
+ return toggleInterface(ctx, sandboxID, inf, vcTypes.NetworkOpUpdate)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListInterfaces is the virtcontainers list interfaces entry point.
|
||||||
|
diff --git a/virtcontainers/implementation.go b/virtcontainers/implementation.go
|
||||||
|
index 0265d1ed..e4bc4ae6 100644
|
||||||
|
--- a/virtcontainers/implementation.go
|
||||||
|
+++ b/virtcontainers/implementation.go
|
||||||
|
@@ -163,6 +163,11 @@ func (impl *VCImpl) ListInterfaces(ctx context.Context, sandboxID string) ([]*vc
|
||||||
|
return ListInterfaces(ctx, sandboxID)
|
||||||
|
}
|
||||||
|
|
||||||
|
+// ListInterfaces implements the VC function of the same name.
|
||||||
|
+func (impl *VCImpl) UpdateInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) {
|
||||||
|
+ return UpdateInterface(ctx, sandboxID, inf)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// UpdateRoutes implements the VC function of the same name.
|
||||||
|
func (impl *VCImpl) UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) {
|
||||||
|
return UpdateRoutes(ctx, sandboxID, routes, op)
|
||||||
|
diff --git a/virtcontainers/interfaces.go b/virtcontainers/interfaces.go
|
||||||
|
index d8f0be69..0fd12d84 100644
|
||||||
|
--- a/virtcontainers/interfaces.go
|
||||||
|
+++ b/virtcontainers/interfaces.go
|
||||||
|
@@ -51,6 +51,7 @@ type VC interface {
|
||||||
|
|
||||||
|
AddInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error)
|
||||||
|
RemoveInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error)
|
||||||
|
+ UpdateInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error)
|
||||||
|
ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTypes.Interface, error)
|
||||||
|
UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error)
|
||||||
|
ListRoutes(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error)
|
||||||
|
diff --git a/virtcontainers/pkg/vcmock/mock.go b/virtcontainers/pkg/vcmock/mock.go
|
||||||
|
index aa9bae9a..d3bf201f 100644
|
||||||
|
--- a/virtcontainers/pkg/vcmock/mock.go
|
||||||
|
+++ b/virtcontainers/pkg/vcmock/mock.go
|
||||||
|
@@ -273,6 +273,14 @@ func (m *VCMock) ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTyp
|
||||||
|
return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID)
|
||||||
|
}
|
||||||
|
|
||||||
|
+// UpdateInterface implements the VC function of the same name
|
||||||
|
+func (m *VCMock) UpdateInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) {
|
||||||
|
+ if m.UpdateRoutesFunc != nil {
|
||||||
|
+ return m.UpdateInterfaceFunc(ctx, sandboxID, inf)
|
||||||
|
+ }
|
||||||
|
+ return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// UpdateRoutes implements the VC function of the same name.
|
||||||
|
func (m *VCMock) UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) {
|
||||||
|
if m.UpdateRoutesFunc != nil {
|
||||||
|
diff --git a/virtcontainers/pkg/vcmock/types.go b/virtcontainers/pkg/vcmock/types.go
|
||||||
|
index 610b4602..152e66b7 100644
|
||||||
|
--- a/virtcontainers/pkg/vcmock/types.go
|
||||||
|
+++ b/virtcontainers/pkg/vcmock/types.go
|
||||||
|
@@ -73,6 +73,7 @@ type VCMock struct {
|
||||||
|
AddInterfaceFunc func(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error)
|
||||||
|
RemoveInterfaceFunc func(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error)
|
||||||
|
ListInterfacesFunc func(ctx context.Context, sandboxID string) ([]*vcTypes.Interface, error)
|
||||||
|
+ UpdateInterfaceFunc func(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error)
|
||||||
|
UpdateRoutesFunc func(ctx context.Context, sandboxID string, routes []*vcTypes.Route) ([]*vcTypes.Route, error)
|
||||||
|
ListRoutesFunc func(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error)
|
||||||
|
CleanupContainerFunc func(ctx context.Context, sandboxID, containerID string, force bool) error
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index d8ab6c1a..174e6cb6 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -1031,6 +1031,54 @@ func (s *Sandbox) RemoveInterface(inf *vcTypes.Interface) (*vcTypes.Interface, e
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+// UpdateInterface updates the nic.
|
||||||
|
+func (s *Sandbox) UpdateInterface(inf *vcTypes.Interface) (*vcTypes.Interface, error) {
|
||||||
|
+ err := checkUpdateInterfaceInfo(inf)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ nicInGuest := false
|
||||||
|
+
|
||||||
|
+ endpoint := s.getEndpointByName(inf.Name)
|
||||||
|
+ // nic is not in local endpoints
|
||||||
|
+ if endpoint == nil {
|
||||||
|
+ interfaceInGuestInf := s.getInterfaceInfByName(inf.Name)
|
||||||
|
+ if interfaceInGuestInf == nil {
|
||||||
|
+ return nil, fmt.Errorf("can not find the network interface")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ nicInGuest = true
|
||||||
|
+ endpoint, err = s.generateEndpointForStore(interfaceInGuestInf)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // update endpoint ip
|
||||||
|
+ err = s.updateEndpointIP(endpoint, inf)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ s.Logger().WithField("endpoint-type", endpoint.Type()).Info("Update endpoint")
|
||||||
|
+ grpcIf, err := s.agent.updateInterface(inf)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if nicInGuest {
|
||||||
|
+ s.networkNS.Endpoints = append(s.networkNS.Endpoints, endpoint)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err = s.Save(); err != nil {
|
||||||
|
+ s.Logger().WithError(err).Error("failed to store the network information")
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return grpcIf, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// ListInterfaces lists all nics and their configurations in the sandbox.
|
||||||
|
func (s *Sandbox) ListInterfaces() ([]*vcTypes.Interface, error) {
|
||||||
|
return s.agent.listInterfaces()
|
||||||
|
@@ -2431,6 +2479,95 @@ func (s *Sandbox) IsCompatOldCNI() bool {
|
||||||
|
return s.config.NetworkConfig.EnableCompatOldCNI
|
||||||
|
}
|
||||||
|
|
||||||
|
+func checkUpdateInterfaceInfo(inf *vcTypes.Interface) error {
|
||||||
|
+ if inf.Mtu > 0 || inf.Queues > 0 || inf.Device != "" || inf.VhostUserSocket != "" || inf.LinkType != "" || inf.HwAddr != "" {
|
||||||
|
+ return fmt.Errorf("device,mtu,hwAddr,linkType,queues and vhostUserSocket should be 0 or empty")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// getEndpointByName returns the endpoint by name.
|
||||||
|
+func (s *Sandbox) getEndpointByName(name string) Endpoint {
|
||||||
|
+ if name == "" {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ for i, endpoint := range s.networkNS.Endpoints {
|
||||||
|
+ if endpoint.Name() == name {
|
||||||
|
+ return s.networkNS.Endpoints[i]
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// getInterfaceInfByName returns the interface information.
|
||||||
|
+func (s *Sandbox) getInterfaceInfByName(name string) *vcTypes.Interface {
|
||||||
|
+ if name == "" {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ nics, err := s.agent.listInterfaces()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ for _, i := range nics {
|
||||||
|
+ if i.Name == name {
|
||||||
|
+ return i
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// generateEndpointForStore only generates endpoint information for store,it does not create a endpoint
|
||||||
|
+func (s *Sandbox) generateEndpointForStore(inf *vcTypes.Interface) (*TapEndpoint, error) {
|
||||||
|
+ netInfo, err := s.generateNetInfo(inf)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ endpoint := &TapEndpoint{
|
||||||
|
+ TapInterface: TapInterface{
|
||||||
|
+ Name: inf.Name,
|
||||||
|
+ TAPIface: NetworkInterface{
|
||||||
|
+ HardAddr: inf.HwAddr,
|
||||||
|
+ },
|
||||||
|
+ },
|
||||||
|
+ EndpointType: TapEndpointType,
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ endpoint.SetProperties(netInfo)
|
||||||
|
+
|
||||||
|
+ return endpoint, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *Sandbox) updateEndpointIP(endpoint Endpoint, inf *vcTypes.Interface) error {
|
||||||
|
+ if endpoint == nil {
|
||||||
|
+ return fmt.Errorf("endpoint is nil")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ netInfo := endpoint.Properties()
|
||||||
|
+
|
||||||
|
+ var addrs []netlink.Addr
|
||||||
|
+ for _, addr := range inf.IPAddresses {
|
||||||
|
+ netlinkAddrStr := fmt.Sprintf("%s/%s", addr.Address, addr.Mask)
|
||||||
|
+ netlinkAddr, err := netlink.ParseAddr(netlinkAddrStr)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return fmt.Errorf("could not parse %q: %v", netlinkAddrStr, err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ addrs = append(addrs, *netlinkAddr)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ netInfo.Addrs = addrs
|
||||||
|
+ endpoint.SetProperties(netInfo)
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// updateStaticSandboxResources update sandbox's cpu and memory resource passed by
|
||||||
|
// sandbox_cpu and sandbox_mem annotations
|
||||||
|
func updateStaticSandboxResources(sandboxConfig *SandboxConfig) error {
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,78 @@
|
|||||||
|
From be0d60f5fa88267afb26125681a217ef9476e133 Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20102011@163.com>
|
||||||
|
Date: Sat, 19 Sep 2020 12:47:14 +0800
|
||||||
|
Subject: [PATCH] runtime: fix del-iface doesn't delete the tap interface in
|
||||||
|
the host problem
|
||||||
|
|
||||||
|
reason: kata-runtime add-iface support add exist tap device in the host or
|
||||||
|
new created tap device by kata-network add-iface process. If add interface
|
||||||
|
is exist tap device, del-iface will not delete the tap device, so we need
|
||||||
|
to judge the deleted interface is exist device or new created, which is judged
|
||||||
|
by the VMFds of the tap Link and del-iface will load this info from the
|
||||||
|
network.json in 1.7.0 version. However, 1.11.1 version don't write the VMFds
|
||||||
|
into the persiste.json file, del-iface can not figure out which tap is exist
|
||||||
|
already or new created, so cause the this problem.
|
||||||
|
---
|
||||||
|
virtcontainers/endpoint.go | 2 ++
|
||||||
|
virtcontainers/persist/api/network.go | 5 +++++
|
||||||
|
virtcontainers/tap_endpoint.go | 1 +
|
||||||
|
3 files changed, 8 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/endpoint.go b/virtcontainers/endpoint.go
|
||||||
|
index 7efcf49c..3618792e 100644
|
||||||
|
--- a/virtcontainers/endpoint.go
|
||||||
|
+++ b/virtcontainers/endpoint.go
|
||||||
|
@@ -129,6 +129,7 @@ func saveTapIf(tapif *TapInterface) *persistapi.TapInterface {
|
||||||
|
HardAddr: tapif.TAPIface.HardAddr,
|
||||||
|
Addrs: tapif.TAPIface.Addrs,
|
||||||
|
},
|
||||||
|
+ VMFds: tapif.VMFds,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -167,6 +168,7 @@ func loadTapIf(tapif *persistapi.TapInterface) *TapInterface {
|
||||||
|
HardAddr: tapif.TAPIface.HardAddr,
|
||||||
|
Addrs: tapif.TAPIface.Addrs,
|
||||||
|
},
|
||||||
|
+ VMFds: tapif.VMFds,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/persist/api/network.go b/virtcontainers/persist/api/network.go
|
||||||
|
index 53c6de44..c5611767 100644
|
||||||
|
--- a/virtcontainers/persist/api/network.go
|
||||||
|
+++ b/virtcontainers/persist/api/network.go
|
||||||
|
@@ -7,6 +7,8 @@
|
||||||
|
package persistapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
+ "os"
|
||||||
|
+
|
||||||
|
"github.com/vishvananda/netlink"
|
||||||
|
)
|
||||||
|
|
||||||
|
@@ -45,6 +47,9 @@ type TapInterface struct {
|
||||||
|
Name string
|
||||||
|
TAPIface NetworkInterface
|
||||||
|
// remove VMFds and VhostFds
|
||||||
|
+ // add VMFds back to judge a tap interface is exist before add-iface
|
||||||
|
+ // or new created by kata-network add-iface
|
||||||
|
+ VMFds []*os.File
|
||||||
|
}
|
||||||
|
|
||||||
|
// TuntapInterface defines a tap interface
|
||||||
|
diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go
|
||||||
|
index 5a3e7f7e..0b6002aa 100644
|
||||||
|
--- a/virtcontainers/tap_endpoint.go
|
||||||
|
+++ b/virtcontainers/tap_endpoint.go
|
||||||
|
@@ -199,6 +199,7 @@ func unTapNetwork(endpoint *TapEndpoint) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+ networkLogger().Debug("untap the new created tap interface")
|
||||||
|
name := endpoint.TapInterface.TAPIface.Name
|
||||||
|
netHandle, err := netlink.NewHandle()
|
||||||
|
if err != nil {
|
||||||
|
--
|
||||||
|
2.11.0
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,199 @@
|
|||||||
|
From 31bbb64a0682e326b354ac54c6596402fc20a977 Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Tue, 8 Sep 2020 20:03:41 +0800
|
||||||
|
Subject: [PATCH 1/5] kata-runtime: add interface for host cgroup
|
||||||
|
|
||||||
|
reason: add interface for host cgroup, including
|
||||||
|
CreateSandboxCgroup, DestroySandboxCgroup and
|
||||||
|
AddPidToSandboxCgroup, GetSandboxCgroupPath
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/cgroups.go | 111 +++++++++++++++++++++++++++++++++++++++
|
||||||
|
virtcontainers/implementation.go | 20 +++++++
|
||||||
|
virtcontainers/interfaces.go | 5 ++
|
||||||
|
3 files changed, 136 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/cgroups.go b/virtcontainers/cgroups.go
|
||||||
|
index 4459df5..df0ec30 100644
|
||||||
|
--- a/virtcontainers/cgroups.go
|
||||||
|
+++ b/virtcontainers/cgroups.go
|
||||||
|
@@ -8,13 +8,20 @@ package virtcontainers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
+ "context"
|
||||||
|
+ "encoding/json"
|
||||||
|
"fmt"
|
||||||
|
+ "io/ioutil"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/containerd/cgroups"
|
||||||
|
specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
+ "github.com/sirupsen/logrus"
|
||||||
|
+
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/store"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
type cgroupPather interface {
|
||||||
|
@@ -30,6 +37,110 @@ const cgroupKataPath = "/kata/"
|
||||||
|
var cgroupsLoadFunc = cgroups.Load
|
||||||
|
var cgroupsNewFunc = cgroups.New
|
||||||
|
|
||||||
|
+// CreateSandboxCgroup create cgroup based on the first container's cgroupPath of sandbox
|
||||||
|
+func CreateSandboxCgroup(ctx context.Context, path string) error {
|
||||||
|
+ if path == "" {
|
||||||
|
+ return fmt.Errorf("sandbox cgroupPath shouldn't be empty!")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ var cgroupPath string
|
||||||
|
+ vcpuCgroupPath := filepath.Join(path, "vcpu")
|
||||||
|
+ if filepath.IsAbs(vcpuCgroupPath) {
|
||||||
|
+ cgroupPath = filepath.Clean(vcpuCgroupPath)
|
||||||
|
+ } else {
|
||||||
|
+ cgroupPath = filepath.Join(filepath.Clean("/" + vcpuCgroupPath))
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ resources := specs.LinuxResources{}
|
||||||
|
+ if _, err := cgroupsNewFunc(cgroups.V1, cgroups.StaticPath(cgroupPath), &resources); err != nil {
|
||||||
|
+ return fmt.Errorf("Could not create cgroup for %v: %v", cgroupPath, err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// DestroySandboxCgroup destroy the cgroup dir created for sandbox
|
||||||
|
+func DestroySandboxCgroup(ctx context.Context, cgroupPath string) error {
|
||||||
|
+ return deleteCgroup(cgroups.V1, cgroupPath)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func deleteCgroup(hierarchy cgroups.Hierarchy, cgroupPath string) error {
|
||||||
|
+ if cgroupPath == "" {
|
||||||
|
+ logrus.Warn("delete cgroupPath shouldn't be empty!")
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ cgroup, err := cgroupsLoadFunc(hierarchy,
|
||||||
|
+ cgroups.StaticPath(cgroupPath))
|
||||||
|
+
|
||||||
|
+ if err == cgroups.ErrCgroupDeleted {
|
||||||
|
+ // cgroup already deleted
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err != nil {
|
||||||
|
+ return fmt.Errorf("Could not load cgroup %v: %v", cgroupPath, err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // move running process here, that way cgroup can be removed
|
||||||
|
+ parent, err := parentCgroup(hierarchy, cgroupPath)
|
||||||
|
+ if err != nil {
|
||||||
|
+ // parent cgroup doesn't exist, that means there are no process running
|
||||||
|
+ // and the container cgroup was removed.
|
||||||
|
+ logrus.Warn(err)
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := cgroup.MoveTo(parent); err != nil {
|
||||||
|
+ // Don't fail, cgroup can be deleted
|
||||||
|
+ logrus.Warnf("Could not move container process into parent cgroup: %v", err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := cgroup.Delete(); err != nil {
|
||||||
|
+ return fmt.Errorf("Could not delete cgroup %v: %v", cgroupPath, err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// GetSandboxCgroupPath return the cgroup path of specified sandbox
|
||||||
|
+func GetSandboxCgroupPath(ctx context.Context, sandboxID string) (string, error) {
|
||||||
|
+ stateFilePath := filepath.Join(store.RunStoragePath(), sandboxID, store.StateFile)
|
||||||
|
+
|
||||||
|
+ fileData, err := ioutil.ReadFile(stateFilePath)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return "", err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ state := types.SandboxState{}
|
||||||
|
+
|
||||||
|
+ if err := json.Unmarshal(fileData, &state); err != nil {
|
||||||
|
+ return "", err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if state.CgroupPath == "" {
|
||||||
|
+ return "", fmt.Errorf("get sandbox cgroup path error: cgroupPath is empty")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return state.CgroupPath, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// AddPidToSandboxCgroup add kata-runtime create process to cgroup
|
||||||
|
+// the pid will be added to the cgroup of "<path>/vcpu"
|
||||||
|
+func AddPidToSandboxCgroup(ctx context.Context, pid int, cgroupPath string) error {
|
||||||
|
+ cgroup, err := cgroupsLoadFunc(cgroups.V1, cgroups.StaticPath(cgroupPath))
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ err = cgroup.Add(cgroups.Process{Pid: pid})
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// V1Constraints returns the cgroups that are compatible with the VC architecture
|
||||||
|
// and hypervisor, constraints can be applied to these cgroups.
|
||||||
|
func V1Constraints() ([]cgroups.Subsystem, error) {
|
||||||
|
diff --git a/virtcontainers/implementation.go b/virtcontainers/implementation.go
|
||||||
|
index e4bc4ae..fedc51f 100644
|
||||||
|
--- a/virtcontainers/implementation.go
|
||||||
|
+++ b/virtcontainers/implementation.go
|
||||||
|
@@ -188,3 +188,23 @@ func (impl *VCImpl) CleanupContainer(ctx context.Context, sandboxID, containerID
|
||||||
|
func (impl *VCImpl) UpdateIPVSRule(ctx context.Context, sandboxID string, IPVSRule *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
|
||||||
|
return UpdateIPVSRule(ctx, sandboxID, IPVSRule)
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+// CreateSandboxCgroup implements the VC function of the same name.
|
||||||
|
+func (impl *VCImpl) CreateSandboxCgroup(ctx context.Context, sandboxCgroupPath string) error {
|
||||||
|
+ return CreateSandboxCgroup(ctx, sandboxCgroupPath)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// DestroySandboxCgroup implements the VC function of the same name.
|
||||||
|
+func (impl *VCImpl) DestroySandboxCgroup(ctx context.Context, sandboxCgroupPath string) error {
|
||||||
|
+ return DestroySandboxCgroup(ctx, sandboxCgroupPath)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// AddPidToSandboxCgroup implements the VC function of the same name.
|
||||||
|
+func (impl *VCImpl) AddPidToSandboxCgroup(ctx context.Context, pid int, sandboxCgroupPath string) error {
|
||||||
|
+ return AddPidToSandboxCgroup(ctx, pid, sandboxCgroupPath)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// GetSandboxCgroupPath implements the VC function of the same name.
|
||||||
|
+func (impl *VCImpl) GetSandboxCgroupPath(ctx context.Context, sandboxID string) (string, error) {
|
||||||
|
+ return GetSandboxCgroupPath(ctx, sandboxID)
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/interfaces.go b/virtcontainers/interfaces.go
|
||||||
|
index 0fd12d8..4d166e0 100644
|
||||||
|
--- a/virtcontainers/interfaces.go
|
||||||
|
+++ b/virtcontainers/interfaces.go
|
||||||
|
@@ -24,6 +24,11 @@ type VC interface {
|
||||||
|
SetLogger(ctx context.Context, logger *logrus.Entry)
|
||||||
|
SetFactory(ctx context.Context, factory Factory)
|
||||||
|
|
||||||
|
+ CreateSandboxCgroup(ctx context.Context, sandboxCgroupPath string) error
|
||||||
|
+ DestroySandboxCgroup(ctx context.Context, sandboxCgroupPath string) error
|
||||||
|
+ AddPidToSandboxCgroup(ctx context.Context, pid int, sandboxCgroupPath string) error
|
||||||
|
+ GetSandboxCgroupPath(ctx context.Context, sandboxID string) (string, error)
|
||||||
|
+
|
||||||
|
CreateSandbox(ctx context.Context, sandboxConfig SandboxConfig) (VCSandbox, error)
|
||||||
|
DeleteSandbox(ctx context.Context, sandboxID string) (VCSandbox, error)
|
||||||
|
FetchSandbox(ctx context.Context, sandboxID string) (VCSandbox, error)
|
||||||
|
--
|
||||||
|
1.8.3.1
|
||||||
|
|
||||||
@ -0,0 +1,376 @@
|
|||||||
|
From 98a3c4677261e1c0364015f36928cddfb0af253e Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Wed, 9 Sep 2020 16:45:24 +0800
|
||||||
|
Subject: [PATCH 2/5] kata-runtime: add sandbox cgroup with vcpu and emulator
|
||||||
|
switch
|
||||||
|
|
||||||
|
reason: add sandbox cgroup with vcpu and emulator switch, if
|
||||||
|
sandbox_cgroup_with_emulator is true, it will overload the feature
|
||||||
|
of sandbox_cgroup_only, there will be two cgroups, vcpu and emulator
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
cli/config/configuration-qemu.toml.in | 12 +++++++++
|
||||||
|
cli/kata-env.go | 38 ++++++++++++++-------------
|
||||||
|
pkg/katautils/config.go | 18 +++++++------
|
||||||
|
virtcontainers/api.go | 4 ++-
|
||||||
|
virtcontainers/container.go | 6 ++---
|
||||||
|
virtcontainers/persist.go | 30 +++++++++++----------
|
||||||
|
virtcontainers/persist/api/config.go | 2 ++
|
||||||
|
virtcontainers/pkg/annotations/annotations.go | 2 ++
|
||||||
|
virtcontainers/pkg/oci/utils.go | 13 +++++++++
|
||||||
|
virtcontainers/sandbox.go | 23 +++++++++++-----
|
||||||
|
10 files changed, 97 insertions(+), 51 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in
|
||||||
|
index e57a954..fae88f9 100644
|
||||||
|
--- a/cli/config/configuration-qemu.toml.in
|
||||||
|
+++ b/cli/config/configuration-qemu.toml.in
|
||||||
|
@@ -477,6 +477,18 @@ enable_compat_old_cni = true
|
||||||
|
# See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType
|
||||||
|
sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
|
||||||
|
|
||||||
|
+# It is a new host cgroup solution to limit the kata resouce in the host different from the
|
||||||
|
+# community original solution.If sandbox_cgroup_with_emulator is enabled, it will override
|
||||||
|
+# the config of sandbox_cgroup_only. Each Pod corresponds to a pod level cgroup directory
|
||||||
|
+# which is named with sandboxID. In each pod level cgroup, it contains two sub cgroup
|
||||||
|
+# directory: vcpu and emulator, these two sub cgroup only valid in the CPU cgroup subsystem,
|
||||||
|
+# because we just want to distinguish the emulator main thread and vcpu thread in the CPU
|
||||||
|
+# cgroup subsystem.And with this config enabled, kata-runtime and related sub processes will
|
||||||
|
+# added into the vcpu cgroup directory with resource limited, and qemu main thread and other
|
||||||
|
+# non-vcpu threads will be moved into the emulator cgroup without resource limit, which will
|
||||||
|
+# improve the IO throughput for kata-containers.
|
||||||
|
+sandbox_cgroup_with_emulator = true
|
||||||
|
+
|
||||||
|
# Enabled experimental feature list, format: ["a", "b"].
|
||||||
|
# Experimental features are features not stable enough for production,
|
||||||
|
# they may break compatibility, and are prepared for a big version bump.
|
||||||
|
diff --git a/cli/kata-env.go b/cli/kata-env.go
|
||||||
|
index d8a6068..48026fe 100644
|
||||||
|
--- a/cli/kata-env.go
|
||||||
|
+++ b/cli/kata-env.go
|
||||||
|
@@ -63,15 +63,16 @@ type RuntimeConfigInfo struct {
|
||||||
|
|
||||||
|
// RuntimeInfo stores runtime details.
|
||||||
|
type RuntimeInfo struct {
|
||||||
|
- Version RuntimeVersionInfo
|
||||||
|
- Config RuntimeConfigInfo
|
||||||
|
- Debug bool
|
||||||
|
- Trace bool
|
||||||
|
- DisableGuestSeccomp bool
|
||||||
|
- DisableNewNetNs bool
|
||||||
|
- SandboxCgroupOnly bool
|
||||||
|
- Experimental []exp.Feature
|
||||||
|
- Path string
|
||||||
|
+ Version RuntimeVersionInfo
|
||||||
|
+ Config RuntimeConfigInfo
|
||||||
|
+ Debug bool
|
||||||
|
+ Trace bool
|
||||||
|
+ DisableGuestSeccomp bool
|
||||||
|
+ DisableNewNetNs bool
|
||||||
|
+ SandboxCgroupOnly bool
|
||||||
|
+ SandboxCgroupWithEmulator bool
|
||||||
|
+ Experimental []exp.Feature
|
||||||
|
+ Path string
|
||||||
|
}
|
||||||
|
|
||||||
|
type VersionInfo struct {
|
||||||
|
@@ -194,15 +195,16 @@ func getRuntimeInfo(configFile string, config oci.RuntimeConfig) RuntimeInfo {
|
||||||
|
runtimePath, _ := os.Executable()
|
||||||
|
|
||||||
|
return RuntimeInfo{
|
||||||
|
- Debug: config.Debug,
|
||||||
|
- Trace: config.Trace,
|
||||||
|
- Version: runtimeVersion,
|
||||||
|
- Config: runtimeConfig,
|
||||||
|
- Path: runtimePath,
|
||||||
|
- DisableNewNetNs: config.DisableNewNetNs,
|
||||||
|
- SandboxCgroupOnly: config.SandboxCgroupOnly,
|
||||||
|
- Experimental: config.Experimental,
|
||||||
|
- DisableGuestSeccomp: config.DisableGuestSeccomp,
|
||||||
|
+ Debug: config.Debug,
|
||||||
|
+ Trace: config.Trace,
|
||||||
|
+ Version: runtimeVersion,
|
||||||
|
+ Config: runtimeConfig,
|
||||||
|
+ Path: runtimePath,
|
||||||
|
+ DisableNewNetNs: config.DisableNewNetNs,
|
||||||
|
+ SandboxCgroupOnly: config.SandboxCgroupOnly,
|
||||||
|
+ SandboxCgroupWithEmulator: config.SandboxCgroupWithEmulator,
|
||||||
|
+ Experimental: config.Experimental,
|
||||||
|
+ DisableGuestSeccomp: config.DisableGuestSeccomp,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
||||||
|
index 3365b3f..89e46f6 100644
|
||||||
|
--- a/pkg/katautils/config.go
|
||||||
|
+++ b/pkg/katautils/config.go
|
||||||
|
@@ -139,14 +139,15 @@ type proxy struct {
|
||||||
|
}
|
||||||
|
|
||||||
|
type runtime struct {
|
||||||
|
- Debug bool `toml:"enable_debug"`
|
||||||
|
- Tracing bool `toml:"enable_tracing"`
|
||||||
|
- DisableNewNetNs bool `toml:"disable_new_netns"`
|
||||||
|
- EnableCompatOldCNI bool `toml:"enable_compat_old_cni"`
|
||||||
|
- DisableGuestSeccomp bool `toml:"disable_guest_seccomp"`
|
||||||
|
- SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"`
|
||||||
|
- Experimental []string `toml:"experimental"`
|
||||||
|
- InterNetworkModel string `toml:"internetworking_model"`
|
||||||
|
+ Debug bool `toml:"enable_debug"`
|
||||||
|
+ Tracing bool `toml:"enable_tracing"`
|
||||||
|
+ DisableNewNetNs bool `toml:"disable_new_netns"`
|
||||||
|
+ EnableCompatOldCNI bool `toml:"enable_compat_old_cni"`
|
||||||
|
+ DisableGuestSeccomp bool `toml:"disable_guest_seccomp"`
|
||||||
|
+ SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"`
|
||||||
|
+ SandboxCgroupWithEmulator bool `toml:"sandbox_cgroup_with_emulator"`
|
||||||
|
+ Experimental []string `toml:"experimental"`
|
||||||
|
+ InterNetworkModel string `toml:"internetworking_model"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type shim struct {
|
||||||
|
@@ -1252,6 +1253,7 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool, debugFlag
|
||||||
|
}
|
||||||
|
|
||||||
|
config.SandboxCgroupOnly = tomlConf.Runtime.SandboxCgroupOnly
|
||||||
|
+ config.SandboxCgroupWithEmulator = tomlConf.Runtime.SandboxCgroupWithEmulator
|
||||||
|
config.DisableNewNetNs = tomlConf.Runtime.DisableNewNetNs
|
||||||
|
config.EnableCompatOldCNI = tomlConf.Runtime.EnableCompatOldCNI
|
||||||
|
for _, f := range tomlConf.Runtime.Experimental {
|
||||||
|
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
|
||||||
|
index ca5412a..08bcbb5 100644
|
||||||
|
--- a/virtcontainers/api.go
|
||||||
|
+++ b/virtcontainers/api.go
|
||||||
|
@@ -103,7 +103,9 @@ func createSandboxFromConfig(ctx context.Context, sandboxConfig SandboxConfig, f
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Move runtime to sandbox cgroup so all process are created there.
|
||||||
|
- if s.config.SandboxCgroupOnly {
|
||||||
|
+ if s.config.SandboxCgroupWithEmulator{
|
||||||
|
+ // emulator
|
||||||
|
+ } else if s.config.SandboxCgroupOnly {
|
||||||
|
if err := s.setupSandboxCgroup(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/container.go b/virtcontainers/container.go
|
||||||
|
index 4060ebb..1b70382 100644
|
||||||
|
--- a/virtcontainers/container.go
|
||||||
|
+++ b/virtcontainers/container.go
|
||||||
|
@@ -1009,7 +1009,7 @@ func (c *Container) create() (err error) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
- if !rootless.IsRootless() && !c.sandbox.config.SandboxCgroupOnly {
|
||||||
|
+ if !rootless.IsRootless() && !c.sandbox.config.SandboxCgroupOnly && !c.sandbox.config.SandboxCgroupWithEmulator {
|
||||||
|
if err = c.cgroupsCreate(); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
@@ -1034,7 +1034,7 @@ func (c *Container) delete() error {
|
||||||
|
}
|
||||||
|
|
||||||
|
// If running rootless, there are no cgroups to remove
|
||||||
|
- if !c.sandbox.config.SandboxCgroupOnly || !rootless.IsRootless() {
|
||||||
|
+ if !c.sandbox.config.SandboxCgroupWithEmulator && (!c.sandbox.config.SandboxCgroupOnly || !rootless.IsRootless()) {
|
||||||
|
if err := c.cgroupsDelete(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
@@ -1348,7 +1348,7 @@ func (c *Container) update(resources specs.LinuxResources) error {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
- if !c.sandbox.config.SandboxCgroupOnly {
|
||||||
|
+ if !c.sandbox.config.SandboxCgroupWithEmulator && !c.sandbox.config.SandboxCgroupOnly {
|
||||||
|
if err := c.cgroupsUpdate(resources); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go
|
||||||
|
index fe00bf9..efa4506 100644
|
||||||
|
--- a/virtcontainers/persist.go
|
||||||
|
+++ b/virtcontainers/persist.go
|
||||||
|
@@ -194,13 +194,14 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
|
||||||
|
InterworkingModel: int(sconfig.NetworkConfig.InterworkingModel),
|
||||||
|
},
|
||||||
|
|
||||||
|
- ShmSize: sconfig.ShmSize,
|
||||||
|
- SharePidNs: sconfig.SharePidNs,
|
||||||
|
- Stateful: sconfig.Stateful,
|
||||||
|
- SystemdCgroup: sconfig.SystemdCgroup,
|
||||||
|
- SandboxCgroupOnly: sconfig.SandboxCgroupOnly,
|
||||||
|
- DisableGuestSeccomp: sconfig.DisableGuestSeccomp,
|
||||||
|
- Cgroups: sconfig.Cgroups,
|
||||||
|
+ ShmSize: sconfig.ShmSize,
|
||||||
|
+ SharePidNs: sconfig.SharePidNs,
|
||||||
|
+ Stateful: sconfig.Stateful,
|
||||||
|
+ SystemdCgroup: sconfig.SystemdCgroup,
|
||||||
|
+ SandboxCgroupOnly: sconfig.SandboxCgroupOnly,
|
||||||
|
+ SandboxCgroupWithEmulator: sconfig.SandboxCgroupWithEmulator,
|
||||||
|
+ DisableGuestSeccomp: sconfig.DisableGuestSeccomp,
|
||||||
|
+ Cgroups: sconfig.Cgroups,
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, e := range sconfig.Experimental {
|
||||||
|
@@ -485,13 +486,14 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
|
||||||
|
InterworkingModel: NetInterworkingModel(savedConf.NetworkConfig.InterworkingModel),
|
||||||
|
},
|
||||||
|
|
||||||
|
- ShmSize: savedConf.ShmSize,
|
||||||
|
- SharePidNs: savedConf.SharePidNs,
|
||||||
|
- Stateful: savedConf.Stateful,
|
||||||
|
- SystemdCgroup: savedConf.SystemdCgroup,
|
||||||
|
- SandboxCgroupOnly: savedConf.SandboxCgroupOnly,
|
||||||
|
- DisableGuestSeccomp: savedConf.DisableGuestSeccomp,
|
||||||
|
- Cgroups: savedConf.Cgroups,
|
||||||
|
+ ShmSize: savedConf.ShmSize,
|
||||||
|
+ SharePidNs: savedConf.SharePidNs,
|
||||||
|
+ Stateful: savedConf.Stateful,
|
||||||
|
+ SystemdCgroup: savedConf.SystemdCgroup,
|
||||||
|
+ SandboxCgroupOnly: savedConf.SandboxCgroupOnly,
|
||||||
|
+ SandboxCgroupWithEmulator: savedConf.SandboxCgroupWithEmulator,
|
||||||
|
+ DisableGuestSeccomp: savedConf.DisableGuestSeccomp,
|
||||||
|
+ Cgroups: savedConf.Cgroups,
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, name := range savedConf.Experimental {
|
||||||
|
diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go
|
||||||
|
index 3a2df32..28204fc 100644
|
||||||
|
--- a/virtcontainers/persist/api/config.go
|
||||||
|
+++ b/virtcontainers/persist/api/config.go
|
||||||
|
@@ -258,6 +258,8 @@ type SandboxConfig struct {
|
||||||
|
// SandboxCgroupOnly enables cgroup only at podlevel in the host
|
||||||
|
SandboxCgroupOnly bool
|
||||||
|
|
||||||
|
+ SandboxCgroupWithEmulator bool
|
||||||
|
+
|
||||||
|
DisableGuestSeccomp bool
|
||||||
|
|
||||||
|
// Experimental enables experimental features
|
||||||
|
diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go
|
||||||
|
index 528dfa6..96c4ef2 100644
|
||||||
|
--- a/virtcontainers/pkg/annotations/annotations.go
|
||||||
|
+++ b/virtcontainers/pkg/annotations/annotations.go
|
||||||
|
@@ -215,6 +215,8 @@ const (
|
||||||
|
// SandboxCgroupOnly is a sandbox annotation that determines if kata processes are managed only in sandbox cgroup.
|
||||||
|
SandboxCgroupOnly = kataAnnotRuntimePrefix + "sandbox_cgroup_only"
|
||||||
|
|
||||||
|
+ SandboxCgroupWithEmulator = kataAnnotRuntimePrefix + "sandbox_cgroup_with_emulator"
|
||||||
|
+
|
||||||
|
// Experimental is a sandbox annotation that determines if experimental features enabled.
|
||||||
|
Experimental = kataAnnotRuntimePrefix + "experimental"
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
||||||
|
index 3b2af75..91067fb 100644
|
||||||
|
--- a/virtcontainers/pkg/oci/utils.go
|
||||||
|
+++ b/virtcontainers/pkg/oci/utils.go
|
||||||
|
@@ -139,6 +139,8 @@ type RuntimeConfig struct {
|
||||||
|
//Determines kata processes are managed only in sandbox cgroup
|
||||||
|
SandboxCgroupOnly bool
|
||||||
|
|
||||||
|
+ SandboxCgroupWithEmulator bool
|
||||||
|
+
|
||||||
|
//Experimental features enabled
|
||||||
|
Experimental []exp.Feature
|
||||||
|
}
|
||||||
|
@@ -746,6 +748,15 @@ func addRuntimeConfigOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e
|
||||||
|
sbConfig.SandboxCgroupOnly = sandboxCgroupOnly
|
||||||
|
}
|
||||||
|
|
||||||
|
+ if value, ok := ocispec.Annotations[vcAnnotations.SandboxCgroupWithEmulator]; ok {
|
||||||
|
+ sandboxCgroupWithEmulator, err := strconv.ParseBool(value)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return fmt.Errorf("error parsing annotation for sandbox_cgroup_with_emulator : Please specify boolean value 'true|false'")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ sbConfig.SandboxCgroupWithEmulator = sandboxCgroupWithEmulator
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
if value, ok := ocispec.Annotations[vcAnnotations.Experimental]; ok {
|
||||||
|
features := strings.Split(value, " ")
|
||||||
|
sbConfig.Experimental = []exp.Feature{}
|
||||||
|
@@ -869,6 +880,8 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid, c
|
||||||
|
|
||||||
|
SandboxCgroupOnly: runtime.SandboxCgroupOnly,
|
||||||
|
|
||||||
|
+ SandboxCgroupWithEmulator: runtime.SandboxCgroupWithEmulator,
|
||||||
|
+
|
||||||
|
DisableGuestSeccomp: runtime.DisableGuestSeccomp,
|
||||||
|
|
||||||
|
// Q: Is this really necessary? @weizhang555
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index 174e6cb..b479cf5 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -126,6 +126,8 @@ type SandboxConfig struct {
|
||||||
|
// SandboxCgroupOnly enables cgroup only at podlevel in the host
|
||||||
|
SandboxCgroupOnly bool
|
||||||
|
|
||||||
|
+ SandboxCgroupWithEmulator bool
|
||||||
|
+
|
||||||
|
DisableGuestSeccomp bool
|
||||||
|
|
||||||
|
// Experimental features enabled
|
||||||
|
@@ -1532,8 +1534,9 @@ func (s *Sandbox) Stats() (SandboxStats, error) {
|
||||||
|
|
||||||
|
var path string
|
||||||
|
var cgroupSubsystems cgroups.Hierarchy
|
||||||
|
-
|
||||||
|
- if s.config.SandboxCgroupOnly {
|
||||||
|
+ if !s.config.SandboxCgroupWithEmulator {
|
||||||
|
+ // vcpu and emulator
|
||||||
|
+ } else if s.config.SandboxCgroupOnly {
|
||||||
|
cgroupSubsystems = cgroups.V1
|
||||||
|
path = s.state.CgroupPath
|
||||||
|
} else {
|
||||||
|
@@ -1793,7 +1796,9 @@ func (s *Sandbox) HotplugAddDevice(device api.Device, devType config.DeviceType)
|
||||||
|
span, _ := s.trace("HotplugAddDevice")
|
||||||
|
defer span.Finish()
|
||||||
|
|
||||||
|
- if s.config.SandboxCgroupOnly {
|
||||||
|
+ if s.config.SandboxCgroupWithEmulator {
|
||||||
|
+ // emulator
|
||||||
|
+ } else if s.config.SandboxCgroupOnly {
|
||||||
|
// We are about to add a device to the hypervisor,
|
||||||
|
// the device cgroup MUST be updated since the hypervisor
|
||||||
|
// will need access to such device
|
||||||
|
@@ -1849,7 +1854,9 @@ func (s *Sandbox) HotplugAddDevice(device api.Device, devType config.DeviceType)
|
||||||
|
// Sandbox implement DeviceReceiver interface from device/api/interface.go
|
||||||
|
func (s *Sandbox) HotplugRemoveDevice(device api.Device, devType config.DeviceType) error {
|
||||||
|
defer func() {
|
||||||
|
- if s.config.SandboxCgroupOnly {
|
||||||
|
+ if s.config.SandboxCgroupWithEmulator {
|
||||||
|
+
|
||||||
|
+ } else if s.config.SandboxCgroupOnly {
|
||||||
|
// Remove device from cgroup, the hypervisor
|
||||||
|
// should not have access to such device anymore.
|
||||||
|
hdev := device.GetHostPath()
|
||||||
|
@@ -2107,7 +2114,7 @@ func (s *Sandbox) cgroupsUpdate() error {
|
||||||
|
// If Kata is configured for SandboxCgroupOnly, the VMM and its processes are already
|
||||||
|
// in the Kata sandbox cgroup (inherited). No need to move threads/processes, and we should
|
||||||
|
// rely on parent's cgroup CPU/memory values
|
||||||
|
- if s.config.SandboxCgroupOnly {
|
||||||
|
+ if s.config.SandboxCgroupWithEmulator || s.config.SandboxCgroupOnly {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -2154,7 +2161,9 @@ func (s *Sandbox) cgroupsDelete() error {
|
||||||
|
var path string
|
||||||
|
var cgroupSubsystems cgroups.Hierarchy
|
||||||
|
|
||||||
|
- if s.config.SandboxCgroupOnly {
|
||||||
|
+ if s.config.SandboxCgroupWithEmulator {
|
||||||
|
+ // emulator
|
||||||
|
+ } else if s.config.SandboxCgroupOnly {
|
||||||
|
return s.cgroupMgr.Destroy()
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -2197,7 +2206,7 @@ func (s *Sandbox) constrainHypervisor(cgroup cgroups.Cgroup) error {
|
||||||
|
// Kata/VMM into account, Kata may fail to boot due to being overconstrained.
|
||||||
|
// If !SandboxCgroupOnly, place the VMM into an unconstrained cgroup, and the vCPU threads into constrained
|
||||||
|
// cgroup
|
||||||
|
- if s.config.SandboxCgroupOnly {
|
||||||
|
+ if s.config.SandboxCgroupOnly || s.config.SandboxCgroupWithEmulator {
|
||||||
|
// Kata components were moved into the sandbox-cgroup already, so VMM
|
||||||
|
// will already land there as well. No need to take action
|
||||||
|
return nil
|
||||||
|
--
|
||||||
|
1.8.3.1
|
||||||
|
|
||||||
@ -0,0 +1,428 @@
|
|||||||
|
From ce7523dfe1bb60cf54254e16a103fd3fc9503618 Mon Sep 17 00:00:00 2001
|
||||||
|
From: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
Date: Thu, 17 Sep 2020 10:38:38 +0800
|
||||||
|
Subject: [PATCH 3/5] kata_runtime: support host cgroup with emulator policy
|
||||||
|
|
||||||
|
reason: support host cgroup with emulator policy when
|
||||||
|
sandbox_cgroup_with_emulator is set true
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
cli/create.go | 38 ++++++++++++
|
||||||
|
virtcontainers/api.go | 10 ++-
|
||||||
|
virtcontainers/cgroups.go | 132 ++++++++++++++++++++++++++++++++++------
|
||||||
|
virtcontainers/persist/fs/fs.go | 8 +++
|
||||||
|
virtcontainers/pkg/oci/utils.go | 14 +++++
|
||||||
|
virtcontainers/sandbox.go | 70 ++++++++++++++++++++-
|
||||||
|
6 files changed, 250 insertions(+), 22 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/cli/create.go b/cli/create.go
|
||||||
|
index 02cb2c5..b14434b 100644
|
||||||
|
--- a/cli/create.go
|
||||||
|
+++ b/cli/create.go
|
||||||
|
@@ -11,6 +11,7 @@ import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
+ "path/filepath"
|
||||||
|
|
||||||
|
"github.com/kata-containers/runtime/pkg/katautils"
|
||||||
|
vc "github.com/kata-containers/runtime/virtcontainers"
|
||||||
|
@@ -134,11 +135,48 @@ func create(ctx context.Context, containerID, bundlePath, console, pidFilePath s
|
||||||
|
var process vc.Process
|
||||||
|
switch containerType {
|
||||||
|
case vc.PodSandbox:
|
||||||
|
+ if runtimeConfig.SandboxCgroupWithEmulator {
|
||||||
|
+ // create the sandbox level cgroup
|
||||||
|
+ cgroupPath := ociSpec.Linux.CgroupsPath
|
||||||
|
+ if err = vci.CreateSandboxCgroup(ctx, cgroupPath); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ defer func() {
|
||||||
|
+ if err != nil {
|
||||||
|
+ _ = vci.DestroySandboxCgroup(ctx, cgroupPath)
|
||||||
|
+ }
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
+ // add kata-runtime create process into <path>/vcpu cgroup
|
||||||
|
+ vcpuCgroupPath := filepath.Join(cgroupPath, "vcpu")
|
||||||
|
+ if err = vci.AddPidToSandboxCgroup(ctx, os.Getpid(), vcpuCgroupPath); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
_, process, err = katautils.CreateSandbox(ctx, vci, ociSpec, runtimeConfig, rootFs, containerID, bundlePath, console, disableOutput, systemdCgroup, false)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
case vc.PodContainer:
|
||||||
|
+ if runtimeConfig.SandboxCgroupWithEmulator {
|
||||||
|
+ sandboxID, err := oci.GetSandboxIDFromAnnotations(&ociSpec)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return fmt.Errorf("container annotation doesn't contain sandboxID")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ sandboxCgroupPath, err := vci.GetSandboxCgroupPath(ctx, sandboxID)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // add kata-runtime create process into <path>/vcpu cgroup
|
||||||
|
+ vcpuCgroupPath := filepath.Join(sandboxCgroupPath, "vcpu")
|
||||||
|
+ if err = vci.AddPidToSandboxCgroup(ctx, os.Getpid(), vcpuCgroupPath); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
process, err = katautils.CreateContainer(ctx, vci, nil, ociSpec, rootFs, containerID, bundlePath, console, disableOutput, false)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
|
||||||
|
index 08bcbb5..38c8235 100644
|
||||||
|
--- a/virtcontainers/api.go
|
||||||
|
+++ b/virtcontainers/api.go
|
||||||
|
@@ -103,9 +103,7 @@ func createSandboxFromConfig(ctx context.Context, sandboxConfig SandboxConfig, f
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Move runtime to sandbox cgroup so all process are created there.
|
||||||
|
- if s.config.SandboxCgroupWithEmulator{
|
||||||
|
- // emulator
|
||||||
|
- } else if s.config.SandboxCgroupOnly {
|
||||||
|
+ if !s.config.SandboxCgroupWithEmulator && s.config.SandboxCgroupOnly {
|
||||||
|
if err := s.setupSandboxCgroup(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
@@ -129,6 +127,12 @@ func createSandboxFromConfig(ctx context.Context, sandboxConfig SandboxConfig, f
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
+ if s.config.SandboxCgroupWithEmulator {
|
||||||
|
+ if err := s.setupHostCgroupsWithEmulator(); err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
// Create Containers
|
||||||
|
if err = s.createContainers(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
diff --git a/virtcontainers/cgroups.go b/virtcontainers/cgroups.go
|
||||||
|
index df0ec30..65d2001 100644
|
||||||
|
--- a/virtcontainers/cgroups.go
|
||||||
|
+++ b/virtcontainers/cgroups.go
|
||||||
|
@@ -9,19 +9,15 @@ package virtcontainers
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"context"
|
||||||
|
- "encoding/json"
|
||||||
|
"fmt"
|
||||||
|
- "io/ioutil"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
+ "strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/containerd/cgroups"
|
||||||
|
specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
-
|
||||||
|
- "github.com/kata-containers/runtime/virtcontainers/store"
|
||||||
|
- "github.com/kata-containers/runtime/virtcontainers/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
type cgroupPather interface {
|
||||||
|
@@ -32,7 +28,11 @@ type cgroupPather interface {
|
||||||
|
// unconstrained cgroups are placed here.
|
||||||
|
// for example /sys/fs/cgroup/memory/kata/$CGPATH
|
||||||
|
// where path is defined by the containers manager
|
||||||
|
-const cgroupKataPath = "/kata/"
|
||||||
|
+const (
|
||||||
|
+ cgroupKataPath = "/kata/"
|
||||||
|
+ vcpuCgroupName = "vcpu"
|
||||||
|
+ emulatorCgroupName = "emulator"
|
||||||
|
+)
|
||||||
|
|
||||||
|
var cgroupsLoadFunc = cgroups.Load
|
||||||
|
var cgroupsNewFunc = cgroups.New
|
||||||
|
@@ -105,24 +105,16 @@ func deleteCgroup(hierarchy cgroups.Hierarchy, cgroupPath string) error {
|
||||||
|
|
||||||
|
// GetSandboxCgroupPath return the cgroup path of specified sandbox
|
||||||
|
func GetSandboxCgroupPath(ctx context.Context, sandboxID string) (string, error) {
|
||||||
|
- stateFilePath := filepath.Join(store.RunStoragePath(), sandboxID, store.StateFile)
|
||||||
|
-
|
||||||
|
- fileData, err := ioutil.ReadFile(stateFilePath)
|
||||||
|
+ config, err := loadSandboxConfig(sandboxID)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
- state := types.SandboxState{}
|
||||||
|
-
|
||||||
|
- if err := json.Unmarshal(fileData, &state); err != nil {
|
||||||
|
- return "", err
|
||||||
|
- }
|
||||||
|
-
|
||||||
|
- if state.CgroupPath == "" {
|
||||||
|
- return "", fmt.Errorf("get sandbox cgroup path error: cgroupPath is empty")
|
||||||
|
+ if config.Cgroups == nil {
|
||||||
|
+ return "", fmt.Errorf("the cgroups of sandbox %s is nil", sandboxID)
|
||||||
|
}
|
||||||
|
|
||||||
|
- return state.CgroupPath, nil
|
||||||
|
+ return config.Cgroups.Path, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddPidToSandboxCgroup add kata-runtime create process to cgroup
|
||||||
|
@@ -276,3 +268,107 @@ func validCPUResources(cpuSpec *specs.LinuxCPU) *specs.LinuxCPU {
|
||||||
|
|
||||||
|
return &cpu
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+// getQemuTaskWithoutVcpu filter out tasks under /proc/{qemu pid}/task, to find out the task of not VCPU,
|
||||||
|
+// VCPU task is filtered by "query-cpus" qmp command
|
||||||
|
+func getQemuTaskWithoutVcpu(sandbox *Sandbox, vmPid int) []int {
|
||||||
|
+ procPath := fmt.Sprintf("/proc/%d/task", vmPid)
|
||||||
|
+
|
||||||
|
+ dirReader, err := os.Open(procPath)
|
||||||
|
+ if err != nil {
|
||||||
|
+ logrus.Warningf("cannot open %s: %s", procPath, err)
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ defer dirReader.Close()
|
||||||
|
+
|
||||||
|
+ dirs, err := dirReader.Readdirnames(0)
|
||||||
|
+ if err != nil {
|
||||||
|
+ logrus.Warningf("walking dirs in %s failed: %s", procPath, err)
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ vcpuThreadInfo, err := sandbox.hypervisor.getThreadIDs()
|
||||||
|
+ if err != nil {
|
||||||
|
+ logrus.Warnf("get hypervisor Thread ID failed: %v", err)
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ var vcpuThreadIDs []int
|
||||||
|
+ for _, value := range vcpuThreadInfo.vcpus {
|
||||||
|
+ vcpuThreadIDs = append(vcpuThreadIDs, value)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ var allThreadIDs []int
|
||||||
|
+ for _, dir := range dirs {
|
||||||
|
+ p, err := strconv.Atoi(dir)
|
||||||
|
+ if err != nil {
|
||||||
|
+ logrus.Warnf("can not change string dir: %s to int type", dir)
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ allThreadIDs = append(allThreadIDs, p)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ nonVCPUThreads := diffSlice(allThreadIDs, vcpuThreadIDs)
|
||||||
|
+
|
||||||
|
+ return nonVCPUThreads
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func pulloutQemuThread(sandbox *Sandbox, vmPid int, path string) error {
|
||||||
|
+ control, err := cgroups.New(cgroups.SingleSubsystem(cgroups.V1, cgroups.Cpu),
|
||||||
|
+ cgroups.StaticPath(path),
|
||||||
|
+ &specs.LinuxResources{})
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ taskIds := getQemuTaskWithoutVcpu(sandbox, vmPid)
|
||||||
|
+ if len(taskIds) == 0 {
|
||||||
|
+ logrus.Warnf("no taskId id in qemu other than vcpu found of pid %d", vmPid)
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+ for _, taskId := range taskIds {
|
||||||
|
+ if err := control.AddTask(cgroups.Process{
|
||||||
|
+ Pid: taskId,
|
||||||
|
+ }); err != nil {
|
||||||
|
+ logrus.Errorf("failed to add task %d to cgroup of %s", taskId, path)
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// checkCgroupExist check cgroup exist or not
|
||||||
|
+func checkCgroupExist(hierarchy cgroups.Hierarchy, path string) bool {
|
||||||
|
+ subSystems, _ := hierarchy()
|
||||||
|
+ for _, s := range cgroupPathers(subSystems) {
|
||||||
|
+ if _, err := os.Lstat(s.Path(path)); err != nil {
|
||||||
|
+ if os.IsNotExist(err) {
|
||||||
|
+ return false
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return true
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// diffSlice return the s1 - s2
|
||||||
|
+func diffSlice(s1, s2 []int) []int {
|
||||||
|
+ var diffSlice []int
|
||||||
|
+ for _, p := range s1 {
|
||||||
|
+ if !isInSlice(p, s2) {
|
||||||
|
+ diffSlice = append(diffSlice, p)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ return diffSlice
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func isInSlice(i int, s []int) bool {
|
||||||
|
+ for _, v := range s {
|
||||||
|
+ if i == v {
|
||||||
|
+ return true
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ return false
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/persist/fs/fs.go b/virtcontainers/persist/fs/fs.go
|
||||||
|
index 38efdba..641d64e 100644
|
||||||
|
--- a/virtcontainers/persist/fs/fs.go
|
||||||
|
+++ b/virtcontainers/persist/fs/fs.go
|
||||||
|
@@ -14,6 +14,8 @@ import (
|
||||||
|
"path/filepath"
|
||||||
|
"syscall"
|
||||||
|
|
||||||
|
+ "github.com/opencontainers/runc/libcontainer/configs"
|
||||||
|
+
|
||||||
|
persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
)
|
||||||
|
@@ -78,6 +80,12 @@ func (fs *FS) ToDisk(ss persistapi.SandboxState, cs map[string]persistapi.Contai
|
||||||
|
return fmt.Errorf("sandbox container id required")
|
||||||
|
}
|
||||||
|
|
||||||
|
+ if ss.Config.Cgroups == nil {
|
||||||
|
+ ss.Config.Cgroups = &configs.Cgroup{
|
||||||
|
+ Path: ss.CgroupPath,
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
fs.sandboxState = &ss
|
||||||
|
fs.containerState = cs
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
||||||
|
index 91067fb..e8ef41b 100644
|
||||||
|
--- a/virtcontainers/pkg/oci/utils.go
|
||||||
|
+++ b/virtcontainers/pkg/oci/utils.go
|
||||||
|
@@ -1136,3 +1136,17 @@ func validateSandboxDNS(value string) error {
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func GetSandboxIDFromAnnotations(s *specs.Spec) (string, error) {
|
||||||
|
+ if s == nil {
|
||||||
|
+ return "", fmt.Errorf("spec is nil")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ for _, v := range CRISandboxNameKeyList {
|
||||||
|
+ if sandboxID, ok := s.Annotations[v]; ok {
|
||||||
|
+ return sandboxID, nil
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return "", fmt.Errorf("failed to find the sandbox ID")
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index b479cf5..ca4e700 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -2162,7 +2162,9 @@ func (s *Sandbox) cgroupsDelete() error {
|
||||||
|
var cgroupSubsystems cgroups.Hierarchy
|
||||||
|
|
||||||
|
if s.config.SandboxCgroupWithEmulator {
|
||||||
|
- // emulator
|
||||||
|
+ if err := deleteCgroup(cgroups.V1, s.state.CgroupPath); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
} else if s.config.SandboxCgroupOnly {
|
||||||
|
return s.cgroupMgr.Destroy()
|
||||||
|
}
|
||||||
|
@@ -2381,6 +2383,68 @@ func (s *Sandbox) setupSandboxCgroup() error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (s *Sandbox) setupHostCgroupsWithEmulator() error {
|
||||||
|
+ if len(s.config.Containers) == 0 {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ sandboxContainerSpec := s.GetPatchedOCISpec()
|
||||||
|
+ if sandboxContainerSpec == nil {
|
||||||
|
+ return fmt.Errorf("sandbox container should not be empty")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // Set sandbox's cgroup path
|
||||||
|
+ s.state.CgroupPath = sandboxContainerSpec.Linux.CgroupsPath
|
||||||
|
+
|
||||||
|
+ if !checkCgroupExist(cgroups.V1, s.state.CgroupPath) {
|
||||||
|
+ return fmt.Errorf("sandbox's cgroup %s doesn't exist", s.state.CgroupPath)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // pull out qemu threads other than vcpu to the cgroup of "<path>/emulator"
|
||||||
|
+ if s.config.HypervisorType == QemuHypervisor {
|
||||||
|
+ emulatorCgroupPath := filepath.Join(s.state.CgroupPath, emulatorCgroupName)
|
||||||
|
+ hypervisorPids := s.hypervisor.getPids()
|
||||||
|
+ if len(hypervisorPids) == 0 || hypervisorPids[0] == 0 {
|
||||||
|
+ return fmt.Errorf("hypervisor pid: %v invalid", hypervisorPids)
|
||||||
|
+ }
|
||||||
|
+ if err := pulloutQemuThread(s, hypervisorPids[0], emulatorCgroupPath); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // limit cpu to "<path>/vcpu"
|
||||||
|
+ vcpuCgroupPath := filepath.Join(s.state.CgroupPath, vcpuCgroupName)
|
||||||
|
+ vcpuResources := specs.LinuxResources{
|
||||||
|
+ CPU: s.cpuResources(),
|
||||||
|
+ }
|
||||||
|
+ if err := applyResourceLimit(&vcpuResources, vcpuCgroupPath); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // limit blkio resource to "<path>"
|
||||||
|
+
|
||||||
|
+ // limit files resource
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func applyResourceLimit(resources *specs.LinuxResources, cgroupPath string) error {
|
||||||
|
+ if resources == nil {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ control, err := cgroupsLoadFunc(cgroups.V1, cgroups.StaticPath(cgroupPath))
|
||||||
|
+ if err != nil {
|
||||||
|
+ return fmt.Errorf("could not load cgroup %v: %v", cgroupPath, err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err = control.Update(resources); err != nil {
|
||||||
|
+ return fmt.Errorf("could not update cgroup %v: %v", cgroupPath, err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// GetPatchedOCISpec returns sandbox's OCI specification
|
||||||
|
// This OCI specification was patched when the sandbox was created
|
||||||
|
// by containerCapabilities(), SetEphemeralStorageType() and others
|
||||||
|
@@ -2452,6 +2516,10 @@ func (s *Sandbox) forceDeleteSandbox() {
|
||||||
|
c.forceDeleteContainer()
|
||||||
|
}
|
||||||
|
|
||||||
|
+ if err := deleteCgroup(cgroups.V1, s.state.CgroupPath); err != nil {
|
||||||
|
+ s.Logger().Warnf("sandbox forceDelete cgroups failed: %v", err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
globalSandboxList.removeSandbox(s.id)
|
||||||
|
|
||||||
|
if s.monitor != nil {
|
||||||
|
--
|
||||||
|
1.8.3.1
|
||||||
|
|
||||||
@ -0,0 +1,318 @@
|
|||||||
|
From f4b899b933a3a30fc378ceb4d8855778cd783d9a Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Fri, 18 Sep 2020 16:54:11 +0800
|
||||||
|
Subject: [PATCH 4/5] kata_runtime: support the blkio in host cgroups
|
||||||
|
|
||||||
|
reason: support the blkio in host cgroups, run with paras
|
||||||
|
--annotation io.kubernetes.docker.type=podsandbox --annotation io.katacontainers.blkio_cgroup=
|
||||||
|
'{"blkiocgroup":[{"path":"/dev/sda","limits":[{"type":"throttle_read_bps","value":400}]}]}'
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
vendor/github.com/containerd/cgroups/blkio.go | 6 ++
|
||||||
|
virtcontainers/cgroups.go | 102 ++++++++++++++++++++++++++
|
||||||
|
virtcontainers/pkg/annotations/annotations.go | 3 +
|
||||||
|
virtcontainers/pkg/oci/utils.go | 50 ++++++++++++-
|
||||||
|
virtcontainers/sandbox.go | 6 ++
|
||||||
|
virtcontainers/utils/utils.go | 22 ++++++
|
||||||
|
6 files changed, 186 insertions(+), 3 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/vendor/github.com/containerd/cgroups/blkio.go b/vendor/github.com/containerd/cgroups/blkio.go
|
||||||
|
index 7c498de..485ff7b 100644
|
||||||
|
--- a/vendor/github.com/containerd/cgroups/blkio.go
|
||||||
|
+++ b/vendor/github.com/containerd/cgroups/blkio.go
|
||||||
|
@@ -23,6 +23,7 @@ import (
|
||||||
|
"io/ioutil"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
+ "reflect"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
@@ -55,6 +56,11 @@ func (b *blkioController) Create(path string, resources *specs.LinuxResources) e
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, t := range createBlkioSettings(resources.BlockIO) {
|
||||||
|
+ ptr := reflect.ValueOf(t.value)
|
||||||
|
+ if ptr.Kind() == reflect.Ptr && ptr.IsNil() {
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
if t.value != nil {
|
||||||
|
if err := ioutil.WriteFile(
|
||||||
|
filepath.Join(b.Path(path), fmt.Sprintf("blkio.%s", t.name)),
|
||||||
|
diff --git a/virtcontainers/cgroups.go b/virtcontainers/cgroups.go
|
||||||
|
index 65d2001..e8c5a7b 100644
|
||||||
|
--- a/virtcontainers/cgroups.go
|
||||||
|
+++ b/virtcontainers/cgroups.go
|
||||||
|
@@ -9,6 +9,7 @@ package virtcontainers
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"context"
|
||||||
|
+ "encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
@@ -16,6 +17,8 @@ import (
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/containerd/cgroups"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||||
|
specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
)
|
||||||
|
@@ -32,8 +35,46 @@ const (
|
||||||
|
cgroupKataPath = "/kata/"
|
||||||
|
vcpuCgroupName = "vcpu"
|
||||||
|
emulatorCgroupName = "emulator"
|
||||||
|
+
|
||||||
|
+ // BlkioThrottleReadBps is the key to fetch throttle_read_bps
|
||||||
|
+ BlkioThrottleReadBps = "throttle_read_bps"
|
||||||
|
+
|
||||||
|
+ // BlkioThrottleWriteBps is the key to fetch throttle_write_bps
|
||||||
|
+ BlkioThrottleWriteBps = "throttle_write_bps"
|
||||||
|
+
|
||||||
|
+ // BlkioThrottleReadIOPS is the key to fetch throttle_read_iops
|
||||||
|
+ BlkioThrottleReadIOPS = "throttle_read_iops"
|
||||||
|
+
|
||||||
|
+ // BlkioThrottleWriteIOPS is the key to fetch throttle_write_iops
|
||||||
|
+ BlkioThrottleWriteIOPS = "throttle_write_iops"
|
||||||
|
+
|
||||||
|
+ // BlkioWeight is the key to fetch blkio_weight
|
||||||
|
+ BlkioWeight = "blkio_weight"
|
||||||
|
+
|
||||||
|
+ // BlkioLeafWeight is the key to fetch blkio_leaf_weight
|
||||||
|
+ BlkioLeafWeight = "blkio_leaf_weight"
|
||||||
|
)
|
||||||
|
|
||||||
|
+// BlkioCgroup for Linux cgroup 'blkio' data exchange
|
||||||
|
+type BlkioCgroup struct {
|
||||||
|
+ // Items specifies per cgroup values
|
||||||
|
+ Items []BlockIOCgroupItem `json:"blkiocgroup,omitempty"`
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+type BlockIOCgroupItem struct {
|
||||||
|
+ // Path represent path of blkio device
|
||||||
|
+ Path string `json:"path,omitempty"`
|
||||||
|
+ // Limits specifies the blkio type and value
|
||||||
|
+ Limits []IOLimit `json:"limits,omitempty"`
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+type IOLimit struct {
|
||||||
|
+ // Type specifies IO type
|
||||||
|
+ Type string `json:"type,omitempty"`
|
||||||
|
+ // Value specifies rate or weight value
|
||||||
|
+ Value uint64 `json:"value,omitempty"`
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
var cgroupsLoadFunc = cgroups.Load
|
||||||
|
var cgroupsNewFunc = cgroups.New
|
||||||
|
|
||||||
|
@@ -372,3 +413,64 @@ func isInSlice(i int, s []int) bool {
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func (s *Sandbox) blockIOResource() *specs.LinuxBlockIO {
|
||||||
|
+ value, ok := s.config.Annotations[annotations.BlkioCgroupTypeKey]
|
||||||
|
+ if !ok {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ var blkioCgroupParse BlkioCgroup
|
||||||
|
+ var linuxBlkio specs.LinuxBlockIO
|
||||||
|
+ if err := json.Unmarshal([]byte(value), &blkioCgroupParse); err != nil {
|
||||||
|
+ s.Logger().Errorf("blkio_cgroup Unmarshal error:%v", err)
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ for _, item := range blkioCgroupParse.Items {
|
||||||
|
+ if item.Limits == nil {
|
||||||
|
+ s.Logger().Errorf("%v:limits have none data", item)
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+ for _, limit := range item.Limits {
|
||||||
|
+ if item.Path != "" {
|
||||||
|
+ major, minor, err := utils.GetDeviceByPath(item.Path)
|
||||||
|
+ if err != nil {
|
||||||
|
+ s.Logger().Errorf("failed to find major and minor of device %s: %v", item.Path, err)
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+ td := specs.LinuxThrottleDevice{
|
||||||
|
+ Rate: limit.Value,
|
||||||
|
+ }
|
||||||
|
+ td.Major = major
|
||||||
|
+ td.Minor = minor
|
||||||
|
+ switch limit.Type {
|
||||||
|
+ case BlkioThrottleReadBps:
|
||||||
|
+ linuxBlkio.ThrottleReadBpsDevice = append(linuxBlkio.ThrottleReadBpsDevice, td)
|
||||||
|
+ case BlkioThrottleWriteBps:
|
||||||
|
+ linuxBlkio.ThrottleWriteBpsDevice = append(linuxBlkio.ThrottleWriteBpsDevice, td)
|
||||||
|
+ case BlkioThrottleReadIOPS:
|
||||||
|
+ linuxBlkio.ThrottleReadIOPSDevice = append(linuxBlkio.ThrottleReadIOPSDevice, td)
|
||||||
|
+ case BlkioThrottleWriteIOPS:
|
||||||
|
+ linuxBlkio.ThrottleWriteIOPSDevice = append(linuxBlkio.ThrottleWriteIOPSDevice, td)
|
||||||
|
+ default:
|
||||||
|
+ s.Logger().Errorf("the type of throtlle device：%s is not surpport", limit.Type)
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+ } else {
|
||||||
|
+ switch limit.Type {
|
||||||
|
+ case BlkioWeight:
|
||||||
|
+ weightUint16 := uint16(limit.Value)
|
||||||
|
+ linuxBlkio.Weight = &weightUint16
|
||||||
|
+ case BlkioLeafWeight:
|
||||||
|
+ weightLeafUint16 := uint16(limit.Value)
|
||||||
|
+ linuxBlkio.LeafWeight = &weightLeafUint16
|
||||||
|
+ default:
|
||||||
|
+ s.Logger().Errorf("the type:%s is not one of the supported types of blkio_weight and blkio_leaf_weight", limit.Type)
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ return &linuxBlkio
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go
|
||||||
|
index 96c4ef2..c35993e 100644
|
||||||
|
--- a/virtcontainers/pkg/annotations/annotations.go
|
||||||
|
+++ b/virtcontainers/pkg/annotations/annotations.go
|
||||||
|
@@ -21,6 +21,9 @@ const (
|
||||||
|
ContainerTypeKey = kataAnnotationsPrefix + "pkg.oci.container_type"
|
||||||
|
|
||||||
|
SandboxConfigPathKey = kataAnnotationsPrefix + "config_path"
|
||||||
|
+
|
||||||
|
+ // BlkioCgroupTypeKey is the annotation key to fetch sandbox blkio cgroup values
|
||||||
|
+ BlkioCgroupTypeKey = kataAnnotationsPrefix + "blkio_cgroup"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Annotations related to Hypervisor configuration
|
||||||
|
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
||||||
|
index e8ef41b..643753b 100644
|
||||||
|
--- a/virtcontainers/pkg/oci/utils.go
|
||||||
|
+++ b/virtcontainers/pkg/oci/utils.go
|
||||||
|
@@ -38,9 +38,10 @@ type annotationContainerType struct {
|
||||||
|
type annotationHandler func(value string) error
|
||||||
|
|
||||||
|
var annotationHandlerList = map[string]annotationHandler{
|
||||||
|
- vcAnnotations.StaticCPUTypeKey: validateSandboxCPU,
|
||||||
|
- vcAnnotations.StaticMemTypeKey: validateSandboxMem,
|
||||||
|
- vcAnnotations.SandboxDNSTypeKey: validateSandboxDNS,
|
||||||
|
+ vcAnnotations.StaticCPUTypeKey: validateSandboxCPU,
|
||||||
|
+ vcAnnotations.StaticMemTypeKey: validateSandboxMem,
|
||||||
|
+ vcAnnotations.SandboxDNSTypeKey: validateSandboxDNS,
|
||||||
|
+ vcAnnotations.BlkioCgroupTypeKey: validateBlkioCgroup,
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
@@ -1076,6 +1077,7 @@ func validateOtherSandboxAnnotations(annotation, value string) error {
|
||||||
|
// addOtherSandboxAnnotation add self defined annotation for sandbox
|
||||||
|
func addOtherSandboxAnnotation(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error {
|
||||||
|
otherSandboxAnnotationsKey := []string{
|
||||||
|
+ vcAnnotations.BlkioCgroupTypeKey,
|
||||||
|
vcAnnotations.StaticCPUTypeKey,
|
||||||
|
vcAnnotations.StaticMemTypeKey,
|
||||||
|
}
|
||||||
|
@@ -1137,6 +1139,48 @@ func validateSandboxDNS(value string) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+func validateBlkioCgroup(value string) error {
|
||||||
|
+ var linuxBlkioCgroup vc.BlkioCgroup
|
||||||
|
+ if err := json.Unmarshal([]byte(value), &linuxBlkioCgroup); err != nil {
|
||||||
|
+ return fmt.Errorf("blkio_cgroup Unmarshal error:%v, value is %s", err, value)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if linuxBlkioCgroup.Items == nil {
|
||||||
|
+ return fmt.Errorf("BlkioCgroup:%v fetch none data", linuxBlkioCgroup)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // pathNull used to judge path for blkio.weight/blkio.leaf_weight only once
|
||||||
|
+ pathNull := false
|
||||||
|
+ for _, item := range linuxBlkioCgroup.Items {
|
||||||
|
+ if pathNull {
|
||||||
|
+ return fmt.Errorf("too many paths for blkio.weight/blkio.leaf_weight")
|
||||||
|
+ }
|
||||||
|
+ if item.Path == "" && !pathNull {
|
||||||
|
+ pathNull = true
|
||||||
|
+ if len(item.Limits) > 2 || len(item.Limits) == 0 {
|
||||||
|
+ return fmt.Errorf("the format of values for blkio.weight or blkio.leaf_weight is wrong")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ for _, ioLimit := range item.Limits {
|
||||||
|
+ switch ioLimit.Type {
|
||||||
|
+ case vc.BlkioWeight:
|
||||||
|
+ // Blkio.weight, between 10 and 1000
|
||||||
|
+ if ioLimit.Value < 10 || ioLimit.Value > 1000 {
|
||||||
|
+ return fmt.Errorf("blkio.weight:%v must be between 10 and 1000", ioLimit.Value)
|
||||||
|
+ }
|
||||||
|
+ case vc.BlkioLeafWeight:
|
||||||
|
+ return fmt.Errorf("the blkio.leaf_weight is not supported now")
|
||||||
|
+ default:
|
||||||
|
+ return fmt.Errorf("the type of blkio device：%s is not surpport", ioLimit.Type)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ } else if _, _, err := utils.GetDeviceByPath(item.Path); err != nil {
|
||||||
|
+ return fmt.Errorf("failed to find major and minor of device %s: %v", item.Path, err)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func GetSandboxIDFromAnnotations(s *specs.Spec) (string, error) {
|
||||||
|
if s == nil {
|
||||||
|
return "", fmt.Errorf("spec is nil")
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index ca4e700..9284f99 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -2422,6 +2422,12 @@ func (s *Sandbox) setupHostCgroupsWithEmulator() error {
|
||||||
|
}
|
||||||
|
|
||||||
|
// limit blkio resource to "<path>"
|
||||||
|
+ blkioResources := specs.LinuxResources{
|
||||||
|
+ BlockIO: s.blockIOResource(),
|
||||||
|
+ }
|
||||||
|
+ if err := applyResourceLimit(&blkioResources, vcpuCgroupPath); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
|
||||||
|
// limit files resource
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go
|
||||||
|
index 36ac67a..d4dad40 100644
|
||||||
|
--- a/virtcontainers/utils/utils.go
|
||||||
|
+++ b/virtcontainers/utils/utils.go
|
||||||
|
@@ -370,6 +370,28 @@ func GetPhysicalCPUNumber() int {
|
||||||
|
return cpuNum
|
||||||
|
}
|
||||||
|
|
||||||
|
+// GetDeviceByPath returns the device No.
|
||||||
|
+func GetDeviceByPath(path string) (int64, int64, error) {
|
||||||
|
+ if path == "" {
|
||||||
|
+ return -1, -1, fmt.Errorf("Path cannot be empty")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ path, err := filepath.EvalSymlinks(path)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return -1, -1, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ stat := syscall.Stat_t{}
|
||||||
|
+ err = syscall.Stat(path, &stat)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return -1, -1, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ major := int64(stat.Rdev / 256)
|
||||||
|
+ minor := int64(stat.Rdev % 256)
|
||||||
|
+ return major, minor, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func RoundVCPUNumber(value string) (int, error) {
|
||||||
|
cpuNum, err := strconv.ParseFloat(value, 64)
|
||||||
|
if err != nil || cpuNum < minCPUs {
|
||||||
|
--
|
||||||
|
1.8.3.1
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,69 @@
|
|||||||
|
From faffb26c307556e1d84399060d7aef1753e9f41a Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Mon, 21 Sep 2020 21:52:48 -0400
|
||||||
|
Subject: [PATCH] runtime: fix sandboxRuntimeRootPath left problem
|
||||||
|
|
||||||
|
reason: If pod_sandbox type container deleted before pod_container
|
||||||
|
type containers in the sandbox, which leads to kata-runtime delete
|
||||||
|
pod_container process create a new sandboxRuntimeRootPath in the
|
||||||
|
/run/vc/sbs/<sandbox-id> dir.
|
||||||
|
|
||||||
|
So, we fix this problem by check sandboxRuntimeRootPath is exist
|
||||||
|
before fetchSandbox function is called.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
cli/delete.go | 2 +-
|
||||||
|
virtcontainers/sandbox.go | 18 ++++++++++++++++++
|
||||||
|
2 files changed, 19 insertions(+), 1 deletion(-)
|
||||||
|
|
||||||
|
diff --git a/cli/delete.go b/cli/delete.go
|
||||||
|
index 871ac40d..09552b9a 100644
|
||||||
|
--- a/cli/delete.go
|
||||||
|
+++ b/cli/delete.go
|
||||||
|
@@ -87,7 +87,7 @@ func delete(ctx context.Context, containerID string, force bool) error {
|
||||||
|
// container is deleted before pod_container type container, just return nil
|
||||||
|
// and let containerd delete container operations continue
|
||||||
|
if strings.Contains(err.Error(), "no such file or directory") {
|
||||||
|
- kataLog.Warnf("pod_sandbox deleted before pod_container: %v", err)
|
||||||
|
+ kataLog.Warnf("pod_sandbox container deleted before pod_container in the sandbox: %v", err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index de8652fc..9f5d3c08 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -729,6 +729,13 @@ func fetchSandbox(ctx context.Context, sandboxID string) (sandbox *Sandbox, err
|
||||||
|
|
||||||
|
var config SandboxConfig
|
||||||
|
|
||||||
|
+ // check sandbox runtime root path(for example: /run/vc/sbs/<sandbox-id>)
|
||||||
|
+ // exist or not before fetch sandbox config file
|
||||||
|
+ if !checkSandboxRuntimeRootPathExist(sandboxID) {
|
||||||
|
+ sandboxRuntimeRootPath := store.SandboxRuntimeRootPath(sandboxID)
|
||||||
|
+ return nil, fmt.Errorf("sandbox %s does not exist, %s no such file or directory", sandboxID,sandboxRuntimeRootPath)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
// Try to load sandbox config from old store at first.
|
||||||
|
c, ctx, err := loadSandboxConfigFromOldStore(ctx, sandboxID)
|
||||||
|
if err != nil {
|
||||||
|
@@ -2707,3 +2714,14 @@ func updateStaticSandboxResources(sandboxConfig *SandboxConfig) error {
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+// checkSandboxRuntimeRootPathExist check /run/vc/sbs/<sandbox-id>/ dir exist or not
|
||||||
|
+func checkSandboxRuntimeRootPathExist(sandboxID string) bool {
|
||||||
|
+ sandboxRuntimeRootPath := store.SandboxRuntimeRootPath(sandboxID)
|
||||||
|
+ _, err := os.Stat(sandboxRuntimeRootPath)
|
||||||
|
+ if os.IsNotExist(err) {
|
||||||
|
+ return false
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return true
|
||||||
|
+}
|
||||||
|
\ No newline at end of file
|
||||||
|
--
|
||||||
|
2.11.0
|
||||||
|
|
||||||
@ -0,0 +1,42 @@
|
|||||||
|
From e10fe6ecf2d895fe009a080bb34334ff63739f3e Mon Sep 17 00:00:00 2001
|
||||||
|
From: LiangZhang <zhangliang5@Huawei.com>
|
||||||
|
Date: Thu, 24 Sep 2020 14:38:04 +0800
|
||||||
|
Subject: [PATCH] runtime: fix invalid cmdline when start sandbox stratovirt
|
||||||
|
|
||||||
|
reason:
|
||||||
|
1. MemorySize is in type of uint32, value larger than 4096 will trigger overflow.
|
||||||
|
2. cannot disable seccomp now.
|
||||||
|
|
||||||
|
Signed-off-by: LiangZhang <zhangliang5@Huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/stratovirt.go | 7 ++++++-
|
||||||
|
1 file changed, 6 insertions(+), 1 deletion(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/stratovirt.go b/virtcontainers/stratovirt.go
|
||||||
|
index 7c156d5..6db8dc0 100644
|
||||||
|
--- a/virtcontainers/stratovirt.go
|
||||||
|
+++ b/virtcontainers/stratovirt.go
|
||||||
|
@@ -146,7 +146,7 @@ func (s *stratovirt) startSandbox(timeout int) error {
|
||||||
|
|
||||||
|
params = append(params, "-append "+s.getKernelCmdLine())
|
||||||
|
params = append(params, fmt.Sprintf("-smp %d", s.config.NumVCPUs))
|
||||||
|
- params = append(params, fmt.Sprintf("-m %d", s.config.MemorySize*1024*1024))
|
||||||
|
+ params = append(params, fmt.Sprintf("-m %d", uint64(s.config.MemorySize)*1024*1024))
|
||||||
|
params = append(params, fmt.Sprintf("-chardev id=charconsole0,path=%s", s.consolePath))
|
||||||
|
|
||||||
|
// add devices to cmdline
|
||||||
|
@@ -178,6 +178,11 @@ func (s *stratovirt) startSandbox(timeout int) error {
|
||||||
|
params = append(params, fmt.Sprintf("-D %s/stratovirt.log", dir))
|
||||||
|
}
|
||||||
|
|
||||||
|
+ // disable Seccomp
|
||||||
|
+ if s.sandbox.config.DisableGuestSeccomp {
|
||||||
|
+ params = append(params, "-disable-seccomp")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
s.Logger().Info("StratoVirt start with params: ", strings.Join(params, " "))
|
||||||
|
|
||||||
|
dir := filepath.Join(store.RunVMStoragePath(), s.id)
|
||||||
|
--
|
||||||
|
1.8.3.1
|
||||||
|
|
||||||
@ -0,0 +1,94 @@
|
|||||||
|
From b2b11cc8fb2144c13f4f9138b4420248ea200fb6 Mon Sep 17 00:00:00 2001
|
||||||
|
From: LiangZhang <zhangliang5@Huawei.com>
|
||||||
|
Date: Sun, 27 Sep 2020 18:10:18 +0800
|
||||||
|
Subject: [PATCH] runtime: fix cmd params of direct use stratovirt binary
|
||||||
|
|
||||||
|
reason: when directly use stratovirt binary, not through bash, it will cause problem.
|
||||||
|
|
||||||
|
Signed-off-by: LiangZhang <zhangliang5@Huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/stratovirt.go | 28 ++++++++++++++--------------
|
||||||
|
1 file changed, 14 insertions(+), 14 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/stratovirt.go b/virtcontainers/stratovirt.go
|
||||||
|
index 6db8dc0..020135e 100644
|
||||||
|
--- a/virtcontainers/stratovirt.go
|
||||||
|
+++ b/virtcontainers/stratovirt.go
|
||||||
|
@@ -133,21 +133,21 @@ func (s *stratovirt) startSandbox(timeout int) error {
|
||||||
|
defer span.Finish()
|
||||||
|
|
||||||
|
var params []string
|
||||||
|
- params = append(params, "-name "+fmt.Sprintf("sandbox-%s", s.id))
|
||||||
|
- params = append(params, "-api-channel unix:"+s.socketPath)
|
||||||
|
+ params = append(params, "-name", fmt.Sprintf("sandbox-%s", s.id))
|
||||||
|
+ params = append(params, "-api-channel", fmt.Sprintf("unix:%s", s.socketPath))
|
||||||
|
|
||||||
|
if kernelPath, err := s.config.KernelAssetPath(); err == nil {
|
||||||
|
- params = append(params, "-kernel "+kernelPath)
|
||||||
|
+ params = append(params, "-kernel", kernelPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
if initrdPath, err := s.config.InitrdAssetPath(); err == nil {
|
||||||
|
- params = append(params, "-initrd "+initrdPath)
|
||||||
|
+ params = append(params, "-initrd", initrdPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
- params = append(params, "-append "+s.getKernelCmdLine())
|
||||||
|
- params = append(params, fmt.Sprintf("-smp %d", s.config.NumVCPUs))
|
||||||
|
- params = append(params, fmt.Sprintf("-m %d", uint64(s.config.MemorySize)*1024*1024))
|
||||||
|
- params = append(params, fmt.Sprintf("-chardev id=charconsole0,path=%s", s.consolePath))
|
||||||
|
+ params = append(params, "-append", s.getKernelCmdLine())
|
||||||
|
+ params = append(params, "-smp", fmt.Sprintf("%d", s.config.NumVCPUs))
|
||||||
|
+ params = append(params, "-m", fmt.Sprintf("%d", uint64(s.config.MemorySize)*1024*1024))
|
||||||
|
+ params = append(params, "-chardev", fmt.Sprintf("id=charconsole0,path=%s", s.consolePath))
|
||||||
|
|
||||||
|
// add devices to cmdline
|
||||||
|
for _, d := range s.devices {
|
||||||
|
@@ -156,14 +156,14 @@ func (s *stratovirt) startSandbox(timeout int) error {
|
||||||
|
name := v.Name()
|
||||||
|
mac := v.HardwareAddr()
|
||||||
|
tapName := v.NetworkPair().TapInterface.TAPIface.Name
|
||||||
|
- params = append(params, fmt.Sprintf("-net id=%s,mac=%s,host_dev_name=%s", name, mac, tapName))
|
||||||
|
+ params = append(params, "-net", fmt.Sprintf("id=%s,mac=%s,host_dev_name=%s", name, mac, tapName))
|
||||||
|
case config.BlockDrive:
|
||||||
|
id := v.ID
|
||||||
|
path := v.File
|
||||||
|
- params = append(params, fmt.Sprintf("-drive id=%s,file=%s", id, path))
|
||||||
|
+ params = append(params, "-drive", fmt.Sprintf("id=%s,file=%s", id, path))
|
||||||
|
case types.VSock:
|
||||||
|
v.VhostFd.Close()
|
||||||
|
- params = append(params, fmt.Sprintf("-device vsock,id=vsock-id,guest-cid=%d", v.ContextID))
|
||||||
|
+ params = append(params, "-device", fmt.Sprintf("vsock,id=vsock-id,guest-cid=%d", v.ContextID))
|
||||||
|
default:
|
||||||
|
s.Logger().Error("Adding device type is unsupported")
|
||||||
|
}
|
||||||
|
@@ -175,7 +175,7 @@ func (s *stratovirt) startSandbox(timeout int) error {
|
||||||
|
// append logfile only on debug
|
||||||
|
if s.config.Debug {
|
||||||
|
dir := filepath.Join(store.RunVMStoragePath(), s.id)
|
||||||
|
- params = append(params, fmt.Sprintf("-D %s/stratovirt.log", dir))
|
||||||
|
+ params = append(params, "-D", fmt.Sprintf("%s/stratovirt.log", dir))
|
||||||
|
}
|
||||||
|
|
||||||
|
// disable Seccomp
|
||||||
|
@@ -183,8 +183,6 @@ func (s *stratovirt) startSandbox(timeout int) error {
|
||||||
|
params = append(params, "-disable-seccomp")
|
||||||
|
}
|
||||||
|
|
||||||
|
- s.Logger().Info("StratoVirt start with params: ", strings.Join(params, " "))
|
||||||
|
-
|
||||||
|
dir := filepath.Join(store.RunVMStoragePath(), s.id)
|
||||||
|
err := os.MkdirAll(dir, store.DirMode)
|
||||||
|
if err != nil {
|
||||||
|
@@ -205,6 +203,8 @@ func (s *stratovirt) startSandbox(timeout int) error {
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd := exec.CommandContext(s.ctx, binPath, params...)
|
||||||
|
+ s.Logger().Info("StratoVirt start with params: ", cmd)
|
||||||
|
+
|
||||||
|
if err := cmd.Run(); err != nil {
|
||||||
|
s.Logger().WithField("Error starting hypervisor, please check the params", err).Error()
|
||||||
|
return err
|
||||||
|
--
|
||||||
|
1.8.3.1
|
||||||
|
|
||||||
@ -0,0 +1,223 @@
|
|||||||
|
From babe7b3028d601a9b00aedda673e6a472f29ad07 Mon Sep 17 00:00:00 2001
|
||||||
|
From: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
Date: Mon, 28 Sep 2020 21:01:00 +0800
|
||||||
|
Subject: [PATCH] kata-runtime: retry inserting of CNI interface
|
||||||
|
|
||||||
|
reason: when netmon is enable, a interface is inserted into the
|
||||||
|
netns created by the sandbox, it should wait for the generating of
|
||||||
|
IP of the new interface, and get the ipv4 address
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
netmon/netmon.go | 128 +++++++++++++++++++++++++++++++++++++++++------
|
||||||
|
1 file changed, 112 insertions(+), 16 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/netmon/netmon.go b/netmon/netmon.go
|
||||||
|
index 94305ce2..57beacfb 100644
|
||||||
|
--- a/netmon/netmon.go
|
||||||
|
+++ b/netmon/netmon.go
|
||||||
|
@@ -52,7 +52,7 @@ var (
|
||||||
|
version = "unknown"
|
||||||
|
|
||||||
|
// For simplicity the code will only focus on IPv4 addresses for now.
|
||||||
|
- netlinkFamily = netlink.FAMILY_ALL
|
||||||
|
+ netlinkFamily = netlink.FAMILY_V4
|
||||||
|
|
||||||
|
storageParentPath = "/var/run/kata-containers/netmon/sbs"
|
||||||
|
)
|
||||||
|
@@ -70,7 +70,9 @@ type netmon struct {
|
||||||
|
storagePath string
|
||||||
|
sharedFile string
|
||||||
|
|
||||||
|
- netIfaces map[int]vcTypes.Interface
|
||||||
|
+ netIfaces map[int]vcTypes.Interface
|
||||||
|
+ plugedIfaces map[string]vcTypes.Interface
|
||||||
|
+ pendingRoutes map[string][]vcTypes.Route
|
||||||
|
|
||||||
|
linkUpdateCh chan netlink.LinkUpdate
|
||||||
|
linkDoneCh chan struct{}
|
||||||
|
@@ -148,15 +150,17 @@ func newNetmon(params netmonParams) (*netmon, error) {
|
||||||
|
}
|
||||||
|
|
||||||
|
n := &netmon{
|
||||||
|
- netmonParams: params,
|
||||||
|
- storagePath: filepath.Join(storageParentPath, params.sandboxID),
|
||||||
|
- sharedFile: filepath.Join(storageParentPath, params.sandboxID, sharedFile),
|
||||||
|
- netIfaces: make(map[int]vcTypes.Interface),
|
||||||
|
- linkUpdateCh: make(chan netlink.LinkUpdate),
|
||||||
|
- linkDoneCh: make(chan struct{}),
|
||||||
|
- rtUpdateCh: make(chan netlink.RouteUpdate),
|
||||||
|
- rtDoneCh: make(chan struct{}),
|
||||||
|
- netHandler: handler,
|
||||||
|
+ netmonParams: params,
|
||||||
|
+ storagePath: filepath.Join(storageParentPath, params.sandboxID),
|
||||||
|
+ sharedFile: filepath.Join(storageParentPath, params.sandboxID, sharedFile),
|
||||||
|
+ netIfaces: make(map[int]vcTypes.Interface),
|
||||||
|
+ plugedIfaces: make(map[string]vcTypes.Interface),
|
||||||
|
+ pendingRoutes: make(map[string][]vcTypes.Route),
|
||||||
|
+ linkUpdateCh: make(chan netlink.LinkUpdate),
|
||||||
|
+ linkDoneCh: make(chan struct{}),
|
||||||
|
+ rtUpdateCh: make(chan netlink.RouteUpdate),
|
||||||
|
+ rtDoneCh: make(chan struct{}),
|
||||||
|
+ netHandler: handler,
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.MkdirAll(n.storagePath, storageDirPerm); err != nil {
|
||||||
|
@@ -266,7 +270,6 @@ func convertInterface(linkAttrs *netlink.LinkAttrs, linkType string, addrs []net
|
||||||
|
}
|
||||||
|
|
||||||
|
var ipAddrs []*vcTypes.IPAddress
|
||||||
|
-
|
||||||
|
for _, addr := range addrs {
|
||||||
|
if addr.IPNet == nil {
|
||||||
|
continue
|
||||||
|
@@ -450,10 +453,24 @@ func (n *netmon) updateRoutes() error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
+ if len(netlinkRoutes) == 0 {
|
||||||
|
+ n.logger().Debug("get 0 routes")
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
|
||||||
|
// Translate them into Route structures.
|
||||||
|
routes := convertRoutes(netlinkRoutes)
|
||||||
|
|
||||||
|
+ // if the device of the routes have not be hotplug to guest,
|
||||||
|
+ // the update operation will be failed, pending them.
|
||||||
|
+ // For all routes are belong the same device, so we just need
|
||||||
|
+ // judge the device of the first route
|
||||||
|
+ if _, pluged := n.plugedIfaces[routes[0].Device]; !pluged {
|
||||||
|
+ n.pendingRoutes[routes[0].Device] = routes
|
||||||
|
+ n.logger().Infof("dev %s have not been added, pending:%v", routes[0].Device, routes)
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
// Update the routes through the Kata CLI.
|
||||||
|
return n.updateRoutesCLI(routes)
|
||||||
|
}
|
||||||
|
@@ -489,6 +506,9 @@ func (n *netmon) handleRTMNewLink(ev netlink.LinkUpdate) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+ // the link is usually not ready, and `sleep 3` is an empirical value.
|
||||||
|
+ time.Sleep(3 * time.Second)
|
||||||
|
+
|
||||||
|
// Check if the interface exist in the internal list.
|
||||||
|
if _, exist := n.netIfaces[int(ev.Index)]; exist {
|
||||||
|
n.logger().Debugf("Ignoring interface %s because already exist",
|
||||||
|
@@ -504,10 +524,26 @@ func (n *netmon) handleRTMNewLink(ev netlink.LinkUpdate) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
- // Get the list of IP addresses associated with this interface.
|
||||||
|
- addrs, err := n.netHandler.AddrList(ev.Link, netlinkFamily)
|
||||||
|
- if err != nil {
|
||||||
|
- return err
|
||||||
|
+ // In some scenarios, the ip have not prepared,so we should do some retries.
|
||||||
|
+ var addrs []netlink.Addr
|
||||||
|
+ var err error
|
||||||
|
+ for i := 0; i < 5; i++ {
|
||||||
|
+ // Get the list of IP addresses associated with this interface.
|
||||||
|
+ addrs, err = n.netHandler.AddrList(ev.Link, netlinkFamily)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ if len(addrs) > 0 {
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+ time.Sleep(500 * time.Millisecond)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // In some scenarios, the link reported by event can not found, do extras check here.
|
||||||
|
+ if n.checkLinkByHw(linkAttrs.HardwareAddr.String()) != true {
|
||||||
|
+ n.logger().Infof("Ignore %v because can not find link by HW %s",
|
||||||
|
+ linkAttrs.Name, linkAttrs.HardwareAddr.String())
|
||||||
|
+ return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert the interfaces in the appropriate structure format.
|
||||||
|
@@ -520,6 +556,17 @@ func (n *netmon) handleRTMNewLink(ev netlink.LinkUpdate) error {
|
||||||
|
|
||||||
|
// Add the interface to the internal list.
|
||||||
|
n.netIfaces[linkAttrs.Index] = iface
|
||||||
|
+ n.plugedIfaces[iface.Name] = iface
|
||||||
|
+
|
||||||
|
+ // The pending routes is preferentially selected.
|
||||||
|
+ if routes, ok := n.pendingRoutes[iface.Name]; ok {
|
||||||
|
+ n.logger().Infof("dev %s find pending routes:%v", iface.Name, routes)
|
||||||
|
+ err = n.updateRoutesCLI(routes)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ delete(n.pendingRoutes, iface.Name)
|
||||||
|
+ }
|
||||||
|
|
||||||
|
// Complete by updating the routes.
|
||||||
|
return n.updateRoutes()
|
||||||
|
@@ -556,6 +603,8 @@ func (n *netmon) handleRTMDelLink(ev netlink.LinkUpdate) error {
|
||||||
|
|
||||||
|
// Delete the interface from the internal list.
|
||||||
|
delete(n.netIfaces, linkAttrs.Index)
|
||||||
|
+ delete(n.plugedIfaces, iface.Name)
|
||||||
|
+ delete(n.pendingRoutes, iface.Name)
|
||||||
|
|
||||||
|
// Complete by updating the routes.
|
||||||
|
return n.updateRoutes()
|
||||||
|
@@ -640,6 +689,53 @@ func (n *netmon) handleEvents() (err error) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
+func linkByHwAddr(netHandle *netlink.Handle, hwAddr string) (netlink.Link, error) {
|
||||||
|
+ if netHandle == nil {
|
||||||
|
+ return nil, fmt.Errorf("no handler ")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ links, err := netHandle.LinkList()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ for _, link := range links {
|
||||||
|
+ if link == nil {
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ lAttrs := link.Attrs()
|
||||||
|
+ if lAttrs == nil {
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if lAttrs.HardwareAddr == nil {
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if lAttrs.HardwareAddr.String() == hwAddr {
|
||||||
|
+ return link, nil
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil, fmt.Errorf("could not find the link corresponding to HwAddr %s", hwAddr)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (n *netmon) checkLinkByHw(hw string) bool {
|
||||||
|
+ netHandle, err := netlink.NewHandle(unix.NETLINK_ROUTE)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return false
|
||||||
|
+ }
|
||||||
|
+ defer netHandle.Delete()
|
||||||
|
+
|
||||||
|
+ link, err := linkByHwAddr(netHandle, hw)
|
||||||
|
+ if err != nil || link == nil {
|
||||||
|
+ return false
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return true
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func main() {
|
||||||
|
// Parse parameters.
|
||||||
|
params := parseOptions()
|
||||||
|
--
|
||||||
|
2.23.0
|
||||||
|
|
||||||
@ -0,0 +1,43 @@
|
|||||||
|
From f0d2f8a19956045b4b53ac5f2c4b59940016ca41 Mon Sep 17 00:00:00 2001
|
||||||
|
From: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
Date: Fri, 9 Oct 2020 16:02:27 +0800
|
||||||
|
Subject: [PATCH] kata-runtime: support using CNI plugin to insert mutiple
|
||||||
|
network interfaces at the same time
|
||||||
|
|
||||||
|
reason: support using CNI plugin to insert mutiple network interfaces at the same time
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
netmon/netmon.go | 16 +++++++++++-----
|
||||||
|
1 file changed, 11 insertions(+), 5 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/netmon/netmon.go b/netmon/netmon.go
|
||||||
|
index 57beacfb..a519e5ba 100644
|
||||||
|
--- a/netmon/netmon.go
|
||||||
|
+++ b/netmon/netmon.go
|
||||||
|
@@ -463,11 +463,17 @@ func (n *netmon) updateRoutes() error {
|
||||||
|
|
||||||
|
// if the device of the routes have not be hotplug to guest,
|
||||||
|
// the update operation will be failed, pending them.
|
||||||
|
- // For all routes are belong the same device, so we just need
|
||||||
|
- // judge the device of the first route
|
||||||
|
- if _, pluged := n.plugedIfaces[routes[0].Device]; !pluged {
|
||||||
|
- n.pendingRoutes[routes[0].Device] = routes
|
||||||
|
- n.logger().Infof("dev %s have not been added, pending:%v", routes[0].Device, routes)
|
||||||
|
+ var pendingFlag bool
|
||||||
|
+ for _, route := range routes{
|
||||||
|
+ if _, pluged := n.plugedIfaces[route.Device]; !pluged {
|
||||||
|
+ pendingFlag = true
|
||||||
|
+ n.pendingRoutes[route.Device] = append(n.pendingRoutes[route.Device],route)
|
||||||
|
+ n.logger().Infof("dev %s have not been added, pending:%v", route.Device, n.pendingRoutes[route.Device])
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // find pending route
|
||||||
|
+ if pendingFlag {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
--
|
||||||
|
2.23.0
|
||||||
|
|
||||||
@ -0,0 +1,90 @@
|
|||||||
|
From bac206ee5b4ccb90fd8c06c0b6244ba163ac82b9 Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu2@huawei.com>
|
||||||
|
Date: Sun, 15 Nov 2020 21:31:49 +0800
|
||||||
|
Subject: [PATCH] kata-runtime: fix get sandbox cpu resources problem
|
||||||
|
|
||||||
|
reason: If sandox_cgroup_with_emulator config is enabled,
|
||||||
|
kata-runtime should get cpu resources by cpuResourcesWithEmulator
|
||||||
|
func instead of the original cpuResource func.
|
||||||
|
|
||||||
|
Signed-off-by: holyfei <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/cgroups.go | 2 ++
|
||||||
|
virtcontainers/sandbox.go | 34 +++++++++++++++++++++++++++++++++-
|
||||||
|
2 files changed, 35 insertions(+), 1 deletion(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/cgroups.go b/virtcontainers/cgroups.go
|
||||||
|
index 1b6b04da..21708ebf 100644
|
||||||
|
--- a/virtcontainers/cgroups.go
|
||||||
|
+++ b/virtcontainers/cgroups.go
|
||||||
|
@@ -40,6 +40,8 @@ const (
|
||||||
|
|
||||||
|
defaultMinFilesLimit uint64 = 1024
|
||||||
|
defaultMaxContainers uint64 = 200
|
||||||
|
+ defaultCPUPeriod uint64 = 1000000
|
||||||
|
+ defaultCPUShares uint64 = 1024
|
||||||
|
|
||||||
|
procFileMaxPath = "/proc/sys/fs/file-max"
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index 9f5d3c08..d3c64b4f 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -14,6 +14,7 @@ import (
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
+ "strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"syscall"
|
||||||
|
@@ -2351,6 +2352,37 @@ func (s *Sandbox) cpuResources() *specs.LinuxCPU {
|
||||||
|
return validCPUResources(cpu)
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (s *Sandbox) cpuResourcesWithEmulator() *specs.LinuxCPU {
|
||||||
|
+ // Use default period and quota if they are not specified.
|
||||||
|
+ // Container will inherit the constraints from its parent.
|
||||||
|
+ quota := int64(0)
|
||||||
|
+ period := uint64(0)
|
||||||
|
+ shares := uint64(0)
|
||||||
|
+
|
||||||
|
+ cpu := &specs.LinuxCPU{
|
||||||
|
+ Quota: "a,
|
||||||
|
+ Period: &period,
|
||||||
|
+ Shares: &shares,
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ period = defaultCPUPeriod
|
||||||
|
+
|
||||||
|
+ cpuValue, exist := s.config.Annotations[annotations.StaticCPUTypeKey]
|
||||||
|
+ if exist {
|
||||||
|
+ // If sandbox_cpu is set, we have validate at first, so we don't need to check it again
|
||||||
|
+ cpuNum, _ := strconv.ParseFloat(cpuValue, 64)
|
||||||
|
+ quota = int64(cpuNum * float64(period))
|
||||||
|
+ shares = uint64(cpuNum * float64(defaultCPUShares))
|
||||||
|
+ } else {
|
||||||
|
+ // If sandbox_cpu is not set, we use the hypervisor's cpu number as cpu limit
|
||||||
|
+ hypervisorConfig := s.hypervisor.hypervisorConfig()
|
||||||
|
+ quota = int64(uint64(hypervisorConfig.NumVCPUs) * period)
|
||||||
|
+ shares = uint64(hypervisorConfig.NumVCPUs) * defaultCPUShares
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return cpu
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// setupSandboxCgroup creates and joins sandbox cgroups for the sandbox config
|
||||||
|
func (s *Sandbox) setupSandboxCgroup() error {
|
||||||
|
var err error
|
||||||
|
@@ -2424,7 +2456,7 @@ func (s *Sandbox) setupHostCgroupsWithEmulator() error {
|
||||||
|
// limit cpu to "<path>/vcpu"
|
||||||
|
vcpuCgroupPath := filepath.Join(s.state.CgroupPath, vcpuCgroupName)
|
||||||
|
vcpuResources := specs.LinuxResources{
|
||||||
|
- CPU: s.cpuResources(),
|
||||||
|
+ CPU: s.cpuResourcesWithEmulator(),
|
||||||
|
}
|
||||||
|
if err := applyResourceLimit(&vcpuResources, vcpuCgroupPath); err != nil {
|
||||||
|
return err
|
||||||
|
--
|
||||||
|
2.11.0
|
||||||
|
|
||||||
BIN
runtime/runtime-1.11.1.tar.gz
Normal file
BIN
runtime/runtime-1.11.1.tar.gz
Normal file
Binary file not shown.
63
runtime/series.conf
Normal file
63
runtime/series.conf
Normal file
@ -0,0 +1,63 @@
|
|||||||
|
0001-qmp-fix-kata-runtime-hungs-when-qemu-process-is-D-T-.patch
|
||||||
|
0002-kata-runtime-fix-kata-runtime-skip-read-lines-in-pro.patch
|
||||||
|
0003-kata-runtime-fix-kata-proxy-process-left-problem.patch
|
||||||
|
0004-kata-runtime-keep-the-process-name-of-qemu-same-as-c.patch
|
||||||
|
0005-cgroups-increase-delete-cgroup-retry-times.patch
|
||||||
|
0006-kata-runtime-fix-umount-container-rootfs-dir-return-.patch
|
||||||
|
0007-kata-runtime-enhance-reliability-when-kata-related-p.patch
|
||||||
|
0008-kata-runtime-fix-kata-runtime-resource-left-problem.patch
|
||||||
|
0009-kata-runtime-add-kata-runtime-global-flag-debug.patch
|
||||||
|
0010-kata-runtime-fix-kata-shim-pid-reused-problem.patch
|
||||||
|
0011-kata-runtime-check-the-process-info-before-send-SIGK.patch
|
||||||
|
0012-kata-runtime-truncate-the-log.json-file-before-kata-.patch
|
||||||
|
0013-kata-runtime-get-container-info-by-containerID-prefi.patch
|
||||||
|
0014-kata-runtime-add-self-defined-annotations-framework.patch
|
||||||
|
0015-kata-runtime-add-reuse-hypervisor-cpu-and-memory-fea.patch
|
||||||
|
0016-virtcontainers-fix-hotplug-huge-size-memory-cause-ag.patch
|
||||||
|
0017-kata-runtime-validate-sandbox-cpu-and-memory-size.patch
|
||||||
|
0018-sandbox-Stop-and-clean-up-containers-that-fail-to-cr.patch
|
||||||
|
0019-virtcontainers-fix-delete-sandbox-failed-problem.patch
|
||||||
|
0020-virtcontainers-add-enable_cpu_memory_hotplug-config-.patch
|
||||||
|
0021-kata-runtime-add-sandbox_cpu-and-sandbox_mem-annotat.patch
|
||||||
|
0022-kata-runtime-skip-go-version-check-and-do-not-build-.patch
|
||||||
|
0023-kata-runtime-set-PCIBridgeMaxCapacity-limit-to-25.patch
|
||||||
|
0024-kata-runtime-support-hotplug-tap-interface-into-kata.patch
|
||||||
|
0025-network-keep-list-ifaces-result-compatible-with-cni.patch
|
||||||
|
0026-network-add-enable_compat_old_cni-config.patch
|
||||||
|
0027-network-add-more-strict-check-for-input-network-inte.patch
|
||||||
|
0028-network-add-kata-network-add-route-subcommand.patch
|
||||||
|
0029-network-add-kata-network-del-route-subcommand.patch
|
||||||
|
0030-network-kata-network-list-routes-support-display-com.patch
|
||||||
|
0031-device_mangaer-check-VFIO-when-create-device.patch
|
||||||
|
0032-network-add-more-detail-usage-for-update-routes-subc.patch
|
||||||
|
0033-network-do-not-delete-the-exist-tap-device-in-the-ho.patch
|
||||||
|
0034-kata-runtime-add-kata-ipvs-command.patch
|
||||||
|
0035-device-mount-blockdevices-in-the-guest-VM.patch
|
||||||
|
0036-mount-limit-the-maximum-number-of-virtio-scsi-bus-sl.patch
|
||||||
|
0037-runtime-add-IPVS-test.patch
|
||||||
|
0038-pcie-using-pcie-root-port-driver-to-hotplug-device-i.patch
|
||||||
|
0039-storage-add-storage-common-functions-and-structs.patch
|
||||||
|
0040-storage-add-go-tests-for-storage.patch
|
||||||
|
0041-storage-mount-nfs-and-gpath-with-given-annotation.patch
|
||||||
|
0042-kata-runtime-do-not-ignore-updateInterface-return-er.patch
|
||||||
|
0043-kata-runtime-support-add-hypervisor-global-parameter.patch
|
||||||
|
0044-network-support-dpdk-vhost_user-net-device.patch
|
||||||
|
0045-network-support-set-dns.patch
|
||||||
|
0046-network-support-multiqueue-when-inserting-interface-.patch
|
||||||
|
0047-network-add-support-to-get-network-stats-of-vm-throu.patch
|
||||||
|
0048-console-fix-the-file-resource-leak.patch
|
||||||
|
0049-container-fix-the-write-operation-transparently-tran.patch
|
||||||
|
0050-runtime-add-kata-network-upate-iface-subcommand.patch
|
||||||
|
0051-network-fix-del-iface-doesn-t-delete-the-tap-interfa.patch
|
||||||
|
0052-runtime-add-support-of-new-sandbox-StratoVirt.patch
|
||||||
|
0053-kata-runtime-add-interface-for-host-cgroup.patch
|
||||||
|
0054-kata-runtime-add-sandbox-cgroup-with-vcpu-and-emulat.patch
|
||||||
|
0055-kata_runtime-support-host-cgroup-with-emulator-polic.patch
|
||||||
|
0056-kata_runtime-support-the-blkio-in-host-cgroups.patch
|
||||||
|
0057-kata-runtime-support-files-limit-in-host-cgroups.patch
|
||||||
|
0058-runtime-fix-sandboxRuntimeRootPath-left-problem.patch
|
||||||
|
0059-runtime-fix-invalid-cmdline-when-start-sandbox-stratovirt.patch
|
||||||
|
0060-runtime-fix-cmd-params-of-direct-use-stratovirt-bina.patch
|
||||||
|
0061-kata-runtime-retry-inserting-of-CNI-interface.patch
|
||||||
|
0062-kata-runtime-support-using-CNI-plugin-to-insert-muti.patch
|
||||||
|
0063-kata-runtime-fix-get-sandbox-cpu-resources-problem.patch
|
||||||
21
shim/apply-patches
Executable file
21
shim/apply-patches
Executable file
@ -0,0 +1,21 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
if [[ -f ./patch_flag ]];then
|
||||||
|
echo "shim patched!"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
tar -zxvf shim-*.tar.gz
|
||||||
|
cp -fr ./shim-*/* ./
|
||||||
|
rm -rf ./shim-*
|
||||||
|
cat ./series.conf | while read line
|
||||||
|
do
|
||||||
|
if [[ $line == '' || $line =~ ^\s*# ]]; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
echo "====patch $line======"
|
||||||
|
pwd
|
||||||
|
patch -p1 -F1 -s < ./patches/$line
|
||||||
|
done
|
||||||
|
|
||||||
|
touch ./patch_flag
|
||||||
@ -0,0 +1,38 @@
|
|||||||
|
From f5d031ef7d3437a05fe41f80aaf0e59ac54a575b Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei9 <jiangpengfei9@huawei.com>
|
||||||
|
Date: Mon, 1 Apr 2019 12:35:59 -0400
|
||||||
|
Subject: [PATCH 2/2] kata-shim: fix kata-shim process wait long time
|
||||||
|
to exit problem
|
||||||
|
|
||||||
|
reason: If containerd-shim process is killed and call kata-runtime delete -f command
|
||||||
|
to delete container, it will wait 10s for kata-shim process exit in the waitForShim
|
||||||
|
function, however 10s wait time is too long in the case of containerd-shim and containerd
|
||||||
|
process is killed.So when containerd-shim process is killed, kata-shim process will
|
||||||
|
receive a SIGHUP signal, we can handle this signal as exit signal which makes kata-shim
|
||||||
|
process exit.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei9 <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
shim.go | 6 ++++++
|
||||||
|
1 file changed, 6 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/shim.go b/shim.go
|
||||||
|
index a93a00d..90172fe 100644
|
||||||
|
--- a/shim.go
|
||||||
|
+++ b/shim.go
|
||||||
|
@@ -134,6 +134,12 @@ func (s *shim) handleSignals(ctx context.Context, tty *os.File) chan os.Signal {
|
||||||
|
backtrace()
|
||||||
|
}
|
||||||
|
|
||||||
|
+ // if containerd-shim process is killed, kata-shim will receive SIGHUP signal
|
||||||
|
+ if sysSig == syscall.SIGHUP {
|
||||||
|
+ logger().WithField("signal", sig).Warn("received SIGHUP signal")
|
||||||
|
+ os.Exit(1)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
// forward this signal to container
|
||||||
|
_, err := s.agent.SignalProcess(s.ctx, &pb.SignalProcessRequest{
|
||||||
|
ContainerId: s.containerID,
|
||||||
|
--
|
||||||
|
1.8.3.1
|
||||||
|
|
||||||
1
shim/series.conf
Normal file
1
shim/series.conf
Normal file
@ -0,0 +1 @@
|
|||||||
|
0001-kata-shim-fix-kata-shim-process-wait-long-tim.patch
|
||||||
BIN
shim/shim-1.11.1.tar.gz
Normal file
BIN
shim/shim-1.11.1.tar.gz
Normal file
Binary file not shown.
Loading…
x
Reference in New Issue
Block a user