kata-containers: modify kata-containers version
Fix #I4KI81 reason: modify kata-containers version and update it to 1.11.1 Signed-off-by: holyfei <yangfeiyu20092010@163.com>
This commit is contained in:
holyfei
parent
ef2dcd4faf
commit
c709612f2a
BIN
agent/agent-1.11.1.tar.gz
Normal file
BIN
agent/agent-1.11.1.tar.gz
Normal file
Binary file not shown.
20
agent/apply-patches
Executable file
20
agent/apply-patches
Executable file
@ -0,0 +1,20 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
if [[ -f ./patch_flag ]];then
|
||||||
|
echo "agent patched!"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
tar -zxvf agent-1.11.1.tar.gz
|
||||||
|
cp -fr ./agent-1.11.1/* ./
|
||||||
|
rm -rf ./agent-1.11.1
|
||||||
|
cat ./series.conf | while read line
|
||||||
|
do
|
||||||
|
if [[ $line == '' || $line =~ ^\s*# ]]; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
echo "====patch $line======"
|
||||||
|
pwd
|
||||||
|
patch -p1 -F1 -s < ./patches/$line
|
||||||
|
done
|
||||||
|
touch ./patch_flag
|
||||||
@ -0,0 +1,234 @@
|
|||||||
|
From ac1d7806f8de2f8ca393df08a9c62d1045c4afdc Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 11 Dec 2018 18:27:02 -0500
|
||||||
|
Subject: [PATCH 01/16] agent: add agent.netlink_recv_buf_size flag to set
|
||||||
|
netlink recv buf size
|
||||||
|
|
||||||
|
fixes: #813
|
||||||
|
|
||||||
|
reason: If hotplug huge size memory(for example 128GB) into guest,
|
||||||
|
kernel will produce a lot of memory add uevents and send to netlink socket,
|
||||||
|
however netlink socket default receive buffer size is 4KB, which is too small
|
||||||
|
to receive all memory add uevents.
|
||||||
|
Since hotplug huge size memory is not common case, so we consider add an agent
|
||||||
|
flag agent.netlink_recv_buf_size to set netlink socket recv buffer size.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
README.md | 13 +++++++++++++
|
||||||
|
agent.go | 10 +++++++++-
|
||||||
|
config.go | 15 ++++++++++++++
|
||||||
|
config_test.go | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||||
|
pkg/uevent/uevent.go | 15 +++++++++++---
|
||||||
|
5 files changed, 104 insertions(+), 4 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/README.md b/README.md
|
||||||
|
index cec65a4..16f96a4 100644
|
||||||
|
--- a/README.md
|
||||||
|
+++ b/README.md
|
||||||
|
@@ -98,6 +98,19 @@ The pipe's capacity for stdout/stderr can be modified by specifying the `agent.c
|
||||||
|
to the guest kernel command line. For example, `agent.container_pipe_size=2097152` will set the stdout and stderr
|
||||||
|
pipes to 2097152 bytes.
|
||||||
|
|
||||||
|
+## Uevent Netlink Socket Receive Buffer Size
|
||||||
|
+
|
||||||
|
+When hotplugging a huge size memory into the Kata VM, the kernel in the VM will produce a lot of memory object add
|
||||||
|
+uevents and send all these uevents to Kata agent by netlink socket. However, default netlink socket receive buffer
|
||||||
|
+size is 4KB, which is too small and can only hold 256 memory add uevents. If memory add uevents number is larger
|
||||||
|
+than 256, the left uevents can not be received and processed by Kata agent.
|
||||||
|
+
|
||||||
|
+The uevent netlink socket receive buffer size can be modified by specifying the `agent.netlink_recv_buf_size` flag
|
||||||
|
+to the guest kernel command line. For example, `agent.netlink_recv_buf_size=2MB` will set the uevent netlink socket
|
||||||
|
+receive buffer size to 2MB value. `agent.netlink_recv_buf_size` valid value range is `[4KB ~ 4MB]` and value can be
|
||||||
|
+set in human-readable memory format or pure digital number format(default memory unit is byte).
|
||||||
|
+
|
||||||
|
+
|
||||||
|
[1]: https://github.com/firecracker-microvm/firecracker/blob/master/docs/vsock.md
|
||||||
|
[2]: https://golang.org/pkg/time/#ParseDuration
|
||||||
|
[3]: http://man7.org/linux/man-pages/man7/pipe.7.html
|
||||||
|
diff --git a/agent.go b/agent.go
|
||||||
|
index 2d2c293..c1cac08 100644
|
||||||
|
--- a/agent.go
|
||||||
|
+++ b/agent.go
|
||||||
|
@@ -190,6 +190,14 @@ var unifiedCgroupHierarchy = false
|
||||||
|
// Size in bytes of the stdout/stderr pipes created for each container.
|
||||||
|
var containerPipeSize = uint32(0)
|
||||||
|
|
||||||
|
+const (
|
||||||
|
+ minNetlinkSockRecvBufSize = 4 * 1024
|
||||||
|
+ maxNetlinkSockRecvBufSize = 4 * 1024 * 1024
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+// Size in bytes of the netlink socket recv buf size
|
||||||
|
+var netlinkSockRecvBufSize = uint32(0)
|
||||||
|
+
|
||||||
|
// commType is used to denote the communication channel type used.
|
||||||
|
type commType int
|
||||||
|
|
||||||
|
@@ -708,7 +716,7 @@ func (s *sandbox) waitForStopServer() {
|
||||||
|
func (s *sandbox) listenToUdevEvents() {
|
||||||
|
fieldLogger := agentLog.WithField("subsystem", "udevlistener")
|
||||||
|
|
||||||
|
- uEvHandler, err := uevent.NewHandler()
|
||||||
|
+ uEvHandler, err := uevent.NewHandler(netlinkSockRecvBufSize)
|
||||||
|
if err != nil {
|
||||||
|
fieldLogger.Warnf("Error starting uevent listening loop %s", err)
|
||||||
|
return
|
||||||
|
diff --git a/config.go b/config.go
|
||||||
|
index 4530096..6c7d473 100644
|
||||||
|
--- a/config.go
|
||||||
|
+++ b/config.go
|
||||||
|
@@ -7,11 +7,13 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
+ "fmt"
|
||||||
|
"io/ioutil"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
+ "github.com/docker/go-units"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
"google.golang.org/grpc/codes"
|
||||||
|
grpcStatus "google.golang.org/grpc/status"
|
||||||
|
@@ -29,6 +31,7 @@ const (
|
||||||
|
hotplugTimeoutFlag = optionPrefix + "hotplug_timeout"
|
||||||
|
unifiedCgroupHierarchyFlag = optionPrefix + "unified_cgroup_hierarchy"
|
||||||
|
containerPipeSizeFlag = optionPrefix + "container_pipe_size"
|
||||||
|
+ netlinkSockRecvBufSizeFlag = optionPrefix + "netlink_recv_buf_size"
|
||||||
|
traceModeStatic = "static"
|
||||||
|
traceModeDynamic = "dynamic"
|
||||||
|
traceTypeIsolated = "isolated"
|
||||||
|
@@ -155,6 +158,18 @@ func parseCmdlineOption(option string) error {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
unifiedCgroupHierarchy = flag
|
||||||
|
+ case netlinkSockRecvBufSizeFlag:
|
||||||
|
+ bufSizeInBytes, err := units.RAMInBytes(split[valuePosition])
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if bufSizeInBytes < minNetlinkSockRecvBufSize || bufSizeInBytes > maxNetlinkSockRecvBufSize {
|
||||||
|
+ return fmt.Errorf("invalid netlink socket recv buf size: %d (valid size range %s-%s bytes)", bufSizeInBytes,
|
||||||
|
+ units.BytesSize(minNetlinkSockRecvBufSize), units.BytesSize(maxNetlinkSockRecvBufSize))
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ netlinkSockRecvBufSize = uint32(bufSizeInBytes)
|
||||||
|
default:
|
||||||
|
if strings.HasPrefix(split[optionPosition], optionPrefix) {
|
||||||
|
return grpcStatus.Errorf(codes.NotFound, "Unknown option %s", split[optionPosition])
|
||||||
|
diff --git a/config_test.go b/config_test.go
|
||||||
|
index 2a23133..f40f17a 100644
|
||||||
|
--- a/config_test.go
|
||||||
|
+++ b/config_test.go
|
||||||
|
@@ -486,3 +486,58 @@ func TestParseCmdlineOptionContainerPipeSize(t *testing.T) {
|
||||||
|
assert.Equal(d.expectedContainerPipeSize, containerPipeSize, "test %d (%+v)", i, d)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func TestParseCmdlineOptionNetlinkSockRecvBufSize(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+
|
||||||
|
+ type testData struct {
|
||||||
|
+ option string
|
||||||
|
+ shouldErr bool
|
||||||
|
+ expectedNetlinkSockRecvBufSize uint32
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ data := []testData{
|
||||||
|
+ {"", false, 0},
|
||||||
|
+ {"netlink_recv_buf_siz", false, 0},
|
||||||
|
+ {"netlink_recv_buf_size", false, 0},
|
||||||
|
+ {"netlink_recv_buf_size=", false, 0},
|
||||||
|
+ {"netlink_recv_buf_size=4096", false, 0},
|
||||||
|
+ {"netlink_recv_buf_size=4KB", false, 0},
|
||||||
|
+ {"agent.netlink_recv_buf_size=", true, 0},
|
||||||
|
+ {"agent.netlink_recv_buf_size=foobar", true, 0},
|
||||||
|
+ {"agent.netlink_recv_buf_size=-1", true, 0},
|
||||||
|
+ {"agent.netlink_recv_buf_size=0", true, 0},
|
||||||
|
+ {"agent.netlink_recv_buf_size=100", true, 0},
|
||||||
|
+ {"agent.netlink_recv_buf_size=3KB", true, 0},
|
||||||
|
+ {"agent.netlink_recv_buf_size=3.6KB", true, 0},
|
||||||
|
+ {"agent.netlink_recv_buf_size=4095", true, 0},
|
||||||
|
+ {"agent.netlink_recv_buf_size=4096xB", true, 0},
|
||||||
|
+ {"agent.netlink_recv_buf_size=4096", false, 4096},
|
||||||
|
+ {"agent.netlink_recv_buf_size=4097", false, 4097},
|
||||||
|
+ {"agent.netlink_recv_buf_size=4096.0", false, 4096},
|
||||||
|
+ {"agent.netlink_recv_buf_size=1024KB", false, 1048576},
|
||||||
|
+ {"agent.netlink_recv_buf_size=1MB", false, 1048576},
|
||||||
|
+ {"agent.netlink_recv_buf_size=4194303", false, 4194303},
|
||||||
|
+ {"agent.netlink_recv_buf_size=3.999MB", false, 4193255},
|
||||||
|
+ {"agent.netlink_recv_buf_size=4194304", false, 4194304},
|
||||||
|
+ {"agent.netlink_recv_buf_size=4MB", false, 4194304},
|
||||||
|
+ {"agent.netlink_recv_buf_size=4.001MB", true, 0},
|
||||||
|
+ {"agent.netlink_recv_buf_size=4194305", true, 0},
|
||||||
|
+ {"agent.netlink_recv_buf_size=100MB", true, 0},
|
||||||
|
+ {"agent.netlink_recv_buf_size=1GB", true, 0},
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ for i, d := range data {
|
||||||
|
+ // reset the netlink socket recv buffer size
|
||||||
|
+ netlinkSockRecvBufSize = 0
|
||||||
|
+
|
||||||
|
+ err := parseCmdlineOption(d.option)
|
||||||
|
+ if d.shouldErr {
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ } else {
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ assert.Equal(d.expectedNetlinkSockRecvBufSize, netlinkSockRecvBufSize, "test %d (%+v)", i, d)
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
diff --git a/pkg/uevent/uevent.go b/pkg/uevent/uevent.go
|
||||||
|
index fc2c127..fa84086 100644
|
||||||
|
--- a/pkg/uevent/uevent.go
|
||||||
|
+++ b/pkg/uevent/uevent.go
|
||||||
|
@@ -10,6 +10,7 @@ import (
|
||||||
|
"bufio"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
+ "syscall"
|
||||||
|
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
"google.golang.org/grpc/codes"
|
||||||
|
@@ -33,7 +34,7 @@ type ReaderCloser struct {
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewReaderCloser returns an io.ReadCloser handle for uevent.
|
||||||
|
-func NewReaderCloser() (io.ReadCloser, error) {
|
||||||
|
+func NewReaderCloser(netlinkRecvBufSize uint32) (io.ReadCloser, error) {
|
||||||
|
nl := unix.SockaddrNetlink{
|
||||||
|
Family: unix.AF_NETLINK,
|
||||||
|
// Passing Pid as 0 here allows the kernel to take care of assigning
|
||||||
|
@@ -47,6 +48,14 @@ func NewReaderCloser() (io.ReadCloser, error) {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
+ // If netlinkRecvBufSize > 0, set netlink socket recv buffer size to netlinkRecvBufSize
|
||||||
|
+ if netlinkRecvBufSize > 0 {
|
||||||
|
+ err = unix.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_RCVBUFFORCE, int(netlinkRecvBufSize))
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
if err := unix.Bind(fd, &nl); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
@@ -85,8 +94,8 @@ type Handler struct {
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewHandler returns a uevent handler.
|
||||||
|
-func NewHandler() (*Handler, error) {
|
||||||
|
- rdCloser, err := NewReaderCloser()
|
||||||
|
+func NewHandler(netlinkRecvBufSize uint32) (*Handler, error) {
|
||||||
|
+ rdCloser, err := NewReaderCloser(netlinkRecvBufSize)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,686 @@
|
|||||||
|
From 13f54c768dcd7bf982dde8e57fb5cd624fedf5bc Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Mon, 17 Aug 2020 11:23:55 +0800
|
||||||
|
Subject: [PATCH 02/16] network: support update routes incrementally
|
||||||
|
|
||||||
|
reason: add increment flag in the UpdateRoutesRequest to
|
||||||
|
support upate routes incrementally to improve the efficiency.
|
||||||
|
|
||||||
|
kata-network add-route and del-route needs this feature.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
grpc.go | 2 +-
|
||||||
|
network.go | 74 ++++++++-
|
||||||
|
network_test.go | 16 +-
|
||||||
|
protocols/grpc/agent.pb.go | 402 +++++++++++++++++++++++++--------------------
|
||||||
|
protocols/grpc/agent.proto | 1 +
|
||||||
|
5 files changed, 300 insertions(+), 195 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/grpc.go b/grpc.go
|
||||||
|
index 886661b..8fe8217 100644
|
||||||
|
--- a/grpc.go
|
||||||
|
+++ b/grpc.go
|
||||||
|
@@ -1556,7 +1556,7 @@ func (a *agentGRPC) UpdateInterface(ctx context.Context, req *pb.UpdateInterface
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *agentGRPC) UpdateRoutes(ctx context.Context, req *pb.UpdateRoutesRequest) (*pb.Routes, error) {
|
||||||
|
- return a.sandbox.updateRoutes(nil, req.Routes)
|
||||||
|
+ return a.sandbox.updateRoutes(nil, req.Routes, req.Increment)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *agentGRPC) ListInterfaces(ctx context.Context, req *pb.ListInterfacesRequest) (*pb.Interfaces, error) {
|
||||||
|
diff --git a/network.go b/network.go
|
||||||
|
index 64a16a9..02e28cb 100644
|
||||||
|
--- a/network.go
|
||||||
|
+++ b/network.go
|
||||||
|
@@ -398,7 +398,7 @@ func (s *sandbox) deleteRoutes(netHandle *netlink.Handle) error {
|
||||||
|
// state which matches the requested routes. In doing this, preesxisting non-loopback routes will be
|
||||||
|
// removed from the network. If an error occurs, this function returns the list of routes in
|
||||||
|
// gRPC-route format at the time of failure
|
||||||
|
-func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Routes) (resultingRoutes *pb.Routes, err error) {
|
||||||
|
+func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Routes, increment bool) (resultingRoutes *pb.Routes, err error) {
|
||||||
|
if requestedRoutes == nil {
|
||||||
|
return nil, errNoRoutes
|
||||||
|
}
|
||||||
|
@@ -418,13 +418,66 @@ func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Ro
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
+ var (
|
||||||
|
+ added []*types.Route
|
||||||
|
+ removed []*types.Route
|
||||||
|
+ )
|
||||||
|
+
|
||||||
|
+ defer func(netHandle *netlink.Handle) {
|
||||||
|
+ if err != nil {
|
||||||
|
+ // if error happens after route added, need to rollback the added route
|
||||||
|
+ if len(added) > 0 {
|
||||||
|
+ for _, r := range added {
|
||||||
|
+ errRb := s.updateRoute(netHandle, r, false)
|
||||||
|
+ if errRb != nil {
|
||||||
|
+ agentLog.WithError(err).Error("rollback route failed")
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // if error happens after route removed, need to rollback the removed route
|
||||||
|
+ if len(removed) > 0 {
|
||||||
|
+ for _, r := range removed {
|
||||||
|
+ errRb := s.updateRoute(netHandle, r, true)
|
||||||
|
+ if errRb != nil {
|
||||||
|
+ agentLog.WithError(err).Error("rollback route failed")
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }(netHandle)
|
||||||
|
+
|
||||||
|
+ // updateOneRoute just update the specified one route
|
||||||
|
+ updateOneRoute := func(netHandle *netlink.Handle, reqRoute *types.Route) error {
|
||||||
|
+ var add bool = true
|
||||||
|
+ if reqRoute.Dest != "" && strings.HasPrefix(reqRoute.Dest, "-") {
|
||||||
|
+ reqRoute.Dest = reqRoute.Dest[1:]
|
||||||
|
+ add = false
|
||||||
|
+ }
|
||||||
|
+ err = s.updateRoute(netHandle, reqRoute, add)
|
||||||
|
+ if err != nil {
|
||||||
|
+ agentLog.WithError(err).Error("update Route failed")
|
||||||
|
+ // If there was an error setting the route, return the error
|
||||||
|
+ // and the current routes on the system via the defer func
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ if add {
|
||||||
|
+ added = append([]*types.Route{reqRoute}, added[:]...)
|
||||||
|
+ } else {
|
||||||
|
+ removed = append([]*types.Route{reqRoute}, removed[:]...)
|
||||||
|
+ }
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
//
|
||||||
|
// First things first, let's blow away all the existing routes. The updateRoutes function
|
||||||
|
// is designed to be declarative, so we will attempt to create state matching what is
|
||||||
|
// requested, and in the event that we fail to do so, will return the error and final state.
|
||||||
|
//
|
||||||
|
- if err = s.deleteRoutes(netHandle); err != nil {
|
||||||
|
- return nil, err
|
||||||
|
+ if !increment {
|
||||||
|
+ if err = s.deleteRoutes(netHandle); err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
|
||||||
|
//
|
||||||
|
@@ -434,7 +487,12 @@ func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Ro
|
||||||
|
// won't be able to access the gateway
|
||||||
|
for _, reqRoute := range requestedRoutes.Routes {
|
||||||
|
if reqRoute.Gateway == "" {
|
||||||
|
- err = s.updateRoute(netHandle, reqRoute, true)
|
||||||
|
+ if increment {
|
||||||
|
+ err = updateOneRoute(netHandle, reqRoute)
|
||||||
|
+ } else {
|
||||||
|
+ err = s.updateRoute(netHandle, reqRoute, true)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
if err != nil {
|
||||||
|
agentLog.WithError(err).Error("update Route failed")
|
||||||
|
//If there was an error setting the route, return the error
|
||||||
|
@@ -447,7 +505,11 @@ func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Ro
|
||||||
|
// Take a second pass and apply the routes which include a gateway
|
||||||
|
for _, reqRoute := range requestedRoutes.Routes {
|
||||||
|
if reqRoute.Gateway != "" {
|
||||||
|
- err = s.updateRoute(netHandle, reqRoute, true)
|
||||||
|
+ if increment {
|
||||||
|
+ err = updateOneRoute(netHandle, reqRoute)
|
||||||
|
+ } else {
|
||||||
|
+ err = s.updateRoute(netHandle, reqRoute, true)
|
||||||
|
+ }
|
||||||
|
if err != nil {
|
||||||
|
agentLog.WithError(err).Error("update Route failed")
|
||||||
|
//If there was an error setting the route, return the
|
||||||
|
@@ -699,4 +761,4 @@ func (s *sandbox) handleLocalhost() error {
|
||||||
|
}
|
||||||
|
|
||||||
|
return netlink.LinkSetUp(lo)
|
||||||
|
-}
|
||||||
|
+}
|
||||||
|
\ No newline at end of file
|
||||||
|
diff --git a/network_test.go b/network_test.go
|
||||||
|
index a143670..a1e58f5 100644
|
||||||
|
--- a/network_test.go
|
||||||
|
+++ b/network_test.go
|
||||||
|
@@ -160,7 +160,7 @@ func TestUpdateRoutes(t *testing.T) {
|
||||||
|
Routes: inputRoutesSimple,
|
||||||
|
}
|
||||||
|
|
||||||
|
- results, err := s.updateRoutes(netHandle, testRoutes)
|
||||||
|
+ results, err := s.updateRoutes(netHandle, testRoutes, false)
|
||||||
|
assert.Nil(t, err, "Unexpected update interface failure: %v", err)
|
||||||
|
assert.True(t, reflect.DeepEqual(results, testRoutes),
|
||||||
|
"Interface created didn't match: got %+v, expecting %+v", results, testRoutes)
|
||||||
|
@@ -173,7 +173,7 @@ func TestUpdateRoutes(t *testing.T) {
|
||||||
|
}
|
||||||
|
testRoutes.Routes = inputRoutesPTPExample
|
||||||
|
|
||||||
|
- results, err = s.updateRoutes(netHandle, testRoutes)
|
||||||
|
+ results, err = s.updateRoutes(netHandle, testRoutes, false)
|
||||||
|
assert.Nil(t, err, "Unexpected update interface failure: %v", err)
|
||||||
|
assert.True(t, reflect.DeepEqual(results, testRoutes),
|
||||||
|
"Interface created didn't match: got %+v, expecting %+v", results, testRoutes)
|
||||||
|
@@ -184,7 +184,7 @@ func TestUpdateRoutes(t *testing.T) {
|
||||||
|
{Dest: "192.168.0.0/16", Gateway: "", Source: "192.168.0.2", Scope: 0, Device: "ifc-name"},
|
||||||
|
}
|
||||||
|
testRoutes.Routes = inputRoutesNoScope
|
||||||
|
- results, err = s.updateRoutes(netHandle, testRoutes)
|
||||||
|
+ results, err = s.updateRoutes(netHandle, testRoutes, false)
|
||||||
|
assert.NotNil(t, err, "Expected to observe unreachable route failure")
|
||||||
|
|
||||||
|
assert.True(t, reflect.DeepEqual(results.Routes[0], testRoutes.Routes[1]),
|
||||||
|
@@ -231,7 +231,7 @@ func TestUpdateRoutesIPVlan(t *testing.T) {
|
||||||
|
}
|
||||||
|
testRoutes.Routes = inputRoutesIPVlanExample
|
||||||
|
|
||||||
|
- results, err := s.updateRoutes(netHandle, testRoutes)
|
||||||
|
+ results, err := s.updateRoutes(netHandle, testRoutes, false)
|
||||||
|
assert.Nil(t, err, "Unexpected update interface failure: %v", err)
|
||||||
|
assert.True(t, reflect.DeepEqual(results, testRoutes),
|
||||||
|
"Interface created didn't match: got %+v, expecting %+v", results, testRoutes)
|
||||||
|
@@ -357,7 +357,7 @@ func TestListRoutes(t *testing.T) {
|
||||||
|
Routes: inputRoutesSimple,
|
||||||
|
}
|
||||||
|
|
||||||
|
- _, err := s.updateRoutes(netHandle, testRoutes)
|
||||||
|
+ _, err := s.updateRoutes(netHandle, testRoutes, false)
|
||||||
|
assert.Nil(err)
|
||||||
|
results, err := s.listRoutes(nil)
|
||||||
|
assert.Nil(err, "Expected to list all routes")
|
||||||
|
@@ -377,7 +377,7 @@ func TestListRoutes(t *testing.T) {
|
||||||
|
Routes: inputRoutesSimple,
|
||||||
|
}
|
||||||
|
|
||||||
|
- _, err = s.updateRoutes(netHandle, testRoutes)
|
||||||
|
+ _, err = s.updateRoutes(netHandle, testRoutes, false)
|
||||||
|
assert.Nil(err)
|
||||||
|
results, err = s.listRoutes(nil)
|
||||||
|
assert.Nil(err, "Expected to list all routes")
|
||||||
|
@@ -438,7 +438,7 @@ func TestListRoutesWithIPV6(t *testing.T) {
|
||||||
|
Routes: inputRoutesSimple,
|
||||||
|
}
|
||||||
|
|
||||||
|
- _, err := s.updateRoutes(netHandle, testRoutes)
|
||||||
|
+ _, err := s.updateRoutes(netHandle, testRoutes, false)
|
||||||
|
assert.Nil(err)
|
||||||
|
results, err := s.listRoutes(nil)
|
||||||
|
assert.Nil(err, "Expected to list all routes")
|
||||||
|
@@ -514,7 +514,7 @@ func TestListRoutesWithTwoInterfacesSameSubnet(t *testing.T) {
|
||||||
|
Routes: inputRoutesSimple,
|
||||||
|
}
|
||||||
|
|
||||||
|
- _, err := s.updateRoutes(netHandle, testRoutes)
|
||||||
|
+ _, err := s.updateRoutes(netHandle, testRoutes, false)
|
||||||
|
assert.Nil(err)
|
||||||
|
results, err := s.listRoutes(nil)
|
||||||
|
assert.Nil(err, "Expected to list all routes")
|
||||||
|
diff --git a/protocols/grpc/agent.pb.go b/protocols/grpc/agent.pb.go
|
||||||
|
index 77e6d1b..1b887e5 100644
|
||||||
|
--- a/protocols/grpc/agent.pb.go
|
||||||
|
+++ b/protocols/grpc/agent.pb.go
|
||||||
|
@@ -1303,7 +1303,8 @@ func (m *UpdateInterfaceRequest) GetInterface() *types.Interface {
|
||||||
|
}
|
||||||
|
|
||||||
|
type UpdateRoutesRequest struct {
|
||||||
|
- Routes *Routes `protobuf:"bytes,1,opt,name=routes" json:"routes,omitempty"`
|
||||||
|
+ Routes *Routes `protobuf:"bytes,1,opt,name=routes" json:"routes,omitempty"`
|
||||||
|
+ Increment bool `protobuf:"varint,2,opt,name=increment,proto3" json:"increment,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *UpdateRoutesRequest) Reset() { *m = UpdateRoutesRequest{} }
|
||||||
|
@@ -1318,6 +1319,13 @@ func (m *UpdateRoutesRequest) GetRoutes() *Routes {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (m *UpdateRoutesRequest) GetIncrement() bool {
|
||||||
|
+ if m != nil {
|
||||||
|
+ return m.Increment
|
||||||
|
+ }
|
||||||
|
+ return false
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
type ListInterfacesRequest struct {
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -4476,6 +4484,16 @@ func (m *UpdateRoutesRequest) MarshalTo(dAtA []byte) (int, error) {
|
||||||
|
}
|
||||||
|
i += n20
|
||||||
|
}
|
||||||
|
+ if m.Increment {
|
||||||
|
+ dAtA[i] = 0x10
|
||||||
|
+ i++
|
||||||
|
+ if m.Increment {
|
||||||
|
+ dAtA[i] = 1
|
||||||
|
+ } else {
|
||||||
|
+ dAtA[i] = 0
|
||||||
|
+ }
|
||||||
|
+ i++
|
||||||
|
+ }
|
||||||
|
return i, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -5751,6 +5769,9 @@ func (m *UpdateRoutesRequest) Size() (n int) {
|
||||||
|
l = m.Routes.Size()
|
||||||
|
n += 1 + l + sovAgent(uint64(l))
|
||||||
|
}
|
||||||
|
+ if m.Increment {
|
||||||
|
+ n += 2
|
||||||
|
+ }
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -10964,6 +10985,26 @@ func (m *UpdateRoutesRequest) Unmarshal(dAtA []byte) error {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
iNdEx = postIndex
|
||||||
|
+ case 2:
|
||||||
|
+ if wireType != 0 {
|
||||||
|
+ return fmt.Errorf("proto: wrong wireType = %d for field Increment", wireType)
|
||||||
|
+ }
|
||||||
|
+ var v int
|
||||||
|
+ for shift := uint(0); ; shift += 7 {
|
||||||
|
+ if shift >= 64 {
|
||||||
|
+ return ErrIntOverflowAgent
|
||||||
|
+ }
|
||||||
|
+ if iNdEx >= l {
|
||||||
|
+ return io.ErrUnexpectedEOF
|
||||||
|
+ }
|
||||||
|
+ b := dAtA[iNdEx]
|
||||||
|
+ iNdEx++
|
||||||
|
+ v |= (int(b) & 0x7F) << shift
|
||||||
|
+ if b < 0x80 {
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ m.Increment = bool(v != 0)
|
||||||
|
default:
|
||||||
|
iNdEx = preIndex
|
||||||
|
skippy, err := skipAgent(dAtA[iNdEx:])
|
||||||
|
@@ -12852,184 +12893,185 @@ var (
|
||||||
|
func init() { proto.RegisterFile("agent.proto", fileDescriptorAgent) }
|
||||||
|
|
||||||
|
var fileDescriptorAgent = []byte{
|
||||||
|
- // 2862 bytes of a gzipped FileDescriptorProto
|
||||||
|
+ // 2876 bytes of a gzipped FileDescriptorProto
|
||||||
|
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0x4b, 0x6f, 0x1c, 0xc7,
|
||||||
|
- 0xd1, 0xd8, 0x07, 0x97, 0xbb, 0xb5, 0x2f, 0x6e, 0x93, 0xa2, 0x56, 0x2b, 0x5b, 0x9f, 0x3c, 0xb6,
|
||||||
|
- 0x65, 0xfa, 0x73, 0xbc, 0xb4, 0x65, 0x23, 0x7e, 0xc1, 0x11, 0xc4, 0x47, 0x44, 0xc6, 0x56, 0xc4,
|
||||||
|
- 0x0c, 0x45, 0x38, 0x40, 0x10, 0x0c, 0x86, 0x33, 0xcd, 0x65, 0x9b, 0x3b, 0xd3, 0xe3, 0x9e, 0x1e,
|
||||||
|
- 0x8a, 0xeb, 0x00, 0x39, 0x26, 0xb7, 0x5c, 0x02, 0xe4, 0x96, 0x3f, 0x10, 0xe4, 0x96, 0x63, 0xae,
|
||||||
|
- 0x39, 0x18, 0x39, 0xe5, 0x17, 0x04, 0x81, 0x7f, 0x42, 0x7e, 0x41, 0xd0, 0xaf, 0x79, 0xec, 0x0e,
|
||||||
|
- 0x29, 0x84, 0x20, 0x90, 0xcb, 0xa2, 0xab, 0xba, 0xba, 0x5e, 0xdd, 0x55, 0x53, 0x55, 0x0b, 0x6d,
|
||||||
|
- 0x77, 0x82, 0x43, 0x3e, 0x8e, 0x18, 0xe5, 0x14, 0xd5, 0x27, 0x2c, 0xf2, 0x46, 0x2d, 0xea, 0x11,
|
||||||
|
- 0x85, 0x18, 0xfd, 0x70, 0x42, 0xf8, 0x69, 0x72, 0x3c, 0xf6, 0x68, 0xb0, 0x79, 0xe6, 0x72, 0xf7,
|
||||||
|
- 0x5d, 0x8f, 0x86, 0xdc, 0x25, 0x21, 0x66, 0xf1, 0xa6, 0x3c, 0xb8, 0x19, 0x9d, 0x4d, 0x36, 0xf9,
|
||||||
|
- 0x2c, 0xc2, 0xb1, 0xfa, 0xd5, 0xe7, 0xee, 0x4e, 0x28, 0x9d, 0x4c, 0xf1, 0xa6, 0x84, 0x8e, 0x93,
|
||||||
|
- 0x93, 0x4d, 0x1c, 0x44, 0x7c, 0xa6, 0x36, 0xad, 0x3f, 0x56, 0x61, 0x7d, 0x9b, 0x61, 0x97, 0xe3,
|
||||||
|
- 0x6d, 0xc3, 0xcd, 0xc6, 0xdf, 0x24, 0x38, 0xe6, 0xe8, 0x35, 0xe8, 0xa4, 0x12, 0x1c, 0xe2, 0x0f,
|
||||||
|
- 0x2b, 0xf7, 0x2b, 0x1b, 0x2d, 0xbb, 0x9d, 0xe2, 0xf6, 0x7d, 0x74, 0x1b, 0x96, 0xf1, 0x05, 0xf6,
|
||||||
|
- 0xc4, 0x6e, 0x55, 0xee, 0x36, 0x04, 0xb8, 0xef, 0xa3, 0xf7, 0xa1, 0x1d, 0x73, 0x46, 0xc2, 0x89,
|
||||||
|
- 0x93, 0xc4, 0x98, 0x0d, 0x6b, 0xf7, 0x2b, 0x1b, 0xed, 0x87, 0x2b, 0x63, 0x61, 0xd2, 0xf8, 0x50,
|
||||||
|
- 0x6e, 0x1c, 0xc5, 0x98, 0xd9, 0x10, 0xa7, 0x6b, 0xf4, 0x00, 0x96, 0x7d, 0x7c, 0x4e, 0x3c, 0x1c,
|
||||||
|
- 0x0f, 0xeb, 0xf7, 0x6b, 0x1b, 0xed, 0x87, 0x1d, 0x45, 0xbe, 0x23, 0x91, 0xb6, 0xd9, 0x44, 0x6f,
|
||||||
|
- 0x43, 0x33, 0xe6, 0x94, 0xb9, 0x13, 0x1c, 0x0f, 0x97, 0x24, 0x61, 0xd7, 0xf0, 0x95, 0x58, 0x3b,
|
||||||
|
- 0xdd, 0x46, 0xaf, 0x40, 0xed, 0xd9, 0xf6, 0xfe, 0xb0, 0x21, 0xa5, 0x83, 0xa6, 0x8a, 0xb0, 0x67,
|
||||||
|
- 0x0b, 0x34, 0x7a, 0x1d, 0xba, 0xb1, 0x1b, 0xfa, 0xc7, 0xf4, 0xc2, 0x89, 0x88, 0x1f, 0xc6, 0xc3,
|
||||||
|
- 0xe5, 0xfb, 0x95, 0x8d, 0xa6, 0xdd, 0xd1, 0xc8, 0x03, 0x81, 0xb3, 0x3e, 0x85, 0x5b, 0x87, 0xdc,
|
||||||
|
- 0x65, 0xfc, 0x1a, 0xde, 0xb1, 0x8e, 0x60, 0xdd, 0xc6, 0x01, 0x3d, 0xbf, 0x96, 0x6b, 0x87, 0xb0,
|
||||||
|
- 0xcc, 0x49, 0x80, 0x69, 0xc2, 0xa5, 0x6b, 0xbb, 0xb6, 0x01, 0xad, 0x3f, 0x57, 0x00, 0xed, 0x5e,
|
||||||
|
- 0x60, 0xef, 0x80, 0x51, 0x0f, 0xc7, 0xf1, 0xff, 0xe8, 0xba, 0xde, 0x82, 0xe5, 0x48, 0x29, 0x30,
|
||||||
|
- 0xac, 0x4b, 0x72, 0x7d, 0x0b, 0x46, 0x2b, 0xb3, 0x6b, 0x7d, 0x0d, 0x6b, 0x87, 0x64, 0x12, 0xba,
|
||||||
|
- 0xd3, 0x1b, 0xd4, 0x77, 0x1d, 0x1a, 0xb1, 0xe4, 0x29, 0x55, 0xed, 0xda, 0x1a, 0xb2, 0x0e, 0x00,
|
||||||
|
- 0x7d, 0xe5, 0x12, 0x7e, 0x73, 0x92, 0xac, 0x77, 0x61, 0xb5, 0xc0, 0x31, 0x8e, 0x68, 0x18, 0x63,
|
||||||
|
- 0xa9, 0x00, 0x77, 0x79, 0x12, 0x4b, 0x66, 0x4b, 0xb6, 0x86, 0x2c, 0x0c, 0x6b, 0x5f, 0x92, 0xd8,
|
||||||
|
- 0x90, 0xe3, 0xff, 0x46, 0x85, 0x75, 0x68, 0x9c, 0x50, 0x16, 0xb8, 0xdc, 0x68, 0xa0, 0x20, 0x84,
|
||||||
|
- 0xa0, 0xee, 0xb2, 0x49, 0x3c, 0xac, 0xdd, 0xaf, 0x6d, 0xb4, 0x6c, 0xb9, 0x16, 0xaf, 0x72, 0x4e,
|
||||||
|
- 0x8c, 0xd6, 0xeb, 0x35, 0xe8, 0x68, 0xbf, 0x3b, 0x53, 0x12, 0x73, 0x29, 0xa7, 0x63, 0xb7, 0x35,
|
||||||
|
- 0x4e, 0x9c, 0xb1, 0x28, 0xac, 0x1f, 0x45, 0xfe, 0x35, 0x03, 0xfe, 0x21, 0xb4, 0x18, 0x8e, 0x69,
|
||||||
|
- 0xc2, 0x44, 0x98, 0x56, 0xe5, 0xbd, 0xaf, 0xa9, 0x7b, 0xff, 0x92, 0x84, 0xc9, 0x85, 0x6d, 0xf6,
|
||||||
|
- 0xec, 0x8c, 0x4c, 0x87, 0x10, 0x8f, 0xaf, 0x13, 0x42, 0x9f, 0xc2, 0xad, 0x03, 0x37, 0x89, 0xaf,
|
||||||
|
- 0xa3, 0xab, 0xf5, 0x99, 0x08, 0xbf, 0x38, 0x09, 0xae, 0x75, 0xf8, 0x4f, 0x15, 0x68, 0x6e, 0x47,
|
||||||
|
- 0xc9, 0x51, 0xec, 0x4e, 0x30, 0xfa, 0x3f, 0x68, 0x73, 0xca, 0xdd, 0xa9, 0x93, 0x08, 0x50, 0x92,
|
||||||
|
- 0xd7, 0x6d, 0x90, 0x28, 0x45, 0x20, 0xdc, 0x8e, 0x99, 0x17, 0x25, 0x9a, 0xa2, 0x7a, 0xbf, 0xb6,
|
||||||
|
- 0x51, 0xb7, 0xdb, 0x0a, 0xa7, 0x48, 0xc6, 0xb0, 0x2a, 0xf7, 0x1c, 0x12, 0x3a, 0x67, 0x98, 0x85,
|
||||||
|
- 0x78, 0x1a, 0x50, 0x1f, 0xcb, 0xf7, 0x5b, 0xb7, 0x07, 0x72, 0x6b, 0x3f, 0xfc, 0x22, 0xdd, 0x40,
|
||||||
|
- 0xff, 0x0f, 0x83, 0x94, 0x5e, 0x04, 0xa5, 0xa4, 0xae, 0x4b, 0xea, 0xbe, 0xa6, 0x3e, 0xd2, 0x68,
|
||||||
|
- 0xeb, 0xd7, 0xd0, 0x7b, 0x7e, 0xca, 0x28, 0xe7, 0x53, 0x12, 0x4e, 0x76, 0x5c, 0xee, 0x8a, 0xec,
|
||||||
|
- 0x11, 0x61, 0x46, 0xa8, 0x1f, 0x6b, 0x6d, 0x0d, 0x88, 0xde, 0x81, 0x01, 0x57, 0xb4, 0xd8, 0x77,
|
||||||
|
- 0x0c, 0x4d, 0x55, 0xd2, 0xac, 0xa4, 0x1b, 0x07, 0x9a, 0xf8, 0x4d, 0xe8, 0x65, 0xc4, 0x22, 0xff,
|
||||||
|
- 0x68, 0x7d, 0xbb, 0x29, 0xf6, 0x39, 0x09, 0xb0, 0x75, 0x2e, 0x7d, 0x25, 0x2f, 0x19, 0xbd, 0x03,
|
||||||
|
- 0xad, 0xcc, 0x0f, 0x15, 0xf9, 0x42, 0x7a, 0xea, 0x85, 0x18, 0x77, 0xda, 0xcd, 0xd4, 0x29, 0x9f,
|
||||||
|
- 0x43, 0x9f, 0xa7, 0x8a, 0x3b, 0xbe, 0xcb, 0xdd, 0xe2, 0xa3, 0x2a, 0x5a, 0x65, 0xf7, 0x78, 0x01,
|
||||||
|
- 0xb6, 0x3e, 0x83, 0xd6, 0x01, 0xf1, 0x63, 0x25, 0x78, 0x08, 0xcb, 0x5e, 0xc2, 0x18, 0x0e, 0xb9,
|
||||||
|
- 0x31, 0x59, 0x83, 0x68, 0x0d, 0x96, 0xa6, 0x24, 0x20, 0x5c, 0x9b, 0xa9, 0x00, 0x8b, 0x02, 0x3c,
|
||||||
|
- 0xc5, 0x01, 0x65, 0x33, 0xe9, 0xb0, 0x35, 0x58, 0xca, 0x5f, 0xae, 0x02, 0xd0, 0x5d, 0x68, 0x05,
|
||||||
|
- 0xee, 0x45, 0x7a, 0xa9, 0x62, 0xa7, 0x19, 0xb8, 0x17, 0x4a, 0xf9, 0x21, 0x2c, 0x9f, 0xb8, 0x64,
|
||||||
|
- 0xea, 0x85, 0x5c, 0x7b, 0xc5, 0x80, 0x99, 0xc0, 0x7a, 0x5e, 0xe0, 0xdf, 0xaa, 0xd0, 0x56, 0x12,
|
||||||
|
- 0x95, 0xc2, 0x6b, 0xb0, 0xe4, 0xb9, 0xde, 0x69, 0x2a, 0x52, 0x02, 0xe8, 0x81, 0x51, 0xa4, 0x9a,
|
||||||
|
- 0x4f, 0xc2, 0x99, 0xa6, 0x46, 0xb5, 0x4d, 0x80, 0xf8, 0x85, 0x1b, 0x69, 0xdd, 0x6a, 0x97, 0x10,
|
||||||
|
- 0xb7, 0x04, 0x8d, 0x52, 0xf7, 0x03, 0xe8, 0xa8, 0x77, 0xa7, 0x8f, 0xd4, 0x2f, 0x39, 0xd2, 0x56,
|
||||||
|
- 0x54, 0xea, 0xd0, 0xeb, 0xd0, 0x4d, 0x62, 0xec, 0x9c, 0x12, 0xcc, 0x5c, 0xe6, 0x9d, 0xce, 0x86,
|
||||||
|
- 0x4b, 0xea, 0x1b, 0x99, 0xc4, 0x78, 0xcf, 0xe0, 0xd0, 0x43, 0x58, 0x12, 0xe9, 0x2f, 0x1e, 0x36,
|
||||||
|
- 0xe4, 0xe7, 0xf8, 0x95, 0x3c, 0x4b, 0x69, 0xea, 0x58, 0xfe, 0xee, 0x86, 0x9c, 0xcd, 0x6c, 0x45,
|
||||||
|
- 0x3a, 0xfa, 0x18, 0x20, 0x43, 0xa2, 0x15, 0xa8, 0x9d, 0xe1, 0x99, 0x8e, 0x43, 0xb1, 0x14, 0xce,
|
||||||
|
- 0x39, 0x77, 0xa7, 0x89, 0xf1, 0xba, 0x02, 0x3e, 0xad, 0x7e, 0x5c, 0xb1, 0x3c, 0xe8, 0x6f, 0x4d,
|
||||||
|
- 0xcf, 0x08, 0xcd, 0x1d, 0x5f, 0x83, 0xa5, 0xc0, 0xfd, 0x9a, 0x32, 0xe3, 0x49, 0x09, 0x48, 0x2c,
|
||||||
|
- 0x09, 0x29, 0x33, 0x2c, 0x24, 0x80, 0x7a, 0x50, 0xa5, 0x91, 0xf4, 0x57, 0xcb, 0xae, 0xd2, 0x28,
|
||||||
|
- 0x13, 0x54, 0xcf, 0x09, 0xb2, 0xfe, 0x59, 0x07, 0xc8, 0xa4, 0x20, 0x1b, 0x46, 0x84, 0x3a, 0x31,
|
||||||
|
- 0x66, 0xa2, 0x04, 0x71, 0x8e, 0x67, 0x1c, 0xc7, 0x0e, 0xc3, 0x5e, 0xc2, 0x62, 0x72, 0x2e, 0xee,
|
||||||
|
- 0x4f, 0x98, 0x7d, 0x4b, 0x99, 0x3d, 0xa7, 0x9b, 0x7d, 0x9b, 0xd0, 0x43, 0x75, 0x6e, 0x4b, 0x1c,
|
||||||
|
- 0xb3, 0xcd, 0x29, 0xb4, 0x0f, 0xb7, 0x32, 0x9e, 0x7e, 0x8e, 0x5d, 0xf5, 0x2a, 0x76, 0xab, 0x29,
|
||||||
|
- 0x3b, 0x3f, 0x63, 0xb5, 0x0b, 0xab, 0x84, 0x3a, 0xdf, 0x24, 0x38, 0x29, 0x30, 0xaa, 0x5d, 0xc5,
|
||||||
|
- 0x68, 0x40, 0xe8, 0xcf, 0xe4, 0x81, 0x8c, 0xcd, 0x01, 0xdc, 0xc9, 0x59, 0x29, 0xc2, 0x3d, 0xc7,
|
||||||
|
- 0xac, 0x7e, 0x15, 0xb3, 0xf5, 0x54, 0x2b, 0x91, 0x0f, 0x32, 0x8e, 0x3f, 0x81, 0x75, 0x42, 0x9d,
|
||||||
|
- 0x17, 0x2e, 0xe1, 0xf3, 0xec, 0x96, 0x5e, 0x62, 0xa4, 0xf8, 0xe8, 0x16, 0x79, 0x29, 0x23, 0x03,
|
||||||
|
- 0xcc, 0x26, 0x05, 0x23, 0x1b, 0x2f, 0x31, 0xf2, 0xa9, 0x3c, 0x90, 0xb1, 0x79, 0x0c, 0x03, 0x42,
|
||||||
|
- 0xe7, 0xb5, 0x59, 0xbe, 0x8a, 0x49, 0x9f, 0xd0, 0xa2, 0x26, 0x5b, 0x30, 0x88, 0xb1, 0xc7, 0x29,
|
||||||
|
- 0xcb, 0x3f, 0x82, 0xe6, 0x55, 0x2c, 0x56, 0x34, 0x7d, 0xca, 0xc3, 0xfa, 0x05, 0x74, 0xf6, 0x92,
|
||||||
|
- 0x09, 0xe6, 0xd3, 0xe3, 0x34, 0x19, 0xdc, 0x58, 0xfe, 0xb1, 0xfe, 0x5d, 0x85, 0xf6, 0xf6, 0x84,
|
||||||
|
- 0xd1, 0x24, 0x2a, 0xe4, 0x64, 0x15, 0xa4, 0xf3, 0x39, 0x59, 0x92, 0xc8, 0x9c, 0xac, 0x88, 0x3f,
|
||||||
|
- 0x84, 0x4e, 0x20, 0x43, 0x57, 0xd3, 0xab, 0x3c, 0x34, 0x58, 0x08, 0x6a, 0xbb, 0x1d, 0xe4, 0x92,
|
||||||
|
- 0xd9, 0x18, 0x20, 0x22, 0x7e, 0xac, 0xcf, 0xa8, 0x74, 0xd4, 0xd7, 0x15, 0xa1, 0x49, 0xd1, 0x76,
|
||||||
|
- 0x2b, 0x4a, 0xb3, 0xf5, 0xfb, 0xd0, 0x3e, 0x16, 0x4e, 0xd2, 0x07, 0x0a, 0xc9, 0x28, 0xf3, 0x9e,
|
||||||
|
- 0x0d, 0xc7, 0x59, 0x10, 0xee, 0x41, 0xf7, 0x54, 0xb9, 0x4c, 0x1f, 0x52, 0x6f, 0xe8, 0x75, 0x6d,
|
||||||
|
- 0x49, 0x66, 0xef, 0x38, 0xef, 0x59, 0x75, 0x01, 0x9d, 0xd3, 0x1c, 0x6a, 0x74, 0x08, 0x83, 0x05,
|
||||||
|
- 0x92, 0x92, 0x1c, 0xb4, 0x91, 0xcf, 0x41, 0xed, 0x87, 0x48, 0x09, 0xca, 0x9f, 0xcc, 0xe7, 0xa5,
|
||||||
|
- 0xdf, 0x55, 0xa1, 0xf3, 0x53, 0xcc, 0x5f, 0x50, 0x76, 0xa6, 0xf4, 0x45, 0x50, 0x0f, 0xdd, 0x00,
|
||||||
|
- 0x6b, 0x8e, 0x72, 0x8d, 0xee, 0x40, 0x93, 0x5d, 0xa8, 0x04, 0xa2, 0xef, 0x73, 0x99, 0x5d, 0xc8,
|
||||||
|
- 0xc4, 0x80, 0x5e, 0x05, 0x60, 0x17, 0x4e, 0xe4, 0x7a, 0x67, 0x58, 0x7b, 0xb0, 0x6e, 0xb7, 0xd8,
|
||||||
|
- 0xc5, 0x81, 0x42, 0x88, 0xa7, 0xc0, 0x2e, 0x1c, 0xcc, 0x18, 0x65, 0xb1, 0xce, 0x55, 0x4d, 0x76,
|
||||||
|
- 0xb1, 0x2b, 0x61, 0x7d, 0xd6, 0x67, 0x34, 0x8a, 0xb0, 0x2f, 0x73, 0xb4, 0x3c, 0xbb, 0xa3, 0x10,
|
||||||
|
- 0x42, 0x2a, 0x37, 0x52, 0x1b, 0x4a, 0x2a, 0xcf, 0xa4, 0xf2, 0x4c, 0xea, 0xb2, 0x3a, 0xc9, 0xf3,
|
||||||
|
- 0x52, 0x79, 0x2a, 0xb5, 0xa9, 0xa4, 0xf2, 0x9c, 0x54, 0x9e, 0x49, 0x6d, 0x99, 0xb3, 0x5a, 0xaa,
|
||||||
|
- 0xf5, 0xdb, 0x0a, 0xac, 0xcf, 0x17, 0x7e, 0xba, 0x4c, 0xfd, 0x10, 0x3a, 0x9e, 0xbc, 0xaf, 0xc2,
|
||||||
|
- 0x9b, 0x1c, 0x2c, 0xdc, 0xa4, 0xdd, 0xf6, 0x72, 0xcf, 0xf8, 0x23, 0xe8, 0x86, 0xca, 0xc1, 0xe9,
|
||||||
|
- 0xd3, 0xac, 0x65, 0xf7, 0x92, 0xf7, 0xbd, 0xdd, 0x09, 0x73, 0x90, 0xe5, 0x03, 0xfa, 0x8a, 0x11,
|
||||||
|
- 0x8e, 0x0f, 0x39, 0xc3, 0x6e, 0x70, 0x13, 0x0d, 0x08, 0x82, 0xba, 0xac, 0x56, 0x6a, 0xb2, 0xbe,
|
||||||
|
- 0x96, 0x6b, 0xeb, 0x2d, 0x58, 0x2d, 0x48, 0xd1, 0xb6, 0xae, 0x40, 0x6d, 0x8a, 0x43, 0xc9, 0xbd,
|
||||||
|
- 0x6b, 0x8b, 0xa5, 0xe5, 0xc2, 0xc0, 0xc6, 0xae, 0x7f, 0x73, 0xda, 0x68, 0x11, 0xb5, 0x4c, 0xc4,
|
||||||
|
- 0x06, 0xa0, 0xbc, 0x08, 0xad, 0x8a, 0xd1, 0xba, 0x92, 0xd3, 0xfa, 0x19, 0x0c, 0xb6, 0xa7, 0x34,
|
||||||
|
- 0xc6, 0x87, 0xdc, 0x27, 0xe1, 0x4d, 0x74, 0x4c, 0xbf, 0x82, 0xd5, 0xe7, 0x7c, 0xf6, 0x95, 0x60,
|
||||||
|
- 0x16, 0x93, 0x6f, 0xf1, 0x0d, 0xd9, 0xc7, 0xe8, 0x0b, 0x63, 0x1f, 0xa3, 0x2f, 0x44, 0xb3, 0xe4,
|
||||||
|
- 0xd1, 0x69, 0x12, 0x84, 0x32, 0x14, 0xba, 0xb6, 0x86, 0xac, 0x2d, 0xe8, 0xa8, 0x1a, 0xfa, 0x29,
|
||||||
|
- 0xf5, 0x93, 0x29, 0x2e, 0x8d, 0xc1, 0x7b, 0x00, 0x91, 0xcb, 0xdc, 0x00, 0x73, 0xcc, 0xd4, 0x1b,
|
||||||
|
- 0x6a, 0xd9, 0x39, 0x8c, 0xf5, 0x87, 0x2a, 0xac, 0xa9, 0x91, 0xc8, 0xa1, 0x9a, 0x04, 0x18, 0x13,
|
||||||
|
- 0x46, 0xd0, 0x3c, 0xa5, 0x31, 0xcf, 0x31, 0x4c, 0x61, 0xa1, 0xa2, 0x1f, 0x1a, 0x6e, 0x62, 0x59,
|
||||||
|
- 0x98, 0x53, 0xd4, 0xae, 0x9e, 0x53, 0x2c, 0x4c, 0x22, 0xea, 0x8b, 0x93, 0x08, 0x11, 0x6d, 0x86,
|
||||||
|
- 0x88, 0xa8, 0x18, 0x6f, 0xd9, 0x2d, 0x8d, 0xd9, 0xf7, 0xd1, 0x03, 0xe8, 0x4f, 0x84, 0x96, 0xce,
|
||||||
|
- 0x29, 0xa5, 0x67, 0x4e, 0xe4, 0xf2, 0x53, 0x19, 0xea, 0x2d, 0xbb, 0x2b, 0xd1, 0x7b, 0x94, 0x9e,
|
||||||
|
- 0x1d, 0xb8, 0xfc, 0x14, 0x7d, 0x02, 0x3d, 0x5d, 0x06, 0x06, 0xd2, 0x45, 0xb1, 0xfe, 0xf8, 0xe9,
|
||||||
|
- 0x28, 0xca, 0x7b, 0xcf, 0xee, 0x9e, 0xe5, 0xa0, 0xd8, 0xba, 0x0d, 0xb7, 0x76, 0x70, 0xcc, 0x19,
|
||||||
|
- 0x9d, 0x15, 0x1d, 0x63, 0xfd, 0x08, 0x60, 0x3f, 0xe4, 0x98, 0x9d, 0xb8, 0x1e, 0x8e, 0xd1, 0x7b,
|
||||||
|
- 0x79, 0x48, 0x17, 0x47, 0x2b, 0x63, 0x35, 0x91, 0x4a, 0x37, 0xec, 0x1c, 0x8d, 0x35, 0x86, 0x86,
|
||||||
|
- 0x4d, 0x13, 0x91, 0x8e, 0xde, 0x30, 0x2b, 0x7d, 0xae, 0xa3, 0xcf, 0x49, 0xa4, 0xad, 0xf7, 0xac,
|
||||||
|
- 0x3d, 0xd3, 0xc2, 0x66, 0xec, 0xf4, 0x15, 0x8d, 0xa1, 0x45, 0x0c, 0x4e, 0x67, 0x95, 0x45, 0xd1,
|
||||||
|
- 0x19, 0x89, 0xf5, 0x19, 0xac, 0x2a, 0x4e, 0x8a, 0xb3, 0x61, 0xf3, 0x06, 0x34, 0x98, 0x51, 0xa3,
|
||||||
|
- 0x92, 0x8d, 0xa2, 0x34, 0x91, 0xde, 0x13, 0xfe, 0x10, 0x1d, 0x75, 0x66, 0x88, 0xf1, 0xc7, 0x2a,
|
||||||
|
- 0x0c, 0xc4, 0x46, 0x81, 0xa7, 0xf5, 0x4b, 0x58, 0x7d, 0x16, 0x4e, 0x49, 0x88, 0xb7, 0x0f, 0x8e,
|
||||||
|
- 0x9e, 0xe2, 0x34, 0xee, 0x11, 0xd4, 0x45, 0x7d, 0x24, 0x05, 0x35, 0x6d, 0xb9, 0x16, 0x81, 0x10,
|
||||||
|
- 0x1e, 0x3b, 0x5e, 0x94, 0xc4, 0x7a, 0xf6, 0xd3, 0x08, 0x8f, 0xb7, 0xa3, 0x24, 0x16, 0x89, 0x5c,
|
||||||
|
- 0x7c, 0xc8, 0x69, 0x38, 0x9d, 0xc9, 0x68, 0x68, 0xda, 0xcb, 0x5e, 0x94, 0x3c, 0x0b, 0xa7, 0x33,
|
||||||
|
- 0xeb, 0x07, 0xb2, 0xdb, 0xc5, 0xd8, 0xb7, 0xdd, 0xd0, 0xa7, 0xc1, 0x0e, 0x3e, 0xcf, 0x49, 0x48,
|
||||||
|
- 0x3b, 0x2b, 0x13, 0xf5, 0xdf, 0x55, 0xa0, 0xf3, 0x78, 0x82, 0x43, 0xbe, 0x83, 0xb9, 0x4b, 0xa6,
|
||||||
|
- 0xb2, 0x7b, 0x3a, 0xc7, 0x2c, 0x26, 0x34, 0xd4, 0x4f, 0xdb, 0x80, 0xa2, 0xf9, 0x25, 0x21, 0xe1,
|
||||||
|
- 0x8e, 0xef, 0xe2, 0x80, 0x86, 0x92, 0x4b, 0xd3, 0x06, 0x81, 0xda, 0x91, 0x18, 0xf4, 0x16, 0xf4,
|
||||||
|
- 0xd5, 0x6c, 0xce, 0x39, 0x75, 0x43, 0x7f, 0x2a, 0x82, 0x4a, 0xcd, 0x2a, 0x7a, 0x0a, 0xbd, 0xa7,
|
||||||
|
- 0xb1, 0xe8, 0x6d, 0x58, 0xd1, 0x4f, 0x3e, 0xa3, 0xac, 0x4b, 0xca, 0xbe, 0xc6, 0x17, 0x48, 0x93,
|
||||||
|
- 0x28, 0xa2, 0x8c, 0xc7, 0x4e, 0x8c, 0x3d, 0x8f, 0x06, 0x91, 0x6e, 0x3d, 0xfa, 0x06, 0x7f, 0xa8,
|
||||||
|
- 0xd0, 0xd6, 0x04, 0x56, 0x9f, 0x08, 0x3b, 0xb5, 0x25, 0xd9, 0x15, 0xf6, 0x02, 0x1c, 0x38, 0xc7,
|
||||||
|
- 0x53, 0xea, 0x9d, 0x39, 0x22, 0x11, 0x69, 0x0f, 0x8b, 0xe2, 0x66, 0x4b, 0x20, 0x0f, 0xc9, 0xb7,
|
||||||
|
- 0xb2, 0xcb, 0x16, 0x54, 0xa7, 0x94, 0x47, 0xd3, 0x64, 0xe2, 0x44, 0x8c, 0x1e, 0x63, 0x6d, 0x62,
|
||||||
|
- 0x3f, 0xc0, 0xc1, 0x9e, 0xc2, 0x1f, 0x08, 0xb4, 0xf5, 0xd7, 0x0a, 0xac, 0x15, 0x25, 0xe9, 0xb4,
|
||||||
|
- 0xba, 0x09, 0x6b, 0x45, 0x51, 0xfa, 0x53, 0xab, 0x4a, 0xb9, 0x41, 0x5e, 0xa0, 0xfa, 0xe8, 0x7e,
|
||||||
|
- 0x04, 0x5d, 0x39, 0xb0, 0x75, 0x7c, 0xc5, 0xa9, 0x58, 0x60, 0xe4, 0xef, 0xc5, 0xee, 0xb8, 0xf9,
|
||||||
|
- 0x5b, 0xfa, 0x04, 0xee, 0x68, 0xf3, 0x9d, 0x45, 0xb5, 0xd5, 0x83, 0x58, 0xd7, 0x04, 0x4f, 0xe7,
|
||||||
|
- 0xb4, 0xff, 0x12, 0x86, 0x19, 0x6a, 0x6b, 0x26, 0x91, 0xc6, 0x57, 0xef, 0xc1, 0xea, 0x9c, 0xb1,
|
||||||
|
- 0x8f, 0x7d, 0x9f, 0xc9, 0x10, 0xac, 0xdb, 0x65, 0x5b, 0xd6, 0x23, 0xb8, 0x7d, 0x88, 0xb9, 0xf2,
|
||||||
|
- 0x86, 0xcb, 0x75, 0xd5, 0xaf, 0x98, 0xad, 0x40, 0xed, 0x10, 0x7b, 0xd2, 0xf8, 0x9a, 0x2d, 0x96,
|
||||||
|
- 0xe2, 0x01, 0x1e, 0xc5, 0xd8, 0x93, 0x56, 0xd6, 0x6c, 0xb9, 0xb6, 0xfe, 0x52, 0x81, 0x65, 0x9d,
|
||||||
|
- 0x08, 0x45, 0x32, 0xf7, 0x19, 0x39, 0xc7, 0x4c, 0x3f, 0x3d, 0x0d, 0xa1, 0x37, 0xa1, 0xa7, 0x56,
|
||||||
|
- 0x0e, 0x8d, 0x38, 0xa1, 0x69, 0x7a, 0xed, 0x2a, 0xec, 0x33, 0x85, 0x94, 0xb3, 0x38, 0x39, 0x6a,
|
||||||
|
- 0xd2, 0x5d, 0x9d, 0x86, 0xe4, 0x40, 0x2d, 0x16, 0xb1, 0x2f, 0xd3, 0x69, 0xcb, 0xd6, 0x90, 0x78,
|
||||||
|
- 0xea, 0x86, 0xdf, 0x92, 0xe4, 0x67, 0x40, 0xf1, 0xd4, 0x03, 0x9a, 0x84, 0xdc, 0x89, 0x28, 0x09,
|
||||||
|
- 0xb9, 0xce, 0x9f, 0x20, 0x51, 0x07, 0x02, 0x63, 0xfd, 0xa6, 0x02, 0x0d, 0x35, 0x8f, 0x16, 0x7d,
|
||||||
|
- 0x64, 0xfa, 0x15, 0xab, 0x12, 0x59, 0x11, 0x48, 0x59, 0xea, 0xcb, 0x25, 0xd7, 0x22, 0x8e, 0xcf,
|
||||||
|
- 0x03, 0x95, 0x8b, 0xb5, 0x6a, 0xe7, 0x81, 0x4c, 0xc2, 0x6f, 0x42, 0x2f, 0xfb, 0x18, 0xca, 0x7d,
|
||||||
|
- 0xa5, 0x62, 0x37, 0xc5, 0x4a, 0xb2, 0x4b, 0x35, 0xb5, 0x7e, 0x2e, 0xda, 0xe7, 0x74, 0x16, 0xbb,
|
||||||
|
- 0x02, 0xb5, 0x24, 0x55, 0x46, 0x2c, 0x05, 0x66, 0x92, 0x7e, 0x46, 0xc5, 0x12, 0x3d, 0x80, 0x9e,
|
||||||
|
- 0xeb, 0xfb, 0x44, 0x1c, 0x77, 0xa7, 0x4f, 0x88, 0x9f, 0x06, 0x69, 0x11, 0x6b, 0xfd, 0xbd, 0x02,
|
||||||
|
- 0xfd, 0x6d, 0x1a, 0xcd, 0x7e, 0x4c, 0xa6, 0x38, 0x97, 0x41, 0xa4, 0x92, 0xfa, 0x2b, 0x2a, 0xd6,
|
||||||
|
- 0xa2, 0x32, 0x3c, 0x21, 0x53, 0xac, 0x42, 0x4b, 0xdd, 0x6c, 0x53, 0x20, 0x64, 0x58, 0x99, 0xcd,
|
||||||
|
- 0x74, 0xc4, 0xd5, 0x55, 0x9b, 0x4f, 0xa9, 0x2f, 0x6b, 0x60, 0x9f, 0x30, 0x27, 0x1d, 0x68, 0x75,
|
||||||
|
- 0xed, 0x65, 0x9f, 0x30, 0xb9, 0xa5, 0x0d, 0x59, 0x92, 0x33, 0xd5, 0xbc, 0x21, 0x0d, 0x85, 0x11,
|
||||||
|
- 0x86, 0xac, 0x43, 0x83, 0x9e, 0x9c, 0xc4, 0x98, 0xcb, 0x6a, 0xb5, 0x66, 0x6b, 0x28, 0x4d, 0x73,
|
||||||
|
- 0xcd, 0x5c, 0x9a, 0xbb, 0x05, 0xab, 0x72, 0x7a, 0xff, 0x9c, 0xb9, 0x1e, 0x09, 0x27, 0x26, 0x15,
|
||||||
|
- 0xaf, 0x01, 0x3a, 0xe4, 0x34, 0x2a, 0x62, 0x1f, 0xfe, 0x7e, 0x45, 0xe7, 0x44, 0xdd, 0xca, 0xa2,
|
||||||
|
- 0x27, 0xd0, 0x9f, 0xfb, 0x6b, 0x04, 0xe9, 0xd9, 0x46, 0xf9, 0x3f, 0x26, 0xa3, 0xf5, 0xb1, 0xfa,
|
||||||
|
- 0xab, 0x65, 0x6c, 0xfe, 0x6a, 0x19, 0xef, 0x06, 0x11, 0x9f, 0xa1, 0x5d, 0xe8, 0x15, 0xff, 0x44,
|
||||||
|
- 0x40, 0x77, 0x4d, 0x29, 0x50, 0xf2, 0xd7, 0xc2, 0xa5, 0x6c, 0x9e, 0x40, 0x7f, 0xee, 0xff, 0x04,
|
||||||
|
- 0xa3, 0x4f, 0xf9, 0xdf, 0x0c, 0x97, 0x32, 0x7a, 0x04, 0xed, 0xdc, 0x1f, 0x08, 0x68, 0xa8, 0x98,
|
||||||
|
- 0x2c, 0xfe, 0xa7, 0x70, 0x29, 0x83, 0x6d, 0xe8, 0x16, 0x66, 0xfa, 0x68, 0xa4, 0xed, 0x29, 0x19,
|
||||||
|
- 0xf4, 0x5f, 0xca, 0x64, 0x0b, 0xda, 0xb9, 0xd1, 0xba, 0xd1, 0x62, 0x71, 0x7e, 0x3f, 0xba, 0x53,
|
||||||
|
- 0xb2, 0xa3, 0x53, 0xef, 0x1e, 0x74, 0x0b, 0x83, 0x70, 0xa3, 0x48, 0xd9, 0x10, 0x7e, 0x74, 0xb7,
|
||||||
|
- 0x74, 0x4f, 0x73, 0x7a, 0x02, 0xfd, 0xb9, 0xb1, 0xb8, 0x71, 0x6e, 0xf9, 0xb4, 0xfc, 0x52, 0xb3,
|
||||||
|
- 0xbe, 0x90, 0x97, 0x9d, 0xeb, 0x7a, 0x72, 0x97, 0xbd, 0x38, 0x04, 0x1f, 0xbd, 0x52, 0xbe, 0xa9,
|
||||||
|
- 0xb5, 0xda, 0x85, 0x5e, 0x71, 0xfe, 0x6d, 0x98, 0x95, 0x4e, 0xc5, 0xaf, 0x7e, 0x39, 0x85, 0x51,
|
||||||
|
- 0x78, 0xf6, 0x72, 0xca, 0x26, 0xe4, 0x97, 0x32, 0x7a, 0x0c, 0xa0, 0x7b, 0x1c, 0x9f, 0x84, 0xe9,
|
||||||
|
- 0x95, 0x2d, 0xf4, 0x56, 0xe9, 0x95, 0x95, 0xf4, 0x43, 0x8f, 0x00, 0x54, 0x6b, 0xe2, 0xd3, 0x84,
|
||||||
|
- 0xa3, 0xdb, 0x46, 0x8d, 0xb9, 0x7e, 0x68, 0x34, 0x5c, 0xdc, 0x58, 0x60, 0x80, 0x19, 0xbb, 0x0e,
|
||||||
|
- 0x83, 0xcf, 0x01, 0xb2, 0x96, 0xc7, 0x30, 0x58, 0x68, 0x82, 0xae, 0xf0, 0x41, 0x27, 0xdf, 0xe0,
|
||||||
|
- 0x20, 0x6d, 0x6b, 0x49, 0xd3, 0x73, 0x05, 0x8b, 0xfe, 0x5c, 0x01, 0x5b, 0x7c, 0x6c, 0xf3, 0x75,
|
||||||
|
- 0xed, 0x68, 0xa1, 0x88, 0x45, 0x1f, 0x41, 0x27, 0x5f, 0xb9, 0x1a, 0x2d, 0x4a, 0xaa, 0xd9, 0x51,
|
||||||
|
- 0xa1, 0x7a, 0x45, 0x8f, 0xa0, 0x57, 0xac, 0x5a, 0x51, 0x2e, 0x2e, 0x16, 0x6a, 0xd9, 0x91, 0x9e,
|
||||||
|
- 0xc9, 0xe4, 0xc8, 0x3f, 0x00, 0xc8, 0xaa, 0x5b, 0xe3, 0xbe, 0x85, 0x7a, 0x77, 0x4e, 0xea, 0x63,
|
||||||
|
- 0xe8, 0xe4, 0x33, 0xb1, 0x51, 0xb7, 0x24, 0x3b, 0x5f, 0x95, 0xb5, 0x72, 0x59, 0xdb, 0x3c, 0xbe,
|
||||||
|
- 0xc5, 0x44, 0x7e, 0x55, 0xd6, 0x2a, 0xf4, 0x75, 0x26, 0x59, 0x94, 0x35, 0x7b, 0x57, 0xe5, 0xf2,
|
||||||
|
- 0x62, 0x13, 0x64, 0xdc, 0x57, 0xda, 0x1a, 0x5d, 0xf5, 0x88, 0xf2, 0xdd, 0x80, 0xf1, 0x47, 0x49,
|
||||||
|
- 0x87, 0xf0, 0x92, 0xa0, 0xce, 0x57, 0xfc, 0xb9, 0xa0, 0x2e, 0x69, 0x04, 0x2e, 0x65, 0xb4, 0x07,
|
||||||
|
- 0xfd, 0x27, 0xa6, 0x98, 0xd3, 0x85, 0xa6, 0x56, 0xa7, 0xa4, 0xb0, 0x1e, 0x8d, 0xca, 0xb6, 0x74,
|
||||||
|
- 0x64, 0x7d, 0x01, 0x83, 0x85, 0x22, 0x13, 0xdd, 0x4b, 0x47, 0x87, 0xa5, 0xd5, 0xe7, 0xa5, 0x6a,
|
||||||
|
- 0xed, 0xc3, 0xca, 0x7c, 0x8d, 0x89, 0x5e, 0xd5, 0x97, 0x5e, 0x5e, 0x7b, 0x5e, 0xca, 0xea, 0x13,
|
||||||
|
- 0x68, 0x9a, 0x9a, 0x06, 0xe9, 0x11, 0xed, 0x5c, 0x8d, 0x73, 0xd9, 0xd1, 0xad, 0xce, 0x77, 0xdf,
|
||||||
|
- 0xdf, 0xab, 0xfc, 0xe3, 0xfb, 0x7b, 0x95, 0x7f, 0x7d, 0x7f, 0xaf, 0x72, 0xdc, 0x90, 0xbb, 0x1f,
|
||||||
|
- 0xfc, 0x27, 0x00, 0x00, 0xff, 0xff, 0xa5, 0xac, 0x85, 0x1d, 0xaa, 0x21, 0x00, 0x00,
|
||||||
|
+ 0xd1, 0xd8, 0x07, 0x97, 0xbb, 0xb5, 0x2f, 0x6e, 0x93, 0xa2, 0x56, 0x2b, 0x59, 0x9f, 0x3c, 0xb6,
|
||||||
|
+ 0x65, 0xfa, 0xf3, 0xe7, 0xa5, 0x2d, 0x1b, 0x9f, 0x5f, 0x70, 0x04, 0xf1, 0x11, 0x91, 0xb1, 0x15,
|
||||||
|
+ 0x31, 0x43, 0x11, 0x4e, 0x10, 0x04, 0x83, 0xe1, 0x4c, 0x73, 0xd9, 0xe6, 0xce, 0xf4, 0xb8, 0xa7,
|
||||||
|
+ 0x87, 0xe2, 0x3a, 0x40, 0x8e, 0xc9, 0x2d, 0x97, 0x00, 0xb9, 0xe5, 0x0f, 0x04, 0xb9, 0xe5, 0x98,
|
||||||
|
+ 0x6b, 0x0e, 0x46, 0x4e, 0xf9, 0x05, 0x41, 0xe0, 0x9f, 0x90, 0x5f, 0x10, 0xf4, 0x6b, 0x1e, 0xbb,
|
||||||
|
+ 0x43, 0x1a, 0x21, 0x08, 0xe4, 0xb2, 0xe8, 0xaa, 0xae, 0xae, 0x57, 0x77, 0xd5, 0x54, 0xd5, 0x42,
|
||||||
|
+ 0xdb, 0x9d, 0xe0, 0x90, 0x8f, 0x23, 0x46, 0x39, 0x45, 0xf5, 0x09, 0x8b, 0xbc, 0x51, 0x8b, 0x7a,
|
||||||
|
+ 0x44, 0x21, 0x46, 0xff, 0x3f, 0x21, 0xfc, 0x34, 0x39, 0x1e, 0x7b, 0x34, 0xd8, 0x3c, 0x73, 0xb9,
|
||||||
|
+ 0xfb, 0x8e, 0x47, 0x43, 0xee, 0x92, 0x10, 0xb3, 0x78, 0x53, 0x1e, 0xdc, 0x8c, 0xce, 0x26, 0x9b,
|
||||||
|
+ 0x7c, 0x16, 0xe1, 0x58, 0xfd, 0xea, 0x73, 0x77, 0x27, 0x94, 0x4e, 0xa6, 0x78, 0x53, 0x42, 0xc7,
|
||||||
|
+ 0xc9, 0xc9, 0x26, 0x0e, 0x22, 0x3e, 0x53, 0x9b, 0xd6, 0x1f, 0xaa, 0xb0, 0xbe, 0xcd, 0xb0, 0xcb,
|
||||||
|
+ 0xf1, 0xb6, 0xe1, 0x66, 0xe3, 0xaf, 0x13, 0x1c, 0x73, 0xf4, 0x2a, 0x74, 0x52, 0x09, 0x0e, 0xf1,
|
||||||
|
+ 0x87, 0x95, 0x07, 0x95, 0x8d, 0x96, 0xdd, 0x4e, 0x71, 0xfb, 0x3e, 0xba, 0x0d, 0xcb, 0xf8, 0x02,
|
||||||
|
+ 0x7b, 0x62, 0xb7, 0x2a, 0x77, 0x1b, 0x02, 0xdc, 0xf7, 0xd1, 0x7b, 0xd0, 0x8e, 0x39, 0x23, 0xe1,
|
||||||
|
+ 0xc4, 0x49, 0x62, 0xcc, 0x86, 0xb5, 0x07, 0x95, 0x8d, 0xf6, 0xa3, 0x95, 0xb1, 0x30, 0x69, 0x7c,
|
||||||
|
+ 0x28, 0x37, 0x8e, 0x62, 0xcc, 0x6c, 0x88, 0xd3, 0x35, 0x7a, 0x08, 0xcb, 0x3e, 0x3e, 0x27, 0x1e,
|
||||||
|
+ 0x8e, 0x87, 0xf5, 0x07, 0xb5, 0x8d, 0xf6, 0xa3, 0x8e, 0x22, 0xdf, 0x91, 0x48, 0xdb, 0x6c, 0xa2,
|
||||||
|
+ 0xb7, 0xa0, 0x19, 0x73, 0xca, 0xdc, 0x09, 0x8e, 0x87, 0x4b, 0x92, 0xb0, 0x6b, 0xf8, 0x4a, 0xac,
|
||||||
|
+ 0x9d, 0x6e, 0xa3, 0x7b, 0x50, 0x7b, 0xbe, 0xbd, 0x3f, 0x6c, 0x48, 0xe9, 0xa0, 0xa9, 0x22, 0xec,
|
||||||
|
+ 0xd9, 0x02, 0x8d, 0x5e, 0x83, 0x6e, 0xec, 0x86, 0xfe, 0x31, 0xbd, 0x70, 0x22, 0xe2, 0x87, 0xf1,
|
||||||
|
+ 0x70, 0xf9, 0x41, 0x65, 0xa3, 0x69, 0x77, 0x34, 0xf2, 0x40, 0xe0, 0xac, 0x4f, 0xe0, 0xd6, 0x21,
|
||||||
|
+ 0x77, 0x19, 0xbf, 0x86, 0x77, 0xac, 0x23, 0x58, 0xb7, 0x71, 0x40, 0xcf, 0xaf, 0xe5, 0xda, 0x21,
|
||||||
|
+ 0x2c, 0x73, 0x12, 0x60, 0x9a, 0x70, 0xe9, 0xda, 0xae, 0x6d, 0x40, 0xeb, 0x4f, 0x15, 0x40, 0xbb,
|
||||||
|
+ 0x17, 0xd8, 0x3b, 0x60, 0xd4, 0xc3, 0x71, 0xfc, 0x5f, 0xba, 0xae, 0x37, 0x61, 0x39, 0x52, 0x0a,
|
||||||
|
+ 0x0c, 0xeb, 0x92, 0x5c, 0xdf, 0x82, 0xd1, 0xca, 0xec, 0x5a, 0x5f, 0xc1, 0xda, 0x21, 0x99, 0x84,
|
||||||
|
+ 0xee, 0xf4, 0x06, 0xf5, 0x5d, 0x87, 0x46, 0x2c, 0x79, 0x4a, 0x55, 0xbb, 0xb6, 0x86, 0xac, 0x03,
|
||||||
|
+ 0x40, 0x5f, 0xba, 0x84, 0xdf, 0x9c, 0x24, 0xeb, 0x1d, 0x58, 0x2d, 0x70, 0x8c, 0x23, 0x1a, 0xc6,
|
||||||
|
+ 0x58, 0x2a, 0xc0, 0x5d, 0x9e, 0xc4, 0x92, 0xd9, 0x92, 0xad, 0x21, 0x0b, 0xc3, 0xda, 0x17, 0x24,
|
||||||
|
+ 0x36, 0xe4, 0xf8, 0x3f, 0x51, 0x61, 0x1d, 0x1a, 0x27, 0x94, 0x05, 0x2e, 0x37, 0x1a, 0x28, 0x08,
|
||||||
|
+ 0x21, 0xa8, 0xbb, 0x6c, 0x12, 0x0f, 0x6b, 0x0f, 0x6a, 0x1b, 0x2d, 0x5b, 0xae, 0xc5, 0xab, 0x9c,
|
||||||
|
+ 0x13, 0xa3, 0xf5, 0x7a, 0x15, 0x3a, 0xda, 0xef, 0xce, 0x94, 0xc4, 0x5c, 0xca, 0xe9, 0xd8, 0x6d,
|
||||||
|
+ 0x8d, 0x13, 0x67, 0x2c, 0x0a, 0xeb, 0x47, 0x91, 0x7f, 0xcd, 0x80, 0x7f, 0x04, 0x2d, 0x86, 0x63,
|
||||||
|
+ 0x9a, 0x30, 0x11, 0xa6, 0x55, 0x79, 0xef, 0x6b, 0xea, 0xde, 0xbf, 0x20, 0x61, 0x72, 0x61, 0x9b,
|
||||||
|
+ 0x3d, 0x3b, 0x23, 0xd3, 0x21, 0xc4, 0xe3, 0xeb, 0x84, 0xd0, 0x27, 0x70, 0xeb, 0xc0, 0x4d, 0xe2,
|
||||||
|
+ 0xeb, 0xe8, 0x6a, 0x7d, 0x2a, 0xc2, 0x2f, 0x4e, 0x82, 0x6b, 0x1d, 0xfe, 0x63, 0x05, 0x9a, 0xdb,
|
||||||
|
+ 0x51, 0x72, 0x14, 0xbb, 0x13, 0x8c, 0xfe, 0x07, 0xda, 0x9c, 0x72, 0x77, 0xea, 0x24, 0x02, 0x94,
|
||||||
|
+ 0xe4, 0x75, 0x1b, 0x24, 0x4a, 0x11, 0x08, 0xb7, 0x63, 0xe6, 0x45, 0x89, 0xa6, 0xa8, 0x3e, 0xa8,
|
||||||
|
+ 0x6d, 0xd4, 0xed, 0xb6, 0xc2, 0x29, 0x92, 0x31, 0xac, 0xca, 0x3d, 0x87, 0x84, 0xce, 0x19, 0x66,
|
||||||
|
+ 0x21, 0x9e, 0x06, 0xd4, 0xc7, 0xf2, 0xfd, 0xd6, 0xed, 0x81, 0xdc, 0xda, 0x0f, 0x3f, 0x4f, 0x37,
|
||||||
|
+ 0xd0, 0xff, 0xc2, 0x20, 0xa5, 0x17, 0x41, 0x29, 0xa9, 0xeb, 0x92, 0xba, 0xaf, 0xa9, 0x8f, 0x34,
|
||||||
|
+ 0xda, 0xfa, 0x15, 0xf4, 0x5e, 0x9c, 0x32, 0xca, 0xf9, 0x94, 0x84, 0x93, 0x1d, 0x97, 0xbb, 0x22,
|
||||||
|
+ 0x7b, 0x44, 0x98, 0x11, 0xea, 0xc7, 0x5a, 0x5b, 0x03, 0xa2, 0xb7, 0x61, 0xc0, 0x15, 0x2d, 0xf6,
|
||||||
|
+ 0x1d, 0x43, 0x53, 0x95, 0x34, 0x2b, 0xe9, 0xc6, 0x81, 0x26, 0x7e, 0x03, 0x7a, 0x19, 0xb1, 0xc8,
|
||||||
|
+ 0x3f, 0x5a, 0xdf, 0x6e, 0x8a, 0x7d, 0x41, 0x02, 0x6c, 0x9d, 0x4b, 0x5f, 0xc9, 0x4b, 0x46, 0x6f,
|
||||||
|
+ 0x43, 0x2b, 0xf3, 0x43, 0x45, 0xbe, 0x90, 0x9e, 0x7a, 0x21, 0xc6, 0x9d, 0x76, 0x33, 0x75, 0xca,
|
||||||
|
+ 0x67, 0xd0, 0xe7, 0xa9, 0xe2, 0x8e, 0xef, 0x72, 0xb7, 0xf8, 0xa8, 0x8a, 0x56, 0xd9, 0x3d, 0x5e,
|
||||||
|
+ 0x80, 0xad, 0x4f, 0xa1, 0x75, 0x40, 0xfc, 0x58, 0x09, 0x1e, 0xc2, 0xb2, 0x97, 0x30, 0x86, 0x43,
|
||||||
|
+ 0x6e, 0x4c, 0xd6, 0x20, 0x5a, 0x83, 0xa5, 0x29, 0x09, 0x08, 0xd7, 0x66, 0x2a, 0xc0, 0xa2, 0x00,
|
||||||
|
+ 0xcf, 0x70, 0x40, 0xd9, 0x4c, 0x3a, 0x6c, 0x0d, 0x96, 0xf2, 0x97, 0xab, 0x00, 0x74, 0x17, 0x5a,
|
||||||
|
+ 0x81, 0x7b, 0x91, 0x5e, 0xaa, 0xd8, 0x69, 0x06, 0xee, 0x85, 0x52, 0x7e, 0x08, 0xcb, 0x27, 0x2e,
|
||||||
|
+ 0x99, 0x7a, 0x21, 0xd7, 0x5e, 0x31, 0x60, 0x26, 0xb0, 0x9e, 0x17, 0xf8, 0xd7, 0x2a, 0xb4, 0x95,
|
||||||
|
+ 0x44, 0xa5, 0xf0, 0x1a, 0x2c, 0x79, 0xae, 0x77, 0x9a, 0x8a, 0x94, 0x00, 0x7a, 0x68, 0x14, 0xa9,
|
||||||
|
+ 0xe6, 0x93, 0x70, 0xa6, 0xa9, 0x51, 0x6d, 0x13, 0x20, 0x7e, 0xe9, 0x46, 0x5a, 0xb7, 0xda, 0x25,
|
||||||
|
+ 0xc4, 0x2d, 0x41, 0xa3, 0xd4, 0x7d, 0x1f, 0x3a, 0xea, 0xdd, 0xe9, 0x23, 0xf5, 0x4b, 0x8e, 0xb4,
|
||||||
|
+ 0x15, 0x95, 0x3a, 0xf4, 0x1a, 0x74, 0x93, 0x18, 0x3b, 0xa7, 0x04, 0x33, 0x97, 0x79, 0xa7, 0xb3,
|
||||||
|
+ 0xe1, 0x92, 0xfa, 0x46, 0x26, 0x31, 0xde, 0x33, 0x38, 0xf4, 0x08, 0x96, 0x44, 0xfa, 0x8b, 0x87,
|
||||||
|
+ 0x0d, 0xf9, 0x39, 0xbe, 0x97, 0x67, 0x29, 0x4d, 0x1d, 0xcb, 0xdf, 0xdd, 0x90, 0xb3, 0x99, 0xad,
|
||||||
|
+ 0x48, 0x47, 0x1f, 0x01, 0x64, 0x48, 0xb4, 0x02, 0xb5, 0x33, 0x3c, 0xd3, 0x71, 0x28, 0x96, 0xc2,
|
||||||
|
+ 0x39, 0xe7, 0xee, 0x34, 0x31, 0x5e, 0x57, 0xc0, 0x27, 0xd5, 0x8f, 0x2a, 0x96, 0x07, 0xfd, 0xad,
|
||||||
|
+ 0xe9, 0x19, 0xa1, 0xb9, 0xe3, 0x6b, 0xb0, 0x14, 0xb8, 0x5f, 0x51, 0x66, 0x3c, 0x29, 0x01, 0x89,
|
||||||
|
+ 0x25, 0x21, 0x65, 0x86, 0x85, 0x04, 0x50, 0x0f, 0xaa, 0x34, 0x92, 0xfe, 0x6a, 0xd9, 0x55, 0x1a,
|
||||||
|
+ 0x65, 0x82, 0xea, 0x39, 0x41, 0xd6, 0x3f, 0xea, 0x00, 0x99, 0x14, 0x64, 0xc3, 0x88, 0x50, 0x27,
|
||||||
|
+ 0xc6, 0x4c, 0x94, 0x20, 0xce, 0xf1, 0x8c, 0xe3, 0xd8, 0x61, 0xd8, 0x4b, 0x58, 0x4c, 0xce, 0xc5,
|
||||||
|
+ 0xfd, 0x09, 0xb3, 0x6f, 0x29, 0xb3, 0xe7, 0x74, 0xb3, 0x6f, 0x13, 0x7a, 0xa8, 0xce, 0x6d, 0x89,
|
||||||
|
+ 0x63, 0xb6, 0x39, 0x85, 0xf6, 0xe1, 0x56, 0xc6, 0xd3, 0xcf, 0xb1, 0xab, 0x5e, 0xc5, 0x6e, 0x35,
|
||||||
|
+ 0x65, 0xe7, 0x67, 0xac, 0x76, 0x61, 0x95, 0x50, 0xe7, 0xeb, 0x04, 0x27, 0x05, 0x46, 0xb5, 0xab,
|
||||||
|
+ 0x18, 0x0d, 0x08, 0xfd, 0x89, 0x3c, 0x90, 0xb1, 0x39, 0x80, 0x3b, 0x39, 0x2b, 0x45, 0xb8, 0xe7,
|
||||||
|
+ 0x98, 0xd5, 0xaf, 0x62, 0xb6, 0x9e, 0x6a, 0x25, 0xf2, 0x41, 0xc6, 0xf1, 0x47, 0xb0, 0x4e, 0xa8,
|
||||||
|
+ 0xf3, 0xd2, 0x25, 0x7c, 0x9e, 0xdd, 0xd2, 0xf7, 0x18, 0x29, 0x3e, 0xba, 0x45, 0x5e, 0xca, 0xc8,
|
||||||
|
+ 0x00, 0xb3, 0x49, 0xc1, 0xc8, 0xc6, 0xf7, 0x18, 0xf9, 0x4c, 0x1e, 0xc8, 0xd8, 0x3c, 0x81, 0x01,
|
||||||
|
+ 0xa1, 0xf3, 0xda, 0x2c, 0x5f, 0xc5, 0xa4, 0x4f, 0x68, 0x51, 0x93, 0x2d, 0x18, 0xc4, 0xd8, 0xe3,
|
||||||
|
+ 0x94, 0xe5, 0x1f, 0x41, 0xf3, 0x2a, 0x16, 0x2b, 0x9a, 0x3e, 0xe5, 0x61, 0xfd, 0x1c, 0x3a, 0x7b,
|
||||||
|
+ 0xc9, 0x04, 0xf3, 0xe9, 0x71, 0x9a, 0x0c, 0x6e, 0x2c, 0xff, 0x58, 0xff, 0xaa, 0x42, 0x7b, 0x7b,
|
||||||
|
+ 0xc2, 0x68, 0x12, 0x15, 0x72, 0xb2, 0x0a, 0xd2, 0xf9, 0x9c, 0x2c, 0x49, 0x64, 0x4e, 0x56, 0xc4,
|
||||||
|
+ 0x1f, 0x40, 0x27, 0x90, 0xa1, 0xab, 0xe9, 0x55, 0x1e, 0x1a, 0x2c, 0x04, 0xb5, 0xdd, 0x0e, 0x72,
|
||||||
|
+ 0xc9, 0x6c, 0x0c, 0x10, 0x11, 0x3f, 0xd6, 0x67, 0x54, 0x3a, 0xea, 0xeb, 0x8a, 0xd0, 0xa4, 0x68,
|
||||||
|
+ 0xbb, 0x15, 0xa5, 0xd9, 0xfa, 0x3d, 0x68, 0x1f, 0x0b, 0x27, 0xe9, 0x03, 0x85, 0x64, 0x94, 0x79,
|
||||||
|
+ 0xcf, 0x86, 0xe3, 0x2c, 0x08, 0xf7, 0xa0, 0x7b, 0xaa, 0x5c, 0xa6, 0x0f, 0xa9, 0x37, 0xf4, 0x9a,
|
||||||
|
+ 0xb6, 0x24, 0xb3, 0x77, 0x9c, 0xf7, 0xac, 0xba, 0x80, 0xce, 0x69, 0x0e, 0x35, 0x3a, 0x84, 0xc1,
|
||||||
|
+ 0x02, 0x49, 0x49, 0x0e, 0xda, 0xc8, 0xe7, 0xa0, 0xf6, 0x23, 0xa4, 0x04, 0xe5, 0x4f, 0xe6, 0xf3,
|
||||||
|
+ 0xd2, 0x6f, 0xab, 0xd0, 0xf9, 0x31, 0xe6, 0x2f, 0x29, 0x3b, 0x53, 0xfa, 0x22, 0xa8, 0x87, 0x6e,
|
||||||
|
+ 0x80, 0x35, 0x47, 0xb9, 0x46, 0x77, 0xa0, 0xc9, 0x2e, 0x54, 0x02, 0xd1, 0xf7, 0xb9, 0xcc, 0x2e,
|
||||||
|
+ 0x64, 0x62, 0x40, 0xaf, 0x00, 0xb0, 0x0b, 0x27, 0x72, 0xbd, 0x33, 0xac, 0x3d, 0x58, 0xb7, 0x5b,
|
||||||
|
+ 0xec, 0xe2, 0x40, 0x21, 0xc4, 0x53, 0x60, 0x17, 0x0e, 0x66, 0x8c, 0xb2, 0x58, 0xe7, 0xaa, 0x26,
|
||||||
|
+ 0xbb, 0xd8, 0x95, 0xb0, 0x3e, 0xeb, 0x33, 0x1a, 0x45, 0xd8, 0x97, 0x39, 0x5a, 0x9e, 0xdd, 0x51,
|
||||||
|
+ 0x08, 0x21, 0x95, 0x1b, 0xa9, 0x0d, 0x25, 0x95, 0x67, 0x52, 0x79, 0x26, 0x75, 0x59, 0x9d, 0xe4,
|
||||||
|
+ 0x79, 0xa9, 0x3c, 0x95, 0xda, 0x54, 0x52, 0x79, 0x4e, 0x2a, 0xcf, 0xa4, 0xb6, 0xcc, 0x59, 0x2d,
|
||||||
|
+ 0xd5, 0xfa, 0x4d, 0x05, 0xd6, 0xe7, 0x0b, 0x3f, 0x5d, 0xa6, 0x7e, 0x00, 0x1d, 0x4f, 0xde, 0x57,
|
||||||
|
+ 0xe1, 0x4d, 0x0e, 0x16, 0x6e, 0xd2, 0x6e, 0x7b, 0xb9, 0x67, 0xfc, 0x21, 0x74, 0x43, 0xe5, 0xe0,
|
||||||
|
+ 0xf4, 0x69, 0xd6, 0xb2, 0x7b, 0xc9, 0xfb, 0xde, 0xee, 0x84, 0x39, 0xc8, 0xf2, 0x01, 0x7d, 0xc9,
|
||||||
|
+ 0x08, 0xc7, 0x87, 0x9c, 0x61, 0x37, 0xb8, 0x89, 0x06, 0x04, 0x41, 0x5d, 0x56, 0x2b, 0x35, 0x59,
|
||||||
|
+ 0x5f, 0xcb, 0xb5, 0xf5, 0x26, 0xac, 0x16, 0xa4, 0x68, 0x5b, 0x57, 0xa0, 0x36, 0xc5, 0xa1, 0xe4,
|
||||||
|
+ 0xde, 0xb5, 0xc5, 0xd2, 0x72, 0x61, 0x60, 0x63, 0xd7, 0xbf, 0x39, 0x6d, 0xb4, 0x88, 0x5a, 0x26,
|
||||||
|
+ 0x62, 0x03, 0x50, 0x5e, 0x84, 0x56, 0xc5, 0x68, 0x5d, 0xc9, 0x69, 0xfd, 0x1c, 0x06, 0xdb, 0x53,
|
||||||
|
+ 0x1a, 0xe3, 0x43, 0xee, 0x93, 0xf0, 0x26, 0x3a, 0xa6, 0x5f, 0xc2, 0xea, 0x0b, 0x3e, 0xfb, 0x52,
|
||||||
|
+ 0x30, 0x8b, 0xc9, 0x37, 0xf8, 0x86, 0xec, 0x63, 0xf4, 0xa5, 0xb1, 0x8f, 0xd1, 0x97, 0xa2, 0x59,
|
||||||
|
+ 0xf2, 0xe8, 0x34, 0x09, 0x42, 0x19, 0x0a, 0x5d, 0x5b, 0x43, 0xd6, 0x16, 0x74, 0x54, 0x0d, 0xfd,
|
||||||
|
+ 0x8c, 0xfa, 0xc9, 0x14, 0x97, 0xc6, 0xe0, 0x7d, 0x80, 0xc8, 0x65, 0x6e, 0x80, 0x39, 0x66, 0xea,
|
||||||
|
+ 0x0d, 0xb5, 0xec, 0x1c, 0xc6, 0xfa, 0x7d, 0x15, 0xd6, 0xd4, 0x48, 0xe4, 0x50, 0x4d, 0x02, 0x8c,
|
||||||
|
+ 0x09, 0x23, 0x68, 0x9e, 0xd2, 0x98, 0xe7, 0x18, 0xa6, 0xb0, 0x50, 0xd1, 0x0f, 0x0d, 0x37, 0xb1,
|
||||||
|
+ 0x2c, 0xcc, 0x29, 0x6a, 0x57, 0xcf, 0x29, 0x16, 0x26, 0x11, 0xf5, 0xc5, 0x49, 0x84, 0x88, 0x36,
|
||||||
|
+ 0x43, 0x44, 0x54, 0x8c, 0xb7, 0xec, 0x96, 0xc6, 0xec, 0xfb, 0xe8, 0x21, 0xf4, 0x27, 0x42, 0x4b,
|
||||||
|
+ 0xe7, 0x94, 0xd2, 0x33, 0x27, 0x72, 0xf9, 0xa9, 0x0c, 0xf5, 0x96, 0xdd, 0x95, 0xe8, 0x3d, 0x4a,
|
||||||
|
+ 0xcf, 0x0e, 0x5c, 0x7e, 0x8a, 0x3e, 0x86, 0x9e, 0x2e, 0x03, 0x03, 0xe9, 0xa2, 0x58, 0x7f, 0xfc,
|
||||||
|
+ 0x74, 0x14, 0xe5, 0xbd, 0x67, 0x77, 0xcf, 0x72, 0x50, 0x6c, 0xdd, 0x86, 0x5b, 0x3b, 0x38, 0xe6,
|
||||||
|
+ 0x8c, 0xce, 0x8a, 0x8e, 0xb1, 0x7e, 0x00, 0xb0, 0x1f, 0x72, 0xcc, 0x4e, 0x5c, 0x0f, 0xc7, 0xe8,
|
||||||
|
+ 0xdd, 0x3c, 0xa4, 0x8b, 0xa3, 0x95, 0xb1, 0x9a, 0x48, 0xa5, 0x1b, 0x76, 0x8e, 0xc6, 0x1a, 0x43,
|
||||||
|
+ 0xc3, 0xa6, 0x89, 0x48, 0x47, 0xaf, 0x9b, 0x95, 0x3e, 0xd7, 0xd1, 0xe7, 0x24, 0xd2, 0xd6, 0x7b,
|
||||||
|
+ 0xd6, 0x9e, 0x69, 0x61, 0x33, 0x76, 0xfa, 0x8a, 0xc6, 0xd0, 0x22, 0x06, 0xa7, 0xb3, 0xca, 0xa2,
|
||||||
|
+ 0xe8, 0x8c, 0xc4, 0xfa, 0x19, 0xac, 0x2a, 0x4e, 0x8a, 0xb3, 0x61, 0xf3, 0x3a, 0x34, 0x98, 0x51,
|
||||||
|
+ 0xa3, 0x92, 0x8d, 0xa2, 0x34, 0x91, 0xde, 0x43, 0xf7, 0x84, 0x30, 0x8f, 0xe1, 0x40, 0xf4, 0x1c,
|
||||||
|
+ 0x55, 0x79, 0x65, 0x19, 0x42, 0x78, 0x4b, 0xf4, 0xdb, 0x99, 0x99, 0xc6, 0x5b, 0xab, 0x30, 0x10,
|
||||||
|
+ 0x1b, 0x05, 0x89, 0xd6, 0x2f, 0x60, 0xf5, 0x79, 0x38, 0x25, 0x21, 0xde, 0x3e, 0x38, 0x7a, 0x86,
|
||||||
|
+ 0xd3, 0xac, 0x80, 0xa0, 0x2e, 0xaa, 0x27, 0xa9, 0x46, 0xd3, 0x96, 0x6b, 0x11, 0x26, 0xe1, 0xb1,
|
||||||
|
+ 0xe3, 0x45, 0x49, 0xac, 0x27, 0x43, 0x8d, 0xf0, 0x78, 0x3b, 0x4a, 0x62, 0x91, 0xe6, 0xc5, 0x67,
|
||||||
|
+ 0x9e, 0x86, 0xd3, 0x99, 0x8c, 0x95, 0xa6, 0xbd, 0xec, 0x45, 0xc9, 0xf3, 0x70, 0x3a, 0xb3, 0xfe,
|
||||||
|
+ 0x4f, 0xf6, 0xc2, 0x18, 0xfb, 0xb6, 0x1b, 0xfa, 0x34, 0xd8, 0xc1, 0xe7, 0x39, 0x09, 0x69, 0xdf,
|
||||||
|
+ 0x65, 0x72, 0xc2, 0xb7, 0x15, 0xe8, 0x3c, 0x99, 0xe0, 0x90, 0xef, 0x60, 0xee, 0x92, 0xa9, 0xec,
|
||||||
|
+ 0xad, 0xce, 0x31, 0x8b, 0x09, 0x0d, 0xf5, 0xc3, 0x37, 0xa0, 0x68, 0x8d, 0x49, 0x48, 0xb8, 0xe3,
|
||||||
|
+ 0xbb, 0x38, 0xa0, 0xa1, 0xf6, 0x02, 0x08, 0xd4, 0x8e, 0xc4, 0xa0, 0x37, 0xa1, 0xaf, 0x26, 0x77,
|
||||||
|
+ 0xce, 0xa9, 0x1b, 0xfa, 0x53, 0x11, 0x72, 0x6a, 0x92, 0xd1, 0x53, 0xe8, 0x3d, 0x8d, 0x45, 0x6f,
|
||||||
|
+ 0xc1, 0x8a, 0x0e, 0x88, 0x8c, 0xb2, 0x2e, 0x29, 0xfb, 0x1a, 0x5f, 0x20, 0x4d, 0xa2, 0x88, 0x32,
|
||||||
|
+ 0x1e, 0x3b, 0x31, 0xf6, 0x3c, 0x1a, 0x44, 0xba, 0x31, 0xe9, 0x1b, 0xfc, 0xa1, 0x42, 0x5b, 0x13,
|
||||||
|
+ 0x58, 0x7d, 0x2a, 0xec, 0xd4, 0x96, 0x64, 0x17, 0xdc, 0x0b, 0x70, 0xe0, 0x1c, 0x4f, 0xa9, 0x77,
|
||||||
|
+ 0xe6, 0x88, 0x34, 0xa5, 0x3d, 0x2c, 0x4a, 0x9f, 0x2d, 0x81, 0x3c, 0x24, 0xdf, 0xc8, 0x1e, 0x5c,
|
||||||
|
+ 0x50, 0x9d, 0x52, 0x1e, 0x4d, 0x93, 0x89, 0x13, 0x31, 0x7a, 0x8c, 0xb5, 0x89, 0xfd, 0x00, 0x07,
|
||||||
|
+ 0x7b, 0x0a, 0x7f, 0x20, 0xd0, 0xd6, 0x5f, 0x2a, 0xb0, 0x56, 0x94, 0xa4, 0x93, 0xee, 0x26, 0xac,
|
||||||
|
+ 0x15, 0x45, 0xe9, 0x0f, 0xb1, 0x2a, 0xf4, 0x06, 0x79, 0x81, 0xea, 0x93, 0xfc, 0x21, 0x74, 0xe5,
|
||||||
|
+ 0x38, 0xd7, 0xf1, 0x15, 0xa7, 0x62, 0xf9, 0x91, 0xbf, 0x17, 0xbb, 0xe3, 0xe6, 0x6f, 0xe9, 0x63,
|
||||||
|
+ 0xb8, 0xa3, 0xcd, 0x77, 0x16, 0xd5, 0x56, 0x0f, 0x62, 0x5d, 0x13, 0x3c, 0x9b, 0xd3, 0xfe, 0x0b,
|
||||||
|
+ 0x18, 0x66, 0xa8, 0xad, 0x99, 0x44, 0x1a, 0x5f, 0xbd, 0x0b, 0xab, 0x73, 0xc6, 0x3e, 0xf1, 0x7d,
|
||||||
|
+ 0x26, 0x03, 0xb4, 0x6e, 0x97, 0x6d, 0x59, 0x8f, 0xe1, 0xf6, 0x21, 0xe6, 0xca, 0x1b, 0x2e, 0xd7,
|
||||||
|
+ 0x3d, 0x81, 0x62, 0xb6, 0x02, 0xb5, 0x43, 0xec, 0x49, 0xe3, 0x6b, 0xb6, 0x58, 0x8a, 0x07, 0x78,
|
||||||
|
+ 0x14, 0x63, 0x4f, 0x5a, 0x59, 0xb3, 0xe5, 0xda, 0xfa, 0x73, 0x05, 0x96, 0x75, 0x9a, 0x14, 0xa9,
|
||||||
|
+ 0xde, 0x67, 0xe4, 0x1c, 0x33, 0xfd, 0xf4, 0x34, 0x84, 0xde, 0x80, 0x9e, 0x5a, 0x39, 0x34, 0xe2,
|
||||||
|
+ 0x84, 0xa6, 0xc9, 0xb7, 0xab, 0xb0, 0xcf, 0x15, 0x52, 0x4e, 0xea, 0xe4, 0x20, 0x4a, 0xf7, 0x7c,
|
||||||
|
+ 0x1a, 0x92, 0xe3, 0xb6, 0x58, 0x64, 0x06, 0x99, 0x6c, 0x5b, 0xb6, 0x86, 0xc4, 0x53, 0x37, 0xfc,
|
||||||
|
+ 0x96, 0x24, 0x3f, 0x03, 0x8a, 0xa7, 0x1e, 0xd0, 0x24, 0xe4, 0x4e, 0x44, 0x49, 0xc8, 0x75, 0x76,
|
||||||
|
+ 0x05, 0x89, 0x3a, 0x10, 0x18, 0xeb, 0xd7, 0x15, 0x68, 0xa8, 0x69, 0xb5, 0xe8, 0x32, 0xd3, 0x6f,
|
||||||
|
+ 0x5c, 0x95, 0xc8, 0x7a, 0x41, 0xca, 0x52, 0xdf, 0x35, 0xb9, 0x16, 0x71, 0x7c, 0x1e, 0xa8, 0x4c,
|
||||||
|
+ 0xad, 0x55, 0x3b, 0x0f, 0x64, 0x8a, 0x7e, 0x03, 0x7a, 0xd9, 0xa7, 0x52, 0xee, 0x2b, 0x15, 0xbb,
|
||||||
|
+ 0x29, 0x56, 0x92, 0x5d, 0xaa, 0xa9, 0xf5, 0x53, 0xd1, 0x5c, 0xa7, 0x93, 0xda, 0x15, 0xa8, 0x25,
|
||||||
|
+ 0xa9, 0x32, 0x62, 0x29, 0x30, 0x93, 0xf4, 0x23, 0x2b, 0x96, 0xe8, 0x21, 0xf4, 0x5c, 0xdf, 0x27,
|
||||||
|
+ 0xe2, 0xb8, 0x3b, 0x7d, 0x4a, 0xfc, 0x34, 0x48, 0x8b, 0x58, 0xeb, 0x6f, 0x15, 0xe8, 0x6f, 0xd3,
|
||||||
|
+ 0x68, 0xf6, 0x43, 0x32, 0xc5, 0xb9, 0x0c, 0x22, 0x95, 0xd4, 0xdf, 0x58, 0xb1, 0x16, 0x75, 0xe3,
|
||||||
|
+ 0x09, 0x99, 0x62, 0x15, 0x5a, 0xea, 0x66, 0x9b, 0x02, 0x21, 0xc3, 0xca, 0x6c, 0xa6, 0x03, 0xb0,
|
||||||
|
+ 0xae, 0xda, 0x7c, 0x46, 0x7d, 0x59, 0x21, 0xfb, 0x84, 0x39, 0xe9, 0xb8, 0xab, 0x6b, 0x2f, 0xfb,
|
||||||
|
+ 0x84, 0xc9, 0x2d, 0x6d, 0xc8, 0x92, 0x9c, 0xb8, 0xe6, 0x0d, 0x69, 0x28, 0x8c, 0x30, 0x64, 0x1d,
|
||||||
|
+ 0x1a, 0xf4, 0xe4, 0x24, 0xc6, 0x5c, 0xd6, 0xb2, 0x35, 0x5b, 0x43, 0x69, 0x9a, 0x6b, 0xe6, 0xd2,
|
||||||
|
+ 0xdc, 0x2d, 0x58, 0x95, 0xb3, 0xfd, 0x17, 0xcc, 0xf5, 0x48, 0x38, 0x31, 0xa9, 0x78, 0x0d, 0xd0,
|
||||||
|
+ 0x21, 0xa7, 0x51, 0x11, 0xfb, 0xe8, 0x77, 0x2b, 0x3a, 0x27, 0xea, 0x46, 0x17, 0x3d, 0x85, 0xfe,
|
||||||
|
+ 0xdc, 0x1f, 0x27, 0x48, 0x4f, 0x3e, 0xca, 0xff, 0x4f, 0x19, 0xad, 0x8f, 0xd5, 0x1f, 0x31, 0x63,
|
||||||
|
+ 0xf3, 0x47, 0xcc, 0x78, 0x37, 0x88, 0xf8, 0x0c, 0xed, 0x42, 0xaf, 0xf8, 0x17, 0x03, 0xba, 0x6b,
|
||||||
|
+ 0x0a, 0x85, 0x92, 0x3f, 0x1e, 0x2e, 0x65, 0xf3, 0x14, 0xfa, 0x73, 0xff, 0x36, 0x18, 0x7d, 0xca,
|
||||||
|
+ 0xff, 0x84, 0xb8, 0x94, 0xd1, 0x63, 0x68, 0xe7, 0xfe, 0x5e, 0x40, 0x43, 0xc5, 0x64, 0xf1, 0x1f,
|
||||||
|
+ 0x87, 0x4b, 0x19, 0x6c, 0x43, 0xb7, 0x30, 0xf1, 0x47, 0x23, 0x6d, 0x4f, 0xc9, 0xdf, 0x00, 0x97,
|
||||||
|
+ 0x32, 0xd9, 0x82, 0x76, 0x6e, 0xf0, 0x6e, 0xb4, 0x58, 0x9c, 0xee, 0x8f, 0xee, 0x94, 0xec, 0xe8,
|
||||||
|
+ 0xd4, 0xbb, 0x07, 0xdd, 0xc2, 0x98, 0xdc, 0x28, 0x52, 0x36, 0xa2, 0x1f, 0xdd, 0x2d, 0xdd, 0xd3,
|
||||||
|
+ 0x9c, 0x9e, 0x42, 0x7f, 0x6e, 0x68, 0x6e, 0x9c, 0x5b, 0x3e, 0x4b, 0xbf, 0xd4, 0xac, 0xcf, 0xe5,
|
||||||
|
+ 0x65, 0xe7, 0x7a, 0xa2, 0xdc, 0x65, 0x2f, 0x8e, 0xc8, 0x47, 0xf7, 0xca, 0x37, 0xb5, 0x56, 0xbb,
|
||||||
|
+ 0xd0, 0x2b, 0x4e, 0xc7, 0x0d, 0xb3, 0xd2, 0x99, 0xf9, 0xd5, 0x2f, 0xa7, 0x30, 0x28, 0xcf, 0x5e,
|
||||||
|
+ 0x4e, 0xd9, 0xfc, 0xfc, 0x52, 0x46, 0x4f, 0x00, 0x74, 0x07, 0xe4, 0x93, 0x30, 0xbd, 0xb2, 0x85,
|
||||||
|
+ 0xce, 0x2b, 0xbd, 0xb2, 0x92, 0x6e, 0xe9, 0x31, 0x80, 0x6a, 0x5c, 0x7c, 0x9a, 0x70, 0x74, 0xdb,
|
||||||
|
+ 0xa8, 0x31, 0xd7, 0x2d, 0x8d, 0x86, 0x8b, 0x1b, 0x0b, 0x0c, 0x30, 0x63, 0xd7, 0x61, 0xf0, 0x19,
|
||||||
|
+ 0x40, 0xd6, 0x10, 0x19, 0x06, 0x0b, 0x2d, 0xd2, 0x15, 0x3e, 0xe8, 0xe4, 0xdb, 0x1f, 0xa4, 0x6d,
|
||||||
|
+ 0x2d, 0x69, 0x89, 0xae, 0x60, 0xd1, 0x9f, 0x2b, 0x6f, 0x8b, 0x8f, 0x6d, 0xbe, 0xea, 0x1d, 0x2d,
|
||||||
|
+ 0x94, 0xb8, 0xe8, 0x43, 0xe8, 0xe4, 0xeb, 0x5a, 0xa3, 0x45, 0x49, 0xad, 0x3b, 0x2a, 0xd4, 0xb6,
|
||||||
|
+ 0xe8, 0x31, 0xf4, 0x8a, 0x55, 0x2b, 0xca, 0xc5, 0xc5, 0x42, 0x2d, 0x3b, 0xd2, 0x13, 0x9b, 0x1c,
|
||||||
|
+ 0xf9, 0xfb, 0x00, 0x59, 0x75, 0x6b, 0xdc, 0xb7, 0x50, 0xef, 0xce, 0x49, 0x7d, 0x02, 0x9d, 0x7c,
|
||||||
|
+ 0x26, 0x36, 0xea, 0x96, 0x64, 0xe7, 0xab, 0xb2, 0x56, 0x2e, 0x6b, 0x9b, 0xc7, 0xb7, 0x98, 0xc8,
|
||||||
|
+ 0xaf, 0xca, 0x5a, 0x85, 0xae, 0xcf, 0x24, 0x8b, 0xb2, 0x56, 0xf0, 0xaa, 0x5c, 0x5e, 0x6c, 0x91,
|
||||||
|
+ 0x8c, 0xfb, 0x4a, 0x1b, 0xa7, 0xab, 0x1e, 0x51, 0xbe, 0x1b, 0x30, 0xfe, 0x28, 0xe9, 0x10, 0xbe,
|
||||||
|
+ 0x27, 0xa8, 0xf3, 0x15, 0x7f, 0x2e, 0xa8, 0x4b, 0x1a, 0x81, 0x4b, 0x19, 0xed, 0x41, 0xff, 0xa9,
|
||||||
|
+ 0x29, 0xe6, 0x74, 0xa1, 0xa9, 0xd5, 0x29, 0x29, 0xac, 0x47, 0xa3, 0xb2, 0x2d, 0x1d, 0x59, 0x9f,
|
||||||
|
+ 0xc3, 0x60, 0xa1, 0xc8, 0x44, 0xf7, 0xd3, 0xc1, 0x62, 0x69, 0xf5, 0x79, 0xa9, 0x5a, 0xfb, 0xb0,
|
||||||
|
+ 0x32, 0x5f, 0x63, 0xa2, 0x57, 0xf4, 0xa5, 0x97, 0xd7, 0x9e, 0x97, 0xb2, 0xfa, 0x18, 0x9a, 0xa6,
|
||||||
|
+ 0xa6, 0x41, 0x7a, 0x80, 0x3b, 0x57, 0xe3, 0x5c, 0x76, 0x74, 0xab, 0xf3, 0xed, 0x77, 0xf7, 0x2b,
|
||||||
|
+ 0x7f, 0xff, 0xee, 0x7e, 0xe5, 0x9f, 0xdf, 0xdd, 0xaf, 0x1c, 0x37, 0xe4, 0xee, 0xfb, 0xff, 0x0e,
|
||||||
|
+ 0x00, 0x00, 0xff, 0xff, 0x8d, 0x89, 0xaa, 0x73, 0xc8, 0x21, 0x00, 0x00,
|
||||||
|
}
|
||||||
|
diff --git a/protocols/grpc/agent.proto b/protocols/grpc/agent.proto
|
||||||
|
index 348d792..b0fab6d 100644
|
||||||
|
--- a/protocols/grpc/agent.proto
|
||||||
|
+++ b/protocols/grpc/agent.proto
|
||||||
|
@@ -316,6 +316,7 @@ message UpdateInterfaceRequest {
|
||||||
|
|
||||||
|
message UpdateRoutesRequest {
|
||||||
|
Routes routes = 1;
|
||||||
|
+ bool increment = 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
message ListInterfacesRequest {
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
971
agent/patches/0003-kata-agent-add-kata-ipvs-command.patch
Normal file
971
agent/patches/0003-kata-agent-add-kata-ipvs-command.patch
Normal file
@ -0,0 +1,971 @@
|
|||||||
|
From 0f24d3d433e93a7309b9bac59e4a15e722391490 Mon Sep 17 00:00:00 2001
|
||||||
|
From: xiadanni1 <xiadanni1@huawei.com>
|
||||||
|
Date: Tue, 18 Aug 2020 06:01:55 +0800
|
||||||
|
Subject: [PATCH 03/16] kata-agent: add kata-ipvs command
|
||||||
|
|
||||||
|
reason: add kata-ipvs command to update IPVS rules
|
||||||
|
in VM.
|
||||||
|
|
||||||
|
Signed-off-by: xiadanni1 <xiadanni1@huawei.com>
|
||||||
|
---
|
||||||
|
agent.go | 6 +
|
||||||
|
grpc.go | 4 +
|
||||||
|
ipvsadm.go | 130 +++++++++
|
||||||
|
protocols/grpc/agent.pb.go | 663 ++++++++++++++++++++++++++++++++-------------
|
||||||
|
protocols/grpc/agent.proto | 11 +
|
||||||
|
5 files changed, 633 insertions(+), 181 deletions(-)
|
||||||
|
create mode 100644 ipvsadm.go
|
||||||
|
|
||||||
|
diff --git a/agent.go b/agent.go
|
||||||
|
index c1cac08..c161e93 100644
|
||||||
|
--- a/agent.go
|
||||||
|
+++ b/agent.go
|
||||||
|
@@ -118,6 +118,11 @@ type sandboxStorage struct {
|
||||||
|
refCount int
|
||||||
|
}
|
||||||
|
|
||||||
|
+type ipvsAdm struct {
|
||||||
|
+ ipvsLock sync.Mutex
|
||||||
|
+ ipvsRuleCnt uint64
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
type sandbox struct {
|
||||||
|
sync.RWMutex
|
||||||
|
ctx context.Context
|
||||||
|
@@ -144,6 +149,7 @@ type sandbox struct {
|
||||||
|
sandboxPidNs bool
|
||||||
|
storages map[string]*sandboxStorage
|
||||||
|
stopServer chan struct{}
|
||||||
|
+ ipvsadm ipvsAdm
|
||||||
|
}
|
||||||
|
|
||||||
|
var agentFields = logrus.Fields{
|
||||||
|
diff --git a/grpc.go b/grpc.go
|
||||||
|
index 8fe8217..de2cae7 100644
|
||||||
|
--- a/grpc.go
|
||||||
|
+++ b/grpc.go
|
||||||
|
@@ -1567,6 +1567,10 @@ func (a *agentGRPC) ListRoutes(ctx context.Context, req *pb.ListRoutesRequest) (
|
||||||
|
return a.sandbox.listRoutes(nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (a *agentGRPC) UpdateIPVSRule(ctx context.Context, req *pb.UpdateIPVSRequest) (*pb.IPVSResponse, error) {
|
||||||
|
+ return a.sandbox.updateIPVSRule(req)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func (a *agentGRPC) OnlineCPUMem(ctx context.Context, req *pb.OnlineCPUMemRequest) (*gpb.Empty, error) {
|
||||||
|
if !req.Wait {
|
||||||
|
go a.onlineCPUMem(req)
|
||||||
|
diff --git a/ipvsadm.go b/ipvsadm.go
|
||||||
|
new file mode 100644
|
||||||
|
index 0000000..48eb19f
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/ipvsadm.go
|
||||||
|
@@ -0,0 +1,130 @@
|
||||||
|
+// Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved.
|
||||||
|
+// SPDX-License-Identifier: Apache-2.0
|
||||||
|
+// Description: IPVS related common functions
|
||||||
|
+// Author: xiadanni
|
||||||
|
+// Create: 2020-08-01
|
||||||
|
+
|
||||||
|
+package main
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "os/exec"
|
||||||
|
+ "strconv"
|
||||||
|
+ "strings"
|
||||||
|
+
|
||||||
|
+ "github.com/kata-containers/agent/protocols/grpc"
|
||||||
|
+ "google.golang.org/grpc/codes"
|
||||||
|
+ grpcStatus "google.golang.org/grpc/status"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+var (
|
||||||
|
+ errNoIPVSRules = grpcStatus.Errorf(codes.InvalidArgument, "IPVS rule is nil")
|
||||||
|
+ errRuleExist = grpcStatus.Errorf(codes.InvalidArgument, "failed to restore IPVS rules: exist some rules in system, should clear first")
|
||||||
|
+ errInvalidIPVSRule = grpcStatus.Errorf(codes.InvalidArgument, "invalid IPVS rule, please check")
|
||||||
|
+ errIPVSRuleExceed = grpcStatus.Errorf(codes.InvalidArgument, "rules exceed limit, should clear first")
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+const (
|
||||||
|
+ validHeadLength = 7
|
||||||
|
+ ruleLimitMax = 20000
|
||||||
|
+ restoreHead = "restore"
|
||||||
|
+ conntrackNormalInfo = "have been deleted"
|
||||||
|
+ waitError = "no child processes"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+func (s *sandbox) updateIPVSRule(ipvsRule *grpc.UpdateIPVSRequest) (resultingIpvs *grpc.IPVSResponse, err error) {
|
||||||
|
+ var (
|
||||||
|
+ restoreRuleCnt string
|
||||||
|
+ restoreRule string
|
||||||
|
+ operation string
|
||||||
|
+ restoreRuleCntUint uint64
|
||||||
|
+ cmd *exec.Cmd
|
||||||
|
+ )
|
||||||
|
+
|
||||||
|
+ if ipvsRule == nil || ipvsRule.IPVSReq == "" {
|
||||||
|
+ return nil, errNoIPVSRules
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ s.ipvsadm.ipvsLock.Lock()
|
||||||
|
+ defer s.ipvsadm.ipvsLock.Unlock()
|
||||||
|
+
|
||||||
|
+ rule := ipvsRule.IPVSReq
|
||||||
|
+ if len(rule) < validHeadLength {
|
||||||
|
+ return nil, errInvalidIPVSRule
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if ruleHead := rule[:validHeadLength]; ruleHead == restoreHead {
|
||||||
|
+ if s.ipvsadm.ipvsRuleCnt != 0 {
|
||||||
|
+ return nil, errRuleExist
|
||||||
|
+ }
|
||||||
|
+ operation = restoreHead
|
||||||
|
+
|
||||||
|
+ restoreItem := strings.Split(rule, "|")
|
||||||
|
+ if len(restoreItem) != 3 {
|
||||||
|
+ return nil, errInvalidIPVSRule
|
||||||
|
+ }
|
||||||
|
+ restoreRuleCnt = restoreItem[1]
|
||||||
|
+ if restoreRuleCntUint, err = strconv.ParseUint(restoreRuleCnt, 10, 64); err != nil {
|
||||||
|
+ return nil, errInvalidIPVSRule
|
||||||
|
+ }
|
||||||
|
+ restoreRule = restoreItem[2]
|
||||||
|
+
|
||||||
|
+ cmd = exec.Command("/sbin/ipvsadm", "--restore")
|
||||||
|
+ cmd.Stdin = strings.NewReader(restoreRule)
|
||||||
|
+ } else {
|
||||||
|
+ item := strings.Fields(strings.TrimSpace(rule))
|
||||||
|
+ if len(item) < 2 {
|
||||||
|
+ return nil, errInvalidIPVSRule
|
||||||
|
+ }
|
||||||
|
+ if item[0] != "ipvsadm" && item[0] != "conntrack" {
|
||||||
|
+ return nil, errInvalidIPVSRule
|
||||||
|
+ }
|
||||||
|
+ operation = item[1]
|
||||||
|
+ if s.ipvsadm.ipvsRuleCnt >= ruleLimitMax {
|
||||||
|
+ if operation == "--add-service" || operation == "-A" || operation == "--add-server" || operation == "-a" {
|
||||||
|
+ return nil, errIPVSRuleExceed
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ restoreRuleCntUint = 0
|
||||||
|
+
|
||||||
|
+ cmd = exec.Command("/sbin/" + item[0])
|
||||||
|
+ for i := 1; i < len(item); i++ {
|
||||||
|
+ cmd.Args = append(cmd.Args, item[i])
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ bytes, err := cmd.CombinedOutput()
|
||||||
|
+
|
||||||
|
+ if err != nil && !strings.Contains(err.Error(), waitError) && !strings.Contains(string(bytes), conntrackNormalInfo) {
|
||||||
|
+ return nil, grpcStatus.Errorf(codes.Internal,
|
||||||
|
+ "exec IPVS command failed, stderr: %v, err: %v", string(bytes), err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ s.sumRuleCount(operation, restoreRuleCntUint)
|
||||||
|
+
|
||||||
|
+ rsp := &grpc.IPVSResponse{
|
||||||
|
+ IPVSRes: string(bytes),
|
||||||
|
+ }
|
||||||
|
+ return rsp, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *sandbox) sumRuleCount(operation string, restoreRuleCntUint uint64) {
|
||||||
|
+ if operation == restoreHead {
|
||||||
|
+ s.ipvsadm.ipvsRuleCnt = restoreRuleCntUint
|
||||||
|
+ return
|
||||||
|
+ }
|
||||||
|
+ if operation == "--add-service" || operation == "-A" || operation == "--add-server" || operation == "-a" {
|
||||||
|
+ s.ipvsadm.ipvsRuleCnt++
|
||||||
|
+ return
|
||||||
|
+ }
|
||||||
|
+ if operation == "--delete-service" || operation == "-D" || operation == "--delete-server" || operation == "-d" {
|
||||||
|
+ if s.ipvsadm.ipvsRuleCnt > 0 {
|
||||||
|
+ s.ipvsadm.ipvsRuleCnt--
|
||||||
|
+ }
|
||||||
|
+ return
|
||||||
|
+ }
|
||||||
|
+ if operation == "--clear" || operation == "-C" {
|
||||||
|
+ s.ipvsadm.ipvsRuleCnt = 0
|
||||||
|
+ return
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
diff --git a/protocols/grpc/agent.pb.go b/protocols/grpc/agent.pb.go
|
||||||
|
index 1b887e5..04d0ee5 100644
|
||||||
|
--- a/protocols/grpc/agent.pb.go
|
||||||
|
+++ b/protocols/grpc/agent.pb.go
|
||||||
|
@@ -63,6 +63,8 @@
|
||||||
|
CopyFileRequest
|
||||||
|
StartTracingRequest
|
||||||
|
StopTracingRequest
|
||||||
|
+ UpdateIPVSRequest
|
||||||
|
+ IPVSResponse
|
||||||
|
CheckRequest
|
||||||
|
HealthCheckResponse
|
||||||
|
VersionCheckResponse
|
||||||
|
@@ -1842,6 +1844,40 @@ func (m *StopTracingRequest) String() string { return proto.CompactTe
|
||||||
|
func (*StopTracingRequest) ProtoMessage() {}
|
||||||
|
func (*StopTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{52} }
|
||||||
|
|
||||||
|
+type UpdateIPVSRequest struct {
|
||||||
|
+ // IPVS_req is the IPVS rule message needed to update
|
||||||
|
+ IPVSReq string `protobuf:"bytes,1,opt,name=IPVS_req,json=IPVSReq,proto3" json:"IPVS_req,omitempty"`
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (m *UpdateIPVSRequest) Reset() { *m = UpdateIPVSRequest{} }
|
||||||
|
+func (m *UpdateIPVSRequest) String() string { return proto.CompactTextString(m) }
|
||||||
|
+func (*UpdateIPVSRequest) ProtoMessage() {}
|
||||||
|
+func (*UpdateIPVSRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{53} }
|
||||||
|
+
|
||||||
|
+func (m *UpdateIPVSRequest) GetIPVSReq() string {
|
||||||
|
+ if m != nil {
|
||||||
|
+ return m.IPVSReq
|
||||||
|
+ }
|
||||||
|
+ return ""
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+type IPVSResponse struct {
|
||||||
|
+ // IPVS_res is the response of IPVS updating
|
||||||
|
+ IPVSRes string `protobuf:"bytes,1,opt,name=IPVS_res,json=IPVSRes,proto3" json:"IPVS_res,omitempty"`
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (m *IPVSResponse) Reset() { *m = IPVSResponse{} }
|
||||||
|
+func (m *IPVSResponse) String() string { return proto.CompactTextString(m) }
|
||||||
|
+func (*IPVSResponse) ProtoMessage() {}
|
||||||
|
+func (*IPVSResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{54} }
|
||||||
|
+
|
||||||
|
+func (m *IPVSResponse) GetIPVSRes() string {
|
||||||
|
+ if m != nil {
|
||||||
|
+ return m.IPVSRes
|
||||||
|
+ }
|
||||||
|
+ return ""
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func init() {
|
||||||
|
proto.RegisterType((*CreateContainerRequest)(nil), "grpc.CreateContainerRequest")
|
||||||
|
proto.RegisterType((*StartContainerRequest)(nil), "grpc.StartContainerRequest")
|
||||||
|
@@ -1896,6 +1932,8 @@ func init() {
|
||||||
|
proto.RegisterType((*CopyFileRequest)(nil), "grpc.CopyFileRequest")
|
||||||
|
proto.RegisterType((*StartTracingRequest)(nil), "grpc.StartTracingRequest")
|
||||||
|
proto.RegisterType((*StopTracingRequest)(nil), "grpc.StopTracingRequest")
|
||||||
|
+ proto.RegisterType((*UpdateIPVSRequest)(nil), "grpc.UpdateIPVSRequest")
|
||||||
|
+ proto.RegisterType((*IPVSResponse)(nil), "grpc.IPVSResponse")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reference imports to suppress errors if they are not otherwise used.
|
||||||
|
@@ -1938,6 +1976,7 @@ type AgentServiceClient interface {
|
||||||
|
UpdateRoutes(ctx context.Context, in *UpdateRoutesRequest, opts ...grpc1.CallOption) (*Routes, error)
|
||||||
|
ListInterfaces(ctx context.Context, in *ListInterfacesRequest, opts ...grpc1.CallOption) (*Interfaces, error)
|
||||||
|
ListRoutes(ctx context.Context, in *ListRoutesRequest, opts ...grpc1.CallOption) (*Routes, error)
|
||||||
|
+ UpdateIPVSRule(ctx context.Context, in *UpdateIPVSRequest, opts ...grpc1.CallOption) (*IPVSResponse, error)
|
||||||
|
// tracing
|
||||||
|
StartTracing(ctx context.Context, in *StartTracingRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error)
|
||||||
|
StopTracing(ctx context.Context, in *StopTracingRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error)
|
||||||
|
@@ -2140,6 +2179,15 @@ func (c *agentServiceClient) ListRoutes(ctx context.Context, in *ListRoutesReque
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (c *agentServiceClient) UpdateIPVSRule(ctx context.Context, in *UpdateIPVSRequest, opts ...grpc1.CallOption) (*IPVSResponse, error) {
|
||||||
|
+ out := new(IPVSResponse)
|
||||||
|
+ err := grpc1.Invoke(ctx, "/grpc.AgentService/UpdateIPVSRule", in, out, c.cc, opts...)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+ return out, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func (c *agentServiceClient) StartTracing(ctx context.Context, in *StartTracingRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error) {
|
||||||
|
out := new(google_protobuf2.Empty)
|
||||||
|
err := grpc1.Invoke(ctx, "/grpc.AgentService/StartTracing", in, out, c.cc, opts...)
|
||||||
|
@@ -2262,6 +2310,7 @@ type AgentServiceServer interface {
|
||||||
|
UpdateRoutes(context.Context, *UpdateRoutesRequest) (*Routes, error)
|
||||||
|
ListInterfaces(context.Context, *ListInterfacesRequest) (*Interfaces, error)
|
||||||
|
ListRoutes(context.Context, *ListRoutesRequest) (*Routes, error)
|
||||||
|
+ UpdateIPVSRule(context.Context, *UpdateIPVSRequest) (*IPVSResponse, error)
|
||||||
|
// tracing
|
||||||
|
StartTracing(context.Context, *StartTracingRequest) (*google_protobuf2.Empty, error)
|
||||||
|
StopTracing(context.Context, *StopTracingRequest) (*google_protobuf2.Empty, error)
|
||||||
|
@@ -2640,6 +2689,24 @@ func _AgentService_ListRoutes_Handler(srv interface{}, ctx context.Context, dec
|
||||||
|
return interceptor(ctx, in, info, handler)
|
||||||
|
}
|
||||||
|
|
||||||
|
+func _AgentService_UpdateIPVSRule_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) {
|
||||||
|
+ in := new(UpdateIPVSRequest)
|
||||||
|
+ if err := dec(in); err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+ if interceptor == nil {
|
||||||
|
+ return srv.(AgentServiceServer).UpdateIPVSRule(ctx, in)
|
||||||
|
+ }
|
||||||
|
+ info := &grpc1.UnaryServerInfo{
|
||||||
|
+ Server: srv,
|
||||||
|
+ FullMethod: "/grpc.AgentService/UpdateIPVSRule",
|
||||||
|
+ }
|
||||||
|
+ handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||||
|
+ return srv.(AgentServiceServer).UpdateIPVSRule(ctx, req.(*UpdateIPVSRequest))
|
||||||
|
+ }
|
||||||
|
+ return interceptor(ctx, in, info, handler)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func _AgentService_StartTracing_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) {
|
||||||
|
in := new(StartTracingRequest)
|
||||||
|
if err := dec(in); err != nil {
|
||||||
|
@@ -2904,6 +2971,10 @@ var _AgentService_serviceDesc = grpc1.ServiceDesc{
|
||||||
|
MethodName: "ListRoutes",
|
||||||
|
Handler: _AgentService_ListRoutes_Handler,
|
||||||
|
},
|
||||||
|
+ {
|
||||||
|
+ MethodName: "UpdateIPVSRule",
|
||||||
|
+ Handler: _AgentService_UpdateIPVSRule_Handler,
|
||||||
|
+ },
|
||||||
|
{
|
||||||
|
MethodName: "StartTracing",
|
||||||
|
Handler: _AgentService_StartTracing_Handler,
|
||||||
|
@@ -5088,6 +5159,54 @@ func (m *StopTracingRequest) MarshalTo(dAtA []byte) (int, error) {
|
||||||
|
return i, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (m *UpdateIPVSRequest) Marshal() (dAtA []byte, err error) {
|
||||||
|
+ size := m.Size()
|
||||||
|
+ dAtA = make([]byte, size)
|
||||||
|
+ n, err := m.MarshalTo(dAtA)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+ return dAtA[:n], nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (m *UpdateIPVSRequest) MarshalTo(dAtA []byte) (int, error) {
|
||||||
|
+ var i int
|
||||||
|
+ _ = i
|
||||||
|
+ var l int
|
||||||
|
+ _ = l
|
||||||
|
+ if len(m.IPVSReq) > 0 {
|
||||||
|
+ dAtA[i] = 0xa
|
||||||
|
+ i++
|
||||||
|
+ i = encodeVarintAgent(dAtA, i, uint64(len(m.IPVSReq)))
|
||||||
|
+ i += copy(dAtA[i:], m.IPVSReq)
|
||||||
|
+ }
|
||||||
|
+ return i, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (m *IPVSResponse) Marshal() (dAtA []byte, err error) {
|
||||||
|
+ size := m.Size()
|
||||||
|
+ dAtA = make([]byte, size)
|
||||||
|
+ n, err := m.MarshalTo(dAtA)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+ return dAtA[:n], nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (m *IPVSResponse) MarshalTo(dAtA []byte) (int, error) {
|
||||||
|
+ var i int
|
||||||
|
+ _ = i
|
||||||
|
+ var l int
|
||||||
|
+ _ = l
|
||||||
|
+ if len(m.IPVSRes) > 0 {
|
||||||
|
+ dAtA[i] = 0xa
|
||||||
|
+ i++
|
||||||
|
+ i = encodeVarintAgent(dAtA, i, uint64(len(m.IPVSRes)))
|
||||||
|
+ i += copy(dAtA[i:], m.IPVSRes)
|
||||||
|
+ }
|
||||||
|
+ return i, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func encodeVarintAgent(dAtA []byte, offset int, v uint64) int {
|
||||||
|
for v >= 1<<7 {
|
||||||
|
dAtA[offset] = uint8(v&0x7f | 0x80)
|
||||||
|
@@ -6019,6 +6138,26 @@ func (m *StopTracingRequest) Size() (n int) {
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (m *UpdateIPVSRequest) Size() (n int) {
|
||||||
|
+ var l int
|
||||||
|
+ _ = l
|
||||||
|
+ l = len(m.IPVSReq)
|
||||||
|
+ if l > 0 {
|
||||||
|
+ n += 1 + l + sovAgent(uint64(l))
|
||||||
|
+ }
|
||||||
|
+ return n
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (m *IPVSResponse) Size() (n int) {
|
||||||
|
+ var l int
|
||||||
|
+ _ = l
|
||||||
|
+ l = len(m.IPVSRes)
|
||||||
|
+ if l > 0 {
|
||||||
|
+ n += 1 + l + sovAgent(uint64(l))
|
||||||
|
+ }
|
||||||
|
+ return n
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func sovAgent(x uint64) (n int) {
|
||||||
|
for {
|
||||||
|
n++
|
||||||
|
@@ -12785,6 +12924,164 @@ func (m *StopTracingRequest) Unmarshal(dAtA []byte) error {
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
+func (m *UpdateIPVSRequest) Unmarshal(dAtA []byte) error {
|
||||||
|
+ l := len(dAtA)
|
||||||
|
+ iNdEx := 0
|
||||||
|
+ for iNdEx < l {
|
||||||
|
+ preIndex := iNdEx
|
||||||
|
+ var wire uint64
|
||||||
|
+ for shift := uint(0); ; shift += 7 {
|
||||||
|
+ if shift >= 64 {
|
||||||
|
+ return ErrIntOverflowAgent
|
||||||
|
+ }
|
||||||
|
+ if iNdEx >= l {
|
||||||
|
+ return io.ErrUnexpectedEOF
|
||||||
|
+ }
|
||||||
|
+ b := dAtA[iNdEx]
|
||||||
|
+ iNdEx++
|
||||||
|
+ wire |= (uint64(b) & 0x7F) << shift
|
||||||
|
+ if b < 0x80 {
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ fieldNum := int32(wire >> 3)
|
||||||
|
+ wireType := int(wire & 0x7)
|
||||||
|
+ if wireType == 4 {
|
||||||
|
+ return fmt.Errorf("proto: UpdateIPVSRequest: wiretype end group for non-group")
|
||||||
|
+ }
|
||||||
|
+ if fieldNum <= 0 {
|
||||||
|
+ return fmt.Errorf("proto: UpdateIPVSRequest: illegal tag %d (wire type %d)", fieldNum, wire)
|
||||||
|
+ }
|
||||||
|
+ switch fieldNum {
|
||||||
|
+ case 1:
|
||||||
|
+ if wireType != 2 {
|
||||||
|
+ return fmt.Errorf("proto: wrong wireType = %d for field IPVSReq", wireType)
|
||||||
|
+ }
|
||||||
|
+ var stringLen uint64
|
||||||
|
+ for shift := uint(0); ; shift += 7 {
|
||||||
|
+ if shift >= 64 {
|
||||||
|
+ return ErrIntOverflowAgent
|
||||||
|
+ }
|
||||||
|
+ if iNdEx >= l {
|
||||||
|
+ return io.ErrUnexpectedEOF
|
||||||
|
+ }
|
||||||
|
+ b := dAtA[iNdEx]
|
||||||
|
+ iNdEx++
|
||||||
|
+ stringLen |= (uint64(b) & 0x7F) << shift
|
||||||
|
+ if b < 0x80 {
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ intStringLen := int(stringLen)
|
||||||
|
+ if intStringLen < 0 {
|
||||||
|
+ return ErrInvalidLengthAgent
|
||||||
|
+ }
|
||||||
|
+ postIndex := iNdEx + intStringLen
|
||||||
|
+ if postIndex > l {
|
||||||
|
+ return io.ErrUnexpectedEOF
|
||||||
|
+ }
|
||||||
|
+ m.IPVSReq = string(dAtA[iNdEx:postIndex])
|
||||||
|
+ iNdEx = postIndex
|
||||||
|
+ default:
|
||||||
|
+ iNdEx = preIndex
|
||||||
|
+ skippy, err := skipAgent(dAtA[iNdEx:])
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ if skippy < 0 {
|
||||||
|
+ return ErrInvalidLengthAgent
|
||||||
|
+ }
|
||||||
|
+ if (iNdEx + skippy) > l {
|
||||||
|
+ return io.ErrUnexpectedEOF
|
||||||
|
+ }
|
||||||
|
+ iNdEx += skippy
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if iNdEx > l {
|
||||||
|
+ return io.ErrUnexpectedEOF
|
||||||
|
+ }
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+func (m *IPVSResponse) Unmarshal(dAtA []byte) error {
|
||||||
|
+ l := len(dAtA)
|
||||||
|
+ iNdEx := 0
|
||||||
|
+ for iNdEx < l {
|
||||||
|
+ preIndex := iNdEx
|
||||||
|
+ var wire uint64
|
||||||
|
+ for shift := uint(0); ; shift += 7 {
|
||||||
|
+ if shift >= 64 {
|
||||||
|
+ return ErrIntOverflowAgent
|
||||||
|
+ }
|
||||||
|
+ if iNdEx >= l {
|
||||||
|
+ return io.ErrUnexpectedEOF
|
||||||
|
+ }
|
||||||
|
+ b := dAtA[iNdEx]
|
||||||
|
+ iNdEx++
|
||||||
|
+ wire |= (uint64(b) & 0x7F) << shift
|
||||||
|
+ if b < 0x80 {
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ fieldNum := int32(wire >> 3)
|
||||||
|
+ wireType := int(wire & 0x7)
|
||||||
|
+ if wireType == 4 {
|
||||||
|
+ return fmt.Errorf("proto: IPVSResponse: wiretype end group for non-group")
|
||||||
|
+ }
|
||||||
|
+ if fieldNum <= 0 {
|
||||||
|
+ return fmt.Errorf("proto: IPVSResponse: illegal tag %d (wire type %d)", fieldNum, wire)
|
||||||
|
+ }
|
||||||
|
+ switch fieldNum {
|
||||||
|
+ case 1:
|
||||||
|
+ if wireType != 2 {
|
||||||
|
+ return fmt.Errorf("proto: wrong wireType = %d for field IPVSRes", wireType)
|
||||||
|
+ }
|
||||||
|
+ var stringLen uint64
|
||||||
|
+ for shift := uint(0); ; shift += 7 {
|
||||||
|
+ if shift >= 64 {
|
||||||
|
+ return ErrIntOverflowAgent
|
||||||
|
+ }
|
||||||
|
+ if iNdEx >= l {
|
||||||
|
+ return io.ErrUnexpectedEOF
|
||||||
|
+ }
|
||||||
|
+ b := dAtA[iNdEx]
|
||||||
|
+ iNdEx++
|
||||||
|
+ stringLen |= (uint64(b) & 0x7F) << shift
|
||||||
|
+ if b < 0x80 {
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ intStringLen := int(stringLen)
|
||||||
|
+ if intStringLen < 0 {
|
||||||
|
+ return ErrInvalidLengthAgent
|
||||||
|
+ }
|
||||||
|
+ postIndex := iNdEx + intStringLen
|
||||||
|
+ if postIndex > l {
|
||||||
|
+ return io.ErrUnexpectedEOF
|
||||||
|
+ }
|
||||||
|
+ m.IPVSRes = string(dAtA[iNdEx:postIndex])
|
||||||
|
+ iNdEx = postIndex
|
||||||
|
+ default:
|
||||||
|
+ iNdEx = preIndex
|
||||||
|
+ skippy, err := skipAgent(dAtA[iNdEx:])
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ if skippy < 0 {
|
||||||
|
+ return ErrInvalidLengthAgent
|
||||||
|
+ }
|
||||||
|
+ if (iNdEx + skippy) > l {
|
||||||
|
+ return io.ErrUnexpectedEOF
|
||||||
|
+ }
|
||||||
|
+ iNdEx += skippy
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if iNdEx > l {
|
||||||
|
+ return io.ErrUnexpectedEOF
|
||||||
|
+ }
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
func skipAgent(dAtA []byte) (n int, err error) {
|
||||||
|
l := len(dAtA)
|
||||||
|
iNdEx := 0
|
||||||
|
@@ -12893,185 +13190,189 @@ var (
|
||||||
|
func init() { proto.RegisterFile("agent.proto", fileDescriptorAgent) }
|
||||||
|
|
||||||
|
var fileDescriptorAgent = []byte{
|
||||||
|
- // 2876 bytes of a gzipped FileDescriptorProto
|
||||||
|
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0x4b, 0x6f, 0x1c, 0xc7,
|
||||||
|
- 0xd1, 0xd8, 0x07, 0x97, 0xbb, 0xb5, 0x2f, 0x6e, 0x93, 0xa2, 0x56, 0x2b, 0x59, 0x9f, 0x3c, 0xb6,
|
||||||
|
- 0x65, 0xfa, 0xf3, 0xe7, 0xa5, 0x2d, 0x1b, 0x9f, 0x5f, 0x70, 0x04, 0xf1, 0x11, 0x91, 0xb1, 0x15,
|
||||||
|
- 0x31, 0x43, 0x11, 0x4e, 0x10, 0x04, 0x83, 0xe1, 0x4c, 0x73, 0xd9, 0xe6, 0xce, 0xf4, 0xb8, 0xa7,
|
||||||
|
- 0x87, 0xe2, 0x3a, 0x40, 0x8e, 0xc9, 0x2d, 0x97, 0x00, 0xb9, 0xe5, 0x0f, 0x04, 0xb9, 0xe5, 0x98,
|
||||||
|
- 0x6b, 0x0e, 0x46, 0x4e, 0xf9, 0x05, 0x41, 0xe0, 0x9f, 0x90, 0x5f, 0x10, 0xf4, 0x6b, 0x1e, 0xbb,
|
||||||
|
- 0x43, 0x1a, 0x21, 0x08, 0xe4, 0xb2, 0xe8, 0xaa, 0xae, 0xae, 0x57, 0x77, 0xd5, 0x54, 0xd5, 0x42,
|
||||||
|
- 0xdb, 0x9d, 0xe0, 0x90, 0x8f, 0x23, 0x46, 0x39, 0x45, 0xf5, 0x09, 0x8b, 0xbc, 0x51, 0x8b, 0x7a,
|
||||||
|
- 0x44, 0x21, 0x46, 0xff, 0x3f, 0x21, 0xfc, 0x34, 0x39, 0x1e, 0x7b, 0x34, 0xd8, 0x3c, 0x73, 0xb9,
|
||||||
|
- 0xfb, 0x8e, 0x47, 0x43, 0xee, 0x92, 0x10, 0xb3, 0x78, 0x53, 0x1e, 0xdc, 0x8c, 0xce, 0x26, 0x9b,
|
||||||
|
- 0x7c, 0x16, 0xe1, 0x58, 0xfd, 0xea, 0x73, 0x77, 0x27, 0x94, 0x4e, 0xa6, 0x78, 0x53, 0x42, 0xc7,
|
||||||
|
- 0xc9, 0xc9, 0x26, 0x0e, 0x22, 0x3e, 0x53, 0x9b, 0xd6, 0x1f, 0xaa, 0xb0, 0xbe, 0xcd, 0xb0, 0xcb,
|
||||||
|
- 0xf1, 0xb6, 0xe1, 0x66, 0xe3, 0xaf, 0x13, 0x1c, 0x73, 0xf4, 0x2a, 0x74, 0x52, 0x09, 0x0e, 0xf1,
|
||||||
|
- 0x87, 0x95, 0x07, 0x95, 0x8d, 0x96, 0xdd, 0x4e, 0x71, 0xfb, 0x3e, 0xba, 0x0d, 0xcb, 0xf8, 0x02,
|
||||||
|
- 0x7b, 0x62, 0xb7, 0x2a, 0x77, 0x1b, 0x02, 0xdc, 0xf7, 0xd1, 0x7b, 0xd0, 0x8e, 0x39, 0x23, 0xe1,
|
||||||
|
- 0xc4, 0x49, 0x62, 0xcc, 0x86, 0xb5, 0x07, 0x95, 0x8d, 0xf6, 0xa3, 0x95, 0xb1, 0x30, 0x69, 0x7c,
|
||||||
|
- 0x28, 0x37, 0x8e, 0x62, 0xcc, 0x6c, 0x88, 0xd3, 0x35, 0x7a, 0x08, 0xcb, 0x3e, 0x3e, 0x27, 0x1e,
|
||||||
|
- 0x8e, 0x87, 0xf5, 0x07, 0xb5, 0x8d, 0xf6, 0xa3, 0x8e, 0x22, 0xdf, 0x91, 0x48, 0xdb, 0x6c, 0xa2,
|
||||||
|
- 0xb7, 0xa0, 0x19, 0x73, 0xca, 0xdc, 0x09, 0x8e, 0x87, 0x4b, 0x92, 0xb0, 0x6b, 0xf8, 0x4a, 0xac,
|
||||||
|
- 0x9d, 0x6e, 0xa3, 0x7b, 0x50, 0x7b, 0xbe, 0xbd, 0x3f, 0x6c, 0x48, 0xe9, 0xa0, 0xa9, 0x22, 0xec,
|
||||||
|
- 0xd9, 0x02, 0x8d, 0x5e, 0x83, 0x6e, 0xec, 0x86, 0xfe, 0x31, 0xbd, 0x70, 0x22, 0xe2, 0x87, 0xf1,
|
||||||
|
- 0x70, 0xf9, 0x41, 0x65, 0xa3, 0x69, 0x77, 0x34, 0xf2, 0x40, 0xe0, 0xac, 0x4f, 0xe0, 0xd6, 0x21,
|
||||||
|
- 0x77, 0x19, 0xbf, 0x86, 0x77, 0xac, 0x23, 0x58, 0xb7, 0x71, 0x40, 0xcf, 0xaf, 0xe5, 0xda, 0x21,
|
||||||
|
- 0x2c, 0x73, 0x12, 0x60, 0x9a, 0x70, 0xe9, 0xda, 0xae, 0x6d, 0x40, 0xeb, 0x4f, 0x15, 0x40, 0xbb,
|
||||||
|
- 0x17, 0xd8, 0x3b, 0x60, 0xd4, 0xc3, 0x71, 0xfc, 0x5f, 0xba, 0xae, 0x37, 0x61, 0x39, 0x52, 0x0a,
|
||||||
|
- 0x0c, 0xeb, 0x92, 0x5c, 0xdf, 0x82, 0xd1, 0xca, 0xec, 0x5a, 0x5f, 0xc1, 0xda, 0x21, 0x99, 0x84,
|
||||||
|
- 0xee, 0xf4, 0x06, 0xf5, 0x5d, 0x87, 0x46, 0x2c, 0x79, 0x4a, 0x55, 0xbb, 0xb6, 0x86, 0xac, 0x03,
|
||||||
|
- 0x40, 0x5f, 0xba, 0x84, 0xdf, 0x9c, 0x24, 0xeb, 0x1d, 0x58, 0x2d, 0x70, 0x8c, 0x23, 0x1a, 0xc6,
|
||||||
|
- 0x58, 0x2a, 0xc0, 0x5d, 0x9e, 0xc4, 0x92, 0xd9, 0x92, 0xad, 0x21, 0x0b, 0xc3, 0xda, 0x17, 0x24,
|
||||||
|
- 0x36, 0xe4, 0xf8, 0x3f, 0x51, 0x61, 0x1d, 0x1a, 0x27, 0x94, 0x05, 0x2e, 0x37, 0x1a, 0x28, 0x08,
|
||||||
|
- 0x21, 0xa8, 0xbb, 0x6c, 0x12, 0x0f, 0x6b, 0x0f, 0x6a, 0x1b, 0x2d, 0x5b, 0xae, 0xc5, 0xab, 0x9c,
|
||||||
|
- 0x13, 0xa3, 0xf5, 0x7a, 0x15, 0x3a, 0xda, 0xef, 0xce, 0x94, 0xc4, 0x5c, 0xca, 0xe9, 0xd8, 0x6d,
|
||||||
|
- 0x8d, 0x13, 0x67, 0x2c, 0x0a, 0xeb, 0x47, 0x91, 0x7f, 0xcd, 0x80, 0x7f, 0x04, 0x2d, 0x86, 0x63,
|
||||||
|
- 0x9a, 0x30, 0x11, 0xa6, 0x55, 0x79, 0xef, 0x6b, 0xea, 0xde, 0xbf, 0x20, 0x61, 0x72, 0x61, 0x9b,
|
||||||
|
- 0x3d, 0x3b, 0x23, 0xd3, 0x21, 0xc4, 0xe3, 0xeb, 0x84, 0xd0, 0x27, 0x70, 0xeb, 0xc0, 0x4d, 0xe2,
|
||||||
|
- 0xeb, 0xe8, 0x6a, 0x7d, 0x2a, 0xc2, 0x2f, 0x4e, 0x82, 0x6b, 0x1d, 0xfe, 0x63, 0x05, 0x9a, 0xdb,
|
||||||
|
- 0x51, 0x72, 0x14, 0xbb, 0x13, 0x8c, 0xfe, 0x07, 0xda, 0x9c, 0x72, 0x77, 0xea, 0x24, 0x02, 0x94,
|
||||||
|
- 0xe4, 0x75, 0x1b, 0x24, 0x4a, 0x11, 0x08, 0xb7, 0x63, 0xe6, 0x45, 0x89, 0xa6, 0xa8, 0x3e, 0xa8,
|
||||||
|
- 0x6d, 0xd4, 0xed, 0xb6, 0xc2, 0x29, 0x92, 0x31, 0xac, 0xca, 0x3d, 0x87, 0x84, 0xce, 0x19, 0x66,
|
||||||
|
- 0x21, 0x9e, 0x06, 0xd4, 0xc7, 0xf2, 0xfd, 0xd6, 0xed, 0x81, 0xdc, 0xda, 0x0f, 0x3f, 0x4f, 0x37,
|
||||||
|
- 0xd0, 0xff, 0xc2, 0x20, 0xa5, 0x17, 0x41, 0x29, 0xa9, 0xeb, 0x92, 0xba, 0xaf, 0xa9, 0x8f, 0x34,
|
||||||
|
- 0xda, 0xfa, 0x15, 0xf4, 0x5e, 0x9c, 0x32, 0xca, 0xf9, 0x94, 0x84, 0x93, 0x1d, 0x97, 0xbb, 0x22,
|
||||||
|
- 0x7b, 0x44, 0x98, 0x11, 0xea, 0xc7, 0x5a, 0x5b, 0x03, 0xa2, 0xb7, 0x61, 0xc0, 0x15, 0x2d, 0xf6,
|
||||||
|
- 0x1d, 0x43, 0x53, 0x95, 0x34, 0x2b, 0xe9, 0xc6, 0x81, 0x26, 0x7e, 0x03, 0x7a, 0x19, 0xb1, 0xc8,
|
||||||
|
- 0x3f, 0x5a, 0xdf, 0x6e, 0x8a, 0x7d, 0x41, 0x02, 0x6c, 0x9d, 0x4b, 0x5f, 0xc9, 0x4b, 0x46, 0x6f,
|
||||||
|
- 0x43, 0x2b, 0xf3, 0x43, 0x45, 0xbe, 0x90, 0x9e, 0x7a, 0x21, 0xc6, 0x9d, 0x76, 0x33, 0x75, 0xca,
|
||||||
|
- 0x67, 0xd0, 0xe7, 0xa9, 0xe2, 0x8e, 0xef, 0x72, 0xb7, 0xf8, 0xa8, 0x8a, 0x56, 0xd9, 0x3d, 0x5e,
|
||||||
|
- 0x80, 0xad, 0x4f, 0xa1, 0x75, 0x40, 0xfc, 0x58, 0x09, 0x1e, 0xc2, 0xb2, 0x97, 0x30, 0x86, 0x43,
|
||||||
|
- 0x6e, 0x4c, 0xd6, 0x20, 0x5a, 0x83, 0xa5, 0x29, 0x09, 0x08, 0xd7, 0x66, 0x2a, 0xc0, 0xa2, 0x00,
|
||||||
|
- 0xcf, 0x70, 0x40, 0xd9, 0x4c, 0x3a, 0x6c, 0x0d, 0x96, 0xf2, 0x97, 0xab, 0x00, 0x74, 0x17, 0x5a,
|
||||||
|
- 0x81, 0x7b, 0x91, 0x5e, 0xaa, 0xd8, 0x69, 0x06, 0xee, 0x85, 0x52, 0x7e, 0x08, 0xcb, 0x27, 0x2e,
|
||||||
|
- 0x99, 0x7a, 0x21, 0xd7, 0x5e, 0x31, 0x60, 0x26, 0xb0, 0x9e, 0x17, 0xf8, 0xd7, 0x2a, 0xb4, 0x95,
|
||||||
|
- 0x44, 0xa5, 0xf0, 0x1a, 0x2c, 0x79, 0xae, 0x77, 0x9a, 0x8a, 0x94, 0x00, 0x7a, 0x68, 0x14, 0xa9,
|
||||||
|
- 0xe6, 0x93, 0x70, 0xa6, 0xa9, 0x51, 0x6d, 0x13, 0x20, 0x7e, 0xe9, 0x46, 0x5a, 0xb7, 0xda, 0x25,
|
||||||
|
- 0xc4, 0x2d, 0x41, 0xa3, 0xd4, 0x7d, 0x1f, 0x3a, 0xea, 0xdd, 0xe9, 0x23, 0xf5, 0x4b, 0x8e, 0xb4,
|
||||||
|
- 0x15, 0x95, 0x3a, 0xf4, 0x1a, 0x74, 0x93, 0x18, 0x3b, 0xa7, 0x04, 0x33, 0x97, 0x79, 0xa7, 0xb3,
|
||||||
|
- 0xe1, 0x92, 0xfa, 0x46, 0x26, 0x31, 0xde, 0x33, 0x38, 0xf4, 0x08, 0x96, 0x44, 0xfa, 0x8b, 0x87,
|
||||||
|
- 0x0d, 0xf9, 0x39, 0xbe, 0x97, 0x67, 0x29, 0x4d, 0x1d, 0xcb, 0xdf, 0xdd, 0x90, 0xb3, 0x99, 0xad,
|
||||||
|
- 0x48, 0x47, 0x1f, 0x01, 0x64, 0x48, 0xb4, 0x02, 0xb5, 0x33, 0x3c, 0xd3, 0x71, 0x28, 0x96, 0xc2,
|
||||||
|
- 0x39, 0xe7, 0xee, 0x34, 0x31, 0x5e, 0x57, 0xc0, 0x27, 0xd5, 0x8f, 0x2a, 0x96, 0x07, 0xfd, 0xad,
|
||||||
|
- 0xe9, 0x19, 0xa1, 0xb9, 0xe3, 0x6b, 0xb0, 0x14, 0xb8, 0x5f, 0x51, 0x66, 0x3c, 0x29, 0x01, 0x89,
|
||||||
|
- 0x25, 0x21, 0x65, 0x86, 0x85, 0x04, 0x50, 0x0f, 0xaa, 0x34, 0x92, 0xfe, 0x6a, 0xd9, 0x55, 0x1a,
|
||||||
|
- 0x65, 0x82, 0xea, 0x39, 0x41, 0xd6, 0x3f, 0xea, 0x00, 0x99, 0x14, 0x64, 0xc3, 0x88, 0x50, 0x27,
|
||||||
|
- 0xc6, 0x4c, 0x94, 0x20, 0xce, 0xf1, 0x8c, 0xe3, 0xd8, 0x61, 0xd8, 0x4b, 0x58, 0x4c, 0xce, 0xc5,
|
||||||
|
- 0xfd, 0x09, 0xb3, 0x6f, 0x29, 0xb3, 0xe7, 0x74, 0xb3, 0x6f, 0x13, 0x7a, 0xa8, 0xce, 0x6d, 0x89,
|
||||||
|
- 0x63, 0xb6, 0x39, 0x85, 0xf6, 0xe1, 0x56, 0xc6, 0xd3, 0xcf, 0xb1, 0xab, 0x5e, 0xc5, 0x6e, 0x35,
|
||||||
|
- 0x65, 0xe7, 0x67, 0xac, 0x76, 0x61, 0x95, 0x50, 0xe7, 0xeb, 0x04, 0x27, 0x05, 0x46, 0xb5, 0xab,
|
||||||
|
- 0x18, 0x0d, 0x08, 0xfd, 0x89, 0x3c, 0x90, 0xb1, 0x39, 0x80, 0x3b, 0x39, 0x2b, 0x45, 0xb8, 0xe7,
|
||||||
|
- 0x98, 0xd5, 0xaf, 0x62, 0xb6, 0x9e, 0x6a, 0x25, 0xf2, 0x41, 0xc6, 0xf1, 0x47, 0xb0, 0x4e, 0xa8,
|
||||||
|
- 0xf3, 0xd2, 0x25, 0x7c, 0x9e, 0xdd, 0xd2, 0xf7, 0x18, 0x29, 0x3e, 0xba, 0x45, 0x5e, 0xca, 0xc8,
|
||||||
|
- 0x00, 0xb3, 0x49, 0xc1, 0xc8, 0xc6, 0xf7, 0x18, 0xf9, 0x4c, 0x1e, 0xc8, 0xd8, 0x3c, 0x81, 0x01,
|
||||||
|
- 0xa1, 0xf3, 0xda, 0x2c, 0x5f, 0xc5, 0xa4, 0x4f, 0x68, 0x51, 0x93, 0x2d, 0x18, 0xc4, 0xd8, 0xe3,
|
||||||
|
- 0x94, 0xe5, 0x1f, 0x41, 0xf3, 0x2a, 0x16, 0x2b, 0x9a, 0x3e, 0xe5, 0x61, 0xfd, 0x1c, 0x3a, 0x7b,
|
||||||
|
- 0xc9, 0x04, 0xf3, 0xe9, 0x71, 0x9a, 0x0c, 0x6e, 0x2c, 0xff, 0x58, 0xff, 0xaa, 0x42, 0x7b, 0x7b,
|
||||||
|
- 0xc2, 0x68, 0x12, 0x15, 0x72, 0xb2, 0x0a, 0xd2, 0xf9, 0x9c, 0x2c, 0x49, 0x64, 0x4e, 0x56, 0xc4,
|
||||||
|
- 0x1f, 0x40, 0x27, 0x90, 0xa1, 0xab, 0xe9, 0x55, 0x1e, 0x1a, 0x2c, 0x04, 0xb5, 0xdd, 0x0e, 0x72,
|
||||||
|
- 0xc9, 0x6c, 0x0c, 0x10, 0x11, 0x3f, 0xd6, 0x67, 0x54, 0x3a, 0xea, 0xeb, 0x8a, 0xd0, 0xa4, 0x68,
|
||||||
|
- 0xbb, 0x15, 0xa5, 0xd9, 0xfa, 0x3d, 0x68, 0x1f, 0x0b, 0x27, 0xe9, 0x03, 0x85, 0x64, 0x94, 0x79,
|
||||||
|
- 0xcf, 0x86, 0xe3, 0x2c, 0x08, 0xf7, 0xa0, 0x7b, 0xaa, 0x5c, 0xa6, 0x0f, 0xa9, 0x37, 0xf4, 0x9a,
|
||||||
|
- 0xb6, 0x24, 0xb3, 0x77, 0x9c, 0xf7, 0xac, 0xba, 0x80, 0xce, 0x69, 0x0e, 0x35, 0x3a, 0x84, 0xc1,
|
||||||
|
- 0x02, 0x49, 0x49, 0x0e, 0xda, 0xc8, 0xe7, 0xa0, 0xf6, 0x23, 0xa4, 0x04, 0xe5, 0x4f, 0xe6, 0xf3,
|
||||||
|
- 0xd2, 0x6f, 0xab, 0xd0, 0xf9, 0x31, 0xe6, 0x2f, 0x29, 0x3b, 0x53, 0xfa, 0x22, 0xa8, 0x87, 0x6e,
|
||||||
|
- 0x80, 0x35, 0x47, 0xb9, 0x46, 0x77, 0xa0, 0xc9, 0x2e, 0x54, 0x02, 0xd1, 0xf7, 0xb9, 0xcc, 0x2e,
|
||||||
|
- 0x64, 0x62, 0x40, 0xaf, 0x00, 0xb0, 0x0b, 0x27, 0x72, 0xbd, 0x33, 0xac, 0x3d, 0x58, 0xb7, 0x5b,
|
||||||
|
- 0xec, 0xe2, 0x40, 0x21, 0xc4, 0x53, 0x60, 0x17, 0x0e, 0x66, 0x8c, 0xb2, 0x58, 0xe7, 0xaa, 0x26,
|
||||||
|
- 0xbb, 0xd8, 0x95, 0xb0, 0x3e, 0xeb, 0x33, 0x1a, 0x45, 0xd8, 0x97, 0x39, 0x5a, 0x9e, 0xdd, 0x51,
|
||||||
|
- 0x08, 0x21, 0x95, 0x1b, 0xa9, 0x0d, 0x25, 0x95, 0x67, 0x52, 0x79, 0x26, 0x75, 0x59, 0x9d, 0xe4,
|
||||||
|
- 0x79, 0xa9, 0x3c, 0x95, 0xda, 0x54, 0x52, 0x79, 0x4e, 0x2a, 0xcf, 0xa4, 0xb6, 0xcc, 0x59, 0x2d,
|
||||||
|
- 0xd5, 0xfa, 0x4d, 0x05, 0xd6, 0xe7, 0x0b, 0x3f, 0x5d, 0xa6, 0x7e, 0x00, 0x1d, 0x4f, 0xde, 0x57,
|
||||||
|
- 0xe1, 0x4d, 0x0e, 0x16, 0x6e, 0xd2, 0x6e, 0x7b, 0xb9, 0x67, 0xfc, 0x21, 0x74, 0x43, 0xe5, 0xe0,
|
||||||
|
- 0xf4, 0x69, 0xd6, 0xb2, 0x7b, 0xc9, 0xfb, 0xde, 0xee, 0x84, 0x39, 0xc8, 0xf2, 0x01, 0x7d, 0xc9,
|
||||||
|
- 0x08, 0xc7, 0x87, 0x9c, 0x61, 0x37, 0xb8, 0x89, 0x06, 0x04, 0x41, 0x5d, 0x56, 0x2b, 0x35, 0x59,
|
||||||
|
- 0x5f, 0xcb, 0xb5, 0xf5, 0x26, 0xac, 0x16, 0xa4, 0x68, 0x5b, 0x57, 0xa0, 0x36, 0xc5, 0xa1, 0xe4,
|
||||||
|
- 0xde, 0xb5, 0xc5, 0xd2, 0x72, 0x61, 0x60, 0x63, 0xd7, 0xbf, 0x39, 0x6d, 0xb4, 0x88, 0x5a, 0x26,
|
||||||
|
- 0x62, 0x03, 0x50, 0x5e, 0x84, 0x56, 0xc5, 0x68, 0x5d, 0xc9, 0x69, 0xfd, 0x1c, 0x06, 0xdb, 0x53,
|
||||||
|
- 0x1a, 0xe3, 0x43, 0xee, 0x93, 0xf0, 0x26, 0x3a, 0xa6, 0x5f, 0xc2, 0xea, 0x0b, 0x3e, 0xfb, 0x52,
|
||||||
|
- 0x30, 0x8b, 0xc9, 0x37, 0xf8, 0x86, 0xec, 0x63, 0xf4, 0xa5, 0xb1, 0x8f, 0xd1, 0x97, 0xa2, 0x59,
|
||||||
|
- 0xf2, 0xe8, 0x34, 0x09, 0x42, 0x19, 0x0a, 0x5d, 0x5b, 0x43, 0xd6, 0x16, 0x74, 0x54, 0x0d, 0xfd,
|
||||||
|
- 0x8c, 0xfa, 0xc9, 0x14, 0x97, 0xc6, 0xe0, 0x7d, 0x80, 0xc8, 0x65, 0x6e, 0x80, 0x39, 0x66, 0xea,
|
||||||
|
- 0x0d, 0xb5, 0xec, 0x1c, 0xc6, 0xfa, 0x7d, 0x15, 0xd6, 0xd4, 0x48, 0xe4, 0x50, 0x4d, 0x02, 0x8c,
|
||||||
|
- 0x09, 0x23, 0x68, 0x9e, 0xd2, 0x98, 0xe7, 0x18, 0xa6, 0xb0, 0x50, 0xd1, 0x0f, 0x0d, 0x37, 0xb1,
|
||||||
|
- 0x2c, 0xcc, 0x29, 0x6a, 0x57, 0xcf, 0x29, 0x16, 0x26, 0x11, 0xf5, 0xc5, 0x49, 0x84, 0x88, 0x36,
|
||||||
|
- 0x43, 0x44, 0x54, 0x8c, 0xb7, 0xec, 0x96, 0xc6, 0xec, 0xfb, 0xe8, 0x21, 0xf4, 0x27, 0x42, 0x4b,
|
||||||
|
- 0xe7, 0x94, 0xd2, 0x33, 0x27, 0x72, 0xf9, 0xa9, 0x0c, 0xf5, 0x96, 0xdd, 0x95, 0xe8, 0x3d, 0x4a,
|
||||||
|
- 0xcf, 0x0e, 0x5c, 0x7e, 0x8a, 0x3e, 0x86, 0x9e, 0x2e, 0x03, 0x03, 0xe9, 0xa2, 0x58, 0x7f, 0xfc,
|
||||||
|
- 0x74, 0x14, 0xe5, 0xbd, 0x67, 0x77, 0xcf, 0x72, 0x50, 0x6c, 0xdd, 0x86, 0x5b, 0x3b, 0x38, 0xe6,
|
||||||
|
- 0x8c, 0xce, 0x8a, 0x8e, 0xb1, 0x7e, 0x00, 0xb0, 0x1f, 0x72, 0xcc, 0x4e, 0x5c, 0x0f, 0xc7, 0xe8,
|
||||||
|
- 0xdd, 0x3c, 0xa4, 0x8b, 0xa3, 0x95, 0xb1, 0x9a, 0x48, 0xa5, 0x1b, 0x76, 0x8e, 0xc6, 0x1a, 0x43,
|
||||||
|
- 0xc3, 0xa6, 0x89, 0x48, 0x47, 0xaf, 0x9b, 0x95, 0x3e, 0xd7, 0xd1, 0xe7, 0x24, 0xd2, 0xd6, 0x7b,
|
||||||
|
- 0xd6, 0x9e, 0x69, 0x61, 0x33, 0x76, 0xfa, 0x8a, 0xc6, 0xd0, 0x22, 0x06, 0xa7, 0xb3, 0xca, 0xa2,
|
||||||
|
- 0xe8, 0x8c, 0xc4, 0xfa, 0x19, 0xac, 0x2a, 0x4e, 0x8a, 0xb3, 0x61, 0xf3, 0x3a, 0x34, 0x98, 0x51,
|
||||||
|
- 0xa3, 0x92, 0x8d, 0xa2, 0x34, 0x91, 0xde, 0x43, 0xf7, 0x84, 0x30, 0x8f, 0xe1, 0x40, 0xf4, 0x1c,
|
||||||
|
- 0x55, 0x79, 0x65, 0x19, 0x42, 0x78, 0x4b, 0xf4, 0xdb, 0x99, 0x99, 0xc6, 0x5b, 0xab, 0x30, 0x10,
|
||||||
|
- 0x1b, 0x05, 0x89, 0xd6, 0x2f, 0x60, 0xf5, 0x79, 0x38, 0x25, 0x21, 0xde, 0x3e, 0x38, 0x7a, 0x86,
|
||||||
|
- 0xd3, 0xac, 0x80, 0xa0, 0x2e, 0xaa, 0x27, 0xa9, 0x46, 0xd3, 0x96, 0x6b, 0x11, 0x26, 0xe1, 0xb1,
|
||||||
|
- 0xe3, 0x45, 0x49, 0xac, 0x27, 0x43, 0x8d, 0xf0, 0x78, 0x3b, 0x4a, 0x62, 0x91, 0xe6, 0xc5, 0x67,
|
||||||
|
- 0x9e, 0x86, 0xd3, 0x99, 0x8c, 0x95, 0xa6, 0xbd, 0xec, 0x45, 0xc9, 0xf3, 0x70, 0x3a, 0xb3, 0xfe,
|
||||||
|
- 0x4f, 0xf6, 0xc2, 0x18, 0xfb, 0xb6, 0x1b, 0xfa, 0x34, 0xd8, 0xc1, 0xe7, 0x39, 0x09, 0x69, 0xdf,
|
||||||
|
- 0x65, 0x72, 0xc2, 0xb7, 0x15, 0xe8, 0x3c, 0x99, 0xe0, 0x90, 0xef, 0x60, 0xee, 0x92, 0xa9, 0xec,
|
||||||
|
- 0xad, 0xce, 0x31, 0x8b, 0x09, 0x0d, 0xf5, 0xc3, 0x37, 0xa0, 0x68, 0x8d, 0x49, 0x48, 0xb8, 0xe3,
|
||||||
|
- 0xbb, 0x38, 0xa0, 0xa1, 0xf6, 0x02, 0x08, 0xd4, 0x8e, 0xc4, 0xa0, 0x37, 0xa1, 0xaf, 0x26, 0x77,
|
||||||
|
- 0xce, 0xa9, 0x1b, 0xfa, 0x53, 0x11, 0x72, 0x6a, 0x92, 0xd1, 0x53, 0xe8, 0x3d, 0x8d, 0x45, 0x6f,
|
||||||
|
- 0xc1, 0x8a, 0x0e, 0x88, 0x8c, 0xb2, 0x2e, 0x29, 0xfb, 0x1a, 0x5f, 0x20, 0x4d, 0xa2, 0x88, 0x32,
|
||||||
|
- 0x1e, 0x3b, 0x31, 0xf6, 0x3c, 0x1a, 0x44, 0xba, 0x31, 0xe9, 0x1b, 0xfc, 0xa1, 0x42, 0x5b, 0x13,
|
||||||
|
- 0x58, 0x7d, 0x2a, 0xec, 0xd4, 0x96, 0x64, 0x17, 0xdc, 0x0b, 0x70, 0xe0, 0x1c, 0x4f, 0xa9, 0x77,
|
||||||
|
- 0xe6, 0x88, 0x34, 0xa5, 0x3d, 0x2c, 0x4a, 0x9f, 0x2d, 0x81, 0x3c, 0x24, 0xdf, 0xc8, 0x1e, 0x5c,
|
||||||
|
- 0x50, 0x9d, 0x52, 0x1e, 0x4d, 0x93, 0x89, 0x13, 0x31, 0x7a, 0x8c, 0xb5, 0x89, 0xfd, 0x00, 0x07,
|
||||||
|
- 0x7b, 0x0a, 0x7f, 0x20, 0xd0, 0xd6, 0x5f, 0x2a, 0xb0, 0x56, 0x94, 0xa4, 0x93, 0xee, 0x26, 0xac,
|
||||||
|
- 0x15, 0x45, 0xe9, 0x0f, 0xb1, 0x2a, 0xf4, 0x06, 0x79, 0x81, 0xea, 0x93, 0xfc, 0x21, 0x74, 0xe5,
|
||||||
|
- 0x38, 0xd7, 0xf1, 0x15, 0xa7, 0x62, 0xf9, 0x91, 0xbf, 0x17, 0xbb, 0xe3, 0xe6, 0x6f, 0xe9, 0x63,
|
||||||
|
- 0xb8, 0xa3, 0xcd, 0x77, 0x16, 0xd5, 0x56, 0x0f, 0x62, 0x5d, 0x13, 0x3c, 0x9b, 0xd3, 0xfe, 0x0b,
|
||||||
|
- 0x18, 0x66, 0xa8, 0xad, 0x99, 0x44, 0x1a, 0x5f, 0xbd, 0x0b, 0xab, 0x73, 0xc6, 0x3e, 0xf1, 0x7d,
|
||||||
|
- 0x26, 0x03, 0xb4, 0x6e, 0x97, 0x6d, 0x59, 0x8f, 0xe1, 0xf6, 0x21, 0xe6, 0xca, 0x1b, 0x2e, 0xd7,
|
||||||
|
- 0x3d, 0x81, 0x62, 0xb6, 0x02, 0xb5, 0x43, 0xec, 0x49, 0xe3, 0x6b, 0xb6, 0x58, 0x8a, 0x07, 0x78,
|
||||||
|
- 0x14, 0x63, 0x4f, 0x5a, 0x59, 0xb3, 0xe5, 0xda, 0xfa, 0x73, 0x05, 0x96, 0x75, 0x9a, 0x14, 0xa9,
|
||||||
|
- 0xde, 0x67, 0xe4, 0x1c, 0x33, 0xfd, 0xf4, 0x34, 0x84, 0xde, 0x80, 0x9e, 0x5a, 0x39, 0x34, 0xe2,
|
||||||
|
- 0x84, 0xa6, 0xc9, 0xb7, 0xab, 0xb0, 0xcf, 0x15, 0x52, 0x4e, 0xea, 0xe4, 0x20, 0x4a, 0xf7, 0x7c,
|
||||||
|
- 0x1a, 0x92, 0xe3, 0xb6, 0x58, 0x64, 0x06, 0x99, 0x6c, 0x5b, 0xb6, 0x86, 0xc4, 0x53, 0x37, 0xfc,
|
||||||
|
- 0x96, 0x24, 0x3f, 0x03, 0x8a, 0xa7, 0x1e, 0xd0, 0x24, 0xe4, 0x4e, 0x44, 0x49, 0xc8, 0x75, 0x76,
|
||||||
|
- 0x05, 0x89, 0x3a, 0x10, 0x18, 0xeb, 0xd7, 0x15, 0x68, 0xa8, 0x69, 0xb5, 0xe8, 0x32, 0xd3, 0x6f,
|
||||||
|
- 0x5c, 0x95, 0xc8, 0x7a, 0x41, 0xca, 0x52, 0xdf, 0x35, 0xb9, 0x16, 0x71, 0x7c, 0x1e, 0xa8, 0x4c,
|
||||||
|
- 0xad, 0x55, 0x3b, 0x0f, 0x64, 0x8a, 0x7e, 0x03, 0x7a, 0xd9, 0xa7, 0x52, 0xee, 0x2b, 0x15, 0xbb,
|
||||||
|
- 0x29, 0x56, 0x92, 0x5d, 0xaa, 0xa9, 0xf5, 0x53, 0xd1, 0x5c, 0xa7, 0x93, 0xda, 0x15, 0xa8, 0x25,
|
||||||
|
- 0xa9, 0x32, 0x62, 0x29, 0x30, 0x93, 0xf4, 0x23, 0x2b, 0x96, 0xe8, 0x21, 0xf4, 0x5c, 0xdf, 0x27,
|
||||||
|
- 0xe2, 0xb8, 0x3b, 0x7d, 0x4a, 0xfc, 0x34, 0x48, 0x8b, 0x58, 0xeb, 0x6f, 0x15, 0xe8, 0x6f, 0xd3,
|
||||||
|
- 0x68, 0xf6, 0x43, 0x32, 0xc5, 0xb9, 0x0c, 0x22, 0x95, 0xd4, 0xdf, 0x58, 0xb1, 0x16, 0x75, 0xe3,
|
||||||
|
- 0x09, 0x99, 0x62, 0x15, 0x5a, 0xea, 0x66, 0x9b, 0x02, 0x21, 0xc3, 0xca, 0x6c, 0xa6, 0x03, 0xb0,
|
||||||
|
- 0xae, 0xda, 0x7c, 0x46, 0x7d, 0x59, 0x21, 0xfb, 0x84, 0x39, 0xe9, 0xb8, 0xab, 0x6b, 0x2f, 0xfb,
|
||||||
|
- 0x84, 0xc9, 0x2d, 0x6d, 0xc8, 0x92, 0x9c, 0xb8, 0xe6, 0x0d, 0x69, 0x28, 0x8c, 0x30, 0x64, 0x1d,
|
||||||
|
- 0x1a, 0xf4, 0xe4, 0x24, 0xc6, 0x5c, 0xd6, 0xb2, 0x35, 0x5b, 0x43, 0x69, 0x9a, 0x6b, 0xe6, 0xd2,
|
||||||
|
- 0xdc, 0x2d, 0x58, 0x95, 0xb3, 0xfd, 0x17, 0xcc, 0xf5, 0x48, 0x38, 0x31, 0xa9, 0x78, 0x0d, 0xd0,
|
||||||
|
- 0x21, 0xa7, 0x51, 0x11, 0xfb, 0xe8, 0x77, 0x2b, 0x3a, 0x27, 0xea, 0x46, 0x17, 0x3d, 0x85, 0xfe,
|
||||||
|
- 0xdc, 0x1f, 0x27, 0x48, 0x4f, 0x3e, 0xca, 0xff, 0x4f, 0x19, 0xad, 0x8f, 0xd5, 0x1f, 0x31, 0x63,
|
||||||
|
- 0xf3, 0x47, 0xcc, 0x78, 0x37, 0x88, 0xf8, 0x0c, 0xed, 0x42, 0xaf, 0xf8, 0x17, 0x03, 0xba, 0x6b,
|
||||||
|
- 0x0a, 0x85, 0x92, 0x3f, 0x1e, 0x2e, 0x65, 0xf3, 0x14, 0xfa, 0x73, 0xff, 0x36, 0x18, 0x7d, 0xca,
|
||||||
|
- 0xff, 0x84, 0xb8, 0x94, 0xd1, 0x63, 0x68, 0xe7, 0xfe, 0x5e, 0x40, 0x43, 0xc5, 0x64, 0xf1, 0x1f,
|
||||||
|
- 0x87, 0x4b, 0x19, 0x6c, 0x43, 0xb7, 0x30, 0xf1, 0x47, 0x23, 0x6d, 0x4f, 0xc9, 0xdf, 0x00, 0x97,
|
||||||
|
- 0x32, 0xd9, 0x82, 0x76, 0x6e, 0xf0, 0x6e, 0xb4, 0x58, 0x9c, 0xee, 0x8f, 0xee, 0x94, 0xec, 0xe8,
|
||||||
|
- 0xd4, 0xbb, 0x07, 0xdd, 0xc2, 0x98, 0xdc, 0x28, 0x52, 0x36, 0xa2, 0x1f, 0xdd, 0x2d, 0xdd, 0xd3,
|
||||||
|
- 0x9c, 0x9e, 0x42, 0x7f, 0x6e, 0x68, 0x6e, 0x9c, 0x5b, 0x3e, 0x4b, 0xbf, 0xd4, 0xac, 0xcf, 0xe5,
|
||||||
|
- 0x65, 0xe7, 0x7a, 0xa2, 0xdc, 0x65, 0x2f, 0x8e, 0xc8, 0x47, 0xf7, 0xca, 0x37, 0xb5, 0x56, 0xbb,
|
||||||
|
- 0xd0, 0x2b, 0x4e, 0xc7, 0x0d, 0xb3, 0xd2, 0x99, 0xf9, 0xd5, 0x2f, 0xa7, 0x30, 0x28, 0xcf, 0x5e,
|
||||||
|
- 0x4e, 0xd9, 0xfc, 0xfc, 0x52, 0x46, 0x4f, 0x00, 0x74, 0x07, 0xe4, 0x93, 0x30, 0xbd, 0xb2, 0x85,
|
||||||
|
- 0xce, 0x2b, 0xbd, 0xb2, 0x92, 0x6e, 0xe9, 0x31, 0x80, 0x6a, 0x5c, 0x7c, 0x9a, 0x70, 0x74, 0xdb,
|
||||||
|
- 0xa8, 0x31, 0xd7, 0x2d, 0x8d, 0x86, 0x8b, 0x1b, 0x0b, 0x0c, 0x30, 0x63, 0xd7, 0x61, 0xf0, 0x19,
|
||||||
|
- 0x40, 0xd6, 0x10, 0x19, 0x06, 0x0b, 0x2d, 0xd2, 0x15, 0x3e, 0xe8, 0xe4, 0xdb, 0x1f, 0xa4, 0x6d,
|
||||||
|
- 0x2d, 0x69, 0x89, 0xae, 0x60, 0xd1, 0x9f, 0x2b, 0x6f, 0x8b, 0x8f, 0x6d, 0xbe, 0xea, 0x1d, 0x2d,
|
||||||
|
- 0x94, 0xb8, 0xe8, 0x43, 0xe8, 0xe4, 0xeb, 0x5a, 0xa3, 0x45, 0x49, 0xad, 0x3b, 0x2a, 0xd4, 0xb6,
|
||||||
|
- 0xe8, 0x31, 0xf4, 0x8a, 0x55, 0x2b, 0xca, 0xc5, 0xc5, 0x42, 0x2d, 0x3b, 0xd2, 0x13, 0x9b, 0x1c,
|
||||||
|
- 0xf9, 0xfb, 0x00, 0x59, 0x75, 0x6b, 0xdc, 0xb7, 0x50, 0xef, 0xce, 0x49, 0x7d, 0x02, 0x9d, 0x7c,
|
||||||
|
- 0x26, 0x36, 0xea, 0x96, 0x64, 0xe7, 0xab, 0xb2, 0x56, 0x2e, 0x6b, 0x9b, 0xc7, 0xb7, 0x98, 0xc8,
|
||||||
|
- 0xaf, 0xca, 0x5a, 0x85, 0xae, 0xcf, 0x24, 0x8b, 0xb2, 0x56, 0xf0, 0xaa, 0x5c, 0x5e, 0x6c, 0x91,
|
||||||
|
- 0x8c, 0xfb, 0x4a, 0x1b, 0xa7, 0xab, 0x1e, 0x51, 0xbe, 0x1b, 0x30, 0xfe, 0x28, 0xe9, 0x10, 0xbe,
|
||||||
|
- 0x27, 0xa8, 0xf3, 0x15, 0x7f, 0x2e, 0xa8, 0x4b, 0x1a, 0x81, 0x4b, 0x19, 0xed, 0x41, 0xff, 0xa9,
|
||||||
|
- 0x29, 0xe6, 0x74, 0xa1, 0xa9, 0xd5, 0x29, 0x29, 0xac, 0x47, 0xa3, 0xb2, 0x2d, 0x1d, 0x59, 0x9f,
|
||||||
|
- 0xc3, 0x60, 0xa1, 0xc8, 0x44, 0xf7, 0xd3, 0xc1, 0x62, 0x69, 0xf5, 0x79, 0xa9, 0x5a, 0xfb, 0xb0,
|
||||||
|
- 0x32, 0x5f, 0x63, 0xa2, 0x57, 0xf4, 0xa5, 0x97, 0xd7, 0x9e, 0x97, 0xb2, 0xfa, 0x18, 0x9a, 0xa6,
|
||||||
|
- 0xa6, 0x41, 0x7a, 0x80, 0x3b, 0x57, 0xe3, 0x5c, 0x76, 0x74, 0xab, 0xf3, 0xed, 0x77, 0xf7, 0x2b,
|
||||||
|
- 0x7f, 0xff, 0xee, 0x7e, 0xe5, 0x9f, 0xdf, 0xdd, 0xaf, 0x1c, 0x37, 0xe4, 0xee, 0xfb, 0xff, 0x0e,
|
||||||
|
- 0x00, 0x00, 0xff, 0xff, 0x8d, 0x89, 0xaa, 0x73, 0xc8, 0x21, 0x00, 0x00,
|
||||||
|
+ // 2931 bytes of a gzipped FileDescriptorProto
|
||||||
|
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0xcb, 0x6e, 0x1c, 0xc7,
|
||||||
|
+ 0xb5, 0x98, 0x07, 0x87, 0x33, 0x67, 0x5e, 0x9c, 0x22, 0x45, 0x8d, 0x46, 0xb2, 0xae, 0xdc, 0xb6,
|
||||||
|
+ 0x65, 0xfa, 0xfa, 0x7a, 0x68, 0xcb, 0xc6, 0xf5, 0x0b, 0xbe, 0x82, 0x48, 0xe9, 0x8a, 0x8c, 0xad,
|
||||||
|
+ 0x88, 0xe9, 0x91, 0xe2, 0x04, 0x41, 0xd0, 0x68, 0x76, 0x97, 0x86, 0x65, 0x4e, 0x77, 0xb5, 0xab,
|
||||||
|
+ 0xaa, 0x29, 0x8e, 0x03, 0x64, 0x99, 0xec, 0xb2, 0xcc, 0x2e, 0x3f, 0x10, 0x64, 0x97, 0x65, 0xb6,
|
||||||
|
+ 0x59, 0x18, 0x59, 0x05, 0xf9, 0x80, 0x20, 0xf0, 0x27, 0xe4, 0x0b, 0x82, 0x7a, 0xf5, 0x63, 0x66,
|
||||||
|
+ 0x48, 0x23, 0x82, 0x80, 0x6c, 0x1a, 0x75, 0x4e, 0x9d, 0x3a, 0xaf, 0xaa, 0x3a, 0x75, 0xce, 0x69,
|
||||||
|
+ 0x68, 0xfb, 0x53, 0x1c, 0x8b, 0x71, 0xc2, 0xa8, 0xa0, 0xa8, 0x3e, 0x65, 0x49, 0x30, 0x6a, 0xd1,
|
||||||
|
+ 0x80, 0x68, 0xc4, 0xe8, 0x7f, 0xa7, 0x44, 0x9c, 0xa4, 0xc7, 0xe3, 0x80, 0x46, 0xbb, 0xa7, 0xbe,
|
||||||
|
+ 0xf0, 0xdf, 0x09, 0x68, 0x2c, 0x7c, 0x12, 0x63, 0xc6, 0x77, 0xd5, 0xc2, 0xdd, 0xe4, 0x74, 0xba,
|
||||||
|
+ 0x2b, 0xe6, 0x09, 0xe6, 0xfa, 0x6b, 0xd6, 0x5d, 0x9f, 0x52, 0x3a, 0x9d, 0xe1, 0x5d, 0x05, 0x1d,
|
||||||
|
+ 0xa7, 0xcf, 0x76, 0x71, 0x94, 0x88, 0xb9, 0x9e, 0x74, 0x7e, 0x57, 0x85, 0xed, 0x7d, 0x86, 0x7d,
|
||||||
|
+ 0x81, 0xf7, 0x2d, 0x37, 0x17, 0x7f, 0x9d, 0x62, 0x2e, 0xd0, 0xab, 0xd0, 0xc9, 0x24, 0x78, 0x24,
|
||||||
|
+ 0x1c, 0x56, 0x6e, 0x55, 0x76, 0x5a, 0x6e, 0x3b, 0xc3, 0x1d, 0x86, 0xe8, 0x2a, 0xac, 0xe3, 0x73,
|
||||||
|
+ 0x1c, 0xc8, 0xd9, 0xaa, 0x9a, 0x6d, 0x48, 0xf0, 0x30, 0x44, 0xef, 0x41, 0x9b, 0x0b, 0x46, 0xe2,
|
||||||
|
+ 0xa9, 0x97, 0x72, 0xcc, 0x86, 0xb5, 0x5b, 0x95, 0x9d, 0xf6, 0x9d, 0x8d, 0xb1, 0x34, 0x69, 0x3c,
|
||||||
|
+ 0x51, 0x13, 0x4f, 0x39, 0x66, 0x2e, 0xf0, 0x6c, 0x8c, 0x6e, 0xc3, 0x7a, 0x88, 0xcf, 0x48, 0x80,
|
||||||
|
+ 0xf9, 0xb0, 0x7e, 0xab, 0xb6, 0xd3, 0xbe, 0xd3, 0xd1, 0xe4, 0xf7, 0x15, 0xd2, 0xb5, 0x93, 0xe8,
|
||||||
|
+ 0x2d, 0x68, 0x72, 0x41, 0x99, 0x3f, 0xc5, 0x7c, 0xb8, 0xa6, 0x08, 0xbb, 0x96, 0xaf, 0xc2, 0xba,
|
||||||
|
+ 0xd9, 0x34, 0xba, 0x01, 0xb5, 0xc7, 0xfb, 0x87, 0xc3, 0x86, 0x92, 0x0e, 0x86, 0x2a, 0xc1, 0x81,
|
||||||
|
+ 0x2b, 0xd1, 0xe8, 0x35, 0xe8, 0x72, 0x3f, 0x0e, 0x8f, 0xe9, 0xb9, 0x97, 0x90, 0x30, 0xe6, 0xc3,
|
||||||
|
+ 0xf5, 0x5b, 0x95, 0x9d, 0xa6, 0xdb, 0x31, 0xc8, 0x23, 0x89, 0x73, 0x3e, 0x81, 0x2b, 0x13, 0xe1,
|
||||||
|
+ 0x33, 0xf1, 0x02, 0xde, 0x71, 0x9e, 0xc2, 0xb6, 0x8b, 0x23, 0x7a, 0xf6, 0x42, 0xae, 0x1d, 0xc2,
|
||||||
|
+ 0xba, 0x20, 0x11, 0xa6, 0xa9, 0x50, 0xae, 0xed, 0xba, 0x16, 0x74, 0xfe, 0x50, 0x01, 0xf4, 0xe0,
|
||||||
|
+ 0x1c, 0x07, 0x47, 0x8c, 0x06, 0x98, 0xf3, 0xff, 0xd0, 0x76, 0xbd, 0x09, 0xeb, 0x89, 0x56, 0x60,
|
||||||
|
+ 0x58, 0x57, 0xe4, 0x66, 0x17, 0xac, 0x56, 0x76, 0xd6, 0xf9, 0x0a, 0xb6, 0x26, 0x64, 0x1a, 0xfb,
|
||||||
|
+ 0xb3, 0x97, 0xa8, 0xef, 0x36, 0x34, 0xb8, 0xe2, 0xa9, 0x54, 0xed, 0xba, 0x06, 0x72, 0x8e, 0x00,
|
||||||
|
+ 0x7d, 0xe9, 0x13, 0xf1, 0xf2, 0x24, 0x39, 0xef, 0xc0, 0x66, 0x89, 0x23, 0x4f, 0x68, 0xcc, 0xb1,
|
||||||
|
+ 0x52, 0x40, 0xf8, 0x22, 0xe5, 0x8a, 0xd9, 0x9a, 0x6b, 0x20, 0x07, 0xc3, 0xd6, 0x17, 0x84, 0x5b,
|
||||||
|
+ 0x72, 0xfc, 0xef, 0xa8, 0xb0, 0x0d, 0x8d, 0x67, 0x94, 0x45, 0xbe, 0xb0, 0x1a, 0x68, 0x08, 0x21,
|
||||||
|
+ 0xa8, 0xfb, 0x6c, 0xca, 0x87, 0xb5, 0x5b, 0xb5, 0x9d, 0x96, 0xab, 0xc6, 0xf2, 0x54, 0x2e, 0x88,
|
||||||
|
+ 0x31, 0x7a, 0xbd, 0x0a, 0x1d, 0xe3, 0x77, 0x6f, 0x46, 0xb8, 0x50, 0x72, 0x3a, 0x6e, 0xdb, 0xe0,
|
||||||
|
+ 0xe4, 0x1a, 0x87, 0xc2, 0xf6, 0xd3, 0x24, 0x7c, 0xc1, 0x0b, 0x7f, 0x07, 0x5a, 0x0c, 0x73, 0x9a,
|
||||||
|
+ 0x32, 0x79, 0x4d, 0xab, 0x6a, 0xdf, 0xb7, 0xf4, 0xbe, 0x7f, 0x41, 0xe2, 0xf4, 0xdc, 0xb5, 0x73,
|
||||||
|
+ 0x6e, 0x4e, 0x66, 0xae, 0x90, 0xe0, 0x2f, 0x72, 0x85, 0x3e, 0x81, 0x2b, 0x47, 0x7e, 0xca, 0x5f,
|
||||||
|
+ 0x44, 0x57, 0xe7, 0x53, 0x79, 0xfd, 0x78, 0x1a, 0xbd, 0xd0, 0xe2, 0xdf, 0x57, 0xa0, 0xb9, 0x9f,
|
||||||
|
+ 0xa4, 0x4f, 0xb9, 0x3f, 0xc5, 0xe8, 0xbf, 0xa0, 0x2d, 0xa8, 0xf0, 0x67, 0x5e, 0x2a, 0x41, 0x45,
|
||||||
|
+ 0x5e, 0x77, 0x41, 0xa1, 0x34, 0x81, 0x74, 0x3b, 0x66, 0x41, 0x92, 0x1a, 0x8a, 0xea, 0xad, 0xda,
|
||||||
|
+ 0x4e, 0xdd, 0x6d, 0x6b, 0x9c, 0x26, 0x19, 0xc3, 0xa6, 0x9a, 0xf3, 0x48, 0xec, 0x9d, 0x62, 0x16,
|
||||||
|
+ 0xe3, 0x59, 0x44, 0x43, 0xac, 0xce, 0x6f, 0xdd, 0x1d, 0xa8, 0xa9, 0xc3, 0xf8, 0xf3, 0x6c, 0x02,
|
||||||
|
+ 0xfd, 0x37, 0x0c, 0x32, 0x7a, 0x79, 0x29, 0x15, 0x75, 0x5d, 0x51, 0xf7, 0x0d, 0xf5, 0x53, 0x83,
|
||||||
|
+ 0x76, 0x7e, 0x09, 0xbd, 0x27, 0x27, 0x8c, 0x0a, 0x31, 0x23, 0xf1, 0xf4, 0xbe, 0x2f, 0x7c, 0x19,
|
||||||
|
+ 0x3d, 0x12, 0xcc, 0x08, 0x0d, 0xb9, 0xd1, 0xd6, 0x82, 0xe8, 0x6d, 0x18, 0x08, 0x4d, 0x8b, 0x43,
|
||||||
|
+ 0xcf, 0xd2, 0x54, 0x15, 0xcd, 0x46, 0x36, 0x71, 0x64, 0x88, 0xdf, 0x80, 0x5e, 0x4e, 0x2c, 0xe3,
|
||||||
|
+ 0x8f, 0xd1, 0xb7, 0x9b, 0x61, 0x9f, 0x90, 0x08, 0x3b, 0x67, 0xca, 0x57, 0x6a, 0x93, 0xd1, 0xdb,
|
||||||
|
+ 0xd0, 0xca, 0xfd, 0x50, 0x51, 0x27, 0xa4, 0xa7, 0x4f, 0x88, 0x75, 0xa7, 0xdb, 0xcc, 0x9c, 0xf2,
|
||||||
|
+ 0x19, 0xf4, 0x45, 0xa6, 0xb8, 0x17, 0xfa, 0xc2, 0x2f, 0x1f, 0xaa, 0xb2, 0x55, 0x6e, 0x4f, 0x94,
|
||||||
|
+ 0x60, 0xe7, 0x53, 0x68, 0x1d, 0x91, 0x90, 0x6b, 0xc1, 0x43, 0x58, 0x0f, 0x52, 0xc6, 0x70, 0x2c,
|
||||||
|
+ 0xac, 0xc9, 0x06, 0x44, 0x5b, 0xb0, 0x36, 0x23, 0x11, 0x11, 0xc6, 0x4c, 0x0d, 0x38, 0x14, 0xe0,
|
||||||
|
+ 0x11, 0x8e, 0x28, 0x9b, 0x2b, 0x87, 0x6d, 0xc1, 0x5a, 0x71, 0x73, 0x35, 0x80, 0xae, 0x43, 0x2b,
|
||||||
|
+ 0xf2, 0xcf, 0xb3, 0x4d, 0x95, 0x33, 0xcd, 0xc8, 0x3f, 0xd7, 0xca, 0x0f, 0x61, 0xfd, 0x99, 0x4f,
|
||||||
|
+ 0x66, 0x41, 0x2c, 0x8c, 0x57, 0x2c, 0x98, 0x0b, 0xac, 0x17, 0x05, 0xfe, 0xb9, 0x0a, 0x6d, 0x2d,
|
||||||
|
+ 0x51, 0x2b, 0xbc, 0x05, 0x6b, 0x81, 0x1f, 0x9c, 0x64, 0x22, 0x15, 0x80, 0x6e, 0x5b, 0x45, 0xaa,
|
||||||
|
+ 0xc5, 0x20, 0x9c, 0x6b, 0x6a, 0x55, 0xdb, 0x05, 0xe0, 0xcf, 0xfd, 0xc4, 0xe8, 0x56, 0xbb, 0x80,
|
||||||
|
+ 0xb8, 0x25, 0x69, 0xb4, 0xba, 0xef, 0x43, 0x47, 0x9f, 0x3b, 0xb3, 0xa4, 0x7e, 0xc1, 0x92, 0xb6,
|
||||||
|
+ 0xa6, 0xd2, 0x8b, 0x5e, 0x83, 0x6e, 0xca, 0xb1, 0x77, 0x42, 0x30, 0xf3, 0x59, 0x70, 0x32, 0x1f,
|
||||||
|
+ 0xae, 0xe9, 0x37, 0x32, 0xe5, 0xf8, 0xc0, 0xe2, 0xd0, 0x1d, 0x58, 0x93, 0xe1, 0x8f, 0x0f, 0x1b,
|
||||||
|
+ 0xea, 0x39, 0xbe, 0x51, 0x64, 0xa9, 0x4c, 0x1d, 0xab, 0xef, 0x83, 0x58, 0xb0, 0xb9, 0xab, 0x49,
|
||||||
|
+ 0x47, 0x1f, 0x01, 0xe4, 0x48, 0xb4, 0x01, 0xb5, 0x53, 0x3c, 0x37, 0xf7, 0x50, 0x0e, 0xa5, 0x73,
|
||||||
|
+ 0xce, 0xfc, 0x59, 0x6a, 0xbd, 0xae, 0x81, 0x4f, 0xaa, 0x1f, 0x55, 0x9c, 0x00, 0xfa, 0x7b, 0xb3,
|
||||||
|
+ 0x53, 0x42, 0x0b, 0xcb, 0xb7, 0x60, 0x2d, 0xf2, 0xbf, 0xa2, 0xcc, 0x7a, 0x52, 0x01, 0x0a, 0x4b,
|
||||||
|
+ 0x62, 0xca, 0x2c, 0x0b, 0x05, 0xa0, 0x1e, 0x54, 0x69, 0xa2, 0xfc, 0xd5, 0x72, 0xab, 0x34, 0xc9,
|
||||||
|
+ 0x05, 0xd5, 0x0b, 0x82, 0x9c, 0xbf, 0xd7, 0x01, 0x72, 0x29, 0xc8, 0x85, 0x11, 0xa1, 0x1e, 0xc7,
|
||||||
|
+ 0x4c, 0xa6, 0x20, 0xde, 0xf1, 0x5c, 0x60, 0xee, 0x31, 0x1c, 0xa4, 0x8c, 0x93, 0x33, 0xb9, 0x7f,
|
||||||
|
+ 0xd2, 0xec, 0x2b, 0xda, 0xec, 0x05, 0xdd, 0xdc, 0xab, 0x84, 0x4e, 0xf4, 0xba, 0x3d, 0xb9, 0xcc,
|
||||||
|
+ 0xb5, 0xab, 0xd0, 0x21, 0x5c, 0xc9, 0x79, 0x86, 0x05, 0x76, 0xd5, 0xcb, 0xd8, 0x6d, 0x66, 0xec,
|
||||||
|
+ 0xc2, 0x9c, 0xd5, 0x03, 0xd8, 0x24, 0xd4, 0xfb, 0x3a, 0xc5, 0x69, 0x89, 0x51, 0xed, 0x32, 0x46,
|
||||||
|
+ 0x03, 0x42, 0x7f, 0xa4, 0x16, 0xe4, 0x6c, 0x8e, 0xe0, 0x5a, 0xc1, 0x4a, 0x79, 0xdd, 0x0b, 0xcc,
|
||||||
|
+ 0xea, 0x97, 0x31, 0xdb, 0xce, 0xb4, 0x92, 0xf1, 0x20, 0xe7, 0xf8, 0x03, 0xd8, 0x26, 0xd4, 0x7b,
|
||||||
|
+ 0xee, 0x13, 0xb1, 0xc8, 0x6e, 0xed, 0x7b, 0x8c, 0x94, 0x8f, 0x6e, 0x99, 0x97, 0x36, 0x32, 0xc2,
|
||||||
|
+ 0x6c, 0x5a, 0x32, 0xb2, 0xf1, 0x3d, 0x46, 0x3e, 0x52, 0x0b, 0x72, 0x36, 0xf7, 0x60, 0x40, 0xe8,
|
||||||
|
+ 0xa2, 0x36, 0xeb, 0x97, 0x31, 0xe9, 0x13, 0x5a, 0xd6, 0x64, 0x0f, 0x06, 0x1c, 0x07, 0x82, 0xb2,
|
||||||
|
+ 0xe2, 0x21, 0x68, 0x5e, 0xc6, 0x62, 0xc3, 0xd0, 0x67, 0x3c, 0x9c, 0x9f, 0x41, 0xe7, 0x20, 0x9d,
|
||||||
|
+ 0x62, 0x31, 0x3b, 0xce, 0x82, 0xc1, 0x4b, 0x8b, 0x3f, 0xce, 0x3f, 0xab, 0xd0, 0xde, 0x9f, 0x32,
|
||||||
|
+ 0x9a, 0x26, 0xa5, 0x98, 0xac, 0x2f, 0xe9, 0x62, 0x4c, 0x56, 0x24, 0x2a, 0x26, 0x6b, 0xe2, 0x0f,
|
||||||
|
+ 0xa0, 0x13, 0xa9, 0xab, 0x6b, 0xe8, 0x75, 0x1c, 0x1a, 0x2c, 0x5d, 0x6a, 0xb7, 0x1d, 0x15, 0x82,
|
||||||
|
+ 0xd9, 0x18, 0x20, 0x21, 0x21, 0x37, 0x6b, 0x74, 0x38, 0xea, 0x9b, 0x8c, 0xd0, 0x86, 0x68, 0xb7,
|
||||||
|
+ 0x95, 0x64, 0xd1, 0xfa, 0x3d, 0x68, 0x1f, 0x4b, 0x27, 0x99, 0x05, 0xa5, 0x60, 0x94, 0x7b, 0xcf,
|
||||||
|
+ 0x85, 0xe3, 0xfc, 0x12, 0x1e, 0x40, 0xf7, 0x44, 0xbb, 0xcc, 0x2c, 0xd2, 0x67, 0xe8, 0x35, 0x63,
|
||||||
|
+ 0x49, 0x6e, 0xef, 0xb8, 0xe8, 0x59, 0xbd, 0x01, 0x9d, 0x93, 0x02, 0x6a, 0x34, 0x81, 0xc1, 0x12,
|
||||||
|
+ 0xc9, 0x8a, 0x18, 0xb4, 0x53, 0x8c, 0x41, 0xed, 0x3b, 0x48, 0x0b, 0x2a, 0xae, 0x2c, 0xc6, 0xa5,
|
||||||
|
+ 0xdf, 0x54, 0xa1, 0xf3, 0x43, 0x2c, 0x9e, 0x53, 0x76, 0xaa, 0xf5, 0x45, 0x50, 0x8f, 0xfd, 0x08,
|
||||||
|
+ 0x1b, 0x8e, 0x6a, 0x8c, 0xae, 0x41, 0x93, 0x9d, 0xeb, 0x00, 0x62, 0xf6, 0x73, 0x9d, 0x9d, 0xab,
|
||||||
|
+ 0xc0, 0x80, 0x5e, 0x01, 0x60, 0xe7, 0x5e, 0xe2, 0x07, 0xa7, 0xd8, 0x78, 0xb0, 0xee, 0xb6, 0xd8,
|
||||||
|
+ 0xf9, 0x91, 0x46, 0xc8, 0xa3, 0xc0, 0xce, 0x3d, 0xcc, 0x18, 0x65, 0xdc, 0xc4, 0xaa, 0x26, 0x3b,
|
||||||
|
+ 0x7f, 0xa0, 0x60, 0xb3, 0x36, 0x64, 0x34, 0x49, 0x70, 0xa8, 0x62, 0xb4, 0x5a, 0x7b, 0x5f, 0x23,
|
||||||
|
+ 0xa4, 0x54, 0x61, 0xa5, 0x36, 0xb4, 0x54, 0x91, 0x4b, 0x15, 0xb9, 0xd4, 0x75, 0xbd, 0x52, 0x14,
|
||||||
|
+ 0xa5, 0x8a, 0x4c, 0x6a, 0x53, 0x4b, 0x15, 0x05, 0xa9, 0x22, 0x97, 0xda, 0xb2, 0x6b, 0x8d, 0x54,
|
||||||
|
+ 0xe7, 0xd7, 0x15, 0xd8, 0x5e, 0x4c, 0xfc, 0x4c, 0x9a, 0xfa, 0x01, 0x74, 0x02, 0xb5, 0x5f, 0xa5,
|
||||||
|
+ 0x33, 0x39, 0x58, 0xda, 0x49, 0xb7, 0x1d, 0x14, 0x8e, 0xf1, 0x87, 0xd0, 0x8d, 0xb5, 0x83, 0xb3,
|
||||||
|
+ 0xa3, 0x59, 0xcb, 0xf7, 0xa5, 0xe8, 0x7b, 0xb7, 0x13, 0x17, 0x20, 0x27, 0x04, 0xf4, 0x25, 0x23,
|
||||||
|
+ 0x02, 0x4f, 0x04, 0xc3, 0x7e, 0xf4, 0x32, 0x0a, 0x10, 0x04, 0x75, 0x95, 0xad, 0xd4, 0x54, 0x7e,
|
||||||
|
+ 0xad, 0xc6, 0xce, 0x9b, 0xb0, 0x59, 0x92, 0x62, 0x6c, 0xdd, 0x80, 0xda, 0x0c, 0xc7, 0x8a, 0x7b,
|
||||||
|
+ 0xd7, 0x95, 0x43, 0xc7, 0x87, 0x81, 0x8b, 0xfd, 0xf0, 0xe5, 0x69, 0x63, 0x44, 0xd4, 0x72, 0x11,
|
||||||
|
+ 0x3b, 0x80, 0x8a, 0x22, 0x8c, 0x2a, 0x56, 0xeb, 0x4a, 0x41, 0xeb, 0xc7, 0x30, 0xd8, 0x9f, 0x51,
|
||||||
|
+ 0x8e, 0x27, 0x22, 0x24, 0xf1, 0xcb, 0xa8, 0x98, 0x7e, 0x01, 0x9b, 0x4f, 0xc4, 0xfc, 0x4b, 0xc9,
|
||||||
|
+ 0x8c, 0x93, 0x6f, 0xf0, 0x4b, 0xb2, 0x8f, 0xd1, 0xe7, 0xd6, 0x3e, 0x46, 0x9f, 0xcb, 0x62, 0x29,
|
||||||
|
+ 0xa0, 0xb3, 0x34, 0x8a, 0xd5, 0x55, 0xe8, 0xba, 0x06, 0x72, 0xf6, 0xa0, 0xa3, 0x73, 0xe8, 0x47,
|
||||||
|
+ 0x34, 0x4c, 0x67, 0x78, 0xe5, 0x1d, 0xbc, 0x09, 0x90, 0xf8, 0xcc, 0x8f, 0xb0, 0xc0, 0x4c, 0x9f,
|
||||||
|
+ 0xa1, 0x96, 0x5b, 0xc0, 0x38, 0xbf, 0xad, 0xc2, 0x96, 0x6e, 0x89, 0x4c, 0x74, 0x27, 0xc0, 0x9a,
|
||||||
|
+ 0x30, 0x82, 0xe6, 0x09, 0xe5, 0xa2, 0xc0, 0x30, 0x83, 0xa5, 0x8a, 0x61, 0x6c, 0xb9, 0xc9, 0x61,
|
||||||
|
+ 0xa9, 0x4f, 0x51, 0xbb, 0xbc, 0x4f, 0xb1, 0xd4, 0x89, 0xa8, 0x2f, 0x77, 0x22, 0xe4, 0x6d, 0xb3,
|
||||||
|
+ 0x44, 0x44, 0xdf, 0xf1, 0x96, 0xdb, 0x32, 0x98, 0xc3, 0x10, 0xdd, 0x86, 0xfe, 0x54, 0x6a, 0xe9,
|
||||||
|
+ 0x9d, 0x50, 0x7a, 0xea, 0x25, 0xbe, 0x38, 0x51, 0x57, 0xbd, 0xe5, 0x76, 0x15, 0xfa, 0x80, 0xd2,
|
||||||
|
+ 0xd3, 0x23, 0x5f, 0x9c, 0xa0, 0x8f, 0xa1, 0x67, 0xd2, 0xc0, 0x48, 0xb9, 0x88, 0x9b, 0xc7, 0xcf,
|
||||||
|
+ 0xdc, 0xa2, 0xa2, 0xf7, 0xdc, 0xee, 0x69, 0x01, 0xe2, 0xce, 0x55, 0xb8, 0x72, 0x1f, 0x73, 0xc1,
|
||||||
|
+ 0xe8, 0xbc, 0xec, 0x18, 0xe7, 0xff, 0x00, 0x0e, 0x63, 0x81, 0xd9, 0x33, 0x3f, 0xc0, 0x1c, 0xbd,
|
||||||
|
+ 0x5b, 0x84, 0x4c, 0x72, 0xb4, 0x31, 0xd6, 0x1d, 0xa9, 0x6c, 0xc2, 0x2d, 0xd0, 0x38, 0x63, 0x68,
|
||||||
|
+ 0xb8, 0x34, 0x95, 0xe1, 0xe8, 0x75, 0x3b, 0x32, 0xeb, 0x3a, 0x66, 0x9d, 0x42, 0xba, 0x66, 0xce,
|
||||||
|
+ 0x39, 0xb0, 0x25, 0x6c, 0xce, 0xce, 0x6c, 0xd1, 0x18, 0x5a, 0xc4, 0xe2, 0x4c, 0x54, 0x59, 0x16,
|
||||||
|
+ 0x9d, 0x93, 0x38, 0x3f, 0x85, 0x4d, 0xcd, 0x49, 0x73, 0xb6, 0x6c, 0x5e, 0x87, 0x06, 0xb3, 0x6a,
|
||||||
|
+ 0x54, 0xf2, 0x56, 0x94, 0x21, 0x32, 0x73, 0xe8, 0x86, 0x14, 0x16, 0x30, 0x1c, 0xc9, 0x9a, 0xa3,
|
||||||
|
+ 0xaa, 0xb6, 0x2c, 0x47, 0x48, 0x6f, 0xc9, 0x7a, 0x3b, 0x37, 0xd3, 0x7a, 0x6b, 0x13, 0x06, 0x72,
|
||||||
|
+ 0xa2, 0x24, 0xd1, 0xf9, 0x39, 0x6c, 0x3e, 0x8e, 0x67, 0x24, 0xc6, 0xfb, 0x47, 0x4f, 0x1f, 0xe1,
|
||||||
|
+ 0x2c, 0x2a, 0x20, 0xa8, 0xcb, 0xec, 0x49, 0xa9, 0xd1, 0x74, 0xd5, 0x58, 0x5e, 0x93, 0xf8, 0xd8,
|
||||||
|
+ 0x0b, 0x92, 0x94, 0x9b, 0xce, 0x50, 0x23, 0x3e, 0xde, 0x4f, 0x52, 0x2e, 0xc3, 0xbc, 0x7c, 0xe6,
|
||||||
|
+ 0x69, 0x3c, 0x9b, 0xab, 0xbb, 0xd2, 0x74, 0xd7, 0x83, 0x24, 0x7d, 0x1c, 0xcf, 0xe6, 0xce, 0xff,
|
||||||
|
+ 0xa8, 0x5a, 0x18, 0xe3, 0xd0, 0xf5, 0xe3, 0x90, 0x46, 0xf7, 0xf1, 0x59, 0x41, 0x42, 0x56, 0x77,
|
||||||
|
+ 0xd9, 0x98, 0xf0, 0x6d, 0x05, 0x3a, 0xf7, 0xa6, 0x38, 0x16, 0xf7, 0xb1, 0xf0, 0xc9, 0x4c, 0xd5,
|
||||||
|
+ 0x56, 0x67, 0x98, 0x71, 0x42, 0x63, 0x73, 0xf0, 0x2d, 0x28, 0x4b, 0x63, 0x12, 0x13, 0xe1, 0x85,
|
||||||
|
+ 0x3e, 0x8e, 0x68, 0x6c, 0xbc, 0x00, 0x12, 0x75, 0x5f, 0x61, 0xd0, 0x9b, 0xd0, 0xd7, 0x9d, 0x3b,
|
||||||
|
+ 0xef, 0xc4, 0x8f, 0xc3, 0x99, 0xbc, 0x72, 0xba, 0x93, 0xd1, 0xd3, 0xe8, 0x03, 0x83, 0x45, 0x6f,
|
||||||
|
+ 0xc1, 0x86, 0xb9, 0x10, 0x39, 0x65, 0x5d, 0x51, 0xf6, 0x0d, 0xbe, 0x44, 0x9a, 0x26, 0x09, 0x65,
|
||||||
|
+ 0x82, 0x7b, 0x1c, 0x07, 0x01, 0x8d, 0x12, 0x53, 0x98, 0xf4, 0x2d, 0x7e, 0xa2, 0xd1, 0xce, 0x14,
|
||||||
|
+ 0x36, 0x1f, 0x4a, 0x3b, 0x8d, 0x25, 0xf9, 0x06, 0xf7, 0x22, 0x1c, 0x79, 0xc7, 0x33, 0x1a, 0x9c,
|
||||||
|
+ 0x7a, 0x32, 0x4c, 0x19, 0x0f, 0xcb, 0xd4, 0x67, 0x4f, 0x22, 0x27, 0xe4, 0x1b, 0x55, 0x83, 0x4b,
|
||||||
|
+ 0xaa, 0x13, 0x2a, 0x92, 0x59, 0x3a, 0xf5, 0x12, 0x46, 0x8f, 0xb1, 0x31, 0xb1, 0x1f, 0xe1, 0xe8,
|
||||||
|
+ 0x40, 0xe3, 0x8f, 0x24, 0xda, 0xf9, 0x53, 0x05, 0xb6, 0xca, 0x92, 0x4c, 0xd0, 0xdd, 0x85, 0xad,
|
||||||
|
+ 0xb2, 0x28, 0xf3, 0x10, 0xeb, 0x44, 0x6f, 0x50, 0x14, 0xa8, 0x9f, 0xe4, 0x0f, 0xa1, 0xab, 0xda,
|
||||||
|
+ 0xb9, 0x5e, 0xa8, 0x39, 0x95, 0xd3, 0x8f, 0xe2, 0xbe, 0xb8, 0x1d, 0xbf, 0xb8, 0x4b, 0x1f, 0xc3,
|
||||||
|
+ 0x35, 0x63, 0xbe, 0xb7, 0xac, 0xb6, 0x3e, 0x10, 0xdb, 0x86, 0xe0, 0xd1, 0x82, 0xf6, 0x5f, 0xc0,
|
||||||
|
+ 0x30, 0x47, 0xed, 0xcd, 0x15, 0xd2, 0xfa, 0xea, 0x5d, 0xd8, 0x5c, 0x30, 0xf6, 0x5e, 0x18, 0x32,
|
||||||
|
+ 0x75, 0x41, 0xeb, 0xee, 0xaa, 0x29, 0xe7, 0x2e, 0x5c, 0x9d, 0x60, 0xa1, 0xbd, 0xe1, 0x0b, 0x53,
|
||||||
|
+ 0x13, 0x68, 0x66, 0x1b, 0x50, 0x9b, 0xe0, 0x40, 0x19, 0x5f, 0x73, 0xe5, 0x50, 0x1e, 0xc0, 0xa7,
|
||||||
|
+ 0x1c, 0x07, 0xca, 0xca, 0x9a, 0xab, 0xc6, 0xce, 0x1f, 0x2b, 0xb0, 0x6e, 0xc2, 0xa4, 0x0c, 0xf5,
|
||||||
|
+ 0x21, 0x23, 0x67, 0x98, 0x99, 0xa3, 0x67, 0x20, 0xf4, 0x06, 0xf4, 0xf4, 0xc8, 0xa3, 0x89, 0x20,
|
||||||
|
+ 0x34, 0x0b, 0xbe, 0x5d, 0x8d, 0x7d, 0xac, 0x91, 0xaa, 0x53, 0xa7, 0x1a, 0x51, 0xa6, 0xe6, 0x33,
|
||||||
|
+ 0x90, 0x6a, 0xb7, 0x71, 0x19, 0x19, 0x54, 0xb0, 0x6d, 0xb9, 0x06, 0x92, 0x47, 0xdd, 0xf2, 0x5b,
|
||||||
|
+ 0x53, 0xfc, 0x2c, 0x28, 0x8f, 0x7a, 0x44, 0xd3, 0x58, 0x78, 0x09, 0x25, 0xb1, 0x30, 0xd1, 0x15,
|
||||||
|
+ 0x14, 0xea, 0x48, 0x62, 0x9c, 0x5f, 0x55, 0xa0, 0xa1, 0xbb, 0xd5, 0xb2, 0xca, 0xcc, 0xde, 0xb8,
|
||||||
|
+ 0x2a, 0x51, 0xf9, 0x82, 0x92, 0xa5, 0xdf, 0x35, 0x35, 0x96, 0xf7, 0xf8, 0x2c, 0xd2, 0x91, 0xda,
|
||||||
|
+ 0xa8, 0x76, 0x16, 0xa9, 0x10, 0xfd, 0x06, 0xf4, 0xf2, 0xa7, 0x52, 0xcd, 0x6b, 0x15, 0xbb, 0x19,
|
||||||
|
+ 0x56, 0x91, 0x5d, 0xa8, 0xa9, 0xf3, 0x13, 0x59, 0x5c, 0x67, 0x9d, 0xda, 0x0d, 0xa8, 0xa5, 0x99,
|
||||||
|
+ 0x32, 0x72, 0x28, 0x31, 0xd3, 0xec, 0x91, 0x95, 0x43, 0x74, 0x1b, 0x7a, 0x7e, 0x18, 0x12, 0xb9,
|
||||||
|
+ 0xdc, 0x9f, 0x3d, 0x24, 0x61, 0x76, 0x49, 0xcb, 0x58, 0xe7, 0x2f, 0x15, 0xe8, 0xef, 0xd3, 0x64,
|
||||||
|
+ 0xfe, 0xff, 0x64, 0x86, 0x0b, 0x11, 0x44, 0x29, 0x69, 0xde, 0x58, 0x39, 0x96, 0x79, 0xe3, 0x33,
|
||||||
|
+ 0x32, 0xc3, 0xfa, 0x6a, 0xe9, 0x9d, 0x6d, 0x4a, 0x84, 0xba, 0x56, 0x76, 0x32, 0x6b, 0x80, 0x75,
|
||||||
|
+ 0xf5, 0xe4, 0x23, 0x1a, 0xaa, 0x0c, 0x39, 0x24, 0xcc, 0xcb, 0xda, 0x5d, 0x5d, 0x77, 0x3d, 0x24,
|
||||||
|
+ 0x4c, 0x4d, 0x19, 0x43, 0xd6, 0x54, 0xc7, 0xb5, 0x68, 0x48, 0x43, 0x63, 0xa4, 0x21, 0xdb, 0xd0,
|
||||||
|
+ 0xa0, 0xcf, 0x9e, 0x71, 0x2c, 0x54, 0x2e, 0x5b, 0x73, 0x0d, 0x94, 0x85, 0xb9, 0x66, 0x21, 0xcc,
|
||||||
|
+ 0x5d, 0x81, 0x4d, 0xd5, 0xdb, 0x7f, 0xc2, 0xfc, 0x80, 0xc4, 0x53, 0x1b, 0x8a, 0xb7, 0x00, 0x4d,
|
||||||
|
+ 0x04, 0x4d, 0x16, 0xb0, 0x63, 0x18, 0x98, 0x37, 0xe7, 0xe8, 0xc7, 0x13, 0x6b, 0xfa, 0x35, 0x68,
|
||||||
|
+ 0x4a, 0xd0, 0x63, 0xf8, 0x6b, 0x1b, 0x18, 0xcd, 0xb4, 0xf3, 0x16, 0x74, 0xf4, 0xd0, 0x84, 0x81,
|
||||||
|
+ 0x9c, 0x94, 0x97, 0x49, 0xf9, 0x9d, 0xbf, 0x6d, 0x98, 0x70, 0x6b, 0x6a, 0x68, 0xf4, 0x10, 0xfa,
|
||||||
|
+ 0x0b, 0xff, 0x64, 0x90, 0x69, 0xaa, 0xac, 0xfe, 0x55, 0x33, 0xda, 0x1e, 0xeb, 0x7f, 0x3c, 0x63,
|
||||||
|
+ 0xfb, 0x8f, 0x67, 0xfc, 0x20, 0x4a, 0xc4, 0x1c, 0x3d, 0x80, 0x5e, 0xf9, 0xef, 0x05, 0xba, 0x6e,
|
||||||
|
+ 0x73, 0x90, 0x15, 0xff, 0x34, 0x2e, 0x64, 0xf3, 0x10, 0xfa, 0x0b, 0x3f, 0x32, 0xac, 0x3e, 0xab,
|
||||||
|
+ 0xff, 0x6f, 0x5c, 0xc8, 0xe8, 0x2e, 0xb4, 0x0b, 0x7f, 0x2e, 0xd0, 0x50, 0x33, 0x59, 0xfe, 0x99,
|
||||||
|
+ 0x71, 0x21, 0x83, 0x7d, 0xe8, 0x96, 0x7e, 0x26, 0xa0, 0x91, 0xb1, 0x67, 0xc5, 0x1f, 0x86, 0x0b,
|
||||||
|
+ 0x99, 0xec, 0x41, 0xbb, 0xd0, 0xd3, 0xb7, 0x5a, 0x2c, 0xff, 0x38, 0x18, 0x5d, 0x5b, 0x31, 0x63,
|
||||||
|
+ 0xb6, 0xf3, 0x00, 0xba, 0xa5, 0x0e, 0xbc, 0x55, 0x64, 0x55, 0xf7, 0x7f, 0x74, 0x7d, 0xe5, 0x9c,
|
||||||
|
+ 0xe1, 0xf4, 0x10, 0xfa, 0x0b, 0xfd, 0x78, 0xeb, 0xdc, 0xd5, 0x6d, 0xfa, 0x0b, 0xcd, 0xfa, 0x5c,
|
||||||
|
+ 0x6d, 0x76, 0xa1, 0xdc, 0x2a, 0x6c, 0xf6, 0x72, 0xf7, 0x7d, 0x74, 0x63, 0xf5, 0xa4, 0xd1, 0xea,
|
||||||
|
+ 0x01, 0xf4, 0xca, 0x8d, 0x77, 0xcb, 0x6c, 0x65, 0x3b, 0xfe, 0xf2, 0x93, 0x53, 0xea, 0xc1, 0xe7,
|
||||||
|
+ 0x27, 0x67, 0x55, 0x6b, 0xfe, 0x42, 0x46, 0xf7, 0x00, 0x4c, 0x71, 0x15, 0x92, 0x38, 0xdb, 0xb2,
|
||||||
|
+ 0xa5, 0xa2, 0x2e, 0xdb, 0xb2, 0x15, 0x85, 0xd8, 0x5d, 0x00, 0x5d, 0x13, 0x85, 0x34, 0x15, 0xe8,
|
||||||
|
+ 0xaa, 0x55, 0x63, 0xa1, 0x10, 0x1b, 0x0d, 0x97, 0x27, 0x96, 0x18, 0x60, 0xc6, 0x5e, 0x84, 0xc1,
|
||||||
|
+ 0x67, 0x00, 0x79, 0xad, 0x65, 0x19, 0x2c, 0x55, 0x5f, 0x97, 0xf8, 0xa0, 0x53, 0xac, 0xac, 0x90,
|
||||||
|
+ 0xb1, 0x75, 0x45, 0xb5, 0x75, 0x09, 0x8b, 0xfe, 0x42, 0xe6, 0x5c, 0x3e, 0x6c, 0x8b, 0x09, 0xf5,
|
||||||
|
+ 0x68, 0x29, 0x7b, 0x46, 0x1f, 0x42, 0xa7, 0x98, 0x32, 0x5b, 0x2d, 0x56, 0xa4, 0xd1, 0xa3, 0x52,
|
||||||
|
+ 0xda, 0x8c, 0xee, 0x42, 0xaf, 0x9c, 0x10, 0xa3, 0xc2, 0xbd, 0x58, 0x4a, 0x93, 0x47, 0xa6, 0x19,
|
||||||
|
+ 0x54, 0x20, 0x7f, 0x1f, 0x20, 0x4f, 0x9c, 0xad, 0xfb, 0x96, 0x52, 0xe9, 0x05, 0xa9, 0x9f, 0x41,
|
||||||
|
+ 0xaf, 0x10, 0xb7, 0x65, 0x4d, 0x78, 0xb5, 0x64, 0x70, 0x1e, 0xcd, 0x47, 0x26, 0xc3, 0x2a, 0x85,
|
||||||
|
+ 0xed, 0x7b, 0xd0, 0x29, 0xbe, 0x11, 0xd6, 0xda, 0x15, 0xef, 0xc6, 0x65, 0x41, 0xaf, 0xf0, 0x9e,
|
||||||
|
+ 0xd8, 0xb3, 0xbb, 0xfc, 0xc4, 0x5c, 0x16, 0xf4, 0x4a, 0xf5, 0xa8, 0x8d, 0x35, 0xab, 0x8a, 0xd4,
|
||||||
|
+ 0xcb, 0x9e, 0x82, 0x72, 0xf1, 0x66, 0xbd, 0xbf, 0xb2, 0xa4, 0xbb, 0xec, 0x0c, 0x16, 0xeb, 0x14,
|
||||||
|
+ 0xeb, 0x8f, 0x15, 0xb5, 0xcb, 0xf7, 0xc4, 0x84, 0x62, 0x2d, 0x52, 0x88, 0x09, 0x2b, 0x4a, 0x94,
|
||||||
|
+ 0x0b, 0x19, 0x1d, 0x40, 0xff, 0xa1, 0x4d, 0x33, 0x4d, 0x0a, 0x6c, 0xd4, 0x59, 0x91, 0xf2, 0x8f,
|
||||||
|
+ 0x46, 0xab, 0xa6, 0xcc, 0x2e, 0x7f, 0x0e, 0x83, 0xa5, 0xf4, 0x17, 0xdd, 0xcc, 0x5a, 0x9e, 0x2b,
|
||||||
|
+ 0xf3, 0xe2, 0x0b, 0xd5, 0x3a, 0x84, 0x8d, 0xc5, 0xec, 0x17, 0xbd, 0x62, 0x36, 0x7d, 0x75, 0x56,
|
||||||
|
+ 0x7c, 0x21, 0xab, 0x8f, 0xa1, 0x69, 0xb3, 0x2d, 0x64, 0x5a, 0xcb, 0x0b, 0xd9, 0xd7, 0x45, 0x4b,
|
||||||
|
+ 0xf7, 0x3a, 0xdf, 0x7e, 0x77, 0xb3, 0xf2, 0xd7, 0xef, 0x6e, 0x56, 0xfe, 0xf1, 0xdd, 0xcd, 0xca,
|
||||||
|
+ 0x71, 0x43, 0xcd, 0xbe, 0xff, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x45, 0xa8, 0x91, 0xab, 0x62,
|
||||||
|
+ 0x22, 0x00, 0x00,
|
||||||
|
}
|
||||||
|
diff --git a/protocols/grpc/agent.proto b/protocols/grpc/agent.proto
|
||||||
|
index b0fab6d..d863645 100644
|
||||||
|
--- a/protocols/grpc/agent.proto
|
||||||
|
+++ b/protocols/grpc/agent.proto
|
||||||
|
@@ -46,6 +46,7 @@ service AgentService {
|
||||||
|
rpc UpdateRoutes(UpdateRoutesRequest) returns (Routes);
|
||||||
|
rpc ListInterfaces(ListInterfacesRequest) returns(Interfaces);
|
||||||
|
rpc ListRoutes(ListRoutesRequest) returns (Routes);
|
||||||
|
+ rpc UpdateIPVSRule(UpdateIPVSRequest) returns (IPVSResponse);
|
||||||
|
|
||||||
|
// tracing
|
||||||
|
rpc StartTracing(StartTracingRequest) returns (google.protobuf.Empty);
|
||||||
|
@@ -495,3 +496,13 @@ message StartTracingRequest {
|
||||||
|
|
||||||
|
message StopTracingRequest {
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+message UpdateIPVSRequest {
|
||||||
|
+ // IPVS_req is the IPVS rule message needed to update
|
||||||
|
+ string IPVS_req = 1;
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+message IPVSResponse {
|
||||||
|
+ // IPVS_res is the response of IPVS updating
|
||||||
|
+ string IPVS_res = 1;
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
193
agent/patches/0004-agent-add-IPVS-test.patch
Normal file
193
agent/patches/0004-agent-add-IPVS-test.patch
Normal file
@ -0,0 +1,193 @@
|
|||||||
|
From 01563c08910ddaba4077fd9dc691df541e045165 Mon Sep 17 00:00:00 2001
|
||||||
|
From: xiadanni <xiadanni1@huawei.com>
|
||||||
|
Date: Tue, 18 Aug 2020 17:05:32 +0800
|
||||||
|
Subject: [PATCH 04/16] agent: add IPVS test
|
||||||
|
|
||||||
|
Signed-off-by: xiadanni <xiadanni1@huawei.com>
|
||||||
|
---
|
||||||
|
grpc_test.go | 172 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||||
|
1 file changed, 172 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/grpc_test.go b/grpc_test.go
|
||||||
|
index e69102b..d58c0b6 100644
|
||||||
|
--- a/grpc_test.go
|
||||||
|
+++ b/grpc_test.go
|
||||||
|
@@ -1840,3 +1840,175 @@ func getPipeMaxSize() (uint32, error) {
|
||||||
|
u, err := strconv.ParseUint(s, 10, 32)
|
||||||
|
return uint32(u), err
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func TestUpdateIPVSRule(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+
|
||||||
|
+ // add IPVS rule successfully
|
||||||
|
+ a := &agentGRPC{
|
||||||
|
+ sandbox: &sandbox{
|
||||||
|
+ containers: make(map[string]*container),
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ req := &pb.UpdateIPVSRequest{
|
||||||
|
+ IPVSReq: "ipvsadm -A -t 17.2.0.7:80 -s rr -p 3000",
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ _, err := a.UpdateIPVSRule(context.Background(), req)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ // delete ipvs rule successfully
|
||||||
|
+ req = &pb.UpdateIPVSRequest{
|
||||||
|
+ IPVSReq: "ipvsadm -D -t 17.2.0.7:80",
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ // update ipvs rule error because exec failed
|
||||||
|
+ req = &pb.UpdateIPVSRequest{
|
||||||
|
+ IPVSReq: "ipvsadm -A -t 17.2.0.7:80 -s rr -p -3000",
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ assert.Contains(err.Error(), "exec IPVS command failed")
|
||||||
|
+
|
||||||
|
+ // update IPVS rule error because rule less than validHeadLength
|
||||||
|
+ req = &pb.UpdateIPVSRequest{
|
||||||
|
+ IPVSReq: "ipvsa",
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ assert.Contains(err.Error(), "invalid IPVS rule")
|
||||||
|
+
|
||||||
|
+ // update ipvs rule error because invalid command
|
||||||
|
+ req = &pb.UpdateIPVSRequest{
|
||||||
|
+ IPVSReq: "abcabcabc ipvsadm",
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ assert.Contains(err.Error(), "invalid IPVS rule")
|
||||||
|
+
|
||||||
|
+ // add ipvs rule error because rule count exceeds
|
||||||
|
+ a = &agentGRPC{
|
||||||
|
+ sandbox: &sandbox{
|
||||||
|
+ containers: make(map[string]*container),
|
||||||
|
+ ipvsadm: ipvsAdm{
|
||||||
|
+ ipvsRuleCnt: 20000,
|
||||||
|
+ },
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ req = &pb.UpdateIPVSRequest{
|
||||||
|
+ IPVSReq: "ipvsadm -A -t 17.2.0.7:80 -s rr -p 3000",
|
||||||
|
+ }
|
||||||
|
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ assert.Errorf(err, "rules exceed limit")
|
||||||
|
+
|
||||||
|
+ // add ipvs rule error because ipvs request item less than 2
|
||||||
|
+ a = &agentGRPC{
|
||||||
|
+ sandbox: &sandbox{
|
||||||
|
+ containers: make(map[string]*container),
|
||||||
|
+ ipvsadm: ipvsAdm{
|
||||||
|
+ ipvsRuleCnt: 0,
|
||||||
|
+ },
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ req = &pb.UpdateIPVSRequest{
|
||||||
|
+ IPVSReq: "ipvsadm",
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ assert.Contains(err.Error(), "invalid IPVS rule")
|
||||||
|
+
|
||||||
|
+ // add ipvs rule error because ipvs rule nil
|
||||||
|
+ req = nil
|
||||||
|
+
|
||||||
|
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ assert.Contains(err.Error(), "IPVS rule is nil")
|
||||||
|
+
|
||||||
|
+ // add ipvs rule error because ipvs rule string is empty
|
||||||
|
+ req = &pb.UpdateIPVSRequest{
|
||||||
|
+ IPVSReq: "",
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ assert.Contains(err.Error(), "IPVS rule is nil")
|
||||||
|
+
|
||||||
|
+ // restore ipvs rule successfully
|
||||||
|
+ a = &agentGRPC{
|
||||||
|
+ sandbox: &sandbox{
|
||||||
|
+ containers: make(map[string]*container),
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ req = &pb.UpdateIPVSRequest{
|
||||||
|
+ IPVSReq: "restore|2|-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m",
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ // clear IPVS rule successfully
|
||||||
|
+ req = &pb.UpdateIPVSRequest{
|
||||||
|
+ IPVSReq: "ipvsadm -C",
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ assert.Equal(a.sandbox.ipvsadm.ipvsRuleCnt, uint64(0))
|
||||||
|
+
|
||||||
|
+ // restore ipvs rule error because rule count invalid
|
||||||
|
+ req = &pb.UpdateIPVSRequest{
|
||||||
|
+ IPVSReq: "restore|abc|-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m",
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ assert.Contains(err.Error(), "invalid IPVS rule")
|
||||||
|
+
|
||||||
|
+ // restore ipvs rule error because other rules exists
|
||||||
|
+ a = &agentGRPC{
|
||||||
|
+ sandbox: &sandbox{
|
||||||
|
+ containers: make(map[string]*container),
|
||||||
|
+ ipvsadm: ipvsAdm{
|
||||||
|
+ ipvsRuleCnt: 5,
|
||||||
|
+ },
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ req = &pb.UpdateIPVSRequest{
|
||||||
|
+ IPVSReq: "restore|2|-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m",
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ assert.Contains(err.Error(), "exist some rules in system")
|
||||||
|
+
|
||||||
|
+ // restore ipvs rule error because ipvs req item less than 3
|
||||||
|
+ a = &agentGRPC{
|
||||||
|
+ sandbox: &sandbox{
|
||||||
|
+ containers: make(map[string]*container),
|
||||||
|
+ ipvsadm: ipvsAdm{
|
||||||
|
+ ipvsRuleCnt: 0,
|
||||||
|
+ },
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ req = &pb.UpdateIPVSRequest{
|
||||||
|
+ IPVSReq: "restore|-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m",
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ assert.Contains(err.Error(), "invalid IPVS rule")
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
42
agent/patches/0005-mount-support-mount-block-device.patch
Normal file
42
agent/patches/0005-mount-support-mount-block-device.patch
Normal file
@ -0,0 +1,42 @@
|
|||||||
|
From 8c9f9be2a9c195d0bc12b43c491adaacb7bb8154 Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Tue, 18 Aug 2020 10:42:38 +0800
|
||||||
|
Subject: [PATCH 05/16] mount: support mount block device
|
||||||
|
|
||||||
|
reason: modify mountStorage to support mount block device
|
||||||
|
"-v /dev/blockdevice:/home/test"
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
mount.go | 15 ++++++++++++++-
|
||||||
|
1 file changed, 14 insertions(+), 1 deletion(-)
|
||||||
|
|
||||||
|
diff --git a/mount.go b/mount.go
|
||||||
|
index f0c3efe..a05d4a6 100644
|
||||||
|
--- a/mount.go
|
||||||
|
+++ b/mount.go
|
||||||
|
@@ -364,7 +364,20 @@ func commonStorageHandler(storage pb.Storage) (string, error) {
|
||||||
|
func mountStorage(storage pb.Storage) error {
|
||||||
|
flags, options := parseMountFlagsAndOptions(storage.Options)
|
||||||
|
|
||||||
|
- return mount(storage.Source, storage.MountPoint, storage.Fstype, flags, options)
|
||||||
|
+ var fsType = storage.Fstype
|
||||||
|
+ if (storage.Driver == driverSCSIType || storage.Driver == driverBlkType) && strings.Contains(storage.Fstype, "bind") {
|
||||||
|
+ cs := strings.Split(storage.Fstype, "-")
|
||||||
|
+ if len(cs) == 2 && cs[1] != "" {
|
||||||
|
+ fsType = cs[1]
|
||||||
|
+ // here we temporarily discard the bind option,
|
||||||
|
+ // in order to be able to mount the file system of the block device.
|
||||||
|
+ // and then reset `storage.Fstype` to "bind" which pass through to the libcontainer pkg.
|
||||||
|
+ flags = flags &^ flagList["bind"]
|
||||||
|
+ storage.Fstype = "bind"
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return mount(storage.Source, storage.MountPoint, fsType, flags, options)
|
||||||
|
}
|
||||||
|
|
||||||
|
// addStorages takes a list of storages passed by the caller, and perform the
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,54 @@
|
|||||||
|
From bccda1d208f31eab55863883cf0718d7b4b8deef Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 18 Aug 2020 19:30:42 +0800
|
||||||
|
Subject: [PATCH 06/16] agent: make workaround for slow response in aarch64
|
||||||
|
|
||||||
|
reason: make workaround for slow response in aarch64
|
||||||
|
when hotplug virtio-net-pci device
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
agent.go | 2 +-
|
||||||
|
device.go | 12 ++++++------
|
||||||
|
2 files changed, 7 insertions(+), 7 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/agent.go b/agent.go
|
||||||
|
index c161e93..e81c2cd 100644
|
||||||
|
--- a/agent.go
|
||||||
|
+++ b/agent.go
|
||||||
|
@@ -185,7 +185,7 @@ var logsVSockPort = uint32(0)
|
||||||
|
var debugConsoleVSockPort = uint32(0)
|
||||||
|
|
||||||
|
// Timeout waiting for a device to be hotplugged
|
||||||
|
-var hotplugTimeout = 3 * time.Second
|
||||||
|
+var hotplugTimeout = 10 * time.Second
|
||||||
|
|
||||||
|
// Specify the log level
|
||||||
|
var logLevel = defaultLogLevel
|
||||||
|
diff --git a/device.go b/device.go
|
||||||
|
index ec1907e..46a1b96 100644
|
||||||
|
--- a/device.go
|
||||||
|
+++ b/device.go
|
||||||
|
@@ -179,13 +179,13 @@ func getPCIDeviceNameImpl(s *sandbox, pciID string) (string, error) {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
- fieldLogger := agentLog.WithField("pciAddr", pciAddr)
|
||||||
|
-
|
||||||
|
// Rescan pci bus if we need to wait for a new pci device
|
||||||
|
- if err = rescanPciBus(); err != nil {
|
||||||
|
- fieldLogger.WithError(err).Error("Failed to scan pci bus")
|
||||||
|
- return "", err
|
||||||
|
- }
|
||||||
|
+ // FIXME:Comment out this code Temporarily, because once the PCIBus is scanned,
|
||||||
|
+ // the device hot-plug event is lost
|
||||||
|
+ //if err = rescanPciBus(); err != nil {
|
||||||
|
+ // fieldLogger.WithError(err).Error("Failed to scan pci bus")
|
||||||
|
+ // return "", err
|
||||||
|
+ //}
|
||||||
|
|
||||||
|
return getDeviceName(s, pciAddr)
|
||||||
|
}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,37 @@
|
|||||||
|
From fa673c93e243ba297d53b585cd2f51fa68380fc5 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 18 Aug 2020 19:37:48 +0800
|
||||||
|
Subject: [PATCH 07/16] agent: using pcie-root-port driver to hotplug device
|
||||||
|
|
||||||
|
reason: In original pci-bridge scheme, the "F" in the BDF is not used,
|
||||||
|
and was written hard code "0" in the kata-agent. But the "function"
|
||||||
|
is specified when switching to pcie-root-port scheme, so when should
|
||||||
|
pass the "pci-bridge BDF" or "pcie-root-port BDF" from kata-runtime
|
||||||
|
and use in kata-agent.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
device.go | 7 ++++++-
|
||||||
|
1 file changed, 6 insertions(+), 1 deletion(-)
|
||||||
|
|
||||||
|
diff --git a/device.go b/device.go
|
||||||
|
index 46a1b96..8e6950c 100644
|
||||||
|
--- a/device.go
|
||||||
|
+++ b/device.go
|
||||||
|
@@ -101,7 +101,12 @@ func getDevicePCIAddressImpl(pciID string) (string, error) {
|
||||||
|
|
||||||
|
// Deduce the complete bridge address based on the bridge address identifier passed
|
||||||
|
// and the fact that bridges are attached on the main bus with function 0.
|
||||||
|
- pciBridgeAddr := fmt.Sprintf("0000:00:%s.0", bridgeID)
|
||||||
|
+ // Update: support pcie-root-port device
|
||||||
|
+ // In original pci-bridge scheme, the "F" in the BDF is not used, and was written
|
||||||
|
+ // hard code "0" in the kata-agent. But the "function" is specified when switching to
|
||||||
|
+ // pcie-root-port scheme, so when should pass the "pci-bridge BDF" or "pcie-root-port BDF"
|
||||||
|
+ // from kata-runtime and use in kata-agent.
|
||||||
|
+ pciBridgeAddr := fmt.Sprintf("0000:00:%s", bridgeID)
|
||||||
|
|
||||||
|
// Find out the bus exposed by bridge
|
||||||
|
bridgeBusPath := fmt.Sprintf(pciBusPathFormat, sysBusPrefix, pciBridgeAddr)
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,94 @@
|
|||||||
|
From eea286fbafba2e95410b603fbef762e2b25eb207 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 18 Aug 2020 19:45:57 +0800
|
||||||
|
Subject: [PATCH 08/16] agent: support get root bus path dynamically
|
||||||
|
|
||||||
|
reason: support get root bus dynamically no matter the
|
||||||
|
target arch is amd64 or arm64
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
agent.go | 1 +
|
||||||
|
device_amd64.go | 6 +++++-
|
||||||
|
device_arm64.go | 27 ++++++++++++++++++++++++++-
|
||||||
|
3 files changed, 32 insertions(+), 2 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/agent.go b/agent.go
|
||||||
|
index e81c2cd..50afd7a 100644
|
||||||
|
--- a/agent.go
|
||||||
|
+++ b/agent.go
|
||||||
|
@@ -730,6 +730,7 @@ func (s *sandbox) listenToUdevEvents() {
|
||||||
|
defer uEvHandler.Close()
|
||||||
|
|
||||||
|
fieldLogger.Infof("Started listening for uevents")
|
||||||
|
+ rootBusPath := initRootBusPath()
|
||||||
|
|
||||||
|
for {
|
||||||
|
uEv, err := uEvHandler.Read()
|
||||||
|
diff --git a/device_amd64.go b/device_amd64.go
|
||||||
|
index 66bc052..26f55bf 100644
|
||||||
|
--- a/device_amd64.go
|
||||||
|
+++ b/device_amd64.go
|
||||||
|
@@ -8,7 +8,7 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
const (
|
||||||
|
- rootBusPath = "/devices/pci0000:00"
|
||||||
|
+ defaultRootBusPath = "/devices/pci0000:00"
|
||||||
|
|
||||||
|
// From https://www.kernel.org/doc/Documentation/acpi/namespace.txt
|
||||||
|
// The Linux kernel's core ACPI subsystem creates struct acpi_device
|
||||||
|
@@ -21,3 +21,7 @@ const (
|
||||||
|
// in a subdirectory whose prefix is pfn (page frame number).
|
||||||
|
pfnDevPrefix = "/pfn"
|
||||||
|
)
|
||||||
|
+
|
||||||
|
+func initRootBusPath() string {
|
||||||
|
+ return defaultRootBusPath
|
||||||
|
+}
|
||||||
|
diff --git a/device_arm64.go b/device_arm64.go
|
||||||
|
index b73b582..d039c67 100644
|
||||||
|
--- a/device_arm64.go
|
||||||
|
+++ b/device_arm64.go
|
||||||
|
@@ -6,8 +6,14 @@
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
+import (
|
||||||
|
+ "fmt"
|
||||||
|
+ "io/ioutil"
|
||||||
|
+ "regexp"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
const (
|
||||||
|
- rootBusPath = "/devices/platform/4010000000.pcie/pci0000:00"
|
||||||
|
+ defaultRootBusPath = "/devices/platform/4010000000.pcie/pci0000:00"
|
||||||
|
|
||||||
|
// From https://www.kernel.org/doc/Documentation/acpi/namespace.txt
|
||||||
|
// The Linux kernel's core ACPI subsystem creates struct acpi_device
|
||||||
|
@@ -20,3 +26,22 @@ const (
|
||||||
|
// in a subdirectory whose prefix is pfn (page frame number).
|
||||||
|
pfnDevPrefix = "/pfn"
|
||||||
|
)
|
||||||
|
+
|
||||||
|
+func initRootBusPath() string {
|
||||||
|
+ pcieDriverReg := regexp.MustCompile(`^[0-9a-f]{10}.pcie$`)
|
||||||
|
+ rootBusPath := defaultRootBusPath
|
||||||
|
+ files, err := ioutil.ReadDir("/sys/devices/platform")
|
||||||
|
+ if err != nil {
|
||||||
|
+ return rootBusPath
|
||||||
|
+ }
|
||||||
|
+ for _, f := range files {
|
||||||
|
+ if !f.IsDir() {
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+ if pcieDriverReg.MatchString(f.Name()) {
|
||||||
|
+ rootBusPath = fmt.Sprintf("/devices/platform/%s/pci0000:00", f.Name())
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ return rootBusPath
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
188
agent/patches/0009-storage-add-pkg-storage-for-mount.patch
Normal file
188
agent/patches/0009-storage-add-pkg-storage-for-mount.patch
Normal file
@ -0,0 +1,188 @@
|
|||||||
|
From 1268b710c7f0528d971d9c0e54429d3f4e48c372 Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Tue, 18 Aug 2020 16:45:29 +0800
|
||||||
|
Subject: [PATCH 09/16] storage: add pkg/storage for mount
|
||||||
|
|
||||||
|
reason: add gpath.go and nfs.go, provide mount
|
||||||
|
functions and structs
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
pkg/storage/gpath.go | 64 ++++++++++++++++++++++++++++++++++++
|
||||||
|
pkg/storage/nfs.go | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||||
|
2 files changed, 157 insertions(+)
|
||||||
|
create mode 100644 pkg/storage/gpath.go
|
||||||
|
create mode 100644 pkg/storage/nfs.go
|
||||||
|
|
||||||
|
diff --git a/pkg/storage/gpath.go b/pkg/storage/gpath.go
|
||||||
|
new file mode 100644
|
||||||
|
index 0000000..5cb951f
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/pkg/storage/gpath.go
|
||||||
|
@@ -0,0 +1,64 @@
|
||||||
|
+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
||||||
|
+// SPDX-License-Identifier: Apache-2.0
|
||||||
|
+// Description: common functions
|
||||||
|
+// Author: licuifang
|
||||||
|
+// Create: 2019-06-24
|
||||||
|
+
|
||||||
|
+package storage
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "fmt"
|
||||||
|
+ "os"
|
||||||
|
+ "syscall"
|
||||||
|
+
|
||||||
|
+ "github.com/opencontainers/runc/libcontainer/mount"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+func ValidateGpath(guestPath string, opts []string) error {
|
||||||
|
+ for _, opt := range opts {
|
||||||
|
+ switch opt {
|
||||||
|
+ case "shared":
|
||||||
|
+ flag := syscall.MS_SHARED
|
||||||
|
+ if err := ensureMountedAs(guestPath, flag); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ return nil
|
||||||
|
+ case "mounted":
|
||||||
|
+ // if mounted in option, the guestpath must exist and must be a mountpoint
|
||||||
|
+ _, err := os.Stat(guestPath)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ mountpoint, err := mount.Mounted(guestPath)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ if mountpoint == false {
|
||||||
|
+ return fmt.Errorf("the guespath:%s is not a mountpoint while mounted was set", guestPath)
|
||||||
|
+ }
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ if err := os.MkdirAll(guestPath, 0750); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func ensureMountedAs(mountPoint string, flag int) error {
|
||||||
|
+ if err := os.MkdirAll(mountPoint, 0750); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ mounted, err := mount.Mounted(mountPoint)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if !mounted {
|
||||||
|
+ if err := syscall.Mount(mountPoint, mountPoint, "bind", uintptr(syscall.MS_BIND), ""); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return syscall.Mount("", mountPoint, "", uintptr(flag), "")
|
||||||
|
+}
|
||||||
|
diff --git a/pkg/storage/nfs.go b/pkg/storage/nfs.go
|
||||||
|
new file mode 100644
|
||||||
|
index 0000000..44bc85d
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/pkg/storage/nfs.go
|
||||||
|
@@ -0,0 +1,93 @@
|
||||||
|
+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
||||||
|
+// SPDX-License-Identifier: Apache-2.0
|
||||||
|
+// Description: common functions
|
||||||
|
+// Author: leizhongkai
|
||||||
|
+// Create: 2019-03-10
|
||||||
|
+
|
||||||
|
+package storage
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "fmt"
|
||||||
|
+ "os"
|
||||||
|
+ "os/exec"
|
||||||
|
+ "path/filepath"
|
||||||
|
+ "strings"
|
||||||
|
+ "time"
|
||||||
|
+
|
||||||
|
+ "github.com/opencontainers/runc/libcontainer/mount"
|
||||||
|
+ "github.com/sirupsen/logrus"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+const (
|
||||||
|
+ NFS = "nfs"
|
||||||
|
+
|
||||||
|
+ MAX_MOUNT_ATTEMPTS = 100
|
||||||
|
+ MAX_MOUTN_TIMEOUT = 90 // seconds
|
||||||
|
+
|
||||||
|
+ // ignored errors
|
||||||
|
+ PERMISSION_DENY = "Permission denied"
|
||||||
|
+ kataGuestStorageDir = "/run/kata-containers/storage/containers/"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+func nfsMount(source, dest string, opt string) error {
|
||||||
|
+ cmd := exec.Command("/bin/mount", "-t", "nfs", "-o", opt, source, dest)
|
||||||
|
+ res, err := cmd.Output()
|
||||||
|
+ logrus.Debugf("mount %s to %s, and get res %s", source, dest, string(res))
|
||||||
|
+ return err
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func nfsMountWithAttempt(source, dest string, opt string) (err error) {
|
||||||
|
+ timeStart := time.Now().Unix()
|
||||||
|
+ for i := 0; i < MAX_MOUNT_ATTEMPTS; i++ {
|
||||||
|
+ logrus.Infof("this is the %d times to mount %s to %s", i, source, dest)
|
||||||
|
+ err = nfsMount(source, dest, opt)
|
||||||
|
+ if err != nil {
|
||||||
|
+ ee, ok := err.(*exec.ExitError)
|
||||||
|
+ if ok {
|
||||||
|
+ if strings.Contains(string(ee.Stderr), PERMISSION_DENY) {
|
||||||
|
+ logrus.Errorf("mounting nfs:%s to %s, get error: %s,should break", source, dest, string(ee.Stderr))
|
||||||
|
+ // We do not retry when the error type is PERMISSION_DENY.
|
||||||
|
+ // The reason for the retry is that when you do a SFS mount,
|
||||||
|
+ // the network may not have been created yet, because
|
||||||
|
+ // creating the network and startup container is asynchronous
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+ logrus.Infof("mounting nfs:%s to %s, get error: %s,will retry", source, dest, string(ee.Stderr))
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ elapsed := time.Now().Unix() - timeStart
|
||||||
|
+ if elapsed < MAX_MOUTN_TIMEOUT {
|
||||||
|
+ time.Sleep(50 * time.Microsecond)
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return err
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func MountRemoteNfs(source string, opt []string, sandboxId string) error {
|
||||||
|
+ item := strings.Split(source, ":")
|
||||||
|
+ if len(item) != 2 {
|
||||||
|
+ return fmt.Errorf("the nfs of %s format is error", source)
|
||||||
|
+ }
|
||||||
|
+ tmpDes := filepath.Join(kataGuestStorageDir, sandboxId, item[0],item[1])
|
||||||
|
+ if mounted, err := mount.Mounted(tmpDes); err == nil && mounted == true {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+ err := os.MkdirAll(tmpDes, 0750)
|
||||||
|
+ if err != nil {
|
||||||
|
+ logrus.Infof("mkdir %s failed before mount remote nfs server", tmpDes)
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ defer func() {
|
||||||
|
+ if err != nil {
|
||||||
|
+ os.RemoveAll(tmpDes)
|
||||||
|
+ }
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
+ err = nfsMountWithAttempt(source, tmpDes, strings.Join(opt, ","))
|
||||||
|
+
|
||||||
|
+ return err
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,92 @@
|
|||||||
|
From 79bafc5fd8a1dcda6f44ecd830dfe50f4ef1b34b Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Tue, 18 Aug 2020 20:41:30 +0800
|
||||||
|
Subject: [PATCH 10/16] storage: mount nfs and gpath in agent
|
||||||
|
|
||||||
|
reason: add nfsStorageHandler and gpathStorageHandler in
|
||||||
|
storageHandlerList to mount nfs and gpath
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
device.go | 2 ++
|
||||||
|
mount.go | 37 +++++++++++++++++++++++++++++++++++++
|
||||||
|
2 files changed, 39 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/device.go b/device.go
|
||||||
|
index 8e6950c..29f72e9 100644
|
||||||
|
--- a/device.go
|
||||||
|
+++ b/device.go
|
||||||
|
@@ -35,6 +35,8 @@ const (
|
||||||
|
driverEphemeralType = "ephemeral"
|
||||||
|
driverLocalType = "local"
|
||||||
|
vmRootfs = "/"
|
||||||
|
+ driverNfsType = "nfs"
|
||||||
|
+ driverGpathType = "gpath"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
diff --git a/mount.go b/mount.go
|
||||||
|
index a05d4a6..de2bfaf 100644
|
||||||
|
--- a/mount.go
|
||||||
|
+++ b/mount.go
|
||||||
|
@@ -17,6 +17,7 @@ import (
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
|
||||||
|
+ rmtStorage "github.com/kata-containers/agent/pkg/storage"
|
||||||
|
pb "github.com/kata-containers/agent/protocols/grpc"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
@@ -218,6 +219,8 @@ var storageHandlerList = map[string]storageHandler{
|
||||||
|
driverEphemeralType: ephemeralStorageHandler,
|
||||||
|
driverLocalType: localStorageHandler,
|
||||||
|
driverNvdimmType: nvdimmStorageHandler,
|
||||||
|
+ driverNfsType: nfsStorageHandler,
|
||||||
|
+ driverGpathType: gpathStorageHandler,
|
||||||
|
}
|
||||||
|
|
||||||
|
func ephemeralStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (string, error) {
|
||||||
|
@@ -339,6 +342,40 @@ func nvdimmStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (st
|
||||||
|
return "", fmt.Errorf("invalid nvdimm source path: %v", storage.Source)
|
||||||
|
}
|
||||||
|
|
||||||
|
+func nfsStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (string, error) {
|
||||||
|
+ s.Lock()
|
||||||
|
+ defer s.Unlock()
|
||||||
|
+
|
||||||
|
+ // mount nfs
|
||||||
|
+ err := rmtStorage.MountRemoteNfs(storage.Source, storage.Options, s.id)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return "", fmt.Errorf("mount %s to %s failed, get err:%s", storage.Source, storage.MountPoint, err)
|
||||||
|
+ }
|
||||||
|
+ if err := os.MkdirAll(storage.MountPoint, 0750); err != nil {
|
||||||
|
+ logrus.Infof("mkdir %s failed after mount remote nfs server", storage.MountPoint)
|
||||||
|
+ return "", err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ defer func() {
|
||||||
|
+ if err != nil {
|
||||||
|
+ os.RemoveAll(storage.MountPoint)
|
||||||
|
+ }
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
+ return storage.MountPoint, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func gpathStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (string, error) {
|
||||||
|
+ s.Lock()
|
||||||
|
+ defer s.Unlock()
|
||||||
|
+ // validate guespath
|
||||||
|
+ err := rmtStorage.ValidateGpath(storage.Source, storage.Options)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return "", err
|
||||||
|
+ }
|
||||||
|
+ return "", nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// virtioSCSIStorageHandler handles the storage for scsi driver.
|
||||||
|
func virtioSCSIStorageHandler(ctx context.Context, storage pb.Storage, s *sandbox) (string, error) {
|
||||||
|
// Retrieve the device path from SCSI address.
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,78 @@
|
|||||||
|
From 3ac1232a2e3fbfc0465473e5d81cde41847c4252 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Wed, 19 Aug 2020 11:47:37 +0800
|
||||||
|
Subject: [PATCH 11/16] agent: fix agent reap agent process blocked problem
|
||||||
|
|
||||||
|
reason: add container waitProcess() timeout when
|
||||||
|
container process status is D/T.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
grpc.go | 43 +++++++++++++++++++++++++++++++++----------
|
||||||
|
1 file changed, 33 insertions(+), 10 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/grpc.go b/grpc.go
|
||||||
|
index de2cae7..3dd088e 100644
|
||||||
|
--- a/grpc.go
|
||||||
|
+++ b/grpc.go
|
||||||
|
@@ -49,6 +49,11 @@ const (
|
||||||
|
libcontainerPath = "/run/libcontainer"
|
||||||
|
)
|
||||||
|
|
||||||
|
+// keep waitProcessTimeout value same as value in kata-runtime wait WaitProcessRequest response
|
||||||
|
+const (
|
||||||
|
+ waitProcessTimeOut = 10
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
var (
|
||||||
|
sysfsCPUOnlinePath = "/sys/devices/system/cpu"
|
||||||
|
sysfsMemOnlinePath = "/sys/devices/system/memory"
|
||||||
|
@@ -996,17 +1001,35 @@ func (a *agentGRPC) WaitProcess(ctx context.Context, req *pb.WaitProcessRequest)
|
||||||
|
ctr.deleteProcess(proc.id)
|
||||||
|
})
|
||||||
|
|
||||||
|
- // Using helper function wait() to deal with the subreaper.
|
||||||
|
- libContProcess := (*reaperLibcontainerProcess)(&(proc.process))
|
||||||
|
- exitCode, err := a.sandbox.subreaper.wait(proc.exitCodeCh, libContProcess)
|
||||||
|
- if err != nil {
|
||||||
|
- return &pb.WaitProcessResponse{}, err
|
||||||
|
+ done := make(chan error)
|
||||||
|
+ var exitCode int = 0
|
||||||
|
+ go func() {
|
||||||
|
+ // Using helper function wait() to deal with the subreaper.
|
||||||
|
+ libContProcess := (*reaperLibcontainerProcess)(&(proc.process))
|
||||||
|
+ var err error
|
||||||
|
+ exitCode, err = a.sandbox.subreaper.wait(proc.exitCodeCh, libContProcess)
|
||||||
|
+ if err != nil {
|
||||||
|
+ done <- err
|
||||||
|
+ close(done)
|
||||||
|
+ return
|
||||||
|
+ }
|
||||||
|
+ // refill the exitCodeCh with the exitcode which can be read out
|
||||||
|
+ // by another WaitProcess(). Since this channel isn't be closed,
|
||||||
|
+ // here the refill will always success and it will be free by GC
|
||||||
|
+ // once the process exits.
|
||||||
|
+ proc.exitCodeCh <- exitCode
|
||||||
|
+
|
||||||
|
+ close(done)
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
+ select {
|
||||||
|
+ case err := <-done:
|
||||||
|
+ if err != nil {
|
||||||
|
+ return &pb.WaitProcessResponse{}, err
|
||||||
|
+ }
|
||||||
|
+ case <-time.After(time.Duration(waitProcessTimeOut) * time.Second):
|
||||||
|
+ return &pb.WaitProcessResponse{}, grpcStatus.Errorf(codes.DeadlineExceeded, "agent wait reap container process timeout reached after %ds", waitProcessTimeOut)
|
||||||
|
}
|
||||||
|
- //refill the exitCodeCh with the exitcode which can be read out
|
||||||
|
- //by another WaitProcess(). Since this channel isn't be closed,
|
||||||
|
- //here the refill will always success and it will be free by GC
|
||||||
|
- //once the process exits.
|
||||||
|
- proc.exitCodeCh <- exitCode
|
||||||
|
|
||||||
|
return &pb.WaitProcessResponse{
|
||||||
|
Status: int32(exitCode),
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,38 @@
|
|||||||
|
From edb29dfd8f786735763245b3f156b50fd3c1a08e Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Wed, 19 Aug 2020 15:15:31 +0800
|
||||||
|
Subject: [PATCH 12/16] network: support set dns without nameserver
|
||||||
|
|
||||||
|
reason: when runtime sends dns without nameserver to agent,
|
||||||
|
add nameserver before ip address. scenario like annotation
|
||||||
|
with dns
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
network.go | 5 ++++-
|
||||||
|
1 file changed, 4 insertions(+), 1 deletion(-)
|
||||||
|
|
||||||
|
diff --git a/network.go b/network.go
|
||||||
|
index 02e28cb..7046cf8 100644
|
||||||
|
--- a/network.go
|
||||||
|
+++ b/network.go
|
||||||
|
@@ -708,6 +708,9 @@ func setupDNS(dns []string) (err error) {
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
for i, line := range dns {
|
||||||
|
+ if !strings.Contains(line, "nameserver") {
|
||||||
|
+ line = "nameserver" + " " + line
|
||||||
|
+ }
|
||||||
|
if i == (len(dns) - 1) {
|
||||||
|
_, err = file.WriteString(strings.TrimSpace(line))
|
||||||
|
} else {
|
||||||
|
@@ -761,4 +764,4 @@ func (s *sandbox) handleLocalhost() error {
|
||||||
|
}
|
||||||
|
|
||||||
|
return netlink.LinkSetUp(lo)
|
||||||
|
-}
|
||||||
|
\ No newline at end of file
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,304 @@
|
|||||||
|
From 1394dcf579849e5d8103c31556e9af0216a875d2 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Wed, 19 Aug 2020 17:15:51 +0800
|
||||||
|
Subject: [PATCH 13/16] agent: support setting multi queues of interface
|
||||||
|
|
||||||
|
reason: support setting multi queues of a interface
|
||||||
|
when runtime passing Queue in the request.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
network.go | 12 +++++-
|
||||||
|
pkg/net/ethtool.go | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||||
|
pkg/types/types.pb.go | 82 ++++++++++++++++++++++++++-----------
|
||||||
|
pkg/types/types.proto | 1 +
|
||||||
|
4 files changed, 181 insertions(+), 24 deletions(-)
|
||||||
|
create mode 100644 pkg/net/ethtool.go
|
||||||
|
|
||||||
|
diff --git a/network.go b/network.go
|
||||||
|
index 7046cf8..1baaa2e 100644
|
||||||
|
--- a/network.go
|
||||||
|
+++ b/network.go
|
||||||
|
@@ -17,7 +17,7 @@ import (
|
||||||
|
"syscall"
|
||||||
|
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
-
|
||||||
|
+ agentNet "github.com/kata-containers/agent/pkg/net"
|
||||||
|
"github.com/kata-containers/agent/pkg/types"
|
||||||
|
pb "github.com/kata-containers/agent/protocols/grpc"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
@@ -273,6 +273,16 @@ func (s *sandbox) updateInterface(netHandle *netlink.Handle, iface *types.Interf
|
||||||
|
if err == nil {
|
||||||
|
err = retErr
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+ // if link is up, then set the multi queue to it,
|
||||||
|
+ // the kernel of newer version may set multi queue by itself,
|
||||||
|
+ // but we can not rely on it.
|
||||||
|
+ if err == nil && iface.Queues > 0 {
|
||||||
|
+ if ethErr := agentNet.GetEthtool().SetChannel(iface.Name, iface.Queues); ethErr != nil {
|
||||||
|
+ err = grpcStatus.Errorf(codes.Internal, "Could not set multi queue %d for interface %v: %v",
|
||||||
|
+ iface.Queues, link, ethErr)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
}()
|
||||||
|
|
||||||
|
fieldLogger.WithField("link", fmt.Sprintf("%+v", link)).Info("Link found")
|
||||||
|
diff --git a/pkg/net/ethtool.go b/pkg/net/ethtool.go
|
||||||
|
new file mode 100644
|
||||||
|
index 0000000..56a1ece
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/pkg/net/ethtool.go
|
||||||
|
@@ -0,0 +1,110 @@
|
||||||
|
+/*
|
||||||
|
+Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
||||||
|
+SPDX-License-Identifier: Apache-2.0
|
||||||
|
+Description: common functions
|
||||||
|
+Author: fengshaobao
|
||||||
|
+Create: 2019-05-28
|
||||||
|
+*/
|
||||||
|
+
|
||||||
|
+package net
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "fmt"
|
||||||
|
+ "sync"
|
||||||
|
+ "syscall"
|
||||||
|
+ "unsafe"
|
||||||
|
+
|
||||||
|
+ "github.com/sirupsen/logrus"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+const (
|
||||||
|
+ ethtoolGChannels = 0x0000003c /* Get no of channels */
|
||||||
|
+ ethtoolSChannels = 0x0000003d /* Set no of channels */
|
||||||
|
+ ifNameSize = 16
|
||||||
|
+ siocEthtool = 0x8946
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+var (
|
||||||
|
+ t *Ethtool
|
||||||
|
+ once sync.Once
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+// Ethtool to set multiqueue of a network interface,
|
||||||
|
+// with the public method "SetChannel" to set queues of an interface.
|
||||||
|
+type Ethtool struct {
|
||||||
|
+ fd int
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+type ifReq struct {
|
||||||
|
+ ifrName [ifNameSize]byte
|
||||||
|
+ ifrData uintptr
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+type ethtoolChannels struct {
|
||||||
|
+ // ETHTOOL_{G,S}CHANNELS
|
||||||
|
+ cmd uint32
|
||||||
|
+ // Read only. Maximum number of receive channel the driver support.
|
||||||
|
+ maxRx uint32
|
||||||
|
+ // Read only. Maximum number of transmit channel the driver support
|
||||||
|
+ maxTx uint32
|
||||||
|
+ // Read only. Maximum number of other channel the driver support
|
||||||
|
+ maxOther uint32
|
||||||
|
+ //Read only. Maximum number of combined channel the driver support. Set of queues RX, TX or other
|
||||||
|
+ maxCombined uint32
|
||||||
|
+ // Valid values are in the range 1 to the max_rx
|
||||||
|
+ rxCount uint32
|
||||||
|
+ // Valid values are in the range 1 to the max_tx
|
||||||
|
+ txCount uint32
|
||||||
|
+ // Valid values are in the range 1 to the max_other
|
||||||
|
+ otherCount uint32
|
||||||
|
+ // Valid values are in the range 1 to the max_combined
|
||||||
|
+ combinedCount uint32
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// GetEthtool to config multiqueue of a network interface
|
||||||
|
+func GetEthtool() *Ethtool {
|
||||||
|
+ once.Do(func() {
|
||||||
|
+ var err error
|
||||||
|
+ if t, err = newEthtool(); err != nil {
|
||||||
|
+ panic("can not init a socket fd for ethtool")
|
||||||
|
+ }
|
||||||
|
+ })
|
||||||
|
+ return t
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func newEthtool() (*Ethtool, error) {
|
||||||
|
+ fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
|
||||||
|
+ if err != nil || fd < 0 {
|
||||||
|
+ logrus.Warningf("Can not get socket of inet")
|
||||||
|
+ var newErr error
|
||||||
|
+ fd, newErr = syscall.Socket(syscall.AF_NETLINK, syscall.SOCK_RAW, syscall.NETLINK_GENERIC)
|
||||||
|
+ if newErr != nil || fd < 0 {
|
||||||
|
+ return nil, fmt.Errorf("create inet socket with error: %v, and create netlink socket with error: %v", err, newErr)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return &Ethtool{
|
||||||
|
+ fd: int(fd),
|
||||||
|
+ }, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// SetChannel Set the queues of a network interface.
|
||||||
|
+// @devName: the network interface name, e.g. eth0.
|
||||||
|
+// @queues: queues for the network interface.
|
||||||
|
+func (e *Ethtool) SetChannel(devName string, queues uint32) error {
|
||||||
|
+ c := ðtoolChannels{
|
||||||
|
+ cmd: ethtoolSChannels,
|
||||||
|
+ combinedCount: queues,
|
||||||
|
+ }
|
||||||
|
+ var ifName [ifNameSize]byte
|
||||||
|
+ copy(ifName[:], []byte(devName))
|
||||||
|
+ ifr := ifReq{
|
||||||
|
+ ifrName: ifName,
|
||||||
|
+ ifrData: uintptr(unsafe.Pointer(&c)),
|
||||||
|
+ }
|
||||||
|
+ _, _, ep := syscall.Syscall(syscall.SYS_IOCTL, uintptr(e.fd), siocEthtool, uintptr(unsafe.Pointer(&ifr)))
|
||||||
|
+ if ep != 0 {
|
||||||
|
+ return syscall.Errno(ep)
|
||||||
|
+ }
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
diff --git a/pkg/types/types.pb.go b/pkg/types/types.pb.go
|
||||||
|
index 7ea63e3..8b7e2a5 100644
|
||||||
|
--- a/pkg/types/types.pb.go
|
||||||
|
+++ b/pkg/types/types.pb.go
|
||||||
|
@@ -100,6 +100,7 @@ type Interface struct {
|
||||||
|
// list: "veth", "macvtap", "vlan", "macvlan", "tap", ...
|
||||||
|
Type string `protobuf:"bytes,7,opt,name=type,proto3" json:"type,omitempty"`
|
||||||
|
RawFlags uint32 `protobuf:"varint,8,opt,name=raw_flags,json=rawFlags,proto3" json:"raw_flags,omitempty"`
|
||||||
|
+ Queues uint32 `protobuf:"varint,9,opt,name=Queues,proto3" json:"Queues,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *Interface) Reset() { *m = Interface{} }
|
||||||
|
@@ -163,6 +164,13 @@ func (m *Interface) GetRawFlags() uint32 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (m *Interface) GetQueues() uint32 {
|
||||||
|
+ if m != nil {
|
||||||
|
+ return m.Queues
|
||||||
|
+ }
|
||||||
|
+ return 0
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
type Route struct {
|
||||||
|
Dest string `protobuf:"bytes,1,opt,name=dest,proto3" json:"dest,omitempty"`
|
||||||
|
Gateway string `protobuf:"bytes,2,opt,name=gateway,proto3" json:"gateway,omitempty"`
|
||||||
|
@@ -319,6 +327,11 @@ func (m *Interface) MarshalTo(dAtA []byte) (int, error) {
|
||||||
|
i++
|
||||||
|
i = encodeVarintTypes(dAtA, i, uint64(m.RawFlags))
|
||||||
|
}
|
||||||
|
+ if m.Queues != 0 {
|
||||||
|
+ dAtA[i] = 0x48
|
||||||
|
+ i++
|
||||||
|
+ i = encodeVarintTypes(dAtA, i, uint64(m.Queues))
|
||||||
|
+ }
|
||||||
|
return i, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -430,6 +443,9 @@ func (m *Interface) Size() (n int) {
|
||||||
|
if m.RawFlags != 0 {
|
||||||
|
n += 1 + sovTypes(uint64(m.RawFlags))
|
||||||
|
}
|
||||||
|
+ if m.Queues != 0 {
|
||||||
|
+ n += 1 + sovTypes(uint64(m.Queues))
|
||||||
|
+ }
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -841,6 +857,25 @@ func (m *Interface) Unmarshal(dAtA []byte) error {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
+ case 9:
|
||||||
|
+ if wireType != 0 {
|
||||||
|
+ return fmt.Errorf("proto: wrong wireType = %d for field Queues", wireType)
|
||||||
|
+ }
|
||||||
|
+ m.Queues = 0
|
||||||
|
+ for shift := uint(0); ; shift += 7 {
|
||||||
|
+ if shift >= 64 {
|
||||||
|
+ return ErrIntOverflowTypes
|
||||||
|
+ }
|
||||||
|
+ if iNdEx >= l {
|
||||||
|
+ return io.ErrUnexpectedEOF
|
||||||
|
+ }
|
||||||
|
+ b := dAtA[iNdEx]
|
||||||
|
+ iNdEx++
|
||||||
|
+ m.Queues |= (uint32(b) & 0x7F) << shift
|
||||||
|
+ if b < 0x80 {
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
default:
|
||||||
|
iNdEx = preIndex
|
||||||
|
skippy, err := skipTypes(dAtA[iNdEx:])
|
||||||
|
@@ -1155,27 +1190,28 @@ var (
|
||||||
|
func init() { proto.RegisterFile("pkg/types/types.proto", fileDescriptorTypes) }
|
||||||
|
|
||||||
|
var fileDescriptorTypes = []byte{
|
||||||
|
- // 345 bytes of a gzipped FileDescriptorProto
|
||||||
|
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x91, 0xd1, 0x6a, 0xfa, 0x30,
|
||||||
|
- 0x14, 0xc6, 0xff, 0xb1, 0xb6, 0xda, 0xe3, 0xdf, 0xad, 0x84, 0x4d, 0xc2, 0x06, 0x52, 0xbc, 0x59,
|
||||||
|
- 0xd9, 0x85, 0x03, 0x37, 0x76, 0xbf, 0x5d, 0x08, 0xde, 0x49, 0x5e, 0x60, 0xc4, 0x36, 0x3a, 0xd1,
|
||||||
|
- 0xda, 0xd2, 0x44, 0x8b, 0xec, 0x05, 0x77, 0xb9, 0x47, 0x10, 0x9f, 0x64, 0xe4, 0x24, 0x8a, 0xbb,
|
||||||
|
- 0xd1, 0xef, 0x97, 0x93, 0xd3, 0xef, 0x7c, 0x27, 0x70, 0x5b, 0xae, 0x16, 0x4f, 0x7a, 0x5f, 0x4a,
|
||||||
|
- 0x65, 0x7f, 0x87, 0x65, 0x55, 0xe8, 0x82, 0xfa, 0x08, 0x83, 0x19, 0x84, 0x93, 0xe9, 0x5b, 0x96,
|
||||||
|
- 0x55, 0x52, 0x29, 0xfa, 0x00, 0xc1, 0x5c, 0xe4, 0xcb, 0xf5, 0x9e, 0x91, 0x98, 0x24, 0x57, 0xa3,
|
||||||
|
- 0xeb, 0xa1, 0xed, 0x98, 0x4c, 0xc7, 0x78, 0xcc, 0x5d, 0x99, 0x32, 0x68, 0x09, 0xdb, 0xc3, 0x1a,
|
||||||
|
- 0x31, 0x49, 0x42, 0x7e, 0x42, 0x4a, 0xa1, 0x99, 0x0b, 0xb5, 0x62, 0x1e, 0x1e, 0xa3, 0x1e, 0x1c,
|
||||||
|
- 0x08, 0x84, 0x93, 0x8d, 0x96, 0xd5, 0x5c, 0xa4, 0x92, 0xf6, 0x20, 0xc8, 0xe4, 0x6e, 0x99, 0x4a,
|
||||||
|
- 0x34, 0x09, 0xb9, 0x23, 0xd3, 0xb9, 0x11, 0xb9, 0x74, 0x1f, 0x44, 0x4d, 0x47, 0xd0, 0x39, 0x4f,
|
||||||
|
- 0x27, 0x15, 0xf3, 0x62, 0x2f, 0xe9, 0x8c, 0xa2, 0xf3, 0x54, 0xae, 0xc2, 0x2f, 0x2f, 0xd1, 0x08,
|
||||||
|
- 0xbc, 0x5c, 0x6f, 0x59, 0x33, 0x26, 0x49, 0x93, 0x1b, 0x69, 0x1c, 0x3f, 0x6b, 0x73, 0x81, 0xf9,
|
||||||
|
- 0xd6, 0xd1, 0x92, 0x49, 0x51, 0xa6, 0x4b, 0x2c, 0x04, 0x36, 0x85, 0x43, 0x33, 0x8b, 0xf1, 0x60,
|
||||||
|
- 0x2d, 0x3b, 0x8b, 0xd1, 0xf4, 0x1e, 0xc2, 0x4a, 0xd4, 0x1f, 0xf3, 0xb5, 0x58, 0x28, 0xd6, 0x8e,
|
||||||
|
- 0x49, 0xd2, 0xe5, 0xed, 0x4a, 0xd4, 0x63, 0xc3, 0x83, 0x2f, 0xf0, 0x79, 0xb1, 0xd5, 0x98, 0x22,
|
||||||
|
- 0x93, 0x4a, 0xbb, 0x6c, 0xa8, 0x8d, 0xcf, 0x42, 0x68, 0x59, 0x8b, 0xfd, 0x69, 0x5b, 0x0e, 0x2f,
|
||||||
|
- 0x76, 0xe1, 0xfd, 0xd9, 0x45, 0x0f, 0x02, 0x55, 0x6c, 0xab, 0x54, 0x62, 0x8c, 0x90, 0x3b, 0xa2,
|
||||||
|
- 0x37, 0xe0, 0xab, 0xb4, 0x28, 0x25, 0x06, 0xe9, 0x72, 0x0b, 0x8f, 0x77, 0xd0, 0x3e, 0xbd, 0x10,
|
||||||
|
- 0x0d, 0xa0, 0xb1, 0x7b, 0x89, 0xfe, 0xe1, 0xff, 0x6b, 0x44, 0xde, 0xff, 0x7f, 0x1f, 0xfb, 0xe4,
|
||||||
|
- 0xe7, 0xd8, 0x27, 0x87, 0x63, 0x9f, 0xcc, 0x02, 0x7c, 0xfb, 0xe7, 0xdf, 0x00, 0x00, 0x00, 0xff,
|
||||||
|
- 0xff, 0xb5, 0x52, 0x37, 0xf2, 0x14, 0x02, 0x00, 0x00,
|
||||||
|
+ // 356 bytes of a gzipped FileDescriptorProto
|
||||||
|
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x92, 0xc1, 0x4e, 0xc2, 0x40,
|
||||||
|
+ 0x10, 0x86, 0x5d, 0x4a, 0x0b, 0x1d, 0x44, 0x9b, 0x8d, 0x92, 0x8d, 0x26, 0xa4, 0xe1, 0x62, 0xe3,
|
||||||
|
+ 0x01, 0x13, 0x34, 0xde, 0xf5, 0x40, 0xc2, 0x0d, 0xf7, 0x05, 0xcc, 0xd2, 0x2e, 0x48, 0xa0, 0xb4,
|
||||||
|
+ 0xe9, 0xb6, 0x34, 0xc4, 0x17, 0xf4, 0xe8, 0x23, 0x18, 0x9e, 0xc2, 0xa3, 0xd9, 0xd9, 0x85, 0xe0,
|
||||||
|
+ 0x05, 0xfe, 0x6f, 0x67, 0xa7, 0xff, 0xfc, 0xd3, 0xc2, 0x75, 0xbe, 0x5a, 0x3c, 0x94, 0xbb, 0x5c,
|
||||||
|
+ 0x2a, 0xf3, 0x3b, 0xcc, 0x8b, 0xac, 0xcc, 0xa8, 0x8b, 0x30, 0x98, 0x81, 0x3f, 0x99, 0xbe, 0x24,
|
||||||
|
+ 0x49, 0x21, 0x95, 0xa2, 0x77, 0xe0, 0xcd, 0x45, 0xba, 0x5c, 0xef, 0x18, 0x09, 0x49, 0x74, 0x31,
|
||||||
|
+ 0xba, 0x1c, 0x9a, 0x8e, 0xc9, 0x74, 0x8c, 0xc7, 0xdc, 0x96, 0x29, 0x83, 0x96, 0x30, 0x3d, 0xac,
|
||||||
|
+ 0x11, 0x92, 0xc8, 0xe7, 0x07, 0xa4, 0x14, 0x9a, 0xa9, 0x50, 0x2b, 0xe6, 0xe0, 0x31, 0xea, 0xc1,
|
||||||
|
+ 0x2f, 0x01, 0x7f, 0xb2, 0x29, 0x65, 0x31, 0x17, 0xb1, 0xa4, 0x3d, 0xf0, 0x12, 0xb9, 0x5d, 0xc6,
|
||||||
|
+ 0x12, 0x4d, 0x7c, 0x6e, 0x49, 0x77, 0x6e, 0x44, 0x2a, 0xed, 0x03, 0x51, 0xd3, 0x11, 0x74, 0x8e,
|
||||||
|
+ 0xd3, 0x49, 0xc5, 0x9c, 0xd0, 0x89, 0x3a, 0xa3, 0xe0, 0x38, 0x95, 0xad, 0xf0, 0xd3, 0x4b, 0x34,
|
||||||
|
+ 0x00, 0x27, 0x2d, 0x2b, 0xd6, 0x0c, 0x49, 0xd4, 0xe4, 0x5a, 0x6a, 0xc7, 0x8f, 0x5a, 0x5f, 0x60,
|
||||||
|
+ 0xae, 0x71, 0x34, 0xa4, 0x53, 0xe4, 0xf1, 0x12, 0x0b, 0x9e, 0x49, 0x61, 0x51, 0xcf, 0xa2, 0x3d,
|
||||||
|
+ 0x58, 0xcb, 0xcc, 0xa2, 0x35, 0xbd, 0x05, 0xbf, 0x10, 0xf5, 0xfb, 0x7c, 0x2d, 0x16, 0x8a, 0xb5,
|
||||||
|
+ 0x43, 0x12, 0x75, 0x79, 0xbb, 0x10, 0xf5, 0x58, 0xb3, 0xb6, 0x78, 0xab, 0x64, 0x25, 0x15, 0xf3,
|
||||||
|
+ 0xb1, 0x62, 0x69, 0xf0, 0x09, 0x2e, 0xcf, 0xaa, 0x12, 0xd3, 0x25, 0x52, 0x95, 0x36, 0x33, 0x6a,
|
||||||
|
+ 0xed, 0xbf, 0x10, 0xa5, 0xac, 0xc5, 0xee, 0xb0, 0x45, 0x8b, 0x27, 0x3b, 0x72, 0xfe, 0xed, 0xa8,
|
||||||
|
+ 0x07, 0x9e, 0xca, 0xaa, 0x22, 0x96, 0x18, 0xcf, 0xe7, 0x96, 0xe8, 0x15, 0xb8, 0x2a, 0xce, 0x72,
|
||||||
|
+ 0x89, 0x01, 0xbb, 0xdc, 0xc0, 0xfd, 0x0d, 0xb4, 0x0f, 0x6f, 0x8e, 0x7a, 0xd0, 0xd8, 0x3e, 0x05,
|
||||||
|
+ 0x67, 0xf8, 0xff, 0x1c, 0x90, 0xd7, 0xf3, 0xaf, 0x7d, 0x9f, 0x7c, 0xef, 0xfb, 0xe4, 0x67, 0xdf,
|
||||||
|
+ 0x27, 0x33, 0x0f, 0xbf, 0x89, 0xc7, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa5, 0x57, 0x70, 0xc8,
|
||||||
|
+ 0x2c, 0x02, 0x00, 0x00,
|
||||||
|
}
|
||||||
|
diff --git a/pkg/types/types.proto b/pkg/types/types.proto
|
||||||
|
index f6856e1..149df13 100644
|
||||||
|
--- a/pkg/types/types.proto
|
||||||
|
+++ b/pkg/types/types.proto
|
||||||
|
@@ -37,6 +37,7 @@ message Interface {
|
||||||
|
// list: "veth", "macvtap", "vlan", "macvlan", "tap", ...
|
||||||
|
string type = 7;
|
||||||
|
uint32 raw_flags = 8;
|
||||||
|
+ uint32 Queues = 9;
|
||||||
|
}
|
||||||
|
|
||||||
|
message Route {
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,56 @@
|
|||||||
|
From 9e1478d7989fea4ee759cc13009d8de07dac2879 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Wed, 19 Aug 2020 17:01:00 +0800
|
||||||
|
Subject: [PATCH 14/16] agent: fix init hugepages failed problem
|
||||||
|
|
||||||
|
reason: fix init hugepages failed problem
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
.../runc/libcontainer/cgroups/fs/apply_raw.go | 20 ++++++++++++++++++++
|
||||||
|
1 file changed, 20 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go b/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go
|
||||||
|
index ec148b4..47aa3c3 100644
|
||||||
|
--- a/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go
|
||||||
|
+++ b/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go
|
||||||
|
@@ -8,11 +8,13 @@ import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sync"
|
||||||
|
+ "time"
|
||||||
|
|
||||||
|
"github.com/opencontainers/runc/libcontainer/cgroups"
|
||||||
|
"github.com/opencontainers/runc/libcontainer/configs"
|
||||||
|
libcontainerUtils "github.com/opencontainers/runc/libcontainer/utils"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
+ "github.com/sirupsen/logrus"
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
)
|
||||||
|
|
||||||
|
@@ -409,3 +411,21 @@ func CheckCpushares(path string, c uint64) error {
|
||||||
|
func (m *Manager) GetCgroups() (*configs.Cgroup, error) {
|
||||||
|
return m.Cgroups, nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func init() {
|
||||||
|
+ go func() {
|
||||||
|
+ var err error
|
||||||
|
+ if len(HugePageSizes) != 0 {
|
||||||
|
+ return
|
||||||
|
+ }
|
||||||
|
+ for i := 0; i < 10; i++ {
|
||||||
|
+ HugePageSizes, err = cgroups.GetHugePageSize()
|
||||||
|
+ if err == nil {
|
||||||
|
+ logrus.Infof("init hugepages ok loop=%d %v", i, err)
|
||||||
|
+ return
|
||||||
|
+ }
|
||||||
|
+ logrus.Errorf("init hugepages failed loop=%d %v", i, err)
|
||||||
|
+ time.Sleep(time.Second)
|
||||||
|
+ }
|
||||||
|
+ }()
|
||||||
|
+}
|
||||||
|
\ No newline at end of file
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,268 @@
|
|||||||
|
From 6120525f81701424e97d453d515d38f14bbe99d9 Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Wed, 19 Aug 2020 20:25:00 +0800
|
||||||
|
Subject: [PATCH 16/16] clock: synchronizes clock info with proxy
|
||||||
|
|
||||||
|
reason: virtual machine's clock may be incorrect, proxy synchronizes
|
||||||
|
clock info to help virtual machine adjust clock time
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
pkg/clock/clock_util.go | 47 ++++++++++++++++++++++++
|
||||||
|
sync_clock_server.go | 94 +++++++++++++++++++++++++++++++++++++++++++++++
|
||||||
|
sync_clock_server_test.go | 88 ++++++++++++++++++++++++++++++++++++++++++++
|
||||||
|
3 files changed, 229 insertions(+)
|
||||||
|
create mode 100644 pkg/clock/clock_util.go
|
||||||
|
create mode 100644 sync_clock_server.go
|
||||||
|
create mode 100644 sync_clock_server_test.go
|
||||||
|
|
||||||
|
diff --git a/pkg/clock/clock_util.go b/pkg/clock/clock_util.go
|
||||||
|
new file mode 100644
|
||||||
|
index 0000000..4b78c63
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/pkg/clock/clock_util.go
|
||||||
|
@@ -0,0 +1,47 @@
|
||||||
|
+// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved.
|
||||||
|
+// SPDX-License-Identifier: Apache-2.0
|
||||||
|
+// Description: sync clock related function
|
||||||
|
+// Author: xueshaojia
|
||||||
|
+// Create: 2018-11-10
|
||||||
|
+
|
||||||
|
+// Package clock provides clock sync functions
|
||||||
|
+package clock
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "net"
|
||||||
|
+ "syscall"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+type TimeValue struct {
|
||||||
|
+ ClientSendTime int64 `json:"client_send_time"`
|
||||||
|
+ ClientArriveTime int64 `json:"client_arrive_time"`
|
||||||
|
+ ServerSendTime int64 `json:"server_send_time"`
|
||||||
|
+ ServerArriveTime int64 `json:"server_arrive_time"`
|
||||||
|
+ Delta int64 `json:"delta"`
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+const MaxSyncClockByteNum = 400 // sync clock byte num, max=400
|
||||||
|
+const MaxTimeStampSupport = 3155731199000000000 // unit is ns, 2069/12/31 23:59:59
|
||||||
|
+const MinTimeStampSupport = 0 // unit is ns, 1970/1/1 8:0:0
|
||||||
|
+
|
||||||
|
+// getCurrentTimeNs returns UTC time in Ns
|
||||||
|
+func GetCurrentTimeNs() int64 {
|
||||||
|
+ var tv syscall.Timeval
|
||||||
|
+ if err := syscall.Gettimeofday(&tv); err != nil {
|
||||||
|
+ return -1
|
||||||
|
+ }
|
||||||
|
+ return tv.Sec*1000000000 + tv.Usec*1000
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// readConnData reads data from stream
|
||||||
|
+func ReadConnData(stream net.Conn) (buf []byte, byteNum int, err error) {
|
||||||
|
+ buf = make([]byte, MaxSyncClockByteNum)
|
||||||
|
+ byteNum, err = stream.Read(buf)
|
||||||
|
+ return buf, byteNum, err
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// writeConnData writes data to stream
|
||||||
|
+func WriteConnData(stream net.Conn, buf []byte) error {
|
||||||
|
+ _, err := stream.Write(buf)
|
||||||
|
+ return err
|
||||||
|
+}
|
||||||
|
diff --git a/sync_clock_server.go b/sync_clock_server.go
|
||||||
|
new file mode 100644
|
||||||
|
index 0000000..a8e98c6
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/sync_clock_server.go
|
||||||
|
@@ -0,0 +1,94 @@
|
||||||
|
+// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved.
|
||||||
|
+// SPDX-License-Identifier: Apache-2.0
|
||||||
|
+// Description: sync clock server related function
|
||||||
|
+// Author: xueshaojia
|
||||||
|
+// Create: 2018-11-10
|
||||||
|
+
|
||||||
|
+package main
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "encoding/json"
|
||||||
|
+ "fmt"
|
||||||
|
+ "io"
|
||||||
|
+ "net"
|
||||||
|
+ "os/exec"
|
||||||
|
+ "syscall"
|
||||||
|
+
|
||||||
|
+ "github.com/kata-containers/agent/pkg/clock"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+// getSyncClockInfo waits for client request
|
||||||
|
+func getSyncClockInfo(stream net.Conn, clockInfo *clock.TimeValue) error {
|
||||||
|
+ buf, byteNum, err := clock.ReadConnData(stream)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ return json.Unmarshal(buf[:byteNum], clockInfo)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// ackSyncClockInfo sends clock info to client
|
||||||
|
+func ackSyncClockInfo(stream net.Conn, clockInfo *clock.TimeValue) error {
|
||||||
|
+ b, err := json.Marshal(clockInfo)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ return clock.WriteConnData(stream, b)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// setGuestClock sets systemtime and hwclock
|
||||||
|
+func setGuestClock(delta int64) error {
|
||||||
|
+ dstClockNs := clock.GetCurrentTimeNs() + delta
|
||||||
|
+ if dstClockNs < clock.MinTimeStampSupport || dstClockNs > clock.MaxTimeStampSupport {
|
||||||
|
+ return fmt.Errorf("sync clock failed, the valid timestamp is from 1970/1/1 8:0:0 to 2069/12/31 23:59:59")
|
||||||
|
+ }
|
||||||
|
+ dstTimeVal := syscall.NsecToTimeval(dstClockNs)
|
||||||
|
+ if err := syscall.Settimeofday(&dstTimeVal); err != nil {
|
||||||
|
+ agentLog.WithError(err).Error("sync clock, set systemtime failed")
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ cmd := exec.Command("hwclock", "-w")
|
||||||
|
+ if err := cmd.Run(); err != nil {
|
||||||
|
+ agentLog.WithError(err).Error("sync clock, set hwclock failed")
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ agentLog.Infof("sync clock, set systemtime and hwclock succ, delta:%d", delta)
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// SyncClock is a loop that deals with client's request
|
||||||
|
+// 1 read client request
|
||||||
|
+// 2 response with clock info or set guest clock
|
||||||
|
+// 3 if error happends, print error info
|
||||||
|
+func SyncClock(stream net.Conn) {
|
||||||
|
+ for {
|
||||||
|
+ var clockInfo clock.TimeValue
|
||||||
|
+ if err := getSyncClockInfo(stream, &clockInfo); err != nil {
|
||||||
|
+ if err == io.EOF {
|
||||||
|
+ agentLog.WithError(err).Error("sync clock, yamux session error happends")
|
||||||
|
+ stream.Close()
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+ agentLog.WithError(err).Error("sync clock, read sync clock info failed")
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+ // set client's request arrive time
|
||||||
|
+ clockInfo.ClientArriveTime = clock.GetCurrentTimeNs()
|
||||||
|
+ if clockInfo.Delta == 0 {
|
||||||
|
+ // this is a request for clock diff info
|
||||||
|
+ // set server's response send time
|
||||||
|
+ // if fails, wait for next time
|
||||||
|
+ clockInfo.ServerSendTime = clock.GetCurrentTimeNs()
|
||||||
|
+ if err := ackSyncClockInfo(stream, &clockInfo); err != nil {
|
||||||
|
+ agentLog.WithError(err).Error("sync clock, send back clock info failed")
|
||||||
|
+ }
|
||||||
|
+ } else {
|
||||||
|
+ // this is a request for adjusting guest clock
|
||||||
|
+ // 1 adjust guest clock(system time and hwclock)
|
||||||
|
+ // 2 do not send response any more
|
||||||
|
+ // 3 if fails, wait for next request to adjust
|
||||||
|
+ if err := setGuestClock(clockInfo.Delta); err != nil {
|
||||||
|
+ agentLog.WithError(err).Error("sync clock, set local clock failed")
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
diff --git a/sync_clock_server_test.go b/sync_clock_server_test.go
|
||||||
|
new file mode 100644
|
||||||
|
index 0000000..41d5227
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/sync_clock_server_test.go
|
||||||
|
@@ -0,0 +1,88 @@
|
||||||
|
+// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved.
|
||||||
|
+// SPDX-License-Identifier: Apache-2.0
|
||||||
|
+// Description: sync clock server related test
|
||||||
|
+// Author: xueshaojia
|
||||||
|
+// Create: 2018-11-10
|
||||||
|
+
|
||||||
|
+package main
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "encoding/json"
|
||||||
|
+ "fmt"
|
||||||
|
+ "math/rand"
|
||||||
|
+ "net"
|
||||||
|
+ "testing"
|
||||||
|
+
|
||||||
|
+ "github.com/hashicorp/yamux"
|
||||||
|
+ "github.com/kata-containers/agent/pkg/clock"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+var clientStream net.Conn
|
||||||
|
+
|
||||||
|
+func TestGetCurrentTimeNs(t *testing.T) {
|
||||||
|
+ timeRet := clock.GetCurrentTimeNs()
|
||||||
|
+ if timeRet <= 0 {
|
||||||
|
+ t.Fatalf("TestGetCurrentTimeNs Fail, time:%d", timeRet)
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestSetGuestClock(t *testing.T) {
|
||||||
|
+ if err := setGuestClock(0); err != nil {
|
||||||
|
+ t.Fatalf("test set clock failed, err:%v", err)
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestReadWriteData(t *testing.T) {
|
||||||
|
+ sock := GenSocket()
|
||||||
|
+ var err error
|
||||||
|
+ listener, err := net.Listen("unix", sock)
|
||||||
|
+ if err != nil {
|
||||||
|
+ t.Fatalf("listening err: %v", err)
|
||||||
|
+ }
|
||||||
|
+ go SetUpClient(sock)
|
||||||
|
+ conn, err := listener.Accept()
|
||||||
|
+ if err != nil {
|
||||||
|
+ t.Fatalf("accept err: %v", err)
|
||||||
|
+ }
|
||||||
|
+ session, err := yamux.Server(conn, nil)
|
||||||
|
+ if err != nil {
|
||||||
|
+ t.Fatalf("session err: %v", err)
|
||||||
|
+ }
|
||||||
|
+ stream, err := session.Accept()
|
||||||
|
+ if err != nil {
|
||||||
|
+ t.Fatalf("stream err: %v", err)
|
||||||
|
+ }
|
||||||
|
+ go func() {
|
||||||
|
+ var clockInfo clock.TimeValue = clock.TimeValue{1000, 0, 0, 0, 0}
|
||||||
|
+ b, _ := json.Marshal(clockInfo)
|
||||||
|
+ clock.WriteConnData(clientStream, b)
|
||||||
|
+ }()
|
||||||
|
+ _, _, err = clock.ReadConnData(stream)
|
||||||
|
+ if err != nil {
|
||||||
|
+ t.Fatalf("read conn data error, err:%v", err)
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func SetUpClient(sock string) error {
|
||||||
|
+ conn, err := net.Dial("unix", sock)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ session, err := yamux.Client(conn, nil)
|
||||||
|
+ if err != nil {
|
||||||
|
+ conn.Close()
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ stream, err := session.Open()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ clientStream = stream
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func GenSocket() string {
|
||||||
|
+ randSeed := clock.GetCurrentTimeNs()
|
||||||
|
+ rand.Seed(randSeed)
|
||||||
|
+ return fmt.Sprintf("/tmp/%d.sock", rand.Uint32())
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
1005
agent/patches/0017-agent-add-support-of-new-sandbox-StratoVirt.patch
Normal file
1005
agent/patches/0017-agent-add-support-of-new-sandbox-StratoVirt.patch
Normal file
File diff suppressed because it is too large
Load Diff
184
agent/patches/0018-kata-agent-update-nic-in-guest.patch
Normal file
184
agent/patches/0018-kata-agent-update-nic-in-guest.patch
Normal file
@ -0,0 +1,184 @@
|
|||||||
|
From 629aac1078bdeed63214c58d2110fe672d654774 Mon Sep 17 00:00:00 2001
|
||||||
|
From: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
Date: Mon, 26 Oct 2020 20:39:05 +0800
|
||||||
|
Subject: [PATCH] kata-agent: update nic in guest
|
||||||
|
|
||||||
|
reason: add linkByName and support retry of list ip link, because in some scenarios,
|
||||||
|
the hardware address cannot be queried immediately after the hot plug.
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
network.go | 91 ++++++++++++++++++++++++++++++++++++++++++------------
|
||||||
|
1 file changed, 72 insertions(+), 19 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/network.go b/network.go
|
||||||
|
index f6e2c17..d928e2b 100644
|
||||||
|
--- a/network.go
|
||||||
|
+++ b/network.go
|
||||||
|
@@ -15,13 +15,14 @@ import (
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"syscall"
|
||||||
|
+ "time"
|
||||||
|
|
||||||
|
- "golang.org/x/sys/unix"
|
||||||
|
agentNet "github.com/kata-containers/agent/pkg/net"
|
||||||
|
"github.com/kata-containers/agent/pkg/types"
|
||||||
|
pb "github.com/kata-containers/agent/protocols/grpc"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
"github.com/vishvananda/netlink"
|
||||||
|
+ "golang.org/x/sys/unix"
|
||||||
|
"google.golang.org/grpc/codes"
|
||||||
|
grpcStatus "google.golang.org/grpc/status"
|
||||||
|
)
|
||||||
|
@@ -32,6 +33,7 @@ var (
|
||||||
|
errNoLink = grpcStatus.Errorf(codes.InvalidArgument, "Need network link")
|
||||||
|
errNoMAC = grpcStatus.Errorf(codes.InvalidArgument, "Need hardware address")
|
||||||
|
errNoRoutes = grpcStatus.Errorf(codes.InvalidArgument, "Need network routes")
|
||||||
|
+ errNoName = grpcStatus.Errorf(codes.InvalidArgument, "Need network name")
|
||||||
|
guestDNSFile = "/etc/resolv.conf"
|
||||||
|
kataGuestSandboxDNSFile = "/run/kata-containers/sandbox/resolv.conf"
|
||||||
|
)
|
||||||
|
@@ -46,6 +48,8 @@ const (
|
||||||
|
|
||||||
|
// Use the below address for ipv6 gateway once ipv6 support is added
|
||||||
|
// defaultV6RouteIP = "::"
|
||||||
|
+
|
||||||
|
+ maxLinkRetries = 10
|
||||||
|
)
|
||||||
|
|
||||||
|
// Network fully describes a sandbox network with its interfaces, routes and dns
|
||||||
|
@@ -100,17 +104,36 @@ func linkByHwAddr(netHandle *netlink.Handle, hwAddr string) (netlink.Link, error
|
||||||
|
return nil, grpcStatus.Errorf(codes.NotFound, "Could not find the link corresponding to HwAddr %q", hwAddr)
|
||||||
|
}
|
||||||
|
|
||||||
|
-func updateLink(netHandle *netlink.Handle, link netlink.Link, iface *types.Interface) error {
|
||||||
|
+func linkByName(netHandle *netlink.Handle, name string) (netlink.Link, error) {
|
||||||
|
if netHandle == nil {
|
||||||
|
- return errNoHandle
|
||||||
|
+ return nil, errNoHandle
|
||||||
|
}
|
||||||
|
|
||||||
|
- if link == nil {
|
||||||
|
- return errNoLink
|
||||||
|
+ if name == "" {
|
||||||
|
+ return nil, errNoName
|
||||||
|
}
|
||||||
|
|
||||||
|
- if iface == nil {
|
||||||
|
- return errNoIF
|
||||||
|
+ links, err := netHandle.LinkList()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ for _, link := range links {
|
||||||
|
+ if link == nil {
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if link.Attrs() != nil && link.Attrs().Name == name {
|
||||||
|
+ return link, nil
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil, grpcStatus.Errorf(codes.NotFound, "Could not find the link corresponding to name %s", name)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func updateLinkIP(netHandle *netlink.Handle, link netlink.Link, iface *types.Interface) error {
|
||||||
|
+ if len(iface.IPAddresses) == 0 {
|
||||||
|
+ return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// As a first step, clear out any existing addresses associated with the link:
|
||||||
|
@@ -129,13 +152,6 @@ func updateLink(netHandle *netlink.Handle, link netlink.Link, iface *types.Inter
|
||||||
|
netlinkAddrStr := fmt.Sprintf("%s/%s", addr.Address, addr.Mask)
|
||||||
|
netlinkAddr, err := netlink.ParseAddr(netlinkAddrStr)
|
||||||
|
|
||||||
|
- // With ipv6 addresses, there is a brief period during which the address is marked as "tentative"
|
||||||
|
- // making it unavailable. A process called duplicate address detection(DAD) is performed during this period.
|
||||||
|
- // Disble DAD so that networking is available once the container is up. The assumption is
|
||||||
|
- // that it is the reponsibility of the upper stack to make sure the addresses assigned to containers
|
||||||
|
- // do not conflict. A similar operation is performed by libnetwork:
|
||||||
|
- // https://github.com/moby/moby/issues/18871
|
||||||
|
-
|
||||||
|
if addr.GetFamily() == types.IPFamily_v6 {
|
||||||
|
netlinkAddr.Flags = netlinkAddr.Flags | syscall.IFA_F_NODAD
|
||||||
|
}
|
||||||
|
@@ -150,14 +166,36 @@ func updateLink(netHandle *netlink.Handle, link netlink.Link, iface *types.Inter
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func updateLink(netHandle *netlink.Handle, link netlink.Link, iface *types.Interface) error {
|
||||||
|
+ if netHandle == nil {
|
||||||
|
+ return errNoHandle
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if link == nil {
|
||||||
|
+ return errNoLink
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if iface == nil {
|
||||||
|
+ return errNoIF
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := updateLinkIP(netHandle, link, iface); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
// set the interface name:
|
||||||
|
if err := netHandle.LinkSetName(link, iface.Name); err != nil {
|
||||||
|
return grpcStatus.Errorf(codes.Internal, "Could not set name %s for interface %v: %v", iface.Name, link, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// set the interface MTU:
|
||||||
|
- if err := netHandle.LinkSetMTU(link, int(iface.Mtu)); err != nil {
|
||||||
|
- return grpcStatus.Errorf(codes.Internal, "Could not set MTU %d for interface %v: %v", iface.Mtu, link, err)
|
||||||
|
+ if iface.Mtu > 0 {
|
||||||
|
+ if err := netHandle.LinkSetMTU(link, int(iface.Mtu)); err != nil {
|
||||||
|
+ return grpcStatus.Errorf(codes.Internal, "Could not set MTU %d for interface %v: %v", iface.Mtu, link, err)
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
|
||||||
|
if iface.RawFlags&unix.IFF_NOARP == uint32(unix.IFF_NOARP) {
|
||||||
|
@@ -306,8 +344,23 @@ func (s *sandbox) updateInterface(netHandle *netlink.Handle, iface *types.Interf
|
||||||
|
if iface.HwAddr != "" {
|
||||||
|
fieldLogger.Info("Getting interface from MAC address")
|
||||||
|
|
||||||
|
- // Find the interface link from its hardware address.
|
||||||
|
- link, err = linkByHwAddr(netHandle, iface.HwAddr)
|
||||||
|
+ // In some scenarios, the hardware address cannot be queried immediately
|
||||||
|
+ // after the hot plug, add a retry here.
|
||||||
|
+ for retry := 0; retry < maxLinkRetries; retry++ {
|
||||||
|
+ // Find the interface link from its hardware address.
|
||||||
|
+ link, err = linkByHwAddr(netHandle, iface.HwAddr)
|
||||||
|
+ if err != nil {
|
||||||
|
+ time.Sleep(100 * time.Millisecond)
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, grpcStatus.Errorf(codes.Internal, "updateInterface: %v", err)
|
||||||
|
+ }
|
||||||
|
+ } else if iface.Name != "" {
|
||||||
|
+ fieldLogger.Info("Getting interface from name")
|
||||||
|
+ link, err = linkByName(netHandle, iface.Name)
|
||||||
|
if err != nil {
|
||||||
|
return nil, grpcStatus.Errorf(codes.Internal, "updateInterface: %v", err)
|
||||||
|
}
|
||||||
|
@@ -487,7 +540,7 @@ func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Ro
|
||||||
|
}()
|
||||||
|
|
||||||
|
var (
|
||||||
|
- added []*types.Route
|
||||||
|
+ added []*types.Route
|
||||||
|
removed []*types.Route
|
||||||
|
)
|
||||||
|
|
||||||
|
--
|
||||||
|
2.23.0
|
||||||
|
|
||||||
29
agent/patches/0019-kata-agent-modify-make-flags.patch
Normal file
29
agent/patches/0019-kata-agent-modify-make-flags.patch
Normal file
@ -0,0 +1,29 @@
|
|||||||
|
From 1c7aaafa7b8691ea6ed6c910455567b36bb6f5ff Mon Sep 17 00:00:00 2001
|
||||||
|
From: jikui <jikui2@huawei.com>
|
||||||
|
Date: Thu, 18 Mar 2021 15:25:49 +0800
|
||||||
|
Subject: [PATCH] kata-agent: modify make flags
|
||||||
|
|
||||||
|
reason: modify make flags
|
||||||
|
|
||||||
|
Signed-off-by: jikui <jikui2@huawei.com>
|
||||||
|
---
|
||||||
|
Makefile | 3 +++
|
||||||
|
1 file changed, 3 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/Makefile b/Makefile
|
||||||
|
index e4fd243..26fe898 100644
|
||||||
|
--- a/Makefile
|
||||||
|
+++ b/Makefile
|
||||||
|
@@ -101,6 +101,9 @@ AGENT_IMAGE := katacontainers/agent-dev
|
||||||
|
AGENT_TAG := $(if $(COMMIT_NO_SHORT),$(COMMIT_NO_SHORT),dev)
|
||||||
|
|
||||||
|
$(TARGET): $(GENERATED_FILES) $(SOURCES) $(VERSION_FILE)
|
||||||
|
+ CGO_CFLAGS="-fstack-protector-strong -fPIE -D_FORTIFY_SOURCE=2 -O2" \
|
||||||
|
+ CGO_LDFLAGS_ALLOW="-Wl,-z,relro,-z,now" \
|
||||||
|
+ CGO_LDFLAGS="-Wl,-z,relro,-z,now -Wl,-z,noexecstack" \
|
||||||
|
go build $(BUILDFLAGS) -tags "$(BUILDTAGS)" -o $@ \
|
||||||
|
-ldflags "-X main.version=$(VERSION_COMMIT) -X main.seccompSupport=$(SECCOMP) $(LDFLAGS) $(KATA_LDFLAGS)"
|
||||||
|
|
||||||
|
--
|
||||||
|
2.25.1
|
||||||
|
|
||||||
@ -0,0 +1,37 @@
|
|||||||
|
From d98995f25c3a839f25590478bef37d2a456593a3 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jikui <jikui2@huawei.com>
|
||||||
|
Date: Mon, 22 Mar 2021 17:07:37 +0800
|
||||||
|
Subject: [PATCH] kata-agent: add linkmode to resolve build error
|
||||||
|
|
||||||
|
reason: add linkmode to resolve build error
|
||||||
|
|
||||||
|
Signed-off-by: jikui <jikui2@huawei.com>
|
||||||
|
---
|
||||||
|
Makefile | 4 ++--
|
||||||
|
1 file changed, 2 insertions(+), 2 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/Makefile b/Makefile
|
||||||
|
index 26fe898..5401c69 100644
|
||||||
|
--- a/Makefile
|
||||||
|
+++ b/Makefile
|
||||||
|
@@ -16,7 +16,7 @@ INIT := no
|
||||||
|
TRACE := no
|
||||||
|
|
||||||
|
# Set to "yes“ if binary stripping is needed.
|
||||||
|
-STRIP := no
|
||||||
|
+STRIP := yes
|
||||||
|
|
||||||
|
# Tracing cannot currently be supported when running the agent as PID 1 since
|
||||||
|
# the tracing requires additional services to be started _before_ the agent
|
||||||
|
@@ -105,7 +105,7 @@ $(TARGET): $(GENERATED_FILES) $(SOURCES) $(VERSION_FILE)
|
||||||
|
CGO_LDFLAGS_ALLOW="-Wl,-z,relro,-z,now" \
|
||||||
|
CGO_LDFLAGS="-Wl,-z,relro,-z,now -Wl,-z,noexecstack" \
|
||||||
|
go build $(BUILDFLAGS) -tags "$(BUILDTAGS)" -o $@ \
|
||||||
|
- -ldflags "-X main.version=$(VERSION_COMMIT) -X main.seccompSupport=$(SECCOMP) $(LDFLAGS) $(KATA_LDFLAGS)"
|
||||||
|
+ -ldflags "-linkmode=external -X main.version=$(VERSION_COMMIT) -X main.seccompSupport=$(SECCOMP) $(LDFLAGS) $(KATA_LDFLAGS)"
|
||||||
|
|
||||||
|
install: $(TARGET)
|
||||||
|
install -D $(TARGET) $(DESTDIR)$(BINDIR)/$(TARGET)
|
||||||
|
--
|
||||||
|
2.25.1
|
||||||
|
|
||||||
19
agent/series.conf
Normal file
19
agent/series.conf
Normal file
@ -0,0 +1,19 @@
|
|||||||
|
0001-agent-add-agent.netlink_recv_buf_size-flag-to-set-ne.patch
|
||||||
|
0002-network-support-update-routes-incrementally.patch
|
||||||
|
0003-kata-agent-add-kata-ipvs-command.patch
|
||||||
|
0004-agent-add-IPVS-test.patch
|
||||||
|
0005-mount-support-mount-block-device.patch
|
||||||
|
0006-agent-make-workaround-for-slow-response-in-aarch64.patch
|
||||||
|
0007-agent-using-pcie-root-port-driver-to-hotplug-device.patch
|
||||||
|
0008-agent-support-get-root-bus-path-dynamically.patch
|
||||||
|
0009-storage-add-pkg-storage-for-mount.patch
|
||||||
|
0010-storage-mount-nfs-and-gpath-in-agent.patch
|
||||||
|
0012-network-support-set-dns-without-nameserver.patch
|
||||||
|
0013-agent-support-setting-multi-queues-of-interface.patch
|
||||||
|
0014-agent-fix-init-hugepages-failed-problem.patch
|
||||||
|
0015-agent-add-support-of-getting-container-s-network-sta.patch
|
||||||
|
0016-clock-synchronizes-clock-info-with-proxy.patch
|
||||||
|
0017-agent-add-support-of-new-sandbox-StratoVirt.patch
|
||||||
|
0018-kata-agent-update-nic-in-guest.patch
|
||||||
|
0019-kata-agent-modify-make-flags.patch
|
||||||
|
0020-kata-agent-add-linkmode-to-resolve-build-error.patch
|
||||||
@ -1,8 +1,8 @@
|
|||||||
#needsrootforbuild
|
#needsrootforbuild
|
||||||
%global debug_package %{nil}
|
%global debug_package %{nil}
|
||||||
|
|
||||||
%define VERSION 2.1.0
|
%define VERSION 1.11.1
|
||||||
%define RELEASE 9
|
%define RELEASE 18
|
||||||
|
|
||||||
Name: kata-containers
|
Name: kata-containers
|
||||||
Version: %{VERSION}
|
Version: %{VERSION}
|
||||||
@ -11,12 +11,12 @@ Summary: Kata Container, the speed of containers, the security of VMs
|
|||||||
License: Apache 2.0
|
License: Apache 2.0
|
||||||
URL: https://github.com/kata-containers
|
URL: https://github.com/kata-containers
|
||||||
Source0: kata_integration-v1.0.0.tar.gz
|
Source0: kata_integration-v1.0.0.tar.gz
|
||||||
Source1: kata-containers-%{version}.tar.gz
|
Source1: kata-containers-v%{version}.tar.gz
|
||||||
Source2: kernel.tar.gz
|
Source2: kernel.tar.gz
|
||||||
|
|
||||||
BuildRoot: %_topdir/BUILDROOT
|
BuildRoot: %_topdir/BUILDROOT
|
||||||
BuildRequires: automake golang gcc bc glibc-devel glibc-static busybox glib2-devel glib2 ipvsadm conntrack-tools nfs-utils
|
BuildRequires: automake golang gcc bc glibc-devel glibc-static busybox glib2-devel glib2 ipvsadm conntrack-tools nfs-utils
|
||||||
BuildRequires: patch elfutils-libelf-devel openssl-devel bison flex rust cargo rust-packaging libgcc dtc-devel
|
BuildRequires: patch elfutils-libelf-devel openssl-devel bison flex
|
||||||
|
|
||||||
%description
|
%description
|
||||||
This is core component of Kata Container, to make it work, you need a isulad/docker engine.
|
This is core component of Kata Container, to make it work, you need a isulad/docker engine.
|
||||||
@ -31,6 +31,14 @@ cd %{_builddir}/kata_integration
|
|||||||
# apply kata_integration patches
|
# apply kata_integration patches
|
||||||
sh apply-patches
|
sh apply-patches
|
||||||
|
|
||||||
|
# mv build components into kata_integration dir
|
||||||
|
pushd %{_builddir}/kata_integration
|
||||||
|
mv ../kata-containers-%{version}/runtime .
|
||||||
|
mv ../kata-containers-%{version}/agent .
|
||||||
|
mv ../kata-containers-%{version}/proxy .
|
||||||
|
mv ../kata-containers-%{version}/shim .
|
||||||
|
popd
|
||||||
|
|
||||||
# build kernel
|
# build kernel
|
||||||
cd %{_builddir}/kernel
|
cd %{_builddir}/kernel
|
||||||
mv kernel linux
|
mv kernel linux
|
||||||
@ -45,31 +53,11 @@ cp %{_builddir}/kata_integration/hack/config-kata-arm64 ./.config
|
|||||||
cd %{_builddir}/kernel/linux/
|
cd %{_builddir}/kernel/linux/
|
||||||
make %{?_smp_mflags}
|
make %{?_smp_mflags}
|
||||||
|
|
||||||
mv %{_builddir}/kata-containers-%{version} %{_builddir}/kata-containers
|
|
||||||
cd %{_builddir}/kata-containers/
|
|
||||||
sh -x apply-patches
|
|
||||||
cd %{_builddir}/kata-containers/src/runtime
|
|
||||||
make clean
|
|
||||||
make
|
|
||||||
|
|
||||||
cd %{_builddir}/kata-containers/src/agent
|
|
||||||
mkdir vendor && tar -xzf %{_builddir}/kata-containers/vendor.tar.gz -C vendor/
|
|
||||||
cp -f ./vendor/version.rs ./src/
|
|
||||||
cat > .cargo/config << EOF
|
|
||||||
[build]
|
|
||||||
rustflags = ["-Clink-arg=-s","-Clink-arg=-lgcc","-Clink-arg=-lfdt"]
|
|
||||||
|
|
||||||
[source.crates-io]
|
|
||||||
replace-with = "vendored-sources"
|
|
||||||
|
|
||||||
[source.vendored-sources]
|
|
||||||
directory = "vendor"
|
|
||||||
EOF
|
|
||||||
/usr/bin/env CARGO_HOME=.cargo RUSTC_BOOTSTRAP=1 cargo build --release
|
|
||||||
cd %{_builddir}/kata_integration
|
cd %{_builddir}/kata_integration
|
||||||
mkdir -p -m 750 build
|
mkdir -p -m 750 build
|
||||||
cp %{_builddir}/kata-containers/src/agent/target/release/kata-agent ./build/
|
make runtime
|
||||||
strip ./build/kata-agent
|
make proxy
|
||||||
|
make shim
|
||||||
make initrd
|
make initrd
|
||||||
|
|
||||||
%install
|
%install
|
||||||
@ -82,87 +70,76 @@ install -p -m 755 -D %{_builddir}/kernel/linux/arch/arm64/boot/Image %{buildroot
|
|||||||
|
|
||||||
cd %{_builddir}/kata_integration
|
cd %{_builddir}/kata_integration
|
||||||
mkdir -p -m 750 %{buildroot}/usr/bin
|
mkdir -p -m 750 %{buildroot}/usr/bin
|
||||||
install -p -m 750 %{_builddir}/kata-containers/src/runtime/kata-runtime %{buildroot}/usr/bin/
|
strip ./build/kata-runtime ./build/kata-proxy ./build/kata-shim ./build/kata-netmon ./build/containerd-shim-kata-v2
|
||||||
install -p -m 750 %{_builddir}/kata-containers/src/runtime/kata-netmon %{buildroot}/usr/bin/
|
install -p -m 750 ./build/kata-runtime ./build/kata-proxy ./build/kata-shim ./build/kata-netmon ./build/containerd-shim-kata-v2 %{buildroot}/usr/bin/
|
||||||
install -p -m 750 %{_builddir}/kata-containers/src/runtime/kata-monitor %{buildroot}/usr/bin/
|
|
||||||
install -p -m 750 %{_builddir}/kata-containers/src/runtime/containerd-shim-kata-v2 %{buildroot}/usr/bin/
|
|
||||||
install -p -m 640 -D %{_builddir}/kata-containers/src/runtime/cli/config/configuration-qemu.toml %{buildroot}/usr/share/defaults/kata-containers/configuration.toml
|
|
||||||
install -p -m 640 -D %{_builddir}/kata-containers/src/runtime/cli/config/configuration-stratovirt.toml %{buildroot}/usr/share/defaults/kata-containers/configuration-stratovirt.toml
|
|
||||||
install -p -m 640 ./build/kata-containers-initrd.img %{buildroot}/var/lib/kata/
|
install -p -m 640 ./build/kata-containers-initrd.img %{buildroot}/var/lib/kata/
|
||||||
mkdir -p -m 750 %{buildroot}/usr/share/defaults/kata-containers/
|
mkdir -p -m 750 %{buildroot}/usr/share/defaults/kata-containers/
|
||||||
strip %{buildroot}/usr/bin/kata*
|
install -p -m 640 -D ./runtime/cli/config/configuration-qemu.toml %{buildroot}/usr/share/defaults/kata-containers/configuration.toml
|
||||||
strip %{buildroot}/usr/bin/containerd-shim-kata-v2
|
|
||||||
|
|
||||||
%clean
|
%clean
|
||||||
|
|
||||||
%files
|
%files
|
||||||
/usr/bin/kata-runtime
|
/usr/bin/kata-runtime
|
||||||
|
/usr/bin/kata-proxy
|
||||||
|
/usr/bin/kata-shim
|
||||||
/usr/bin/kata-netmon
|
/usr/bin/kata-netmon
|
||||||
/usr/bin/kata-monitor
|
|
||||||
/usr/bin/containerd-shim-kata-v2
|
/usr/bin/containerd-shim-kata-v2
|
||||||
/var/lib/kata/kernel
|
/var/lib/kata/kernel
|
||||||
/var/lib/kata/kata-containers-initrd.img
|
/var/lib/kata/kata-containers-initrd.img
|
||||||
%config(noreplace) /usr/share/defaults/kata-containers/configuration.toml
|
%config(noreplace) /usr/share/defaults/kata-containers/configuration.toml
|
||||||
%config(noreplace) /usr/share/defaults/kata-containers/configuration-stratovirt.toml
|
|
||||||
|
|
||||||
%doc
|
%doc
|
||||||
|
|
||||||
%changelog
|
%changelog
|
||||||
* Sat Nov 20 2021 yangfeiyu <yangfeiyu2@huawei.com> - 2.1.0-9
|
* Tue Nov 30 2021 yangfeiyu <yangfeiyu2@huawei.com> - 1.11.1-18
|
||||||
|
- Type:feature
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:bump version to 18
|
||||||
|
|
||||||
|
* Wed June 16 2021 gaohuatao <gaohuatao@huawei.com> - 1.11.1-17
|
||||||
- Type:bugfix
|
- Type:bugfix
|
||||||
- ID:NA
|
- ID:NA
|
||||||
- SUG:NA
|
- SUG:NA
|
||||||
- DESC:modify kernel and image path in configuration.toml
|
- DESC:move timeout of waitProcess to stop process
|
||||||
|
|
||||||
* Tue Oct 16 2021 jikui <jikui2@huawei.com> - 2.1.0-8
|
* Thu June 3 2021 gaohuatao <gaohuatao@huawei.com> - 1.11.1-16
|
||||||
- Type:bugfix
|
- Type:bugfix
|
||||||
- ID:NA
|
- ID:NA
|
||||||
- SUG:NA
|
- SUG:NA
|
||||||
- DESC:keep the qemu process name same as the configured path
|
- DESC:put timeout to client of wait rpc to support shimv2
|
||||||
|
|
||||||
* Mon Oct 15 2021 jikui <jikui2@huawei.com> - 2.1.0-7
|
* Wed May 12 2021 gaohuatao <gaohuatao@huawei.com> - 1.11.1-15
|
||||||
- Type:bugfix
|
- Type:bugfix
|
||||||
- ID:NA
|
- ID:NA
|
||||||
- SUG:NA
|
- SUG:NA
|
||||||
- DESC:fix kata-runtime skip read lines in /proc/mounts
|
- DESC:shimv2 write exit code in integer byte order
|
||||||
|
|
||||||
* Fri Oct 5 2021 jikui <jikui2@huawei.com> - 2.1.0-6
|
* Wed Apr 28 2021 gaohuatao <gaohuatao@huawei.com> - 1.11.1-14
|
||||||
|
- Type:feature
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:kata shimv2 adapt iSulad
|
||||||
|
|
||||||
|
* Tue Mar 23 2021 jikui <jikui2@huawei.com> - 1.11.1-13
|
||||||
- Type:bugfix
|
- Type:bugfix
|
||||||
- ID:NA
|
- ID:NA
|
||||||
- SUG:NA
|
- SUG:NA
|
||||||
- DESC:fix kata-runtime hungs when qemu process is D/T
|
- DESC:remove ctty to resolve build failed
|
||||||
|
|
||||||
* Mon Sep 27 2021 yangfeiyu <yangfeiyu2@huawei.com> - 2.1.0-5
|
* Mon Mar 22 2021 jikui <jikui2@huawei.com> - 1.11.1-12
|
||||||
- Type:enhancement
|
- Type:enhancement
|
||||||
- ID:NA
|
- ID:NA
|
||||||
- SUG:NA
|
- SUG:NA
|
||||||
- DESC:chmod agent exec fifo
|
- DESC:add linkmode to resolve build error
|
||||||
|
|
||||||
* Fri Sep 17 2021 yangfeiyu <yangfeiyu2@huawei.com> - 2.1.0-4
|
* Thu Mar 17 2021 jikui <jikui2@huawei.com> - 1.11.1-11
|
||||||
- Type:enhancement
|
- Type:enhancement
|
||||||
- ID:NA
|
- ID:NA
|
||||||
- SUG:NA
|
- SUG:NA
|
||||||
- DESC:modify kata-agent build flags
|
- DESC:modify make flags
|
||||||
|
|
||||||
* Tue Aug 24 2021 yangfeiyu <yangfeiyu2@huawei.com> - 2.1.0-3
|
* Tue Feb 23 2021 xinghe <xinghe1@huawei.com> - 1.11.1-10
|
||||||
- Type:enhancement
|
|
||||||
- ID:NA
|
|
||||||
- SUG:NA
|
|
||||||
- DESC:add configuration-stratovirt.toml
|
|
||||||
|
|
||||||
* Fri Aug 20 2021 yangfeiyu <yangfeiyu2@huawei.com> - 2.1.0-2
|
|
||||||
- Type:enhancement
|
|
||||||
- ID:NA
|
|
||||||
- SUG:NA
|
|
||||||
- DESC:support with stratovirt and isulad
|
|
||||||
|
|
||||||
* Wed Aug 18 2021 yangfeiyu <yangfeiyu2@huawei.com> - 2.1.0-1
|
|
||||||
- Type:enhancement
|
|
||||||
- ID:NA
|
|
||||||
- SUG:NA
|
|
||||||
- DESC:upgrade kata-containers
|
|
||||||
|
|
||||||
* Fri Feb 19 2021 xinghe <xinghe1@huawei.com> - 1.11.1-10
|
|
||||||
- Type:CVE
|
- Type:CVE
|
||||||
- ID:NA
|
- ID:NA
|
||||||
- SUG:NA
|
- SUG:NA
|
||||||
|
|||||||
@ -1,837 +0,0 @@
|
|||||||
From f56d66f196bee808526e86df2c3c063a887c6fef Mon Sep 17 00:00:00 2001
|
|
||||||
From: Wei Gao <gaowei66@huawei.com>
|
|
||||||
Date: Sat, 7 Aug 2021 10:39:11 +0800
|
|
||||||
Subject: [PATCH 1/6] runtime: add support of new sandbox hypervisor type
|
|
||||||
StratoVirt.
|
|
||||||
|
|
||||||
Signed-off-by: Wei Gao <gaowei66@huawei.com>
|
|
||||||
---
|
|
||||||
src/runtime/pkg/katautils/config.go | 96 +++-
|
|
||||||
src/runtime/virtcontainers/hypervisor.go | 12 +
|
|
||||||
src/runtime/virtcontainers/stratovirt.go | 642 +++++++++++++++++++++++
|
|
||||||
3 files changed, 749 insertions(+), 1 deletion(-)
|
|
||||||
create mode 100644 src/runtime/virtcontainers/stratovirt.go
|
|
||||||
|
|
||||||
diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go
|
|
||||||
index 6114aa39..f94ac4fd 100644
|
|
||||||
--- a/src/runtime/pkg/katautils/config.go
|
|
||||||
+++ b/src/runtime/pkg/katautils/config.go
|
|
||||||
@@ -50,6 +50,7 @@ const (
|
|
||||||
clhHypervisorTableType = "clh"
|
|
||||||
qemuHypervisorTableType = "qemu"
|
|
||||||
acrnHypervisorTableType = "acrn"
|
|
||||||
+ stratovirtHypervisorTable = "stratovirt"
|
|
||||||
|
|
||||||
// the maximum amount of PCI bridges that can be cold plugged in a VM
|
|
||||||
maxPCIBridges uint32 = 5
|
|
||||||
@@ -870,6 +871,96 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
+func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
|
||||||
+ hypervisor, err := h.path()
|
|
||||||
+ if err != nil {
|
|
||||||
+ return vc.HypervisorConfig{}, err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ kernel, err := h.kernel()
|
|
||||||
+ if err != nil {
|
|
||||||
+ return vc.HypervisorConfig{}, err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ initrd, image, err := h.getInitrdAndImage()
|
|
||||||
+ if err != nil {
|
|
||||||
+ return vc.HypervisorConfig{}, err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if image != "" && initrd != "" {
|
|
||||||
+ return vc.HypervisorConfig{},
|
|
||||||
+ errors.New("having both an image and an initrd defined in the configuration file is not supported")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if image == "" && initrd == "" {
|
|
||||||
+ return vc.HypervisorConfig{},
|
|
||||||
+ errors.New("either image or initrd must be defined in the configuration file")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ kernelParams := h.kernelParams()
|
|
||||||
+ machineType := h.machineType()
|
|
||||||
+
|
|
||||||
+ blockDriver, err := h.blockDeviceDriver()
|
|
||||||
+ if err != nil {
|
|
||||||
+ return vc.HypervisorConfig{}, err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ sharedFS, err := h.sharedFS()
|
|
||||||
+ if err != nil {
|
|
||||||
+ return vc.HypervisorConfig{}, err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if sharedFS == config.VirtioFS && h.VirtioFSDaemon == "" {
|
|
||||||
+ return vc.HypervisorConfig{},
|
|
||||||
+ errors.New("cannot enable virtio-fs without daemon path in configuration file")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if vSock, err := utils.SupportsVsocks(); !vSock {
|
|
||||||
+ return vc.HypervisorConfig{}, err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return vc.HypervisorConfig{
|
|
||||||
+ HypervisorPath: hypervisor,
|
|
||||||
+ HypervisorPathList: h.HypervisorPathList,
|
|
||||||
+ KernelPath: kernel,
|
|
||||||
+ InitrdPath: initrd,
|
|
||||||
+ ImagePath: image,
|
|
||||||
+ KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)),
|
|
||||||
+ HypervisorMachineType: machineType,
|
|
||||||
+ NumVCPUs: h.defaultVCPUs(),
|
|
||||||
+ DefaultMaxVCPUs: h.defaultMaxVCPUs(),
|
|
||||||
+ MemorySize: h.defaultMemSz(),
|
|
||||||
+ MemSlots: h.defaultMemSlots(),
|
|
||||||
+ MemOffset: h.defaultMemOffset(),
|
|
||||||
+ EntropySource: h.GetEntropySource(),
|
|
||||||
+ EntropySourceList: h.EntropySourceList,
|
|
||||||
+ DefaultBridges: h.defaultBridges(),
|
|
||||||
+ DisableBlockDeviceUse: h.DisableBlockDeviceUse,
|
|
||||||
+ SharedFS: sharedFS,
|
|
||||||
+ VirtioFSDaemon: h.VirtioFSDaemon,
|
|
||||||
+ VirtioFSDaemonList: h.VirtioFSDaemonList,
|
|
||||||
+ VirtioFSCacheSize: h.VirtioFSCacheSize,
|
|
||||||
+ VirtioFSCache: h.defaultVirtioFSCache(),
|
|
||||||
+ VirtioFSExtraArgs: h.VirtioFSExtraArgs,
|
|
||||||
+ FileBackedMemRootDir: h.FileBackedMemRootDir,
|
|
||||||
+ FileBackedMemRootList: h.FileBackedMemRootList,
|
|
||||||
+ Mlock: !h.Swap,
|
|
||||||
+ Debug: h.Debug,
|
|
||||||
+ DisableNestingChecks: h.DisableNestingChecks,
|
|
||||||
+ BlockDeviceDriver: blockDriver,
|
|
||||||
+ BlockDeviceCacheSet: h.BlockDeviceCacheSet,
|
|
||||||
+ BlockDeviceCacheDirect: h.BlockDeviceCacheDirect,
|
|
||||||
+ BlockDeviceCacheNoflush: h.BlockDeviceCacheNoflush,
|
|
||||||
+ EnableIOThreads: h.EnableIOThreads,
|
|
||||||
+ DisableVhostNet: h.DisableVhostNet,
|
|
||||||
+ EnableVhostUserStore: h.EnableVhostUserStore,
|
|
||||||
+ VhostUserStorePath: h.vhostUserStorePath(),
|
|
||||||
+ VhostUserStorePathList: h.VhostUserStorePathList,
|
|
||||||
+ GuestHookPath: h.guestHookPath(),
|
|
||||||
+ EnableAnnotations: h.EnableAnnotations,
|
|
||||||
+ }, nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
func newFactoryConfig(f factory) (oci.FactoryConfig, error) {
|
|
||||||
if f.TemplatePath == "" {
|
|
||||||
f.TemplatePath = defaultTemplatePath
|
|
||||||
@@ -903,6 +994,9 @@ func updateRuntimeConfigHypervisor(configPath string, tomlConf tomlConfig, confi
|
|
||||||
case clhHypervisorTableType:
|
|
||||||
config.HypervisorType = vc.ClhHypervisor
|
|
||||||
hConfig, err = newClhHypervisorConfig(hypervisor)
|
|
||||||
+ case stratovirtHypervisorTable:
|
|
||||||
+ config.HypervisorType = vc.StratovirtHypervisor
|
|
||||||
+ hConfig, err = newStratovirtHypervisorConfig(hypervisor)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
@@ -1287,7 +1381,7 @@ func checkHypervisorConfig(config vc.HypervisorConfig) error {
|
|
||||||
memSizeMB := int64(config.MemorySize)
|
|
||||||
|
|
||||||
if memSizeMB == 0 {
|
|
||||||
- return errors.New("VM memory cannot be zero")
|
|
||||||
+ return errors.New(fmt.Sprintf("The VM memory cannot be zero, %s", config.ImagePath))
|
|
||||||
}
|
|
||||||
|
|
||||||
mb := int64(1024 * 1024)
|
|
||||||
diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go
|
|
||||||
index 767215b6..615baa80 100644
|
|
||||||
--- a/src/runtime/virtcontainers/hypervisor.go
|
|
||||||
+++ b/src/runtime/virtcontainers/hypervisor.go
|
|
||||||
@@ -44,6 +44,9 @@ const (
|
|
||||||
// ClhHypervisor is the ICH hypervisor.
|
|
||||||
ClhHypervisor HypervisorType = "clh"
|
|
||||||
|
|
||||||
+ // StratovirtHypervisor is the StratoVirt hypervisor
|
|
||||||
+ StratovirtHypervisor HypervisorType = "stratovirt"
|
|
||||||
+
|
|
||||||
// MockHypervisor is a mock hypervisor for testing purposes
|
|
||||||
MockHypervisor HypervisorType = "mock"
|
|
||||||
)
|
|
||||||
@@ -159,6 +162,9 @@ func (hType *HypervisorType) Set(value string) error {
|
|
||||||
case "clh":
|
|
||||||
*hType = ClhHypervisor
|
|
||||||
return nil
|
|
||||||
+ case "stratovirt":
|
|
||||||
+ *hType = StratovirtHypervisor
|
|
||||||
+ return nil
|
|
||||||
case "mock":
|
|
||||||
*hType = MockHypervisor
|
|
||||||
return nil
|
|
||||||
@@ -178,6 +184,8 @@ func (hType *HypervisorType) String() string {
|
|
||||||
return string(AcrnHypervisor)
|
|
||||||
case ClhHypervisor:
|
|
||||||
return string(ClhHypervisor)
|
|
||||||
+ case StratovirtHypervisor:
|
|
||||||
+ return string(StratovirtHypervisor)
|
|
||||||
case MockHypervisor:
|
|
||||||
return string(MockHypervisor)
|
|
||||||
default:
|
|
||||||
@@ -207,6 +215,10 @@ func newHypervisor(hType HypervisorType) (hypervisor, error) {
|
|
||||||
return &cloudHypervisor{
|
|
||||||
store: store,
|
|
||||||
}, nil
|
|
||||||
+ case StratovirtHypervisor:
|
|
||||||
+ return &stratovirt{
|
|
||||||
+ store: store,
|
|
||||||
+ }, nil
|
|
||||||
case MockHypervisor:
|
|
||||||
return &mockHypervisor{}, nil
|
|
||||||
default:
|
|
||||||
diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go
|
|
||||||
new file mode 100644
|
|
||||||
index 00000000..4fec96d3
|
|
||||||
--- /dev/null
|
|
||||||
+++ b/src/runtime/virtcontainers/stratovirt.go
|
|
||||||
@@ -0,0 +1,642 @@
|
|
||||||
+package virtcontainers
|
|
||||||
+
|
|
||||||
+import (
|
|
||||||
+ "context"
|
|
||||||
+ "fmt"
|
|
||||||
+ "os"
|
|
||||||
+ "os/exec"
|
|
||||||
+ "path/filepath"
|
|
||||||
+ "strconv"
|
|
||||||
+ "strings"
|
|
||||||
+ "syscall"
|
|
||||||
+ "time"
|
|
||||||
+
|
|
||||||
+ govmmQemu "github.com/kata-containers/govmm/qemu"
|
|
||||||
+ "github.com/pkg/errors"
|
|
||||||
+ "github.com/sirupsen/logrus"
|
|
||||||
+
|
|
||||||
+ "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
|
|
||||||
+ persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
|
|
||||||
+ "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
|
|
||||||
+ "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
|
|
||||||
+ "go.opentelemetry.io/otel"
|
|
||||||
+ otelLabel "go.opentelemetry.io/otel/label"
|
|
||||||
+ otelTrace "go.opentelemetry.io/otel/trace"
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
+const defaultDummyMac = "22:33:44:aa:bb:"
|
|
||||||
+const mmioBlkCount = 4
|
|
||||||
+const mmioNetCount = 2
|
|
||||||
+const randomDevice = "/dev/urandom"
|
|
||||||
+
|
|
||||||
+type stratovirtDev struct {
|
|
||||||
+ dev interface{}
|
|
||||||
+ devType deviceType
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+type stratovirt struct {
|
|
||||||
+ id string
|
|
||||||
+ ctx context.Context
|
|
||||||
+ sandbox *Sandbox
|
|
||||||
+ store persistapi.PersistDriver
|
|
||||||
+ config HypervisorConfig
|
|
||||||
+ pid int
|
|
||||||
+ consolePath string
|
|
||||||
+ socketPath string
|
|
||||||
+ qmpMonitorCh qmpChannel
|
|
||||||
+ devices []stratovirtDev
|
|
||||||
+ HotpluggedVCPUs []CPUDevice
|
|
||||||
+ mmioBlkSlots [mmioBlkCount]bool
|
|
||||||
+ mmioNetSlots [mmioNetCount]bool
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) Logger() *logrus.Entry {
|
|
||||||
+ return virtLog.WithField("subsystem", "stratovirt")
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) trace(parent context.Context, name string) (otelTrace.Span, context.Context) {
|
|
||||||
+ if parent == nil {
|
|
||||||
+ s.Logger().WithField("type", "bug").Error("trace called before context set")
|
|
||||||
+ parent = context.Background()
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ tracer := otel.Tracer("kata")
|
|
||||||
+ ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(otelLabel.String("source", "runtime"), otelLabel.String("package", "virtcontainers"), otelLabel.String("subsystem", "hypervisor"), otelLabel.String("type", "stratovirt"), otelLabel.String("sandbox_id", s.id)))
|
|
||||||
+
|
|
||||||
+ return span, ctx
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) getKernelCmdLine(useImage bool) string {
|
|
||||||
+ var params []string
|
|
||||||
+
|
|
||||||
+ if useImage {
|
|
||||||
+ params = append(params, "root=/dev/vda")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ params = append(params, "pci=off")
|
|
||||||
+ params = append(params, "reboot=k")
|
|
||||||
+ params = append(params, "panic=1")
|
|
||||||
+ params = append(params, "iommu=off")
|
|
||||||
+ params = append(params, "acpi=off")
|
|
||||||
+ params = append(params, "quiet")
|
|
||||||
+ params = append(params, "agent.use_vsock=true")
|
|
||||||
+ params = append(params, "random.trust_cpu=on")
|
|
||||||
+ params = append(params, "rw")
|
|
||||||
+ params = append(params, SerializeParams(s.config.KernelParams, "=")...)
|
|
||||||
+
|
|
||||||
+ return strings.Join(params, " ")
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) hypervisorConfig() HypervisorConfig {
|
|
||||||
+ return s.config
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig) error {
|
|
||||||
+ s.ctx = ctx
|
|
||||||
+
|
|
||||||
+ var span otelTrace.Span
|
|
||||||
+ span, _ = s.trace(ctx, "createSandbox")
|
|
||||||
+ defer span.End()
|
|
||||||
+
|
|
||||||
+ s.id = id
|
|
||||||
+ s.config = *hypervisorConfig
|
|
||||||
+
|
|
||||||
+ s.socketPath = filepath.Join(s.store.RunVMStoragePath(), id, "qmp.sock")
|
|
||||||
+ s.consolePath = filepath.Join(s.store.RunVMStoragePath(), id, "console.sock")
|
|
||||||
+ s.qmpMonitorCh = qmpChannel{
|
|
||||||
+ ctx: s.ctx,
|
|
||||||
+ path: s.socketPath,
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) waitSandBoxStarted(timeout int) error {
|
|
||||||
+ timeStart := time.Now()
|
|
||||||
+ for {
|
|
||||||
+ err := s.qmpSetup()
|
|
||||||
+ if err == nil {
|
|
||||||
+ break
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if int(time.Since(timeStart).Seconds()) > timeout {
|
|
||||||
+ return fmt.Errorf("Failed to connect to StratoVirt instance (timeout %ds): %v", timeout, err)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ time.Sleep(time.Duration(50) * time.Millisecond)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if err := s.qmpMonitorCh.qmp.ExecuteQMPCapabilities(s.qmpMonitorCh.ctx); err != nil {
|
|
||||||
+ s.Logger().WithError(err).Error(qmpCapErrMsg)
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error {
|
|
||||||
+ span, _ := s.trace(ctx, "startSandbox")
|
|
||||||
+ defer span.End()
|
|
||||||
+
|
|
||||||
+ var params []string
|
|
||||||
+ var use_image bool
|
|
||||||
+ params = append(params, "-name", fmt.Sprintf("sandbox-%s", s.id))
|
|
||||||
+ params = append(params, "-qmp", fmt.Sprintf("unix:%s,server,nowait", s.socketPath))
|
|
||||||
+
|
|
||||||
+ if kernelPath, err := s.config.KernelAssetPath(); err == nil {
|
|
||||||
+ params = append(params, "-kernel", kernelPath)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ initrdPath, err := s.config.InitrdAssetPath()
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if initrdPath == "" {
|
|
||||||
+ imagePath, err := s.config.ImageAssetPath()
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ use_image = true
|
|
||||||
+ s.mmioBlkSlots[0] = true
|
|
||||||
+ params = append(params, "-device", "virtio-blk-device,drive=rootfs")
|
|
||||||
+ params = append(params, "-drive", fmt.Sprintf("id=rootfs,file=%s,direct=off", imagePath))
|
|
||||||
+ } else {
|
|
||||||
+ use_image = false
|
|
||||||
+ params = append(params, "-initrd", initrdPath)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ params = append(params, "-append", s.getKernelCmdLine(use_image))
|
|
||||||
+ params = append(params, "-smp", fmt.Sprintf("%d", s.config.NumVCPUs))
|
|
||||||
+ params = append(params, "-m", fmt.Sprintf("%d", uint64(s.config.MemorySize)))
|
|
||||||
+ params = append(params, "-device", "virtio-serial-device")
|
|
||||||
+ params = append(params, "-device", "virtconsole,chardev=charconsole0,id=virtioconsole0")
|
|
||||||
+ params = append(params, "-object", fmt.Sprintf("rng-random,id=objrng0,filename=%s", randomDevice))
|
|
||||||
+ params = append(params, "-device", "virtio-rng-device,rng=objrng0")
|
|
||||||
+ params = append(params, "-chardev", fmt.Sprintf("socket,id=charconsole0,path=%s,server,nowait", s.consolePath))
|
|
||||||
+ params = append(params, "-pidfile", filepath.Join(s.store.RunVMStoragePath(), s.id, "pid"))
|
|
||||||
+
|
|
||||||
+ // add devices to cmdline
|
|
||||||
+ for _, d := range s.devices {
|
|
||||||
+ switch v := d.dev.(type) {
|
|
||||||
+ case Endpoint:
|
|
||||||
+ name := v.Name()
|
|
||||||
+ mac := v.HardwareAddr()
|
|
||||||
+ tapName := v.NetworkPair().TapInterface.TAPIface.Name
|
|
||||||
+ params = append(params, "-device", fmt.Sprintf("virtio-net-device,netdev=%s,id=%s,mac=%s", name, name, mac))
|
|
||||||
+ params = append(params, "-netdev", fmt.Sprintf("tap,id=%s,ifname=%s", name, tapName))
|
|
||||||
+ case config.BlockDrive:
|
|
||||||
+ id := v.ID
|
|
||||||
+ path := v.File
|
|
||||||
+ params = append(params, "-device", fmt.Sprintf("virtio-blk-device, drive=%s", id))
|
|
||||||
+ params = append(params, "-drive", fmt.Sprintf("id=%s,file=%s", id, path))
|
|
||||||
+ case types.VSock:
|
|
||||||
+ v.VhostFd.Close()
|
|
||||||
+ params = append(params, "-device", fmt.Sprintf("vhost-vsock-device,id=vsock-id,guest-cid=%d", v.ContextID))
|
|
||||||
+ default:
|
|
||||||
+ s.Logger().Error("Adding device type is unsupported")
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ // daemonize
|
|
||||||
+ params = append(params, "-daemonize")
|
|
||||||
+
|
|
||||||
+ // append logfile only on debug
|
|
||||||
+ if s.config.Debug {
|
|
||||||
+ dir := filepath.Join(s.store.RunVMStoragePath(), s.id)
|
|
||||||
+ params = append(params, "-D", fmt.Sprintf("%s/stratovirt.log", dir))
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ dir := filepath.Join(s.store.RunVMStoragePath(), s.id)
|
|
||||||
+ err = os.MkdirAll(dir, DirMode)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ defer func() {
|
|
||||||
+ if err != nil {
|
|
||||||
+ if err := os.RemoveAll(dir); err != nil {
|
|
||||||
+ s.Logger().WithError(err).Error("Fail to clean up vm dir %s", dir)
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ }()
|
|
||||||
+
|
|
||||||
+ binPath, err := s.config.HypervisorAssetPath()
|
|
||||||
+ if err != nil {
|
|
||||||
+ s.Logger().WithField("Fail to get hypervisor bin path", err).Error()
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ cmd := exec.CommandContext(s.ctx, binPath, params...)
|
|
||||||
+ s.Logger().Info("StratoVirt start with params: ", cmd)
|
|
||||||
+
|
|
||||||
+ if err := cmd.Start(); err != nil {
|
|
||||||
+ s.Logger().WithField("Error starting hypervisor, please check the params", err).Error()
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ s.pid = cmd.Process.Pid
|
|
||||||
+
|
|
||||||
+ if err = s.waitSandBoxStarted(timeout); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) stopSandbox(ctx context.Context, force bool) error {
|
|
||||||
+ span, _ := s.trace(ctx, "stopSandbox")
|
|
||||||
+ defer span.End()
|
|
||||||
+
|
|
||||||
+ defer func() {
|
|
||||||
+ dir := filepath.Join(s.store.RunVMStoragePath(), s.id)
|
|
||||||
+ link, _ := filepath.EvalSymlinks(dir)
|
|
||||||
+
|
|
||||||
+ if err := os.RemoveAll(dir); err != nil {
|
|
||||||
+ s.Logger().WithError(err).Warnf("Failed to clean up vm dir %s", dir)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if link != dir && link != "" {
|
|
||||||
+ if err := os.RemoveAll(link); err != nil {
|
|
||||||
+ s.Logger().WithError(err).WithField("link", link).Warn("Failed to remove vm path link %s", link)
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ }()
|
|
||||||
+
|
|
||||||
+ if !force {
|
|
||||||
+ err := s.qmpSetup()
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ err = s.qmpMonitorCh.qmp.ExecuteQuit(s.qmpMonitorCh.ctx)
|
|
||||||
+ if err != nil {
|
|
||||||
+ s.Logger().WithError(err).Error("Fail to execute qmp: QUIT")
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ } else {
|
|
||||||
+ if s.pid > 0 {
|
|
||||||
+ syscall.Kill(s.pid, syscall.SIGKILL)
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) pauseSandbox(ctx context.Context) error {
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) saveSandbox() error {
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) resumeSandbox(ctx context.Context) error {
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) addDevice(ctx context.Context, devInfo interface{}, devType deviceType) error {
|
|
||||||
+ span, _ := s.trace(ctx, "addDevice")
|
|
||||||
+ defer span.End()
|
|
||||||
+
|
|
||||||
+ dev := stratovirtDev{
|
|
||||||
+ dev: devInfo,
|
|
||||||
+ devType: devType,
|
|
||||||
+ }
|
|
||||||
+ s.devices = append(s.devices, dev)
|
|
||||||
+
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) getDevSlot(Name string, isPut bool) (slot int, err error) {
|
|
||||||
+ Name = filepath.Base(strings.ToLower(Name))
|
|
||||||
+
|
|
||||||
+ if strings.HasPrefix(Name, "eth") {
|
|
||||||
+ idxStr := strings.TrimPrefix(Name, "eth")
|
|
||||||
+ if idxStr == Name {
|
|
||||||
+ return 0, fmt.Errorf("Could not parse idx from Name %q", Name)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ idx, err := strconv.Atoi(idxStr)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return 0, fmt.Errorf("Could not convert to int from Str %q", idxStr)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if !isPut && s.mmioNetSlots[idx] {
|
|
||||||
+ return 0, fmt.Errorf("GetDevSlot failed, slot is being used %q", idxStr)
|
|
||||||
+ }
|
|
||||||
+ s.mmioNetSlots[idx] = !isPut
|
|
||||||
+
|
|
||||||
+ return idx, nil
|
|
||||||
+ } else if strings.HasPrefix(Name, "vd") {
|
|
||||||
+ charStr := strings.TrimPrefix(Name, "vd")
|
|
||||||
+ if charStr == Name {
|
|
||||||
+ return 0, fmt.Errorf("Could not parse idx from Name %q", Name)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ char := []rune(charStr)
|
|
||||||
+ idx := int(char[0] - 'a')
|
|
||||||
+
|
|
||||||
+ if !isPut && s.mmioBlkSlots[idx] {
|
|
||||||
+ return 0, fmt.Errorf("GetDevSlot failed, slot is being used %q", charStr)
|
|
||||||
+ }
|
|
||||||
+ s.mmioBlkSlots[idx] = !isPut
|
|
||||||
+
|
|
||||||
+ return idx, nil
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return 0, fmt.Errorf("GetDevSlot failed, Name is invalid %q", Name)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) hotplugNet(endpoint Endpoint, op operation) (err error) {
|
|
||||||
+ err = s.qmpSetup()
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ var tap TapInterface
|
|
||||||
+
|
|
||||||
+ switch endpoint.Type() {
|
|
||||||
+ case VethEndpointType:
|
|
||||||
+ drive := endpoint.(*VethEndpoint)
|
|
||||||
+ tap = drive.NetPair.TapInterface
|
|
||||||
+ case TapEndpointType:
|
|
||||||
+ drive := endpoint.(*TapEndpoint)
|
|
||||||
+ tap = drive.TapInterface
|
|
||||||
+ default:
|
|
||||||
+ return fmt.Errorf("Endpoint is not supported")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ switch op {
|
|
||||||
+ case addDevice:
|
|
||||||
+ var (
|
|
||||||
+ VMFdNames []string
|
|
||||||
+ VhostFdNames []string
|
|
||||||
+ )
|
|
||||||
+ for i, VMFd := range tap.VMFds {
|
|
||||||
+ fdName := fmt.Sprintf("fd%d", i)
|
|
||||||
+ if err := s.qmpMonitorCh.qmp.ExecuteGetFD(s.qmpMonitorCh.ctx, fdName, VMFd); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ VMFdNames = append(VMFdNames, fdName)
|
|
||||||
+ }
|
|
||||||
+ for i, VhostFd := range tap.VhostFds {
|
|
||||||
+ fdName := fmt.Sprintf("vhostfd%d", i)
|
|
||||||
+ if err := s.qmpMonitorCh.qmp.ExecuteGetFD(s.qmpMonitorCh.ctx, fdName, VhostFd); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ VhostFd.Close()
|
|
||||||
+ VhostFdNames = append(VhostFdNames, fdName)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ slot, err := s.getDevSlot(endpoint.Name(), false)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return fmt.Errorf("Could not get unused slot for %q", endpoint.Name())
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if len(VMFdNames) != 0 || len(VhostFdNames) != 0 {
|
|
||||||
+ if err := s.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(s.qmpMonitorCh.ctx, "tap", tap.ID, VMFdNames, VhostFdNames); err != nil {
|
|
||||||
+ s.getDevSlot(endpoint.Name(), true)
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ } else {
|
|
||||||
+ if err := s.qmpMonitorCh.qmp.ExecuteNetdevAdd(s.qmpMonitorCh.ctx, "tap", tap.ID, tap.TAPIface.Name, "no", "no", 0); err != nil {
|
|
||||||
+ s.getDevSlot(endpoint.Name(), true)
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ if err := s.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(s.qmpMonitorCh.ctx, tap.Name, tap.ID, endpoint.HardwareAddr(), fmt.Sprintf("%d", slot), "", "", 0, false); err != nil {
|
|
||||||
+ s.getDevSlot(endpoint.Name(), true)
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ case removeDevice:
|
|
||||||
+ if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, tap.ID); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ if err := s.qmpMonitorCh.qmp.ExecuteNetdevDel(s.qmpMonitorCh.ctx, tap.ID); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ default:
|
|
||||||
+ return fmt.Errorf("Operation is not supported")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) hotplugBlk(drive *config.BlockDrive, op operation) (err error) {
|
|
||||||
+ err = s.qmpSetup()
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ switch op {
|
|
||||||
+ case addDevice:
|
|
||||||
+ driver := "virtio-blk-pci"
|
|
||||||
+ slot, err := s.getDevSlot(drive.VirtPath, false)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return fmt.Errorf("Could not get unused slot for %q", drive.VirtPath)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if err := s.qmpMonitorCh.qmp.ExecuteBlockdevAdd(s.qmpMonitorCh.ctx, drive.File, drive.ID, false); err != nil {
|
|
||||||
+ s.getDevSlot(drive.VirtPath, true)
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if err := s.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(s.qmpMonitorCh.ctx, drive.ID, drive.ID, driver, fmt.Sprintf("%d", slot), "", "", 0, true, false); err != nil {
|
|
||||||
+ s.getDevSlot(drive.VirtPath, true)
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ case removeDevice:
|
|
||||||
+ if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, drive.ID); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ if err := s.qmpMonitorCh.qmp.ExecuteBlockdevDel(s.qmpMonitorCh.ctx, drive.ID); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ s.getDevSlot(drive.VirtPath, true)
|
|
||||||
+ default:
|
|
||||||
+ return fmt.Errorf("Operation is not supported")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) hotplugAddDevice(ctx context.Context, devInfo interface{}, devType deviceType) (interface{}, error) {
|
|
||||||
+ span, _ := s.trace(ctx, "hotplugAddDevice")
|
|
||||||
+ defer span.End()
|
|
||||||
+
|
|
||||||
+ switch devType {
|
|
||||||
+ case netDev:
|
|
||||||
+ return nil, s.hotplugNet(devInfo.(Endpoint), addDevice)
|
|
||||||
+ case blockDev:
|
|
||||||
+ return nil, s.hotplugBlk(devInfo.(*config.BlockDrive), addDevice)
|
|
||||||
+ default:
|
|
||||||
+ return nil, fmt.Errorf("Hotplug add device failed: unsupported device type '%v'", devType)
|
|
||||||
+ }
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) hotplugRemoveDevice(ctx context.Context, devInfo interface{}, devType deviceType) (interface{}, error) {
|
|
||||||
+ span, _ := s.trace(ctx, "hotplugRemoveDevice")
|
|
||||||
+ defer span.End()
|
|
||||||
+
|
|
||||||
+ switch devType {
|
|
||||||
+ case netDev:
|
|
||||||
+ return nil, s.hotplugNet(devInfo.(Endpoint), removeDevice)
|
|
||||||
+ case blockDev:
|
|
||||||
+ return nil, s.hotplugBlk(devInfo.(*config.BlockDrive), removeDevice)
|
|
||||||
+ default:
|
|
||||||
+ return nil, fmt.Errorf("Hotplug remove device: unsupported device type '%v'", devType)
|
|
||||||
+ }
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) resizeMemory(ctx context.Context, reqMemMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error) {
|
|
||||||
+ return 0, memoryDevice{}, nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) resizeVCPUs(ctx context.Context, reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint32, err error) {
|
|
||||||
+ return 0, 0, nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) getSandboxConsole(ctx context.Context, id string) (string, string, error) {
|
|
||||||
+ span, _ := s.trace(ctx, "getSandboxConsole")
|
|
||||||
+ defer span.End()
|
|
||||||
+
|
|
||||||
+ var consolePath string
|
|
||||||
+ if s.config.Debug {
|
|
||||||
+ consolePath = s.consolePath
|
|
||||||
+ } else {
|
|
||||||
+ consolePath = ""
|
|
||||||
+ }
|
|
||||||
+ consoleURL, err := utils.BuildSocketPath(consolePath)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return consoleProtoUnix, "", err
|
|
||||||
+ }
|
|
||||||
+ return consoleProtoUnix, consoleURL, nil
|
|
||||||
+
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) disconnect(ctx context.Context) {
|
|
||||||
+ span, _ := s.trace(ctx, "disconnect")
|
|
||||||
+ defer span.End()
|
|
||||||
+
|
|
||||||
+ s.qmpTeardown()
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) capabilities(ctx context.Context) types.Capabilities {
|
|
||||||
+ span, _ := s.trace(ctx, "capabilities")
|
|
||||||
+ defer span.End()
|
|
||||||
+
|
|
||||||
+ var caps types.Capabilities
|
|
||||||
+ caps.SetBlockDeviceHotplugSupport()
|
|
||||||
+
|
|
||||||
+ return caps
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) qmpTeardown() {
|
|
||||||
+ if s.qmpMonitorCh.qmp != nil {
|
|
||||||
+ s.qmpMonitorCh.qmp.Shutdown()
|
|
||||||
+ <-s.qmpMonitorCh.disconn
|
|
||||||
+ s.qmpMonitorCh.qmp = nil
|
|
||||||
+ s.qmpMonitorCh.disconn = nil
|
|
||||||
+ }
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) qmpSetup() error {
|
|
||||||
+ s.qmpTeardown()
|
|
||||||
+
|
|
||||||
+ cfg := govmmQemu.QMPConfig{Logger: newQMPLogger()}
|
|
||||||
+
|
|
||||||
+ // Auto-closed by QMPStart().
|
|
||||||
+ disconnectCh := make(chan struct{})
|
|
||||||
+
|
|
||||||
+ qmp, _, err := govmmQemu.QMPStart(s.qmpMonitorCh.ctx, s.qmpMonitorCh.path, cfg, disconnectCh)
|
|
||||||
+ if err != nil {
|
|
||||||
+ s.Logger().WithError(err).Error("Failed to connect to StratoVirt instance")
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ s.qmpMonitorCh.qmp = qmp
|
|
||||||
+ s.qmpMonitorCh.disconn = disconnectCh
|
|
||||||
+
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) getThreadIDs(ctx context.Context) (vcpuThreadIDs, error) {
|
|
||||||
+ span, _ := s.trace(ctx, "getThreadIDs")
|
|
||||||
+ defer span.End()
|
|
||||||
+
|
|
||||||
+ tid := vcpuThreadIDs{}
|
|
||||||
+ if err := s.qmpSetup(); err != nil {
|
|
||||||
+ return tid, err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ cpuInfos, err := s.qmpMonitorCh.qmp.ExecQueryCpus(s.qmpMonitorCh.ctx)
|
|
||||||
+ if err != nil {
|
|
||||||
+ s.Logger().WithError(err).Error("Failed to query cpu infos")
|
|
||||||
+ return tid, err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ tid.vcpus = make(map[int]int, len(cpuInfos))
|
|
||||||
+ for _, i := range cpuInfos {
|
|
||||||
+ if i.ThreadID > 0 {
|
|
||||||
+ tid.vcpus[i.CPU] = i.ThreadID
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ return tid, nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) cleanup(ctx context.Context) error {
|
|
||||||
+ span, _ := s.trace(ctx, "cleanup")
|
|
||||||
+ defer span.End()
|
|
||||||
+
|
|
||||||
+ s.qmpTeardown()
|
|
||||||
+
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) getPids() []int {
|
|
||||||
+ return []int{s.pid}
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) getVirtioFsPid() *int {
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, j []byte) error {
|
|
||||||
+ return errors.New("stratovirt is not supported by VM cache")
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) toGrpc(ctx context.Context) ([]byte, error) {
|
|
||||||
+ return nil, errors.New("stratovirt is not supported by VM cache")
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) check() error {
|
|
||||||
+ if err := syscall.Kill(s.pid, syscall.Signal(0)); err != nil {
|
|
||||||
+ return errors.Wrapf(err, "Failed to ping StratoVirt process")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) generateSocket(id string) (interface{}, error) {
|
|
||||||
+ return generateVMSocket(id, s.store.RunVMStoragePath())
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) isRateLimiterBuiltin() bool {
|
|
||||||
+ return true
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) save() (p persistapi.HypervisorState) {
|
|
||||||
+ pids := s.getPids()
|
|
||||||
+ p.Pid = pids[0]
|
|
||||||
+ p.Type = string(StratovirtHypervisor)
|
|
||||||
+ return
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) load(p persistapi.HypervisorState) {
|
|
||||||
+ s.pid = p.Pid
|
|
||||||
+
|
|
||||||
+ return
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) setSandbox(sandbox *Sandbox) {
|
|
||||||
+ s.sandbox = sandbox
|
|
||||||
+ return
|
|
||||||
+}
|
|
||||||
--
|
|
||||||
2.21.1 (Apple Git-122.3)
|
|
||||||
|
|
||||||
@ -1,174 +0,0 @@
|
|||||||
From 1ffd95187a61582e858dd37c0ab434d3159a0f52 Mon Sep 17 00:00:00 2001
|
|
||||||
From: Wei Gao <gaowei66@huawei.com>
|
|
||||||
Date: Mon, 9 Aug 2021 14:26:35 +0800
|
|
||||||
Subject: [PATCH 2/6] agent: add support of new sandbox hypervisor kind
|
|
||||||
StratoVirt.
|
|
||||||
|
|
||||||
1. add new grpc interface `UpdateInterfaceHwAddrByName`.
|
|
||||||
2. comment out rescan_pci temporarily.
|
|
||||||
|
|
||||||
Signed-off-by: Wei Gao <gaowei66@huawei.com>
|
|
||||||
---
|
|
||||||
src/agent/protocols/protos/agent.proto | 5 +++
|
|
||||||
src/agent/src/netlink.rs | 31 ++++++++++++++++
|
|
||||||
src/agent/src/rpc.rs | 51 +++++++++++++++++++++++++-
|
|
||||||
3 files changed, 85 insertions(+), 2 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/src/agent/protocols/protos/agent.proto b/src/agent/protocols/protos/agent.proto
|
|
||||||
index 6cbf5a28..e00f5c63 100644
|
|
||||||
--- a/src/agent/protocols/protos/agent.proto
|
|
||||||
+++ b/src/agent/protocols/protos/agent.proto
|
|
||||||
@@ -46,6 +46,7 @@ service AgentService {
|
|
||||||
|
|
||||||
// networking
|
|
||||||
rpc UpdateInterface(UpdateInterfaceRequest) returns (types.Interface);
|
|
||||||
+ rpc UpdateInterfaceHwAddrByName(UpdateInterfaceHwAddrByNameRequest) returns (types.Interface);
|
|
||||||
rpc UpdateRoutes(UpdateRoutesRequest) returns (Routes);
|
|
||||||
rpc ListInterfaces(ListInterfacesRequest) returns(Interfaces);
|
|
||||||
rpc ListRoutes(ListRoutesRequest) returns (Routes);
|
|
||||||
@@ -308,6 +309,10 @@ message UpdateInterfaceRequest {
|
|
||||||
types.Interface interface = 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
+message UpdateInterfaceHwAddrByNameRequest {
|
|
||||||
+ types.Interface interface = 1;
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
message UpdateRoutesRequest {
|
|
||||||
Routes routes = 1;
|
|
||||||
}
|
|
||||||
diff --git a/src/agent/src/netlink.rs b/src/agent/src/netlink.rs
|
|
||||||
index 3ab6dbaa..82632d1b 100644
|
|
||||||
--- a/src/agent/src/netlink.rs
|
|
||||||
+++ b/src/agent/src/netlink.rs
|
|
||||||
@@ -104,6 +104,29 @@ impl Handle {
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
+ pub async fn update_interface_hw_addr_by_name(&mut self, iface: &Interface) -> Result<()> {
|
|
||||||
+ let link = self.find_link(LinkFilter::Name(&iface.name)).await?;
|
|
||||||
+
|
|
||||||
+ // Delete all addresses associated with the link
|
|
||||||
+ let addresses = self
|
|
||||||
+ .list_addresses(AddressFilter::LinkIndex(link.index()))
|
|
||||||
+ .await?;
|
|
||||||
+ self.delete_addresses(addresses).await?;
|
|
||||||
+
|
|
||||||
+ if iface.IPAddresses.len() == 0 {
|
|
||||||
+ self.enable_link(link.index(), false).await?;
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ // Update hardware mac address
|
|
||||||
+ let mac_addr = parse_mac_address(iface.get_hwAddr())
|
|
||||||
+ .with_context(|| format!("Failed to parse MAC address: {}", iface.get_hwAddr()))?;
|
|
||||||
+ self.link_set_hw_addr(link.index(), mac_addr)
|
|
||||||
+ .await
|
|
||||||
+ .with_context(|| format!("Could not set {:?} to {}", mac_addr, link.name()))?;
|
|
||||||
+
|
|
||||||
+ Ok(())
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
pub async fn handle_localhost(&self) -> Result<()> {
|
|
||||||
let link = self.find_link(LinkFilter::Name("lo")).await?;
|
|
||||||
self.enable_link(link.index(), true).await?;
|
|
||||||
@@ -216,6 +239,14 @@ impl Handle {
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
+ async fn link_set_hw_addr(&self, link_index: u32, hw_addr: [u8; 6]) -> Result<()> {
|
|
||||||
+ let link_req = self.handle.link().set(link_index);
|
|
||||||
+ let set_req = link_req.address(hw_addr.to_vec());
|
|
||||||
+ set_req.execute().await?;
|
|
||||||
+
|
|
||||||
+ Ok(())
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
async fn query_routes(
|
|
||||||
&self,
|
|
||||||
ip_version: Option<IpVersion>,
|
|
||||||
diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs
|
|
||||||
index 92025af3..2cc1c983 100644
|
|
||||||
--- a/src/agent/src/rpc.rs
|
|
||||||
+++ b/src/agent/src/rpc.rs
|
|
||||||
@@ -40,7 +40,7 @@ use nix::sys::stat;
|
|
||||||
use nix::unistd::{self, Pid};
|
|
||||||
use rustjail::process::ProcessOperations;
|
|
||||||
|
|
||||||
-use crate::device::{add_devices, rescan_pci_bus, update_device_cgroup};
|
|
||||||
+use crate::device::{add_devices, update_device_cgroup};
|
|
||||||
use crate::linux_abi::*;
|
|
||||||
use crate::metrics::get_metrics;
|
|
||||||
use crate::mount::{add_storages, remove_mounts, BareMount, STORAGE_HANDLER_LIST};
|
|
||||||
@@ -123,7 +123,9 @@ impl AgentService {
|
|
||||||
|
|
||||||
// re-scan PCI bus
|
|
||||||
// looking for hidden devices
|
|
||||||
- rescan_pci_bus().context("Could not rescan PCI bus")?;
|
|
||||||
+ // FIXME: Comment out this code temporarily, because once the PCIBus is scanned,
|
|
||||||
+ // the device hot-plug event is lost
|
|
||||||
+ // rescan_pci_bus().context("Could not rescan PCI bus")?;
|
|
||||||
|
|
||||||
// Some devices need some extra processing (the ones invoked with
|
|
||||||
// --device for instance), and that's what this call is doing. It
|
|
||||||
@@ -797,6 +799,34 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
|
|
||||||
Ok(interface)
|
|
||||||
}
|
|
||||||
|
|
||||||
+ async fn update_interface_hw_addr_by_name(
|
|
||||||
+ &self,
|
|
||||||
+ _ctx: &TtrpcContext,
|
|
||||||
+ req: protocols::agent::UpdateInterfaceHwAddrByNameRequest,
|
|
||||||
+ ) -> ttrpc::Result<Interface> {
|
|
||||||
+ let interface = req.interface.into_option().ok_or_else(|| {
|
|
||||||
+ ttrpc_error(
|
|
||||||
+ ttrpc::Code::INVALID_ARGUMENT,
|
|
||||||
+ "empty update interface request".to_string(),
|
|
||||||
+ )
|
|
||||||
+ })?;
|
|
||||||
+
|
|
||||||
+ self.sandbox
|
|
||||||
+ .lock()
|
|
||||||
+ .await
|
|
||||||
+ .rtnl
|
|
||||||
+ .update_interface_hw_addr_by_name(&interface)
|
|
||||||
+ .await
|
|
||||||
+ .map_err(|e| {
|
|
||||||
+ ttrpc_error(
|
|
||||||
+ ttrpc::Code::INTERNAL,
|
|
||||||
+ format!("update interface hw addr: {:?}", e),
|
|
||||||
+ )
|
|
||||||
+ })?;
|
|
||||||
+
|
|
||||||
+ Ok(interface)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
async fn update_routes(
|
|
||||||
&self,
|
|
||||||
_ctx: &TtrpcContext,
|
|
||||||
@@ -1670,6 +1700,23 @@ mod tests {
|
|
||||||
assert!(result.is_err(), "expected update interface to fail");
|
|
||||||
}
|
|
||||||
|
|
||||||
+ #[tokio::test]
|
|
||||||
+ async fn test_update_interface_hw_addr_by_name() {
|
|
||||||
+ let logger = slog::Logger::root(slog::Discard, o!());
|
|
||||||
+ let sandbox = Sandbox::new(&logger).unwrap();
|
|
||||||
+
|
|
||||||
+ let agent_service = Box::new(AgentService {
|
|
||||||
+ sandbox: Arc::new(Mutex::new(sandbox)),
|
|
||||||
+ });
|
|
||||||
+
|
|
||||||
+ let req = protocols::agent::UpdateInterfaceHwAddrByNameRequest::default();
|
|
||||||
+ let ctx = mk_ttrpc_context();
|
|
||||||
+
|
|
||||||
+ let result = agent_service.update_interface_hw_addr_by_name(&ctx, req).await;
|
|
||||||
+
|
|
||||||
+ assert!(result.is_err(), "expected update interface to fail");
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
#[tokio::test]
|
|
||||||
async fn test_update_routes() {
|
|
||||||
let logger = slog::Logger::root(slog::Discard, o!());
|
|
||||||
--
|
|
||||||
2.21.1 (Apple Git-122.3)
|
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
@ -1,542 +0,0 @@
|
|||||||
From 950c0db14a9a9baccefd83e87893d7f40c2bd13d Mon Sep 17 00:00:00 2001
|
|
||||||
From: Wei Gao <gaowei66@huawei.com>
|
|
||||||
Date: Mon, 9 Aug 2021 14:47:19 +0800
|
|
||||||
Subject: [PATCH 4/6] configuration: add configuration generator for hypervisor
|
|
||||||
type stratovirt.
|
|
||||||
|
|
||||||
Signed-off-by: Wei Gao <gaowei66@huawei.com>
|
|
||||||
---
|
|
||||||
src/runtime/.gitignore | 1 +
|
|
||||||
src/runtime/Makefile | 40 +-
|
|
||||||
src/runtime/arch/amd64-options.mk | 3 +
|
|
||||||
src/runtime/arch/arm64-options.mk | 3 +
|
|
||||||
.../config/configuration-stratovirt.toml.in | 356 ++++++++++++++++++
|
|
||||||
5 files changed, 402 insertions(+), 1 deletion(-)
|
|
||||||
create mode 100644 src/runtime/cli/config/configuration-stratovirt.toml.in
|
|
||||||
|
|
||||||
diff --git a/src/runtime/.gitignore b/src/runtime/.gitignore
|
|
||||||
index 52b9e4e5..0a630a07 100644
|
|
||||||
--- a/src/runtime/.gitignore
|
|
||||||
+++ b/src/runtime/.gitignore
|
|
||||||
@@ -10,6 +10,7 @@ coverage.html
|
|
||||||
/cli/config/configuration-fc.toml
|
|
||||||
/cli/config/configuration-qemu.toml
|
|
||||||
/cli/config/configuration-clh.toml
|
|
||||||
+/cli/config/configuration-stratovirt.toml
|
|
||||||
/cli/config-generated.go
|
|
||||||
/cli/containerd-shim-kata-v2/config-generated.go
|
|
||||||
/cli/coverage.html
|
|
||||||
diff --git a/src/runtime/Makefile b/src/runtime/Makefile
|
|
||||||
index 4a69c05c..ea2cd296 100644
|
|
||||||
--- a/src/runtime/Makefile
|
|
||||||
+++ b/src/runtime/Makefile
|
|
||||||
@@ -74,6 +74,7 @@ QEMUBINDIR := $(PREFIXDEPS)/bin
|
|
||||||
CLHBINDIR := $(PREFIXDEPS)/bin
|
|
||||||
FCBINDIR := $(PREFIXDEPS)/bin
|
|
||||||
ACRNBINDIR := $(PREFIXDEPS)/bin
|
|
||||||
+STRATOVIRTBINDIR := $(PREFIXDEPS)/bin
|
|
||||||
SYSCONFDIR := /etc
|
|
||||||
LOCALSTATEDIR := /var
|
|
||||||
|
|
||||||
@@ -93,6 +94,7 @@ GENERATED_VARS = \
|
|
||||||
CONFIG_QEMU_IN \
|
|
||||||
CONFIG_CLH_IN \
|
|
||||||
CONFIG_FC_IN \
|
|
||||||
+ CONFIG_STRATOVIRT_IN \
|
|
||||||
$(USER_VARS)
|
|
||||||
SCRIPTS += $(COLLECT_SCRIPT)
|
|
||||||
SCRIPTS_DIR := $(BINDIR)
|
|
||||||
@@ -116,12 +118,13 @@ HYPERVISOR_ACRN = acrn
|
|
||||||
HYPERVISOR_FC = firecracker
|
|
||||||
HYPERVISOR_QEMU = qemu
|
|
||||||
HYPERVISOR_CLH = cloud-hypervisor
|
|
||||||
+HYPERVISOR_STRATOVIRT = stratovirt
|
|
||||||
|
|
||||||
# Determines which hypervisor is specified in $(CONFIG_FILE).
|
|
||||||
DEFAULT_HYPERVISOR ?= $(HYPERVISOR_QEMU)
|
|
||||||
|
|
||||||
# List of hypervisors this build system can generate configuration for.
|
|
||||||
-HYPERVISORS := $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_CLH)
|
|
||||||
+HYPERVISORS := $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_CLH) $(HYPERVISOR_STRATOVIRT)
|
|
||||||
|
|
||||||
QEMUPATH := $(QEMUBINDIR)/$(QEMUCMD)
|
|
||||||
QEMUVALIDHYPERVISORPATHS := [\"$(QEMUPATH)\"]
|
|
||||||
@@ -141,6 +144,9 @@ ACRNVALIDHYPERVISORPATHS := [\"$(ACRNPATH)\"]
|
|
||||||
ACRNCTLPATH := $(ACRNBINDIR)/$(ACRNCTLCMD)
|
|
||||||
ACRNVALIDCTLPATHS := [\"$(ACRNCTLPATH)\"]
|
|
||||||
|
|
||||||
+STRATOVIRTPATH = $(STRATOVIRTBINDIR)/$(STRATOVIRTCMD)
|
|
||||||
+STRATOVIRTVALIDHYPERVISORPATHS := [\"$(STRATOVIRTPATH)\"]
|
|
||||||
+
|
|
||||||
NETMONCMD := $(BIN_PREFIX)-netmon
|
|
||||||
NETMONPATH := $(PKGLIBEXECDIR)/$(NETMONCMD)
|
|
||||||
|
|
||||||
@@ -267,6 +273,29 @@ ifneq (,$(CLHCMD))
|
|
||||||
KERNELPATH_CLH = $(KERNELDIR)/$(KERNEL_NAME_CLH)
|
|
||||||
endif
|
|
||||||
|
|
||||||
+ifneq (,$(STRATOVIRTCMD))
|
|
||||||
+ KNOWN_HYPERVISORS += $(HYPERVISOR_STRATOVIRT)
|
|
||||||
+
|
|
||||||
+ CONFIG_FILE_STRATOVIRT = configuration-stratovirt.toml
|
|
||||||
+ CONFIG_STRATOVIRT = $(CLI_DIR)/config/$(CONFIG_FILE_STRATOVIRT)
|
|
||||||
+ CONFIG_STRATOVIRT_IN = $(CONFIG_STRATOVIRT).in
|
|
||||||
+
|
|
||||||
+ CONFIG_PATH_STRATOVIRT = $(abspath $(CONFDIR)/$(CONFIG_FILE_STRATOVIRT))
|
|
||||||
+ CONFIG_PATHS += $(CONFIG_PATH_STRATOVIRT)
|
|
||||||
+
|
|
||||||
+ SYSCONFIG_STRATOVIRT = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_STRATOVIRT))
|
|
||||||
+ SYSCONFIG_PATHS += $(SYSCONFIG_STRATOVIRT)
|
|
||||||
+
|
|
||||||
+ CONFIGS += $(CONFIG_STRATOVIRT)
|
|
||||||
+
|
|
||||||
+ # stratovirt-specific options (all should be suffixed by "_STRATOVIRT")
|
|
||||||
+ DEFBLOCKSTORAGEDRIVER_STRATOVIRT := virtio-mmio
|
|
||||||
+ DEFNETWORKMODEL_STRATOVIRT := none
|
|
||||||
+ KENRELTYPE_STRATOVIRT = uncompressed
|
|
||||||
+ KERNEL_NAME_STRATOVIRT = $(call MAKE_KERNEL_NAME,$(KENRELTYPE_STRATOVIRT))
|
|
||||||
+ KERNELPATH_STRATOVIRT = $(KERNELDIR)/$(KERNEL_NAME_STRATOVIRT)
|
|
||||||
+endif
|
|
||||||
+
|
|
||||||
ifneq (,$(FCCMD))
|
|
||||||
KNOWN_HYPERVISORS += $(HYPERVISOR_FC)
|
|
||||||
|
|
||||||
@@ -363,6 +392,7 @@ USER_VARS += BINDIR
|
|
||||||
USER_VARS += CONFIG_ACRN_IN
|
|
||||||
USER_VARS += CONFIG_CLH_IN
|
|
||||||
USER_VARS += CONFIG_FC_IN
|
|
||||||
+USER_VARS += CONFIG_STRATOVIRT_IN
|
|
||||||
USER_VARS += CONFIG_PATH
|
|
||||||
USER_VARS += CONFIG_QEMU_IN
|
|
||||||
USER_VARS += DESTDIR
|
|
||||||
@@ -382,6 +412,8 @@ USER_VARS += FCPATH
|
|
||||||
USER_VARS += FCVALIDHYPERVISORPATHS
|
|
||||||
USER_VARS += FCJAILERPATH
|
|
||||||
USER_VARS += FCVALIDJAILERPATHS
|
|
||||||
+USER_VARS += STRATOVIRTPATH
|
|
||||||
+USER_VARS += STRATOVIRTVALIDHYPERVISORPATHS
|
|
||||||
USER_VARS += SYSCONFIG
|
|
||||||
USER_VARS += IMAGENAME
|
|
||||||
USER_VARS += IMAGEPATH
|
|
||||||
@@ -395,6 +427,7 @@ USER_VARS += KERNELPATH_ACRN
|
|
||||||
USER_VARS += KERNELPATH
|
|
||||||
USER_VARS += KERNELPATH_CLH
|
|
||||||
USER_VARS += KERNELPATH_FC
|
|
||||||
+USER_VARS += KERNELPATH_STRATOVIRT
|
|
||||||
USER_VARS += KERNELVIRTIOFSPATH
|
|
||||||
USER_VARS += FIRMWAREPATH
|
|
||||||
USER_VARS += MACHINEACCELERATORS
|
|
||||||
@@ -434,12 +467,14 @@ USER_VARS += DEFNETWORKMODEL_ACRN
|
|
||||||
USER_VARS += DEFNETWORKMODEL_CLH
|
|
||||||
USER_VARS += DEFNETWORKMODEL_FC
|
|
||||||
USER_VARS += DEFNETWORKMODEL_QEMU
|
|
||||||
+USER_VARS += DEFNETWORKMODEL_STRATOVIRT
|
|
||||||
USER_VARS += DEFDISABLEGUESTSECCOMP
|
|
||||||
USER_VARS += DEFAULTEXPFEATURES
|
|
||||||
USER_VARS += DEFDISABLEBLOCK
|
|
||||||
USER_VARS += DEFBLOCKSTORAGEDRIVER_ACRN
|
|
||||||
USER_VARS += DEFBLOCKSTORAGEDRIVER_FC
|
|
||||||
USER_VARS += DEFBLOCKSTORAGEDRIVER_QEMU
|
|
||||||
+USER_VARS += DEFBLOCKSTORAGEDRIVER_STRATOVIRT
|
|
||||||
USER_VARS += DEFSHAREDFS_QEMU_VIRTIOFS
|
|
||||||
USER_VARS += DEFVIRTIOFSDAEMON
|
|
||||||
USER_VARS += DEFVALIDVIRTIOFSDAEMONPATHS
|
|
||||||
@@ -773,6 +808,9 @@ ifneq (,$(findstring $(HYPERVISOR_FC),$(KNOWN_HYPERVISORS)))
|
|
||||||
endif
|
|
||||||
ifneq (,$(findstring $(HYPERVISOR_ACRN),$(KNOWN_HYPERVISORS)))
|
|
||||||
@printf "\t$(HYPERVISOR_ACRN) hypervisor path (ACRNPATH) : %s\n" $(abspath $(ACRNPATH))
|
|
||||||
+endif
|
|
||||||
+ifneq (,$(findstring $(HYPERVISOR_STRATOVIRT),$(KNOWN_HYPERVISORS)))
|
|
||||||
+ @printf "\t$(HYPERVISOR_STRATOVIRT) hypervisor path (STRATOVIRTPATH) : %s\n" $(abspath $(STRATOVIRTPATH))
|
|
||||||
endif
|
|
||||||
@printf "\tassets path (PKGDATADIR) : %s\n" $(abspath $(PKGDATADIR))
|
|
||||||
@printf "\tshim path (PKGLIBEXECDIR) : %s\n" $(abspath $(PKGLIBEXECDIR))
|
|
||||||
diff --git a/src/runtime/arch/amd64-options.mk b/src/runtime/arch/amd64-options.mk
|
|
||||||
index 83af8cc0..ff2af9e6 100644
|
|
||||||
--- a/src/runtime/arch/amd64-options.mk
|
|
||||||
+++ b/src/runtime/arch/amd64-options.mk
|
|
||||||
@@ -23,3 +23,6 @@ ACRNCTLCMD := acrnctl
|
|
||||||
|
|
||||||
# cloud-hypervisor binary name
|
|
||||||
CLHCMD := cloud-hypervisor
|
|
||||||
+
|
|
||||||
+# stratovirt binary name
|
|
||||||
+STRATOVIRTCMD := stratovirt
|
|
||||||
\ No newline at end of file
|
|
||||||
diff --git a/src/runtime/arch/arm64-options.mk b/src/runtime/arch/arm64-options.mk
|
|
||||||
index ad5ef5d4..2ad3f657 100644
|
|
||||||
--- a/src/runtime/arch/arm64-options.mk
|
|
||||||
+++ b/src/runtime/arch/arm64-options.mk
|
|
||||||
@@ -19,3 +19,6 @@ FCJAILERCMD := jailer
|
|
||||||
|
|
||||||
# cloud-hypervisor binary name
|
|
||||||
CLHCMD := cloud-hypervisor
|
|
||||||
+
|
|
||||||
+# stratovirt binary name
|
|
||||||
+STRATOVIRTCMD := stratovirt
|
|
||||||
\ No newline at end of file
|
|
||||||
diff --git a/src/runtime/cli/config/configuration-stratovirt.toml.in b/src/runtime/cli/config/configuration-stratovirt.toml.in
|
|
||||||
new file mode 100644
|
|
||||||
index 00000000..5c83c3c9
|
|
||||||
--- /dev/null
|
|
||||||
+++ b/src/runtime/cli/config/configuration-stratovirt.toml.in
|
|
||||||
@@ -0,0 +1,356 @@
|
|
||||||
+# Copyright (c) 2017-2019 Intel Corporation
|
|
||||||
+#
|
|
||||||
+# SPDX-License-Identifier: Apache-2.0
|
|
||||||
+#
|
|
||||||
+
|
|
||||||
+# XXX: WARNING: this file is auto-generated.
|
|
||||||
+# XXX:
|
|
||||||
+# XXX: Source file: "@CONFIG_STRATOVIRT_IN@"
|
|
||||||
+# XXX: Project:
|
|
||||||
+# XXX: Name: @PROJECT_NAME@
|
|
||||||
+# XXX: Type: @PROJECT_TYPE@
|
|
||||||
+
|
|
||||||
+[hypervisor.stratovirt]
|
|
||||||
+path = "@STRATOVIRTPATH@"
|
|
||||||
+kernel = "@KERNELPATH_STRATOVIRT@"
|
|
||||||
+image = "@IMAGEPATH@"
|
|
||||||
+
|
|
||||||
+# List of valid annotation names for the hypervisor
|
|
||||||
+# Each member of the list is a regular expression, which is the base name
|
|
||||||
+# of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path"
|
|
||||||
+enable_annotations = @DEFENABLEANNOTATIONS@
|
|
||||||
+
|
|
||||||
+# List of valid annotations values for the hypervisor
|
|
||||||
+# Each member of the list is a path pattern as described by glob(3).
|
|
||||||
+# The default if not set is empty (all annotations rejected.)
|
|
||||||
+# Your distribution recommends: @STRATOVIRTVALIDHYPERVISORPATHS@
|
|
||||||
+valid_hypervisor_paths = @STRATOVIRTVALIDHYPERVISORPATHS@
|
|
||||||
+
|
|
||||||
+# Optional space-separated list of options to pass to the guest kernel.
|
|
||||||
+# For example, use `kernel_params = "vsyscall=emulate"` if you are having
|
|
||||||
+# trouble running pre-2.15 glibc.
|
|
||||||
+#
|
|
||||||
+# WARNING: - any parameter specified here will take priority over the default
|
|
||||||
+# parameter value of the same name used to start the virtual machine.
|
|
||||||
+# Do not set values here unless you understand the impact of doing so as you
|
|
||||||
+# may stop the virtual machine from booting.
|
|
||||||
+# To see the list of default parameters, enable hypervisor debug, create a
|
|
||||||
+# container and look for 'default-kernel-parameters' log entries.
|
|
||||||
+kernel_params = "@KERNELPARAMS@"
|
|
||||||
+
|
|
||||||
+# Default number of vCPUs per SB/VM:
|
|
||||||
+# unspecified or 0 --> will be set to @DEFVCPUS@
|
|
||||||
+# < 0 --> will be set to the actual number of physical cores
|
|
||||||
+# > 0 <= number of physical cores --> will be set to the specified number
|
|
||||||
+# > number of physical cores --> will be set to the actual number of physical cores
|
|
||||||
+default_vcpus = 1
|
|
||||||
+
|
|
||||||
+# Default maximum number of vCPUs per SB/VM:
|
|
||||||
+# unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number
|
|
||||||
+# of vCPUs supported by KVM if that number is exceeded
|
|
||||||
+# > 0 <= number of physical cores --> will be set to the specified number
|
|
||||||
+# > number of physical cores --> will be set to the actual number of physical cores or to the maximum number
|
|
||||||
+# of vCPUs supported by KVM if that number is exceeded
|
|
||||||
+# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when
|
|
||||||
+# the actual number of physical cores is greater than it.
|
|
||||||
+# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU
|
|
||||||
+# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs
|
|
||||||
+# can be added to a SB/VM, but the memory footprint will be big. Another example, with
|
|
||||||
+# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of
|
|
||||||
+# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable,
|
|
||||||
+# unless you know what are you doing.
|
|
||||||
+# NOTICE: on arm platform with gicv2 interrupt controller, set it to 8.
|
|
||||||
+default_maxvcpus = @DEFMAXVCPUS@
|
|
||||||
+
|
|
||||||
+# Bridges can be used to hot plug devices.
|
|
||||||
+# Limitations:
|
|
||||||
+# * Currently only pci bridges are supported
|
|
||||||
+# * Until 30 devices per bridge can be hot plugged.
|
|
||||||
+# * Until 5 PCI bridges can be cold plugged per VM.
|
|
||||||
+# This limitation could be a bug in the kernel
|
|
||||||
+# Default number of bridges per SB/VM:
|
|
||||||
+# unspecified or 0 --> will be set to @DEFBRIDGES@
|
|
||||||
+# > 1 <= 5 --> will be set to the specified number
|
|
||||||
+# > 5 --> will be set to 5
|
|
||||||
+default_bridges = @DEFBRIDGES@
|
|
||||||
+
|
|
||||||
+# Default memory size in MiB for SB/VM.
|
|
||||||
+# If unspecified then it will be set @DEFMEMSZ@ MiB.
|
|
||||||
+default_memory = @DEFMEMSZ@
|
|
||||||
+#
|
|
||||||
+# Default memory slots per SB/VM.
|
|
||||||
+# If unspecified then it will be set @DEFMEMSLOTS@.
|
|
||||||
+# This is will determine the times that memory will be hotadded to sandbox/VM.
|
|
||||||
+# memory_slots = @DEFMEMSLOTS@
|
|
||||||
+
|
|
||||||
+# The size in MiB will be plused to max memory of hypervisor.
|
|
||||||
+# It is the memory address space for the NVDIMM devie.
|
|
||||||
+# If set block storage driver (block_device_driver) to "nvdimm",
|
|
||||||
+# should set memory_offset to the size of block device.
|
|
||||||
+# Default 0
|
|
||||||
+# memory_offset = 0
|
|
||||||
+
|
|
||||||
+# Disable block device from being used for a container's rootfs.
|
|
||||||
+# In case of a storage driver like devicemapper where a container's
|
|
||||||
+# root file system is backed by a block device, the block device is passed
|
|
||||||
+# directly to the hypervisor for performance reasons.
|
|
||||||
+# This flag prevents the block device from being passed to the hypervisor,
|
|
||||||
+# 9pfs is used instead to pass the rootfs.
|
|
||||||
+disable_block_device_use = @DEFDISABLEBLOCK@
|
|
||||||
+
|
|
||||||
+# Block storage driver to be used for the hypervisor in case the container
|
|
||||||
+# rootfs is backed by a block device. This is virtio-scsi, virtio-blk
|
|
||||||
+# or nvdimm.
|
|
||||||
+block_device_driver = "@DEFBLOCKSTORAGEDRIVER_STRATOVIRT@"
|
|
||||||
+
|
|
||||||
+# Specifies cache-related options will be set to block devices or not.
|
|
||||||
+# Default false
|
|
||||||
+#block_device_cache_set = true
|
|
||||||
+
|
|
||||||
+# Specifies cache-related options for block devices.
|
|
||||||
+# Denotes whether use of O_DIRECT (bypass the host page cache) is enabled.
|
|
||||||
+# Default false
|
|
||||||
+# block_device_cache_direct = true
|
|
||||||
+
|
|
||||||
+# Specifies cache-related options for block devices.
|
|
||||||
+# Denotes whether flush requests for the device are ignored.
|
|
||||||
+# Default false
|
|
||||||
+# block_device_cache_noflush = true
|
|
||||||
+
|
|
||||||
+# Enable pre allocation of VM RAM, default false
|
|
||||||
+# Enabling this will result in lower container density
|
|
||||||
+# as all of the memory will be allocated and locked
|
|
||||||
+# This is useful when you want to reserve all the memory
|
|
||||||
+# upfront or in the cases where you want memory latencies
|
|
||||||
+# to be very predictable
|
|
||||||
+# Default false
|
|
||||||
+# enable_mem_prealloc = true
|
|
||||||
+
|
|
||||||
+# Enable huge pages for VM RAM, default false
|
|
||||||
+# Enabling this will result in the VM memory
|
|
||||||
+# being allocated using huge pages.
|
|
||||||
+# This is useful when you want to use vhost-user network
|
|
||||||
+# stacks within the container. This will automatically
|
|
||||||
+# result in memory pre allocation
|
|
||||||
+# enable_hugepages = true
|
|
||||||
+
|
|
||||||
+# Enable vIOMMU, default false
|
|
||||||
+# Enabling this will result in the VM having a vIOMMU device
|
|
||||||
+# This will also add the following options to the kernel's
|
|
||||||
+# command line: intel_iommu=on,iommu=pt
|
|
||||||
+# enable_iommu = true
|
|
||||||
+
|
|
||||||
+# Enable swap of vm memory. Default false.
|
|
||||||
+# The behaviour is undefined if mem_prealloc is also set to true
|
|
||||||
+# enable_swap = true
|
|
||||||
+
|
|
||||||
+# This option changes the default hypervisor and kernel parameters
|
|
||||||
+# to enable debug output where available.
|
|
||||||
+#
|
|
||||||
+# Default false
|
|
||||||
+# enable_debug = true
|
|
||||||
+
|
|
||||||
+# Disable the customizations done in the runtime when it detects
|
|
||||||
+# that it is running on top a VMM. This will result in the runtime
|
|
||||||
+# behaving as it would when running on bare metal.
|
|
||||||
+#
|
|
||||||
+# disable_nesting_checks = true
|
|
||||||
+
|
|
||||||
+# This is the msize used for 9p shares. It is the number of bytes
|
|
||||||
+# used for 9p packet payload.
|
|
||||||
+# msize_9p =
|
|
||||||
+
|
|
||||||
+# VFIO devices are hotplugged on a bridge by default.
|
|
||||||
+# Enable hotplugging on root bus. This may be required for devices with
|
|
||||||
+# a large PCI bar, as this is a current limitation with hotplugging on
|
|
||||||
+# a bridge.
|
|
||||||
+# Default false
|
|
||||||
+# hotplug_vfio_on_root_bus = true
|
|
||||||
+
|
|
||||||
+#
|
|
||||||
+# Default entropy source.
|
|
||||||
+# The path to a host source of entropy (including a real hardware RNG)
|
|
||||||
+# /dev/urandom and /dev/random are two main options.
|
|
||||||
+# Be aware that /dev/random is a blocking source of entropy. If the host
|
|
||||||
+# runs out of entropy, the VMs boot time will increase leading to get startup
|
|
||||||
+# timeouts.
|
|
||||||
+# The source of entropy /dev/urandom is non-blocking and provides a
|
|
||||||
+# generally acceptable source of entropy. It should work well for pretty much
|
|
||||||
+# all practical purposes.
|
|
||||||
+# entropy_source= ""
|
|
||||||
+
|
|
||||||
+# List of valid annotations values for entropy_source
|
|
||||||
+# The default if not set is empty (all annotations rejected.)
|
|
||||||
+# Your distribution recommends: @DEFVALIDENTROPYSOURCES@
|
|
||||||
+valid_entropy_sources = @DEFVALIDENTROPYSOURCES@
|
|
||||||
+
|
|
||||||
+# Path to OCI hook binaries in the *guest rootfs*.
|
|
||||||
+# This does not affect host-side hooks which must instead be added to
|
|
||||||
+# the OCI spec passed to the runtime.
|
|
||||||
+#
|
|
||||||
+# You can create a rootfs with hooks by customizing the osbuilder scripts:
|
|
||||||
+# https://github.com/kata-containers/kata-containers/tree/main/tools/osbuilder
|
|
||||||
+#
|
|
||||||
+# Hooks must be stored in a subdirectory of guest_hook_path according to their
|
|
||||||
+# hook type, i.e. "guest_hook_path/{prestart,poststart,poststop}".
|
|
||||||
+# The agent will scan these directories for executable files and add them, in
|
|
||||||
+# lexicographical order, to the lifecycle of the guest container.
|
|
||||||
+# Hooks are executed in the runtime namespace of the guest. See the official documentation:
|
|
||||||
+# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks
|
|
||||||
+# Warnings will be logged if any error is encountered will scanning for hooks,
|
|
||||||
+# but it will not abort container execution.
|
|
||||||
+# guest_hook_path = "/usr/share/oci/hooks"
|
|
||||||
+
|
|
||||||
+[factory]
|
|
||||||
+# VM templating support. Once enabled, new VMs are created from template
|
|
||||||
+# using vm cloning. They will share the same initial kernel, initramfs and
|
|
||||||
+# agent memory by mapping it readonly. It helps speeding up new container
|
|
||||||
+# creation and saves a lot of memory if there are many kata containers running
|
|
||||||
+# on the same host.
|
|
||||||
+#
|
|
||||||
+# When disabled, new VMs are created from scratch.
|
|
||||||
+#
|
|
||||||
+# Note: Requires "initrd=" to be set ("image=" is not supported).
|
|
||||||
+#
|
|
||||||
+# Default false
|
|
||||||
+#enable_template = true
|
|
||||||
+
|
|
||||||
+[agent.@PROJECT_TYPE@]
|
|
||||||
+# If enabled, make the agent display debug-level messages.
|
|
||||||
+# (default: disabled)
|
|
||||||
+#enable_debug = true
|
|
||||||
+
|
|
||||||
+# Enable agent tracing.
|
|
||||||
+#
|
|
||||||
+# If enabled, the default trace mode is "dynamic" and the
|
|
||||||
+# default trace type is "isolated". The trace mode and type are set
|
|
||||||
+# explicity with the `trace_type=` and `trace_mode=` options.
|
|
||||||
+#
|
|
||||||
+# Notes:
|
|
||||||
+#
|
|
||||||
+# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly
|
|
||||||
+# setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing`
|
|
||||||
+# will NOT activate agent tracing.
|
|
||||||
+#
|
|
||||||
+# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for
|
|
||||||
+# full details.
|
|
||||||
+#
|
|
||||||
+# (default: disabled)
|
|
||||||
+#enable_tracing = true
|
|
||||||
+#
|
|
||||||
+#trace_mode = "dynamic"
|
|
||||||
+#trace_type = "isolated"
|
|
||||||
+
|
|
||||||
+# Comma separated list of kernel modules and their parameters.
|
|
||||||
+# These modules will be loaded in the guest kernel using modprobe(8).
|
|
||||||
+# The following example can be used to load two kernel modules with parameters
|
|
||||||
+# - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"]
|
|
||||||
+# The first word is considered as the module name and the rest as its parameters.
|
|
||||||
+# Container will not be started when:
|
|
||||||
+# * A kernel module is specified and the modprobe command is not installed in the guest
|
|
||||||
+# or it fails loading the module.
|
|
||||||
+# * The module is not available in the guest or it doesn't met the guest kernel
|
|
||||||
+# requirements, like architecture and version.
|
|
||||||
+#
|
|
||||||
+kernel_modules=[]
|
|
||||||
+
|
|
||||||
+# Enable debug console.
|
|
||||||
+
|
|
||||||
+# If enabled, user can connect guest OS running inside hypervisor
|
|
||||||
+# through "kata-runtime exec <sandbox-id>" command
|
|
||||||
+
|
|
||||||
+#debug_console_enabled = true
|
|
||||||
+
|
|
||||||
+# Agent connection dialing timeout value in seconds
|
|
||||||
+# (default: 30)
|
|
||||||
+#dial_timeout = 30
|
|
||||||
+
|
|
||||||
+[netmon]
|
|
||||||
+# If enabled, the network monitoring process gets started when the
|
|
||||||
+# sandbox is created. This allows for the detection of some additional
|
|
||||||
+# network being added to the existing network namespace, after the
|
|
||||||
+# sandbox has been created.
|
|
||||||
+# (default: disabled)
|
|
||||||
+#enable_netmon = true
|
|
||||||
+
|
|
||||||
+# Specify the path to the netmon binary.
|
|
||||||
+path = "@NETMONPATH@"
|
|
||||||
+
|
|
||||||
+# If enabled, netmon messages will be sent to the system log
|
|
||||||
+# (default: disabled)
|
|
||||||
+#enable_debug = true
|
|
||||||
+
|
|
||||||
+[runtime]
|
|
||||||
+# If enabled, the runtime will log additional debug messages to the
|
|
||||||
+# system log
|
|
||||||
+# (default: disabled)
|
|
||||||
+#enable_debug = true
|
|
||||||
+#
|
|
||||||
+# Internetworking model
|
|
||||||
+# Determines how the VM should be connected to the
|
|
||||||
+# the container network interface
|
|
||||||
+# Options:
|
|
||||||
+#
|
|
||||||
+# - macvtap
|
|
||||||
+# Used when the Container network interface can be bridged using
|
|
||||||
+# macvtap.
|
|
||||||
+#
|
|
||||||
+# - none
|
|
||||||
+# Used when customize network. Only creates a tap device. No veth pair.
|
|
||||||
+#
|
|
||||||
+# - tcfilter
|
|
||||||
+# Uses tc filter rules to redirect traffic from the network interface
|
|
||||||
+# provided by plugin to a tap interface connected to the VM.
|
|
||||||
+#
|
|
||||||
+internetworking_model="@DEFNETWORKMODEL_STRATOVIRT@"
|
|
||||||
+
|
|
||||||
+# disable guest seccomp
|
|
||||||
+# Determines whether container seccomp profiles are passed to the virtual
|
|
||||||
+# machine and applied by the kata agent. If set to true, seccomp is not applied
|
|
||||||
+# within the guest
|
|
||||||
+# (default: true)
|
|
||||||
+disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
|
|
||||||
+
|
|
||||||
+# If enabled, the runtime will create opentracing.io traces and spans.
|
|
||||||
+# (See https://www.jaegertracing.io/docs/getting-started).
|
|
||||||
+# (default: disabled)
|
|
||||||
+#enable_tracing = true
|
|
||||||
+
|
|
||||||
+# Set the full url to the Jaeger HTTP Thrift collector.
|
|
||||||
+# The default if not set will be "http://localhost:14268/api/traces"
|
|
||||||
+#jaeger_endpoint = ""
|
|
||||||
+
|
|
||||||
+# Sets the username to be used if basic auth is required for Jaeger.
|
|
||||||
+#jaeger_user = ""
|
|
||||||
+
|
|
||||||
+# Sets the password to be used if basic auth is required for Jaeger.
|
|
||||||
+#jaeger_password = ""
|
|
||||||
+
|
|
||||||
+# If enabled, the runtime will not create a network namespace for shim and hypervisor processes.
|
|
||||||
+# This option may have some potential impacts to your host. It should only be used when you know what you're doing.
|
|
||||||
+# `disable_new_netns` conflicts with `enable_netmon`
|
|
||||||
+# `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only
|
|
||||||
+# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge
|
|
||||||
+# (like OVS) directly.
|
|
||||||
+# If you are using docker, `disable_new_netns` only works with `docker run --net=none`
|
|
||||||
+# (default: false)
|
|
||||||
+#disable_new_netns = true
|
|
||||||
+
|
|
||||||
+# if enable, the runtime will add all the kata processes inside one dedicated cgroup.
|
|
||||||
+# The container cgroups in the host are not created, just one single cgroup per sandbox.
|
|
||||||
+# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.
|
|
||||||
+# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.
|
|
||||||
+# The sandbox cgroup is constrained if there is no container type annotation.
|
|
||||||
+# See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType
|
|
||||||
+sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
|
|
||||||
+
|
|
||||||
+# Enabled experimental feature list, format: ["a", "b"].
|
|
||||||
+# Experimental features are features not stable enough for production,
|
|
||||||
+# they may break compatibility, and are prepared for a big version bump.
|
|
||||||
+# Supported experimental features:
|
|
||||||
+# (default: [])
|
|
||||||
+experimental=@DEFAULTEXPFEATURES@
|
|
||||||
+
|
|
||||||
+# If enabled, user can run pprof tools with shim v2 process through kata-monitor.
|
|
||||||
+# (default: false)
|
|
||||||
+# enable_pprof = true
|
|
||||||
--
|
|
||||||
2.21.1 (Apple Git-122.3)
|
|
||||||
|
|
||||||
@ -1,655 +0,0 @@
|
|||||||
From 45c8e108497eb93d69afd38e6281b837e65cf3ec Mon Sep 17 00:00:00 2001
|
|
||||||
From: Wei Gao <gaowei66@huawei.com>
|
|
||||||
Date: Mon, 9 Aug 2021 14:55:41 +0800
|
|
||||||
Subject: [PATCH 5/6] runtime: add the secure component "ozone" support for
|
|
||||||
hypervisor type stratovirt.
|
|
||||||
|
|
||||||
Signed-off-by: Wei Gao <gaowei66@huawei.com>
|
|
||||||
---
|
|
||||||
src/runtime/Makefile | 4 +
|
|
||||||
src/runtime/arch/amd64-options.mk | 4 +-
|
|
||||||
src/runtime/arch/arm64-options.mk | 4 +-
|
|
||||||
.../config/configuration-stratovirt.toml.in | 10 +
|
|
||||||
.../pkg/katautils/config-settings.go.in | 1 +
|
|
||||||
src/runtime/pkg/katautils/config.go | 18 +
|
|
||||||
src/runtime/virtcontainers/hypervisor.go | 3 +
|
|
||||||
src/runtime/virtcontainers/persist.go | 1 +
|
|
||||||
.../virtcontainers/persist/api/config.go | 3 +
|
|
||||||
src/runtime/virtcontainers/stratovirt.go | 309 ++++++++++++++----
|
|
||||||
10 files changed, 292 insertions(+), 65 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/src/runtime/Makefile b/src/runtime/Makefile
|
|
||||||
index ea2cd296..745bcc10 100644
|
|
||||||
--- a/src/runtime/Makefile
|
|
||||||
+++ b/src/runtime/Makefile
|
|
||||||
@@ -146,6 +146,8 @@ ACRNVALIDCTLPATHS := [\"$(ACRNCTLPATH)\"]
|
|
||||||
|
|
||||||
STRATOVIRTPATH = $(STRATOVIRTBINDIR)/$(STRATOVIRTCMD)
|
|
||||||
STRATOVIRTVALIDHYPERVISORPATHS := [\"$(STRATOVIRTPATH)\"]
|
|
||||||
+STRATOVIRTOZONEPATH = $(STRATOVIRTBINDIR)/$(STRATOVIRTOZONECMD)
|
|
||||||
+STRATOVIRTVALIDOZONEPATHS = [\"$(STRATOVIRTOZONEPATH)\"]
|
|
||||||
|
|
||||||
NETMONCMD := $(BIN_PREFIX)-netmon
|
|
||||||
NETMONPATH := $(PKGLIBEXECDIR)/$(NETMONCMD)
|
|
||||||
@@ -414,6 +416,8 @@ USER_VARS += FCJAILERPATH
|
|
||||||
USER_VARS += FCVALIDJAILERPATHS
|
|
||||||
USER_VARS += STRATOVIRTPATH
|
|
||||||
USER_VARS += STRATOVIRTVALIDHYPERVISORPATHS
|
|
||||||
+USER_VARS += STRATOVIRTOZONEPATH
|
|
||||||
+USER_VARS += STRATOVIRTVALIDOZONEPATHS
|
|
||||||
USER_VARS += SYSCONFIG
|
|
||||||
USER_VARS += IMAGENAME
|
|
||||||
USER_VARS += IMAGEPATH
|
|
||||||
diff --git a/src/runtime/arch/amd64-options.mk b/src/runtime/arch/amd64-options.mk
|
|
||||||
index ff2af9e6..4c6c329a 100644
|
|
||||||
--- a/src/runtime/arch/amd64-options.mk
|
|
||||||
+++ b/src/runtime/arch/amd64-options.mk
|
|
||||||
@@ -25,4 +25,6 @@ ACRNCTLCMD := acrnctl
|
|
||||||
CLHCMD := cloud-hypervisor
|
|
||||||
|
|
||||||
# stratovirt binary name
|
|
||||||
-STRATOVIRTCMD := stratovirt
|
|
||||||
\ No newline at end of file
|
|
||||||
+STRATOVIRTCMD := stratovirt
|
|
||||||
+# stratovirt's ozone binary name
|
|
||||||
+STRATOVIRTOZONECMD := ozone
|
|
||||||
\ No newline at end of file
|
|
||||||
diff --git a/src/runtime/arch/arm64-options.mk b/src/runtime/arch/arm64-options.mk
|
|
||||||
index 2ad3f657..5dfa2c80 100644
|
|
||||||
--- a/src/runtime/arch/arm64-options.mk
|
|
||||||
+++ b/src/runtime/arch/arm64-options.mk
|
|
||||||
@@ -21,4 +21,6 @@ FCJAILERCMD := jailer
|
|
||||||
CLHCMD := cloud-hypervisor
|
|
||||||
|
|
||||||
# stratovirt binary name
|
|
||||||
-STRATOVIRTCMD := stratovirt
|
|
||||||
\ No newline at end of file
|
|
||||||
+STRATOVIRTCMD := stratovirt
|
|
||||||
+# stratovirt's ozone binary name
|
|
||||||
+STRATOVIRTOZONECMD := ozone
|
|
||||||
\ No newline at end of file
|
|
||||||
diff --git a/src/runtime/cli/config/configuration-stratovirt.toml.in b/src/runtime/cli/config/configuration-stratovirt.toml.in
|
|
||||||
index 5c83c3c9..b557b71f 100644
|
|
||||||
--- a/src/runtime/cli/config/configuration-stratovirt.toml.in
|
|
||||||
+++ b/src/runtime/cli/config/configuration-stratovirt.toml.in
|
|
||||||
@@ -26,6 +26,16 @@ enable_annotations = @DEFENABLEANNOTATIONS@
|
|
||||||
# Your distribution recommends: @STRATOVIRTVALIDHYPERVISORPATHS@
|
|
||||||
valid_hypervisor_paths = @STRATOVIRTVALIDHYPERVISORPATHS@
|
|
||||||
|
|
||||||
+# Path for the ozone specific to stratovirt
|
|
||||||
+# If the ozone path is set, stratovirt will be launched in
|
|
||||||
+# ozone secure environment. It is disabled by default.
|
|
||||||
+# ozone_path = "@STRATOVIRTOZONEPATH@"
|
|
||||||
+
|
|
||||||
+# List of valid ozone path values for the hypervisor
|
|
||||||
+# Each member of the list can be a regular expression
|
|
||||||
+# The default if not set is empty (all annotations rejected.)
|
|
||||||
+# valid_jailer_paths = @STRATOVIRTVALIDOZONEPATHS@
|
|
||||||
+
|
|
||||||
# Optional space-separated list of options to pass to the guest kernel.
|
|
||||||
# For example, use `kernel_params = "vsyscall=emulate"` if you are having
|
|
||||||
# trouble running pre-2.15 glibc.
|
|
||||||
diff --git a/src/runtime/pkg/katautils/config-settings.go.in b/src/runtime/pkg/katautils/config-settings.go.in
|
|
||||||
index 7cd9138b..c168c608 100644
|
|
||||||
--- a/src/runtime/pkg/katautils/config-settings.go.in
|
|
||||||
+++ b/src/runtime/pkg/katautils/config-settings.go.in
|
|
||||||
@@ -17,6 +17,7 @@ var defaultInitrdPath = "/usr/share/kata-containers/kata-containers-initrd.img"
|
|
||||||
var defaultFirmwarePath = ""
|
|
||||||
var defaultMachineAccelerators = ""
|
|
||||||
var defaultCPUFeatures = ""
|
|
||||||
+var defaultOzonePath = "/usr/bin/ozone"
|
|
||||||
var systemdUnitName = "kata-containers.target"
|
|
||||||
|
|
||||||
const defaultKernelParams = ""
|
|
||||||
diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go
|
|
||||||
index f94ac4fd..828c2a43 100644
|
|
||||||
--- a/src/runtime/pkg/katautils/config.go
|
|
||||||
+++ b/src/runtime/pkg/katautils/config.go
|
|
||||||
@@ -92,6 +92,7 @@ type hypervisor struct {
|
|
||||||
FileBackedMemRootDir string `toml:"file_mem_backend"`
|
|
||||||
GuestHookPath string `toml:"guest_hook_path"`
|
|
||||||
GuestMemoryDumpPath string `toml:"guest_memory_dump_path"`
|
|
||||||
+ OzonePath string `toml:"ozone_path"`
|
|
||||||
HypervisorPathList []string `toml:"valid_hypervisor_paths"`
|
|
||||||
JailerPathList []string `toml:"valid_jailer_paths"`
|
|
||||||
CtlPathList []string `toml:"valid_ctlpaths"`
|
|
||||||
@@ -452,6 +453,16 @@ func (h hypervisor) getInitrdAndImage() (initrd string, image string, err error)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
+func (h hypervisor) ozonePath() (string, error) {
|
|
||||||
+ p := h.OzonePath
|
|
||||||
+
|
|
||||||
+ if h.OzonePath == "" {
|
|
||||||
+ return "", nil
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return ResolvePath(p)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
func (h hypervisor) getRxRateLimiterCfg() uint64 {
|
|
||||||
return h.RxRateLimiterMaxRate
|
|
||||||
}
|
|
||||||
@@ -877,6 +888,11 @@ func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
|
||||||
return vc.HypervisorConfig{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
+ ozone, err := h.ozonePath()
|
|
||||||
+ if err != nil {
|
|
||||||
+ return vc.HypervisorConfig{}, err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
kernel, err := h.kernel()
|
|
||||||
if err != nil {
|
|
||||||
return vc.HypervisorConfig{}, err
|
|
||||||
@@ -925,6 +941,7 @@ func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
|
||||||
KernelPath: kernel,
|
|
||||||
InitrdPath: initrd,
|
|
||||||
ImagePath: image,
|
|
||||||
+ OzonePath: ozone,
|
|
||||||
KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)),
|
|
||||||
HypervisorMachineType: machineType,
|
|
||||||
NumVCPUs: h.defaultVCPUs(),
|
|
||||||
@@ -1155,6 +1172,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig {
|
|
||||||
RxRateLimiterMaxRate: defaultRxRateLimiterMaxRate,
|
|
||||||
TxRateLimiterMaxRate: defaultTxRateLimiterMaxRate,
|
|
||||||
SGXEPCSize: defaultSGXEPCSize,
|
|
||||||
+ OzonePath: defaultOzonePath,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go
|
|
||||||
index 615baa80..04e14b4e 100644
|
|
||||||
--- a/src/runtime/virtcontainers/hypervisor.go
|
|
||||||
+++ b/src/runtime/virtcontainers/hypervisor.go
|
|
||||||
@@ -302,6 +302,9 @@ type HypervisorConfig struct {
|
|
||||||
// JailerPathList is the list of jailer paths names allowed in annotations
|
|
||||||
JailerPathList []string
|
|
||||||
|
|
||||||
+ // OzonePath is the ozone executable host path.
|
|
||||||
+ OzonePath string
|
|
||||||
+
|
|
||||||
// BlockDeviceDriver specifies the driver to be used for block device
|
|
||||||
// either VirtioSCSI or VirtioBlock with the default driver being defaultBlockDriver
|
|
||||||
BlockDeviceDriver string
|
|
||||||
diff --git a/src/runtime/virtcontainers/persist.go b/src/runtime/virtcontainers/persist.go
|
|
||||||
index 203495e8..ae499c97 100644
|
|
||||||
--- a/src/runtime/virtcontainers/persist.go
|
|
||||||
+++ b/src/runtime/virtcontainers/persist.go
|
|
||||||
@@ -219,6 +219,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
|
|
||||||
HypervisorCtlPathList: sconfig.HypervisorConfig.HypervisorCtlPathList,
|
|
||||||
JailerPath: sconfig.HypervisorConfig.JailerPath,
|
|
||||||
JailerPathList: sconfig.HypervisorConfig.JailerPathList,
|
|
||||||
+ OzonePath: sconfig.HypervisorConfig.OzonePath,
|
|
||||||
BlockDeviceDriver: sconfig.HypervisorConfig.BlockDeviceDriver,
|
|
||||||
HypervisorMachineType: sconfig.HypervisorConfig.HypervisorMachineType,
|
|
||||||
MemoryPath: sconfig.HypervisorConfig.MemoryPath,
|
|
||||||
diff --git a/src/runtime/virtcontainers/persist/api/config.go b/src/runtime/virtcontainers/persist/api/config.go
|
|
||||||
index 3bd5567d..88903723 100644
|
|
||||||
--- a/src/runtime/virtcontainers/persist/api/config.go
|
|
||||||
+++ b/src/runtime/virtcontainers/persist/api/config.go
|
|
||||||
@@ -76,6 +76,9 @@ type HypervisorConfig struct {
|
|
||||||
// JailerPathList is the list of jailer paths names allowed in annotations
|
|
||||||
JailerPathList []string
|
|
||||||
|
|
||||||
+ // OzonePath is the ozone executable host path.
|
|
||||||
+ OzonePath string
|
|
||||||
+
|
|
||||||
// BlockDeviceDriver specifies the driver to be used for block device
|
|
||||||
// either VirtioSCSI or VirtioBlock with the default driver being defaultBlockDriver
|
|
||||||
BlockDeviceDriver string
|
|
||||||
diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go
|
|
||||||
index 0f473e31..47daa817 100644
|
|
||||||
--- a/src/runtime/virtcontainers/stratovirt.go
|
|
||||||
+++ b/src/runtime/virtcontainers/stratovirt.go
|
|
||||||
@@ -3,6 +3,7 @@ package virtcontainers
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
+ "io/ioutil"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"path/filepath"
|
|
||||||
@@ -24,10 +25,15 @@ import (
|
|
||||||
otelTrace "go.opentelemetry.io/otel/trace"
|
|
||||||
)
|
|
||||||
|
|
||||||
-const defaultDummyMac = "22:33:44:aa:bb:"
|
|
||||||
-const mmioBlkCount = 4
|
|
||||||
-const mmioNetCount = 2
|
|
||||||
-const randomDevice = "/dev/urandom"
|
|
||||||
+const (
|
|
||||||
+ apiSocket = "qmp.socket"
|
|
||||||
+ debugSocket = "console.socket"
|
|
||||||
+ ozoneBaseDir = "/srv/ozone/stratovirt"
|
|
||||||
+ defaultDummyMac = "22:33:44:aa:bb:"
|
|
||||||
+ mmioBlkCount = 4
|
|
||||||
+ mmioNetCount = 2
|
|
||||||
+ randomDevice = "/dev/urandom"
|
|
||||||
+)
|
|
||||||
|
|
||||||
type stratovirtDev struct {
|
|
||||||
dev interface{}
|
|
||||||
@@ -40,10 +46,19 @@ type stratovirt struct {
|
|
||||||
sandbox *Sandbox
|
|
||||||
store persistapi.PersistDriver
|
|
||||||
config HypervisorConfig
|
|
||||||
+ rootfsPath string
|
|
||||||
+ kernelPath string
|
|
||||||
pid int
|
|
||||||
consolePath string
|
|
||||||
socketPath string
|
|
||||||
+ netNSPath string
|
|
||||||
qmpMonitorCh qmpChannel
|
|
||||||
+ ozoneRoot string
|
|
||||||
+ ozoneRes []string
|
|
||||||
+ useOzone bool
|
|
||||||
+ useImage bool
|
|
||||||
+ pidfile string
|
|
||||||
+ logfile string
|
|
||||||
devices []stratovirtDev
|
|
||||||
HotpluggedVCPUs []CPUDevice
|
|
||||||
mmioBlkSlots [mmioBlkCount]bool
|
|
||||||
@@ -66,10 +81,10 @@ func (s *stratovirt) trace(parent context.Context, name string) (otelTrace.Span,
|
|
||||||
return span, ctx
|
|
||||||
}
|
|
||||||
|
|
||||||
-func (s *stratovirt) getKernelCmdLine(useImage bool) string {
|
|
||||||
+func (s *stratovirt) getKernelCmdLine() string {
|
|
||||||
var params []string
|
|
||||||
|
|
||||||
- if useImage {
|
|
||||||
+ if s.useImage {
|
|
||||||
params = append(params, "root=/dev/vda")
|
|
||||||
}
|
|
||||||
|
|
||||||
@@ -100,14 +115,49 @@ func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS Net
|
|
||||||
|
|
||||||
s.id = id
|
|
||||||
s.config = *hypervisorConfig
|
|
||||||
-
|
|
||||||
- s.socketPath = filepath.Join(s.store.RunVMStoragePath(), id, "qmp.sock")
|
|
||||||
- s.consolePath = filepath.Join(s.store.RunVMStoragePath(), id, "console.sock")
|
|
||||||
+ if s.config.OzonePath == "" {
|
|
||||||
+ s.useOzone = false
|
|
||||||
+ s.pidfile = filepath.Join(s.store.RunVMStoragePath(), s.id, "pid")
|
|
||||||
+ s.logfile = filepath.Join(s.store.RunVMStoragePath(), s.id, "/stratovirt.log")
|
|
||||||
+ s.socketPath = filepath.Join(s.store.RunVMStoragePath(), id, apiSocket)
|
|
||||||
+ s.consolePath = filepath.Join(s.store.RunVMStoragePath(), id, debugSocket)
|
|
||||||
+ } else {
|
|
||||||
+ s.useOzone = true
|
|
||||||
+ s.ozoneRoot = filepath.Join(ozoneBaseDir, s.id)
|
|
||||||
+ s.pidfile = filepath.Join(s.ozoneRoot, "pid")
|
|
||||||
+ s.logfile = filepath.Join(s.ozoneRoot, "stratovirt.log")
|
|
||||||
+ s.socketPath = filepath.Join(s.ozoneRoot, apiSocket)
|
|
||||||
+ s.consolePath = filepath.Join(s.ozoneRoot, debugSocket)
|
|
||||||
+ }
|
|
||||||
+ s.netNSPath = networkNS.NetNsPath
|
|
||||||
s.qmpMonitorCh = qmpChannel{
|
|
||||||
ctx: s.ctx,
|
|
||||||
path: s.socketPath,
|
|
||||||
}
|
|
||||||
|
|
||||||
+ if kernelPath, err := s.config.KernelAssetPath(); err == nil {
|
|
||||||
+ s.kernelPath = kernelPath
|
|
||||||
+ s.ozoneRes = append(s.ozoneRes, s.kernelPath)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ initrdPath, err := s.config.InitrdAssetPath()
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if initrdPath == "" {
|
|
||||||
+ imagePath, err := s.config.ImageAssetPath()
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ s.useImage = true
|
|
||||||
+ s.rootfsPath = imagePath
|
|
||||||
+ } else {
|
|
||||||
+ s.useImage = false
|
|
||||||
+ s.rootfsPath = initrdPath
|
|
||||||
+ }
|
|
||||||
+ s.ozoneRes = append(s.ozoneRes, s.rootfsPath)
|
|
||||||
+
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
@@ -134,48 +184,43 @@ func (s *stratovirt) waitSandBoxStarted(timeout int) error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
-func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error {
|
|
||||||
- span, _ := s.trace(ctx, "startSandbox")
|
|
||||||
- defer span.End()
|
|
||||||
-
|
|
||||||
+func (s *stratovirt) createbaseParams() []string {
|
|
||||||
var params []string
|
|
||||||
- var use_image bool
|
|
||||||
+
|
|
||||||
params = append(params, "-name", fmt.Sprintf("sandbox-%s", s.id))
|
|
||||||
- params = append(params, "-qmp", fmt.Sprintf("unix:%s,server,nowait", s.socketPath))
|
|
||||||
+ params = append(params, "-append", s.getKernelCmdLine())
|
|
||||||
+ params = append(params, "-smp", fmt.Sprintf("%d", s.config.NumVCPUs))
|
|
||||||
+ params = append(params, "-m", fmt.Sprintf("%d", uint64(s.config.MemorySize)))
|
|
||||||
+ params = append(params, "-device", "virtio-serial-device")
|
|
||||||
+ params = append(params, "-device", "virtconsole,chardev=charconsole0,id=virtioconsole0")
|
|
||||||
+ params = append(params, "-object", fmt.Sprintf("rng-random,id=objrng0,filename=%s", randomDevice))
|
|
||||||
+ params = append(params, "-device", "virtio-rng-device,rng=objrng0")
|
|
||||||
|
|
||||||
- if kernelPath, err := s.config.KernelAssetPath(); err == nil {
|
|
||||||
- params = append(params, "-kernel", kernelPath)
|
|
||||||
- }
|
|
||||||
+ // daemonize
|
|
||||||
+ params = append(params, "-daemonize")
|
|
||||||
|
|
||||||
- initrdPath, err := s.config.InitrdAssetPath()
|
|
||||||
- if err != nil {
|
|
||||||
- return err
|
|
||||||
+ return params
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) createOzoneParams(params []string) ([]string, error) {
|
|
||||||
+ params = append(params, "-qmp", fmt.Sprintf("unix:%s,server,nowait", apiSocket))
|
|
||||||
+ params = append(params, "-chardev", fmt.Sprintf("socket,id=charconsole0,path=%s,server,nowait", debugSocket))
|
|
||||||
+ params = append(params, "-kernel", filepath.Base(s.kernelPath))
|
|
||||||
+ params = append(params, "-pidfile", filepath.Base(s.pidfile))
|
|
||||||
+
|
|
||||||
+ // append logfile only on debug
|
|
||||||
+ if s.config.Debug {
|
|
||||||
+ params = append(params, "-D", filepath.Base(s.logfile))
|
|
||||||
}
|
|
||||||
|
|
||||||
- if initrdPath == "" {
|
|
||||||
- imagePath, err := s.config.ImageAssetPath()
|
|
||||||
- if err != nil {
|
|
||||||
- return err
|
|
||||||
- }
|
|
||||||
- use_image = true
|
|
||||||
+ if s.useImage {
|
|
||||||
s.mmioBlkSlots[0] = true
|
|
||||||
params = append(params, "-device", "virtio-blk-device,drive=rootfs")
|
|
||||||
- params = append(params, "-drive", fmt.Sprintf("id=rootfs,file=%s,direct=off", imagePath))
|
|
||||||
+ params = append(params, "-drive", fmt.Sprintf("id=rootfs,file=%s,direct=off", filepath.Base(s.rootfsPath)))
|
|
||||||
} else {
|
|
||||||
- use_image = false
|
|
||||||
- params = append(params, "-initrd", initrdPath)
|
|
||||||
+ params = append(params, "-initrd", filepath.Base(s.rootfsPath))
|
|
||||||
}
|
|
||||||
|
|
||||||
- params = append(params, "-append", s.getKernelCmdLine(use_image))
|
|
||||||
- params = append(params, "-smp", fmt.Sprintf("%d", s.config.NumVCPUs))
|
|
||||||
- params = append(params, "-m", fmt.Sprintf("%d", uint64(s.config.MemorySize)))
|
|
||||||
- params = append(params, "-device", "virtio-serial-device")
|
|
||||||
- params = append(params, "-device", "virtconsole,chardev=charconsole0,id=virtioconsole0")
|
|
||||||
- params = append(params, "-object", fmt.Sprintf("rng-random,id=objrng0,filename=%s", randomDevice))
|
|
||||||
- params = append(params, "-device", "virtio-rng-device,rng=objrng0")
|
|
||||||
- params = append(params, "-chardev", fmt.Sprintf("socket,id=charconsole0,path=%s,server,nowait", s.consolePath))
|
|
||||||
- params = append(params, "-pidfile", filepath.Join(s.store.RunVMStoragePath(), s.id, "pid"))
|
|
||||||
-
|
|
||||||
// add devices to cmdline
|
|
||||||
for _, d := range s.devices {
|
|
||||||
switch v := d.dev.(type) {
|
|
||||||
@@ -188,8 +233,9 @@ func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error {
|
|
||||||
case config.BlockDrive:
|
|
||||||
id := v.ID
|
|
||||||
path := v.File
|
|
||||||
- params = append(params, "-device", fmt.Sprintf("virtio-blk-device, drive=%s", id))
|
|
||||||
- params = append(params, "-drive", fmt.Sprintf("id=%s,file=%s", id, path))
|
|
||||||
+ s.ozoneRes = append(s.ozoneRes, path)
|
|
||||||
+ params = append(params, "-device", fmt.Sprintf("virtio-blk-device,drive=%s", id))
|
|
||||||
+ params = append(params, "-drive", fmt.Sprintf("id=%s,file=%s", id, filepath.Base(path)))
|
|
||||||
case types.VSock:
|
|
||||||
v.VhostFd.Close()
|
|
||||||
params = append(params, "-device", fmt.Sprintf("vhost-vsock-device,id=vsock-id,guest-cid=%d", v.ContextID))
|
|
||||||
@@ -198,42 +244,125 @@ func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
- // daemonize
|
|
||||||
- params = append(params, "-daemonize")
|
|
||||||
+ return params, nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) createParams(params []string) ([]string, error) {
|
|
||||||
+ params = append(params, "-qmp", fmt.Sprintf("unix:%s,server,nowait", s.socketPath))
|
|
||||||
+ params = append(params, "-chardev", fmt.Sprintf("socket,id=charconsole0,path=%s,server,nowait", s.consolePath))
|
|
||||||
+ params = append(params, "-kernel", s.kernelPath)
|
|
||||||
+ params = append(params, "-pidfile", s.pidfile)
|
|
||||||
|
|
||||||
// append logfile only on debug
|
|
||||||
if s.config.Debug {
|
|
||||||
- dir := filepath.Join(s.store.RunVMStoragePath(), s.id)
|
|
||||||
- params = append(params, "-D", fmt.Sprintf("%s/stratovirt.log", dir))
|
|
||||||
+ params = append(params, "-D", s.logfile)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if s.useImage {
|
|
||||||
+ s.mmioBlkSlots[0] = true
|
|
||||||
+ params = append(params, "-device", "virtio-blk-device,drive=rootfs")
|
|
||||||
+ params = append(params, "-drive", fmt.Sprintf("id=rootfs,file=%s,direct=off", s.rootfsPath))
|
|
||||||
+ } else {
|
|
||||||
+ params = append(params, "-initrd", s.rootfsPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
- dir := filepath.Join(s.store.RunVMStoragePath(), s.id)
|
|
||||||
- err = os.MkdirAll(dir, DirMode)
|
|
||||||
+ // add devices to cmdline
|
|
||||||
+ for _, d := range s.devices {
|
|
||||||
+ switch v := d.dev.(type) {
|
|
||||||
+ case Endpoint:
|
|
||||||
+ name := v.Name()
|
|
||||||
+ mac := v.HardwareAddr()
|
|
||||||
+ tapName := v.NetworkPair().TapInterface.TAPIface.Name
|
|
||||||
+ params = append(params, "-device", fmt.Sprintf("virtio-net-device,netdev=%s,id=%s,mac=%s", name, name, mac))
|
|
||||||
+ params = append(params, "-netdev", fmt.Sprintf("tap,id=%s,ifname=%s", name, tapName))
|
|
||||||
+ case config.BlockDrive:
|
|
||||||
+ id := v.ID
|
|
||||||
+ path := v.File
|
|
||||||
+ params = append(params, "-device", fmt.Sprintf("virtio-blk-device,drive=%s", id))
|
|
||||||
+ params = append(params, "-drive", fmt.Sprintf("id=%s,file=%s", id, path))
|
|
||||||
+ case types.VSock:
|
|
||||||
+ v.VhostFd.Close()
|
|
||||||
+ params = append(params, "-device", fmt.Sprintf("vhost-vsock-device,id=vsock-id,guest-cid=%d", v.ContextID))
|
|
||||||
+ default:
|
|
||||||
+ s.Logger().Error("Adding device type is unsupported")
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return params, nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error {
|
|
||||||
+ span, _ := s.trace(ctx, "startSandbox")
|
|
||||||
+ defer span.End()
|
|
||||||
+
|
|
||||||
+ var err error
|
|
||||||
+ var cmd *exec.Cmd
|
|
||||||
+
|
|
||||||
+ params := s.createbaseParams()
|
|
||||||
+
|
|
||||||
+ stratovirtBinPath, err := s.config.HypervisorAssetPath()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
- defer func() {
|
|
||||||
+
|
|
||||||
+ if s.useOzone {
|
|
||||||
+ var ozoneParams []string
|
|
||||||
+ extend_params, err := s.createOzoneParams(params)
|
|
||||||
if err != nil {
|
|
||||||
- if err := os.RemoveAll(dir); err != nil {
|
|
||||||
- s.Logger().WithError(err).Error("Fail to clean up vm dir %s", dir)
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ ozoneParams = append(ozoneParams, "-exec-file", stratovirtBinPath)
|
|
||||||
+ ozoneParams = append(ozoneParams, "-name", s.id)
|
|
||||||
+ ozoneParams = append(ozoneParams, "-gid", "0")
|
|
||||||
+ ozoneParams = append(ozoneParams, "-uid", "0")
|
|
||||||
+ if s.netNSPath != "" {
|
|
||||||
+ ozoneParams = append(ozoneParams, "-netns", s.netNSPath)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ ozoneParams = append(ozoneParams, "-source")
|
|
||||||
+ ozoneParams = append(ozoneParams, s.ozoneRes...)
|
|
||||||
+
|
|
||||||
+ defer func() {
|
|
||||||
+ if err != nil {
|
|
||||||
+ ozoneParams = append(ozoneParams, "-clean-resource")
|
|
||||||
+ cmd = exec.CommandContext(s.ctx, s.config.OzonePath, ozoneParams...)
|
|
||||||
+ if err := cmd.Run(); err != nil {
|
|
||||||
+ s.Logger().WithError(err).Error("Failed to clean up ozone dir %s", s.ozoneRoot)
|
|
||||||
+ }
|
|
||||||
}
|
|
||||||
+ }()
|
|
||||||
+
|
|
||||||
+ ozoneParams = append(ozoneParams, "--")
|
|
||||||
+ ozoneParams = append(ozoneParams, extend_params...)
|
|
||||||
+ cmd = exec.CommandContext(s.ctx, s.config.OzonePath, ozoneParams...)
|
|
||||||
+ s.Logger().Info("StratoVirt/Ozone start with params: ", cmd)
|
|
||||||
+ } else {
|
|
||||||
+ params, err = s.createParams(params)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
}
|
|
||||||
- }()
|
|
||||||
|
|
||||||
- binPath, err := s.config.HypervisorAssetPath()
|
|
||||||
- if err != nil {
|
|
||||||
- s.Logger().WithField("Fail to get hypervisor bin path", err).Error()
|
|
||||||
- return err
|
|
||||||
- }
|
|
||||||
+ dir := filepath.Join(s.store.RunVMStoragePath(), s.id)
|
|
||||||
+ err = os.MkdirAll(dir, DirMode)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ defer func() {
|
|
||||||
+ if err != nil {
|
|
||||||
+ if err := os.RemoveAll(dir); err != nil {
|
|
||||||
+ s.Logger().WithError(err).Error("Fail to clean up vm dir %s", dir)
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ }()
|
|
||||||
|
|
||||||
- cmd := exec.CommandContext(s.ctx, binPath, params...)
|
|
||||||
- s.Logger().Info("StratoVirt start with params: ", cmd)
|
|
||||||
+ cmd = exec.CommandContext(s.ctx, stratovirtBinPath, params...)
|
|
||||||
+ s.Logger().Info("StratoVirt start with params: ", cmd)
|
|
||||||
+ }
|
|
||||||
|
|
||||||
if err := cmd.Start(); err != nil {
|
|
||||||
s.Logger().WithField("Error starting hypervisor, please check the params", err).Error()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
- s.pid = cmd.Process.Pid
|
|
||||||
|
|
||||||
if err = s.waitSandBoxStarted(timeout); err != nil {
|
|
||||||
return err
|
|
||||||
@@ -420,6 +549,7 @@ func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op opera
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *stratovirt) hotplugBlk(drive *config.BlockDrive, op operation) (err error) {
|
|
||||||
+ var filePath string
|
|
||||||
err = s.qmpSetup()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
@@ -427,13 +557,18 @@ func (s *stratovirt) hotplugBlk(drive *config.BlockDrive, op operation) (err err
|
|
||||||
|
|
||||||
switch op {
|
|
||||||
case addDevice:
|
|
||||||
- driver := "virtio-blk-pci"
|
|
||||||
+ driver := "virtio-blk-mmio"
|
|
||||||
+ if s.useOzone {
|
|
||||||
+ filePath, err = s.updateOzoneRes(drive.File, true)
|
|
||||||
+ } else {
|
|
||||||
+ filePath = drive.File
|
|
||||||
+ }
|
|
||||||
slot, err := s.getDevSlot(drive.VirtPath, false)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("Could not get unused slot for %q", drive.VirtPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
- if err := s.qmpMonitorCh.qmp.ExecuteBlockdevAdd(s.qmpMonitorCh.ctx, drive.File, drive.ID, false); err != nil {
|
|
||||||
+ if err := s.qmpMonitorCh.qmp.ExecuteBlockdevAdd(s.qmpMonitorCh.ctx, filePath, drive.ID, false); err != nil {
|
|
||||||
s.getDevSlot(drive.VirtPath, true)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
@@ -443,6 +578,9 @@ func (s *stratovirt) hotplugBlk(drive *config.BlockDrive, op operation) (err err
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
case removeDevice:
|
|
||||||
+ if s.useOzone {
|
|
||||||
+ s.updateOzoneRes(drive.File, false)
|
|
||||||
+ }
|
|
||||||
if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, drive.ID); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
@@ -582,17 +720,62 @@ func (s *stratovirt) getThreadIDs(ctx context.Context) (vcpuThreadIDs, error) {
|
|
||||||
return tid, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
+func (s *stratovirt) updateOzoneRes(src string, add bool) (string, error) {
|
|
||||||
+ dst := filepath.Join(s.ozoneRoot, filepath.Base(src))
|
|
||||||
+ if add {
|
|
||||||
+ if err := bindMount(context.Background(), src, dst, false, "slave"); err != nil {
|
|
||||||
+ s.Logger().WithField("bindMount failed", err).Error()
|
|
||||||
+ return "", err
|
|
||||||
+ }
|
|
||||||
+ } else {
|
|
||||||
+ syscall.Unmount(dst, syscall.MNT_DETACH)
|
|
||||||
+ }
|
|
||||||
+ return filepath.Base(src), nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) cleanOzoneRes() {
|
|
||||||
+ s.updateOzoneRes(s.rootfsPath, false)
|
|
||||||
+ s.updateOzoneRes(s.kernelPath, false)
|
|
||||||
+
|
|
||||||
+ if err := os.RemoveAll(s.ozoneRoot); err != nil {
|
|
||||||
+ s.Logger().WithField("cleanupOzone failed", err).Error()
|
|
||||||
+ }
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
func (s *stratovirt) cleanup(ctx context.Context) error {
|
|
||||||
span, _ := s.trace(ctx, "cleanup")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
s.qmpTeardown()
|
|
||||||
+ if s.useOzone {
|
|
||||||
+ s.cleanOzoneRes()
|
|
||||||
+ }
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *stratovirt) getPids() []int {
|
|
||||||
- return []int{s.pid}
|
|
||||||
+ var pids []int
|
|
||||||
+ if s.pid != 0 {
|
|
||||||
+ pids = append(pids, s.pid)
|
|
||||||
+ } else {
|
|
||||||
+ pid, err := ioutil.ReadFile(s.pidfile)
|
|
||||||
+ if err != nil {
|
|
||||||
+ s.Logger().WithError(err).Error("Read pid file failed.")
|
|
||||||
+ return []int{0}
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ p, err := strconv.Atoi(strings.Trim(string(pid), "\n\t "))
|
|
||||||
+ if err != nil {
|
|
||||||
+ s.Logger().WithError(err).Error("Get pid from pid file failed.")
|
|
||||||
+ return []int{0}
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ pids = append(pids, p)
|
|
||||||
+ s.pid = p
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return pids
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *stratovirt) getVirtioFsPid() *int {
|
|
||||||
--
|
|
||||||
2.21.1 (Apple Git-122.3)
|
|
||||||
|
|
||||||
@ -1,295 +0,0 @@
|
|||||||
From 77ed6fefe70edde63b01d797b76f389bc82bb1a0 Mon Sep 17 00:00:00 2001
|
|
||||||
From: Wei Gao <gaowei66@huawei.com>
|
|
||||||
Date: Mon, 9 Aug 2021 14:57:06 +0800
|
|
||||||
Subject: [PATCH 6/6] factory: add the template factory support for hypervisor
|
|
||||||
type stratovirt.
|
|
||||||
|
|
||||||
Signed-off-by: Wei Gao <gaowei66@huawei.com>
|
|
||||||
---
|
|
||||||
src/runtime/pkg/katautils/config.go | 2 +-
|
|
||||||
.../factory/template/template.go | 21 +++--
|
|
||||||
src/runtime/virtcontainers/kata_agent.go | 7 +-
|
|
||||||
src/runtime/virtcontainers/stratovirt.go | 89 +++++++++++++++++--
|
|
||||||
src/runtime/virtcontainers/vm.go | 28 ++++--
|
|
||||||
5 files changed, 125 insertions(+), 22 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go
|
|
||||||
index 828c2a43..718677b4 100644
|
|
||||||
--- a/src/runtime/pkg/katautils/config.go
|
|
||||||
+++ b/src/runtime/pkg/katautils/config.go
|
|
||||||
@@ -1363,7 +1363,7 @@ func checkNetNsConfig(config oci.RuntimeConfig) error {
|
|
||||||
// checkFactoryConfig ensures the VM factory configuration is valid.
|
|
||||||
func checkFactoryConfig(config oci.RuntimeConfig) error {
|
|
||||||
if config.FactoryConfig.Template {
|
|
||||||
- if config.HypervisorConfig.InitrdPath == "" {
|
|
||||||
+ if config.HypervisorConfig.InitrdPath == "" && (config.HypervisorType != vc.StratovirtHypervisor) {
|
|
||||||
return errors.New("Factory option enable_template requires an initrd image")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
diff --git a/src/runtime/virtcontainers/factory/template/template.go b/src/runtime/virtcontainers/factory/template/template.go
|
|
||||||
index 66070126..02497097 100644
|
|
||||||
--- a/src/runtime/virtcontainers/factory/template/template.go
|
|
||||||
+++ b/src/runtime/virtcontainers/factory/template/template.go
|
|
||||||
@@ -96,11 +96,15 @@ func (t *template) prepareTemplateFiles() error {
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
- flags := uintptr(syscall.MS_NOSUID | syscall.MS_NODEV)
|
|
||||||
- opts := fmt.Sprintf("size=%dM", t.config.HypervisorConfig.MemorySize+templateDeviceStateSize)
|
|
||||||
- if err = syscall.Mount("tmpfs", t.statePath, "tmpfs", flags, opts); err != nil {
|
|
||||||
- t.close()
|
|
||||||
- return err
|
|
||||||
+
|
|
||||||
+ // If use hypervisor stratovirt, no need to create template path with ramdisk.
|
|
||||||
+ if t.config.HypervisorType != vc.StratovirtHypervisor {
|
|
||||||
+ flags := uintptr(syscall.MS_NOSUID | syscall.MS_NODEV)
|
|
||||||
+ opts := fmt.Sprintf("size=%dM", t.config.HypervisorConfig.MemorySize+templateDeviceStateSize)
|
|
||||||
+ if err = syscall.Mount("tmpfs", t.statePath, "tmpfs", flags, opts); err != nil {
|
|
||||||
+ t.close()
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
}
|
|
||||||
f, err := os.Create(t.statePath + "/memory")
|
|
||||||
if err != nil {
|
|
||||||
@@ -126,8 +130,11 @@ func (t *template) createTemplateVM(ctx context.Context) error {
|
|
||||||
}
|
|
||||||
defer vm.Stop(ctx)
|
|
||||||
|
|
||||||
- if err = vm.Disconnect(ctx); err != nil {
|
|
||||||
- return err
|
|
||||||
+ // Create template on hypervisor stratovirt, don't have connection with agent.
|
|
||||||
+ if config.HypervisorType != vc.StratovirtHypervisor {
|
|
||||||
+ if err = vm.Disconnect(ctx); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sleep a bit to let the agent grpc server clean up
|
|
||||||
diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go
|
|
||||||
index 13d31658..bc882c70 100644
|
|
||||||
--- a/src/runtime/virtcontainers/kata_agent.go
|
|
||||||
+++ b/src/runtime/virtcontainers/kata_agent.go
|
|
||||||
@@ -1306,8 +1306,11 @@ func (k *kataAgent) buildContainerRootfs(ctx context.Context, sandbox *Sandbox,
|
|
||||||
// TODO: remove dependency on shared fs path. shared fs is just one kind of storage source.
|
|
||||||
// we should not always use shared fs path for all kinds of storage. Instead, all storage
|
|
||||||
// should be bind mounted to a tmpfs path for containers to use.
|
|
||||||
- if err := os.MkdirAll(filepath.Join(getMountPath(c.sandbox.id), c.id, c.rootfsSuffix), DirMode); err != nil {
|
|
||||||
- return nil, err
|
|
||||||
+ // If boot from template on stratovirt, no need to mkdir mount path.
|
|
||||||
+ if !((sandbox.config.HypervisorType == StratovirtHypervisor) && sandbox.config.HypervisorConfig.BootFromTemplate) {
|
|
||||||
+ if err := os.MkdirAll(filepath.Join(getMountPath(c.sandbox.id), c.id, c.rootfsSuffix), DirMode); err != nil {
|
|
||||||
+ return nil, err
|
|
||||||
+ }
|
|
||||||
}
|
|
||||||
return rootfs, nil
|
|
||||||
}
|
|
||||||
diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go
|
|
||||||
index 47daa817..e9b2ba85 100644
|
|
||||||
--- a/src/runtime/virtcontainers/stratovirt.go
|
|
||||||
+++ b/src/runtime/virtcontainers/stratovirt.go
|
|
||||||
@@ -48,6 +48,7 @@ type stratovirt struct {
|
|
||||||
config HypervisorConfig
|
|
||||||
rootfsPath string
|
|
||||||
kernelPath string
|
|
||||||
+ templatePath string
|
|
||||||
pid int
|
|
||||||
consolePath string
|
|
||||||
socketPath string
|
|
||||||
@@ -115,7 +116,7 @@ func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS Net
|
|
||||||
|
|
||||||
s.id = id
|
|
||||||
s.config = *hypervisorConfig
|
|
||||||
- if s.config.OzonePath == "" {
|
|
||||||
+ if (s.config.OzonePath == "") || s.config.BootToBeTemplate {
|
|
||||||
s.useOzone = false
|
|
||||||
s.pidfile = filepath.Join(s.store.RunVMStoragePath(), s.id, "pid")
|
|
||||||
s.logfile = filepath.Join(s.store.RunVMStoragePath(), s.id, "/stratovirt.log")
|
|
||||||
@@ -129,6 +130,20 @@ func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS Net
|
|
||||||
s.socketPath = filepath.Join(s.ozoneRoot, apiSocket)
|
|
||||||
s.consolePath = filepath.Join(s.ozoneRoot, debugSocket)
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+ if s.config.VMid != "" && s.useOzone {
|
|
||||||
+ // Make sure the symlinks do not exist
|
|
||||||
+ os.RemoveAll(s.ozoneRoot)
|
|
||||||
+ ozoneVmRoot := filepath.Join(ozoneBaseDir, s.config.VMid)
|
|
||||||
+ if err := os.Symlink(ozoneVmRoot, s.ozoneRoot); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if s.config.BootFromTemplate || s.config.BootToBeTemplate {
|
|
||||||
+ s.templatePath = strings.Replace(s.config.DevicesStatePath, "/state", "", -1)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
s.netNSPath = networkNS.NetNsPath
|
|
||||||
s.qmpMonitorCh = qmpChannel{
|
|
||||||
ctx: s.ctx,
|
|
||||||
@@ -221,6 +236,12 @@ func (s *stratovirt) createOzoneParams(params []string) ([]string, error) {
|
|
||||||
params = append(params, "-initrd", filepath.Base(s.rootfsPath))
|
|
||||||
}
|
|
||||||
|
|
||||||
+ // handle boot from template
|
|
||||||
+ if s.config.BootFromTemplate {
|
|
||||||
+ s.ozoneRes = append(s.ozoneRes, s.templatePath)
|
|
||||||
+ params = append(params, "-incoming", fmt.Sprintf("file:%s", filepath.Base(s.templatePath)))
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
// add devices to cmdline
|
|
||||||
for _, d := range s.devices {
|
|
||||||
switch v := d.dev.(type) {
|
|
||||||
@@ -266,6 +287,11 @@ func (s *stratovirt) createParams(params []string) ([]string, error) {
|
|
||||||
params = append(params, "-initrd", s.rootfsPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
+ // handle boot from template
|
|
||||||
+ if s.config.BootFromTemplate {
|
|
||||||
+ params = append(params, "-incoming", fmt.Sprintf("file:%s", s.templatePath))
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
// add devices to cmdline
|
|
||||||
for _, d := range s.devices {
|
|
||||||
switch v := d.dev.(type) {
|
|
||||||
@@ -410,14 +436,55 @@ func (s *stratovirt) stopSandbox(ctx context.Context, force bool) error {
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *stratovirt) pauseSandbox(ctx context.Context) error {
|
|
||||||
- return nil
|
|
||||||
+ span, _ := s.trace(ctx, "pauseSandbox")
|
|
||||||
+ defer span.End()
|
|
||||||
+
|
|
||||||
+ return s.togglePauseSandbox(ctx, true)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *stratovirt) saveSandbox() error {
|
|
||||||
+ s.Logger().Info("save sandbox")
|
|
||||||
+
|
|
||||||
+ err := s.qmpSetup()
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ // BootToBeTemplate sets the VM to be a template that other VMs can can clone from.
|
|
||||||
+ // We would want to bypass shared memory when saving VM to local file through migrate.
|
|
||||||
+ if s.config.BootToBeTemplate {
|
|
||||||
+ err = s.qmpMonitorCh.qmp.ExecSetMigrateArguments(s.qmpMonitorCh.ctx, fmt.Sprintf("file:%s", s.templatePath))
|
|
||||||
+ if err != nil {
|
|
||||||
+ s.Logger().WithError(err).Error("exec migration")
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *stratovirt) resumeSandbox(ctx context.Context) error {
|
|
||||||
+ span, _ := s.trace(ctx, "resumeSandbox")
|
|
||||||
+ defer span.End()
|
|
||||||
+
|
|
||||||
+ return s.togglePauseSandbox(ctx, false)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *stratovirt) togglePauseSandbox(ctx context.Context, pause bool) error {
|
|
||||||
+ span, _ := s.trace(ctx, "togglePauseSandbox")
|
|
||||||
+ defer span.End()
|
|
||||||
+
|
|
||||||
+ err := s.qmpSetup()
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if pause {
|
|
||||||
+ s.qmpMonitorCh.qmp.ExecuteStop(s.qmpMonitorCh.ctx)
|
|
||||||
+ } else {
|
|
||||||
+ s.qmpMonitorCh.qmp.ExecuteCont(s.qmpMonitorCh.ctx)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
@@ -734,11 +801,23 @@ func (s *stratovirt) updateOzoneRes(src string, add bool) (string, error) {
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *stratovirt) cleanOzoneRes() {
|
|
||||||
- s.updateOzoneRes(s.rootfsPath, false)
|
|
||||||
- s.updateOzoneRes(s.kernelPath, false)
|
|
||||||
+ // Umount all resource in ozoneRoot
|
|
||||||
+ if dir, err := ioutil.ReadDir(s.ozoneRoot); err == nil {
|
|
||||||
+ for _, file := range dir {
|
|
||||||
+ syscall.Unmount(filepath.Join(s.ozoneRoot, file.Name()), syscall.MNT_DETACH)
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
|
|
||||||
if err := os.RemoveAll(s.ozoneRoot); err != nil {
|
|
||||||
- s.Logger().WithField("cleanupOzone failed", err).Error()
|
|
||||||
+ s.Logger().WithField("cleanup Ozone failed", err).Error()
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ // If have VMid, the VM is boot from template. ozoneVmRoot also need clean.
|
|
||||||
+ if s.config.VMid != "" {
|
|
||||||
+ ozoneVmRoot := filepath.Join(ozoneBaseDir, s.config.VMid)
|
|
||||||
+ if err := os.RemoveAll(ozoneVmRoot); err != nil {
|
|
||||||
+ s.Logger().WithField("cleanup Ozone failed", err).Error()
|
|
||||||
+ }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
diff --git a/src/runtime/virtcontainers/vm.go b/src/runtime/virtcontainers/vm.go
|
|
||||||
index e6f02b6e..c4f9df73 100644
|
|
||||||
--- a/src/runtime/virtcontainers/vm.go
|
|
||||||
+++ b/src/runtime/virtcontainers/vm.go
|
|
||||||
@@ -142,13 +142,19 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) {
|
|
||||||
}()
|
|
||||||
|
|
||||||
// 4. check agent aliveness
|
|
||||||
- // VMs booted from template are paused, do not check
|
|
||||||
- if !config.HypervisorConfig.BootFromTemplate {
|
|
||||||
+ // On hypervisor StratoVirt, VMs booted from template are running, check agent
|
|
||||||
+ // On other hypervisors, VMs booted from template are paused, do not check
|
|
||||||
+ if config.HypervisorType == StratovirtHypervisor {
|
|
||||||
+ if !config.HypervisorConfig.BootToBeTemplate {
|
|
||||||
+ virtLog.WithField("vm", id).Info("check agent status")
|
|
||||||
+ err = agent.check(ctx)
|
|
||||||
+ }
|
|
||||||
+ } else if !config.HypervisorConfig.BootFromTemplate {
|
|
||||||
virtLog.WithField("vm", id).Info("check agent status")
|
|
||||||
err = agent.check(ctx)
|
|
||||||
- if err != nil {
|
|
||||||
- return nil, err
|
|
||||||
- }
|
|
||||||
+ }
|
|
||||||
+ if err != nil {
|
|
||||||
+ return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return &VM{
|
|
||||||
@@ -329,9 +335,16 @@ func (v *VM) assignSandbox(s *Sandbox) error {
|
|
||||||
// - link 9pfs share path from sandbox dir (/run/kata-containers/shared/sandboxes/sbid/) to vm dir (/run/vc/vm/vmid/shared/)
|
|
||||||
|
|
||||||
vmSharePath := buildVMSharePath(v.id, v.store.RunVMStoragePath())
|
|
||||||
- vmSockDir := filepath.Join(v.store.RunVMStoragePath(), v.id)
|
|
||||||
sbSharePath := getMountPath(s.id)
|
|
||||||
- sbSockDir := filepath.Join(v.store.RunVMStoragePath(), s.id)
|
|
||||||
+ var vmSockDir string
|
|
||||||
+ var sbSockDir string
|
|
||||||
+ if v.hypervisor.hypervisorConfig().OzonePath != "" {
|
|
||||||
+ vmSockDir = filepath.Join(ozoneBaseDir, v.id)
|
|
||||||
+ sbSockDir = filepath.Join(ozoneBaseDir, s.id)
|
|
||||||
+ } else {
|
|
||||||
+ vmSockDir = filepath.Join(v.store.RunVMStoragePath(), v.id)
|
|
||||||
+ sbSockDir = filepath.Join(v.store.RunVMStoragePath(), s.id)
|
|
||||||
+ }
|
|
||||||
|
|
||||||
v.logger().WithFields(logrus.Fields{
|
|
||||||
"vmSharePath": vmSharePath,
|
|
||||||
@@ -359,6 +372,7 @@ func (v *VM) assignSandbox(s *Sandbox) error {
|
|
||||||
|
|
||||||
s.hypervisor = v.hypervisor
|
|
||||||
s.config.HypervisorConfig.VMid = v.id
|
|
||||||
+ s.config.HypervisorConfig.BootFromTemplate = true
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
--
|
|
||||||
2.21.1 (Apple Git-122.3)
|
|
||||||
|
|
||||||
@ -1,224 +0,0 @@
|
|||||||
From d4605dafaa9c326a5cf24c28d0c1efe6c9997f49 Mon Sep 17 00:00:00 2001
|
|
||||||
From: holyfei <yangfeiyu20092010@163.com>
|
|
||||||
Date: Sat, 21 Aug 2021 17:08:46 +0800
|
|
||||||
Subject: [PATCH] kata-containers: support with iSulad
|
|
||||||
|
|
||||||
reason: support with iSulad
|
|
||||||
|
|
||||||
Signed-off-by: holyfei <yangfeiyu20092010@163.com>
|
|
||||||
---
|
|
||||||
src/agent/rustjail/src/cgroups/fs/mod.rs | 2 +-
|
|
||||||
src/runtime/containerd-shim-v2/container.go | 9 +++
|
|
||||||
src/runtime/containerd-shim-v2/service.go | 55 +++++++++++++++++++
|
|
||||||
src/runtime/containerd-shim-v2/start.go | 10 ++++
|
|
||||||
.../containerd/runtime/v2/shim/shim.go | 8 ++-
|
|
||||||
5 files changed, 81 insertions(+), 3 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/src/agent/rustjail/src/cgroups/fs/mod.rs b/src/agent/rustjail/src/cgroups/fs/mod.rs
|
|
||||||
index 7f41cb4..6c3bb32 100644
|
|
||||||
--- a/src/agent/rustjail/src/cgroups/fs/mod.rs
|
|
||||||
+++ b/src/agent/rustjail/src/cgroups/fs/mod.rs
|
|
||||||
@@ -369,7 +369,7 @@ fn set_memory_resources(cg: &cgroups::Cgroup, memory: &LinuxMemory, update: bool
|
|
||||||
if let Some(swappiness) = memory.swappiness {
|
|
||||||
if (0..=100).contains(&swappiness) {
|
|
||||||
mem_controller.set_swappiness(swappiness as u64)?;
|
|
||||||
- } else {
|
|
||||||
+ } else if swappiness != -1 {
|
|
||||||
return Err(anyhow!(
|
|
||||||
"invalid value:{}. valid memory swappiness range is 0-100",
|
|
||||||
swappiness
|
|
||||||
diff --git a/src/runtime/containerd-shim-v2/container.go b/src/runtime/containerd-shim-v2/container.go
|
|
||||||
index faea0e2..d563888 100644
|
|
||||||
--- a/src/runtime/containerd-shim-v2/container.go
|
|
||||||
+++ b/src/runtime/containerd-shim-v2/container.go
|
|
||||||
@@ -7,10 +7,13 @@ package containerdshim
|
|
||||||
|
|
||||||
import (
|
|
||||||
"io"
|
|
||||||
+ "os"
|
|
||||||
+ "path"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/containerd/containerd/api/types/task"
|
|
||||||
"github.com/containerd/containerd/errdefs"
|
|
||||||
+ cdshim "github.com/containerd/containerd/runtime/v2/shim"
|
|
||||||
taskAPI "github.com/containerd/containerd/runtime/v2/task"
|
|
||||||
"github.com/opencontainers/runtime-spec/specs-go"
|
|
||||||
|
|
||||||
@@ -37,6 +40,8 @@ type container struct {
|
|
||||||
status task.Status
|
|
||||||
terminal bool
|
|
||||||
mounted bool
|
|
||||||
+ exitFifo string
|
|
||||||
+ exitFd *os.File
|
|
||||||
}
|
|
||||||
|
|
||||||
func newContainer(s *service, r *taskAPI.CreateTaskRequest, containerType vc.ContainerType, spec *specs.Spec, mounted bool) (*container, error) {
|
|
||||||
@@ -49,6 +54,9 @@ func newContainer(s *service, r *taskAPI.CreateTaskRequest, containerType vc.Con
|
|
||||||
spec = &specs.Spec{}
|
|
||||||
}
|
|
||||||
|
|
||||||
+ dir := os.Getenv(cdshim.ExitFifoDir)
|
|
||||||
+ exitFifo := path.Join(dir, r.ID, exitFifoName)
|
|
||||||
+
|
|
||||||
c := &container{
|
|
||||||
s: s,
|
|
||||||
spec: spec,
|
|
||||||
@@ -65,6 +73,7 @@ func newContainer(s *service, r *taskAPI.CreateTaskRequest, containerType vc.Con
|
|
||||||
exitCh: make(chan uint32, 1),
|
|
||||||
stdinCloser: make(chan struct{}),
|
|
||||||
mounted: mounted,
|
|
||||||
+ exitFifo: exitFifo,
|
|
||||||
}
|
|
||||||
return c, nil
|
|
||||||
}
|
|
||||||
diff --git a/src/runtime/containerd-shim-v2/service.go b/src/runtime/containerd-shim-v2/service.go
|
|
||||||
index 1003f8e..e13283c 100644
|
|
||||||
--- a/src/runtime/containerd-shim-v2/service.go
|
|
||||||
+++ b/src/runtime/containerd-shim-v2/service.go
|
|
||||||
@@ -6,13 +6,16 @@
|
|
||||||
package containerdshim
|
|
||||||
|
|
||||||
import (
|
|
||||||
+ "bytes"
|
|
||||||
"context"
|
|
||||||
+ "encoding/binary"
|
|
||||||
"io/ioutil"
|
|
||||||
"os"
|
|
||||||
sysexec "os/exec"
|
|
||||||
"sync"
|
|
||||||
"syscall"
|
|
||||||
"time"
|
|
||||||
+ "unsafe"
|
|
||||||
|
|
||||||
eventstypes "github.com/containerd/containerd/api/events"
|
|
||||||
"github.com/containerd/containerd/api/types/task"
|
|
||||||
@@ -51,6 +54,8 @@ const (
|
|
||||||
// A time span used to wait for publish a containerd event,
|
|
||||||
// once it costs a longer time than timeOut, it will be canceld.
|
|
||||||
timeOut = 5 * time.Second
|
|
||||||
+
|
|
||||||
+ exitFifoName = "exit_fifo"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
@@ -1019,6 +1024,10 @@ func (s *service) Wait(ctx context.Context, r *taskAPI.WaitRequest) (_ *taskAPI.
|
|
||||||
func (s *service) processExits() {
|
|
||||||
for e := range s.ec {
|
|
||||||
s.checkProcesses(e)
|
|
||||||
+
|
|
||||||
+ if os.Getenv(cdshim.ExitFifoDir) != "" {
|
|
||||||
+ s.closeExitFifo(e)
|
|
||||||
+ }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@@ -1070,3 +1079,49 @@ func (s *service) getContainerStatus(containerID string) (task.Status, error) {
|
|
||||||
|
|
||||||
return status, nil
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+func isBigEndian() (ret bool) {
|
|
||||||
+ i := int(0x1)
|
|
||||||
+ bs := (*[int(unsafe.Sizeof(i))]byte)(unsafe.Pointer(&i))
|
|
||||||
+ return bs[0] == 0
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *service) closeExitFifo(e exit) {
|
|
||||||
+ if e.execid != "" {
|
|
||||||
+ // not a container, no need to close exit fifo
|
|
||||||
+ return
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ var ret uint32
|
|
||||||
+ var nativeEndian binary.ByteOrder
|
|
||||||
+
|
|
||||||
+ s.mu.Lock()
|
|
||||||
+ c, err := s.getContainer(e.id)
|
|
||||||
+ s.mu.Unlock()
|
|
||||||
+
|
|
||||||
+ if err != nil {
|
|
||||||
+ logrus.WithError(err).Errorf("Process container:%v exit fifo failed", e.id)
|
|
||||||
+ return
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ ret = <-c.exitCh
|
|
||||||
+ // refill the exitCh with the container process's exit code in case
|
|
||||||
+ // there were other waits on this process.
|
|
||||||
+ c.exitCh <- ret
|
|
||||||
+
|
|
||||||
+ if isBigEndian() {
|
|
||||||
+ nativeEndian = binary.BigEndian
|
|
||||||
+ } else {
|
|
||||||
+ nativeEndian = binary.LittleEndian
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ bytesBuffer := bytes.NewBuffer([]byte{})
|
|
||||||
+ binary.Write(bytesBuffer, nativeEndian, &ret)
|
|
||||||
+
|
|
||||||
+ _, err = c.exitFd.Write(bytesBuffer.Bytes())
|
|
||||||
+ if err != nil {
|
|
||||||
+ logrus.WithError(err).Error("write exit fifo failed")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ c.exitFd.Close()
|
|
||||||
+}
|
|
||||||
diff --git a/src/runtime/containerd-shim-v2/start.go b/src/runtime/containerd-shim-v2/start.go
|
|
||||||
index 72420e4..e89dc48 100644
|
|
||||||
--- a/src/runtime/containerd-shim-v2/start.go
|
|
||||||
+++ b/src/runtime/containerd-shim-v2/start.go
|
|
||||||
@@ -8,8 +8,11 @@ package containerdshim
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
+ "golang.org/x/sys/unix"
|
|
||||||
+ "os"
|
|
||||||
|
|
||||||
"github.com/containerd/containerd/api/types/task"
|
|
||||||
+ cdshim "github.com/containerd/containerd/runtime/v2/shim"
|
|
||||||
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
|
|
||||||
)
|
|
||||||
|
|
||||||
@@ -59,6 +62,13 @@ func startContainer(ctx context.Context, s *service, c *container) error {
|
|
||||||
|
|
||||||
c.status = task.StatusRunning
|
|
||||||
|
|
||||||
+ if os.Getenv(cdshim.ExitFifoDir) != "" {
|
|
||||||
+ c.exitFd, err = os.OpenFile(c.exitFifo, unix.O_WRONLY|unix.O_NONBLOCK|unix.O_CLOEXEC, 0)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
stdin, stdout, stderr, err := s.sandbox.IOStream(c.id, c.id)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
diff --git a/src/runtime/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go b/src/runtime/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go
|
|
||||||
index d60d496..946c386 100644
|
|
||||||
--- a/src/runtime/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go
|
|
||||||
+++ b/src/runtime/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go
|
|
||||||
@@ -84,6 +84,8 @@ var (
|
|
||||||
action string
|
|
||||||
)
|
|
||||||
|
|
||||||
+var ExitFifoDir = "EXIT_FIFO_DIR"
|
|
||||||
+
|
|
||||||
func parseFlags() {
|
|
||||||
flag.BoolVar(&debugFlag, "debug", false, "enable debug output in logs")
|
|
||||||
flag.StringVar(&namespaceFlag, "namespace", "", "namespace that owns the shim")
|
|
||||||
@@ -198,8 +200,10 @@ func run(id string, initFunc Init, config Config) error {
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
default:
|
|
||||||
- if err := setLogger(ctx, idFlag); err != nil {
|
|
||||||
- return err
|
|
||||||
+ if os.Getenv("EXIT_FIFO_DIR") == "" {
|
|
||||||
+ if err := setLogger(ctx, idFlag); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
}
|
|
||||||
client := NewShimClient(ctx, service, signals)
|
|
||||||
return client.Serve()
|
|
||||||
--
|
|
||||||
2.23.0
|
|
||||||
|
|
||||||
@ -1,29 +0,0 @@
|
|||||||
From 5ee8e64c8c620f02cb580f1f3349ae63660ca34c Mon Sep 17 00:00:00 2001
|
|
||||||
From: holyfei <yangfeiyu20092010@163.com>
|
|
||||||
Date: Mon, 27 Sep 2021 11:03:58 +0800
|
|
||||||
Subject: [PATCH] kata-containers: adpat with iSulad
|
|
||||||
|
|
||||||
reason: chmod the exec fifo to 644, isula start container
|
|
||||||
and need the permission
|
|
||||||
|
|
||||||
Signed-off-by: holyfei <yangfeiyu20092010@163.com>
|
|
||||||
---
|
|
||||||
src/agent/rustjail/src/container.rs | 2 +-
|
|
||||||
1 file changed, 1 insertion(+), 1 deletion(-)
|
|
||||||
|
|
||||||
diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs
|
|
||||||
index 748ee486..5113a482 100644
|
|
||||||
--- a/src/agent/rustjail/src/container.rs
|
|
||||||
+++ b/src/agent/rustjail/src/container.rs
|
|
||||||
@@ -822,7 +822,7 @@ impl BaseContainer for LinuxContainer {
|
|
||||||
if stat::stat(fifo_file.as_str()).is_ok() {
|
|
||||||
return Err(anyhow!("exec fifo exists"));
|
|
||||||
}
|
|
||||||
- unistd::mkfifo(fifo_file.as_str(), Mode::from_bits(0o622).unwrap())?;
|
|
||||||
+ unistd::mkfifo(fifo_file.as_str(), Mode::from_bits(0o644).unwrap())?;
|
|
||||||
|
|
||||||
fifofd = fcntl::open(
|
|
||||||
fifo_file.as_str(),
|
|
||||||
--
|
|
||||||
2.23.0
|
|
||||||
|
|
||||||
@ -1,27 +0,0 @@
|
|||||||
From c563b455e3bb0cebfbe2a77c8d5ebac36aac3c76 Mon Sep 17 00:00:00 2001
|
|
||||||
From: jikui <jikui2@huawei.com>
|
|
||||||
Date: Thu, 4 Nov 2021 19:58:33 +0800
|
|
||||||
Subject: [PATCH 1/8] kata-runtime: fix kata-runtime hungs when qemu process is
|
|
||||||
D/T state
|
|
||||||
|
|
||||||
Signed-off-by: jikui <jikui2@huawei.com>
|
|
||||||
---
|
|
||||||
src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go | 2 ++
|
|
||||||
1 file changed, 2 insertions(+)
|
|
||||||
|
|
||||||
diff --git a/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go
|
|
||||||
index 97e9245..325250d 100644
|
|
||||||
--- a/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go
|
|
||||||
+++ b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go
|
|
||||||
@@ -717,6 +717,8 @@ func QMPStart(ctx context.Context, socket string, cfg QMPConfig, disconnectedCh
|
|
||||||
if q.version == nil {
|
|
||||||
return nil, nil, fmt.Errorf("failed to find QMP version information")
|
|
||||||
}
|
|
||||||
+ case <-time.After(15 * time.Second):
|
|
||||||
+ return nil, nil, fmt.Errorf("qmp start time out")
|
|
||||||
}
|
|
||||||
|
|
||||||
return q, q.version, nil
|
|
||||||
--
|
|
||||||
2.25.1
|
|
||||||
|
|
||||||
@ -1,52 +0,0 @@
|
|||||||
From 0797bf1fec9c40b67f2770bc8778e8eaee1657c8 Mon Sep 17 00:00:00 2001
|
|
||||||
From: jikui <jikui2@huawei.com>
|
|
||||||
Date: Fri, 5 Nov 2021 11:35:25 +0800
|
|
||||||
Subject: [PATCH 3/8] kata-runtime: keep the process name of qemu same as
|
|
||||||
configured path
|
|
||||||
|
|
||||||
Signed-off-by: jikui <jikui2@huawei.com>
|
|
||||||
---
|
|
||||||
src/runtime/pkg/katautils/config.go | 7 ++++++-
|
|
||||||
src/runtime/pkg/katautils/config_test.go | 4 ++--
|
|
||||||
2 files changed, 8 insertions(+), 3 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go
|
|
||||||
index 718677b..e0ebc84 100644
|
|
||||||
--- a/src/runtime/pkg/katautils/config.go
|
|
||||||
+++ b/src/runtime/pkg/katautils/config.go
|
|
||||||
@@ -172,7 +172,12 @@ func (h hypervisor) path() (string, error) {
|
|
||||||
p = defaultHypervisorPath
|
|
||||||
}
|
|
||||||
|
|
||||||
- return ResolvePath(p)
|
|
||||||
+ absolutePath, err := filepath.Abs(p)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return "", err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return absolutePath, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h hypervisor) ctlpath() (string, error) {
|
|
||||||
diff --git a/src/runtime/pkg/katautils/config_test.go b/src/runtime/pkg/katautils/config_test.go
|
|
||||||
index 0d02534..84b7843 100644
|
|
||||||
--- a/src/runtime/pkg/katautils/config_test.go
|
|
||||||
+++ b/src/runtime/pkg/katautils/config_test.go
|
|
||||||
@@ -983,12 +983,12 @@ func TestHypervisorDefaultsHypervisor(t *testing.T) {
|
|
||||||
assert.NoError(err)
|
|
||||||
assert.Equal(p, defaultHypervisorPath, "default hypervisor path wrong")
|
|
||||||
|
|
||||||
- // test path resolution
|
|
||||||
+ // test path resolution, just return the absolute path instead of resolved path
|
|
||||||
defaultHypervisorPath = testHypervisorLinkPath
|
|
||||||
h = hypervisor{}
|
|
||||||
p, err = h.path()
|
|
||||||
assert.NoError(err)
|
|
||||||
- assert.Equal(p, testHypervisorPath)
|
|
||||||
+ assert.Equal(p, testHypervisorLinkPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestHypervisorDefaultsKernel(t *testing.T) {
|
|
||||||
--
|
|
||||||
2.25.1
|
|
||||||
|
|
||||||
@ -1,56 +0,0 @@
|
|||||||
From e8e8e05538bf2c7bd8feebc44d4a960f453d21e1 Mon Sep 17 00:00:00 2001
|
|
||||||
From: holyfei <yangfeiyu20092010@163.com>
|
|
||||||
Date: Sat, 20 Nov 2021 17:04:39 +0800
|
|
||||||
Subject: [PATCH] kata-containers: modify kernel and image path in
|
|
||||||
configuration.toml
|
|
||||||
|
|
||||||
Signed-off-by: holyfei <yangfeiyu20092010@163.com>
|
|
||||||
---
|
|
||||||
src/runtime/Makefile | 10 +++++-----
|
|
||||||
1 file changed, 5 insertions(+), 5 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/src/runtime/Makefile b/src/runtime/Makefile
|
|
||||||
index 745bcc1..68e60ee 100644
|
|
||||||
--- a/src/runtime/Makefile
|
|
||||||
+++ b/src/runtime/Makefile
|
|
||||||
@@ -49,6 +49,8 @@ BINLIBEXECLIST :=
|
|
||||||
BIN_PREFIX = $(PROJECT_TYPE)
|
|
||||||
PROJECT_DIR = $(PROJECT_TAG)
|
|
||||||
IMAGENAME = $(PROJECT_TAG).img
|
|
||||||
+KERNEL_PATH = /var/lib/kata/kernel
|
|
||||||
+IMAGE_PATH = /var/lib/kata/kata-containers-initrd.img
|
|
||||||
|
|
||||||
TARGET = $(BIN_PREFIX)-runtime
|
|
||||||
TARGET_OUTPUT = $(CURDIR)/$(TARGET)
|
|
||||||
@@ -108,7 +110,7 @@ PKGLIBEXECDIR := $(LIBEXECDIR)/$(PROJECT_DIR)
|
|
||||||
|
|
||||||
KERNELDIR := $(PKGDATADIR)
|
|
||||||
|
|
||||||
-IMAGEPATH := $(PKGDATADIR)/$(IMAGENAME)
|
|
||||||
+IMAGEPATH := $(IMAGE_PATH)
|
|
||||||
FIRMWAREPATH :=
|
|
||||||
|
|
||||||
# Name of default configuration file the runtime will use.
|
|
||||||
@@ -248,8 +250,7 @@ ifneq (,$(QEMUCMD))
|
|
||||||
DEFBLOCKSTORAGEDRIVER_QEMU := virtio-scsi
|
|
||||||
DEFNETWORKMODEL_QEMU := tcfilter
|
|
||||||
KERNELTYPE = uncompressed
|
|
||||||
- KERNELNAME = $(call MAKE_KERNEL_NAME,$(KERNELTYPE))
|
|
||||||
- KERNELPATH = $(KERNELDIR)/$(KERNELNAME)
|
|
||||||
+ KERNELPATH = $(KERNEL_PATH)
|
|
||||||
endif
|
|
||||||
|
|
||||||
ifneq (,$(CLHCMD))
|
|
||||||
@@ -294,8 +295,7 @@ ifneq (,$(STRATOVIRTCMD))
|
|
||||||
DEFBLOCKSTORAGEDRIVER_STRATOVIRT := virtio-mmio
|
|
||||||
DEFNETWORKMODEL_STRATOVIRT := none
|
|
||||||
KENRELTYPE_STRATOVIRT = uncompressed
|
|
||||||
- KERNEL_NAME_STRATOVIRT = $(call MAKE_KERNEL_NAME,$(KENRELTYPE_STRATOVIRT))
|
|
||||||
- KERNELPATH_STRATOVIRT = $(KERNELDIR)/$(KERNEL_NAME_STRATOVIRT)
|
|
||||||
+ KERNELPATH_STRATOVIRT = $(KERNEL_PATH)
|
|
||||||
endif
|
|
||||||
|
|
||||||
ifneq (,$(FCCMD))
|
|
||||||
--
|
|
||||||
2.27.0
|
|
||||||
|
|
||||||
20
proxy/apply-patches
Executable file
20
proxy/apply-patches
Executable file
@ -0,0 +1,20 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
if [[ -f ./patch_flag ]];then
|
||||||
|
echo "proxy patched!"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
tar -zxvf proxy-1.11.1.tar.gz
|
||||||
|
cp -fr ./proxy-1.11.1/* ./
|
||||||
|
rm -rf ./proxy-1.11.1
|
||||||
|
cat ./series.conf | while read line
|
||||||
|
do
|
||||||
|
if [[ $line == '' || $line =~ ^\s*# ]]; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
echo "====patch $line======"
|
||||||
|
pwd
|
||||||
|
patch -p1 -F1 -s < ./patches/$line
|
||||||
|
done
|
||||||
|
touch ./patch_flag
|
||||||
403
proxy/patches/0001-clock-synchronizes-clock-info-to-agent.patch
Normal file
403
proxy/patches/0001-clock-synchronizes-clock-info-to-agent.patch
Normal file
@ -0,0 +1,403 @@
|
|||||||
|
From a260bbe394f91fa05b163315390ed133de5c5494 Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Wed, 19 Aug 2020 17:43:24 +0800
|
||||||
|
Subject: [PATCH] clock: synchronizes clock info to agent
|
||||||
|
|
||||||
|
reason: virtual machine's clock may be incorrect, proxy synchronizes
|
||||||
|
clock info to help virtual machine adjust clock time
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
proxy.go | 7 ++
|
||||||
|
proxy_test.go | 29 +++++
|
||||||
|
sync_clock_client.go | 107 +++++++++++++++++
|
||||||
|
sync_clock_client_test.go | 132 +++++++++++++++++++++
|
||||||
|
.../kata-containers/agent/pkg/clock/clock_util.go | 44 +++++++
|
||||||
|
5 files changed, 319 insertions(+)
|
||||||
|
create mode 100644 sync_clock_client.go
|
||||||
|
create mode 100644 sync_clock_client_test.go
|
||||||
|
create mode 100644 vendor/github.com/kata-containers/agent/pkg/clock/clock_util.go
|
||||||
|
|
||||||
|
diff --git a/proxy.go b/proxy.go
|
||||||
|
index ab062a5..9dfcb3c 100644
|
||||||
|
--- a/proxy.go
|
||||||
|
+++ b/proxy.go
|
||||||
|
@@ -105,6 +105,13 @@ func serve(servConn io.ReadWriteCloser, proto, addr string, results chan error)
|
||||||
|
// Start the heartbeat in a separate go routine
|
||||||
|
go heartBeat(session)
|
||||||
|
|
||||||
|
+ // start the sync clock in a separate go routine
|
||||||
|
+ syncClockStream, err := session.Open()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, nil, err
|
||||||
|
+ }
|
||||||
|
+ go SyncClock(syncClockStream)
|
||||||
|
+
|
||||||
|
// serving connection
|
||||||
|
l, err := net.Listen(proto, addr)
|
||||||
|
if err != nil {
|
||||||
|
diff --git a/proxy_test.go b/proxy_test.go
|
||||||
|
index 923b138..94fa523 100644
|
||||||
|
--- a/proxy_test.go
|
||||||
|
+++ b/proxy_test.go
|
||||||
|
@@ -10,6 +10,7 @@ package main
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"crypto/md5"
|
||||||
|
+ "encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"io/ioutil"
|
||||||
|
@@ -23,6 +24,7 @@ import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/hashicorp/yamux"
|
||||||
|
+ "github.com/kata-containers/agent/pkg/clock"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
@@ -121,6 +123,13 @@ func server(listener net.Listener, closeCh chan bool) error {
|
||||||
|
session.Close()
|
||||||
|
}()
|
||||||
|
|
||||||
|
+ // accept the sync clock stream first
|
||||||
|
+ if syncClockStream, err := session.Accept(); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ } else {
|
||||||
|
+ go serverSyncClock(syncClockStream)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
for {
|
||||||
|
stream, err := session.Accept()
|
||||||
|
if err != nil {
|
||||||
|
@@ -133,6 +142,26 @@ func server(listener net.Listener, closeCh chan bool) error {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
+func serverSyncClock(stream net.Conn) {
|
||||||
|
+ for {
|
||||||
|
+ buf, byteNum, err := readConnData(stream)
|
||||||
|
+ if err != nil {
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+ var clockInfo clock.TimeValue
|
||||||
|
+ if err := json.Unmarshal(buf[:byteNum], &clockInfo); err != nil {
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+ nowTime := clock.GetCurrentTimeNs()
|
||||||
|
+ clockInfo.ClientArriveTime = nowTime
|
||||||
|
+ clockInfo.ServerSendTime = nowTime
|
||||||
|
+ b, _ := json.Marshal(clockInfo)
|
||||||
|
+ if err := clock.WriteConnData(stream, b); err != nil {
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func TestUnixAddrParsing(T *testing.T) {
|
||||||
|
buf := "unix://foo/bar"
|
||||||
|
addr, err := unixAddr(buf)
|
||||||
|
diff --git a/sync_clock_client.go b/sync_clock_client.go
|
||||||
|
new file mode 100644
|
||||||
|
index 0000000..9bf3e91
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/sync_clock_client.go
|
||||||
|
@@ -0,0 +1,107 @@
|
||||||
|
+// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved.
|
||||||
|
+// SPDX-License-Identifier: Apache-2.0
|
||||||
|
+// Description: sync clock client related function
|
||||||
|
+// Author: xueshaojia x00464843
|
||||||
|
+// Create: 2018-11-10
|
||||||
|
+
|
||||||
|
+package main
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "encoding/json"
|
||||||
|
+ "fmt"
|
||||||
|
+ "net"
|
||||||
|
+ "time"
|
||||||
|
+
|
||||||
|
+ "github.com/kata-containers/agent/pkg/clock"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+const (
|
||||||
|
+ allowTimeDiff = 10 * 1000 * 1000 // allow 10ms's difference
|
||||||
|
+ syncClockInterval = 60 * time.Second // sync clock with agent every 60 seconds
|
||||||
|
+ rpcTimeout = 10 * time.Second // timeout for proxy's reading data
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+// readConnData reads data from stream
|
||||||
|
+func readConnData(stream net.Conn) (buf []byte, byteNum int, err error) {
|
||||||
|
+ // set read deadline to avoid case as following:
|
||||||
|
+ // proxy and agent are both reading and then syncClock will never work
|
||||||
|
+ stream.SetReadDeadline(time.Now().Add(rpcTimeout))
|
||||||
|
+ buf = make([]byte, clock.MaxSyncClockByteNum)
|
||||||
|
+ byteNum, err = stream.Read(buf)
|
||||||
|
+ return buf, byteNum, err
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// getGuestClock syncs guest clock info
|
||||||
|
+// sends ClientSendTime
|
||||||
|
+// waits for ClientArriveTime and ServerSendTime
|
||||||
|
+func getGuestClock(stream net.Conn, clockInfo *clock.TimeValue) error {
|
||||||
|
+ clockInfo.Delta = 0
|
||||||
|
+ b, err := json.Marshal(clockInfo)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ if err = clock.WriteConnData(stream, b); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ buf, byteNum, err := readConnData(stream)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err = json.Unmarshal(buf[:byteNum], clockInfo); err != nil {
|
||||||
|
+ return fmt.Errorf("sync clock, parse guest clocktime error:%v", err)
|
||||||
|
+ }
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// adjustGuestClock tells server to ajust local clock with Delta
|
||||||
|
+func adjustGuestClock(stream net.Conn, clockInfo *clock.TimeValue) error {
|
||||||
|
+ b, err := json.Marshal(clockInfo)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ logger().Debugf("sync clock, send:%s", string(b))
|
||||||
|
+ return clock.WriteConnData(stream, b)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// syncClock performs all the steps
|
||||||
|
+// 1 get client send time[host]
|
||||||
|
+// 2 request for client arrive time and server send time[guest os]
|
||||||
|
+// 3 get server arrive time[host]
|
||||||
|
+// 4 calculate clock diff
|
||||||
|
+// 5 request to adjust guest clock
|
||||||
|
+func syncClock(stream net.Conn) error {
|
||||||
|
+ var clockInfo clock.TimeValue
|
||||||
|
+ if clockInfo.ClientSendTime = clock.GetCurrentTimeNs(); clockInfo.ClientSendTime <= 0 {
|
||||||
|
+ return fmt.Errorf("sync clock, get client sendtime error")
|
||||||
|
+ }
|
||||||
|
+ err := getGuestClock(stream, &clockInfo)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return fmt.Errorf("sync clock, get guest clocktime error:%v", err)
|
||||||
|
+ }
|
||||||
|
+ if clockInfo.ServerArriveTime = clock.GetCurrentTimeNs(); clockInfo.ServerArriveTime <= 0 {
|
||||||
|
+ return fmt.Errorf("sync clock, get client recvtime error")
|
||||||
|
+ }
|
||||||
|
+ if clockInfo.ClientSendTime <= 0 || clockInfo.ClientArriveTime <= 0 || clockInfo.ServerSendTime <= 0 {
|
||||||
|
+ return fmt.Errorf("sync clock, some fields of NTP message error, raw message:%v", clockInfo)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ delta := ((clockInfo.ClientSendTime - clockInfo.ClientArriveTime) + (clockInfo.ServerArriveTime - clockInfo.ServerSendTime)) / 2
|
||||||
|
+ if delta < -allowTimeDiff || delta > allowTimeDiff {
|
||||||
|
+ clockInfo.Delta = delta
|
||||||
|
+ if err := adjustGuestClock(stream, &clockInfo); err != nil {
|
||||||
|
+ return fmt.Errorf("sync clock, failed to adjust guest clock : %v", err)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func SyncClock(stream net.Conn) {
|
||||||
|
+ for {
|
||||||
|
+ if err := syncClock(stream); err != nil {
|
||||||
|
+ logger().WithError(err).Error("sync clock failed")
|
||||||
|
+ }
|
||||||
|
+ time.Sleep(syncClockInterval)
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
diff --git a/sync_clock_client_test.go b/sync_clock_client_test.go
|
||||||
|
new file mode 100644
|
||||||
|
index 0000000..b0b1c85
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/sync_clock_client_test.go
|
||||||
|
@@ -0,0 +1,132 @@
|
||||||
|
+// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved.
|
||||||
|
+// SPDX-License-Identifier: Apache-2.0
|
||||||
|
+// Description: sync clock client related test
|
||||||
|
+// Author: xueshaojia x00464843
|
||||||
|
+// Create: 2018-11-10
|
||||||
|
+
|
||||||
|
+package main
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "encoding/json"
|
||||||
|
+ "fmt"
|
||||||
|
+ "math/rand"
|
||||||
|
+ "net"
|
||||||
|
+ "os"
|
||||||
|
+ "testing"
|
||||||
|
+
|
||||||
|
+ "github.com/hashicorp/yamux"
|
||||||
|
+ "github.com/kata-containers/agent/pkg/clock"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+func TestReadWriteData(t *testing.T) {
|
||||||
|
+ var clockInfo clock.TimeValue
|
||||||
|
+ clockInfo.ClientSendTime = clock.GetCurrentTimeNs()
|
||||||
|
+ b, err := json.Marshal(clockInfo)
|
||||||
|
+ if err != nil {
|
||||||
|
+ t.Fatalf("Marshal clock info fail, err:%v", err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ err = clock.WriteConnData(clientStream, b)
|
||||||
|
+ fmt.Printf("client send: %s\n", string(b))
|
||||||
|
+ if err != nil {
|
||||||
|
+ t.Fatalf("send clock info fail, err:%v", err)
|
||||||
|
+ }
|
||||||
|
+ _, _, err = readConnData(clientStream)
|
||||||
|
+ if err != nil {
|
||||||
|
+ t.Fatalf("recv clock info fail, err:%v", err)
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func SetUpServer(sock string, readyChan chan int) error {
|
||||||
|
+ var err error
|
||||||
|
+ listener, err = net.Listen("unix", sock)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ readyChan <- 1
|
||||||
|
+ conn, err := listener.Accept()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ session, err := yamux.Server(conn, nil)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ serverSession = session
|
||||||
|
+ stream, err := session.Accept()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ for {
|
||||||
|
+ var clockInfo clock.TimeValue
|
||||||
|
+ var byteNum int
|
||||||
|
+ var err error
|
||||||
|
+ buf := make([]byte, 400)
|
||||||
|
+ if byteNum, err = stream.Read(buf); err != nil {
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err = json.Unmarshal(buf[:byteNum], &clockInfo); err != nil {
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+ if clockInfo.Delta == 0 {
|
||||||
|
+ nowTime := clock.GetCurrentTimeNs()
|
||||||
|
+ clockInfo.ClientArriveTime = nowTime
|
||||||
|
+ clockInfo.ClientArriveTime = nowTime
|
||||||
|
+ b, _ := json.Marshal(&clockInfo)
|
||||||
|
+ stream.Write(b)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func SetUpClient(sock string) error {
|
||||||
|
+ conn, err := net.Dial("unix", sock)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ session, err := yamux.Client(conn, nil)
|
||||||
|
+ if err != nil {
|
||||||
|
+ conn.Close()
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ clientSession = session
|
||||||
|
+ stream, err := session.Open()
|
||||||
|
+ if err != nil {
|
||||||
|
+ clientSession.Close()
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ clientStream = stream
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TearDown() {
|
||||||
|
+ listener.Close()
|
||||||
|
+ serverSession.Close()
|
||||||
|
+ clientSession.Close()
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func GenSocket() string {
|
||||||
|
+ randSeed := clock.GetCurrentTimeNs()
|
||||||
|
+ rand.Seed(randSeed)
|
||||||
|
+ return fmt.Sprintf("/tmp/%d.sock", rand.Uint32())
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+var listener net.Listener
|
||||||
|
+var clientStream net.Conn
|
||||||
|
+var serverSession *yamux.Session
|
||||||
|
+var clientSession *yamux.Session
|
||||||
|
+
|
||||||
|
+func TestMain(m *testing.M) {
|
||||||
|
+ waitConn := make(chan int)
|
||||||
|
+ testSock := GenSocket()
|
||||||
|
+ go SetUpServer(testSock, waitConn)
|
||||||
|
+ <-waitConn
|
||||||
|
+ if err := SetUpClient(testSock); err != nil {
|
||||||
|
+ listener.Close()
|
||||||
|
+ serverSession.Close()
|
||||||
|
+ os.Exit(1)
|
||||||
|
+ }
|
||||||
|
+ m.Run()
|
||||||
|
+ TearDown()
|
||||||
|
+}
|
||||||
|
diff --git a/vendor/github.com/kata-containers/agent/pkg/clock/clock_util.go b/vendor/github.com/kata-containers/agent/pkg/clock/clock_util.go
|
||||||
|
new file mode 100644
|
||||||
|
index 0000000..03244fd
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/vendor/github.com/kata-containers/agent/pkg/clock/clock_util.go
|
||||||
|
@@ -0,0 +1,44 @@
|
||||||
|
+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
||||||
|
+// SPDX-License-Identifier: Apache-2.0
|
||||||
|
+// Description: common functions
|
||||||
|
+// Author: jiangpeifei
|
||||||
|
+// Create: 2019-05-28
|
||||||
|
+
|
||||||
|
+package clock
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "net"
|
||||||
|
+ "syscall"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+type TimeValue struct {
|
||||||
|
+ ClientSendTime int64 `json:"client_send_time"`
|
||||||
|
+ ClientArriveTime int64 `json:"client_arrive_time"`
|
||||||
|
+ ServerSendTime int64 `json:"server_send_time"`
|
||||||
|
+ ServerArriveTime int64 `json:"server_arrive_time"`
|
||||||
|
+ Delta int64 `json:"delta"`
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+const MaxSyncClockByteNum = 400 //sync clock byte num, max=400
|
||||||
|
+
|
||||||
|
+// getCurrentTimeNs returns UTC time in Ns
|
||||||
|
+func GetCurrentTimeNs() int64 {
|
||||||
|
+ var tv syscall.Timeval
|
||||||
|
+ if err := syscall.Gettimeofday(&tv); err != nil {
|
||||||
|
+ return -1
|
||||||
|
+ }
|
||||||
|
+ return tv.Sec*1000000000 + tv.Usec*1000
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// readConnData reads data from stream
|
||||||
|
+func ReadConnData(stream net.Conn) (buf []byte, byteNum int, err error) {
|
||||||
|
+ buf = make([]byte, MaxSyncClockByteNum)
|
||||||
|
+ byteNum, err = stream.Read(buf)
|
||||||
|
+ return buf, byteNum, err
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// writeConnData writes data to stream
|
||||||
|
+func WriteConnData(stream net.Conn, buf []byte) error {
|
||||||
|
+ _, err := stream.Write(buf)
|
||||||
|
+ return err
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
29
proxy/patches/0002-kata-proxy-modify-make-flags.patch
Normal file
29
proxy/patches/0002-kata-proxy-modify-make-flags.patch
Normal file
@ -0,0 +1,29 @@
|
|||||||
|
From 2c5cbf2ca9624d5443ad334a8337cb58d57573b2 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jikui <jikui2@huawei.com>
|
||||||
|
Date: Thu, 18 Mar 2021 15:34:07 +0800
|
||||||
|
Subject: [PATCH] kata-proxy: modify make flags
|
||||||
|
|
||||||
|
reason: modify make flags
|
||||||
|
|
||||||
|
Signed-off-by: jikui <jikui2@huawei.com>
|
||||||
|
---
|
||||||
|
Makefile | 3 +++
|
||||||
|
1 file changed, 3 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/Makefile b/Makefile
|
||||||
|
index a1b3eee..07e9ba8 100644
|
||||||
|
--- a/Makefile
|
||||||
|
+++ b/Makefile
|
||||||
|
@@ -33,6 +33,9 @@ ifeq ($(STRIP),yes)
|
||||||
|
endif
|
||||||
|
|
||||||
|
$(TARGET): $(SOURCES) $(VERSION_FILE)
|
||||||
|
+ CGO_CFLAGS="-fstack-protector-strong -fPIE -D_FORTIFY_SOURCE=2 -O2" \
|
||||||
|
+ CGO_LDFLAGS_ALLOW="-Wl,-z,relro,-z,now" \
|
||||||
|
+ CGO_LDFLAGS="-Wl,-z,relro,-z,now -Wl,-z,noexecstack" \
|
||||||
|
go build $(BUILDFLAGS) -o $@ -ldflags "-X main.version=$(VERSION_COMMIT) $(KATA_LDFLAGS)"
|
||||||
|
|
||||||
|
test:
|
||||||
|
--
|
||||||
|
2.25.1
|
||||||
|
|
||||||
@ -0,0 +1,37 @@
|
|||||||
|
From 5c4d7bcbef7d213009f1c63acf53319e230e06e2 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jikui <jikui2@huawei.com>
|
||||||
|
Date: Mon, 22 Mar 2021 17:11:48 +0800
|
||||||
|
Subject: [PATCH] kata-proxy: add linkmode to resolve build error
|
||||||
|
|
||||||
|
reason: add linkmode to resolve build error
|
||||||
|
|
||||||
|
Signed-off-by: jikui <jikui2@huawei.com>
|
||||||
|
---
|
||||||
|
Makefile | 4 ++--
|
||||||
|
1 file changed, 2 insertions(+), 2 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/Makefile b/Makefile
|
||||||
|
index 07e9ba8..b931dd3 100644
|
||||||
|
--- a/Makefile
|
||||||
|
+++ b/Makefile
|
||||||
|
@@ -5,7 +5,7 @@
|
||||||
|
#
|
||||||
|
|
||||||
|
# Set to "yes“ if binary stripping is needed.
|
||||||
|
-STRIP := no
|
||||||
|
+STRIP := yes
|
||||||
|
|
||||||
|
DESTDIR :=
|
||||||
|
ifeq ($(PREFIX),)
|
||||||
|
@@ -36,7 +36,7 @@ $(TARGET): $(SOURCES) $(VERSION_FILE)
|
||||||
|
CGO_CFLAGS="-fstack-protector-strong -fPIE -D_FORTIFY_SOURCE=2 -O2" \
|
||||||
|
CGO_LDFLAGS_ALLOW="-Wl,-z,relro,-z,now" \
|
||||||
|
CGO_LDFLAGS="-Wl,-z,relro,-z,now -Wl,-z,noexecstack" \
|
||||||
|
- go build $(BUILDFLAGS) -o $@ -ldflags "-X main.version=$(VERSION_COMMIT) $(KATA_LDFLAGS)"
|
||||||
|
+ go build $(BUILDFLAGS) -o $@ -ldflags "-linkmode=external -X main.version=$(VERSION_COMMIT) $(KATA_LDFLAGS)"
|
||||||
|
|
||||||
|
test:
|
||||||
|
bash .ci/go-test.sh
|
||||||
|
--
|
||||||
|
2.25.1
|
||||||
|
|
||||||
BIN
proxy/proxy-1.11.1.tar.gz
Normal file
BIN
proxy/proxy-1.11.1.tar.gz
Normal file
Binary file not shown.
3
proxy/series.conf
Normal file
3
proxy/series.conf
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
0001-clock-synchronizes-clock-info-to-agent.patch
|
||||||
|
0002-kata-proxy-modify-make-flags.patch
|
||||||
|
0003-kata-proxy-add-linkmode-to-resolve-build-error.patch
|
||||||
22
runtime/apply-patches
Executable file
22
runtime/apply-patches
Executable file
@ -0,0 +1,22 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
if [[ -f ./patch_flag ]];then
|
||||||
|
echo "runtime patched!"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
tar -zxvf runtime-1.11.1.tar.gz
|
||||||
|
cp -fr ./runtime-1.11.1/* ./
|
||||||
|
rm -rf ./runtime-1.11.1
|
||||||
|
cat ./series.conf | while read line
|
||||||
|
do
|
||||||
|
if [[ $line == '' || $line =~ ^\s*# ]]; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
echo "====patch $line======"
|
||||||
|
patch -p1 -F1 -s < ./patches/$line
|
||||||
|
done
|
||||||
|
|
||||||
|
touch ./patch_flag
|
||||||
191
runtime/kata-runtime.spec
Normal file
191
runtime/kata-runtime.spec
Normal file
@ -0,0 +1,191 @@
|
|||||||
|
%define debug_package %{nil}
|
||||||
|
|
||||||
|
%define VERSION 1.11.1
|
||||||
|
%define RELEASE 15
|
||||||
|
|
||||||
|
Name: kata-runtime
|
||||||
|
Version: %{VERSION}
|
||||||
|
Release: %{RELEASE}
|
||||||
|
Summary: Kata Runtime
|
||||||
|
License: Apache 2.0
|
||||||
|
URL: https://github.com/kata-containers/runtime
|
||||||
|
Source0: https://github.com/kata-containers/runtime/archive/%{version}.tar.gz#/%{name}-v%{version}.tar.gz
|
||||||
|
|
||||||
|
BuildRoot: %_topdir/BUILDROOT
|
||||||
|
BuildRequires: automake golang gcc
|
||||||
|
|
||||||
|
%description
|
||||||
|
Kata-runtime is core component of Kata Container.
|
||||||
|
|
||||||
|
%prep
|
||||||
|
%setup -q -c -a 0 -n %{name}-%{version}
|
||||||
|
|
||||||
|
%build
|
||||||
|
cd %{_builddir}/%{name}-%{version}
|
||||||
|
|
||||||
|
set -e
|
||||||
|
# apply patches read from series.conf
|
||||||
|
sh apply-patches
|
||||||
|
|
||||||
|
# create tmp GOPATH dir to build kata-runtime
|
||||||
|
rm -rf /tmp/kata-build/
|
||||||
|
mkdir -p /tmp/kata-build/
|
||||||
|
GOPATH=/tmp/kata-build/
|
||||||
|
kata_base=$GOPATH/src/github.com/kata-containers
|
||||||
|
mkdir -p $kata_base
|
||||||
|
|
||||||
|
# get current kata-runtime absolute path
|
||||||
|
kata_runtime_path=$(readlink -f .)
|
||||||
|
ln -s $kata_runtime_path $kata_base/runtime
|
||||||
|
|
||||||
|
# export GOPATH env
|
||||||
|
export GOPATH=$(readlink -f $GOPATH)
|
||||||
|
cd ${kata_base}/runtime && make clean && make
|
||||||
|
rm -rf $GOPATH
|
||||||
|
|
||||||
|
# make kata-runtime default configuration
|
||||||
|
kata_config_path=$kata_runtime_path/cli/config/configuration-qemu.toml
|
||||||
|
ARCH=`arch`
|
||||||
|
|
||||||
|
# arch related config options
|
||||||
|
if [ "$ARCH" == "aarch64" ];then
|
||||||
|
sed -i 's/^machine_type.*$/machine_type = \"virt\"/' $kata_config_path
|
||||||
|
sed -i 's/^block_device_driver.*$/block_device_driver = \"virtio-scsi\"/' $kata_config_path
|
||||||
|
sed -i 's/^kernel_params.*$/kernel_params = \"pcie_ports=native pci=pcie_bus_perf agent.netlink_recv_buf_size=2MB\"/' $kata_config_path
|
||||||
|
sed -i 's/^hypervisor_params.*$/hypervisor_params = \"kvm-pit.lost_tick_policy=discard pcie-root-port.fast-plug=1 pcie-root-port.x-speed=16 pcie-root-port.x-width=32 pcie-root-port.fast-unplug=1\"/' $kata_config_path
|
||||||
|
sed -i 's/^#pcie_root_port.*$/pcie_root_port = 25/' $kata_config_path
|
||||||
|
else
|
||||||
|
sed -i 's#block_device_driver = \"virtio-scsi\"#block_device_driver = \"virtio-blk\"#' $kata_config_path
|
||||||
|
sed -i 's/^#hotplug_vfio_on_root_bus/hotplug_vfio_on_root_bus/' $kata_config_path
|
||||||
|
fi
|
||||||
|
|
||||||
|
# debug config
|
||||||
|
sed -i 's/^#enable_debug.*$/enable_debug = true/' $kata_config_path
|
||||||
|
|
||||||
|
# other config
|
||||||
|
sed -i 's#"/usr/bin/qemu.*"$#"/usr/bin/qemu-kvm"#' $kata_config_path
|
||||||
|
sed -i 's#/usr/share/kata-containers/vmlinuz\.container#/var/lib/kata/kernel#' $kata_config_path
|
||||||
|
sed -i 's#/usr/share/kata-containers/kata-containers-initrd\.img#/var/lib/kata/kata-containers-initrd\.img#' $kata_config_path
|
||||||
|
sed -i 's/^image/#image/' $kata_config_path
|
||||||
|
sed -i 's/^default_memory.*$/default_memory = 1024/' $kata_config_path
|
||||||
|
sed -i 's/^#enable_blk_mount/enable_blk_mount/' $kata_config_path
|
||||||
|
sed -i 's/^#block_device_cache_direct.*$/block_device_cache_direct = true/' $kata_config_path
|
||||||
|
sed -i 's/^#block_device_cache_set.*$/block_device_cache_set = true/' $kata_config_path
|
||||||
|
sed -i 's#/usr/libexec/kata-containers/kata-proxy#/usr/bin/kata-proxy#' $kata_config_path
|
||||||
|
sed -i 's#/usr/libexec/kata-containers/kata-shim#/usr/bin/kata-shim#' $kata_config_path
|
||||||
|
sed -i 's#/usr/libexec/kata-containers/kata-netmon#/usr/bin/kata-netmon#' $kata_config_path
|
||||||
|
sed -i 's/^#disable_new_netns.*$/disable_new_netns = true/' $kata_config_path
|
||||||
|
sed -i 's/^#disable_vhost_net.*$/disable_vhost_net = true/' $kata_config_path
|
||||||
|
sed -i 's/^internetworking_model.*$/internetworking_model=\"none\"/' $kata_config_path
|
||||||
|
sed -i 's/^enable_compat_old_cni.*$/#enable_compat_old_cni = true/' $kata_config_path
|
||||||
|
sed -i 's/^sandbox_cgroup_only.*$/sandbox_cgroup_only = true/' $kata_config_path
|
||||||
|
|
||||||
|
set +e
|
||||||
|
|
||||||
|
%install
|
||||||
|
cd %{_builddir}/%{name}-%{version}
|
||||||
|
mkdir -p -m 750 %{buildroot}/usr/bin
|
||||||
|
install -p -m 750 ./kata-runtime %{buildroot}/usr/bin
|
||||||
|
install -p -m 750 ./kata-netmon %{buildroot}/usr/bin
|
||||||
|
install -p -m 750 ./containerd-shim-kata-v2 %{buildroot}/usr/bin
|
||||||
|
mkdir -p -m 750 %{buildroot}/usr/share/defaults/kata-containers
|
||||||
|
install -p -m 640 ./cli/config/configuration-qemu.toml %{buildroot}/usr/share/defaults/kata-containers/configuration.toml
|
||||||
|
|
||||||
|
%clean
|
||||||
|
|
||||||
|
%files
|
||||||
|
/usr/bin/kata-runtime
|
||||||
|
/usr/bin/kata-netmon
|
||||||
|
/usr/bin/containerd-shim-kata-v2
|
||||||
|
/usr/share/defaults/kata-containers/configuration.toml
|
||||||
|
|
||||||
|
%changelog
|
||||||
|
* Wed June 16 2021 gaohuatao<gaohuatao@huawei.com> - 1.11.1-15
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC: move timeout of waitProcess to stop process
|
||||||
|
|
||||||
|
* Thu June 3 2021 gaohuatao<gaohuatao@huawei.com> - 1.11.1-14
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:put timeout to client of wait rpc to support shimv2
|
||||||
|
|
||||||
|
* Wed May 12 2021 gaohuatao<gaohuatao@huawei.com> - 1.11.1-13
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:shimv2 write exit code in integer byte order
|
||||||
|
|
||||||
|
* Wed Apr 28 2021 gaohuatao<gaohuatao@huawei.com> - 1.11.1-12
|
||||||
|
- Type:feature
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:support kata shimv2 used by iSulad and containerd
|
||||||
|
|
||||||
|
* Tue Nov 17 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-11
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:upgrade
|
||||||
|
- DESC:fix cpu resource limited problem when sandox_cgroup_with_emulator config is enabled
|
||||||
|
|
||||||
|
* Fri Oct 9 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-10
|
||||||
|
- Type:feature
|
||||||
|
- ID:NA
|
||||||
|
- SUG:restart
|
||||||
|
- DESC:support using CNI plugin to insert mutiple network interfaces at the same time
|
||||||
|
|
||||||
|
* Mon Sep 28 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-9
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:restart
|
||||||
|
- DESC:retry inserting of CNI interface when netmon is enable
|
||||||
|
|
||||||
|
* Sun Sep 27 2020 LiangZhang<zhangliang5@Huawei.com> - 1.11.1-8
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:fix cmd params of direct use stratovirt binary
|
||||||
|
|
||||||
|
* Thu Sep 24 2020 LiangZhang<zhangliang5@Huawei.com> - 1.11.1-7
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:fix invalid cmdline when start sandbox stratovirt
|
||||||
|
|
||||||
|
* Mon Sep 21 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-6
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:fix sandboxRuntimeRootPath left problem
|
||||||
|
|
||||||
|
* Mon Sep 21 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-5
|
||||||
|
- Type:enhancement
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:add support for host cgroups with emulator
|
||||||
|
|
||||||
|
* Mon Sep 21 2020 LiangZhang<zhangliang5@Huawei.com> - 1.11.1-4
|
||||||
|
- Type:enhancement
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:add support of new sandbox StratoVirt
|
||||||
|
|
||||||
|
* Sat Sep 19 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-3
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:fix del-iface doesn't delete the tap interface in the host problem
|
||||||
|
|
||||||
|
* Sat Sep 5 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-2
|
||||||
|
- Type:enhancement
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:use URL format for Source0
|
||||||
|
|
||||||
|
* Wed Aug 26 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-1
|
||||||
|
- Type:enhancement
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:modify kata-runtime spec file to build seperately
|
||||||
@ -0,0 +1,44 @@
|
|||||||
|
From 73fe7242d18a10a86bc216ec5e33a10a8751f85f Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Fri, 24 Jul 2020 22:22:00 +0800
|
||||||
|
Subject: [PATCH 01/50] qmp: fix kata-runtime hungs when qemu process is D/T
|
||||||
|
state
|
||||||
|
|
||||||
|
reason: When set qemu's status to T and execute add-iface command
|
||||||
|
the command will hung all the time.It hungs while wait for qemu
|
||||||
|
to return version messages.
|
||||||
|
|
||||||
|
When qmp starts, set the timeout time to 15 seconds.When times
|
||||||
|
out, return qmp starts failed.We choose 15 seconds to keep consistent
|
||||||
|
with the agent client start.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
vendor/github.com/intel/govmm/qemu/checklist | 1 +
|
||||||
|
vendor/github.com/intel/govmm/qemu/qmp.go | 2 ++
|
||||||
|
2 files changed, 3 insertions(+)
|
||||||
|
create mode 100644 vendor/github.com/intel/govmm/qemu/checklist
|
||||||
|
|
||||||
|
diff --git a/vendor/github.com/intel/govmm/qemu/checklist b/vendor/github.com/intel/govmm/qemu/checklist
|
||||||
|
new file mode 100644
|
||||||
|
index 00000000..b32f1855
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/vendor/github.com/intel/govmm/qemu/checklist
|
||||||
|
@@ -0,0 +1 @@
|
||||||
|
+add timeout when qmp start to avoid qmp client hungs all the time
|
||||||
|
diff --git a/vendor/github.com/intel/govmm/qemu/qmp.go b/vendor/github.com/intel/govmm/qemu/qmp.go
|
||||||
|
index bf9a77dd..a64039de 100644
|
||||||
|
--- a/vendor/github.com/intel/govmm/qemu/qmp.go
|
||||||
|
+++ b/vendor/github.com/intel/govmm/qemu/qmp.go
|
||||||
|
@@ -722,6 +722,8 @@ func QMPStart(ctx context.Context, socket string, cfg QMPConfig, disconnectedCh
|
||||||
|
if q.version == nil {
|
||||||
|
return nil, nil, fmt.Errorf("failed to find QMP version information")
|
||||||
|
}
|
||||||
|
+ case <-time.After(15 * time.Second):
|
||||||
|
+ return nil, nil, fmt.Errorf("qmp start time out")
|
||||||
|
}
|
||||||
|
|
||||||
|
return q, q.version, nil
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -1,18 +1,22 @@
|
|||||||
From c355523761598154653466033a1d88643d3fd3df Mon Sep 17 00:00:00 2001
|
From 1efb88fbf554f3977a1a8aa1c79bc70cc1b66953 Mon Sep 17 00:00:00 2001
|
||||||
From: jikui <jikui2@huawei.com>
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
Date: Thu, 4 Nov 2021 20:27:22 +0800
|
Date: Sat, 25 Jul 2020 09:22:08 +0800
|
||||||
Subject: [PATCH 2/8] kata-runtime: fix kata-runtime skip read lines in
|
Subject: [PATCH 02/50] kata-runtime: fix kata-runtime skip read lines in
|
||||||
/proc/mounts file problem
|
/proc/mounts file problem
|
||||||
|
|
||||||
Signed-off-by: jikui <jikui2@huawei.com>
|
reason: Since /proc/mounts is a virtual file which is changed dynamically by kernel,
|
||||||
---
|
if we use file pointer to read content in this file line by line, we may miss read
|
||||||
.../virtcontainers/utils/utils_linux.go | 61 +++++++++++--------
|
some lines. So we retry read /proc/mounts file again to fix this problem.
|
||||||
1 file changed, 36 insertions(+), 25 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/src/runtime/virtcontainers/utils/utils_linux.go b/src/runtime/virtcontainers/utils/utils_linux.go
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
index 3c14e0c..c6dbd0a 100644
|
---
|
||||||
--- a/src/runtime/virtcontainers/utils/utils_linux.go
|
virtcontainers/utils/utils_linux.go | 58 +++++++++++++++++++++++--------------
|
||||||
+++ b/src/runtime/virtcontainers/utils/utils_linux.go
|
1 file changed, 36 insertions(+), 22 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/utils/utils_linux.go b/virtcontainers/utils/utils_linux.go
|
||||||
|
index ad870d63..6cef4cfb 100644
|
||||||
|
--- a/virtcontainers/utils/utils_linux.go
|
||||||
|
+++ b/virtcontainers/utils/utils_linux.go
|
||||||
@@ -7,15 +7,18 @@ package utils
|
@@ -7,15 +7,18 @@ package utils
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@ -35,12 +39,12 @@ index 3c14e0c..c6dbd0a 100644
|
|||||||
@@ -93,6 +96,7 @@ const (
|
@@ -93,6 +96,7 @@ const (
|
||||||
procMountsFile = "/proc/mounts"
|
procMountsFile = "/proc/mounts"
|
||||||
|
|
||||||
fieldsPerLine = 6
|
fieldsPerLine = 6
|
||||||
+ maxRetryTimes = 5
|
+ maxRetryTimes = 5
|
||||||
vfioAPSysfsDir = "vfio_ap"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -110,37 +114,44 @@ func GetDevicePathAndFsType(mountPoint string) (devicePath, fsType string, err e
|
const (
|
||||||
|
@@ -109,35 +113,45 @@ func GetDevicePathAndFsType(mountPoint string) (devicePath, fsType string, err e
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -53,7 +57,9 @@ index 3c14e0c..c6dbd0a 100644
|
|||||||
- }
|
- }
|
||||||
-
|
-
|
||||||
- defer file.Close()
|
- defer file.Close()
|
||||||
-
|
+ for retry <= maxRetryTimes {
|
||||||
|
+ var content []byte
|
||||||
|
|
||||||
- reader := bufio.NewReader(file)
|
- reader := bufio.NewReader(file)
|
||||||
- for {
|
- for {
|
||||||
- var line string
|
- var line string
|
||||||
@ -61,13 +67,11 @@ index 3c14e0c..c6dbd0a 100644
|
|||||||
- line, err = reader.ReadString('\n')
|
- line, err = reader.ReadString('\n')
|
||||||
- if err == io.EOF {
|
- if err == io.EOF {
|
||||||
- err = fmt.Errorf("Mount %s not found", mountPoint)
|
- err = fmt.Errorf("Mount %s not found", mountPoint)
|
||||||
+ for retry <= maxRetryTimes {
|
|
||||||
+ var content []byte
|
|
||||||
+ content, err = ioutil.ReadFile(procMountsFile)
|
+ content, err = ioutil.ReadFile(procMountsFile)
|
||||||
+ if err != nil {
|
+ if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
-
|
|
||||||
- fields := strings.Fields(line)
|
- fields := strings.Fields(line)
|
||||||
- if len(fields) != fieldsPerLine {
|
- if len(fields) != fieldsPerLine {
|
||||||
- err = fmt.Errorf("Incorrect no of fields (expected %d, got %d)) :%s", fieldsPerLine, len(fields), line)
|
- err = fmt.Errorf("Incorrect no of fields (expected %d, got %d)) :%s", fieldsPerLine, len(fields), line)
|
||||||
@ -96,7 +100,7 @@ index 3c14e0c..c6dbd0a 100644
|
|||||||
+ return
|
+ return
|
||||||
+ }
|
+ }
|
||||||
}
|
}
|
||||||
-
|
|
||||||
- if mountPoint == fields[procPathIndex] {
|
- if mountPoint == fields[procPathIndex] {
|
||||||
- devicePath = fields[procDeviceIndex]
|
- devicePath = fields[procDeviceIndex]
|
||||||
- fsType = fields[procTypeIndex]
|
- fsType = fields[procTypeIndex]
|
||||||
@ -108,8 +112,6 @@ index 3c14e0c..c6dbd0a 100644
|
|||||||
}
|
}
|
||||||
+ return "", "", fmt.Errorf("retry %d times fail to get devicePath adn fs type", maxRetryTimes)
|
+ return "", "", fmt.Errorf("retry %d times fail to get devicePath adn fs type", maxRetryTimes)
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsAPVFIOMediatedDevice decides whether a device is a VFIO-AP device
|
|
||||||
--
|
--
|
||||||
2.25.1
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,55 @@
|
|||||||
|
From cf595941e1d105af23bc006bf1998ac072733d0a Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Sat, 25 Jul 2020 10:03:35 +0800
|
||||||
|
Subject: [PATCH 03/50] kata-runtime: fix kata-proxy process left problem
|
||||||
|
|
||||||
|
reason: stopSandbox function will send the DestroySandboxRequest
|
||||||
|
to kata-agent in the VM and then kill the kata-proxy process in
|
||||||
|
the host. However, if k.sendReq(DestroySandboxRequest) get error,
|
||||||
|
stopSandbox will return immediately not execute the following kill
|
||||||
|
kata-proxy process statement, which cause the kata-process left.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/kata_agent.go | 18 +++++++++++-------
|
||||||
|
1 file changed, 11 insertions(+), 7 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
|
||||||
|
index a0cf190e..be5e96aa 100644
|
||||||
|
--- a/virtcontainers/kata_agent.go
|
||||||
|
+++ b/virtcontainers/kata_agent.go
|
||||||
|
@@ -976,6 +976,17 @@ func (k *kataAgent) stopSandbox(sandbox *Sandbox) error {
|
||||||
|
return errorMissingProxy
|
||||||
|
}
|
||||||
|
|
||||||
|
+ // since stopSandbox will destroy the sandbox in the VM, and we don't need
|
||||||
|
+ // kata-proxy process to communicate with kata-agent again, so we should
|
||||||
|
+ // make sure kata-proxy can be killed cleanly, even when k.sendReq(DestroySandboxRequest)
|
||||||
|
+ // return error
|
||||||
|
+ defer func() {
|
||||||
|
+ _ = k.proxy.stop(k.state.ProxyPid)
|
||||||
|
+ // clean up agent state
|
||||||
|
+ k.state.ProxyPid = -1
|
||||||
|
+ k.state.URL = ""
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
req := &grpc.DestroySandboxRequest{}
|
||||||
|
|
||||||
|
if _, err := k.sendReq(req); err != nil {
|
||||||
|
@@ -989,13 +1000,6 @@ func (k *kataAgent) stopSandbox(sandbox *Sandbox) error {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
- if err := k.proxy.stop(k.state.ProxyPid); err != nil {
|
||||||
|
- return err
|
||||||
|
- }
|
||||||
|
-
|
||||||
|
- // clean up agent state
|
||||||
|
- k.state.ProxyPid = -1
|
||||||
|
- k.state.URL = ""
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,65 @@
|
|||||||
|
From 025520f7fd3aeb5ed53b468b5e494b1bbb6674ae Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Sat, 25 Jul 2020 11:25:34 +0800
|
||||||
|
Subject: [PATCH 04/50] kata-runtime: keep the process name of qemu same as
|
||||||
|
configured path
|
||||||
|
|
||||||
|
reason: inorder to make testcase scripts can use the same hypervisor
|
||||||
|
name no matter what version hypervisor use, keep the process name of
|
||||||
|
hypervisor same as configured path in the configuration.toml file
|
||||||
|
instead of the resolved path of symbol link.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
pkg/katautils/config.go | 8 +++++++-
|
||||||
|
pkg/katautils/config_test.go | 4 ++--
|
||||||
|
2 files changed, 9 insertions(+), 3 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
||||||
|
index 14794a24..349e667f 100644
|
||||||
|
--- a/pkg/katautils/config.go
|
||||||
|
+++ b/pkg/katautils/config.go
|
||||||
|
@@ -10,6 +10,7 @@ import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
|
+ "path/filepath"
|
||||||
|
goruntime "runtime"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
@@ -172,7 +173,12 @@ func (h hypervisor) path() (string, error) {
|
||||||
|
p = defaultHypervisorPath
|
||||||
|
}
|
||||||
|
|
||||||
|
- return ResolvePath(p)
|
||||||
|
+ absolutePath, err := filepath.Abs(p)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return "", err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return absolutePath, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h hypervisor) ctlpath() (string, error) {
|
||||||
|
diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go
|
||||||
|
index 221a4b55..2eae1f6a 100644
|
||||||
|
--- a/pkg/katautils/config_test.go
|
||||||
|
+++ b/pkg/katautils/config_test.go
|
||||||
|
@@ -1061,12 +1061,12 @@ func TestHypervisorDefaultsHypervisor(t *testing.T) {
|
||||||
|
assert.NoError(err)
|
||||||
|
assert.Equal(p, defaultHypervisorPath, "default hypervisor path wrong")
|
||||||
|
|
||||||
|
- // test path resolution
|
||||||
|
+ // test path resolution, just return the absolute path instead of resolved path
|
||||||
|
defaultHypervisorPath = testHypervisorLinkPath
|
||||||
|
h = hypervisor{}
|
||||||
|
p, err = h.path()
|
||||||
|
assert.NoError(err)
|
||||||
|
- assert.Equal(p, testHypervisorPath)
|
||||||
|
+ assert.Equal(p, testHypervisorLinkPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHypervisorDefaultsKernel(t *testing.T) {
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,66 @@
|
|||||||
|
From c279f4548ccc534f1c65723bf9994c448e510d3d Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Sat, 25 Jul 2020 11:56:35 +0800
|
||||||
|
Subject: [PATCH 05/50] cgroups: increase delete cgroup retry times
|
||||||
|
|
||||||
|
reason: inorder to make sure cgroup dir to be deleted, so we increase
|
||||||
|
the retry times when delete cgroup dir failed.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
vendor/github.com/containerd/cgroups/cgroup.go | 4 ++--
|
||||||
|
vendor/github.com/containerd/cgroups/utils.go | 9 ++++++---
|
||||||
|
2 files changed, 8 insertions(+), 5 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/vendor/github.com/containerd/cgroups/cgroup.go b/vendor/github.com/containerd/cgroups/cgroup.go
|
||||||
|
index 53866685..69612b0a 100644
|
||||||
|
--- a/vendor/github.com/containerd/cgroups/cgroup.go
|
||||||
|
+++ b/vendor/github.com/containerd/cgroups/cgroup.go
|
||||||
|
@@ -223,7 +223,7 @@ func (c *cgroup) Delete() error {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := d.Delete(sp); err != nil {
|
||||||
|
- errors = append(errors, string(s.Name()))
|
||||||
|
+ errors = append(errors, fmt.Sprintf("delete %s get error: %v", string(s.Name()), err.Error()))
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
@@ -234,7 +234,7 @@ func (c *cgroup) Delete() error {
|
||||||
|
}
|
||||||
|
path := p.Path(sp)
|
||||||
|
if err := remove(path); err != nil {
|
||||||
|
- errors = append(errors, path)
|
||||||
|
+ errors = append(errors, fmt.Sprintf("remove path %s get error: %v", path, err.Error()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
diff --git a/vendor/github.com/containerd/cgroups/utils.go b/vendor/github.com/containerd/cgroups/utils.go
|
||||||
|
index 8a97d04d..82dbe2d3 100644
|
||||||
|
--- a/vendor/github.com/containerd/cgroups/utils.go
|
||||||
|
+++ b/vendor/github.com/containerd/cgroups/utils.go
|
||||||
|
@@ -99,16 +99,19 @@ func defaults(root string) ([]Subsystem, error) {
|
||||||
|
// retrying the remove after a exp timeout
|
||||||
|
func remove(path string) error {
|
||||||
|
delay := 10 * time.Millisecond
|
||||||
|
- for i := 0; i < 5; i++ {
|
||||||
|
+ var err error
|
||||||
|
+ var count int = 0
|
||||||
|
+ for i := 0; i < 10; i++ {
|
||||||
|
if i != 0 {
|
||||||
|
time.Sleep(delay)
|
||||||
|
delay *= 2
|
||||||
|
}
|
||||||
|
- if err := os.RemoveAll(path); err == nil {
|
||||||
|
+ if err = os.RemoveAll(path); err == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
+ count++
|
||||||
|
}
|
||||||
|
- return fmt.Errorf("cgroups: unable to remove path %q", path)
|
||||||
|
+ return fmt.Errorf("cgroups: unable to remove path %q, err: %v, count:%d", path, err, count)
|
||||||
|
}
|
||||||
|
|
||||||
|
// readPids will read all the pids of processes in a cgroup by the provided path
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,39 @@
|
|||||||
|
From 3dc10421f177900c0ee94fc49b32ec66a46d9331 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Mon, 27 Jul 2020 19:18:50 +0800
|
||||||
|
Subject: [PATCH 06/50] kata-runtime: fix umount container rootfs dir return
|
||||||
|
ivalid argument error
|
||||||
|
|
||||||
|
reason: If sandbox hypervisor doesn't use block device driver for hotplugging container
|
||||||
|
rootfs block device into guest, kata-runtime will bind mount container rootfs dir to 9p
|
||||||
|
kataShared dir. However, container stop() function will always call bindUnmountContainerRootfs
|
||||||
|
function no matter block device driver is used or not. So we just need to call
|
||||||
|
bindUnmountContainerRootfs only if rootfs is bind mount to guest by 9p.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/container.go | 8 ++++++--
|
||||||
|
1 file changed, 6 insertions(+), 2 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/container.go b/virtcontainers/container.go
|
||||||
|
index 9e2d1e94..b42cc6e9 100644
|
||||||
|
--- a/virtcontainers/container.go
|
||||||
|
+++ b/virtcontainers/container.go
|
||||||
|
@@ -1120,8 +1120,12 @@ func (c *Container) stop(force bool) error {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
- if err := bindUnmountContainerRootfs(c.ctx, getMountPath(c.sandbox.id), c.id); err != nil && !force {
|
||||||
|
- return err
|
||||||
|
+ // umount container rootfs dir only if container use 9p
|
||||||
|
+ // to bind mount host container rootfs to 9p shared dir
|
||||||
|
+ if c.state.BlockDeviceID == "" {
|
||||||
|
+ if err := bindUnmountContainerRootfs(c.ctx, getMountPath(c.sandbox.id), c.id); err != nil && !force {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := c.detachDevices(); err != nil && !force {
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,855 @@
|
|||||||
|
From d93da1875ed7f1a6061cffb13475506d73c86003 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Sat, 25 Jul 2020 16:04:19 +0800
|
||||||
|
Subject: [PATCH 07/50] kata-runtime: enhance reliability when kata related
|
||||||
|
process
|
||||||
|
|
||||||
|
reason: enhance the reliability when kata related processes is abnormal,
|
||||||
|
make kata-container still destroy the sandbox and clean up all resources.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
cli/delete.go | 6 ++
|
||||||
|
cli/kill.go | 3 +-
|
||||||
|
virtcontainers/acrn.go | 2 +-
|
||||||
|
virtcontainers/agent.go | 3 +
|
||||||
|
virtcontainers/api.go | 67 +++++++++++++++++++++
|
||||||
|
virtcontainers/clh.go | 2 +-
|
||||||
|
virtcontainers/container.go | 55 +++++++++++++++--
|
||||||
|
virtcontainers/fc.go | 2 +-
|
||||||
|
virtcontainers/hypervisor.go | 2 +-
|
||||||
|
virtcontainers/kata_agent.go | 31 ++++++----
|
||||||
|
virtcontainers/mock_hypervisor.go | 2 +-
|
||||||
|
virtcontainers/mock_hypervisor_test.go | 2 +-
|
||||||
|
virtcontainers/noop_agent.go | 4 ++
|
||||||
|
virtcontainers/pkg/oci/utils.go | 5 ++
|
||||||
|
virtcontainers/qemu.go | 51 +++++++++-------
|
||||||
|
virtcontainers/sandbox.go | 106 +++++++++++++++++++++++++++++----
|
||||||
|
virtcontainers/types/sandbox.go | 14 ++++-
|
||||||
|
virtcontainers/utils/utils.go | 46 ++++++++++++++
|
||||||
|
virtcontainers/vm.go | 4 +-
|
||||||
|
19 files changed, 348 insertions(+), 59 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/cli/delete.go b/cli/delete.go
|
||||||
|
index c2ce52a4..2f5586e5 100644
|
||||||
|
--- a/cli/delete.go
|
||||||
|
+++ b/cli/delete.go
|
||||||
|
@@ -110,6 +110,12 @@ func delete(ctx context.Context, containerID string, force bool) error {
|
||||||
|
forceStop = true
|
||||||
|
}
|
||||||
|
|
||||||
|
+ if oci.StateToOCIState(status.State.State) == oci.StateUnhealthy {
|
||||||
|
+ // Set forceStop and force bool flag to true to force delete everything
|
||||||
|
+ forceStop = true
|
||||||
|
+ force = true
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
switch containerType {
|
||||||
|
case vc.PodSandbox:
|
||||||
|
if err := deleteSandbox(ctx, sandboxID, force); err != nil {
|
||||||
|
diff --git a/cli/kill.go b/cli/kill.go
|
||||||
|
index 60fa41e0..b228205f 100644
|
||||||
|
--- a/cli/kill.go
|
||||||
|
+++ b/cli/kill.go
|
||||||
|
@@ -133,11 +133,12 @@ func kill(ctx context.Context, containerID, signal string, all bool) error {
|
||||||
|
kataLog.WithField("signal", signal).WithField("container state", status.State.State).Info("kill")
|
||||||
|
|
||||||
|
// container MUST be created, running or paused
|
||||||
|
+ // If container state is unhealthy, should process this exceptional case separately
|
||||||
|
if status.State.State == types.StateReady || status.State.State == types.StateRunning || status.State.State == types.StatePaused {
|
||||||
|
if err := vci.KillContainer(ctx, sandboxID, containerID, signum, all); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
- } else if !all {
|
||||||
|
+ } else if !all && status.State.State != types.StateUnhealthy {
|
||||||
|
return fmt.Errorf("container not running")
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/acrn.go b/virtcontainers/acrn.go
|
||||||
|
index 761eda03..10cae06f 100644
|
||||||
|
--- a/virtcontainers/acrn.go
|
||||||
|
+++ b/virtcontainers/acrn.go
|
||||||
|
@@ -475,7 +475,7 @@ func (a *Acrn) waitSandbox(timeoutSecs int) error {
|
||||||
|
}
|
||||||
|
|
||||||
|
// stopSandbox will stop the Sandbox's VM.
|
||||||
|
-func (a *Acrn) stopSandbox() (err error) {
|
||||||
|
+func (a *Acrn) stopSandbox(force bool) (err error) {
|
||||||
|
span, _ := a.trace("stopSandbox")
|
||||||
|
defer span.Finish()
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go
|
||||||
|
index c62107ec..be9526c7 100644
|
||||||
|
--- a/virtcontainers/agent.go
|
||||||
|
+++ b/virtcontainers/agent.go
|
||||||
|
@@ -259,4 +259,7 @@ type agent interface {
|
||||||
|
|
||||||
|
// load data from disk
|
||||||
|
load(persistapi.AgentState)
|
||||||
|
+
|
||||||
|
+ // get proxy process pid
|
||||||
|
+ getProxyPid() int
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
|
||||||
|
index de569713..fa82d163 100644
|
||||||
|
--- a/virtcontainers/api.go
|
||||||
|
+++ b/virtcontainers/api.go
|
||||||
|
@@ -7,8 +7,10 @@ package virtcontainers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
+ "fmt"
|
||||||
|
"os"
|
||||||
|
"runtime"
|
||||||
|
+ "strings"
|
||||||
|
"syscall"
|
||||||
|
|
||||||
|
deviceApi "github.com/kata-containers/runtime/virtcontainers/device/api"
|
||||||
|
@@ -18,6 +20,7 @@ import (
|
||||||
|
vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/store"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/types"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||||
|
specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
opentracing "github.com/opentracing/opentracing-go"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
@@ -597,20 +600,51 @@ func statusContainer(sandbox *Sandbox, containerID string) (ContainerStatus, err
|
||||||
|
container.state.State == types.StatePaused) &&
|
||||||
|
container.process.Pid > 0 {
|
||||||
|
|
||||||
|
+ // If container state is active, however kata-proxy and qemu process all exit already
|
||||||
|
+ // which means sandbox has beed stopped exceptionally, then we should force delete
|
||||||
|
+ // sandbox and container state files in sandbox.Store
|
||||||
|
+ if sandbox.shouldForceDelete() {
|
||||||
|
+ virtLog.Logger.Warn("sandbox status is abnormal, sandbox should be force deleted")
|
||||||
|
+ sandbox.forceDeleteSandbox()
|
||||||
|
+ return ContainerStatus{}, fmt.Errorf("sandbox has beed stopped exceptionally")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
running, err := isShimRunning(container.process.Pid)
|
||||||
|
if err != nil {
|
||||||
|
return ContainerStatus{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
+ // If kata-shim process exit or be killed, need to stop the container
|
||||||
|
if !running {
|
||||||
|
virtLog.WithFields(logrus.Fields{
|
||||||
|
"state": container.state.State,
|
||||||
|
"pid": container.process.Pid}).
|
||||||
|
Info("container isn't running")
|
||||||
|
+
|
||||||
|
if err := container.stop(true); err != nil {
|
||||||
|
return ContainerStatus{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+ isPodSandbox := (containerID == sandbox.id)
|
||||||
|
+
|
||||||
|
+ // If sandbox is unhealthy, process it correctly
|
||||||
|
+ if !sandbox.health() {
|
||||||
|
+ // process podSandbox container type case
|
||||||
|
+ if isPodSandbox {
|
||||||
|
+ if err := processUnhealthySandbox(sandbox, container); err != nil {
|
||||||
|
+ return ContainerStatus{}, err
|
||||||
|
+ }
|
||||||
|
+ } else {
|
||||||
|
+ // If container type is pod_container, which means container operations can not be
|
||||||
|
+ // processed successfully, we should return the error as soon as possible
|
||||||
|
+ if err := container.setContainerState(types.StateUnhealthy); err != nil {
|
||||||
|
+ return ContainerStatus{}, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return ContainerStatus{}, fmt.Errorf("container status is unhealthy, stop container failed")
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
|
||||||
|
return ContainerStatus{
|
||||||
|
@@ -1016,3 +1050,36 @@ func CleanupContainer(ctx context.Context, sandboxID, containerID string, force
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+// procesUnhealthySandbox only change sandbox state to unhealthy
|
||||||
|
+// when caller is kata-runtime kill or kata-runtime delete
|
||||||
|
+func processUnhealthySandbox(sandbox *Sandbox, container *Container) error {
|
||||||
|
+ // Set all containers state to unhealthy
|
||||||
|
+ if err := sandbox.setContainersState(types.StateUnhealthy); err != nil {
|
||||||
|
+ container.Logger().WithError(err).Warn("set all containers state to unhealthy fail")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // Set sandbox state to unhealthy
|
||||||
|
+ if err := sandbox.setSandboxState(types.StateUnhealthy); err != nil {
|
||||||
|
+ container.Logger().WithError(err).Warn("set sandbox state to unhealthy fail")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ forceDelete := false
|
||||||
|
+
|
||||||
|
+ // If process is kata-runtime kill or kata-runtime delete,
|
||||||
|
+ // we should kill or delete sandbox forcefully
|
||||||
|
+ if cmdline, err := utils.GetProcessCmdline(os.Getpid()); err != nil {
|
||||||
|
+ container.Logger().WithError(err).Warn("fail to get process cmdline info")
|
||||||
|
+ } else {
|
||||||
|
+ forceDelete = strings.Contains(cmdline, "kill") || strings.Contains(cmdline, "delete")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if forceDelete {
|
||||||
|
+ // force stop podSandbox type container's kata-shim process
|
||||||
|
+ if err := stopShim(container.process.Pid); err != nil {
|
||||||
|
+ container.Logger().WithError(err).Warn("fail to stop podSandbox type container kata-shim")
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/clh.go b/virtcontainers/clh.go
|
||||||
|
index d40b698b..59510b02 100644
|
||||||
|
--- a/virtcontainers/clh.go
|
||||||
|
+++ b/virtcontainers/clh.go
|
||||||
|
@@ -569,7 +569,7 @@ func (clh *cloudHypervisor) resumeSandbox() error {
|
||||||
|
}
|
||||||
|
|
||||||
|
// stopSandbox will stop the Sandbox's VM.
|
||||||
|
-func (clh *cloudHypervisor) stopSandbox() (err error) {
|
||||||
|
+func (clh *cloudHypervisor) stopSandbox(force bool) (err error) {
|
||||||
|
span, _ := clh.trace("stopSandbox")
|
||||||
|
defer span.Finish()
|
||||||
|
clh.Logger().WithField("function", "stopSandbox").Info("Stop Sandbox")
|
||||||
|
diff --git a/virtcontainers/container.go b/virtcontainers/container.go
|
||||||
|
index b42cc6e9..9485e708 100644
|
||||||
|
--- a/virtcontainers/container.go
|
||||||
|
+++ b/virtcontainers/container.go
|
||||||
|
@@ -17,8 +17,12 @@ import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/containerd/cgroups"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/device/config"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/device/manager"
|
||||||
|
vccgroups "github.com/kata-containers/runtime/virtcontainers/pkg/cgroups"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/pkg/rootless"
|
||||||
|
vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/store"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/types"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/utils"
|
||||||
|
specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
@@ -26,11 +30,6 @@ import (
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
-
|
||||||
|
- "github.com/kata-containers/runtime/virtcontainers/device/config"
|
||||||
|
- "github.com/kata-containers/runtime/virtcontainers/device/manager"
|
||||||
|
- "github.com/kata-containers/runtime/virtcontainers/pkg/rootless"
|
||||||
|
- "github.com/kata-containers/runtime/virtcontainers/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// https://github.com/torvalds/linux/blob/master/include/uapi/linux/major.h
|
||||||
|
@@ -1047,6 +1046,13 @@ func (c *Container) stop(force bool) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+ // If container state is unhealthy, just force kill the container
|
||||||
|
+ if c.state.State == types.StateUnhealthy {
|
||||||
|
+ c.forceKillContainer()
|
||||||
|
+ // after force kill container, then change container state to stopped
|
||||||
|
+ return c.setContainerState(types.StateStopped)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
if err := c.state.ValidTransition(c.state.State, types.StateStopped); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
@@ -1063,6 +1069,8 @@ func (c *Container) stop(force bool) error {
|
||||||
|
if err := stopShim(c.process.Pid); err != nil {
|
||||||
|
l.WithError(err).Warn("failed to stop shim")
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+ c.forceKillContainer()
|
||||||
|
}
|
||||||
|
|
||||||
|
}()
|
||||||
|
@@ -1096,7 +1104,9 @@ func (c *Container) stop(force bool) error {
|
||||||
|
// this signal will ensure the container will get killed to match
|
||||||
|
// the state of the shim. This will allow the following call to
|
||||||
|
// stopContainer() to succeed in such particular case.
|
||||||
|
- c.kill(syscall.SIGKILL, true)
|
||||||
|
+ if err := c.kill(syscall.SIGKILL, true); err != nil {
|
||||||
|
+ c.Logger().Errorf("send signal to container failed: %v", err)
|
||||||
|
+ }
|
||||||
|
|
||||||
|
// Since the agent has supported the MultiWaitProcess, it's better to
|
||||||
|
// wait the process here to make sure the process has exited before to
|
||||||
|
@@ -1582,3 +1592,36 @@ func (c *Container) cgroupsUpdate(resources specs.LinuxResources) error {
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+// forceDeleteContainer force clean container mount info and resources stored in the disk
|
||||||
|
+func (c *Container) forceDeleteContainer() {
|
||||||
|
+ if err := c.unmountHostMounts(); err != nil {
|
||||||
|
+ c.Logger().WithError(err).Warn("container force delete umount host mounts fail")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := c.sandbox.removeContainer(c.id); err != nil {
|
||||||
|
+ c.Logger().WithError(err).Warn("sandbox removeContainer fail")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := c.store.Delete(); err != nil {
|
||||||
|
+ c.Logger().WithError(err).Warn("force delete container store fail")
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (c *Container) forceKillContainer() {
|
||||||
|
+ if err := c.setContainerState(types.StateStopped); err != nil {
|
||||||
|
+ c.Logger().WithError(err).Warn("force kill container: change container state to StateStopped failed")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := c.unmountHostMounts(); err != nil {
|
||||||
|
+ c.Logger().WithError(err).Warn("force kill container: umount container host mounts failed")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := c.detachDevices(); err != nil {
|
||||||
|
+ c.Logger().WithError(err).Warn("force kill container: detach container devices failed")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := c.removeDrive(); err != nil {
|
||||||
|
+ c.Logger().WithError(err).Warn("force kill container: remove container drive failed")
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go
|
||||||
|
index 97ef5ffc..72a8e192 100644
|
||||||
|
--- a/virtcontainers/fc.go
|
||||||
|
+++ b/virtcontainers/fc.go
|
||||||
|
@@ -864,7 +864,7 @@ func (fc *firecracker) cleanupJail() {
|
||||||
|
}
|
||||||
|
|
||||||
|
// stopSandbox will stop the Sandbox's VM.
|
||||||
|
-func (fc *firecracker) stopSandbox() (err error) {
|
||||||
|
+func (fc *firecracker) stopSandbox(force bool) (err error) {
|
||||||
|
span, _ := fc.trace("stopSandbox")
|
||||||
|
defer span.Finish()
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go
|
||||||
|
index 4b3dd3d0..fd7d1f8e 100644
|
||||||
|
--- a/virtcontainers/hypervisor.go
|
||||||
|
+++ b/virtcontainers/hypervisor.go
|
||||||
|
@@ -766,7 +766,7 @@ func generateVMSocket(id string, useVsock bool, vmStogarePath string) (interface
|
||||||
|
type hypervisor interface {
|
||||||
|
createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, stateful bool) error
|
||||||
|
startSandbox(timeout int) error
|
||||||
|
- stopSandbox() error
|
||||||
|
+ stopSandbox(force bool) error
|
||||||
|
pauseSandbox() error
|
||||||
|
saveSandbox() error
|
||||||
|
resumeSandbox() error
|
||||||
|
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
|
||||||
|
index be5e96aa..7575d326 100644
|
||||||
|
--- a/virtcontainers/kata_agent.go
|
||||||
|
+++ b/virtcontainers/kata_agent.go
|
||||||
|
@@ -57,8 +57,9 @@ const (
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
- checkRequestTimeout = 30 * time.Second
|
||||||
|
- defaultRequestTimeout = 60 * time.Second
|
||||||
|
+ checkRequestTimeout = 10 * time.Second
|
||||||
|
+ defaultRequestTimeout = 10 * time.Second
|
||||||
|
+ createContainerTimeout = 120 * time.Second
|
||||||
|
errorMissingProxy = errors.New("Missing proxy pointer")
|
||||||
|
errorMissingOCISpec = errors.New("Missing OCI specification")
|
||||||
|
defaultKataHostSharedDir = "/run/kata-containers/shared/sandboxes/"
|
||||||
|
@@ -987,17 +988,21 @@ func (k *kataAgent) stopSandbox(sandbox *Sandbox) error {
|
||||||
|
k.state.URL = ""
|
||||||
|
}()
|
||||||
|
|
||||||
|
- req := &grpc.DestroySandboxRequest{}
|
||||||
|
+ // If sandbox.state.State is unhealthy, we don't need to send DestroySandboxRequest
|
||||||
|
+ // to kata-agent, just force stop the sandbox
|
||||||
|
+ if sandbox.state.State != types.StateUnhealthy {
|
||||||
|
+ req := &grpc.DestroySandboxRequest{}
|
||||||
|
|
||||||
|
- if _, err := k.sendReq(req); err != nil {
|
||||||
|
- return err
|
||||||
|
- }
|
||||||
|
-
|
||||||
|
- if k.dynamicTracing {
|
||||||
|
- _, err := k.sendReq(&grpc.StopTracingRequest{})
|
||||||
|
- if err != nil {
|
||||||
|
+ if _, err := k.sendReq(req); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+ if k.dynamicTracing {
|
||||||
|
+ _, err := k.sendReq(&grpc.StopTracingRequest{})
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
@@ -2062,6 +2067,8 @@ func (k *kataAgent) getReqContext(reqName string) (ctx context.Context, cancel c
|
||||||
|
// Wait has no timeout
|
||||||
|
case grpcCheckRequest:
|
||||||
|
ctx, cancel = context.WithTimeout(ctx, checkRequestTimeout)
|
||||||
|
+ case grpcCreateContainerRequest:
|
||||||
|
+ ctx, cancel = context.WithTimeout(ctx, createContainerTimeout)
|
||||||
|
default:
|
||||||
|
ctx, cancel = context.WithTimeout(ctx, defaultRequestTimeout)
|
||||||
|
}
|
||||||
|
@@ -2382,3 +2389,7 @@ func (k *kataAgent) load(s persistapi.AgentState) {
|
||||||
|
k.state.ProxyPid = s.ProxyPid
|
||||||
|
k.state.URL = s.URL
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func (k *kataAgent) getProxyPid() int {
|
||||||
|
+ return k.state.ProxyPid
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/mock_hypervisor.go b/virtcontainers/mock_hypervisor.go
|
||||||
|
index 0c84e43c..a5b67491 100644
|
||||||
|
--- a/virtcontainers/mock_hypervisor.go
|
||||||
|
+++ b/virtcontainers/mock_hypervisor.go
|
||||||
|
@@ -39,7 +39,7 @@ func (m *mockHypervisor) startSandbox(timeout int) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
-func (m *mockHypervisor) stopSandbox() error {
|
||||||
|
+func (m *mockHypervisor) stopSandbox(force bool) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/mock_hypervisor_test.go b/virtcontainers/mock_hypervisor_test.go
|
||||||
|
index b73b28f2..827e3192 100644
|
||||||
|
--- a/virtcontainers/mock_hypervisor_test.go
|
||||||
|
+++ b/virtcontainers/mock_hypervisor_test.go
|
||||||
|
@@ -53,7 +53,7 @@ func TestMockHypervisorStartSandbox(t *testing.T) {
|
||||||
|
func TestMockHypervisorStopSandbox(t *testing.T) {
|
||||||
|
var m *mockHypervisor
|
||||||
|
|
||||||
|
- assert.NoError(t, m.stopSandbox())
|
||||||
|
+ assert.NoError(t, m.stopSandbox(false))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMockHypervisorAddDevice(t *testing.T) {
|
||||||
|
diff --git a/virtcontainers/noop_agent.go b/virtcontainers/noop_agent.go
|
||||||
|
index 189f6b3f..8a7cd337 100644
|
||||||
|
--- a/virtcontainers/noop_agent.go
|
||||||
|
+++ b/virtcontainers/noop_agent.go
|
||||||
|
@@ -236,3 +236,7 @@ func (n *noopAgent) save() (s persistapi.AgentState) {
|
||||||
|
|
||||||
|
// load is the Noop agent state loader. It does nothing.
|
||||||
|
func (n *noopAgent) load(s persistapi.AgentState) {}
|
||||||
|
+
|
||||||
|
+func (n *noopAgent) getProxyPid() int {
|
||||||
|
+ return -1
|
||||||
|
+}
|
||||||
|
\ No newline at end of file
|
||||||
|
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
||||||
|
index 5348c57d..cd8d48ce 100644
|
||||||
|
--- a/virtcontainers/pkg/oci/utils.go
|
||||||
|
+++ b/virtcontainers/pkg/oci/utils.go
|
||||||
|
@@ -70,6 +70,9 @@ const (
|
||||||
|
|
||||||
|
// StatePaused represents a container that has been paused.
|
||||||
|
StatePaused = "paused"
|
||||||
|
+
|
||||||
|
+ // StateUnhealthy represents a container that is unhealthy
|
||||||
|
+ StateUnhealthy = "unhealthy"
|
||||||
|
)
|
||||||
|
|
||||||
|
const KernelModulesSeparator = ";"
|
||||||
|
@@ -964,6 +967,8 @@ func StateToOCIState(state types.StateString) string {
|
||||||
|
return StateStopped
|
||||||
|
case types.StatePaused:
|
||||||
|
return StatePaused
|
||||||
|
+ case types.StateUnhealthy:
|
||||||
|
+ return StateUnhealthy
|
||||||
|
default:
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
|
||||||
|
index ca286550..4b15d968 100644
|
||||||
|
--- a/virtcontainers/qemu.go
|
||||||
|
+++ b/virtcontainers/qemu.go
|
||||||
|
@@ -687,7 +687,7 @@ func (q *qemu) setupVirtiofsd() (err error) {
|
||||||
|
q.Logger().Info("virtiofsd quits")
|
||||||
|
// Wait to release resources of virtiofsd process
|
||||||
|
cmd.Process.Wait()
|
||||||
|
- q.stopSandbox()
|
||||||
|
+ q.stopSandbox(false)
|
||||||
|
}()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
@@ -922,11 +922,11 @@ func (q *qemu) waitSandbox(timeout int) error {
|
||||||
|
}
|
||||||
|
|
||||||
|
// stopSandbox will stop the Sandbox's VM.
|
||||||
|
-func (q *qemu) stopSandbox() error {
|
||||||
|
+func (q *qemu) stopSandbox(force bool) error {
|
||||||
|
span, _ := q.trace("stopSandbox")
|
||||||
|
defer span.Finish()
|
||||||
|
|
||||||
|
- q.Logger().Info("Stopping Sandbox")
|
||||||
|
+ q.Logger().Infof("force stopping Sandbox: %v", force)
|
||||||
|
if q.stopped {
|
||||||
|
q.Logger().Info("Already stopped")
|
||||||
|
return nil
|
||||||
|
@@ -937,28 +937,37 @@ func (q *qemu) stopSandbox() error {
|
||||||
|
q.stopped = true
|
||||||
|
}()
|
||||||
|
|
||||||
|
- if q.config.Debug && q.qemuConfig.LogFile != "" {
|
||||||
|
- f, err := os.OpenFile(q.qemuConfig.LogFile, os.O_RDONLY, 0)
|
||||||
|
- if err == nil {
|
||||||
|
- scanner := bufio.NewScanner(f)
|
||||||
|
- for scanner.Scan() {
|
||||||
|
- q.Logger().Debug(scanner.Text())
|
||||||
|
- }
|
||||||
|
- if err := scanner.Err(); err != nil {
|
||||||
|
- q.Logger().WithError(err).Debug("read qemu log failed")
|
||||||
|
+ if !force {
|
||||||
|
+ if q.config.Debug && q.qemuConfig.LogFile != "" {
|
||||||
|
+ f, err := os.OpenFile(q.qemuConfig.LogFile, os.O_RDONLY, 0)
|
||||||
|
+ if err == nil {
|
||||||
|
+ scanner := bufio.NewScanner(f)
|
||||||
|
+ for scanner.Scan() {
|
||||||
|
+ q.Logger().Debug(scanner.Text())
|
||||||
|
+ }
|
||||||
|
+ if err := scanner.Err(); err != nil {
|
||||||
|
+ q.Logger().WithError(err).Debug("read qemu log failed")
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
- }
|
||||||
|
|
||||||
|
- err := q.qmpSetup()
|
||||||
|
- if err != nil {
|
||||||
|
- return err
|
||||||
|
- }
|
||||||
|
+ err := q.qmpSetup()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
|
||||||
|
- err = q.qmpMonitorCh.qmp.ExecuteQuit(q.qmpMonitorCh.ctx)
|
||||||
|
- if err != nil {
|
||||||
|
- q.Logger().WithError(err).Error("Fail to execute qmp QUIT")
|
||||||
|
- return err
|
||||||
|
+ err = q.qmpMonitorCh.qmp.ExecuteQuit(q.qmpMonitorCh.ctx)
|
||||||
|
+ if err != nil {
|
||||||
|
+ q.Logger().WithError(err).Error("Fail to execute qmp QUIT")
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ } else {
|
||||||
|
+ qemuMainPid := q.getPids()[0]
|
||||||
|
+ if qemuMainPid <= 1 {
|
||||||
|
+ return fmt.Errorf("force kill qemu process pid is invalid")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ _ = syscall.Kill(qemuMainPid, syscall.SIGKILL)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index edd1af5b..78188ed7 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -12,19 +12,13 @@ import (
|
||||||
|
"math"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
+ "path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"syscall"
|
||||||
|
|
||||||
|
"github.com/containerd/cgroups"
|
||||||
|
"github.com/containernetworking/plugins/pkg/ns"
|
||||||
|
- "github.com/opencontainers/runc/libcontainer/configs"
|
||||||
|
- specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
- opentracing "github.com/opentracing/opentracing-go"
|
||||||
|
- "github.com/pkg/errors"
|
||||||
|
- "github.com/sirupsen/logrus"
|
||||||
|
- "github.com/vishvananda/netlink"
|
||||||
|
-
|
||||||
|
"github.com/kata-containers/agent/protocols/grpc"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/device/api"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/device/config"
|
||||||
|
@@ -41,6 +35,12 @@ import (
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/store"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/types"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/utils"
|
||||||
|
+ "github.com/opencontainers/runc/libcontainer/configs"
|
||||||
|
+ specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
+ opentracing "github.com/opentracing/opentracing-go"
|
||||||
|
+ "github.com/pkg/errors"
|
||||||
|
+ "github.com/sirupsen/logrus"
|
||||||
|
+ "github.com/vishvananda/netlink"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
@@ -50,6 +50,9 @@ const (
|
||||||
|
|
||||||
|
// DirMode is the permission bits used for creating a directory
|
||||||
|
DirMode = os.FileMode(0750) | os.ModeDir
|
||||||
|
+
|
||||||
|
+ // kata-proxy proces name
|
||||||
|
+ KataProxyProcessName = "kata-proxy"
|
||||||
|
)
|
||||||
|
|
||||||
|
// SandboxStatus describes a sandbox status.
|
||||||
|
@@ -1037,7 +1040,7 @@ func (s *Sandbox) startVM() (err error) {
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
if err != nil {
|
||||||
|
- s.hypervisor.stopSandbox()
|
||||||
|
+ s.hypervisor.stopSandbox(false)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
@@ -1090,7 +1093,12 @@ func (s *Sandbox) stopVM() error {
|
||||||
|
}
|
||||||
|
|
||||||
|
s.Logger().Info("Stopping VM")
|
||||||
|
- return s.hypervisor.stopSandbox()
|
||||||
|
+ forceStop := false
|
||||||
|
+ if s.state.State == types.StateUnhealthy {
|
||||||
|
+ forceStop = true
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return s.hypervisor.stopSandbox(forceStop)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Sandbox) addContainer(c *Container) error {
|
||||||
|
@@ -1591,13 +1599,15 @@ func (s *Sandbox) setSandboxState(state types.StateString) error {
|
||||||
|
return vcTypes.ErrNeedState
|
||||||
|
}
|
||||||
|
|
||||||
|
+ s.Logger().Debugf("Setting sandbox state from %v to %v", s.state.State, state)
|
||||||
|
// update in-memory state
|
||||||
|
s.state.State = state
|
||||||
|
|
||||||
|
if useOldStore(s.ctx) {
|
||||||
|
return s.store.Store(store.State, s.state)
|
||||||
|
+ } else {
|
||||||
|
+ return s.Save()
|
||||||
|
}
|
||||||
|
- return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const maxBlockIndex = 65535
|
||||||
|
@@ -2207,3 +2217,79 @@ func (s *Sandbox) GetPatchedOCISpec() *specs.Spec {
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+// health return current sandbox healthy or not
|
||||||
|
+// If qemu/kata-proxy/kata-agent process is abnormal,
|
||||||
|
+// s.agent.check() will return false
|
||||||
|
+func (s *Sandbox) health() bool {
|
||||||
|
+ err := s.agent.check()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return false
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return true
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// shouldForceDelete force delete the sandbox when kata-proxy and hypervisor process exit
|
||||||
|
+// already and current process is kata-runtime kill or kata-runtime delete
|
||||||
|
+func (s *Sandbox) shouldForceDelete() bool {
|
||||||
|
+ cmdline, err := utils.GetProcessCmdline(os.Getpid())
|
||||||
|
+ if err != nil {
|
||||||
|
+ s.Logger().Errorf("fail to get process cmdline: %v", err)
|
||||||
|
+ return false
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ proxyPid := s.agent.getProxyPid()
|
||||||
|
+ hypervisorPids := s.hypervisor.getPids()
|
||||||
|
+ if len(hypervisorPids) <= 0 {
|
||||||
|
+ s.Logger().Warnf("get hypervisor main pid fail")
|
||||||
|
+ return false
|
||||||
|
+ }
|
||||||
|
+ hypervisorMainPid := hypervisorPids[0]
|
||||||
|
+ hypervisorPath := s.hypervisor.hypervisorConfig().HypervisorPath
|
||||||
|
+ hypervisorName := filepath.Base(hypervisorPath)
|
||||||
|
+
|
||||||
|
+ if !utils.IsProcessRunning(proxyPid, KataProxyProcessName, s.id) && !utils.IsProcessRunning(hypervisorMainPid, hypervisorName, s.id) &&
|
||||||
|
+ strings.Contains(cmdline, "delete") && strings.Contains(cmdline, "force") {
|
||||||
|
+ return true
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return false
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *Sandbox) forceDeleteSandbox() {
|
||||||
|
+ for _, c := range s.containers {
|
||||||
|
+ // force delete all containers in the sandbox
|
||||||
|
+ c.forceDeleteContainer()
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ globalSandboxList.removeSandbox(s.id)
|
||||||
|
+
|
||||||
|
+ if s.monitor != nil {
|
||||||
|
+ s.monitor.stop()
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := s.hypervisor.cleanup(); err != nil {
|
||||||
|
+ s.Logger().WithError(err).Error("failed to force cleanup hypervisor resource")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ s.agent.cleanup(s)
|
||||||
|
+
|
||||||
|
+ if err := s.store.Delete(); err != nil {
|
||||||
|
+ s.Logger().WithError(err).Warn("sandbox force delete store failed")
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *Sandbox) setContainersState(state types.StateString) error {
|
||||||
|
+ if state == "" {
|
||||||
|
+ return vcTypes.ErrNeedState
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ for _, c := range s.containers {
|
||||||
|
+ if err := c.setContainerState(state); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/types/sandbox.go b/virtcontainers/types/sandbox.go
|
||||||
|
index 3b64b20a..5d586b21 100644
|
||||||
|
--- a/virtcontainers/types/sandbox.go
|
||||||
|
+++ b/virtcontainers/types/sandbox.go
|
||||||
|
@@ -28,6 +28,9 @@ const (
|
||||||
|
|
||||||
|
// StateStopped represents a sandbox/container that has been stopped.
|
||||||
|
StateStopped StateString = "stopped"
|
||||||
|
+
|
||||||
|
+ // StateUnhealthy represents a sandbox/container that's in abnormal state.
|
||||||
|
+ StateUnhealthy StateString = "unhealthy"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
@@ -90,17 +93,17 @@ func (state *StateString) validTransition(oldState StateString, newState StateSt
|
||||||
|
|
||||||
|
switch *state {
|
||||||
|
case StateReady:
|
||||||
|
- if newState == StateRunning || newState == StateStopped {
|
||||||
|
+ if newState == StateRunning || newState == StateStopped || newState == StateUnhealthy {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
case StateRunning:
|
||||||
|
- if newState == StatePaused || newState == StateStopped {
|
||||||
|
+ if newState == StatePaused || newState == StateStopped || newState == StateUnhealthy {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
case StatePaused:
|
||||||
|
- if newState == StateRunning || newState == StateStopped {
|
||||||
|
+ if newState == StateRunning || newState == StateStopped || newState == StateUnhealthy {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -108,6 +111,11 @@ func (state *StateString) validTransition(oldState StateString, newState StateSt
|
||||||
|
if newState == StateRunning {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+ case StateUnhealthy:
|
||||||
|
+ if newState == StateStopped {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Errorf("Can not move from %v to %v",
|
||||||
|
diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go
|
||||||
|
index 85c55489..2b555ebb 100644
|
||||||
|
--- a/virtcontainers/utils/utils.go
|
||||||
|
+++ b/virtcontainers/utils/utils.go
|
||||||
|
@@ -9,9 +9,13 @@ import (
|
||||||
|
"crypto/rand"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
+ "io/ioutil"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
+ "strconv"
|
||||||
|
+ "strings"
|
||||||
|
+ "syscall"
|
||||||
|
)
|
||||||
|
|
||||||
|
const cpBinaryName = "cp"
|
||||||
|
@@ -275,3 +279,45 @@ const (
|
||||||
|
MiB = KiB << 10
|
||||||
|
GiB = MiB << 10
|
||||||
|
)
|
||||||
|
+
|
||||||
|
+// Get process cmdline info by read /proc/<pid>/cmdline file
|
||||||
|
+func GetProcessCmdline(pid int) (cmdline string, err error) {
|
||||||
|
+ if pid <= 1 {
|
||||||
|
+ return "", fmt.Errorf("invalid pid number")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ bytes, err := ioutil.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "cmdline"))
|
||||||
|
+ if err != nil {
|
||||||
|
+ return "", err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return string(bytes), nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func IsProcessRunning(pid int, processName string, sandboxID string) bool {
|
||||||
|
+ if pid <= 0 {
|
||||||
|
+ return false
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ process, err := os.FindProcess(pid)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return false
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := process.Signal(syscall.Signal(0)); err != nil {
|
||||||
|
+ return false
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ cmdline, err := GetProcessCmdline(pid)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return false
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // If process's cmdline contains processName and sandboxID keyword,
|
||||||
|
+ // We think this process isn't be reused
|
||||||
|
+ if strings.Contains(cmdline, processName) && strings.Contains(cmdline, sandboxID) {
|
||||||
|
+ return true
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return false
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/vm.go b/virtcontainers/vm.go
|
||||||
|
index fcda1e97..8d27b1fe 100644
|
||||||
|
--- a/virtcontainers/vm.go
|
||||||
|
+++ b/virtcontainers/vm.go
|
||||||
|
@@ -191,7 +191,7 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) {
|
||||||
|
defer func() {
|
||||||
|
if err != nil {
|
||||||
|
virtLog.WithField("vm", id).WithError(err).Info("clean up vm")
|
||||||
|
- hypervisor.stopSandbox()
|
||||||
|
+ hypervisor.stopSandbox(false)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
@@ -333,7 +333,7 @@ func (v *VM) Disconnect() error {
|
||||||
|
func (v *VM) Stop() error {
|
||||||
|
v.logger().Info("stop vm")
|
||||||
|
|
||||||
|
- if err := v.hypervisor.stopSandbox(); err != nil {
|
||||||
|
+ if err := v.hypervisor.stopSandbox(false); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,177 @@
|
|||||||
|
From a1bf2e1c696b703935f4b81ca087a60cc2559464 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Mon, 27 Jul 2020 21:45:25 +0800
|
||||||
|
Subject: [PATCH 08/50] kata-runtime: fix kata-runtime resource left problem
|
||||||
|
|
||||||
|
reason: fix the following resource left problem
|
||||||
|
- sandbox stored files and directory are deleted before work container delete
|
||||||
|
- ignore StatusSandbox error, let stopSandbox function call be executed
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
cli/delete.go | 25 +++++++++++++++++++++++--
|
||||||
|
cli/delete_test.go | 15 +++++++++++++++
|
||||||
|
cli/state.go | 6 ++++++
|
||||||
|
virtcontainers/api.go | 4 +++-
|
||||||
|
4 files changed, 47 insertions(+), 3 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/cli/delete.go b/cli/delete.go
|
||||||
|
index 2f5586e5..871ac40d 100644
|
||||||
|
--- a/cli/delete.go
|
||||||
|
+++ b/cli/delete.go
|
||||||
|
@@ -10,6 +10,7 @@ import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
+ "strings"
|
||||||
|
|
||||||
|
"github.com/kata-containers/runtime/pkg/katautils"
|
||||||
|
vc "github.com/kata-containers/runtime/virtcontainers"
|
||||||
|
@@ -68,6 +69,12 @@ func delete(ctx context.Context, containerID string, force bool) error {
|
||||||
|
setExternalLoggers(ctx, kataLog)
|
||||||
|
span.SetTag("container", containerID)
|
||||||
|
|
||||||
|
+ // Remove the containerID to sandboxID mapping
|
||||||
|
+ // no matter what error return
|
||||||
|
+ defer func() {
|
||||||
|
+ _ = katautils.DelContainerIDMapping(ctx, containerID)
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
// Checks the MUST and MUST NOT from OCI runtime specification
|
||||||
|
status, sandboxID, err := getExistingContainerInfo(ctx, containerID)
|
||||||
|
if err != nil {
|
||||||
|
@@ -75,6 +82,15 @@ func delete(ctx context.Context, containerID string, force bool) error {
|
||||||
|
kataLog.Warnf("Failed to get container, force will not fail: %s", err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+ // If err info containers "no such file or directory, because pod_sandbox type
|
||||||
|
+ // container is deleted before pod_container type container, just return nil
|
||||||
|
+ // and let containerd delete container operations continue
|
||||||
|
+ if strings.Contains(err.Error(), "no such file or directory") {
|
||||||
|
+ kataLog.Warnf("pod_sandbox deleted before pod_container: %v", err)
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -123,7 +139,12 @@ func delete(ctx context.Context, containerID string, force bool) error {
|
||||||
|
}
|
||||||
|
case vc.PodContainer:
|
||||||
|
if err := deleteContainer(ctx, sandboxID, containerID, forceStop); err != nil {
|
||||||
|
- return err
|
||||||
|
+ // If err info containers "no such file or directory, because pod_sandbox type
|
||||||
|
+ // container is deleted before pod_container type container, just return nil
|
||||||
|
+ // and let containerd delete container operations continue
|
||||||
|
+ if !strings.Contains(err.Error(), "no such file or directory") {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("Invalid container type found")
|
||||||
|
@@ -134,7 +155,7 @@ func delete(ctx context.Context, containerID string, force bool) error {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
- return katautils.DelContainerIDMapping(ctx, containerID)
|
||||||
|
+ return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func deleteSandbox(ctx context.Context, sandboxID string, force bool) error {
|
||||||
|
diff --git a/cli/delete_test.go b/cli/delete_test.go
|
||||||
|
index a2455dee..ae421cd7 100644
|
||||||
|
--- a/cli/delete_test.go
|
||||||
|
+++ b/cli/delete_test.go
|
||||||
|
@@ -184,6 +184,9 @@ func TestDeleteSandbox(t *testing.T) {
|
||||||
|
assert.Error(err)
|
||||||
|
assert.True(vcmock.IsMockError(err))
|
||||||
|
|
||||||
|
+ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
testingImpl.StatusSandboxFunc = func(ctx context.Context, sandboxID string) (vc.SandboxStatus, error) {
|
||||||
|
return vc.SandboxStatus{
|
||||||
|
ID: sandbox.ID(),
|
||||||
|
@@ -201,6 +204,9 @@ func TestDeleteSandbox(t *testing.T) {
|
||||||
|
assert.Error(err)
|
||||||
|
assert.True(vcmock.IsMockError(err))
|
||||||
|
|
||||||
|
+ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
testingImpl.StopSandboxFunc = func(ctx context.Context, sandboxID string, force bool) (vc.VCSandbox, error) {
|
||||||
|
return sandbox, nil
|
||||||
|
}
|
||||||
|
@@ -213,6 +219,9 @@ func TestDeleteSandbox(t *testing.T) {
|
||||||
|
assert.Error(err)
|
||||||
|
assert.True(vcmock.IsMockError(err))
|
||||||
|
|
||||||
|
+ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
testingImpl.DeleteSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) {
|
||||||
|
return sandbox, nil
|
||||||
|
}
|
||||||
|
@@ -302,6 +311,9 @@ func TestDeleteSandboxRunning(t *testing.T) {
|
||||||
|
assert.Error(err)
|
||||||
|
assert.False(vcmock.IsMockError(err))
|
||||||
|
|
||||||
|
+ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
testingImpl.StatusSandboxFunc = func(ctx context.Context, sandboxID string) (vc.SandboxStatus, error) {
|
||||||
|
return vc.SandboxStatus{
|
||||||
|
ID: sandbox.ID(),
|
||||||
|
@@ -325,6 +337,9 @@ func TestDeleteSandboxRunning(t *testing.T) {
|
||||||
|
assert.Error(err)
|
||||||
|
assert.True(vcmock.IsMockError(err))
|
||||||
|
|
||||||
|
+ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
testingImpl.DeleteSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) {
|
||||||
|
return sandbox, nil
|
||||||
|
}
|
||||||
|
diff --git a/cli/state.go b/cli/state.go
|
||||||
|
index a2fcc12e..de843d34 100644
|
||||||
|
--- a/cli/state.go
|
||||||
|
+++ b/cli/state.go
|
||||||
|
@@ -11,6 +11,7 @@ import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
+ "strings"
|
||||||
|
|
||||||
|
"github.com/kata-containers/runtime/pkg/katautils"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
|
||||||
|
@@ -52,6 +53,11 @@ func state(ctx context.Context, containerID string) error {
|
||||||
|
// Checks the MUST and MUST NOT from OCI runtime specification
|
||||||
|
status, _, err := getExistingContainerInfo(ctx, containerID)
|
||||||
|
if err != nil {
|
||||||
|
+ // If err info containers "no such file or directory, because pod_sandbox type
|
||||||
|
+ // container is deleted before pod_container type container, just return nil
|
||||||
|
+ if strings.Contains(err.Error(), "no such file or directory") {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
|
||||||
|
index fa82d163..449a03e0 100644
|
||||||
|
--- a/virtcontainers/api.go
|
||||||
|
+++ b/virtcontainers/api.go
|
||||||
|
@@ -374,7 +374,9 @@ func StatusSandbox(ctx context.Context, sandboxID string) (SandboxStatus, error)
|
||||||
|
for _, container := range s.containers {
|
||||||
|
contStatus, err := statusContainer(s, container.id)
|
||||||
|
if err != nil {
|
||||||
|
- return SandboxStatus{}, err
|
||||||
|
+ // Since statusContainer may get error because of qemu process D or T,
|
||||||
|
+ // So just ignore this error and let StatusSandbox function return actually SandboxStatus
|
||||||
|
+ s.Logger().Warnf("Status container's status in sandbox get error: %v", contStatus)
|
||||||
|
}
|
||||||
|
|
||||||
|
contStatusList = append(contStatusList, contStatus)
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,132 @@
|
|||||||
|
From 508fd9b94b1b12be3167342b03a47f8f97245e9c Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 28 Jul 2020 10:53:30 +0800
|
||||||
|
Subject: [PATCH 09/50] kata-runtime: add kata-runtime global flag --debug
|
||||||
|
|
||||||
|
reason: add the same debug flag with runc to adapt to
|
||||||
|
containerd 1.2.0
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
cli/kata-env_test.go | 2 +-
|
||||||
|
cli/main.go | 6 +++++-
|
||||||
|
containerd-shim-v2/create.go | 2 +-
|
||||||
|
pkg/katautils/config.go | 4 ++--
|
||||||
|
pkg/katautils/config_test.go | 8 ++++----
|
||||||
|
5 files changed, 13 insertions(+), 9 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/cli/kata-env_test.go b/cli/kata-env_test.go
|
||||||
|
index b31b6cc2..75bb697f 100644
|
||||||
|
--- a/cli/kata-env_test.go
|
||||||
|
+++ b/cli/kata-env_test.go
|
||||||
|
@@ -178,7 +178,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
|
||||||
|
return "", oci.RuntimeConfig{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
- _, config, err = katautils.LoadConfiguration(configFile, true, false)
|
||||||
|
+ _, config, err = katautils.LoadConfiguration(configFile, true, false, false)
|
||||||
|
if err != nil {
|
||||||
|
return "", oci.RuntimeConfig{}, err
|
||||||
|
}
|
||||||
|
diff --git a/cli/main.go b/cli/main.go
|
||||||
|
index df16c5f3..1362a8fb 100644
|
||||||
|
--- a/cli/main.go
|
||||||
|
+++ b/cli/main.go
|
||||||
|
@@ -115,6 +115,10 @@ var runtimeFlags = []cli.Flag{
|
||||||
|
Name: "systemd-cgroup",
|
||||||
|
Usage: "enable systemd cgroup support, expects cgroupsPath to be of form \"slice:prefix:name\" for e.g. \"system.slice:runc:434234\"",
|
||||||
|
},
|
||||||
|
+ cli.BoolFlag{
|
||||||
|
+ Name: "debug",
|
||||||
|
+ Usage: "enable debug output for logging",
|
||||||
|
+ },
|
||||||
|
}
|
||||||
|
|
||||||
|
// runtimeCommands is the list of supported command-line (sub-)
|
||||||
|
@@ -334,7 +338,7 @@ func beforeSubcommands(c *cli.Context) error {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
- configFile, runtimeConfig, err = katautils.LoadConfiguration(c.GlobalString(configFilePathOption), ignoreConfigLogs, false)
|
||||||
|
+ configFile, runtimeConfig, err = katautils.LoadConfiguration(c.GlobalString(configFilePathOption), ignoreConfigLogs, false, c.GlobalBool("debug"))
|
||||||
|
if err != nil {
|
||||||
|
fatal(err)
|
||||||
|
}
|
||||||
|
diff --git a/containerd-shim-v2/create.go b/containerd-shim-v2/create.go
|
||||||
|
index affdbae2..9749073d 100644
|
||||||
|
--- a/containerd-shim-v2/create.go
|
||||||
|
+++ b/containerd-shim-v2/create.go
|
||||||
|
@@ -167,7 +167,7 @@ func loadRuntimeConfig(s *service, r *taskAPI.CreateTaskRequest, anno map[string
|
||||||
|
configPath = os.Getenv("KATA_CONF_FILE")
|
||||||
|
}
|
||||||
|
|
||||||
|
- _, runtimeConfig, err := katautils.LoadConfiguration(configPath, false, true)
|
||||||
|
+ _, runtimeConfig, err := katautils.LoadConfiguration(configPath, false, true, false)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
||||||
|
index 349e667f..448d23ac 100644
|
||||||
|
--- a/pkg/katautils/config.go
|
||||||
|
+++ b/pkg/katautils/config.go
|
||||||
|
@@ -1141,7 +1141,7 @@ func initConfig() (config oci.RuntimeConfig, err error) {
|
||||||
|
//
|
||||||
|
// All paths are resolved fully meaning if this function does not return an
|
||||||
|
// error, all paths are valid at the time of the call.
|
||||||
|
-func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolvedConfigPath string, config oci.RuntimeConfig, err error) {
|
||||||
|
+func LoadConfiguration(configPath string, ignoreLogging, builtIn bool, debugFlag bool) (resolvedConfigPath string, config oci.RuntimeConfig, err error) {
|
||||||
|
|
||||||
|
config, err = initConfig()
|
||||||
|
if err != nil {
|
||||||
|
@@ -1154,7 +1154,7 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved
|
||||||
|
}
|
||||||
|
|
||||||
|
config.Debug = tomlConf.Runtime.Debug
|
||||||
|
- if !tomlConf.Runtime.Debug {
|
||||||
|
+ if !tomlConf.Runtime.Debug && !debugFlag {
|
||||||
|
// If debug is not required, switch back to the original
|
||||||
|
// default log priority, otherwise continue in debug mode.
|
||||||
|
kataUtilsLogger.Logger.Level = originalLoggerLevel
|
||||||
|
diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go
|
||||||
|
index 2eae1f6a..31afcca6 100644
|
||||||
|
--- a/pkg/katautils/config_test.go
|
||||||
|
+++ b/pkg/katautils/config_test.go
|
||||||
|
@@ -264,7 +264,7 @@ func testLoadConfiguration(t *testing.T, dir string,
|
||||||
|
assert.NoError(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
- resolvedConfigPath, config, err := LoadConfiguration(file, ignoreLogging, false)
|
||||||
|
+ resolvedConfigPath, config, err := LoadConfiguration(file, ignoreLogging, false, false)
|
||||||
|
if expectFail {
|
||||||
|
assert.Error(t, err)
|
||||||
|
|
||||||
|
@@ -578,7 +578,7 @@ func TestMinimalRuntimeConfig(t *testing.T) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
- _, config, err := LoadConfiguration(configPath, false, false)
|
||||||
|
+ _, config, err := LoadConfiguration(configPath, false, false, false)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("Expected loadConfiguration to fail as shim path does not exist: %+v", config)
|
||||||
|
}
|
||||||
|
@@ -608,7 +608,7 @@ func TestMinimalRuntimeConfig(t *testing.T) {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
- _, config, err = LoadConfiguration(configPath, false, false)
|
||||||
|
+ _, config, err = LoadConfiguration(configPath, false, false, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
@@ -748,7 +748,7 @@ func TestMinimalRuntimeConfigWithVsock(t *testing.T) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
- _, config, err := LoadConfiguration(configPath, false, false)
|
||||||
|
+ _, config, err := LoadConfiguration(configPath, false, false, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,172 @@
|
|||||||
|
From 76cbca91608e94c1855705ad1a8d06ffa2273115 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 28 Jul 2020 18:18:54 +0800
|
||||||
|
Subject: [PATCH 10/50] kata-runtime: fix kata-shim pid reused problem
|
||||||
|
|
||||||
|
reason: If kata-shim process exit and it's pid reused by other process,
|
||||||
|
it may cause kill other proecss and cause some problem.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/api.go | 2 +-
|
||||||
|
virtcontainers/container.go | 6 +++---
|
||||||
|
virtcontainers/shim.go | 21 +++++++++++++++++----
|
||||||
|
virtcontainers/shim_test.go | 10 +++++-----
|
||||||
|
4 files changed, 26 insertions(+), 13 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
|
||||||
|
index 449a03e0..5e8c9c9e 100644
|
||||||
|
--- a/virtcontainers/api.go
|
||||||
|
+++ b/virtcontainers/api.go
|
||||||
|
@@ -611,7 +611,7 @@ func statusContainer(sandbox *Sandbox, containerID string) (ContainerStatus, err
|
||||||
|
return ContainerStatus{}, fmt.Errorf("sandbox has beed stopped exceptionally")
|
||||||
|
}
|
||||||
|
|
||||||
|
- running, err := isShimRunning(container.process.Pid)
|
||||||
|
+ running, err := isShimRunning(container.process.Pid, containerID)
|
||||||
|
if err != nil {
|
||||||
|
return ContainerStatus{}, err
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/container.go b/virtcontainers/container.go
|
||||||
|
index 9485e708..75f590eb 100644
|
||||||
|
--- a/virtcontainers/container.go
|
||||||
|
+++ b/virtcontainers/container.go
|
||||||
|
@@ -1063,7 +1063,7 @@ func (c *Container) stop(force bool) error {
|
||||||
|
|
||||||
|
// If shim is still running something went wrong
|
||||||
|
// Make sure we stop the shim process
|
||||||
|
- if running, _ := isShimRunning(c.process.Pid); running {
|
||||||
|
+ if running, _ := isShimRunning(c.process.Pid, c.id); running {
|
||||||
|
l := c.Logger()
|
||||||
|
l.Error("Failed to stop container so stopping dangling shim")
|
||||||
|
if err := stopShim(c.process.Pid); err != nil {
|
||||||
|
@@ -1081,7 +1081,7 @@ func (c *Container) stop(force bool) error {
|
||||||
|
// However, if the signal didn't reach its goal, the caller still
|
||||||
|
// expects this container to be stopped, that's why we should not
|
||||||
|
// return an error, but instead try to kill it forcefully.
|
||||||
|
- if err := waitForShim(c.process.Pid); err != nil {
|
||||||
|
+ if err := waitForShim(c.process.Pid, c.id); err != nil {
|
||||||
|
// Force the container to be killed.
|
||||||
|
if err := c.kill(syscall.SIGKILL, true); err != nil && !force {
|
||||||
|
return err
|
||||||
|
@@ -1091,7 +1091,7 @@ func (c *Container) stop(force bool) error {
|
||||||
|
// to succeed. Indeed, we have already given a second chance
|
||||||
|
// to the container by trying to kill it with SIGKILL, there
|
||||||
|
// is no reason to try to go further if we got an error.
|
||||||
|
- if err := waitForShim(c.process.Pid); err != nil && !force {
|
||||||
|
+ if err := waitForShim(c.process.Pid, c.id); err != nil && !force {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/shim.go b/virtcontainers/shim.go
|
||||||
|
index 8ec7458b..6f784a03 100644
|
||||||
|
--- a/virtcontainers/shim.go
|
||||||
|
+++ b/virtcontainers/shim.go
|
||||||
|
@@ -9,11 +9,13 @@ import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
+ "strings"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/types"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||||
|
"github.com/mitchellh/mapstructure"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
)
|
||||||
|
@@ -227,7 +229,7 @@ func startShim(args []string, params ShimParams) (int, error) {
|
||||||
|
return cmd.Process.Pid, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
-func isShimRunning(pid int) (bool, error) {
|
||||||
|
+func isShimRunning(pid int, containerID string) (bool, error) {
|
||||||
|
if pid <= 0 {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
@@ -241,19 +243,30 @@ func isShimRunning(pid int) (bool, error) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
- return true, nil
|
||||||
|
+ cmdline, err := utils.GetProcessCmdline(pid)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return false, nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // If process's cmdline contains kata-shim and containerID keyword, we think this process pid isn't be reused
|
||||||
|
+ if strings.Contains(cmdline, "kata-shim") && strings.Contains(cmdline, containerID) {
|
||||||
|
+ return true, nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ shimLogger().Errorf("%d process isn't a kata-shim process", pid)
|
||||||
|
+ return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitForShim waits for the end of the shim unless it reaches the timeout
|
||||||
|
// first, returning an error in that case.
|
||||||
|
-func waitForShim(pid int) error {
|
||||||
|
+func waitForShim(pid int, containerID string) error {
|
||||||
|
if pid <= 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
tInit := time.Now()
|
||||||
|
for {
|
||||||
|
- running, err := isShimRunning(pid)
|
||||||
|
+ running, err := isShimRunning(pid, containerID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/shim_test.go b/virtcontainers/shim_test.go
|
||||||
|
index e9bd027c..62471311 100644
|
||||||
|
--- a/virtcontainers/shim_test.go
|
||||||
|
+++ b/virtcontainers/shim_test.go
|
||||||
|
@@ -190,7 +190,7 @@ func TestStopShimSuccessfulProcessRunning(t *testing.T) {
|
||||||
|
|
||||||
|
func testIsShimRunning(t *testing.T, pid int, expected bool) {
|
||||||
|
assert := assert.New(t)
|
||||||
|
- running, err := isShimRunning(pid)
|
||||||
|
+ running, err := isShimRunning(pid, containerID)
|
||||||
|
assert.NoError(err)
|
||||||
|
assert.Equal(running, expected)
|
||||||
|
}
|
||||||
|
@@ -205,7 +205,7 @@ func TestIsShimRunningTrue(t *testing.T) {
|
||||||
|
cmd := testRunSleep999AndGetCmd(t)
|
||||||
|
assert := assert.New(t)
|
||||||
|
|
||||||
|
- testIsShimRunning(t, cmd.Process.Pid, true)
|
||||||
|
+ testIsShimRunning(t, cmd.Process.Pid, false)
|
||||||
|
|
||||||
|
err := syscall.Kill(cmd.Process.Pid, syscall.SIGKILL)
|
||||||
|
assert.NoError(err)
|
||||||
|
@@ -216,7 +216,7 @@ func TestWaitForShimInvalidPidSuccessful(t *testing.T) {
|
||||||
|
assert := assert.New(t)
|
||||||
|
|
||||||
|
for _, val := range wrongValuesList {
|
||||||
|
- err := waitForShim(val)
|
||||||
|
+ err := waitForShim(val, containerID)
|
||||||
|
assert.NoError(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
@@ -224,7 +224,7 @@ func TestWaitForShimInvalidPidSuccessful(t *testing.T) {
|
||||||
|
func TestWaitForShimNotRunningSuccessful(t *testing.T) {
|
||||||
|
pid := testRunSleep0AndGetPid(t)
|
||||||
|
assert := assert.New(t)
|
||||||
|
- assert.NoError(waitForShim(pid))
|
||||||
|
+ assert.NoError(waitForShim(pid, containerID))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWaitForShimRunningForTooLongFailure(t *testing.T) {
|
||||||
|
@@ -232,6 +232,6 @@ func TestWaitForShimRunningForTooLongFailure(t *testing.T) {
|
||||||
|
assert := assert.New(t)
|
||||||
|
|
||||||
|
waitForShimTimeout = 0.1
|
||||||
|
- assert.Error(waitForShim(cmd.Process.Pid))
|
||||||
|
+ assert.NoError(waitForShim(cmd.Process.Pid, containerID))
|
||||||
|
assert.NoError(syscall.Kill(cmd.Process.Pid, syscall.SIGKILL))
|
||||||
|
}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,119 @@
|
|||||||
|
From 0aeff2632eac58eefdc8ae438891303332831ec5 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 28 Jul 2020 20:48:24 +0800
|
||||||
|
Subject: [PATCH 11/50] kata-runtime: check the process info before send
|
||||||
|
SIGKILL
|
||||||
|
|
||||||
|
reason: In order to avoid the pid reuse problem, check the
|
||||||
|
process info before send SIGKILL signal to process.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/kata_proxy.go | 18 ++++++++++++++++++
|
||||||
|
virtcontainers/qemu.go | 5 +++++
|
||||||
|
virtcontainers/shim.go | 9 +++++++++
|
||||||
|
virtcontainers/shim_test.go | 8 ++++----
|
||||||
|
4 files changed, 36 insertions(+), 4 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/kata_proxy.go b/virtcontainers/kata_proxy.go
|
||||||
|
index e04b4cff..ed272bad 100644
|
||||||
|
--- a/virtcontainers/kata_proxy.go
|
||||||
|
+++ b/virtcontainers/kata_proxy.go
|
||||||
|
@@ -6,8 +6,12 @@
|
||||||
|
package virtcontainers
|
||||||
|
|
||||||
|
import (
|
||||||
|
+ "fmt"
|
||||||
|
"os/exec"
|
||||||
|
+ "strings"
|
||||||
|
"syscall"
|
||||||
|
+
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||||
|
)
|
||||||
|
|
||||||
|
// This is the Kata Containers implementation of the proxy interface.
|
||||||
|
@@ -61,6 +65,20 @@ func (p *kataProxy) start(params proxyParams) (int, string, error) {
|
||||||
|
|
||||||
|
// stop is kataProxy stop implementation for proxy interface.
|
||||||
|
func (p *kataProxy) stop(pid int) error {
|
||||||
|
+ if pid <= 1 {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // check process info before send SIGKILL signal
|
||||||
|
+ cmdline, err := utils.GetProcessCmdline(pid)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return fmt.Errorf("get kata-proxy %d cmdline error: %v", pid, err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if !strings.Contains(cmdline, KataProxyProcessName) {
|
||||||
|
+ return fmt.Errorf("%d is not kata-proxy process, don't kill wrong process", pid)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
// Signal the proxy with SIGTERM.
|
||||||
|
return syscall.Kill(pid, syscall.SIGTERM)
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
|
||||||
|
index 4b15d968..4789101d 100644
|
||||||
|
--- a/virtcontainers/qemu.go
|
||||||
|
+++ b/virtcontainers/qemu.go
|
||||||
|
@@ -967,6 +967,11 @@ func (q *qemu) stopSandbox(force bool) error {
|
||||||
|
return fmt.Errorf("force kill qemu process pid is invalid")
|
||||||
|
}
|
||||||
|
|
||||||
|
+ cmdline, _ := utils.GetProcessCmdline(qemuMainPid)
|
||||||
|
+ if !strings.Contains(cmdline, string(QemuHypervisor)) {
|
||||||
|
+ return fmt.Errorf("force kill %d process is not qemu process, don't kill wrong process", qemuMainPid)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
_ = syscall.Kill(qemuMainPid, syscall.SIGKILL)
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/shim.go b/virtcontainers/shim.go
|
||||||
|
index 6f784a03..b192b258 100644
|
||||||
|
--- a/virtcontainers/shim.go
|
||||||
|
+++ b/virtcontainers/shim.go
|
||||||
|
@@ -143,6 +143,15 @@ func stopShim(pid int) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+ cmdline, err := utils.GetProcessCmdline(pid)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if !strings.Contains(cmdline, "kata-shim") {
|
||||||
|
+ return fmt.Errorf("%d process is not kata-shim process, don't kill wrong process", pid)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
if err := signalShim(pid, syscall.SIGKILL); err != nil && err != syscall.ESRCH {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/shim_test.go b/virtcontainers/shim_test.go
|
||||||
|
index 62471311..dc15eab0 100644
|
||||||
|
--- a/virtcontainers/shim_test.go
|
||||||
|
+++ b/virtcontainers/shim_test.go
|
||||||
|
@@ -176,16 +176,16 @@ func testRunSleep999AndGetCmd(t *testing.T) *exec.Cmd {
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
-func TestStopShimSuccessfulProcessNotRunning(t *testing.T) {
|
||||||
|
+func TestStopShimFailProcessNotRunning(t *testing.T) {
|
||||||
|
assert := assert.New(t)
|
||||||
|
pid := testRunSleep0AndGetPid(t)
|
||||||
|
- assert.NoError(stopShim(pid))
|
||||||
|
+ assert.Error(stopShim(pid))
|
||||||
|
}
|
||||||
|
|
||||||
|
-func TestStopShimSuccessfulProcessRunning(t *testing.T) {
|
||||||
|
+func TestStopShimFailProcessRunning(t *testing.T) {
|
||||||
|
assert := assert.New(t)
|
||||||
|
cmd := testRunSleep999AndGetCmd(t)
|
||||||
|
- assert.NoError(stopShim(cmd.Process.Pid))
|
||||||
|
+ assert.Error(stopShim(cmd.Process.Pid))
|
||||||
|
}
|
||||||
|
|
||||||
|
func testIsShimRunning(t *testing.T, pid int, expected bool) {
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,37 @@
|
|||||||
|
From 441d80f55f4dc5efb4c92d91608a3c8db3d087cb Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 28 Jul 2020 21:43:15 +0800
|
||||||
|
Subject: [PATCH 12/50] kata-runtime: truncate the log.json file before
|
||||||
|
kata-runtime subcommand executed
|
||||||
|
|
||||||
|
reason: since we have redirect the kata-runtime log to /var/log/messages, and avoid the
|
||||||
|
path of log.json file to be large in the tmpfs, so we truncate the log.json file every
|
||||||
|
time before subcommand is executed.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
cli/main.go | 8 ++++++++
|
||||||
|
1 file changed, 8 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/cli/main.go b/cli/main.go
|
||||||
|
index 1362a8fb..5eb2fb19 100644
|
||||||
|
--- a/cli/main.go
|
||||||
|
+++ b/cli/main.go
|
||||||
|
@@ -300,6 +300,14 @@ func beforeSubcommands(c *cli.Context) error {
|
||||||
|
ignoreConfigLogs = true
|
||||||
|
} else {
|
||||||
|
if path := c.GlobalString("log"); path != "" {
|
||||||
|
+ // since we have redirect the kata-runtime log to /var/log/messages, and avoid the
|
||||||
|
+ // path of log.json file to be large in the tmpfs, so we truncate the log.json file
|
||||||
|
+ // every time before subcommand is executed.
|
||||||
|
+ if path != "/dev/null" && katautils.FileExists(path) {
|
||||||
|
+ if err := os.Truncate(path, 0); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND|os.O_SYNC, 0640)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,98 @@
|
|||||||
|
From fd63d26a5b0542f35d61b0c19c80795f052b4518 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 28 Jul 2020 22:05:44 +0800
|
||||||
|
Subject: [PATCH 13/50] kata-runtime: get container info by containerID prefix
|
||||||
|
|
||||||
|
reason: get container info by containerID prefix, so we just
|
||||||
|
need to input the prefix containerID when call kata-runtime
|
||||||
|
subcommand
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
cli/oci.go | 35 +++++++++++++++++++++++++++++++++++
|
||||||
|
pkg/katautils/oci.go | 5 +++++
|
||||||
|
2 files changed, 40 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/cli/oci.go b/cli/oci.go
|
||||||
|
index 8ffac2df..bf962d03 100644
|
||||||
|
--- a/cli/oci.go
|
||||||
|
+++ b/cli/oci.go
|
||||||
|
@@ -9,6 +9,7 @@ import (
|
||||||
|
"bufio"
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
+ "io/ioutil"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
@@ -25,6 +26,8 @@ const (
|
||||||
|
// Filesystem type corresponding to CGROUP_SUPER_MAGIC as listed
|
||||||
|
// here: http://man7.org/linux/man-pages/man2/statfs.2.html
|
||||||
|
cgroupFsType = 0x27e0eb
|
||||||
|
+
|
||||||
|
+ maxIDLength = 64
|
||||||
|
)
|
||||||
|
|
||||||
|
var cgroupsDirPath string
|
||||||
|
@@ -38,6 +41,14 @@ func getContainerInfo(ctx context.Context, containerID string) (vc.ContainerStat
|
||||||
|
return vc.ContainerStatus{}, "", fmt.Errorf("Missing container ID")
|
||||||
|
}
|
||||||
|
|
||||||
|
+ if len(containerID) < maxIDLength {
|
||||||
|
+ fullContainerID, err := getContainerIDbyPrefix(containerID)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return vc.ContainerStatus{}, "", err
|
||||||
|
+ }
|
||||||
|
+ containerID = fullContainerID
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
sandboxID, err := katautils.FetchContainerIDMapping(containerID)
|
||||||
|
if err != nil {
|
||||||
|
return vc.ContainerStatus{}, "", err
|
||||||
|
@@ -211,3 +222,27 @@ func getCgroupsDirPath(mountInfoFile string) (string, error) {
|
||||||
|
|
||||||
|
return cgroupRootPath, nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func getContainerIDbyPrefix(prefix string) (string, error) {
|
||||||
|
+ files, err := ioutil.ReadDir(katautils.GetCtrsMapTreePath())
|
||||||
|
+ if err != nil {
|
||||||
|
+ return "", err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ containers := []string{}
|
||||||
|
+ for _, file := range files {
|
||||||
|
+ if file.IsDir() && strings.HasPrefix(file.Name(), prefix) {
|
||||||
|
+ containers = append(containers, file.Name())
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if len(containers) == 0 {
|
||||||
|
+ return "", fmt.Errorf("no such container ID (%v)", prefix)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if len(containers) > 1 {
|
||||||
|
+ return "", fmt.Errorf("multiple containers found (%v)", prefix)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return containers[0], nil
|
||||||
|
+}
|
||||||
|
diff --git a/pkg/katautils/oci.go b/pkg/katautils/oci.go
|
||||||
|
index 6de8101e..1334af35 100644
|
||||||
|
--- a/pkg/katautils/oci.go
|
||||||
|
+++ b/pkg/katautils/oci.go
|
||||||
|
@@ -25,6 +25,11 @@ func SetCtrsMapTreePath(path string) {
|
||||||
|
ctrsMapTreePath = path
|
||||||
|
}
|
||||||
|
|
||||||
|
+// GetCtrsMapTreePath return the containerID to SandboxID mapping dir
|
||||||
|
+func GetCtrsMapTreePath() string {
|
||||||
|
+ return ctrsMapTreePath
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// doUpdatePath returns whether a ctrsMapTreePath needs to be updated with a rootless prefix
|
||||||
|
func doUpdatePath() bool {
|
||||||
|
return rootless.IsRootless() && !strings.HasPrefix(ctrsMapTreePath, rootless.GetRootlessDir())
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,76 @@
|
|||||||
|
From 1bd3cb85a1cf0e94b3280412d5fb47cecc4721fd Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Wed, 29 Jul 2020 10:42:49 +0800
|
||||||
|
Subject: [PATCH 14/50] kata-runtime: add self defined annotations framework
|
||||||
|
|
||||||
|
reason: add self defined annotations framework to pass some
|
||||||
|
self defined resource for sandbox
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/pkg/oci/utils.go | 38 ++++++++++++++++++++++++++++++++++++++
|
||||||
|
1 file changed, 38 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
||||||
|
index cd8d48ce..05181efd 100644
|
||||||
|
--- a/virtcontainers/pkg/oci/utils.go
|
||||||
|
+++ b/virtcontainers/pkg/oci/utils.go
|
||||||
|
@@ -33,6 +33,10 @@ type annotationContainerType struct {
|
||||||
|
containerType vc.ContainerType
|
||||||
|
}
|
||||||
|
|
||||||
|
+type annotationHandler func(value string) error
|
||||||
|
+
|
||||||
|
+var annotationHandlerList = map[string]annotationHandler{}
|
||||||
|
+
|
||||||
|
var (
|
||||||
|
// ErrNoLinux is an error for missing Linux sections in the OCI configuration file.
|
||||||
|
ErrNoLinux = errors.New("missing Linux section")
|
||||||
|
@@ -342,6 +346,10 @@ func addAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) error {
|
||||||
|
if err := addAgentConfigOverrides(ocispec, config); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+ if err := addOtherSandboxAnnotation(ocispec, config); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -1016,3 +1024,33 @@ func GetOCIConfig(status vc.ContainerStatus) (specs.Spec, error) {
|
||||||
|
|
||||||
|
return *status.Spec, nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+// validateOtherSandboxAnnotations validate the value of annotation
|
||||||
|
+func validateOtherSandboxAnnotations(annotation, value string) error {
|
||||||
|
+ validateHandler, ok := annotationHandlerList[annotation]
|
||||||
|
+ if !ok {
|
||||||
|
+ return fmt.Errorf("unsupport Sandbox annotation type")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return validateHandler(value)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// addOtherSandboxAnnotation add self defined annotation for sandbox
|
||||||
|
+func addOtherSandboxAnnotation(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error {
|
||||||
|
+ otherSandboxAnnotationsKey := []string{}
|
||||||
|
+
|
||||||
|
+ for _, a := range otherSandboxAnnotationsKey {
|
||||||
|
+ value, ok := ocispec.Annotations[a]
|
||||||
|
+ if !ok {
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := validateOtherSandboxAnnotations(a, value); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ sbConfig.Annotations[a] = value
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,143 @@
|
|||||||
|
From c0a33c4584e1fbd9b39ff1ca3ed632efe85de65e Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Sun, 2 Aug 2020 15:51:10 +0800
|
||||||
|
Subject: [PATCH 15/50] kata-runtime: add reuse hypervisor cpu and memory
|
||||||
|
feature
|
||||||
|
|
||||||
|
reason: If default hypervisor cpu and memory is set too large,
|
||||||
|
which may waste resource, so we add enable_reuse_cpu_memory
|
||||||
|
config in the configuration.toml file to choose share the
|
||||||
|
hypervisor cpu and memory with container or not.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
cli/config/configuration-qemu.toml.in | 5 +++++
|
||||||
|
pkg/katautils/config.go | 2 ++
|
||||||
|
virtcontainers/hypervisor.go | 3 +++
|
||||||
|
virtcontainers/persist.go | 2 ++
|
||||||
|
virtcontainers/persist/api/config.go | 3 +++
|
||||||
|
virtcontainers/sandbox.go | 22 ++++++++++++++++++----
|
||||||
|
6 files changed, 33 insertions(+), 4 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in
|
||||||
|
index 46ce7d9b..82461732 100644
|
||||||
|
--- a/cli/config/configuration-qemu.toml.in
|
||||||
|
+++ b/cli/config/configuration-qemu.toml.in
|
||||||
|
@@ -95,6 +95,11 @@ default_memory = @DEFMEMSZ@
|
||||||
|
# Default false
|
||||||
|
#enable_virtio_mem = true
|
||||||
|
|
||||||
|
+# Specifies share hypervisor default cpu and memory resource
|
||||||
|
+# between hypervisor and container process.
|
||||||
|
+# Default false
|
||||||
|
+#enable_reuse_cpu_memory = false
|
||||||
|
+
|
||||||
|
# Disable block device from being used for a container's rootfs.
|
||||||
|
# In case of a storage driver like devicemapper where a container's
|
||||||
|
# root file system is backed by a block device, the block device is passed
|
||||||
|
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
||||||
|
index 448d23ac..d1883c94 100644
|
||||||
|
--- a/pkg/katautils/config.go
|
||||||
|
+++ b/pkg/katautils/config.go
|
||||||
|
@@ -112,6 +112,7 @@ type hypervisor struct {
|
||||||
|
MemorySize uint32 `toml:"default_memory"`
|
||||||
|
MemSlots uint32 `toml:"memory_slots"`
|
||||||
|
MemOffset uint32 `toml:"memory_offset"`
|
||||||
|
+ EnableCPUMemoryReuse bool `toml:"enable_reuse_cpu_memory"`
|
||||||
|
DefaultBridges uint32 `toml:"default_bridges"`
|
||||||
|
Msize9p uint32 `toml:"msize_9p"`
|
||||||
|
PCIeRootPort uint32 `toml:"pcie_root_port"`
|
||||||
|
@@ -640,6 +641,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
||||||
|
MemorySize: h.defaultMemSz(),
|
||||||
|
MemSlots: h.defaultMemSlots(),
|
||||||
|
MemOffset: h.defaultMemOffset(),
|
||||||
|
+ EnableCPUMemoryReuse: h.EnableCPUMemoryReuse,
|
||||||
|
VirtioMem: h.VirtioMem,
|
||||||
|
EntropySource: h.GetEntropySource(),
|
||||||
|
DefaultBridges: h.defaultBridges(),
|
||||||
|
diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go
|
||||||
|
index fd7d1f8e..5f8d24f9 100644
|
||||||
|
--- a/virtcontainers/hypervisor.go
|
||||||
|
+++ b/virtcontainers/hypervisor.go
|
||||||
|
@@ -250,6 +250,9 @@ type HypervisorConfig struct {
|
||||||
|
// MemOffset specifies memory space for nvdimm device
|
||||||
|
MemOffset uint32
|
||||||
|
|
||||||
|
+ // Enable hypervisor cpu and memory resource share with container
|
||||||
|
+ EnableCPUMemoryReuse bool
|
||||||
|
+
|
||||||
|
// VirtioFSCacheSize is the DAX cache size in MiB
|
||||||
|
VirtioFSCacheSize uint32
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go
|
||||||
|
index d96a3890..aef4d18d 100644
|
||||||
|
--- a/virtcontainers/persist.go
|
||||||
|
+++ b/virtcontainers/persist.go
|
||||||
|
@@ -214,6 +214,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
|
||||||
|
Msize9p: sconfig.HypervisorConfig.Msize9p,
|
||||||
|
MemSlots: sconfig.HypervisorConfig.MemSlots,
|
||||||
|
MemOffset: sconfig.HypervisorConfig.MemOffset,
|
||||||
|
+ EnableCPUMemoryReuse: sconfig.HypervisorConfig.EnableCPUMemoryReuse,
|
||||||
|
VirtioMem: sconfig.HypervisorConfig.VirtioMem,
|
||||||
|
VirtioFSCacheSize: sconfig.HypervisorConfig.VirtioFSCacheSize,
|
||||||
|
KernelPath: sconfig.HypervisorConfig.KernelPath,
|
||||||
|
@@ -503,6 +504,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
|
||||||
|
Msize9p: hconf.Msize9p,
|
||||||
|
MemSlots: hconf.MemSlots,
|
||||||
|
MemOffset: hconf.MemOffset,
|
||||||
|
+ EnableCPUMemoryReuse: hconf.EnableCPUMemoryReuse,
|
||||||
|
VirtioMem: hconf.VirtioMem,
|
||||||
|
VirtioFSCacheSize: hconf.VirtioFSCacheSize,
|
||||||
|
KernelPath: hconf.KernelPath,
|
||||||
|
diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go
|
||||||
|
index 34a5fd0f..a3c6ec91 100644
|
||||||
|
--- a/virtcontainers/persist/api/config.go
|
||||||
|
+++ b/virtcontainers/persist/api/config.go
|
||||||
|
@@ -35,6 +35,9 @@ type HypervisorConfig struct {
|
||||||
|
// MemOffset specifies memory space for nvdimm device
|
||||||
|
MemOffset uint32
|
||||||
|
|
||||||
|
+ // Enable hypervisor cpu and memory resource share with container
|
||||||
|
+ EnableCPUMemoryReuse bool
|
||||||
|
+
|
||||||
|
// VirtioFSCacheSize is the DAX cache size in MiB
|
||||||
|
VirtioFSCacheSize uint32
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index 78188ed7..e766d1f7 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -1833,12 +1833,26 @@ func (s *Sandbox) updateResources() error {
|
||||||
|
}
|
||||||
|
|
||||||
|
sandboxVCPUs := s.calculateSandboxCPUs()
|
||||||
|
- // Add default vcpus for sandbox
|
||||||
|
- sandboxVCPUs += s.hypervisor.hypervisorConfig().NumVCPUs
|
||||||
|
+ // Share the hypervisor vcpu with container process
|
||||||
|
+ if s.config.HypervisorConfig.EnableCPUMemoryReuse {
|
||||||
|
+ if sandboxVCPUs <= s.config.HypervisorConfig.NumVCPUs {
|
||||||
|
+ sandboxVCPUs = s.config.HypervisorConfig.NumVCPUs
|
||||||
|
+ }
|
||||||
|
+ } else {
|
||||||
|
+ // Add default vcpus for sandbox
|
||||||
|
+ sandboxVCPUs += s.hypervisor.hypervisorConfig().NumVCPUs
|
||||||
|
+ }
|
||||||
|
|
||||||
|
sandboxMemoryByte := s.calculateSandboxMemory()
|
||||||
|
- // Add default / rsvd memory for sandbox.
|
||||||
|
- sandboxMemoryByte += int64(s.hypervisor.hypervisorConfig().MemorySize) << utils.MibToBytesShift
|
||||||
|
+ // Share the hypervisor memory with container process
|
||||||
|
+ if s.config.HypervisorConfig.EnableCPUMemoryReuse {
|
||||||
|
+ if sandboxMemoryByte <= (int64(s.config.HypervisorConfig.MemorySize) << utils.MibToBytesShift) {
|
||||||
|
+ sandboxMemoryByte = int64(s.config.HypervisorConfig.MemorySize) << utils.MibToBytesShift
|
||||||
|
+ }
|
||||||
|
+ } else {
|
||||||
|
+ // Add default / rsvd memory for sandbox.
|
||||||
|
+ sandboxMemoryByte += int64(s.hypervisor.hypervisorConfig().MemorySize) << utils.MibToBytesShift
|
||||||
|
+ }
|
||||||
|
|
||||||
|
// Update VCPUs
|
||||||
|
s.Logger().WithField("cpus-sandbox", sandboxVCPUs).Debugf("Request to hypervisor to update vCPUs")
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,288 @@
|
|||||||
|
From dc9de8bb181e2cec2f3e0a76d02833fef45b46af Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Thu, 6 Aug 2020 09:28:34 -0400
|
||||||
|
Subject: [PATCH 16/50] virtcontainers: fix hotplug huge size memory cause
|
||||||
|
agent hang bug
|
||||||
|
|
||||||
|
fixes: #2872
|
||||||
|
|
||||||
|
reason: If hotplug huge size memory into kata VM at once time,
|
||||||
|
guest kernel will allocate some extra memory for memory management,
|
||||||
|
which may cause kata-agent hang and out of responding.
|
||||||
|
And hotplug more memory into VM, more extra memory is needed.
|
||||||
|
|
||||||
|
Inorder to solve this problem, we divide hotplug huge memory into
|
||||||
|
two steps. First, hotplug the max allowed memory into VM and wait
|
||||||
|
all first step hotplugged memory online. Second Step, hotplug the
|
||||||
|
left memory into VM.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/acrn.go | 4 ++++
|
||||||
|
virtcontainers/agent.go | 3 ++-
|
||||||
|
virtcontainers/clh.go | 4 ++++
|
||||||
|
virtcontainers/fc.go | 4 ++++
|
||||||
|
virtcontainers/hypervisor.go | 3 +++
|
||||||
|
virtcontainers/kata_agent.go | 4 ++--
|
||||||
|
virtcontainers/kata_agent_test.go | 2 +-
|
||||||
|
virtcontainers/mock_hypervisor.go | 4 ++++
|
||||||
|
virtcontainers/noop_agent.go | 2 +-
|
||||||
|
virtcontainers/qemu.go | 4 ++++
|
||||||
|
virtcontainers/sandbox.go | 30 ++++++++++++++++++++++++++++--
|
||||||
|
virtcontainers/sandbox_test.go | 12 ++++++++++++
|
||||||
|
virtcontainers/utils/utils.go | 3 +++
|
||||||
|
virtcontainers/vm.go | 2 +-
|
||||||
|
14 files changed, 73 insertions(+), 8 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/acrn.go b/virtcontainers/acrn.go
|
||||||
|
index 10cae06f..c9a0fe0b 100644
|
||||||
|
--- a/virtcontainers/acrn.go
|
||||||
|
+++ b/virtcontainers/acrn.go
|
||||||
|
@@ -811,3 +811,7 @@ func (a *Acrn) loadInfo() error {
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func (a *Acrn) getMemorySize() uint32 {
|
||||||
|
+ return a.config.MemorySize
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go
|
||||||
|
index be9526c7..b1dea816 100644
|
||||||
|
--- a/virtcontainers/agent.go
|
||||||
|
+++ b/virtcontainers/agent.go
|
||||||
|
@@ -201,7 +201,8 @@ type agent interface {
|
||||||
|
// This function should be called after hot adding vCPUs or Memory.
|
||||||
|
// cpus specifies the number of CPUs that were added and the agent should online
|
||||||
|
// cpuOnly specifies that we should online cpu or online memory or both
|
||||||
|
- onlineCPUMem(cpus uint32, cpuOnly bool) error
|
||||||
|
+ // wait specifies that we should wait all cpu or memory online in the VM synchronously
|
||||||
|
+ onlineCPUMem(cpus uint32, cpuOnly bool, wait bool) error
|
||||||
|
|
||||||
|
// memHotplugByProbe will notify the guest kernel about memory hotplug event through
|
||||||
|
// probe interface.
|
||||||
|
diff --git a/virtcontainers/clh.go b/virtcontainers/clh.go
|
||||||
|
index 59510b02..8afcd4bf 100644
|
||||||
|
--- a/virtcontainers/clh.go
|
||||||
|
+++ b/virtcontainers/clh.go
|
||||||
|
@@ -1210,3 +1210,7 @@ func (clh *cloudHypervisor) vmInfo() (chclient.VmInfo, error) {
|
||||||
|
return info, openAPIClientError(err)
|
||||||
|
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func (clh *cloudHypervisor) getMemorySize() uint32 {
|
||||||
|
+ return clh.config.MemorySize
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go
|
||||||
|
index 72a8e192..15726156 100644
|
||||||
|
--- a/virtcontainers/fc.go
|
||||||
|
+++ b/virtcontainers/fc.go
|
||||||
|
@@ -1212,3 +1212,7 @@ func (fc *firecracker) watchConsole() (*os.File, error) {
|
||||||
|
|
||||||
|
return stdio, nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func (fc *firecracker) getMemorySize() uint32 {
|
||||||
|
+ return fc.config.MemorySize
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go
|
||||||
|
index 5f8d24f9..9cd685ad 100644
|
||||||
|
--- a/virtcontainers/hypervisor.go
|
||||||
|
+++ b/virtcontainers/hypervisor.go
|
||||||
|
@@ -778,6 +778,9 @@ type hypervisor interface {
|
||||||
|
hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error)
|
||||||
|
resizeMemory(memMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error)
|
||||||
|
resizeVCPUs(vcpus uint32) (uint32, uint32, error)
|
||||||
|
+ // getMemorySize return the total memory in the guest include default memory size + hot plugged memory
|
||||||
|
+ // return memory size unit is MB
|
||||||
|
+ getMemorySize() uint32
|
||||||
|
getSandboxConsole(sandboxID string) (string, error)
|
||||||
|
disconnect()
|
||||||
|
capabilities() types.Capabilities
|
||||||
|
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
|
||||||
|
index 7575d326..8e073339 100644
|
||||||
|
--- a/virtcontainers/kata_agent.go
|
||||||
|
+++ b/virtcontainers/kata_agent.go
|
||||||
|
@@ -1806,9 +1806,9 @@ func (k *kataAgent) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionS
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
-func (k *kataAgent) onlineCPUMem(cpus uint32, cpuOnly bool) error {
|
||||||
|
+func (k *kataAgent) onlineCPUMem(cpus uint32, cpuOnly bool, wait bool) error {
|
||||||
|
req := &grpc.OnlineCPUMemRequest{
|
||||||
|
- Wait: false,
|
||||||
|
+ Wait: wait,
|
||||||
|
NbCpus: cpus,
|
||||||
|
CpuOnly: cpuOnly,
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go
|
||||||
|
index 62d31c93..2a2ddada 100644
|
||||||
|
--- a/virtcontainers/kata_agent_test.go
|
||||||
|
+++ b/virtcontainers/kata_agent_test.go
|
||||||
|
@@ -324,7 +324,7 @@ func TestKataAgentSendReq(t *testing.T) {
|
||||||
|
err = k.resumeContainer(sandbox, Container{})
|
||||||
|
assert.Nil(err)
|
||||||
|
|
||||||
|
- err = k.onlineCPUMem(1, true)
|
||||||
|
+ err = k.onlineCPUMem(1, true, false)
|
||||||
|
assert.Nil(err)
|
||||||
|
|
||||||
|
_, err = k.statsContainer(sandbox, Container{})
|
||||||
|
diff --git a/virtcontainers/mock_hypervisor.go b/virtcontainers/mock_hypervisor.go
|
||||||
|
index a5b67491..f1c6106d 100644
|
||||||
|
--- a/virtcontainers/mock_hypervisor.go
|
||||||
|
+++ b/virtcontainers/mock_hypervisor.go
|
||||||
|
@@ -128,3 +128,7 @@ func (m *mockHypervisor) check() error {
|
||||||
|
func (m *mockHypervisor) generateSocket(id string, useVsock bool) (interface{}, error) {
|
||||||
|
return types.Socket{HostPath: "/tmp/socket", Name: "socket"}, nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func (m *mockHypervisor) getMemorySize() uint32 {
|
||||||
|
+ return 0
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/noop_agent.go b/virtcontainers/noop_agent.go
|
||||||
|
index 8a7cd337..6e211bca 100644
|
||||||
|
--- a/virtcontainers/noop_agent.go
|
||||||
|
+++ b/virtcontainers/noop_agent.go
|
||||||
|
@@ -102,7 +102,7 @@ func (n *noopAgent) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionS
|
||||||
|
}
|
||||||
|
|
||||||
|
// onlineCPUMem is the Noop agent Container online CPU and Memory implementation. It does nothing.
|
||||||
|
-func (n *noopAgent) onlineCPUMem(cpus uint32, cpuOnly bool) error {
|
||||||
|
+func (n *noopAgent) onlineCPUMem(cpus uint32, cpuOnly bool, wait bool) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
|
||||||
|
index 4789101d..7bae3278 100644
|
||||||
|
--- a/virtcontainers/qemu.go
|
||||||
|
+++ b/virtcontainers/qemu.go
|
||||||
|
@@ -2273,3 +2273,7 @@ func (q *qemu) check() error {
|
||||||
|
func (q *qemu) generateSocket(id string, useVsock bool) (interface{}, error) {
|
||||||
|
return generateVMSocket(id, useVsock, q.store.RunVMStoragePath())
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func (q *qemu) getMemorySize() uint32 {
|
||||||
|
+ return q.config.MemorySize + uint32(q.state.HotpluggedMemory)
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index e766d1f7..a318d677 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -1864,7 +1864,7 @@ func (s *Sandbox) updateResources() error {
|
||||||
|
// If the CPUs were increased, ask agent to online them
|
||||||
|
if oldCPUs < newCPUs {
|
||||||
|
vcpusAdded := newCPUs - oldCPUs
|
||||||
|
- if err := s.agent.onlineCPUMem(vcpusAdded, true); err != nil {
|
||||||
|
+ if err := s.agent.onlineCPUMem(vcpusAdded, true, false); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
@@ -1872,6 +1872,20 @@ func (s *Sandbox) updateResources() error {
|
||||||
|
|
||||||
|
// Update Memory
|
||||||
|
s.Logger().WithField("memory-sandbox-size-byte", sandboxMemoryByte).Debugf("Request to hypervisor to update memory")
|
||||||
|
+ reqMemMB := uint32(sandboxMemoryByte >> utils.MibToBytesShift)
|
||||||
|
+ currentMemMB := s.hypervisor.getMemorySize()
|
||||||
|
+
|
||||||
|
+ // If request hotplug memory size larger than utils.MaxHotplugMemMBOnceTime,
|
||||||
|
+ // inorder to avoid hotplug memory oom problem, we need to hotplug large memory
|
||||||
|
+ // with two steps. First, hotplug utils.MaxHotplugMemMBOnceTime size memory into
|
||||||
|
+ // guest and wait all hotplug memory online. Then, hotplug the left unplugged memory
|
||||||
|
+ // into the guest
|
||||||
|
+ if currentMemMB < reqMemMB && (reqMemMB-currentMemMB) > utils.MaxHotplugMemMBOnceTime {
|
||||||
|
+ if err := s.beforeHotplugHugeMem(currentMemMB); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
newMemory, updatedMemoryDevice, err := s.hypervisor.resizeMemory(uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
@@ -1884,7 +1898,7 @@ func (s *Sandbox) updateResources() error {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
- if err := s.agent.onlineCPUMem(0, false); err != nil {
|
||||||
|
+ if err := s.agent.onlineCPUMem(0, false, false); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
@@ -1926,6 +1940,18 @@ func (s *Sandbox) calculateSandboxCPUs() uint32 {
|
||||||
|
return utils.CalculateVCpusFromMilliCpus(mCPU)
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (s *Sandbox) beforeHotplugHugeMem(currentMemSizeInMB uint32) error {
|
||||||
|
+ wantedTotalMemSize := currentMemSizeInMB + utils.MaxHotplugMemMBOnceTime
|
||||||
|
+ newMemory, _, err := s.hypervisor.resizeMemory(wantedTotalMemSize, s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ s.Logger().Debugf("first part hotplug memory size: %d MB", newMemory)
|
||||||
|
+ // wait all first part hotplugged memory online in the guest
|
||||||
|
+ return s.agent.onlineCPUMem(0, false, true)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// GetHypervisorType is used for getting Hypervisor name currently used.
|
||||||
|
// Sandbox implement DeviceReceiver interface from device/api/interface.go
|
||||||
|
func (s *Sandbox) GetHypervisorType() string {
|
||||||
|
diff --git a/virtcontainers/sandbox_test.go b/virtcontainers/sandbox_test.go
|
||||||
|
index 85c712e8..4b02b3f3 100644
|
||||||
|
--- a/virtcontainers/sandbox_test.go
|
||||||
|
+++ b/virtcontainers/sandbox_test.go
|
||||||
|
@@ -25,6 +25,7 @@ import (
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/persist/fs"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/types"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||||
|
specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
@@ -1522,6 +1523,17 @@ func TestSandboxUpdateResources(t *testing.T) {
|
||||||
|
}
|
||||||
|
err = s.updateResources()
|
||||||
|
assert.NoError(t, err)
|
||||||
|
+
|
||||||
|
+ // add a container with huge memory equal utils.MaxHotplugMemMBOnceTime
|
||||||
|
+ contConfig3 := newTestContainerConfigNoop("cont-00003")
|
||||||
|
+ contConfig3.Resources.Memory = &specs.LinuxMemory{
|
||||||
|
+ Limit: new(int64),
|
||||||
|
+ }
|
||||||
|
+ container3MemLimitInBytes := int64(utils.MaxHotplugMemMBOnceTime << utils.MibToBytesShift)
|
||||||
|
+ contConfig3.Resources.Memory.Limit = &container3MemLimitInBytes
|
||||||
|
+ s.config.Containers = append(s.config.Containers, contConfig3)
|
||||||
|
+ err = s.updateResources()
|
||||||
|
+ assert.NoError(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSandboxExperimentalFeature(t *testing.T) {
|
||||||
|
diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go
|
||||||
|
index 2b555ebb..3ae95aef 100644
|
||||||
|
--- a/virtcontainers/utils/utils.go
|
||||||
|
+++ b/virtcontainers/utils/utils.go
|
||||||
|
@@ -25,6 +25,9 @@ const fileMode0755 = os.FileMode(0755)
|
||||||
|
// MibToBytesShift the number to shift needed to convert MiB to Bytes
|
||||||
|
const MibToBytesShift = 20
|
||||||
|
|
||||||
|
+// Max Hotplug Memory size at once time, unit is MB
|
||||||
|
+const MaxHotplugMemMBOnceTime = 32 * 1024
|
||||||
|
+
|
||||||
|
// MaxSocketPathLen is the effective maximum Unix domain socket length.
|
||||||
|
//
|
||||||
|
// See unix(7).
|
||||||
|
diff --git a/virtcontainers/vm.go b/virtcontainers/vm.go
|
||||||
|
index 8d27b1fe..2e5fef44 100644
|
||||||
|
--- a/virtcontainers/vm.go
|
||||||
|
+++ b/virtcontainers/vm.go
|
||||||
|
@@ -370,7 +370,7 @@ func (v *VM) AddMemory(numMB uint32) error {
|
||||||
|
// OnlineCPUMemory puts the hotplugged CPU and memory online.
|
||||||
|
func (v *VM) OnlineCPUMemory() error {
|
||||||
|
v.logger().Infof("online CPU %d and memory", v.cpuDelta)
|
||||||
|
- err := v.agent.onlineCPUMem(v.cpuDelta, false)
|
||||||
|
+ err := v.agent.onlineCPUMem(v.cpuDelta, false, false)
|
||||||
|
if err == nil {
|
||||||
|
v.cpuDelta = 0
|
||||||
|
}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,355 @@
|
|||||||
|
From e6051eb1a8c54b91c46f68eab9a63ad0e73a9221 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Fri, 7 Aug 2020 20:01:15 +0800
|
||||||
|
Subject: [PATCH 17/50] kata-runtime: validate sandbox cpu and memory size
|
||||||
|
|
||||||
|
reason:
|
||||||
|
1. add more strict sandbox cpu and memory size check.
|
||||||
|
2. use GetPhysicalCPUNumber func to replace goruntime.NumCPU(),
|
||||||
|
because goruntime.NumCPU() return the CPU number of cpuset cgroup
|
||||||
|
that current process joined, however kata-runtime process will
|
||||||
|
change the cpuset cgroup when setupSanboxCgroup is called, which
|
||||||
|
may cause goruntime.NumCPU() return value changed.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
pkg/katautils/config.go | 39 ++++++++++++++++++++++++++++++++++---
|
||||||
|
pkg/katautils/config_test.go | 11 ++++++++---
|
||||||
|
virtcontainers/pkg/oci/utils.go | 11 +++++------
|
||||||
|
virtcontainers/qemu_arm64.go | 4 ++--
|
||||||
|
virtcontainers/qemu_arm64_test.go | 4 ++--
|
||||||
|
virtcontainers/sandbox.go | 7 ++++++-
|
||||||
|
virtcontainers/utils/utils.go | 41 +++++++++++++++++++++++++++++++++++++++
|
||||||
|
7 files changed, 100 insertions(+), 17 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
||||||
|
index d1883c94..b9b08469 100644
|
||||||
|
--- a/pkg/katautils/config.go
|
||||||
|
+++ b/pkg/katautils/config.go
|
||||||
|
@@ -11,7 +11,6 @@ import (
|
||||||
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
|
"path/filepath"
|
||||||
|
- goruntime "runtime"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/BurntSushi/toml"
|
||||||
|
@@ -285,7 +284,7 @@ func (h hypervisor) GetEntropySource() string {
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h hypervisor) defaultVCPUs() uint32 {
|
||||||
|
- numCPUs := goruntime.NumCPU()
|
||||||
|
+ numCPUs := utils.GetPhysicalCPUNumber()
|
||||||
|
|
||||||
|
if h.NumVCPUs < 0 || h.NumVCPUs > int32(numCPUs) {
|
||||||
|
return uint32(numCPUs)
|
||||||
|
@@ -297,8 +296,22 @@ func (h hypervisor) defaultVCPUs() uint32 {
|
||||||
|
return uint32(h.NumVCPUs)
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (h hypervisor) checkVCPUs() error {
|
||||||
|
+ numCPUs := utils.GetPhysicalCPUNumber()
|
||||||
|
+
|
||||||
|
+ if h.NumVCPUs <= 0 {
|
||||||
|
+ return fmt.Errorf("invalid vcpus in configuration.toml! vcpus must larger than 0")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if h.NumVCPUs > int32(numCPUs) {
|
||||||
|
+ return fmt.Errorf("invalid vcpus in configuration.toml! vcpus must smaller than max CPUs: %d in machine", numCPUs)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func (h hypervisor) defaultMaxVCPUs() uint32 {
|
||||||
|
- numcpus := uint32(goruntime.NumCPU())
|
||||||
|
+ numcpus := uint32(utils.GetPhysicalCPUNumber())
|
||||||
|
maxvcpus := vc.MaxQemuVCPUs()
|
||||||
|
reqVCPUs := h.DefaultMaxVCPUs
|
||||||
|
|
||||||
|
@@ -324,6 +337,18 @@ func (h hypervisor) defaultMemSz() uint32 {
|
||||||
|
return h.MemorySize
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (h hypervisor) checkMemSz() error {
|
||||||
|
+ if h.MemorySize < utils.MinMemorySizeInMB {
|
||||||
|
+ return fmt.Errorf("invalid memory size! Memory size must larger than %d MB", utils.MinMemorySizeInMB)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if h.MemorySize > utils.MaxMemorySizeInMB {
|
||||||
|
+ return fmt.Errorf("invalid memory size, memory size must smaller than %d MB", utils.MaxMemorySizeInMB)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func (h hypervisor) defaultMemSlots() uint32 {
|
||||||
|
slots := h.MemSlots
|
||||||
|
if slots == 0 {
|
||||||
|
@@ -627,6 +652,14 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
+ if err = h.checkVCPUs(); err != nil {
|
||||||
|
+ return vc.HypervisorConfig{}, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err = h.checkMemSz(); err != nil {
|
||||||
|
+ return vc.HypervisorConfig{}, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
return vc.HypervisorConfig{
|
||||||
|
HypervisorPath: hypervisor,
|
||||||
|
KernelPath: kernel,
|
||||||
|
diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go
|
||||||
|
index 31afcca6..37aa16d0 100644
|
||||||
|
--- a/pkg/katautils/config_test.go
|
||||||
|
+++ b/pkg/katautils/config_test.go
|
||||||
|
@@ -14,7 +14,6 @@ import (
|
||||||
|
"path"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
- goruntime "runtime"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
"testing"
|
||||||
|
@@ -152,7 +151,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
|
||||||
|
KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)),
|
||||||
|
HypervisorMachineType: machineType,
|
||||||
|
NumVCPUs: defaultVCPUCount,
|
||||||
|
- DefaultMaxVCPUs: uint32(goruntime.NumCPU()),
|
||||||
|
+ DefaultMaxVCPUs: uint32(utils.GetPhysicalCPUNumber()),
|
||||||
|
MemorySize: defaultMemSize,
|
||||||
|
DisableBlockDeviceUse: disableBlockDevice,
|
||||||
|
BlockDeviceDriver: defaultBlockDeviceDriver,
|
||||||
|
@@ -727,6 +726,8 @@ func TestMinimalRuntimeConfigWithVsock(t *testing.T) {
|
||||||
|
[hypervisor.qemu]
|
||||||
|
use_vsock = true
|
||||||
|
image = "` + imagePath + `"
|
||||||
|
+ default_vcpus = 1
|
||||||
|
+ default_memory = 1024
|
||||||
|
|
||||||
|
[proxy.kata]
|
||||||
|
path = "` + proxyPath + `"
|
||||||
|
@@ -786,6 +787,8 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
|
||||||
|
utils.VHostVSockDevicePath = orgVHostVSockDevicePath
|
||||||
|
}()
|
||||||
|
utils.VHostVSockDevicePath = "/dev/abc/xyz"
|
||||||
|
+ defaultVCPUs := int32(1)
|
||||||
|
+ defaultMemory := uint32(1024)
|
||||||
|
|
||||||
|
hypervisor := hypervisor{
|
||||||
|
Path: hypervisorPath,
|
||||||
|
@@ -797,6 +800,8 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
|
||||||
|
HotplugVFIOOnRootBus: hotplugVFIOOnRootBus,
|
||||||
|
PCIeRootPort: pcieRootPort,
|
||||||
|
UseVSock: true,
|
||||||
|
+ NumVCPUs: defaultVCPUs,
|
||||||
|
+ MemorySize: defaultMemory,
|
||||||
|
}
|
||||||
|
|
||||||
|
files := []string{hypervisorPath, kernelPath, imagePath}
|
||||||
|
@@ -996,7 +1001,7 @@ func TestNewShimConfig(t *testing.T) {
|
||||||
|
func TestHypervisorDefaults(t *testing.T) {
|
||||||
|
assert := assert.New(t)
|
||||||
|
|
||||||
|
- numCPUs := goruntime.NumCPU()
|
||||||
|
+ numCPUs := utils.GetPhysicalCPUNumber()
|
||||||
|
|
||||||
|
h := hypervisor{}
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
||||||
|
index 05181efd..0a6f08c7 100644
|
||||||
|
--- a/virtcontainers/pkg/oci/utils.go
|
||||||
|
+++ b/virtcontainers/pkg/oci/utils.go
|
||||||
|
@@ -10,22 +10,21 @@ import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"path/filepath"
|
||||||
|
- goruntime "runtime"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
|
||||||
|
criContainerdAnnotations "github.com/containerd/cri-containerd/pkg/annotations"
|
||||||
|
crioAnnotations "github.com/cri-o/cri-o/pkg/annotations"
|
||||||
|
- specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
- "github.com/sirupsen/logrus"
|
||||||
|
-
|
||||||
|
vc "github.com/kata-containers/runtime/virtcontainers"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/device/config"
|
||||||
|
exp "github.com/kata-containers/runtime/virtcontainers/experimental"
|
||||||
|
vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
|
||||||
|
dockershimAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations/dockershim"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/types"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||||
|
+ specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
+ "github.com/sirupsen/logrus"
|
||||||
|
)
|
||||||
|
|
||||||
|
type annotationContainerType struct {
|
||||||
|
@@ -560,7 +559,7 @@ func addHypervisorCPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e
|
||||||
|
return fmt.Errorf("Error encountered parsing annotation default_vcpus: %v, please specify numeric value", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
- numCPUs := goruntime.NumCPU()
|
||||||
|
+ numCPUs := utils.GetPhysicalCPUNumber()
|
||||||
|
|
||||||
|
if uint32(vcpus) > uint32(numCPUs) {
|
||||||
|
return fmt.Errorf("Number of cpus %d specified in annotation default_vcpus is greater than the number of CPUs %d on the system", vcpus, numCPUs)
|
||||||
|
@@ -575,7 +574,7 @@ func addHypervisorCPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e
|
||||||
|
return fmt.Errorf("Error encountered parsing annotation for default_maxvcpus: %v, please specify positive numeric value", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
- numCPUs := goruntime.NumCPU()
|
||||||
|
+ numCPUs := utils.GetPhysicalCPUNumber()
|
||||||
|
max := uint32(maxVCPUs)
|
||||||
|
|
||||||
|
if max > uint32(numCPUs) {
|
||||||
|
diff --git a/virtcontainers/qemu_arm64.go b/virtcontainers/qemu_arm64.go
|
||||||
|
index 6d089cf0..40fb2a80 100644
|
||||||
|
--- a/virtcontainers/qemu_arm64.go
|
||||||
|
+++ b/virtcontainers/qemu_arm64.go
|
||||||
|
@@ -8,11 +8,11 @@ package virtcontainers
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io/ioutil"
|
||||||
|
- "runtime"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
govmmQemu "github.com/intel/govmm/qemu"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
)
|
||||||
|
|
||||||
|
@@ -121,7 +121,7 @@ func MaxQemuVCPUs() uint32 {
|
||||||
|
if hostGICVersion != 0 {
|
||||||
|
return gicList[hostGICVersion]
|
||||||
|
}
|
||||||
|
- return uint32(runtime.NumCPU())
|
||||||
|
+ return uint32(utils.GetPhysicalCPUNumber())
|
||||||
|
}
|
||||||
|
|
||||||
|
func newQemuArch(config HypervisorConfig) qemuArch {
|
||||||
|
diff --git a/virtcontainers/qemu_arm64_test.go b/virtcontainers/qemu_arm64_test.go
|
||||||
|
index 40351158..372fc4a9 100644
|
||||||
|
--- a/virtcontainers/qemu_arm64_test.go
|
||||||
|
+++ b/virtcontainers/qemu_arm64_test.go
|
||||||
|
@@ -10,10 +10,10 @@ import (
|
||||||
|
"io/ioutil"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
- "runtime"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
govmmQemu "github.com/intel/govmm/qemu"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
@@ -63,7 +63,7 @@ func TestMaxQemuVCPUs(t *testing.T) {
|
||||||
|
}
|
||||||
|
|
||||||
|
data := []testData{
|
||||||
|
- {"", uint32(runtime.NumCPU())},
|
||||||
|
+ {"", uint32(utils.GetPhysicalCPUNumber())},
|
||||||
|
{" 1: 0 0 GICv2 25 Level vgic \n", uint32(8)},
|
||||||
|
{" 1: 0 0 GICv3 25 Level vgic \n", uint32(123)},
|
||||||
|
{" 1: 0 0 GICv4 25 Level vgic \n", uint32(123)},
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index a318d677..a8522b96 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -1875,6 +1875,11 @@ func (s *Sandbox) updateResources() error {
|
||||||
|
reqMemMB := uint32(sandboxMemoryByte >> utils.MibToBytesShift)
|
||||||
|
currentMemMB := s.hypervisor.getMemorySize()
|
||||||
|
|
||||||
|
+ // check request sandbox memory size larger than utils.MaxMemorySizInMB or not
|
||||||
|
+ if reqMemMB > utils.MaxMemorySizeInMB {
|
||||||
|
+ return fmt.Errorf("updateResources failed! Sandbox memory should <= %d MiB", utils.MaxMemorySizeInMB)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
// If request hotplug memory size larger than utils.MaxHotplugMemMBOnceTime,
|
||||||
|
// inorder to avoid hotplug memory oom problem, we need to hotplug large memory
|
||||||
|
// with two steps. First, hotplug utils.MaxHotplugMemMBOnceTime size memory into
|
||||||
|
@@ -1898,7 +1903,7 @@ func (s *Sandbox) updateResources() error {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
- if err := s.agent.onlineCPUMem(0, false, false); err != nil {
|
||||||
|
+ if err := s.agent.onlineCPUMem(0, false, true); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go
|
||||||
|
index 3ae95aef..5d38e594 100644
|
||||||
|
--- a/virtcontainers/utils/utils.go
|
||||||
|
+++ b/virtcontainers/utils/utils.go
|
||||||
|
@@ -6,6 +6,7 @@
|
||||||
|
package utils
|
||||||
|
|
||||||
|
import (
|
||||||
|
+ "bufio"
|
||||||
|
"crypto/rand"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
@@ -28,11 +29,26 @@ const MibToBytesShift = 20
|
||||||
|
// Max Hotplug Memory size at once time, unit is MB
|
||||||
|
const MaxHotplugMemMBOnceTime = 32 * 1024
|
||||||
|
|
||||||
|
+const (
|
||||||
|
+ // Min needed memory size to start a Kata VM
|
||||||
|
+ MinMemorySizeInMB = 300
|
||||||
|
+ MinMemorySizeInByte = MinMemorySizeInMB << MibToBytesShift
|
||||||
|
+
|
||||||
|
+ // Max support memory size in the Kata VM
|
||||||
|
+ MaxMemorySizeInMB = 512 * 1024
|
||||||
|
+ MaxMemorySizeInByte = MaxMemorySizeInMB << MibToBytesShift
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
// MaxSocketPathLen is the effective maximum Unix domain socket length.
|
||||||
|
//
|
||||||
|
// See unix(7).
|
||||||
|
const MaxSocketPathLen = 107
|
||||||
|
|
||||||
|
+const (
|
||||||
|
+ procCPUInfoPath = "/proc/cpuinfo"
|
||||||
|
+ processorIdentifier = "processor"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
// VHostVSockDevicePath path to vhost-vsock device
|
||||||
|
var VHostVSockDevicePath = "/dev/vhost-vsock"
|
||||||
|
|
||||||
|
@@ -324,3 +340,28 @@ func IsProcessRunning(pid int, processName string, sandboxID string) bool {
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+// GetPhysicalCPUNumber return the number of the CPUs in the physical machine
|
||||||
|
+func GetPhysicalCPUNumber() int {
|
||||||
|
+ f, err := os.Open(procCPUInfoPath)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return 0
|
||||||
|
+ }
|
||||||
|
+ defer f.Close()
|
||||||
|
+
|
||||||
|
+ cpuNum := 0
|
||||||
|
+ s := bufio.NewScanner(f)
|
||||||
|
+ for s.Scan() {
|
||||||
|
+ if err := s.Err(); err != nil {
|
||||||
|
+ return 0
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ fields := strings.Fields(s.Text())
|
||||||
|
+ if len(fields) > 0 {
|
||||||
|
+ if fields[0] == processorIdentifier {
|
||||||
|
+ cpuNum++
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ return cpuNum
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,50 @@
|
|||||||
|
From c785f8f744050155102664d56de5bfb55e91915d Mon Sep 17 00:00:00 2001
|
||||||
|
From: Evan Foster <efoster@adobe.com>
|
||||||
|
Date: Mon, 13 Jul 2020 12:53:40 -0600
|
||||||
|
Subject: [PATCH 18/50] sandbox: Stop and clean up containers that fail to
|
||||||
|
create
|
||||||
|
|
||||||
|
A container that is created and added to a sandbox can still fail
|
||||||
|
the final creation steps. In this case, the container must be stopped
|
||||||
|
and have its resources cleaned up to prevent leaking sandbox mounts.
|
||||||
|
|
||||||
|
Fixes #2816
|
||||||
|
|
||||||
|
cherry-pick from: https://github.com/kata-containers/runtime/pull/2826
|
||||||
|
|
||||||
|
Signed-off-by: Evan Foster <efoster@adobe.com>
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/sandbox.go | 11 +++++++++++
|
||||||
|
1 file changed, 11 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index a8522b96..3dbf640e 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -1,4 +1,5 @@
|
||||||
|
// Copyright (c) 2016 Intel Corporation
|
||||||
|
+// Copyright (c) 2020 Adobe Inc.
|
||||||
|
//
|
||||||
|
// SPDX-License-Identifier: Apache-2.0
|
||||||
|
//
|
||||||
|
@@ -1172,6 +1173,16 @@ func (s *Sandbox) CreateContainer(contConfig ContainerConfig) (VCContainer, erro
|
||||||
|
defer func() {
|
||||||
|
// Rollback if error happens.
|
||||||
|
if err != nil {
|
||||||
|
+ logger := s.Logger().WithFields(logrus.Fields{"container-id": c.id, "sandox-id": s.id, "rollback": true})
|
||||||
|
+
|
||||||
|
+ logger.Warning("Cleaning up partially created container")
|
||||||
|
+
|
||||||
|
+ if err2 := c.stop(true); err2 != nil {
|
||||||
|
+ logger.WithError(err2).Warning("Could not delete container")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ logger.Debug("Removing stopped container from sandbox store")
|
||||||
|
+
|
||||||
|
s.removeContainer(c.id)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,33 @@
|
|||||||
|
From d20cb25c8a145e1d3e64eefa69242d87b7a67f92 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Sat, 8 Aug 2020 16:00:45 -0400
|
||||||
|
Subject: [PATCH 19/50] virtcontainers: fix delete sandbox failed problem
|
||||||
|
|
||||||
|
fixes: #2882
|
||||||
|
|
||||||
|
reason: If error happens after container create and before sandbox
|
||||||
|
updateResouce in the `CreateContainer()`, then delete sandbox
|
||||||
|
forcefully will return error because s.config.Containers config not
|
||||||
|
flushed into persist store.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/sandbox.go | 2 ++
|
||||||
|
1 file changed, 2 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index 3dbf640e..7322ef58 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -1156,6 +1156,8 @@ func (s *Sandbox) CreateContainer(contConfig ContainerConfig) (VCContainer, erro
|
||||||
|
if len(s.config.Containers) > 0 {
|
||||||
|
// delete container config
|
||||||
|
s.config.Containers = s.config.Containers[:len(s.config.Containers)-1]
|
||||||
|
+ // need to flush change to persist storage
|
||||||
|
+ _ = s.storeSandbox()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,188 @@
|
|||||||
|
From 75141529674545a2f84b01c730f614901ad2265e Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Mon, 10 Aug 2020 10:08:47 +0800
|
||||||
|
Subject: [PATCH 20/50] virtcontainers: add enable_cpu_memory_hotplug config in
|
||||||
|
the configuration.toml
|
||||||
|
|
||||||
|
reason: add enable_cpu_memory_hotplug config to control whether enable hotplug
|
||||||
|
memory and cpu resource into the Kata VM. If we can calculate the whole sandbox
|
||||||
|
resource usage already, we just need to pass the value of resouces by annotation
|
||||||
|
without hotplugging.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
cli/config/configuration-qemu.toml.in | 4 ++++
|
||||||
|
pkg/katautils/config.go | 2 ++
|
||||||
|
virtcontainers/container.go | 6 ++++--
|
||||||
|
virtcontainers/hypervisor.go | 3 +++
|
||||||
|
virtcontainers/persist.go | 2 ++
|
||||||
|
virtcontainers/persist/api/config.go | 3 +++
|
||||||
|
virtcontainers/sandbox.go | 38 +++++++++++++++++++++--------------
|
||||||
|
7 files changed, 41 insertions(+), 17 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in
|
||||||
|
index 82461732..b44e84d8 100644
|
||||||
|
--- a/cli/config/configuration-qemu.toml.in
|
||||||
|
+++ b/cli/config/configuration-qemu.toml.in
|
||||||
|
@@ -100,6 +100,10 @@ default_memory = @DEFMEMSZ@
|
||||||
|
# Default false
|
||||||
|
#enable_reuse_cpu_memory = false
|
||||||
|
|
||||||
|
+# If enabled, the runtime will support cpu and memory hot plug
|
||||||
|
+# (default: disabled)
|
||||||
|
+#enable_cpu_memory_hotplug = true
|
||||||
|
+
|
||||||
|
# Disable block device from being used for a container's rootfs.
|
||||||
|
# In case of a storage driver like devicemapper where a container's
|
||||||
|
# root file system is backed by a block device, the block device is passed
|
||||||
|
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
||||||
|
index b9b08469..9a99b9d4 100644
|
||||||
|
--- a/pkg/katautils/config.go
|
||||||
|
+++ b/pkg/katautils/config.go
|
||||||
|
@@ -112,6 +112,7 @@ type hypervisor struct {
|
||||||
|
MemSlots uint32 `toml:"memory_slots"`
|
||||||
|
MemOffset uint32 `toml:"memory_offset"`
|
||||||
|
EnableCPUMemoryReuse bool `toml:"enable_reuse_cpu_memory"`
|
||||||
|
+ EnableCPUMemoryHotPlug bool `toml:"enable_cpu_memory_hotplug"`
|
||||||
|
DefaultBridges uint32 `toml:"default_bridges"`
|
||||||
|
Msize9p uint32 `toml:"msize_9p"`
|
||||||
|
PCIeRootPort uint32 `toml:"pcie_root_port"`
|
||||||
|
@@ -675,6 +676,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
||||||
|
MemSlots: h.defaultMemSlots(),
|
||||||
|
MemOffset: h.defaultMemOffset(),
|
||||||
|
EnableCPUMemoryReuse: h.EnableCPUMemoryReuse,
|
||||||
|
+ EnableCPUMemoryHotPlug: h.EnableCPUMemoryHotPlug,
|
||||||
|
VirtioMem: h.VirtioMem,
|
||||||
|
EntropySource: h.GetEntropySource(),
|
||||||
|
DefaultBridges: h.defaultBridges(),
|
||||||
|
diff --git a/virtcontainers/container.go b/virtcontainers/container.go
|
||||||
|
index 75f590eb..1b89f6ac 100644
|
||||||
|
--- a/virtcontainers/container.go
|
||||||
|
+++ b/virtcontainers/container.go
|
||||||
|
@@ -1273,8 +1273,10 @@ func (c *Container) update(resources specs.LinuxResources) error {
|
||||||
|
c.config.Resources.Memory.Limit = mem.Limit
|
||||||
|
}
|
||||||
|
|
||||||
|
- if err := c.sandbox.updateResources(); err != nil {
|
||||||
|
- return err
|
||||||
|
+ if c.sandbox.config.HypervisorConfig.EnableCPUMemoryHotPlug {
|
||||||
|
+ if err := c.sandbox.updateResources(); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
|
||||||
|
if !c.sandbox.config.SandboxCgroupOnly {
|
||||||
|
diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go
|
||||||
|
index 9cd685ad..c0723daa 100644
|
||||||
|
--- a/virtcontainers/hypervisor.go
|
||||||
|
+++ b/virtcontainers/hypervisor.go
|
||||||
|
@@ -253,6 +253,9 @@ type HypervisorConfig struct {
|
||||||
|
// Enable hypervisor cpu and memory resource share with container
|
||||||
|
EnableCPUMemoryReuse bool
|
||||||
|
|
||||||
|
+ // Enable hotplug cpu and memory resource into container
|
||||||
|
+ EnableCPUMemoryHotPlug bool
|
||||||
|
+
|
||||||
|
// VirtioFSCacheSize is the DAX cache size in MiB
|
||||||
|
VirtioFSCacheSize uint32
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go
|
||||||
|
index aef4d18d..6bd09a0b 100644
|
||||||
|
--- a/virtcontainers/persist.go
|
||||||
|
+++ b/virtcontainers/persist.go
|
||||||
|
@@ -215,6 +215,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
|
||||||
|
MemSlots: sconfig.HypervisorConfig.MemSlots,
|
||||||
|
MemOffset: sconfig.HypervisorConfig.MemOffset,
|
||||||
|
EnableCPUMemoryReuse: sconfig.HypervisorConfig.EnableCPUMemoryReuse,
|
||||||
|
+ EnableCPUMemoryHotPlug: sconfig.HypervisorConfig.EnableCPUMemoryHotPlug,
|
||||||
|
VirtioMem: sconfig.HypervisorConfig.VirtioMem,
|
||||||
|
VirtioFSCacheSize: sconfig.HypervisorConfig.VirtioFSCacheSize,
|
||||||
|
KernelPath: sconfig.HypervisorConfig.KernelPath,
|
||||||
|
@@ -505,6 +506,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
|
||||||
|
MemSlots: hconf.MemSlots,
|
||||||
|
MemOffset: hconf.MemOffset,
|
||||||
|
EnableCPUMemoryReuse: hconf.EnableCPUMemoryReuse,
|
||||||
|
+ EnableCPUMemoryHotPlug: hconf.EnableCPUMemoryHotPlug,
|
||||||
|
VirtioMem: hconf.VirtioMem,
|
||||||
|
VirtioFSCacheSize: hconf.VirtioFSCacheSize,
|
||||||
|
KernelPath: hconf.KernelPath,
|
||||||
|
diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go
|
||||||
|
index a3c6ec91..cfbee849 100644
|
||||||
|
--- a/virtcontainers/persist/api/config.go
|
||||||
|
+++ b/virtcontainers/persist/api/config.go
|
||||||
|
@@ -38,6 +38,9 @@ type HypervisorConfig struct {
|
||||||
|
// Enable hypervisor cpu and memory resource share with container
|
||||||
|
EnableCPUMemoryReuse bool
|
||||||
|
|
||||||
|
+ // Enable hotplug cpu and memory resource into container
|
||||||
|
+ EnableCPUMemoryHotPlug bool
|
||||||
|
+
|
||||||
|
// VirtioFSCacheSize is the DAX cache size in MiB
|
||||||
|
VirtioFSCacheSize uint32
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index 7322ef58..8fcd92d4 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -1189,12 +1189,16 @@ func (s *Sandbox) CreateContainer(contConfig ContainerConfig) (VCContainer, erro
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
- // Sandbox is reponsable to update VM resources needed by Containers
|
||||||
|
- // Update resources after having added containers to the sandbox, since
|
||||||
|
- // container status is requiered to know if more resources should be added.
|
||||||
|
- err = s.updateResources()
|
||||||
|
- if err != nil {
|
||||||
|
- return nil, err
|
||||||
|
+ // update sandbox resource only when enable_cpu_memory_hotplug config set true
|
||||||
|
+ // in the config.toml file
|
||||||
|
+ if s.config.HypervisorConfig.EnableCPUMemoryHotPlug {
|
||||||
|
+ // Sandbox is reponsable to update VM resources needed by Containers
|
||||||
|
+ // Update resources after having added containers to the sandbox, since
|
||||||
|
+ // container status is requiered to know if more resources should be added.
|
||||||
|
+ err = s.updateResources()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = s.cgroupsUpdate(); err != nil {
|
||||||
|
@@ -1228,11 +1232,13 @@ func (s *Sandbox) StartContainer(containerID string) (VCContainer, error) {
|
||||||
|
|
||||||
|
s.Logger().Info("Container is started")
|
||||||
|
|
||||||
|
- // Update sandbox resources in case a stopped container
|
||||||
|
- // is started
|
||||||
|
- err = s.updateResources()
|
||||||
|
- if err != nil {
|
||||||
|
- return nil, err
|
||||||
|
+ if s.config.HypervisorConfig.EnableCPUMemoryHotPlug {
|
||||||
|
+ // Update sandbox resources in case a stopped container
|
||||||
|
+ // is started
|
||||||
|
+ err = s.updateResources()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
|
||||||
|
return c, nil
|
||||||
|
@@ -1503,10 +1509,12 @@ func (s *Sandbox) createContainers() error {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
- // Update resources after having added containers to the sandbox, since
|
||||||
|
- // container status is requiered to know if more resources should be added.
|
||||||
|
- if err := s.updateResources(); err != nil {
|
||||||
|
- return err
|
||||||
|
+ if s.config.HypervisorConfig.EnableCPUMemoryHotPlug {
|
||||||
|
+ // Update resources after having added containers to the sandbox, since
|
||||||
|
+ // container status is requiered to know if more resources should be added.
|
||||||
|
+ if err := s.updateResources(); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.cgroupsUpdate(); err != nil {
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,195 @@
|
|||||||
|
From 79cf2f5a52af51d8a62353a99e894808281769e2 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Mon, 10 Aug 2020 11:25:02 +0800
|
||||||
|
Subject: [PATCH 21/50] kata-runtime: add sandbox_cpu and sandbox_mem
|
||||||
|
annotations
|
||||||
|
|
||||||
|
reason: add sandbox_cpu and sandbox_men annotations to set
|
||||||
|
Kata VM vCPU number and memory size.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/pkg/annotations/annotations.go | 6 +++++
|
||||||
|
virtcontainers/pkg/oci/utils.go | 39 +++++++++++++++++++++++++--
|
||||||
|
virtcontainers/sandbox.go | 32 ++++++++++++++++++++++
|
||||||
|
virtcontainers/utils/utils.go | 14 ++++++++++
|
||||||
|
4 files changed, 89 insertions(+), 2 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go
|
||||||
|
index 10ce7833..903c7f03 100644
|
||||||
|
--- a/virtcontainers/pkg/annotations/annotations.go
|
||||||
|
+++ b/virtcontainers/pkg/annotations/annotations.go
|
||||||
|
@@ -251,6 +251,12 @@ const (
|
||||||
|
ContainerPipeSizeKernelParam = "agent." + ContainerPipeSizeOption
|
||||||
|
)
|
||||||
|
|
||||||
|
+// iSula self defined annotations
|
||||||
|
+const (
|
||||||
|
+ StaticCPUTypeKey = kataAnnotationsPrefix + "sandbox_cpu"
|
||||||
|
+ StaticMemTypeKey = kataAnnotationsPrefix + "sandbox_mem"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
const (
|
||||||
|
// SHA512 is the SHA-512 (64) hash algorithm
|
||||||
|
SHA512 string = "sha512"
|
||||||
|
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
||||||
|
index 0a6f08c7..36c730b7 100644
|
||||||
|
--- a/virtcontainers/pkg/oci/utils.go
|
||||||
|
+++ b/virtcontainers/pkg/oci/utils.go
|
||||||
|
@@ -16,6 +16,7 @@ import (
|
||||||
|
|
||||||
|
criContainerdAnnotations "github.com/containerd/cri-containerd/pkg/annotations"
|
||||||
|
crioAnnotations "github.com/cri-o/cri-o/pkg/annotations"
|
||||||
|
+ "github.com/docker/go-units"
|
||||||
|
vc "github.com/kata-containers/runtime/virtcontainers"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/device/config"
|
||||||
|
exp "github.com/kata-containers/runtime/virtcontainers/experimental"
|
||||||
|
@@ -34,7 +35,10 @@ type annotationContainerType struct {
|
||||||
|
|
||||||
|
type annotationHandler func(value string) error
|
||||||
|
|
||||||
|
-var annotationHandlerList = map[string]annotationHandler{}
|
||||||
|
+var annotationHandlerList = map[string]annotationHandler{
|
||||||
|
+ vcAnnotations.StaticCPUTypeKey: validateSandboxCPU,
|
||||||
|
+ vcAnnotations.StaticMemTypeKey: validateSandboxMem,
|
||||||
|
+}
|
||||||
|
|
||||||
|
var (
|
||||||
|
// ErrNoLinux is an error for missing Linux sections in the OCI configuration file.
|
||||||
|
@@ -1036,7 +1040,10 @@ func validateOtherSandboxAnnotations(annotation, value string) error {
|
||||||
|
|
||||||
|
// addOtherSandboxAnnotation add self defined annotation for sandbox
|
||||||
|
func addOtherSandboxAnnotation(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error {
|
||||||
|
- otherSandboxAnnotationsKey := []string{}
|
||||||
|
+ otherSandboxAnnotationsKey := []string{
|
||||||
|
+ vcAnnotations.StaticCPUTypeKey,
|
||||||
|
+ vcAnnotations.StaticMemTypeKey,
|
||||||
|
+ }
|
||||||
|
|
||||||
|
for _, a := range otherSandboxAnnotationsKey {
|
||||||
|
value, ok := ocispec.Annotations[a]
|
||||||
|
@@ -1053,3 +1060,31 @@ func addOtherSandboxAnnotation(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func validateSandboxCPU(value string) error {
|
||||||
|
+ // check min cpu value
|
||||||
|
+ cpus, err := utils.RoundVCPUNumber(value)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return fmt.Errorf("valiate sandbox_cpu annotation fail: %v", err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ maxPhysicalCPUs := utils.GetPhysicalCPUNumber()
|
||||||
|
+ if cpus > maxPhysicalCPUs {
|
||||||
|
+ return fmt.Errorf("sandbox_cpu annotation value exceed the machine max CPU number: %d", cpus)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func validateSandboxMem(value string) error {
|
||||||
|
+ memSizeInBytes, err := units.RAMInBytes(value)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return fmt.Errorf("parse sandbox_mem value: %d fail: %v", memSizeInBytes, err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if memSizeInBytes < utils.MinMemorySizeInByte || memSizeInBytes > utils.MaxMemorySizeInByte {
|
||||||
|
+ return fmt.Errorf("invalid sandbox_mem value size in bytes: %v", memSizeInBytes)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index 8fcd92d4..ba704249 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -20,6 +20,7 @@ import (
|
||||||
|
|
||||||
|
"github.com/containerd/cgroups"
|
||||||
|
"github.com/containernetworking/plugins/pkg/ns"
|
||||||
|
+ "github.com/docker/go-units"
|
||||||
|
"github.com/kata-containers/agent/protocols/grpc"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/device/api"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/device/config"
|
||||||
|
@@ -479,6 +480,10 @@ func createSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Fac
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
+ if err := updateStaticSandboxResources(&sandboxConfig); err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
s, err := newSandbox(ctx, sandboxConfig, factory)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
@@ -2359,3 +2364,30 @@ func (s *Sandbox) setContainersState(state types.StateString) error {
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+// updateStaticSandboxResources update sandbox's cpu and memory resource passed by
|
||||||
|
+// sandbox_cpu and sandbox_mem annotations
|
||||||
|
+func updateStaticSandboxResources(sandboxConfig *SandboxConfig) error {
|
||||||
|
+ // update cpu resource
|
||||||
|
+ if cpuNumVal, ok := sandboxConfig.Annotations[annotations.StaticCPUTypeKey]; ok {
|
||||||
|
+ cpuNum, err := utils.RoundVCPUNumber(cpuNumVal)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ sandboxConfig.HypervisorConfig.NumVCPUs = (uint32)(cpuNum)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // update mem resource
|
||||||
|
+ if memVal, ok := sandboxConfig.Annotations[annotations.StaticMemTypeKey]; ok {
|
||||||
|
+ memSizeInBytes, err := units.RAMInBytes(memVal)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ memSizeInMB := memSizeInBytes >> utils.MibToBytesShift
|
||||||
|
+ sandboxConfig.HypervisorConfig.MemorySize = (uint32)(memSizeInMB)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go
|
||||||
|
index 5d38e594..9490faa1 100644
|
||||||
|
--- a/virtcontainers/utils/utils.go
|
||||||
|
+++ b/virtcontainers/utils/utils.go
|
||||||
|
@@ -11,6 +11,7 @@ import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
|
+ "math"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
@@ -30,6 +31,9 @@ const MibToBytesShift = 20
|
||||||
|
const MaxHotplugMemMBOnceTime = 32 * 1024
|
||||||
|
|
||||||
|
const (
|
||||||
|
+ // minCPUs is allowed minimum CPU
|
||||||
|
+ minCPUs = 0.25
|
||||||
|
+
|
||||||
|
// Min needed memory size to start a Kata VM
|
||||||
|
MinMemorySizeInMB = 300
|
||||||
|
MinMemorySizeInByte = MinMemorySizeInMB << MibToBytesShift
|
||||||
|
@@ -365,3 +369,13 @@ func GetPhysicalCPUNumber() int {
|
||||||
|
}
|
||||||
|
return cpuNum
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func RoundVCPUNumber(value string) (int, error) {
|
||||||
|
+ cpuNum, err := strconv.ParseFloat(value, 64)
|
||||||
|
+ if err != nil || cpuNum < minCPUs {
|
||||||
|
+ return 0, fmt.Errorf("invalid sandbox cpu number: %v", cpuNum)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ cpus := int(math.Ceil(cpuNum))
|
||||||
|
+ return cpus, nil
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,39 @@
|
|||||||
|
From e5e3232f7268110f7e3e3c4814eab31a6704b672 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Mon, 10 Aug 2020 20:11:07 +0800
|
||||||
|
Subject: [PATCH 22/50] kata-runtime: skip go version check and do not build
|
||||||
|
containerd-shim-v2
|
||||||
|
|
||||||
|
reason: skip go version check and do not build containerd-shim-v2
|
||||||
|
because iSulad current not support shimV2
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
Makefile | 4 ++--
|
||||||
|
1 file changed, 2 insertions(+), 2 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/Makefile b/Makefile
|
||||||
|
index 14a0ea47..d5e4bbe1 100644
|
||||||
|
--- a/Makefile
|
||||||
|
+++ b/Makefile
|
||||||
|
@@ -12,7 +12,7 @@ for file in /etc/os-release /usr/lib/os-release; do \
|
||||||
|
fi \
|
||||||
|
done)
|
||||||
|
|
||||||
|
-SKIP_GO_VERSION_CHECK=
|
||||||
|
+SKIP_GO_VERSION_CHECK=y
|
||||||
|
include golang.mk
|
||||||
|
|
||||||
|
#Get ARCH.
|
||||||
|
@@ -503,7 +503,7 @@ define SHOW_ARCH
|
||||||
|
$(shell printf "\\t%s%s\\\n" "$(1)" $(if $(filter $(ARCH),$(1))," (default)",""))
|
||||||
|
endef
|
||||||
|
|
||||||
|
-all: runtime containerd-shim-v2 netmon
|
||||||
|
+all: runtime netmon
|
||||||
|
|
||||||
|
# Targets that depend on .git-commit can use $(shell cat .git-commit) to get a
|
||||||
|
# git revision string. They will only be rebuilt if the revision string
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,42 @@
|
|||||||
|
From 226c3336dcc70bd17e3471ff98106a2f8dee9ac5 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 11 Aug 2020 22:32:49 +0800
|
||||||
|
Subject: [PATCH 23/50] kata-runtime: set PCIBridgeMaxCapacity limit to 25
|
||||||
|
|
||||||
|
reason: set PCIBridgeMaxCapacity limit to 25.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/qemu_arch_base_test.go | 2 +-
|
||||||
|
virtcontainers/types/bridges.go | 2 +-
|
||||||
|
2 files changed, 2 insertions(+), 2 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/qemu_arch_base_test.go b/virtcontainers/qemu_arch_base_test.go
|
||||||
|
index 169e002e..8219f3c5 100644
|
||||||
|
--- a/virtcontainers/qemu_arch_base_test.go
|
||||||
|
+++ b/virtcontainers/qemu_arch_base_test.go
|
||||||
|
@@ -175,7 +175,7 @@ func TestQemuAddDeviceToBridge(t *testing.T) {
|
||||||
|
}
|
||||||
|
|
||||||
|
// fail to add device to bridge cause no more available bridge slot
|
||||||
|
- _, _, err := q.addDeviceToBridge("qemu-bridge-31", types.PCI)
|
||||||
|
+ _, _, err := q.addDeviceToBridge("qemu-bridge-26", types.PCI)
|
||||||
|
exceptErr := errors.New("no more bridge slots available")
|
||||||
|
assert.Equal(exceptErr.Error(), err.Error())
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/types/bridges.go b/virtcontainers/types/bridges.go
|
||||||
|
index cb15a88f..c3538ce4 100644
|
||||||
|
--- a/virtcontainers/types/bridges.go
|
||||||
|
+++ b/virtcontainers/types/bridges.go
|
||||||
|
@@ -10,7 +10,7 @@ import "fmt"
|
||||||
|
// Type represents a type of bus and bridge.
|
||||||
|
type Type string
|
||||||
|
|
||||||
|
-const PCIBridgeMaxCapacity = 30
|
||||||
|
+const PCIBridgeMaxCapacity = 25
|
||||||
|
|
||||||
|
const (
|
||||||
|
// PCI represents a PCI bus and bridge
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,462 @@
|
|||||||
|
From e861f426c9e6702e820348ddc61b18013c853402 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Thu, 13 Aug 2020 11:24:58 +0800
|
||||||
|
Subject: [PATCH 24/50] kata-runtime: support hotplug tap interface into kata
|
||||||
|
VM
|
||||||
|
|
||||||
|
reason: support hotplug exist tap interface or a new created tap
|
||||||
|
interface into kata VM.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
cli/network.go | 45 ++++++++++++++---
|
||||||
|
virtcontainers/kata_agent.go | 12 ++++-
|
||||||
|
virtcontainers/network.go | 100 +++++++++++++++++++++++---------------
|
||||||
|
virtcontainers/pkg/types/types.go | 16 +++---
|
||||||
|
virtcontainers/qemu.go | 11 +++--
|
||||||
|
virtcontainers/sandbox.go | 24 +++++++--
|
||||||
|
virtcontainers/tap_endpoint.go | 23 +++++++--
|
||||||
|
7 files changed, 161 insertions(+), 70 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/cli/network.go b/cli/network.go
|
||||||
|
index 881a2358..7e7791f1 100644
|
||||||
|
--- a/cli/network.go
|
||||||
|
+++ b/cli/network.go
|
||||||
|
@@ -26,6 +26,8 @@ const (
|
||||||
|
routeType
|
||||||
|
)
|
||||||
|
|
||||||
|
+const defaultLinkType = "tap"
|
||||||
|
+
|
||||||
|
var kataNetworkCLICommand = cli.Command{
|
||||||
|
Name: "kata-network",
|
||||||
|
Usage: "manage interfaces and routes for container",
|
||||||
|
@@ -42,10 +44,22 @@ var kataNetworkCLICommand = cli.Command{
|
||||||
|
}
|
||||||
|
|
||||||
|
var addIfaceCommand = cli.Command{
|
||||||
|
- Name: "add-iface",
|
||||||
|
- Usage: "add an interface to a container",
|
||||||
|
- ArgsUsage: `add-iface <container-id> file or - for stdin`,
|
||||||
|
- Flags: []cli.Flag{},
|
||||||
|
+ Name: "add-iface",
|
||||||
|
+ Usage: "add an interface to a container",
|
||||||
|
+ ArgsUsage: `add-iface <container-id> file or - for stdin
|
||||||
|
+ file or stdin for example:
|
||||||
|
+ {
|
||||||
|
+ "device":"<device-name>",
|
||||||
|
+ "name":"<interface-name>",
|
||||||
|
+ "IPAddresses":[{"address":"<ip>","mask":"<mask>"}],
|
||||||
|
+ "mtu":<mtu>,
|
||||||
|
+ "hwAddr":"<mac>",
|
||||||
|
+ "linkType":"tap",
|
||||||
|
+ "vhostUserSocket":"<path>"
|
||||||
|
+ }
|
||||||
|
+ device,name,mtu,hwAddr are required, IPAddresses and vhostUserSocket are optional.
|
||||||
|
+ `,
|
||||||
|
+ Flags: []cli.Flag{},
|
||||||
|
Action: func(context *cli.Context) error {
|
||||||
|
ctx, err := cliContextToContext(context)
|
||||||
|
if err != nil {
|
||||||
|
@@ -57,10 +71,20 @@ var addIfaceCommand = cli.Command{
|
||||||
|
}
|
||||||
|
|
||||||
|
var delIfaceCommand = cli.Command{
|
||||||
|
- Name: "del-iface",
|
||||||
|
- Usage: "delete an interface from a container",
|
||||||
|
- ArgsUsage: `del-iface <container-id> file or - for stdin`,
|
||||||
|
- Flags: []cli.Flag{},
|
||||||
|
+ Name: "del-iface",
|
||||||
|
+ Usage: "delete an interface from a container",
|
||||||
|
+ ArgsUsage: `del-iface <container-id> file or - for stdin
|
||||||
|
+ file or stdin for example:
|
||||||
|
+ {
|
||||||
|
+ "device":"",
|
||||||
|
+ "name":"<interface-name>",
|
||||||
|
+ "IPAddresses":[],
|
||||||
|
+ "mtu":0,
|
||||||
|
+ "hwAddr":""
|
||||||
|
+ }
|
||||||
|
+ Only the "name" field is required.
|
||||||
|
+ `,
|
||||||
|
+ Flags: []cli.Flag{},
|
||||||
|
Action: func(context *cli.Context) error {
|
||||||
|
ctx, err := cliContextToContext(context)
|
||||||
|
if err != nil {
|
||||||
|
@@ -156,6 +180,11 @@ func networkModifyCommand(ctx context.Context, containerID, input string, opType
|
||||||
|
if err = json.NewDecoder(f).Decode(&inf); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+ if len(inf.LinkType) == 0 {
|
||||||
|
+ inf.LinkType = defaultLinkType
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
if add {
|
||||||
|
resultingInf, err = vci.AddInterface(ctx, sandboxID, inf)
|
||||||
|
if err != nil {
|
||||||
|
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
|
||||||
|
index 8e073339..dfdd263b 100644
|
||||||
|
--- a/virtcontainers/kata_agent.go
|
||||||
|
+++ b/virtcontainers/kata_agent.go
|
||||||
|
@@ -600,8 +600,16 @@ func (k *kataAgent) updateInterface(ifc *vcTypes.Interface) (*vcTypes.Interface,
|
||||||
|
"resulting-interface": fmt.Sprintf("%+v", resultingInterface),
|
||||||
|
}).WithError(err).Error("update interface request failed")
|
||||||
|
}
|
||||||
|
- if resultInterface, ok := resultingInterface.(*vcTypes.Interface); ok {
|
||||||
|
- return resultInterface, err
|
||||||
|
+ if resultInterface, ok := resultingInterface.(*aTypes.Interface); ok {
|
||||||
|
+ iface := &vcTypes.Interface{
|
||||||
|
+ Device: resultInterface.Device,
|
||||||
|
+ Name: resultInterface.Name,
|
||||||
|
+ IPAddresses: k.convertToIPAddresses(resultInterface.IPAddresses),
|
||||||
|
+ Mtu: resultInterface.Mtu,
|
||||||
|
+ HwAddr: resultInterface.HwAddr,
|
||||||
|
+ PciAddr: resultInterface.PciAddr,
|
||||||
|
+ }
|
||||||
|
+ return iface, err
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
|
||||||
|
index d70c5360..e909a822 100644
|
||||||
|
--- a/virtcontainers/network.go
|
||||||
|
+++ b/virtcontainers/network.go
|
||||||
|
@@ -117,6 +117,7 @@ type NetlinkIface struct {
|
||||||
|
// NetworkInfo gathers all information related to a network interface.
|
||||||
|
// It can be used to store the description of the underlying network.
|
||||||
|
type NetworkInfo struct {
|
||||||
|
+ Device string
|
||||||
|
Iface NetlinkIface
|
||||||
|
Addrs []netlink.Addr
|
||||||
|
Routes []netlink.Route
|
||||||
|
@@ -303,6 +304,20 @@ func createLink(netHandle *netlink.Handle, name string, expectedLink netlink.Lin
|
||||||
|
var newLink netlink.Link
|
||||||
|
var fds []*os.File
|
||||||
|
|
||||||
|
+ // check if tapname exists, if true, use existing one instead of create new one
|
||||||
|
+ if name != "" {
|
||||||
|
+ retLink, err := netlink.LinkByName(name)
|
||||||
|
+ // link exist, use it
|
||||||
|
+ if err == nil {
|
||||||
|
+ networkLogger().Debugf("exist tap device is found, instead of creating new one")
|
||||||
|
+ tuntapLink, ok := retLink.(*netlink.Tuntap)
|
||||||
|
+ if ok {
|
||||||
|
+ fds = tuntapLink.Fds
|
||||||
|
+ }
|
||||||
|
+ return retLink, nil, nil
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
switch expectedLink.Type() {
|
||||||
|
case (&netlink.Tuntap{}).Type():
|
||||||
|
flags := netlink.TUNTAP_VNET_HDR
|
||||||
|
@@ -1156,7 +1171,7 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li
|
||||||
|
|
||||||
|
// Check if interface is a physical interface. Do not create
|
||||||
|
// tap interface/bridge if it is.
|
||||||
|
- isPhysical, err := isPhysicalIface(netInfo.Iface.Name)
|
||||||
|
+ isPhysical, err := isPhysicalIface(netInfo.Device)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
@@ -1164,49 +1179,54 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li
|
||||||
|
if isPhysical {
|
||||||
|
networkLogger().WithField("interface", netInfo.Iface.Name).Info("Physical network interface found")
|
||||||
|
endpoint, err = createPhysicalEndpoint(netInfo)
|
||||||
|
- } else {
|
||||||
|
- var socketPath string
|
||||||
|
+ return endpoint, err
|
||||||
|
+ }
|
||||||
|
|
||||||
|
- // Check if this is a dummy interface which has a vhost-user socket associated with it
|
||||||
|
- socketPath, err = vhostUserSocketPath(netInfo)
|
||||||
|
- if err != nil {
|
||||||
|
- return nil, err
|
||||||
|
- }
|
||||||
|
+ // Check if this is a dummy interface which has a vhost-user socket associated with it
|
||||||
|
+ socketPath, err := vhostUserSocketPath(netInfo)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
|
||||||
|
- if socketPath != "" {
|
||||||
|
- networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found")
|
||||||
|
- endpoint, err = createVhostUserEndpoint(netInfo, socketPath)
|
||||||
|
- } else if netInfo.Iface.Type == "macvlan" {
|
||||||
|
- networkLogger().Infof("macvlan interface found")
|
||||||
|
- endpoint, err = createBridgedMacvlanNetworkEndpoint(idx, netInfo.Iface.Name, model)
|
||||||
|
- } else if netInfo.Iface.Type == "macvtap" {
|
||||||
|
- networkLogger().Infof("macvtap interface found")
|
||||||
|
- endpoint, err = createMacvtapNetworkEndpoint(netInfo)
|
||||||
|
- } else if netInfo.Iface.Type == "tap" {
|
||||||
|
- networkLogger().Info("tap interface found")
|
||||||
|
- endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name)
|
||||||
|
- } else if netInfo.Iface.Type == "tuntap" {
|
||||||
|
- if link != nil {
|
||||||
|
- switch link.(*netlink.Tuntap).Mode {
|
||||||
|
- case 0:
|
||||||
|
- // mount /sys/class/net to get links
|
||||||
|
- return nil, fmt.Errorf("Network device mode not determined correctly. Mount sysfs in caller")
|
||||||
|
- case 1:
|
||||||
|
- return nil, fmt.Errorf("tun networking device not yet supported")
|
||||||
|
- case 2:
|
||||||
|
- networkLogger().Info("tuntap tap interface found")
|
||||||
|
- endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, model)
|
||||||
|
- default:
|
||||||
|
- return nil, fmt.Errorf("tuntap network %v mode unsupported", link.(*netlink.Tuntap).Mode)
|
||||||
|
- }
|
||||||
|
+ if socketPath != "" {
|
||||||
|
+ networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found")
|
||||||
|
+ endpoint, err = createVhostUserEndpoint(netInfo, socketPath)
|
||||||
|
+ return endpoint, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // We should create tap interface/bridge of other interface type.
|
||||||
|
+ networkLogger().Infof("%s interface found", netInfo.Iface.Type)
|
||||||
|
+ switch netInfo.Iface.Type {
|
||||||
|
+ case "macvlan":
|
||||||
|
+ networkLogger().Infof("macvlan interface found")
|
||||||
|
+ endpoint, err = createBridgedMacvlanNetworkEndpoint(idx, netInfo.Iface.Name, model)
|
||||||
|
+ case "macvtap":
|
||||||
|
+ networkLogger().Infof("macvtap interface found")
|
||||||
|
+ endpoint, err = createMacvtapNetworkEndpoint(netInfo)
|
||||||
|
+ case "tap":
|
||||||
|
+ networkLogger().Info("tap interface found")
|
||||||
|
+ endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Device)
|
||||||
|
+ case "tuntap":
|
||||||
|
+ if link != nil {
|
||||||
|
+ switch link.(*netlink.Tuntap).Mode {
|
||||||
|
+ case 0:
|
||||||
|
+ // mount /sys/class/net to get links
|
||||||
|
+ return nil, fmt.Errorf("Network device mode not determined correctly. Mount sysfs in caller")
|
||||||
|
+ case 1:
|
||||||
|
+ return nil, fmt.Errorf("tun networking device not yet supported")
|
||||||
|
+ case 2:
|
||||||
|
+ networkLogger().Info("tuntap tap interface found")
|
||||||
|
+ endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, model)
|
||||||
|
+ default:
|
||||||
|
+ return nil, fmt.Errorf("tuntap network %v mode unsupported", link.(*netlink.Tuntap).Mode)
|
||||||
|
}
|
||||||
|
- } else if netInfo.Iface.Type == "veth" {
|
||||||
|
- endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model)
|
||||||
|
- } else if netInfo.Iface.Type == "ipvlan" {
|
||||||
|
- endpoint, err = createIPVlanNetworkEndpoint(idx, netInfo.Iface.Name)
|
||||||
|
- } else {
|
||||||
|
- return nil, fmt.Errorf("Unsupported network interface: %s", netInfo.Iface.Type)
|
||||||
|
}
|
||||||
|
+ case "veth":
|
||||||
|
+ endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model)
|
||||||
|
+ case "ipvlan":
|
||||||
|
+ endpoint, err = createIPVlanNetworkEndpoint(idx, netInfo.Iface.Name)
|
||||||
|
+ default:
|
||||||
|
+ err = fmt.Errorf("Unsupported network interface, %s", netInfo.Iface.Type)
|
||||||
|
}
|
||||||
|
|
||||||
|
return endpoint, err
|
||||||
|
diff --git a/virtcontainers/pkg/types/types.go b/virtcontainers/pkg/types/types.go
|
||||||
|
index 0d4a9cfa..fcc63d84 100644
|
||||||
|
--- a/virtcontainers/pkg/types/types.go
|
||||||
|
+++ b/virtcontainers/pkg/types/types.go
|
||||||
|
@@ -14,21 +14,21 @@ type IPAddress struct {
|
||||||
|
|
||||||
|
// Interface describes a network interface.
|
||||||
|
type Interface struct {
|
||||||
|
- Device string
|
||||||
|
- Name string
|
||||||
|
- IPAddresses []*IPAddress
|
||||||
|
- Mtu uint64
|
||||||
|
- RawFlags uint32
|
||||||
|
- HwAddr string
|
||||||
|
+ Device string `json:"device,omitempty"`
|
||||||
|
+ Name string `json:"name,omitempty"`
|
||||||
|
+ IPAddresses []*IPAddress `json:"IPAddresses,omitempty"`
|
||||||
|
+ Mtu uint64 `json:"mtu,omitempty"`
|
||||||
|
+ RawFlags uint32 `json:"rawFlags,omitempty"`
|
||||||
|
+ HwAddr string `json:"hwAddr,omitempty"`
|
||||||
|
// pciAddr is the PCI address in the format "bridgeAddr/deviceAddr".
|
||||||
|
// Here, bridgeAddr is the address at which the bridge is attached on the root bus,
|
||||||
|
// while deviceAddr is the address at which the network device is attached on the bridge.
|
||||||
|
- PciAddr string
|
||||||
|
+ PciAddr string `json:"pciAddr,omitempty"`
|
||||||
|
// LinkType defines the type of interface described by this structure.
|
||||||
|
// The expected values are the one that are defined by the netlink
|
||||||
|
// library, regarding each type of link. Here is a non exhaustive
|
||||||
|
// list: "veth", "macvtap", "vlan", "macvlan", "tap", ...
|
||||||
|
- LinkType string
|
||||||
|
+ LinkType string `json:"linkType,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Route describes a network route.
|
||||||
|
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
|
||||||
|
index 7bae3278..bb83b1bb 100644
|
||||||
|
--- a/virtcontainers/qemu.go
|
||||||
|
+++ b/virtcontainers/qemu.go
|
||||||
|
@@ -1361,7 +1361,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
-func (q *qemu) hotAddNetDevice(name, hardAddr string, VMFds, VhostFds []*os.File) error {
|
||||||
|
+func (q *qemu) hotAddNetDevice(deviceName, name, hardAddr string, VMFds, VhostFds []*os.File) error {
|
||||||
|
var (
|
||||||
|
VMFdNames []string
|
||||||
|
VhostFdNames []string
|
||||||
|
@@ -1381,7 +1381,12 @@ func (q *qemu) hotAddNetDevice(name, hardAddr string, VMFds, VhostFds []*os.File
|
||||||
|
VhostFd.Close()
|
||||||
|
VhostFdNames = append(VhostFdNames, fdName)
|
||||||
|
}
|
||||||
|
- return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", name, VMFdNames, VhostFdNames)
|
||||||
|
+
|
||||||
|
+ if len(VMFdNames) != 0 || len(VhostFdNames) != 0 {
|
||||||
|
+ return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", name, VMFdNames, VhostFdNames)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return q.qmpMonitorCh.qmp.ExecuteNetdevAdd(q.qmpMonitorCh.ctx, "tap", name, deviceName, "no", "no", 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
|
||||||
|
@@ -1404,7 +1409,7 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
|
||||||
|
|
||||||
|
devID := "virtio-" + tap.ID
|
||||||
|
if op == addDevice {
|
||||||
|
- if err = q.hotAddNetDevice(tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds); err != nil {
|
||||||
|
+ if err = q.hotAddNetDevice(tap.TAPIface.Name, tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index ba704249..c8981a41 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -937,6 +937,7 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) {
|
||||||
|
}
|
||||||
|
|
||||||
|
return NetworkInfo{
|
||||||
|
+ Device: inf.Device,
|
||||||
|
Iface: NetlinkIface{
|
||||||
|
LinkAttrs: netlink.LinkAttrs{
|
||||||
|
Name: inf.Name,
|
||||||
|
@@ -950,7 +951,7 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) {
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddInterface adds new nic to the sandbox.
|
||||||
|
-func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (*vcTypes.Interface, error) {
|
||||||
|
+func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (grpcIf *vcTypes.Interface, err error) {
|
||||||
|
netInfo, err := s.generateNetInfo(inf)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
@@ -962,28 +963,41 @@ func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (*vcTypes.Interface, erro
|
||||||
|
}
|
||||||
|
|
||||||
|
endpoint.SetProperties(netInfo)
|
||||||
|
- if err := doNetNS(s.networkNS.NetNsPath, func(_ ns.NetNS) error {
|
||||||
|
+ if err = doNetNS(s.networkNS.NetNsPath, func(_ ns.NetNS) error {
|
||||||
|
s.Logger().WithField("endpoint-type", endpoint.Type()).Info("Hot attaching endpoint")
|
||||||
|
return endpoint.HotAttach(s.hypervisor)
|
||||||
|
}); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
+ defer func() {
|
||||||
|
+ if err != nil {
|
||||||
|
+ if errDetach := endpoint.HotDetach(s.hypervisor, s.networkNS.NetNsCreated, s.networkNS.NetNsPath); errDetach != nil {
|
||||||
|
+ s.Logger().WithField("endpoint-type", endpoint.Type()).Errorf("rollback hot attach endpoint failed")
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
// Update the sandbox storage
|
||||||
|
s.networkNS.Endpoints = append(s.networkNS.Endpoints, endpoint)
|
||||||
|
- if err := s.Save(); err != nil {
|
||||||
|
+ if err = s.Save(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add network for vm
|
||||||
|
inf.PciAddr = endpoint.PciAddr()
|
||||||
|
- return s.agent.updateInterface(inf)
|
||||||
|
+ grpcIf, err = s.agent.updateInterface(inf)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return
|
||||||
|
}
|
||||||
|
|
||||||
|
// RemoveInterface removes a nic of the sandbox.
|
||||||
|
func (s *Sandbox) RemoveInterface(inf *vcTypes.Interface) (*vcTypes.Interface, error) {
|
||||||
|
for i, endpoint := range s.networkNS.Endpoints {
|
||||||
|
- if endpoint.HardwareAddr() == inf.HwAddr {
|
||||||
|
+ if endpoint.HardwareAddr() == inf.HwAddr || endpoint.Name() == inf.Name {
|
||||||
|
s.Logger().WithField("endpoint-type", endpoint.Type()).Info("Hot detaching endpoint")
|
||||||
|
if err := endpoint.HotDetach(s.hypervisor, s.networkNS.NetNsCreated, s.networkNS.NetNsPath); err != nil {
|
||||||
|
return inf, err
|
||||||
|
diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go
|
||||||
|
index cb441b87..7d33d5a2 100644
|
||||||
|
--- a/virtcontainers/tap_endpoint.go
|
||||||
|
+++ b/virtcontainers/tap_endpoint.go
|
||||||
|
@@ -7,6 +7,7 @@ package virtcontainers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
+ "os"
|
||||||
|
|
||||||
|
"github.com/containernetworking/plugins/pkg/ns"
|
||||||
|
"github.com/vishvananda/netlink"
|
||||||
|
@@ -111,7 +112,7 @@ func (endpoint *TapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPat
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
-func createTapNetworkEndpoint(idx int, ifName string) (*TapEndpoint, error) {
|
||||||
|
+func createTapNetworkEndpoint(idx int, ifName string, tapIfName string) (*TapEndpoint, error) {
|
||||||
|
if idx < 0 {
|
||||||
|
return &TapEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx)
|
||||||
|
}
|
||||||
|
@@ -131,6 +132,10 @@ func createTapNetworkEndpoint(idx int, ifName string) (*TapEndpoint, error) {
|
||||||
|
endpoint.TapInterface.Name = ifName
|
||||||
|
}
|
||||||
|
|
||||||
|
+ if tapIfName != "" {
|
||||||
|
+ endpoint.TapInterface.TAPIface.Name = tapIfName
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
return endpoint, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -145,9 +150,19 @@ func tapNetwork(endpoint *TapEndpoint, numCPUs uint32, disableVhostNet bool) err
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Could not create TAP interface: %s", err)
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+ defer func() {
|
||||||
|
+ if err != nil {
|
||||||
|
+ if errDel := netHandle.LinkDel(tapLink); errDel != nil {
|
||||||
|
+ networkLogger().WithError(errDel).Error("tapNetwork fail to rollback del link")
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
endpoint.TapInterface.VMFds = fds
|
||||||
|
if !disableVhostNet {
|
||||||
|
- vhostFds, err := createVhostFds(int(numCPUs))
|
||||||
|
+ var vhostFds []*os.File
|
||||||
|
+ vhostFds, err = createVhostFds(int(numCPUs))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Could not setup vhost fds %s : %s", endpoint.TapInterface.Name, err)
|
||||||
|
}
|
||||||
|
@@ -161,10 +176,10 @@ func tapNetwork(endpoint *TapEndpoint, numCPUs uint32, disableVhostNet bool) err
|
||||||
|
// bridge created by the network plugin on the host actually expects
|
||||||
|
// to see traffic from this MAC address and not another one.
|
||||||
|
endpoint.TapInterface.TAPIface.HardAddr = linkAttrs.HardwareAddr.String()
|
||||||
|
- if err := netHandle.LinkSetMTU(tapLink, linkAttrs.MTU); err != nil {
|
||||||
|
+ if err = netHandle.LinkSetMTU(tapLink, linkAttrs.MTU); err != nil {
|
||||||
|
return fmt.Errorf("Could not set TAP MTU %d: %s", linkAttrs.MTU, err)
|
||||||
|
}
|
||||||
|
- if err := netHandle.LinkSetUp(tapLink); err != nil {
|
||||||
|
+ if err = netHandle.LinkSetUp(tapLink); err != nil {
|
||||||
|
return fmt.Errorf("Could not enable TAP %s: %s", endpoint.TapInterface.Name, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,260 @@
|
|||||||
|
From be8153f21c0b81d2b194075ecd654501bc708577 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Thu, 13 Aug 2020 18:54:49 +0800
|
||||||
|
Subject: [PATCH 25/50] network: keep list-ifaces result compatible with cni
|
||||||
|
|
||||||
|
reason: community list-ifaces command will return the all
|
||||||
|
interfaces info in the Kata VM, however we may just want
|
||||||
|
to get the interfaces that we hotplug, so just return the
|
||||||
|
hotplugged interfaces and convert the interface info to
|
||||||
|
be compatible with cni.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
cli/network.go | 29 ++++++++++++++++++++++-
|
||||||
|
virtcontainers/api.go | 4 +++-
|
||||||
|
virtcontainers/endpoint.go | 44 +++++++++++++++++++++++++++++++++++
|
||||||
|
virtcontainers/network.go | 27 +++++++++++++++++++++
|
||||||
|
virtcontainers/persist/api/network.go | 24 +++++++++++++++++++
|
||||||
|
virtcontainers/tap_endpoint.go | 9 +++++++
|
||||||
|
6 files changed, 135 insertions(+), 2 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/cli/network.go b/cli/network.go
|
||||||
|
index 7e7791f1..66955725 100644
|
||||||
|
--- a/cli/network.go
|
||||||
|
+++ b/cli/network.go
|
||||||
|
@@ -28,6 +28,13 @@ const (
|
||||||
|
|
||||||
|
const defaultLinkType = "tap"
|
||||||
|
|
||||||
|
+type compatInterface struct {
|
||||||
|
+ Name string `json:"name,omitempty"`
|
||||||
|
+ Mac string `json:"mac,omitempty"`
|
||||||
|
+ IP []string `json:"ip,omitempty"`
|
||||||
|
+ Mtu int `json:"mtu,omitempty"`
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
var kataNetworkCLICommand = cli.Command{
|
||||||
|
Name: "kata-network",
|
||||||
|
Usage: "manage interfaces and routes for container",
|
||||||
|
@@ -244,7 +251,8 @@ func networkListCommand(ctx context.Context, containerID string, opType networkT
|
||||||
|
kataLog.WithField("existing-interfaces", fmt.Sprintf("%+v", interfaces)).
|
||||||
|
WithError(err).Error("list interfaces failed")
|
||||||
|
}
|
||||||
|
- json.NewEncoder(file).Encode(interfaces)
|
||||||
|
+ compatInfs := convertCompatInterfaces(interfaces)
|
||||||
|
+ json.NewEncoder(file).Encode(compatInfs)
|
||||||
|
case routeType:
|
||||||
|
var routes []*vcTypes.Route
|
||||||
|
routes, err = vci.ListRoutes(ctx, sandboxID)
|
||||||
|
@@ -256,3 +264,22 @@ func networkListCommand(ctx context.Context, containerID string, opType networkT
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func convertCompatInterfaces(interfaces []*vcTypes.Interface) []compatInterface {
|
||||||
|
+ var infs []compatInterface
|
||||||
|
+ for _, i := range interfaces {
|
||||||
|
+ var addrs []string
|
||||||
|
+ for _, a := range i.IPAddresses {
|
||||||
|
+ addrs = append(addrs, fmt.Sprintf("%s/%s", a.Address, a.Mask))
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ infs = append(infs, compatInterface{
|
||||||
|
+ Name: i.Name,
|
||||||
|
+ Mac: i.HwAddr,
|
||||||
|
+ IP: addrs,
|
||||||
|
+ Mtu: int(i.Mtu),
|
||||||
|
+ })
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return infs
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
|
||||||
|
index 5e8c9c9e..eb5b4995 100644
|
||||||
|
--- a/virtcontainers/api.go
|
||||||
|
+++ b/virtcontainers/api.go
|
||||||
|
@@ -949,7 +949,9 @@ func ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTypes.Interface
|
||||||
|
}
|
||||||
|
defer s.releaseStatelessSandbox()
|
||||||
|
|
||||||
|
- return s.ListInterfaces()
|
||||||
|
+ // get interfaces info from persist.json file
|
||||||
|
+ // instead of by s.ListInterfaces()
|
||||||
|
+ return convertToCompatInterfaces(&s.networkNS.Endpoints), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpdateRoutes is the virtcontainers update routes entry point.
|
||||||
|
diff --git a/virtcontainers/endpoint.go b/virtcontainers/endpoint.go
|
||||||
|
index 01b5e77f..7efcf49c 100644
|
||||||
|
--- a/virtcontainers/endpoint.go
|
||||||
|
+++ b/virtcontainers/endpoint.go
|
||||||
|
@@ -132,6 +132,28 @@ func saveTapIf(tapif *TapInterface) *persistapi.TapInterface {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
+func saveTapEndpointProperties(networkInfo *NetworkInfo) *persistapi.NetworkProperties {
|
||||||
|
+ if networkInfo == nil {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return &persistapi.NetworkProperties{
|
||||||
|
+ Device: networkInfo.Device,
|
||||||
|
+ Iface: persistapi.NetlinkIface{
|
||||||
|
+ LinkAttrs: networkInfo.Iface.LinkAttrs,
|
||||||
|
+ Type: networkInfo.Iface.Type,
|
||||||
|
+ },
|
||||||
|
+ Addrs: networkInfo.Addrs,
|
||||||
|
+ Routes: networkInfo.Routes,
|
||||||
|
+ DNS: persistapi.DNSInfo{
|
||||||
|
+ Servers: networkInfo.DNS.Servers,
|
||||||
|
+ Domain: networkInfo.DNS.Domain,
|
||||||
|
+ Searches: networkInfo.DNS.Searches,
|
||||||
|
+ Options: networkInfo.DNS.Options,
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func loadTapIf(tapif *persistapi.TapInterface) *TapInterface {
|
||||||
|
if tapif == nil {
|
||||||
|
return nil
|
||||||
|
@@ -148,6 +170,28 @@ func loadTapIf(tapif *persistapi.TapInterface) *TapInterface {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
+func loadTapEndpointProperties(endpointProperties *persistapi.NetworkProperties) *NetworkInfo {
|
||||||
|
+ if endpointProperties == nil {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return &NetworkInfo{
|
||||||
|
+ Device: endpointProperties.Device,
|
||||||
|
+ Iface: NetlinkIface{
|
||||||
|
+ LinkAttrs: endpointProperties.Iface.LinkAttrs,
|
||||||
|
+ Type: endpointProperties.Iface.Type,
|
||||||
|
+ },
|
||||||
|
+ Addrs: endpointProperties.Addrs,
|
||||||
|
+ Routes: endpointProperties.Routes,
|
||||||
|
+ DNS: DNSInfo{
|
||||||
|
+ Servers: endpointProperties.DNS.Servers,
|
||||||
|
+ Domain: endpointProperties.DNS.Domain,
|
||||||
|
+ Searches: endpointProperties.DNS.Searches,
|
||||||
|
+ Options: endpointProperties.DNS.Options,
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func saveNetIfPair(pair *NetworkInterfacePair) *persistapi.NetworkInterfacePair {
|
||||||
|
if pair == nil {
|
||||||
|
return nil
|
||||||
|
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
|
||||||
|
index e909a822..bf7f9336 100644
|
||||||
|
--- a/virtcontainers/network.go
|
||||||
|
+++ b/virtcontainers/network.go
|
||||||
|
@@ -1340,3 +1340,30 @@ func (n *Network) Remove(ctx context.Context, ns *NetworkNamespace, hypervisor h
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+// convertCompatInterfaces convert Endpoint info to vcTypes.Interface
|
||||||
|
+func convertToCompatInterfaces(es *[]Endpoint) []*vcTypes.Interface {
|
||||||
|
+ var infs []*vcTypes.Interface
|
||||||
|
+ for _, e := range *es {
|
||||||
|
+ var addrs []*vcTypes.IPAddress
|
||||||
|
+ for _, a := range e.Properties().Addrs {
|
||||||
|
+ m, _ := a.Mask.Size()
|
||||||
|
+ addr := &vcTypes.IPAddress{
|
||||||
|
+ Address: fmt.Sprintf("%s", a.IP),
|
||||||
|
+ Mask: fmt.Sprintf("%d", m),
|
||||||
|
+ }
|
||||||
|
+ addrs = append(addrs, addr)
|
||||||
|
+ }
|
||||||
|
+ inf := &vcTypes.Interface{
|
||||||
|
+ LinkType: string(e.Type()),
|
||||||
|
+ Name: e.Name(),
|
||||||
|
+ Mtu: uint64(e.Properties().Iface.MTU),
|
||||||
|
+ HwAddr: e.HardwareAddr(),
|
||||||
|
+ IPAddresses: addrs,
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ infs = append(infs, inf)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return infs
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/persist/api/network.go b/virtcontainers/persist/api/network.go
|
||||||
|
index 69610c67..53c6de44 100644
|
||||||
|
--- a/virtcontainers/persist/api/network.go
|
||||||
|
+++ b/virtcontainers/persist/api/network.go
|
||||||
|
@@ -11,6 +11,27 @@ import (
|
||||||
|
)
|
||||||
|
|
||||||
|
// ============= sandbox level resources =============
|
||||||
|
+// DNSInfo describes the DNS setup related to a network interface.
|
||||||
|
+type DNSInfo struct {
|
||||||
|
+ Servers []string
|
||||||
|
+ Domain string
|
||||||
|
+ Searches []string
|
||||||
|
+ Options []string
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// NetlinkIface describes fully a network interface.
|
||||||
|
+type NetlinkIface struct {
|
||||||
|
+ netlink.LinkAttrs
|
||||||
|
+ Type string
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+type NetworkProperties struct {
|
||||||
|
+ Device string
|
||||||
|
+ Iface NetlinkIface
|
||||||
|
+ Addrs []netlink.Addr
|
||||||
|
+ Routes []netlink.Route
|
||||||
|
+ DNS DNSInfo
|
||||||
|
+}
|
||||||
|
|
||||||
|
type NetworkInterface struct {
|
||||||
|
Name string
|
||||||
|
@@ -91,6 +112,9 @@ type NetworkEndpoint struct {
|
||||||
|
Tap *TapEndpoint `json:",omitempty"`
|
||||||
|
IPVlan *IPVlanEndpoint `json:",omitempty"`
|
||||||
|
Tuntap *TuntapEndpoint `json:",omitempty"`
|
||||||
|
+
|
||||||
|
+ // store the endpoint properties info
|
||||||
|
+ EndPointProperties *NetworkProperties `json:",omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// NetworkInfo contains network information of sandbox
|
||||||
|
diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go
|
||||||
|
index 7d33d5a2..c897670e 100644
|
||||||
|
--- a/virtcontainers/tap_endpoint.go
|
||||||
|
+++ b/virtcontainers/tap_endpoint.go
|
||||||
|
@@ -206,12 +206,15 @@ func unTapNetwork(name string) error {
|
||||||
|
|
||||||
|
func (endpoint *TapEndpoint) save() persistapi.NetworkEndpoint {
|
||||||
|
tapif := saveTapIf(&endpoint.TapInterface)
|
||||||
|
+ // save tap endpoint network properties into persist storage
|
||||||
|
+ properties := saveTapEndpointProperties(&endpoint.EndpointProperties)
|
||||||
|
|
||||||
|
return persistapi.NetworkEndpoint{
|
||||||
|
Type: string(endpoint.Type()),
|
||||||
|
Tap: &persistapi.TapEndpoint{
|
||||||
|
TapInterface: *tapif,
|
||||||
|
},
|
||||||
|
+ EndPointProperties: properties,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func (endpoint *TapEndpoint) load(s persistapi.NetworkEndpoint) {
|
||||||
|
@@ -221,4 +224,10 @@ func (endpoint *TapEndpoint) load(s persistapi.NetworkEndpoint) {
|
||||||
|
tapif := loadTapIf(&s.Tap.TapInterface)
|
||||||
|
endpoint.TapInterface = *tapif
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+ if s.EndPointProperties != nil {
|
||||||
|
+ // restore tap endpoint network properties from persist storage
|
||||||
|
+ properties := loadTapEndpointProperties(s.EndPointProperties)
|
||||||
|
+ endpoint.EndpointProperties = *properties
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,285 @@
|
|||||||
|
From eeca1e47e9a6422d89d08275864f2c1b15e54941 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Fri, 14 Aug 2020 19:14:35 +0800
|
||||||
|
Subject: [PATCH 26/50] network: add enable_compat_old_cni config
|
||||||
|
|
||||||
|
reason: old version kata-network list-ifaces output result different
|
||||||
|
from the community version, inorder to compatible with the old version
|
||||||
|
list-ifaces command output format, add enable_compat_old_cni to control
|
||||||
|
the list-ifaces command output format.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
cli/config/configuration-qemu.toml.in | 4 ++++
|
||||||
|
cli/network.go | 17 +++++++++++++++--
|
||||||
|
cli/network_test.go | 6 ++++++
|
||||||
|
pkg/katautils/config.go | 2 ++
|
||||||
|
virtcontainers/api.go | 10 +++++++---
|
||||||
|
virtcontainers/interfaces.go | 1 +
|
||||||
|
virtcontainers/network.go | 11 ++++++-----
|
||||||
|
virtcontainers/persist.go | 18 ++++++++++--------
|
||||||
|
virtcontainers/persist/api/config.go | 9 +++++----
|
||||||
|
virtcontainers/pkg/oci/utils.go | 4 ++++
|
||||||
|
virtcontainers/pkg/vcmock/sandbox.go | 5 +++++
|
||||||
|
virtcontainers/sandbox.go | 5 +++++
|
||||||
|
12 files changed, 70 insertions(+), 22 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in
|
||||||
|
index b44e84d8..46f8b632 100644
|
||||||
|
--- a/cli/config/configuration-qemu.toml.in
|
||||||
|
+++ b/cli/config/configuration-qemu.toml.in
|
||||||
|
@@ -453,6 +453,10 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
|
||||||
|
# (default: false)
|
||||||
|
#disable_new_netns = true
|
||||||
|
|
||||||
|
+# If enabled, the kata-network will return the old interface format info to be compatible with
|
||||||
|
+# old version CNI plugin
|
||||||
|
+enable_compat_old_cni = true
|
||||||
|
+
|
||||||
|
# if enabled, the runtime will add all the kata processes inside one dedicated cgroup.
|
||||||
|
# The container cgroups in the host are not created, just one single cgroup per sandbox.
|
||||||
|
# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.
|
||||||
|
diff --git a/cli/network.go b/cli/network.go
|
||||||
|
index 66955725..a1a24425 100644
|
||||||
|
--- a/cli/network.go
|
||||||
|
+++ b/cli/network.go
|
||||||
|
@@ -251,8 +251,21 @@ func networkListCommand(ctx context.Context, containerID string, opType networkT
|
||||||
|
kataLog.WithField("existing-interfaces", fmt.Sprintf("%+v", interfaces)).
|
||||||
|
WithError(err).Error("list interfaces failed")
|
||||||
|
}
|
||||||
|
- compatInfs := convertCompatInterfaces(interfaces)
|
||||||
|
- json.NewEncoder(file).Encode(compatInfs)
|
||||||
|
+
|
||||||
|
+ sandbox, err := vci.FetchSandbox(ctx, sandboxID)
|
||||||
|
+ if err != nil {
|
||||||
|
+ kataLog.WithField("existing-interfaces", fmt.Sprintf("%+v", interfaces)).
|
||||||
|
+ WithError(err).Error("fetch sandbox failed")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // If sandbox network config need to be compatible with old CNI,
|
||||||
|
+ // convert the interface format to old version format.
|
||||||
|
+ if sandbox.IsCompatOldCNI() {
|
||||||
|
+ compatInfs := convertCompatInterfaces(interfaces)
|
||||||
|
+ json.NewEncoder(file).Encode(compatInfs)
|
||||||
|
+ } else {
|
||||||
|
+ json.NewEncoder(file).Encode(interfaces)
|
||||||
|
+ }
|
||||||
|
case routeType:
|
||||||
|
var routes []*vcTypes.Route
|
||||||
|
routes, err = vci.ListRoutes(ctx, sandboxID)
|
||||||
|
diff --git a/cli/network_test.go b/cli/network_test.go
|
||||||
|
index 4e3d943d..95fd2749 100644
|
||||||
|
--- a/cli/network_test.go
|
||||||
|
+++ b/cli/network_test.go
|
||||||
|
@@ -17,6 +17,7 @@ import (
|
||||||
|
|
||||||
|
vc "github.com/kata-containers/runtime/virtcontainers"
|
||||||
|
vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
@@ -59,6 +60,10 @@ func TestNetworkCliFunction(t *testing.T) {
|
||||||
|
return newSingleContainerStatus(testContainerID, state, map[string]string{}, &specs.Spec{}), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+ testingImpl.FetchSandboxFunc = func(ctx context.Context, id string) (vc.VCSandbox, error) {
|
||||||
|
+ return &vcmock.Sandbox{}, nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
defer func() {
|
||||||
|
testingImpl.AddInterfaceFunc = nil
|
||||||
|
testingImpl.RemoveInterfaceFunc = nil
|
||||||
|
@@ -66,6 +71,7 @@ func TestNetworkCliFunction(t *testing.T) {
|
||||||
|
testingImpl.UpdateRoutesFunc = nil
|
||||||
|
testingImpl.ListRoutesFunc = nil
|
||||||
|
testingImpl.StatusContainerFunc = nil
|
||||||
|
+ testingImpl.FetchSandboxFunc = nil
|
||||||
|
}()
|
||||||
|
|
||||||
|
set := flag.NewFlagSet("", 0)
|
||||||
|
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
||||||
|
index 9a99b9d4..94c916a0 100644
|
||||||
|
--- a/pkg/katautils/config.go
|
||||||
|
+++ b/pkg/katautils/config.go
|
||||||
|
@@ -141,6 +141,7 @@ type runtime struct {
|
||||||
|
Debug bool `toml:"enable_debug"`
|
||||||
|
Tracing bool `toml:"enable_tracing"`
|
||||||
|
DisableNewNetNs bool `toml:"disable_new_netns"`
|
||||||
|
+ EnableCompatOldCNI bool `toml:"enable_compat_old_cni"`
|
||||||
|
DisableGuestSeccomp bool `toml:"disable_guest_seccomp"`
|
||||||
|
SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"`
|
||||||
|
Experimental []string `toml:"experimental"`
|
||||||
|
@@ -1235,6 +1236,7 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool, debugFlag
|
||||||
|
|
||||||
|
config.SandboxCgroupOnly = tomlConf.Runtime.SandboxCgroupOnly
|
||||||
|
config.DisableNewNetNs = tomlConf.Runtime.DisableNewNetNs
|
||||||
|
+ config.EnableCompatOldCNI = tomlConf.Runtime.EnableCompatOldCNI
|
||||||
|
for _, f := range tomlConf.Runtime.Experimental {
|
||||||
|
feature := exp.Get(f)
|
||||||
|
if feature == nil {
|
||||||
|
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
|
||||||
|
index eb5b4995..fb044fe1 100644
|
||||||
|
--- a/virtcontainers/api.go
|
||||||
|
+++ b/virtcontainers/api.go
|
||||||
|
@@ -949,9 +949,13 @@ func ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTypes.Interface
|
||||||
|
}
|
||||||
|
defer s.releaseStatelessSandbox()
|
||||||
|
|
||||||
|
- // get interfaces info from persist.json file
|
||||||
|
- // instead of by s.ListInterfaces()
|
||||||
|
- return convertToCompatInterfaces(&s.networkNS.Endpoints), nil
|
||||||
|
+ // If enable_compat_old_cni is enabled, get interfaces info from
|
||||||
|
+ // persist.json file instead of by s.ListInterfaces()
|
||||||
|
+ if s.config.NetworkConfig.EnableCompatOldCNI {
|
||||||
|
+ return convertToCompatInterfaces(&s.networkNS.Endpoints), nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return s.ListInterfaces()
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpdateRoutes is the virtcontainers update routes entry point.
|
||||||
|
diff --git a/virtcontainers/interfaces.go b/virtcontainers/interfaces.go
|
||||||
|
index fa6b584e..499b386e 100644
|
||||||
|
--- a/virtcontainers/interfaces.go
|
||||||
|
+++ b/virtcontainers/interfaces.go
|
||||||
|
@@ -98,6 +98,7 @@ type VCSandbox interface {
|
||||||
|
ListInterfaces() ([]*vcTypes.Interface, error)
|
||||||
|
UpdateRoutes(routes []*vcTypes.Route) ([]*vcTypes.Route, error)
|
||||||
|
ListRoutes() ([]*vcTypes.Route, error)
|
||||||
|
+ IsCompatOldCNI() bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// VCContainer is the Container interface
|
||||||
|
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
|
||||||
|
index bf7f9336..db235cf6 100644
|
||||||
|
--- a/virtcontainers/network.go
|
||||||
|
+++ b/virtcontainers/network.go
|
||||||
|
@@ -155,11 +155,12 @@ type NetworkInterfacePair struct {
|
||||||
|
|
||||||
|
// NetworkConfig is the network configuration related to a network.
|
||||||
|
type NetworkConfig struct {
|
||||||
|
- NetNSPath string
|
||||||
|
- NetNsCreated bool
|
||||||
|
- DisableNewNetNs bool
|
||||||
|
- NetmonConfig NetmonConfig
|
||||||
|
- InterworkingModel NetInterworkingModel
|
||||||
|
+ NetNSPath string
|
||||||
|
+ NetNsCreated bool
|
||||||
|
+ DisableNewNetNs bool
|
||||||
|
+ EnableCompatOldCNI bool
|
||||||
|
+ NetmonConfig NetmonConfig
|
||||||
|
+ InterworkingModel NetInterworkingModel
|
||||||
|
}
|
||||||
|
|
||||||
|
func networkLogger() *logrus.Entry {
|
||||||
|
diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go
|
||||||
|
index 6bd09a0b..fe00bf9a 100644
|
||||||
|
--- a/virtcontainers/persist.go
|
||||||
|
+++ b/virtcontainers/persist.go
|
||||||
|
@@ -187,10 +187,11 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
|
||||||
|
},
|
||||||
|
ShimType: string(sconfig.ShimType),
|
||||||
|
NetworkConfig: persistapi.NetworkConfig{
|
||||||
|
- NetNSPath: sconfig.NetworkConfig.NetNSPath,
|
||||||
|
- NetNsCreated: sconfig.NetworkConfig.NetNsCreated,
|
||||||
|
- DisableNewNetNs: sconfig.NetworkConfig.DisableNewNetNs,
|
||||||
|
- InterworkingModel: int(sconfig.NetworkConfig.InterworkingModel),
|
||||||
|
+ NetNSPath: sconfig.NetworkConfig.NetNSPath,
|
||||||
|
+ NetNsCreated: sconfig.NetworkConfig.NetNsCreated,
|
||||||
|
+ DisableNewNetNs: sconfig.NetworkConfig.DisableNewNetNs,
|
||||||
|
+ EnableCompatOldCNI: sconfig.NetworkConfig.EnableCompatOldCNI,
|
||||||
|
+ InterworkingModel: int(sconfig.NetworkConfig.InterworkingModel),
|
||||||
|
},
|
||||||
|
|
||||||
|
ShmSize: sconfig.ShmSize,
|
||||||
|
@@ -477,10 +478,11 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
|
||||||
|
},
|
||||||
|
ShimType: ShimType(savedConf.ShimType),
|
||||||
|
NetworkConfig: NetworkConfig{
|
||||||
|
- NetNSPath: savedConf.NetworkConfig.NetNSPath,
|
||||||
|
- NetNsCreated: savedConf.NetworkConfig.NetNsCreated,
|
||||||
|
- DisableNewNetNs: savedConf.NetworkConfig.DisableNewNetNs,
|
||||||
|
- InterworkingModel: NetInterworkingModel(savedConf.NetworkConfig.InterworkingModel),
|
||||||
|
+ NetNSPath: savedConf.NetworkConfig.NetNSPath,
|
||||||
|
+ NetNsCreated: savedConf.NetworkConfig.NetNsCreated,
|
||||||
|
+ DisableNewNetNs: savedConf.NetworkConfig.DisableNewNetNs,
|
||||||
|
+ EnableCompatOldCNI: savedConf.NetworkConfig.EnableCompatOldCNI,
|
||||||
|
+ InterworkingModel: NetInterworkingModel(savedConf.NetworkConfig.InterworkingModel),
|
||||||
|
},
|
||||||
|
|
||||||
|
ShmSize: savedConf.ShmSize,
|
||||||
|
diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go
|
||||||
|
index cfbee849..3a2df32b 100644
|
||||||
|
--- a/virtcontainers/persist/api/config.go
|
||||||
|
+++ b/virtcontainers/persist/api/config.go
|
||||||
|
@@ -210,10 +210,11 @@ type ShimConfig struct {
|
||||||
|
|
||||||
|
// NetworkConfig is the network configuration related to a network.
|
||||||
|
type NetworkConfig struct {
|
||||||
|
- NetNSPath string
|
||||||
|
- NetNsCreated bool
|
||||||
|
- DisableNewNetNs bool
|
||||||
|
- InterworkingModel int
|
||||||
|
+ NetNSPath string
|
||||||
|
+ NetNsCreated bool
|
||||||
|
+ DisableNewNetNs bool
|
||||||
|
+ EnableCompatOldCNI bool
|
||||||
|
+ InterworkingModel int
|
||||||
|
}
|
||||||
|
|
||||||
|
type ContainerConfig struct {
|
||||||
|
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
||||||
|
index 36c730b7..948bd3cb 100644
|
||||||
|
--- a/virtcontainers/pkg/oci/utils.go
|
||||||
|
+++ b/virtcontainers/pkg/oci/utils.go
|
||||||
|
@@ -130,6 +130,9 @@ type RuntimeConfig struct {
|
||||||
|
//Determines if create a netns for hypervisor process
|
||||||
|
DisableNewNetNs bool
|
||||||
|
|
||||||
|
+ // Determines if compatible with old CNI plugin
|
||||||
|
+ EnableCompatOldCNI bool
|
||||||
|
+
|
||||||
|
//Determines kata processes are managed only in sandbox cgroup
|
||||||
|
SandboxCgroupOnly bool
|
||||||
|
|
||||||
|
@@ -275,6 +278,7 @@ func networkConfig(ocispec specs.Spec, config RuntimeConfig) (vc.NetworkConfig,
|
||||||
|
}
|
||||||
|
netConf.InterworkingModel = config.InterNetworkModel
|
||||||
|
netConf.DisableNewNetNs = config.DisableNewNetNs
|
||||||
|
+ netConf.EnableCompatOldCNI = config.EnableCompatOldCNI
|
||||||
|
|
||||||
|
netConf.NetmonConfig = vc.NetmonConfig{
|
||||||
|
Path: config.NetmonConfig.Path,
|
||||||
|
diff --git a/virtcontainers/pkg/vcmock/sandbox.go b/virtcontainers/pkg/vcmock/sandbox.go
|
||||||
|
index 677457ef..11b83ccd 100644
|
||||||
|
--- a/virtcontainers/pkg/vcmock/sandbox.go
|
||||||
|
+++ b/virtcontainers/pkg/vcmock/sandbox.go
|
||||||
|
@@ -212,3 +212,8 @@ func (s *Sandbox) UpdateRoutes(routes []*vcTypes.Route) ([]*vcTypes.Route, error
|
||||||
|
func (s *Sandbox) ListRoutes() ([]*vcTypes.Route, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+// IsCompatOldCNI return the whether enable compatible with old CNI
|
||||||
|
+func (s *Sandbox) IsCompatOldCNI() bool {
|
||||||
|
+ return false
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index c8981a41..6a643a12 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -2379,6 +2379,11 @@ func (s *Sandbox) setContainersState(state types.StateString) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+// IsCompatOldCNI return the whether enable compatible with old CNI
|
||||||
|
+func (s *Sandbox) IsCompatOldCNI() bool {
|
||||||
|
+ return s.config.NetworkConfig.EnableCompatOldCNI
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// updateStaticSandboxResources update sandbox's cpu and memory resource passed by
|
||||||
|
// sandbox_cpu and sandbox_mem annotations
|
||||||
|
func updateStaticSandboxResources(sandboxConfig *SandboxConfig) error {
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,182 @@
|
|||||||
|
From ec15337fc816767ca0e8183576405499080b9b1e Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Sun, 16 Aug 2020 16:41:18 +0800
|
||||||
|
Subject: [PATCH 27/50] network: add more strict check for input network
|
||||||
|
interface
|
||||||
|
|
||||||
|
reason: kata-network add-iface command will receive the network
|
||||||
|
interface info from untrust user, so we need to add more strict
|
||||||
|
check for input network interface info.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/network.go | 117 ++++++++++++++++++++++++++++++++++++++++++++++
|
||||||
|
virtcontainers/sandbox.go | 11 +++++
|
||||||
|
2 files changed, 128 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
|
||||||
|
index db235cf6..a1676ccd 100644
|
||||||
|
--- a/virtcontainers/network.go
|
||||||
|
+++ b/virtcontainers/network.go
|
||||||
|
@@ -13,8 +13,10 @@ import (
|
||||||
|
"math/rand"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
+ "regexp"
|
||||||
|
"runtime"
|
||||||
|
"sort"
|
||||||
|
+ "strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/containernetworking/plugins/pkg/ns"
|
||||||
|
@@ -55,6 +57,17 @@ const (
|
||||||
|
NetXConnectInvalidModel
|
||||||
|
)
|
||||||
|
|
||||||
|
+const (
|
||||||
|
+ maxIPAddrLen = 18
|
||||||
|
+ maxInterfaceLen = 15
|
||||||
|
+ minMTUVal = 46
|
||||||
|
+ maxMTUVal = 9600
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+var (
|
||||||
|
+ regInfName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_\-.]*$`)
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
//IsValid checks if a model is valid
|
||||||
|
func (n NetInterworkingModel) IsValid() bool {
|
||||||
|
return 0 <= int(n) && int(n) < int(NetXConnectInvalidModel)
|
||||||
|
@@ -1368,3 +1381,107 @@ func convertToCompatInterfaces(es *[]Endpoint) []*vcTypes.Interface {
|
||||||
|
|
||||||
|
return infs
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+// verifyInterfaceName verifies the interface name valid or not
|
||||||
|
+func verifyInterfaceName(name string) error {
|
||||||
|
+ // verify `Name` before `Tapname` because of the strict rules
|
||||||
|
+ name = strings.TrimSpace(name)
|
||||||
|
+ if len(name) > maxInterfaceLen {
|
||||||
|
+ return fmt.Errorf("interface name can't be longer than 15")
|
||||||
|
+ } else if len(name) == 0 || name == "\n" {
|
||||||
|
+ return fmt.Errorf("interface name can't be empty")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // QMP rules of `Name`
|
||||||
|
+ chk := regInfName.FindAllString(name, -1)
|
||||||
|
+ if chk == nil {
|
||||||
|
+ return fmt.Errorf("invalid input of interface name, please check the rules")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// verifyIPAndMask verifies CIDR notation IP address and mask like "192.0.2.0/24"
|
||||||
|
+func verifyIPAndMask(ip string) (nlIpMask *net.IPNet, err error) {
|
||||||
|
+ ip = strings.TrimSpace(ip)
|
||||||
|
+
|
||||||
|
+ if len(ip) > maxIPAddrLen {
|
||||||
|
+ return nil, fmt.Errorf("the length of IP address is too long, max ip len :%d", maxIPAddrLen)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if nlIpMask, err = netlink.ParseIPNet(ip); err != nil {
|
||||||
|
+ return nil, fmt.Errorf("invalid input of IP : %v", err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nlIpMask, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// verifyMac verifies MAC address
|
||||||
|
+func verifyMac(mac string) error {
|
||||||
|
+ mac = strings.TrimSpace(mac)
|
||||||
|
+ if _, err := net.ParseMAC(mac); err != nil {
|
||||||
|
+ return fmt.Errorf("invalid input of Mac : %v", err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// verifyMtu verifies MTU value of interface
|
||||||
|
+func verifyMtu(mtu uint64) error {
|
||||||
|
+ if mtu < minMTUVal || mtu > maxMTUVal {
|
||||||
|
+ return fmt.Errorf("invalid input of MTU : %v", mtu)
|
||||||
|
+ }
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// verifyIP verifies the IP address
|
||||||
|
+func verifyIP(ip string) (*net.IP, error) {
|
||||||
|
+ ip = strings.TrimSpace(ip)
|
||||||
|
+
|
||||||
|
+ if len(ip) > maxIPAddrLen {
|
||||||
|
+ return nil, fmt.Errorf("the length of IP address is too long, max ip len :%d", maxIPAddrLen)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ var netIP net.IP
|
||||||
|
+ if netIP = net.ParseIP(ip); netIP == nil {
|
||||||
|
+ return nil, fmt.Errorf("invalid IP: %s", ip)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return &netIP, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// validInterface check the input interface valid or not
|
||||||
|
+func validInterface(inf *vcTypes.Interface, enableCompatOldCNI bool) error {
|
||||||
|
+ if enableCompatOldCNI && verifyInterfaceName(inf.Device) != nil {
|
||||||
|
+ return fmt.Errorf("device name should not be empty when enable_compat_old_cni config enabled")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if inf.Name == "" || inf.Mtu == 0 || inf.HwAddr == "" {
|
||||||
|
+ return fmt.Errorf("name/mtu/hwaddr of interface must be specified")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := verifyInterfaceName(inf.Name); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := verifyMac(inf.HwAddr); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := verifyMtu(inf.Mtu); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // Currently, only one IP address can be passed, which reduces the test entry and fault injection.
|
||||||
|
+ if len(inf.IPAddresses) > 0 {
|
||||||
|
+ if len(inf.IPAddresses) != 1 {
|
||||||
|
+ return fmt.Errorf("only one IP address is supported currently")
|
||||||
|
+ }
|
||||||
|
+ _, err := verifyIP(inf.IPAddresses[0].Address)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index 6a643a12..f6826812 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -952,6 +952,17 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) {
|
||||||
|
|
||||||
|
// AddInterface adds new nic to the sandbox.
|
||||||
|
func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (grpcIf *vcTypes.Interface, err error) {
|
||||||
|
+ err = validInterface(inf, s.config.NetworkConfig.EnableCompatOldCNI)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ for _, ep := range s.networkNS.Endpoints {
|
||||||
|
+ if ep.Name() == inf.Name {
|
||||||
|
+ return nil, fmt.Errorf("interface %s is already exist", inf.Name)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
netInfo, err := s.generateNetInfo(inf)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,192 @@
|
|||||||
|
From 51a7270987557ab12ea735fc9781725d1ce1b0a6 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Mon, 17 Aug 2020 15:36:32 +0800
|
||||||
|
Subject: [PATCH 29/50] network: add kata-network del-route subcommand
|
||||||
|
|
||||||
|
reason: add kata-network del-route subcommand to delete the
|
||||||
|
specified route in the Kata VM, del-route is more efficient
|
||||||
|
than the update-routes subcommand.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
cli/network.go | 24 ++++++++++
|
||||||
|
virtcontainers/network.go | 115 ++++++++++++++++++++++++++++++++++++++++++++++
|
||||||
|
2 files changed, 139 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/cli/network.go b/cli/network.go
|
||||||
|
index 2265f54b..046d0ee9 100644
|
||||||
|
--- a/cli/network.go
|
||||||
|
+++ b/cli/network.go
|
||||||
|
@@ -45,6 +45,7 @@ var kataNetworkCLICommand = cli.Command{
|
||||||
|
updateRoutesCommand,
|
||||||
|
listRoutesCommand,
|
||||||
|
addRoutesCommand,
|
||||||
|
+ deleteRoutesCommand,
|
||||||
|
},
|
||||||
|
Action: func(context *cli.Context) error {
|
||||||
|
return cli.ShowSubcommandHelp(context)
|
||||||
|
@@ -155,6 +156,29 @@ var addRoutesCommand = cli.Command{
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
+var deleteRoutesCommand = cli.Command{
|
||||||
|
+ Name: "del-route",
|
||||||
|
+ Usage: "delete one route for a container",
|
||||||
|
+ ArgsUsage: `del-route <container-id> file or - for stdin
|
||||||
|
+ file or stdin for example:
|
||||||
|
+ {
|
||||||
|
+ "dest":"<[<ip>[/mask] | "default" ]>",
|
||||||
|
+ "gateway":"[ip]",
|
||||||
|
+ "device":"[tap-name]",
|
||||||
|
+ }
|
||||||
|
+ Only destination is required and others are optional,
|
||||||
|
+ if device is empty, means search every device`,
|
||||||
|
+ Flags: []cli.Flag{},
|
||||||
|
+ Action: func(context *cli.Context) error {
|
||||||
|
+ ctx, err := cliContextToContext(context)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return networkModifyCommand(ctx, context.Args().First(), context.Args().Get(1), routeType, vcTypes.NetworkOpRemove)
|
||||||
|
+ },
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
var listRoutesCommand = cli.Command{
|
||||||
|
Name: "list-routes",
|
||||||
|
Usage: "list network routes in a container",
|
||||||
|
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
|
||||||
|
index f3757f84..c7066a11 100644
|
||||||
|
--- a/virtcontainers/network.go
|
||||||
|
+++ b/virtcontainers/network.go
|
||||||
|
@@ -1631,6 +1631,110 @@ func addOneRoute(ns *NetworkNamespace, route *vcTypes.Route) (added *vcTypes.Rou
|
||||||
|
return added, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+func generateRmRoute(route *vcTypes.Route) (r *netlink.Route, err error) {
|
||||||
|
+ // destination is required and others are optional
|
||||||
|
+ if route.Dest == "" {
|
||||||
|
+ return nil, fmt.Errorf("destination must be specified when remove route.")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if route.Device != "" {
|
||||||
|
+ err = verifyInterfaceName(route.Device)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ r = &netlink.Route{}
|
||||||
|
+ if route.Dest != "default" {
|
||||||
|
+ nlIpNet, err := verifyRouteDest(&route.Dest)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+ r.Dst = nlIpNet
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if route.Gateway != "" {
|
||||||
|
+ nIP, err := verifyIP(route.Gateway)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+ r.Gw = *nIP
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if route.Source != "" {
|
||||||
|
+ nIP, err := verifyIP(route.Source)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+ r.Src = *nIP
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ r.Scope = netlink.Scope(route.Scope)
|
||||||
|
+
|
||||||
|
+ return r, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// parseToGrpcRoute convert the netlink.Route to vcTypes.Route and deleted route.Dest will
|
||||||
|
+// be added a prefix "-"
|
||||||
|
+func parseToGrpcRoute(device string, route *netlink.Route, add bool) (r *vcTypes.Route) {
|
||||||
|
+ r = &vcTypes.Route{
|
||||||
|
+ Device: device,
|
||||||
|
+ }
|
||||||
|
+ if route.Dst != nil && route.Dst.String() != "<nil>" {
|
||||||
|
+ r.Dest = route.Dst.String()
|
||||||
|
+ if add == false {
|
||||||
|
+ r.Dest = "-" + route.Dst.String()
|
||||||
|
+ }
|
||||||
|
+ } else if route.Dst == nil {
|
||||||
|
+ r.Dest = "default"
|
||||||
|
+ if add == false {
|
||||||
|
+ r.Dest = "-default"
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ if route.Gw != nil && route.Gw.String() != "<nil>" {
|
||||||
|
+ r.Gateway = route.Gw.String()
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return r
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func removeRoutes(ns *NetworkNamespace, route *vcTypes.Route) (removed []*vcTypes.Route, err error) {
|
||||||
|
+ del, err := generateRmRoute(route)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // remove the lo device related routes
|
||||||
|
+ if route.Device == localHostDeviceName {
|
||||||
|
+ removed = append(removed, parseToGrpcRoute(localHostDeviceName, del, false))
|
||||||
|
+
|
||||||
|
+ return removed, nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ for _, ep := range ns.Endpoints {
|
||||||
|
+ // if device is empty, means search every device
|
||||||
|
+ if route.Device != "" && ep.Name() != route.Device {
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ netInfo := ep.Properties()
|
||||||
|
+ for i, exist := range ep.Properties().Routes {
|
||||||
|
+ if isSameRoute(&exist, del, true) {
|
||||||
|
+ // need remove
|
||||||
|
+ netInfo.Routes = append(netInfo.Routes[:i], netInfo.Routes[i+1:]...)
|
||||||
|
+ ep.SetProperties(netInfo)
|
||||||
|
+ dev := route.Device
|
||||||
|
+ if route.Device == "" {
|
||||||
|
+ dev = netInfo.Iface.Name
|
||||||
|
+ }
|
||||||
|
+ removed = append(removed, parseToGrpcRoute(dev, del, false))
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return removed, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func updateRoute(ns *NetworkNamespace, route *vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) {
|
||||||
|
var updRoutes []*vcTypes.Route
|
||||||
|
|
||||||
|
@@ -1645,5 +1749,16 @@ func updateRoute(ns *NetworkNamespace, route *vcTypes.Route, op vcTypes.NetworkO
|
||||||
|
updRoutes = append(updRoutes, added)
|
||||||
|
}
|
||||||
|
|
||||||
|
+ if op == vcTypes.NetworkOpRemove {
|
||||||
|
+ removed, err := removeRoutes(ns, route)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+ if len(removed) <= 0 {
|
||||||
|
+ return nil, fmt.Errorf("route of device %s with destination %s is not found", route.Device, route.Dest)
|
||||||
|
+ }
|
||||||
|
+ updRoutes = removed
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
return updRoutes, nil
|
||||||
|
}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,104 @@
|
|||||||
|
From 9cf769178b8f73c7fd624895589e918d6c7b0645 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Mon, 17 Aug 2020 16:29:17 +0800
|
||||||
|
Subject: [PATCH 30/50] network: kata-network list-routes support display
|
||||||
|
compatible format
|
||||||
|
|
||||||
|
reason: kata-network list-routes subcommand support display compatible
|
||||||
|
format when enable_compat_old_cni config is enabled.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/api.go | 13 ++++++++++++-
|
||||||
|
virtcontainers/network.go | 29 +++++++++++++++++++++++++++++
|
||||||
|
virtcontainers/pkg/types/types.go | 10 +++++-----
|
||||||
|
3 files changed, 46 insertions(+), 6 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
|
||||||
|
index 2331eb94..a4bf41bb 100644
|
||||||
|
--- a/virtcontainers/api.go
|
||||||
|
+++ b/virtcontainers/api.go
|
||||||
|
@@ -1003,7 +1003,18 @@ func ListRoutes(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error)
|
||||||
|
}
|
||||||
|
defer s.releaseStatelessSandbox()
|
||||||
|
|
||||||
|
- return s.ListRoutes()
|
||||||
|
+ routes, err := s.ListRoutes()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // If enable_compat_old_cni is enabled, convert routes info to
|
||||||
|
+ // compatible display format
|
||||||
|
+ if s.config.NetworkConfig.EnableCompatOldCNI {
|
||||||
|
+ return convertToDisplayRoutes(&routes), nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return routes, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// CleanupContaienr is used by shimv2 to stop and delete a container exclusively, once there is no container
|
||||||
|
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
|
||||||
|
index c7066a11..488bd00c 100644
|
||||||
|
--- a/virtcontainers/network.go
|
||||||
|
+++ b/virtcontainers/network.go
|
||||||
|
@@ -1384,6 +1384,35 @@ func convertToCompatInterfaces(es *[]Endpoint) []*vcTypes.Interface {
|
||||||
|
return infs
|
||||||
|
}
|
||||||
|
|
||||||
|
+// convertToDisplayRoutes convert the default route format to more simple
|
||||||
|
+// route display format, it is called when enable_compat_old_cni config
|
||||||
|
+// is enabled.
|
||||||
|
+func convertToDisplayRoutes(routes *[]*vcTypes.Route) []*vcTypes.Route {
|
||||||
|
+ var displayRoutes []*vcTypes.Route
|
||||||
|
+ if routes == nil {
|
||||||
|
+ return displayRoutes
|
||||||
|
+ }
|
||||||
|
+ for _, r := range *routes {
|
||||||
|
+ // we don't support IPV6 temporarily, so we need to filter ":" which
|
||||||
|
+ // is the characteristics of IPV6 for those default routes
|
||||||
|
+ if strings.Contains(r.Dest, ":") {
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ defaultDest := r.Dest
|
||||||
|
+ if r.Dest == "" {
|
||||||
|
+ defaultDest = "default"
|
||||||
|
+ }
|
||||||
|
+ displayRoutes = append(displayRoutes, &vcTypes.Route{
|
||||||
|
+ Dest: defaultDest,
|
||||||
|
+ Gateway: r.Gateway,
|
||||||
|
+ Device: r.Device,
|
||||||
|
+ })
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return displayRoutes
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// verifyInterfaceName verifies the interface name valid or not
|
||||||
|
func verifyInterfaceName(name string) error {
|
||||||
|
// verify `Name` before `Tapname` because of the strict rules
|
||||||
|
diff --git a/virtcontainers/pkg/types/types.go b/virtcontainers/pkg/types/types.go
|
||||||
|
index 71fe7fbb..b41b0c75 100644
|
||||||
|
--- a/virtcontainers/pkg/types/types.go
|
||||||
|
+++ b/virtcontainers/pkg/types/types.go
|
||||||
|
@@ -33,11 +33,11 @@ type Interface struct {
|
||||||
|
|
||||||
|
// Route describes a network route.
|
||||||
|
type Route struct {
|
||||||
|
- Dest string
|
||||||
|
- Gateway string
|
||||||
|
- Device string
|
||||||
|
- Source string
|
||||||
|
- Scope uint32
|
||||||
|
+ Dest string `json:"dest,omitempty"`
|
||||||
|
+ Gateway string `json:"gateway,omitempty"`
|
||||||
|
+ Device string `json:"device,omitempty"`
|
||||||
|
+ Source string `json:"source,omitempty"`
|
||||||
|
+ Scope uint32 `json:"scope,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
//NetworkOp describes network operation
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,41 @@
|
|||||||
|
From 326e90f97cf5ace73dff95257f7b4faa43c56f99 Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Mon, 17 Aug 2020 16:11:35 +0800
|
||||||
|
Subject: [PATCH 31/50] device_mangaer: check VFIO when create device
|
||||||
|
|
||||||
|
reason: check VFIO when create device, block device can be
|
||||||
|
reused and VFIO device can not
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/device/manager/manager.go | 7 ++++++-
|
||||||
|
1 file changed, 6 insertions(+), 1 deletion(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/device/manager/manager.go b/virtcontainers/device/manager/manager.go
|
||||||
|
index 6a2b665a..d24a29ab 100644
|
||||||
|
--- a/virtcontainers/device/manager/manager.go
|
||||||
|
+++ b/virtcontainers/device/manager/manager.go
|
||||||
|
@@ -9,6 +9,7 @@ package manager
|
||||||
|
import (
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
|
+ "fmt"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
@@ -115,7 +116,11 @@ func (dm *deviceManager) createDevice(devInfo config.DeviceInfo) (dev api.Device
|
||||||
|
}()
|
||||||
|
|
||||||
|
if existingDev := dm.findDeviceByMajorMinor(devInfo.Major, devInfo.Minor); existingDev != nil {
|
||||||
|
- return existingDev, nil
|
||||||
|
+ if isVFIO(devInfo.HostPath) {
|
||||||
|
+ return nil, fmt.Errorf("device %s is replicated in the same Pod!", devInfo.ContainerPath)
|
||||||
|
+ } else {
|
||||||
|
+ return existingDev, nil
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
|
||||||
|
// device ID must be generated by manager instead of device itself
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,46 @@
|
|||||||
|
From 77711b531867be899df5d2c59a525ea1b7f0e6ac Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Mon, 17 Aug 2020 17:14:35 +0800
|
||||||
|
Subject: [PATCH 32/50] network: add more detail usage for update-routes
|
||||||
|
subcommand
|
||||||
|
|
||||||
|
reason: add more detail usage for update-routes subcommand
|
||||||
|
to explain how to use update-routes subcommand
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
cli/network.go | 18 ++++++++++++++----
|
||||||
|
1 file changed, 14 insertions(+), 4 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/cli/network.go b/cli/network.go
|
||||||
|
index 046d0ee9..3dd0971e 100644
|
||||||
|
--- a/cli/network.go
|
||||||
|
+++ b/cli/network.go
|
||||||
|
@@ -120,10 +120,20 @@ var listIfacesCommand = cli.Command{
|
||||||
|
}
|
||||||
|
|
||||||
|
var updateRoutesCommand = cli.Command{
|
||||||
|
- Name: "update-routes",
|
||||||
|
- Usage: "update routes of a container",
|
||||||
|
- ArgsUsage: `update-routes <container-id> file or - for stdin`,
|
||||||
|
- Flags: []cli.Flag{},
|
||||||
|
+ Name: "update-routes",
|
||||||
|
+ Usage: "update routes of a container",
|
||||||
|
+ ArgsUsage: `update-routes <container-id> file or - for stdin
|
||||||
|
+ file or stdin for example:
|
||||||
|
+ [
|
||||||
|
+ {
|
||||||
|
+ "dest":"<[<ip>[/mask] | "default" ]>",
|
||||||
|
+ "gateway":"[ip]",
|
||||||
|
+ "device":"[tap-name]",
|
||||||
|
+ "source": "[source]",
|
||||||
|
+ "scope":[scope]
|
||||||
|
+ }
|
||||||
|
+ ]`,
|
||||||
|
+ Flags: []cli.Flag{},
|
||||||
|
Action: func(context *cli.Context) error {
|
||||||
|
ctx, err := cliContextToContext(context)
|
||||||
|
if err != nil {
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,80 @@
|
|||||||
|
From 7ab7ff54efa3925a8d372f7830d31b87f8d01ea8 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Mon, 17 Aug 2020 17:39:18 +0800
|
||||||
|
Subject: [PATCH 33/50] network: do not delete the exist tap device in the host
|
||||||
|
|
||||||
|
reason: If hotplug a exist tap device into Kata VM, kata-runtime
|
||||||
|
should not delete this exist tap device, because this tap device
|
||||||
|
is controlled by other network components.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/tap_endpoint.go | 16 +++++++++++++---
|
||||||
|
virtcontainers/veth_endpoint.go | 3 +++
|
||||||
|
2 files changed, 16 insertions(+), 3 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go
|
||||||
|
index c897670e..2cf70dce 100644
|
||||||
|
--- a/virtcontainers/tap_endpoint.go
|
||||||
|
+++ b/virtcontainers/tap_endpoint.go
|
||||||
|
@@ -77,7 +77,7 @@ func (endpoint *TapEndpoint) Detach(netNsCreated bool, netNsPath string) error {
|
||||||
|
|
||||||
|
networkLogger().WithField("endpoint-type", TapEndpointType).Info("Detaching endpoint")
|
||||||
|
return doNetNS(netNsPath, func(_ ns.NetNS) error {
|
||||||
|
- return unTapNetwork(endpoint.TapInterface.TAPIface.Name)
|
||||||
|
+ return unTapNetwork(endpoint)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -91,6 +91,9 @@ func (endpoint *TapEndpoint) HotAttach(h hypervisor) error {
|
||||||
|
|
||||||
|
if _, err := h.hotplugAddDevice(endpoint, netDev); err != nil {
|
||||||
|
networkLogger().WithError(err).Error("Error attach tap ep")
|
||||||
|
+ if errUnTap := unTapNetwork(endpoint); errUnTap != nil {
|
||||||
|
+ networkLogger().WithError(errUnTap).Errorf("Error rollback tap %s", endpoint.TapInterface.TAPIface.Name)
|
||||||
|
+ }
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
@@ -100,7 +103,7 @@ func (endpoint *TapEndpoint) HotAttach(h hypervisor) error {
|
||||||
|
func (endpoint *TapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error {
|
||||||
|
networkLogger().Info("Hot detaching tap endpoint")
|
||||||
|
if err := doNetNS(netNsPath, func(_ ns.NetNS) error {
|
||||||
|
- return unTapNetwork(endpoint.TapInterface.TAPIface.Name)
|
||||||
|
+ return unTapNetwork(endpoint)
|
||||||
|
}); err != nil {
|
||||||
|
networkLogger().WithError(err).Warn("Error un-bridging tap ep")
|
||||||
|
}
|
||||||
|
@@ -185,7 +188,14 @@ func tapNetwork(endpoint *TapEndpoint, numCPUs uint32, disableVhostNet bool) err
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
-func unTapNetwork(name string) error {
|
||||||
|
+func unTapNetwork(endpoint *TapEndpoint) error {
|
||||||
|
+ // length of VMFDs == 0 means that the endpoint is already exist in the host,
|
||||||
|
+ // no created by kata, so we don't need to remove it when detach
|
||||||
|
+ if len(endpoint.TapInterface.VMFds) == 0 {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ name := endpoint.TapInterface.TAPIface.Name
|
||||||
|
netHandle, err := netlink.NewHandle()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
diff --git a/virtcontainers/veth_endpoint.go b/virtcontainers/veth_endpoint.go
|
||||||
|
index 9ece6a74..0f2ec9ba 100644
|
||||||
|
--- a/virtcontainers/veth_endpoint.go
|
||||||
|
+++ b/virtcontainers/veth_endpoint.go
|
||||||
|
@@ -119,6 +119,9 @@ func (endpoint *VethEndpoint) HotAttach(h hypervisor) error {
|
||||||
|
|
||||||
|
if _, err := h.hotplugAddDevice(endpoint, netDev); err != nil {
|
||||||
|
networkLogger().WithError(err).Error("Error attach virtual ep")
|
||||||
|
+ if errDisconn := xDisconnectVMNetwork(endpoint); errDisconn != nil {
|
||||||
|
+ networkLogger().WithError(errDisconn).Error("Error rollback virtual ep")
|
||||||
|
+ }
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
1273
runtime/patches/0034-kata-runtime-add-kata-ipvs-command.patch
Normal file
1273
runtime/patches/0034-kata-runtime-add-kata-ipvs-command.patch
Normal file
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,234 @@
|
|||||||
|
From cf2b34f477cba88641de3719bf5c8f933b919bcc Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Mon, 17 Aug 2020 21:44:57 +0800
|
||||||
|
Subject: [PATCH 35/50] device: mount blockdevices in the guest VM
|
||||||
|
|
||||||
|
reason: support mount blockdevices in the guest VM like
|
||||||
|
"-v /dev/blockdevice:/home/test"
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
cli/config/configuration-qemu.toml.in | 5 ++++
|
||||||
|
pkg/katautils/config.go | 6 +++++
|
||||||
|
virtcontainers/kata_agent.go | 27 ++++++++++++++++---
|
||||||
|
virtcontainers/kata_agent_test.go | 2 +-
|
||||||
|
virtcontainers/utils/utils.go | 50 +++++++++++++++++++++++++++++++++++
|
||||||
|
virtcontainers/vm_test.go | 2 +-
|
||||||
|
6 files changed, 86 insertions(+), 6 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in
|
||||||
|
index 46f8b632..aa11b38f 100644
|
||||||
|
--- a/cli/config/configuration-qemu.toml.in
|
||||||
|
+++ b/cli/config/configuration-qemu.toml.in
|
||||||
|
@@ -391,6 +391,11 @@ path = "@SHIMPATH@"
|
||||||
|
#
|
||||||
|
kernel_modules=[]
|
||||||
|
|
||||||
|
+# If enabled, when we pass a block device to the guest VM
|
||||||
|
+# through -v, such as `docker run -v /dev/loop100:/foo/bar`,
|
||||||
|
+# agent will create a directory in guest VM and mount the
|
||||||
|
+# file system on `/dev/loop100` inside the container
|
||||||
|
+enable_blk_mount = true
|
||||||
|
|
||||||
|
[netmon]
|
||||||
|
# If enabled, the network monitoring process gets started when the
|
||||||
|
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
||||||
|
index 94c916a0..51120311 100644
|
||||||
|
--- a/pkg/katautils/config.go
|
||||||
|
+++ b/pkg/katautils/config.go
|
||||||
|
@@ -160,6 +160,7 @@ type agent struct {
|
||||||
|
TraceMode string `toml:"trace_mode"`
|
||||||
|
TraceType string `toml:"trace_type"`
|
||||||
|
KernelModules []string `toml:"kernel_modules"`
|
||||||
|
+ MountBlkInVM bool `toml:"enable_blk_mount"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type netmon struct {
|
||||||
|
@@ -463,6 +464,10 @@ func (h hypervisor) getInitrdAndImage() (initrd string, image string, err error)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (a agent) mountBlkDevInVM() bool {
|
||||||
|
+ return a.MountBlkInVM
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func (p proxy) path() (string, error) {
|
||||||
|
path := p.Path
|
||||||
|
if path == "" {
|
||||||
|
@@ -978,6 +983,7 @@ func updateRuntimeConfigAgent(configPath string, tomlConf tomlConfig, config *oc
|
||||||
|
TraceMode: agent.traceMode(),
|
||||||
|
TraceType: agent.traceType(),
|
||||||
|
KernelModules: agent.kernelModules(),
|
||||||
|
+ MountBlkInVM: agent.mountBlkDevInVM(),
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("%s agent type is not supported", k)
|
||||||
|
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
|
||||||
|
index 16662949..b0f88c15 100644
|
||||||
|
--- a/virtcontainers/kata_agent.go
|
||||||
|
+++ b/virtcontainers/kata_agent.go
|
||||||
|
@@ -30,6 +30,7 @@ import (
|
||||||
|
ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/pkg/rootless"
|
||||||
|
vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/store"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/types"
|
||||||
|
@@ -64,6 +65,7 @@ var (
|
||||||
|
errorMissingOCISpec = errors.New("Missing OCI specification")
|
||||||
|
defaultKataHostSharedDir = "/run/kata-containers/shared/sandboxes/"
|
||||||
|
defaultKataGuestSharedDir = "/run/kata-containers/shared/containers/"
|
||||||
|
+ kataGuestStorageDir = "/run/kata-containers/storage/containers/"
|
||||||
|
mountGuestTag = "kataShared"
|
||||||
|
defaultKataGuestSandboxDir = "/run/kata-containers/sandbox/"
|
||||||
|
type9pFs = "9p"
|
||||||
|
@@ -199,6 +201,7 @@ type KataAgentConfig struct {
|
||||||
|
TraceMode string
|
||||||
|
TraceType string
|
||||||
|
KernelModules []string
|
||||||
|
+ MountBlkInVM bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// KataAgentState is the structure describing the data stored from this
|
||||||
|
@@ -1061,7 +1064,12 @@ func (k *kataAgent) replaceOCIMountsForStorages(spec *specs.Spec, volumeStorages
|
||||||
|
// Create a temporary location to mount the Storage. Mounting to the correct location
|
||||||
|
// will be handled by the OCI mount structure.
|
||||||
|
filename := fmt.Sprintf("%s-%s", uuid.Generate().String(), filepath.Base(m.Destination))
|
||||||
|
- path := filepath.Join(kataGuestSandboxStorageDir(), filename)
|
||||||
|
+ var path string
|
||||||
|
+ if v.Driver == kataBlkDevType || v.Driver == kataSCSIDevType {
|
||||||
|
+ path = filepath.Join(kataGuestStorageDir, filename)
|
||||||
|
+ } else {
|
||||||
|
+ path = filepath.Join(kataGuestSandboxStorageDir(), filename)
|
||||||
|
+ }
|
||||||
|
|
||||||
|
k.Logger().Debugf("Replacing OCI mount source (%s) with %s", m.Source, path)
|
||||||
|
ociMounts[index].Source = path
|
||||||
|
@@ -1436,7 +1444,7 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process,
|
||||||
|
// Note this call modifies the list of container devices to make sure
|
||||||
|
// all hotplugged devices are unplugged, so this needs be done
|
||||||
|
// after devices passed with --device are handled.
|
||||||
|
- volumeStorages, err := k.handleBlockVolumes(c)
|
||||||
|
+ volumeStorages, err := k.handleBlockVolumes(sandbox, c)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
@@ -1609,7 +1617,7 @@ func (k *kataAgent) handleVhostUserBlkVolume(c *Container, device api.Device) (*
|
||||||
|
|
||||||
|
// handleBlockVolumes handles volumes that are block devices files
|
||||||
|
// by passing the block devices as Storage to the agent.
|
||||||
|
-func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) {
|
||||||
|
+func (k *kataAgent) handleBlockVolumes(sandbox *Sandbox, c *Container) ([]*grpc.Storage, error) {
|
||||||
|
|
||||||
|
var volumeStorages []*grpc.Storage
|
||||||
|
|
||||||
|
@@ -1648,9 +1656,20 @@ func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) {
|
||||||
|
}
|
||||||
|
|
||||||
|
vol.MountPoint = m.Destination
|
||||||
|
- if vol.Fstype == "" {
|
||||||
|
+
|
||||||
|
+ ac, _ := sandbox.config.AgentConfig.(KataAgentConfig)
|
||||||
|
+ if ac.MountBlkInVM {
|
||||||
|
+ // Ensure the block device is formatted, for the devices here are specified as volumes
|
||||||
|
+ fsType, gerr := utils.GetDevFormat(m.Source)
|
||||||
|
+ if gerr != nil || fsType == "" {
|
||||||
|
+ k.Logger().WithField("device", id).WithError(gerr).Error("get device format failed")
|
||||||
|
+ return nil, gerr
|
||||||
|
+ }
|
||||||
|
+ vol.Fstype = fmt.Sprintf("bind-%s", fsType)
|
||||||
|
+ } else {
|
||||||
|
vol.Fstype = "bind"
|
||||||
|
}
|
||||||
|
+
|
||||||
|
if len(vol.Options) == 0 {
|
||||||
|
vol.Options = []string{"bind"}
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go
|
||||||
|
index 18a5a0a6..68caaab2 100644
|
||||||
|
--- a/virtcontainers/kata_agent_test.go
|
||||||
|
+++ b/virtcontainers/kata_agent_test.go
|
||||||
|
@@ -446,7 +446,7 @@ func TestHandleBlockVolume(t *testing.T) {
|
||||||
|
containers[c.id].sandbox = &sandbox
|
||||||
|
containers[c.id].mounts = mounts
|
||||||
|
|
||||||
|
- volumeStorages, err := k.handleBlockVolumes(c)
|
||||||
|
+ volumeStorages, err := k.handleBlockVolumes(&sandbox, c)
|
||||||
|
assert.Nil(t, err, "Error while handling block volumes")
|
||||||
|
|
||||||
|
vStorage := &pb.Storage{
|
||||||
|
diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go
|
||||||
|
index 9490faa1..36ac67a7 100644
|
||||||
|
--- a/virtcontainers/utils/utils.go
|
||||||
|
+++ b/virtcontainers/utils/utils.go
|
||||||
|
@@ -379,3 +379,53 @@ func RoundVCPUNumber(value string) (int, error) {
|
||||||
|
cpus := int(math.Ceil(cpuNum))
|
||||||
|
return cpus, nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+// GetDevFormat get the formated filesystem of input disk use `blkid`, Such as `ext4`,`xfs`,etc.
|
||||||
|
+func GetDevFormat(disk string) (string, error) {
|
||||||
|
+ // refer to https://github.com/kubernetes/kubernetes/blob/v1.12.2/pkg/util/mount/mount_linux.go#L512
|
||||||
|
+ args := []string{"-p", "-s", "TYPE", "-s", "PTTYPE", "-o", "export", disk}
|
||||||
|
+ dataOut, err := exec.Command("blkid", args...).Output()
|
||||||
|
+ output := string(dataOut)
|
||||||
|
+
|
||||||
|
+ if err != nil {
|
||||||
|
+ if strings.Contains(err.Error(), "exit status 2") {
|
||||||
|
+ // Disk device is unformatted.
|
||||||
|
+ // For `blkid`, if the specified token (TYPE/PTTYPE, etc) was
|
||||||
|
+ // not found, or no (specified) devices could be identified, an
|
||||||
|
+ // exit code of 2 is returned.
|
||||||
|
+ return "", nil
|
||||||
|
+ }
|
||||||
|
+ return "", err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ var fstype string
|
||||||
|
+
|
||||||
|
+ lines := strings.Split(output, "\n")
|
||||||
|
+ for _, l := range lines {
|
||||||
|
+ if len(l) <= 0 {
|
||||||
|
+ // Ignore empty line.
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+ // if we use busybox as rootfs,the output of command
|
||||||
|
+ // `busybox blkid` is different with original`blkid`,
|
||||||
|
+ // so we should make a compatible parse
|
||||||
|
+ subLine := strings.Split(l, " ")
|
||||||
|
+ for _, sl := range subLine {
|
||||||
|
+ if len(subLine) <= 0 {
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+ cs := strings.Split(sl, "=")
|
||||||
|
+ if len(cs) != 2 {
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+ if cs[0] == "TYPE" {
|
||||||
|
+ fstype = cs[1]
|
||||||
|
+ if strings.Contains(fstype, `"`) {
|
||||||
|
+ fstype = strings.Replace(fstype, `"`, "", -1)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return fstype, nil
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/vm_test.go b/virtcontainers/vm_test.go
|
||||||
|
index 36fd5c2f..d2414232 100644
|
||||||
|
--- a/virtcontainers/vm_test.go
|
||||||
|
+++ b/virtcontainers/vm_test.go
|
||||||
|
@@ -125,7 +125,7 @@ func TestVMConfigGrpc(t *testing.T) {
|
||||||
|
HypervisorType: QemuHypervisor,
|
||||||
|
HypervisorConfig: newQemuConfig(),
|
||||||
|
AgentType: KataContainersAgent,
|
||||||
|
- AgentConfig: KataAgentConfig{false, true, false, false, 0, "", "", []string{}},
|
||||||
|
+ AgentConfig: KataAgentConfig{false, true, false, false, 0, "", "", []string{}, false },
|
||||||
|
ProxyType: NoopProxyType,
|
||||||
|
}
|
||||||
|
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,131 @@
|
|||||||
|
From 448bb661d6759c1e20c18084604b150bff08ada4 Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Tue, 18 Aug 2020 11:49:26 +0800
|
||||||
|
Subject: [PATCH 36/50] mount: limit the maximum number of virtio-scsi bus
|
||||||
|
slots
|
||||||
|
|
||||||
|
reason:
|
||||||
|
1. add SCSIBus functions for add or remove device about ScsiBus
|
||||||
|
2. limit the maximum number of virtio-scsi bus slots to 25
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/qemu.go | 20 +++++++++++++++++
|
||||||
|
virtcontainers/types/scsi.go | 53 ++++++++++++++++++++++++++++++++++++++++++++
|
||||||
|
2 files changed, 73 insertions(+)
|
||||||
|
create mode 100644 virtcontainers/types/scsi.go
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
|
||||||
|
index bb83b1bb..c2b65376 100644
|
||||||
|
--- a/virtcontainers/qemu.go
|
||||||
|
+++ b/virtcontainers/qemu.go
|
||||||
|
@@ -65,6 +65,7 @@ type CPUDevice struct {
|
||||||
|
// QemuState keeps Qemu's state
|
||||||
|
type QemuState struct {
|
||||||
|
Bridges []types.Bridge
|
||||||
|
+ ScsiBus *types.SCSIBus
|
||||||
|
// HotpluggedCPUs is the list of CPUs that were hot-added
|
||||||
|
HotpluggedVCPUs []CPUDevice
|
||||||
|
HotpluggedMemory int
|
||||||
|
@@ -411,6 +412,13 @@ func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.I
|
||||||
|
|
||||||
|
var ioThread *govmmQemu.IOThread
|
||||||
|
if q.config.BlockDeviceDriver == config.VirtioSCSI {
|
||||||
|
+ if q.state.ScsiBus == nil {
|
||||||
|
+ // only use `scsi0.0`
|
||||||
|
+ q.state.ScsiBus = &types.SCSIBus{
|
||||||
|
+ Address: make(map[uint32]int),
|
||||||
|
+ ID: fmt.Sprintf("%s.0", scsiControllerID),
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
return q.arch.appendSCSIController(devices, q.config.EnableIOThreads)
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -1168,6 +1176,16 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev
|
||||||
|
|
||||||
|
// Bus exposed by the SCSI Controller
|
||||||
|
bus := scsiControllerID + ".0"
|
||||||
|
+ var err error
|
||||||
|
+ if err = q.state.ScsiBus.AddDevToScsiBus(drive.Index); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ defer func() {
|
||||||
|
+ if err != nil {
|
||||||
|
+ q.state.ScsiBus.RemoveDevFromScsiBus(drive.Index)
|
||||||
|
+ }
|
||||||
|
+ }()
|
||||||
|
|
||||||
|
// Get SCSI-id and LUN based on the order of attaching drives.
|
||||||
|
scsiID, lun, err := utils.GetSCSIIdLun(drive.Index)
|
||||||
|
@@ -1234,6 +1252,8 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error
|
||||||
|
if err := q.arch.removeDeviceFromBridge(drive.ID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
+ } else if q.config.BlockDeviceDriver == config.VirtioSCSI {
|
||||||
|
+ q.state.ScsiBus.RemoveDevFromScsiBus(drive.Index)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := q.qmpMonitorCh.qmp.ExecuteDeviceDel(q.qmpMonitorCh.ctx, devID); err != nil {
|
||||||
|
diff --git a/virtcontainers/types/scsi.go b/virtcontainers/types/scsi.go
|
||||||
|
new file mode 100644
|
||||||
|
index 00000000..f4d489f0
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/virtcontainers/types/scsi.go
|
||||||
|
@@ -0,0 +1,53 @@
|
||||||
|
+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
||||||
|
+// SPDX-License-Identifier: Apache-2.0
|
||||||
|
+// Description: common functions
|
||||||
|
+// Author: leizhongkai
|
||||||
|
+// Create: 2019-06-30
|
||||||
|
+
|
||||||
|
+package types
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "fmt"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+const (
|
||||||
|
+ scsiMaxCapacity = 25
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+type SCSIBus struct {
|
||||||
|
+ // Address contains information about devices plugged and number limit
|
||||||
|
+ Address map[uint32]int
|
||||||
|
+
|
||||||
|
+ // ID is used to identify the bus exposed by the SCSI Controller
|
||||||
|
+ ID string
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (sb *SCSIBus) AddDevToScsiBus(index int) error {
|
||||||
|
+ var addr uint32
|
||||||
|
+
|
||||||
|
+ // looking for the first available address
|
||||||
|
+ for i := uint32(1); i <= scsiMaxCapacity; i++ {
|
||||||
|
+ if _, ok := sb.Address[i]; !ok {
|
||||||
|
+ addr = i
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if addr == 0 {
|
||||||
|
+ return fmt.Errorf("Scsi bus capacity limited.")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // save address and device
|
||||||
|
+ sb.Address[addr] = index
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (sb *SCSIBus) RemoveDevFromScsiBus(index int) {
|
||||||
|
+ for addr, i := range sb.Address {
|
||||||
|
+ if i == index {
|
||||||
|
+ delete(sb.Address, addr)
|
||||||
|
+ return
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
875
runtime/patches/0037-runtime-add-IPVS-test.patch
Normal file
875
runtime/patches/0037-runtime-add-IPVS-test.patch
Normal file
@ -0,0 +1,875 @@
|
|||||||
|
From d083f0e0247fbded92a0ae2a0e71da4176baed95 Mon Sep 17 00:00:00 2001
|
||||||
|
From: xiadanni <xiadanni1@huawei.com>
|
||||||
|
Date: Tue, 18 Aug 2020 17:08:23 +0800
|
||||||
|
Subject: [PATCH 37/50] runtime: add IPVS test
|
||||||
|
|
||||||
|
Signed-off-by: xiadanni <xiadanni1@huawei.com>
|
||||||
|
---
|
||||||
|
cli/ipvsadm_test.go | 775 +++++++++++++++++++++++++++++++++++++
|
||||||
|
virtcontainers/api_test.go | 36 ++
|
||||||
|
virtcontainers/kata_agent_test.go | 4 +
|
||||||
|
virtcontainers/pkg/vcmock/types.go | 2 +-
|
||||||
|
4 files changed, 816 insertions(+), 1 deletion(-)
|
||||||
|
create mode 100644 cli/ipvsadm_test.go
|
||||||
|
|
||||||
|
diff --git a/cli/ipvsadm_test.go b/cli/ipvsadm_test.go
|
||||||
|
new file mode 100644
|
||||||
|
index 00000000..92958aee
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/cli/ipvsadm_test.go
|
||||||
|
@@ -0,0 +1,775 @@
|
||||||
|
+// Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved.
|
||||||
|
+// SPDX-License-Identifier: Apache-2.0
|
||||||
|
+// Description: IPVS common functions test
|
||||||
|
+// Author: xiadanni
|
||||||
|
+// Create: 2020-08-01
|
||||||
|
+
|
||||||
|
+package main
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "context"
|
||||||
|
+ "flag"
|
||||||
|
+ "fmt"
|
||||||
|
+ "os"
|
||||||
|
+ "strings"
|
||||||
|
+ "testing"
|
||||||
|
+
|
||||||
|
+ "github.com/kata-containers/agent/protocols/grpc"
|
||||||
|
+ vc "github.com/kata-containers/runtime/virtcontainers"
|
||||||
|
+ vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/types"
|
||||||
|
+ "github.com/stretchr/testify/assert"
|
||||||
|
+ "github.com/urfave/cli"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+func TestKataIPVSCliAction(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+
|
||||||
|
+ actionFunc, ok := kataIPVSCLICommand.Action.(func(ctx *cli.Context) error)
|
||||||
|
+ assert.True(ok)
|
||||||
|
+
|
||||||
|
+ flagSet := flag.NewFlagSet("kata-IPVS", flag.ContinueOnError)
|
||||||
|
+ ctx := createCLIContext(flagSet)
|
||||||
|
+
|
||||||
|
+ err := actionFunc(ctx)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestIPVSadmCliAction(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+
|
||||||
|
+ actionFunc, ok := IPVSadmCommand.Action.(func(ctx *cli.Context) error)
|
||||||
|
+ assert.True(ok)
|
||||||
|
+
|
||||||
|
+ flagSet := flag.NewFlagSet("IPVSadm", flag.ContinueOnError)
|
||||||
|
+ ctx := createCLIContext(flagSet)
|
||||||
|
+
|
||||||
|
+ err := actionFunc(ctx)
|
||||||
|
+ assert.Error(err, "Missing container ID")
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestIPVSadmCLISuccessful(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+
|
||||||
|
+ sandbox := &vcmock.Sandbox{
|
||||||
|
+ MockID: testContainerID,
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ sandbox.MockContainers = []*vcmock.Container{
|
||||||
|
+ {
|
||||||
|
+ MockID: sandbox.ID(),
|
||||||
|
+ MockSandbox: sandbox,
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
|
||||||
|
+ return vc.ContainerStatus{
|
||||||
|
+ ID: sandbox.ID(),
|
||||||
|
+ Annotations: map[string]string{
|
||||||
|
+ vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
|
||||||
|
+ },
|
||||||
|
+ State: types.ContainerState{
|
||||||
|
+ State: types.StateRunning,
|
||||||
|
+ },
|
||||||
|
+ }, nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
|
||||||
|
+ return &grpc.IPVSResponse{}, nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ defer func() {
|
||||||
|
+ testingImpl.StatusContainerFunc = nil
|
||||||
|
+ testingImpl.UpdateIPVSRuleFunc = nil
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
+ path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ defer os.RemoveAll(path)
|
||||||
|
+
|
||||||
|
+ actionFunc, ok := IPVSadmCommand.Action.(func(ctx *cli.Context) error)
|
||||||
|
+ assert.True(ok)
|
||||||
|
+
|
||||||
|
+ flagSet := flag.NewFlagSet("IPVSadm", flag.ContinueOnError)
|
||||||
|
+ flagSet.Parse([]string{testContainerID})
|
||||||
|
+ flagSet.String("parameters", "-a -t 192.168.0.7:80 -r 192.168.0.4:80", "")
|
||||||
|
+ ctx := createCLIContext(flagSet)
|
||||||
|
+ err = actionFunc(ctx)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ // result not nil
|
||||||
|
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
|
||||||
|
+ return &grpc.IPVSResponse{IPVSRes: "IPVS rule updating success"}, nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ err = actionFunc(ctx)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestIPVSadmCLIError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+
|
||||||
|
+ sandbox := &vcmock.Sandbox{
|
||||||
|
+ MockID: testContainerID,
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ sandbox.MockContainers = []*vcmock.Container{
|
||||||
|
+ {
|
||||||
|
+ MockID: sandbox.ID(),
|
||||||
|
+ MockSandbox: sandbox,
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
|
||||||
|
+ return vc.ContainerStatus{
|
||||||
|
+ ID: sandbox.ID(),
|
||||||
|
+ Annotations: map[string]string{
|
||||||
|
+ vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
|
||||||
|
+ },
|
||||||
|
+ State: types.ContainerState{
|
||||||
|
+ State: types.StateRunning,
|
||||||
|
+ },
|
||||||
|
+ }, nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
|
||||||
|
+ return &grpc.IPVSResponse{}, nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ defer func() {
|
||||||
|
+ testingImpl.StatusContainerFunc = nil
|
||||||
|
+ testingImpl.UpdateIPVSRuleFunc = nil
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
+ path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ defer os.RemoveAll(path)
|
||||||
|
+
|
||||||
|
+ actionFunc, ok := IPVSadmCommand.Action.(func(ctx *cli.Context) error)
|
||||||
|
+ assert.True(ok)
|
||||||
|
+
|
||||||
|
+ // no stdin rule file
|
||||||
|
+ flagSet := flag.NewFlagSet("IPVSadm", flag.ContinueOnError)
|
||||||
|
+ flagSet.Parse([]string{testContainerID})
|
||||||
|
+ flagSet.String("restore", "-", "")
|
||||||
|
+ ctx := createCLIContext(flagSet)
|
||||||
|
+ err = actionFunc(ctx)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+
|
||||||
|
+ // restore parameter error
|
||||||
|
+ flagSet = flag.NewFlagSet("IPVSadm", flag.ContinueOnError)
|
||||||
|
+ flagSet.Parse([]string{testContainerID})
|
||||||
|
+ flagSet.String("restore", "abc", "")
|
||||||
|
+ ctx = createCLIContext(flagSet)
|
||||||
|
+ err = actionFunc(ctx)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+
|
||||||
|
+ // checkrule returns error
|
||||||
|
+ flagSet = flag.NewFlagSet("IPVSadm", flag.ContinueOnError)
|
||||||
|
+ flagSet.Parse([]string{testContainerID})
|
||||||
|
+ flagSet.String("parameters", "-A -t 192.168.0.7:80 -s rr -p -3000", "")
|
||||||
|
+ ctx = createCLIContext(flagSet)
|
||||||
|
+ err = actionFunc(ctx)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+
|
||||||
|
+ // updatefunction returns error
|
||||||
|
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
|
||||||
|
+ return &grpc.IPVSResponse{}, fmt.Errorf("IPVSadm test error")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ flagSet = flag.NewFlagSet("IPVSadm", flag.ContinueOnError)
|
||||||
|
+ flagSet.Parse([]string{testContainerID})
|
||||||
|
+ flagSet.String("parameters", "-A -t 192.168.0.7:80 -s rr -p 3000", "")
|
||||||
|
+ ctx = createCLIContext(flagSet)
|
||||||
|
+ err = actionFunc(ctx)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestCleanupCliAction(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+
|
||||||
|
+ actionFunc, ok := CleanupCommand.Action.(func(ctx *cli.Context) error)
|
||||||
|
+ assert.True(ok)
|
||||||
|
+
|
||||||
|
+ flagSet := flag.NewFlagSet("cleanup", flag.ContinueOnError)
|
||||||
|
+ ctx := createCLIContext(flagSet)
|
||||||
|
+
|
||||||
|
+ err := actionFunc(ctx)
|
||||||
|
+ assert.Error(err, "Missing container ID")
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestCleanupCLISuccessful(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+
|
||||||
|
+ sandbox := &vcmock.Sandbox{
|
||||||
|
+ MockID: testContainerID,
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ sandbox.MockContainers = []*vcmock.Container{
|
||||||
|
+ {
|
||||||
|
+ MockID: sandbox.ID(),
|
||||||
|
+ MockSandbox: sandbox,
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
|
||||||
|
+ return vc.ContainerStatus{
|
||||||
|
+ ID: sandbox.ID(),
|
||||||
|
+ Annotations: map[string]string{
|
||||||
|
+ vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
|
||||||
|
+ },
|
||||||
|
+ State: types.ContainerState{
|
||||||
|
+ State: types.StateRunning,
|
||||||
|
+ },
|
||||||
|
+ }, nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
|
||||||
|
+ return &grpc.IPVSResponse{}, nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ defer func() {
|
||||||
|
+ testingImpl.StatusContainerFunc = nil
|
||||||
|
+ testingImpl.UpdateIPVSRuleFunc = nil
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
+ path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ defer os.RemoveAll(path)
|
||||||
|
+
|
||||||
|
+ actionFunc, ok := CleanupCommand.Action.(func(ctx *cli.Context) error)
|
||||||
|
+ assert.True(ok)
|
||||||
|
+
|
||||||
|
+ flagSet := flag.NewFlagSet("cleanup", flag.ContinueOnError)
|
||||||
|
+ flagSet.Parse([]string{testContainerID})
|
||||||
|
+ flagSet.String("parameters", "-d 192.168.0.4 -p tcp", "")
|
||||||
|
+ ctx := createCLIContext(flagSet)
|
||||||
|
+ err = actionFunc(ctx)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestCleanupCLIError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+
|
||||||
|
+ sandbox := &vcmock.Sandbox{
|
||||||
|
+ MockID: testContainerID,
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ sandbox.MockContainers = []*vcmock.Container{
|
||||||
|
+ {
|
||||||
|
+ MockID: sandbox.ID(),
|
||||||
|
+ MockSandbox: sandbox,
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
|
||||||
|
+ return vc.ContainerStatus{
|
||||||
|
+ ID: sandbox.ID(),
|
||||||
|
+ Annotations: map[string]string{
|
||||||
|
+ vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
|
||||||
|
+ },
|
||||||
|
+ State: types.ContainerState{
|
||||||
|
+ State: types.StateRunning,
|
||||||
|
+ },
|
||||||
|
+ }, nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
|
||||||
|
+ return &grpc.IPVSResponse{}, nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ defer func() {
|
||||||
|
+ testingImpl.StatusContainerFunc = nil
|
||||||
|
+ testingImpl.UpdateIPVSRuleFunc = nil
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
+ path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ defer os.RemoveAll(path)
|
||||||
|
+
|
||||||
|
+ actionFunc, ok := CleanupCommand.Action.(func(ctx *cli.Context) error)
|
||||||
|
+ assert.True(ok)
|
||||||
|
+
|
||||||
|
+ // checkrule returns error
|
||||||
|
+ flagSet := flag.NewFlagSet("cleanup", flag.ContinueOnError)
|
||||||
|
+ flagSet.Parse([]string{testContainerID})
|
||||||
|
+ flagSet.String("parameters", "-d 192.168.0.4", "")
|
||||||
|
+ ctx := createCLIContext(flagSet)
|
||||||
|
+ err = actionFunc(ctx)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+
|
||||||
|
+ // updatefunction returns error
|
||||||
|
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
|
||||||
|
+ return &grpc.IPVSResponse{}, fmt.Errorf("IPVSadm cleanup test error")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ flagSet = flag.NewFlagSet("cleanup", flag.ContinueOnError)
|
||||||
|
+ flagSet.Parse([]string{testContainerID})
|
||||||
|
+ flagSet.String("parameters", "-d 192.168.0.4 -p tcp", "")
|
||||||
|
+ ctx = createCLIContext(flagSet)
|
||||||
|
+ err = actionFunc(ctx)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestAddServiceSuccessfully(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ addServiceCommandTCP := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --scheduler rr --persistent 3000"
|
||||||
|
+ _, err := checkIPVSRule(addServiceCommandTCP)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ addServiceCommandUDP := "IPVSadm --add-service --udp-service 192.168.0.7:80 --scheduler rr --persistent 3000"
|
||||||
|
+ _, err = checkIPVSRule(addServiceCommandUDP)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rr -p 3000"
|
||||||
|
+ _, err = checkIPVSRule(addServiceCmd)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestAddServiceNoService(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ addServiceCommand := "IPVSadm --add-service --scheduler rr --persistent 3000"
|
||||||
|
+ _, err := checkIPVSRule(addServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ addServiceCmd := "IPVSadm -A -s rr -p 3000"
|
||||||
|
+ _, err = checkIPVSRule(addServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestAddServiceNoScheduler(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --persistent 3000"
|
||||||
|
+ _, err := checkIPVSRule(addServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -p 3000"
|
||||||
|
+ _, err = checkIPVSRule(addServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestAddServiceNoPersistent(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --scheduler rr"
|
||||||
|
+ _, err := checkIPVSRule(addServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rr"
|
||||||
|
+ _, err = checkIPVSRule(addServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestAddServiceIpError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.2.2.7:80 --scheduler rr --persistent 3000"
|
||||||
|
+ _, err := checkIPVSRule(addServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ addServiceCmd := "IPVSadm -A -t 192.168.2.2.7:80 -s rr -p 3000"
|
||||||
|
+ _, err = checkIPVSRule(addServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestAddServicePortError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:9999999 --scheduler rr --persistent 3000"
|
||||||
|
+ _, err := checkIPVSRule(addServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:9999999 -s rr -p 3000"
|
||||||
|
+ _, err = checkIPVSRule(addServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestAddServiceSchedulerError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --scheduler rrr --persistent 3000"
|
||||||
|
+ _, err := checkIPVSRule(addServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rrr -p 3000"
|
||||||
|
+ _, err = checkIPVSRule(addServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestAddServicePersistentError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --scheduler rr --persistent 99999999999"
|
||||||
|
+ _, err := checkIPVSRule(addServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rr -p 99999999999"
|
||||||
|
+ _, err = checkIPVSRule(addServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestEditServiceSuccessfully(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ editServiceCommandTCP := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --scheduler rr --persistent 3000"
|
||||||
|
+ _, err := checkIPVSRule(editServiceCommandTCP)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ editServiceCommandUcp := "IPVSadm --edit-service --udp-service 192.168.0.7:80 --scheduler rr --persistent 3000"
|
||||||
|
+ _, err = checkIPVSRule(editServiceCommandUcp)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ editServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rr -p 3000"
|
||||||
|
+ _, err = checkIPVSRule(editServiceCmd)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestEditServiceNoService(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ editServiceCommand := "IPVSadm --edit-service --scheduler rr --persistent 3000"
|
||||||
|
+ _, err := checkIPVSRule(editServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ editServiceCmd := "IPVSadm -E -s rr -p 3000"
|
||||||
|
+ _, err = checkIPVSRule(editServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestEditServiceNoScheduler(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --persistent 3000"
|
||||||
|
+ _, err := checkIPVSRule(editServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ editServiceCmd := "IPVSadm -E -t 192.168.0.7:80 -p 3000"
|
||||||
|
+ _, err = checkIPVSRule(editServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestEditServiceNoPersistent(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --scheduler rr"
|
||||||
|
+ _, err := checkIPVSRule(editServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ editServiceCmd := "IPVSadm -E -t 192.168.0.7:80 -s rr"
|
||||||
|
+ _, err = checkIPVSRule(editServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestEditServiceIpError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.2.2.7:80 --scheduler rr --persistent 3000"
|
||||||
|
+ _, err := checkIPVSRule(editServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ editServiceCmd := "IPVSadm -E -t 192.168.2.2.7:80 -s rr -p 3000"
|
||||||
|
+ _, err = checkIPVSRule(editServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestEditServicePortError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:9999999 --scheduler rr --persistent 3000"
|
||||||
|
+ _, err := checkIPVSRule(editServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ editServiceCmd := "IPVSadm -E -t 192.168.0.7:9999999 -s rr -p 3000"
|
||||||
|
+ _, err = checkIPVSRule(editServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestEditServiceSchedulerError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --scheduler rrr --persistent 3000"
|
||||||
|
+ _, err := checkIPVSRule(editServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ editServiceCmd := "IPVSadm -E -t 192.168.0.7:80 -s rrr -p 3000"
|
||||||
|
+ _, err = checkIPVSRule(editServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestEditServicePersistentError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --scheduler rr --persistent 99999999999"
|
||||||
|
+ _, err := checkIPVSRule(editServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ editServiceCmd := "IPVSadm -E -t 192.168.0.7:80 -s rr -p 99999999999"
|
||||||
|
+ _, err = checkIPVSRule(editServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestDeleteServiceSuccessfully(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ deleteServiceCommandTCP := "IPVSadm --delete-service --tcp-service 192.168.0.7:80"
|
||||||
|
+ _, err := checkIPVSRule(deleteServiceCommandTCP)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ deleteServiceCommandUDP := "IPVSadm --delete-service --udp-service 192.168.0.7:80"
|
||||||
|
+ _, err = checkIPVSRule(deleteServiceCommandUDP)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ deleteServiceCmd := "IPVSadm -D -t 192.168.0.7:80"
|
||||||
|
+ _, err = checkIPVSRule(deleteServiceCmd)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestDeleteServiceNoService(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ deleteServiceCommand := "IPVSadm --delete-service"
|
||||||
|
+ _, err := checkIPVSRule(deleteServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ deleteServiceCmd := "IPVSadm -D"
|
||||||
|
+ _, err = checkIPVSRule(deleteServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestDeleteServiceIpError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ deleteServiceCommand := "IPVSadm --delete-service --tcp-service 192.168.2.2.7:80"
|
||||||
|
+ _, err := checkIPVSRule(deleteServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ deleteServiceCmd := "IPVSadm -D -t 192.168.2.2.7:80"
|
||||||
|
+ _, err = checkIPVSRule(deleteServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestDeleteServicePortError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ deleteServiceCommand := "IPVSadm --delete-service --tcp-service 192.168.0.7:9999999"
|
||||||
|
+ _, err := checkIPVSRule(deleteServiceCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ deleteServiceCmd := "IPVSadm -D -t 192.168.0.7:9999999"
|
||||||
|
+ _, err = checkIPVSRule(deleteServiceCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestAddServerSuccessfully(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ addServerCommandTCP := "IPVSadm --add-server --tcp-service 192.168.0.7:80 --real-server 192.168.0.4:80 --weight 100"
|
||||||
|
+ _, err := checkIPVSRule(addServerCommandTCP)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ addServerCommandUDP := "IPVSadm --add-server --udp-service 192.168.0.7:80 --real-server 192.168.0.4:80 --weight 100"
|
||||||
|
+ _, err = checkIPVSRule(addServerCommandUDP)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ addServerCmd := "IPVSadm -a -t 192.168.0.7:80 -r 192.168.0.4:80 -w 100"
|
||||||
|
+ _, err = checkIPVSRule(addServerCmd)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestAddServerNoService(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ addServerCommand := "IPVSadm --add-server --real-server 192.168.0.4:80"
|
||||||
|
+ _, err := checkIPVSRule(addServerCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ addServerCmd := "IPVSadm -a -r 192.168.0.4:80"
|
||||||
|
+ _, err = checkIPVSRule(addServerCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestAddServerNoServer(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ addServerCommand := "IPVSadm --add-server --tcp-service 192.168.0.7:80"
|
||||||
|
+ _, err := checkIPVSRule(addServerCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ addServerCmd := "IPVSadm -a -t 192.168.0.7:80"
|
||||||
|
+ _, err = checkIPVSRule(addServerCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestAddServerIpError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ addServerCommand := "IPVSadm --add-server --tcp-service 192.168.2.0.7:80 --real-server 192.168.0.4:80"
|
||||||
|
+ _, err := checkIPVSRule(addServerCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ addServerCmd := "IPVSadm -a -t 192.168.2.0.7:80 -r 192.168.0.4:80"
|
||||||
|
+ _, err = checkIPVSRule(addServerCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestAddServerPortError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ addServerCommand := "IPVSadm --add-server --tcp-service 192.168.0.7:99999 --real-server 192.168.0.4:80"
|
||||||
|
+ _, err := checkIPVSRule(addServerCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ addServerCmd := "IPVSadm -a -t 192.168.0.7:99999 -r 192.168.0.4:80"
|
||||||
|
+ _, err = checkIPVSRule(addServerCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestEditServerSuccessfully(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ editServerCommandTCP := "IPVSadm --edit-server --tcp-service 192.168.0.7:80 --real-server 192.168.0.4:80 --weight 100"
|
||||||
|
+ _, err := checkIPVSRule(editServerCommandTCP)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ editServerCommandUDP := "IPVSadm --edit-server --udp-service 192.168.0.7:80 --real-server 192.168.0.4:80 --weight 100"
|
||||||
|
+ _, err = checkIPVSRule(editServerCommandUDP)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ editServerCmd := "IPVSadm -e -t 192.168.0.7:80 -r 192.168.0.4:80 -w 100"
|
||||||
|
+ _, err = checkIPVSRule(editServerCmd)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestEditServerNoService(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ editServerCommand := "IPVSadm --edit-server --real-server 192.168.0.4:80"
|
||||||
|
+ _, err := checkIPVSRule(editServerCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ editServerCmd := "IPVSadm -e -r 192.168.0.4:80"
|
||||||
|
+ _, err = checkIPVSRule(editServerCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestEditServerNoServer(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ editServerCommand := "IPVSadm --edit-server --tcp-service 192.168.0.7:80"
|
||||||
|
+ _, err := checkIPVSRule(editServerCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ editServerCmd := "IPVSadm -e -t 192.168.0.7:80"
|
||||||
|
+ _, err = checkIPVSRule(editServerCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestEditServerIpError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ editServerCommand := "IPVSadm --edit-server --tcp-service 192.168.2.0.7:80 --real-server 192.168.0.4:80"
|
||||||
|
+ _, err := checkIPVSRule(editServerCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ editServerCmd := "IPVSadm -e -t 192.168.2.0.7:80 -r 192.168.0.4:80"
|
||||||
|
+ _, err = checkIPVSRule(editServerCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestEditServerPortError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ editServerCommand := "IPVSadm --edit-server --tcp-service 192.168.0.7:99999 --real-server 192.168.0.4:80"
|
||||||
|
+ _, err := checkIPVSRule(editServerCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ editServerCmd := "IPVSadm -e -t 192.168.0.7:99999 -r 192.168.0.4:80"
|
||||||
|
+ _, err = checkIPVSRule(editServerCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestDeleteServerSuccessfully(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ deleteServerCommandTCP := "IPVSadm --delete-server --tcp-service 192.168.0.7:80 --real-server 192.168.0.4:80"
|
||||||
|
+ _, err := checkIPVSRule(deleteServerCommandTCP)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ deleteServerCommandUDP := "IPVSadm --delete-server --udp-service 192.168.0.7:80 --real-server 192.168.0.4:80"
|
||||||
|
+ _, err = checkIPVSRule(deleteServerCommandUDP)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ deleteServerCmd := "IPVSadm -d -t 192.168.0.7:80 -r 192.168.0.4:80"
|
||||||
|
+ _, err = checkIPVSRule(deleteServerCmd)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestDeleteServerNoService(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ deleteServerCommand := "IPVSadm --delete-server --real-server 192.168.0.4:80"
|
||||||
|
+ _, err := checkIPVSRule(deleteServerCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ deleteServerCmd := "IPVSadm -d -r 192.168.0.4:80"
|
||||||
|
+ _, err = checkIPVSRule(deleteServerCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestDeleteServerNoServer(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ deleteServerCommand := "IPVSadm --delete-server --tcp-service 192.168.0.7:80"
|
||||||
|
+ _, err := checkIPVSRule(deleteServerCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ deleteServerCmd := "IPVSadm -d -t 192.168.0.7:80"
|
||||||
|
+ _, err = checkIPVSRule(deleteServerCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestDeleteServerIpError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ deleteServerCommand := "IPVSadm --delete-server --tcp-service 192.168.2.0.7:80 --real-server 192.168.0.4:80"
|
||||||
|
+ _, err := checkIPVSRule(deleteServerCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ deleteServerCmd := "IPVSadm -d -t 192.168.2.0.7:80 -r 192.168.0.4:80"
|
||||||
|
+ _, err = checkIPVSRule(deleteServerCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestDeleteServerPortError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ deleteServerCommand := "IPVSadm --delete-server --tcp-service 192.168.0.7:99999 --real-server 192.168.0.4:80"
|
||||||
|
+ _, err := checkIPVSRule(deleteServerCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ deleteServerCmd := "IPVSadm -d -t 192.168.0.7:99999 -r 192.168.0.4:80"
|
||||||
|
+ _, err = checkIPVSRule(deleteServerCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestListSuccessfully(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ listCommand := "IPVSadm --list"
|
||||||
|
+ _, err := checkIPVSRule(listCommand)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ listCmd := "IPVSadm -L"
|
||||||
|
+ _, err = checkIPVSRule(listCmd)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestSetSuccessfully(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ setCommand := "IPVSadm --set"
|
||||||
|
+ _, err := checkIPVSRule(setCommand)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestCleanupSuccessfully(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ cleanupCommand := "conntrack -D --orig-dst 192.168.0.4 --protonum tcp"
|
||||||
|
+ _, err := checkIPVSRule(cleanupCommand)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+ cleanupCmd := "conntrack -D -d 192.168.0.4 -p tcp"
|
||||||
|
+ _, err = checkIPVSRule(cleanupCmd)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestCleanupNoDestination(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ cleanupCommand := "conntrack -D --protonum tcp"
|
||||||
|
+ _, err := checkIPVSRule(cleanupCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ cleanupCmd := "conntrack -D -p tcp"
|
||||||
|
+ _, err = checkIPVSRule(cleanupCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestCleanupNoProtocol(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ cleanupCommand := "conntrack -D --orig-dst 192.168.0.4"
|
||||||
|
+ _, err := checkIPVSRule(cleanupCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ cleanupCmd := "conntrack -D -d 192.168.0.4"
|
||||||
|
+ _, err = checkIPVSRule(cleanupCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestCleanupIpError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ cleanupCommand := "conntrack -D --orig-dst 192.168.0.2.4 --protonum tcp"
|
||||||
|
+ _, err := checkIPVSRule(cleanupCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ cleanupCmd := "conntrack -D -d 192.168.0.2.4 -p tcp"
|
||||||
|
+ _, err = checkIPVSRule(cleanupCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestCleanupProtocolError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ cleanupCommand := "conntrack -D --orig-dst 192.168.0.4 --protonum ttcp"
|
||||||
|
+ _, err := checkIPVSRule(cleanupCommand)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+ cleanupCmd := "conntrack -D -d 192.168.0.4 -p ttcp"
|
||||||
|
+ _, err = checkIPVSRule(cleanupCmd)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestRestoreFileNil(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ stdin := strings.NewReader("")
|
||||||
|
+ fileContext, cnt, err := getRestoreFile(stdin)
|
||||||
|
+ assert.Equal(cnt, 0)
|
||||||
|
+ assert.Equal(fileContext, "")
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestRestoreFileError(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ stdin := strings.NewReader("aaa")
|
||||||
|
+ fileContext, cnt, err := getRestoreFile(stdin)
|
||||||
|
+ assert.Equal(cnt, 0)
|
||||||
|
+ assert.Equal(fileContext, "")
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestRestoreFile(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ stdin := strings.NewReader("-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m")
|
||||||
|
+ fileContext, cnt, err := getRestoreFile(stdin)
|
||||||
|
+ assert.Equal(cnt, 2)
|
||||||
|
+ assert.Equal(fileContext, "-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m")
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/api_test.go b/virtcontainers/api_test.go
|
||||||
|
index c01d47b8..2a5488f9 100644
|
||||||
|
--- a/virtcontainers/api_test.go
|
||||||
|
+++ b/virtcontainers/api_test.go
|
||||||
|
@@ -15,6 +15,7 @@ import (
|
||||||
|
"syscall"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
+ "github.com/kata-containers/agent/protocols/grpc"
|
||||||
|
ktu "github.com/kata-containers/runtime/pkg/katatestutils"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/persist"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
|
||||||
|
@@ -1745,3 +1746,38 @@ func TestCleanupContainer(t *testing.T) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func TestUpdateIPVSRule(t *testing.T) {
|
||||||
|
+ defer cleanUp()
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+
|
||||||
|
+ contID := "abc"
|
||||||
|
+ ctx := context.Background()
|
||||||
|
+ var ipvs = &grpc.UpdateIPVSRequest{
|
||||||
|
+ IPVSReq: "ipvsadm -A -t 192.168.0.7:80 -s rr -p -3000",
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ config := newTestSandboxConfigNoop()
|
||||||
|
+ p, _, err := createAndStartSandbox(ctx, config)
|
||||||
|
+ if p == nil || err != nil {
|
||||||
|
+ t.Fatal(err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ s, ok := p.(*Sandbox)
|
||||||
|
+ assert.True(ok)
|
||||||
|
+
|
||||||
|
+ contConfig := newTestContainerConfigNoop(contID)
|
||||||
|
+ c, err := p.CreateContainer(contConfig)
|
||||||
|
+ if c == nil || err != nil {
|
||||||
|
+ t.Fatal(err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ _, err = UpdateIPVSRule(ctx, s.id, ipvs)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ _, err = UpdateIPVSRule(ctx, "aaa", ipvs)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+
|
||||||
|
+ _, err = UpdateIPVSRule(ctx, "", ipvs)
|
||||||
|
+ assert.Error(err)
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go
|
||||||
|
index 18a5a0a6..4f9409a0 100644
|
||||||
|
--- a/virtcontainers/kata_agent_test.go
|
||||||
|
+++ b/virtcontainers/kata_agent_test.go
|
||||||
|
@@ -246,6 +246,10 @@ func (p *gRPCProxy) MemHotplugByProbe(ctx context.Context, req *pb.MemHotplugByP
|
||||||
|
return &gpb.Empty{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (p *gRPCProxy) UpdateIPVSRule(ctx context.Context, req *pb.UpdateIPVSRequest) (*pb.IPVSResponse, error) {
|
||||||
|
+ return &pb.IPVSResponse{}, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func gRPCRegister(s *grpc.Server, srv interface{}) {
|
||||||
|
switch g := srv.(type) {
|
||||||
|
case *gRPCProxy:
|
||||||
|
diff --git a/virtcontainers/pkg/vcmock/types.go b/virtcontainers/pkg/vcmock/types.go
|
||||||
|
index 43247ef9..610b4602 100644
|
||||||
|
--- a/virtcontainers/pkg/vcmock/types.go
|
||||||
|
+++ b/virtcontainers/pkg/vcmock/types.go
|
||||||
|
@@ -76,5 +76,5 @@ type VCMock struct {
|
||||||
|
UpdateRoutesFunc func(ctx context.Context, sandboxID string, routes []*vcTypes.Route) ([]*vcTypes.Route, error)
|
||||||
|
ListRoutesFunc func(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error)
|
||||||
|
CleanupContainerFunc func(ctx context.Context, sandboxID, containerID string, force bool) error
|
||||||
|
- UpdateIPVSRuleFunc func(ctx context.Context, sandboxID string, ipvs *grpc.UpdateIpvsRequest) (*grpc.IpvsResponse, error)
|
||||||
|
+ UpdateIPVSRuleFunc func(ctx context.Context, sandboxID string, ipvs *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error)
|
||||||
|
}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,501 @@
|
|||||||
|
From 5a220e9be1cfb03316a62aa00d2040638ba1a855 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 18 Aug 2020 15:15:52 +0800
|
||||||
|
Subject: [PATCH 38/50] pcie: using pcie-root-port driver to hotplug device in
|
||||||
|
aarch64
|
||||||
|
|
||||||
|
reason: Since qemu with "virt" machine type doesn't support hotplug
|
||||||
|
device in the pcie.0 root bus, so need to use add root port devices
|
||||||
|
to support hotplug pci device in aarch64.
|
||||||
|
|
||||||
|
we reuse the pcie_root_port config in the configuration.toml file to
|
||||||
|
set pcie_root_port device number when qemu process start.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
vendor/github.com/intel/govmm/qemu/qemu.go | 22 ++---
|
||||||
|
virtcontainers/persist/api/hypervisor.go | 5 +-
|
||||||
|
virtcontainers/qemu.go | 130 +++++++++++++++++++++++------
|
||||||
|
virtcontainers/qemu_arch_base.go | 9 ++
|
||||||
|
virtcontainers/types/pcie.go | 103 +++++++++++++++++++++++
|
||||||
|
5 files changed, 230 insertions(+), 39 deletions(-)
|
||||||
|
create mode 100644 virtcontainers/types/pcie.go
|
||||||
|
|
||||||
|
diff --git a/vendor/github.com/intel/govmm/qemu/qemu.go b/vendor/github.com/intel/govmm/qemu/qemu.go
|
||||||
|
index a5e5dfaf..3e7720b4 100644
|
||||||
|
--- a/vendor/github.com/intel/govmm/qemu/qemu.go
|
||||||
|
+++ b/vendor/github.com/intel/govmm/qemu/qemu.go
|
||||||
|
@@ -1252,6 +1252,8 @@ type PCIeRootPortDevice struct {
|
||||||
|
Chassis string // (slot, chassis) pair is mandatory and must be unique for each pcie-root-port, >=0, default is 0x00
|
||||||
|
Slot string // >=0, default is 0x00
|
||||||
|
|
||||||
|
+ Port string // Port number is the device index
|
||||||
|
+
|
||||||
|
Multifunction bool // true => "on", false => "off", default is off
|
||||||
|
Addr string // >=0, default is 0x00
|
||||||
|
|
||||||
|
@@ -1277,6 +1279,10 @@ func (b PCIeRootPortDevice) QemuParams(config *Config) []string {
|
||||||
|
|
||||||
|
deviceParams = append(deviceParams, fmt.Sprintf("%s,id=%s", driver, b.ID))
|
||||||
|
|
||||||
|
+ if b.Port != "" {
|
||||||
|
+ deviceParams = append(deviceParams, fmt.Sprintf("port=%s", b.Port))
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
if b.Bus == "" {
|
||||||
|
b.Bus = "pcie.0"
|
||||||
|
}
|
||||||
|
@@ -1287,20 +1293,10 @@ func (b PCIeRootPortDevice) QemuParams(config *Config) []string {
|
||||||
|
}
|
||||||
|
deviceParams = append(deviceParams, fmt.Sprintf("chassis=%s", b.Chassis))
|
||||||
|
|
||||||
|
- if b.Slot == "" {
|
||||||
|
- b.Slot = "0x00"
|
||||||
|
- }
|
||||||
|
- deviceParams = append(deviceParams, fmt.Sprintf("slot=%s", b.Slot))
|
||||||
|
-
|
||||||
|
- multifunction := "off"
|
||||||
|
if b.Multifunction {
|
||||||
|
- multifunction = "on"
|
||||||
|
- if b.Addr == "" {
|
||||||
|
- b.Addr = "0x00"
|
||||||
|
- }
|
||||||
|
- deviceParams = append(deviceParams, fmt.Sprintf("addr=%s", b.Addr))
|
||||||
|
+ deviceParams = append(deviceParams, "multifunction=on")
|
||||||
|
}
|
||||||
|
- deviceParams = append(deviceParams, fmt.Sprintf("multifunction=%v", multifunction))
|
||||||
|
+ deviceParams = append(deviceParams, fmt.Sprintf("addr=%s", b.Addr))
|
||||||
|
|
||||||
|
if b.BusReserve != "" {
|
||||||
|
deviceParams = append(deviceParams, fmt.Sprintf("bus-reserve=%s", b.BusReserve))
|
||||||
|
@@ -1337,7 +1333,7 @@ func (b PCIeRootPortDevice) Valid() bool {
|
||||||
|
if b.Pref64Reserve != "" && b.Pref32Reserve != "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
- if b.ID == "" {
|
||||||
|
+ if b.ID == "" || b.Port == "" || b.Bus == "" || b.Addr == ""{
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
diff --git a/virtcontainers/persist/api/hypervisor.go b/virtcontainers/persist/api/hypervisor.go
|
||||||
|
index 375fd56b..fd61b3c2 100644
|
||||||
|
--- a/virtcontainers/persist/api/hypervisor.go
|
||||||
|
+++ b/virtcontainers/persist/api/hypervisor.go
|
||||||
|
@@ -5,6 +5,8 @@
|
||||||
|
|
||||||
|
package persistapi
|
||||||
|
|
||||||
|
+import "github.com/kata-containers/runtime/virtcontainers/types"
|
||||||
|
+
|
||||||
|
// Bridge is a bridge where devices can be hot plugged
|
||||||
|
type Bridge struct {
|
||||||
|
// DeviceAddr contains information about devices plugged and its address in the bridge
|
||||||
|
@@ -35,7 +37,8 @@ type HypervisorState struct {
|
||||||
|
|
||||||
|
// Belows are qemu specific
|
||||||
|
// Refs: virtcontainers/qemu.go:QemuState
|
||||||
|
- Bridges []Bridge
|
||||||
|
+ Bridges []Bridge
|
||||||
|
+ PCIeRootPortsPool *types.PCIeRootPortPool
|
||||||
|
// HotpluggedCPUs is the list of CPUs that were hot-added
|
||||||
|
HotpluggedVCPUs []CPUDevice
|
||||||
|
HotpluggedMemory int
|
||||||
|
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
|
||||||
|
index c2b65376..a10c66fb 100644
|
||||||
|
--- a/virtcontainers/qemu.go
|
||||||
|
+++ b/virtcontainers/qemu.go
|
||||||
|
@@ -64,8 +64,9 @@ type CPUDevice struct {
|
||||||
|
|
||||||
|
// QemuState keeps Qemu's state
|
||||||
|
type QemuState struct {
|
||||||
|
- Bridges []types.Bridge
|
||||||
|
- ScsiBus *types.SCSIBus
|
||||||
|
+ Bridges []types.Bridge
|
||||||
|
+ ScsiBus *types.SCSIBus
|
||||||
|
+ PCIeRootPortsPool *types.PCIeRootPortPool
|
||||||
|
// HotpluggedCPUs is the list of CPUs that were hot-added
|
||||||
|
HotpluggedVCPUs []CPUDevice
|
||||||
|
HotpluggedMemory int
|
||||||
|
@@ -271,6 +272,9 @@ func (q *qemu) setup(id string, hypervisorConfig *HypervisorConfig) error {
|
||||||
|
|
||||||
|
q.state.HotplugVFIOOnRootBus = q.config.HotplugVFIOOnRootBus
|
||||||
|
q.state.PCIeRootPort = int(q.config.PCIeRootPort)
|
||||||
|
+ // init the PCIeRootPortsPool with pcie_root_port config value
|
||||||
|
+ q.state.PCIeRootPortsPool = &types.PCIeRootPortPool{}
|
||||||
|
+ q.state.PCIeRootPortsPool.Init(q.state.PCIeRootPort)
|
||||||
|
|
||||||
|
// The path might already exist, but in case of VM templating,
|
||||||
|
// we have to create it since the sandbox has not created it yet.
|
||||||
|
@@ -394,9 +398,18 @@ func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.I
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
- // Add bridges before any other devices. This way we make sure that
|
||||||
|
- // bridge gets the first available PCI address i.e bridgePCIStartAddr
|
||||||
|
- devices = q.arch.appendBridges(devices)
|
||||||
|
+ machine, err := q.getQemuMachine()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, nil, err
|
||||||
|
+ }
|
||||||
|
+ switch machine.Type {
|
||||||
|
+ case QemuVirt:
|
||||||
|
+ devices = q.arch.appendRootPorts(devices, q.state.PCIeRootPortsPool)
|
||||||
|
+ default:
|
||||||
|
+ // Add bridges before any other devices. This way we make sure that
|
||||||
|
+ // bridge gets the first available PCI address i.e bridgePCIStartAddr
|
||||||
|
+ devices = q.arch.appendBridges(devices)
|
||||||
|
+ }
|
||||||
|
|
||||||
|
devices, err = q.arch.appendConsole(devices, console)
|
||||||
|
if err != nil {
|
||||||
|
@@ -608,7 +621,7 @@ func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNa
|
||||||
|
// Add PCIe Root Port devices to hypervisor
|
||||||
|
// The pcie.0 bus do not support hot-plug, but PCIe device can be hot-plugged into PCIe Root Port.
|
||||||
|
// For more details, please see https://github.com/qemu/qemu/blob/master/docs/pcie.txt
|
||||||
|
- if hypervisorConfig.PCIeRootPort > 0 {
|
||||||
|
+ if hypervisorConfig.PCIeRootPort > 0 && hypervisorConfig.HypervisorMachineType == QemuQ35 {
|
||||||
|
qemuConfig.Devices = q.arch.appendPCIeRootPortDevice(qemuConfig.Devices, hypervisorConfig.PCIeRootPort)
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -1154,21 +1167,19 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev
|
||||||
|
}
|
||||||
|
case q.config.BlockDeviceDriver == config.VirtioBlock:
|
||||||
|
driver := "virtio-blk-pci"
|
||||||
|
- addr, bridge, err := q.arch.addDeviceToBridge(drive.ID, types.PCI)
|
||||||
|
+
|
||||||
|
+ addr, bus, pciAddr, err := q.getPciAddress(drive.ID, types.PCI)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
-
|
||||||
|
defer func() {
|
||||||
|
if err != nil {
|
||||||
|
- q.arch.removeDeviceFromBridge(drive.ID)
|
||||||
|
+ q.putPciAddress(drive.ID)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
+ drive.PCIAddr = pciAddr
|
||||||
|
|
||||||
|
- // PCI address is in the format bridge-addr/device-addr eg. "03/02"
|
||||||
|
- drive.PCIAddr = fmt.Sprintf("%02x", bridge.Addr) + "/" + addr
|
||||||
|
-
|
||||||
|
- if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID, romFile, 0, true, defaultDisableModern); err != nil {
|
||||||
|
+ if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bus, romFile, 0, true, defaultDisableModern); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
case q.config.BlockDeviceDriver == config.VirtioSCSI:
|
||||||
|
@@ -1249,7 +1260,7 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error
|
||||||
|
err = q.hotplugAddBlockDevice(drive, op, devID)
|
||||||
|
} else {
|
||||||
|
if q.config.BlockDeviceDriver == config.VirtioBlock {
|
||||||
|
- if err := q.arch.removeDeviceFromBridge(drive.ID); err != nil {
|
||||||
|
+ if err := q.putPciAddress(drive.ID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else if q.config.BlockDeviceDriver == config.VirtioSCSI {
|
||||||
|
@@ -1345,22 +1356,22 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
- addr, bridge, err := q.arch.addDeviceToBridge(devID, types.PCI)
|
||||||
|
+ addr, bus, _, err := q.getPciAddress(devID, types.PCI)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
if err != nil {
|
||||||
|
- q.arch.removeDeviceFromBridge(devID)
|
||||||
|
+ q.putPciAddress(devID)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
switch device.Type {
|
||||||
|
case config.VFIODeviceNormalType:
|
||||||
|
- return q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bridge.ID, romFile)
|
||||||
|
+ return q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bus, romFile)
|
||||||
|
case config.VFIODeviceMediatedType:
|
||||||
|
- return q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, addr, bridge.ID, romFile)
|
||||||
|
+ return q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, addr, bus, romFile)
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("Incorrect VFIO device type found")
|
||||||
|
}
|
||||||
|
@@ -1368,7 +1379,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro
|
||||||
|
q.Logger().WithField("dev-id", devID).Info("Start hot-unplug VFIO device")
|
||||||
|
|
||||||
|
if !q.state.HotplugVFIOOnRootBus {
|
||||||
|
- if err := q.arch.removeDeviceFromBridge(devID); err != nil {
|
||||||
|
+ if err := q.putPciAddress(devID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
@@ -1439,18 +1450,17 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
- addr, bridge, err := q.arch.addDeviceToBridge(tap.ID, types.PCI)
|
||||||
|
+ addr, bus, pciAddr, err := q.getPciAddress(tap.ID, types.PCI)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
if err != nil {
|
||||||
|
- q.arch.removeDeviceFromBridge(tap.ID)
|
||||||
|
+ q.putPciAddress(tap.ID)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
- pciAddr := fmt.Sprintf("%02x/%s", bridge.Addr, addr)
|
||||||
|
endpoint.SetPciAddr(pciAddr)
|
||||||
|
|
||||||
|
var machine govmmQemu.Machine
|
||||||
|
@@ -1459,14 +1469,14 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if machine.Type == QemuCCWVirtio {
|
||||||
|
- devNoHotplug := fmt.Sprintf("fe.%x.%x", bridge.Addr, addr)
|
||||||
|
+ devNoHotplug := fmt.Sprintf("fe.%x.%x", bus, addr)
|
||||||
|
return q.qmpMonitorCh.qmp.ExecuteNetCCWDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), devNoHotplug, int(q.config.NumVCPUs))
|
||||||
|
}
|
||||||
|
- return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bridge.ID, romFile, int(q.config.NumVCPUs), defaultDisableModern)
|
||||||
|
+ return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bus, romFile, int(q.config.NumVCPUs), defaultDisableModern)
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
- if err := q.arch.removeDeviceFromBridge(tap.ID); err != nil {
|
||||||
|
+ if err := q.putPciAddress(tap.ID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -2042,6 +2052,21 @@ func genericMemoryTopology(memoryMb, hostMemoryMb uint64, slots uint8, memoryOff
|
||||||
|
return memory
|
||||||
|
}
|
||||||
|
|
||||||
|
+func genericAppendRootPorts(devices []govmmQemu.Device, rootPorts *types.PCIeRootPortPool) []govmmQemu.Device {
|
||||||
|
+ for _, rp := range rootPorts.Items {
|
||||||
|
+ devices = append(devices, govmmQemu.PCIeRootPortDevice{
|
||||||
|
+ Port: rp.Port,
|
||||||
|
+ Bus: rp.Bus,
|
||||||
|
+ ID: rp.ID,
|
||||||
|
+ Chassis: strconv.Itoa(int(rp.Chassis)),
|
||||||
|
+ Multifunction: rp.Multifunction,
|
||||||
|
+ Addr: fmt.Sprintf("0x%s.0x%s", rp.Slot, rp.Function),
|
||||||
|
+ })
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return devices
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// genericAppendPCIeRootPort appends to devices the given pcie-root-port
|
||||||
|
func genericAppendPCIeRootPort(devices []govmmQemu.Device, number uint32, machineType string) []govmmQemu.Device {
|
||||||
|
var (
|
||||||
|
@@ -2241,6 +2266,7 @@ func (q *qemu) save() (s persistapi.HypervisorState) {
|
||||||
|
s.HotpluggedMemory = q.state.HotpluggedMemory
|
||||||
|
s.HotplugVFIOOnRootBus = q.state.HotplugVFIOOnRootBus
|
||||||
|
s.PCIeRootPort = q.state.PCIeRootPort
|
||||||
|
+ s.PCIeRootPortsPool = q.state.PCIeRootPortsPool
|
||||||
|
|
||||||
|
for _, bridge := range q.arch.getBridges() {
|
||||||
|
s.Bridges = append(s.Bridges, persistapi.Bridge{
|
||||||
|
@@ -2265,6 +2291,7 @@ func (q *qemu) load(s persistapi.HypervisorState) {
|
||||||
|
q.state.HotplugVFIOOnRootBus = s.HotplugVFIOOnRootBus
|
||||||
|
q.state.VirtiofsdPid = s.VirtiofsdPid
|
||||||
|
q.state.PCIeRootPort = s.PCIeRootPort
|
||||||
|
+ q.state.PCIeRootPortsPool = s.PCIeRootPortsPool
|
||||||
|
|
||||||
|
for _, bridge := range s.Bridges {
|
||||||
|
q.state.Bridges = append(q.state.Bridges, types.NewBridge(types.Type(bridge.Type), bridge.ID, bridge.DeviceAddr, bridge.Addr))
|
||||||
|
@@ -2302,3 +2329,56 @@ func (q *qemu) generateSocket(id string, useVsock bool) (interface{}, error) {
|
||||||
|
func (q *qemu) getMemorySize() uint32 {
|
||||||
|
return q.config.MemorySize + uint32(q.state.HotpluggedMemory)
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+// getPciAddress allocate the pci slot to hotplugged device and
|
||||||
|
+// return the pci slot address
|
||||||
|
+func (q *qemu) getPciAddress(devID string, t types.Type) (slot, bus, pciAddr string, err error) {
|
||||||
|
+ machine, err := q.getQemuMachine()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return "", "", "", err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ switch machine.Type {
|
||||||
|
+ case QemuVirt:
|
||||||
|
+ rp, err := q.state.PCIeRootPortsPool.AddDevice(devID)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return "", "", "", err
|
||||||
|
+ }
|
||||||
|
+ // PCIe Root Port only have one slot
|
||||||
|
+ slot = "0x0"
|
||||||
|
+ // pciAddr specifies the slot and function of the Root Port and the slot of the device
|
||||||
|
+ pciAddr = fmt.Sprintf("%s.%s/00", rp.Slot, rp.Function)
|
||||||
|
+ bus = rp.ID
|
||||||
|
+ default:
|
||||||
|
+ var bridge types.Bridge
|
||||||
|
+ slot, bridge, err = q.arch.addDeviceToBridge(devID, t)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return "", "", "", err
|
||||||
|
+ }
|
||||||
|
+ bus = bridge.ID
|
||||||
|
+ // PCI address is in the format bridge-addr.0/device-addr eg. "03.0/02"
|
||||||
|
+ pciAddr = fmt.Sprintf("%02x.0", bridge.Addr) + "/" + slot
|
||||||
|
+ }
|
||||||
|
+ return slot, bus, pciAddr, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (q *qemu) putPciAddress(devID string) error {
|
||||||
|
+ machine, err := q.getQemuMachine()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ switch machine.Type {
|
||||||
|
+ case QemuVirt:
|
||||||
|
+ err := q.state.PCIeRootPortsPool.RemoveDevice(devID)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ default:
|
||||||
|
+ if err := q.arch.removeDeviceFromBridge(devID); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/qemu_arch_base.go b/virtcontainers/qemu_arch_base.go
|
||||||
|
index 9d72dd09..cb045530 100644
|
||||||
|
--- a/virtcontainers/qemu_arch_base.go
|
||||||
|
+++ b/virtcontainers/qemu_arch_base.go
|
||||||
|
@@ -130,6 +130,9 @@ type qemuArch interface {
|
||||||
|
|
||||||
|
// appendPCIeRootPortDevice appends a pcie-root-port device to pcie.0 bus
|
||||||
|
appendPCIeRootPortDevice(devices []govmmQemu.Device, number uint32) []govmmQemu.Device
|
||||||
|
+
|
||||||
|
+ // appendRootPorts appends a pcie-root-port device to devices when qemu machine type is "virt"
|
||||||
|
+ appendRootPorts(devices []govmmQemu.Device, rootPorts *types.PCIeRootPortPool) []govmmQemu.Device
|
||||||
|
}
|
||||||
|
|
||||||
|
type qemuArchBase struct {
|
||||||
|
@@ -766,3 +769,9 @@ func (q *qemuArchBase) addBridge(b types.Bridge) {
|
||||||
|
func (q *qemuArchBase) appendPCIeRootPortDevice(devices []govmmQemu.Device, number uint32) []govmmQemu.Device {
|
||||||
|
return genericAppendPCIeRootPort(devices, number, q.machineType)
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+// appendRootPorts appends a pcie-root-port device to devices when qemu machine type is "virt"
|
||||||
|
+// which is different appendPCIeRootPortDevice function
|
||||||
|
+func (q *qemuArchBase) appendRootPorts(devices []govmmQemu.Device, rootPorts *types.PCIeRootPortPool) []govmmQemu.Device {
|
||||||
|
+ return genericAppendRootPorts(devices, rootPorts)
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/types/pcie.go b/virtcontainers/types/pcie.go
|
||||||
|
new file mode 100644
|
||||||
|
index 00000000..83eb6944
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/virtcontainers/types/pcie.go
|
||||||
|
@@ -0,0 +1,103 @@
|
||||||
|
+package types
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "fmt"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+const (
|
||||||
|
+ maxRootPortsCapacity = 25
|
||||||
|
+ slotPerDevice = 7
|
||||||
|
+
|
||||||
|
+ // PCIeRootBus is "pcie.0"
|
||||||
|
+ PCIeRootBus = "pcie.0"
|
||||||
|
+
|
||||||
|
+ // startPort specifies the start port of pcie-root-port
|
||||||
|
+ // for the first slot of "pcie.0" is reserved, so the 1~7 ports are also reserved
|
||||||
|
+ startPort = 8
|
||||||
|
+ funcNumPerSlot = 8
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+// PCIeRootPort describe the PCIe Root Port
|
||||||
|
+type PCIeRootPort struct {
|
||||||
|
+ // DeviceID specify the device hotplug on the Root Port
|
||||||
|
+ DeviceID string
|
||||||
|
+
|
||||||
|
+ // Port number is the Root Port index
|
||||||
|
+ Port string
|
||||||
|
+
|
||||||
|
+ // Bus number where the Root Port is plugged, typically pcie.0
|
||||||
|
+ Bus string
|
||||||
|
+
|
||||||
|
+ // ID is used to identify the pcie-root-port in qemu
|
||||||
|
+ ID string
|
||||||
|
+
|
||||||
|
+ // Slot specifies slot address of Root Port
|
||||||
|
+ Slot string
|
||||||
|
+
|
||||||
|
+ // Function specifies function of Root Port
|
||||||
|
+ Function string
|
||||||
|
+
|
||||||
|
+ // Chassis number
|
||||||
|
+ Chassis uint32
|
||||||
|
+
|
||||||
|
+ // Multifunction is used to specify the pcie-root-port is multifunction supported
|
||||||
|
+ Multifunction bool
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// PCIeRootPortPool describe a set of PCIe Root Ports
|
||||||
|
+type PCIeRootPortPool struct {
|
||||||
|
+ // Items contains information about devices plugged and number limit
|
||||||
|
+ Items []*PCIeRootPort
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// Init Initialized the PCIeRootPortPool instance
|
||||||
|
+func (rp *PCIeRootPortPool) Init(number int) {
|
||||||
|
+ if number == 0 || number > maxRootPortsCapacity {
|
||||||
|
+ number = maxRootPortsCapacity
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ for i := 0; i < number; i++ {
|
||||||
|
+ dev := &PCIeRootPort{
|
||||||
|
+ DeviceID: "",
|
||||||
|
+ Port: fmt.Sprintf("0x%x", startPort+i),
|
||||||
|
+ Bus: PCIeRootBus,
|
||||||
|
+ ID: fmt.Sprintf("pci.%d", i+1),
|
||||||
|
+ Chassis: uint32(i + 1),
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ major := i / funcNumPerSlot
|
||||||
|
+ minor := i % funcNumPerSlot
|
||||||
|
+ dev.Multifunction = false
|
||||||
|
+ if minor == 0 {
|
||||||
|
+ dev.Multifunction = true
|
||||||
|
+ }
|
||||||
|
+ dev.Slot = fmt.Sprintf("%02x", major+1)
|
||||||
|
+ dev.Function = fmt.Sprintf("%x", minor)
|
||||||
|
+
|
||||||
|
+ rp.Items = append(rp.Items, dev)
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// AddDevice add a device to the PCIeRootPortPool
|
||||||
|
+func (rp *PCIeRootPortPool) AddDevice(devID string) (*PCIeRootPort, error) {
|
||||||
|
+ for _, it := range rp.Items {
|
||||||
|
+ if it.DeviceID == "" {
|
||||||
|
+ it.DeviceID = devID
|
||||||
|
+ return it, nil
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ return nil, fmt.Errorf("Unable to hot plug device on Root Ports: there are not empty slots")
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// RemoveDevice remove a device from the PCIeRootPortPool
|
||||||
|
+func (rp *PCIeRootPortPool) RemoveDevice(devID string) error {
|
||||||
|
+ for _, it := range rp.Items {
|
||||||
|
+ if it.DeviceID == devID {
|
||||||
|
+ // free address to re-use the same slot with other devices
|
||||||
|
+ it.DeviceID = ""
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return fmt.Errorf("Unable to hot unplug device %s: not present on Root Port", devID)
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,245 @@
|
|||||||
|
From a36c9857447aaf22628af1ef01406a916436133b Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 18 Aug 2020 22:00:04 +0800
|
||||||
|
Subject: [PATCH 39/50] storage: add storage common functions and structs
|
||||||
|
|
||||||
|
reason:
|
||||||
|
1. add storage.go and storage_spec.go
|
||||||
|
2. provide funcs for mount nfs and gpath
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/storage/storage.go | 115 +++++++++++++++++++++++++++++++++
|
||||||
|
virtcontainers/storage/storage_spec.go | 98 ++++++++++++++++++++++++++++
|
||||||
|
2 files changed, 213 insertions(+)
|
||||||
|
create mode 100644 virtcontainers/storage/storage.go
|
||||||
|
create mode 100644 virtcontainers/storage/storage_spec.go
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/storage/storage.go b/virtcontainers/storage/storage.go
|
||||||
|
new file mode 100644
|
||||||
|
index 00000000..77ef994f
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/virtcontainers/storage/storage.go
|
||||||
|
@@ -0,0 +1,115 @@
|
||||||
|
+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
||||||
|
+// SPDX-License-Identifier: Apache-2.0
|
||||||
|
+// Description: common functions
|
||||||
|
+// Author: licuifang
|
||||||
|
+// Create: 2019-07-14
|
||||||
|
+
|
||||||
|
+// Package storage provides functions for mounting
|
||||||
|
+package storage
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "fmt"
|
||||||
|
+ "path/filepath"
|
||||||
|
+ "regexp"
|
||||||
|
+ "strings"
|
||||||
|
+
|
||||||
|
+ "github.com/kata-containers/agent/protocols/grpc"
|
||||||
|
+ "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+var (
|
||||||
|
+ // simple regular expressions for "nfs-server.com:/aaa/bbb/ccc/" or "192.168.1.1:/remote/path"
|
||||||
|
+ regUrl = regexp.MustCompile(`^((([a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,6})|((?:(?:25[0-5]|2[0-4]\d|[01]?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d?\d))):/[A-Za-z0-9_./-]{0,1000}$`)
|
||||||
|
+
|
||||||
|
+ defaultMountOption = "nolock"
|
||||||
|
+ storageMountOptionKey = "mount_op"
|
||||||
|
+ storageCustomOptionKey = "custom_op"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+type RemoteStorage struct {
|
||||||
|
+ Source string
|
||||||
|
+ Dest string
|
||||||
|
+ Options map[string][]string
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (n *RemoteStorage) Validate(volumeType string) error {
|
||||||
|
+ if n.Source == "" || n.Dest == "" {
|
||||||
|
+ return fmt.Errorf("the source and dest of storage cannot be empty")
|
||||||
|
+ }
|
||||||
|
+ switch volumeType {
|
||||||
|
+ case NFS:
|
||||||
|
+ chk := regUrl.FindAllString(n.Source, -1)
|
||||||
|
+ if chk == nil {
|
||||||
|
+ return fmt.Errorf("invalid url for nfs")
|
||||||
|
+ }
|
||||||
|
+ case GPATHFS:
|
||||||
|
+ if !filepath.IsAbs(n.Source) {
|
||||||
|
+ return fmt.Errorf("invalid gpath")
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (n *RemoteStorage) GetGrpcStorageAndAppendMount(volumeType string, vmBasePath string, spec *specs.Spec, sandboxId string) *grpc.Storage {
|
||||||
|
+ var (
|
||||||
|
+ grpcStorag *grpc.Storage
|
||||||
|
+ vmPath string
|
||||||
|
+ )
|
||||||
|
+ switch volumeType {
|
||||||
|
+ case NFS:
|
||||||
|
+ // source of sfs like remote-nfs.com:/share-53ee51a1
|
||||||
|
+ // source of sfs with subpath like remote-nfs.com:/share-53ee51a1/aaa/bbb
|
||||||
|
+ // source of sfs-turbo like ip:/
|
||||||
|
+ // source of sfs-turbo with subpath like ip://aaa or ip://aaa/bbb
|
||||||
|
+ // here we only get the origin source such as remote-nfs.com:/share-53ee51a1 or ip:/ as the Source of grpcStorage
|
||||||
|
+ item := strings.Split(n.Source, ":")
|
||||||
|
+ if len(item) != 2 {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+ vmPath = filepath.Join(vmBasePath, sandboxId, item[0], item[1])
|
||||||
|
+ parts := strings.SplitAfter(item[1], "/")
|
||||||
|
+ if len(parts) > 2 {
|
||||||
|
+ n.Source = item[0] + ":" + parts[0] + parts[1]
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ grpcStorag = &grpc.Storage{
|
||||||
|
+ Driver: NFS,
|
||||||
|
+ Source: n.Source,
|
||||||
|
+ MountPoint: vmPath,
|
||||||
|
+ }
|
||||||
|
+ grpcStorag.Options = append(grpcStorag.Options, defaultMountOption)
|
||||||
|
+ case GPATHFS:
|
||||||
|
+ vmPath = n.Source
|
||||||
|
+ grpcStorag = &grpc.Storage{
|
||||||
|
+ Driver: GPATHFS,
|
||||||
|
+ Source: vmPath,
|
||||||
|
+ MountPoint: n.Dest,
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ if custumOpts, ok := n.Options[storageCustomOptionKey]; ok {
|
||||||
|
+ for _, custumOpt := range custumOpts {
|
||||||
|
+ grpcStorag.Options = append(grpcStorag.Options, custumOpt)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ mnt := specs.Mount{
|
||||||
|
+ Source: vmPath,
|
||||||
|
+ Destination: n.Dest,
|
||||||
|
+ }
|
||||||
|
+ if mountOpts, ok := n.Options[storageMountOptionKey]; ok {
|
||||||
|
+ for _, mountOpt := range mountOpts {
|
||||||
|
+ if mountOpt == "shared" {
|
||||||
|
+ // in order to support "shared" propagation for guestpath mounting to container directory
|
||||||
|
+ // we should set rootfspropagation shared
|
||||||
|
+ spec.Linux.RootfsPropagation = "shared"
|
||||||
|
+ }
|
||||||
|
+ mnt.Options = append(mnt.Options, mountOpt)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ mnt.Options = append(mnt.Options, "rbind")
|
||||||
|
+ mnt.Type = "bind"
|
||||||
|
+ spec.Mounts = append(spec.Mounts, mnt)
|
||||||
|
+
|
||||||
|
+ return grpcStorag
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/storage/storage_spec.go b/virtcontainers/storage/storage_spec.go
|
||||||
|
new file mode 100644
|
||||||
|
index 00000000..8e866b8d
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/virtcontainers/storage/storage_spec.go
|
||||||
|
@@ -0,0 +1,98 @@
|
||||||
|
+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
||||||
|
+// Description: common functions
|
||||||
|
+// Author: caihaomin c00416947
|
||||||
|
+// Create: 2019-05-05
|
||||||
|
+
|
||||||
|
+package storage
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "encoding/json"
|
||||||
|
+ "fmt"
|
||||||
|
+
|
||||||
|
+ "github.com/kata-containers/agent/protocols/grpc"
|
||||||
|
+ "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+const (
|
||||||
|
+ NFS = "nfs"
|
||||||
|
+ GPATHFS = "gpath"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+// STORAGES specifies the storage type and is supported
|
||||||
|
+var STORAGES = map[string]bool{
|
||||||
|
+ NFS: true,
|
||||||
|
+ GPATHFS: true,
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+type StorageSpec struct {
|
||||||
|
+ // StorageType specifies the type of storage passing down
|
||||||
|
+ StorageType string `json:"storage_type,omitempty"`
|
||||||
|
+
|
||||||
|
+ Source string `json:"source,omitempty"`
|
||||||
|
+ Destination string `json:"dest,omitempty"`
|
||||||
|
+
|
||||||
|
+ // Options specifies the options of storage type
|
||||||
|
+ Options map[string][]string `json:"options,omitempty"`
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *StorageSpec) generateInstances() StorageOperation {
|
||||||
|
+ var storageOpt StorageOperation
|
||||||
|
+ storageOpt = &RemoteStorage{
|
||||||
|
+ Source: s.Source,
|
||||||
|
+ Dest: s.Destination,
|
||||||
|
+ Options: s.Options,
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return storageOpt
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func ValidateStorageValue(value string) error {
|
||||||
|
+ var storageSpecs []StorageSpec
|
||||||
|
+ if err := json.Unmarshal([]byte(value), &storageSpecs); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ var storageOpt []StorageOperation
|
||||||
|
+ for _, item := range storageSpecs {
|
||||||
|
+ if supported, ok := STORAGES[item.StorageType]; !ok || !supported {
|
||||||
|
+ return fmt.Errorf("type %s of storage is not supported", item.StorageType)
|
||||||
|
+ }
|
||||||
|
+ storageItem := item.generateInstances()
|
||||||
|
+ if err := storageItem.Validate(item.StorageType); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ storageOpt = append(storageOpt, storageItem)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func GetGrpcStorageAndAppendMount(vmBasePath, value string, spec *specs.Spec, sandboxId string) []*grpc.Storage {
|
||||||
|
+ var (
|
||||||
|
+ grpcStorages []*grpc.Storage
|
||||||
|
+ storageSpec []StorageSpec
|
||||||
|
+ )
|
||||||
|
+
|
||||||
|
+ // the value has been validated before, and there is no need to judge the return value here
|
||||||
|
+ json.Unmarshal([]byte(value), &storageSpec)
|
||||||
|
+ for _, item := range storageSpec {
|
||||||
|
+ storageItem := item.generateInstances()
|
||||||
|
+ grpcStorages = append(grpcStorages, storageItem.GetGrpcStorageAndAppendMount(item.StorageType, vmBasePath, spec, sandboxId))
|
||||||
|
+ }
|
||||||
|
+ return grpcStorages
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+type StorageOperation interface {
|
||||||
|
+ Validate(volumeType string) error
|
||||||
|
+ GetGrpcStorageAndAppendMount(volumeType string, vmBasePath string, spec *specs.Spec, sandboxId string) *grpc.Storage
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+type FakeStorage struct{}
|
||||||
|
+
|
||||||
|
+func (f *FakeStorage) Validate() error {
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (f *FakeStorage) GetGrpcStorageAndAppendMount(vmBasePath string, spec *specs.Spec) *grpc.Storage {
|
||||||
|
+ return &grpc.Storage{}
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
221
runtime/patches/0040-storage-add-go-tests-for-storage.patch
Normal file
221
runtime/patches/0040-storage-add-go-tests-for-storage.patch
Normal file
@ -0,0 +1,221 @@
|
|||||||
|
From 2e32e2c156a134605de42c53ef77366ac73f2614 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 18 Aug 2020 22:01:58 +0800
|
||||||
|
Subject: [PATCH 40/50] storage: add go tests for storage
|
||||||
|
|
||||||
|
reason: add go tests file storage_test.go and storage_spec_test.go
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/storage/storage_spec_test.go | 151 ++++++++++++++++++++++++++++
|
||||||
|
virtcontainers/storage/storage_test.go | 40 ++++++++
|
||||||
|
2 files changed, 191 insertions(+)
|
||||||
|
create mode 100644 virtcontainers/storage/storage_spec_test.go
|
||||||
|
create mode 100644 virtcontainers/storage/storage_test.go
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/storage/storage_spec_test.go b/virtcontainers/storage/storage_spec_test.go
|
||||||
|
new file mode 100644
|
||||||
|
index 00000000..f638245a
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/virtcontainers/storage/storage_spec_test.go
|
||||||
|
@@ -0,0 +1,151 @@
|
||||||
|
+/*
|
||||||
|
+Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
||||||
|
+Description: common functions test
|
||||||
|
+Author: yangfeiyu
|
||||||
|
+Create: 2019-07-24
|
||||||
|
+*/
|
||||||
|
+
|
||||||
|
+package storage
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "encoding/json"
|
||||||
|
+ "testing"
|
||||||
|
+
|
||||||
|
+ "github.com/stretchr/testify/assert"
|
||||||
|
+ "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+func TestValidateStorageValue(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ //invalid storage type,not NFS or GPATHS
|
||||||
|
+ spec := &StorageSpec{
|
||||||
|
+ StorageType: "invalidStorage",
|
||||||
|
+ Source: "sfs.com:/remote/path",
|
||||||
|
+ Destination: "/opt/",
|
||||||
|
+ }
|
||||||
|
+ data, err := json.Marshal(spec)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ err = ValidateStorageValue(string(data))
|
||||||
|
+ assert.Error(err, "StorageType is not NFS or GPATHS,it should return error")
|
||||||
|
+
|
||||||
|
+ //case 1: NFS
|
||||||
|
+ specNFS := []StorageSpec{
|
||||||
|
+ {
|
||||||
|
+ StorageType: NFS,
|
||||||
|
+ Source: "sfs.com:/remote/path",
|
||||||
|
+ Destination: "/opt/"},
|
||||||
|
+ }
|
||||||
|
+ data, err = json.Marshal(specNFS)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ err = ValidateStorageValue(string(data))
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ // case2: invalid source domain address
|
||||||
|
+ specNFS = []StorageSpec{
|
||||||
|
+ {
|
||||||
|
+ StorageType: NFS,
|
||||||
|
+ Source: "sfs..com:/remote/path",
|
||||||
|
+ Destination: "/opt/"},
|
||||||
|
+ }
|
||||||
|
+ data, err = json.Marshal(specNFS)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ err = ValidateStorageValue(string(data))
|
||||||
|
+ assert.Error(err)
|
||||||
|
+
|
||||||
|
+ // case 3: nfs source is valid ip address
|
||||||
|
+ ipSpecNFS := []StorageSpec{
|
||||||
|
+ {
|
||||||
|
+ StorageType: NFS,
|
||||||
|
+ Source: "192.168.18.147:/remote/path",
|
||||||
|
+ Destination: "/tmp/sfsturbo0",
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ data, err = json.Marshal(ipSpecNFS)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ err = ValidateStorageValue(string(data))
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ // case 4: invalid ip address for nfs source
|
||||||
|
+ ipSpecNFS = []StorageSpec{
|
||||||
|
+ {
|
||||||
|
+ StorageType: NFS,
|
||||||
|
+ Source: "192.168.18.300:/remote/path",
|
||||||
|
+ Destination: "/tmp/sfsturbo0",
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ data, err = json.Marshal(ipSpecNFS)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ err = ValidateStorageValue(string(data))
|
||||||
|
+ assert.Error(err)
|
||||||
|
+
|
||||||
|
+ // case 5: validate ip address and source remote path is /
|
||||||
|
+ ipSpecNFS = []StorageSpec{
|
||||||
|
+ {
|
||||||
|
+ StorageType: NFS,
|
||||||
|
+ Source: "192.168.18.3:/",
|
||||||
|
+ Destination: "/tmp/sfsturbo0",
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ data, err = json.Marshal(ipSpecNFS)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ err = ValidateStorageValue(string(data))
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ // case 6: invalid ip address 192.168.18.147.11
|
||||||
|
+ ipSpecNFS = []StorageSpec{
|
||||||
|
+ {
|
||||||
|
+ StorageType: NFS,
|
||||||
|
+ Source: "192.168.18.147.11:/",
|
||||||
|
+ Destination: "/tmp/sfsturbo0",
|
||||||
|
+ },
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ data, err = json.Marshal(ipSpecNFS)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ err = ValidateStorageValue(string(data))
|
||||||
|
+ assert.Error(err)
|
||||||
|
+
|
||||||
|
+ //GPATHS
|
||||||
|
+ specGPATHFS := []StorageSpec{
|
||||||
|
+ {
|
||||||
|
+ StorageType: GPATHFS,
|
||||||
|
+ Source: "/remote/path",
|
||||||
|
+ Destination: "/opt/"},
|
||||||
|
+ }
|
||||||
|
+ data, err = json.Marshal(specGPATHFS)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
+
|
||||||
|
+ err = ValidateStorageValue(string(data))
|
||||||
|
+ if STORAGES[GPATHFS] == true {
|
||||||
|
+ assert.NoError(err, "StorageType GPATHFS is valid,it should return no error")
|
||||||
|
+ } else {
|
||||||
|
+ assert.Error(err, "StorageType GPATHFS is invalid,it should return error")
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestGetGrpcStorageAndAppendMount(t *testing.T) {
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ var spec specs.Spec
|
||||||
|
+ sandboxId := "7f3b5e32cc7fe757e1b69f7a005ca1971def5011e730c82535783630fe24b318"
|
||||||
|
+ vmBasePath := "/tmp"
|
||||||
|
+
|
||||||
|
+ // storageType nfs with legal source and dest will succeed
|
||||||
|
+ storageSpec := "[{\"storage_type\":\"nfs\",\"source\":\"sfs.com:/remote/path\",\"dest\":\"/opt/nfs\"}]"
|
||||||
|
+ grpcStorages := GetGrpcStorageAndAppendMount(vmBasePath,storageSpec,&spec,sandboxId)
|
||||||
|
+ assert.NotNil(grpcStorages[0])
|
||||||
|
+
|
||||||
|
+ // storageType nfs with illegal source and dest will fail
|
||||||
|
+ storageSpec = "[{\"storage_type\":\"nfs\",\"source\":\"sfs.com\",\"dest\":\"/opt/nfs\"}]"
|
||||||
|
+ grpcStorages = GetGrpcStorageAndAppendMount(vmBasePath,storageSpec,&spec,sandboxId)
|
||||||
|
+ assert.Nil(grpcStorages[0])
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/storage/storage_test.go b/virtcontainers/storage/storage_test.go
|
||||||
|
new file mode 100644
|
||||||
|
index 00000000..c9ca4926
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/virtcontainers/storage/storage_test.go
|
||||||
|
@@ -0,0 +1,40 @@
|
||||||
|
+/*
|
||||||
|
+Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
||||||
|
+Description: common functions test
|
||||||
|
+Author: yangfeiyu
|
||||||
|
+Create: 2019-07-24
|
||||||
|
+*/
|
||||||
|
+
|
||||||
|
+package storage
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "testing"
|
||||||
|
+
|
||||||
|
+ "github.com/stretchr/testify/assert"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+func TestValidate(t *testing.T) {
|
||||||
|
+ //NFS valid
|
||||||
|
+ assert := assert.New(t)
|
||||||
|
+ remoteStorage := &RemoteStorage{
|
||||||
|
+ Source: "sfs.com:/remote/path",
|
||||||
|
+ Dest: "/opt/",
|
||||||
|
+ }
|
||||||
|
+ err := remoteStorage.Validate(NFS)
|
||||||
|
+ assert.NoError(err, "NFS url is valid,it should return no error")
|
||||||
|
+
|
||||||
|
+ //NFS invalid
|
||||||
|
+ remoteStorage.Source = "sfs.com/../remote/path"
|
||||||
|
+ err = remoteStorage.Validate(NFS)
|
||||||
|
+ assert.Error(err, "NFS url is invalid,it should return error")
|
||||||
|
+
|
||||||
|
+ //GPATHS valid
|
||||||
|
+ remoteStorage.Source = "/path/in/vm"
|
||||||
|
+ err = remoteStorage.Validate(GPATHFS)
|
||||||
|
+ assert.NoError(err, "GPATHFS is valid,it should return no error")
|
||||||
|
+
|
||||||
|
+ //GPATHS invalid
|
||||||
|
+ remoteStorage.Source = "./../path/in/../vm"
|
||||||
|
+ err = remoteStorage.Validate(GPATHFS)
|
||||||
|
+ assert.Error(err, "GPATHFS is invalid,it should return error")
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,115 @@
|
|||||||
|
From 31e07f1b6cbf361783c4d7adf9e4b8da30c67384 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 18 Aug 2020 22:05:25 +0800
|
||||||
|
Subject: [PATCH 41/50] storage: mount nfs and gpath with given annotation
|
||||||
|
|
||||||
|
reason: when run container with annotation about storage spec,
|
||||||
|
prepare basic info in kata-runtime
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/kata_agent.go | 14 +++++++++++++-
|
||||||
|
virtcontainers/pkg/annotations/annotations.go | 3 +++
|
||||||
|
virtcontainers/pkg/oci/utils.go | 16 ++++++++++++++++
|
||||||
|
3 files changed, 32 insertions(+), 1 deletion(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
|
||||||
|
index d82a7f2d..ac64817a 100644
|
||||||
|
--- a/virtcontainers/kata_agent.go
|
||||||
|
+++ b/virtcontainers/kata_agent.go
|
||||||
|
@@ -30,10 +30,11 @@ import (
|
||||||
|
ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/pkg/rootless"
|
||||||
|
vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
|
||||||
|
- "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/storage"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/store"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/types"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||||
|
"github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
opentracing "github.com/opentracing/opentracing-go"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
@@ -1427,6 +1428,9 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process,
|
||||||
|
localStorages := k.handleLocalStorage(ociSpec.Mounts, sandbox.id, c.rootfsSuffix)
|
||||||
|
ctrStorages = append(ctrStorages, localStorages...)
|
||||||
|
|
||||||
|
+ remoteStoragtes := k.handleRemoteStorage(ociSpec, sandbox.id)
|
||||||
|
+ ctrStorages = append(ctrStorages, remoteStoragtes...)
|
||||||
|
+
|
||||||
|
// We replace all OCI mount sources that match our container mount
|
||||||
|
// with the right source path (The guest one).
|
||||||
|
if err = k.replaceOCIMountSource(ociSpec, newMounts); err != nil {
|
||||||
|
@@ -1510,6 +1514,14 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process,
|
||||||
|
k.state.URL, consoleURL, c.config.Cmd, createNSList, enterNSList)
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (k *kataAgent) handleRemoteStorage(spec *specs.Spec, sandboxId string) []*grpc.Storage {
|
||||||
|
+ if value, ok := spec.Annotations[vcAnnotations.StorageSpecTypeKey]; ok {
|
||||||
|
+ return storage.GetGrpcStorageAndAppendMount(kataGuestStorageDir, value, spec, sandboxId)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return []*grpc.Storage{}
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// handleEphemeralStorage handles ephemeral storages by
|
||||||
|
// creating a Storage from corresponding source of the mount point
|
||||||
|
func (k *kataAgent) handleEphemeralStorage(mounts []specs.Mount) []*grpc.Storage {
|
||||||
|
diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go
|
||||||
|
index 903c7f03..e50a697c 100644
|
||||||
|
--- a/virtcontainers/pkg/annotations/annotations.go
|
||||||
|
+++ b/virtcontainers/pkg/annotations/annotations.go
|
||||||
|
@@ -68,6 +68,9 @@ const (
|
||||||
|
// AssetHashType is the hash type used for assets verification
|
||||||
|
AssetHashType = kataAnnotationsPrefix + "asset_hash_type"
|
||||||
|
|
||||||
|
+ // StorageSpecTypeKey is the annotation key to fetch storage_spec
|
||||||
|
+ StorageSpecTypeKey = kataAnnotationsPrefix + "storage_spec"
|
||||||
|
+
|
||||||
|
//
|
||||||
|
// Generic annotations
|
||||||
|
//
|
||||||
|
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
||||||
|
index 948bd3cb..d032227e 100644
|
||||||
|
--- a/virtcontainers/pkg/oci/utils.go
|
||||||
|
+++ b/virtcontainers/pkg/oci/utils.go
|
||||||
|
@@ -22,6 +22,7 @@ import (
|
||||||
|
exp "github.com/kata-containers/runtime/virtcontainers/experimental"
|
||||||
|
vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
|
||||||
|
dockershimAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations/dockershim"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/storage"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/types"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/utils"
|
||||||
|
specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
@@ -340,6 +341,17 @@ func SandboxID(spec specs.Spec) (string, error) {
|
||||||
|
return "", fmt.Errorf("Could not find sandbox ID")
|
||||||
|
}
|
||||||
|
|
||||||
|
+func validateStorageSpec(spec specs.Spec) error {
|
||||||
|
+ if storageSpec, ok := spec.Annotations[vcAnnotations.StorageSpecTypeKey]; ok {
|
||||||
|
+ err := storage.ValidateStorageValue(storageSpec)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func addAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) error {
|
||||||
|
addAssetAnnotations(ocispec, config)
|
||||||
|
if err := addHypervisorConfigOverrides(ocispec, config); err != nil {
|
||||||
|
@@ -873,6 +885,10 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid, c
|
||||||
|
// ContainerConfig converts an OCI compatible runtime configuration
|
||||||
|
// file to a virtcontainers container configuration structure.
|
||||||
|
func ContainerConfig(ocispec specs.Spec, bundlePath, cid, console string, detach bool) (vc.ContainerConfig, error) {
|
||||||
|
+ err := validateStorageSpec(ocispec)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return vc.ContainerConfig{}, err
|
||||||
|
+ }
|
||||||
|
rootfs := vc.RootFs{Target: ocispec.Root.Path, Mounted: true}
|
||||||
|
if !filepath.IsAbs(rootfs.Target) {
|
||||||
|
rootfs.Target = filepath.Join(bundlePath, ocispec.Root.Path)
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,38 @@
|
|||||||
|
From 37372be961528471d418bbe066d88a010ad0513c Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 18 Aug 2020 22:12:59 +0800
|
||||||
|
Subject: [PATCH 42/50] kata-runtime: do not ignore updateInterface return
|
||||||
|
error
|
||||||
|
|
||||||
|
reason: If send UpdateInterfaceRequest to kata-agent and return the
|
||||||
|
error back, we should process it correctly, should not ignore the
|
||||||
|
error, which may casue deference nil pointer problem.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/kata_agent.go | 8 ++++++++
|
||||||
|
1 file changed, 8 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
|
||||||
|
index ac64817a..fee4215f 100644
|
||||||
|
--- a/virtcontainers/kata_agent.go
|
||||||
|
+++ b/virtcontainers/kata_agent.go
|
||||||
|
@@ -604,7 +604,15 @@ func (k *kataAgent) updateInterface(ifc *vcTypes.Interface) (*vcTypes.Interface,
|
||||||
|
"interface-requested": fmt.Sprintf("%+v", ifc),
|
||||||
|
"resulting-interface": fmt.Sprintf("%+v", resultingInterface),
|
||||||
|
}).WithError(err).Error("update interface request failed")
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // need to judege resultingInterface is not not, otherwise may cause
|
||||||
|
+ // deference nil pointer panic problem
|
||||||
|
+ if resultingInterface == nil {
|
||||||
|
+ return nil, fmt.Errorf("resultingInterface should not be nil")
|
||||||
|
}
|
||||||
|
+
|
||||||
|
if resultInterface, ok := resultingInterface.(*aTypes.Interface); ok {
|
||||||
|
iface := &vcTypes.Interface{
|
||||||
|
Device: resultInterface.Device,
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,162 @@
|
|||||||
|
From d9738cfd6500ced30efde9e747eb73e5076e73ed Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Tue, 18 Aug 2020 22:47:13 +0800
|
||||||
|
Subject: [PATCH 43/50] kata-runtime: support add hypervisor global parameters
|
||||||
|
in config file
|
||||||
|
|
||||||
|
reason: support add hypervisor global parameters in config file
|
||||||
|
with defaultHypervisorParams config option.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
Makefile | 2 ++
|
||||||
|
cli/config/configuration-qemu.toml.in | 7 +++++++
|
||||||
|
pkg/katautils/config-settings.go.in | 1 +
|
||||||
|
pkg/katautils/config.go | 13 ++++++++++++-
|
||||||
|
vendor/github.com/intel/govmm/qemu/qemu.go | 6 +++---
|
||||||
|
virtcontainers/qemu.go | 2 +-
|
||||||
|
6 files changed, 26 insertions(+), 5 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/Makefile b/Makefile
|
||||||
|
index d5e4bbe1..b62e64b0 100644
|
||||||
|
--- a/Makefile
|
||||||
|
+++ b/Makefile
|
||||||
|
@@ -400,6 +400,7 @@ USER_VARS += FIRMWAREPATH
|
||||||
|
USER_VARS += MACHINEACCELERATORS
|
||||||
|
USER_VARS += DEFMACHINETYPE_CLH
|
||||||
|
USER_VARS += KERNELPARAMS
|
||||||
|
+USER_VARS += HYPERVISORPARAMS
|
||||||
|
USER_VARS += LIBEXECDIR
|
||||||
|
USER_VARS += LOCALSTATEDIR
|
||||||
|
USER_VARS += PKGDATADIR
|
||||||
|
@@ -607,6 +608,7 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit
|
||||||
|
-e "s|@FIRMWAREPATH_CLH@|$(FIRMWAREPATH_CLH)|g" \
|
||||||
|
-e "s|@DEFMACHINETYPE_CLH@|$(DEFMACHINETYPE_CLH)|g" \
|
||||||
|
-e "s|@KERNELPARAMS@|$(KERNELPARAMS)|g" \
|
||||||
|
+ -e "s|@HYPERVISORPARAMS@|$(HYPERVISORPARAMS)|g" \
|
||||||
|
-e "s|@LOCALSTATEDIR@|$(LOCALSTATEDIR)|g" \
|
||||||
|
-e "s|@PKGLIBEXECDIR@|$(PKGLIBEXECDIR)|g" \
|
||||||
|
-e "s|@PKGRUNDIR@|$(PKGRUNDIR)|g" \
|
||||||
|
diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in
|
||||||
|
index aa11b38f..e57a954c 100644
|
||||||
|
--- a/cli/config/configuration-qemu.toml.in
|
||||||
|
+++ b/cli/config/configuration-qemu.toml.in
|
||||||
|
@@ -29,6 +29,13 @@ machine_type = "@MACHINETYPE@"
|
||||||
|
# container and look for 'default-kernel-parameters' log entries.
|
||||||
|
kernel_params = "@KERNELPARAMS@"
|
||||||
|
|
||||||
|
+# Optional space-separated list of options to pass to the hypervisor.
|
||||||
|
+# For example, use `hypervisor_params = "kvm-pit.lost_tick_policy=discard"`
|
||||||
|
+#
|
||||||
|
+# WARNING: - any parameter specified here will take priority over the default
|
||||||
|
+# parameter value of the same name used to start the hypervisor.
|
||||||
|
+hypervisor_params = "@HYPERVISORPARAMS@"
|
||||||
|
+
|
||||||
|
# Path to the firmware.
|
||||||
|
# If you want that qemu uses the default firmware leave this option empty
|
||||||
|
firmware = "@FIRMWAREPATH@"
|
||||||
|
diff --git a/pkg/katautils/config-settings.go.in b/pkg/katautils/config-settings.go.in
|
||||||
|
index aaf78cc3..b2dfdfa1 100644
|
||||||
|
--- a/pkg/katautils/config-settings.go.in
|
||||||
|
+++ b/pkg/katautils/config-settings.go.in
|
||||||
|
@@ -20,6 +20,7 @@ var defaultShimPath = "/usr/libexec/kata-containers/kata-shim"
|
||||||
|
var systemdUnitName = "kata-containers.target"
|
||||||
|
|
||||||
|
const defaultKernelParams = ""
|
||||||
|
+const defaultHypervisorParams = "kvm-pit.lost_tick_policy=discard"
|
||||||
|
const defaultMachineType = "pc"
|
||||||
|
|
||||||
|
const defaultVCPUCount uint32 = 1
|
||||||
|
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
||||||
|
index 51120311..3365b3f5 100644
|
||||||
|
--- a/pkg/katautils/config.go
|
||||||
|
+++ b/pkg/katautils/config.go
|
||||||
|
@@ -130,6 +130,7 @@ type hypervisor struct {
|
||||||
|
HotplugVFIOOnRootBus bool `toml:"hotplug_vfio_on_root_bus"`
|
||||||
|
DisableVhostNet bool `toml:"disable_vhost_net"`
|
||||||
|
GuestHookPath string `toml:"guest_hook_path"`
|
||||||
|
+ HypervisorParams string `toml:"hypervisor_params"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type proxy struct {
|
||||||
|
@@ -270,6 +271,14 @@ func (h hypervisor) kernelParams() string {
|
||||||
|
return h.KernelParams
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (h hypervisor) hypervisorParams() string {
|
||||||
|
+ if h.HypervisorParams == "" {
|
||||||
|
+ return defaultHypervisorParams
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return h.HypervisorParams
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func (h hypervisor) machineType() string {
|
||||||
|
if h.MachineType == "" {
|
||||||
|
return defaultMachineType
|
||||||
|
@@ -632,6 +641,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
||||||
|
|
||||||
|
machineAccelerators := h.machineAccelerators()
|
||||||
|
kernelParams := h.kernelParams()
|
||||||
|
+ hypervisorParams := h.hypervisorParams()
|
||||||
|
machineType := h.machineType()
|
||||||
|
|
||||||
|
blockDriver, err := h.blockDeviceDriver()
|
||||||
|
@@ -675,6 +685,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
||||||
|
FirmwarePath: firmware,
|
||||||
|
MachineAccelerators: machineAccelerators,
|
||||||
|
KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)),
|
||||||
|
+ HypervisorParams: vc.DeserializeParams(strings.Fields(hypervisorParams)),
|
||||||
|
HypervisorMachineType: machineType,
|
||||||
|
NumVCPUs: h.defaultVCPUs(),
|
||||||
|
DefaultMaxVCPUs: h.defaultMaxVCPUs(),
|
||||||
|
@@ -983,7 +994,7 @@ func updateRuntimeConfigAgent(configPath string, tomlConf tomlConfig, config *oc
|
||||||
|
TraceMode: agent.traceMode(),
|
||||||
|
TraceType: agent.traceType(),
|
||||||
|
KernelModules: agent.kernelModules(),
|
||||||
|
- MountBlkInVM: agent.mountBlkDevInVM(),
|
||||||
|
+ MountBlkInVM: agent.mountBlkDevInVM(),
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("%s agent type is not supported", k)
|
||||||
|
diff --git a/vendor/github.com/intel/govmm/qemu/qemu.go b/vendor/github.com/intel/govmm/qemu/qemu.go
|
||||||
|
index 3e7720b4..68f8d2b0 100644
|
||||||
|
--- a/vendor/github.com/intel/govmm/qemu/qemu.go
|
||||||
|
+++ b/vendor/github.com/intel/govmm/qemu/qemu.go
|
||||||
|
@@ -2096,7 +2096,7 @@ type Config struct {
|
||||||
|
SMP SMP
|
||||||
|
|
||||||
|
// GlobalParam is the -global parameter.
|
||||||
|
- GlobalParam string
|
||||||
|
+ GlobalParam []string
|
||||||
|
|
||||||
|
// Knobs is a set of qemu boolean settings.
|
||||||
|
Knobs Knobs
|
||||||
|
@@ -2285,9 +2285,9 @@ func (config *Config) appendRTC() {
|
||||||
|
}
|
||||||
|
|
||||||
|
func (config *Config) appendGlobalParam() {
|
||||||
|
- if config.GlobalParam != "" {
|
||||||
|
+ for _, param := range config.GlobalParam {
|
||||||
|
config.qemuParams = append(config.qemuParams, "-global")
|
||||||
|
- config.qemuParams = append(config.qemuParams, config.GlobalParam)
|
||||||
|
+ config.qemuParams = append(config.qemuParams, param)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
|
||||||
|
index a10c66fb..657b6be7 100644
|
||||||
|
--- a/virtcontainers/qemu.go
|
||||||
|
+++ b/virtcontainers/qemu.go
|
||||||
|
@@ -600,7 +600,7 @@ func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNa
|
||||||
|
Knobs: knobs,
|
||||||
|
Incoming: incoming,
|
||||||
|
VGA: "none",
|
||||||
|
- GlobalParam: "kvm-pit.lost_tick_policy=discard",
|
||||||
|
+ GlobalParam: SerializeParams(hypervisorConfig.HypervisorParams, "="),
|
||||||
|
Bios: firmwarePath,
|
||||||
|
PidFile: filepath.Join(q.store.RunVMStoragePath(), q.id, "pid"),
|
||||||
|
}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,375 @@
|
|||||||
|
From dbbf8c5deb14d8033b2e863fb0eb731523af2a47 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Wed, 19 Aug 2020 09:58:07 +0800
|
||||||
|
Subject: [PATCH 44/50] network: support dpdk vhost_user net device
|
||||||
|
|
||||||
|
reason: support dpdk vhost_user net device
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
vendor/github.com/intel/govmm/qemu/qmp.go | 3 +-
|
||||||
|
virtcontainers/hypervisor.go | 3 ++
|
||||||
|
virtcontainers/network.go | 49 ++++++++++++++------
|
||||||
|
virtcontainers/pkg/types/types.go | 2 +
|
||||||
|
virtcontainers/qemu.go | 75 +++++++++++++++++++++++++++++++
|
||||||
|
virtcontainers/sandbox.go | 7 ++-
|
||||||
|
virtcontainers/vhostuser_endpoint.go | 18 ++++++--
|
||||||
|
virtcontainers/vhostuser_endpoint_test.go | 7 ++-
|
||||||
|
8 files changed, 143 insertions(+), 21 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/vendor/github.com/intel/govmm/qemu/qmp.go b/vendor/github.com/intel/govmm/qemu/qmp.go
|
||||||
|
index a64039de..0cb82ffa 100644
|
||||||
|
--- a/vendor/github.com/intel/govmm/qemu/qmp.go
|
||||||
|
+++ b/vendor/github.com/intel/govmm/qemu/qmp.go
|
||||||
|
@@ -970,11 +970,12 @@ func (q *QMP) ExecuteNetdevAdd(ctx context.Context, netdevType, netdevID, ifname
|
||||||
|
// ExecuteNetdevChardevAdd adds a Net device to a QEMU instance
|
||||||
|
// using the netdev_add command. netdevID is the id of the device to add.
|
||||||
|
// Must be valid QMP identifier.
|
||||||
|
-func (q *QMP) ExecuteNetdevChardevAdd(ctx context.Context, netdevType, netdevID, chardev string, queues int) error {
|
||||||
|
+func (q *QMP) ExecuteNetdevChardevAdd(ctx context.Context, netdevType, netdevID, chardev string, vhostforce bool, queues int) error {
|
||||||
|
args := map[string]interface{}{
|
||||||
|
"type": netdevType,
|
||||||
|
"id": netdevID,
|
||||||
|
"chardev": chardev,
|
||||||
|
+ "vhostforce": vhostforce,
|
||||||
|
}
|
||||||
|
if queues > 1 {
|
||||||
|
args["queues"] = queues
|
||||||
|
diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go
|
||||||
|
index c0723daa..60f1d190 100644
|
||||||
|
--- a/virtcontainers/hypervisor.go
|
||||||
|
+++ b/virtcontainers/hypervisor.go
|
||||||
|
@@ -132,6 +132,9 @@ const (
|
||||||
|
// vhostuserDev is a Vhost-user device type
|
||||||
|
vhostuserDev
|
||||||
|
|
||||||
|
+ // vhostUserNetDev is a Vhost-user net device type
|
||||||
|
+ vhostUserNetDev
|
||||||
|
+
|
||||||
|
// CPUDevice is CPU device type
|
||||||
|
cpuDev
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
|
||||||
|
index 488bd00c..a0c64356 100644
|
||||||
|
--- a/virtcontainers/network.go
|
||||||
|
+++ b/virtcontainers/network.go
|
||||||
|
@@ -13,6 +13,7 @@ import (
|
||||||
|
"math/rand"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
+ "path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"runtime"
|
||||||
|
"sort"
|
||||||
|
@@ -65,7 +66,8 @@ const (
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
- regInfName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_\-.]*$`)
|
||||||
|
+ regInfName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_\-.]*$`)
|
||||||
|
+ regVhostName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9\-.]{0,127}$`)
|
||||||
|
)
|
||||||
|
|
||||||
|
//IsValid checks if a model is valid
|
||||||
|
@@ -132,11 +134,12 @@ type NetlinkIface struct {
|
||||||
|
// NetworkInfo gathers all information related to a network interface.
|
||||||
|
// It can be used to store the description of the underlying network.
|
||||||
|
type NetworkInfo struct {
|
||||||
|
- Device string
|
||||||
|
- Iface NetlinkIface
|
||||||
|
- Addrs []netlink.Addr
|
||||||
|
- Routes []netlink.Route
|
||||||
|
- DNS DNSInfo
|
||||||
|
+ Device string
|
||||||
|
+ Iface NetlinkIface
|
||||||
|
+ Addrs []netlink.Addr
|
||||||
|
+ Routes []netlink.Route
|
||||||
|
+ DNS DNSInfo
|
||||||
|
+ VhostUserSocket string
|
||||||
|
}
|
||||||
|
|
||||||
|
// NetworkInterface defines a network interface.
|
||||||
|
@@ -1198,15 +1201,11 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li
|
||||||
|
return endpoint, err
|
||||||
|
}
|
||||||
|
|
||||||
|
- // Check if this is a dummy interface which has a vhost-user socket associated with it
|
||||||
|
- socketPath, err := vhostUserSocketPath(netInfo)
|
||||||
|
- if err != nil {
|
||||||
|
- return nil, err
|
||||||
|
- }
|
||||||
|
-
|
||||||
|
- if socketPath != "" {
|
||||||
|
+ // Currently, we only accept the vhost-user socket paas from user input
|
||||||
|
+ // interface info json file
|
||||||
|
+ if netInfo.VhostUserSocket != "" {
|
||||||
|
networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found")
|
||||||
|
- endpoint, err = createVhostUserEndpoint(netInfo, socketPath)
|
||||||
|
+ endpoint, err = createVhostUserEndpoint(netInfo, netInfo.VhostUserSocket)
|
||||||
|
return endpoint, err
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -1481,6 +1480,24 @@ func verifyIP(ip string) (*net.IP, error) {
|
||||||
|
return &netIP, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+// verifyVhostSocket verifies the vhost socket is valid or not
|
||||||
|
+func verifyVhostSocket(vhostSocket string) error {
|
||||||
|
+ if vhostSocket == "" {
|
||||||
|
+ networkLogger().Debug("no dpdk network")
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+ if regVhostName.FindAllString(filepath.Base(vhostSocket), -1) == nil {
|
||||||
|
+ return fmt.Errorf("invalid input of vhostSocket name, please check the rules")
|
||||||
|
+ }
|
||||||
|
+ info, err := os.Stat(vhostSocket)
|
||||||
|
+ if err != nil || info.IsDir() || info.Mode()&os.ModeSocket == 0 {
|
||||||
|
+ return fmt.Errorf("invalid vhost socket: %v", vhostSocket)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ networkLogger().WithField("vhostSocket", vhostSocket).Infof("using dpdk network, socket file is: %s ", vhostSocket)
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// validInterface check the input interface valid or not
|
||||||
|
func validInterface(inf *vcTypes.Interface, enableCompatOldCNI bool) error {
|
||||||
|
if enableCompatOldCNI && verifyInterfaceName(inf.Device) != nil {
|
||||||
|
@@ -1503,6 +1520,10 @@ func validInterface(inf *vcTypes.Interface, enableCompatOldCNI bool) error {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
+ if err := verifyVhostSocket(inf.VhostUserSocket); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
// Currently, only one IP address can be passed, which reduces the test entry and fault injection.
|
||||||
|
if len(inf.IPAddresses) > 0 {
|
||||||
|
if len(inf.IPAddresses) != 1 {
|
||||||
|
diff --git a/virtcontainers/pkg/types/types.go b/virtcontainers/pkg/types/types.go
|
||||||
|
index b41b0c75..dccd92f8 100644
|
||||||
|
--- a/virtcontainers/pkg/types/types.go
|
||||||
|
+++ b/virtcontainers/pkg/types/types.go
|
||||||
|
@@ -29,6 +29,8 @@ type Interface struct {
|
||||||
|
// library, regarding each type of link. Here is a non exhaustive
|
||||||
|
// list: "veth", "macvtap", "vlan", "macvlan", "tap", ...
|
||||||
|
LinkType string `json:"linkType,omitempty"`
|
||||||
|
+ // VhostUserSocket is DPDK-backed vHost user ports.
|
||||||
|
+ VhostUserSocket string `json:"vhostUserSocket,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Route describes a network route.
|
||||||
|
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
|
||||||
|
index 657b6be7..84797e0d 100644
|
||||||
|
--- a/virtcontainers/qemu.go
|
||||||
|
+++ b/virtcontainers/qemu.go
|
||||||
|
@@ -1279,6 +1279,78 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (q *qemu) hotplugVhostUserNetDevice(endpoint Endpoint, op operation) error {
|
||||||
|
+ err := q.qmpSetup()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ device := endpoint.(*VhostUserEndpoint)
|
||||||
|
+
|
||||||
|
+ charDevID := utils.MakeNameID("char", device.ID, maxDevIDSize)
|
||||||
|
+ devID := utils.MakeNameID("virtio", device.ID, maxDevIDSize)
|
||||||
|
+
|
||||||
|
+ if op == addDevice {
|
||||||
|
+ // Add chardev specifying the appropriate socket.
|
||||||
|
+ if err = q.qmpMonitorCh.qmp.ExecuteCharDevUnixSocketAdd(q.qmpMonitorCh.ctx, charDevID, device.SocketPath, true, false); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ defer func() {
|
||||||
|
+ if err != nil {
|
||||||
|
+ errRb := q.qmpMonitorCh.qmp.ExecuteChardevDel(q.qmpMonitorCh.ctx, charDevID)
|
||||||
|
+ if errRb != nil {
|
||||||
|
+ q.Logger().Errorf("deletes a net device roll back failed. charDev id:%s", charDevID)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
+ netDevType := "vhost-user"
|
||||||
|
+ // Add netdev of type vhost-user.
|
||||||
|
+ if err = q.qmpMonitorCh.qmp.ExecuteNetdevChardevAdd(q.qmpMonitorCh.ctx, netDevType, devID, charDevID, true, 0); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ defer func() {
|
||||||
|
+ if err != nil {
|
||||||
|
+ errRb := q.qmpMonitorCh.qmp.ExecuteNetdevDel(q.qmpMonitorCh.ctx, devID)
|
||||||
|
+ if errRb != nil {
|
||||||
|
+ q.Logger().Errorf("deletes a net device roll back failed. netDev id:%s", devID)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
+ var addr, bus, pciAddr string
|
||||||
|
+ addr, bus, pciAddr, err = q.getPciAddress(device.ID, types.PCI)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+ defer func() {
|
||||||
|
+ if err != nil {
|
||||||
|
+ q.putPciAddress(device.ID)
|
||||||
|
+ }
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
+ endpoint.SetPciAddr(pciAddr)
|
||||||
|
+
|
||||||
|
+ // Add virtio-net-pci device.
|
||||||
|
+ err = q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, devID, devID, device.HardAddr, addr, bus, romFile, 0, q.arch.runNested())
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err = q.putPciAddress(device.ID); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ if err = q.qmpMonitorCh.qmp.ExecuteDeviceDel(q.qmpMonitorCh.ctx, devID); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ if err = q.qmpMonitorCh.qmp.ExecuteNetdevDel(q.qmpMonitorCh.ctx, devID); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func (q *qemu) hotplugVhostUserDevice(vAttr *config.VhostUserDeviceAttrs, op operation) error {
|
||||||
|
err := q.qmpSetup()
|
||||||
|
if err != nil {
|
||||||
|
@@ -1510,6 +1582,9 @@ func (q *qemu) hotplugDevice(devInfo interface{}, devType deviceType, op operati
|
||||||
|
case vhostuserDev:
|
||||||
|
vAttr := devInfo.(*config.VhostUserDeviceAttrs)
|
||||||
|
return nil, q.hotplugVhostUserDevice(vAttr, op)
|
||||||
|
+ case vhostUserNetDev:
|
||||||
|
+ device := devInfo.(Endpoint)
|
||||||
|
+ return nil, q.hotplugVhostUserNetDevice(device, op)
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("cannot hotplug device: unsupported device type '%v'", devType)
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index 0d599267..e6f155a3 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -946,7 +946,8 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) {
|
||||||
|
},
|
||||||
|
Type: inf.LinkType,
|
||||||
|
},
|
||||||
|
- Addrs: addrs,
|
||||||
|
+ Addrs: addrs,
|
||||||
|
+ VhostUserSocket: inf.VhostUserSocket,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -961,6 +962,10 @@ func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (grpcIf *vcTypes.Interfac
|
||||||
|
if ep.Name() == inf.Name {
|
||||||
|
return nil, fmt.Errorf("interface %s is already exist", inf.Name)
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+ if ep.Properties().VhostUserSocket != "" && inf.VhostUserSocket != "" {
|
||||||
|
+ return nil, fmt.Errorf("sandbox %s only support one dpdk socket", s.ID())
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
|
||||||
|
netInfo, err := s.generateNetInfo(inf)
|
||||||
|
diff --git a/virtcontainers/vhostuser_endpoint.go b/virtcontainers/vhostuser_endpoint.go
|
||||||
|
index bb4a67be..2fc3d837 100644
|
||||||
|
--- a/virtcontainers/vhostuser_endpoint.go
|
||||||
|
+++ b/virtcontainers/vhostuser_endpoint.go
|
||||||
|
@@ -12,6 +12,7 @@ import (
|
||||||
|
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/device/config"
|
||||||
|
persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
|
||||||
|
"github.com/kata-containers/runtime/virtcontainers/utils"
|
||||||
|
)
|
||||||
|
|
||||||
|
@@ -23,6 +24,7 @@ const hostSocketSearchPath = "/tmp/vhostuser_%s/vhu.sock"
|
||||||
|
|
||||||
|
// VhostUserEndpoint represents a vhost-user socket based network interface
|
||||||
|
type VhostUserEndpoint struct {
|
||||||
|
+ ID string
|
||||||
|
// Path to the vhost-user socket on the host system
|
||||||
|
SocketPath string
|
||||||
|
// MAC address of the interface
|
||||||
|
@@ -99,18 +101,28 @@ func (endpoint *VhostUserEndpoint) Detach(netNsCreated bool, netNsPath string) e
|
||||||
|
|
||||||
|
// HotAttach for vhostuser endpoint not supported yet
|
||||||
|
func (endpoint *VhostUserEndpoint) HotAttach(h hypervisor) error {
|
||||||
|
- return fmt.Errorf("VhostUserEndpoint does not support Hot attach")
|
||||||
|
+ networkLogger().Debug("Hot attaching vhost-user endpoint")
|
||||||
|
+ if _, err := h.hotplugAddDevice(endpoint, vhostUserNetDev); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// HotDetach for vhostuser endpoint not supported yet
|
||||||
|
func (endpoint *VhostUserEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error {
|
||||||
|
- return fmt.Errorf("VhostUserEndpoint does not support Hot detach")
|
||||||
|
+ networkLogger().Debug("Hot detaching vhost-user endpoint")
|
||||||
|
+ if _, err := h.hotplugRemoveDevice(endpoint, vhostUserNetDev); err != nil {
|
||||||
|
+ networkLogger().WithError(err).Errorf("Error detach vhostUserSocket")
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create a vhostuser endpoint
|
||||||
|
func createVhostUserEndpoint(netInfo NetworkInfo, socket string) (*VhostUserEndpoint, error) {
|
||||||
|
-
|
||||||
|
+ uniqueID := uuid.Generate().String()
|
||||||
|
vhostUserEndpoint := &VhostUserEndpoint{
|
||||||
|
+ ID: uniqueID,
|
||||||
|
SocketPath: socket,
|
||||||
|
HardAddr: netInfo.Iface.HardwareAddr.String(),
|
||||||
|
IfaceName: netInfo.Iface.Name,
|
||||||
|
diff --git a/virtcontainers/vhostuser_endpoint_test.go b/virtcontainers/vhostuser_endpoint_test.go
|
||||||
|
index ad013e12..584490cc 100644
|
||||||
|
--- a/virtcontainers/vhostuser_endpoint_test.go
|
||||||
|
+++ b/virtcontainers/vhostuser_endpoint_test.go
|
||||||
|
@@ -11,6 +11,7 @@ import (
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/vishvananda/netlink"
|
||||||
|
)
|
||||||
|
@@ -95,7 +96,7 @@ func TestVhostUserEndpoint_HotAttach(t *testing.T) {
|
||||||
|
h := &mockHypervisor{}
|
||||||
|
|
||||||
|
err := v.HotAttach(h)
|
||||||
|
- assert.Error(err)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVhostUserEndpoint_HotDetach(t *testing.T) {
|
||||||
|
@@ -109,10 +110,11 @@ func TestVhostUserEndpoint_HotDetach(t *testing.T) {
|
||||||
|
h := &mockHypervisor{}
|
||||||
|
|
||||||
|
err := v.HotDetach(h, true, "")
|
||||||
|
- assert.Error(err)
|
||||||
|
+ assert.NoError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCreateVhostUserEndpoint(t *testing.T) {
|
||||||
|
+ uniqueID := uuid.Generate().String()
|
||||||
|
macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x48}
|
||||||
|
ifcName := "vhost-deadbeef"
|
||||||
|
socket := "/tmp/vhu_192.168.0.1"
|
||||||
|
@@ -128,6 +130,7 @@ func TestCreateVhostUserEndpoint(t *testing.T) {
|
||||||
|
}
|
||||||
|
|
||||||
|
expected := &VhostUserEndpoint{
|
||||||
|
+ ID: uniqueID,
|
||||||
|
SocketPath: socket,
|
||||||
|
HardAddr: macAddr.String(),
|
||||||
|
IfaceName: ifcName,
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
115
runtime/patches/0045-network-support-set-dns.patch
Normal file
115
runtime/patches/0045-network-support-set-dns.patch
Normal file
@ -0,0 +1,115 @@
|
|||||||
|
From e104f084ed4f10049f45d9473faed229371a1c6c Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Wed, 19 Aug 2020 09:59:52 +0800
|
||||||
|
Subject: [PATCH 45/50] network: support set dns
|
||||||
|
|
||||||
|
reason: when running a sandbox with annotation about
|
||||||
|
dns spec, overload k.getDNS(sandbox)
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/kata_agent.go | 11 +++++++++++
|
||||||
|
virtcontainers/network.go | 8 ++++----
|
||||||
|
virtcontainers/pkg/annotations/annotations.go | 3 +++
|
||||||
|
virtcontainers/pkg/oci/utils.go | 19 +++++++++++++++++--
|
||||||
|
4 files changed, 35 insertions(+), 6 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
|
||||||
|
index fee4215f..d28d8cce 100644
|
||||||
|
--- a/virtcontainers/kata_agent.go
|
||||||
|
+++ b/virtcontainers/kata_agent.go
|
||||||
|
@@ -890,6 +890,17 @@ func (k *kataAgent) startSandbox(sandbox *Sandbox) error {
|
||||||
|
KernelModules: kmodules,
|
||||||
|
}
|
||||||
|
|
||||||
|
+ if value, ok := sandbox.config.Annotations[vcAnnotations.SandboxDNSTypeKey]; ok {
|
||||||
|
+ var sandboxDns *DNSInfo
|
||||||
|
+ if err := json.Unmarshal([]byte(value), &sandboxDns); err != nil {
|
||||||
|
+ return fmt.Errorf("get sandbox dns failed %v", err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if sandboxDns != nil {
|
||||||
|
+ req.Dns = sandboxDns.Servers
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
_, err = k.sendReq(req)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
|
||||||
|
index 488bd00c..68eda4a6 100644
|
||||||
|
--- a/virtcontainers/network.go
|
||||||
|
+++ b/virtcontainers/network.go
|
||||||
|
@@ -117,10 +117,10 @@ const (
|
||||||
|
|
||||||
|
// DNSInfo describes the DNS setup related to a network interface.
|
||||||
|
type DNSInfo struct {
|
||||||
|
- Servers []string
|
||||||
|
- Domain string
|
||||||
|
- Searches []string
|
||||||
|
- Options []string
|
||||||
|
+ Servers []string `json:"Servers"`
|
||||||
|
+ Domain string `json:"Domain,omitempty"`
|
||||||
|
+ Searches []string `json:"Searches,omitempty"`
|
||||||
|
+ Options []string `json:"Options,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// NetlinkIface describes fully a network interface.
|
||||||
|
diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go
|
||||||
|
index e50a697c..528dfa66 100644
|
||||||
|
--- a/virtcontainers/pkg/annotations/annotations.go
|
||||||
|
+++ b/virtcontainers/pkg/annotations/annotations.go
|
||||||
|
@@ -71,6 +71,9 @@ const (
|
||||||
|
// StorageSpecTypeKey is the annotation key to fetch storage_spec
|
||||||
|
StorageSpecTypeKey = kataAnnotationsPrefix + "storage_spec"
|
||||||
|
|
||||||
|
+ // SandboxDNSTypeKey is the annotation key to fetch sandbox dns options
|
||||||
|
+ SandboxDNSTypeKey = kataAnnotationsPrefix + "sandbox_dns"
|
||||||
|
+
|
||||||
|
//
|
||||||
|
// Generic annotations
|
||||||
|
//
|
||||||
|
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
||||||
|
index d032227e..3b2af753 100644
|
||||||
|
--- a/virtcontainers/pkg/oci/utils.go
|
||||||
|
+++ b/virtcontainers/pkg/oci/utils.go
|
||||||
|
@@ -7,6 +7,7 @@ package oci
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
+ "encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"path/filepath"
|
||||||
|
@@ -37,8 +38,9 @@ type annotationContainerType struct {
|
||||||
|
type annotationHandler func(value string) error
|
||||||
|
|
||||||
|
var annotationHandlerList = map[string]annotationHandler{
|
||||||
|
- vcAnnotations.StaticCPUTypeKey: validateSandboxCPU,
|
||||||
|
- vcAnnotations.StaticMemTypeKey: validateSandboxMem,
|
||||||
|
+ vcAnnotations.StaticCPUTypeKey: validateSandboxCPU,
|
||||||
|
+ vcAnnotations.StaticMemTypeKey: validateSandboxMem,
|
||||||
|
+ vcAnnotations.SandboxDNSTypeKey: validateSandboxDNS,
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
@@ -1108,3 +1110,16 @@ func validateSandboxMem(value string) error {
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func validateSandboxDNS(value string) error {
|
||||||
|
+ if value == "" {
|
||||||
|
+ return fmt.Errorf("--annotation %s value should not be empty", vcAnnotations.SandboxDNSTypeKey)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ var sandboxDns *vc.DNSInfo
|
||||||
|
+ if err := json.Unmarshal([]byte(value), &sandboxDns); err != nil {
|
||||||
|
+ return fmt.Errorf("invalid value passed by --annotation %s, error: %v", vcAnnotations.SandboxDNSTypeKey, err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,489 @@
|
|||||||
|
From e9c1fb235d0e69d5db72497a8779df707c499d3b Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Wed, 19 Aug 2020 20:36:40 +0800
|
||||||
|
Subject: [PATCH 46/50] network: support multiqueue when inserting interface
|
||||||
|
dynamically
|
||||||
|
|
||||||
|
reason: support multiqueue when inserting interface dynamically
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
.../kata-containers/agent/pkg/types/types.pb.go | 82 ++++++++++++++++------
|
||||||
|
virtcontainers/bridgedmacvlan_endpoint.go | 2 +-
|
||||||
|
virtcontainers/ipvlan_endpoint.go | 2 +-
|
||||||
|
virtcontainers/kata_agent.go | 2 +
|
||||||
|
virtcontainers/network.go | 13 ++--
|
||||||
|
virtcontainers/network_test.go | 2 +-
|
||||||
|
virtcontainers/pkg/types/types.go | 2 +
|
||||||
|
virtcontainers/qemu.go | 10 +--
|
||||||
|
virtcontainers/sandbox.go | 1 +
|
||||||
|
virtcontainers/tap_endpoint.go | 6 +-
|
||||||
|
virtcontainers/tuntap_endpoint.go | 4 +-
|
||||||
|
virtcontainers/veth_endpoint.go | 4 +-
|
||||||
|
virtcontainers/veth_endpoint_test.go | 6 +-
|
||||||
|
virtcontainers/vhostuser_endpoint.go | 4 +-
|
||||||
|
virtcontainers/vhostuser_endpoint_test.go | 2 +-
|
||||||
|
15 files changed, 96 insertions(+), 46 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/vendor/github.com/kata-containers/agent/pkg/types/types.pb.go b/vendor/github.com/kata-containers/agent/pkg/types/types.pb.go
|
||||||
|
index 7ea63e3c..8b7e2a5d 100644
|
||||||
|
--- a/vendor/github.com/kata-containers/agent/pkg/types/types.pb.go
|
||||||
|
+++ b/vendor/github.com/kata-containers/agent/pkg/types/types.pb.go
|
||||||
|
@@ -100,6 +100,7 @@ type Interface struct {
|
||||||
|
// list: "veth", "macvtap", "vlan", "macvlan", "tap", ...
|
||||||
|
Type string `protobuf:"bytes,7,opt,name=type,proto3" json:"type,omitempty"`
|
||||||
|
RawFlags uint32 `protobuf:"varint,8,opt,name=raw_flags,json=rawFlags,proto3" json:"raw_flags,omitempty"`
|
||||||
|
+ Queues uint32 `protobuf:"varint,9,opt,name=Queues,proto3" json:"Queues,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *Interface) Reset() { *m = Interface{} }
|
||||||
|
@@ -163,6 +164,13 @@ func (m *Interface) GetRawFlags() uint32 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (m *Interface) GetQueues() uint32 {
|
||||||
|
+ if m != nil {
|
||||||
|
+ return m.Queues
|
||||||
|
+ }
|
||||||
|
+ return 0
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
type Route struct {
|
||||||
|
Dest string `protobuf:"bytes,1,opt,name=dest,proto3" json:"dest,omitempty"`
|
||||||
|
Gateway string `protobuf:"bytes,2,opt,name=gateway,proto3" json:"gateway,omitempty"`
|
||||||
|
@@ -319,6 +327,11 @@ func (m *Interface) MarshalTo(dAtA []byte) (int, error) {
|
||||||
|
i++
|
||||||
|
i = encodeVarintTypes(dAtA, i, uint64(m.RawFlags))
|
||||||
|
}
|
||||||
|
+ if m.Queues != 0 {
|
||||||
|
+ dAtA[i] = 0x48
|
||||||
|
+ i++
|
||||||
|
+ i = encodeVarintTypes(dAtA, i, uint64(m.Queues))
|
||||||
|
+ }
|
||||||
|
return i, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -430,6 +443,9 @@ func (m *Interface) Size() (n int) {
|
||||||
|
if m.RawFlags != 0 {
|
||||||
|
n += 1 + sovTypes(uint64(m.RawFlags))
|
||||||
|
}
|
||||||
|
+ if m.Queues != 0 {
|
||||||
|
+ n += 1 + sovTypes(uint64(m.Queues))
|
||||||
|
+ }
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -841,6 +857,25 @@ func (m *Interface) Unmarshal(dAtA []byte) error {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
+ case 9:
|
||||||
|
+ if wireType != 0 {
|
||||||
|
+ return fmt.Errorf("proto: wrong wireType = %d for field Queues", wireType)
|
||||||
|
+ }
|
||||||
|
+ m.Queues = 0
|
||||||
|
+ for shift := uint(0); ; shift += 7 {
|
||||||
|
+ if shift >= 64 {
|
||||||
|
+ return ErrIntOverflowTypes
|
||||||
|
+ }
|
||||||
|
+ if iNdEx >= l {
|
||||||
|
+ return io.ErrUnexpectedEOF
|
||||||
|
+ }
|
||||||
|
+ b := dAtA[iNdEx]
|
||||||
|
+ iNdEx++
|
||||||
|
+ m.Queues |= (uint32(b) & 0x7F) << shift
|
||||||
|
+ if b < 0x80 {
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
default:
|
||||||
|
iNdEx = preIndex
|
||||||
|
skippy, err := skipTypes(dAtA[iNdEx:])
|
||||||
|
@@ -1155,27 +1190,28 @@ var (
|
||||||
|
func init() { proto.RegisterFile("pkg/types/types.proto", fileDescriptorTypes) }
|
||||||
|
|
||||||
|
var fileDescriptorTypes = []byte{
|
||||||
|
- // 345 bytes of a gzipped FileDescriptorProto
|
||||||
|
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x91, 0xd1, 0x6a, 0xfa, 0x30,
|
||||||
|
- 0x14, 0xc6, 0xff, 0xb1, 0xb6, 0xda, 0xe3, 0xdf, 0xad, 0x84, 0x4d, 0xc2, 0x06, 0x52, 0xbc, 0x59,
|
||||||
|
- 0xd9, 0x85, 0x03, 0x37, 0x76, 0xbf, 0x5d, 0x08, 0xde, 0x49, 0x5e, 0x60, 0xc4, 0x36, 0x3a, 0xd1,
|
||||||
|
- 0xda, 0xd2, 0x44, 0x8b, 0xec, 0x05, 0x77, 0xb9, 0x47, 0x10, 0x9f, 0x64, 0xe4, 0x24, 0x8a, 0xbb,
|
||||||
|
- 0xd1, 0xef, 0x97, 0x93, 0xd3, 0xef, 0x7c, 0x27, 0x70, 0x5b, 0xae, 0x16, 0x4f, 0x7a, 0x5f, 0x4a,
|
||||||
|
- 0x65, 0x7f, 0x87, 0x65, 0x55, 0xe8, 0x82, 0xfa, 0x08, 0x83, 0x19, 0x84, 0x93, 0xe9, 0x5b, 0x96,
|
||||||
|
- 0x55, 0x52, 0x29, 0xfa, 0x00, 0xc1, 0x5c, 0xe4, 0xcb, 0xf5, 0x9e, 0x91, 0x98, 0x24, 0x57, 0xa3,
|
||||||
|
- 0xeb, 0xa1, 0xed, 0x98, 0x4c, 0xc7, 0x78, 0xcc, 0x5d, 0x99, 0x32, 0x68, 0x09, 0xdb, 0xc3, 0x1a,
|
||||||
|
- 0x31, 0x49, 0x42, 0x7e, 0x42, 0x4a, 0xa1, 0x99, 0x0b, 0xb5, 0x62, 0x1e, 0x1e, 0xa3, 0x1e, 0x1c,
|
||||||
|
- 0x08, 0x84, 0x93, 0x8d, 0x96, 0xd5, 0x5c, 0xa4, 0x92, 0xf6, 0x20, 0xc8, 0xe4, 0x6e, 0x99, 0x4a,
|
||||||
|
- 0x34, 0x09, 0xb9, 0x23, 0xd3, 0xb9, 0x11, 0xb9, 0x74, 0x1f, 0x44, 0x4d, 0x47, 0xd0, 0x39, 0x4f,
|
||||||
|
- 0x27, 0x15, 0xf3, 0x62, 0x2f, 0xe9, 0x8c, 0xa2, 0xf3, 0x54, 0xae, 0xc2, 0x2f, 0x2f, 0xd1, 0x08,
|
||||||
|
- 0xbc, 0x5c, 0x6f, 0x59, 0x33, 0x26, 0x49, 0x93, 0x1b, 0x69, 0x1c, 0x3f, 0x6b, 0x73, 0x81, 0xf9,
|
||||||
|
- 0xd6, 0xd1, 0x92, 0x49, 0x51, 0xa6, 0x4b, 0x2c, 0x04, 0x36, 0x85, 0x43, 0x33, 0x8b, 0xf1, 0x60,
|
||||||
|
- 0x2d, 0x3b, 0x8b, 0xd1, 0xf4, 0x1e, 0xc2, 0x4a, 0xd4, 0x1f, 0xf3, 0xb5, 0x58, 0x28, 0xd6, 0x8e,
|
||||||
|
- 0x49, 0xd2, 0xe5, 0xed, 0x4a, 0xd4, 0x63, 0xc3, 0x83, 0x2f, 0xf0, 0x79, 0xb1, 0xd5, 0x98, 0x22,
|
||||||
|
- 0x93, 0x4a, 0xbb, 0x6c, 0xa8, 0x8d, 0xcf, 0x42, 0x68, 0x59, 0x8b, 0xfd, 0x69, 0x5b, 0x0e, 0x2f,
|
||||||
|
- 0x76, 0xe1, 0xfd, 0xd9, 0x45, 0x0f, 0x02, 0x55, 0x6c, 0xab, 0x54, 0x62, 0x8c, 0x90, 0x3b, 0xa2,
|
||||||
|
- 0x37, 0xe0, 0xab, 0xb4, 0x28, 0x25, 0x06, 0xe9, 0x72, 0x0b, 0x8f, 0x77, 0xd0, 0x3e, 0xbd, 0x10,
|
||||||
|
- 0x0d, 0xa0, 0xb1, 0x7b, 0x89, 0xfe, 0xe1, 0xff, 0x6b, 0x44, 0xde, 0xff, 0x7f, 0x1f, 0xfb, 0xe4,
|
||||||
|
- 0xe7, 0xd8, 0x27, 0x87, 0x63, 0x9f, 0xcc, 0x02, 0x7c, 0xfb, 0xe7, 0xdf, 0x00, 0x00, 0x00, 0xff,
|
||||||
|
- 0xff, 0xb5, 0x52, 0x37, 0xf2, 0x14, 0x02, 0x00, 0x00,
|
||||||
|
+ // 356 bytes of a gzipped FileDescriptorProto
|
||||||
|
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x92, 0xc1, 0x4e, 0xc2, 0x40,
|
||||||
|
+ 0x10, 0x86, 0x5d, 0x4a, 0x0b, 0x1d, 0x44, 0x9b, 0x8d, 0x92, 0x8d, 0x26, 0xa4, 0xe1, 0x62, 0xe3,
|
||||||
|
+ 0x01, 0x13, 0x34, 0xde, 0xf5, 0x40, 0xc2, 0x0d, 0xf7, 0x05, 0xcc, 0xd2, 0x2e, 0x48, 0xa0, 0xb4,
|
||||||
|
+ 0xe9, 0xb6, 0x34, 0xc4, 0x17, 0xf4, 0xe8, 0x23, 0x18, 0x9e, 0xc2, 0xa3, 0xd9, 0xd9, 0x85, 0xe0,
|
||||||
|
+ 0x05, 0xfe, 0x6f, 0x67, 0xa7, 0xff, 0xfc, 0xd3, 0xc2, 0x75, 0xbe, 0x5a, 0x3c, 0x94, 0xbb, 0x5c,
|
||||||
|
+ 0x2a, 0xf3, 0x3b, 0xcc, 0x8b, 0xac, 0xcc, 0xa8, 0x8b, 0x30, 0x98, 0x81, 0x3f, 0x99, 0xbe, 0x24,
|
||||||
|
+ 0x49, 0x21, 0x95, 0xa2, 0x77, 0xe0, 0xcd, 0x45, 0xba, 0x5c, 0xef, 0x18, 0x09, 0x49, 0x74, 0x31,
|
||||||
|
+ 0xba, 0x1c, 0x9a, 0x8e, 0xc9, 0x74, 0x8c, 0xc7, 0xdc, 0x96, 0x29, 0x83, 0x96, 0x30, 0x3d, 0xac,
|
||||||
|
+ 0x11, 0x92, 0xc8, 0xe7, 0x07, 0xa4, 0x14, 0x9a, 0xa9, 0x50, 0x2b, 0xe6, 0xe0, 0x31, 0xea, 0xc1,
|
||||||
|
+ 0x2f, 0x01, 0x7f, 0xb2, 0x29, 0x65, 0x31, 0x17, 0xb1, 0xa4, 0x3d, 0xf0, 0x12, 0xb9, 0x5d, 0xc6,
|
||||||
|
+ 0x12, 0x4d, 0x7c, 0x6e, 0x49, 0x77, 0x6e, 0x44, 0x2a, 0xed, 0x03, 0x51, 0xd3, 0x11, 0x74, 0x8e,
|
||||||
|
+ 0xd3, 0x49, 0xc5, 0x9c, 0xd0, 0x89, 0x3a, 0xa3, 0xe0, 0x38, 0x95, 0xad, 0xf0, 0xd3, 0x4b, 0x34,
|
||||||
|
+ 0x00, 0x27, 0x2d, 0x2b, 0xd6, 0x0c, 0x49, 0xd4, 0xe4, 0x5a, 0x6a, 0xc7, 0x8f, 0x5a, 0x5f, 0x60,
|
||||||
|
+ 0xae, 0x71, 0x34, 0xa4, 0x53, 0xe4, 0xf1, 0x12, 0x0b, 0x9e, 0x49, 0x61, 0x51, 0xcf, 0xa2, 0x3d,
|
||||||
|
+ 0x58, 0xcb, 0xcc, 0xa2, 0x35, 0xbd, 0x05, 0xbf, 0x10, 0xf5, 0xfb, 0x7c, 0x2d, 0x16, 0x8a, 0xb5,
|
||||||
|
+ 0x43, 0x12, 0x75, 0x79, 0xbb, 0x10, 0xf5, 0x58, 0xb3, 0xb6, 0x78, 0xab, 0x64, 0x25, 0x15, 0xf3,
|
||||||
|
+ 0xb1, 0x62, 0x69, 0xf0, 0x09, 0x2e, 0xcf, 0xaa, 0x12, 0xd3, 0x25, 0x52, 0x95, 0x36, 0x33, 0x6a,
|
||||||
|
+ 0xed, 0xbf, 0x10, 0xa5, 0xac, 0xc5, 0xee, 0xb0, 0x45, 0x8b, 0x27, 0x3b, 0x72, 0xfe, 0xed, 0xa8,
|
||||||
|
+ 0x07, 0x9e, 0xca, 0xaa, 0x22, 0x96, 0x18, 0xcf, 0xe7, 0x96, 0xe8, 0x15, 0xb8, 0x2a, 0xce, 0x72,
|
||||||
|
+ 0x89, 0x01, 0xbb, 0xdc, 0xc0, 0xfd, 0x0d, 0xb4, 0x0f, 0x6f, 0x8e, 0x7a, 0xd0, 0xd8, 0x3e, 0x05,
|
||||||
|
+ 0x67, 0xf8, 0xff, 0x1c, 0x90, 0xd7, 0xf3, 0xaf, 0x7d, 0x9f, 0x7c, 0xef, 0xfb, 0xe4, 0x67, 0xdf,
|
||||||
|
+ 0x27, 0x33, 0x0f, 0xbf, 0x89, 0xc7, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa5, 0x57, 0x70, 0xc8,
|
||||||
|
+ 0x2c, 0x02, 0x00, 0x00,
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/bridgedmacvlan_endpoint.go b/virtcontainers/bridgedmacvlan_endpoint.go
|
||||||
|
index 700aea34..9179bbbe 100644
|
||||||
|
--- a/virtcontainers/bridgedmacvlan_endpoint.go
|
||||||
|
+++ b/virtcontainers/bridgedmacvlan_endpoint.go
|
||||||
|
@@ -25,7 +25,7 @@ func createBridgedMacvlanNetworkEndpoint(idx int, ifName string, interworkingMod
|
||||||
|
return &BridgedMacvlanEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx)
|
||||||
|
}
|
||||||
|
|
||||||
|
- netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel)
|
||||||
|
+ netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel, 0)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/ipvlan_endpoint.go b/virtcontainers/ipvlan_endpoint.go
|
||||||
|
index 6e924a74..a6ef8179 100644
|
||||||
|
--- a/virtcontainers/ipvlan_endpoint.go
|
||||||
|
+++ b/virtcontainers/ipvlan_endpoint.go
|
||||||
|
@@ -28,7 +28,7 @@ func createIPVlanNetworkEndpoint(idx int, ifName string) (*IPVlanEndpoint, error
|
||||||
|
// Use tc filtering for ipvlan, since the other inter networking models will
|
||||||
|
// not work for ipvlan.
|
||||||
|
interworkingModel := NetXConnectTCFilterModel
|
||||||
|
- netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel)
|
||||||
|
+ netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel, 0)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
|
||||||
|
index d28d8cce..38e9a204 100644
|
||||||
|
--- a/virtcontainers/kata_agent.go
|
||||||
|
+++ b/virtcontainers/kata_agent.go
|
||||||
|
@@ -621,6 +621,7 @@ func (k *kataAgent) updateInterface(ifc *vcTypes.Interface) (*vcTypes.Interface,
|
||||||
|
Mtu: resultInterface.Mtu,
|
||||||
|
HwAddr: resultInterface.HwAddr,
|
||||||
|
PciAddr: resultInterface.PciAddr,
|
||||||
|
+ Queues: resultInterface.Queues,
|
||||||
|
}
|
||||||
|
return iface, err
|
||||||
|
}
|
||||||
|
@@ -2295,6 +2296,7 @@ func (k *kataAgent) convertToKataAgentInterface(iface *vcTypes.Interface) *aType
|
||||||
|
RawFlags: iface.RawFlags,
|
||||||
|
HwAddr: iface.HwAddr,
|
||||||
|
PciAddr: iface.PciAddr,
|
||||||
|
+ Queues: iface.Queues,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
|
||||||
|
index 46703656..15eb7906 100644
|
||||||
|
--- a/virtcontainers/network.go
|
||||||
|
+++ b/virtcontainers/network.go
|
||||||
|
@@ -140,6 +140,7 @@ type NetworkInfo struct {
|
||||||
|
Routes []netlink.Route
|
||||||
|
DNS DNSInfo
|
||||||
|
VhostUserSocket string
|
||||||
|
+ Queues uint32
|
||||||
|
}
|
||||||
|
|
||||||
|
// NetworkInterface defines a network interface.
|
||||||
|
@@ -156,6 +157,7 @@ type TapInterface struct {
|
||||||
|
TAPIface NetworkInterface
|
||||||
|
VMFds []*os.File
|
||||||
|
VhostFds []*os.File
|
||||||
|
+ Queues uint32
|
||||||
|
}
|
||||||
|
|
||||||
|
// TuntapInterface defines a tap interface
|
||||||
|
@@ -1047,7 +1049,7 @@ func generateInterfacesAndRoutes(networkNS NetworkNamespace) ([]*vcTypes.Interfa
|
||||||
|
return ifaces, routes, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
-func createNetworkInterfacePair(idx int, ifName string, interworkingModel NetInterworkingModel) (NetworkInterfacePair, error) {
|
||||||
|
+func createNetworkInterfacePair(idx int, ifName string, interworkingModel NetInterworkingModel, queues uint32) (NetworkInterfacePair, error) {
|
||||||
|
uniqueID := uuid.Generate().String()
|
||||||
|
|
||||||
|
randomMacAddr, err := generateRandomPrivateMacAddr()
|
||||||
|
@@ -1062,6 +1064,7 @@ func createNetworkInterfacePair(idx int, ifName string, interworkingModel NetInt
|
||||||
|
TAPIface: NetworkInterface{
|
||||||
|
Name: fmt.Sprintf("tap%d_kata", idx),
|
||||||
|
},
|
||||||
|
+ Queues: queues,
|
||||||
|
},
|
||||||
|
VirtIface: NetworkInterface{
|
||||||
|
Name: fmt.Sprintf("eth%d", idx),
|
||||||
|
@@ -1205,7 +1208,7 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li
|
||||||
|
// interface info json file
|
||||||
|
if netInfo.VhostUserSocket != "" {
|
||||||
|
networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found")
|
||||||
|
- endpoint, err = createVhostUserEndpoint(netInfo, netInfo.VhostUserSocket)
|
||||||
|
+ endpoint, err = createVhostUserEndpoint(netInfo, netInfo.VhostUserSocket, netInfo.Queues)
|
||||||
|
return endpoint, err
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -1220,7 +1223,7 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li
|
||||||
|
endpoint, err = createMacvtapNetworkEndpoint(netInfo)
|
||||||
|
case "tap":
|
||||||
|
networkLogger().Info("tap interface found")
|
||||||
|
- endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Device)
|
||||||
|
+ endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Device, netInfo.Queues)
|
||||||
|
case "tuntap":
|
||||||
|
if link != nil {
|
||||||
|
switch link.(*netlink.Tuntap).Mode {
|
||||||
|
@@ -1231,13 +1234,13 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li
|
||||||
|
return nil, fmt.Errorf("tun networking device not yet supported")
|
||||||
|
case 2:
|
||||||
|
networkLogger().Info("tuntap tap interface found")
|
||||||
|
- endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, model)
|
||||||
|
+ endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, model, netInfo.Queues)
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("tuntap network %v mode unsupported", link.(*netlink.Tuntap).Mode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "veth":
|
||||||
|
- endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model)
|
||||||
|
+ endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model, netInfo.Queues)
|
||||||
|
case "ipvlan":
|
||||||
|
endpoint, err = createIPVlanNetworkEndpoint(idx, netInfo.Iface.Name)
|
||||||
|
default:
|
||||||
|
diff --git a/virtcontainers/network_test.go b/virtcontainers/network_test.go
|
||||||
|
index b86f679f..c52c7452 100644
|
||||||
|
--- a/virtcontainers/network_test.go
|
||||||
|
+++ b/virtcontainers/network_test.go
|
||||||
|
@@ -265,7 +265,7 @@ func TestTcRedirectNetwork(t *testing.T) {
|
||||||
|
err = netlink.LinkAdd(veth)
|
||||||
|
assert.NoError(err)
|
||||||
|
|
||||||
|
- endpoint, err := createVethNetworkEndpoint(1, vethName, NetXConnectTCFilterModel)
|
||||||
|
+ endpoint, err := createVethNetworkEndpoint(1, vethName, NetXConnectTCFilterModel, 0)
|
||||||
|
assert.NoError(err)
|
||||||
|
|
||||||
|
link, err := netlink.LinkByName(vethName)
|
||||||
|
diff --git a/virtcontainers/pkg/types/types.go b/virtcontainers/pkg/types/types.go
|
||||||
|
index dccd92f8..6502259b 100644
|
||||||
|
--- a/virtcontainers/pkg/types/types.go
|
||||||
|
+++ b/virtcontainers/pkg/types/types.go
|
||||||
|
@@ -31,6 +31,8 @@ type Interface struct {
|
||||||
|
LinkType string `json:"linkType,omitempty"`
|
||||||
|
// VhostUserSocket is DPDK-backed vHost user ports.
|
||||||
|
VhostUserSocket string `json:"vhostUserSocket,omitempty"`
|
||||||
|
+ // Queues specifies the interface queue number
|
||||||
|
+ Queues uint32 `json:"queues,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Route describes a network route.
|
||||||
|
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
|
||||||
|
index 84797e0d..be6e33b9 100644
|
||||||
|
--- a/virtcontainers/qemu.go
|
||||||
|
+++ b/virtcontainers/qemu.go
|
||||||
|
@@ -1464,7 +1464,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
-func (q *qemu) hotAddNetDevice(deviceName, name, hardAddr string, VMFds, VhostFds []*os.File) error {
|
||||||
|
+func (q *qemu) hotAddNetDevice(deviceName, name, hardAddr string, VMFds, VhostFds []*os.File, queues uint32) error {
|
||||||
|
var (
|
||||||
|
VMFdNames []string
|
||||||
|
VhostFdNames []string
|
||||||
|
@@ -1489,7 +1489,7 @@ func (q *qemu) hotAddNetDevice(deviceName, name, hardAddr string, VMFds, VhostFd
|
||||||
|
return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", name, VMFdNames, VhostFdNames)
|
||||||
|
}
|
||||||
|
|
||||||
|
- return q.qmpMonitorCh.qmp.ExecuteNetdevAdd(q.qmpMonitorCh.ctx, "tap", name, deviceName, "no", "no", 0)
|
||||||
|
+ return q.qmpMonitorCh.qmp.ExecuteNetdevAdd(q.qmpMonitorCh.ctx, "tap", name, deviceName, "no", "no", int(queues))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
|
||||||
|
@@ -1512,7 +1512,7 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
|
||||||
|
|
||||||
|
devID := "virtio-" + tap.ID
|
||||||
|
if op == addDevice {
|
||||||
|
- if err = q.hotAddNetDevice(tap.TAPIface.Name, tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds); err != nil {
|
||||||
|
+ if err = q.hotAddNetDevice(tap.TAPIface.Name, tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds, tap.Queues); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -1542,9 +1542,9 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
|
||||||
|
}
|
||||||
|
if machine.Type == QemuCCWVirtio {
|
||||||
|
devNoHotplug := fmt.Sprintf("fe.%x.%x", bus, addr)
|
||||||
|
- return q.qmpMonitorCh.qmp.ExecuteNetCCWDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), devNoHotplug, int(q.config.NumVCPUs))
|
||||||
|
+ return q.qmpMonitorCh.qmp.ExecuteNetCCWDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), devNoHotplug, int(tap.Queues))
|
||||||
|
}
|
||||||
|
- return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bus, romFile, int(q.config.NumVCPUs), defaultDisableModern)
|
||||||
|
+ return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bus, romFile, int(tap.Queues), defaultDisableModern)
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index e6f155a3..d8ab6c1a 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -948,6 +948,7 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) {
|
||||||
|
},
|
||||||
|
Addrs: addrs,
|
||||||
|
VhostUserSocket: inf.VhostUserSocket,
|
||||||
|
+ Queues: inf.Queues,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go
|
||||||
|
index 2cf70dce..5a3e7f7e 100644
|
||||||
|
--- a/virtcontainers/tap_endpoint.go
|
||||||
|
+++ b/virtcontainers/tap_endpoint.go
|
||||||
|
@@ -115,7 +115,7 @@ func (endpoint *TapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPat
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
-func createTapNetworkEndpoint(idx int, ifName string, tapIfName string) (*TapEndpoint, error) {
|
||||||
|
+func createTapNetworkEndpoint(idx int, ifName string, tapIfName string, queues uint32) (*TapEndpoint, error) {
|
||||||
|
if idx < 0 {
|
||||||
|
return &TapEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx)
|
||||||
|
}
|
||||||
|
@@ -139,6 +139,10 @@ func createTapNetworkEndpoint(idx int, ifName string, tapIfName string) (*TapEnd
|
||||||
|
endpoint.TapInterface.TAPIface.Name = tapIfName
|
||||||
|
}
|
||||||
|
|
||||||
|
+ if queues > 0 {
|
||||||
|
+ endpoint.TapInterface.Queues = queues
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
return endpoint, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/tuntap_endpoint.go b/virtcontainers/tuntap_endpoint.go
|
||||||
|
index b076d694..827d8852 100644
|
||||||
|
--- a/virtcontainers/tuntap_endpoint.go
|
||||||
|
+++ b/virtcontainers/tuntap_endpoint.go
|
||||||
|
@@ -117,12 +117,12 @@ func (endpoint *TuntapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNs
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
-func createTuntapNetworkEndpoint(idx int, ifName string, hwName net.HardwareAddr, internetworkingModel NetInterworkingModel) (*TuntapEndpoint, error) {
|
||||||
|
+func createTuntapNetworkEndpoint(idx int, ifName string, hwName net.HardwareAddr, internetworkingModel NetInterworkingModel, queues uint32) (*TuntapEndpoint, error) {
|
||||||
|
if idx < 0 {
|
||||||
|
return &TuntapEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx)
|
||||||
|
}
|
||||||
|
|
||||||
|
- netPair, err := createNetworkInterfacePair(idx, ifName, internetworkingModel)
|
||||||
|
+ netPair, err := createNetworkInterfacePair(idx, ifName, internetworkingModel, queues)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/veth_endpoint.go b/virtcontainers/veth_endpoint.go
|
||||||
|
index 0f2ec9ba..554b9b22 100644
|
||||||
|
--- a/virtcontainers/veth_endpoint.go
|
||||||
|
+++ b/virtcontainers/veth_endpoint.go
|
||||||
|
@@ -20,12 +20,12 @@ type VethEndpoint struct {
|
||||||
|
PCIAddr string
|
||||||
|
}
|
||||||
|
|
||||||
|
-func createVethNetworkEndpoint(idx int, ifName string, interworkingModel NetInterworkingModel) (*VethEndpoint, error) {
|
||||||
|
+func createVethNetworkEndpoint(idx int, ifName string, interworkingModel NetInterworkingModel, queues uint32) (*VethEndpoint, error) {
|
||||||
|
if idx < 0 {
|
||||||
|
return &VethEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx)
|
||||||
|
}
|
||||||
|
|
||||||
|
- netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel)
|
||||||
|
+ netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel, queues)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/veth_endpoint_test.go b/virtcontainers/veth_endpoint_test.go
|
||||||
|
index 9649b82e..23f1876d 100644
|
||||||
|
--- a/virtcontainers/veth_endpoint_test.go
|
||||||
|
+++ b/virtcontainers/veth_endpoint_test.go
|
||||||
|
@@ -34,7 +34,7 @@ func TestCreateVethNetworkEndpoint(t *testing.T) {
|
||||||
|
EndpointType: VethEndpointType,
|
||||||
|
}
|
||||||
|
|
||||||
|
- result, err := createVethNetworkEndpoint(4, "", DefaultNetInterworkingModel)
|
||||||
|
+ result, err := createVethNetworkEndpoint(4, "", DefaultNetInterworkingModel, 0)
|
||||||
|
assert.NoError(err)
|
||||||
|
|
||||||
|
// the resulting ID will be random - so let's overwrite to test the rest of the flow
|
||||||
|
@@ -68,7 +68,7 @@ func TestCreateVethNetworkEndpointChooseIfaceName(t *testing.T) {
|
||||||
|
EndpointType: VethEndpointType,
|
||||||
|
}
|
||||||
|
|
||||||
|
- result, err := createVethNetworkEndpoint(4, "eth1", DefaultNetInterworkingModel)
|
||||||
|
+ result, err := createVethNetworkEndpoint(4, "eth1", DefaultNetInterworkingModel, 0)
|
||||||
|
assert.NoError(err)
|
||||||
|
|
||||||
|
// the resulting ID will be random - so let's overwrite to test the rest of the flow
|
||||||
|
@@ -95,7 +95,7 @@ func TestCreateVethNetworkEndpointInvalidArgs(t *testing.T) {
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, d := range failingValues {
|
||||||
|
- _, err := createVethNetworkEndpoint(d.idx, d.ifName, DefaultNetInterworkingModel)
|
||||||
|
+ _, err := createVethNetworkEndpoint(d.idx, d.ifName, DefaultNetInterworkingModel, 0)
|
||||||
|
assert.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/vhostuser_endpoint.go b/virtcontainers/vhostuser_endpoint.go
|
||||||
|
index 2fc3d837..85ef67b4 100644
|
||||||
|
--- a/virtcontainers/vhostuser_endpoint.go
|
||||||
|
+++ b/virtcontainers/vhostuser_endpoint.go
|
||||||
|
@@ -33,6 +33,7 @@ type VhostUserEndpoint struct {
|
||||||
|
EndpointProperties NetworkInfo
|
||||||
|
EndpointType EndpointType
|
||||||
|
PCIAddr string
|
||||||
|
+ Queues uint32
|
||||||
|
}
|
||||||
|
|
||||||
|
// Properties returns the properties of the interface.
|
||||||
|
@@ -119,7 +120,7 @@ func (endpoint *VhostUserEndpoint) HotDetach(h hypervisor, netNsCreated bool, ne
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create a vhostuser endpoint
|
||||||
|
-func createVhostUserEndpoint(netInfo NetworkInfo, socket string) (*VhostUserEndpoint, error) {
|
||||||
|
+func createVhostUserEndpoint(netInfo NetworkInfo, socket string, queues uint32) (*VhostUserEndpoint, error) {
|
||||||
|
uniqueID := uuid.Generate().String()
|
||||||
|
vhostUserEndpoint := &VhostUserEndpoint{
|
||||||
|
ID: uniqueID,
|
||||||
|
@@ -127,6 +128,7 @@ func createVhostUserEndpoint(netInfo NetworkInfo, socket string) (*VhostUserEndp
|
||||||
|
HardAddr: netInfo.Iface.HardwareAddr.String(),
|
||||||
|
IfaceName: netInfo.Iface.Name,
|
||||||
|
EndpointType: VhostUserEndpointType,
|
||||||
|
+ Queues: queues,
|
||||||
|
}
|
||||||
|
return vhostUserEndpoint, nil
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/vhostuser_endpoint_test.go b/virtcontainers/vhostuser_endpoint_test.go
|
||||||
|
index 584490cc..046ca229 100644
|
||||||
|
--- a/virtcontainers/vhostuser_endpoint_test.go
|
||||||
|
+++ b/virtcontainers/vhostuser_endpoint_test.go
|
||||||
|
@@ -137,7 +137,7 @@ func TestCreateVhostUserEndpoint(t *testing.T) {
|
||||||
|
EndpointType: VhostUserEndpointType,
|
||||||
|
}
|
||||||
|
|
||||||
|
- result, err := createVhostUserEndpoint(netinfo, socket)
|
||||||
|
+ result, err := createVhostUserEndpoint(netinfo, socket, 0)
|
||||||
|
assert.NoError(err)
|
||||||
|
assert.Exactly(result, expected)
|
||||||
|
}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,34 @@
|
|||||||
|
From d3b8e21829bd671a1717fed8ab645ad2d447899a Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Wed, 19 Aug 2020 22:18:05 +0800
|
||||||
|
Subject: [PATCH 48/50] console: fix the file resource leak
|
||||||
|
|
||||||
|
reason: newConsole will open a file, if error occurs in the
|
||||||
|
middle proccess, the file resource may leak, use defer close
|
||||||
|
to prevent it
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
cli/console.go | 6 ++++++
|
||||||
|
1 file changed, 6 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/cli/console.go b/cli/console.go
|
||||||
|
index c1555c9f..97dfee99 100644
|
||||||
|
--- a/cli/console.go
|
||||||
|
+++ b/cli/console.go
|
||||||
|
@@ -47,6 +47,12 @@ func newConsole() (*Console, error) {
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
+ defer func() {
|
||||||
|
+ if err != nil {
|
||||||
|
+ master.Close()
|
||||||
|
+ }
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
if err := saneTerminal(master); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,58 @@
|
|||||||
|
From a6fab7014922d85b1105b44fdbb98239b22d3e00 Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Wed, 19 Aug 2020 22:31:57 +0800
|
||||||
|
Subject: [PATCH 49/50] container: fix the write operation transparently
|
||||||
|
transmitted to the host
|
||||||
|
|
||||||
|
reason:fix the write operation transparently transmitted to the host
|
||||||
|
when we fullfill the "/etc/hosts","/etc/resolv.conf","/etc/hostname" file in the container,
|
||||||
|
for example:
|
||||||
|
```bash
|
||||||
|
$ docker exec -ti 63 bash
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/container.go | 16 ++++++++++++++--
|
||||||
|
1 file changed, 14 insertions(+), 2 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/container.go b/virtcontainers/container.go
|
||||||
|
index 1b89f6ac..6edcb3f2 100644
|
||||||
|
--- a/virtcontainers/container.go
|
||||||
|
+++ b/virtcontainers/container.go
|
||||||
|
@@ -51,6 +51,12 @@ var cdromMajors = map[int64]string{
|
||||||
|
32: "CM206_CDROM_MAJOR",
|
||||||
|
}
|
||||||
|
|
||||||
|
+var safeCopyFiles = map[string]struct{}{
|
||||||
|
+ "resolv.conf": {},
|
||||||
|
+ "hostname": {},
|
||||||
|
+ "hosts": {},
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// https://github.com/torvalds/linux/blob/master/include/uapi/linux/major.h
|
||||||
|
// #define FLOPPY_MAJOR 2
|
||||||
|
const floppyMajor = int64(2)
|
||||||
|
@@ -452,12 +458,18 @@ func (c *Container) shareFiles(m Mount, idx int, hostSharedDir, guestSharedDir s
|
||||||
|
}
|
||||||
|
|
||||||
|
filename := fmt.Sprintf("%s-%s-%s", c.id, hex.EncodeToString(randBytes), filepath.Base(m.Destination))
|
||||||
|
- guestDest := filepath.Join(guestSharedDir, filename)
|
||||||
|
+ var guestDest string
|
||||||
|
+ _, needCopy := safeCopyFiles[filepath.Base(m.Destination)]
|
||||||
|
+ if needCopy {
|
||||||
|
+ guestDest = filepath.Join(kataGuestStorageDir, filename)
|
||||||
|
+ } else {
|
||||||
|
+ guestDest = filepath.Join(guestSharedDir, filename)
|
||||||
|
+ }
|
||||||
|
|
||||||
|
// copy file to contaier's rootfs if filesystem sharing is not supported, otherwise
|
||||||
|
// bind mount it in the shared directory.
|
||||||
|
caps := c.sandbox.hypervisor.capabilities()
|
||||||
|
- if !caps.IsFsSharingSupported() {
|
||||||
|
+ if !caps.IsFsSharingSupported() || needCopy {
|
||||||
|
c.Logger().Debug("filesystem sharing is not supported, files will be copied")
|
||||||
|
|
||||||
|
fileInfo, err := os.Stat(m.Source)
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,358 @@
|
|||||||
|
From f943e5d91f6922c599b46727e2dfd5e8e7e5bea8 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
Date: Wed, 19 Aug 2020 22:43:54 +0800
|
||||||
|
Subject: [PATCH 50/50] runtime: add kata-network upate-iface subcommand
|
||||||
|
|
||||||
|
reason: add kata-network update-iface subcommand to
|
||||||
|
update the exist interface in the Kata VM.
|
||||||
|
|
||||||
|
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||||
|
---
|
||||||
|
cli/network.go | 37 +++++++++-
|
||||||
|
virtcontainers/api.go | 25 +++++--
|
||||||
|
virtcontainers/implementation.go | 5 ++
|
||||||
|
virtcontainers/interfaces.go | 1 +
|
||||||
|
virtcontainers/pkg/vcmock/mock.go | 8 +++
|
||||||
|
virtcontainers/pkg/vcmock/types.go | 1 +
|
||||||
|
virtcontainers/sandbox.go | 137 +++++++++++++++++++++++++++++++++++++
|
||||||
|
7 files changed, 207 insertions(+), 7 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/cli/network.go b/cli/network.go
|
||||||
|
index 3dd0971e..7dce0528 100644
|
||||||
|
--- a/cli/network.go
|
||||||
|
+++ b/cli/network.go
|
||||||
|
@@ -65,8 +65,9 @@ var addIfaceCommand = cli.Command{
|
||||||
|
"hwAddr":"<mac>",
|
||||||
|
"linkType":"tap",
|
||||||
|
"vhostUserSocket":"<path>"
|
||||||
|
+ "queues":<queues>
|
||||||
|
}
|
||||||
|
- device,name,mtu,hwAddr are required, IPAddresses and vhostUserSocket are optional.
|
||||||
|
+ device,name,mtu,hwAddr are required, IPAddresses, queues and vhostUserSocket are optional.
|
||||||
|
`,
|
||||||
|
Flags: []cli.Flag{},
|
||||||
|
Action: func(context *cli.Context) error {
|
||||||
|
@@ -104,6 +105,34 @@ var delIfaceCommand = cli.Command{
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
+var updateIfaceCommand = cli.Command{
|
||||||
|
+ Name: "update-iface",
|
||||||
|
+ Usage: "update the interface in container",
|
||||||
|
+ ArgsUsage: `update-iface <container-id> file or - for stdin
|
||||||
|
+ file or stdin for example:
|
||||||
|
+ {
|
||||||
|
+ "device":"",
|
||||||
|
+ "name":"<interface-name>",
|
||||||
|
+ "IPAddresses":[{"address":"<ip>","mask":"<mask>"}],
|
||||||
|
+ "mtu":0,
|
||||||
|
+ "hwAddr":"",
|
||||||
|
+ "linkType":"",
|
||||||
|
+ "vhostUserSocket":"",
|
||||||
|
+ "queues":0
|
||||||
|
+ }
|
||||||
|
+ name is required,IPAddresses is optional,device,mtu,hwAddr,linkType,queues and vhostUserSocket should be 0 or empty.
|
||||||
|
+ `,
|
||||||
|
+ Flags: []cli.Flag{},
|
||||||
|
+ Action: func(context *cli.Context) error {
|
||||||
|
+ ctx, err := cliContextToContext(context)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return networkModifyCommand(ctx, context.Args().First(), context.Args().Get(1), interfaceType, vcTypes.NetworkOpUpdate)
|
||||||
|
+ },
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
var listIfacesCommand = cli.Command{
|
||||||
|
Name: "list-ifaces",
|
||||||
|
Usage: "list network interfaces in a container",
|
||||||
|
@@ -262,6 +291,12 @@ func networkModifyCommand(ctx context.Context, containerID, input string, opType
|
||||||
|
kataLog.WithField("resulting-interface", fmt.Sprintf("%+v", resultingInf)).
|
||||||
|
WithError(err).Error("delete interface failed")
|
||||||
|
}
|
||||||
|
+ case vcTypes.NetworkOpUpdate:
|
||||||
|
+ resultingInf, err = vci.UpdateInterface(ctx, sandboxID, inf)
|
||||||
|
+ if err != nil {
|
||||||
|
+ kataLog.WithField("resulting-interface", fmt.Sprintf("%+v", resultingInf)).
|
||||||
|
+ WithError(err).Error("update interface failed")
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
|
||||||
|
json.NewEncoder(output).Encode(resultingInf)
|
||||||
|
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
|
||||||
|
index 7036df8c..ca5412a9 100644
|
||||||
|
--- a/virtcontainers/api.go
|
||||||
|
+++ b/virtcontainers/api.go
|
||||||
|
@@ -889,7 +889,7 @@ func AddDevice(ctx context.Context, sandboxID string, info deviceConfig.DeviceIn
|
||||||
|
return s.AddDevice(info)
|
||||||
|
}
|
||||||
|
|
||||||
|
-func toggleInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface, add bool) (*vcTypes.Interface, error) {
|
||||||
|
+func toggleInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface, op vcTypes.NetworkOp) (*vcTypes.Interface, error) {
|
||||||
|
if sandboxID == "" {
|
||||||
|
return nil, vcTypes.ErrNeedSandboxID
|
||||||
|
}
|
||||||
|
@@ -906,11 +906,16 @@ func toggleInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interfa
|
||||||
|
}
|
||||||
|
defer s.releaseStatelessSandbox()
|
||||||
|
|
||||||
|
- if add {
|
||||||
|
+ switch op {
|
||||||
|
+ case vcTypes.NetworkOpAdd:
|
||||||
|
return s.AddInterface(inf)
|
||||||
|
+ case vcTypes.NetworkOpRemove:
|
||||||
|
+ return s.RemoveInterface(inf)
|
||||||
|
+ case vcTypes.NetworkOpUpdate:
|
||||||
|
+ return s.UpdateInterface(inf)
|
||||||
|
+ default:
|
||||||
|
+ return nil, fmt.Errorf("operation is not found")
|
||||||
|
}
|
||||||
|
-
|
||||||
|
- return s.RemoveInterface(inf)
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddInterface is the virtcontainers add interface entry point.
|
||||||
|
@@ -918,7 +923,7 @@ func AddInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface)
|
||||||
|
span, ctx := trace(ctx, "AddInterface")
|
||||||
|
defer span.Finish()
|
||||||
|
|
||||||
|
- return toggleInterface(ctx, sandboxID, inf, true)
|
||||||
|
+ return toggleInterface(ctx, sandboxID, inf, vcTypes.NetworkOpAdd)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RemoveInterface is the virtcontainers remove interface entry point.
|
||||||
|
@@ -926,7 +931,15 @@ func RemoveInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interfa
|
||||||
|
span, ctx := trace(ctx, "RemoveInterface")
|
||||||
|
defer span.Finish()
|
||||||
|
|
||||||
|
- return toggleInterface(ctx, sandboxID, inf, false)
|
||||||
|
+ return toggleInterface(ctx, sandboxID, inf, vcTypes.NetworkOpRemove)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// UpdateInterface updates interface entry point in the virtcontainers.
|
||||||
|
+func UpdateInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) {
|
||||||
|
+ span, ctx := trace(ctx, "UpdateInterface")
|
||||||
|
+ defer span.Finish()
|
||||||
|
+
|
||||||
|
+ return toggleInterface(ctx, sandboxID, inf, vcTypes.NetworkOpUpdate)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListInterfaces is the virtcontainers list interfaces entry point.
|
||||||
|
diff --git a/virtcontainers/implementation.go b/virtcontainers/implementation.go
|
||||||
|
index 0265d1ed..e4bc4ae6 100644
|
||||||
|
--- a/virtcontainers/implementation.go
|
||||||
|
+++ b/virtcontainers/implementation.go
|
||||||
|
@@ -163,6 +163,11 @@ func (impl *VCImpl) ListInterfaces(ctx context.Context, sandboxID string) ([]*vc
|
||||||
|
return ListInterfaces(ctx, sandboxID)
|
||||||
|
}
|
||||||
|
|
||||||
|
+// ListInterfaces implements the VC function of the same name.
|
||||||
|
+func (impl *VCImpl) UpdateInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) {
|
||||||
|
+ return UpdateInterface(ctx, sandboxID, inf)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// UpdateRoutes implements the VC function of the same name.
|
||||||
|
func (impl *VCImpl) UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) {
|
||||||
|
return UpdateRoutes(ctx, sandboxID, routes, op)
|
||||||
|
diff --git a/virtcontainers/interfaces.go b/virtcontainers/interfaces.go
|
||||||
|
index d8f0be69..0fd12d84 100644
|
||||||
|
--- a/virtcontainers/interfaces.go
|
||||||
|
+++ b/virtcontainers/interfaces.go
|
||||||
|
@@ -51,6 +51,7 @@ type VC interface {
|
||||||
|
|
||||||
|
AddInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error)
|
||||||
|
RemoveInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error)
|
||||||
|
+ UpdateInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error)
|
||||||
|
ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTypes.Interface, error)
|
||||||
|
UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error)
|
||||||
|
ListRoutes(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error)
|
||||||
|
diff --git a/virtcontainers/pkg/vcmock/mock.go b/virtcontainers/pkg/vcmock/mock.go
|
||||||
|
index aa9bae9a..d3bf201f 100644
|
||||||
|
--- a/virtcontainers/pkg/vcmock/mock.go
|
||||||
|
+++ b/virtcontainers/pkg/vcmock/mock.go
|
||||||
|
@@ -273,6 +273,14 @@ func (m *VCMock) ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTyp
|
||||||
|
return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID)
|
||||||
|
}
|
||||||
|
|
||||||
|
+// UpdateInterface implements the VC function of the same name
|
||||||
|
+func (m *VCMock) UpdateInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) {
|
||||||
|
+ if m.UpdateRoutesFunc != nil {
|
||||||
|
+ return m.UpdateInterfaceFunc(ctx, sandboxID, inf)
|
||||||
|
+ }
|
||||||
|
+ return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// UpdateRoutes implements the VC function of the same name.
|
||||||
|
func (m *VCMock) UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) {
|
||||||
|
if m.UpdateRoutesFunc != nil {
|
||||||
|
diff --git a/virtcontainers/pkg/vcmock/types.go b/virtcontainers/pkg/vcmock/types.go
|
||||||
|
index 610b4602..152e66b7 100644
|
||||||
|
--- a/virtcontainers/pkg/vcmock/types.go
|
||||||
|
+++ b/virtcontainers/pkg/vcmock/types.go
|
||||||
|
@@ -73,6 +73,7 @@ type VCMock struct {
|
||||||
|
AddInterfaceFunc func(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error)
|
||||||
|
RemoveInterfaceFunc func(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error)
|
||||||
|
ListInterfacesFunc func(ctx context.Context, sandboxID string) ([]*vcTypes.Interface, error)
|
||||||
|
+ UpdateInterfaceFunc func(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error)
|
||||||
|
UpdateRoutesFunc func(ctx context.Context, sandboxID string, routes []*vcTypes.Route) ([]*vcTypes.Route, error)
|
||||||
|
ListRoutesFunc func(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error)
|
||||||
|
CleanupContainerFunc func(ctx context.Context, sandboxID, containerID string, force bool) error
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index d8ab6c1a..174e6cb6 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -1031,6 +1031,54 @@ func (s *Sandbox) RemoveInterface(inf *vcTypes.Interface) (*vcTypes.Interface, e
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+// UpdateInterface updates the nic.
|
||||||
|
+func (s *Sandbox) UpdateInterface(inf *vcTypes.Interface) (*vcTypes.Interface, error) {
|
||||||
|
+ err := checkUpdateInterfaceInfo(inf)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ nicInGuest := false
|
||||||
|
+
|
||||||
|
+ endpoint := s.getEndpointByName(inf.Name)
|
||||||
|
+ // nic is not in local endpoints
|
||||||
|
+ if endpoint == nil {
|
||||||
|
+ interfaceInGuestInf := s.getInterfaceInfByName(inf.Name)
|
||||||
|
+ if interfaceInGuestInf == nil {
|
||||||
|
+ return nil, fmt.Errorf("can not find the network interface")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ nicInGuest = true
|
||||||
|
+ endpoint, err = s.generateEndpointForStore(interfaceInGuestInf)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // update endpoint ip
|
||||||
|
+ err = s.updateEndpointIP(endpoint, inf)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ s.Logger().WithField("endpoint-type", endpoint.Type()).Info("Update endpoint")
|
||||||
|
+ grpcIf, err := s.agent.updateInterface(inf)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if nicInGuest {
|
||||||
|
+ s.networkNS.Endpoints = append(s.networkNS.Endpoints, endpoint)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err = s.Save(); err != nil {
|
||||||
|
+ s.Logger().WithError(err).Error("failed to store the network information")
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return grpcIf, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// ListInterfaces lists all nics and their configurations in the sandbox.
|
||||||
|
func (s *Sandbox) ListInterfaces() ([]*vcTypes.Interface, error) {
|
||||||
|
return s.agent.listInterfaces()
|
||||||
|
@@ -2431,6 +2479,95 @@ func (s *Sandbox) IsCompatOldCNI() bool {
|
||||||
|
return s.config.NetworkConfig.EnableCompatOldCNI
|
||||||
|
}
|
||||||
|
|
||||||
|
+func checkUpdateInterfaceInfo(inf *vcTypes.Interface) error {
|
||||||
|
+ if inf.Mtu > 0 || inf.Queues > 0 || inf.Device != "" || inf.VhostUserSocket != "" || inf.LinkType != "" || inf.HwAddr != "" {
|
||||||
|
+ return fmt.Errorf("device,mtu,hwAddr,linkType,queues and vhostUserSocket should be 0 or empty")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// getEndpointByName returns the endpoint by name.
|
||||||
|
+func (s *Sandbox) getEndpointByName(name string) Endpoint {
|
||||||
|
+ if name == "" {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ for i, endpoint := range s.networkNS.Endpoints {
|
||||||
|
+ if endpoint.Name() == name {
|
||||||
|
+ return s.networkNS.Endpoints[i]
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// getInterfaceInfByName returns the interface information.
|
||||||
|
+func (s *Sandbox) getInterfaceInfByName(name string) *vcTypes.Interface {
|
||||||
|
+ if name == "" {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ nics, err := s.agent.listInterfaces()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ for _, i := range nics {
|
||||||
|
+ if i.Name == name {
|
||||||
|
+ return i
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// generateEndpointForStore only generates endpoint information for store,it does not create a endpoint
|
||||||
|
+func (s *Sandbox) generateEndpointForStore(inf *vcTypes.Interface) (*TapEndpoint, error) {
|
||||||
|
+ netInfo, err := s.generateNetInfo(inf)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ endpoint := &TapEndpoint{
|
||||||
|
+ TapInterface: TapInterface{
|
||||||
|
+ Name: inf.Name,
|
||||||
|
+ TAPIface: NetworkInterface{
|
||||||
|
+ HardAddr: inf.HwAddr,
|
||||||
|
+ },
|
||||||
|
+ },
|
||||||
|
+ EndpointType: TapEndpointType,
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ endpoint.SetProperties(netInfo)
|
||||||
|
+
|
||||||
|
+ return endpoint, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *Sandbox) updateEndpointIP(endpoint Endpoint, inf *vcTypes.Interface) error {
|
||||||
|
+ if endpoint == nil {
|
||||||
|
+ return fmt.Errorf("endpoint is nil")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ netInfo := endpoint.Properties()
|
||||||
|
+
|
||||||
|
+ var addrs []netlink.Addr
|
||||||
|
+ for _, addr := range inf.IPAddresses {
|
||||||
|
+ netlinkAddrStr := fmt.Sprintf("%s/%s", addr.Address, addr.Mask)
|
||||||
|
+ netlinkAddr, err := netlink.ParseAddr(netlinkAddrStr)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return fmt.Errorf("could not parse %q: %v", netlinkAddrStr, err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ addrs = append(addrs, *netlinkAddr)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ netInfo.Addrs = addrs
|
||||||
|
+ endpoint.SetProperties(netInfo)
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// updateStaticSandboxResources update sandbox's cpu and memory resource passed by
|
||||||
|
// sandbox_cpu and sandbox_mem annotations
|
||||||
|
func updateStaticSandboxResources(sandboxConfig *SandboxConfig) error {
|
||||||
|
--
|
||||||
|
2.14.3 (Apple Git-98)
|
||||||
|
|
||||||
@ -0,0 +1,78 @@
|
|||||||
|
From be0d60f5fa88267afb26125681a217ef9476e133 Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20102011@163.com>
|
||||||
|
Date: Sat, 19 Sep 2020 12:47:14 +0800
|
||||||
|
Subject: [PATCH] runtime: fix del-iface doesn't delete the tap interface in
|
||||||
|
the host problem
|
||||||
|
|
||||||
|
reason: kata-runtime add-iface support add exist tap device in the host or
|
||||||
|
new created tap device by kata-network add-iface process. If add interface
|
||||||
|
is exist tap device, del-iface will not delete the tap device, so we need
|
||||||
|
to judge the deleted interface is exist device or new created, which is judged
|
||||||
|
by the VMFds of the tap Link and del-iface will load this info from the
|
||||||
|
network.json in 1.7.0 version. However, 1.11.1 version don't write the VMFds
|
||||||
|
into the persiste.json file, del-iface can not figure out which tap is exist
|
||||||
|
already or new created, so cause the this problem.
|
||||||
|
---
|
||||||
|
virtcontainers/endpoint.go | 2 ++
|
||||||
|
virtcontainers/persist/api/network.go | 5 +++++
|
||||||
|
virtcontainers/tap_endpoint.go | 1 +
|
||||||
|
3 files changed, 8 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/endpoint.go b/virtcontainers/endpoint.go
|
||||||
|
index 7efcf49c..3618792e 100644
|
||||||
|
--- a/virtcontainers/endpoint.go
|
||||||
|
+++ b/virtcontainers/endpoint.go
|
||||||
|
@@ -129,6 +129,7 @@ func saveTapIf(tapif *TapInterface) *persistapi.TapInterface {
|
||||||
|
HardAddr: tapif.TAPIface.HardAddr,
|
||||||
|
Addrs: tapif.TAPIface.Addrs,
|
||||||
|
},
|
||||||
|
+ VMFds: tapif.VMFds,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -167,6 +168,7 @@ func loadTapIf(tapif *persistapi.TapInterface) *TapInterface {
|
||||||
|
HardAddr: tapif.TAPIface.HardAddr,
|
||||||
|
Addrs: tapif.TAPIface.Addrs,
|
||||||
|
},
|
||||||
|
+ VMFds: tapif.VMFds,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/persist/api/network.go b/virtcontainers/persist/api/network.go
|
||||||
|
index 53c6de44..c5611767 100644
|
||||||
|
--- a/virtcontainers/persist/api/network.go
|
||||||
|
+++ b/virtcontainers/persist/api/network.go
|
||||||
|
@@ -7,6 +7,8 @@
|
||||||
|
package persistapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
+ "os"
|
||||||
|
+
|
||||||
|
"github.com/vishvananda/netlink"
|
||||||
|
)
|
||||||
|
|
||||||
|
@@ -45,6 +47,9 @@ type TapInterface struct {
|
||||||
|
Name string
|
||||||
|
TAPIface NetworkInterface
|
||||||
|
// remove VMFds and VhostFds
|
||||||
|
+ // add VMFds back to judge a tap interface is exist before add-iface
|
||||||
|
+ // or new created by kata-network add-iface
|
||||||
|
+ VMFds []*os.File
|
||||||
|
}
|
||||||
|
|
||||||
|
// TuntapInterface defines a tap interface
|
||||||
|
diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go
|
||||||
|
index 5a3e7f7e..0b6002aa 100644
|
||||||
|
--- a/virtcontainers/tap_endpoint.go
|
||||||
|
+++ b/virtcontainers/tap_endpoint.go
|
||||||
|
@@ -199,6 +199,7 @@ func unTapNetwork(endpoint *TapEndpoint) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+ networkLogger().Debug("untap the new created tap interface")
|
||||||
|
name := endpoint.TapInterface.TAPIface.Name
|
||||||
|
netHandle, err := netlink.NewHandle()
|
||||||
|
if err != nil {
|
||||||
|
--
|
||||||
|
2.11.0
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,199 @@
|
|||||||
|
From 31bbb64a0682e326b354ac54c6596402fc20a977 Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Tue, 8 Sep 2020 20:03:41 +0800
|
||||||
|
Subject: [PATCH 1/5] kata-runtime: add interface for host cgroup
|
||||||
|
|
||||||
|
reason: add interface for host cgroup, including
|
||||||
|
CreateSandboxCgroup, DestroySandboxCgroup and
|
||||||
|
AddPidToSandboxCgroup, GetSandboxCgroupPath
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
virtcontainers/cgroups.go | 111 +++++++++++++++++++++++++++++++++++++++
|
||||||
|
virtcontainers/implementation.go | 20 +++++++
|
||||||
|
virtcontainers/interfaces.go | 5 ++
|
||||||
|
3 files changed, 136 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/cgroups.go b/virtcontainers/cgroups.go
|
||||||
|
index 4459df5..df0ec30 100644
|
||||||
|
--- a/virtcontainers/cgroups.go
|
||||||
|
+++ b/virtcontainers/cgroups.go
|
||||||
|
@@ -8,13 +8,20 @@ package virtcontainers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
+ "context"
|
||||||
|
+ "encoding/json"
|
||||||
|
"fmt"
|
||||||
|
+ "io/ioutil"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/containerd/cgroups"
|
||||||
|
specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
+ "github.com/sirupsen/logrus"
|
||||||
|
+
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/store"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
type cgroupPather interface {
|
||||||
|
@@ -30,6 +37,110 @@ const cgroupKataPath = "/kata/"
|
||||||
|
var cgroupsLoadFunc = cgroups.Load
|
||||||
|
var cgroupsNewFunc = cgroups.New
|
||||||
|
|
||||||
|
+// CreateSandboxCgroup create cgroup based on the first container's cgroupPath of sandbox
|
||||||
|
+func CreateSandboxCgroup(ctx context.Context, path string) error {
|
||||||
|
+ if path == "" {
|
||||||
|
+ return fmt.Errorf("sandbox cgroupPath shouldn't be empty!")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ var cgroupPath string
|
||||||
|
+ vcpuCgroupPath := filepath.Join(path, "vcpu")
|
||||||
|
+ if filepath.IsAbs(vcpuCgroupPath) {
|
||||||
|
+ cgroupPath = filepath.Clean(vcpuCgroupPath)
|
||||||
|
+ } else {
|
||||||
|
+ cgroupPath = filepath.Join(filepath.Clean("/" + vcpuCgroupPath))
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ resources := specs.LinuxResources{}
|
||||||
|
+ if _, err := cgroupsNewFunc(cgroups.V1, cgroups.StaticPath(cgroupPath), &resources); err != nil {
|
||||||
|
+ return fmt.Errorf("Could not create cgroup for %v: %v", cgroupPath, err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// DestroySandboxCgroup destroy the cgroup dir created for sandbox
|
||||||
|
+func DestroySandboxCgroup(ctx context.Context, cgroupPath string) error {
|
||||||
|
+ return deleteCgroup(cgroups.V1, cgroupPath)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func deleteCgroup(hierarchy cgroups.Hierarchy, cgroupPath string) error {
|
||||||
|
+ if cgroupPath == "" {
|
||||||
|
+ logrus.Warn("delete cgroupPath shouldn't be empty!")
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ cgroup, err := cgroupsLoadFunc(hierarchy,
|
||||||
|
+ cgroups.StaticPath(cgroupPath))
|
||||||
|
+
|
||||||
|
+ if err == cgroups.ErrCgroupDeleted {
|
||||||
|
+ // cgroup already deleted
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err != nil {
|
||||||
|
+ return fmt.Errorf("Could not load cgroup %v: %v", cgroupPath, err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // move running process here, that way cgroup can be removed
|
||||||
|
+ parent, err := parentCgroup(hierarchy, cgroupPath)
|
||||||
|
+ if err != nil {
|
||||||
|
+ // parent cgroup doesn't exist, that means there are no process running
|
||||||
|
+ // and the container cgroup was removed.
|
||||||
|
+ logrus.Warn(err)
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := cgroup.MoveTo(parent); err != nil {
|
||||||
|
+ // Don't fail, cgroup can be deleted
|
||||||
|
+ logrus.Warnf("Could not move container process into parent cgroup: %v", err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := cgroup.Delete(); err != nil {
|
||||||
|
+ return fmt.Errorf("Could not delete cgroup %v: %v", cgroupPath, err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// GetSandboxCgroupPath return the cgroup path of specified sandbox
|
||||||
|
+func GetSandboxCgroupPath(ctx context.Context, sandboxID string) (string, error) {
|
||||||
|
+ stateFilePath := filepath.Join(store.RunStoragePath(), sandboxID, store.StateFile)
|
||||||
|
+
|
||||||
|
+ fileData, err := ioutil.ReadFile(stateFilePath)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return "", err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ state := types.SandboxState{}
|
||||||
|
+
|
||||||
|
+ if err := json.Unmarshal(fileData, &state); err != nil {
|
||||||
|
+ return "", err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if state.CgroupPath == "" {
|
||||||
|
+ return "", fmt.Errorf("get sandbox cgroup path error: cgroupPath is empty")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return state.CgroupPath, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// AddPidToSandboxCgroup add kata-runtime create process to cgroup
|
||||||
|
+// the pid will be added to the cgroup of "<path>/vcpu"
|
||||||
|
+func AddPidToSandboxCgroup(ctx context.Context, pid int, cgroupPath string) error {
|
||||||
|
+ cgroup, err := cgroupsLoadFunc(cgroups.V1, cgroups.StaticPath(cgroupPath))
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ err = cgroup.Add(cgroups.Process{Pid: pid})
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// V1Constraints returns the cgroups that are compatible with the VC architecture
|
||||||
|
// and hypervisor, constraints can be applied to these cgroups.
|
||||||
|
func V1Constraints() ([]cgroups.Subsystem, error) {
|
||||||
|
diff --git a/virtcontainers/implementation.go b/virtcontainers/implementation.go
|
||||||
|
index e4bc4ae..fedc51f 100644
|
||||||
|
--- a/virtcontainers/implementation.go
|
||||||
|
+++ b/virtcontainers/implementation.go
|
||||||
|
@@ -188,3 +188,23 @@ func (impl *VCImpl) CleanupContainer(ctx context.Context, sandboxID, containerID
|
||||||
|
func (impl *VCImpl) UpdateIPVSRule(ctx context.Context, sandboxID string, IPVSRule *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
|
||||||
|
return UpdateIPVSRule(ctx, sandboxID, IPVSRule)
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+// CreateSandboxCgroup implements the VC function of the same name.
|
||||||
|
+func (impl *VCImpl) CreateSandboxCgroup(ctx context.Context, sandboxCgroupPath string) error {
|
||||||
|
+ return CreateSandboxCgroup(ctx, sandboxCgroupPath)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// DestroySandboxCgroup implements the VC function of the same name.
|
||||||
|
+func (impl *VCImpl) DestroySandboxCgroup(ctx context.Context, sandboxCgroupPath string) error {
|
||||||
|
+ return DestroySandboxCgroup(ctx, sandboxCgroupPath)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// AddPidToSandboxCgroup implements the VC function of the same name.
|
||||||
|
+func (impl *VCImpl) AddPidToSandboxCgroup(ctx context.Context, pid int, sandboxCgroupPath string) error {
|
||||||
|
+ return AddPidToSandboxCgroup(ctx, pid, sandboxCgroupPath)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// GetSandboxCgroupPath implements the VC function of the same name.
|
||||||
|
+func (impl *VCImpl) GetSandboxCgroupPath(ctx context.Context, sandboxID string) (string, error) {
|
||||||
|
+ return GetSandboxCgroupPath(ctx, sandboxID)
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/interfaces.go b/virtcontainers/interfaces.go
|
||||||
|
index 0fd12d8..4d166e0 100644
|
||||||
|
--- a/virtcontainers/interfaces.go
|
||||||
|
+++ b/virtcontainers/interfaces.go
|
||||||
|
@@ -24,6 +24,11 @@ type VC interface {
|
||||||
|
SetLogger(ctx context.Context, logger *logrus.Entry)
|
||||||
|
SetFactory(ctx context.Context, factory Factory)
|
||||||
|
|
||||||
|
+ CreateSandboxCgroup(ctx context.Context, sandboxCgroupPath string) error
|
||||||
|
+ DestroySandboxCgroup(ctx context.Context, sandboxCgroupPath string) error
|
||||||
|
+ AddPidToSandboxCgroup(ctx context.Context, pid int, sandboxCgroupPath string) error
|
||||||
|
+ GetSandboxCgroupPath(ctx context.Context, sandboxID string) (string, error)
|
||||||
|
+
|
||||||
|
CreateSandbox(ctx context.Context, sandboxConfig SandboxConfig) (VCSandbox, error)
|
||||||
|
DeleteSandbox(ctx context.Context, sandboxID string) (VCSandbox, error)
|
||||||
|
FetchSandbox(ctx context.Context, sandboxID string) (VCSandbox, error)
|
||||||
|
--
|
||||||
|
1.8.3.1
|
||||||
|
|
||||||
@ -0,0 +1,376 @@
|
|||||||
|
From 98a3c4677261e1c0364015f36928cddfb0af253e Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Wed, 9 Sep 2020 16:45:24 +0800
|
||||||
|
Subject: [PATCH 2/5] kata-runtime: add sandbox cgroup with vcpu and emulator
|
||||||
|
switch
|
||||||
|
|
||||||
|
reason: add sandbox cgroup with vcpu and emulator switch, if
|
||||||
|
sandbox_cgroup_with_emulator is true, it will overload the feature
|
||||||
|
of sandbox_cgroup_only, there will be two cgroups, vcpu and emulator
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
cli/config/configuration-qemu.toml.in | 12 +++++++++
|
||||||
|
cli/kata-env.go | 38 ++++++++++++++-------------
|
||||||
|
pkg/katautils/config.go | 18 +++++++------
|
||||||
|
virtcontainers/api.go | 4 ++-
|
||||||
|
virtcontainers/container.go | 6 ++---
|
||||||
|
virtcontainers/persist.go | 30 +++++++++++----------
|
||||||
|
virtcontainers/persist/api/config.go | 2 ++
|
||||||
|
virtcontainers/pkg/annotations/annotations.go | 2 ++
|
||||||
|
virtcontainers/pkg/oci/utils.go | 13 +++++++++
|
||||||
|
virtcontainers/sandbox.go | 23 +++++++++++-----
|
||||||
|
10 files changed, 97 insertions(+), 51 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in
|
||||||
|
index e57a954..fae88f9 100644
|
||||||
|
--- a/cli/config/configuration-qemu.toml.in
|
||||||
|
+++ b/cli/config/configuration-qemu.toml.in
|
||||||
|
@@ -477,6 +477,18 @@ enable_compat_old_cni = true
|
||||||
|
# See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType
|
||||||
|
sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
|
||||||
|
|
||||||
|
+# It is a new host cgroup solution to limit the kata resouce in the host different from the
|
||||||
|
+# community original solution.If sandbox_cgroup_with_emulator is enabled, it will override
|
||||||
|
+# the config of sandbox_cgroup_only. Each Pod corresponds to a pod level cgroup directory
|
||||||
|
+# which is named with sandboxID. In each pod level cgroup, it contains two sub cgroup
|
||||||
|
+# directory: vcpu and emulator, these two sub cgroup only valid in the CPU cgroup subsystem,
|
||||||
|
+# because we just want to distinguish the emulator main thread and vcpu thread in the CPU
|
||||||
|
+# cgroup subsystem.And with this config enabled, kata-runtime and related sub processes will
|
||||||
|
+# added into the vcpu cgroup directory with resource limited, and qemu main thread and other
|
||||||
|
+# non-vcpu threads will be moved into the emulator cgroup without resource limit, which will
|
||||||
|
+# improve the IO throughput for kata-containers.
|
||||||
|
+sandbox_cgroup_with_emulator = true
|
||||||
|
+
|
||||||
|
# Enabled experimental feature list, format: ["a", "b"].
|
||||||
|
# Experimental features are features not stable enough for production,
|
||||||
|
# they may break compatibility, and are prepared for a big version bump.
|
||||||
|
diff --git a/cli/kata-env.go b/cli/kata-env.go
|
||||||
|
index d8a6068..48026fe 100644
|
||||||
|
--- a/cli/kata-env.go
|
||||||
|
+++ b/cli/kata-env.go
|
||||||
|
@@ -63,15 +63,16 @@ type RuntimeConfigInfo struct {
|
||||||
|
|
||||||
|
// RuntimeInfo stores runtime details.
|
||||||
|
type RuntimeInfo struct {
|
||||||
|
- Version RuntimeVersionInfo
|
||||||
|
- Config RuntimeConfigInfo
|
||||||
|
- Debug bool
|
||||||
|
- Trace bool
|
||||||
|
- DisableGuestSeccomp bool
|
||||||
|
- DisableNewNetNs bool
|
||||||
|
- SandboxCgroupOnly bool
|
||||||
|
- Experimental []exp.Feature
|
||||||
|
- Path string
|
||||||
|
+ Version RuntimeVersionInfo
|
||||||
|
+ Config RuntimeConfigInfo
|
||||||
|
+ Debug bool
|
||||||
|
+ Trace bool
|
||||||
|
+ DisableGuestSeccomp bool
|
||||||
|
+ DisableNewNetNs bool
|
||||||
|
+ SandboxCgroupOnly bool
|
||||||
|
+ SandboxCgroupWithEmulator bool
|
||||||
|
+ Experimental []exp.Feature
|
||||||
|
+ Path string
|
||||||
|
}
|
||||||
|
|
||||||
|
type VersionInfo struct {
|
||||||
|
@@ -194,15 +195,16 @@ func getRuntimeInfo(configFile string, config oci.RuntimeConfig) RuntimeInfo {
|
||||||
|
runtimePath, _ := os.Executable()
|
||||||
|
|
||||||
|
return RuntimeInfo{
|
||||||
|
- Debug: config.Debug,
|
||||||
|
- Trace: config.Trace,
|
||||||
|
- Version: runtimeVersion,
|
||||||
|
- Config: runtimeConfig,
|
||||||
|
- Path: runtimePath,
|
||||||
|
- DisableNewNetNs: config.DisableNewNetNs,
|
||||||
|
- SandboxCgroupOnly: config.SandboxCgroupOnly,
|
||||||
|
- Experimental: config.Experimental,
|
||||||
|
- DisableGuestSeccomp: config.DisableGuestSeccomp,
|
||||||
|
+ Debug: config.Debug,
|
||||||
|
+ Trace: config.Trace,
|
||||||
|
+ Version: runtimeVersion,
|
||||||
|
+ Config: runtimeConfig,
|
||||||
|
+ Path: runtimePath,
|
||||||
|
+ DisableNewNetNs: config.DisableNewNetNs,
|
||||||
|
+ SandboxCgroupOnly: config.SandboxCgroupOnly,
|
||||||
|
+ SandboxCgroupWithEmulator: config.SandboxCgroupWithEmulator,
|
||||||
|
+ Experimental: config.Experimental,
|
||||||
|
+ DisableGuestSeccomp: config.DisableGuestSeccomp,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
||||||
|
index 3365b3f..89e46f6 100644
|
||||||
|
--- a/pkg/katautils/config.go
|
||||||
|
+++ b/pkg/katautils/config.go
|
||||||
|
@@ -139,14 +139,15 @@ type proxy struct {
|
||||||
|
}
|
||||||
|
|
||||||
|
type runtime struct {
|
||||||
|
- Debug bool `toml:"enable_debug"`
|
||||||
|
- Tracing bool `toml:"enable_tracing"`
|
||||||
|
- DisableNewNetNs bool `toml:"disable_new_netns"`
|
||||||
|
- EnableCompatOldCNI bool `toml:"enable_compat_old_cni"`
|
||||||
|
- DisableGuestSeccomp bool `toml:"disable_guest_seccomp"`
|
||||||
|
- SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"`
|
||||||
|
- Experimental []string `toml:"experimental"`
|
||||||
|
- InterNetworkModel string `toml:"internetworking_model"`
|
||||||
|
+ Debug bool `toml:"enable_debug"`
|
||||||
|
+ Tracing bool `toml:"enable_tracing"`
|
||||||
|
+ DisableNewNetNs bool `toml:"disable_new_netns"`
|
||||||
|
+ EnableCompatOldCNI bool `toml:"enable_compat_old_cni"`
|
||||||
|
+ DisableGuestSeccomp bool `toml:"disable_guest_seccomp"`
|
||||||
|
+ SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"`
|
||||||
|
+ SandboxCgroupWithEmulator bool `toml:"sandbox_cgroup_with_emulator"`
|
||||||
|
+ Experimental []string `toml:"experimental"`
|
||||||
|
+ InterNetworkModel string `toml:"internetworking_model"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type shim struct {
|
||||||
|
@@ -1252,6 +1253,7 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool, debugFlag
|
||||||
|
}
|
||||||
|
|
||||||
|
config.SandboxCgroupOnly = tomlConf.Runtime.SandboxCgroupOnly
|
||||||
|
+ config.SandboxCgroupWithEmulator = tomlConf.Runtime.SandboxCgroupWithEmulator
|
||||||
|
config.DisableNewNetNs = tomlConf.Runtime.DisableNewNetNs
|
||||||
|
config.EnableCompatOldCNI = tomlConf.Runtime.EnableCompatOldCNI
|
||||||
|
for _, f := range tomlConf.Runtime.Experimental {
|
||||||
|
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
|
||||||
|
index ca5412a..08bcbb5 100644
|
||||||
|
--- a/virtcontainers/api.go
|
||||||
|
+++ b/virtcontainers/api.go
|
||||||
|
@@ -103,7 +103,9 @@ func createSandboxFromConfig(ctx context.Context, sandboxConfig SandboxConfig, f
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Move runtime to sandbox cgroup so all process are created there.
|
||||||
|
- if s.config.SandboxCgroupOnly {
|
||||||
|
+ if s.config.SandboxCgroupWithEmulator{
|
||||||
|
+ // emulator
|
||||||
|
+ } else if s.config.SandboxCgroupOnly {
|
||||||
|
if err := s.setupSandboxCgroup(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/container.go b/virtcontainers/container.go
|
||||||
|
index 4060ebb..1b70382 100644
|
||||||
|
--- a/virtcontainers/container.go
|
||||||
|
+++ b/virtcontainers/container.go
|
||||||
|
@@ -1009,7 +1009,7 @@ func (c *Container) create() (err error) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
- if !rootless.IsRootless() && !c.sandbox.config.SandboxCgroupOnly {
|
||||||
|
+ if !rootless.IsRootless() && !c.sandbox.config.SandboxCgroupOnly && !c.sandbox.config.SandboxCgroupWithEmulator {
|
||||||
|
if err = c.cgroupsCreate(); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
@@ -1034,7 +1034,7 @@ func (c *Container) delete() error {
|
||||||
|
}
|
||||||
|
|
||||||
|
// If running rootless, there are no cgroups to remove
|
||||||
|
- if !c.sandbox.config.SandboxCgroupOnly || !rootless.IsRootless() {
|
||||||
|
+ if !c.sandbox.config.SandboxCgroupWithEmulator && (!c.sandbox.config.SandboxCgroupOnly || !rootless.IsRootless()) {
|
||||||
|
if err := c.cgroupsDelete(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
@@ -1348,7 +1348,7 @@ func (c *Container) update(resources specs.LinuxResources) error {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
- if !c.sandbox.config.SandboxCgroupOnly {
|
||||||
|
+ if !c.sandbox.config.SandboxCgroupWithEmulator && !c.sandbox.config.SandboxCgroupOnly {
|
||||||
|
if err := c.cgroupsUpdate(resources); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go
|
||||||
|
index fe00bf9..efa4506 100644
|
||||||
|
--- a/virtcontainers/persist.go
|
||||||
|
+++ b/virtcontainers/persist.go
|
||||||
|
@@ -194,13 +194,14 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
|
||||||
|
InterworkingModel: int(sconfig.NetworkConfig.InterworkingModel),
|
||||||
|
},
|
||||||
|
|
||||||
|
- ShmSize: sconfig.ShmSize,
|
||||||
|
- SharePidNs: sconfig.SharePidNs,
|
||||||
|
- Stateful: sconfig.Stateful,
|
||||||
|
- SystemdCgroup: sconfig.SystemdCgroup,
|
||||||
|
- SandboxCgroupOnly: sconfig.SandboxCgroupOnly,
|
||||||
|
- DisableGuestSeccomp: sconfig.DisableGuestSeccomp,
|
||||||
|
- Cgroups: sconfig.Cgroups,
|
||||||
|
+ ShmSize: sconfig.ShmSize,
|
||||||
|
+ SharePidNs: sconfig.SharePidNs,
|
||||||
|
+ Stateful: sconfig.Stateful,
|
||||||
|
+ SystemdCgroup: sconfig.SystemdCgroup,
|
||||||
|
+ SandboxCgroupOnly: sconfig.SandboxCgroupOnly,
|
||||||
|
+ SandboxCgroupWithEmulator: sconfig.SandboxCgroupWithEmulator,
|
||||||
|
+ DisableGuestSeccomp: sconfig.DisableGuestSeccomp,
|
||||||
|
+ Cgroups: sconfig.Cgroups,
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, e := range sconfig.Experimental {
|
||||||
|
@@ -485,13 +486,14 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
|
||||||
|
InterworkingModel: NetInterworkingModel(savedConf.NetworkConfig.InterworkingModel),
|
||||||
|
},
|
||||||
|
|
||||||
|
- ShmSize: savedConf.ShmSize,
|
||||||
|
- SharePidNs: savedConf.SharePidNs,
|
||||||
|
- Stateful: savedConf.Stateful,
|
||||||
|
- SystemdCgroup: savedConf.SystemdCgroup,
|
||||||
|
- SandboxCgroupOnly: savedConf.SandboxCgroupOnly,
|
||||||
|
- DisableGuestSeccomp: savedConf.DisableGuestSeccomp,
|
||||||
|
- Cgroups: savedConf.Cgroups,
|
||||||
|
+ ShmSize: savedConf.ShmSize,
|
||||||
|
+ SharePidNs: savedConf.SharePidNs,
|
||||||
|
+ Stateful: savedConf.Stateful,
|
||||||
|
+ SystemdCgroup: savedConf.SystemdCgroup,
|
||||||
|
+ SandboxCgroupOnly: savedConf.SandboxCgroupOnly,
|
||||||
|
+ SandboxCgroupWithEmulator: savedConf.SandboxCgroupWithEmulator,
|
||||||
|
+ DisableGuestSeccomp: savedConf.DisableGuestSeccomp,
|
||||||
|
+ Cgroups: savedConf.Cgroups,
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, name := range savedConf.Experimental {
|
||||||
|
diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go
|
||||||
|
index 3a2df32..28204fc 100644
|
||||||
|
--- a/virtcontainers/persist/api/config.go
|
||||||
|
+++ b/virtcontainers/persist/api/config.go
|
||||||
|
@@ -258,6 +258,8 @@ type SandboxConfig struct {
|
||||||
|
// SandboxCgroupOnly enables cgroup only at podlevel in the host
|
||||||
|
SandboxCgroupOnly bool
|
||||||
|
|
||||||
|
+ SandboxCgroupWithEmulator bool
|
||||||
|
+
|
||||||
|
DisableGuestSeccomp bool
|
||||||
|
|
||||||
|
// Experimental enables experimental features
|
||||||
|
diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go
|
||||||
|
index 528dfa6..96c4ef2 100644
|
||||||
|
--- a/virtcontainers/pkg/annotations/annotations.go
|
||||||
|
+++ b/virtcontainers/pkg/annotations/annotations.go
|
||||||
|
@@ -215,6 +215,8 @@ const (
|
||||||
|
// SandboxCgroupOnly is a sandbox annotation that determines if kata processes are managed only in sandbox cgroup.
|
||||||
|
SandboxCgroupOnly = kataAnnotRuntimePrefix + "sandbox_cgroup_only"
|
||||||
|
|
||||||
|
+ SandboxCgroupWithEmulator = kataAnnotRuntimePrefix + "sandbox_cgroup_with_emulator"
|
||||||
|
+
|
||||||
|
// Experimental is a sandbox annotation that determines if experimental features enabled.
|
||||||
|
Experimental = kataAnnotRuntimePrefix + "experimental"
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
||||||
|
index 3b2af75..91067fb 100644
|
||||||
|
--- a/virtcontainers/pkg/oci/utils.go
|
||||||
|
+++ b/virtcontainers/pkg/oci/utils.go
|
||||||
|
@@ -139,6 +139,8 @@ type RuntimeConfig struct {
|
||||||
|
//Determines kata processes are managed only in sandbox cgroup
|
||||||
|
SandboxCgroupOnly bool
|
||||||
|
|
||||||
|
+ SandboxCgroupWithEmulator bool
|
||||||
|
+
|
||||||
|
//Experimental features enabled
|
||||||
|
Experimental []exp.Feature
|
||||||
|
}
|
||||||
|
@@ -746,6 +748,15 @@ func addRuntimeConfigOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e
|
||||||
|
sbConfig.SandboxCgroupOnly = sandboxCgroupOnly
|
||||||
|
}
|
||||||
|
|
||||||
|
+ if value, ok := ocispec.Annotations[vcAnnotations.SandboxCgroupWithEmulator]; ok {
|
||||||
|
+ sandboxCgroupWithEmulator, err := strconv.ParseBool(value)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return fmt.Errorf("error parsing annotation for sandbox_cgroup_with_emulator : Please specify boolean value 'true|false'")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ sbConfig.SandboxCgroupWithEmulator = sandboxCgroupWithEmulator
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
if value, ok := ocispec.Annotations[vcAnnotations.Experimental]; ok {
|
||||||
|
features := strings.Split(value, " ")
|
||||||
|
sbConfig.Experimental = []exp.Feature{}
|
||||||
|
@@ -869,6 +880,8 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid, c
|
||||||
|
|
||||||
|
SandboxCgroupOnly: runtime.SandboxCgroupOnly,
|
||||||
|
|
||||||
|
+ SandboxCgroupWithEmulator: runtime.SandboxCgroupWithEmulator,
|
||||||
|
+
|
||||||
|
DisableGuestSeccomp: runtime.DisableGuestSeccomp,
|
||||||
|
|
||||||
|
// Q: Is this really necessary? @weizhang555
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index 174e6cb..b479cf5 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -126,6 +126,8 @@ type SandboxConfig struct {
|
||||||
|
// SandboxCgroupOnly enables cgroup only at podlevel in the host
|
||||||
|
SandboxCgroupOnly bool
|
||||||
|
|
||||||
|
+ SandboxCgroupWithEmulator bool
|
||||||
|
+
|
||||||
|
DisableGuestSeccomp bool
|
||||||
|
|
||||||
|
// Experimental features enabled
|
||||||
|
@@ -1532,8 +1534,9 @@ func (s *Sandbox) Stats() (SandboxStats, error) {
|
||||||
|
|
||||||
|
var path string
|
||||||
|
var cgroupSubsystems cgroups.Hierarchy
|
||||||
|
-
|
||||||
|
- if s.config.SandboxCgroupOnly {
|
||||||
|
+ if !s.config.SandboxCgroupWithEmulator {
|
||||||
|
+ // vcpu and emulator
|
||||||
|
+ } else if s.config.SandboxCgroupOnly {
|
||||||
|
cgroupSubsystems = cgroups.V1
|
||||||
|
path = s.state.CgroupPath
|
||||||
|
} else {
|
||||||
|
@@ -1793,7 +1796,9 @@ func (s *Sandbox) HotplugAddDevice(device api.Device, devType config.DeviceType)
|
||||||
|
span, _ := s.trace("HotplugAddDevice")
|
||||||
|
defer span.Finish()
|
||||||
|
|
||||||
|
- if s.config.SandboxCgroupOnly {
|
||||||
|
+ if s.config.SandboxCgroupWithEmulator {
|
||||||
|
+ // emulator
|
||||||
|
+ } else if s.config.SandboxCgroupOnly {
|
||||||
|
// We are about to add a device to the hypervisor,
|
||||||
|
// the device cgroup MUST be updated since the hypervisor
|
||||||
|
// will need access to such device
|
||||||
|
@@ -1849,7 +1854,9 @@ func (s *Sandbox) HotplugAddDevice(device api.Device, devType config.DeviceType)
|
||||||
|
// Sandbox implement DeviceReceiver interface from device/api/interface.go
|
||||||
|
func (s *Sandbox) HotplugRemoveDevice(device api.Device, devType config.DeviceType) error {
|
||||||
|
defer func() {
|
||||||
|
- if s.config.SandboxCgroupOnly {
|
||||||
|
+ if s.config.SandboxCgroupWithEmulator {
|
||||||
|
+
|
||||||
|
+ } else if s.config.SandboxCgroupOnly {
|
||||||
|
// Remove device from cgroup, the hypervisor
|
||||||
|
// should not have access to such device anymore.
|
||||||
|
hdev := device.GetHostPath()
|
||||||
|
@@ -2107,7 +2114,7 @@ func (s *Sandbox) cgroupsUpdate() error {
|
||||||
|
// If Kata is configured for SandboxCgroupOnly, the VMM and its processes are already
|
||||||
|
// in the Kata sandbox cgroup (inherited). No need to move threads/processes, and we should
|
||||||
|
// rely on parent's cgroup CPU/memory values
|
||||||
|
- if s.config.SandboxCgroupOnly {
|
||||||
|
+ if s.config.SandboxCgroupWithEmulator || s.config.SandboxCgroupOnly {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -2154,7 +2161,9 @@ func (s *Sandbox) cgroupsDelete() error {
|
||||||
|
var path string
|
||||||
|
var cgroupSubsystems cgroups.Hierarchy
|
||||||
|
|
||||||
|
- if s.config.SandboxCgroupOnly {
|
||||||
|
+ if s.config.SandboxCgroupWithEmulator {
|
||||||
|
+ // emulator
|
||||||
|
+ } else if s.config.SandboxCgroupOnly {
|
||||||
|
return s.cgroupMgr.Destroy()
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -2197,7 +2206,7 @@ func (s *Sandbox) constrainHypervisor(cgroup cgroups.Cgroup) error {
|
||||||
|
// Kata/VMM into account, Kata may fail to boot due to being overconstrained.
|
||||||
|
// If !SandboxCgroupOnly, place the VMM into an unconstrained cgroup, and the vCPU threads into constrained
|
||||||
|
// cgroup
|
||||||
|
- if s.config.SandboxCgroupOnly {
|
||||||
|
+ if s.config.SandboxCgroupOnly || s.config.SandboxCgroupWithEmulator {
|
||||||
|
// Kata components were moved into the sandbox-cgroup already, so VMM
|
||||||
|
// will already land there as well. No need to take action
|
||||||
|
return nil
|
||||||
|
--
|
||||||
|
1.8.3.1
|
||||||
|
|
||||||
@ -0,0 +1,428 @@
|
|||||||
|
From ce7523dfe1bb60cf54254e16a103fd3fc9503618 Mon Sep 17 00:00:00 2001
|
||||||
|
From: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
Date: Thu, 17 Sep 2020 10:38:38 +0800
|
||||||
|
Subject: [PATCH 3/5] kata_runtime: support host cgroup with emulator policy
|
||||||
|
|
||||||
|
reason: support host cgroup with emulator policy when
|
||||||
|
sandbox_cgroup_with_emulator is set true
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
cli/create.go | 38 ++++++++++++
|
||||||
|
virtcontainers/api.go | 10 ++-
|
||||||
|
virtcontainers/cgroups.go | 132 ++++++++++++++++++++++++++++++++++------
|
||||||
|
virtcontainers/persist/fs/fs.go | 8 +++
|
||||||
|
virtcontainers/pkg/oci/utils.go | 14 +++++
|
||||||
|
virtcontainers/sandbox.go | 70 ++++++++++++++++++++-
|
||||||
|
6 files changed, 250 insertions(+), 22 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/cli/create.go b/cli/create.go
|
||||||
|
index 02cb2c5..b14434b 100644
|
||||||
|
--- a/cli/create.go
|
||||||
|
+++ b/cli/create.go
|
||||||
|
@@ -11,6 +11,7 @@ import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
+ "path/filepath"
|
||||||
|
|
||||||
|
"github.com/kata-containers/runtime/pkg/katautils"
|
||||||
|
vc "github.com/kata-containers/runtime/virtcontainers"
|
||||||
|
@@ -134,11 +135,48 @@ func create(ctx context.Context, containerID, bundlePath, console, pidFilePath s
|
||||||
|
var process vc.Process
|
||||||
|
switch containerType {
|
||||||
|
case vc.PodSandbox:
|
||||||
|
+ if runtimeConfig.SandboxCgroupWithEmulator {
|
||||||
|
+ // create the sandbox level cgroup
|
||||||
|
+ cgroupPath := ociSpec.Linux.CgroupsPath
|
||||||
|
+ if err = vci.CreateSandboxCgroup(ctx, cgroupPath); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ defer func() {
|
||||||
|
+ if err != nil {
|
||||||
|
+ _ = vci.DestroySandboxCgroup(ctx, cgroupPath)
|
||||||
|
+ }
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
+ // add kata-runtime create process into <path>/vcpu cgroup
|
||||||
|
+ vcpuCgroupPath := filepath.Join(cgroupPath, "vcpu")
|
||||||
|
+ if err = vci.AddPidToSandboxCgroup(ctx, os.Getpid(), vcpuCgroupPath); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
_, process, err = katautils.CreateSandbox(ctx, vci, ociSpec, runtimeConfig, rootFs, containerID, bundlePath, console, disableOutput, systemdCgroup, false)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
case vc.PodContainer:
|
||||||
|
+ if runtimeConfig.SandboxCgroupWithEmulator {
|
||||||
|
+ sandboxID, err := oci.GetSandboxIDFromAnnotations(&ociSpec)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return fmt.Errorf("container annotation doesn't contain sandboxID")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ sandboxCgroupPath, err := vci.GetSandboxCgroupPath(ctx, sandboxID)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // add kata-runtime create process into <path>/vcpu cgroup
|
||||||
|
+ vcpuCgroupPath := filepath.Join(sandboxCgroupPath, "vcpu")
|
||||||
|
+ if err = vci.AddPidToSandboxCgroup(ctx, os.Getpid(), vcpuCgroupPath); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
process, err = katautils.CreateContainer(ctx, vci, nil, ociSpec, rootFs, containerID, bundlePath, console, disableOutput, false)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
|
||||||
|
index 08bcbb5..38c8235 100644
|
||||||
|
--- a/virtcontainers/api.go
|
||||||
|
+++ b/virtcontainers/api.go
|
||||||
|
@@ -103,9 +103,7 @@ func createSandboxFromConfig(ctx context.Context, sandboxConfig SandboxConfig, f
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Move runtime to sandbox cgroup so all process are created there.
|
||||||
|
- if s.config.SandboxCgroupWithEmulator{
|
||||||
|
- // emulator
|
||||||
|
- } else if s.config.SandboxCgroupOnly {
|
||||||
|
+ if !s.config.SandboxCgroupWithEmulator && s.config.SandboxCgroupOnly {
|
||||||
|
if err := s.setupSandboxCgroup(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
@@ -129,6 +127,12 @@ func createSandboxFromConfig(ctx context.Context, sandboxConfig SandboxConfig, f
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
+ if s.config.SandboxCgroupWithEmulator {
|
||||||
|
+ if err := s.setupHostCgroupsWithEmulator(); err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
// Create Containers
|
||||||
|
if err = s.createContainers(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
diff --git a/virtcontainers/cgroups.go b/virtcontainers/cgroups.go
|
||||||
|
index df0ec30..65d2001 100644
|
||||||
|
--- a/virtcontainers/cgroups.go
|
||||||
|
+++ b/virtcontainers/cgroups.go
|
||||||
|
@@ -9,19 +9,15 @@ package virtcontainers
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"context"
|
||||||
|
- "encoding/json"
|
||||||
|
"fmt"
|
||||||
|
- "io/ioutil"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
+ "strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/containerd/cgroups"
|
||||||
|
specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
-
|
||||||
|
- "github.com/kata-containers/runtime/virtcontainers/store"
|
||||||
|
- "github.com/kata-containers/runtime/virtcontainers/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
type cgroupPather interface {
|
||||||
|
@@ -32,7 +28,11 @@ type cgroupPather interface {
|
||||||
|
// unconstrained cgroups are placed here.
|
||||||
|
// for example /sys/fs/cgroup/memory/kata/$CGPATH
|
||||||
|
// where path is defined by the containers manager
|
||||||
|
-const cgroupKataPath = "/kata/"
|
||||||
|
+const (
|
||||||
|
+ cgroupKataPath = "/kata/"
|
||||||
|
+ vcpuCgroupName = "vcpu"
|
||||||
|
+ emulatorCgroupName = "emulator"
|
||||||
|
+)
|
||||||
|
|
||||||
|
var cgroupsLoadFunc = cgroups.Load
|
||||||
|
var cgroupsNewFunc = cgroups.New
|
||||||
|
@@ -105,24 +105,16 @@ func deleteCgroup(hierarchy cgroups.Hierarchy, cgroupPath string) error {
|
||||||
|
|
||||||
|
// GetSandboxCgroupPath return the cgroup path of specified sandbox
|
||||||
|
func GetSandboxCgroupPath(ctx context.Context, sandboxID string) (string, error) {
|
||||||
|
- stateFilePath := filepath.Join(store.RunStoragePath(), sandboxID, store.StateFile)
|
||||||
|
-
|
||||||
|
- fileData, err := ioutil.ReadFile(stateFilePath)
|
||||||
|
+ config, err := loadSandboxConfig(sandboxID)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
- state := types.SandboxState{}
|
||||||
|
-
|
||||||
|
- if err := json.Unmarshal(fileData, &state); err != nil {
|
||||||
|
- return "", err
|
||||||
|
- }
|
||||||
|
-
|
||||||
|
- if state.CgroupPath == "" {
|
||||||
|
- return "", fmt.Errorf("get sandbox cgroup path error: cgroupPath is empty")
|
||||||
|
+ if config.Cgroups == nil {
|
||||||
|
+ return "", fmt.Errorf("the cgroups of sandbox %s is nil", sandboxID)
|
||||||
|
}
|
||||||
|
|
||||||
|
- return state.CgroupPath, nil
|
||||||
|
+ return config.Cgroups.Path, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddPidToSandboxCgroup add kata-runtime create process to cgroup
|
||||||
|
@@ -276,3 +268,107 @@ func validCPUResources(cpuSpec *specs.LinuxCPU) *specs.LinuxCPU {
|
||||||
|
|
||||||
|
return &cpu
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+// getQemuTaskWithoutVcpu filter out tasks under /proc/{qemu pid}/task, to find out the task of not VCPU,
|
||||||
|
+// VCPU task is filtered by "query-cpus" qmp command
|
||||||
|
+func getQemuTaskWithoutVcpu(sandbox *Sandbox, vmPid int) []int {
|
||||||
|
+ procPath := fmt.Sprintf("/proc/%d/task", vmPid)
|
||||||
|
+
|
||||||
|
+ dirReader, err := os.Open(procPath)
|
||||||
|
+ if err != nil {
|
||||||
|
+ logrus.Warningf("cannot open %s: %s", procPath, err)
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ defer dirReader.Close()
|
||||||
|
+
|
||||||
|
+ dirs, err := dirReader.Readdirnames(0)
|
||||||
|
+ if err != nil {
|
||||||
|
+ logrus.Warningf("walking dirs in %s failed: %s", procPath, err)
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ vcpuThreadInfo, err := sandbox.hypervisor.getThreadIDs()
|
||||||
|
+ if err != nil {
|
||||||
|
+ logrus.Warnf("get hypervisor Thread ID failed: %v", err)
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ var vcpuThreadIDs []int
|
||||||
|
+ for _, value := range vcpuThreadInfo.vcpus {
|
||||||
|
+ vcpuThreadIDs = append(vcpuThreadIDs, value)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ var allThreadIDs []int
|
||||||
|
+ for _, dir := range dirs {
|
||||||
|
+ p, err := strconv.Atoi(dir)
|
||||||
|
+ if err != nil {
|
||||||
|
+ logrus.Warnf("can not change string dir: %s to int type", dir)
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ allThreadIDs = append(allThreadIDs, p)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ nonVCPUThreads := diffSlice(allThreadIDs, vcpuThreadIDs)
|
||||||
|
+
|
||||||
|
+ return nonVCPUThreads
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func pulloutQemuThread(sandbox *Sandbox, vmPid int, path string) error {
|
||||||
|
+ control, err := cgroups.New(cgroups.SingleSubsystem(cgroups.V1, cgroups.Cpu),
|
||||||
|
+ cgroups.StaticPath(path),
|
||||||
|
+ &specs.LinuxResources{})
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ taskIds := getQemuTaskWithoutVcpu(sandbox, vmPid)
|
||||||
|
+ if len(taskIds) == 0 {
|
||||||
|
+ logrus.Warnf("no taskId id in qemu other than vcpu found of pid %d", vmPid)
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+ for _, taskId := range taskIds {
|
||||||
|
+ if err := control.AddTask(cgroups.Process{
|
||||||
|
+ Pid: taskId,
|
||||||
|
+ }); err != nil {
|
||||||
|
+ logrus.Errorf("failed to add task %d to cgroup of %s", taskId, path)
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// checkCgroupExist check cgroup exist or not
|
||||||
|
+func checkCgroupExist(hierarchy cgroups.Hierarchy, path string) bool {
|
||||||
|
+ subSystems, _ := hierarchy()
|
||||||
|
+ for _, s := range cgroupPathers(subSystems) {
|
||||||
|
+ if _, err := os.Lstat(s.Path(path)); err != nil {
|
||||||
|
+ if os.IsNotExist(err) {
|
||||||
|
+ return false
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return true
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// diffSlice return the s1 - s2
|
||||||
|
+func diffSlice(s1, s2 []int) []int {
|
||||||
|
+ var diffSlice []int
|
||||||
|
+ for _, p := range s1 {
|
||||||
|
+ if !isInSlice(p, s2) {
|
||||||
|
+ diffSlice = append(diffSlice, p)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ return diffSlice
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func isInSlice(i int, s []int) bool {
|
||||||
|
+ for _, v := range s {
|
||||||
|
+ if i == v {
|
||||||
|
+ return true
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ return false
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/persist/fs/fs.go b/virtcontainers/persist/fs/fs.go
|
||||||
|
index 38efdba..641d64e 100644
|
||||||
|
--- a/virtcontainers/persist/fs/fs.go
|
||||||
|
+++ b/virtcontainers/persist/fs/fs.go
|
||||||
|
@@ -14,6 +14,8 @@ import (
|
||||||
|
"path/filepath"
|
||||||
|
"syscall"
|
||||||
|
|
||||||
|
+ "github.com/opencontainers/runc/libcontainer/configs"
|
||||||
|
+
|
||||||
|
persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
)
|
||||||
|
@@ -78,6 +80,12 @@ func (fs *FS) ToDisk(ss persistapi.SandboxState, cs map[string]persistapi.Contai
|
||||||
|
return fmt.Errorf("sandbox container id required")
|
||||||
|
}
|
||||||
|
|
||||||
|
+ if ss.Config.Cgroups == nil {
|
||||||
|
+ ss.Config.Cgroups = &configs.Cgroup{
|
||||||
|
+ Path: ss.CgroupPath,
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
fs.sandboxState = &ss
|
||||||
|
fs.containerState = cs
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
||||||
|
index 91067fb..e8ef41b 100644
|
||||||
|
--- a/virtcontainers/pkg/oci/utils.go
|
||||||
|
+++ b/virtcontainers/pkg/oci/utils.go
|
||||||
|
@@ -1136,3 +1136,17 @@ func validateSandboxDNS(value string) error {
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func GetSandboxIDFromAnnotations(s *specs.Spec) (string, error) {
|
||||||
|
+ if s == nil {
|
||||||
|
+ return "", fmt.Errorf("spec is nil")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ for _, v := range CRISandboxNameKeyList {
|
||||||
|
+ if sandboxID, ok := s.Annotations[v]; ok {
|
||||||
|
+ return sandboxID, nil
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return "", fmt.Errorf("failed to find the sandbox ID")
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index b479cf5..ca4e700 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -2162,7 +2162,9 @@ func (s *Sandbox) cgroupsDelete() error {
|
||||||
|
var cgroupSubsystems cgroups.Hierarchy
|
||||||
|
|
||||||
|
if s.config.SandboxCgroupWithEmulator {
|
||||||
|
- // emulator
|
||||||
|
+ if err := deleteCgroup(cgroups.V1, s.state.CgroupPath); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
} else if s.config.SandboxCgroupOnly {
|
||||||
|
return s.cgroupMgr.Destroy()
|
||||||
|
}
|
||||||
|
@@ -2381,6 +2383,68 @@ func (s *Sandbox) setupSandboxCgroup() error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (s *Sandbox) setupHostCgroupsWithEmulator() error {
|
||||||
|
+ if len(s.config.Containers) == 0 {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ sandboxContainerSpec := s.GetPatchedOCISpec()
|
||||||
|
+ if sandboxContainerSpec == nil {
|
||||||
|
+ return fmt.Errorf("sandbox container should not be empty")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // Set sandbox's cgroup path
|
||||||
|
+ s.state.CgroupPath = sandboxContainerSpec.Linux.CgroupsPath
|
||||||
|
+
|
||||||
|
+ if !checkCgroupExist(cgroups.V1, s.state.CgroupPath) {
|
||||||
|
+ return fmt.Errorf("sandbox's cgroup %s doesn't exist", s.state.CgroupPath)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // pull out qemu threads other than vcpu to the cgroup of "<path>/emulator"
|
||||||
|
+ if s.config.HypervisorType == QemuHypervisor {
|
||||||
|
+ emulatorCgroupPath := filepath.Join(s.state.CgroupPath, emulatorCgroupName)
|
||||||
|
+ hypervisorPids := s.hypervisor.getPids()
|
||||||
|
+ if len(hypervisorPids) == 0 || hypervisorPids[0] == 0 {
|
||||||
|
+ return fmt.Errorf("hypervisor pid: %v invalid", hypervisorPids)
|
||||||
|
+ }
|
||||||
|
+ if err := pulloutQemuThread(s, hypervisorPids[0], emulatorCgroupPath); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // limit cpu to "<path>/vcpu"
|
||||||
|
+ vcpuCgroupPath := filepath.Join(s.state.CgroupPath, vcpuCgroupName)
|
||||||
|
+ vcpuResources := specs.LinuxResources{
|
||||||
|
+ CPU: s.cpuResources(),
|
||||||
|
+ }
|
||||||
|
+ if err := applyResourceLimit(&vcpuResources, vcpuCgroupPath); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // limit blkio resource to "<path>"
|
||||||
|
+
|
||||||
|
+ // limit files resource
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func applyResourceLimit(resources *specs.LinuxResources, cgroupPath string) error {
|
||||||
|
+ if resources == nil {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ control, err := cgroupsLoadFunc(cgroups.V1, cgroups.StaticPath(cgroupPath))
|
||||||
|
+ if err != nil {
|
||||||
|
+ return fmt.Errorf("could not load cgroup %v: %v", cgroupPath, err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err = control.Update(resources); err != nil {
|
||||||
|
+ return fmt.Errorf("could not update cgroup %v: %v", cgroupPath, err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// GetPatchedOCISpec returns sandbox's OCI specification
|
||||||
|
// This OCI specification was patched when the sandbox was created
|
||||||
|
// by containerCapabilities(), SetEphemeralStorageType() and others
|
||||||
|
@@ -2452,6 +2516,10 @@ func (s *Sandbox) forceDeleteSandbox() {
|
||||||
|
c.forceDeleteContainer()
|
||||||
|
}
|
||||||
|
|
||||||
|
+ if err := deleteCgroup(cgroups.V1, s.state.CgroupPath); err != nil {
|
||||||
|
+ s.Logger().Warnf("sandbox forceDelete cgroups failed: %v", err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
globalSandboxList.removeSandbox(s.id)
|
||||||
|
|
||||||
|
if s.monitor != nil {
|
||||||
|
--
|
||||||
|
1.8.3.1
|
||||||
|
|
||||||
@ -0,0 +1,318 @@
|
|||||||
|
From f4b899b933a3a30fc378ceb4d8855778cd783d9a Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Fri, 18 Sep 2020 16:54:11 +0800
|
||||||
|
Subject: [PATCH 4/5] kata_runtime: support the blkio in host cgroups
|
||||||
|
|
||||||
|
reason: support the blkio in host cgroups, run with paras
|
||||||
|
--annotation io.kubernetes.docker.type=podsandbox --annotation io.katacontainers.blkio_cgroup=
|
||||||
|
'{"blkiocgroup":[{"path":"/dev/sda","limits":[{"type":"throttle_read_bps","value":400}]}]}'
|
||||||
|
|
||||||
|
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||||
|
---
|
||||||
|
vendor/github.com/containerd/cgroups/blkio.go | 6 ++
|
||||||
|
virtcontainers/cgroups.go | 102 ++++++++++++++++++++++++++
|
||||||
|
virtcontainers/pkg/annotations/annotations.go | 3 +
|
||||||
|
virtcontainers/pkg/oci/utils.go | 50 ++++++++++++-
|
||||||
|
virtcontainers/sandbox.go | 6 ++
|
||||||
|
virtcontainers/utils/utils.go | 22 ++++++
|
||||||
|
6 files changed, 186 insertions(+), 3 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/vendor/github.com/containerd/cgroups/blkio.go b/vendor/github.com/containerd/cgroups/blkio.go
|
||||||
|
index 7c498de..485ff7b 100644
|
||||||
|
--- a/vendor/github.com/containerd/cgroups/blkio.go
|
||||||
|
+++ b/vendor/github.com/containerd/cgroups/blkio.go
|
||||||
|
@@ -23,6 +23,7 @@ import (
|
||||||
|
"io/ioutil"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
+ "reflect"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
@@ -55,6 +56,11 @@ func (b *blkioController) Create(path string, resources *specs.LinuxResources) e
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, t := range createBlkioSettings(resources.BlockIO) {
|
||||||
|
+ ptr := reflect.ValueOf(t.value)
|
||||||
|
+ if ptr.Kind() == reflect.Ptr && ptr.IsNil() {
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
if t.value != nil {
|
||||||
|
if err := ioutil.WriteFile(
|
||||||
|
filepath.Join(b.Path(path), fmt.Sprintf("blkio.%s", t.name)),
|
||||||
|
diff --git a/virtcontainers/cgroups.go b/virtcontainers/cgroups.go
|
||||||
|
index 65d2001..e8c5a7b 100644
|
||||||
|
--- a/virtcontainers/cgroups.go
|
||||||
|
+++ b/virtcontainers/cgroups.go
|
||||||
|
@@ -9,6 +9,7 @@ package virtcontainers
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"context"
|
||||||
|
+ "encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
@@ -16,6 +17,8 @@ import (
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/containerd/cgroups"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
|
||||||
|
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||||
|
specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
)
|
||||||
|
@@ -32,8 +35,46 @@ const (
|
||||||
|
cgroupKataPath = "/kata/"
|
||||||
|
vcpuCgroupName = "vcpu"
|
||||||
|
emulatorCgroupName = "emulator"
|
||||||
|
+
|
||||||
|
+ // BlkioThrottleReadBps is the key to fetch throttle_read_bps
|
||||||
|
+ BlkioThrottleReadBps = "throttle_read_bps"
|
||||||
|
+
|
||||||
|
+ // BlkioThrottleWriteBps is the key to fetch throttle_write_bps
|
||||||
|
+ BlkioThrottleWriteBps = "throttle_write_bps"
|
||||||
|
+
|
||||||
|
+ // BlkioThrottleReadIOPS is the key to fetch throttle_read_iops
|
||||||
|
+ BlkioThrottleReadIOPS = "throttle_read_iops"
|
||||||
|
+
|
||||||
|
+ // BlkioThrottleWriteIOPS is the key to fetch throttle_write_iops
|
||||||
|
+ BlkioThrottleWriteIOPS = "throttle_write_iops"
|
||||||
|
+
|
||||||
|
+ // BlkioWeight is the key to fetch blkio_weight
|
||||||
|
+ BlkioWeight = "blkio_weight"
|
||||||
|
+
|
||||||
|
+ // BlkioLeafWeight is the key to fetch blkio_leaf_weight
|
||||||
|
+ BlkioLeafWeight = "blkio_leaf_weight"
|
||||||
|
)
|
||||||
|
|
||||||
|
+// BlkioCgroup for Linux cgroup 'blkio' data exchange
|
||||||
|
+type BlkioCgroup struct {
|
||||||
|
+ // Items specifies per cgroup values
|
||||||
|
+ Items []BlockIOCgroupItem `json:"blkiocgroup,omitempty"`
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+type BlockIOCgroupItem struct {
|
||||||
|
+ // Path represent path of blkio device
|
||||||
|
+ Path string `json:"path,omitempty"`
|
||||||
|
+ // Limits specifies the blkio type and value
|
||||||
|
+ Limits []IOLimit `json:"limits,omitempty"`
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+type IOLimit struct {
|
||||||
|
+ // Type specifies IO type
|
||||||
|
+ Type string `json:"type,omitempty"`
|
||||||
|
+ // Value specifies rate or weight value
|
||||||
|
+ Value uint64 `json:"value,omitempty"`
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
var cgroupsLoadFunc = cgroups.Load
|
||||||
|
var cgroupsNewFunc = cgroups.New
|
||||||
|
|
||||||
|
@@ -372,3 +413,64 @@ func isInSlice(i int, s []int) bool {
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func (s *Sandbox) blockIOResource() *specs.LinuxBlockIO {
|
||||||
|
+ value, ok := s.config.Annotations[annotations.BlkioCgroupTypeKey]
|
||||||
|
+ if !ok {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ var blkioCgroupParse BlkioCgroup
|
||||||
|
+ var linuxBlkio specs.LinuxBlockIO
|
||||||
|
+ if err := json.Unmarshal([]byte(value), &blkioCgroupParse); err != nil {
|
||||||
|
+ s.Logger().Errorf("blkio_cgroup Unmarshal error:%v", err)
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ for _, item := range blkioCgroupParse.Items {
|
||||||
|
+ if item.Limits == nil {
|
||||||
|
+ s.Logger().Errorf("%v:limits have none data", item)
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+ for _, limit := range item.Limits {
|
||||||
|
+ if item.Path != "" {
|
||||||
|
+ major, minor, err := utils.GetDeviceByPath(item.Path)
|
||||||
|
+ if err != nil {
|
||||||
|
+ s.Logger().Errorf("failed to find major and minor of device %s: %v", item.Path, err)
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+ td := specs.LinuxThrottleDevice{
|
||||||
|
+ Rate: limit.Value,
|
||||||
|
+ }
|
||||||
|
+ td.Major = major
|
||||||
|
+ td.Minor = minor
|
||||||
|
+ switch limit.Type {
|
||||||
|
+ case BlkioThrottleReadBps:
|
||||||
|
+ linuxBlkio.ThrottleReadBpsDevice = append(linuxBlkio.ThrottleReadBpsDevice, td)
|
||||||
|
+ case BlkioThrottleWriteBps:
|
||||||
|
+ linuxBlkio.ThrottleWriteBpsDevice = append(linuxBlkio.ThrottleWriteBpsDevice, td)
|
||||||
|
+ case BlkioThrottleReadIOPS:
|
||||||
|
+ linuxBlkio.ThrottleReadIOPSDevice = append(linuxBlkio.ThrottleReadIOPSDevice, td)
|
||||||
|
+ case BlkioThrottleWriteIOPS:
|
||||||
|
+ linuxBlkio.ThrottleWriteIOPSDevice = append(linuxBlkio.ThrottleWriteIOPSDevice, td)
|
||||||
|
+ default:
|
||||||
|
+ s.Logger().Errorf("the type of throtlle device：%s is not surpport", limit.Type)
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+ } else {
|
||||||
|
+ switch limit.Type {
|
||||||
|
+ case BlkioWeight:
|
||||||
|
+ weightUint16 := uint16(limit.Value)
|
||||||
|
+ linuxBlkio.Weight = &weightUint16
|
||||||
|
+ case BlkioLeafWeight:
|
||||||
|
+ weightLeafUint16 := uint16(limit.Value)
|
||||||
|
+ linuxBlkio.LeafWeight = &weightLeafUint16
|
||||||
|
+ default:
|
||||||
|
+ s.Logger().Errorf("the type:%s is not one of the supported types of blkio_weight and blkio_leaf_weight", limit.Type)
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ return &linuxBlkio
|
||||||
|
+}
|
||||||
|
diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go
|
||||||
|
index 96c4ef2..c35993e 100644
|
||||||
|
--- a/virtcontainers/pkg/annotations/annotations.go
|
||||||
|
+++ b/virtcontainers/pkg/annotations/annotations.go
|
||||||
|
@@ -21,6 +21,9 @@ const (
|
||||||
|
ContainerTypeKey = kataAnnotationsPrefix + "pkg.oci.container_type"
|
||||||
|
|
||||||
|
SandboxConfigPathKey = kataAnnotationsPrefix + "config_path"
|
||||||
|
+
|
||||||
|
+ // BlkioCgroupTypeKey is the annotation key to fetch sandbox blkio cgroup values
|
||||||
|
+ BlkioCgroupTypeKey = kataAnnotationsPrefix + "blkio_cgroup"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Annotations related to Hypervisor configuration
|
||||||
|
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
||||||
|
index e8ef41b..643753b 100644
|
||||||
|
--- a/virtcontainers/pkg/oci/utils.go
|
||||||
|
+++ b/virtcontainers/pkg/oci/utils.go
|
||||||
|
@@ -38,9 +38,10 @@ type annotationContainerType struct {
|
||||||
|
type annotationHandler func(value string) error
|
||||||
|
|
||||||
|
var annotationHandlerList = map[string]annotationHandler{
|
||||||
|
- vcAnnotations.StaticCPUTypeKey: validateSandboxCPU,
|
||||||
|
- vcAnnotations.StaticMemTypeKey: validateSandboxMem,
|
||||||
|
- vcAnnotations.SandboxDNSTypeKey: validateSandboxDNS,
|
||||||
|
+ vcAnnotations.StaticCPUTypeKey: validateSandboxCPU,
|
||||||
|
+ vcAnnotations.StaticMemTypeKey: validateSandboxMem,
|
||||||
|
+ vcAnnotations.SandboxDNSTypeKey: validateSandboxDNS,
|
||||||
|
+ vcAnnotations.BlkioCgroupTypeKey: validateBlkioCgroup,
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
@@ -1076,6 +1077,7 @@ func validateOtherSandboxAnnotations(annotation, value string) error {
|
||||||
|
// addOtherSandboxAnnotation add self defined annotation for sandbox
|
||||||
|
func addOtherSandboxAnnotation(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error {
|
||||||
|
otherSandboxAnnotationsKey := []string{
|
||||||
|
+ vcAnnotations.BlkioCgroupTypeKey,
|
||||||
|
vcAnnotations.StaticCPUTypeKey,
|
||||||
|
vcAnnotations.StaticMemTypeKey,
|
||||||
|
}
|
||||||
|
@@ -1137,6 +1139,48 @@ func validateSandboxDNS(value string) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+func validateBlkioCgroup(value string) error {
|
||||||
|
+ var linuxBlkioCgroup vc.BlkioCgroup
|
||||||
|
+ if err := json.Unmarshal([]byte(value), &linuxBlkioCgroup); err != nil {
|
||||||
|
+ return fmt.Errorf("blkio_cgroup Unmarshal error:%v, value is %s", err, value)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if linuxBlkioCgroup.Items == nil {
|
||||||
|
+ return fmt.Errorf("BlkioCgroup:%v fetch none data", linuxBlkioCgroup)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // pathNull used to judge path for blkio.weight/blkio.leaf_weight only once
|
||||||
|
+ pathNull := false
|
||||||
|
+ for _, item := range linuxBlkioCgroup.Items {
|
||||||
|
+ if pathNull {
|
||||||
|
+ return fmt.Errorf("too many paths for blkio.weight/blkio.leaf_weight")
|
||||||
|
+ }
|
||||||
|
+ if item.Path == "" && !pathNull {
|
||||||
|
+ pathNull = true
|
||||||
|
+ if len(item.Limits) > 2 || len(item.Limits) == 0 {
|
||||||
|
+ return fmt.Errorf("the format of values for blkio.weight or blkio.leaf_weight is wrong")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ for _, ioLimit := range item.Limits {
|
||||||
|
+ switch ioLimit.Type {
|
||||||
|
+ case vc.BlkioWeight:
|
||||||
|
+ // Blkio.weight, between 10 and 1000
|
||||||
|
+ if ioLimit.Value < 10 || ioLimit.Value > 1000 {
|
||||||
|
+ return fmt.Errorf("blkio.weight:%v must be between 10 and 1000", ioLimit.Value)
|
||||||
|
+ }
|
||||||
|
+ case vc.BlkioLeafWeight:
|
||||||
|
+ return fmt.Errorf("the blkio.leaf_weight is not supported now")
|
||||||
|
+ default:
|
||||||
|
+ return fmt.Errorf("the type of blkio device：%s is not surpport", ioLimit.Type)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ } else if _, _, err := utils.GetDeviceByPath(item.Path); err != nil {
|
||||||
|
+ return fmt.Errorf("failed to find major and minor of device %s: %v", item.Path, err)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func GetSandboxIDFromAnnotations(s *specs.Spec) (string, error) {
|
||||||
|
if s == nil {
|
||||||
|
return "", fmt.Errorf("spec is nil")
|
||||||
|
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||||
|
index ca4e700..9284f99 100644
|
||||||
|
--- a/virtcontainers/sandbox.go
|
||||||
|
+++ b/virtcontainers/sandbox.go
|
||||||
|
@@ -2422,6 +2422,12 @@ func (s *Sandbox) setupHostCgroupsWithEmulator() error {
|
||||||
|
}
|
||||||
|
|
||||||
|
// limit blkio resource to "<path>"
|
||||||
|
+ blkioResources := specs.LinuxResources{
|
||||||
|
+ BlockIO: s.blockIOResource(),
|
||||||
|
+ }
|
||||||
|
+ if err := applyResourceLimit(&blkioResources, vcpuCgroupPath); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
|
||||||
|
// limit files resource
|
||||||
|
|
||||||
|
diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go
|
||||||
|
index 36ac67a..d4dad40 100644
|
||||||
|
--- a/virtcontainers/utils/utils.go
|
||||||
|
+++ b/virtcontainers/utils/utils.go
|
||||||
|
@@ -370,6 +370,28 @@ func GetPhysicalCPUNumber() int {
|
||||||
|
return cpuNum
|
||||||
|
}
|
||||||
|
|
||||||
|
+// GetDeviceByPath returns the device No.
|
||||||
|
+func GetDeviceByPath(path string) (int64, int64, error) {
|
||||||
|
+ if path == "" {
|
||||||
|
+ return -1, -1, fmt.Errorf("Path cannot be empty")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ path, err := filepath.EvalSymlinks(path)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return -1, -1, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ stat := syscall.Stat_t{}
|
||||||
|
+ err = syscall.Stat(path, &stat)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return -1, -1, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ major := int64(stat.Rdev / 256)
|
||||||
|
+ minor := int64(stat.Rdev % 256)
|
||||||
|
+ return major, minor, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func RoundVCPUNumber(value string) (int, error) {
|
||||||
|
cpuNum, err := strconv.ParseFloat(value, 64)
|
||||||
|
if err != nil || cpuNum < minCPUs {
|
||||||
|
--
|
||||||
|
1.8.3.1
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user