packages/kata-containers
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

!1 move all kata-containers related source repo into the only one repo

Browse Source 
From: @flyflyflypeng
Reviewed-by: 
Signed-off-by:
This commit is contained in:
openeuler-ci-bot 2021-01-05 19:18:06 +08:00 committed by Gitee
commit f06ff0ea4e
98 changed files with 31394 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
agent/agent-1.11.1.tar.gz Normal file
View File

Binary file not shown.

20
agent/apply-patches Executable file
View File

@ -0,0 +1,20 @@
#!/bin/bash
if [[ -f ./patch_flag ]];then
echo "agent patched!"
exit 0
fi
tar -zxvf agent-1.11.1.tar.gz
cp -fr ./agent-1.11.1/* ./
rm -rf ./agent-1.11.1
cat ./series.conf | while read line
do
if [[ $line == '' || $line =~ ^\s*# ]]; then
continue
fi
echo "====patch $line======"
pwd
patch -p1 -F1 -s < ./patches/$line
done
touch ./patch_flag

234
agent/patches/0001-agent-add-agent.netlink_recv_buf_size-flag-to-set-ne.patch Normal file
View File

@ -0,0 +1,234 @@
From ac1d7806f8de2f8ca393df08a9c62d1045c4afdc Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Tue, 11 Dec 2018 18:27:02 -0500
Subject: [PATCH 01/16] agent: add agent.netlink_recv_buf_size flag to set
netlink recv buf size
fixes: #813
reason: If hotplug huge size memory(for example 128GB) into guest,
kernel will produce a lot of memory add uevents and send to netlink socket,
however netlink socket default receive buffer size is 4KB, which is too small
to receive all memory add uevents.
Since hotplug huge size memory is not common case, so we consider add an agent
flag agent.netlink_recv_buf_size to set netlink socket recv buffer size.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
README.md | 13 +++++++++++++
agent.go | 10 +++++++++-
config.go | 15 ++++++++++++++
config_test.go | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++
pkg/uevent/uevent.go | 15 +++++++++++---
5 files changed, 104 insertions(+), 4 deletions(-)
diff --git a/README.md b/README.md
index cec65a4..16f96a4 100644
--- a/README.md
+++ b/README.md
@@ -98,6 +98,19 @@ The pipe's capacity for stdout/stderr can be modified by specifying the `agent.c
to the guest kernel command line. For example, `agent.container_pipe_size=2097152` will set the stdout and stderr
pipes to 2097152 bytes.
+## Uevent Netlink Socket Receive Buffer Size
+
+When hotplugging a huge size memory into the Kata VM, the kernel in the VM will produce a lot of memory object add
+uevents and send all these uevents to Kata agent by netlink socket. However, default netlink socket receive buffer
+size is 4KB, which is too small and can only hold 256 memory add uevents. If memory add uevents number is larger
+than 256, the left uevents can not be received and processed by Kata agent.
+
+The uevent netlink socket receive buffer size can be modified by specifying the `agent.netlink_recv_buf_size` flag
+to the guest kernel command line. For example, `agent.netlink_recv_buf_size=2MB` will set the uevent netlink socket
+receive buffer size to 2MB value. `agent.netlink_recv_buf_size` valid value range is `[4KB ~ 4MB]` and value can be
+set in human-readable memory format or pure digital number format(default memory unit is byte).
+
+
[1]: https://github.com/firecracker-microvm/firecracker/blob/master/docs/vsock.md
[2]: https://golang.org/pkg/time/#ParseDuration
[3]: http://man7.org/linux/man-pages/man7/pipe.7.html
diff --git a/agent.go b/agent.go
index 2d2c293..c1cac08 100644
--- a/agent.go
+++ b/agent.go
@@ -190,6 +190,14 @@ var unifiedCgroupHierarchy = false
// Size in bytes of the stdout/stderr pipes created for each container.
var containerPipeSize = uint32(0)
+const (
+ minNetlinkSockRecvBufSize = 4 * 1024
+ maxNetlinkSockRecvBufSize = 4 * 1024 * 1024
+)
+
+// Size in bytes of the netlink socket recv buf size
+var netlinkSockRecvBufSize = uint32(0)
+
// commType is used to denote the communication channel type used.
type commType int
@@ -708,7 +716,7 @@ func (s *sandbox) waitForStopServer() {
func (s *sandbox) listenToUdevEvents() {
fieldLogger := agentLog.WithField("subsystem", "udevlistener")
- uEvHandler, err := uevent.NewHandler()
+ uEvHandler, err := uevent.NewHandler(netlinkSockRecvBufSize)
if err != nil {
fieldLogger.Warnf("Error starting uevent listening loop %s", err)
return
diff --git a/config.go b/config.go
index 4530096..6c7d473 100644
--- a/config.go
+++ b/config.go
@@ -7,11 +7,13 @@
package main
import (
+ "fmt"
"io/ioutil"
"strconv"
"strings"
"time"
+ "github.com/docker/go-units"
"github.com/sirupsen/logrus"
"google.golang.org/grpc/codes"
grpcStatus "google.golang.org/grpc/status"
@@ -29,6 +31,7 @@ const (
hotplugTimeoutFlag = optionPrefix + "hotplug_timeout"
unifiedCgroupHierarchyFlag = optionPrefix + "unified_cgroup_hierarchy"
containerPipeSizeFlag = optionPrefix + "container_pipe_size"
+ netlinkSockRecvBufSizeFlag = optionPrefix + "netlink_recv_buf_size"
traceModeStatic = "static"
traceModeDynamic = "dynamic"
traceTypeIsolated = "isolated"
@@ -155,6 +158,18 @@ func parseCmdlineOption(option string) error {
return err
}
unifiedCgroupHierarchy = flag
+ case netlinkSockRecvBufSizeFlag:
+ bufSizeInBytes, err := units.RAMInBytes(split[valuePosition])
+ if err != nil {
+ return err
+ }
+
+ if bufSizeInBytes < minNetlinkSockRecvBufSize || bufSizeInBytes > maxNetlinkSockRecvBufSize {
+ return fmt.Errorf("invalid netlink socket recv buf size: %d (valid size range %s-%s bytes)", bufSizeInBytes,
+ units.BytesSize(minNetlinkSockRecvBufSize), units.BytesSize(maxNetlinkSockRecvBufSize))
+ }
+
+ netlinkSockRecvBufSize = uint32(bufSizeInBytes)
default:
if strings.HasPrefix(split[optionPosition], optionPrefix) {
return grpcStatus.Errorf(codes.NotFound, "Unknown option %s", split[optionPosition])
diff --git a/config_test.go b/config_test.go
index 2a23133..f40f17a 100644
--- a/config_test.go
+++ b/config_test.go
@@ -486,3 +486,58 @@ func TestParseCmdlineOptionContainerPipeSize(t *testing.T) {
assert.Equal(d.expectedContainerPipeSize, containerPipeSize, "test %d (%+v)", i, d)
}
}
+
+func TestParseCmdlineOptionNetlinkSockRecvBufSize(t *testing.T) {
+ assert := assert.New(t)
+
+ type testData struct {
+ option string
+ shouldErr bool
+ expectedNetlinkSockRecvBufSize uint32
+ }
+
+ data := []testData{
+ {"", false, 0},
+ {"netlink_recv_buf_siz", false, 0},
+ {"netlink_recv_buf_size", false, 0},
+ {"netlink_recv_buf_size=", false, 0},
+ {"netlink_recv_buf_size=4096", false, 0},
+ {"netlink_recv_buf_size=4KB", false, 0},
+ {"agent.netlink_recv_buf_size=", true, 0},
+ {"agent.netlink_recv_buf_size=foobar", true, 0},
+ {"agent.netlink_recv_buf_size=-1", true, 0},
+ {"agent.netlink_recv_buf_size=0", true, 0},
+ {"agent.netlink_recv_buf_size=100", true, 0},
+ {"agent.netlink_recv_buf_size=3KB", true, 0},
+ {"agent.netlink_recv_buf_size=3.6KB", true, 0},
+ {"agent.netlink_recv_buf_size=4095", true, 0},
+ {"agent.netlink_recv_buf_size=4096xB", true, 0},
+ {"agent.netlink_recv_buf_size=4096", false, 4096},
+ {"agent.netlink_recv_buf_size=4097", false, 4097},
+ {"agent.netlink_recv_buf_size=4096.0", false, 4096},
+ {"agent.netlink_recv_buf_size=1024KB", false, 1048576},
+ {"agent.netlink_recv_buf_size=1MB", false, 1048576},
+ {"agent.netlink_recv_buf_size=4194303", false, 4194303},
+ {"agent.netlink_recv_buf_size=3.999MB", false, 4193255},
+ {"agent.netlink_recv_buf_size=4194304", false, 4194304},
+ {"agent.netlink_recv_buf_size=4MB", false, 4194304},
+ {"agent.netlink_recv_buf_size=4.001MB", true, 0},
+ {"agent.netlink_recv_buf_size=4194305", true, 0},
+ {"agent.netlink_recv_buf_size=100MB", true, 0},
+ {"agent.netlink_recv_buf_size=1GB", true, 0},
+ }
+
+ for i, d := range data {
+ // reset the netlink socket recv buffer size
+ netlinkSockRecvBufSize = 0
+
+ err := parseCmdlineOption(d.option)
+ if d.shouldErr {
+ assert.Error(err)
+ } else {
+ assert.NoError(err)
+ }
+
+ assert.Equal(d.expectedNetlinkSockRecvBufSize, netlinkSockRecvBufSize, "test %d (%+v)", i, d)
+ }
+}
diff --git a/pkg/uevent/uevent.go b/pkg/uevent/uevent.go
index fc2c127..fa84086 100644
--- a/pkg/uevent/uevent.go
+++ b/pkg/uevent/uevent.go
@@ -10,6 +10,7 @@ import (
"bufio"
"io"
"strings"
+ "syscall"
"golang.org/x/sys/unix"
"google.golang.org/grpc/codes"
@@ -33,7 +34,7 @@ type ReaderCloser struct {
}
// NewReaderCloser returns an io.ReadCloser handle for uevent.
-func NewReaderCloser() (io.ReadCloser, error) {
+func NewReaderCloser(netlinkRecvBufSize uint32) (io.ReadCloser, error) {
nl := unix.SockaddrNetlink{
Family: unix.AF_NETLINK,
// Passing Pid as 0 here allows the kernel to take care of assigning
@@ -47,6 +48,14 @@ func NewReaderCloser() (io.ReadCloser, error) {
return nil, err
}
+ // If netlinkRecvBufSize > 0, set netlink socket recv buffer size to netlinkRecvBufSize
+ if netlinkRecvBufSize > 0 {
+ err = unix.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_RCVBUFFORCE, int(netlinkRecvBufSize))
+ if err != nil {
+ return nil, err
+ }
+ }
+
if err := unix.Bind(fd, &nl); err != nil {
return nil, err
}
@@ -85,8 +94,8 @@ type Handler struct {
}
// NewHandler returns a uevent handler.
-func NewHandler() (*Handler, error) {
- rdCloser, err := NewReaderCloser()
+func NewHandler(netlinkRecvBufSize uint32) (*Handler, error) {
+ rdCloser, err := NewReaderCloser(netlinkRecvBufSize)
if err != nil {
return nil, err
}
--
2.14.3 (Apple Git-98)

686
agent/patches/0002-network-support-update-routes-incrementally.patch Normal file
View File

@ -0,0 +1,686 @@
From 13f54c768dcd7bf982dde8e57fb5cd624fedf5bc Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Mon, 17 Aug 2020 11:23:55 +0800
Subject: [PATCH 02/16] network: support update routes incrementally
reason: add increment flag in the UpdateRoutesRequest to
support upate routes incrementally to improve the efficiency.
kata-network add-route and del-route needs this feature.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
grpc.go | 2 +-
network.go | 74 ++++++++-
network_test.go | 16 +-
protocols/grpc/agent.pb.go | 402 +++++++++++++++++++++++++--------------------
protocols/grpc/agent.proto | 1 +
5 files changed, 300 insertions(+), 195 deletions(-)
diff --git a/grpc.go b/grpc.go
index 886661b..8fe8217 100644
--- a/grpc.go
+++ b/grpc.go
@@ -1556,7 +1556,7 @@ func (a *agentGRPC) UpdateInterface(ctx context.Context, req *pb.UpdateInterface
}
func (a *agentGRPC) UpdateRoutes(ctx context.Context, req *pb.UpdateRoutesRequest) (*pb.Routes, error) {
- return a.sandbox.updateRoutes(nil, req.Routes)
+ return a.sandbox.updateRoutes(nil, req.Routes, req.Increment)
}
func (a *agentGRPC) ListInterfaces(ctx context.Context, req *pb.ListInterfacesRequest) (*pb.Interfaces, error) {
diff --git a/network.go b/network.go
index 64a16a9..02e28cb 100644
--- a/network.go
+++ b/network.go
@@ -398,7 +398,7 @@ func (s *sandbox) deleteRoutes(netHandle *netlink.Handle) error {
// state which matches the requested routes. In doing this, preesxisting non-loopback routes will be
// removed from the network. If an error occurs, this function returns the list of routes in
// gRPC-route format at the time of failure
-func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Routes) (resultingRoutes *pb.Routes, err error) {
+func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Routes, increment bool) (resultingRoutes *pb.Routes, err error) {
if requestedRoutes == nil {
return nil, errNoRoutes
}
@@ -418,13 +418,66 @@ func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Ro
}
}()
+ var (
+ added []*types.Route
+ removed []*types.Route
+ )
+
+ defer func(netHandle *netlink.Handle) {
+ if err != nil {
+ // if error happens after route added, need to rollback the added route
+ if len(added) > 0 {
+ for _, r := range added {
+ errRb := s.updateRoute(netHandle, r, false)
+ if errRb != nil {
+ agentLog.WithError(err).Error("rollback route failed")
+ }
+ }
+ }
+
+ // if error happens after route removed, need to rollback the removed route
+ if len(removed) > 0 {
+ for _, r := range removed {
+ errRb := s.updateRoute(netHandle, r, true)
+ if errRb != nil {
+ agentLog.WithError(err).Error("rollback route failed")
+ }
+ }
+ }
+ }
+ }(netHandle)
+
+ // updateOneRoute just update the specified one route
+ updateOneRoute := func(netHandle *netlink.Handle, reqRoute *types.Route) error {
+ var add bool = true
+ if reqRoute.Dest != "" && strings.HasPrefix(reqRoute.Dest, "-") {
+ reqRoute.Dest = reqRoute.Dest[1:]
+ add = false
+ }
+ err = s.updateRoute(netHandle, reqRoute, add)
+ if err != nil {
+ agentLog.WithError(err).Error("update Route failed")
+ // If there was an error setting the route, return the error
+ // and the current routes on the system via the defer func
+ return err
+ }
+ if add {
+ added = append([]*types.Route{reqRoute}, added[:]...)
+ } else {
+ removed = append([]*types.Route{reqRoute}, removed[:]...)
+ }
+ return nil
+ }
+
//
// First things first, let's blow away all the existing routes. The updateRoutes function
// is designed to be declarative, so we will attempt to create state matching what is
// requested, and in the event that we fail to do so, will return the error and final state.
//
- if err = s.deleteRoutes(netHandle); err != nil {
- return nil, err
+ if !increment {
+ if err = s.deleteRoutes(netHandle); err != nil {
+ return nil, err
+ }
}
//
@@ -434,7 +487,12 @@ func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Ro
// won't be able to access the gateway
for _, reqRoute := range requestedRoutes.Routes {
if reqRoute.Gateway == "" {
- err = s.updateRoute(netHandle, reqRoute, true)
+ if increment {
+ err = updateOneRoute(netHandle, reqRoute)
+ } else {
+ err = s.updateRoute(netHandle, reqRoute, true)
+ }
+
if err != nil {
agentLog.WithError(err).Error("update Route failed")
//If there was an error setting the route, return the error
@@ -447,7 +505,11 @@ func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Ro
// Take a second pass and apply the routes which include a gateway
for _, reqRoute := range requestedRoutes.Routes {
if reqRoute.Gateway != "" {
- err = s.updateRoute(netHandle, reqRoute, true)
+ if increment {
+ err = updateOneRoute(netHandle, reqRoute)
+ } else {
+ err = s.updateRoute(netHandle, reqRoute, true)
+ }
if err != nil {
agentLog.WithError(err).Error("update Route failed")
//If there was an error setting the route, return the
@@ -699,4 +761,4 @@ func (s *sandbox) handleLocalhost() error {
}
return netlink.LinkSetUp(lo)
-}
+}
\ No newline at end of file
diff --git a/network_test.go b/network_test.go
index a143670..a1e58f5 100644
--- a/network_test.go
+++ b/network_test.go
@@ -160,7 +160,7 @@ func TestUpdateRoutes(t *testing.T) {
Routes: inputRoutesSimple,
}
- results, err := s.updateRoutes(netHandle, testRoutes)
+ results, err := s.updateRoutes(netHandle, testRoutes, false)
assert.Nil(t, err, "Unexpected update interface failure: %v", err)
assert.True(t, reflect.DeepEqual(results, testRoutes),
"Interface created didn't match: got %+v, expecting %+v", results, testRoutes)
@@ -173,7 +173,7 @@ func TestUpdateRoutes(t *testing.T) {
}
testRoutes.Routes = inputRoutesPTPExample
- results, err = s.updateRoutes(netHandle, testRoutes)
+ results, err = s.updateRoutes(netHandle, testRoutes, false)
assert.Nil(t, err, "Unexpected update interface failure: %v", err)
assert.True(t, reflect.DeepEqual(results, testRoutes),
"Interface created didn't match: got %+v, expecting %+v", results, testRoutes)
@@ -184,7 +184,7 @@ func TestUpdateRoutes(t *testing.T) {
{Dest: "192.168.0.0/16", Gateway: "", Source: "192.168.0.2", Scope: 0, Device: "ifc-name"},
}
testRoutes.Routes = inputRoutesNoScope
- results, err = s.updateRoutes(netHandle, testRoutes)
+ results, err = s.updateRoutes(netHandle, testRoutes, false)
assert.NotNil(t, err, "Expected to observe unreachable route failure")
assert.True(t, reflect.DeepEqual(results.Routes[0], testRoutes.Routes[1]),
@@ -231,7 +231,7 @@ func TestUpdateRoutesIPVlan(t *testing.T) {
}
testRoutes.Routes = inputRoutesIPVlanExample
- results, err := s.updateRoutes(netHandle, testRoutes)
+ results, err := s.updateRoutes(netHandle, testRoutes, false)
assert.Nil(t, err, "Unexpected update interface failure: %v", err)
assert.True(t, reflect.DeepEqual(results, testRoutes),
"Interface created didn't match: got %+v, expecting %+v", results, testRoutes)
@@ -357,7 +357,7 @@ func TestListRoutes(t *testing.T) {
Routes: inputRoutesSimple,
}
- _, err := s.updateRoutes(netHandle, testRoutes)
+ _, err := s.updateRoutes(netHandle, testRoutes, false)
assert.Nil(err)
results, err := s.listRoutes(nil)
assert.Nil(err, "Expected to list all routes")
@@ -377,7 +377,7 @@ func TestListRoutes(t *testing.T) {
Routes: inputRoutesSimple,
}
- _, err = s.updateRoutes(netHandle, testRoutes)
+ _, err = s.updateRoutes(netHandle, testRoutes, false)
assert.Nil(err)
results, err = s.listRoutes(nil)
assert.Nil(err, "Expected to list all routes")
@@ -438,7 +438,7 @@ func TestListRoutesWithIPV6(t *testing.T) {
Routes: inputRoutesSimple,
}
- _, err := s.updateRoutes(netHandle, testRoutes)
+ _, err := s.updateRoutes(netHandle, testRoutes, false)
assert.Nil(err)
results, err := s.listRoutes(nil)
assert.Nil(err, "Expected to list all routes")
@@ -514,7 +514,7 @@ func TestListRoutesWithTwoInterfacesSameSubnet(t *testing.T) {
Routes: inputRoutesSimple,
}
- _, err := s.updateRoutes(netHandle, testRoutes)
+ _, err := s.updateRoutes(netHandle, testRoutes, false)
assert.Nil(err)
results, err := s.listRoutes(nil)
assert.Nil(err, "Expected to list all routes")
diff --git a/protocols/grpc/agent.pb.go b/protocols/grpc/agent.pb.go
index 77e6d1b..1b887e5 100644
--- a/protocols/grpc/agent.pb.go
+++ b/protocols/grpc/agent.pb.go
@@ -1303,7 +1303,8 @@ func (m *UpdateInterfaceRequest) GetInterface() *types.Interface {
}
type UpdateRoutesRequest struct {
- Routes *Routes `protobuf:"bytes,1,opt,name=routes" json:"routes,omitempty"`
+ Routes *Routes `protobuf:"bytes,1,opt,name=routes" json:"routes,omitempty"`
+ Increment bool `protobuf:"varint,2,opt,name=increment,proto3" json:"increment,omitempty"`
}
func (m *UpdateRoutesRequest) Reset() { *m = UpdateRoutesRequest{} }
@@ -1318,6 +1319,13 @@ func (m *UpdateRoutesRequest) GetRoutes() *Routes {
return nil
}
+func (m *UpdateRoutesRequest) GetIncrement() bool {
+ if m != nil {
+ return m.Increment
+ }
+ return false
+}
+
type ListInterfacesRequest struct {
}
@@ -4476,6 +4484,16 @@ func (m *UpdateRoutesRequest) MarshalTo(dAtA []byte) (int, error) {
}
i += n20
}
+ if m.Increment {
+ dAtA[i] = 0x10
+ i++
+ if m.Increment {
+ dAtA[i] = 1
+ } else {
+ dAtA[i] = 0
+ }
+ i++
+ }
return i, nil
}
@@ -5751,6 +5769,9 @@ func (m *UpdateRoutesRequest) Size() (n int) {
l = m.Routes.Size()
n += 1 + l + sovAgent(uint64(l))
}
+ if m.Increment {
+ n += 2
+ }
return n
}
@@ -10964,6 +10985,26 @@ func (m *UpdateRoutesRequest) Unmarshal(dAtA []byte) error {
return err
}
iNdEx = postIndex
+ case 2:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field Increment", wireType)
+ }
+ var v int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowAgent
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ v |= (int(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ m.Increment = bool(v != 0)
default:
iNdEx = preIndex
skippy, err := skipAgent(dAtA[iNdEx:])
@@ -12852,184 +12893,185 @@ var (
func init() { proto.RegisterFile("agent.proto", fileDescriptorAgent) }
var fileDescriptorAgent = []byte{
- // 2862 bytes of a gzipped FileDescriptorProto
+ // 2876 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0x4b, 0x6f, 0x1c, 0xc7,
- 0xd1, 0xd8, 0x07, 0x97, 0xbb, 0xb5, 0x2f, 0x6e, 0x93, 0xa2, 0x56, 0x2b, 0x5b, 0x9f, 0x3c, 0xb6,
- 0x65, 0xfa, 0x73, 0xbc, 0xb4, 0x65, 0x23, 0x7e, 0xc1, 0x11, 0xc4, 0x47, 0x44, 0xc6, 0x56, 0xc4,
- 0x0c, 0x45, 0x38, 0x40, 0x10, 0x0c, 0x86, 0x33, 0xcd, 0x65, 0x9b, 0x3b, 0xd3, 0xe3, 0x9e, 0x1e,
- 0x8a, 0xeb, 0x00, 0x39, 0x26, 0xb7, 0x5c, 0x02, 0xe4, 0x96, 0x3f, 0x10, 0xe4, 0x96, 0x63, 0xae,
- 0x39, 0x18, 0x39, 0xe5, 0x17, 0x04, 0x81, 0x7f, 0x42, 0x7e, 0x41, 0xd0, 0xaf, 0x79, 0xec, 0x0e,
- 0x29, 0x84, 0x20, 0x90, 0xcb, 0xa2, 0xab, 0xba, 0xba, 0x5e, 0xdd, 0x55, 0x53, 0x55, 0x0b, 0x6d,
- 0x77, 0x82, 0x43, 0x3e, 0x8e, 0x18, 0xe5, 0x14, 0xd5, 0x27, 0x2c, 0xf2, 0x46, 0x2d, 0xea, 0x11,
- 0x85, 0x18, 0xfd, 0x70, 0x42, 0xf8, 0x69, 0x72, 0x3c, 0xf6, 0x68, 0xb0, 0x79, 0xe6, 0x72, 0xf7,
- 0x5d, 0x8f, 0x86, 0xdc, 0x25, 0x21, 0x66, 0xf1, 0xa6, 0x3c, 0xb8, 0x19, 0x9d, 0x4d, 0x36, 0xf9,
- 0x2c, 0xc2, 0xb1, 0xfa, 0xd5, 0xe7, 0xee, 0x4e, 0x28, 0x9d, 0x4c, 0xf1, 0xa6, 0x84, 0x8e, 0x93,
- 0x93, 0x4d, 0x1c, 0x44, 0x7c, 0xa6, 0x36, 0xad, 0x3f, 0x56, 0x61, 0x7d, 0x9b, 0x61, 0x97, 0xe3,
- 0x6d, 0xc3, 0xcd, 0xc6, 0xdf, 0x24, 0x38, 0xe6, 0xe8, 0x35, 0xe8, 0xa4, 0x12, 0x1c, 0xe2, 0x0f,
- 0x2b, 0xf7, 0x2b, 0x1b, 0x2d, 0xbb, 0x9d, 0xe2, 0xf6, 0x7d, 0x74, 0x1b, 0x96, 0xf1, 0x05, 0xf6,
- 0xc4, 0x6e, 0x55, 0xee, 0x36, 0x04, 0xb8, 0xef, 0xa3, 0xf7, 0xa1, 0x1d, 0x73, 0x46, 0xc2, 0x89,
- 0x93, 0xc4, 0x98, 0x0d, 0x6b, 0xf7, 0x2b, 0x1b, 0xed, 0x87, 0x2b, 0x63, 0x61, 0xd2, 0xf8, 0x50,
- 0x6e, 0x1c, 0xc5, 0x98, 0xd9, 0x10, 0xa7, 0x6b, 0xf4, 0x00, 0x96, 0x7d, 0x7c, 0x4e, 0x3c, 0x1c,
- 0x0f, 0xeb, 0xf7, 0x6b, 0x1b, 0xed, 0x87, 0x1d, 0x45, 0xbe, 0x23, 0x91, 0xb6, 0xd9, 0x44, 0x6f,
- 0x43, 0x33, 0xe6, 0x94, 0xb9, 0x13, 0x1c, 0x0f, 0x97, 0x24, 0x61, 0xd7, 0xf0, 0x95, 0x58, 0x3b,
- 0xdd, 0x46, 0xaf, 0x40, 0xed, 0xd9, 0xf6, 0xfe, 0xb0, 0x21, 0xa5, 0x83, 0xa6, 0x8a, 0xb0, 0x67,
- 0x0b, 0x34, 0x7a, 0x1d, 0xba, 0xb1, 0x1b, 0xfa, 0xc7, 0xf4, 0xc2, 0x89, 0x88, 0x1f, 0xc6, 0xc3,
- 0xe5, 0xfb, 0x95, 0x8d, 0xa6, 0xdd, 0xd1, 0xc8, 0x03, 0x81, 0xb3, 0x3e, 0x85, 0x5b, 0x87, 0xdc,
- 0x65, 0xfc, 0x1a, 0xde, 0xb1, 0x8e, 0x60, 0xdd, 0xc6, 0x01, 0x3d, 0xbf, 0x96, 0x6b, 0x87, 0xb0,
- 0xcc, 0x49, 0x80, 0x69, 0xc2, 0xa5, 0x6b, 0xbb, 0xb6, 0x01, 0xad, 0x3f, 0x57, 0x00, 0xed, 0x5e,
- 0x60, 0xef, 0x80, 0x51, 0x0f, 0xc7, 0xf1, 0xff, 0xe8, 0xba, 0xde, 0x82, 0xe5, 0x48, 0x29, 0x30,
- 0xac, 0x4b, 0x72, 0x7d, 0x0b, 0x46, 0x2b, 0xb3, 0x6b, 0x7d, 0x0d, 0x6b, 0x87, 0x64, 0x12, 0xba,
- 0xd3, 0x1b, 0xd4, 0x77, 0x1d, 0x1a, 0xb1, 0xe4, 0x29, 0x55, 0xed, 0xda, 0x1a, 0xb2, 0x0e, 0x00,
- 0x7d, 0xe5, 0x12, 0x7e, 0x73, 0x92, 0xac, 0x77, 0x61, 0xb5, 0xc0, 0x31, 0x8e, 0x68, 0x18, 0x63,
- 0xa9, 0x00, 0x77, 0x79, 0x12, 0x4b, 0x66, 0x4b, 0xb6, 0x86, 0x2c, 0x0c, 0x6b, 0x5f, 0x92, 0xd8,
- 0x90, 0xe3, 0xff, 0x46, 0x85, 0x75, 0x68, 0x9c, 0x50, 0x16, 0xb8, 0xdc, 0x68, 0xa0, 0x20, 0x84,
- 0xa0, 0xee, 0xb2, 0x49, 0x3c, 0xac, 0xdd, 0xaf, 0x6d, 0xb4, 0x6c, 0xb9, 0x16, 0xaf, 0x72, 0x4e,
- 0x8c, 0xd6, 0xeb, 0x35, 0xe8, 0x68, 0xbf, 0x3b, 0x53, 0x12, 0x73, 0x29, 0xa7, 0x63, 0xb7, 0x35,
- 0x4e, 0x9c, 0xb1, 0x28, 0xac, 0x1f, 0x45, 0xfe, 0x35, 0x03, 0xfe, 0x21, 0xb4, 0x18, 0x8e, 0x69,
- 0xc2, 0x44, 0x98, 0x56, 0xe5, 0xbd, 0xaf, 0xa9, 0x7b, 0xff, 0x92, 0x84, 0xc9, 0x85, 0x6d, 0xf6,
- 0xec, 0x8c, 0x4c, 0x87, 0x10, 0x8f, 0xaf, 0x13, 0x42, 0x9f, 0xc2, 0xad, 0x03, 0x37, 0x89, 0xaf,
- 0xa3, 0xab, 0xf5, 0x99, 0x08, 0xbf, 0x38, 0x09, 0xae, 0x75, 0xf8, 0x4f, 0x15, 0x68, 0x6e, 0x47,
- 0xc9, 0x51, 0xec, 0x4e, 0x30, 0xfa, 0x3f, 0x68, 0x73, 0xca, 0xdd, 0xa9, 0x93, 0x08, 0x50, 0x92,
- 0xd7, 0x6d, 0x90, 0x28, 0x45, 0x20, 0xdc, 0x8e, 0x99, 0x17, 0x25, 0x9a, 0xa2, 0x7a, 0xbf, 0xb6,
- 0x51, 0xb7, 0xdb, 0x0a, 0xa7, 0x48, 0xc6, 0xb0, 0x2a, 0xf7, 0x1c, 0x12, 0x3a, 0x67, 0x98, 0x85,
- 0x78, 0x1a, 0x50, 0x1f, 0xcb, 0xf7, 0x5b, 0xb7, 0x07, 0x72, 0x6b, 0x3f, 0xfc, 0x22, 0xdd, 0x40,
- 0xff, 0x0f, 0x83, 0x94, 0x5e, 0x04, 0xa5, 0xa4, 0xae, 0x4b, 0xea, 0xbe, 0xa6, 0x3e, 0xd2, 0x68,
- 0xeb, 0xd7, 0xd0, 0x7b, 0x7e, 0xca, 0x28, 0xe7, 0x53, 0x12, 0x4e, 0x76, 0x5c, 0xee, 0x8a, 0xec,
- 0x11, 0x61, 0x46, 0xa8, 0x1f, 0x6b, 0x6d, 0x0d, 0x88, 0xde, 0x81, 0x01, 0x57, 0xb4, 0xd8, 0x77,
- 0x0c, 0x4d, 0x55, 0xd2, 0xac, 0xa4, 0x1b, 0x07, 0x9a, 0xf8, 0x4d, 0xe8, 0x65, 0xc4, 0x22, 0xff,
- 0x68, 0x7d, 0xbb, 0x29, 0xf6, 0x39, 0x09, 0xb0, 0x75, 0x2e, 0x7d, 0x25, 0x2f, 0x19, 0xbd, 0x03,
- 0xad, 0xcc, 0x0f, 0x15, 0xf9, 0x42, 0x7a, 0xea, 0x85, 0x18, 0x77, 0xda, 0xcd, 0xd4, 0x29, 0x9f,
- 0x43, 0x9f, 0xa7, 0x8a, 0x3b, 0xbe, 0xcb, 0xdd, 0xe2, 0xa3, 0x2a, 0x5a, 0x65, 0xf7, 0x78, 0x01,
- 0xb6, 0x3e, 0x83, 0xd6, 0x01, 0xf1, 0x63, 0x25, 0x78, 0x08, 0xcb, 0x5e, 0xc2, 0x18, 0x0e, 0xb9,
- 0x31, 0x59, 0x83, 0x68, 0x0d, 0x96, 0xa6, 0x24, 0x20, 0x5c, 0x9b, 0xa9, 0x00, 0x8b, 0x02, 0x3c,
- 0xc5, 0x01, 0x65, 0x33, 0xe9, 0xb0, 0x35, 0x58, 0xca, 0x5f, 0xae, 0x02, 0xd0, 0x5d, 0x68, 0x05,
- 0xee, 0x45, 0x7a, 0xa9, 0x62, 0xa7, 0x19, 0xb8, 0x17, 0x4a, 0xf9, 0x21, 0x2c, 0x9f, 0xb8, 0x64,
- 0xea, 0x85, 0x5c, 0x7b, 0xc5, 0x80, 0x99, 0xc0, 0x7a, 0x5e, 0xe0, 0xdf, 0xaa, 0xd0, 0x56, 0x12,
- 0x95, 0xc2, 0x6b, 0xb0, 0xe4, 0xb9, 0xde, 0x69, 0x2a, 0x52, 0x02, 0xe8, 0x81, 0x51, 0xa4, 0x9a,
- 0x4f, 0xc2, 0x99, 0xa6, 0x46, 0xb5, 0x4d, 0x80, 0xf8, 0x85, 0x1b, 0x69, 0xdd, 0x6a, 0x97, 0x10,
- 0xb7, 0x04, 0x8d, 0x52, 0xf7, 0x03, 0xe8, 0xa8, 0x77, 0xa7, 0x8f, 0xd4, 0x2f, 0x39, 0xd2, 0x56,
- 0x54, 0xea, 0xd0, 0xeb, 0xd0, 0x4d, 0x62, 0xec, 0x9c, 0x12, 0xcc, 0x5c, 0xe6, 0x9d, 0xce, 0x86,
- 0x4b, 0xea, 0x1b, 0x99, 0xc4, 0x78, 0xcf, 0xe0, 0xd0, 0x43, 0x58, 0x12, 0xe9, 0x2f, 0x1e, 0x36,
- 0xe4, 0xe7, 0xf8, 0x95, 0x3c, 0x4b, 0x69, 0xea, 0x58, 0xfe, 0xee, 0x86, 0x9c, 0xcd, 0x6c, 0x45,
- 0x3a, 0xfa, 0x18, 0x20, 0x43, 0xa2, 0x15, 0xa8, 0x9d, 0xe1, 0x99, 0x8e, 0x43, 0xb1, 0x14, 0xce,
- 0x39, 0x77, 0xa7, 0x89, 0xf1, 0xba, 0x02, 0x3e, 0xad, 0x7e, 0x5c, 0xb1, 0x3c, 0xe8, 0x6f, 0x4d,
- 0xcf, 0x08, 0xcd, 0x1d, 0x5f, 0x83, 0xa5, 0xc0, 0xfd, 0x9a, 0x32, 0xe3, 0x49, 0x09, 0x48, 0x2c,
- 0x09, 0x29, 0x33, 0x2c, 0x24, 0x80, 0x7a, 0x50, 0xa5, 0x91, 0xf4, 0x57, 0xcb, 0xae, 0xd2, 0x28,
- 0x13, 0x54, 0xcf, 0x09, 0xb2, 0xfe, 0x59, 0x07, 0xc8, 0xa4, 0x20, 0x1b, 0x46, 0x84, 0x3a, 0x31,
- 0x66, 0xa2, 0x04, 0x71, 0x8e, 0x67, 0x1c, 0xc7, 0x0e, 0xc3, 0x5e, 0xc2, 0x62, 0x72, 0x2e, 0xee,
- 0x4f, 0x98, 0x7d, 0x4b, 0x99, 0x3d, 0xa7, 0x9b, 0x7d, 0x9b, 0xd0, 0x43, 0x75, 0x6e, 0x4b, 0x1c,
- 0xb3, 0xcd, 0x29, 0xb4, 0x0f, 0xb7, 0x32, 0x9e, 0x7e, 0x8e, 0x5d, 0xf5, 0x2a, 0x76, 0xab, 0x29,
- 0x3b, 0x3f, 0x63, 0xb5, 0x0b, 0xab, 0x84, 0x3a, 0xdf, 0x24, 0x38, 0x29, 0x30, 0xaa, 0x5d, 0xc5,
- 0x68, 0x40, 0xe8, 0xcf, 0xe4, 0x81, 0x8c, 0xcd, 0x01, 0xdc, 0xc9, 0x59, 0x29, 0xc2, 0x3d, 0xc7,
- 0xac, 0x7e, 0x15, 0xb3, 0xf5, 0x54, 0x2b, 0x91, 0x0f, 0x32, 0x8e, 0x3f, 0x81, 0x75, 0x42, 0x9d,
- 0x17, 0x2e, 0xe1, 0xf3, 0xec, 0x96, 0x5e, 0x62, 0xa4, 0xf8, 0xe8, 0x16, 0x79, 0x29, 0x23, 0x03,
- 0xcc, 0x26, 0x05, 0x23, 0x1b, 0x2f, 0x31, 0xf2, 0xa9, 0x3c, 0x90, 0xb1, 0x79, 0x0c, 0x03, 0x42,
- 0xe7, 0xb5, 0x59, 0xbe, 0x8a, 0x49, 0x9f, 0xd0, 0xa2, 0x26, 0x5b, 0x30, 0x88, 0xb1, 0xc7, 0x29,
- 0xcb, 0x3f, 0x82, 0xe6, 0x55, 0x2c, 0x56, 0x34, 0x7d, 0xca, 0xc3, 0xfa, 0x05, 0x74, 0xf6, 0x92,
- 0x09, 0xe6, 0xd3, 0xe3, 0x34, 0x19, 0xdc, 0x58, 0xfe, 0xb1, 0xfe, 0x5d, 0x85, 0xf6, 0xf6, 0x84,
- 0xd1, 0x24, 0x2a, 0xe4, 0x64, 0x15, 0xa4, 0xf3, 0x39, 0x59, 0x92, 0xc8, 0x9c, 0xac, 0x88, 0x3f,
- 0x84, 0x4e, 0x20, 0x43, 0x57, 0xd3, 0xab, 0x3c, 0x34, 0x58, 0x08, 0x6a, 0xbb, 0x1d, 0xe4, 0x92,
- 0xd9, 0x18, 0x20, 0x22, 0x7e, 0xac, 0xcf, 0xa8, 0x74, 0xd4, 0xd7, 0x15, 0xa1, 0x49, 0xd1, 0x76,
- 0x2b, 0x4a, 0xb3, 0xf5, 0xfb, 0xd0, 0x3e, 0x16, 0x4e, 0xd2, 0x07, 0x0a, 0xc9, 0x28, 0xf3, 0x9e,
- 0x0d, 0xc7, 0x59, 0x10, 0xee, 0x41, 0xf7, 0x54, 0xb9, 0x4c, 0x1f, 0x52, 0x6f, 0xe8, 0x75, 0x6d,
- 0x49, 0x66, 0xef, 0x38, 0xef, 0x59, 0x75, 0x01, 0x9d, 0xd3, 0x1c, 0x6a, 0x74, 0x08, 0x83, 0x05,
- 0x92, 0x92, 0x1c, 0xb4, 0x91, 0xcf, 0x41, 0xed, 0x87, 0x48, 0x09, 0xca, 0x9f, 0xcc, 0xe7, 0xa5,
- 0xdf, 0x55, 0xa1, 0xf3, 0x53, 0xcc, 0x5f, 0x50, 0x76, 0xa6, 0xf4, 0x45, 0x50, 0x0f, 0xdd, 0x00,
- 0x6b, 0x8e, 0x72, 0x8d, 0xee, 0x40, 0x93, 0x5d, 0xa8, 0x04, 0xa2, 0xef, 0x73, 0x99, 0x5d, 0xc8,
- 0xc4, 0x80, 0x5e, 0x05, 0x60, 0x17, 0x4e, 0xe4, 0x7a, 0x67, 0x58, 0x7b, 0xb0, 0x6e, 0xb7, 0xd8,
- 0xc5, 0x81, 0x42, 0x88, 0xa7, 0xc0, 0x2e, 0x1c, 0xcc, 0x18, 0x65, 0xb1, 0xce, 0x55, 0x4d, 0x76,
- 0xb1, 0x2b, 0x61, 0x7d, 0xd6, 0x67, 0x34, 0x8a, 0xb0, 0x2f, 0x73, 0xb4, 0x3c, 0xbb, 0xa3, 0x10,
- 0x42, 0x2a, 0x37, 0x52, 0x1b, 0x4a, 0x2a, 0xcf, 0xa4, 0xf2, 0x4c, 0xea, 0xb2, 0x3a, 0xc9, 0xf3,
- 0x52, 0x79, 0x2a, 0xb5, 0xa9, 0xa4, 0xf2, 0x9c, 0x54, 0x9e, 0x49, 0x6d, 0x99, 0xb3, 0x5a, 0xaa,
- 0xf5, 0xdb, 0x0a, 0xac, 0xcf, 0x17, 0x7e, 0xba, 0x4c, 0xfd, 0x10, 0x3a, 0x9e, 0xbc, 0xaf, 0xc2,
- 0x9b, 0x1c, 0x2c, 0xdc, 0xa4, 0xdd, 0xf6, 0x72, 0xcf, 0xf8, 0x23, 0xe8, 0x86, 0xca, 0xc1, 0xe9,
- 0xd3, 0xac, 0x65, 0xf7, 0x92, 0xf7, 0xbd, 0xdd, 0x09, 0x73, 0x90, 0xe5, 0x03, 0xfa, 0x8a, 0x11,
- 0x8e, 0x0f, 0x39, 0xc3, 0x6e, 0x70, 0x13, 0x0d, 0x08, 0x82, 0xba, 0xac, 0x56, 0x6a, 0xb2, 0xbe,
- 0x96, 0x6b, 0xeb, 0x2d, 0x58, 0x2d, 0x48, 0xd1, 0xb6, 0xae, 0x40, 0x6d, 0x8a, 0x43, 0xc9, 0xbd,
- 0x6b, 0x8b, 0xa5, 0xe5, 0xc2, 0xc0, 0xc6, 0xae, 0x7f, 0x73, 0xda, 0x68, 0x11, 0xb5, 0x4c, 0xc4,
- 0x06, 0xa0, 0xbc, 0x08, 0xad, 0x8a, 0xd1, 0xba, 0x92, 0xd3, 0xfa, 0x19, 0x0c, 0xb6, 0xa7, 0x34,
- 0xc6, 0x87, 0xdc, 0x27, 0xe1, 0x4d, 0x74, 0x4c, 0xbf, 0x82, 0xd5, 0xe7, 0x7c, 0xf6, 0x95, 0x60,
- 0x16, 0x93, 0x6f, 0xf1, 0x0d, 0xd9, 0xc7, 0xe8, 0x0b, 0x63, 0x1f, 0xa3, 0x2f, 0x44, 0xb3, 0xe4,
- 0xd1, 0x69, 0x12, 0x84, 0x32, 0x14, 0xba, 0xb6, 0x86, 0xac, 0x2d, 0xe8, 0xa8, 0x1a, 0xfa, 0x29,
- 0xf5, 0x93, 0x29, 0x2e, 0x8d, 0xc1, 0x7b, 0x00, 0x91, 0xcb, 0xdc, 0x00, 0x73, 0xcc, 0xd4, 0x1b,
- 0x6a, 0xd9, 0x39, 0x8c, 0xf5, 0x87, 0x2a, 0xac, 0xa9, 0x91, 0xc8, 0xa1, 0x9a, 0x04, 0x18, 0x13,
- 0x46, 0xd0, 0x3c, 0xa5, 0x31, 0xcf, 0x31, 0x4c, 0x61, 0xa1, 0xa2, 0x1f, 0x1a, 0x6e, 0x62, 0x59,
- 0x98, 0x53, 0xd4, 0xae, 0x9e, 0x53, 0x2c, 0x4c, 0x22, 0xea, 0x8b, 0x93, 0x08, 0x11, 0x6d, 0x86,
- 0x88, 0xa8, 0x18, 0x6f, 0xd9, 0x2d, 0x8d, 0xd9, 0xf7, 0xd1, 0x03, 0xe8, 0x4f, 0x84, 0x96, 0xce,
- 0x29, 0xa5, 0x67, 0x4e, 0xe4, 0xf2, 0x53, 0x19, 0xea, 0x2d, 0xbb, 0x2b, 0xd1, 0x7b, 0x94, 0x9e,
- 0x1d, 0xb8, 0xfc, 0x14, 0x7d, 0x02, 0x3d, 0x5d, 0x06, 0x06, 0xd2, 0x45, 0xb1, 0xfe, 0xf8, 0xe9,
- 0x28, 0xca, 0x7b, 0xcf, 0xee, 0x9e, 0xe5, 0xa0, 0xd8, 0xba, 0x0d, 0xb7, 0x76, 0x70, 0xcc, 0x19,
- 0x9d, 0x15, 0x1d, 0x63, 0xfd, 0x08, 0x60, 0x3f, 0xe4, 0x98, 0x9d, 0xb8, 0x1e, 0x8e, 0xd1, 0x7b,
- 0x79, 0x48, 0x17, 0x47, 0x2b, 0x63, 0x35, 0x91, 0x4a, 0x37, 0xec, 0x1c, 0x8d, 0x35, 0x86, 0x86,
- 0x4d, 0x13, 0x91, 0x8e, 0xde, 0x30, 0x2b, 0x7d, 0xae, 0xa3, 0xcf, 0x49, 0xa4, 0xad, 0xf7, 0xac,
- 0x3d, 0xd3, 0xc2, 0x66, 0xec, 0xf4, 0x15, 0x8d, 0xa1, 0x45, 0x0c, 0x4e, 0x67, 0x95, 0x45, 0xd1,
- 0x19, 0x89, 0xf5, 0x19, 0xac, 0x2a, 0x4e, 0x8a, 0xb3, 0x61, 0xf3, 0x06, 0x34, 0x98, 0x51, 0xa3,
- 0x92, 0x8d, 0xa2, 0x34, 0x91, 0xde, 0x13, 0xfe, 0x10, 0x1d, 0x75, 0x66, 0x88, 0xf1, 0xc7, 0x2a,
- 0x0c, 0xc4, 0x46, 0x81, 0xa7, 0xf5, 0x4b, 0x58, 0x7d, 0x16, 0x4e, 0x49, 0x88, 0xb7, 0x0f, 0x8e,
- 0x9e, 0xe2, 0x34, 0xee, 0x11, 0xd4, 0x45, 0x7d, 0x24, 0x05, 0x35, 0x6d, 0xb9, 0x16, 0x81, 0x10,
- 0x1e, 0x3b, 0x5e, 0x94, 0xc4, 0x7a, 0xf6, 0xd3, 0x08, 0x8f, 0xb7, 0xa3, 0x24, 0x16, 0x89, 0x5c,
- 0x7c, 0xc8, 0x69, 0x38, 0x9d, 0xc9, 0x68, 0x68, 0xda, 0xcb, 0x5e, 0x94, 0x3c, 0x0b, 0xa7, 0x33,
- 0xeb, 0x07, 0xb2, 0xdb, 0xc5, 0xd8, 0xb7, 0xdd, 0xd0, 0xa7, 0xc1, 0x0e, 0x3e, 0xcf, 0x49, 0x48,
- 0x3b, 0x2b, 0x13, 0xf5, 0xdf, 0x55, 0xa0, 0xf3, 0x78, 0x82, 0x43, 0xbe, 0x83, 0xb9, 0x4b, 0xa6,
- 0xb2, 0x7b, 0x3a, 0xc7, 0x2c, 0x26, 0x34, 0xd4, 0x4f, 0xdb, 0x80, 0xa2, 0xf9, 0x25, 0x21, 0xe1,
- 0x8e, 0xef, 0xe2, 0x80, 0x86, 0x92, 0x4b, 0xd3, 0x06, 0x81, 0xda, 0x91, 0x18, 0xf4, 0x16, 0xf4,
- 0xd5, 0x6c, 0xce, 0x39, 0x75, 0x43, 0x7f, 0x2a, 0x82, 0x4a, 0xcd, 0x2a, 0x7a, 0x0a, 0xbd, 0xa7,
- 0xb1, 0xe8, 0x6d, 0x58, 0xd1, 0x4f, 0x3e, 0xa3, 0xac, 0x4b, 0xca, 0xbe, 0xc6, 0x17, 0x48, 0x93,
- 0x28, 0xa2, 0x8c, 0xc7, 0x4e, 0x8c, 0x3d, 0x8f, 0x06, 0x91, 0x6e, 0x3d, 0xfa, 0x06, 0x7f, 0xa8,
- 0xd0, 0xd6, 0x04, 0x56, 0x9f, 0x08, 0x3b, 0xb5, 0x25, 0xd9, 0x15, 0xf6, 0x02, 0x1c, 0x38, 0xc7,
- 0x53, 0xea, 0x9d, 0x39, 0x22, 0x11, 0x69, 0x0f, 0x8b, 0xe2, 0x66, 0x4b, 0x20, 0x0f, 0xc9, 0xb7,
- 0xb2, 0xcb, 0x16, 0x54, 0xa7, 0x94, 0x47, 0xd3, 0x64, 0xe2, 0x44, 0x8c, 0x1e, 0x63, 0x6d, 0x62,
- 0x3f, 0xc0, 0xc1, 0x9e, 0xc2, 0x1f, 0x08, 0xb4, 0xf5, 0xd7, 0x0a, 0xac, 0x15, 0x25, 0xe9, 0xb4,
- 0xba, 0x09, 0x6b, 0x45, 0x51, 0xfa, 0x53, 0xab, 0x4a, 0xb9, 0x41, 0x5e, 0xa0, 0xfa, 0xe8, 0x7e,
- 0x04, 0x5d, 0x39, 0xb0, 0x75, 0x7c, 0xc5, 0xa9, 0x58, 0x60, 0xe4, 0xef, 0xc5, 0xee, 0xb8, 0xf9,
- 0x5b, 0xfa, 0x04, 0xee, 0x68, 0xf3, 0x9d, 0x45, 0xb5, 0xd5, 0x83, 0x58, 0xd7, 0x04, 0x4f, 0xe7,
- 0xb4, 0xff, 0x12, 0x86, 0x19, 0x6a, 0x6b, 0x26, 0x91, 0xc6, 0x57, 0xef, 0xc1, 0xea, 0x9c, 0xb1,
- 0x8f, 0x7d, 0x9f, 0xc9, 0x10, 0xac, 0xdb, 0x65, 0x5b, 0xd6, 0x23, 0xb8, 0x7d, 0x88, 0xb9, 0xf2,
- 0x86, 0xcb, 0x75, 0xd5, 0xaf, 0x98, 0xad, 0x40, 0xed, 0x10, 0x7b, 0xd2, 0xf8, 0x9a, 0x2d, 0x96,
- 0xe2, 0x01, 0x1e, 0xc5, 0xd8, 0x93, 0x56, 0xd6, 0x6c, 0xb9, 0xb6, 0xfe, 0x52, 0x81, 0x65, 0x9d,
- 0x08, 0x45, 0x32, 0xf7, 0x19, 0x39, 0xc7, 0x4c, 0x3f, 0x3d, 0x0d, 0xa1, 0x37, 0xa1, 0xa7, 0x56,
- 0x0e, 0x8d, 0x38, 0xa1, 0x69, 0x7a, 0xed, 0x2a, 0xec, 0x33, 0x85, 0x94, 0xb3, 0x38, 0x39, 0x6a,
- 0xd2, 0x5d, 0x9d, 0x86, 0xe4, 0x40, 0x2d, 0x16, 0xb1, 0x2f, 0xd3, 0x69, 0xcb, 0xd6, 0x90, 0x78,
- 0xea, 0x86, 0xdf, 0x92, 0xe4, 0x67, 0x40, 0xf1, 0xd4, 0x03, 0x9a, 0x84, 0xdc, 0x89, 0x28, 0x09,
- 0xb9, 0xce, 0x9f, 0x20, 0x51, 0x07, 0x02, 0x63, 0xfd, 0xa6, 0x02, 0x0d, 0x35, 0x8f, 0x16, 0x7d,
- 0x64, 0xfa, 0x15, 0xab, 0x12, 0x59, 0x11, 0x48, 0x59, 0xea, 0xcb, 0x25, 0xd7, 0x22, 0x8e, 0xcf,
- 0x03, 0x95, 0x8b, 0xb5, 0x6a, 0xe7, 0x81, 0x4c, 0xc2, 0x6f, 0x42, 0x2f, 0xfb, 0x18, 0xca, 0x7d,
- 0xa5, 0x62, 0x37, 0xc5, 0x4a, 0xb2, 0x4b, 0x35, 0xb5, 0x7e, 0x2e, 0xda, 0xe7, 0x74, 0x16, 0xbb,
- 0x02, 0xb5, 0x24, 0x55, 0x46, 0x2c, 0x05, 0x66, 0x92, 0x7e, 0x46, 0xc5, 0x12, 0x3d, 0x80, 0x9e,
- 0xeb, 0xfb, 0x44, 0x1c, 0x77, 0xa7, 0x4f, 0x88, 0x9f, 0x06, 0x69, 0x11, 0x6b, 0xfd, 0xbd, 0x02,
- 0xfd, 0x6d, 0x1a, 0xcd, 0x7e, 0x4c, 0xa6, 0x38, 0x97, 0x41, 0xa4, 0x92, 0xfa, 0x2b, 0x2a, 0xd6,
- 0xa2, 0x32, 0x3c, 0x21, 0x53, 0xac, 0x42, 0x4b, 0xdd, 0x6c, 0x53, 0x20, 0x64, 0x58, 0x99, 0xcd,
- 0x74, 0xc4, 0xd5, 0x55, 0x9b, 0x4f, 0xa9, 0x2f, 0x6b, 0x60, 0x9f, 0x30, 0x27, 0x1d, 0x68, 0x75,
- 0xed, 0x65, 0x9f, 0x30, 0xb9, 0xa5, 0x0d, 0x59, 0x92, 0x33, 0xd5, 0xbc, 0x21, 0x0d, 0x85, 0x11,
- 0x86, 0xac, 0x43, 0x83, 0x9e, 0x9c, 0xc4, 0x98, 0xcb, 0x6a, 0xb5, 0x66, 0x6b, 0x28, 0x4d, 0x73,
- 0xcd, 0x5c, 0x9a, 0xbb, 0x05, 0xab, 0x72, 0x7a, 0xff, 0x9c, 0xb9, 0x1e, 0x09, 0x27, 0x26, 0x15,
- 0xaf, 0x01, 0x3a, 0xe4, 0x34, 0x2a, 0x62, 0x1f, 0xfe, 0x7e, 0x45, 0xe7, 0x44, 0xdd, 0xca, 0xa2,
- 0x27, 0xd0, 0x9f, 0xfb, 0x6b, 0x04, 0xe9, 0xd9, 0x46, 0xf9, 0x3f, 0x26, 0xa3, 0xf5, 0xb1, 0xfa,
- 0xab, 0x65, 0x6c, 0xfe, 0x6a, 0x19, 0xef, 0x06, 0x11, 0x9f, 0xa1, 0x5d, 0xe8, 0x15, 0xff, 0x44,
- 0x40, 0x77, 0x4d, 0x29, 0x50, 0xf2, 0xd7, 0xc2, 0xa5, 0x6c, 0x9e, 0x40, 0x7f, 0xee, 0xff, 0x04,
- 0xa3, 0x4f, 0xf9, 0xdf, 0x0c, 0x97, 0x32, 0x7a, 0x04, 0xed, 0xdc, 0x1f, 0x08, 0x68, 0xa8, 0x98,
- 0x2c, 0xfe, 0xa7, 0x70, 0x29, 0x83, 0x6d, 0xe8, 0x16, 0x66, 0xfa, 0x68, 0xa4, 0xed, 0x29, 0x19,
- 0xf4, 0x5f, 0xca, 0x64, 0x0b, 0xda, 0xb9, 0xd1, 0xba, 0xd1, 0x62, 0x71, 0x7e, 0x3f, 0xba, 0x53,
- 0xb2, 0xa3, 0x53, 0xef, 0x1e, 0x74, 0x0b, 0x83, 0x70, 0xa3, 0x48, 0xd9, 0x10, 0x7e, 0x74, 0xb7,
- 0x74, 0x4f, 0x73, 0x7a, 0x02, 0xfd, 0xb9, 0xb1, 0xb8, 0x71, 0x6e, 0xf9, 0xb4, 0xfc, 0x52, 0xb3,
- 0xbe, 0x90, 0x97, 0x9d, 0xeb, 0x7a, 0x72, 0x97, 0xbd, 0x38, 0x04, 0x1f, 0xbd, 0x52, 0xbe, 0xa9,
- 0xb5, 0xda, 0x85, 0x5e, 0x71, 0xfe, 0x6d, 0x98, 0x95, 0x4e, 0xc5, 0xaf, 0x7e, 0x39, 0x85, 0x51,
- 0x78, 0xf6, 0x72, 0xca, 0x26, 0xe4, 0x97, 0x32, 0x7a, 0x0c, 0xa0, 0x7b, 0x1c, 0x9f, 0x84, 0xe9,
- 0x95, 0x2d, 0xf4, 0x56, 0xe9, 0x95, 0x95, 0xf4, 0x43, 0x8f, 0x00, 0x54, 0x6b, 0xe2, 0xd3, 0x84,
- 0xa3, 0xdb, 0x46, 0x8d, 0xb9, 0x7e, 0x68, 0x34, 0x5c, 0xdc, 0x58, 0x60, 0x80, 0x19, 0xbb, 0x0e,
- 0x83, 0xcf, 0x01, 0xb2, 0x96, 0xc7, 0x30, 0x58, 0x68, 0x82, 0xae, 0xf0, 0x41, 0x27, 0xdf, 0xe0,
- 0x20, 0x6d, 0x6b, 0x49, 0xd3, 0x73, 0x05, 0x8b, 0xfe, 0x5c, 0x01, 0x5b, 0x7c, 0x6c, 0xf3, 0x75,
- 0xed, 0x68, 0xa1, 0x88, 0x45, 0x1f, 0x41, 0x27, 0x5f, 0xb9, 0x1a, 0x2d, 0x4a, 0xaa, 0xd9, 0x51,
- 0xa1, 0x7a, 0x45, 0x8f, 0xa0, 0x57, 0xac, 0x5a, 0x51, 0x2e, 0x2e, 0x16, 0x6a, 0xd9, 0x91, 0x9e,
- 0xc9, 0xe4, 0xc8, 0x3f, 0x00, 0xc8, 0xaa, 0x5b, 0xe3, 0xbe, 0x85, 0x7a, 0x77, 0x4e, 0xea, 0x63,
- 0xe8, 0xe4, 0x33, 0xb1, 0x51, 0xb7, 0x24, 0x3b, 0x5f, 0x95, 0xb5, 0x72, 0x59, 0xdb, 0x3c, 0xbe,
- 0xc5, 0x44, 0x7e, 0x55, 0xd6, 0x2a, 0xf4, 0x75, 0x26, 0x59, 0x94, 0x35, 0x7b, 0x57, 0xe5, 0xf2,
- 0x62, 0x13, 0x64, 0xdc, 0x57, 0xda, 0x1a, 0x5d, 0xf5, 0x88, 0xf2, 0xdd, 0x80, 0xf1, 0x47, 0x49,
- 0x87, 0xf0, 0x92, 0xa0, 0xce, 0x57, 0xfc, 0xb9, 0xa0, 0x2e, 0x69, 0x04, 0x2e, 0x65, 0xb4, 0x07,
- 0xfd, 0x27, 0xa6, 0x98, 0xd3, 0x85, 0xa6, 0x56, 0xa7, 0xa4, 0xb0, 0x1e, 0x8d, 0xca, 0xb6, 0x74,
- 0x64, 0x7d, 0x01, 0x83, 0x85, 0x22, 0x13, 0xdd, 0x4b, 0x47, 0x87, 0xa5, 0xd5, 0xe7, 0xa5, 0x6a,
- 0xed, 0xc3, 0xca, 0x7c, 0x8d, 0x89, 0x5e, 0xd5, 0x97, 0x5e, 0x5e, 0x7b, 0x5e, 0xca, 0xea, 0x13,
- 0x68, 0x9a, 0x9a, 0x06, 0xe9, 0x11, 0xed, 0x5c, 0x8d, 0x73, 0xd9, 0xd1, 0xad, 0xce, 0x77, 0xdf,
- 0xdf, 0xab, 0xfc, 0xe3, 0xfb, 0x7b, 0x95, 0x7f, 0x7d, 0x7f, 0xaf, 0x72, 0xdc, 0x90, 0xbb, 0x1f,
- 0xfc, 0x27, 0x00, 0x00, 0xff, 0xff, 0xa5, 0xac, 0x85, 0x1d, 0xaa, 0x21, 0x00, 0x00,
+ 0xd1, 0xd8, 0x07, 0x97, 0xbb, 0xb5, 0x2f, 0x6e, 0x93, 0xa2, 0x56, 0x2b, 0x59, 0x9f, 0x3c, 0xb6,
+ 0x65, 0xfa, 0xf3, 0xe7, 0xa5, 0x2d, 0x1b, 0x9f, 0x5f, 0x70, 0x04, 0xf1, 0x11, 0x91, 0xb1, 0x15,
+ 0x31, 0x43, 0x11, 0x4e, 0x10, 0x04, 0x83, 0xe1, 0x4c, 0x73, 0xd9, 0xe6, 0xce, 0xf4, 0xb8, 0xa7,
+ 0x87, 0xe2, 0x3a, 0x40, 0x8e, 0xc9, 0x2d, 0x97, 0x00, 0xb9, 0xe5, 0x0f, 0x04, 0xb9, 0xe5, 0x98,
+ 0x6b, 0x0e, 0x46, 0x4e, 0xf9, 0x05, 0x41, 0xe0, 0x9f, 0x90, 0x5f, 0x10, 0xf4, 0x6b, 0x1e, 0xbb,
+ 0x43, 0x1a, 0x21, 0x08, 0xe4, 0xb2, 0xe8, 0xaa, 0xae, 0xae, 0x57, 0x77, 0xd5, 0x54, 0xd5, 0x42,
+ 0xdb, 0x9d, 0xe0, 0x90, 0x8f, 0x23, 0x46, 0x39, 0x45, 0xf5, 0x09, 0x8b, 0xbc, 0x51, 0x8b, 0x7a,
+ 0x44, 0x21, 0x46, 0xff, 0x3f, 0x21, 0xfc, 0x34, 0x39, 0x1e, 0x7b, 0x34, 0xd8, 0x3c, 0x73, 0xb9,
+ 0xfb, 0x8e, 0x47, 0x43, 0xee, 0x92, 0x10, 0xb3, 0x78, 0x53, 0x1e, 0xdc, 0x8c, 0xce, 0x26, 0x9b,
+ 0x7c, 0x16, 0xe1, 0x58, 0xfd, 0xea, 0x73, 0x77, 0x27, 0x94, 0x4e, 0xa6, 0x78, 0x53, 0x42, 0xc7,
+ 0xc9, 0xc9, 0x26, 0x0e, 0x22, 0x3e, 0x53, 0x9b, 0xd6, 0x1f, 0xaa, 0xb0, 0xbe, 0xcd, 0xb0, 0xcb,
+ 0xf1, 0xb6, 0xe1, 0x66, 0xe3, 0xaf, 0x13, 0x1c, 0x73, 0xf4, 0x2a, 0x74, 0x52, 0x09, 0x0e, 0xf1,
+ 0x87, 0x95, 0x07, 0x95, 0x8d, 0x96, 0xdd, 0x4e, 0x71, 0xfb, 0x3e, 0xba, 0x0d, 0xcb, 0xf8, 0x02,
+ 0x7b, 0x62, 0xb7, 0x2a, 0x77, 0x1b, 0x02, 0xdc, 0xf7, 0xd1, 0x7b, 0xd0, 0x8e, 0x39, 0x23, 0xe1,
+ 0xc4, 0x49, 0x62, 0xcc, 0x86, 0xb5, 0x07, 0x95, 0x8d, 0xf6, 0xa3, 0x95, 0xb1, 0x30, 0x69, 0x7c,
+ 0x28, 0x37, 0x8e, 0x62, 0xcc, 0x6c, 0x88, 0xd3, 0x35, 0x7a, 0x08, 0xcb, 0x3e, 0x3e, 0x27, 0x1e,
+ 0x8e, 0x87, 0xf5, 0x07, 0xb5, 0x8d, 0xf6, 0xa3, 0x8e, 0x22, 0xdf, 0x91, 0x48, 0xdb, 0x6c, 0xa2,
+ 0xb7, 0xa0, 0x19, 0x73, 0xca, 0xdc, 0x09, 0x8e, 0x87, 0x4b, 0x92, 0xb0, 0x6b, 0xf8, 0x4a, 0xac,
+ 0x9d, 0x6e, 0xa3, 0x7b, 0x50, 0x7b, 0xbe, 0xbd, 0x3f, 0x6c, 0x48, 0xe9, 0xa0, 0xa9, 0x22, 0xec,
+ 0xd9, 0x02, 0x8d, 0x5e, 0x83, 0x6e, 0xec, 0x86, 0xfe, 0x31, 0xbd, 0x70, 0x22, 0xe2, 0x87, 0xf1,
+ 0x70, 0xf9, 0x41, 0x65, 0xa3, 0x69, 0x77, 0x34, 0xf2, 0x40, 0xe0, 0xac, 0x4f, 0xe0, 0xd6, 0x21,
+ 0x77, 0x19, 0xbf, 0x86, 0x77, 0xac, 0x23, 0x58, 0xb7, 0x71, 0x40, 0xcf, 0xaf, 0xe5, 0xda, 0x21,
+ 0x2c, 0x73, 0x12, 0x60, 0x9a, 0x70, 0xe9, 0xda, 0xae, 0x6d, 0x40, 0xeb, 0x4f, 0x15, 0x40, 0xbb,
+ 0x17, 0xd8, 0x3b, 0x60, 0xd4, 0xc3, 0x71, 0xfc, 0x5f, 0xba, 0xae, 0x37, 0x61, 0x39, 0x52, 0x0a,
+ 0x0c, 0xeb, 0x92, 0x5c, 0xdf, 0x82, 0xd1, 0xca, 0xec, 0x5a, 0x5f, 0xc1, 0xda, 0x21, 0x99, 0x84,
+ 0xee, 0xf4, 0x06, 0xf5, 0x5d, 0x87, 0x46, 0x2c, 0x79, 0x4a, 0x55, 0xbb, 0xb6, 0x86, 0xac, 0x03,
+ 0x40, 0x5f, 0xba, 0x84, 0xdf, 0x9c, 0x24, 0xeb, 0x1d, 0x58, 0x2d, 0x70, 0x8c, 0x23, 0x1a, 0xc6,
+ 0x58, 0x2a, 0xc0, 0x5d, 0x9e, 0xc4, 0x92, 0xd9, 0x92, 0xad, 0x21, 0x0b, 0xc3, 0xda, 0x17, 0x24,
+ 0x36, 0xe4, 0xf8, 0x3f, 0x51, 0x61, 0x1d, 0x1a, 0x27, 0x94, 0x05, 0x2e, 0x37, 0x1a, 0x28, 0x08,
+ 0x21, 0xa8, 0xbb, 0x6c, 0x12, 0x0f, 0x6b, 0x0f, 0x6a, 0x1b, 0x2d, 0x5b, 0xae, 0xc5, 0xab, 0x9c,
+ 0x13, 0xa3, 0xf5, 0x7a, 0x15, 0x3a, 0xda, 0xef, 0xce, 0x94, 0xc4, 0x5c, 0xca, 0xe9, 0xd8, 0x6d,
+ 0x8d, 0x13, 0x67, 0x2c, 0x0a, 0xeb, 0x47, 0x91, 0x7f, 0xcd, 0x80, 0x7f, 0x04, 0x2d, 0x86, 0x63,
+ 0x9a, 0x30, 0x11, 0xa6, 0x55, 0x79, 0xef, 0x6b, 0xea, 0xde, 0xbf, 0x20, 0x61, 0x72, 0x61, 0x9b,
+ 0x3d, 0x3b, 0x23, 0xd3, 0x21, 0xc4, 0xe3, 0xeb, 0x84, 0xd0, 0x27, 0x70, 0xeb, 0xc0, 0x4d, 0xe2,
+ 0xeb, 0xe8, 0x6a, 0x7d, 0x2a, 0xc2, 0x2f, 0x4e, 0x82, 0x6b, 0x1d, 0xfe, 0x63, 0x05, 0x9a, 0xdb,
+ 0x51, 0x72, 0x14, 0xbb, 0x13, 0x8c, 0xfe, 0x07, 0xda, 0x9c, 0x72, 0x77, 0xea, 0x24, 0x02, 0x94,
+ 0xe4, 0x75, 0x1b, 0x24, 0x4a, 0x11, 0x08, 0xb7, 0x63, 0xe6, 0x45, 0x89, 0xa6, 0xa8, 0x3e, 0xa8,
+ 0x6d, 0xd4, 0xed, 0xb6, 0xc2, 0x29, 0x92, 0x31, 0xac, 0xca, 0x3d, 0x87, 0x84, 0xce, 0x19, 0x66,
+ 0x21, 0x9e, 0x06, 0xd4, 0xc7, 0xf2, 0xfd, 0xd6, 0xed, 0x81, 0xdc, 0xda, 0x0f, 0x3f, 0x4f, 0x37,
+ 0xd0, 0xff, 0xc2, 0x20, 0xa5, 0x17, 0x41, 0x29, 0xa9, 0xeb, 0x92, 0xba, 0xaf, 0xa9, 0x8f, 0x34,
+ 0xda, 0xfa, 0x15, 0xf4, 0x5e, 0x9c, 0x32, 0xca, 0xf9, 0x94, 0x84, 0x93, 0x1d, 0x97, 0xbb, 0x22,
+ 0x7b, 0x44, 0x98, 0x11, 0xea, 0xc7, 0x5a, 0x5b, 0x03, 0xa2, 0xb7, 0x61, 0xc0, 0x15, 0x2d, 0xf6,
+ 0x1d, 0x43, 0x53, 0x95, 0x34, 0x2b, 0xe9, 0xc6, 0x81, 0x26, 0x7e, 0x03, 0x7a, 0x19, 0xb1, 0xc8,
+ 0x3f, 0x5a, 0xdf, 0x6e, 0x8a, 0x7d, 0x41, 0x02, 0x6c, 0x9d, 0x4b, 0x5f, 0xc9, 0x4b, 0x46, 0x6f,
+ 0x43, 0x2b, 0xf3, 0x43, 0x45, 0xbe, 0x90, 0x9e, 0x7a, 0x21, 0xc6, 0x9d, 0x76, 0x33, 0x75, 0xca,
+ 0x67, 0xd0, 0xe7, 0xa9, 0xe2, 0x8e, 0xef, 0x72, 0xb7, 0xf8, 0xa8, 0x8a, 0x56, 0xd9, 0x3d, 0x5e,
+ 0x80, 0xad, 0x4f, 0xa1, 0x75, 0x40, 0xfc, 0x58, 0x09, 0x1e, 0xc2, 0xb2, 0x97, 0x30, 0x86, 0x43,
+ 0x6e, 0x4c, 0xd6, 0x20, 0x5a, 0x83, 0xa5, 0x29, 0x09, 0x08, 0xd7, 0x66, 0x2a, 0xc0, 0xa2, 0x00,
+ 0xcf, 0x70, 0x40, 0xd9, 0x4c, 0x3a, 0x6c, 0x0d, 0x96, 0xf2, 0x97, 0xab, 0x00, 0x74, 0x17, 0x5a,
+ 0x81, 0x7b, 0x91, 0x5e, 0xaa, 0xd8, 0x69, 0x06, 0xee, 0x85, 0x52, 0x7e, 0x08, 0xcb, 0x27, 0x2e,
+ 0x99, 0x7a, 0x21, 0xd7, 0x5e, 0x31, 0x60, 0x26, 0xb0, 0x9e, 0x17, 0xf8, 0xd7, 0x2a, 0xb4, 0x95,
+ 0x44, 0xa5, 0xf0, 0x1a, 0x2c, 0x79, 0xae, 0x77, 0x9a, 0x8a, 0x94, 0x00, 0x7a, 0x68, 0x14, 0xa9,
+ 0xe6, 0x93, 0x70, 0xa6, 0xa9, 0x51, 0x6d, 0x13, 0x20, 0x7e, 0xe9, 0x46, 0x5a, 0xb7, 0xda, 0x25,
+ 0xc4, 0x2d, 0x41, 0xa3, 0xd4, 0x7d, 0x1f, 0x3a, 0xea, 0xdd, 0xe9, 0x23, 0xf5, 0x4b, 0x8e, 0xb4,
+ 0x15, 0x95, 0x3a, 0xf4, 0x1a, 0x74, 0x93, 0x18, 0x3b, 0xa7, 0x04, 0x33, 0x97, 0x79, 0xa7, 0xb3,
+ 0xe1, 0x92, 0xfa, 0x46, 0x26, 0x31, 0xde, 0x33, 0x38, 0xf4, 0x08, 0x96, 0x44, 0xfa, 0x8b, 0x87,
+ 0x0d, 0xf9, 0x39, 0xbe, 0x97, 0x67, 0x29, 0x4d, 0x1d, 0xcb, 0xdf, 0xdd, 0x90, 0xb3, 0x99, 0xad,
+ 0x48, 0x47, 0x1f, 0x01, 0x64, 0x48, 0xb4, 0x02, 0xb5, 0x33, 0x3c, 0xd3, 0x71, 0x28, 0x96, 0xc2,
+ 0x39, 0xe7, 0xee, 0x34, 0x31, 0x5e, 0x57, 0xc0, 0x27, 0xd5, 0x8f, 0x2a, 0x96, 0x07, 0xfd, 0xad,
+ 0xe9, 0x19, 0xa1, 0xb9, 0xe3, 0x6b, 0xb0, 0x14, 0xb8, 0x5f, 0x51, 0x66, 0x3c, 0x29, 0x01, 0x89,
+ 0x25, 0x21, 0x65, 0x86, 0x85, 0x04, 0x50, 0x0f, 0xaa, 0x34, 0x92, 0xfe, 0x6a, 0xd9, 0x55, 0x1a,
+ 0x65, 0x82, 0xea, 0x39, 0x41, 0xd6, 0x3f, 0xea, 0x00, 0x99, 0x14, 0x64, 0xc3, 0x88, 0x50, 0x27,
+ 0xc6, 0x4c, 0x94, 0x20, 0xce, 0xf1, 0x8c, 0xe3, 0xd8, 0x61, 0xd8, 0x4b, 0x58, 0x4c, 0xce, 0xc5,
+ 0xfd, 0x09, 0xb3, 0x6f, 0x29, 0xb3, 0xe7, 0x74, 0xb3, 0x6f, 0x13, 0x7a, 0xa8, 0xce, 0x6d, 0x89,
+ 0x63, 0xb6, 0x39, 0x85, 0xf6, 0xe1, 0x56, 0xc6, 0xd3, 0xcf, 0xb1, 0xab, 0x5e, 0xc5, 0x6e, 0x35,
+ 0x65, 0xe7, 0x67, 0xac, 0x76, 0x61, 0x95, 0x50, 0xe7, 0xeb, 0x04, 0x27, 0x05, 0x46, 0xb5, 0xab,
+ 0x18, 0x0d, 0x08, 0xfd, 0x89, 0x3c, 0x90, 0xb1, 0x39, 0x80, 0x3b, 0x39, 0x2b, 0x45, 0xb8, 0xe7,
+ 0x98, 0xd5, 0xaf, 0x62, 0xb6, 0x9e, 0x6a, 0x25, 0xf2, 0x41, 0xc6, 0xf1, 0x47, 0xb0, 0x4e, 0xa8,
+ 0xf3, 0xd2, 0x25, 0x7c, 0x9e, 0xdd, 0xd2, 0xf7, 0x18, 0x29, 0x3e, 0xba, 0x45, 0x5e, 0xca, 0xc8,
+ 0x00, 0xb3, 0x49, 0xc1, 0xc8, 0xc6, 0xf7, 0x18, 0xf9, 0x4c, 0x1e, 0xc8, 0xd8, 0x3c, 0x81, 0x01,
+ 0xa1, 0xf3, 0xda, 0x2c, 0x5f, 0xc5, 0xa4, 0x4f, 0x68, 0x51, 0x93, 0x2d, 0x18, 0xc4, 0xd8, 0xe3,
+ 0x94, 0xe5, 0x1f, 0x41, 0xf3, 0x2a, 0x16, 0x2b, 0x9a, 0x3e, 0xe5, 0x61, 0xfd, 0x1c, 0x3a, 0x7b,
+ 0xc9, 0x04, 0xf3, 0xe9, 0x71, 0x9a, 0x0c, 0x6e, 0x2c, 0xff, 0x58, 0xff, 0xaa, 0x42, 0x7b, 0x7b,
+ 0xc2, 0x68, 0x12, 0x15, 0x72, 0xb2, 0x0a, 0xd2, 0xf9, 0x9c, 0x2c, 0x49, 0x64, 0x4e, 0x56, 0xc4,
+ 0x1f, 0x40, 0x27, 0x90, 0xa1, 0xab, 0xe9, 0x55, 0x1e, 0x1a, 0x2c, 0x04, 0xb5, 0xdd, 0x0e, 0x72,
+ 0xc9, 0x6c, 0x0c, 0x10, 0x11, 0x3f, 0xd6, 0x67, 0x54, 0x3a, 0xea, 0xeb, 0x8a, 0xd0, 0xa4, 0x68,
+ 0xbb, 0x15, 0xa5, 0xd9, 0xfa, 0x3d, 0x68, 0x1f, 0x0b, 0x27, 0xe9, 0x03, 0x85, 0x64, 0x94, 0x79,
+ 0xcf, 0x86, 0xe3, 0x2c, 0x08, 0xf7, 0xa0, 0x7b, 0xaa, 0x5c, 0xa6, 0x0f, 0xa9, 0x37, 0xf4, 0x9a,
+ 0xb6, 0x24, 0xb3, 0x77, 0x9c, 0xf7, 0xac, 0xba, 0x80, 0xce, 0x69, 0x0e, 0x35, 0x3a, 0x84, 0xc1,
+ 0x02, 0x49, 0x49, 0x0e, 0xda, 0xc8, 0xe7, 0xa0, 0xf6, 0x23, 0xa4, 0x04, 0xe5, 0x4f, 0xe6, 0xf3,
+ 0xd2, 0x6f, 0xab, 0xd0, 0xf9, 0x31, 0xe6, 0x2f, 0x29, 0x3b, 0x53, 0xfa, 0x22, 0xa8, 0x87, 0x6e,
+ 0x80, 0x35, 0x47, 0xb9, 0x46, 0x77, 0xa0, 0xc9, 0x2e, 0x54, 0x02, 0xd1, 0xf7, 0xb9, 0xcc, 0x2e,
+ 0x64, 0x62, 0x40, 0xaf, 0x00, 0xb0, 0x0b, 0x27, 0x72, 0xbd, 0x33, 0xac, 0x3d, 0x58, 0xb7, 0x5b,
+ 0xec, 0xe2, 0x40, 0x21, 0xc4, 0x53, 0x60, 0x17, 0x0e, 0x66, 0x8c, 0xb2, 0x58, 0xe7, 0xaa, 0x26,
+ 0xbb, 0xd8, 0x95, 0xb0, 0x3e, 0xeb, 0x33, 0x1a, 0x45, 0xd8, 0x97, 0x39, 0x5a, 0x9e, 0xdd, 0x51,
+ 0x08, 0x21, 0x95, 0x1b, 0xa9, 0x0d, 0x25, 0x95, 0x67, 0x52, 0x79, 0x26, 0x75, 0x59, 0x9d, 0xe4,
+ 0x79, 0xa9, 0x3c, 0x95, 0xda, 0x54, 0x52, 0x79, 0x4e, 0x2a, 0xcf, 0xa4, 0xb6, 0xcc, 0x59, 0x2d,
+ 0xd5, 0xfa, 0x4d, 0x05, 0xd6, 0xe7, 0x0b, 0x3f, 0x5d, 0xa6, 0x7e, 0x00, 0x1d, 0x4f, 0xde, 0x57,
+ 0xe1, 0x4d, 0x0e, 0x16, 0x6e, 0xd2, 0x6e, 0x7b, 0xb9, 0x67, 0xfc, 0x21, 0x74, 0x43, 0xe5, 0xe0,
+ 0xf4, 0x69, 0xd6, 0xb2, 0x7b, 0xc9, 0xfb, 0xde, 0xee, 0x84, 0x39, 0xc8, 0xf2, 0x01, 0x7d, 0xc9,
+ 0x08, 0xc7, 0x87, 0x9c, 0x61, 0x37, 0xb8, 0x89, 0x06, 0x04, 0x41, 0x5d, 0x56, 0x2b, 0x35, 0x59,
+ 0x5f, 0xcb, 0xb5, 0xf5, 0x26, 0xac, 0x16, 0xa4, 0x68, 0x5b, 0x57, 0xa0, 0x36, 0xc5, 0xa1, 0xe4,
+ 0xde, 0xb5, 0xc5, 0xd2, 0x72, 0x61, 0x60, 0x63, 0xd7, 0xbf, 0x39, 0x6d, 0xb4, 0x88, 0x5a, 0x26,
+ 0x62, 0x03, 0x50, 0x5e, 0x84, 0x56, 0xc5, 0x68, 0x5d, 0xc9, 0x69, 0xfd, 0x1c, 0x06, 0xdb, 0x53,
+ 0x1a, 0xe3, 0x43, 0xee, 0x93, 0xf0, 0x26, 0x3a, 0xa6, 0x5f, 0xc2, 0xea, 0x0b, 0x3e, 0xfb, 0x52,
+ 0x30, 0x8b, 0xc9, 0x37, 0xf8, 0x86, 0xec, 0x63, 0xf4, 0xa5, 0xb1, 0x8f, 0xd1, 0x97, 0xa2, 0x59,
+ 0xf2, 0xe8, 0x34, 0x09, 0x42, 0x19, 0x0a, 0x5d, 0x5b, 0x43, 0xd6, 0x16, 0x74, 0x54, 0x0d, 0xfd,
+ 0x8c, 0xfa, 0xc9, 0x14, 0x97, 0xc6, 0xe0, 0x7d, 0x80, 0xc8, 0x65, 0x6e, 0x80, 0x39, 0x66, 0xea,
+ 0x0d, 0xb5, 0xec, 0x1c, 0xc6, 0xfa, 0x7d, 0x15, 0xd6, 0xd4, 0x48, 0xe4, 0x50, 0x4d, 0x02, 0x8c,
+ 0x09, 0x23, 0x68, 0x9e, 0xd2, 0x98, 0xe7, 0x18, 0xa6, 0xb0, 0x50, 0xd1, 0x0f, 0x0d, 0x37, 0xb1,
+ 0x2c, 0xcc, 0x29, 0x6a, 0x57, 0xcf, 0x29, 0x16, 0x26, 0x11, 0xf5, 0xc5, 0x49, 0x84, 0x88, 0x36,
+ 0x43, 0x44, 0x54, 0x8c, 0xb7, 0xec, 0x96, 0xc6, 0xec, 0xfb, 0xe8, 0x21, 0xf4, 0x27, 0x42, 0x4b,
+ 0xe7, 0x94, 0xd2, 0x33, 0x27, 0x72, 0xf9, 0xa9, 0x0c, 0xf5, 0x96, 0xdd, 0x95, 0xe8, 0x3d, 0x4a,
+ 0xcf, 0x0e, 0x5c, 0x7e, 0x8a, 0x3e, 0x86, 0x9e, 0x2e, 0x03, 0x03, 0xe9, 0xa2, 0x58, 0x7f, 0xfc,
+ 0x74, 0x14, 0xe5, 0xbd, 0x67, 0x77, 0xcf, 0x72, 0x50, 0x6c, 0xdd, 0x86, 0x5b, 0x3b, 0x38, 0xe6,
+ 0x8c, 0xce, 0x8a, 0x8e, 0xb1, 0x7e, 0x00, 0xb0, 0x1f, 0x72, 0xcc, 0x4e, 0x5c, 0x0f, 0xc7, 0xe8,
+ 0xdd, 0x3c, 0xa4, 0x8b, 0xa3, 0x95, 0xb1, 0x9a, 0x48, 0xa5, 0x1b, 0x76, 0x8e, 0xc6, 0x1a, 0x43,
+ 0xc3, 0xa6, 0x89, 0x48, 0x47, 0xaf, 0x9b, 0x95, 0x3e, 0xd7, 0xd1, 0xe7, 0x24, 0xd2, 0xd6, 0x7b,
+ 0xd6, 0x9e, 0x69, 0x61, 0x33, 0x76, 0xfa, 0x8a, 0xc6, 0xd0, 0x22, 0x06, 0xa7, 0xb3, 0xca, 0xa2,
+ 0xe8, 0x8c, 0xc4, 0xfa, 0x19, 0xac, 0x2a, 0x4e, 0x8a, 0xb3, 0x61, 0xf3, 0x3a, 0x34, 0x98, 0x51,
+ 0xa3, 0x92, 0x8d, 0xa2, 0x34, 0x91, 0xde, 0x43, 0xf7, 0x84, 0x30, 0x8f, 0xe1, 0x40, 0xf4, 0x1c,
+ 0x55, 0x79, 0x65, 0x19, 0x42, 0x78, 0x4b, 0xf4, 0xdb, 0x99, 0x99, 0xc6, 0x5b, 0xab, 0x30, 0x10,
+ 0x1b, 0x05, 0x89, 0xd6, 0x2f, 0x60, 0xf5, 0x79, 0x38, 0x25, 0x21, 0xde, 0x3e, 0x38, 0x7a, 0x86,
+ 0xd3, 0xac, 0x80, 0xa0, 0x2e, 0xaa, 0x27, 0xa9, 0x46, 0xd3, 0x96, 0x6b, 0x11, 0x26, 0xe1, 0xb1,
+ 0xe3, 0x45, 0x49, 0xac, 0x27, 0x43, 0x8d, 0xf0, 0x78, 0x3b, 0x4a, 0x62, 0x91, 0xe6, 0xc5, 0x67,
+ 0x9e, 0x86, 0xd3, 0x99, 0x8c, 0x95, 0xa6, 0xbd, 0xec, 0x45, 0xc9, 0xf3, 0x70, 0x3a, 0xb3, 0xfe,
+ 0x4f, 0xf6, 0xc2, 0x18, 0xfb, 0xb6, 0x1b, 0xfa, 0x34, 0xd8, 0xc1, 0xe7, 0x39, 0x09, 0x69, 0xdf,
+ 0x65, 0x72, 0xc2, 0xb7, 0x15, 0xe8, 0x3c, 0x99, 0xe0, 0x90, 0xef, 0x60, 0xee, 0x92, 0xa9, 0xec,
+ 0xad, 0xce, 0x31, 0x8b, 0x09, 0x0d, 0xf5, 0xc3, 0x37, 0xa0, 0x68, 0x8d, 0x49, 0x48, 0xb8, 0xe3,
+ 0xbb, 0x38, 0xa0, 0xa1, 0xf6, 0x02, 0x08, 0xd4, 0x8e, 0xc4, 0xa0, 0x37, 0xa1, 0xaf, 0x26, 0x77,
+ 0xce, 0xa9, 0x1b, 0xfa, 0x53, 0x11, 0x72, 0x6a, 0x92, 0xd1, 0x53, 0xe8, 0x3d, 0x8d, 0x45, 0x6f,
+ 0xc1, 0x8a, 0x0e, 0x88, 0x8c, 0xb2, 0x2e, 0x29, 0xfb, 0x1a, 0x5f, 0x20, 0x4d, 0xa2, 0x88, 0x32,
+ 0x1e, 0x3b, 0x31, 0xf6, 0x3c, 0x1a, 0x44, 0xba, 0x31, 0xe9, 0x1b, 0xfc, 0xa1, 0x42, 0x5b, 0x13,
+ 0x58, 0x7d, 0x2a, 0xec, 0xd4, 0x96, 0x64, 0x17, 0xdc, 0x0b, 0x70, 0xe0, 0x1c, 0x4f, 0xa9, 0x77,
+ 0xe6, 0x88, 0x34, 0xa5, 0x3d, 0x2c, 0x4a, 0x9f, 0x2d, 0x81, 0x3c, 0x24, 0xdf, 0xc8, 0x1e, 0x5c,
+ 0x50, 0x9d, 0x52, 0x1e, 0x4d, 0x93, 0x89, 0x13, 0x31, 0x7a, 0x8c, 0xb5, 0x89, 0xfd, 0x00, 0x07,
+ 0x7b, 0x0a, 0x7f, 0x20, 0xd0, 0xd6, 0x5f, 0x2a, 0xb0, 0x56, 0x94, 0xa4, 0x93, 0xee, 0x26, 0xac,
+ 0x15, 0x45, 0xe9, 0x0f, 0xb1, 0x2a, 0xf4, 0x06, 0x79, 0x81, 0xea, 0x93, 0xfc, 0x21, 0x74, 0xe5,
+ 0x38, 0xd7, 0xf1, 0x15, 0xa7, 0x62, 0xf9, 0x91, 0xbf, 0x17, 0xbb, 0xe3, 0xe6, 0x6f, 0xe9, 0x63,
+ 0xb8, 0xa3, 0xcd, 0x77, 0x16, 0xd5, 0x56, 0x0f, 0x62, 0x5d, 0x13, 0x3c, 0x9b, 0xd3, 0xfe, 0x0b,
+ 0x18, 0x66, 0xa8, 0xad, 0x99, 0x44, 0x1a, 0x5f, 0xbd, 0x0b, 0xab, 0x73, 0xc6, 0x3e, 0xf1, 0x7d,
+ 0x26, 0x03, 0xb4, 0x6e, 0x97, 0x6d, 0x59, 0x8f, 0xe1, 0xf6, 0x21, 0xe6, 0xca, 0x1b, 0x2e, 0xd7,
+ 0x3d, 0x81, 0x62, 0xb6, 0x02, 0xb5, 0x43, 0xec, 0x49, 0xe3, 0x6b, 0xb6, 0x58, 0x8a, 0x07, 0x78,
+ 0x14, 0x63, 0x4f, 0x5a, 0x59, 0xb3, 0xe5, 0xda, 0xfa, 0x73, 0x05, 0x96, 0x75, 0x9a, 0x14, 0xa9,
+ 0xde, 0x67, 0xe4, 0x1c, 0x33, 0xfd, 0xf4, 0x34, 0x84, 0xde, 0x80, 0x9e, 0x5a, 0x39, 0x34, 0xe2,
+ 0x84, 0xa6, 0xc9, 0xb7, 0xab, 0xb0, 0xcf, 0x15, 0x52, 0x4e, 0xea, 0xe4, 0x20, 0x4a, 0xf7, 0x7c,
+ 0x1a, 0x92, 0xe3, 0xb6, 0x58, 0x64, 0x06, 0x99, 0x6c, 0x5b, 0xb6, 0x86, 0xc4, 0x53, 0x37, 0xfc,
+ 0x96, 0x24, 0x3f, 0x03, 0x8a, 0xa7, 0x1e, 0xd0, 0x24, 0xe4, 0x4e, 0x44, 0x49, 0xc8, 0x75, 0x76,
+ 0x05, 0x89, 0x3a, 0x10, 0x18, 0xeb, 0xd7, 0x15, 0x68, 0xa8, 0x69, 0xb5, 0xe8, 0x32, 0xd3, 0x6f,
+ 0x5c, 0x95, 0xc8, 0x7a, 0x41, 0xca, 0x52, 0xdf, 0x35, 0xb9, 0x16, 0x71, 0x7c, 0x1e, 0xa8, 0x4c,
+ 0xad, 0x55, 0x3b, 0x0f, 0x64, 0x8a, 0x7e, 0x03, 0x7a, 0xd9, 0xa7, 0x52, 0xee, 0x2b, 0x15, 0xbb,
+ 0x29, 0x56, 0x92, 0x5d, 0xaa, 0xa9, 0xf5, 0x53, 0xd1, 0x5c, 0xa7, 0x93, 0xda, 0x15, 0xa8, 0x25,
+ 0xa9, 0x32, 0x62, 0x29, 0x30, 0x93, 0xf4, 0x23, 0x2b, 0x96, 0xe8, 0x21, 0xf4, 0x5c, 0xdf, 0x27,
+ 0xe2, 0xb8, 0x3b, 0x7d, 0x4a, 0xfc, 0x34, 0x48, 0x8b, 0x58, 0xeb, 0x6f, 0x15, 0xe8, 0x6f, 0xd3,
+ 0x68, 0xf6, 0x43, 0x32, 0xc5, 0xb9, 0x0c, 0x22, 0x95, 0xd4, 0xdf, 0x58, 0xb1, 0x16, 0x75, 0xe3,
+ 0x09, 0x99, 0x62, 0x15, 0x5a, 0xea, 0x66, 0x9b, 0x02, 0x21, 0xc3, 0xca, 0x6c, 0xa6, 0x03, 0xb0,
+ 0xae, 0xda, 0x7c, 0x46, 0x7d, 0x59, 0x21, 0xfb, 0x84, 0x39, 0xe9, 0xb8, 0xab, 0x6b, 0x2f, 0xfb,
+ 0x84, 0xc9, 0x2d, 0x6d, 0xc8, 0x92, 0x9c, 0xb8, 0xe6, 0x0d, 0x69, 0x28, 0x8c, 0x30, 0x64, 0x1d,
+ 0x1a, 0xf4, 0xe4, 0x24, 0xc6, 0x5c, 0xd6, 0xb2, 0x35, 0x5b, 0x43, 0x69, 0x9a, 0x6b, 0xe6, 0xd2,
+ 0xdc, 0x2d, 0x58, 0x95, 0xb3, 0xfd, 0x17, 0xcc, 0xf5, 0x48, 0x38, 0x31, 0xa9, 0x78, 0x0d, 0xd0,
+ 0x21, 0xa7, 0x51, 0x11, 0xfb, 0xe8, 0x77, 0x2b, 0x3a, 0x27, 0xea, 0x46, 0x17, 0x3d, 0x85, 0xfe,
+ 0xdc, 0x1f, 0x27, 0x48, 0x4f, 0x3e, 0xca, 0xff, 0x4f, 0x19, 0xad, 0x8f, 0xd5, 0x1f, 0x31, 0x63,
+ 0xf3, 0x47, 0xcc, 0x78, 0x37, 0x88, 0xf8, 0x0c, 0xed, 0x42, 0xaf, 0xf8, 0x17, 0x03, 0xba, 0x6b,
+ 0x0a, 0x85, 0x92, 0x3f, 0x1e, 0x2e, 0x65, 0xf3, 0x14, 0xfa, 0x73, 0xff, 0x36, 0x18, 0x7d, 0xca,
+ 0xff, 0x84, 0xb8, 0x94, 0xd1, 0x63, 0x68, 0xe7, 0xfe, 0x5e, 0x40, 0x43, 0xc5, 0x64, 0xf1, 0x1f,
+ 0x87, 0x4b, 0x19, 0x6c, 0x43, 0xb7, 0x30, 0xf1, 0x47, 0x23, 0x6d, 0x4f, 0xc9, 0xdf, 0x00, 0x97,
+ 0x32, 0xd9, 0x82, 0x76, 0x6e, 0xf0, 0x6e, 0xb4, 0x58, 0x9c, 0xee, 0x8f, 0xee, 0x94, 0xec, 0xe8,
+ 0xd4, 0xbb, 0x07, 0xdd, 0xc2, 0x98, 0xdc, 0x28, 0x52, 0x36, 0xa2, 0x1f, 0xdd, 0x2d, 0xdd, 0xd3,
+ 0x9c, 0x9e, 0x42, 0x7f, 0x6e, 0x68, 0x6e, 0x9c, 0x5b, 0x3e, 0x4b, 0xbf, 0xd4, 0xac, 0xcf, 0xe5,
+ 0x65, 0xe7, 0x7a, 0xa2, 0xdc, 0x65, 0x2f, 0x8e, 0xc8, 0x47, 0xf7, 0xca, 0x37, 0xb5, 0x56, 0xbb,
+ 0xd0, 0x2b, 0x4e, 0xc7, 0x0d, 0xb3, 0xd2, 0x99, 0xf9, 0xd5, 0x2f, 0xa7, 0x30, 0x28, 0xcf, 0x5e,
+ 0x4e, 0xd9, 0xfc, 0xfc, 0x52, 0x46, 0x4f, 0x00, 0x74, 0x07, 0xe4, 0x93, 0x30, 0xbd, 0xb2, 0x85,
+ 0xce, 0x2b, 0xbd, 0xb2, 0x92, 0x6e, 0xe9, 0x31, 0x80, 0x6a, 0x5c, 0x7c, 0x9a, 0x70, 0x74, 0xdb,
+ 0xa8, 0x31, 0xd7, 0x2d, 0x8d, 0x86, 0x8b, 0x1b, 0x0b, 0x0c, 0x30, 0x63, 0xd7, 0x61, 0xf0, 0x19,
+ 0x40, 0xd6, 0x10, 0x19, 0x06, 0x0b, 0x2d, 0xd2, 0x15, 0x3e, 0xe8, 0xe4, 0xdb, 0x1f, 0xa4, 0x6d,
+ 0x2d, 0x69, 0x89, 0xae, 0x60, 0xd1, 0x9f, 0x2b, 0x6f, 0x8b, 0x8f, 0x6d, 0xbe, 0xea, 0x1d, 0x2d,
+ 0x94, 0xb8, 0xe8, 0x43, 0xe8, 0xe4, 0xeb, 0x5a, 0xa3, 0x45, 0x49, 0xad, 0x3b, 0x2a, 0xd4, 0xb6,
+ 0xe8, 0x31, 0xf4, 0x8a, 0x55, 0x2b, 0xca, 0xc5, 0xc5, 0x42, 0x2d, 0x3b, 0xd2, 0x13, 0x9b, 0x1c,
+ 0xf9, 0xfb, 0x00, 0x59, 0x75, 0x6b, 0xdc, 0xb7, 0x50, 0xef, 0xce, 0x49, 0x7d, 0x02, 0x9d, 0x7c,
+ 0x26, 0x36, 0xea, 0x96, 0x64, 0xe7, 0xab, 0xb2, 0x56, 0x2e, 0x6b, 0x9b, 0xc7, 0xb7, 0x98, 0xc8,
+ 0xaf, 0xca, 0x5a, 0x85, 0xae, 0xcf, 0x24, 0x8b, 0xb2, 0x56, 0xf0, 0xaa, 0x5c, 0x5e, 0x6c, 0x91,
+ 0x8c, 0xfb, 0x4a, 0x1b, 0xa7, 0xab, 0x1e, 0x51, 0xbe, 0x1b, 0x30, 0xfe, 0x28, 0xe9, 0x10, 0xbe,
+ 0x27, 0xa8, 0xf3, 0x15, 0x7f, 0x2e, 0xa8, 0x4b, 0x1a, 0x81, 0x4b, 0x19, 0xed, 0x41, 0xff, 0xa9,
+ 0x29, 0xe6, 0x74, 0xa1, 0xa9, 0xd5, 0x29, 0x29, 0xac, 0x47, 0xa3, 0xb2, 0x2d, 0x1d, 0x59, 0x9f,
+ 0xc3, 0x60, 0xa1, 0xc8, 0x44, 0xf7, 0xd3, 0xc1, 0x62, 0x69, 0xf5, 0x79, 0xa9, 0x5a, 0xfb, 0xb0,
+ 0x32, 0x5f, 0x63, 0xa2, 0x57, 0xf4, 0xa5, 0x97, 0xd7, 0x9e, 0x97, 0xb2, 0xfa, 0x18, 0x9a, 0xa6,
+ 0xa6, 0x41, 0x7a, 0x80, 0x3b, 0x57, 0xe3, 0x5c, 0x76, 0x74, 0xab, 0xf3, 0xed, 0x77, 0xf7, 0x2b,
+ 0x7f, 0xff, 0xee, 0x7e, 0xe5, 0x9f, 0xdf, 0xdd, 0xaf, 0x1c, 0x37, 0xe4, 0xee, 0xfb, 0xff, 0x0e,
+ 0x00, 0x00, 0xff, 0xff, 0x8d, 0x89, 0xaa, 0x73, 0xc8, 0x21, 0x00, 0x00,
}
diff --git a/protocols/grpc/agent.proto b/protocols/grpc/agent.proto
index 348d792..b0fab6d 100644
--- a/protocols/grpc/agent.proto
+++ b/protocols/grpc/agent.proto
@@ -316,6 +316,7 @@ message UpdateInterfaceRequest {
message UpdateRoutesRequest {
Routes routes = 1;
+ bool increment = 2;
}
message ListInterfacesRequest {
--
2.14.3 (Apple Git-98)

971
agent/patches/0003-kata-agent-add-kata-ipvs-command.patch Normal file
View File

@ -0,0 +1,971 @@
From 0f24d3d433e93a7309b9bac59e4a15e722391490 Mon Sep 17 00:00:00 2001
From: xiadanni1 <xiadanni1@huawei.com>
Date: Tue, 18 Aug 2020 06:01:55 +0800
Subject: [PATCH 03/16] kata-agent: add kata-ipvs command
reason: add kata-ipvs command to update IPVS rules
in VM.
Signed-off-by: xiadanni1 <xiadanni1@huawei.com>
---
agent.go | 6 +
grpc.go | 4 +
ipvsadm.go | 130 +++++++++
protocols/grpc/agent.pb.go | 663 ++++++++++++++++++++++++++++++++-------------
protocols/grpc/agent.proto | 11 +
5 files changed, 633 insertions(+), 181 deletions(-)
create mode 100644 ipvsadm.go
diff --git a/agent.go b/agent.go
index c1cac08..c161e93 100644
--- a/agent.go
+++ b/agent.go
@@ -118,6 +118,11 @@ type sandboxStorage struct {
refCount int
}
+type ipvsAdm struct {
+ ipvsLock sync.Mutex
+ ipvsRuleCnt uint64
+}
+
type sandbox struct {
sync.RWMutex
ctx context.Context
@@ -144,6 +149,7 @@ type sandbox struct {
sandboxPidNs bool
storages map[string]*sandboxStorage
stopServer chan struct{}
+ ipvsadm ipvsAdm
}
var agentFields = logrus.Fields{
diff --git a/grpc.go b/grpc.go
index 8fe8217..de2cae7 100644
--- a/grpc.go
+++ b/grpc.go
@@ -1567,6 +1567,10 @@ func (a *agentGRPC) ListRoutes(ctx context.Context, req *pb.ListRoutesRequest) (
return a.sandbox.listRoutes(nil)
}
+func (a *agentGRPC) UpdateIPVSRule(ctx context.Context, req *pb.UpdateIPVSRequest) (*pb.IPVSResponse, error) {
+ return a.sandbox.updateIPVSRule(req)
+}
+
func (a *agentGRPC) OnlineCPUMem(ctx context.Context, req *pb.OnlineCPUMemRequest) (*gpb.Empty, error) {
if !req.Wait {
go a.onlineCPUMem(req)
diff --git a/ipvsadm.go b/ipvsadm.go
new file mode 100644
index 0000000..48eb19f
--- /dev/null
+++ b/ipvsadm.go
@@ -0,0 +1,130 @@
+// Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+// Description: IPVS related common functions
+// Author: xiadanni
+// Create: 2020-08-01
+
+package main
+
+import (
+ "os/exec"
+ "strconv"
+ "strings"
+
+ "github.com/kata-containers/agent/protocols/grpc"
+ "google.golang.org/grpc/codes"
+ grpcStatus "google.golang.org/grpc/status"
+)
+
+var (
+ errNoIPVSRules = grpcStatus.Errorf(codes.InvalidArgument, "IPVS rule is nil")
+ errRuleExist = grpcStatus.Errorf(codes.InvalidArgument, "failed to restore IPVS rules: exist some rules in system, should clear first")
+ errInvalidIPVSRule = grpcStatus.Errorf(codes.InvalidArgument, "invalid IPVS rule, please check")
+ errIPVSRuleExceed = grpcStatus.Errorf(codes.InvalidArgument, "rules exceed limit, should clear first")
+)
+
+const (
+ validHeadLength = 7
+ ruleLimitMax = 20000
+ restoreHead = "restore"
+ conntrackNormalInfo = "have been deleted"
+ waitError = "no child processes"
+)
+
+func (s *sandbox) updateIPVSRule(ipvsRule *grpc.UpdateIPVSRequest) (resultingIpvs *grpc.IPVSResponse, err error) {
+ var (
+ restoreRuleCnt string
+ restoreRule string
+ operation string
+ restoreRuleCntUint uint64
+ cmd *exec.Cmd
+ )
+
+ if ipvsRule == nil || ipvsRule.IPVSReq == "" {
+ return nil, errNoIPVSRules
+ }
+
+ s.ipvsadm.ipvsLock.Lock()
+ defer s.ipvsadm.ipvsLock.Unlock()
+
+ rule := ipvsRule.IPVSReq
+ if len(rule) < validHeadLength {
+ return nil, errInvalidIPVSRule
+ }
+
+ if ruleHead := rule[:validHeadLength]; ruleHead == restoreHead {
+ if s.ipvsadm.ipvsRuleCnt != 0 {
+ return nil, errRuleExist
+ }
+ operation = restoreHead
+
+ restoreItem := strings.Split(rule, "|")
+ if len(restoreItem) != 3 {
+ return nil, errInvalidIPVSRule
+ }
+ restoreRuleCnt = restoreItem[1]
+ if restoreRuleCntUint, err = strconv.ParseUint(restoreRuleCnt, 10, 64); err != nil {
+ return nil, errInvalidIPVSRule
+ }
+ restoreRule = restoreItem[2]
+
+ cmd = exec.Command("/sbin/ipvsadm", "--restore")
+ cmd.Stdin = strings.NewReader(restoreRule)
+ } else {
+ item := strings.Fields(strings.TrimSpace(rule))
+ if len(item) < 2 {
+ return nil, errInvalidIPVSRule
+ }
+ if item[0] != "ipvsadm" && item[0] != "conntrack" {
+ return nil, errInvalidIPVSRule
+ }
+ operation = item[1]
+ if s.ipvsadm.ipvsRuleCnt >= ruleLimitMax {
+ if operation == "--add-service" || operation == "-A" || operation == "--add-server" || operation == "-a" {
+ return nil, errIPVSRuleExceed
+ }
+ }
+
+ restoreRuleCntUint = 0
+
+ cmd = exec.Command("/sbin/" + item[0])
+ for i := 1; i < len(item); i++ {
+ cmd.Args = append(cmd.Args, item[i])
+ }
+ }
+
+ bytes, err := cmd.CombinedOutput()
+
+ if err != nil && !strings.Contains(err.Error(), waitError) && !strings.Contains(string(bytes), conntrackNormalInfo) {
+ return nil, grpcStatus.Errorf(codes.Internal,
+ "exec IPVS command failed, stderr: %v, err: %v", string(bytes), err)
+ }
+
+ s.sumRuleCount(operation, restoreRuleCntUint)
+
+ rsp := &grpc.IPVSResponse{
+ IPVSRes: string(bytes),
+ }
+ return rsp, nil
+}
+
+func (s *sandbox) sumRuleCount(operation string, restoreRuleCntUint uint64) {
+ if operation == restoreHead {
+ s.ipvsadm.ipvsRuleCnt = restoreRuleCntUint
+ return
+ }
+ if operation == "--add-service" || operation == "-A" || operation == "--add-server" || operation == "-a" {
+ s.ipvsadm.ipvsRuleCnt++
+ return
+ }
+ if operation == "--delete-service" || operation == "-D" || operation == "--delete-server" || operation == "-d" {
+ if s.ipvsadm.ipvsRuleCnt > 0 {
+ s.ipvsadm.ipvsRuleCnt--
+ }
+ return
+ }
+ if operation == "--clear" || operation == "-C" {
+ s.ipvsadm.ipvsRuleCnt = 0
+ return
+ }
+}
diff --git a/protocols/grpc/agent.pb.go b/protocols/grpc/agent.pb.go
index 1b887e5..04d0ee5 100644
--- a/protocols/grpc/agent.pb.go
+++ b/protocols/grpc/agent.pb.go
@@ -63,6 +63,8 @@
CopyFileRequest
StartTracingRequest
StopTracingRequest
+ UpdateIPVSRequest
+ IPVSResponse
CheckRequest
HealthCheckResponse
VersionCheckResponse
@@ -1842,6 +1844,40 @@ func (m *StopTracingRequest) String() string { return proto.CompactTe
func (*StopTracingRequest) ProtoMessage() {}
func (*StopTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{52} }
+type UpdateIPVSRequest struct {
+ // IPVS_req is the IPVS rule message needed to update
+ IPVSReq string `protobuf:"bytes,1,opt,name=IPVS_req,json=IPVSReq,proto3" json:"IPVS_req,omitempty"`
+}
+
+func (m *UpdateIPVSRequest) Reset() { *m = UpdateIPVSRequest{} }
+func (m *UpdateIPVSRequest) String() string { return proto.CompactTextString(m) }
+func (*UpdateIPVSRequest) ProtoMessage() {}
+func (*UpdateIPVSRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{53} }
+
+func (m *UpdateIPVSRequest) GetIPVSReq() string {
+ if m != nil {
+ return m.IPVSReq
+ }
+ return ""
+}
+
+type IPVSResponse struct {
+ // IPVS_res is the response of IPVS updating
+ IPVSRes string `protobuf:"bytes,1,opt,name=IPVS_res,json=IPVSRes,proto3" json:"IPVS_res,omitempty"`
+}
+
+func (m *IPVSResponse) Reset() { *m = IPVSResponse{} }
+func (m *IPVSResponse) String() string { return proto.CompactTextString(m) }
+func (*IPVSResponse) ProtoMessage() {}
+func (*IPVSResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{54} }
+
+func (m *IPVSResponse) GetIPVSRes() string {
+ if m != nil {
+ return m.IPVSRes
+ }
+ return ""
+}
+
func init() {
proto.RegisterType((*CreateContainerRequest)(nil), "grpc.CreateContainerRequest")
proto.RegisterType((*StartContainerRequest)(nil), "grpc.StartContainerRequest")
@@ -1896,6 +1932,8 @@ func init() {
proto.RegisterType((*CopyFileRequest)(nil), "grpc.CopyFileRequest")
proto.RegisterType((*StartTracingRequest)(nil), "grpc.StartTracingRequest")
proto.RegisterType((*StopTracingRequest)(nil), "grpc.StopTracingRequest")
+ proto.RegisterType((*UpdateIPVSRequest)(nil), "grpc.UpdateIPVSRequest")
+ proto.RegisterType((*IPVSResponse)(nil), "grpc.IPVSResponse")
}
// Reference imports to suppress errors if they are not otherwise used.
@@ -1938,6 +1976,7 @@ type AgentServiceClient interface {
UpdateRoutes(ctx context.Context, in *UpdateRoutesRequest, opts ...grpc1.CallOption) (*Routes, error)
ListInterfaces(ctx context.Context, in *ListInterfacesRequest, opts ...grpc1.CallOption) (*Interfaces, error)
ListRoutes(ctx context.Context, in *ListRoutesRequest, opts ...grpc1.CallOption) (*Routes, error)
+ UpdateIPVSRule(ctx context.Context, in *UpdateIPVSRequest, opts ...grpc1.CallOption) (*IPVSResponse, error)
// tracing
StartTracing(ctx context.Context, in *StartTracingRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error)
StopTracing(ctx context.Context, in *StopTracingRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error)
@@ -2140,6 +2179,15 @@ func (c *agentServiceClient) ListRoutes(ctx context.Context, in *ListRoutesReque
return out, nil
}
+func (c *agentServiceClient) UpdateIPVSRule(ctx context.Context, in *UpdateIPVSRequest, opts ...grpc1.CallOption) (*IPVSResponse, error) {
+ out := new(IPVSResponse)
+ err := grpc1.Invoke(ctx, "/grpc.AgentService/UpdateIPVSRule", in, out, c.cc, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
func (c *agentServiceClient) StartTracing(ctx context.Context, in *StartTracingRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error) {
out := new(google_protobuf2.Empty)
err := grpc1.Invoke(ctx, "/grpc.AgentService/StartTracing", in, out, c.cc, opts...)
@@ -2262,6 +2310,7 @@ type AgentServiceServer interface {
UpdateRoutes(context.Context, *UpdateRoutesRequest) (*Routes, error)
ListInterfaces(context.Context, *ListInterfacesRequest) (*Interfaces, error)
ListRoutes(context.Context, *ListRoutesRequest) (*Routes, error)
+ UpdateIPVSRule(context.Context, *UpdateIPVSRequest) (*IPVSResponse, error)
// tracing
StartTracing(context.Context, *StartTracingRequest) (*google_protobuf2.Empty, error)
StopTracing(context.Context, *StopTracingRequest) (*google_protobuf2.Empty, error)
@@ -2640,6 +2689,24 @@ func _AgentService_ListRoutes_Handler(srv interface{}, ctx context.Context, dec
return interceptor(ctx, in, info, handler)
}
+func _AgentService_UpdateIPVSRule_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) {
+ in := new(UpdateIPVSRequest)
+ if err := dec(in); err != nil {
+ return nil, err
+ }
+ if interceptor == nil {
+ return srv.(AgentServiceServer).UpdateIPVSRule(ctx, in)
+ }
+ info := &grpc1.UnaryServerInfo{
+ Server: srv,
+ FullMethod: "/grpc.AgentService/UpdateIPVSRule",
+ }
+ handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+ return srv.(AgentServiceServer).UpdateIPVSRule(ctx, req.(*UpdateIPVSRequest))
+ }
+ return interceptor(ctx, in, info, handler)
+}
+
func _AgentService_StartTracing_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) {
in := new(StartTracingRequest)
if err := dec(in); err != nil {
@@ -2904,6 +2971,10 @@ var _AgentService_serviceDesc = grpc1.ServiceDesc{
MethodName: "ListRoutes",
Handler: _AgentService_ListRoutes_Handler,
},
+ {
+ MethodName: "UpdateIPVSRule",
+ Handler: _AgentService_UpdateIPVSRule_Handler,
+ },
{
MethodName: "StartTracing",
Handler: _AgentService_StartTracing_Handler,
@@ -5088,6 +5159,54 @@ func (m *StopTracingRequest) MarshalTo(dAtA []byte) (int, error) {
return i, nil
}
+func (m *UpdateIPVSRequest) Marshal() (dAtA []byte, err error) {
+ size := m.Size()
+ dAtA = make([]byte, size)
+ n, err := m.MarshalTo(dAtA)
+ if err != nil {
+ return nil, err
+ }
+ return dAtA[:n], nil
+}
+
+func (m *UpdateIPVSRequest) MarshalTo(dAtA []byte) (int, error) {
+ var i int
+ _ = i
+ var l int
+ _ = l
+ if len(m.IPVSReq) > 0 {
+ dAtA[i] = 0xa
+ i++
+ i = encodeVarintAgent(dAtA, i, uint64(len(m.IPVSReq)))
+ i += copy(dAtA[i:], m.IPVSReq)
+ }
+ return i, nil
+}
+
+func (m *IPVSResponse) Marshal() (dAtA []byte, err error) {
+ size := m.Size()
+ dAtA = make([]byte, size)
+ n, err := m.MarshalTo(dAtA)
+ if err != nil {
+ return nil, err
+ }
+ return dAtA[:n], nil
+}
+
+func (m *IPVSResponse) MarshalTo(dAtA []byte) (int, error) {
+ var i int
+ _ = i
+ var l int
+ _ = l
+ if len(m.IPVSRes) > 0 {
+ dAtA[i] = 0xa
+ i++
+ i = encodeVarintAgent(dAtA, i, uint64(len(m.IPVSRes)))
+ i += copy(dAtA[i:], m.IPVSRes)
+ }
+ return i, nil
+}
+
func encodeVarintAgent(dAtA []byte, offset int, v uint64) int {
for v >= 1<<7 {
dAtA[offset] = uint8(v&0x7f | 0x80)
@@ -6019,6 +6138,26 @@ func (m *StopTracingRequest) Size() (n int) {
return n
}
+func (m *UpdateIPVSRequest) Size() (n int) {
+ var l int
+ _ = l
+ l = len(m.IPVSReq)
+ if l > 0 {
+ n += 1 + l + sovAgent(uint64(l))
+ }
+ return n
+}
+
+func (m *IPVSResponse) Size() (n int) {
+ var l int
+ _ = l
+ l = len(m.IPVSRes)
+ if l > 0 {
+ n += 1 + l + sovAgent(uint64(l))
+ }
+ return n
+}
+
func sovAgent(x uint64) (n int) {
for {
n++
@@ -12785,6 +12924,164 @@ func (m *StopTracingRequest) Unmarshal(dAtA []byte) error {
}
return nil
}
+func (m *UpdateIPVSRequest) Unmarshal(dAtA []byte) error {
+ l := len(dAtA)
+ iNdEx := 0
+ for iNdEx < l {
+ preIndex := iNdEx
+ var wire uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowAgent
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ wire |= (uint64(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ fieldNum := int32(wire >> 3)
+ wireType := int(wire & 0x7)
+ if wireType == 4 {
+ return fmt.Errorf("proto: UpdateIPVSRequest: wiretype end group for non-group")
+ }
+ if fieldNum <= 0 {
+ return fmt.Errorf("proto: UpdateIPVSRequest: illegal tag %d (wire type %d)", fieldNum, wire)
+ }
+ switch fieldNum {
+ case 1:
+ if wireType != 2 {
+ return fmt.Errorf("proto: wrong wireType = %d for field IPVSReq", wireType)
+ }
+ var stringLen uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowAgent
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ stringLen |= (uint64(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ intStringLen := int(stringLen)
+ if intStringLen < 0 {
+ return ErrInvalidLengthAgent
+ }
+ postIndex := iNdEx + intStringLen
+ if postIndex > l {
+ return io.ErrUnexpectedEOF
+ }
+ m.IPVSReq = string(dAtA[iNdEx:postIndex])
+ iNdEx = postIndex
+ default:
+ iNdEx = preIndex
+ skippy, err := skipAgent(dAtA[iNdEx:])
+ if err != nil {
+ return err
+ }
+ if skippy < 0 {
+ return ErrInvalidLengthAgent
+ }
+ if (iNdEx + skippy) > l {
+ return io.ErrUnexpectedEOF
+ }
+ iNdEx += skippy
+ }
+ }
+
+ if iNdEx > l {
+ return io.ErrUnexpectedEOF
+ }
+ return nil
+}
+func (m *IPVSResponse) Unmarshal(dAtA []byte) error {
+ l := len(dAtA)
+ iNdEx := 0
+ for iNdEx < l {
+ preIndex := iNdEx
+ var wire uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowAgent
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ wire |= (uint64(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ fieldNum := int32(wire >> 3)
+ wireType := int(wire & 0x7)
+ if wireType == 4 {
+ return fmt.Errorf("proto: IPVSResponse: wiretype end group for non-group")
+ }
+ if fieldNum <= 0 {
+ return fmt.Errorf("proto: IPVSResponse: illegal tag %d (wire type %d)", fieldNum, wire)
+ }
+ switch fieldNum {
+ case 1:
+ if wireType != 2 {
+ return fmt.Errorf("proto: wrong wireType = %d for field IPVSRes", wireType)
+ }
+ var stringLen uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowAgent
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ stringLen |= (uint64(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ intStringLen := int(stringLen)
+ if intStringLen < 0 {
+ return ErrInvalidLengthAgent
+ }
+ postIndex := iNdEx + intStringLen
+ if postIndex > l {
+ return io.ErrUnexpectedEOF
+ }
+ m.IPVSRes = string(dAtA[iNdEx:postIndex])
+ iNdEx = postIndex
+ default:
+ iNdEx = preIndex
+ skippy, err := skipAgent(dAtA[iNdEx:])
+ if err != nil {
+ return err
+ }
+ if skippy < 0 {
+ return ErrInvalidLengthAgent
+ }
+ if (iNdEx + skippy) > l {
+ return io.ErrUnexpectedEOF
+ }
+ iNdEx += skippy
+ }
+ }
+
+ if iNdEx > l {
+ return io.ErrUnexpectedEOF
+ }
+ return nil
+}
func skipAgent(dAtA []byte) (n int, err error) {
l := len(dAtA)
iNdEx := 0
@@ -12893,185 +13190,189 @@ var (
func init() { proto.RegisterFile("agent.proto", fileDescriptorAgent) }
var fileDescriptorAgent = []byte{
- // 2876 bytes of a gzipped FileDescriptorProto
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0x4b, 0x6f, 0x1c, 0xc7,
- 0xd1, 0xd8, 0x07, 0x97, 0xbb, 0xb5, 0x2f, 0x6e, 0x93, 0xa2, 0x56, 0x2b, 0x59, 0x9f, 0x3c, 0xb6,
- 0x65, 0xfa, 0xf3, 0xe7, 0xa5, 0x2d, 0x1b, 0x9f, 0x5f, 0x70, 0x04, 0xf1, 0x11, 0x91, 0xb1, 0x15,
- 0x31, 0x43, 0x11, 0x4e, 0x10, 0x04, 0x83, 0xe1, 0x4c, 0x73, 0xd9, 0xe6, 0xce, 0xf4, 0xb8, 0xa7,
- 0x87, 0xe2, 0x3a, 0x40, 0x8e, 0xc9, 0x2d, 0x97, 0x00, 0xb9, 0xe5, 0x0f, 0x04, 0xb9, 0xe5, 0x98,
- 0x6b, 0x0e, 0x46, 0x4e, 0xf9, 0x05, 0x41, 0xe0, 0x9f, 0x90, 0x5f, 0x10, 0xf4, 0x6b, 0x1e, 0xbb,
- 0x43, 0x1a, 0x21, 0x08, 0xe4, 0xb2, 0xe8, 0xaa, 0xae, 0xae, 0x57, 0x77, 0xd5, 0x54, 0xd5, 0x42,
- 0xdb, 0x9d, 0xe0, 0x90, 0x8f, 0x23, 0x46, 0x39, 0x45, 0xf5, 0x09, 0x8b, 0xbc, 0x51, 0x8b, 0x7a,
- 0x44, 0x21, 0x46, 0xff, 0x3f, 0x21, 0xfc, 0x34, 0x39, 0x1e, 0x7b, 0x34, 0xd8, 0x3c, 0x73, 0xb9,
- 0xfb, 0x8e, 0x47, 0x43, 0xee, 0x92, 0x10, 0xb3, 0x78, 0x53, 0x1e, 0xdc, 0x8c, 0xce, 0x26, 0x9b,
- 0x7c, 0x16, 0xe1, 0x58, 0xfd, 0xea, 0x73, 0x77, 0x27, 0x94, 0x4e, 0xa6, 0x78, 0x53, 0x42, 0xc7,
- 0xc9, 0xc9, 0x26, 0x0e, 0x22, 0x3e, 0x53, 0x9b, 0xd6, 0x1f, 0xaa, 0xb0, 0xbe, 0xcd, 0xb0, 0xcb,
- 0xf1, 0xb6, 0xe1, 0x66, 0xe3, 0xaf, 0x13, 0x1c, 0x73, 0xf4, 0x2a, 0x74, 0x52, 0x09, 0x0e, 0xf1,
- 0x87, 0x95, 0x07, 0x95, 0x8d, 0x96, 0xdd, 0x4e, 0x71, 0xfb, 0x3e, 0xba, 0x0d, 0xcb, 0xf8, 0x02,
- 0x7b, 0x62, 0xb7, 0x2a, 0x77, 0x1b, 0x02, 0xdc, 0xf7, 0xd1, 0x7b, 0xd0, 0x8e, 0x39, 0x23, 0xe1,
- 0xc4, 0x49, 0x62, 0xcc, 0x86, 0xb5, 0x07, 0x95, 0x8d, 0xf6, 0xa3, 0x95, 0xb1, 0x30, 0x69, 0x7c,
- 0x28, 0x37, 0x8e, 0x62, 0xcc, 0x6c, 0x88, 0xd3, 0x35, 0x7a, 0x08, 0xcb, 0x3e, 0x3e, 0x27, 0x1e,
- 0x8e, 0x87, 0xf5, 0x07, 0xb5, 0x8d, 0xf6, 0xa3, 0x8e, 0x22, 0xdf, 0x91, 0x48, 0xdb, 0x6c, 0xa2,
- 0xb7, 0xa0, 0x19, 0x73, 0xca, 0xdc, 0x09, 0x8e, 0x87, 0x4b, 0x92, 0xb0, 0x6b, 0xf8, 0x4a, 0xac,
- 0x9d, 0x6e, 0xa3, 0x7b, 0x50, 0x7b, 0xbe, 0xbd, 0x3f, 0x6c, 0x48, 0xe9, 0xa0, 0xa9, 0x22, 0xec,
- 0xd9, 0x02, 0x8d, 0x5e, 0x83, 0x6e, 0xec, 0x86, 0xfe, 0x31, 0xbd, 0x70, 0x22, 0xe2, 0x87, 0xf1,
- 0x70, 0xf9, 0x41, 0x65, 0xa3, 0x69, 0x77, 0x34, 0xf2, 0x40, 0xe0, 0xac, 0x4f, 0xe0, 0xd6, 0x21,
- 0x77, 0x19, 0xbf, 0x86, 0x77, 0xac, 0x23, 0x58, 0xb7, 0x71, 0x40, 0xcf, 0xaf, 0xe5, 0xda, 0x21,
- 0x2c, 0x73, 0x12, 0x60, 0x9a, 0x70, 0xe9, 0xda, 0xae, 0x6d, 0x40, 0xeb, 0x4f, 0x15, 0x40, 0xbb,
- 0x17, 0xd8, 0x3b, 0x60, 0xd4, 0xc3, 0x71, 0xfc, 0x5f, 0xba, 0xae, 0x37, 0x61, 0x39, 0x52, 0x0a,
- 0x0c, 0xeb, 0x92, 0x5c, 0xdf, 0x82, 0xd1, 0xca, 0xec, 0x5a, 0x5f, 0xc1, 0xda, 0x21, 0x99, 0x84,
- 0xee, 0xf4, 0x06, 0xf5, 0x5d, 0x87, 0x46, 0x2c, 0x79, 0x4a, 0x55, 0xbb, 0xb6, 0x86, 0xac, 0x03,
- 0x40, 0x5f, 0xba, 0x84, 0xdf, 0x9c, 0x24, 0xeb, 0x1d, 0x58, 0x2d, 0x70, 0x8c, 0x23, 0x1a, 0xc6,
- 0x58, 0x2a, 0xc0, 0x5d, 0x9e, 0xc4, 0x92, 0xd9, 0x92, 0xad, 0x21, 0x0b, 0xc3, 0xda, 0x17, 0x24,
- 0x36, 0xe4, 0xf8, 0x3f, 0x51, 0x61, 0x1d, 0x1a, 0x27, 0x94, 0x05, 0x2e, 0x37, 0x1a, 0x28, 0x08,
- 0x21, 0xa8, 0xbb, 0x6c, 0x12, 0x0f, 0x6b, 0x0f, 0x6a, 0x1b, 0x2d, 0x5b, 0xae, 0xc5, 0xab, 0x9c,
- 0x13, 0xa3, 0xf5, 0x7a, 0x15, 0x3a, 0xda, 0xef, 0xce, 0x94, 0xc4, 0x5c, 0xca, 0xe9, 0xd8, 0x6d,
- 0x8d, 0x13, 0x67, 0x2c, 0x0a, 0xeb, 0x47, 0x91, 0x7f, 0xcd, 0x80, 0x7f, 0x04, 0x2d, 0x86, 0x63,
- 0x9a, 0x30, 0x11, 0xa6, 0x55, 0x79, 0xef, 0x6b, 0xea, 0xde, 0xbf, 0x20, 0x61, 0x72, 0x61, 0x9b,
- 0x3d, 0x3b, 0x23, 0xd3, 0x21, 0xc4, 0xe3, 0xeb, 0x84, 0xd0, 0x27, 0x70, 0xeb, 0xc0, 0x4d, 0xe2,
- 0xeb, 0xe8, 0x6a, 0x7d, 0x2a, 0xc2, 0x2f, 0x4e, 0x82, 0x6b, 0x1d, 0xfe, 0x63, 0x05, 0x9a, 0xdb,
- 0x51, 0x72, 0x14, 0xbb, 0x13, 0x8c, 0xfe, 0x07, 0xda, 0x9c, 0x72, 0x77, 0xea, 0x24, 0x02, 0x94,
- 0xe4, 0x75, 0x1b, 0x24, 0x4a, 0x11, 0x08, 0xb7, 0x63, 0xe6, 0x45, 0x89, 0xa6, 0xa8, 0x3e, 0xa8,
- 0x6d, 0xd4, 0xed, 0xb6, 0xc2, 0x29, 0x92, 0x31, 0xac, 0xca, 0x3d, 0x87, 0x84, 0xce, 0x19, 0x66,
- 0x21, 0x9e, 0x06, 0xd4, 0xc7, 0xf2, 0xfd, 0xd6, 0xed, 0x81, 0xdc, 0xda, 0x0f, 0x3f, 0x4f, 0x37,
- 0xd0, 0xff, 0xc2, 0x20, 0xa5, 0x17, 0x41, 0x29, 0xa9, 0xeb, 0x92, 0xba, 0xaf, 0xa9, 0x8f, 0x34,
- 0xda, 0xfa, 0x15, 0xf4, 0x5e, 0x9c, 0x32, 0xca, 0xf9, 0x94, 0x84, 0x93, 0x1d, 0x97, 0xbb, 0x22,
- 0x7b, 0x44, 0x98, 0x11, 0xea, 0xc7, 0x5a, 0x5b, 0x03, 0xa2, 0xb7, 0x61, 0xc0, 0x15, 0x2d, 0xf6,
- 0x1d, 0x43, 0x53, 0x95, 0x34, 0x2b, 0xe9, 0xc6, 0x81, 0x26, 0x7e, 0x03, 0x7a, 0x19, 0xb1, 0xc8,
- 0x3f, 0x5a, 0xdf, 0x6e, 0x8a, 0x7d, 0x41, 0x02, 0x6c, 0x9d, 0x4b, 0x5f, 0xc9, 0x4b, 0x46, 0x6f,
- 0x43, 0x2b, 0xf3, 0x43, 0x45, 0xbe, 0x90, 0x9e, 0x7a, 0x21, 0xc6, 0x9d, 0x76, 0x33, 0x75, 0xca,
- 0x67, 0xd0, 0xe7, 0xa9, 0xe2, 0x8e, 0xef, 0x72, 0xb7, 0xf8, 0xa8, 0x8a, 0x56, 0xd9, 0x3d, 0x5e,
- 0x80, 0xad, 0x4f, 0xa1, 0x75, 0x40, 0xfc, 0x58, 0x09, 0x1e, 0xc2, 0xb2, 0x97, 0x30, 0x86, 0x43,
- 0x6e, 0x4c, 0xd6, 0x20, 0x5a, 0x83, 0xa5, 0x29, 0x09, 0x08, 0xd7, 0x66, 0x2a, 0xc0, 0xa2, 0x00,
- 0xcf, 0x70, 0x40, 0xd9, 0x4c, 0x3a, 0x6c, 0x0d, 0x96, 0xf2, 0x97, 0xab, 0x00, 0x74, 0x17, 0x5a,
- 0x81, 0x7b, 0x91, 0x5e, 0xaa, 0xd8, 0x69, 0x06, 0xee, 0x85, 0x52, 0x7e, 0x08, 0xcb, 0x27, 0x2e,
- 0x99, 0x7a, 0x21, 0xd7, 0x5e, 0x31, 0x60, 0x26, 0xb0, 0x9e, 0x17, 0xf8, 0xd7, 0x2a, 0xb4, 0x95,
- 0x44, 0xa5, 0xf0, 0x1a, 0x2c, 0x79, 0xae, 0x77, 0x9a, 0x8a, 0x94, 0x00, 0x7a, 0x68, 0x14, 0xa9,
- 0xe6, 0x93, 0x70, 0xa6, 0xa9, 0x51, 0x6d, 0x13, 0x20, 0x7e, 0xe9, 0x46, 0x5a, 0xb7, 0xda, 0x25,
- 0xc4, 0x2d, 0x41, 0xa3, 0xd4, 0x7d, 0x1f, 0x3a, 0xea, 0xdd, 0xe9, 0x23, 0xf5, 0x4b, 0x8e, 0xb4,
- 0x15, 0x95, 0x3a, 0xf4, 0x1a, 0x74, 0x93, 0x18, 0x3b, 0xa7, 0x04, 0x33, 0x97, 0x79, 0xa7, 0xb3,
- 0xe1, 0x92, 0xfa, 0x46, 0x26, 0x31, 0xde, 0x33, 0x38, 0xf4, 0x08, 0x96, 0x44, 0xfa, 0x8b, 0x87,
- 0x0d, 0xf9, 0x39, 0xbe, 0x97, 0x67, 0x29, 0x4d, 0x1d, 0xcb, 0xdf, 0xdd, 0x90, 0xb3, 0x99, 0xad,
- 0x48, 0x47, 0x1f, 0x01, 0x64, 0x48, 0xb4, 0x02, 0xb5, 0x33, 0x3c, 0xd3, 0x71, 0x28, 0x96, 0xc2,
- 0x39, 0xe7, 0xee, 0x34, 0x31, 0x5e, 0x57, 0xc0, 0x27, 0xd5, 0x8f, 0x2a, 0x96, 0x07, 0xfd, 0xad,
- 0xe9, 0x19, 0xa1, 0xb9, 0xe3, 0x6b, 0xb0, 0x14, 0xb8, 0x5f, 0x51, 0x66, 0x3c, 0x29, 0x01, 0x89,
- 0x25, 0x21, 0x65, 0x86, 0x85, 0x04, 0x50, 0x0f, 0xaa, 0x34, 0x92, 0xfe, 0x6a, 0xd9, 0x55, 0x1a,
- 0x65, 0x82, 0xea, 0x39, 0x41, 0xd6, 0x3f, 0xea, 0x00, 0x99, 0x14, 0x64, 0xc3, 0x88, 0x50, 0x27,
- 0xc6, 0x4c, 0x94, 0x20, 0xce, 0xf1, 0x8c, 0xe3, 0xd8, 0x61, 0xd8, 0x4b, 0x58, 0x4c, 0xce, 0xc5,
- 0xfd, 0x09, 0xb3, 0x6f, 0x29, 0xb3, 0xe7, 0x74, 0xb3, 0x6f, 0x13, 0x7a, 0xa8, 0xce, 0x6d, 0x89,
- 0x63, 0xb6, 0x39, 0x85, 0xf6, 0xe1, 0x56, 0xc6, 0xd3, 0xcf, 0xb1, 0xab, 0x5e, 0xc5, 0x6e, 0x35,
- 0x65, 0xe7, 0x67, 0xac, 0x76, 0x61, 0x95, 0x50, 0xe7, 0xeb, 0x04, 0x27, 0x05, 0x46, 0xb5, 0xab,
- 0x18, 0x0d, 0x08, 0xfd, 0x89, 0x3c, 0x90, 0xb1, 0x39, 0x80, 0x3b, 0x39, 0x2b, 0x45, 0xb8, 0xe7,
- 0x98, 0xd5, 0xaf, 0x62, 0xb6, 0x9e, 0x6a, 0x25, 0xf2, 0x41, 0xc6, 0xf1, 0x47, 0xb0, 0x4e, 0xa8,
- 0xf3, 0xd2, 0x25, 0x7c, 0x9e, 0xdd, 0xd2, 0xf7, 0x18, 0x29, 0x3e, 0xba, 0x45, 0x5e, 0xca, 0xc8,
- 0x00, 0xb3, 0x49, 0xc1, 0xc8, 0xc6, 0xf7, 0x18, 0xf9, 0x4c, 0x1e, 0xc8, 0xd8, 0x3c, 0x81, 0x01,
- 0xa1, 0xf3, 0xda, 0x2c, 0x5f, 0xc5, 0xa4, 0x4f, 0x68, 0x51, 0x93, 0x2d, 0x18, 0xc4, 0xd8, 0xe3,
- 0x94, 0xe5, 0x1f, 0x41, 0xf3, 0x2a, 0x16, 0x2b, 0x9a, 0x3e, 0xe5, 0x61, 0xfd, 0x1c, 0x3a, 0x7b,
- 0xc9, 0x04, 0xf3, 0xe9, 0x71, 0x9a, 0x0c, 0x6e, 0x2c, 0xff, 0x58, 0xff, 0xaa, 0x42, 0x7b, 0x7b,
- 0xc2, 0x68, 0x12, 0x15, 0x72, 0xb2, 0x0a, 0xd2, 0xf9, 0x9c, 0x2c, 0x49, 0x64, 0x4e, 0x56, 0xc4,
- 0x1f, 0x40, 0x27, 0x90, 0xa1, 0xab, 0xe9, 0x55, 0x1e, 0x1a, 0x2c, 0x04, 0xb5, 0xdd, 0x0e, 0x72,
- 0xc9, 0x6c, 0x0c, 0x10, 0x11, 0x3f, 0xd6, 0x67, 0x54, 0x3a, 0xea, 0xeb, 0x8a, 0xd0, 0xa4, 0x68,
- 0xbb, 0x15, 0xa5, 0xd9, 0xfa, 0x3d, 0x68, 0x1f, 0x0b, 0x27, 0xe9, 0x03, 0x85, 0x64, 0x94, 0x79,
- 0xcf, 0x86, 0xe3, 0x2c, 0x08, 0xf7, 0xa0, 0x7b, 0xaa, 0x5c, 0xa6, 0x0f, 0xa9, 0x37, 0xf4, 0x9a,
- 0xb6, 0x24, 0xb3, 0x77, 0x9c, 0xf7, 0xac, 0xba, 0x80, 0xce, 0x69, 0x0e, 0x35, 0x3a, 0x84, 0xc1,
- 0x02, 0x49, 0x49, 0x0e, 0xda, 0xc8, 0xe7, 0xa0, 0xf6, 0x23, 0xa4, 0x04, 0xe5, 0x4f, 0xe6, 0xf3,
- 0xd2, 0x6f, 0xab, 0xd0, 0xf9, 0x31, 0xe6, 0x2f, 0x29, 0x3b, 0x53, 0xfa, 0x22, 0xa8, 0x87, 0x6e,
- 0x80, 0x35, 0x47, 0xb9, 0x46, 0x77, 0xa0, 0xc9, 0x2e, 0x54, 0x02, 0xd1, 0xf7, 0xb9, 0xcc, 0x2e,
- 0x64, 0x62, 0x40, 0xaf, 0x00, 0xb0, 0x0b, 0x27, 0x72, 0xbd, 0x33, 0xac, 0x3d, 0x58, 0xb7, 0x5b,
- 0xec, 0xe2, 0x40, 0x21, 0xc4, 0x53, 0x60, 0x17, 0x0e, 0x66, 0x8c, 0xb2, 0x58, 0xe7, 0xaa, 0x26,
- 0xbb, 0xd8, 0x95, 0xb0, 0x3e, 0xeb, 0x33, 0x1a, 0x45, 0xd8, 0x97, 0x39, 0x5a, 0x9e, 0xdd, 0x51,
- 0x08, 0x21, 0x95, 0x1b, 0xa9, 0x0d, 0x25, 0x95, 0x67, 0x52, 0x79, 0x26, 0x75, 0x59, 0x9d, 0xe4,
- 0x79, 0xa9, 0x3c, 0x95, 0xda, 0x54, 0x52, 0x79, 0x4e, 0x2a, 0xcf, 0xa4, 0xb6, 0xcc, 0x59, 0x2d,
- 0xd5, 0xfa, 0x4d, 0x05, 0xd6, 0xe7, 0x0b, 0x3f, 0x5d, 0xa6, 0x7e, 0x00, 0x1d, 0x4f, 0xde, 0x57,
- 0xe1, 0x4d, 0x0e, 0x16, 0x6e, 0xd2, 0x6e, 0x7b, 0xb9, 0x67, 0xfc, 0x21, 0x74, 0x43, 0xe5, 0xe0,
- 0xf4, 0x69, 0xd6, 0xb2, 0x7b, 0xc9, 0xfb, 0xde, 0xee, 0x84, 0x39, 0xc8, 0xf2, 0x01, 0x7d, 0xc9,
- 0x08, 0xc7, 0x87, 0x9c, 0x61, 0x37, 0xb8, 0x89, 0x06, 0x04, 0x41, 0x5d, 0x56, 0x2b, 0x35, 0x59,
- 0x5f, 0xcb, 0xb5, 0xf5, 0x26, 0xac, 0x16, 0xa4, 0x68, 0x5b, 0x57, 0xa0, 0x36, 0xc5, 0xa1, 0xe4,
- 0xde, 0xb5, 0xc5, 0xd2, 0x72, 0x61, 0x60, 0x63, 0xd7, 0xbf, 0x39, 0x6d, 0xb4, 0x88, 0x5a, 0x26,
- 0x62, 0x03, 0x50, 0x5e, 0x84, 0x56, 0xc5, 0x68, 0x5d, 0xc9, 0x69, 0xfd, 0x1c, 0x06, 0xdb, 0x53,
- 0x1a, 0xe3, 0x43, 0xee, 0x93, 0xf0, 0x26, 0x3a, 0xa6, 0x5f, 0xc2, 0xea, 0x0b, 0x3e, 0xfb, 0x52,
- 0x30, 0x8b, 0xc9, 0x37, 0xf8, 0x86, 0xec, 0x63, 0xf4, 0xa5, 0xb1, 0x8f, 0xd1, 0x97, 0xa2, 0x59,
- 0xf2, 0xe8, 0x34, 0x09, 0x42, 0x19, 0x0a, 0x5d, 0x5b, 0x43, 0xd6, 0x16, 0x74, 0x54, 0x0d, 0xfd,
- 0x8c, 0xfa, 0xc9, 0x14, 0x97, 0xc6, 0xe0, 0x7d, 0x80, 0xc8, 0x65, 0x6e, 0x80, 0x39, 0x66, 0xea,
- 0x0d, 0xb5, 0xec, 0x1c, 0xc6, 0xfa, 0x7d, 0x15, 0xd6, 0xd4, 0x48, 0xe4, 0x50, 0x4d, 0x02, 0x8c,
- 0x09, 0x23, 0x68, 0x9e, 0xd2, 0x98, 0xe7, 0x18, 0xa6, 0xb0, 0x50, 0xd1, 0x0f, 0x0d, 0x37, 0xb1,
- 0x2c, 0xcc, 0x29, 0x6a, 0x57, 0xcf, 0x29, 0x16, 0x26, 0x11, 0xf5, 0xc5, 0x49, 0x84, 0x88, 0x36,
- 0x43, 0x44, 0x54, 0x8c, 0xb7, 0xec, 0x96, 0xc6, 0xec, 0xfb, 0xe8, 0x21, 0xf4, 0x27, 0x42, 0x4b,
- 0xe7, 0x94, 0xd2, 0x33, 0x27, 0x72, 0xf9, 0xa9, 0x0c, 0xf5, 0x96, 0xdd, 0x95, 0xe8, 0x3d, 0x4a,
- 0xcf, 0x0e, 0x5c, 0x7e, 0x8a, 0x3e, 0x86, 0x9e, 0x2e, 0x03, 0x03, 0xe9, 0xa2, 0x58, 0x7f, 0xfc,
- 0x74, 0x14, 0xe5, 0xbd, 0x67, 0x77, 0xcf, 0x72, 0x50, 0x6c, 0xdd, 0x86, 0x5b, 0x3b, 0x38, 0xe6,
- 0x8c, 0xce, 0x8a, 0x8e, 0xb1, 0x7e, 0x00, 0xb0, 0x1f, 0x72, 0xcc, 0x4e, 0x5c, 0x0f, 0xc7, 0xe8,
- 0xdd, 0x3c, 0xa4, 0x8b, 0xa3, 0x95, 0xb1, 0x9a, 0x48, 0xa5, 0x1b, 0x76, 0x8e, 0xc6, 0x1a, 0x43,
- 0xc3, 0xa6, 0x89, 0x48, 0x47, 0xaf, 0x9b, 0x95, 0x3e, 0xd7, 0xd1, 0xe7, 0x24, 0xd2, 0xd6, 0x7b,
- 0xd6, 0x9e, 0x69, 0x61, 0x33, 0x76, 0xfa, 0x8a, 0xc6, 0xd0, 0x22, 0x06, 0xa7, 0xb3, 0xca, 0xa2,
- 0xe8, 0x8c, 0xc4, 0xfa, 0x19, 0xac, 0x2a, 0x4e, 0x8a, 0xb3, 0x61, 0xf3, 0x3a, 0x34, 0x98, 0x51,
- 0xa3, 0x92, 0x8d, 0xa2, 0x34, 0x91, 0xde, 0x43, 0xf7, 0x84, 0x30, 0x8f, 0xe1, 0x40, 0xf4, 0x1c,
- 0x55, 0x79, 0x65, 0x19, 0x42, 0x78, 0x4b, 0xf4, 0xdb, 0x99, 0x99, 0xc6, 0x5b, 0xab, 0x30, 0x10,
- 0x1b, 0x05, 0x89, 0xd6, 0x2f, 0x60, 0xf5, 0x79, 0x38, 0x25, 0x21, 0xde, 0x3e, 0x38, 0x7a, 0x86,
- 0xd3, 0xac, 0x80, 0xa0, 0x2e, 0xaa, 0x27, 0xa9, 0x46, 0xd3, 0x96, 0x6b, 0x11, 0x26, 0xe1, 0xb1,
- 0xe3, 0x45, 0x49, 0xac, 0x27, 0x43, 0x8d, 0xf0, 0x78, 0x3b, 0x4a, 0x62, 0x91, 0xe6, 0xc5, 0x67,
- 0x9e, 0x86, 0xd3, 0x99, 0x8c, 0x95, 0xa6, 0xbd, 0xec, 0x45, 0xc9, 0xf3, 0x70, 0x3a, 0xb3, 0xfe,
- 0x4f, 0xf6, 0xc2, 0x18, 0xfb, 0xb6, 0x1b, 0xfa, 0x34, 0xd8, 0xc1, 0xe7, 0x39, 0x09, 0x69, 0xdf,
- 0x65, 0x72, 0xc2, 0xb7, 0x15, 0xe8, 0x3c, 0x99, 0xe0, 0x90, 0xef, 0x60, 0xee, 0x92, 0xa9, 0xec,
- 0xad, 0xce, 0x31, 0x8b, 0x09, 0x0d, 0xf5, 0xc3, 0x37, 0xa0, 0x68, 0x8d, 0x49, 0x48, 0xb8, 0xe3,
- 0xbb, 0x38, 0xa0, 0xa1, 0xf6, 0x02, 0x08, 0xd4, 0x8e, 0xc4, 0xa0, 0x37, 0xa1, 0xaf, 0x26, 0x77,
- 0xce, 0xa9, 0x1b, 0xfa, 0x53, 0x11, 0x72, 0x6a, 0x92, 0xd1, 0x53, 0xe8, 0x3d, 0x8d, 0x45, 0x6f,
- 0xc1, 0x8a, 0x0e, 0x88, 0x8c, 0xb2, 0x2e, 0x29, 0xfb, 0x1a, 0x5f, 0x20, 0x4d, 0xa2, 0x88, 0x32,
- 0x1e, 0x3b, 0x31, 0xf6, 0x3c, 0x1a, 0x44, 0xba, 0x31, 0xe9, 0x1b, 0xfc, 0xa1, 0x42, 0x5b, 0x13,
- 0x58, 0x7d, 0x2a, 0xec, 0xd4, 0x96, 0x64, 0x17, 0xdc, 0x0b, 0x70, 0xe0, 0x1c, 0x4f, 0xa9, 0x77,
- 0xe6, 0x88, 0x34, 0xa5, 0x3d, 0x2c, 0x4a, 0x9f, 0x2d, 0x81, 0x3c, 0x24, 0xdf, 0xc8, 0x1e, 0x5c,
- 0x50, 0x9d, 0x52, 0x1e, 0x4d, 0x93, 0x89, 0x13, 0x31, 0x7a, 0x8c, 0xb5, 0x89, 0xfd, 0x00, 0x07,
- 0x7b, 0x0a, 0x7f, 0x20, 0xd0, 0xd6, 0x5f, 0x2a, 0xb0, 0x56, 0x94, 0xa4, 0x93, 0xee, 0x26, 0xac,
- 0x15, 0x45, 0xe9, 0x0f, 0xb1, 0x2a, 0xf4, 0x06, 0x79, 0x81, 0xea, 0x93, 0xfc, 0x21, 0x74, 0xe5,
- 0x38, 0xd7, 0xf1, 0x15, 0xa7, 0x62, 0xf9, 0x91, 0xbf, 0x17, 0xbb, 0xe3, 0xe6, 0x6f, 0xe9, 0x63,
- 0xb8, 0xa3, 0xcd, 0x77, 0x16, 0xd5, 0x56, 0x0f, 0x62, 0x5d, 0x13, 0x3c, 0x9b, 0xd3, 0xfe, 0x0b,
- 0x18, 0x66, 0xa8, 0xad, 0x99, 0x44, 0x1a, 0x5f, 0xbd, 0x0b, 0xab, 0x73, 0xc6, 0x3e, 0xf1, 0x7d,
- 0x26, 0x03, 0xb4, 0x6e, 0x97, 0x6d, 0x59, 0x8f, 0xe1, 0xf6, 0x21, 0xe6, 0xca, 0x1b, 0x2e, 0xd7,
- 0x3d, 0x81, 0x62, 0xb6, 0x02, 0xb5, 0x43, 0xec, 0x49, 0xe3, 0x6b, 0xb6, 0x58, 0x8a, 0x07, 0x78,
- 0x14, 0x63, 0x4f, 0x5a, 0x59, 0xb3, 0xe5, 0xda, 0xfa, 0x73, 0x05, 0x96, 0x75, 0x9a, 0x14, 0xa9,
- 0xde, 0x67, 0xe4, 0x1c, 0x33, 0xfd, 0xf4, 0x34, 0x84, 0xde, 0x80, 0x9e, 0x5a, 0x39, 0x34, 0xe2,
- 0x84, 0xa6, 0xc9, 0xb7, 0xab, 0xb0, 0xcf, 0x15, 0x52, 0x4e, 0xea, 0xe4, 0x20, 0x4a, 0xf7, 0x7c,
- 0x1a, 0x92, 0xe3, 0xb6, 0x58, 0x64, 0x06, 0x99, 0x6c, 0x5b, 0xb6, 0x86, 0xc4, 0x53, 0x37, 0xfc,
- 0x96, 0x24, 0x3f, 0x03, 0x8a, 0xa7, 0x1e, 0xd0, 0x24, 0xe4, 0x4e, 0x44, 0x49, 0xc8, 0x75, 0x76,
- 0x05, 0x89, 0x3a, 0x10, 0x18, 0xeb, 0xd7, 0x15, 0x68, 0xa8, 0x69, 0xb5, 0xe8, 0x32, 0xd3, 0x6f,
- 0x5c, 0x95, 0xc8, 0x7a, 0x41, 0xca, 0x52, 0xdf, 0x35, 0xb9, 0x16, 0x71, 0x7c, 0x1e, 0xa8, 0x4c,
- 0xad, 0x55, 0x3b, 0x0f, 0x64, 0x8a, 0x7e, 0x03, 0x7a, 0xd9, 0xa7, 0x52, 0xee, 0x2b, 0x15, 0xbb,
- 0x29, 0x56, 0x92, 0x5d, 0xaa, 0xa9, 0xf5, 0x53, 0xd1, 0x5c, 0xa7, 0x93, 0xda, 0x15, 0xa8, 0x25,
- 0xa9, 0x32, 0x62, 0x29, 0x30, 0x93, 0xf4, 0x23, 0x2b, 0x96, 0xe8, 0x21, 0xf4, 0x5c, 0xdf, 0x27,
- 0xe2, 0xb8, 0x3b, 0x7d, 0x4a, 0xfc, 0x34, 0x48, 0x8b, 0x58, 0xeb, 0x6f, 0x15, 0xe8, 0x6f, 0xd3,
- 0x68, 0xf6, 0x43, 0x32, 0xc5, 0xb9, 0x0c, 0x22, 0x95, 0xd4, 0xdf, 0x58, 0xb1, 0x16, 0x75, 0xe3,
- 0x09, 0x99, 0x62, 0x15, 0x5a, 0xea, 0x66, 0x9b, 0x02, 0x21, 0xc3, 0xca, 0x6c, 0xa6, 0x03, 0xb0,
- 0xae, 0xda, 0x7c, 0x46, 0x7d, 0x59, 0x21, 0xfb, 0x84, 0x39, 0xe9, 0xb8, 0xab, 0x6b, 0x2f, 0xfb,
- 0x84, 0xc9, 0x2d, 0x6d, 0xc8, 0x92, 0x9c, 0xb8, 0xe6, 0x0d, 0x69, 0x28, 0x8c, 0x30, 0x64, 0x1d,
- 0x1a, 0xf4, 0xe4, 0x24, 0xc6, 0x5c, 0xd6, 0xb2, 0x35, 0x5b, 0x43, 0x69, 0x9a, 0x6b, 0xe6, 0xd2,
- 0xdc, 0x2d, 0x58, 0x95, 0xb3, 0xfd, 0x17, 0xcc, 0xf5, 0x48, 0x38, 0x31, 0xa9, 0x78, 0x0d, 0xd0,
- 0x21, 0xa7, 0x51, 0x11, 0xfb, 0xe8, 0x77, 0x2b, 0x3a, 0x27, 0xea, 0x46, 0x17, 0x3d, 0x85, 0xfe,
- 0xdc, 0x1f, 0x27, 0x48, 0x4f, 0x3e, 0xca, 0xff, 0x4f, 0x19, 0xad, 0x8f, 0xd5, 0x1f, 0x31, 0x63,
- 0xf3, 0x47, 0xcc, 0x78, 0x37, 0x88, 0xf8, 0x0c, 0xed, 0x42, 0xaf, 0xf8, 0x17, 0x03, 0xba, 0x6b,
- 0x0a, 0x85, 0x92, 0x3f, 0x1e, 0x2e, 0x65, 0xf3, 0x14, 0xfa, 0x73, 0xff, 0x36, 0x18, 0x7d, 0xca,
- 0xff, 0x84, 0xb8, 0x94, 0xd1, 0x63, 0x68, 0xe7, 0xfe, 0x5e, 0x40, 0x43, 0xc5, 0x64, 0xf1, 0x1f,
- 0x87, 0x4b, 0x19, 0x6c, 0x43, 0xb7, 0x30, 0xf1, 0x47, 0x23, 0x6d, 0x4f, 0xc9, 0xdf, 0x00, 0x97,
- 0x32, 0xd9, 0x82, 0x76, 0x6e, 0xf0, 0x6e, 0xb4, 0x58, 0x9c, 0xee, 0x8f, 0xee, 0x94, 0xec, 0xe8,
- 0xd4, 0xbb, 0x07, 0xdd, 0xc2, 0x98, 0xdc, 0x28, 0x52, 0x36, 0xa2, 0x1f, 0xdd, 0x2d, 0xdd, 0xd3,
- 0x9c, 0x9e, 0x42, 0x7f, 0x6e, 0x68, 0x6e, 0x9c, 0x5b, 0x3e, 0x4b, 0xbf, 0xd4, 0xac, 0xcf, 0xe5,
- 0x65, 0xe7, 0x7a, 0xa2, 0xdc, 0x65, 0x2f, 0x8e, 0xc8, 0x47, 0xf7, 0xca, 0x37, 0xb5, 0x56, 0xbb,
- 0xd0, 0x2b, 0x4e, 0xc7, 0x0d, 0xb3, 0xd2, 0x99, 0xf9, 0xd5, 0x2f, 0xa7, 0x30, 0x28, 0xcf, 0x5e,
- 0x4e, 0xd9, 0xfc, 0xfc, 0x52, 0x46, 0x4f, 0x00, 0x74, 0x07, 0xe4, 0x93, 0x30, 0xbd, 0xb2, 0x85,
- 0xce, 0x2b, 0xbd, 0xb2, 0x92, 0x6e, 0xe9, 0x31, 0x80, 0x6a, 0x5c, 0x7c, 0x9a, 0x70, 0x74, 0xdb,
- 0xa8, 0x31, 0xd7, 0x2d, 0x8d, 0x86, 0x8b, 0x1b, 0x0b, 0x0c, 0x30, 0x63, 0xd7, 0x61, 0xf0, 0x19,
- 0x40, 0xd6, 0x10, 0x19, 0x06, 0x0b, 0x2d, 0xd2, 0x15, 0x3e, 0xe8, 0xe4, 0xdb, 0x1f, 0xa4, 0x6d,
- 0x2d, 0x69, 0x89, 0xae, 0x60, 0xd1, 0x9f, 0x2b, 0x6f, 0x8b, 0x8f, 0x6d, 0xbe, 0xea, 0x1d, 0x2d,
- 0x94, 0xb8, 0xe8, 0x43, 0xe8, 0xe4, 0xeb, 0x5a, 0xa3, 0x45, 0x49, 0xad, 0x3b, 0x2a, 0xd4, 0xb6,
- 0xe8, 0x31, 0xf4, 0x8a, 0x55, 0x2b, 0xca, 0xc5, 0xc5, 0x42, 0x2d, 0x3b, 0xd2, 0x13, 0x9b, 0x1c,
- 0xf9, 0xfb, 0x00, 0x59, 0x75, 0x6b, 0xdc, 0xb7, 0x50, 0xef, 0xce, 0x49, 0x7d, 0x02, 0x9d, 0x7c,
- 0x26, 0x36, 0xea, 0x96, 0x64, 0xe7, 0xab, 0xb2, 0x56, 0x2e, 0x6b, 0x9b, 0xc7, 0xb7, 0x98, 0xc8,
- 0xaf, 0xca, 0x5a, 0x85, 0xae, 0xcf, 0x24, 0x8b, 0xb2, 0x56, 0xf0, 0xaa, 0x5c, 0x5e, 0x6c, 0x91,
- 0x8c, 0xfb, 0x4a, 0x1b, 0xa7, 0xab, 0x1e, 0x51, 0xbe, 0x1b, 0x30, 0xfe, 0x28, 0xe9, 0x10, 0xbe,
- 0x27, 0xa8, 0xf3, 0x15, 0x7f, 0x2e, 0xa8, 0x4b, 0x1a, 0x81, 0x4b, 0x19, 0xed, 0x41, 0xff, 0xa9,
- 0x29, 0xe6, 0x74, 0xa1, 0xa9, 0xd5, 0x29, 0x29, 0xac, 0x47, 0xa3, 0xb2, 0x2d, 0x1d, 0x59, 0x9f,
- 0xc3, 0x60, 0xa1, 0xc8, 0x44, 0xf7, 0xd3, 0xc1, 0x62, 0x69, 0xf5, 0x79, 0xa9, 0x5a, 0xfb, 0xb0,
- 0x32, 0x5f, 0x63, 0xa2, 0x57, 0xf4, 0xa5, 0x97, 0xd7, 0x9e, 0x97, 0xb2, 0xfa, 0x18, 0x9a, 0xa6,
- 0xa6, 0x41, 0x7a, 0x80, 0x3b, 0x57, 0xe3, 0x5c, 0x76, 0x74, 0xab, 0xf3, 0xed, 0x77, 0xf7, 0x2b,
- 0x7f, 0xff, 0xee, 0x7e, 0xe5, 0x9f, 0xdf, 0xdd, 0xaf, 0x1c, 0x37, 0xe4, 0xee, 0xfb, 0xff, 0x0e,
- 0x00, 0x00, 0xff, 0xff, 0x8d, 0x89, 0xaa, 0x73, 0xc8, 0x21, 0x00, 0x00,
+ // 2931 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0xcb, 0x6e, 0x1c, 0xc7,
+ 0xb5, 0x98, 0x07, 0x87, 0x33, 0x67, 0x5e, 0x9c, 0x22, 0x45, 0x8d, 0x46, 0xb2, 0xae, 0xdc, 0xb6,
+ 0x65, 0xfa, 0xfa, 0x7a, 0x68, 0xcb, 0xc6, 0xf5, 0x0b, 0xbe, 0x82, 0x48, 0xe9, 0x8a, 0x8c, 0xad,
+ 0x88, 0xe9, 0x91, 0xe2, 0x04, 0x41, 0xd0, 0x68, 0x76, 0x97, 0x86, 0x65, 0x4e, 0x77, 0xb5, 0xab,
+ 0xaa, 0x29, 0x8e, 0x03, 0x64, 0x99, 0xec, 0xb2, 0xcc, 0x2e, 0x3f, 0x10, 0x64, 0x97, 0x65, 0xb6,
+ 0x59, 0x18, 0x59, 0x05, 0xf9, 0x80, 0x20, 0xf0, 0x27, 0xe4, 0x0b, 0x82, 0x7a, 0xf5, 0x63, 0x66,
+ 0x48, 0x23, 0x82, 0x80, 0x6c, 0x1a, 0x75, 0x4e, 0x9d, 0x3a, 0xaf, 0xaa, 0x3a, 0x75, 0xce, 0x69,
+ 0x68, 0xfb, 0x53, 0x1c, 0x8b, 0x71, 0xc2, 0xa8, 0xa0, 0xa8, 0x3e, 0x65, 0x49, 0x30, 0x6a, 0xd1,
+ 0x80, 0x68, 0xc4, 0xe8, 0x7f, 0xa7, 0x44, 0x9c, 0xa4, 0xc7, 0xe3, 0x80, 0x46, 0xbb, 0xa7, 0xbe,
+ 0xf0, 0xdf, 0x09, 0x68, 0x2c, 0x7c, 0x12, 0x63, 0xc6, 0x77, 0xd5, 0xc2, 0xdd, 0xe4, 0x74, 0xba,
+ 0x2b, 0xe6, 0x09, 0xe6, 0xfa, 0x6b, 0xd6, 0x5d, 0x9f, 0x52, 0x3a, 0x9d, 0xe1, 0x5d, 0x05, 0x1d,
+ 0xa7, 0xcf, 0x76, 0x71, 0x94, 0x88, 0xb9, 0x9e, 0x74, 0x7e, 0x57, 0x85, 0xed, 0x7d, 0x86, 0x7d,
+ 0x81, 0xf7, 0x2d, 0x37, 0x17, 0x7f, 0x9d, 0x62, 0x2e, 0xd0, 0xab, 0xd0, 0xc9, 0x24, 0x78, 0x24,
+ 0x1c, 0x56, 0x6e, 0x55, 0x76, 0x5a, 0x6e, 0x3b, 0xc3, 0x1d, 0x86, 0xe8, 0x2a, 0xac, 0xe3, 0x73,
+ 0x1c, 0xc8, 0xd9, 0xaa, 0x9a, 0x6d, 0x48, 0xf0, 0x30, 0x44, 0xef, 0x41, 0x9b, 0x0b, 0x46, 0xe2,
+ 0xa9, 0x97, 0x72, 0xcc, 0x86, 0xb5, 0x5b, 0x95, 0x9d, 0xf6, 0x9d, 0x8d, 0xb1, 0x34, 0x69, 0x3c,
+ 0x51, 0x13, 0x4f, 0x39, 0x66, 0x2e, 0xf0, 0x6c, 0x8c, 0x6e, 0xc3, 0x7a, 0x88, 0xcf, 0x48, 0x80,
+ 0xf9, 0xb0, 0x7e, 0xab, 0xb6, 0xd3, 0xbe, 0xd3, 0xd1, 0xe4, 0xf7, 0x15, 0xd2, 0xb5, 0x93, 0xe8,
+ 0x2d, 0x68, 0x72, 0x41, 0x99, 0x3f, 0xc5, 0x7c, 0xb8, 0xa6, 0x08, 0xbb, 0x96, 0xaf, 0xc2, 0xba,
+ 0xd9, 0x34, 0xba, 0x01, 0xb5, 0xc7, 0xfb, 0x87, 0xc3, 0x86, 0x92, 0x0e, 0x86, 0x2a, 0xc1, 0x81,
+ 0x2b, 0xd1, 0xe8, 0x35, 0xe8, 0x72, 0x3f, 0x0e, 0x8f, 0xe9, 0xb9, 0x97, 0x90, 0x30, 0xe6, 0xc3,
+ 0xf5, 0x5b, 0x95, 0x9d, 0xa6, 0xdb, 0x31, 0xc8, 0x23, 0x89, 0x73, 0x3e, 0x81, 0x2b, 0x13, 0xe1,
+ 0x33, 0xf1, 0x02, 0xde, 0x71, 0x9e, 0xc2, 0xb6, 0x8b, 0x23, 0x7a, 0xf6, 0x42, 0xae, 0x1d, 0xc2,
+ 0xba, 0x20, 0x11, 0xa6, 0xa9, 0x50, 0xae, 0xed, 0xba, 0x16, 0x74, 0xfe, 0x50, 0x01, 0xf4, 0xe0,
+ 0x1c, 0x07, 0x47, 0x8c, 0x06, 0x98, 0xf3, 0xff, 0xd0, 0x76, 0xbd, 0x09, 0xeb, 0x89, 0x56, 0x60,
+ 0x58, 0x57, 0xe4, 0x66, 0x17, 0xac, 0x56, 0x76, 0xd6, 0xf9, 0x0a, 0xb6, 0x26, 0x64, 0x1a, 0xfb,
+ 0xb3, 0x97, 0xa8, 0xef, 0x36, 0x34, 0xb8, 0xe2, 0xa9, 0x54, 0xed, 0xba, 0x06, 0x72, 0x8e, 0x00,
+ 0x7d, 0xe9, 0x13, 0xf1, 0xf2, 0x24, 0x39, 0xef, 0xc0, 0x66, 0x89, 0x23, 0x4f, 0x68, 0xcc, 0xb1,
+ 0x52, 0x40, 0xf8, 0x22, 0xe5, 0x8a, 0xd9, 0x9a, 0x6b, 0x20, 0x07, 0xc3, 0xd6, 0x17, 0x84, 0x5b,
+ 0x72, 0xfc, 0xef, 0xa8, 0xb0, 0x0d, 0x8d, 0x67, 0x94, 0x45, 0xbe, 0xb0, 0x1a, 0x68, 0x08, 0x21,
+ 0xa8, 0xfb, 0x6c, 0xca, 0x87, 0xb5, 0x5b, 0xb5, 0x9d, 0x96, 0xab, 0xc6, 0xf2, 0x54, 0x2e, 0x88,
+ 0x31, 0x7a, 0xbd, 0x0a, 0x1d, 0xe3, 0x77, 0x6f, 0x46, 0xb8, 0x50, 0x72, 0x3a, 0x6e, 0xdb, 0xe0,
+ 0xe4, 0x1a, 0x87, 0xc2, 0xf6, 0xd3, 0x24, 0x7c, 0xc1, 0x0b, 0x7f, 0x07, 0x5a, 0x0c, 0x73, 0x9a,
+ 0x32, 0x79, 0x4d, 0xab, 0x6a, 0xdf, 0xb7, 0xf4, 0xbe, 0x7f, 0x41, 0xe2, 0xf4, 0xdc, 0xb5, 0x73,
+ 0x6e, 0x4e, 0x66, 0xae, 0x90, 0xe0, 0x2f, 0x72, 0x85, 0x3e, 0x81, 0x2b, 0x47, 0x7e, 0xca, 0x5f,
+ 0x44, 0x57, 0xe7, 0x53, 0x79, 0xfd, 0x78, 0x1a, 0xbd, 0xd0, 0xe2, 0xdf, 0x57, 0xa0, 0xb9, 0x9f,
+ 0xa4, 0x4f, 0xb9, 0x3f, 0xc5, 0xe8, 0xbf, 0xa0, 0x2d, 0xa8, 0xf0, 0x67, 0x5e, 0x2a, 0x41, 0x45,
+ 0x5e, 0x77, 0x41, 0xa1, 0x34, 0x81, 0x74, 0x3b, 0x66, 0x41, 0x92, 0x1a, 0x8a, 0xea, 0xad, 0xda,
+ 0x4e, 0xdd, 0x6d, 0x6b, 0x9c, 0x26, 0x19, 0xc3, 0xa6, 0x9a, 0xf3, 0x48, 0xec, 0x9d, 0x62, 0x16,
+ 0xe3, 0x59, 0x44, 0x43, 0xac, 0xce, 0x6f, 0xdd, 0x1d, 0xa8, 0xa9, 0xc3, 0xf8, 0xf3, 0x6c, 0x02,
+ 0xfd, 0x37, 0x0c, 0x32, 0x7a, 0x79, 0x29, 0x15, 0x75, 0x5d, 0x51, 0xf7, 0x0d, 0xf5, 0x53, 0x83,
+ 0x76, 0x7e, 0x09, 0xbd, 0x27, 0x27, 0x8c, 0x0a, 0x31, 0x23, 0xf1, 0xf4, 0xbe, 0x2f, 0x7c, 0x19,
+ 0x3d, 0x12, 0xcc, 0x08, 0x0d, 0xb9, 0xd1, 0xd6, 0x82, 0xe8, 0x6d, 0x18, 0x08, 0x4d, 0x8b, 0x43,
+ 0xcf, 0xd2, 0x54, 0x15, 0xcd, 0x46, 0x36, 0x71, 0x64, 0x88, 0xdf, 0x80, 0x5e, 0x4e, 0x2c, 0xe3,
+ 0x8f, 0xd1, 0xb7, 0x9b, 0x61, 0x9f, 0x90, 0x08, 0x3b, 0x67, 0xca, 0x57, 0x6a, 0x93, 0xd1, 0xdb,
+ 0xd0, 0xca, 0xfd, 0x50, 0x51, 0x27, 0xa4, 0xa7, 0x4f, 0x88, 0x75, 0xa7, 0xdb, 0xcc, 0x9c, 0xf2,
+ 0x19, 0xf4, 0x45, 0xa6, 0xb8, 0x17, 0xfa, 0xc2, 0x2f, 0x1f, 0xaa, 0xb2, 0x55, 0x6e, 0x4f, 0x94,
+ 0x60, 0xe7, 0x53, 0x68, 0x1d, 0x91, 0x90, 0x6b, 0xc1, 0x43, 0x58, 0x0f, 0x52, 0xc6, 0x70, 0x2c,
+ 0xac, 0xc9, 0x06, 0x44, 0x5b, 0xb0, 0x36, 0x23, 0x11, 0x11, 0xc6, 0x4c, 0x0d, 0x38, 0x14, 0xe0,
+ 0x11, 0x8e, 0x28, 0x9b, 0x2b, 0x87, 0x6d, 0xc1, 0x5a, 0x71, 0x73, 0x35, 0x80, 0xae, 0x43, 0x2b,
+ 0xf2, 0xcf, 0xb3, 0x4d, 0x95, 0x33, 0xcd, 0xc8, 0x3f, 0xd7, 0xca, 0x0f, 0x61, 0xfd, 0x99, 0x4f,
+ 0x66, 0x41, 0x2c, 0x8c, 0x57, 0x2c, 0x98, 0x0b, 0xac, 0x17, 0x05, 0xfe, 0xb9, 0x0a, 0x6d, 0x2d,
+ 0x51, 0x2b, 0xbc, 0x05, 0x6b, 0x81, 0x1f, 0x9c, 0x64, 0x22, 0x15, 0x80, 0x6e, 0x5b, 0x45, 0xaa,
+ 0xc5, 0x20, 0x9c, 0x6b, 0x6a, 0x55, 0xdb, 0x05, 0xe0, 0xcf, 0xfd, 0xc4, 0xe8, 0x56, 0xbb, 0x80,
+ 0xb8, 0x25, 0x69, 0xb4, 0xba, 0xef, 0x43, 0x47, 0x9f, 0x3b, 0xb3, 0xa4, 0x7e, 0xc1, 0x92, 0xb6,
+ 0xa6, 0xd2, 0x8b, 0x5e, 0x83, 0x6e, 0xca, 0xb1, 0x77, 0x42, 0x30, 0xf3, 0x59, 0x70, 0x32, 0x1f,
+ 0xae, 0xe9, 0x37, 0x32, 0xe5, 0xf8, 0xc0, 0xe2, 0xd0, 0x1d, 0x58, 0x93, 0xe1, 0x8f, 0x0f, 0x1b,
+ 0xea, 0x39, 0xbe, 0x51, 0x64, 0xa9, 0x4c, 0x1d, 0xab, 0xef, 0x83, 0x58, 0xb0, 0xb9, 0xab, 0x49,
+ 0x47, 0x1f, 0x01, 0xe4, 0x48, 0xb4, 0x01, 0xb5, 0x53, 0x3c, 0x37, 0xf7, 0x50, 0x0e, 0xa5, 0x73,
+ 0xce, 0xfc, 0x59, 0x6a, 0xbd, 0xae, 0x81, 0x4f, 0xaa, 0x1f, 0x55, 0x9c, 0x00, 0xfa, 0x7b, 0xb3,
+ 0x53, 0x42, 0x0b, 0xcb, 0xb7, 0x60, 0x2d, 0xf2, 0xbf, 0xa2, 0xcc, 0x7a, 0x52, 0x01, 0x0a, 0x4b,
+ 0x62, 0xca, 0x2c, 0x0b, 0x05, 0xa0, 0x1e, 0x54, 0x69, 0xa2, 0xfc, 0xd5, 0x72, 0xab, 0x34, 0xc9,
+ 0x05, 0xd5, 0x0b, 0x82, 0x9c, 0xbf, 0xd7, 0x01, 0x72, 0x29, 0xc8, 0x85, 0x11, 0xa1, 0x1e, 0xc7,
+ 0x4c, 0xa6, 0x20, 0xde, 0xf1, 0x5c, 0x60, 0xee, 0x31, 0x1c, 0xa4, 0x8c, 0x93, 0x33, 0xb9, 0x7f,
+ 0xd2, 0xec, 0x2b, 0xda, 0xec, 0x05, 0xdd, 0xdc, 0xab, 0x84, 0x4e, 0xf4, 0xba, 0x3d, 0xb9, 0xcc,
+ 0xb5, 0xab, 0xd0, 0x21, 0x5c, 0xc9, 0x79, 0x86, 0x05, 0x76, 0xd5, 0xcb, 0xd8, 0x6d, 0x66, 0xec,
+ 0xc2, 0x9c, 0xd5, 0x03, 0xd8, 0x24, 0xd4, 0xfb, 0x3a, 0xc5, 0x69, 0x89, 0x51, 0xed, 0x32, 0x46,
+ 0x03, 0x42, 0x7f, 0xa4, 0x16, 0xe4, 0x6c, 0x8e, 0xe0, 0x5a, 0xc1, 0x4a, 0x79, 0xdd, 0x0b, 0xcc,
+ 0xea, 0x97, 0x31, 0xdb, 0xce, 0xb4, 0x92, 0xf1, 0x20, 0xe7, 0xf8, 0x03, 0xd8, 0x26, 0xd4, 0x7b,
+ 0xee, 0x13, 0xb1, 0xc8, 0x6e, 0xed, 0x7b, 0x8c, 0x94, 0x8f, 0x6e, 0x99, 0x97, 0x36, 0x32, 0xc2,
+ 0x6c, 0x5a, 0x32, 0xb2, 0xf1, 0x3d, 0x46, 0x3e, 0x52, 0x0b, 0x72, 0x36, 0xf7, 0x60, 0x40, 0xe8,
+ 0xa2, 0x36, 0xeb, 0x97, 0x31, 0xe9, 0x13, 0x5a, 0xd6, 0x64, 0x0f, 0x06, 0x1c, 0x07, 0x82, 0xb2,
+ 0xe2, 0x21, 0x68, 0x5e, 0xc6, 0x62, 0xc3, 0xd0, 0x67, 0x3c, 0x9c, 0x9f, 0x41, 0xe7, 0x20, 0x9d,
+ 0x62, 0x31, 0x3b, 0xce, 0x82, 0xc1, 0x4b, 0x8b, 0x3f, 0xce, 0x3f, 0xab, 0xd0, 0xde, 0x9f, 0x32,
+ 0x9a, 0x26, 0xa5, 0x98, 0xac, 0x2f, 0xe9, 0x62, 0x4c, 0x56, 0x24, 0x2a, 0x26, 0x6b, 0xe2, 0x0f,
+ 0xa0, 0x13, 0xa9, 0xab, 0x6b, 0xe8, 0x75, 0x1c, 0x1a, 0x2c, 0x5d, 0x6a, 0xb7, 0x1d, 0x15, 0x82,
+ 0xd9, 0x18, 0x20, 0x21, 0x21, 0x37, 0x6b, 0x74, 0x38, 0xea, 0x9b, 0x8c, 0xd0, 0x86, 0x68, 0xb7,
+ 0x95, 0x64, 0xd1, 0xfa, 0x3d, 0x68, 0x1f, 0x4b, 0x27, 0x99, 0x05, 0xa5, 0x60, 0x94, 0x7b, 0xcf,
+ 0x85, 0xe3, 0xfc, 0x12, 0x1e, 0x40, 0xf7, 0x44, 0xbb, 0xcc, 0x2c, 0xd2, 0x67, 0xe8, 0x35, 0x63,
+ 0x49, 0x6e, 0xef, 0xb8, 0xe8, 0x59, 0xbd, 0x01, 0x9d, 0x93, 0x02, 0x6a, 0x34, 0x81, 0xc1, 0x12,
+ 0xc9, 0x8a, 0x18, 0xb4, 0x53, 0x8c, 0x41, 0xed, 0x3b, 0x48, 0x0b, 0x2a, 0xae, 0x2c, 0xc6, 0xa5,
+ 0xdf, 0x54, 0xa1, 0xf3, 0x43, 0x2c, 0x9e, 0x53, 0x76, 0xaa, 0xf5, 0x45, 0x50, 0x8f, 0xfd, 0x08,
+ 0x1b, 0x8e, 0x6a, 0x8c, 0xae, 0x41, 0x93, 0x9d, 0xeb, 0x00, 0x62, 0xf6, 0x73, 0x9d, 0x9d, 0xab,
+ 0xc0, 0x80, 0x5e, 0x01, 0x60, 0xe7, 0x5e, 0xe2, 0x07, 0xa7, 0xd8, 0x78, 0xb0, 0xee, 0xb6, 0xd8,
+ 0xf9, 0x91, 0x46, 0xc8, 0xa3, 0xc0, 0xce, 0x3d, 0xcc, 0x18, 0x65, 0xdc, 0xc4, 0xaa, 0x26, 0x3b,
+ 0x7f, 0xa0, 0x60, 0xb3, 0x36, 0x64, 0x34, 0x49, 0x70, 0xa8, 0x62, 0xb4, 0x5a, 0x7b, 0x5f, 0x23,
+ 0xa4, 0x54, 0x61, 0xa5, 0x36, 0xb4, 0x54, 0x91, 0x4b, 0x15, 0xb9, 0xd4, 0x75, 0xbd, 0x52, 0x14,
+ 0xa5, 0x8a, 0x4c, 0x6a, 0x53, 0x4b, 0x15, 0x05, 0xa9, 0x22, 0x97, 0xda, 0xb2, 0x6b, 0x8d, 0x54,
+ 0xe7, 0xd7, 0x15, 0xd8, 0x5e, 0x4c, 0xfc, 0x4c, 0x9a, 0xfa, 0x01, 0x74, 0x02, 0xb5, 0x5f, 0xa5,
+ 0x33, 0x39, 0x58, 0xda, 0x49, 0xb7, 0x1d, 0x14, 0x8e, 0xf1, 0x87, 0xd0, 0x8d, 0xb5, 0x83, 0xb3,
+ 0xa3, 0x59, 0xcb, 0xf7, 0xa5, 0xe8, 0x7b, 0xb7, 0x13, 0x17, 0x20, 0x27, 0x04, 0xf4, 0x25, 0x23,
+ 0x02, 0x4f, 0x04, 0xc3, 0x7e, 0xf4, 0x32, 0x0a, 0x10, 0x04, 0x75, 0x95, 0xad, 0xd4, 0x54, 0x7e,
+ 0xad, 0xc6, 0xce, 0x9b, 0xb0, 0x59, 0x92, 0x62, 0x6c, 0xdd, 0x80, 0xda, 0x0c, 0xc7, 0x8a, 0x7b,
+ 0xd7, 0x95, 0x43, 0xc7, 0x87, 0x81, 0x8b, 0xfd, 0xf0, 0xe5, 0x69, 0x63, 0x44, 0xd4, 0x72, 0x11,
+ 0x3b, 0x80, 0x8a, 0x22, 0x8c, 0x2a, 0x56, 0xeb, 0x4a, 0x41, 0xeb, 0xc7, 0x30, 0xd8, 0x9f, 0x51,
+ 0x8e, 0x27, 0x22, 0x24, 0xf1, 0xcb, 0xa8, 0x98, 0x7e, 0x01, 0x9b, 0x4f, 0xc4, 0xfc, 0x4b, 0xc9,
+ 0x8c, 0x93, 0x6f, 0xf0, 0x4b, 0xb2, 0x8f, 0xd1, 0xe7, 0xd6, 0x3e, 0x46, 0x9f, 0xcb, 0x62, 0x29,
+ 0xa0, 0xb3, 0x34, 0x8a, 0xd5, 0x55, 0xe8, 0xba, 0x06, 0x72, 0xf6, 0xa0, 0xa3, 0x73, 0xe8, 0x47,
+ 0x34, 0x4c, 0x67, 0x78, 0xe5, 0x1d, 0xbc, 0x09, 0x90, 0xf8, 0xcc, 0x8f, 0xb0, 0xc0, 0x4c, 0x9f,
+ 0xa1, 0x96, 0x5b, 0xc0, 0x38, 0xbf, 0xad, 0xc2, 0x96, 0x6e, 0x89, 0x4c, 0x74, 0x27, 0xc0, 0x9a,
+ 0x30, 0x82, 0xe6, 0x09, 0xe5, 0xa2, 0xc0, 0x30, 0x83, 0xa5, 0x8a, 0x61, 0x6c, 0xb9, 0xc9, 0x61,
+ 0xa9, 0x4f, 0x51, 0xbb, 0xbc, 0x4f, 0xb1, 0xd4, 0x89, 0xa8, 0x2f, 0x77, 0x22, 0xe4, 0x6d, 0xb3,
+ 0x44, 0x44, 0xdf, 0xf1, 0x96, 0xdb, 0x32, 0x98, 0xc3, 0x10, 0xdd, 0x86, 0xfe, 0x54, 0x6a, 0xe9,
+ 0x9d, 0x50, 0x7a, 0xea, 0x25, 0xbe, 0x38, 0x51, 0x57, 0xbd, 0xe5, 0x76, 0x15, 0xfa, 0x80, 0xd2,
+ 0xd3, 0x23, 0x5f, 0x9c, 0xa0, 0x8f, 0xa1, 0x67, 0xd2, 0xc0, 0x48, 0xb9, 0x88, 0x9b, 0xc7, 0xcf,
+ 0xdc, 0xa2, 0xa2, 0xf7, 0xdc, 0xee, 0x69, 0x01, 0xe2, 0xce, 0x55, 0xb8, 0x72, 0x1f, 0x73, 0xc1,
+ 0xe8, 0xbc, 0xec, 0x18, 0xe7, 0xff, 0x00, 0x0e, 0x63, 0x81, 0xd9, 0x33, 0x3f, 0xc0, 0x1c, 0xbd,
+ 0x5b, 0x84, 0x4c, 0x72, 0xb4, 0x31, 0xd6, 0x1d, 0xa9, 0x6c, 0xc2, 0x2d, 0xd0, 0x38, 0x63, 0x68,
+ 0xb8, 0x34, 0x95, 0xe1, 0xe8, 0x75, 0x3b, 0x32, 0xeb, 0x3a, 0x66, 0x9d, 0x42, 0xba, 0x66, 0xce,
+ 0x39, 0xb0, 0x25, 0x6c, 0xce, 0xce, 0x6c, 0xd1, 0x18, 0x5a, 0xc4, 0xe2, 0x4c, 0x54, 0x59, 0x16,
+ 0x9d, 0x93, 0x38, 0x3f, 0x85, 0x4d, 0xcd, 0x49, 0x73, 0xb6, 0x6c, 0x5e, 0x87, 0x06, 0xb3, 0x6a,
+ 0x54, 0xf2, 0x56, 0x94, 0x21, 0x32, 0x73, 0xe8, 0x86, 0x14, 0x16, 0x30, 0x1c, 0xc9, 0x9a, 0xa3,
+ 0xaa, 0xb6, 0x2c, 0x47, 0x48, 0x6f, 0xc9, 0x7a, 0x3b, 0x37, 0xd3, 0x7a, 0x6b, 0x13, 0x06, 0x72,
+ 0xa2, 0x24, 0xd1, 0xf9, 0x39, 0x6c, 0x3e, 0x8e, 0x67, 0x24, 0xc6, 0xfb, 0x47, 0x4f, 0x1f, 0xe1,
+ 0x2c, 0x2a, 0x20, 0xa8, 0xcb, 0xec, 0x49, 0xa9, 0xd1, 0x74, 0xd5, 0x58, 0x5e, 0x93, 0xf8, 0xd8,
+ 0x0b, 0x92, 0x94, 0x9b, 0xce, 0x50, 0x23, 0x3e, 0xde, 0x4f, 0x52, 0x2e, 0xc3, 0xbc, 0x7c, 0xe6,
+ 0x69, 0x3c, 0x9b, 0xab, 0xbb, 0xd2, 0x74, 0xd7, 0x83, 0x24, 0x7d, 0x1c, 0xcf, 0xe6, 0xce, 0xff,
+ 0xa8, 0x5a, 0x18, 0xe3, 0xd0, 0xf5, 0xe3, 0x90, 0x46, 0xf7, 0xf1, 0x59, 0x41, 0x42, 0x56, 0x77,
+ 0xd9, 0x98, 0xf0, 0x6d, 0x05, 0x3a, 0xf7, 0xa6, 0x38, 0x16, 0xf7, 0xb1, 0xf0, 0xc9, 0x4c, 0xd5,
+ 0x56, 0x67, 0x98, 0x71, 0x42, 0x63, 0x73, 0xf0, 0x2d, 0x28, 0x4b, 0x63, 0x12, 0x13, 0xe1, 0x85,
+ 0x3e, 0x8e, 0x68, 0x6c, 0xbc, 0x00, 0x12, 0x75, 0x5f, 0x61, 0xd0, 0x9b, 0xd0, 0xd7, 0x9d, 0x3b,
+ 0xef, 0xc4, 0x8f, 0xc3, 0x99, 0xbc, 0x72, 0xba, 0x93, 0xd1, 0xd3, 0xe8, 0x03, 0x83, 0x45, 0x6f,
+ 0xc1, 0x86, 0xb9, 0x10, 0x39, 0x65, 0x5d, 0x51, 0xf6, 0x0d, 0xbe, 0x44, 0x9a, 0x26, 0x09, 0x65,
+ 0x82, 0x7b, 0x1c, 0x07, 0x01, 0x8d, 0x12, 0x53, 0x98, 0xf4, 0x2d, 0x7e, 0xa2, 0xd1, 0xce, 0x14,
+ 0x36, 0x1f, 0x4a, 0x3b, 0x8d, 0x25, 0xf9, 0x06, 0xf7, 0x22, 0x1c, 0x79, 0xc7, 0x33, 0x1a, 0x9c,
+ 0x7a, 0x32, 0x4c, 0x19, 0x0f, 0xcb, 0xd4, 0x67, 0x4f, 0x22, 0x27, 0xe4, 0x1b, 0x55, 0x83, 0x4b,
+ 0xaa, 0x13, 0x2a, 0x92, 0x59, 0x3a, 0xf5, 0x12, 0x46, 0x8f, 0xb1, 0x31, 0xb1, 0x1f, 0xe1, 0xe8,
+ 0x40, 0xe3, 0x8f, 0x24, 0xda, 0xf9, 0x53, 0x05, 0xb6, 0xca, 0x92, 0x4c, 0xd0, 0xdd, 0x85, 0xad,
+ 0xb2, 0x28, 0xf3, 0x10, 0xeb, 0x44, 0x6f, 0x50, 0x14, 0xa8, 0x9f, 0xe4, 0x0f, 0xa1, 0xab, 0xda,
+ 0xb9, 0x5e, 0xa8, 0x39, 0x95, 0xd3, 0x8f, 0xe2, 0xbe, 0xb8, 0x1d, 0xbf, 0xb8, 0x4b, 0x1f, 0xc3,
+ 0x35, 0x63, 0xbe, 0xb7, 0xac, 0xb6, 0x3e, 0x10, 0xdb, 0x86, 0xe0, 0xd1, 0x82, 0xf6, 0x5f, 0xc0,
+ 0x30, 0x47, 0xed, 0xcd, 0x15, 0xd2, 0xfa, 0xea, 0x5d, 0xd8, 0x5c, 0x30, 0xf6, 0x5e, 0x18, 0x32,
+ 0x75, 0x41, 0xeb, 0xee, 0xaa, 0x29, 0xe7, 0x2e, 0x5c, 0x9d, 0x60, 0xa1, 0xbd, 0xe1, 0x0b, 0x53,
+ 0x13, 0x68, 0x66, 0x1b, 0x50, 0x9b, 0xe0, 0x40, 0x19, 0x5f, 0x73, 0xe5, 0x50, 0x1e, 0xc0, 0xa7,
+ 0x1c, 0x07, 0xca, 0xca, 0x9a, 0xab, 0xc6, 0xce, 0x1f, 0x2b, 0xb0, 0x6e, 0xc2, 0xa4, 0x0c, 0xf5,
+ 0x21, 0x23, 0x67, 0x98, 0x99, 0xa3, 0x67, 0x20, 0xf4, 0x06, 0xf4, 0xf4, 0xc8, 0xa3, 0x89, 0x20,
+ 0x34, 0x0b, 0xbe, 0x5d, 0x8d, 0x7d, 0xac, 0x91, 0xaa, 0x53, 0xa7, 0x1a, 0x51, 0xa6, 0xe6, 0x33,
+ 0x90, 0x6a, 0xb7, 0x71, 0x19, 0x19, 0x54, 0xb0, 0x6d, 0xb9, 0x06, 0x92, 0x47, 0xdd, 0xf2, 0x5b,
+ 0x53, 0xfc, 0x2c, 0x28, 0x8f, 0x7a, 0x44, 0xd3, 0x58, 0x78, 0x09, 0x25, 0xb1, 0x30, 0xd1, 0x15,
+ 0x14, 0xea, 0x48, 0x62, 0x9c, 0x5f, 0x55, 0xa0, 0xa1, 0xbb, 0xd5, 0xb2, 0xca, 0xcc, 0xde, 0xb8,
+ 0x2a, 0x51, 0xf9, 0x82, 0x92, 0xa5, 0xdf, 0x35, 0x35, 0x96, 0xf7, 0xf8, 0x2c, 0xd2, 0x91, 0xda,
+ 0xa8, 0x76, 0x16, 0xa9, 0x10, 0xfd, 0x06, 0xf4, 0xf2, 0xa7, 0x52, 0xcd, 0x6b, 0x15, 0xbb, 0x19,
+ 0x56, 0x91, 0x5d, 0xa8, 0xa9, 0xf3, 0x13, 0x59, 0x5c, 0x67, 0x9d, 0xda, 0x0d, 0xa8, 0xa5, 0x99,
+ 0x32, 0x72, 0x28, 0x31, 0xd3, 0xec, 0x91, 0x95, 0x43, 0x74, 0x1b, 0x7a, 0x7e, 0x18, 0x12, 0xb9,
+ 0xdc, 0x9f, 0x3d, 0x24, 0x61, 0x76, 0x49, 0xcb, 0x58, 0xe7, 0x2f, 0x15, 0xe8, 0xef, 0xd3, 0x64,
+ 0xfe, 0xff, 0x64, 0x86, 0x0b, 0x11, 0x44, 0x29, 0x69, 0xde, 0x58, 0x39, 0x96, 0x79, 0xe3, 0x33,
+ 0x32, 0xc3, 0xfa, 0x6a, 0xe9, 0x9d, 0x6d, 0x4a, 0x84, 0xba, 0x56, 0x76, 0x32, 0x6b, 0x80, 0x75,
+ 0xf5, 0xe4, 0x23, 0x1a, 0xaa, 0x0c, 0x39, 0x24, 0xcc, 0xcb, 0xda, 0x5d, 0x5d, 0x77, 0x3d, 0x24,
+ 0x4c, 0x4d, 0x19, 0x43, 0xd6, 0x54, 0xc7, 0xb5, 0x68, 0x48, 0x43, 0x63, 0xa4, 0x21, 0xdb, 0xd0,
+ 0xa0, 0xcf, 0x9e, 0x71, 0x2c, 0x54, 0x2e, 0x5b, 0x73, 0x0d, 0x94, 0x85, 0xb9, 0x66, 0x21, 0xcc,
+ 0x5d, 0x81, 0x4d, 0xd5, 0xdb, 0x7f, 0xc2, 0xfc, 0x80, 0xc4, 0x53, 0x1b, 0x8a, 0xb7, 0x00, 0x4d,
+ 0x04, 0x4d, 0x16, 0xb0, 0x63, 0x18, 0x98, 0x37, 0xe7, 0xe8, 0xc7, 0x13, 0x6b, 0xfa, 0x35, 0x68,
+ 0x4a, 0xd0, 0x63, 0xf8, 0x6b, 0x1b, 0x18, 0xcd, 0xb4, 0xf3, 0x16, 0x74, 0xf4, 0xd0, 0x84, 0x81,
+ 0x9c, 0x94, 0x97, 0x49, 0xf9, 0x9d, 0xbf, 0x6d, 0x98, 0x70, 0x6b, 0x6a, 0x68, 0xf4, 0x10, 0xfa,
+ 0x0b, 0xff, 0x64, 0x90, 0x69, 0xaa, 0xac, 0xfe, 0x55, 0x33, 0xda, 0x1e, 0xeb, 0x7f, 0x3c, 0x63,
+ 0xfb, 0x8f, 0x67, 0xfc, 0x20, 0x4a, 0xc4, 0x1c, 0x3d, 0x80, 0x5e, 0xf9, 0xef, 0x05, 0xba, 0x6e,
+ 0x73, 0x90, 0x15, 0xff, 0x34, 0x2e, 0x64, 0xf3, 0x10, 0xfa, 0x0b, 0x3f, 0x32, 0xac, 0x3e, 0xab,
+ 0xff, 0x6f, 0x5c, 0xc8, 0xe8, 0x2e, 0xb4, 0x0b, 0x7f, 0x2e, 0xd0, 0x50, 0x33, 0x59, 0xfe, 0x99,
+ 0x71, 0x21, 0x83, 0x7d, 0xe8, 0x96, 0x7e, 0x26, 0xa0, 0x91, 0xb1, 0x67, 0xc5, 0x1f, 0x86, 0x0b,
+ 0x99, 0xec, 0x41, 0xbb, 0xd0, 0xd3, 0xb7, 0x5a, 0x2c, 0xff, 0x38, 0x18, 0x5d, 0x5b, 0x31, 0x63,
+ 0xb6, 0xf3, 0x00, 0xba, 0xa5, 0x0e, 0xbc, 0x55, 0x64, 0x55, 0xf7, 0x7f, 0x74, 0x7d, 0xe5, 0x9c,
+ 0xe1, 0xf4, 0x10, 0xfa, 0x0b, 0xfd, 0x78, 0xeb, 0xdc, 0xd5, 0x6d, 0xfa, 0x0b, 0xcd, 0xfa, 0x5c,
+ 0x6d, 0x76, 0xa1, 0xdc, 0x2a, 0x6c, 0xf6, 0x72, 0xf7, 0x7d, 0x74, 0x63, 0xf5, 0xa4, 0xd1, 0xea,
+ 0x01, 0xf4, 0xca, 0x8d, 0x77, 0xcb, 0x6c, 0x65, 0x3b, 0xfe, 0xf2, 0x93, 0x53, 0xea, 0xc1, 0xe7,
+ 0x27, 0x67, 0x55, 0x6b, 0xfe, 0x42, 0x46, 0xf7, 0x00, 0x4c, 0x71, 0x15, 0x92, 0x38, 0xdb, 0xb2,
+ 0xa5, 0xa2, 0x2e, 0xdb, 0xb2, 0x15, 0x85, 0xd8, 0x5d, 0x00, 0x5d, 0x13, 0x85, 0x34, 0x15, 0xe8,
+ 0xaa, 0x55, 0x63, 0xa1, 0x10, 0x1b, 0x0d, 0x97, 0x27, 0x96, 0x18, 0x60, 0xc6, 0x5e, 0x84, 0xc1,
+ 0x67, 0x00, 0x79, 0xad, 0x65, 0x19, 0x2c, 0x55, 0x5f, 0x97, 0xf8, 0xa0, 0x53, 0xac, 0xac, 0x90,
+ 0xb1, 0x75, 0x45, 0xb5, 0x75, 0x09, 0x8b, 0xfe, 0x42, 0xe6, 0x5c, 0x3e, 0x6c, 0x8b, 0x09, 0xf5,
+ 0x68, 0x29, 0x7b, 0x46, 0x1f, 0x42, 0xa7, 0x98, 0x32, 0x5b, 0x2d, 0x56, 0xa4, 0xd1, 0xa3, 0x52,
+ 0xda, 0x8c, 0xee, 0x42, 0xaf, 0x9c, 0x10, 0xa3, 0xc2, 0xbd, 0x58, 0x4a, 0x93, 0x47, 0xa6, 0x19,
+ 0x54, 0x20, 0x7f, 0x1f, 0x20, 0x4f, 0x9c, 0xad, 0xfb, 0x96, 0x52, 0xe9, 0x05, 0xa9, 0x9f, 0x41,
+ 0xaf, 0x10, 0xb7, 0x65, 0x4d, 0x78, 0xb5, 0x64, 0x70, 0x1e, 0xcd, 0x47, 0x26, 0xc3, 0x2a, 0x85,
+ 0xed, 0x7b, 0xd0, 0x29, 0xbe, 0x11, 0xd6, 0xda, 0x15, 0xef, 0xc6, 0x65, 0x41, 0xaf, 0xf0, 0x9e,
+ 0xd8, 0xb3, 0xbb, 0xfc, 0xc4, 0x5c, 0x16, 0xf4, 0x4a, 0xf5, 0xa8, 0x8d, 0x35, 0xab, 0x8a, 0xd4,
+ 0xcb, 0x9e, 0x82, 0x72, 0xf1, 0x66, 0xbd, 0xbf, 0xb2, 0xa4, 0xbb, 0xec, 0x0c, 0x16, 0xeb, 0x14,
+ 0xeb, 0x8f, 0x15, 0xb5, 0xcb, 0xf7, 0xc4, 0x84, 0x62, 0x2d, 0x52, 0x88, 0x09, 0x2b, 0x4a, 0x94,
+ 0x0b, 0x19, 0x1d, 0x40, 0xff, 0xa1, 0x4d, 0x33, 0x4d, 0x0a, 0x6c, 0xd4, 0x59, 0x91, 0xf2, 0x8f,
+ 0x46, 0xab, 0xa6, 0xcc, 0x2e, 0x7f, 0x0e, 0x83, 0xa5, 0xf4, 0x17, 0xdd, 0xcc, 0x5a, 0x9e, 0x2b,
+ 0xf3, 0xe2, 0x0b, 0xd5, 0x3a, 0x84, 0x8d, 0xc5, 0xec, 0x17, 0xbd, 0x62, 0x36, 0x7d, 0x75, 0x56,
+ 0x7c, 0x21, 0xab, 0x8f, 0xa1, 0x69, 0xb3, 0x2d, 0x64, 0x5a, 0xcb, 0x0b, 0xd9, 0xd7, 0x45, 0x4b,
+ 0xf7, 0x3a, 0xdf, 0x7e, 0x77, 0xb3, 0xf2, 0xd7, 0xef, 0x6e, 0x56, 0xfe, 0xf1, 0xdd, 0xcd, 0xca,
+ 0x71, 0x43, 0xcd, 0xbe, 0xff, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x45, 0xa8, 0x91, 0xab, 0x62,
+ 0x22, 0x00, 0x00,
}
diff --git a/protocols/grpc/agent.proto b/protocols/grpc/agent.proto
index b0fab6d..d863645 100644
--- a/protocols/grpc/agent.proto
+++ b/protocols/grpc/agent.proto
@@ -46,6 +46,7 @@ service AgentService {
rpc UpdateRoutes(UpdateRoutesRequest) returns (Routes);
rpc ListInterfaces(ListInterfacesRequest) returns(Interfaces);
rpc ListRoutes(ListRoutesRequest) returns (Routes);
+ rpc UpdateIPVSRule(UpdateIPVSRequest) returns (IPVSResponse);
// tracing
rpc StartTracing(StartTracingRequest) returns (google.protobuf.Empty);
@@ -495,3 +496,13 @@ message StartTracingRequest {
message StopTracingRequest {
}
+
+message UpdateIPVSRequest {
+ // IPVS_req is the IPVS rule message needed to update
+ string IPVS_req = 1;
+}
+
+message IPVSResponse {
+ // IPVS_res is the response of IPVS updating
+ string IPVS_res = 1;
+}
--
2.14.3 (Apple Git-98)

193
agent/patches/0004-agent-add-IPVS-test.patch Normal file
View File

@ -0,0 +1,193 @@
From 01563c08910ddaba4077fd9dc691df541e045165 Mon Sep 17 00:00:00 2001
From: xiadanni <xiadanni1@huawei.com>
Date: Tue, 18 Aug 2020 17:05:32 +0800
Subject: [PATCH 04/16] agent: add IPVS test
Signed-off-by: xiadanni <xiadanni1@huawei.com>
---
grpc_test.go | 172 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 172 insertions(+)
diff --git a/grpc_test.go b/grpc_test.go
index e69102b..d58c0b6 100644
--- a/grpc_test.go
+++ b/grpc_test.go
@@ -1840,3 +1840,175 @@ func getPipeMaxSize() (uint32, error) {
u, err := strconv.ParseUint(s, 10, 32)
return uint32(u), err
}
+
+func TestUpdateIPVSRule(t *testing.T) {
+ assert := assert.New(t)
+
+ // add IPVS rule successfully
+ a := &agentGRPC{
+ sandbox: &sandbox{
+ containers: make(map[string]*container),
+ },
+ }
+
+ req := &pb.UpdateIPVSRequest{
+ IPVSReq: "ipvsadm -A -t 17.2.0.7:80 -s rr -p 3000",
+ }
+
+ _, err := a.UpdateIPVSRule(context.Background(), req)
+ assert.NoError(err)
+
+ // delete ipvs rule successfully
+ req = &pb.UpdateIPVSRequest{
+ IPVSReq: "ipvsadm -D -t 17.2.0.7:80",
+ }
+
+ _, err = a.UpdateIPVSRule(context.Background(), req)
+ assert.NoError(err)
+
+ // update ipvs rule error because exec failed
+ req = &pb.UpdateIPVSRequest{
+ IPVSReq: "ipvsadm -A -t 17.2.0.7:80 -s rr -p -3000",
+ }
+
+ _, err = a.UpdateIPVSRule(context.Background(), req)
+ assert.Error(err)
+ assert.Contains(err.Error(), "exec IPVS command failed")
+
+ // update IPVS rule error because rule less than validHeadLength
+ req = &pb.UpdateIPVSRequest{
+ IPVSReq: "ipvsa",
+ }
+
+ _, err = a.UpdateIPVSRule(context.Background(), req)
+ assert.Error(err)
+ assert.Contains(err.Error(), "invalid IPVS rule")
+
+ // update ipvs rule error because invalid command
+ req = &pb.UpdateIPVSRequest{
+ IPVSReq: "abcabcabc ipvsadm",
+ }
+
+ _, err = a.UpdateIPVSRule(context.Background(), req)
+ assert.Error(err)
+ assert.Contains(err.Error(), "invalid IPVS rule")
+
+ // add ipvs rule error because rule count exceeds
+ a = &agentGRPC{
+ sandbox: &sandbox{
+ containers: make(map[string]*container),
+ ipvsadm: ipvsAdm{
+ ipvsRuleCnt: 20000,
+ },
+ },
+ }
+
+ req = &pb.UpdateIPVSRequest{
+ IPVSReq: "ipvsadm -A -t 17.2.0.7:80 -s rr -p 3000",
+ }
+ _, err = a.UpdateIPVSRule(context.Background(), req)
+ assert.Error(err)
+ assert.Errorf(err, "rules exceed limit")
+
+ // add ipvs rule error because ipvs request item less than 2
+ a = &agentGRPC{
+ sandbox: &sandbox{
+ containers: make(map[string]*container),
+ ipvsadm: ipvsAdm{
+ ipvsRuleCnt: 0,
+ },
+ },
+ }
+
+ req = &pb.UpdateIPVSRequest{
+ IPVSReq: "ipvsadm",
+ }
+
+ _, err = a.UpdateIPVSRule(context.Background(), req)
+ assert.Error(err)
+ assert.Contains(err.Error(), "invalid IPVS rule")
+
+ // add ipvs rule error because ipvs rule nil
+ req = nil
+
+ _, err = a.UpdateIPVSRule(context.Background(), req)
+ assert.Error(err)
+ assert.Contains(err.Error(), "IPVS rule is nil")
+
+ // add ipvs rule error because ipvs rule string is empty
+ req = &pb.UpdateIPVSRequest{
+ IPVSReq: "",
+ }
+
+ _, err = a.UpdateIPVSRule(context.Background(), req)
+ assert.Error(err)
+ assert.Contains(err.Error(), "IPVS rule is nil")
+
+ // restore ipvs rule successfully
+ a = &agentGRPC{
+ sandbox: &sandbox{
+ containers: make(map[string]*container),
+ },
+ }
+
+ req = &pb.UpdateIPVSRequest{
+ IPVSReq: "restore|2|-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m",
+ }
+
+ _, err = a.UpdateIPVSRule(context.Background(), req)
+ assert.NoError(err)
+
+ // clear IPVS rule successfully
+ req = &pb.UpdateIPVSRequest{
+ IPVSReq: "ipvsadm -C",
+ }
+
+ _, err = a.UpdateIPVSRule(context.Background(), req)
+ assert.NoError(err)
+ assert.Equal(a.sandbox.ipvsadm.ipvsRuleCnt, uint64(0))
+
+ // restore ipvs rule error because rule count invalid
+ req = &pb.UpdateIPVSRequest{
+ IPVSReq: "restore|abc|-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m",
+ }
+
+ _, err = a.UpdateIPVSRule(context.Background(), req)
+ assert.Error(err)
+ assert.Contains(err.Error(), "invalid IPVS rule")
+
+ // restore ipvs rule error because other rules exists
+ a = &agentGRPC{
+ sandbox: &sandbox{
+ containers: make(map[string]*container),
+ ipvsadm: ipvsAdm{
+ ipvsRuleCnt: 5,
+ },
+ },
+ }
+
+ req = &pb.UpdateIPVSRequest{
+ IPVSReq: "restore|2|-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m",
+ }
+
+ _, err = a.UpdateIPVSRule(context.Background(), req)
+ assert.Error(err)
+ assert.Contains(err.Error(), "exist some rules in system")
+
+ // restore ipvs rule error because ipvs req item less than 3
+ a = &agentGRPC{
+ sandbox: &sandbox{
+ containers: make(map[string]*container),
+ ipvsadm: ipvsAdm{
+ ipvsRuleCnt: 0,
+ },
+ },
+ }
+
+ req = &pb.UpdateIPVSRequest{
+ IPVSReq: "restore|-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m",
+ }
+
+ _, err = a.UpdateIPVSRule(context.Background(), req)
+ assert.Error(err)
+ assert.Contains(err.Error(), "invalid IPVS rule")
+}
--
2.14.3 (Apple Git-98)

42
agent/patches/0005-mount-support-mount-block-device.patch Normal file
View File

@ -0,0 +1,42 @@
From 8c9f9be2a9c195d0bc12b43c491adaacb7bb8154 Mon Sep 17 00:00:00 2001
From: holyfei <yangfeiyu20092010@163.com>
Date: Tue, 18 Aug 2020 10:42:38 +0800
Subject: [PATCH 05/16] mount: support mount block device
reason: modify mountStorage to support mount block device
"-v /dev/blockdevice:/home/test"
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
---
mount.go | 15 ++++++++++++++-
1 file changed, 14 insertions(+), 1 deletion(-)
diff --git a/mount.go b/mount.go
index f0c3efe..a05d4a6 100644
--- a/mount.go
+++ b/mount.go
@@ -364,7 +364,20 @@ func commonStorageHandler(storage pb.Storage) (string, error) {
func mountStorage(storage pb.Storage) error {
flags, options := parseMountFlagsAndOptions(storage.Options)
- return mount(storage.Source, storage.MountPoint, storage.Fstype, flags, options)
+ var fsType = storage.Fstype
+ if (storage.Driver == driverSCSIType || storage.Driver == driverBlkType) && strings.Contains(storage.Fstype, "bind") {
+ cs := strings.Split(storage.Fstype, "-")
+ if len(cs) == 2 && cs[1] != "" {
+ fsType = cs[1]
+ // here we temporarily discard the bind option,
+ // in order to be able to mount the file system of the block device.
+ // and then reset `storage.Fstype` to "bind" which pass through to the libcontainer pkg.
+ flags = flags &^ flagList["bind"]
+ storage.Fstype = "bind"
+ }
+ }
+
+ return mount(storage.Source, storage.MountPoint, fsType, flags, options)
}
// addStorages takes a list of storages passed by the caller, and perform the
--
2.14.3 (Apple Git-98)

54
agent/patches/0006-agent-make-workaround-for-slow-response-in-aarch64.patch Normal file
View File

@ -0,0 +1,54 @@
From bccda1d208f31eab55863883cf0718d7b4b8deef Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Tue, 18 Aug 2020 19:30:42 +0800
Subject: [PATCH 06/16] agent: make workaround for slow response in aarch64
reason: make workaround for slow response in aarch64
when hotplug virtio-net-pci device
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
agent.go | 2 +-
device.go | 12 ++++++------
2 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/agent.go b/agent.go
index c161e93..e81c2cd 100644
--- a/agent.go
+++ b/agent.go
@@ -185,7 +185,7 @@ var logsVSockPort = uint32(0)
var debugConsoleVSockPort = uint32(0)
// Timeout waiting for a device to be hotplugged
-var hotplugTimeout = 3 * time.Second
+var hotplugTimeout = 10 * time.Second
// Specify the log level
var logLevel = defaultLogLevel
diff --git a/device.go b/device.go
index ec1907e..46a1b96 100644
--- a/device.go
+++ b/device.go
@@ -179,13 +179,13 @@ func getPCIDeviceNameImpl(s *sandbox, pciID string) (string, error) {
return "", err
}
- fieldLogger := agentLog.WithField("pciAddr", pciAddr)
-
// Rescan pci bus if we need to wait for a new pci device
- if err = rescanPciBus(); err != nil {
- fieldLogger.WithError(err).Error("Failed to scan pci bus")
- return "", err
- }
+ // FIXME:Comment out this code Temporarily, because once the PCIBus is scanned,
+ // the device hot-plug event is lost
+ //if err = rescanPciBus(); err != nil {
+ // fieldLogger.WithError(err).Error("Failed to scan pci bus")
+ // return "", err
+ //}
return getDeviceName(s, pciAddr)
}
--
2.14.3 (Apple Git-98)

37
agent/patches/0007-agent-using-pcie-root-port-driver-to-hotplug-device.patch Normal file
View File

@ -0,0 +1,37 @@
From fa673c93e243ba297d53b585cd2f51fa68380fc5 Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Tue, 18 Aug 2020 19:37:48 +0800
Subject: [PATCH 07/16] agent: using pcie-root-port driver to hotplug device
reason: In original pci-bridge scheme, the "F" in the BDF is not used,
and was written hard code "0" in the kata-agent. But the "function"
is specified when switching to pcie-root-port scheme, so when should
pass the "pci-bridge BDF" or "pcie-root-port BDF" from kata-runtime
and use in kata-agent.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
device.go | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/device.go b/device.go
index 46a1b96..8e6950c 100644
--- a/device.go
+++ b/device.go
@@ -101,7 +101,12 @@ func getDevicePCIAddressImpl(pciID string) (string, error) {
// Deduce the complete bridge address based on the bridge address identifier passed
// and the fact that bridges are attached on the main bus with function 0.
- pciBridgeAddr := fmt.Sprintf("0000:00:%s.0", bridgeID)
+ // Update: support pcie-root-port device
+ // In original pci-bridge scheme, the "F" in the BDF is not used, and was written
+ // hard code "0" in the kata-agent. But the "function" is specified when switching to
+ // pcie-root-port scheme, so when should pass the "pci-bridge BDF" or "pcie-root-port BDF"
+ // from kata-runtime and use in kata-agent.
+ pciBridgeAddr := fmt.Sprintf("0000:00:%s", bridgeID)
// Find out the bus exposed by bridge
bridgeBusPath := fmt.Sprintf(pciBusPathFormat, sysBusPrefix, pciBridgeAddr)
--
2.14.3 (Apple Git-98)

94
agent/patches/0008-agent-support-get-root-bus-path-dynamically.patch Normal file
View File

@ -0,0 +1,94 @@
From eea286fbafba2e95410b603fbef762e2b25eb207 Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Tue, 18 Aug 2020 19:45:57 +0800
Subject: [PATCH 08/16] agent: support get root bus path dynamically
reason: support get root bus dynamically no matter the
target arch is amd64 or arm64
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
agent.go | 1 +
device_amd64.go | 6 +++++-
device_arm64.go | 27 ++++++++++++++++++++++++++-
3 files changed, 32 insertions(+), 2 deletions(-)
diff --git a/agent.go b/agent.go
index e81c2cd..50afd7a 100644
--- a/agent.go
+++ b/agent.go
@@ -730,6 +730,7 @@ func (s *sandbox) listenToUdevEvents() {
defer uEvHandler.Close()
fieldLogger.Infof("Started listening for uevents")
+ rootBusPath := initRootBusPath()
for {
uEv, err := uEvHandler.Read()
diff --git a/device_amd64.go b/device_amd64.go
index 66bc052..26f55bf 100644
--- a/device_amd64.go
+++ b/device_amd64.go
@@ -8,7 +8,7 @@
package main
const (
- rootBusPath = "/devices/pci0000:00"
+ defaultRootBusPath = "/devices/pci0000:00"
// From https://www.kernel.org/doc/Documentation/acpi/namespace.txt
// The Linux kernel's core ACPI subsystem creates struct acpi_device
@@ -21,3 +21,7 @@ const (
// in a subdirectory whose prefix is pfn (page frame number).
pfnDevPrefix = "/pfn"
)
+
+func initRootBusPath() string {
+ return defaultRootBusPath
+}
diff --git a/device_arm64.go b/device_arm64.go
index b73b582..d039c67 100644
--- a/device_arm64.go
+++ b/device_arm64.go
@@ -6,8 +6,14 @@
package main
+import (
+ "fmt"
+ "io/ioutil"
+ "regexp"
+)
+
const (
- rootBusPath = "/devices/platform/4010000000.pcie/pci0000:00"
+ defaultRootBusPath = "/devices/platform/4010000000.pcie/pci0000:00"
// From https://www.kernel.org/doc/Documentation/acpi/namespace.txt
// The Linux kernel's core ACPI subsystem creates struct acpi_device
@@ -20,3 +26,22 @@ const (
// in a subdirectory whose prefix is pfn (page frame number).
pfnDevPrefix = "/pfn"
)
+
+func initRootBusPath() string {
+ pcieDriverReg := regexp.MustCompile(`^[0-9a-f]{10}.pcie$`)
+ rootBusPath := defaultRootBusPath
+ files, err := ioutil.ReadDir("/sys/devices/platform")
+ if err != nil {
+ return rootBusPath
+ }
+ for _, f := range files {
+ if !f.IsDir() {
+ continue
+ }
+ if pcieDriverReg.MatchString(f.Name()) {
+ rootBusPath = fmt.Sprintf("/devices/platform/%s/pci0000:00", f.Name())
+ break
+ }
+ }
+ return rootBusPath
+}
--
2.14.3 (Apple Git-98)

188
agent/patches/0009-storage-add-pkg-storage-for-mount.patch Normal file
View File

@ -0,0 +1,188 @@
From 1268b710c7f0528d971d9c0e54429d3f4e48c372 Mon Sep 17 00:00:00 2001
From: holyfei <yangfeiyu20092010@163.com>
Date: Tue, 18 Aug 2020 16:45:29 +0800
Subject: [PATCH 09/16] storage: add pkg/storage for mount
reason: add gpath.go and nfs.go, provide mount
functions and structs
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
---
pkg/storage/gpath.go | 64 ++++++++++++++++++++++++++++++++++++
pkg/storage/nfs.go | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 157 insertions(+)
create mode 100644 pkg/storage/gpath.go
create mode 100644 pkg/storage/nfs.go
diff --git a/pkg/storage/gpath.go b/pkg/storage/gpath.go
new file mode 100644
index 0000000..5cb951f
--- /dev/null
+++ b/pkg/storage/gpath.go
@@ -0,0 +1,64 @@
+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+// Description: common functions
+// Author: licuifang
+// Create: 2019-06-24
+
+package storage
+
+import (
+ "fmt"
+ "os"
+ "syscall"
+
+ "github.com/opencontainers/runc/libcontainer/mount"
+)
+
+func ValidateGpath(guestPath string, opts []string) error {
+ for _, opt := range opts {
+ switch opt {
+ case "shared":
+ flag := syscall.MS_SHARED
+ if err := ensureMountedAs(guestPath, flag); err != nil {
+ return err
+ }
+ return nil
+ case "mounted":
+ // if mounted in option, the guestpath must exist and must be a mountpoint
+ _, err := os.Stat(guestPath)
+ if err != nil {
+ return err
+ }
+ mountpoint, err := mount.Mounted(guestPath)
+ if err != nil {
+ return err
+ }
+ if mountpoint == false {
+ return fmt.Errorf("the guespath:%s is not a mountpoint while mounted was set", guestPath)
+ }
+ return nil
+ }
+ }
+ if err := os.MkdirAll(guestPath, 0750); err != nil {
+ return err
+ }
+ return nil
+}
+
+func ensureMountedAs(mountPoint string, flag int) error {
+ if err := os.MkdirAll(mountPoint, 0750); err != nil {
+ return err
+ }
+ mounted, err := mount.Mounted(mountPoint)
+ if err != nil {
+ return err
+ }
+
+ if !mounted {
+ if err := syscall.Mount(mountPoint, mountPoint, "bind", uintptr(syscall.MS_BIND), ""); err != nil {
+ return err
+ }
+ }
+
+ return syscall.Mount("", mountPoint, "", uintptr(flag), "")
+}
diff --git a/pkg/storage/nfs.go b/pkg/storage/nfs.go
new file mode 100644
index 0000000..44bc85d
--- /dev/null
+++ b/pkg/storage/nfs.go
@@ -0,0 +1,93 @@
+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+// Description: common functions
+// Author: leizhongkai
+// Create: 2019-03-10
+
+package storage
+
+import (
+ "fmt"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "time"
+
+ "github.com/opencontainers/runc/libcontainer/mount"
+ "github.com/sirupsen/logrus"
+)
+
+const (
+ NFS = "nfs"
+
+ MAX_MOUNT_ATTEMPTS = 100
+ MAX_MOUTN_TIMEOUT = 90 // seconds
+
+ // ignored errors
+ PERMISSION_DENY = "Permission denied"
+ kataGuestStorageDir = "/run/kata-containers/storage/containers/"
+)
+
+func nfsMount(source, dest string, opt string) error {
+ cmd := exec.Command("/bin/mount", "-t", "nfs", "-o", opt, source, dest)
+ res, err := cmd.Output()
+ logrus.Debugf("mount %s to %s, and get res %s", source, dest, string(res))
+ return err
+}
+
+func nfsMountWithAttempt(source, dest string, opt string) (err error) {
+ timeStart := time.Now().Unix()
+ for i := 0; i < MAX_MOUNT_ATTEMPTS; i++ {
+ logrus.Infof("this is the %d times to mount %s to %s", i, source, dest)
+ err = nfsMount(source, dest, opt)
+ if err != nil {
+ ee, ok := err.(*exec.ExitError)
+ if ok {
+ if strings.Contains(string(ee.Stderr), PERMISSION_DENY) {
+ logrus.Errorf("mounting nfs:%s to %s, get error: %s,should break", source, dest, string(ee.Stderr))
+ // We do not retry when the error type is PERMISSION_DENY.
+ // The reason for the retry is that when you do a SFS mount,
+ // the network may not have been created yet, because
+ // creating the network and startup container is asynchronous
+ break
+ }
+ logrus.Infof("mounting nfs:%s to %s, get error: %s,will retry", source, dest, string(ee.Stderr))
+ }
+
+ elapsed := time.Now().Unix() - timeStart
+ if elapsed < MAX_MOUTN_TIMEOUT {
+ time.Sleep(50 * time.Microsecond)
+ continue
+ }
+ }
+ break
+ }
+
+ return err
+}
+
+func MountRemoteNfs(source string, opt []string, sandboxId string) error {
+ item := strings.Split(source, ":")
+ if len(item) != 2 {
+ return fmt.Errorf("the nfs of %s format is error", source)
+ }
+ tmpDes := filepath.Join(kataGuestStorageDir, sandboxId, item[0],item[1])
+ if mounted, err := mount.Mounted(tmpDes); err == nil && mounted == true {
+ return nil
+ }
+ err := os.MkdirAll(tmpDes, 0750)
+ if err != nil {
+ logrus.Infof("mkdir %s failed before mount remote nfs server", tmpDes)
+ return err
+ }
+ defer func() {
+ if err != nil {
+ os.RemoveAll(tmpDes)
+ }
+ }()
+
+ err = nfsMountWithAttempt(source, tmpDes, strings.Join(opt, ","))
+
+ return err
+}
--
2.14.3 (Apple Git-98)

92
agent/patches/0010-storage-mount-nfs-and-gpath-in-agent.patch Normal file
View File

@ -0,0 +1,92 @@
From 79bafc5fd8a1dcda6f44ecd830dfe50f4ef1b34b Mon Sep 17 00:00:00 2001
From: holyfei <yangfeiyu20092010@163.com>
Date: Tue, 18 Aug 2020 20:41:30 +0800
Subject: [PATCH 10/16] storage: mount nfs and gpath in agent
reason: add nfsStorageHandler and gpathStorageHandler in
storageHandlerList to mount nfs and gpath
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
---
device.go | 2 ++
mount.go | 37 +++++++++++++++++++++++++++++++++++++
2 files changed, 39 insertions(+)
diff --git a/device.go b/device.go
index 8e6950c..29f72e9 100644
--- a/device.go
+++ b/device.go
@@ -35,6 +35,8 @@ const (
driverEphemeralType = "ephemeral"
driverLocalType = "local"
vmRootfs = "/"
+ driverNfsType = "nfs"
+ driverGpathType = "gpath"
)
const (
diff --git a/mount.go b/mount.go
index a05d4a6..de2bfaf 100644
--- a/mount.go
+++ b/mount.go
@@ -17,6 +17,7 @@ import (
"strings"
"syscall"
+ rmtStorage "github.com/kata-containers/agent/pkg/storage"
pb "github.com/kata-containers/agent/protocols/grpc"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
@@ -218,6 +219,8 @@ var storageHandlerList = map[string]storageHandler{
driverEphemeralType: ephemeralStorageHandler,
driverLocalType: localStorageHandler,
driverNvdimmType: nvdimmStorageHandler,
+ driverNfsType: nfsStorageHandler,
+ driverGpathType: gpathStorageHandler,
}
func ephemeralStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (string, error) {
@@ -339,6 +342,40 @@ func nvdimmStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (st
return "", fmt.Errorf("invalid nvdimm source path: %v", storage.Source)
}
+func nfsStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (string, error) {
+ s.Lock()
+ defer s.Unlock()
+
+ // mount nfs
+ err := rmtStorage.MountRemoteNfs(storage.Source, storage.Options, s.id)
+ if err != nil {
+ return "", fmt.Errorf("mount %s to %s failed, get err:%s", storage.Source, storage.MountPoint, err)
+ }
+ if err := os.MkdirAll(storage.MountPoint, 0750); err != nil {
+ logrus.Infof("mkdir %s failed after mount remote nfs server", storage.MountPoint)
+ return "", err
+ }
+
+ defer func() {
+ if err != nil {
+ os.RemoveAll(storage.MountPoint)
+ }
+ }()
+
+ return storage.MountPoint, nil
+}
+
+func gpathStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (string, error) {
+ s.Lock()
+ defer s.Unlock()
+ // validate guespath
+ err := rmtStorage.ValidateGpath(storage.Source, storage.Options)
+ if err != nil {
+ return "", err
+ }
+ return "", nil
+}
+
// virtioSCSIStorageHandler handles the storage for scsi driver.
func virtioSCSIStorageHandler(ctx context.Context, storage pb.Storage, s *sandbox) (string, error) {
// Retrieve the device path from SCSI address.
--
2.14.3 (Apple Git-98)

78
agent/patches/0011-agent-fix-agent-reap-agent-process-blocked-problem.patch Normal file
View File

@ -0,0 +1,78 @@
From 3ac1232a2e3fbfc0465473e5d81cde41847c4252 Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Wed, 19 Aug 2020 11:47:37 +0800
Subject: [PATCH 11/16] agent: fix agent reap agent process blocked problem
reason: add container waitProcess() timeout when
container process status is D/T.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
grpc.go | 43 +++++++++++++++++++++++++++++++++----------
1 file changed, 33 insertions(+), 10 deletions(-)
diff --git a/grpc.go b/grpc.go
index de2cae7..3dd088e 100644
--- a/grpc.go
+++ b/grpc.go
@@ -49,6 +49,11 @@ const (
libcontainerPath = "/run/libcontainer"
)
+// keep waitProcessTimeout value same as value in kata-runtime wait WaitProcessRequest response
+const (
+ waitProcessTimeOut = 10
+)
+
var (
sysfsCPUOnlinePath = "/sys/devices/system/cpu"
sysfsMemOnlinePath = "/sys/devices/system/memory"
@@ -996,17 +1001,35 @@ func (a *agentGRPC) WaitProcess(ctx context.Context, req *pb.WaitProcessRequest)
ctr.deleteProcess(proc.id)
})
- // Using helper function wait() to deal with the subreaper.
- libContProcess := (*reaperLibcontainerProcess)(&(proc.process))
- exitCode, err := a.sandbox.subreaper.wait(proc.exitCodeCh, libContProcess)
- if err != nil {
- return &pb.WaitProcessResponse{}, err
+ done := make(chan error)
+ var exitCode int = 0
+ go func() {
+ // Using helper function wait() to deal with the subreaper.
+ libContProcess := (*reaperLibcontainerProcess)(&(proc.process))
+ var err error
+ exitCode, err = a.sandbox.subreaper.wait(proc.exitCodeCh, libContProcess)
+ if err != nil {
+ done <- err
+ close(done)
+ return
+ }
+ // refill the exitCodeCh with the exitcode which can be read out
+ // by another WaitProcess(). Since this channel isn't be closed,
+ // here the refill will always success and it will be free by GC
+ // once the process exits.
+ proc.exitCodeCh <- exitCode
+
+ close(done)
+ }()
+
+ select {
+ case err := <-done:
+ if err != nil {
+ return &pb.WaitProcessResponse{}, err
+ }
+ case <-time.After(time.Duration(waitProcessTimeOut) * time.Second):
+ return &pb.WaitProcessResponse{}, grpcStatus.Errorf(codes.DeadlineExceeded, "agent wait reap container process timeout reached after %ds", waitProcessTimeOut)
}
- //refill the exitCodeCh with the exitcode which can be read out
- //by another WaitProcess(). Since this channel isn't be closed,
- //here the refill will always success and it will be free by GC
- //once the process exits.
- proc.exitCodeCh <- exitCode
return &pb.WaitProcessResponse{
Status: int32(exitCode),
--
2.14.3 (Apple Git-98)

38
agent/patches/0012-network-support-set-dns-without-nameserver.patch Normal file
View File

@ -0,0 +1,38 @@
From edb29dfd8f786735763245b3f156b50fd3c1a08e Mon Sep 17 00:00:00 2001
From: holyfei <yangfeiyu20092010@163.com>
Date: Wed, 19 Aug 2020 15:15:31 +0800
Subject: [PATCH 12/16] network: support set dns without nameserver
reason: when runtime sends dns without nameserver to agent,
add nameserver before ip address. scenario like annotation
with dns
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
---
network.go | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/network.go b/network.go
index 02e28cb..7046cf8 100644
--- a/network.go
+++ b/network.go
@@ -708,6 +708,9 @@ func setupDNS(dns []string) (err error) {
defer file.Close()
for i, line := range dns {
+ if !strings.Contains(line, "nameserver") {
+ line = "nameserver" + " " + line
+ }
if i == (len(dns) - 1) {
_, err = file.WriteString(strings.TrimSpace(line))
} else {
@@ -761,4 +764,4 @@ func (s *sandbox) handleLocalhost() error {
}
return netlink.LinkSetUp(lo)
-}
\ No newline at end of file
+}
--
2.14.3 (Apple Git-98)

304
agent/patches/0013-agent-support-setting-multi-queues-of-interface.patch Normal file
View File

@ -0,0 +1,304 @@
From 1394dcf579849e5d8103c31556e9af0216a875d2 Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Wed, 19 Aug 2020 17:15:51 +0800
Subject: [PATCH 13/16] agent: support setting multi queues of interface
reason: support setting multi queues of a interface
when runtime passing Queue in the request.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
network.go | 12 +++++-
pkg/net/ethtool.go | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++
pkg/types/types.pb.go | 82 ++++++++++++++++++++++++++-----------
pkg/types/types.proto | 1 +
4 files changed, 181 insertions(+), 24 deletions(-)
create mode 100644 pkg/net/ethtool.go
diff --git a/network.go b/network.go
index 7046cf8..1baaa2e 100644
--- a/network.go
+++ b/network.go
@@ -17,7 +17,7 @@ import (
"syscall"
"golang.org/x/sys/unix"
-
+ agentNet "github.com/kata-containers/agent/pkg/net"
"github.com/kata-containers/agent/pkg/types"
pb "github.com/kata-containers/agent/protocols/grpc"
"github.com/sirupsen/logrus"
@@ -273,6 +273,16 @@ func (s *sandbox) updateInterface(netHandle *netlink.Handle, iface *types.Interf
if err == nil {
err = retErr
}
+
+ // if link is up, then set the multi queue to it,
+ // the kernel of newer version may set multi queue by itself,
+ // but we can not rely on it.
+ if err == nil && iface.Queues > 0 {
+ if ethErr := agentNet.GetEthtool().SetChannel(iface.Name, iface.Queues); ethErr != nil {
+ err = grpcStatus.Errorf(codes.Internal, "Could not set multi queue %d for interface %v: %v",
+ iface.Queues, link, ethErr)
+ }
+ }
}()
fieldLogger.WithField("link", fmt.Sprintf("%+v", link)).Info("Link found")
diff --git a/pkg/net/ethtool.go b/pkg/net/ethtool.go
new file mode 100644
index 0000000..56a1ece
--- /dev/null
+++ b/pkg/net/ethtool.go
@@ -0,0 +1,110 @@
+/*
+Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+Description: common functions
+Author: fengshaobao
+Create: 2019-05-28
+*/
+
+package net
+
+import (
+ "fmt"
+ "sync"
+ "syscall"
+ "unsafe"
+
+ "github.com/sirupsen/logrus"
+)
+
+const (
+ ethtoolGChannels = 0x0000003c /* Get no of channels */
+ ethtoolSChannels = 0x0000003d /* Set no of channels */
+ ifNameSize = 16
+ siocEthtool = 0x8946
+)
+
+var (
+ t *Ethtool
+ once sync.Once
+)
+
+// Ethtool to set multiqueue of a network interface,
+// with the public method "SetChannel" to set queues of an interface.
+type Ethtool struct {
+ fd int
+}
+
+type ifReq struct {
+ ifrName [ifNameSize]byte
+ ifrData uintptr
+}
+
+type ethtoolChannels struct {
+ // ETHTOOL_{G,S}CHANNELS
+ cmd uint32
+ // Read only. Maximum number of receive channel the driver support.
+ maxRx uint32
+ // Read only. Maximum number of transmit channel the driver support
+ maxTx uint32
+ // Read only. Maximum number of other channel the driver support
+ maxOther uint32
+ //Read only. Maximum number of combined channel the driver support. Set of queues RX, TX or other
+ maxCombined uint32
+ // Valid values are in the range 1 to the max_rx
+ rxCount uint32
+ // Valid values are in the range 1 to the max_tx
+ txCount uint32
+ // Valid values are in the range 1 to the max_other
+ otherCount uint32
+ // Valid values are in the range 1 to the max_combined
+ combinedCount uint32
+}
+
+// GetEthtool to config multiqueue of a network interface
+func GetEthtool() *Ethtool {
+ once.Do(func() {
+ var err error
+ if t, err = newEthtool(); err != nil {
+ panic("can not init a socket fd for ethtool")
+ }
+ })
+ return t
+}
+
+func newEthtool() (*Ethtool, error) {
+ fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
+ if err != nil || fd < 0 {
+ logrus.Warningf("Can not get socket of inet")
+ var newErr error
+ fd, newErr = syscall.Socket(syscall.AF_NETLINK, syscall.SOCK_RAW, syscall.NETLINK_GENERIC)
+ if newErr != nil || fd < 0 {
+ return nil, fmt.Errorf("create inet socket with error: %v, and create netlink socket with error: %v", err, newErr)
+ }
+ }
+
+ return &Ethtool{
+ fd: int(fd),
+ }, nil
+}
+
+// SetChannel Set the queues of a network interface.
+// @devName: the network interface name, e.g. eth0.
+// @queues: queues for the network interface.
+func (e *Ethtool) SetChannel(devName string, queues uint32) error {
+ c := &ethtoolChannels{
+ cmd: ethtoolSChannels,
+ combinedCount: queues,
+ }
+ var ifName [ifNameSize]byte
+ copy(ifName[:], []byte(devName))
+ ifr := ifReq{
+ ifrName: ifName,
+ ifrData: uintptr(unsafe.Pointer(&c)),
+ }
+ _, _, ep := syscall.Syscall(syscall.SYS_IOCTL, uintptr(e.fd), siocEthtool, uintptr(unsafe.Pointer(&ifr)))
+ if ep != 0 {
+ return syscall.Errno(ep)
+ }
+ return nil
+}
diff --git a/pkg/types/types.pb.go b/pkg/types/types.pb.go
index 7ea63e3..8b7e2a5 100644
--- a/pkg/types/types.pb.go
+++ b/pkg/types/types.pb.go
@@ -100,6 +100,7 @@ type Interface struct {
// list: "veth", "macvtap", "vlan", "macvlan", "tap", ...
Type string `protobuf:"bytes,7,opt,name=type,proto3" json:"type,omitempty"`
RawFlags uint32 `protobuf:"varint,8,opt,name=raw_flags,json=rawFlags,proto3" json:"raw_flags,omitempty"`
+ Queues uint32 `protobuf:"varint,9,opt,name=Queues,proto3" json:"Queues,omitempty"`
}
func (m *Interface) Reset() { *m = Interface{} }
@@ -163,6 +164,13 @@ func (m *Interface) GetRawFlags() uint32 {
return 0
}
+func (m *Interface) GetQueues() uint32 {
+ if m != nil {
+ return m.Queues
+ }
+ return 0
+}
+
type Route struct {
Dest string `protobuf:"bytes,1,opt,name=dest,proto3" json:"dest,omitempty"`
Gateway string `protobuf:"bytes,2,opt,name=gateway,proto3" json:"gateway,omitempty"`
@@ -319,6 +327,11 @@ func (m *Interface) MarshalTo(dAtA []byte) (int, error) {
i++
i = encodeVarintTypes(dAtA, i, uint64(m.RawFlags))
}
+ if m.Queues != 0 {
+ dAtA[i] = 0x48
+ i++
+ i = encodeVarintTypes(dAtA, i, uint64(m.Queues))
+ }
return i, nil
}
@@ -430,6 +443,9 @@ func (m *Interface) Size() (n int) {
if m.RawFlags != 0 {
n += 1 + sovTypes(uint64(m.RawFlags))
}
+ if m.Queues != 0 {
+ n += 1 + sovTypes(uint64(m.Queues))
+ }
return n
}
@@ -841,6 +857,25 @@ func (m *Interface) Unmarshal(dAtA []byte) error {
break
}
}
+ case 9:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field Queues", wireType)
+ }
+ m.Queues = 0
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowTypes
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ m.Queues |= (uint32(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
default:
iNdEx = preIndex
skippy, err := skipTypes(dAtA[iNdEx:])
@@ -1155,27 +1190,28 @@ var (
func init() { proto.RegisterFile("pkg/types/types.proto", fileDescriptorTypes) }
var fileDescriptorTypes = []byte{
- // 345 bytes of a gzipped FileDescriptorProto
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x91, 0xd1, 0x6a, 0xfa, 0x30,
- 0x14, 0xc6, 0xff, 0xb1, 0xb6, 0xda, 0xe3, 0xdf, 0xad, 0x84, 0x4d, 0xc2, 0x06, 0x52, 0xbc, 0x59,
- 0xd9, 0x85, 0x03, 0x37, 0x76, 0xbf, 0x5d, 0x08, 0xde, 0x49, 0x5e, 0x60, 0xc4, 0x36, 0x3a, 0xd1,
- 0xda, 0xd2, 0x44, 0x8b, 0xec, 0x05, 0x77, 0xb9, 0x47, 0x10, 0x9f, 0x64, 0xe4, 0x24, 0x8a, 0xbb,
- 0xd1, 0xef, 0x97, 0x93, 0xd3, 0xef, 0x7c, 0x27, 0x70, 0x5b, 0xae, 0x16, 0x4f, 0x7a, 0x5f, 0x4a,
- 0x65, 0x7f, 0x87, 0x65, 0x55, 0xe8, 0x82, 0xfa, 0x08, 0x83, 0x19, 0x84, 0x93, 0xe9, 0x5b, 0x96,
- 0x55, 0x52, 0x29, 0xfa, 0x00, 0xc1, 0x5c, 0xe4, 0xcb, 0xf5, 0x9e, 0x91, 0x98, 0x24, 0x57, 0xa3,
- 0xeb, 0xa1, 0xed, 0x98, 0x4c, 0xc7, 0x78, 0xcc, 0x5d, 0x99, 0x32, 0x68, 0x09, 0xdb, 0xc3, 0x1a,
- 0x31, 0x49, 0x42, 0x7e, 0x42, 0x4a, 0xa1, 0x99, 0x0b, 0xb5, 0x62, 0x1e, 0x1e, 0xa3, 0x1e, 0x1c,
- 0x08, 0x84, 0x93, 0x8d, 0x96, 0xd5, 0x5c, 0xa4, 0x92, 0xf6, 0x20, 0xc8, 0xe4, 0x6e, 0x99, 0x4a,
- 0x34, 0x09, 0xb9, 0x23, 0xd3, 0xb9, 0x11, 0xb9, 0x74, 0x1f, 0x44, 0x4d, 0x47, 0xd0, 0x39, 0x4f,
- 0x27, 0x15, 0xf3, 0x62, 0x2f, 0xe9, 0x8c, 0xa2, 0xf3, 0x54, 0xae, 0xc2, 0x2f, 0x2f, 0xd1, 0x08,
- 0xbc, 0x5c, 0x6f, 0x59, 0x33, 0x26, 0x49, 0x93, 0x1b, 0x69, 0x1c, 0x3f, 0x6b, 0x73, 0x81, 0xf9,
- 0xd6, 0xd1, 0x92, 0x49, 0x51, 0xa6, 0x4b, 0x2c, 0x04, 0x36, 0x85, 0x43, 0x33, 0x8b, 0xf1, 0x60,
- 0x2d, 0x3b, 0x8b, 0xd1, 0xf4, 0x1e, 0xc2, 0x4a, 0xd4, 0x1f, 0xf3, 0xb5, 0x58, 0x28, 0xd6, 0x8e,
- 0x49, 0xd2, 0xe5, 0xed, 0x4a, 0xd4, 0x63, 0xc3, 0x83, 0x2f, 0xf0, 0x79, 0xb1, 0xd5, 0x98, 0x22,
- 0x93, 0x4a, 0xbb, 0x6c, 0xa8, 0x8d, 0xcf, 0x42, 0x68, 0x59, 0x8b, 0xfd, 0x69, 0x5b, 0x0e, 0x2f,
- 0x76, 0xe1, 0xfd, 0xd9, 0x45, 0x0f, 0x02, 0x55, 0x6c, 0xab, 0x54, 0x62, 0x8c, 0x90, 0x3b, 0xa2,
- 0x37, 0xe0, 0xab, 0xb4, 0x28, 0x25, 0x06, 0xe9, 0x72, 0x0b, 0x8f, 0x77, 0xd0, 0x3e, 0xbd, 0x10,
- 0x0d, 0xa0, 0xb1, 0x7b, 0x89, 0xfe, 0xe1, 0xff, 0x6b, 0x44, 0xde, 0xff, 0x7f, 0x1f, 0xfb, 0xe4,
- 0xe7, 0xd8, 0x27, 0x87, 0x63, 0x9f, 0xcc, 0x02, 0x7c, 0xfb, 0xe7, 0xdf, 0x00, 0x00, 0x00, 0xff,
- 0xff, 0xb5, 0x52, 0x37, 0xf2, 0x14, 0x02, 0x00, 0x00,
+ // 356 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x92, 0xc1, 0x4e, 0xc2, 0x40,
+ 0x10, 0x86, 0x5d, 0x4a, 0x0b, 0x1d, 0x44, 0x9b, 0x8d, 0x92, 0x8d, 0x26, 0xa4, 0xe1, 0x62, 0xe3,
+ 0x01, 0x13, 0x34, 0xde, 0xf5, 0x40, 0xc2, 0x0d, 0xf7, 0x05, 0xcc, 0xd2, 0x2e, 0x48, 0xa0, 0xb4,
+ 0xe9, 0xb6, 0x34, 0xc4, 0x17, 0xf4, 0xe8, 0x23, 0x18, 0x9e, 0xc2, 0xa3, 0xd9, 0xd9, 0x85, 0xe0,
+ 0x05, 0xfe, 0x6f, 0x67, 0xa7, 0xff, 0xfc, 0xd3, 0xc2, 0x75, 0xbe, 0x5a, 0x3c, 0x94, 0xbb, 0x5c,
+ 0x2a, 0xf3, 0x3b, 0xcc, 0x8b, 0xac, 0xcc, 0xa8, 0x8b, 0x30, 0x98, 0x81, 0x3f, 0x99, 0xbe, 0x24,
+ 0x49, 0x21, 0x95, 0xa2, 0x77, 0xe0, 0xcd, 0x45, 0xba, 0x5c, 0xef, 0x18, 0x09, 0x49, 0x74, 0x31,
+ 0xba, 0x1c, 0x9a, 0x8e, 0xc9, 0x74, 0x8c, 0xc7, 0xdc, 0x96, 0x29, 0x83, 0x96, 0x30, 0x3d, 0xac,
+ 0x11, 0x92, 0xc8, 0xe7, 0x07, 0xa4, 0x14, 0x9a, 0xa9, 0x50, 0x2b, 0xe6, 0xe0, 0x31, 0xea, 0xc1,
+ 0x2f, 0x01, 0x7f, 0xb2, 0x29, 0x65, 0x31, 0x17, 0xb1, 0xa4, 0x3d, 0xf0, 0x12, 0xb9, 0x5d, 0xc6,
+ 0x12, 0x4d, 0x7c, 0x6e, 0x49, 0x77, 0x6e, 0x44, 0x2a, 0xed, 0x03, 0x51, 0xd3, 0x11, 0x74, 0x8e,
+ 0xd3, 0x49, 0xc5, 0x9c, 0xd0, 0x89, 0x3a, 0xa3, 0xe0, 0x38, 0x95, 0xad, 0xf0, 0xd3, 0x4b, 0x34,
+ 0x00, 0x27, 0x2d, 0x2b, 0xd6, 0x0c, 0x49, 0xd4, 0xe4, 0x5a, 0x6a, 0xc7, 0x8f, 0x5a, 0x5f, 0x60,
+ 0xae, 0x71, 0x34, 0xa4, 0x53, 0xe4, 0xf1, 0x12, 0x0b, 0x9e, 0x49, 0x61, 0x51, 0xcf, 0xa2, 0x3d,
+ 0x58, 0xcb, 0xcc, 0xa2, 0x35, 0xbd, 0x05, 0xbf, 0x10, 0xf5, 0xfb, 0x7c, 0x2d, 0x16, 0x8a, 0xb5,
+ 0x43, 0x12, 0x75, 0x79, 0xbb, 0x10, 0xf5, 0x58, 0xb3, 0xb6, 0x78, 0xab, 0x64, 0x25, 0x15, 0xf3,
+ 0xb1, 0x62, 0x69, 0xf0, 0x09, 0x2e, 0xcf, 0xaa, 0x12, 0xd3, 0x25, 0x52, 0x95, 0x36, 0x33, 0x6a,
+ 0xed, 0xbf, 0x10, 0xa5, 0xac, 0xc5, 0xee, 0xb0, 0x45, 0x8b, 0x27, 0x3b, 0x72, 0xfe, 0xed, 0xa8,
+ 0x07, 0x9e, 0xca, 0xaa, 0x22, 0x96, 0x18, 0xcf, 0xe7, 0x96, 0xe8, 0x15, 0xb8, 0x2a, 0xce, 0x72,
+ 0x89, 0x01, 0xbb, 0xdc, 0xc0, 0xfd, 0x0d, 0xb4, 0x0f, 0x6f, 0x8e, 0x7a, 0xd0, 0xd8, 0x3e, 0x05,
+ 0x67, 0xf8, 0xff, 0x1c, 0x90, 0xd7, 0xf3, 0xaf, 0x7d, 0x9f, 0x7c, 0xef, 0xfb, 0xe4, 0x67, 0xdf,
+ 0x27, 0x33, 0x0f, 0xbf, 0x89, 0xc7, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa5, 0x57, 0x70, 0xc8,
+ 0x2c, 0x02, 0x00, 0x00,
}
diff --git a/pkg/types/types.proto b/pkg/types/types.proto
index f6856e1..149df13 100644
--- a/pkg/types/types.proto
+++ b/pkg/types/types.proto
@@ -37,6 +37,7 @@ message Interface {
// list: "veth", "macvtap", "vlan", "macvlan", "tap", ...
string type = 7;
uint32 raw_flags = 8;
+ uint32 Queues = 9;
}
message Route {
--
2.14.3 (Apple Git-98)

56
agent/patches/0014-agent-fix-init-hugepages-failed-problem.patch Normal file
View File

@ -0,0 +1,56 @@
From 9e1478d7989fea4ee759cc13009d8de07dac2879 Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Wed, 19 Aug 2020 17:01:00 +0800
Subject: [PATCH 14/16] agent: fix init hugepages failed problem
reason: fix init hugepages failed problem
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
.../runc/libcontainer/cgroups/fs/apply_raw.go | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go b/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go
index ec148b4..47aa3c3 100644
--- a/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go
+++ b/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go
@@ -8,11 +8,13 @@ import (
"os"
"path/filepath"
"sync"
+ "time"
"github.com/opencontainers/runc/libcontainer/cgroups"
"github.com/opencontainers/runc/libcontainer/configs"
libcontainerUtils "github.com/opencontainers/runc/libcontainer/utils"
"github.com/pkg/errors"
+ "github.com/sirupsen/logrus"
"golang.org/x/sys/unix"
)
@@ -409,3 +411,21 @@ func CheckCpushares(path string, c uint64) error {
func (m *Manager) GetCgroups() (*configs.Cgroup, error) {
return m.Cgroups, nil
}
+
+func init() {
+ go func() {
+ var err error
+ if len(HugePageSizes) != 0 {
+ return
+ }
+ for i := 0; i < 10; i++ {
+ HugePageSizes, err = cgroups.GetHugePageSize()
+ if err == nil {
+ logrus.Infof("init hugepages ok loop=%d %v", i, err)
+ return
+ }
+ logrus.Errorf("init hugepages failed loop=%d %v", i, err)
+ time.Sleep(time.Second)
+ }
+ }()
+}
\ No newline at end of file
--
2.14.3 (Apple Git-98)

2649
agent/patches/0015-agent-add-support-of-getting-container-s-network-sta.patch Normal file
View File

File diff suppressed because it is too large Load Diff

268
agent/patches/0016-clock-synchronizes-clock-info-with-proxy.patch Normal file
View File

@ -0,0 +1,268 @@
From 6120525f81701424e97d453d515d38f14bbe99d9 Mon Sep 17 00:00:00 2001
From: holyfei <yangfeiyu20092010@163.com>
Date: Wed, 19 Aug 2020 20:25:00 +0800
Subject: [PATCH 16/16] clock: synchronizes clock info with proxy
reason: virtual machine's clock may be incorrect, proxy synchronizes
clock info to help virtual machine adjust clock time
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
---
pkg/clock/clock_util.go | 47 ++++++++++++++++++++++++
sync_clock_server.go | 94 +++++++++++++++++++++++++++++++++++++++++++++++
sync_clock_server_test.go | 88 ++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 229 insertions(+)
create mode 100644 pkg/clock/clock_util.go
create mode 100644 sync_clock_server.go
create mode 100644 sync_clock_server_test.go
diff --git a/pkg/clock/clock_util.go b/pkg/clock/clock_util.go
new file mode 100644
index 0000000..4b78c63
--- /dev/null
+++ b/pkg/clock/clock_util.go
@@ -0,0 +1,47 @@
+// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+// Description: sync clock related function
+// Author: xueshaojia
+// Create: 2018-11-10
+
+// Package clock provides clock sync functions
+package clock
+
+import (
+ "net"
+ "syscall"
+)
+
+type TimeValue struct {
+ ClientSendTime int64 `json:"client_send_time"`
+ ClientArriveTime int64 `json:"client_arrive_time"`
+ ServerSendTime int64 `json:"server_send_time"`
+ ServerArriveTime int64 `json:"server_arrive_time"`
+ Delta int64 `json:"delta"`
+}
+
+const MaxSyncClockByteNum = 400 // sync clock byte num, max=400
+const MaxTimeStampSupport = 3155731199000000000 // unit is ns, 2069/12/31 23:59:59
+const MinTimeStampSupport = 0 // unit is ns, 1970/1/1 8:0:0
+
+// getCurrentTimeNs returns UTC time in Ns
+func GetCurrentTimeNs() int64 {
+ var tv syscall.Timeval
+ if err := syscall.Gettimeofday(&tv); err != nil {
+ return -1
+ }
+ return tv.Sec*1000000000 + tv.Usec*1000
+}
+
+// readConnData reads data from stream
+func ReadConnData(stream net.Conn) (buf []byte, byteNum int, err error) {
+ buf = make([]byte, MaxSyncClockByteNum)
+ byteNum, err = stream.Read(buf)
+ return buf, byteNum, err
+}
+
+// writeConnData writes data to stream
+func WriteConnData(stream net.Conn, buf []byte) error {
+ _, err := stream.Write(buf)
+ return err
+}
diff --git a/sync_clock_server.go b/sync_clock_server.go
new file mode 100644
index 0000000..a8e98c6
--- /dev/null
+++ b/sync_clock_server.go
@@ -0,0 +1,94 @@
+// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+// Description: sync clock server related function
+// Author: xueshaojia
+// Create: 2018-11-10
+
+package main
+
+import (
+ "encoding/json"
+ "fmt"
+ "io"
+ "net"
+ "os/exec"
+ "syscall"
+
+ "github.com/kata-containers/agent/pkg/clock"
+)
+
+// getSyncClockInfo waits for client request
+func getSyncClockInfo(stream net.Conn, clockInfo *clock.TimeValue) error {
+ buf, byteNum, err := clock.ReadConnData(stream)
+ if err != nil {
+ return err
+ }
+ return json.Unmarshal(buf[:byteNum], clockInfo)
+}
+
+// ackSyncClockInfo sends clock info to client
+func ackSyncClockInfo(stream net.Conn, clockInfo *clock.TimeValue) error {
+ b, err := json.Marshal(clockInfo)
+ if err != nil {
+ return err
+ }
+ return clock.WriteConnData(stream, b)
+}
+
+// setGuestClock sets systemtime and hwclock
+func setGuestClock(delta int64) error {
+ dstClockNs := clock.GetCurrentTimeNs() + delta
+ if dstClockNs < clock.MinTimeStampSupport || dstClockNs > clock.MaxTimeStampSupport {
+ return fmt.Errorf("sync clock failed, the valid timestamp is from 1970/1/1 8:0:0 to 2069/12/31 23:59:59")
+ }
+ dstTimeVal := syscall.NsecToTimeval(dstClockNs)
+ if err := syscall.Settimeofday(&dstTimeVal); err != nil {
+ agentLog.WithError(err).Error("sync clock, set systemtime failed")
+ return err
+ }
+ cmd := exec.Command("hwclock", "-w")
+ if err := cmd.Run(); err != nil {
+ agentLog.WithError(err).Error("sync clock, set hwclock failed")
+ return err
+ }
+ agentLog.Infof("sync clock, set systemtime and hwclock succ, delta:%d", delta)
+ return nil
+}
+
+// SyncClock is a loop that deals with client's request
+// 1 read client request
+// 2 response with clock info or set guest clock
+// 3 if error happends, print error info
+func SyncClock(stream net.Conn) {
+ for {
+ var clockInfo clock.TimeValue
+ if err := getSyncClockInfo(stream, &clockInfo); err != nil {
+ if err == io.EOF {
+ agentLog.WithError(err).Error("sync clock, yamux session error happends")
+ stream.Close()
+ break
+ }
+ agentLog.WithError(err).Error("sync clock, read sync clock info failed")
+ continue
+ }
+ // set client's request arrive time
+ clockInfo.ClientArriveTime = clock.GetCurrentTimeNs()
+ if clockInfo.Delta == 0 {
+ // this is a request for clock diff info
+ // set server's response send time
+ // if fails, wait for next time
+ clockInfo.ServerSendTime = clock.GetCurrentTimeNs()
+ if err := ackSyncClockInfo(stream, &clockInfo); err != nil {
+ agentLog.WithError(err).Error("sync clock, send back clock info failed")
+ }
+ } else {
+ // this is a request for adjusting guest clock
+ // 1 adjust guest clock(system time and hwclock)
+ // 2 do not send response any more
+ // 3 if fails, wait for next request to adjust
+ if err := setGuestClock(clockInfo.Delta); err != nil {
+ agentLog.WithError(err).Error("sync clock, set local clock failed")
+ }
+ }
+ }
+}
diff --git a/sync_clock_server_test.go b/sync_clock_server_test.go
new file mode 100644
index 0000000..41d5227
--- /dev/null
+++ b/sync_clock_server_test.go
@@ -0,0 +1,88 @@
+// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+// Description: sync clock server related test
+// Author: xueshaojia
+// Create: 2018-11-10
+
+package main
+
+import (
+ "encoding/json"
+ "fmt"
+ "math/rand"
+ "net"
+ "testing"
+
+ "github.com/hashicorp/yamux"
+ "github.com/kata-containers/agent/pkg/clock"
+)
+
+var clientStream net.Conn
+
+func TestGetCurrentTimeNs(t *testing.T) {
+ timeRet := clock.GetCurrentTimeNs()
+ if timeRet <= 0 {
+ t.Fatalf("TestGetCurrentTimeNs Fail, time:%d", timeRet)
+ }
+}
+
+func TestSetGuestClock(t *testing.T) {
+ if err := setGuestClock(0); err != nil {
+ t.Fatalf("test set clock failed, err:%v", err)
+ }
+}
+
+func TestReadWriteData(t *testing.T) {
+ sock := GenSocket()
+ var err error
+ listener, err := net.Listen("unix", sock)
+ if err != nil {
+ t.Fatalf("listening err: %v", err)
+ }
+ go SetUpClient(sock)
+ conn, err := listener.Accept()
+ if err != nil {
+ t.Fatalf("accept err: %v", err)
+ }
+ session, err := yamux.Server(conn, nil)
+ if err != nil {
+ t.Fatalf("session err: %v", err)
+ }
+ stream, err := session.Accept()
+ if err != nil {
+ t.Fatalf("stream err: %v", err)
+ }
+ go func() {
+ var clockInfo clock.TimeValue = clock.TimeValue{1000, 0, 0, 0, 0}
+ b, _ := json.Marshal(clockInfo)
+ clock.WriteConnData(clientStream, b)
+ }()
+ _, _, err = clock.ReadConnData(stream)
+ if err != nil {
+ t.Fatalf("read conn data error, err:%v", err)
+ }
+}
+
+func SetUpClient(sock string) error {
+ conn, err := net.Dial("unix", sock)
+ if err != nil {
+ return err
+ }
+ session, err := yamux.Client(conn, nil)
+ if err != nil {
+ conn.Close()
+ return err
+ }
+ stream, err := session.Open()
+ if err != nil {
+ return err
+ }
+ clientStream = stream
+ return nil
+}
+
+func GenSocket() string {
+ randSeed := clock.GetCurrentTimeNs()
+ rand.Seed(randSeed)
+ return fmt.Sprintf("/tmp/%d.sock", rand.Uint32())
+}
--
2.14.3 (Apple Git-98)

1005
agent/patches/0017-agent-add-support-of-new-sandbox-StratoVirt.patch Normal file
View File

File diff suppressed because it is too large Load Diff

184
agent/patches/0018-kata-agent-update-nic-in-guest.patch Normal file
View File

@ -0,0 +1,184 @@
From 629aac1078bdeed63214c58d2110fe672d654774 Mon Sep 17 00:00:00 2001
From: yangfeiyu <yangfeiyu2@huawei.com>
Date: Mon, 26 Oct 2020 20:39:05 +0800
Subject: [PATCH] kata-agent: update nic in guest
reason: add linkByName and support retry of list ip link, because in some scenarios,
the hardware address cannot be queried immediately after the hot plug.
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
---
network.go | 91 ++++++++++++++++++++++++++++++++++++++++++------------
1 file changed, 72 insertions(+), 19 deletions(-)
diff --git a/network.go b/network.go
index f6e2c17..d928e2b 100644
--- a/network.go
+++ b/network.go
@@ -15,13 +15,14 @@ import (
"strings"
"sync"
"syscall"
+ "time"
- "golang.org/x/sys/unix"
agentNet "github.com/kata-containers/agent/pkg/net"
"github.com/kata-containers/agent/pkg/types"
pb "github.com/kata-containers/agent/protocols/grpc"
"github.com/sirupsen/logrus"
"github.com/vishvananda/netlink"
+ "golang.org/x/sys/unix"
"google.golang.org/grpc/codes"
grpcStatus "google.golang.org/grpc/status"
)
@@ -32,6 +33,7 @@ var (
errNoLink = grpcStatus.Errorf(codes.InvalidArgument, "Need network link")
errNoMAC = grpcStatus.Errorf(codes.InvalidArgument, "Need hardware address")
errNoRoutes = grpcStatus.Errorf(codes.InvalidArgument, "Need network routes")
+ errNoName = grpcStatus.Errorf(codes.InvalidArgument, "Need network name")
guestDNSFile = "/etc/resolv.conf"
kataGuestSandboxDNSFile = "/run/kata-containers/sandbox/resolv.conf"
)
@@ -46,6 +48,8 @@ const (
// Use the below address for ipv6 gateway once ipv6 support is added
// defaultV6RouteIP = "::"
+
+ maxLinkRetries = 10
)
// Network fully describes a sandbox network with its interfaces, routes and dns
@@ -100,17 +104,36 @@ func linkByHwAddr(netHandle *netlink.Handle, hwAddr string) (netlink.Link, error
return nil, grpcStatus.Errorf(codes.NotFound, "Could not find the link corresponding to HwAddr %q", hwAddr)
}
-func updateLink(netHandle *netlink.Handle, link netlink.Link, iface *types.Interface) error {
+func linkByName(netHandle *netlink.Handle, name string) (netlink.Link, error) {
if netHandle == nil {
- return errNoHandle
+ return nil, errNoHandle
}
- if link == nil {
- return errNoLink
+ if name == "" {
+ return nil, errNoName
}
- if iface == nil {
- return errNoIF
+ links, err := netHandle.LinkList()
+ if err != nil {
+ return nil, err
+ }
+
+ for _, link := range links {
+ if link == nil {
+ continue
+ }
+
+ if link.Attrs() != nil && link.Attrs().Name == name {
+ return link, nil
+ }
+ }
+
+ return nil, grpcStatus.Errorf(codes.NotFound, "Could not find the link corresponding to name %s", name)
+}
+
+func updateLinkIP(netHandle *netlink.Handle, link netlink.Link, iface *types.Interface) error {
+ if len(iface.IPAddresses) == 0 {
+ return nil
}
// As a first step, clear out any existing addresses associated with the link:
@@ -129,13 +152,6 @@ func updateLink(netHandle *netlink.Handle, link netlink.Link, iface *types.Inter
netlinkAddrStr := fmt.Sprintf("%s/%s", addr.Address, addr.Mask)
netlinkAddr, err := netlink.ParseAddr(netlinkAddrStr)
- // With ipv6 addresses, there is a brief period during which the address is marked as "tentative"
- // making it unavailable. A process called duplicate address detection(DAD) is performed during this period.
- // Disble DAD so that networking is available once the container is up. The assumption is
- // that it is the reponsibility of the upper stack to make sure the addresses assigned to containers
- // do not conflict. A similar operation is performed by libnetwork:
- // https://github.com/moby/moby/issues/18871
-
if addr.GetFamily() == types.IPFamily_v6 {
netlinkAddr.Flags = netlinkAddr.Flags | syscall.IFA_F_NODAD
}
@@ -150,14 +166,36 @@ func updateLink(netHandle *netlink.Handle, link netlink.Link, iface *types.Inter
}
}
+ return nil
+}
+
+func updateLink(netHandle *netlink.Handle, link netlink.Link, iface *types.Interface) error {
+ if netHandle == nil {
+ return errNoHandle
+ }
+
+ if link == nil {
+ return errNoLink
+ }
+
+ if iface == nil {
+ return errNoIF
+ }
+
+ if err := updateLinkIP(netHandle, link, iface); err != nil {
+ return err
+ }
+
// set the interface name:
if err := netHandle.LinkSetName(link, iface.Name); err != nil {
return grpcStatus.Errorf(codes.Internal, "Could not set name %s for interface %v: %v", iface.Name, link, err)
}
// set the interface MTU:
- if err := netHandle.LinkSetMTU(link, int(iface.Mtu)); err != nil {
- return grpcStatus.Errorf(codes.Internal, "Could not set MTU %d for interface %v: %v", iface.Mtu, link, err)
+ if iface.Mtu > 0 {
+ if err := netHandle.LinkSetMTU(link, int(iface.Mtu)); err != nil {
+ return grpcStatus.Errorf(codes.Internal, "Could not set MTU %d for interface %v: %v", iface.Mtu, link, err)
+ }
}
if iface.RawFlags&unix.IFF_NOARP == uint32(unix.IFF_NOARP) {
@@ -306,8 +344,23 @@ func (s *sandbox) updateInterface(netHandle *netlink.Handle, iface *types.Interf
if iface.HwAddr != "" {
fieldLogger.Info("Getting interface from MAC address")
- // Find the interface link from its hardware address.
- link, err = linkByHwAddr(netHandle, iface.HwAddr)
+ // In some scenarios, the hardware address cannot be queried immediately
+ // after the hot plug, add a retry here.
+ for retry := 0; retry < maxLinkRetries; retry++ {
+ // Find the interface link from its hardware address.
+ link, err = linkByHwAddr(netHandle, iface.HwAddr)
+ if err != nil {
+ time.Sleep(100 * time.Millisecond)
+ continue
+ }
+ break
+ }
+ if err != nil {
+ return nil, grpcStatus.Errorf(codes.Internal, "updateInterface: %v", err)
+ }
+ } else if iface.Name != "" {
+ fieldLogger.Info("Getting interface from name")
+ link, err = linkByName(netHandle, iface.Name)
if err != nil {
return nil, grpcStatus.Errorf(codes.Internal, "updateInterface: %v", err)
}
@@ -487,7 +540,7 @@ func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Ro
}()
var (
- added []*types.Route
+ added []*types.Route
removed []*types.Route
)
--
2.23.0

18
agent/series.conf Normal file
View File

@ -0,0 +1,18 @@
0001-agent-add-agent.netlink_recv_buf_size-flag-to-set-ne.patch
0002-network-support-update-routes-incrementally.patch
0003-kata-agent-add-kata-ipvs-command.patch
0004-agent-add-IPVS-test.patch
0005-mount-support-mount-block-device.patch
0006-agent-make-workaround-for-slow-response-in-aarch64.patch
0007-agent-using-pcie-root-port-driver-to-hotplug-device.patch
0008-agent-support-get-root-bus-path-dynamically.patch
0009-storage-add-pkg-storage-for-mount.patch
0010-storage-mount-nfs-and-gpath-in-agent.patch
0011-agent-fix-agent-reap-agent-process-blocked-problem.patch
0012-network-support-set-dns-without-nameserver.patch
0013-agent-support-setting-multi-queues-of-interface.patch
0014-agent-fix-init-hugepages-failed-problem.patch
0015-agent-add-support-of-getting-container-s-network-sta.patch
0016-clock-synchronizes-clock-info-with-proxy.patch
0017-agent-add-support-of-new-sandbox-StratoVirt.patch
0018-kata-agent-update-nic-in-guest.patch

133
kata-containers.spec Normal file
View File

@ -0,0 +1,133 @@
#needsrootforbuild
%global debug_package %{nil}
%define VERSION v1.11.1
%define RELEASE 7
Name: kata-containers
Version: %{VERSION}
Release: %{RELEASE}
Summary: Kata Container, the speed of containers, the security of VMs
License: Apache 2.0
URL: https://github.com/kata-containers
Source0: kata_integration-v1.0.0.tar.gz
Source1: kata-containers-%{version}.tar.gz
Source2: kernel.tar.gz
BuildRoot: %_topdir/BUILDROOT
BuildRequires: automake golang gcc bc glibc-devel glibc-static busybox glib2-devel glib2 ipvsadm conntrack-tools nfs-utils
BuildRequires: patch elfutils-libelf-devel openssl-devel bison flex
%description
This is core component of Kata Container, to make it work, you need a isulad/docker engine.
%prep
%setup -T -c -a 0 -n kata_integration
%setup -T -c -a 1 -n kata-containers-%{version}
%setup -T -c -a 2 -n kernel
# extract the kata_integration.tar.gz file
cd %{_builddir}/kata_integration
# apply kata_integration patches
sh apply-patches
# mv build components into kata_integration dir
pushd %{_builddir}/kata_integration
mv ../kata-containers-%{version}/runtime .
mv ../kata-containers-%{version}/agent .
mv ../kata-containers-%{version}/proxy .
mv ../kata-containers-%{version}/shim .
popd
# build kernel
cd %{_builddir}/kernel
mv kernel linux
cd %{_builddir}/kernel/linux/
%ifarch %{ix86} x86_64
cp %{_builddir}/kata_integration/hack/config-kata-x86_64 ./.config
%else
cp %{_builddir}/kata_integration/hack/config-kata-arm64 ./.config
%endif
%build
cd %{_builddir}/kernel/linux/
make %{?_smp_mflags}
cd %{_builddir}/kata_integration
mkdir -p -m 750 build
make runtime
make proxy
make shim
make initrd
%install
mkdir -p -m 755 %{buildroot}/var/lib/kata
%ifarch %{ix86} x86_64
install -p -m 755 -D %{_builddir}/kernel/linux/arch/x86_64/boot/bzImage %{buildroot}/var/lib/kata/kernel
%else
install -p -m 755 -D %{_builddir}/kernel/linux/arch/arm64/boot/Image %{buildroot}/var/lib/kata/kernel
%endif
cd %{_builddir}/kata_integration
mkdir -p -m 750 %{buildroot}/usr/bin
install -p -m 750 ./build/kata-runtime ./build/kata-proxy ./build/kata-shim ./build/kata-netmon %{buildroot}/usr/bin/
install -p -m 640 ./build/kata-containers-initrd.img %{buildroot}/var/lib/kata/
mkdir -p -m 750 %{buildroot}/usr/share/defaults/kata-containers/
install -p -m 640 -D ./runtime/cli/config/configuration-qemu.toml %{buildroot}/usr/share/defaults/kata-containers/configuration.toml
%clean
%files
/usr/bin/kata-runtime
/usr/bin/kata-proxy
/usr/bin/kata-shim
/usr/bin/kata-netmon
/var/lib/kata/kernel
/var/lib/kata/kata-containers-initrd.img
%config(noreplace) /usr/share/defaults/kata-containers/configuration.toml
%doc
%changelog
* Tue Dec 22 2020 jiangpengfei<jiangpengfei9@huawei.com> - 1.11.1-7
- Type:enhancement
- ID:NA
- SUG:update
- DESC:update kata-containers source forms of organization to move all kata-containers related source repo into one repo kata-containers
* Fri Nov 6 2020 yangfeiyu<yangfeiyu2@huawei.com> - 1.11.1-6
- Type:bugfix
- ID:NA
- SUG:NA
- DESC:revert the kata-containers.spec to still build kata-containers components into one package
* Fri Oct 9 2020 yangfeiyu<yangfeiyu2@huawei.com> - 1.11.1-5
- Type:enhancement
- ID:NA
- SUG:restart
- DESC:directly copy kata binary files instead of building them
* Wed Sep 30 2020 yangfeiyu<yangfeiyu2@huawei.com> - 1.11.1-4
- Type:bugfix
- ID:NA
- SUG:restart
- DESC:kata-runtime retry inserting of CNI interface
* Sun Sep 27 2020 LiangZhang<zhangliang5@Huawei.com> - 1.11.1-3
- Type:bugfix
- ID:NA
- SUG:NA
- DESC:fix cmd params of direct use stratovirt binary
* Thu Sep 20 2020 jiangpengf<jiangpengfei9@huawei.com> - 1.11.1-2
- Type:bugfix
- ID:NA
- SUG:NA
- DESC:fix del-iface doesn't delete the tap interface in the host problem
* Thu Aug 27 2020 jiangpengf<jiangpengfei9@huawei.com> - 1.11.1-1
- Type:enhancement
- ID:NA
- SUG:NA
- DESC:update kata-containers version to v1.11.1-1

4
kata-containers.yaml Normal file
View File

@ -0,0 +1,4 @@
version_control: github
src_repo: kata-containers/runtime
tag_prefix:
seperator: "."

20
proxy/apply-patches Executable file
View File

@ -0,0 +1,20 @@
#!/bin/bash
if [[ -f ./patch_flag ]];then
echo "proxy patched!"
exit 0
fi
tar -zxvf proxy-1.11.1.tar.gz
cp -fr ./proxy-1.11.1/* ./
rm -rf ./proxy-1.11.1
cat ./series.conf | while read line
do
if [[ $line == '' || $line =~ ^\s*# ]]; then
continue
fi
echo "====patch $line======"
pwd
patch -p1 -F1 -s < ./patches/$line
done
touch ./patch_flag

403
proxy/patches/0001-clock-synchronizes-clock-info-to-agent.patch Normal file
View File

@ -0,0 +1,403 @@
From a260bbe394f91fa05b163315390ed133de5c5494 Mon Sep 17 00:00:00 2001
From: holyfei <yangfeiyu20092010@163.com>
Date: Wed, 19 Aug 2020 17:43:24 +0800
Subject: [PATCH] clock: synchronizes clock info to agent
reason: virtual machine's clock may be incorrect, proxy synchronizes
clock info to help virtual machine adjust clock time
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
---
proxy.go | 7 ++
proxy_test.go | 29 +++++
sync_clock_client.go | 107 +++++++++++++++++
sync_clock_client_test.go | 132 +++++++++++++++++++++
.../kata-containers/agent/pkg/clock/clock_util.go | 44 +++++++
5 files changed, 319 insertions(+)
create mode 100644 sync_clock_client.go
create mode 100644 sync_clock_client_test.go
create mode 100644 vendor/github.com/kata-containers/agent/pkg/clock/clock_util.go
diff --git a/proxy.go b/proxy.go
index ab062a5..9dfcb3c 100644
--- a/proxy.go
+++ b/proxy.go
@@ -105,6 +105,13 @@ func serve(servConn io.ReadWriteCloser, proto, addr string, results chan error)
// Start the heartbeat in a separate go routine
go heartBeat(session)
+ // start the sync clock in a separate go routine
+ syncClockStream, err := session.Open()
+ if err != nil {
+ return nil, nil, err
+ }
+ go SyncClock(syncClockStream)
+
// serving connection
l, err := net.Listen(proto, addr)
if err != nil {
diff --git a/proxy_test.go b/proxy_test.go
index 923b138..94fa523 100644
--- a/proxy_test.go
+++ b/proxy_test.go
@@ -10,6 +10,7 @@ package main
import (
"bytes"
"crypto/md5"
+ "encoding/json"
"fmt"
"io"
"io/ioutil"
@@ -23,6 +24,7 @@ import (
"testing"
"github.com/hashicorp/yamux"
+ "github.com/kata-containers/agent/pkg/clock"
"github.com/stretchr/testify/assert"
)
@@ -121,6 +123,13 @@ func server(listener net.Listener, closeCh chan bool) error {
session.Close()
}()
+ // accept the sync clock stream first
+ if syncClockStream, err := session.Accept(); err != nil {
+ return err
+ } else {
+ go serverSyncClock(syncClockStream)
+ }
+
for {
stream, err := session.Accept()
if err != nil {
@@ -133,6 +142,26 @@ func server(listener net.Listener, closeCh chan bool) error {
}
}
+func serverSyncClock(stream net.Conn) {
+ for {
+ buf, byteNum, err := readConnData(stream)
+ if err != nil {
+ continue
+ }
+ var clockInfo clock.TimeValue
+ if err := json.Unmarshal(buf[:byteNum], &clockInfo); err != nil {
+ continue
+ }
+ nowTime := clock.GetCurrentTimeNs()
+ clockInfo.ClientArriveTime = nowTime
+ clockInfo.ServerSendTime = nowTime
+ b, _ := json.Marshal(clockInfo)
+ if err := clock.WriteConnData(stream, b); err != nil {
+ continue
+ }
+ }
+}
+
func TestUnixAddrParsing(T *testing.T) {
buf := "unix://foo/bar"
addr, err := unixAddr(buf)
diff --git a/sync_clock_client.go b/sync_clock_client.go
new file mode 100644
index 0000000..9bf3e91
--- /dev/null
+++ b/sync_clock_client.go
@@ -0,0 +1,107 @@
+// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+// Description: sync clock client related function
+// Author: xueshaojia x00464843
+// Create: 2018-11-10
+
+package main
+
+import (
+ "encoding/json"
+ "fmt"
+ "net"
+ "time"
+
+ "github.com/kata-containers/agent/pkg/clock"
+)
+
+const (
+ allowTimeDiff = 10 * 1000 * 1000 // allow 10ms's difference
+ syncClockInterval = 60 * time.Second // sync clock with agent every 60 seconds
+ rpcTimeout = 10 * time.Second // timeout for proxy's reading data
+)
+
+// readConnData reads data from stream
+func readConnData(stream net.Conn) (buf []byte, byteNum int, err error) {
+ // set read deadline to avoid case as following:
+ // proxy and agent are both reading and then syncClock will never work
+ stream.SetReadDeadline(time.Now().Add(rpcTimeout))
+ buf = make([]byte, clock.MaxSyncClockByteNum)
+ byteNum, err = stream.Read(buf)
+ return buf, byteNum, err
+}
+
+// getGuestClock syncs guest clock info
+// sends ClientSendTime
+// waits for ClientArriveTime and ServerSendTime
+func getGuestClock(stream net.Conn, clockInfo *clock.TimeValue) error {
+ clockInfo.Delta = 0
+ b, err := json.Marshal(clockInfo)
+ if err != nil {
+ return err
+ }
+ if err = clock.WriteConnData(stream, b); err != nil {
+ return err
+ }
+
+ buf, byteNum, err := readConnData(stream)
+ if err != nil {
+ return err
+ }
+
+ if err = json.Unmarshal(buf[:byteNum], clockInfo); err != nil {
+ return fmt.Errorf("sync clock, parse guest clocktime error:%v", err)
+ }
+ return nil
+}
+
+// adjustGuestClock tells server to ajust local clock with Delta
+func adjustGuestClock(stream net.Conn, clockInfo *clock.TimeValue) error {
+ b, err := json.Marshal(clockInfo)
+ if err != nil {
+ return err
+ }
+ logger().Debugf("sync clock, send:%s", string(b))
+ return clock.WriteConnData(stream, b)
+}
+
+// syncClock performs all the steps
+// 1 get client send time[host]
+// 2 request for client arrive time and server send time[guest os]
+// 3 get server arrive time[host]
+// 4 calculate clock diff
+// 5 request to adjust guest clock
+func syncClock(stream net.Conn) error {
+ var clockInfo clock.TimeValue
+ if clockInfo.ClientSendTime = clock.GetCurrentTimeNs(); clockInfo.ClientSendTime <= 0 {
+ return fmt.Errorf("sync clock, get client sendtime error")
+ }
+ err := getGuestClock(stream, &clockInfo)
+ if err != nil {
+ return fmt.Errorf("sync clock, get guest clocktime error:%v", err)
+ }
+ if clockInfo.ServerArriveTime = clock.GetCurrentTimeNs(); clockInfo.ServerArriveTime <= 0 {
+ return fmt.Errorf("sync clock, get client recvtime error")
+ }
+ if clockInfo.ClientSendTime <= 0 || clockInfo.ClientArriveTime <= 0 || clockInfo.ServerSendTime <= 0 {
+ return fmt.Errorf("sync clock, some fields of NTP message error, raw message:%v", clockInfo)
+ }
+
+ delta := ((clockInfo.ClientSendTime - clockInfo.ClientArriveTime) + (clockInfo.ServerArriveTime - clockInfo.ServerSendTime)) / 2
+ if delta < -allowTimeDiff || delta > allowTimeDiff {
+ clockInfo.Delta = delta
+ if err := adjustGuestClock(stream, &clockInfo); err != nil {
+ return fmt.Errorf("sync clock, failed to adjust guest clock : %v", err)
+ }
+ }
+ return nil
+}
+
+func SyncClock(stream net.Conn) {
+ for {
+ if err := syncClock(stream); err != nil {
+ logger().WithError(err).Error("sync clock failed")
+ }
+ time.Sleep(syncClockInterval)
+ }
+}
diff --git a/sync_clock_client_test.go b/sync_clock_client_test.go
new file mode 100644
index 0000000..b0b1c85
--- /dev/null
+++ b/sync_clock_client_test.go
@@ -0,0 +1,132 @@
+// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+// Description: sync clock client related test
+// Author: xueshaojia x00464843
+// Create: 2018-11-10
+
+package main
+
+import (
+ "encoding/json"
+ "fmt"
+ "math/rand"
+ "net"
+ "os"
+ "testing"
+
+ "github.com/hashicorp/yamux"
+ "github.com/kata-containers/agent/pkg/clock"
+)
+
+func TestReadWriteData(t *testing.T) {
+ var clockInfo clock.TimeValue
+ clockInfo.ClientSendTime = clock.GetCurrentTimeNs()
+ b, err := json.Marshal(clockInfo)
+ if err != nil {
+ t.Fatalf("Marshal clock info fail, err:%v", err)
+ }
+
+ err = clock.WriteConnData(clientStream, b)
+ fmt.Printf("client send: %s\n", string(b))
+ if err != nil {
+ t.Fatalf("send clock info fail, err:%v", err)
+ }
+ _, _, err = readConnData(clientStream)
+ if err != nil {
+ t.Fatalf("recv clock info fail, err:%v", err)
+ }
+}
+
+func SetUpServer(sock string, readyChan chan int) error {
+ var err error
+ listener, err = net.Listen("unix", sock)
+ if err != nil {
+ return err
+ }
+ readyChan <- 1
+ conn, err := listener.Accept()
+ if err != nil {
+ return err
+ }
+ session, err := yamux.Server(conn, nil)
+ if err != nil {
+ return err
+ }
+ serverSession = session
+ stream, err := session.Accept()
+ if err != nil {
+ return err
+ }
+ for {
+ var clockInfo clock.TimeValue
+ var byteNum int
+ var err error
+ buf := make([]byte, 400)
+ if byteNum, err = stream.Read(buf); err != nil {
+ break
+ }
+
+ if err = json.Unmarshal(buf[:byteNum], &clockInfo); err != nil {
+ break
+ }
+ if clockInfo.Delta == 0 {
+ nowTime := clock.GetCurrentTimeNs()
+ clockInfo.ClientArriveTime = nowTime
+ clockInfo.ClientArriveTime = nowTime
+ b, _ := json.Marshal(&clockInfo)
+ stream.Write(b)
+ }
+ }
+ return nil
+}
+
+func SetUpClient(sock string) error {
+ conn, err := net.Dial("unix", sock)
+ if err != nil {
+ return err
+ }
+ session, err := yamux.Client(conn, nil)
+ if err != nil {
+ conn.Close()
+ return err
+ }
+ clientSession = session
+ stream, err := session.Open()
+ if err != nil {
+ clientSession.Close()
+ return err
+ }
+ clientStream = stream
+ return nil
+}
+
+func TearDown() {
+ listener.Close()
+ serverSession.Close()
+ clientSession.Close()
+}
+
+func GenSocket() string {
+ randSeed := clock.GetCurrentTimeNs()
+ rand.Seed(randSeed)
+ return fmt.Sprintf("/tmp/%d.sock", rand.Uint32())
+}
+
+var listener net.Listener
+var clientStream net.Conn
+var serverSession *yamux.Session
+var clientSession *yamux.Session
+
+func TestMain(m *testing.M) {
+ waitConn := make(chan int)
+ testSock := GenSocket()
+ go SetUpServer(testSock, waitConn)
+ <-waitConn
+ if err := SetUpClient(testSock); err != nil {
+ listener.Close()
+ serverSession.Close()
+ os.Exit(1)
+ }
+ m.Run()
+ TearDown()
+}
diff --git a/vendor/github.com/kata-containers/agent/pkg/clock/clock_util.go b/vendor/github.com/kata-containers/agent/pkg/clock/clock_util.go
new file mode 100644
index 0000000..03244fd
--- /dev/null
+++ b/vendor/github.com/kata-containers/agent/pkg/clock/clock_util.go
@@ -0,0 +1,44 @@
+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+// Description: common functions
+// Author: jiangpeifei
+// Create: 2019-05-28
+
+package clock
+
+import (
+ "net"
+ "syscall"
+)
+
+type TimeValue struct {
+ ClientSendTime int64 `json:"client_send_time"`
+ ClientArriveTime int64 `json:"client_arrive_time"`
+ ServerSendTime int64 `json:"server_send_time"`
+ ServerArriveTime int64 `json:"server_arrive_time"`
+ Delta int64 `json:"delta"`
+}
+
+const MaxSyncClockByteNum = 400 //sync clock byte num, max=400
+
+// getCurrentTimeNs returns UTC time in Ns
+func GetCurrentTimeNs() int64 {
+ var tv syscall.Timeval
+ if err := syscall.Gettimeofday(&tv); err != nil {
+ return -1
+ }
+ return tv.Sec*1000000000 + tv.Usec*1000
+}
+
+// readConnData reads data from stream
+func ReadConnData(stream net.Conn) (buf []byte, byteNum int, err error) {
+ buf = make([]byte, MaxSyncClockByteNum)
+ byteNum, err = stream.Read(buf)
+ return buf, byteNum, err
+}
+
+// writeConnData writes data to stream
+func WriteConnData(stream net.Conn, buf []byte) error {
+ _, err := stream.Write(buf)
+ return err
+}
--
2.14.3 (Apple Git-98)

BIN
proxy/proxy-1.11.1.tar.gz Normal file
View File

Binary file not shown.

1
proxy/series.conf Normal file
View File

@ -0,0 +1 @@
0001-clock-synchronizes-clock-info-to-agent.patch

22
runtime/apply-patches Executable file
View File

@ -0,0 +1,22 @@
#!/bin/bash
set -e
if [[ -f ./patch_flag ]];then
echo "runtime patched!"
exit 0
fi
tar -zxvf runtime-1.11.1.tar.gz
cp -fr ./runtime-1.11.1/* ./
rm -rf ./runtime-1.11.1
cat ./series.conf | while read line
do
if [[ $line == '' || $line =~ ^\s*# ]]; then
continue
fi
echo "====patch $line======"
patch -p1 -F1 -s < ./patches/$line
done
touch ./patch_flag

165
runtime/kata-runtime.spec Normal file
View File

@ -0,0 +1,165 @@
%define debug_package %{nil}
%define VERSION 1.11.1
%define RELEASE 11
Name: kata-runtime
Version: %{VERSION}
Release: %{RELEASE}
Summary: Kata Runtime
License: Apache 2.0
URL: https://github.com/kata-containers/runtime
Source0: https://github.com/kata-containers/runtime/archive/%{version}.tar.gz#/%{name}-v%{version}.tar.gz
BuildRoot: %_topdir/BUILDROOT
BuildRequires: automake golang gcc
%description
Kata-runtime is core component of Kata Container.
%prep
%setup -q -c -a 0 -n %{name}-%{version}
%build
cd %{_builddir}/%{name}-%{version}
set -e
# apply patches read from series.conf
sh apply-patches
# create tmp GOPATH dir to build kata-runtime
rm -rf /tmp/kata-build/
mkdir -p /tmp/kata-build/
GOPATH=/tmp/kata-build/
kata_base=$GOPATH/src/github.com/kata-containers
mkdir -p $kata_base
# get current kata-runtime absolute path
kata_runtime_path=$(readlink -f .)
ln -s $kata_runtime_path $kata_base/runtime
# export GOPATH env
export GOPATH=$(readlink -f $GOPATH)
cd ${kata_base}/runtime && make clean && make
rm -rf $GOPATH
# make kata-runtime default configuration
kata_config_path=$kata_runtime_path/cli/config/configuration-qemu.toml
ARCH=`arch`
# arch related config options
if [ "$ARCH" == "aarch64" ];then
sed -i 's/^machine_type.*$/machine_type = \"virt\"/' $kata_config_path
sed -i 's/^block_device_driver.*$/block_device_driver = \"virtio-scsi\"/' $kata_config_path
sed -i 's/^kernel_params.*$/kernel_params = \"pcie_ports=native pci=pcie_bus_perf agent.netlink_recv_buf_size=2MB\"/' $kata_config_path
sed -i 's/^hypervisor_params.*$/hypervisor_params = \"kvm-pit.lost_tick_policy=discard pcie-root-port.fast-plug=1 pcie-root-port.x-speed=16 pcie-root-port.x-width=32 pcie-root-port.fast-unplug=1\"/' $kata_config_path
sed -i 's/^#pcie_root_port.*$/pcie_root_port = 25/' $kata_config_path
else
sed -i 's#block_device_driver = \"virtio-scsi\"#block_device_driver = \"virtio-blk\"#' $kata_config_path
sed -i 's/^#hotplug_vfio_on_root_bus/hotplug_vfio_on_root_bus/' $kata_config_path
fi
# debug config
sed -i 's/^#enable_debug.*$/enable_debug = true/' $kata_config_path
# other config
sed -i 's#"/usr/bin/qemu.*"$#"/usr/bin/qemu-kvm"#' $kata_config_path
sed -i 's#/usr/share/kata-containers/vmlinuz\.container#/var/lib/kata/kernel#' $kata_config_path
sed -i 's#/usr/share/kata-containers/kata-containers-initrd\.img#/var/lib/kata/kata-containers-initrd\.img#' $kata_config_path
sed -i 's/^image/#image/' $kata_config_path
sed -i 's/^default_memory.*$/default_memory = 1024/' $kata_config_path
sed -i 's/^#enable_blk_mount/enable_blk_mount/' $kata_config_path
sed -i 's/^#block_device_cache_direct.*$/block_device_cache_direct = true/' $kata_config_path
sed -i 's/^#block_device_cache_set.*$/block_device_cache_set = true/' $kata_config_path
sed -i 's#/usr/libexec/kata-containers/kata-proxy#/usr/bin/kata-proxy#' $kata_config_path
sed -i 's#/usr/libexec/kata-containers/kata-shim#/usr/bin/kata-shim#' $kata_config_path
sed -i 's#/usr/libexec/kata-containers/kata-netmon#/usr/bin/kata-netmon#' $kata_config_path
sed -i 's/^#disable_new_netns.*$/disable_new_netns = true/' $kata_config_path
sed -i 's/^#disable_vhost_net.*$/disable_vhost_net = true/' $kata_config_path
sed -i 's/^internetworking_model.*$/internetworking_model=\"none\"/' $kata_config_path
sed -i 's/^enable_compat_old_cni.*$/#enable_compat_old_cni = true/' $kata_config_path
sed -i 's/^sandbox_cgroup_only.*$/sandbox_cgroup_only = true/' $kata_config_path
set +e
%install
cd %{_builddir}/%{name}-%{version}
mkdir -p -m 750 %{buildroot}/usr/bin
install -p -m 750 ./kata-runtime %{buildroot}/usr/bin
install -p -m 750 ./kata-netmon %{buildroot}/usr/bin
mkdir -p -m 750 %{buildroot}/usr/share/defaults/kata-containers
install -p -m 640 ./cli/config/configuration-qemu.toml %{buildroot}/usr/share/defaults/kata-containers/configuration.toml
%clean
%files
/usr/bin/kata-runtime
/usr/bin/kata-netmon
/usr/share/defaults/kata-containers/configuration.toml
%changelog
* Tue Nov 17 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-11
- Type:bugfix
- ID:NA
- SUG:upgrade
- DESC:fix cpu resource limited problem when sandox_cgroup_with_emulator config is enabled
* Fri Oct 9 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-10
- Type:feature
- ID:NA
- SUG:restart
- DESC:support using CNI plugin to insert mutiple network interfaces at the same time
* Mon Sep 28 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-9
- Type:bugfix
- ID:NA
- SUG:restart
- DESC:retry inserting of CNI interface when netmon is enable
* Sun Sep 27 2020 LiangZhang<zhangliang5@Huawei.com> - 1.11.1-8
- Type:bugfix
- ID:NA
- SUG:NA
- DESC:fix cmd params of direct use stratovirt binary
* Thu Sep 24 2020 LiangZhang<zhangliang5@Huawei.com> - 1.11.1-7
- Type:bugfix
- ID:NA
- SUG:NA
- DESC:fix invalid cmdline when start sandbox stratovirt
* Mon Sep 21 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-6
- Type:bugfix
- ID:NA
- SUG:NA
- DESC:fix sandboxRuntimeRootPath left problem
* Mon Sep 21 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-5
- Type:enhancement
- ID:NA
- SUG:NA
- DESC:add support for host cgroups with emulator
* Mon Sep 21 2020 LiangZhang<zhangliang5@Huawei.com> - 1.11.1-4
- Type:enhancement
- ID:NA
- SUG:NA
- DESC:add support of new sandbox StratoVirt
* Sat Sep 19 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-3
- Type:bugfix
- ID:NA
- SUG:NA
- DESC:fix del-iface doesn't delete the tap interface in the host problem
* Sat Sep 5 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-2
- Type:enhancement
- ID:NA
- SUG:NA
- DESC:use URL format for Source0
* Wed Aug 26 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-1
- Type:enhancement
- ID:NA
- SUG:NA
- DESC:modify kata-runtime spec file to build seperately

44
runtime/patches/0001-qmp-fix-kata-runtime-hungs-when-qemu-process-is-D-T-.patch Normal file
View File

@ -0,0 +1,44 @@
From 73fe7242d18a10a86bc216ec5e33a10a8751f85f Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Fri, 24 Jul 2020 22:22:00 +0800
Subject: [PATCH 01/50] qmp: fix kata-runtime hungs when qemu process is D/T
state
reason: When set qemu's status to T and execute add-iface command
the command will hung all the time.It hungs while wait for qemu
to return version messages.
When qmp starts, set the timeout time to 15 seconds.When times
out, return qmp starts failed.We choose 15 seconds to keep consistent
with the agent client start.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
vendor/github.com/intel/govmm/qemu/checklist | 1 +
vendor/github.com/intel/govmm/qemu/qmp.go | 2 ++
2 files changed, 3 insertions(+)
create mode 100644 vendor/github.com/intel/govmm/qemu/checklist
diff --git a/vendor/github.com/intel/govmm/qemu/checklist b/vendor/github.com/intel/govmm/qemu/checklist
new file mode 100644
index 00000000..b32f1855
--- /dev/null
+++ b/vendor/github.com/intel/govmm/qemu/checklist
@@ -0,0 +1 @@
+add timeout when qmp start to avoid qmp client hungs all the time
diff --git a/vendor/github.com/intel/govmm/qemu/qmp.go b/vendor/github.com/intel/govmm/qemu/qmp.go
index bf9a77dd..a64039de 100644
--- a/vendor/github.com/intel/govmm/qemu/qmp.go
+++ b/vendor/github.com/intel/govmm/qemu/qmp.go
@@ -722,6 +722,8 @@ func QMPStart(ctx context.Context, socket string, cfg QMPConfig, disconnectedCh
if q.version == nil {
return nil, nil, fmt.Errorf("failed to find QMP version information")
}
+ case <-time.After(15 * time.Second):
+ return nil, nil, fmt.Errorf("qmp start time out")
}
return q, q.version, nil
--
2.14.3 (Apple Git-98)

117
runtime/patches/0002-kata-runtime-fix-kata-runtime-skip-read-lines-in-pro.patch Normal file
View File

@ -0,0 +1,117 @@
From 1efb88fbf554f3977a1a8aa1c79bc70cc1b66953 Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Sat, 25 Jul 2020 09:22:08 +0800
Subject: [PATCH 02/50] kata-runtime: fix kata-runtime skip read lines in
/proc/mounts file problem
reason: Since /proc/mounts is a virtual file which is changed dynamically by kernel,
if we use file pointer to read content in this file line by line, we may miss read
some lines. So we retry read /proc/mounts file again to fix this problem.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
virtcontainers/utils/utils_linux.go | 58 +++++++++++++++++++++++--------------
1 file changed, 36 insertions(+), 22 deletions(-)
diff --git a/virtcontainers/utils/utils_linux.go b/virtcontainers/utils/utils_linux.go
index ad870d63..6cef4cfb 100644
--- a/virtcontainers/utils/utils_linux.go
+++ b/virtcontainers/utils/utils_linux.go
@@ -7,15 +7,18 @@ package utils
import (
"bufio"
+ "bytes"
"crypto/rand"
"fmt"
"io"
+ "io/ioutil"
"math/big"
"os"
"strings"
"syscall"
"unsafe"
+ "github.com/sirupsen/logrus"
"golang.org/x/sys/unix"
)
@@ -93,6 +96,7 @@ const (
procMountsFile = "/proc/mounts"
fieldsPerLine = 6
+ maxRetryTimes = 5
)
const (
@@ -109,35 +113,45 @@ func GetDevicePathAndFsType(mountPoint string) (devicePath, fsType string, err e
return
}
- var file *os.File
+ var retry int = 0
- file, err = os.Open(procMountsFile)
- if err != nil {
- return
- }
-
- defer file.Close()
+ for retry <= maxRetryTimes {
+ var content []byte
- reader := bufio.NewReader(file)
- for {
- var line string
-
- line, err = reader.ReadString('\n')
- if err == io.EOF {
- err = fmt.Errorf("Mount %s not found", mountPoint)
+ content, err = ioutil.ReadFile(procMountsFile)
+ if err != nil {
return
}
- fields := strings.Fields(line)
- if len(fields) != fieldsPerLine {
- err = fmt.Errorf("Incorrect no of fields (expected %d, got %d)) :%s", fieldsPerLine, len(fields), line)
- return
+ bytesReader := bytes.NewReader(content)
+ reader := bufio.NewReader(bytesReader)
+
+ for {
+ var line string
+
+ line, err = reader.ReadString('\n')
+ if err == io.EOF {
+ err = fmt.Errorf("Mount %s not found", mountPoint)
+ break
+ }
+
+ fields := strings.Fields(line)
+ if len(fields) != fieldsPerLine {
+ err = fmt.Errorf("Incorrect no of fields (expected %d, got %d)) :%s", fieldsPerLine, len(fields), line)
+ return
+ }
+
+ if mountPoint == fields[procPathIndex] {
+ devicePath = fields[procDeviceIndex]
+ fsType = fields[procTypeIndex]
+ return
+ }
}
- if mountPoint == fields[procPathIndex] {
- devicePath = fields[procDeviceIndex]
- fsType = fields[procTypeIndex]
- return
+ retry = retry + 1
+ if retry <= maxRetryTimes {
+ logrus.Warnf("can not find %s in %s, retry %d times again......", mountPoint, procMountsFile, retry)
}
}
+ return "", "", fmt.Errorf("retry %d times fail to get devicePath adn fs type", maxRetryTimes)
}
--
2.14.3 (Apple Git-98)

55
runtime/patches/0003-kata-runtime-fix-kata-proxy-process-left-problem.patch Normal file
View File

@ -0,0 +1,55 @@
From cf595941e1d105af23bc006bf1998ac072733d0a Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Sat, 25 Jul 2020 10:03:35 +0800
Subject: [PATCH 03/50] kata-runtime: fix kata-proxy process left problem
reason: stopSandbox function will send the DestroySandboxRequest
to kata-agent in the VM and then kill the kata-proxy process in
the host. However, if k.sendReq(DestroySandboxRequest) get error,
stopSandbox will return immediately not execute the following kill
kata-proxy process statement, which cause the kata-process left.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
virtcontainers/kata_agent.go | 18 +++++++++++-------
1 file changed, 11 insertions(+), 7 deletions(-)
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
index a0cf190e..be5e96aa 100644
--- a/virtcontainers/kata_agent.go
+++ b/virtcontainers/kata_agent.go
@@ -976,6 +976,17 @@ func (k *kataAgent) stopSandbox(sandbox *Sandbox) error {
return errorMissingProxy
}
+ // since stopSandbox will destroy the sandbox in the VM, and we don't need
+ // kata-proxy process to communicate with kata-agent again, so we should
+ // make sure kata-proxy can be killed cleanly, even when k.sendReq(DestroySandboxRequest)
+ // return error
+ defer func() {
+ _ = k.proxy.stop(k.state.ProxyPid)
+ // clean up agent state
+ k.state.ProxyPid = -1
+ k.state.URL = ""
+ }()
+
req := &grpc.DestroySandboxRequest{}
if _, err := k.sendReq(req); err != nil {
@@ -989,13 +1000,6 @@ func (k *kataAgent) stopSandbox(sandbox *Sandbox) error {
}
}
- if err := k.proxy.stop(k.state.ProxyPid); err != nil {
- return err
- }
-
- // clean up agent state
- k.state.ProxyPid = -1
- k.state.URL = ""
return nil
}
--
2.14.3 (Apple Git-98)

65
runtime/patches/0004-kata-runtime-keep-the-process-name-of-qemu-same-as-c.patch Normal file
View File

@ -0,0 +1,65 @@
From 025520f7fd3aeb5ed53b468b5e494b1bbb6674ae Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Sat, 25 Jul 2020 11:25:34 +0800
Subject: [PATCH 04/50] kata-runtime: keep the process name of qemu same as
configured path
reason: inorder to make testcase scripts can use the same hypervisor
name no matter what version hypervisor use, keep the process name of
hypervisor same as configured path in the configuration.toml file
instead of the resolved path of symbol link.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
pkg/katautils/config.go | 8 +++++++-
pkg/katautils/config_test.go | 4 ++--
2 files changed, 9 insertions(+), 3 deletions(-)
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
index 14794a24..349e667f 100644
--- a/pkg/katautils/config.go
+++ b/pkg/katautils/config.go
@@ -10,6 +10,7 @@ import (
"errors"
"fmt"
"io/ioutil"
+ "path/filepath"
goruntime "runtime"
"strings"
@@ -172,7 +173,12 @@ func (h hypervisor) path() (string, error) {
p = defaultHypervisorPath
}
- return ResolvePath(p)
+ absolutePath, err := filepath.Abs(p)
+ if err != nil {
+ return "", err
+ }
+
+ return absolutePath, nil
}
func (h hypervisor) ctlpath() (string, error) {
diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go
index 221a4b55..2eae1f6a 100644
--- a/pkg/katautils/config_test.go
+++ b/pkg/katautils/config_test.go
@@ -1061,12 +1061,12 @@ func TestHypervisorDefaultsHypervisor(t *testing.T) {
assert.NoError(err)
assert.Equal(p, defaultHypervisorPath, "default hypervisor path wrong")
- // test path resolution
+ // test path resolution, just return the absolute path instead of resolved path
defaultHypervisorPath = testHypervisorLinkPath
h = hypervisor{}
p, err = h.path()
assert.NoError(err)
- assert.Equal(p, testHypervisorPath)
+ assert.Equal(p, testHypervisorLinkPath)
}
func TestHypervisorDefaultsKernel(t *testing.T) {
--
2.14.3 (Apple Git-98)

66
runtime/patches/0005-cgroups-increase-delete-cgroup-retry-times.patch Normal file
View File

@ -0,0 +1,66 @@
From c279f4548ccc534f1c65723bf9994c448e510d3d Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Sat, 25 Jul 2020 11:56:35 +0800
Subject: [PATCH 05/50] cgroups: increase delete cgroup retry times
reason: inorder to make sure cgroup dir to be deleted, so we increase
the retry times when delete cgroup dir failed.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
vendor/github.com/containerd/cgroups/cgroup.go | 4 ++--
vendor/github.com/containerd/cgroups/utils.go | 9 ++++++---
2 files changed, 8 insertions(+), 5 deletions(-)
diff --git a/vendor/github.com/containerd/cgroups/cgroup.go b/vendor/github.com/containerd/cgroups/cgroup.go
index 53866685..69612b0a 100644
--- a/vendor/github.com/containerd/cgroups/cgroup.go
+++ b/vendor/github.com/containerd/cgroups/cgroup.go
@@ -223,7 +223,7 @@ func (c *cgroup) Delete() error {
return err
}
if err := d.Delete(sp); err != nil {
- errors = append(errors, string(s.Name()))
+ errors = append(errors, fmt.Sprintf("delete %s get error: %v", string(s.Name()), err.Error()))
}
continue
}
@@ -234,7 +234,7 @@ func (c *cgroup) Delete() error {
}
path := p.Path(sp)
if err := remove(path); err != nil {
- errors = append(errors, path)
+ errors = append(errors, fmt.Sprintf("remove path %s get error: %v", path, err.Error()))
}
}
}
diff --git a/vendor/github.com/containerd/cgroups/utils.go b/vendor/github.com/containerd/cgroups/utils.go
index 8a97d04d..82dbe2d3 100644
--- a/vendor/github.com/containerd/cgroups/utils.go
+++ b/vendor/github.com/containerd/cgroups/utils.go
@@ -99,16 +99,19 @@ func defaults(root string) ([]Subsystem, error) {
// retrying the remove after a exp timeout
func remove(path string) error {
delay := 10 * time.Millisecond
- for i := 0; i < 5; i++ {
+ var err error
+ var count int = 0
+ for i := 0; i < 10; i++ {
if i != 0 {
time.Sleep(delay)
delay *= 2
}
- if err := os.RemoveAll(path); err == nil {
+ if err = os.RemoveAll(path); err == nil {
return nil
}
+ count++
}
- return fmt.Errorf("cgroups: unable to remove path %q", path)
+ return fmt.Errorf("cgroups: unable to remove path %q, err: %v, count:%d", path, err, count)
}
// readPids will read all the pids of processes in a cgroup by the provided path
--
2.14.3 (Apple Git-98)

39
runtime/patches/0006-kata-runtime-fix-umount-container-rootfs-dir-return-.patch Normal file
View File

@ -0,0 +1,39 @@
From 3dc10421f177900c0ee94fc49b32ec66a46d9331 Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Mon, 27 Jul 2020 19:18:50 +0800
Subject: [PATCH 06/50] kata-runtime: fix umount container rootfs dir return
ivalid argument error
reason: If sandbox hypervisor doesn't use block device driver for hotplugging container
rootfs block device into guest, kata-runtime will bind mount container rootfs dir to 9p
kataShared dir. However, container stop() function will always call bindUnmountContainerRootfs
function no matter block device driver is used or not. So we just need to call
bindUnmountContainerRootfs only if rootfs is bind mount to guest by 9p.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
virtcontainers/container.go | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/virtcontainers/container.go b/virtcontainers/container.go
index 9e2d1e94..b42cc6e9 100644
--- a/virtcontainers/container.go
+++ b/virtcontainers/container.go
@@ -1120,8 +1120,12 @@ func (c *Container) stop(force bool) error {
return err
}
- if err := bindUnmountContainerRootfs(c.ctx, getMountPath(c.sandbox.id), c.id); err != nil && !force {
- return err
+ // umount container rootfs dir only if container use 9p
+ // to bind mount host container rootfs to 9p shared dir
+ if c.state.BlockDeviceID == "" {
+ if err := bindUnmountContainerRootfs(c.ctx, getMountPath(c.sandbox.id), c.id); err != nil && !force {
+ return err
+ }
}
if err := c.detachDevices(); err != nil && !force {
--
2.14.3 (Apple Git-98)

855
runtime/patches/0007-kata-runtime-enhance-reliability-when-kata-related-p.patch Normal file
View File

@ -0,0 +1,855 @@
From d93da1875ed7f1a6061cffb13475506d73c86003 Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Sat, 25 Jul 2020 16:04:19 +0800
Subject: [PATCH 07/50] kata-runtime: enhance reliability when kata related
process
reason: enhance the reliability when kata related processes is abnormal,
make kata-container still destroy the sandbox and clean up all resources.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
cli/delete.go | 6 ++
cli/kill.go | 3 +-
virtcontainers/acrn.go | 2 +-
virtcontainers/agent.go | 3 +
virtcontainers/api.go | 67 +++++++++++++++++++++
virtcontainers/clh.go | 2 +-
virtcontainers/container.go | 55 +++++++++++++++--
virtcontainers/fc.go | 2 +-
virtcontainers/hypervisor.go | 2 +-
virtcontainers/kata_agent.go | 31 ++++++----
virtcontainers/mock_hypervisor.go | 2 +-
virtcontainers/mock_hypervisor_test.go | 2 +-
virtcontainers/noop_agent.go | 4 ++
virtcontainers/pkg/oci/utils.go | 5 ++
virtcontainers/qemu.go | 51 +++++++++-------
virtcontainers/sandbox.go | 106 +++++++++++++++++++++++++++++----
virtcontainers/types/sandbox.go | 14 ++++-
virtcontainers/utils/utils.go | 46 ++++++++++++++
virtcontainers/vm.go | 4 +-
19 files changed, 348 insertions(+), 59 deletions(-)
diff --git a/cli/delete.go b/cli/delete.go
index c2ce52a4..2f5586e5 100644
--- a/cli/delete.go
+++ b/cli/delete.go
@@ -110,6 +110,12 @@ func delete(ctx context.Context, containerID string, force bool) error {
forceStop = true
}
+ if oci.StateToOCIState(status.State.State) == oci.StateUnhealthy {
+ // Set forceStop and force bool flag to true to force delete everything
+ forceStop = true
+ force = true
+ }
+
switch containerType {
case vc.PodSandbox:
if err := deleteSandbox(ctx, sandboxID, force); err != nil {
diff --git a/cli/kill.go b/cli/kill.go
index 60fa41e0..b228205f 100644
--- a/cli/kill.go
+++ b/cli/kill.go
@@ -133,11 +133,12 @@ func kill(ctx context.Context, containerID, signal string, all bool) error {
kataLog.WithField("signal", signal).WithField("container state", status.State.State).Info("kill")
// container MUST be created, running or paused
+ // If container state is unhealthy, should process this exceptional case separately
if status.State.State == types.StateReady || status.State.State == types.StateRunning || status.State.State == types.StatePaused {
if err := vci.KillContainer(ctx, sandboxID, containerID, signum, all); err != nil {
return err
}
- } else if !all {
+ } else if !all && status.State.State != types.StateUnhealthy {
return fmt.Errorf("container not running")
}
diff --git a/virtcontainers/acrn.go b/virtcontainers/acrn.go
index 761eda03..10cae06f 100644
--- a/virtcontainers/acrn.go
+++ b/virtcontainers/acrn.go
@@ -475,7 +475,7 @@ func (a *Acrn) waitSandbox(timeoutSecs int) error {
}
// stopSandbox will stop the Sandbox's VM.
-func (a *Acrn) stopSandbox() (err error) {
+func (a *Acrn) stopSandbox(force bool) (err error) {
span, _ := a.trace("stopSandbox")
defer span.Finish()
diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go
index c62107ec..be9526c7 100644
--- a/virtcontainers/agent.go
+++ b/virtcontainers/agent.go
@@ -259,4 +259,7 @@ type agent interface {
// load data from disk
load(persistapi.AgentState)
+
+ // get proxy process pid
+ getProxyPid() int
}
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
index de569713..fa82d163 100644
--- a/virtcontainers/api.go
+++ b/virtcontainers/api.go
@@ -7,8 +7,10 @@ package virtcontainers
import (
"context"
+ "fmt"
"os"
"runtime"
+ "strings"
"syscall"
deviceApi "github.com/kata-containers/runtime/virtcontainers/device/api"
@@ -18,6 +20,7 @@ import (
vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
"github.com/kata-containers/runtime/virtcontainers/store"
"github.com/kata-containers/runtime/virtcontainers/types"
+ "github.com/kata-containers/runtime/virtcontainers/utils"
specs "github.com/opencontainers/runtime-spec/specs-go"
opentracing "github.com/opentracing/opentracing-go"
"github.com/sirupsen/logrus"
@@ -597,20 +600,51 @@ func statusContainer(sandbox *Sandbox, containerID string) (ContainerStatus, err
container.state.State == types.StatePaused) &&
container.process.Pid > 0 {
+ // If container state is active, however kata-proxy and qemu process all exit already
+ // which means sandbox has beed stopped exceptionally, then we should force delete
+ // sandbox and container state files in sandbox.Store
+ if sandbox.shouldForceDelete() {
+ virtLog.Logger.Warn("sandbox status is abnormal, sandbox should be force deleted")
+ sandbox.forceDeleteSandbox()
+ return ContainerStatus{}, fmt.Errorf("sandbox has beed stopped exceptionally")
+ }
+
running, err := isShimRunning(container.process.Pid)
if err != nil {
return ContainerStatus{}, err
}
+ // If kata-shim process exit or be killed, need to stop the container
if !running {
virtLog.WithFields(logrus.Fields{
"state": container.state.State,
"pid": container.process.Pid}).
Info("container isn't running")
+
if err := container.stop(true); err != nil {
return ContainerStatus{}, err
}
}
+
+ isPodSandbox := (containerID == sandbox.id)
+
+ // If sandbox is unhealthy, process it correctly
+ if !sandbox.health() {
+ // process podSandbox container type case
+ if isPodSandbox {
+ if err := processUnhealthySandbox(sandbox, container); err != nil {
+ return ContainerStatus{}, err
+ }
+ } else {
+ // If container type is pod_container, which means container operations can not be
+ // processed successfully, we should return the error as soon as possible
+ if err := container.setContainerState(types.StateUnhealthy); err != nil {
+ return ContainerStatus{}, err
+ }
+
+ return ContainerStatus{}, fmt.Errorf("container status is unhealthy, stop container failed")
+ }
+ }
}
return ContainerStatus{
@@ -1016,3 +1050,36 @@ func CleanupContainer(ctx context.Context, sandboxID, containerID string, force
return nil
}
+
+// procesUnhealthySandbox only change sandbox state to unhealthy
+// when caller is kata-runtime kill or kata-runtime delete
+func processUnhealthySandbox(sandbox *Sandbox, container *Container) error {
+ // Set all containers state to unhealthy
+ if err := sandbox.setContainersState(types.StateUnhealthy); err != nil {
+ container.Logger().WithError(err).Warn("set all containers state to unhealthy fail")
+ }
+
+ // Set sandbox state to unhealthy
+ if err := sandbox.setSandboxState(types.StateUnhealthy); err != nil {
+ container.Logger().WithError(err).Warn("set sandbox state to unhealthy fail")
+ }
+
+ forceDelete := false
+
+ // If process is kata-runtime kill or kata-runtime delete,
+ // we should kill or delete sandbox forcefully
+ if cmdline, err := utils.GetProcessCmdline(os.Getpid()); err != nil {
+ container.Logger().WithError(err).Warn("fail to get process cmdline info")
+ } else {
+ forceDelete = strings.Contains(cmdline, "kill") || strings.Contains(cmdline, "delete")
+ }
+
+ if forceDelete {
+ // force stop podSandbox type container's kata-shim process
+ if err := stopShim(container.process.Pid); err != nil {
+ container.Logger().WithError(err).Warn("fail to stop podSandbox type container kata-shim")
+ }
+ }
+
+ return nil
+}
diff --git a/virtcontainers/clh.go b/virtcontainers/clh.go
index d40b698b..59510b02 100644
--- a/virtcontainers/clh.go
+++ b/virtcontainers/clh.go
@@ -569,7 +569,7 @@ func (clh *cloudHypervisor) resumeSandbox() error {
}
// stopSandbox will stop the Sandbox's VM.
-func (clh *cloudHypervisor) stopSandbox() (err error) {
+func (clh *cloudHypervisor) stopSandbox(force bool) (err error) {
span, _ := clh.trace("stopSandbox")
defer span.Finish()
clh.Logger().WithField("function", "stopSandbox").Info("Stop Sandbox")
diff --git a/virtcontainers/container.go b/virtcontainers/container.go
index b42cc6e9..9485e708 100644
--- a/virtcontainers/container.go
+++ b/virtcontainers/container.go
@@ -17,8 +17,12 @@ import (
"time"
"github.com/containerd/cgroups"
+ "github.com/kata-containers/runtime/virtcontainers/device/config"
+ "github.com/kata-containers/runtime/virtcontainers/device/manager"
vccgroups "github.com/kata-containers/runtime/virtcontainers/pkg/cgroups"
+ "github.com/kata-containers/runtime/virtcontainers/pkg/rootless"
vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
+ "github.com/kata-containers/runtime/virtcontainers/store"
"github.com/kata-containers/runtime/virtcontainers/types"
"github.com/kata-containers/runtime/virtcontainers/utils"
specs "github.com/opencontainers/runtime-spec/specs-go"
@@ -26,11 +30,6 @@ import (
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/sys/unix"
-
- "github.com/kata-containers/runtime/virtcontainers/device/config"
- "github.com/kata-containers/runtime/virtcontainers/device/manager"
- "github.com/kata-containers/runtime/virtcontainers/pkg/rootless"
- "github.com/kata-containers/runtime/virtcontainers/store"
)
// https://github.com/torvalds/linux/blob/master/include/uapi/linux/major.h
@@ -1047,6 +1046,13 @@ func (c *Container) stop(force bool) error {
return nil
}
+ // If container state is unhealthy, just force kill the container
+ if c.state.State == types.StateUnhealthy {
+ c.forceKillContainer()
+ // after force kill container, then change container state to stopped
+ return c.setContainerState(types.StateStopped)
+ }
+
if err := c.state.ValidTransition(c.state.State, types.StateStopped); err != nil {
return err
}
@@ -1063,6 +1069,8 @@ func (c *Container) stop(force bool) error {
if err := stopShim(c.process.Pid); err != nil {
l.WithError(err).Warn("failed to stop shim")
}
+
+ c.forceKillContainer()
}
}()
@@ -1096,7 +1104,9 @@ func (c *Container) stop(force bool) error {
// this signal will ensure the container will get killed to match
// the state of the shim. This will allow the following call to
// stopContainer() to succeed in such particular case.
- c.kill(syscall.SIGKILL, true)
+ if err := c.kill(syscall.SIGKILL, true); err != nil {
+ c.Logger().Errorf("send signal to container failed: %v", err)
+ }
// Since the agent has supported the MultiWaitProcess, it's better to
// wait the process here to make sure the process has exited before to
@@ -1582,3 +1592,36 @@ func (c *Container) cgroupsUpdate(resources specs.LinuxResources) error {
return nil
}
+
+// forceDeleteContainer force clean container mount info and resources stored in the disk
+func (c *Container) forceDeleteContainer() {
+ if err := c.unmountHostMounts(); err != nil {
+ c.Logger().WithError(err).Warn("container force delete umount host mounts fail")
+ }
+
+ if err := c.sandbox.removeContainer(c.id); err != nil {
+ c.Logger().WithError(err).Warn("sandbox removeContainer fail")
+ }
+
+ if err := c.store.Delete(); err != nil {
+ c.Logger().WithError(err).Warn("force delete container store fail")
+ }
+}
+
+func (c *Container) forceKillContainer() {
+ if err := c.setContainerState(types.StateStopped); err != nil {
+ c.Logger().WithError(err).Warn("force kill container: change container state to StateStopped failed")
+ }
+
+ if err := c.unmountHostMounts(); err != nil {
+ c.Logger().WithError(err).Warn("force kill container: umount container host mounts failed")
+ }
+
+ if err := c.detachDevices(); err != nil {
+ c.Logger().WithError(err).Warn("force kill container: detach container devices failed")
+ }
+
+ if err := c.removeDrive(); err != nil {
+ c.Logger().WithError(err).Warn("force kill container: remove container drive failed")
+ }
+}
diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go
index 97ef5ffc..72a8e192 100644
--- a/virtcontainers/fc.go
+++ b/virtcontainers/fc.go
@@ -864,7 +864,7 @@ func (fc *firecracker) cleanupJail() {
}
// stopSandbox will stop the Sandbox's VM.
-func (fc *firecracker) stopSandbox() (err error) {
+func (fc *firecracker) stopSandbox(force bool) (err error) {
span, _ := fc.trace("stopSandbox")
defer span.Finish()
diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go
index 4b3dd3d0..fd7d1f8e 100644
--- a/virtcontainers/hypervisor.go
+++ b/virtcontainers/hypervisor.go
@@ -766,7 +766,7 @@ func generateVMSocket(id string, useVsock bool, vmStogarePath string) (interface
type hypervisor interface {
createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, stateful bool) error
startSandbox(timeout int) error
- stopSandbox() error
+ stopSandbox(force bool) error
pauseSandbox() error
saveSandbox() error
resumeSandbox() error
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
index be5e96aa..7575d326 100644
--- a/virtcontainers/kata_agent.go
+++ b/virtcontainers/kata_agent.go
@@ -57,8 +57,9 @@ const (
)
var (
- checkRequestTimeout = 30 * time.Second
- defaultRequestTimeout = 60 * time.Second
+ checkRequestTimeout = 10 * time.Second
+ defaultRequestTimeout = 10 * time.Second
+ createContainerTimeout = 120 * time.Second
errorMissingProxy = errors.New("Missing proxy pointer")
errorMissingOCISpec = errors.New("Missing OCI specification")
defaultKataHostSharedDir = "/run/kata-containers/shared/sandboxes/"
@@ -987,17 +988,21 @@ func (k *kataAgent) stopSandbox(sandbox *Sandbox) error {
k.state.URL = ""
}()
- req := &grpc.DestroySandboxRequest{}
+ // If sandbox.state.State is unhealthy, we don't need to send DestroySandboxRequest
+ // to kata-agent, just force stop the sandbox
+ if sandbox.state.State != types.StateUnhealthy {
+ req := &grpc.DestroySandboxRequest{}
- if _, err := k.sendReq(req); err != nil {
- return err
- }
-
- if k.dynamicTracing {
- _, err := k.sendReq(&grpc.StopTracingRequest{})
- if err != nil {
+ if _, err := k.sendReq(req); err != nil {
return err
}
+
+ if k.dynamicTracing {
+ _, err := k.sendReq(&grpc.StopTracingRequest{})
+ if err != nil {
+ return err
+ }
+ }
}
return nil
@@ -2062,6 +2067,8 @@ func (k *kataAgent) getReqContext(reqName string) (ctx context.Context, cancel c
// Wait has no timeout
case grpcCheckRequest:
ctx, cancel = context.WithTimeout(ctx, checkRequestTimeout)
+ case grpcCreateContainerRequest:
+ ctx, cancel = context.WithTimeout(ctx, createContainerTimeout)
default:
ctx, cancel = context.WithTimeout(ctx, defaultRequestTimeout)
}
@@ -2382,3 +2389,7 @@ func (k *kataAgent) load(s persistapi.AgentState) {
k.state.ProxyPid = s.ProxyPid
k.state.URL = s.URL
}
+
+func (k *kataAgent) getProxyPid() int {
+ return k.state.ProxyPid
+}
diff --git a/virtcontainers/mock_hypervisor.go b/virtcontainers/mock_hypervisor.go
index 0c84e43c..a5b67491 100644
--- a/virtcontainers/mock_hypervisor.go
+++ b/virtcontainers/mock_hypervisor.go
@@ -39,7 +39,7 @@ func (m *mockHypervisor) startSandbox(timeout int) error {
return nil
}
-func (m *mockHypervisor) stopSandbox() error {
+func (m *mockHypervisor) stopSandbox(force bool) error {
return nil
}
diff --git a/virtcontainers/mock_hypervisor_test.go b/virtcontainers/mock_hypervisor_test.go
index b73b28f2..827e3192 100644
--- a/virtcontainers/mock_hypervisor_test.go
+++ b/virtcontainers/mock_hypervisor_test.go
@@ -53,7 +53,7 @@ func TestMockHypervisorStartSandbox(t *testing.T) {
func TestMockHypervisorStopSandbox(t *testing.T) {
var m *mockHypervisor
- assert.NoError(t, m.stopSandbox())
+ assert.NoError(t, m.stopSandbox(false))
}
func TestMockHypervisorAddDevice(t *testing.T) {
diff --git a/virtcontainers/noop_agent.go b/virtcontainers/noop_agent.go
index 189f6b3f..8a7cd337 100644
--- a/virtcontainers/noop_agent.go
+++ b/virtcontainers/noop_agent.go
@@ -236,3 +236,7 @@ func (n *noopAgent) save() (s persistapi.AgentState) {
// load is the Noop agent state loader. It does nothing.
func (n *noopAgent) load(s persistapi.AgentState) {}
+
+func (n *noopAgent) getProxyPid() int {
+ return -1
+}
\ No newline at end of file
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
index 5348c57d..cd8d48ce 100644
--- a/virtcontainers/pkg/oci/utils.go
+++ b/virtcontainers/pkg/oci/utils.go
@@ -70,6 +70,9 @@ const (
// StatePaused represents a container that has been paused.
StatePaused = "paused"
+
+ // StateUnhealthy represents a container that is unhealthy
+ StateUnhealthy = "unhealthy"
)
const KernelModulesSeparator = ";"
@@ -964,6 +967,8 @@ func StateToOCIState(state types.StateString) string {
return StateStopped
case types.StatePaused:
return StatePaused
+ case types.StateUnhealthy:
+ return StateUnhealthy
default:
return ""
}
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
index ca286550..4b15d968 100644
--- a/virtcontainers/qemu.go
+++ b/virtcontainers/qemu.go
@@ -687,7 +687,7 @@ func (q *qemu) setupVirtiofsd() (err error) {
q.Logger().Info("virtiofsd quits")
// Wait to release resources of virtiofsd process
cmd.Process.Wait()
- q.stopSandbox()
+ q.stopSandbox(false)
}()
return err
}
@@ -922,11 +922,11 @@ func (q *qemu) waitSandbox(timeout int) error {
}
// stopSandbox will stop the Sandbox's VM.
-func (q *qemu) stopSandbox() error {
+func (q *qemu) stopSandbox(force bool) error {
span, _ := q.trace("stopSandbox")
defer span.Finish()
- q.Logger().Info("Stopping Sandbox")
+ q.Logger().Infof("force stopping Sandbox: %v", force)
if q.stopped {
q.Logger().Info("Already stopped")
return nil
@@ -937,28 +937,37 @@ func (q *qemu) stopSandbox() error {
q.stopped = true
}()
- if q.config.Debug && q.qemuConfig.LogFile != "" {
- f, err := os.OpenFile(q.qemuConfig.LogFile, os.O_RDONLY, 0)
- if err == nil {
- scanner := bufio.NewScanner(f)
- for scanner.Scan() {
- q.Logger().Debug(scanner.Text())
- }
- if err := scanner.Err(); err != nil {
- q.Logger().WithError(err).Debug("read qemu log failed")
+ if !force {
+ if q.config.Debug && q.qemuConfig.LogFile != "" {
+ f, err := os.OpenFile(q.qemuConfig.LogFile, os.O_RDONLY, 0)
+ if err == nil {
+ scanner := bufio.NewScanner(f)
+ for scanner.Scan() {
+ q.Logger().Debug(scanner.Text())
+ }
+ if err := scanner.Err(); err != nil {
+ q.Logger().WithError(err).Debug("read qemu log failed")
+ }
}
}
- }
- err := q.qmpSetup()
- if err != nil {
- return err
- }
+ err := q.qmpSetup()
+ if err != nil {
+ return err
+ }
- err = q.qmpMonitorCh.qmp.ExecuteQuit(q.qmpMonitorCh.ctx)
- if err != nil {
- q.Logger().WithError(err).Error("Fail to execute qmp QUIT")
- return err
+ err = q.qmpMonitorCh.qmp.ExecuteQuit(q.qmpMonitorCh.ctx)
+ if err != nil {
+ q.Logger().WithError(err).Error("Fail to execute qmp QUIT")
+ return err
+ }
+ } else {
+ qemuMainPid := q.getPids()[0]
+ if qemuMainPid <= 1 {
+ return fmt.Errorf("force kill qemu process pid is invalid")
+ }
+
+ _ = syscall.Kill(qemuMainPid, syscall.SIGKILL)
}
return nil
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
index edd1af5b..78188ed7 100644
--- a/virtcontainers/sandbox.go
+++ b/virtcontainers/sandbox.go
@@ -12,19 +12,13 @@ import (
"math"
"net"
"os"
+ "path/filepath"
"strings"
"sync"
"syscall"
"github.com/containerd/cgroups"
"github.com/containernetworking/plugins/pkg/ns"
- "github.com/opencontainers/runc/libcontainer/configs"
- specs "github.com/opencontainers/runtime-spec/specs-go"
- opentracing "github.com/opentracing/opentracing-go"
- "github.com/pkg/errors"
- "github.com/sirupsen/logrus"
- "github.com/vishvananda/netlink"
-
"github.com/kata-containers/agent/protocols/grpc"
"github.com/kata-containers/runtime/virtcontainers/device/api"
"github.com/kata-containers/runtime/virtcontainers/device/config"
@@ -41,6 +35,12 @@ import (
"github.com/kata-containers/runtime/virtcontainers/store"
"github.com/kata-containers/runtime/virtcontainers/types"
"github.com/kata-containers/runtime/virtcontainers/utils"
+ "github.com/opencontainers/runc/libcontainer/configs"
+ specs "github.com/opencontainers/runtime-spec/specs-go"
+ opentracing "github.com/opentracing/opentracing-go"
+ "github.com/pkg/errors"
+ "github.com/sirupsen/logrus"
+ "github.com/vishvananda/netlink"
)
const (
@@ -50,6 +50,9 @@ const (
// DirMode is the permission bits used for creating a directory
DirMode = os.FileMode(0750) | os.ModeDir
+
+ // kata-proxy proces name
+ KataProxyProcessName = "kata-proxy"
)
// SandboxStatus describes a sandbox status.
@@ -1037,7 +1040,7 @@ func (s *Sandbox) startVM() (err error) {
defer func() {
if err != nil {
- s.hypervisor.stopSandbox()
+ s.hypervisor.stopSandbox(false)
}
}()
@@ -1090,7 +1093,12 @@ func (s *Sandbox) stopVM() error {
}
s.Logger().Info("Stopping VM")
- return s.hypervisor.stopSandbox()
+ forceStop := false
+ if s.state.State == types.StateUnhealthy {
+ forceStop = true
+ }
+
+ return s.hypervisor.stopSandbox(forceStop)
}
func (s *Sandbox) addContainer(c *Container) error {
@@ -1591,13 +1599,15 @@ func (s *Sandbox) setSandboxState(state types.StateString) error {
return vcTypes.ErrNeedState
}
+ s.Logger().Debugf("Setting sandbox state from %v to %v", s.state.State, state)
// update in-memory state
s.state.State = state
if useOldStore(s.ctx) {
return s.store.Store(store.State, s.state)
+ } else {
+ return s.Save()
}
- return nil
}
const maxBlockIndex = 65535
@@ -2207,3 +2217,79 @@ func (s *Sandbox) GetPatchedOCISpec() *specs.Spec {
return nil
}
+
+// health return current sandbox healthy or not
+// If qemu/kata-proxy/kata-agent process is abnormal,
+// s.agent.check() will return false
+func (s *Sandbox) health() bool {
+ err := s.agent.check()
+ if err != nil {
+ return false
+ }
+
+ return true
+}
+
+// shouldForceDelete force delete the sandbox when kata-proxy and hypervisor process exit
+// already and current process is kata-runtime kill or kata-runtime delete
+func (s *Sandbox) shouldForceDelete() bool {
+ cmdline, err := utils.GetProcessCmdline(os.Getpid())
+ if err != nil {
+ s.Logger().Errorf("fail to get process cmdline: %v", err)
+ return false
+ }
+
+ proxyPid := s.agent.getProxyPid()
+ hypervisorPids := s.hypervisor.getPids()
+ if len(hypervisorPids) <= 0 {
+ s.Logger().Warnf("get hypervisor main pid fail")
+ return false
+ }
+ hypervisorMainPid := hypervisorPids[0]
+ hypervisorPath := s.hypervisor.hypervisorConfig().HypervisorPath
+ hypervisorName := filepath.Base(hypervisorPath)
+
+ if !utils.IsProcessRunning(proxyPid, KataProxyProcessName, s.id) && !utils.IsProcessRunning(hypervisorMainPid, hypervisorName, s.id) &&
+ strings.Contains(cmdline, "delete") && strings.Contains(cmdline, "force") {
+ return true
+ }
+
+ return false
+}
+
+func (s *Sandbox) forceDeleteSandbox() {
+ for _, c := range s.containers {
+ // force delete all containers in the sandbox
+ c.forceDeleteContainer()
+ }
+
+ globalSandboxList.removeSandbox(s.id)
+
+ if s.monitor != nil {
+ s.monitor.stop()
+ }
+
+ if err := s.hypervisor.cleanup(); err != nil {
+ s.Logger().WithError(err).Error("failed to force cleanup hypervisor resource")
+ }
+
+ s.agent.cleanup(s)
+
+ if err := s.store.Delete(); err != nil {
+ s.Logger().WithError(err).Warn("sandbox force delete store failed")
+ }
+}
+
+func (s *Sandbox) setContainersState(state types.StateString) error {
+ if state == "" {
+ return vcTypes.ErrNeedState
+ }
+
+ for _, c := range s.containers {
+ if err := c.setContainerState(state); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
diff --git a/virtcontainers/types/sandbox.go b/virtcontainers/types/sandbox.go
index 3b64b20a..5d586b21 100644
--- a/virtcontainers/types/sandbox.go
+++ b/virtcontainers/types/sandbox.go
@@ -28,6 +28,9 @@ const (
// StateStopped represents a sandbox/container that has been stopped.
StateStopped StateString = "stopped"
+
+ // StateUnhealthy represents a sandbox/container that's in abnormal state.
+ StateUnhealthy StateString = "unhealthy"
)
const (
@@ -90,17 +93,17 @@ func (state *StateString) validTransition(oldState StateString, newState StateSt
switch *state {
case StateReady:
- if newState == StateRunning || newState == StateStopped {
+ if newState == StateRunning || newState == StateStopped || newState == StateUnhealthy {
return nil
}
case StateRunning:
- if newState == StatePaused || newState == StateStopped {
+ if newState == StatePaused || newState == StateStopped || newState == StateUnhealthy {
return nil
}
case StatePaused:
- if newState == StateRunning || newState == StateStopped {
+ if newState == StateRunning || newState == StateStopped || newState == StateUnhealthy {
return nil
}
@@ -108,6 +111,11 @@ func (state *StateString) validTransition(oldState StateString, newState StateSt
if newState == StateRunning {
return nil
}
+
+ case StateUnhealthy:
+ if newState == StateStopped {
+ return nil
+ }
}
return fmt.Errorf("Can not move from %v to %v",
diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go
index 85c55489..2b555ebb 100644
--- a/virtcontainers/utils/utils.go
+++ b/virtcontainers/utils/utils.go
@@ -9,9 +9,13 @@ import (
"crypto/rand"
"errors"
"fmt"
+ "io/ioutil"
"os"
"os/exec"
"path/filepath"
+ "strconv"
+ "strings"
+ "syscall"
)
const cpBinaryName = "cp"
@@ -275,3 +279,45 @@ const (
MiB = KiB << 10
GiB = MiB << 10
)
+
+// Get process cmdline info by read /proc/<pid>/cmdline file
+func GetProcessCmdline(pid int) (cmdline string, err error) {
+ if pid <= 1 {
+ return "", fmt.Errorf("invalid pid number")
+ }
+
+ bytes, err := ioutil.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "cmdline"))
+ if err != nil {
+ return "", err
+ }
+
+ return string(bytes), nil
+}
+
+func IsProcessRunning(pid int, processName string, sandboxID string) bool {
+ if pid <= 0 {
+ return false
+ }
+
+ process, err := os.FindProcess(pid)
+ if err != nil {
+ return false
+ }
+
+ if err := process.Signal(syscall.Signal(0)); err != nil {
+ return false
+ }
+
+ cmdline, err := GetProcessCmdline(pid)
+ if err != nil {
+ return false
+ }
+
+ // If process's cmdline contains processName and sandboxID keyword,
+ // We think this process isn't be reused
+ if strings.Contains(cmdline, processName) && strings.Contains(cmdline, sandboxID) {
+ return true
+ }
+
+ return false
+}
diff --git a/virtcontainers/vm.go b/virtcontainers/vm.go
index fcda1e97..8d27b1fe 100644
--- a/virtcontainers/vm.go
+++ b/virtcontainers/vm.go
@@ -191,7 +191,7 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) {
defer func() {
if err != nil {
virtLog.WithField("vm", id).WithError(err).Info("clean up vm")
- hypervisor.stopSandbox()
+ hypervisor.stopSandbox(false)
}
}()
@@ -333,7 +333,7 @@ func (v *VM) Disconnect() error {
func (v *VM) Stop() error {
v.logger().Info("stop vm")
- if err := v.hypervisor.stopSandbox(); err != nil {
+ if err := v.hypervisor.stopSandbox(false); err != nil {
return err
}
--
2.14.3 (Apple Git-98)

177
runtime/patches/0008-kata-runtime-fix-kata-runtime-resource-left-problem.patch Normal file
View File

@ -0,0 +1,177 @@
From a1bf2e1c696b703935f4b81ca087a60cc2559464 Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Mon, 27 Jul 2020 21:45:25 +0800
Subject: [PATCH 08/50] kata-runtime: fix kata-runtime resource left problem
reason: fix the following resource left problem
- sandbox stored files and directory are deleted before work container delete
- ignore StatusSandbox error, let stopSandbox function call be executed
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
cli/delete.go | 25 +++++++++++++++++++++++--
cli/delete_test.go | 15 +++++++++++++++
cli/state.go | 6 ++++++
virtcontainers/api.go | 4 +++-
4 files changed, 47 insertions(+), 3 deletions(-)
diff --git a/cli/delete.go b/cli/delete.go
index 2f5586e5..871ac40d 100644
--- a/cli/delete.go
+++ b/cli/delete.go
@@ -10,6 +10,7 @@ import (
"context"
"fmt"
"os"
+ "strings"
"github.com/kata-containers/runtime/pkg/katautils"
vc "github.com/kata-containers/runtime/virtcontainers"
@@ -68,6 +69,12 @@ func delete(ctx context.Context, containerID string, force bool) error {
setExternalLoggers(ctx, kataLog)
span.SetTag("container", containerID)
+ // Remove the containerID to sandboxID mapping
+ // no matter what error return
+ defer func() {
+ _ = katautils.DelContainerIDMapping(ctx, containerID)
+ }()
+
// Checks the MUST and MUST NOT from OCI runtime specification
status, sandboxID, err := getExistingContainerInfo(ctx, containerID)
if err != nil {
@@ -75,6 +82,15 @@ func delete(ctx context.Context, containerID string, force bool) error {
kataLog.Warnf("Failed to get container, force will not fail: %s", err)
return nil
}
+
+ // If err info containers "no such file or directory, because pod_sandbox type
+ // container is deleted before pod_container type container, just return nil
+ // and let containerd delete container operations continue
+ if strings.Contains(err.Error(), "no such file or directory") {
+ kataLog.Warnf("pod_sandbox deleted before pod_container: %v", err)
+ return nil
+ }
+
return err
}
@@ -123,7 +139,12 @@ func delete(ctx context.Context, containerID string, force bool) error {
}
case vc.PodContainer:
if err := deleteContainer(ctx, sandboxID, containerID, forceStop); err != nil {
- return err
+ // If err info containers "no such file or directory, because pod_sandbox type
+ // container is deleted before pod_container type container, just return nil
+ // and let containerd delete container operations continue
+ if !strings.Contains(err.Error(), "no such file or directory") {
+ return err
+ }
}
default:
return fmt.Errorf("Invalid container type found")
@@ -134,7 +155,7 @@ func delete(ctx context.Context, containerID string, force bool) error {
return err
}
- return katautils.DelContainerIDMapping(ctx, containerID)
+ return nil
}
func deleteSandbox(ctx context.Context, sandboxID string, force bool) error {
diff --git a/cli/delete_test.go b/cli/delete_test.go
index a2455dee..ae421cd7 100644
--- a/cli/delete_test.go
+++ b/cli/delete_test.go
@@ -184,6 +184,9 @@ func TestDeleteSandbox(t *testing.T) {
assert.Error(err)
assert.True(vcmock.IsMockError(err))
+ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
+ assert.NoError(err)
+
testingImpl.StatusSandboxFunc = func(ctx context.Context, sandboxID string) (vc.SandboxStatus, error) {
return vc.SandboxStatus{
ID: sandbox.ID(),
@@ -201,6 +204,9 @@ func TestDeleteSandbox(t *testing.T) {
assert.Error(err)
assert.True(vcmock.IsMockError(err))
+ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
+ assert.NoError(err)
+
testingImpl.StopSandboxFunc = func(ctx context.Context, sandboxID string, force bool) (vc.VCSandbox, error) {
return sandbox, nil
}
@@ -213,6 +219,9 @@ func TestDeleteSandbox(t *testing.T) {
assert.Error(err)
assert.True(vcmock.IsMockError(err))
+ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
+ assert.NoError(err)
+
testingImpl.DeleteSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) {
return sandbox, nil
}
@@ -302,6 +311,9 @@ func TestDeleteSandboxRunning(t *testing.T) {
assert.Error(err)
assert.False(vcmock.IsMockError(err))
+ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
+ assert.NoError(err)
+
testingImpl.StatusSandboxFunc = func(ctx context.Context, sandboxID string) (vc.SandboxStatus, error) {
return vc.SandboxStatus{
ID: sandbox.ID(),
@@ -325,6 +337,9 @@ func TestDeleteSandboxRunning(t *testing.T) {
assert.Error(err)
assert.True(vcmock.IsMockError(err))
+ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
+ assert.NoError(err)
+
testingImpl.DeleteSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) {
return sandbox, nil
}
diff --git a/cli/state.go b/cli/state.go
index a2fcc12e..de843d34 100644
--- a/cli/state.go
+++ b/cli/state.go
@@ -11,6 +11,7 @@ import (
"encoding/json"
"fmt"
"os"
+ "strings"
"github.com/kata-containers/runtime/pkg/katautils"
"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
@@ -52,6 +53,11 @@ func state(ctx context.Context, containerID string) error {
// Checks the MUST and MUST NOT from OCI runtime specification
status, _, err := getExistingContainerInfo(ctx, containerID)
if err != nil {
+ // If err info containers "no such file or directory, because pod_sandbox type
+ // container is deleted before pod_container type container, just return nil
+ if strings.Contains(err.Error(), "no such file or directory") {
+ return nil
+ }
return err
}
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
index fa82d163..449a03e0 100644
--- a/virtcontainers/api.go
+++ b/virtcontainers/api.go
@@ -374,7 +374,9 @@ func StatusSandbox(ctx context.Context, sandboxID string) (SandboxStatus, error)
for _, container := range s.containers {
contStatus, err := statusContainer(s, container.id)
if err != nil {
- return SandboxStatus{}, err
+ // Since statusContainer may get error because of qemu process D or T,
+ // So just ignore this error and let StatusSandbox function return actually SandboxStatus
+ s.Logger().Warnf("Status container's status in sandbox get error: %v", contStatus)
}
contStatusList = append(contStatusList, contStatus)
--
2.14.3 (Apple Git-98)

132
runtime/patches/0009-kata-runtime-add-kata-runtime-global-flag-debug.patch Normal file
View File

@ -0,0 +1,132 @@
From 508fd9b94b1b12be3167342b03a47f8f97245e9c Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Tue, 28 Jul 2020 10:53:30 +0800
Subject: [PATCH 09/50] kata-runtime: add kata-runtime global flag --debug
reason: add the same debug flag with runc to adapt to
containerd 1.2.0
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
cli/kata-env_test.go | 2 +-
cli/main.go | 6 +++++-
containerd-shim-v2/create.go | 2 +-
pkg/katautils/config.go | 4 ++--
pkg/katautils/config_test.go | 8 ++++----
5 files changed, 13 insertions(+), 9 deletions(-)
diff --git a/cli/kata-env_test.go b/cli/kata-env_test.go
index b31b6cc2..75bb697f 100644
--- a/cli/kata-env_test.go
+++ b/cli/kata-env_test.go
@@ -178,7 +178,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
return "", oci.RuntimeConfig{}, err
}
- _, config, err = katautils.LoadConfiguration(configFile, true, false)
+ _, config, err = katautils.LoadConfiguration(configFile, true, false, false)
if err != nil {
return "", oci.RuntimeConfig{}, err
}
diff --git a/cli/main.go b/cli/main.go
index df16c5f3..1362a8fb 100644
--- a/cli/main.go
+++ b/cli/main.go
@@ -115,6 +115,10 @@ var runtimeFlags = []cli.Flag{
Name: "systemd-cgroup",
Usage: "enable systemd cgroup support, expects cgroupsPath to be of form \"slice:prefix:name\" for e.g. \"system.slice:runc:434234\"",
},
+ cli.BoolFlag{
+ Name: "debug",
+ Usage: "enable debug output for logging",
+ },
}
// runtimeCommands is the list of supported command-line (sub-)
@@ -334,7 +338,7 @@ func beforeSubcommands(c *cli.Context) error {
}
}
- configFile, runtimeConfig, err = katautils.LoadConfiguration(c.GlobalString(configFilePathOption), ignoreConfigLogs, false)
+ configFile, runtimeConfig, err = katautils.LoadConfiguration(c.GlobalString(configFilePathOption), ignoreConfigLogs, false, c.GlobalBool("debug"))
if err != nil {
fatal(err)
}
diff --git a/containerd-shim-v2/create.go b/containerd-shim-v2/create.go
index affdbae2..9749073d 100644
--- a/containerd-shim-v2/create.go
+++ b/containerd-shim-v2/create.go
@@ -167,7 +167,7 @@ func loadRuntimeConfig(s *service, r *taskAPI.CreateTaskRequest, anno map[string
configPath = os.Getenv("KATA_CONF_FILE")
}
- _, runtimeConfig, err := katautils.LoadConfiguration(configPath, false, true)
+ _, runtimeConfig, err := katautils.LoadConfiguration(configPath, false, true, false)
if err != nil {
return nil, err
}
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
index 349e667f..448d23ac 100644
--- a/pkg/katautils/config.go
+++ b/pkg/katautils/config.go
@@ -1141,7 +1141,7 @@ func initConfig() (config oci.RuntimeConfig, err error) {
//
// All paths are resolved fully meaning if this function does not return an
// error, all paths are valid at the time of the call.
-func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolvedConfigPath string, config oci.RuntimeConfig, err error) {
+func LoadConfiguration(configPath string, ignoreLogging, builtIn bool, debugFlag bool) (resolvedConfigPath string, config oci.RuntimeConfig, err error) {
config, err = initConfig()
if err != nil {
@@ -1154,7 +1154,7 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved
}
config.Debug = tomlConf.Runtime.Debug
- if !tomlConf.Runtime.Debug {
+ if !tomlConf.Runtime.Debug && !debugFlag {
// If debug is not required, switch back to the original
// default log priority, otherwise continue in debug mode.
kataUtilsLogger.Logger.Level = originalLoggerLevel
diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go
index 2eae1f6a..31afcca6 100644
--- a/pkg/katautils/config_test.go
+++ b/pkg/katautils/config_test.go
@@ -264,7 +264,7 @@ func testLoadConfiguration(t *testing.T, dir string,
assert.NoError(t, err)
}
- resolvedConfigPath, config, err := LoadConfiguration(file, ignoreLogging, false)
+ resolvedConfigPath, config, err := LoadConfiguration(file, ignoreLogging, false, false)
if expectFail {
assert.Error(t, err)
@@ -578,7 +578,7 @@ func TestMinimalRuntimeConfig(t *testing.T) {
t.Fatal(err)
}
- _, config, err := LoadConfiguration(configPath, false, false)
+ _, config, err := LoadConfiguration(configPath, false, false, false)
if err == nil {
t.Fatalf("Expected loadConfiguration to fail as shim path does not exist: %+v", config)
}
@@ -608,7 +608,7 @@ func TestMinimalRuntimeConfig(t *testing.T) {
t.Error(err)
}
- _, config, err = LoadConfiguration(configPath, false, false)
+ _, config, err = LoadConfiguration(configPath, false, false, false)
if err != nil {
t.Fatal(err)
}
@@ -748,7 +748,7 @@ func TestMinimalRuntimeConfigWithVsock(t *testing.T) {
t.Fatal(err)
}
- _, config, err := LoadConfiguration(configPath, false, false)
+ _, config, err := LoadConfiguration(configPath, false, false, false)
if err != nil {
t.Fatal(err)
}
--
2.14.3 (Apple Git-98)

172
runtime/patches/0010-kata-runtime-fix-kata-shim-pid-reused-problem.patch Normal file
View File

@ -0,0 +1,172 @@
From 76cbca91608e94c1855705ad1a8d06ffa2273115 Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Tue, 28 Jul 2020 18:18:54 +0800
Subject: [PATCH 10/50] kata-runtime: fix kata-shim pid reused problem
reason: If kata-shim process exit and it's pid reused by other process,
it may cause kill other proecss and cause some problem.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
virtcontainers/api.go | 2 +-
virtcontainers/container.go | 6 +++---
virtcontainers/shim.go | 21 +++++++++++++++++----
virtcontainers/shim_test.go | 10 +++++-----
4 files changed, 26 insertions(+), 13 deletions(-)
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
index 449a03e0..5e8c9c9e 100644
--- a/virtcontainers/api.go
+++ b/virtcontainers/api.go
@@ -611,7 +611,7 @@ func statusContainer(sandbox *Sandbox, containerID string) (ContainerStatus, err
return ContainerStatus{}, fmt.Errorf("sandbox has beed stopped exceptionally")
}
- running, err := isShimRunning(container.process.Pid)
+ running, err := isShimRunning(container.process.Pid, containerID)
if err != nil {
return ContainerStatus{}, err
}
diff --git a/virtcontainers/container.go b/virtcontainers/container.go
index 9485e708..75f590eb 100644
--- a/virtcontainers/container.go
+++ b/virtcontainers/container.go
@@ -1063,7 +1063,7 @@ func (c *Container) stop(force bool) error {
// If shim is still running something went wrong
// Make sure we stop the shim process
- if running, _ := isShimRunning(c.process.Pid); running {
+ if running, _ := isShimRunning(c.process.Pid, c.id); running {
l := c.Logger()
l.Error("Failed to stop container so stopping dangling shim")
if err := stopShim(c.process.Pid); err != nil {
@@ -1081,7 +1081,7 @@ func (c *Container) stop(force bool) error {
// However, if the signal didn't reach its goal, the caller still
// expects this container to be stopped, that's why we should not
// return an error, but instead try to kill it forcefully.
- if err := waitForShim(c.process.Pid); err != nil {
+ if err := waitForShim(c.process.Pid, c.id); err != nil {
// Force the container to be killed.
if err := c.kill(syscall.SIGKILL, true); err != nil && !force {
return err
@@ -1091,7 +1091,7 @@ func (c *Container) stop(force bool) error {
// to succeed. Indeed, we have already given a second chance
// to the container by trying to kill it with SIGKILL, there
// is no reason to try to go further if we got an error.
- if err := waitForShim(c.process.Pid); err != nil && !force {
+ if err := waitForShim(c.process.Pid, c.id); err != nil && !force {
return err
}
}
diff --git a/virtcontainers/shim.go b/virtcontainers/shim.go
index 8ec7458b..6f784a03 100644
--- a/virtcontainers/shim.go
+++ b/virtcontainers/shim.go
@@ -9,11 +9,13 @@ import (
"fmt"
"os"
"os/exec"
+ "strings"
"syscall"
"time"
ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter"
"github.com/kata-containers/runtime/virtcontainers/types"
+ "github.com/kata-containers/runtime/virtcontainers/utils"
"github.com/mitchellh/mapstructure"
"github.com/sirupsen/logrus"
)
@@ -227,7 +229,7 @@ func startShim(args []string, params ShimParams) (int, error) {
return cmd.Process.Pid, nil
}
-func isShimRunning(pid int) (bool, error) {
+func isShimRunning(pid int, containerID string) (bool, error) {
if pid <= 0 {
return false, nil
}
@@ -241,19 +243,30 @@ func isShimRunning(pid int) (bool, error) {
return false, nil
}
- return true, nil
+ cmdline, err := utils.GetProcessCmdline(pid)
+ if err != nil {
+ return false, nil
+ }
+
+ // If process's cmdline contains kata-shim and containerID keyword, we think this process pid isn't be reused
+ if strings.Contains(cmdline, "kata-shim") && strings.Contains(cmdline, containerID) {
+ return true, nil
+ }
+
+ shimLogger().Errorf("%d process isn't a kata-shim process", pid)
+ return false, nil
}
// waitForShim waits for the end of the shim unless it reaches the timeout
// first, returning an error in that case.
-func waitForShim(pid int) error {
+func waitForShim(pid int, containerID string) error {
if pid <= 0 {
return nil
}
tInit := time.Now()
for {
- running, err := isShimRunning(pid)
+ running, err := isShimRunning(pid, containerID)
if err != nil {
return err
}
diff --git a/virtcontainers/shim_test.go b/virtcontainers/shim_test.go
index e9bd027c..62471311 100644
--- a/virtcontainers/shim_test.go
+++ b/virtcontainers/shim_test.go
@@ -190,7 +190,7 @@ func TestStopShimSuccessfulProcessRunning(t *testing.T) {
func testIsShimRunning(t *testing.T, pid int, expected bool) {
assert := assert.New(t)
- running, err := isShimRunning(pid)
+ running, err := isShimRunning(pid, containerID)
assert.NoError(err)
assert.Equal(running, expected)
}
@@ -205,7 +205,7 @@ func TestIsShimRunningTrue(t *testing.T) {
cmd := testRunSleep999AndGetCmd(t)
assert := assert.New(t)
- testIsShimRunning(t, cmd.Process.Pid, true)
+ testIsShimRunning(t, cmd.Process.Pid, false)
err := syscall.Kill(cmd.Process.Pid, syscall.SIGKILL)
assert.NoError(err)
@@ -216,7 +216,7 @@ func TestWaitForShimInvalidPidSuccessful(t *testing.T) {
assert := assert.New(t)
for _, val := range wrongValuesList {
- err := waitForShim(val)
+ err := waitForShim(val, containerID)
assert.NoError(err)
}
}
@@ -224,7 +224,7 @@ func TestWaitForShimInvalidPidSuccessful(t *testing.T) {
func TestWaitForShimNotRunningSuccessful(t *testing.T) {
pid := testRunSleep0AndGetPid(t)
assert := assert.New(t)
- assert.NoError(waitForShim(pid))
+ assert.NoError(waitForShim(pid, containerID))
}
func TestWaitForShimRunningForTooLongFailure(t *testing.T) {
@@ -232,6 +232,6 @@ func TestWaitForShimRunningForTooLongFailure(t *testing.T) {
assert := assert.New(t)
waitForShimTimeout = 0.1
- assert.Error(waitForShim(cmd.Process.Pid))
+ assert.NoError(waitForShim(cmd.Process.Pid, containerID))
assert.NoError(syscall.Kill(cmd.Process.Pid, syscall.SIGKILL))
}
--
2.14.3 (Apple Git-98)

119
runtime/patches/0011-kata-runtime-check-the-process-info-before-send-SIGK.patch Normal file
View File

@ -0,0 +1,119 @@
From 0aeff2632eac58eefdc8ae438891303332831ec5 Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Tue, 28 Jul 2020 20:48:24 +0800
Subject: [PATCH 11/50] kata-runtime: check the process info before send
SIGKILL
reason: In order to avoid the pid reuse problem, check the
process info before send SIGKILL signal to process.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
virtcontainers/kata_proxy.go | 18 ++++++++++++++++++
virtcontainers/qemu.go | 5 +++++
virtcontainers/shim.go | 9 +++++++++
virtcontainers/shim_test.go | 8 ++++----
4 files changed, 36 insertions(+), 4 deletions(-)
diff --git a/virtcontainers/kata_proxy.go b/virtcontainers/kata_proxy.go
index e04b4cff..ed272bad 100644
--- a/virtcontainers/kata_proxy.go
+++ b/virtcontainers/kata_proxy.go
@@ -6,8 +6,12 @@
package virtcontainers
import (
+ "fmt"
"os/exec"
+ "strings"
"syscall"
+
+ "github.com/kata-containers/runtime/virtcontainers/utils"
)
// This is the Kata Containers implementation of the proxy interface.
@@ -61,6 +65,20 @@ func (p *kataProxy) start(params proxyParams) (int, string, error) {
// stop is kataProxy stop implementation for proxy interface.
func (p *kataProxy) stop(pid int) error {
+ if pid <= 1 {
+ return nil
+ }
+
+ // check process info before send SIGKILL signal
+ cmdline, err := utils.GetProcessCmdline(pid)
+ if err != nil {
+ return fmt.Errorf("get kata-proxy %d cmdline error: %v", pid, err)
+ }
+
+ if !strings.Contains(cmdline, KataProxyProcessName) {
+ return fmt.Errorf("%d is not kata-proxy process, don't kill wrong process", pid)
+ }
+
// Signal the proxy with SIGTERM.
return syscall.Kill(pid, syscall.SIGTERM)
}
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
index 4b15d968..4789101d 100644
--- a/virtcontainers/qemu.go
+++ b/virtcontainers/qemu.go
@@ -967,6 +967,11 @@ func (q *qemu) stopSandbox(force bool) error {
return fmt.Errorf("force kill qemu process pid is invalid")
}
+ cmdline, _ := utils.GetProcessCmdline(qemuMainPid)
+ if !strings.Contains(cmdline, string(QemuHypervisor)) {
+ return fmt.Errorf("force kill %d process is not qemu process, don't kill wrong process", qemuMainPid)
+ }
+
_ = syscall.Kill(qemuMainPid, syscall.SIGKILL)
}
diff --git a/virtcontainers/shim.go b/virtcontainers/shim.go
index 6f784a03..b192b258 100644
--- a/virtcontainers/shim.go
+++ b/virtcontainers/shim.go
@@ -143,6 +143,15 @@ func stopShim(pid int) error {
return nil
}
+ cmdline, err := utils.GetProcessCmdline(pid)
+ if err != nil {
+ return err
+ }
+
+ if !strings.Contains(cmdline, "kata-shim") {
+ return fmt.Errorf("%d process is not kata-shim process, don't kill wrong process", pid)
+ }
+
if err := signalShim(pid, syscall.SIGKILL); err != nil && err != syscall.ESRCH {
return err
}
diff --git a/virtcontainers/shim_test.go b/virtcontainers/shim_test.go
index 62471311..dc15eab0 100644
--- a/virtcontainers/shim_test.go
+++ b/virtcontainers/shim_test.go
@@ -176,16 +176,16 @@ func testRunSleep999AndGetCmd(t *testing.T) *exec.Cmd {
return cmd
}
-func TestStopShimSuccessfulProcessNotRunning(t *testing.T) {
+func TestStopShimFailProcessNotRunning(t *testing.T) {
assert := assert.New(t)
pid := testRunSleep0AndGetPid(t)
- assert.NoError(stopShim(pid))
+ assert.Error(stopShim(pid))
}
-func TestStopShimSuccessfulProcessRunning(t *testing.T) {
+func TestStopShimFailProcessRunning(t *testing.T) {
assert := assert.New(t)
cmd := testRunSleep999AndGetCmd(t)
- assert.NoError(stopShim(cmd.Process.Pid))
+ assert.Error(stopShim(cmd.Process.Pid))
}
func testIsShimRunning(t *testing.T, pid int, expected bool) {
--
2.14.3 (Apple Git-98)

37
runtime/patches/0012-kata-runtime-truncate-the-log.json-file-before-kata-.patch Normal file
View File

@ -0,0 +1,37 @@
From 441d80f55f4dc5efb4c92d91608a3c8db3d087cb Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Tue, 28 Jul 2020 21:43:15 +0800
Subject: [PATCH 12/50] kata-runtime: truncate the log.json file before
kata-runtime subcommand executed
reason: since we have redirect the kata-runtime log to /var/log/messages, and avoid the
path of log.json file to be large in the tmpfs, so we truncate the log.json file every
time before subcommand is executed.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
cli/main.go | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/cli/main.go b/cli/main.go
index 1362a8fb..5eb2fb19 100644
--- a/cli/main.go
+++ b/cli/main.go
@@ -300,6 +300,14 @@ func beforeSubcommands(c *cli.Context) error {
ignoreConfigLogs = true
} else {
if path := c.GlobalString("log"); path != "" {
+ // since we have redirect the kata-runtime log to /var/log/messages, and avoid the
+ // path of log.json file to be large in the tmpfs, so we truncate the log.json file
+ // every time before subcommand is executed.
+ if path != "/dev/null" && katautils.FileExists(path) {
+ if err := os.Truncate(path, 0); err != nil {
+ return err
+ }
+ }
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND|os.O_SYNC, 0640)
if err != nil {
return err
--
2.14.3 (Apple Git-98)

98
runtime/patches/0013-kata-runtime-get-container-info-by-containerID-prefi.patch Normal file
View File

@ -0,0 +1,98 @@
From fd63d26a5b0542f35d61b0c19c80795f052b4518 Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Tue, 28 Jul 2020 22:05:44 +0800
Subject: [PATCH 13/50] kata-runtime: get container info by containerID prefix
reason: get container info by containerID prefix, so we just
need to input the prefix containerID when call kata-runtime
subcommand
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
cli/oci.go | 35 +++++++++++++++++++++++++++++++++++
pkg/katautils/oci.go | 5 +++++
2 files changed, 40 insertions(+)
diff --git a/cli/oci.go b/cli/oci.go
index 8ffac2df..bf962d03 100644
--- a/cli/oci.go
+++ b/cli/oci.go
@@ -9,6 +9,7 @@ import (
"bufio"
"context"
"fmt"
+ "io/ioutil"
"net"
"os"
"path/filepath"
@@ -25,6 +26,8 @@ const (
// Filesystem type corresponding to CGROUP_SUPER_MAGIC as listed
// here: http://man7.org/linux/man-pages/man2/statfs.2.html
cgroupFsType = 0x27e0eb
+
+ maxIDLength = 64
)
var cgroupsDirPath string
@@ -38,6 +41,14 @@ func getContainerInfo(ctx context.Context, containerID string) (vc.ContainerStat
return vc.ContainerStatus{}, "", fmt.Errorf("Missing container ID")
}
+ if len(containerID) < maxIDLength {
+ fullContainerID, err := getContainerIDbyPrefix(containerID)
+ if err != nil {
+ return vc.ContainerStatus{}, "", err
+ }
+ containerID = fullContainerID
+ }
+
sandboxID, err := katautils.FetchContainerIDMapping(containerID)
if err != nil {
return vc.ContainerStatus{}, "", err
@@ -211,3 +222,27 @@ func getCgroupsDirPath(mountInfoFile string) (string, error) {
return cgroupRootPath, nil
}
+
+func getContainerIDbyPrefix(prefix string) (string, error) {
+ files, err := ioutil.ReadDir(katautils.GetCtrsMapTreePath())
+ if err != nil {
+ return "", err
+ }
+
+ containers := []string{}
+ for _, file := range files {
+ if file.IsDir() && strings.HasPrefix(file.Name(), prefix) {
+ containers = append(containers, file.Name())
+ }
+ }
+
+ if len(containers) == 0 {
+ return "", fmt.Errorf("no such container ID (%v)", prefix)
+ }
+
+ if len(containers) > 1 {
+ return "", fmt.Errorf("multiple containers found (%v)", prefix)
+ }
+
+ return containers[0], nil
+}
diff --git a/pkg/katautils/oci.go b/pkg/katautils/oci.go
index 6de8101e..1334af35 100644
--- a/pkg/katautils/oci.go
+++ b/pkg/katautils/oci.go
@@ -25,6 +25,11 @@ func SetCtrsMapTreePath(path string) {
ctrsMapTreePath = path
}
+// GetCtrsMapTreePath return the containerID to SandboxID mapping dir
+func GetCtrsMapTreePath() string {
+ return ctrsMapTreePath
+}
+
// doUpdatePath returns whether a ctrsMapTreePath needs to be updated with a rootless prefix
func doUpdatePath() bool {
return rootless.IsRootless() && !strings.HasPrefix(ctrsMapTreePath, rootless.GetRootlessDir())
--
2.14.3 (Apple Git-98)

76
runtime/patches/0014-kata-runtime-add-self-defined-annotations-framework.patch Normal file
View File

@ -0,0 +1,76 @@
From 1bd3cb85a1cf0e94b3280412d5fb47cecc4721fd Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Wed, 29 Jul 2020 10:42:49 +0800
Subject: [PATCH 14/50] kata-runtime: add self defined annotations framework
reason: add self defined annotations framework to pass some
self defined resource for sandbox
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
virtcontainers/pkg/oci/utils.go | 38 ++++++++++++++++++++++++++++++++++++++
1 file changed, 38 insertions(+)
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
index cd8d48ce..05181efd 100644
--- a/virtcontainers/pkg/oci/utils.go
+++ b/virtcontainers/pkg/oci/utils.go
@@ -33,6 +33,10 @@ type annotationContainerType struct {
containerType vc.ContainerType
}
+type annotationHandler func(value string) error
+
+var annotationHandlerList = map[string]annotationHandler{}
+
var (
// ErrNoLinux is an error for missing Linux sections in the OCI configuration file.
ErrNoLinux = errors.New("missing Linux section")
@@ -342,6 +346,10 @@ func addAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) error {
if err := addAgentConfigOverrides(ocispec, config); err != nil {
return err
}
+
+ if err := addOtherSandboxAnnotation(ocispec, config); err != nil {
+ return err
+ }
return nil
}
@@ -1016,3 +1024,33 @@ func GetOCIConfig(status vc.ContainerStatus) (specs.Spec, error) {
return *status.Spec, nil
}
+
+// validateOtherSandboxAnnotations validate the value of annotation
+func validateOtherSandboxAnnotations(annotation, value string) error {
+ validateHandler, ok := annotationHandlerList[annotation]
+ if !ok {
+ return fmt.Errorf("unsupport Sandbox annotation type")
+ }
+
+ return validateHandler(value)
+}
+
+// addOtherSandboxAnnotation add self defined annotation for sandbox
+func addOtherSandboxAnnotation(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error {
+ otherSandboxAnnotationsKey := []string{}
+
+ for _, a := range otherSandboxAnnotationsKey {
+ value, ok := ocispec.Annotations[a]
+ if !ok {
+ continue
+ }
+
+ if err := validateOtherSandboxAnnotations(a, value); err != nil {
+ return err
+ }
+
+ sbConfig.Annotations[a] = value
+ }
+
+ return nil
+}
--
2.14.3 (Apple Git-98)

143
runtime/patches/0015-kata-runtime-add-reuse-hypervisor-cpu-and-memory-fea.patch Normal file
View File

@ -0,0 +1,143 @@
From c0a33c4584e1fbd9b39ff1ca3ed632efe85de65e Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Sun, 2 Aug 2020 15:51:10 +0800
Subject: [PATCH 15/50] kata-runtime: add reuse hypervisor cpu and memory
feature
reason: If default hypervisor cpu and memory is set too large,
which may waste resource, so we add enable_reuse_cpu_memory
config in the configuration.toml file to choose share the
hypervisor cpu and memory with container or not.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
cli/config/configuration-qemu.toml.in | 5 +++++
pkg/katautils/config.go | 2 ++
virtcontainers/hypervisor.go | 3 +++
virtcontainers/persist.go | 2 ++
virtcontainers/persist/api/config.go | 3 +++
virtcontainers/sandbox.go | 22 ++++++++++++++++++----
6 files changed, 33 insertions(+), 4 deletions(-)
diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in
index 46ce7d9b..82461732 100644
--- a/cli/config/configuration-qemu.toml.in
+++ b/cli/config/configuration-qemu.toml.in
@@ -95,6 +95,11 @@ default_memory = @DEFMEMSZ@
# Default false
#enable_virtio_mem = true
+# Specifies share hypervisor default cpu and memory resource
+# between hypervisor and container process.
+# Default false
+#enable_reuse_cpu_memory = false
+
# Disable block device from being used for a container's rootfs.
# In case of a storage driver like devicemapper where a container's
# root file system is backed by a block device, the block device is passed
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
index 448d23ac..d1883c94 100644
--- a/pkg/katautils/config.go
+++ b/pkg/katautils/config.go
@@ -112,6 +112,7 @@ type hypervisor struct {
MemorySize uint32 `toml:"default_memory"`
MemSlots uint32 `toml:"memory_slots"`
MemOffset uint32 `toml:"memory_offset"`
+ EnableCPUMemoryReuse bool `toml:"enable_reuse_cpu_memory"`
DefaultBridges uint32 `toml:"default_bridges"`
Msize9p uint32 `toml:"msize_9p"`
PCIeRootPort uint32 `toml:"pcie_root_port"`
@@ -640,6 +641,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
MemorySize: h.defaultMemSz(),
MemSlots: h.defaultMemSlots(),
MemOffset: h.defaultMemOffset(),
+ EnableCPUMemoryReuse: h.EnableCPUMemoryReuse,
VirtioMem: h.VirtioMem,
EntropySource: h.GetEntropySource(),
DefaultBridges: h.defaultBridges(),
diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go
index fd7d1f8e..5f8d24f9 100644
--- a/virtcontainers/hypervisor.go
+++ b/virtcontainers/hypervisor.go
@@ -250,6 +250,9 @@ type HypervisorConfig struct {
// MemOffset specifies memory space for nvdimm device
MemOffset uint32
+ // Enable hypervisor cpu and memory resource share with container
+ EnableCPUMemoryReuse bool
+
// VirtioFSCacheSize is the DAX cache size in MiB
VirtioFSCacheSize uint32
diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go
index d96a3890..aef4d18d 100644
--- a/virtcontainers/persist.go
+++ b/virtcontainers/persist.go
@@ -214,6 +214,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
Msize9p: sconfig.HypervisorConfig.Msize9p,
MemSlots: sconfig.HypervisorConfig.MemSlots,
MemOffset: sconfig.HypervisorConfig.MemOffset,
+ EnableCPUMemoryReuse: sconfig.HypervisorConfig.EnableCPUMemoryReuse,
VirtioMem: sconfig.HypervisorConfig.VirtioMem,
VirtioFSCacheSize: sconfig.HypervisorConfig.VirtioFSCacheSize,
KernelPath: sconfig.HypervisorConfig.KernelPath,
@@ -503,6 +504,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
Msize9p: hconf.Msize9p,
MemSlots: hconf.MemSlots,
MemOffset: hconf.MemOffset,
+ EnableCPUMemoryReuse: hconf.EnableCPUMemoryReuse,
VirtioMem: hconf.VirtioMem,
VirtioFSCacheSize: hconf.VirtioFSCacheSize,
KernelPath: hconf.KernelPath,
diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go
index 34a5fd0f..a3c6ec91 100644
--- a/virtcontainers/persist/api/config.go
+++ b/virtcontainers/persist/api/config.go
@@ -35,6 +35,9 @@ type HypervisorConfig struct {
// MemOffset specifies memory space for nvdimm device
MemOffset uint32
+ // Enable hypervisor cpu and memory resource share with container
+ EnableCPUMemoryReuse bool
+
// VirtioFSCacheSize is the DAX cache size in MiB
VirtioFSCacheSize uint32
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
index 78188ed7..e766d1f7 100644
--- a/virtcontainers/sandbox.go
+++ b/virtcontainers/sandbox.go
@@ -1833,12 +1833,26 @@ func (s *Sandbox) updateResources() error {
}
sandboxVCPUs := s.calculateSandboxCPUs()
- // Add default vcpus for sandbox
- sandboxVCPUs += s.hypervisor.hypervisorConfig().NumVCPUs
+ // Share the hypervisor vcpu with container process
+ if s.config.HypervisorConfig.EnableCPUMemoryReuse {
+ if sandboxVCPUs <= s.config.HypervisorConfig.NumVCPUs {
+ sandboxVCPUs = s.config.HypervisorConfig.NumVCPUs
+ }
+ } else {
+ // Add default vcpus for sandbox
+ sandboxVCPUs += s.hypervisor.hypervisorConfig().NumVCPUs
+ }
sandboxMemoryByte := s.calculateSandboxMemory()
- // Add default / rsvd memory for sandbox.
- sandboxMemoryByte += int64(s.hypervisor.hypervisorConfig().MemorySize) << utils.MibToBytesShift
+ // Share the hypervisor memory with container process
+ if s.config.HypervisorConfig.EnableCPUMemoryReuse {
+ if sandboxMemoryByte <= (int64(s.config.HypervisorConfig.MemorySize) << utils.MibToBytesShift) {
+ sandboxMemoryByte = int64(s.config.HypervisorConfig.MemorySize) << utils.MibToBytesShift
+ }
+ } else {
+ // Add default / rsvd memory for sandbox.
+ sandboxMemoryByte += int64(s.hypervisor.hypervisorConfig().MemorySize) << utils.MibToBytesShift
+ }
// Update VCPUs
s.Logger().WithField("cpus-sandbox", sandboxVCPUs).Debugf("Request to hypervisor to update vCPUs")
--
2.14.3 (Apple Git-98)

288
runtime/patches/0016-virtcontainers-fix-hotplug-huge-size-memory-cause-ag.patch Normal file
View File

@ -0,0 +1,288 @@
From dc9de8bb181e2cec2f3e0a76d02833fef45b46af Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Thu, 6 Aug 2020 09:28:34 -0400
Subject: [PATCH 16/50] virtcontainers: fix hotplug huge size memory cause
agent hang bug
fixes: #2872
reason: If hotplug huge size memory into kata VM at once time,
guest kernel will allocate some extra memory for memory management,
which may cause kata-agent hang and out of responding.
And hotplug more memory into VM, more extra memory is needed.
Inorder to solve this problem, we divide hotplug huge memory into
two steps. First, hotplug the max allowed memory into VM and wait
all first step hotplugged memory online. Second Step, hotplug the
left memory into VM.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
virtcontainers/acrn.go | 4 ++++
virtcontainers/agent.go | 3 ++-
virtcontainers/clh.go | 4 ++++
virtcontainers/fc.go | 4 ++++
virtcontainers/hypervisor.go | 3 +++
virtcontainers/kata_agent.go | 4 ++--
virtcontainers/kata_agent_test.go | 2 +-
virtcontainers/mock_hypervisor.go | 4 ++++
virtcontainers/noop_agent.go | 2 +-
virtcontainers/qemu.go | 4 ++++
virtcontainers/sandbox.go | 30 ++++++++++++++++++++++++++++--
virtcontainers/sandbox_test.go | 12 ++++++++++++
virtcontainers/utils/utils.go | 3 +++
virtcontainers/vm.go | 2 +-
14 files changed, 73 insertions(+), 8 deletions(-)
diff --git a/virtcontainers/acrn.go b/virtcontainers/acrn.go
index 10cae06f..c9a0fe0b 100644
--- a/virtcontainers/acrn.go
+++ b/virtcontainers/acrn.go
@@ -811,3 +811,7 @@ func (a *Acrn) loadInfo() error {
}
return nil
}
+
+func (a *Acrn) getMemorySize() uint32 {
+ return a.config.MemorySize
+}
diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go
index be9526c7..b1dea816 100644
--- a/virtcontainers/agent.go
+++ b/virtcontainers/agent.go
@@ -201,7 +201,8 @@ type agent interface {
// This function should be called after hot adding vCPUs or Memory.
// cpus specifies the number of CPUs that were added and the agent should online
// cpuOnly specifies that we should online cpu or online memory or both
- onlineCPUMem(cpus uint32, cpuOnly bool) error
+ // wait specifies that we should wait all cpu or memory online in the VM synchronously
+ onlineCPUMem(cpus uint32, cpuOnly bool, wait bool) error
// memHotplugByProbe will notify the guest kernel about memory hotplug event through
// probe interface.
diff --git a/virtcontainers/clh.go b/virtcontainers/clh.go
index 59510b02..8afcd4bf 100644
--- a/virtcontainers/clh.go
+++ b/virtcontainers/clh.go
@@ -1210,3 +1210,7 @@ func (clh *cloudHypervisor) vmInfo() (chclient.VmInfo, error) {
return info, openAPIClientError(err)
}
+
+func (clh *cloudHypervisor) getMemorySize() uint32 {
+ return clh.config.MemorySize
+}
diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go
index 72a8e192..15726156 100644
--- a/virtcontainers/fc.go
+++ b/virtcontainers/fc.go
@@ -1212,3 +1212,7 @@ func (fc *firecracker) watchConsole() (*os.File, error) {
return stdio, nil
}
+
+func (fc *firecracker) getMemorySize() uint32 {
+ return fc.config.MemorySize
+}
diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go
index 5f8d24f9..9cd685ad 100644
--- a/virtcontainers/hypervisor.go
+++ b/virtcontainers/hypervisor.go
@@ -778,6 +778,9 @@ type hypervisor interface {
hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error)
resizeMemory(memMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error)
resizeVCPUs(vcpus uint32) (uint32, uint32, error)
+ // getMemorySize return the total memory in the guest include default memory size + hot plugged memory
+ // return memory size unit is MB
+ getMemorySize() uint32
getSandboxConsole(sandboxID string) (string, error)
disconnect()
capabilities() types.Capabilities
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
index 7575d326..8e073339 100644
--- a/virtcontainers/kata_agent.go
+++ b/virtcontainers/kata_agent.go
@@ -1806,9 +1806,9 @@ func (k *kataAgent) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionS
return err
}
-func (k *kataAgent) onlineCPUMem(cpus uint32, cpuOnly bool) error {
+func (k *kataAgent) onlineCPUMem(cpus uint32, cpuOnly bool, wait bool) error {
req := &grpc.OnlineCPUMemRequest{
- Wait: false,
+ Wait: wait,
NbCpus: cpus,
CpuOnly: cpuOnly,
}
diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go
index 62d31c93..2a2ddada 100644
--- a/virtcontainers/kata_agent_test.go
+++ b/virtcontainers/kata_agent_test.go
@@ -324,7 +324,7 @@ func TestKataAgentSendReq(t *testing.T) {
err = k.resumeContainer(sandbox, Container{})
assert.Nil(err)
- err = k.onlineCPUMem(1, true)
+ err = k.onlineCPUMem(1, true, false)
assert.Nil(err)
_, err = k.statsContainer(sandbox, Container{})
diff --git a/virtcontainers/mock_hypervisor.go b/virtcontainers/mock_hypervisor.go
index a5b67491..f1c6106d 100644
--- a/virtcontainers/mock_hypervisor.go
+++ b/virtcontainers/mock_hypervisor.go
@@ -128,3 +128,7 @@ func (m *mockHypervisor) check() error {
func (m *mockHypervisor) generateSocket(id string, useVsock bool) (interface{}, error) {
return types.Socket{HostPath: "/tmp/socket", Name: "socket"}, nil
}
+
+func (m *mockHypervisor) getMemorySize() uint32 {
+ return 0
+}
diff --git a/virtcontainers/noop_agent.go b/virtcontainers/noop_agent.go
index 8a7cd337..6e211bca 100644
--- a/virtcontainers/noop_agent.go
+++ b/virtcontainers/noop_agent.go
@@ -102,7 +102,7 @@ func (n *noopAgent) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionS
}
// onlineCPUMem is the Noop agent Container online CPU and Memory implementation. It does nothing.
-func (n *noopAgent) onlineCPUMem(cpus uint32, cpuOnly bool) error {
+func (n *noopAgent) onlineCPUMem(cpus uint32, cpuOnly bool, wait bool) error {
return nil
}
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
index 4789101d..7bae3278 100644
--- a/virtcontainers/qemu.go
+++ b/virtcontainers/qemu.go
@@ -2273,3 +2273,7 @@ func (q *qemu) check() error {
func (q *qemu) generateSocket(id string, useVsock bool) (interface{}, error) {
return generateVMSocket(id, useVsock, q.store.RunVMStoragePath())
}
+
+func (q *qemu) getMemorySize() uint32 {
+ return q.config.MemorySize + uint32(q.state.HotpluggedMemory)
+}
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
index e766d1f7..a318d677 100644
--- a/virtcontainers/sandbox.go
+++ b/virtcontainers/sandbox.go
@@ -1864,7 +1864,7 @@ func (s *Sandbox) updateResources() error {
// If the CPUs were increased, ask agent to online them
if oldCPUs < newCPUs {
vcpusAdded := newCPUs - oldCPUs
- if err := s.agent.onlineCPUMem(vcpusAdded, true); err != nil {
+ if err := s.agent.onlineCPUMem(vcpusAdded, true, false); err != nil {
return err
}
}
@@ -1872,6 +1872,20 @@ func (s *Sandbox) updateResources() error {
// Update Memory
s.Logger().WithField("memory-sandbox-size-byte", sandboxMemoryByte).Debugf("Request to hypervisor to update memory")
+ reqMemMB := uint32(sandboxMemoryByte >> utils.MibToBytesShift)
+ currentMemMB := s.hypervisor.getMemorySize()
+
+ // If request hotplug memory size larger than utils.MaxHotplugMemMBOnceTime,
+ // inorder to avoid hotplug memory oom problem, we need to hotplug large memory
+ // with two steps. First, hotplug utils.MaxHotplugMemMBOnceTime size memory into
+ // guest and wait all hotplug memory online. Then, hotplug the left unplugged memory
+ // into the guest
+ if currentMemMB < reqMemMB && (reqMemMB-currentMemMB) > utils.MaxHotplugMemMBOnceTime {
+ if err := s.beforeHotplugHugeMem(currentMemMB); err != nil {
+ return err
+ }
+ }
+
newMemory, updatedMemoryDevice, err := s.hypervisor.resizeMemory(uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe)
if err != nil {
return err
@@ -1884,7 +1898,7 @@ func (s *Sandbox) updateResources() error {
return err
}
}
- if err := s.agent.onlineCPUMem(0, false); err != nil {
+ if err := s.agent.onlineCPUMem(0, false, false); err != nil {
return err
}
return nil
@@ -1926,6 +1940,18 @@ func (s *Sandbox) calculateSandboxCPUs() uint32 {
return utils.CalculateVCpusFromMilliCpus(mCPU)
}
+func (s *Sandbox) beforeHotplugHugeMem(currentMemSizeInMB uint32) error {
+ wantedTotalMemSize := currentMemSizeInMB + utils.MaxHotplugMemMBOnceTime
+ newMemory, _, err := s.hypervisor.resizeMemory(wantedTotalMemSize, s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe)
+ if err != nil {
+ return err
+ }
+
+ s.Logger().Debugf("first part hotplug memory size: %d MB", newMemory)
+ // wait all first part hotplugged memory online in the guest
+ return s.agent.onlineCPUMem(0, false, true)
+}
+
// GetHypervisorType is used for getting Hypervisor name currently used.
// Sandbox implement DeviceReceiver interface from device/api/interface.go
func (s *Sandbox) GetHypervisorType() string {
diff --git a/virtcontainers/sandbox_test.go b/virtcontainers/sandbox_test.go
index 85c712e8..4b02b3f3 100644
--- a/virtcontainers/sandbox_test.go
+++ b/virtcontainers/sandbox_test.go
@@ -25,6 +25,7 @@ import (
"github.com/kata-containers/runtime/virtcontainers/persist/fs"
"github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
"github.com/kata-containers/runtime/virtcontainers/types"
+ "github.com/kata-containers/runtime/virtcontainers/utils"
specs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/stretchr/testify/assert"
"golang.org/x/sys/unix"
@@ -1522,6 +1523,17 @@ func TestSandboxUpdateResources(t *testing.T) {
}
err = s.updateResources()
assert.NoError(t, err)
+
+ // add a container with huge memory equal utils.MaxHotplugMemMBOnceTime
+ contConfig3 := newTestContainerConfigNoop("cont-00003")
+ contConfig3.Resources.Memory = &specs.LinuxMemory{
+ Limit: new(int64),
+ }
+ container3MemLimitInBytes := int64(utils.MaxHotplugMemMBOnceTime << utils.MibToBytesShift)
+ contConfig3.Resources.Memory.Limit = &container3MemLimitInBytes
+ s.config.Containers = append(s.config.Containers, contConfig3)
+ err = s.updateResources()
+ assert.NoError(t, err)
}
func TestSandboxExperimentalFeature(t *testing.T) {
diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go
index 2b555ebb..3ae95aef 100644
--- a/virtcontainers/utils/utils.go
+++ b/virtcontainers/utils/utils.go
@@ -25,6 +25,9 @@ const fileMode0755 = os.FileMode(0755)
// MibToBytesShift the number to shift needed to convert MiB to Bytes
const MibToBytesShift = 20
+// Max Hotplug Memory size at once time, unit is MB
+const MaxHotplugMemMBOnceTime = 32 * 1024
+
// MaxSocketPathLen is the effective maximum Unix domain socket length.
//
// See unix(7).
diff --git a/virtcontainers/vm.go b/virtcontainers/vm.go
index 8d27b1fe..2e5fef44 100644
--- a/virtcontainers/vm.go
+++ b/virtcontainers/vm.go
@@ -370,7 +370,7 @@ func (v *VM) AddMemory(numMB uint32) error {
// OnlineCPUMemory puts the hotplugged CPU and memory online.
func (v *VM) OnlineCPUMemory() error {
v.logger().Infof("online CPU %d and memory", v.cpuDelta)
- err := v.agent.onlineCPUMem(v.cpuDelta, false)
+ err := v.agent.onlineCPUMem(v.cpuDelta, false, false)
if err == nil {
v.cpuDelta = 0
}
--
2.14.3 (Apple Git-98)

355
runtime/patches/0017-kata-runtime-validate-sandbox-cpu-and-memory-size.patch Normal file
View File

@ -0,0 +1,355 @@
From e6051eb1a8c54b91c46f68eab9a63ad0e73a9221 Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Fri, 7 Aug 2020 20:01:15 +0800
Subject: [PATCH 17/50] kata-runtime: validate sandbox cpu and memory size
reason:
1. add more strict sandbox cpu and memory size check.
2. use GetPhysicalCPUNumber func to replace goruntime.NumCPU(),
because goruntime.NumCPU() return the CPU number of cpuset cgroup
that current process joined, however kata-runtime process will
change the cpuset cgroup when setupSanboxCgroup is called, which
may cause goruntime.NumCPU() return value changed.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
pkg/katautils/config.go | 39 ++++++++++++++++++++++++++++++++++---
pkg/katautils/config_test.go | 11 ++++++++---
virtcontainers/pkg/oci/utils.go | 11 +++++------
virtcontainers/qemu_arm64.go | 4 ++--
virtcontainers/qemu_arm64_test.go | 4 ++--
virtcontainers/sandbox.go | 7 ++++++-
virtcontainers/utils/utils.go | 41 +++++++++++++++++++++++++++++++++++++++
7 files changed, 100 insertions(+), 17 deletions(-)
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
index d1883c94..b9b08469 100644
--- a/pkg/katautils/config.go
+++ b/pkg/katautils/config.go
@@ -11,7 +11,6 @@ import (
"fmt"
"io/ioutil"
"path/filepath"
- goruntime "runtime"
"strings"
"github.com/BurntSushi/toml"
@@ -285,7 +284,7 @@ func (h hypervisor) GetEntropySource() string {
}
func (h hypervisor) defaultVCPUs() uint32 {
- numCPUs := goruntime.NumCPU()
+ numCPUs := utils.GetPhysicalCPUNumber()
if h.NumVCPUs < 0 || h.NumVCPUs > int32(numCPUs) {
return uint32(numCPUs)
@@ -297,8 +296,22 @@ func (h hypervisor) defaultVCPUs() uint32 {
return uint32(h.NumVCPUs)
}
+func (h hypervisor) checkVCPUs() error {
+ numCPUs := utils.GetPhysicalCPUNumber()
+
+ if h.NumVCPUs <= 0 {
+ return fmt.Errorf("invalid vcpus in configuration.toml! vcpus must larger than 0")
+ }
+
+ if h.NumVCPUs > int32(numCPUs) {
+ return fmt.Errorf("invalid vcpus in configuration.toml! vcpus must smaller than max CPUs: %d in machine", numCPUs)
+ }
+
+ return nil
+}
+
func (h hypervisor) defaultMaxVCPUs() uint32 {
- numcpus := uint32(goruntime.NumCPU())
+ numcpus := uint32(utils.GetPhysicalCPUNumber())
maxvcpus := vc.MaxQemuVCPUs()
reqVCPUs := h.DefaultMaxVCPUs
@@ -324,6 +337,18 @@ func (h hypervisor) defaultMemSz() uint32 {
return h.MemorySize
}
+func (h hypervisor) checkMemSz() error {
+ if h.MemorySize < utils.MinMemorySizeInMB {
+ return fmt.Errorf("invalid memory size! Memory size must larger than %d MB", utils.MinMemorySizeInMB)
+ }
+
+ if h.MemorySize > utils.MaxMemorySizeInMB {
+ return fmt.Errorf("invalid memory size, memory size must smaller than %d MB", utils.MaxMemorySizeInMB)
+ }
+
+ return nil
+}
+
func (h hypervisor) defaultMemSlots() uint32 {
slots := h.MemSlots
if slots == 0 {
@@ -627,6 +652,14 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
}
}
+ if err = h.checkVCPUs(); err != nil {
+ return vc.HypervisorConfig{}, err
+ }
+
+ if err = h.checkMemSz(); err != nil {
+ return vc.HypervisorConfig{}, err
+ }
+
return vc.HypervisorConfig{
HypervisorPath: hypervisor,
KernelPath: kernel,
diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go
index 31afcca6..37aa16d0 100644
--- a/pkg/katautils/config_test.go
+++ b/pkg/katautils/config_test.go
@@ -14,7 +14,6 @@ import (
"path"
"path/filepath"
"reflect"
- goruntime "runtime"
"strings"
"syscall"
"testing"
@@ -152,7 +151,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)),
HypervisorMachineType: machineType,
NumVCPUs: defaultVCPUCount,
- DefaultMaxVCPUs: uint32(goruntime.NumCPU()),
+ DefaultMaxVCPUs: uint32(utils.GetPhysicalCPUNumber()),
MemorySize: defaultMemSize,
DisableBlockDeviceUse: disableBlockDevice,
BlockDeviceDriver: defaultBlockDeviceDriver,
@@ -727,6 +726,8 @@ func TestMinimalRuntimeConfigWithVsock(t *testing.T) {
[hypervisor.qemu]
use_vsock = true
image = "` + imagePath + `"
+ default_vcpus = 1
+ default_memory = 1024
[proxy.kata]
path = "` + proxyPath + `"
@@ -786,6 +787,8 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
utils.VHostVSockDevicePath = orgVHostVSockDevicePath
}()
utils.VHostVSockDevicePath = "/dev/abc/xyz"
+ defaultVCPUs := int32(1)
+ defaultMemory := uint32(1024)
hypervisor := hypervisor{
Path: hypervisorPath,
@@ -797,6 +800,8 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
HotplugVFIOOnRootBus: hotplugVFIOOnRootBus,
PCIeRootPort: pcieRootPort,
UseVSock: true,
+ NumVCPUs: defaultVCPUs,
+ MemorySize: defaultMemory,
}
files := []string{hypervisorPath, kernelPath, imagePath}
@@ -996,7 +1001,7 @@ func TestNewShimConfig(t *testing.T) {
func TestHypervisorDefaults(t *testing.T) {
assert := assert.New(t)
- numCPUs := goruntime.NumCPU()
+ numCPUs := utils.GetPhysicalCPUNumber()
h := hypervisor{}
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
index 05181efd..0a6f08c7 100644
--- a/virtcontainers/pkg/oci/utils.go
+++ b/virtcontainers/pkg/oci/utils.go
@@ -10,22 +10,21 @@ import (
"errors"
"fmt"
"path/filepath"
- goruntime "runtime"
"strconv"
"strings"
"syscall"
criContainerdAnnotations "github.com/containerd/cri-containerd/pkg/annotations"
crioAnnotations "github.com/cri-o/cri-o/pkg/annotations"
- specs "github.com/opencontainers/runtime-spec/specs-go"
- "github.com/sirupsen/logrus"
-
vc "github.com/kata-containers/runtime/virtcontainers"
"github.com/kata-containers/runtime/virtcontainers/device/config"
exp "github.com/kata-containers/runtime/virtcontainers/experimental"
vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
dockershimAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations/dockershim"
"github.com/kata-containers/runtime/virtcontainers/types"
+ "github.com/kata-containers/runtime/virtcontainers/utils"
+ specs "github.com/opencontainers/runtime-spec/specs-go"
+ "github.com/sirupsen/logrus"
)
type annotationContainerType struct {
@@ -560,7 +559,7 @@ func addHypervisorCPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e
return fmt.Errorf("Error encountered parsing annotation default_vcpus: %v, please specify numeric value", err)
}
- numCPUs := goruntime.NumCPU()
+ numCPUs := utils.GetPhysicalCPUNumber()
if uint32(vcpus) > uint32(numCPUs) {
return fmt.Errorf("Number of cpus %d specified in annotation default_vcpus is greater than the number of CPUs %d on the system", vcpus, numCPUs)
@@ -575,7 +574,7 @@ func addHypervisorCPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e
return fmt.Errorf("Error encountered parsing annotation for default_maxvcpus: %v, please specify positive numeric value", err)
}
- numCPUs := goruntime.NumCPU()
+ numCPUs := utils.GetPhysicalCPUNumber()
max := uint32(maxVCPUs)
if max > uint32(numCPUs) {
diff --git a/virtcontainers/qemu_arm64.go b/virtcontainers/qemu_arm64.go
index 6d089cf0..40fb2a80 100644
--- a/virtcontainers/qemu_arm64.go
+++ b/virtcontainers/qemu_arm64.go
@@ -8,11 +8,11 @@ package virtcontainers
import (
"context"
"io/ioutil"
- "runtime"
"strings"
"time"
govmmQemu "github.com/intel/govmm/qemu"
+ "github.com/kata-containers/runtime/virtcontainers/utils"
"github.com/sirupsen/logrus"
)
@@ -121,7 +121,7 @@ func MaxQemuVCPUs() uint32 {
if hostGICVersion != 0 {
return gicList[hostGICVersion]
}
- return uint32(runtime.NumCPU())
+ return uint32(utils.GetPhysicalCPUNumber())
}
func newQemuArch(config HypervisorConfig) qemuArch {
diff --git a/virtcontainers/qemu_arm64_test.go b/virtcontainers/qemu_arm64_test.go
index 40351158..372fc4a9 100644
--- a/virtcontainers/qemu_arm64_test.go
+++ b/virtcontainers/qemu_arm64_test.go
@@ -10,10 +10,10 @@ import (
"io/ioutil"
"os"
"path/filepath"
- "runtime"
"testing"
govmmQemu "github.com/intel/govmm/qemu"
+ "github.com/kata-containers/runtime/virtcontainers/utils"
"github.com/stretchr/testify/assert"
)
@@ -63,7 +63,7 @@ func TestMaxQemuVCPUs(t *testing.T) {
}
data := []testData{
- {"", uint32(runtime.NumCPU())},
+ {"", uint32(utils.GetPhysicalCPUNumber())},
{" 1: 0 0 GICv2 25 Level vgic \n", uint32(8)},
{" 1: 0 0 GICv3 25 Level vgic \n", uint32(123)},
{" 1: 0 0 GICv4 25 Level vgic \n", uint32(123)},
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
index a318d677..a8522b96 100644
--- a/virtcontainers/sandbox.go
+++ b/virtcontainers/sandbox.go
@@ -1875,6 +1875,11 @@ func (s *Sandbox) updateResources() error {
reqMemMB := uint32(sandboxMemoryByte >> utils.MibToBytesShift)
currentMemMB := s.hypervisor.getMemorySize()
+ // check request sandbox memory size larger than utils.MaxMemorySizInMB or not
+ if reqMemMB > utils.MaxMemorySizeInMB {
+ return fmt.Errorf("updateResources failed! Sandbox memory should <= %d MiB", utils.MaxMemorySizeInMB)
+ }
+
// If request hotplug memory size larger than utils.MaxHotplugMemMBOnceTime,
// inorder to avoid hotplug memory oom problem, we need to hotplug large memory
// with two steps. First, hotplug utils.MaxHotplugMemMBOnceTime size memory into
@@ -1898,7 +1903,7 @@ func (s *Sandbox) updateResources() error {
return err
}
}
- if err := s.agent.onlineCPUMem(0, false, false); err != nil {
+ if err := s.agent.onlineCPUMem(0, false, true); err != nil {
return err
}
return nil
diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go
index 3ae95aef..5d38e594 100644
--- a/virtcontainers/utils/utils.go
+++ b/virtcontainers/utils/utils.go
@@ -6,6 +6,7 @@
package utils
import (
+ "bufio"
"crypto/rand"
"errors"
"fmt"
@@ -28,11 +29,26 @@ const MibToBytesShift = 20
// Max Hotplug Memory size at once time, unit is MB
const MaxHotplugMemMBOnceTime = 32 * 1024
+const (
+ // Min needed memory size to start a Kata VM
+ MinMemorySizeInMB = 300
+ MinMemorySizeInByte = MinMemorySizeInMB << MibToBytesShift
+
+ // Max support memory size in the Kata VM
+ MaxMemorySizeInMB = 512 * 1024
+ MaxMemorySizeInByte = MaxMemorySizeInMB << MibToBytesShift
+)
+
// MaxSocketPathLen is the effective maximum Unix domain socket length.
//
// See unix(7).
const MaxSocketPathLen = 107
+const (
+ procCPUInfoPath = "/proc/cpuinfo"
+ processorIdentifier = "processor"
+)
+
// VHostVSockDevicePath path to vhost-vsock device
var VHostVSockDevicePath = "/dev/vhost-vsock"
@@ -324,3 +340,28 @@ func IsProcessRunning(pid int, processName string, sandboxID string) bool {
return false
}
+
+// GetPhysicalCPUNumber return the number of the CPUs in the physical machine
+func GetPhysicalCPUNumber() int {
+ f, err := os.Open(procCPUInfoPath)
+ if err != nil {
+ return 0
+ }
+ defer f.Close()
+
+ cpuNum := 0
+ s := bufio.NewScanner(f)
+ for s.Scan() {
+ if err := s.Err(); err != nil {
+ return 0
+ }
+
+ fields := strings.Fields(s.Text())
+ if len(fields) > 0 {
+ if fields[0] == processorIdentifier {
+ cpuNum++
+ }
+ }
+ }
+ return cpuNum
+}
--
2.14.3 (Apple Git-98)

50
runtime/patches/0018-sandbox-Stop-and-clean-up-containers-that-fail-to-cr.patch Normal file
View File

@ -0,0 +1,50 @@
From c785f8f744050155102664d56de5bfb55e91915d Mon Sep 17 00:00:00 2001
From: Evan Foster <efoster@adobe.com>
Date: Mon, 13 Jul 2020 12:53:40 -0600
Subject: [PATCH 18/50] sandbox: Stop and clean up containers that fail to
create
A container that is created and added to a sandbox can still fail
the final creation steps. In this case, the container must be stopped
and have its resources cleaned up to prevent leaking sandbox mounts.
Fixes #2816
cherry-pick from: https://github.com/kata-containers/runtime/pull/2826
Signed-off-by: Evan Foster <efoster@adobe.com>
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
virtcontainers/sandbox.go | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
index a8522b96..3dbf640e 100644
--- a/virtcontainers/sandbox.go
+++ b/virtcontainers/sandbox.go
@@ -1,4 +1,5 @@
// Copyright (c) 2016 Intel Corporation
+// Copyright (c) 2020 Adobe Inc.
//
// SPDX-License-Identifier: Apache-2.0
//
@@ -1172,6 +1173,16 @@ func (s *Sandbox) CreateContainer(contConfig ContainerConfig) (VCContainer, erro
defer func() {
// Rollback if error happens.
if err != nil {
+ logger := s.Logger().WithFields(logrus.Fields{"container-id": c.id, "sandox-id": s.id, "rollback": true})
+
+ logger.Warning("Cleaning up partially created container")
+
+ if err2 := c.stop(true); err2 != nil {
+ logger.WithError(err2).Warning("Could not delete container")
+ }
+
+ logger.Debug("Removing stopped container from sandbox store")
+
s.removeContainer(c.id)
}
}()
--
2.14.3 (Apple Git-98)

33
runtime/patches/0019-virtcontainers-fix-delete-sandbox-failed-problem.patch Normal file
View File

@ -0,0 +1,33 @@
From d20cb25c8a145e1d3e64eefa69242d87b7a67f92 Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Sat, 8 Aug 2020 16:00:45 -0400
Subject: [PATCH 19/50] virtcontainers: fix delete sandbox failed problem
fixes: #2882
reason: If error happens after container create and before sandbox
updateResouce in the `CreateContainer()`, then delete sandbox
forcefully will return error because s.config.Containers config not
flushed into persist store.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
virtcontainers/sandbox.go | 2 ++
1 file changed, 2 insertions(+)
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
index 3dbf640e..7322ef58 100644
--- a/virtcontainers/sandbox.go
+++ b/virtcontainers/sandbox.go
@@ -1156,6 +1156,8 @@ func (s *Sandbox) CreateContainer(contConfig ContainerConfig) (VCContainer, erro
if len(s.config.Containers) > 0 {
// delete container config
s.config.Containers = s.config.Containers[:len(s.config.Containers)-1]
+ // need to flush change to persist storage
+ _ = s.storeSandbox()
}
}
}()
--
2.14.3 (Apple Git-98)

188
runtime/patches/0020-virtcontainers-add-enable_cpu_memory_hotplug-config-.patch Normal file
View File

@ -0,0 +1,188 @@
From 75141529674545a2f84b01c730f614901ad2265e Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Mon, 10 Aug 2020 10:08:47 +0800
Subject: [PATCH 20/50] virtcontainers: add enable_cpu_memory_hotplug config in
the configuration.toml
reason: add enable_cpu_memory_hotplug config to control whether enable hotplug
memory and cpu resource into the Kata VM. If we can calculate the whole sandbox
resource usage already, we just need to pass the value of resouces by annotation
without hotplugging.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
cli/config/configuration-qemu.toml.in | 4 ++++
pkg/katautils/config.go | 2 ++
virtcontainers/container.go | 6 ++++--
virtcontainers/hypervisor.go | 3 +++
virtcontainers/persist.go | 2 ++
virtcontainers/persist/api/config.go | 3 +++
virtcontainers/sandbox.go | 38 +++++++++++++++++++++--------------
7 files changed, 41 insertions(+), 17 deletions(-)
diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in
index 82461732..b44e84d8 100644
--- a/cli/config/configuration-qemu.toml.in
+++ b/cli/config/configuration-qemu.toml.in
@@ -100,6 +100,10 @@ default_memory = @DEFMEMSZ@
# Default false
#enable_reuse_cpu_memory = false
+# If enabled, the runtime will support cpu and memory hot plug
+# (default: disabled)
+#enable_cpu_memory_hotplug = true
+
# Disable block device from being used for a container's rootfs.
# In case of a storage driver like devicemapper where a container's
# root file system is backed by a block device, the block device is passed
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
index b9b08469..9a99b9d4 100644
--- a/pkg/katautils/config.go
+++ b/pkg/katautils/config.go
@@ -112,6 +112,7 @@ type hypervisor struct {
MemSlots uint32 `toml:"memory_slots"`
MemOffset uint32 `toml:"memory_offset"`
EnableCPUMemoryReuse bool `toml:"enable_reuse_cpu_memory"`
+ EnableCPUMemoryHotPlug bool `toml:"enable_cpu_memory_hotplug"`
DefaultBridges uint32 `toml:"default_bridges"`
Msize9p uint32 `toml:"msize_9p"`
PCIeRootPort uint32 `toml:"pcie_root_port"`
@@ -675,6 +676,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
MemSlots: h.defaultMemSlots(),
MemOffset: h.defaultMemOffset(),
EnableCPUMemoryReuse: h.EnableCPUMemoryReuse,
+ EnableCPUMemoryHotPlug: h.EnableCPUMemoryHotPlug,
VirtioMem: h.VirtioMem,
EntropySource: h.GetEntropySource(),
DefaultBridges: h.defaultBridges(),
diff --git a/virtcontainers/container.go b/virtcontainers/container.go
index 75f590eb..1b89f6ac 100644
--- a/virtcontainers/container.go
+++ b/virtcontainers/container.go
@@ -1273,8 +1273,10 @@ func (c *Container) update(resources specs.LinuxResources) error {
c.config.Resources.Memory.Limit = mem.Limit
}
- if err := c.sandbox.updateResources(); err != nil {
- return err
+ if c.sandbox.config.HypervisorConfig.EnableCPUMemoryHotPlug {
+ if err := c.sandbox.updateResources(); err != nil {
+ return err
+ }
}
if !c.sandbox.config.SandboxCgroupOnly {
diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go
index 9cd685ad..c0723daa 100644
--- a/virtcontainers/hypervisor.go
+++ b/virtcontainers/hypervisor.go
@@ -253,6 +253,9 @@ type HypervisorConfig struct {
// Enable hypervisor cpu and memory resource share with container
EnableCPUMemoryReuse bool
+ // Enable hotplug cpu and memory resource into container
+ EnableCPUMemoryHotPlug bool
+
// VirtioFSCacheSize is the DAX cache size in MiB
VirtioFSCacheSize uint32
diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go
index aef4d18d..6bd09a0b 100644
--- a/virtcontainers/persist.go
+++ b/virtcontainers/persist.go
@@ -215,6 +215,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
MemSlots: sconfig.HypervisorConfig.MemSlots,
MemOffset: sconfig.HypervisorConfig.MemOffset,
EnableCPUMemoryReuse: sconfig.HypervisorConfig.EnableCPUMemoryReuse,
+ EnableCPUMemoryHotPlug: sconfig.HypervisorConfig.EnableCPUMemoryHotPlug,
VirtioMem: sconfig.HypervisorConfig.VirtioMem,
VirtioFSCacheSize: sconfig.HypervisorConfig.VirtioFSCacheSize,
KernelPath: sconfig.HypervisorConfig.KernelPath,
@@ -505,6 +506,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
MemSlots: hconf.MemSlots,
MemOffset: hconf.MemOffset,
EnableCPUMemoryReuse: hconf.EnableCPUMemoryReuse,
+ EnableCPUMemoryHotPlug: hconf.EnableCPUMemoryHotPlug,
VirtioMem: hconf.VirtioMem,
VirtioFSCacheSize: hconf.VirtioFSCacheSize,
KernelPath: hconf.KernelPath,
diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go
index a3c6ec91..cfbee849 100644
--- a/virtcontainers/persist/api/config.go
+++ b/virtcontainers/persist/api/config.go
@@ -38,6 +38,9 @@ type HypervisorConfig struct {
// Enable hypervisor cpu and memory resource share with container
EnableCPUMemoryReuse bool
+ // Enable hotplug cpu and memory resource into container
+ EnableCPUMemoryHotPlug bool
+
// VirtioFSCacheSize is the DAX cache size in MiB
VirtioFSCacheSize uint32
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
index 7322ef58..8fcd92d4 100644
--- a/virtcontainers/sandbox.go
+++ b/virtcontainers/sandbox.go
@@ -1189,12 +1189,16 @@ func (s *Sandbox) CreateContainer(contConfig ContainerConfig) (VCContainer, erro
}
}()
- // Sandbox is reponsable to update VM resources needed by Containers
- // Update resources after having added containers to the sandbox, since
- // container status is requiered to know if more resources should be added.
- err = s.updateResources()
- if err != nil {
- return nil, err
+ // update sandbox resource only when enable_cpu_memory_hotplug config set true
+ // in the config.toml file
+ if s.config.HypervisorConfig.EnableCPUMemoryHotPlug {
+ // Sandbox is reponsable to update VM resources needed by Containers
+ // Update resources after having added containers to the sandbox, since
+ // container status is requiered to know if more resources should be added.
+ err = s.updateResources()
+ if err != nil {
+ return nil, err
+ }
}
if err = s.cgroupsUpdate(); err != nil {
@@ -1228,11 +1232,13 @@ func (s *Sandbox) StartContainer(containerID string) (VCContainer, error) {
s.Logger().Info("Container is started")
- // Update sandbox resources in case a stopped container
- // is started
- err = s.updateResources()
- if err != nil {
- return nil, err
+ if s.config.HypervisorConfig.EnableCPUMemoryHotPlug {
+ // Update sandbox resources in case a stopped container
+ // is started
+ err = s.updateResources()
+ if err != nil {
+ return nil, err
+ }
}
return c, nil
@@ -1503,10 +1509,12 @@ func (s *Sandbox) createContainers() error {
}
}
- // Update resources after having added containers to the sandbox, since
- // container status is requiered to know if more resources should be added.
- if err := s.updateResources(); err != nil {
- return err
+ if s.config.HypervisorConfig.EnableCPUMemoryHotPlug {
+ // Update resources after having added containers to the sandbox, since
+ // container status is requiered to know if more resources should be added.
+ if err := s.updateResources(); err != nil {
+ return err
+ }
}
if err := s.cgroupsUpdate(); err != nil {
--
2.14.3 (Apple Git-98)

195
runtime/patches/0021-kata-runtime-add-sandbox_cpu-and-sandbox_mem-annotat.patch Normal file
View File

@ -0,0 +1,195 @@
From 79cf2f5a52af51d8a62353a99e894808281769e2 Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Mon, 10 Aug 2020 11:25:02 +0800
Subject: [PATCH 21/50] kata-runtime: add sandbox_cpu and sandbox_mem
annotations
reason: add sandbox_cpu and sandbox_men annotations to set
Kata VM vCPU number and memory size.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
virtcontainers/pkg/annotations/annotations.go | 6 +++++
virtcontainers/pkg/oci/utils.go | 39 +++++++++++++++++++++++++--
virtcontainers/sandbox.go | 32 ++++++++++++++++++++++
virtcontainers/utils/utils.go | 14 ++++++++++
4 files changed, 89 insertions(+), 2 deletions(-)
diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go
index 10ce7833..903c7f03 100644
--- a/virtcontainers/pkg/annotations/annotations.go
+++ b/virtcontainers/pkg/annotations/annotations.go
@@ -251,6 +251,12 @@ const (
ContainerPipeSizeKernelParam = "agent." + ContainerPipeSizeOption
)
+// iSula self defined annotations
+const (
+ StaticCPUTypeKey = kataAnnotationsPrefix + "sandbox_cpu"
+ StaticMemTypeKey = kataAnnotationsPrefix + "sandbox_mem"
+)
+
const (
// SHA512 is the SHA-512 (64) hash algorithm
SHA512 string = "sha512"
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
index 0a6f08c7..36c730b7 100644
--- a/virtcontainers/pkg/oci/utils.go
+++ b/virtcontainers/pkg/oci/utils.go
@@ -16,6 +16,7 @@ import (
criContainerdAnnotations "github.com/containerd/cri-containerd/pkg/annotations"
crioAnnotations "github.com/cri-o/cri-o/pkg/annotations"
+ "github.com/docker/go-units"
vc "github.com/kata-containers/runtime/virtcontainers"
"github.com/kata-containers/runtime/virtcontainers/device/config"
exp "github.com/kata-containers/runtime/virtcontainers/experimental"
@@ -34,7 +35,10 @@ type annotationContainerType struct {
type annotationHandler func(value string) error
-var annotationHandlerList = map[string]annotationHandler{}
+var annotationHandlerList = map[string]annotationHandler{
+ vcAnnotations.StaticCPUTypeKey: validateSandboxCPU,
+ vcAnnotations.StaticMemTypeKey: validateSandboxMem,
+}
var (
// ErrNoLinux is an error for missing Linux sections in the OCI configuration file.
@@ -1036,7 +1040,10 @@ func validateOtherSandboxAnnotations(annotation, value string) error {
// addOtherSandboxAnnotation add self defined annotation for sandbox
func addOtherSandboxAnnotation(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error {
- otherSandboxAnnotationsKey := []string{}
+ otherSandboxAnnotationsKey := []string{
+ vcAnnotations.StaticCPUTypeKey,
+ vcAnnotations.StaticMemTypeKey,
+ }
for _, a := range otherSandboxAnnotationsKey {
value, ok := ocispec.Annotations[a]
@@ -1053,3 +1060,31 @@ func addOtherSandboxAnnotation(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e
return nil
}
+
+func validateSandboxCPU(value string) error {
+ // check min cpu value
+ cpus, err := utils.RoundVCPUNumber(value)
+ if err != nil {
+ return fmt.Errorf("valiate sandbox_cpu annotation fail: %v", err)
+ }
+
+ maxPhysicalCPUs := utils.GetPhysicalCPUNumber()
+ if cpus > maxPhysicalCPUs {
+ return fmt.Errorf("sandbox_cpu annotation value exceed the machine max CPU number: %d", cpus)
+ }
+
+ return nil
+}
+
+func validateSandboxMem(value string) error {
+ memSizeInBytes, err := units.RAMInBytes(value)
+ if err != nil {
+ return fmt.Errorf("parse sandbox_mem value: %d fail: %v", memSizeInBytes, err)
+ }
+
+ if memSizeInBytes < utils.MinMemorySizeInByte || memSizeInBytes > utils.MaxMemorySizeInByte {
+ return fmt.Errorf("invalid sandbox_mem value size in bytes: %v", memSizeInBytes)
+ }
+
+ return nil
+}
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
index 8fcd92d4..ba704249 100644
--- a/virtcontainers/sandbox.go
+++ b/virtcontainers/sandbox.go
@@ -20,6 +20,7 @@ import (
"github.com/containerd/cgroups"
"github.com/containernetworking/plugins/pkg/ns"
+ "github.com/docker/go-units"
"github.com/kata-containers/agent/protocols/grpc"
"github.com/kata-containers/runtime/virtcontainers/device/api"
"github.com/kata-containers/runtime/virtcontainers/device/config"
@@ -479,6 +480,10 @@ func createSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Fac
return nil, err
}
+ if err := updateStaticSandboxResources(&sandboxConfig); err != nil {
+ return nil, err
+ }
+
s, err := newSandbox(ctx, sandboxConfig, factory)
if err != nil {
return nil, err
@@ -2359,3 +2364,30 @@ func (s *Sandbox) setContainersState(state types.StateString) error {
return nil
}
+
+// updateStaticSandboxResources update sandbox's cpu and memory resource passed by
+// sandbox_cpu and sandbox_mem annotations
+func updateStaticSandboxResources(sandboxConfig *SandboxConfig) error {
+ // update cpu resource
+ if cpuNumVal, ok := sandboxConfig.Annotations[annotations.StaticCPUTypeKey]; ok {
+ cpuNum, err := utils.RoundVCPUNumber(cpuNumVal)
+ if err != nil {
+ return err
+ }
+
+ sandboxConfig.HypervisorConfig.NumVCPUs = (uint32)(cpuNum)
+ }
+
+ // update mem resource
+ if memVal, ok := sandboxConfig.Annotations[annotations.StaticMemTypeKey]; ok {
+ memSizeInBytes, err := units.RAMInBytes(memVal)
+ if err != nil {
+ return err
+ }
+
+ memSizeInMB := memSizeInBytes >> utils.MibToBytesShift
+ sandboxConfig.HypervisorConfig.MemorySize = (uint32)(memSizeInMB)
+ }
+
+ return nil
+}
diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go
index 5d38e594..9490faa1 100644
--- a/virtcontainers/utils/utils.go
+++ b/virtcontainers/utils/utils.go
@@ -11,6 +11,7 @@ import (
"errors"
"fmt"
"io/ioutil"
+ "math"
"os"
"os/exec"
"path/filepath"
@@ -30,6 +31,9 @@ const MibToBytesShift = 20
const MaxHotplugMemMBOnceTime = 32 * 1024
const (
+ // minCPUs is allowed minimum CPU
+ minCPUs = 0.25
+
// Min needed memory size to start a Kata VM
MinMemorySizeInMB = 300
MinMemorySizeInByte = MinMemorySizeInMB << MibToBytesShift
@@ -365,3 +369,13 @@ func GetPhysicalCPUNumber() int {
}
return cpuNum
}
+
+func RoundVCPUNumber(value string) (int, error) {
+ cpuNum, err := strconv.ParseFloat(value, 64)
+ if err != nil || cpuNum < minCPUs {
+ return 0, fmt.Errorf("invalid sandbox cpu number: %v", cpuNum)
+ }
+
+ cpus := int(math.Ceil(cpuNum))
+ return cpus, nil
+}
--
2.14.3 (Apple Git-98)

39
runtime/patches/0022-kata-runtime-skip-go-version-check-and-do-not-build-.patch Normal file
View File

@ -0,0 +1,39 @@
From e5e3232f7268110f7e3e3c4814eab31a6704b672 Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Mon, 10 Aug 2020 20:11:07 +0800
Subject: [PATCH 22/50] kata-runtime: skip go version check and do not build
containerd-shim-v2
reason: skip go version check and do not build containerd-shim-v2
because iSulad current not support shimV2
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
Makefile | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/Makefile b/Makefile
index 14a0ea47..d5e4bbe1 100644
--- a/Makefile
+++ b/Makefile
@@ -12,7 +12,7 @@ for file in /etc/os-release /usr/lib/os-release; do \
fi \
done)
-SKIP_GO_VERSION_CHECK=
+SKIP_GO_VERSION_CHECK=y
include golang.mk
#Get ARCH.
@@ -503,7 +503,7 @@ define SHOW_ARCH
$(shell printf "\\t%s%s\\\n" "$(1)" $(if $(filter $(ARCH),$(1))," (default)",""))
endef
-all: runtime containerd-shim-v2 netmon
+all: runtime netmon
# Targets that depend on .git-commit can use $(shell cat .git-commit) to get a
# git revision string. They will only be rebuilt if the revision string
--
2.14.3 (Apple Git-98)

42
runtime/patches/0023-kata-runtime-set-PCIBridgeMaxCapacity-limit-to-25.patch Normal file
View File

@ -0,0 +1,42 @@
From 226c3336dcc70bd17e3471ff98106a2f8dee9ac5 Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Tue, 11 Aug 2020 22:32:49 +0800
Subject: [PATCH 23/50] kata-runtime: set PCIBridgeMaxCapacity limit to 25
reason: set PCIBridgeMaxCapacity limit to 25.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
virtcontainers/qemu_arch_base_test.go | 2 +-
virtcontainers/types/bridges.go | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/virtcontainers/qemu_arch_base_test.go b/virtcontainers/qemu_arch_base_test.go
index 169e002e..8219f3c5 100644
--- a/virtcontainers/qemu_arch_base_test.go
+++ b/virtcontainers/qemu_arch_base_test.go
@@ -175,7 +175,7 @@ func TestQemuAddDeviceToBridge(t *testing.T) {
}
// fail to add device to bridge cause no more available bridge slot
- _, _, err := q.addDeviceToBridge("qemu-bridge-31", types.PCI)
+ _, _, err := q.addDeviceToBridge("qemu-bridge-26", types.PCI)
exceptErr := errors.New("no more bridge slots available")
assert.Equal(exceptErr.Error(), err.Error())
diff --git a/virtcontainers/types/bridges.go b/virtcontainers/types/bridges.go
index cb15a88f..c3538ce4 100644
--- a/virtcontainers/types/bridges.go
+++ b/virtcontainers/types/bridges.go
@@ -10,7 +10,7 @@ import "fmt"
// Type represents a type of bus and bridge.
type Type string
-const PCIBridgeMaxCapacity = 30
+const PCIBridgeMaxCapacity = 25
const (
// PCI represents a PCI bus and bridge
--
2.14.3 (Apple Git-98)

462
runtime/patches/0024-kata-runtime-support-hotplug-tap-interface-into-kata.patch Normal file
View File

@ -0,0 +1,462 @@
From e861f426c9e6702e820348ddc61b18013c853402 Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Thu, 13 Aug 2020 11:24:58 +0800
Subject: [PATCH 24/50] kata-runtime: support hotplug tap interface into kata
VM
reason: support hotplug exist tap interface or a new created tap
interface into kata VM.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
cli/network.go | 45 ++++++++++++++---
virtcontainers/kata_agent.go | 12 ++++-
virtcontainers/network.go | 100 +++++++++++++++++++++++---------------
virtcontainers/pkg/types/types.go | 16 +++---
virtcontainers/qemu.go | 11 +++--
virtcontainers/sandbox.go | 24 +++++++--
virtcontainers/tap_endpoint.go | 23 +++++++--
7 files changed, 161 insertions(+), 70 deletions(-)
diff --git a/cli/network.go b/cli/network.go
index 881a2358..7e7791f1 100644
--- a/cli/network.go
+++ b/cli/network.go
@@ -26,6 +26,8 @@ const (
routeType
)
+const defaultLinkType = "tap"
+
var kataNetworkCLICommand = cli.Command{
Name: "kata-network",
Usage: "manage interfaces and routes for container",
@@ -42,10 +44,22 @@ var kataNetworkCLICommand = cli.Command{
}
var addIfaceCommand = cli.Command{
- Name: "add-iface",
- Usage: "add an interface to a container",
- ArgsUsage: `add-iface <container-id> file or - for stdin`,
- Flags: []cli.Flag{},
+ Name: "add-iface",
+ Usage: "add an interface to a container",
+ ArgsUsage: `add-iface <container-id> file or - for stdin
+ file or stdin for example:
+ {
+ "device":"<device-name>",
+ "name":"<interface-name>",
+ "IPAddresses":[{"address":"<ip>","mask":"<mask>"}],
+ "mtu":<mtu>,
+ "hwAddr":"<mac>",
+ "linkType":"tap",
+ "vhostUserSocket":"<path>"
+ }
+ device,name,mtu,hwAddr are required, IPAddresses and vhostUserSocket are optional.
+ `,
+ Flags: []cli.Flag{},
Action: func(context *cli.Context) error {
ctx, err := cliContextToContext(context)
if err != nil {
@@ -57,10 +71,20 @@ var addIfaceCommand = cli.Command{
}
var delIfaceCommand = cli.Command{
- Name: "del-iface",
- Usage: "delete an interface from a container",
- ArgsUsage: `del-iface <container-id> file or - for stdin`,
- Flags: []cli.Flag{},
+ Name: "del-iface",
+ Usage: "delete an interface from a container",
+ ArgsUsage: `del-iface <container-id> file or - for stdin
+ file or stdin for example:
+ {
+ "device":"",
+ "name":"<interface-name>",
+ "IPAddresses":[],
+ "mtu":0,
+ "hwAddr":""
+ }
+ Only the "name" field is required.
+ `,
+ Flags: []cli.Flag{},
Action: func(context *cli.Context) error {
ctx, err := cliContextToContext(context)
if err != nil {
@@ -156,6 +180,11 @@ func networkModifyCommand(ctx context.Context, containerID, input string, opType
if err = json.NewDecoder(f).Decode(&inf); err != nil {
return err
}
+
+ if len(inf.LinkType) == 0 {
+ inf.LinkType = defaultLinkType
+ }
+
if add {
resultingInf, err = vci.AddInterface(ctx, sandboxID, inf)
if err != nil {
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
index 8e073339..dfdd263b 100644
--- a/virtcontainers/kata_agent.go
+++ b/virtcontainers/kata_agent.go
@@ -600,8 +600,16 @@ func (k *kataAgent) updateInterface(ifc *vcTypes.Interface) (*vcTypes.Interface,
"resulting-interface": fmt.Sprintf("%+v", resultingInterface),
}).WithError(err).Error("update interface request failed")
}
- if resultInterface, ok := resultingInterface.(*vcTypes.Interface); ok {
- return resultInterface, err
+ if resultInterface, ok := resultingInterface.(*aTypes.Interface); ok {
+ iface := &vcTypes.Interface{
+ Device: resultInterface.Device,
+ Name: resultInterface.Name,
+ IPAddresses: k.convertToIPAddresses(resultInterface.IPAddresses),
+ Mtu: resultInterface.Mtu,
+ HwAddr: resultInterface.HwAddr,
+ PciAddr: resultInterface.PciAddr,
+ }
+ return iface, err
}
return nil, err
}
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
index d70c5360..e909a822 100644
--- a/virtcontainers/network.go
+++ b/virtcontainers/network.go
@@ -117,6 +117,7 @@ type NetlinkIface struct {
// NetworkInfo gathers all information related to a network interface.
// It can be used to store the description of the underlying network.
type NetworkInfo struct {
+ Device string
Iface NetlinkIface
Addrs []netlink.Addr
Routes []netlink.Route
@@ -303,6 +304,20 @@ func createLink(netHandle *netlink.Handle, name string, expectedLink netlink.Lin
var newLink netlink.Link
var fds []*os.File
+ // check if tapname exists, if true, use existing one instead of create new one
+ if name != "" {
+ retLink, err := netlink.LinkByName(name)
+ // link exist, use it
+ if err == nil {
+ networkLogger().Debugf("exist tap device is found, instead of creating new one")
+ tuntapLink, ok := retLink.(*netlink.Tuntap)
+ if ok {
+ fds = tuntapLink.Fds
+ }
+ return retLink, nil, nil
+ }
+ }
+
switch expectedLink.Type() {
case (&netlink.Tuntap{}).Type():
flags := netlink.TUNTAP_VNET_HDR
@@ -1156,7 +1171,7 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li
// Check if interface is a physical interface. Do not create
// tap interface/bridge if it is.
- isPhysical, err := isPhysicalIface(netInfo.Iface.Name)
+ isPhysical, err := isPhysicalIface(netInfo.Device)
if err != nil {
return nil, err
}
@@ -1164,49 +1179,54 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li
if isPhysical {
networkLogger().WithField("interface", netInfo.Iface.Name).Info("Physical network interface found")
endpoint, err = createPhysicalEndpoint(netInfo)
- } else {
- var socketPath string
+ return endpoint, err
+ }
- // Check if this is a dummy interface which has a vhost-user socket associated with it
- socketPath, err = vhostUserSocketPath(netInfo)
- if err != nil {
- return nil, err
- }
+ // Check if this is a dummy interface which has a vhost-user socket associated with it
+ socketPath, err := vhostUserSocketPath(netInfo)
+ if err != nil {
+ return nil, err
+ }
- if socketPath != "" {
- networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found")
- endpoint, err = createVhostUserEndpoint(netInfo, socketPath)
- } else if netInfo.Iface.Type == "macvlan" {
- networkLogger().Infof("macvlan interface found")
- endpoint, err = createBridgedMacvlanNetworkEndpoint(idx, netInfo.Iface.Name, model)
- } else if netInfo.Iface.Type == "macvtap" {
- networkLogger().Infof("macvtap interface found")
- endpoint, err = createMacvtapNetworkEndpoint(netInfo)
- } else if netInfo.Iface.Type == "tap" {
- networkLogger().Info("tap interface found")
- endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name)
- } else if netInfo.Iface.Type == "tuntap" {
- if link != nil {
- switch link.(*netlink.Tuntap).Mode {
- case 0:
- // mount /sys/class/net to get links
- return nil, fmt.Errorf("Network device mode not determined correctly. Mount sysfs in caller")
- case 1:
- return nil, fmt.Errorf("tun networking device not yet supported")
- case 2:
- networkLogger().Info("tuntap tap interface found")
- endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, model)
- default:
- return nil, fmt.Errorf("tuntap network %v mode unsupported", link.(*netlink.Tuntap).Mode)
- }
+ if socketPath != "" {
+ networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found")
+ endpoint, err = createVhostUserEndpoint(netInfo, socketPath)
+ return endpoint, err
+ }
+
+ // We should create tap interface/bridge of other interface type.
+ networkLogger().Infof("%s interface found", netInfo.Iface.Type)
+ switch netInfo.Iface.Type {
+ case "macvlan":
+ networkLogger().Infof("macvlan interface found")
+ endpoint, err = createBridgedMacvlanNetworkEndpoint(idx, netInfo.Iface.Name, model)
+ case "macvtap":
+ networkLogger().Infof("macvtap interface found")
+ endpoint, err = createMacvtapNetworkEndpoint(netInfo)
+ case "tap":
+ networkLogger().Info("tap interface found")
+ endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Device)
+ case "tuntap":
+ if link != nil {
+ switch link.(*netlink.Tuntap).Mode {
+ case 0:
+ // mount /sys/class/net to get links
+ return nil, fmt.Errorf("Network device mode not determined correctly. Mount sysfs in caller")
+ case 1:
+ return nil, fmt.Errorf("tun networking device not yet supported")
+ case 2:
+ networkLogger().Info("tuntap tap interface found")
+ endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, model)
+ default:
+ return nil, fmt.Errorf("tuntap network %v mode unsupported", link.(*netlink.Tuntap).Mode)
}
- } else if netInfo.Iface.Type == "veth" {
- endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model)
- } else if netInfo.Iface.Type == "ipvlan" {
- endpoint, err = createIPVlanNetworkEndpoint(idx, netInfo.Iface.Name)
- } else {
- return nil, fmt.Errorf("Unsupported network interface: %s", netInfo.Iface.Type)
}
+ case "veth":
+ endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model)
+ case "ipvlan":
+ endpoint, err = createIPVlanNetworkEndpoint(idx, netInfo.Iface.Name)
+ default:
+ err = fmt.Errorf("Unsupported network interface, %s", netInfo.Iface.Type)
}
return endpoint, err
diff --git a/virtcontainers/pkg/types/types.go b/virtcontainers/pkg/types/types.go
index 0d4a9cfa..fcc63d84 100644
--- a/virtcontainers/pkg/types/types.go
+++ b/virtcontainers/pkg/types/types.go
@@ -14,21 +14,21 @@ type IPAddress struct {
// Interface describes a network interface.
type Interface struct {
- Device string
- Name string
- IPAddresses []*IPAddress
- Mtu uint64
- RawFlags uint32
- HwAddr string
+ Device string `json:"device,omitempty"`
+ Name string `json:"name,omitempty"`
+ IPAddresses []*IPAddress `json:"IPAddresses,omitempty"`
+ Mtu uint64 `json:"mtu,omitempty"`
+ RawFlags uint32 `json:"rawFlags,omitempty"`
+ HwAddr string `json:"hwAddr,omitempty"`
// pciAddr is the PCI address in the format "bridgeAddr/deviceAddr".
// Here, bridgeAddr is the address at which the bridge is attached on the root bus,
// while deviceAddr is the address at which the network device is attached on the bridge.
- PciAddr string
+ PciAddr string `json:"pciAddr,omitempty"`
// LinkType defines the type of interface described by this structure.
// The expected values are the one that are defined by the netlink
// library, regarding each type of link. Here is a non exhaustive
// list: "veth", "macvtap", "vlan", "macvlan", "tap", ...
- LinkType string
+ LinkType string `json:"linkType,omitempty"`
}
// Route describes a network route.
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
index 7bae3278..bb83b1bb 100644
--- a/virtcontainers/qemu.go
+++ b/virtcontainers/qemu.go
@@ -1361,7 +1361,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro
return nil
}
-func (q *qemu) hotAddNetDevice(name, hardAddr string, VMFds, VhostFds []*os.File) error {
+func (q *qemu) hotAddNetDevice(deviceName, name, hardAddr string, VMFds, VhostFds []*os.File) error {
var (
VMFdNames []string
VhostFdNames []string
@@ -1381,7 +1381,12 @@ func (q *qemu) hotAddNetDevice(name, hardAddr string, VMFds, VhostFds []*os.File
VhostFd.Close()
VhostFdNames = append(VhostFdNames, fdName)
}
- return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", name, VMFdNames, VhostFdNames)
+
+ if len(VMFdNames) != 0 || len(VhostFdNames) != 0 {
+ return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", name, VMFdNames, VhostFdNames)
+ }
+
+ return q.qmpMonitorCh.qmp.ExecuteNetdevAdd(q.qmpMonitorCh.ctx, "tap", name, deviceName, "no", "no", 0)
}
func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
@@ -1404,7 +1409,7 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
devID := "virtio-" + tap.ID
if op == addDevice {
- if err = q.hotAddNetDevice(tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds); err != nil {
+ if err = q.hotAddNetDevice(tap.TAPIface.Name, tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds); err != nil {
return err
}
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
index ba704249..c8981a41 100644
--- a/virtcontainers/sandbox.go
+++ b/virtcontainers/sandbox.go
@@ -937,6 +937,7 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) {
}
return NetworkInfo{
+ Device: inf.Device,
Iface: NetlinkIface{
LinkAttrs: netlink.LinkAttrs{
Name: inf.Name,
@@ -950,7 +951,7 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) {
}
// AddInterface adds new nic to the sandbox.
-func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (*vcTypes.Interface, error) {
+func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (grpcIf *vcTypes.Interface, err error) {
netInfo, err := s.generateNetInfo(inf)
if err != nil {
return nil, err
@@ -962,28 +963,41 @@ func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (*vcTypes.Interface, erro
}
endpoint.SetProperties(netInfo)
- if err := doNetNS(s.networkNS.NetNsPath, func(_ ns.NetNS) error {
+ if err = doNetNS(s.networkNS.NetNsPath, func(_ ns.NetNS) error {
s.Logger().WithField("endpoint-type", endpoint.Type()).Info("Hot attaching endpoint")
return endpoint.HotAttach(s.hypervisor)
}); err != nil {
return nil, err
}
+ defer func() {
+ if err != nil {
+ if errDetach := endpoint.HotDetach(s.hypervisor, s.networkNS.NetNsCreated, s.networkNS.NetNsPath); errDetach != nil {
+ s.Logger().WithField("endpoint-type", endpoint.Type()).Errorf("rollback hot attach endpoint failed")
+ }
+ }
+ }()
+
// Update the sandbox storage
s.networkNS.Endpoints = append(s.networkNS.Endpoints, endpoint)
- if err := s.Save(); err != nil {
+ if err = s.Save(); err != nil {
return nil, err
}
// Add network for vm
inf.PciAddr = endpoint.PciAddr()
- return s.agent.updateInterface(inf)
+ grpcIf, err = s.agent.updateInterface(inf)
+ if err != nil {
+ return nil, err
+ }
+
+ return
}
// RemoveInterface removes a nic of the sandbox.
func (s *Sandbox) RemoveInterface(inf *vcTypes.Interface) (*vcTypes.Interface, error) {
for i, endpoint := range s.networkNS.Endpoints {
- if endpoint.HardwareAddr() == inf.HwAddr {
+ if endpoint.HardwareAddr() == inf.HwAddr || endpoint.Name() == inf.Name {
s.Logger().WithField("endpoint-type", endpoint.Type()).Info("Hot detaching endpoint")
if err := endpoint.HotDetach(s.hypervisor, s.networkNS.NetNsCreated, s.networkNS.NetNsPath); err != nil {
return inf, err
diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go
index cb441b87..7d33d5a2 100644
--- a/virtcontainers/tap_endpoint.go
+++ b/virtcontainers/tap_endpoint.go
@@ -7,6 +7,7 @@ package virtcontainers
import (
"fmt"
+ "os"
"github.com/containernetworking/plugins/pkg/ns"
"github.com/vishvananda/netlink"
@@ -111,7 +112,7 @@ func (endpoint *TapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPat
return nil
}
-func createTapNetworkEndpoint(idx int, ifName string) (*TapEndpoint, error) {
+func createTapNetworkEndpoint(idx int, ifName string, tapIfName string) (*TapEndpoint, error) {
if idx < 0 {
return &TapEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx)
}
@@ -131,6 +132,10 @@ func createTapNetworkEndpoint(idx int, ifName string) (*TapEndpoint, error) {
endpoint.TapInterface.Name = ifName
}
+ if tapIfName != "" {
+ endpoint.TapInterface.TAPIface.Name = tapIfName
+ }
+
return endpoint, nil
}
@@ -145,9 +150,19 @@ func tapNetwork(endpoint *TapEndpoint, numCPUs uint32, disableVhostNet bool) err
if err != nil {
return fmt.Errorf("Could not create TAP interface: %s", err)
}
+
+ defer func() {
+ if err != nil {
+ if errDel := netHandle.LinkDel(tapLink); errDel != nil {
+ networkLogger().WithError(errDel).Error("tapNetwork fail to rollback del link")
+ }
+ }
+ }()
+
endpoint.TapInterface.VMFds = fds
if !disableVhostNet {
- vhostFds, err := createVhostFds(int(numCPUs))
+ var vhostFds []*os.File
+ vhostFds, err = createVhostFds(int(numCPUs))
if err != nil {
return fmt.Errorf("Could not setup vhost fds %s : %s", endpoint.TapInterface.Name, err)
}
@@ -161,10 +176,10 @@ func tapNetwork(endpoint *TapEndpoint, numCPUs uint32, disableVhostNet bool) err
// bridge created by the network plugin on the host actually expects
// to see traffic from this MAC address and not another one.
endpoint.TapInterface.TAPIface.HardAddr = linkAttrs.HardwareAddr.String()
- if err := netHandle.LinkSetMTU(tapLink, linkAttrs.MTU); err != nil {
+ if err = netHandle.LinkSetMTU(tapLink, linkAttrs.MTU); err != nil {
return fmt.Errorf("Could not set TAP MTU %d: %s", linkAttrs.MTU, err)
}
- if err := netHandle.LinkSetUp(tapLink); err != nil {
+ if err = netHandle.LinkSetUp(tapLink); err != nil {
return fmt.Errorf("Could not enable TAP %s: %s", endpoint.TapInterface.Name, err)
}
return nil
--
2.14.3 (Apple Git-98)

260
runtime/patches/0025-network-keep-list-ifaces-result-compatible-with-cni.patch Normal file
View File

@ -0,0 +1,260 @@
From be8153f21c0b81d2b194075ecd654501bc708577 Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Thu, 13 Aug 2020 18:54:49 +0800
Subject: [PATCH 25/50] network: keep list-ifaces result compatible with cni
reason: community list-ifaces command will return the all
interfaces info in the Kata VM, however we may just want
to get the interfaces that we hotplug, so just return the
hotplugged interfaces and convert the interface info to
be compatible with cni.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
cli/network.go | 29 ++++++++++++++++++++++-
virtcontainers/api.go | 4 +++-
virtcontainers/endpoint.go | 44 +++++++++++++++++++++++++++++++++++
virtcontainers/network.go | 27 +++++++++++++++++++++
virtcontainers/persist/api/network.go | 24 +++++++++++++++++++
virtcontainers/tap_endpoint.go | 9 +++++++
6 files changed, 135 insertions(+), 2 deletions(-)
diff --git a/cli/network.go b/cli/network.go
index 7e7791f1..66955725 100644
--- a/cli/network.go
+++ b/cli/network.go
@@ -28,6 +28,13 @@ const (
const defaultLinkType = "tap"
+type compatInterface struct {
+ Name string `json:"name,omitempty"`
+ Mac string `json:"mac,omitempty"`
+ IP []string `json:"ip,omitempty"`
+ Mtu int `json:"mtu,omitempty"`
+}
+
var kataNetworkCLICommand = cli.Command{
Name: "kata-network",
Usage: "manage interfaces and routes for container",
@@ -244,7 +251,8 @@ func networkListCommand(ctx context.Context, containerID string, opType networkT
kataLog.WithField("existing-interfaces", fmt.Sprintf("%+v", interfaces)).
WithError(err).Error("list interfaces failed")
}
- json.NewEncoder(file).Encode(interfaces)
+ compatInfs := convertCompatInterfaces(interfaces)
+ json.NewEncoder(file).Encode(compatInfs)
case routeType:
var routes []*vcTypes.Route
routes, err = vci.ListRoutes(ctx, sandboxID)
@@ -256,3 +264,22 @@ func networkListCommand(ctx context.Context, containerID string, opType networkT
}
return err
}
+
+func convertCompatInterfaces(interfaces []*vcTypes.Interface) []compatInterface {
+ var infs []compatInterface
+ for _, i := range interfaces {
+ var addrs []string
+ for _, a := range i.IPAddresses {
+ addrs = append(addrs, fmt.Sprintf("%s/%s", a.Address, a.Mask))
+ }
+
+ infs = append(infs, compatInterface{
+ Name: i.Name,
+ Mac: i.HwAddr,
+ IP: addrs,
+ Mtu: int(i.Mtu),
+ })
+ }
+
+ return infs
+}
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
index 5e8c9c9e..eb5b4995 100644
--- a/virtcontainers/api.go
+++ b/virtcontainers/api.go
@@ -949,7 +949,9 @@ func ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTypes.Interface
}
defer s.releaseStatelessSandbox()
- return s.ListInterfaces()
+ // get interfaces info from persist.json file
+ // instead of by s.ListInterfaces()
+ return convertToCompatInterfaces(&s.networkNS.Endpoints), nil
}
// UpdateRoutes is the virtcontainers update routes entry point.
diff --git a/virtcontainers/endpoint.go b/virtcontainers/endpoint.go
index 01b5e77f..7efcf49c 100644
--- a/virtcontainers/endpoint.go
+++ b/virtcontainers/endpoint.go
@@ -132,6 +132,28 @@ func saveTapIf(tapif *TapInterface) *persistapi.TapInterface {
}
}
+func saveTapEndpointProperties(networkInfo *NetworkInfo) *persistapi.NetworkProperties {
+ if networkInfo == nil {
+ return nil
+ }
+
+ return &persistapi.NetworkProperties{
+ Device: networkInfo.Device,
+ Iface: persistapi.NetlinkIface{
+ LinkAttrs: networkInfo.Iface.LinkAttrs,
+ Type: networkInfo.Iface.Type,
+ },
+ Addrs: networkInfo.Addrs,
+ Routes: networkInfo.Routes,
+ DNS: persistapi.DNSInfo{
+ Servers: networkInfo.DNS.Servers,
+ Domain: networkInfo.DNS.Domain,
+ Searches: networkInfo.DNS.Searches,
+ Options: networkInfo.DNS.Options,
+ },
+ }
+}
+
func loadTapIf(tapif *persistapi.TapInterface) *TapInterface {
if tapif == nil {
return nil
@@ -148,6 +170,28 @@ func loadTapIf(tapif *persistapi.TapInterface) *TapInterface {
}
}
+func loadTapEndpointProperties(endpointProperties *persistapi.NetworkProperties) *NetworkInfo {
+ if endpointProperties == nil {
+ return nil
+ }
+
+ return &NetworkInfo{
+ Device: endpointProperties.Device,
+ Iface: NetlinkIface{
+ LinkAttrs: endpointProperties.Iface.LinkAttrs,
+ Type: endpointProperties.Iface.Type,
+ },
+ Addrs: endpointProperties.Addrs,
+ Routes: endpointProperties.Routes,
+ DNS: DNSInfo{
+ Servers: endpointProperties.DNS.Servers,
+ Domain: endpointProperties.DNS.Domain,
+ Searches: endpointProperties.DNS.Searches,
+ Options: endpointProperties.DNS.Options,
+ },
+ }
+}
+
func saveNetIfPair(pair *NetworkInterfacePair) *persistapi.NetworkInterfacePair {
if pair == nil {
return nil
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
index e909a822..bf7f9336 100644
--- a/virtcontainers/network.go
+++ b/virtcontainers/network.go
@@ -1340,3 +1340,30 @@ func (n *Network) Remove(ctx context.Context, ns *NetworkNamespace, hypervisor h
return nil
}
+
+// convertCompatInterfaces convert Endpoint info to vcTypes.Interface
+func convertToCompatInterfaces(es *[]Endpoint) []*vcTypes.Interface {
+ var infs []*vcTypes.Interface
+ for _, e := range *es {
+ var addrs []*vcTypes.IPAddress
+ for _, a := range e.Properties().Addrs {
+ m, _ := a.Mask.Size()
+ addr := &vcTypes.IPAddress{
+ Address: fmt.Sprintf("%s", a.IP),
+ Mask: fmt.Sprintf("%d", m),
+ }
+ addrs = append(addrs, addr)
+ }
+ inf := &vcTypes.Interface{
+ LinkType: string(e.Type()),
+ Name: e.Name(),
+ Mtu: uint64(e.Properties().Iface.MTU),
+ HwAddr: e.HardwareAddr(),
+ IPAddresses: addrs,
+ }
+
+ infs = append(infs, inf)
+ }
+
+ return infs
+}
diff --git a/virtcontainers/persist/api/network.go b/virtcontainers/persist/api/network.go
index 69610c67..53c6de44 100644
--- a/virtcontainers/persist/api/network.go
+++ b/virtcontainers/persist/api/network.go
@@ -11,6 +11,27 @@ import (
)
// ============= sandbox level resources =============
+// DNSInfo describes the DNS setup related to a network interface.
+type DNSInfo struct {
+ Servers []string
+ Domain string
+ Searches []string
+ Options []string
+}
+
+// NetlinkIface describes fully a network interface.
+type NetlinkIface struct {
+ netlink.LinkAttrs
+ Type string
+}
+
+type NetworkProperties struct {
+ Device string
+ Iface NetlinkIface
+ Addrs []netlink.Addr
+ Routes []netlink.Route
+ DNS DNSInfo
+}
type NetworkInterface struct {
Name string
@@ -91,6 +112,9 @@ type NetworkEndpoint struct {
Tap *TapEndpoint `json:",omitempty"`
IPVlan *IPVlanEndpoint `json:",omitempty"`
Tuntap *TuntapEndpoint `json:",omitempty"`
+
+ // store the endpoint properties info
+ EndPointProperties *NetworkProperties `json:",omitempty"`
}
// NetworkInfo contains network information of sandbox
diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go
index 7d33d5a2..c897670e 100644
--- a/virtcontainers/tap_endpoint.go
+++ b/virtcontainers/tap_endpoint.go
@@ -206,12 +206,15 @@ func unTapNetwork(name string) error {
func (endpoint *TapEndpoint) save() persistapi.NetworkEndpoint {
tapif := saveTapIf(&endpoint.TapInterface)
+ // save tap endpoint network properties into persist storage
+ properties := saveTapEndpointProperties(&endpoint.EndpointProperties)
return persistapi.NetworkEndpoint{
Type: string(endpoint.Type()),
Tap: &persistapi.TapEndpoint{
TapInterface: *tapif,
},
+ EndPointProperties: properties,
}
}
func (endpoint *TapEndpoint) load(s persistapi.NetworkEndpoint) {
@@ -221,4 +224,10 @@ func (endpoint *TapEndpoint) load(s persistapi.NetworkEndpoint) {
tapif := loadTapIf(&s.Tap.TapInterface)
endpoint.TapInterface = *tapif
}
+
+ if s.EndPointProperties != nil {
+ // restore tap endpoint network properties from persist storage
+ properties := loadTapEndpointProperties(s.EndPointProperties)
+ endpoint.EndpointProperties = *properties
+ }
}
--
2.14.3 (Apple Git-98)

285
runtime/patches/0026-network-add-enable_compat_old_cni-config.patch Normal file
View File

@ -0,0 +1,285 @@
From eeca1e47e9a6422d89d08275864f2c1b15e54941 Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Fri, 14 Aug 2020 19:14:35 +0800
Subject: [PATCH 26/50] network: add enable_compat_old_cni config
reason: old version kata-network list-ifaces output result different
from the community version, inorder to compatible with the old version
list-ifaces command output format, add enable_compat_old_cni to control
the list-ifaces command output format.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
cli/config/configuration-qemu.toml.in | 4 ++++
cli/network.go | 17 +++++++++++++++--
cli/network_test.go | 6 ++++++
pkg/katautils/config.go | 2 ++
virtcontainers/api.go | 10 +++++++---
virtcontainers/interfaces.go | 1 +
virtcontainers/network.go | 11 ++++++-----
virtcontainers/persist.go | 18 ++++++++++--------
virtcontainers/persist/api/config.go | 9 +++++----
virtcontainers/pkg/oci/utils.go | 4 ++++
virtcontainers/pkg/vcmock/sandbox.go | 5 +++++
virtcontainers/sandbox.go | 5 +++++
12 files changed, 70 insertions(+), 22 deletions(-)
diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in
index b44e84d8..46f8b632 100644
--- a/cli/config/configuration-qemu.toml.in
+++ b/cli/config/configuration-qemu.toml.in
@@ -453,6 +453,10 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
# (default: false)
#disable_new_netns = true
+# If enabled, the kata-network will return the old interface format info to be compatible with
+# old version CNI plugin
+enable_compat_old_cni = true
+
# if enabled, the runtime will add all the kata processes inside one dedicated cgroup.
# The container cgroups in the host are not created, just one single cgroup per sandbox.
# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.
diff --git a/cli/network.go b/cli/network.go
index 66955725..a1a24425 100644
--- a/cli/network.go
+++ b/cli/network.go
@@ -251,8 +251,21 @@ func networkListCommand(ctx context.Context, containerID string, opType networkT
kataLog.WithField("existing-interfaces", fmt.Sprintf("%+v", interfaces)).
WithError(err).Error("list interfaces failed")
}
- compatInfs := convertCompatInterfaces(interfaces)
- json.NewEncoder(file).Encode(compatInfs)
+
+ sandbox, err := vci.FetchSandbox(ctx, sandboxID)
+ if err != nil {
+ kataLog.WithField("existing-interfaces", fmt.Sprintf("%+v", interfaces)).
+ WithError(err).Error("fetch sandbox failed")
+ }
+
+ // If sandbox network config need to be compatible with old CNI,
+ // convert the interface format to old version format.
+ if sandbox.IsCompatOldCNI() {
+ compatInfs := convertCompatInterfaces(interfaces)
+ json.NewEncoder(file).Encode(compatInfs)
+ } else {
+ json.NewEncoder(file).Encode(interfaces)
+ }
case routeType:
var routes []*vcTypes.Route
routes, err = vci.ListRoutes(ctx, sandboxID)
diff --git a/cli/network_test.go b/cli/network_test.go
index 4e3d943d..95fd2749 100644
--- a/cli/network_test.go
+++ b/cli/network_test.go
@@ -17,6 +17,7 @@ import (
vc "github.com/kata-containers/runtime/virtcontainers"
vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
+ "github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
"github.com/kata-containers/runtime/virtcontainers/types"
)
@@ -59,6 +60,10 @@ func TestNetworkCliFunction(t *testing.T) {
return newSingleContainerStatus(testContainerID, state, map[string]string{}, &specs.Spec{}), nil
}
+ testingImpl.FetchSandboxFunc = func(ctx context.Context, id string) (vc.VCSandbox, error) {
+ return &vcmock.Sandbox{}, nil
+ }
+
defer func() {
testingImpl.AddInterfaceFunc = nil
testingImpl.RemoveInterfaceFunc = nil
@@ -66,6 +71,7 @@ func TestNetworkCliFunction(t *testing.T) {
testingImpl.UpdateRoutesFunc = nil
testingImpl.ListRoutesFunc = nil
testingImpl.StatusContainerFunc = nil
+ testingImpl.FetchSandboxFunc = nil
}()
set := flag.NewFlagSet("", 0)
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
index 9a99b9d4..94c916a0 100644
--- a/pkg/katautils/config.go
+++ b/pkg/katautils/config.go
@@ -141,6 +141,7 @@ type runtime struct {
Debug bool `toml:"enable_debug"`
Tracing bool `toml:"enable_tracing"`
DisableNewNetNs bool `toml:"disable_new_netns"`
+ EnableCompatOldCNI bool `toml:"enable_compat_old_cni"`
DisableGuestSeccomp bool `toml:"disable_guest_seccomp"`
SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"`
Experimental []string `toml:"experimental"`
@@ -1235,6 +1236,7 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool, debugFlag
config.SandboxCgroupOnly = tomlConf.Runtime.SandboxCgroupOnly
config.DisableNewNetNs = tomlConf.Runtime.DisableNewNetNs
+ config.EnableCompatOldCNI = tomlConf.Runtime.EnableCompatOldCNI
for _, f := range tomlConf.Runtime.Experimental {
feature := exp.Get(f)
if feature == nil {
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
index eb5b4995..fb044fe1 100644
--- a/virtcontainers/api.go
+++ b/virtcontainers/api.go
@@ -949,9 +949,13 @@ func ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTypes.Interface
}
defer s.releaseStatelessSandbox()
- // get interfaces info from persist.json file
- // instead of by s.ListInterfaces()
- return convertToCompatInterfaces(&s.networkNS.Endpoints), nil
+ // If enable_compat_old_cni is enabled, get interfaces info from
+ // persist.json file instead of by s.ListInterfaces()
+ if s.config.NetworkConfig.EnableCompatOldCNI {
+ return convertToCompatInterfaces(&s.networkNS.Endpoints), nil
+ }
+
+ return s.ListInterfaces()
}
// UpdateRoutes is the virtcontainers update routes entry point.
diff --git a/virtcontainers/interfaces.go b/virtcontainers/interfaces.go
index fa6b584e..499b386e 100644
--- a/virtcontainers/interfaces.go
+++ b/virtcontainers/interfaces.go
@@ -98,6 +98,7 @@ type VCSandbox interface {
ListInterfaces() ([]*vcTypes.Interface, error)
UpdateRoutes(routes []*vcTypes.Route) ([]*vcTypes.Route, error)
ListRoutes() ([]*vcTypes.Route, error)
+ IsCompatOldCNI() bool
}
// VCContainer is the Container interface
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
index bf7f9336..db235cf6 100644
--- a/virtcontainers/network.go
+++ b/virtcontainers/network.go
@@ -155,11 +155,12 @@ type NetworkInterfacePair struct {
// NetworkConfig is the network configuration related to a network.
type NetworkConfig struct {
- NetNSPath string
- NetNsCreated bool
- DisableNewNetNs bool
- NetmonConfig NetmonConfig
- InterworkingModel NetInterworkingModel
+ NetNSPath string
+ NetNsCreated bool
+ DisableNewNetNs bool
+ EnableCompatOldCNI bool
+ NetmonConfig NetmonConfig
+ InterworkingModel NetInterworkingModel
}
func networkLogger() *logrus.Entry {
diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go
index 6bd09a0b..fe00bf9a 100644
--- a/virtcontainers/persist.go
+++ b/virtcontainers/persist.go
@@ -187,10 +187,11 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
},
ShimType: string(sconfig.ShimType),
NetworkConfig: persistapi.NetworkConfig{
- NetNSPath: sconfig.NetworkConfig.NetNSPath,
- NetNsCreated: sconfig.NetworkConfig.NetNsCreated,
- DisableNewNetNs: sconfig.NetworkConfig.DisableNewNetNs,
- InterworkingModel: int(sconfig.NetworkConfig.InterworkingModel),
+ NetNSPath: sconfig.NetworkConfig.NetNSPath,
+ NetNsCreated: sconfig.NetworkConfig.NetNsCreated,
+ DisableNewNetNs: sconfig.NetworkConfig.DisableNewNetNs,
+ EnableCompatOldCNI: sconfig.NetworkConfig.EnableCompatOldCNI,
+ InterworkingModel: int(sconfig.NetworkConfig.InterworkingModel),
},
ShmSize: sconfig.ShmSize,
@@ -477,10 +478,11 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
},
ShimType: ShimType(savedConf.ShimType),
NetworkConfig: NetworkConfig{
- NetNSPath: savedConf.NetworkConfig.NetNSPath,
- NetNsCreated: savedConf.NetworkConfig.NetNsCreated,
- DisableNewNetNs: savedConf.NetworkConfig.DisableNewNetNs,
- InterworkingModel: NetInterworkingModel(savedConf.NetworkConfig.InterworkingModel),
+ NetNSPath: savedConf.NetworkConfig.NetNSPath,
+ NetNsCreated: savedConf.NetworkConfig.NetNsCreated,
+ DisableNewNetNs: savedConf.NetworkConfig.DisableNewNetNs,
+ EnableCompatOldCNI: savedConf.NetworkConfig.EnableCompatOldCNI,
+ InterworkingModel: NetInterworkingModel(savedConf.NetworkConfig.InterworkingModel),
},
ShmSize: savedConf.ShmSize,
diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go
index cfbee849..3a2df32b 100644
--- a/virtcontainers/persist/api/config.go
+++ b/virtcontainers/persist/api/config.go
@@ -210,10 +210,11 @@ type ShimConfig struct {
// NetworkConfig is the network configuration related to a network.
type NetworkConfig struct {
- NetNSPath string
- NetNsCreated bool
- DisableNewNetNs bool
- InterworkingModel int
+ NetNSPath string
+ NetNsCreated bool
+ DisableNewNetNs bool
+ EnableCompatOldCNI bool
+ InterworkingModel int
}
type ContainerConfig struct {
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
index 36c730b7..948bd3cb 100644
--- a/virtcontainers/pkg/oci/utils.go
+++ b/virtcontainers/pkg/oci/utils.go
@@ -130,6 +130,9 @@ type RuntimeConfig struct {
//Determines if create a netns for hypervisor process
DisableNewNetNs bool
+ // Determines if compatible with old CNI plugin
+ EnableCompatOldCNI bool
+
//Determines kata processes are managed only in sandbox cgroup
SandboxCgroupOnly bool
@@ -275,6 +278,7 @@ func networkConfig(ocispec specs.Spec, config RuntimeConfig) (vc.NetworkConfig,
}
netConf.InterworkingModel = config.InterNetworkModel
netConf.DisableNewNetNs = config.DisableNewNetNs
+ netConf.EnableCompatOldCNI = config.EnableCompatOldCNI
netConf.NetmonConfig = vc.NetmonConfig{
Path: config.NetmonConfig.Path,
diff --git a/virtcontainers/pkg/vcmock/sandbox.go b/virtcontainers/pkg/vcmock/sandbox.go
index 677457ef..11b83ccd 100644
--- a/virtcontainers/pkg/vcmock/sandbox.go
+++ b/virtcontainers/pkg/vcmock/sandbox.go
@@ -212,3 +212,8 @@ func (s *Sandbox) UpdateRoutes(routes []*vcTypes.Route) ([]*vcTypes.Route, error
func (s *Sandbox) ListRoutes() ([]*vcTypes.Route, error) {
return nil, nil
}
+
+// IsCompatOldCNI return the whether enable compatible with old CNI
+func (s *Sandbox) IsCompatOldCNI() bool {
+ return false
+}
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
index c8981a41..6a643a12 100644
--- a/virtcontainers/sandbox.go
+++ b/virtcontainers/sandbox.go
@@ -2379,6 +2379,11 @@ func (s *Sandbox) setContainersState(state types.StateString) error {
return nil
}
+// IsCompatOldCNI return the whether enable compatible with old CNI
+func (s *Sandbox) IsCompatOldCNI() bool {
+ return s.config.NetworkConfig.EnableCompatOldCNI
+}
+
// updateStaticSandboxResources update sandbox's cpu and memory resource passed by
// sandbox_cpu and sandbox_mem annotations
func updateStaticSandboxResources(sandboxConfig *SandboxConfig) error {
--
2.14.3 (Apple Git-98)

182
runtime/patches/0027-network-add-more-strict-check-for-input-network-inte.patch Normal file
View File

@ -0,0 +1,182 @@
From ec15337fc816767ca0e8183576405499080b9b1e Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Sun, 16 Aug 2020 16:41:18 +0800
Subject: [PATCH 27/50] network: add more strict check for input network
interface
reason: kata-network add-iface command will receive the network
interface info from untrust user, so we need to add more strict
check for input network interface info.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
virtcontainers/network.go | 117 ++++++++++++++++++++++++++++++++++++++++++++++
virtcontainers/sandbox.go | 11 +++++
2 files changed, 128 insertions(+)
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
index db235cf6..a1676ccd 100644
--- a/virtcontainers/network.go
+++ b/virtcontainers/network.go
@@ -13,8 +13,10 @@ import (
"math/rand"
"net"
"os"
+ "regexp"
"runtime"
"sort"
+ "strings"
"time"
"github.com/containernetworking/plugins/pkg/ns"
@@ -55,6 +57,17 @@ const (
NetXConnectInvalidModel
)
+const (
+ maxIPAddrLen = 18
+ maxInterfaceLen = 15
+ minMTUVal = 46
+ maxMTUVal = 9600
+)
+
+var (
+ regInfName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_\-.]*$`)
+)
+
//IsValid checks if a model is valid
func (n NetInterworkingModel) IsValid() bool {
return 0 <= int(n) && int(n) < int(NetXConnectInvalidModel)
@@ -1368,3 +1381,107 @@ func convertToCompatInterfaces(es *[]Endpoint) []*vcTypes.Interface {
return infs
}
+
+// verifyInterfaceName verifies the interface name valid or not
+func verifyInterfaceName(name string) error {
+ // verify `Name` before `Tapname` because of the strict rules
+ name = strings.TrimSpace(name)
+ if len(name) > maxInterfaceLen {
+ return fmt.Errorf("interface name can't be longer than 15")
+ } else if len(name) == 0 || name == "\n" {
+ return fmt.Errorf("interface name can't be empty")
+ }
+
+ // QMP rules of `Name`
+ chk := regInfName.FindAllString(name, -1)
+ if chk == nil {
+ return fmt.Errorf("invalid input of interface name, please check the rules")
+ }
+
+ return nil
+}
+
+// verifyIPAndMask verifies CIDR notation IP address and mask like "192.0.2.0/24"
+func verifyIPAndMask(ip string) (nlIpMask *net.IPNet, err error) {
+ ip = strings.TrimSpace(ip)
+
+ if len(ip) > maxIPAddrLen {
+ return nil, fmt.Errorf("the length of IP address is too long, max ip len :%d", maxIPAddrLen)
+ }
+
+ if nlIpMask, err = netlink.ParseIPNet(ip); err != nil {
+ return nil, fmt.Errorf("invalid input of IP : %v", err)
+ }
+
+ return nlIpMask, nil
+}
+
+// verifyMac verifies MAC address
+func verifyMac(mac string) error {
+ mac = strings.TrimSpace(mac)
+ if _, err := net.ParseMAC(mac); err != nil {
+ return fmt.Errorf("invalid input of Mac : %v", err)
+ }
+
+ return nil
+}
+
+// verifyMtu verifies MTU value of interface
+func verifyMtu(mtu uint64) error {
+ if mtu < minMTUVal || mtu > maxMTUVal {
+ return fmt.Errorf("invalid input of MTU : %v", mtu)
+ }
+ return nil
+}
+
+// verifyIP verifies the IP address
+func verifyIP(ip string) (*net.IP, error) {
+ ip = strings.TrimSpace(ip)
+
+ if len(ip) > maxIPAddrLen {
+ return nil, fmt.Errorf("the length of IP address is too long, max ip len :%d", maxIPAddrLen)
+ }
+
+ var netIP net.IP
+ if netIP = net.ParseIP(ip); netIP == nil {
+ return nil, fmt.Errorf("invalid IP: %s", ip)
+ }
+
+ return &netIP, nil
+}
+
+// validInterface check the input interface valid or not
+func validInterface(inf *vcTypes.Interface, enableCompatOldCNI bool) error {
+ if enableCompatOldCNI && verifyInterfaceName(inf.Device) != nil {
+ return fmt.Errorf("device name should not be empty when enable_compat_old_cni config enabled")
+ }
+
+ if inf.Name == "" || inf.Mtu == 0 || inf.HwAddr == "" {
+ return fmt.Errorf("name/mtu/hwaddr of interface must be specified")
+ }
+
+ if err := verifyInterfaceName(inf.Name); err != nil {
+ return err
+ }
+
+ if err := verifyMac(inf.HwAddr); err != nil {
+ return err
+ }
+
+ if err := verifyMtu(inf.Mtu); err != nil {
+ return err
+ }
+
+ // Currently, only one IP address can be passed, which reduces the test entry and fault injection.
+ if len(inf.IPAddresses) > 0 {
+ if len(inf.IPAddresses) != 1 {
+ return fmt.Errorf("only one IP address is supported currently")
+ }
+ _, err := verifyIP(inf.IPAddresses[0].Address)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
index 6a643a12..f6826812 100644
--- a/virtcontainers/sandbox.go
+++ b/virtcontainers/sandbox.go
@@ -952,6 +952,17 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) {
// AddInterface adds new nic to the sandbox.
func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (grpcIf *vcTypes.Interface, err error) {
+ err = validInterface(inf, s.config.NetworkConfig.EnableCompatOldCNI)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, ep := range s.networkNS.Endpoints {
+ if ep.Name() == inf.Name {
+ return nil, fmt.Errorf("interface %s is already exist", inf.Name)
+ }
+ }
+
netInfo, err := s.generateNetInfo(inf)
if err != nil {
return nil, err
--
2.14.3 (Apple Git-98)

1026
runtime/patches/0028-network-add-kata-network-add-route-subcommand.patch Normal file
View File

File diff suppressed because it is too large Load Diff

192
runtime/patches/0029-network-add-kata-network-del-route-subcommand.patch Normal file
View File

@ -0,0 +1,192 @@
From 51a7270987557ab12ea735fc9781725d1ce1b0a6 Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Mon, 17 Aug 2020 15:36:32 +0800
Subject: [PATCH 29/50] network: add kata-network del-route subcommand
reason: add kata-network del-route subcommand to delete the
specified route in the Kata VM, del-route is more efficient
than the update-routes subcommand.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
cli/network.go | 24 ++++++++++
virtcontainers/network.go | 115 ++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 139 insertions(+)
diff --git a/cli/network.go b/cli/network.go
index 2265f54b..046d0ee9 100644
--- a/cli/network.go
+++ b/cli/network.go
@@ -45,6 +45,7 @@ var kataNetworkCLICommand = cli.Command{
updateRoutesCommand,
listRoutesCommand,
addRoutesCommand,
+ deleteRoutesCommand,
},
Action: func(context *cli.Context) error {
return cli.ShowSubcommandHelp(context)
@@ -155,6 +156,29 @@ var addRoutesCommand = cli.Command{
},
}
+var deleteRoutesCommand = cli.Command{
+ Name: "del-route",
+ Usage: "delete one route for a container",
+ ArgsUsage: `del-route <container-id> file or - for stdin
+ file or stdin for example:
+ {
+ "dest":"<[<ip>[/mask] | "default" ]>",
+ "gateway":"[ip]",
+ "device":"[tap-name]",
+ }
+ Only destination is required and others are optional,
+ if device is empty, means search every device`,
+ Flags: []cli.Flag{},
+ Action: func(context *cli.Context) error {
+ ctx, err := cliContextToContext(context)
+ if err != nil {
+ return err
+ }
+
+ return networkModifyCommand(ctx, context.Args().First(), context.Args().Get(1), routeType, vcTypes.NetworkOpRemove)
+ },
+}
+
var listRoutesCommand = cli.Command{
Name: "list-routes",
Usage: "list network routes in a container",
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
index f3757f84..c7066a11 100644
--- a/virtcontainers/network.go
+++ b/virtcontainers/network.go
@@ -1631,6 +1631,110 @@ func addOneRoute(ns *NetworkNamespace, route *vcTypes.Route) (added *vcTypes.Rou
return added, nil
}
+func generateRmRoute(route *vcTypes.Route) (r *netlink.Route, err error) {
+ // destination is required and others are optional
+ if route.Dest == "" {
+ return nil, fmt.Errorf("destination must be specified when remove route.")
+ }
+
+ if route.Device != "" {
+ err = verifyInterfaceName(route.Device)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ r = &netlink.Route{}
+ if route.Dest != "default" {
+ nlIpNet, err := verifyRouteDest(&route.Dest)
+ if err != nil {
+ return nil, err
+ }
+ r.Dst = nlIpNet
+ }
+
+ if route.Gateway != "" {
+ nIP, err := verifyIP(route.Gateway)
+ if err != nil {
+ return nil, err
+ }
+ r.Gw = *nIP
+ }
+
+ if route.Source != "" {
+ nIP, err := verifyIP(route.Source)
+ if err != nil {
+ return nil, err
+ }
+ r.Src = *nIP
+ }
+
+ r.Scope = netlink.Scope(route.Scope)
+
+ return r, nil
+}
+
+// parseToGrpcRoute convert the netlink.Route to vcTypes.Route and deleted route.Dest will
+// be added a prefix "-"
+func parseToGrpcRoute(device string, route *netlink.Route, add bool) (r *vcTypes.Route) {
+ r = &vcTypes.Route{
+ Device: device,
+ }
+ if route.Dst != nil && route.Dst.String() != "<nil>" {
+ r.Dest = route.Dst.String()
+ if add == false {
+ r.Dest = "-" + route.Dst.String()
+ }
+ } else if route.Dst == nil {
+ r.Dest = "default"
+ if add == false {
+ r.Dest = "-default"
+ }
+ }
+ if route.Gw != nil && route.Gw.String() != "<nil>" {
+ r.Gateway = route.Gw.String()
+ }
+
+ return r
+}
+
+func removeRoutes(ns *NetworkNamespace, route *vcTypes.Route) (removed []*vcTypes.Route, err error) {
+ del, err := generateRmRoute(route)
+ if err != nil {
+ return nil, err
+ }
+
+ // remove the lo device related routes
+ if route.Device == localHostDeviceName {
+ removed = append(removed, parseToGrpcRoute(localHostDeviceName, del, false))
+
+ return removed, nil
+ }
+
+ for _, ep := range ns.Endpoints {
+ // if device is empty, means search every device
+ if route.Device != "" && ep.Name() != route.Device {
+ continue
+ }
+
+ netInfo := ep.Properties()
+ for i, exist := range ep.Properties().Routes {
+ if isSameRoute(&exist, del, true) {
+ // need remove
+ netInfo.Routes = append(netInfo.Routes[:i], netInfo.Routes[i+1:]...)
+ ep.SetProperties(netInfo)
+ dev := route.Device
+ if route.Device == "" {
+ dev = netInfo.Iface.Name
+ }
+ removed = append(removed, parseToGrpcRoute(dev, del, false))
+ }
+ }
+ }
+
+ return removed, nil
+}
+
func updateRoute(ns *NetworkNamespace, route *vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) {
var updRoutes []*vcTypes.Route
@@ -1645,5 +1749,16 @@ func updateRoute(ns *NetworkNamespace, route *vcTypes.Route, op vcTypes.NetworkO
updRoutes = append(updRoutes, added)
}
+ if op == vcTypes.NetworkOpRemove {
+ removed, err := removeRoutes(ns, route)
+ if err != nil {
+ return nil, err
+ }
+ if len(removed) <= 0 {
+ return nil, fmt.Errorf("route of device %s with destination %s is not found", route.Device, route.Dest)
+ }
+ updRoutes = removed
+ }
+
return updRoutes, nil
}
--
2.14.3 (Apple Git-98)

104
runtime/patches/0030-network-kata-network-list-routes-support-display-com.patch Normal file
View File

@ -0,0 +1,104 @@
From 9cf769178b8f73c7fd624895589e918d6c7b0645 Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Mon, 17 Aug 2020 16:29:17 +0800
Subject: [PATCH 30/50] network: kata-network list-routes support display
compatible format
reason: kata-network list-routes subcommand support display compatible
format when enable_compat_old_cni config is enabled.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
virtcontainers/api.go | 13 ++++++++++++-
virtcontainers/network.go | 29 +++++++++++++++++++++++++++++
virtcontainers/pkg/types/types.go | 10 +++++-----
3 files changed, 46 insertions(+), 6 deletions(-)
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
index 2331eb94..a4bf41bb 100644
--- a/virtcontainers/api.go
+++ b/virtcontainers/api.go
@@ -1003,7 +1003,18 @@ func ListRoutes(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error)
}
defer s.releaseStatelessSandbox()
- return s.ListRoutes()
+ routes, err := s.ListRoutes()
+ if err != nil {
+ return nil, err
+ }
+
+ // If enable_compat_old_cni is enabled, convert routes info to
+ // compatible display format
+ if s.config.NetworkConfig.EnableCompatOldCNI {
+ return convertToDisplayRoutes(&routes), nil
+ }
+
+ return routes, nil
}
// CleanupContaienr is used by shimv2 to stop and delete a container exclusively, once there is no container
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
index c7066a11..488bd00c 100644
--- a/virtcontainers/network.go
+++ b/virtcontainers/network.go
@@ -1384,6 +1384,35 @@ func convertToCompatInterfaces(es *[]Endpoint) []*vcTypes.Interface {
return infs
}
+// convertToDisplayRoutes convert the default route format to more simple
+// route display format, it is called when enable_compat_old_cni config
+// is enabled.
+func convertToDisplayRoutes(routes *[]*vcTypes.Route) []*vcTypes.Route {
+ var displayRoutes []*vcTypes.Route
+ if routes == nil {
+ return displayRoutes
+ }
+ for _, r := range *routes {
+ // we don't support IPV6 temporarily, so we need to filter ":" which
+ // is the characteristics of IPV6 for those default routes
+ if strings.Contains(r.Dest, ":") {
+ continue
+ }
+
+ defaultDest := r.Dest
+ if r.Dest == "" {
+ defaultDest = "default"
+ }
+ displayRoutes = append(displayRoutes, &vcTypes.Route{
+ Dest: defaultDest,
+ Gateway: r.Gateway,
+ Device: r.Device,
+ })
+ }
+
+ return displayRoutes
+}
+
// verifyInterfaceName verifies the interface name valid or not
func verifyInterfaceName(name string) error {
// verify `Name` before `Tapname` because of the strict rules
diff --git a/virtcontainers/pkg/types/types.go b/virtcontainers/pkg/types/types.go
index 71fe7fbb..b41b0c75 100644
--- a/virtcontainers/pkg/types/types.go
+++ b/virtcontainers/pkg/types/types.go
@@ -33,11 +33,11 @@ type Interface struct {
// Route describes a network route.
type Route struct {
- Dest string
- Gateway string
- Device string
- Source string
- Scope uint32
+ Dest string `json:"dest,omitempty"`
+ Gateway string `json:"gateway,omitempty"`
+ Device string `json:"device,omitempty"`
+ Source string `json:"source,omitempty"`
+ Scope uint32 `json:"scope,omitempty"`
}
//NetworkOp describes network operation
--
2.14.3 (Apple Git-98)

41
runtime/patches/0031-device_mangaer-check-VFIO-when-create-device.patch Normal file
View File

@ -0,0 +1,41 @@
From 326e90f97cf5ace73dff95257f7b4faa43c56f99 Mon Sep 17 00:00:00 2001
From: holyfei <yangfeiyu20092010@163.com>
Date: Mon, 17 Aug 2020 16:11:35 +0800
Subject: [PATCH 31/50] device_mangaer: check VFIO when create device
reason: check VFIO when create device, block device can be
reused and VFIO device can not
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
---
virtcontainers/device/manager/manager.go | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/virtcontainers/device/manager/manager.go b/virtcontainers/device/manager/manager.go
index 6a2b665a..d24a29ab 100644
--- a/virtcontainers/device/manager/manager.go
+++ b/virtcontainers/device/manager/manager.go
@@ -9,6 +9,7 @@ package manager
import (
"encoding/hex"
"errors"
+ "fmt"
"sync"
"github.com/sirupsen/logrus"
@@ -115,7 +116,11 @@ func (dm *deviceManager) createDevice(devInfo config.DeviceInfo) (dev api.Device
}()
if existingDev := dm.findDeviceByMajorMinor(devInfo.Major, devInfo.Minor); existingDev != nil {
- return existingDev, nil
+ if isVFIO(devInfo.HostPath) {
+ return nil, fmt.Errorf("device %s is replicated in the same Pod!", devInfo.ContainerPath)
+ } else {
+ return existingDev, nil
+ }
}
// device ID must be generated by manager instead of device itself
--
2.14.3 (Apple Git-98)

46
runtime/patches/0032-network-add-more-detail-usage-for-update-routes-subc.patch Normal file
View File

@ -0,0 +1,46 @@
From 77711b531867be899df5d2c59a525ea1b7f0e6ac Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Mon, 17 Aug 2020 17:14:35 +0800
Subject: [PATCH 32/50] network: add more detail usage for update-routes
subcommand
reason: add more detail usage for update-routes subcommand
to explain how to use update-routes subcommand
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
cli/network.go | 18 ++++++++++++++----
1 file changed, 14 insertions(+), 4 deletions(-)
diff --git a/cli/network.go b/cli/network.go
index 046d0ee9..3dd0971e 100644
--- a/cli/network.go
+++ b/cli/network.go
@@ -120,10 +120,20 @@ var listIfacesCommand = cli.Command{
}
var updateRoutesCommand = cli.Command{
- Name: "update-routes",
- Usage: "update routes of a container",
- ArgsUsage: `update-routes <container-id> file or - for stdin`,
- Flags: []cli.Flag{},
+ Name: "update-routes",
+ Usage: "update routes of a container",
+ ArgsUsage: `update-routes <container-id> file or - for stdin
+ file or stdin for example:
+ [
+ {
+ "dest":"<[<ip>[/mask] | "default" ]>",
+ "gateway":"[ip]",
+ "device":"[tap-name]",
+ "source": "[source]",
+ "scope":[scope]
+ }
+ ]`,
+ Flags: []cli.Flag{},
Action: func(context *cli.Context) error {
ctx, err := cliContextToContext(context)
if err != nil {
--
2.14.3 (Apple Git-98)

80
runtime/patches/0033-network-do-not-delete-the-exist-tap-device-in-the-ho.patch Normal file
View File

@ -0,0 +1,80 @@
From 7ab7ff54efa3925a8d372f7830d31b87f8d01ea8 Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Mon, 17 Aug 2020 17:39:18 +0800
Subject: [PATCH 33/50] network: do not delete the exist tap device in the host
reason: If hotplug a exist tap device into Kata VM, kata-runtime
should not delete this exist tap device, because this tap device
is controlled by other network components.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
virtcontainers/tap_endpoint.go | 16 +++++++++++++---
virtcontainers/veth_endpoint.go | 3 +++
2 files changed, 16 insertions(+), 3 deletions(-)
diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go
index c897670e..2cf70dce 100644
--- a/virtcontainers/tap_endpoint.go
+++ b/virtcontainers/tap_endpoint.go
@@ -77,7 +77,7 @@ func (endpoint *TapEndpoint) Detach(netNsCreated bool, netNsPath string) error {
networkLogger().WithField("endpoint-type", TapEndpointType).Info("Detaching endpoint")
return doNetNS(netNsPath, func(_ ns.NetNS) error {
- return unTapNetwork(endpoint.TapInterface.TAPIface.Name)
+ return unTapNetwork(endpoint)
})
}
@@ -91,6 +91,9 @@ func (endpoint *TapEndpoint) HotAttach(h hypervisor) error {
if _, err := h.hotplugAddDevice(endpoint, netDev); err != nil {
networkLogger().WithError(err).Error("Error attach tap ep")
+ if errUnTap := unTapNetwork(endpoint); errUnTap != nil {
+ networkLogger().WithError(errUnTap).Errorf("Error rollback tap %s", endpoint.TapInterface.TAPIface.Name)
+ }
return err
}
return nil
@@ -100,7 +103,7 @@ func (endpoint *TapEndpoint) HotAttach(h hypervisor) error {
func (endpoint *TapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error {
networkLogger().Info("Hot detaching tap endpoint")
if err := doNetNS(netNsPath, func(_ ns.NetNS) error {
- return unTapNetwork(endpoint.TapInterface.TAPIface.Name)
+ return unTapNetwork(endpoint)
}); err != nil {
networkLogger().WithError(err).Warn("Error un-bridging tap ep")
}
@@ -185,7 +188,14 @@ func tapNetwork(endpoint *TapEndpoint, numCPUs uint32, disableVhostNet bool) err
return nil
}
-func unTapNetwork(name string) error {
+func unTapNetwork(endpoint *TapEndpoint) error {
+ // length of VMFDs == 0 means that the endpoint is already exist in the host,
+ // no created by kata, so we don't need to remove it when detach
+ if len(endpoint.TapInterface.VMFds) == 0 {
+ return nil
+ }
+
+ name := endpoint.TapInterface.TAPIface.Name
netHandle, err := netlink.NewHandle()
if err != nil {
return err
diff --git a/virtcontainers/veth_endpoint.go b/virtcontainers/veth_endpoint.go
index 9ece6a74..0f2ec9ba 100644
--- a/virtcontainers/veth_endpoint.go
+++ b/virtcontainers/veth_endpoint.go
@@ -119,6 +119,9 @@ func (endpoint *VethEndpoint) HotAttach(h hypervisor) error {
if _, err := h.hotplugAddDevice(endpoint, netDev); err != nil {
networkLogger().WithError(err).Error("Error attach virtual ep")
+ if errDisconn := xDisconnectVMNetwork(endpoint); errDisconn != nil {
+ networkLogger().WithError(errDisconn).Error("Error rollback virtual ep")
+ }
return err
}
return nil
--
2.14.3 (Apple Git-98)

1273
runtime/patches/0034-kata-runtime-add-kata-ipvs-command.patch Normal file
View File

File diff suppressed because it is too large Load Diff

234
runtime/patches/0035-device-mount-blockdevices-in-the-guest-VM.patch Normal file
View File

@ -0,0 +1,234 @@
From cf2b34f477cba88641de3719bf5c8f933b919bcc Mon Sep 17 00:00:00 2001
From: holyfei <yangfeiyu20092010@163.com>
Date: Mon, 17 Aug 2020 21:44:57 +0800
Subject: [PATCH 35/50] device: mount blockdevices in the guest VM
reason: support mount blockdevices in the guest VM like
"-v /dev/blockdevice:/home/test"
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
---
cli/config/configuration-qemu.toml.in | 5 ++++
pkg/katautils/config.go | 6 +++++
virtcontainers/kata_agent.go | 27 ++++++++++++++++---
virtcontainers/kata_agent_test.go | 2 +-
virtcontainers/utils/utils.go | 50 +++++++++++++++++++++++++++++++++++
virtcontainers/vm_test.go | 2 +-
6 files changed, 86 insertions(+), 6 deletions(-)
diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in
index 46f8b632..aa11b38f 100644
--- a/cli/config/configuration-qemu.toml.in
+++ b/cli/config/configuration-qemu.toml.in
@@ -391,6 +391,11 @@ path = "@SHIMPATH@"
#
kernel_modules=[]
+# If enabled, when we pass a block device to the guest VM
+# through -v, such as `docker run -v /dev/loop100:/foo/bar`,
+# agent will create a directory in guest VM and mount the
+# file system on `/dev/loop100` inside the container
+enable_blk_mount = true
[netmon]
# If enabled, the network monitoring process gets started when the
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
index 94c916a0..51120311 100644
--- a/pkg/katautils/config.go
+++ b/pkg/katautils/config.go
@@ -160,6 +160,7 @@ type agent struct {
TraceMode string `toml:"trace_mode"`
TraceType string `toml:"trace_type"`
KernelModules []string `toml:"kernel_modules"`
+ MountBlkInVM bool `toml:"enable_blk_mount"`
}
type netmon struct {
@@ -463,6 +464,10 @@ func (h hypervisor) getInitrdAndImage() (initrd string, image string, err error)
return
}
+func (a agent) mountBlkDevInVM() bool {
+ return a.MountBlkInVM
+}
+
func (p proxy) path() (string, error) {
path := p.Path
if path == "" {
@@ -978,6 +983,7 @@ func updateRuntimeConfigAgent(configPath string, tomlConf tomlConfig, config *oc
TraceMode: agent.traceMode(),
TraceType: agent.traceType(),
KernelModules: agent.kernelModules(),
+ MountBlkInVM: agent.mountBlkDevInVM(),
}
default:
return fmt.Errorf("%s agent type is not supported", k)
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
index 16662949..b0f88c15 100644
--- a/virtcontainers/kata_agent.go
+++ b/virtcontainers/kata_agent.go
@@ -30,6 +30,7 @@ import (
ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter"
"github.com/kata-containers/runtime/virtcontainers/pkg/rootless"
vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
+ "github.com/kata-containers/runtime/virtcontainers/utils"
"github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
"github.com/kata-containers/runtime/virtcontainers/store"
"github.com/kata-containers/runtime/virtcontainers/types"
@@ -64,6 +65,7 @@ var (
errorMissingOCISpec = errors.New("Missing OCI specification")
defaultKataHostSharedDir = "/run/kata-containers/shared/sandboxes/"
defaultKataGuestSharedDir = "/run/kata-containers/shared/containers/"
+ kataGuestStorageDir = "/run/kata-containers/storage/containers/"
mountGuestTag = "kataShared"
defaultKataGuestSandboxDir = "/run/kata-containers/sandbox/"
type9pFs = "9p"
@@ -199,6 +201,7 @@ type KataAgentConfig struct {
TraceMode string
TraceType string
KernelModules []string
+ MountBlkInVM bool
}
// KataAgentState is the structure describing the data stored from this
@@ -1061,7 +1064,12 @@ func (k *kataAgent) replaceOCIMountsForStorages(spec *specs.Spec, volumeStorages
// Create a temporary location to mount the Storage. Mounting to the correct location
// will be handled by the OCI mount structure.
filename := fmt.Sprintf("%s-%s", uuid.Generate().String(), filepath.Base(m.Destination))
- path := filepath.Join(kataGuestSandboxStorageDir(), filename)
+ var path string
+ if v.Driver == kataBlkDevType || v.Driver == kataSCSIDevType {
+ path = filepath.Join(kataGuestStorageDir, filename)
+ } else {
+ path = filepath.Join(kataGuestSandboxStorageDir(), filename)
+ }
k.Logger().Debugf("Replacing OCI mount source (%s) with %s", m.Source, path)
ociMounts[index].Source = path
@@ -1436,7 +1444,7 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process,
// Note this call modifies the list of container devices to make sure
// all hotplugged devices are unplugged, so this needs be done
// after devices passed with --device are handled.
- volumeStorages, err := k.handleBlockVolumes(c)
+ volumeStorages, err := k.handleBlockVolumes(sandbox, c)
if err != nil {
return nil, err
}
@@ -1609,7 +1617,7 @@ func (k *kataAgent) handleVhostUserBlkVolume(c *Container, device api.Device) (*
// handleBlockVolumes handles volumes that are block devices files
// by passing the block devices as Storage to the agent.
-func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) {
+func (k *kataAgent) handleBlockVolumes(sandbox *Sandbox, c *Container) ([]*grpc.Storage, error) {
var volumeStorages []*grpc.Storage
@@ -1648,9 +1656,20 @@ func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) {
}
vol.MountPoint = m.Destination
- if vol.Fstype == "" {
+
+ ac, _ := sandbox.config.AgentConfig.(KataAgentConfig)
+ if ac.MountBlkInVM {
+ // Ensure the block device is formatted, for the devices here are specified as volumes
+ fsType, gerr := utils.GetDevFormat(m.Source)
+ if gerr != nil || fsType == "" {
+ k.Logger().WithField("device", id).WithError(gerr).Error("get device format failed")
+ return nil, gerr
+ }
+ vol.Fstype = fmt.Sprintf("bind-%s", fsType)
+ } else {
vol.Fstype = "bind"
}
+
if len(vol.Options) == 0 {
vol.Options = []string{"bind"}
}
diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go
index 18a5a0a6..68caaab2 100644
--- a/virtcontainers/kata_agent_test.go
+++ b/virtcontainers/kata_agent_test.go
@@ -446,7 +446,7 @@ func TestHandleBlockVolume(t *testing.T) {
containers[c.id].sandbox = &sandbox
containers[c.id].mounts = mounts
- volumeStorages, err := k.handleBlockVolumes(c)
+ volumeStorages, err := k.handleBlockVolumes(&sandbox, c)
assert.Nil(t, err, "Error while handling block volumes")
vStorage := &pb.Storage{
diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go
index 9490faa1..36ac67a7 100644
--- a/virtcontainers/utils/utils.go
+++ b/virtcontainers/utils/utils.go
@@ -379,3 +379,53 @@ func RoundVCPUNumber(value string) (int, error) {
cpus := int(math.Ceil(cpuNum))
return cpus, nil
}
+
+// GetDevFormat get the formated filesystem of input disk use `blkid`, Such as `ext4`,`xfs`,etc.
+func GetDevFormat(disk string) (string, error) {
+ // refer to https://github.com/kubernetes/kubernetes/blob/v1.12.2/pkg/util/mount/mount_linux.go#L512
+ args := []string{"-p", "-s", "TYPE", "-s", "PTTYPE", "-o", "export", disk}
+ dataOut, err := exec.Command("blkid", args...).Output()
+ output := string(dataOut)
+
+ if err != nil {
+ if strings.Contains(err.Error(), "exit status 2") {
+ // Disk device is unformatted.
+ // For `blkid`, if the specified token (TYPE/PTTYPE, etc) was
+ // not found, or no (specified) devices could be identified, an
+ // exit code of 2 is returned.
+ return "", nil
+ }
+ return "", err
+ }
+
+ var fstype string
+
+ lines := strings.Split(output, "\n")
+ for _, l := range lines {
+ if len(l) <= 0 {
+ // Ignore empty line.
+ continue
+ }
+ // if we use busybox as rootfs,the output of command
+ // `busybox blkid` is different with original`blkid`,
+ // so we should make a compatible parse
+ subLine := strings.Split(l, " ")
+ for _, sl := range subLine {
+ if len(subLine) <= 0 {
+ continue
+ }
+ cs := strings.Split(sl, "=")
+ if len(cs) != 2 {
+ continue
+ }
+ if cs[0] == "TYPE" {
+ fstype = cs[1]
+ if strings.Contains(fstype, `"`) {
+ fstype = strings.Replace(fstype, `"`, "", -1)
+ }
+ }
+ }
+ }
+
+ return fstype, nil
+}
diff --git a/virtcontainers/vm_test.go b/virtcontainers/vm_test.go
index 36fd5c2f..d2414232 100644
--- a/virtcontainers/vm_test.go
+++ b/virtcontainers/vm_test.go
@@ -125,7 +125,7 @@ func TestVMConfigGrpc(t *testing.T) {
HypervisorType: QemuHypervisor,
HypervisorConfig: newQemuConfig(),
AgentType: KataContainersAgent,
- AgentConfig: KataAgentConfig{false, true, false, false, 0, "", "", []string{}},
+ AgentConfig: KataAgentConfig{false, true, false, false, 0, "", "", []string{}, false },
ProxyType: NoopProxyType,
}
--
2.14.3 (Apple Git-98)

131
runtime/patches/0036-mount-limit-the-maximum-number-of-virtio-scsi-bus-sl.patch Normal file
View File

@ -0,0 +1,131 @@
From 448bb661d6759c1e20c18084604b150bff08ada4 Mon Sep 17 00:00:00 2001
From: holyfei <yangfeiyu20092010@163.com>
Date: Tue, 18 Aug 2020 11:49:26 +0800
Subject: [PATCH 36/50] mount: limit the maximum number of virtio-scsi bus
slots
reason:
1. add SCSIBus functions for add or remove device about ScsiBus
2. limit the maximum number of virtio-scsi bus slots to 25
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
---
virtcontainers/qemu.go | 20 +++++++++++++++++
virtcontainers/types/scsi.go | 53 ++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 73 insertions(+)
create mode 100644 virtcontainers/types/scsi.go
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
index bb83b1bb..c2b65376 100644
--- a/virtcontainers/qemu.go
+++ b/virtcontainers/qemu.go
@@ -65,6 +65,7 @@ type CPUDevice struct {
// QemuState keeps Qemu's state
type QemuState struct {
Bridges []types.Bridge
+ ScsiBus *types.SCSIBus
// HotpluggedCPUs is the list of CPUs that were hot-added
HotpluggedVCPUs []CPUDevice
HotpluggedMemory int
@@ -411,6 +412,13 @@ func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.I
var ioThread *govmmQemu.IOThread
if q.config.BlockDeviceDriver == config.VirtioSCSI {
+ if q.state.ScsiBus == nil {
+ // only use `scsi0.0`
+ q.state.ScsiBus = &types.SCSIBus{
+ Address: make(map[uint32]int),
+ ID: fmt.Sprintf("%s.0", scsiControllerID),
+ }
+ }
return q.arch.appendSCSIController(devices, q.config.EnableIOThreads)
}
@@ -1168,6 +1176,16 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev
// Bus exposed by the SCSI Controller
bus := scsiControllerID + ".0"
+ var err error
+ if err = q.state.ScsiBus.AddDevToScsiBus(drive.Index); err != nil {
+ return err
+ }
+
+ defer func() {
+ if err != nil {
+ q.state.ScsiBus.RemoveDevFromScsiBus(drive.Index)
+ }
+ }()
// Get SCSI-id and LUN based on the order of attaching drives.
scsiID, lun, err := utils.GetSCSIIdLun(drive.Index)
@@ -1234,6 +1252,8 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error
if err := q.arch.removeDeviceFromBridge(drive.ID); err != nil {
return err
}
+ } else if q.config.BlockDeviceDriver == config.VirtioSCSI {
+ q.state.ScsiBus.RemoveDevFromScsiBus(drive.Index)
}
if err := q.qmpMonitorCh.qmp.ExecuteDeviceDel(q.qmpMonitorCh.ctx, devID); err != nil {
diff --git a/virtcontainers/types/scsi.go b/virtcontainers/types/scsi.go
new file mode 100644
index 00000000..f4d489f0
--- /dev/null
+++ b/virtcontainers/types/scsi.go
@@ -0,0 +1,53 @@
+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+// Description: common functions
+// Author: leizhongkai
+// Create: 2019-06-30
+
+package types
+
+import (
+ "fmt"
+)
+
+const (
+ scsiMaxCapacity = 25
+)
+
+type SCSIBus struct {
+ // Address contains information about devices plugged and number limit
+ Address map[uint32]int
+
+ // ID is used to identify the bus exposed by the SCSI Controller
+ ID string
+}
+
+func (sb *SCSIBus) AddDevToScsiBus(index int) error {
+ var addr uint32
+
+ // looking for the first available address
+ for i := uint32(1); i <= scsiMaxCapacity; i++ {
+ if _, ok := sb.Address[i]; !ok {
+ addr = i
+ break
+ }
+ }
+
+ if addr == 0 {
+ return fmt.Errorf("Scsi bus capacity limited.")
+ }
+
+ // save address and device
+ sb.Address[addr] = index
+
+ return nil
+}
+
+func (sb *SCSIBus) RemoveDevFromScsiBus(index int) {
+ for addr, i := range sb.Address {
+ if i == index {
+ delete(sb.Address, addr)
+ return
+ }
+ }
+}
--
2.14.3 (Apple Git-98)

875
runtime/patches/0037-runtime-add-IPVS-test.patch Normal file
View File

@ -0,0 +1,875 @@
From d083f0e0247fbded92a0ae2a0e71da4176baed95 Mon Sep 17 00:00:00 2001
From: xiadanni <xiadanni1@huawei.com>
Date: Tue, 18 Aug 2020 17:08:23 +0800
Subject: [PATCH 37/50] runtime: add IPVS test
Signed-off-by: xiadanni <xiadanni1@huawei.com>
---
cli/ipvsadm_test.go | 775 +++++++++++++++++++++++++++++++++++++
virtcontainers/api_test.go | 36 ++
virtcontainers/kata_agent_test.go | 4 +
virtcontainers/pkg/vcmock/types.go | 2 +-
4 files changed, 816 insertions(+), 1 deletion(-)
create mode 100644 cli/ipvsadm_test.go
diff --git a/cli/ipvsadm_test.go b/cli/ipvsadm_test.go
new file mode 100644
index 00000000..92958aee
--- /dev/null
+++ b/cli/ipvsadm_test.go
@@ -0,0 +1,775 @@
+// Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+// Description: IPVS common functions test
+// Author: xiadanni
+// Create: 2020-08-01
+
+package main
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "os"
+ "strings"
+ "testing"
+
+ "github.com/kata-containers/agent/protocols/grpc"
+ vc "github.com/kata-containers/runtime/virtcontainers"
+ vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
+ "github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
+ "github.com/kata-containers/runtime/virtcontainers/types"
+ "github.com/stretchr/testify/assert"
+ "github.com/urfave/cli"
+)
+
+func TestKataIPVSCliAction(t *testing.T) {
+ assert := assert.New(t)
+
+ actionFunc, ok := kataIPVSCLICommand.Action.(func(ctx *cli.Context) error)
+ assert.True(ok)
+
+ flagSet := flag.NewFlagSet("kata-IPVS", flag.ContinueOnError)
+ ctx := createCLIContext(flagSet)
+
+ err := actionFunc(ctx)
+ assert.NoError(err)
+}
+
+func TestIPVSadmCliAction(t *testing.T) {
+ assert := assert.New(t)
+
+ actionFunc, ok := IPVSadmCommand.Action.(func(ctx *cli.Context) error)
+ assert.True(ok)
+
+ flagSet := flag.NewFlagSet("IPVSadm", flag.ContinueOnError)
+ ctx := createCLIContext(flagSet)
+
+ err := actionFunc(ctx)
+ assert.Error(err, "Missing container ID")
+}
+
+func TestIPVSadmCLISuccessful(t *testing.T) {
+ assert := assert.New(t)
+
+ sandbox := &vcmock.Sandbox{
+ MockID: testContainerID,
+ }
+
+ sandbox.MockContainers = []*vcmock.Container{
+ {
+ MockID: sandbox.ID(),
+ MockSandbox: sandbox,
+ },
+ }
+
+ testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+ return vc.ContainerStatus{
+ ID: sandbox.ID(),
+ Annotations: map[string]string{
+ vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
+ },
+ State: types.ContainerState{
+ State: types.StateRunning,
+ },
+ }, nil
+ }
+
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
+ return &grpc.IPVSResponse{}, nil
+ }
+
+ defer func() {
+ testingImpl.StatusContainerFunc = nil
+ testingImpl.UpdateIPVSRuleFunc = nil
+ }()
+
+ path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
+ assert.NoError(err)
+ defer os.RemoveAll(path)
+
+ actionFunc, ok := IPVSadmCommand.Action.(func(ctx *cli.Context) error)
+ assert.True(ok)
+
+ flagSet := flag.NewFlagSet("IPVSadm", flag.ContinueOnError)
+ flagSet.Parse([]string{testContainerID})
+ flagSet.String("parameters", "-a -t 192.168.0.7:80 -r 192.168.0.4:80", "")
+ ctx := createCLIContext(flagSet)
+ err = actionFunc(ctx)
+ assert.NoError(err)
+
+ // result not nil
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
+ return &grpc.IPVSResponse{IPVSRes: "IPVS rule updating success"}, nil
+ }
+
+ err = actionFunc(ctx)
+ assert.NoError(err)
+}
+
+func TestIPVSadmCLIError(t *testing.T) {
+ assert := assert.New(t)
+
+ sandbox := &vcmock.Sandbox{
+ MockID: testContainerID,
+ }
+
+ sandbox.MockContainers = []*vcmock.Container{
+ {
+ MockID: sandbox.ID(),
+ MockSandbox: sandbox,
+ },
+ }
+
+ testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+ return vc.ContainerStatus{
+ ID: sandbox.ID(),
+ Annotations: map[string]string{
+ vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
+ },
+ State: types.ContainerState{
+ State: types.StateRunning,
+ },
+ }, nil
+ }
+
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
+ return &grpc.IPVSResponse{}, nil
+ }
+
+ defer func() {
+ testingImpl.StatusContainerFunc = nil
+ testingImpl.UpdateIPVSRuleFunc = nil
+ }()
+
+ path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
+ assert.NoError(err)
+ defer os.RemoveAll(path)
+
+ actionFunc, ok := IPVSadmCommand.Action.(func(ctx *cli.Context) error)
+ assert.True(ok)
+
+ // no stdin rule file
+ flagSet := flag.NewFlagSet("IPVSadm", flag.ContinueOnError)
+ flagSet.Parse([]string{testContainerID})
+ flagSet.String("restore", "-", "")
+ ctx := createCLIContext(flagSet)
+ err = actionFunc(ctx)
+ assert.Error(err)
+
+ // restore parameter error
+ flagSet = flag.NewFlagSet("IPVSadm", flag.ContinueOnError)
+ flagSet.Parse([]string{testContainerID})
+ flagSet.String("restore", "abc", "")
+ ctx = createCLIContext(flagSet)
+ err = actionFunc(ctx)
+ assert.Error(err)
+
+ // checkrule returns error
+ flagSet = flag.NewFlagSet("IPVSadm", flag.ContinueOnError)
+ flagSet.Parse([]string{testContainerID})
+ flagSet.String("parameters", "-A -t 192.168.0.7:80 -s rr -p -3000", "")
+ ctx = createCLIContext(flagSet)
+ err = actionFunc(ctx)
+ assert.Error(err)
+
+ // updatefunction returns error
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
+ return &grpc.IPVSResponse{}, fmt.Errorf("IPVSadm test error")
+ }
+
+ flagSet = flag.NewFlagSet("IPVSadm", flag.ContinueOnError)
+ flagSet.Parse([]string{testContainerID})
+ flagSet.String("parameters", "-A -t 192.168.0.7:80 -s rr -p 3000", "")
+ ctx = createCLIContext(flagSet)
+ err = actionFunc(ctx)
+ assert.Error(err)
+}
+
+func TestCleanupCliAction(t *testing.T) {
+ assert := assert.New(t)
+
+ actionFunc, ok := CleanupCommand.Action.(func(ctx *cli.Context) error)
+ assert.True(ok)
+
+ flagSet := flag.NewFlagSet("cleanup", flag.ContinueOnError)
+ ctx := createCLIContext(flagSet)
+
+ err := actionFunc(ctx)
+ assert.Error(err, "Missing container ID")
+}
+
+func TestCleanupCLISuccessful(t *testing.T) {
+ assert := assert.New(t)
+
+ sandbox := &vcmock.Sandbox{
+ MockID: testContainerID,
+ }
+
+ sandbox.MockContainers = []*vcmock.Container{
+ {
+ MockID: sandbox.ID(),
+ MockSandbox: sandbox,
+ },
+ }
+
+ testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+ return vc.ContainerStatus{
+ ID: sandbox.ID(),
+ Annotations: map[string]string{
+ vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
+ },
+ State: types.ContainerState{
+ State: types.StateRunning,
+ },
+ }, nil
+ }
+
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
+ return &grpc.IPVSResponse{}, nil
+ }
+
+ defer func() {
+ testingImpl.StatusContainerFunc = nil
+ testingImpl.UpdateIPVSRuleFunc = nil
+ }()
+
+ path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
+ assert.NoError(err)
+ defer os.RemoveAll(path)
+
+ actionFunc, ok := CleanupCommand.Action.(func(ctx *cli.Context) error)
+ assert.True(ok)
+
+ flagSet := flag.NewFlagSet("cleanup", flag.ContinueOnError)
+ flagSet.Parse([]string{testContainerID})
+ flagSet.String("parameters", "-d 192.168.0.4 -p tcp", "")
+ ctx := createCLIContext(flagSet)
+ err = actionFunc(ctx)
+ assert.NoError(err)
+}
+
+func TestCleanupCLIError(t *testing.T) {
+ assert := assert.New(t)
+
+ sandbox := &vcmock.Sandbox{
+ MockID: testContainerID,
+ }
+
+ sandbox.MockContainers = []*vcmock.Container{
+ {
+ MockID: sandbox.ID(),
+ MockSandbox: sandbox,
+ },
+ }
+
+ testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+ return vc.ContainerStatus{
+ ID: sandbox.ID(),
+ Annotations: map[string]string{
+ vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
+ },
+ State: types.ContainerState{
+ State: types.StateRunning,
+ },
+ }, nil
+ }
+
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
+ return &grpc.IPVSResponse{}, nil
+ }
+
+ defer func() {
+ testingImpl.StatusContainerFunc = nil
+ testingImpl.UpdateIPVSRuleFunc = nil
+ }()
+
+ path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
+ assert.NoError(err)
+ defer os.RemoveAll(path)
+
+ actionFunc, ok := CleanupCommand.Action.(func(ctx *cli.Context) error)
+ assert.True(ok)
+
+ // checkrule returns error
+ flagSet := flag.NewFlagSet("cleanup", flag.ContinueOnError)
+ flagSet.Parse([]string{testContainerID})
+ flagSet.String("parameters", "-d 192.168.0.4", "")
+ ctx := createCLIContext(flagSet)
+ err = actionFunc(ctx)
+ assert.Error(err)
+
+ // updatefunction returns error
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
+ return &grpc.IPVSResponse{}, fmt.Errorf("IPVSadm cleanup test error")
+ }
+
+ flagSet = flag.NewFlagSet("cleanup", flag.ContinueOnError)
+ flagSet.Parse([]string{testContainerID})
+ flagSet.String("parameters", "-d 192.168.0.4 -p tcp", "")
+ ctx = createCLIContext(flagSet)
+ err = actionFunc(ctx)
+ assert.Error(err)
+}
+
+func TestAddServiceSuccessfully(t *testing.T) {
+ assert := assert.New(t)
+ addServiceCommandTCP := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --scheduler rr --persistent 3000"
+ _, err := checkIPVSRule(addServiceCommandTCP)
+ assert.NoError(err)
+ addServiceCommandUDP := "IPVSadm --add-service --udp-service 192.168.0.7:80 --scheduler rr --persistent 3000"
+ _, err = checkIPVSRule(addServiceCommandUDP)
+ assert.NoError(err)
+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rr -p 3000"
+ _, err = checkIPVSRule(addServiceCmd)
+ assert.NoError(err)
+}
+
+func TestAddServiceNoService(t *testing.T) {
+ assert := assert.New(t)
+ addServiceCommand := "IPVSadm --add-service --scheduler rr --persistent 3000"
+ _, err := checkIPVSRule(addServiceCommand)
+ assert.Error(err)
+ addServiceCmd := "IPVSadm -A -s rr -p 3000"
+ _, err = checkIPVSRule(addServiceCmd)
+ assert.Error(err)
+}
+
+func TestAddServiceNoScheduler(t *testing.T) {
+ assert := assert.New(t)
+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --persistent 3000"
+ _, err := checkIPVSRule(addServiceCommand)
+ assert.Error(err)
+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -p 3000"
+ _, err = checkIPVSRule(addServiceCmd)
+ assert.Error(err)
+}
+
+func TestAddServiceNoPersistent(t *testing.T) {
+ assert := assert.New(t)
+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --scheduler rr"
+ _, err := checkIPVSRule(addServiceCommand)
+ assert.Error(err)
+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rr"
+ _, err = checkIPVSRule(addServiceCmd)
+ assert.Error(err)
+}
+
+func TestAddServiceIpError(t *testing.T) {
+ assert := assert.New(t)
+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.2.2.7:80 --scheduler rr --persistent 3000"
+ _, err := checkIPVSRule(addServiceCommand)
+ assert.Error(err)
+ addServiceCmd := "IPVSadm -A -t 192.168.2.2.7:80 -s rr -p 3000"
+ _, err = checkIPVSRule(addServiceCmd)
+ assert.Error(err)
+}
+
+func TestAddServicePortError(t *testing.T) {
+ assert := assert.New(t)
+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:9999999 --scheduler rr --persistent 3000"
+ _, err := checkIPVSRule(addServiceCommand)
+ assert.Error(err)
+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:9999999 -s rr -p 3000"
+ _, err = checkIPVSRule(addServiceCmd)
+ assert.Error(err)
+}
+
+func TestAddServiceSchedulerError(t *testing.T) {
+ assert := assert.New(t)
+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --scheduler rrr --persistent 3000"
+ _, err := checkIPVSRule(addServiceCommand)
+ assert.Error(err)
+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rrr -p 3000"
+ _, err = checkIPVSRule(addServiceCmd)
+ assert.Error(err)
+}
+
+func TestAddServicePersistentError(t *testing.T) {
+ assert := assert.New(t)
+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --scheduler rr --persistent 99999999999"
+ _, err := checkIPVSRule(addServiceCommand)
+ assert.Error(err)
+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rr -p 99999999999"
+ _, err = checkIPVSRule(addServiceCmd)
+ assert.Error(err)
+}
+
+func TestEditServiceSuccessfully(t *testing.T) {
+ assert := assert.New(t)
+ editServiceCommandTCP := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --scheduler rr --persistent 3000"
+ _, err := checkIPVSRule(editServiceCommandTCP)
+ assert.NoError(err)
+ editServiceCommandUcp := "IPVSadm --edit-service --udp-service 192.168.0.7:80 --scheduler rr --persistent 3000"
+ _, err = checkIPVSRule(editServiceCommandUcp)
+ assert.NoError(err)
+ editServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rr -p 3000"
+ _, err = checkIPVSRule(editServiceCmd)
+ assert.NoError(err)
+}
+
+func TestEditServiceNoService(t *testing.T) {
+ assert := assert.New(t)
+ editServiceCommand := "IPVSadm --edit-service --scheduler rr --persistent 3000"
+ _, err := checkIPVSRule(editServiceCommand)
+ assert.Error(err)
+ editServiceCmd := "IPVSadm -E -s rr -p 3000"
+ _, err = checkIPVSRule(editServiceCmd)
+ assert.Error(err)
+}
+
+func TestEditServiceNoScheduler(t *testing.T) {
+ assert := assert.New(t)
+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --persistent 3000"
+ _, err := checkIPVSRule(editServiceCommand)
+ assert.Error(err)
+ editServiceCmd := "IPVSadm -E -t 192.168.0.7:80 -p 3000"
+ _, err = checkIPVSRule(editServiceCmd)
+ assert.Error(err)
+}
+
+func TestEditServiceNoPersistent(t *testing.T) {
+ assert := assert.New(t)
+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --scheduler rr"
+ _, err := checkIPVSRule(editServiceCommand)
+ assert.Error(err)
+ editServiceCmd := "IPVSadm -E -t 192.168.0.7:80 -s rr"
+ _, err = checkIPVSRule(editServiceCmd)
+ assert.Error(err)
+}
+
+func TestEditServiceIpError(t *testing.T) {
+ assert := assert.New(t)
+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.2.2.7:80 --scheduler rr --persistent 3000"
+ _, err := checkIPVSRule(editServiceCommand)
+ assert.Error(err)
+ editServiceCmd := "IPVSadm -E -t 192.168.2.2.7:80 -s rr -p 3000"
+ _, err = checkIPVSRule(editServiceCmd)
+ assert.Error(err)
+}
+
+func TestEditServicePortError(t *testing.T) {
+ assert := assert.New(t)
+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:9999999 --scheduler rr --persistent 3000"
+ _, err := checkIPVSRule(editServiceCommand)
+ assert.Error(err)
+ editServiceCmd := "IPVSadm -E -t 192.168.0.7:9999999 -s rr -p 3000"
+ _, err = checkIPVSRule(editServiceCmd)
+ assert.Error(err)
+}
+
+func TestEditServiceSchedulerError(t *testing.T) {
+ assert := assert.New(t)
+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --scheduler rrr --persistent 3000"
+ _, err := checkIPVSRule(editServiceCommand)
+ assert.Error(err)
+ editServiceCmd := "IPVSadm -E -t 192.168.0.7:80 -s rrr -p 3000"
+ _, err = checkIPVSRule(editServiceCmd)
+ assert.Error(err)
+}
+
+func TestEditServicePersistentError(t *testing.T) {
+ assert := assert.New(t)
+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --scheduler rr --persistent 99999999999"
+ _, err := checkIPVSRule(editServiceCommand)
+ assert.Error(err)
+ editServiceCmd := "IPVSadm -E -t 192.168.0.7:80 -s rr -p 99999999999"
+ _, err = checkIPVSRule(editServiceCmd)
+ assert.Error(err)
+}
+
+func TestDeleteServiceSuccessfully(t *testing.T) {
+ assert := assert.New(t)
+ deleteServiceCommandTCP := "IPVSadm --delete-service --tcp-service 192.168.0.7:80"
+ _, err := checkIPVSRule(deleteServiceCommandTCP)
+ assert.NoError(err)
+ deleteServiceCommandUDP := "IPVSadm --delete-service --udp-service 192.168.0.7:80"
+ _, err = checkIPVSRule(deleteServiceCommandUDP)
+ assert.NoError(err)
+ deleteServiceCmd := "IPVSadm -D -t 192.168.0.7:80"
+ _, err = checkIPVSRule(deleteServiceCmd)
+ assert.NoError(err)
+}
+
+func TestDeleteServiceNoService(t *testing.T) {
+ assert := assert.New(t)
+ deleteServiceCommand := "IPVSadm --delete-service"
+ _, err := checkIPVSRule(deleteServiceCommand)
+ assert.Error(err)
+ deleteServiceCmd := "IPVSadm -D"
+ _, err = checkIPVSRule(deleteServiceCmd)
+ assert.Error(err)
+}
+
+func TestDeleteServiceIpError(t *testing.T) {
+ assert := assert.New(t)
+ deleteServiceCommand := "IPVSadm --delete-service --tcp-service 192.168.2.2.7:80"
+ _, err := checkIPVSRule(deleteServiceCommand)
+ assert.Error(err)
+ deleteServiceCmd := "IPVSadm -D -t 192.168.2.2.7:80"
+ _, err = checkIPVSRule(deleteServiceCmd)
+ assert.Error(err)
+}
+
+func TestDeleteServicePortError(t *testing.T) {
+ assert := assert.New(t)
+ deleteServiceCommand := "IPVSadm --delete-service --tcp-service 192.168.0.7:9999999"
+ _, err := checkIPVSRule(deleteServiceCommand)
+ assert.Error(err)
+ deleteServiceCmd := "IPVSadm -D -t 192.168.0.7:9999999"
+ _, err = checkIPVSRule(deleteServiceCmd)
+ assert.Error(err)
+}
+
+func TestAddServerSuccessfully(t *testing.T) {
+ assert := assert.New(t)
+ addServerCommandTCP := "IPVSadm --add-server --tcp-service 192.168.0.7:80 --real-server 192.168.0.4:80 --weight 100"
+ _, err := checkIPVSRule(addServerCommandTCP)
+ assert.NoError(err)
+ addServerCommandUDP := "IPVSadm --add-server --udp-service 192.168.0.7:80 --real-server 192.168.0.4:80 --weight 100"
+ _, err = checkIPVSRule(addServerCommandUDP)
+ assert.NoError(err)
+ addServerCmd := "IPVSadm -a -t 192.168.0.7:80 -r 192.168.0.4:80 -w 100"
+ _, err = checkIPVSRule(addServerCmd)
+ assert.NoError(err)
+}
+
+func TestAddServerNoService(t *testing.T) {
+ assert := assert.New(t)
+ addServerCommand := "IPVSadm --add-server --real-server 192.168.0.4:80"
+ _, err := checkIPVSRule(addServerCommand)
+ assert.Error(err)
+ addServerCmd := "IPVSadm -a -r 192.168.0.4:80"
+ _, err = checkIPVSRule(addServerCmd)
+ assert.Error(err)
+}
+
+func TestAddServerNoServer(t *testing.T) {
+ assert := assert.New(t)
+ addServerCommand := "IPVSadm --add-server --tcp-service 192.168.0.7:80"
+ _, err := checkIPVSRule(addServerCommand)
+ assert.Error(err)
+ addServerCmd := "IPVSadm -a -t 192.168.0.7:80"
+ _, err = checkIPVSRule(addServerCmd)
+ assert.Error(err)
+}
+
+func TestAddServerIpError(t *testing.T) {
+ assert := assert.New(t)
+ addServerCommand := "IPVSadm --add-server --tcp-service 192.168.2.0.7:80 --real-server 192.168.0.4:80"
+ _, err := checkIPVSRule(addServerCommand)
+ assert.Error(err)
+ addServerCmd := "IPVSadm -a -t 192.168.2.0.7:80 -r 192.168.0.4:80"
+ _, err = checkIPVSRule(addServerCmd)
+ assert.Error(err)
+}
+
+func TestAddServerPortError(t *testing.T) {
+ assert := assert.New(t)
+ addServerCommand := "IPVSadm --add-server --tcp-service 192.168.0.7:99999 --real-server 192.168.0.4:80"
+ _, err := checkIPVSRule(addServerCommand)
+ assert.Error(err)
+ addServerCmd := "IPVSadm -a -t 192.168.0.7:99999 -r 192.168.0.4:80"
+ _, err = checkIPVSRule(addServerCmd)
+ assert.Error(err)
+}
+
+func TestEditServerSuccessfully(t *testing.T) {
+ assert := assert.New(t)
+ editServerCommandTCP := "IPVSadm --edit-server --tcp-service 192.168.0.7:80 --real-server 192.168.0.4:80 --weight 100"
+ _, err := checkIPVSRule(editServerCommandTCP)
+ assert.NoError(err)
+ editServerCommandUDP := "IPVSadm --edit-server --udp-service 192.168.0.7:80 --real-server 192.168.0.4:80 --weight 100"
+ _, err = checkIPVSRule(editServerCommandUDP)
+ assert.NoError(err)
+ editServerCmd := "IPVSadm -e -t 192.168.0.7:80 -r 192.168.0.4:80 -w 100"
+ _, err = checkIPVSRule(editServerCmd)
+ assert.NoError(err)
+}
+
+func TestEditServerNoService(t *testing.T) {
+ assert := assert.New(t)
+ editServerCommand := "IPVSadm --edit-server --real-server 192.168.0.4:80"
+ _, err := checkIPVSRule(editServerCommand)
+ assert.Error(err)
+ editServerCmd := "IPVSadm -e -r 192.168.0.4:80"
+ _, err = checkIPVSRule(editServerCmd)
+ assert.Error(err)
+}
+
+func TestEditServerNoServer(t *testing.T) {
+ assert := assert.New(t)
+ editServerCommand := "IPVSadm --edit-server --tcp-service 192.168.0.7:80"
+ _, err := checkIPVSRule(editServerCommand)
+ assert.Error(err)
+ editServerCmd := "IPVSadm -e -t 192.168.0.7:80"
+ _, err = checkIPVSRule(editServerCmd)
+ assert.Error(err)
+}
+
+func TestEditServerIpError(t *testing.T) {
+ assert := assert.New(t)
+ editServerCommand := "IPVSadm --edit-server --tcp-service 192.168.2.0.7:80 --real-server 192.168.0.4:80"
+ _, err := checkIPVSRule(editServerCommand)
+ assert.Error(err)
+ editServerCmd := "IPVSadm -e -t 192.168.2.0.7:80 -r 192.168.0.4:80"
+ _, err = checkIPVSRule(editServerCmd)
+ assert.Error(err)
+}
+
+func TestEditServerPortError(t *testing.T) {
+ assert := assert.New(t)
+ editServerCommand := "IPVSadm --edit-server --tcp-service 192.168.0.7:99999 --real-server 192.168.0.4:80"
+ _, err := checkIPVSRule(editServerCommand)
+ assert.Error(err)
+ editServerCmd := "IPVSadm -e -t 192.168.0.7:99999 -r 192.168.0.4:80"
+ _, err = checkIPVSRule(editServerCmd)
+ assert.Error(err)
+}
+
+func TestDeleteServerSuccessfully(t *testing.T) {
+ assert := assert.New(t)
+ deleteServerCommandTCP := "IPVSadm --delete-server --tcp-service 192.168.0.7:80 --real-server 192.168.0.4:80"
+ _, err := checkIPVSRule(deleteServerCommandTCP)
+ assert.NoError(err)
+ deleteServerCommandUDP := "IPVSadm --delete-server --udp-service 192.168.0.7:80 --real-server 192.168.0.4:80"
+ _, err = checkIPVSRule(deleteServerCommandUDP)
+ assert.NoError(err)
+ deleteServerCmd := "IPVSadm -d -t 192.168.0.7:80 -r 192.168.0.4:80"
+ _, err = checkIPVSRule(deleteServerCmd)
+ assert.NoError(err)
+}
+
+func TestDeleteServerNoService(t *testing.T) {
+ assert := assert.New(t)
+ deleteServerCommand := "IPVSadm --delete-server --real-server 192.168.0.4:80"
+ _, err := checkIPVSRule(deleteServerCommand)
+ assert.Error(err)
+ deleteServerCmd := "IPVSadm -d -r 192.168.0.4:80"
+ _, err = checkIPVSRule(deleteServerCmd)
+ assert.Error(err)
+}
+
+func TestDeleteServerNoServer(t *testing.T) {
+ assert := assert.New(t)
+ deleteServerCommand := "IPVSadm --delete-server --tcp-service 192.168.0.7:80"
+ _, err := checkIPVSRule(deleteServerCommand)
+ assert.Error(err)
+ deleteServerCmd := "IPVSadm -d -t 192.168.0.7:80"
+ _, err = checkIPVSRule(deleteServerCmd)
+ assert.Error(err)
+}
+
+func TestDeleteServerIpError(t *testing.T) {
+ assert := assert.New(t)
+ deleteServerCommand := "IPVSadm --delete-server --tcp-service 192.168.2.0.7:80 --real-server 192.168.0.4:80"
+ _, err := checkIPVSRule(deleteServerCommand)
+ assert.Error(err)
+ deleteServerCmd := "IPVSadm -d -t 192.168.2.0.7:80 -r 192.168.0.4:80"
+ _, err = checkIPVSRule(deleteServerCmd)
+ assert.Error(err)
+}
+
+func TestDeleteServerPortError(t *testing.T) {
+ assert := assert.New(t)
+ deleteServerCommand := "IPVSadm --delete-server --tcp-service 192.168.0.7:99999 --real-server 192.168.0.4:80"
+ _, err := checkIPVSRule(deleteServerCommand)
+ assert.Error(err)
+ deleteServerCmd := "IPVSadm -d -t 192.168.0.7:99999 -r 192.168.0.4:80"
+ _, err = checkIPVSRule(deleteServerCmd)
+ assert.Error(err)
+}
+
+func TestListSuccessfully(t *testing.T) {
+ assert := assert.New(t)
+ listCommand := "IPVSadm --list"
+ _, err := checkIPVSRule(listCommand)
+ assert.NoError(err)
+ listCmd := "IPVSadm -L"
+ _, err = checkIPVSRule(listCmd)
+ assert.NoError(err)
+}
+
+func TestSetSuccessfully(t *testing.T) {
+ assert := assert.New(t)
+ setCommand := "IPVSadm --set"
+ _, err := checkIPVSRule(setCommand)
+ assert.NoError(err)
+}
+
+func TestCleanupSuccessfully(t *testing.T) {
+ assert := assert.New(t)
+ cleanupCommand := "conntrack -D --orig-dst 192.168.0.4 --protonum tcp"
+ _, err := checkIPVSRule(cleanupCommand)
+ assert.NoError(err)
+ cleanupCmd := "conntrack -D -d 192.168.0.4 -p tcp"
+ _, err = checkIPVSRule(cleanupCmd)
+ assert.NoError(err)
+}
+
+func TestCleanupNoDestination(t *testing.T) {
+ assert := assert.New(t)
+ cleanupCommand := "conntrack -D --protonum tcp"
+ _, err := checkIPVSRule(cleanupCommand)
+ assert.Error(err)
+ cleanupCmd := "conntrack -D -p tcp"
+ _, err = checkIPVSRule(cleanupCmd)
+ assert.Error(err)
+}
+
+func TestCleanupNoProtocol(t *testing.T) {
+ assert := assert.New(t)
+ cleanupCommand := "conntrack -D --orig-dst 192.168.0.4"
+ _, err := checkIPVSRule(cleanupCommand)
+ assert.Error(err)
+ cleanupCmd := "conntrack -D -d 192.168.0.4"
+ _, err = checkIPVSRule(cleanupCmd)
+ assert.Error(err)
+}
+
+func TestCleanupIpError(t *testing.T) {
+ assert := assert.New(t)
+ cleanupCommand := "conntrack -D --orig-dst 192.168.0.2.4 --protonum tcp"
+ _, err := checkIPVSRule(cleanupCommand)
+ assert.Error(err)
+ cleanupCmd := "conntrack -D -d 192.168.0.2.4 -p tcp"
+ _, err = checkIPVSRule(cleanupCmd)
+ assert.Error(err)
+}
+
+func TestCleanupProtocolError(t *testing.T) {
+ assert := assert.New(t)
+ cleanupCommand := "conntrack -D --orig-dst 192.168.0.4 --protonum ttcp"
+ _, err := checkIPVSRule(cleanupCommand)
+ assert.Error(err)
+ cleanupCmd := "conntrack -D -d 192.168.0.4 -p ttcp"
+ _, err = checkIPVSRule(cleanupCmd)
+ assert.Error(err)
+}
+
+func TestRestoreFileNil(t *testing.T) {
+ assert := assert.New(t)
+ stdin := strings.NewReader("")
+ fileContext, cnt, err := getRestoreFile(stdin)
+ assert.Equal(cnt, 0)
+ assert.Equal(fileContext, "")
+ assert.Error(err)
+}
+
+func TestRestoreFileError(t *testing.T) {
+ assert := assert.New(t)
+ stdin := strings.NewReader("aaa")
+ fileContext, cnt, err := getRestoreFile(stdin)
+ assert.Equal(cnt, 0)
+ assert.Equal(fileContext, "")
+ assert.Error(err)
+}
+
+func TestRestoreFile(t *testing.T) {
+ assert := assert.New(t)
+ stdin := strings.NewReader("-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m")
+ fileContext, cnt, err := getRestoreFile(stdin)
+ assert.Equal(cnt, 2)
+ assert.Equal(fileContext, "-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m")
+ assert.NoError(err)
+}
diff --git a/virtcontainers/api_test.go b/virtcontainers/api_test.go
index c01d47b8..2a5488f9 100644
--- a/virtcontainers/api_test.go
+++ b/virtcontainers/api_test.go
@@ -15,6 +15,7 @@ import (
"syscall"
"testing"
+ "github.com/kata-containers/agent/protocols/grpc"
ktu "github.com/kata-containers/runtime/pkg/katatestutils"
"github.com/kata-containers/runtime/virtcontainers/persist"
"github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
@@ -1745,3 +1746,38 @@ func TestCleanupContainer(t *testing.T) {
t.Fatal(err)
}
}
+
+func TestUpdateIPVSRule(t *testing.T) {
+ defer cleanUp()
+ assert := assert.New(t)
+
+ contID := "abc"
+ ctx := context.Background()
+ var ipvs = &grpc.UpdateIPVSRequest{
+ IPVSReq: "ipvsadm -A -t 192.168.0.7:80 -s rr -p -3000",
+ }
+
+ config := newTestSandboxConfigNoop()
+ p, _, err := createAndStartSandbox(ctx, config)
+ if p == nil || err != nil {
+ t.Fatal(err)
+ }
+
+ s, ok := p.(*Sandbox)
+ assert.True(ok)
+
+ contConfig := newTestContainerConfigNoop(contID)
+ c, err := p.CreateContainer(contConfig)
+ if c == nil || err != nil {
+ t.Fatal(err)
+ }
+
+ _, err = UpdateIPVSRule(ctx, s.id, ipvs)
+ assert.NoError(err)
+
+ _, err = UpdateIPVSRule(ctx, "aaa", ipvs)
+ assert.Error(err)
+
+ _, err = UpdateIPVSRule(ctx, "", ipvs)
+ assert.Error(err)
+}
diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go
index 18a5a0a6..4f9409a0 100644
--- a/virtcontainers/kata_agent_test.go
+++ b/virtcontainers/kata_agent_test.go
@@ -246,6 +246,10 @@ func (p *gRPCProxy) MemHotplugByProbe(ctx context.Context, req *pb.MemHotplugByP
return &gpb.Empty{}, nil
}
+func (p *gRPCProxy) UpdateIPVSRule(ctx context.Context, req *pb.UpdateIPVSRequest) (*pb.IPVSResponse, error) {
+ return &pb.IPVSResponse{}, nil
+}
+
func gRPCRegister(s *grpc.Server, srv interface{}) {
switch g := srv.(type) {
case *gRPCProxy:
diff --git a/virtcontainers/pkg/vcmock/types.go b/virtcontainers/pkg/vcmock/types.go
index 43247ef9..610b4602 100644
--- a/virtcontainers/pkg/vcmock/types.go
+++ b/virtcontainers/pkg/vcmock/types.go
@@ -76,5 +76,5 @@ type VCMock struct {
UpdateRoutesFunc func(ctx context.Context, sandboxID string, routes []*vcTypes.Route) ([]*vcTypes.Route, error)
ListRoutesFunc func(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error)
CleanupContainerFunc func(ctx context.Context, sandboxID, containerID string, force bool) error
- UpdateIPVSRuleFunc func(ctx context.Context, sandboxID string, ipvs *grpc.UpdateIpvsRequest) (*grpc.IpvsResponse, error)
+ UpdateIPVSRuleFunc func(ctx context.Context, sandboxID string, ipvs *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error)
}
--
2.14.3 (Apple Git-98)

501
runtime/patches/0038-pcie-using-pcie-root-port-driver-to-hotplug-device-i.patch Normal file
View File

@ -0,0 +1,501 @@
From 5a220e9be1cfb03316a62aa00d2040638ba1a855 Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Tue, 18 Aug 2020 15:15:52 +0800
Subject: [PATCH 38/50] pcie: using pcie-root-port driver to hotplug device in
aarch64
reason: Since qemu with "virt" machine type doesn't support hotplug
device in the pcie.0 root bus, so need to use add root port devices
to support hotplug pci device in aarch64.
we reuse the pcie_root_port config in the configuration.toml file to
set pcie_root_port device number when qemu process start.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
vendor/github.com/intel/govmm/qemu/qemu.go | 22 ++---
virtcontainers/persist/api/hypervisor.go | 5 +-
virtcontainers/qemu.go | 130 +++++++++++++++++++++++------
virtcontainers/qemu_arch_base.go | 9 ++
virtcontainers/types/pcie.go | 103 +++++++++++++++++++++++
5 files changed, 230 insertions(+), 39 deletions(-)
create mode 100644 virtcontainers/types/pcie.go
diff --git a/vendor/github.com/intel/govmm/qemu/qemu.go b/vendor/github.com/intel/govmm/qemu/qemu.go
index a5e5dfaf..3e7720b4 100644
--- a/vendor/github.com/intel/govmm/qemu/qemu.go
+++ b/vendor/github.com/intel/govmm/qemu/qemu.go
@@ -1252,6 +1252,8 @@ type PCIeRootPortDevice struct {
Chassis string // (slot, chassis) pair is mandatory and must be unique for each pcie-root-port, >=0, default is 0x00
Slot string // >=0, default is 0x00
+ Port string // Port number is the device index
+
Multifunction bool // true => "on", false => "off", default is off
Addr string // >=0, default is 0x00
@@ -1277,6 +1279,10 @@ func (b PCIeRootPortDevice) QemuParams(config *Config) []string {
deviceParams = append(deviceParams, fmt.Sprintf("%s,id=%s", driver, b.ID))
+ if b.Port != "" {
+ deviceParams = append(deviceParams, fmt.Sprintf("port=%s", b.Port))
+ }
+
if b.Bus == "" {
b.Bus = "pcie.0"
}
@@ -1287,20 +1293,10 @@ func (b PCIeRootPortDevice) QemuParams(config *Config) []string {
}
deviceParams = append(deviceParams, fmt.Sprintf("chassis=%s", b.Chassis))
- if b.Slot == "" {
- b.Slot = "0x00"
- }
- deviceParams = append(deviceParams, fmt.Sprintf("slot=%s", b.Slot))
-
- multifunction := "off"
if b.Multifunction {
- multifunction = "on"
- if b.Addr == "" {
- b.Addr = "0x00"
- }
- deviceParams = append(deviceParams, fmt.Sprintf("addr=%s", b.Addr))
+ deviceParams = append(deviceParams, "multifunction=on")
}
- deviceParams = append(deviceParams, fmt.Sprintf("multifunction=%v", multifunction))
+ deviceParams = append(deviceParams, fmt.Sprintf("addr=%s", b.Addr))
if b.BusReserve != "" {
deviceParams = append(deviceParams, fmt.Sprintf("bus-reserve=%s", b.BusReserve))
@@ -1337,7 +1333,7 @@ func (b PCIeRootPortDevice) Valid() bool {
if b.Pref64Reserve != "" && b.Pref32Reserve != "" {
return false
}
- if b.ID == "" {
+ if b.ID == "" || b.Port == "" || b.Bus == "" || b.Addr == ""{
return false
}
return true
diff --git a/virtcontainers/persist/api/hypervisor.go b/virtcontainers/persist/api/hypervisor.go
index 375fd56b..fd61b3c2 100644
--- a/virtcontainers/persist/api/hypervisor.go
+++ b/virtcontainers/persist/api/hypervisor.go
@@ -5,6 +5,8 @@
package persistapi
+import "github.com/kata-containers/runtime/virtcontainers/types"
+
// Bridge is a bridge where devices can be hot plugged
type Bridge struct {
// DeviceAddr contains information about devices plugged and its address in the bridge
@@ -35,7 +37,8 @@ type HypervisorState struct {
// Belows are qemu specific
// Refs: virtcontainers/qemu.go:QemuState
- Bridges []Bridge
+ Bridges []Bridge
+ PCIeRootPortsPool *types.PCIeRootPortPool
// HotpluggedCPUs is the list of CPUs that were hot-added
HotpluggedVCPUs []CPUDevice
HotpluggedMemory int
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
index c2b65376..a10c66fb 100644
--- a/virtcontainers/qemu.go
+++ b/virtcontainers/qemu.go
@@ -64,8 +64,9 @@ type CPUDevice struct {
// QemuState keeps Qemu's state
type QemuState struct {
- Bridges []types.Bridge
- ScsiBus *types.SCSIBus
+ Bridges []types.Bridge
+ ScsiBus *types.SCSIBus
+ PCIeRootPortsPool *types.PCIeRootPortPool
// HotpluggedCPUs is the list of CPUs that were hot-added
HotpluggedVCPUs []CPUDevice
HotpluggedMemory int
@@ -271,6 +272,9 @@ func (q *qemu) setup(id string, hypervisorConfig *HypervisorConfig) error {
q.state.HotplugVFIOOnRootBus = q.config.HotplugVFIOOnRootBus
q.state.PCIeRootPort = int(q.config.PCIeRootPort)
+ // init the PCIeRootPortsPool with pcie_root_port config value
+ q.state.PCIeRootPortsPool = &types.PCIeRootPortPool{}
+ q.state.PCIeRootPortsPool.Init(q.state.PCIeRootPort)
// The path might already exist, but in case of VM templating,
// we have to create it since the sandbox has not created it yet.
@@ -394,9 +398,18 @@ func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.I
return nil, nil, err
}
- // Add bridges before any other devices. This way we make sure that
- // bridge gets the first available PCI address i.e bridgePCIStartAddr
- devices = q.arch.appendBridges(devices)
+ machine, err := q.getQemuMachine()
+ if err != nil {
+ return nil, nil, err
+ }
+ switch machine.Type {
+ case QemuVirt:
+ devices = q.arch.appendRootPorts(devices, q.state.PCIeRootPortsPool)
+ default:
+ // Add bridges before any other devices. This way we make sure that
+ // bridge gets the first available PCI address i.e bridgePCIStartAddr
+ devices = q.arch.appendBridges(devices)
+ }
devices, err = q.arch.appendConsole(devices, console)
if err != nil {
@@ -608,7 +621,7 @@ func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNa
// Add PCIe Root Port devices to hypervisor
// The pcie.0 bus do not support hot-plug, but PCIe device can be hot-plugged into PCIe Root Port.
// For more details, please see https://github.com/qemu/qemu/blob/master/docs/pcie.txt
- if hypervisorConfig.PCIeRootPort > 0 {
+ if hypervisorConfig.PCIeRootPort > 0 && hypervisorConfig.HypervisorMachineType == QemuQ35 {
qemuConfig.Devices = q.arch.appendPCIeRootPortDevice(qemuConfig.Devices, hypervisorConfig.PCIeRootPort)
}
@@ -1154,21 +1167,19 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev
}
case q.config.BlockDeviceDriver == config.VirtioBlock:
driver := "virtio-blk-pci"
- addr, bridge, err := q.arch.addDeviceToBridge(drive.ID, types.PCI)
+
+ addr, bus, pciAddr, err := q.getPciAddress(drive.ID, types.PCI)
if err != nil {
return err
}
-
defer func() {
if err != nil {
- q.arch.removeDeviceFromBridge(drive.ID)
+ q.putPciAddress(drive.ID)
}
}()
+ drive.PCIAddr = pciAddr
- // PCI address is in the format bridge-addr/device-addr eg. "03/02"
- drive.PCIAddr = fmt.Sprintf("%02x", bridge.Addr) + "/" + addr
-
- if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID, romFile, 0, true, defaultDisableModern); err != nil {
+ if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bus, romFile, 0, true, defaultDisableModern); err != nil {
return err
}
case q.config.BlockDeviceDriver == config.VirtioSCSI:
@@ -1249,7 +1260,7 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error
err = q.hotplugAddBlockDevice(drive, op, devID)
} else {
if q.config.BlockDeviceDriver == config.VirtioBlock {
- if err := q.arch.removeDeviceFromBridge(drive.ID); err != nil {
+ if err := q.putPciAddress(drive.ID); err != nil {
return err
}
} else if q.config.BlockDeviceDriver == config.VirtioSCSI {
@@ -1345,22 +1356,22 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro
}
}
- addr, bridge, err := q.arch.addDeviceToBridge(devID, types.PCI)
+ addr, bus, _, err := q.getPciAddress(devID, types.PCI)
if err != nil {
return err
}
defer func() {
if err != nil {
- q.arch.removeDeviceFromBridge(devID)
+ q.putPciAddress(devID)
}
}()
switch device.Type {
case config.VFIODeviceNormalType:
- return q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bridge.ID, romFile)
+ return q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bus, romFile)
case config.VFIODeviceMediatedType:
- return q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, addr, bridge.ID, romFile)
+ return q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, addr, bus, romFile)
default:
return fmt.Errorf("Incorrect VFIO device type found")
}
@@ -1368,7 +1379,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro
q.Logger().WithField("dev-id", devID).Info("Start hot-unplug VFIO device")
if !q.state.HotplugVFIOOnRootBus {
- if err := q.arch.removeDeviceFromBridge(devID); err != nil {
+ if err := q.putPciAddress(devID); err != nil {
return err
}
}
@@ -1439,18 +1450,17 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
}
}()
- addr, bridge, err := q.arch.addDeviceToBridge(tap.ID, types.PCI)
+ addr, bus, pciAddr, err := q.getPciAddress(tap.ID, types.PCI)
if err != nil {
return err
}
defer func() {
if err != nil {
- q.arch.removeDeviceFromBridge(tap.ID)
+ q.putPciAddress(tap.ID)
}
}()
- pciAddr := fmt.Sprintf("%02x/%s", bridge.Addr, addr)
endpoint.SetPciAddr(pciAddr)
var machine govmmQemu.Machine
@@ -1459,14 +1469,14 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
return err
}
if machine.Type == QemuCCWVirtio {
- devNoHotplug := fmt.Sprintf("fe.%x.%x", bridge.Addr, addr)
+ devNoHotplug := fmt.Sprintf("fe.%x.%x", bus, addr)
return q.qmpMonitorCh.qmp.ExecuteNetCCWDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), devNoHotplug, int(q.config.NumVCPUs))
}
- return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bridge.ID, romFile, int(q.config.NumVCPUs), defaultDisableModern)
+ return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bus, romFile, int(q.config.NumVCPUs), defaultDisableModern)
}
- if err := q.arch.removeDeviceFromBridge(tap.ID); err != nil {
+ if err := q.putPciAddress(tap.ID); err != nil {
return err
}
@@ -2042,6 +2052,21 @@ func genericMemoryTopology(memoryMb, hostMemoryMb uint64, slots uint8, memoryOff
return memory
}
+func genericAppendRootPorts(devices []govmmQemu.Device, rootPorts *types.PCIeRootPortPool) []govmmQemu.Device {
+ for _, rp := range rootPorts.Items {
+ devices = append(devices, govmmQemu.PCIeRootPortDevice{
+ Port: rp.Port,
+ Bus: rp.Bus,
+ ID: rp.ID,
+ Chassis: strconv.Itoa(int(rp.Chassis)),
+ Multifunction: rp.Multifunction,
+ Addr: fmt.Sprintf("0x%s.0x%s", rp.Slot, rp.Function),
+ })
+ }
+
+ return devices
+}
+
// genericAppendPCIeRootPort appends to devices the given pcie-root-port
func genericAppendPCIeRootPort(devices []govmmQemu.Device, number uint32, machineType string) []govmmQemu.Device {
var (
@@ -2241,6 +2266,7 @@ func (q *qemu) save() (s persistapi.HypervisorState) {
s.HotpluggedMemory = q.state.HotpluggedMemory
s.HotplugVFIOOnRootBus = q.state.HotplugVFIOOnRootBus
s.PCIeRootPort = q.state.PCIeRootPort
+ s.PCIeRootPortsPool = q.state.PCIeRootPortsPool
for _, bridge := range q.arch.getBridges() {
s.Bridges = append(s.Bridges, persistapi.Bridge{
@@ -2265,6 +2291,7 @@ func (q *qemu) load(s persistapi.HypervisorState) {
q.state.HotplugVFIOOnRootBus = s.HotplugVFIOOnRootBus
q.state.VirtiofsdPid = s.VirtiofsdPid
q.state.PCIeRootPort = s.PCIeRootPort
+ q.state.PCIeRootPortsPool = s.PCIeRootPortsPool
for _, bridge := range s.Bridges {
q.state.Bridges = append(q.state.Bridges, types.NewBridge(types.Type(bridge.Type), bridge.ID, bridge.DeviceAddr, bridge.Addr))
@@ -2302,3 +2329,56 @@ func (q *qemu) generateSocket(id string, useVsock bool) (interface{}, error) {
func (q *qemu) getMemorySize() uint32 {
return q.config.MemorySize + uint32(q.state.HotpluggedMemory)
}
+
+// getPciAddress allocate the pci slot to hotplugged device and
+// return the pci slot address
+func (q *qemu) getPciAddress(devID string, t types.Type) (slot, bus, pciAddr string, err error) {
+ machine, err := q.getQemuMachine()
+ if err != nil {
+ return "", "", "", err
+ }
+
+ switch machine.Type {
+ case QemuVirt:
+ rp, err := q.state.PCIeRootPortsPool.AddDevice(devID)
+ if err != nil {
+ return "", "", "", err
+ }
+ // PCIe Root Port only have one slot
+ slot = "0x0"
+ // pciAddr specifies the slot and function of the Root Port and the slot of the device
+ pciAddr = fmt.Sprintf("%s.%s/00", rp.Slot, rp.Function)
+ bus = rp.ID
+ default:
+ var bridge types.Bridge
+ slot, bridge, err = q.arch.addDeviceToBridge(devID, t)
+ if err != nil {
+ return "", "", "", err
+ }
+ bus = bridge.ID
+ // PCI address is in the format bridge-addr.0/device-addr eg. "03.0/02"
+ pciAddr = fmt.Sprintf("%02x.0", bridge.Addr) + "/" + slot
+ }
+ return slot, bus, pciAddr, nil
+}
+
+func (q *qemu) putPciAddress(devID string) error {
+ machine, err := q.getQemuMachine()
+ if err != nil {
+ return err
+ }
+
+ switch machine.Type {
+ case QemuVirt:
+ err := q.state.PCIeRootPortsPool.RemoveDevice(devID)
+ if err != nil {
+ return err
+ }
+ default:
+ if err := q.arch.removeDeviceFromBridge(devID); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
diff --git a/virtcontainers/qemu_arch_base.go b/virtcontainers/qemu_arch_base.go
index 9d72dd09..cb045530 100644
--- a/virtcontainers/qemu_arch_base.go
+++ b/virtcontainers/qemu_arch_base.go
@@ -130,6 +130,9 @@ type qemuArch interface {
// appendPCIeRootPortDevice appends a pcie-root-port device to pcie.0 bus
appendPCIeRootPortDevice(devices []govmmQemu.Device, number uint32) []govmmQemu.Device
+
+ // appendRootPorts appends a pcie-root-port device to devices when qemu machine type is "virt"
+ appendRootPorts(devices []govmmQemu.Device, rootPorts *types.PCIeRootPortPool) []govmmQemu.Device
}
type qemuArchBase struct {
@@ -766,3 +769,9 @@ func (q *qemuArchBase) addBridge(b types.Bridge) {
func (q *qemuArchBase) appendPCIeRootPortDevice(devices []govmmQemu.Device, number uint32) []govmmQemu.Device {
return genericAppendPCIeRootPort(devices, number, q.machineType)
}
+
+// appendRootPorts appends a pcie-root-port device to devices when qemu machine type is "virt"
+// which is different appendPCIeRootPortDevice function
+func (q *qemuArchBase) appendRootPorts(devices []govmmQemu.Device, rootPorts *types.PCIeRootPortPool) []govmmQemu.Device {
+ return genericAppendRootPorts(devices, rootPorts)
+}
diff --git a/virtcontainers/types/pcie.go b/virtcontainers/types/pcie.go
new file mode 100644
index 00000000..83eb6944
--- /dev/null
+++ b/virtcontainers/types/pcie.go
@@ -0,0 +1,103 @@
+package types
+
+import (
+ "fmt"
+)
+
+const (
+ maxRootPortsCapacity = 25
+ slotPerDevice = 7
+
+ // PCIeRootBus is "pcie.0"
+ PCIeRootBus = "pcie.0"
+
+ // startPort specifies the start port of pcie-root-port
+ // for the first slot of "pcie.0" is reserved, so the 1~7 ports are also reserved
+ startPort = 8
+ funcNumPerSlot = 8
+)
+
+// PCIeRootPort describe the PCIe Root Port
+type PCIeRootPort struct {
+ // DeviceID specify the device hotplug on the Root Port
+ DeviceID string
+
+ // Port number is the Root Port index
+ Port string
+
+ // Bus number where the Root Port is plugged, typically pcie.0
+ Bus string
+
+ // ID is used to identify the pcie-root-port in qemu
+ ID string
+
+ // Slot specifies slot address of Root Port
+ Slot string
+
+ // Function specifies function of Root Port
+ Function string
+
+ // Chassis number
+ Chassis uint32
+
+ // Multifunction is used to specify the pcie-root-port is multifunction supported
+ Multifunction bool
+}
+
+// PCIeRootPortPool describe a set of PCIe Root Ports
+type PCIeRootPortPool struct {
+ // Items contains information about devices plugged and number limit
+ Items []*PCIeRootPort
+}
+
+// Init Initialized the PCIeRootPortPool instance
+func (rp *PCIeRootPortPool) Init(number int) {
+ if number == 0 || number > maxRootPortsCapacity {
+ number = maxRootPortsCapacity
+ }
+
+ for i := 0; i < number; i++ {
+ dev := &PCIeRootPort{
+ DeviceID: "",
+ Port: fmt.Sprintf("0x%x", startPort+i),
+ Bus: PCIeRootBus,
+ ID: fmt.Sprintf("pci.%d", i+1),
+ Chassis: uint32(i + 1),
+ }
+
+ major := i / funcNumPerSlot
+ minor := i % funcNumPerSlot
+ dev.Multifunction = false
+ if minor == 0 {
+ dev.Multifunction = true
+ }
+ dev.Slot = fmt.Sprintf("%02x", major+1)
+ dev.Function = fmt.Sprintf("%x", minor)
+
+ rp.Items = append(rp.Items, dev)
+ }
+}
+
+// AddDevice add a device to the PCIeRootPortPool
+func (rp *PCIeRootPortPool) AddDevice(devID string) (*PCIeRootPort, error) {
+ for _, it := range rp.Items {
+ if it.DeviceID == "" {
+ it.DeviceID = devID
+ return it, nil
+ }
+ }
+ return nil, fmt.Errorf("Unable to hot plug device on Root Ports: there are not empty slots")
+}
+
+// RemoveDevice remove a device from the PCIeRootPortPool
+func (rp *PCIeRootPortPool) RemoveDevice(devID string) error {
+ for _, it := range rp.Items {
+ if it.DeviceID == devID {
+ // free address to re-use the same slot with other devices
+ it.DeviceID = ""
+ return nil
+ }
+ }
+
+ return fmt.Errorf("Unable to hot unplug device %s: not present on Root Port", devID)
+}
--
2.14.3 (Apple Git-98)

245
runtime/patches/0039-storage-add-storage-common-functions-and-structs.patch Normal file
View File

@ -0,0 +1,245 @@
From a36c9857447aaf22628af1ef01406a916436133b Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Tue, 18 Aug 2020 22:00:04 +0800
Subject: [PATCH 39/50] storage: add storage common functions and structs
reason:
1. add storage.go and storage_spec.go
2. provide funcs for mount nfs and gpath
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
virtcontainers/storage/storage.go | 115 +++++++++++++++++++++++++++++++++
virtcontainers/storage/storage_spec.go | 98 ++++++++++++++++++++++++++++
2 files changed, 213 insertions(+)
create mode 100644 virtcontainers/storage/storage.go
create mode 100644 virtcontainers/storage/storage_spec.go
diff --git a/virtcontainers/storage/storage.go b/virtcontainers/storage/storage.go
new file mode 100644
index 00000000..77ef994f
--- /dev/null
+++ b/virtcontainers/storage/storage.go
@@ -0,0 +1,115 @@
+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+// Description: common functions
+// Author: licuifang
+// Create: 2019-07-14
+
+// Package storage provides functions for mounting
+package storage
+
+import (
+ "fmt"
+ "path/filepath"
+ "regexp"
+ "strings"
+
+ "github.com/kata-containers/agent/protocols/grpc"
+ "github.com/opencontainers/runtime-spec/specs-go"
+)
+
+var (
+ // simple regular expressions for "nfs-server.com:/aaa/bbb/ccc/" or "192.168.1.1:/remote/path"
+ regUrl = regexp.MustCompile(`^((([a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,6})|((?:(?:25[0-5]|2[0-4]\d|[01]?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d?\d))):/[A-Za-z0-9_./-]{0,1000}$`)
+
+ defaultMountOption = "nolock"
+ storageMountOptionKey = "mount_op"
+ storageCustomOptionKey = "custom_op"
+)
+
+type RemoteStorage struct {
+ Source string
+ Dest string
+ Options map[string][]string
+}
+
+func (n *RemoteStorage) Validate(volumeType string) error {
+ if n.Source == "" || n.Dest == "" {
+ return fmt.Errorf("the source and dest of storage cannot be empty")
+ }
+ switch volumeType {
+ case NFS:
+ chk := regUrl.FindAllString(n.Source, -1)
+ if chk == nil {
+ return fmt.Errorf("invalid url for nfs")
+ }
+ case GPATHFS:
+ if !filepath.IsAbs(n.Source) {
+ return fmt.Errorf("invalid gpath")
+ }
+ }
+
+ return nil
+}
+
+func (n *RemoteStorage) GetGrpcStorageAndAppendMount(volumeType string, vmBasePath string, spec *specs.Spec, sandboxId string) *grpc.Storage {
+ var (
+ grpcStorag *grpc.Storage
+ vmPath string
+ )
+ switch volumeType {
+ case NFS:
+ // source of sfs like remote-nfs.com:/share-53ee51a1
+ // source of sfs with subpath like remote-nfs.com:/share-53ee51a1/aaa/bbb
+ // source of sfs-turbo like ip:/
+ // source of sfs-turbo with subpath like ip://aaa or ip://aaa/bbb
+ // here we only get the origin source such as remote-nfs.com:/share-53ee51a1 or ip:/ as the Source of grpcStorage
+ item := strings.Split(n.Source, ":")
+ if len(item) != 2 {
+ return nil
+ }
+ vmPath = filepath.Join(vmBasePath, sandboxId, item[0], item[1])
+ parts := strings.SplitAfter(item[1], "/")
+ if len(parts) > 2 {
+ n.Source = item[0] + ":" + parts[0] + parts[1]
+ }
+
+ grpcStorag = &grpc.Storage{
+ Driver: NFS,
+ Source: n.Source,
+ MountPoint: vmPath,
+ }
+ grpcStorag.Options = append(grpcStorag.Options, defaultMountOption)
+ case GPATHFS:
+ vmPath = n.Source
+ grpcStorag = &grpc.Storage{
+ Driver: GPATHFS,
+ Source: vmPath,
+ MountPoint: n.Dest,
+ }
+ }
+ if custumOpts, ok := n.Options[storageCustomOptionKey]; ok {
+ for _, custumOpt := range custumOpts {
+ grpcStorag.Options = append(grpcStorag.Options, custumOpt)
+ }
+ }
+
+ mnt := specs.Mount{
+ Source: vmPath,
+ Destination: n.Dest,
+ }
+ if mountOpts, ok := n.Options[storageMountOptionKey]; ok {
+ for _, mountOpt := range mountOpts {
+ if mountOpt == "shared" {
+ // in order to support "shared" propagation for guestpath mounting to container directory
+ // we should set rootfspropagation shared
+ spec.Linux.RootfsPropagation = "shared"
+ }
+ mnt.Options = append(mnt.Options, mountOpt)
+ }
+ }
+ mnt.Options = append(mnt.Options, "rbind")
+ mnt.Type = "bind"
+ spec.Mounts = append(spec.Mounts, mnt)
+
+ return grpcStorag
+}
diff --git a/virtcontainers/storage/storage_spec.go b/virtcontainers/storage/storage_spec.go
new file mode 100644
index 00000000..8e866b8d
--- /dev/null
+++ b/virtcontainers/storage/storage_spec.go
@@ -0,0 +1,98 @@
+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
+// Description: common functions
+// Author: caihaomin c00416947
+// Create: 2019-05-05
+
+package storage
+
+import (
+ "encoding/json"
+ "fmt"
+
+ "github.com/kata-containers/agent/protocols/grpc"
+ "github.com/opencontainers/runtime-spec/specs-go"
+)
+
+const (
+ NFS = "nfs"
+ GPATHFS = "gpath"
+)
+
+// STORAGES specifies the storage type and is supported
+var STORAGES = map[string]bool{
+ NFS: true,
+ GPATHFS: true,
+}
+
+type StorageSpec struct {
+ // StorageType specifies the type of storage passing down
+ StorageType string `json:"storage_type,omitempty"`
+
+ Source string `json:"source,omitempty"`
+ Destination string `json:"dest,omitempty"`
+
+ // Options specifies the options of storage type
+ Options map[string][]string `json:"options,omitempty"`
+}
+
+func (s *StorageSpec) generateInstances() StorageOperation {
+ var storageOpt StorageOperation
+ storageOpt = &RemoteStorage{
+ Source: s.Source,
+ Dest: s.Destination,
+ Options: s.Options,
+ }
+
+ return storageOpt
+}
+
+func ValidateStorageValue(value string) error {
+ var storageSpecs []StorageSpec
+ if err := json.Unmarshal([]byte(value), &storageSpecs); err != nil {
+ return err
+ }
+
+ var storageOpt []StorageOperation
+ for _, item := range storageSpecs {
+ if supported, ok := STORAGES[item.StorageType]; !ok || !supported {
+ return fmt.Errorf("type %s of storage is not supported", item.StorageType)
+ }
+ storageItem := item.generateInstances()
+ if err := storageItem.Validate(item.StorageType); err != nil {
+ return err
+ }
+ storageOpt = append(storageOpt, storageItem)
+ }
+
+ return nil
+}
+
+func GetGrpcStorageAndAppendMount(vmBasePath, value string, spec *specs.Spec, sandboxId string) []*grpc.Storage {
+ var (
+ grpcStorages []*grpc.Storage
+ storageSpec []StorageSpec
+ )
+
+ // the value has been validated before, and there is no need to judge the return value here
+ json.Unmarshal([]byte(value), &storageSpec)
+ for _, item := range storageSpec {
+ storageItem := item.generateInstances()
+ grpcStorages = append(grpcStorages, storageItem.GetGrpcStorageAndAppendMount(item.StorageType, vmBasePath, spec, sandboxId))
+ }
+ return grpcStorages
+}
+
+type StorageOperation interface {
+ Validate(volumeType string) error
+ GetGrpcStorageAndAppendMount(volumeType string, vmBasePath string, spec *specs.Spec, sandboxId string) *grpc.Storage
+}
+
+type FakeStorage struct{}
+
+func (f *FakeStorage) Validate() error {
+ return nil
+}
+
+func (f *FakeStorage) GetGrpcStorageAndAppendMount(vmBasePath string, spec *specs.Spec) *grpc.Storage {
+ return &grpc.Storage{}
+}
--
2.14.3 (Apple Git-98)

221
runtime/patches/0040-storage-add-go-tests-for-storage.patch Normal file
View File

@ -0,0 +1,221 @@
From 2e32e2c156a134605de42c53ef77366ac73f2614 Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Tue, 18 Aug 2020 22:01:58 +0800
Subject: [PATCH 40/50] storage: add go tests for storage
reason: add go tests file storage_test.go and storage_spec_test.go
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
virtcontainers/storage/storage_spec_test.go | 151 ++++++++++++++++++++++++++++
virtcontainers/storage/storage_test.go | 40 ++++++++
2 files changed, 191 insertions(+)
create mode 100644 virtcontainers/storage/storage_spec_test.go
create mode 100644 virtcontainers/storage/storage_test.go
diff --git a/virtcontainers/storage/storage_spec_test.go b/virtcontainers/storage/storage_spec_test.go
new file mode 100644
index 00000000..f638245a
--- /dev/null
+++ b/virtcontainers/storage/storage_spec_test.go
@@ -0,0 +1,151 @@
+/*
+Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
+Description: common functions test
+Author: yangfeiyu
+Create: 2019-07-24
+*/
+
+package storage
+
+import (
+ "encoding/json"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/opencontainers/runtime-spec/specs-go"
+)
+
+func TestValidateStorageValue(t *testing.T) {
+ assert := assert.New(t)
+ //invalid storage type,not NFS or GPATHS
+ spec := &StorageSpec{
+ StorageType: "invalidStorage",
+ Source: "sfs.com:/remote/path",
+ Destination: "/opt/",
+ }
+ data, err := json.Marshal(spec)
+ assert.NoError(err)
+
+ err = ValidateStorageValue(string(data))
+ assert.Error(err, "StorageType is not NFS or GPATHS,it should return error")
+
+ //case 1: NFS
+ specNFS := []StorageSpec{
+ {
+ StorageType: NFS,
+ Source: "sfs.com:/remote/path",
+ Destination: "/opt/"},
+ }
+ data, err = json.Marshal(specNFS)
+ assert.NoError(err)
+
+ err = ValidateStorageValue(string(data))
+ assert.NoError(err)
+
+ // case2: invalid source domain address
+ specNFS = []StorageSpec{
+ {
+ StorageType: NFS,
+ Source: "sfs..com:/remote/path",
+ Destination: "/opt/"},
+ }
+ data, err = json.Marshal(specNFS)
+ assert.NoError(err)
+
+ err = ValidateStorageValue(string(data))
+ assert.Error(err)
+
+ // case 3: nfs source is valid ip address
+ ipSpecNFS := []StorageSpec{
+ {
+ StorageType: NFS,
+ Source: "192.168.18.147:/remote/path",
+ Destination: "/tmp/sfsturbo0",
+ },
+ }
+
+ data, err = json.Marshal(ipSpecNFS)
+ assert.NoError(err)
+
+ err = ValidateStorageValue(string(data))
+ assert.NoError(err)
+
+ // case 4: invalid ip address for nfs source
+ ipSpecNFS = []StorageSpec{
+ {
+ StorageType: NFS,
+ Source: "192.168.18.300:/remote/path",
+ Destination: "/tmp/sfsturbo0",
+ },
+ }
+
+ data, err = json.Marshal(ipSpecNFS)
+ assert.NoError(err)
+
+ err = ValidateStorageValue(string(data))
+ assert.Error(err)
+
+ // case 5: validate ip address and source remote path is /
+ ipSpecNFS = []StorageSpec{
+ {
+ StorageType: NFS,
+ Source: "192.168.18.3:/",
+ Destination: "/tmp/sfsturbo0",
+ },
+ }
+
+ data, err = json.Marshal(ipSpecNFS)
+ assert.NoError(err)
+
+ err = ValidateStorageValue(string(data))
+ assert.NoError(err)
+
+ // case 6: invalid ip address 192.168.18.147.11
+ ipSpecNFS = []StorageSpec{
+ {
+ StorageType: NFS,
+ Source: "192.168.18.147.11:/",
+ Destination: "/tmp/sfsturbo0",
+ },
+ }
+
+ data, err = json.Marshal(ipSpecNFS)
+ assert.NoError(err)
+
+ err = ValidateStorageValue(string(data))
+ assert.Error(err)
+
+ //GPATHS
+ specGPATHFS := []StorageSpec{
+ {
+ StorageType: GPATHFS,
+ Source: "/remote/path",
+ Destination: "/opt/"},
+ }
+ data, err = json.Marshal(specGPATHFS)
+ assert.NoError(err)
+
+ err = ValidateStorageValue(string(data))
+ if STORAGES[GPATHFS] == true {
+ assert.NoError(err, "StorageType GPATHFS is valid,it should return no error")
+ } else {
+ assert.Error(err, "StorageType GPATHFS is invalid,it should return error")
+ }
+}
+
+func TestGetGrpcStorageAndAppendMount(t *testing.T) {
+ assert := assert.New(t)
+ var spec specs.Spec
+ sandboxId := "7f3b5e32cc7fe757e1b69f7a005ca1971def5011e730c82535783630fe24b318"
+ vmBasePath := "/tmp"
+
+ // storageType nfs with legal source and dest will succeed
+ storageSpec := "[{\"storage_type\":\"nfs\",\"source\":\"sfs.com:/remote/path\",\"dest\":\"/opt/nfs\"}]"
+ grpcStorages := GetGrpcStorageAndAppendMount(vmBasePath,storageSpec,&spec,sandboxId)
+ assert.NotNil(grpcStorages[0])
+
+ // storageType nfs with illegal source and dest will fail
+ storageSpec = "[{\"storage_type\":\"nfs\",\"source\":\"sfs.com\",\"dest\":\"/opt/nfs\"}]"
+ grpcStorages = GetGrpcStorageAndAppendMount(vmBasePath,storageSpec,&spec,sandboxId)
+ assert.Nil(grpcStorages[0])
+}
diff --git a/virtcontainers/storage/storage_test.go b/virtcontainers/storage/storage_test.go
new file mode 100644
index 00000000..c9ca4926
--- /dev/null
+++ b/virtcontainers/storage/storage_test.go
@@ -0,0 +1,40 @@
+/*
+Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
+Description: common functions test
+Author: yangfeiyu
+Create: 2019-07-24
+*/
+
+package storage
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestValidate(t *testing.T) {
+ //NFS valid
+ assert := assert.New(t)
+ remoteStorage := &RemoteStorage{
+ Source: "sfs.com:/remote/path",
+ Dest: "/opt/",
+ }
+ err := remoteStorage.Validate(NFS)
+ assert.NoError(err, "NFS url is valid,it should return no error")
+
+ //NFS invalid
+ remoteStorage.Source = "sfs.com/../remote/path"
+ err = remoteStorage.Validate(NFS)
+ assert.Error(err, "NFS url is invalid,it should return error")
+
+ //GPATHS valid
+ remoteStorage.Source = "/path/in/vm"
+ err = remoteStorage.Validate(GPATHFS)
+ assert.NoError(err, "GPATHFS is valid,it should return no error")
+
+ //GPATHS invalid
+ remoteStorage.Source = "./../path/in/../vm"
+ err = remoteStorage.Validate(GPATHFS)
+ assert.Error(err, "GPATHFS is invalid,it should return error")
+}
--
2.14.3 (Apple Git-98)

115
runtime/patches/0041-storage-mount-nfs-and-gpath-with-given-annotation.patch Normal file
View File

@ -0,0 +1,115 @@
From 31e07f1b6cbf361783c4d7adf9e4b8da30c67384 Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Tue, 18 Aug 2020 22:05:25 +0800
Subject: [PATCH 41/50] storage: mount nfs and gpath with given annotation
reason: when run container with annotation about storage spec,
prepare basic info in kata-runtime
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
virtcontainers/kata_agent.go | 14 +++++++++++++-
virtcontainers/pkg/annotations/annotations.go | 3 +++
virtcontainers/pkg/oci/utils.go | 16 ++++++++++++++++
3 files changed, 32 insertions(+), 1 deletion(-)
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
index d82a7f2d..ac64817a 100644
--- a/virtcontainers/kata_agent.go
+++ b/virtcontainers/kata_agent.go
@@ -30,10 +30,11 @@ import (
ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter"
"github.com/kata-containers/runtime/virtcontainers/pkg/rootless"
vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
- "github.com/kata-containers/runtime/virtcontainers/utils"
"github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
+ "github.com/kata-containers/runtime/virtcontainers/storage"
"github.com/kata-containers/runtime/virtcontainers/store"
"github.com/kata-containers/runtime/virtcontainers/types"
+ "github.com/kata-containers/runtime/virtcontainers/utils"
"github.com/opencontainers/runtime-spec/specs-go"
opentracing "github.com/opentracing/opentracing-go"
"github.com/sirupsen/logrus"
@@ -1427,6 +1428,9 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process,
localStorages := k.handleLocalStorage(ociSpec.Mounts, sandbox.id, c.rootfsSuffix)
ctrStorages = append(ctrStorages, localStorages...)
+ remoteStoragtes := k.handleRemoteStorage(ociSpec, sandbox.id)
+ ctrStorages = append(ctrStorages, remoteStoragtes...)
+
// We replace all OCI mount sources that match our container mount
// with the right source path (The guest one).
if err = k.replaceOCIMountSource(ociSpec, newMounts); err != nil {
@@ -1510,6 +1514,14 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process,
k.state.URL, consoleURL, c.config.Cmd, createNSList, enterNSList)
}
+func (k *kataAgent) handleRemoteStorage(spec *specs.Spec, sandboxId string) []*grpc.Storage {
+ if value, ok := spec.Annotations[vcAnnotations.StorageSpecTypeKey]; ok {
+ return storage.GetGrpcStorageAndAppendMount(kataGuestStorageDir, value, spec, sandboxId)
+ }
+
+ return []*grpc.Storage{}
+}
+
// handleEphemeralStorage handles ephemeral storages by
// creating a Storage from corresponding source of the mount point
func (k *kataAgent) handleEphemeralStorage(mounts []specs.Mount) []*grpc.Storage {
diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go
index 903c7f03..e50a697c 100644
--- a/virtcontainers/pkg/annotations/annotations.go
+++ b/virtcontainers/pkg/annotations/annotations.go
@@ -68,6 +68,9 @@ const (
// AssetHashType is the hash type used for assets verification
AssetHashType = kataAnnotationsPrefix + "asset_hash_type"
+ // StorageSpecTypeKey is the annotation key to fetch storage_spec
+ StorageSpecTypeKey = kataAnnotationsPrefix + "storage_spec"
+
//
// Generic annotations
//
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
index 948bd3cb..d032227e 100644
--- a/virtcontainers/pkg/oci/utils.go
+++ b/virtcontainers/pkg/oci/utils.go
@@ -22,6 +22,7 @@ import (
exp "github.com/kata-containers/runtime/virtcontainers/experimental"
vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
dockershimAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations/dockershim"
+ "github.com/kata-containers/runtime/virtcontainers/storage"
"github.com/kata-containers/runtime/virtcontainers/types"
"github.com/kata-containers/runtime/virtcontainers/utils"
specs "github.com/opencontainers/runtime-spec/specs-go"
@@ -340,6 +341,17 @@ func SandboxID(spec specs.Spec) (string, error) {
return "", fmt.Errorf("Could not find sandbox ID")
}
+func validateStorageSpec(spec specs.Spec) error {
+ if storageSpec, ok := spec.Annotations[vcAnnotations.StorageSpecTypeKey]; ok {
+ err := storage.ValidateStorageValue(storageSpec)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
func addAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) error {
addAssetAnnotations(ocispec, config)
if err := addHypervisorConfigOverrides(ocispec, config); err != nil {
@@ -873,6 +885,10 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid, c
// ContainerConfig converts an OCI compatible runtime configuration
// file to a virtcontainers container configuration structure.
func ContainerConfig(ocispec specs.Spec, bundlePath, cid, console string, detach bool) (vc.ContainerConfig, error) {
+ err := validateStorageSpec(ocispec)
+ if err != nil {
+ return vc.ContainerConfig{}, err
+ }
rootfs := vc.RootFs{Target: ocispec.Root.Path, Mounted: true}
if !filepath.IsAbs(rootfs.Target) {
rootfs.Target = filepath.Join(bundlePath, ocispec.Root.Path)
--
2.14.3 (Apple Git-98)

38
runtime/patches/0042-kata-runtime-do-not-ignore-updateInterface-return-er.patch Normal file
View File

@ -0,0 +1,38 @@
From 37372be961528471d418bbe066d88a010ad0513c Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Tue, 18 Aug 2020 22:12:59 +0800
Subject: [PATCH 42/50] kata-runtime: do not ignore updateInterface return
error
reason: If send UpdateInterfaceRequest to kata-agent and return the
error back, we should process it correctly, should not ignore the
error, which may casue deference nil pointer problem.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
virtcontainers/kata_agent.go | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
index ac64817a..fee4215f 100644
--- a/virtcontainers/kata_agent.go
+++ b/virtcontainers/kata_agent.go
@@ -604,7 +604,15 @@ func (k *kataAgent) updateInterface(ifc *vcTypes.Interface) (*vcTypes.Interface,
"interface-requested": fmt.Sprintf("%+v", ifc),
"resulting-interface": fmt.Sprintf("%+v", resultingInterface),
}).WithError(err).Error("update interface request failed")
+ return nil, err
+ }
+
+ // need to judege resultingInterface is not not, otherwise may cause
+ // deference nil pointer panic problem
+ if resultingInterface == nil {
+ return nil, fmt.Errorf("resultingInterface should not be nil")
}
+
if resultInterface, ok := resultingInterface.(*aTypes.Interface); ok {
iface := &vcTypes.Interface{
Device: resultInterface.Device,
--
2.14.3 (Apple Git-98)

162
runtime/patches/0043-kata-runtime-support-add-hypervisor-global-parameter.patch Normal file
View File

@ -0,0 +1,162 @@
From d9738cfd6500ced30efde9e747eb73e5076e73ed Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Tue, 18 Aug 2020 22:47:13 +0800
Subject: [PATCH 43/50] kata-runtime: support add hypervisor global parameters
in config file
reason: support add hypervisor global parameters in config file
with defaultHypervisorParams config option.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
Makefile | 2 ++
cli/config/configuration-qemu.toml.in | 7 +++++++
pkg/katautils/config-settings.go.in | 1 +
pkg/katautils/config.go | 13 ++++++++++++-
vendor/github.com/intel/govmm/qemu/qemu.go | 6 +++---
virtcontainers/qemu.go | 2 +-
6 files changed, 26 insertions(+), 5 deletions(-)
diff --git a/Makefile b/Makefile
index d5e4bbe1..b62e64b0 100644
--- a/Makefile
+++ b/Makefile
@@ -400,6 +400,7 @@ USER_VARS += FIRMWAREPATH
USER_VARS += MACHINEACCELERATORS
USER_VARS += DEFMACHINETYPE_CLH
USER_VARS += KERNELPARAMS
+USER_VARS += HYPERVISORPARAMS
USER_VARS += LIBEXECDIR
USER_VARS += LOCALSTATEDIR
USER_VARS += PKGDATADIR
@@ -607,6 +608,7 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit
-e "s|@FIRMWAREPATH_CLH@|$(FIRMWAREPATH_CLH)|g" \
-e "s|@DEFMACHINETYPE_CLH@|$(DEFMACHINETYPE_CLH)|g" \
-e "s|@KERNELPARAMS@|$(KERNELPARAMS)|g" \
+ -e "s|@HYPERVISORPARAMS@|$(HYPERVISORPARAMS)|g" \
-e "s|@LOCALSTATEDIR@|$(LOCALSTATEDIR)|g" \
-e "s|@PKGLIBEXECDIR@|$(PKGLIBEXECDIR)|g" \
-e "s|@PKGRUNDIR@|$(PKGRUNDIR)|g" \
diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in
index aa11b38f..e57a954c 100644
--- a/cli/config/configuration-qemu.toml.in
+++ b/cli/config/configuration-qemu.toml.in
@@ -29,6 +29,13 @@ machine_type = "@MACHINETYPE@"
# container and look for 'default-kernel-parameters' log entries.
kernel_params = "@KERNELPARAMS@"
+# Optional space-separated list of options to pass to the hypervisor.
+# For example, use `hypervisor_params = "kvm-pit.lost_tick_policy=discard"`
+#
+# WARNING: - any parameter specified here will take priority over the default
+# parameter value of the same name used to start the hypervisor.
+hypervisor_params = "@HYPERVISORPARAMS@"
+
# Path to the firmware.
# If you want that qemu uses the default firmware leave this option empty
firmware = "@FIRMWAREPATH@"
diff --git a/pkg/katautils/config-settings.go.in b/pkg/katautils/config-settings.go.in
index aaf78cc3..b2dfdfa1 100644
--- a/pkg/katautils/config-settings.go.in
+++ b/pkg/katautils/config-settings.go.in
@@ -20,6 +20,7 @@ var defaultShimPath = "/usr/libexec/kata-containers/kata-shim"
var systemdUnitName = "kata-containers.target"
const defaultKernelParams = ""
+const defaultHypervisorParams = "kvm-pit.lost_tick_policy=discard"
const defaultMachineType = "pc"
const defaultVCPUCount uint32 = 1
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
index 51120311..3365b3f5 100644
--- a/pkg/katautils/config.go
+++ b/pkg/katautils/config.go
@@ -130,6 +130,7 @@ type hypervisor struct {
HotplugVFIOOnRootBus bool `toml:"hotplug_vfio_on_root_bus"`
DisableVhostNet bool `toml:"disable_vhost_net"`
GuestHookPath string `toml:"guest_hook_path"`
+ HypervisorParams string `toml:"hypervisor_params"`
}
type proxy struct {
@@ -270,6 +271,14 @@ func (h hypervisor) kernelParams() string {
return h.KernelParams
}
+func (h hypervisor) hypervisorParams() string {
+ if h.HypervisorParams == "" {
+ return defaultHypervisorParams
+ }
+
+ return h.HypervisorParams
+}
+
func (h hypervisor) machineType() string {
if h.MachineType == "" {
return defaultMachineType
@@ -632,6 +641,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
machineAccelerators := h.machineAccelerators()
kernelParams := h.kernelParams()
+ hypervisorParams := h.hypervisorParams()
machineType := h.machineType()
blockDriver, err := h.blockDeviceDriver()
@@ -675,6 +685,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
FirmwarePath: firmware,
MachineAccelerators: machineAccelerators,
KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)),
+ HypervisorParams: vc.DeserializeParams(strings.Fields(hypervisorParams)),
HypervisorMachineType: machineType,
NumVCPUs: h.defaultVCPUs(),
DefaultMaxVCPUs: h.defaultMaxVCPUs(),
@@ -983,7 +994,7 @@ func updateRuntimeConfigAgent(configPath string, tomlConf tomlConfig, config *oc
TraceMode: agent.traceMode(),
TraceType: agent.traceType(),
KernelModules: agent.kernelModules(),
- MountBlkInVM: agent.mountBlkDevInVM(),
+ MountBlkInVM: agent.mountBlkDevInVM(),
}
default:
return fmt.Errorf("%s agent type is not supported", k)
diff --git a/vendor/github.com/intel/govmm/qemu/qemu.go b/vendor/github.com/intel/govmm/qemu/qemu.go
index 3e7720b4..68f8d2b0 100644
--- a/vendor/github.com/intel/govmm/qemu/qemu.go
+++ b/vendor/github.com/intel/govmm/qemu/qemu.go
@@ -2096,7 +2096,7 @@ type Config struct {
SMP SMP
// GlobalParam is the -global parameter.
- GlobalParam string
+ GlobalParam []string
// Knobs is a set of qemu boolean settings.
Knobs Knobs
@@ -2285,9 +2285,9 @@ func (config *Config) appendRTC() {
}
func (config *Config) appendGlobalParam() {
- if config.GlobalParam != "" {
+ for _, param := range config.GlobalParam {
config.qemuParams = append(config.qemuParams, "-global")
- config.qemuParams = append(config.qemuParams, config.GlobalParam)
+ config.qemuParams = append(config.qemuParams, param)
}
}
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
index a10c66fb..657b6be7 100644
--- a/virtcontainers/qemu.go
+++ b/virtcontainers/qemu.go
@@ -600,7 +600,7 @@ func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNa
Knobs: knobs,
Incoming: incoming,
VGA: "none",
- GlobalParam: "kvm-pit.lost_tick_policy=discard",
+ GlobalParam: SerializeParams(hypervisorConfig.HypervisorParams, "="),
Bios: firmwarePath,
PidFile: filepath.Join(q.store.RunVMStoragePath(), q.id, "pid"),
}
--
2.14.3 (Apple Git-98)

375
runtime/patches/0044-network-support-dpdk-vhost_user-net-device.patch Normal file
View File

@ -0,0 +1,375 @@
From dbbf8c5deb14d8033b2e863fb0eb731523af2a47 Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Wed, 19 Aug 2020 09:58:07 +0800
Subject: [PATCH 44/50] network: support dpdk vhost_user net device
reason: support dpdk vhost_user net device
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
vendor/github.com/intel/govmm/qemu/qmp.go | 3 +-
virtcontainers/hypervisor.go | 3 ++
virtcontainers/network.go | 49 ++++++++++++++------
virtcontainers/pkg/types/types.go | 2 +
virtcontainers/qemu.go | 75 +++++++++++++++++++++++++++++++
virtcontainers/sandbox.go | 7 ++-
virtcontainers/vhostuser_endpoint.go | 18 ++++++--
virtcontainers/vhostuser_endpoint_test.go | 7 ++-
8 files changed, 143 insertions(+), 21 deletions(-)
diff --git a/vendor/github.com/intel/govmm/qemu/qmp.go b/vendor/github.com/intel/govmm/qemu/qmp.go
index a64039de..0cb82ffa 100644
--- a/vendor/github.com/intel/govmm/qemu/qmp.go
+++ b/vendor/github.com/intel/govmm/qemu/qmp.go
@@ -970,11 +970,12 @@ func (q *QMP) ExecuteNetdevAdd(ctx context.Context, netdevType, netdevID, ifname
// ExecuteNetdevChardevAdd adds a Net device to a QEMU instance
// using the netdev_add command. netdevID is the id of the device to add.
// Must be valid QMP identifier.
-func (q *QMP) ExecuteNetdevChardevAdd(ctx context.Context, netdevType, netdevID, chardev string, queues int) error {
+func (q *QMP) ExecuteNetdevChardevAdd(ctx context.Context, netdevType, netdevID, chardev string, vhostforce bool, queues int) error {
args := map[string]interface{}{
"type": netdevType,
"id": netdevID,
"chardev": chardev,
+ "vhostforce": vhostforce,
}
if queues > 1 {
args["queues"] = queues
diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go
index c0723daa..60f1d190 100644
--- a/virtcontainers/hypervisor.go
+++ b/virtcontainers/hypervisor.go
@@ -132,6 +132,9 @@ const (
// vhostuserDev is a Vhost-user device type
vhostuserDev
+ // vhostUserNetDev is a Vhost-user net device type
+ vhostUserNetDev
+
// CPUDevice is CPU device type
cpuDev
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
index 488bd00c..a0c64356 100644
--- a/virtcontainers/network.go
+++ b/virtcontainers/network.go
@@ -13,6 +13,7 @@ import (
"math/rand"
"net"
"os"
+ "path/filepath"
"regexp"
"runtime"
"sort"
@@ -65,7 +66,8 @@ const (
)
var (
- regInfName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_\-.]*$`)
+ regInfName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_\-.]*$`)
+ regVhostName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9\-.]{0,127}$`)
)
//IsValid checks if a model is valid
@@ -132,11 +134,12 @@ type NetlinkIface struct {
// NetworkInfo gathers all information related to a network interface.
// It can be used to store the description of the underlying network.
type NetworkInfo struct {
- Device string
- Iface NetlinkIface
- Addrs []netlink.Addr
- Routes []netlink.Route
- DNS DNSInfo
+ Device string
+ Iface NetlinkIface
+ Addrs []netlink.Addr
+ Routes []netlink.Route
+ DNS DNSInfo
+ VhostUserSocket string
}
// NetworkInterface defines a network interface.
@@ -1198,15 +1201,11 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li
return endpoint, err
}
- // Check if this is a dummy interface which has a vhost-user socket associated with it
- socketPath, err := vhostUserSocketPath(netInfo)
- if err != nil {
- return nil, err
- }
-
- if socketPath != "" {
+ // Currently, we only accept the vhost-user socket paas from user input
+ // interface info json file
+ if netInfo.VhostUserSocket != "" {
networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found")
- endpoint, err = createVhostUserEndpoint(netInfo, socketPath)
+ endpoint, err = createVhostUserEndpoint(netInfo, netInfo.VhostUserSocket)
return endpoint, err
}
@@ -1481,6 +1480,24 @@ func verifyIP(ip string) (*net.IP, error) {
return &netIP, nil
}
+// verifyVhostSocket verifies the vhost socket is valid or not
+func verifyVhostSocket(vhostSocket string) error {
+ if vhostSocket == "" {
+ networkLogger().Debug("no dpdk network")
+ return nil
+ }
+ if regVhostName.FindAllString(filepath.Base(vhostSocket), -1) == nil {
+ return fmt.Errorf("invalid input of vhostSocket name, please check the rules")
+ }
+ info, err := os.Stat(vhostSocket)
+ if err != nil || info.IsDir() || info.Mode()&os.ModeSocket == 0 {
+ return fmt.Errorf("invalid vhost socket: %v", vhostSocket)
+ }
+
+ networkLogger().WithField("vhostSocket", vhostSocket).Infof("using dpdk network, socket file is: %s ", vhostSocket)
+ return nil
+}
+
// validInterface check the input interface valid or not
func validInterface(inf *vcTypes.Interface, enableCompatOldCNI bool) error {
if enableCompatOldCNI && verifyInterfaceName(inf.Device) != nil {
@@ -1503,6 +1520,10 @@ func validInterface(inf *vcTypes.Interface, enableCompatOldCNI bool) error {
return err
}
+ if err := verifyVhostSocket(inf.VhostUserSocket); err != nil {
+ return err
+ }
+
// Currently, only one IP address can be passed, which reduces the test entry and fault injection.
if len(inf.IPAddresses) > 0 {
if len(inf.IPAddresses) != 1 {
diff --git a/virtcontainers/pkg/types/types.go b/virtcontainers/pkg/types/types.go
index b41b0c75..dccd92f8 100644
--- a/virtcontainers/pkg/types/types.go
+++ b/virtcontainers/pkg/types/types.go
@@ -29,6 +29,8 @@ type Interface struct {
// library, regarding each type of link. Here is a non exhaustive
// list: "veth", "macvtap", "vlan", "macvlan", "tap", ...
LinkType string `json:"linkType,omitempty"`
+ // VhostUserSocket is DPDK-backed vHost user ports.
+ VhostUserSocket string `json:"vhostUserSocket,omitempty"`
}
// Route describes a network route.
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
index 657b6be7..84797e0d 100644
--- a/virtcontainers/qemu.go
+++ b/virtcontainers/qemu.go
@@ -1279,6 +1279,78 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error
return err
}
+func (q *qemu) hotplugVhostUserNetDevice(endpoint Endpoint, op operation) error {
+ err := q.qmpSetup()
+ if err != nil {
+ return err
+ }
+
+ device := endpoint.(*VhostUserEndpoint)
+
+ charDevID := utils.MakeNameID("char", device.ID, maxDevIDSize)
+ devID := utils.MakeNameID("virtio", device.ID, maxDevIDSize)
+
+ if op == addDevice {
+ // Add chardev specifying the appropriate socket.
+ if err = q.qmpMonitorCh.qmp.ExecuteCharDevUnixSocketAdd(q.qmpMonitorCh.ctx, charDevID, device.SocketPath, true, false); err != nil {
+ return err
+ }
+
+ defer func() {
+ if err != nil {
+ errRb := q.qmpMonitorCh.qmp.ExecuteChardevDel(q.qmpMonitorCh.ctx, charDevID)
+ if errRb != nil {
+ q.Logger().Errorf("deletes a net device roll back failed. charDev id:%s", charDevID)
+ }
+ }
+ }()
+
+ netDevType := "vhost-user"
+ // Add netdev of type vhost-user.
+ if err = q.qmpMonitorCh.qmp.ExecuteNetdevChardevAdd(q.qmpMonitorCh.ctx, netDevType, devID, charDevID, true, 0); err != nil {
+ return err
+ }
+
+ defer func() {
+ if err != nil {
+ errRb := q.qmpMonitorCh.qmp.ExecuteNetdevDel(q.qmpMonitorCh.ctx, devID)
+ if errRb != nil {
+ q.Logger().Errorf("deletes a net device roll back failed. netDev id:%s", devID)
+ }
+ }
+ }()
+
+ var addr, bus, pciAddr string
+ addr, bus, pciAddr, err = q.getPciAddress(device.ID, types.PCI)
+ if err != nil {
+ return nil
+ }
+ defer func() {
+ if err != nil {
+ q.putPciAddress(device.ID)
+ }
+ }()
+
+ endpoint.SetPciAddr(pciAddr)
+
+ // Add virtio-net-pci device.
+ err = q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, devID, devID, device.HardAddr, addr, bus, romFile, 0, q.arch.runNested())
+ return err
+ }
+
+ if err = q.putPciAddress(device.ID); err != nil {
+ return err
+ }
+ if err = q.qmpMonitorCh.qmp.ExecuteDeviceDel(q.qmpMonitorCh.ctx, devID); err != nil {
+ return err
+ }
+ if err = q.qmpMonitorCh.qmp.ExecuteNetdevDel(q.qmpMonitorCh.ctx, devID); err != nil {
+ return err
+ }
+
+ return nil
+}
+
func (q *qemu) hotplugVhostUserDevice(vAttr *config.VhostUserDeviceAttrs, op operation) error {
err := q.qmpSetup()
if err != nil {
@@ -1510,6 +1582,9 @@ func (q *qemu) hotplugDevice(devInfo interface{}, devType deviceType, op operati
case vhostuserDev:
vAttr := devInfo.(*config.VhostUserDeviceAttrs)
return nil, q.hotplugVhostUserDevice(vAttr, op)
+ case vhostUserNetDev:
+ device := devInfo.(Endpoint)
+ return nil, q.hotplugVhostUserNetDevice(device, op)
default:
return nil, fmt.Errorf("cannot hotplug device: unsupported device type '%v'", devType)
}
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
index 0d599267..e6f155a3 100644
--- a/virtcontainers/sandbox.go
+++ b/virtcontainers/sandbox.go
@@ -946,7 +946,8 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) {
},
Type: inf.LinkType,
},
- Addrs: addrs,
+ Addrs: addrs,
+ VhostUserSocket: inf.VhostUserSocket,
}, nil
}
@@ -961,6 +962,10 @@ func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (grpcIf *vcTypes.Interfac
if ep.Name() == inf.Name {
return nil, fmt.Errorf("interface %s is already exist", inf.Name)
}
+
+ if ep.Properties().VhostUserSocket != "" && inf.VhostUserSocket != "" {
+ return nil, fmt.Errorf("sandbox %s only support one dpdk socket", s.ID())
+ }
}
netInfo, err := s.generateNetInfo(inf)
diff --git a/virtcontainers/vhostuser_endpoint.go b/virtcontainers/vhostuser_endpoint.go
index bb4a67be..2fc3d837 100644
--- a/virtcontainers/vhostuser_endpoint.go
+++ b/virtcontainers/vhostuser_endpoint.go
@@ -12,6 +12,7 @@ import (
"github.com/kata-containers/runtime/virtcontainers/device/config"
persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
+ "github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
"github.com/kata-containers/runtime/virtcontainers/utils"
)
@@ -23,6 +24,7 @@ const hostSocketSearchPath = "/tmp/vhostuser_%s/vhu.sock"
// VhostUserEndpoint represents a vhost-user socket based network interface
type VhostUserEndpoint struct {
+ ID string
// Path to the vhost-user socket on the host system
SocketPath string
// MAC address of the interface
@@ -99,18 +101,28 @@ func (endpoint *VhostUserEndpoint) Detach(netNsCreated bool, netNsPath string) e
// HotAttach for vhostuser endpoint not supported yet
func (endpoint *VhostUserEndpoint) HotAttach(h hypervisor) error {
- return fmt.Errorf("VhostUserEndpoint does not support Hot attach")
+ networkLogger().Debug("Hot attaching vhost-user endpoint")
+ if _, err := h.hotplugAddDevice(endpoint, vhostUserNetDev); err != nil {
+ return err
+ }
+ return nil
}
// HotDetach for vhostuser endpoint not supported yet
func (endpoint *VhostUserEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error {
- return fmt.Errorf("VhostUserEndpoint does not support Hot detach")
+ networkLogger().Debug("Hot detaching vhost-user endpoint")
+ if _, err := h.hotplugRemoveDevice(endpoint, vhostUserNetDev); err != nil {
+ networkLogger().WithError(err).Errorf("Error detach vhostUserSocket")
+ return err
+ }
+ return nil
}
// Create a vhostuser endpoint
func createVhostUserEndpoint(netInfo NetworkInfo, socket string) (*VhostUserEndpoint, error) {
-
+ uniqueID := uuid.Generate().String()
vhostUserEndpoint := &VhostUserEndpoint{
+ ID: uniqueID,
SocketPath: socket,
HardAddr: netInfo.Iface.HardwareAddr.String(),
IfaceName: netInfo.Iface.Name,
diff --git a/virtcontainers/vhostuser_endpoint_test.go b/virtcontainers/vhostuser_endpoint_test.go
index ad013e12..584490cc 100644
--- a/virtcontainers/vhostuser_endpoint_test.go
+++ b/virtcontainers/vhostuser_endpoint_test.go
@@ -11,6 +11,7 @@ import (
"os"
"testing"
+ "github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
"github.com/stretchr/testify/assert"
"github.com/vishvananda/netlink"
)
@@ -95,7 +96,7 @@ func TestVhostUserEndpoint_HotAttach(t *testing.T) {
h := &mockHypervisor{}
err := v.HotAttach(h)
- assert.Error(err)
+ assert.NoError(err)
}
func TestVhostUserEndpoint_HotDetach(t *testing.T) {
@@ -109,10 +110,11 @@ func TestVhostUserEndpoint_HotDetach(t *testing.T) {
h := &mockHypervisor{}
err := v.HotDetach(h, true, "")
- assert.Error(err)
+ assert.NoError(err)
}
func TestCreateVhostUserEndpoint(t *testing.T) {
+ uniqueID := uuid.Generate().String()
macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x48}
ifcName := "vhost-deadbeef"
socket := "/tmp/vhu_192.168.0.1"
@@ -128,6 +130,7 @@ func TestCreateVhostUserEndpoint(t *testing.T) {
}
expected := &VhostUserEndpoint{
+ ID: uniqueID,
SocketPath: socket,
HardAddr: macAddr.String(),
IfaceName: ifcName,
--
2.14.3 (Apple Git-98)

115
runtime/patches/0045-network-support-set-dns.patch Normal file
View File

@ -0,0 +1,115 @@
From e104f084ed4f10049f45d9473faed229371a1c6c Mon Sep 17 00:00:00 2001
From: holyfei <yangfeiyu20092010@163.com>
Date: Wed, 19 Aug 2020 09:59:52 +0800
Subject: [PATCH 45/50] network: support set dns
reason: when running a sandbox with annotation about
dns spec, overload k.getDNS(sandbox)
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
---
virtcontainers/kata_agent.go | 11 +++++++++++
virtcontainers/network.go | 8 ++++----
virtcontainers/pkg/annotations/annotations.go | 3 +++
virtcontainers/pkg/oci/utils.go | 19 +++++++++++++++++--
4 files changed, 35 insertions(+), 6 deletions(-)
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
index fee4215f..d28d8cce 100644
--- a/virtcontainers/kata_agent.go
+++ b/virtcontainers/kata_agent.go
@@ -890,6 +890,17 @@ func (k *kataAgent) startSandbox(sandbox *Sandbox) error {
KernelModules: kmodules,
}
+ if value, ok := sandbox.config.Annotations[vcAnnotations.SandboxDNSTypeKey]; ok {
+ var sandboxDns *DNSInfo
+ if err := json.Unmarshal([]byte(value), &sandboxDns); err != nil {
+ return fmt.Errorf("get sandbox dns failed %v", err)
+ }
+
+ if sandboxDns != nil {
+ req.Dns = sandboxDns.Servers
+ }
+ }
+
_, err = k.sendReq(req)
if err != nil {
return err
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
index 488bd00c..68eda4a6 100644
--- a/virtcontainers/network.go
+++ b/virtcontainers/network.go
@@ -117,10 +117,10 @@ const (
// DNSInfo describes the DNS setup related to a network interface.
type DNSInfo struct {
- Servers []string
- Domain string
- Searches []string
- Options []string
+ Servers []string `json:"Servers"`
+ Domain string `json:"Domain,omitempty"`
+ Searches []string `json:"Searches,omitempty"`
+ Options []string `json:"Options,omitempty"`
}
// NetlinkIface describes fully a network interface.
diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go
index e50a697c..528dfa66 100644
--- a/virtcontainers/pkg/annotations/annotations.go
+++ b/virtcontainers/pkg/annotations/annotations.go
@@ -71,6 +71,9 @@ const (
// StorageSpecTypeKey is the annotation key to fetch storage_spec
StorageSpecTypeKey = kataAnnotationsPrefix + "storage_spec"
+ // SandboxDNSTypeKey is the annotation key to fetch sandbox dns options
+ SandboxDNSTypeKey = kataAnnotationsPrefix + "sandbox_dns"
+
//
// Generic annotations
//
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
index d032227e..3b2af753 100644
--- a/virtcontainers/pkg/oci/utils.go
+++ b/virtcontainers/pkg/oci/utils.go
@@ -7,6 +7,7 @@ package oci
import (
"context"
+ "encoding/json"
"errors"
"fmt"
"path/filepath"
@@ -37,8 +38,9 @@ type annotationContainerType struct {
type annotationHandler func(value string) error
var annotationHandlerList = map[string]annotationHandler{
- vcAnnotations.StaticCPUTypeKey: validateSandboxCPU,
- vcAnnotations.StaticMemTypeKey: validateSandboxMem,
+ vcAnnotations.StaticCPUTypeKey: validateSandboxCPU,
+ vcAnnotations.StaticMemTypeKey: validateSandboxMem,
+ vcAnnotations.SandboxDNSTypeKey: validateSandboxDNS,
}
var (
@@ -1108,3 +1110,16 @@ func validateSandboxMem(value string) error {
return nil
}
+
+func validateSandboxDNS(value string) error {
+ if value == "" {
+ return fmt.Errorf("--annotation %s value should not be empty", vcAnnotations.SandboxDNSTypeKey)
+ }
+
+ var sandboxDns *vc.DNSInfo
+ if err := json.Unmarshal([]byte(value), &sandboxDns); err != nil {
+ return fmt.Errorf("invalid value passed by --annotation %s, error: %v", vcAnnotations.SandboxDNSTypeKey, err)
+ }
+
+ return nil
+}
--
2.14.3 (Apple Git-98)

489
runtime/patches/0046-network-support-multiqueue-when-inserting-interface-.patch Normal file
View File

@ -0,0 +1,489 @@
From e9c1fb235d0e69d5db72497a8779df707c499d3b Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Wed, 19 Aug 2020 20:36:40 +0800
Subject: [PATCH 46/50] network: support multiqueue when inserting interface
dynamically
reason: support multiqueue when inserting interface dynamically
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
.../kata-containers/agent/pkg/types/types.pb.go | 82 ++++++++++++++++------
virtcontainers/bridgedmacvlan_endpoint.go | 2 +-
virtcontainers/ipvlan_endpoint.go | 2 +-
virtcontainers/kata_agent.go | 2 +
virtcontainers/network.go | 13 ++--
virtcontainers/network_test.go | 2 +-
virtcontainers/pkg/types/types.go | 2 +
virtcontainers/qemu.go | 10 +--
virtcontainers/sandbox.go | 1 +
virtcontainers/tap_endpoint.go | 6 +-
virtcontainers/tuntap_endpoint.go | 4 +-
virtcontainers/veth_endpoint.go | 4 +-
virtcontainers/veth_endpoint_test.go | 6 +-
virtcontainers/vhostuser_endpoint.go | 4 +-
virtcontainers/vhostuser_endpoint_test.go | 2 +-
15 files changed, 96 insertions(+), 46 deletions(-)
diff --git a/vendor/github.com/kata-containers/agent/pkg/types/types.pb.go b/vendor/github.com/kata-containers/agent/pkg/types/types.pb.go
index 7ea63e3c..8b7e2a5d 100644
--- a/vendor/github.com/kata-containers/agent/pkg/types/types.pb.go
+++ b/vendor/github.com/kata-containers/agent/pkg/types/types.pb.go
@@ -100,6 +100,7 @@ type Interface struct {
// list: "veth", "macvtap", "vlan", "macvlan", "tap", ...
Type string `protobuf:"bytes,7,opt,name=type,proto3" json:"type,omitempty"`
RawFlags uint32 `protobuf:"varint,8,opt,name=raw_flags,json=rawFlags,proto3" json:"raw_flags,omitempty"`
+ Queues uint32 `protobuf:"varint,9,opt,name=Queues,proto3" json:"Queues,omitempty"`
}
func (m *Interface) Reset() { *m = Interface{} }
@@ -163,6 +164,13 @@ func (m *Interface) GetRawFlags() uint32 {
return 0
}
+func (m *Interface) GetQueues() uint32 {
+ if m != nil {
+ return m.Queues
+ }
+ return 0
+}
+
type Route struct {
Dest string `protobuf:"bytes,1,opt,name=dest,proto3" json:"dest,omitempty"`
Gateway string `protobuf:"bytes,2,opt,name=gateway,proto3" json:"gateway,omitempty"`
@@ -319,6 +327,11 @@ func (m *Interface) MarshalTo(dAtA []byte) (int, error) {
i++
i = encodeVarintTypes(dAtA, i, uint64(m.RawFlags))
}
+ if m.Queues != 0 {
+ dAtA[i] = 0x48
+ i++
+ i = encodeVarintTypes(dAtA, i, uint64(m.Queues))
+ }
return i, nil
}
@@ -430,6 +443,9 @@ func (m *Interface) Size() (n int) {
if m.RawFlags != 0 {
n += 1 + sovTypes(uint64(m.RawFlags))
}
+ if m.Queues != 0 {
+ n += 1 + sovTypes(uint64(m.Queues))
+ }
return n
}
@@ -841,6 +857,25 @@ func (m *Interface) Unmarshal(dAtA []byte) error {
break
}
}
+ case 9:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field Queues", wireType)
+ }
+ m.Queues = 0
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowTypes
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ m.Queues |= (uint32(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
default:
iNdEx = preIndex
skippy, err := skipTypes(dAtA[iNdEx:])
@@ -1155,27 +1190,28 @@ var (
func init() { proto.RegisterFile("pkg/types/types.proto", fileDescriptorTypes) }
var fileDescriptorTypes = []byte{
- // 345 bytes of a gzipped FileDescriptorProto
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x91, 0xd1, 0x6a, 0xfa, 0x30,
- 0x14, 0xc6, 0xff, 0xb1, 0xb6, 0xda, 0xe3, 0xdf, 0xad, 0x84, 0x4d, 0xc2, 0x06, 0x52, 0xbc, 0x59,
- 0xd9, 0x85, 0x03, 0x37, 0x76, 0xbf, 0x5d, 0x08, 0xde, 0x49, 0x5e, 0x60, 0xc4, 0x36, 0x3a, 0xd1,
- 0xda, 0xd2, 0x44, 0x8b, 0xec, 0x05, 0x77, 0xb9, 0x47, 0x10, 0x9f, 0x64, 0xe4, 0x24, 0x8a, 0xbb,
- 0xd1, 0xef, 0x97, 0x93, 0xd3, 0xef, 0x7c, 0x27, 0x70, 0x5b, 0xae, 0x16, 0x4f, 0x7a, 0x5f, 0x4a,
- 0x65, 0x7f, 0x87, 0x65, 0x55, 0xe8, 0x82, 0xfa, 0x08, 0x83, 0x19, 0x84, 0x93, 0xe9, 0x5b, 0x96,
- 0x55, 0x52, 0x29, 0xfa, 0x00, 0xc1, 0x5c, 0xe4, 0xcb, 0xf5, 0x9e, 0x91, 0x98, 0x24, 0x57, 0xa3,
- 0xeb, 0xa1, 0xed, 0x98, 0x4c, 0xc7, 0x78, 0xcc, 0x5d, 0x99, 0x32, 0x68, 0x09, 0xdb, 0xc3, 0x1a,
- 0x31, 0x49, 0x42, 0x7e, 0x42, 0x4a, 0xa1, 0x99, 0x0b, 0xb5, 0x62, 0x1e, 0x1e, 0xa3, 0x1e, 0x1c,
- 0x08, 0x84, 0x93, 0x8d, 0x96, 0xd5, 0x5c, 0xa4, 0x92, 0xf6, 0x20, 0xc8, 0xe4, 0x6e, 0x99, 0x4a,
- 0x34, 0x09, 0xb9, 0x23, 0xd3, 0xb9, 0x11, 0xb9, 0x74, 0x1f, 0x44, 0x4d, 0x47, 0xd0, 0x39, 0x4f,
- 0x27, 0x15, 0xf3, 0x62, 0x2f, 0xe9, 0x8c, 0xa2, 0xf3, 0x54, 0xae, 0xc2, 0x2f, 0x2f, 0xd1, 0x08,
- 0xbc, 0x5c, 0x6f, 0x59, 0x33, 0x26, 0x49, 0x93, 0x1b, 0x69, 0x1c, 0x3f, 0x6b, 0x73, 0x81, 0xf9,
- 0xd6, 0xd1, 0x92, 0x49, 0x51, 0xa6, 0x4b, 0x2c, 0x04, 0x36, 0x85, 0x43, 0x33, 0x8b, 0xf1, 0x60,
- 0x2d, 0x3b, 0x8b, 0xd1, 0xf4, 0x1e, 0xc2, 0x4a, 0xd4, 0x1f, 0xf3, 0xb5, 0x58, 0x28, 0xd6, 0x8e,
- 0x49, 0xd2, 0xe5, 0xed, 0x4a, 0xd4, 0x63, 0xc3, 0x83, 0x2f, 0xf0, 0x79, 0xb1, 0xd5, 0x98, 0x22,
- 0x93, 0x4a, 0xbb, 0x6c, 0xa8, 0x8d, 0xcf, 0x42, 0x68, 0x59, 0x8b, 0xfd, 0x69, 0x5b, 0x0e, 0x2f,
- 0x76, 0xe1, 0xfd, 0xd9, 0x45, 0x0f, 0x02, 0x55, 0x6c, 0xab, 0x54, 0x62, 0x8c, 0x90, 0x3b, 0xa2,
- 0x37, 0xe0, 0xab, 0xb4, 0x28, 0x25, 0x06, 0xe9, 0x72, 0x0b, 0x8f, 0x77, 0xd0, 0x3e, 0xbd, 0x10,
- 0x0d, 0xa0, 0xb1, 0x7b, 0x89, 0xfe, 0xe1, 0xff, 0x6b, 0x44, 0xde, 0xff, 0x7f, 0x1f, 0xfb, 0xe4,
- 0xe7, 0xd8, 0x27, 0x87, 0x63, 0x9f, 0xcc, 0x02, 0x7c, 0xfb, 0xe7, 0xdf, 0x00, 0x00, 0x00, 0xff,
- 0xff, 0xb5, 0x52, 0x37, 0xf2, 0x14, 0x02, 0x00, 0x00,
+ // 356 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x92, 0xc1, 0x4e, 0xc2, 0x40,
+ 0x10, 0x86, 0x5d, 0x4a, 0x0b, 0x1d, 0x44, 0x9b, 0x8d, 0x92, 0x8d, 0x26, 0xa4, 0xe1, 0x62, 0xe3,
+ 0x01, 0x13, 0x34, 0xde, 0xf5, 0x40, 0xc2, 0x0d, 0xf7, 0x05, 0xcc, 0xd2, 0x2e, 0x48, 0xa0, 0xb4,
+ 0xe9, 0xb6, 0x34, 0xc4, 0x17, 0xf4, 0xe8, 0x23, 0x18, 0x9e, 0xc2, 0xa3, 0xd9, 0xd9, 0x85, 0xe0,
+ 0x05, 0xfe, 0x6f, 0x67, 0xa7, 0xff, 0xfc, 0xd3, 0xc2, 0x75, 0xbe, 0x5a, 0x3c, 0x94, 0xbb, 0x5c,
+ 0x2a, 0xf3, 0x3b, 0xcc, 0x8b, 0xac, 0xcc, 0xa8, 0x8b, 0x30, 0x98, 0x81, 0x3f, 0x99, 0xbe, 0x24,
+ 0x49, 0x21, 0x95, 0xa2, 0x77, 0xe0, 0xcd, 0x45, 0xba, 0x5c, 0xef, 0x18, 0x09, 0x49, 0x74, 0x31,
+ 0xba, 0x1c, 0x9a, 0x8e, 0xc9, 0x74, 0x8c, 0xc7, 0xdc, 0x96, 0x29, 0x83, 0x96, 0x30, 0x3d, 0xac,
+ 0x11, 0x92, 0xc8, 0xe7, 0x07, 0xa4, 0x14, 0x9a, 0xa9, 0x50, 0x2b, 0xe6, 0xe0, 0x31, 0xea, 0xc1,
+ 0x2f, 0x01, 0x7f, 0xb2, 0x29, 0x65, 0x31, 0x17, 0xb1, 0xa4, 0x3d, 0xf0, 0x12, 0xb9, 0x5d, 0xc6,
+ 0x12, 0x4d, 0x7c, 0x6e, 0x49, 0x77, 0x6e, 0x44, 0x2a, 0xed, 0x03, 0x51, 0xd3, 0x11, 0x74, 0x8e,
+ 0xd3, 0x49, 0xc5, 0x9c, 0xd0, 0x89, 0x3a, 0xa3, 0xe0, 0x38, 0x95, 0xad, 0xf0, 0xd3, 0x4b, 0x34,
+ 0x00, 0x27, 0x2d, 0x2b, 0xd6, 0x0c, 0x49, 0xd4, 0xe4, 0x5a, 0x6a, 0xc7, 0x8f, 0x5a, 0x5f, 0x60,
+ 0xae, 0x71, 0x34, 0xa4, 0x53, 0xe4, 0xf1, 0x12, 0x0b, 0x9e, 0x49, 0x61, 0x51, 0xcf, 0xa2, 0x3d,
+ 0x58, 0xcb, 0xcc, 0xa2, 0x35, 0xbd, 0x05, 0xbf, 0x10, 0xf5, 0xfb, 0x7c, 0x2d, 0x16, 0x8a, 0xb5,
+ 0x43, 0x12, 0x75, 0x79, 0xbb, 0x10, 0xf5, 0x58, 0xb3, 0xb6, 0x78, 0xab, 0x64, 0x25, 0x15, 0xf3,
+ 0xb1, 0x62, 0x69, 0xf0, 0x09, 0x2e, 0xcf, 0xaa, 0x12, 0xd3, 0x25, 0x52, 0x95, 0x36, 0x33, 0x6a,
+ 0xed, 0xbf, 0x10, 0xa5, 0xac, 0xc5, 0xee, 0xb0, 0x45, 0x8b, 0x27, 0x3b, 0x72, 0xfe, 0xed, 0xa8,
+ 0x07, 0x9e, 0xca, 0xaa, 0x22, 0x96, 0x18, 0xcf, 0xe7, 0x96, 0xe8, 0x15, 0xb8, 0x2a, 0xce, 0x72,
+ 0x89, 0x01, 0xbb, 0xdc, 0xc0, 0xfd, 0x0d, 0xb4, 0x0f, 0x6f, 0x8e, 0x7a, 0xd0, 0xd8, 0x3e, 0x05,
+ 0x67, 0xf8, 0xff, 0x1c, 0x90, 0xd7, 0xf3, 0xaf, 0x7d, 0x9f, 0x7c, 0xef, 0xfb, 0xe4, 0x67, 0xdf,
+ 0x27, 0x33, 0x0f, 0xbf, 0x89, 0xc7, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa5, 0x57, 0x70, 0xc8,
+ 0x2c, 0x02, 0x00, 0x00,
}
diff --git a/virtcontainers/bridgedmacvlan_endpoint.go b/virtcontainers/bridgedmacvlan_endpoint.go
index 700aea34..9179bbbe 100644
--- a/virtcontainers/bridgedmacvlan_endpoint.go
+++ b/virtcontainers/bridgedmacvlan_endpoint.go
@@ -25,7 +25,7 @@ func createBridgedMacvlanNetworkEndpoint(idx int, ifName string, interworkingMod
return &BridgedMacvlanEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx)
}
- netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel)
+ netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel, 0)
if err != nil {
return nil, err
}
diff --git a/virtcontainers/ipvlan_endpoint.go b/virtcontainers/ipvlan_endpoint.go
index 6e924a74..a6ef8179 100644
--- a/virtcontainers/ipvlan_endpoint.go
+++ b/virtcontainers/ipvlan_endpoint.go
@@ -28,7 +28,7 @@ func createIPVlanNetworkEndpoint(idx int, ifName string) (*IPVlanEndpoint, error
// Use tc filtering for ipvlan, since the other inter networking models will
// not work for ipvlan.
interworkingModel := NetXConnectTCFilterModel
- netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel)
+ netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel, 0)
if err != nil {
return nil, err
}
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
index d28d8cce..38e9a204 100644
--- a/virtcontainers/kata_agent.go
+++ b/virtcontainers/kata_agent.go
@@ -621,6 +621,7 @@ func (k *kataAgent) updateInterface(ifc *vcTypes.Interface) (*vcTypes.Interface,
Mtu: resultInterface.Mtu,
HwAddr: resultInterface.HwAddr,
PciAddr: resultInterface.PciAddr,
+ Queues: resultInterface.Queues,
}
return iface, err
}
@@ -2295,6 +2296,7 @@ func (k *kataAgent) convertToKataAgentInterface(iface *vcTypes.Interface) *aType
RawFlags: iface.RawFlags,
HwAddr: iface.HwAddr,
PciAddr: iface.PciAddr,
+ Queues: iface.Queues,
}
}
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
index 46703656..15eb7906 100644
--- a/virtcontainers/network.go
+++ b/virtcontainers/network.go
@@ -140,6 +140,7 @@ type NetworkInfo struct {
Routes []netlink.Route
DNS DNSInfo
VhostUserSocket string
+ Queues uint32
}
// NetworkInterface defines a network interface.
@@ -156,6 +157,7 @@ type TapInterface struct {
TAPIface NetworkInterface
VMFds []*os.File
VhostFds []*os.File
+ Queues uint32
}
// TuntapInterface defines a tap interface
@@ -1047,7 +1049,7 @@ func generateInterfacesAndRoutes(networkNS NetworkNamespace) ([]*vcTypes.Interfa
return ifaces, routes, nil
}
-func createNetworkInterfacePair(idx int, ifName string, interworkingModel NetInterworkingModel) (NetworkInterfacePair, error) {
+func createNetworkInterfacePair(idx int, ifName string, interworkingModel NetInterworkingModel, queues uint32) (NetworkInterfacePair, error) {
uniqueID := uuid.Generate().String()
randomMacAddr, err := generateRandomPrivateMacAddr()
@@ -1062,6 +1064,7 @@ func createNetworkInterfacePair(idx int, ifName string, interworkingModel NetInt
TAPIface: NetworkInterface{
Name: fmt.Sprintf("tap%d_kata", idx),
},
+ Queues: queues,
},
VirtIface: NetworkInterface{
Name: fmt.Sprintf("eth%d", idx),
@@ -1205,7 +1208,7 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li
// interface info json file
if netInfo.VhostUserSocket != "" {
networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found")
- endpoint, err = createVhostUserEndpoint(netInfo, netInfo.VhostUserSocket)
+ endpoint, err = createVhostUserEndpoint(netInfo, netInfo.VhostUserSocket, netInfo.Queues)
return endpoint, err
}
@@ -1220,7 +1223,7 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li
endpoint, err = createMacvtapNetworkEndpoint(netInfo)
case "tap":
networkLogger().Info("tap interface found")
- endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Device)
+ endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Device, netInfo.Queues)
case "tuntap":
if link != nil {
switch link.(*netlink.Tuntap).Mode {
@@ -1231,13 +1234,13 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li
return nil, fmt.Errorf("tun networking device not yet supported")
case 2:
networkLogger().Info("tuntap tap interface found")
- endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, model)
+ endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, model, netInfo.Queues)
default:
return nil, fmt.Errorf("tuntap network %v mode unsupported", link.(*netlink.Tuntap).Mode)
}
}
case "veth":
- endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model)
+ endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model, netInfo.Queues)
case "ipvlan":
endpoint, err = createIPVlanNetworkEndpoint(idx, netInfo.Iface.Name)
default:
diff --git a/virtcontainers/network_test.go b/virtcontainers/network_test.go
index b86f679f..c52c7452 100644
--- a/virtcontainers/network_test.go
+++ b/virtcontainers/network_test.go
@@ -265,7 +265,7 @@ func TestTcRedirectNetwork(t *testing.T) {
err = netlink.LinkAdd(veth)
assert.NoError(err)
- endpoint, err := createVethNetworkEndpoint(1, vethName, NetXConnectTCFilterModel)
+ endpoint, err := createVethNetworkEndpoint(1, vethName, NetXConnectTCFilterModel, 0)
assert.NoError(err)
link, err := netlink.LinkByName(vethName)
diff --git a/virtcontainers/pkg/types/types.go b/virtcontainers/pkg/types/types.go
index dccd92f8..6502259b 100644
--- a/virtcontainers/pkg/types/types.go
+++ b/virtcontainers/pkg/types/types.go
@@ -31,6 +31,8 @@ type Interface struct {
LinkType string `json:"linkType,omitempty"`
// VhostUserSocket is DPDK-backed vHost user ports.
VhostUserSocket string `json:"vhostUserSocket,omitempty"`
+ // Queues specifies the interface queue number
+ Queues uint32 `json:"queues,omitempty"`
}
// Route describes a network route.
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
index 84797e0d..be6e33b9 100644
--- a/virtcontainers/qemu.go
+++ b/virtcontainers/qemu.go
@@ -1464,7 +1464,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro
return nil
}
-func (q *qemu) hotAddNetDevice(deviceName, name, hardAddr string, VMFds, VhostFds []*os.File) error {
+func (q *qemu) hotAddNetDevice(deviceName, name, hardAddr string, VMFds, VhostFds []*os.File, queues uint32) error {
var (
VMFdNames []string
VhostFdNames []string
@@ -1489,7 +1489,7 @@ func (q *qemu) hotAddNetDevice(deviceName, name, hardAddr string, VMFds, VhostFd
return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", name, VMFdNames, VhostFdNames)
}
- return q.qmpMonitorCh.qmp.ExecuteNetdevAdd(q.qmpMonitorCh.ctx, "tap", name, deviceName, "no", "no", 0)
+ return q.qmpMonitorCh.qmp.ExecuteNetdevAdd(q.qmpMonitorCh.ctx, "tap", name, deviceName, "no", "no", int(queues))
}
func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
@@ -1512,7 +1512,7 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
devID := "virtio-" + tap.ID
if op == addDevice {
- if err = q.hotAddNetDevice(tap.TAPIface.Name, tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds); err != nil {
+ if err = q.hotAddNetDevice(tap.TAPIface.Name, tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds, tap.Queues); err != nil {
return err
}
@@ -1542,9 +1542,9 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
}
if machine.Type == QemuCCWVirtio {
devNoHotplug := fmt.Sprintf("fe.%x.%x", bus, addr)
- return q.qmpMonitorCh.qmp.ExecuteNetCCWDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), devNoHotplug, int(q.config.NumVCPUs))
+ return q.qmpMonitorCh.qmp.ExecuteNetCCWDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), devNoHotplug, int(tap.Queues))
}
- return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bus, romFile, int(q.config.NumVCPUs), defaultDisableModern)
+ return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bus, romFile, int(tap.Queues), defaultDisableModern)
}
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
index e6f155a3..d8ab6c1a 100644
--- a/virtcontainers/sandbox.go
+++ b/virtcontainers/sandbox.go
@@ -948,6 +948,7 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) {
},
Addrs: addrs,
VhostUserSocket: inf.VhostUserSocket,
+ Queues: inf.Queues,
}, nil
}
diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go
index 2cf70dce..5a3e7f7e 100644
--- a/virtcontainers/tap_endpoint.go
+++ b/virtcontainers/tap_endpoint.go
@@ -115,7 +115,7 @@ func (endpoint *TapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPat
return nil
}
-func createTapNetworkEndpoint(idx int, ifName string, tapIfName string) (*TapEndpoint, error) {
+func createTapNetworkEndpoint(idx int, ifName string, tapIfName string, queues uint32) (*TapEndpoint, error) {
if idx < 0 {
return &TapEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx)
}
@@ -139,6 +139,10 @@ func createTapNetworkEndpoint(idx int, ifName string, tapIfName string) (*TapEnd
endpoint.TapInterface.TAPIface.Name = tapIfName
}
+ if queues > 0 {
+ endpoint.TapInterface.Queues = queues
+ }
+
return endpoint, nil
}
diff --git a/virtcontainers/tuntap_endpoint.go b/virtcontainers/tuntap_endpoint.go
index b076d694..827d8852 100644
--- a/virtcontainers/tuntap_endpoint.go
+++ b/virtcontainers/tuntap_endpoint.go
@@ -117,12 +117,12 @@ func (endpoint *TuntapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNs
return nil
}
-func createTuntapNetworkEndpoint(idx int, ifName string, hwName net.HardwareAddr, internetworkingModel NetInterworkingModel) (*TuntapEndpoint, error) {
+func createTuntapNetworkEndpoint(idx int, ifName string, hwName net.HardwareAddr, internetworkingModel NetInterworkingModel, queues uint32) (*TuntapEndpoint, error) {
if idx < 0 {
return &TuntapEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx)
}
- netPair, err := createNetworkInterfacePair(idx, ifName, internetworkingModel)
+ netPair, err := createNetworkInterfacePair(idx, ifName, internetworkingModel, queues)
if err != nil {
return nil, err
}
diff --git a/virtcontainers/veth_endpoint.go b/virtcontainers/veth_endpoint.go
index 0f2ec9ba..554b9b22 100644
--- a/virtcontainers/veth_endpoint.go
+++ b/virtcontainers/veth_endpoint.go
@@ -20,12 +20,12 @@ type VethEndpoint struct {
PCIAddr string
}
-func createVethNetworkEndpoint(idx int, ifName string, interworkingModel NetInterworkingModel) (*VethEndpoint, error) {
+func createVethNetworkEndpoint(idx int, ifName string, interworkingModel NetInterworkingModel, queues uint32) (*VethEndpoint, error) {
if idx < 0 {
return &VethEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx)
}
- netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel)
+ netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel, queues)
if err != nil {
return nil, err
}
diff --git a/virtcontainers/veth_endpoint_test.go b/virtcontainers/veth_endpoint_test.go
index 9649b82e..23f1876d 100644
--- a/virtcontainers/veth_endpoint_test.go
+++ b/virtcontainers/veth_endpoint_test.go
@@ -34,7 +34,7 @@ func TestCreateVethNetworkEndpoint(t *testing.T) {
EndpointType: VethEndpointType,
}
- result, err := createVethNetworkEndpoint(4, "", DefaultNetInterworkingModel)
+ result, err := createVethNetworkEndpoint(4, "", DefaultNetInterworkingModel, 0)
assert.NoError(err)
// the resulting ID will be random - so let's overwrite to test the rest of the flow
@@ -68,7 +68,7 @@ func TestCreateVethNetworkEndpointChooseIfaceName(t *testing.T) {
EndpointType: VethEndpointType,
}
- result, err := createVethNetworkEndpoint(4, "eth1", DefaultNetInterworkingModel)
+ result, err := createVethNetworkEndpoint(4, "eth1", DefaultNetInterworkingModel, 0)
assert.NoError(err)
// the resulting ID will be random - so let's overwrite to test the rest of the flow
@@ -95,7 +95,7 @@ func TestCreateVethNetworkEndpointInvalidArgs(t *testing.T) {
}
for _, d := range failingValues {
- _, err := createVethNetworkEndpoint(d.idx, d.ifName, DefaultNetInterworkingModel)
+ _, err := createVethNetworkEndpoint(d.idx, d.ifName, DefaultNetInterworkingModel, 0)
assert.Error(err)
}
}
diff --git a/virtcontainers/vhostuser_endpoint.go b/virtcontainers/vhostuser_endpoint.go
index 2fc3d837..85ef67b4 100644
--- a/virtcontainers/vhostuser_endpoint.go
+++ b/virtcontainers/vhostuser_endpoint.go
@@ -33,6 +33,7 @@ type VhostUserEndpoint struct {
EndpointProperties NetworkInfo
EndpointType EndpointType
PCIAddr string
+ Queues uint32
}
// Properties returns the properties of the interface.
@@ -119,7 +120,7 @@ func (endpoint *VhostUserEndpoint) HotDetach(h hypervisor, netNsCreated bool, ne
}
// Create a vhostuser endpoint
-func createVhostUserEndpoint(netInfo NetworkInfo, socket string) (*VhostUserEndpoint, error) {
+func createVhostUserEndpoint(netInfo NetworkInfo, socket string, queues uint32) (*VhostUserEndpoint, error) {
uniqueID := uuid.Generate().String()
vhostUserEndpoint := &VhostUserEndpoint{
ID: uniqueID,
@@ -127,6 +128,7 @@ func createVhostUserEndpoint(netInfo NetworkInfo, socket string) (*VhostUserEndp
HardAddr: netInfo.Iface.HardwareAddr.String(),
IfaceName: netInfo.Iface.Name,
EndpointType: VhostUserEndpointType,
+ Queues: queues,
}
return vhostUserEndpoint, nil
}
diff --git a/virtcontainers/vhostuser_endpoint_test.go b/virtcontainers/vhostuser_endpoint_test.go
index 584490cc..046ca229 100644
--- a/virtcontainers/vhostuser_endpoint_test.go
+++ b/virtcontainers/vhostuser_endpoint_test.go
@@ -137,7 +137,7 @@ func TestCreateVhostUserEndpoint(t *testing.T) {
EndpointType: VhostUserEndpointType,
}
- result, err := createVhostUserEndpoint(netinfo, socket)
+ result, err := createVhostUserEndpoint(netinfo, socket, 0)
assert.NoError(err)
assert.Exactly(result, expected)
}
--
2.14.3 (Apple Git-98)

2511
runtime/patches/0047-network-add-support-to-get-network-stats-of-vm-throu.patch Normal file
View File

File diff suppressed because it is too large Load Diff

34
runtime/patches/0048-console-fix-the-file-resource-leak.patch Normal file
View File

@ -0,0 +1,34 @@
From d3b8e21829bd671a1717fed8ab645ad2d447899a Mon Sep 17 00:00:00 2001
From: holyfei <yangfeiyu20092010@163.com>
Date: Wed, 19 Aug 2020 22:18:05 +0800
Subject: [PATCH 48/50] console: fix the file resource leak
reason: newConsole will open a file, if error occurs in the
middle proccess, the file resource may leak, use defer close
to prevent it
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
---
cli/console.go | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/cli/console.go b/cli/console.go
index c1555c9f..97dfee99 100644
--- a/cli/console.go
+++ b/cli/console.go
@@ -47,6 +47,12 @@ func newConsole() (*Console, error) {
if err != nil {
return nil, err
}
+ defer func() {
+ if err != nil {
+ master.Close()
+ }
+ }()
+
if err := saneTerminal(master); err != nil {
return nil, err
}
--
2.14.3 (Apple Git-98)

58
runtime/patches/0049-container-fix-the-write-operation-transparently-tran.patch Normal file
View File

@ -0,0 +1,58 @@
From a6fab7014922d85b1105b44fdbb98239b22d3e00 Mon Sep 17 00:00:00 2001
From: holyfei <yangfeiyu20092010@163.com>
Date: Wed, 19 Aug 2020 22:31:57 +0800
Subject: [PATCH 49/50] container: fix the write operation transparently
transmitted to the host
reason:fix the write operation transparently transmitted to the host
when we fullfill the "/etc/hosts","/etc/resolv.conf","/etc/hostname" file in the container,
for example:
```bash
$ docker exec -ti 63 bash
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
---
virtcontainers/container.go | 16 ++++++++++++++--
1 file changed, 14 insertions(+), 2 deletions(-)
diff --git a/virtcontainers/container.go b/virtcontainers/container.go
index 1b89f6ac..6edcb3f2 100644
--- a/virtcontainers/container.go
+++ b/virtcontainers/container.go
@@ -51,6 +51,12 @@ var cdromMajors = map[int64]string{
32: "CM206_CDROM_MAJOR",
}
+var safeCopyFiles = map[string]struct{}{
+ "resolv.conf": {},
+ "hostname": {},
+ "hosts": {},
+}
+
// https://github.com/torvalds/linux/blob/master/include/uapi/linux/major.h
// #define FLOPPY_MAJOR 2
const floppyMajor = int64(2)
@@ -452,12 +458,18 @@ func (c *Container) shareFiles(m Mount, idx int, hostSharedDir, guestSharedDir s
}
filename := fmt.Sprintf("%s-%s-%s", c.id, hex.EncodeToString(randBytes), filepath.Base(m.Destination))
- guestDest := filepath.Join(guestSharedDir, filename)
+ var guestDest string
+ _, needCopy := safeCopyFiles[filepath.Base(m.Destination)]
+ if needCopy {
+ guestDest = filepath.Join(kataGuestStorageDir, filename)
+ } else {
+ guestDest = filepath.Join(guestSharedDir, filename)
+ }
// copy file to contaier's rootfs if filesystem sharing is not supported, otherwise
// bind mount it in the shared directory.
caps := c.sandbox.hypervisor.capabilities()
- if !caps.IsFsSharingSupported() {
+ if !caps.IsFsSharingSupported() || needCopy {
c.Logger().Debug("filesystem sharing is not supported, files will be copied")
fileInfo, err := os.Stat(m.Source)
--
2.14.3 (Apple Git-98)

358
runtime/patches/0050-runtime-add-kata-network-upate-iface-subcommand.patch Normal file
View File

@ -0,0 +1,358 @@
From f943e5d91f6922c599b46727e2dfd5e8e7e5bea8 Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Wed, 19 Aug 2020 22:43:54 +0800
Subject: [PATCH 50/50] runtime: add kata-network upate-iface subcommand
reason: add kata-network update-iface subcommand to
update the exist interface in the Kata VM.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
cli/network.go | 37 +++++++++-
virtcontainers/api.go | 25 +++++--
virtcontainers/implementation.go | 5 ++
virtcontainers/interfaces.go | 1 +
virtcontainers/pkg/vcmock/mock.go | 8 +++
virtcontainers/pkg/vcmock/types.go | 1 +
virtcontainers/sandbox.go | 137 +++++++++++++++++++++++++++++++++++++
7 files changed, 207 insertions(+), 7 deletions(-)
diff --git a/cli/network.go b/cli/network.go
index 3dd0971e..7dce0528 100644
--- a/cli/network.go
+++ b/cli/network.go
@@ -65,8 +65,9 @@ var addIfaceCommand = cli.Command{
"hwAddr":"<mac>",
"linkType":"tap",
"vhostUserSocket":"<path>"
+ "queues":<queues>
}
- device,name,mtu,hwAddr are required, IPAddresses and vhostUserSocket are optional.
+ device,name,mtu,hwAddr are required, IPAddresses, queues and vhostUserSocket are optional.
`,
Flags: []cli.Flag{},
Action: func(context *cli.Context) error {
@@ -104,6 +105,34 @@ var delIfaceCommand = cli.Command{
},
}
+var updateIfaceCommand = cli.Command{
+ Name: "update-iface",
+ Usage: "update the interface in container",
+ ArgsUsage: `update-iface <container-id> file or - for stdin
+ file or stdin for example:
+ {
+ "device":"",
+ "name":"<interface-name>",
+ "IPAddresses":[{"address":"<ip>","mask":"<mask>"}],
+ "mtu":0,
+ "hwAddr":"",
+ "linkType":"",
+ "vhostUserSocket":"",
+ "queues":0
+ }
+ name is required,IPAddresses is optional,device,mtu,hwAddr,linkType,queues and vhostUserSocket should be 0 or empty.
+ `,
+ Flags: []cli.Flag{},
+ Action: func(context *cli.Context) error {
+ ctx, err := cliContextToContext(context)
+ if err != nil {
+ return err
+ }
+
+ return networkModifyCommand(ctx, context.Args().First(), context.Args().Get(1), interfaceType, vcTypes.NetworkOpUpdate)
+ },
+}
+
var listIfacesCommand = cli.Command{
Name: "list-ifaces",
Usage: "list network interfaces in a container",
@@ -262,6 +291,12 @@ func networkModifyCommand(ctx context.Context, containerID, input string, opType
kataLog.WithField("resulting-interface", fmt.Sprintf("%+v", resultingInf)).
WithError(err).Error("delete interface failed")
}
+ case vcTypes.NetworkOpUpdate:
+ resultingInf, err = vci.UpdateInterface(ctx, sandboxID, inf)
+ if err != nil {
+ kataLog.WithField("resulting-interface", fmt.Sprintf("%+v", resultingInf)).
+ WithError(err).Error("update interface failed")
+ }
}
json.NewEncoder(output).Encode(resultingInf)
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
index 7036df8c..ca5412a9 100644
--- a/virtcontainers/api.go
+++ b/virtcontainers/api.go
@@ -889,7 +889,7 @@ func AddDevice(ctx context.Context, sandboxID string, info deviceConfig.DeviceIn
return s.AddDevice(info)
}
-func toggleInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface, add bool) (*vcTypes.Interface, error) {
+func toggleInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface, op vcTypes.NetworkOp) (*vcTypes.Interface, error) {
if sandboxID == "" {
return nil, vcTypes.ErrNeedSandboxID
}
@@ -906,11 +906,16 @@ func toggleInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interfa
}
defer s.releaseStatelessSandbox()
- if add {
+ switch op {
+ case vcTypes.NetworkOpAdd:
return s.AddInterface(inf)
+ case vcTypes.NetworkOpRemove:
+ return s.RemoveInterface(inf)
+ case vcTypes.NetworkOpUpdate:
+ return s.UpdateInterface(inf)
+ default:
+ return nil, fmt.Errorf("operation is not found")
}
-
- return s.RemoveInterface(inf)
}
// AddInterface is the virtcontainers add interface entry point.
@@ -918,7 +923,7 @@ func AddInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface)
span, ctx := trace(ctx, "AddInterface")
defer span.Finish()
- return toggleInterface(ctx, sandboxID, inf, true)
+ return toggleInterface(ctx, sandboxID, inf, vcTypes.NetworkOpAdd)
}
// RemoveInterface is the virtcontainers remove interface entry point.
@@ -926,7 +931,15 @@ func RemoveInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interfa
span, ctx := trace(ctx, "RemoveInterface")
defer span.Finish()
- return toggleInterface(ctx, sandboxID, inf, false)
+ return toggleInterface(ctx, sandboxID, inf, vcTypes.NetworkOpRemove)
+}
+
+// UpdateInterface updates interface entry point in the virtcontainers.
+func UpdateInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) {
+ span, ctx := trace(ctx, "UpdateInterface")
+ defer span.Finish()
+
+ return toggleInterface(ctx, sandboxID, inf, vcTypes.NetworkOpUpdate)
}
// ListInterfaces is the virtcontainers list interfaces entry point.
diff --git a/virtcontainers/implementation.go b/virtcontainers/implementation.go
index 0265d1ed..e4bc4ae6 100644
--- a/virtcontainers/implementation.go
+++ b/virtcontainers/implementation.go
@@ -163,6 +163,11 @@ func (impl *VCImpl) ListInterfaces(ctx context.Context, sandboxID string) ([]*vc
return ListInterfaces(ctx, sandboxID)
}
+// ListInterfaces implements the VC function of the same name.
+func (impl *VCImpl) UpdateInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) {
+ return UpdateInterface(ctx, sandboxID, inf)
+}
+
// UpdateRoutes implements the VC function of the same name.
func (impl *VCImpl) UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) {
return UpdateRoutes(ctx, sandboxID, routes, op)
diff --git a/virtcontainers/interfaces.go b/virtcontainers/interfaces.go
index d8f0be69..0fd12d84 100644
--- a/virtcontainers/interfaces.go
+++ b/virtcontainers/interfaces.go
@@ -51,6 +51,7 @@ type VC interface {
AddInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error)
RemoveInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error)
+ UpdateInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error)
ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTypes.Interface, error)
UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error)
ListRoutes(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error)
diff --git a/virtcontainers/pkg/vcmock/mock.go b/virtcontainers/pkg/vcmock/mock.go
index aa9bae9a..d3bf201f 100644
--- a/virtcontainers/pkg/vcmock/mock.go
+++ b/virtcontainers/pkg/vcmock/mock.go
@@ -273,6 +273,14 @@ func (m *VCMock) ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTyp
return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID)
}
+// UpdateInterface implements the VC function of the same name
+func (m *VCMock) UpdateInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) {
+ if m.UpdateRoutesFunc != nil {
+ return m.UpdateInterfaceFunc(ctx, sandboxID, inf)
+ }
+ return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID)
+}
+
// UpdateRoutes implements the VC function of the same name.
func (m *VCMock) UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) {
if m.UpdateRoutesFunc != nil {
diff --git a/virtcontainers/pkg/vcmock/types.go b/virtcontainers/pkg/vcmock/types.go
index 610b4602..152e66b7 100644
--- a/virtcontainers/pkg/vcmock/types.go
+++ b/virtcontainers/pkg/vcmock/types.go
@@ -73,6 +73,7 @@ type VCMock struct {
AddInterfaceFunc func(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error)
RemoveInterfaceFunc func(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error)
ListInterfacesFunc func(ctx context.Context, sandboxID string) ([]*vcTypes.Interface, error)
+ UpdateInterfaceFunc func(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error)
UpdateRoutesFunc func(ctx context.Context, sandboxID string, routes []*vcTypes.Route) ([]*vcTypes.Route, error)
ListRoutesFunc func(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error)
CleanupContainerFunc func(ctx context.Context, sandboxID, containerID string, force bool) error
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
index d8ab6c1a..174e6cb6 100644
--- a/virtcontainers/sandbox.go
+++ b/virtcontainers/sandbox.go
@@ -1031,6 +1031,54 @@ func (s *Sandbox) RemoveInterface(inf *vcTypes.Interface) (*vcTypes.Interface, e
return nil, nil
}
+// UpdateInterface updates the nic.
+func (s *Sandbox) UpdateInterface(inf *vcTypes.Interface) (*vcTypes.Interface, error) {
+ err := checkUpdateInterfaceInfo(inf)
+ if err != nil {
+ return nil, err
+ }
+
+ nicInGuest := false
+
+ endpoint := s.getEndpointByName(inf.Name)
+ // nic is not in local endpoints
+ if endpoint == nil {
+ interfaceInGuestInf := s.getInterfaceInfByName(inf.Name)
+ if interfaceInGuestInf == nil {
+ return nil, fmt.Errorf("can not find the network interface")
+ }
+
+ nicInGuest = true
+ endpoint, err = s.generateEndpointForStore(interfaceInGuestInf)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ // update endpoint ip
+ err = s.updateEndpointIP(endpoint, inf)
+ if err != nil {
+ return nil, err
+ }
+
+ s.Logger().WithField("endpoint-type", endpoint.Type()).Info("Update endpoint")
+ grpcIf, err := s.agent.updateInterface(inf)
+ if err != nil {
+ return nil, err
+ }
+
+ if nicInGuest {
+ s.networkNS.Endpoints = append(s.networkNS.Endpoints, endpoint)
+ }
+
+ if err = s.Save(); err != nil {
+ s.Logger().WithError(err).Error("failed to store the network information")
+ return nil, err
+ }
+
+ return grpcIf, nil
+}
+
// ListInterfaces lists all nics and their configurations in the sandbox.
func (s *Sandbox) ListInterfaces() ([]*vcTypes.Interface, error) {
return s.agent.listInterfaces()
@@ -2431,6 +2479,95 @@ func (s *Sandbox) IsCompatOldCNI() bool {
return s.config.NetworkConfig.EnableCompatOldCNI
}
+func checkUpdateInterfaceInfo(inf *vcTypes.Interface) error {
+ if inf.Mtu > 0 || inf.Queues > 0 || inf.Device != "" || inf.VhostUserSocket != "" || inf.LinkType != "" || inf.HwAddr != "" {
+ return fmt.Errorf("device,mtu,hwAddr,linkType,queues and vhostUserSocket should be 0 or empty")
+ }
+
+ return nil
+}
+
+// getEndpointByName returns the endpoint by name.
+func (s *Sandbox) getEndpointByName(name string) Endpoint {
+ if name == "" {
+ return nil
+ }
+
+ for i, endpoint := range s.networkNS.Endpoints {
+ if endpoint.Name() == name {
+ return s.networkNS.Endpoints[i]
+ }
+ }
+
+ return nil
+}
+
+// getInterfaceInfByName returns the interface information.
+func (s *Sandbox) getInterfaceInfByName(name string) *vcTypes.Interface {
+ if name == "" {
+ return nil
+ }
+
+ nics, err := s.agent.listInterfaces()
+ if err != nil {
+ return nil
+ }
+
+ for _, i := range nics {
+ if i.Name == name {
+ return i
+ }
+ }
+
+ return nil
+}
+
+// generateEndpointForStore only generates endpoint information for store,it does not create a endpoint
+func (s *Sandbox) generateEndpointForStore(inf *vcTypes.Interface) (*TapEndpoint, error) {
+ netInfo, err := s.generateNetInfo(inf)
+ if err != nil {
+ return nil, err
+ }
+
+ endpoint := &TapEndpoint{
+ TapInterface: TapInterface{
+ Name: inf.Name,
+ TAPIface: NetworkInterface{
+ HardAddr: inf.HwAddr,
+ },
+ },
+ EndpointType: TapEndpointType,
+ }
+
+ endpoint.SetProperties(netInfo)
+
+ return endpoint, nil
+}
+
+func (s *Sandbox) updateEndpointIP(endpoint Endpoint, inf *vcTypes.Interface) error {
+ if endpoint == nil {
+ return fmt.Errorf("endpoint is nil")
+ }
+
+ netInfo := endpoint.Properties()
+
+ var addrs []netlink.Addr
+ for _, addr := range inf.IPAddresses {
+ netlinkAddrStr := fmt.Sprintf("%s/%s", addr.Address, addr.Mask)
+ netlinkAddr, err := netlink.ParseAddr(netlinkAddrStr)
+ if err != nil {
+ return fmt.Errorf("could not parse %q: %v", netlinkAddrStr, err)
+ }
+
+ addrs = append(addrs, *netlinkAddr)
+ }
+
+ netInfo.Addrs = addrs
+ endpoint.SetProperties(netInfo)
+
+ return nil
+}
+
// updateStaticSandboxResources update sandbox's cpu and memory resource passed by
// sandbox_cpu and sandbox_mem annotations
func updateStaticSandboxResources(sandboxConfig *SandboxConfig) error {
--
2.14.3 (Apple Git-98)

78
runtime/patches/0051-network-fix-del-iface-doesn-t-delete-the-tap-interfa.patch Normal file
View File

@ -0,0 +1,78 @@
From be0d60f5fa88267afb26125681a217ef9476e133 Mon Sep 17 00:00:00 2001
From: holyfei <yangfeiyu20102011@163.com>
Date: Sat, 19 Sep 2020 12:47:14 +0800
Subject: [PATCH] runtime: fix del-iface doesn't delete the tap interface in
the host problem
reason: kata-runtime add-iface support add exist tap device in the host or
new created tap device by kata-network add-iface process. If add interface
is exist tap device, del-iface will not delete the tap device, so we need
to judge the deleted interface is exist device or new created, which is judged
by the VMFds of the tap Link and del-iface will load this info from the
network.json in 1.7.0 version. However, 1.11.1 version don't write the VMFds
into the persiste.json file, del-iface can not figure out which tap is exist
already or new created, so cause the this problem.
---
virtcontainers/endpoint.go | 2 ++
virtcontainers/persist/api/network.go | 5 +++++
virtcontainers/tap_endpoint.go | 1 +
3 files changed, 8 insertions(+)
diff --git a/virtcontainers/endpoint.go b/virtcontainers/endpoint.go
index 7efcf49c..3618792e 100644
--- a/virtcontainers/endpoint.go
+++ b/virtcontainers/endpoint.go
@@ -129,6 +129,7 @@ func saveTapIf(tapif *TapInterface) *persistapi.TapInterface {
HardAddr: tapif.TAPIface.HardAddr,
Addrs: tapif.TAPIface.Addrs,
},
+ VMFds: tapif.VMFds,
}
}
@@ -167,6 +168,7 @@ func loadTapIf(tapif *persistapi.TapInterface) *TapInterface {
HardAddr: tapif.TAPIface.HardAddr,
Addrs: tapif.TAPIface.Addrs,
},
+ VMFds: tapif.VMFds,
}
}
diff --git a/virtcontainers/persist/api/network.go b/virtcontainers/persist/api/network.go
index 53c6de44..c5611767 100644
--- a/virtcontainers/persist/api/network.go
+++ b/virtcontainers/persist/api/network.go
@@ -7,6 +7,8 @@
package persistapi
import (
+ "os"
+
"github.com/vishvananda/netlink"
)
@@ -45,6 +47,9 @@ type TapInterface struct {
Name string
TAPIface NetworkInterface
// remove VMFds and VhostFds
+ // add VMFds back to judge a tap interface is exist before add-iface
+ // or new created by kata-network add-iface
+ VMFds []*os.File
}
// TuntapInterface defines a tap interface
diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go
index 5a3e7f7e..0b6002aa 100644
--- a/virtcontainers/tap_endpoint.go
+++ b/virtcontainers/tap_endpoint.go
@@ -199,6 +199,7 @@ func unTapNetwork(endpoint *TapEndpoint) error {
return nil
}
+ networkLogger().Debug("untap the new created tap interface")
name := endpoint.TapInterface.TAPIface.Name
netHandle, err := netlink.NewHandle()
if err != nil {
--
2.11.0

1720
runtime/patches/0052-runtime-add-support-of-new-sandbox-StratoVirt.patch Normal file
View File

File diff suppressed because it is too large Load Diff

199
runtime/patches/0053-kata-runtime-add-interface-for-host-cgroup.patch Normal file
View File

@ -0,0 +1,199 @@
From 31bbb64a0682e326b354ac54c6596402fc20a977 Mon Sep 17 00:00:00 2001
From: holyfei <yangfeiyu20092010@163.com>
Date: Tue, 8 Sep 2020 20:03:41 +0800
Subject: [PATCH 1/5] kata-runtime: add interface for host cgroup
reason: add interface for host cgroup, including
CreateSandboxCgroup, DestroySandboxCgroup and
AddPidToSandboxCgroup, GetSandboxCgroupPath
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
---
virtcontainers/cgroups.go | 111 +++++++++++++++++++++++++++++++++++++++
virtcontainers/implementation.go | 20 +++++++
virtcontainers/interfaces.go | 5 ++
3 files changed, 136 insertions(+)
diff --git a/virtcontainers/cgroups.go b/virtcontainers/cgroups.go
index 4459df5..df0ec30 100644
--- a/virtcontainers/cgroups.go
+++ b/virtcontainers/cgroups.go
@@ -8,13 +8,20 @@ package virtcontainers
import (
"bufio"
+ "context"
+ "encoding/json"
"fmt"
+ "io/ioutil"
"os"
"path/filepath"
"strings"
"github.com/containerd/cgroups"
specs "github.com/opencontainers/runtime-spec/specs-go"
+ "github.com/sirupsen/logrus"
+
+ "github.com/kata-containers/runtime/virtcontainers/store"
+ "github.com/kata-containers/runtime/virtcontainers/types"
)
type cgroupPather interface {
@@ -30,6 +37,110 @@ const cgroupKataPath = "/kata/"
var cgroupsLoadFunc = cgroups.Load
var cgroupsNewFunc = cgroups.New
+// CreateSandboxCgroup create cgroup based on the first container's cgroupPath of sandbox
+func CreateSandboxCgroup(ctx context.Context, path string) error {
+ if path == "" {
+ return fmt.Errorf("sandbox cgroupPath shouldn't be empty!")
+ }
+
+ var cgroupPath string
+ vcpuCgroupPath := filepath.Join(path, "vcpu")
+ if filepath.IsAbs(vcpuCgroupPath) {
+ cgroupPath = filepath.Clean(vcpuCgroupPath)
+ } else {
+ cgroupPath = filepath.Join(filepath.Clean("/" + vcpuCgroupPath))
+ }
+
+ resources := specs.LinuxResources{}
+ if _, err := cgroupsNewFunc(cgroups.V1, cgroups.StaticPath(cgroupPath), &resources); err != nil {
+ return fmt.Errorf("Could not create cgroup for %v: %v", cgroupPath, err)
+ }
+
+ return nil
+}
+
+// DestroySandboxCgroup destroy the cgroup dir created for sandbox
+func DestroySandboxCgroup(ctx context.Context, cgroupPath string) error {
+ return deleteCgroup(cgroups.V1, cgroupPath)
+}
+
+func deleteCgroup(hierarchy cgroups.Hierarchy, cgroupPath string) error {
+ if cgroupPath == "" {
+ logrus.Warn("delete cgroupPath shouldn't be empty!")
+ return nil
+ }
+
+ cgroup, err := cgroupsLoadFunc(hierarchy,
+ cgroups.StaticPath(cgroupPath))
+
+ if err == cgroups.ErrCgroupDeleted {
+ // cgroup already deleted
+ return nil
+ }
+
+ if err != nil {
+ return fmt.Errorf("Could not load cgroup %v: %v", cgroupPath, err)
+ }
+
+ // move running process here, that way cgroup can be removed
+ parent, err := parentCgroup(hierarchy, cgroupPath)
+ if err != nil {
+ // parent cgroup doesn't exist, that means there are no process running
+ // and the container cgroup was removed.
+ logrus.Warn(err)
+ return nil
+ }
+
+ if err := cgroup.MoveTo(parent); err != nil {
+ // Don't fail, cgroup can be deleted
+ logrus.Warnf("Could not move container process into parent cgroup: %v", err)
+ }
+
+ if err := cgroup.Delete(); err != nil {
+ return fmt.Errorf("Could not delete cgroup %v: %v", cgroupPath, err)
+ }
+
+ return nil
+}
+
+// GetSandboxCgroupPath return the cgroup path of specified sandbox
+func GetSandboxCgroupPath(ctx context.Context, sandboxID string) (string, error) {
+ stateFilePath := filepath.Join(store.RunStoragePath(), sandboxID, store.StateFile)
+
+ fileData, err := ioutil.ReadFile(stateFilePath)
+ if err != nil {
+ return "", err
+ }
+
+ state := types.SandboxState{}
+
+ if err := json.Unmarshal(fileData, &state); err != nil {
+ return "", err
+ }
+
+ if state.CgroupPath == "" {
+ return "", fmt.Errorf("get sandbox cgroup path error: cgroupPath is empty")
+ }
+
+ return state.CgroupPath, nil
+}
+
+// AddPidToSandboxCgroup add kata-runtime create process to cgroup
+// the pid will be added to the cgroup of "<path>/vcpu"
+func AddPidToSandboxCgroup(ctx context.Context, pid int, cgroupPath string) error {
+ cgroup, err := cgroupsLoadFunc(cgroups.V1, cgroups.StaticPath(cgroupPath))
+ if err != nil {
+ return err
+ }
+
+ err = cgroup.Add(cgroups.Process{Pid: pid})
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
// V1Constraints returns the cgroups that are compatible with the VC architecture
// and hypervisor, constraints can be applied to these cgroups.
func V1Constraints() ([]cgroups.Subsystem, error) {
diff --git a/virtcontainers/implementation.go b/virtcontainers/implementation.go
index e4bc4ae..fedc51f 100644
--- a/virtcontainers/implementation.go
+++ b/virtcontainers/implementation.go
@@ -188,3 +188,23 @@ func (impl *VCImpl) CleanupContainer(ctx context.Context, sandboxID, containerID
func (impl *VCImpl) UpdateIPVSRule(ctx context.Context, sandboxID string, IPVSRule *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
return UpdateIPVSRule(ctx, sandboxID, IPVSRule)
}
+
+// CreateSandboxCgroup implements the VC function of the same name.
+func (impl *VCImpl) CreateSandboxCgroup(ctx context.Context, sandboxCgroupPath string) error {
+ return CreateSandboxCgroup(ctx, sandboxCgroupPath)
+}
+
+// DestroySandboxCgroup implements the VC function of the same name.
+func (impl *VCImpl) DestroySandboxCgroup(ctx context.Context, sandboxCgroupPath string) error {
+ return DestroySandboxCgroup(ctx, sandboxCgroupPath)
+}
+
+// AddPidToSandboxCgroup implements the VC function of the same name.
+func (impl *VCImpl) AddPidToSandboxCgroup(ctx context.Context, pid int, sandboxCgroupPath string) error {
+ return AddPidToSandboxCgroup(ctx, pid, sandboxCgroupPath)
+}
+
+// GetSandboxCgroupPath implements the VC function of the same name.
+func (impl *VCImpl) GetSandboxCgroupPath(ctx context.Context, sandboxID string) (string, error) {
+ return GetSandboxCgroupPath(ctx, sandboxID)
+}
diff --git a/virtcontainers/interfaces.go b/virtcontainers/interfaces.go
index 0fd12d8..4d166e0 100644
--- a/virtcontainers/interfaces.go
+++ b/virtcontainers/interfaces.go
@@ -24,6 +24,11 @@ type VC interface {
SetLogger(ctx context.Context, logger *logrus.Entry)
SetFactory(ctx context.Context, factory Factory)
+ CreateSandboxCgroup(ctx context.Context, sandboxCgroupPath string) error
+ DestroySandboxCgroup(ctx context.Context, sandboxCgroupPath string) error
+ AddPidToSandboxCgroup(ctx context.Context, pid int, sandboxCgroupPath string) error
+ GetSandboxCgroupPath(ctx context.Context, sandboxID string) (string, error)
+
CreateSandbox(ctx context.Context, sandboxConfig SandboxConfig) (VCSandbox, error)
DeleteSandbox(ctx context.Context, sandboxID string) (VCSandbox, error)
FetchSandbox(ctx context.Context, sandboxID string) (VCSandbox, error)
--
1.8.3.1

376
runtime/patches/0054-kata-runtime-add-sandbox-cgroup-with-vcpu-and-emulat.patch Normal file
View File

@ -0,0 +1,376 @@
From 98a3c4677261e1c0364015f36928cddfb0af253e Mon Sep 17 00:00:00 2001
From: holyfei <yangfeiyu20092010@163.com>
Date: Wed, 9 Sep 2020 16:45:24 +0800
Subject: [PATCH 2/5] kata-runtime: add sandbox cgroup with vcpu and emulator
switch
reason: add sandbox cgroup with vcpu and emulator switch, if
sandbox_cgroup_with_emulator is true, it will overload the feature
of sandbox_cgroup_only, there will be two cgroups, vcpu and emulator
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
---
cli/config/configuration-qemu.toml.in | 12 +++++++++
cli/kata-env.go | 38 ++++++++++++++-------------
pkg/katautils/config.go | 18 +++++++------
virtcontainers/api.go | 4 ++-
virtcontainers/container.go | 6 ++---
virtcontainers/persist.go | 30 +++++++++++----------
virtcontainers/persist/api/config.go | 2 ++
virtcontainers/pkg/annotations/annotations.go | 2 ++
virtcontainers/pkg/oci/utils.go | 13 +++++++++
virtcontainers/sandbox.go | 23 +++++++++++-----
10 files changed, 97 insertions(+), 51 deletions(-)
diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in
index e57a954..fae88f9 100644
--- a/cli/config/configuration-qemu.toml.in
+++ b/cli/config/configuration-qemu.toml.in
@@ -477,6 +477,18 @@ enable_compat_old_cni = true
# See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType
sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
+# It is a new host cgroup solution to limit the kata resouce in the host different from the
+# community original solution.If sandbox_cgroup_with_emulator is enabled, it will override
+# the config of sandbox_cgroup_only. Each Pod corresponds to a pod level cgroup directory
+# which is named with sandboxID. In each pod level cgroup, it contains two sub cgroup
+# directory: vcpu and emulator, these two sub cgroup only valid in the CPU cgroup subsystem,
+# because we just want to distinguish the emulator main thread and vcpu thread in the CPU
+# cgroup subsystem.And with this config enabled, kata-runtime and related sub processes will
+# added into the vcpu cgroup directory with resource limited, and qemu main thread and other
+# non-vcpu threads will be moved into the emulator cgroup without resource limit, which will
+# improve the IO throughput for kata-containers.
+sandbox_cgroup_with_emulator = true
+
# Enabled experimental feature list, format: ["a", "b"].
# Experimental features are features not stable enough for production,
# they may break compatibility, and are prepared for a big version bump.
diff --git a/cli/kata-env.go b/cli/kata-env.go
index d8a6068..48026fe 100644
--- a/cli/kata-env.go
+++ b/cli/kata-env.go
@@ -63,15 +63,16 @@ type RuntimeConfigInfo struct {
// RuntimeInfo stores runtime details.
type RuntimeInfo struct {
- Version RuntimeVersionInfo
- Config RuntimeConfigInfo
- Debug bool
- Trace bool
- DisableGuestSeccomp bool
- DisableNewNetNs bool
- SandboxCgroupOnly bool
- Experimental []exp.Feature
- Path string
+ Version RuntimeVersionInfo
+ Config RuntimeConfigInfo
+ Debug bool
+ Trace bool
+ DisableGuestSeccomp bool
+ DisableNewNetNs bool
+ SandboxCgroupOnly bool
+ SandboxCgroupWithEmulator bool
+ Experimental []exp.Feature
+ Path string
}
type VersionInfo struct {
@@ -194,15 +195,16 @@ func getRuntimeInfo(configFile string, config oci.RuntimeConfig) RuntimeInfo {
runtimePath, _ := os.Executable()
return RuntimeInfo{
- Debug: config.Debug,
- Trace: config.Trace,
- Version: runtimeVersion,
- Config: runtimeConfig,
- Path: runtimePath,
- DisableNewNetNs: config.DisableNewNetNs,
- SandboxCgroupOnly: config.SandboxCgroupOnly,
- Experimental: config.Experimental,
- DisableGuestSeccomp: config.DisableGuestSeccomp,
+ Debug: config.Debug,
+ Trace: config.Trace,
+ Version: runtimeVersion,
+ Config: runtimeConfig,
+ Path: runtimePath,
+ DisableNewNetNs: config.DisableNewNetNs,
+ SandboxCgroupOnly: config.SandboxCgroupOnly,
+ SandboxCgroupWithEmulator: config.SandboxCgroupWithEmulator,
+ Experimental: config.Experimental,
+ DisableGuestSeccomp: config.DisableGuestSeccomp,
}
}
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
index 3365b3f..89e46f6 100644
--- a/pkg/katautils/config.go
+++ b/pkg/katautils/config.go
@@ -139,14 +139,15 @@ type proxy struct {
}
type runtime struct {
- Debug bool `toml:"enable_debug"`
- Tracing bool `toml:"enable_tracing"`
- DisableNewNetNs bool `toml:"disable_new_netns"`
- EnableCompatOldCNI bool `toml:"enable_compat_old_cni"`
- DisableGuestSeccomp bool `toml:"disable_guest_seccomp"`
- SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"`
- Experimental []string `toml:"experimental"`
- InterNetworkModel string `toml:"internetworking_model"`
+ Debug bool `toml:"enable_debug"`
+ Tracing bool `toml:"enable_tracing"`
+ DisableNewNetNs bool `toml:"disable_new_netns"`
+ EnableCompatOldCNI bool `toml:"enable_compat_old_cni"`
+ DisableGuestSeccomp bool `toml:"disable_guest_seccomp"`
+ SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"`
+ SandboxCgroupWithEmulator bool `toml:"sandbox_cgroup_with_emulator"`
+ Experimental []string `toml:"experimental"`
+ InterNetworkModel string `toml:"internetworking_model"`
}
type shim struct {
@@ -1252,6 +1253,7 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool, debugFlag
}
config.SandboxCgroupOnly = tomlConf.Runtime.SandboxCgroupOnly
+ config.SandboxCgroupWithEmulator = tomlConf.Runtime.SandboxCgroupWithEmulator
config.DisableNewNetNs = tomlConf.Runtime.DisableNewNetNs
config.EnableCompatOldCNI = tomlConf.Runtime.EnableCompatOldCNI
for _, f := range tomlConf.Runtime.Experimental {
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
index ca5412a..08bcbb5 100644
--- a/virtcontainers/api.go
+++ b/virtcontainers/api.go
@@ -103,7 +103,9 @@ func createSandboxFromConfig(ctx context.Context, sandboxConfig SandboxConfig, f
}()
// Move runtime to sandbox cgroup so all process are created there.
- if s.config.SandboxCgroupOnly {
+ if s.config.SandboxCgroupWithEmulator{
+ // emulator
+ } else if s.config.SandboxCgroupOnly {
if err := s.setupSandboxCgroup(); err != nil {
return nil, err
}
diff --git a/virtcontainers/container.go b/virtcontainers/container.go
index 4060ebb..1b70382 100644
--- a/virtcontainers/container.go
+++ b/virtcontainers/container.go
@@ -1009,7 +1009,7 @@ func (c *Container) create() (err error) {
}
}
- if !rootless.IsRootless() && !c.sandbox.config.SandboxCgroupOnly {
+ if !rootless.IsRootless() && !c.sandbox.config.SandboxCgroupOnly && !c.sandbox.config.SandboxCgroupWithEmulator {
if err = c.cgroupsCreate(); err != nil {
return
}
@@ -1034,7 +1034,7 @@ func (c *Container) delete() error {
}
// If running rootless, there are no cgroups to remove
- if !c.sandbox.config.SandboxCgroupOnly || !rootless.IsRootless() {
+ if !c.sandbox.config.SandboxCgroupWithEmulator && (!c.sandbox.config.SandboxCgroupOnly || !rootless.IsRootless()) {
if err := c.cgroupsDelete(); err != nil {
return err
}
@@ -1348,7 +1348,7 @@ func (c *Container) update(resources specs.LinuxResources) error {
}
}
- if !c.sandbox.config.SandboxCgroupOnly {
+ if !c.sandbox.config.SandboxCgroupWithEmulator && !c.sandbox.config.SandboxCgroupOnly {
if err := c.cgroupsUpdate(resources); err != nil {
return err
}
diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go
index fe00bf9..efa4506 100644
--- a/virtcontainers/persist.go
+++ b/virtcontainers/persist.go
@@ -194,13 +194,14 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
InterworkingModel: int(sconfig.NetworkConfig.InterworkingModel),
},
- ShmSize: sconfig.ShmSize,
- SharePidNs: sconfig.SharePidNs,
- Stateful: sconfig.Stateful,
- SystemdCgroup: sconfig.SystemdCgroup,
- SandboxCgroupOnly: sconfig.SandboxCgroupOnly,
- DisableGuestSeccomp: sconfig.DisableGuestSeccomp,
- Cgroups: sconfig.Cgroups,
+ ShmSize: sconfig.ShmSize,
+ SharePidNs: sconfig.SharePidNs,
+ Stateful: sconfig.Stateful,
+ SystemdCgroup: sconfig.SystemdCgroup,
+ SandboxCgroupOnly: sconfig.SandboxCgroupOnly,
+ SandboxCgroupWithEmulator: sconfig.SandboxCgroupWithEmulator,
+ DisableGuestSeccomp: sconfig.DisableGuestSeccomp,
+ Cgroups: sconfig.Cgroups,
}
for _, e := range sconfig.Experimental {
@@ -485,13 +486,14 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
InterworkingModel: NetInterworkingModel(savedConf.NetworkConfig.InterworkingModel),
},
- ShmSize: savedConf.ShmSize,
- SharePidNs: savedConf.SharePidNs,
- Stateful: savedConf.Stateful,
- SystemdCgroup: savedConf.SystemdCgroup,
- SandboxCgroupOnly: savedConf.SandboxCgroupOnly,
- DisableGuestSeccomp: savedConf.DisableGuestSeccomp,
- Cgroups: savedConf.Cgroups,
+ ShmSize: savedConf.ShmSize,
+ SharePidNs: savedConf.SharePidNs,
+ Stateful: savedConf.Stateful,
+ SystemdCgroup: savedConf.SystemdCgroup,
+ SandboxCgroupOnly: savedConf.SandboxCgroupOnly,
+ SandboxCgroupWithEmulator: savedConf.SandboxCgroupWithEmulator,
+ DisableGuestSeccomp: savedConf.DisableGuestSeccomp,
+ Cgroups: savedConf.Cgroups,
}
for _, name := range savedConf.Experimental {
diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go
index 3a2df32..28204fc 100644
--- a/virtcontainers/persist/api/config.go
+++ b/virtcontainers/persist/api/config.go
@@ -258,6 +258,8 @@ type SandboxConfig struct {
// SandboxCgroupOnly enables cgroup only at podlevel in the host
SandboxCgroupOnly bool
+ SandboxCgroupWithEmulator bool
+
DisableGuestSeccomp bool
// Experimental enables experimental features
diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go
index 528dfa6..96c4ef2 100644
--- a/virtcontainers/pkg/annotations/annotations.go
+++ b/virtcontainers/pkg/annotations/annotations.go
@@ -215,6 +215,8 @@ const (
// SandboxCgroupOnly is a sandbox annotation that determines if kata processes are managed only in sandbox cgroup.
SandboxCgroupOnly = kataAnnotRuntimePrefix + "sandbox_cgroup_only"
+ SandboxCgroupWithEmulator = kataAnnotRuntimePrefix + "sandbox_cgroup_with_emulator"
+
// Experimental is a sandbox annotation that determines if experimental features enabled.
Experimental = kataAnnotRuntimePrefix + "experimental"
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
index 3b2af75..91067fb 100644
--- a/virtcontainers/pkg/oci/utils.go
+++ b/virtcontainers/pkg/oci/utils.go
@@ -139,6 +139,8 @@ type RuntimeConfig struct {
//Determines kata processes are managed only in sandbox cgroup
SandboxCgroupOnly bool
+ SandboxCgroupWithEmulator bool
+
//Experimental features enabled
Experimental []exp.Feature
}
@@ -746,6 +748,15 @@ func addRuntimeConfigOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e
sbConfig.SandboxCgroupOnly = sandboxCgroupOnly
}
+ if value, ok := ocispec.Annotations[vcAnnotations.SandboxCgroupWithEmulator]; ok {
+ sandboxCgroupWithEmulator, err := strconv.ParseBool(value)
+ if err != nil {
+ return fmt.Errorf("error parsing annotation for sandbox_cgroup_with_emulator : Please specify boolean value 'true|false'")
+ }
+
+ sbConfig.SandboxCgroupWithEmulator = sandboxCgroupWithEmulator
+ }
+
if value, ok := ocispec.Annotations[vcAnnotations.Experimental]; ok {
features := strings.Split(value, " ")
sbConfig.Experimental = []exp.Feature{}
@@ -869,6 +880,8 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid, c
SandboxCgroupOnly: runtime.SandboxCgroupOnly,
+ SandboxCgroupWithEmulator: runtime.SandboxCgroupWithEmulator,
+
DisableGuestSeccomp: runtime.DisableGuestSeccomp,
// Q: Is this really necessary? @weizhang555
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
index 174e6cb..b479cf5 100644
--- a/virtcontainers/sandbox.go
+++ b/virtcontainers/sandbox.go
@@ -126,6 +126,8 @@ type SandboxConfig struct {
// SandboxCgroupOnly enables cgroup only at podlevel in the host
SandboxCgroupOnly bool
+ SandboxCgroupWithEmulator bool
+
DisableGuestSeccomp bool
// Experimental features enabled
@@ -1532,8 +1534,9 @@ func (s *Sandbox) Stats() (SandboxStats, error) {
var path string
var cgroupSubsystems cgroups.Hierarchy
-
- if s.config.SandboxCgroupOnly {
+ if !s.config.SandboxCgroupWithEmulator {
+ // vcpu and emulator
+ } else if s.config.SandboxCgroupOnly {
cgroupSubsystems = cgroups.V1
path = s.state.CgroupPath
} else {
@@ -1793,7 +1796,9 @@ func (s *Sandbox) HotplugAddDevice(device api.Device, devType config.DeviceType)
span, _ := s.trace("HotplugAddDevice")
defer span.Finish()
- if s.config.SandboxCgroupOnly {
+ if s.config.SandboxCgroupWithEmulator {
+ // emulator
+ } else if s.config.SandboxCgroupOnly {
// We are about to add a device to the hypervisor,
// the device cgroup MUST be updated since the hypervisor
// will need access to such device
@@ -1849,7 +1854,9 @@ func (s *Sandbox) HotplugAddDevice(device api.Device, devType config.DeviceType)
// Sandbox implement DeviceReceiver interface from device/api/interface.go
func (s *Sandbox) HotplugRemoveDevice(device api.Device, devType config.DeviceType) error {
defer func() {
- if s.config.SandboxCgroupOnly {
+ if s.config.SandboxCgroupWithEmulator {
+
+ } else if s.config.SandboxCgroupOnly {
// Remove device from cgroup, the hypervisor
// should not have access to such device anymore.
hdev := device.GetHostPath()
@@ -2107,7 +2114,7 @@ func (s *Sandbox) cgroupsUpdate() error {
// If Kata is configured for SandboxCgroupOnly, the VMM and its processes are already
// in the Kata sandbox cgroup (inherited). No need to move threads/processes, and we should
// rely on parent's cgroup CPU/memory values
- if s.config.SandboxCgroupOnly {
+ if s.config.SandboxCgroupWithEmulator || s.config.SandboxCgroupOnly {
return nil
}
@@ -2154,7 +2161,9 @@ func (s *Sandbox) cgroupsDelete() error {
var path string
var cgroupSubsystems cgroups.Hierarchy
- if s.config.SandboxCgroupOnly {
+ if s.config.SandboxCgroupWithEmulator {
+ // emulator
+ } else if s.config.SandboxCgroupOnly {
return s.cgroupMgr.Destroy()
}
@@ -2197,7 +2206,7 @@ func (s *Sandbox) constrainHypervisor(cgroup cgroups.Cgroup) error {
// Kata/VMM into account, Kata may fail to boot due to being overconstrained.
// If !SandboxCgroupOnly, place the VMM into an unconstrained cgroup, and the vCPU threads into constrained
// cgroup
- if s.config.SandboxCgroupOnly {
+ if s.config.SandboxCgroupOnly || s.config.SandboxCgroupWithEmulator {
// Kata components were moved into the sandbox-cgroup already, so VMM
// will already land there as well. No need to take action
return nil
--
1.8.3.1

428
runtime/patches/0055-kata_runtime-support-host-cgroup-with-emulator-polic.patch Normal file
View File

@ -0,0 +1,428 @@
From ce7523dfe1bb60cf54254e16a103fd3fc9503618 Mon Sep 17 00:00:00 2001
From: yangfeiyu <yangfeiyu2@huawei.com>
Date: Thu, 17 Sep 2020 10:38:38 +0800
Subject: [PATCH 3/5] kata_runtime: support host cgroup with emulator policy
reason: support host cgroup with emulator policy when
sandbox_cgroup_with_emulator is set true
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
---
cli/create.go | 38 ++++++++++++
virtcontainers/api.go | 10 ++-
virtcontainers/cgroups.go | 132 ++++++++++++++++++++++++++++++++++------
virtcontainers/persist/fs/fs.go | 8 +++
virtcontainers/pkg/oci/utils.go | 14 +++++
virtcontainers/sandbox.go | 70 ++++++++++++++++++++-
6 files changed, 250 insertions(+), 22 deletions(-)
diff --git a/cli/create.go b/cli/create.go
index 02cb2c5..b14434b 100644
--- a/cli/create.go
+++ b/cli/create.go
@@ -11,6 +11,7 @@ import (
"errors"
"fmt"
"os"
+ "path/filepath"
"github.com/kata-containers/runtime/pkg/katautils"
vc "github.com/kata-containers/runtime/virtcontainers"
@@ -134,11 +135,48 @@ func create(ctx context.Context, containerID, bundlePath, console, pidFilePath s
var process vc.Process
switch containerType {
case vc.PodSandbox:
+ if runtimeConfig.SandboxCgroupWithEmulator {
+ // create the sandbox level cgroup
+ cgroupPath := ociSpec.Linux.CgroupsPath
+ if err = vci.CreateSandboxCgroup(ctx, cgroupPath); err != nil {
+ return err
+ }
+
+ defer func() {
+ if err != nil {
+ _ = vci.DestroySandboxCgroup(ctx, cgroupPath)
+ }
+ }()
+
+ // add kata-runtime create process into <path>/vcpu cgroup
+ vcpuCgroupPath := filepath.Join(cgroupPath, "vcpu")
+ if err = vci.AddPidToSandboxCgroup(ctx, os.Getpid(), vcpuCgroupPath); err != nil {
+ return err
+ }
+ }
+
_, process, err = katautils.CreateSandbox(ctx, vci, ociSpec, runtimeConfig, rootFs, containerID, bundlePath, console, disableOutput, systemdCgroup, false)
if err != nil {
return err
}
case vc.PodContainer:
+ if runtimeConfig.SandboxCgroupWithEmulator {
+ sandboxID, err := oci.GetSandboxIDFromAnnotations(&ociSpec)
+ if err != nil {
+ return fmt.Errorf("container annotation doesn't contain sandboxID")
+ }
+
+ sandboxCgroupPath, err := vci.GetSandboxCgroupPath(ctx, sandboxID)
+ if err != nil {
+ return err
+ }
+
+ // add kata-runtime create process into <path>/vcpu cgroup
+ vcpuCgroupPath := filepath.Join(sandboxCgroupPath, "vcpu")
+ if err = vci.AddPidToSandboxCgroup(ctx, os.Getpid(), vcpuCgroupPath); err != nil {
+ return err
+ }
+ }
process, err = katautils.CreateContainer(ctx, vci, nil, ociSpec, rootFs, containerID, bundlePath, console, disableOutput, false)
if err != nil {
return err
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
index 08bcbb5..38c8235 100644
--- a/virtcontainers/api.go
+++ b/virtcontainers/api.go
@@ -103,9 +103,7 @@ func createSandboxFromConfig(ctx context.Context, sandboxConfig SandboxConfig, f
}()
// Move runtime to sandbox cgroup so all process are created there.
- if s.config.SandboxCgroupWithEmulator{
- // emulator
- } else if s.config.SandboxCgroupOnly {
+ if !s.config.SandboxCgroupWithEmulator && s.config.SandboxCgroupOnly {
if err := s.setupSandboxCgroup(); err != nil {
return nil, err
}
@@ -129,6 +127,12 @@ func createSandboxFromConfig(ctx context.Context, sandboxConfig SandboxConfig, f
return nil, err
}
+ if s.config.SandboxCgroupWithEmulator {
+ if err := s.setupHostCgroupsWithEmulator(); err != nil {
+ return nil, err
+ }
+ }
+
// Create Containers
if err = s.createContainers(); err != nil {
return nil, err
diff --git a/virtcontainers/cgroups.go b/virtcontainers/cgroups.go
index df0ec30..65d2001 100644
--- a/virtcontainers/cgroups.go
+++ b/virtcontainers/cgroups.go
@@ -9,19 +9,15 @@ package virtcontainers
import (
"bufio"
"context"
- "encoding/json"
"fmt"
- "io/ioutil"
"os"
"path/filepath"
+ "strconv"
"strings"
"github.com/containerd/cgroups"
specs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/sirupsen/logrus"
-
- "github.com/kata-containers/runtime/virtcontainers/store"
- "github.com/kata-containers/runtime/virtcontainers/types"
)
type cgroupPather interface {
@@ -32,7 +28,11 @@ type cgroupPather interface {
// unconstrained cgroups are placed here.
// for example /sys/fs/cgroup/memory/kata/$CGPATH
// where path is defined by the containers manager
-const cgroupKataPath = "/kata/"
+const (
+ cgroupKataPath = "/kata/"
+ vcpuCgroupName = "vcpu"
+ emulatorCgroupName = "emulator"
+)
var cgroupsLoadFunc = cgroups.Load
var cgroupsNewFunc = cgroups.New
@@ -105,24 +105,16 @@ func deleteCgroup(hierarchy cgroups.Hierarchy, cgroupPath string) error {
// GetSandboxCgroupPath return the cgroup path of specified sandbox
func GetSandboxCgroupPath(ctx context.Context, sandboxID string) (string, error) {
- stateFilePath := filepath.Join(store.RunStoragePath(), sandboxID, store.StateFile)
-
- fileData, err := ioutil.ReadFile(stateFilePath)
+ config, err := loadSandboxConfig(sandboxID)
if err != nil {
return "", err
}
- state := types.SandboxState{}
-
- if err := json.Unmarshal(fileData, &state); err != nil {
- return "", err
- }
-
- if state.CgroupPath == "" {
- return "", fmt.Errorf("get sandbox cgroup path error: cgroupPath is empty")
+ if config.Cgroups == nil {
+ return "", fmt.Errorf("the cgroups of sandbox %s is nil", sandboxID)
}
- return state.CgroupPath, nil
+ return config.Cgroups.Path, nil
}
// AddPidToSandboxCgroup add kata-runtime create process to cgroup
@@ -276,3 +268,107 @@ func validCPUResources(cpuSpec *specs.LinuxCPU) *specs.LinuxCPU {
return &cpu
}
+
+// getQemuTaskWithoutVcpu filter out tasks under /proc/{qemu pid}/task, to find out the task of not VCPU,
+// VCPU task is filtered by "query-cpus" qmp command
+func getQemuTaskWithoutVcpu(sandbox *Sandbox, vmPid int) []int {
+ procPath := fmt.Sprintf("/proc/%d/task", vmPid)
+
+ dirReader, err := os.Open(procPath)
+ if err != nil {
+ logrus.Warningf("cannot open %s: %s", procPath, err)
+ return nil
+ }
+
+ defer dirReader.Close()
+
+ dirs, err := dirReader.Readdirnames(0)
+ if err != nil {
+ logrus.Warningf("walking dirs in %s failed: %s", procPath, err)
+ return nil
+ }
+
+ vcpuThreadInfo, err := sandbox.hypervisor.getThreadIDs()
+ if err != nil {
+ logrus.Warnf("get hypervisor Thread ID failed: %v", err)
+ return nil
+ }
+
+ var vcpuThreadIDs []int
+ for _, value := range vcpuThreadInfo.vcpus {
+ vcpuThreadIDs = append(vcpuThreadIDs, value)
+ }
+
+ var allThreadIDs []int
+ for _, dir := range dirs {
+ p, err := strconv.Atoi(dir)
+ if err != nil {
+ logrus.Warnf("can not change string dir: %s to int type", dir)
+ return nil
+ }
+
+ allThreadIDs = append(allThreadIDs, p)
+ }
+
+ nonVCPUThreads := diffSlice(allThreadIDs, vcpuThreadIDs)
+
+ return nonVCPUThreads
+}
+
+func pulloutQemuThread(sandbox *Sandbox, vmPid int, path string) error {
+ control, err := cgroups.New(cgroups.SingleSubsystem(cgroups.V1, cgroups.Cpu),
+ cgroups.StaticPath(path),
+ &specs.LinuxResources{})
+ if err != nil {
+ return err
+ }
+ taskIds := getQemuTaskWithoutVcpu(sandbox, vmPid)
+ if len(taskIds) == 0 {
+ logrus.Warnf("no taskId id in qemu other than vcpu found of pid %d", vmPid)
+ return nil
+ }
+ for _, taskId := range taskIds {
+ if err := control.AddTask(cgroups.Process{
+ Pid: taskId,
+ }); err != nil {
+ logrus.Errorf("failed to add task %d to cgroup of %s", taskId, path)
+ return err
+ }
+ }
+
+ return nil
+}
+
+// checkCgroupExist check cgroup exist or not
+func checkCgroupExist(hierarchy cgroups.Hierarchy, path string) bool {
+ subSystems, _ := hierarchy()
+ for _, s := range cgroupPathers(subSystems) {
+ if _, err := os.Lstat(s.Path(path)); err != nil {
+ if os.IsNotExist(err) {
+ return false
+ }
+ }
+ }
+
+ return true
+}
+
+// diffSlice return the s1 - s2
+func diffSlice(s1, s2 []int) []int {
+ var diffSlice []int
+ for _, p := range s1 {
+ if !isInSlice(p, s2) {
+ diffSlice = append(diffSlice, p)
+ }
+ }
+ return diffSlice
+}
+
+func isInSlice(i int, s []int) bool {
+ for _, v := range s {
+ if i == v {
+ return true
+ }
+ }
+ return false
+}
diff --git a/virtcontainers/persist/fs/fs.go b/virtcontainers/persist/fs/fs.go
index 38efdba..641d64e 100644
--- a/virtcontainers/persist/fs/fs.go
+++ b/virtcontainers/persist/fs/fs.go
@@ -14,6 +14,8 @@ import (
"path/filepath"
"syscall"
+ "github.com/opencontainers/runc/libcontainer/configs"
+
persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
"github.com/sirupsen/logrus"
)
@@ -78,6 +80,12 @@ func (fs *FS) ToDisk(ss persistapi.SandboxState, cs map[string]persistapi.Contai
return fmt.Errorf("sandbox container id required")
}
+ if ss.Config.Cgroups == nil {
+ ss.Config.Cgroups = &configs.Cgroup{
+ Path: ss.CgroupPath,
+ }
+ }
+
fs.sandboxState = &ss
fs.containerState = cs
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
index 91067fb..e8ef41b 100644
--- a/virtcontainers/pkg/oci/utils.go
+++ b/virtcontainers/pkg/oci/utils.go
@@ -1136,3 +1136,17 @@ func validateSandboxDNS(value string) error {
return nil
}
+
+func GetSandboxIDFromAnnotations(s *specs.Spec) (string, error) {
+ if s == nil {
+ return "", fmt.Errorf("spec is nil")
+ }
+
+ for _, v := range CRISandboxNameKeyList {
+ if sandboxID, ok := s.Annotations[v]; ok {
+ return sandboxID, nil
+ }
+ }
+
+ return "", fmt.Errorf("failed to find the sandbox ID")
+}
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
index b479cf5..ca4e700 100644
--- a/virtcontainers/sandbox.go
+++ b/virtcontainers/sandbox.go
@@ -2162,7 +2162,9 @@ func (s *Sandbox) cgroupsDelete() error {
var cgroupSubsystems cgroups.Hierarchy
if s.config.SandboxCgroupWithEmulator {
- // emulator
+ if err := deleteCgroup(cgroups.V1, s.state.CgroupPath); err != nil {
+ return err
+ }
} else if s.config.SandboxCgroupOnly {
return s.cgroupMgr.Destroy()
}
@@ -2381,6 +2383,68 @@ func (s *Sandbox) setupSandboxCgroup() error {
return nil
}
+func (s *Sandbox) setupHostCgroupsWithEmulator() error {
+ if len(s.config.Containers) == 0 {
+ return nil
+ }
+
+ sandboxContainerSpec := s.GetPatchedOCISpec()
+ if sandboxContainerSpec == nil {
+ return fmt.Errorf("sandbox container should not be empty")
+ }
+
+ // Set sandbox's cgroup path
+ s.state.CgroupPath = sandboxContainerSpec.Linux.CgroupsPath
+
+ if !checkCgroupExist(cgroups.V1, s.state.CgroupPath) {
+ return fmt.Errorf("sandbox's cgroup %s doesn't exist", s.state.CgroupPath)
+ }
+
+ // pull out qemu threads other than vcpu to the cgroup of "<path>/emulator"
+ if s.config.HypervisorType == QemuHypervisor {
+ emulatorCgroupPath := filepath.Join(s.state.CgroupPath, emulatorCgroupName)
+ hypervisorPids := s.hypervisor.getPids()
+ if len(hypervisorPids) == 0 || hypervisorPids[0] == 0 {
+ return fmt.Errorf("hypervisor pid: %v invalid", hypervisorPids)
+ }
+ if err := pulloutQemuThread(s, hypervisorPids[0], emulatorCgroupPath); err != nil {
+ return err
+ }
+ }
+
+ // limit cpu to "<path>/vcpu"
+ vcpuCgroupPath := filepath.Join(s.state.CgroupPath, vcpuCgroupName)
+ vcpuResources := specs.LinuxResources{
+ CPU: s.cpuResources(),
+ }
+ if err := applyResourceLimit(&vcpuResources, vcpuCgroupPath); err != nil {
+ return err
+ }
+
+ // limit blkio resource to "<path>"
+
+ // limit files resource
+
+ return nil
+}
+
+func applyResourceLimit(resources *specs.LinuxResources, cgroupPath string) error {
+ if resources == nil {
+ return nil
+ }
+
+ control, err := cgroupsLoadFunc(cgroups.V1, cgroups.StaticPath(cgroupPath))
+ if err != nil {
+ return fmt.Errorf("could not load cgroup %v: %v", cgroupPath, err)
+ }
+
+ if err = control.Update(resources); err != nil {
+ return fmt.Errorf("could not update cgroup %v: %v", cgroupPath, err)
+ }
+
+ return nil
+}
+
// GetPatchedOCISpec returns sandbox's OCI specification
// This OCI specification was patched when the sandbox was created
// by containerCapabilities(), SetEphemeralStorageType() and others
@@ -2452,6 +2516,10 @@ func (s *Sandbox) forceDeleteSandbox() {
c.forceDeleteContainer()
}
+ if err := deleteCgroup(cgroups.V1, s.state.CgroupPath); err != nil {
+ s.Logger().Warnf("sandbox forceDelete cgroups failed: %v", err)
+ }
+
globalSandboxList.removeSandbox(s.id)
if s.monitor != nil {
--
1.8.3.1

318
runtime/patches/0056-kata_runtime-support-the-blkio-in-host-cgroups.patch Normal file
View File

@ -0,0 +1,318 @@
From f4b899b933a3a30fc378ceb4d8855778cd783d9a Mon Sep 17 00:00:00 2001
From: holyfei <yangfeiyu20092010@163.com>
Date: Fri, 18 Sep 2020 16:54:11 +0800
Subject: [PATCH 4/5] kata_runtime: support the blkio in host cgroups
reason: support the blkio in host cgroups, run with paras
--annotation io.kubernetes.docker.type=podsandbox --annotation io.katacontainers.blkio_cgroup=
'{"blkiocgroup":[{"path":"/dev/sda","limits":[{"type":"throttle_read_bps","value":400}]}]}'
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
---
vendor/github.com/containerd/cgroups/blkio.go | 6 ++
virtcontainers/cgroups.go | 102 ++++++++++++++++++++++++++
virtcontainers/pkg/annotations/annotations.go | 3 +
virtcontainers/pkg/oci/utils.go | 50 ++++++++++++-
virtcontainers/sandbox.go | 6 ++
virtcontainers/utils/utils.go | 22 ++++++
6 files changed, 186 insertions(+), 3 deletions(-)
diff --git a/vendor/github.com/containerd/cgroups/blkio.go b/vendor/github.com/containerd/cgroups/blkio.go
index 7c498de..485ff7b 100644
--- a/vendor/github.com/containerd/cgroups/blkio.go
+++ b/vendor/github.com/containerd/cgroups/blkio.go
@@ -23,6 +23,7 @@ import (
"io/ioutil"
"os"
"path/filepath"
+ "reflect"
"strconv"
"strings"
@@ -55,6 +56,11 @@ func (b *blkioController) Create(path string, resources *specs.LinuxResources) e
return nil
}
for _, t := range createBlkioSettings(resources.BlockIO) {
+ ptr := reflect.ValueOf(t.value)
+ if ptr.Kind() == reflect.Ptr && ptr.IsNil() {
+ continue
+ }
+
if t.value != nil {
if err := ioutil.WriteFile(
filepath.Join(b.Path(path), fmt.Sprintf("blkio.%s", t.name)),
diff --git a/virtcontainers/cgroups.go b/virtcontainers/cgroups.go
index 65d2001..e8c5a7b 100644
--- a/virtcontainers/cgroups.go
+++ b/virtcontainers/cgroups.go
@@ -9,6 +9,7 @@ package virtcontainers
import (
"bufio"
"context"
+ "encoding/json"
"fmt"
"os"
"path/filepath"
@@ -16,6 +17,8 @@ import (
"strings"
"github.com/containerd/cgroups"
+ "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
+ "github.com/kata-containers/runtime/virtcontainers/utils"
specs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/sirupsen/logrus"
)
@@ -32,8 +35,46 @@ const (
cgroupKataPath = "/kata/"
vcpuCgroupName = "vcpu"
emulatorCgroupName = "emulator"
+
+ // BlkioThrottleReadBps is the key to fetch throttle_read_bps
+ BlkioThrottleReadBps = "throttle_read_bps"
+
+ // BlkioThrottleWriteBps is the key to fetch throttle_write_bps
+ BlkioThrottleWriteBps = "throttle_write_bps"
+
+ // BlkioThrottleReadIOPS is the key to fetch throttle_read_iops
+ BlkioThrottleReadIOPS = "throttle_read_iops"
+
+ // BlkioThrottleWriteIOPS is the key to fetch throttle_write_iops
+ BlkioThrottleWriteIOPS = "throttle_write_iops"
+
+ // BlkioWeight is the key to fetch blkio_weight
+ BlkioWeight = "blkio_weight"
+
+ // BlkioLeafWeight is the key to fetch blkio_leaf_weight
+ BlkioLeafWeight = "blkio_leaf_weight"
)
+// BlkioCgroup for Linux cgroup 'blkio' data exchange
+type BlkioCgroup struct {
+ // Items specifies per cgroup values
+ Items []BlockIOCgroupItem `json:"blkiocgroup,omitempty"`
+}
+
+type BlockIOCgroupItem struct {
+ // Path represent path of blkio device
+ Path string `json:"path,omitempty"`
+ // Limits specifies the blkio type and value
+ Limits []IOLimit `json:"limits,omitempty"`
+}
+
+type IOLimit struct {
+ // Type specifies IO type
+ Type string `json:"type,omitempty"`
+ // Value specifies rate or weight value
+ Value uint64 `json:"value,omitempty"`
+}
+
var cgroupsLoadFunc = cgroups.Load
var cgroupsNewFunc = cgroups.New
@@ -372,3 +413,64 @@ func isInSlice(i int, s []int) bool {
}
return false
}
+
+func (s *Sandbox) blockIOResource() *specs.LinuxBlockIO {
+ value, ok := s.config.Annotations[annotations.BlkioCgroupTypeKey]
+ if !ok {
+ return nil
+ }
+
+ var blkioCgroupParse BlkioCgroup
+ var linuxBlkio specs.LinuxBlockIO
+ if err := json.Unmarshal([]byte(value), &blkioCgroupParse); err != nil {
+ s.Logger().Errorf("blkio_cgroup Unmarshal error:%v", err)
+ return nil
+ }
+
+ for _, item := range blkioCgroupParse.Items {
+ if item.Limits == nil {
+ s.Logger().Errorf("%v:limits have none data", item)
+ return nil
+ }
+ for _, limit := range item.Limits {
+ if item.Path != "" {
+ major, minor, err := utils.GetDeviceByPath(item.Path)
+ if err != nil {
+ s.Logger().Errorf("failed to find major and minor of device %s: %v", item.Path, err)
+ return nil
+ }
+ td := specs.LinuxThrottleDevice{
+ Rate: limit.Value,
+ }
+ td.Major = major
+ td.Minor = minor
+ switch limit.Type {
+ case BlkioThrottleReadBps:
+ linuxBlkio.ThrottleReadBpsDevice = append(linuxBlkio.ThrottleReadBpsDevice, td)
+ case BlkioThrottleWriteBps:
+ linuxBlkio.ThrottleWriteBpsDevice = append(linuxBlkio.ThrottleWriteBpsDevice, td)
+ case BlkioThrottleReadIOPS:
+ linuxBlkio.ThrottleReadIOPSDevice = append(linuxBlkio.ThrottleReadIOPSDevice, td)
+ case BlkioThrottleWriteIOPS:
+ linuxBlkio.ThrottleWriteIOPSDevice = append(linuxBlkio.ThrottleWriteIOPSDevice, td)
+ default:
+ s.Logger().Errorf("the type of throtlle device：%s is not surpport", limit.Type)
+ return nil
+ }
+ } else {
+ switch limit.Type {
+ case BlkioWeight:
+ weightUint16 := uint16(limit.Value)
+ linuxBlkio.Weight = &weightUint16
+ case BlkioLeafWeight:
+ weightLeafUint16 := uint16(limit.Value)
+ linuxBlkio.LeafWeight = &weightLeafUint16
+ default:
+ s.Logger().Errorf("the type:%s is not one of the supported types of blkio_weight and blkio_leaf_weight", limit.Type)
+ return nil
+ }
+ }
+ }
+ }
+ return &linuxBlkio
+}
diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go
index 96c4ef2..c35993e 100644
--- a/virtcontainers/pkg/annotations/annotations.go
+++ b/virtcontainers/pkg/annotations/annotations.go
@@ -21,6 +21,9 @@ const (
ContainerTypeKey = kataAnnotationsPrefix + "pkg.oci.container_type"
SandboxConfigPathKey = kataAnnotationsPrefix + "config_path"
+
+ // BlkioCgroupTypeKey is the annotation key to fetch sandbox blkio cgroup values
+ BlkioCgroupTypeKey = kataAnnotationsPrefix + "blkio_cgroup"
)
// Annotations related to Hypervisor configuration
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
index e8ef41b..643753b 100644
--- a/virtcontainers/pkg/oci/utils.go
+++ b/virtcontainers/pkg/oci/utils.go
@@ -38,9 +38,10 @@ type annotationContainerType struct {
type annotationHandler func(value string) error
var annotationHandlerList = map[string]annotationHandler{
- vcAnnotations.StaticCPUTypeKey: validateSandboxCPU,
- vcAnnotations.StaticMemTypeKey: validateSandboxMem,
- vcAnnotations.SandboxDNSTypeKey: validateSandboxDNS,
+ vcAnnotations.StaticCPUTypeKey: validateSandboxCPU,
+ vcAnnotations.StaticMemTypeKey: validateSandboxMem,
+ vcAnnotations.SandboxDNSTypeKey: validateSandboxDNS,
+ vcAnnotations.BlkioCgroupTypeKey: validateBlkioCgroup,
}
var (
@@ -1076,6 +1077,7 @@ func validateOtherSandboxAnnotations(annotation, value string) error {
// addOtherSandboxAnnotation add self defined annotation for sandbox
func addOtherSandboxAnnotation(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error {
otherSandboxAnnotationsKey := []string{
+ vcAnnotations.BlkioCgroupTypeKey,
vcAnnotations.StaticCPUTypeKey,
vcAnnotations.StaticMemTypeKey,
}
@@ -1137,6 +1139,48 @@ func validateSandboxDNS(value string) error {
return nil
}
+func validateBlkioCgroup(value string) error {
+ var linuxBlkioCgroup vc.BlkioCgroup
+ if err := json.Unmarshal([]byte(value), &linuxBlkioCgroup); err != nil {
+ return fmt.Errorf("blkio_cgroup Unmarshal error:%v, value is %s", err, value)
+ }
+
+ if linuxBlkioCgroup.Items == nil {
+ return fmt.Errorf("BlkioCgroup:%v fetch none data", linuxBlkioCgroup)
+ }
+
+ // pathNull used to judge path for blkio.weight/blkio.leaf_weight only once
+ pathNull := false
+ for _, item := range linuxBlkioCgroup.Items {
+ if pathNull {
+ return fmt.Errorf("too many paths for blkio.weight/blkio.leaf_weight")
+ }
+ if item.Path == "" && !pathNull {
+ pathNull = true
+ if len(item.Limits) > 2 || len(item.Limits) == 0 {
+ return fmt.Errorf("the format of values for blkio.weight or blkio.leaf_weight is wrong")
+ }
+
+ for _, ioLimit := range item.Limits {
+ switch ioLimit.Type {
+ case vc.BlkioWeight:
+ // Blkio.weight, between 10 and 1000
+ if ioLimit.Value < 10 || ioLimit.Value > 1000 {
+ return fmt.Errorf("blkio.weight:%v must be between 10 and 1000", ioLimit.Value)
+ }
+ case vc.BlkioLeafWeight:
+ return fmt.Errorf("the blkio.leaf_weight is not supported now")
+ default:
+ return fmt.Errorf("the type of blkio device：%s is not surpport", ioLimit.Type)
+ }
+ }
+ } else if _, _, err := utils.GetDeviceByPath(item.Path); err != nil {
+ return fmt.Errorf("failed to find major and minor of device %s: %v", item.Path, err)
+ }
+ }
+ return nil
+}
+
func GetSandboxIDFromAnnotations(s *specs.Spec) (string, error) {
if s == nil {
return "", fmt.Errorf("spec is nil")
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
index ca4e700..9284f99 100644
--- a/virtcontainers/sandbox.go
+++ b/virtcontainers/sandbox.go
@@ -2422,6 +2422,12 @@ func (s *Sandbox) setupHostCgroupsWithEmulator() error {
}
// limit blkio resource to "<path>"
+ blkioResources := specs.LinuxResources{
+ BlockIO: s.blockIOResource(),
+ }
+ if err := applyResourceLimit(&blkioResources, vcpuCgroupPath); err != nil {
+ return err
+ }
// limit files resource
diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go
index 36ac67a..d4dad40 100644
--- a/virtcontainers/utils/utils.go
+++ b/virtcontainers/utils/utils.go
@@ -370,6 +370,28 @@ func GetPhysicalCPUNumber() int {
return cpuNum
}
+// GetDeviceByPath returns the device No.
+func GetDeviceByPath(path string) (int64, int64, error) {
+ if path == "" {
+ return -1, -1, fmt.Errorf("Path cannot be empty")
+ }
+
+ path, err := filepath.EvalSymlinks(path)
+ if err != nil {
+ return -1, -1, err
+ }
+
+ stat := syscall.Stat_t{}
+ err = syscall.Stat(path, &stat)
+ if err != nil {
+ return -1, -1, err
+ }
+
+ major := int64(stat.Rdev / 256)
+ minor := int64(stat.Rdev % 256)
+ return major, minor, nil
+}
+
func RoundVCPUNumber(value string) (int, error) {
cpuNum, err := strconv.ParseFloat(value, 64)
if err != nil || cpuNum < minCPUs {
--
1.8.3.1

5834
runtime/patches/0057-kata-runtime-support-files-limit-in-host-cgroups.patch Normal file
View File

File diff suppressed because it is too large Load Diff

69
runtime/patches/0058-runtime-fix-sandboxRuntimeRootPath-left-problem.patch Normal file
View File

@ -0,0 +1,69 @@
From faffb26c307556e1d84399060d7aef1753e9f41a Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Mon, 21 Sep 2020 21:52:48 -0400
Subject: [PATCH] runtime: fix sandboxRuntimeRootPath left problem
reason: If pod_sandbox type container deleted before pod_container
type containers in the sandbox, which leads to kata-runtime delete
pod_container process create a new sandboxRuntimeRootPath in the
/run/vc/sbs/<sandbox-id> dir.
So, we fix this problem by check sandboxRuntimeRootPath is exist
before fetchSandbox function is called.
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
cli/delete.go | 2 +-
virtcontainers/sandbox.go | 18 ++++++++++++++++++
2 files changed, 19 insertions(+), 1 deletion(-)
diff --git a/cli/delete.go b/cli/delete.go
index 871ac40d..09552b9a 100644
--- a/cli/delete.go
+++ b/cli/delete.go
@@ -87,7 +87,7 @@ func delete(ctx context.Context, containerID string, force bool) error {
// container is deleted before pod_container type container, just return nil
// and let containerd delete container operations continue
if strings.Contains(err.Error(), "no such file or directory") {
- kataLog.Warnf("pod_sandbox deleted before pod_container: %v", err)
+ kataLog.Warnf("pod_sandbox container deleted before pod_container in the sandbox: %v", err)
return nil
}
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
index de8652fc..9f5d3c08 100644
--- a/virtcontainers/sandbox.go
+++ b/virtcontainers/sandbox.go
@@ -729,6 +729,13 @@ func fetchSandbox(ctx context.Context, sandboxID string) (sandbox *Sandbox, err
var config SandboxConfig
+ // check sandbox runtime root path(for example: /run/vc/sbs/<sandbox-id>)
+ // exist or not before fetch sandbox config file
+ if !checkSandboxRuntimeRootPathExist(sandboxID) {
+ sandboxRuntimeRootPath := store.SandboxRuntimeRootPath(sandboxID)
+ return nil, fmt.Errorf("sandbox %s does not exist, %s no such file or directory", sandboxID,sandboxRuntimeRootPath)
+ }
+
// Try to load sandbox config from old store at first.
c, ctx, err := loadSandboxConfigFromOldStore(ctx, sandboxID)
if err != nil {
@@ -2707,3 +2714,14 @@ func updateStaticSandboxResources(sandboxConfig *SandboxConfig) error {
return nil
}
+
+// checkSandboxRuntimeRootPathExist check /run/vc/sbs/<sandbox-id>/ dir exist or not
+func checkSandboxRuntimeRootPathExist(sandboxID string) bool {
+ sandboxRuntimeRootPath := store.SandboxRuntimeRootPath(sandboxID)
+ _, err := os.Stat(sandboxRuntimeRootPath)
+ if os.IsNotExist(err) {
+ return false
+ }
+
+ return true
+}
\ No newline at end of file
--
2.11.0

42
runtime/patches/0059-runtime-fix-invalid-cmdline-when-start-sandbox-stratovirt.patch Normal file
View File

@ -0,0 +1,42 @@
From e10fe6ecf2d895fe009a080bb34334ff63739f3e Mon Sep 17 00:00:00 2001
From: LiangZhang <zhangliang5@Huawei.com>
Date: Thu, 24 Sep 2020 14:38:04 +0800
Subject: [PATCH] runtime: fix invalid cmdline when start sandbox stratovirt
reason:
1. MemorySize is in type of uint32, value larger than 4096 will trigger overflow.
2. cannot disable seccomp now.
Signed-off-by: LiangZhang <zhangliang5@Huawei.com>
---
virtcontainers/stratovirt.go | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/virtcontainers/stratovirt.go b/virtcontainers/stratovirt.go
index 7c156d5..6db8dc0 100644
--- a/virtcontainers/stratovirt.go
+++ b/virtcontainers/stratovirt.go
@@ -146,7 +146,7 @@ func (s *stratovirt) startSandbox(timeout int) error {
params = append(params, "-append "+s.getKernelCmdLine())
params = append(params, fmt.Sprintf("-smp %d", s.config.NumVCPUs))
- params = append(params, fmt.Sprintf("-m %d", s.config.MemorySize*1024*1024))
+ params = append(params, fmt.Sprintf("-m %d", uint64(s.config.MemorySize)*1024*1024))
params = append(params, fmt.Sprintf("-chardev id=charconsole0,path=%s", s.consolePath))
// add devices to cmdline
@@ -178,6 +178,11 @@ func (s *stratovirt) startSandbox(timeout int) error {
params = append(params, fmt.Sprintf("-D %s/stratovirt.log", dir))
}
+ // disable Seccomp
+ if s.sandbox.config.DisableGuestSeccomp {
+ params = append(params, "-disable-seccomp")
+ }
+
s.Logger().Info("StratoVirt start with params: ", strings.Join(params, " "))
dir := filepath.Join(store.RunVMStoragePath(), s.id)
--
1.8.3.1

94
runtime/patches/0060-runtime-fix-cmd-params-of-direct-use-stratovirt-bina.patch Normal file
View File

@ -0,0 +1,94 @@
From b2b11cc8fb2144c13f4f9138b4420248ea200fb6 Mon Sep 17 00:00:00 2001
From: LiangZhang <zhangliang5@Huawei.com>
Date: Sun, 27 Sep 2020 18:10:18 +0800
Subject: [PATCH] runtime: fix cmd params of direct use stratovirt binary
reason: when directly use stratovirt binary, not through bash, it will cause problem.
Signed-off-by: LiangZhang <zhangliang5@Huawei.com>
---
virtcontainers/stratovirt.go | 28 ++++++++++++++--------------
1 file changed, 14 insertions(+), 14 deletions(-)
diff --git a/virtcontainers/stratovirt.go b/virtcontainers/stratovirt.go
index 6db8dc0..020135e 100644
--- a/virtcontainers/stratovirt.go
+++ b/virtcontainers/stratovirt.go
@@ -133,21 +133,21 @@ func (s *stratovirt) startSandbox(timeout int) error {
defer span.Finish()
var params []string
- params = append(params, "-name "+fmt.Sprintf("sandbox-%s", s.id))
- params = append(params, "-api-channel unix:"+s.socketPath)
+ params = append(params, "-name", fmt.Sprintf("sandbox-%s", s.id))
+ params = append(params, "-api-channel", fmt.Sprintf("unix:%s", s.socketPath))
if kernelPath, err := s.config.KernelAssetPath(); err == nil {
- params = append(params, "-kernel "+kernelPath)
+ params = append(params, "-kernel", kernelPath)
}
if initrdPath, err := s.config.InitrdAssetPath(); err == nil {
- params = append(params, "-initrd "+initrdPath)
+ params = append(params, "-initrd", initrdPath)
}
- params = append(params, "-append "+s.getKernelCmdLine())
- params = append(params, fmt.Sprintf("-smp %d", s.config.NumVCPUs))
- params = append(params, fmt.Sprintf("-m %d", uint64(s.config.MemorySize)*1024*1024))
- params = append(params, fmt.Sprintf("-chardev id=charconsole0,path=%s", s.consolePath))
+ params = append(params, "-append", s.getKernelCmdLine())
+ params = append(params, "-smp", fmt.Sprintf("%d", s.config.NumVCPUs))
+ params = append(params, "-m", fmt.Sprintf("%d", uint64(s.config.MemorySize)*1024*1024))
+ params = append(params, "-chardev", fmt.Sprintf("id=charconsole0,path=%s", s.consolePath))
// add devices to cmdline
for _, d := range s.devices {
@@ -156,14 +156,14 @@ func (s *stratovirt) startSandbox(timeout int) error {
name := v.Name()
mac := v.HardwareAddr()
tapName := v.NetworkPair().TapInterface.TAPIface.Name
- params = append(params, fmt.Sprintf("-net id=%s,mac=%s,host_dev_name=%s", name, mac, tapName))
+ params = append(params, "-net", fmt.Sprintf("id=%s,mac=%s,host_dev_name=%s", name, mac, tapName))
case config.BlockDrive:
id := v.ID
path := v.File
- params = append(params, fmt.Sprintf("-drive id=%s,file=%s", id, path))
+ params = append(params, "-drive", fmt.Sprintf("id=%s,file=%s", id, path))
case types.VSock:
v.VhostFd.Close()
- params = append(params, fmt.Sprintf("-device vsock,id=vsock-id,guest-cid=%d", v.ContextID))
+ params = append(params, "-device", fmt.Sprintf("vsock,id=vsock-id,guest-cid=%d", v.ContextID))
default:
s.Logger().Error("Adding device type is unsupported")
}
@@ -175,7 +175,7 @@ func (s *stratovirt) startSandbox(timeout int) error {
// append logfile only on debug
if s.config.Debug {
dir := filepath.Join(store.RunVMStoragePath(), s.id)
- params = append(params, fmt.Sprintf("-D %s/stratovirt.log", dir))
+ params = append(params, "-D", fmt.Sprintf("%s/stratovirt.log", dir))
}
// disable Seccomp
@@ -183,8 +183,6 @@ func (s *stratovirt) startSandbox(timeout int) error {
params = append(params, "-disable-seccomp")
}
- s.Logger().Info("StratoVirt start with params: ", strings.Join(params, " "))
-
dir := filepath.Join(store.RunVMStoragePath(), s.id)
err := os.MkdirAll(dir, store.DirMode)
if err != nil {
@@ -205,6 +203,8 @@ func (s *stratovirt) startSandbox(timeout int) error {
}
cmd := exec.CommandContext(s.ctx, binPath, params...)
+ s.Logger().Info("StratoVirt start with params: ", cmd)
+
if err := cmd.Run(); err != nil {
s.Logger().WithField("Error starting hypervisor, please check the params", err).Error()
return err
--
1.8.3.1

223
runtime/patches/0061-kata-runtime-retry-inserting-of-CNI-interface.patch Normal file
View File

@ -0,0 +1,223 @@
From babe7b3028d601a9b00aedda673e6a472f29ad07 Mon Sep 17 00:00:00 2001
From: yangfeiyu <yangfeiyu2@huawei.com>
Date: Mon, 28 Sep 2020 21:01:00 +0800
Subject: [PATCH] kata-runtime: retry inserting of CNI interface
reason: when netmon is enable, a interface is inserted into the
netns created by the sandbox, it should wait for the generating of
IP of the new interface, and get the ipv4 address
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
---
netmon/netmon.go | 128 +++++++++++++++++++++++++++++++++++++++++------
1 file changed, 112 insertions(+), 16 deletions(-)
diff --git a/netmon/netmon.go b/netmon/netmon.go
index 94305ce2..57beacfb 100644
--- a/netmon/netmon.go
+++ b/netmon/netmon.go
@@ -52,7 +52,7 @@ var (
version = "unknown"
// For simplicity the code will only focus on IPv4 addresses for now.
- netlinkFamily = netlink.FAMILY_ALL
+ netlinkFamily = netlink.FAMILY_V4
storageParentPath = "/var/run/kata-containers/netmon/sbs"
)
@@ -70,7 +70,9 @@ type netmon struct {
storagePath string
sharedFile string
- netIfaces map[int]vcTypes.Interface
+ netIfaces map[int]vcTypes.Interface
+ plugedIfaces map[string]vcTypes.Interface
+ pendingRoutes map[string][]vcTypes.Route
linkUpdateCh chan netlink.LinkUpdate
linkDoneCh chan struct{}
@@ -148,15 +150,17 @@ func newNetmon(params netmonParams) (*netmon, error) {
}
n := &netmon{
- netmonParams: params,
- storagePath: filepath.Join(storageParentPath, params.sandboxID),
- sharedFile: filepath.Join(storageParentPath, params.sandboxID, sharedFile),
- netIfaces: make(map[int]vcTypes.Interface),
- linkUpdateCh: make(chan netlink.LinkUpdate),
- linkDoneCh: make(chan struct{}),
- rtUpdateCh: make(chan netlink.RouteUpdate),
- rtDoneCh: make(chan struct{}),
- netHandler: handler,
+ netmonParams: params,
+ storagePath: filepath.Join(storageParentPath, params.sandboxID),
+ sharedFile: filepath.Join(storageParentPath, params.sandboxID, sharedFile),
+ netIfaces: make(map[int]vcTypes.Interface),
+ plugedIfaces: make(map[string]vcTypes.Interface),
+ pendingRoutes: make(map[string][]vcTypes.Route),
+ linkUpdateCh: make(chan netlink.LinkUpdate),
+ linkDoneCh: make(chan struct{}),
+ rtUpdateCh: make(chan netlink.RouteUpdate),
+ rtDoneCh: make(chan struct{}),
+ netHandler: handler,
}
if err := os.MkdirAll(n.storagePath, storageDirPerm); err != nil {
@@ -266,7 +270,6 @@ func convertInterface(linkAttrs *netlink.LinkAttrs, linkType string, addrs []net
}
var ipAddrs []*vcTypes.IPAddress
-
for _, addr := range addrs {
if addr.IPNet == nil {
continue
@@ -450,10 +453,24 @@ func (n *netmon) updateRoutes() error {
if err != nil {
return err
}
+ if len(netlinkRoutes) == 0 {
+ n.logger().Debug("get 0 routes")
+ return nil
+ }
// Translate them into Route structures.
routes := convertRoutes(netlinkRoutes)
+ // if the device of the routes have not be hotplug to guest,
+ // the update operation will be failed, pending them.
+ // For all routes are belong the same device, so we just need
+ // judge the device of the first route
+ if _, pluged := n.plugedIfaces[routes[0].Device]; !pluged {
+ n.pendingRoutes[routes[0].Device] = routes
+ n.logger().Infof("dev %s have not been added, pending:%v", routes[0].Device, routes)
+ return nil
+ }
+
// Update the routes through the Kata CLI.
return n.updateRoutesCLI(routes)
}
@@ -489,6 +506,9 @@ func (n *netmon) handleRTMNewLink(ev netlink.LinkUpdate) error {
return nil
}
+ // the link is usually not ready, and `sleep 3` is an empirical value.
+ time.Sleep(3 * time.Second)
+
// Check if the interface exist in the internal list.
if _, exist := n.netIfaces[int(ev.Index)]; exist {
n.logger().Debugf("Ignoring interface %s because already exist",
@@ -504,10 +524,26 @@ func (n *netmon) handleRTMNewLink(ev netlink.LinkUpdate) error {
return nil
}
- // Get the list of IP addresses associated with this interface.
- addrs, err := n.netHandler.AddrList(ev.Link, netlinkFamily)
- if err != nil {
- return err
+ // In some scenarios, the ip have not prepared,so we should do some retries.
+ var addrs []netlink.Addr
+ var err error
+ for i := 0; i < 5; i++ {
+ // Get the list of IP addresses associated with this interface.
+ addrs, err = n.netHandler.AddrList(ev.Link, netlinkFamily)
+ if err != nil {
+ return err
+ }
+ if len(addrs) > 0 {
+ break
+ }
+ time.Sleep(500 * time.Millisecond)
+ }
+
+ // In some scenarios, the link reported by event can not found, do extras check here.
+ if n.checkLinkByHw(linkAttrs.HardwareAddr.String()) != true {
+ n.logger().Infof("Ignore %v because can not find link by HW %s",
+ linkAttrs.Name, linkAttrs.HardwareAddr.String())
+ return nil
}
// Convert the interfaces in the appropriate structure format.
@@ -520,6 +556,17 @@ func (n *netmon) handleRTMNewLink(ev netlink.LinkUpdate) error {
// Add the interface to the internal list.
n.netIfaces[linkAttrs.Index] = iface
+ n.plugedIfaces[iface.Name] = iface
+
+ // The pending routes is preferentially selected.
+ if routes, ok := n.pendingRoutes[iface.Name]; ok {
+ n.logger().Infof("dev %s find pending routes:%v", iface.Name, routes)
+ err = n.updateRoutesCLI(routes)
+ if err != nil {
+ return err
+ }
+ delete(n.pendingRoutes, iface.Name)
+ }
// Complete by updating the routes.
return n.updateRoutes()
@@ -556,6 +603,8 @@ func (n *netmon) handleRTMDelLink(ev netlink.LinkUpdate) error {
// Delete the interface from the internal list.
delete(n.netIfaces, linkAttrs.Index)
+ delete(n.plugedIfaces, iface.Name)
+ delete(n.pendingRoutes, iface.Name)
// Complete by updating the routes.
return n.updateRoutes()
@@ -640,6 +689,53 @@ func (n *netmon) handleEvents() (err error) {
}
}
+func linkByHwAddr(netHandle *netlink.Handle, hwAddr string) (netlink.Link, error) {
+ if netHandle == nil {
+ return nil, fmt.Errorf("no handler ")
+ }
+
+ links, err := netHandle.LinkList()
+ if err != nil {
+ return nil, err
+ }
+
+ for _, link := range links {
+ if link == nil {
+ continue
+ }
+
+ lAttrs := link.Attrs()
+ if lAttrs == nil {
+ continue
+ }
+
+ if lAttrs.HardwareAddr == nil {
+ continue
+ }
+
+ if lAttrs.HardwareAddr.String() == hwAddr {
+ return link, nil
+ }
+ }
+
+ return nil, fmt.Errorf("could not find the link corresponding to HwAddr %s", hwAddr)
+}
+
+func (n *netmon) checkLinkByHw(hw string) bool {
+ netHandle, err := netlink.NewHandle(unix.NETLINK_ROUTE)
+ if err != nil {
+ return false
+ }
+ defer netHandle.Delete()
+
+ link, err := linkByHwAddr(netHandle, hw)
+ if err != nil || link == nil {
+ return false
+ }
+
+ return true
+}
+
func main() {
// Parse parameters.
params := parseOptions()
--
2.23.0

43
runtime/patches/0062-kata-runtime-support-using-CNI-plugin-to-insert-muti.patch Normal file
View File

@ -0,0 +1,43 @@
From f0d2f8a19956045b4b53ac5f2c4b59940016ca41 Mon Sep 17 00:00:00 2001
From: yangfeiyu <yangfeiyu2@huawei.com>
Date: Fri, 9 Oct 2020 16:02:27 +0800
Subject: [PATCH] kata-runtime: support using CNI plugin to insert mutiple
network interfaces at the same time
reason: support using CNI plugin to insert mutiple network interfaces at the same time
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
---
netmon/netmon.go | 16 +++++++++++-----
1 file changed, 11 insertions(+), 5 deletions(-)
diff --git a/netmon/netmon.go b/netmon/netmon.go
index 57beacfb..a519e5ba 100644
--- a/netmon/netmon.go
+++ b/netmon/netmon.go
@@ -463,11 +463,17 @@ func (n *netmon) updateRoutes() error {
// if the device of the routes have not be hotplug to guest,
// the update operation will be failed, pending them.
- // For all routes are belong the same device, so we just need
- // judge the device of the first route
- if _, pluged := n.plugedIfaces[routes[0].Device]; !pluged {
- n.pendingRoutes[routes[0].Device] = routes
- n.logger().Infof("dev %s have not been added, pending:%v", routes[0].Device, routes)
+ var pendingFlag bool
+ for _, route := range routes{
+ if _, pluged := n.plugedIfaces[route.Device]; !pluged {
+ pendingFlag = true
+ n.pendingRoutes[route.Device] = append(n.pendingRoutes[route.Device],route)
+ n.logger().Infof("dev %s have not been added, pending:%v", route.Device, n.pendingRoutes[route.Device])
+ }
+ }
+
+ // find pending route
+ if pendingFlag {
return nil
}
--
2.23.0

90
runtime/patches/0063-kata-runtime-fix-get-sandbox-cpu-resources-problem.patch Normal file
View File

@ -0,0 +1,90 @@
From bac206ee5b4ccb90fd8c06c0b6244ba163ac82b9 Mon Sep 17 00:00:00 2001
From: holyfei <yangfeiyu2@huawei.com>
Date: Sun, 15 Nov 2020 21:31:49 +0800
Subject: [PATCH] kata-runtime: fix get sandbox cpu resources problem
reason: If sandox_cgroup_with_emulator config is enabled,
kata-runtime should get cpu resources by cpuResourcesWithEmulator
func instead of the original cpuResource func.
Signed-off-by: holyfei <yangfeiyu2@huawei.com>
---
virtcontainers/cgroups.go | 2 ++
virtcontainers/sandbox.go | 34 +++++++++++++++++++++++++++++++++-
2 files changed, 35 insertions(+), 1 deletion(-)
diff --git a/virtcontainers/cgroups.go b/virtcontainers/cgroups.go
index 1b6b04da..21708ebf 100644
--- a/virtcontainers/cgroups.go
+++ b/virtcontainers/cgroups.go
@@ -40,6 +40,8 @@ const (
defaultMinFilesLimit uint64 = 1024
defaultMaxContainers uint64 = 200
+ defaultCPUPeriod uint64 = 1000000
+ defaultCPUShares uint64 = 1024
procFileMaxPath = "/proc/sys/fs/file-max"
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
index 9f5d3c08..d3c64b4f 100644
--- a/virtcontainers/sandbox.go
+++ b/virtcontainers/sandbox.go
@@ -14,6 +14,7 @@ import (
"net"
"os"
"path/filepath"
+ "strconv"
"strings"
"sync"
"syscall"
@@ -2351,6 +2352,37 @@ func (s *Sandbox) cpuResources() *specs.LinuxCPU {
return validCPUResources(cpu)
}
+func (s *Sandbox) cpuResourcesWithEmulator() *specs.LinuxCPU {
+ // Use default period and quota if they are not specified.
+ // Container will inherit the constraints from its parent.
+ quota := int64(0)
+ period := uint64(0)
+ shares := uint64(0)
+
+ cpu := &specs.LinuxCPU{
+ Quota: &quota,
+ Period: &period,
+ Shares: &shares,
+ }
+
+ period = defaultCPUPeriod
+
+ cpuValue, exist := s.config.Annotations[annotations.StaticCPUTypeKey]
+ if exist {
+ // If sandbox_cpu is set, we have validate at first, so we don't need to check it again
+ cpuNum, _ := strconv.ParseFloat(cpuValue, 64)
+ quota = int64(cpuNum * float64(period))
+ shares = uint64(cpuNum * float64(defaultCPUShares))
+ } else {
+ // If sandbox_cpu is not set, we use the hypervisor's cpu number as cpu limit
+ hypervisorConfig := s.hypervisor.hypervisorConfig()
+ quota = int64(uint64(hypervisorConfig.NumVCPUs) * period)
+ shares = uint64(hypervisorConfig.NumVCPUs) * defaultCPUShares
+ }
+
+ return cpu
+}
+
// setupSandboxCgroup creates and joins sandbox cgroups for the sandbox config
func (s *Sandbox) setupSandboxCgroup() error {
var err error
@@ -2424,7 +2456,7 @@ func (s *Sandbox) setupHostCgroupsWithEmulator() error {
// limit cpu to "<path>/vcpu"
vcpuCgroupPath := filepath.Join(s.state.CgroupPath, vcpuCgroupName)
vcpuResources := specs.LinuxResources{
- CPU: s.cpuResources(),
+ CPU: s.cpuResourcesWithEmulator(),
}
if err := applyResourceLimit(&vcpuResources, vcpuCgroupPath); err != nil {
return err
--
2.11.0

BIN
runtime/runtime-1.11.1.tar.gz Normal file
View File

Binary file not shown.

63
runtime/series.conf Normal file
View File

@ -0,0 +1,63 @@
0001-qmp-fix-kata-runtime-hungs-when-qemu-process-is-D-T-.patch
0002-kata-runtime-fix-kata-runtime-skip-read-lines-in-pro.patch
0003-kata-runtime-fix-kata-proxy-process-left-problem.patch
0004-kata-runtime-keep-the-process-name-of-qemu-same-as-c.patch
0005-cgroups-increase-delete-cgroup-retry-times.patch
0006-kata-runtime-fix-umount-container-rootfs-dir-return-.patch
0007-kata-runtime-enhance-reliability-when-kata-related-p.patch
0008-kata-runtime-fix-kata-runtime-resource-left-problem.patch
0009-kata-runtime-add-kata-runtime-global-flag-debug.patch
0010-kata-runtime-fix-kata-shim-pid-reused-problem.patch
0011-kata-runtime-check-the-process-info-before-send-SIGK.patch
0012-kata-runtime-truncate-the-log.json-file-before-kata-.patch
0013-kata-runtime-get-container-info-by-containerID-prefi.patch
0014-kata-runtime-add-self-defined-annotations-framework.patch
0015-kata-runtime-add-reuse-hypervisor-cpu-and-memory-fea.patch
0016-virtcontainers-fix-hotplug-huge-size-memory-cause-ag.patch
0017-kata-runtime-validate-sandbox-cpu-and-memory-size.patch
0018-sandbox-Stop-and-clean-up-containers-that-fail-to-cr.patch
0019-virtcontainers-fix-delete-sandbox-failed-problem.patch
0020-virtcontainers-add-enable_cpu_memory_hotplug-config-.patch
0021-kata-runtime-add-sandbox_cpu-and-sandbox_mem-annotat.patch
0022-kata-runtime-skip-go-version-check-and-do-not-build-.patch
0023-kata-runtime-set-PCIBridgeMaxCapacity-limit-to-25.patch
0024-kata-runtime-support-hotplug-tap-interface-into-kata.patch
0025-network-keep-list-ifaces-result-compatible-with-cni.patch
0026-network-add-enable_compat_old_cni-config.patch
0027-network-add-more-strict-check-for-input-network-inte.patch
0028-network-add-kata-network-add-route-subcommand.patch
0029-network-add-kata-network-del-route-subcommand.patch
0030-network-kata-network-list-routes-support-display-com.patch
0031-device_mangaer-check-VFIO-when-create-device.patch
0032-network-add-more-detail-usage-for-update-routes-subc.patch
0033-network-do-not-delete-the-exist-tap-device-in-the-ho.patch
0034-kata-runtime-add-kata-ipvs-command.patch
0035-device-mount-blockdevices-in-the-guest-VM.patch
0036-mount-limit-the-maximum-number-of-virtio-scsi-bus-sl.patch
0037-runtime-add-IPVS-test.patch
0038-pcie-using-pcie-root-port-driver-to-hotplug-device-i.patch
0039-storage-add-storage-common-functions-and-structs.patch
0040-storage-add-go-tests-for-storage.patch
0041-storage-mount-nfs-and-gpath-with-given-annotation.patch
0042-kata-runtime-do-not-ignore-updateInterface-return-er.patch
0043-kata-runtime-support-add-hypervisor-global-parameter.patch
0044-network-support-dpdk-vhost_user-net-device.patch
0045-network-support-set-dns.patch
0046-network-support-multiqueue-when-inserting-interface-.patch
0047-network-add-support-to-get-network-stats-of-vm-throu.patch
0048-console-fix-the-file-resource-leak.patch
0049-container-fix-the-write-operation-transparently-tran.patch
0050-runtime-add-kata-network-upate-iface-subcommand.patch
0051-network-fix-del-iface-doesn-t-delete-the-tap-interfa.patch
0052-runtime-add-support-of-new-sandbox-StratoVirt.patch
0053-kata-runtime-add-interface-for-host-cgroup.patch
0054-kata-runtime-add-sandbox-cgroup-with-vcpu-and-emulat.patch
0055-kata_runtime-support-host-cgroup-with-emulator-polic.patch
0056-kata_runtime-support-the-blkio-in-host-cgroups.patch
0057-kata-runtime-support-files-limit-in-host-cgroups.patch
0058-runtime-fix-sandboxRuntimeRootPath-left-problem.patch
0059-runtime-fix-invalid-cmdline-when-start-sandbox-stratovirt.patch
0060-runtime-fix-cmd-params-of-direct-use-stratovirt-bina.patch
0061-kata-runtime-retry-inserting-of-CNI-interface.patch
0062-kata-runtime-support-using-CNI-plugin-to-insert-muti.patch
0063-kata-runtime-fix-get-sandbox-cpu-resources-problem.patch

21
shim/apply-patches Executable file
View File

@ -0,0 +1,21 @@
#!/bin/bash
if [[ -f ./patch_flag ]];then
echo "shim patched!"
exit 0
fi
tar -zxvf shim-*.tar.gz
cp -fr ./shim-*/* ./
rm -rf ./shim-*
cat ./series.conf | while read line
do
if [[ $line == '' || $line =~ ^\s*# ]]; then
continue
fi
echo "====patch $line======"
pwd
patch -p1 -F1 -s < ./patches/$line
done
touch ./patch_flag

38
shim/patches/0001-kata-shim-fix-kata-shim-process-wait-long-tim.patch Normal file
View File

@ -0,0 +1,38 @@
From f5d031ef7d3437a05fe41f80aaf0e59ac54a575b Mon Sep 17 00:00:00 2001
From: jiangpengfei9 <jiangpengfei9@huawei.com>
Date: Mon, 1 Apr 2019 12:35:59 -0400
Subject: [PATCH 2/2] kata-shim: fix kata-shim process wait long time
to exit problem
reason: If containerd-shim process is killed and call kata-runtime delete -f command
to delete container, it will wait 10s for kata-shim process exit in the waitForShim
function, however 10s wait time is too long in the case of containerd-shim and containerd
process is killed.So when containerd-shim process is killed, kata-shim process will
receive a SIGHUP signal, we can handle this signal as exit signal which makes kata-shim
process exit.
Signed-off-by: jiangpengfei9 <jiangpengfei9@huawei.com>
---
shim.go | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/shim.go b/shim.go
index a93a00d..90172fe 100644
--- a/shim.go
+++ b/shim.go
@@ -134,6 +134,12 @@ func (s *shim) handleSignals(ctx context.Context, tty *os.File) chan os.Signal {
backtrace()
}
+ // if containerd-shim process is killed, kata-shim will receive SIGHUP signal
+ if sysSig == syscall.SIGHUP {
+ logger().WithField("signal", sig).Warn("received SIGHUP signal")
+ os.Exit(1)
+ }
+
// forward this signal to container
_, err := s.agent.SignalProcess(s.ctx, &pb.SignalProcessRequest{
ContainerId: s.containerID,
--
1.8.3.1

1
shim/series.conf Normal file
View File

@ -0,0 +1 @@
0001-kata-shim-fix-kata-shim-process-wait-long-tim.patch

BIN
shim/shim-1.11.1.tar.gz Normal file
View File

Binary file not shown.