diff --git a/agent/agent-1.11.1.tar.gz b/agent/agent-1.11.1.tar.gz deleted file mode 100644 index 5500b5d..0000000 Binary files a/agent/agent-1.11.1.tar.gz and /dev/null differ diff --git a/agent/apply-patches b/agent/apply-patches deleted file mode 100755 index 87d9b08..0000000 --- a/agent/apply-patches +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -if [[ -f ./patch_flag ]];then - echo "agent patched!" - exit 0 -fi - -tar -zxvf agent-1.11.1.tar.gz -cp -fr ./agent-1.11.1/* ./ -rm -rf ./agent-1.11.1 -cat ./series.conf | while read line -do - if [[ $line == '' || $line =~ ^\s*# ]]; then - continue - fi - echo "====patch $line======" - pwd - patch -p1 -F1 -s < ./patches/$line -done -touch ./patch_flag diff --git a/agent/patches/0001-agent-add-agent.netlink_recv_buf_size-flag-to-set-ne.patch b/agent/patches/0001-agent-add-agent.netlink_recv_buf_size-flag-to-set-ne.patch deleted file mode 100644 index 24bc18a..0000000 --- a/agent/patches/0001-agent-add-agent.netlink_recv_buf_size-flag-to-set-ne.patch +++ /dev/null @@ -1,234 +0,0 @@ -From ac1d7806f8de2f8ca393df08a9c62d1045c4afdc Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Tue, 11 Dec 2018 18:27:02 -0500 -Subject: [PATCH 01/16] agent: add agent.netlink_recv_buf_size flag to set - netlink recv buf size - -fixes: #813 - -reason: If hotplug huge size memory(for example 128GB) into guest, -kernel will produce a lot of memory add uevents and send to netlink socket, -however netlink socket default receive buffer size is 4KB, which is too small -to receive all memory add uevents. -Since hotplug huge size memory is not common case, so we consider add an agent -flag agent.netlink_recv_buf_size to set netlink socket recv buffer size. - -Signed-off-by: jiangpengfei ---- - README.md | 13 +++++++++++++ - agent.go | 10 +++++++++- - config.go | 15 ++++++++++++++ - config_test.go | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++ - pkg/uevent/uevent.go | 15 +++++++++++--- - 5 files changed, 104 insertions(+), 4 deletions(-) - -diff --git a/README.md b/README.md -index cec65a4..16f96a4 100644 ---- a/README.md -+++ b/README.md -@@ -98,6 +98,19 @@ The pipe's capacity for stdout/stderr can be modified by specifying the `agent.c - to the guest kernel command line. For example, `agent.container_pipe_size=2097152` will set the stdout and stderr - pipes to 2097152 bytes. - -+## Uevent Netlink Socket Receive Buffer Size -+ -+When hotplugging a huge size memory into the Kata VM, the kernel in the VM will produce a lot of memory object add -+uevents and send all these uevents to Kata agent by netlink socket. However, default netlink socket receive buffer -+size is 4KB, which is too small and can only hold 256 memory add uevents. If memory add uevents number is larger -+than 256, the left uevents can not be received and processed by Kata agent. -+ -+The uevent netlink socket receive buffer size can be modified by specifying the `agent.netlink_recv_buf_size` flag -+to the guest kernel command line. For example, `agent.netlink_recv_buf_size=2MB` will set the uevent netlink socket -+receive buffer size to 2MB value. `agent.netlink_recv_buf_size` valid value range is `[4KB ~ 4MB]` and value can be -+set in human-readable memory format or pure digital number format(default memory unit is byte). -+ -+ - [1]: https://github.com/firecracker-microvm/firecracker/blob/master/docs/vsock.md - [2]: https://golang.org/pkg/time/#ParseDuration - [3]: http://man7.org/linux/man-pages/man7/pipe.7.html -diff --git a/agent.go b/agent.go -index 2d2c293..c1cac08 100644 ---- a/agent.go -+++ b/agent.go -@@ -190,6 +190,14 @@ var unifiedCgroupHierarchy = false - // Size in bytes of the stdout/stderr pipes created for each container. - var containerPipeSize = uint32(0) - -+const ( -+ minNetlinkSockRecvBufSize = 4 * 1024 -+ maxNetlinkSockRecvBufSize = 4 * 1024 * 1024 -+) -+ -+// Size in bytes of the netlink socket recv buf size -+var netlinkSockRecvBufSize = uint32(0) -+ - // commType is used to denote the communication channel type used. - type commType int - -@@ -708,7 +716,7 @@ func (s *sandbox) waitForStopServer() { - func (s *sandbox) listenToUdevEvents() { - fieldLogger := agentLog.WithField("subsystem", "udevlistener") - -- uEvHandler, err := uevent.NewHandler() -+ uEvHandler, err := uevent.NewHandler(netlinkSockRecvBufSize) - if err != nil { - fieldLogger.Warnf("Error starting uevent listening loop %s", err) - return -diff --git a/config.go b/config.go -index 4530096..6c7d473 100644 ---- a/config.go -+++ b/config.go -@@ -7,11 +7,13 @@ - package main - - import ( -+ "fmt" - "io/ioutil" - "strconv" - "strings" - "time" - -+ "github.com/docker/go-units" - "github.com/sirupsen/logrus" - "google.golang.org/grpc/codes" - grpcStatus "google.golang.org/grpc/status" -@@ -29,6 +31,7 @@ const ( - hotplugTimeoutFlag = optionPrefix + "hotplug_timeout" - unifiedCgroupHierarchyFlag = optionPrefix + "unified_cgroup_hierarchy" - containerPipeSizeFlag = optionPrefix + "container_pipe_size" -+ netlinkSockRecvBufSizeFlag = optionPrefix + "netlink_recv_buf_size" - traceModeStatic = "static" - traceModeDynamic = "dynamic" - traceTypeIsolated = "isolated" -@@ -155,6 +158,18 @@ func parseCmdlineOption(option string) error { - return err - } - unifiedCgroupHierarchy = flag -+ case netlinkSockRecvBufSizeFlag: -+ bufSizeInBytes, err := units.RAMInBytes(split[valuePosition]) -+ if err != nil { -+ return err -+ } -+ -+ if bufSizeInBytes < minNetlinkSockRecvBufSize || bufSizeInBytes > maxNetlinkSockRecvBufSize { -+ return fmt.Errorf("invalid netlink socket recv buf size: %d (valid size range %s-%s bytes)", bufSizeInBytes, -+ units.BytesSize(minNetlinkSockRecvBufSize), units.BytesSize(maxNetlinkSockRecvBufSize)) -+ } -+ -+ netlinkSockRecvBufSize = uint32(bufSizeInBytes) - default: - if strings.HasPrefix(split[optionPosition], optionPrefix) { - return grpcStatus.Errorf(codes.NotFound, "Unknown option %s", split[optionPosition]) -diff --git a/config_test.go b/config_test.go -index 2a23133..f40f17a 100644 ---- a/config_test.go -+++ b/config_test.go -@@ -486,3 +486,58 @@ func TestParseCmdlineOptionContainerPipeSize(t *testing.T) { - assert.Equal(d.expectedContainerPipeSize, containerPipeSize, "test %d (%+v)", i, d) - } - } -+ -+func TestParseCmdlineOptionNetlinkSockRecvBufSize(t *testing.T) { -+ assert := assert.New(t) -+ -+ type testData struct { -+ option string -+ shouldErr bool -+ expectedNetlinkSockRecvBufSize uint32 -+ } -+ -+ data := []testData{ -+ {"", false, 0}, -+ {"netlink_recv_buf_siz", false, 0}, -+ {"netlink_recv_buf_size", false, 0}, -+ {"netlink_recv_buf_size=", false, 0}, -+ {"netlink_recv_buf_size=4096", false, 0}, -+ {"netlink_recv_buf_size=4KB", false, 0}, -+ {"agent.netlink_recv_buf_size=", true, 0}, -+ {"agent.netlink_recv_buf_size=foobar", true, 0}, -+ {"agent.netlink_recv_buf_size=-1", true, 0}, -+ {"agent.netlink_recv_buf_size=0", true, 0}, -+ {"agent.netlink_recv_buf_size=100", true, 0}, -+ {"agent.netlink_recv_buf_size=3KB", true, 0}, -+ {"agent.netlink_recv_buf_size=3.6KB", true, 0}, -+ {"agent.netlink_recv_buf_size=4095", true, 0}, -+ {"agent.netlink_recv_buf_size=4096xB", true, 0}, -+ {"agent.netlink_recv_buf_size=4096", false, 4096}, -+ {"agent.netlink_recv_buf_size=4097", false, 4097}, -+ {"agent.netlink_recv_buf_size=4096.0", false, 4096}, -+ {"agent.netlink_recv_buf_size=1024KB", false, 1048576}, -+ {"agent.netlink_recv_buf_size=1MB", false, 1048576}, -+ {"agent.netlink_recv_buf_size=4194303", false, 4194303}, -+ {"agent.netlink_recv_buf_size=3.999MB", false, 4193255}, -+ {"agent.netlink_recv_buf_size=4194304", false, 4194304}, -+ {"agent.netlink_recv_buf_size=4MB", false, 4194304}, -+ {"agent.netlink_recv_buf_size=4.001MB", true, 0}, -+ {"agent.netlink_recv_buf_size=4194305", true, 0}, -+ {"agent.netlink_recv_buf_size=100MB", true, 0}, -+ {"agent.netlink_recv_buf_size=1GB", true, 0}, -+ } -+ -+ for i, d := range data { -+ // reset the netlink socket recv buffer size -+ netlinkSockRecvBufSize = 0 -+ -+ err := parseCmdlineOption(d.option) -+ if d.shouldErr { -+ assert.Error(err) -+ } else { -+ assert.NoError(err) -+ } -+ -+ assert.Equal(d.expectedNetlinkSockRecvBufSize, netlinkSockRecvBufSize, "test %d (%+v)", i, d) -+ } -+} -diff --git a/pkg/uevent/uevent.go b/pkg/uevent/uevent.go -index fc2c127..fa84086 100644 ---- a/pkg/uevent/uevent.go -+++ b/pkg/uevent/uevent.go -@@ -10,6 +10,7 @@ import ( - "bufio" - "io" - "strings" -+ "syscall" - - "golang.org/x/sys/unix" - "google.golang.org/grpc/codes" -@@ -33,7 +34,7 @@ type ReaderCloser struct { - } - - // NewReaderCloser returns an io.ReadCloser handle for uevent. --func NewReaderCloser() (io.ReadCloser, error) { -+func NewReaderCloser(netlinkRecvBufSize uint32) (io.ReadCloser, error) { - nl := unix.SockaddrNetlink{ - Family: unix.AF_NETLINK, - // Passing Pid as 0 here allows the kernel to take care of assigning -@@ -47,6 +48,14 @@ func NewReaderCloser() (io.ReadCloser, error) { - return nil, err - } - -+ // If netlinkRecvBufSize > 0, set netlink socket recv buffer size to netlinkRecvBufSize -+ if netlinkRecvBufSize > 0 { -+ err = unix.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_RCVBUFFORCE, int(netlinkRecvBufSize)) -+ if err != nil { -+ return nil, err -+ } -+ } -+ - if err := unix.Bind(fd, &nl); err != nil { - return nil, err - } -@@ -85,8 +94,8 @@ type Handler struct { - } - - // NewHandler returns a uevent handler. --func NewHandler() (*Handler, error) { -- rdCloser, err := NewReaderCloser() -+func NewHandler(netlinkRecvBufSize uint32) (*Handler, error) { -+ rdCloser, err := NewReaderCloser(netlinkRecvBufSize) - if err != nil { - return nil, err - } --- -2.14.3 (Apple Git-98) - diff --git a/agent/patches/0002-network-support-update-routes-incrementally.patch b/agent/patches/0002-network-support-update-routes-incrementally.patch deleted file mode 100644 index 4cc9c27..0000000 --- a/agent/patches/0002-network-support-update-routes-incrementally.patch +++ /dev/null @@ -1,686 +0,0 @@ -From 13f54c768dcd7bf982dde8e57fb5cd624fedf5bc Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Mon, 17 Aug 2020 11:23:55 +0800 -Subject: [PATCH 02/16] network: support update routes incrementally - -reason: add increment flag in the UpdateRoutesRequest to -support upate routes incrementally to improve the efficiency. - -kata-network add-route and del-route needs this feature. - -Signed-off-by: jiangpengfei ---- - grpc.go | 2 +- - network.go | 74 ++++++++- - network_test.go | 16 +- - protocols/grpc/agent.pb.go | 402 +++++++++++++++++++++++++-------------------- - protocols/grpc/agent.proto | 1 + - 5 files changed, 300 insertions(+), 195 deletions(-) - -diff --git a/grpc.go b/grpc.go -index 886661b..8fe8217 100644 ---- a/grpc.go -+++ b/grpc.go -@@ -1556,7 +1556,7 @@ func (a *agentGRPC) UpdateInterface(ctx context.Context, req *pb.UpdateInterface - } - - func (a *agentGRPC) UpdateRoutes(ctx context.Context, req *pb.UpdateRoutesRequest) (*pb.Routes, error) { -- return a.sandbox.updateRoutes(nil, req.Routes) -+ return a.sandbox.updateRoutes(nil, req.Routes, req.Increment) - } - - func (a *agentGRPC) ListInterfaces(ctx context.Context, req *pb.ListInterfacesRequest) (*pb.Interfaces, error) { -diff --git a/network.go b/network.go -index 64a16a9..02e28cb 100644 ---- a/network.go -+++ b/network.go -@@ -398,7 +398,7 @@ func (s *sandbox) deleteRoutes(netHandle *netlink.Handle) error { - // state which matches the requested routes. In doing this, preesxisting non-loopback routes will be - // removed from the network. If an error occurs, this function returns the list of routes in - // gRPC-route format at the time of failure --func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Routes) (resultingRoutes *pb.Routes, err error) { -+func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Routes, increment bool) (resultingRoutes *pb.Routes, err error) { - if requestedRoutes == nil { - return nil, errNoRoutes - } -@@ -418,13 +418,66 @@ func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Ro - } - }() - -+ var ( -+ added []*types.Route -+ removed []*types.Route -+ ) -+ -+ defer func(netHandle *netlink.Handle) { -+ if err != nil { -+ // if error happens after route added, need to rollback the added route -+ if len(added) > 0 { -+ for _, r := range added { -+ errRb := s.updateRoute(netHandle, r, false) -+ if errRb != nil { -+ agentLog.WithError(err).Error("rollback route failed") -+ } -+ } -+ } -+ -+ // if error happens after route removed, need to rollback the removed route -+ if len(removed) > 0 { -+ for _, r := range removed { -+ errRb := s.updateRoute(netHandle, r, true) -+ if errRb != nil { -+ agentLog.WithError(err).Error("rollback route failed") -+ } -+ } -+ } -+ } -+ }(netHandle) -+ -+ // updateOneRoute just update the specified one route -+ updateOneRoute := func(netHandle *netlink.Handle, reqRoute *types.Route) error { -+ var add bool = true -+ if reqRoute.Dest != "" && strings.HasPrefix(reqRoute.Dest, "-") { -+ reqRoute.Dest = reqRoute.Dest[1:] -+ add = false -+ } -+ err = s.updateRoute(netHandle, reqRoute, add) -+ if err != nil { -+ agentLog.WithError(err).Error("update Route failed") -+ // If there was an error setting the route, return the error -+ // and the current routes on the system via the defer func -+ return err -+ } -+ if add { -+ added = append([]*types.Route{reqRoute}, added[:]...) -+ } else { -+ removed = append([]*types.Route{reqRoute}, removed[:]...) -+ } -+ return nil -+ } -+ - // - // First things first, let's blow away all the existing routes. The updateRoutes function - // is designed to be declarative, so we will attempt to create state matching what is - // requested, and in the event that we fail to do so, will return the error and final state. - // -- if err = s.deleteRoutes(netHandle); err != nil { -- return nil, err -+ if !increment { -+ if err = s.deleteRoutes(netHandle); err != nil { -+ return nil, err -+ } - } - - // -@@ -434,7 +487,12 @@ func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Ro - // won't be able to access the gateway - for _, reqRoute := range requestedRoutes.Routes { - if reqRoute.Gateway == "" { -- err = s.updateRoute(netHandle, reqRoute, true) -+ if increment { -+ err = updateOneRoute(netHandle, reqRoute) -+ } else { -+ err = s.updateRoute(netHandle, reqRoute, true) -+ } -+ - if err != nil { - agentLog.WithError(err).Error("update Route failed") - //If there was an error setting the route, return the error -@@ -447,7 +505,11 @@ func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Ro - // Take a second pass and apply the routes which include a gateway - for _, reqRoute := range requestedRoutes.Routes { - if reqRoute.Gateway != "" { -- err = s.updateRoute(netHandle, reqRoute, true) -+ if increment { -+ err = updateOneRoute(netHandle, reqRoute) -+ } else { -+ err = s.updateRoute(netHandle, reqRoute, true) -+ } - if err != nil { - agentLog.WithError(err).Error("update Route failed") - //If there was an error setting the route, return the -@@ -699,4 +761,4 @@ func (s *sandbox) handleLocalhost() error { - } - - return netlink.LinkSetUp(lo) --} -+} -\ No newline at end of file -diff --git a/network_test.go b/network_test.go -index a143670..a1e58f5 100644 ---- a/network_test.go -+++ b/network_test.go -@@ -160,7 +160,7 @@ func TestUpdateRoutes(t *testing.T) { - Routes: inputRoutesSimple, - } - -- results, err := s.updateRoutes(netHandle, testRoutes) -+ results, err := s.updateRoutes(netHandle, testRoutes, false) - assert.Nil(t, err, "Unexpected update interface failure: %v", err) - assert.True(t, reflect.DeepEqual(results, testRoutes), - "Interface created didn't match: got %+v, expecting %+v", results, testRoutes) -@@ -173,7 +173,7 @@ func TestUpdateRoutes(t *testing.T) { - } - testRoutes.Routes = inputRoutesPTPExample - -- results, err = s.updateRoutes(netHandle, testRoutes) -+ results, err = s.updateRoutes(netHandle, testRoutes, false) - assert.Nil(t, err, "Unexpected update interface failure: %v", err) - assert.True(t, reflect.DeepEqual(results, testRoutes), - "Interface created didn't match: got %+v, expecting %+v", results, testRoutes) -@@ -184,7 +184,7 @@ func TestUpdateRoutes(t *testing.T) { - {Dest: "192.168.0.0/16", Gateway: "", Source: "192.168.0.2", Scope: 0, Device: "ifc-name"}, - } - testRoutes.Routes = inputRoutesNoScope -- results, err = s.updateRoutes(netHandle, testRoutes) -+ results, err = s.updateRoutes(netHandle, testRoutes, false) - assert.NotNil(t, err, "Expected to observe unreachable route failure") - - assert.True(t, reflect.DeepEqual(results.Routes[0], testRoutes.Routes[1]), -@@ -231,7 +231,7 @@ func TestUpdateRoutesIPVlan(t *testing.T) { - } - testRoutes.Routes = inputRoutesIPVlanExample - -- results, err := s.updateRoutes(netHandle, testRoutes) -+ results, err := s.updateRoutes(netHandle, testRoutes, false) - assert.Nil(t, err, "Unexpected update interface failure: %v", err) - assert.True(t, reflect.DeepEqual(results, testRoutes), - "Interface created didn't match: got %+v, expecting %+v", results, testRoutes) -@@ -357,7 +357,7 @@ func TestListRoutes(t *testing.T) { - Routes: inputRoutesSimple, - } - -- _, err := s.updateRoutes(netHandle, testRoutes) -+ _, err := s.updateRoutes(netHandle, testRoutes, false) - assert.Nil(err) - results, err := s.listRoutes(nil) - assert.Nil(err, "Expected to list all routes") -@@ -377,7 +377,7 @@ func TestListRoutes(t *testing.T) { - Routes: inputRoutesSimple, - } - -- _, err = s.updateRoutes(netHandle, testRoutes) -+ _, err = s.updateRoutes(netHandle, testRoutes, false) - assert.Nil(err) - results, err = s.listRoutes(nil) - assert.Nil(err, "Expected to list all routes") -@@ -438,7 +438,7 @@ func TestListRoutesWithIPV6(t *testing.T) { - Routes: inputRoutesSimple, - } - -- _, err := s.updateRoutes(netHandle, testRoutes) -+ _, err := s.updateRoutes(netHandle, testRoutes, false) - assert.Nil(err) - results, err := s.listRoutes(nil) - assert.Nil(err, "Expected to list all routes") -@@ -514,7 +514,7 @@ func TestListRoutesWithTwoInterfacesSameSubnet(t *testing.T) { - Routes: inputRoutesSimple, - } - -- _, err := s.updateRoutes(netHandle, testRoutes) -+ _, err := s.updateRoutes(netHandle, testRoutes, false) - assert.Nil(err) - results, err := s.listRoutes(nil) - assert.Nil(err, "Expected to list all routes") -diff --git a/protocols/grpc/agent.pb.go b/protocols/grpc/agent.pb.go -index 77e6d1b..1b887e5 100644 ---- a/protocols/grpc/agent.pb.go -+++ b/protocols/grpc/agent.pb.go -@@ -1303,7 +1303,8 @@ func (m *UpdateInterfaceRequest) GetInterface() *types.Interface { - } - - type UpdateRoutesRequest struct { -- Routes *Routes `protobuf:"bytes,1,opt,name=routes" json:"routes,omitempty"` -+ Routes *Routes `protobuf:"bytes,1,opt,name=routes" json:"routes,omitempty"` -+ Increment bool `protobuf:"varint,2,opt,name=increment,proto3" json:"increment,omitempty"` - } - - func (m *UpdateRoutesRequest) Reset() { *m = UpdateRoutesRequest{} } -@@ -1318,6 +1319,13 @@ func (m *UpdateRoutesRequest) GetRoutes() *Routes { - return nil - } - -+func (m *UpdateRoutesRequest) GetIncrement() bool { -+ if m != nil { -+ return m.Increment -+ } -+ return false -+} -+ - type ListInterfacesRequest struct { - } - -@@ -4476,6 +4484,16 @@ func (m *UpdateRoutesRequest) MarshalTo(dAtA []byte) (int, error) { - } - i += n20 - } -+ if m.Increment { -+ dAtA[i] = 0x10 -+ i++ -+ if m.Increment { -+ dAtA[i] = 1 -+ } else { -+ dAtA[i] = 0 -+ } -+ i++ -+ } - return i, nil - } - -@@ -5751,6 +5769,9 @@ func (m *UpdateRoutesRequest) Size() (n int) { - l = m.Routes.Size() - n += 1 + l + sovAgent(uint64(l)) - } -+ if m.Increment { -+ n += 2 -+ } - return n - } - -@@ -10964,6 +10985,26 @@ func (m *UpdateRoutesRequest) Unmarshal(dAtA []byte) error { - return err - } - iNdEx = postIndex -+ case 2: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field Increment", wireType) -+ } -+ var v int -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ v |= (int(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ m.Increment = bool(v != 0) - default: - iNdEx = preIndex - skippy, err := skipAgent(dAtA[iNdEx:]) -@@ -12852,184 +12893,185 @@ var ( - func init() { proto.RegisterFile("agent.proto", fileDescriptorAgent) } - - var fileDescriptorAgent = []byte{ -- // 2862 bytes of a gzipped FileDescriptorProto -+ // 2876 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0x4b, 0x6f, 0x1c, 0xc7, -- 0xd1, 0xd8, 0x07, 0x97, 0xbb, 0xb5, 0x2f, 0x6e, 0x93, 0xa2, 0x56, 0x2b, 0x5b, 0x9f, 0x3c, 0xb6, -- 0x65, 0xfa, 0x73, 0xbc, 0xb4, 0x65, 0x23, 0x7e, 0xc1, 0x11, 0xc4, 0x47, 0x44, 0xc6, 0x56, 0xc4, -- 0x0c, 0x45, 0x38, 0x40, 0x10, 0x0c, 0x86, 0x33, 0xcd, 0x65, 0x9b, 0x3b, 0xd3, 0xe3, 0x9e, 0x1e, -- 0x8a, 0xeb, 0x00, 0x39, 0x26, 0xb7, 0x5c, 0x02, 0xe4, 0x96, 0x3f, 0x10, 0xe4, 0x96, 0x63, 0xae, -- 0x39, 0x18, 0x39, 0xe5, 0x17, 0x04, 0x81, 0x7f, 0x42, 0x7e, 0x41, 0xd0, 0xaf, 0x79, 0xec, 0x0e, -- 0x29, 0x84, 0x20, 0x90, 0xcb, 0xa2, 0xab, 0xba, 0xba, 0x5e, 0xdd, 0x55, 0x53, 0x55, 0x0b, 0x6d, -- 0x77, 0x82, 0x43, 0x3e, 0x8e, 0x18, 0xe5, 0x14, 0xd5, 0x27, 0x2c, 0xf2, 0x46, 0x2d, 0xea, 0x11, -- 0x85, 0x18, 0xfd, 0x70, 0x42, 0xf8, 0x69, 0x72, 0x3c, 0xf6, 0x68, 0xb0, 0x79, 0xe6, 0x72, 0xf7, -- 0x5d, 0x8f, 0x86, 0xdc, 0x25, 0x21, 0x66, 0xf1, 0xa6, 0x3c, 0xb8, 0x19, 0x9d, 0x4d, 0x36, 0xf9, -- 0x2c, 0xc2, 0xb1, 0xfa, 0xd5, 0xe7, 0xee, 0x4e, 0x28, 0x9d, 0x4c, 0xf1, 0xa6, 0x84, 0x8e, 0x93, -- 0x93, 0x4d, 0x1c, 0x44, 0x7c, 0xa6, 0x36, 0xad, 0x3f, 0x56, 0x61, 0x7d, 0x9b, 0x61, 0x97, 0xe3, -- 0x6d, 0xc3, 0xcd, 0xc6, 0xdf, 0x24, 0x38, 0xe6, 0xe8, 0x35, 0xe8, 0xa4, 0x12, 0x1c, 0xe2, 0x0f, -- 0x2b, 0xf7, 0x2b, 0x1b, 0x2d, 0xbb, 0x9d, 0xe2, 0xf6, 0x7d, 0x74, 0x1b, 0x96, 0xf1, 0x05, 0xf6, -- 0xc4, 0x6e, 0x55, 0xee, 0x36, 0x04, 0xb8, 0xef, 0xa3, 0xf7, 0xa1, 0x1d, 0x73, 0x46, 0xc2, 0x89, -- 0x93, 0xc4, 0x98, 0x0d, 0x6b, 0xf7, 0x2b, 0x1b, 0xed, 0x87, 0x2b, 0x63, 0x61, 0xd2, 0xf8, 0x50, -- 0x6e, 0x1c, 0xc5, 0x98, 0xd9, 0x10, 0xa7, 0x6b, 0xf4, 0x00, 0x96, 0x7d, 0x7c, 0x4e, 0x3c, 0x1c, -- 0x0f, 0xeb, 0xf7, 0x6b, 0x1b, 0xed, 0x87, 0x1d, 0x45, 0xbe, 0x23, 0x91, 0xb6, 0xd9, 0x44, 0x6f, -- 0x43, 0x33, 0xe6, 0x94, 0xb9, 0x13, 0x1c, 0x0f, 0x97, 0x24, 0x61, 0xd7, 0xf0, 0x95, 0x58, 0x3b, -- 0xdd, 0x46, 0xaf, 0x40, 0xed, 0xd9, 0xf6, 0xfe, 0xb0, 0x21, 0xa5, 0x83, 0xa6, 0x8a, 0xb0, 0x67, -- 0x0b, 0x34, 0x7a, 0x1d, 0xba, 0xb1, 0x1b, 0xfa, 0xc7, 0xf4, 0xc2, 0x89, 0x88, 0x1f, 0xc6, 0xc3, -- 0xe5, 0xfb, 0x95, 0x8d, 0xa6, 0xdd, 0xd1, 0xc8, 0x03, 0x81, 0xb3, 0x3e, 0x85, 0x5b, 0x87, 0xdc, -- 0x65, 0xfc, 0x1a, 0xde, 0xb1, 0x8e, 0x60, 0xdd, 0xc6, 0x01, 0x3d, 0xbf, 0x96, 0x6b, 0x87, 0xb0, -- 0xcc, 0x49, 0x80, 0x69, 0xc2, 0xa5, 0x6b, 0xbb, 0xb6, 0x01, 0xad, 0x3f, 0x57, 0x00, 0xed, 0x5e, -- 0x60, 0xef, 0x80, 0x51, 0x0f, 0xc7, 0xf1, 0xff, 0xe8, 0xba, 0xde, 0x82, 0xe5, 0x48, 0x29, 0x30, -- 0xac, 0x4b, 0x72, 0x7d, 0x0b, 0x46, 0x2b, 0xb3, 0x6b, 0x7d, 0x0d, 0x6b, 0x87, 0x64, 0x12, 0xba, -- 0xd3, 0x1b, 0xd4, 0x77, 0x1d, 0x1a, 0xb1, 0xe4, 0x29, 0x55, 0xed, 0xda, 0x1a, 0xb2, 0x0e, 0x00, -- 0x7d, 0xe5, 0x12, 0x7e, 0x73, 0x92, 0xac, 0x77, 0x61, 0xb5, 0xc0, 0x31, 0x8e, 0x68, 0x18, 0x63, -- 0xa9, 0x00, 0x77, 0x79, 0x12, 0x4b, 0x66, 0x4b, 0xb6, 0x86, 0x2c, 0x0c, 0x6b, 0x5f, 0x92, 0xd8, -- 0x90, 0xe3, 0xff, 0x46, 0x85, 0x75, 0x68, 0x9c, 0x50, 0x16, 0xb8, 0xdc, 0x68, 0xa0, 0x20, 0x84, -- 0xa0, 0xee, 0xb2, 0x49, 0x3c, 0xac, 0xdd, 0xaf, 0x6d, 0xb4, 0x6c, 0xb9, 0x16, 0xaf, 0x72, 0x4e, -- 0x8c, 0xd6, 0xeb, 0x35, 0xe8, 0x68, 0xbf, 0x3b, 0x53, 0x12, 0x73, 0x29, 0xa7, 0x63, 0xb7, 0x35, -- 0x4e, 0x9c, 0xb1, 0x28, 0xac, 0x1f, 0x45, 0xfe, 0x35, 0x03, 0xfe, 0x21, 0xb4, 0x18, 0x8e, 0x69, -- 0xc2, 0x44, 0x98, 0x56, 0xe5, 0xbd, 0xaf, 0xa9, 0x7b, 0xff, 0x92, 0x84, 0xc9, 0x85, 0x6d, 0xf6, -- 0xec, 0x8c, 0x4c, 0x87, 0x10, 0x8f, 0xaf, 0x13, 0x42, 0x9f, 0xc2, 0xad, 0x03, 0x37, 0x89, 0xaf, -- 0xa3, 0xab, 0xf5, 0x99, 0x08, 0xbf, 0x38, 0x09, 0xae, 0x75, 0xf8, 0x4f, 0x15, 0x68, 0x6e, 0x47, -- 0xc9, 0x51, 0xec, 0x4e, 0x30, 0xfa, 0x3f, 0x68, 0x73, 0xca, 0xdd, 0xa9, 0x93, 0x08, 0x50, 0x92, -- 0xd7, 0x6d, 0x90, 0x28, 0x45, 0x20, 0xdc, 0x8e, 0x99, 0x17, 0x25, 0x9a, 0xa2, 0x7a, 0xbf, 0xb6, -- 0x51, 0xb7, 0xdb, 0x0a, 0xa7, 0x48, 0xc6, 0xb0, 0x2a, 0xf7, 0x1c, 0x12, 0x3a, 0x67, 0x98, 0x85, -- 0x78, 0x1a, 0x50, 0x1f, 0xcb, 0xf7, 0x5b, 0xb7, 0x07, 0x72, 0x6b, 0x3f, 0xfc, 0x22, 0xdd, 0x40, -- 0xff, 0x0f, 0x83, 0x94, 0x5e, 0x04, 0xa5, 0xa4, 0xae, 0x4b, 0xea, 0xbe, 0xa6, 0x3e, 0xd2, 0x68, -- 0xeb, 0xd7, 0xd0, 0x7b, 0x7e, 0xca, 0x28, 0xe7, 0x53, 0x12, 0x4e, 0x76, 0x5c, 0xee, 0x8a, 0xec, -- 0x11, 0x61, 0x46, 0xa8, 0x1f, 0x6b, 0x6d, 0x0d, 0x88, 0xde, 0x81, 0x01, 0x57, 0xb4, 0xd8, 0x77, -- 0x0c, 0x4d, 0x55, 0xd2, 0xac, 0xa4, 0x1b, 0x07, 0x9a, 0xf8, 0x4d, 0xe8, 0x65, 0xc4, 0x22, 0xff, -- 0x68, 0x7d, 0xbb, 0x29, 0xf6, 0x39, 0x09, 0xb0, 0x75, 0x2e, 0x7d, 0x25, 0x2f, 0x19, 0xbd, 0x03, -- 0xad, 0xcc, 0x0f, 0x15, 0xf9, 0x42, 0x7a, 0xea, 0x85, 0x18, 0x77, 0xda, 0xcd, 0xd4, 0x29, 0x9f, -- 0x43, 0x9f, 0xa7, 0x8a, 0x3b, 0xbe, 0xcb, 0xdd, 0xe2, 0xa3, 0x2a, 0x5a, 0x65, 0xf7, 0x78, 0x01, -- 0xb6, 0x3e, 0x83, 0xd6, 0x01, 0xf1, 0x63, 0x25, 0x78, 0x08, 0xcb, 0x5e, 0xc2, 0x18, 0x0e, 0xb9, -- 0x31, 0x59, 0x83, 0x68, 0x0d, 0x96, 0xa6, 0x24, 0x20, 0x5c, 0x9b, 0xa9, 0x00, 0x8b, 0x02, 0x3c, -- 0xc5, 0x01, 0x65, 0x33, 0xe9, 0xb0, 0x35, 0x58, 0xca, 0x5f, 0xae, 0x02, 0xd0, 0x5d, 0x68, 0x05, -- 0xee, 0x45, 0x7a, 0xa9, 0x62, 0xa7, 0x19, 0xb8, 0x17, 0x4a, 0xf9, 0x21, 0x2c, 0x9f, 0xb8, 0x64, -- 0xea, 0x85, 0x5c, 0x7b, 0xc5, 0x80, 0x99, 0xc0, 0x7a, 0x5e, 0xe0, 0xdf, 0xaa, 0xd0, 0x56, 0x12, -- 0x95, 0xc2, 0x6b, 0xb0, 0xe4, 0xb9, 0xde, 0x69, 0x2a, 0x52, 0x02, 0xe8, 0x81, 0x51, 0xa4, 0x9a, -- 0x4f, 0xc2, 0x99, 0xa6, 0x46, 0xb5, 0x4d, 0x80, 0xf8, 0x85, 0x1b, 0x69, 0xdd, 0x6a, 0x97, 0x10, -- 0xb7, 0x04, 0x8d, 0x52, 0xf7, 0x03, 0xe8, 0xa8, 0x77, 0xa7, 0x8f, 0xd4, 0x2f, 0x39, 0xd2, 0x56, -- 0x54, 0xea, 0xd0, 0xeb, 0xd0, 0x4d, 0x62, 0xec, 0x9c, 0x12, 0xcc, 0x5c, 0xe6, 0x9d, 0xce, 0x86, -- 0x4b, 0xea, 0x1b, 0x99, 0xc4, 0x78, 0xcf, 0xe0, 0xd0, 0x43, 0x58, 0x12, 0xe9, 0x2f, 0x1e, 0x36, -- 0xe4, 0xe7, 0xf8, 0x95, 0x3c, 0x4b, 0x69, 0xea, 0x58, 0xfe, 0xee, 0x86, 0x9c, 0xcd, 0x6c, 0x45, -- 0x3a, 0xfa, 0x18, 0x20, 0x43, 0xa2, 0x15, 0xa8, 0x9d, 0xe1, 0x99, 0x8e, 0x43, 0xb1, 0x14, 0xce, -- 0x39, 0x77, 0xa7, 0x89, 0xf1, 0xba, 0x02, 0x3e, 0xad, 0x7e, 0x5c, 0xb1, 0x3c, 0xe8, 0x6f, 0x4d, -- 0xcf, 0x08, 0xcd, 0x1d, 0x5f, 0x83, 0xa5, 0xc0, 0xfd, 0x9a, 0x32, 0xe3, 0x49, 0x09, 0x48, 0x2c, -- 0x09, 0x29, 0x33, 0x2c, 0x24, 0x80, 0x7a, 0x50, 0xa5, 0x91, 0xf4, 0x57, 0xcb, 0xae, 0xd2, 0x28, -- 0x13, 0x54, 0xcf, 0x09, 0xb2, 0xfe, 0x59, 0x07, 0xc8, 0xa4, 0x20, 0x1b, 0x46, 0x84, 0x3a, 0x31, -- 0x66, 0xa2, 0x04, 0x71, 0x8e, 0x67, 0x1c, 0xc7, 0x0e, 0xc3, 0x5e, 0xc2, 0x62, 0x72, 0x2e, 0xee, -- 0x4f, 0x98, 0x7d, 0x4b, 0x99, 0x3d, 0xa7, 0x9b, 0x7d, 0x9b, 0xd0, 0x43, 0x75, 0x6e, 0x4b, 0x1c, -- 0xb3, 0xcd, 0x29, 0xb4, 0x0f, 0xb7, 0x32, 0x9e, 0x7e, 0x8e, 0x5d, 0xf5, 0x2a, 0x76, 0xab, 0x29, -- 0x3b, 0x3f, 0x63, 0xb5, 0x0b, 0xab, 0x84, 0x3a, 0xdf, 0x24, 0x38, 0x29, 0x30, 0xaa, 0x5d, 0xc5, -- 0x68, 0x40, 0xe8, 0xcf, 0xe4, 0x81, 0x8c, 0xcd, 0x01, 0xdc, 0xc9, 0x59, 0x29, 0xc2, 0x3d, 0xc7, -- 0xac, 0x7e, 0x15, 0xb3, 0xf5, 0x54, 0x2b, 0x91, 0x0f, 0x32, 0x8e, 0x3f, 0x81, 0x75, 0x42, 0x9d, -- 0x17, 0x2e, 0xe1, 0xf3, 0xec, 0x96, 0x5e, 0x62, 0xa4, 0xf8, 0xe8, 0x16, 0x79, 0x29, 0x23, 0x03, -- 0xcc, 0x26, 0x05, 0x23, 0x1b, 0x2f, 0x31, 0xf2, 0xa9, 0x3c, 0x90, 0xb1, 0x79, 0x0c, 0x03, 0x42, -- 0xe7, 0xb5, 0x59, 0xbe, 0x8a, 0x49, 0x9f, 0xd0, 0xa2, 0x26, 0x5b, 0x30, 0x88, 0xb1, 0xc7, 0x29, -- 0xcb, 0x3f, 0x82, 0xe6, 0x55, 0x2c, 0x56, 0x34, 0x7d, 0xca, 0xc3, 0xfa, 0x05, 0x74, 0xf6, 0x92, -- 0x09, 0xe6, 0xd3, 0xe3, 0x34, 0x19, 0xdc, 0x58, 0xfe, 0xb1, 0xfe, 0x5d, 0x85, 0xf6, 0xf6, 0x84, -- 0xd1, 0x24, 0x2a, 0xe4, 0x64, 0x15, 0xa4, 0xf3, 0x39, 0x59, 0x92, 0xc8, 0x9c, 0xac, 0x88, 0x3f, -- 0x84, 0x4e, 0x20, 0x43, 0x57, 0xd3, 0xab, 0x3c, 0x34, 0x58, 0x08, 0x6a, 0xbb, 0x1d, 0xe4, 0x92, -- 0xd9, 0x18, 0x20, 0x22, 0x7e, 0xac, 0xcf, 0xa8, 0x74, 0xd4, 0xd7, 0x15, 0xa1, 0x49, 0xd1, 0x76, -- 0x2b, 0x4a, 0xb3, 0xf5, 0xfb, 0xd0, 0x3e, 0x16, 0x4e, 0xd2, 0x07, 0x0a, 0xc9, 0x28, 0xf3, 0x9e, -- 0x0d, 0xc7, 0x59, 0x10, 0xee, 0x41, 0xf7, 0x54, 0xb9, 0x4c, 0x1f, 0x52, 0x6f, 0xe8, 0x75, 0x6d, -- 0x49, 0x66, 0xef, 0x38, 0xef, 0x59, 0x75, 0x01, 0x9d, 0xd3, 0x1c, 0x6a, 0x74, 0x08, 0x83, 0x05, -- 0x92, 0x92, 0x1c, 0xb4, 0x91, 0xcf, 0x41, 0xed, 0x87, 0x48, 0x09, 0xca, 0x9f, 0xcc, 0xe7, 0xa5, -- 0xdf, 0x55, 0xa1, 0xf3, 0x53, 0xcc, 0x5f, 0x50, 0x76, 0xa6, 0xf4, 0x45, 0x50, 0x0f, 0xdd, 0x00, -- 0x6b, 0x8e, 0x72, 0x8d, 0xee, 0x40, 0x93, 0x5d, 0xa8, 0x04, 0xa2, 0xef, 0x73, 0x99, 0x5d, 0xc8, -- 0xc4, 0x80, 0x5e, 0x05, 0x60, 0x17, 0x4e, 0xe4, 0x7a, 0x67, 0x58, 0x7b, 0xb0, 0x6e, 0xb7, 0xd8, -- 0xc5, 0x81, 0x42, 0x88, 0xa7, 0xc0, 0x2e, 0x1c, 0xcc, 0x18, 0x65, 0xb1, 0xce, 0x55, 0x4d, 0x76, -- 0xb1, 0x2b, 0x61, 0x7d, 0xd6, 0x67, 0x34, 0x8a, 0xb0, 0x2f, 0x73, 0xb4, 0x3c, 0xbb, 0xa3, 0x10, -- 0x42, 0x2a, 0x37, 0x52, 0x1b, 0x4a, 0x2a, 0xcf, 0xa4, 0xf2, 0x4c, 0xea, 0xb2, 0x3a, 0xc9, 0xf3, -- 0x52, 0x79, 0x2a, 0xb5, 0xa9, 0xa4, 0xf2, 0x9c, 0x54, 0x9e, 0x49, 0x6d, 0x99, 0xb3, 0x5a, 0xaa, -- 0xf5, 0xdb, 0x0a, 0xac, 0xcf, 0x17, 0x7e, 0xba, 0x4c, 0xfd, 0x10, 0x3a, 0x9e, 0xbc, 0xaf, 0xc2, -- 0x9b, 0x1c, 0x2c, 0xdc, 0xa4, 0xdd, 0xf6, 0x72, 0xcf, 0xf8, 0x23, 0xe8, 0x86, 0xca, 0xc1, 0xe9, -- 0xd3, 0xac, 0x65, 0xf7, 0x92, 0xf7, 0xbd, 0xdd, 0x09, 0x73, 0x90, 0xe5, 0x03, 0xfa, 0x8a, 0x11, -- 0x8e, 0x0f, 0x39, 0xc3, 0x6e, 0x70, 0x13, 0x0d, 0x08, 0x82, 0xba, 0xac, 0x56, 0x6a, 0xb2, 0xbe, -- 0x96, 0x6b, 0xeb, 0x2d, 0x58, 0x2d, 0x48, 0xd1, 0xb6, 0xae, 0x40, 0x6d, 0x8a, 0x43, 0xc9, 0xbd, -- 0x6b, 0x8b, 0xa5, 0xe5, 0xc2, 0xc0, 0xc6, 0xae, 0x7f, 0x73, 0xda, 0x68, 0x11, 0xb5, 0x4c, 0xc4, -- 0x06, 0xa0, 0xbc, 0x08, 0xad, 0x8a, 0xd1, 0xba, 0x92, 0xd3, 0xfa, 0x19, 0x0c, 0xb6, 0xa7, 0x34, -- 0xc6, 0x87, 0xdc, 0x27, 0xe1, 0x4d, 0x74, 0x4c, 0xbf, 0x82, 0xd5, 0xe7, 0x7c, 0xf6, 0x95, 0x60, -- 0x16, 0x93, 0x6f, 0xf1, 0x0d, 0xd9, 0xc7, 0xe8, 0x0b, 0x63, 0x1f, 0xa3, 0x2f, 0x44, 0xb3, 0xe4, -- 0xd1, 0x69, 0x12, 0x84, 0x32, 0x14, 0xba, 0xb6, 0x86, 0xac, 0x2d, 0xe8, 0xa8, 0x1a, 0xfa, 0x29, -- 0xf5, 0x93, 0x29, 0x2e, 0x8d, 0xc1, 0x7b, 0x00, 0x91, 0xcb, 0xdc, 0x00, 0x73, 0xcc, 0xd4, 0x1b, -- 0x6a, 0xd9, 0x39, 0x8c, 0xf5, 0x87, 0x2a, 0xac, 0xa9, 0x91, 0xc8, 0xa1, 0x9a, 0x04, 0x18, 0x13, -- 0x46, 0xd0, 0x3c, 0xa5, 0x31, 0xcf, 0x31, 0x4c, 0x61, 0xa1, 0xa2, 0x1f, 0x1a, 0x6e, 0x62, 0x59, -- 0x98, 0x53, 0xd4, 0xae, 0x9e, 0x53, 0x2c, 0x4c, 0x22, 0xea, 0x8b, 0x93, 0x08, 0x11, 0x6d, 0x86, -- 0x88, 0xa8, 0x18, 0x6f, 0xd9, 0x2d, 0x8d, 0xd9, 0xf7, 0xd1, 0x03, 0xe8, 0x4f, 0x84, 0x96, 0xce, -- 0x29, 0xa5, 0x67, 0x4e, 0xe4, 0xf2, 0x53, 0x19, 0xea, 0x2d, 0xbb, 0x2b, 0xd1, 0x7b, 0x94, 0x9e, -- 0x1d, 0xb8, 0xfc, 0x14, 0x7d, 0x02, 0x3d, 0x5d, 0x06, 0x06, 0xd2, 0x45, 0xb1, 0xfe, 0xf8, 0xe9, -- 0x28, 0xca, 0x7b, 0xcf, 0xee, 0x9e, 0xe5, 0xa0, 0xd8, 0xba, 0x0d, 0xb7, 0x76, 0x70, 0xcc, 0x19, -- 0x9d, 0x15, 0x1d, 0x63, 0xfd, 0x08, 0x60, 0x3f, 0xe4, 0x98, 0x9d, 0xb8, 0x1e, 0x8e, 0xd1, 0x7b, -- 0x79, 0x48, 0x17, 0x47, 0x2b, 0x63, 0x35, 0x91, 0x4a, 0x37, 0xec, 0x1c, 0x8d, 0x35, 0x86, 0x86, -- 0x4d, 0x13, 0x91, 0x8e, 0xde, 0x30, 0x2b, 0x7d, 0xae, 0xa3, 0xcf, 0x49, 0xa4, 0xad, 0xf7, 0xac, -- 0x3d, 0xd3, 0xc2, 0x66, 0xec, 0xf4, 0x15, 0x8d, 0xa1, 0x45, 0x0c, 0x4e, 0x67, 0x95, 0x45, 0xd1, -- 0x19, 0x89, 0xf5, 0x19, 0xac, 0x2a, 0x4e, 0x8a, 0xb3, 0x61, 0xf3, 0x06, 0x34, 0x98, 0x51, 0xa3, -- 0x92, 0x8d, 0xa2, 0x34, 0x91, 0xde, 0x13, 0xfe, 0x10, 0x1d, 0x75, 0x66, 0x88, 0xf1, 0xc7, 0x2a, -- 0x0c, 0xc4, 0x46, 0x81, 0xa7, 0xf5, 0x4b, 0x58, 0x7d, 0x16, 0x4e, 0x49, 0x88, 0xb7, 0x0f, 0x8e, -- 0x9e, 0xe2, 0x34, 0xee, 0x11, 0xd4, 0x45, 0x7d, 0x24, 0x05, 0x35, 0x6d, 0xb9, 0x16, 0x81, 0x10, -- 0x1e, 0x3b, 0x5e, 0x94, 0xc4, 0x7a, 0xf6, 0xd3, 0x08, 0x8f, 0xb7, 0xa3, 0x24, 0x16, 0x89, 0x5c, -- 0x7c, 0xc8, 0x69, 0x38, 0x9d, 0xc9, 0x68, 0x68, 0xda, 0xcb, 0x5e, 0x94, 0x3c, 0x0b, 0xa7, 0x33, -- 0xeb, 0x07, 0xb2, 0xdb, 0xc5, 0xd8, 0xb7, 0xdd, 0xd0, 0xa7, 0xc1, 0x0e, 0x3e, 0xcf, 0x49, 0x48, -- 0x3b, 0x2b, 0x13, 0xf5, 0xdf, 0x55, 0xa0, 0xf3, 0x78, 0x82, 0x43, 0xbe, 0x83, 0xb9, 0x4b, 0xa6, -- 0xb2, 0x7b, 0x3a, 0xc7, 0x2c, 0x26, 0x34, 0xd4, 0x4f, 0xdb, 0x80, 0xa2, 0xf9, 0x25, 0x21, 0xe1, -- 0x8e, 0xef, 0xe2, 0x80, 0x86, 0x92, 0x4b, 0xd3, 0x06, 0x81, 0xda, 0x91, 0x18, 0xf4, 0x16, 0xf4, -- 0xd5, 0x6c, 0xce, 0x39, 0x75, 0x43, 0x7f, 0x2a, 0x82, 0x4a, 0xcd, 0x2a, 0x7a, 0x0a, 0xbd, 0xa7, -- 0xb1, 0xe8, 0x6d, 0x58, 0xd1, 0x4f, 0x3e, 0xa3, 0xac, 0x4b, 0xca, 0xbe, 0xc6, 0x17, 0x48, 0x93, -- 0x28, 0xa2, 0x8c, 0xc7, 0x4e, 0x8c, 0x3d, 0x8f, 0x06, 0x91, 0x6e, 0x3d, 0xfa, 0x06, 0x7f, 0xa8, -- 0xd0, 0xd6, 0x04, 0x56, 0x9f, 0x08, 0x3b, 0xb5, 0x25, 0xd9, 0x15, 0xf6, 0x02, 0x1c, 0x38, 0xc7, -- 0x53, 0xea, 0x9d, 0x39, 0x22, 0x11, 0x69, 0x0f, 0x8b, 0xe2, 0x66, 0x4b, 0x20, 0x0f, 0xc9, 0xb7, -- 0xb2, 0xcb, 0x16, 0x54, 0xa7, 0x94, 0x47, 0xd3, 0x64, 0xe2, 0x44, 0x8c, 0x1e, 0x63, 0x6d, 0x62, -- 0x3f, 0xc0, 0xc1, 0x9e, 0xc2, 0x1f, 0x08, 0xb4, 0xf5, 0xd7, 0x0a, 0xac, 0x15, 0x25, 0xe9, 0xb4, -- 0xba, 0x09, 0x6b, 0x45, 0x51, 0xfa, 0x53, 0xab, 0x4a, 0xb9, 0x41, 0x5e, 0xa0, 0xfa, 0xe8, 0x7e, -- 0x04, 0x5d, 0x39, 0xb0, 0x75, 0x7c, 0xc5, 0xa9, 0x58, 0x60, 0xe4, 0xef, 0xc5, 0xee, 0xb8, 0xf9, -- 0x5b, 0xfa, 0x04, 0xee, 0x68, 0xf3, 0x9d, 0x45, 0xb5, 0xd5, 0x83, 0x58, 0xd7, 0x04, 0x4f, 0xe7, -- 0xb4, 0xff, 0x12, 0x86, 0x19, 0x6a, 0x6b, 0x26, 0x91, 0xc6, 0x57, 0xef, 0xc1, 0xea, 0x9c, 0xb1, -- 0x8f, 0x7d, 0x9f, 0xc9, 0x10, 0xac, 0xdb, 0x65, 0x5b, 0xd6, 0x23, 0xb8, 0x7d, 0x88, 0xb9, 0xf2, -- 0x86, 0xcb, 0x75, 0xd5, 0xaf, 0x98, 0xad, 0x40, 0xed, 0x10, 0x7b, 0xd2, 0xf8, 0x9a, 0x2d, 0x96, -- 0xe2, 0x01, 0x1e, 0xc5, 0xd8, 0x93, 0x56, 0xd6, 0x6c, 0xb9, 0xb6, 0xfe, 0x52, 0x81, 0x65, 0x9d, -- 0x08, 0x45, 0x32, 0xf7, 0x19, 0x39, 0xc7, 0x4c, 0x3f, 0x3d, 0x0d, 0xa1, 0x37, 0xa1, 0xa7, 0x56, -- 0x0e, 0x8d, 0x38, 0xa1, 0x69, 0x7a, 0xed, 0x2a, 0xec, 0x33, 0x85, 0x94, 0xb3, 0x38, 0x39, 0x6a, -- 0xd2, 0x5d, 0x9d, 0x86, 0xe4, 0x40, 0x2d, 0x16, 0xb1, 0x2f, 0xd3, 0x69, 0xcb, 0xd6, 0x90, 0x78, -- 0xea, 0x86, 0xdf, 0x92, 0xe4, 0x67, 0x40, 0xf1, 0xd4, 0x03, 0x9a, 0x84, 0xdc, 0x89, 0x28, 0x09, -- 0xb9, 0xce, 0x9f, 0x20, 0x51, 0x07, 0x02, 0x63, 0xfd, 0xa6, 0x02, 0x0d, 0x35, 0x8f, 0x16, 0x7d, -- 0x64, 0xfa, 0x15, 0xab, 0x12, 0x59, 0x11, 0x48, 0x59, 0xea, 0xcb, 0x25, 0xd7, 0x22, 0x8e, 0xcf, -- 0x03, 0x95, 0x8b, 0xb5, 0x6a, 0xe7, 0x81, 0x4c, 0xc2, 0x6f, 0x42, 0x2f, 0xfb, 0x18, 0xca, 0x7d, -- 0xa5, 0x62, 0x37, 0xc5, 0x4a, 0xb2, 0x4b, 0x35, 0xb5, 0x7e, 0x2e, 0xda, 0xe7, 0x74, 0x16, 0xbb, -- 0x02, 0xb5, 0x24, 0x55, 0x46, 0x2c, 0x05, 0x66, 0x92, 0x7e, 0x46, 0xc5, 0x12, 0x3d, 0x80, 0x9e, -- 0xeb, 0xfb, 0x44, 0x1c, 0x77, 0xa7, 0x4f, 0x88, 0x9f, 0x06, 0x69, 0x11, 0x6b, 0xfd, 0xbd, 0x02, -- 0xfd, 0x6d, 0x1a, 0xcd, 0x7e, 0x4c, 0xa6, 0x38, 0x97, 0x41, 0xa4, 0x92, 0xfa, 0x2b, 0x2a, 0xd6, -- 0xa2, 0x32, 0x3c, 0x21, 0x53, 0xac, 0x42, 0x4b, 0xdd, 0x6c, 0x53, 0x20, 0x64, 0x58, 0x99, 0xcd, -- 0x74, 0xc4, 0xd5, 0x55, 0x9b, 0x4f, 0xa9, 0x2f, 0x6b, 0x60, 0x9f, 0x30, 0x27, 0x1d, 0x68, 0x75, -- 0xed, 0x65, 0x9f, 0x30, 0xb9, 0xa5, 0x0d, 0x59, 0x92, 0x33, 0xd5, 0xbc, 0x21, 0x0d, 0x85, 0x11, -- 0x86, 0xac, 0x43, 0x83, 0x9e, 0x9c, 0xc4, 0x98, 0xcb, 0x6a, 0xb5, 0x66, 0x6b, 0x28, 0x4d, 0x73, -- 0xcd, 0x5c, 0x9a, 0xbb, 0x05, 0xab, 0x72, 0x7a, 0xff, 0x9c, 0xb9, 0x1e, 0x09, 0x27, 0x26, 0x15, -- 0xaf, 0x01, 0x3a, 0xe4, 0x34, 0x2a, 0x62, 0x1f, 0xfe, 0x7e, 0x45, 0xe7, 0x44, 0xdd, 0xca, 0xa2, -- 0x27, 0xd0, 0x9f, 0xfb, 0x6b, 0x04, 0xe9, 0xd9, 0x46, 0xf9, 0x3f, 0x26, 0xa3, 0xf5, 0xb1, 0xfa, -- 0xab, 0x65, 0x6c, 0xfe, 0x6a, 0x19, 0xef, 0x06, 0x11, 0x9f, 0xa1, 0x5d, 0xe8, 0x15, 0xff, 0x44, -- 0x40, 0x77, 0x4d, 0x29, 0x50, 0xf2, 0xd7, 0xc2, 0xa5, 0x6c, 0x9e, 0x40, 0x7f, 0xee, 0xff, 0x04, -- 0xa3, 0x4f, 0xf9, 0xdf, 0x0c, 0x97, 0x32, 0x7a, 0x04, 0xed, 0xdc, 0x1f, 0x08, 0x68, 0xa8, 0x98, -- 0x2c, 0xfe, 0xa7, 0x70, 0x29, 0x83, 0x6d, 0xe8, 0x16, 0x66, 0xfa, 0x68, 0xa4, 0xed, 0x29, 0x19, -- 0xf4, 0x5f, 0xca, 0x64, 0x0b, 0xda, 0xb9, 0xd1, 0xba, 0xd1, 0x62, 0x71, 0x7e, 0x3f, 0xba, 0x53, -- 0xb2, 0xa3, 0x53, 0xef, 0x1e, 0x74, 0x0b, 0x83, 0x70, 0xa3, 0x48, 0xd9, 0x10, 0x7e, 0x74, 0xb7, -- 0x74, 0x4f, 0x73, 0x7a, 0x02, 0xfd, 0xb9, 0xb1, 0xb8, 0x71, 0x6e, 0xf9, 0xb4, 0xfc, 0x52, 0xb3, -- 0xbe, 0x90, 0x97, 0x9d, 0xeb, 0x7a, 0x72, 0x97, 0xbd, 0x38, 0x04, 0x1f, 0xbd, 0x52, 0xbe, 0xa9, -- 0xb5, 0xda, 0x85, 0x5e, 0x71, 0xfe, 0x6d, 0x98, 0x95, 0x4e, 0xc5, 0xaf, 0x7e, 0x39, 0x85, 0x51, -- 0x78, 0xf6, 0x72, 0xca, 0x26, 0xe4, 0x97, 0x32, 0x7a, 0x0c, 0xa0, 0x7b, 0x1c, 0x9f, 0x84, 0xe9, -- 0x95, 0x2d, 0xf4, 0x56, 0xe9, 0x95, 0x95, 0xf4, 0x43, 0x8f, 0x00, 0x54, 0x6b, 0xe2, 0xd3, 0x84, -- 0xa3, 0xdb, 0x46, 0x8d, 0xb9, 0x7e, 0x68, 0x34, 0x5c, 0xdc, 0x58, 0x60, 0x80, 0x19, 0xbb, 0x0e, -- 0x83, 0xcf, 0x01, 0xb2, 0x96, 0xc7, 0x30, 0x58, 0x68, 0x82, 0xae, 0xf0, 0x41, 0x27, 0xdf, 0xe0, -- 0x20, 0x6d, 0x6b, 0x49, 0xd3, 0x73, 0x05, 0x8b, 0xfe, 0x5c, 0x01, 0x5b, 0x7c, 0x6c, 0xf3, 0x75, -- 0xed, 0x68, 0xa1, 0x88, 0x45, 0x1f, 0x41, 0x27, 0x5f, 0xb9, 0x1a, 0x2d, 0x4a, 0xaa, 0xd9, 0x51, -- 0xa1, 0x7a, 0x45, 0x8f, 0xa0, 0x57, 0xac, 0x5a, 0x51, 0x2e, 0x2e, 0x16, 0x6a, 0xd9, 0x91, 0x9e, -- 0xc9, 0xe4, 0xc8, 0x3f, 0x00, 0xc8, 0xaa, 0x5b, 0xe3, 0xbe, 0x85, 0x7a, 0x77, 0x4e, 0xea, 0x63, -- 0xe8, 0xe4, 0x33, 0xb1, 0x51, 0xb7, 0x24, 0x3b, 0x5f, 0x95, 0xb5, 0x72, 0x59, 0xdb, 0x3c, 0xbe, -- 0xc5, 0x44, 0x7e, 0x55, 0xd6, 0x2a, 0xf4, 0x75, 0x26, 0x59, 0x94, 0x35, 0x7b, 0x57, 0xe5, 0xf2, -- 0x62, 0x13, 0x64, 0xdc, 0x57, 0xda, 0x1a, 0x5d, 0xf5, 0x88, 0xf2, 0xdd, 0x80, 0xf1, 0x47, 0x49, -- 0x87, 0xf0, 0x92, 0xa0, 0xce, 0x57, 0xfc, 0xb9, 0xa0, 0x2e, 0x69, 0x04, 0x2e, 0x65, 0xb4, 0x07, -- 0xfd, 0x27, 0xa6, 0x98, 0xd3, 0x85, 0xa6, 0x56, 0xa7, 0xa4, 0xb0, 0x1e, 0x8d, 0xca, 0xb6, 0x74, -- 0x64, 0x7d, 0x01, 0x83, 0x85, 0x22, 0x13, 0xdd, 0x4b, 0x47, 0x87, 0xa5, 0xd5, 0xe7, 0xa5, 0x6a, -- 0xed, 0xc3, 0xca, 0x7c, 0x8d, 0x89, 0x5e, 0xd5, 0x97, 0x5e, 0x5e, 0x7b, 0x5e, 0xca, 0xea, 0x13, -- 0x68, 0x9a, 0x9a, 0x06, 0xe9, 0x11, 0xed, 0x5c, 0x8d, 0x73, 0xd9, 0xd1, 0xad, 0xce, 0x77, 0xdf, -- 0xdf, 0xab, 0xfc, 0xe3, 0xfb, 0x7b, 0x95, 0x7f, 0x7d, 0x7f, 0xaf, 0x72, 0xdc, 0x90, 0xbb, 0x1f, -- 0xfc, 0x27, 0x00, 0x00, 0xff, 0xff, 0xa5, 0xac, 0x85, 0x1d, 0xaa, 0x21, 0x00, 0x00, -+ 0xd1, 0xd8, 0x07, 0x97, 0xbb, 0xb5, 0x2f, 0x6e, 0x93, 0xa2, 0x56, 0x2b, 0x59, 0x9f, 0x3c, 0xb6, -+ 0x65, 0xfa, 0xf3, 0xe7, 0xa5, 0x2d, 0x1b, 0x9f, 0x5f, 0x70, 0x04, 0xf1, 0x11, 0x91, 0xb1, 0x15, -+ 0x31, 0x43, 0x11, 0x4e, 0x10, 0x04, 0x83, 0xe1, 0x4c, 0x73, 0xd9, 0xe6, 0xce, 0xf4, 0xb8, 0xa7, -+ 0x87, 0xe2, 0x3a, 0x40, 0x8e, 0xc9, 0x2d, 0x97, 0x00, 0xb9, 0xe5, 0x0f, 0x04, 0xb9, 0xe5, 0x98, -+ 0x6b, 0x0e, 0x46, 0x4e, 0xf9, 0x05, 0x41, 0xe0, 0x9f, 0x90, 0x5f, 0x10, 0xf4, 0x6b, 0x1e, 0xbb, -+ 0x43, 0x1a, 0x21, 0x08, 0xe4, 0xb2, 0xe8, 0xaa, 0xae, 0xae, 0x57, 0x77, 0xd5, 0x54, 0xd5, 0x42, -+ 0xdb, 0x9d, 0xe0, 0x90, 0x8f, 0x23, 0x46, 0x39, 0x45, 0xf5, 0x09, 0x8b, 0xbc, 0x51, 0x8b, 0x7a, -+ 0x44, 0x21, 0x46, 0xff, 0x3f, 0x21, 0xfc, 0x34, 0x39, 0x1e, 0x7b, 0x34, 0xd8, 0x3c, 0x73, 0xb9, -+ 0xfb, 0x8e, 0x47, 0x43, 0xee, 0x92, 0x10, 0xb3, 0x78, 0x53, 0x1e, 0xdc, 0x8c, 0xce, 0x26, 0x9b, -+ 0x7c, 0x16, 0xe1, 0x58, 0xfd, 0xea, 0x73, 0x77, 0x27, 0x94, 0x4e, 0xa6, 0x78, 0x53, 0x42, 0xc7, -+ 0xc9, 0xc9, 0x26, 0x0e, 0x22, 0x3e, 0x53, 0x9b, 0xd6, 0x1f, 0xaa, 0xb0, 0xbe, 0xcd, 0xb0, 0xcb, -+ 0xf1, 0xb6, 0xe1, 0x66, 0xe3, 0xaf, 0x13, 0x1c, 0x73, 0xf4, 0x2a, 0x74, 0x52, 0x09, 0x0e, 0xf1, -+ 0x87, 0x95, 0x07, 0x95, 0x8d, 0x96, 0xdd, 0x4e, 0x71, 0xfb, 0x3e, 0xba, 0x0d, 0xcb, 0xf8, 0x02, -+ 0x7b, 0x62, 0xb7, 0x2a, 0x77, 0x1b, 0x02, 0xdc, 0xf7, 0xd1, 0x7b, 0xd0, 0x8e, 0x39, 0x23, 0xe1, -+ 0xc4, 0x49, 0x62, 0xcc, 0x86, 0xb5, 0x07, 0x95, 0x8d, 0xf6, 0xa3, 0x95, 0xb1, 0x30, 0x69, 0x7c, -+ 0x28, 0x37, 0x8e, 0x62, 0xcc, 0x6c, 0x88, 0xd3, 0x35, 0x7a, 0x08, 0xcb, 0x3e, 0x3e, 0x27, 0x1e, -+ 0x8e, 0x87, 0xf5, 0x07, 0xb5, 0x8d, 0xf6, 0xa3, 0x8e, 0x22, 0xdf, 0x91, 0x48, 0xdb, 0x6c, 0xa2, -+ 0xb7, 0xa0, 0x19, 0x73, 0xca, 0xdc, 0x09, 0x8e, 0x87, 0x4b, 0x92, 0xb0, 0x6b, 0xf8, 0x4a, 0xac, -+ 0x9d, 0x6e, 0xa3, 0x7b, 0x50, 0x7b, 0xbe, 0xbd, 0x3f, 0x6c, 0x48, 0xe9, 0xa0, 0xa9, 0x22, 0xec, -+ 0xd9, 0x02, 0x8d, 0x5e, 0x83, 0x6e, 0xec, 0x86, 0xfe, 0x31, 0xbd, 0x70, 0x22, 0xe2, 0x87, 0xf1, -+ 0x70, 0xf9, 0x41, 0x65, 0xa3, 0x69, 0x77, 0x34, 0xf2, 0x40, 0xe0, 0xac, 0x4f, 0xe0, 0xd6, 0x21, -+ 0x77, 0x19, 0xbf, 0x86, 0x77, 0xac, 0x23, 0x58, 0xb7, 0x71, 0x40, 0xcf, 0xaf, 0xe5, 0xda, 0x21, -+ 0x2c, 0x73, 0x12, 0x60, 0x9a, 0x70, 0xe9, 0xda, 0xae, 0x6d, 0x40, 0xeb, 0x4f, 0x15, 0x40, 0xbb, -+ 0x17, 0xd8, 0x3b, 0x60, 0xd4, 0xc3, 0x71, 0xfc, 0x5f, 0xba, 0xae, 0x37, 0x61, 0x39, 0x52, 0x0a, -+ 0x0c, 0xeb, 0x92, 0x5c, 0xdf, 0x82, 0xd1, 0xca, 0xec, 0x5a, 0x5f, 0xc1, 0xda, 0x21, 0x99, 0x84, -+ 0xee, 0xf4, 0x06, 0xf5, 0x5d, 0x87, 0x46, 0x2c, 0x79, 0x4a, 0x55, 0xbb, 0xb6, 0x86, 0xac, 0x03, -+ 0x40, 0x5f, 0xba, 0x84, 0xdf, 0x9c, 0x24, 0xeb, 0x1d, 0x58, 0x2d, 0x70, 0x8c, 0x23, 0x1a, 0xc6, -+ 0x58, 0x2a, 0xc0, 0x5d, 0x9e, 0xc4, 0x92, 0xd9, 0x92, 0xad, 0x21, 0x0b, 0xc3, 0xda, 0x17, 0x24, -+ 0x36, 0xe4, 0xf8, 0x3f, 0x51, 0x61, 0x1d, 0x1a, 0x27, 0x94, 0x05, 0x2e, 0x37, 0x1a, 0x28, 0x08, -+ 0x21, 0xa8, 0xbb, 0x6c, 0x12, 0x0f, 0x6b, 0x0f, 0x6a, 0x1b, 0x2d, 0x5b, 0xae, 0xc5, 0xab, 0x9c, -+ 0x13, 0xa3, 0xf5, 0x7a, 0x15, 0x3a, 0xda, 0xef, 0xce, 0x94, 0xc4, 0x5c, 0xca, 0xe9, 0xd8, 0x6d, -+ 0x8d, 0x13, 0x67, 0x2c, 0x0a, 0xeb, 0x47, 0x91, 0x7f, 0xcd, 0x80, 0x7f, 0x04, 0x2d, 0x86, 0x63, -+ 0x9a, 0x30, 0x11, 0xa6, 0x55, 0x79, 0xef, 0x6b, 0xea, 0xde, 0xbf, 0x20, 0x61, 0x72, 0x61, 0x9b, -+ 0x3d, 0x3b, 0x23, 0xd3, 0x21, 0xc4, 0xe3, 0xeb, 0x84, 0xd0, 0x27, 0x70, 0xeb, 0xc0, 0x4d, 0xe2, -+ 0xeb, 0xe8, 0x6a, 0x7d, 0x2a, 0xc2, 0x2f, 0x4e, 0x82, 0x6b, 0x1d, 0xfe, 0x63, 0x05, 0x9a, 0xdb, -+ 0x51, 0x72, 0x14, 0xbb, 0x13, 0x8c, 0xfe, 0x07, 0xda, 0x9c, 0x72, 0x77, 0xea, 0x24, 0x02, 0x94, -+ 0xe4, 0x75, 0x1b, 0x24, 0x4a, 0x11, 0x08, 0xb7, 0x63, 0xe6, 0x45, 0x89, 0xa6, 0xa8, 0x3e, 0xa8, -+ 0x6d, 0xd4, 0xed, 0xb6, 0xc2, 0x29, 0x92, 0x31, 0xac, 0xca, 0x3d, 0x87, 0x84, 0xce, 0x19, 0x66, -+ 0x21, 0x9e, 0x06, 0xd4, 0xc7, 0xf2, 0xfd, 0xd6, 0xed, 0x81, 0xdc, 0xda, 0x0f, 0x3f, 0x4f, 0x37, -+ 0xd0, 0xff, 0xc2, 0x20, 0xa5, 0x17, 0x41, 0x29, 0xa9, 0xeb, 0x92, 0xba, 0xaf, 0xa9, 0x8f, 0x34, -+ 0xda, 0xfa, 0x15, 0xf4, 0x5e, 0x9c, 0x32, 0xca, 0xf9, 0x94, 0x84, 0x93, 0x1d, 0x97, 0xbb, 0x22, -+ 0x7b, 0x44, 0x98, 0x11, 0xea, 0xc7, 0x5a, 0x5b, 0x03, 0xa2, 0xb7, 0x61, 0xc0, 0x15, 0x2d, 0xf6, -+ 0x1d, 0x43, 0x53, 0x95, 0x34, 0x2b, 0xe9, 0xc6, 0x81, 0x26, 0x7e, 0x03, 0x7a, 0x19, 0xb1, 0xc8, -+ 0x3f, 0x5a, 0xdf, 0x6e, 0x8a, 0x7d, 0x41, 0x02, 0x6c, 0x9d, 0x4b, 0x5f, 0xc9, 0x4b, 0x46, 0x6f, -+ 0x43, 0x2b, 0xf3, 0x43, 0x45, 0xbe, 0x90, 0x9e, 0x7a, 0x21, 0xc6, 0x9d, 0x76, 0x33, 0x75, 0xca, -+ 0x67, 0xd0, 0xe7, 0xa9, 0xe2, 0x8e, 0xef, 0x72, 0xb7, 0xf8, 0xa8, 0x8a, 0x56, 0xd9, 0x3d, 0x5e, -+ 0x80, 0xad, 0x4f, 0xa1, 0x75, 0x40, 0xfc, 0x58, 0x09, 0x1e, 0xc2, 0xb2, 0x97, 0x30, 0x86, 0x43, -+ 0x6e, 0x4c, 0xd6, 0x20, 0x5a, 0x83, 0xa5, 0x29, 0x09, 0x08, 0xd7, 0x66, 0x2a, 0xc0, 0xa2, 0x00, -+ 0xcf, 0x70, 0x40, 0xd9, 0x4c, 0x3a, 0x6c, 0x0d, 0x96, 0xf2, 0x97, 0xab, 0x00, 0x74, 0x17, 0x5a, -+ 0x81, 0x7b, 0x91, 0x5e, 0xaa, 0xd8, 0x69, 0x06, 0xee, 0x85, 0x52, 0x7e, 0x08, 0xcb, 0x27, 0x2e, -+ 0x99, 0x7a, 0x21, 0xd7, 0x5e, 0x31, 0x60, 0x26, 0xb0, 0x9e, 0x17, 0xf8, 0xd7, 0x2a, 0xb4, 0x95, -+ 0x44, 0xa5, 0xf0, 0x1a, 0x2c, 0x79, 0xae, 0x77, 0x9a, 0x8a, 0x94, 0x00, 0x7a, 0x68, 0x14, 0xa9, -+ 0xe6, 0x93, 0x70, 0xa6, 0xa9, 0x51, 0x6d, 0x13, 0x20, 0x7e, 0xe9, 0x46, 0x5a, 0xb7, 0xda, 0x25, -+ 0xc4, 0x2d, 0x41, 0xa3, 0xd4, 0x7d, 0x1f, 0x3a, 0xea, 0xdd, 0xe9, 0x23, 0xf5, 0x4b, 0x8e, 0xb4, -+ 0x15, 0x95, 0x3a, 0xf4, 0x1a, 0x74, 0x93, 0x18, 0x3b, 0xa7, 0x04, 0x33, 0x97, 0x79, 0xa7, 0xb3, -+ 0xe1, 0x92, 0xfa, 0x46, 0x26, 0x31, 0xde, 0x33, 0x38, 0xf4, 0x08, 0x96, 0x44, 0xfa, 0x8b, 0x87, -+ 0x0d, 0xf9, 0x39, 0xbe, 0x97, 0x67, 0x29, 0x4d, 0x1d, 0xcb, 0xdf, 0xdd, 0x90, 0xb3, 0x99, 0xad, -+ 0x48, 0x47, 0x1f, 0x01, 0x64, 0x48, 0xb4, 0x02, 0xb5, 0x33, 0x3c, 0xd3, 0x71, 0x28, 0x96, 0xc2, -+ 0x39, 0xe7, 0xee, 0x34, 0x31, 0x5e, 0x57, 0xc0, 0x27, 0xd5, 0x8f, 0x2a, 0x96, 0x07, 0xfd, 0xad, -+ 0xe9, 0x19, 0xa1, 0xb9, 0xe3, 0x6b, 0xb0, 0x14, 0xb8, 0x5f, 0x51, 0x66, 0x3c, 0x29, 0x01, 0x89, -+ 0x25, 0x21, 0x65, 0x86, 0x85, 0x04, 0x50, 0x0f, 0xaa, 0x34, 0x92, 0xfe, 0x6a, 0xd9, 0x55, 0x1a, -+ 0x65, 0x82, 0xea, 0x39, 0x41, 0xd6, 0x3f, 0xea, 0x00, 0x99, 0x14, 0x64, 0xc3, 0x88, 0x50, 0x27, -+ 0xc6, 0x4c, 0x94, 0x20, 0xce, 0xf1, 0x8c, 0xe3, 0xd8, 0x61, 0xd8, 0x4b, 0x58, 0x4c, 0xce, 0xc5, -+ 0xfd, 0x09, 0xb3, 0x6f, 0x29, 0xb3, 0xe7, 0x74, 0xb3, 0x6f, 0x13, 0x7a, 0xa8, 0xce, 0x6d, 0x89, -+ 0x63, 0xb6, 0x39, 0x85, 0xf6, 0xe1, 0x56, 0xc6, 0xd3, 0xcf, 0xb1, 0xab, 0x5e, 0xc5, 0x6e, 0x35, -+ 0x65, 0xe7, 0x67, 0xac, 0x76, 0x61, 0x95, 0x50, 0xe7, 0xeb, 0x04, 0x27, 0x05, 0x46, 0xb5, 0xab, -+ 0x18, 0x0d, 0x08, 0xfd, 0x89, 0x3c, 0x90, 0xb1, 0x39, 0x80, 0x3b, 0x39, 0x2b, 0x45, 0xb8, 0xe7, -+ 0x98, 0xd5, 0xaf, 0x62, 0xb6, 0x9e, 0x6a, 0x25, 0xf2, 0x41, 0xc6, 0xf1, 0x47, 0xb0, 0x4e, 0xa8, -+ 0xf3, 0xd2, 0x25, 0x7c, 0x9e, 0xdd, 0xd2, 0xf7, 0x18, 0x29, 0x3e, 0xba, 0x45, 0x5e, 0xca, 0xc8, -+ 0x00, 0xb3, 0x49, 0xc1, 0xc8, 0xc6, 0xf7, 0x18, 0xf9, 0x4c, 0x1e, 0xc8, 0xd8, 0x3c, 0x81, 0x01, -+ 0xa1, 0xf3, 0xda, 0x2c, 0x5f, 0xc5, 0xa4, 0x4f, 0x68, 0x51, 0x93, 0x2d, 0x18, 0xc4, 0xd8, 0xe3, -+ 0x94, 0xe5, 0x1f, 0x41, 0xf3, 0x2a, 0x16, 0x2b, 0x9a, 0x3e, 0xe5, 0x61, 0xfd, 0x1c, 0x3a, 0x7b, -+ 0xc9, 0x04, 0xf3, 0xe9, 0x71, 0x9a, 0x0c, 0x6e, 0x2c, 0xff, 0x58, 0xff, 0xaa, 0x42, 0x7b, 0x7b, -+ 0xc2, 0x68, 0x12, 0x15, 0x72, 0xb2, 0x0a, 0xd2, 0xf9, 0x9c, 0x2c, 0x49, 0x64, 0x4e, 0x56, 0xc4, -+ 0x1f, 0x40, 0x27, 0x90, 0xa1, 0xab, 0xe9, 0x55, 0x1e, 0x1a, 0x2c, 0x04, 0xb5, 0xdd, 0x0e, 0x72, -+ 0xc9, 0x6c, 0x0c, 0x10, 0x11, 0x3f, 0xd6, 0x67, 0x54, 0x3a, 0xea, 0xeb, 0x8a, 0xd0, 0xa4, 0x68, -+ 0xbb, 0x15, 0xa5, 0xd9, 0xfa, 0x3d, 0x68, 0x1f, 0x0b, 0x27, 0xe9, 0x03, 0x85, 0x64, 0x94, 0x79, -+ 0xcf, 0x86, 0xe3, 0x2c, 0x08, 0xf7, 0xa0, 0x7b, 0xaa, 0x5c, 0xa6, 0x0f, 0xa9, 0x37, 0xf4, 0x9a, -+ 0xb6, 0x24, 0xb3, 0x77, 0x9c, 0xf7, 0xac, 0xba, 0x80, 0xce, 0x69, 0x0e, 0x35, 0x3a, 0x84, 0xc1, -+ 0x02, 0x49, 0x49, 0x0e, 0xda, 0xc8, 0xe7, 0xa0, 0xf6, 0x23, 0xa4, 0x04, 0xe5, 0x4f, 0xe6, 0xf3, -+ 0xd2, 0x6f, 0xab, 0xd0, 0xf9, 0x31, 0xe6, 0x2f, 0x29, 0x3b, 0x53, 0xfa, 0x22, 0xa8, 0x87, 0x6e, -+ 0x80, 0x35, 0x47, 0xb9, 0x46, 0x77, 0xa0, 0xc9, 0x2e, 0x54, 0x02, 0xd1, 0xf7, 0xb9, 0xcc, 0x2e, -+ 0x64, 0x62, 0x40, 0xaf, 0x00, 0xb0, 0x0b, 0x27, 0x72, 0xbd, 0x33, 0xac, 0x3d, 0x58, 0xb7, 0x5b, -+ 0xec, 0xe2, 0x40, 0x21, 0xc4, 0x53, 0x60, 0x17, 0x0e, 0x66, 0x8c, 0xb2, 0x58, 0xe7, 0xaa, 0x26, -+ 0xbb, 0xd8, 0x95, 0xb0, 0x3e, 0xeb, 0x33, 0x1a, 0x45, 0xd8, 0x97, 0x39, 0x5a, 0x9e, 0xdd, 0x51, -+ 0x08, 0x21, 0x95, 0x1b, 0xa9, 0x0d, 0x25, 0x95, 0x67, 0x52, 0x79, 0x26, 0x75, 0x59, 0x9d, 0xe4, -+ 0x79, 0xa9, 0x3c, 0x95, 0xda, 0x54, 0x52, 0x79, 0x4e, 0x2a, 0xcf, 0xa4, 0xb6, 0xcc, 0x59, 0x2d, -+ 0xd5, 0xfa, 0x4d, 0x05, 0xd6, 0xe7, 0x0b, 0x3f, 0x5d, 0xa6, 0x7e, 0x00, 0x1d, 0x4f, 0xde, 0x57, -+ 0xe1, 0x4d, 0x0e, 0x16, 0x6e, 0xd2, 0x6e, 0x7b, 0xb9, 0x67, 0xfc, 0x21, 0x74, 0x43, 0xe5, 0xe0, -+ 0xf4, 0x69, 0xd6, 0xb2, 0x7b, 0xc9, 0xfb, 0xde, 0xee, 0x84, 0x39, 0xc8, 0xf2, 0x01, 0x7d, 0xc9, -+ 0x08, 0xc7, 0x87, 0x9c, 0x61, 0x37, 0xb8, 0x89, 0x06, 0x04, 0x41, 0x5d, 0x56, 0x2b, 0x35, 0x59, -+ 0x5f, 0xcb, 0xb5, 0xf5, 0x26, 0xac, 0x16, 0xa4, 0x68, 0x5b, 0x57, 0xa0, 0x36, 0xc5, 0xa1, 0xe4, -+ 0xde, 0xb5, 0xc5, 0xd2, 0x72, 0x61, 0x60, 0x63, 0xd7, 0xbf, 0x39, 0x6d, 0xb4, 0x88, 0x5a, 0x26, -+ 0x62, 0x03, 0x50, 0x5e, 0x84, 0x56, 0xc5, 0x68, 0x5d, 0xc9, 0x69, 0xfd, 0x1c, 0x06, 0xdb, 0x53, -+ 0x1a, 0xe3, 0x43, 0xee, 0x93, 0xf0, 0x26, 0x3a, 0xa6, 0x5f, 0xc2, 0xea, 0x0b, 0x3e, 0xfb, 0x52, -+ 0x30, 0x8b, 0xc9, 0x37, 0xf8, 0x86, 0xec, 0x63, 0xf4, 0xa5, 0xb1, 0x8f, 0xd1, 0x97, 0xa2, 0x59, -+ 0xf2, 0xe8, 0x34, 0x09, 0x42, 0x19, 0x0a, 0x5d, 0x5b, 0x43, 0xd6, 0x16, 0x74, 0x54, 0x0d, 0xfd, -+ 0x8c, 0xfa, 0xc9, 0x14, 0x97, 0xc6, 0xe0, 0x7d, 0x80, 0xc8, 0x65, 0x6e, 0x80, 0x39, 0x66, 0xea, -+ 0x0d, 0xb5, 0xec, 0x1c, 0xc6, 0xfa, 0x7d, 0x15, 0xd6, 0xd4, 0x48, 0xe4, 0x50, 0x4d, 0x02, 0x8c, -+ 0x09, 0x23, 0x68, 0x9e, 0xd2, 0x98, 0xe7, 0x18, 0xa6, 0xb0, 0x50, 0xd1, 0x0f, 0x0d, 0x37, 0xb1, -+ 0x2c, 0xcc, 0x29, 0x6a, 0x57, 0xcf, 0x29, 0x16, 0x26, 0x11, 0xf5, 0xc5, 0x49, 0x84, 0x88, 0x36, -+ 0x43, 0x44, 0x54, 0x8c, 0xb7, 0xec, 0x96, 0xc6, 0xec, 0xfb, 0xe8, 0x21, 0xf4, 0x27, 0x42, 0x4b, -+ 0xe7, 0x94, 0xd2, 0x33, 0x27, 0x72, 0xf9, 0xa9, 0x0c, 0xf5, 0x96, 0xdd, 0x95, 0xe8, 0x3d, 0x4a, -+ 0xcf, 0x0e, 0x5c, 0x7e, 0x8a, 0x3e, 0x86, 0x9e, 0x2e, 0x03, 0x03, 0xe9, 0xa2, 0x58, 0x7f, 0xfc, -+ 0x74, 0x14, 0xe5, 0xbd, 0x67, 0x77, 0xcf, 0x72, 0x50, 0x6c, 0xdd, 0x86, 0x5b, 0x3b, 0x38, 0xe6, -+ 0x8c, 0xce, 0x8a, 0x8e, 0xb1, 0x7e, 0x00, 0xb0, 0x1f, 0x72, 0xcc, 0x4e, 0x5c, 0x0f, 0xc7, 0xe8, -+ 0xdd, 0x3c, 0xa4, 0x8b, 0xa3, 0x95, 0xb1, 0x9a, 0x48, 0xa5, 0x1b, 0x76, 0x8e, 0xc6, 0x1a, 0x43, -+ 0xc3, 0xa6, 0x89, 0x48, 0x47, 0xaf, 0x9b, 0x95, 0x3e, 0xd7, 0xd1, 0xe7, 0x24, 0xd2, 0xd6, 0x7b, -+ 0xd6, 0x9e, 0x69, 0x61, 0x33, 0x76, 0xfa, 0x8a, 0xc6, 0xd0, 0x22, 0x06, 0xa7, 0xb3, 0xca, 0xa2, -+ 0xe8, 0x8c, 0xc4, 0xfa, 0x19, 0xac, 0x2a, 0x4e, 0x8a, 0xb3, 0x61, 0xf3, 0x3a, 0x34, 0x98, 0x51, -+ 0xa3, 0x92, 0x8d, 0xa2, 0x34, 0x91, 0xde, 0x43, 0xf7, 0x84, 0x30, 0x8f, 0xe1, 0x40, 0xf4, 0x1c, -+ 0x55, 0x79, 0x65, 0x19, 0x42, 0x78, 0x4b, 0xf4, 0xdb, 0x99, 0x99, 0xc6, 0x5b, 0xab, 0x30, 0x10, -+ 0x1b, 0x05, 0x89, 0xd6, 0x2f, 0x60, 0xf5, 0x79, 0x38, 0x25, 0x21, 0xde, 0x3e, 0x38, 0x7a, 0x86, -+ 0xd3, 0xac, 0x80, 0xa0, 0x2e, 0xaa, 0x27, 0xa9, 0x46, 0xd3, 0x96, 0x6b, 0x11, 0x26, 0xe1, 0xb1, -+ 0xe3, 0x45, 0x49, 0xac, 0x27, 0x43, 0x8d, 0xf0, 0x78, 0x3b, 0x4a, 0x62, 0x91, 0xe6, 0xc5, 0x67, -+ 0x9e, 0x86, 0xd3, 0x99, 0x8c, 0x95, 0xa6, 0xbd, 0xec, 0x45, 0xc9, 0xf3, 0x70, 0x3a, 0xb3, 0xfe, -+ 0x4f, 0xf6, 0xc2, 0x18, 0xfb, 0xb6, 0x1b, 0xfa, 0x34, 0xd8, 0xc1, 0xe7, 0x39, 0x09, 0x69, 0xdf, -+ 0x65, 0x72, 0xc2, 0xb7, 0x15, 0xe8, 0x3c, 0x99, 0xe0, 0x90, 0xef, 0x60, 0xee, 0x92, 0xa9, 0xec, -+ 0xad, 0xce, 0x31, 0x8b, 0x09, 0x0d, 0xf5, 0xc3, 0x37, 0xa0, 0x68, 0x8d, 0x49, 0x48, 0xb8, 0xe3, -+ 0xbb, 0x38, 0xa0, 0xa1, 0xf6, 0x02, 0x08, 0xd4, 0x8e, 0xc4, 0xa0, 0x37, 0xa1, 0xaf, 0x26, 0x77, -+ 0xce, 0xa9, 0x1b, 0xfa, 0x53, 0x11, 0x72, 0x6a, 0x92, 0xd1, 0x53, 0xe8, 0x3d, 0x8d, 0x45, 0x6f, -+ 0xc1, 0x8a, 0x0e, 0x88, 0x8c, 0xb2, 0x2e, 0x29, 0xfb, 0x1a, 0x5f, 0x20, 0x4d, 0xa2, 0x88, 0x32, -+ 0x1e, 0x3b, 0x31, 0xf6, 0x3c, 0x1a, 0x44, 0xba, 0x31, 0xe9, 0x1b, 0xfc, 0xa1, 0x42, 0x5b, 0x13, -+ 0x58, 0x7d, 0x2a, 0xec, 0xd4, 0x96, 0x64, 0x17, 0xdc, 0x0b, 0x70, 0xe0, 0x1c, 0x4f, 0xa9, 0x77, -+ 0xe6, 0x88, 0x34, 0xa5, 0x3d, 0x2c, 0x4a, 0x9f, 0x2d, 0x81, 0x3c, 0x24, 0xdf, 0xc8, 0x1e, 0x5c, -+ 0x50, 0x9d, 0x52, 0x1e, 0x4d, 0x93, 0x89, 0x13, 0x31, 0x7a, 0x8c, 0xb5, 0x89, 0xfd, 0x00, 0x07, -+ 0x7b, 0x0a, 0x7f, 0x20, 0xd0, 0xd6, 0x5f, 0x2a, 0xb0, 0x56, 0x94, 0xa4, 0x93, 0xee, 0x26, 0xac, -+ 0x15, 0x45, 0xe9, 0x0f, 0xb1, 0x2a, 0xf4, 0x06, 0x79, 0x81, 0xea, 0x93, 0xfc, 0x21, 0x74, 0xe5, -+ 0x38, 0xd7, 0xf1, 0x15, 0xa7, 0x62, 0xf9, 0x91, 0xbf, 0x17, 0xbb, 0xe3, 0xe6, 0x6f, 0xe9, 0x63, -+ 0xb8, 0xa3, 0xcd, 0x77, 0x16, 0xd5, 0x56, 0x0f, 0x62, 0x5d, 0x13, 0x3c, 0x9b, 0xd3, 0xfe, 0x0b, -+ 0x18, 0x66, 0xa8, 0xad, 0x99, 0x44, 0x1a, 0x5f, 0xbd, 0x0b, 0xab, 0x73, 0xc6, 0x3e, 0xf1, 0x7d, -+ 0x26, 0x03, 0xb4, 0x6e, 0x97, 0x6d, 0x59, 0x8f, 0xe1, 0xf6, 0x21, 0xe6, 0xca, 0x1b, 0x2e, 0xd7, -+ 0x3d, 0x81, 0x62, 0xb6, 0x02, 0xb5, 0x43, 0xec, 0x49, 0xe3, 0x6b, 0xb6, 0x58, 0x8a, 0x07, 0x78, -+ 0x14, 0x63, 0x4f, 0x5a, 0x59, 0xb3, 0xe5, 0xda, 0xfa, 0x73, 0x05, 0x96, 0x75, 0x9a, 0x14, 0xa9, -+ 0xde, 0x67, 0xe4, 0x1c, 0x33, 0xfd, 0xf4, 0x34, 0x84, 0xde, 0x80, 0x9e, 0x5a, 0x39, 0x34, 0xe2, -+ 0x84, 0xa6, 0xc9, 0xb7, 0xab, 0xb0, 0xcf, 0x15, 0x52, 0x4e, 0xea, 0xe4, 0x20, 0x4a, 0xf7, 0x7c, -+ 0x1a, 0x92, 0xe3, 0xb6, 0x58, 0x64, 0x06, 0x99, 0x6c, 0x5b, 0xb6, 0x86, 0xc4, 0x53, 0x37, 0xfc, -+ 0x96, 0x24, 0x3f, 0x03, 0x8a, 0xa7, 0x1e, 0xd0, 0x24, 0xe4, 0x4e, 0x44, 0x49, 0xc8, 0x75, 0x76, -+ 0x05, 0x89, 0x3a, 0x10, 0x18, 0xeb, 0xd7, 0x15, 0x68, 0xa8, 0x69, 0xb5, 0xe8, 0x32, 0xd3, 0x6f, -+ 0x5c, 0x95, 0xc8, 0x7a, 0x41, 0xca, 0x52, 0xdf, 0x35, 0xb9, 0x16, 0x71, 0x7c, 0x1e, 0xa8, 0x4c, -+ 0xad, 0x55, 0x3b, 0x0f, 0x64, 0x8a, 0x7e, 0x03, 0x7a, 0xd9, 0xa7, 0x52, 0xee, 0x2b, 0x15, 0xbb, -+ 0x29, 0x56, 0x92, 0x5d, 0xaa, 0xa9, 0xf5, 0x53, 0xd1, 0x5c, 0xa7, 0x93, 0xda, 0x15, 0xa8, 0x25, -+ 0xa9, 0x32, 0x62, 0x29, 0x30, 0x93, 0xf4, 0x23, 0x2b, 0x96, 0xe8, 0x21, 0xf4, 0x5c, 0xdf, 0x27, -+ 0xe2, 0xb8, 0x3b, 0x7d, 0x4a, 0xfc, 0x34, 0x48, 0x8b, 0x58, 0xeb, 0x6f, 0x15, 0xe8, 0x6f, 0xd3, -+ 0x68, 0xf6, 0x43, 0x32, 0xc5, 0xb9, 0x0c, 0x22, 0x95, 0xd4, 0xdf, 0x58, 0xb1, 0x16, 0x75, 0xe3, -+ 0x09, 0x99, 0x62, 0x15, 0x5a, 0xea, 0x66, 0x9b, 0x02, 0x21, 0xc3, 0xca, 0x6c, 0xa6, 0x03, 0xb0, -+ 0xae, 0xda, 0x7c, 0x46, 0x7d, 0x59, 0x21, 0xfb, 0x84, 0x39, 0xe9, 0xb8, 0xab, 0x6b, 0x2f, 0xfb, -+ 0x84, 0xc9, 0x2d, 0x6d, 0xc8, 0x92, 0x9c, 0xb8, 0xe6, 0x0d, 0x69, 0x28, 0x8c, 0x30, 0x64, 0x1d, -+ 0x1a, 0xf4, 0xe4, 0x24, 0xc6, 0x5c, 0xd6, 0xb2, 0x35, 0x5b, 0x43, 0x69, 0x9a, 0x6b, 0xe6, 0xd2, -+ 0xdc, 0x2d, 0x58, 0x95, 0xb3, 0xfd, 0x17, 0xcc, 0xf5, 0x48, 0x38, 0x31, 0xa9, 0x78, 0x0d, 0xd0, -+ 0x21, 0xa7, 0x51, 0x11, 0xfb, 0xe8, 0x77, 0x2b, 0x3a, 0x27, 0xea, 0x46, 0x17, 0x3d, 0x85, 0xfe, -+ 0xdc, 0x1f, 0x27, 0x48, 0x4f, 0x3e, 0xca, 0xff, 0x4f, 0x19, 0xad, 0x8f, 0xd5, 0x1f, 0x31, 0x63, -+ 0xf3, 0x47, 0xcc, 0x78, 0x37, 0x88, 0xf8, 0x0c, 0xed, 0x42, 0xaf, 0xf8, 0x17, 0x03, 0xba, 0x6b, -+ 0x0a, 0x85, 0x92, 0x3f, 0x1e, 0x2e, 0x65, 0xf3, 0x14, 0xfa, 0x73, 0xff, 0x36, 0x18, 0x7d, 0xca, -+ 0xff, 0x84, 0xb8, 0x94, 0xd1, 0x63, 0x68, 0xe7, 0xfe, 0x5e, 0x40, 0x43, 0xc5, 0x64, 0xf1, 0x1f, -+ 0x87, 0x4b, 0x19, 0x6c, 0x43, 0xb7, 0x30, 0xf1, 0x47, 0x23, 0x6d, 0x4f, 0xc9, 0xdf, 0x00, 0x97, -+ 0x32, 0xd9, 0x82, 0x76, 0x6e, 0xf0, 0x6e, 0xb4, 0x58, 0x9c, 0xee, 0x8f, 0xee, 0x94, 0xec, 0xe8, -+ 0xd4, 0xbb, 0x07, 0xdd, 0xc2, 0x98, 0xdc, 0x28, 0x52, 0x36, 0xa2, 0x1f, 0xdd, 0x2d, 0xdd, 0xd3, -+ 0x9c, 0x9e, 0x42, 0x7f, 0x6e, 0x68, 0x6e, 0x9c, 0x5b, 0x3e, 0x4b, 0xbf, 0xd4, 0xac, 0xcf, 0xe5, -+ 0x65, 0xe7, 0x7a, 0xa2, 0xdc, 0x65, 0x2f, 0x8e, 0xc8, 0x47, 0xf7, 0xca, 0x37, 0xb5, 0x56, 0xbb, -+ 0xd0, 0x2b, 0x4e, 0xc7, 0x0d, 0xb3, 0xd2, 0x99, 0xf9, 0xd5, 0x2f, 0xa7, 0x30, 0x28, 0xcf, 0x5e, -+ 0x4e, 0xd9, 0xfc, 0xfc, 0x52, 0x46, 0x4f, 0x00, 0x74, 0x07, 0xe4, 0x93, 0x30, 0xbd, 0xb2, 0x85, -+ 0xce, 0x2b, 0xbd, 0xb2, 0x92, 0x6e, 0xe9, 0x31, 0x80, 0x6a, 0x5c, 0x7c, 0x9a, 0x70, 0x74, 0xdb, -+ 0xa8, 0x31, 0xd7, 0x2d, 0x8d, 0x86, 0x8b, 0x1b, 0x0b, 0x0c, 0x30, 0x63, 0xd7, 0x61, 0xf0, 0x19, -+ 0x40, 0xd6, 0x10, 0x19, 0x06, 0x0b, 0x2d, 0xd2, 0x15, 0x3e, 0xe8, 0xe4, 0xdb, 0x1f, 0xa4, 0x6d, -+ 0x2d, 0x69, 0x89, 0xae, 0x60, 0xd1, 0x9f, 0x2b, 0x6f, 0x8b, 0x8f, 0x6d, 0xbe, 0xea, 0x1d, 0x2d, -+ 0x94, 0xb8, 0xe8, 0x43, 0xe8, 0xe4, 0xeb, 0x5a, 0xa3, 0x45, 0x49, 0xad, 0x3b, 0x2a, 0xd4, 0xb6, -+ 0xe8, 0x31, 0xf4, 0x8a, 0x55, 0x2b, 0xca, 0xc5, 0xc5, 0x42, 0x2d, 0x3b, 0xd2, 0x13, 0x9b, 0x1c, -+ 0xf9, 0xfb, 0x00, 0x59, 0x75, 0x6b, 0xdc, 0xb7, 0x50, 0xef, 0xce, 0x49, 0x7d, 0x02, 0x9d, 0x7c, -+ 0x26, 0x36, 0xea, 0x96, 0x64, 0xe7, 0xab, 0xb2, 0x56, 0x2e, 0x6b, 0x9b, 0xc7, 0xb7, 0x98, 0xc8, -+ 0xaf, 0xca, 0x5a, 0x85, 0xae, 0xcf, 0x24, 0x8b, 0xb2, 0x56, 0xf0, 0xaa, 0x5c, 0x5e, 0x6c, 0x91, -+ 0x8c, 0xfb, 0x4a, 0x1b, 0xa7, 0xab, 0x1e, 0x51, 0xbe, 0x1b, 0x30, 0xfe, 0x28, 0xe9, 0x10, 0xbe, -+ 0x27, 0xa8, 0xf3, 0x15, 0x7f, 0x2e, 0xa8, 0x4b, 0x1a, 0x81, 0x4b, 0x19, 0xed, 0x41, 0xff, 0xa9, -+ 0x29, 0xe6, 0x74, 0xa1, 0xa9, 0xd5, 0x29, 0x29, 0xac, 0x47, 0xa3, 0xb2, 0x2d, 0x1d, 0x59, 0x9f, -+ 0xc3, 0x60, 0xa1, 0xc8, 0x44, 0xf7, 0xd3, 0xc1, 0x62, 0x69, 0xf5, 0x79, 0xa9, 0x5a, 0xfb, 0xb0, -+ 0x32, 0x5f, 0x63, 0xa2, 0x57, 0xf4, 0xa5, 0x97, 0xd7, 0x9e, 0x97, 0xb2, 0xfa, 0x18, 0x9a, 0xa6, -+ 0xa6, 0x41, 0x7a, 0x80, 0x3b, 0x57, 0xe3, 0x5c, 0x76, 0x74, 0xab, 0xf3, 0xed, 0x77, 0xf7, 0x2b, -+ 0x7f, 0xff, 0xee, 0x7e, 0xe5, 0x9f, 0xdf, 0xdd, 0xaf, 0x1c, 0x37, 0xe4, 0xee, 0xfb, 0xff, 0x0e, -+ 0x00, 0x00, 0xff, 0xff, 0x8d, 0x89, 0xaa, 0x73, 0xc8, 0x21, 0x00, 0x00, - } -diff --git a/protocols/grpc/agent.proto b/protocols/grpc/agent.proto -index 348d792..b0fab6d 100644 ---- a/protocols/grpc/agent.proto -+++ b/protocols/grpc/agent.proto -@@ -316,6 +316,7 @@ message UpdateInterfaceRequest { - - message UpdateRoutesRequest { - Routes routes = 1; -+ bool increment = 2; - } - - message ListInterfacesRequest { --- -2.14.3 (Apple Git-98) - diff --git a/agent/patches/0003-kata-agent-add-kata-ipvs-command.patch b/agent/patches/0003-kata-agent-add-kata-ipvs-command.patch deleted file mode 100644 index c3d6c86..0000000 --- a/agent/patches/0003-kata-agent-add-kata-ipvs-command.patch +++ /dev/null @@ -1,971 +0,0 @@ -From 0f24d3d433e93a7309b9bac59e4a15e722391490 Mon Sep 17 00:00:00 2001 -From: xiadanni1 -Date: Tue, 18 Aug 2020 06:01:55 +0800 -Subject: [PATCH 03/16] kata-agent: add kata-ipvs command - -reason: add kata-ipvs command to update IPVS rules -in VM. - -Signed-off-by: xiadanni1 ---- - agent.go | 6 + - grpc.go | 4 + - ipvsadm.go | 130 +++++++++ - protocols/grpc/agent.pb.go | 663 ++++++++++++++++++++++++++++++++------------- - protocols/grpc/agent.proto | 11 + - 5 files changed, 633 insertions(+), 181 deletions(-) - create mode 100644 ipvsadm.go - -diff --git a/agent.go b/agent.go -index c1cac08..c161e93 100644 ---- a/agent.go -+++ b/agent.go -@@ -118,6 +118,11 @@ type sandboxStorage struct { - refCount int - } - -+type ipvsAdm struct { -+ ipvsLock sync.Mutex -+ ipvsRuleCnt uint64 -+} -+ - type sandbox struct { - sync.RWMutex - ctx context.Context -@@ -144,6 +149,7 @@ type sandbox struct { - sandboxPidNs bool - storages map[string]*sandboxStorage - stopServer chan struct{} -+ ipvsadm ipvsAdm - } - - var agentFields = logrus.Fields{ -diff --git a/grpc.go b/grpc.go -index 8fe8217..de2cae7 100644 ---- a/grpc.go -+++ b/grpc.go -@@ -1567,6 +1567,10 @@ func (a *agentGRPC) ListRoutes(ctx context.Context, req *pb.ListRoutesRequest) ( - return a.sandbox.listRoutes(nil) - } - -+func (a *agentGRPC) UpdateIPVSRule(ctx context.Context, req *pb.UpdateIPVSRequest) (*pb.IPVSResponse, error) { -+ return a.sandbox.updateIPVSRule(req) -+} -+ - func (a *agentGRPC) OnlineCPUMem(ctx context.Context, req *pb.OnlineCPUMemRequest) (*gpb.Empty, error) { - if !req.Wait { - go a.onlineCPUMem(req) -diff --git a/ipvsadm.go b/ipvsadm.go -new file mode 100644 -index 0000000..48eb19f ---- /dev/null -+++ b/ipvsadm.go -@@ -0,0 +1,130 @@ -+// Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. -+// SPDX-License-Identifier: Apache-2.0 -+// Description: IPVS related common functions -+// Author: xiadanni -+// Create: 2020-08-01 -+ -+package main -+ -+import ( -+ "os/exec" -+ "strconv" -+ "strings" -+ -+ "github.com/kata-containers/agent/protocols/grpc" -+ "google.golang.org/grpc/codes" -+ grpcStatus "google.golang.org/grpc/status" -+) -+ -+var ( -+ errNoIPVSRules = grpcStatus.Errorf(codes.InvalidArgument, "IPVS rule is nil") -+ errRuleExist = grpcStatus.Errorf(codes.InvalidArgument, "failed to restore IPVS rules: exist some rules in system, should clear first") -+ errInvalidIPVSRule = grpcStatus.Errorf(codes.InvalidArgument, "invalid IPVS rule, please check") -+ errIPVSRuleExceed = grpcStatus.Errorf(codes.InvalidArgument, "rules exceed limit, should clear first") -+) -+ -+const ( -+ validHeadLength = 7 -+ ruleLimitMax = 20000 -+ restoreHead = "restore" -+ conntrackNormalInfo = "have been deleted" -+ waitError = "no child processes" -+) -+ -+func (s *sandbox) updateIPVSRule(ipvsRule *grpc.UpdateIPVSRequest) (resultingIpvs *grpc.IPVSResponse, err error) { -+ var ( -+ restoreRuleCnt string -+ restoreRule string -+ operation string -+ restoreRuleCntUint uint64 -+ cmd *exec.Cmd -+ ) -+ -+ if ipvsRule == nil || ipvsRule.IPVSReq == "" { -+ return nil, errNoIPVSRules -+ } -+ -+ s.ipvsadm.ipvsLock.Lock() -+ defer s.ipvsadm.ipvsLock.Unlock() -+ -+ rule := ipvsRule.IPVSReq -+ if len(rule) < validHeadLength { -+ return nil, errInvalidIPVSRule -+ } -+ -+ if ruleHead := rule[:validHeadLength]; ruleHead == restoreHead { -+ if s.ipvsadm.ipvsRuleCnt != 0 { -+ return nil, errRuleExist -+ } -+ operation = restoreHead -+ -+ restoreItem := strings.Split(rule, "|") -+ if len(restoreItem) != 3 { -+ return nil, errInvalidIPVSRule -+ } -+ restoreRuleCnt = restoreItem[1] -+ if restoreRuleCntUint, err = strconv.ParseUint(restoreRuleCnt, 10, 64); err != nil { -+ return nil, errInvalidIPVSRule -+ } -+ restoreRule = restoreItem[2] -+ -+ cmd = exec.Command("/sbin/ipvsadm", "--restore") -+ cmd.Stdin = strings.NewReader(restoreRule) -+ } else { -+ item := strings.Fields(strings.TrimSpace(rule)) -+ if len(item) < 2 { -+ return nil, errInvalidIPVSRule -+ } -+ if item[0] != "ipvsadm" && item[0] != "conntrack" { -+ return nil, errInvalidIPVSRule -+ } -+ operation = item[1] -+ if s.ipvsadm.ipvsRuleCnt >= ruleLimitMax { -+ if operation == "--add-service" || operation == "-A" || operation == "--add-server" || operation == "-a" { -+ return nil, errIPVSRuleExceed -+ } -+ } -+ -+ restoreRuleCntUint = 0 -+ -+ cmd = exec.Command("/sbin/" + item[0]) -+ for i := 1; i < len(item); i++ { -+ cmd.Args = append(cmd.Args, item[i]) -+ } -+ } -+ -+ bytes, err := cmd.CombinedOutput() -+ -+ if err != nil && !strings.Contains(err.Error(), waitError) && !strings.Contains(string(bytes), conntrackNormalInfo) { -+ return nil, grpcStatus.Errorf(codes.Internal, -+ "exec IPVS command failed, stderr: %v, err: %v", string(bytes), err) -+ } -+ -+ s.sumRuleCount(operation, restoreRuleCntUint) -+ -+ rsp := &grpc.IPVSResponse{ -+ IPVSRes: string(bytes), -+ } -+ return rsp, nil -+} -+ -+func (s *sandbox) sumRuleCount(operation string, restoreRuleCntUint uint64) { -+ if operation == restoreHead { -+ s.ipvsadm.ipvsRuleCnt = restoreRuleCntUint -+ return -+ } -+ if operation == "--add-service" || operation == "-A" || operation == "--add-server" || operation == "-a" { -+ s.ipvsadm.ipvsRuleCnt++ -+ return -+ } -+ if operation == "--delete-service" || operation == "-D" || operation == "--delete-server" || operation == "-d" { -+ if s.ipvsadm.ipvsRuleCnt > 0 { -+ s.ipvsadm.ipvsRuleCnt-- -+ } -+ return -+ } -+ if operation == "--clear" || operation == "-C" { -+ s.ipvsadm.ipvsRuleCnt = 0 -+ return -+ } -+} -diff --git a/protocols/grpc/agent.pb.go b/protocols/grpc/agent.pb.go -index 1b887e5..04d0ee5 100644 ---- a/protocols/grpc/agent.pb.go -+++ b/protocols/grpc/agent.pb.go -@@ -63,6 +63,8 @@ - CopyFileRequest - StartTracingRequest - StopTracingRequest -+ UpdateIPVSRequest -+ IPVSResponse - CheckRequest - HealthCheckResponse - VersionCheckResponse -@@ -1842,6 +1844,40 @@ func (m *StopTracingRequest) String() string { return proto.CompactTe - func (*StopTracingRequest) ProtoMessage() {} - func (*StopTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{52} } - -+type UpdateIPVSRequest struct { -+ // IPVS_req is the IPVS rule message needed to update -+ IPVSReq string `protobuf:"bytes,1,opt,name=IPVS_req,json=IPVSReq,proto3" json:"IPVS_req,omitempty"` -+} -+ -+func (m *UpdateIPVSRequest) Reset() { *m = UpdateIPVSRequest{} } -+func (m *UpdateIPVSRequest) String() string { return proto.CompactTextString(m) } -+func (*UpdateIPVSRequest) ProtoMessage() {} -+func (*UpdateIPVSRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{53} } -+ -+func (m *UpdateIPVSRequest) GetIPVSReq() string { -+ if m != nil { -+ return m.IPVSReq -+ } -+ return "" -+} -+ -+type IPVSResponse struct { -+ // IPVS_res is the response of IPVS updating -+ IPVSRes string `protobuf:"bytes,1,opt,name=IPVS_res,json=IPVSRes,proto3" json:"IPVS_res,omitempty"` -+} -+ -+func (m *IPVSResponse) Reset() { *m = IPVSResponse{} } -+func (m *IPVSResponse) String() string { return proto.CompactTextString(m) } -+func (*IPVSResponse) ProtoMessage() {} -+func (*IPVSResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{54} } -+ -+func (m *IPVSResponse) GetIPVSRes() string { -+ if m != nil { -+ return m.IPVSRes -+ } -+ return "" -+} -+ - func init() { - proto.RegisterType((*CreateContainerRequest)(nil), "grpc.CreateContainerRequest") - proto.RegisterType((*StartContainerRequest)(nil), "grpc.StartContainerRequest") -@@ -1896,6 +1932,8 @@ func init() { - proto.RegisterType((*CopyFileRequest)(nil), "grpc.CopyFileRequest") - proto.RegisterType((*StartTracingRequest)(nil), "grpc.StartTracingRequest") - proto.RegisterType((*StopTracingRequest)(nil), "grpc.StopTracingRequest") -+ proto.RegisterType((*UpdateIPVSRequest)(nil), "grpc.UpdateIPVSRequest") -+ proto.RegisterType((*IPVSResponse)(nil), "grpc.IPVSResponse") - } - - // Reference imports to suppress errors if they are not otherwise used. -@@ -1938,6 +1976,7 @@ type AgentServiceClient interface { - UpdateRoutes(ctx context.Context, in *UpdateRoutesRequest, opts ...grpc1.CallOption) (*Routes, error) - ListInterfaces(ctx context.Context, in *ListInterfacesRequest, opts ...grpc1.CallOption) (*Interfaces, error) - ListRoutes(ctx context.Context, in *ListRoutesRequest, opts ...grpc1.CallOption) (*Routes, error) -+ UpdateIPVSRule(ctx context.Context, in *UpdateIPVSRequest, opts ...grpc1.CallOption) (*IPVSResponse, error) - // tracing - StartTracing(ctx context.Context, in *StartTracingRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error) - StopTracing(ctx context.Context, in *StopTracingRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error) -@@ -2140,6 +2179,15 @@ func (c *agentServiceClient) ListRoutes(ctx context.Context, in *ListRoutesReque - return out, nil - } - -+func (c *agentServiceClient) UpdateIPVSRule(ctx context.Context, in *UpdateIPVSRequest, opts ...grpc1.CallOption) (*IPVSResponse, error) { -+ out := new(IPVSResponse) -+ err := grpc1.Invoke(ctx, "/grpc.AgentService/UpdateIPVSRule", in, out, c.cc, opts...) -+ if err != nil { -+ return nil, err -+ } -+ return out, nil -+} -+ - func (c *agentServiceClient) StartTracing(ctx context.Context, in *StartTracingRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error) { - out := new(google_protobuf2.Empty) - err := grpc1.Invoke(ctx, "/grpc.AgentService/StartTracing", in, out, c.cc, opts...) -@@ -2262,6 +2310,7 @@ type AgentServiceServer interface { - UpdateRoutes(context.Context, *UpdateRoutesRequest) (*Routes, error) - ListInterfaces(context.Context, *ListInterfacesRequest) (*Interfaces, error) - ListRoutes(context.Context, *ListRoutesRequest) (*Routes, error) -+ UpdateIPVSRule(context.Context, *UpdateIPVSRequest) (*IPVSResponse, error) - // tracing - StartTracing(context.Context, *StartTracingRequest) (*google_protobuf2.Empty, error) - StopTracing(context.Context, *StopTracingRequest) (*google_protobuf2.Empty, error) -@@ -2640,6 +2689,24 @@ func _AgentService_ListRoutes_Handler(srv interface{}, ctx context.Context, dec - return interceptor(ctx, in, info, handler) - } - -+func _AgentService_UpdateIPVSRule_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) { -+ in := new(UpdateIPVSRequest) -+ if err := dec(in); err != nil { -+ return nil, err -+ } -+ if interceptor == nil { -+ return srv.(AgentServiceServer).UpdateIPVSRule(ctx, in) -+ } -+ info := &grpc1.UnaryServerInfo{ -+ Server: srv, -+ FullMethod: "/grpc.AgentService/UpdateIPVSRule", -+ } -+ handler := func(ctx context.Context, req interface{}) (interface{}, error) { -+ return srv.(AgentServiceServer).UpdateIPVSRule(ctx, req.(*UpdateIPVSRequest)) -+ } -+ return interceptor(ctx, in, info, handler) -+} -+ - func _AgentService_StartTracing_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) { - in := new(StartTracingRequest) - if err := dec(in); err != nil { -@@ -2904,6 +2971,10 @@ var _AgentService_serviceDesc = grpc1.ServiceDesc{ - MethodName: "ListRoutes", - Handler: _AgentService_ListRoutes_Handler, - }, -+ { -+ MethodName: "UpdateIPVSRule", -+ Handler: _AgentService_UpdateIPVSRule_Handler, -+ }, - { - MethodName: "StartTracing", - Handler: _AgentService_StartTracing_Handler, -@@ -5088,6 +5159,54 @@ func (m *StopTracingRequest) MarshalTo(dAtA []byte) (int, error) { - return i, nil - } - -+func (m *UpdateIPVSRequest) Marshal() (dAtA []byte, err error) { -+ size := m.Size() -+ dAtA = make([]byte, size) -+ n, err := m.MarshalTo(dAtA) -+ if err != nil { -+ return nil, err -+ } -+ return dAtA[:n], nil -+} -+ -+func (m *UpdateIPVSRequest) MarshalTo(dAtA []byte) (int, error) { -+ var i int -+ _ = i -+ var l int -+ _ = l -+ if len(m.IPVSReq) > 0 { -+ dAtA[i] = 0xa -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(len(m.IPVSReq))) -+ i += copy(dAtA[i:], m.IPVSReq) -+ } -+ return i, nil -+} -+ -+func (m *IPVSResponse) Marshal() (dAtA []byte, err error) { -+ size := m.Size() -+ dAtA = make([]byte, size) -+ n, err := m.MarshalTo(dAtA) -+ if err != nil { -+ return nil, err -+ } -+ return dAtA[:n], nil -+} -+ -+func (m *IPVSResponse) MarshalTo(dAtA []byte) (int, error) { -+ var i int -+ _ = i -+ var l int -+ _ = l -+ if len(m.IPVSRes) > 0 { -+ dAtA[i] = 0xa -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(len(m.IPVSRes))) -+ i += copy(dAtA[i:], m.IPVSRes) -+ } -+ return i, nil -+} -+ - func encodeVarintAgent(dAtA []byte, offset int, v uint64) int { - for v >= 1<<7 { - dAtA[offset] = uint8(v&0x7f | 0x80) -@@ -6019,6 +6138,26 @@ func (m *StopTracingRequest) Size() (n int) { - return n - } - -+func (m *UpdateIPVSRequest) Size() (n int) { -+ var l int -+ _ = l -+ l = len(m.IPVSReq) -+ if l > 0 { -+ n += 1 + l + sovAgent(uint64(l)) -+ } -+ return n -+} -+ -+func (m *IPVSResponse) Size() (n int) { -+ var l int -+ _ = l -+ l = len(m.IPVSRes) -+ if l > 0 { -+ n += 1 + l + sovAgent(uint64(l)) -+ } -+ return n -+} -+ - func sovAgent(x uint64) (n int) { - for { - n++ -@@ -12785,6 +12924,164 @@ func (m *StopTracingRequest) Unmarshal(dAtA []byte) error { - } - return nil - } -+func (m *UpdateIPVSRequest) Unmarshal(dAtA []byte) error { -+ l := len(dAtA) -+ iNdEx := 0 -+ for iNdEx < l { -+ preIndex := iNdEx -+ var wire uint64 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ wire |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ fieldNum := int32(wire >> 3) -+ wireType := int(wire & 0x7) -+ if wireType == 4 { -+ return fmt.Errorf("proto: UpdateIPVSRequest: wiretype end group for non-group") -+ } -+ if fieldNum <= 0 { -+ return fmt.Errorf("proto: UpdateIPVSRequest: illegal tag %d (wire type %d)", fieldNum, wire) -+ } -+ switch fieldNum { -+ case 1: -+ if wireType != 2 { -+ return fmt.Errorf("proto: wrong wireType = %d for field IPVSReq", wireType) -+ } -+ var stringLen uint64 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ stringLen |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ intStringLen := int(stringLen) -+ if intStringLen < 0 { -+ return ErrInvalidLengthAgent -+ } -+ postIndex := iNdEx + intStringLen -+ if postIndex > l { -+ return io.ErrUnexpectedEOF -+ } -+ m.IPVSReq = string(dAtA[iNdEx:postIndex]) -+ iNdEx = postIndex -+ default: -+ iNdEx = preIndex -+ skippy, err := skipAgent(dAtA[iNdEx:]) -+ if err != nil { -+ return err -+ } -+ if skippy < 0 { -+ return ErrInvalidLengthAgent -+ } -+ if (iNdEx + skippy) > l { -+ return io.ErrUnexpectedEOF -+ } -+ iNdEx += skippy -+ } -+ } -+ -+ if iNdEx > l { -+ return io.ErrUnexpectedEOF -+ } -+ return nil -+} -+func (m *IPVSResponse) Unmarshal(dAtA []byte) error { -+ l := len(dAtA) -+ iNdEx := 0 -+ for iNdEx < l { -+ preIndex := iNdEx -+ var wire uint64 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ wire |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ fieldNum := int32(wire >> 3) -+ wireType := int(wire & 0x7) -+ if wireType == 4 { -+ return fmt.Errorf("proto: IPVSResponse: wiretype end group for non-group") -+ } -+ if fieldNum <= 0 { -+ return fmt.Errorf("proto: IPVSResponse: illegal tag %d (wire type %d)", fieldNum, wire) -+ } -+ switch fieldNum { -+ case 1: -+ if wireType != 2 { -+ return fmt.Errorf("proto: wrong wireType = %d for field IPVSRes", wireType) -+ } -+ var stringLen uint64 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ stringLen |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ intStringLen := int(stringLen) -+ if intStringLen < 0 { -+ return ErrInvalidLengthAgent -+ } -+ postIndex := iNdEx + intStringLen -+ if postIndex > l { -+ return io.ErrUnexpectedEOF -+ } -+ m.IPVSRes = string(dAtA[iNdEx:postIndex]) -+ iNdEx = postIndex -+ default: -+ iNdEx = preIndex -+ skippy, err := skipAgent(dAtA[iNdEx:]) -+ if err != nil { -+ return err -+ } -+ if skippy < 0 { -+ return ErrInvalidLengthAgent -+ } -+ if (iNdEx + skippy) > l { -+ return io.ErrUnexpectedEOF -+ } -+ iNdEx += skippy -+ } -+ } -+ -+ if iNdEx > l { -+ return io.ErrUnexpectedEOF -+ } -+ return nil -+} - func skipAgent(dAtA []byte) (n int, err error) { - l := len(dAtA) - iNdEx := 0 -@@ -12893,185 +13190,189 @@ var ( - func init() { proto.RegisterFile("agent.proto", fileDescriptorAgent) } - - var fileDescriptorAgent = []byte{ -- // 2876 bytes of a gzipped FileDescriptorProto -- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0x4b, 0x6f, 0x1c, 0xc7, -- 0xd1, 0xd8, 0x07, 0x97, 0xbb, 0xb5, 0x2f, 0x6e, 0x93, 0xa2, 0x56, 0x2b, 0x59, 0x9f, 0x3c, 0xb6, -- 0x65, 0xfa, 0xf3, 0xe7, 0xa5, 0x2d, 0x1b, 0x9f, 0x5f, 0x70, 0x04, 0xf1, 0x11, 0x91, 0xb1, 0x15, -- 0x31, 0x43, 0x11, 0x4e, 0x10, 0x04, 0x83, 0xe1, 0x4c, 0x73, 0xd9, 0xe6, 0xce, 0xf4, 0xb8, 0xa7, -- 0x87, 0xe2, 0x3a, 0x40, 0x8e, 0xc9, 0x2d, 0x97, 0x00, 0xb9, 0xe5, 0x0f, 0x04, 0xb9, 0xe5, 0x98, -- 0x6b, 0x0e, 0x46, 0x4e, 0xf9, 0x05, 0x41, 0xe0, 0x9f, 0x90, 0x5f, 0x10, 0xf4, 0x6b, 0x1e, 0xbb, -- 0x43, 0x1a, 0x21, 0x08, 0xe4, 0xb2, 0xe8, 0xaa, 0xae, 0xae, 0x57, 0x77, 0xd5, 0x54, 0xd5, 0x42, -- 0xdb, 0x9d, 0xe0, 0x90, 0x8f, 0x23, 0x46, 0x39, 0x45, 0xf5, 0x09, 0x8b, 0xbc, 0x51, 0x8b, 0x7a, -- 0x44, 0x21, 0x46, 0xff, 0x3f, 0x21, 0xfc, 0x34, 0x39, 0x1e, 0x7b, 0x34, 0xd8, 0x3c, 0x73, 0xb9, -- 0xfb, 0x8e, 0x47, 0x43, 0xee, 0x92, 0x10, 0xb3, 0x78, 0x53, 0x1e, 0xdc, 0x8c, 0xce, 0x26, 0x9b, -- 0x7c, 0x16, 0xe1, 0x58, 0xfd, 0xea, 0x73, 0x77, 0x27, 0x94, 0x4e, 0xa6, 0x78, 0x53, 0x42, 0xc7, -- 0xc9, 0xc9, 0x26, 0x0e, 0x22, 0x3e, 0x53, 0x9b, 0xd6, 0x1f, 0xaa, 0xb0, 0xbe, 0xcd, 0xb0, 0xcb, -- 0xf1, 0xb6, 0xe1, 0x66, 0xe3, 0xaf, 0x13, 0x1c, 0x73, 0xf4, 0x2a, 0x74, 0x52, 0x09, 0x0e, 0xf1, -- 0x87, 0x95, 0x07, 0x95, 0x8d, 0x96, 0xdd, 0x4e, 0x71, 0xfb, 0x3e, 0xba, 0x0d, 0xcb, 0xf8, 0x02, -- 0x7b, 0x62, 0xb7, 0x2a, 0x77, 0x1b, 0x02, 0xdc, 0xf7, 0xd1, 0x7b, 0xd0, 0x8e, 0x39, 0x23, 0xe1, -- 0xc4, 0x49, 0x62, 0xcc, 0x86, 0xb5, 0x07, 0x95, 0x8d, 0xf6, 0xa3, 0x95, 0xb1, 0x30, 0x69, 0x7c, -- 0x28, 0x37, 0x8e, 0x62, 0xcc, 0x6c, 0x88, 0xd3, 0x35, 0x7a, 0x08, 0xcb, 0x3e, 0x3e, 0x27, 0x1e, -- 0x8e, 0x87, 0xf5, 0x07, 0xb5, 0x8d, 0xf6, 0xa3, 0x8e, 0x22, 0xdf, 0x91, 0x48, 0xdb, 0x6c, 0xa2, -- 0xb7, 0xa0, 0x19, 0x73, 0xca, 0xdc, 0x09, 0x8e, 0x87, 0x4b, 0x92, 0xb0, 0x6b, 0xf8, 0x4a, 0xac, -- 0x9d, 0x6e, 0xa3, 0x7b, 0x50, 0x7b, 0xbe, 0xbd, 0x3f, 0x6c, 0x48, 0xe9, 0xa0, 0xa9, 0x22, 0xec, -- 0xd9, 0x02, 0x8d, 0x5e, 0x83, 0x6e, 0xec, 0x86, 0xfe, 0x31, 0xbd, 0x70, 0x22, 0xe2, 0x87, 0xf1, -- 0x70, 0xf9, 0x41, 0x65, 0xa3, 0x69, 0x77, 0x34, 0xf2, 0x40, 0xe0, 0xac, 0x4f, 0xe0, 0xd6, 0x21, -- 0x77, 0x19, 0xbf, 0x86, 0x77, 0xac, 0x23, 0x58, 0xb7, 0x71, 0x40, 0xcf, 0xaf, 0xe5, 0xda, 0x21, -- 0x2c, 0x73, 0x12, 0x60, 0x9a, 0x70, 0xe9, 0xda, 0xae, 0x6d, 0x40, 0xeb, 0x4f, 0x15, 0x40, 0xbb, -- 0x17, 0xd8, 0x3b, 0x60, 0xd4, 0xc3, 0x71, 0xfc, 0x5f, 0xba, 0xae, 0x37, 0x61, 0x39, 0x52, 0x0a, -- 0x0c, 0xeb, 0x92, 0x5c, 0xdf, 0x82, 0xd1, 0xca, 0xec, 0x5a, 0x5f, 0xc1, 0xda, 0x21, 0x99, 0x84, -- 0xee, 0xf4, 0x06, 0xf5, 0x5d, 0x87, 0x46, 0x2c, 0x79, 0x4a, 0x55, 0xbb, 0xb6, 0x86, 0xac, 0x03, -- 0x40, 0x5f, 0xba, 0x84, 0xdf, 0x9c, 0x24, 0xeb, 0x1d, 0x58, 0x2d, 0x70, 0x8c, 0x23, 0x1a, 0xc6, -- 0x58, 0x2a, 0xc0, 0x5d, 0x9e, 0xc4, 0x92, 0xd9, 0x92, 0xad, 0x21, 0x0b, 0xc3, 0xda, 0x17, 0x24, -- 0x36, 0xe4, 0xf8, 0x3f, 0x51, 0x61, 0x1d, 0x1a, 0x27, 0x94, 0x05, 0x2e, 0x37, 0x1a, 0x28, 0x08, -- 0x21, 0xa8, 0xbb, 0x6c, 0x12, 0x0f, 0x6b, 0x0f, 0x6a, 0x1b, 0x2d, 0x5b, 0xae, 0xc5, 0xab, 0x9c, -- 0x13, 0xa3, 0xf5, 0x7a, 0x15, 0x3a, 0xda, 0xef, 0xce, 0x94, 0xc4, 0x5c, 0xca, 0xe9, 0xd8, 0x6d, -- 0x8d, 0x13, 0x67, 0x2c, 0x0a, 0xeb, 0x47, 0x91, 0x7f, 0xcd, 0x80, 0x7f, 0x04, 0x2d, 0x86, 0x63, -- 0x9a, 0x30, 0x11, 0xa6, 0x55, 0x79, 0xef, 0x6b, 0xea, 0xde, 0xbf, 0x20, 0x61, 0x72, 0x61, 0x9b, -- 0x3d, 0x3b, 0x23, 0xd3, 0x21, 0xc4, 0xe3, 0xeb, 0x84, 0xd0, 0x27, 0x70, 0xeb, 0xc0, 0x4d, 0xe2, -- 0xeb, 0xe8, 0x6a, 0x7d, 0x2a, 0xc2, 0x2f, 0x4e, 0x82, 0x6b, 0x1d, 0xfe, 0x63, 0x05, 0x9a, 0xdb, -- 0x51, 0x72, 0x14, 0xbb, 0x13, 0x8c, 0xfe, 0x07, 0xda, 0x9c, 0x72, 0x77, 0xea, 0x24, 0x02, 0x94, -- 0xe4, 0x75, 0x1b, 0x24, 0x4a, 0x11, 0x08, 0xb7, 0x63, 0xe6, 0x45, 0x89, 0xa6, 0xa8, 0x3e, 0xa8, -- 0x6d, 0xd4, 0xed, 0xb6, 0xc2, 0x29, 0x92, 0x31, 0xac, 0xca, 0x3d, 0x87, 0x84, 0xce, 0x19, 0x66, -- 0x21, 0x9e, 0x06, 0xd4, 0xc7, 0xf2, 0xfd, 0xd6, 0xed, 0x81, 0xdc, 0xda, 0x0f, 0x3f, 0x4f, 0x37, -- 0xd0, 0xff, 0xc2, 0x20, 0xa5, 0x17, 0x41, 0x29, 0xa9, 0xeb, 0x92, 0xba, 0xaf, 0xa9, 0x8f, 0x34, -- 0xda, 0xfa, 0x15, 0xf4, 0x5e, 0x9c, 0x32, 0xca, 0xf9, 0x94, 0x84, 0x93, 0x1d, 0x97, 0xbb, 0x22, -- 0x7b, 0x44, 0x98, 0x11, 0xea, 0xc7, 0x5a, 0x5b, 0x03, 0xa2, 0xb7, 0x61, 0xc0, 0x15, 0x2d, 0xf6, -- 0x1d, 0x43, 0x53, 0x95, 0x34, 0x2b, 0xe9, 0xc6, 0x81, 0x26, 0x7e, 0x03, 0x7a, 0x19, 0xb1, 0xc8, -- 0x3f, 0x5a, 0xdf, 0x6e, 0x8a, 0x7d, 0x41, 0x02, 0x6c, 0x9d, 0x4b, 0x5f, 0xc9, 0x4b, 0x46, 0x6f, -- 0x43, 0x2b, 0xf3, 0x43, 0x45, 0xbe, 0x90, 0x9e, 0x7a, 0x21, 0xc6, 0x9d, 0x76, 0x33, 0x75, 0xca, -- 0x67, 0xd0, 0xe7, 0xa9, 0xe2, 0x8e, 0xef, 0x72, 0xb7, 0xf8, 0xa8, 0x8a, 0x56, 0xd9, 0x3d, 0x5e, -- 0x80, 0xad, 0x4f, 0xa1, 0x75, 0x40, 0xfc, 0x58, 0x09, 0x1e, 0xc2, 0xb2, 0x97, 0x30, 0x86, 0x43, -- 0x6e, 0x4c, 0xd6, 0x20, 0x5a, 0x83, 0xa5, 0x29, 0x09, 0x08, 0xd7, 0x66, 0x2a, 0xc0, 0xa2, 0x00, -- 0xcf, 0x70, 0x40, 0xd9, 0x4c, 0x3a, 0x6c, 0x0d, 0x96, 0xf2, 0x97, 0xab, 0x00, 0x74, 0x17, 0x5a, -- 0x81, 0x7b, 0x91, 0x5e, 0xaa, 0xd8, 0x69, 0x06, 0xee, 0x85, 0x52, 0x7e, 0x08, 0xcb, 0x27, 0x2e, -- 0x99, 0x7a, 0x21, 0xd7, 0x5e, 0x31, 0x60, 0x26, 0xb0, 0x9e, 0x17, 0xf8, 0xd7, 0x2a, 0xb4, 0x95, -- 0x44, 0xa5, 0xf0, 0x1a, 0x2c, 0x79, 0xae, 0x77, 0x9a, 0x8a, 0x94, 0x00, 0x7a, 0x68, 0x14, 0xa9, -- 0xe6, 0x93, 0x70, 0xa6, 0xa9, 0x51, 0x6d, 0x13, 0x20, 0x7e, 0xe9, 0x46, 0x5a, 0xb7, 0xda, 0x25, -- 0xc4, 0x2d, 0x41, 0xa3, 0xd4, 0x7d, 0x1f, 0x3a, 0xea, 0xdd, 0xe9, 0x23, 0xf5, 0x4b, 0x8e, 0xb4, -- 0x15, 0x95, 0x3a, 0xf4, 0x1a, 0x74, 0x93, 0x18, 0x3b, 0xa7, 0x04, 0x33, 0x97, 0x79, 0xa7, 0xb3, -- 0xe1, 0x92, 0xfa, 0x46, 0x26, 0x31, 0xde, 0x33, 0x38, 0xf4, 0x08, 0x96, 0x44, 0xfa, 0x8b, 0x87, -- 0x0d, 0xf9, 0x39, 0xbe, 0x97, 0x67, 0x29, 0x4d, 0x1d, 0xcb, 0xdf, 0xdd, 0x90, 0xb3, 0x99, 0xad, -- 0x48, 0x47, 0x1f, 0x01, 0x64, 0x48, 0xb4, 0x02, 0xb5, 0x33, 0x3c, 0xd3, 0x71, 0x28, 0x96, 0xc2, -- 0x39, 0xe7, 0xee, 0x34, 0x31, 0x5e, 0x57, 0xc0, 0x27, 0xd5, 0x8f, 0x2a, 0x96, 0x07, 0xfd, 0xad, -- 0xe9, 0x19, 0xa1, 0xb9, 0xe3, 0x6b, 0xb0, 0x14, 0xb8, 0x5f, 0x51, 0x66, 0x3c, 0x29, 0x01, 0x89, -- 0x25, 0x21, 0x65, 0x86, 0x85, 0x04, 0x50, 0x0f, 0xaa, 0x34, 0x92, 0xfe, 0x6a, 0xd9, 0x55, 0x1a, -- 0x65, 0x82, 0xea, 0x39, 0x41, 0xd6, 0x3f, 0xea, 0x00, 0x99, 0x14, 0x64, 0xc3, 0x88, 0x50, 0x27, -- 0xc6, 0x4c, 0x94, 0x20, 0xce, 0xf1, 0x8c, 0xe3, 0xd8, 0x61, 0xd8, 0x4b, 0x58, 0x4c, 0xce, 0xc5, -- 0xfd, 0x09, 0xb3, 0x6f, 0x29, 0xb3, 0xe7, 0x74, 0xb3, 0x6f, 0x13, 0x7a, 0xa8, 0xce, 0x6d, 0x89, -- 0x63, 0xb6, 0x39, 0x85, 0xf6, 0xe1, 0x56, 0xc6, 0xd3, 0xcf, 0xb1, 0xab, 0x5e, 0xc5, 0x6e, 0x35, -- 0x65, 0xe7, 0x67, 0xac, 0x76, 0x61, 0x95, 0x50, 0xe7, 0xeb, 0x04, 0x27, 0x05, 0x46, 0xb5, 0xab, -- 0x18, 0x0d, 0x08, 0xfd, 0x89, 0x3c, 0x90, 0xb1, 0x39, 0x80, 0x3b, 0x39, 0x2b, 0x45, 0xb8, 0xe7, -- 0x98, 0xd5, 0xaf, 0x62, 0xb6, 0x9e, 0x6a, 0x25, 0xf2, 0x41, 0xc6, 0xf1, 0x47, 0xb0, 0x4e, 0xa8, -- 0xf3, 0xd2, 0x25, 0x7c, 0x9e, 0xdd, 0xd2, 0xf7, 0x18, 0x29, 0x3e, 0xba, 0x45, 0x5e, 0xca, 0xc8, -- 0x00, 0xb3, 0x49, 0xc1, 0xc8, 0xc6, 0xf7, 0x18, 0xf9, 0x4c, 0x1e, 0xc8, 0xd8, 0x3c, 0x81, 0x01, -- 0xa1, 0xf3, 0xda, 0x2c, 0x5f, 0xc5, 0xa4, 0x4f, 0x68, 0x51, 0x93, 0x2d, 0x18, 0xc4, 0xd8, 0xe3, -- 0x94, 0xe5, 0x1f, 0x41, 0xf3, 0x2a, 0x16, 0x2b, 0x9a, 0x3e, 0xe5, 0x61, 0xfd, 0x1c, 0x3a, 0x7b, -- 0xc9, 0x04, 0xf3, 0xe9, 0x71, 0x9a, 0x0c, 0x6e, 0x2c, 0xff, 0x58, 0xff, 0xaa, 0x42, 0x7b, 0x7b, -- 0xc2, 0x68, 0x12, 0x15, 0x72, 0xb2, 0x0a, 0xd2, 0xf9, 0x9c, 0x2c, 0x49, 0x64, 0x4e, 0x56, 0xc4, -- 0x1f, 0x40, 0x27, 0x90, 0xa1, 0xab, 0xe9, 0x55, 0x1e, 0x1a, 0x2c, 0x04, 0xb5, 0xdd, 0x0e, 0x72, -- 0xc9, 0x6c, 0x0c, 0x10, 0x11, 0x3f, 0xd6, 0x67, 0x54, 0x3a, 0xea, 0xeb, 0x8a, 0xd0, 0xa4, 0x68, -- 0xbb, 0x15, 0xa5, 0xd9, 0xfa, 0x3d, 0x68, 0x1f, 0x0b, 0x27, 0xe9, 0x03, 0x85, 0x64, 0x94, 0x79, -- 0xcf, 0x86, 0xe3, 0x2c, 0x08, 0xf7, 0xa0, 0x7b, 0xaa, 0x5c, 0xa6, 0x0f, 0xa9, 0x37, 0xf4, 0x9a, -- 0xb6, 0x24, 0xb3, 0x77, 0x9c, 0xf7, 0xac, 0xba, 0x80, 0xce, 0x69, 0x0e, 0x35, 0x3a, 0x84, 0xc1, -- 0x02, 0x49, 0x49, 0x0e, 0xda, 0xc8, 0xe7, 0xa0, 0xf6, 0x23, 0xa4, 0x04, 0xe5, 0x4f, 0xe6, 0xf3, -- 0xd2, 0x6f, 0xab, 0xd0, 0xf9, 0x31, 0xe6, 0x2f, 0x29, 0x3b, 0x53, 0xfa, 0x22, 0xa8, 0x87, 0x6e, -- 0x80, 0x35, 0x47, 0xb9, 0x46, 0x77, 0xa0, 0xc9, 0x2e, 0x54, 0x02, 0xd1, 0xf7, 0xb9, 0xcc, 0x2e, -- 0x64, 0x62, 0x40, 0xaf, 0x00, 0xb0, 0x0b, 0x27, 0x72, 0xbd, 0x33, 0xac, 0x3d, 0x58, 0xb7, 0x5b, -- 0xec, 0xe2, 0x40, 0x21, 0xc4, 0x53, 0x60, 0x17, 0x0e, 0x66, 0x8c, 0xb2, 0x58, 0xe7, 0xaa, 0x26, -- 0xbb, 0xd8, 0x95, 0xb0, 0x3e, 0xeb, 0x33, 0x1a, 0x45, 0xd8, 0x97, 0x39, 0x5a, 0x9e, 0xdd, 0x51, -- 0x08, 0x21, 0x95, 0x1b, 0xa9, 0x0d, 0x25, 0x95, 0x67, 0x52, 0x79, 0x26, 0x75, 0x59, 0x9d, 0xe4, -- 0x79, 0xa9, 0x3c, 0x95, 0xda, 0x54, 0x52, 0x79, 0x4e, 0x2a, 0xcf, 0xa4, 0xb6, 0xcc, 0x59, 0x2d, -- 0xd5, 0xfa, 0x4d, 0x05, 0xd6, 0xe7, 0x0b, 0x3f, 0x5d, 0xa6, 0x7e, 0x00, 0x1d, 0x4f, 0xde, 0x57, -- 0xe1, 0x4d, 0x0e, 0x16, 0x6e, 0xd2, 0x6e, 0x7b, 0xb9, 0x67, 0xfc, 0x21, 0x74, 0x43, 0xe5, 0xe0, -- 0xf4, 0x69, 0xd6, 0xb2, 0x7b, 0xc9, 0xfb, 0xde, 0xee, 0x84, 0x39, 0xc8, 0xf2, 0x01, 0x7d, 0xc9, -- 0x08, 0xc7, 0x87, 0x9c, 0x61, 0x37, 0xb8, 0x89, 0x06, 0x04, 0x41, 0x5d, 0x56, 0x2b, 0x35, 0x59, -- 0x5f, 0xcb, 0xb5, 0xf5, 0x26, 0xac, 0x16, 0xa4, 0x68, 0x5b, 0x57, 0xa0, 0x36, 0xc5, 0xa1, 0xe4, -- 0xde, 0xb5, 0xc5, 0xd2, 0x72, 0x61, 0x60, 0x63, 0xd7, 0xbf, 0x39, 0x6d, 0xb4, 0x88, 0x5a, 0x26, -- 0x62, 0x03, 0x50, 0x5e, 0x84, 0x56, 0xc5, 0x68, 0x5d, 0xc9, 0x69, 0xfd, 0x1c, 0x06, 0xdb, 0x53, -- 0x1a, 0xe3, 0x43, 0xee, 0x93, 0xf0, 0x26, 0x3a, 0xa6, 0x5f, 0xc2, 0xea, 0x0b, 0x3e, 0xfb, 0x52, -- 0x30, 0x8b, 0xc9, 0x37, 0xf8, 0x86, 0xec, 0x63, 0xf4, 0xa5, 0xb1, 0x8f, 0xd1, 0x97, 0xa2, 0x59, -- 0xf2, 0xe8, 0x34, 0x09, 0x42, 0x19, 0x0a, 0x5d, 0x5b, 0x43, 0xd6, 0x16, 0x74, 0x54, 0x0d, 0xfd, -- 0x8c, 0xfa, 0xc9, 0x14, 0x97, 0xc6, 0xe0, 0x7d, 0x80, 0xc8, 0x65, 0x6e, 0x80, 0x39, 0x66, 0xea, -- 0x0d, 0xb5, 0xec, 0x1c, 0xc6, 0xfa, 0x7d, 0x15, 0xd6, 0xd4, 0x48, 0xe4, 0x50, 0x4d, 0x02, 0x8c, -- 0x09, 0x23, 0x68, 0x9e, 0xd2, 0x98, 0xe7, 0x18, 0xa6, 0xb0, 0x50, 0xd1, 0x0f, 0x0d, 0x37, 0xb1, -- 0x2c, 0xcc, 0x29, 0x6a, 0x57, 0xcf, 0x29, 0x16, 0x26, 0x11, 0xf5, 0xc5, 0x49, 0x84, 0x88, 0x36, -- 0x43, 0x44, 0x54, 0x8c, 0xb7, 0xec, 0x96, 0xc6, 0xec, 0xfb, 0xe8, 0x21, 0xf4, 0x27, 0x42, 0x4b, -- 0xe7, 0x94, 0xd2, 0x33, 0x27, 0x72, 0xf9, 0xa9, 0x0c, 0xf5, 0x96, 0xdd, 0x95, 0xe8, 0x3d, 0x4a, -- 0xcf, 0x0e, 0x5c, 0x7e, 0x8a, 0x3e, 0x86, 0x9e, 0x2e, 0x03, 0x03, 0xe9, 0xa2, 0x58, 0x7f, 0xfc, -- 0x74, 0x14, 0xe5, 0xbd, 0x67, 0x77, 0xcf, 0x72, 0x50, 0x6c, 0xdd, 0x86, 0x5b, 0x3b, 0x38, 0xe6, -- 0x8c, 0xce, 0x8a, 0x8e, 0xb1, 0x7e, 0x00, 0xb0, 0x1f, 0x72, 0xcc, 0x4e, 0x5c, 0x0f, 0xc7, 0xe8, -- 0xdd, 0x3c, 0xa4, 0x8b, 0xa3, 0x95, 0xb1, 0x9a, 0x48, 0xa5, 0x1b, 0x76, 0x8e, 0xc6, 0x1a, 0x43, -- 0xc3, 0xa6, 0x89, 0x48, 0x47, 0xaf, 0x9b, 0x95, 0x3e, 0xd7, 0xd1, 0xe7, 0x24, 0xd2, 0xd6, 0x7b, -- 0xd6, 0x9e, 0x69, 0x61, 0x33, 0x76, 0xfa, 0x8a, 0xc6, 0xd0, 0x22, 0x06, 0xa7, 0xb3, 0xca, 0xa2, -- 0xe8, 0x8c, 0xc4, 0xfa, 0x19, 0xac, 0x2a, 0x4e, 0x8a, 0xb3, 0x61, 0xf3, 0x3a, 0x34, 0x98, 0x51, -- 0xa3, 0x92, 0x8d, 0xa2, 0x34, 0x91, 0xde, 0x43, 0xf7, 0x84, 0x30, 0x8f, 0xe1, 0x40, 0xf4, 0x1c, -- 0x55, 0x79, 0x65, 0x19, 0x42, 0x78, 0x4b, 0xf4, 0xdb, 0x99, 0x99, 0xc6, 0x5b, 0xab, 0x30, 0x10, -- 0x1b, 0x05, 0x89, 0xd6, 0x2f, 0x60, 0xf5, 0x79, 0x38, 0x25, 0x21, 0xde, 0x3e, 0x38, 0x7a, 0x86, -- 0xd3, 0xac, 0x80, 0xa0, 0x2e, 0xaa, 0x27, 0xa9, 0x46, 0xd3, 0x96, 0x6b, 0x11, 0x26, 0xe1, 0xb1, -- 0xe3, 0x45, 0x49, 0xac, 0x27, 0x43, 0x8d, 0xf0, 0x78, 0x3b, 0x4a, 0x62, 0x91, 0xe6, 0xc5, 0x67, -- 0x9e, 0x86, 0xd3, 0x99, 0x8c, 0x95, 0xa6, 0xbd, 0xec, 0x45, 0xc9, 0xf3, 0x70, 0x3a, 0xb3, 0xfe, -- 0x4f, 0xf6, 0xc2, 0x18, 0xfb, 0xb6, 0x1b, 0xfa, 0x34, 0xd8, 0xc1, 0xe7, 0x39, 0x09, 0x69, 0xdf, -- 0x65, 0x72, 0xc2, 0xb7, 0x15, 0xe8, 0x3c, 0x99, 0xe0, 0x90, 0xef, 0x60, 0xee, 0x92, 0xa9, 0xec, -- 0xad, 0xce, 0x31, 0x8b, 0x09, 0x0d, 0xf5, 0xc3, 0x37, 0xa0, 0x68, 0x8d, 0x49, 0x48, 0xb8, 0xe3, -- 0xbb, 0x38, 0xa0, 0xa1, 0xf6, 0x02, 0x08, 0xd4, 0x8e, 0xc4, 0xa0, 0x37, 0xa1, 0xaf, 0x26, 0x77, -- 0xce, 0xa9, 0x1b, 0xfa, 0x53, 0x11, 0x72, 0x6a, 0x92, 0xd1, 0x53, 0xe8, 0x3d, 0x8d, 0x45, 0x6f, -- 0xc1, 0x8a, 0x0e, 0x88, 0x8c, 0xb2, 0x2e, 0x29, 0xfb, 0x1a, 0x5f, 0x20, 0x4d, 0xa2, 0x88, 0x32, -- 0x1e, 0x3b, 0x31, 0xf6, 0x3c, 0x1a, 0x44, 0xba, 0x31, 0xe9, 0x1b, 0xfc, 0xa1, 0x42, 0x5b, 0x13, -- 0x58, 0x7d, 0x2a, 0xec, 0xd4, 0x96, 0x64, 0x17, 0xdc, 0x0b, 0x70, 0xe0, 0x1c, 0x4f, 0xa9, 0x77, -- 0xe6, 0x88, 0x34, 0xa5, 0x3d, 0x2c, 0x4a, 0x9f, 0x2d, 0x81, 0x3c, 0x24, 0xdf, 0xc8, 0x1e, 0x5c, -- 0x50, 0x9d, 0x52, 0x1e, 0x4d, 0x93, 0x89, 0x13, 0x31, 0x7a, 0x8c, 0xb5, 0x89, 0xfd, 0x00, 0x07, -- 0x7b, 0x0a, 0x7f, 0x20, 0xd0, 0xd6, 0x5f, 0x2a, 0xb0, 0x56, 0x94, 0xa4, 0x93, 0xee, 0x26, 0xac, -- 0x15, 0x45, 0xe9, 0x0f, 0xb1, 0x2a, 0xf4, 0x06, 0x79, 0x81, 0xea, 0x93, 0xfc, 0x21, 0x74, 0xe5, -- 0x38, 0xd7, 0xf1, 0x15, 0xa7, 0x62, 0xf9, 0x91, 0xbf, 0x17, 0xbb, 0xe3, 0xe6, 0x6f, 0xe9, 0x63, -- 0xb8, 0xa3, 0xcd, 0x77, 0x16, 0xd5, 0x56, 0x0f, 0x62, 0x5d, 0x13, 0x3c, 0x9b, 0xd3, 0xfe, 0x0b, -- 0x18, 0x66, 0xa8, 0xad, 0x99, 0x44, 0x1a, 0x5f, 0xbd, 0x0b, 0xab, 0x73, 0xc6, 0x3e, 0xf1, 0x7d, -- 0x26, 0x03, 0xb4, 0x6e, 0x97, 0x6d, 0x59, 0x8f, 0xe1, 0xf6, 0x21, 0xe6, 0xca, 0x1b, 0x2e, 0xd7, -- 0x3d, 0x81, 0x62, 0xb6, 0x02, 0xb5, 0x43, 0xec, 0x49, 0xe3, 0x6b, 0xb6, 0x58, 0x8a, 0x07, 0x78, -- 0x14, 0x63, 0x4f, 0x5a, 0x59, 0xb3, 0xe5, 0xda, 0xfa, 0x73, 0x05, 0x96, 0x75, 0x9a, 0x14, 0xa9, -- 0xde, 0x67, 0xe4, 0x1c, 0x33, 0xfd, 0xf4, 0x34, 0x84, 0xde, 0x80, 0x9e, 0x5a, 0x39, 0x34, 0xe2, -- 0x84, 0xa6, 0xc9, 0xb7, 0xab, 0xb0, 0xcf, 0x15, 0x52, 0x4e, 0xea, 0xe4, 0x20, 0x4a, 0xf7, 0x7c, -- 0x1a, 0x92, 0xe3, 0xb6, 0x58, 0x64, 0x06, 0x99, 0x6c, 0x5b, 0xb6, 0x86, 0xc4, 0x53, 0x37, 0xfc, -- 0x96, 0x24, 0x3f, 0x03, 0x8a, 0xa7, 0x1e, 0xd0, 0x24, 0xe4, 0x4e, 0x44, 0x49, 0xc8, 0x75, 0x76, -- 0x05, 0x89, 0x3a, 0x10, 0x18, 0xeb, 0xd7, 0x15, 0x68, 0xa8, 0x69, 0xb5, 0xe8, 0x32, 0xd3, 0x6f, -- 0x5c, 0x95, 0xc8, 0x7a, 0x41, 0xca, 0x52, 0xdf, 0x35, 0xb9, 0x16, 0x71, 0x7c, 0x1e, 0xa8, 0x4c, -- 0xad, 0x55, 0x3b, 0x0f, 0x64, 0x8a, 0x7e, 0x03, 0x7a, 0xd9, 0xa7, 0x52, 0xee, 0x2b, 0x15, 0xbb, -- 0x29, 0x56, 0x92, 0x5d, 0xaa, 0xa9, 0xf5, 0x53, 0xd1, 0x5c, 0xa7, 0x93, 0xda, 0x15, 0xa8, 0x25, -- 0xa9, 0x32, 0x62, 0x29, 0x30, 0x93, 0xf4, 0x23, 0x2b, 0x96, 0xe8, 0x21, 0xf4, 0x5c, 0xdf, 0x27, -- 0xe2, 0xb8, 0x3b, 0x7d, 0x4a, 0xfc, 0x34, 0x48, 0x8b, 0x58, 0xeb, 0x6f, 0x15, 0xe8, 0x6f, 0xd3, -- 0x68, 0xf6, 0x43, 0x32, 0xc5, 0xb9, 0x0c, 0x22, 0x95, 0xd4, 0xdf, 0x58, 0xb1, 0x16, 0x75, 0xe3, -- 0x09, 0x99, 0x62, 0x15, 0x5a, 0xea, 0x66, 0x9b, 0x02, 0x21, 0xc3, 0xca, 0x6c, 0xa6, 0x03, 0xb0, -- 0xae, 0xda, 0x7c, 0x46, 0x7d, 0x59, 0x21, 0xfb, 0x84, 0x39, 0xe9, 0xb8, 0xab, 0x6b, 0x2f, 0xfb, -- 0x84, 0xc9, 0x2d, 0x6d, 0xc8, 0x92, 0x9c, 0xb8, 0xe6, 0x0d, 0x69, 0x28, 0x8c, 0x30, 0x64, 0x1d, -- 0x1a, 0xf4, 0xe4, 0x24, 0xc6, 0x5c, 0xd6, 0xb2, 0x35, 0x5b, 0x43, 0x69, 0x9a, 0x6b, 0xe6, 0xd2, -- 0xdc, 0x2d, 0x58, 0x95, 0xb3, 0xfd, 0x17, 0xcc, 0xf5, 0x48, 0x38, 0x31, 0xa9, 0x78, 0x0d, 0xd0, -- 0x21, 0xa7, 0x51, 0x11, 0xfb, 0xe8, 0x77, 0x2b, 0x3a, 0x27, 0xea, 0x46, 0x17, 0x3d, 0x85, 0xfe, -- 0xdc, 0x1f, 0x27, 0x48, 0x4f, 0x3e, 0xca, 0xff, 0x4f, 0x19, 0xad, 0x8f, 0xd5, 0x1f, 0x31, 0x63, -- 0xf3, 0x47, 0xcc, 0x78, 0x37, 0x88, 0xf8, 0x0c, 0xed, 0x42, 0xaf, 0xf8, 0x17, 0x03, 0xba, 0x6b, -- 0x0a, 0x85, 0x92, 0x3f, 0x1e, 0x2e, 0x65, 0xf3, 0x14, 0xfa, 0x73, 0xff, 0x36, 0x18, 0x7d, 0xca, -- 0xff, 0x84, 0xb8, 0x94, 0xd1, 0x63, 0x68, 0xe7, 0xfe, 0x5e, 0x40, 0x43, 0xc5, 0x64, 0xf1, 0x1f, -- 0x87, 0x4b, 0x19, 0x6c, 0x43, 0xb7, 0x30, 0xf1, 0x47, 0x23, 0x6d, 0x4f, 0xc9, 0xdf, 0x00, 0x97, -- 0x32, 0xd9, 0x82, 0x76, 0x6e, 0xf0, 0x6e, 0xb4, 0x58, 0x9c, 0xee, 0x8f, 0xee, 0x94, 0xec, 0xe8, -- 0xd4, 0xbb, 0x07, 0xdd, 0xc2, 0x98, 0xdc, 0x28, 0x52, 0x36, 0xa2, 0x1f, 0xdd, 0x2d, 0xdd, 0xd3, -- 0x9c, 0x9e, 0x42, 0x7f, 0x6e, 0x68, 0x6e, 0x9c, 0x5b, 0x3e, 0x4b, 0xbf, 0xd4, 0xac, 0xcf, 0xe5, -- 0x65, 0xe7, 0x7a, 0xa2, 0xdc, 0x65, 0x2f, 0x8e, 0xc8, 0x47, 0xf7, 0xca, 0x37, 0xb5, 0x56, 0xbb, -- 0xd0, 0x2b, 0x4e, 0xc7, 0x0d, 0xb3, 0xd2, 0x99, 0xf9, 0xd5, 0x2f, 0xa7, 0x30, 0x28, 0xcf, 0x5e, -- 0x4e, 0xd9, 0xfc, 0xfc, 0x52, 0x46, 0x4f, 0x00, 0x74, 0x07, 0xe4, 0x93, 0x30, 0xbd, 0xb2, 0x85, -- 0xce, 0x2b, 0xbd, 0xb2, 0x92, 0x6e, 0xe9, 0x31, 0x80, 0x6a, 0x5c, 0x7c, 0x9a, 0x70, 0x74, 0xdb, -- 0xa8, 0x31, 0xd7, 0x2d, 0x8d, 0x86, 0x8b, 0x1b, 0x0b, 0x0c, 0x30, 0x63, 0xd7, 0x61, 0xf0, 0x19, -- 0x40, 0xd6, 0x10, 0x19, 0x06, 0x0b, 0x2d, 0xd2, 0x15, 0x3e, 0xe8, 0xe4, 0xdb, 0x1f, 0xa4, 0x6d, -- 0x2d, 0x69, 0x89, 0xae, 0x60, 0xd1, 0x9f, 0x2b, 0x6f, 0x8b, 0x8f, 0x6d, 0xbe, 0xea, 0x1d, 0x2d, -- 0x94, 0xb8, 0xe8, 0x43, 0xe8, 0xe4, 0xeb, 0x5a, 0xa3, 0x45, 0x49, 0xad, 0x3b, 0x2a, 0xd4, 0xb6, -- 0xe8, 0x31, 0xf4, 0x8a, 0x55, 0x2b, 0xca, 0xc5, 0xc5, 0x42, 0x2d, 0x3b, 0xd2, 0x13, 0x9b, 0x1c, -- 0xf9, 0xfb, 0x00, 0x59, 0x75, 0x6b, 0xdc, 0xb7, 0x50, 0xef, 0xce, 0x49, 0x7d, 0x02, 0x9d, 0x7c, -- 0x26, 0x36, 0xea, 0x96, 0x64, 0xe7, 0xab, 0xb2, 0x56, 0x2e, 0x6b, 0x9b, 0xc7, 0xb7, 0x98, 0xc8, -- 0xaf, 0xca, 0x5a, 0x85, 0xae, 0xcf, 0x24, 0x8b, 0xb2, 0x56, 0xf0, 0xaa, 0x5c, 0x5e, 0x6c, 0x91, -- 0x8c, 0xfb, 0x4a, 0x1b, 0xa7, 0xab, 0x1e, 0x51, 0xbe, 0x1b, 0x30, 0xfe, 0x28, 0xe9, 0x10, 0xbe, -- 0x27, 0xa8, 0xf3, 0x15, 0x7f, 0x2e, 0xa8, 0x4b, 0x1a, 0x81, 0x4b, 0x19, 0xed, 0x41, 0xff, 0xa9, -- 0x29, 0xe6, 0x74, 0xa1, 0xa9, 0xd5, 0x29, 0x29, 0xac, 0x47, 0xa3, 0xb2, 0x2d, 0x1d, 0x59, 0x9f, -- 0xc3, 0x60, 0xa1, 0xc8, 0x44, 0xf7, 0xd3, 0xc1, 0x62, 0x69, 0xf5, 0x79, 0xa9, 0x5a, 0xfb, 0xb0, -- 0x32, 0x5f, 0x63, 0xa2, 0x57, 0xf4, 0xa5, 0x97, 0xd7, 0x9e, 0x97, 0xb2, 0xfa, 0x18, 0x9a, 0xa6, -- 0xa6, 0x41, 0x7a, 0x80, 0x3b, 0x57, 0xe3, 0x5c, 0x76, 0x74, 0xab, 0xf3, 0xed, 0x77, 0xf7, 0x2b, -- 0x7f, 0xff, 0xee, 0x7e, 0xe5, 0x9f, 0xdf, 0xdd, 0xaf, 0x1c, 0x37, 0xe4, 0xee, 0xfb, 0xff, 0x0e, -- 0x00, 0x00, 0xff, 0xff, 0x8d, 0x89, 0xaa, 0x73, 0xc8, 0x21, 0x00, 0x00, -+ // 2931 bytes of a gzipped FileDescriptorProto -+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0xcb, 0x6e, 0x1c, 0xc7, -+ 0xb5, 0x98, 0x07, 0x87, 0x33, 0x67, 0x5e, 0x9c, 0x22, 0x45, 0x8d, 0x46, 0xb2, 0xae, 0xdc, 0xb6, -+ 0x65, 0xfa, 0xfa, 0x7a, 0x68, 0xcb, 0xc6, 0xf5, 0x0b, 0xbe, 0x82, 0x48, 0xe9, 0x8a, 0x8c, 0xad, -+ 0x88, 0xe9, 0x91, 0xe2, 0x04, 0x41, 0xd0, 0x68, 0x76, 0x97, 0x86, 0x65, 0x4e, 0x77, 0xb5, 0xab, -+ 0xaa, 0x29, 0x8e, 0x03, 0x64, 0x99, 0xec, 0xb2, 0xcc, 0x2e, 0x3f, 0x10, 0x64, 0x97, 0x65, 0xb6, -+ 0x59, 0x18, 0x59, 0x05, 0xf9, 0x80, 0x20, 0xf0, 0x27, 0xe4, 0x0b, 0x82, 0x7a, 0xf5, 0x63, 0x66, -+ 0x48, 0x23, 0x82, 0x80, 0x6c, 0x1a, 0x75, 0x4e, 0x9d, 0x3a, 0xaf, 0xaa, 0x3a, 0x75, 0xce, 0x69, -+ 0x68, 0xfb, 0x53, 0x1c, 0x8b, 0x71, 0xc2, 0xa8, 0xa0, 0xa8, 0x3e, 0x65, 0x49, 0x30, 0x6a, 0xd1, -+ 0x80, 0x68, 0xc4, 0xe8, 0x7f, 0xa7, 0x44, 0x9c, 0xa4, 0xc7, 0xe3, 0x80, 0x46, 0xbb, 0xa7, 0xbe, -+ 0xf0, 0xdf, 0x09, 0x68, 0x2c, 0x7c, 0x12, 0x63, 0xc6, 0x77, 0xd5, 0xc2, 0xdd, 0xe4, 0x74, 0xba, -+ 0x2b, 0xe6, 0x09, 0xe6, 0xfa, 0x6b, 0xd6, 0x5d, 0x9f, 0x52, 0x3a, 0x9d, 0xe1, 0x5d, 0x05, 0x1d, -+ 0xa7, 0xcf, 0x76, 0x71, 0x94, 0x88, 0xb9, 0x9e, 0x74, 0x7e, 0x57, 0x85, 0xed, 0x7d, 0x86, 0x7d, -+ 0x81, 0xf7, 0x2d, 0x37, 0x17, 0x7f, 0x9d, 0x62, 0x2e, 0xd0, 0xab, 0xd0, 0xc9, 0x24, 0x78, 0x24, -+ 0x1c, 0x56, 0x6e, 0x55, 0x76, 0x5a, 0x6e, 0x3b, 0xc3, 0x1d, 0x86, 0xe8, 0x2a, 0xac, 0xe3, 0x73, -+ 0x1c, 0xc8, 0xd9, 0xaa, 0x9a, 0x6d, 0x48, 0xf0, 0x30, 0x44, 0xef, 0x41, 0x9b, 0x0b, 0x46, 0xe2, -+ 0xa9, 0x97, 0x72, 0xcc, 0x86, 0xb5, 0x5b, 0x95, 0x9d, 0xf6, 0x9d, 0x8d, 0xb1, 0x34, 0x69, 0x3c, -+ 0x51, 0x13, 0x4f, 0x39, 0x66, 0x2e, 0xf0, 0x6c, 0x8c, 0x6e, 0xc3, 0x7a, 0x88, 0xcf, 0x48, 0x80, -+ 0xf9, 0xb0, 0x7e, 0xab, 0xb6, 0xd3, 0xbe, 0xd3, 0xd1, 0xe4, 0xf7, 0x15, 0xd2, 0xb5, 0x93, 0xe8, -+ 0x2d, 0x68, 0x72, 0x41, 0x99, 0x3f, 0xc5, 0x7c, 0xb8, 0xa6, 0x08, 0xbb, 0x96, 0xaf, 0xc2, 0xba, -+ 0xd9, 0x34, 0xba, 0x01, 0xb5, 0xc7, 0xfb, 0x87, 0xc3, 0x86, 0x92, 0x0e, 0x86, 0x2a, 0xc1, 0x81, -+ 0x2b, 0xd1, 0xe8, 0x35, 0xe8, 0x72, 0x3f, 0x0e, 0x8f, 0xe9, 0xb9, 0x97, 0x90, 0x30, 0xe6, 0xc3, -+ 0xf5, 0x5b, 0x95, 0x9d, 0xa6, 0xdb, 0x31, 0xc8, 0x23, 0x89, 0x73, 0x3e, 0x81, 0x2b, 0x13, 0xe1, -+ 0x33, 0xf1, 0x02, 0xde, 0x71, 0x9e, 0xc2, 0xb6, 0x8b, 0x23, 0x7a, 0xf6, 0x42, 0xae, 0x1d, 0xc2, -+ 0xba, 0x20, 0x11, 0xa6, 0xa9, 0x50, 0xae, 0xed, 0xba, 0x16, 0x74, 0xfe, 0x50, 0x01, 0xf4, 0xe0, -+ 0x1c, 0x07, 0x47, 0x8c, 0x06, 0x98, 0xf3, 0xff, 0xd0, 0x76, 0xbd, 0x09, 0xeb, 0x89, 0x56, 0x60, -+ 0x58, 0x57, 0xe4, 0x66, 0x17, 0xac, 0x56, 0x76, 0xd6, 0xf9, 0x0a, 0xb6, 0x26, 0x64, 0x1a, 0xfb, -+ 0xb3, 0x97, 0xa8, 0xef, 0x36, 0x34, 0xb8, 0xe2, 0xa9, 0x54, 0xed, 0xba, 0x06, 0x72, 0x8e, 0x00, -+ 0x7d, 0xe9, 0x13, 0xf1, 0xf2, 0x24, 0x39, 0xef, 0xc0, 0x66, 0x89, 0x23, 0x4f, 0x68, 0xcc, 0xb1, -+ 0x52, 0x40, 0xf8, 0x22, 0xe5, 0x8a, 0xd9, 0x9a, 0x6b, 0x20, 0x07, 0xc3, 0xd6, 0x17, 0x84, 0x5b, -+ 0x72, 0xfc, 0xef, 0xa8, 0xb0, 0x0d, 0x8d, 0x67, 0x94, 0x45, 0xbe, 0xb0, 0x1a, 0x68, 0x08, 0x21, -+ 0xa8, 0xfb, 0x6c, 0xca, 0x87, 0xb5, 0x5b, 0xb5, 0x9d, 0x96, 0xab, 0xc6, 0xf2, 0x54, 0x2e, 0x88, -+ 0x31, 0x7a, 0xbd, 0x0a, 0x1d, 0xe3, 0x77, 0x6f, 0x46, 0xb8, 0x50, 0x72, 0x3a, 0x6e, 0xdb, 0xe0, -+ 0xe4, 0x1a, 0x87, 0xc2, 0xf6, 0xd3, 0x24, 0x7c, 0xc1, 0x0b, 0x7f, 0x07, 0x5a, 0x0c, 0x73, 0x9a, -+ 0x32, 0x79, 0x4d, 0xab, 0x6a, 0xdf, 0xb7, 0xf4, 0xbe, 0x7f, 0x41, 0xe2, 0xf4, 0xdc, 0xb5, 0x73, -+ 0x6e, 0x4e, 0x66, 0xae, 0x90, 0xe0, 0x2f, 0x72, 0x85, 0x3e, 0x81, 0x2b, 0x47, 0x7e, 0xca, 0x5f, -+ 0x44, 0x57, 0xe7, 0x53, 0x79, 0xfd, 0x78, 0x1a, 0xbd, 0xd0, 0xe2, 0xdf, 0x57, 0xa0, 0xb9, 0x9f, -+ 0xa4, 0x4f, 0xb9, 0x3f, 0xc5, 0xe8, 0xbf, 0xa0, 0x2d, 0xa8, 0xf0, 0x67, 0x5e, 0x2a, 0x41, 0x45, -+ 0x5e, 0x77, 0x41, 0xa1, 0x34, 0x81, 0x74, 0x3b, 0x66, 0x41, 0x92, 0x1a, 0x8a, 0xea, 0xad, 0xda, -+ 0x4e, 0xdd, 0x6d, 0x6b, 0x9c, 0x26, 0x19, 0xc3, 0xa6, 0x9a, 0xf3, 0x48, 0xec, 0x9d, 0x62, 0x16, -+ 0xe3, 0x59, 0x44, 0x43, 0xac, 0xce, 0x6f, 0xdd, 0x1d, 0xa8, 0xa9, 0xc3, 0xf8, 0xf3, 0x6c, 0x02, -+ 0xfd, 0x37, 0x0c, 0x32, 0x7a, 0x79, 0x29, 0x15, 0x75, 0x5d, 0x51, 0xf7, 0x0d, 0xf5, 0x53, 0x83, -+ 0x76, 0x7e, 0x09, 0xbd, 0x27, 0x27, 0x8c, 0x0a, 0x31, 0x23, 0xf1, 0xf4, 0xbe, 0x2f, 0x7c, 0x19, -+ 0x3d, 0x12, 0xcc, 0x08, 0x0d, 0xb9, 0xd1, 0xd6, 0x82, 0xe8, 0x6d, 0x18, 0x08, 0x4d, 0x8b, 0x43, -+ 0xcf, 0xd2, 0x54, 0x15, 0xcd, 0x46, 0x36, 0x71, 0x64, 0x88, 0xdf, 0x80, 0x5e, 0x4e, 0x2c, 0xe3, -+ 0x8f, 0xd1, 0xb7, 0x9b, 0x61, 0x9f, 0x90, 0x08, 0x3b, 0x67, 0xca, 0x57, 0x6a, 0x93, 0xd1, 0xdb, -+ 0xd0, 0xca, 0xfd, 0x50, 0x51, 0x27, 0xa4, 0xa7, 0x4f, 0x88, 0x75, 0xa7, 0xdb, 0xcc, 0x9c, 0xf2, -+ 0x19, 0xf4, 0x45, 0xa6, 0xb8, 0x17, 0xfa, 0xc2, 0x2f, 0x1f, 0xaa, 0xb2, 0x55, 0x6e, 0x4f, 0x94, -+ 0x60, 0xe7, 0x53, 0x68, 0x1d, 0x91, 0x90, 0x6b, 0xc1, 0x43, 0x58, 0x0f, 0x52, 0xc6, 0x70, 0x2c, -+ 0xac, 0xc9, 0x06, 0x44, 0x5b, 0xb0, 0x36, 0x23, 0x11, 0x11, 0xc6, 0x4c, 0x0d, 0x38, 0x14, 0xe0, -+ 0x11, 0x8e, 0x28, 0x9b, 0x2b, 0x87, 0x6d, 0xc1, 0x5a, 0x71, 0x73, 0x35, 0x80, 0xae, 0x43, 0x2b, -+ 0xf2, 0xcf, 0xb3, 0x4d, 0x95, 0x33, 0xcd, 0xc8, 0x3f, 0xd7, 0xca, 0x0f, 0x61, 0xfd, 0x99, 0x4f, -+ 0x66, 0x41, 0x2c, 0x8c, 0x57, 0x2c, 0x98, 0x0b, 0xac, 0x17, 0x05, 0xfe, 0xb9, 0x0a, 0x6d, 0x2d, -+ 0x51, 0x2b, 0xbc, 0x05, 0x6b, 0x81, 0x1f, 0x9c, 0x64, 0x22, 0x15, 0x80, 0x6e, 0x5b, 0x45, 0xaa, -+ 0xc5, 0x20, 0x9c, 0x6b, 0x6a, 0x55, 0xdb, 0x05, 0xe0, 0xcf, 0xfd, 0xc4, 0xe8, 0x56, 0xbb, 0x80, -+ 0xb8, 0x25, 0x69, 0xb4, 0xba, 0xef, 0x43, 0x47, 0x9f, 0x3b, 0xb3, 0xa4, 0x7e, 0xc1, 0x92, 0xb6, -+ 0xa6, 0xd2, 0x8b, 0x5e, 0x83, 0x6e, 0xca, 0xb1, 0x77, 0x42, 0x30, 0xf3, 0x59, 0x70, 0x32, 0x1f, -+ 0xae, 0xe9, 0x37, 0x32, 0xe5, 0xf8, 0xc0, 0xe2, 0xd0, 0x1d, 0x58, 0x93, 0xe1, 0x8f, 0x0f, 0x1b, -+ 0xea, 0x39, 0xbe, 0x51, 0x64, 0xa9, 0x4c, 0x1d, 0xab, 0xef, 0x83, 0x58, 0xb0, 0xb9, 0xab, 0x49, -+ 0x47, 0x1f, 0x01, 0xe4, 0x48, 0xb4, 0x01, 0xb5, 0x53, 0x3c, 0x37, 0xf7, 0x50, 0x0e, 0xa5, 0x73, -+ 0xce, 0xfc, 0x59, 0x6a, 0xbd, 0xae, 0x81, 0x4f, 0xaa, 0x1f, 0x55, 0x9c, 0x00, 0xfa, 0x7b, 0xb3, -+ 0x53, 0x42, 0x0b, 0xcb, 0xb7, 0x60, 0x2d, 0xf2, 0xbf, 0xa2, 0xcc, 0x7a, 0x52, 0x01, 0x0a, 0x4b, -+ 0x62, 0xca, 0x2c, 0x0b, 0x05, 0xa0, 0x1e, 0x54, 0x69, 0xa2, 0xfc, 0xd5, 0x72, 0xab, 0x34, 0xc9, -+ 0x05, 0xd5, 0x0b, 0x82, 0x9c, 0xbf, 0xd7, 0x01, 0x72, 0x29, 0xc8, 0x85, 0x11, 0xa1, 0x1e, 0xc7, -+ 0x4c, 0xa6, 0x20, 0xde, 0xf1, 0x5c, 0x60, 0xee, 0x31, 0x1c, 0xa4, 0x8c, 0x93, 0x33, 0xb9, 0x7f, -+ 0xd2, 0xec, 0x2b, 0xda, 0xec, 0x05, 0xdd, 0xdc, 0xab, 0x84, 0x4e, 0xf4, 0xba, 0x3d, 0xb9, 0xcc, -+ 0xb5, 0xab, 0xd0, 0x21, 0x5c, 0xc9, 0x79, 0x86, 0x05, 0x76, 0xd5, 0xcb, 0xd8, 0x6d, 0x66, 0xec, -+ 0xc2, 0x9c, 0xd5, 0x03, 0xd8, 0x24, 0xd4, 0xfb, 0x3a, 0xc5, 0x69, 0x89, 0x51, 0xed, 0x32, 0x46, -+ 0x03, 0x42, 0x7f, 0xa4, 0x16, 0xe4, 0x6c, 0x8e, 0xe0, 0x5a, 0xc1, 0x4a, 0x79, 0xdd, 0x0b, 0xcc, -+ 0xea, 0x97, 0x31, 0xdb, 0xce, 0xb4, 0x92, 0xf1, 0x20, 0xe7, 0xf8, 0x03, 0xd8, 0x26, 0xd4, 0x7b, -+ 0xee, 0x13, 0xb1, 0xc8, 0x6e, 0xed, 0x7b, 0x8c, 0x94, 0x8f, 0x6e, 0x99, 0x97, 0x36, 0x32, 0xc2, -+ 0x6c, 0x5a, 0x32, 0xb2, 0xf1, 0x3d, 0x46, 0x3e, 0x52, 0x0b, 0x72, 0x36, 0xf7, 0x60, 0x40, 0xe8, -+ 0xa2, 0x36, 0xeb, 0x97, 0x31, 0xe9, 0x13, 0x5a, 0xd6, 0x64, 0x0f, 0x06, 0x1c, 0x07, 0x82, 0xb2, -+ 0xe2, 0x21, 0x68, 0x5e, 0xc6, 0x62, 0xc3, 0xd0, 0x67, 0x3c, 0x9c, 0x9f, 0x41, 0xe7, 0x20, 0x9d, -+ 0x62, 0x31, 0x3b, 0xce, 0x82, 0xc1, 0x4b, 0x8b, 0x3f, 0xce, 0x3f, 0xab, 0xd0, 0xde, 0x9f, 0x32, -+ 0x9a, 0x26, 0xa5, 0x98, 0xac, 0x2f, 0xe9, 0x62, 0x4c, 0x56, 0x24, 0x2a, 0x26, 0x6b, 0xe2, 0x0f, -+ 0xa0, 0x13, 0xa9, 0xab, 0x6b, 0xe8, 0x75, 0x1c, 0x1a, 0x2c, 0x5d, 0x6a, 0xb7, 0x1d, 0x15, 0x82, -+ 0xd9, 0x18, 0x20, 0x21, 0x21, 0x37, 0x6b, 0x74, 0x38, 0xea, 0x9b, 0x8c, 0xd0, 0x86, 0x68, 0xb7, -+ 0x95, 0x64, 0xd1, 0xfa, 0x3d, 0x68, 0x1f, 0x4b, 0x27, 0x99, 0x05, 0xa5, 0x60, 0x94, 0x7b, 0xcf, -+ 0x85, 0xe3, 0xfc, 0x12, 0x1e, 0x40, 0xf7, 0x44, 0xbb, 0xcc, 0x2c, 0xd2, 0x67, 0xe8, 0x35, 0x63, -+ 0x49, 0x6e, 0xef, 0xb8, 0xe8, 0x59, 0xbd, 0x01, 0x9d, 0x93, 0x02, 0x6a, 0x34, 0x81, 0xc1, 0x12, -+ 0xc9, 0x8a, 0x18, 0xb4, 0x53, 0x8c, 0x41, 0xed, 0x3b, 0x48, 0x0b, 0x2a, 0xae, 0x2c, 0xc6, 0xa5, -+ 0xdf, 0x54, 0xa1, 0xf3, 0x43, 0x2c, 0x9e, 0x53, 0x76, 0xaa, 0xf5, 0x45, 0x50, 0x8f, 0xfd, 0x08, -+ 0x1b, 0x8e, 0x6a, 0x8c, 0xae, 0x41, 0x93, 0x9d, 0xeb, 0x00, 0x62, 0xf6, 0x73, 0x9d, 0x9d, 0xab, -+ 0xc0, 0x80, 0x5e, 0x01, 0x60, 0xe7, 0x5e, 0xe2, 0x07, 0xa7, 0xd8, 0x78, 0xb0, 0xee, 0xb6, 0xd8, -+ 0xf9, 0x91, 0x46, 0xc8, 0xa3, 0xc0, 0xce, 0x3d, 0xcc, 0x18, 0x65, 0xdc, 0xc4, 0xaa, 0x26, 0x3b, -+ 0x7f, 0xa0, 0x60, 0xb3, 0x36, 0x64, 0x34, 0x49, 0x70, 0xa8, 0x62, 0xb4, 0x5a, 0x7b, 0x5f, 0x23, -+ 0xa4, 0x54, 0x61, 0xa5, 0x36, 0xb4, 0x54, 0x91, 0x4b, 0x15, 0xb9, 0xd4, 0x75, 0xbd, 0x52, 0x14, -+ 0xa5, 0x8a, 0x4c, 0x6a, 0x53, 0x4b, 0x15, 0x05, 0xa9, 0x22, 0x97, 0xda, 0xb2, 0x6b, 0x8d, 0x54, -+ 0xe7, 0xd7, 0x15, 0xd8, 0x5e, 0x4c, 0xfc, 0x4c, 0x9a, 0xfa, 0x01, 0x74, 0x02, 0xb5, 0x5f, 0xa5, -+ 0x33, 0x39, 0x58, 0xda, 0x49, 0xb7, 0x1d, 0x14, 0x8e, 0xf1, 0x87, 0xd0, 0x8d, 0xb5, 0x83, 0xb3, -+ 0xa3, 0x59, 0xcb, 0xf7, 0xa5, 0xe8, 0x7b, 0xb7, 0x13, 0x17, 0x20, 0x27, 0x04, 0xf4, 0x25, 0x23, -+ 0x02, 0x4f, 0x04, 0xc3, 0x7e, 0xf4, 0x32, 0x0a, 0x10, 0x04, 0x75, 0x95, 0xad, 0xd4, 0x54, 0x7e, -+ 0xad, 0xc6, 0xce, 0x9b, 0xb0, 0x59, 0x92, 0x62, 0x6c, 0xdd, 0x80, 0xda, 0x0c, 0xc7, 0x8a, 0x7b, -+ 0xd7, 0x95, 0x43, 0xc7, 0x87, 0x81, 0x8b, 0xfd, 0xf0, 0xe5, 0x69, 0x63, 0x44, 0xd4, 0x72, 0x11, -+ 0x3b, 0x80, 0x8a, 0x22, 0x8c, 0x2a, 0x56, 0xeb, 0x4a, 0x41, 0xeb, 0xc7, 0x30, 0xd8, 0x9f, 0x51, -+ 0x8e, 0x27, 0x22, 0x24, 0xf1, 0xcb, 0xa8, 0x98, 0x7e, 0x01, 0x9b, 0x4f, 0xc4, 0xfc, 0x4b, 0xc9, -+ 0x8c, 0x93, 0x6f, 0xf0, 0x4b, 0xb2, 0x8f, 0xd1, 0xe7, 0xd6, 0x3e, 0x46, 0x9f, 0xcb, 0x62, 0x29, -+ 0xa0, 0xb3, 0x34, 0x8a, 0xd5, 0x55, 0xe8, 0xba, 0x06, 0x72, 0xf6, 0xa0, 0xa3, 0x73, 0xe8, 0x47, -+ 0x34, 0x4c, 0x67, 0x78, 0xe5, 0x1d, 0xbc, 0x09, 0x90, 0xf8, 0xcc, 0x8f, 0xb0, 0xc0, 0x4c, 0x9f, -+ 0xa1, 0x96, 0x5b, 0xc0, 0x38, 0xbf, 0xad, 0xc2, 0x96, 0x6e, 0x89, 0x4c, 0x74, 0x27, 0xc0, 0x9a, -+ 0x30, 0x82, 0xe6, 0x09, 0xe5, 0xa2, 0xc0, 0x30, 0x83, 0xa5, 0x8a, 0x61, 0x6c, 0xb9, 0xc9, 0x61, -+ 0xa9, 0x4f, 0x51, 0xbb, 0xbc, 0x4f, 0xb1, 0xd4, 0x89, 0xa8, 0x2f, 0x77, 0x22, 0xe4, 0x6d, 0xb3, -+ 0x44, 0x44, 0xdf, 0xf1, 0x96, 0xdb, 0x32, 0x98, 0xc3, 0x10, 0xdd, 0x86, 0xfe, 0x54, 0x6a, 0xe9, -+ 0x9d, 0x50, 0x7a, 0xea, 0x25, 0xbe, 0x38, 0x51, 0x57, 0xbd, 0xe5, 0x76, 0x15, 0xfa, 0x80, 0xd2, -+ 0xd3, 0x23, 0x5f, 0x9c, 0xa0, 0x8f, 0xa1, 0x67, 0xd2, 0xc0, 0x48, 0xb9, 0x88, 0x9b, 0xc7, 0xcf, -+ 0xdc, 0xa2, 0xa2, 0xf7, 0xdc, 0xee, 0x69, 0x01, 0xe2, 0xce, 0x55, 0xb8, 0x72, 0x1f, 0x73, 0xc1, -+ 0xe8, 0xbc, 0xec, 0x18, 0xe7, 0xff, 0x00, 0x0e, 0x63, 0x81, 0xd9, 0x33, 0x3f, 0xc0, 0x1c, 0xbd, -+ 0x5b, 0x84, 0x4c, 0x72, 0xb4, 0x31, 0xd6, 0x1d, 0xa9, 0x6c, 0xc2, 0x2d, 0xd0, 0x38, 0x63, 0x68, -+ 0xb8, 0x34, 0x95, 0xe1, 0xe8, 0x75, 0x3b, 0x32, 0xeb, 0x3a, 0x66, 0x9d, 0x42, 0xba, 0x66, 0xce, -+ 0x39, 0xb0, 0x25, 0x6c, 0xce, 0xce, 0x6c, 0xd1, 0x18, 0x5a, 0xc4, 0xe2, 0x4c, 0x54, 0x59, 0x16, -+ 0x9d, 0x93, 0x38, 0x3f, 0x85, 0x4d, 0xcd, 0x49, 0x73, 0xb6, 0x6c, 0x5e, 0x87, 0x06, 0xb3, 0x6a, -+ 0x54, 0xf2, 0x56, 0x94, 0x21, 0x32, 0x73, 0xe8, 0x86, 0x14, 0x16, 0x30, 0x1c, 0xc9, 0x9a, 0xa3, -+ 0xaa, 0xb6, 0x2c, 0x47, 0x48, 0x6f, 0xc9, 0x7a, 0x3b, 0x37, 0xd3, 0x7a, 0x6b, 0x13, 0x06, 0x72, -+ 0xa2, 0x24, 0xd1, 0xf9, 0x39, 0x6c, 0x3e, 0x8e, 0x67, 0x24, 0xc6, 0xfb, 0x47, 0x4f, 0x1f, 0xe1, -+ 0x2c, 0x2a, 0x20, 0xa8, 0xcb, 0xec, 0x49, 0xa9, 0xd1, 0x74, 0xd5, 0x58, 0x5e, 0x93, 0xf8, 0xd8, -+ 0x0b, 0x92, 0x94, 0x9b, 0xce, 0x50, 0x23, 0x3e, 0xde, 0x4f, 0x52, 0x2e, 0xc3, 0xbc, 0x7c, 0xe6, -+ 0x69, 0x3c, 0x9b, 0xab, 0xbb, 0xd2, 0x74, 0xd7, 0x83, 0x24, 0x7d, 0x1c, 0xcf, 0xe6, 0xce, 0xff, -+ 0xa8, 0x5a, 0x18, 0xe3, 0xd0, 0xf5, 0xe3, 0x90, 0x46, 0xf7, 0xf1, 0x59, 0x41, 0x42, 0x56, 0x77, -+ 0xd9, 0x98, 0xf0, 0x6d, 0x05, 0x3a, 0xf7, 0xa6, 0x38, 0x16, 0xf7, 0xb1, 0xf0, 0xc9, 0x4c, 0xd5, -+ 0x56, 0x67, 0x98, 0x71, 0x42, 0x63, 0x73, 0xf0, 0x2d, 0x28, 0x4b, 0x63, 0x12, 0x13, 0xe1, 0x85, -+ 0x3e, 0x8e, 0x68, 0x6c, 0xbc, 0x00, 0x12, 0x75, 0x5f, 0x61, 0xd0, 0x9b, 0xd0, 0xd7, 0x9d, 0x3b, -+ 0xef, 0xc4, 0x8f, 0xc3, 0x99, 0xbc, 0x72, 0xba, 0x93, 0xd1, 0xd3, 0xe8, 0x03, 0x83, 0x45, 0x6f, -+ 0xc1, 0x86, 0xb9, 0x10, 0x39, 0x65, 0x5d, 0x51, 0xf6, 0x0d, 0xbe, 0x44, 0x9a, 0x26, 0x09, 0x65, -+ 0x82, 0x7b, 0x1c, 0x07, 0x01, 0x8d, 0x12, 0x53, 0x98, 0xf4, 0x2d, 0x7e, 0xa2, 0xd1, 0xce, 0x14, -+ 0x36, 0x1f, 0x4a, 0x3b, 0x8d, 0x25, 0xf9, 0x06, 0xf7, 0x22, 0x1c, 0x79, 0xc7, 0x33, 0x1a, 0x9c, -+ 0x7a, 0x32, 0x4c, 0x19, 0x0f, 0xcb, 0xd4, 0x67, 0x4f, 0x22, 0x27, 0xe4, 0x1b, 0x55, 0x83, 0x4b, -+ 0xaa, 0x13, 0x2a, 0x92, 0x59, 0x3a, 0xf5, 0x12, 0x46, 0x8f, 0xb1, 0x31, 0xb1, 0x1f, 0xe1, 0xe8, -+ 0x40, 0xe3, 0x8f, 0x24, 0xda, 0xf9, 0x53, 0x05, 0xb6, 0xca, 0x92, 0x4c, 0xd0, 0xdd, 0x85, 0xad, -+ 0xb2, 0x28, 0xf3, 0x10, 0xeb, 0x44, 0x6f, 0x50, 0x14, 0xa8, 0x9f, 0xe4, 0x0f, 0xa1, 0xab, 0xda, -+ 0xb9, 0x5e, 0xa8, 0x39, 0x95, 0xd3, 0x8f, 0xe2, 0xbe, 0xb8, 0x1d, 0xbf, 0xb8, 0x4b, 0x1f, 0xc3, -+ 0x35, 0x63, 0xbe, 0xb7, 0xac, 0xb6, 0x3e, 0x10, 0xdb, 0x86, 0xe0, 0xd1, 0x82, 0xf6, 0x5f, 0xc0, -+ 0x30, 0x47, 0xed, 0xcd, 0x15, 0xd2, 0xfa, 0xea, 0x5d, 0xd8, 0x5c, 0x30, 0xf6, 0x5e, 0x18, 0x32, -+ 0x75, 0x41, 0xeb, 0xee, 0xaa, 0x29, 0xe7, 0x2e, 0x5c, 0x9d, 0x60, 0xa1, 0xbd, 0xe1, 0x0b, 0x53, -+ 0x13, 0x68, 0x66, 0x1b, 0x50, 0x9b, 0xe0, 0x40, 0x19, 0x5f, 0x73, 0xe5, 0x50, 0x1e, 0xc0, 0xa7, -+ 0x1c, 0x07, 0xca, 0xca, 0x9a, 0xab, 0xc6, 0xce, 0x1f, 0x2b, 0xb0, 0x6e, 0xc2, 0xa4, 0x0c, 0xf5, -+ 0x21, 0x23, 0x67, 0x98, 0x99, 0xa3, 0x67, 0x20, 0xf4, 0x06, 0xf4, 0xf4, 0xc8, 0xa3, 0x89, 0x20, -+ 0x34, 0x0b, 0xbe, 0x5d, 0x8d, 0x7d, 0xac, 0x91, 0xaa, 0x53, 0xa7, 0x1a, 0x51, 0xa6, 0xe6, 0x33, -+ 0x90, 0x6a, 0xb7, 0x71, 0x19, 0x19, 0x54, 0xb0, 0x6d, 0xb9, 0x06, 0x92, 0x47, 0xdd, 0xf2, 0x5b, -+ 0x53, 0xfc, 0x2c, 0x28, 0x8f, 0x7a, 0x44, 0xd3, 0x58, 0x78, 0x09, 0x25, 0xb1, 0x30, 0xd1, 0x15, -+ 0x14, 0xea, 0x48, 0x62, 0x9c, 0x5f, 0x55, 0xa0, 0xa1, 0xbb, 0xd5, 0xb2, 0xca, 0xcc, 0xde, 0xb8, -+ 0x2a, 0x51, 0xf9, 0x82, 0x92, 0xa5, 0xdf, 0x35, 0x35, 0x96, 0xf7, 0xf8, 0x2c, 0xd2, 0x91, 0xda, -+ 0xa8, 0x76, 0x16, 0xa9, 0x10, 0xfd, 0x06, 0xf4, 0xf2, 0xa7, 0x52, 0xcd, 0x6b, 0x15, 0xbb, 0x19, -+ 0x56, 0x91, 0x5d, 0xa8, 0xa9, 0xf3, 0x13, 0x59, 0x5c, 0x67, 0x9d, 0xda, 0x0d, 0xa8, 0xa5, 0x99, -+ 0x32, 0x72, 0x28, 0x31, 0xd3, 0xec, 0x91, 0x95, 0x43, 0x74, 0x1b, 0x7a, 0x7e, 0x18, 0x12, 0xb9, -+ 0xdc, 0x9f, 0x3d, 0x24, 0x61, 0x76, 0x49, 0xcb, 0x58, 0xe7, 0x2f, 0x15, 0xe8, 0xef, 0xd3, 0x64, -+ 0xfe, 0xff, 0x64, 0x86, 0x0b, 0x11, 0x44, 0x29, 0x69, 0xde, 0x58, 0x39, 0x96, 0x79, 0xe3, 0x33, -+ 0x32, 0xc3, 0xfa, 0x6a, 0xe9, 0x9d, 0x6d, 0x4a, 0x84, 0xba, 0x56, 0x76, 0x32, 0x6b, 0x80, 0x75, -+ 0xf5, 0xe4, 0x23, 0x1a, 0xaa, 0x0c, 0x39, 0x24, 0xcc, 0xcb, 0xda, 0x5d, 0x5d, 0x77, 0x3d, 0x24, -+ 0x4c, 0x4d, 0x19, 0x43, 0xd6, 0x54, 0xc7, 0xb5, 0x68, 0x48, 0x43, 0x63, 0xa4, 0x21, 0xdb, 0xd0, -+ 0xa0, 0xcf, 0x9e, 0x71, 0x2c, 0x54, 0x2e, 0x5b, 0x73, 0x0d, 0x94, 0x85, 0xb9, 0x66, 0x21, 0xcc, -+ 0x5d, 0x81, 0x4d, 0xd5, 0xdb, 0x7f, 0xc2, 0xfc, 0x80, 0xc4, 0x53, 0x1b, 0x8a, 0xb7, 0x00, 0x4d, -+ 0x04, 0x4d, 0x16, 0xb0, 0x63, 0x18, 0x98, 0x37, 0xe7, 0xe8, 0xc7, 0x13, 0x6b, 0xfa, 0x35, 0x68, -+ 0x4a, 0xd0, 0x63, 0xf8, 0x6b, 0x1b, 0x18, 0xcd, 0xb4, 0xf3, 0x16, 0x74, 0xf4, 0xd0, 0x84, 0x81, -+ 0x9c, 0x94, 0x97, 0x49, 0xf9, 0x9d, 0xbf, 0x6d, 0x98, 0x70, 0x6b, 0x6a, 0x68, 0xf4, 0x10, 0xfa, -+ 0x0b, 0xff, 0x64, 0x90, 0x69, 0xaa, 0xac, 0xfe, 0x55, 0x33, 0xda, 0x1e, 0xeb, 0x7f, 0x3c, 0x63, -+ 0xfb, 0x8f, 0x67, 0xfc, 0x20, 0x4a, 0xc4, 0x1c, 0x3d, 0x80, 0x5e, 0xf9, 0xef, 0x05, 0xba, 0x6e, -+ 0x73, 0x90, 0x15, 0xff, 0x34, 0x2e, 0x64, 0xf3, 0x10, 0xfa, 0x0b, 0x3f, 0x32, 0xac, 0x3e, 0xab, -+ 0xff, 0x6f, 0x5c, 0xc8, 0xe8, 0x2e, 0xb4, 0x0b, 0x7f, 0x2e, 0xd0, 0x50, 0x33, 0x59, 0xfe, 0x99, -+ 0x71, 0x21, 0x83, 0x7d, 0xe8, 0x96, 0x7e, 0x26, 0xa0, 0x91, 0xb1, 0x67, 0xc5, 0x1f, 0x86, 0x0b, -+ 0x99, 0xec, 0x41, 0xbb, 0xd0, 0xd3, 0xb7, 0x5a, 0x2c, 0xff, 0x38, 0x18, 0x5d, 0x5b, 0x31, 0x63, -+ 0xb6, 0xf3, 0x00, 0xba, 0xa5, 0x0e, 0xbc, 0x55, 0x64, 0x55, 0xf7, 0x7f, 0x74, 0x7d, 0xe5, 0x9c, -+ 0xe1, 0xf4, 0x10, 0xfa, 0x0b, 0xfd, 0x78, 0xeb, 0xdc, 0xd5, 0x6d, 0xfa, 0x0b, 0xcd, 0xfa, 0x5c, -+ 0x6d, 0x76, 0xa1, 0xdc, 0x2a, 0x6c, 0xf6, 0x72, 0xf7, 0x7d, 0x74, 0x63, 0xf5, 0xa4, 0xd1, 0xea, -+ 0x01, 0xf4, 0xca, 0x8d, 0x77, 0xcb, 0x6c, 0x65, 0x3b, 0xfe, 0xf2, 0x93, 0x53, 0xea, 0xc1, 0xe7, -+ 0x27, 0x67, 0x55, 0x6b, 0xfe, 0x42, 0x46, 0xf7, 0x00, 0x4c, 0x71, 0x15, 0x92, 0x38, 0xdb, 0xb2, -+ 0xa5, 0xa2, 0x2e, 0xdb, 0xb2, 0x15, 0x85, 0xd8, 0x5d, 0x00, 0x5d, 0x13, 0x85, 0x34, 0x15, 0xe8, -+ 0xaa, 0x55, 0x63, 0xa1, 0x10, 0x1b, 0x0d, 0x97, 0x27, 0x96, 0x18, 0x60, 0xc6, 0x5e, 0x84, 0xc1, -+ 0x67, 0x00, 0x79, 0xad, 0x65, 0x19, 0x2c, 0x55, 0x5f, 0x97, 0xf8, 0xa0, 0x53, 0xac, 0xac, 0x90, -+ 0xb1, 0x75, 0x45, 0xb5, 0x75, 0x09, 0x8b, 0xfe, 0x42, 0xe6, 0x5c, 0x3e, 0x6c, 0x8b, 0x09, 0xf5, -+ 0x68, 0x29, 0x7b, 0x46, 0x1f, 0x42, 0xa7, 0x98, 0x32, 0x5b, 0x2d, 0x56, 0xa4, 0xd1, 0xa3, 0x52, -+ 0xda, 0x8c, 0xee, 0x42, 0xaf, 0x9c, 0x10, 0xa3, 0xc2, 0xbd, 0x58, 0x4a, 0x93, 0x47, 0xa6, 0x19, -+ 0x54, 0x20, 0x7f, 0x1f, 0x20, 0x4f, 0x9c, 0xad, 0xfb, 0x96, 0x52, 0xe9, 0x05, 0xa9, 0x9f, 0x41, -+ 0xaf, 0x10, 0xb7, 0x65, 0x4d, 0x78, 0xb5, 0x64, 0x70, 0x1e, 0xcd, 0x47, 0x26, 0xc3, 0x2a, 0x85, -+ 0xed, 0x7b, 0xd0, 0x29, 0xbe, 0x11, 0xd6, 0xda, 0x15, 0xef, 0xc6, 0x65, 0x41, 0xaf, 0xf0, 0x9e, -+ 0xd8, 0xb3, 0xbb, 0xfc, 0xc4, 0x5c, 0x16, 0xf4, 0x4a, 0xf5, 0xa8, 0x8d, 0x35, 0xab, 0x8a, 0xd4, -+ 0xcb, 0x9e, 0x82, 0x72, 0xf1, 0x66, 0xbd, 0xbf, 0xb2, 0xa4, 0xbb, 0xec, 0x0c, 0x16, 0xeb, 0x14, -+ 0xeb, 0x8f, 0x15, 0xb5, 0xcb, 0xf7, 0xc4, 0x84, 0x62, 0x2d, 0x52, 0x88, 0x09, 0x2b, 0x4a, 0x94, -+ 0x0b, 0x19, 0x1d, 0x40, 0xff, 0xa1, 0x4d, 0x33, 0x4d, 0x0a, 0x6c, 0xd4, 0x59, 0x91, 0xf2, 0x8f, -+ 0x46, 0xab, 0xa6, 0xcc, 0x2e, 0x7f, 0x0e, 0x83, 0xa5, 0xf4, 0x17, 0xdd, 0xcc, 0x5a, 0x9e, 0x2b, -+ 0xf3, 0xe2, 0x0b, 0xd5, 0x3a, 0x84, 0x8d, 0xc5, 0xec, 0x17, 0xbd, 0x62, 0x36, 0x7d, 0x75, 0x56, -+ 0x7c, 0x21, 0xab, 0x8f, 0xa1, 0x69, 0xb3, 0x2d, 0x64, 0x5a, 0xcb, 0x0b, 0xd9, 0xd7, 0x45, 0x4b, -+ 0xf7, 0x3a, 0xdf, 0x7e, 0x77, 0xb3, 0xf2, 0xd7, 0xef, 0x6e, 0x56, 0xfe, 0xf1, 0xdd, 0xcd, 0xca, -+ 0x71, 0x43, 0xcd, 0xbe, 0xff, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x45, 0xa8, 0x91, 0xab, 0x62, -+ 0x22, 0x00, 0x00, - } -diff --git a/protocols/grpc/agent.proto b/protocols/grpc/agent.proto -index b0fab6d..d863645 100644 ---- a/protocols/grpc/agent.proto -+++ b/protocols/grpc/agent.proto -@@ -46,6 +46,7 @@ service AgentService { - rpc UpdateRoutes(UpdateRoutesRequest) returns (Routes); - rpc ListInterfaces(ListInterfacesRequest) returns(Interfaces); - rpc ListRoutes(ListRoutesRequest) returns (Routes); -+ rpc UpdateIPVSRule(UpdateIPVSRequest) returns (IPVSResponse); - - // tracing - rpc StartTracing(StartTracingRequest) returns (google.protobuf.Empty); -@@ -495,3 +496,13 @@ message StartTracingRequest { - - message StopTracingRequest { - } -+ -+message UpdateIPVSRequest { -+ // IPVS_req is the IPVS rule message needed to update -+ string IPVS_req = 1; -+} -+ -+message IPVSResponse { -+ // IPVS_res is the response of IPVS updating -+ string IPVS_res = 1; -+} --- -2.14.3 (Apple Git-98) - diff --git a/agent/patches/0004-agent-add-IPVS-test.patch b/agent/patches/0004-agent-add-IPVS-test.patch deleted file mode 100644 index 5c73f8c..0000000 --- a/agent/patches/0004-agent-add-IPVS-test.patch +++ /dev/null @@ -1,193 +0,0 @@ -From 01563c08910ddaba4077fd9dc691df541e045165 Mon Sep 17 00:00:00 2001 -From: xiadanni -Date: Tue, 18 Aug 2020 17:05:32 +0800 -Subject: [PATCH 04/16] agent: add IPVS test - -Signed-off-by: xiadanni ---- - grpc_test.go | 172 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - 1 file changed, 172 insertions(+) - -diff --git a/grpc_test.go b/grpc_test.go -index e69102b..d58c0b6 100644 ---- a/grpc_test.go -+++ b/grpc_test.go -@@ -1840,3 +1840,175 @@ func getPipeMaxSize() (uint32, error) { - u, err := strconv.ParseUint(s, 10, 32) - return uint32(u), err - } -+ -+func TestUpdateIPVSRule(t *testing.T) { -+ assert := assert.New(t) -+ -+ // add IPVS rule successfully -+ a := &agentGRPC{ -+ sandbox: &sandbox{ -+ containers: make(map[string]*container), -+ }, -+ } -+ -+ req := &pb.UpdateIPVSRequest{ -+ IPVSReq: "ipvsadm -A -t 17.2.0.7:80 -s rr -p 3000", -+ } -+ -+ _, err := a.UpdateIPVSRule(context.Background(), req) -+ assert.NoError(err) -+ -+ // delete ipvs rule successfully -+ req = &pb.UpdateIPVSRequest{ -+ IPVSReq: "ipvsadm -D -t 17.2.0.7:80", -+ } -+ -+ _, err = a.UpdateIPVSRule(context.Background(), req) -+ assert.NoError(err) -+ -+ // update ipvs rule error because exec failed -+ req = &pb.UpdateIPVSRequest{ -+ IPVSReq: "ipvsadm -A -t 17.2.0.7:80 -s rr -p -3000", -+ } -+ -+ _, err = a.UpdateIPVSRule(context.Background(), req) -+ assert.Error(err) -+ assert.Contains(err.Error(), "exec IPVS command failed") -+ -+ // update IPVS rule error because rule less than validHeadLength -+ req = &pb.UpdateIPVSRequest{ -+ IPVSReq: "ipvsa", -+ } -+ -+ _, err = a.UpdateIPVSRule(context.Background(), req) -+ assert.Error(err) -+ assert.Contains(err.Error(), "invalid IPVS rule") -+ -+ // update ipvs rule error because invalid command -+ req = &pb.UpdateIPVSRequest{ -+ IPVSReq: "abcabcabc ipvsadm", -+ } -+ -+ _, err = a.UpdateIPVSRule(context.Background(), req) -+ assert.Error(err) -+ assert.Contains(err.Error(), "invalid IPVS rule") -+ -+ // add ipvs rule error because rule count exceeds -+ a = &agentGRPC{ -+ sandbox: &sandbox{ -+ containers: make(map[string]*container), -+ ipvsadm: ipvsAdm{ -+ ipvsRuleCnt: 20000, -+ }, -+ }, -+ } -+ -+ req = &pb.UpdateIPVSRequest{ -+ IPVSReq: "ipvsadm -A -t 17.2.0.7:80 -s rr -p 3000", -+ } -+ _, err = a.UpdateIPVSRule(context.Background(), req) -+ assert.Error(err) -+ assert.Errorf(err, "rules exceed limit") -+ -+ // add ipvs rule error because ipvs request item less than 2 -+ a = &agentGRPC{ -+ sandbox: &sandbox{ -+ containers: make(map[string]*container), -+ ipvsadm: ipvsAdm{ -+ ipvsRuleCnt: 0, -+ }, -+ }, -+ } -+ -+ req = &pb.UpdateIPVSRequest{ -+ IPVSReq: "ipvsadm", -+ } -+ -+ _, err = a.UpdateIPVSRule(context.Background(), req) -+ assert.Error(err) -+ assert.Contains(err.Error(), "invalid IPVS rule") -+ -+ // add ipvs rule error because ipvs rule nil -+ req = nil -+ -+ _, err = a.UpdateIPVSRule(context.Background(), req) -+ assert.Error(err) -+ assert.Contains(err.Error(), "IPVS rule is nil") -+ -+ // add ipvs rule error because ipvs rule string is empty -+ req = &pb.UpdateIPVSRequest{ -+ IPVSReq: "", -+ } -+ -+ _, err = a.UpdateIPVSRule(context.Background(), req) -+ assert.Error(err) -+ assert.Contains(err.Error(), "IPVS rule is nil") -+ -+ // restore ipvs rule successfully -+ a = &agentGRPC{ -+ sandbox: &sandbox{ -+ containers: make(map[string]*container), -+ }, -+ } -+ -+ req = &pb.UpdateIPVSRequest{ -+ IPVSReq: "restore|2|-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m", -+ } -+ -+ _, err = a.UpdateIPVSRule(context.Background(), req) -+ assert.NoError(err) -+ -+ // clear IPVS rule successfully -+ req = &pb.UpdateIPVSRequest{ -+ IPVSReq: "ipvsadm -C", -+ } -+ -+ _, err = a.UpdateIPVSRule(context.Background(), req) -+ assert.NoError(err) -+ assert.Equal(a.sandbox.ipvsadm.ipvsRuleCnt, uint64(0)) -+ -+ // restore ipvs rule error because rule count invalid -+ req = &pb.UpdateIPVSRequest{ -+ IPVSReq: "restore|abc|-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m", -+ } -+ -+ _, err = a.UpdateIPVSRule(context.Background(), req) -+ assert.Error(err) -+ assert.Contains(err.Error(), "invalid IPVS rule") -+ -+ // restore ipvs rule error because other rules exists -+ a = &agentGRPC{ -+ sandbox: &sandbox{ -+ containers: make(map[string]*container), -+ ipvsadm: ipvsAdm{ -+ ipvsRuleCnt: 5, -+ }, -+ }, -+ } -+ -+ req = &pb.UpdateIPVSRequest{ -+ IPVSReq: "restore|2|-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m", -+ } -+ -+ _, err = a.UpdateIPVSRule(context.Background(), req) -+ assert.Error(err) -+ assert.Contains(err.Error(), "exist some rules in system") -+ -+ // restore ipvs rule error because ipvs req item less than 3 -+ a = &agentGRPC{ -+ sandbox: &sandbox{ -+ containers: make(map[string]*container), -+ ipvsadm: ipvsAdm{ -+ ipvsRuleCnt: 0, -+ }, -+ }, -+ } -+ -+ req = &pb.UpdateIPVSRequest{ -+ IPVSReq: "restore|-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m", -+ } -+ -+ _, err = a.UpdateIPVSRule(context.Background(), req) -+ assert.Error(err) -+ assert.Contains(err.Error(), "invalid IPVS rule") -+} --- -2.14.3 (Apple Git-98) - diff --git a/agent/patches/0005-mount-support-mount-block-device.patch b/agent/patches/0005-mount-support-mount-block-device.patch deleted file mode 100644 index 4da993b..0000000 --- a/agent/patches/0005-mount-support-mount-block-device.patch +++ /dev/null @@ -1,42 +0,0 @@ -From 8c9f9be2a9c195d0bc12b43c491adaacb7bb8154 Mon Sep 17 00:00:00 2001 -From: holyfei -Date: Tue, 18 Aug 2020 10:42:38 +0800 -Subject: [PATCH 05/16] mount: support mount block device - -reason: modify mountStorage to support mount block device -"-v /dev/blockdevice:/home/test" - -Signed-off-by: yangfeiyu ---- - mount.go | 15 ++++++++++++++- - 1 file changed, 14 insertions(+), 1 deletion(-) - -diff --git a/mount.go b/mount.go -index f0c3efe..a05d4a6 100644 ---- a/mount.go -+++ b/mount.go -@@ -364,7 +364,20 @@ func commonStorageHandler(storage pb.Storage) (string, error) { - func mountStorage(storage pb.Storage) error { - flags, options := parseMountFlagsAndOptions(storage.Options) - -- return mount(storage.Source, storage.MountPoint, storage.Fstype, flags, options) -+ var fsType = storage.Fstype -+ if (storage.Driver == driverSCSIType || storage.Driver == driverBlkType) && strings.Contains(storage.Fstype, "bind") { -+ cs := strings.Split(storage.Fstype, "-") -+ if len(cs) == 2 && cs[1] != "" { -+ fsType = cs[1] -+ // here we temporarily discard the bind option, -+ // in order to be able to mount the file system of the block device. -+ // and then reset `storage.Fstype` to "bind" which pass through to the libcontainer pkg. -+ flags = flags &^ flagList["bind"] -+ storage.Fstype = "bind" -+ } -+ } -+ -+ return mount(storage.Source, storage.MountPoint, fsType, flags, options) - } - - // addStorages takes a list of storages passed by the caller, and perform the --- -2.14.3 (Apple Git-98) - diff --git a/agent/patches/0006-agent-make-workaround-for-slow-response-in-aarch64.patch b/agent/patches/0006-agent-make-workaround-for-slow-response-in-aarch64.patch deleted file mode 100644 index 84b10d6..0000000 --- a/agent/patches/0006-agent-make-workaround-for-slow-response-in-aarch64.patch +++ /dev/null @@ -1,54 +0,0 @@ -From bccda1d208f31eab55863883cf0718d7b4b8deef Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Tue, 18 Aug 2020 19:30:42 +0800 -Subject: [PATCH 06/16] agent: make workaround for slow response in aarch64 - -reason: make workaround for slow response in aarch64 -when hotplug virtio-net-pci device - -Signed-off-by: jiangpengfei ---- - agent.go | 2 +- - device.go | 12 ++++++------ - 2 files changed, 7 insertions(+), 7 deletions(-) - -diff --git a/agent.go b/agent.go -index c161e93..e81c2cd 100644 ---- a/agent.go -+++ b/agent.go -@@ -185,7 +185,7 @@ var logsVSockPort = uint32(0) - var debugConsoleVSockPort = uint32(0) - - // Timeout waiting for a device to be hotplugged --var hotplugTimeout = 3 * time.Second -+var hotplugTimeout = 10 * time.Second - - // Specify the log level - var logLevel = defaultLogLevel -diff --git a/device.go b/device.go -index ec1907e..46a1b96 100644 ---- a/device.go -+++ b/device.go -@@ -179,13 +179,13 @@ func getPCIDeviceNameImpl(s *sandbox, pciID string) (string, error) { - return "", err - } - -- fieldLogger := agentLog.WithField("pciAddr", pciAddr) -- - // Rescan pci bus if we need to wait for a new pci device -- if err = rescanPciBus(); err != nil { -- fieldLogger.WithError(err).Error("Failed to scan pci bus") -- return "", err -- } -+ // FIXME:Comment out this code Temporarily, because once the PCIBus is scanned, -+ // the device hot-plug event is lost -+ //if err = rescanPciBus(); err != nil { -+ // fieldLogger.WithError(err).Error("Failed to scan pci bus") -+ // return "", err -+ //} - - return getDeviceName(s, pciAddr) - } --- -2.14.3 (Apple Git-98) - diff --git a/agent/patches/0007-agent-using-pcie-root-port-driver-to-hotplug-device.patch b/agent/patches/0007-agent-using-pcie-root-port-driver-to-hotplug-device.patch deleted file mode 100644 index 661aa5a..0000000 --- a/agent/patches/0007-agent-using-pcie-root-port-driver-to-hotplug-device.patch +++ /dev/null @@ -1,37 +0,0 @@ -From fa673c93e243ba297d53b585cd2f51fa68380fc5 Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Tue, 18 Aug 2020 19:37:48 +0800 -Subject: [PATCH 07/16] agent: using pcie-root-port driver to hotplug device - -reason: In original pci-bridge scheme, the "F" in the BDF is not used, -and was written hard code "0" in the kata-agent. But the "function" -is specified when switching to pcie-root-port scheme, so when should -pass the "pci-bridge BDF" or "pcie-root-port BDF" from kata-runtime -and use in kata-agent. - -Signed-off-by: jiangpengfei ---- - device.go | 7 ++++++- - 1 file changed, 6 insertions(+), 1 deletion(-) - -diff --git a/device.go b/device.go -index 46a1b96..8e6950c 100644 ---- a/device.go -+++ b/device.go -@@ -101,7 +101,12 @@ func getDevicePCIAddressImpl(pciID string) (string, error) { - - // Deduce the complete bridge address based on the bridge address identifier passed - // and the fact that bridges are attached on the main bus with function 0. -- pciBridgeAddr := fmt.Sprintf("0000:00:%s.0", bridgeID) -+ // Update: support pcie-root-port device -+ // In original pci-bridge scheme, the "F" in the BDF is not used, and was written -+ // hard code "0" in the kata-agent. But the "function" is specified when switching to -+ // pcie-root-port scheme, so when should pass the "pci-bridge BDF" or "pcie-root-port BDF" -+ // from kata-runtime and use in kata-agent. -+ pciBridgeAddr := fmt.Sprintf("0000:00:%s", bridgeID) - - // Find out the bus exposed by bridge - bridgeBusPath := fmt.Sprintf(pciBusPathFormat, sysBusPrefix, pciBridgeAddr) --- -2.14.3 (Apple Git-98) - diff --git a/agent/patches/0008-agent-support-get-root-bus-path-dynamically.patch b/agent/patches/0008-agent-support-get-root-bus-path-dynamically.patch deleted file mode 100644 index 86167c6..0000000 --- a/agent/patches/0008-agent-support-get-root-bus-path-dynamically.patch +++ /dev/null @@ -1,94 +0,0 @@ -From eea286fbafba2e95410b603fbef762e2b25eb207 Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Tue, 18 Aug 2020 19:45:57 +0800 -Subject: [PATCH 08/16] agent: support get root bus path dynamically - -reason: support get root bus dynamically no matter the -target arch is amd64 or arm64 - -Signed-off-by: jiangpengfei ---- - agent.go | 1 + - device_amd64.go | 6 +++++- - device_arm64.go | 27 ++++++++++++++++++++++++++- - 3 files changed, 32 insertions(+), 2 deletions(-) - -diff --git a/agent.go b/agent.go -index e81c2cd..50afd7a 100644 ---- a/agent.go -+++ b/agent.go -@@ -730,6 +730,7 @@ func (s *sandbox) listenToUdevEvents() { - defer uEvHandler.Close() - - fieldLogger.Infof("Started listening for uevents") -+ rootBusPath := initRootBusPath() - - for { - uEv, err := uEvHandler.Read() -diff --git a/device_amd64.go b/device_amd64.go -index 66bc052..26f55bf 100644 ---- a/device_amd64.go -+++ b/device_amd64.go -@@ -8,7 +8,7 @@ - package main - - const ( -- rootBusPath = "/devices/pci0000:00" -+ defaultRootBusPath = "/devices/pci0000:00" - - // From https://www.kernel.org/doc/Documentation/acpi/namespace.txt - // The Linux kernel's core ACPI subsystem creates struct acpi_device -@@ -21,3 +21,7 @@ const ( - // in a subdirectory whose prefix is pfn (page frame number). - pfnDevPrefix = "/pfn" - ) -+ -+func initRootBusPath() string { -+ return defaultRootBusPath -+} -diff --git a/device_arm64.go b/device_arm64.go -index b73b582..d039c67 100644 ---- a/device_arm64.go -+++ b/device_arm64.go -@@ -6,8 +6,14 @@ - - package main - -+import ( -+ "fmt" -+ "io/ioutil" -+ "regexp" -+) -+ - const ( -- rootBusPath = "/devices/platform/4010000000.pcie/pci0000:00" -+ defaultRootBusPath = "/devices/platform/4010000000.pcie/pci0000:00" - - // From https://www.kernel.org/doc/Documentation/acpi/namespace.txt - // The Linux kernel's core ACPI subsystem creates struct acpi_device -@@ -20,3 +26,22 @@ const ( - // in a subdirectory whose prefix is pfn (page frame number). - pfnDevPrefix = "/pfn" - ) -+ -+func initRootBusPath() string { -+ pcieDriverReg := regexp.MustCompile(`^[0-9a-f]{10}.pcie$`) -+ rootBusPath := defaultRootBusPath -+ files, err := ioutil.ReadDir("/sys/devices/platform") -+ if err != nil { -+ return rootBusPath -+ } -+ for _, f := range files { -+ if !f.IsDir() { -+ continue -+ } -+ if pcieDriverReg.MatchString(f.Name()) { -+ rootBusPath = fmt.Sprintf("/devices/platform/%s/pci0000:00", f.Name()) -+ break -+ } -+ } -+ return rootBusPath -+} --- -2.14.3 (Apple Git-98) - diff --git a/agent/patches/0009-storage-add-pkg-storage-for-mount.patch b/agent/patches/0009-storage-add-pkg-storage-for-mount.patch deleted file mode 100644 index aabfb25..0000000 --- a/agent/patches/0009-storage-add-pkg-storage-for-mount.patch +++ /dev/null @@ -1,188 +0,0 @@ -From 1268b710c7f0528d971d9c0e54429d3f4e48c372 Mon Sep 17 00:00:00 2001 -From: holyfei -Date: Tue, 18 Aug 2020 16:45:29 +0800 -Subject: [PATCH 09/16] storage: add pkg/storage for mount - -reason: add gpath.go and nfs.go, provide mount -functions and structs - -Signed-off-by: yangfeiyu ---- - pkg/storage/gpath.go | 64 ++++++++++++++++++++++++++++++++++++ - pkg/storage/nfs.go | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++++ - 2 files changed, 157 insertions(+) - create mode 100644 pkg/storage/gpath.go - create mode 100644 pkg/storage/nfs.go - -diff --git a/pkg/storage/gpath.go b/pkg/storage/gpath.go -new file mode 100644 -index 0000000..5cb951f ---- /dev/null -+++ b/pkg/storage/gpath.go -@@ -0,0 +1,64 @@ -+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. -+// SPDX-License-Identifier: Apache-2.0 -+// Description: common functions -+// Author: licuifang -+// Create: 2019-06-24 -+ -+package storage -+ -+import ( -+ "fmt" -+ "os" -+ "syscall" -+ -+ "github.com/opencontainers/runc/libcontainer/mount" -+) -+ -+func ValidateGpath(guestPath string, opts []string) error { -+ for _, opt := range opts { -+ switch opt { -+ case "shared": -+ flag := syscall.MS_SHARED -+ if err := ensureMountedAs(guestPath, flag); err != nil { -+ return err -+ } -+ return nil -+ case "mounted": -+ // if mounted in option, the guestpath must exist and must be a mountpoint -+ _, err := os.Stat(guestPath) -+ if err != nil { -+ return err -+ } -+ mountpoint, err := mount.Mounted(guestPath) -+ if err != nil { -+ return err -+ } -+ if mountpoint == false { -+ return fmt.Errorf("the guespath:%s is not a mountpoint while mounted was set", guestPath) -+ } -+ return nil -+ } -+ } -+ if err := os.MkdirAll(guestPath, 0750); err != nil { -+ return err -+ } -+ return nil -+} -+ -+func ensureMountedAs(mountPoint string, flag int) error { -+ if err := os.MkdirAll(mountPoint, 0750); err != nil { -+ return err -+ } -+ mounted, err := mount.Mounted(mountPoint) -+ if err != nil { -+ return err -+ } -+ -+ if !mounted { -+ if err := syscall.Mount(mountPoint, mountPoint, "bind", uintptr(syscall.MS_BIND), ""); err != nil { -+ return err -+ } -+ } -+ -+ return syscall.Mount("", mountPoint, "", uintptr(flag), "") -+} -diff --git a/pkg/storage/nfs.go b/pkg/storage/nfs.go -new file mode 100644 -index 0000000..44bc85d ---- /dev/null -+++ b/pkg/storage/nfs.go -@@ -0,0 +1,93 @@ -+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. -+// SPDX-License-Identifier: Apache-2.0 -+// Description: common functions -+// Author: leizhongkai -+// Create: 2019-03-10 -+ -+package storage -+ -+import ( -+ "fmt" -+ "os" -+ "os/exec" -+ "path/filepath" -+ "strings" -+ "time" -+ -+ "github.com/opencontainers/runc/libcontainer/mount" -+ "github.com/sirupsen/logrus" -+) -+ -+const ( -+ NFS = "nfs" -+ -+ MAX_MOUNT_ATTEMPTS = 100 -+ MAX_MOUTN_TIMEOUT = 90 // seconds -+ -+ // ignored errors -+ PERMISSION_DENY = "Permission denied" -+ kataGuestStorageDir = "/run/kata-containers/storage/containers/" -+) -+ -+func nfsMount(source, dest string, opt string) error { -+ cmd := exec.Command("/bin/mount", "-t", "nfs", "-o", opt, source, dest) -+ res, err := cmd.Output() -+ logrus.Debugf("mount %s to %s, and get res %s", source, dest, string(res)) -+ return err -+} -+ -+func nfsMountWithAttempt(source, dest string, opt string) (err error) { -+ timeStart := time.Now().Unix() -+ for i := 0; i < MAX_MOUNT_ATTEMPTS; i++ { -+ logrus.Infof("this is the %d times to mount %s to %s", i, source, dest) -+ err = nfsMount(source, dest, opt) -+ if err != nil { -+ ee, ok := err.(*exec.ExitError) -+ if ok { -+ if strings.Contains(string(ee.Stderr), PERMISSION_DENY) { -+ logrus.Errorf("mounting nfs:%s to %s, get error: %s,should break", source, dest, string(ee.Stderr)) -+ // We do not retry when the error type is PERMISSION_DENY. -+ // The reason for the retry is that when you do a SFS mount, -+ // the network may not have been created yet, because -+ // creating the network and startup container is asynchronous -+ break -+ } -+ logrus.Infof("mounting nfs:%s to %s, get error: %s,will retry", source, dest, string(ee.Stderr)) -+ } -+ -+ elapsed := time.Now().Unix() - timeStart -+ if elapsed < MAX_MOUTN_TIMEOUT { -+ time.Sleep(50 * time.Microsecond) -+ continue -+ } -+ } -+ break -+ } -+ -+ return err -+} -+ -+func MountRemoteNfs(source string, opt []string, sandboxId string) error { -+ item := strings.Split(source, ":") -+ if len(item) != 2 { -+ return fmt.Errorf("the nfs of %s format is error", source) -+ } -+ tmpDes := filepath.Join(kataGuestStorageDir, sandboxId, item[0],item[1]) -+ if mounted, err := mount.Mounted(tmpDes); err == nil && mounted == true { -+ return nil -+ } -+ err := os.MkdirAll(tmpDes, 0750) -+ if err != nil { -+ logrus.Infof("mkdir %s failed before mount remote nfs server", tmpDes) -+ return err -+ } -+ defer func() { -+ if err != nil { -+ os.RemoveAll(tmpDes) -+ } -+ }() -+ -+ err = nfsMountWithAttempt(source, tmpDes, strings.Join(opt, ",")) -+ -+ return err -+} --- -2.14.3 (Apple Git-98) - diff --git a/agent/patches/0010-storage-mount-nfs-and-gpath-in-agent.patch b/agent/patches/0010-storage-mount-nfs-and-gpath-in-agent.patch deleted file mode 100644 index 8e5fdb6..0000000 --- a/agent/patches/0010-storage-mount-nfs-and-gpath-in-agent.patch +++ /dev/null @@ -1,92 +0,0 @@ -From 79bafc5fd8a1dcda6f44ecd830dfe50f4ef1b34b Mon Sep 17 00:00:00 2001 -From: holyfei -Date: Tue, 18 Aug 2020 20:41:30 +0800 -Subject: [PATCH 10/16] storage: mount nfs and gpath in agent - -reason: add nfsStorageHandler and gpathStorageHandler in -storageHandlerList to mount nfs and gpath - -Signed-off-by: yangfeiyu ---- - device.go | 2 ++ - mount.go | 37 +++++++++++++++++++++++++++++++++++++ - 2 files changed, 39 insertions(+) - -diff --git a/device.go b/device.go -index 8e6950c..29f72e9 100644 ---- a/device.go -+++ b/device.go -@@ -35,6 +35,8 @@ const ( - driverEphemeralType = "ephemeral" - driverLocalType = "local" - vmRootfs = "/" -+ driverNfsType = "nfs" -+ driverGpathType = "gpath" - ) - - const ( -diff --git a/mount.go b/mount.go -index a05d4a6..de2bfaf 100644 ---- a/mount.go -+++ b/mount.go -@@ -17,6 +17,7 @@ import ( - "strings" - "syscall" - -+ rmtStorage "github.com/kata-containers/agent/pkg/storage" - pb "github.com/kata-containers/agent/protocols/grpc" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" -@@ -218,6 +219,8 @@ var storageHandlerList = map[string]storageHandler{ - driverEphemeralType: ephemeralStorageHandler, - driverLocalType: localStorageHandler, - driverNvdimmType: nvdimmStorageHandler, -+ driverNfsType: nfsStorageHandler, -+ driverGpathType: gpathStorageHandler, - } - - func ephemeralStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (string, error) { -@@ -339,6 +342,40 @@ func nvdimmStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (st - return "", fmt.Errorf("invalid nvdimm source path: %v", storage.Source) - } - -+func nfsStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (string, error) { -+ s.Lock() -+ defer s.Unlock() -+ -+ // mount nfs -+ err := rmtStorage.MountRemoteNfs(storage.Source, storage.Options, s.id) -+ if err != nil { -+ return "", fmt.Errorf("mount %s to %s failed, get err:%s", storage.Source, storage.MountPoint, err) -+ } -+ if err := os.MkdirAll(storage.MountPoint, 0750); err != nil { -+ logrus.Infof("mkdir %s failed after mount remote nfs server", storage.MountPoint) -+ return "", err -+ } -+ -+ defer func() { -+ if err != nil { -+ os.RemoveAll(storage.MountPoint) -+ } -+ }() -+ -+ return storage.MountPoint, nil -+} -+ -+func gpathStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (string, error) { -+ s.Lock() -+ defer s.Unlock() -+ // validate guespath -+ err := rmtStorage.ValidateGpath(storage.Source, storage.Options) -+ if err != nil { -+ return "", err -+ } -+ return "", nil -+} -+ - // virtioSCSIStorageHandler handles the storage for scsi driver. - func virtioSCSIStorageHandler(ctx context.Context, storage pb.Storage, s *sandbox) (string, error) { - // Retrieve the device path from SCSI address. --- -2.14.3 (Apple Git-98) - diff --git a/agent/patches/0011-agent-fix-agent-reap-agent-process-blocked-problem.patch b/agent/patches/0011-agent-fix-agent-reap-agent-process-blocked-problem.patch deleted file mode 100644 index 272e0b5..0000000 --- a/agent/patches/0011-agent-fix-agent-reap-agent-process-blocked-problem.patch +++ /dev/null @@ -1,78 +0,0 @@ -From 3ac1232a2e3fbfc0465473e5d81cde41847c4252 Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Wed, 19 Aug 2020 11:47:37 +0800 -Subject: [PATCH 11/16] agent: fix agent reap agent process blocked problem - -reason: add container waitProcess() timeout when -container process status is D/T. - -Signed-off-by: jiangpengfei ---- - grpc.go | 43 +++++++++++++++++++++++++++++++++---------- - 1 file changed, 33 insertions(+), 10 deletions(-) - -diff --git a/grpc.go b/grpc.go -index de2cae7..3dd088e 100644 ---- a/grpc.go -+++ b/grpc.go -@@ -49,6 +49,11 @@ const ( - libcontainerPath = "/run/libcontainer" - ) - -+// keep waitProcessTimeout value same as value in kata-runtime wait WaitProcessRequest response -+const ( -+ waitProcessTimeOut = 10 -+) -+ - var ( - sysfsCPUOnlinePath = "/sys/devices/system/cpu" - sysfsMemOnlinePath = "/sys/devices/system/memory" -@@ -996,17 +1001,35 @@ func (a *agentGRPC) WaitProcess(ctx context.Context, req *pb.WaitProcessRequest) - ctr.deleteProcess(proc.id) - }) - -- // Using helper function wait() to deal with the subreaper. -- libContProcess := (*reaperLibcontainerProcess)(&(proc.process)) -- exitCode, err := a.sandbox.subreaper.wait(proc.exitCodeCh, libContProcess) -- if err != nil { -- return &pb.WaitProcessResponse{}, err -+ done := make(chan error) -+ var exitCode int = 0 -+ go func() { -+ // Using helper function wait() to deal with the subreaper. -+ libContProcess := (*reaperLibcontainerProcess)(&(proc.process)) -+ var err error -+ exitCode, err = a.sandbox.subreaper.wait(proc.exitCodeCh, libContProcess) -+ if err != nil { -+ done <- err -+ close(done) -+ return -+ } -+ // refill the exitCodeCh with the exitcode which can be read out -+ // by another WaitProcess(). Since this channel isn't be closed, -+ // here the refill will always success and it will be free by GC -+ // once the process exits. -+ proc.exitCodeCh <- exitCode -+ -+ close(done) -+ }() -+ -+ select { -+ case err := <-done: -+ if err != nil { -+ return &pb.WaitProcessResponse{}, err -+ } -+ case <-time.After(time.Duration(waitProcessTimeOut) * time.Second): -+ return &pb.WaitProcessResponse{}, grpcStatus.Errorf(codes.DeadlineExceeded, "agent wait reap container process timeout reached after %ds", waitProcessTimeOut) - } -- //refill the exitCodeCh with the exitcode which can be read out -- //by another WaitProcess(). Since this channel isn't be closed, -- //here the refill will always success and it will be free by GC -- //once the process exits. -- proc.exitCodeCh <- exitCode - - return &pb.WaitProcessResponse{ - Status: int32(exitCode), --- -2.14.3 (Apple Git-98) - diff --git a/agent/patches/0012-network-support-set-dns-without-nameserver.patch b/agent/patches/0012-network-support-set-dns-without-nameserver.patch deleted file mode 100644 index 850c88c..0000000 --- a/agent/patches/0012-network-support-set-dns-without-nameserver.patch +++ /dev/null @@ -1,38 +0,0 @@ -From edb29dfd8f786735763245b3f156b50fd3c1a08e Mon Sep 17 00:00:00 2001 -From: holyfei -Date: Wed, 19 Aug 2020 15:15:31 +0800 -Subject: [PATCH 12/16] network: support set dns without nameserver - -reason: when runtime sends dns without nameserver to agent, -add nameserver before ip address. scenario like annotation -with dns - -Signed-off-by: yangfeiyu ---- - network.go | 5 ++++- - 1 file changed, 4 insertions(+), 1 deletion(-) - -diff --git a/network.go b/network.go -index 02e28cb..7046cf8 100644 ---- a/network.go -+++ b/network.go -@@ -708,6 +708,9 @@ func setupDNS(dns []string) (err error) { - defer file.Close() - - for i, line := range dns { -+ if !strings.Contains(line, "nameserver") { -+ line = "nameserver" + " " + line -+ } - if i == (len(dns) - 1) { - _, err = file.WriteString(strings.TrimSpace(line)) - } else { -@@ -761,4 +764,4 @@ func (s *sandbox) handleLocalhost() error { - } - - return netlink.LinkSetUp(lo) --} -\ No newline at end of file -+} --- -2.14.3 (Apple Git-98) - diff --git a/agent/patches/0013-agent-support-setting-multi-queues-of-interface.patch b/agent/patches/0013-agent-support-setting-multi-queues-of-interface.patch deleted file mode 100644 index 34c2bb9..0000000 --- a/agent/patches/0013-agent-support-setting-multi-queues-of-interface.patch +++ /dev/null @@ -1,304 +0,0 @@ -From 1394dcf579849e5d8103c31556e9af0216a875d2 Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Wed, 19 Aug 2020 17:15:51 +0800 -Subject: [PATCH 13/16] agent: support setting multi queues of interface - -reason: support setting multi queues of a interface -when runtime passing Queue in the request. - -Signed-off-by: jiangpengfei ---- - network.go | 12 +++++- - pkg/net/ethtool.go | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++ - pkg/types/types.pb.go | 82 ++++++++++++++++++++++++++----------- - pkg/types/types.proto | 1 + - 4 files changed, 181 insertions(+), 24 deletions(-) - create mode 100644 pkg/net/ethtool.go - -diff --git a/network.go b/network.go -index 7046cf8..1baaa2e 100644 ---- a/network.go -+++ b/network.go -@@ -17,7 +17,7 @@ import ( - "syscall" - - "golang.org/x/sys/unix" -- -+ agentNet "github.com/kata-containers/agent/pkg/net" - "github.com/kata-containers/agent/pkg/types" - pb "github.com/kata-containers/agent/protocols/grpc" - "github.com/sirupsen/logrus" -@@ -273,6 +273,16 @@ func (s *sandbox) updateInterface(netHandle *netlink.Handle, iface *types.Interf - if err == nil { - err = retErr - } -+ -+ // if link is up, then set the multi queue to it, -+ // the kernel of newer version may set multi queue by itself, -+ // but we can not rely on it. -+ if err == nil && iface.Queues > 0 { -+ if ethErr := agentNet.GetEthtool().SetChannel(iface.Name, iface.Queues); ethErr != nil { -+ err = grpcStatus.Errorf(codes.Internal, "Could not set multi queue %d for interface %v: %v", -+ iface.Queues, link, ethErr) -+ } -+ } - }() - - fieldLogger.WithField("link", fmt.Sprintf("%+v", link)).Info("Link found") -diff --git a/pkg/net/ethtool.go b/pkg/net/ethtool.go -new file mode 100644 -index 0000000..56a1ece ---- /dev/null -+++ b/pkg/net/ethtool.go -@@ -0,0 +1,110 @@ -+/* -+Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. -+SPDX-License-Identifier: Apache-2.0 -+Description: common functions -+Author: fengshaobao -+Create: 2019-05-28 -+*/ -+ -+package net -+ -+import ( -+ "fmt" -+ "sync" -+ "syscall" -+ "unsafe" -+ -+ "github.com/sirupsen/logrus" -+) -+ -+const ( -+ ethtoolGChannels = 0x0000003c /* Get no of channels */ -+ ethtoolSChannels = 0x0000003d /* Set no of channels */ -+ ifNameSize = 16 -+ siocEthtool = 0x8946 -+) -+ -+var ( -+ t *Ethtool -+ once sync.Once -+) -+ -+// Ethtool to set multiqueue of a network interface, -+// with the public method "SetChannel" to set queues of an interface. -+type Ethtool struct { -+ fd int -+} -+ -+type ifReq struct { -+ ifrName [ifNameSize]byte -+ ifrData uintptr -+} -+ -+type ethtoolChannels struct { -+ // ETHTOOL_{G,S}CHANNELS -+ cmd uint32 -+ // Read only. Maximum number of receive channel the driver support. -+ maxRx uint32 -+ // Read only. Maximum number of transmit channel the driver support -+ maxTx uint32 -+ // Read only. Maximum number of other channel the driver support -+ maxOther uint32 -+ //Read only. Maximum number of combined channel the driver support. Set of queues RX, TX or other -+ maxCombined uint32 -+ // Valid values are in the range 1 to the max_rx -+ rxCount uint32 -+ // Valid values are in the range 1 to the max_tx -+ txCount uint32 -+ // Valid values are in the range 1 to the max_other -+ otherCount uint32 -+ // Valid values are in the range 1 to the max_combined -+ combinedCount uint32 -+} -+ -+// GetEthtool to config multiqueue of a network interface -+func GetEthtool() *Ethtool { -+ once.Do(func() { -+ var err error -+ if t, err = newEthtool(); err != nil { -+ panic("can not init a socket fd for ethtool") -+ } -+ }) -+ return t -+} -+ -+func newEthtool() (*Ethtool, error) { -+ fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP) -+ if err != nil || fd < 0 { -+ logrus.Warningf("Can not get socket of inet") -+ var newErr error -+ fd, newErr = syscall.Socket(syscall.AF_NETLINK, syscall.SOCK_RAW, syscall.NETLINK_GENERIC) -+ if newErr != nil || fd < 0 { -+ return nil, fmt.Errorf("create inet socket with error: %v, and create netlink socket with error: %v", err, newErr) -+ } -+ } -+ -+ return &Ethtool{ -+ fd: int(fd), -+ }, nil -+} -+ -+// SetChannel Set the queues of a network interface. -+// @devName: the network interface name, e.g. eth0. -+// @queues: queues for the network interface. -+func (e *Ethtool) SetChannel(devName string, queues uint32) error { -+ c := ðtoolChannels{ -+ cmd: ethtoolSChannels, -+ combinedCount: queues, -+ } -+ var ifName [ifNameSize]byte -+ copy(ifName[:], []byte(devName)) -+ ifr := ifReq{ -+ ifrName: ifName, -+ ifrData: uintptr(unsafe.Pointer(&c)), -+ } -+ _, _, ep := syscall.Syscall(syscall.SYS_IOCTL, uintptr(e.fd), siocEthtool, uintptr(unsafe.Pointer(&ifr))) -+ if ep != 0 { -+ return syscall.Errno(ep) -+ } -+ return nil -+} -diff --git a/pkg/types/types.pb.go b/pkg/types/types.pb.go -index 7ea63e3..8b7e2a5 100644 ---- a/pkg/types/types.pb.go -+++ b/pkg/types/types.pb.go -@@ -100,6 +100,7 @@ type Interface struct { - // list: "veth", "macvtap", "vlan", "macvlan", "tap", ... - Type string `protobuf:"bytes,7,opt,name=type,proto3" json:"type,omitempty"` - RawFlags uint32 `protobuf:"varint,8,opt,name=raw_flags,json=rawFlags,proto3" json:"raw_flags,omitempty"` -+ Queues uint32 `protobuf:"varint,9,opt,name=Queues,proto3" json:"Queues,omitempty"` - } - - func (m *Interface) Reset() { *m = Interface{} } -@@ -163,6 +164,13 @@ func (m *Interface) GetRawFlags() uint32 { - return 0 - } - -+func (m *Interface) GetQueues() uint32 { -+ if m != nil { -+ return m.Queues -+ } -+ return 0 -+} -+ - type Route struct { - Dest string `protobuf:"bytes,1,opt,name=dest,proto3" json:"dest,omitempty"` - Gateway string `protobuf:"bytes,2,opt,name=gateway,proto3" json:"gateway,omitempty"` -@@ -319,6 +327,11 @@ func (m *Interface) MarshalTo(dAtA []byte) (int, error) { - i++ - i = encodeVarintTypes(dAtA, i, uint64(m.RawFlags)) - } -+ if m.Queues != 0 { -+ dAtA[i] = 0x48 -+ i++ -+ i = encodeVarintTypes(dAtA, i, uint64(m.Queues)) -+ } - return i, nil - } - -@@ -430,6 +443,9 @@ func (m *Interface) Size() (n int) { - if m.RawFlags != 0 { - n += 1 + sovTypes(uint64(m.RawFlags)) - } -+ if m.Queues != 0 { -+ n += 1 + sovTypes(uint64(m.Queues)) -+ } - return n - } - -@@ -841,6 +857,25 @@ func (m *Interface) Unmarshal(dAtA []byte) error { - break - } - } -+ case 9: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field Queues", wireType) -+ } -+ m.Queues = 0 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowTypes -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ m.Queues |= (uint32(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } - default: - iNdEx = preIndex - skippy, err := skipTypes(dAtA[iNdEx:]) -@@ -1155,27 +1190,28 @@ var ( - func init() { proto.RegisterFile("pkg/types/types.proto", fileDescriptorTypes) } - - var fileDescriptorTypes = []byte{ -- // 345 bytes of a gzipped FileDescriptorProto -- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x91, 0xd1, 0x6a, 0xfa, 0x30, -- 0x14, 0xc6, 0xff, 0xb1, 0xb6, 0xda, 0xe3, 0xdf, 0xad, 0x84, 0x4d, 0xc2, 0x06, 0x52, 0xbc, 0x59, -- 0xd9, 0x85, 0x03, 0x37, 0x76, 0xbf, 0x5d, 0x08, 0xde, 0x49, 0x5e, 0x60, 0xc4, 0x36, 0x3a, 0xd1, -- 0xda, 0xd2, 0x44, 0x8b, 0xec, 0x05, 0x77, 0xb9, 0x47, 0x10, 0x9f, 0x64, 0xe4, 0x24, 0x8a, 0xbb, -- 0xd1, 0xef, 0x97, 0x93, 0xd3, 0xef, 0x7c, 0x27, 0x70, 0x5b, 0xae, 0x16, 0x4f, 0x7a, 0x5f, 0x4a, -- 0x65, 0x7f, 0x87, 0x65, 0x55, 0xe8, 0x82, 0xfa, 0x08, 0x83, 0x19, 0x84, 0x93, 0xe9, 0x5b, 0x96, -- 0x55, 0x52, 0x29, 0xfa, 0x00, 0xc1, 0x5c, 0xe4, 0xcb, 0xf5, 0x9e, 0x91, 0x98, 0x24, 0x57, 0xa3, -- 0xeb, 0xa1, 0xed, 0x98, 0x4c, 0xc7, 0x78, 0xcc, 0x5d, 0x99, 0x32, 0x68, 0x09, 0xdb, 0xc3, 0x1a, -- 0x31, 0x49, 0x42, 0x7e, 0x42, 0x4a, 0xa1, 0x99, 0x0b, 0xb5, 0x62, 0x1e, 0x1e, 0xa3, 0x1e, 0x1c, -- 0x08, 0x84, 0x93, 0x8d, 0x96, 0xd5, 0x5c, 0xa4, 0x92, 0xf6, 0x20, 0xc8, 0xe4, 0x6e, 0x99, 0x4a, -- 0x34, 0x09, 0xb9, 0x23, 0xd3, 0xb9, 0x11, 0xb9, 0x74, 0x1f, 0x44, 0x4d, 0x47, 0xd0, 0x39, 0x4f, -- 0x27, 0x15, 0xf3, 0x62, 0x2f, 0xe9, 0x8c, 0xa2, 0xf3, 0x54, 0xae, 0xc2, 0x2f, 0x2f, 0xd1, 0x08, -- 0xbc, 0x5c, 0x6f, 0x59, 0x33, 0x26, 0x49, 0x93, 0x1b, 0x69, 0x1c, 0x3f, 0x6b, 0x73, 0x81, 0xf9, -- 0xd6, 0xd1, 0x92, 0x49, 0x51, 0xa6, 0x4b, 0x2c, 0x04, 0x36, 0x85, 0x43, 0x33, 0x8b, 0xf1, 0x60, -- 0x2d, 0x3b, 0x8b, 0xd1, 0xf4, 0x1e, 0xc2, 0x4a, 0xd4, 0x1f, 0xf3, 0xb5, 0x58, 0x28, 0xd6, 0x8e, -- 0x49, 0xd2, 0xe5, 0xed, 0x4a, 0xd4, 0x63, 0xc3, 0x83, 0x2f, 0xf0, 0x79, 0xb1, 0xd5, 0x98, 0x22, -- 0x93, 0x4a, 0xbb, 0x6c, 0xa8, 0x8d, 0xcf, 0x42, 0x68, 0x59, 0x8b, 0xfd, 0x69, 0x5b, 0x0e, 0x2f, -- 0x76, 0xe1, 0xfd, 0xd9, 0x45, 0x0f, 0x02, 0x55, 0x6c, 0xab, 0x54, 0x62, 0x8c, 0x90, 0x3b, 0xa2, -- 0x37, 0xe0, 0xab, 0xb4, 0x28, 0x25, 0x06, 0xe9, 0x72, 0x0b, 0x8f, 0x77, 0xd0, 0x3e, 0xbd, 0x10, -- 0x0d, 0xa0, 0xb1, 0x7b, 0x89, 0xfe, 0xe1, 0xff, 0x6b, 0x44, 0xde, 0xff, 0x7f, 0x1f, 0xfb, 0xe4, -- 0xe7, 0xd8, 0x27, 0x87, 0x63, 0x9f, 0xcc, 0x02, 0x7c, 0xfb, 0xe7, 0xdf, 0x00, 0x00, 0x00, 0xff, -- 0xff, 0xb5, 0x52, 0x37, 0xf2, 0x14, 0x02, 0x00, 0x00, -+ // 356 bytes of a gzipped FileDescriptorProto -+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x92, 0xc1, 0x4e, 0xc2, 0x40, -+ 0x10, 0x86, 0x5d, 0x4a, 0x0b, 0x1d, 0x44, 0x9b, 0x8d, 0x92, 0x8d, 0x26, 0xa4, 0xe1, 0x62, 0xe3, -+ 0x01, 0x13, 0x34, 0xde, 0xf5, 0x40, 0xc2, 0x0d, 0xf7, 0x05, 0xcc, 0xd2, 0x2e, 0x48, 0xa0, 0xb4, -+ 0xe9, 0xb6, 0x34, 0xc4, 0x17, 0xf4, 0xe8, 0x23, 0x18, 0x9e, 0xc2, 0xa3, 0xd9, 0xd9, 0x85, 0xe0, -+ 0x05, 0xfe, 0x6f, 0x67, 0xa7, 0xff, 0xfc, 0xd3, 0xc2, 0x75, 0xbe, 0x5a, 0x3c, 0x94, 0xbb, 0x5c, -+ 0x2a, 0xf3, 0x3b, 0xcc, 0x8b, 0xac, 0xcc, 0xa8, 0x8b, 0x30, 0x98, 0x81, 0x3f, 0x99, 0xbe, 0x24, -+ 0x49, 0x21, 0x95, 0xa2, 0x77, 0xe0, 0xcd, 0x45, 0xba, 0x5c, 0xef, 0x18, 0x09, 0x49, 0x74, 0x31, -+ 0xba, 0x1c, 0x9a, 0x8e, 0xc9, 0x74, 0x8c, 0xc7, 0xdc, 0x96, 0x29, 0x83, 0x96, 0x30, 0x3d, 0xac, -+ 0x11, 0x92, 0xc8, 0xe7, 0x07, 0xa4, 0x14, 0x9a, 0xa9, 0x50, 0x2b, 0xe6, 0xe0, 0x31, 0xea, 0xc1, -+ 0x2f, 0x01, 0x7f, 0xb2, 0x29, 0x65, 0x31, 0x17, 0xb1, 0xa4, 0x3d, 0xf0, 0x12, 0xb9, 0x5d, 0xc6, -+ 0x12, 0x4d, 0x7c, 0x6e, 0x49, 0x77, 0x6e, 0x44, 0x2a, 0xed, 0x03, 0x51, 0xd3, 0x11, 0x74, 0x8e, -+ 0xd3, 0x49, 0xc5, 0x9c, 0xd0, 0x89, 0x3a, 0xa3, 0xe0, 0x38, 0x95, 0xad, 0xf0, 0xd3, 0x4b, 0x34, -+ 0x00, 0x27, 0x2d, 0x2b, 0xd6, 0x0c, 0x49, 0xd4, 0xe4, 0x5a, 0x6a, 0xc7, 0x8f, 0x5a, 0x5f, 0x60, -+ 0xae, 0x71, 0x34, 0xa4, 0x53, 0xe4, 0xf1, 0x12, 0x0b, 0x9e, 0x49, 0x61, 0x51, 0xcf, 0xa2, 0x3d, -+ 0x58, 0xcb, 0xcc, 0xa2, 0x35, 0xbd, 0x05, 0xbf, 0x10, 0xf5, 0xfb, 0x7c, 0x2d, 0x16, 0x8a, 0xb5, -+ 0x43, 0x12, 0x75, 0x79, 0xbb, 0x10, 0xf5, 0x58, 0xb3, 0xb6, 0x78, 0xab, 0x64, 0x25, 0x15, 0xf3, -+ 0xb1, 0x62, 0x69, 0xf0, 0x09, 0x2e, 0xcf, 0xaa, 0x12, 0xd3, 0x25, 0x52, 0x95, 0x36, 0x33, 0x6a, -+ 0xed, 0xbf, 0x10, 0xa5, 0xac, 0xc5, 0xee, 0xb0, 0x45, 0x8b, 0x27, 0x3b, 0x72, 0xfe, 0xed, 0xa8, -+ 0x07, 0x9e, 0xca, 0xaa, 0x22, 0x96, 0x18, 0xcf, 0xe7, 0x96, 0xe8, 0x15, 0xb8, 0x2a, 0xce, 0x72, -+ 0x89, 0x01, 0xbb, 0xdc, 0xc0, 0xfd, 0x0d, 0xb4, 0x0f, 0x6f, 0x8e, 0x7a, 0xd0, 0xd8, 0x3e, 0x05, -+ 0x67, 0xf8, 0xff, 0x1c, 0x90, 0xd7, 0xf3, 0xaf, 0x7d, 0x9f, 0x7c, 0xef, 0xfb, 0xe4, 0x67, 0xdf, -+ 0x27, 0x33, 0x0f, 0xbf, 0x89, 0xc7, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa5, 0x57, 0x70, 0xc8, -+ 0x2c, 0x02, 0x00, 0x00, - } -diff --git a/pkg/types/types.proto b/pkg/types/types.proto -index f6856e1..149df13 100644 ---- a/pkg/types/types.proto -+++ b/pkg/types/types.proto -@@ -37,6 +37,7 @@ message Interface { - // list: "veth", "macvtap", "vlan", "macvlan", "tap", ... - string type = 7; - uint32 raw_flags = 8; -+ uint32 Queues = 9; - } - - message Route { --- -2.14.3 (Apple Git-98) - diff --git a/agent/patches/0014-agent-fix-init-hugepages-failed-problem.patch b/agent/patches/0014-agent-fix-init-hugepages-failed-problem.patch deleted file mode 100644 index a29b0e6..0000000 --- a/agent/patches/0014-agent-fix-init-hugepages-failed-problem.patch +++ /dev/null @@ -1,56 +0,0 @@ -From 9e1478d7989fea4ee759cc13009d8de07dac2879 Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Wed, 19 Aug 2020 17:01:00 +0800 -Subject: [PATCH 14/16] agent: fix init hugepages failed problem - -reason: fix init hugepages failed problem - -Signed-off-by: jiangpengfei ---- - .../runc/libcontainer/cgroups/fs/apply_raw.go | 20 ++++++++++++++++++++ - 1 file changed, 20 insertions(+) - -diff --git a/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go b/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go -index ec148b4..47aa3c3 100644 ---- a/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go -+++ b/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go -@@ -8,11 +8,13 @@ import ( - "os" - "path/filepath" - "sync" -+ "time" - - "github.com/opencontainers/runc/libcontainer/cgroups" - "github.com/opencontainers/runc/libcontainer/configs" - libcontainerUtils "github.com/opencontainers/runc/libcontainer/utils" - "github.com/pkg/errors" -+ "github.com/sirupsen/logrus" - "golang.org/x/sys/unix" - ) - -@@ -409,3 +411,21 @@ func CheckCpushares(path string, c uint64) error { - func (m *Manager) GetCgroups() (*configs.Cgroup, error) { - return m.Cgroups, nil - } -+ -+func init() { -+ go func() { -+ var err error -+ if len(HugePageSizes) != 0 { -+ return -+ } -+ for i := 0; i < 10; i++ { -+ HugePageSizes, err = cgroups.GetHugePageSize() -+ if err == nil { -+ logrus.Infof("init hugepages ok loop=%d %v", i, err) -+ return -+ } -+ logrus.Errorf("init hugepages failed loop=%d %v", i, err) -+ time.Sleep(time.Second) -+ } -+ }() -+} -\ No newline at end of file --- -2.14.3 (Apple Git-98) - diff --git a/agent/patches/0015-agent-add-support-of-getting-container-s-network-sta.patch b/agent/patches/0015-agent-add-support-of-getting-container-s-network-sta.patch deleted file mode 100644 index 5ca41d6..0000000 --- a/agent/patches/0015-agent-add-support-of-getting-container-s-network-sta.patch +++ /dev/null @@ -1,2649 +0,0 @@ -From 555db828d0dae24330bfdc4e3d10a145ce44a66b Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Wed, 19 Aug 2020 20:52:35 +0800 -Subject: [PATCH 15/16] agent: add support of getting container's network stats - -reason: support to get more detailed container's network -stats info - -Signed-off-by: jiangpengfei ---- - grpc.go | 12 +- - network_stats.go | 241 ++++++ - protocols/grpc/agent.pb.go | 1749 +++++++++++++++++++++++++++++++++++--------- - protocols/grpc/agent.proto | 33 +- - 4 files changed, 1684 insertions(+), 351 deletions(-) - create mode 100644 network_stats.go - -diff --git a/grpc.go b/grpc.go -index 3dd088e..1b63cde 100644 ---- a/grpc.go -+++ b/grpc.go -@@ -1206,32 +1206,28 @@ func (a *agentGRPC) StatsContainer(ctx context.Context, req *pb.StatsContainerRe - return nil, err - } - -+ agentLog.Debugf("container stats start : %s", c.container.ID()) - stats, err := c.container.Stats() - if err != nil { - return nil, err - } - -- cgroupData, err := json.Marshal(stats.CgroupStats) -+ networkStats, err := getNetworkStats() - if err != nil { - return nil, err - } - -- netData, err := json.Marshal(stats.Interfaces) -+ cgroupData, err := json.Marshal(stats.CgroupStats) - if err != nil { - return nil, err - } - - var cgroupStats pb.CgroupStats -- networkStats := make([]*pb.NetworkStats, 0) -- - err = json.Unmarshal(cgroupData, &cgroupStats) - if err != nil { - return nil, err - } -- err = json.Unmarshal(netData, &networkStats) -- if err != nil { -- return nil, err -- } -+ - resp := &pb.StatsContainerResponse{ - CgroupStats: &cgroupStats, - NetworkStats: networkStats, -diff --git a/network_stats.go b/network_stats.go -new file mode 100644 -index 0000000..6954be1 ---- /dev/null -+++ b/network_stats.go -@@ -0,0 +1,241 @@ -+/* -+Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. -+SPDX-License-Identifier: Apache-2.0 -+Description: common functions -+Author: yanlei -+Create: 2019-06-27 -+*/ -+ -+package main -+ -+import ( -+ "bufio" -+ "fmt" -+ "io/ioutil" -+ "net" -+ "path" -+ "strconv" -+ "strings" -+ -+ pb "github.com/kata-containers/agent/protocols/grpc" -+) -+ -+func getNetworkStats() (*pb.NetworkStats, error) { -+ networkStats := &pb.NetworkStats{} -+ ifaces, err := net.Interfaces() -+ if err != nil { -+ return nil, err -+ } -+ for _, iface := range ifaces { -+ ifaceStats, err := getNetworkInterfaceStats(iface.Name) -+ if err != nil { -+ return nil, err -+ } -+ networkStats.Interfaces = append(networkStats.Interfaces, ifaceStats) -+ } -+ -+ t, err := getNetworkTcpStats("tcp") -+ if err != nil { -+ return nil, err -+ } -+ networkStats.Tcp = t -+ -+ t6, err := getNetworkTcpStats("tcp6") -+ if err != nil { -+ return nil, err -+ } -+ networkStats.Tcp6 = t6 -+ -+ u, err := getNetworkUdpStats("udp") -+ if err != nil { -+ return nil, err -+ } -+ networkStats.Udp = u -+ -+ u6, err := getNetworkUdpStats("udp6") -+ if err != nil { -+ return nil, err -+ } -+ networkStats.Udp6 = u6 -+ -+ return networkStats, nil -+} -+ -+func getNetworkInterfaceStats(interfaceName string) (*pb.InterfaceStats, error) { -+ out := &pb.InterfaceStats{Name: interfaceName} -+ // This can happen if the network runtime information is missing - possible if the -+ // container was created by an old version of libcontainer. -+ if interfaceName == "" { -+ return out, nil -+ } -+ type netStatsPair struct { -+ // Where to write the output. -+ Out *uint64 -+ // The network stats file to read. -+ File string -+ } -+ // Ingress for host veth is from the container. Hence tx_bytes stat on the host veth is actually number of bytes received by the container. -+ netStats := []netStatsPair{ -+ {Out: &out.RxBytes, File: "rx_bytes"}, -+ {Out: &out.RxPackets, File: "rx_packets"}, -+ {Out: &out.RxErrors, File: "rx_errors"}, -+ {Out: &out.RxDropped, File: "rx_dropped"}, -+ -+ {Out: &out.TxBytes, File: "tx_bytes"}, -+ {Out: &out.TxPackets, File: "tx_packets"}, -+ {Out: &out.TxErrors, File: "tx_errors"}, -+ {Out: &out.TxDropped, File: "tx_dropped"}, -+ } -+ for _, netStat := range netStats { -+ data, err := readSysfsNetworkStats(interfaceName, netStat.File) -+ if err != nil { -+ return nil, err -+ } -+ *(netStat.Out) = data -+ } -+ return out, nil -+} -+ -+// Reads the specified statistics available under /sys/class/net//statistics -+func readSysfsNetworkStats(ethInterface, statsFile string) (uint64, error) { -+ data, err := ioutil.ReadFile(path.Join("/sys/class/net", ethInterface, "statistics", statsFile)) -+ if err != nil { -+ return 0, err -+ } -+ return strconv.ParseUint(strings.TrimSpace(string(data)), 10, 64) -+} -+ -+func getNetworkTcpStats(file string) (*pb.TcpStat, error) { -+ tcpStatsFile := path.Join("/proc/net", file) -+ -+ tcpStats, err := scanTcpStats(tcpStatsFile) -+ if err != nil { -+ return nil, err -+ } -+ -+ return tcpStats, nil -+} -+ -+func scanTcpStats(tcpStatsFile string) (*pb.TcpStat, error) { -+ data, err := ioutil.ReadFile(tcpStatsFile) -+ if err != nil { -+ return nil, fmt.Errorf("fail to open %s: %v", tcpStatsFile, err) -+ } -+ -+ tcpStateMap := map[string]uint64{ -+ "01": 0, //ESTABLISHED -+ "02": 0, //SYN_SENT -+ "03": 0, //SYN_RECV -+ "04": 0, //FIN_WAIT1 -+ "05": 0, //FIN_WAIT2 -+ "06": 0, //TIME_WAIT -+ "07": 0, //CLOSE -+ "08": 0, //CLOSE_WAIT -+ "09": 0, //LAST_ACK -+ "0A": 0, //LISTEN -+ "0B": 0, //CLOSING -+ } -+ -+ reader := strings.NewReader(string(data)) -+ scanner := bufio.NewScanner(reader) -+ -+ scanner.Split(bufio.ScanLines) -+ -+ if b := scanner.Scan(); !b { -+ return nil, scanner.Err() -+ } -+ -+ for scanner.Scan() { -+ line := scanner.Text() -+ -+ state := strings.Fields(line) -+ // TCP state is the 4th field. -+ // Format: sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode -+ tcpState := state[3] -+ _, ok := tcpStateMap[tcpState] -+ if !ok { -+ return nil, fmt.Errorf("invalid TCP state line: %v", line) -+ } -+ tcpStateMap[tcpState]++ -+ } -+ -+ stats := &pb.TcpStat{ -+ Established: tcpStateMap["01"], -+ SynSent: tcpStateMap["02"], -+ SynRecv: tcpStateMap["03"], -+ FinWait1: tcpStateMap["04"], -+ FinWait2: tcpStateMap["05"], -+ TimeWait: tcpStateMap["06"], -+ Close: tcpStateMap["07"], -+ CloseWait: tcpStateMap["08"], -+ LastAck: tcpStateMap["09"], -+ Listen: tcpStateMap["0A"], -+ Closing: tcpStateMap["0B"], -+ } -+ -+ return stats, nil -+} -+ -+func getNetworkUdpStats(file string) (*pb.UdpStat, error) { -+ udpStatsFile := path.Join("/proc/net", file) -+ -+ udpStats, err := scanUdpStats(udpStatsFile) -+ if err != nil { -+ return nil, err -+ } -+ -+ return udpStats, nil -+} -+ -+func scanUdpStats(udpStatsFile string) (*pb.UdpStat, error) { -+ data, err := ioutil.ReadFile(udpStatsFile) -+ if err != nil { -+ return nil, fmt.Errorf("fail to open %s: %v", udpStatsFile, err) -+ } -+ -+ reader := strings.NewReader(string(data)) -+ scanner := bufio.NewScanner(reader) -+ -+ scanner.Split(bufio.ScanLines) -+ -+ if b := scanner.Scan(); !b { -+ return nil, scanner.Err() -+ } -+ -+ listening := uint64(0) -+ dropped := uint64(0) -+ rxQueued := uint64(0) -+ txQueued := uint64(0) -+ -+ for scanner.Scan() { -+ line := scanner.Text() -+ // Format: sl local_address rem_address st tx_queue:rx_queue tr:tm->when retrnsmt uid timeout inode ref pointer drops -+ -+ listening++ -+ -+ fs := strings.Fields(line) -+ if len(fs) != 13 { -+ continue -+ } -+ -+ rx, tx := uint64(0), uint64(0) -+ fmt.Sscanf(fs[4], "%X:%X", &rx, &tx) -+ rxQueued += rx -+ txQueued += tx -+ -+ d, err := strconv.Atoi(string(fs[12])) -+ if err != nil { -+ continue -+ } -+ dropped += uint64(d) -+ } -+ -+ stats := &pb.UdpStat{ -+ Listen: listening, -+ Dropped: dropped, -+ RxQueued: rxQueued, -+ TxQueued: txQueued, -+ } -+ -+ return stats, nil -+} -diff --git a/protocols/grpc/agent.pb.go b/protocols/grpc/agent.pb.go -index 04d0ee5..c50ecb5 100644 ---- a/protocols/grpc/agent.pb.go -+++ b/protocols/grpc/agent.pb.go -@@ -33,6 +33,9 @@ - BlkioStats - HugetlbStats - CgroupStats -+ InterfaceStats -+ TcpStat -+ UdpStat - NetworkStats - StatsContainerResponse - WriteStreamRequest -@@ -883,7 +886,7 @@ func (m *CgroupStats) GetHugetlbStats() map[string]*HugetlbStats { - return nil - } - --type NetworkStats struct { -+type InterfaceStats struct { - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - RxBytes uint64 `protobuf:"varint,2,opt,name=rx_bytes,json=rxBytes,proto3" json:"rx_bytes,omitempty"` - RxPackets uint64 `protobuf:"varint,3,opt,name=rx_packets,json=rxPackets,proto3" json:"rx_packets,omitempty"` -@@ -895,83 +898,267 @@ type NetworkStats struct { - TxDropped uint64 `protobuf:"varint,9,opt,name=tx_dropped,json=txDropped,proto3" json:"tx_dropped,omitempty"` - } - --func (m *NetworkStats) Reset() { *m = NetworkStats{} } --func (m *NetworkStats) String() string { return proto.CompactTextString(m) } --func (*NetworkStats) ProtoMessage() {} --func (*NetworkStats) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{23} } -+func (m *InterfaceStats) Reset() { *m = InterfaceStats{} } -+func (m *InterfaceStats) String() string { return proto.CompactTextString(m) } -+func (*InterfaceStats) ProtoMessage() {} -+func (*InterfaceStats) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{23} } - --func (m *NetworkStats) GetName() string { -+func (m *InterfaceStats) GetName() string { - if m != nil { - return m.Name - } - return "" - } - --func (m *NetworkStats) GetRxBytes() uint64 { -+func (m *InterfaceStats) GetRxBytes() uint64 { - if m != nil { - return m.RxBytes - } - return 0 - } - --func (m *NetworkStats) GetRxPackets() uint64 { -+func (m *InterfaceStats) GetRxPackets() uint64 { - if m != nil { - return m.RxPackets - } - return 0 - } - --func (m *NetworkStats) GetRxErrors() uint64 { -+func (m *InterfaceStats) GetRxErrors() uint64 { - if m != nil { - return m.RxErrors - } - return 0 - } - --func (m *NetworkStats) GetRxDropped() uint64 { -+func (m *InterfaceStats) GetRxDropped() uint64 { - if m != nil { - return m.RxDropped - } - return 0 - } - --func (m *NetworkStats) GetTxBytes() uint64 { -+func (m *InterfaceStats) GetTxBytes() uint64 { - if m != nil { - return m.TxBytes - } - return 0 - } - --func (m *NetworkStats) GetTxPackets() uint64 { -+func (m *InterfaceStats) GetTxPackets() uint64 { - if m != nil { - return m.TxPackets - } - return 0 - } - --func (m *NetworkStats) GetTxErrors() uint64 { -+func (m *InterfaceStats) GetTxErrors() uint64 { - if m != nil { - return m.TxErrors - } - return 0 - } - --func (m *NetworkStats) GetTxDropped() uint64 { -+func (m *InterfaceStats) GetTxDropped() uint64 { - if m != nil { - return m.TxDropped - } - return 0 - } - -+type TcpStat struct { -+ Established uint64 `protobuf:"varint,1,opt,name=established,proto3" json:"established,omitempty"` -+ SynSent uint64 `protobuf:"varint,2,opt,name=syn_sent,json=synSent,proto3" json:"syn_sent,omitempty"` -+ SynRecv uint64 `protobuf:"varint,3,opt,name=syn_recv,json=synRecv,proto3" json:"syn_recv,omitempty"` -+ FinWait1 uint64 `protobuf:"varint,4,opt,name=fin_wait1,json=finWait1,proto3" json:"fin_wait1,omitempty"` -+ FinWait2 uint64 `protobuf:"varint,5,opt,name=fin_wait2,json=finWait2,proto3" json:"fin_wait2,omitempty"` -+ TimeWait uint64 `protobuf:"varint,6,opt,name=time_wait,json=timeWait,proto3" json:"time_wait,omitempty"` -+ Close uint64 `protobuf:"varint,7,opt,name=close,proto3" json:"close,omitempty"` -+ CloseWait uint64 `protobuf:"varint,8,opt,name=close_wait,json=closeWait,proto3" json:"close_wait,omitempty"` -+ LastAck uint64 `protobuf:"varint,9,opt,name=last_ack,json=lastAck,proto3" json:"last_ack,omitempty"` -+ Listen uint64 `protobuf:"varint,10,opt,name=listen,proto3" json:"listen,omitempty"` -+ Closing uint64 `protobuf:"varint,11,opt,name=closing,proto3" json:"closing,omitempty"` -+} -+ -+func (m *TcpStat) Reset() { *m = TcpStat{} } -+func (m *TcpStat) String() string { return proto.CompactTextString(m) } -+func (*TcpStat) ProtoMessage() {} -+func (*TcpStat) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{24} } -+ -+func (m *TcpStat) GetEstablished() uint64 { -+ if m != nil { -+ return m.Established -+ } -+ return 0 -+} -+ -+func (m *TcpStat) GetSynSent() uint64 { -+ if m != nil { -+ return m.SynSent -+ } -+ return 0 -+} -+ -+func (m *TcpStat) GetSynRecv() uint64 { -+ if m != nil { -+ return m.SynRecv -+ } -+ return 0 -+} -+ -+func (m *TcpStat) GetFinWait1() uint64 { -+ if m != nil { -+ return m.FinWait1 -+ } -+ return 0 -+} -+ -+func (m *TcpStat) GetFinWait2() uint64 { -+ if m != nil { -+ return m.FinWait2 -+ } -+ return 0 -+} -+ -+func (m *TcpStat) GetTimeWait() uint64 { -+ if m != nil { -+ return m.TimeWait -+ } -+ return 0 -+} -+ -+func (m *TcpStat) GetClose() uint64 { -+ if m != nil { -+ return m.Close -+ } -+ return 0 -+} -+ -+func (m *TcpStat) GetCloseWait() uint64 { -+ if m != nil { -+ return m.CloseWait -+ } -+ return 0 -+} -+ -+func (m *TcpStat) GetLastAck() uint64 { -+ if m != nil { -+ return m.LastAck -+ } -+ return 0 -+} -+ -+func (m *TcpStat) GetListen() uint64 { -+ if m != nil { -+ return m.Listen -+ } -+ return 0 -+} -+ -+func (m *TcpStat) GetClosing() uint64 { -+ if m != nil { -+ return m.Closing -+ } -+ return 0 -+} -+ -+type UdpStat struct { -+ Listen uint64 `protobuf:"varint,1,opt,name=listen,proto3" json:"listen,omitempty"` -+ Dropped uint64 `protobuf:"varint,2,opt,name=dropped,proto3" json:"dropped,omitempty"` -+ RxQueued uint64 `protobuf:"varint,3,opt,name=rx_queued,json=rxQueued,proto3" json:"rx_queued,omitempty"` -+ TxQueued uint64 `protobuf:"varint,4,opt,name=tx_queued,json=txQueued,proto3" json:"tx_queued,omitempty"` -+} -+ -+func (m *UdpStat) Reset() { *m = UdpStat{} } -+func (m *UdpStat) String() string { return proto.CompactTextString(m) } -+func (*UdpStat) ProtoMessage() {} -+func (*UdpStat) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{25} } -+ -+func (m *UdpStat) GetListen() uint64 { -+ if m != nil { -+ return m.Listen -+ } -+ return 0 -+} -+ -+func (m *UdpStat) GetDropped() uint64 { -+ if m != nil { -+ return m.Dropped -+ } -+ return 0 -+} -+ -+func (m *UdpStat) GetRxQueued() uint64 { -+ if m != nil { -+ return m.RxQueued -+ } -+ return 0 -+} -+ -+func (m *UdpStat) GetTxQueued() uint64 { -+ if m != nil { -+ return m.TxQueued -+ } -+ return 0 -+} -+ -+type NetworkStats struct { -+ Interfaces []*InterfaceStats `protobuf:"bytes,1,rep,name=interfaces" json:"interfaces,omitempty"` -+ Tcp *TcpStat `protobuf:"bytes,2,opt,name=tcp" json:"tcp,omitempty"` -+ Tcp6 *TcpStat `protobuf:"bytes,3,opt,name=tcp6" json:"tcp6,omitempty"` -+ Udp *UdpStat `protobuf:"bytes,4,opt,name=udp" json:"udp,omitempty"` -+ Udp6 *UdpStat `protobuf:"bytes,5,opt,name=udp6" json:"udp6,omitempty"` -+} -+ -+func (m *NetworkStats) Reset() { *m = NetworkStats{} } -+func (m *NetworkStats) String() string { return proto.CompactTextString(m) } -+func (*NetworkStats) ProtoMessage() {} -+func (*NetworkStats) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{26} } -+ -+func (m *NetworkStats) GetInterfaces() []*InterfaceStats { -+ if m != nil { -+ return m.Interfaces -+ } -+ return nil -+} -+ -+func (m *NetworkStats) GetTcp() *TcpStat { -+ if m != nil { -+ return m.Tcp -+ } -+ return nil -+} -+ -+func (m *NetworkStats) GetTcp6() *TcpStat { -+ if m != nil { -+ return m.Tcp6 -+ } -+ return nil -+} -+ -+func (m *NetworkStats) GetUdp() *UdpStat { -+ if m != nil { -+ return m.Udp -+ } -+ return nil -+} -+ -+func (m *NetworkStats) GetUdp6() *UdpStat { -+ if m != nil { -+ return m.Udp6 -+ } -+ return nil -+} -+ - type StatsContainerResponse struct { -- CgroupStats *CgroupStats `protobuf:"bytes,1,opt,name=cgroup_stats,json=cgroupStats" json:"cgroup_stats,omitempty"` -- NetworkStats []*NetworkStats `protobuf:"bytes,2,rep,name=network_stats,json=networkStats" json:"network_stats,omitempty"` -+ CgroupStats *CgroupStats `protobuf:"bytes,1,opt,name=cgroup_stats,json=cgroupStats" json:"cgroup_stats,omitempty"` -+ NetworkStats *NetworkStats `protobuf:"bytes,2,opt,name=network_stats,json=networkStats" json:"network_stats,omitempty"` - } - - func (m *StatsContainerResponse) Reset() { *m = StatsContainerResponse{} } - func (m *StatsContainerResponse) String() string { return proto.CompactTextString(m) } - func (*StatsContainerResponse) ProtoMessage() {} --func (*StatsContainerResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{24} } -+func (*StatsContainerResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{27} } - - func (m *StatsContainerResponse) GetCgroupStats() *CgroupStats { - if m != nil { -@@ -980,7 +1167,7 @@ func (m *StatsContainerResponse) GetCgroupStats() *CgroupStats { - return nil - } - --func (m *StatsContainerResponse) GetNetworkStats() []*NetworkStats { -+func (m *StatsContainerResponse) GetNetworkStats() *NetworkStats { - if m != nil { - return m.NetworkStats - } -@@ -996,7 +1183,7 @@ type WriteStreamRequest struct { - func (m *WriteStreamRequest) Reset() { *m = WriteStreamRequest{} } - func (m *WriteStreamRequest) String() string { return proto.CompactTextString(m) } - func (*WriteStreamRequest) ProtoMessage() {} --func (*WriteStreamRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{25} } -+func (*WriteStreamRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{28} } - - func (m *WriteStreamRequest) GetContainerId() string { - if m != nil { -@@ -1026,7 +1213,7 @@ type WriteStreamResponse struct { - func (m *WriteStreamResponse) Reset() { *m = WriteStreamResponse{} } - func (m *WriteStreamResponse) String() string { return proto.CompactTextString(m) } - func (*WriteStreamResponse) ProtoMessage() {} --func (*WriteStreamResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{26} } -+func (*WriteStreamResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{29} } - - func (m *WriteStreamResponse) GetLen() uint32 { - if m != nil { -@@ -1044,7 +1231,7 @@ type ReadStreamRequest struct { - func (m *ReadStreamRequest) Reset() { *m = ReadStreamRequest{} } - func (m *ReadStreamRequest) String() string { return proto.CompactTextString(m) } - func (*ReadStreamRequest) ProtoMessage() {} --func (*ReadStreamRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{27} } -+func (*ReadStreamRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{30} } - - func (m *ReadStreamRequest) GetContainerId() string { - if m != nil { -@@ -1074,7 +1261,7 @@ type ReadStreamResponse struct { - func (m *ReadStreamResponse) Reset() { *m = ReadStreamResponse{} } - func (m *ReadStreamResponse) String() string { return proto.CompactTextString(m) } - func (*ReadStreamResponse) ProtoMessage() {} --func (*ReadStreamResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{28} } -+func (*ReadStreamResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{31} } - - func (m *ReadStreamResponse) GetData() []byte { - if m != nil { -@@ -1091,7 +1278,7 @@ type CloseStdinRequest struct { - func (m *CloseStdinRequest) Reset() { *m = CloseStdinRequest{} } - func (m *CloseStdinRequest) String() string { return proto.CompactTextString(m) } - func (*CloseStdinRequest) ProtoMessage() {} --func (*CloseStdinRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{29} } -+func (*CloseStdinRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{32} } - - func (m *CloseStdinRequest) GetContainerId() string { - if m != nil { -@@ -1117,7 +1304,7 @@ type TtyWinResizeRequest struct { - func (m *TtyWinResizeRequest) Reset() { *m = TtyWinResizeRequest{} } - func (m *TtyWinResizeRequest) String() string { return proto.CompactTextString(m) } - func (*TtyWinResizeRequest) ProtoMessage() {} --func (*TtyWinResizeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{30} } -+func (*TtyWinResizeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{33} } - - func (m *TtyWinResizeRequest) GetContainerId() string { - if m != nil { -@@ -1158,7 +1345,7 @@ type KernelModule struct { - func (m *KernelModule) Reset() { *m = KernelModule{} } - func (m *KernelModule) String() string { return proto.CompactTextString(m) } - func (*KernelModule) ProtoMessage() {} --func (*KernelModule) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{31} } -+func (*KernelModule) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{34} } - - func (m *KernelModule) GetName() string { - if m != nil { -@@ -1197,7 +1384,7 @@ type CreateSandboxRequest struct { - func (m *CreateSandboxRequest) Reset() { *m = CreateSandboxRequest{} } - func (m *CreateSandboxRequest) String() string { return proto.CompactTextString(m) } - func (*CreateSandboxRequest) ProtoMessage() {} --func (*CreateSandboxRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{32} } -+func (*CreateSandboxRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{35} } - - func (m *CreateSandboxRequest) GetHostname() string { - if m != nil { -@@ -1254,7 +1441,7 @@ type DestroySandboxRequest struct { - func (m *DestroySandboxRequest) Reset() { *m = DestroySandboxRequest{} } - func (m *DestroySandboxRequest) String() string { return proto.CompactTextString(m) } - func (*DestroySandboxRequest) ProtoMessage() {} --func (*DestroySandboxRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{33} } -+func (*DestroySandboxRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{36} } - - type Interfaces struct { - Interfaces []*types.Interface `protobuf:"bytes,1,rep,name=Interfaces" json:"Interfaces,omitempty"` -@@ -1263,7 +1450,7 @@ type Interfaces struct { - func (m *Interfaces) Reset() { *m = Interfaces{} } - func (m *Interfaces) String() string { return proto.CompactTextString(m) } - func (*Interfaces) ProtoMessage() {} --func (*Interfaces) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{34} } -+func (*Interfaces) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{37} } - - func (m *Interfaces) GetInterfaces() []*types.Interface { - if m != nil { -@@ -1279,7 +1466,7 @@ type Routes struct { - func (m *Routes) Reset() { *m = Routes{} } - func (m *Routes) String() string { return proto.CompactTextString(m) } - func (*Routes) ProtoMessage() {} --func (*Routes) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{35} } -+func (*Routes) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{38} } - - func (m *Routes) GetRoutes() []*types.Route { - if m != nil { -@@ -1295,7 +1482,7 @@ type UpdateInterfaceRequest struct { - func (m *UpdateInterfaceRequest) Reset() { *m = UpdateInterfaceRequest{} } - func (m *UpdateInterfaceRequest) String() string { return proto.CompactTextString(m) } - func (*UpdateInterfaceRequest) ProtoMessage() {} --func (*UpdateInterfaceRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{36} } -+func (*UpdateInterfaceRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{39} } - - func (m *UpdateInterfaceRequest) GetInterface() *types.Interface { - if m != nil { -@@ -1312,7 +1499,7 @@ type UpdateRoutesRequest struct { - func (m *UpdateRoutesRequest) Reset() { *m = UpdateRoutesRequest{} } - func (m *UpdateRoutesRequest) String() string { return proto.CompactTextString(m) } - func (*UpdateRoutesRequest) ProtoMessage() {} --func (*UpdateRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{37} } -+func (*UpdateRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{40} } - - func (m *UpdateRoutesRequest) GetRoutes() *Routes { - if m != nil { -@@ -1334,7 +1521,7 @@ type ListInterfacesRequest struct { - func (m *ListInterfacesRequest) Reset() { *m = ListInterfacesRequest{} } - func (m *ListInterfacesRequest) String() string { return proto.CompactTextString(m) } - func (*ListInterfacesRequest) ProtoMessage() {} --func (*ListInterfacesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{38} } -+func (*ListInterfacesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{41} } - - type ListRoutesRequest struct { - } -@@ -1342,7 +1529,7 @@ type ListRoutesRequest struct { - func (m *ListRoutesRequest) Reset() { *m = ListRoutesRequest{} } - func (m *ListRoutesRequest) String() string { return proto.CompactTextString(m) } - func (*ListRoutesRequest) ProtoMessage() {} --func (*ListRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{39} } -+func (*ListRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{42} } - - type OnlineCPUMemRequest struct { - // Wait specifies if the caller waits for the agent to online all resources. -@@ -1358,7 +1545,7 @@ type OnlineCPUMemRequest struct { - func (m *OnlineCPUMemRequest) Reset() { *m = OnlineCPUMemRequest{} } - func (m *OnlineCPUMemRequest) String() string { return proto.CompactTextString(m) } - func (*OnlineCPUMemRequest) ProtoMessage() {} --func (*OnlineCPUMemRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{40} } -+func (*OnlineCPUMemRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{43} } - - func (m *OnlineCPUMemRequest) GetWait() bool { - if m != nil { -@@ -1389,7 +1576,7 @@ type ReseedRandomDevRequest struct { - func (m *ReseedRandomDevRequest) Reset() { *m = ReseedRandomDevRequest{} } - func (m *ReseedRandomDevRequest) String() string { return proto.CompactTextString(m) } - func (*ReseedRandomDevRequest) ProtoMessage() {} --func (*ReseedRandomDevRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{41} } -+func (*ReseedRandomDevRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{44} } - - func (m *ReseedRandomDevRequest) GetData() []byte { - if m != nil { -@@ -1416,7 +1603,7 @@ type AgentDetails struct { - func (m *AgentDetails) Reset() { *m = AgentDetails{} } - func (m *AgentDetails) String() string { return proto.CompactTextString(m) } - func (*AgentDetails) ProtoMessage() {} --func (*AgentDetails) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{42} } -+func (*AgentDetails) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{45} } - - func (m *AgentDetails) GetVersion() string { - if m != nil { -@@ -1467,7 +1654,7 @@ type GuestDetailsRequest struct { - func (m *GuestDetailsRequest) Reset() { *m = GuestDetailsRequest{} } - func (m *GuestDetailsRequest) String() string { return proto.CompactTextString(m) } - func (*GuestDetailsRequest) ProtoMessage() {} --func (*GuestDetailsRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{43} } -+func (*GuestDetailsRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{46} } - - func (m *GuestDetailsRequest) GetMemBlockSize() bool { - if m != nil { -@@ -1493,7 +1680,7 @@ type GuestDetailsResponse struct { - func (m *GuestDetailsResponse) Reset() { *m = GuestDetailsResponse{} } - func (m *GuestDetailsResponse) String() string { return proto.CompactTextString(m) } - func (*GuestDetailsResponse) ProtoMessage() {} --func (*GuestDetailsResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{44} } -+func (*GuestDetailsResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{47} } - - func (m *GuestDetailsResponse) GetMemBlockSizeBytes() uint64 { - if m != nil { -@@ -1525,7 +1712,7 @@ type MemHotplugByProbeRequest struct { - func (m *MemHotplugByProbeRequest) Reset() { *m = MemHotplugByProbeRequest{} } - func (m *MemHotplugByProbeRequest) String() string { return proto.CompactTextString(m) } - func (*MemHotplugByProbeRequest) ProtoMessage() {} --func (*MemHotplugByProbeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{45} } -+func (*MemHotplugByProbeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{48} } - - func (m *MemHotplugByProbeRequest) GetMemHotplugProbeAddr() []uint64 { - if m != nil { -@@ -1544,7 +1731,7 @@ type SetGuestDateTimeRequest struct { - func (m *SetGuestDateTimeRequest) Reset() { *m = SetGuestDateTimeRequest{} } - func (m *SetGuestDateTimeRequest) String() string { return proto.CompactTextString(m) } - func (*SetGuestDateTimeRequest) ProtoMessage() {} --func (*SetGuestDateTimeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{46} } -+func (*SetGuestDateTimeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{49} } - - func (m *SetGuestDateTimeRequest) GetSec() int64 { - if m != nil { -@@ -1593,7 +1780,7 @@ type Storage struct { - func (m *Storage) Reset() { *m = Storage{} } - func (m *Storage) String() string { return proto.CompactTextString(m) } - func (*Storage) ProtoMessage() {} --func (*Storage) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{47} } -+func (*Storage) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{50} } - - func (m *Storage) GetDriver() string { - if m != nil { -@@ -1676,7 +1863,7 @@ type Device struct { - func (m *Device) Reset() { *m = Device{} } - func (m *Device) String() string { return proto.CompactTextString(m) } - func (*Device) ProtoMessage() {} --func (*Device) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{48} } -+func (*Device) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{51} } - - func (m *Device) GetId() string { - if m != nil { -@@ -1722,7 +1909,7 @@ type StringUser struct { - func (m *StringUser) Reset() { *m = StringUser{} } - func (m *StringUser) String() string { return proto.CompactTextString(m) } - func (*StringUser) ProtoMessage() {} --func (*StringUser) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{49} } -+func (*StringUser) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{52} } - - func (m *StringUser) GetUid() string { - if m != nil { -@@ -1770,7 +1957,7 @@ type CopyFileRequest struct { - func (m *CopyFileRequest) Reset() { *m = CopyFileRequest{} } - func (m *CopyFileRequest) String() string { return proto.CompactTextString(m) } - func (*CopyFileRequest) ProtoMessage() {} --func (*CopyFileRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{50} } -+func (*CopyFileRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{53} } - - func (m *CopyFileRequest) GetPath() string { - if m != nil { -@@ -1834,7 +2021,7 @@ type StartTracingRequest struct { - func (m *StartTracingRequest) Reset() { *m = StartTracingRequest{} } - func (m *StartTracingRequest) String() string { return proto.CompactTextString(m) } - func (*StartTracingRequest) ProtoMessage() {} --func (*StartTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{51} } -+func (*StartTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{54} } - - type StopTracingRequest struct { - } -@@ -1842,7 +2029,7 @@ type StopTracingRequest struct { - func (m *StopTracingRequest) Reset() { *m = StopTracingRequest{} } - func (m *StopTracingRequest) String() string { return proto.CompactTextString(m) } - func (*StopTracingRequest) ProtoMessage() {} --func (*StopTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{52} } -+func (*StopTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{55} } - - type UpdateIPVSRequest struct { - // IPVS_req is the IPVS rule message needed to update -@@ -1852,7 +2039,7 @@ type UpdateIPVSRequest struct { - func (m *UpdateIPVSRequest) Reset() { *m = UpdateIPVSRequest{} } - func (m *UpdateIPVSRequest) String() string { return proto.CompactTextString(m) } - func (*UpdateIPVSRequest) ProtoMessage() {} --func (*UpdateIPVSRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{53} } -+func (*UpdateIPVSRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{56} } - - func (m *UpdateIPVSRequest) GetIPVSReq() string { - if m != nil { -@@ -1869,7 +2056,7 @@ type IPVSResponse struct { - func (m *IPVSResponse) Reset() { *m = IPVSResponse{} } - func (m *IPVSResponse) String() string { return proto.CompactTextString(m) } - func (*IPVSResponse) ProtoMessage() {} --func (*IPVSResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{54} } -+func (*IPVSResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{57} } - - func (m *IPVSResponse) GetIPVSRes() string { - if m != nil { -@@ -1902,6 +2089,9 @@ func init() { - proto.RegisterType((*BlkioStats)(nil), "grpc.BlkioStats") - proto.RegisterType((*HugetlbStats)(nil), "grpc.HugetlbStats") - proto.RegisterType((*CgroupStats)(nil), "grpc.CgroupStats") -+ proto.RegisterType((*InterfaceStats)(nil), "grpc.InterfaceStats") -+ proto.RegisterType((*TcpStat)(nil), "grpc.TcpStat") -+ proto.RegisterType((*UdpStat)(nil), "grpc.UdpStat") - proto.RegisterType((*NetworkStats)(nil), "grpc.NetworkStats") - proto.RegisterType((*StatsContainerResponse)(nil), "grpc.StatsContainerResponse") - proto.RegisterType((*WriteStreamRequest)(nil), "grpc.WriteStreamRequest") -@@ -4008,7 +4198,7 @@ func (m *CgroupStats) MarshalTo(dAtA []byte) (int, error) { - return i, nil - } - --func (m *NetworkStats) Marshal() (dAtA []byte, err error) { -+func (m *InterfaceStats) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) -@@ -4018,7 +4208,7 @@ func (m *NetworkStats) Marshal() (dAtA []byte, err error) { - return dAtA[:n], nil - } - --func (m *NetworkStats) MarshalTo(dAtA []byte) (int, error) { -+func (m *InterfaceStats) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int -@@ -4072,7 +4262,7 @@ func (m *NetworkStats) MarshalTo(dAtA []byte) (int, error) { - return i, nil - } - --func (m *StatsContainerResponse) Marshal() (dAtA []byte, err error) { -+func (m *TcpStat) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) -@@ -4082,37 +4272,70 @@ func (m *StatsContainerResponse) Marshal() (dAtA []byte, err error) { - return dAtA[:n], nil - } - --func (m *StatsContainerResponse) MarshalTo(dAtA []byte) (int, error) { -+func (m *TcpStat) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l -- if m.CgroupStats != nil { -- dAtA[i] = 0xa -+ if m.Established != 0 { -+ dAtA[i] = 0x8 - i++ -- i = encodeVarintAgent(dAtA, i, uint64(m.CgroupStats.Size())) -- n18, err := m.CgroupStats.MarshalTo(dAtA[i:]) -- if err != nil { -- return 0, err -- } -- i += n18 -+ i = encodeVarintAgent(dAtA, i, uint64(m.Established)) - } -- if len(m.NetworkStats) > 0 { -- for _, msg := range m.NetworkStats { -- dAtA[i] = 0x12 -- i++ -- i = encodeVarintAgent(dAtA, i, uint64(msg.Size())) -- n, err := msg.MarshalTo(dAtA[i:]) -- if err != nil { -- return 0, err -- } -- i += n -- } -+ if m.SynSent != 0 { -+ dAtA[i] = 0x10 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.SynSent)) -+ } -+ if m.SynRecv != 0 { -+ dAtA[i] = 0x18 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.SynRecv)) -+ } -+ if m.FinWait1 != 0 { -+ dAtA[i] = 0x20 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.FinWait1)) -+ } -+ if m.FinWait2 != 0 { -+ dAtA[i] = 0x28 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.FinWait2)) -+ } -+ if m.TimeWait != 0 { -+ dAtA[i] = 0x30 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.TimeWait)) -+ } -+ if m.Close != 0 { -+ dAtA[i] = 0x38 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.Close)) -+ } -+ if m.CloseWait != 0 { -+ dAtA[i] = 0x40 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.CloseWait)) -+ } -+ if m.LastAck != 0 { -+ dAtA[i] = 0x48 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.LastAck)) -+ } -+ if m.Listen != 0 { -+ dAtA[i] = 0x50 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.Listen)) -+ } -+ if m.Closing != 0 { -+ dAtA[i] = 0x58 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.Closing)) - } - return i, nil - } - --func (m *WriteStreamRequest) Marshal() (dAtA []byte, err error) { -+func (m *UdpStat) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) -@@ -4122,33 +4345,35 @@ func (m *WriteStreamRequest) Marshal() (dAtA []byte, err error) { - return dAtA[:n], nil - } - --func (m *WriteStreamRequest) MarshalTo(dAtA []byte) (int, error) { -+func (m *UdpStat) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l -- if len(m.ContainerId) > 0 { -- dAtA[i] = 0xa -+ if m.Listen != 0 { -+ dAtA[i] = 0x8 - i++ -- i = encodeVarintAgent(dAtA, i, uint64(len(m.ContainerId))) -- i += copy(dAtA[i:], m.ContainerId) -+ i = encodeVarintAgent(dAtA, i, uint64(m.Listen)) - } -- if len(m.ExecId) > 0 { -- dAtA[i] = 0x12 -+ if m.Dropped != 0 { -+ dAtA[i] = 0x10 - i++ -- i = encodeVarintAgent(dAtA, i, uint64(len(m.ExecId))) -- i += copy(dAtA[i:], m.ExecId) -+ i = encodeVarintAgent(dAtA, i, uint64(m.Dropped)) - } -- if len(m.Data) > 0 { -- dAtA[i] = 0x1a -+ if m.RxQueued != 0 { -+ dAtA[i] = 0x18 - i++ -- i = encodeVarintAgent(dAtA, i, uint64(len(m.Data))) -- i += copy(dAtA[i:], m.Data) -+ i = encodeVarintAgent(dAtA, i, uint64(m.RxQueued)) -+ } -+ if m.TxQueued != 0 { -+ dAtA[i] = 0x20 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.TxQueued)) - } - return i, nil - } - --func (m *WriteStreamResponse) Marshal() (dAtA []byte, err error) { -+func (m *NetworkStats) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) -@@ -4158,20 +4383,67 @@ func (m *WriteStreamResponse) Marshal() (dAtA []byte, err error) { - return dAtA[:n], nil - } - --func (m *WriteStreamResponse) MarshalTo(dAtA []byte) (int, error) { -+func (m *NetworkStats) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l -- if m.Len != 0 { -- dAtA[i] = 0x8 -+ if len(m.Interfaces) > 0 { -+ for _, msg := range m.Interfaces { -+ dAtA[i] = 0xa -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(msg.Size())) -+ n, err := msg.MarshalTo(dAtA[i:]) -+ if err != nil { -+ return 0, err -+ } -+ i += n -+ } -+ } -+ if m.Tcp != nil { -+ dAtA[i] = 0x12 - i++ -- i = encodeVarintAgent(dAtA, i, uint64(m.Len)) -+ i = encodeVarintAgent(dAtA, i, uint64(m.Tcp.Size())) -+ n18, err := m.Tcp.MarshalTo(dAtA[i:]) -+ if err != nil { -+ return 0, err -+ } -+ i += n18 -+ } -+ if m.Tcp6 != nil { -+ dAtA[i] = 0x1a -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.Tcp6.Size())) -+ n19, err := m.Tcp6.MarshalTo(dAtA[i:]) -+ if err != nil { -+ return 0, err -+ } -+ i += n19 -+ } -+ if m.Udp != nil { -+ dAtA[i] = 0x22 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.Udp.Size())) -+ n20, err := m.Udp.MarshalTo(dAtA[i:]) -+ if err != nil { -+ return 0, err -+ } -+ i += n20 -+ } -+ if m.Udp6 != nil { -+ dAtA[i] = 0x2a -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.Udp6.Size())) -+ n21, err := m.Udp6.MarshalTo(dAtA[i:]) -+ if err != nil { -+ return 0, err -+ } -+ i += n21 - } - return i, nil - } - --func (m *ReadStreamRequest) Marshal() (dAtA []byte, err error) { -+func (m *StatsContainerResponse) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) -@@ -4181,32 +4453,129 @@ func (m *ReadStreamRequest) Marshal() (dAtA []byte, err error) { - return dAtA[:n], nil - } - --func (m *ReadStreamRequest) MarshalTo(dAtA []byte) (int, error) { -+func (m *StatsContainerResponse) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l -- if len(m.ContainerId) > 0 { -+ if m.CgroupStats != nil { - dAtA[i] = 0xa - i++ -- i = encodeVarintAgent(dAtA, i, uint64(len(m.ContainerId))) -- i += copy(dAtA[i:], m.ContainerId) -+ i = encodeVarintAgent(dAtA, i, uint64(m.CgroupStats.Size())) -+ n22, err := m.CgroupStats.MarshalTo(dAtA[i:]) -+ if err != nil { -+ return 0, err -+ } -+ i += n22 - } -- if len(m.ExecId) > 0 { -+ if m.NetworkStats != nil { - dAtA[i] = 0x12 - i++ -- i = encodeVarintAgent(dAtA, i, uint64(len(m.ExecId))) -- i += copy(dAtA[i:], m.ExecId) -- } -- if m.Len != 0 { -- dAtA[i] = 0x18 -- i++ -- i = encodeVarintAgent(dAtA, i, uint64(m.Len)) -+ i = encodeVarintAgent(dAtA, i, uint64(m.NetworkStats.Size())) -+ n23, err := m.NetworkStats.MarshalTo(dAtA[i:]) -+ if err != nil { -+ return 0, err -+ } -+ i += n23 - } - return i, nil - } - --func (m *ReadStreamResponse) Marshal() (dAtA []byte, err error) { -+func (m *WriteStreamRequest) Marshal() (dAtA []byte, err error) { -+ size := m.Size() -+ dAtA = make([]byte, size) -+ n, err := m.MarshalTo(dAtA) -+ if err != nil { -+ return nil, err -+ } -+ return dAtA[:n], nil -+} -+ -+func (m *WriteStreamRequest) MarshalTo(dAtA []byte) (int, error) { -+ var i int -+ _ = i -+ var l int -+ _ = l -+ if len(m.ContainerId) > 0 { -+ dAtA[i] = 0xa -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(len(m.ContainerId))) -+ i += copy(dAtA[i:], m.ContainerId) -+ } -+ if len(m.ExecId) > 0 { -+ dAtA[i] = 0x12 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(len(m.ExecId))) -+ i += copy(dAtA[i:], m.ExecId) -+ } -+ if len(m.Data) > 0 { -+ dAtA[i] = 0x1a -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(len(m.Data))) -+ i += copy(dAtA[i:], m.Data) -+ } -+ return i, nil -+} -+ -+func (m *WriteStreamResponse) Marshal() (dAtA []byte, err error) { -+ size := m.Size() -+ dAtA = make([]byte, size) -+ n, err := m.MarshalTo(dAtA) -+ if err != nil { -+ return nil, err -+ } -+ return dAtA[:n], nil -+} -+ -+func (m *WriteStreamResponse) MarshalTo(dAtA []byte) (int, error) { -+ var i int -+ _ = i -+ var l int -+ _ = l -+ if m.Len != 0 { -+ dAtA[i] = 0x8 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.Len)) -+ } -+ return i, nil -+} -+ -+func (m *ReadStreamRequest) Marshal() (dAtA []byte, err error) { -+ size := m.Size() -+ dAtA = make([]byte, size) -+ n, err := m.MarshalTo(dAtA) -+ if err != nil { -+ return nil, err -+ } -+ return dAtA[:n], nil -+} -+ -+func (m *ReadStreamRequest) MarshalTo(dAtA []byte) (int, error) { -+ var i int -+ _ = i -+ var l int -+ _ = l -+ if len(m.ContainerId) > 0 { -+ dAtA[i] = 0xa -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(len(m.ContainerId))) -+ i += copy(dAtA[i:], m.ContainerId) -+ } -+ if len(m.ExecId) > 0 { -+ dAtA[i] = 0x12 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(len(m.ExecId))) -+ i += copy(dAtA[i:], m.ExecId) -+ } -+ if m.Len != 0 { -+ dAtA[i] = 0x18 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.Len)) -+ } -+ return i, nil -+} -+ -+func (m *ReadStreamResponse) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) -@@ -4521,11 +4890,11 @@ func (m *UpdateInterfaceRequest) MarshalTo(dAtA []byte) (int, error) { - dAtA[i] = 0xa - i++ - i = encodeVarintAgent(dAtA, i, uint64(m.Interface.Size())) -- n19, err := m.Interface.MarshalTo(dAtA[i:]) -+ n24, err := m.Interface.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } -- i += n19 -+ i += n24 - } - return i, nil - } -@@ -4549,11 +4918,11 @@ func (m *UpdateRoutesRequest) MarshalTo(dAtA []byte) (int, error) { - dAtA[i] = 0xa - i++ - i = encodeVarintAgent(dAtA, i, uint64(m.Routes.Size())) -- n20, err := m.Routes.MarshalTo(dAtA[i:]) -+ n25, err := m.Routes.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } -- i += n20 -+ i += n25 - } - if m.Increment { - dAtA[i] = 0x10 -@@ -4807,11 +5176,11 @@ func (m *GuestDetailsResponse) MarshalTo(dAtA []byte) (int, error) { - dAtA[i] = 0x12 - i++ - i = encodeVarintAgent(dAtA, i, uint64(m.AgentDetails.Size())) -- n21, err := m.AgentDetails.MarshalTo(dAtA[i:]) -+ n26, err := m.AgentDetails.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } -- i += n21 -+ i += n26 - } - if m.SupportMemHotplugProbe { - dAtA[i] = 0x18 -@@ -4842,21 +5211,21 @@ func (m *MemHotplugByProbeRequest) MarshalTo(dAtA []byte) (int, error) { - var l int - _ = l - if len(m.MemHotplugProbeAddr) > 0 { -- dAtA23 := make([]byte, len(m.MemHotplugProbeAddr)*10) -- var j22 int -+ dAtA28 := make([]byte, len(m.MemHotplugProbeAddr)*10) -+ var j27 int - for _, num := range m.MemHotplugProbeAddr { - for num >= 1<<7 { -- dAtA23[j22] = uint8(uint64(num)&0x7f | 0x80) -+ dAtA28[j27] = uint8(uint64(num)&0x7f | 0x80) - num >>= 7 -- j22++ -+ j27++ - } -- dAtA23[j22] = uint8(num) -- j22++ -+ dAtA28[j27] = uint8(num) -+ j27++ - } - dAtA[i] = 0xa - i++ -- i = encodeVarintAgent(dAtA, i, uint64(j22)) -- i += copy(dAtA[i:], dAtA23[:j22]) -+ i = encodeVarintAgent(dAtA, i, uint64(j27)) -+ i += copy(dAtA[i:], dAtA28[:j27]) - } - return i, nil - } -@@ -5648,7 +6017,7 @@ func (m *CgroupStats) Size() (n int) { - return n - } - --func (m *NetworkStats) Size() (n int) { -+func (m *InterfaceStats) Size() (n int) { - var l int - _ = l - l = len(m.Name) -@@ -5682,6 +6051,91 @@ func (m *NetworkStats) Size() (n int) { - return n - } - -+func (m *TcpStat) Size() (n int) { -+ var l int -+ _ = l -+ if m.Established != 0 { -+ n += 1 + sovAgent(uint64(m.Established)) -+ } -+ if m.SynSent != 0 { -+ n += 1 + sovAgent(uint64(m.SynSent)) -+ } -+ if m.SynRecv != 0 { -+ n += 1 + sovAgent(uint64(m.SynRecv)) -+ } -+ if m.FinWait1 != 0 { -+ n += 1 + sovAgent(uint64(m.FinWait1)) -+ } -+ if m.FinWait2 != 0 { -+ n += 1 + sovAgent(uint64(m.FinWait2)) -+ } -+ if m.TimeWait != 0 { -+ n += 1 + sovAgent(uint64(m.TimeWait)) -+ } -+ if m.Close != 0 { -+ n += 1 + sovAgent(uint64(m.Close)) -+ } -+ if m.CloseWait != 0 { -+ n += 1 + sovAgent(uint64(m.CloseWait)) -+ } -+ if m.LastAck != 0 { -+ n += 1 + sovAgent(uint64(m.LastAck)) -+ } -+ if m.Listen != 0 { -+ n += 1 + sovAgent(uint64(m.Listen)) -+ } -+ if m.Closing != 0 { -+ n += 1 + sovAgent(uint64(m.Closing)) -+ } -+ return n -+} -+ -+func (m *UdpStat) Size() (n int) { -+ var l int -+ _ = l -+ if m.Listen != 0 { -+ n += 1 + sovAgent(uint64(m.Listen)) -+ } -+ if m.Dropped != 0 { -+ n += 1 + sovAgent(uint64(m.Dropped)) -+ } -+ if m.RxQueued != 0 { -+ n += 1 + sovAgent(uint64(m.RxQueued)) -+ } -+ if m.TxQueued != 0 { -+ n += 1 + sovAgent(uint64(m.TxQueued)) -+ } -+ return n -+} -+ -+func (m *NetworkStats) Size() (n int) { -+ var l int -+ _ = l -+ if len(m.Interfaces) > 0 { -+ for _, e := range m.Interfaces { -+ l = e.Size() -+ n += 1 + l + sovAgent(uint64(l)) -+ } -+ } -+ if m.Tcp != nil { -+ l = m.Tcp.Size() -+ n += 1 + l + sovAgent(uint64(l)) -+ } -+ if m.Tcp6 != nil { -+ l = m.Tcp6.Size() -+ n += 1 + l + sovAgent(uint64(l)) -+ } -+ if m.Udp != nil { -+ l = m.Udp.Size() -+ n += 1 + l + sovAgent(uint64(l)) -+ } -+ if m.Udp6 != nil { -+ l = m.Udp6.Size() -+ n += 1 + l + sovAgent(uint64(l)) -+ } -+ return n -+} -+ - func (m *StatsContainerResponse) Size() (n int) { - var l int - _ = l -@@ -5689,11 +6143,9 @@ func (m *StatsContainerResponse) Size() (n int) { - l = m.CgroupStats.Size() - n += 1 + l + sovAgent(uint64(l)) - } -- if len(m.NetworkStats) > 0 { -- for _, e := range m.NetworkStats { -- l = e.Size() -- n += 1 + l + sovAgent(uint64(l)) -- } -+ if m.NetworkStats != nil { -+ l = m.NetworkStats.Size() -+ n += 1 + l + sovAgent(uint64(l)) - } - return n - } -@@ -9396,7 +9848,7 @@ func (m *CgroupStats) Unmarshal(dAtA []byte) error { - } - return nil - } --func (m *NetworkStats) Unmarshal(dAtA []byte) error { -+func (m *InterfaceStats) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { -@@ -9419,10 +9871,10 @@ func (m *NetworkStats) Unmarshal(dAtA []byte) error { - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { -- return fmt.Errorf("proto: NetworkStats: wiretype end group for non-group") -+ return fmt.Errorf("proto: InterfaceStats: wiretype end group for non-group") - } - if fieldNum <= 0 { -- return fmt.Errorf("proto: NetworkStats: illegal tag %d (wire type %d)", fieldNum, wire) -+ return fmt.Errorf("proto: InterfaceStats: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: -@@ -9627,7 +10079,7 @@ func (m *NetworkStats) Unmarshal(dAtA []byte) error { - } - return nil - } --func (m *StatsContainerResponse) Unmarshal(dAtA []byte) error { -+func (m *TcpStat) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { -@@ -9650,17 +10102,17 @@ func (m *StatsContainerResponse) Unmarshal(dAtA []byte) error { - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { -- return fmt.Errorf("proto: StatsContainerResponse: wiretype end group for non-group") -+ return fmt.Errorf("proto: TcpStat: wiretype end group for non-group") - } - if fieldNum <= 0 { -- return fmt.Errorf("proto: StatsContainerResponse: illegal tag %d (wire type %d)", fieldNum, wire) -+ return fmt.Errorf("proto: TcpStat: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field CgroupStats", wireType) -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field Established", wireType) - } -- var msglen int -+ m.Established = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowAgent -@@ -9670,30 +10122,54 @@ func (m *StatsContainerResponse) Unmarshal(dAtA []byte) error { - } - b := dAtA[iNdEx] - iNdEx++ -- msglen |= (int(b) & 0x7F) << shift -+ m.Established |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } -- if msglen < 0 { -- return ErrInvalidLengthAgent -+ case 2: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field SynSent", wireType) - } -- postIndex := iNdEx + msglen -- if postIndex > l { -- return io.ErrUnexpectedEOF -+ m.SynSent = 0 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ m.SynSent |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } - } -- if m.CgroupStats == nil { -- m.CgroupStats = &CgroupStats{} -+ case 3: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field SynRecv", wireType) - } -- if err := m.CgroupStats.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -- return err -+ m.SynRecv = 0 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ m.SynRecv |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } - } -- iNdEx = postIndex -- case 2: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field NetworkStats", wireType) -+ case 4: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field FinWait1", wireType) - } -- var msglen int -+ m.FinWait1 = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowAgent -@@ -9703,20 +10179,596 @@ func (m *StatsContainerResponse) Unmarshal(dAtA []byte) error { - } - b := dAtA[iNdEx] - iNdEx++ -- msglen |= (int(b) & 0x7F) << shift -+ m.FinWait1 |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } -- if msglen < 0 { -- return ErrInvalidLengthAgent -+ case 5: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field FinWait2", wireType) - } -- postIndex := iNdEx + msglen -- if postIndex > l { -- return io.ErrUnexpectedEOF -+ m.FinWait2 = 0 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ m.FinWait2 |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ case 6: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field TimeWait", wireType) -+ } -+ m.TimeWait = 0 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ m.TimeWait |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ case 7: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field Close", wireType) -+ } -+ m.Close = 0 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ m.Close |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ case 8: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field CloseWait", wireType) -+ } -+ m.CloseWait = 0 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ m.CloseWait |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ case 9: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field LastAck", wireType) -+ } -+ m.LastAck = 0 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ m.LastAck |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ case 10: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field Listen", wireType) -+ } -+ m.Listen = 0 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ m.Listen |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ case 11: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field Closing", wireType) -+ } -+ m.Closing = 0 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ m.Closing |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ default: -+ iNdEx = preIndex -+ skippy, err := skipAgent(dAtA[iNdEx:]) -+ if err != nil { -+ return err -+ } -+ if skippy < 0 { -+ return ErrInvalidLengthAgent -+ } -+ if (iNdEx + skippy) > l { -+ return io.ErrUnexpectedEOF -+ } -+ iNdEx += skippy -+ } -+ } -+ -+ if iNdEx > l { -+ return io.ErrUnexpectedEOF -+ } -+ return nil -+} -+func (m *UdpStat) Unmarshal(dAtA []byte) error { -+ l := len(dAtA) -+ iNdEx := 0 -+ for iNdEx < l { -+ preIndex := iNdEx -+ var wire uint64 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ wire |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ fieldNum := int32(wire >> 3) -+ wireType := int(wire & 0x7) -+ if wireType == 4 { -+ return fmt.Errorf("proto: UdpStat: wiretype end group for non-group") -+ } -+ if fieldNum <= 0 { -+ return fmt.Errorf("proto: UdpStat: illegal tag %d (wire type %d)", fieldNum, wire) -+ } -+ switch fieldNum { -+ case 1: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field Listen", wireType) -+ } -+ m.Listen = 0 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ m.Listen |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ case 2: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field Dropped", wireType) -+ } -+ m.Dropped = 0 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ m.Dropped |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ case 3: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field RxQueued", wireType) -+ } -+ m.RxQueued = 0 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ m.RxQueued |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ case 4: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field TxQueued", wireType) -+ } -+ m.TxQueued = 0 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ m.TxQueued |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ default: -+ iNdEx = preIndex -+ skippy, err := skipAgent(dAtA[iNdEx:]) -+ if err != nil { -+ return err -+ } -+ if skippy < 0 { -+ return ErrInvalidLengthAgent -+ } -+ if (iNdEx + skippy) > l { -+ return io.ErrUnexpectedEOF -+ } -+ iNdEx += skippy -+ } -+ } -+ -+ if iNdEx > l { -+ return io.ErrUnexpectedEOF -+ } -+ return nil -+} -+func (m *NetworkStats) Unmarshal(dAtA []byte) error { -+ l := len(dAtA) -+ iNdEx := 0 -+ for iNdEx < l { -+ preIndex := iNdEx -+ var wire uint64 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ wire |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ fieldNum := int32(wire >> 3) -+ wireType := int(wire & 0x7) -+ if wireType == 4 { -+ return fmt.Errorf("proto: NetworkStats: wiretype end group for non-group") -+ } -+ if fieldNum <= 0 { -+ return fmt.Errorf("proto: NetworkStats: illegal tag %d (wire type %d)", fieldNum, wire) -+ } -+ switch fieldNum { -+ case 1: -+ if wireType != 2 { -+ return fmt.Errorf("proto: wrong wireType = %d for field Interfaces", wireType) -+ } -+ var msglen int -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ msglen |= (int(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ if msglen < 0 { -+ return ErrInvalidLengthAgent -+ } -+ postIndex := iNdEx + msglen -+ if postIndex > l { -+ return io.ErrUnexpectedEOF -+ } -+ m.Interfaces = append(m.Interfaces, &InterfaceStats{}) -+ if err := m.Interfaces[len(m.Interfaces)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -+ return err -+ } -+ iNdEx = postIndex -+ case 2: -+ if wireType != 2 { -+ return fmt.Errorf("proto: wrong wireType = %d for field Tcp", wireType) -+ } -+ var msglen int -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ msglen |= (int(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ if msglen < 0 { -+ return ErrInvalidLengthAgent -+ } -+ postIndex := iNdEx + msglen -+ if postIndex > l { -+ return io.ErrUnexpectedEOF -+ } -+ if m.Tcp == nil { -+ m.Tcp = &TcpStat{} -+ } -+ if err := m.Tcp.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -+ return err -+ } -+ iNdEx = postIndex -+ case 3: -+ if wireType != 2 { -+ return fmt.Errorf("proto: wrong wireType = %d for field Tcp6", wireType) -+ } -+ var msglen int -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ msglen |= (int(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ if msglen < 0 { -+ return ErrInvalidLengthAgent -+ } -+ postIndex := iNdEx + msglen -+ if postIndex > l { -+ return io.ErrUnexpectedEOF -+ } -+ if m.Tcp6 == nil { -+ m.Tcp6 = &TcpStat{} -+ } -+ if err := m.Tcp6.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -+ return err -+ } -+ iNdEx = postIndex -+ case 4: -+ if wireType != 2 { -+ return fmt.Errorf("proto: wrong wireType = %d for field Udp", wireType) -+ } -+ var msglen int -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ msglen |= (int(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ if msglen < 0 { -+ return ErrInvalidLengthAgent -+ } -+ postIndex := iNdEx + msglen -+ if postIndex > l { -+ return io.ErrUnexpectedEOF -+ } -+ if m.Udp == nil { -+ m.Udp = &UdpStat{} -+ } -+ if err := m.Udp.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -+ return err -+ } -+ iNdEx = postIndex -+ case 5: -+ if wireType != 2 { -+ return fmt.Errorf("proto: wrong wireType = %d for field Udp6", wireType) -+ } -+ var msglen int -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ msglen |= (int(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ if msglen < 0 { -+ return ErrInvalidLengthAgent -+ } -+ postIndex := iNdEx + msglen -+ if postIndex > l { -+ return io.ErrUnexpectedEOF -+ } -+ if m.Udp6 == nil { -+ m.Udp6 = &UdpStat{} -+ } -+ if err := m.Udp6.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -+ return err -+ } -+ iNdEx = postIndex -+ default: -+ iNdEx = preIndex -+ skippy, err := skipAgent(dAtA[iNdEx:]) -+ if err != nil { -+ return err -+ } -+ if skippy < 0 { -+ return ErrInvalidLengthAgent -+ } -+ if (iNdEx + skippy) > l { -+ return io.ErrUnexpectedEOF -+ } -+ iNdEx += skippy -+ } -+ } -+ -+ if iNdEx > l { -+ return io.ErrUnexpectedEOF -+ } -+ return nil -+} -+func (m *StatsContainerResponse) Unmarshal(dAtA []byte) error { -+ l := len(dAtA) -+ iNdEx := 0 -+ for iNdEx < l { -+ preIndex := iNdEx -+ var wire uint64 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ wire |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ fieldNum := int32(wire >> 3) -+ wireType := int(wire & 0x7) -+ if wireType == 4 { -+ return fmt.Errorf("proto: StatsContainerResponse: wiretype end group for non-group") -+ } -+ if fieldNum <= 0 { -+ return fmt.Errorf("proto: StatsContainerResponse: illegal tag %d (wire type %d)", fieldNum, wire) -+ } -+ switch fieldNum { -+ case 1: -+ if wireType != 2 { -+ return fmt.Errorf("proto: wrong wireType = %d for field CgroupStats", wireType) -+ } -+ var msglen int -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ msglen |= (int(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ if msglen < 0 { -+ return ErrInvalidLengthAgent -+ } -+ postIndex := iNdEx + msglen -+ if postIndex > l { -+ return io.ErrUnexpectedEOF -+ } -+ if m.CgroupStats == nil { -+ m.CgroupStats = &CgroupStats{} -+ } -+ if err := m.CgroupStats.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -+ return err -+ } -+ iNdEx = postIndex -+ case 2: -+ if wireType != 2 { -+ return fmt.Errorf("proto: wrong wireType = %d for field NetworkStats", wireType) -+ } -+ var msglen int -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ msglen |= (int(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ if msglen < 0 { -+ return ErrInvalidLengthAgent -+ } -+ postIndex := iNdEx + msglen -+ if postIndex > l { -+ return io.ErrUnexpectedEOF -+ } -+ if m.NetworkStats == nil { -+ m.NetworkStats = &NetworkStats{} - } -- m.NetworkStats = append(m.NetworkStats, &NetworkStats{}) -- if err := m.NetworkStats[len(m.NetworkStats)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -+ if err := m.NetworkStats.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex -@@ -13190,189 +14242,204 @@ var ( - func init() { proto.RegisterFile("agent.proto", fileDescriptorAgent) } - - var fileDescriptorAgent = []byte{ -- // 2931 bytes of a gzipped FileDescriptorProto -- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0xcb, 0x6e, 0x1c, 0xc7, -- 0xb5, 0x98, 0x07, 0x87, 0x33, 0x67, 0x5e, 0x9c, 0x22, 0x45, 0x8d, 0x46, 0xb2, 0xae, 0xdc, 0xb6, -- 0x65, 0xfa, 0xfa, 0x7a, 0x68, 0xcb, 0xc6, 0xf5, 0x0b, 0xbe, 0x82, 0x48, 0xe9, 0x8a, 0x8c, 0xad, -- 0x88, 0xe9, 0x91, 0xe2, 0x04, 0x41, 0xd0, 0x68, 0x76, 0x97, 0x86, 0x65, 0x4e, 0x77, 0xb5, 0xab, -- 0xaa, 0x29, 0x8e, 0x03, 0x64, 0x99, 0xec, 0xb2, 0xcc, 0x2e, 0x3f, 0x10, 0x64, 0x97, 0x65, 0xb6, -- 0x59, 0x18, 0x59, 0x05, 0xf9, 0x80, 0x20, 0xf0, 0x27, 0xe4, 0x0b, 0x82, 0x7a, 0xf5, 0x63, 0x66, -- 0x48, 0x23, 0x82, 0x80, 0x6c, 0x1a, 0x75, 0x4e, 0x9d, 0x3a, 0xaf, 0xaa, 0x3a, 0x75, 0xce, 0x69, -- 0x68, 0xfb, 0x53, 0x1c, 0x8b, 0x71, 0xc2, 0xa8, 0xa0, 0xa8, 0x3e, 0x65, 0x49, 0x30, 0x6a, 0xd1, -- 0x80, 0x68, 0xc4, 0xe8, 0x7f, 0xa7, 0x44, 0x9c, 0xa4, 0xc7, 0xe3, 0x80, 0x46, 0xbb, 0xa7, 0xbe, -- 0xf0, 0xdf, 0x09, 0x68, 0x2c, 0x7c, 0x12, 0x63, 0xc6, 0x77, 0xd5, 0xc2, 0xdd, 0xe4, 0x74, 0xba, -- 0x2b, 0xe6, 0x09, 0xe6, 0xfa, 0x6b, 0xd6, 0x5d, 0x9f, 0x52, 0x3a, 0x9d, 0xe1, 0x5d, 0x05, 0x1d, -- 0xa7, 0xcf, 0x76, 0x71, 0x94, 0x88, 0xb9, 0x9e, 0x74, 0x7e, 0x57, 0x85, 0xed, 0x7d, 0x86, 0x7d, -- 0x81, 0xf7, 0x2d, 0x37, 0x17, 0x7f, 0x9d, 0x62, 0x2e, 0xd0, 0xab, 0xd0, 0xc9, 0x24, 0x78, 0x24, -- 0x1c, 0x56, 0x6e, 0x55, 0x76, 0x5a, 0x6e, 0x3b, 0xc3, 0x1d, 0x86, 0xe8, 0x2a, 0xac, 0xe3, 0x73, -- 0x1c, 0xc8, 0xd9, 0xaa, 0x9a, 0x6d, 0x48, 0xf0, 0x30, 0x44, 0xef, 0x41, 0x9b, 0x0b, 0x46, 0xe2, -- 0xa9, 0x97, 0x72, 0xcc, 0x86, 0xb5, 0x5b, 0x95, 0x9d, 0xf6, 0x9d, 0x8d, 0xb1, 0x34, 0x69, 0x3c, -- 0x51, 0x13, 0x4f, 0x39, 0x66, 0x2e, 0xf0, 0x6c, 0x8c, 0x6e, 0xc3, 0x7a, 0x88, 0xcf, 0x48, 0x80, -- 0xf9, 0xb0, 0x7e, 0xab, 0xb6, 0xd3, 0xbe, 0xd3, 0xd1, 0xe4, 0xf7, 0x15, 0xd2, 0xb5, 0x93, 0xe8, -- 0x2d, 0x68, 0x72, 0x41, 0x99, 0x3f, 0xc5, 0x7c, 0xb8, 0xa6, 0x08, 0xbb, 0x96, 0xaf, 0xc2, 0xba, -- 0xd9, 0x34, 0xba, 0x01, 0xb5, 0xc7, 0xfb, 0x87, 0xc3, 0x86, 0x92, 0x0e, 0x86, 0x2a, 0xc1, 0x81, -- 0x2b, 0xd1, 0xe8, 0x35, 0xe8, 0x72, 0x3f, 0x0e, 0x8f, 0xe9, 0xb9, 0x97, 0x90, 0x30, 0xe6, 0xc3, -- 0xf5, 0x5b, 0x95, 0x9d, 0xa6, 0xdb, 0x31, 0xc8, 0x23, 0x89, 0x73, 0x3e, 0x81, 0x2b, 0x13, 0xe1, -- 0x33, 0xf1, 0x02, 0xde, 0x71, 0x9e, 0xc2, 0xb6, 0x8b, 0x23, 0x7a, 0xf6, 0x42, 0xae, 0x1d, 0xc2, -- 0xba, 0x20, 0x11, 0xa6, 0xa9, 0x50, 0xae, 0xed, 0xba, 0x16, 0x74, 0xfe, 0x50, 0x01, 0xf4, 0xe0, -- 0x1c, 0x07, 0x47, 0x8c, 0x06, 0x98, 0xf3, 0xff, 0xd0, 0x76, 0xbd, 0x09, 0xeb, 0x89, 0x56, 0x60, -- 0x58, 0x57, 0xe4, 0x66, 0x17, 0xac, 0x56, 0x76, 0xd6, 0xf9, 0x0a, 0xb6, 0x26, 0x64, 0x1a, 0xfb, -- 0xb3, 0x97, 0xa8, 0xef, 0x36, 0x34, 0xb8, 0xe2, 0xa9, 0x54, 0xed, 0xba, 0x06, 0x72, 0x8e, 0x00, -- 0x7d, 0xe9, 0x13, 0xf1, 0xf2, 0x24, 0x39, 0xef, 0xc0, 0x66, 0x89, 0x23, 0x4f, 0x68, 0xcc, 0xb1, -- 0x52, 0x40, 0xf8, 0x22, 0xe5, 0x8a, 0xd9, 0x9a, 0x6b, 0x20, 0x07, 0xc3, 0xd6, 0x17, 0x84, 0x5b, -- 0x72, 0xfc, 0xef, 0xa8, 0xb0, 0x0d, 0x8d, 0x67, 0x94, 0x45, 0xbe, 0xb0, 0x1a, 0x68, 0x08, 0x21, -- 0xa8, 0xfb, 0x6c, 0xca, 0x87, 0xb5, 0x5b, 0xb5, 0x9d, 0x96, 0xab, 0xc6, 0xf2, 0x54, 0x2e, 0x88, -- 0x31, 0x7a, 0xbd, 0x0a, 0x1d, 0xe3, 0x77, 0x6f, 0x46, 0xb8, 0x50, 0x72, 0x3a, 0x6e, 0xdb, 0xe0, -- 0xe4, 0x1a, 0x87, 0xc2, 0xf6, 0xd3, 0x24, 0x7c, 0xc1, 0x0b, 0x7f, 0x07, 0x5a, 0x0c, 0x73, 0x9a, -- 0x32, 0x79, 0x4d, 0xab, 0x6a, 0xdf, 0xb7, 0xf4, 0xbe, 0x7f, 0x41, 0xe2, 0xf4, 0xdc, 0xb5, 0x73, -- 0x6e, 0x4e, 0x66, 0xae, 0x90, 0xe0, 0x2f, 0x72, 0x85, 0x3e, 0x81, 0x2b, 0x47, 0x7e, 0xca, 0x5f, -- 0x44, 0x57, 0xe7, 0x53, 0x79, 0xfd, 0x78, 0x1a, 0xbd, 0xd0, 0xe2, 0xdf, 0x57, 0xa0, 0xb9, 0x9f, -- 0xa4, 0x4f, 0xb9, 0x3f, 0xc5, 0xe8, 0xbf, 0xa0, 0x2d, 0xa8, 0xf0, 0x67, 0x5e, 0x2a, 0x41, 0x45, -- 0x5e, 0x77, 0x41, 0xa1, 0x34, 0x81, 0x74, 0x3b, 0x66, 0x41, 0x92, 0x1a, 0x8a, 0xea, 0xad, 0xda, -- 0x4e, 0xdd, 0x6d, 0x6b, 0x9c, 0x26, 0x19, 0xc3, 0xa6, 0x9a, 0xf3, 0x48, 0xec, 0x9d, 0x62, 0x16, -- 0xe3, 0x59, 0x44, 0x43, 0xac, 0xce, 0x6f, 0xdd, 0x1d, 0xa8, 0xa9, 0xc3, 0xf8, 0xf3, 0x6c, 0x02, -- 0xfd, 0x37, 0x0c, 0x32, 0x7a, 0x79, 0x29, 0x15, 0x75, 0x5d, 0x51, 0xf7, 0x0d, 0xf5, 0x53, 0x83, -- 0x76, 0x7e, 0x09, 0xbd, 0x27, 0x27, 0x8c, 0x0a, 0x31, 0x23, 0xf1, 0xf4, 0xbe, 0x2f, 0x7c, 0x19, -- 0x3d, 0x12, 0xcc, 0x08, 0x0d, 0xb9, 0xd1, 0xd6, 0x82, 0xe8, 0x6d, 0x18, 0x08, 0x4d, 0x8b, 0x43, -- 0xcf, 0xd2, 0x54, 0x15, 0xcd, 0x46, 0x36, 0x71, 0x64, 0x88, 0xdf, 0x80, 0x5e, 0x4e, 0x2c, 0xe3, -- 0x8f, 0xd1, 0xb7, 0x9b, 0x61, 0x9f, 0x90, 0x08, 0x3b, 0x67, 0xca, 0x57, 0x6a, 0x93, 0xd1, 0xdb, -- 0xd0, 0xca, 0xfd, 0x50, 0x51, 0x27, 0xa4, 0xa7, 0x4f, 0x88, 0x75, 0xa7, 0xdb, 0xcc, 0x9c, 0xf2, -- 0x19, 0xf4, 0x45, 0xa6, 0xb8, 0x17, 0xfa, 0xc2, 0x2f, 0x1f, 0xaa, 0xb2, 0x55, 0x6e, 0x4f, 0x94, -- 0x60, 0xe7, 0x53, 0x68, 0x1d, 0x91, 0x90, 0x6b, 0xc1, 0x43, 0x58, 0x0f, 0x52, 0xc6, 0x70, 0x2c, -- 0xac, 0xc9, 0x06, 0x44, 0x5b, 0xb0, 0x36, 0x23, 0x11, 0x11, 0xc6, 0x4c, 0x0d, 0x38, 0x14, 0xe0, -- 0x11, 0x8e, 0x28, 0x9b, 0x2b, 0x87, 0x6d, 0xc1, 0x5a, 0x71, 0x73, 0x35, 0x80, 0xae, 0x43, 0x2b, -- 0xf2, 0xcf, 0xb3, 0x4d, 0x95, 0x33, 0xcd, 0xc8, 0x3f, 0xd7, 0xca, 0x0f, 0x61, 0xfd, 0x99, 0x4f, -- 0x66, 0x41, 0x2c, 0x8c, 0x57, 0x2c, 0x98, 0x0b, 0xac, 0x17, 0x05, 0xfe, 0xb9, 0x0a, 0x6d, 0x2d, -- 0x51, 0x2b, 0xbc, 0x05, 0x6b, 0x81, 0x1f, 0x9c, 0x64, 0x22, 0x15, 0x80, 0x6e, 0x5b, 0x45, 0xaa, -- 0xc5, 0x20, 0x9c, 0x6b, 0x6a, 0x55, 0xdb, 0x05, 0xe0, 0xcf, 0xfd, 0xc4, 0xe8, 0x56, 0xbb, 0x80, -- 0xb8, 0x25, 0x69, 0xb4, 0xba, 0xef, 0x43, 0x47, 0x9f, 0x3b, 0xb3, 0xa4, 0x7e, 0xc1, 0x92, 0xb6, -- 0xa6, 0xd2, 0x8b, 0x5e, 0x83, 0x6e, 0xca, 0xb1, 0x77, 0x42, 0x30, 0xf3, 0x59, 0x70, 0x32, 0x1f, -- 0xae, 0xe9, 0x37, 0x32, 0xe5, 0xf8, 0xc0, 0xe2, 0xd0, 0x1d, 0x58, 0x93, 0xe1, 0x8f, 0x0f, 0x1b, -- 0xea, 0x39, 0xbe, 0x51, 0x64, 0xa9, 0x4c, 0x1d, 0xab, 0xef, 0x83, 0x58, 0xb0, 0xb9, 0xab, 0x49, -- 0x47, 0x1f, 0x01, 0xe4, 0x48, 0xb4, 0x01, 0xb5, 0x53, 0x3c, 0x37, 0xf7, 0x50, 0x0e, 0xa5, 0x73, -- 0xce, 0xfc, 0x59, 0x6a, 0xbd, 0xae, 0x81, 0x4f, 0xaa, 0x1f, 0x55, 0x9c, 0x00, 0xfa, 0x7b, 0xb3, -- 0x53, 0x42, 0x0b, 0xcb, 0xb7, 0x60, 0x2d, 0xf2, 0xbf, 0xa2, 0xcc, 0x7a, 0x52, 0x01, 0x0a, 0x4b, -- 0x62, 0xca, 0x2c, 0x0b, 0x05, 0xa0, 0x1e, 0x54, 0x69, 0xa2, 0xfc, 0xd5, 0x72, 0xab, 0x34, 0xc9, -- 0x05, 0xd5, 0x0b, 0x82, 0x9c, 0xbf, 0xd7, 0x01, 0x72, 0x29, 0xc8, 0x85, 0x11, 0xa1, 0x1e, 0xc7, -- 0x4c, 0xa6, 0x20, 0xde, 0xf1, 0x5c, 0x60, 0xee, 0x31, 0x1c, 0xa4, 0x8c, 0x93, 0x33, 0xb9, 0x7f, -- 0xd2, 0xec, 0x2b, 0xda, 0xec, 0x05, 0xdd, 0xdc, 0xab, 0x84, 0x4e, 0xf4, 0xba, 0x3d, 0xb9, 0xcc, -- 0xb5, 0xab, 0xd0, 0x21, 0x5c, 0xc9, 0x79, 0x86, 0x05, 0x76, 0xd5, 0xcb, 0xd8, 0x6d, 0x66, 0xec, -- 0xc2, 0x9c, 0xd5, 0x03, 0xd8, 0x24, 0xd4, 0xfb, 0x3a, 0xc5, 0x69, 0x89, 0x51, 0xed, 0x32, 0x46, -- 0x03, 0x42, 0x7f, 0xa4, 0x16, 0xe4, 0x6c, 0x8e, 0xe0, 0x5a, 0xc1, 0x4a, 0x79, 0xdd, 0x0b, 0xcc, -- 0xea, 0x97, 0x31, 0xdb, 0xce, 0xb4, 0x92, 0xf1, 0x20, 0xe7, 0xf8, 0x03, 0xd8, 0x26, 0xd4, 0x7b, -- 0xee, 0x13, 0xb1, 0xc8, 0x6e, 0xed, 0x7b, 0x8c, 0x94, 0x8f, 0x6e, 0x99, 0x97, 0x36, 0x32, 0xc2, -- 0x6c, 0x5a, 0x32, 0xb2, 0xf1, 0x3d, 0x46, 0x3e, 0x52, 0x0b, 0x72, 0x36, 0xf7, 0x60, 0x40, 0xe8, -- 0xa2, 0x36, 0xeb, 0x97, 0x31, 0xe9, 0x13, 0x5a, 0xd6, 0x64, 0x0f, 0x06, 0x1c, 0x07, 0x82, 0xb2, -- 0xe2, 0x21, 0x68, 0x5e, 0xc6, 0x62, 0xc3, 0xd0, 0x67, 0x3c, 0x9c, 0x9f, 0x41, 0xe7, 0x20, 0x9d, -- 0x62, 0x31, 0x3b, 0xce, 0x82, 0xc1, 0x4b, 0x8b, 0x3f, 0xce, 0x3f, 0xab, 0xd0, 0xde, 0x9f, 0x32, -- 0x9a, 0x26, 0xa5, 0x98, 0xac, 0x2f, 0xe9, 0x62, 0x4c, 0x56, 0x24, 0x2a, 0x26, 0x6b, 0xe2, 0x0f, -- 0xa0, 0x13, 0xa9, 0xab, 0x6b, 0xe8, 0x75, 0x1c, 0x1a, 0x2c, 0x5d, 0x6a, 0xb7, 0x1d, 0x15, 0x82, -- 0xd9, 0x18, 0x20, 0x21, 0x21, 0x37, 0x6b, 0x74, 0x38, 0xea, 0x9b, 0x8c, 0xd0, 0x86, 0x68, 0xb7, -- 0x95, 0x64, 0xd1, 0xfa, 0x3d, 0x68, 0x1f, 0x4b, 0x27, 0x99, 0x05, 0xa5, 0x60, 0x94, 0x7b, 0xcf, -- 0x85, 0xe3, 0xfc, 0x12, 0x1e, 0x40, 0xf7, 0x44, 0xbb, 0xcc, 0x2c, 0xd2, 0x67, 0xe8, 0x35, 0x63, -- 0x49, 0x6e, 0xef, 0xb8, 0xe8, 0x59, 0xbd, 0x01, 0x9d, 0x93, 0x02, 0x6a, 0x34, 0x81, 0xc1, 0x12, -- 0xc9, 0x8a, 0x18, 0xb4, 0x53, 0x8c, 0x41, 0xed, 0x3b, 0x48, 0x0b, 0x2a, 0xae, 0x2c, 0xc6, 0xa5, -- 0xdf, 0x54, 0xa1, 0xf3, 0x43, 0x2c, 0x9e, 0x53, 0x76, 0xaa, 0xf5, 0x45, 0x50, 0x8f, 0xfd, 0x08, -- 0x1b, 0x8e, 0x6a, 0x8c, 0xae, 0x41, 0x93, 0x9d, 0xeb, 0x00, 0x62, 0xf6, 0x73, 0x9d, 0x9d, 0xab, -- 0xc0, 0x80, 0x5e, 0x01, 0x60, 0xe7, 0x5e, 0xe2, 0x07, 0xa7, 0xd8, 0x78, 0xb0, 0xee, 0xb6, 0xd8, -- 0xf9, 0x91, 0x46, 0xc8, 0xa3, 0xc0, 0xce, 0x3d, 0xcc, 0x18, 0x65, 0xdc, 0xc4, 0xaa, 0x26, 0x3b, -- 0x7f, 0xa0, 0x60, 0xb3, 0x36, 0x64, 0x34, 0x49, 0x70, 0xa8, 0x62, 0xb4, 0x5a, 0x7b, 0x5f, 0x23, -- 0xa4, 0x54, 0x61, 0xa5, 0x36, 0xb4, 0x54, 0x91, 0x4b, 0x15, 0xb9, 0xd4, 0x75, 0xbd, 0x52, 0x14, -- 0xa5, 0x8a, 0x4c, 0x6a, 0x53, 0x4b, 0x15, 0x05, 0xa9, 0x22, 0x97, 0xda, 0xb2, 0x6b, 0x8d, 0x54, -- 0xe7, 0xd7, 0x15, 0xd8, 0x5e, 0x4c, 0xfc, 0x4c, 0x9a, 0xfa, 0x01, 0x74, 0x02, 0xb5, 0x5f, 0xa5, -- 0x33, 0x39, 0x58, 0xda, 0x49, 0xb7, 0x1d, 0x14, 0x8e, 0xf1, 0x87, 0xd0, 0x8d, 0xb5, 0x83, 0xb3, -- 0xa3, 0x59, 0xcb, 0xf7, 0xa5, 0xe8, 0x7b, 0xb7, 0x13, 0x17, 0x20, 0x27, 0x04, 0xf4, 0x25, 0x23, -- 0x02, 0x4f, 0x04, 0xc3, 0x7e, 0xf4, 0x32, 0x0a, 0x10, 0x04, 0x75, 0x95, 0xad, 0xd4, 0x54, 0x7e, -- 0xad, 0xc6, 0xce, 0x9b, 0xb0, 0x59, 0x92, 0x62, 0x6c, 0xdd, 0x80, 0xda, 0x0c, 0xc7, 0x8a, 0x7b, -- 0xd7, 0x95, 0x43, 0xc7, 0x87, 0x81, 0x8b, 0xfd, 0xf0, 0xe5, 0x69, 0x63, 0x44, 0xd4, 0x72, 0x11, -- 0x3b, 0x80, 0x8a, 0x22, 0x8c, 0x2a, 0x56, 0xeb, 0x4a, 0x41, 0xeb, 0xc7, 0x30, 0xd8, 0x9f, 0x51, -- 0x8e, 0x27, 0x22, 0x24, 0xf1, 0xcb, 0xa8, 0x98, 0x7e, 0x01, 0x9b, 0x4f, 0xc4, 0xfc, 0x4b, 0xc9, -- 0x8c, 0x93, 0x6f, 0xf0, 0x4b, 0xb2, 0x8f, 0xd1, 0xe7, 0xd6, 0x3e, 0x46, 0x9f, 0xcb, 0x62, 0x29, -- 0xa0, 0xb3, 0x34, 0x8a, 0xd5, 0x55, 0xe8, 0xba, 0x06, 0x72, 0xf6, 0xa0, 0xa3, 0x73, 0xe8, 0x47, -- 0x34, 0x4c, 0x67, 0x78, 0xe5, 0x1d, 0xbc, 0x09, 0x90, 0xf8, 0xcc, 0x8f, 0xb0, 0xc0, 0x4c, 0x9f, -- 0xa1, 0x96, 0x5b, 0xc0, 0x38, 0xbf, 0xad, 0xc2, 0x96, 0x6e, 0x89, 0x4c, 0x74, 0x27, 0xc0, 0x9a, -- 0x30, 0x82, 0xe6, 0x09, 0xe5, 0xa2, 0xc0, 0x30, 0x83, 0xa5, 0x8a, 0x61, 0x6c, 0xb9, 0xc9, 0x61, -- 0xa9, 0x4f, 0x51, 0xbb, 0xbc, 0x4f, 0xb1, 0xd4, 0x89, 0xa8, 0x2f, 0x77, 0x22, 0xe4, 0x6d, 0xb3, -- 0x44, 0x44, 0xdf, 0xf1, 0x96, 0xdb, 0x32, 0x98, 0xc3, 0x10, 0xdd, 0x86, 0xfe, 0x54, 0x6a, 0xe9, -- 0x9d, 0x50, 0x7a, 0xea, 0x25, 0xbe, 0x38, 0x51, 0x57, 0xbd, 0xe5, 0x76, 0x15, 0xfa, 0x80, 0xd2, -- 0xd3, 0x23, 0x5f, 0x9c, 0xa0, 0x8f, 0xa1, 0x67, 0xd2, 0xc0, 0x48, 0xb9, 0x88, 0x9b, 0xc7, 0xcf, -- 0xdc, 0xa2, 0xa2, 0xf7, 0xdc, 0xee, 0x69, 0x01, 0xe2, 0xce, 0x55, 0xb8, 0x72, 0x1f, 0x73, 0xc1, -- 0xe8, 0xbc, 0xec, 0x18, 0xe7, 0xff, 0x00, 0x0e, 0x63, 0x81, 0xd9, 0x33, 0x3f, 0xc0, 0x1c, 0xbd, -- 0x5b, 0x84, 0x4c, 0x72, 0xb4, 0x31, 0xd6, 0x1d, 0xa9, 0x6c, 0xc2, 0x2d, 0xd0, 0x38, 0x63, 0x68, -- 0xb8, 0x34, 0x95, 0xe1, 0xe8, 0x75, 0x3b, 0x32, 0xeb, 0x3a, 0x66, 0x9d, 0x42, 0xba, 0x66, 0xce, -- 0x39, 0xb0, 0x25, 0x6c, 0xce, 0xce, 0x6c, 0xd1, 0x18, 0x5a, 0xc4, 0xe2, 0x4c, 0x54, 0x59, 0x16, -- 0x9d, 0x93, 0x38, 0x3f, 0x85, 0x4d, 0xcd, 0x49, 0x73, 0xb6, 0x6c, 0x5e, 0x87, 0x06, 0xb3, 0x6a, -- 0x54, 0xf2, 0x56, 0x94, 0x21, 0x32, 0x73, 0xe8, 0x86, 0x14, 0x16, 0x30, 0x1c, 0xc9, 0x9a, 0xa3, -- 0xaa, 0xb6, 0x2c, 0x47, 0x48, 0x6f, 0xc9, 0x7a, 0x3b, 0x37, 0xd3, 0x7a, 0x6b, 0x13, 0x06, 0x72, -- 0xa2, 0x24, 0xd1, 0xf9, 0x39, 0x6c, 0x3e, 0x8e, 0x67, 0x24, 0xc6, 0xfb, 0x47, 0x4f, 0x1f, 0xe1, -- 0x2c, 0x2a, 0x20, 0xa8, 0xcb, 0xec, 0x49, 0xa9, 0xd1, 0x74, 0xd5, 0x58, 0x5e, 0x93, 0xf8, 0xd8, -- 0x0b, 0x92, 0x94, 0x9b, 0xce, 0x50, 0x23, 0x3e, 0xde, 0x4f, 0x52, 0x2e, 0xc3, 0xbc, 0x7c, 0xe6, -- 0x69, 0x3c, 0x9b, 0xab, 0xbb, 0xd2, 0x74, 0xd7, 0x83, 0x24, 0x7d, 0x1c, 0xcf, 0xe6, 0xce, 0xff, -- 0xa8, 0x5a, 0x18, 0xe3, 0xd0, 0xf5, 0xe3, 0x90, 0x46, 0xf7, 0xf1, 0x59, 0x41, 0x42, 0x56, 0x77, -- 0xd9, 0x98, 0xf0, 0x6d, 0x05, 0x3a, 0xf7, 0xa6, 0x38, 0x16, 0xf7, 0xb1, 0xf0, 0xc9, 0x4c, 0xd5, -- 0x56, 0x67, 0x98, 0x71, 0x42, 0x63, 0x73, 0xf0, 0x2d, 0x28, 0x4b, 0x63, 0x12, 0x13, 0xe1, 0x85, -- 0x3e, 0x8e, 0x68, 0x6c, 0xbc, 0x00, 0x12, 0x75, 0x5f, 0x61, 0xd0, 0x9b, 0xd0, 0xd7, 0x9d, 0x3b, -- 0xef, 0xc4, 0x8f, 0xc3, 0x99, 0xbc, 0x72, 0xba, 0x93, 0xd1, 0xd3, 0xe8, 0x03, 0x83, 0x45, 0x6f, -- 0xc1, 0x86, 0xb9, 0x10, 0x39, 0x65, 0x5d, 0x51, 0xf6, 0x0d, 0xbe, 0x44, 0x9a, 0x26, 0x09, 0x65, -- 0x82, 0x7b, 0x1c, 0x07, 0x01, 0x8d, 0x12, 0x53, 0x98, 0xf4, 0x2d, 0x7e, 0xa2, 0xd1, 0xce, 0x14, -- 0x36, 0x1f, 0x4a, 0x3b, 0x8d, 0x25, 0xf9, 0x06, 0xf7, 0x22, 0x1c, 0x79, 0xc7, 0x33, 0x1a, 0x9c, -- 0x7a, 0x32, 0x4c, 0x19, 0x0f, 0xcb, 0xd4, 0x67, 0x4f, 0x22, 0x27, 0xe4, 0x1b, 0x55, 0x83, 0x4b, -- 0xaa, 0x13, 0x2a, 0x92, 0x59, 0x3a, 0xf5, 0x12, 0x46, 0x8f, 0xb1, 0x31, 0xb1, 0x1f, 0xe1, 0xe8, -- 0x40, 0xe3, 0x8f, 0x24, 0xda, 0xf9, 0x53, 0x05, 0xb6, 0xca, 0x92, 0x4c, 0xd0, 0xdd, 0x85, 0xad, -- 0xb2, 0x28, 0xf3, 0x10, 0xeb, 0x44, 0x6f, 0x50, 0x14, 0xa8, 0x9f, 0xe4, 0x0f, 0xa1, 0xab, 0xda, -- 0xb9, 0x5e, 0xa8, 0x39, 0x95, 0xd3, 0x8f, 0xe2, 0xbe, 0xb8, 0x1d, 0xbf, 0xb8, 0x4b, 0x1f, 0xc3, -- 0x35, 0x63, 0xbe, 0xb7, 0xac, 0xb6, 0x3e, 0x10, 0xdb, 0x86, 0xe0, 0xd1, 0x82, 0xf6, 0x5f, 0xc0, -- 0x30, 0x47, 0xed, 0xcd, 0x15, 0xd2, 0xfa, 0xea, 0x5d, 0xd8, 0x5c, 0x30, 0xf6, 0x5e, 0x18, 0x32, -- 0x75, 0x41, 0xeb, 0xee, 0xaa, 0x29, 0xe7, 0x2e, 0x5c, 0x9d, 0x60, 0xa1, 0xbd, 0xe1, 0x0b, 0x53, -- 0x13, 0x68, 0x66, 0x1b, 0x50, 0x9b, 0xe0, 0x40, 0x19, 0x5f, 0x73, 0xe5, 0x50, 0x1e, 0xc0, 0xa7, -- 0x1c, 0x07, 0xca, 0xca, 0x9a, 0xab, 0xc6, 0xce, 0x1f, 0x2b, 0xb0, 0x6e, 0xc2, 0xa4, 0x0c, 0xf5, -- 0x21, 0x23, 0x67, 0x98, 0x99, 0xa3, 0x67, 0x20, 0xf4, 0x06, 0xf4, 0xf4, 0xc8, 0xa3, 0x89, 0x20, -- 0x34, 0x0b, 0xbe, 0x5d, 0x8d, 0x7d, 0xac, 0x91, 0xaa, 0x53, 0xa7, 0x1a, 0x51, 0xa6, 0xe6, 0x33, -- 0x90, 0x6a, 0xb7, 0x71, 0x19, 0x19, 0x54, 0xb0, 0x6d, 0xb9, 0x06, 0x92, 0x47, 0xdd, 0xf2, 0x5b, -- 0x53, 0xfc, 0x2c, 0x28, 0x8f, 0x7a, 0x44, 0xd3, 0x58, 0x78, 0x09, 0x25, 0xb1, 0x30, 0xd1, 0x15, -- 0x14, 0xea, 0x48, 0x62, 0x9c, 0x5f, 0x55, 0xa0, 0xa1, 0xbb, 0xd5, 0xb2, 0xca, 0xcc, 0xde, 0xb8, -- 0x2a, 0x51, 0xf9, 0x82, 0x92, 0xa5, 0xdf, 0x35, 0x35, 0x96, 0xf7, 0xf8, 0x2c, 0xd2, 0x91, 0xda, -- 0xa8, 0x76, 0x16, 0xa9, 0x10, 0xfd, 0x06, 0xf4, 0xf2, 0xa7, 0x52, 0xcd, 0x6b, 0x15, 0xbb, 0x19, -- 0x56, 0x91, 0x5d, 0xa8, 0xa9, 0xf3, 0x13, 0x59, 0x5c, 0x67, 0x9d, 0xda, 0x0d, 0xa8, 0xa5, 0x99, -- 0x32, 0x72, 0x28, 0x31, 0xd3, 0xec, 0x91, 0x95, 0x43, 0x74, 0x1b, 0x7a, 0x7e, 0x18, 0x12, 0xb9, -- 0xdc, 0x9f, 0x3d, 0x24, 0x61, 0x76, 0x49, 0xcb, 0x58, 0xe7, 0x2f, 0x15, 0xe8, 0xef, 0xd3, 0x64, -- 0xfe, 0xff, 0x64, 0x86, 0x0b, 0x11, 0x44, 0x29, 0x69, 0xde, 0x58, 0x39, 0x96, 0x79, 0xe3, 0x33, -- 0x32, 0xc3, 0xfa, 0x6a, 0xe9, 0x9d, 0x6d, 0x4a, 0x84, 0xba, 0x56, 0x76, 0x32, 0x6b, 0x80, 0x75, -- 0xf5, 0xe4, 0x23, 0x1a, 0xaa, 0x0c, 0x39, 0x24, 0xcc, 0xcb, 0xda, 0x5d, 0x5d, 0x77, 0x3d, 0x24, -- 0x4c, 0x4d, 0x19, 0x43, 0xd6, 0x54, 0xc7, 0xb5, 0x68, 0x48, 0x43, 0x63, 0xa4, 0x21, 0xdb, 0xd0, -- 0xa0, 0xcf, 0x9e, 0x71, 0x2c, 0x54, 0x2e, 0x5b, 0x73, 0x0d, 0x94, 0x85, 0xb9, 0x66, 0x21, 0xcc, -- 0x5d, 0x81, 0x4d, 0xd5, 0xdb, 0x7f, 0xc2, 0xfc, 0x80, 0xc4, 0x53, 0x1b, 0x8a, 0xb7, 0x00, 0x4d, -- 0x04, 0x4d, 0x16, 0xb0, 0x63, 0x18, 0x98, 0x37, 0xe7, 0xe8, 0xc7, 0x13, 0x6b, 0xfa, 0x35, 0x68, -- 0x4a, 0xd0, 0x63, 0xf8, 0x6b, 0x1b, 0x18, 0xcd, 0xb4, 0xf3, 0x16, 0x74, 0xf4, 0xd0, 0x84, 0x81, -- 0x9c, 0x94, 0x97, 0x49, 0xf9, 0x9d, 0xbf, 0x6d, 0x98, 0x70, 0x6b, 0x6a, 0x68, 0xf4, 0x10, 0xfa, -- 0x0b, 0xff, 0x64, 0x90, 0x69, 0xaa, 0xac, 0xfe, 0x55, 0x33, 0xda, 0x1e, 0xeb, 0x7f, 0x3c, 0x63, -- 0xfb, 0x8f, 0x67, 0xfc, 0x20, 0x4a, 0xc4, 0x1c, 0x3d, 0x80, 0x5e, 0xf9, 0xef, 0x05, 0xba, 0x6e, -- 0x73, 0x90, 0x15, 0xff, 0x34, 0x2e, 0x64, 0xf3, 0x10, 0xfa, 0x0b, 0x3f, 0x32, 0xac, 0x3e, 0xab, -- 0xff, 0x6f, 0x5c, 0xc8, 0xe8, 0x2e, 0xb4, 0x0b, 0x7f, 0x2e, 0xd0, 0x50, 0x33, 0x59, 0xfe, 0x99, -- 0x71, 0x21, 0x83, 0x7d, 0xe8, 0x96, 0x7e, 0x26, 0xa0, 0x91, 0xb1, 0x67, 0xc5, 0x1f, 0x86, 0x0b, -- 0x99, 0xec, 0x41, 0xbb, 0xd0, 0xd3, 0xb7, 0x5a, 0x2c, 0xff, 0x38, 0x18, 0x5d, 0x5b, 0x31, 0x63, -- 0xb6, 0xf3, 0x00, 0xba, 0xa5, 0x0e, 0xbc, 0x55, 0x64, 0x55, 0xf7, 0x7f, 0x74, 0x7d, 0xe5, 0x9c, -- 0xe1, 0xf4, 0x10, 0xfa, 0x0b, 0xfd, 0x78, 0xeb, 0xdc, 0xd5, 0x6d, 0xfa, 0x0b, 0xcd, 0xfa, 0x5c, -- 0x6d, 0x76, 0xa1, 0xdc, 0x2a, 0x6c, 0xf6, 0x72, 0xf7, 0x7d, 0x74, 0x63, 0xf5, 0xa4, 0xd1, 0xea, -- 0x01, 0xf4, 0xca, 0x8d, 0x77, 0xcb, 0x6c, 0x65, 0x3b, 0xfe, 0xf2, 0x93, 0x53, 0xea, 0xc1, 0xe7, -- 0x27, 0x67, 0x55, 0x6b, 0xfe, 0x42, 0x46, 0xf7, 0x00, 0x4c, 0x71, 0x15, 0x92, 0x38, 0xdb, 0xb2, -- 0xa5, 0xa2, 0x2e, 0xdb, 0xb2, 0x15, 0x85, 0xd8, 0x5d, 0x00, 0x5d, 0x13, 0x85, 0x34, 0x15, 0xe8, -- 0xaa, 0x55, 0x63, 0xa1, 0x10, 0x1b, 0x0d, 0x97, 0x27, 0x96, 0x18, 0x60, 0xc6, 0x5e, 0x84, 0xc1, -- 0x67, 0x00, 0x79, 0xad, 0x65, 0x19, 0x2c, 0x55, 0x5f, 0x97, 0xf8, 0xa0, 0x53, 0xac, 0xac, 0x90, -- 0xb1, 0x75, 0x45, 0xb5, 0x75, 0x09, 0x8b, 0xfe, 0x42, 0xe6, 0x5c, 0x3e, 0x6c, 0x8b, 0x09, 0xf5, -- 0x68, 0x29, 0x7b, 0x46, 0x1f, 0x42, 0xa7, 0x98, 0x32, 0x5b, 0x2d, 0x56, 0xa4, 0xd1, 0xa3, 0x52, -- 0xda, 0x8c, 0xee, 0x42, 0xaf, 0x9c, 0x10, 0xa3, 0xc2, 0xbd, 0x58, 0x4a, 0x93, 0x47, 0xa6, 0x19, -- 0x54, 0x20, 0x7f, 0x1f, 0x20, 0x4f, 0x9c, 0xad, 0xfb, 0x96, 0x52, 0xe9, 0x05, 0xa9, 0x9f, 0x41, -- 0xaf, 0x10, 0xb7, 0x65, 0x4d, 0x78, 0xb5, 0x64, 0x70, 0x1e, 0xcd, 0x47, 0x26, 0xc3, 0x2a, 0x85, -- 0xed, 0x7b, 0xd0, 0x29, 0xbe, 0x11, 0xd6, 0xda, 0x15, 0xef, 0xc6, 0x65, 0x41, 0xaf, 0xf0, 0x9e, -- 0xd8, 0xb3, 0xbb, 0xfc, 0xc4, 0x5c, 0x16, 0xf4, 0x4a, 0xf5, 0xa8, 0x8d, 0x35, 0xab, 0x8a, 0xd4, -- 0xcb, 0x9e, 0x82, 0x72, 0xf1, 0x66, 0xbd, 0xbf, 0xb2, 0xa4, 0xbb, 0xec, 0x0c, 0x16, 0xeb, 0x14, -- 0xeb, 0x8f, 0x15, 0xb5, 0xcb, 0xf7, 0xc4, 0x84, 0x62, 0x2d, 0x52, 0x88, 0x09, 0x2b, 0x4a, 0x94, -- 0x0b, 0x19, 0x1d, 0x40, 0xff, 0xa1, 0x4d, 0x33, 0x4d, 0x0a, 0x6c, 0xd4, 0x59, 0x91, 0xf2, 0x8f, -- 0x46, 0xab, 0xa6, 0xcc, 0x2e, 0x7f, 0x0e, 0x83, 0xa5, 0xf4, 0x17, 0xdd, 0xcc, 0x5a, 0x9e, 0x2b, -- 0xf3, 0xe2, 0x0b, 0xd5, 0x3a, 0x84, 0x8d, 0xc5, 0xec, 0x17, 0xbd, 0x62, 0x36, 0x7d, 0x75, 0x56, -- 0x7c, 0x21, 0xab, 0x8f, 0xa1, 0x69, 0xb3, 0x2d, 0x64, 0x5a, 0xcb, 0x0b, 0xd9, 0xd7, 0x45, 0x4b, -- 0xf7, 0x3a, 0xdf, 0x7e, 0x77, 0xb3, 0xf2, 0xd7, 0xef, 0x6e, 0x56, 0xfe, 0xf1, 0xdd, 0xcd, 0xca, -- 0x71, 0x43, 0xcd, 0xbe, 0xff, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x45, 0xa8, 0x91, 0xab, 0x62, -- 0x22, 0x00, 0x00, -+ // 3183 bytes of a gzipped FileDescriptorProto -+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0x4b, 0x6f, 0x1c, 0xc7, -+ 0x99, 0x98, 0x07, 0x39, 0x33, 0xdf, 0xbc, 0xc8, 0x22, 0x45, 0x8d, 0x46, 0xb2, 0x2c, 0xb7, 0x6d, -+ 0x99, 0x5e, 0xaf, 0x49, 0x8b, 0x36, 0xfc, 0x84, 0x57, 0x10, 0x29, 0xad, 0xc8, 0xb5, 0xb4, 0xe2, -+ 0xf6, 0x88, 0xeb, 0x5d, 0x2c, 0x16, 0x8d, 0x66, 0x77, 0x71, 0x58, 0xe6, 0x74, 0x57, 0xbb, 0xaa, -+ 0x9a, 0xe2, 0x78, 0x81, 0x3d, 0x26, 0xd7, 0x9c, 0x72, 0xcb, 0x1f, 0x08, 0x72, 0xcb, 0x2d, 0xb9, -+ 0xe6, 0x60, 0x04, 0x39, 0x04, 0xf9, 0x01, 0x41, 0xe0, 0x9f, 0x90, 0x5f, 0x10, 0xd4, 0xab, 0x1f, -+ 0x33, 0x43, 0x1a, 0x11, 0x04, 0xe4, 0xd2, 0xe8, 0xef, 0x51, 0xdf, 0xab, 0xaa, 0xbe, 0xfa, 0xbe, -+ 0x2a, 0x68, 0xfb, 0x63, 0x1c, 0x8b, 0xad, 0x84, 0x51, 0x41, 0x51, 0x7d, 0xcc, 0x92, 0x60, 0xd8, -+ 0xa2, 0x01, 0xd1, 0x88, 0xe1, 0xc7, 0x63, 0x22, 0x4e, 0xd3, 0xe3, 0xad, 0x80, 0x46, 0xdb, 0x67, -+ 0xbe, 0xf0, 0xdf, 0x0f, 0x68, 0x2c, 0x7c, 0x12, 0x63, 0xc6, 0xb7, 0xd5, 0xc0, 0xed, 0xe4, 0x6c, -+ 0xbc, 0x2d, 0xa6, 0x09, 0xe6, 0xfa, 0x6b, 0xc6, 0xdd, 0x1c, 0x53, 0x3a, 0x9e, 0xe0, 0x6d, 0x05, -+ 0x1d, 0xa7, 0x27, 0xdb, 0x38, 0x4a, 0xc4, 0x54, 0x13, 0x9d, 0x5f, 0x54, 0x61, 0x63, 0x8f, 0x61, -+ 0x5f, 0xe0, 0x3d, 0x2b, 0xcd, 0xc5, 0xdf, 0xa6, 0x98, 0x0b, 0xf4, 0x06, 0x74, 0x32, 0x0d, 0x1e, -+ 0x09, 0x07, 0x95, 0x3b, 0x95, 0xcd, 0x96, 0xdb, 0xce, 0x70, 0x07, 0x21, 0xba, 0x0e, 0x0d, 0x7c, -+ 0x81, 0x03, 0x49, 0xad, 0x2a, 0xea, 0xb2, 0x04, 0x0f, 0x42, 0x74, 0x0f, 0xda, 0x5c, 0x30, 0x12, -+ 0x8f, 0xbd, 0x94, 0x63, 0x36, 0xa8, 0xdd, 0xa9, 0x6c, 0xb6, 0x77, 0x56, 0xb6, 0xa4, 0x4b, 0x5b, -+ 0x23, 0x45, 0x38, 0xe2, 0x98, 0xb9, 0xc0, 0xb3, 0x7f, 0x74, 0x17, 0x1a, 0x21, 0x3e, 0x27, 0x01, -+ 0xe6, 0x83, 0xfa, 0x9d, 0xda, 0x66, 0x7b, 0xa7, 0xa3, 0xd9, 0x1f, 0x2a, 0xa4, 0x6b, 0x89, 0xe8, -+ 0x5d, 0x68, 0x72, 0x41, 0x99, 0x3f, 0xc6, 0x7c, 0xb0, 0xa4, 0x18, 0xbb, 0x56, 0xae, 0xc2, 0xba, -+ 0x19, 0x19, 0xdd, 0x82, 0xda, 0xb3, 0xbd, 0x83, 0xc1, 0xb2, 0xd2, 0x0e, 0x86, 0x2b, 0xc1, 0x81, -+ 0x2b, 0xd1, 0xe8, 0x4d, 0xe8, 0x72, 0x3f, 0x0e, 0x8f, 0xe9, 0x85, 0x97, 0x90, 0x30, 0xe6, 0x83, -+ 0xc6, 0x9d, 0xca, 0x66, 0xd3, 0xed, 0x18, 0xe4, 0xa1, 0xc4, 0x39, 0x9f, 0xc3, 0xb5, 0x91, 0xf0, -+ 0x99, 0x78, 0x89, 0xe8, 0x38, 0x47, 0xb0, 0xe1, 0xe2, 0x88, 0x9e, 0xbf, 0x54, 0x68, 0x07, 0xd0, -+ 0x10, 0x24, 0xc2, 0x34, 0x15, 0x2a, 0xb4, 0x5d, 0xd7, 0x82, 0xce, 0xaf, 0x2a, 0x80, 0x1e, 0x5d, -+ 0xe0, 0xe0, 0x90, 0xd1, 0x00, 0x73, 0xfe, 0x0f, 0x9a, 0xae, 0x77, 0xa0, 0x91, 0x68, 0x03, 0x06, -+ 0x75, 0xc5, 0x6e, 0x66, 0xc1, 0x5a, 0x65, 0xa9, 0xce, 0x37, 0xb0, 0x3e, 0x22, 0xe3, 0xd8, 0x9f, -+ 0xbc, 0x42, 0x7b, 0x37, 0x60, 0x99, 0x2b, 0x99, 0xca, 0xd4, 0xae, 0x6b, 0x20, 0xe7, 0x10, 0xd0, -+ 0xd7, 0x3e, 0x11, 0xaf, 0x4e, 0x93, 0xf3, 0x3e, 0xac, 0x95, 0x24, 0xf2, 0x84, 0xc6, 0x1c, 0x2b, -+ 0x03, 0x84, 0x2f, 0x52, 0xae, 0x84, 0x2d, 0xb9, 0x06, 0x72, 0x30, 0xac, 0x3f, 0x21, 0xdc, 0xb2, -+ 0xe3, 0xbf, 0xc7, 0x84, 0x0d, 0x58, 0x3e, 0xa1, 0x2c, 0xf2, 0x85, 0xb5, 0x40, 0x43, 0x08, 0x41, -+ 0xdd, 0x67, 0x63, 0x3e, 0xa8, 0xdd, 0xa9, 0x6d, 0xb6, 0x5c, 0xf5, 0x2f, 0x57, 0xe5, 0x8c, 0x1a, -+ 0x63, 0xd7, 0x1b, 0xd0, 0x31, 0x71, 0xf7, 0x26, 0x84, 0x0b, 0xa5, 0xa7, 0xe3, 0xb6, 0x0d, 0x4e, -+ 0x8e, 0x71, 0x28, 0x6c, 0x1c, 0x25, 0xe1, 0x4b, 0x6e, 0xf8, 0x1d, 0x68, 0x31, 0xcc, 0x69, 0xca, -+ 0xe4, 0x36, 0xad, 0xaa, 0x79, 0x5f, 0xd7, 0xf3, 0xfe, 0x84, 0xc4, 0xe9, 0x85, 0x6b, 0x69, 0x6e, -+ 0xce, 0x66, 0xb6, 0x90, 0xe0, 0x2f, 0xb3, 0x85, 0x3e, 0x87, 0x6b, 0x87, 0x7e, 0xca, 0x5f, 0xc6, -+ 0x56, 0xe7, 0x0b, 0xb9, 0xfd, 0x78, 0x1a, 0xbd, 0xd4, 0xe0, 0x5f, 0x56, 0xa0, 0xb9, 0x97, 0xa4, -+ 0x47, 0xdc, 0x1f, 0x63, 0xf4, 0x3a, 0xb4, 0x05, 0x15, 0xfe, 0xc4, 0x4b, 0x25, 0xa8, 0xd8, 0xeb, -+ 0x2e, 0x28, 0x94, 0x66, 0x90, 0x61, 0xc7, 0x2c, 0x48, 0x52, 0xc3, 0x51, 0xbd, 0x53, 0xdb, 0xac, -+ 0xbb, 0x6d, 0x8d, 0xd3, 0x2c, 0x5b, 0xb0, 0xa6, 0x68, 0x1e, 0x89, 0xbd, 0x33, 0xcc, 0x62, 0x3c, -+ 0x89, 0x68, 0x88, 0xd5, 0xfa, 0xad, 0xbb, 0xab, 0x8a, 0x74, 0x10, 0x7f, 0x95, 0x11, 0xd0, 0x3f, -+ 0xc1, 0x6a, 0xc6, 0x2f, 0x37, 0xa5, 0xe2, 0xae, 0x2b, 0xee, 0xbe, 0xe1, 0x3e, 0x32, 0x68, 0xe7, -+ 0xff, 0xa1, 0xf7, 0xfc, 0x94, 0x51, 0x21, 0x26, 0x24, 0x1e, 0x3f, 0xf4, 0x85, 0x2f, 0xb3, 0x47, -+ 0x82, 0x19, 0xa1, 0x21, 0x37, 0xd6, 0x5a, 0x10, 0xbd, 0x07, 0xab, 0x42, 0xf3, 0xe2, 0xd0, 0xb3, -+ 0x3c, 0x55, 0xc5, 0xb3, 0x92, 0x11, 0x0e, 0x0d, 0xf3, 0xdb, 0xd0, 0xcb, 0x99, 0x65, 0xfe, 0x31, -+ 0xf6, 0x76, 0x33, 0xec, 0x73, 0x12, 0x61, 0xe7, 0x5c, 0xc5, 0x4a, 0x4d, 0x32, 0x7a, 0x0f, 0x5a, -+ 0x79, 0x1c, 0x2a, 0x6a, 0x85, 0xf4, 0xf4, 0x0a, 0xb1, 0xe1, 0x74, 0x9b, 0x59, 0x50, 0xbe, 0x84, -+ 0xbe, 0xc8, 0x0c, 0xf7, 0x42, 0x5f, 0xf8, 0xe5, 0x45, 0x55, 0xf6, 0xca, 0xed, 0x89, 0x12, 0xec, -+ 0x7c, 0x01, 0xad, 0x43, 0x12, 0x72, 0xad, 0x78, 0x00, 0x8d, 0x20, 0x65, 0x0c, 0xc7, 0xc2, 0xba, -+ 0x6c, 0x40, 0xb4, 0x0e, 0x4b, 0x13, 0x12, 0x11, 0x61, 0xdc, 0xd4, 0x80, 0x43, 0x01, 0x9e, 0xe2, -+ 0x88, 0xb2, 0xa9, 0x0a, 0xd8, 0x3a, 0x2c, 0x15, 0x27, 0x57, 0x03, 0xe8, 0x26, 0xb4, 0x22, 0xff, -+ 0x22, 0x9b, 0x54, 0x49, 0x69, 0x46, 0xfe, 0x85, 0x36, 0x7e, 0x00, 0x8d, 0x13, 0x9f, 0x4c, 0x82, -+ 0x58, 0x98, 0xa8, 0x58, 0x30, 0x57, 0x58, 0x2f, 0x2a, 0xfc, 0x5d, 0x15, 0xda, 0x5a, 0xa3, 0x36, -+ 0x78, 0x1d, 0x96, 0x02, 0x3f, 0x38, 0xcd, 0x54, 0x2a, 0x00, 0xdd, 0xb5, 0x86, 0x54, 0x8b, 0x49, -+ 0x38, 0xb7, 0xd4, 0x9a, 0xb6, 0x0d, 0xc0, 0x5f, 0xf8, 0x89, 0xb1, 0xad, 0x76, 0x09, 0x73, 0x4b, -+ 0xf2, 0x68, 0x73, 0x3f, 0x84, 0x8e, 0x5e, 0x77, 0x66, 0x48, 0xfd, 0x92, 0x21, 0x6d, 0xcd, 0xa5, -+ 0x07, 0xbd, 0x09, 0xdd, 0x94, 0x63, 0xef, 0x94, 0x60, 0xe6, 0xb3, 0xe0, 0x74, 0x3a, 0x58, 0xd2, -+ 0x67, 0x64, 0xca, 0xf1, 0xbe, 0xc5, 0xa1, 0x1d, 0x58, 0x92, 0xe9, 0x8f, 0x0f, 0x96, 0xd5, 0x71, -+ 0x7c, 0xab, 0x28, 0x52, 0xb9, 0xba, 0xa5, 0xbe, 0x8f, 0x62, 0xc1, 0xa6, 0xae, 0x66, 0x1d, 0x7e, -+ 0x0a, 0x90, 0x23, 0xd1, 0x0a, 0xd4, 0xce, 0xf0, 0xd4, 0xec, 0x43, 0xf9, 0x2b, 0x83, 0x73, 0xee, -+ 0x4f, 0x52, 0x1b, 0x75, 0x0d, 0x7c, 0x5e, 0xfd, 0xb4, 0xe2, 0x04, 0xd0, 0xdf, 0x9d, 0x9c, 0x11, -+ 0x5a, 0x18, 0xbe, 0x0e, 0x4b, 0x91, 0xff, 0x0d, 0x65, 0x36, 0x92, 0x0a, 0x50, 0x58, 0x12, 0x53, -+ 0x66, 0x45, 0x28, 0x00, 0xf5, 0xa0, 0x4a, 0x13, 0x15, 0xaf, 0x96, 0x5b, 0xa5, 0x49, 0xae, 0xa8, -+ 0x5e, 0x50, 0xe4, 0xfc, 0xb9, 0x0e, 0x90, 0x6b, 0x41, 0x2e, 0x0c, 0x09, 0xf5, 0x38, 0x66, 0xb2, -+ 0x04, 0xf1, 0x8e, 0xa7, 0x02, 0x73, 0x8f, 0xe1, 0x20, 0x65, 0x9c, 0x9c, 0xcb, 0xf9, 0x93, 0x6e, -+ 0x5f, 0xd3, 0x6e, 0xcf, 0xd8, 0xe6, 0x5e, 0x27, 0x74, 0xa4, 0xc7, 0xed, 0xca, 0x61, 0xae, 0x1d, -+ 0x85, 0x0e, 0xe0, 0x5a, 0x2e, 0x33, 0x2c, 0x88, 0xab, 0x5e, 0x25, 0x6e, 0x2d, 0x13, 0x17, 0xe6, -+ 0xa2, 0x1e, 0xc1, 0x1a, 0xa1, 0xde, 0xb7, 0x29, 0x4e, 0x4b, 0x82, 0x6a, 0x57, 0x09, 0x5a, 0x25, -+ 0xf4, 0x3f, 0xd4, 0x80, 0x5c, 0xcc, 0x21, 0xdc, 0x28, 0x78, 0x29, 0xb7, 0x7b, 0x41, 0x58, 0xfd, -+ 0x2a, 0x61, 0x1b, 0x99, 0x55, 0x32, 0x1f, 0xe4, 0x12, 0xff, 0x0d, 0x36, 0x08, 0xf5, 0x5e, 0xf8, -+ 0x44, 0xcc, 0x8a, 0x5b, 0xfa, 0x11, 0x27, 0xe5, 0xa1, 0x5b, 0x96, 0xa5, 0x9d, 0x8c, 0x30, 0x1b, -+ 0x97, 0x9c, 0x5c, 0xfe, 0x11, 0x27, 0x9f, 0xaa, 0x01, 0xb9, 0x98, 0x07, 0xb0, 0x4a, 0xe8, 0xac, -+ 0x35, 0x8d, 0xab, 0x84, 0xf4, 0x09, 0x2d, 0x5b, 0xb2, 0x0b, 0xab, 0x1c, 0x07, 0x82, 0xb2, 0xe2, -+ 0x22, 0x68, 0x5e, 0x25, 0x62, 0xc5, 0xf0, 0x67, 0x32, 0x9c, 0xff, 0x81, 0xce, 0x7e, 0x3a, 0xc6, -+ 0x62, 0x72, 0x9c, 0x25, 0x83, 0x57, 0x96, 0x7f, 0x9c, 0xbf, 0x56, 0xa1, 0xbd, 0x37, 0x66, 0x34, -+ 0x4d, 0x4a, 0x39, 0x59, 0x6f, 0xd2, 0xd9, 0x9c, 0xac, 0x58, 0x54, 0x4e, 0xd6, 0xcc, 0x1f, 0x41, -+ 0x27, 0x52, 0x5b, 0xd7, 0xf0, 0xeb, 0x3c, 0xb4, 0x3a, 0xb7, 0xa9, 0xdd, 0x76, 0x54, 0x48, 0x66, -+ 0x5b, 0x00, 0x09, 0x09, 0xb9, 0x19, 0xa3, 0xd3, 0x51, 0xdf, 0x54, 0x84, 0x36, 0x45, 0xbb, 0xad, -+ 0x24, 0xcb, 0xd6, 0xf7, 0xa0, 0x7d, 0x2c, 0x83, 0x64, 0x06, 0x94, 0x92, 0x51, 0x1e, 0x3d, 0x17, -+ 0x8e, 0xf3, 0x4d, 0xb8, 0x0f, 0xdd, 0x53, 0x1d, 0x32, 0x33, 0x48, 0xaf, 0xa1, 0x37, 0x8d, 0x27, -+ 0xb9, 0xbf, 0x5b, 0xc5, 0xc8, 0xea, 0x09, 0xe8, 0x9c, 0x16, 0x50, 0xc3, 0x11, 0xac, 0xce, 0xb1, -+ 0x2c, 0xc8, 0x41, 0x9b, 0xc5, 0x1c, 0xd4, 0xde, 0x41, 0x5a, 0x51, 0x71, 0x64, 0x31, 0x2f, 0xfd, -+ 0xac, 0x0a, 0xbd, 0x83, 0x58, 0x60, 0x76, 0xe2, 0x07, 0x58, 0x5b, 0x8c, 0xa0, 0x1e, 0xfb, 0x11, -+ 0x36, 0x32, 0xd5, 0x3f, 0xba, 0x01, 0x4d, 0x76, 0xa1, 0x53, 0x88, 0x99, 0xd1, 0x06, 0xbb, 0x50, -+ 0xa9, 0x01, 0xbd, 0x06, 0xc0, 0x2e, 0xbc, 0xc4, 0x0f, 0xce, 0xb0, 0x89, 0x61, 0xdd, 0x6d, 0xb1, -+ 0x8b, 0x43, 0x8d, 0x90, 0x8b, 0x81, 0x5d, 0x78, 0x98, 0x31, 0xca, 0xb8, 0xc9, 0x56, 0x4d, 0x76, -+ 0xf1, 0x48, 0xc1, 0x66, 0x6c, 0xc8, 0x68, 0x92, 0xe0, 0x50, 0x65, 0x69, 0x35, 0xf6, 0xa1, 0x46, -+ 0x48, 0xad, 0xc2, 0x6a, 0x5d, 0xd6, 0x5a, 0x45, 0xae, 0x55, 0xe4, 0x5a, 0x1b, 0x7a, 0xa4, 0x28, -+ 0x6a, 0x15, 0x99, 0xd6, 0xa6, 0xd6, 0x2a, 0x0a, 0x5a, 0x45, 0xae, 0xb5, 0x65, 0xc7, 0x1a, 0xad, -+ 0xce, 0x6f, 0xaa, 0xd0, 0x78, 0x1e, 0xa8, 0x49, 0x41, 0x77, 0xa0, 0x8d, 0xb9, 0xf0, 0x8f, 0x27, -+ 0x84, 0x9f, 0xe2, 0xd0, 0x2c, 0xf3, 0x22, 0x4a, 0xda, 0xc8, 0xa7, 0xb1, 0xc7, 0xe5, 0x09, 0x6e, -+ 0x22, 0xc3, 0xa7, 0xf1, 0x48, 0x9e, 0xe0, 0x86, 0xc4, 0x70, 0x70, 0x6e, 0xd7, 0x3a, 0x9f, 0xc6, -+ 0x2e, 0x0e, 0xce, 0xa5, 0x7d, 0x27, 0x24, 0x56, 0x39, 0xe6, 0x9e, 0x8d, 0xca, 0x09, 0x89, 0x65, -+ 0xfe, 0xb8, 0x57, 0x24, 0xee, 0x98, 0xa0, 0x58, 0xe2, 0x8e, 0xf2, 0x4c, 0xa6, 0x01, 0x49, 0x35, -+ 0x41, 0x69, 0x4a, 0x84, 0xa4, 0xaa, 0xc3, 0x79, 0x42, 0x39, 0x36, 0x01, 0xd1, 0x80, 0xf4, 0x57, -+ 0xfd, 0xe8, 0x31, 0x3a, 0x1a, 0x2d, 0x85, 0x51, 0x83, 0x6e, 0x40, 0x73, 0xe2, 0x73, 0xe1, 0xf9, -+ 0xc1, 0x99, 0x09, 0x46, 0x43, 0xc2, 0x0f, 0x82, 0x33, 0x59, 0xdd, 0xcb, 0x82, 0x1c, 0xc7, 0x03, -+ 0x50, 0x04, 0x03, 0xa9, 0xaa, 0x65, 0x42, 0x39, 0x89, 0xc7, 0x83, 0xb6, 0xa9, 0x5a, 0x34, 0xe8, -+ 0xa4, 0xd0, 0x38, 0x0a, 0x75, 0xec, 0xf2, 0xc1, 0x95, 0xd9, 0xc1, 0x36, 0xf6, 0x26, 0x60, 0x06, -+ 0x34, 0x6b, 0x45, 0x9f, 0x08, 0x26, 0x62, 0x4d, 0x76, 0xa1, 0x13, 0xbe, 0x99, 0x52, 0x43, 0xac, -+ 0xdb, 0x29, 0xd5, 0x44, 0xe7, 0x0f, 0x15, 0xe8, 0xfc, 0x3b, 0x16, 0x2f, 0x28, 0x3b, 0xb3, 0xf9, -+ 0x00, 0x88, 0x5d, 0xd6, 0xdc, 0x9c, 0x75, 0xa6, 0x3c, 0x2b, 0x2f, 0x77, 0xb7, 0xc0, 0x87, 0x5e, -+ 0x87, 0x9a, 0x08, 0x12, 0xb3, 0x73, 0x4c, 0x6b, 0x68, 0x96, 0x82, 0x2b, 0x29, 0xe8, 0x0d, 0xa8, -+ 0x8b, 0x20, 0xf9, 0xd8, 0xa4, 0x8a, 0x19, 0x0e, 0x45, 0x92, 0x32, 0xd2, 0x30, 0x29, 0xb7, 0x97, -+ 0x26, 0x24, 0xae, 0xa4, 0x48, 0x19, 0x69, 0x98, 0x7c, 0xac, 0x66, 0x76, 0x8e, 0x43, 0x91, 0x9c, -+ 0x9f, 0x56, 0x60, 0x63, 0xb6, 0xfb, 0x30, 0xbd, 0xd2, 0x47, 0xd0, 0x09, 0x54, 0xd2, 0x28, 0x25, -+ 0xc6, 0xd5, 0xb9, 0x74, 0xe2, 0xb6, 0x83, 0x42, 0x2e, 0xfd, 0x04, 0xba, 0xb1, 0x0e, 0x4f, 0x29, -+ 0x3f, 0x9a, 0xe4, 0x50, 0x8c, 0x9c, 0xdb, 0x89, 0x0b, 0x90, 0x13, 0x02, 0xfa, 0x9a, 0x11, 0x81, -+ 0x47, 0x82, 0x61, 0x3f, 0x7a, 0x15, 0x5d, 0x30, 0x82, 0xba, 0x2a, 0x99, 0x6b, 0xaa, 0xc9, 0x53, -+ 0xff, 0xce, 0x3b, 0xb0, 0x56, 0xd2, 0x62, 0x7c, 0x5d, 0x81, 0xda, 0xc4, 0x2c, 0x9f, 0xae, 0x2b, -+ 0x7f, 0x1d, 0x1f, 0x56, 0x5d, 0xec, 0x87, 0xaf, 0xce, 0x1a, 0xa3, 0xa2, 0x96, 0xab, 0xd8, 0x04, -+ 0x54, 0x54, 0x61, 0x4c, 0xb1, 0x56, 0x57, 0x0a, 0x56, 0x3f, 0x83, 0xd5, 0x3d, 0xb9, 0x8b, 0x46, -+ 0x22, 0x24, 0xf1, 0xab, 0x68, 0xdb, 0xff, 0x0f, 0xd6, 0x9e, 0x8b, 0xe9, 0xd7, 0x52, 0x18, 0x27, -+ 0xdf, 0xe1, 0x57, 0xe4, 0x1f, 0xa3, 0x2f, 0xac, 0x7f, 0x8c, 0xbe, 0x90, 0xdb, 0x32, 0xa0, 0x93, -+ 0x34, 0x8a, 0xd5, 0x12, 0xed, 0xba, 0x06, 0x72, 0x76, 0xa1, 0xa3, 0x1b, 0xb9, 0xa7, 0x34, 0x4c, -+ 0x27, 0x78, 0xe1, 0x31, 0x70, 0x1b, 0x20, 0xf1, 0x99, 0x1f, 0x61, 0x81, 0x19, 0x57, 0x25, 0x5f, -+ 0xcb, 0x2d, 0x60, 0x9c, 0x9f, 0x57, 0x61, 0x5d, 0xdf, 0xcb, 0x8d, 0xf4, 0x75, 0x94, 0x75, 0x61, -+ 0x08, 0xcd, 0x53, 0xca, 0x45, 0x41, 0x60, 0x06, 0x4b, 0x13, 0xc3, 0xd8, 0x4a, 0x93, 0xbf, 0xa5, -+ 0xcb, 0xb2, 0xda, 0xd5, 0x97, 0x65, 0x73, 0xd7, 0x61, 0xf5, 0xf9, 0xeb, 0x30, 0x99, 0x00, 0x2d, -+ 0x13, 0xd1, 0xc7, 0x4c, 0xcb, 0x6d, 0x19, 0xcc, 0x41, 0x88, 0xee, 0x42, 0x7f, 0x2c, 0xad, 0xf4, -+ 0x4e, 0x29, 0x3d, 0xf3, 0x12, 0x5f, 0x9c, 0xaa, 0xc4, 0xda, 0x72, 0xbb, 0x0a, 0xbd, 0x4f, 0xe9, -+ 0xd9, 0xa1, 0x2f, 0x4e, 0xd1, 0x67, 0xd0, 0x33, 0xbd, 0x48, 0xa4, 0x42, 0xc4, 0x4d, 0x05, 0x66, -+ 0x76, 0x51, 0x31, 0x7a, 0x6e, 0xf7, 0xac, 0x00, 0x71, 0xe7, 0x3a, 0x5c, 0x7b, 0x88, 0xb9, 0x60, -+ 0x74, 0x5a, 0x0e, 0x8c, 0xf3, 0x2f, 0x00, 0x07, 0x79, 0xfe, 0xf9, 0xa0, 0x08, 0x99, 0xac, 0xb5, -+ 0xb2, 0xa5, 0xaf, 0x45, 0x33, 0x82, 0x5b, 0xe0, 0x71, 0xb6, 0x60, 0xd9, 0xa5, 0xa9, 0x3c, 0x11, -+ 0xdf, 0xb2, 0x7f, 0x66, 0x5c, 0xc7, 0x8c, 0x53, 0x48, 0xd7, 0xd0, 0x9c, 0x7d, 0x7b, 0x8f, 0x92, -+ 0x8b, 0x33, 0x53, 0xb4, 0x05, 0xad, 0x2c, 0x13, 0x9a, 0xac, 0x32, 0xaf, 0x3a, 0x67, 0x71, 0xfe, -+ 0x1b, 0xd6, 0xb4, 0x24, 0x2d, 0xd9, 0x8a, 0x79, 0x0b, 0x96, 0x99, 0x35, 0xa3, 0x92, 0xdf, 0x87, -+ 0x1a, 0x26, 0x43, 0x43, 0xb7, 0xa4, 0xb2, 0x80, 0xe1, 0xc8, 0x1e, 0x9b, 0x4d, 0x37, 0x47, 0xc8, -+ 0x68, 0x3d, 0x21, 0x5c, 0xe4, 0x6e, 0xda, 0x68, 0xad, 0xc1, 0xaa, 0x24, 0x94, 0x34, 0x3a, 0xff, -+ 0x0b, 0x6b, 0xcf, 0xe2, 0x09, 0x89, 0xf1, 0xde, 0xe1, 0xd1, 0x53, 0x9c, 0x65, 0x05, 0x04, 0x75, -+ 0x75, 0xde, 0x55, 0x94, 0x74, 0xf5, 0x2f, 0xb7, 0x49, 0x7c, 0xec, 0x05, 0x49, 0xca, 0xcd, 0xf5, -+ 0xe4, 0x72, 0x7c, 0xbc, 0x97, 0xa4, 0x5c, 0x9e, 0x81, 0xb2, 0xd6, 0xa4, 0xf1, 0x64, 0xaa, 0xf6, -+ 0x4a, 0xd3, 0x6d, 0x04, 0x49, 0xfa, 0x2c, 0x9e, 0x4c, 0x9d, 0x7f, 0x56, 0x17, 0x32, 0x18, 0x87, -+ 0xae, 0x1f, 0x87, 0x34, 0x7a, 0x88, 0xcf, 0x0b, 0x1a, 0xb2, 0xe6, 0xdf, 0xe6, 0x84, 0xef, 0x2b, -+ 0xd0, 0x79, 0x30, 0xc6, 0xb1, 0x78, 0x88, 0x85, 0x4f, 0x26, 0xaa, 0xc1, 0x3f, 0xc7, 0x8c, 0x13, -+ 0x1a, 0x9b, 0x85, 0x6f, 0x41, 0xf4, 0x3a, 0xb4, 0x49, 0x4c, 0x84, 0x17, 0xfa, 0x38, 0xa2, 0xb1, -+ 0x89, 0x02, 0x48, 0xd4, 0x43, 0x85, 0x41, 0xef, 0x40, 0x5f, 0x5f, 0x1f, 0x7b, 0xa7, 0x7e, 0x1c, -+ 0x4e, 0xe4, 0x96, 0xd3, 0xd7, 0x69, 0x3d, 0x8d, 0xde, 0x37, 0x58, 0xf4, 0x2e, 0xac, 0x98, 0x0d, -+ 0x91, 0x73, 0xd6, 0x15, 0x67, 0xdf, 0xe0, 0x4b, 0xac, 0x69, 0x92, 0x50, 0x26, 0xb8, 0xc7, 0x71, -+ 0x10, 0xd0, 0x28, 0x31, 0xdd, 0x71, 0xdf, 0xe2, 0x47, 0x1a, 0xed, 0x8c, 0x61, 0xed, 0xb1, 0xf4, -+ 0xd3, 0x78, 0x92, 0x4f, 0x70, 0x2f, 0xc2, 0x91, 0x77, 0x3c, 0xa1, 0xc1, 0x99, 0x27, 0xd3, 0x94, -+ 0x89, 0xb0, 0xac, 0xbf, 0x77, 0x25, 0x72, 0x44, 0xbe, 0x53, 0x17, 0x41, 0x92, 0xeb, 0x94, 0x8a, -+ 0x64, 0x92, 0x8e, 0xbd, 0x84, 0xd1, 0x63, 0x6c, 0x5c, 0xec, 0x47, 0x38, 0xda, 0xd7, 0xf8, 0x43, -+ 0x89, 0x76, 0x7e, 0x5b, 0x81, 0xf5, 0xb2, 0x26, 0x93, 0x74, 0xb7, 0x61, 0xbd, 0xac, 0xca, 0xd4, -+ 0x82, 0xba, 0x9e, 0x58, 0x2d, 0x2a, 0xd4, 0x55, 0xe1, 0x27, 0xd0, 0x55, 0x6f, 0x0a, 0x5e, 0xa8, -+ 0x25, 0x95, 0x8f, 0xb9, 0xe2, 0xbc, 0xb8, 0x1d, 0xbf, 0x38, 0x4b, 0x9f, 0xc1, 0x0d, 0xe3, 0xbe, -+ 0x37, 0x6f, 0xb6, 0x5e, 0x10, 0x1b, 0x86, 0xe1, 0xe9, 0x8c, 0xf5, 0x4f, 0x60, 0x90, 0xa3, 0x76, -+ 0xa7, 0x0a, 0x69, 0x63, 0xf5, 0x01, 0xac, 0xcd, 0x38, 0xfb, 0x20, 0x0c, 0x99, 0xda, 0xa0, 0x75, -+ 0x77, 0x11, 0xc9, 0xb9, 0x0f, 0xd7, 0x47, 0x58, 0xe8, 0x68, 0xf8, 0xc2, 0x34, 0xa6, 0x5a, 0xd8, -+ 0x0a, 0xd4, 0x46, 0x38, 0x50, 0xce, 0xd7, 0x5c, 0xf9, 0x2b, 0x17, 0xe0, 0x11, 0xc7, 0x81, 0xf2, -+ 0xb2, 0xe6, 0xaa, 0x7f, 0xe7, 0xd7, 0x15, 0x68, 0x98, 0x34, 0x29, 0x53, 0x7d, 0xc8, 0xc8, 0x39, -+ 0x66, 0x66, 0xe9, 0x19, 0x08, 0xbd, 0x0d, 0x3d, 0xfd, 0xe7, 0xd1, 0x44, 0x10, 0x9a, 0x25, 0xdf, -+ 0xae, 0xc6, 0x3e, 0xd3, 0x48, 0x75, 0x5d, 0xac, 0x6e, 0x43, 0xcd, 0xc5, 0x83, 0x81, 0xd4, 0x9d, -+ 0x2f, 0x97, 0x99, 0x41, 0x25, 0xdb, 0x96, 0x6b, 0x20, 0xb9, 0xd4, 0xad, 0xbc, 0x25, 0x25, 0xcf, -+ 0x82, 0x72, 0xa9, 0x47, 0x34, 0x8d, 0x85, 0x97, 0x50, 0x12, 0x0b, 0x93, 0x5d, 0x41, 0xa1, 0x0e, -+ 0x25, 0xc6, 0xf9, 0x49, 0x05, 0x96, 0xf5, 0x93, 0x09, 0xea, 0x41, 0x35, 0x3b, 0xe3, 0xaa, 0x44, -+ 0xd5, 0x0b, 0x4a, 0x97, 0x3e, 0xd7, 0xd4, 0xbf, 0xdc, 0xc7, 0xe7, 0x91, 0xce, 0xd4, 0xc6, 0xb4, -+ 0xf3, 0x48, 0xa5, 0xe8, 0xb7, 0xa1, 0x97, 0x1f, 0x95, 0x8a, 0xae, 0x4d, 0xec, 0x66, 0x58, 0xc5, -+ 0x76, 0xa9, 0xa5, 0xce, 0x7f, 0x01, 0xe4, 0x4f, 0x07, 0x32, 0xe4, 0x69, 0x66, 0x8c, 0xfc, 0x95, -+ 0x98, 0x71, 0x76, 0xc8, 0xca, 0x5f, 0x74, 0x17, 0x7a, 0x7e, 0x18, 0x12, 0x39, 0xdc, 0x9f, 0x3c, -+ 0x26, 0x61, 0xb6, 0x49, 0xcb, 0x58, 0xe7, 0xf7, 0x15, 0xe8, 0xef, 0xd1, 0x64, 0xfa, 0xaf, 0x64, -+ 0x82, 0x0b, 0x19, 0x44, 0x19, 0x69, 0xce, 0x58, 0xf9, 0xaf, 0xab, 0xff, 0x09, 0xd6, 0x5b, 0x4b, -+ 0xcf, 0x6c, 0x53, 0x22, 0xd4, 0xb6, 0xb2, 0xc4, 0xec, 0x16, 0xb6, 0xab, 0x89, 0x4f, 0x69, 0xa8, -+ 0x9a, 0xb4, 0x90, 0x30, 0x2f, 0xbb, 0x73, 0xed, 0xba, 0x8d, 0x90, 0x30, 0x45, 0x32, 0x8e, 0x2c, -+ 0xa9, 0x6b, 0xff, 0xa2, 0x23, 0xcb, 0x1a, 0x23, 0x1d, 0xd9, 0x80, 0x65, 0x7a, 0x72, 0xc2, 0xb1, -+ 0x50, 0xdd, 0x43, 0xcd, 0x35, 0x50, 0x96, 0xe6, 0x9a, 0x85, 0x34, 0x77, 0x0d, 0xd6, 0xd4, 0x03, -+ 0xd3, 0x73, 0xe6, 0x07, 0x24, 0x1e, 0xdb, 0x54, 0xbc, 0x0e, 0x68, 0x24, 0x68, 0x32, 0x83, 0xdd, -+ 0x82, 0x55, 0x73, 0xe6, 0x1c, 0xfe, 0xe7, 0xc8, 0xba, 0x7e, 0x03, 0x9a, 0x12, 0xf4, 0x18, 0xfe, -+ 0xd6, 0x26, 0x46, 0x43, 0x76, 0xde, 0x85, 0x8e, 0xfe, 0x35, 0x69, 0x20, 0x67, 0xe5, 0x65, 0x56, -+ 0xbe, 0xf3, 0xa7, 0x15, 0x93, 0x6e, 0xcd, 0x45, 0x0e, 0x7a, 0x0c, 0xfd, 0x99, 0x87, 0x41, 0x64, -+ 0x6e, 0xf6, 0x16, 0xbf, 0x17, 0x0e, 0x37, 0xb6, 0xf4, 0x43, 0xe3, 0x96, 0x7d, 0x68, 0xdc, 0x7a, -+ 0x14, 0x25, 0x62, 0x8a, 0x1e, 0x41, 0xaf, 0xfc, 0x84, 0x86, 0x6e, 0xda, 0x1a, 0x64, 0xc1, 0xc3, -+ 0xda, 0xa5, 0x62, 0x1e, 0x43, 0x7f, 0xe6, 0x35, 0xcd, 0xda, 0xb3, 0xf8, 0x91, 0xed, 0x52, 0x41, -+ 0xf7, 0xa1, 0x5d, 0x78, 0x3e, 0x43, 0x03, 0x2d, 0x64, 0xfe, 0x45, 0xed, 0x52, 0x01, 0x7b, 0xd0, -+ 0x2d, 0xbd, 0x68, 0xa1, 0xa1, 0xf1, 0x67, 0xc1, 0x33, 0xd7, 0xa5, 0x42, 0x76, 0xa1, 0x5d, 0x78, -+ 0x58, 0xb2, 0x56, 0xcc, 0xbf, 0x5e, 0x0d, 0x6f, 0x2c, 0xa0, 0x98, 0xe9, 0xdc, 0x87, 0x6e, 0xe9, -+ 0x19, 0xc8, 0x1a, 0xb2, 0xe8, 0x09, 0x6a, 0x78, 0x73, 0x21, 0xcd, 0x48, 0x7a, 0x0c, 0xfd, 0x99, -+ 0x47, 0x21, 0x1b, 0xdc, 0xc5, 0x6f, 0x45, 0x97, 0xba, 0xf5, 0x95, 0x9a, 0xec, 0x42, 0xbb, 0x55, -+ 0x98, 0xec, 0xf9, 0x27, 0xa0, 0xe1, 0xad, 0xc5, 0x44, 0x63, 0xd5, 0x23, 0xe8, 0x95, 0x5f, 0x7f, -+ 0xac, 0xb0, 0x85, 0x6f, 0x42, 0x57, 0xaf, 0x9c, 0xd2, 0x43, 0x50, 0xbe, 0x72, 0x16, 0xbd, 0x0f, -+ 0x5d, 0x2a, 0xe8, 0x01, 0x80, 0x69, 0xae, 0x42, 0x12, 0x67, 0x53, 0x36, 0xd7, 0xd4, 0x65, 0x53, -+ 0xb6, 0xa0, 0x11, 0xbb, 0x0f, 0xa0, 0x7b, 0xa2, 0x90, 0xa6, 0x02, 0x5d, 0xb7, 0x66, 0xcc, 0x34, -+ 0x62, 0xc3, 0xc1, 0x3c, 0x61, 0x4e, 0x00, 0x66, 0xec, 0x65, 0x04, 0x7c, 0x09, 0x90, 0xf7, 0x5a, -+ 0x56, 0xc0, 0x5c, 0xf7, 0x75, 0x45, 0x0c, 0x3a, 0xc5, 0xce, 0x0a, 0x19, 0x5f, 0x17, 0x74, 0x5b, -+ 0x57, 0x88, 0xe8, 0xcf, 0x54, 0xce, 0xe5, 0xc5, 0x36, 0x5b, 0x50, 0x0f, 0xe7, 0xaa, 0x67, 0xf4, -+ 0x09, 0x74, 0x8a, 0x25, 0xb3, 0xb5, 0x62, 0x41, 0x19, 0x3d, 0x2c, 0x95, 0xcd, 0xe8, 0x3e, 0xf4, -+ 0xca, 0x05, 0x31, 0x2a, 0xec, 0x8b, 0xb9, 0x32, 0x79, 0xb8, 0x32, 0x73, 0xd1, 0xc1, 0xd1, 0x87, -+ 0x00, 0x79, 0xe1, 0x6c, 0xc3, 0x37, 0x57, 0x4a, 0xcf, 0x68, 0xfd, 0x12, 0x7a, 0x85, 0xbc, 0x2d, -+ 0x7b, 0xc2, 0xeb, 0x25, 0x87, 0xf3, 0x6c, 0x3e, 0x34, 0x15, 0x56, 0x29, 0x6d, 0x3f, 0x80, 0x4e, -+ 0xf1, 0x8c, 0xb0, 0xde, 0x2e, 0x38, 0x37, 0xae, 0x4a, 0x7a, 0x85, 0xf3, 0xc4, 0xae, 0xdd, 0xf9, -+ 0x23, 0xe6, 0xaa, 0xa4, 0x57, 0xea, 0x47, 0x6d, 0xae, 0x59, 0xd4, 0xa4, 0x5e, 0x75, 0x14, 0x94, -+ 0x9b, 0x37, 0x1b, 0xfd, 0x85, 0x2d, 0xdd, 0x55, 0x6b, 0xb0, 0xd8, 0xa7, 0xd8, 0x78, 0x2c, 0xe8, -+ 0x5d, 0x7e, 0x24, 0x27, 0x14, 0x7b, 0x91, 0x42, 0x4e, 0x58, 0xd0, 0xa2, 0x5c, 0x2a, 0x68, 0x1f, -+ 0xfa, 0x8f, 0x6d, 0x99, 0x69, 0x4a, 0x60, 0x63, 0xce, 0x82, 0x92, 0x7f, 0x38, 0x5c, 0x44, 0x32, -+ 0xb3, 0xfc, 0x15, 0xac, 0xce, 0x95, 0xbf, 0xe8, 0x76, 0x76, 0xef, 0xbe, 0xb0, 0x2e, 0xbe, 0xd4, -+ 0xac, 0x03, 0x58, 0x99, 0xad, 0x7e, 0xd1, 0x6b, 0x66, 0xd2, 0x17, 0x57, 0xc5, 0x97, 0x8a, 0xfa, -+ 0x0c, 0x9a, 0xb6, 0xda, 0x42, 0xe6, 0x7d, 0x63, 0xa6, 0xfa, 0xba, 0x6c, 0xe8, 0x6e, 0xe7, 0xfb, -+ 0x1f, 0x6e, 0x57, 0xfe, 0xf8, 0xc3, 0xed, 0xca, 0x5f, 0x7e, 0xb8, 0x5d, 0x39, 0x5e, 0x56, 0xd4, -+ 0x0f, 0xff, 0x16, 0x00, 0x00, 0xff, 0xff, 0x9a, 0x16, 0x45, 0x76, 0xe7, 0x24, 0x00, 0x00, - } -diff --git a/protocols/grpc/agent.proto b/protocols/grpc/agent.proto -index d863645..73f6b37 100644 ---- a/protocols/grpc/agent.proto -+++ b/protocols/grpc/agent.proto -@@ -222,7 +222,7 @@ message CgroupStats { - - } - --message NetworkStats { -+message InterfaceStats { - string name = 1; - uint64 rx_bytes = 2; - uint64 rx_packets = 3; -@@ -234,9 +234,38 @@ message NetworkStats { - uint64 tx_dropped = 9; - } - -+message TcpStat { -+ uint64 established = 1; -+ uint64 syn_sent = 2; -+ uint64 syn_recv = 3; -+ uint64 fin_wait1 = 4; -+ uint64 fin_wait2 = 5; -+ uint64 time_wait = 6; -+ uint64 close = 7; -+ uint64 close_wait = 8; -+ uint64 last_ack = 9; -+ uint64 listen = 10; -+ uint64 closing = 11; -+} -+ -+message UdpStat { -+ uint64 listen = 1; -+ uint64 dropped = 2; -+ uint64 rx_queued = 3; -+ uint64 tx_queued = 4; -+} -+ -+message NetworkStats { -+ repeated InterfaceStats interfaces = 1; -+ TcpStat tcp = 2; -+ TcpStat tcp6 = 3; -+ UdpStat udp = 4; -+ UdpStat udp6 = 5; -+} -+ - message StatsContainerResponse { - CgroupStats cgroup_stats = 1; -- repeated NetworkStats network_stats = 2; -+ NetworkStats network_stats = 2; - } - - message WriteStreamRequest { --- -2.14.3 (Apple Git-98) - diff --git a/agent/patches/0016-clock-synchronizes-clock-info-with-proxy.patch b/agent/patches/0016-clock-synchronizes-clock-info-with-proxy.patch deleted file mode 100644 index f594245..0000000 --- a/agent/patches/0016-clock-synchronizes-clock-info-with-proxy.patch +++ /dev/null @@ -1,268 +0,0 @@ -From 6120525f81701424e97d453d515d38f14bbe99d9 Mon Sep 17 00:00:00 2001 -From: holyfei -Date: Wed, 19 Aug 2020 20:25:00 +0800 -Subject: [PATCH 16/16] clock: synchronizes clock info with proxy - -reason: virtual machine's clock may be incorrect, proxy synchronizes -clock info to help virtual machine adjust clock time - -Signed-off-by: yangfeiyu ---- - pkg/clock/clock_util.go | 47 ++++++++++++++++++++++++ - sync_clock_server.go | 94 +++++++++++++++++++++++++++++++++++++++++++++++ - sync_clock_server_test.go | 88 ++++++++++++++++++++++++++++++++++++++++++++ - 3 files changed, 229 insertions(+) - create mode 100644 pkg/clock/clock_util.go - create mode 100644 sync_clock_server.go - create mode 100644 sync_clock_server_test.go - -diff --git a/pkg/clock/clock_util.go b/pkg/clock/clock_util.go -new file mode 100644 -index 0000000..4b78c63 ---- /dev/null -+++ b/pkg/clock/clock_util.go -@@ -0,0 +1,47 @@ -+// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved. -+// SPDX-License-Identifier: Apache-2.0 -+// Description: sync clock related function -+// Author: xueshaojia -+// Create: 2018-11-10 -+ -+// Package clock provides clock sync functions -+package clock -+ -+import ( -+ "net" -+ "syscall" -+) -+ -+type TimeValue struct { -+ ClientSendTime int64 `json:"client_send_time"` -+ ClientArriveTime int64 `json:"client_arrive_time"` -+ ServerSendTime int64 `json:"server_send_time"` -+ ServerArriveTime int64 `json:"server_arrive_time"` -+ Delta int64 `json:"delta"` -+} -+ -+const MaxSyncClockByteNum = 400 // sync clock byte num, max=400 -+const MaxTimeStampSupport = 3155731199000000000 // unit is ns, 2069/12/31 23:59:59 -+const MinTimeStampSupport = 0 // unit is ns, 1970/1/1 8:0:0 -+ -+// getCurrentTimeNs returns UTC time in Ns -+func GetCurrentTimeNs() int64 { -+ var tv syscall.Timeval -+ if err := syscall.Gettimeofday(&tv); err != nil { -+ return -1 -+ } -+ return tv.Sec*1000000000 + tv.Usec*1000 -+} -+ -+// readConnData reads data from stream -+func ReadConnData(stream net.Conn) (buf []byte, byteNum int, err error) { -+ buf = make([]byte, MaxSyncClockByteNum) -+ byteNum, err = stream.Read(buf) -+ return buf, byteNum, err -+} -+ -+// writeConnData writes data to stream -+func WriteConnData(stream net.Conn, buf []byte) error { -+ _, err := stream.Write(buf) -+ return err -+} -diff --git a/sync_clock_server.go b/sync_clock_server.go -new file mode 100644 -index 0000000..a8e98c6 ---- /dev/null -+++ b/sync_clock_server.go -@@ -0,0 +1,94 @@ -+// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved. -+// SPDX-License-Identifier: Apache-2.0 -+// Description: sync clock server related function -+// Author: xueshaojia -+// Create: 2018-11-10 -+ -+package main -+ -+import ( -+ "encoding/json" -+ "fmt" -+ "io" -+ "net" -+ "os/exec" -+ "syscall" -+ -+ "github.com/kata-containers/agent/pkg/clock" -+) -+ -+// getSyncClockInfo waits for client request -+func getSyncClockInfo(stream net.Conn, clockInfo *clock.TimeValue) error { -+ buf, byteNum, err := clock.ReadConnData(stream) -+ if err != nil { -+ return err -+ } -+ return json.Unmarshal(buf[:byteNum], clockInfo) -+} -+ -+// ackSyncClockInfo sends clock info to client -+func ackSyncClockInfo(stream net.Conn, clockInfo *clock.TimeValue) error { -+ b, err := json.Marshal(clockInfo) -+ if err != nil { -+ return err -+ } -+ return clock.WriteConnData(stream, b) -+} -+ -+// setGuestClock sets systemtime and hwclock -+func setGuestClock(delta int64) error { -+ dstClockNs := clock.GetCurrentTimeNs() + delta -+ if dstClockNs < clock.MinTimeStampSupport || dstClockNs > clock.MaxTimeStampSupport { -+ return fmt.Errorf("sync clock failed, the valid timestamp is from 1970/1/1 8:0:0 to 2069/12/31 23:59:59") -+ } -+ dstTimeVal := syscall.NsecToTimeval(dstClockNs) -+ if err := syscall.Settimeofday(&dstTimeVal); err != nil { -+ agentLog.WithError(err).Error("sync clock, set systemtime failed") -+ return err -+ } -+ cmd := exec.Command("hwclock", "-w") -+ if err := cmd.Run(); err != nil { -+ agentLog.WithError(err).Error("sync clock, set hwclock failed") -+ return err -+ } -+ agentLog.Infof("sync clock, set systemtime and hwclock succ, delta:%d", delta) -+ return nil -+} -+ -+// SyncClock is a loop that deals with client's request -+// 1 read client request -+// 2 response with clock info or set guest clock -+// 3 if error happends, print error info -+func SyncClock(stream net.Conn) { -+ for { -+ var clockInfo clock.TimeValue -+ if err := getSyncClockInfo(stream, &clockInfo); err != nil { -+ if err == io.EOF { -+ agentLog.WithError(err).Error("sync clock, yamux session error happends") -+ stream.Close() -+ break -+ } -+ agentLog.WithError(err).Error("sync clock, read sync clock info failed") -+ continue -+ } -+ // set client's request arrive time -+ clockInfo.ClientArriveTime = clock.GetCurrentTimeNs() -+ if clockInfo.Delta == 0 { -+ // this is a request for clock diff info -+ // set server's response send time -+ // if fails, wait for next time -+ clockInfo.ServerSendTime = clock.GetCurrentTimeNs() -+ if err := ackSyncClockInfo(stream, &clockInfo); err != nil { -+ agentLog.WithError(err).Error("sync clock, send back clock info failed") -+ } -+ } else { -+ // this is a request for adjusting guest clock -+ // 1 adjust guest clock(system time and hwclock) -+ // 2 do not send response any more -+ // 3 if fails, wait for next request to adjust -+ if err := setGuestClock(clockInfo.Delta); err != nil { -+ agentLog.WithError(err).Error("sync clock, set local clock failed") -+ } -+ } -+ } -+} -diff --git a/sync_clock_server_test.go b/sync_clock_server_test.go -new file mode 100644 -index 0000000..41d5227 ---- /dev/null -+++ b/sync_clock_server_test.go -@@ -0,0 +1,88 @@ -+// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved. -+// SPDX-License-Identifier: Apache-2.0 -+// Description: sync clock server related test -+// Author: xueshaojia -+// Create: 2018-11-10 -+ -+package main -+ -+import ( -+ "encoding/json" -+ "fmt" -+ "math/rand" -+ "net" -+ "testing" -+ -+ "github.com/hashicorp/yamux" -+ "github.com/kata-containers/agent/pkg/clock" -+) -+ -+var clientStream net.Conn -+ -+func TestGetCurrentTimeNs(t *testing.T) { -+ timeRet := clock.GetCurrentTimeNs() -+ if timeRet <= 0 { -+ t.Fatalf("TestGetCurrentTimeNs Fail, time:%d", timeRet) -+ } -+} -+ -+func TestSetGuestClock(t *testing.T) { -+ if err := setGuestClock(0); err != nil { -+ t.Fatalf("test set clock failed, err:%v", err) -+ } -+} -+ -+func TestReadWriteData(t *testing.T) { -+ sock := GenSocket() -+ var err error -+ listener, err := net.Listen("unix", sock) -+ if err != nil { -+ t.Fatalf("listening err: %v", err) -+ } -+ go SetUpClient(sock) -+ conn, err := listener.Accept() -+ if err != nil { -+ t.Fatalf("accept err: %v", err) -+ } -+ session, err := yamux.Server(conn, nil) -+ if err != nil { -+ t.Fatalf("session err: %v", err) -+ } -+ stream, err := session.Accept() -+ if err != nil { -+ t.Fatalf("stream err: %v", err) -+ } -+ go func() { -+ var clockInfo clock.TimeValue = clock.TimeValue{1000, 0, 0, 0, 0} -+ b, _ := json.Marshal(clockInfo) -+ clock.WriteConnData(clientStream, b) -+ }() -+ _, _, err = clock.ReadConnData(stream) -+ if err != nil { -+ t.Fatalf("read conn data error, err:%v", err) -+ } -+} -+ -+func SetUpClient(sock string) error { -+ conn, err := net.Dial("unix", sock) -+ if err != nil { -+ return err -+ } -+ session, err := yamux.Client(conn, nil) -+ if err != nil { -+ conn.Close() -+ return err -+ } -+ stream, err := session.Open() -+ if err != nil { -+ return err -+ } -+ clientStream = stream -+ return nil -+} -+ -+func GenSocket() string { -+ randSeed := clock.GetCurrentTimeNs() -+ rand.Seed(randSeed) -+ return fmt.Sprintf("/tmp/%d.sock", rand.Uint32()) -+} --- -2.14.3 (Apple Git-98) - diff --git a/agent/patches/0017-agent-add-support-of-new-sandbox-StratoVirt.patch b/agent/patches/0017-agent-add-support-of-new-sandbox-StratoVirt.patch deleted file mode 100644 index 591eb05..0000000 --- a/agent/patches/0017-agent-add-support-of-new-sandbox-StratoVirt.patch +++ /dev/null @@ -1,1005 +0,0 @@ -From ad3da81d9566807df2de062b0059aafbe9d9d810 Mon Sep 17 00:00:00 2001 -From: LiangZhang -Date: Mon, 21 Sep 2020 10:42:41 +0800 -Subject: [PATCH] agent: add support of new sandbox StratoVirt - -Signed-off-by: LiangZhang ---- - grpc.go | 4 + - network.go | 58 +++++ - protocols/grpc/agent.pb.go | 636 +++++++++++++++++++++++++++++---------------- - protocols/grpc/agent.proto | 5 + - 4 files changed, 473 insertions(+), 230 deletions(-) - -diff --git a/grpc.go b/grpc.go -index 1b63cde..727efb8 100644 ---- a/grpc.go -+++ b/grpc.go -@@ -1574,6 +1574,10 @@ func (a *agentGRPC) UpdateInterface(ctx context.Context, req *pb.UpdateInterface - return a.sandbox.updateInterface(nil, req.Interface) - } - -+func (a *agentGRPC) UpdateInterfaceHwAddrByName(ctx context.Context, req *pb.UpdateInterfaceHwAddrByNameRequest) (*types.Interface, error) { -+ return a.sandbox.updateInterfaceHwAddrByName(nil, req.Interface) -+} -+ - func (a *agentGRPC) UpdateRoutes(ctx context.Context, req *pb.UpdateRoutesRequest) (*pb.Routes, error) { - return a.sandbox.updateRoutes(nil, req.Routes, req.Increment) - } -diff --git a/network.go b/network.go -index 1baaa2e..f6e2c17 100644 ---- a/network.go -+++ b/network.go -@@ -209,6 +209,64 @@ func (s *sandbox) removeInterface(netHandle *netlink.Handle, iface *types.Interf - return nil, nil - } - -+// updateInterfaceHwAddrByName will update an existing interface with the values provided in the types.Interface. -+// It will identify the existing interface via its name. -+func (s *sandbox) updateInterfaceHwAddrByName(netHandle *netlink.Handle, iface *types.Interface) (resultingIfc *types.Interface, err error) { -+ if iface == nil { -+ return nil, errNoIF -+ } -+ s.network.ifacesLock.Lock() -+ defer s.network.ifacesLock.Unlock() -+ -+ if netHandle == nil { -+ netHandle, err = netlink.NewHandle(unix.NETLINK_ROUTE) -+ if err != nil { -+ return nil, err -+ } -+ defer netHandle.Delete() -+ } -+ -+ var link netlink.Link -+ links, err := netHandle.LinkList() -+ if err != nil { -+ return nil, err -+ } -+ -+ for _, l := range links { -+ if l == nil { -+ continue -+ } -+ -+ if l.Attrs() != nil && l.Attrs().Name == iface.Name { -+ link = l -+ } -+ } -+ -+ if link == nil { -+ return nil, grpcStatus.Errorf(codes.NotFound, "Could not find the link corresponding to name %s", iface.Name) -+ } -+ -+ // delete ip -+ if ips, err := netlink.AddrList(link, netlink.FAMILY_ALL); err == nil { -+ for _, ip := range ips { -+ netlink.AddrDel(link, &ip) -+ } -+ } -+ -+ if len(iface.IPAddresses) == 0 { -+ netHandle.LinkSetDown(link) -+ } -+ -+ // update mac -+ if hardAddr, err := net.ParseMAC(iface.HwAddr); err == nil { -+ if err = netHandle.LinkSetHardwareAddr(link, hardAddr); err != nil { -+ return nil, grpcStatus.Errorf(codes.Internal, "Could not set %s to %v: %v", iface.HwAddr, link, err) -+ } -+ } -+ -+ return iface, nil -+} -+ - // updateInterface will update an existing interface with the values provided in the types.Interface. It will identify the - // existing interface via MAC address and will return the state of the interface once the function completes as well an any - // errors observed. -diff --git a/protocols/grpc/agent.pb.go b/protocols/grpc/agent.pb.go -index c50ecb5..181fcb6 100644 ---- a/protocols/grpc/agent.pb.go -+++ b/protocols/grpc/agent.pb.go -@@ -50,6 +50,7 @@ - Interfaces - Routes - UpdateInterfaceRequest -+ UpdateInterfaceHwAddrByNameRequest - UpdateRoutesRequest - ListInterfacesRequest - ListRoutesRequest -@@ -1491,6 +1492,24 @@ func (m *UpdateInterfaceRequest) GetInterface() *types.Interface { - return nil - } - -+type UpdateInterfaceHwAddrByNameRequest struct { -+ Interface *types.Interface `protobuf:"bytes,1,opt,name=interface" json:"interface,omitempty"` -+} -+ -+func (m *UpdateInterfaceHwAddrByNameRequest) Reset() { *m = UpdateInterfaceHwAddrByNameRequest{} } -+func (m *UpdateInterfaceHwAddrByNameRequest) String() string { return proto.CompactTextString(m) } -+func (*UpdateInterfaceHwAddrByNameRequest) ProtoMessage() {} -+func (*UpdateInterfaceHwAddrByNameRequest) Descriptor() ([]byte, []int) { -+ return fileDescriptorAgent, []int{40} -+} -+ -+func (m *UpdateInterfaceHwAddrByNameRequest) GetInterface() *types.Interface { -+ if m != nil { -+ return m.Interface -+ } -+ return nil -+} -+ - type UpdateRoutesRequest struct { - Routes *Routes `protobuf:"bytes,1,opt,name=routes" json:"routes,omitempty"` - Increment bool `protobuf:"varint,2,opt,name=increment,proto3" json:"increment,omitempty"` -@@ -1499,7 +1518,7 @@ type UpdateRoutesRequest struct { - func (m *UpdateRoutesRequest) Reset() { *m = UpdateRoutesRequest{} } - func (m *UpdateRoutesRequest) String() string { return proto.CompactTextString(m) } - func (*UpdateRoutesRequest) ProtoMessage() {} --func (*UpdateRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{40} } -+func (*UpdateRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{41} } - - func (m *UpdateRoutesRequest) GetRoutes() *Routes { - if m != nil { -@@ -1521,7 +1540,7 @@ type ListInterfacesRequest struct { - func (m *ListInterfacesRequest) Reset() { *m = ListInterfacesRequest{} } - func (m *ListInterfacesRequest) String() string { return proto.CompactTextString(m) } - func (*ListInterfacesRequest) ProtoMessage() {} --func (*ListInterfacesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{41} } -+func (*ListInterfacesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{42} } - - type ListRoutesRequest struct { - } -@@ -1529,7 +1548,7 @@ type ListRoutesRequest struct { - func (m *ListRoutesRequest) Reset() { *m = ListRoutesRequest{} } - func (m *ListRoutesRequest) String() string { return proto.CompactTextString(m) } - func (*ListRoutesRequest) ProtoMessage() {} --func (*ListRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{42} } -+func (*ListRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{43} } - - type OnlineCPUMemRequest struct { - // Wait specifies if the caller waits for the agent to online all resources. -@@ -1545,7 +1564,7 @@ type OnlineCPUMemRequest struct { - func (m *OnlineCPUMemRequest) Reset() { *m = OnlineCPUMemRequest{} } - func (m *OnlineCPUMemRequest) String() string { return proto.CompactTextString(m) } - func (*OnlineCPUMemRequest) ProtoMessage() {} --func (*OnlineCPUMemRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{43} } -+func (*OnlineCPUMemRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{44} } - - func (m *OnlineCPUMemRequest) GetWait() bool { - if m != nil { -@@ -1576,7 +1595,7 @@ type ReseedRandomDevRequest struct { - func (m *ReseedRandomDevRequest) Reset() { *m = ReseedRandomDevRequest{} } - func (m *ReseedRandomDevRequest) String() string { return proto.CompactTextString(m) } - func (*ReseedRandomDevRequest) ProtoMessage() {} --func (*ReseedRandomDevRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{44} } -+func (*ReseedRandomDevRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{45} } - - func (m *ReseedRandomDevRequest) GetData() []byte { - if m != nil { -@@ -1603,7 +1622,7 @@ type AgentDetails struct { - func (m *AgentDetails) Reset() { *m = AgentDetails{} } - func (m *AgentDetails) String() string { return proto.CompactTextString(m) } - func (*AgentDetails) ProtoMessage() {} --func (*AgentDetails) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{45} } -+func (*AgentDetails) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{46} } - - func (m *AgentDetails) GetVersion() string { - if m != nil { -@@ -1654,7 +1673,7 @@ type GuestDetailsRequest struct { - func (m *GuestDetailsRequest) Reset() { *m = GuestDetailsRequest{} } - func (m *GuestDetailsRequest) String() string { return proto.CompactTextString(m) } - func (*GuestDetailsRequest) ProtoMessage() {} --func (*GuestDetailsRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{46} } -+func (*GuestDetailsRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{47} } - - func (m *GuestDetailsRequest) GetMemBlockSize() bool { - if m != nil { -@@ -1680,7 +1699,7 @@ type GuestDetailsResponse struct { - func (m *GuestDetailsResponse) Reset() { *m = GuestDetailsResponse{} } - func (m *GuestDetailsResponse) String() string { return proto.CompactTextString(m) } - func (*GuestDetailsResponse) ProtoMessage() {} --func (*GuestDetailsResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{47} } -+func (*GuestDetailsResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{48} } - - func (m *GuestDetailsResponse) GetMemBlockSizeBytes() uint64 { - if m != nil { -@@ -1712,7 +1731,7 @@ type MemHotplugByProbeRequest struct { - func (m *MemHotplugByProbeRequest) Reset() { *m = MemHotplugByProbeRequest{} } - func (m *MemHotplugByProbeRequest) String() string { return proto.CompactTextString(m) } - func (*MemHotplugByProbeRequest) ProtoMessage() {} --func (*MemHotplugByProbeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{48} } -+func (*MemHotplugByProbeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{49} } - - func (m *MemHotplugByProbeRequest) GetMemHotplugProbeAddr() []uint64 { - if m != nil { -@@ -1731,7 +1750,7 @@ type SetGuestDateTimeRequest struct { - func (m *SetGuestDateTimeRequest) Reset() { *m = SetGuestDateTimeRequest{} } - func (m *SetGuestDateTimeRequest) String() string { return proto.CompactTextString(m) } - func (*SetGuestDateTimeRequest) ProtoMessage() {} --func (*SetGuestDateTimeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{49} } -+func (*SetGuestDateTimeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{50} } - - func (m *SetGuestDateTimeRequest) GetSec() int64 { - if m != nil { -@@ -1780,7 +1799,7 @@ type Storage struct { - func (m *Storage) Reset() { *m = Storage{} } - func (m *Storage) String() string { return proto.CompactTextString(m) } - func (*Storage) ProtoMessage() {} --func (*Storage) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{50} } -+func (*Storage) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{51} } - - func (m *Storage) GetDriver() string { - if m != nil { -@@ -1863,7 +1882,7 @@ type Device struct { - func (m *Device) Reset() { *m = Device{} } - func (m *Device) String() string { return proto.CompactTextString(m) } - func (*Device) ProtoMessage() {} --func (*Device) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{51} } -+func (*Device) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{52} } - - func (m *Device) GetId() string { - if m != nil { -@@ -1909,7 +1928,7 @@ type StringUser struct { - func (m *StringUser) Reset() { *m = StringUser{} } - func (m *StringUser) String() string { return proto.CompactTextString(m) } - func (*StringUser) ProtoMessage() {} --func (*StringUser) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{52} } -+func (*StringUser) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{53} } - - func (m *StringUser) GetUid() string { - if m != nil { -@@ -1957,7 +1976,7 @@ type CopyFileRequest struct { - func (m *CopyFileRequest) Reset() { *m = CopyFileRequest{} } - func (m *CopyFileRequest) String() string { return proto.CompactTextString(m) } - func (*CopyFileRequest) ProtoMessage() {} --func (*CopyFileRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{53} } -+func (*CopyFileRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{54} } - - func (m *CopyFileRequest) GetPath() string { - if m != nil { -@@ -2021,7 +2040,7 @@ type StartTracingRequest struct { - func (m *StartTracingRequest) Reset() { *m = StartTracingRequest{} } - func (m *StartTracingRequest) String() string { return proto.CompactTextString(m) } - func (*StartTracingRequest) ProtoMessage() {} --func (*StartTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{54} } -+func (*StartTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{55} } - - type StopTracingRequest struct { - } -@@ -2029,7 +2048,7 @@ type StopTracingRequest struct { - func (m *StopTracingRequest) Reset() { *m = StopTracingRequest{} } - func (m *StopTracingRequest) String() string { return proto.CompactTextString(m) } - func (*StopTracingRequest) ProtoMessage() {} --func (*StopTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{55} } -+func (*StopTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{56} } - - type UpdateIPVSRequest struct { - // IPVS_req is the IPVS rule message needed to update -@@ -2039,7 +2058,7 @@ type UpdateIPVSRequest struct { - func (m *UpdateIPVSRequest) Reset() { *m = UpdateIPVSRequest{} } - func (m *UpdateIPVSRequest) String() string { return proto.CompactTextString(m) } - func (*UpdateIPVSRequest) ProtoMessage() {} --func (*UpdateIPVSRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{56} } -+func (*UpdateIPVSRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{57} } - - func (m *UpdateIPVSRequest) GetIPVSReq() string { - if m != nil { -@@ -2056,7 +2075,7 @@ type IPVSResponse struct { - func (m *IPVSResponse) Reset() { *m = IPVSResponse{} } - func (m *IPVSResponse) String() string { return proto.CompactTextString(m) } - func (*IPVSResponse) ProtoMessage() {} --func (*IPVSResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{57} } -+func (*IPVSResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{58} } - - func (m *IPVSResponse) GetIPVSRes() string { - if m != nil { -@@ -2106,6 +2125,7 @@ func init() { - proto.RegisterType((*Interfaces)(nil), "grpc.Interfaces") - proto.RegisterType((*Routes)(nil), "grpc.Routes") - proto.RegisterType((*UpdateInterfaceRequest)(nil), "grpc.UpdateInterfaceRequest") -+ proto.RegisterType((*UpdateInterfaceHwAddrByNameRequest)(nil), "grpc.UpdateInterfaceHwAddrByNameRequest") - proto.RegisterType((*UpdateRoutesRequest)(nil), "grpc.UpdateRoutesRequest") - proto.RegisterType((*ListInterfacesRequest)(nil), "grpc.ListInterfacesRequest") - proto.RegisterType((*ListRoutesRequest)(nil), "grpc.ListRoutesRequest") -@@ -2163,6 +2183,7 @@ type AgentServiceClient interface { - TtyWinResize(ctx context.Context, in *TtyWinResizeRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error) - // networking - UpdateInterface(ctx context.Context, in *UpdateInterfaceRequest, opts ...grpc1.CallOption) (*types.Interface, error) -+ UpdateInterfaceHwAddrByName(ctx context.Context, in *UpdateInterfaceHwAddrByNameRequest, opts ...grpc1.CallOption) (*types.Interface, error) - UpdateRoutes(ctx context.Context, in *UpdateRoutesRequest, opts ...grpc1.CallOption) (*Routes, error) - ListInterfaces(ctx context.Context, in *ListInterfacesRequest, opts ...grpc1.CallOption) (*Interfaces, error) - ListRoutes(ctx context.Context, in *ListRoutesRequest, opts ...grpc1.CallOption) (*Routes, error) -@@ -2342,6 +2363,15 @@ func (c *agentServiceClient) UpdateInterface(ctx context.Context, in *UpdateInte - return out, nil - } - -+func (c *agentServiceClient) UpdateInterfaceHwAddrByName(ctx context.Context, in *UpdateInterfaceHwAddrByNameRequest, opts ...grpc1.CallOption) (*types.Interface, error) { -+ out := new(types.Interface) -+ err := grpc1.Invoke(ctx, "/grpc.AgentService/UpdateInterfaceHwAddrByName", in, out, c.cc, opts...) -+ if err != nil { -+ return nil, err -+ } -+ return out, nil -+} -+ - func (c *agentServiceClient) UpdateRoutes(ctx context.Context, in *UpdateRoutesRequest, opts ...grpc1.CallOption) (*Routes, error) { - out := new(Routes) - err := grpc1.Invoke(ctx, "/grpc.AgentService/UpdateRoutes", in, out, c.cc, opts...) -@@ -2497,6 +2527,7 @@ type AgentServiceServer interface { - TtyWinResize(context.Context, *TtyWinResizeRequest) (*google_protobuf2.Empty, error) - // networking - UpdateInterface(context.Context, *UpdateInterfaceRequest) (*types.Interface, error) -+ UpdateInterfaceHwAddrByName(context.Context, *UpdateInterfaceHwAddrByNameRequest) (*types.Interface, error) - UpdateRoutes(context.Context, *UpdateRoutesRequest) (*Routes, error) - ListInterfaces(context.Context, *ListInterfacesRequest) (*Interfaces, error) - ListRoutes(context.Context, *ListRoutesRequest) (*Routes, error) -@@ -2825,6 +2856,24 @@ func _AgentService_UpdateInterface_Handler(srv interface{}, ctx context.Context, - return interceptor(ctx, in, info, handler) - } - -+func _AgentService_UpdateInterfaceHwAddrByName_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) { -+ in := new(UpdateInterfaceHwAddrByNameRequest) -+ if err := dec(in); err != nil { -+ return nil, err -+ } -+ if interceptor == nil { -+ return srv.(AgentServiceServer).UpdateInterfaceHwAddrByName(ctx, in) -+ } -+ info := &grpc1.UnaryServerInfo{ -+ Server: srv, -+ FullMethod: "/grpc.AgentService/UpdateInterfaceHwAddrByName", -+ } -+ handler := func(ctx context.Context, req interface{}) (interface{}, error) { -+ return srv.(AgentServiceServer).UpdateInterfaceHwAddrByName(ctx, req.(*UpdateInterfaceHwAddrByNameRequest)) -+ } -+ return interceptor(ctx, in, info, handler) -+} -+ - func _AgentService_UpdateRoutes_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) { - in := new(UpdateRoutesRequest) - if err := dec(in); err != nil { -@@ -3150,6 +3199,10 @@ var _AgentService_serviceDesc = grpc1.ServiceDesc{ - Handler: _AgentService_UpdateInterface_Handler, - }, - { -+ MethodName: "UpdateInterfaceHwAddrByName", -+ Handler: _AgentService_UpdateInterfaceHwAddrByName_Handler, -+ }, -+ { - MethodName: "UpdateRoutes", - Handler: _AgentService_UpdateRoutes_Handler, - }, -@@ -4899,6 +4952,34 @@ func (m *UpdateInterfaceRequest) MarshalTo(dAtA []byte) (int, error) { - return i, nil - } - -+func (m *UpdateInterfaceHwAddrByNameRequest) Marshal() (dAtA []byte, err error) { -+ size := m.Size() -+ dAtA = make([]byte, size) -+ n, err := m.MarshalTo(dAtA) -+ if err != nil { -+ return nil, err -+ } -+ return dAtA[:n], nil -+} -+ -+func (m *UpdateInterfaceHwAddrByNameRequest) MarshalTo(dAtA []byte) (int, error) { -+ var i int -+ _ = i -+ var l int -+ _ = l -+ if m.Interface != nil { -+ dAtA[i] = 0xa -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.Interface.Size())) -+ n25, err := m.Interface.MarshalTo(dAtA[i:]) -+ if err != nil { -+ return 0, err -+ } -+ i += n25 -+ } -+ return i, nil -+} -+ - func (m *UpdateRoutesRequest) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) -@@ -4918,11 +4999,11 @@ func (m *UpdateRoutesRequest) MarshalTo(dAtA []byte) (int, error) { - dAtA[i] = 0xa - i++ - i = encodeVarintAgent(dAtA, i, uint64(m.Routes.Size())) -- n25, err := m.Routes.MarshalTo(dAtA[i:]) -+ n26, err := m.Routes.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } -- i += n25 -+ i += n26 - } - if m.Increment { - dAtA[i] = 0x10 -@@ -5176,11 +5257,11 @@ func (m *GuestDetailsResponse) MarshalTo(dAtA []byte) (int, error) { - dAtA[i] = 0x12 - i++ - i = encodeVarintAgent(dAtA, i, uint64(m.AgentDetails.Size())) -- n26, err := m.AgentDetails.MarshalTo(dAtA[i:]) -+ n27, err := m.AgentDetails.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } -- i += n26 -+ i += n27 - } - if m.SupportMemHotplugProbe { - dAtA[i] = 0x18 -@@ -5211,21 +5292,21 @@ func (m *MemHotplugByProbeRequest) MarshalTo(dAtA []byte) (int, error) { - var l int - _ = l - if len(m.MemHotplugProbeAddr) > 0 { -- dAtA28 := make([]byte, len(m.MemHotplugProbeAddr)*10) -- var j27 int -+ dAtA29 := make([]byte, len(m.MemHotplugProbeAddr)*10) -+ var j28 int - for _, num := range m.MemHotplugProbeAddr { - for num >= 1<<7 { -- dAtA28[j27] = uint8(uint64(num)&0x7f | 0x80) -+ dAtA29[j28] = uint8(uint64(num)&0x7f | 0x80) - num >>= 7 -- j27++ -+ j28++ - } -- dAtA28[j27] = uint8(num) -- j27++ -+ dAtA29[j28] = uint8(num) -+ j28++ - } - dAtA[i] = 0xa - i++ -- i = encodeVarintAgent(dAtA, i, uint64(j27)) -- i += copy(dAtA[i:], dAtA28[:j27]) -+ i = encodeVarintAgent(dAtA, i, uint64(j28)) -+ i += copy(dAtA[i:], dAtA29[:j28]) - } - return i, nil - } -@@ -6333,6 +6414,16 @@ func (m *UpdateInterfaceRequest) Size() (n int) { - return n - } - -+func (m *UpdateInterfaceHwAddrByNameRequest) Size() (n int) { -+ var l int -+ _ = l -+ if m.Interface != nil { -+ l = m.Interface.Size() -+ n += 1 + l + sovAgent(uint64(l)) -+ } -+ return n -+} -+ - func (m *UpdateRoutesRequest) Size() (n int) { - var l int - _ = l -@@ -12114,6 +12205,89 @@ func (m *UpdateInterfaceRequest) Unmarshal(dAtA []byte) error { - } - return nil - } -+func (m *UpdateInterfaceHwAddrByNameRequest) Unmarshal(dAtA []byte) error { -+ l := len(dAtA) -+ iNdEx := 0 -+ for iNdEx < l { -+ preIndex := iNdEx -+ var wire uint64 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ wire |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ fieldNum := int32(wire >> 3) -+ wireType := int(wire & 0x7) -+ if wireType == 4 { -+ return fmt.Errorf("proto: UpdateInterfaceHwAddrByNameRequest: wiretype end group for non-group") -+ } -+ if fieldNum <= 0 { -+ return fmt.Errorf("proto: UpdateInterfaceHwAddrByNameRequest: illegal tag %d (wire type %d)", fieldNum, wire) -+ } -+ switch fieldNum { -+ case 1: -+ if wireType != 2 { -+ return fmt.Errorf("proto: wrong wireType = %d for field Interface", wireType) -+ } -+ var msglen int -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ msglen |= (int(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ if msglen < 0 { -+ return ErrInvalidLengthAgent -+ } -+ postIndex := iNdEx + msglen -+ if postIndex > l { -+ return io.ErrUnexpectedEOF -+ } -+ if m.Interface == nil { -+ m.Interface = &types.Interface{} -+ } -+ if err := m.Interface.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -+ return err -+ } -+ iNdEx = postIndex -+ default: -+ iNdEx = preIndex -+ skippy, err := skipAgent(dAtA[iNdEx:]) -+ if err != nil { -+ return err -+ } -+ if skippy < 0 { -+ return ErrInvalidLengthAgent -+ } -+ if (iNdEx + skippy) > l { -+ return io.ErrUnexpectedEOF -+ } -+ iNdEx += skippy -+ } -+ } -+ -+ if iNdEx > l { -+ return io.ErrUnexpectedEOF -+ } -+ return nil -+} - func (m *UpdateRoutesRequest) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 -@@ -14242,204 +14416,206 @@ var ( - func init() { proto.RegisterFile("agent.proto", fileDescriptorAgent) } - - var fileDescriptorAgent = []byte{ -- // 3183 bytes of a gzipped FileDescriptorProto -- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0x4b, 0x6f, 0x1c, 0xc7, -- 0x99, 0x98, 0x07, 0x39, 0x33, 0xdf, 0xbc, 0xc8, 0x22, 0x45, 0x8d, 0x46, 0xb2, 0x2c, 0xb7, 0x6d, -- 0x99, 0x5e, 0xaf, 0x49, 0x8b, 0x36, 0xfc, 0x84, 0x57, 0x10, 0x29, 0xad, 0xc8, 0xb5, 0xb4, 0xe2, -- 0xf6, 0x88, 0xeb, 0x5d, 0x2c, 0x16, 0x8d, 0x66, 0x77, 0x71, 0x58, 0xe6, 0x74, 0x57, 0xbb, 0xaa, -- 0x9a, 0xe2, 0x78, 0x81, 0x3d, 0x26, 0xd7, 0x9c, 0x72, 0xcb, 0x1f, 0x08, 0x72, 0xcb, 0x2d, 0xb9, -- 0xe6, 0x60, 0x04, 0x39, 0x04, 0xf9, 0x01, 0x41, 0xe0, 0x9f, 0x90, 0x5f, 0x10, 0xd4, 0xab, 0x1f, -- 0x33, 0x43, 0x1a, 0x11, 0x04, 0xe4, 0xd2, 0xe8, 0xef, 0x51, 0xdf, 0xab, 0xaa, 0xbe, 0xfa, 0xbe, -- 0x2a, 0x68, 0xfb, 0x63, 0x1c, 0x8b, 0xad, 0x84, 0x51, 0x41, 0x51, 0x7d, 0xcc, 0x92, 0x60, 0xd8, -- 0xa2, 0x01, 0xd1, 0x88, 0xe1, 0xc7, 0x63, 0x22, 0x4e, 0xd3, 0xe3, 0xad, 0x80, 0x46, 0xdb, 0x67, -- 0xbe, 0xf0, 0xdf, 0x0f, 0x68, 0x2c, 0x7c, 0x12, 0x63, 0xc6, 0xb7, 0xd5, 0xc0, 0xed, 0xe4, 0x6c, -- 0xbc, 0x2d, 0xa6, 0x09, 0xe6, 0xfa, 0x6b, 0xc6, 0xdd, 0x1c, 0x53, 0x3a, 0x9e, 0xe0, 0x6d, 0x05, -- 0x1d, 0xa7, 0x27, 0xdb, 0x38, 0x4a, 0xc4, 0x54, 0x13, 0x9d, 0x5f, 0x54, 0x61, 0x63, 0x8f, 0x61, -- 0x5f, 0xe0, 0x3d, 0x2b, 0xcd, 0xc5, 0xdf, 0xa6, 0x98, 0x0b, 0xf4, 0x06, 0x74, 0x32, 0x0d, 0x1e, -- 0x09, 0x07, 0x95, 0x3b, 0x95, 0xcd, 0x96, 0xdb, 0xce, 0x70, 0x07, 0x21, 0xba, 0x0e, 0x0d, 0x7c, -- 0x81, 0x03, 0x49, 0xad, 0x2a, 0xea, 0xb2, 0x04, 0x0f, 0x42, 0x74, 0x0f, 0xda, 0x5c, 0x30, 0x12, -- 0x8f, 0xbd, 0x94, 0x63, 0x36, 0xa8, 0xdd, 0xa9, 0x6c, 0xb6, 0x77, 0x56, 0xb6, 0xa4, 0x4b, 0x5b, -- 0x23, 0x45, 0x38, 0xe2, 0x98, 0xb9, 0xc0, 0xb3, 0x7f, 0x74, 0x17, 0x1a, 0x21, 0x3e, 0x27, 0x01, -- 0xe6, 0x83, 0xfa, 0x9d, 0xda, 0x66, 0x7b, 0xa7, 0xa3, 0xd9, 0x1f, 0x2a, 0xa4, 0x6b, 0x89, 0xe8, -- 0x5d, 0x68, 0x72, 0x41, 0x99, 0x3f, 0xc6, 0x7c, 0xb0, 0xa4, 0x18, 0xbb, 0x56, 0xae, 0xc2, 0xba, -- 0x19, 0x19, 0xdd, 0x82, 0xda, 0xb3, 0xbd, 0x83, 0xc1, 0xb2, 0xd2, 0x0e, 0x86, 0x2b, 0xc1, 0x81, -- 0x2b, 0xd1, 0xe8, 0x4d, 0xe8, 0x72, 0x3f, 0x0e, 0x8f, 0xe9, 0x85, 0x97, 0x90, 0x30, 0xe6, 0x83, -- 0xc6, 0x9d, 0xca, 0x66, 0xd3, 0xed, 0x18, 0xe4, 0xa1, 0xc4, 0x39, 0x9f, 0xc3, 0xb5, 0x91, 0xf0, -- 0x99, 0x78, 0x89, 0xe8, 0x38, 0x47, 0xb0, 0xe1, 0xe2, 0x88, 0x9e, 0xbf, 0x54, 0x68, 0x07, 0xd0, -- 0x10, 0x24, 0xc2, 0x34, 0x15, 0x2a, 0xb4, 0x5d, 0xd7, 0x82, 0xce, 0xaf, 0x2a, 0x80, 0x1e, 0x5d, -- 0xe0, 0xe0, 0x90, 0xd1, 0x00, 0x73, 0xfe, 0x0f, 0x9a, 0xae, 0x77, 0xa0, 0x91, 0x68, 0x03, 0x06, -- 0x75, 0xc5, 0x6e, 0x66, 0xc1, 0x5a, 0x65, 0xa9, 0xce, 0x37, 0xb0, 0x3e, 0x22, 0xe3, 0xd8, 0x9f, -- 0xbc, 0x42, 0x7b, 0x37, 0x60, 0x99, 0x2b, 0x99, 0xca, 0xd4, 0xae, 0x6b, 0x20, 0xe7, 0x10, 0xd0, -- 0xd7, 0x3e, 0x11, 0xaf, 0x4e, 0x93, 0xf3, 0x3e, 0xac, 0x95, 0x24, 0xf2, 0x84, 0xc6, 0x1c, 0x2b, -- 0x03, 0x84, 0x2f, 0x52, 0xae, 0x84, 0x2d, 0xb9, 0x06, 0x72, 0x30, 0xac, 0x3f, 0x21, 0xdc, 0xb2, -- 0xe3, 0xbf, 0xc7, 0x84, 0x0d, 0x58, 0x3e, 0xa1, 0x2c, 0xf2, 0x85, 0xb5, 0x40, 0x43, 0x08, 0x41, -- 0xdd, 0x67, 0x63, 0x3e, 0xa8, 0xdd, 0xa9, 0x6d, 0xb6, 0x5c, 0xf5, 0x2f, 0x57, 0xe5, 0x8c, 0x1a, -- 0x63, 0xd7, 0x1b, 0xd0, 0x31, 0x71, 0xf7, 0x26, 0x84, 0x0b, 0xa5, 0xa7, 0xe3, 0xb6, 0x0d, 0x4e, -- 0x8e, 0x71, 0x28, 0x6c, 0x1c, 0x25, 0xe1, 0x4b, 0x6e, 0xf8, 0x1d, 0x68, 0x31, 0xcc, 0x69, 0xca, -- 0xe4, 0x36, 0xad, 0xaa, 0x79, 0x5f, 0xd7, 0xf3, 0xfe, 0x84, 0xc4, 0xe9, 0x85, 0x6b, 0x69, 0x6e, -- 0xce, 0x66, 0xb6, 0x90, 0xe0, 0x2f, 0xb3, 0x85, 0x3e, 0x87, 0x6b, 0x87, 0x7e, 0xca, 0x5f, 0xc6, -- 0x56, 0xe7, 0x0b, 0xb9, 0xfd, 0x78, 0x1a, 0xbd, 0xd4, 0xe0, 0x5f, 0x56, 0xa0, 0xb9, 0x97, 0xa4, -- 0x47, 0xdc, 0x1f, 0x63, 0xf4, 0x3a, 0xb4, 0x05, 0x15, 0xfe, 0xc4, 0x4b, 0x25, 0xa8, 0xd8, 0xeb, -- 0x2e, 0x28, 0x94, 0x66, 0x90, 0x61, 0xc7, 0x2c, 0x48, 0x52, 0xc3, 0x51, 0xbd, 0x53, 0xdb, 0xac, -- 0xbb, 0x6d, 0x8d, 0xd3, 0x2c, 0x5b, 0xb0, 0xa6, 0x68, 0x1e, 0x89, 0xbd, 0x33, 0xcc, 0x62, 0x3c, -- 0x89, 0x68, 0x88, 0xd5, 0xfa, 0xad, 0xbb, 0xab, 0x8a, 0x74, 0x10, 0x7f, 0x95, 0x11, 0xd0, 0x3f, -- 0xc1, 0x6a, 0xc6, 0x2f, 0x37, 0xa5, 0xe2, 0xae, 0x2b, 0xee, 0xbe, 0xe1, 0x3e, 0x32, 0x68, 0xe7, -- 0xff, 0xa1, 0xf7, 0xfc, 0x94, 0x51, 0x21, 0x26, 0x24, 0x1e, 0x3f, 0xf4, 0x85, 0x2f, 0xb3, 0x47, -- 0x82, 0x19, 0xa1, 0x21, 0x37, 0xd6, 0x5a, 0x10, 0xbd, 0x07, 0xab, 0x42, 0xf3, 0xe2, 0xd0, 0xb3, -- 0x3c, 0x55, 0xc5, 0xb3, 0x92, 0x11, 0x0e, 0x0d, 0xf3, 0xdb, 0xd0, 0xcb, 0x99, 0x65, 0xfe, 0x31, -- 0xf6, 0x76, 0x33, 0xec, 0x73, 0x12, 0x61, 0xe7, 0x5c, 0xc5, 0x4a, 0x4d, 0x32, 0x7a, 0x0f, 0x5a, -- 0x79, 0x1c, 0x2a, 0x6a, 0x85, 0xf4, 0xf4, 0x0a, 0xb1, 0xe1, 0x74, 0x9b, 0x59, 0x50, 0xbe, 0x84, -- 0xbe, 0xc8, 0x0c, 0xf7, 0x42, 0x5f, 0xf8, 0xe5, 0x45, 0x55, 0xf6, 0xca, 0xed, 0x89, 0x12, 0xec, -- 0x7c, 0x01, 0xad, 0x43, 0x12, 0x72, 0xad, 0x78, 0x00, 0x8d, 0x20, 0x65, 0x0c, 0xc7, 0xc2, 0xba, -- 0x6c, 0x40, 0xb4, 0x0e, 0x4b, 0x13, 0x12, 0x11, 0x61, 0xdc, 0xd4, 0x80, 0x43, 0x01, 0x9e, 0xe2, -- 0x88, 0xb2, 0xa9, 0x0a, 0xd8, 0x3a, 0x2c, 0x15, 0x27, 0x57, 0x03, 0xe8, 0x26, 0xb4, 0x22, 0xff, -- 0x22, 0x9b, 0x54, 0x49, 0x69, 0x46, 0xfe, 0x85, 0x36, 0x7e, 0x00, 0x8d, 0x13, 0x9f, 0x4c, 0x82, -- 0x58, 0x98, 0xa8, 0x58, 0x30, 0x57, 0x58, 0x2f, 0x2a, 0xfc, 0x5d, 0x15, 0xda, 0x5a, 0xa3, 0x36, -- 0x78, 0x1d, 0x96, 0x02, 0x3f, 0x38, 0xcd, 0x54, 0x2a, 0x00, 0xdd, 0xb5, 0x86, 0x54, 0x8b, 0x49, -- 0x38, 0xb7, 0xd4, 0x9a, 0xb6, 0x0d, 0xc0, 0x5f, 0xf8, 0x89, 0xb1, 0xad, 0x76, 0x09, 0x73, 0x4b, -- 0xf2, 0x68, 0x73, 0x3f, 0x84, 0x8e, 0x5e, 0x77, 0x66, 0x48, 0xfd, 0x92, 0x21, 0x6d, 0xcd, 0xa5, -- 0x07, 0xbd, 0x09, 0xdd, 0x94, 0x63, 0xef, 0x94, 0x60, 0xe6, 0xb3, 0xe0, 0x74, 0x3a, 0x58, 0xd2, -- 0x67, 0x64, 0xca, 0xf1, 0xbe, 0xc5, 0xa1, 0x1d, 0x58, 0x92, 0xe9, 0x8f, 0x0f, 0x96, 0xd5, 0x71, -- 0x7c, 0xab, 0x28, 0x52, 0xb9, 0xba, 0xa5, 0xbe, 0x8f, 0x62, 0xc1, 0xa6, 0xae, 0x66, 0x1d, 0x7e, -- 0x0a, 0x90, 0x23, 0xd1, 0x0a, 0xd4, 0xce, 0xf0, 0xd4, 0xec, 0x43, 0xf9, 0x2b, 0x83, 0x73, 0xee, -- 0x4f, 0x52, 0x1b, 0x75, 0x0d, 0x7c, 0x5e, 0xfd, 0xb4, 0xe2, 0x04, 0xd0, 0xdf, 0x9d, 0x9c, 0x11, -- 0x5a, 0x18, 0xbe, 0x0e, 0x4b, 0x91, 0xff, 0x0d, 0x65, 0x36, 0x92, 0x0a, 0x50, 0x58, 0x12, 0x53, -- 0x66, 0x45, 0x28, 0x00, 0xf5, 0xa0, 0x4a, 0x13, 0x15, 0xaf, 0x96, 0x5b, 0xa5, 0x49, 0xae, 0xa8, -- 0x5e, 0x50, 0xe4, 0xfc, 0xb9, 0x0e, 0x90, 0x6b, 0x41, 0x2e, 0x0c, 0x09, 0xf5, 0x38, 0x66, 0xb2, -- 0x04, 0xf1, 0x8e, 0xa7, 0x02, 0x73, 0x8f, 0xe1, 0x20, 0x65, 0x9c, 0x9c, 0xcb, 0xf9, 0x93, 0x6e, -- 0x5f, 0xd3, 0x6e, 0xcf, 0xd8, 0xe6, 0x5e, 0x27, 0x74, 0xa4, 0xc7, 0xed, 0xca, 0x61, 0xae, 0x1d, -- 0x85, 0x0e, 0xe0, 0x5a, 0x2e, 0x33, 0x2c, 0x88, 0xab, 0x5e, 0x25, 0x6e, 0x2d, 0x13, 0x17, 0xe6, -- 0xa2, 0x1e, 0xc1, 0x1a, 0xa1, 0xde, 0xb7, 0x29, 0x4e, 0x4b, 0x82, 0x6a, 0x57, 0x09, 0x5a, 0x25, -- 0xf4, 0x3f, 0xd4, 0x80, 0x5c, 0xcc, 0x21, 0xdc, 0x28, 0x78, 0x29, 0xb7, 0x7b, 0x41, 0x58, 0xfd, -- 0x2a, 0x61, 0x1b, 0x99, 0x55, 0x32, 0x1f, 0xe4, 0x12, 0xff, 0x0d, 0x36, 0x08, 0xf5, 0x5e, 0xf8, -- 0x44, 0xcc, 0x8a, 0x5b, 0xfa, 0x11, 0x27, 0xe5, 0xa1, 0x5b, 0x96, 0xa5, 0x9d, 0x8c, 0x30, 0x1b, -- 0x97, 0x9c, 0x5c, 0xfe, 0x11, 0x27, 0x9f, 0xaa, 0x01, 0xb9, 0x98, 0x07, 0xb0, 0x4a, 0xe8, 0xac, -- 0x35, 0x8d, 0xab, 0x84, 0xf4, 0x09, 0x2d, 0x5b, 0xb2, 0x0b, 0xab, 0x1c, 0x07, 0x82, 0xb2, 0xe2, -- 0x22, 0x68, 0x5e, 0x25, 0x62, 0xc5, 0xf0, 0x67, 0x32, 0x9c, 0xff, 0x81, 0xce, 0x7e, 0x3a, 0xc6, -- 0x62, 0x72, 0x9c, 0x25, 0x83, 0x57, 0x96, 0x7f, 0x9c, 0xbf, 0x56, 0xa1, 0xbd, 0x37, 0x66, 0x34, -- 0x4d, 0x4a, 0x39, 0x59, 0x6f, 0xd2, 0xd9, 0x9c, 0xac, 0x58, 0x54, 0x4e, 0xd6, 0xcc, 0x1f, 0x41, -- 0x27, 0x52, 0x5b, 0xd7, 0xf0, 0xeb, 0x3c, 0xb4, 0x3a, 0xb7, 0xa9, 0xdd, 0x76, 0x54, 0x48, 0x66, -- 0x5b, 0x00, 0x09, 0x09, 0xb9, 0x19, 0xa3, 0xd3, 0x51, 0xdf, 0x54, 0x84, 0x36, 0x45, 0xbb, 0xad, -- 0x24, 0xcb, 0xd6, 0xf7, 0xa0, 0x7d, 0x2c, 0x83, 0x64, 0x06, 0x94, 0x92, 0x51, 0x1e, 0x3d, 0x17, -- 0x8e, 0xf3, 0x4d, 0xb8, 0x0f, 0xdd, 0x53, 0x1d, 0x32, 0x33, 0x48, 0xaf, 0xa1, 0x37, 0x8d, 0x27, -- 0xb9, 0xbf, 0x5b, 0xc5, 0xc8, 0xea, 0x09, 0xe8, 0x9c, 0x16, 0x50, 0xc3, 0x11, 0xac, 0xce, 0xb1, -- 0x2c, 0xc8, 0x41, 0x9b, 0xc5, 0x1c, 0xd4, 0xde, 0x41, 0x5a, 0x51, 0x71, 0x64, 0x31, 0x2f, 0xfd, -- 0xac, 0x0a, 0xbd, 0x83, 0x58, 0x60, 0x76, 0xe2, 0x07, 0x58, 0x5b, 0x8c, 0xa0, 0x1e, 0xfb, 0x11, -- 0x36, 0x32, 0xd5, 0x3f, 0xba, 0x01, 0x4d, 0x76, 0xa1, 0x53, 0x88, 0x99, 0xd1, 0x06, 0xbb, 0x50, -- 0xa9, 0x01, 0xbd, 0x06, 0xc0, 0x2e, 0xbc, 0xc4, 0x0f, 0xce, 0xb0, 0x89, 0x61, 0xdd, 0x6d, 0xb1, -- 0x8b, 0x43, 0x8d, 0x90, 0x8b, 0x81, 0x5d, 0x78, 0x98, 0x31, 0xca, 0xb8, 0xc9, 0x56, 0x4d, 0x76, -- 0xf1, 0x48, 0xc1, 0x66, 0x6c, 0xc8, 0x68, 0x92, 0xe0, 0x50, 0x65, 0x69, 0x35, 0xf6, 0xa1, 0x46, -- 0x48, 0xad, 0xc2, 0x6a, 0x5d, 0xd6, 0x5a, 0x45, 0xae, 0x55, 0xe4, 0x5a, 0x1b, 0x7a, 0xa4, 0x28, -- 0x6a, 0x15, 0x99, 0xd6, 0xa6, 0xd6, 0x2a, 0x0a, 0x5a, 0x45, 0xae, 0xb5, 0x65, 0xc7, 0x1a, 0xad, -- 0xce, 0x6f, 0xaa, 0xd0, 0x78, 0x1e, 0xa8, 0x49, 0x41, 0x77, 0xa0, 0x8d, 0xb9, 0xf0, 0x8f, 0x27, -- 0x84, 0x9f, 0xe2, 0xd0, 0x2c, 0xf3, 0x22, 0x4a, 0xda, 0xc8, 0xa7, 0xb1, 0xc7, 0xe5, 0x09, 0x6e, -- 0x22, 0xc3, 0xa7, 0xf1, 0x48, 0x9e, 0xe0, 0x86, 0xc4, 0x70, 0x70, 0x6e, 0xd7, 0x3a, 0x9f, 0xc6, -- 0x2e, 0x0e, 0xce, 0xa5, 0x7d, 0x27, 0x24, 0x56, 0x39, 0xe6, 0x9e, 0x8d, 0xca, 0x09, 0x89, 0x65, -- 0xfe, 0xb8, 0x57, 0x24, 0xee, 0x98, 0xa0, 0x58, 0xe2, 0x8e, 0xf2, 0x4c, 0xa6, 0x01, 0x49, 0x35, -- 0x41, 0x69, 0x4a, 0x84, 0xa4, 0xaa, 0xc3, 0x79, 0x42, 0x39, 0x36, 0x01, 0xd1, 0x80, 0xf4, 0x57, -- 0xfd, 0xe8, 0x31, 0x3a, 0x1a, 0x2d, 0x85, 0x51, 0x83, 0x6e, 0x40, 0x73, 0xe2, 0x73, 0xe1, 0xf9, -- 0xc1, 0x99, 0x09, 0x46, 0x43, 0xc2, 0x0f, 0x82, 0x33, 0x59, 0xdd, 0xcb, 0x82, 0x1c, 0xc7, 0x03, -- 0x50, 0x04, 0x03, 0xa9, 0xaa, 0x65, 0x42, 0x39, 0x89, 0xc7, 0x83, 0xb6, 0xa9, 0x5a, 0x34, 0xe8, -- 0xa4, 0xd0, 0x38, 0x0a, 0x75, 0xec, 0xf2, 0xc1, 0x95, 0xd9, 0xc1, 0x36, 0xf6, 0x26, 0x60, 0x06, -- 0x34, 0x6b, 0x45, 0x9f, 0x08, 0x26, 0x62, 0x4d, 0x76, 0xa1, 0x13, 0xbe, 0x99, 0x52, 0x43, 0xac, -- 0xdb, 0x29, 0xd5, 0x44, 0xe7, 0x0f, 0x15, 0xe8, 0xfc, 0x3b, 0x16, 0x2f, 0x28, 0x3b, 0xb3, 0xf9, -- 0x00, 0x88, 0x5d, 0xd6, 0xdc, 0x9c, 0x75, 0xa6, 0x3c, 0x2b, 0x2f, 0x77, 0xb7, 0xc0, 0x87, 0x5e, -- 0x87, 0x9a, 0x08, 0x12, 0xb3, 0x73, 0x4c, 0x6b, 0x68, 0x96, 0x82, 0x2b, 0x29, 0xe8, 0x0d, 0xa8, -- 0x8b, 0x20, 0xf9, 0xd8, 0xa4, 0x8a, 0x19, 0x0e, 0x45, 0x92, 0x32, 0xd2, 0x30, 0x29, 0xb7, 0x97, -- 0x26, 0x24, 0xae, 0xa4, 0x48, 0x19, 0x69, 0x98, 0x7c, 0xac, 0x66, 0x76, 0x8e, 0x43, 0x91, 0x9c, -- 0x9f, 0x56, 0x60, 0x63, 0xb6, 0xfb, 0x30, 0xbd, 0xd2, 0x47, 0xd0, 0x09, 0x54, 0xd2, 0x28, 0x25, -- 0xc6, 0xd5, 0xb9, 0x74, 0xe2, 0xb6, 0x83, 0x42, 0x2e, 0xfd, 0x04, 0xba, 0xb1, 0x0e, 0x4f, 0x29, -- 0x3f, 0x9a, 0xe4, 0x50, 0x8c, 0x9c, 0xdb, 0x89, 0x0b, 0x90, 0x13, 0x02, 0xfa, 0x9a, 0x11, 0x81, -- 0x47, 0x82, 0x61, 0x3f, 0x7a, 0x15, 0x5d, 0x30, 0x82, 0xba, 0x2a, 0x99, 0x6b, 0xaa, 0xc9, 0x53, -- 0xff, 0xce, 0x3b, 0xb0, 0x56, 0xd2, 0x62, 0x7c, 0x5d, 0x81, 0xda, 0xc4, 0x2c, 0x9f, 0xae, 0x2b, -- 0x7f, 0x1d, 0x1f, 0x56, 0x5d, 0xec, 0x87, 0xaf, 0xce, 0x1a, 0xa3, 0xa2, 0x96, 0xab, 0xd8, 0x04, -- 0x54, 0x54, 0x61, 0x4c, 0xb1, 0x56, 0x57, 0x0a, 0x56, 0x3f, 0x83, 0xd5, 0x3d, 0xb9, 0x8b, 0x46, -- 0x22, 0x24, 0xf1, 0xab, 0x68, 0xdb, 0xff, 0x0f, 0xd6, 0x9e, 0x8b, 0xe9, 0xd7, 0x52, 0x18, 0x27, -- 0xdf, 0xe1, 0x57, 0xe4, 0x1f, 0xa3, 0x2f, 0xac, 0x7f, 0x8c, 0xbe, 0x90, 0xdb, 0x32, 0xa0, 0x93, -- 0x34, 0x8a, 0xd5, 0x12, 0xed, 0xba, 0x06, 0x72, 0x76, 0xa1, 0xa3, 0x1b, 0xb9, 0xa7, 0x34, 0x4c, -- 0x27, 0x78, 0xe1, 0x31, 0x70, 0x1b, 0x20, 0xf1, 0x99, 0x1f, 0x61, 0x81, 0x19, 0x57, 0x25, 0x5f, -- 0xcb, 0x2d, 0x60, 0x9c, 0x9f, 0x57, 0x61, 0x5d, 0xdf, 0xcb, 0x8d, 0xf4, 0x75, 0x94, 0x75, 0x61, -- 0x08, 0xcd, 0x53, 0xca, 0x45, 0x41, 0x60, 0x06, 0x4b, 0x13, 0xc3, 0xd8, 0x4a, 0x93, 0xbf, 0xa5, -- 0xcb, 0xb2, 0xda, 0xd5, 0x97, 0x65, 0x73, 0xd7, 0x61, 0xf5, 0xf9, 0xeb, 0x30, 0x99, 0x00, 0x2d, -- 0x13, 0xd1, 0xc7, 0x4c, 0xcb, 0x6d, 0x19, 0xcc, 0x41, 0x88, 0xee, 0x42, 0x7f, 0x2c, 0xad, 0xf4, -- 0x4e, 0x29, 0x3d, 0xf3, 0x12, 0x5f, 0x9c, 0xaa, 0xc4, 0xda, 0x72, 0xbb, 0x0a, 0xbd, 0x4f, 0xe9, -- 0xd9, 0xa1, 0x2f, 0x4e, 0xd1, 0x67, 0xd0, 0x33, 0xbd, 0x48, 0xa4, 0x42, 0xc4, 0x4d, 0x05, 0x66, -- 0x76, 0x51, 0x31, 0x7a, 0x6e, 0xf7, 0xac, 0x00, 0x71, 0xe7, 0x3a, 0x5c, 0x7b, 0x88, 0xb9, 0x60, -- 0x74, 0x5a, 0x0e, 0x8c, 0xf3, 0x2f, 0x00, 0x07, 0x79, 0xfe, 0xf9, 0xa0, 0x08, 0x99, 0xac, 0xb5, -- 0xb2, 0xa5, 0xaf, 0x45, 0x33, 0x82, 0x5b, 0xe0, 0x71, 0xb6, 0x60, 0xd9, 0xa5, 0xa9, 0x3c, 0x11, -- 0xdf, 0xb2, 0x7f, 0x66, 0x5c, 0xc7, 0x8c, 0x53, 0x48, 0xd7, 0xd0, 0x9c, 0x7d, 0x7b, 0x8f, 0x92, -- 0x8b, 0x33, 0x53, 0xb4, 0x05, 0xad, 0x2c, 0x13, 0x9a, 0xac, 0x32, 0xaf, 0x3a, 0x67, 0x71, 0xfe, -- 0x1b, 0xd6, 0xb4, 0x24, 0x2d, 0xd9, 0x8a, 0x79, 0x0b, 0x96, 0x99, 0x35, 0xa3, 0x92, 0xdf, 0x87, -- 0x1a, 0x26, 0x43, 0x43, 0xb7, 0xa4, 0xb2, 0x80, 0xe1, 0xc8, 0x1e, 0x9b, 0x4d, 0x37, 0x47, 0xc8, -- 0x68, 0x3d, 0x21, 0x5c, 0xe4, 0x6e, 0xda, 0x68, 0xad, 0xc1, 0xaa, 0x24, 0x94, 0x34, 0x3a, 0xff, -- 0x0b, 0x6b, 0xcf, 0xe2, 0x09, 0x89, 0xf1, 0xde, 0xe1, 0xd1, 0x53, 0x9c, 0x65, 0x05, 0x04, 0x75, -- 0x75, 0xde, 0x55, 0x94, 0x74, 0xf5, 0x2f, 0xb7, 0x49, 0x7c, 0xec, 0x05, 0x49, 0xca, 0xcd, 0xf5, -- 0xe4, 0x72, 0x7c, 0xbc, 0x97, 0xa4, 0x5c, 0x9e, 0x81, 0xb2, 0xd6, 0xa4, 0xf1, 0x64, 0xaa, 0xf6, -- 0x4a, 0xd3, 0x6d, 0x04, 0x49, 0xfa, 0x2c, 0x9e, 0x4c, 0x9d, 0x7f, 0x56, 0x17, 0x32, 0x18, 0x87, -- 0xae, 0x1f, 0x87, 0x34, 0x7a, 0x88, 0xcf, 0x0b, 0x1a, 0xb2, 0xe6, 0xdf, 0xe6, 0x84, 0xef, 0x2b, -- 0xd0, 0x79, 0x30, 0xc6, 0xb1, 0x78, 0x88, 0x85, 0x4f, 0x26, 0xaa, 0xc1, 0x3f, 0xc7, 0x8c, 0x13, -- 0x1a, 0x9b, 0x85, 0x6f, 0x41, 0xf4, 0x3a, 0xb4, 0x49, 0x4c, 0x84, 0x17, 0xfa, 0x38, 0xa2, 0xb1, -- 0x89, 0x02, 0x48, 0xd4, 0x43, 0x85, 0x41, 0xef, 0x40, 0x5f, 0x5f, 0x1f, 0x7b, 0xa7, 0x7e, 0x1c, -- 0x4e, 0xe4, 0x96, 0xd3, 0xd7, 0x69, 0x3d, 0x8d, 0xde, 0x37, 0x58, 0xf4, 0x2e, 0xac, 0x98, 0x0d, -- 0x91, 0x73, 0xd6, 0x15, 0x67, 0xdf, 0xe0, 0x4b, 0xac, 0x69, 0x92, 0x50, 0x26, 0xb8, 0xc7, 0x71, -- 0x10, 0xd0, 0x28, 0x31, 0xdd, 0x71, 0xdf, 0xe2, 0x47, 0x1a, 0xed, 0x8c, 0x61, 0xed, 0xb1, 0xf4, -- 0xd3, 0x78, 0x92, 0x4f, 0x70, 0x2f, 0xc2, 0x91, 0x77, 0x3c, 0xa1, 0xc1, 0x99, 0x27, 0xd3, 0x94, -- 0x89, 0xb0, 0xac, 0xbf, 0x77, 0x25, 0x72, 0x44, 0xbe, 0x53, 0x17, 0x41, 0x92, 0xeb, 0x94, 0x8a, -- 0x64, 0x92, 0x8e, 0xbd, 0x84, 0xd1, 0x63, 0x6c, 0x5c, 0xec, 0x47, 0x38, 0xda, 0xd7, 0xf8, 0x43, -- 0x89, 0x76, 0x7e, 0x5b, 0x81, 0xf5, 0xb2, 0x26, 0x93, 0x74, 0xb7, 0x61, 0xbd, 0xac, 0xca, 0xd4, -- 0x82, 0xba, 0x9e, 0x58, 0x2d, 0x2a, 0xd4, 0x55, 0xe1, 0x27, 0xd0, 0x55, 0x6f, 0x0a, 0x5e, 0xa8, -- 0x25, 0x95, 0x8f, 0xb9, 0xe2, 0xbc, 0xb8, 0x1d, 0xbf, 0x38, 0x4b, 0x9f, 0xc1, 0x0d, 0xe3, 0xbe, -- 0x37, 0x6f, 0xb6, 0x5e, 0x10, 0x1b, 0x86, 0xe1, 0xe9, 0x8c, 0xf5, 0x4f, 0x60, 0x90, 0xa3, 0x76, -- 0xa7, 0x0a, 0x69, 0x63, 0xf5, 0x01, 0xac, 0xcd, 0x38, 0xfb, 0x20, 0x0c, 0x99, 0xda, 0xa0, 0x75, -- 0x77, 0x11, 0xc9, 0xb9, 0x0f, 0xd7, 0x47, 0x58, 0xe8, 0x68, 0xf8, 0xc2, 0x34, 0xa6, 0x5a, 0xd8, -- 0x0a, 0xd4, 0x46, 0x38, 0x50, 0xce, 0xd7, 0x5c, 0xf9, 0x2b, 0x17, 0xe0, 0x11, 0xc7, 0x81, 0xf2, -- 0xb2, 0xe6, 0xaa, 0x7f, 0xe7, 0xd7, 0x15, 0x68, 0x98, 0x34, 0x29, 0x53, 0x7d, 0xc8, 0xc8, 0x39, -- 0x66, 0x66, 0xe9, 0x19, 0x08, 0xbd, 0x0d, 0x3d, 0xfd, 0xe7, 0xd1, 0x44, 0x10, 0x9a, 0x25, 0xdf, -- 0xae, 0xc6, 0x3e, 0xd3, 0x48, 0x75, 0x5d, 0xac, 0x6e, 0x43, 0xcd, 0xc5, 0x83, 0x81, 0xd4, 0x9d, -- 0x2f, 0x97, 0x99, 0x41, 0x25, 0xdb, 0x96, 0x6b, 0x20, 0xb9, 0xd4, 0xad, 0xbc, 0x25, 0x25, 0xcf, -- 0x82, 0x72, 0xa9, 0x47, 0x34, 0x8d, 0x85, 0x97, 0x50, 0x12, 0x0b, 0x93, 0x5d, 0x41, 0xa1, 0x0e, -- 0x25, 0xc6, 0xf9, 0x49, 0x05, 0x96, 0xf5, 0x93, 0x09, 0xea, 0x41, 0x35, 0x3b, 0xe3, 0xaa, 0x44, -- 0xd5, 0x0b, 0x4a, 0x97, 0x3e, 0xd7, 0xd4, 0xbf, 0xdc, 0xc7, 0xe7, 0x91, 0xce, 0xd4, 0xc6, 0xb4, -- 0xf3, 0x48, 0xa5, 0xe8, 0xb7, 0xa1, 0x97, 0x1f, 0x95, 0x8a, 0xae, 0x4d, 0xec, 0x66, 0x58, 0xc5, -- 0x76, 0xa9, 0xa5, 0xce, 0x7f, 0x01, 0xe4, 0x4f, 0x07, 0x32, 0xe4, 0x69, 0x66, 0x8c, 0xfc, 0x95, -- 0x98, 0x71, 0x76, 0xc8, 0xca, 0x5f, 0x74, 0x17, 0x7a, 0x7e, 0x18, 0x12, 0x39, 0xdc, 0x9f, 0x3c, -- 0x26, 0x61, 0xb6, 0x49, 0xcb, 0x58, 0xe7, 0xf7, 0x15, 0xe8, 0xef, 0xd1, 0x64, 0xfa, 0xaf, 0x64, -- 0x82, 0x0b, 0x19, 0x44, 0x19, 0x69, 0xce, 0x58, 0xf9, 0xaf, 0xab, 0xff, 0x09, 0xd6, 0x5b, 0x4b, -- 0xcf, 0x6c, 0x53, 0x22, 0xd4, 0xb6, 0xb2, 0xc4, 0xec, 0x16, 0xb6, 0xab, 0x89, 0x4f, 0x69, 0xa8, -- 0x9a, 0xb4, 0x90, 0x30, 0x2f, 0xbb, 0x73, 0xed, 0xba, 0x8d, 0x90, 0x30, 0x45, 0x32, 0x8e, 0x2c, -- 0xa9, 0x6b, 0xff, 0xa2, 0x23, 0xcb, 0x1a, 0x23, 0x1d, 0xd9, 0x80, 0x65, 0x7a, 0x72, 0xc2, 0xb1, -- 0x50, 0xdd, 0x43, 0xcd, 0x35, 0x50, 0x96, 0xe6, 0x9a, 0x85, 0x34, 0x77, 0x0d, 0xd6, 0xd4, 0x03, -- 0xd3, 0x73, 0xe6, 0x07, 0x24, 0x1e, 0xdb, 0x54, 0xbc, 0x0e, 0x68, 0x24, 0x68, 0x32, 0x83, 0xdd, -- 0x82, 0x55, 0x73, 0xe6, 0x1c, 0xfe, 0xe7, 0xc8, 0xba, 0x7e, 0x03, 0x9a, 0x12, 0xf4, 0x18, 0xfe, -- 0xd6, 0x26, 0x46, 0x43, 0x76, 0xde, 0x85, 0x8e, 0xfe, 0x35, 0x69, 0x20, 0x67, 0xe5, 0x65, 0x56, -- 0xbe, 0xf3, 0xa7, 0x15, 0x93, 0x6e, 0xcd, 0x45, 0x0e, 0x7a, 0x0c, 0xfd, 0x99, 0x87, 0x41, 0x64, -- 0x6e, 0xf6, 0x16, 0xbf, 0x17, 0x0e, 0x37, 0xb6, 0xf4, 0x43, 0xe3, 0x96, 0x7d, 0x68, 0xdc, 0x7a, -- 0x14, 0x25, 0x62, 0x8a, 0x1e, 0x41, 0xaf, 0xfc, 0x84, 0x86, 0x6e, 0xda, 0x1a, 0x64, 0xc1, 0xc3, -- 0xda, 0xa5, 0x62, 0x1e, 0x43, 0x7f, 0xe6, 0x35, 0xcd, 0xda, 0xb3, 0xf8, 0x91, 0xed, 0x52, 0x41, -- 0xf7, 0xa1, 0x5d, 0x78, 0x3e, 0x43, 0x03, 0x2d, 0x64, 0xfe, 0x45, 0xed, 0x52, 0x01, 0x7b, 0xd0, -- 0x2d, 0xbd, 0x68, 0xa1, 0xa1, 0xf1, 0x67, 0xc1, 0x33, 0xd7, 0xa5, 0x42, 0x76, 0xa1, 0x5d, 0x78, -- 0x58, 0xb2, 0x56, 0xcc, 0xbf, 0x5e, 0x0d, 0x6f, 0x2c, 0xa0, 0x98, 0xe9, 0xdc, 0x87, 0x6e, 0xe9, -- 0x19, 0xc8, 0x1a, 0xb2, 0xe8, 0x09, 0x6a, 0x78, 0x73, 0x21, 0xcd, 0x48, 0x7a, 0x0c, 0xfd, 0x99, -- 0x47, 0x21, 0x1b, 0xdc, 0xc5, 0x6f, 0x45, 0x97, 0xba, 0xf5, 0x95, 0x9a, 0xec, 0x42, 0xbb, 0x55, -- 0x98, 0xec, 0xf9, 0x27, 0xa0, 0xe1, 0xad, 0xc5, 0x44, 0x63, 0xd5, 0x23, 0xe8, 0x95, 0x5f, 0x7f, -- 0xac, 0xb0, 0x85, 0x6f, 0x42, 0x57, 0xaf, 0x9c, 0xd2, 0x43, 0x50, 0xbe, 0x72, 0x16, 0xbd, 0x0f, -- 0x5d, 0x2a, 0xe8, 0x01, 0x80, 0x69, 0xae, 0x42, 0x12, 0x67, 0x53, 0x36, 0xd7, 0xd4, 0x65, 0x53, -- 0xb6, 0xa0, 0x11, 0xbb, 0x0f, 0xa0, 0x7b, 0xa2, 0x90, 0xa6, 0x02, 0x5d, 0xb7, 0x66, 0xcc, 0x34, -- 0x62, 0xc3, 0xc1, 0x3c, 0x61, 0x4e, 0x00, 0x66, 0xec, 0x65, 0x04, 0x7c, 0x09, 0x90, 0xf7, 0x5a, -- 0x56, 0xc0, 0x5c, 0xf7, 0x75, 0x45, 0x0c, 0x3a, 0xc5, 0xce, 0x0a, 0x19, 0x5f, 0x17, 0x74, 0x5b, -- 0x57, 0x88, 0xe8, 0xcf, 0x54, 0xce, 0xe5, 0xc5, 0x36, 0x5b, 0x50, 0x0f, 0xe7, 0xaa, 0x67, 0xf4, -- 0x09, 0x74, 0x8a, 0x25, 0xb3, 0xb5, 0x62, 0x41, 0x19, 0x3d, 0x2c, 0x95, 0xcd, 0xe8, 0x3e, 0xf4, -- 0xca, 0x05, 0x31, 0x2a, 0xec, 0x8b, 0xb9, 0x32, 0x79, 0xb8, 0x32, 0x73, 0xd1, 0xc1, 0xd1, 0x87, -- 0x00, 0x79, 0xe1, 0x6c, 0xc3, 0x37, 0x57, 0x4a, 0xcf, 0x68, 0xfd, 0x12, 0x7a, 0x85, 0xbc, 0x2d, -- 0x7b, 0xc2, 0xeb, 0x25, 0x87, 0xf3, 0x6c, 0x3e, 0x34, 0x15, 0x56, 0x29, 0x6d, 0x3f, 0x80, 0x4e, -- 0xf1, 0x8c, 0xb0, 0xde, 0x2e, 0x38, 0x37, 0xae, 0x4a, 0x7a, 0x85, 0xf3, 0xc4, 0xae, 0xdd, 0xf9, -- 0x23, 0xe6, 0xaa, 0xa4, 0x57, 0xea, 0x47, 0x6d, 0xae, 0x59, 0xd4, 0xa4, 0x5e, 0x75, 0x14, 0x94, -- 0x9b, 0x37, 0x1b, 0xfd, 0x85, 0x2d, 0xdd, 0x55, 0x6b, 0xb0, 0xd8, 0xa7, 0xd8, 0x78, 0x2c, 0xe8, -- 0x5d, 0x7e, 0x24, 0x27, 0x14, 0x7b, 0x91, 0x42, 0x4e, 0x58, 0xd0, 0xa2, 0x5c, 0x2a, 0x68, 0x1f, -- 0xfa, 0x8f, 0x6d, 0x99, 0x69, 0x4a, 0x60, 0x63, 0xce, 0x82, 0x92, 0x7f, 0x38, 0x5c, 0x44, 0x32, -- 0xb3, 0xfc, 0x15, 0xac, 0xce, 0x95, 0xbf, 0xe8, 0x76, 0x76, 0xef, 0xbe, 0xb0, 0x2e, 0xbe, 0xd4, -- 0xac, 0x03, 0x58, 0x99, 0xad, 0x7e, 0xd1, 0x6b, 0x66, 0xd2, 0x17, 0x57, 0xc5, 0x97, 0x8a, 0xfa, -- 0x0c, 0x9a, 0xb6, 0xda, 0x42, 0xe6, 0x7d, 0x63, 0xa6, 0xfa, 0xba, 0x6c, 0xe8, 0x6e, 0xe7, 0xfb, -- 0x1f, 0x6e, 0x57, 0xfe, 0xf8, 0xc3, 0xed, 0xca, 0x5f, 0x7e, 0xb8, 0x5d, 0x39, 0x5e, 0x56, 0xd4, -- 0x0f, 0xff, 0x16, 0x00, 0x00, 0xff, 0xff, 0x9a, 0x16, 0x45, 0x76, 0xe7, 0x24, 0x00, 0x00, -+ // 3214 bytes of a gzipped FileDescriptorProto -+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0xcb, 0x6e, 0x1c, 0xc7, -+ 0x76, 0x98, 0x07, 0x39, 0x33, 0x67, 0x5e, 0x9c, 0x22, 0x45, 0x8d, 0x46, 0xb2, 0x2c, 0xb7, 0x6d, -+ 0x99, 0x8e, 0x63, 0xd2, 0xa2, 0x0d, 0x3f, 0xe1, 0x08, 0x22, 0xa5, 0x88, 0x8c, 0x25, 0x8b, 0x69, -+ 0x8a, 0x71, 0x8c, 0x20, 0x68, 0x34, 0xbb, 0x8b, 0xc3, 0x32, 0xa7, 0xbb, 0xda, 0x55, 0xd5, 0x14, -+ 0xc7, 0x01, 0xb2, 0x4c, 0xb6, 0x59, 0x65, 0x97, 0x1f, 0x08, 0x92, 0x55, 0x76, 0xc9, 0x36, 0x0b, -+ 0x23, 0xc8, 0xe2, 0x7e, 0xc1, 0xc5, 0x85, 0x3f, 0xe1, 0x7e, 0xc1, 0x45, 0xbd, 0xfa, 0x31, 0x33, -+ 0xa4, 0x71, 0x09, 0x01, 0x77, 0xd3, 0xe8, 0xf3, 0xa8, 0xf3, 0xaa, 0xaa, 0x53, 0xe7, 0x54, 0x41, -+ 0xdb, 0x1f, 0xe3, 0x58, 0x6c, 0x26, 0x8c, 0x0a, 0x8a, 0xea, 0x63, 0x96, 0x04, 0xa3, 0x16, 0x0d, -+ 0x88, 0x46, 0x8c, 0x3e, 0x1d, 0x13, 0x71, 0x9a, 0x1e, 0x6f, 0x06, 0x34, 0xda, 0x3a, 0xf3, 0x85, -+ 0xff, 0x61, 0x40, 0x63, 0xe1, 0x93, 0x18, 0x33, 0xbe, 0xa5, 0x06, 0x6e, 0x25, 0x67, 0xe3, 0x2d, -+ 0x31, 0x4d, 0x30, 0xd7, 0x5f, 0x33, 0xee, 0xf6, 0x98, 0xd2, 0xf1, 0x04, 0x6f, 0x29, 0xe8, 0x38, -+ 0x3d, 0xd9, 0xc2, 0x51, 0x22, 0xa6, 0x9a, 0xe8, 0xfc, 0x5b, 0x15, 0xd6, 0x77, 0x19, 0xf6, 0x05, -+ 0xde, 0xb5, 0xd2, 0x5c, 0xfc, 0x63, 0x8a, 0xb9, 0x40, 0x6f, 0x41, 0x27, 0xd3, 0xe0, 0x91, 0x70, -+ 0x58, 0xb9, 0x57, 0xd9, 0x68, 0xb9, 0xed, 0x0c, 0xb7, 0x1f, 0xa2, 0x9b, 0xd0, 0xc0, 0x17, 0x38, -+ 0x90, 0xd4, 0xaa, 0xa2, 0x2e, 0x4b, 0x70, 0x3f, 0x44, 0x0f, 0xa0, 0xcd, 0x05, 0x23, 0xf1, 0xd8, -+ 0x4b, 0x39, 0x66, 0xc3, 0xda, 0xbd, 0xca, 0x46, 0x7b, 0x7b, 0x65, 0x53, 0xba, 0xb4, 0x79, 0xa8, -+ 0x08, 0x47, 0x1c, 0x33, 0x17, 0x78, 0xf6, 0x8f, 0xee, 0x43, 0x23, 0xc4, 0xe7, 0x24, 0xc0, 0x7c, -+ 0x58, 0xbf, 0x57, 0xdb, 0x68, 0x6f, 0x77, 0x34, 0xfb, 0x63, 0x85, 0x74, 0x2d, 0x11, 0xbd, 0x0f, -+ 0x4d, 0x2e, 0x28, 0xf3, 0xc7, 0x98, 0x0f, 0x97, 0x14, 0x63, 0xd7, 0xca, 0x55, 0x58, 0x37, 0x23, -+ 0xa3, 0x3b, 0x50, 0x7b, 0xb1, 0xbb, 0x3f, 0x5c, 0x56, 0xda, 0xc1, 0x70, 0x25, 0x38, 0x70, 0x25, -+ 0x1a, 0xbd, 0x0d, 0x5d, 0xee, 0xc7, 0xe1, 0x31, 0xbd, 0xf0, 0x12, 0x12, 0xc6, 0x7c, 0xd8, 0xb8, -+ 0x57, 0xd9, 0x68, 0xba, 0x1d, 0x83, 0x3c, 0x90, 0x38, 0xe7, 0x4b, 0xb8, 0x71, 0x28, 0x7c, 0x26, -+ 0xae, 0x11, 0x1d, 0xe7, 0x08, 0xd6, 0x5d, 0x1c, 0xd1, 0xf3, 0x6b, 0x85, 0x76, 0x08, 0x0d, 0x41, -+ 0x22, 0x4c, 0x53, 0xa1, 0x42, 0xdb, 0x75, 0x2d, 0xe8, 0xfc, 0x47, 0x05, 0xd0, 0x93, 0x0b, 0x1c, -+ 0x1c, 0x30, 0x1a, 0x60, 0xce, 0xff, 0x44, 0xd3, 0xf5, 0x1e, 0x34, 0x12, 0x6d, 0xc0, 0xb0, 0xae, -+ 0xd8, 0xcd, 0x2c, 0x58, 0xab, 0x2c, 0xd5, 0xf9, 0x01, 0xd6, 0x0e, 0xc9, 0x38, 0xf6, 0x27, 0xaf, -+ 0xd1, 0xde, 0x75, 0x58, 0xe6, 0x4a, 0xa6, 0x32, 0xb5, 0xeb, 0x1a, 0xc8, 0x39, 0x00, 0xf4, 0x9d, -+ 0x4f, 0xc4, 0xeb, 0xd3, 0xe4, 0x7c, 0x08, 0xab, 0x25, 0x89, 0x3c, 0xa1, 0x31, 0xc7, 0xca, 0x00, -+ 0xe1, 0x8b, 0x94, 0x2b, 0x61, 0x4b, 0xae, 0x81, 0x1c, 0x0c, 0x6b, 0xcf, 0x08, 0xb7, 0xec, 0xf8, -+ 0x8f, 0x31, 0x61, 0x1d, 0x96, 0x4f, 0x28, 0x8b, 0x7c, 0x61, 0x2d, 0xd0, 0x10, 0x42, 0x50, 0xf7, -+ 0xd9, 0x98, 0x0f, 0x6b, 0xf7, 0x6a, 0x1b, 0x2d, 0x57, 0xfd, 0xcb, 0x55, 0x39, 0xa3, 0xc6, 0xd8, -+ 0xf5, 0x16, 0x74, 0x4c, 0xdc, 0xbd, 0x09, 0xe1, 0x42, 0xe9, 0xe9, 0xb8, 0x6d, 0x83, 0x93, 0x63, -+ 0x1c, 0x0a, 0xeb, 0x47, 0x49, 0x78, 0xcd, 0x0d, 0xbf, 0x0d, 0x2d, 0x86, 0x39, 0x4d, 0x99, 0xdc, -+ 0xa6, 0x55, 0x35, 0xef, 0x6b, 0x7a, 0xde, 0x9f, 0x91, 0x38, 0xbd, 0x70, 0x2d, 0xcd, 0xcd, 0xd9, -+ 0xcc, 0x16, 0x12, 0xfc, 0x3a, 0x5b, 0xe8, 0x4b, 0xb8, 0x71, 0xe0, 0xa7, 0xfc, 0x3a, 0xb6, 0x3a, -+ 0x5f, 0xc9, 0xed, 0xc7, 0xd3, 0xe8, 0x5a, 0x83, 0xff, 0xbd, 0x02, 0xcd, 0xdd, 0x24, 0x3d, 0xe2, -+ 0xfe, 0x18, 0xa3, 0x37, 0xa1, 0x2d, 0xa8, 0xf0, 0x27, 0x5e, 0x2a, 0x41, 0xc5, 0x5e, 0x77, 0x41, -+ 0xa1, 0x34, 0x83, 0x0c, 0x3b, 0x66, 0x41, 0x92, 0x1a, 0x8e, 0xea, 0xbd, 0xda, 0x46, 0xdd, 0x6d, -+ 0x6b, 0x9c, 0x66, 0xd9, 0x84, 0x55, 0x45, 0xf3, 0x48, 0xec, 0x9d, 0x61, 0x16, 0xe3, 0x49, 0x44, -+ 0x43, 0xac, 0xd6, 0x6f, 0xdd, 0x1d, 0x28, 0xd2, 0x7e, 0xfc, 0x4d, 0x46, 0x40, 0x7f, 0x06, 0x83, -+ 0x8c, 0x5f, 0x6e, 0x4a, 0xc5, 0x5d, 0x57, 0xdc, 0x7d, 0xc3, 0x7d, 0x64, 0xd0, 0xce, 0x3f, 0x42, -+ 0xef, 0xe5, 0x29, 0xa3, 0x42, 0x4c, 0x48, 0x3c, 0x7e, 0xec, 0x0b, 0x5f, 0x66, 0x8f, 0x04, 0x33, -+ 0x42, 0x43, 0x6e, 0xac, 0xb5, 0x20, 0xfa, 0x00, 0x06, 0x42, 0xf3, 0xe2, 0xd0, 0xb3, 0x3c, 0x55, -+ 0xc5, 0xb3, 0x92, 0x11, 0x0e, 0x0c, 0xf3, 0xbb, 0xd0, 0xcb, 0x99, 0x65, 0xfe, 0x31, 0xf6, 0x76, -+ 0x33, 0xec, 0x4b, 0x12, 0x61, 0xe7, 0x5c, 0xc5, 0x4a, 0x4d, 0x32, 0xfa, 0x00, 0x5a, 0x79, 0x1c, -+ 0x2a, 0x6a, 0x85, 0xf4, 0xf4, 0x0a, 0xb1, 0xe1, 0x74, 0x9b, 0x59, 0x50, 0xbe, 0x86, 0xbe, 0xc8, -+ 0x0c, 0xf7, 0x42, 0x5f, 0xf8, 0xe5, 0x45, 0x55, 0xf6, 0xca, 0xed, 0x89, 0x12, 0xec, 0x7c, 0x05, -+ 0xad, 0x03, 0x12, 0x72, 0xad, 0x78, 0x08, 0x8d, 0x20, 0x65, 0x0c, 0xc7, 0xc2, 0xba, 0x6c, 0x40, -+ 0xb4, 0x06, 0x4b, 0x13, 0x12, 0x11, 0x61, 0xdc, 0xd4, 0x80, 0x43, 0x01, 0x9e, 0xe3, 0x88, 0xb2, -+ 0xa9, 0x0a, 0xd8, 0x1a, 0x2c, 0x15, 0x27, 0x57, 0x03, 0xe8, 0x36, 0xb4, 0x22, 0xff, 0x22, 0x9b, -+ 0x54, 0x49, 0x69, 0x46, 0xfe, 0x85, 0x36, 0x7e, 0x08, 0x8d, 0x13, 0x9f, 0x4c, 0x82, 0x58, 0x98, -+ 0xa8, 0x58, 0x30, 0x57, 0x58, 0x2f, 0x2a, 0xfc, 0xdf, 0x2a, 0xb4, 0xb5, 0x46, 0x6d, 0xf0, 0x1a, -+ 0x2c, 0x05, 0x7e, 0x70, 0x9a, 0xa9, 0x54, 0x00, 0xba, 0x6f, 0x0d, 0xa9, 0x16, 0x93, 0x70, 0x6e, -+ 0xa9, 0x35, 0x6d, 0x0b, 0x80, 0xbf, 0xf2, 0x13, 0x63, 0x5b, 0xed, 0x12, 0xe6, 0x96, 0xe4, 0xd1, -+ 0xe6, 0x7e, 0x0c, 0x1d, 0xbd, 0xee, 0xcc, 0x90, 0xfa, 0x25, 0x43, 0xda, 0x9a, 0x4b, 0x0f, 0x7a, -+ 0x1b, 0xba, 0x29, 0xc7, 0xde, 0x29, 0xc1, 0xcc, 0x67, 0xc1, 0xe9, 0x74, 0xb8, 0xa4, 0xcf, 0xc8, -+ 0x94, 0xe3, 0x3d, 0x8b, 0x43, 0xdb, 0xb0, 0x24, 0xd3, 0x1f, 0x1f, 0x2e, 0xab, 0xe3, 0xf8, 0x4e, -+ 0x51, 0xa4, 0x72, 0x75, 0x53, 0x7d, 0x9f, 0xc4, 0x82, 0x4d, 0x5d, 0xcd, 0x3a, 0xfa, 0x1c, 0x20, -+ 0x47, 0xa2, 0x15, 0xa8, 0x9d, 0xe1, 0xa9, 0xd9, 0x87, 0xf2, 0x57, 0x06, 0xe7, 0xdc, 0x9f, 0xa4, -+ 0x36, 0xea, 0x1a, 0xf8, 0xb2, 0xfa, 0x79, 0xc5, 0x09, 0xa0, 0xbf, 0x33, 0x39, 0x23, 0xb4, 0x30, -+ 0x7c, 0x0d, 0x96, 0x22, 0xff, 0x07, 0xca, 0x6c, 0x24, 0x15, 0xa0, 0xb0, 0x24, 0xa6, 0xcc, 0x8a, -+ 0x50, 0x00, 0xea, 0x41, 0x95, 0x26, 0x2a, 0x5e, 0x2d, 0xb7, 0x4a, 0x93, 0x5c, 0x51, 0xbd, 0xa0, -+ 0xc8, 0xf9, 0x6d, 0x1d, 0x20, 0xd7, 0x82, 0x5c, 0x18, 0x11, 0xea, 0x71, 0xcc, 0x64, 0x09, 0xe2, -+ 0x1d, 0x4f, 0x05, 0xe6, 0x1e, 0xc3, 0x41, 0xca, 0x38, 0x39, 0x97, 0xf3, 0x27, 0xdd, 0xbe, 0xa1, -+ 0xdd, 0x9e, 0xb1, 0xcd, 0xbd, 0x49, 0xe8, 0xa1, 0x1e, 0xb7, 0x23, 0x87, 0xb9, 0x76, 0x14, 0xda, -+ 0x87, 0x1b, 0xb9, 0xcc, 0xb0, 0x20, 0xae, 0x7a, 0x95, 0xb8, 0xd5, 0x4c, 0x5c, 0x98, 0x8b, 0x7a, -+ 0x02, 0xab, 0x84, 0x7a, 0x3f, 0xa6, 0x38, 0x2d, 0x09, 0xaa, 0x5d, 0x25, 0x68, 0x40, 0xe8, 0x5f, -+ 0xab, 0x01, 0xb9, 0x98, 0x03, 0xb8, 0x55, 0xf0, 0x52, 0x6e, 0xf7, 0x82, 0xb0, 0xfa, 0x55, 0xc2, -+ 0xd6, 0x33, 0xab, 0x64, 0x3e, 0xc8, 0x25, 0xfe, 0x15, 0xac, 0x13, 0xea, 0xbd, 0xf2, 0x89, 0x98, -+ 0x15, 0xb7, 0xf4, 0x2b, 0x4e, 0xca, 0x43, 0xb7, 0x2c, 0x4b, 0x3b, 0x19, 0x61, 0x36, 0x2e, 0x39, -+ 0xb9, 0xfc, 0x2b, 0x4e, 0x3e, 0x57, 0x03, 0x72, 0x31, 0x8f, 0x60, 0x40, 0xe8, 0xac, 0x35, 0x8d, -+ 0xab, 0x84, 0xf4, 0x09, 0x2d, 0x5b, 0xb2, 0x03, 0x03, 0x8e, 0x03, 0x41, 0x59, 0x71, 0x11, 0x34, -+ 0xaf, 0x12, 0xb1, 0x62, 0xf8, 0x33, 0x19, 0xce, 0xdf, 0x41, 0x67, 0x2f, 0x1d, 0x63, 0x31, 0x39, -+ 0xce, 0x92, 0xc1, 0x6b, 0xcb, 0x3f, 0xce, 0xef, 0xab, 0xd0, 0xde, 0x1d, 0x33, 0x9a, 0x26, 0xa5, -+ 0x9c, 0xac, 0x37, 0xe9, 0x6c, 0x4e, 0x56, 0x2c, 0x2a, 0x27, 0x6b, 0xe6, 0x4f, 0xa0, 0x13, 0xa9, -+ 0xad, 0x6b, 0xf8, 0x75, 0x1e, 0x1a, 0xcc, 0x6d, 0x6a, 0xb7, 0x1d, 0x15, 0x92, 0xd9, 0x26, 0x40, -+ 0x42, 0x42, 0x6e, 0xc6, 0xe8, 0x74, 0xd4, 0x37, 0x15, 0xa1, 0x4d, 0xd1, 0x6e, 0x2b, 0xc9, 0xb2, -+ 0xf5, 0x03, 0x68, 0x1f, 0xcb, 0x20, 0x99, 0x01, 0xa5, 0x64, 0x94, 0x47, 0xcf, 0x85, 0xe3, 0x7c, -+ 0x13, 0xee, 0x41, 0xf7, 0x54, 0x87, 0xcc, 0x0c, 0xd2, 0x6b, 0xe8, 0x6d, 0xe3, 0x49, 0xee, 0xef, -+ 0x66, 0x31, 0xb2, 0x7a, 0x02, 0x3a, 0xa7, 0x05, 0xd4, 0xe8, 0x10, 0x06, 0x73, 0x2c, 0x0b, 0x72, -+ 0xd0, 0x46, 0x31, 0x07, 0xb5, 0xb7, 0x91, 0x56, 0x54, 0x1c, 0x59, 0xcc, 0x4b, 0xff, 0x52, 0x85, -+ 0xde, 0x7e, 0x2c, 0x30, 0x3b, 0xf1, 0x03, 0xac, 0x2d, 0x46, 0x50, 0x8f, 0xfd, 0x08, 0x1b, 0x99, -+ 0xea, 0x1f, 0xdd, 0x82, 0x26, 0xbb, 0xd0, 0x29, 0xc4, 0xcc, 0x68, 0x83, 0x5d, 0xa8, 0xd4, 0x80, -+ 0xde, 0x00, 0x60, 0x17, 0x5e, 0xe2, 0x07, 0x67, 0xd8, 0xc4, 0xb0, 0xee, 0xb6, 0xd8, 0xc5, 0x81, -+ 0x46, 0xc8, 0xc5, 0xc0, 0x2e, 0x3c, 0xcc, 0x18, 0x65, 0xdc, 0x64, 0xab, 0x26, 0xbb, 0x78, 0xa2, -+ 0x60, 0x33, 0x36, 0x64, 0x34, 0x49, 0x70, 0xa8, 0xb2, 0xb4, 0x1a, 0xfb, 0x58, 0x23, 0xa4, 0x56, -+ 0x61, 0xb5, 0x2e, 0x6b, 0xad, 0x22, 0xd7, 0x2a, 0x72, 0xad, 0x0d, 0x3d, 0x52, 0x14, 0xb5, 0x8a, -+ 0x4c, 0x6b, 0x53, 0x6b, 0x15, 0x05, 0xad, 0x22, 0xd7, 0xda, 0xb2, 0x63, 0x8d, 0x56, 0xe7, 0xbf, -+ 0xab, 0xd0, 0x78, 0x19, 0xa8, 0x49, 0x41, 0xf7, 0xa0, 0x8d, 0xb9, 0xf0, 0x8f, 0x27, 0x84, 0x9f, -+ 0xe2, 0xd0, 0x2c, 0xf3, 0x22, 0x4a, 0xda, 0xc8, 0xa7, 0xb1, 0xc7, 0xe5, 0x09, 0x6e, 0x22, 0xc3, -+ 0xa7, 0xf1, 0xa1, 0x3c, 0xc1, 0x0d, 0x89, 0xe1, 0xe0, 0xdc, 0xae, 0x75, 0x3e, 0x8d, 0x5d, 0x1c, -+ 0x9c, 0x4b, 0xfb, 0x4e, 0x48, 0xac, 0x72, 0xcc, 0x03, 0x1b, 0x95, 0x13, 0x12, 0xcb, 0xfc, 0xf1, -+ 0xa0, 0x48, 0xdc, 0x36, 0x41, 0xb1, 0xc4, 0x6d, 0xe5, 0x99, 0x4c, 0x03, 0x92, 0x6a, 0x82, 0xd2, -+ 0x94, 0x08, 0x49, 0x55, 0x87, 0xf3, 0x84, 0x72, 0x6c, 0x02, 0xa2, 0x01, 0xe9, 0xaf, 0xfa, 0xd1, -+ 0x63, 0x74, 0x34, 0x5a, 0x0a, 0xa3, 0x06, 0xdd, 0x82, 0xe6, 0xc4, 0xe7, 0xc2, 0xf3, 0x83, 0x33, -+ 0x13, 0x8c, 0x86, 0x84, 0x1f, 0x05, 0x67, 0xb2, 0xba, 0x97, 0x05, 0x39, 0x8e, 0x87, 0xa0, 0x08, -+ 0x06, 0x52, 0x55, 0xcb, 0x84, 0x72, 0x12, 0x8f, 0x87, 0x6d, 0x53, 0xb5, 0x68, 0xd0, 0x49, 0xa1, -+ 0x71, 0x14, 0xea, 0xd8, 0xe5, 0x83, 0x2b, 0xb3, 0x83, 0x6d, 0xec, 0x4d, 0xc0, 0x0c, 0x68, 0xd6, -+ 0x8a, 0x3e, 0x11, 0x4c, 0xc4, 0x9a, 0xec, 0x42, 0x27, 0x7c, 0x33, 0xa5, 0x86, 0x58, 0xb7, 0x53, -+ 0xaa, 0x89, 0xce, 0xff, 0x57, 0xa0, 0xf3, 0x2d, 0x16, 0xaf, 0x28, 0x3b, 0xb3, 0xf9, 0x00, 0x88, -+ 0x5d, 0xd6, 0xdc, 0x9c, 0x75, 0xa6, 0x3c, 0x2b, 0x2f, 0x77, 0xb7, 0xc0, 0x87, 0xde, 0x84, 0x9a, -+ 0x08, 0x12, 0xb3, 0x73, 0x4c, 0x6b, 0x68, 0x96, 0x82, 0x2b, 0x29, 0xe8, 0x2d, 0xa8, 0x8b, 0x20, -+ 0xf9, 0xd4, 0xa4, 0x8a, 0x19, 0x0e, 0x45, 0x92, 0x32, 0xd2, 0x30, 0x29, 0xb7, 0x97, 0x26, 0x24, -+ 0xae, 0xa4, 0x48, 0x19, 0x69, 0x98, 0x7c, 0xaa, 0x66, 0x76, 0x8e, 0x43, 0x91, 0x9c, 0x7f, 0xae, -+ 0xc0, 0xfa, 0x6c, 0xf7, 0x61, 0x7a, 0xa5, 0x4f, 0xa0, 0x13, 0xa8, 0xa4, 0x51, 0x4a, 0x8c, 0x83, -+ 0xb9, 0x74, 0xe2, 0xb6, 0x83, 0x42, 0x2e, 0xfd, 0x0c, 0xba, 0xb1, 0x0e, 0x4f, 0x29, 0x3f, 0x9a, -+ 0xe4, 0x50, 0x8c, 0x9c, 0xdb, 0x89, 0x0b, 0x90, 0x13, 0x02, 0xfa, 0x8e, 0x11, 0x81, 0x0f, 0x05, -+ 0xc3, 0x7e, 0xf4, 0x3a, 0xba, 0x60, 0x04, 0x75, 0x55, 0x32, 0xd7, 0x54, 0x93, 0xa7, 0xfe, 0x9d, -+ 0xf7, 0x60, 0xb5, 0xa4, 0xc5, 0xf8, 0xba, 0x02, 0xb5, 0x89, 0x59, 0x3e, 0x5d, 0x57, 0xfe, 0x3a, -+ 0x3e, 0x0c, 0x5c, 0xec, 0x87, 0xaf, 0xcf, 0x1a, 0xa3, 0xa2, 0x96, 0xab, 0xd8, 0x00, 0x54, 0x54, -+ 0x61, 0x4c, 0xb1, 0x56, 0x57, 0x0a, 0x56, 0xbf, 0x80, 0xc1, 0xae, 0xdc, 0x45, 0x87, 0x22, 0x24, -+ 0xf1, 0xeb, 0x68, 0xdb, 0xff, 0x01, 0x56, 0x5f, 0x8a, 0xe9, 0x77, 0x52, 0x18, 0x27, 0x3f, 0xe1, -+ 0xd7, 0xe4, 0x1f, 0xa3, 0xaf, 0xac, 0x7f, 0x8c, 0xbe, 0x92, 0xdb, 0x32, 0xa0, 0x93, 0x34, 0x8a, -+ 0xd5, 0x12, 0xed, 0xba, 0x06, 0x72, 0x76, 0xa0, 0xa3, 0x1b, 0xb9, 0xe7, 0x34, 0x4c, 0x27, 0x78, -+ 0xe1, 0x31, 0x70, 0x17, 0x20, 0xf1, 0x99, 0x1f, 0x61, 0x81, 0x19, 0x57, 0x25, 0x5f, 0xcb, 0x2d, -+ 0x60, 0x9c, 0x7f, 0xad, 0xc2, 0x9a, 0xbe, 0x97, 0x3b, 0xd4, 0xd7, 0x51, 0xd6, 0x85, 0x11, 0x34, -+ 0x4f, 0x29, 0x17, 0x05, 0x81, 0x19, 0x2c, 0x4d, 0x0c, 0x63, 0x2b, 0x4d, 0xfe, 0x96, 0x2e, 0xcb, -+ 0x6a, 0x57, 0x5f, 0x96, 0xcd, 0x5d, 0x87, 0xd5, 0xe7, 0xaf, 0xc3, 0x64, 0x02, 0xb4, 0x4c, 0x44, -+ 0x1f, 0x33, 0x2d, 0xb7, 0x65, 0x30, 0xfb, 0x21, 0xba, 0x0f, 0xfd, 0xb1, 0xb4, 0xd2, 0x3b, 0xa5, -+ 0xf4, 0xcc, 0x4b, 0x7c, 0x71, 0xaa, 0x12, 0x6b, 0xcb, 0xed, 0x2a, 0xf4, 0x1e, 0xa5, 0x67, 0x07, -+ 0xbe, 0x38, 0x45, 0x5f, 0x40, 0xcf, 0xf4, 0x22, 0x91, 0x0a, 0x11, 0x37, 0x15, 0x98, 0xd9, 0x45, -+ 0xc5, 0xe8, 0xb9, 0xdd, 0xb3, 0x02, 0xc4, 0x9d, 0x9b, 0x70, 0xe3, 0x31, 0xe6, 0x82, 0xd1, 0x69, -+ 0x39, 0x30, 0xce, 0x5f, 0x00, 0xec, 0xe7, 0xf9, 0xe7, 0xa3, 0x22, 0x64, 0xb2, 0xd6, 0xca, 0xa6, -+ 0xbe, 0x16, 0xcd, 0x08, 0x6e, 0x81, 0xc7, 0xd9, 0x84, 0x65, 0x97, 0xa6, 0xf2, 0x44, 0x7c, 0xc7, -+ 0xfe, 0x99, 0x71, 0x1d, 0x33, 0x4e, 0x21, 0x5d, 0x43, 0x73, 0xf6, 0xec, 0x3d, 0x4a, 0x2e, 0xce, -+ 0x4c, 0xd1, 0x26, 0xb4, 0xb2, 0x4c, 0x68, 0xb2, 0xca, 0xbc, 0xea, 0x9c, 0xc5, 0x79, 0x09, 0xce, -+ 0x8c, 0xa4, 0xbd, 0x57, 0x8f, 0xc2, 0x90, 0xed, 0x4c, 0xbf, 0xf5, 0xa3, 0x6b, 0x4b, 0xfd, 0x1e, -+ 0x56, 0xb5, 0x54, 0x6d, 0xaf, 0x15, 0xf3, 0x0e, 0x2c, 0x33, 0xeb, 0x5c, 0x25, 0xbf, 0x65, 0x35, -+ 0x4c, 0x86, 0x86, 0xee, 0x48, 0x65, 0x01, 0xc3, 0x91, 0x3d, 0x8c, 0x9b, 0x6e, 0x8e, 0x90, 0x73, -+ 0xf0, 0x8c, 0x70, 0x91, 0x07, 0xcf, 0xce, 0xc1, 0x2a, 0x0c, 0x24, 0xa1, 0xa4, 0xd1, 0xf9, 0x7b, -+ 0x58, 0x7d, 0x11, 0x4f, 0x48, 0x8c, 0x77, 0x0f, 0x8e, 0x9e, 0xe3, 0x2c, 0xd7, 0x20, 0xa8, 0xab, -+ 0x53, 0xb4, 0xa2, 0xa4, 0xab, 0x7f, 0xb9, 0xf9, 0xe2, 0x63, 0x2f, 0x48, 0x52, 0x6e, 0x2e, 0x3d, -+ 0x97, 0xe3, 0xe3, 0xdd, 0x24, 0xe5, 0xf2, 0x64, 0x95, 0x15, 0x2c, 0x8d, 0x27, 0x53, 0xb5, 0x03, -+ 0x9b, 0x6e, 0x23, 0x48, 0xd2, 0x17, 0xf1, 0x64, 0xea, 0xfc, 0xb9, 0xba, 0xe6, 0xc1, 0x38, 0x74, -+ 0xfd, 0x38, 0xa4, 0xd1, 0x63, 0x7c, 0x5e, 0xd0, 0x90, 0x5d, 0x29, 0xd8, 0x4c, 0xf3, 0x73, 0x05, -+ 0x3a, 0x8f, 0xc6, 0x38, 0x16, 0x8f, 0xb1, 0xf0, 0xc9, 0x44, 0x5d, 0x1b, 0x9c, 0x63, 0xc6, 0x09, -+ 0x8d, 0xcd, 0x76, 0xb2, 0x20, 0x7a, 0x13, 0xda, 0x24, 0x26, 0xc2, 0x0b, 0x7d, 0x1c, 0xd1, 0xd8, -+ 0x44, 0x01, 0x24, 0xea, 0xb1, 0xc2, 0xa0, 0xf7, 0xa0, 0xaf, 0x2f, 0xa5, 0xbd, 0x53, 0x3f, 0x0e, -+ 0x27, 0x72, 0x23, 0xeb, 0x4b, 0xba, 0x9e, 0x46, 0xef, 0x19, 0x2c, 0x7a, 0x1f, 0x56, 0xcc, 0x36, -+ 0xcb, 0x39, 0xeb, 0x8a, 0xb3, 0x6f, 0xf0, 0x25, 0xd6, 0x34, 0x49, 0x28, 0x13, 0xdc, 0xe3, 0x38, -+ 0x08, 0x68, 0x94, 0x98, 0x9e, 0xbb, 0x6f, 0xf1, 0x87, 0x1a, 0xed, 0x8c, 0x61, 0xf5, 0xa9, 0xf4, -+ 0xd3, 0x78, 0x92, 0x4f, 0x70, 0x2f, 0xc2, 0x91, 0x77, 0x3c, 0xa1, 0xc1, 0x99, 0x27, 0x93, 0x9f, -+ 0x89, 0xb0, 0xac, 0xea, 0x77, 0x24, 0xf2, 0x90, 0xfc, 0xa4, 0xae, 0x97, 0x24, 0xd7, 0x29, 0x15, -+ 0xc9, 0x24, 0x1d, 0x7b, 0x09, 0xa3, 0xc7, 0xd8, 0xb8, 0xd8, 0x8f, 0x70, 0xb4, 0xa7, 0xf1, 0x07, -+ 0x12, 0xed, 0xfc, 0x4f, 0x05, 0xd6, 0xca, 0x9a, 0x4c, 0x2a, 0xdf, 0x82, 0xb5, 0xb2, 0x2a, 0x53, -+ 0x61, 0xea, 0x2a, 0x65, 0x50, 0x54, 0xa8, 0x6b, 0xcd, 0xcf, 0xa0, 0xab, 0x5e, 0x2a, 0xbc, 0x50, -+ 0x4b, 0x2a, 0x1f, 0x9e, 0xc5, 0x79, 0x71, 0x3b, 0x7e, 0x71, 0x96, 0xbe, 0x80, 0x5b, 0xc6, 0x7d, -+ 0x6f, 0xde, 0x6c, 0xbd, 0x20, 0xd6, 0x0d, 0xc3, 0xf3, 0x19, 0xeb, 0x9f, 0xc1, 0x30, 0x47, 0xed, -+ 0x4c, 0x15, 0xd2, 0xc6, 0xea, 0x23, 0x58, 0x9d, 0x71, 0x56, 0xee, 0x3b, 0xb5, 0xed, 0xeb, 0xee, -+ 0x22, 0x92, 0xf3, 0x10, 0x6e, 0x1e, 0x62, 0xa1, 0xa3, 0xe1, 0x0b, 0xd3, 0xee, 0x6a, 0x61, 0x2b, -+ 0x50, 0x3b, 0xc4, 0x81, 0x72, 0xbe, 0xe6, 0xca, 0x5f, 0xb9, 0x00, 0x8f, 0x38, 0x0e, 0x94, 0x97, -+ 0x35, 0x57, 0xfd, 0x3b, 0xff, 0x55, 0x81, 0x86, 0x49, 0xbe, 0xf2, 0x00, 0x09, 0x19, 0x39, 0xc7, -+ 0xcc, 0x2c, 0x3d, 0x03, 0xa1, 0x77, 0xa1, 0xa7, 0xff, 0x3c, 0x9a, 0x08, 0x42, 0xb3, 0x94, 0xde, -+ 0xd5, 0xd8, 0x17, 0x1a, 0xa9, 0x2e, 0xa1, 0xd5, 0x1d, 0xab, 0xb9, 0xce, 0x30, 0x90, 0xba, 0x49, -+ 0xe6, 0x32, 0x33, 0xa8, 0x14, 0xde, 0x72, 0x0d, 0x24, 0x97, 0xba, 0x95, 0xb7, 0xa4, 0xe4, 0x59, -+ 0x50, 0x2e, 0xf5, 0x88, 0xa6, 0xb1, 0xf0, 0x12, 0x4a, 0x62, 0x61, 0x72, 0x36, 0x28, 0xd4, 0x81, -+ 0xc4, 0x38, 0xff, 0x54, 0x81, 0x65, 0xfd, 0x10, 0x83, 0x7a, 0x50, 0xcd, 0x4e, 0xce, 0x2a, 0x51, -+ 0x55, 0x88, 0xd2, 0xa5, 0x4f, 0x4b, 0xf5, 0x2f, 0xf7, 0xf1, 0x79, 0xa4, 0xf3, 0xbf, 0x31, 0xed, -+ 0x3c, 0x52, 0x89, 0xff, 0x5d, 0xe8, 0xe5, 0x07, 0xb0, 0xa2, 0x6b, 0x13, 0xbb, 0x19, 0x56, 0xb1, -+ 0x5d, 0x6a, 0xa9, 0xf3, 0xb7, 0x00, 0xf9, 0x83, 0x84, 0x0c, 0x79, 0x9a, 0x19, 0x23, 0x7f, 0x25, -+ 0x66, 0x9c, 0x1d, 0xdd, 0xf2, 0x17, 0xdd, 0x87, 0x9e, 0x1f, 0x86, 0x44, 0x0e, 0xf7, 0x27, 0x4f, -+ 0x49, 0x98, 0x6d, 0xd2, 0x32, 0xd6, 0xf9, 0xbf, 0x0a, 0xf4, 0x77, 0x69, 0x32, 0xfd, 0x4b, 0x32, -+ 0xc1, 0x85, 0x0c, 0xa2, 0x8c, 0x34, 0x27, 0xb7, 0xfc, 0xd7, 0x3d, 0xc5, 0x04, 0xeb, 0xad, 0xa5, -+ 0x67, 0xb6, 0x29, 0x11, 0x6a, 0x5b, 0x59, 0x62, 0x76, 0xb7, 0xdb, 0xd5, 0xc4, 0xe7, 0x34, 0x54, -+ 0xad, 0x5f, 0x48, 0x98, 0x97, 0xdd, 0xe4, 0x76, 0xdd, 0x46, 0x48, 0x98, 0x22, 0x19, 0x47, 0x96, -+ 0xd4, 0x63, 0x42, 0xd1, 0x91, 0x65, 0x8d, 0x91, 0x8e, 0xac, 0xc3, 0x32, 0x3d, 0x39, 0xe1, 0x58, -+ 0xa8, 0x9e, 0xa4, 0xe6, 0x1a, 0x28, 0x4b, 0x73, 0xcd, 0x42, 0x9a, 0xbb, 0x01, 0xab, 0xea, 0xd9, -+ 0xea, 0x25, 0xf3, 0x03, 0x12, 0x8f, 0x6d, 0x2a, 0x5e, 0x03, 0x74, 0x28, 0x68, 0x32, 0x83, 0xdd, -+ 0x84, 0x81, 0x39, 0x7f, 0x0e, 0xfe, 0xe6, 0xd0, 0xba, 0x7e, 0x0b, 0x9a, 0x12, 0xf4, 0x18, 0xfe, -+ 0xd1, 0x26, 0x46, 0x43, 0x76, 0xde, 0x87, 0x8e, 0xfe, 0x35, 0x69, 0x20, 0x67, 0xe5, 0x65, 0x56, -+ 0xbe, 0xfd, 0x9f, 0x03, 0x93, 0x6e, 0xcd, 0xf5, 0x10, 0x7a, 0x0a, 0xfd, 0x99, 0xe7, 0x46, 0x64, -+ 0xee, 0x0b, 0x17, 0xbf, 0x42, 0x8e, 0xd6, 0x37, 0xf5, 0xf3, 0xe5, 0xa6, 0x7d, 0xbe, 0xdc, 0x7c, -+ 0x12, 0x25, 0x62, 0x8a, 0x9e, 0x40, 0xaf, 0xfc, 0x30, 0x87, 0x6e, 0xdb, 0xca, 0x66, 0xc1, 0x73, -+ 0xdd, 0xa5, 0x62, 0x9e, 0x42, 0x7f, 0xe6, 0x8d, 0xce, 0xda, 0xb3, 0xf8, 0xe9, 0xee, 0x52, 0x41, -+ 0x0f, 0xa1, 0x5d, 0x78, 0x94, 0x43, 0x43, 0x2d, 0x64, 0xfe, 0x9d, 0xee, 0x52, 0x01, 0xbb, 0xd0, -+ 0x2d, 0xbd, 0x93, 0xa1, 0x91, 0xf1, 0x67, 0xc1, 0xe3, 0xd9, 0xa5, 0x42, 0x76, 0xa0, 0x5d, 0x78, -+ 0xae, 0xb2, 0x56, 0xcc, 0xbf, 0x89, 0x8d, 0x6e, 0x2d, 0xa0, 0x98, 0xe9, 0xdc, 0x83, 0x6e, 0xe9, -+ 0x71, 0xc9, 0x1a, 0xb2, 0xe8, 0x61, 0x6b, 0x74, 0x7b, 0x21, 0xcd, 0x48, 0x7a, 0x0a, 0xfd, 0x99, -+ 0xa7, 0x26, 0x1b, 0xdc, 0xc5, 0x2f, 0x50, 0x97, 0xba, 0xf5, 0x8d, 0x9a, 0xec, 0x42, 0x13, 0x57, -+ 0x98, 0xec, 0xf9, 0x87, 0xa5, 0xd1, 0x9d, 0xc5, 0x44, 0x63, 0xd5, 0x13, 0xe8, 0x95, 0xdf, 0x94, -+ 0xac, 0xb0, 0x85, 0x2f, 0x4d, 0x57, 0xaf, 0x9c, 0xd2, 0xf3, 0x52, 0xbe, 0x72, 0x16, 0xbd, 0x3a, -+ 0x5d, 0x2a, 0xe8, 0x11, 0x80, 0x69, 0xd9, 0x42, 0x12, 0x67, 0x53, 0x36, 0xd7, 0x2a, 0x66, 0x53, -+ 0xb6, 0xa0, 0xbd, 0x7b, 0x08, 0xa0, 0x3b, 0xad, 0x90, 0xa6, 0x02, 0xdd, 0xb4, 0x66, 0xcc, 0xb4, -+ 0x77, 0xa3, 0xe1, 0x3c, 0x61, 0x4e, 0x00, 0x66, 0xec, 0x3a, 0x02, 0xbe, 0x06, 0xc8, 0x3b, 0x38, -+ 0x2b, 0x60, 0xae, 0xa7, 0xbb, 0x22, 0x06, 0x9d, 0x62, 0xbf, 0x86, 0x8c, 0xaf, 0x0b, 0x7a, 0xb8, -+ 0x2b, 0x44, 0xf4, 0x67, 0xaa, 0xe8, 0xf2, 0x62, 0x9b, 0x2d, 0xd3, 0x47, 0x73, 0xd5, 0x33, 0xfa, -+ 0x1e, 0x6e, 0x5f, 0x51, 0x88, 0xa3, 0x8d, 0x85, 0xe2, 0x16, 0xd4, 0xea, 0x0b, 0x44, 0x7f, 0x06, -+ 0x9d, 0x62, 0x35, 0x6e, 0x1d, 0x5c, 0x50, 0xa1, 0x8f, 0x4a, 0x15, 0x39, 0x7a, 0x08, 0xbd, 0x72, -+ 0xad, 0x8d, 0x0a, 0x5b, 0x6e, 0xae, 0x02, 0x1f, 0xad, 0xcc, 0xdc, 0xcc, 0x70, 0xf4, 0x31, 0x40, -+ 0x5e, 0x93, 0xdb, 0x99, 0x99, 0xab, 0xd2, 0x67, 0xb4, 0x7e, 0x0d, 0xbd, 0xc2, 0x91, 0x20, 0x9b, -+ 0xd8, 0x9b, 0x25, 0xe7, 0xf3, 0x83, 0x62, 0x64, 0x8a, 0xb7, 0xd2, 0x89, 0xf0, 0x08, 0x3a, 0xc5, -+ 0xe3, 0xc7, 0x7a, 0xbb, 0xe0, 0x48, 0xba, 0x2a, 0x9f, 0x16, 0x8e, 0x2a, 0xbb, 0x2d, 0xe6, 0x4f, -+ 0xaf, 0xab, 0xf2, 0x69, 0xa9, 0x81, 0xb6, 0x69, 0x6c, 0x51, 0x57, 0x7d, 0xd5, 0x29, 0x53, 0xee, -+ 0x36, 0x6d, 0xf4, 0x17, 0xf6, 0xa0, 0x57, 0x2d, 0xef, 0x62, 0x0b, 0x64, 0xe3, 0xb1, 0xa0, 0x2d, -+ 0xfa, 0x95, 0x74, 0x53, 0x6c, 0x73, 0x0a, 0xe9, 0x66, 0x41, 0xf7, 0x73, 0xa9, 0xa0, 0x3d, 0xe8, -+ 0x3f, 0xb5, 0x15, 0xac, 0xa9, 0xae, 0x8d, 0x39, 0x0b, 0xba, 0x89, 0xd1, 0x68, 0x11, 0xc9, 0xcc, -+ 0xf2, 0x37, 0x30, 0x98, 0xab, 0xac, 0xd1, 0xdd, 0xec, 0xa1, 0x60, 0x61, 0xc9, 0x7d, 0xa9, 0x59, -+ 0xfb, 0xb0, 0x32, 0x5b, 0x58, 0xa3, 0x37, 0xcc, 0xa4, 0x2f, 0x2e, 0xb8, 0x2f, 0x15, 0xf5, 0x05, -+ 0x34, 0x6d, 0x21, 0x87, 0xcc, 0x83, 0xcc, 0x4c, 0x61, 0x77, 0xd9, 0xd0, 0x9d, 0xce, 0xcf, 0xbf, -+ 0xdc, 0xad, 0xfc, 0xe6, 0x97, 0xbb, 0x95, 0xdf, 0xfd, 0x72, 0xb7, 0x72, 0xbc, 0xac, 0xa8, 0x1f, -+ 0xff, 0x21, 0x00, 0x00, 0xff, 0xff, 0x1b, 0x62, 0x55, 0x85, 0x98, 0x25, 0x00, 0x00, - } -diff --git a/protocols/grpc/agent.proto b/protocols/grpc/agent.proto -index 73f6b37..ac329d4 100644 ---- a/protocols/grpc/agent.proto -+++ b/protocols/grpc/agent.proto -@@ -43,6 +43,7 @@ service AgentService { - - // networking - rpc UpdateInterface(UpdateInterfaceRequest) returns (types.Interface); -+ rpc UpdateInterfaceHwAddrByName(UpdateInterfaceHwAddrByNameRequest) returns (types.Interface); - rpc UpdateRoutes(UpdateRoutesRequest) returns (Routes); - rpc ListInterfaces(ListInterfacesRequest) returns(Interfaces); - rpc ListRoutes(ListRoutesRequest) returns (Routes); -@@ -344,6 +345,10 @@ message UpdateInterfaceRequest { - types.Interface interface = 1; - } - -+message UpdateInterfaceHwAddrByNameRequest { -+ types.Interface interface = 1; -+} -+ - message UpdateRoutesRequest { - Routes routes = 1; - bool increment = 2; --- -1.8.3.1 - diff --git a/agent/patches/0018-kata-agent-update-nic-in-guest.patch b/agent/patches/0018-kata-agent-update-nic-in-guest.patch deleted file mode 100644 index c0454dd..0000000 --- a/agent/patches/0018-kata-agent-update-nic-in-guest.patch +++ /dev/null @@ -1,184 +0,0 @@ -From 629aac1078bdeed63214c58d2110fe672d654774 Mon Sep 17 00:00:00 2001 -From: yangfeiyu -Date: Mon, 26 Oct 2020 20:39:05 +0800 -Subject: [PATCH] kata-agent: update nic in guest - -reason: add linkByName and support retry of list ip link, because in some scenarios, -the hardware address cannot be queried immediately after the hot plug. - -Signed-off-by: yangfeiyu ---- - network.go | 91 ++++++++++++++++++++++++++++++++++++++++++------------ - 1 file changed, 72 insertions(+), 19 deletions(-) - -diff --git a/network.go b/network.go -index f6e2c17..d928e2b 100644 ---- a/network.go -+++ b/network.go -@@ -15,13 +15,14 @@ import ( - "strings" - "sync" - "syscall" -+ "time" - -- "golang.org/x/sys/unix" - agentNet "github.com/kata-containers/agent/pkg/net" - "github.com/kata-containers/agent/pkg/types" - pb "github.com/kata-containers/agent/protocols/grpc" - "github.com/sirupsen/logrus" - "github.com/vishvananda/netlink" -+ "golang.org/x/sys/unix" - "google.golang.org/grpc/codes" - grpcStatus "google.golang.org/grpc/status" - ) -@@ -32,6 +33,7 @@ var ( - errNoLink = grpcStatus.Errorf(codes.InvalidArgument, "Need network link") - errNoMAC = grpcStatus.Errorf(codes.InvalidArgument, "Need hardware address") - errNoRoutes = grpcStatus.Errorf(codes.InvalidArgument, "Need network routes") -+ errNoName = grpcStatus.Errorf(codes.InvalidArgument, "Need network name") - guestDNSFile = "/etc/resolv.conf" - kataGuestSandboxDNSFile = "/run/kata-containers/sandbox/resolv.conf" - ) -@@ -46,6 +48,8 @@ const ( - - // Use the below address for ipv6 gateway once ipv6 support is added - // defaultV6RouteIP = "::" -+ -+ maxLinkRetries = 10 - ) - - // Network fully describes a sandbox network with its interfaces, routes and dns -@@ -100,17 +104,36 @@ func linkByHwAddr(netHandle *netlink.Handle, hwAddr string) (netlink.Link, error - return nil, grpcStatus.Errorf(codes.NotFound, "Could not find the link corresponding to HwAddr %q", hwAddr) - } - --func updateLink(netHandle *netlink.Handle, link netlink.Link, iface *types.Interface) error { -+func linkByName(netHandle *netlink.Handle, name string) (netlink.Link, error) { - if netHandle == nil { -- return errNoHandle -+ return nil, errNoHandle - } - -- if link == nil { -- return errNoLink -+ if name == "" { -+ return nil, errNoName - } - -- if iface == nil { -- return errNoIF -+ links, err := netHandle.LinkList() -+ if err != nil { -+ return nil, err -+ } -+ -+ for _, link := range links { -+ if link == nil { -+ continue -+ } -+ -+ if link.Attrs() != nil && link.Attrs().Name == name { -+ return link, nil -+ } -+ } -+ -+ return nil, grpcStatus.Errorf(codes.NotFound, "Could not find the link corresponding to name %s", name) -+} -+ -+func updateLinkIP(netHandle *netlink.Handle, link netlink.Link, iface *types.Interface) error { -+ if len(iface.IPAddresses) == 0 { -+ return nil - } - - // As a first step, clear out any existing addresses associated with the link: -@@ -129,13 +152,6 @@ func updateLink(netHandle *netlink.Handle, link netlink.Link, iface *types.Inter - netlinkAddrStr := fmt.Sprintf("%s/%s", addr.Address, addr.Mask) - netlinkAddr, err := netlink.ParseAddr(netlinkAddrStr) - -- // With ipv6 addresses, there is a brief period during which the address is marked as "tentative" -- // making it unavailable. A process called duplicate address detection(DAD) is performed during this period. -- // Disble DAD so that networking is available once the container is up. The assumption is -- // that it is the reponsibility of the upper stack to make sure the addresses assigned to containers -- // do not conflict. A similar operation is performed by libnetwork: -- // https://github.com/moby/moby/issues/18871 -- - if addr.GetFamily() == types.IPFamily_v6 { - netlinkAddr.Flags = netlinkAddr.Flags | syscall.IFA_F_NODAD - } -@@ -150,14 +166,36 @@ func updateLink(netHandle *netlink.Handle, link netlink.Link, iface *types.Inter - } - } - -+ return nil -+} -+ -+func updateLink(netHandle *netlink.Handle, link netlink.Link, iface *types.Interface) error { -+ if netHandle == nil { -+ return errNoHandle -+ } -+ -+ if link == nil { -+ return errNoLink -+ } -+ -+ if iface == nil { -+ return errNoIF -+ } -+ -+ if err := updateLinkIP(netHandle, link, iface); err != nil { -+ return err -+ } -+ - // set the interface name: - if err := netHandle.LinkSetName(link, iface.Name); err != nil { - return grpcStatus.Errorf(codes.Internal, "Could not set name %s for interface %v: %v", iface.Name, link, err) - } - - // set the interface MTU: -- if err := netHandle.LinkSetMTU(link, int(iface.Mtu)); err != nil { -- return grpcStatus.Errorf(codes.Internal, "Could not set MTU %d for interface %v: %v", iface.Mtu, link, err) -+ if iface.Mtu > 0 { -+ if err := netHandle.LinkSetMTU(link, int(iface.Mtu)); err != nil { -+ return grpcStatus.Errorf(codes.Internal, "Could not set MTU %d for interface %v: %v", iface.Mtu, link, err) -+ } - } - - if iface.RawFlags&unix.IFF_NOARP == uint32(unix.IFF_NOARP) { -@@ -306,8 +344,23 @@ func (s *sandbox) updateInterface(netHandle *netlink.Handle, iface *types.Interf - if iface.HwAddr != "" { - fieldLogger.Info("Getting interface from MAC address") - -- // Find the interface link from its hardware address. -- link, err = linkByHwAddr(netHandle, iface.HwAddr) -+ // In some scenarios, the hardware address cannot be queried immediately -+ // after the hot plug, add a retry here. -+ for retry := 0; retry < maxLinkRetries; retry++ { -+ // Find the interface link from its hardware address. -+ link, err = linkByHwAddr(netHandle, iface.HwAddr) -+ if err != nil { -+ time.Sleep(100 * time.Millisecond) -+ continue -+ } -+ break -+ } -+ if err != nil { -+ return nil, grpcStatus.Errorf(codes.Internal, "updateInterface: %v", err) -+ } -+ } else if iface.Name != "" { -+ fieldLogger.Info("Getting interface from name") -+ link, err = linkByName(netHandle, iface.Name) - if err != nil { - return nil, grpcStatus.Errorf(codes.Internal, "updateInterface: %v", err) - } -@@ -487,7 +540,7 @@ func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Ro - }() - - var ( -- added []*types.Route -+ added []*types.Route - removed []*types.Route - ) - --- -2.23.0 - diff --git a/agent/series.conf b/agent/series.conf deleted file mode 100644 index fc5adee..0000000 --- a/agent/series.conf +++ /dev/null @@ -1,18 +0,0 @@ -0001-agent-add-agent.netlink_recv_buf_size-flag-to-set-ne.patch -0002-network-support-update-routes-incrementally.patch -0003-kata-agent-add-kata-ipvs-command.patch -0004-agent-add-IPVS-test.patch -0005-mount-support-mount-block-device.patch -0006-agent-make-workaround-for-slow-response-in-aarch64.patch -0007-agent-using-pcie-root-port-driver-to-hotplug-device.patch -0008-agent-support-get-root-bus-path-dynamically.patch -0009-storage-add-pkg-storage-for-mount.patch -0010-storage-mount-nfs-and-gpath-in-agent.patch -0011-agent-fix-agent-reap-agent-process-blocked-problem.patch -0012-network-support-set-dns-without-nameserver.patch -0013-agent-support-setting-multi-queues-of-interface.patch -0014-agent-fix-init-hugepages-failed-problem.patch -0015-agent-add-support-of-getting-container-s-network-sta.patch -0016-clock-synchronizes-clock-info-with-proxy.patch -0017-agent-add-support-of-new-sandbox-StratoVirt.patch -0018-kata-agent-update-nic-in-guest.patch diff --git a/shim/apply-patches b/apply-patches similarity index 72% rename from shim/apply-patches rename to apply-patches index 5ac8bba..1d34b02 100755 --- a/shim/apply-patches +++ b/apply-patches @@ -1,20 +1,20 @@ #!/bin/bash +set -e + if [[ -f ./patch_flag ]];then - echo "shim patched!" + echo "patched!" exit 0 fi -tar -zxvf shim-*.tar.gz -cp -fr ./shim-*/* ./ -rm -rf ./shim-* +tar -zxvf kata-containers-2.1.0.tar.gz +cp -rf ./kata-containers-2.1.0/* ./ cat ./series.conf | while read line do if [[ $line == '' || $line =~ ^\s*# ]]; then continue fi echo "====patch $line======" - pwd patch -p1 -F1 -s < ./patches/$line done diff --git a/shim/shim-1.11.1.tar.gz b/kata-containers-2.1.0.tar.gz similarity index 52% rename from shim/shim-1.11.1.tar.gz rename to kata-containers-2.1.0.tar.gz index 8a6d21b..3b2255e 100644 Binary files a/shim/shim-1.11.1.tar.gz and b/kata-containers-2.1.0.tar.gz differ diff --git a/kata-containers.spec b/kata-containers.spec index bb9449a..6d5ac8d 100644 --- a/kata-containers.spec +++ b/kata-containers.spec @@ -1,8 +1,8 @@ #needsrootforbuild %global debug_package %{nil} -%define VERSION v1.11.1 -%define RELEASE 10 +%define VERSION 2.1.0 +%define RELEASE 1 Name: kata-containers Version: %{VERSION} @@ -16,7 +16,7 @@ Source2: kernel.tar.gz BuildRoot: %_topdir/BUILDROOT BuildRequires: automake golang gcc bc glibc-devel glibc-static busybox glib2-devel glib2 ipvsadm conntrack-tools nfs-utils -BuildRequires: patch elfutils-libelf-devel openssl-devel bison flex +BuildRequires: patch elfutils-libelf-devel openssl-devel bison flex rust cargo rust-packaging libgcc dtc-devel %description This is core component of Kata Container, to make it work, you need a isulad/docker engine. @@ -31,14 +31,6 @@ cd %{_builddir}/kata_integration # apply kata_integration patches sh apply-patches -# mv build components into kata_integration dir -pushd %{_builddir}/kata_integration -mv ../kata-containers-%{version}/runtime . -mv ../kata-containers-%{version}/agent . -mv ../kata-containers-%{version}/proxy . -mv ../kata-containers-%{version}/shim . -popd - # build kernel cd %{_builddir}/kernel mv kernel linux @@ -53,11 +45,31 @@ cp %{_builddir}/kata_integration/hack/config-kata-arm64 ./.config cd %{_builddir}/kernel/linux/ make %{?_smp_mflags} +mv %{_builddir}/kata-containers-%{version} %{_builddir}/kata-containers +cd %{_builddir}/kata-containers/ +sh -x apply-patches +cd %{_builddir}/kata-containers/src/runtime +make clean +make + +cd %{_builddir}/kata-containers/src/agent +mkdir vendor && tar -xzf %{_builddir}/kata-containers/vendor.tar.gz -C vendor/ +cp -f ./vendor/version.rs ./src/ +cat > .cargo/config << EOF +[build] +rustflags = ["-Clink-arg=-s","-Clink-arg=-lgcc","-Clink-arg=-lfdt"] + +[source.crates-io] +replace-with = "vendored-sources" + +[source.vendored-sources] +directory = "vendor" +EOF +/usr/bin/env CARGO_HOME=.cargo RUSTC_BOOTSTRAP=1 cargo build cd %{_builddir}/kata_integration mkdir -p -m 750 build -make runtime -make proxy -make shim +cp %{_builddir}/kata-containers/src/agent/target/debug/kata-agent ./build/ +strip ./build/kata-agent make initrd %install @@ -70,26 +82,37 @@ install -p -m 755 -D %{_builddir}/kernel/linux/arch/arm64/boot/Image %{buildroot cd %{_builddir}/kata_integration mkdir -p -m 750 %{buildroot}/usr/bin -install -p -m 750 ./build/kata-runtime ./build/kata-proxy ./build/kata-shim ./build/kata-netmon %{buildroot}/usr/bin/ +install -p -m 750 %{_builddir}/kata-containers/src/runtime/kata-runtime %{buildroot}/usr/bin/ +install -p -m 750 %{_builddir}/kata-containers/src/runtime/kata-netmon %{buildroot}/usr/bin/ +install -p -m 750 %{_builddir}/kata-containers/src/runtime/kata-monitor %{buildroot}/usr/bin/ +install -p -m 750 %{_builddir}/kata-containers/src/runtime/containerd-shim-kata-v2 %{buildroot}/usr/bin/ +install -p -m 640 -D %{_builddir}/kata-containers/src/runtime/cli/config/configuration-qemu.toml %{buildroot}/usr/share/defaults/kata-containers/configuration.toml install -p -m 640 ./build/kata-containers-initrd.img %{buildroot}/var/lib/kata/ mkdir -p -m 750 %{buildroot}/usr/share/defaults/kata-containers/ -install -p -m 640 -D ./runtime/cli/config/configuration-qemu.toml %{buildroot}/usr/share/defaults/kata-containers/configuration.toml +install -p -m 640 -D %{_builddir}/kata-containers/src/runtime/cli/config/configuration-qemu.toml %{buildroot}/usr/share/defaults/kata-containers/configuration.toml +strip %{buildroot}/usr/bin/kata* +strip %{buildroot}/usr/bin/containerd-shim-kata-v2 %clean %files /usr/bin/kata-runtime -/usr/bin/kata-proxy -/usr/bin/kata-shim /usr/bin/kata-netmon +/usr/bin/kata-monitor +/usr/bin/containerd-shim-kata-v2 /var/lib/kata/kernel /var/lib/kata/kata-containers-initrd.img %config(noreplace) /usr/share/defaults/kata-containers/configuration.toml %doc - %changelog +* Wed Aug 18 2021 yangfeiyu - 2.1.0-1 +- Type:enhancement +- ID:NA +- SUG:NA +- DESC:upgrade kata-containers + * Fri Feb 19 2021 xinghe - 1.11.1-10 - Type:CVE - ID:NA diff --git a/proxy/apply-patches b/proxy/apply-patches deleted file mode 100755 index 99f657b..0000000 --- a/proxy/apply-patches +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -if [[ -f ./patch_flag ]];then - echo "proxy patched!" - exit 0 -fi - -tar -zxvf proxy-1.11.1.tar.gz -cp -fr ./proxy-1.11.1/* ./ -rm -rf ./proxy-1.11.1 -cat ./series.conf | while read line -do - if [[ $line == '' || $line =~ ^\s*# ]]; then - continue - fi - echo "====patch $line======" - pwd - patch -p1 -F1 -s < ./patches/$line -done -touch ./patch_flag diff --git a/proxy/patches/0001-clock-synchronizes-clock-info-to-agent.patch b/proxy/patches/0001-clock-synchronizes-clock-info-to-agent.patch deleted file mode 100644 index e294d3a..0000000 --- a/proxy/patches/0001-clock-synchronizes-clock-info-to-agent.patch +++ /dev/null @@ -1,403 +0,0 @@ -From a260bbe394f91fa05b163315390ed133de5c5494 Mon Sep 17 00:00:00 2001 -From: holyfei -Date: Wed, 19 Aug 2020 17:43:24 +0800 -Subject: [PATCH] clock: synchronizes clock info to agent - -reason: virtual machine's clock may be incorrect, proxy synchronizes -clock info to help virtual machine adjust clock time - -Signed-off-by: yangfeiyu ---- - proxy.go | 7 ++ - proxy_test.go | 29 +++++ - sync_clock_client.go | 107 +++++++++++++++++ - sync_clock_client_test.go | 132 +++++++++++++++++++++ - .../kata-containers/agent/pkg/clock/clock_util.go | 44 +++++++ - 5 files changed, 319 insertions(+) - create mode 100644 sync_clock_client.go - create mode 100644 sync_clock_client_test.go - create mode 100644 vendor/github.com/kata-containers/agent/pkg/clock/clock_util.go - -diff --git a/proxy.go b/proxy.go -index ab062a5..9dfcb3c 100644 ---- a/proxy.go -+++ b/proxy.go -@@ -105,6 +105,13 @@ func serve(servConn io.ReadWriteCloser, proto, addr string, results chan error) - // Start the heartbeat in a separate go routine - go heartBeat(session) - -+ // start the sync clock in a separate go routine -+ syncClockStream, err := session.Open() -+ if err != nil { -+ return nil, nil, err -+ } -+ go SyncClock(syncClockStream) -+ - // serving connection - l, err := net.Listen(proto, addr) - if err != nil { -diff --git a/proxy_test.go b/proxy_test.go -index 923b138..94fa523 100644 ---- a/proxy_test.go -+++ b/proxy_test.go -@@ -10,6 +10,7 @@ package main - import ( - "bytes" - "crypto/md5" -+ "encoding/json" - "fmt" - "io" - "io/ioutil" -@@ -23,6 +24,7 @@ import ( - "testing" - - "github.com/hashicorp/yamux" -+ "github.com/kata-containers/agent/pkg/clock" - "github.com/stretchr/testify/assert" - ) - -@@ -121,6 +123,13 @@ func server(listener net.Listener, closeCh chan bool) error { - session.Close() - }() - -+ // accept the sync clock stream first -+ if syncClockStream, err := session.Accept(); err != nil { -+ return err -+ } else { -+ go serverSyncClock(syncClockStream) -+ } -+ - for { - stream, err := session.Accept() - if err != nil { -@@ -133,6 +142,26 @@ func server(listener net.Listener, closeCh chan bool) error { - } - } - -+func serverSyncClock(stream net.Conn) { -+ for { -+ buf, byteNum, err := readConnData(stream) -+ if err != nil { -+ continue -+ } -+ var clockInfo clock.TimeValue -+ if err := json.Unmarshal(buf[:byteNum], &clockInfo); err != nil { -+ continue -+ } -+ nowTime := clock.GetCurrentTimeNs() -+ clockInfo.ClientArriveTime = nowTime -+ clockInfo.ServerSendTime = nowTime -+ b, _ := json.Marshal(clockInfo) -+ if err := clock.WriteConnData(stream, b); err != nil { -+ continue -+ } -+ } -+} -+ - func TestUnixAddrParsing(T *testing.T) { - buf := "unix://foo/bar" - addr, err := unixAddr(buf) -diff --git a/sync_clock_client.go b/sync_clock_client.go -new file mode 100644 -index 0000000..9bf3e91 ---- /dev/null -+++ b/sync_clock_client.go -@@ -0,0 +1,107 @@ -+// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved. -+// SPDX-License-Identifier: Apache-2.0 -+// Description: sync clock client related function -+// Author: xueshaojia x00464843 -+// Create: 2018-11-10 -+ -+package main -+ -+import ( -+ "encoding/json" -+ "fmt" -+ "net" -+ "time" -+ -+ "github.com/kata-containers/agent/pkg/clock" -+) -+ -+const ( -+ allowTimeDiff = 10 * 1000 * 1000 // allow 10ms's difference -+ syncClockInterval = 60 * time.Second // sync clock with agent every 60 seconds -+ rpcTimeout = 10 * time.Second // timeout for proxy's reading data -+) -+ -+// readConnData reads data from stream -+func readConnData(stream net.Conn) (buf []byte, byteNum int, err error) { -+ // set read deadline to avoid case as following: -+ // proxy and agent are both reading and then syncClock will never work -+ stream.SetReadDeadline(time.Now().Add(rpcTimeout)) -+ buf = make([]byte, clock.MaxSyncClockByteNum) -+ byteNum, err = stream.Read(buf) -+ return buf, byteNum, err -+} -+ -+// getGuestClock syncs guest clock info -+// sends ClientSendTime -+// waits for ClientArriveTime and ServerSendTime -+func getGuestClock(stream net.Conn, clockInfo *clock.TimeValue) error { -+ clockInfo.Delta = 0 -+ b, err := json.Marshal(clockInfo) -+ if err != nil { -+ return err -+ } -+ if err = clock.WriteConnData(stream, b); err != nil { -+ return err -+ } -+ -+ buf, byteNum, err := readConnData(stream) -+ if err != nil { -+ return err -+ } -+ -+ if err = json.Unmarshal(buf[:byteNum], clockInfo); err != nil { -+ return fmt.Errorf("sync clock, parse guest clocktime error:%v", err) -+ } -+ return nil -+} -+ -+// adjustGuestClock tells server to ajust local clock with Delta -+func adjustGuestClock(stream net.Conn, clockInfo *clock.TimeValue) error { -+ b, err := json.Marshal(clockInfo) -+ if err != nil { -+ return err -+ } -+ logger().Debugf("sync clock, send:%s", string(b)) -+ return clock.WriteConnData(stream, b) -+} -+ -+// syncClock performs all the steps -+// 1 get client send time[host] -+// 2 request for client arrive time and server send time[guest os] -+// 3 get server arrive time[host] -+// 4 calculate clock diff -+// 5 request to adjust guest clock -+func syncClock(stream net.Conn) error { -+ var clockInfo clock.TimeValue -+ if clockInfo.ClientSendTime = clock.GetCurrentTimeNs(); clockInfo.ClientSendTime <= 0 { -+ return fmt.Errorf("sync clock, get client sendtime error") -+ } -+ err := getGuestClock(stream, &clockInfo) -+ if err != nil { -+ return fmt.Errorf("sync clock, get guest clocktime error:%v", err) -+ } -+ if clockInfo.ServerArriveTime = clock.GetCurrentTimeNs(); clockInfo.ServerArriveTime <= 0 { -+ return fmt.Errorf("sync clock, get client recvtime error") -+ } -+ if clockInfo.ClientSendTime <= 0 || clockInfo.ClientArriveTime <= 0 || clockInfo.ServerSendTime <= 0 { -+ return fmt.Errorf("sync clock, some fields of NTP message error, raw message:%v", clockInfo) -+ } -+ -+ delta := ((clockInfo.ClientSendTime - clockInfo.ClientArriveTime) + (clockInfo.ServerArriveTime - clockInfo.ServerSendTime)) / 2 -+ if delta < -allowTimeDiff || delta > allowTimeDiff { -+ clockInfo.Delta = delta -+ if err := adjustGuestClock(stream, &clockInfo); err != nil { -+ return fmt.Errorf("sync clock, failed to adjust guest clock : %v", err) -+ } -+ } -+ return nil -+} -+ -+func SyncClock(stream net.Conn) { -+ for { -+ if err := syncClock(stream); err != nil { -+ logger().WithError(err).Error("sync clock failed") -+ } -+ time.Sleep(syncClockInterval) -+ } -+} -diff --git a/sync_clock_client_test.go b/sync_clock_client_test.go -new file mode 100644 -index 0000000..b0b1c85 ---- /dev/null -+++ b/sync_clock_client_test.go -@@ -0,0 +1,132 @@ -+// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved. -+// SPDX-License-Identifier: Apache-2.0 -+// Description: sync clock client related test -+// Author: xueshaojia x00464843 -+// Create: 2018-11-10 -+ -+package main -+ -+import ( -+ "encoding/json" -+ "fmt" -+ "math/rand" -+ "net" -+ "os" -+ "testing" -+ -+ "github.com/hashicorp/yamux" -+ "github.com/kata-containers/agent/pkg/clock" -+) -+ -+func TestReadWriteData(t *testing.T) { -+ var clockInfo clock.TimeValue -+ clockInfo.ClientSendTime = clock.GetCurrentTimeNs() -+ b, err := json.Marshal(clockInfo) -+ if err != nil { -+ t.Fatalf("Marshal clock info fail, err:%v", err) -+ } -+ -+ err = clock.WriteConnData(clientStream, b) -+ fmt.Printf("client send: %s\n", string(b)) -+ if err != nil { -+ t.Fatalf("send clock info fail, err:%v", err) -+ } -+ _, _, err = readConnData(clientStream) -+ if err != nil { -+ t.Fatalf("recv clock info fail, err:%v", err) -+ } -+} -+ -+func SetUpServer(sock string, readyChan chan int) error { -+ var err error -+ listener, err = net.Listen("unix", sock) -+ if err != nil { -+ return err -+ } -+ readyChan <- 1 -+ conn, err := listener.Accept() -+ if err != nil { -+ return err -+ } -+ session, err := yamux.Server(conn, nil) -+ if err != nil { -+ return err -+ } -+ serverSession = session -+ stream, err := session.Accept() -+ if err != nil { -+ return err -+ } -+ for { -+ var clockInfo clock.TimeValue -+ var byteNum int -+ var err error -+ buf := make([]byte, 400) -+ if byteNum, err = stream.Read(buf); err != nil { -+ break -+ } -+ -+ if err = json.Unmarshal(buf[:byteNum], &clockInfo); err != nil { -+ break -+ } -+ if clockInfo.Delta == 0 { -+ nowTime := clock.GetCurrentTimeNs() -+ clockInfo.ClientArriveTime = nowTime -+ clockInfo.ClientArriveTime = nowTime -+ b, _ := json.Marshal(&clockInfo) -+ stream.Write(b) -+ } -+ } -+ return nil -+} -+ -+func SetUpClient(sock string) error { -+ conn, err := net.Dial("unix", sock) -+ if err != nil { -+ return err -+ } -+ session, err := yamux.Client(conn, nil) -+ if err != nil { -+ conn.Close() -+ return err -+ } -+ clientSession = session -+ stream, err := session.Open() -+ if err != nil { -+ clientSession.Close() -+ return err -+ } -+ clientStream = stream -+ return nil -+} -+ -+func TearDown() { -+ listener.Close() -+ serverSession.Close() -+ clientSession.Close() -+} -+ -+func GenSocket() string { -+ randSeed := clock.GetCurrentTimeNs() -+ rand.Seed(randSeed) -+ return fmt.Sprintf("/tmp/%d.sock", rand.Uint32()) -+} -+ -+var listener net.Listener -+var clientStream net.Conn -+var serverSession *yamux.Session -+var clientSession *yamux.Session -+ -+func TestMain(m *testing.M) { -+ waitConn := make(chan int) -+ testSock := GenSocket() -+ go SetUpServer(testSock, waitConn) -+ <-waitConn -+ if err := SetUpClient(testSock); err != nil { -+ listener.Close() -+ serverSession.Close() -+ os.Exit(1) -+ } -+ m.Run() -+ TearDown() -+} -diff --git a/vendor/github.com/kata-containers/agent/pkg/clock/clock_util.go b/vendor/github.com/kata-containers/agent/pkg/clock/clock_util.go -new file mode 100644 -index 0000000..03244fd ---- /dev/null -+++ b/vendor/github.com/kata-containers/agent/pkg/clock/clock_util.go -@@ -0,0 +1,44 @@ -+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. -+// SPDX-License-Identifier: Apache-2.0 -+// Description: common functions -+// Author: jiangpeifei -+// Create: 2019-05-28 -+ -+package clock -+ -+import ( -+ "net" -+ "syscall" -+) -+ -+type TimeValue struct { -+ ClientSendTime int64 `json:"client_send_time"` -+ ClientArriveTime int64 `json:"client_arrive_time"` -+ ServerSendTime int64 `json:"server_send_time"` -+ ServerArriveTime int64 `json:"server_arrive_time"` -+ Delta int64 `json:"delta"` -+} -+ -+const MaxSyncClockByteNum = 400 //sync clock byte num, max=400 -+ -+// getCurrentTimeNs returns UTC time in Ns -+func GetCurrentTimeNs() int64 { -+ var tv syscall.Timeval -+ if err := syscall.Gettimeofday(&tv); err != nil { -+ return -1 -+ } -+ return tv.Sec*1000000000 + tv.Usec*1000 -+} -+ -+// readConnData reads data from stream -+func ReadConnData(stream net.Conn) (buf []byte, byteNum int, err error) { -+ buf = make([]byte, MaxSyncClockByteNum) -+ byteNum, err = stream.Read(buf) -+ return buf, byteNum, err -+} -+ -+// writeConnData writes data to stream -+func WriteConnData(stream net.Conn, buf []byte) error { -+ _, err := stream.Write(buf) -+ return err -+} --- -2.14.3 (Apple Git-98) - diff --git a/proxy/proxy-1.11.1.tar.gz b/proxy/proxy-1.11.1.tar.gz deleted file mode 100644 index 45699f9..0000000 Binary files a/proxy/proxy-1.11.1.tar.gz and /dev/null differ diff --git a/proxy/series.conf b/proxy/series.conf deleted file mode 100644 index 1f29a6e..0000000 --- a/proxy/series.conf +++ /dev/null @@ -1 +0,0 @@ -0001-clock-synchronizes-clock-info-to-agent.patch diff --git a/runtime/apply-patches b/runtime/apply-patches deleted file mode 100755 index 2a86e4e..0000000 --- a/runtime/apply-patches +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -set -e - -if [[ -f ./patch_flag ]];then - echo "runtime patched!" - exit 0 -fi - -tar -zxvf runtime-1.11.1.tar.gz -cp -fr ./runtime-1.11.1/* ./ -rm -rf ./runtime-1.11.1 -cat ./series.conf | while read line -do - if [[ $line == '' || $line =~ ^\s*# ]]; then - continue - fi - echo "====patch $line======" - patch -p1 -F1 -s < ./patches/$line -done - -touch ./patch_flag diff --git a/runtime/kata-runtime.spec b/runtime/kata-runtime.spec deleted file mode 100644 index acef8ce..0000000 --- a/runtime/kata-runtime.spec +++ /dev/null @@ -1,165 +0,0 @@ -%define debug_package %{nil} - -%define VERSION 1.11.1 -%define RELEASE 11 - -Name: kata-runtime -Version: %{VERSION} -Release: %{RELEASE} -Summary: Kata Runtime -License: Apache 2.0 -URL: https://github.com/kata-containers/runtime -Source0: https://github.com/kata-containers/runtime/archive/%{version}.tar.gz#/%{name}-v%{version}.tar.gz - -BuildRoot: %_topdir/BUILDROOT -BuildRequires: automake golang gcc - -%description -Kata-runtime is core component of Kata Container. - -%prep -%setup -q -c -a 0 -n %{name}-%{version} - -%build -cd %{_builddir}/%{name}-%{version} - -set -e -# apply patches read from series.conf -sh apply-patches - -# create tmp GOPATH dir to build kata-runtime -rm -rf /tmp/kata-build/ -mkdir -p /tmp/kata-build/ -GOPATH=/tmp/kata-build/ -kata_base=$GOPATH/src/github.com/kata-containers -mkdir -p $kata_base - -# get current kata-runtime absolute path -kata_runtime_path=$(readlink -f .) -ln -s $kata_runtime_path $kata_base/runtime - -# export GOPATH env -export GOPATH=$(readlink -f $GOPATH) -cd ${kata_base}/runtime && make clean && make -rm -rf $GOPATH - -# make kata-runtime default configuration -kata_config_path=$kata_runtime_path/cli/config/configuration-qemu.toml -ARCH=`arch` - -# arch related config options -if [ "$ARCH" == "aarch64" ];then - sed -i 's/^machine_type.*$/machine_type = \"virt\"/' $kata_config_path - sed -i 's/^block_device_driver.*$/block_device_driver = \"virtio-scsi\"/' $kata_config_path - sed -i 's/^kernel_params.*$/kernel_params = \"pcie_ports=native pci=pcie_bus_perf agent.netlink_recv_buf_size=2MB\"/' $kata_config_path - sed -i 's/^hypervisor_params.*$/hypervisor_params = \"kvm-pit.lost_tick_policy=discard pcie-root-port.fast-plug=1 pcie-root-port.x-speed=16 pcie-root-port.x-width=32 pcie-root-port.fast-unplug=1\"/' $kata_config_path - sed -i 's/^#pcie_root_port.*$/pcie_root_port = 25/' $kata_config_path -else - sed -i 's#block_device_driver = \"virtio-scsi\"#block_device_driver = \"virtio-blk\"#' $kata_config_path - sed -i 's/^#hotplug_vfio_on_root_bus/hotplug_vfio_on_root_bus/' $kata_config_path -fi - -# debug config -sed -i 's/^#enable_debug.*$/enable_debug = true/' $kata_config_path - -# other config -sed -i 's#"/usr/bin/qemu.*"$#"/usr/bin/qemu-kvm"#' $kata_config_path -sed -i 's#/usr/share/kata-containers/vmlinuz\.container#/var/lib/kata/kernel#' $kata_config_path -sed -i 's#/usr/share/kata-containers/kata-containers-initrd\.img#/var/lib/kata/kata-containers-initrd\.img#' $kata_config_path -sed -i 's/^image/#image/' $kata_config_path -sed -i 's/^default_memory.*$/default_memory = 1024/' $kata_config_path -sed -i 's/^#enable_blk_mount/enable_blk_mount/' $kata_config_path -sed -i 's/^#block_device_cache_direct.*$/block_device_cache_direct = true/' $kata_config_path -sed -i 's/^#block_device_cache_set.*$/block_device_cache_set = true/' $kata_config_path -sed -i 's#/usr/libexec/kata-containers/kata-proxy#/usr/bin/kata-proxy#' $kata_config_path -sed -i 's#/usr/libexec/kata-containers/kata-shim#/usr/bin/kata-shim#' $kata_config_path -sed -i 's#/usr/libexec/kata-containers/kata-netmon#/usr/bin/kata-netmon#' $kata_config_path -sed -i 's/^#disable_new_netns.*$/disable_new_netns = true/' $kata_config_path -sed -i 's/^#disable_vhost_net.*$/disable_vhost_net = true/' $kata_config_path -sed -i 's/^internetworking_model.*$/internetworking_model=\"none\"/' $kata_config_path -sed -i 's/^enable_compat_old_cni.*$/#enable_compat_old_cni = true/' $kata_config_path -sed -i 's/^sandbox_cgroup_only.*$/sandbox_cgroup_only = true/' $kata_config_path - -set +e - -%install -cd %{_builddir}/%{name}-%{version} -mkdir -p -m 750 %{buildroot}/usr/bin -install -p -m 750 ./kata-runtime %{buildroot}/usr/bin -install -p -m 750 ./kata-netmon %{buildroot}/usr/bin -mkdir -p -m 750 %{buildroot}/usr/share/defaults/kata-containers -install -p -m 640 ./cli/config/configuration-qemu.toml %{buildroot}/usr/share/defaults/kata-containers/configuration.toml - -%clean - -%files -/usr/bin/kata-runtime -/usr/bin/kata-netmon -/usr/share/defaults/kata-containers/configuration.toml - -%changelog -* Tue Nov 17 2020 yangfeiyu - 1.11.1-11 -- Type:bugfix -- ID:NA -- SUG:upgrade -- DESC:fix cpu resource limited problem when sandox_cgroup_with_emulator config is enabled - -* Fri Oct 9 2020 yangfeiyu - 1.11.1-10 -- Type:feature -- ID:NA -- SUG:restart -- DESC:support using CNI plugin to insert mutiple network interfaces at the same time - -* Mon Sep 28 2020 yangfeiyu - 1.11.1-9 -- Type:bugfix -- ID:NA -- SUG:restart -- DESC:retry inserting of CNI interface when netmon is enable - -* Sun Sep 27 2020 LiangZhang - 1.11.1-8 -- Type:bugfix -- ID:NA -- SUG:NA -- DESC:fix cmd params of direct use stratovirt binary - -* Thu Sep 24 2020 LiangZhang - 1.11.1-7 -- Type:bugfix -- ID:NA -- SUG:NA -- DESC:fix invalid cmdline when start sandbox stratovirt - -* Mon Sep 21 2020 yangfeiyu - 1.11.1-6 -- Type:bugfix -- ID:NA -- SUG:NA -- DESC:fix sandboxRuntimeRootPath left problem - -* Mon Sep 21 2020 yangfeiyu - 1.11.1-5 -- Type:enhancement -- ID:NA -- SUG:NA -- DESC:add support for host cgroups with emulator - -* Mon Sep 21 2020 LiangZhang - 1.11.1-4 -- Type:enhancement -- ID:NA -- SUG:NA -- DESC:add support of new sandbox StratoVirt - -* Sat Sep 19 2020 yangfeiyu - 1.11.1-3 -- Type:bugfix -- ID:NA -- SUG:NA -- DESC:fix del-iface doesn't delete the tap interface in the host problem - -* Sat Sep 5 2020 yangfeiyu - 1.11.1-2 -- Type:enhancement -- ID:NA -- SUG:NA -- DESC:use URL format for Source0 - -* Wed Aug 26 2020 yangfeiyu - 1.11.1-1 -- Type:enhancement -- ID:NA -- SUG:NA -- DESC:modify kata-runtime spec file to build seperately diff --git a/runtime/patches/0001-qmp-fix-kata-runtime-hungs-when-qemu-process-is-D-T-.patch b/runtime/patches/0001-qmp-fix-kata-runtime-hungs-when-qemu-process-is-D-T-.patch deleted file mode 100644 index c2c605c..0000000 --- a/runtime/patches/0001-qmp-fix-kata-runtime-hungs-when-qemu-process-is-D-T-.patch +++ /dev/null @@ -1,44 +0,0 @@ -From 73fe7242d18a10a86bc216ec5e33a10a8751f85f Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Fri, 24 Jul 2020 22:22:00 +0800 -Subject: [PATCH 01/50] qmp: fix kata-runtime hungs when qemu process is D/T - state - -reason: When set qemu's status to T and execute add-iface command -the command will hung all the time.It hungs while wait for qemu -to return version messages. - -When qmp starts, set the timeout time to 15 seconds.When times -out, return qmp starts failed.We choose 15 seconds to keep consistent -with the agent client start. - -Signed-off-by: jiangpengfei ---- - vendor/github.com/intel/govmm/qemu/checklist | 1 + - vendor/github.com/intel/govmm/qemu/qmp.go | 2 ++ - 2 files changed, 3 insertions(+) - create mode 100644 vendor/github.com/intel/govmm/qemu/checklist - -diff --git a/vendor/github.com/intel/govmm/qemu/checklist b/vendor/github.com/intel/govmm/qemu/checklist -new file mode 100644 -index 00000000..b32f1855 ---- /dev/null -+++ b/vendor/github.com/intel/govmm/qemu/checklist -@@ -0,0 +1 @@ -+add timeout when qmp start to avoid qmp client hungs all the time -diff --git a/vendor/github.com/intel/govmm/qemu/qmp.go b/vendor/github.com/intel/govmm/qemu/qmp.go -index bf9a77dd..a64039de 100644 ---- a/vendor/github.com/intel/govmm/qemu/qmp.go -+++ b/vendor/github.com/intel/govmm/qemu/qmp.go -@@ -722,6 +722,8 @@ func QMPStart(ctx context.Context, socket string, cfg QMPConfig, disconnectedCh - if q.version == nil { - return nil, nil, fmt.Errorf("failed to find QMP version information") - } -+ case <-time.After(15 * time.Second): -+ return nil, nil, fmt.Errorf("qmp start time out") - } - - return q, q.version, nil --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0002-kata-runtime-fix-kata-runtime-skip-read-lines-in-pro.patch b/runtime/patches/0002-kata-runtime-fix-kata-runtime-skip-read-lines-in-pro.patch deleted file mode 100644 index 99f3137..0000000 --- a/runtime/patches/0002-kata-runtime-fix-kata-runtime-skip-read-lines-in-pro.patch +++ /dev/null @@ -1,117 +0,0 @@ -From 1efb88fbf554f3977a1a8aa1c79bc70cc1b66953 Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Sat, 25 Jul 2020 09:22:08 +0800 -Subject: [PATCH 02/50] kata-runtime: fix kata-runtime skip read lines in - /proc/mounts file problem - -reason: Since /proc/mounts is a virtual file which is changed dynamically by kernel, -if we use file pointer to read content in this file line by line, we may miss read -some lines. So we retry read /proc/mounts file again to fix this problem. - -Signed-off-by: jiangpengfei ---- - virtcontainers/utils/utils_linux.go | 58 +++++++++++++++++++++++-------------- - 1 file changed, 36 insertions(+), 22 deletions(-) - -diff --git a/virtcontainers/utils/utils_linux.go b/virtcontainers/utils/utils_linux.go -index ad870d63..6cef4cfb 100644 ---- a/virtcontainers/utils/utils_linux.go -+++ b/virtcontainers/utils/utils_linux.go -@@ -7,15 +7,18 @@ package utils - - import ( - "bufio" -+ "bytes" - "crypto/rand" - "fmt" - "io" -+ "io/ioutil" - "math/big" - "os" - "strings" - "syscall" - "unsafe" - -+ "github.com/sirupsen/logrus" - "golang.org/x/sys/unix" - ) - -@@ -93,6 +96,7 @@ const ( - procMountsFile = "/proc/mounts" - - fieldsPerLine = 6 -+ maxRetryTimes = 5 - ) - - const ( -@@ -109,35 +113,45 @@ func GetDevicePathAndFsType(mountPoint string) (devicePath, fsType string, err e - return - } - -- var file *os.File -+ var retry int = 0 - -- file, err = os.Open(procMountsFile) -- if err != nil { -- return -- } -- -- defer file.Close() -+ for retry <= maxRetryTimes { -+ var content []byte - -- reader := bufio.NewReader(file) -- for { -- var line string -- -- line, err = reader.ReadString('\n') -- if err == io.EOF { -- err = fmt.Errorf("Mount %s not found", mountPoint) -+ content, err = ioutil.ReadFile(procMountsFile) -+ if err != nil { - return - } - -- fields := strings.Fields(line) -- if len(fields) != fieldsPerLine { -- err = fmt.Errorf("Incorrect no of fields (expected %d, got %d)) :%s", fieldsPerLine, len(fields), line) -- return -+ bytesReader := bytes.NewReader(content) -+ reader := bufio.NewReader(bytesReader) -+ -+ for { -+ var line string -+ -+ line, err = reader.ReadString('\n') -+ if err == io.EOF { -+ err = fmt.Errorf("Mount %s not found", mountPoint) -+ break -+ } -+ -+ fields := strings.Fields(line) -+ if len(fields) != fieldsPerLine { -+ err = fmt.Errorf("Incorrect no of fields (expected %d, got %d)) :%s", fieldsPerLine, len(fields), line) -+ return -+ } -+ -+ if mountPoint == fields[procPathIndex] { -+ devicePath = fields[procDeviceIndex] -+ fsType = fields[procTypeIndex] -+ return -+ } - } - -- if mountPoint == fields[procPathIndex] { -- devicePath = fields[procDeviceIndex] -- fsType = fields[procTypeIndex] -- return -+ retry = retry + 1 -+ if retry <= maxRetryTimes { -+ logrus.Warnf("can not find %s in %s, retry %d times again......", mountPoint, procMountsFile, retry) - } - } -+ return "", "", fmt.Errorf("retry %d times fail to get devicePath adn fs type", maxRetryTimes) - } --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0003-kata-runtime-fix-kata-proxy-process-left-problem.patch b/runtime/patches/0003-kata-runtime-fix-kata-proxy-process-left-problem.patch deleted file mode 100644 index 9556441..0000000 --- a/runtime/patches/0003-kata-runtime-fix-kata-proxy-process-left-problem.patch +++ /dev/null @@ -1,55 +0,0 @@ -From cf595941e1d105af23bc006bf1998ac072733d0a Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Sat, 25 Jul 2020 10:03:35 +0800 -Subject: [PATCH 03/50] kata-runtime: fix kata-proxy process left problem - -reason: stopSandbox function will send the DestroySandboxRequest -to kata-agent in the VM and then kill the kata-proxy process in -the host. However, if k.sendReq(DestroySandboxRequest) get error, -stopSandbox will return immediately not execute the following kill -kata-proxy process statement, which cause the kata-process left. - -Signed-off-by: jiangpengfei ---- - virtcontainers/kata_agent.go | 18 +++++++++++------- - 1 file changed, 11 insertions(+), 7 deletions(-) - -diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go -index a0cf190e..be5e96aa 100644 ---- a/virtcontainers/kata_agent.go -+++ b/virtcontainers/kata_agent.go -@@ -976,6 +976,17 @@ func (k *kataAgent) stopSandbox(sandbox *Sandbox) error { - return errorMissingProxy - } - -+ // since stopSandbox will destroy the sandbox in the VM, and we don't need -+ // kata-proxy process to communicate with kata-agent again, so we should -+ // make sure kata-proxy can be killed cleanly, even when k.sendReq(DestroySandboxRequest) -+ // return error -+ defer func() { -+ _ = k.proxy.stop(k.state.ProxyPid) -+ // clean up agent state -+ k.state.ProxyPid = -1 -+ k.state.URL = "" -+ }() -+ - req := &grpc.DestroySandboxRequest{} - - if _, err := k.sendReq(req); err != nil { -@@ -989,13 +1000,6 @@ func (k *kataAgent) stopSandbox(sandbox *Sandbox) error { - } - } - -- if err := k.proxy.stop(k.state.ProxyPid); err != nil { -- return err -- } -- -- // clean up agent state -- k.state.ProxyPid = -1 -- k.state.URL = "" - return nil - } - --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0004-kata-runtime-keep-the-process-name-of-qemu-same-as-c.patch b/runtime/patches/0004-kata-runtime-keep-the-process-name-of-qemu-same-as-c.patch deleted file mode 100644 index e212506..0000000 --- a/runtime/patches/0004-kata-runtime-keep-the-process-name-of-qemu-same-as-c.patch +++ /dev/null @@ -1,65 +0,0 @@ -From 025520f7fd3aeb5ed53b468b5e494b1bbb6674ae Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Sat, 25 Jul 2020 11:25:34 +0800 -Subject: [PATCH 04/50] kata-runtime: keep the process name of qemu same as - configured path - -reason: inorder to make testcase scripts can use the same hypervisor -name no matter what version hypervisor use, keep the process name of -hypervisor same as configured path in the configuration.toml file -instead of the resolved path of symbol link. - -Signed-off-by: jiangpengfei ---- - pkg/katautils/config.go | 8 +++++++- - pkg/katautils/config_test.go | 4 ++-- - 2 files changed, 9 insertions(+), 3 deletions(-) - -diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go -index 14794a24..349e667f 100644 ---- a/pkg/katautils/config.go -+++ b/pkg/katautils/config.go -@@ -10,6 +10,7 @@ import ( - "errors" - "fmt" - "io/ioutil" -+ "path/filepath" - goruntime "runtime" - "strings" - -@@ -172,7 +173,12 @@ func (h hypervisor) path() (string, error) { - p = defaultHypervisorPath - } - -- return ResolvePath(p) -+ absolutePath, err := filepath.Abs(p) -+ if err != nil { -+ return "", err -+ } -+ -+ return absolutePath, nil - } - - func (h hypervisor) ctlpath() (string, error) { -diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go -index 221a4b55..2eae1f6a 100644 ---- a/pkg/katautils/config_test.go -+++ b/pkg/katautils/config_test.go -@@ -1061,12 +1061,12 @@ func TestHypervisorDefaultsHypervisor(t *testing.T) { - assert.NoError(err) - assert.Equal(p, defaultHypervisorPath, "default hypervisor path wrong") - -- // test path resolution -+ // test path resolution, just return the absolute path instead of resolved path - defaultHypervisorPath = testHypervisorLinkPath - h = hypervisor{} - p, err = h.path() - assert.NoError(err) -- assert.Equal(p, testHypervisorPath) -+ assert.Equal(p, testHypervisorLinkPath) - } - - func TestHypervisorDefaultsKernel(t *testing.T) { --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0005-cgroups-increase-delete-cgroup-retry-times.patch b/runtime/patches/0005-cgroups-increase-delete-cgroup-retry-times.patch deleted file mode 100644 index cc9bd5a..0000000 --- a/runtime/patches/0005-cgroups-increase-delete-cgroup-retry-times.patch +++ /dev/null @@ -1,66 +0,0 @@ -From c279f4548ccc534f1c65723bf9994c448e510d3d Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Sat, 25 Jul 2020 11:56:35 +0800 -Subject: [PATCH 05/50] cgroups: increase delete cgroup retry times - -reason: inorder to make sure cgroup dir to be deleted, so we increase -the retry times when delete cgroup dir failed. - -Signed-off-by: jiangpengfei ---- - vendor/github.com/containerd/cgroups/cgroup.go | 4 ++-- - vendor/github.com/containerd/cgroups/utils.go | 9 ++++++--- - 2 files changed, 8 insertions(+), 5 deletions(-) - -diff --git a/vendor/github.com/containerd/cgroups/cgroup.go b/vendor/github.com/containerd/cgroups/cgroup.go -index 53866685..69612b0a 100644 ---- a/vendor/github.com/containerd/cgroups/cgroup.go -+++ b/vendor/github.com/containerd/cgroups/cgroup.go -@@ -223,7 +223,7 @@ func (c *cgroup) Delete() error { - return err - } - if err := d.Delete(sp); err != nil { -- errors = append(errors, string(s.Name())) -+ errors = append(errors, fmt.Sprintf("delete %s get error: %v", string(s.Name()), err.Error())) - } - continue - } -@@ -234,7 +234,7 @@ func (c *cgroup) Delete() error { - } - path := p.Path(sp) - if err := remove(path); err != nil { -- errors = append(errors, path) -+ errors = append(errors, fmt.Sprintf("remove path %s get error: %v", path, err.Error())) - } - } - } -diff --git a/vendor/github.com/containerd/cgroups/utils.go b/vendor/github.com/containerd/cgroups/utils.go -index 8a97d04d..82dbe2d3 100644 ---- a/vendor/github.com/containerd/cgroups/utils.go -+++ b/vendor/github.com/containerd/cgroups/utils.go -@@ -99,16 +99,19 @@ func defaults(root string) ([]Subsystem, error) { - // retrying the remove after a exp timeout - func remove(path string) error { - delay := 10 * time.Millisecond -- for i := 0; i < 5; i++ { -+ var err error -+ var count int = 0 -+ for i := 0; i < 10; i++ { - if i != 0 { - time.Sleep(delay) - delay *= 2 - } -- if err := os.RemoveAll(path); err == nil { -+ if err = os.RemoveAll(path); err == nil { - return nil - } -+ count++ - } -- return fmt.Errorf("cgroups: unable to remove path %q", path) -+ return fmt.Errorf("cgroups: unable to remove path %q, err: %v, count:%d", path, err, count) - } - - // readPids will read all the pids of processes in a cgroup by the provided path --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0006-kata-runtime-fix-umount-container-rootfs-dir-return-.patch b/runtime/patches/0006-kata-runtime-fix-umount-container-rootfs-dir-return-.patch deleted file mode 100644 index 12e60f1..0000000 --- a/runtime/patches/0006-kata-runtime-fix-umount-container-rootfs-dir-return-.patch +++ /dev/null @@ -1,39 +0,0 @@ -From 3dc10421f177900c0ee94fc49b32ec66a46d9331 Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Mon, 27 Jul 2020 19:18:50 +0800 -Subject: [PATCH 06/50] kata-runtime: fix umount container rootfs dir return - ivalid argument error - -reason: If sandbox hypervisor doesn't use block device driver for hotplugging container -rootfs block device into guest, kata-runtime will bind mount container rootfs dir to 9p -kataShared dir. However, container stop() function will always call bindUnmountContainerRootfs -function no matter block device driver is used or not. So we just need to call -bindUnmountContainerRootfs only if rootfs is bind mount to guest by 9p. - -Signed-off-by: jiangpengfei ---- - virtcontainers/container.go | 8 ++++++-- - 1 file changed, 6 insertions(+), 2 deletions(-) - -diff --git a/virtcontainers/container.go b/virtcontainers/container.go -index 9e2d1e94..b42cc6e9 100644 ---- a/virtcontainers/container.go -+++ b/virtcontainers/container.go -@@ -1120,8 +1120,12 @@ func (c *Container) stop(force bool) error { - return err - } - -- if err := bindUnmountContainerRootfs(c.ctx, getMountPath(c.sandbox.id), c.id); err != nil && !force { -- return err -+ // umount container rootfs dir only if container use 9p -+ // to bind mount host container rootfs to 9p shared dir -+ if c.state.BlockDeviceID == "" { -+ if err := bindUnmountContainerRootfs(c.ctx, getMountPath(c.sandbox.id), c.id); err != nil && !force { -+ return err -+ } - } - - if err := c.detachDevices(); err != nil && !force { --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0007-kata-runtime-enhance-reliability-when-kata-related-p.patch b/runtime/patches/0007-kata-runtime-enhance-reliability-when-kata-related-p.patch deleted file mode 100644 index 2fdecfb..0000000 --- a/runtime/patches/0007-kata-runtime-enhance-reliability-when-kata-related-p.patch +++ /dev/null @@ -1,855 +0,0 @@ -From d93da1875ed7f1a6061cffb13475506d73c86003 Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Sat, 25 Jul 2020 16:04:19 +0800 -Subject: [PATCH 07/50] kata-runtime: enhance reliability when kata related - process - -reason: enhance the reliability when kata related processes is abnormal, -make kata-container still destroy the sandbox and clean up all resources. - -Signed-off-by: jiangpengfei ---- - cli/delete.go | 6 ++ - cli/kill.go | 3 +- - virtcontainers/acrn.go | 2 +- - virtcontainers/agent.go | 3 + - virtcontainers/api.go | 67 +++++++++++++++++++++ - virtcontainers/clh.go | 2 +- - virtcontainers/container.go | 55 +++++++++++++++-- - virtcontainers/fc.go | 2 +- - virtcontainers/hypervisor.go | 2 +- - virtcontainers/kata_agent.go | 31 ++++++---- - virtcontainers/mock_hypervisor.go | 2 +- - virtcontainers/mock_hypervisor_test.go | 2 +- - virtcontainers/noop_agent.go | 4 ++ - virtcontainers/pkg/oci/utils.go | 5 ++ - virtcontainers/qemu.go | 51 +++++++++------- - virtcontainers/sandbox.go | 106 +++++++++++++++++++++++++++++---- - virtcontainers/types/sandbox.go | 14 ++++- - virtcontainers/utils/utils.go | 46 ++++++++++++++ - virtcontainers/vm.go | 4 +- - 19 files changed, 348 insertions(+), 59 deletions(-) - -diff --git a/cli/delete.go b/cli/delete.go -index c2ce52a4..2f5586e5 100644 ---- a/cli/delete.go -+++ b/cli/delete.go -@@ -110,6 +110,12 @@ func delete(ctx context.Context, containerID string, force bool) error { - forceStop = true - } - -+ if oci.StateToOCIState(status.State.State) == oci.StateUnhealthy { -+ // Set forceStop and force bool flag to true to force delete everything -+ forceStop = true -+ force = true -+ } -+ - switch containerType { - case vc.PodSandbox: - if err := deleteSandbox(ctx, sandboxID, force); err != nil { -diff --git a/cli/kill.go b/cli/kill.go -index 60fa41e0..b228205f 100644 ---- a/cli/kill.go -+++ b/cli/kill.go -@@ -133,11 +133,12 @@ func kill(ctx context.Context, containerID, signal string, all bool) error { - kataLog.WithField("signal", signal).WithField("container state", status.State.State).Info("kill") - - // container MUST be created, running or paused -+ // If container state is unhealthy, should process this exceptional case separately - if status.State.State == types.StateReady || status.State.State == types.StateRunning || status.State.State == types.StatePaused { - if err := vci.KillContainer(ctx, sandboxID, containerID, signum, all); err != nil { - return err - } -- } else if !all { -+ } else if !all && status.State.State != types.StateUnhealthy { - return fmt.Errorf("container not running") - } - -diff --git a/virtcontainers/acrn.go b/virtcontainers/acrn.go -index 761eda03..10cae06f 100644 ---- a/virtcontainers/acrn.go -+++ b/virtcontainers/acrn.go -@@ -475,7 +475,7 @@ func (a *Acrn) waitSandbox(timeoutSecs int) error { - } - - // stopSandbox will stop the Sandbox's VM. --func (a *Acrn) stopSandbox() (err error) { -+func (a *Acrn) stopSandbox(force bool) (err error) { - span, _ := a.trace("stopSandbox") - defer span.Finish() - -diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go -index c62107ec..be9526c7 100644 ---- a/virtcontainers/agent.go -+++ b/virtcontainers/agent.go -@@ -259,4 +259,7 @@ type agent interface { - - // load data from disk - load(persistapi.AgentState) -+ -+ // get proxy process pid -+ getProxyPid() int - } -diff --git a/virtcontainers/api.go b/virtcontainers/api.go -index de569713..fa82d163 100644 ---- a/virtcontainers/api.go -+++ b/virtcontainers/api.go -@@ -7,8 +7,10 @@ package virtcontainers - - import ( - "context" -+ "fmt" - "os" - "runtime" -+ "strings" - "syscall" - - deviceApi "github.com/kata-containers/runtime/virtcontainers/device/api" -@@ -18,6 +20,7 @@ import ( - vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" - "github.com/kata-containers/runtime/virtcontainers/store" - "github.com/kata-containers/runtime/virtcontainers/types" -+ "github.com/kata-containers/runtime/virtcontainers/utils" - specs "github.com/opencontainers/runtime-spec/specs-go" - opentracing "github.com/opentracing/opentracing-go" - "github.com/sirupsen/logrus" -@@ -597,20 +600,51 @@ func statusContainer(sandbox *Sandbox, containerID string) (ContainerStatus, err - container.state.State == types.StatePaused) && - container.process.Pid > 0 { - -+ // If container state is active, however kata-proxy and qemu process all exit already -+ // which means sandbox has beed stopped exceptionally, then we should force delete -+ // sandbox and container state files in sandbox.Store -+ if sandbox.shouldForceDelete() { -+ virtLog.Logger.Warn("sandbox status is abnormal, sandbox should be force deleted") -+ sandbox.forceDeleteSandbox() -+ return ContainerStatus{}, fmt.Errorf("sandbox has beed stopped exceptionally") -+ } -+ - running, err := isShimRunning(container.process.Pid) - if err != nil { - return ContainerStatus{}, err - } - -+ // If kata-shim process exit or be killed, need to stop the container - if !running { - virtLog.WithFields(logrus.Fields{ - "state": container.state.State, - "pid": container.process.Pid}). - Info("container isn't running") -+ - if err := container.stop(true); err != nil { - return ContainerStatus{}, err - } - } -+ -+ isPodSandbox := (containerID == sandbox.id) -+ -+ // If sandbox is unhealthy, process it correctly -+ if !sandbox.health() { -+ // process podSandbox container type case -+ if isPodSandbox { -+ if err := processUnhealthySandbox(sandbox, container); err != nil { -+ return ContainerStatus{}, err -+ } -+ } else { -+ // If container type is pod_container, which means container operations can not be -+ // processed successfully, we should return the error as soon as possible -+ if err := container.setContainerState(types.StateUnhealthy); err != nil { -+ return ContainerStatus{}, err -+ } -+ -+ return ContainerStatus{}, fmt.Errorf("container status is unhealthy, stop container failed") -+ } -+ } - } - - return ContainerStatus{ -@@ -1016,3 +1050,36 @@ func CleanupContainer(ctx context.Context, sandboxID, containerID string, force - - return nil - } -+ -+// procesUnhealthySandbox only change sandbox state to unhealthy -+// when caller is kata-runtime kill or kata-runtime delete -+func processUnhealthySandbox(sandbox *Sandbox, container *Container) error { -+ // Set all containers state to unhealthy -+ if err := sandbox.setContainersState(types.StateUnhealthy); err != nil { -+ container.Logger().WithError(err).Warn("set all containers state to unhealthy fail") -+ } -+ -+ // Set sandbox state to unhealthy -+ if err := sandbox.setSandboxState(types.StateUnhealthy); err != nil { -+ container.Logger().WithError(err).Warn("set sandbox state to unhealthy fail") -+ } -+ -+ forceDelete := false -+ -+ // If process is kata-runtime kill or kata-runtime delete, -+ // we should kill or delete sandbox forcefully -+ if cmdline, err := utils.GetProcessCmdline(os.Getpid()); err != nil { -+ container.Logger().WithError(err).Warn("fail to get process cmdline info") -+ } else { -+ forceDelete = strings.Contains(cmdline, "kill") || strings.Contains(cmdline, "delete") -+ } -+ -+ if forceDelete { -+ // force stop podSandbox type container's kata-shim process -+ if err := stopShim(container.process.Pid); err != nil { -+ container.Logger().WithError(err).Warn("fail to stop podSandbox type container kata-shim") -+ } -+ } -+ -+ return nil -+} -diff --git a/virtcontainers/clh.go b/virtcontainers/clh.go -index d40b698b..59510b02 100644 ---- a/virtcontainers/clh.go -+++ b/virtcontainers/clh.go -@@ -569,7 +569,7 @@ func (clh *cloudHypervisor) resumeSandbox() error { - } - - // stopSandbox will stop the Sandbox's VM. --func (clh *cloudHypervisor) stopSandbox() (err error) { -+func (clh *cloudHypervisor) stopSandbox(force bool) (err error) { - span, _ := clh.trace("stopSandbox") - defer span.Finish() - clh.Logger().WithField("function", "stopSandbox").Info("Stop Sandbox") -diff --git a/virtcontainers/container.go b/virtcontainers/container.go -index b42cc6e9..9485e708 100644 ---- a/virtcontainers/container.go -+++ b/virtcontainers/container.go -@@ -17,8 +17,12 @@ import ( - "time" - - "github.com/containerd/cgroups" -+ "github.com/kata-containers/runtime/virtcontainers/device/config" -+ "github.com/kata-containers/runtime/virtcontainers/device/manager" - vccgroups "github.com/kata-containers/runtime/virtcontainers/pkg/cgroups" -+ "github.com/kata-containers/runtime/virtcontainers/pkg/rootless" - vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" -+ "github.com/kata-containers/runtime/virtcontainers/store" - "github.com/kata-containers/runtime/virtcontainers/types" - "github.com/kata-containers/runtime/virtcontainers/utils" - specs "github.com/opencontainers/runtime-spec/specs-go" -@@ -26,11 +30,6 @@ import ( - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "golang.org/x/sys/unix" -- -- "github.com/kata-containers/runtime/virtcontainers/device/config" -- "github.com/kata-containers/runtime/virtcontainers/device/manager" -- "github.com/kata-containers/runtime/virtcontainers/pkg/rootless" -- "github.com/kata-containers/runtime/virtcontainers/store" - ) - - // https://github.com/torvalds/linux/blob/master/include/uapi/linux/major.h -@@ -1047,6 +1046,13 @@ func (c *Container) stop(force bool) error { - return nil - } - -+ // If container state is unhealthy, just force kill the container -+ if c.state.State == types.StateUnhealthy { -+ c.forceKillContainer() -+ // after force kill container, then change container state to stopped -+ return c.setContainerState(types.StateStopped) -+ } -+ - if err := c.state.ValidTransition(c.state.State, types.StateStopped); err != nil { - return err - } -@@ -1063,6 +1069,8 @@ func (c *Container) stop(force bool) error { - if err := stopShim(c.process.Pid); err != nil { - l.WithError(err).Warn("failed to stop shim") - } -+ -+ c.forceKillContainer() - } - - }() -@@ -1096,7 +1104,9 @@ func (c *Container) stop(force bool) error { - // this signal will ensure the container will get killed to match - // the state of the shim. This will allow the following call to - // stopContainer() to succeed in such particular case. -- c.kill(syscall.SIGKILL, true) -+ if err := c.kill(syscall.SIGKILL, true); err != nil { -+ c.Logger().Errorf("send signal to container failed: %v", err) -+ } - - // Since the agent has supported the MultiWaitProcess, it's better to - // wait the process here to make sure the process has exited before to -@@ -1582,3 +1592,36 @@ func (c *Container) cgroupsUpdate(resources specs.LinuxResources) error { - - return nil - } -+ -+// forceDeleteContainer force clean container mount info and resources stored in the disk -+func (c *Container) forceDeleteContainer() { -+ if err := c.unmountHostMounts(); err != nil { -+ c.Logger().WithError(err).Warn("container force delete umount host mounts fail") -+ } -+ -+ if err := c.sandbox.removeContainer(c.id); err != nil { -+ c.Logger().WithError(err).Warn("sandbox removeContainer fail") -+ } -+ -+ if err := c.store.Delete(); err != nil { -+ c.Logger().WithError(err).Warn("force delete container store fail") -+ } -+} -+ -+func (c *Container) forceKillContainer() { -+ if err := c.setContainerState(types.StateStopped); err != nil { -+ c.Logger().WithError(err).Warn("force kill container: change container state to StateStopped failed") -+ } -+ -+ if err := c.unmountHostMounts(); err != nil { -+ c.Logger().WithError(err).Warn("force kill container: umount container host mounts failed") -+ } -+ -+ if err := c.detachDevices(); err != nil { -+ c.Logger().WithError(err).Warn("force kill container: detach container devices failed") -+ } -+ -+ if err := c.removeDrive(); err != nil { -+ c.Logger().WithError(err).Warn("force kill container: remove container drive failed") -+ } -+} -diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go -index 97ef5ffc..72a8e192 100644 ---- a/virtcontainers/fc.go -+++ b/virtcontainers/fc.go -@@ -864,7 +864,7 @@ func (fc *firecracker) cleanupJail() { - } - - // stopSandbox will stop the Sandbox's VM. --func (fc *firecracker) stopSandbox() (err error) { -+func (fc *firecracker) stopSandbox(force bool) (err error) { - span, _ := fc.trace("stopSandbox") - defer span.Finish() - -diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go -index 4b3dd3d0..fd7d1f8e 100644 ---- a/virtcontainers/hypervisor.go -+++ b/virtcontainers/hypervisor.go -@@ -766,7 +766,7 @@ func generateVMSocket(id string, useVsock bool, vmStogarePath string) (interface - type hypervisor interface { - createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, stateful bool) error - startSandbox(timeout int) error -- stopSandbox() error -+ stopSandbox(force bool) error - pauseSandbox() error - saveSandbox() error - resumeSandbox() error -diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go -index be5e96aa..7575d326 100644 ---- a/virtcontainers/kata_agent.go -+++ b/virtcontainers/kata_agent.go -@@ -57,8 +57,9 @@ const ( - ) - - var ( -- checkRequestTimeout = 30 * time.Second -- defaultRequestTimeout = 60 * time.Second -+ checkRequestTimeout = 10 * time.Second -+ defaultRequestTimeout = 10 * time.Second -+ createContainerTimeout = 120 * time.Second - errorMissingProxy = errors.New("Missing proxy pointer") - errorMissingOCISpec = errors.New("Missing OCI specification") - defaultKataHostSharedDir = "/run/kata-containers/shared/sandboxes/" -@@ -987,17 +988,21 @@ func (k *kataAgent) stopSandbox(sandbox *Sandbox) error { - k.state.URL = "" - }() - -- req := &grpc.DestroySandboxRequest{} -+ // If sandbox.state.State is unhealthy, we don't need to send DestroySandboxRequest -+ // to kata-agent, just force stop the sandbox -+ if sandbox.state.State != types.StateUnhealthy { -+ req := &grpc.DestroySandboxRequest{} - -- if _, err := k.sendReq(req); err != nil { -- return err -- } -- -- if k.dynamicTracing { -- _, err := k.sendReq(&grpc.StopTracingRequest{}) -- if err != nil { -+ if _, err := k.sendReq(req); err != nil { - return err - } -+ -+ if k.dynamicTracing { -+ _, err := k.sendReq(&grpc.StopTracingRequest{}) -+ if err != nil { -+ return err -+ } -+ } - } - - return nil -@@ -2062,6 +2067,8 @@ func (k *kataAgent) getReqContext(reqName string) (ctx context.Context, cancel c - // Wait has no timeout - case grpcCheckRequest: - ctx, cancel = context.WithTimeout(ctx, checkRequestTimeout) -+ case grpcCreateContainerRequest: -+ ctx, cancel = context.WithTimeout(ctx, createContainerTimeout) - default: - ctx, cancel = context.WithTimeout(ctx, defaultRequestTimeout) - } -@@ -2382,3 +2389,7 @@ func (k *kataAgent) load(s persistapi.AgentState) { - k.state.ProxyPid = s.ProxyPid - k.state.URL = s.URL - } -+ -+func (k *kataAgent) getProxyPid() int { -+ return k.state.ProxyPid -+} -diff --git a/virtcontainers/mock_hypervisor.go b/virtcontainers/mock_hypervisor.go -index 0c84e43c..a5b67491 100644 ---- a/virtcontainers/mock_hypervisor.go -+++ b/virtcontainers/mock_hypervisor.go -@@ -39,7 +39,7 @@ func (m *mockHypervisor) startSandbox(timeout int) error { - return nil - } - --func (m *mockHypervisor) stopSandbox() error { -+func (m *mockHypervisor) stopSandbox(force bool) error { - return nil - } - -diff --git a/virtcontainers/mock_hypervisor_test.go b/virtcontainers/mock_hypervisor_test.go -index b73b28f2..827e3192 100644 ---- a/virtcontainers/mock_hypervisor_test.go -+++ b/virtcontainers/mock_hypervisor_test.go -@@ -53,7 +53,7 @@ func TestMockHypervisorStartSandbox(t *testing.T) { - func TestMockHypervisorStopSandbox(t *testing.T) { - var m *mockHypervisor - -- assert.NoError(t, m.stopSandbox()) -+ assert.NoError(t, m.stopSandbox(false)) - } - - func TestMockHypervisorAddDevice(t *testing.T) { -diff --git a/virtcontainers/noop_agent.go b/virtcontainers/noop_agent.go -index 189f6b3f..8a7cd337 100644 ---- a/virtcontainers/noop_agent.go -+++ b/virtcontainers/noop_agent.go -@@ -236,3 +236,7 @@ func (n *noopAgent) save() (s persistapi.AgentState) { - - // load is the Noop agent state loader. It does nothing. - func (n *noopAgent) load(s persistapi.AgentState) {} -+ -+func (n *noopAgent) getProxyPid() int { -+ return -1 -+} -\ No newline at end of file -diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go -index 5348c57d..cd8d48ce 100644 ---- a/virtcontainers/pkg/oci/utils.go -+++ b/virtcontainers/pkg/oci/utils.go -@@ -70,6 +70,9 @@ const ( - - // StatePaused represents a container that has been paused. - StatePaused = "paused" -+ -+ // StateUnhealthy represents a container that is unhealthy -+ StateUnhealthy = "unhealthy" - ) - - const KernelModulesSeparator = ";" -@@ -964,6 +967,8 @@ func StateToOCIState(state types.StateString) string { - return StateStopped - case types.StatePaused: - return StatePaused -+ case types.StateUnhealthy: -+ return StateUnhealthy - default: - return "" - } -diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go -index ca286550..4b15d968 100644 ---- a/virtcontainers/qemu.go -+++ b/virtcontainers/qemu.go -@@ -687,7 +687,7 @@ func (q *qemu) setupVirtiofsd() (err error) { - q.Logger().Info("virtiofsd quits") - // Wait to release resources of virtiofsd process - cmd.Process.Wait() -- q.stopSandbox() -+ q.stopSandbox(false) - }() - return err - } -@@ -922,11 +922,11 @@ func (q *qemu) waitSandbox(timeout int) error { - } - - // stopSandbox will stop the Sandbox's VM. --func (q *qemu) stopSandbox() error { -+func (q *qemu) stopSandbox(force bool) error { - span, _ := q.trace("stopSandbox") - defer span.Finish() - -- q.Logger().Info("Stopping Sandbox") -+ q.Logger().Infof("force stopping Sandbox: %v", force) - if q.stopped { - q.Logger().Info("Already stopped") - return nil -@@ -937,28 +937,37 @@ func (q *qemu) stopSandbox() error { - q.stopped = true - }() - -- if q.config.Debug && q.qemuConfig.LogFile != "" { -- f, err := os.OpenFile(q.qemuConfig.LogFile, os.O_RDONLY, 0) -- if err == nil { -- scanner := bufio.NewScanner(f) -- for scanner.Scan() { -- q.Logger().Debug(scanner.Text()) -- } -- if err := scanner.Err(); err != nil { -- q.Logger().WithError(err).Debug("read qemu log failed") -+ if !force { -+ if q.config.Debug && q.qemuConfig.LogFile != "" { -+ f, err := os.OpenFile(q.qemuConfig.LogFile, os.O_RDONLY, 0) -+ if err == nil { -+ scanner := bufio.NewScanner(f) -+ for scanner.Scan() { -+ q.Logger().Debug(scanner.Text()) -+ } -+ if err := scanner.Err(); err != nil { -+ q.Logger().WithError(err).Debug("read qemu log failed") -+ } - } - } -- } - -- err := q.qmpSetup() -- if err != nil { -- return err -- } -+ err := q.qmpSetup() -+ if err != nil { -+ return err -+ } - -- err = q.qmpMonitorCh.qmp.ExecuteQuit(q.qmpMonitorCh.ctx) -- if err != nil { -- q.Logger().WithError(err).Error("Fail to execute qmp QUIT") -- return err -+ err = q.qmpMonitorCh.qmp.ExecuteQuit(q.qmpMonitorCh.ctx) -+ if err != nil { -+ q.Logger().WithError(err).Error("Fail to execute qmp QUIT") -+ return err -+ } -+ } else { -+ qemuMainPid := q.getPids()[0] -+ if qemuMainPid <= 1 { -+ return fmt.Errorf("force kill qemu process pid is invalid") -+ } -+ -+ _ = syscall.Kill(qemuMainPid, syscall.SIGKILL) - } - - return nil -diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go -index edd1af5b..78188ed7 100644 ---- a/virtcontainers/sandbox.go -+++ b/virtcontainers/sandbox.go -@@ -12,19 +12,13 @@ import ( - "math" - "net" - "os" -+ "path/filepath" - "strings" - "sync" - "syscall" - - "github.com/containerd/cgroups" - "github.com/containernetworking/plugins/pkg/ns" -- "github.com/opencontainers/runc/libcontainer/configs" -- specs "github.com/opencontainers/runtime-spec/specs-go" -- opentracing "github.com/opentracing/opentracing-go" -- "github.com/pkg/errors" -- "github.com/sirupsen/logrus" -- "github.com/vishvananda/netlink" -- - "github.com/kata-containers/agent/protocols/grpc" - "github.com/kata-containers/runtime/virtcontainers/device/api" - "github.com/kata-containers/runtime/virtcontainers/device/config" -@@ -41,6 +35,12 @@ import ( - "github.com/kata-containers/runtime/virtcontainers/store" - "github.com/kata-containers/runtime/virtcontainers/types" - "github.com/kata-containers/runtime/virtcontainers/utils" -+ "github.com/opencontainers/runc/libcontainer/configs" -+ specs "github.com/opencontainers/runtime-spec/specs-go" -+ opentracing "github.com/opentracing/opentracing-go" -+ "github.com/pkg/errors" -+ "github.com/sirupsen/logrus" -+ "github.com/vishvananda/netlink" - ) - - const ( -@@ -50,6 +50,9 @@ const ( - - // DirMode is the permission bits used for creating a directory - DirMode = os.FileMode(0750) | os.ModeDir -+ -+ // kata-proxy proces name -+ KataProxyProcessName = "kata-proxy" - ) - - // SandboxStatus describes a sandbox status. -@@ -1037,7 +1040,7 @@ func (s *Sandbox) startVM() (err error) { - - defer func() { - if err != nil { -- s.hypervisor.stopSandbox() -+ s.hypervisor.stopSandbox(false) - } - }() - -@@ -1090,7 +1093,12 @@ func (s *Sandbox) stopVM() error { - } - - s.Logger().Info("Stopping VM") -- return s.hypervisor.stopSandbox() -+ forceStop := false -+ if s.state.State == types.StateUnhealthy { -+ forceStop = true -+ } -+ -+ return s.hypervisor.stopSandbox(forceStop) - } - - func (s *Sandbox) addContainer(c *Container) error { -@@ -1591,13 +1599,15 @@ func (s *Sandbox) setSandboxState(state types.StateString) error { - return vcTypes.ErrNeedState - } - -+ s.Logger().Debugf("Setting sandbox state from %v to %v", s.state.State, state) - // update in-memory state - s.state.State = state - - if useOldStore(s.ctx) { - return s.store.Store(store.State, s.state) -+ } else { -+ return s.Save() - } -- return nil - } - - const maxBlockIndex = 65535 -@@ -2207,3 +2217,79 @@ func (s *Sandbox) GetPatchedOCISpec() *specs.Spec { - - return nil - } -+ -+// health return current sandbox healthy or not -+// If qemu/kata-proxy/kata-agent process is abnormal, -+// s.agent.check() will return false -+func (s *Sandbox) health() bool { -+ err := s.agent.check() -+ if err != nil { -+ return false -+ } -+ -+ return true -+} -+ -+// shouldForceDelete force delete the sandbox when kata-proxy and hypervisor process exit -+// already and current process is kata-runtime kill or kata-runtime delete -+func (s *Sandbox) shouldForceDelete() bool { -+ cmdline, err := utils.GetProcessCmdline(os.Getpid()) -+ if err != nil { -+ s.Logger().Errorf("fail to get process cmdline: %v", err) -+ return false -+ } -+ -+ proxyPid := s.agent.getProxyPid() -+ hypervisorPids := s.hypervisor.getPids() -+ if len(hypervisorPids) <= 0 { -+ s.Logger().Warnf("get hypervisor main pid fail") -+ return false -+ } -+ hypervisorMainPid := hypervisorPids[0] -+ hypervisorPath := s.hypervisor.hypervisorConfig().HypervisorPath -+ hypervisorName := filepath.Base(hypervisorPath) -+ -+ if !utils.IsProcessRunning(proxyPid, KataProxyProcessName, s.id) && !utils.IsProcessRunning(hypervisorMainPid, hypervisorName, s.id) && -+ strings.Contains(cmdline, "delete") && strings.Contains(cmdline, "force") { -+ return true -+ } -+ -+ return false -+} -+ -+func (s *Sandbox) forceDeleteSandbox() { -+ for _, c := range s.containers { -+ // force delete all containers in the sandbox -+ c.forceDeleteContainer() -+ } -+ -+ globalSandboxList.removeSandbox(s.id) -+ -+ if s.monitor != nil { -+ s.monitor.stop() -+ } -+ -+ if err := s.hypervisor.cleanup(); err != nil { -+ s.Logger().WithError(err).Error("failed to force cleanup hypervisor resource") -+ } -+ -+ s.agent.cleanup(s) -+ -+ if err := s.store.Delete(); err != nil { -+ s.Logger().WithError(err).Warn("sandbox force delete store failed") -+ } -+} -+ -+func (s *Sandbox) setContainersState(state types.StateString) error { -+ if state == "" { -+ return vcTypes.ErrNeedState -+ } -+ -+ for _, c := range s.containers { -+ if err := c.setContainerState(state); err != nil { -+ return err -+ } -+ } -+ -+ return nil -+} -diff --git a/virtcontainers/types/sandbox.go b/virtcontainers/types/sandbox.go -index 3b64b20a..5d586b21 100644 ---- a/virtcontainers/types/sandbox.go -+++ b/virtcontainers/types/sandbox.go -@@ -28,6 +28,9 @@ const ( - - // StateStopped represents a sandbox/container that has been stopped. - StateStopped StateString = "stopped" -+ -+ // StateUnhealthy represents a sandbox/container that's in abnormal state. -+ StateUnhealthy StateString = "unhealthy" - ) - - const ( -@@ -90,17 +93,17 @@ func (state *StateString) validTransition(oldState StateString, newState StateSt - - switch *state { - case StateReady: -- if newState == StateRunning || newState == StateStopped { -+ if newState == StateRunning || newState == StateStopped || newState == StateUnhealthy { - return nil - } - - case StateRunning: -- if newState == StatePaused || newState == StateStopped { -+ if newState == StatePaused || newState == StateStopped || newState == StateUnhealthy { - return nil - } - - case StatePaused: -- if newState == StateRunning || newState == StateStopped { -+ if newState == StateRunning || newState == StateStopped || newState == StateUnhealthy { - return nil - } - -@@ -108,6 +111,11 @@ func (state *StateString) validTransition(oldState StateString, newState StateSt - if newState == StateRunning { - return nil - } -+ -+ case StateUnhealthy: -+ if newState == StateStopped { -+ return nil -+ } - } - - return fmt.Errorf("Can not move from %v to %v", -diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go -index 85c55489..2b555ebb 100644 ---- a/virtcontainers/utils/utils.go -+++ b/virtcontainers/utils/utils.go -@@ -9,9 +9,13 @@ import ( - "crypto/rand" - "errors" - "fmt" -+ "io/ioutil" - "os" - "os/exec" - "path/filepath" -+ "strconv" -+ "strings" -+ "syscall" - ) - - const cpBinaryName = "cp" -@@ -275,3 +279,45 @@ const ( - MiB = KiB << 10 - GiB = MiB << 10 - ) -+ -+// Get process cmdline info by read /proc//cmdline file -+func GetProcessCmdline(pid int) (cmdline string, err error) { -+ if pid <= 1 { -+ return "", fmt.Errorf("invalid pid number") -+ } -+ -+ bytes, err := ioutil.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "cmdline")) -+ if err != nil { -+ return "", err -+ } -+ -+ return string(bytes), nil -+} -+ -+func IsProcessRunning(pid int, processName string, sandboxID string) bool { -+ if pid <= 0 { -+ return false -+ } -+ -+ process, err := os.FindProcess(pid) -+ if err != nil { -+ return false -+ } -+ -+ if err := process.Signal(syscall.Signal(0)); err != nil { -+ return false -+ } -+ -+ cmdline, err := GetProcessCmdline(pid) -+ if err != nil { -+ return false -+ } -+ -+ // If process's cmdline contains processName and sandboxID keyword, -+ // We think this process isn't be reused -+ if strings.Contains(cmdline, processName) && strings.Contains(cmdline, sandboxID) { -+ return true -+ } -+ -+ return false -+} -diff --git a/virtcontainers/vm.go b/virtcontainers/vm.go -index fcda1e97..8d27b1fe 100644 ---- a/virtcontainers/vm.go -+++ b/virtcontainers/vm.go -@@ -191,7 +191,7 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) { - defer func() { - if err != nil { - virtLog.WithField("vm", id).WithError(err).Info("clean up vm") -- hypervisor.stopSandbox() -+ hypervisor.stopSandbox(false) - } - }() - -@@ -333,7 +333,7 @@ func (v *VM) Disconnect() error { - func (v *VM) Stop() error { - v.logger().Info("stop vm") - -- if err := v.hypervisor.stopSandbox(); err != nil { -+ if err := v.hypervisor.stopSandbox(false); err != nil { - return err - } - --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0008-kata-runtime-fix-kata-runtime-resource-left-problem.patch b/runtime/patches/0008-kata-runtime-fix-kata-runtime-resource-left-problem.patch deleted file mode 100644 index e0022d5..0000000 --- a/runtime/patches/0008-kata-runtime-fix-kata-runtime-resource-left-problem.patch +++ /dev/null @@ -1,177 +0,0 @@ -From a1bf2e1c696b703935f4b81ca087a60cc2559464 Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Mon, 27 Jul 2020 21:45:25 +0800 -Subject: [PATCH 08/50] kata-runtime: fix kata-runtime resource left problem - -reason: fix the following resource left problem -- sandbox stored files and directory are deleted before work container delete -- ignore StatusSandbox error, let stopSandbox function call be executed - -Signed-off-by: jiangpengfei ---- - cli/delete.go | 25 +++++++++++++++++++++++-- - cli/delete_test.go | 15 +++++++++++++++ - cli/state.go | 6 ++++++ - virtcontainers/api.go | 4 +++- - 4 files changed, 47 insertions(+), 3 deletions(-) - -diff --git a/cli/delete.go b/cli/delete.go -index 2f5586e5..871ac40d 100644 ---- a/cli/delete.go -+++ b/cli/delete.go -@@ -10,6 +10,7 @@ import ( - "context" - "fmt" - "os" -+ "strings" - - "github.com/kata-containers/runtime/pkg/katautils" - vc "github.com/kata-containers/runtime/virtcontainers" -@@ -68,6 +69,12 @@ func delete(ctx context.Context, containerID string, force bool) error { - setExternalLoggers(ctx, kataLog) - span.SetTag("container", containerID) - -+ // Remove the containerID to sandboxID mapping -+ // no matter what error return -+ defer func() { -+ _ = katautils.DelContainerIDMapping(ctx, containerID) -+ }() -+ - // Checks the MUST and MUST NOT from OCI runtime specification - status, sandboxID, err := getExistingContainerInfo(ctx, containerID) - if err != nil { -@@ -75,6 +82,15 @@ func delete(ctx context.Context, containerID string, force bool) error { - kataLog.Warnf("Failed to get container, force will not fail: %s", err) - return nil - } -+ -+ // If err info containers "no such file or directory, because pod_sandbox type -+ // container is deleted before pod_container type container, just return nil -+ // and let containerd delete container operations continue -+ if strings.Contains(err.Error(), "no such file or directory") { -+ kataLog.Warnf("pod_sandbox deleted before pod_container: %v", err) -+ return nil -+ } -+ - return err - } - -@@ -123,7 +139,12 @@ func delete(ctx context.Context, containerID string, force bool) error { - } - case vc.PodContainer: - if err := deleteContainer(ctx, sandboxID, containerID, forceStop); err != nil { -- return err -+ // If err info containers "no such file or directory, because pod_sandbox type -+ // container is deleted before pod_container type container, just return nil -+ // and let containerd delete container operations continue -+ if !strings.Contains(err.Error(), "no such file or directory") { -+ return err -+ } - } - default: - return fmt.Errorf("Invalid container type found") -@@ -134,7 +155,7 @@ func delete(ctx context.Context, containerID string, force bool) error { - return err - } - -- return katautils.DelContainerIDMapping(ctx, containerID) -+ return nil - } - - func deleteSandbox(ctx context.Context, sandboxID string, force bool) error { -diff --git a/cli/delete_test.go b/cli/delete_test.go -index a2455dee..ae421cd7 100644 ---- a/cli/delete_test.go -+++ b/cli/delete_test.go -@@ -184,6 +184,9 @@ func TestDeleteSandbox(t *testing.T) { - assert.Error(err) - assert.True(vcmock.IsMockError(err)) - -+ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID()) -+ assert.NoError(err) -+ - testingImpl.StatusSandboxFunc = func(ctx context.Context, sandboxID string) (vc.SandboxStatus, error) { - return vc.SandboxStatus{ - ID: sandbox.ID(), -@@ -201,6 +204,9 @@ func TestDeleteSandbox(t *testing.T) { - assert.Error(err) - assert.True(vcmock.IsMockError(err)) - -+ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID()) -+ assert.NoError(err) -+ - testingImpl.StopSandboxFunc = func(ctx context.Context, sandboxID string, force bool) (vc.VCSandbox, error) { - return sandbox, nil - } -@@ -213,6 +219,9 @@ func TestDeleteSandbox(t *testing.T) { - assert.Error(err) - assert.True(vcmock.IsMockError(err)) - -+ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID()) -+ assert.NoError(err) -+ - testingImpl.DeleteSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) { - return sandbox, nil - } -@@ -302,6 +311,9 @@ func TestDeleteSandboxRunning(t *testing.T) { - assert.Error(err) - assert.False(vcmock.IsMockError(err)) - -+ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID()) -+ assert.NoError(err) -+ - testingImpl.StatusSandboxFunc = func(ctx context.Context, sandboxID string) (vc.SandboxStatus, error) { - return vc.SandboxStatus{ - ID: sandbox.ID(), -@@ -325,6 +337,9 @@ func TestDeleteSandboxRunning(t *testing.T) { - assert.Error(err) - assert.True(vcmock.IsMockError(err)) - -+ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID()) -+ assert.NoError(err) -+ - testingImpl.DeleteSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) { - return sandbox, nil - } -diff --git a/cli/state.go b/cli/state.go -index a2fcc12e..de843d34 100644 ---- a/cli/state.go -+++ b/cli/state.go -@@ -11,6 +11,7 @@ import ( - "encoding/json" - "fmt" - "os" -+ "strings" - - "github.com/kata-containers/runtime/pkg/katautils" - "github.com/kata-containers/runtime/virtcontainers/pkg/oci" -@@ -52,6 +53,11 @@ func state(ctx context.Context, containerID string) error { - // Checks the MUST and MUST NOT from OCI runtime specification - status, _, err := getExistingContainerInfo(ctx, containerID) - if err != nil { -+ // If err info containers "no such file or directory, because pod_sandbox type -+ // container is deleted before pod_container type container, just return nil -+ if strings.Contains(err.Error(), "no such file or directory") { -+ return nil -+ } - return err - } - -diff --git a/virtcontainers/api.go b/virtcontainers/api.go -index fa82d163..449a03e0 100644 ---- a/virtcontainers/api.go -+++ b/virtcontainers/api.go -@@ -374,7 +374,9 @@ func StatusSandbox(ctx context.Context, sandboxID string) (SandboxStatus, error) - for _, container := range s.containers { - contStatus, err := statusContainer(s, container.id) - if err != nil { -- return SandboxStatus{}, err -+ // Since statusContainer may get error because of qemu process D or T, -+ // So just ignore this error and let StatusSandbox function return actually SandboxStatus -+ s.Logger().Warnf("Status container's status in sandbox get error: %v", contStatus) - } - - contStatusList = append(contStatusList, contStatus) --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0009-kata-runtime-add-kata-runtime-global-flag-debug.patch b/runtime/patches/0009-kata-runtime-add-kata-runtime-global-flag-debug.patch deleted file mode 100644 index 19596fc..0000000 --- a/runtime/patches/0009-kata-runtime-add-kata-runtime-global-flag-debug.patch +++ /dev/null @@ -1,132 +0,0 @@ -From 508fd9b94b1b12be3167342b03a47f8f97245e9c Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Tue, 28 Jul 2020 10:53:30 +0800 -Subject: [PATCH 09/50] kata-runtime: add kata-runtime global flag --debug - -reason: add the same debug flag with runc to adapt to -containerd 1.2.0 - -Signed-off-by: jiangpengfei ---- - cli/kata-env_test.go | 2 +- - cli/main.go | 6 +++++- - containerd-shim-v2/create.go | 2 +- - pkg/katautils/config.go | 4 ++-- - pkg/katautils/config_test.go | 8 ++++---- - 5 files changed, 13 insertions(+), 9 deletions(-) - -diff --git a/cli/kata-env_test.go b/cli/kata-env_test.go -index b31b6cc2..75bb697f 100644 ---- a/cli/kata-env_test.go -+++ b/cli/kata-env_test.go -@@ -178,7 +178,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC - return "", oci.RuntimeConfig{}, err - } - -- _, config, err = katautils.LoadConfiguration(configFile, true, false) -+ _, config, err = katautils.LoadConfiguration(configFile, true, false, false) - if err != nil { - return "", oci.RuntimeConfig{}, err - } -diff --git a/cli/main.go b/cli/main.go -index df16c5f3..1362a8fb 100644 ---- a/cli/main.go -+++ b/cli/main.go -@@ -115,6 +115,10 @@ var runtimeFlags = []cli.Flag{ - Name: "systemd-cgroup", - Usage: "enable systemd cgroup support, expects cgroupsPath to be of form \"slice:prefix:name\" for e.g. \"system.slice:runc:434234\"", - }, -+ cli.BoolFlag{ -+ Name: "debug", -+ Usage: "enable debug output for logging", -+ }, - } - - // runtimeCommands is the list of supported command-line (sub-) -@@ -334,7 +338,7 @@ func beforeSubcommands(c *cli.Context) error { - } - } - -- configFile, runtimeConfig, err = katautils.LoadConfiguration(c.GlobalString(configFilePathOption), ignoreConfigLogs, false) -+ configFile, runtimeConfig, err = katautils.LoadConfiguration(c.GlobalString(configFilePathOption), ignoreConfigLogs, false, c.GlobalBool("debug")) - if err != nil { - fatal(err) - } -diff --git a/containerd-shim-v2/create.go b/containerd-shim-v2/create.go -index affdbae2..9749073d 100644 ---- a/containerd-shim-v2/create.go -+++ b/containerd-shim-v2/create.go -@@ -167,7 +167,7 @@ func loadRuntimeConfig(s *service, r *taskAPI.CreateTaskRequest, anno map[string - configPath = os.Getenv("KATA_CONF_FILE") - } - -- _, runtimeConfig, err := katautils.LoadConfiguration(configPath, false, true) -+ _, runtimeConfig, err := katautils.LoadConfiguration(configPath, false, true, false) - if err != nil { - return nil, err - } -diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go -index 349e667f..448d23ac 100644 ---- a/pkg/katautils/config.go -+++ b/pkg/katautils/config.go -@@ -1141,7 +1141,7 @@ func initConfig() (config oci.RuntimeConfig, err error) { - // - // All paths are resolved fully meaning if this function does not return an - // error, all paths are valid at the time of the call. --func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolvedConfigPath string, config oci.RuntimeConfig, err error) { -+func LoadConfiguration(configPath string, ignoreLogging, builtIn bool, debugFlag bool) (resolvedConfigPath string, config oci.RuntimeConfig, err error) { - - config, err = initConfig() - if err != nil { -@@ -1154,7 +1154,7 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved - } - - config.Debug = tomlConf.Runtime.Debug -- if !tomlConf.Runtime.Debug { -+ if !tomlConf.Runtime.Debug && !debugFlag { - // If debug is not required, switch back to the original - // default log priority, otherwise continue in debug mode. - kataUtilsLogger.Logger.Level = originalLoggerLevel -diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go -index 2eae1f6a..31afcca6 100644 ---- a/pkg/katautils/config_test.go -+++ b/pkg/katautils/config_test.go -@@ -264,7 +264,7 @@ func testLoadConfiguration(t *testing.T, dir string, - assert.NoError(t, err) - } - -- resolvedConfigPath, config, err := LoadConfiguration(file, ignoreLogging, false) -+ resolvedConfigPath, config, err := LoadConfiguration(file, ignoreLogging, false, false) - if expectFail { - assert.Error(t, err) - -@@ -578,7 +578,7 @@ func TestMinimalRuntimeConfig(t *testing.T) { - t.Fatal(err) - } - -- _, config, err := LoadConfiguration(configPath, false, false) -+ _, config, err := LoadConfiguration(configPath, false, false, false) - if err == nil { - t.Fatalf("Expected loadConfiguration to fail as shim path does not exist: %+v", config) - } -@@ -608,7 +608,7 @@ func TestMinimalRuntimeConfig(t *testing.T) { - t.Error(err) - } - -- _, config, err = LoadConfiguration(configPath, false, false) -+ _, config, err = LoadConfiguration(configPath, false, false, false) - if err != nil { - t.Fatal(err) - } -@@ -748,7 +748,7 @@ func TestMinimalRuntimeConfigWithVsock(t *testing.T) { - t.Fatal(err) - } - -- _, config, err := LoadConfiguration(configPath, false, false) -+ _, config, err := LoadConfiguration(configPath, false, false, false) - if err != nil { - t.Fatal(err) - } --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0010-kata-runtime-fix-kata-shim-pid-reused-problem.patch b/runtime/patches/0010-kata-runtime-fix-kata-shim-pid-reused-problem.patch deleted file mode 100644 index 31dc1ba..0000000 --- a/runtime/patches/0010-kata-runtime-fix-kata-shim-pid-reused-problem.patch +++ /dev/null @@ -1,172 +0,0 @@ -From 76cbca91608e94c1855705ad1a8d06ffa2273115 Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Tue, 28 Jul 2020 18:18:54 +0800 -Subject: [PATCH 10/50] kata-runtime: fix kata-shim pid reused problem - -reason: If kata-shim process exit and it's pid reused by other process, -it may cause kill other proecss and cause some problem. - -Signed-off-by: jiangpengfei ---- - virtcontainers/api.go | 2 +- - virtcontainers/container.go | 6 +++--- - virtcontainers/shim.go | 21 +++++++++++++++++---- - virtcontainers/shim_test.go | 10 +++++----- - 4 files changed, 26 insertions(+), 13 deletions(-) - -diff --git a/virtcontainers/api.go b/virtcontainers/api.go -index 449a03e0..5e8c9c9e 100644 ---- a/virtcontainers/api.go -+++ b/virtcontainers/api.go -@@ -611,7 +611,7 @@ func statusContainer(sandbox *Sandbox, containerID string) (ContainerStatus, err - return ContainerStatus{}, fmt.Errorf("sandbox has beed stopped exceptionally") - } - -- running, err := isShimRunning(container.process.Pid) -+ running, err := isShimRunning(container.process.Pid, containerID) - if err != nil { - return ContainerStatus{}, err - } -diff --git a/virtcontainers/container.go b/virtcontainers/container.go -index 9485e708..75f590eb 100644 ---- a/virtcontainers/container.go -+++ b/virtcontainers/container.go -@@ -1063,7 +1063,7 @@ func (c *Container) stop(force bool) error { - - // If shim is still running something went wrong - // Make sure we stop the shim process -- if running, _ := isShimRunning(c.process.Pid); running { -+ if running, _ := isShimRunning(c.process.Pid, c.id); running { - l := c.Logger() - l.Error("Failed to stop container so stopping dangling shim") - if err := stopShim(c.process.Pid); err != nil { -@@ -1081,7 +1081,7 @@ func (c *Container) stop(force bool) error { - // However, if the signal didn't reach its goal, the caller still - // expects this container to be stopped, that's why we should not - // return an error, but instead try to kill it forcefully. -- if err := waitForShim(c.process.Pid); err != nil { -+ if err := waitForShim(c.process.Pid, c.id); err != nil { - // Force the container to be killed. - if err := c.kill(syscall.SIGKILL, true); err != nil && !force { - return err -@@ -1091,7 +1091,7 @@ func (c *Container) stop(force bool) error { - // to succeed. Indeed, we have already given a second chance - // to the container by trying to kill it with SIGKILL, there - // is no reason to try to go further if we got an error. -- if err := waitForShim(c.process.Pid); err != nil && !force { -+ if err := waitForShim(c.process.Pid, c.id); err != nil && !force { - return err - } - } -diff --git a/virtcontainers/shim.go b/virtcontainers/shim.go -index 8ec7458b..6f784a03 100644 ---- a/virtcontainers/shim.go -+++ b/virtcontainers/shim.go -@@ -9,11 +9,13 @@ import ( - "fmt" - "os" - "os/exec" -+ "strings" - "syscall" - "time" - - ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter" - "github.com/kata-containers/runtime/virtcontainers/types" -+ "github.com/kata-containers/runtime/virtcontainers/utils" - "github.com/mitchellh/mapstructure" - "github.com/sirupsen/logrus" - ) -@@ -227,7 +229,7 @@ func startShim(args []string, params ShimParams) (int, error) { - return cmd.Process.Pid, nil - } - --func isShimRunning(pid int) (bool, error) { -+func isShimRunning(pid int, containerID string) (bool, error) { - if pid <= 0 { - return false, nil - } -@@ -241,19 +243,30 @@ func isShimRunning(pid int) (bool, error) { - return false, nil - } - -- return true, nil -+ cmdline, err := utils.GetProcessCmdline(pid) -+ if err != nil { -+ return false, nil -+ } -+ -+ // If process's cmdline contains kata-shim and containerID keyword, we think this process pid isn't be reused -+ if strings.Contains(cmdline, "kata-shim") && strings.Contains(cmdline, containerID) { -+ return true, nil -+ } -+ -+ shimLogger().Errorf("%d process isn't a kata-shim process", pid) -+ return false, nil - } - - // waitForShim waits for the end of the shim unless it reaches the timeout - // first, returning an error in that case. --func waitForShim(pid int) error { -+func waitForShim(pid int, containerID string) error { - if pid <= 0 { - return nil - } - - tInit := time.Now() - for { -- running, err := isShimRunning(pid) -+ running, err := isShimRunning(pid, containerID) - if err != nil { - return err - } -diff --git a/virtcontainers/shim_test.go b/virtcontainers/shim_test.go -index e9bd027c..62471311 100644 ---- a/virtcontainers/shim_test.go -+++ b/virtcontainers/shim_test.go -@@ -190,7 +190,7 @@ func TestStopShimSuccessfulProcessRunning(t *testing.T) { - - func testIsShimRunning(t *testing.T, pid int, expected bool) { - assert := assert.New(t) -- running, err := isShimRunning(pid) -+ running, err := isShimRunning(pid, containerID) - assert.NoError(err) - assert.Equal(running, expected) - } -@@ -205,7 +205,7 @@ func TestIsShimRunningTrue(t *testing.T) { - cmd := testRunSleep999AndGetCmd(t) - assert := assert.New(t) - -- testIsShimRunning(t, cmd.Process.Pid, true) -+ testIsShimRunning(t, cmd.Process.Pid, false) - - err := syscall.Kill(cmd.Process.Pid, syscall.SIGKILL) - assert.NoError(err) -@@ -216,7 +216,7 @@ func TestWaitForShimInvalidPidSuccessful(t *testing.T) { - assert := assert.New(t) - - for _, val := range wrongValuesList { -- err := waitForShim(val) -+ err := waitForShim(val, containerID) - assert.NoError(err) - } - } -@@ -224,7 +224,7 @@ func TestWaitForShimInvalidPidSuccessful(t *testing.T) { - func TestWaitForShimNotRunningSuccessful(t *testing.T) { - pid := testRunSleep0AndGetPid(t) - assert := assert.New(t) -- assert.NoError(waitForShim(pid)) -+ assert.NoError(waitForShim(pid, containerID)) - } - - func TestWaitForShimRunningForTooLongFailure(t *testing.T) { -@@ -232,6 +232,6 @@ func TestWaitForShimRunningForTooLongFailure(t *testing.T) { - assert := assert.New(t) - - waitForShimTimeout = 0.1 -- assert.Error(waitForShim(cmd.Process.Pid)) -+ assert.NoError(waitForShim(cmd.Process.Pid, containerID)) - assert.NoError(syscall.Kill(cmd.Process.Pid, syscall.SIGKILL)) - } --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0011-kata-runtime-check-the-process-info-before-send-SIGK.patch b/runtime/patches/0011-kata-runtime-check-the-process-info-before-send-SIGK.patch deleted file mode 100644 index 6a021be..0000000 --- a/runtime/patches/0011-kata-runtime-check-the-process-info-before-send-SIGK.patch +++ /dev/null @@ -1,119 +0,0 @@ -From 0aeff2632eac58eefdc8ae438891303332831ec5 Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Tue, 28 Jul 2020 20:48:24 +0800 -Subject: [PATCH 11/50] kata-runtime: check the process info before send - SIGKILL - -reason: In order to avoid the pid reuse problem, check the -process info before send SIGKILL signal to process. - -Signed-off-by: jiangpengfei ---- - virtcontainers/kata_proxy.go | 18 ++++++++++++++++++ - virtcontainers/qemu.go | 5 +++++ - virtcontainers/shim.go | 9 +++++++++ - virtcontainers/shim_test.go | 8 ++++---- - 4 files changed, 36 insertions(+), 4 deletions(-) - -diff --git a/virtcontainers/kata_proxy.go b/virtcontainers/kata_proxy.go -index e04b4cff..ed272bad 100644 ---- a/virtcontainers/kata_proxy.go -+++ b/virtcontainers/kata_proxy.go -@@ -6,8 +6,12 @@ - package virtcontainers - - import ( -+ "fmt" - "os/exec" -+ "strings" - "syscall" -+ -+ "github.com/kata-containers/runtime/virtcontainers/utils" - ) - - // This is the Kata Containers implementation of the proxy interface. -@@ -61,6 +65,20 @@ func (p *kataProxy) start(params proxyParams) (int, string, error) { - - // stop is kataProxy stop implementation for proxy interface. - func (p *kataProxy) stop(pid int) error { -+ if pid <= 1 { -+ return nil -+ } -+ -+ // check process info before send SIGKILL signal -+ cmdline, err := utils.GetProcessCmdline(pid) -+ if err != nil { -+ return fmt.Errorf("get kata-proxy %d cmdline error: %v", pid, err) -+ } -+ -+ if !strings.Contains(cmdline, KataProxyProcessName) { -+ return fmt.Errorf("%d is not kata-proxy process, don't kill wrong process", pid) -+ } -+ - // Signal the proxy with SIGTERM. - return syscall.Kill(pid, syscall.SIGTERM) - } -diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go -index 4b15d968..4789101d 100644 ---- a/virtcontainers/qemu.go -+++ b/virtcontainers/qemu.go -@@ -967,6 +967,11 @@ func (q *qemu) stopSandbox(force bool) error { - return fmt.Errorf("force kill qemu process pid is invalid") - } - -+ cmdline, _ := utils.GetProcessCmdline(qemuMainPid) -+ if !strings.Contains(cmdline, string(QemuHypervisor)) { -+ return fmt.Errorf("force kill %d process is not qemu process, don't kill wrong process", qemuMainPid) -+ } -+ - _ = syscall.Kill(qemuMainPid, syscall.SIGKILL) - } - -diff --git a/virtcontainers/shim.go b/virtcontainers/shim.go -index 6f784a03..b192b258 100644 ---- a/virtcontainers/shim.go -+++ b/virtcontainers/shim.go -@@ -143,6 +143,15 @@ func stopShim(pid int) error { - return nil - } - -+ cmdline, err := utils.GetProcessCmdline(pid) -+ if err != nil { -+ return err -+ } -+ -+ if !strings.Contains(cmdline, "kata-shim") { -+ return fmt.Errorf("%d process is not kata-shim process, don't kill wrong process", pid) -+ } -+ - if err := signalShim(pid, syscall.SIGKILL); err != nil && err != syscall.ESRCH { - return err - } -diff --git a/virtcontainers/shim_test.go b/virtcontainers/shim_test.go -index 62471311..dc15eab0 100644 ---- a/virtcontainers/shim_test.go -+++ b/virtcontainers/shim_test.go -@@ -176,16 +176,16 @@ func testRunSleep999AndGetCmd(t *testing.T) *exec.Cmd { - return cmd - } - --func TestStopShimSuccessfulProcessNotRunning(t *testing.T) { -+func TestStopShimFailProcessNotRunning(t *testing.T) { - assert := assert.New(t) - pid := testRunSleep0AndGetPid(t) -- assert.NoError(stopShim(pid)) -+ assert.Error(stopShim(pid)) - } - --func TestStopShimSuccessfulProcessRunning(t *testing.T) { -+func TestStopShimFailProcessRunning(t *testing.T) { - assert := assert.New(t) - cmd := testRunSleep999AndGetCmd(t) -- assert.NoError(stopShim(cmd.Process.Pid)) -+ assert.Error(stopShim(cmd.Process.Pid)) - } - - func testIsShimRunning(t *testing.T, pid int, expected bool) { --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0012-kata-runtime-truncate-the-log.json-file-before-kata-.patch b/runtime/patches/0012-kata-runtime-truncate-the-log.json-file-before-kata-.patch deleted file mode 100644 index 5fc9679..0000000 --- a/runtime/patches/0012-kata-runtime-truncate-the-log.json-file-before-kata-.patch +++ /dev/null @@ -1,37 +0,0 @@ -From 441d80f55f4dc5efb4c92d91608a3c8db3d087cb Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Tue, 28 Jul 2020 21:43:15 +0800 -Subject: [PATCH 12/50] kata-runtime: truncate the log.json file before - kata-runtime subcommand executed - -reason: since we have redirect the kata-runtime log to /var/log/messages, and avoid the -path of log.json file to be large in the tmpfs, so we truncate the log.json file every -time before subcommand is executed. - -Signed-off-by: jiangpengfei ---- - cli/main.go | 8 ++++++++ - 1 file changed, 8 insertions(+) - -diff --git a/cli/main.go b/cli/main.go -index 1362a8fb..5eb2fb19 100644 ---- a/cli/main.go -+++ b/cli/main.go -@@ -300,6 +300,14 @@ func beforeSubcommands(c *cli.Context) error { - ignoreConfigLogs = true - } else { - if path := c.GlobalString("log"); path != "" { -+ // since we have redirect the kata-runtime log to /var/log/messages, and avoid the -+ // path of log.json file to be large in the tmpfs, so we truncate the log.json file -+ // every time before subcommand is executed. -+ if path != "/dev/null" && katautils.FileExists(path) { -+ if err := os.Truncate(path, 0); err != nil { -+ return err -+ } -+ } - f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND|os.O_SYNC, 0640) - if err != nil { - return err --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0013-kata-runtime-get-container-info-by-containerID-prefi.patch b/runtime/patches/0013-kata-runtime-get-container-info-by-containerID-prefi.patch deleted file mode 100644 index a377bd8..0000000 --- a/runtime/patches/0013-kata-runtime-get-container-info-by-containerID-prefi.patch +++ /dev/null @@ -1,98 +0,0 @@ -From fd63d26a5b0542f35d61b0c19c80795f052b4518 Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Tue, 28 Jul 2020 22:05:44 +0800 -Subject: [PATCH 13/50] kata-runtime: get container info by containerID prefix - -reason: get container info by containerID prefix, so we just -need to input the prefix containerID when call kata-runtime -subcommand - -Signed-off-by: jiangpengfei ---- - cli/oci.go | 35 +++++++++++++++++++++++++++++++++++ - pkg/katautils/oci.go | 5 +++++ - 2 files changed, 40 insertions(+) - -diff --git a/cli/oci.go b/cli/oci.go -index 8ffac2df..bf962d03 100644 ---- a/cli/oci.go -+++ b/cli/oci.go -@@ -9,6 +9,7 @@ import ( - "bufio" - "context" - "fmt" -+ "io/ioutil" - "net" - "os" - "path/filepath" -@@ -25,6 +26,8 @@ const ( - // Filesystem type corresponding to CGROUP_SUPER_MAGIC as listed - // here: http://man7.org/linux/man-pages/man2/statfs.2.html - cgroupFsType = 0x27e0eb -+ -+ maxIDLength = 64 - ) - - var cgroupsDirPath string -@@ -38,6 +41,14 @@ func getContainerInfo(ctx context.Context, containerID string) (vc.ContainerStat - return vc.ContainerStatus{}, "", fmt.Errorf("Missing container ID") - } - -+ if len(containerID) < maxIDLength { -+ fullContainerID, err := getContainerIDbyPrefix(containerID) -+ if err != nil { -+ return vc.ContainerStatus{}, "", err -+ } -+ containerID = fullContainerID -+ } -+ - sandboxID, err := katautils.FetchContainerIDMapping(containerID) - if err != nil { - return vc.ContainerStatus{}, "", err -@@ -211,3 +222,27 @@ func getCgroupsDirPath(mountInfoFile string) (string, error) { - - return cgroupRootPath, nil - } -+ -+func getContainerIDbyPrefix(prefix string) (string, error) { -+ files, err := ioutil.ReadDir(katautils.GetCtrsMapTreePath()) -+ if err != nil { -+ return "", err -+ } -+ -+ containers := []string{} -+ for _, file := range files { -+ if file.IsDir() && strings.HasPrefix(file.Name(), prefix) { -+ containers = append(containers, file.Name()) -+ } -+ } -+ -+ if len(containers) == 0 { -+ return "", fmt.Errorf("no such container ID (%v)", prefix) -+ } -+ -+ if len(containers) > 1 { -+ return "", fmt.Errorf("multiple containers found (%v)", prefix) -+ } -+ -+ return containers[0], nil -+} -diff --git a/pkg/katautils/oci.go b/pkg/katautils/oci.go -index 6de8101e..1334af35 100644 ---- a/pkg/katautils/oci.go -+++ b/pkg/katautils/oci.go -@@ -25,6 +25,11 @@ func SetCtrsMapTreePath(path string) { - ctrsMapTreePath = path - } - -+// GetCtrsMapTreePath return the containerID to SandboxID mapping dir -+func GetCtrsMapTreePath() string { -+ return ctrsMapTreePath -+} -+ - // doUpdatePath returns whether a ctrsMapTreePath needs to be updated with a rootless prefix - func doUpdatePath() bool { - return rootless.IsRootless() && !strings.HasPrefix(ctrsMapTreePath, rootless.GetRootlessDir()) --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0014-kata-runtime-add-self-defined-annotations-framework.patch b/runtime/patches/0014-kata-runtime-add-self-defined-annotations-framework.patch deleted file mode 100644 index 3dc1a5f..0000000 --- a/runtime/patches/0014-kata-runtime-add-self-defined-annotations-framework.patch +++ /dev/null @@ -1,76 +0,0 @@ -From 1bd3cb85a1cf0e94b3280412d5fb47cecc4721fd Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Wed, 29 Jul 2020 10:42:49 +0800 -Subject: [PATCH 14/50] kata-runtime: add self defined annotations framework - -reason: add self defined annotations framework to pass some -self defined resource for sandbox - -Signed-off-by: jiangpengfei ---- - virtcontainers/pkg/oci/utils.go | 38 ++++++++++++++++++++++++++++++++++++++ - 1 file changed, 38 insertions(+) - -diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go -index cd8d48ce..05181efd 100644 ---- a/virtcontainers/pkg/oci/utils.go -+++ b/virtcontainers/pkg/oci/utils.go -@@ -33,6 +33,10 @@ type annotationContainerType struct { - containerType vc.ContainerType - } - -+type annotationHandler func(value string) error -+ -+var annotationHandlerList = map[string]annotationHandler{} -+ - var ( - // ErrNoLinux is an error for missing Linux sections in the OCI configuration file. - ErrNoLinux = errors.New("missing Linux section") -@@ -342,6 +346,10 @@ func addAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) error { - if err := addAgentConfigOverrides(ocispec, config); err != nil { - return err - } -+ -+ if err := addOtherSandboxAnnotation(ocispec, config); err != nil { -+ return err -+ } - return nil - } - -@@ -1016,3 +1024,33 @@ func GetOCIConfig(status vc.ContainerStatus) (specs.Spec, error) { - - return *status.Spec, nil - } -+ -+// validateOtherSandboxAnnotations validate the value of annotation -+func validateOtherSandboxAnnotations(annotation, value string) error { -+ validateHandler, ok := annotationHandlerList[annotation] -+ if !ok { -+ return fmt.Errorf("unsupport Sandbox annotation type") -+ } -+ -+ return validateHandler(value) -+} -+ -+// addOtherSandboxAnnotation add self defined annotation for sandbox -+func addOtherSandboxAnnotation(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error { -+ otherSandboxAnnotationsKey := []string{} -+ -+ for _, a := range otherSandboxAnnotationsKey { -+ value, ok := ocispec.Annotations[a] -+ if !ok { -+ continue -+ } -+ -+ if err := validateOtherSandboxAnnotations(a, value); err != nil { -+ return err -+ } -+ -+ sbConfig.Annotations[a] = value -+ } -+ -+ return nil -+} --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0015-kata-runtime-add-reuse-hypervisor-cpu-and-memory-fea.patch b/runtime/patches/0015-kata-runtime-add-reuse-hypervisor-cpu-and-memory-fea.patch deleted file mode 100644 index 5624a46..0000000 --- a/runtime/patches/0015-kata-runtime-add-reuse-hypervisor-cpu-and-memory-fea.patch +++ /dev/null @@ -1,143 +0,0 @@ -From c0a33c4584e1fbd9b39ff1ca3ed632efe85de65e Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Sun, 2 Aug 2020 15:51:10 +0800 -Subject: [PATCH 15/50] kata-runtime: add reuse hypervisor cpu and memory - feature - -reason: If default hypervisor cpu and memory is set too large, -which may waste resource, so we add enable_reuse_cpu_memory -config in the configuration.toml file to choose share the -hypervisor cpu and memory with container or not. - -Signed-off-by: jiangpengfei ---- - cli/config/configuration-qemu.toml.in | 5 +++++ - pkg/katautils/config.go | 2 ++ - virtcontainers/hypervisor.go | 3 +++ - virtcontainers/persist.go | 2 ++ - virtcontainers/persist/api/config.go | 3 +++ - virtcontainers/sandbox.go | 22 ++++++++++++++++++---- - 6 files changed, 33 insertions(+), 4 deletions(-) - -diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in -index 46ce7d9b..82461732 100644 ---- a/cli/config/configuration-qemu.toml.in -+++ b/cli/config/configuration-qemu.toml.in -@@ -95,6 +95,11 @@ default_memory = @DEFMEMSZ@ - # Default false - #enable_virtio_mem = true - -+# Specifies share hypervisor default cpu and memory resource -+# between hypervisor and container process. -+# Default false -+#enable_reuse_cpu_memory = false -+ - # Disable block device from being used for a container's rootfs. - # In case of a storage driver like devicemapper where a container's - # root file system is backed by a block device, the block device is passed -diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go -index 448d23ac..d1883c94 100644 ---- a/pkg/katautils/config.go -+++ b/pkg/katautils/config.go -@@ -112,6 +112,7 @@ type hypervisor struct { - MemorySize uint32 `toml:"default_memory"` - MemSlots uint32 `toml:"memory_slots"` - MemOffset uint32 `toml:"memory_offset"` -+ EnableCPUMemoryReuse bool `toml:"enable_reuse_cpu_memory"` - DefaultBridges uint32 `toml:"default_bridges"` - Msize9p uint32 `toml:"msize_9p"` - PCIeRootPort uint32 `toml:"pcie_root_port"` -@@ -640,6 +641,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { - MemorySize: h.defaultMemSz(), - MemSlots: h.defaultMemSlots(), - MemOffset: h.defaultMemOffset(), -+ EnableCPUMemoryReuse: h.EnableCPUMemoryReuse, - VirtioMem: h.VirtioMem, - EntropySource: h.GetEntropySource(), - DefaultBridges: h.defaultBridges(), -diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go -index fd7d1f8e..5f8d24f9 100644 ---- a/virtcontainers/hypervisor.go -+++ b/virtcontainers/hypervisor.go -@@ -250,6 +250,9 @@ type HypervisorConfig struct { - // MemOffset specifies memory space for nvdimm device - MemOffset uint32 - -+ // Enable hypervisor cpu and memory resource share with container -+ EnableCPUMemoryReuse bool -+ - // VirtioFSCacheSize is the DAX cache size in MiB - VirtioFSCacheSize uint32 - -diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go -index d96a3890..aef4d18d 100644 ---- a/virtcontainers/persist.go -+++ b/virtcontainers/persist.go -@@ -214,6 +214,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) { - Msize9p: sconfig.HypervisorConfig.Msize9p, - MemSlots: sconfig.HypervisorConfig.MemSlots, - MemOffset: sconfig.HypervisorConfig.MemOffset, -+ EnableCPUMemoryReuse: sconfig.HypervisorConfig.EnableCPUMemoryReuse, - VirtioMem: sconfig.HypervisorConfig.VirtioMem, - VirtioFSCacheSize: sconfig.HypervisorConfig.VirtioFSCacheSize, - KernelPath: sconfig.HypervisorConfig.KernelPath, -@@ -503,6 +504,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) { - Msize9p: hconf.Msize9p, - MemSlots: hconf.MemSlots, - MemOffset: hconf.MemOffset, -+ EnableCPUMemoryReuse: hconf.EnableCPUMemoryReuse, - VirtioMem: hconf.VirtioMem, - VirtioFSCacheSize: hconf.VirtioFSCacheSize, - KernelPath: hconf.KernelPath, -diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go -index 34a5fd0f..a3c6ec91 100644 ---- a/virtcontainers/persist/api/config.go -+++ b/virtcontainers/persist/api/config.go -@@ -35,6 +35,9 @@ type HypervisorConfig struct { - // MemOffset specifies memory space for nvdimm device - MemOffset uint32 - -+ // Enable hypervisor cpu and memory resource share with container -+ EnableCPUMemoryReuse bool -+ - // VirtioFSCacheSize is the DAX cache size in MiB - VirtioFSCacheSize uint32 - -diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go -index 78188ed7..e766d1f7 100644 ---- a/virtcontainers/sandbox.go -+++ b/virtcontainers/sandbox.go -@@ -1833,12 +1833,26 @@ func (s *Sandbox) updateResources() error { - } - - sandboxVCPUs := s.calculateSandboxCPUs() -- // Add default vcpus for sandbox -- sandboxVCPUs += s.hypervisor.hypervisorConfig().NumVCPUs -+ // Share the hypervisor vcpu with container process -+ if s.config.HypervisorConfig.EnableCPUMemoryReuse { -+ if sandboxVCPUs <= s.config.HypervisorConfig.NumVCPUs { -+ sandboxVCPUs = s.config.HypervisorConfig.NumVCPUs -+ } -+ } else { -+ // Add default vcpus for sandbox -+ sandboxVCPUs += s.hypervisor.hypervisorConfig().NumVCPUs -+ } - - sandboxMemoryByte := s.calculateSandboxMemory() -- // Add default / rsvd memory for sandbox. -- sandboxMemoryByte += int64(s.hypervisor.hypervisorConfig().MemorySize) << utils.MibToBytesShift -+ // Share the hypervisor memory with container process -+ if s.config.HypervisorConfig.EnableCPUMemoryReuse { -+ if sandboxMemoryByte <= (int64(s.config.HypervisorConfig.MemorySize) << utils.MibToBytesShift) { -+ sandboxMemoryByte = int64(s.config.HypervisorConfig.MemorySize) << utils.MibToBytesShift -+ } -+ } else { -+ // Add default / rsvd memory for sandbox. -+ sandboxMemoryByte += int64(s.hypervisor.hypervisorConfig().MemorySize) << utils.MibToBytesShift -+ } - - // Update VCPUs - s.Logger().WithField("cpus-sandbox", sandboxVCPUs).Debugf("Request to hypervisor to update vCPUs") --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0016-virtcontainers-fix-hotplug-huge-size-memory-cause-ag.patch b/runtime/patches/0016-virtcontainers-fix-hotplug-huge-size-memory-cause-ag.patch deleted file mode 100644 index 3d12a3f..0000000 --- a/runtime/patches/0016-virtcontainers-fix-hotplug-huge-size-memory-cause-ag.patch +++ /dev/null @@ -1,288 +0,0 @@ -From dc9de8bb181e2cec2f3e0a76d02833fef45b46af Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Thu, 6 Aug 2020 09:28:34 -0400 -Subject: [PATCH 16/50] virtcontainers: fix hotplug huge size memory cause - agent hang bug - -fixes: #2872 - -reason: If hotplug huge size memory into kata VM at once time, -guest kernel will allocate some extra memory for memory management, -which may cause kata-agent hang and out of responding. -And hotplug more memory into VM, more extra memory is needed. - -Inorder to solve this problem, we divide hotplug huge memory into -two steps. First, hotplug the max allowed memory into VM and wait -all first step hotplugged memory online. Second Step, hotplug the -left memory into VM. - -Signed-off-by: jiangpengfei ---- - virtcontainers/acrn.go | 4 ++++ - virtcontainers/agent.go | 3 ++- - virtcontainers/clh.go | 4 ++++ - virtcontainers/fc.go | 4 ++++ - virtcontainers/hypervisor.go | 3 +++ - virtcontainers/kata_agent.go | 4 ++-- - virtcontainers/kata_agent_test.go | 2 +- - virtcontainers/mock_hypervisor.go | 4 ++++ - virtcontainers/noop_agent.go | 2 +- - virtcontainers/qemu.go | 4 ++++ - virtcontainers/sandbox.go | 30 ++++++++++++++++++++++++++++-- - virtcontainers/sandbox_test.go | 12 ++++++++++++ - virtcontainers/utils/utils.go | 3 +++ - virtcontainers/vm.go | 2 +- - 14 files changed, 73 insertions(+), 8 deletions(-) - -diff --git a/virtcontainers/acrn.go b/virtcontainers/acrn.go -index 10cae06f..c9a0fe0b 100644 ---- a/virtcontainers/acrn.go -+++ b/virtcontainers/acrn.go -@@ -811,3 +811,7 @@ func (a *Acrn) loadInfo() error { - } - return nil - } -+ -+func (a *Acrn) getMemorySize() uint32 { -+ return a.config.MemorySize -+} -diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go -index be9526c7..b1dea816 100644 ---- a/virtcontainers/agent.go -+++ b/virtcontainers/agent.go -@@ -201,7 +201,8 @@ type agent interface { - // This function should be called after hot adding vCPUs or Memory. - // cpus specifies the number of CPUs that were added and the agent should online - // cpuOnly specifies that we should online cpu or online memory or both -- onlineCPUMem(cpus uint32, cpuOnly bool) error -+ // wait specifies that we should wait all cpu or memory online in the VM synchronously -+ onlineCPUMem(cpus uint32, cpuOnly bool, wait bool) error - - // memHotplugByProbe will notify the guest kernel about memory hotplug event through - // probe interface. -diff --git a/virtcontainers/clh.go b/virtcontainers/clh.go -index 59510b02..8afcd4bf 100644 ---- a/virtcontainers/clh.go -+++ b/virtcontainers/clh.go -@@ -1210,3 +1210,7 @@ func (clh *cloudHypervisor) vmInfo() (chclient.VmInfo, error) { - return info, openAPIClientError(err) - - } -+ -+func (clh *cloudHypervisor) getMemorySize() uint32 { -+ return clh.config.MemorySize -+} -diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go -index 72a8e192..15726156 100644 ---- a/virtcontainers/fc.go -+++ b/virtcontainers/fc.go -@@ -1212,3 +1212,7 @@ func (fc *firecracker) watchConsole() (*os.File, error) { - - return stdio, nil - } -+ -+func (fc *firecracker) getMemorySize() uint32 { -+ return fc.config.MemorySize -+} -diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go -index 5f8d24f9..9cd685ad 100644 ---- a/virtcontainers/hypervisor.go -+++ b/virtcontainers/hypervisor.go -@@ -778,6 +778,9 @@ type hypervisor interface { - hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) - resizeMemory(memMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error) - resizeVCPUs(vcpus uint32) (uint32, uint32, error) -+ // getMemorySize return the total memory in the guest include default memory size + hot plugged memory -+ // return memory size unit is MB -+ getMemorySize() uint32 - getSandboxConsole(sandboxID string) (string, error) - disconnect() - capabilities() types.Capabilities -diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go -index 7575d326..8e073339 100644 ---- a/virtcontainers/kata_agent.go -+++ b/virtcontainers/kata_agent.go -@@ -1806,9 +1806,9 @@ func (k *kataAgent) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionS - return err - } - --func (k *kataAgent) onlineCPUMem(cpus uint32, cpuOnly bool) error { -+func (k *kataAgent) onlineCPUMem(cpus uint32, cpuOnly bool, wait bool) error { - req := &grpc.OnlineCPUMemRequest{ -- Wait: false, -+ Wait: wait, - NbCpus: cpus, - CpuOnly: cpuOnly, - } -diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go -index 62d31c93..2a2ddada 100644 ---- a/virtcontainers/kata_agent_test.go -+++ b/virtcontainers/kata_agent_test.go -@@ -324,7 +324,7 @@ func TestKataAgentSendReq(t *testing.T) { - err = k.resumeContainer(sandbox, Container{}) - assert.Nil(err) - -- err = k.onlineCPUMem(1, true) -+ err = k.onlineCPUMem(1, true, false) - assert.Nil(err) - - _, err = k.statsContainer(sandbox, Container{}) -diff --git a/virtcontainers/mock_hypervisor.go b/virtcontainers/mock_hypervisor.go -index a5b67491..f1c6106d 100644 ---- a/virtcontainers/mock_hypervisor.go -+++ b/virtcontainers/mock_hypervisor.go -@@ -128,3 +128,7 @@ func (m *mockHypervisor) check() error { - func (m *mockHypervisor) generateSocket(id string, useVsock bool) (interface{}, error) { - return types.Socket{HostPath: "/tmp/socket", Name: "socket"}, nil - } -+ -+func (m *mockHypervisor) getMemorySize() uint32 { -+ return 0 -+} -diff --git a/virtcontainers/noop_agent.go b/virtcontainers/noop_agent.go -index 8a7cd337..6e211bca 100644 ---- a/virtcontainers/noop_agent.go -+++ b/virtcontainers/noop_agent.go -@@ -102,7 +102,7 @@ func (n *noopAgent) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionS - } - - // onlineCPUMem is the Noop agent Container online CPU and Memory implementation. It does nothing. --func (n *noopAgent) onlineCPUMem(cpus uint32, cpuOnly bool) error { -+func (n *noopAgent) onlineCPUMem(cpus uint32, cpuOnly bool, wait bool) error { - return nil - } - -diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go -index 4789101d..7bae3278 100644 ---- a/virtcontainers/qemu.go -+++ b/virtcontainers/qemu.go -@@ -2273,3 +2273,7 @@ func (q *qemu) check() error { - func (q *qemu) generateSocket(id string, useVsock bool) (interface{}, error) { - return generateVMSocket(id, useVsock, q.store.RunVMStoragePath()) - } -+ -+func (q *qemu) getMemorySize() uint32 { -+ return q.config.MemorySize + uint32(q.state.HotpluggedMemory) -+} -diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go -index e766d1f7..a318d677 100644 ---- a/virtcontainers/sandbox.go -+++ b/virtcontainers/sandbox.go -@@ -1864,7 +1864,7 @@ func (s *Sandbox) updateResources() error { - // If the CPUs were increased, ask agent to online them - if oldCPUs < newCPUs { - vcpusAdded := newCPUs - oldCPUs -- if err := s.agent.onlineCPUMem(vcpusAdded, true); err != nil { -+ if err := s.agent.onlineCPUMem(vcpusAdded, true, false); err != nil { - return err - } - } -@@ -1872,6 +1872,20 @@ func (s *Sandbox) updateResources() error { - - // Update Memory - s.Logger().WithField("memory-sandbox-size-byte", sandboxMemoryByte).Debugf("Request to hypervisor to update memory") -+ reqMemMB := uint32(sandboxMemoryByte >> utils.MibToBytesShift) -+ currentMemMB := s.hypervisor.getMemorySize() -+ -+ // If request hotplug memory size larger than utils.MaxHotplugMemMBOnceTime, -+ // inorder to avoid hotplug memory oom problem, we need to hotplug large memory -+ // with two steps. First, hotplug utils.MaxHotplugMemMBOnceTime size memory into -+ // guest and wait all hotplug memory online. Then, hotplug the left unplugged memory -+ // into the guest -+ if currentMemMB < reqMemMB && (reqMemMB-currentMemMB) > utils.MaxHotplugMemMBOnceTime { -+ if err := s.beforeHotplugHugeMem(currentMemMB); err != nil { -+ return err -+ } -+ } -+ - newMemory, updatedMemoryDevice, err := s.hypervisor.resizeMemory(uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe) - if err != nil { - return err -@@ -1884,7 +1898,7 @@ func (s *Sandbox) updateResources() error { - return err - } - } -- if err := s.agent.onlineCPUMem(0, false); err != nil { -+ if err := s.agent.onlineCPUMem(0, false, false); err != nil { - return err - } - return nil -@@ -1926,6 +1940,18 @@ func (s *Sandbox) calculateSandboxCPUs() uint32 { - return utils.CalculateVCpusFromMilliCpus(mCPU) - } - -+func (s *Sandbox) beforeHotplugHugeMem(currentMemSizeInMB uint32) error { -+ wantedTotalMemSize := currentMemSizeInMB + utils.MaxHotplugMemMBOnceTime -+ newMemory, _, err := s.hypervisor.resizeMemory(wantedTotalMemSize, s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe) -+ if err != nil { -+ return err -+ } -+ -+ s.Logger().Debugf("first part hotplug memory size: %d MB", newMemory) -+ // wait all first part hotplugged memory online in the guest -+ return s.agent.onlineCPUMem(0, false, true) -+} -+ - // GetHypervisorType is used for getting Hypervisor name currently used. - // Sandbox implement DeviceReceiver interface from device/api/interface.go - func (s *Sandbox) GetHypervisorType() string { -diff --git a/virtcontainers/sandbox_test.go b/virtcontainers/sandbox_test.go -index 85c712e8..4b02b3f3 100644 ---- a/virtcontainers/sandbox_test.go -+++ b/virtcontainers/sandbox_test.go -@@ -25,6 +25,7 @@ import ( - "github.com/kata-containers/runtime/virtcontainers/persist/fs" - "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" - "github.com/kata-containers/runtime/virtcontainers/types" -+ "github.com/kata-containers/runtime/virtcontainers/utils" - specs "github.com/opencontainers/runtime-spec/specs-go" - "github.com/stretchr/testify/assert" - "golang.org/x/sys/unix" -@@ -1522,6 +1523,17 @@ func TestSandboxUpdateResources(t *testing.T) { - } - err = s.updateResources() - assert.NoError(t, err) -+ -+ // add a container with huge memory equal utils.MaxHotplugMemMBOnceTime -+ contConfig3 := newTestContainerConfigNoop("cont-00003") -+ contConfig3.Resources.Memory = &specs.LinuxMemory{ -+ Limit: new(int64), -+ } -+ container3MemLimitInBytes := int64(utils.MaxHotplugMemMBOnceTime << utils.MibToBytesShift) -+ contConfig3.Resources.Memory.Limit = &container3MemLimitInBytes -+ s.config.Containers = append(s.config.Containers, contConfig3) -+ err = s.updateResources() -+ assert.NoError(t, err) - } - - func TestSandboxExperimentalFeature(t *testing.T) { -diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go -index 2b555ebb..3ae95aef 100644 ---- a/virtcontainers/utils/utils.go -+++ b/virtcontainers/utils/utils.go -@@ -25,6 +25,9 @@ const fileMode0755 = os.FileMode(0755) - // MibToBytesShift the number to shift needed to convert MiB to Bytes - const MibToBytesShift = 20 - -+// Max Hotplug Memory size at once time, unit is MB -+const MaxHotplugMemMBOnceTime = 32 * 1024 -+ - // MaxSocketPathLen is the effective maximum Unix domain socket length. - // - // See unix(7). -diff --git a/virtcontainers/vm.go b/virtcontainers/vm.go -index 8d27b1fe..2e5fef44 100644 ---- a/virtcontainers/vm.go -+++ b/virtcontainers/vm.go -@@ -370,7 +370,7 @@ func (v *VM) AddMemory(numMB uint32) error { - // OnlineCPUMemory puts the hotplugged CPU and memory online. - func (v *VM) OnlineCPUMemory() error { - v.logger().Infof("online CPU %d and memory", v.cpuDelta) -- err := v.agent.onlineCPUMem(v.cpuDelta, false) -+ err := v.agent.onlineCPUMem(v.cpuDelta, false, false) - if err == nil { - v.cpuDelta = 0 - } --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0017-kata-runtime-validate-sandbox-cpu-and-memory-size.patch b/runtime/patches/0017-kata-runtime-validate-sandbox-cpu-and-memory-size.patch deleted file mode 100644 index 62b2847..0000000 --- a/runtime/patches/0017-kata-runtime-validate-sandbox-cpu-and-memory-size.patch +++ /dev/null @@ -1,355 +0,0 @@ -From e6051eb1a8c54b91c46f68eab9a63ad0e73a9221 Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Fri, 7 Aug 2020 20:01:15 +0800 -Subject: [PATCH 17/50] kata-runtime: validate sandbox cpu and memory size - -reason: -1. add more strict sandbox cpu and memory size check. -2. use GetPhysicalCPUNumber func to replace goruntime.NumCPU(), -because goruntime.NumCPU() return the CPU number of cpuset cgroup -that current process joined, however kata-runtime process will -change the cpuset cgroup when setupSanboxCgroup is called, which -may cause goruntime.NumCPU() return value changed. - -Signed-off-by: jiangpengfei ---- - pkg/katautils/config.go | 39 ++++++++++++++++++++++++++++++++++--- - pkg/katautils/config_test.go | 11 ++++++++--- - virtcontainers/pkg/oci/utils.go | 11 +++++------ - virtcontainers/qemu_arm64.go | 4 ++-- - virtcontainers/qemu_arm64_test.go | 4 ++-- - virtcontainers/sandbox.go | 7 ++++++- - virtcontainers/utils/utils.go | 41 +++++++++++++++++++++++++++++++++++++++ - 7 files changed, 100 insertions(+), 17 deletions(-) - -diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go -index d1883c94..b9b08469 100644 ---- a/pkg/katautils/config.go -+++ b/pkg/katautils/config.go -@@ -11,7 +11,6 @@ import ( - "fmt" - "io/ioutil" - "path/filepath" -- goruntime "runtime" - "strings" - - "github.com/BurntSushi/toml" -@@ -285,7 +284,7 @@ func (h hypervisor) GetEntropySource() string { - } - - func (h hypervisor) defaultVCPUs() uint32 { -- numCPUs := goruntime.NumCPU() -+ numCPUs := utils.GetPhysicalCPUNumber() - - if h.NumVCPUs < 0 || h.NumVCPUs > int32(numCPUs) { - return uint32(numCPUs) -@@ -297,8 +296,22 @@ func (h hypervisor) defaultVCPUs() uint32 { - return uint32(h.NumVCPUs) - } - -+func (h hypervisor) checkVCPUs() error { -+ numCPUs := utils.GetPhysicalCPUNumber() -+ -+ if h.NumVCPUs <= 0 { -+ return fmt.Errorf("invalid vcpus in configuration.toml! vcpus must larger than 0") -+ } -+ -+ if h.NumVCPUs > int32(numCPUs) { -+ return fmt.Errorf("invalid vcpus in configuration.toml! vcpus must smaller than max CPUs: %d in machine", numCPUs) -+ } -+ -+ return nil -+} -+ - func (h hypervisor) defaultMaxVCPUs() uint32 { -- numcpus := uint32(goruntime.NumCPU()) -+ numcpus := uint32(utils.GetPhysicalCPUNumber()) - maxvcpus := vc.MaxQemuVCPUs() - reqVCPUs := h.DefaultMaxVCPUs - -@@ -324,6 +337,18 @@ func (h hypervisor) defaultMemSz() uint32 { - return h.MemorySize - } - -+func (h hypervisor) checkMemSz() error { -+ if h.MemorySize < utils.MinMemorySizeInMB { -+ return fmt.Errorf("invalid memory size! Memory size must larger than %d MB", utils.MinMemorySizeInMB) -+ } -+ -+ if h.MemorySize > utils.MaxMemorySizeInMB { -+ return fmt.Errorf("invalid memory size, memory size must smaller than %d MB", utils.MaxMemorySizeInMB) -+ } -+ -+ return nil -+} -+ - func (h hypervisor) defaultMemSlots() uint32 { - slots := h.MemSlots - if slots == 0 { -@@ -627,6 +652,14 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { - } - } - -+ if err = h.checkVCPUs(); err != nil { -+ return vc.HypervisorConfig{}, err -+ } -+ -+ if err = h.checkMemSz(); err != nil { -+ return vc.HypervisorConfig{}, err -+ } -+ - return vc.HypervisorConfig{ - HypervisorPath: hypervisor, - KernelPath: kernel, -diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go -index 31afcca6..37aa16d0 100644 ---- a/pkg/katautils/config_test.go -+++ b/pkg/katautils/config_test.go -@@ -14,7 +14,6 @@ import ( - "path" - "path/filepath" - "reflect" -- goruntime "runtime" - "strings" - "syscall" - "testing" -@@ -152,7 +151,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf - KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), - HypervisorMachineType: machineType, - NumVCPUs: defaultVCPUCount, -- DefaultMaxVCPUs: uint32(goruntime.NumCPU()), -+ DefaultMaxVCPUs: uint32(utils.GetPhysicalCPUNumber()), - MemorySize: defaultMemSize, - DisableBlockDeviceUse: disableBlockDevice, - BlockDeviceDriver: defaultBlockDeviceDriver, -@@ -727,6 +726,8 @@ func TestMinimalRuntimeConfigWithVsock(t *testing.T) { - [hypervisor.qemu] - use_vsock = true - image = "` + imagePath + `" -+ default_vcpus = 1 -+ default_memory = 1024 - - [proxy.kata] - path = "` + proxyPath + `" -@@ -786,6 +787,8 @@ func TestNewQemuHypervisorConfig(t *testing.T) { - utils.VHostVSockDevicePath = orgVHostVSockDevicePath - }() - utils.VHostVSockDevicePath = "/dev/abc/xyz" -+ defaultVCPUs := int32(1) -+ defaultMemory := uint32(1024) - - hypervisor := hypervisor{ - Path: hypervisorPath, -@@ -797,6 +800,8 @@ func TestNewQemuHypervisorConfig(t *testing.T) { - HotplugVFIOOnRootBus: hotplugVFIOOnRootBus, - PCIeRootPort: pcieRootPort, - UseVSock: true, -+ NumVCPUs: defaultVCPUs, -+ MemorySize: defaultMemory, - } - - files := []string{hypervisorPath, kernelPath, imagePath} -@@ -996,7 +1001,7 @@ func TestNewShimConfig(t *testing.T) { - func TestHypervisorDefaults(t *testing.T) { - assert := assert.New(t) - -- numCPUs := goruntime.NumCPU() -+ numCPUs := utils.GetPhysicalCPUNumber() - - h := hypervisor{} - -diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go -index 05181efd..0a6f08c7 100644 ---- a/virtcontainers/pkg/oci/utils.go -+++ b/virtcontainers/pkg/oci/utils.go -@@ -10,22 +10,21 @@ import ( - "errors" - "fmt" - "path/filepath" -- goruntime "runtime" - "strconv" - "strings" - "syscall" - - criContainerdAnnotations "github.com/containerd/cri-containerd/pkg/annotations" - crioAnnotations "github.com/cri-o/cri-o/pkg/annotations" -- specs "github.com/opencontainers/runtime-spec/specs-go" -- "github.com/sirupsen/logrus" -- - vc "github.com/kata-containers/runtime/virtcontainers" - "github.com/kata-containers/runtime/virtcontainers/device/config" - exp "github.com/kata-containers/runtime/virtcontainers/experimental" - vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" - dockershimAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations/dockershim" - "github.com/kata-containers/runtime/virtcontainers/types" -+ "github.com/kata-containers/runtime/virtcontainers/utils" -+ specs "github.com/opencontainers/runtime-spec/specs-go" -+ "github.com/sirupsen/logrus" - ) - - type annotationContainerType struct { -@@ -560,7 +559,7 @@ func addHypervisorCPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e - return fmt.Errorf("Error encountered parsing annotation default_vcpus: %v, please specify numeric value", err) - } - -- numCPUs := goruntime.NumCPU() -+ numCPUs := utils.GetPhysicalCPUNumber() - - if uint32(vcpus) > uint32(numCPUs) { - return fmt.Errorf("Number of cpus %d specified in annotation default_vcpus is greater than the number of CPUs %d on the system", vcpus, numCPUs) -@@ -575,7 +574,7 @@ func addHypervisorCPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e - return fmt.Errorf("Error encountered parsing annotation for default_maxvcpus: %v, please specify positive numeric value", err) - } - -- numCPUs := goruntime.NumCPU() -+ numCPUs := utils.GetPhysicalCPUNumber() - max := uint32(maxVCPUs) - - if max > uint32(numCPUs) { -diff --git a/virtcontainers/qemu_arm64.go b/virtcontainers/qemu_arm64.go -index 6d089cf0..40fb2a80 100644 ---- a/virtcontainers/qemu_arm64.go -+++ b/virtcontainers/qemu_arm64.go -@@ -8,11 +8,11 @@ package virtcontainers - import ( - "context" - "io/ioutil" -- "runtime" - "strings" - "time" - - govmmQemu "github.com/intel/govmm/qemu" -+ "github.com/kata-containers/runtime/virtcontainers/utils" - "github.com/sirupsen/logrus" - ) - -@@ -121,7 +121,7 @@ func MaxQemuVCPUs() uint32 { - if hostGICVersion != 0 { - return gicList[hostGICVersion] - } -- return uint32(runtime.NumCPU()) -+ return uint32(utils.GetPhysicalCPUNumber()) - } - - func newQemuArch(config HypervisorConfig) qemuArch { -diff --git a/virtcontainers/qemu_arm64_test.go b/virtcontainers/qemu_arm64_test.go -index 40351158..372fc4a9 100644 ---- a/virtcontainers/qemu_arm64_test.go -+++ b/virtcontainers/qemu_arm64_test.go -@@ -10,10 +10,10 @@ import ( - "io/ioutil" - "os" - "path/filepath" -- "runtime" - "testing" - - govmmQemu "github.com/intel/govmm/qemu" -+ "github.com/kata-containers/runtime/virtcontainers/utils" - "github.com/stretchr/testify/assert" - ) - -@@ -63,7 +63,7 @@ func TestMaxQemuVCPUs(t *testing.T) { - } - - data := []testData{ -- {"", uint32(runtime.NumCPU())}, -+ {"", uint32(utils.GetPhysicalCPUNumber())}, - {" 1: 0 0 GICv2 25 Level vgic \n", uint32(8)}, - {" 1: 0 0 GICv3 25 Level vgic \n", uint32(123)}, - {" 1: 0 0 GICv4 25 Level vgic \n", uint32(123)}, -diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go -index a318d677..a8522b96 100644 ---- a/virtcontainers/sandbox.go -+++ b/virtcontainers/sandbox.go -@@ -1875,6 +1875,11 @@ func (s *Sandbox) updateResources() error { - reqMemMB := uint32(sandboxMemoryByte >> utils.MibToBytesShift) - currentMemMB := s.hypervisor.getMemorySize() - -+ // check request sandbox memory size larger than utils.MaxMemorySizInMB or not -+ if reqMemMB > utils.MaxMemorySizeInMB { -+ return fmt.Errorf("updateResources failed! Sandbox memory should <= %d MiB", utils.MaxMemorySizeInMB) -+ } -+ - // If request hotplug memory size larger than utils.MaxHotplugMemMBOnceTime, - // inorder to avoid hotplug memory oom problem, we need to hotplug large memory - // with two steps. First, hotplug utils.MaxHotplugMemMBOnceTime size memory into -@@ -1898,7 +1903,7 @@ func (s *Sandbox) updateResources() error { - return err - } - } -- if err := s.agent.onlineCPUMem(0, false, false); err != nil { -+ if err := s.agent.onlineCPUMem(0, false, true); err != nil { - return err - } - return nil -diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go -index 3ae95aef..5d38e594 100644 ---- a/virtcontainers/utils/utils.go -+++ b/virtcontainers/utils/utils.go -@@ -6,6 +6,7 @@ - package utils - - import ( -+ "bufio" - "crypto/rand" - "errors" - "fmt" -@@ -28,11 +29,26 @@ const MibToBytesShift = 20 - // Max Hotplug Memory size at once time, unit is MB - const MaxHotplugMemMBOnceTime = 32 * 1024 - -+const ( -+ // Min needed memory size to start a Kata VM -+ MinMemorySizeInMB = 300 -+ MinMemorySizeInByte = MinMemorySizeInMB << MibToBytesShift -+ -+ // Max support memory size in the Kata VM -+ MaxMemorySizeInMB = 512 * 1024 -+ MaxMemorySizeInByte = MaxMemorySizeInMB << MibToBytesShift -+) -+ - // MaxSocketPathLen is the effective maximum Unix domain socket length. - // - // See unix(7). - const MaxSocketPathLen = 107 - -+const ( -+ procCPUInfoPath = "/proc/cpuinfo" -+ processorIdentifier = "processor" -+) -+ - // VHostVSockDevicePath path to vhost-vsock device - var VHostVSockDevicePath = "/dev/vhost-vsock" - -@@ -324,3 +340,28 @@ func IsProcessRunning(pid int, processName string, sandboxID string) bool { - - return false - } -+ -+// GetPhysicalCPUNumber return the number of the CPUs in the physical machine -+func GetPhysicalCPUNumber() int { -+ f, err := os.Open(procCPUInfoPath) -+ if err != nil { -+ return 0 -+ } -+ defer f.Close() -+ -+ cpuNum := 0 -+ s := bufio.NewScanner(f) -+ for s.Scan() { -+ if err := s.Err(); err != nil { -+ return 0 -+ } -+ -+ fields := strings.Fields(s.Text()) -+ if len(fields) > 0 { -+ if fields[0] == processorIdentifier { -+ cpuNum++ -+ } -+ } -+ } -+ return cpuNum -+} --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0018-sandbox-Stop-and-clean-up-containers-that-fail-to-cr.patch b/runtime/patches/0018-sandbox-Stop-and-clean-up-containers-that-fail-to-cr.patch deleted file mode 100644 index d7f0d10..0000000 --- a/runtime/patches/0018-sandbox-Stop-and-clean-up-containers-that-fail-to-cr.patch +++ /dev/null @@ -1,50 +0,0 @@ -From c785f8f744050155102664d56de5bfb55e91915d Mon Sep 17 00:00:00 2001 -From: Evan Foster -Date: Mon, 13 Jul 2020 12:53:40 -0600 -Subject: [PATCH 18/50] sandbox: Stop and clean up containers that fail to - create - -A container that is created and added to a sandbox can still fail -the final creation steps. In this case, the container must be stopped -and have its resources cleaned up to prevent leaking sandbox mounts. - -Fixes #2816 - -cherry-pick from: https://github.com/kata-containers/runtime/pull/2826 - -Signed-off-by: Evan Foster -Signed-off-by: jiangpengfei ---- - virtcontainers/sandbox.go | 11 +++++++++++ - 1 file changed, 11 insertions(+) - -diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go -index a8522b96..3dbf640e 100644 ---- a/virtcontainers/sandbox.go -+++ b/virtcontainers/sandbox.go -@@ -1,4 +1,5 @@ - // Copyright (c) 2016 Intel Corporation -+// Copyright (c) 2020 Adobe Inc. - // - // SPDX-License-Identifier: Apache-2.0 - // -@@ -1172,6 +1173,16 @@ func (s *Sandbox) CreateContainer(contConfig ContainerConfig) (VCContainer, erro - defer func() { - // Rollback if error happens. - if err != nil { -+ logger := s.Logger().WithFields(logrus.Fields{"container-id": c.id, "sandox-id": s.id, "rollback": true}) -+ -+ logger.Warning("Cleaning up partially created container") -+ -+ if err2 := c.stop(true); err2 != nil { -+ logger.WithError(err2).Warning("Could not delete container") -+ } -+ -+ logger.Debug("Removing stopped container from sandbox store") -+ - s.removeContainer(c.id) - } - }() --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0019-virtcontainers-fix-delete-sandbox-failed-problem.patch b/runtime/patches/0019-virtcontainers-fix-delete-sandbox-failed-problem.patch deleted file mode 100644 index 1bee93b..0000000 --- a/runtime/patches/0019-virtcontainers-fix-delete-sandbox-failed-problem.patch +++ /dev/null @@ -1,33 +0,0 @@ -From d20cb25c8a145e1d3e64eefa69242d87b7a67f92 Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Sat, 8 Aug 2020 16:00:45 -0400 -Subject: [PATCH 19/50] virtcontainers: fix delete sandbox failed problem - -fixes: #2882 - -reason: If error happens after container create and before sandbox -updateResouce in the `CreateContainer()`, then delete sandbox -forcefully will return error because s.config.Containers config not -flushed into persist store. - -Signed-off-by: jiangpengfei ---- - virtcontainers/sandbox.go | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go -index 3dbf640e..7322ef58 100644 ---- a/virtcontainers/sandbox.go -+++ b/virtcontainers/sandbox.go -@@ -1156,6 +1156,8 @@ func (s *Sandbox) CreateContainer(contConfig ContainerConfig) (VCContainer, erro - if len(s.config.Containers) > 0 { - // delete container config - s.config.Containers = s.config.Containers[:len(s.config.Containers)-1] -+ // need to flush change to persist storage -+ _ = s.storeSandbox() - } - } - }() --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0020-virtcontainers-add-enable_cpu_memory_hotplug-config-.patch b/runtime/patches/0020-virtcontainers-add-enable_cpu_memory_hotplug-config-.patch deleted file mode 100644 index b159188..0000000 --- a/runtime/patches/0020-virtcontainers-add-enable_cpu_memory_hotplug-config-.patch +++ /dev/null @@ -1,188 +0,0 @@ -From 75141529674545a2f84b01c730f614901ad2265e Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Mon, 10 Aug 2020 10:08:47 +0800 -Subject: [PATCH 20/50] virtcontainers: add enable_cpu_memory_hotplug config in - the configuration.toml - -reason: add enable_cpu_memory_hotplug config to control whether enable hotplug -memory and cpu resource into the Kata VM. If we can calculate the whole sandbox -resource usage already, we just need to pass the value of resouces by annotation -without hotplugging. - -Signed-off-by: jiangpengfei ---- - cli/config/configuration-qemu.toml.in | 4 ++++ - pkg/katautils/config.go | 2 ++ - virtcontainers/container.go | 6 ++++-- - virtcontainers/hypervisor.go | 3 +++ - virtcontainers/persist.go | 2 ++ - virtcontainers/persist/api/config.go | 3 +++ - virtcontainers/sandbox.go | 38 +++++++++++++++++++++-------------- - 7 files changed, 41 insertions(+), 17 deletions(-) - -diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in -index 82461732..b44e84d8 100644 ---- a/cli/config/configuration-qemu.toml.in -+++ b/cli/config/configuration-qemu.toml.in -@@ -100,6 +100,10 @@ default_memory = @DEFMEMSZ@ - # Default false - #enable_reuse_cpu_memory = false - -+# If enabled, the runtime will support cpu and memory hot plug -+# (default: disabled) -+#enable_cpu_memory_hotplug = true -+ - # Disable block device from being used for a container's rootfs. - # In case of a storage driver like devicemapper where a container's - # root file system is backed by a block device, the block device is passed -diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go -index b9b08469..9a99b9d4 100644 ---- a/pkg/katautils/config.go -+++ b/pkg/katautils/config.go -@@ -112,6 +112,7 @@ type hypervisor struct { - MemSlots uint32 `toml:"memory_slots"` - MemOffset uint32 `toml:"memory_offset"` - EnableCPUMemoryReuse bool `toml:"enable_reuse_cpu_memory"` -+ EnableCPUMemoryHotPlug bool `toml:"enable_cpu_memory_hotplug"` - DefaultBridges uint32 `toml:"default_bridges"` - Msize9p uint32 `toml:"msize_9p"` - PCIeRootPort uint32 `toml:"pcie_root_port"` -@@ -675,6 +676,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { - MemSlots: h.defaultMemSlots(), - MemOffset: h.defaultMemOffset(), - EnableCPUMemoryReuse: h.EnableCPUMemoryReuse, -+ EnableCPUMemoryHotPlug: h.EnableCPUMemoryHotPlug, - VirtioMem: h.VirtioMem, - EntropySource: h.GetEntropySource(), - DefaultBridges: h.defaultBridges(), -diff --git a/virtcontainers/container.go b/virtcontainers/container.go -index 75f590eb..1b89f6ac 100644 ---- a/virtcontainers/container.go -+++ b/virtcontainers/container.go -@@ -1273,8 +1273,10 @@ func (c *Container) update(resources specs.LinuxResources) error { - c.config.Resources.Memory.Limit = mem.Limit - } - -- if err := c.sandbox.updateResources(); err != nil { -- return err -+ if c.sandbox.config.HypervisorConfig.EnableCPUMemoryHotPlug { -+ if err := c.sandbox.updateResources(); err != nil { -+ return err -+ } - } - - if !c.sandbox.config.SandboxCgroupOnly { -diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go -index 9cd685ad..c0723daa 100644 ---- a/virtcontainers/hypervisor.go -+++ b/virtcontainers/hypervisor.go -@@ -253,6 +253,9 @@ type HypervisorConfig struct { - // Enable hypervisor cpu and memory resource share with container - EnableCPUMemoryReuse bool - -+ // Enable hotplug cpu and memory resource into container -+ EnableCPUMemoryHotPlug bool -+ - // VirtioFSCacheSize is the DAX cache size in MiB - VirtioFSCacheSize uint32 - -diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go -index aef4d18d..6bd09a0b 100644 ---- a/virtcontainers/persist.go -+++ b/virtcontainers/persist.go -@@ -215,6 +215,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) { - MemSlots: sconfig.HypervisorConfig.MemSlots, - MemOffset: sconfig.HypervisorConfig.MemOffset, - EnableCPUMemoryReuse: sconfig.HypervisorConfig.EnableCPUMemoryReuse, -+ EnableCPUMemoryHotPlug: sconfig.HypervisorConfig.EnableCPUMemoryHotPlug, - VirtioMem: sconfig.HypervisorConfig.VirtioMem, - VirtioFSCacheSize: sconfig.HypervisorConfig.VirtioFSCacheSize, - KernelPath: sconfig.HypervisorConfig.KernelPath, -@@ -505,6 +506,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) { - MemSlots: hconf.MemSlots, - MemOffset: hconf.MemOffset, - EnableCPUMemoryReuse: hconf.EnableCPUMemoryReuse, -+ EnableCPUMemoryHotPlug: hconf.EnableCPUMemoryHotPlug, - VirtioMem: hconf.VirtioMem, - VirtioFSCacheSize: hconf.VirtioFSCacheSize, - KernelPath: hconf.KernelPath, -diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go -index a3c6ec91..cfbee849 100644 ---- a/virtcontainers/persist/api/config.go -+++ b/virtcontainers/persist/api/config.go -@@ -38,6 +38,9 @@ type HypervisorConfig struct { - // Enable hypervisor cpu and memory resource share with container - EnableCPUMemoryReuse bool - -+ // Enable hotplug cpu and memory resource into container -+ EnableCPUMemoryHotPlug bool -+ - // VirtioFSCacheSize is the DAX cache size in MiB - VirtioFSCacheSize uint32 - -diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go -index 7322ef58..8fcd92d4 100644 ---- a/virtcontainers/sandbox.go -+++ b/virtcontainers/sandbox.go -@@ -1189,12 +1189,16 @@ func (s *Sandbox) CreateContainer(contConfig ContainerConfig) (VCContainer, erro - } - }() - -- // Sandbox is reponsable to update VM resources needed by Containers -- // Update resources after having added containers to the sandbox, since -- // container status is requiered to know if more resources should be added. -- err = s.updateResources() -- if err != nil { -- return nil, err -+ // update sandbox resource only when enable_cpu_memory_hotplug config set true -+ // in the config.toml file -+ if s.config.HypervisorConfig.EnableCPUMemoryHotPlug { -+ // Sandbox is reponsable to update VM resources needed by Containers -+ // Update resources after having added containers to the sandbox, since -+ // container status is requiered to know if more resources should be added. -+ err = s.updateResources() -+ if err != nil { -+ return nil, err -+ } - } - - if err = s.cgroupsUpdate(); err != nil { -@@ -1228,11 +1232,13 @@ func (s *Sandbox) StartContainer(containerID string) (VCContainer, error) { - - s.Logger().Info("Container is started") - -- // Update sandbox resources in case a stopped container -- // is started -- err = s.updateResources() -- if err != nil { -- return nil, err -+ if s.config.HypervisorConfig.EnableCPUMemoryHotPlug { -+ // Update sandbox resources in case a stopped container -+ // is started -+ err = s.updateResources() -+ if err != nil { -+ return nil, err -+ } - } - - return c, nil -@@ -1503,10 +1509,12 @@ func (s *Sandbox) createContainers() error { - } - } - -- // Update resources after having added containers to the sandbox, since -- // container status is requiered to know if more resources should be added. -- if err := s.updateResources(); err != nil { -- return err -+ if s.config.HypervisorConfig.EnableCPUMemoryHotPlug { -+ // Update resources after having added containers to the sandbox, since -+ // container status is requiered to know if more resources should be added. -+ if err := s.updateResources(); err != nil { -+ return err -+ } - } - - if err := s.cgroupsUpdate(); err != nil { --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0021-kata-runtime-add-sandbox_cpu-and-sandbox_mem-annotat.patch b/runtime/patches/0021-kata-runtime-add-sandbox_cpu-and-sandbox_mem-annotat.patch deleted file mode 100644 index eae49d4..0000000 --- a/runtime/patches/0021-kata-runtime-add-sandbox_cpu-and-sandbox_mem-annotat.patch +++ /dev/null @@ -1,195 +0,0 @@ -From 79cf2f5a52af51d8a62353a99e894808281769e2 Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Mon, 10 Aug 2020 11:25:02 +0800 -Subject: [PATCH 21/50] kata-runtime: add sandbox_cpu and sandbox_mem - annotations - -reason: add sandbox_cpu and sandbox_men annotations to set -Kata VM vCPU number and memory size. - -Signed-off-by: jiangpengfei ---- - virtcontainers/pkg/annotations/annotations.go | 6 +++++ - virtcontainers/pkg/oci/utils.go | 39 +++++++++++++++++++++++++-- - virtcontainers/sandbox.go | 32 ++++++++++++++++++++++ - virtcontainers/utils/utils.go | 14 ++++++++++ - 4 files changed, 89 insertions(+), 2 deletions(-) - -diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go -index 10ce7833..903c7f03 100644 ---- a/virtcontainers/pkg/annotations/annotations.go -+++ b/virtcontainers/pkg/annotations/annotations.go -@@ -251,6 +251,12 @@ const ( - ContainerPipeSizeKernelParam = "agent." + ContainerPipeSizeOption - ) - -+// iSula self defined annotations -+const ( -+ StaticCPUTypeKey = kataAnnotationsPrefix + "sandbox_cpu" -+ StaticMemTypeKey = kataAnnotationsPrefix + "sandbox_mem" -+) -+ - const ( - // SHA512 is the SHA-512 (64) hash algorithm - SHA512 string = "sha512" -diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go -index 0a6f08c7..36c730b7 100644 ---- a/virtcontainers/pkg/oci/utils.go -+++ b/virtcontainers/pkg/oci/utils.go -@@ -16,6 +16,7 @@ import ( - - criContainerdAnnotations "github.com/containerd/cri-containerd/pkg/annotations" - crioAnnotations "github.com/cri-o/cri-o/pkg/annotations" -+ "github.com/docker/go-units" - vc "github.com/kata-containers/runtime/virtcontainers" - "github.com/kata-containers/runtime/virtcontainers/device/config" - exp "github.com/kata-containers/runtime/virtcontainers/experimental" -@@ -34,7 +35,10 @@ type annotationContainerType struct { - - type annotationHandler func(value string) error - --var annotationHandlerList = map[string]annotationHandler{} -+var annotationHandlerList = map[string]annotationHandler{ -+ vcAnnotations.StaticCPUTypeKey: validateSandboxCPU, -+ vcAnnotations.StaticMemTypeKey: validateSandboxMem, -+} - - var ( - // ErrNoLinux is an error for missing Linux sections in the OCI configuration file. -@@ -1036,7 +1040,10 @@ func validateOtherSandboxAnnotations(annotation, value string) error { - - // addOtherSandboxAnnotation add self defined annotation for sandbox - func addOtherSandboxAnnotation(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error { -- otherSandboxAnnotationsKey := []string{} -+ otherSandboxAnnotationsKey := []string{ -+ vcAnnotations.StaticCPUTypeKey, -+ vcAnnotations.StaticMemTypeKey, -+ } - - for _, a := range otherSandboxAnnotationsKey { - value, ok := ocispec.Annotations[a] -@@ -1053,3 +1060,31 @@ func addOtherSandboxAnnotation(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e - - return nil - } -+ -+func validateSandboxCPU(value string) error { -+ // check min cpu value -+ cpus, err := utils.RoundVCPUNumber(value) -+ if err != nil { -+ return fmt.Errorf("valiate sandbox_cpu annotation fail: %v", err) -+ } -+ -+ maxPhysicalCPUs := utils.GetPhysicalCPUNumber() -+ if cpus > maxPhysicalCPUs { -+ return fmt.Errorf("sandbox_cpu annotation value exceed the machine max CPU number: %d", cpus) -+ } -+ -+ return nil -+} -+ -+func validateSandboxMem(value string) error { -+ memSizeInBytes, err := units.RAMInBytes(value) -+ if err != nil { -+ return fmt.Errorf("parse sandbox_mem value: %d fail: %v", memSizeInBytes, err) -+ } -+ -+ if memSizeInBytes < utils.MinMemorySizeInByte || memSizeInBytes > utils.MaxMemorySizeInByte { -+ return fmt.Errorf("invalid sandbox_mem value size in bytes: %v", memSizeInBytes) -+ } -+ -+ return nil -+} -diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go -index 8fcd92d4..ba704249 100644 ---- a/virtcontainers/sandbox.go -+++ b/virtcontainers/sandbox.go -@@ -20,6 +20,7 @@ import ( - - "github.com/containerd/cgroups" - "github.com/containernetworking/plugins/pkg/ns" -+ "github.com/docker/go-units" - "github.com/kata-containers/agent/protocols/grpc" - "github.com/kata-containers/runtime/virtcontainers/device/api" - "github.com/kata-containers/runtime/virtcontainers/device/config" -@@ -479,6 +480,10 @@ func createSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Fac - return nil, err - } - -+ if err := updateStaticSandboxResources(&sandboxConfig); err != nil { -+ return nil, err -+ } -+ - s, err := newSandbox(ctx, sandboxConfig, factory) - if err != nil { - return nil, err -@@ -2359,3 +2364,30 @@ func (s *Sandbox) setContainersState(state types.StateString) error { - - return nil - } -+ -+// updateStaticSandboxResources update sandbox's cpu and memory resource passed by -+// sandbox_cpu and sandbox_mem annotations -+func updateStaticSandboxResources(sandboxConfig *SandboxConfig) error { -+ // update cpu resource -+ if cpuNumVal, ok := sandboxConfig.Annotations[annotations.StaticCPUTypeKey]; ok { -+ cpuNum, err := utils.RoundVCPUNumber(cpuNumVal) -+ if err != nil { -+ return err -+ } -+ -+ sandboxConfig.HypervisorConfig.NumVCPUs = (uint32)(cpuNum) -+ } -+ -+ // update mem resource -+ if memVal, ok := sandboxConfig.Annotations[annotations.StaticMemTypeKey]; ok { -+ memSizeInBytes, err := units.RAMInBytes(memVal) -+ if err != nil { -+ return err -+ } -+ -+ memSizeInMB := memSizeInBytes >> utils.MibToBytesShift -+ sandboxConfig.HypervisorConfig.MemorySize = (uint32)(memSizeInMB) -+ } -+ -+ return nil -+} -diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go -index 5d38e594..9490faa1 100644 ---- a/virtcontainers/utils/utils.go -+++ b/virtcontainers/utils/utils.go -@@ -11,6 +11,7 @@ import ( - "errors" - "fmt" - "io/ioutil" -+ "math" - "os" - "os/exec" - "path/filepath" -@@ -30,6 +31,9 @@ const MibToBytesShift = 20 - const MaxHotplugMemMBOnceTime = 32 * 1024 - - const ( -+ // minCPUs is allowed minimum CPU -+ minCPUs = 0.25 -+ - // Min needed memory size to start a Kata VM - MinMemorySizeInMB = 300 - MinMemorySizeInByte = MinMemorySizeInMB << MibToBytesShift -@@ -365,3 +369,13 @@ func GetPhysicalCPUNumber() int { - } - return cpuNum - } -+ -+func RoundVCPUNumber(value string) (int, error) { -+ cpuNum, err := strconv.ParseFloat(value, 64) -+ if err != nil || cpuNum < minCPUs { -+ return 0, fmt.Errorf("invalid sandbox cpu number: %v", cpuNum) -+ } -+ -+ cpus := int(math.Ceil(cpuNum)) -+ return cpus, nil -+} --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0022-kata-runtime-skip-go-version-check-and-do-not-build-.patch b/runtime/patches/0022-kata-runtime-skip-go-version-check-and-do-not-build-.patch deleted file mode 100644 index 025f3ff..0000000 --- a/runtime/patches/0022-kata-runtime-skip-go-version-check-and-do-not-build-.patch +++ /dev/null @@ -1,39 +0,0 @@ -From e5e3232f7268110f7e3e3c4814eab31a6704b672 Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Mon, 10 Aug 2020 20:11:07 +0800 -Subject: [PATCH 22/50] kata-runtime: skip go version check and do not build - containerd-shim-v2 - -reason: skip go version check and do not build containerd-shim-v2 -because iSulad current not support shimV2 - -Signed-off-by: jiangpengfei ---- - Makefile | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/Makefile b/Makefile -index 14a0ea47..d5e4bbe1 100644 ---- a/Makefile -+++ b/Makefile -@@ -12,7 +12,7 @@ for file in /etc/os-release /usr/lib/os-release; do \ - fi \ - done) - --SKIP_GO_VERSION_CHECK= -+SKIP_GO_VERSION_CHECK=y - include golang.mk - - #Get ARCH. -@@ -503,7 +503,7 @@ define SHOW_ARCH - $(shell printf "\\t%s%s\\\n" "$(1)" $(if $(filter $(ARCH),$(1))," (default)","")) - endef - --all: runtime containerd-shim-v2 netmon -+all: runtime netmon - - # Targets that depend on .git-commit can use $(shell cat .git-commit) to get a - # git revision string. They will only be rebuilt if the revision string --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0023-kata-runtime-set-PCIBridgeMaxCapacity-limit-to-25.patch b/runtime/patches/0023-kata-runtime-set-PCIBridgeMaxCapacity-limit-to-25.patch deleted file mode 100644 index ac1c9a1..0000000 --- a/runtime/patches/0023-kata-runtime-set-PCIBridgeMaxCapacity-limit-to-25.patch +++ /dev/null @@ -1,42 +0,0 @@ -From 226c3336dcc70bd17e3471ff98106a2f8dee9ac5 Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Tue, 11 Aug 2020 22:32:49 +0800 -Subject: [PATCH 23/50] kata-runtime: set PCIBridgeMaxCapacity limit to 25 - -reason: set PCIBridgeMaxCapacity limit to 25. - -Signed-off-by: jiangpengfei ---- - virtcontainers/qemu_arch_base_test.go | 2 +- - virtcontainers/types/bridges.go | 2 +- - 2 files changed, 2 insertions(+), 2 deletions(-) - -diff --git a/virtcontainers/qemu_arch_base_test.go b/virtcontainers/qemu_arch_base_test.go -index 169e002e..8219f3c5 100644 ---- a/virtcontainers/qemu_arch_base_test.go -+++ b/virtcontainers/qemu_arch_base_test.go -@@ -175,7 +175,7 @@ func TestQemuAddDeviceToBridge(t *testing.T) { - } - - // fail to add device to bridge cause no more available bridge slot -- _, _, err := q.addDeviceToBridge("qemu-bridge-31", types.PCI) -+ _, _, err := q.addDeviceToBridge("qemu-bridge-26", types.PCI) - exceptErr := errors.New("no more bridge slots available") - assert.Equal(exceptErr.Error(), err.Error()) - -diff --git a/virtcontainers/types/bridges.go b/virtcontainers/types/bridges.go -index cb15a88f..c3538ce4 100644 ---- a/virtcontainers/types/bridges.go -+++ b/virtcontainers/types/bridges.go -@@ -10,7 +10,7 @@ import "fmt" - // Type represents a type of bus and bridge. - type Type string - --const PCIBridgeMaxCapacity = 30 -+const PCIBridgeMaxCapacity = 25 - - const ( - // PCI represents a PCI bus and bridge --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0024-kata-runtime-support-hotplug-tap-interface-into-kata.patch b/runtime/patches/0024-kata-runtime-support-hotplug-tap-interface-into-kata.patch deleted file mode 100644 index bdf9537..0000000 --- a/runtime/patches/0024-kata-runtime-support-hotplug-tap-interface-into-kata.patch +++ /dev/null @@ -1,462 +0,0 @@ -From e861f426c9e6702e820348ddc61b18013c853402 Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Thu, 13 Aug 2020 11:24:58 +0800 -Subject: [PATCH 24/50] kata-runtime: support hotplug tap interface into kata - VM - -reason: support hotplug exist tap interface or a new created tap -interface into kata VM. - -Signed-off-by: jiangpengfei ---- - cli/network.go | 45 ++++++++++++++--- - virtcontainers/kata_agent.go | 12 ++++- - virtcontainers/network.go | 100 +++++++++++++++++++++++--------------- - virtcontainers/pkg/types/types.go | 16 +++--- - virtcontainers/qemu.go | 11 +++-- - virtcontainers/sandbox.go | 24 +++++++-- - virtcontainers/tap_endpoint.go | 23 +++++++-- - 7 files changed, 161 insertions(+), 70 deletions(-) - -diff --git a/cli/network.go b/cli/network.go -index 881a2358..7e7791f1 100644 ---- a/cli/network.go -+++ b/cli/network.go -@@ -26,6 +26,8 @@ const ( - routeType - ) - -+const defaultLinkType = "tap" -+ - var kataNetworkCLICommand = cli.Command{ - Name: "kata-network", - Usage: "manage interfaces and routes for container", -@@ -42,10 +44,22 @@ var kataNetworkCLICommand = cli.Command{ - } - - var addIfaceCommand = cli.Command{ -- Name: "add-iface", -- Usage: "add an interface to a container", -- ArgsUsage: `add-iface file or - for stdin`, -- Flags: []cli.Flag{}, -+ Name: "add-iface", -+ Usage: "add an interface to a container", -+ ArgsUsage: `add-iface file or - for stdin -+ file or stdin for example: -+ { -+ "device":"", -+ "name":"", -+ "IPAddresses":[{"address":"","mask":""}], -+ "mtu":, -+ "hwAddr":"", -+ "linkType":"tap", -+ "vhostUserSocket":"" -+ } -+ device,name,mtu,hwAddr are required, IPAddresses and vhostUserSocket are optional. -+ `, -+ Flags: []cli.Flag{}, - Action: func(context *cli.Context) error { - ctx, err := cliContextToContext(context) - if err != nil { -@@ -57,10 +71,20 @@ var addIfaceCommand = cli.Command{ - } - - var delIfaceCommand = cli.Command{ -- Name: "del-iface", -- Usage: "delete an interface from a container", -- ArgsUsage: `del-iface file or - for stdin`, -- Flags: []cli.Flag{}, -+ Name: "del-iface", -+ Usage: "delete an interface from a container", -+ ArgsUsage: `del-iface file or - for stdin -+ file or stdin for example: -+ { -+ "device":"", -+ "name":"", -+ "IPAddresses":[], -+ "mtu":0, -+ "hwAddr":"" -+ } -+ Only the "name" field is required. -+ `, -+ Flags: []cli.Flag{}, - Action: func(context *cli.Context) error { - ctx, err := cliContextToContext(context) - if err != nil { -@@ -156,6 +180,11 @@ func networkModifyCommand(ctx context.Context, containerID, input string, opType - if err = json.NewDecoder(f).Decode(&inf); err != nil { - return err - } -+ -+ if len(inf.LinkType) == 0 { -+ inf.LinkType = defaultLinkType -+ } -+ - if add { - resultingInf, err = vci.AddInterface(ctx, sandboxID, inf) - if err != nil { -diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go -index 8e073339..dfdd263b 100644 ---- a/virtcontainers/kata_agent.go -+++ b/virtcontainers/kata_agent.go -@@ -600,8 +600,16 @@ func (k *kataAgent) updateInterface(ifc *vcTypes.Interface) (*vcTypes.Interface, - "resulting-interface": fmt.Sprintf("%+v", resultingInterface), - }).WithError(err).Error("update interface request failed") - } -- if resultInterface, ok := resultingInterface.(*vcTypes.Interface); ok { -- return resultInterface, err -+ if resultInterface, ok := resultingInterface.(*aTypes.Interface); ok { -+ iface := &vcTypes.Interface{ -+ Device: resultInterface.Device, -+ Name: resultInterface.Name, -+ IPAddresses: k.convertToIPAddresses(resultInterface.IPAddresses), -+ Mtu: resultInterface.Mtu, -+ HwAddr: resultInterface.HwAddr, -+ PciAddr: resultInterface.PciAddr, -+ } -+ return iface, err - } - return nil, err - } -diff --git a/virtcontainers/network.go b/virtcontainers/network.go -index d70c5360..e909a822 100644 ---- a/virtcontainers/network.go -+++ b/virtcontainers/network.go -@@ -117,6 +117,7 @@ type NetlinkIface struct { - // NetworkInfo gathers all information related to a network interface. - // It can be used to store the description of the underlying network. - type NetworkInfo struct { -+ Device string - Iface NetlinkIface - Addrs []netlink.Addr - Routes []netlink.Route -@@ -303,6 +304,20 @@ func createLink(netHandle *netlink.Handle, name string, expectedLink netlink.Lin - var newLink netlink.Link - var fds []*os.File - -+ // check if tapname exists, if true, use existing one instead of create new one -+ if name != "" { -+ retLink, err := netlink.LinkByName(name) -+ // link exist, use it -+ if err == nil { -+ networkLogger().Debugf("exist tap device is found, instead of creating new one") -+ tuntapLink, ok := retLink.(*netlink.Tuntap) -+ if ok { -+ fds = tuntapLink.Fds -+ } -+ return retLink, nil, nil -+ } -+ } -+ - switch expectedLink.Type() { - case (&netlink.Tuntap{}).Type(): - flags := netlink.TUNTAP_VNET_HDR -@@ -1156,7 +1171,7 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li - - // Check if interface is a physical interface. Do not create - // tap interface/bridge if it is. -- isPhysical, err := isPhysicalIface(netInfo.Iface.Name) -+ isPhysical, err := isPhysicalIface(netInfo.Device) - if err != nil { - return nil, err - } -@@ -1164,49 +1179,54 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li - if isPhysical { - networkLogger().WithField("interface", netInfo.Iface.Name).Info("Physical network interface found") - endpoint, err = createPhysicalEndpoint(netInfo) -- } else { -- var socketPath string -+ return endpoint, err -+ } - -- // Check if this is a dummy interface which has a vhost-user socket associated with it -- socketPath, err = vhostUserSocketPath(netInfo) -- if err != nil { -- return nil, err -- } -+ // Check if this is a dummy interface which has a vhost-user socket associated with it -+ socketPath, err := vhostUserSocketPath(netInfo) -+ if err != nil { -+ return nil, err -+ } - -- if socketPath != "" { -- networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found") -- endpoint, err = createVhostUserEndpoint(netInfo, socketPath) -- } else if netInfo.Iface.Type == "macvlan" { -- networkLogger().Infof("macvlan interface found") -- endpoint, err = createBridgedMacvlanNetworkEndpoint(idx, netInfo.Iface.Name, model) -- } else if netInfo.Iface.Type == "macvtap" { -- networkLogger().Infof("macvtap interface found") -- endpoint, err = createMacvtapNetworkEndpoint(netInfo) -- } else if netInfo.Iface.Type == "tap" { -- networkLogger().Info("tap interface found") -- endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name) -- } else if netInfo.Iface.Type == "tuntap" { -- if link != nil { -- switch link.(*netlink.Tuntap).Mode { -- case 0: -- // mount /sys/class/net to get links -- return nil, fmt.Errorf("Network device mode not determined correctly. Mount sysfs in caller") -- case 1: -- return nil, fmt.Errorf("tun networking device not yet supported") -- case 2: -- networkLogger().Info("tuntap tap interface found") -- endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, model) -- default: -- return nil, fmt.Errorf("tuntap network %v mode unsupported", link.(*netlink.Tuntap).Mode) -- } -+ if socketPath != "" { -+ networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found") -+ endpoint, err = createVhostUserEndpoint(netInfo, socketPath) -+ return endpoint, err -+ } -+ -+ // We should create tap interface/bridge of other interface type. -+ networkLogger().Infof("%s interface found", netInfo.Iface.Type) -+ switch netInfo.Iface.Type { -+ case "macvlan": -+ networkLogger().Infof("macvlan interface found") -+ endpoint, err = createBridgedMacvlanNetworkEndpoint(idx, netInfo.Iface.Name, model) -+ case "macvtap": -+ networkLogger().Infof("macvtap interface found") -+ endpoint, err = createMacvtapNetworkEndpoint(netInfo) -+ case "tap": -+ networkLogger().Info("tap interface found") -+ endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Device) -+ case "tuntap": -+ if link != nil { -+ switch link.(*netlink.Tuntap).Mode { -+ case 0: -+ // mount /sys/class/net to get links -+ return nil, fmt.Errorf("Network device mode not determined correctly. Mount sysfs in caller") -+ case 1: -+ return nil, fmt.Errorf("tun networking device not yet supported") -+ case 2: -+ networkLogger().Info("tuntap tap interface found") -+ endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, model) -+ default: -+ return nil, fmt.Errorf("tuntap network %v mode unsupported", link.(*netlink.Tuntap).Mode) - } -- } else if netInfo.Iface.Type == "veth" { -- endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model) -- } else if netInfo.Iface.Type == "ipvlan" { -- endpoint, err = createIPVlanNetworkEndpoint(idx, netInfo.Iface.Name) -- } else { -- return nil, fmt.Errorf("Unsupported network interface: %s", netInfo.Iface.Type) - } -+ case "veth": -+ endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model) -+ case "ipvlan": -+ endpoint, err = createIPVlanNetworkEndpoint(idx, netInfo.Iface.Name) -+ default: -+ err = fmt.Errorf("Unsupported network interface, %s", netInfo.Iface.Type) - } - - return endpoint, err -diff --git a/virtcontainers/pkg/types/types.go b/virtcontainers/pkg/types/types.go -index 0d4a9cfa..fcc63d84 100644 ---- a/virtcontainers/pkg/types/types.go -+++ b/virtcontainers/pkg/types/types.go -@@ -14,21 +14,21 @@ type IPAddress struct { - - // Interface describes a network interface. - type Interface struct { -- Device string -- Name string -- IPAddresses []*IPAddress -- Mtu uint64 -- RawFlags uint32 -- HwAddr string -+ Device string `json:"device,omitempty"` -+ Name string `json:"name,omitempty"` -+ IPAddresses []*IPAddress `json:"IPAddresses,omitempty"` -+ Mtu uint64 `json:"mtu,omitempty"` -+ RawFlags uint32 `json:"rawFlags,omitempty"` -+ HwAddr string `json:"hwAddr,omitempty"` - // pciAddr is the PCI address in the format "bridgeAddr/deviceAddr". - // Here, bridgeAddr is the address at which the bridge is attached on the root bus, - // while deviceAddr is the address at which the network device is attached on the bridge. -- PciAddr string -+ PciAddr string `json:"pciAddr,omitempty"` - // LinkType defines the type of interface described by this structure. - // The expected values are the one that are defined by the netlink - // library, regarding each type of link. Here is a non exhaustive - // list: "veth", "macvtap", "vlan", "macvlan", "tap", ... -- LinkType string -+ LinkType string `json:"linkType,omitempty"` - } - - // Route describes a network route. -diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go -index 7bae3278..bb83b1bb 100644 ---- a/virtcontainers/qemu.go -+++ b/virtcontainers/qemu.go -@@ -1361,7 +1361,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro - return nil - } - --func (q *qemu) hotAddNetDevice(name, hardAddr string, VMFds, VhostFds []*os.File) error { -+func (q *qemu) hotAddNetDevice(deviceName, name, hardAddr string, VMFds, VhostFds []*os.File) error { - var ( - VMFdNames []string - VhostFdNames []string -@@ -1381,7 +1381,12 @@ func (q *qemu) hotAddNetDevice(name, hardAddr string, VMFds, VhostFds []*os.File - VhostFd.Close() - VhostFdNames = append(VhostFdNames, fdName) - } -- return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", name, VMFdNames, VhostFdNames) -+ -+ if len(VMFdNames) != 0 || len(VhostFdNames) != 0 { -+ return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", name, VMFdNames, VhostFdNames) -+ } -+ -+ return q.qmpMonitorCh.qmp.ExecuteNetdevAdd(q.qmpMonitorCh.ctx, "tap", name, deviceName, "no", "no", 0) - } - - func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) { -@@ -1404,7 +1409,7 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) { - - devID := "virtio-" + tap.ID - if op == addDevice { -- if err = q.hotAddNetDevice(tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds); err != nil { -+ if err = q.hotAddNetDevice(tap.TAPIface.Name, tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds); err != nil { - return err - } - -diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go -index ba704249..c8981a41 100644 ---- a/virtcontainers/sandbox.go -+++ b/virtcontainers/sandbox.go -@@ -937,6 +937,7 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) { - } - - return NetworkInfo{ -+ Device: inf.Device, - Iface: NetlinkIface{ - LinkAttrs: netlink.LinkAttrs{ - Name: inf.Name, -@@ -950,7 +951,7 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) { - } - - // AddInterface adds new nic to the sandbox. --func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (*vcTypes.Interface, error) { -+func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (grpcIf *vcTypes.Interface, err error) { - netInfo, err := s.generateNetInfo(inf) - if err != nil { - return nil, err -@@ -962,28 +963,41 @@ func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (*vcTypes.Interface, erro - } - - endpoint.SetProperties(netInfo) -- if err := doNetNS(s.networkNS.NetNsPath, func(_ ns.NetNS) error { -+ if err = doNetNS(s.networkNS.NetNsPath, func(_ ns.NetNS) error { - s.Logger().WithField("endpoint-type", endpoint.Type()).Info("Hot attaching endpoint") - return endpoint.HotAttach(s.hypervisor) - }); err != nil { - return nil, err - } - -+ defer func() { -+ if err != nil { -+ if errDetach := endpoint.HotDetach(s.hypervisor, s.networkNS.NetNsCreated, s.networkNS.NetNsPath); errDetach != nil { -+ s.Logger().WithField("endpoint-type", endpoint.Type()).Errorf("rollback hot attach endpoint failed") -+ } -+ } -+ }() -+ - // Update the sandbox storage - s.networkNS.Endpoints = append(s.networkNS.Endpoints, endpoint) -- if err := s.Save(); err != nil { -+ if err = s.Save(); err != nil { - return nil, err - } - - // Add network for vm - inf.PciAddr = endpoint.PciAddr() -- return s.agent.updateInterface(inf) -+ grpcIf, err = s.agent.updateInterface(inf) -+ if err != nil { -+ return nil, err -+ } -+ -+ return - } - - // RemoveInterface removes a nic of the sandbox. - func (s *Sandbox) RemoveInterface(inf *vcTypes.Interface) (*vcTypes.Interface, error) { - for i, endpoint := range s.networkNS.Endpoints { -- if endpoint.HardwareAddr() == inf.HwAddr { -+ if endpoint.HardwareAddr() == inf.HwAddr || endpoint.Name() == inf.Name { - s.Logger().WithField("endpoint-type", endpoint.Type()).Info("Hot detaching endpoint") - if err := endpoint.HotDetach(s.hypervisor, s.networkNS.NetNsCreated, s.networkNS.NetNsPath); err != nil { - return inf, err -diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go -index cb441b87..7d33d5a2 100644 ---- a/virtcontainers/tap_endpoint.go -+++ b/virtcontainers/tap_endpoint.go -@@ -7,6 +7,7 @@ package virtcontainers - - import ( - "fmt" -+ "os" - - "github.com/containernetworking/plugins/pkg/ns" - "github.com/vishvananda/netlink" -@@ -111,7 +112,7 @@ func (endpoint *TapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPat - return nil - } - --func createTapNetworkEndpoint(idx int, ifName string) (*TapEndpoint, error) { -+func createTapNetworkEndpoint(idx int, ifName string, tapIfName string) (*TapEndpoint, error) { - if idx < 0 { - return &TapEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx) - } -@@ -131,6 +132,10 @@ func createTapNetworkEndpoint(idx int, ifName string) (*TapEndpoint, error) { - endpoint.TapInterface.Name = ifName - } - -+ if tapIfName != "" { -+ endpoint.TapInterface.TAPIface.Name = tapIfName -+ } -+ - return endpoint, nil - } - -@@ -145,9 +150,19 @@ func tapNetwork(endpoint *TapEndpoint, numCPUs uint32, disableVhostNet bool) err - if err != nil { - return fmt.Errorf("Could not create TAP interface: %s", err) - } -+ -+ defer func() { -+ if err != nil { -+ if errDel := netHandle.LinkDel(tapLink); errDel != nil { -+ networkLogger().WithError(errDel).Error("tapNetwork fail to rollback del link") -+ } -+ } -+ }() -+ - endpoint.TapInterface.VMFds = fds - if !disableVhostNet { -- vhostFds, err := createVhostFds(int(numCPUs)) -+ var vhostFds []*os.File -+ vhostFds, err = createVhostFds(int(numCPUs)) - if err != nil { - return fmt.Errorf("Could not setup vhost fds %s : %s", endpoint.TapInterface.Name, err) - } -@@ -161,10 +176,10 @@ func tapNetwork(endpoint *TapEndpoint, numCPUs uint32, disableVhostNet bool) err - // bridge created by the network plugin on the host actually expects - // to see traffic from this MAC address and not another one. - endpoint.TapInterface.TAPIface.HardAddr = linkAttrs.HardwareAddr.String() -- if err := netHandle.LinkSetMTU(tapLink, linkAttrs.MTU); err != nil { -+ if err = netHandle.LinkSetMTU(tapLink, linkAttrs.MTU); err != nil { - return fmt.Errorf("Could not set TAP MTU %d: %s", linkAttrs.MTU, err) - } -- if err := netHandle.LinkSetUp(tapLink); err != nil { -+ if err = netHandle.LinkSetUp(tapLink); err != nil { - return fmt.Errorf("Could not enable TAP %s: %s", endpoint.TapInterface.Name, err) - } - return nil --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0025-network-keep-list-ifaces-result-compatible-with-cni.patch b/runtime/patches/0025-network-keep-list-ifaces-result-compatible-with-cni.patch deleted file mode 100644 index 8f630dc..0000000 --- a/runtime/patches/0025-network-keep-list-ifaces-result-compatible-with-cni.patch +++ /dev/null @@ -1,260 +0,0 @@ -From be8153f21c0b81d2b194075ecd654501bc708577 Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Thu, 13 Aug 2020 18:54:49 +0800 -Subject: [PATCH 25/50] network: keep list-ifaces result compatible with cni - -reason: community list-ifaces command will return the all -interfaces info in the Kata VM, however we may just want -to get the interfaces that we hotplug, so just return the -hotplugged interfaces and convert the interface info to -be compatible with cni. - -Signed-off-by: jiangpengfei ---- - cli/network.go | 29 ++++++++++++++++++++++- - virtcontainers/api.go | 4 +++- - virtcontainers/endpoint.go | 44 +++++++++++++++++++++++++++++++++++ - virtcontainers/network.go | 27 +++++++++++++++++++++ - virtcontainers/persist/api/network.go | 24 +++++++++++++++++++ - virtcontainers/tap_endpoint.go | 9 +++++++ - 6 files changed, 135 insertions(+), 2 deletions(-) - -diff --git a/cli/network.go b/cli/network.go -index 7e7791f1..66955725 100644 ---- a/cli/network.go -+++ b/cli/network.go -@@ -28,6 +28,13 @@ const ( - - const defaultLinkType = "tap" - -+type compatInterface struct { -+ Name string `json:"name,omitempty"` -+ Mac string `json:"mac,omitempty"` -+ IP []string `json:"ip,omitempty"` -+ Mtu int `json:"mtu,omitempty"` -+} -+ - var kataNetworkCLICommand = cli.Command{ - Name: "kata-network", - Usage: "manage interfaces and routes for container", -@@ -244,7 +251,8 @@ func networkListCommand(ctx context.Context, containerID string, opType networkT - kataLog.WithField("existing-interfaces", fmt.Sprintf("%+v", interfaces)). - WithError(err).Error("list interfaces failed") - } -- json.NewEncoder(file).Encode(interfaces) -+ compatInfs := convertCompatInterfaces(interfaces) -+ json.NewEncoder(file).Encode(compatInfs) - case routeType: - var routes []*vcTypes.Route - routes, err = vci.ListRoutes(ctx, sandboxID) -@@ -256,3 +264,22 @@ func networkListCommand(ctx context.Context, containerID string, opType networkT - } - return err - } -+ -+func convertCompatInterfaces(interfaces []*vcTypes.Interface) []compatInterface { -+ var infs []compatInterface -+ for _, i := range interfaces { -+ var addrs []string -+ for _, a := range i.IPAddresses { -+ addrs = append(addrs, fmt.Sprintf("%s/%s", a.Address, a.Mask)) -+ } -+ -+ infs = append(infs, compatInterface{ -+ Name: i.Name, -+ Mac: i.HwAddr, -+ IP: addrs, -+ Mtu: int(i.Mtu), -+ }) -+ } -+ -+ return infs -+} -diff --git a/virtcontainers/api.go b/virtcontainers/api.go -index 5e8c9c9e..eb5b4995 100644 ---- a/virtcontainers/api.go -+++ b/virtcontainers/api.go -@@ -949,7 +949,9 @@ func ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTypes.Interface - } - defer s.releaseStatelessSandbox() - -- return s.ListInterfaces() -+ // get interfaces info from persist.json file -+ // instead of by s.ListInterfaces() -+ return convertToCompatInterfaces(&s.networkNS.Endpoints), nil - } - - // UpdateRoutes is the virtcontainers update routes entry point. -diff --git a/virtcontainers/endpoint.go b/virtcontainers/endpoint.go -index 01b5e77f..7efcf49c 100644 ---- a/virtcontainers/endpoint.go -+++ b/virtcontainers/endpoint.go -@@ -132,6 +132,28 @@ func saveTapIf(tapif *TapInterface) *persistapi.TapInterface { - } - } - -+func saveTapEndpointProperties(networkInfo *NetworkInfo) *persistapi.NetworkProperties { -+ if networkInfo == nil { -+ return nil -+ } -+ -+ return &persistapi.NetworkProperties{ -+ Device: networkInfo.Device, -+ Iface: persistapi.NetlinkIface{ -+ LinkAttrs: networkInfo.Iface.LinkAttrs, -+ Type: networkInfo.Iface.Type, -+ }, -+ Addrs: networkInfo.Addrs, -+ Routes: networkInfo.Routes, -+ DNS: persistapi.DNSInfo{ -+ Servers: networkInfo.DNS.Servers, -+ Domain: networkInfo.DNS.Domain, -+ Searches: networkInfo.DNS.Searches, -+ Options: networkInfo.DNS.Options, -+ }, -+ } -+} -+ - func loadTapIf(tapif *persistapi.TapInterface) *TapInterface { - if tapif == nil { - return nil -@@ -148,6 +170,28 @@ func loadTapIf(tapif *persistapi.TapInterface) *TapInterface { - } - } - -+func loadTapEndpointProperties(endpointProperties *persistapi.NetworkProperties) *NetworkInfo { -+ if endpointProperties == nil { -+ return nil -+ } -+ -+ return &NetworkInfo{ -+ Device: endpointProperties.Device, -+ Iface: NetlinkIface{ -+ LinkAttrs: endpointProperties.Iface.LinkAttrs, -+ Type: endpointProperties.Iface.Type, -+ }, -+ Addrs: endpointProperties.Addrs, -+ Routes: endpointProperties.Routes, -+ DNS: DNSInfo{ -+ Servers: endpointProperties.DNS.Servers, -+ Domain: endpointProperties.DNS.Domain, -+ Searches: endpointProperties.DNS.Searches, -+ Options: endpointProperties.DNS.Options, -+ }, -+ } -+} -+ - func saveNetIfPair(pair *NetworkInterfacePair) *persistapi.NetworkInterfacePair { - if pair == nil { - return nil -diff --git a/virtcontainers/network.go b/virtcontainers/network.go -index e909a822..bf7f9336 100644 ---- a/virtcontainers/network.go -+++ b/virtcontainers/network.go -@@ -1340,3 +1340,30 @@ func (n *Network) Remove(ctx context.Context, ns *NetworkNamespace, hypervisor h - - return nil - } -+ -+// convertCompatInterfaces convert Endpoint info to vcTypes.Interface -+func convertToCompatInterfaces(es *[]Endpoint) []*vcTypes.Interface { -+ var infs []*vcTypes.Interface -+ for _, e := range *es { -+ var addrs []*vcTypes.IPAddress -+ for _, a := range e.Properties().Addrs { -+ m, _ := a.Mask.Size() -+ addr := &vcTypes.IPAddress{ -+ Address: fmt.Sprintf("%s", a.IP), -+ Mask: fmt.Sprintf("%d", m), -+ } -+ addrs = append(addrs, addr) -+ } -+ inf := &vcTypes.Interface{ -+ LinkType: string(e.Type()), -+ Name: e.Name(), -+ Mtu: uint64(e.Properties().Iface.MTU), -+ HwAddr: e.HardwareAddr(), -+ IPAddresses: addrs, -+ } -+ -+ infs = append(infs, inf) -+ } -+ -+ return infs -+} -diff --git a/virtcontainers/persist/api/network.go b/virtcontainers/persist/api/network.go -index 69610c67..53c6de44 100644 ---- a/virtcontainers/persist/api/network.go -+++ b/virtcontainers/persist/api/network.go -@@ -11,6 +11,27 @@ import ( - ) - - // ============= sandbox level resources ============= -+// DNSInfo describes the DNS setup related to a network interface. -+type DNSInfo struct { -+ Servers []string -+ Domain string -+ Searches []string -+ Options []string -+} -+ -+// NetlinkIface describes fully a network interface. -+type NetlinkIface struct { -+ netlink.LinkAttrs -+ Type string -+} -+ -+type NetworkProperties struct { -+ Device string -+ Iface NetlinkIface -+ Addrs []netlink.Addr -+ Routes []netlink.Route -+ DNS DNSInfo -+} - - type NetworkInterface struct { - Name string -@@ -91,6 +112,9 @@ type NetworkEndpoint struct { - Tap *TapEndpoint `json:",omitempty"` - IPVlan *IPVlanEndpoint `json:",omitempty"` - Tuntap *TuntapEndpoint `json:",omitempty"` -+ -+ // store the endpoint properties info -+ EndPointProperties *NetworkProperties `json:",omitempty"` - } - - // NetworkInfo contains network information of sandbox -diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go -index 7d33d5a2..c897670e 100644 ---- a/virtcontainers/tap_endpoint.go -+++ b/virtcontainers/tap_endpoint.go -@@ -206,12 +206,15 @@ func unTapNetwork(name string) error { - - func (endpoint *TapEndpoint) save() persistapi.NetworkEndpoint { - tapif := saveTapIf(&endpoint.TapInterface) -+ // save tap endpoint network properties into persist storage -+ properties := saveTapEndpointProperties(&endpoint.EndpointProperties) - - return persistapi.NetworkEndpoint{ - Type: string(endpoint.Type()), - Tap: &persistapi.TapEndpoint{ - TapInterface: *tapif, - }, -+ EndPointProperties: properties, - } - } - func (endpoint *TapEndpoint) load(s persistapi.NetworkEndpoint) { -@@ -221,4 +224,10 @@ func (endpoint *TapEndpoint) load(s persistapi.NetworkEndpoint) { - tapif := loadTapIf(&s.Tap.TapInterface) - endpoint.TapInterface = *tapif - } -+ -+ if s.EndPointProperties != nil { -+ // restore tap endpoint network properties from persist storage -+ properties := loadTapEndpointProperties(s.EndPointProperties) -+ endpoint.EndpointProperties = *properties -+ } - } --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0026-network-add-enable_compat_old_cni-config.patch b/runtime/patches/0026-network-add-enable_compat_old_cni-config.patch deleted file mode 100644 index 7578b55..0000000 --- a/runtime/patches/0026-network-add-enable_compat_old_cni-config.patch +++ /dev/null @@ -1,285 +0,0 @@ -From eeca1e47e9a6422d89d08275864f2c1b15e54941 Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Fri, 14 Aug 2020 19:14:35 +0800 -Subject: [PATCH 26/50] network: add enable_compat_old_cni config - -reason: old version kata-network list-ifaces output result different -from the community version, inorder to compatible with the old version -list-ifaces command output format, add enable_compat_old_cni to control -the list-ifaces command output format. - -Signed-off-by: jiangpengfei ---- - cli/config/configuration-qemu.toml.in | 4 ++++ - cli/network.go | 17 +++++++++++++++-- - cli/network_test.go | 6 ++++++ - pkg/katautils/config.go | 2 ++ - virtcontainers/api.go | 10 +++++++--- - virtcontainers/interfaces.go | 1 + - virtcontainers/network.go | 11 ++++++----- - virtcontainers/persist.go | 18 ++++++++++-------- - virtcontainers/persist/api/config.go | 9 +++++---- - virtcontainers/pkg/oci/utils.go | 4 ++++ - virtcontainers/pkg/vcmock/sandbox.go | 5 +++++ - virtcontainers/sandbox.go | 5 +++++ - 12 files changed, 70 insertions(+), 22 deletions(-) - -diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in -index b44e84d8..46f8b632 100644 ---- a/cli/config/configuration-qemu.toml.in -+++ b/cli/config/configuration-qemu.toml.in -@@ -453,6 +453,10 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@ - # (default: false) - #disable_new_netns = true - -+# If enabled, the kata-network will return the old interface format info to be compatible with -+# old version CNI plugin -+enable_compat_old_cni = true -+ - # if enabled, the runtime will add all the kata processes inside one dedicated cgroup. - # The container cgroups in the host are not created, just one single cgroup per sandbox. - # The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox. -diff --git a/cli/network.go b/cli/network.go -index 66955725..a1a24425 100644 ---- a/cli/network.go -+++ b/cli/network.go -@@ -251,8 +251,21 @@ func networkListCommand(ctx context.Context, containerID string, opType networkT - kataLog.WithField("existing-interfaces", fmt.Sprintf("%+v", interfaces)). - WithError(err).Error("list interfaces failed") - } -- compatInfs := convertCompatInterfaces(interfaces) -- json.NewEncoder(file).Encode(compatInfs) -+ -+ sandbox, err := vci.FetchSandbox(ctx, sandboxID) -+ if err != nil { -+ kataLog.WithField("existing-interfaces", fmt.Sprintf("%+v", interfaces)). -+ WithError(err).Error("fetch sandbox failed") -+ } -+ -+ // If sandbox network config need to be compatible with old CNI, -+ // convert the interface format to old version format. -+ if sandbox.IsCompatOldCNI() { -+ compatInfs := convertCompatInterfaces(interfaces) -+ json.NewEncoder(file).Encode(compatInfs) -+ } else { -+ json.NewEncoder(file).Encode(interfaces) -+ } - case routeType: - var routes []*vcTypes.Route - routes, err = vci.ListRoutes(ctx, sandboxID) -diff --git a/cli/network_test.go b/cli/network_test.go -index 4e3d943d..95fd2749 100644 ---- a/cli/network_test.go -+++ b/cli/network_test.go -@@ -17,6 +17,7 @@ import ( - - vc "github.com/kata-containers/runtime/virtcontainers" - vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" -+ "github.com/kata-containers/runtime/virtcontainers/pkg/vcmock" - "github.com/kata-containers/runtime/virtcontainers/types" - ) - -@@ -59,6 +60,10 @@ func TestNetworkCliFunction(t *testing.T) { - return newSingleContainerStatus(testContainerID, state, map[string]string{}, &specs.Spec{}), nil - } - -+ testingImpl.FetchSandboxFunc = func(ctx context.Context, id string) (vc.VCSandbox, error) { -+ return &vcmock.Sandbox{}, nil -+ } -+ - defer func() { - testingImpl.AddInterfaceFunc = nil - testingImpl.RemoveInterfaceFunc = nil -@@ -66,6 +71,7 @@ func TestNetworkCliFunction(t *testing.T) { - testingImpl.UpdateRoutesFunc = nil - testingImpl.ListRoutesFunc = nil - testingImpl.StatusContainerFunc = nil -+ testingImpl.FetchSandboxFunc = nil - }() - - set := flag.NewFlagSet("", 0) -diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go -index 9a99b9d4..94c916a0 100644 ---- a/pkg/katautils/config.go -+++ b/pkg/katautils/config.go -@@ -141,6 +141,7 @@ type runtime struct { - Debug bool `toml:"enable_debug"` - Tracing bool `toml:"enable_tracing"` - DisableNewNetNs bool `toml:"disable_new_netns"` -+ EnableCompatOldCNI bool `toml:"enable_compat_old_cni"` - DisableGuestSeccomp bool `toml:"disable_guest_seccomp"` - SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"` - Experimental []string `toml:"experimental"` -@@ -1235,6 +1236,7 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool, debugFlag - - config.SandboxCgroupOnly = tomlConf.Runtime.SandboxCgroupOnly - config.DisableNewNetNs = tomlConf.Runtime.DisableNewNetNs -+ config.EnableCompatOldCNI = tomlConf.Runtime.EnableCompatOldCNI - for _, f := range tomlConf.Runtime.Experimental { - feature := exp.Get(f) - if feature == nil { -diff --git a/virtcontainers/api.go b/virtcontainers/api.go -index eb5b4995..fb044fe1 100644 ---- a/virtcontainers/api.go -+++ b/virtcontainers/api.go -@@ -949,9 +949,13 @@ func ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTypes.Interface - } - defer s.releaseStatelessSandbox() - -- // get interfaces info from persist.json file -- // instead of by s.ListInterfaces() -- return convertToCompatInterfaces(&s.networkNS.Endpoints), nil -+ // If enable_compat_old_cni is enabled, get interfaces info from -+ // persist.json file instead of by s.ListInterfaces() -+ if s.config.NetworkConfig.EnableCompatOldCNI { -+ return convertToCompatInterfaces(&s.networkNS.Endpoints), nil -+ } -+ -+ return s.ListInterfaces() - } - - // UpdateRoutes is the virtcontainers update routes entry point. -diff --git a/virtcontainers/interfaces.go b/virtcontainers/interfaces.go -index fa6b584e..499b386e 100644 ---- a/virtcontainers/interfaces.go -+++ b/virtcontainers/interfaces.go -@@ -98,6 +98,7 @@ type VCSandbox interface { - ListInterfaces() ([]*vcTypes.Interface, error) - UpdateRoutes(routes []*vcTypes.Route) ([]*vcTypes.Route, error) - ListRoutes() ([]*vcTypes.Route, error) -+ IsCompatOldCNI() bool - } - - // VCContainer is the Container interface -diff --git a/virtcontainers/network.go b/virtcontainers/network.go -index bf7f9336..db235cf6 100644 ---- a/virtcontainers/network.go -+++ b/virtcontainers/network.go -@@ -155,11 +155,12 @@ type NetworkInterfacePair struct { - - // NetworkConfig is the network configuration related to a network. - type NetworkConfig struct { -- NetNSPath string -- NetNsCreated bool -- DisableNewNetNs bool -- NetmonConfig NetmonConfig -- InterworkingModel NetInterworkingModel -+ NetNSPath string -+ NetNsCreated bool -+ DisableNewNetNs bool -+ EnableCompatOldCNI bool -+ NetmonConfig NetmonConfig -+ InterworkingModel NetInterworkingModel - } - - func networkLogger() *logrus.Entry { -diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go -index 6bd09a0b..fe00bf9a 100644 ---- a/virtcontainers/persist.go -+++ b/virtcontainers/persist.go -@@ -187,10 +187,11 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) { - }, - ShimType: string(sconfig.ShimType), - NetworkConfig: persistapi.NetworkConfig{ -- NetNSPath: sconfig.NetworkConfig.NetNSPath, -- NetNsCreated: sconfig.NetworkConfig.NetNsCreated, -- DisableNewNetNs: sconfig.NetworkConfig.DisableNewNetNs, -- InterworkingModel: int(sconfig.NetworkConfig.InterworkingModel), -+ NetNSPath: sconfig.NetworkConfig.NetNSPath, -+ NetNsCreated: sconfig.NetworkConfig.NetNsCreated, -+ DisableNewNetNs: sconfig.NetworkConfig.DisableNewNetNs, -+ EnableCompatOldCNI: sconfig.NetworkConfig.EnableCompatOldCNI, -+ InterworkingModel: int(sconfig.NetworkConfig.InterworkingModel), - }, - - ShmSize: sconfig.ShmSize, -@@ -477,10 +478,11 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) { - }, - ShimType: ShimType(savedConf.ShimType), - NetworkConfig: NetworkConfig{ -- NetNSPath: savedConf.NetworkConfig.NetNSPath, -- NetNsCreated: savedConf.NetworkConfig.NetNsCreated, -- DisableNewNetNs: savedConf.NetworkConfig.DisableNewNetNs, -- InterworkingModel: NetInterworkingModel(savedConf.NetworkConfig.InterworkingModel), -+ NetNSPath: savedConf.NetworkConfig.NetNSPath, -+ NetNsCreated: savedConf.NetworkConfig.NetNsCreated, -+ DisableNewNetNs: savedConf.NetworkConfig.DisableNewNetNs, -+ EnableCompatOldCNI: savedConf.NetworkConfig.EnableCompatOldCNI, -+ InterworkingModel: NetInterworkingModel(savedConf.NetworkConfig.InterworkingModel), - }, - - ShmSize: savedConf.ShmSize, -diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go -index cfbee849..3a2df32b 100644 ---- a/virtcontainers/persist/api/config.go -+++ b/virtcontainers/persist/api/config.go -@@ -210,10 +210,11 @@ type ShimConfig struct { - - // NetworkConfig is the network configuration related to a network. - type NetworkConfig struct { -- NetNSPath string -- NetNsCreated bool -- DisableNewNetNs bool -- InterworkingModel int -+ NetNSPath string -+ NetNsCreated bool -+ DisableNewNetNs bool -+ EnableCompatOldCNI bool -+ InterworkingModel int - } - - type ContainerConfig struct { -diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go -index 36c730b7..948bd3cb 100644 ---- a/virtcontainers/pkg/oci/utils.go -+++ b/virtcontainers/pkg/oci/utils.go -@@ -130,6 +130,9 @@ type RuntimeConfig struct { - //Determines if create a netns for hypervisor process - DisableNewNetNs bool - -+ // Determines if compatible with old CNI plugin -+ EnableCompatOldCNI bool -+ - //Determines kata processes are managed only in sandbox cgroup - SandboxCgroupOnly bool - -@@ -275,6 +278,7 @@ func networkConfig(ocispec specs.Spec, config RuntimeConfig) (vc.NetworkConfig, - } - netConf.InterworkingModel = config.InterNetworkModel - netConf.DisableNewNetNs = config.DisableNewNetNs -+ netConf.EnableCompatOldCNI = config.EnableCompatOldCNI - - netConf.NetmonConfig = vc.NetmonConfig{ - Path: config.NetmonConfig.Path, -diff --git a/virtcontainers/pkg/vcmock/sandbox.go b/virtcontainers/pkg/vcmock/sandbox.go -index 677457ef..11b83ccd 100644 ---- a/virtcontainers/pkg/vcmock/sandbox.go -+++ b/virtcontainers/pkg/vcmock/sandbox.go -@@ -212,3 +212,8 @@ func (s *Sandbox) UpdateRoutes(routes []*vcTypes.Route) ([]*vcTypes.Route, error - func (s *Sandbox) ListRoutes() ([]*vcTypes.Route, error) { - return nil, nil - } -+ -+// IsCompatOldCNI return the whether enable compatible with old CNI -+func (s *Sandbox) IsCompatOldCNI() bool { -+ return false -+} -diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go -index c8981a41..6a643a12 100644 ---- a/virtcontainers/sandbox.go -+++ b/virtcontainers/sandbox.go -@@ -2379,6 +2379,11 @@ func (s *Sandbox) setContainersState(state types.StateString) error { - return nil - } - -+// IsCompatOldCNI return the whether enable compatible with old CNI -+func (s *Sandbox) IsCompatOldCNI() bool { -+ return s.config.NetworkConfig.EnableCompatOldCNI -+} -+ - // updateStaticSandboxResources update sandbox's cpu and memory resource passed by - // sandbox_cpu and sandbox_mem annotations - func updateStaticSandboxResources(sandboxConfig *SandboxConfig) error { --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0027-network-add-more-strict-check-for-input-network-inte.patch b/runtime/patches/0027-network-add-more-strict-check-for-input-network-inte.patch deleted file mode 100644 index d0b0ede..0000000 --- a/runtime/patches/0027-network-add-more-strict-check-for-input-network-inte.patch +++ /dev/null @@ -1,182 +0,0 @@ -From ec15337fc816767ca0e8183576405499080b9b1e Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Sun, 16 Aug 2020 16:41:18 +0800 -Subject: [PATCH 27/50] network: add more strict check for input network - interface - -reason: kata-network add-iface command will receive the network -interface info from untrust user, so we need to add more strict -check for input network interface info. - -Signed-off-by: jiangpengfei ---- - virtcontainers/network.go | 117 ++++++++++++++++++++++++++++++++++++++++++++++ - virtcontainers/sandbox.go | 11 +++++ - 2 files changed, 128 insertions(+) - -diff --git a/virtcontainers/network.go b/virtcontainers/network.go -index db235cf6..a1676ccd 100644 ---- a/virtcontainers/network.go -+++ b/virtcontainers/network.go -@@ -13,8 +13,10 @@ import ( - "math/rand" - "net" - "os" -+ "regexp" - "runtime" - "sort" -+ "strings" - "time" - - "github.com/containernetworking/plugins/pkg/ns" -@@ -55,6 +57,17 @@ const ( - NetXConnectInvalidModel - ) - -+const ( -+ maxIPAddrLen = 18 -+ maxInterfaceLen = 15 -+ minMTUVal = 46 -+ maxMTUVal = 9600 -+) -+ -+var ( -+ regInfName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_\-.]*$`) -+) -+ - //IsValid checks if a model is valid - func (n NetInterworkingModel) IsValid() bool { - return 0 <= int(n) && int(n) < int(NetXConnectInvalidModel) -@@ -1368,3 +1381,107 @@ func convertToCompatInterfaces(es *[]Endpoint) []*vcTypes.Interface { - - return infs - } -+ -+// verifyInterfaceName verifies the interface name valid or not -+func verifyInterfaceName(name string) error { -+ // verify `Name` before `Tapname` because of the strict rules -+ name = strings.TrimSpace(name) -+ if len(name) > maxInterfaceLen { -+ return fmt.Errorf("interface name can't be longer than 15") -+ } else if len(name) == 0 || name == "\n" { -+ return fmt.Errorf("interface name can't be empty") -+ } -+ -+ // QMP rules of `Name` -+ chk := regInfName.FindAllString(name, -1) -+ if chk == nil { -+ return fmt.Errorf("invalid input of interface name, please check the rules") -+ } -+ -+ return nil -+} -+ -+// verifyIPAndMask verifies CIDR notation IP address and mask like "192.0.2.0/24" -+func verifyIPAndMask(ip string) (nlIpMask *net.IPNet, err error) { -+ ip = strings.TrimSpace(ip) -+ -+ if len(ip) > maxIPAddrLen { -+ return nil, fmt.Errorf("the length of IP address is too long, max ip len :%d", maxIPAddrLen) -+ } -+ -+ if nlIpMask, err = netlink.ParseIPNet(ip); err != nil { -+ return nil, fmt.Errorf("invalid input of IP : %v", err) -+ } -+ -+ return nlIpMask, nil -+} -+ -+// verifyMac verifies MAC address -+func verifyMac(mac string) error { -+ mac = strings.TrimSpace(mac) -+ if _, err := net.ParseMAC(mac); err != nil { -+ return fmt.Errorf("invalid input of Mac : %v", err) -+ } -+ -+ return nil -+} -+ -+// verifyMtu verifies MTU value of interface -+func verifyMtu(mtu uint64) error { -+ if mtu < minMTUVal || mtu > maxMTUVal { -+ return fmt.Errorf("invalid input of MTU : %v", mtu) -+ } -+ return nil -+} -+ -+// verifyIP verifies the IP address -+func verifyIP(ip string) (*net.IP, error) { -+ ip = strings.TrimSpace(ip) -+ -+ if len(ip) > maxIPAddrLen { -+ return nil, fmt.Errorf("the length of IP address is too long, max ip len :%d", maxIPAddrLen) -+ } -+ -+ var netIP net.IP -+ if netIP = net.ParseIP(ip); netIP == nil { -+ return nil, fmt.Errorf("invalid IP: %s", ip) -+ } -+ -+ return &netIP, nil -+} -+ -+// validInterface check the input interface valid or not -+func validInterface(inf *vcTypes.Interface, enableCompatOldCNI bool) error { -+ if enableCompatOldCNI && verifyInterfaceName(inf.Device) != nil { -+ return fmt.Errorf("device name should not be empty when enable_compat_old_cni config enabled") -+ } -+ -+ if inf.Name == "" || inf.Mtu == 0 || inf.HwAddr == "" { -+ return fmt.Errorf("name/mtu/hwaddr of interface must be specified") -+ } -+ -+ if err := verifyInterfaceName(inf.Name); err != nil { -+ return err -+ } -+ -+ if err := verifyMac(inf.HwAddr); err != nil { -+ return err -+ } -+ -+ if err := verifyMtu(inf.Mtu); err != nil { -+ return err -+ } -+ -+ // Currently, only one IP address can be passed, which reduces the test entry and fault injection. -+ if len(inf.IPAddresses) > 0 { -+ if len(inf.IPAddresses) != 1 { -+ return fmt.Errorf("only one IP address is supported currently") -+ } -+ _, err := verifyIP(inf.IPAddresses[0].Address) -+ if err != nil { -+ return err -+ } -+ } -+ -+ return nil -+} -diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go -index 6a643a12..f6826812 100644 ---- a/virtcontainers/sandbox.go -+++ b/virtcontainers/sandbox.go -@@ -952,6 +952,17 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) { - - // AddInterface adds new nic to the sandbox. - func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (grpcIf *vcTypes.Interface, err error) { -+ err = validInterface(inf, s.config.NetworkConfig.EnableCompatOldCNI) -+ if err != nil { -+ return nil, err -+ } -+ -+ for _, ep := range s.networkNS.Endpoints { -+ if ep.Name() == inf.Name { -+ return nil, fmt.Errorf("interface %s is already exist", inf.Name) -+ } -+ } -+ - netInfo, err := s.generateNetInfo(inf) - if err != nil { - return nil, err --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0028-network-add-kata-network-add-route-subcommand.patch b/runtime/patches/0028-network-add-kata-network-add-route-subcommand.patch deleted file mode 100644 index 8b636a4..0000000 --- a/runtime/patches/0028-network-add-kata-network-add-route-subcommand.patch +++ /dev/null @@ -1,1026 +0,0 @@ -From 4d76ace47b422fedd9b6ae2f38ee09fc6d4c4bd4 Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Sun, 16 Aug 2020 17:43:41 +0800 -Subject: [PATCH 28/50] network: add kata-network add-route subcommand - -reason: add kata-network add-route subcommand to support add one route -operation. And keep compatible with community update-routes subcommand. - -Signed-off-by: jiangpengfei ---- - cli/network.go | 60 ++- - .../agent/protocols/grpc/agent.pb.go | 402 ++++++++++++--------- - virtcontainers/agent.go | 3 +- - virtcontainers/api.go | 4 +- - virtcontainers/api_test.go | 2 +- - virtcontainers/implementation.go | 4 +- - virtcontainers/interfaces.go | 4 +- - virtcontainers/kata_agent.go | 5 +- - virtcontainers/kata_agent_test.go | 2 +- - virtcontainers/network.go | 162 +++++++++ - virtcontainers/noop_agent.go | 4 +- - virtcontainers/noop_agent_test.go | 2 +- - virtcontainers/pkg/types/types.go | 14 + - virtcontainers/pkg/vcmock/mock.go | 2 +- - virtcontainers/pkg/vcmock/sandbox.go | 2 +- - virtcontainers/sandbox.go | 29 +- - 16 files changed, 492 insertions(+), 209 deletions(-) - -diff --git a/cli/network.go b/cli/network.go -index a1a24425..2265f54b 100644 ---- a/cli/network.go -+++ b/cli/network.go -@@ -44,6 +44,7 @@ var kataNetworkCLICommand = cli.Command{ - listIfacesCommand, - updateRoutesCommand, - listRoutesCommand, -+ addRoutesCommand, - }, - Action: func(context *cli.Context) error { - return cli.ShowSubcommandHelp(context) -@@ -73,7 +74,7 @@ var addIfaceCommand = cli.Command{ - return err - } - -- return networkModifyCommand(ctx, context.Args().First(), context.Args().Get(1), interfaceType, true) -+ return networkModifyCommand(ctx, context.Args().First(), context.Args().Get(1), interfaceType, vcTypes.NetworkOpAdd) - }, - } - -@@ -98,7 +99,7 @@ var delIfaceCommand = cli.Command{ - return err - } - -- return networkModifyCommand(ctx, context.Args().First(), context.Args().Get(1), interfaceType, false) -+ return networkModifyCommand(ctx, context.Args().First(), context.Args().Get(1), interfaceType, vcTypes.NetworkOpRemove) - }, - } - -@@ -128,7 +129,29 @@ var updateRoutesCommand = cli.Command{ - return err - } - -- return networkModifyCommand(ctx, context.Args().First(), context.Args().Get(1), routeType, true) -+ return networkModifyCommand(ctx, context.Args().First(), context.Args().Get(1), routeType, vcTypes.NetworkOpUpdate) -+ }, -+} -+ -+var addRoutesCommand = cli.Command{ -+ Name: "add-route", -+ Usage: "add one route for a container", -+ ArgsUsage: `add-route file or - for stdin -+ file or stdin for example: -+ { -+ "dest":"<[[/mask] | "default" ]>", -+ "gateway":"[ip]", -+ "device":"", -+ } -+ `, -+ Flags: []cli.Flag{}, -+ Action: func(context *cli.Context) error { -+ ctx, err := cliContextToContext(context) -+ if err != nil { -+ return err -+ } -+ -+ return networkModifyCommand(ctx, context.Args().First(), context.Args().Get(1), routeType, vcTypes.NetworkOpAdd) - }, - } - -@@ -147,7 +170,7 @@ var listRoutesCommand = cli.Command{ - }, - } - --func networkModifyCommand(ctx context.Context, containerID, input string, opType networkType, add bool) (err error) { -+func networkModifyCommand(ctx context.Context, containerID, input string, opType networkType, op vcTypes.NetworkOp) (err error) { - status, sandboxID, err := getExistingContainerInfo(ctx, containerID) - if err != nil { - return err -@@ -188,30 +211,45 @@ func networkModifyCommand(ctx context.Context, containerID, input string, opType - return err - } - -- if len(inf.LinkType) == 0 { -+ if op != vcTypes.NetworkOpUpdate && inf.LinkType == "" { - inf.LinkType = defaultLinkType - } - -- if add { -+ switch op { -+ case vcTypes.NetworkOpAdd: - resultingInf, err = vci.AddInterface(ctx, sandboxID, inf) - if err != nil { - kataLog.WithField("resulting-interface", fmt.Sprintf("%+v", resultingInf)). - WithError(err).Error("add interface failed") - } -- } else { -+ case vcTypes.NetworkOpRemove: - resultingInf, err = vci.RemoveInterface(ctx, sandboxID, inf) - if err != nil { - kataLog.WithField("resulting-interface", fmt.Sprintf("%+v", resultingInf)). - WithError(err).Error("delete interface failed") - } - } -+ - json.NewEncoder(output).Encode(resultingInf) - case routeType: -- var routes, resultingRoutes []*vcTypes.Route -- if err = json.NewDecoder(f).Decode(&routes); err != nil { -- return err -+ var ( -+ route *vcTypes.Route -+ routes []*vcTypes.Route -+ resultingRoutes []*vcTypes.Route -+ ) -+ -+ if op == vcTypes.NetworkOpUpdate { -+ if err = json.NewDecoder(f).Decode(&routes); err != nil { -+ return err -+ } -+ } else { -+ if err = json.NewDecoder(f).Decode(&route); err != nil { -+ return err -+ } -+ routes = append(routes, route) - } -- resultingRoutes, err = vci.UpdateRoutes(ctx, sandboxID, routes) -+ -+ resultingRoutes, err = vci.UpdateRoutes(ctx, sandboxID, routes, op) - json.NewEncoder(output).Encode(resultingRoutes) - if err != nil { - kataLog.WithField("resulting-routes", fmt.Sprintf("%+v", resultingRoutes)). -diff --git a/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go b/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go -index 77e6d1bc..1b887e55 100644 ---- a/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go -+++ b/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go -@@ -1303,7 +1303,8 @@ func (m *UpdateInterfaceRequest) GetInterface() *types.Interface { - } - - type UpdateRoutesRequest struct { -- Routes *Routes `protobuf:"bytes,1,opt,name=routes" json:"routes,omitempty"` -+ Routes *Routes `protobuf:"bytes,1,opt,name=routes" json:"routes,omitempty"` -+ Increment bool `protobuf:"varint,2,opt,name=increment,proto3" json:"increment,omitempty"` - } - - func (m *UpdateRoutesRequest) Reset() { *m = UpdateRoutesRequest{} } -@@ -1318,6 +1319,13 @@ func (m *UpdateRoutesRequest) GetRoutes() *Routes { - return nil - } - -+func (m *UpdateRoutesRequest) GetIncrement() bool { -+ if m != nil { -+ return m.Increment -+ } -+ return false -+} -+ - type ListInterfacesRequest struct { - } - -@@ -4476,6 +4484,16 @@ func (m *UpdateRoutesRequest) MarshalTo(dAtA []byte) (int, error) { - } - i += n20 - } -+ if m.Increment { -+ dAtA[i] = 0x10 -+ i++ -+ if m.Increment { -+ dAtA[i] = 1 -+ } else { -+ dAtA[i] = 0 -+ } -+ i++ -+ } - return i, nil - } - -@@ -5751,6 +5769,9 @@ func (m *UpdateRoutesRequest) Size() (n int) { - l = m.Routes.Size() - n += 1 + l + sovAgent(uint64(l)) - } -+ if m.Increment { -+ n += 2 -+ } - return n - } - -@@ -10964,6 +10985,26 @@ func (m *UpdateRoutesRequest) Unmarshal(dAtA []byte) error { - return err - } - iNdEx = postIndex -+ case 2: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field Increment", wireType) -+ } -+ var v int -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ v |= (int(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ m.Increment = bool(v != 0) - default: - iNdEx = preIndex - skippy, err := skipAgent(dAtA[iNdEx:]) -@@ -12852,184 +12893,185 @@ var ( - func init() { proto.RegisterFile("agent.proto", fileDescriptorAgent) } - - var fileDescriptorAgent = []byte{ -- // 2862 bytes of a gzipped FileDescriptorProto -+ // 2876 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0x4b, 0x6f, 0x1c, 0xc7, -- 0xd1, 0xd8, 0x07, 0x97, 0xbb, 0xb5, 0x2f, 0x6e, 0x93, 0xa2, 0x56, 0x2b, 0x5b, 0x9f, 0x3c, 0xb6, -- 0x65, 0xfa, 0x73, 0xbc, 0xb4, 0x65, 0x23, 0x7e, 0xc1, 0x11, 0xc4, 0x47, 0x44, 0xc6, 0x56, 0xc4, -- 0x0c, 0x45, 0x38, 0x40, 0x10, 0x0c, 0x86, 0x33, 0xcd, 0x65, 0x9b, 0x3b, 0xd3, 0xe3, 0x9e, 0x1e, -- 0x8a, 0xeb, 0x00, 0x39, 0x26, 0xb7, 0x5c, 0x02, 0xe4, 0x96, 0x3f, 0x10, 0xe4, 0x96, 0x63, 0xae, -- 0x39, 0x18, 0x39, 0xe5, 0x17, 0x04, 0x81, 0x7f, 0x42, 0x7e, 0x41, 0xd0, 0xaf, 0x79, 0xec, 0x0e, -- 0x29, 0x84, 0x20, 0x90, 0xcb, 0xa2, 0xab, 0xba, 0xba, 0x5e, 0xdd, 0x55, 0x53, 0x55, 0x0b, 0x6d, -- 0x77, 0x82, 0x43, 0x3e, 0x8e, 0x18, 0xe5, 0x14, 0xd5, 0x27, 0x2c, 0xf2, 0x46, 0x2d, 0xea, 0x11, -- 0x85, 0x18, 0xfd, 0x70, 0x42, 0xf8, 0x69, 0x72, 0x3c, 0xf6, 0x68, 0xb0, 0x79, 0xe6, 0x72, 0xf7, -- 0x5d, 0x8f, 0x86, 0xdc, 0x25, 0x21, 0x66, 0xf1, 0xa6, 0x3c, 0xb8, 0x19, 0x9d, 0x4d, 0x36, 0xf9, -- 0x2c, 0xc2, 0xb1, 0xfa, 0xd5, 0xe7, 0xee, 0x4e, 0x28, 0x9d, 0x4c, 0xf1, 0xa6, 0x84, 0x8e, 0x93, -- 0x93, 0x4d, 0x1c, 0x44, 0x7c, 0xa6, 0x36, 0xad, 0x3f, 0x56, 0x61, 0x7d, 0x9b, 0x61, 0x97, 0xe3, -- 0x6d, 0xc3, 0xcd, 0xc6, 0xdf, 0x24, 0x38, 0xe6, 0xe8, 0x35, 0xe8, 0xa4, 0x12, 0x1c, 0xe2, 0x0f, -- 0x2b, 0xf7, 0x2b, 0x1b, 0x2d, 0xbb, 0x9d, 0xe2, 0xf6, 0x7d, 0x74, 0x1b, 0x96, 0xf1, 0x05, 0xf6, -- 0xc4, 0x6e, 0x55, 0xee, 0x36, 0x04, 0xb8, 0xef, 0xa3, 0xf7, 0xa1, 0x1d, 0x73, 0x46, 0xc2, 0x89, -- 0x93, 0xc4, 0x98, 0x0d, 0x6b, 0xf7, 0x2b, 0x1b, 0xed, 0x87, 0x2b, 0x63, 0x61, 0xd2, 0xf8, 0x50, -- 0x6e, 0x1c, 0xc5, 0x98, 0xd9, 0x10, 0xa7, 0x6b, 0xf4, 0x00, 0x96, 0x7d, 0x7c, 0x4e, 0x3c, 0x1c, -- 0x0f, 0xeb, 0xf7, 0x6b, 0x1b, 0xed, 0x87, 0x1d, 0x45, 0xbe, 0x23, 0x91, 0xb6, 0xd9, 0x44, 0x6f, -- 0x43, 0x33, 0xe6, 0x94, 0xb9, 0x13, 0x1c, 0x0f, 0x97, 0x24, 0x61, 0xd7, 0xf0, 0x95, 0x58, 0x3b, -- 0xdd, 0x46, 0xaf, 0x40, 0xed, 0xd9, 0xf6, 0xfe, 0xb0, 0x21, 0xa5, 0x83, 0xa6, 0x8a, 0xb0, 0x67, -- 0x0b, 0x34, 0x7a, 0x1d, 0xba, 0xb1, 0x1b, 0xfa, 0xc7, 0xf4, 0xc2, 0x89, 0x88, 0x1f, 0xc6, 0xc3, -- 0xe5, 0xfb, 0x95, 0x8d, 0xa6, 0xdd, 0xd1, 0xc8, 0x03, 0x81, 0xb3, 0x3e, 0x85, 0x5b, 0x87, 0xdc, -- 0x65, 0xfc, 0x1a, 0xde, 0xb1, 0x8e, 0x60, 0xdd, 0xc6, 0x01, 0x3d, 0xbf, 0x96, 0x6b, 0x87, 0xb0, -- 0xcc, 0x49, 0x80, 0x69, 0xc2, 0xa5, 0x6b, 0xbb, 0xb6, 0x01, 0xad, 0x3f, 0x57, 0x00, 0xed, 0x5e, -- 0x60, 0xef, 0x80, 0x51, 0x0f, 0xc7, 0xf1, 0xff, 0xe8, 0xba, 0xde, 0x82, 0xe5, 0x48, 0x29, 0x30, -- 0xac, 0x4b, 0x72, 0x7d, 0x0b, 0x46, 0x2b, 0xb3, 0x6b, 0x7d, 0x0d, 0x6b, 0x87, 0x64, 0x12, 0xba, -- 0xd3, 0x1b, 0xd4, 0x77, 0x1d, 0x1a, 0xb1, 0xe4, 0x29, 0x55, 0xed, 0xda, 0x1a, 0xb2, 0x0e, 0x00, -- 0x7d, 0xe5, 0x12, 0x7e, 0x73, 0x92, 0xac, 0x77, 0x61, 0xb5, 0xc0, 0x31, 0x8e, 0x68, 0x18, 0x63, -- 0xa9, 0x00, 0x77, 0x79, 0x12, 0x4b, 0x66, 0x4b, 0xb6, 0x86, 0x2c, 0x0c, 0x6b, 0x5f, 0x92, 0xd8, -- 0x90, 0xe3, 0xff, 0x46, 0x85, 0x75, 0x68, 0x9c, 0x50, 0x16, 0xb8, 0xdc, 0x68, 0xa0, 0x20, 0x84, -- 0xa0, 0xee, 0xb2, 0x49, 0x3c, 0xac, 0xdd, 0xaf, 0x6d, 0xb4, 0x6c, 0xb9, 0x16, 0xaf, 0x72, 0x4e, -- 0x8c, 0xd6, 0xeb, 0x35, 0xe8, 0x68, 0xbf, 0x3b, 0x53, 0x12, 0x73, 0x29, 0xa7, 0x63, 0xb7, 0x35, -- 0x4e, 0x9c, 0xb1, 0x28, 0xac, 0x1f, 0x45, 0xfe, 0x35, 0x03, 0xfe, 0x21, 0xb4, 0x18, 0x8e, 0x69, -- 0xc2, 0x44, 0x98, 0x56, 0xe5, 0xbd, 0xaf, 0xa9, 0x7b, 0xff, 0x92, 0x84, 0xc9, 0x85, 0x6d, 0xf6, -- 0xec, 0x8c, 0x4c, 0x87, 0x10, 0x8f, 0xaf, 0x13, 0x42, 0x9f, 0xc2, 0xad, 0x03, 0x37, 0x89, 0xaf, -- 0xa3, 0xab, 0xf5, 0x99, 0x08, 0xbf, 0x38, 0x09, 0xae, 0x75, 0xf8, 0x4f, 0x15, 0x68, 0x6e, 0x47, -- 0xc9, 0x51, 0xec, 0x4e, 0x30, 0xfa, 0x3f, 0x68, 0x73, 0xca, 0xdd, 0xa9, 0x93, 0x08, 0x50, 0x92, -- 0xd7, 0x6d, 0x90, 0x28, 0x45, 0x20, 0xdc, 0x8e, 0x99, 0x17, 0x25, 0x9a, 0xa2, 0x7a, 0xbf, 0xb6, -- 0x51, 0xb7, 0xdb, 0x0a, 0xa7, 0x48, 0xc6, 0xb0, 0x2a, 0xf7, 0x1c, 0x12, 0x3a, 0x67, 0x98, 0x85, -- 0x78, 0x1a, 0x50, 0x1f, 0xcb, 0xf7, 0x5b, 0xb7, 0x07, 0x72, 0x6b, 0x3f, 0xfc, 0x22, 0xdd, 0x40, -- 0xff, 0x0f, 0x83, 0x94, 0x5e, 0x04, 0xa5, 0xa4, 0xae, 0x4b, 0xea, 0xbe, 0xa6, 0x3e, 0xd2, 0x68, -- 0xeb, 0xd7, 0xd0, 0x7b, 0x7e, 0xca, 0x28, 0xe7, 0x53, 0x12, 0x4e, 0x76, 0x5c, 0xee, 0x8a, 0xec, -- 0x11, 0x61, 0x46, 0xa8, 0x1f, 0x6b, 0x6d, 0x0d, 0x88, 0xde, 0x81, 0x01, 0x57, 0xb4, 0xd8, 0x77, -- 0x0c, 0x4d, 0x55, 0xd2, 0xac, 0xa4, 0x1b, 0x07, 0x9a, 0xf8, 0x4d, 0xe8, 0x65, 0xc4, 0x22, 0xff, -- 0x68, 0x7d, 0xbb, 0x29, 0xf6, 0x39, 0x09, 0xb0, 0x75, 0x2e, 0x7d, 0x25, 0x2f, 0x19, 0xbd, 0x03, -- 0xad, 0xcc, 0x0f, 0x15, 0xf9, 0x42, 0x7a, 0xea, 0x85, 0x18, 0x77, 0xda, 0xcd, 0xd4, 0x29, 0x9f, -- 0x43, 0x9f, 0xa7, 0x8a, 0x3b, 0xbe, 0xcb, 0xdd, 0xe2, 0xa3, 0x2a, 0x5a, 0x65, 0xf7, 0x78, 0x01, -- 0xb6, 0x3e, 0x83, 0xd6, 0x01, 0xf1, 0x63, 0x25, 0x78, 0x08, 0xcb, 0x5e, 0xc2, 0x18, 0x0e, 0xb9, -- 0x31, 0x59, 0x83, 0x68, 0x0d, 0x96, 0xa6, 0x24, 0x20, 0x5c, 0x9b, 0xa9, 0x00, 0x8b, 0x02, 0x3c, -- 0xc5, 0x01, 0x65, 0x33, 0xe9, 0xb0, 0x35, 0x58, 0xca, 0x5f, 0xae, 0x02, 0xd0, 0x5d, 0x68, 0x05, -- 0xee, 0x45, 0x7a, 0xa9, 0x62, 0xa7, 0x19, 0xb8, 0x17, 0x4a, 0xf9, 0x21, 0x2c, 0x9f, 0xb8, 0x64, -- 0xea, 0x85, 0x5c, 0x7b, 0xc5, 0x80, 0x99, 0xc0, 0x7a, 0x5e, 0xe0, 0xdf, 0xaa, 0xd0, 0x56, 0x12, -- 0x95, 0xc2, 0x6b, 0xb0, 0xe4, 0xb9, 0xde, 0x69, 0x2a, 0x52, 0x02, 0xe8, 0x81, 0x51, 0xa4, 0x9a, -- 0x4f, 0xc2, 0x99, 0xa6, 0x46, 0xb5, 0x4d, 0x80, 0xf8, 0x85, 0x1b, 0x69, 0xdd, 0x6a, 0x97, 0x10, -- 0xb7, 0x04, 0x8d, 0x52, 0xf7, 0x03, 0xe8, 0xa8, 0x77, 0xa7, 0x8f, 0xd4, 0x2f, 0x39, 0xd2, 0x56, -- 0x54, 0xea, 0xd0, 0xeb, 0xd0, 0x4d, 0x62, 0xec, 0x9c, 0x12, 0xcc, 0x5c, 0xe6, 0x9d, 0xce, 0x86, -- 0x4b, 0xea, 0x1b, 0x99, 0xc4, 0x78, 0xcf, 0xe0, 0xd0, 0x43, 0x58, 0x12, 0xe9, 0x2f, 0x1e, 0x36, -- 0xe4, 0xe7, 0xf8, 0x95, 0x3c, 0x4b, 0x69, 0xea, 0x58, 0xfe, 0xee, 0x86, 0x9c, 0xcd, 0x6c, 0x45, -- 0x3a, 0xfa, 0x18, 0x20, 0x43, 0xa2, 0x15, 0xa8, 0x9d, 0xe1, 0x99, 0x8e, 0x43, 0xb1, 0x14, 0xce, -- 0x39, 0x77, 0xa7, 0x89, 0xf1, 0xba, 0x02, 0x3e, 0xad, 0x7e, 0x5c, 0xb1, 0x3c, 0xe8, 0x6f, 0x4d, -- 0xcf, 0x08, 0xcd, 0x1d, 0x5f, 0x83, 0xa5, 0xc0, 0xfd, 0x9a, 0x32, 0xe3, 0x49, 0x09, 0x48, 0x2c, -- 0x09, 0x29, 0x33, 0x2c, 0x24, 0x80, 0x7a, 0x50, 0xa5, 0x91, 0xf4, 0x57, 0xcb, 0xae, 0xd2, 0x28, -- 0x13, 0x54, 0xcf, 0x09, 0xb2, 0xfe, 0x59, 0x07, 0xc8, 0xa4, 0x20, 0x1b, 0x46, 0x84, 0x3a, 0x31, -- 0x66, 0xa2, 0x04, 0x71, 0x8e, 0x67, 0x1c, 0xc7, 0x0e, 0xc3, 0x5e, 0xc2, 0x62, 0x72, 0x2e, 0xee, -- 0x4f, 0x98, 0x7d, 0x4b, 0x99, 0x3d, 0xa7, 0x9b, 0x7d, 0x9b, 0xd0, 0x43, 0x75, 0x6e, 0x4b, 0x1c, -- 0xb3, 0xcd, 0x29, 0xb4, 0x0f, 0xb7, 0x32, 0x9e, 0x7e, 0x8e, 0x5d, 0xf5, 0x2a, 0x76, 0xab, 0x29, -- 0x3b, 0x3f, 0x63, 0xb5, 0x0b, 0xab, 0x84, 0x3a, 0xdf, 0x24, 0x38, 0x29, 0x30, 0xaa, 0x5d, 0xc5, -- 0x68, 0x40, 0xe8, 0xcf, 0xe4, 0x81, 0x8c, 0xcd, 0x01, 0xdc, 0xc9, 0x59, 0x29, 0xc2, 0x3d, 0xc7, -- 0xac, 0x7e, 0x15, 0xb3, 0xf5, 0x54, 0x2b, 0x91, 0x0f, 0x32, 0x8e, 0x3f, 0x81, 0x75, 0x42, 0x9d, -- 0x17, 0x2e, 0xe1, 0xf3, 0xec, 0x96, 0x5e, 0x62, 0xa4, 0xf8, 0xe8, 0x16, 0x79, 0x29, 0x23, 0x03, -- 0xcc, 0x26, 0x05, 0x23, 0x1b, 0x2f, 0x31, 0xf2, 0xa9, 0x3c, 0x90, 0xb1, 0x79, 0x0c, 0x03, 0x42, -- 0xe7, 0xb5, 0x59, 0xbe, 0x8a, 0x49, 0x9f, 0xd0, 0xa2, 0x26, 0x5b, 0x30, 0x88, 0xb1, 0xc7, 0x29, -- 0xcb, 0x3f, 0x82, 0xe6, 0x55, 0x2c, 0x56, 0x34, 0x7d, 0xca, 0xc3, 0xfa, 0x05, 0x74, 0xf6, 0x92, -- 0x09, 0xe6, 0xd3, 0xe3, 0x34, 0x19, 0xdc, 0x58, 0xfe, 0xb1, 0xfe, 0x5d, 0x85, 0xf6, 0xf6, 0x84, -- 0xd1, 0x24, 0x2a, 0xe4, 0x64, 0x15, 0xa4, 0xf3, 0x39, 0x59, 0x92, 0xc8, 0x9c, 0xac, 0x88, 0x3f, -- 0x84, 0x4e, 0x20, 0x43, 0x57, 0xd3, 0xab, 0x3c, 0x34, 0x58, 0x08, 0x6a, 0xbb, 0x1d, 0xe4, 0x92, -- 0xd9, 0x18, 0x20, 0x22, 0x7e, 0xac, 0xcf, 0xa8, 0x74, 0xd4, 0xd7, 0x15, 0xa1, 0x49, 0xd1, 0x76, -- 0x2b, 0x4a, 0xb3, 0xf5, 0xfb, 0xd0, 0x3e, 0x16, 0x4e, 0xd2, 0x07, 0x0a, 0xc9, 0x28, 0xf3, 0x9e, -- 0x0d, 0xc7, 0x59, 0x10, 0xee, 0x41, 0xf7, 0x54, 0xb9, 0x4c, 0x1f, 0x52, 0x6f, 0xe8, 0x75, 0x6d, -- 0x49, 0x66, 0xef, 0x38, 0xef, 0x59, 0x75, 0x01, 0x9d, 0xd3, 0x1c, 0x6a, 0x74, 0x08, 0x83, 0x05, -- 0x92, 0x92, 0x1c, 0xb4, 0x91, 0xcf, 0x41, 0xed, 0x87, 0x48, 0x09, 0xca, 0x9f, 0xcc, 0xe7, 0xa5, -- 0xdf, 0x55, 0xa1, 0xf3, 0x53, 0xcc, 0x5f, 0x50, 0x76, 0xa6, 0xf4, 0x45, 0x50, 0x0f, 0xdd, 0x00, -- 0x6b, 0x8e, 0x72, 0x8d, 0xee, 0x40, 0x93, 0x5d, 0xa8, 0x04, 0xa2, 0xef, 0x73, 0x99, 0x5d, 0xc8, -- 0xc4, 0x80, 0x5e, 0x05, 0x60, 0x17, 0x4e, 0xe4, 0x7a, 0x67, 0x58, 0x7b, 0xb0, 0x6e, 0xb7, 0xd8, -- 0xc5, 0x81, 0x42, 0x88, 0xa7, 0xc0, 0x2e, 0x1c, 0xcc, 0x18, 0x65, 0xb1, 0xce, 0x55, 0x4d, 0x76, -- 0xb1, 0x2b, 0x61, 0x7d, 0xd6, 0x67, 0x34, 0x8a, 0xb0, 0x2f, 0x73, 0xb4, 0x3c, 0xbb, 0xa3, 0x10, -- 0x42, 0x2a, 0x37, 0x52, 0x1b, 0x4a, 0x2a, 0xcf, 0xa4, 0xf2, 0x4c, 0xea, 0xb2, 0x3a, 0xc9, 0xf3, -- 0x52, 0x79, 0x2a, 0xb5, 0xa9, 0xa4, 0xf2, 0x9c, 0x54, 0x9e, 0x49, 0x6d, 0x99, 0xb3, 0x5a, 0xaa, -- 0xf5, 0xdb, 0x0a, 0xac, 0xcf, 0x17, 0x7e, 0xba, 0x4c, 0xfd, 0x10, 0x3a, 0x9e, 0xbc, 0xaf, 0xc2, -- 0x9b, 0x1c, 0x2c, 0xdc, 0xa4, 0xdd, 0xf6, 0x72, 0xcf, 0xf8, 0x23, 0xe8, 0x86, 0xca, 0xc1, 0xe9, -- 0xd3, 0xac, 0x65, 0xf7, 0x92, 0xf7, 0xbd, 0xdd, 0x09, 0x73, 0x90, 0xe5, 0x03, 0xfa, 0x8a, 0x11, -- 0x8e, 0x0f, 0x39, 0xc3, 0x6e, 0x70, 0x13, 0x0d, 0x08, 0x82, 0xba, 0xac, 0x56, 0x6a, 0xb2, 0xbe, -- 0x96, 0x6b, 0xeb, 0x2d, 0x58, 0x2d, 0x48, 0xd1, 0xb6, 0xae, 0x40, 0x6d, 0x8a, 0x43, 0xc9, 0xbd, -- 0x6b, 0x8b, 0xa5, 0xe5, 0xc2, 0xc0, 0xc6, 0xae, 0x7f, 0x73, 0xda, 0x68, 0x11, 0xb5, 0x4c, 0xc4, -- 0x06, 0xa0, 0xbc, 0x08, 0xad, 0x8a, 0xd1, 0xba, 0x92, 0xd3, 0xfa, 0x19, 0x0c, 0xb6, 0xa7, 0x34, -- 0xc6, 0x87, 0xdc, 0x27, 0xe1, 0x4d, 0x74, 0x4c, 0xbf, 0x82, 0xd5, 0xe7, 0x7c, 0xf6, 0x95, 0x60, -- 0x16, 0x93, 0x6f, 0xf1, 0x0d, 0xd9, 0xc7, 0xe8, 0x0b, 0x63, 0x1f, 0xa3, 0x2f, 0x44, 0xb3, 0xe4, -- 0xd1, 0x69, 0x12, 0x84, 0x32, 0x14, 0xba, 0xb6, 0x86, 0xac, 0x2d, 0xe8, 0xa8, 0x1a, 0xfa, 0x29, -- 0xf5, 0x93, 0x29, 0x2e, 0x8d, 0xc1, 0x7b, 0x00, 0x91, 0xcb, 0xdc, 0x00, 0x73, 0xcc, 0xd4, 0x1b, -- 0x6a, 0xd9, 0x39, 0x8c, 0xf5, 0x87, 0x2a, 0xac, 0xa9, 0x91, 0xc8, 0xa1, 0x9a, 0x04, 0x18, 0x13, -- 0x46, 0xd0, 0x3c, 0xa5, 0x31, 0xcf, 0x31, 0x4c, 0x61, 0xa1, 0xa2, 0x1f, 0x1a, 0x6e, 0x62, 0x59, -- 0x98, 0x53, 0xd4, 0xae, 0x9e, 0x53, 0x2c, 0x4c, 0x22, 0xea, 0x8b, 0x93, 0x08, 0x11, 0x6d, 0x86, -- 0x88, 0xa8, 0x18, 0x6f, 0xd9, 0x2d, 0x8d, 0xd9, 0xf7, 0xd1, 0x03, 0xe8, 0x4f, 0x84, 0x96, 0xce, -- 0x29, 0xa5, 0x67, 0x4e, 0xe4, 0xf2, 0x53, 0x19, 0xea, 0x2d, 0xbb, 0x2b, 0xd1, 0x7b, 0x94, 0x9e, -- 0x1d, 0xb8, 0xfc, 0x14, 0x7d, 0x02, 0x3d, 0x5d, 0x06, 0x06, 0xd2, 0x45, 0xb1, 0xfe, 0xf8, 0xe9, -- 0x28, 0xca, 0x7b, 0xcf, 0xee, 0x9e, 0xe5, 0xa0, 0xd8, 0xba, 0x0d, 0xb7, 0x76, 0x70, 0xcc, 0x19, -- 0x9d, 0x15, 0x1d, 0x63, 0xfd, 0x08, 0x60, 0x3f, 0xe4, 0x98, 0x9d, 0xb8, 0x1e, 0x8e, 0xd1, 0x7b, -- 0x79, 0x48, 0x17, 0x47, 0x2b, 0x63, 0x35, 0x91, 0x4a, 0x37, 0xec, 0x1c, 0x8d, 0x35, 0x86, 0x86, -- 0x4d, 0x13, 0x91, 0x8e, 0xde, 0x30, 0x2b, 0x7d, 0xae, 0xa3, 0xcf, 0x49, 0xa4, 0xad, 0xf7, 0xac, -- 0x3d, 0xd3, 0xc2, 0x66, 0xec, 0xf4, 0x15, 0x8d, 0xa1, 0x45, 0x0c, 0x4e, 0x67, 0x95, 0x45, 0xd1, -- 0x19, 0x89, 0xf5, 0x19, 0xac, 0x2a, 0x4e, 0x8a, 0xb3, 0x61, 0xf3, 0x06, 0x34, 0x98, 0x51, 0xa3, -- 0x92, 0x8d, 0xa2, 0x34, 0x91, 0xde, 0x13, 0xfe, 0x10, 0x1d, 0x75, 0x66, 0x88, 0xf1, 0xc7, 0x2a, -- 0x0c, 0xc4, 0x46, 0x81, 0xa7, 0xf5, 0x4b, 0x58, 0x7d, 0x16, 0x4e, 0x49, 0x88, 0xb7, 0x0f, 0x8e, -- 0x9e, 0xe2, 0x34, 0xee, 0x11, 0xd4, 0x45, 0x7d, 0x24, 0x05, 0x35, 0x6d, 0xb9, 0x16, 0x81, 0x10, -- 0x1e, 0x3b, 0x5e, 0x94, 0xc4, 0x7a, 0xf6, 0xd3, 0x08, 0x8f, 0xb7, 0xa3, 0x24, 0x16, 0x89, 0x5c, -- 0x7c, 0xc8, 0x69, 0x38, 0x9d, 0xc9, 0x68, 0x68, 0xda, 0xcb, 0x5e, 0x94, 0x3c, 0x0b, 0xa7, 0x33, -- 0xeb, 0x07, 0xb2, 0xdb, 0xc5, 0xd8, 0xb7, 0xdd, 0xd0, 0xa7, 0xc1, 0x0e, 0x3e, 0xcf, 0x49, 0x48, -- 0x3b, 0x2b, 0x13, 0xf5, 0xdf, 0x55, 0xa0, 0xf3, 0x78, 0x82, 0x43, 0xbe, 0x83, 0xb9, 0x4b, 0xa6, -- 0xb2, 0x7b, 0x3a, 0xc7, 0x2c, 0x26, 0x34, 0xd4, 0x4f, 0xdb, 0x80, 0xa2, 0xf9, 0x25, 0x21, 0xe1, -- 0x8e, 0xef, 0xe2, 0x80, 0x86, 0x92, 0x4b, 0xd3, 0x06, 0x81, 0xda, 0x91, 0x18, 0xf4, 0x16, 0xf4, -- 0xd5, 0x6c, 0xce, 0x39, 0x75, 0x43, 0x7f, 0x2a, 0x82, 0x4a, 0xcd, 0x2a, 0x7a, 0x0a, 0xbd, 0xa7, -- 0xb1, 0xe8, 0x6d, 0x58, 0xd1, 0x4f, 0x3e, 0xa3, 0xac, 0x4b, 0xca, 0xbe, 0xc6, 0x17, 0x48, 0x93, -- 0x28, 0xa2, 0x8c, 0xc7, 0x4e, 0x8c, 0x3d, 0x8f, 0x06, 0x91, 0x6e, 0x3d, 0xfa, 0x06, 0x7f, 0xa8, -- 0xd0, 0xd6, 0x04, 0x56, 0x9f, 0x08, 0x3b, 0xb5, 0x25, 0xd9, 0x15, 0xf6, 0x02, 0x1c, 0x38, 0xc7, -- 0x53, 0xea, 0x9d, 0x39, 0x22, 0x11, 0x69, 0x0f, 0x8b, 0xe2, 0x66, 0x4b, 0x20, 0x0f, 0xc9, 0xb7, -- 0xb2, 0xcb, 0x16, 0x54, 0xa7, 0x94, 0x47, 0xd3, 0x64, 0xe2, 0x44, 0x8c, 0x1e, 0x63, 0x6d, 0x62, -- 0x3f, 0xc0, 0xc1, 0x9e, 0xc2, 0x1f, 0x08, 0xb4, 0xf5, 0xd7, 0x0a, 0xac, 0x15, 0x25, 0xe9, 0xb4, -- 0xba, 0x09, 0x6b, 0x45, 0x51, 0xfa, 0x53, 0xab, 0x4a, 0xb9, 0x41, 0x5e, 0xa0, 0xfa, 0xe8, 0x7e, -- 0x04, 0x5d, 0x39, 0xb0, 0x75, 0x7c, 0xc5, 0xa9, 0x58, 0x60, 0xe4, 0xef, 0xc5, 0xee, 0xb8, 0xf9, -- 0x5b, 0xfa, 0x04, 0xee, 0x68, 0xf3, 0x9d, 0x45, 0xb5, 0xd5, 0x83, 0x58, 0xd7, 0x04, 0x4f, 0xe7, -- 0xb4, 0xff, 0x12, 0x86, 0x19, 0x6a, 0x6b, 0x26, 0x91, 0xc6, 0x57, 0xef, 0xc1, 0xea, 0x9c, 0xb1, -- 0x8f, 0x7d, 0x9f, 0xc9, 0x10, 0xac, 0xdb, 0x65, 0x5b, 0xd6, 0x23, 0xb8, 0x7d, 0x88, 0xb9, 0xf2, -- 0x86, 0xcb, 0x75, 0xd5, 0xaf, 0x98, 0xad, 0x40, 0xed, 0x10, 0x7b, 0xd2, 0xf8, 0x9a, 0x2d, 0x96, -- 0xe2, 0x01, 0x1e, 0xc5, 0xd8, 0x93, 0x56, 0xd6, 0x6c, 0xb9, 0xb6, 0xfe, 0x52, 0x81, 0x65, 0x9d, -- 0x08, 0x45, 0x32, 0xf7, 0x19, 0x39, 0xc7, 0x4c, 0x3f, 0x3d, 0x0d, 0xa1, 0x37, 0xa1, 0xa7, 0x56, -- 0x0e, 0x8d, 0x38, 0xa1, 0x69, 0x7a, 0xed, 0x2a, 0xec, 0x33, 0x85, 0x94, 0xb3, 0x38, 0x39, 0x6a, -- 0xd2, 0x5d, 0x9d, 0x86, 0xe4, 0x40, 0x2d, 0x16, 0xb1, 0x2f, 0xd3, 0x69, 0xcb, 0xd6, 0x90, 0x78, -- 0xea, 0x86, 0xdf, 0x92, 0xe4, 0x67, 0x40, 0xf1, 0xd4, 0x03, 0x9a, 0x84, 0xdc, 0x89, 0x28, 0x09, -- 0xb9, 0xce, 0x9f, 0x20, 0x51, 0x07, 0x02, 0x63, 0xfd, 0xa6, 0x02, 0x0d, 0x35, 0x8f, 0x16, 0x7d, -- 0x64, 0xfa, 0x15, 0xab, 0x12, 0x59, 0x11, 0x48, 0x59, 0xea, 0xcb, 0x25, 0xd7, 0x22, 0x8e, 0xcf, -- 0x03, 0x95, 0x8b, 0xb5, 0x6a, 0xe7, 0x81, 0x4c, 0xc2, 0x6f, 0x42, 0x2f, 0xfb, 0x18, 0xca, 0x7d, -- 0xa5, 0x62, 0x37, 0xc5, 0x4a, 0xb2, 0x4b, 0x35, 0xb5, 0x7e, 0x2e, 0xda, 0xe7, 0x74, 0x16, 0xbb, -- 0x02, 0xb5, 0x24, 0x55, 0x46, 0x2c, 0x05, 0x66, 0x92, 0x7e, 0x46, 0xc5, 0x12, 0x3d, 0x80, 0x9e, -- 0xeb, 0xfb, 0x44, 0x1c, 0x77, 0xa7, 0x4f, 0x88, 0x9f, 0x06, 0x69, 0x11, 0x6b, 0xfd, 0xbd, 0x02, -- 0xfd, 0x6d, 0x1a, 0xcd, 0x7e, 0x4c, 0xa6, 0x38, 0x97, 0x41, 0xa4, 0x92, 0xfa, 0x2b, 0x2a, 0xd6, -- 0xa2, 0x32, 0x3c, 0x21, 0x53, 0xac, 0x42, 0x4b, 0xdd, 0x6c, 0x53, 0x20, 0x64, 0x58, 0x99, 0xcd, -- 0x74, 0xc4, 0xd5, 0x55, 0x9b, 0x4f, 0xa9, 0x2f, 0x6b, 0x60, 0x9f, 0x30, 0x27, 0x1d, 0x68, 0x75, -- 0xed, 0x65, 0x9f, 0x30, 0xb9, 0xa5, 0x0d, 0x59, 0x92, 0x33, 0xd5, 0xbc, 0x21, 0x0d, 0x85, 0x11, -- 0x86, 0xac, 0x43, 0x83, 0x9e, 0x9c, 0xc4, 0x98, 0xcb, 0x6a, 0xb5, 0x66, 0x6b, 0x28, 0x4d, 0x73, -- 0xcd, 0x5c, 0x9a, 0xbb, 0x05, 0xab, 0x72, 0x7a, 0xff, 0x9c, 0xb9, 0x1e, 0x09, 0x27, 0x26, 0x15, -- 0xaf, 0x01, 0x3a, 0xe4, 0x34, 0x2a, 0x62, 0x1f, 0xfe, 0x7e, 0x45, 0xe7, 0x44, 0xdd, 0xca, 0xa2, -- 0x27, 0xd0, 0x9f, 0xfb, 0x6b, 0x04, 0xe9, 0xd9, 0x46, 0xf9, 0x3f, 0x26, 0xa3, 0xf5, 0xb1, 0xfa, -- 0xab, 0x65, 0x6c, 0xfe, 0x6a, 0x19, 0xef, 0x06, 0x11, 0x9f, 0xa1, 0x5d, 0xe8, 0x15, 0xff, 0x44, -- 0x40, 0x77, 0x4d, 0x29, 0x50, 0xf2, 0xd7, 0xc2, 0xa5, 0x6c, 0x9e, 0x40, 0x7f, 0xee, 0xff, 0x04, -- 0xa3, 0x4f, 0xf9, 0xdf, 0x0c, 0x97, 0x32, 0x7a, 0x04, 0xed, 0xdc, 0x1f, 0x08, 0x68, 0xa8, 0x98, -- 0x2c, 0xfe, 0xa7, 0x70, 0x29, 0x83, 0x6d, 0xe8, 0x16, 0x66, 0xfa, 0x68, 0xa4, 0xed, 0x29, 0x19, -- 0xf4, 0x5f, 0xca, 0x64, 0x0b, 0xda, 0xb9, 0xd1, 0xba, 0xd1, 0x62, 0x71, 0x7e, 0x3f, 0xba, 0x53, -- 0xb2, 0xa3, 0x53, 0xef, 0x1e, 0x74, 0x0b, 0x83, 0x70, 0xa3, 0x48, 0xd9, 0x10, 0x7e, 0x74, 0xb7, -- 0x74, 0x4f, 0x73, 0x7a, 0x02, 0xfd, 0xb9, 0xb1, 0xb8, 0x71, 0x6e, 0xf9, 0xb4, 0xfc, 0x52, 0xb3, -- 0xbe, 0x90, 0x97, 0x9d, 0xeb, 0x7a, 0x72, 0x97, 0xbd, 0x38, 0x04, 0x1f, 0xbd, 0x52, 0xbe, 0xa9, -- 0xb5, 0xda, 0x85, 0x5e, 0x71, 0xfe, 0x6d, 0x98, 0x95, 0x4e, 0xc5, 0xaf, 0x7e, 0x39, 0x85, 0x51, -- 0x78, 0xf6, 0x72, 0xca, 0x26, 0xe4, 0x97, 0x32, 0x7a, 0x0c, 0xa0, 0x7b, 0x1c, 0x9f, 0x84, 0xe9, -- 0x95, 0x2d, 0xf4, 0x56, 0xe9, 0x95, 0x95, 0xf4, 0x43, 0x8f, 0x00, 0x54, 0x6b, 0xe2, 0xd3, 0x84, -- 0xa3, 0xdb, 0x46, 0x8d, 0xb9, 0x7e, 0x68, 0x34, 0x5c, 0xdc, 0x58, 0x60, 0x80, 0x19, 0xbb, 0x0e, -- 0x83, 0xcf, 0x01, 0xb2, 0x96, 0xc7, 0x30, 0x58, 0x68, 0x82, 0xae, 0xf0, 0x41, 0x27, 0xdf, 0xe0, -- 0x20, 0x6d, 0x6b, 0x49, 0xd3, 0x73, 0x05, 0x8b, 0xfe, 0x5c, 0x01, 0x5b, 0x7c, 0x6c, 0xf3, 0x75, -- 0xed, 0x68, 0xa1, 0x88, 0x45, 0x1f, 0x41, 0x27, 0x5f, 0xb9, 0x1a, 0x2d, 0x4a, 0xaa, 0xd9, 0x51, -- 0xa1, 0x7a, 0x45, 0x8f, 0xa0, 0x57, 0xac, 0x5a, 0x51, 0x2e, 0x2e, 0x16, 0x6a, 0xd9, 0x91, 0x9e, -- 0xc9, 0xe4, 0xc8, 0x3f, 0x00, 0xc8, 0xaa, 0x5b, 0xe3, 0xbe, 0x85, 0x7a, 0x77, 0x4e, 0xea, 0x63, -- 0xe8, 0xe4, 0x33, 0xb1, 0x51, 0xb7, 0x24, 0x3b, 0x5f, 0x95, 0xb5, 0x72, 0x59, 0xdb, 0x3c, 0xbe, -- 0xc5, 0x44, 0x7e, 0x55, 0xd6, 0x2a, 0xf4, 0x75, 0x26, 0x59, 0x94, 0x35, 0x7b, 0x57, 0xe5, 0xf2, -- 0x62, 0x13, 0x64, 0xdc, 0x57, 0xda, 0x1a, 0x5d, 0xf5, 0x88, 0xf2, 0xdd, 0x80, 0xf1, 0x47, 0x49, -- 0x87, 0xf0, 0x92, 0xa0, 0xce, 0x57, 0xfc, 0xb9, 0xa0, 0x2e, 0x69, 0x04, 0x2e, 0x65, 0xb4, 0x07, -- 0xfd, 0x27, 0xa6, 0x98, 0xd3, 0x85, 0xa6, 0x56, 0xa7, 0xa4, 0xb0, 0x1e, 0x8d, 0xca, 0xb6, 0x74, -- 0x64, 0x7d, 0x01, 0x83, 0x85, 0x22, 0x13, 0xdd, 0x4b, 0x47, 0x87, 0xa5, 0xd5, 0xe7, 0xa5, 0x6a, -- 0xed, 0xc3, 0xca, 0x7c, 0x8d, 0x89, 0x5e, 0xd5, 0x97, 0x5e, 0x5e, 0x7b, 0x5e, 0xca, 0xea, 0x13, -- 0x68, 0x9a, 0x9a, 0x06, 0xe9, 0x11, 0xed, 0x5c, 0x8d, 0x73, 0xd9, 0xd1, 0xad, 0xce, 0x77, 0xdf, -- 0xdf, 0xab, 0xfc, 0xe3, 0xfb, 0x7b, 0x95, 0x7f, 0x7d, 0x7f, 0xaf, 0x72, 0xdc, 0x90, 0xbb, 0x1f, -- 0xfc, 0x27, 0x00, 0x00, 0xff, 0xff, 0xa5, 0xac, 0x85, 0x1d, 0xaa, 0x21, 0x00, 0x00, -+ 0xd1, 0xd8, 0x07, 0x97, 0xbb, 0xb5, 0x2f, 0x6e, 0x93, 0xa2, 0x56, 0x2b, 0x59, 0x9f, 0x3c, 0xb6, -+ 0x65, 0xfa, 0xf3, 0xe7, 0xa5, 0x2d, 0x1b, 0x9f, 0x5f, 0x70, 0x04, 0xf1, 0x11, 0x91, 0xb1, 0x15, -+ 0x31, 0x43, 0x11, 0x4e, 0x10, 0x04, 0x83, 0xe1, 0x4c, 0x73, 0xd9, 0xe6, 0xce, 0xf4, 0xb8, 0xa7, -+ 0x87, 0xe2, 0x3a, 0x40, 0x8e, 0xc9, 0x2d, 0x97, 0x00, 0xb9, 0xe5, 0x0f, 0x04, 0xb9, 0xe5, 0x98, -+ 0x6b, 0x0e, 0x46, 0x4e, 0xf9, 0x05, 0x41, 0xe0, 0x9f, 0x90, 0x5f, 0x10, 0xf4, 0x6b, 0x1e, 0xbb, -+ 0x43, 0x1a, 0x21, 0x08, 0xe4, 0xb2, 0xe8, 0xaa, 0xae, 0xae, 0x57, 0x77, 0xd5, 0x54, 0xd5, 0x42, -+ 0xdb, 0x9d, 0xe0, 0x90, 0x8f, 0x23, 0x46, 0x39, 0x45, 0xf5, 0x09, 0x8b, 0xbc, 0x51, 0x8b, 0x7a, -+ 0x44, 0x21, 0x46, 0xff, 0x3f, 0x21, 0xfc, 0x34, 0x39, 0x1e, 0x7b, 0x34, 0xd8, 0x3c, 0x73, 0xb9, -+ 0xfb, 0x8e, 0x47, 0x43, 0xee, 0x92, 0x10, 0xb3, 0x78, 0x53, 0x1e, 0xdc, 0x8c, 0xce, 0x26, 0x9b, -+ 0x7c, 0x16, 0xe1, 0x58, 0xfd, 0xea, 0x73, 0x77, 0x27, 0x94, 0x4e, 0xa6, 0x78, 0x53, 0x42, 0xc7, -+ 0xc9, 0xc9, 0x26, 0x0e, 0x22, 0x3e, 0x53, 0x9b, 0xd6, 0x1f, 0xaa, 0xb0, 0xbe, 0xcd, 0xb0, 0xcb, -+ 0xf1, 0xb6, 0xe1, 0x66, 0xe3, 0xaf, 0x13, 0x1c, 0x73, 0xf4, 0x2a, 0x74, 0x52, 0x09, 0x0e, 0xf1, -+ 0x87, 0x95, 0x07, 0x95, 0x8d, 0x96, 0xdd, 0x4e, 0x71, 0xfb, 0x3e, 0xba, 0x0d, 0xcb, 0xf8, 0x02, -+ 0x7b, 0x62, 0xb7, 0x2a, 0x77, 0x1b, 0x02, 0xdc, 0xf7, 0xd1, 0x7b, 0xd0, 0x8e, 0x39, 0x23, 0xe1, -+ 0xc4, 0x49, 0x62, 0xcc, 0x86, 0xb5, 0x07, 0x95, 0x8d, 0xf6, 0xa3, 0x95, 0xb1, 0x30, 0x69, 0x7c, -+ 0x28, 0x37, 0x8e, 0x62, 0xcc, 0x6c, 0x88, 0xd3, 0x35, 0x7a, 0x08, 0xcb, 0x3e, 0x3e, 0x27, 0x1e, -+ 0x8e, 0x87, 0xf5, 0x07, 0xb5, 0x8d, 0xf6, 0xa3, 0x8e, 0x22, 0xdf, 0x91, 0x48, 0xdb, 0x6c, 0xa2, -+ 0xb7, 0xa0, 0x19, 0x73, 0xca, 0xdc, 0x09, 0x8e, 0x87, 0x4b, 0x92, 0xb0, 0x6b, 0xf8, 0x4a, 0xac, -+ 0x9d, 0x6e, 0xa3, 0x7b, 0x50, 0x7b, 0xbe, 0xbd, 0x3f, 0x6c, 0x48, 0xe9, 0xa0, 0xa9, 0x22, 0xec, -+ 0xd9, 0x02, 0x8d, 0x5e, 0x83, 0x6e, 0xec, 0x86, 0xfe, 0x31, 0xbd, 0x70, 0x22, 0xe2, 0x87, 0xf1, -+ 0x70, 0xf9, 0x41, 0x65, 0xa3, 0x69, 0x77, 0x34, 0xf2, 0x40, 0xe0, 0xac, 0x4f, 0xe0, 0xd6, 0x21, -+ 0x77, 0x19, 0xbf, 0x86, 0x77, 0xac, 0x23, 0x58, 0xb7, 0x71, 0x40, 0xcf, 0xaf, 0xe5, 0xda, 0x21, -+ 0x2c, 0x73, 0x12, 0x60, 0x9a, 0x70, 0xe9, 0xda, 0xae, 0x6d, 0x40, 0xeb, 0x4f, 0x15, 0x40, 0xbb, -+ 0x17, 0xd8, 0x3b, 0x60, 0xd4, 0xc3, 0x71, 0xfc, 0x5f, 0xba, 0xae, 0x37, 0x61, 0x39, 0x52, 0x0a, -+ 0x0c, 0xeb, 0x92, 0x5c, 0xdf, 0x82, 0xd1, 0xca, 0xec, 0x5a, 0x5f, 0xc1, 0xda, 0x21, 0x99, 0x84, -+ 0xee, 0xf4, 0x06, 0xf5, 0x5d, 0x87, 0x46, 0x2c, 0x79, 0x4a, 0x55, 0xbb, 0xb6, 0x86, 0xac, 0x03, -+ 0x40, 0x5f, 0xba, 0x84, 0xdf, 0x9c, 0x24, 0xeb, 0x1d, 0x58, 0x2d, 0x70, 0x8c, 0x23, 0x1a, 0xc6, -+ 0x58, 0x2a, 0xc0, 0x5d, 0x9e, 0xc4, 0x92, 0xd9, 0x92, 0xad, 0x21, 0x0b, 0xc3, 0xda, 0x17, 0x24, -+ 0x36, 0xe4, 0xf8, 0x3f, 0x51, 0x61, 0x1d, 0x1a, 0x27, 0x94, 0x05, 0x2e, 0x37, 0x1a, 0x28, 0x08, -+ 0x21, 0xa8, 0xbb, 0x6c, 0x12, 0x0f, 0x6b, 0x0f, 0x6a, 0x1b, 0x2d, 0x5b, 0xae, 0xc5, 0xab, 0x9c, -+ 0x13, 0xa3, 0xf5, 0x7a, 0x15, 0x3a, 0xda, 0xef, 0xce, 0x94, 0xc4, 0x5c, 0xca, 0xe9, 0xd8, 0x6d, -+ 0x8d, 0x13, 0x67, 0x2c, 0x0a, 0xeb, 0x47, 0x91, 0x7f, 0xcd, 0x80, 0x7f, 0x04, 0x2d, 0x86, 0x63, -+ 0x9a, 0x30, 0x11, 0xa6, 0x55, 0x79, 0xef, 0x6b, 0xea, 0xde, 0xbf, 0x20, 0x61, 0x72, 0x61, 0x9b, -+ 0x3d, 0x3b, 0x23, 0xd3, 0x21, 0xc4, 0xe3, 0xeb, 0x84, 0xd0, 0x27, 0x70, 0xeb, 0xc0, 0x4d, 0xe2, -+ 0xeb, 0xe8, 0x6a, 0x7d, 0x2a, 0xc2, 0x2f, 0x4e, 0x82, 0x6b, 0x1d, 0xfe, 0x63, 0x05, 0x9a, 0xdb, -+ 0x51, 0x72, 0x14, 0xbb, 0x13, 0x8c, 0xfe, 0x07, 0xda, 0x9c, 0x72, 0x77, 0xea, 0x24, 0x02, 0x94, -+ 0xe4, 0x75, 0x1b, 0x24, 0x4a, 0x11, 0x08, 0xb7, 0x63, 0xe6, 0x45, 0x89, 0xa6, 0xa8, 0x3e, 0xa8, -+ 0x6d, 0xd4, 0xed, 0xb6, 0xc2, 0x29, 0x92, 0x31, 0xac, 0xca, 0x3d, 0x87, 0x84, 0xce, 0x19, 0x66, -+ 0x21, 0x9e, 0x06, 0xd4, 0xc7, 0xf2, 0xfd, 0xd6, 0xed, 0x81, 0xdc, 0xda, 0x0f, 0x3f, 0x4f, 0x37, -+ 0xd0, 0xff, 0xc2, 0x20, 0xa5, 0x17, 0x41, 0x29, 0xa9, 0xeb, 0x92, 0xba, 0xaf, 0xa9, 0x8f, 0x34, -+ 0xda, 0xfa, 0x15, 0xf4, 0x5e, 0x9c, 0x32, 0xca, 0xf9, 0x94, 0x84, 0x93, 0x1d, 0x97, 0xbb, 0x22, -+ 0x7b, 0x44, 0x98, 0x11, 0xea, 0xc7, 0x5a, 0x5b, 0x03, 0xa2, 0xb7, 0x61, 0xc0, 0x15, 0x2d, 0xf6, -+ 0x1d, 0x43, 0x53, 0x95, 0x34, 0x2b, 0xe9, 0xc6, 0x81, 0x26, 0x7e, 0x03, 0x7a, 0x19, 0xb1, 0xc8, -+ 0x3f, 0x5a, 0xdf, 0x6e, 0x8a, 0x7d, 0x41, 0x02, 0x6c, 0x9d, 0x4b, 0x5f, 0xc9, 0x4b, 0x46, 0x6f, -+ 0x43, 0x2b, 0xf3, 0x43, 0x45, 0xbe, 0x90, 0x9e, 0x7a, 0x21, 0xc6, 0x9d, 0x76, 0x33, 0x75, 0xca, -+ 0x67, 0xd0, 0xe7, 0xa9, 0xe2, 0x8e, 0xef, 0x72, 0xb7, 0xf8, 0xa8, 0x8a, 0x56, 0xd9, 0x3d, 0x5e, -+ 0x80, 0xad, 0x4f, 0xa1, 0x75, 0x40, 0xfc, 0x58, 0x09, 0x1e, 0xc2, 0xb2, 0x97, 0x30, 0x86, 0x43, -+ 0x6e, 0x4c, 0xd6, 0x20, 0x5a, 0x83, 0xa5, 0x29, 0x09, 0x08, 0xd7, 0x66, 0x2a, 0xc0, 0xa2, 0x00, -+ 0xcf, 0x70, 0x40, 0xd9, 0x4c, 0x3a, 0x6c, 0x0d, 0x96, 0xf2, 0x97, 0xab, 0x00, 0x74, 0x17, 0x5a, -+ 0x81, 0x7b, 0x91, 0x5e, 0xaa, 0xd8, 0x69, 0x06, 0xee, 0x85, 0x52, 0x7e, 0x08, 0xcb, 0x27, 0x2e, -+ 0x99, 0x7a, 0x21, 0xd7, 0x5e, 0x31, 0x60, 0x26, 0xb0, 0x9e, 0x17, 0xf8, 0xd7, 0x2a, 0xb4, 0x95, -+ 0x44, 0xa5, 0xf0, 0x1a, 0x2c, 0x79, 0xae, 0x77, 0x9a, 0x8a, 0x94, 0x00, 0x7a, 0x68, 0x14, 0xa9, -+ 0xe6, 0x93, 0x70, 0xa6, 0xa9, 0x51, 0x6d, 0x13, 0x20, 0x7e, 0xe9, 0x46, 0x5a, 0xb7, 0xda, 0x25, -+ 0xc4, 0x2d, 0x41, 0xa3, 0xd4, 0x7d, 0x1f, 0x3a, 0xea, 0xdd, 0xe9, 0x23, 0xf5, 0x4b, 0x8e, 0xb4, -+ 0x15, 0x95, 0x3a, 0xf4, 0x1a, 0x74, 0x93, 0x18, 0x3b, 0xa7, 0x04, 0x33, 0x97, 0x79, 0xa7, 0xb3, -+ 0xe1, 0x92, 0xfa, 0x46, 0x26, 0x31, 0xde, 0x33, 0x38, 0xf4, 0x08, 0x96, 0x44, 0xfa, 0x8b, 0x87, -+ 0x0d, 0xf9, 0x39, 0xbe, 0x97, 0x67, 0x29, 0x4d, 0x1d, 0xcb, 0xdf, 0xdd, 0x90, 0xb3, 0x99, 0xad, -+ 0x48, 0x47, 0x1f, 0x01, 0x64, 0x48, 0xb4, 0x02, 0xb5, 0x33, 0x3c, 0xd3, 0x71, 0x28, 0x96, 0xc2, -+ 0x39, 0xe7, 0xee, 0x34, 0x31, 0x5e, 0x57, 0xc0, 0x27, 0xd5, 0x8f, 0x2a, 0x96, 0x07, 0xfd, 0xad, -+ 0xe9, 0x19, 0xa1, 0xb9, 0xe3, 0x6b, 0xb0, 0x14, 0xb8, 0x5f, 0x51, 0x66, 0x3c, 0x29, 0x01, 0x89, -+ 0x25, 0x21, 0x65, 0x86, 0x85, 0x04, 0x50, 0x0f, 0xaa, 0x34, 0x92, 0xfe, 0x6a, 0xd9, 0x55, 0x1a, -+ 0x65, 0x82, 0xea, 0x39, 0x41, 0xd6, 0x3f, 0xea, 0x00, 0x99, 0x14, 0x64, 0xc3, 0x88, 0x50, 0x27, -+ 0xc6, 0x4c, 0x94, 0x20, 0xce, 0xf1, 0x8c, 0xe3, 0xd8, 0x61, 0xd8, 0x4b, 0x58, 0x4c, 0xce, 0xc5, -+ 0xfd, 0x09, 0xb3, 0x6f, 0x29, 0xb3, 0xe7, 0x74, 0xb3, 0x6f, 0x13, 0x7a, 0xa8, 0xce, 0x6d, 0x89, -+ 0x63, 0xb6, 0x39, 0x85, 0xf6, 0xe1, 0x56, 0xc6, 0xd3, 0xcf, 0xb1, 0xab, 0x5e, 0xc5, 0x6e, 0x35, -+ 0x65, 0xe7, 0x67, 0xac, 0x76, 0x61, 0x95, 0x50, 0xe7, 0xeb, 0x04, 0x27, 0x05, 0x46, 0xb5, 0xab, -+ 0x18, 0x0d, 0x08, 0xfd, 0x89, 0x3c, 0x90, 0xb1, 0x39, 0x80, 0x3b, 0x39, 0x2b, 0x45, 0xb8, 0xe7, -+ 0x98, 0xd5, 0xaf, 0x62, 0xb6, 0x9e, 0x6a, 0x25, 0xf2, 0x41, 0xc6, 0xf1, 0x47, 0xb0, 0x4e, 0xa8, -+ 0xf3, 0xd2, 0x25, 0x7c, 0x9e, 0xdd, 0xd2, 0xf7, 0x18, 0x29, 0x3e, 0xba, 0x45, 0x5e, 0xca, 0xc8, -+ 0x00, 0xb3, 0x49, 0xc1, 0xc8, 0xc6, 0xf7, 0x18, 0xf9, 0x4c, 0x1e, 0xc8, 0xd8, 0x3c, 0x81, 0x01, -+ 0xa1, 0xf3, 0xda, 0x2c, 0x5f, 0xc5, 0xa4, 0x4f, 0x68, 0x51, 0x93, 0x2d, 0x18, 0xc4, 0xd8, 0xe3, -+ 0x94, 0xe5, 0x1f, 0x41, 0xf3, 0x2a, 0x16, 0x2b, 0x9a, 0x3e, 0xe5, 0x61, 0xfd, 0x1c, 0x3a, 0x7b, -+ 0xc9, 0x04, 0xf3, 0xe9, 0x71, 0x9a, 0x0c, 0x6e, 0x2c, 0xff, 0x58, 0xff, 0xaa, 0x42, 0x7b, 0x7b, -+ 0xc2, 0x68, 0x12, 0x15, 0x72, 0xb2, 0x0a, 0xd2, 0xf9, 0x9c, 0x2c, 0x49, 0x64, 0x4e, 0x56, 0xc4, -+ 0x1f, 0x40, 0x27, 0x90, 0xa1, 0xab, 0xe9, 0x55, 0x1e, 0x1a, 0x2c, 0x04, 0xb5, 0xdd, 0x0e, 0x72, -+ 0xc9, 0x6c, 0x0c, 0x10, 0x11, 0x3f, 0xd6, 0x67, 0x54, 0x3a, 0xea, 0xeb, 0x8a, 0xd0, 0xa4, 0x68, -+ 0xbb, 0x15, 0xa5, 0xd9, 0xfa, 0x3d, 0x68, 0x1f, 0x0b, 0x27, 0xe9, 0x03, 0x85, 0x64, 0x94, 0x79, -+ 0xcf, 0x86, 0xe3, 0x2c, 0x08, 0xf7, 0xa0, 0x7b, 0xaa, 0x5c, 0xa6, 0x0f, 0xa9, 0x37, 0xf4, 0x9a, -+ 0xb6, 0x24, 0xb3, 0x77, 0x9c, 0xf7, 0xac, 0xba, 0x80, 0xce, 0x69, 0x0e, 0x35, 0x3a, 0x84, 0xc1, -+ 0x02, 0x49, 0x49, 0x0e, 0xda, 0xc8, 0xe7, 0xa0, 0xf6, 0x23, 0xa4, 0x04, 0xe5, 0x4f, 0xe6, 0xf3, -+ 0xd2, 0x6f, 0xab, 0xd0, 0xf9, 0x31, 0xe6, 0x2f, 0x29, 0x3b, 0x53, 0xfa, 0x22, 0xa8, 0x87, 0x6e, -+ 0x80, 0x35, 0x47, 0xb9, 0x46, 0x77, 0xa0, 0xc9, 0x2e, 0x54, 0x02, 0xd1, 0xf7, 0xb9, 0xcc, 0x2e, -+ 0x64, 0x62, 0x40, 0xaf, 0x00, 0xb0, 0x0b, 0x27, 0x72, 0xbd, 0x33, 0xac, 0x3d, 0x58, 0xb7, 0x5b, -+ 0xec, 0xe2, 0x40, 0x21, 0xc4, 0x53, 0x60, 0x17, 0x0e, 0x66, 0x8c, 0xb2, 0x58, 0xe7, 0xaa, 0x26, -+ 0xbb, 0xd8, 0x95, 0xb0, 0x3e, 0xeb, 0x33, 0x1a, 0x45, 0xd8, 0x97, 0x39, 0x5a, 0x9e, 0xdd, 0x51, -+ 0x08, 0x21, 0x95, 0x1b, 0xa9, 0x0d, 0x25, 0x95, 0x67, 0x52, 0x79, 0x26, 0x75, 0x59, 0x9d, 0xe4, -+ 0x79, 0xa9, 0x3c, 0x95, 0xda, 0x54, 0x52, 0x79, 0x4e, 0x2a, 0xcf, 0xa4, 0xb6, 0xcc, 0x59, 0x2d, -+ 0xd5, 0xfa, 0x4d, 0x05, 0xd6, 0xe7, 0x0b, 0x3f, 0x5d, 0xa6, 0x7e, 0x00, 0x1d, 0x4f, 0xde, 0x57, -+ 0xe1, 0x4d, 0x0e, 0x16, 0x6e, 0xd2, 0x6e, 0x7b, 0xb9, 0x67, 0xfc, 0x21, 0x74, 0x43, 0xe5, 0xe0, -+ 0xf4, 0x69, 0xd6, 0xb2, 0x7b, 0xc9, 0xfb, 0xde, 0xee, 0x84, 0x39, 0xc8, 0xf2, 0x01, 0x7d, 0xc9, -+ 0x08, 0xc7, 0x87, 0x9c, 0x61, 0x37, 0xb8, 0x89, 0x06, 0x04, 0x41, 0x5d, 0x56, 0x2b, 0x35, 0x59, -+ 0x5f, 0xcb, 0xb5, 0xf5, 0x26, 0xac, 0x16, 0xa4, 0x68, 0x5b, 0x57, 0xa0, 0x36, 0xc5, 0xa1, 0xe4, -+ 0xde, 0xb5, 0xc5, 0xd2, 0x72, 0x61, 0x60, 0x63, 0xd7, 0xbf, 0x39, 0x6d, 0xb4, 0x88, 0x5a, 0x26, -+ 0x62, 0x03, 0x50, 0x5e, 0x84, 0x56, 0xc5, 0x68, 0x5d, 0xc9, 0x69, 0xfd, 0x1c, 0x06, 0xdb, 0x53, -+ 0x1a, 0xe3, 0x43, 0xee, 0x93, 0xf0, 0x26, 0x3a, 0xa6, 0x5f, 0xc2, 0xea, 0x0b, 0x3e, 0xfb, 0x52, -+ 0x30, 0x8b, 0xc9, 0x37, 0xf8, 0x86, 0xec, 0x63, 0xf4, 0xa5, 0xb1, 0x8f, 0xd1, 0x97, 0xa2, 0x59, -+ 0xf2, 0xe8, 0x34, 0x09, 0x42, 0x19, 0x0a, 0x5d, 0x5b, 0x43, 0xd6, 0x16, 0x74, 0x54, 0x0d, 0xfd, -+ 0x8c, 0xfa, 0xc9, 0x14, 0x97, 0xc6, 0xe0, 0x7d, 0x80, 0xc8, 0x65, 0x6e, 0x80, 0x39, 0x66, 0xea, -+ 0x0d, 0xb5, 0xec, 0x1c, 0xc6, 0xfa, 0x7d, 0x15, 0xd6, 0xd4, 0x48, 0xe4, 0x50, 0x4d, 0x02, 0x8c, -+ 0x09, 0x23, 0x68, 0x9e, 0xd2, 0x98, 0xe7, 0x18, 0xa6, 0xb0, 0x50, 0xd1, 0x0f, 0x0d, 0x37, 0xb1, -+ 0x2c, 0xcc, 0x29, 0x6a, 0x57, 0xcf, 0x29, 0x16, 0x26, 0x11, 0xf5, 0xc5, 0x49, 0x84, 0x88, 0x36, -+ 0x43, 0x44, 0x54, 0x8c, 0xb7, 0xec, 0x96, 0xc6, 0xec, 0xfb, 0xe8, 0x21, 0xf4, 0x27, 0x42, 0x4b, -+ 0xe7, 0x94, 0xd2, 0x33, 0x27, 0x72, 0xf9, 0xa9, 0x0c, 0xf5, 0x96, 0xdd, 0x95, 0xe8, 0x3d, 0x4a, -+ 0xcf, 0x0e, 0x5c, 0x7e, 0x8a, 0x3e, 0x86, 0x9e, 0x2e, 0x03, 0x03, 0xe9, 0xa2, 0x58, 0x7f, 0xfc, -+ 0x74, 0x14, 0xe5, 0xbd, 0x67, 0x77, 0xcf, 0x72, 0x50, 0x6c, 0xdd, 0x86, 0x5b, 0x3b, 0x38, 0xe6, -+ 0x8c, 0xce, 0x8a, 0x8e, 0xb1, 0x7e, 0x00, 0xb0, 0x1f, 0x72, 0xcc, 0x4e, 0x5c, 0x0f, 0xc7, 0xe8, -+ 0xdd, 0x3c, 0xa4, 0x8b, 0xa3, 0x95, 0xb1, 0x9a, 0x48, 0xa5, 0x1b, 0x76, 0x8e, 0xc6, 0x1a, 0x43, -+ 0xc3, 0xa6, 0x89, 0x48, 0x47, 0xaf, 0x9b, 0x95, 0x3e, 0xd7, 0xd1, 0xe7, 0x24, 0xd2, 0xd6, 0x7b, -+ 0xd6, 0x9e, 0x69, 0x61, 0x33, 0x76, 0xfa, 0x8a, 0xc6, 0xd0, 0x22, 0x06, 0xa7, 0xb3, 0xca, 0xa2, -+ 0xe8, 0x8c, 0xc4, 0xfa, 0x19, 0xac, 0x2a, 0x4e, 0x8a, 0xb3, 0x61, 0xf3, 0x3a, 0x34, 0x98, 0x51, -+ 0xa3, 0x92, 0x8d, 0xa2, 0x34, 0x91, 0xde, 0x43, 0xf7, 0x84, 0x30, 0x8f, 0xe1, 0x40, 0xf4, 0x1c, -+ 0x55, 0x79, 0x65, 0x19, 0x42, 0x78, 0x4b, 0xf4, 0xdb, 0x99, 0x99, 0xc6, 0x5b, 0xab, 0x30, 0x10, -+ 0x1b, 0x05, 0x89, 0xd6, 0x2f, 0x60, 0xf5, 0x79, 0x38, 0x25, 0x21, 0xde, 0x3e, 0x38, 0x7a, 0x86, -+ 0xd3, 0xac, 0x80, 0xa0, 0x2e, 0xaa, 0x27, 0xa9, 0x46, 0xd3, 0x96, 0x6b, 0x11, 0x26, 0xe1, 0xb1, -+ 0xe3, 0x45, 0x49, 0xac, 0x27, 0x43, 0x8d, 0xf0, 0x78, 0x3b, 0x4a, 0x62, 0x91, 0xe6, 0xc5, 0x67, -+ 0x9e, 0x86, 0xd3, 0x99, 0x8c, 0x95, 0xa6, 0xbd, 0xec, 0x45, 0xc9, 0xf3, 0x70, 0x3a, 0xb3, 0xfe, -+ 0x4f, 0xf6, 0xc2, 0x18, 0xfb, 0xb6, 0x1b, 0xfa, 0x34, 0xd8, 0xc1, 0xe7, 0x39, 0x09, 0x69, 0xdf, -+ 0x65, 0x72, 0xc2, 0xb7, 0x15, 0xe8, 0x3c, 0x99, 0xe0, 0x90, 0xef, 0x60, 0xee, 0x92, 0xa9, 0xec, -+ 0xad, 0xce, 0x31, 0x8b, 0x09, 0x0d, 0xf5, 0xc3, 0x37, 0xa0, 0x68, 0x8d, 0x49, 0x48, 0xb8, 0xe3, -+ 0xbb, 0x38, 0xa0, 0xa1, 0xf6, 0x02, 0x08, 0xd4, 0x8e, 0xc4, 0xa0, 0x37, 0xa1, 0xaf, 0x26, 0x77, -+ 0xce, 0xa9, 0x1b, 0xfa, 0x53, 0x11, 0x72, 0x6a, 0x92, 0xd1, 0x53, 0xe8, 0x3d, 0x8d, 0x45, 0x6f, -+ 0xc1, 0x8a, 0x0e, 0x88, 0x8c, 0xb2, 0x2e, 0x29, 0xfb, 0x1a, 0x5f, 0x20, 0x4d, 0xa2, 0x88, 0x32, -+ 0x1e, 0x3b, 0x31, 0xf6, 0x3c, 0x1a, 0x44, 0xba, 0x31, 0xe9, 0x1b, 0xfc, 0xa1, 0x42, 0x5b, 0x13, -+ 0x58, 0x7d, 0x2a, 0xec, 0xd4, 0x96, 0x64, 0x17, 0xdc, 0x0b, 0x70, 0xe0, 0x1c, 0x4f, 0xa9, 0x77, -+ 0xe6, 0x88, 0x34, 0xa5, 0x3d, 0x2c, 0x4a, 0x9f, 0x2d, 0x81, 0x3c, 0x24, 0xdf, 0xc8, 0x1e, 0x5c, -+ 0x50, 0x9d, 0x52, 0x1e, 0x4d, 0x93, 0x89, 0x13, 0x31, 0x7a, 0x8c, 0xb5, 0x89, 0xfd, 0x00, 0x07, -+ 0x7b, 0x0a, 0x7f, 0x20, 0xd0, 0xd6, 0x5f, 0x2a, 0xb0, 0x56, 0x94, 0xa4, 0x93, 0xee, 0x26, 0xac, -+ 0x15, 0x45, 0xe9, 0x0f, 0xb1, 0x2a, 0xf4, 0x06, 0x79, 0x81, 0xea, 0x93, 0xfc, 0x21, 0x74, 0xe5, -+ 0x38, 0xd7, 0xf1, 0x15, 0xa7, 0x62, 0xf9, 0x91, 0xbf, 0x17, 0xbb, 0xe3, 0xe6, 0x6f, 0xe9, 0x63, -+ 0xb8, 0xa3, 0xcd, 0x77, 0x16, 0xd5, 0x56, 0x0f, 0x62, 0x5d, 0x13, 0x3c, 0x9b, 0xd3, 0xfe, 0x0b, -+ 0x18, 0x66, 0xa8, 0xad, 0x99, 0x44, 0x1a, 0x5f, 0xbd, 0x0b, 0xab, 0x73, 0xc6, 0x3e, 0xf1, 0x7d, -+ 0x26, 0x03, 0xb4, 0x6e, 0x97, 0x6d, 0x59, 0x8f, 0xe1, 0xf6, 0x21, 0xe6, 0xca, 0x1b, 0x2e, 0xd7, -+ 0x3d, 0x81, 0x62, 0xb6, 0x02, 0xb5, 0x43, 0xec, 0x49, 0xe3, 0x6b, 0xb6, 0x58, 0x8a, 0x07, 0x78, -+ 0x14, 0x63, 0x4f, 0x5a, 0x59, 0xb3, 0xe5, 0xda, 0xfa, 0x73, 0x05, 0x96, 0x75, 0x9a, 0x14, 0xa9, -+ 0xde, 0x67, 0xe4, 0x1c, 0x33, 0xfd, 0xf4, 0x34, 0x84, 0xde, 0x80, 0x9e, 0x5a, 0x39, 0x34, 0xe2, -+ 0x84, 0xa6, 0xc9, 0xb7, 0xab, 0xb0, 0xcf, 0x15, 0x52, 0x4e, 0xea, 0xe4, 0x20, 0x4a, 0xf7, 0x7c, -+ 0x1a, 0x92, 0xe3, 0xb6, 0x58, 0x64, 0x06, 0x99, 0x6c, 0x5b, 0xb6, 0x86, 0xc4, 0x53, 0x37, 0xfc, -+ 0x96, 0x24, 0x3f, 0x03, 0x8a, 0xa7, 0x1e, 0xd0, 0x24, 0xe4, 0x4e, 0x44, 0x49, 0xc8, 0x75, 0x76, -+ 0x05, 0x89, 0x3a, 0x10, 0x18, 0xeb, 0xd7, 0x15, 0x68, 0xa8, 0x69, 0xb5, 0xe8, 0x32, 0xd3, 0x6f, -+ 0x5c, 0x95, 0xc8, 0x7a, 0x41, 0xca, 0x52, 0xdf, 0x35, 0xb9, 0x16, 0x71, 0x7c, 0x1e, 0xa8, 0x4c, -+ 0xad, 0x55, 0x3b, 0x0f, 0x64, 0x8a, 0x7e, 0x03, 0x7a, 0xd9, 0xa7, 0x52, 0xee, 0x2b, 0x15, 0xbb, -+ 0x29, 0x56, 0x92, 0x5d, 0xaa, 0xa9, 0xf5, 0x53, 0xd1, 0x5c, 0xa7, 0x93, 0xda, 0x15, 0xa8, 0x25, -+ 0xa9, 0x32, 0x62, 0x29, 0x30, 0x93, 0xf4, 0x23, 0x2b, 0x96, 0xe8, 0x21, 0xf4, 0x5c, 0xdf, 0x27, -+ 0xe2, 0xb8, 0x3b, 0x7d, 0x4a, 0xfc, 0x34, 0x48, 0x8b, 0x58, 0xeb, 0x6f, 0x15, 0xe8, 0x6f, 0xd3, -+ 0x68, 0xf6, 0x43, 0x32, 0xc5, 0xb9, 0x0c, 0x22, 0x95, 0xd4, 0xdf, 0x58, 0xb1, 0x16, 0x75, 0xe3, -+ 0x09, 0x99, 0x62, 0x15, 0x5a, 0xea, 0x66, 0x9b, 0x02, 0x21, 0xc3, 0xca, 0x6c, 0xa6, 0x03, 0xb0, -+ 0xae, 0xda, 0x7c, 0x46, 0x7d, 0x59, 0x21, 0xfb, 0x84, 0x39, 0xe9, 0xb8, 0xab, 0x6b, 0x2f, 0xfb, -+ 0x84, 0xc9, 0x2d, 0x6d, 0xc8, 0x92, 0x9c, 0xb8, 0xe6, 0x0d, 0x69, 0x28, 0x8c, 0x30, 0x64, 0x1d, -+ 0x1a, 0xf4, 0xe4, 0x24, 0xc6, 0x5c, 0xd6, 0xb2, 0x35, 0x5b, 0x43, 0x69, 0x9a, 0x6b, 0xe6, 0xd2, -+ 0xdc, 0x2d, 0x58, 0x95, 0xb3, 0xfd, 0x17, 0xcc, 0xf5, 0x48, 0x38, 0x31, 0xa9, 0x78, 0x0d, 0xd0, -+ 0x21, 0xa7, 0x51, 0x11, 0xfb, 0xe8, 0x77, 0x2b, 0x3a, 0x27, 0xea, 0x46, 0x17, 0x3d, 0x85, 0xfe, -+ 0xdc, 0x1f, 0x27, 0x48, 0x4f, 0x3e, 0xca, 0xff, 0x4f, 0x19, 0xad, 0x8f, 0xd5, 0x1f, 0x31, 0x63, -+ 0xf3, 0x47, 0xcc, 0x78, 0x37, 0x88, 0xf8, 0x0c, 0xed, 0x42, 0xaf, 0xf8, 0x17, 0x03, 0xba, 0x6b, -+ 0x0a, 0x85, 0x92, 0x3f, 0x1e, 0x2e, 0x65, 0xf3, 0x14, 0xfa, 0x73, 0xff, 0x36, 0x18, 0x7d, 0xca, -+ 0xff, 0x84, 0xb8, 0x94, 0xd1, 0x63, 0x68, 0xe7, 0xfe, 0x5e, 0x40, 0x43, 0xc5, 0x64, 0xf1, 0x1f, -+ 0x87, 0x4b, 0x19, 0x6c, 0x43, 0xb7, 0x30, 0xf1, 0x47, 0x23, 0x6d, 0x4f, 0xc9, 0xdf, 0x00, 0x97, -+ 0x32, 0xd9, 0x82, 0x76, 0x6e, 0xf0, 0x6e, 0xb4, 0x58, 0x9c, 0xee, 0x8f, 0xee, 0x94, 0xec, 0xe8, -+ 0xd4, 0xbb, 0x07, 0xdd, 0xc2, 0x98, 0xdc, 0x28, 0x52, 0x36, 0xa2, 0x1f, 0xdd, 0x2d, 0xdd, 0xd3, -+ 0x9c, 0x9e, 0x42, 0x7f, 0x6e, 0x68, 0x6e, 0x9c, 0x5b, 0x3e, 0x4b, 0xbf, 0xd4, 0xac, 0xcf, 0xe5, -+ 0x65, 0xe7, 0x7a, 0xa2, 0xdc, 0x65, 0x2f, 0x8e, 0xc8, 0x47, 0xf7, 0xca, 0x37, 0xb5, 0x56, 0xbb, -+ 0xd0, 0x2b, 0x4e, 0xc7, 0x0d, 0xb3, 0xd2, 0x99, 0xf9, 0xd5, 0x2f, 0xa7, 0x30, 0x28, 0xcf, 0x5e, -+ 0x4e, 0xd9, 0xfc, 0xfc, 0x52, 0x46, 0x4f, 0x00, 0x74, 0x07, 0xe4, 0x93, 0x30, 0xbd, 0xb2, 0x85, -+ 0xce, 0x2b, 0xbd, 0xb2, 0x92, 0x6e, 0xe9, 0x31, 0x80, 0x6a, 0x5c, 0x7c, 0x9a, 0x70, 0x74, 0xdb, -+ 0xa8, 0x31, 0xd7, 0x2d, 0x8d, 0x86, 0x8b, 0x1b, 0x0b, 0x0c, 0x30, 0x63, 0xd7, 0x61, 0xf0, 0x19, -+ 0x40, 0xd6, 0x10, 0x19, 0x06, 0x0b, 0x2d, 0xd2, 0x15, 0x3e, 0xe8, 0xe4, 0xdb, 0x1f, 0xa4, 0x6d, -+ 0x2d, 0x69, 0x89, 0xae, 0x60, 0xd1, 0x9f, 0x2b, 0x6f, 0x8b, 0x8f, 0x6d, 0xbe, 0xea, 0x1d, 0x2d, -+ 0x94, 0xb8, 0xe8, 0x43, 0xe8, 0xe4, 0xeb, 0x5a, 0xa3, 0x45, 0x49, 0xad, 0x3b, 0x2a, 0xd4, 0xb6, -+ 0xe8, 0x31, 0xf4, 0x8a, 0x55, 0x2b, 0xca, 0xc5, 0xc5, 0x42, 0x2d, 0x3b, 0xd2, 0x13, 0x9b, 0x1c, -+ 0xf9, 0xfb, 0x00, 0x59, 0x75, 0x6b, 0xdc, 0xb7, 0x50, 0xef, 0xce, 0x49, 0x7d, 0x02, 0x9d, 0x7c, -+ 0x26, 0x36, 0xea, 0x96, 0x64, 0xe7, 0xab, 0xb2, 0x56, 0x2e, 0x6b, 0x9b, 0xc7, 0xb7, 0x98, 0xc8, -+ 0xaf, 0xca, 0x5a, 0x85, 0xae, 0xcf, 0x24, 0x8b, 0xb2, 0x56, 0xf0, 0xaa, 0x5c, 0x5e, 0x6c, 0x91, -+ 0x8c, 0xfb, 0x4a, 0x1b, 0xa7, 0xab, 0x1e, 0x51, 0xbe, 0x1b, 0x30, 0xfe, 0x28, 0xe9, 0x10, 0xbe, -+ 0x27, 0xa8, 0xf3, 0x15, 0x7f, 0x2e, 0xa8, 0x4b, 0x1a, 0x81, 0x4b, 0x19, 0xed, 0x41, 0xff, 0xa9, -+ 0x29, 0xe6, 0x74, 0xa1, 0xa9, 0xd5, 0x29, 0x29, 0xac, 0x47, 0xa3, 0xb2, 0x2d, 0x1d, 0x59, 0x9f, -+ 0xc3, 0x60, 0xa1, 0xc8, 0x44, 0xf7, 0xd3, 0xc1, 0x62, 0x69, 0xf5, 0x79, 0xa9, 0x5a, 0xfb, 0xb0, -+ 0x32, 0x5f, 0x63, 0xa2, 0x57, 0xf4, 0xa5, 0x97, 0xd7, 0x9e, 0x97, 0xb2, 0xfa, 0x18, 0x9a, 0xa6, -+ 0xa6, 0x41, 0x7a, 0x80, 0x3b, 0x57, 0xe3, 0x5c, 0x76, 0x74, 0xab, 0xf3, 0xed, 0x77, 0xf7, 0x2b, -+ 0x7f, 0xff, 0xee, 0x7e, 0xe5, 0x9f, 0xdf, 0xdd, 0xaf, 0x1c, 0x37, 0xe4, 0xee, 0xfb, 0xff, 0x0e, -+ 0x00, 0x00, 0xff, 0xff, 0x8d, 0x89, 0xaa, 0x73, 0xc8, 0x21, 0x00, 0x00, - } -diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go -index b1dea816..c22ed39c 100644 ---- a/virtcontainers/agent.go -+++ b/virtcontainers/agent.go -@@ -235,7 +235,8 @@ type agent interface { - listInterfaces() ([]*vcTypes.Interface, error) - - // updateRoutes will tell the agent to update route table for an existed Sandbox. -- updateRoutes(routes []*vcTypes.Route) ([]*vcTypes.Route, error) -+ // increment specifies update route with increment mode -+ updateRoutes(routes []*vcTypes.Route, increment bool) ([]*vcTypes.Route, error) - - // listRoutes will tell the agent to list routes of an existed Sandbox - listRoutes() ([]*vcTypes.Route, error) -diff --git a/virtcontainers/api.go b/virtcontainers/api.go -index fb044fe1..2331eb94 100644 ---- a/virtcontainers/api.go -+++ b/virtcontainers/api.go -@@ -959,7 +959,7 @@ func ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTypes.Interface - } - - // UpdateRoutes is the virtcontainers update routes entry point. --func UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route) ([]*vcTypes.Route, error) { -+func UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) { - span, ctx := trace(ctx, "UpdateRoutes") - defer span.Finish() - -@@ -979,7 +979,7 @@ func UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route - } - defer s.releaseStatelessSandbox() - -- return s.UpdateRoutes(routes) -+ return s.UpdateRoutes(routes, op) - } - - // ListRoutes is the virtcontainers list routes entry point. -diff --git a/virtcontainers/api_test.go b/virtcontainers/api_test.go -index 5fb8ff4d..c01d47b8 100644 ---- a/virtcontainers/api_test.go -+++ b/virtcontainers/api_test.go -@@ -1699,7 +1699,7 @@ func TestNetworkOperation(t *testing.T) { - _, err = ListInterfaces(ctx, s.ID()) - assert.NoError(err) - -- _, err = UpdateRoutes(ctx, s.ID(), nil) -+ _, err = UpdateRoutes(ctx, s.ID(), nil, vcTypes.NetworkOpUpdate) - assert.NoError(err) - - _, err = ListRoutes(ctx, s.ID()) -diff --git a/virtcontainers/implementation.go b/virtcontainers/implementation.go -index 079afa0a..92cc4c12 100644 ---- a/virtcontainers/implementation.go -+++ b/virtcontainers/implementation.go -@@ -163,8 +163,8 @@ func (impl *VCImpl) ListInterfaces(ctx context.Context, sandboxID string) ([]*vc - } - - // UpdateRoutes implements the VC function of the same name. --func (impl *VCImpl) UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route) ([]*vcTypes.Route, error) { -- return UpdateRoutes(ctx, sandboxID, routes) -+func (impl *VCImpl) UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) { -+ return UpdateRoutes(ctx, sandboxID, routes, op) - } - - // ListRoutes implements the VC function of the same name. -diff --git a/virtcontainers/interfaces.go b/virtcontainers/interfaces.go -index 499b386e..6bb9ea22 100644 ---- a/virtcontainers/interfaces.go -+++ b/virtcontainers/interfaces.go -@@ -51,7 +51,7 @@ type VC interface { - AddInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) - RemoveInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) - ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTypes.Interface, error) -- UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route) ([]*vcTypes.Route, error) -+ UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) - ListRoutes(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error) - - CleanupContainer(ctx context.Context, sandboxID, containerID string, force bool) error -@@ -96,7 +96,7 @@ type VCSandbox interface { - AddInterface(inf *vcTypes.Interface) (*vcTypes.Interface, error) - RemoveInterface(inf *vcTypes.Interface) (*vcTypes.Interface, error) - ListInterfaces() ([]*vcTypes.Interface, error) -- UpdateRoutes(routes []*vcTypes.Route) ([]*vcTypes.Route, error) -+ UpdateRoutes(routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) - ListRoutes() ([]*vcTypes.Route, error) - IsCompatOldCNI() bool - } -diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go -index dfdd263b..16662949 100644 ---- a/virtcontainers/kata_agent.go -+++ b/virtcontainers/kata_agent.go -@@ -623,12 +623,13 @@ func (k *kataAgent) updateInterfaces(interfaces []*vcTypes.Interface) error { - return nil - } - --func (k *kataAgent) updateRoutes(routes []*vcTypes.Route) ([]*vcTypes.Route, error) { -+func (k *kataAgent) updateRoutes(routes []*vcTypes.Route, increment bool) ([]*vcTypes.Route, error) { - if routes != nil { - routesReq := &grpc.UpdateRoutesRequest{ - Routes: &grpc.Routes{ - Routes: k.convertToKataAgentRoutes(routes), - }, -+ Increment: increment, - } - resultingRoutes, err := k.sendReq(routesReq) - if err != nil { -@@ -858,7 +859,7 @@ func (k *kataAgent) startSandbox(sandbox *Sandbox) error { - if err = k.updateInterfaces(interfaces); err != nil { - return err - } -- if _, err = k.updateRoutes(routes); err != nil { -+ if _, err = k.updateRoutes(routes, false); err != nil { - return err - } - -diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go -index 2a2ddada..18a5a0a6 100644 ---- a/virtcontainers/kata_agent_test.go -+++ b/virtcontainers/kata_agent_test.go -@@ -943,7 +943,7 @@ func TestAgentNetworkOperation(t *testing.T) { - _, err = k.listInterfaces() - assert.Nil(err) - -- _, err = k.updateRoutes([]*vcTypes.Route{}) -+ _, err = k.updateRoutes([]*vcTypes.Route{}, false) - assert.Nil(err) - - _, err = k.listRoutes() -diff --git a/virtcontainers/network.go b/virtcontainers/network.go -index a1676ccd..f3757f84 100644 ---- a/virtcontainers/network.go -+++ b/virtcontainers/network.go -@@ -81,6 +81,8 @@ const ( - tcFilterNetModelStr = "tcfilter" - - noneNetModelStr = "none" -+ -+ localHostDeviceName = "lo" - ) - - //SetModel change the model string value -@@ -1485,3 +1487,163 @@ func validInterface(inf *vcTypes.Interface, enableCompatOldCNI bool) error { - - return nil - } -+ -+// verifyRouteDest verifies the route is valid or not -+func verifyRouteDest(ipMask *string) (nlIpMask *net.IPNet, err error) { -+ // set the defaule mask "32", just like the command `ip route add ${ip}`, -+ // it will give the "255.255.255.255"(/32) by default -+ if !strings.Contains(*ipMask, "/") { -+ *ipMask = *ipMask + "/32" -+ } -+ nlIpNet, err := verifyIPAndMask(*ipMask) -+ if err != nil { -+ return nil, err -+ } -+ -+ return nlIpNet, nil -+} -+ -+// generateAddRoute transform input vcTypes.Route info into netlink.Route info -+func generateAddRoute(route *vcTypes.Route) (r *netlink.Route, err error) { -+ // we must specify the device and destination -+ if route.Device == "" || route.Dest == "" { -+ return nil, fmt.Errorf("device and destination must be specified.") -+ } -+ -+ err = verifyInterfaceName(route.Device) -+ if err != nil { -+ return nil, err -+ } -+ -+ r = &netlink.Route{} -+ if route.Dest != "default" { -+ nlIpNet, err := verifyRouteDest(&route.Dest) -+ if err != nil { -+ return nil, err -+ } -+ r.Dst = nlIpNet -+ } else { -+ // if destination of route rule is "default", gateway is needed -+ if route.Gateway == "" { -+ return nil, fmt.Errorf("gateway must be specified when destination is \"default\"") -+ } -+ } -+ -+ if route.Gateway != "" { -+ nIP, err := verifyIP(route.Gateway) -+ if err != nil { -+ return nil, err -+ } -+ r.Gw = *nIP -+ } -+ -+ if route.Source != "" { -+ nIP, err := verifyIP(route.Source) -+ if err != nil { -+ return nil, err -+ } -+ r.Src = *nIP -+ } -+ -+ r.Scope = netlink.Scope(route.Scope) -+ -+ return r, nil -+} -+ -+// isSameRoute checks two input routes is same or not -+func isSameRoute(existing, r *netlink.Route, fuzzy bool) bool { -+ var ( -+ existDst string -+ inputDst string -+ ) -+ if existing.Dst != nil && existing.Dst.String() != "" { -+ existDst = existing.Dst.String() -+ } -+ if r.Dst != nil && r.Dst.String() != "" { -+ inputDst = r.Dst.String() -+ } -+ -+ if fuzzy { -+ // fuzzy matching -+ if inputDst != "" && existDst != inputDst { -+ return false -+ } else if inputDst == "" && existDst != "" { -+ return false -+ } -+ if r.Gw.String() != "" && existing.Gw.String() != r.Gw.String() { -+ return false -+ } -+ } else { -+ // exact matching -+ if existing.Dst != nil && r.Dst != nil && existing.Dst.String() != r.Dst.String() { -+ return false -+ } -+ if (existing.Dst == nil && r.Dst != nil) || (existing.Dst != nil && r.Dst == nil) { -+ return false -+ } -+ if existing.Gw.String() != r.Gw.String() { -+ return false -+ } -+ } -+ -+ return true -+} -+ -+func addOneRoute(ns *NetworkNamespace, route *vcTypes.Route) (added *vcTypes.Route, err error) { -+ add, err := generateAddRoute(route) -+ if err != nil { -+ return nil, err -+ } -+ -+ // add the route for "lo" loopback device -+ if route.Device == localHostDeviceName { -+ added = &vcTypes.Route{ -+ Dest: route.Dest, -+ Gateway: route.Gateway, -+ Device: route.Device, -+ } -+ return added, nil -+ } -+ -+ // add the route for exist network enpoints -+ for _, ep := range ns.Endpoints { -+ if ep.Name() != route.Device { -+ continue -+ } -+ -+ netInfo := ep.Properties() -+ for _, exist := range ep.Properties().Routes { -+ if isSameRoute(&exist, add, false) { -+ return nil, fmt.Errorf("route rule %v already exits", add) -+ } -+ } -+ // flush the netInfo.Routes with new added route -+ netInfo.Routes = append(netInfo.Routes, *add) -+ ep.SetProperties(netInfo) -+ added = &vcTypes.Route{ -+ Dest: route.Dest, -+ Gateway: route.Gateway, -+ Device: route.Device, -+ } -+ break -+ } -+ -+ return added, nil -+} -+ -+func updateRoute(ns *NetworkNamespace, route *vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) { -+ var updRoutes []*vcTypes.Route -+ -+ if op == vcTypes.NetworkOpAdd { -+ added, err := addOneRoute(ns, route) -+ if err != nil { -+ return nil, err -+ } -+ if added == nil { -+ return nil, fmt.Errorf("no matching endpoints for route device: %#v", route.Device) -+ } -+ updRoutes = append(updRoutes, added) -+ } -+ -+ return updRoutes, nil -+} -diff --git a/virtcontainers/noop_agent.go b/virtcontainers/noop_agent.go -index 6e211bca..b19705c3 100644 ---- a/virtcontainers/noop_agent.go -+++ b/virtcontainers/noop_agent.go -@@ -117,7 +117,7 @@ func (n *noopAgent) listInterfaces() ([]*vcTypes.Interface, error) { - } - - // updateRoutes is the Noop agent Routes update implementation. It does nothing. --func (n *noopAgent) updateRoutes(routes []*vcTypes.Route) ([]*vcTypes.Route, error) { -+func (n *noopAgent) updateRoutes(routes []*vcTypes.Route, increment bool) ([]*vcTypes.Route, error) { - return nil, nil - } - -@@ -239,4 +239,4 @@ func (n *noopAgent) load(s persistapi.AgentState) {} - - func (n *noopAgent) getProxyPid() int { - return -1 --} -\ No newline at end of file -+} -diff --git a/virtcontainers/noop_agent_test.go b/virtcontainers/noop_agent_test.go -index 97c56f21..63f52e58 100644 ---- a/virtcontainers/noop_agent_test.go -+++ b/virtcontainers/noop_agent_test.go -@@ -201,7 +201,7 @@ func TestNoopAgentListInterfaces(t *testing.T) { - func TestNoopAgentUpdateRoutes(t *testing.T) { - assert := assert.New(t) - n := &noopAgent{} -- _, err := n.updateRoutes(nil) -+ _, err := n.updateRoutes(nil, false) - assert.NoError(err) - } - -diff --git a/virtcontainers/pkg/types/types.go b/virtcontainers/pkg/types/types.go -index fcc63d84..71fe7fbb 100644 ---- a/virtcontainers/pkg/types/types.go -+++ b/virtcontainers/pkg/types/types.go -@@ -39,3 +39,17 @@ type Route struct { - Source string - Scope uint32 - } -+ -+//NetworkOp describes network operation -+type NetworkOp string -+ -+const ( -+ //NetworkOpAdd adds the network interface -+ NetworkOpAdd NetworkOp = "add" -+ -+ //NetworkOpRemove removes the network interface -+ NetworkOpRemove NetworkOp = "remove" -+ -+ //NetworkOpUpdate updates the network interface -+ NetworkOpUpdate NetworkOp = "update" -+) -diff --git a/virtcontainers/pkg/vcmock/mock.go b/virtcontainers/pkg/vcmock/mock.go -index 51a2c216..9c50e0e4 100644 ---- a/virtcontainers/pkg/vcmock/mock.go -+++ b/virtcontainers/pkg/vcmock/mock.go -@@ -273,7 +273,7 @@ func (m *VCMock) ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTyp - } - - // UpdateRoutes implements the VC function of the same name. --func (m *VCMock) UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route) ([]*vcTypes.Route, error) { -+func (m *VCMock) UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) { - if m.UpdateRoutesFunc != nil { - return m.UpdateRoutesFunc(ctx, sandboxID, routes) - } -diff --git a/virtcontainers/pkg/vcmock/sandbox.go b/virtcontainers/pkg/vcmock/sandbox.go -index 11b83ccd..7329e1a7 100644 ---- a/virtcontainers/pkg/vcmock/sandbox.go -+++ b/virtcontainers/pkg/vcmock/sandbox.go -@@ -204,7 +204,7 @@ func (s *Sandbox) ListInterfaces() ([]*vcTypes.Interface, error) { - } - - // UpdateRoutes implements the VCSandbox function of the same name. --func (s *Sandbox) UpdateRoutes(routes []*vcTypes.Route) ([]*vcTypes.Route, error) { -+func (s *Sandbox) UpdateRoutes(routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) { - return nil, nil - } - -diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go -index f6826812..f226af4e 100644 ---- a/virtcontainers/sandbox.go -+++ b/virtcontainers/sandbox.go -@@ -1031,8 +1031,33 @@ func (s *Sandbox) ListInterfaces() ([]*vcTypes.Interface, error) { - } - - // UpdateRoutes updates the sandbox route table (e.g. for portmapping support). --func (s *Sandbox) UpdateRoutes(routes []*vcTypes.Route) ([]*vcTypes.Route, error) { -- return s.agent.updateRoutes(routes) -+func (s *Sandbox) UpdateRoutes(routes []*vcTypes.Route, op vcTypes.NetworkOp) (r []*vcTypes.Route, err error) { -+ var allUpdateRoutes []*vcTypes.Route -+ -+ increment := true -+ if op == vcTypes.NetworkOpUpdate { -+ allUpdateRoutes = routes -+ increment = false -+ } else { -+ for _, r := range routes { -+ updRoutes, err := updateRoute(&s.networkNS, r, op) -+ if err != nil { -+ return nil, err -+ } -+ allUpdateRoutes = append(allUpdateRoutes, updRoutes...) -+ } -+ } -+ -+ r, err = s.agent.updateRoutes(allUpdateRoutes, increment) -+ if err != nil { -+ return nil, err -+ } -+ -+ if err = s.Save(); err != nil { -+ return nil, err -+ } -+ -+ return r, nil - } - - // ListRoutes lists all routes and their configurations in the sandbox. --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0029-network-add-kata-network-del-route-subcommand.patch b/runtime/patches/0029-network-add-kata-network-del-route-subcommand.patch deleted file mode 100644 index a4c8cc8..0000000 --- a/runtime/patches/0029-network-add-kata-network-del-route-subcommand.patch +++ /dev/null @@ -1,192 +0,0 @@ -From 51a7270987557ab12ea735fc9781725d1ce1b0a6 Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Mon, 17 Aug 2020 15:36:32 +0800 -Subject: [PATCH 29/50] network: add kata-network del-route subcommand - -reason: add kata-network del-route subcommand to delete the -specified route in the Kata VM, del-route is more efficient -than the update-routes subcommand. - -Signed-off-by: jiangpengfei ---- - cli/network.go | 24 ++++++++++ - virtcontainers/network.go | 115 ++++++++++++++++++++++++++++++++++++++++++++++ - 2 files changed, 139 insertions(+) - -diff --git a/cli/network.go b/cli/network.go -index 2265f54b..046d0ee9 100644 ---- a/cli/network.go -+++ b/cli/network.go -@@ -45,6 +45,7 @@ var kataNetworkCLICommand = cli.Command{ - updateRoutesCommand, - listRoutesCommand, - addRoutesCommand, -+ deleteRoutesCommand, - }, - Action: func(context *cli.Context) error { - return cli.ShowSubcommandHelp(context) -@@ -155,6 +156,29 @@ var addRoutesCommand = cli.Command{ - }, - } - -+var deleteRoutesCommand = cli.Command{ -+ Name: "del-route", -+ Usage: "delete one route for a container", -+ ArgsUsage: `del-route file or - for stdin -+ file or stdin for example: -+ { -+ "dest":"<[[/mask] | "default" ]>", -+ "gateway":"[ip]", -+ "device":"[tap-name]", -+ } -+ Only destination is required and others are optional, -+ if device is empty, means search every device`, -+ Flags: []cli.Flag{}, -+ Action: func(context *cli.Context) error { -+ ctx, err := cliContextToContext(context) -+ if err != nil { -+ return err -+ } -+ -+ return networkModifyCommand(ctx, context.Args().First(), context.Args().Get(1), routeType, vcTypes.NetworkOpRemove) -+ }, -+} -+ - var listRoutesCommand = cli.Command{ - Name: "list-routes", - Usage: "list network routes in a container", -diff --git a/virtcontainers/network.go b/virtcontainers/network.go -index f3757f84..c7066a11 100644 ---- a/virtcontainers/network.go -+++ b/virtcontainers/network.go -@@ -1631,6 +1631,110 @@ func addOneRoute(ns *NetworkNamespace, route *vcTypes.Route) (added *vcTypes.Rou - return added, nil - } - -+func generateRmRoute(route *vcTypes.Route) (r *netlink.Route, err error) { -+ // destination is required and others are optional -+ if route.Dest == "" { -+ return nil, fmt.Errorf("destination must be specified when remove route.") -+ } -+ -+ if route.Device != "" { -+ err = verifyInterfaceName(route.Device) -+ if err != nil { -+ return nil, err -+ } -+ } -+ -+ r = &netlink.Route{} -+ if route.Dest != "default" { -+ nlIpNet, err := verifyRouteDest(&route.Dest) -+ if err != nil { -+ return nil, err -+ } -+ r.Dst = nlIpNet -+ } -+ -+ if route.Gateway != "" { -+ nIP, err := verifyIP(route.Gateway) -+ if err != nil { -+ return nil, err -+ } -+ r.Gw = *nIP -+ } -+ -+ if route.Source != "" { -+ nIP, err := verifyIP(route.Source) -+ if err != nil { -+ return nil, err -+ } -+ r.Src = *nIP -+ } -+ -+ r.Scope = netlink.Scope(route.Scope) -+ -+ return r, nil -+} -+ -+// parseToGrpcRoute convert the netlink.Route to vcTypes.Route and deleted route.Dest will -+// be added a prefix "-" -+func parseToGrpcRoute(device string, route *netlink.Route, add bool) (r *vcTypes.Route) { -+ r = &vcTypes.Route{ -+ Device: device, -+ } -+ if route.Dst != nil && route.Dst.String() != "" { -+ r.Dest = route.Dst.String() -+ if add == false { -+ r.Dest = "-" + route.Dst.String() -+ } -+ } else if route.Dst == nil { -+ r.Dest = "default" -+ if add == false { -+ r.Dest = "-default" -+ } -+ } -+ if route.Gw != nil && route.Gw.String() != "" { -+ r.Gateway = route.Gw.String() -+ } -+ -+ return r -+} -+ -+func removeRoutes(ns *NetworkNamespace, route *vcTypes.Route) (removed []*vcTypes.Route, err error) { -+ del, err := generateRmRoute(route) -+ if err != nil { -+ return nil, err -+ } -+ -+ // remove the lo device related routes -+ if route.Device == localHostDeviceName { -+ removed = append(removed, parseToGrpcRoute(localHostDeviceName, del, false)) -+ -+ return removed, nil -+ } -+ -+ for _, ep := range ns.Endpoints { -+ // if device is empty, means search every device -+ if route.Device != "" && ep.Name() != route.Device { -+ continue -+ } -+ -+ netInfo := ep.Properties() -+ for i, exist := range ep.Properties().Routes { -+ if isSameRoute(&exist, del, true) { -+ // need remove -+ netInfo.Routes = append(netInfo.Routes[:i], netInfo.Routes[i+1:]...) -+ ep.SetProperties(netInfo) -+ dev := route.Device -+ if route.Device == "" { -+ dev = netInfo.Iface.Name -+ } -+ removed = append(removed, parseToGrpcRoute(dev, del, false)) -+ } -+ } -+ } -+ -+ return removed, nil -+} -+ - func updateRoute(ns *NetworkNamespace, route *vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) { - var updRoutes []*vcTypes.Route - -@@ -1645,5 +1749,16 @@ func updateRoute(ns *NetworkNamespace, route *vcTypes.Route, op vcTypes.NetworkO - updRoutes = append(updRoutes, added) - } - -+ if op == vcTypes.NetworkOpRemove { -+ removed, err := removeRoutes(ns, route) -+ if err != nil { -+ return nil, err -+ } -+ if len(removed) <= 0 { -+ return nil, fmt.Errorf("route of device %s with destination %s is not found", route.Device, route.Dest) -+ } -+ updRoutes = removed -+ } -+ - return updRoutes, nil - } --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0030-network-kata-network-list-routes-support-display-com.patch b/runtime/patches/0030-network-kata-network-list-routes-support-display-com.patch deleted file mode 100644 index 7d62827..0000000 --- a/runtime/patches/0030-network-kata-network-list-routes-support-display-com.patch +++ /dev/null @@ -1,104 +0,0 @@ -From 9cf769178b8f73c7fd624895589e918d6c7b0645 Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Mon, 17 Aug 2020 16:29:17 +0800 -Subject: [PATCH 30/50] network: kata-network list-routes support display - compatible format - -reason: kata-network list-routes subcommand support display compatible -format when enable_compat_old_cni config is enabled. - -Signed-off-by: jiangpengfei ---- - virtcontainers/api.go | 13 ++++++++++++- - virtcontainers/network.go | 29 +++++++++++++++++++++++++++++ - virtcontainers/pkg/types/types.go | 10 +++++----- - 3 files changed, 46 insertions(+), 6 deletions(-) - -diff --git a/virtcontainers/api.go b/virtcontainers/api.go -index 2331eb94..a4bf41bb 100644 ---- a/virtcontainers/api.go -+++ b/virtcontainers/api.go -@@ -1003,7 +1003,18 @@ func ListRoutes(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error) - } - defer s.releaseStatelessSandbox() - -- return s.ListRoutes() -+ routes, err := s.ListRoutes() -+ if err != nil { -+ return nil, err -+ } -+ -+ // If enable_compat_old_cni is enabled, convert routes info to -+ // compatible display format -+ if s.config.NetworkConfig.EnableCompatOldCNI { -+ return convertToDisplayRoutes(&routes), nil -+ } -+ -+ return routes, nil - } - - // CleanupContaienr is used by shimv2 to stop and delete a container exclusively, once there is no container -diff --git a/virtcontainers/network.go b/virtcontainers/network.go -index c7066a11..488bd00c 100644 ---- a/virtcontainers/network.go -+++ b/virtcontainers/network.go -@@ -1384,6 +1384,35 @@ func convertToCompatInterfaces(es *[]Endpoint) []*vcTypes.Interface { - return infs - } - -+// convertToDisplayRoutes convert the default route format to more simple -+// route display format, it is called when enable_compat_old_cni config -+// is enabled. -+func convertToDisplayRoutes(routes *[]*vcTypes.Route) []*vcTypes.Route { -+ var displayRoutes []*vcTypes.Route -+ if routes == nil { -+ return displayRoutes -+ } -+ for _, r := range *routes { -+ // we don't support IPV6 temporarily, so we need to filter ":" which -+ // is the characteristics of IPV6 for those default routes -+ if strings.Contains(r.Dest, ":") { -+ continue -+ } -+ -+ defaultDest := r.Dest -+ if r.Dest == "" { -+ defaultDest = "default" -+ } -+ displayRoutes = append(displayRoutes, &vcTypes.Route{ -+ Dest: defaultDest, -+ Gateway: r.Gateway, -+ Device: r.Device, -+ }) -+ } -+ -+ return displayRoutes -+} -+ - // verifyInterfaceName verifies the interface name valid or not - func verifyInterfaceName(name string) error { - // verify `Name` before `Tapname` because of the strict rules -diff --git a/virtcontainers/pkg/types/types.go b/virtcontainers/pkg/types/types.go -index 71fe7fbb..b41b0c75 100644 ---- a/virtcontainers/pkg/types/types.go -+++ b/virtcontainers/pkg/types/types.go -@@ -33,11 +33,11 @@ type Interface struct { - - // Route describes a network route. - type Route struct { -- Dest string -- Gateway string -- Device string -- Source string -- Scope uint32 -+ Dest string `json:"dest,omitempty"` -+ Gateway string `json:"gateway,omitempty"` -+ Device string `json:"device,omitempty"` -+ Source string `json:"source,omitempty"` -+ Scope uint32 `json:"scope,omitempty"` - } - - //NetworkOp describes network operation --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0031-device_mangaer-check-VFIO-when-create-device.patch b/runtime/patches/0031-device_mangaer-check-VFIO-when-create-device.patch deleted file mode 100644 index 3784e24..0000000 --- a/runtime/patches/0031-device_mangaer-check-VFIO-when-create-device.patch +++ /dev/null @@ -1,41 +0,0 @@ -From 326e90f97cf5ace73dff95257f7b4faa43c56f99 Mon Sep 17 00:00:00 2001 -From: holyfei -Date: Mon, 17 Aug 2020 16:11:35 +0800 -Subject: [PATCH 31/50] device_mangaer: check VFIO when create device - -reason: check VFIO when create device, block device can be -reused and VFIO device can not - -Signed-off-by: yangfeiyu ---- - virtcontainers/device/manager/manager.go | 7 ++++++- - 1 file changed, 6 insertions(+), 1 deletion(-) - -diff --git a/virtcontainers/device/manager/manager.go b/virtcontainers/device/manager/manager.go -index 6a2b665a..d24a29ab 100644 ---- a/virtcontainers/device/manager/manager.go -+++ b/virtcontainers/device/manager/manager.go -@@ -9,6 +9,7 @@ package manager - import ( - "encoding/hex" - "errors" -+ "fmt" - "sync" - - "github.com/sirupsen/logrus" -@@ -115,7 +116,11 @@ func (dm *deviceManager) createDevice(devInfo config.DeviceInfo) (dev api.Device - }() - - if existingDev := dm.findDeviceByMajorMinor(devInfo.Major, devInfo.Minor); existingDev != nil { -- return existingDev, nil -+ if isVFIO(devInfo.HostPath) { -+ return nil, fmt.Errorf("device %s is replicated in the same Pod!", devInfo.ContainerPath) -+ } else { -+ return existingDev, nil -+ } - } - - // device ID must be generated by manager instead of device itself --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0032-network-add-more-detail-usage-for-update-routes-subc.patch b/runtime/patches/0032-network-add-more-detail-usage-for-update-routes-subc.patch deleted file mode 100644 index 51f5b92..0000000 --- a/runtime/patches/0032-network-add-more-detail-usage-for-update-routes-subc.patch +++ /dev/null @@ -1,46 +0,0 @@ -From 77711b531867be899df5d2c59a525ea1b7f0e6ac Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Mon, 17 Aug 2020 17:14:35 +0800 -Subject: [PATCH 32/50] network: add more detail usage for update-routes - subcommand - -reason: add more detail usage for update-routes subcommand -to explain how to use update-routes subcommand - -Signed-off-by: jiangpengfei ---- - cli/network.go | 18 ++++++++++++++---- - 1 file changed, 14 insertions(+), 4 deletions(-) - -diff --git a/cli/network.go b/cli/network.go -index 046d0ee9..3dd0971e 100644 ---- a/cli/network.go -+++ b/cli/network.go -@@ -120,10 +120,20 @@ var listIfacesCommand = cli.Command{ - } - - var updateRoutesCommand = cli.Command{ -- Name: "update-routes", -- Usage: "update routes of a container", -- ArgsUsage: `update-routes file or - for stdin`, -- Flags: []cli.Flag{}, -+ Name: "update-routes", -+ Usage: "update routes of a container", -+ ArgsUsage: `update-routes file or - for stdin -+ file or stdin for example: -+ [ -+ { -+ "dest":"<[[/mask] | "default" ]>", -+ "gateway":"[ip]", -+ "device":"[tap-name]", -+ "source": "[source]", -+ "scope":[scope] -+ } -+ ]`, -+ Flags: []cli.Flag{}, - Action: func(context *cli.Context) error { - ctx, err := cliContextToContext(context) - if err != nil { --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0033-network-do-not-delete-the-exist-tap-device-in-the-ho.patch b/runtime/patches/0033-network-do-not-delete-the-exist-tap-device-in-the-ho.patch deleted file mode 100644 index c6a30ff..0000000 --- a/runtime/patches/0033-network-do-not-delete-the-exist-tap-device-in-the-ho.patch +++ /dev/null @@ -1,80 +0,0 @@ -From 7ab7ff54efa3925a8d372f7830d31b87f8d01ea8 Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Mon, 17 Aug 2020 17:39:18 +0800 -Subject: [PATCH 33/50] network: do not delete the exist tap device in the host - -reason: If hotplug a exist tap device into Kata VM, kata-runtime -should not delete this exist tap device, because this tap device -is controlled by other network components. - -Signed-off-by: jiangpengfei ---- - virtcontainers/tap_endpoint.go | 16 +++++++++++++--- - virtcontainers/veth_endpoint.go | 3 +++ - 2 files changed, 16 insertions(+), 3 deletions(-) - -diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go -index c897670e..2cf70dce 100644 ---- a/virtcontainers/tap_endpoint.go -+++ b/virtcontainers/tap_endpoint.go -@@ -77,7 +77,7 @@ func (endpoint *TapEndpoint) Detach(netNsCreated bool, netNsPath string) error { - - networkLogger().WithField("endpoint-type", TapEndpointType).Info("Detaching endpoint") - return doNetNS(netNsPath, func(_ ns.NetNS) error { -- return unTapNetwork(endpoint.TapInterface.TAPIface.Name) -+ return unTapNetwork(endpoint) - }) - } - -@@ -91,6 +91,9 @@ func (endpoint *TapEndpoint) HotAttach(h hypervisor) error { - - if _, err := h.hotplugAddDevice(endpoint, netDev); err != nil { - networkLogger().WithError(err).Error("Error attach tap ep") -+ if errUnTap := unTapNetwork(endpoint); errUnTap != nil { -+ networkLogger().WithError(errUnTap).Errorf("Error rollback tap %s", endpoint.TapInterface.TAPIface.Name) -+ } - return err - } - return nil -@@ -100,7 +103,7 @@ func (endpoint *TapEndpoint) HotAttach(h hypervisor) error { - func (endpoint *TapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { - networkLogger().Info("Hot detaching tap endpoint") - if err := doNetNS(netNsPath, func(_ ns.NetNS) error { -- return unTapNetwork(endpoint.TapInterface.TAPIface.Name) -+ return unTapNetwork(endpoint) - }); err != nil { - networkLogger().WithError(err).Warn("Error un-bridging tap ep") - } -@@ -185,7 +188,14 @@ func tapNetwork(endpoint *TapEndpoint, numCPUs uint32, disableVhostNet bool) err - return nil - } - --func unTapNetwork(name string) error { -+func unTapNetwork(endpoint *TapEndpoint) error { -+ // length of VMFDs == 0 means that the endpoint is already exist in the host, -+ // no created by kata, so we don't need to remove it when detach -+ if len(endpoint.TapInterface.VMFds) == 0 { -+ return nil -+ } -+ -+ name := endpoint.TapInterface.TAPIface.Name - netHandle, err := netlink.NewHandle() - if err != nil { - return err -diff --git a/virtcontainers/veth_endpoint.go b/virtcontainers/veth_endpoint.go -index 9ece6a74..0f2ec9ba 100644 ---- a/virtcontainers/veth_endpoint.go -+++ b/virtcontainers/veth_endpoint.go -@@ -119,6 +119,9 @@ func (endpoint *VethEndpoint) HotAttach(h hypervisor) error { - - if _, err := h.hotplugAddDevice(endpoint, netDev); err != nil { - networkLogger().WithError(err).Error("Error attach virtual ep") -+ if errDisconn := xDisconnectVMNetwork(endpoint); errDisconn != nil { -+ networkLogger().WithError(errDisconn).Error("Error rollback virtual ep") -+ } - return err - } - return nil --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0034-kata-runtime-add-kata-ipvs-command.patch b/runtime/patches/0034-kata-runtime-add-kata-ipvs-command.patch deleted file mode 100644 index 407e129..0000000 --- a/runtime/patches/0034-kata-runtime-add-kata-ipvs-command.patch +++ /dev/null @@ -1,1273 +0,0 @@ -From 4113462fe4bef73cd1f3a9c74f1cf0b2a1b643bd Mon Sep 17 00:00:00 2001 -From: xiadanni1 -Date: Tue, 18 Aug 2020 03:26:32 +0800 -Subject: [PATCH 34/50] kata-runtime: add kata-ipvs command - -reason: add kata-ipvs command to update IPVS rules -in VM. - -Signed-off-by: xiadanni1 ---- - cli/ipvsadm.go | 239 ++++++++ - cli/main.go | 1 + - .../agent/protocols/grpc/agent.pb.go | 663 +++++++++++++++------ - virtcontainers/agent.go | 3 + - virtcontainers/api.go | 25 + - virtcontainers/implementation.go | 5 + - virtcontainers/interfaces.go | 3 + - virtcontainers/kata_agent.go | 24 + - virtcontainers/noop_agent.go | 4 + - virtcontainers/pkg/vcmock/mock.go | 8 + - virtcontainers/pkg/vcmock/sandbox.go | 5 + - virtcontainers/pkg/vcmock/types.go | 2 + - virtcontainers/sandbox.go | 5 + - 13 files changed, 806 insertions(+), 181 deletions(-) - create mode 100644 cli/ipvsadm.go - -diff --git a/cli/ipvsadm.go b/cli/ipvsadm.go -new file mode 100644 -index 00000000..51e4bf41 ---- /dev/null -+++ b/cli/ipvsadm.go -@@ -0,0 +1,239 @@ -+// Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. -+// SPDX-License-Identifier: Apache-2.0 -+// Description: IPVS related common functions -+// Author: xiadanni -+// Create: 2020-08-01 -+ -+package main -+ -+import ( -+ "bufio" -+ "bytes" -+ "fmt" -+ "io" -+ "os" -+ "regexp" -+ "strconv" -+ "strings" -+ -+ "github.com/kata-containers/agent/protocols/grpc" -+ "github.com/kata-containers/runtime/virtcontainers/types" -+ "github.com/urfave/cli" -+) -+ -+const ( -+ ruleLimitMax = 20000 -+ restoreHead = "restore" -+) -+ -+var kataIPVSCLICommand = cli.Command{ -+ Name: "kata-ipvs", -+ Usage: "manage IPVS service for container", -+ Subcommands: []cli.Command{ -+ IPVSadmCommand, -+ CleanupCommand, -+ }, -+ Action: func(context *cli.Context) error { -+ return cli.ShowSubcommandHelp(context) -+ }, -+} -+ -+var IPVSadmCommand = cli.Command{ -+ Name: "ipvsadm", -+ Usage: "manage virtual server and real server for container", -+ ArgsUsage: ``, -+ Flags: []cli.Flag{ -+ cli.StringFlag{ -+ Name: "parameters", -+ Usage: "parameters of IPVSadm command", -+ }, -+ cli.StringFlag{ -+ Name: "restore", -+ Usage: "restore rules from stdin, - for stdin", -+ }, -+ }, -+ Action: func(context *cli.Context) error { -+ ctx, err := cliContextToContext(context) -+ if err != nil { -+ return err -+ } -+ -+ status, sandboxID, err := getExistingContainerInfo(ctx, context.Args().First()) -+ if err != nil { -+ return err -+ } -+ -+ if status.State.State != types.StateReady && -+ status.State.State != types.StateRunning { -+ return fmt.Errorf("container with id %s is not running", status.ID) -+ } -+ -+ var IPVSRule string -+ input := context.String("restore") -+ if input != "" { -+ if input != "-" { -+ return fmt.Errorf("invalid kata-IPVS command, please use --restore - for stdin") -+ } -+ ruleFile, ruleCnt, err := getRestoreFile(os.Stdin) -+ if err != nil { -+ return err -+ } -+ // restore format: restore|ruleCnt|ruleFileContext -+ IPVSRule = restoreHead + "|" + strconv.Itoa(ruleCnt) + "|" + ruleFile -+ } else { -+ parameters := context.String("parameters") -+ IPVSRule = "ipvsadm " + parameters -+ IPVSRule, err = checkIPVSRule(IPVSRule) -+ if err != nil { -+ return err -+ } -+ -+ // we just use NAT mode -+ if strings.Contains(IPVSRule, "add-server") || strings.Contains(IPVSRule, "edit-server") || -+ strings.Contains(IPVSRule, " -a ") || strings.Contains(IPVSRule, " -e ") { -+ IPVSRule = IPVSRule + " -m" -+ } -+ } -+ -+ IPVSReq := &grpc.UpdateIPVSRequest{ -+ IPVSReq: IPVSRule, -+ } -+ resultingIPVS, err := vci.UpdateIPVSRule(ctx, sandboxID, IPVSReq) -+ if err != nil { -+ kataLog.WithField("resulting-IPVSadm", fmt.Sprintf("%+v", resultingIPVS)). -+ WithError(err).Error("exec IPVSadm failed") -+ return err -+ } -+ -+ if resultingIPVS.IPVSRes != "" { -+ fmt.Printf(resultingIPVS.IPVSRes) -+ } -+ -+ return nil -+ }, -+} -+ -+var CleanupCommand = cli.Command{ -+ Name: "cleanup", -+ Usage: "delete conntrack of destination address", -+ ArgsUsage: ``, -+ Flags: []cli.Flag{ -+ cli.StringFlag{ -+ Name: "parameters", -+ Usage: "parameters of cleanup command", -+ }, -+ }, -+ Action: func(context *cli.Context) error { -+ ctx, err := cliContextToContext(context) -+ if err != nil { -+ return err -+ } -+ -+ status, sandboxID, err := getExistingContainerInfo(ctx, context.Args().First()) -+ if err != nil { -+ return err -+ } -+ -+ if status.State.State != types.StateReady && -+ status.State.State != types.StateRunning { -+ return fmt.Errorf("container with id %s is not running", status.ID) -+ } -+ -+ parameters := context.String("parameters") -+ IPVSRule := "conntrack -D " + parameters -+ -+ _, err = checkIPVSRule(IPVSRule) -+ if err != nil { -+ return err -+ } -+ -+ IPVSReq := &grpc.UpdateIPVSRequest{ -+ IPVSReq: IPVSRule, -+ } -+ resultingIPVS, err := vci.UpdateIPVSRule(ctx, sandboxID, IPVSReq) -+ if err != nil { -+ kataLog.WithField("resulting-conntrack", fmt.Sprintf("%+v", resultingIPVS)). -+ WithError(err).Error("exec conntrack failed") -+ return err -+ } -+ -+ fmt.Printf(resultingIPVS.IPVSRes) -+ -+ return nil -+ }, -+} -+ -+var ( -+ match = regexp.MustCompile -+ ipReg = `((2[0-4]\d|25[0-5]|[01]?\d\d?)\.){3}(2[0-4]\d|25[0-5]|[01]?\d\d?)` -+ portReg = `:(6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5]|[1-5]\d{4}|[1-9]\d{0,3})\b` -+ -+ addOrEditServiceReg = match(`(--(add|edit)-service|-A|-E)\s+`) -+ serverTopReg = match(`(--(add|edit|delete)-server|-a|-e|-d)\s+`) -+ schedulerReg = match(`(--scheduler|-s)(\s+|=)(rr|wrr|lc|wlc|lblc|lblcr|dh|sh|sed|nq)\b`) -+ persistentReg = match(`(--persistent|-p)(\s+|=)\d{1,7}\b`) -+ persistentZeroReg = match(`(--persistent|-p)\s+0\b`) -+ serviceReg = match(`(--(tcp|udp)-service|-t|-u)(\s+|=)` + ipReg + portReg) -+ serverReg = match(`(--real-server|-r)(\s+|=)` + ipReg + portReg) -+ delServiceReg = match(`(--delete-service|-D)\s+` + serviceReg.String()) -+ conntrackReg = match(`conntrack\s+-D\s+(--orig-dst|-d)\s+` + ipReg + `\s+(--protonum|-p)\s+(tcp|udp)\s*$`) -+ otherReg = match(`((--list|-L|-l|--clear|-C|--set).*)|` + conntrackReg.String()) -+) -+ -+func checkIPVSRule(rule string) (string, error) { -+ switch { -+ // match add-service or edit-service rule format -+ case addOrEditServiceReg.MatchString(rule) && serviceReg.MatchString(rule) && -+ schedulerReg.MatchString(rule) && persistentReg.MatchString(rule): -+ rule = persistentZeroReg.ReplaceAllString(rule, "") -+ return rule, nil -+ // match delete-service rule format -+ case delServiceReg.MatchString(rule): -+ return rule, nil -+ // match add/edit/delete-server rule format -+ case serverTopReg.MatchString(rule) && serviceReg.MatchString(rule) && serverReg.MatchString(rule): -+ return rule, nil -+ // match cleanup/list/set/clear rule format -+ case otherReg.MatchString(rule): -+ return rule, nil -+ default: -+ return "", fmt.Errorf("invalid input of IPVS rule") -+ } -+} -+ -+func getRestoreFile(stdin io.Reader) (string, int, error) { -+ var ruleContent bytes.Buffer -+ firstFlag := true -+ ruleCnt := 0 -+ scanner := bufio.NewScanner(stdin) -+ for scanner.Scan() { -+ if err := scanner.Err(); err != nil { -+ return "", 0, err -+ } -+ newLine := scanner.Text() -+ -+ _, err := checkIPVSRule(newLine) -+ if err != nil { -+ return "", 0, err -+ } -+ -+ ruleCnt++ -+ -+ if ruleCnt > ruleLimitMax { -+ return "", 0, fmt.Errorf("Rules exceed limit %v", ruleLimitMax) -+ } -+ -+ if firstFlag { -+ firstFlag = false -+ } else { -+ ruleContent.WriteString("\n") -+ } -+ ruleContent.WriteString(newLine) -+ } -+ -+ if ruleCnt == 0 { -+ return "", 0, fmt.Errorf("Rule file has no effective rule.") -+ } -+ -+ return ruleContent.String(), ruleCnt, nil -+} -diff --git a/cli/main.go b/cli/main.go -index 5eb2fb19..95a0786b 100644 ---- a/cli/main.go -+++ b/cli/main.go -@@ -146,6 +146,7 @@ var runtimeCommands = []cli.Command{ - kataNetworkCLICommand, - kataOverheadCLICommand, - factoryCLICommand, -+ kataIPVSCLICommand, - } - - // runtimeBeforeSubcommands is the function to run before command-line -diff --git a/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go b/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go -index 1b887e55..04d0ee52 100644 ---- a/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go -+++ b/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go -@@ -63,6 +63,8 @@ - CopyFileRequest - StartTracingRequest - StopTracingRequest -+ UpdateIPVSRequest -+ IPVSResponse - CheckRequest - HealthCheckResponse - VersionCheckResponse -@@ -1842,6 +1844,40 @@ func (m *StopTracingRequest) String() string { return proto.CompactTe - func (*StopTracingRequest) ProtoMessage() {} - func (*StopTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{52} } - -+type UpdateIPVSRequest struct { -+ // IPVS_req is the IPVS rule message needed to update -+ IPVSReq string `protobuf:"bytes,1,opt,name=IPVS_req,json=IPVSReq,proto3" json:"IPVS_req,omitempty"` -+} -+ -+func (m *UpdateIPVSRequest) Reset() { *m = UpdateIPVSRequest{} } -+func (m *UpdateIPVSRequest) String() string { return proto.CompactTextString(m) } -+func (*UpdateIPVSRequest) ProtoMessage() {} -+func (*UpdateIPVSRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{53} } -+ -+func (m *UpdateIPVSRequest) GetIPVSReq() string { -+ if m != nil { -+ return m.IPVSReq -+ } -+ return "" -+} -+ -+type IPVSResponse struct { -+ // IPVS_res is the response of IPVS updating -+ IPVSRes string `protobuf:"bytes,1,opt,name=IPVS_res,json=IPVSRes,proto3" json:"IPVS_res,omitempty"` -+} -+ -+func (m *IPVSResponse) Reset() { *m = IPVSResponse{} } -+func (m *IPVSResponse) String() string { return proto.CompactTextString(m) } -+func (*IPVSResponse) ProtoMessage() {} -+func (*IPVSResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{54} } -+ -+func (m *IPVSResponse) GetIPVSRes() string { -+ if m != nil { -+ return m.IPVSRes -+ } -+ return "" -+} -+ - func init() { - proto.RegisterType((*CreateContainerRequest)(nil), "grpc.CreateContainerRequest") - proto.RegisterType((*StartContainerRequest)(nil), "grpc.StartContainerRequest") -@@ -1896,6 +1932,8 @@ func init() { - proto.RegisterType((*CopyFileRequest)(nil), "grpc.CopyFileRequest") - proto.RegisterType((*StartTracingRequest)(nil), "grpc.StartTracingRequest") - proto.RegisterType((*StopTracingRequest)(nil), "grpc.StopTracingRequest") -+ proto.RegisterType((*UpdateIPVSRequest)(nil), "grpc.UpdateIPVSRequest") -+ proto.RegisterType((*IPVSResponse)(nil), "grpc.IPVSResponse") - } - - // Reference imports to suppress errors if they are not otherwise used. -@@ -1938,6 +1976,7 @@ type AgentServiceClient interface { - UpdateRoutes(ctx context.Context, in *UpdateRoutesRequest, opts ...grpc1.CallOption) (*Routes, error) - ListInterfaces(ctx context.Context, in *ListInterfacesRequest, opts ...grpc1.CallOption) (*Interfaces, error) - ListRoutes(ctx context.Context, in *ListRoutesRequest, opts ...grpc1.CallOption) (*Routes, error) -+ UpdateIPVSRule(ctx context.Context, in *UpdateIPVSRequest, opts ...grpc1.CallOption) (*IPVSResponse, error) - // tracing - StartTracing(ctx context.Context, in *StartTracingRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error) - StopTracing(ctx context.Context, in *StopTracingRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error) -@@ -2140,6 +2179,15 @@ func (c *agentServiceClient) ListRoutes(ctx context.Context, in *ListRoutesReque - return out, nil - } - -+func (c *agentServiceClient) UpdateIPVSRule(ctx context.Context, in *UpdateIPVSRequest, opts ...grpc1.CallOption) (*IPVSResponse, error) { -+ out := new(IPVSResponse) -+ err := grpc1.Invoke(ctx, "/grpc.AgentService/UpdateIPVSRule", in, out, c.cc, opts...) -+ if err != nil { -+ return nil, err -+ } -+ return out, nil -+} -+ - func (c *agentServiceClient) StartTracing(ctx context.Context, in *StartTracingRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error) { - out := new(google_protobuf2.Empty) - err := grpc1.Invoke(ctx, "/grpc.AgentService/StartTracing", in, out, c.cc, opts...) -@@ -2262,6 +2310,7 @@ type AgentServiceServer interface { - UpdateRoutes(context.Context, *UpdateRoutesRequest) (*Routes, error) - ListInterfaces(context.Context, *ListInterfacesRequest) (*Interfaces, error) - ListRoutes(context.Context, *ListRoutesRequest) (*Routes, error) -+ UpdateIPVSRule(context.Context, *UpdateIPVSRequest) (*IPVSResponse, error) - // tracing - StartTracing(context.Context, *StartTracingRequest) (*google_protobuf2.Empty, error) - StopTracing(context.Context, *StopTracingRequest) (*google_protobuf2.Empty, error) -@@ -2640,6 +2689,24 @@ func _AgentService_ListRoutes_Handler(srv interface{}, ctx context.Context, dec - return interceptor(ctx, in, info, handler) - } - -+func _AgentService_UpdateIPVSRule_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) { -+ in := new(UpdateIPVSRequest) -+ if err := dec(in); err != nil { -+ return nil, err -+ } -+ if interceptor == nil { -+ return srv.(AgentServiceServer).UpdateIPVSRule(ctx, in) -+ } -+ info := &grpc1.UnaryServerInfo{ -+ Server: srv, -+ FullMethod: "/grpc.AgentService/UpdateIPVSRule", -+ } -+ handler := func(ctx context.Context, req interface{}) (interface{}, error) { -+ return srv.(AgentServiceServer).UpdateIPVSRule(ctx, req.(*UpdateIPVSRequest)) -+ } -+ return interceptor(ctx, in, info, handler) -+} -+ - func _AgentService_StartTracing_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) { - in := new(StartTracingRequest) - if err := dec(in); err != nil { -@@ -2904,6 +2971,10 @@ var _AgentService_serviceDesc = grpc1.ServiceDesc{ - MethodName: "ListRoutes", - Handler: _AgentService_ListRoutes_Handler, - }, -+ { -+ MethodName: "UpdateIPVSRule", -+ Handler: _AgentService_UpdateIPVSRule_Handler, -+ }, - { - MethodName: "StartTracing", - Handler: _AgentService_StartTracing_Handler, -@@ -5088,6 +5159,54 @@ func (m *StopTracingRequest) MarshalTo(dAtA []byte) (int, error) { - return i, nil - } - -+func (m *UpdateIPVSRequest) Marshal() (dAtA []byte, err error) { -+ size := m.Size() -+ dAtA = make([]byte, size) -+ n, err := m.MarshalTo(dAtA) -+ if err != nil { -+ return nil, err -+ } -+ return dAtA[:n], nil -+} -+ -+func (m *UpdateIPVSRequest) MarshalTo(dAtA []byte) (int, error) { -+ var i int -+ _ = i -+ var l int -+ _ = l -+ if len(m.IPVSReq) > 0 { -+ dAtA[i] = 0xa -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(len(m.IPVSReq))) -+ i += copy(dAtA[i:], m.IPVSReq) -+ } -+ return i, nil -+} -+ -+func (m *IPVSResponse) Marshal() (dAtA []byte, err error) { -+ size := m.Size() -+ dAtA = make([]byte, size) -+ n, err := m.MarshalTo(dAtA) -+ if err != nil { -+ return nil, err -+ } -+ return dAtA[:n], nil -+} -+ -+func (m *IPVSResponse) MarshalTo(dAtA []byte) (int, error) { -+ var i int -+ _ = i -+ var l int -+ _ = l -+ if len(m.IPVSRes) > 0 { -+ dAtA[i] = 0xa -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(len(m.IPVSRes))) -+ i += copy(dAtA[i:], m.IPVSRes) -+ } -+ return i, nil -+} -+ - func encodeVarintAgent(dAtA []byte, offset int, v uint64) int { - for v >= 1<<7 { - dAtA[offset] = uint8(v&0x7f | 0x80) -@@ -6019,6 +6138,26 @@ func (m *StopTracingRequest) Size() (n int) { - return n - } - -+func (m *UpdateIPVSRequest) Size() (n int) { -+ var l int -+ _ = l -+ l = len(m.IPVSReq) -+ if l > 0 { -+ n += 1 + l + sovAgent(uint64(l)) -+ } -+ return n -+} -+ -+func (m *IPVSResponse) Size() (n int) { -+ var l int -+ _ = l -+ l = len(m.IPVSRes) -+ if l > 0 { -+ n += 1 + l + sovAgent(uint64(l)) -+ } -+ return n -+} -+ - func sovAgent(x uint64) (n int) { - for { - n++ -@@ -12785,6 +12924,164 @@ func (m *StopTracingRequest) Unmarshal(dAtA []byte) error { - } - return nil - } -+func (m *UpdateIPVSRequest) Unmarshal(dAtA []byte) error { -+ l := len(dAtA) -+ iNdEx := 0 -+ for iNdEx < l { -+ preIndex := iNdEx -+ var wire uint64 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ wire |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ fieldNum := int32(wire >> 3) -+ wireType := int(wire & 0x7) -+ if wireType == 4 { -+ return fmt.Errorf("proto: UpdateIPVSRequest: wiretype end group for non-group") -+ } -+ if fieldNum <= 0 { -+ return fmt.Errorf("proto: UpdateIPVSRequest: illegal tag %d (wire type %d)", fieldNum, wire) -+ } -+ switch fieldNum { -+ case 1: -+ if wireType != 2 { -+ return fmt.Errorf("proto: wrong wireType = %d for field IPVSReq", wireType) -+ } -+ var stringLen uint64 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ stringLen |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ intStringLen := int(stringLen) -+ if intStringLen < 0 { -+ return ErrInvalidLengthAgent -+ } -+ postIndex := iNdEx + intStringLen -+ if postIndex > l { -+ return io.ErrUnexpectedEOF -+ } -+ m.IPVSReq = string(dAtA[iNdEx:postIndex]) -+ iNdEx = postIndex -+ default: -+ iNdEx = preIndex -+ skippy, err := skipAgent(dAtA[iNdEx:]) -+ if err != nil { -+ return err -+ } -+ if skippy < 0 { -+ return ErrInvalidLengthAgent -+ } -+ if (iNdEx + skippy) > l { -+ return io.ErrUnexpectedEOF -+ } -+ iNdEx += skippy -+ } -+ } -+ -+ if iNdEx > l { -+ return io.ErrUnexpectedEOF -+ } -+ return nil -+} -+func (m *IPVSResponse) Unmarshal(dAtA []byte) error { -+ l := len(dAtA) -+ iNdEx := 0 -+ for iNdEx < l { -+ preIndex := iNdEx -+ var wire uint64 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ wire |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ fieldNum := int32(wire >> 3) -+ wireType := int(wire & 0x7) -+ if wireType == 4 { -+ return fmt.Errorf("proto: IPVSResponse: wiretype end group for non-group") -+ } -+ if fieldNum <= 0 { -+ return fmt.Errorf("proto: IPVSResponse: illegal tag %d (wire type %d)", fieldNum, wire) -+ } -+ switch fieldNum { -+ case 1: -+ if wireType != 2 { -+ return fmt.Errorf("proto: wrong wireType = %d for field IPVSRes", wireType) -+ } -+ var stringLen uint64 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ stringLen |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ intStringLen := int(stringLen) -+ if intStringLen < 0 { -+ return ErrInvalidLengthAgent -+ } -+ postIndex := iNdEx + intStringLen -+ if postIndex > l { -+ return io.ErrUnexpectedEOF -+ } -+ m.IPVSRes = string(dAtA[iNdEx:postIndex]) -+ iNdEx = postIndex -+ default: -+ iNdEx = preIndex -+ skippy, err := skipAgent(dAtA[iNdEx:]) -+ if err != nil { -+ return err -+ } -+ if skippy < 0 { -+ return ErrInvalidLengthAgent -+ } -+ if (iNdEx + skippy) > l { -+ return io.ErrUnexpectedEOF -+ } -+ iNdEx += skippy -+ } -+ } -+ -+ if iNdEx > l { -+ return io.ErrUnexpectedEOF -+ } -+ return nil -+} - func skipAgent(dAtA []byte) (n int, err error) { - l := len(dAtA) - iNdEx := 0 -@@ -12893,185 +13190,189 @@ var ( - func init() { proto.RegisterFile("agent.proto", fileDescriptorAgent) } - - var fileDescriptorAgent = []byte{ -- // 2876 bytes of a gzipped FileDescriptorProto -- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0x4b, 0x6f, 0x1c, 0xc7, -- 0xd1, 0xd8, 0x07, 0x97, 0xbb, 0xb5, 0x2f, 0x6e, 0x93, 0xa2, 0x56, 0x2b, 0x59, 0x9f, 0x3c, 0xb6, -- 0x65, 0xfa, 0xf3, 0xe7, 0xa5, 0x2d, 0x1b, 0x9f, 0x5f, 0x70, 0x04, 0xf1, 0x11, 0x91, 0xb1, 0x15, -- 0x31, 0x43, 0x11, 0x4e, 0x10, 0x04, 0x83, 0xe1, 0x4c, 0x73, 0xd9, 0xe6, 0xce, 0xf4, 0xb8, 0xa7, -- 0x87, 0xe2, 0x3a, 0x40, 0x8e, 0xc9, 0x2d, 0x97, 0x00, 0xb9, 0xe5, 0x0f, 0x04, 0xb9, 0xe5, 0x98, -- 0x6b, 0x0e, 0x46, 0x4e, 0xf9, 0x05, 0x41, 0xe0, 0x9f, 0x90, 0x5f, 0x10, 0xf4, 0x6b, 0x1e, 0xbb, -- 0x43, 0x1a, 0x21, 0x08, 0xe4, 0xb2, 0xe8, 0xaa, 0xae, 0xae, 0x57, 0x77, 0xd5, 0x54, 0xd5, 0x42, -- 0xdb, 0x9d, 0xe0, 0x90, 0x8f, 0x23, 0x46, 0x39, 0x45, 0xf5, 0x09, 0x8b, 0xbc, 0x51, 0x8b, 0x7a, -- 0x44, 0x21, 0x46, 0xff, 0x3f, 0x21, 0xfc, 0x34, 0x39, 0x1e, 0x7b, 0x34, 0xd8, 0x3c, 0x73, 0xb9, -- 0xfb, 0x8e, 0x47, 0x43, 0xee, 0x92, 0x10, 0xb3, 0x78, 0x53, 0x1e, 0xdc, 0x8c, 0xce, 0x26, 0x9b, -- 0x7c, 0x16, 0xe1, 0x58, 0xfd, 0xea, 0x73, 0x77, 0x27, 0x94, 0x4e, 0xa6, 0x78, 0x53, 0x42, 0xc7, -- 0xc9, 0xc9, 0x26, 0x0e, 0x22, 0x3e, 0x53, 0x9b, 0xd6, 0x1f, 0xaa, 0xb0, 0xbe, 0xcd, 0xb0, 0xcb, -- 0xf1, 0xb6, 0xe1, 0x66, 0xe3, 0xaf, 0x13, 0x1c, 0x73, 0xf4, 0x2a, 0x74, 0x52, 0x09, 0x0e, 0xf1, -- 0x87, 0x95, 0x07, 0x95, 0x8d, 0x96, 0xdd, 0x4e, 0x71, 0xfb, 0x3e, 0xba, 0x0d, 0xcb, 0xf8, 0x02, -- 0x7b, 0x62, 0xb7, 0x2a, 0x77, 0x1b, 0x02, 0xdc, 0xf7, 0xd1, 0x7b, 0xd0, 0x8e, 0x39, 0x23, 0xe1, -- 0xc4, 0x49, 0x62, 0xcc, 0x86, 0xb5, 0x07, 0x95, 0x8d, 0xf6, 0xa3, 0x95, 0xb1, 0x30, 0x69, 0x7c, -- 0x28, 0x37, 0x8e, 0x62, 0xcc, 0x6c, 0x88, 0xd3, 0x35, 0x7a, 0x08, 0xcb, 0x3e, 0x3e, 0x27, 0x1e, -- 0x8e, 0x87, 0xf5, 0x07, 0xb5, 0x8d, 0xf6, 0xa3, 0x8e, 0x22, 0xdf, 0x91, 0x48, 0xdb, 0x6c, 0xa2, -- 0xb7, 0xa0, 0x19, 0x73, 0xca, 0xdc, 0x09, 0x8e, 0x87, 0x4b, 0x92, 0xb0, 0x6b, 0xf8, 0x4a, 0xac, -- 0x9d, 0x6e, 0xa3, 0x7b, 0x50, 0x7b, 0xbe, 0xbd, 0x3f, 0x6c, 0x48, 0xe9, 0xa0, 0xa9, 0x22, 0xec, -- 0xd9, 0x02, 0x8d, 0x5e, 0x83, 0x6e, 0xec, 0x86, 0xfe, 0x31, 0xbd, 0x70, 0x22, 0xe2, 0x87, 0xf1, -- 0x70, 0xf9, 0x41, 0x65, 0xa3, 0x69, 0x77, 0x34, 0xf2, 0x40, 0xe0, 0xac, 0x4f, 0xe0, 0xd6, 0x21, -- 0x77, 0x19, 0xbf, 0x86, 0x77, 0xac, 0x23, 0x58, 0xb7, 0x71, 0x40, 0xcf, 0xaf, 0xe5, 0xda, 0x21, -- 0x2c, 0x73, 0x12, 0x60, 0x9a, 0x70, 0xe9, 0xda, 0xae, 0x6d, 0x40, 0xeb, 0x4f, 0x15, 0x40, 0xbb, -- 0x17, 0xd8, 0x3b, 0x60, 0xd4, 0xc3, 0x71, 0xfc, 0x5f, 0xba, 0xae, 0x37, 0x61, 0x39, 0x52, 0x0a, -- 0x0c, 0xeb, 0x92, 0x5c, 0xdf, 0x82, 0xd1, 0xca, 0xec, 0x5a, 0x5f, 0xc1, 0xda, 0x21, 0x99, 0x84, -- 0xee, 0xf4, 0x06, 0xf5, 0x5d, 0x87, 0x46, 0x2c, 0x79, 0x4a, 0x55, 0xbb, 0xb6, 0x86, 0xac, 0x03, -- 0x40, 0x5f, 0xba, 0x84, 0xdf, 0x9c, 0x24, 0xeb, 0x1d, 0x58, 0x2d, 0x70, 0x8c, 0x23, 0x1a, 0xc6, -- 0x58, 0x2a, 0xc0, 0x5d, 0x9e, 0xc4, 0x92, 0xd9, 0x92, 0xad, 0x21, 0x0b, 0xc3, 0xda, 0x17, 0x24, -- 0x36, 0xe4, 0xf8, 0x3f, 0x51, 0x61, 0x1d, 0x1a, 0x27, 0x94, 0x05, 0x2e, 0x37, 0x1a, 0x28, 0x08, -- 0x21, 0xa8, 0xbb, 0x6c, 0x12, 0x0f, 0x6b, 0x0f, 0x6a, 0x1b, 0x2d, 0x5b, 0xae, 0xc5, 0xab, 0x9c, -- 0x13, 0xa3, 0xf5, 0x7a, 0x15, 0x3a, 0xda, 0xef, 0xce, 0x94, 0xc4, 0x5c, 0xca, 0xe9, 0xd8, 0x6d, -- 0x8d, 0x13, 0x67, 0x2c, 0x0a, 0xeb, 0x47, 0x91, 0x7f, 0xcd, 0x80, 0x7f, 0x04, 0x2d, 0x86, 0x63, -- 0x9a, 0x30, 0x11, 0xa6, 0x55, 0x79, 0xef, 0x6b, 0xea, 0xde, 0xbf, 0x20, 0x61, 0x72, 0x61, 0x9b, -- 0x3d, 0x3b, 0x23, 0xd3, 0x21, 0xc4, 0xe3, 0xeb, 0x84, 0xd0, 0x27, 0x70, 0xeb, 0xc0, 0x4d, 0xe2, -- 0xeb, 0xe8, 0x6a, 0x7d, 0x2a, 0xc2, 0x2f, 0x4e, 0x82, 0x6b, 0x1d, 0xfe, 0x63, 0x05, 0x9a, 0xdb, -- 0x51, 0x72, 0x14, 0xbb, 0x13, 0x8c, 0xfe, 0x07, 0xda, 0x9c, 0x72, 0x77, 0xea, 0x24, 0x02, 0x94, -- 0xe4, 0x75, 0x1b, 0x24, 0x4a, 0x11, 0x08, 0xb7, 0x63, 0xe6, 0x45, 0x89, 0xa6, 0xa8, 0x3e, 0xa8, -- 0x6d, 0xd4, 0xed, 0xb6, 0xc2, 0x29, 0x92, 0x31, 0xac, 0xca, 0x3d, 0x87, 0x84, 0xce, 0x19, 0x66, -- 0x21, 0x9e, 0x06, 0xd4, 0xc7, 0xf2, 0xfd, 0xd6, 0xed, 0x81, 0xdc, 0xda, 0x0f, 0x3f, 0x4f, 0x37, -- 0xd0, 0xff, 0xc2, 0x20, 0xa5, 0x17, 0x41, 0x29, 0xa9, 0xeb, 0x92, 0xba, 0xaf, 0xa9, 0x8f, 0x34, -- 0xda, 0xfa, 0x15, 0xf4, 0x5e, 0x9c, 0x32, 0xca, 0xf9, 0x94, 0x84, 0x93, 0x1d, 0x97, 0xbb, 0x22, -- 0x7b, 0x44, 0x98, 0x11, 0xea, 0xc7, 0x5a, 0x5b, 0x03, 0xa2, 0xb7, 0x61, 0xc0, 0x15, 0x2d, 0xf6, -- 0x1d, 0x43, 0x53, 0x95, 0x34, 0x2b, 0xe9, 0xc6, 0x81, 0x26, 0x7e, 0x03, 0x7a, 0x19, 0xb1, 0xc8, -- 0x3f, 0x5a, 0xdf, 0x6e, 0x8a, 0x7d, 0x41, 0x02, 0x6c, 0x9d, 0x4b, 0x5f, 0xc9, 0x4b, 0x46, 0x6f, -- 0x43, 0x2b, 0xf3, 0x43, 0x45, 0xbe, 0x90, 0x9e, 0x7a, 0x21, 0xc6, 0x9d, 0x76, 0x33, 0x75, 0xca, -- 0x67, 0xd0, 0xe7, 0xa9, 0xe2, 0x8e, 0xef, 0x72, 0xb7, 0xf8, 0xa8, 0x8a, 0x56, 0xd9, 0x3d, 0x5e, -- 0x80, 0xad, 0x4f, 0xa1, 0x75, 0x40, 0xfc, 0x58, 0x09, 0x1e, 0xc2, 0xb2, 0x97, 0x30, 0x86, 0x43, -- 0x6e, 0x4c, 0xd6, 0x20, 0x5a, 0x83, 0xa5, 0x29, 0x09, 0x08, 0xd7, 0x66, 0x2a, 0xc0, 0xa2, 0x00, -- 0xcf, 0x70, 0x40, 0xd9, 0x4c, 0x3a, 0x6c, 0x0d, 0x96, 0xf2, 0x97, 0xab, 0x00, 0x74, 0x17, 0x5a, -- 0x81, 0x7b, 0x91, 0x5e, 0xaa, 0xd8, 0x69, 0x06, 0xee, 0x85, 0x52, 0x7e, 0x08, 0xcb, 0x27, 0x2e, -- 0x99, 0x7a, 0x21, 0xd7, 0x5e, 0x31, 0x60, 0x26, 0xb0, 0x9e, 0x17, 0xf8, 0xd7, 0x2a, 0xb4, 0x95, -- 0x44, 0xa5, 0xf0, 0x1a, 0x2c, 0x79, 0xae, 0x77, 0x9a, 0x8a, 0x94, 0x00, 0x7a, 0x68, 0x14, 0xa9, -- 0xe6, 0x93, 0x70, 0xa6, 0xa9, 0x51, 0x6d, 0x13, 0x20, 0x7e, 0xe9, 0x46, 0x5a, 0xb7, 0xda, 0x25, -- 0xc4, 0x2d, 0x41, 0xa3, 0xd4, 0x7d, 0x1f, 0x3a, 0xea, 0xdd, 0xe9, 0x23, 0xf5, 0x4b, 0x8e, 0xb4, -- 0x15, 0x95, 0x3a, 0xf4, 0x1a, 0x74, 0x93, 0x18, 0x3b, 0xa7, 0x04, 0x33, 0x97, 0x79, 0xa7, 0xb3, -- 0xe1, 0x92, 0xfa, 0x46, 0x26, 0x31, 0xde, 0x33, 0x38, 0xf4, 0x08, 0x96, 0x44, 0xfa, 0x8b, 0x87, -- 0x0d, 0xf9, 0x39, 0xbe, 0x97, 0x67, 0x29, 0x4d, 0x1d, 0xcb, 0xdf, 0xdd, 0x90, 0xb3, 0x99, 0xad, -- 0x48, 0x47, 0x1f, 0x01, 0x64, 0x48, 0xb4, 0x02, 0xb5, 0x33, 0x3c, 0xd3, 0x71, 0x28, 0x96, 0xc2, -- 0x39, 0xe7, 0xee, 0x34, 0x31, 0x5e, 0x57, 0xc0, 0x27, 0xd5, 0x8f, 0x2a, 0x96, 0x07, 0xfd, 0xad, -- 0xe9, 0x19, 0xa1, 0xb9, 0xe3, 0x6b, 0xb0, 0x14, 0xb8, 0x5f, 0x51, 0x66, 0x3c, 0x29, 0x01, 0x89, -- 0x25, 0x21, 0x65, 0x86, 0x85, 0x04, 0x50, 0x0f, 0xaa, 0x34, 0x92, 0xfe, 0x6a, 0xd9, 0x55, 0x1a, -- 0x65, 0x82, 0xea, 0x39, 0x41, 0xd6, 0x3f, 0xea, 0x00, 0x99, 0x14, 0x64, 0xc3, 0x88, 0x50, 0x27, -- 0xc6, 0x4c, 0x94, 0x20, 0xce, 0xf1, 0x8c, 0xe3, 0xd8, 0x61, 0xd8, 0x4b, 0x58, 0x4c, 0xce, 0xc5, -- 0xfd, 0x09, 0xb3, 0x6f, 0x29, 0xb3, 0xe7, 0x74, 0xb3, 0x6f, 0x13, 0x7a, 0xa8, 0xce, 0x6d, 0x89, -- 0x63, 0xb6, 0x39, 0x85, 0xf6, 0xe1, 0x56, 0xc6, 0xd3, 0xcf, 0xb1, 0xab, 0x5e, 0xc5, 0x6e, 0x35, -- 0x65, 0xe7, 0x67, 0xac, 0x76, 0x61, 0x95, 0x50, 0xe7, 0xeb, 0x04, 0x27, 0x05, 0x46, 0xb5, 0xab, -- 0x18, 0x0d, 0x08, 0xfd, 0x89, 0x3c, 0x90, 0xb1, 0x39, 0x80, 0x3b, 0x39, 0x2b, 0x45, 0xb8, 0xe7, -- 0x98, 0xd5, 0xaf, 0x62, 0xb6, 0x9e, 0x6a, 0x25, 0xf2, 0x41, 0xc6, 0xf1, 0x47, 0xb0, 0x4e, 0xa8, -- 0xf3, 0xd2, 0x25, 0x7c, 0x9e, 0xdd, 0xd2, 0xf7, 0x18, 0x29, 0x3e, 0xba, 0x45, 0x5e, 0xca, 0xc8, -- 0x00, 0xb3, 0x49, 0xc1, 0xc8, 0xc6, 0xf7, 0x18, 0xf9, 0x4c, 0x1e, 0xc8, 0xd8, 0x3c, 0x81, 0x01, -- 0xa1, 0xf3, 0xda, 0x2c, 0x5f, 0xc5, 0xa4, 0x4f, 0x68, 0x51, 0x93, 0x2d, 0x18, 0xc4, 0xd8, 0xe3, -- 0x94, 0xe5, 0x1f, 0x41, 0xf3, 0x2a, 0x16, 0x2b, 0x9a, 0x3e, 0xe5, 0x61, 0xfd, 0x1c, 0x3a, 0x7b, -- 0xc9, 0x04, 0xf3, 0xe9, 0x71, 0x9a, 0x0c, 0x6e, 0x2c, 0xff, 0x58, 0xff, 0xaa, 0x42, 0x7b, 0x7b, -- 0xc2, 0x68, 0x12, 0x15, 0x72, 0xb2, 0x0a, 0xd2, 0xf9, 0x9c, 0x2c, 0x49, 0x64, 0x4e, 0x56, 0xc4, -- 0x1f, 0x40, 0x27, 0x90, 0xa1, 0xab, 0xe9, 0x55, 0x1e, 0x1a, 0x2c, 0x04, 0xb5, 0xdd, 0x0e, 0x72, -- 0xc9, 0x6c, 0x0c, 0x10, 0x11, 0x3f, 0xd6, 0x67, 0x54, 0x3a, 0xea, 0xeb, 0x8a, 0xd0, 0xa4, 0x68, -- 0xbb, 0x15, 0xa5, 0xd9, 0xfa, 0x3d, 0x68, 0x1f, 0x0b, 0x27, 0xe9, 0x03, 0x85, 0x64, 0x94, 0x79, -- 0xcf, 0x86, 0xe3, 0x2c, 0x08, 0xf7, 0xa0, 0x7b, 0xaa, 0x5c, 0xa6, 0x0f, 0xa9, 0x37, 0xf4, 0x9a, -- 0xb6, 0x24, 0xb3, 0x77, 0x9c, 0xf7, 0xac, 0xba, 0x80, 0xce, 0x69, 0x0e, 0x35, 0x3a, 0x84, 0xc1, -- 0x02, 0x49, 0x49, 0x0e, 0xda, 0xc8, 0xe7, 0xa0, 0xf6, 0x23, 0xa4, 0x04, 0xe5, 0x4f, 0xe6, 0xf3, -- 0xd2, 0x6f, 0xab, 0xd0, 0xf9, 0x31, 0xe6, 0x2f, 0x29, 0x3b, 0x53, 0xfa, 0x22, 0xa8, 0x87, 0x6e, -- 0x80, 0x35, 0x47, 0xb9, 0x46, 0x77, 0xa0, 0xc9, 0x2e, 0x54, 0x02, 0xd1, 0xf7, 0xb9, 0xcc, 0x2e, -- 0x64, 0x62, 0x40, 0xaf, 0x00, 0xb0, 0x0b, 0x27, 0x72, 0xbd, 0x33, 0xac, 0x3d, 0x58, 0xb7, 0x5b, -- 0xec, 0xe2, 0x40, 0x21, 0xc4, 0x53, 0x60, 0x17, 0x0e, 0x66, 0x8c, 0xb2, 0x58, 0xe7, 0xaa, 0x26, -- 0xbb, 0xd8, 0x95, 0xb0, 0x3e, 0xeb, 0x33, 0x1a, 0x45, 0xd8, 0x97, 0x39, 0x5a, 0x9e, 0xdd, 0x51, -- 0x08, 0x21, 0x95, 0x1b, 0xa9, 0x0d, 0x25, 0x95, 0x67, 0x52, 0x79, 0x26, 0x75, 0x59, 0x9d, 0xe4, -- 0x79, 0xa9, 0x3c, 0x95, 0xda, 0x54, 0x52, 0x79, 0x4e, 0x2a, 0xcf, 0xa4, 0xb6, 0xcc, 0x59, 0x2d, -- 0xd5, 0xfa, 0x4d, 0x05, 0xd6, 0xe7, 0x0b, 0x3f, 0x5d, 0xa6, 0x7e, 0x00, 0x1d, 0x4f, 0xde, 0x57, -- 0xe1, 0x4d, 0x0e, 0x16, 0x6e, 0xd2, 0x6e, 0x7b, 0xb9, 0x67, 0xfc, 0x21, 0x74, 0x43, 0xe5, 0xe0, -- 0xf4, 0x69, 0xd6, 0xb2, 0x7b, 0xc9, 0xfb, 0xde, 0xee, 0x84, 0x39, 0xc8, 0xf2, 0x01, 0x7d, 0xc9, -- 0x08, 0xc7, 0x87, 0x9c, 0x61, 0x37, 0xb8, 0x89, 0x06, 0x04, 0x41, 0x5d, 0x56, 0x2b, 0x35, 0x59, -- 0x5f, 0xcb, 0xb5, 0xf5, 0x26, 0xac, 0x16, 0xa4, 0x68, 0x5b, 0x57, 0xa0, 0x36, 0xc5, 0xa1, 0xe4, -- 0xde, 0xb5, 0xc5, 0xd2, 0x72, 0x61, 0x60, 0x63, 0xd7, 0xbf, 0x39, 0x6d, 0xb4, 0x88, 0x5a, 0x26, -- 0x62, 0x03, 0x50, 0x5e, 0x84, 0x56, 0xc5, 0x68, 0x5d, 0xc9, 0x69, 0xfd, 0x1c, 0x06, 0xdb, 0x53, -- 0x1a, 0xe3, 0x43, 0xee, 0x93, 0xf0, 0x26, 0x3a, 0xa6, 0x5f, 0xc2, 0xea, 0x0b, 0x3e, 0xfb, 0x52, -- 0x30, 0x8b, 0xc9, 0x37, 0xf8, 0x86, 0xec, 0x63, 0xf4, 0xa5, 0xb1, 0x8f, 0xd1, 0x97, 0xa2, 0x59, -- 0xf2, 0xe8, 0x34, 0x09, 0x42, 0x19, 0x0a, 0x5d, 0x5b, 0x43, 0xd6, 0x16, 0x74, 0x54, 0x0d, 0xfd, -- 0x8c, 0xfa, 0xc9, 0x14, 0x97, 0xc6, 0xe0, 0x7d, 0x80, 0xc8, 0x65, 0x6e, 0x80, 0x39, 0x66, 0xea, -- 0x0d, 0xb5, 0xec, 0x1c, 0xc6, 0xfa, 0x7d, 0x15, 0xd6, 0xd4, 0x48, 0xe4, 0x50, 0x4d, 0x02, 0x8c, -- 0x09, 0x23, 0x68, 0x9e, 0xd2, 0x98, 0xe7, 0x18, 0xa6, 0xb0, 0x50, 0xd1, 0x0f, 0x0d, 0x37, 0xb1, -- 0x2c, 0xcc, 0x29, 0x6a, 0x57, 0xcf, 0x29, 0x16, 0x26, 0x11, 0xf5, 0xc5, 0x49, 0x84, 0x88, 0x36, -- 0x43, 0x44, 0x54, 0x8c, 0xb7, 0xec, 0x96, 0xc6, 0xec, 0xfb, 0xe8, 0x21, 0xf4, 0x27, 0x42, 0x4b, -- 0xe7, 0x94, 0xd2, 0x33, 0x27, 0x72, 0xf9, 0xa9, 0x0c, 0xf5, 0x96, 0xdd, 0x95, 0xe8, 0x3d, 0x4a, -- 0xcf, 0x0e, 0x5c, 0x7e, 0x8a, 0x3e, 0x86, 0x9e, 0x2e, 0x03, 0x03, 0xe9, 0xa2, 0x58, 0x7f, 0xfc, -- 0x74, 0x14, 0xe5, 0xbd, 0x67, 0x77, 0xcf, 0x72, 0x50, 0x6c, 0xdd, 0x86, 0x5b, 0x3b, 0x38, 0xe6, -- 0x8c, 0xce, 0x8a, 0x8e, 0xb1, 0x7e, 0x00, 0xb0, 0x1f, 0x72, 0xcc, 0x4e, 0x5c, 0x0f, 0xc7, 0xe8, -- 0xdd, 0x3c, 0xa4, 0x8b, 0xa3, 0x95, 0xb1, 0x9a, 0x48, 0xa5, 0x1b, 0x76, 0x8e, 0xc6, 0x1a, 0x43, -- 0xc3, 0xa6, 0x89, 0x48, 0x47, 0xaf, 0x9b, 0x95, 0x3e, 0xd7, 0xd1, 0xe7, 0x24, 0xd2, 0xd6, 0x7b, -- 0xd6, 0x9e, 0x69, 0x61, 0x33, 0x76, 0xfa, 0x8a, 0xc6, 0xd0, 0x22, 0x06, 0xa7, 0xb3, 0xca, 0xa2, -- 0xe8, 0x8c, 0xc4, 0xfa, 0x19, 0xac, 0x2a, 0x4e, 0x8a, 0xb3, 0x61, 0xf3, 0x3a, 0x34, 0x98, 0x51, -- 0xa3, 0x92, 0x8d, 0xa2, 0x34, 0x91, 0xde, 0x43, 0xf7, 0x84, 0x30, 0x8f, 0xe1, 0x40, 0xf4, 0x1c, -- 0x55, 0x79, 0x65, 0x19, 0x42, 0x78, 0x4b, 0xf4, 0xdb, 0x99, 0x99, 0xc6, 0x5b, 0xab, 0x30, 0x10, -- 0x1b, 0x05, 0x89, 0xd6, 0x2f, 0x60, 0xf5, 0x79, 0x38, 0x25, 0x21, 0xde, 0x3e, 0x38, 0x7a, 0x86, -- 0xd3, 0xac, 0x80, 0xa0, 0x2e, 0xaa, 0x27, 0xa9, 0x46, 0xd3, 0x96, 0x6b, 0x11, 0x26, 0xe1, 0xb1, -- 0xe3, 0x45, 0x49, 0xac, 0x27, 0x43, 0x8d, 0xf0, 0x78, 0x3b, 0x4a, 0x62, 0x91, 0xe6, 0xc5, 0x67, -- 0x9e, 0x86, 0xd3, 0x99, 0x8c, 0x95, 0xa6, 0xbd, 0xec, 0x45, 0xc9, 0xf3, 0x70, 0x3a, 0xb3, 0xfe, -- 0x4f, 0xf6, 0xc2, 0x18, 0xfb, 0xb6, 0x1b, 0xfa, 0x34, 0xd8, 0xc1, 0xe7, 0x39, 0x09, 0x69, 0xdf, -- 0x65, 0x72, 0xc2, 0xb7, 0x15, 0xe8, 0x3c, 0x99, 0xe0, 0x90, 0xef, 0x60, 0xee, 0x92, 0xa9, 0xec, -- 0xad, 0xce, 0x31, 0x8b, 0x09, 0x0d, 0xf5, 0xc3, 0x37, 0xa0, 0x68, 0x8d, 0x49, 0x48, 0xb8, 0xe3, -- 0xbb, 0x38, 0xa0, 0xa1, 0xf6, 0x02, 0x08, 0xd4, 0x8e, 0xc4, 0xa0, 0x37, 0xa1, 0xaf, 0x26, 0x77, -- 0xce, 0xa9, 0x1b, 0xfa, 0x53, 0x11, 0x72, 0x6a, 0x92, 0xd1, 0x53, 0xe8, 0x3d, 0x8d, 0x45, 0x6f, -- 0xc1, 0x8a, 0x0e, 0x88, 0x8c, 0xb2, 0x2e, 0x29, 0xfb, 0x1a, 0x5f, 0x20, 0x4d, 0xa2, 0x88, 0x32, -- 0x1e, 0x3b, 0x31, 0xf6, 0x3c, 0x1a, 0x44, 0xba, 0x31, 0xe9, 0x1b, 0xfc, 0xa1, 0x42, 0x5b, 0x13, -- 0x58, 0x7d, 0x2a, 0xec, 0xd4, 0x96, 0x64, 0x17, 0xdc, 0x0b, 0x70, 0xe0, 0x1c, 0x4f, 0xa9, 0x77, -- 0xe6, 0x88, 0x34, 0xa5, 0x3d, 0x2c, 0x4a, 0x9f, 0x2d, 0x81, 0x3c, 0x24, 0xdf, 0xc8, 0x1e, 0x5c, -- 0x50, 0x9d, 0x52, 0x1e, 0x4d, 0x93, 0x89, 0x13, 0x31, 0x7a, 0x8c, 0xb5, 0x89, 0xfd, 0x00, 0x07, -- 0x7b, 0x0a, 0x7f, 0x20, 0xd0, 0xd6, 0x5f, 0x2a, 0xb0, 0x56, 0x94, 0xa4, 0x93, 0xee, 0x26, 0xac, -- 0x15, 0x45, 0xe9, 0x0f, 0xb1, 0x2a, 0xf4, 0x06, 0x79, 0x81, 0xea, 0x93, 0xfc, 0x21, 0x74, 0xe5, -- 0x38, 0xd7, 0xf1, 0x15, 0xa7, 0x62, 0xf9, 0x91, 0xbf, 0x17, 0xbb, 0xe3, 0xe6, 0x6f, 0xe9, 0x63, -- 0xb8, 0xa3, 0xcd, 0x77, 0x16, 0xd5, 0x56, 0x0f, 0x62, 0x5d, 0x13, 0x3c, 0x9b, 0xd3, 0xfe, 0x0b, -- 0x18, 0x66, 0xa8, 0xad, 0x99, 0x44, 0x1a, 0x5f, 0xbd, 0x0b, 0xab, 0x73, 0xc6, 0x3e, 0xf1, 0x7d, -- 0x26, 0x03, 0xb4, 0x6e, 0x97, 0x6d, 0x59, 0x8f, 0xe1, 0xf6, 0x21, 0xe6, 0xca, 0x1b, 0x2e, 0xd7, -- 0x3d, 0x81, 0x62, 0xb6, 0x02, 0xb5, 0x43, 0xec, 0x49, 0xe3, 0x6b, 0xb6, 0x58, 0x8a, 0x07, 0x78, -- 0x14, 0x63, 0x4f, 0x5a, 0x59, 0xb3, 0xe5, 0xda, 0xfa, 0x73, 0x05, 0x96, 0x75, 0x9a, 0x14, 0xa9, -- 0xde, 0x67, 0xe4, 0x1c, 0x33, 0xfd, 0xf4, 0x34, 0x84, 0xde, 0x80, 0x9e, 0x5a, 0x39, 0x34, 0xe2, -- 0x84, 0xa6, 0xc9, 0xb7, 0xab, 0xb0, 0xcf, 0x15, 0x52, 0x4e, 0xea, 0xe4, 0x20, 0x4a, 0xf7, 0x7c, -- 0x1a, 0x92, 0xe3, 0xb6, 0x58, 0x64, 0x06, 0x99, 0x6c, 0x5b, 0xb6, 0x86, 0xc4, 0x53, 0x37, 0xfc, -- 0x96, 0x24, 0x3f, 0x03, 0x8a, 0xa7, 0x1e, 0xd0, 0x24, 0xe4, 0x4e, 0x44, 0x49, 0xc8, 0x75, 0x76, -- 0x05, 0x89, 0x3a, 0x10, 0x18, 0xeb, 0xd7, 0x15, 0x68, 0xa8, 0x69, 0xb5, 0xe8, 0x32, 0xd3, 0x6f, -- 0x5c, 0x95, 0xc8, 0x7a, 0x41, 0xca, 0x52, 0xdf, 0x35, 0xb9, 0x16, 0x71, 0x7c, 0x1e, 0xa8, 0x4c, -- 0xad, 0x55, 0x3b, 0x0f, 0x64, 0x8a, 0x7e, 0x03, 0x7a, 0xd9, 0xa7, 0x52, 0xee, 0x2b, 0x15, 0xbb, -- 0x29, 0x56, 0x92, 0x5d, 0xaa, 0xa9, 0xf5, 0x53, 0xd1, 0x5c, 0xa7, 0x93, 0xda, 0x15, 0xa8, 0x25, -- 0xa9, 0x32, 0x62, 0x29, 0x30, 0x93, 0xf4, 0x23, 0x2b, 0x96, 0xe8, 0x21, 0xf4, 0x5c, 0xdf, 0x27, -- 0xe2, 0xb8, 0x3b, 0x7d, 0x4a, 0xfc, 0x34, 0x48, 0x8b, 0x58, 0xeb, 0x6f, 0x15, 0xe8, 0x6f, 0xd3, -- 0x68, 0xf6, 0x43, 0x32, 0xc5, 0xb9, 0x0c, 0x22, 0x95, 0xd4, 0xdf, 0x58, 0xb1, 0x16, 0x75, 0xe3, -- 0x09, 0x99, 0x62, 0x15, 0x5a, 0xea, 0x66, 0x9b, 0x02, 0x21, 0xc3, 0xca, 0x6c, 0xa6, 0x03, 0xb0, -- 0xae, 0xda, 0x7c, 0x46, 0x7d, 0x59, 0x21, 0xfb, 0x84, 0x39, 0xe9, 0xb8, 0xab, 0x6b, 0x2f, 0xfb, -- 0x84, 0xc9, 0x2d, 0x6d, 0xc8, 0x92, 0x9c, 0xb8, 0xe6, 0x0d, 0x69, 0x28, 0x8c, 0x30, 0x64, 0x1d, -- 0x1a, 0xf4, 0xe4, 0x24, 0xc6, 0x5c, 0xd6, 0xb2, 0x35, 0x5b, 0x43, 0x69, 0x9a, 0x6b, 0xe6, 0xd2, -- 0xdc, 0x2d, 0x58, 0x95, 0xb3, 0xfd, 0x17, 0xcc, 0xf5, 0x48, 0x38, 0x31, 0xa9, 0x78, 0x0d, 0xd0, -- 0x21, 0xa7, 0x51, 0x11, 0xfb, 0xe8, 0x77, 0x2b, 0x3a, 0x27, 0xea, 0x46, 0x17, 0x3d, 0x85, 0xfe, -- 0xdc, 0x1f, 0x27, 0x48, 0x4f, 0x3e, 0xca, 0xff, 0x4f, 0x19, 0xad, 0x8f, 0xd5, 0x1f, 0x31, 0x63, -- 0xf3, 0x47, 0xcc, 0x78, 0x37, 0x88, 0xf8, 0x0c, 0xed, 0x42, 0xaf, 0xf8, 0x17, 0x03, 0xba, 0x6b, -- 0x0a, 0x85, 0x92, 0x3f, 0x1e, 0x2e, 0x65, 0xf3, 0x14, 0xfa, 0x73, 0xff, 0x36, 0x18, 0x7d, 0xca, -- 0xff, 0x84, 0xb8, 0x94, 0xd1, 0x63, 0x68, 0xe7, 0xfe, 0x5e, 0x40, 0x43, 0xc5, 0x64, 0xf1, 0x1f, -- 0x87, 0x4b, 0x19, 0x6c, 0x43, 0xb7, 0x30, 0xf1, 0x47, 0x23, 0x6d, 0x4f, 0xc9, 0xdf, 0x00, 0x97, -- 0x32, 0xd9, 0x82, 0x76, 0x6e, 0xf0, 0x6e, 0xb4, 0x58, 0x9c, 0xee, 0x8f, 0xee, 0x94, 0xec, 0xe8, -- 0xd4, 0xbb, 0x07, 0xdd, 0xc2, 0x98, 0xdc, 0x28, 0x52, 0x36, 0xa2, 0x1f, 0xdd, 0x2d, 0xdd, 0xd3, -- 0x9c, 0x9e, 0x42, 0x7f, 0x6e, 0x68, 0x6e, 0x9c, 0x5b, 0x3e, 0x4b, 0xbf, 0xd4, 0xac, 0xcf, 0xe5, -- 0x65, 0xe7, 0x7a, 0xa2, 0xdc, 0x65, 0x2f, 0x8e, 0xc8, 0x47, 0xf7, 0xca, 0x37, 0xb5, 0x56, 0xbb, -- 0xd0, 0x2b, 0x4e, 0xc7, 0x0d, 0xb3, 0xd2, 0x99, 0xf9, 0xd5, 0x2f, 0xa7, 0x30, 0x28, 0xcf, 0x5e, -- 0x4e, 0xd9, 0xfc, 0xfc, 0x52, 0x46, 0x4f, 0x00, 0x74, 0x07, 0xe4, 0x93, 0x30, 0xbd, 0xb2, 0x85, -- 0xce, 0x2b, 0xbd, 0xb2, 0x92, 0x6e, 0xe9, 0x31, 0x80, 0x6a, 0x5c, 0x7c, 0x9a, 0x70, 0x74, 0xdb, -- 0xa8, 0x31, 0xd7, 0x2d, 0x8d, 0x86, 0x8b, 0x1b, 0x0b, 0x0c, 0x30, 0x63, 0xd7, 0x61, 0xf0, 0x19, -- 0x40, 0xd6, 0x10, 0x19, 0x06, 0x0b, 0x2d, 0xd2, 0x15, 0x3e, 0xe8, 0xe4, 0xdb, 0x1f, 0xa4, 0x6d, -- 0x2d, 0x69, 0x89, 0xae, 0x60, 0xd1, 0x9f, 0x2b, 0x6f, 0x8b, 0x8f, 0x6d, 0xbe, 0xea, 0x1d, 0x2d, -- 0x94, 0xb8, 0xe8, 0x43, 0xe8, 0xe4, 0xeb, 0x5a, 0xa3, 0x45, 0x49, 0xad, 0x3b, 0x2a, 0xd4, 0xb6, -- 0xe8, 0x31, 0xf4, 0x8a, 0x55, 0x2b, 0xca, 0xc5, 0xc5, 0x42, 0x2d, 0x3b, 0xd2, 0x13, 0x9b, 0x1c, -- 0xf9, 0xfb, 0x00, 0x59, 0x75, 0x6b, 0xdc, 0xb7, 0x50, 0xef, 0xce, 0x49, 0x7d, 0x02, 0x9d, 0x7c, -- 0x26, 0x36, 0xea, 0x96, 0x64, 0xe7, 0xab, 0xb2, 0x56, 0x2e, 0x6b, 0x9b, 0xc7, 0xb7, 0x98, 0xc8, -- 0xaf, 0xca, 0x5a, 0x85, 0xae, 0xcf, 0x24, 0x8b, 0xb2, 0x56, 0xf0, 0xaa, 0x5c, 0x5e, 0x6c, 0x91, -- 0x8c, 0xfb, 0x4a, 0x1b, 0xa7, 0xab, 0x1e, 0x51, 0xbe, 0x1b, 0x30, 0xfe, 0x28, 0xe9, 0x10, 0xbe, -- 0x27, 0xa8, 0xf3, 0x15, 0x7f, 0x2e, 0xa8, 0x4b, 0x1a, 0x81, 0x4b, 0x19, 0xed, 0x41, 0xff, 0xa9, -- 0x29, 0xe6, 0x74, 0xa1, 0xa9, 0xd5, 0x29, 0x29, 0xac, 0x47, 0xa3, 0xb2, 0x2d, 0x1d, 0x59, 0x9f, -- 0xc3, 0x60, 0xa1, 0xc8, 0x44, 0xf7, 0xd3, 0xc1, 0x62, 0x69, 0xf5, 0x79, 0xa9, 0x5a, 0xfb, 0xb0, -- 0x32, 0x5f, 0x63, 0xa2, 0x57, 0xf4, 0xa5, 0x97, 0xd7, 0x9e, 0x97, 0xb2, 0xfa, 0x18, 0x9a, 0xa6, -- 0xa6, 0x41, 0x7a, 0x80, 0x3b, 0x57, 0xe3, 0x5c, 0x76, 0x74, 0xab, 0xf3, 0xed, 0x77, 0xf7, 0x2b, -- 0x7f, 0xff, 0xee, 0x7e, 0xe5, 0x9f, 0xdf, 0xdd, 0xaf, 0x1c, 0x37, 0xe4, 0xee, 0xfb, 0xff, 0x0e, -- 0x00, 0x00, 0xff, 0xff, 0x8d, 0x89, 0xaa, 0x73, 0xc8, 0x21, 0x00, 0x00, -+ // 2931 bytes of a gzipped FileDescriptorProto -+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0xcb, 0x6e, 0x1c, 0xc7, -+ 0xb5, 0x98, 0x07, 0x87, 0x33, 0x67, 0x5e, 0x9c, 0x22, 0x45, 0x8d, 0x46, 0xb2, 0xae, 0xdc, 0xb6, -+ 0x65, 0xfa, 0xfa, 0x7a, 0x68, 0xcb, 0xc6, 0xf5, 0x0b, 0xbe, 0x82, 0x48, 0xe9, 0x8a, 0x8c, 0xad, -+ 0x88, 0xe9, 0x91, 0xe2, 0x04, 0x41, 0xd0, 0x68, 0x76, 0x97, 0x86, 0x65, 0x4e, 0x77, 0xb5, 0xab, -+ 0xaa, 0x29, 0x8e, 0x03, 0x64, 0x99, 0xec, 0xb2, 0xcc, 0x2e, 0x3f, 0x10, 0x64, 0x97, 0x65, 0xb6, -+ 0x59, 0x18, 0x59, 0x05, 0xf9, 0x80, 0x20, 0xf0, 0x27, 0xe4, 0x0b, 0x82, 0x7a, 0xf5, 0x63, 0x66, -+ 0x48, 0x23, 0x82, 0x80, 0x6c, 0x1a, 0x75, 0x4e, 0x9d, 0x3a, 0xaf, 0xaa, 0x3a, 0x75, 0xce, 0x69, -+ 0x68, 0xfb, 0x53, 0x1c, 0x8b, 0x71, 0xc2, 0xa8, 0xa0, 0xa8, 0x3e, 0x65, 0x49, 0x30, 0x6a, 0xd1, -+ 0x80, 0x68, 0xc4, 0xe8, 0x7f, 0xa7, 0x44, 0x9c, 0xa4, 0xc7, 0xe3, 0x80, 0x46, 0xbb, 0xa7, 0xbe, -+ 0xf0, 0xdf, 0x09, 0x68, 0x2c, 0x7c, 0x12, 0x63, 0xc6, 0x77, 0xd5, 0xc2, 0xdd, 0xe4, 0x74, 0xba, -+ 0x2b, 0xe6, 0x09, 0xe6, 0xfa, 0x6b, 0xd6, 0x5d, 0x9f, 0x52, 0x3a, 0x9d, 0xe1, 0x5d, 0x05, 0x1d, -+ 0xa7, 0xcf, 0x76, 0x71, 0x94, 0x88, 0xb9, 0x9e, 0x74, 0x7e, 0x57, 0x85, 0xed, 0x7d, 0x86, 0x7d, -+ 0x81, 0xf7, 0x2d, 0x37, 0x17, 0x7f, 0x9d, 0x62, 0x2e, 0xd0, 0xab, 0xd0, 0xc9, 0x24, 0x78, 0x24, -+ 0x1c, 0x56, 0x6e, 0x55, 0x76, 0x5a, 0x6e, 0x3b, 0xc3, 0x1d, 0x86, 0xe8, 0x2a, 0xac, 0xe3, 0x73, -+ 0x1c, 0xc8, 0xd9, 0xaa, 0x9a, 0x6d, 0x48, 0xf0, 0x30, 0x44, 0xef, 0x41, 0x9b, 0x0b, 0x46, 0xe2, -+ 0xa9, 0x97, 0x72, 0xcc, 0x86, 0xb5, 0x5b, 0x95, 0x9d, 0xf6, 0x9d, 0x8d, 0xb1, 0x34, 0x69, 0x3c, -+ 0x51, 0x13, 0x4f, 0x39, 0x66, 0x2e, 0xf0, 0x6c, 0x8c, 0x6e, 0xc3, 0x7a, 0x88, 0xcf, 0x48, 0x80, -+ 0xf9, 0xb0, 0x7e, 0xab, 0xb6, 0xd3, 0xbe, 0xd3, 0xd1, 0xe4, 0xf7, 0x15, 0xd2, 0xb5, 0x93, 0xe8, -+ 0x2d, 0x68, 0x72, 0x41, 0x99, 0x3f, 0xc5, 0x7c, 0xb8, 0xa6, 0x08, 0xbb, 0x96, 0xaf, 0xc2, 0xba, -+ 0xd9, 0x34, 0xba, 0x01, 0xb5, 0xc7, 0xfb, 0x87, 0xc3, 0x86, 0x92, 0x0e, 0x86, 0x2a, 0xc1, 0x81, -+ 0x2b, 0xd1, 0xe8, 0x35, 0xe8, 0x72, 0x3f, 0x0e, 0x8f, 0xe9, 0xb9, 0x97, 0x90, 0x30, 0xe6, 0xc3, -+ 0xf5, 0x5b, 0x95, 0x9d, 0xa6, 0xdb, 0x31, 0xc8, 0x23, 0x89, 0x73, 0x3e, 0x81, 0x2b, 0x13, 0xe1, -+ 0x33, 0xf1, 0x02, 0xde, 0x71, 0x9e, 0xc2, 0xb6, 0x8b, 0x23, 0x7a, 0xf6, 0x42, 0xae, 0x1d, 0xc2, -+ 0xba, 0x20, 0x11, 0xa6, 0xa9, 0x50, 0xae, 0xed, 0xba, 0x16, 0x74, 0xfe, 0x50, 0x01, 0xf4, 0xe0, -+ 0x1c, 0x07, 0x47, 0x8c, 0x06, 0x98, 0xf3, 0xff, 0xd0, 0x76, 0xbd, 0x09, 0xeb, 0x89, 0x56, 0x60, -+ 0x58, 0x57, 0xe4, 0x66, 0x17, 0xac, 0x56, 0x76, 0xd6, 0xf9, 0x0a, 0xb6, 0x26, 0x64, 0x1a, 0xfb, -+ 0xb3, 0x97, 0xa8, 0xef, 0x36, 0x34, 0xb8, 0xe2, 0xa9, 0x54, 0xed, 0xba, 0x06, 0x72, 0x8e, 0x00, -+ 0x7d, 0xe9, 0x13, 0xf1, 0xf2, 0x24, 0x39, 0xef, 0xc0, 0x66, 0x89, 0x23, 0x4f, 0x68, 0xcc, 0xb1, -+ 0x52, 0x40, 0xf8, 0x22, 0xe5, 0x8a, 0xd9, 0x9a, 0x6b, 0x20, 0x07, 0xc3, 0xd6, 0x17, 0x84, 0x5b, -+ 0x72, 0xfc, 0xef, 0xa8, 0xb0, 0x0d, 0x8d, 0x67, 0x94, 0x45, 0xbe, 0xb0, 0x1a, 0x68, 0x08, 0x21, -+ 0xa8, 0xfb, 0x6c, 0xca, 0x87, 0xb5, 0x5b, 0xb5, 0x9d, 0x96, 0xab, 0xc6, 0xf2, 0x54, 0x2e, 0x88, -+ 0x31, 0x7a, 0xbd, 0x0a, 0x1d, 0xe3, 0x77, 0x6f, 0x46, 0xb8, 0x50, 0x72, 0x3a, 0x6e, 0xdb, 0xe0, -+ 0xe4, 0x1a, 0x87, 0xc2, 0xf6, 0xd3, 0x24, 0x7c, 0xc1, 0x0b, 0x7f, 0x07, 0x5a, 0x0c, 0x73, 0x9a, -+ 0x32, 0x79, 0x4d, 0xab, 0x6a, 0xdf, 0xb7, 0xf4, 0xbe, 0x7f, 0x41, 0xe2, 0xf4, 0xdc, 0xb5, 0x73, -+ 0x6e, 0x4e, 0x66, 0xae, 0x90, 0xe0, 0x2f, 0x72, 0x85, 0x3e, 0x81, 0x2b, 0x47, 0x7e, 0xca, 0x5f, -+ 0x44, 0x57, 0xe7, 0x53, 0x79, 0xfd, 0x78, 0x1a, 0xbd, 0xd0, 0xe2, 0xdf, 0x57, 0xa0, 0xb9, 0x9f, -+ 0xa4, 0x4f, 0xb9, 0x3f, 0xc5, 0xe8, 0xbf, 0xa0, 0x2d, 0xa8, 0xf0, 0x67, 0x5e, 0x2a, 0x41, 0x45, -+ 0x5e, 0x77, 0x41, 0xa1, 0x34, 0x81, 0x74, 0x3b, 0x66, 0x41, 0x92, 0x1a, 0x8a, 0xea, 0xad, 0xda, -+ 0x4e, 0xdd, 0x6d, 0x6b, 0x9c, 0x26, 0x19, 0xc3, 0xa6, 0x9a, 0xf3, 0x48, 0xec, 0x9d, 0x62, 0x16, -+ 0xe3, 0x59, 0x44, 0x43, 0xac, 0xce, 0x6f, 0xdd, 0x1d, 0xa8, 0xa9, 0xc3, 0xf8, 0xf3, 0x6c, 0x02, -+ 0xfd, 0x37, 0x0c, 0x32, 0x7a, 0x79, 0x29, 0x15, 0x75, 0x5d, 0x51, 0xf7, 0x0d, 0xf5, 0x53, 0x83, -+ 0x76, 0x7e, 0x09, 0xbd, 0x27, 0x27, 0x8c, 0x0a, 0x31, 0x23, 0xf1, 0xf4, 0xbe, 0x2f, 0x7c, 0x19, -+ 0x3d, 0x12, 0xcc, 0x08, 0x0d, 0xb9, 0xd1, 0xd6, 0x82, 0xe8, 0x6d, 0x18, 0x08, 0x4d, 0x8b, 0x43, -+ 0xcf, 0xd2, 0x54, 0x15, 0xcd, 0x46, 0x36, 0x71, 0x64, 0x88, 0xdf, 0x80, 0x5e, 0x4e, 0x2c, 0xe3, -+ 0x8f, 0xd1, 0xb7, 0x9b, 0x61, 0x9f, 0x90, 0x08, 0x3b, 0x67, 0xca, 0x57, 0x6a, 0x93, 0xd1, 0xdb, -+ 0xd0, 0xca, 0xfd, 0x50, 0x51, 0x27, 0xa4, 0xa7, 0x4f, 0x88, 0x75, 0xa7, 0xdb, 0xcc, 0x9c, 0xf2, -+ 0x19, 0xf4, 0x45, 0xa6, 0xb8, 0x17, 0xfa, 0xc2, 0x2f, 0x1f, 0xaa, 0xb2, 0x55, 0x6e, 0x4f, 0x94, -+ 0x60, 0xe7, 0x53, 0x68, 0x1d, 0x91, 0x90, 0x6b, 0xc1, 0x43, 0x58, 0x0f, 0x52, 0xc6, 0x70, 0x2c, -+ 0xac, 0xc9, 0x06, 0x44, 0x5b, 0xb0, 0x36, 0x23, 0x11, 0x11, 0xc6, 0x4c, 0x0d, 0x38, 0x14, 0xe0, -+ 0x11, 0x8e, 0x28, 0x9b, 0x2b, 0x87, 0x6d, 0xc1, 0x5a, 0x71, 0x73, 0x35, 0x80, 0xae, 0x43, 0x2b, -+ 0xf2, 0xcf, 0xb3, 0x4d, 0x95, 0x33, 0xcd, 0xc8, 0x3f, 0xd7, 0xca, 0x0f, 0x61, 0xfd, 0x99, 0x4f, -+ 0x66, 0x41, 0x2c, 0x8c, 0x57, 0x2c, 0x98, 0x0b, 0xac, 0x17, 0x05, 0xfe, 0xb9, 0x0a, 0x6d, 0x2d, -+ 0x51, 0x2b, 0xbc, 0x05, 0x6b, 0x81, 0x1f, 0x9c, 0x64, 0x22, 0x15, 0x80, 0x6e, 0x5b, 0x45, 0xaa, -+ 0xc5, 0x20, 0x9c, 0x6b, 0x6a, 0x55, 0xdb, 0x05, 0xe0, 0xcf, 0xfd, 0xc4, 0xe8, 0x56, 0xbb, 0x80, -+ 0xb8, 0x25, 0x69, 0xb4, 0xba, 0xef, 0x43, 0x47, 0x9f, 0x3b, 0xb3, 0xa4, 0x7e, 0xc1, 0x92, 0xb6, -+ 0xa6, 0xd2, 0x8b, 0x5e, 0x83, 0x6e, 0xca, 0xb1, 0x77, 0x42, 0x30, 0xf3, 0x59, 0x70, 0x32, 0x1f, -+ 0xae, 0xe9, 0x37, 0x32, 0xe5, 0xf8, 0xc0, 0xe2, 0xd0, 0x1d, 0x58, 0x93, 0xe1, 0x8f, 0x0f, 0x1b, -+ 0xea, 0x39, 0xbe, 0x51, 0x64, 0xa9, 0x4c, 0x1d, 0xab, 0xef, 0x83, 0x58, 0xb0, 0xb9, 0xab, 0x49, -+ 0x47, 0x1f, 0x01, 0xe4, 0x48, 0xb4, 0x01, 0xb5, 0x53, 0x3c, 0x37, 0xf7, 0x50, 0x0e, 0xa5, 0x73, -+ 0xce, 0xfc, 0x59, 0x6a, 0xbd, 0xae, 0x81, 0x4f, 0xaa, 0x1f, 0x55, 0x9c, 0x00, 0xfa, 0x7b, 0xb3, -+ 0x53, 0x42, 0x0b, 0xcb, 0xb7, 0x60, 0x2d, 0xf2, 0xbf, 0xa2, 0xcc, 0x7a, 0x52, 0x01, 0x0a, 0x4b, -+ 0x62, 0xca, 0x2c, 0x0b, 0x05, 0xa0, 0x1e, 0x54, 0x69, 0xa2, 0xfc, 0xd5, 0x72, 0xab, 0x34, 0xc9, -+ 0x05, 0xd5, 0x0b, 0x82, 0x9c, 0xbf, 0xd7, 0x01, 0x72, 0x29, 0xc8, 0x85, 0x11, 0xa1, 0x1e, 0xc7, -+ 0x4c, 0xa6, 0x20, 0xde, 0xf1, 0x5c, 0x60, 0xee, 0x31, 0x1c, 0xa4, 0x8c, 0x93, 0x33, 0xb9, 0x7f, -+ 0xd2, 0xec, 0x2b, 0xda, 0xec, 0x05, 0xdd, 0xdc, 0xab, 0x84, 0x4e, 0xf4, 0xba, 0x3d, 0xb9, 0xcc, -+ 0xb5, 0xab, 0xd0, 0x21, 0x5c, 0xc9, 0x79, 0x86, 0x05, 0x76, 0xd5, 0xcb, 0xd8, 0x6d, 0x66, 0xec, -+ 0xc2, 0x9c, 0xd5, 0x03, 0xd8, 0x24, 0xd4, 0xfb, 0x3a, 0xc5, 0x69, 0x89, 0x51, 0xed, 0x32, 0x46, -+ 0x03, 0x42, 0x7f, 0xa4, 0x16, 0xe4, 0x6c, 0x8e, 0xe0, 0x5a, 0xc1, 0x4a, 0x79, 0xdd, 0x0b, 0xcc, -+ 0xea, 0x97, 0x31, 0xdb, 0xce, 0xb4, 0x92, 0xf1, 0x20, 0xe7, 0xf8, 0x03, 0xd8, 0x26, 0xd4, 0x7b, -+ 0xee, 0x13, 0xb1, 0xc8, 0x6e, 0xed, 0x7b, 0x8c, 0x94, 0x8f, 0x6e, 0x99, 0x97, 0x36, 0x32, 0xc2, -+ 0x6c, 0x5a, 0x32, 0xb2, 0xf1, 0x3d, 0x46, 0x3e, 0x52, 0x0b, 0x72, 0x36, 0xf7, 0x60, 0x40, 0xe8, -+ 0xa2, 0x36, 0xeb, 0x97, 0x31, 0xe9, 0x13, 0x5a, 0xd6, 0x64, 0x0f, 0x06, 0x1c, 0x07, 0x82, 0xb2, -+ 0xe2, 0x21, 0x68, 0x5e, 0xc6, 0x62, 0xc3, 0xd0, 0x67, 0x3c, 0x9c, 0x9f, 0x41, 0xe7, 0x20, 0x9d, -+ 0x62, 0x31, 0x3b, 0xce, 0x82, 0xc1, 0x4b, 0x8b, 0x3f, 0xce, 0x3f, 0xab, 0xd0, 0xde, 0x9f, 0x32, -+ 0x9a, 0x26, 0xa5, 0x98, 0xac, 0x2f, 0xe9, 0x62, 0x4c, 0x56, 0x24, 0x2a, 0x26, 0x6b, 0xe2, 0x0f, -+ 0xa0, 0x13, 0xa9, 0xab, 0x6b, 0xe8, 0x75, 0x1c, 0x1a, 0x2c, 0x5d, 0x6a, 0xb7, 0x1d, 0x15, 0x82, -+ 0xd9, 0x18, 0x20, 0x21, 0x21, 0x37, 0x6b, 0x74, 0x38, 0xea, 0x9b, 0x8c, 0xd0, 0x86, 0x68, 0xb7, -+ 0x95, 0x64, 0xd1, 0xfa, 0x3d, 0x68, 0x1f, 0x4b, 0x27, 0x99, 0x05, 0xa5, 0x60, 0x94, 0x7b, 0xcf, -+ 0x85, 0xe3, 0xfc, 0x12, 0x1e, 0x40, 0xf7, 0x44, 0xbb, 0xcc, 0x2c, 0xd2, 0x67, 0xe8, 0x35, 0x63, -+ 0x49, 0x6e, 0xef, 0xb8, 0xe8, 0x59, 0xbd, 0x01, 0x9d, 0x93, 0x02, 0x6a, 0x34, 0x81, 0xc1, 0x12, -+ 0xc9, 0x8a, 0x18, 0xb4, 0x53, 0x8c, 0x41, 0xed, 0x3b, 0x48, 0x0b, 0x2a, 0xae, 0x2c, 0xc6, 0xa5, -+ 0xdf, 0x54, 0xa1, 0xf3, 0x43, 0x2c, 0x9e, 0x53, 0x76, 0xaa, 0xf5, 0x45, 0x50, 0x8f, 0xfd, 0x08, -+ 0x1b, 0x8e, 0x6a, 0x8c, 0xae, 0x41, 0x93, 0x9d, 0xeb, 0x00, 0x62, 0xf6, 0x73, 0x9d, 0x9d, 0xab, -+ 0xc0, 0x80, 0x5e, 0x01, 0x60, 0xe7, 0x5e, 0xe2, 0x07, 0xa7, 0xd8, 0x78, 0xb0, 0xee, 0xb6, 0xd8, -+ 0xf9, 0x91, 0x46, 0xc8, 0xa3, 0xc0, 0xce, 0x3d, 0xcc, 0x18, 0x65, 0xdc, 0xc4, 0xaa, 0x26, 0x3b, -+ 0x7f, 0xa0, 0x60, 0xb3, 0x36, 0x64, 0x34, 0x49, 0x70, 0xa8, 0x62, 0xb4, 0x5a, 0x7b, 0x5f, 0x23, -+ 0xa4, 0x54, 0x61, 0xa5, 0x36, 0xb4, 0x54, 0x91, 0x4b, 0x15, 0xb9, 0xd4, 0x75, 0xbd, 0x52, 0x14, -+ 0xa5, 0x8a, 0x4c, 0x6a, 0x53, 0x4b, 0x15, 0x05, 0xa9, 0x22, 0x97, 0xda, 0xb2, 0x6b, 0x8d, 0x54, -+ 0xe7, 0xd7, 0x15, 0xd8, 0x5e, 0x4c, 0xfc, 0x4c, 0x9a, 0xfa, 0x01, 0x74, 0x02, 0xb5, 0x5f, 0xa5, -+ 0x33, 0x39, 0x58, 0xda, 0x49, 0xb7, 0x1d, 0x14, 0x8e, 0xf1, 0x87, 0xd0, 0x8d, 0xb5, 0x83, 0xb3, -+ 0xa3, 0x59, 0xcb, 0xf7, 0xa5, 0xe8, 0x7b, 0xb7, 0x13, 0x17, 0x20, 0x27, 0x04, 0xf4, 0x25, 0x23, -+ 0x02, 0x4f, 0x04, 0xc3, 0x7e, 0xf4, 0x32, 0x0a, 0x10, 0x04, 0x75, 0x95, 0xad, 0xd4, 0x54, 0x7e, -+ 0xad, 0xc6, 0xce, 0x9b, 0xb0, 0x59, 0x92, 0x62, 0x6c, 0xdd, 0x80, 0xda, 0x0c, 0xc7, 0x8a, 0x7b, -+ 0xd7, 0x95, 0x43, 0xc7, 0x87, 0x81, 0x8b, 0xfd, 0xf0, 0xe5, 0x69, 0x63, 0x44, 0xd4, 0x72, 0x11, -+ 0x3b, 0x80, 0x8a, 0x22, 0x8c, 0x2a, 0x56, 0xeb, 0x4a, 0x41, 0xeb, 0xc7, 0x30, 0xd8, 0x9f, 0x51, -+ 0x8e, 0x27, 0x22, 0x24, 0xf1, 0xcb, 0xa8, 0x98, 0x7e, 0x01, 0x9b, 0x4f, 0xc4, 0xfc, 0x4b, 0xc9, -+ 0x8c, 0x93, 0x6f, 0xf0, 0x4b, 0xb2, 0x8f, 0xd1, 0xe7, 0xd6, 0x3e, 0x46, 0x9f, 0xcb, 0x62, 0x29, -+ 0xa0, 0xb3, 0x34, 0x8a, 0xd5, 0x55, 0xe8, 0xba, 0x06, 0x72, 0xf6, 0xa0, 0xa3, 0x73, 0xe8, 0x47, -+ 0x34, 0x4c, 0x67, 0x78, 0xe5, 0x1d, 0xbc, 0x09, 0x90, 0xf8, 0xcc, 0x8f, 0xb0, 0xc0, 0x4c, 0x9f, -+ 0xa1, 0x96, 0x5b, 0xc0, 0x38, 0xbf, 0xad, 0xc2, 0x96, 0x6e, 0x89, 0x4c, 0x74, 0x27, 0xc0, 0x9a, -+ 0x30, 0x82, 0xe6, 0x09, 0xe5, 0xa2, 0xc0, 0x30, 0x83, 0xa5, 0x8a, 0x61, 0x6c, 0xb9, 0xc9, 0x61, -+ 0xa9, 0x4f, 0x51, 0xbb, 0xbc, 0x4f, 0xb1, 0xd4, 0x89, 0xa8, 0x2f, 0x77, 0x22, 0xe4, 0x6d, 0xb3, -+ 0x44, 0x44, 0xdf, 0xf1, 0x96, 0xdb, 0x32, 0x98, 0xc3, 0x10, 0xdd, 0x86, 0xfe, 0x54, 0x6a, 0xe9, -+ 0x9d, 0x50, 0x7a, 0xea, 0x25, 0xbe, 0x38, 0x51, 0x57, 0xbd, 0xe5, 0x76, 0x15, 0xfa, 0x80, 0xd2, -+ 0xd3, 0x23, 0x5f, 0x9c, 0xa0, 0x8f, 0xa1, 0x67, 0xd2, 0xc0, 0x48, 0xb9, 0x88, 0x9b, 0xc7, 0xcf, -+ 0xdc, 0xa2, 0xa2, 0xf7, 0xdc, 0xee, 0x69, 0x01, 0xe2, 0xce, 0x55, 0xb8, 0x72, 0x1f, 0x73, 0xc1, -+ 0xe8, 0xbc, 0xec, 0x18, 0xe7, 0xff, 0x00, 0x0e, 0x63, 0x81, 0xd9, 0x33, 0x3f, 0xc0, 0x1c, 0xbd, -+ 0x5b, 0x84, 0x4c, 0x72, 0xb4, 0x31, 0xd6, 0x1d, 0xa9, 0x6c, 0xc2, 0x2d, 0xd0, 0x38, 0x63, 0x68, -+ 0xb8, 0x34, 0x95, 0xe1, 0xe8, 0x75, 0x3b, 0x32, 0xeb, 0x3a, 0x66, 0x9d, 0x42, 0xba, 0x66, 0xce, -+ 0x39, 0xb0, 0x25, 0x6c, 0xce, 0xce, 0x6c, 0xd1, 0x18, 0x5a, 0xc4, 0xe2, 0x4c, 0x54, 0x59, 0x16, -+ 0x9d, 0x93, 0x38, 0x3f, 0x85, 0x4d, 0xcd, 0x49, 0x73, 0xb6, 0x6c, 0x5e, 0x87, 0x06, 0xb3, 0x6a, -+ 0x54, 0xf2, 0x56, 0x94, 0x21, 0x32, 0x73, 0xe8, 0x86, 0x14, 0x16, 0x30, 0x1c, 0xc9, 0x9a, 0xa3, -+ 0xaa, 0xb6, 0x2c, 0x47, 0x48, 0x6f, 0xc9, 0x7a, 0x3b, 0x37, 0xd3, 0x7a, 0x6b, 0x13, 0x06, 0x72, -+ 0xa2, 0x24, 0xd1, 0xf9, 0x39, 0x6c, 0x3e, 0x8e, 0x67, 0x24, 0xc6, 0xfb, 0x47, 0x4f, 0x1f, 0xe1, -+ 0x2c, 0x2a, 0x20, 0xa8, 0xcb, 0xec, 0x49, 0xa9, 0xd1, 0x74, 0xd5, 0x58, 0x5e, 0x93, 0xf8, 0xd8, -+ 0x0b, 0x92, 0x94, 0x9b, 0xce, 0x50, 0x23, 0x3e, 0xde, 0x4f, 0x52, 0x2e, 0xc3, 0xbc, 0x7c, 0xe6, -+ 0x69, 0x3c, 0x9b, 0xab, 0xbb, 0xd2, 0x74, 0xd7, 0x83, 0x24, 0x7d, 0x1c, 0xcf, 0xe6, 0xce, 0xff, -+ 0xa8, 0x5a, 0x18, 0xe3, 0xd0, 0xf5, 0xe3, 0x90, 0x46, 0xf7, 0xf1, 0x59, 0x41, 0x42, 0x56, 0x77, -+ 0xd9, 0x98, 0xf0, 0x6d, 0x05, 0x3a, 0xf7, 0xa6, 0x38, 0x16, 0xf7, 0xb1, 0xf0, 0xc9, 0x4c, 0xd5, -+ 0x56, 0x67, 0x98, 0x71, 0x42, 0x63, 0x73, 0xf0, 0x2d, 0x28, 0x4b, 0x63, 0x12, 0x13, 0xe1, 0x85, -+ 0x3e, 0x8e, 0x68, 0x6c, 0xbc, 0x00, 0x12, 0x75, 0x5f, 0x61, 0xd0, 0x9b, 0xd0, 0xd7, 0x9d, 0x3b, -+ 0xef, 0xc4, 0x8f, 0xc3, 0x99, 0xbc, 0x72, 0xba, 0x93, 0xd1, 0xd3, 0xe8, 0x03, 0x83, 0x45, 0x6f, -+ 0xc1, 0x86, 0xb9, 0x10, 0x39, 0x65, 0x5d, 0x51, 0xf6, 0x0d, 0xbe, 0x44, 0x9a, 0x26, 0x09, 0x65, -+ 0x82, 0x7b, 0x1c, 0x07, 0x01, 0x8d, 0x12, 0x53, 0x98, 0xf4, 0x2d, 0x7e, 0xa2, 0xd1, 0xce, 0x14, -+ 0x36, 0x1f, 0x4a, 0x3b, 0x8d, 0x25, 0xf9, 0x06, 0xf7, 0x22, 0x1c, 0x79, 0xc7, 0x33, 0x1a, 0x9c, -+ 0x7a, 0x32, 0x4c, 0x19, 0x0f, 0xcb, 0xd4, 0x67, 0x4f, 0x22, 0x27, 0xe4, 0x1b, 0x55, 0x83, 0x4b, -+ 0xaa, 0x13, 0x2a, 0x92, 0x59, 0x3a, 0xf5, 0x12, 0x46, 0x8f, 0xb1, 0x31, 0xb1, 0x1f, 0xe1, 0xe8, -+ 0x40, 0xe3, 0x8f, 0x24, 0xda, 0xf9, 0x53, 0x05, 0xb6, 0xca, 0x92, 0x4c, 0xd0, 0xdd, 0x85, 0xad, -+ 0xb2, 0x28, 0xf3, 0x10, 0xeb, 0x44, 0x6f, 0x50, 0x14, 0xa8, 0x9f, 0xe4, 0x0f, 0xa1, 0xab, 0xda, -+ 0xb9, 0x5e, 0xa8, 0x39, 0x95, 0xd3, 0x8f, 0xe2, 0xbe, 0xb8, 0x1d, 0xbf, 0xb8, 0x4b, 0x1f, 0xc3, -+ 0x35, 0x63, 0xbe, 0xb7, 0xac, 0xb6, 0x3e, 0x10, 0xdb, 0x86, 0xe0, 0xd1, 0x82, 0xf6, 0x5f, 0xc0, -+ 0x30, 0x47, 0xed, 0xcd, 0x15, 0xd2, 0xfa, 0xea, 0x5d, 0xd8, 0x5c, 0x30, 0xf6, 0x5e, 0x18, 0x32, -+ 0x75, 0x41, 0xeb, 0xee, 0xaa, 0x29, 0xe7, 0x2e, 0x5c, 0x9d, 0x60, 0xa1, 0xbd, 0xe1, 0x0b, 0x53, -+ 0x13, 0x68, 0x66, 0x1b, 0x50, 0x9b, 0xe0, 0x40, 0x19, 0x5f, 0x73, 0xe5, 0x50, 0x1e, 0xc0, 0xa7, -+ 0x1c, 0x07, 0xca, 0xca, 0x9a, 0xab, 0xc6, 0xce, 0x1f, 0x2b, 0xb0, 0x6e, 0xc2, 0xa4, 0x0c, 0xf5, -+ 0x21, 0x23, 0x67, 0x98, 0x99, 0xa3, 0x67, 0x20, 0xf4, 0x06, 0xf4, 0xf4, 0xc8, 0xa3, 0x89, 0x20, -+ 0x34, 0x0b, 0xbe, 0x5d, 0x8d, 0x7d, 0xac, 0x91, 0xaa, 0x53, 0xa7, 0x1a, 0x51, 0xa6, 0xe6, 0x33, -+ 0x90, 0x6a, 0xb7, 0x71, 0x19, 0x19, 0x54, 0xb0, 0x6d, 0xb9, 0x06, 0x92, 0x47, 0xdd, 0xf2, 0x5b, -+ 0x53, 0xfc, 0x2c, 0x28, 0x8f, 0x7a, 0x44, 0xd3, 0x58, 0x78, 0x09, 0x25, 0xb1, 0x30, 0xd1, 0x15, -+ 0x14, 0xea, 0x48, 0x62, 0x9c, 0x5f, 0x55, 0xa0, 0xa1, 0xbb, 0xd5, 0xb2, 0xca, 0xcc, 0xde, 0xb8, -+ 0x2a, 0x51, 0xf9, 0x82, 0x92, 0xa5, 0xdf, 0x35, 0x35, 0x96, 0xf7, 0xf8, 0x2c, 0xd2, 0x91, 0xda, -+ 0xa8, 0x76, 0x16, 0xa9, 0x10, 0xfd, 0x06, 0xf4, 0xf2, 0xa7, 0x52, 0xcd, 0x6b, 0x15, 0xbb, 0x19, -+ 0x56, 0x91, 0x5d, 0xa8, 0xa9, 0xf3, 0x13, 0x59, 0x5c, 0x67, 0x9d, 0xda, 0x0d, 0xa8, 0xa5, 0x99, -+ 0x32, 0x72, 0x28, 0x31, 0xd3, 0xec, 0x91, 0x95, 0x43, 0x74, 0x1b, 0x7a, 0x7e, 0x18, 0x12, 0xb9, -+ 0xdc, 0x9f, 0x3d, 0x24, 0x61, 0x76, 0x49, 0xcb, 0x58, 0xe7, 0x2f, 0x15, 0xe8, 0xef, 0xd3, 0x64, -+ 0xfe, 0xff, 0x64, 0x86, 0x0b, 0x11, 0x44, 0x29, 0x69, 0xde, 0x58, 0x39, 0x96, 0x79, 0xe3, 0x33, -+ 0x32, 0xc3, 0xfa, 0x6a, 0xe9, 0x9d, 0x6d, 0x4a, 0x84, 0xba, 0x56, 0x76, 0x32, 0x6b, 0x80, 0x75, -+ 0xf5, 0xe4, 0x23, 0x1a, 0xaa, 0x0c, 0x39, 0x24, 0xcc, 0xcb, 0xda, 0x5d, 0x5d, 0x77, 0x3d, 0x24, -+ 0x4c, 0x4d, 0x19, 0x43, 0xd6, 0x54, 0xc7, 0xb5, 0x68, 0x48, 0x43, 0x63, 0xa4, 0x21, 0xdb, 0xd0, -+ 0xa0, 0xcf, 0x9e, 0x71, 0x2c, 0x54, 0x2e, 0x5b, 0x73, 0x0d, 0x94, 0x85, 0xb9, 0x66, 0x21, 0xcc, -+ 0x5d, 0x81, 0x4d, 0xd5, 0xdb, 0x7f, 0xc2, 0xfc, 0x80, 0xc4, 0x53, 0x1b, 0x8a, 0xb7, 0x00, 0x4d, -+ 0x04, 0x4d, 0x16, 0xb0, 0x63, 0x18, 0x98, 0x37, 0xe7, 0xe8, 0xc7, 0x13, 0x6b, 0xfa, 0x35, 0x68, -+ 0x4a, 0xd0, 0x63, 0xf8, 0x6b, 0x1b, 0x18, 0xcd, 0xb4, 0xf3, 0x16, 0x74, 0xf4, 0xd0, 0x84, 0x81, -+ 0x9c, 0x94, 0x97, 0x49, 0xf9, 0x9d, 0xbf, 0x6d, 0x98, 0x70, 0x6b, 0x6a, 0x68, 0xf4, 0x10, 0xfa, -+ 0x0b, 0xff, 0x64, 0x90, 0x69, 0xaa, 0xac, 0xfe, 0x55, 0x33, 0xda, 0x1e, 0xeb, 0x7f, 0x3c, 0x63, -+ 0xfb, 0x8f, 0x67, 0xfc, 0x20, 0x4a, 0xc4, 0x1c, 0x3d, 0x80, 0x5e, 0xf9, 0xef, 0x05, 0xba, 0x6e, -+ 0x73, 0x90, 0x15, 0xff, 0x34, 0x2e, 0x64, 0xf3, 0x10, 0xfa, 0x0b, 0x3f, 0x32, 0xac, 0x3e, 0xab, -+ 0xff, 0x6f, 0x5c, 0xc8, 0xe8, 0x2e, 0xb4, 0x0b, 0x7f, 0x2e, 0xd0, 0x50, 0x33, 0x59, 0xfe, 0x99, -+ 0x71, 0x21, 0x83, 0x7d, 0xe8, 0x96, 0x7e, 0x26, 0xa0, 0x91, 0xb1, 0x67, 0xc5, 0x1f, 0x86, 0x0b, -+ 0x99, 0xec, 0x41, 0xbb, 0xd0, 0xd3, 0xb7, 0x5a, 0x2c, 0xff, 0x38, 0x18, 0x5d, 0x5b, 0x31, 0x63, -+ 0xb6, 0xf3, 0x00, 0xba, 0xa5, 0x0e, 0xbc, 0x55, 0x64, 0x55, 0xf7, 0x7f, 0x74, 0x7d, 0xe5, 0x9c, -+ 0xe1, 0xf4, 0x10, 0xfa, 0x0b, 0xfd, 0x78, 0xeb, 0xdc, 0xd5, 0x6d, 0xfa, 0x0b, 0xcd, 0xfa, 0x5c, -+ 0x6d, 0x76, 0xa1, 0xdc, 0x2a, 0x6c, 0xf6, 0x72, 0xf7, 0x7d, 0x74, 0x63, 0xf5, 0xa4, 0xd1, 0xea, -+ 0x01, 0xf4, 0xca, 0x8d, 0x77, 0xcb, 0x6c, 0x65, 0x3b, 0xfe, 0xf2, 0x93, 0x53, 0xea, 0xc1, 0xe7, -+ 0x27, 0x67, 0x55, 0x6b, 0xfe, 0x42, 0x46, 0xf7, 0x00, 0x4c, 0x71, 0x15, 0x92, 0x38, 0xdb, 0xb2, -+ 0xa5, 0xa2, 0x2e, 0xdb, 0xb2, 0x15, 0x85, 0xd8, 0x5d, 0x00, 0x5d, 0x13, 0x85, 0x34, 0x15, 0xe8, -+ 0xaa, 0x55, 0x63, 0xa1, 0x10, 0x1b, 0x0d, 0x97, 0x27, 0x96, 0x18, 0x60, 0xc6, 0x5e, 0x84, 0xc1, -+ 0x67, 0x00, 0x79, 0xad, 0x65, 0x19, 0x2c, 0x55, 0x5f, 0x97, 0xf8, 0xa0, 0x53, 0xac, 0xac, 0x90, -+ 0xb1, 0x75, 0x45, 0xb5, 0x75, 0x09, 0x8b, 0xfe, 0x42, 0xe6, 0x5c, 0x3e, 0x6c, 0x8b, 0x09, 0xf5, -+ 0x68, 0x29, 0x7b, 0x46, 0x1f, 0x42, 0xa7, 0x98, 0x32, 0x5b, 0x2d, 0x56, 0xa4, 0xd1, 0xa3, 0x52, -+ 0xda, 0x8c, 0xee, 0x42, 0xaf, 0x9c, 0x10, 0xa3, 0xc2, 0xbd, 0x58, 0x4a, 0x93, 0x47, 0xa6, 0x19, -+ 0x54, 0x20, 0x7f, 0x1f, 0x20, 0x4f, 0x9c, 0xad, 0xfb, 0x96, 0x52, 0xe9, 0x05, 0xa9, 0x9f, 0x41, -+ 0xaf, 0x10, 0xb7, 0x65, 0x4d, 0x78, 0xb5, 0x64, 0x70, 0x1e, 0xcd, 0x47, 0x26, 0xc3, 0x2a, 0x85, -+ 0xed, 0x7b, 0xd0, 0x29, 0xbe, 0x11, 0xd6, 0xda, 0x15, 0xef, 0xc6, 0x65, 0x41, 0xaf, 0xf0, 0x9e, -+ 0xd8, 0xb3, 0xbb, 0xfc, 0xc4, 0x5c, 0x16, 0xf4, 0x4a, 0xf5, 0xa8, 0x8d, 0x35, 0xab, 0x8a, 0xd4, -+ 0xcb, 0x9e, 0x82, 0x72, 0xf1, 0x66, 0xbd, 0xbf, 0xb2, 0xa4, 0xbb, 0xec, 0x0c, 0x16, 0xeb, 0x14, -+ 0xeb, 0x8f, 0x15, 0xb5, 0xcb, 0xf7, 0xc4, 0x84, 0x62, 0x2d, 0x52, 0x88, 0x09, 0x2b, 0x4a, 0x94, -+ 0x0b, 0x19, 0x1d, 0x40, 0xff, 0xa1, 0x4d, 0x33, 0x4d, 0x0a, 0x6c, 0xd4, 0x59, 0x91, 0xf2, 0x8f, -+ 0x46, 0xab, 0xa6, 0xcc, 0x2e, 0x7f, 0x0e, 0x83, 0xa5, 0xf4, 0x17, 0xdd, 0xcc, 0x5a, 0x9e, 0x2b, -+ 0xf3, 0xe2, 0x0b, 0xd5, 0x3a, 0x84, 0x8d, 0xc5, 0xec, 0x17, 0xbd, 0x62, 0x36, 0x7d, 0x75, 0x56, -+ 0x7c, 0x21, 0xab, 0x8f, 0xa1, 0x69, 0xb3, 0x2d, 0x64, 0x5a, 0xcb, 0x0b, 0xd9, 0xd7, 0x45, 0x4b, -+ 0xf7, 0x3a, 0xdf, 0x7e, 0x77, 0xb3, 0xf2, 0xd7, 0xef, 0x6e, 0x56, 0xfe, 0xf1, 0xdd, 0xcd, 0xca, -+ 0x71, 0x43, 0xcd, 0xbe, 0xff, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x45, 0xa8, 0x91, 0xab, 0x62, -+ 0x22, 0x00, 0x00, - } -diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go -index c22ed39c..3f22d9f0 100644 ---- a/virtcontainers/agent.go -+++ b/virtcontainers/agent.go -@@ -264,4 +264,7 @@ type agent interface { - - // get proxy process pid - getProxyPid() int -+ -+ // updateIPVSRule updates IPVS rules in VM -+ updateIPVSRule(IPVSRule *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) - } -diff --git a/virtcontainers/api.go b/virtcontainers/api.go -index a4bf41bb..7036df8c 100644 ---- a/virtcontainers/api.go -+++ b/virtcontainers/api.go -@@ -13,6 +13,7 @@ import ( - "strings" - "syscall" - -+ "github.com/kata-containers/agent/protocols/grpc" - deviceApi "github.com/kata-containers/runtime/virtcontainers/device/api" - deviceConfig "github.com/kata-containers/runtime/virtcontainers/device/config" - "github.com/kata-containers/runtime/virtcontainers/persist" -@@ -1017,6 +1018,30 @@ func ListRoutes(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error) - return routes, nil - } - -+// UpdateIPVSRule adds IPVS rule -+func UpdateIPVSRule(ctx context.Context, sandboxID string, IPVSRule *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) { -+ span, ctx := trace(ctx, "UpdateIPVSRule") -+ defer span.Finish() -+ -+ if sandboxID == "" { -+ return nil, vcTypes.ErrNeedSandboxID -+ } -+ -+ unlock, err := rwLockSandbox(sandboxID) -+ if err != nil { -+ return nil, err -+ } -+ defer unlock() -+ -+ s, err := fetchSandbox(ctx, sandboxID) -+ if err != nil { -+ return nil, err -+ } -+ defer s.releaseStatelessSandbox() -+ -+ return s.UpdateIPVSRule(IPVSRule) -+} -+ - // CleanupContaienr is used by shimv2 to stop and delete a container exclusively, once there is no container - // in the sandbox left, do stop the sandbox and delete it. Those serial operations will be done exclusively by - // locking the sandbox. -diff --git a/virtcontainers/implementation.go b/virtcontainers/implementation.go -index 92cc4c12..0265d1ed 100644 ---- a/virtcontainers/implementation.go -+++ b/virtcontainers/implementation.go -@@ -13,6 +13,7 @@ import ( - "context" - "syscall" - -+ "github.com/kata-containers/agent/protocols/grpc" - "github.com/kata-containers/runtime/virtcontainers/device/api" - "github.com/kata-containers/runtime/virtcontainers/device/config" - vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" -@@ -178,3 +179,7 @@ func (impl *VCImpl) ListRoutes(ctx context.Context, sandboxID string) ([]*vcType - func (impl *VCImpl) CleanupContainer(ctx context.Context, sandboxID, containerID string, force bool) error { - return CleanupContainer(ctx, sandboxID, containerID, force) - } -+ -+func (impl *VCImpl) UpdateIPVSRule(ctx context.Context, sandboxID string, IPVSRule *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) { -+ return UpdateIPVSRule(ctx, sandboxID, IPVSRule) -+} -diff --git a/virtcontainers/interfaces.go b/virtcontainers/interfaces.go -index 6bb9ea22..d8f0be69 100644 ---- a/virtcontainers/interfaces.go -+++ b/virtcontainers/interfaces.go -@@ -10,6 +10,7 @@ import ( - "io" - "syscall" - -+ "github.com/kata-containers/agent/protocols/grpc" - "github.com/kata-containers/runtime/virtcontainers/device/api" - "github.com/kata-containers/runtime/virtcontainers/device/config" - vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" -@@ -53,6 +54,7 @@ type VC interface { - ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTypes.Interface, error) - UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) - ListRoutes(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error) -+ UpdateIPVSRule(ctx context.Context, sandboxID string, IPVSRule *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) - - CleanupContainer(ctx context.Context, sandboxID, containerID string, force bool) error - } -@@ -99,6 +101,7 @@ type VCSandbox interface { - UpdateRoutes(routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) - ListRoutes() ([]*vcTypes.Route, error) - IsCompatOldCNI() bool -+ UpdateIPVSRule(IPVSRule *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) - } - - // VCContainer is the Container interface -diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go -index 16662949..29287685 100644 ---- a/virtcontainers/kata_agent.go -+++ b/virtcontainers/kata_agent.go -@@ -128,6 +128,7 @@ const ( - grpcSetGuestDateTimeRequest = "grpc.SetGuestDateTimeRequest" - grpcStartTracingRequest = "grpc.StartTracingRequest" - grpcStopTracingRequest = "grpc.StopTracingRequest" -+ grpcUpdateIPVSRequest = "grpc.UpdateIPVSRequest" - ) - - // The function is declared this way for mocking in unit tests -@@ -2067,6 +2068,10 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) { - k.reqHandlers[grpcStopTracingRequest] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) { - return k.client.StopTracing(ctx, req.(*grpc.StopTracingRequest), opts...) - } -+ k.reqHandlers[grpcUpdateIPVSRequest] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) { -+ return k.client.UpdateIPVSRule(ctx, req.(*grpc.UpdateIPVSRequest), opts...) -+ } -+ - } - - func (k *kataAgent) getReqContext(reqName string) (ctx context.Context, cancel context.CancelFunc) { -@@ -2402,3 +2407,22 @@ func (k *kataAgent) load(s persistapi.AgentState) { - func (k *kataAgent) getProxyPid() int { - return k.state.ProxyPid - } -+ -+func (k *kataAgent) updateIPVSRule(IPVSRule *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) { -+ if IPVSRule == nil { -+ return nil, nil -+ } -+ resultingIPVS, err := k.sendReq(IPVSRule) -+ if err != nil { -+ k.Logger().WithFields(logrus.Fields{ -+ "IPVS-requested": fmt.Sprintf("%+v", IPVSRule), -+ "resulting-IPVS": fmt.Sprintf("%+v", resultingIPVS), -+ }).WithError(err).Error("update IPVS rule request failed") -+ } -+ resultIPVS, ok := resultingIPVS.(*grpc.IPVSResponse) -+ if ok && resultIPVS != nil { -+ return resultIPVS, err -+ } -+ -+ return nil, err -+} -diff --git a/virtcontainers/noop_agent.go b/virtcontainers/noop_agent.go -index b19705c3..d174623b 100644 ---- a/virtcontainers/noop_agent.go -+++ b/virtcontainers/noop_agent.go -@@ -240,3 +240,7 @@ func (n *noopAgent) load(s persistapi.AgentState) {} - func (n *noopAgent) getProxyPid() int { - return -1 - } -+ -+func (n *noopAgent) updateIPVSRule(IPVSRule *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) { -+ return nil, nil -+} -diff --git a/virtcontainers/pkg/vcmock/mock.go b/virtcontainers/pkg/vcmock/mock.go -index 9c50e0e4..aa9bae9a 100644 ---- a/virtcontainers/pkg/vcmock/mock.go -+++ b/virtcontainers/pkg/vcmock/mock.go -@@ -20,6 +20,7 @@ import ( - "fmt" - "syscall" - -+ "github.com/kata-containers/agent/protocols/grpc" - vc "github.com/kata-containers/runtime/virtcontainers" - "github.com/kata-containers/runtime/virtcontainers/device/api" - "github.com/kata-containers/runtime/virtcontainers/device/config" -@@ -296,3 +297,10 @@ func (m *VCMock) CleanupContainer(ctx context.Context, sandboxID, containerID st - } - return fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID) - } -+ -+func (m *VCMock) UpdateIPVSRule(ctx context.Context, sandboxID string, ipvs *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) { -+ if m.UpdateIPVSRuleFunc != nil { -+ return m.UpdateIPVSRuleFunc(ctx, sandboxID, ipvs) -+ } -+ return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID) -+} -diff --git a/virtcontainers/pkg/vcmock/sandbox.go b/virtcontainers/pkg/vcmock/sandbox.go -index 7329e1a7..063c9eca 100644 ---- a/virtcontainers/pkg/vcmock/sandbox.go -+++ b/virtcontainers/pkg/vcmock/sandbox.go -@@ -9,6 +9,7 @@ import ( - "io" - "syscall" - -+ "github.com/kata-containers/agent/protocols/grpc" - vc "github.com/kata-containers/runtime/virtcontainers" - "github.com/kata-containers/runtime/virtcontainers/device/api" - "github.com/kata-containers/runtime/virtcontainers/device/config" -@@ -217,3 +218,7 @@ func (s *Sandbox) ListRoutes() ([]*vcTypes.Route, error) { - func (s *Sandbox) IsCompatOldCNI() bool { - return false - } -+ -+func (s *Sandbox) UpdateIPVSRule(IPVSRule *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) { -+ return nil, nil -+} -diff --git a/virtcontainers/pkg/vcmock/types.go b/virtcontainers/pkg/vcmock/types.go -index 8cf0e0b7..43247ef9 100644 ---- a/virtcontainers/pkg/vcmock/types.go -+++ b/virtcontainers/pkg/vcmock/types.go -@@ -9,6 +9,7 @@ import ( - "context" - "syscall" - -+ "github.com/kata-containers/agent/protocols/grpc" - vc "github.com/kata-containers/runtime/virtcontainers" - "github.com/kata-containers/runtime/virtcontainers/device/api" - "github.com/kata-containers/runtime/virtcontainers/device/config" -@@ -75,4 +76,5 @@ type VCMock struct { - UpdateRoutesFunc func(ctx context.Context, sandboxID string, routes []*vcTypes.Route) ([]*vcTypes.Route, error) - ListRoutesFunc func(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error) - CleanupContainerFunc func(ctx context.Context, sandboxID, containerID string, force bool) error -+ UpdateIPVSRuleFunc func(ctx context.Context, sandboxID string, ipvs *grpc.UpdateIpvsRequest) (*grpc.IpvsResponse, error) - } -diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go -index f226af4e..0d599267 100644 ---- a/virtcontainers/sandbox.go -+++ b/virtcontainers/sandbox.go -@@ -1065,6 +1065,11 @@ func (s *Sandbox) ListRoutes() ([]*vcTypes.Route, error) { - return s.agent.listRoutes() - } - -+// UpdateIPVSRule manages IPVS rule in the sandbox -+func (s *Sandbox) UpdateIPVSRule(IPVSRule *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) { -+ return s.agent.updateIPVSRule(IPVSRule) -+} -+ - // startVM starts the VM. - func (s *Sandbox) startVM() (err error) { - span, ctx := s.trace("startVM") --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0035-device-mount-blockdevices-in-the-guest-VM.patch b/runtime/patches/0035-device-mount-blockdevices-in-the-guest-VM.patch deleted file mode 100644 index 143d6b5..0000000 --- a/runtime/patches/0035-device-mount-blockdevices-in-the-guest-VM.patch +++ /dev/null @@ -1,234 +0,0 @@ -From cf2b34f477cba88641de3719bf5c8f933b919bcc Mon Sep 17 00:00:00 2001 -From: holyfei -Date: Mon, 17 Aug 2020 21:44:57 +0800 -Subject: [PATCH 35/50] device: mount blockdevices in the guest VM - -reason: support mount blockdevices in the guest VM like -"-v /dev/blockdevice:/home/test" - -Signed-off-by: yangfeiyu ---- - cli/config/configuration-qemu.toml.in | 5 ++++ - pkg/katautils/config.go | 6 +++++ - virtcontainers/kata_agent.go | 27 ++++++++++++++++--- - virtcontainers/kata_agent_test.go | 2 +- - virtcontainers/utils/utils.go | 50 +++++++++++++++++++++++++++++++++++ - virtcontainers/vm_test.go | 2 +- - 6 files changed, 86 insertions(+), 6 deletions(-) - -diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in -index 46f8b632..aa11b38f 100644 ---- a/cli/config/configuration-qemu.toml.in -+++ b/cli/config/configuration-qemu.toml.in -@@ -391,6 +391,11 @@ path = "@SHIMPATH@" - # - kernel_modules=[] - -+# If enabled, when we pass a block device to the guest VM -+# through -v, such as `docker run -v /dev/loop100:/foo/bar`, -+# agent will create a directory in guest VM and mount the -+# file system on `/dev/loop100` inside the container -+enable_blk_mount = true - - [netmon] - # If enabled, the network monitoring process gets started when the -diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go -index 94c916a0..51120311 100644 ---- a/pkg/katautils/config.go -+++ b/pkg/katautils/config.go -@@ -160,6 +160,7 @@ type agent struct { - TraceMode string `toml:"trace_mode"` - TraceType string `toml:"trace_type"` - KernelModules []string `toml:"kernel_modules"` -+ MountBlkInVM bool `toml:"enable_blk_mount"` - } - - type netmon struct { -@@ -463,6 +464,10 @@ func (h hypervisor) getInitrdAndImage() (initrd string, image string, err error) - return - } - -+func (a agent) mountBlkDevInVM() bool { -+ return a.MountBlkInVM -+} -+ - func (p proxy) path() (string, error) { - path := p.Path - if path == "" { -@@ -978,6 +983,7 @@ func updateRuntimeConfigAgent(configPath string, tomlConf tomlConfig, config *oc - TraceMode: agent.traceMode(), - TraceType: agent.traceType(), - KernelModules: agent.kernelModules(), -+ MountBlkInVM: agent.mountBlkDevInVM(), - } - default: - return fmt.Errorf("%s agent type is not supported", k) -diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go -index 16662949..b0f88c15 100644 ---- a/virtcontainers/kata_agent.go -+++ b/virtcontainers/kata_agent.go -@@ -30,6 +30,7 @@ import ( - ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter" - "github.com/kata-containers/runtime/virtcontainers/pkg/rootless" - vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" -+ "github.com/kata-containers/runtime/virtcontainers/utils" - "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" - "github.com/kata-containers/runtime/virtcontainers/store" - "github.com/kata-containers/runtime/virtcontainers/types" -@@ -64,6 +65,7 @@ var ( - errorMissingOCISpec = errors.New("Missing OCI specification") - defaultKataHostSharedDir = "/run/kata-containers/shared/sandboxes/" - defaultKataGuestSharedDir = "/run/kata-containers/shared/containers/" -+ kataGuestStorageDir = "/run/kata-containers/storage/containers/" - mountGuestTag = "kataShared" - defaultKataGuestSandboxDir = "/run/kata-containers/sandbox/" - type9pFs = "9p" -@@ -199,6 +201,7 @@ type KataAgentConfig struct { - TraceMode string - TraceType string - KernelModules []string -+ MountBlkInVM bool - } - - // KataAgentState is the structure describing the data stored from this -@@ -1061,7 +1064,12 @@ func (k *kataAgent) replaceOCIMountsForStorages(spec *specs.Spec, volumeStorages - // Create a temporary location to mount the Storage. Mounting to the correct location - // will be handled by the OCI mount structure. - filename := fmt.Sprintf("%s-%s", uuid.Generate().String(), filepath.Base(m.Destination)) -- path := filepath.Join(kataGuestSandboxStorageDir(), filename) -+ var path string -+ if v.Driver == kataBlkDevType || v.Driver == kataSCSIDevType { -+ path = filepath.Join(kataGuestStorageDir, filename) -+ } else { -+ path = filepath.Join(kataGuestSandboxStorageDir(), filename) -+ } - - k.Logger().Debugf("Replacing OCI mount source (%s) with %s", m.Source, path) - ociMounts[index].Source = path -@@ -1436,7 +1444,7 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process, - // Note this call modifies the list of container devices to make sure - // all hotplugged devices are unplugged, so this needs be done - // after devices passed with --device are handled. -- volumeStorages, err := k.handleBlockVolumes(c) -+ volumeStorages, err := k.handleBlockVolumes(sandbox, c) - if err != nil { - return nil, err - } -@@ -1609,7 +1617,7 @@ func (k *kataAgent) handleVhostUserBlkVolume(c *Container, device api.Device) (* - - // handleBlockVolumes handles volumes that are block devices files - // by passing the block devices as Storage to the agent. --func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) { -+func (k *kataAgent) handleBlockVolumes(sandbox *Sandbox, c *Container) ([]*grpc.Storage, error) { - - var volumeStorages []*grpc.Storage - -@@ -1648,9 +1656,20 @@ func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) { - } - - vol.MountPoint = m.Destination -- if vol.Fstype == "" { -+ -+ ac, _ := sandbox.config.AgentConfig.(KataAgentConfig) -+ if ac.MountBlkInVM { -+ // Ensure the block device is formatted, for the devices here are specified as volumes -+ fsType, gerr := utils.GetDevFormat(m.Source) -+ if gerr != nil || fsType == "" { -+ k.Logger().WithField("device", id).WithError(gerr).Error("get device format failed") -+ return nil, gerr -+ } -+ vol.Fstype = fmt.Sprintf("bind-%s", fsType) -+ } else { - vol.Fstype = "bind" - } -+ - if len(vol.Options) == 0 { - vol.Options = []string{"bind"} - } -diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go -index 18a5a0a6..68caaab2 100644 ---- a/virtcontainers/kata_agent_test.go -+++ b/virtcontainers/kata_agent_test.go -@@ -446,7 +446,7 @@ func TestHandleBlockVolume(t *testing.T) { - containers[c.id].sandbox = &sandbox - containers[c.id].mounts = mounts - -- volumeStorages, err := k.handleBlockVolumes(c) -+ volumeStorages, err := k.handleBlockVolumes(&sandbox, c) - assert.Nil(t, err, "Error while handling block volumes") - - vStorage := &pb.Storage{ -diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go -index 9490faa1..36ac67a7 100644 ---- a/virtcontainers/utils/utils.go -+++ b/virtcontainers/utils/utils.go -@@ -379,3 +379,53 @@ func RoundVCPUNumber(value string) (int, error) { - cpus := int(math.Ceil(cpuNum)) - return cpus, nil - } -+ -+// GetDevFormat get the formated filesystem of input disk use `blkid`, Such as `ext4`,`xfs`,etc. -+func GetDevFormat(disk string) (string, error) { -+ // refer to https://github.com/kubernetes/kubernetes/blob/v1.12.2/pkg/util/mount/mount_linux.go#L512 -+ args := []string{"-p", "-s", "TYPE", "-s", "PTTYPE", "-o", "export", disk} -+ dataOut, err := exec.Command("blkid", args...).Output() -+ output := string(dataOut) -+ -+ if err != nil { -+ if strings.Contains(err.Error(), "exit status 2") { -+ // Disk device is unformatted. -+ // For `blkid`, if the specified token (TYPE/PTTYPE, etc) was -+ // not found, or no (specified) devices could be identified, an -+ // exit code of 2 is returned. -+ return "", nil -+ } -+ return "", err -+ } -+ -+ var fstype string -+ -+ lines := strings.Split(output, "\n") -+ for _, l := range lines { -+ if len(l) <= 0 { -+ // Ignore empty line. -+ continue -+ } -+ // if we use busybox as rootfs,the output of command -+ // `busybox blkid` is different with original`blkid`, -+ // so we should make a compatible parse -+ subLine := strings.Split(l, " ") -+ for _, sl := range subLine { -+ if len(subLine) <= 0 { -+ continue -+ } -+ cs := strings.Split(sl, "=") -+ if len(cs) != 2 { -+ continue -+ } -+ if cs[0] == "TYPE" { -+ fstype = cs[1] -+ if strings.Contains(fstype, `"`) { -+ fstype = strings.Replace(fstype, `"`, "", -1) -+ } -+ } -+ } -+ } -+ -+ return fstype, nil -+} -diff --git a/virtcontainers/vm_test.go b/virtcontainers/vm_test.go -index 36fd5c2f..d2414232 100644 ---- a/virtcontainers/vm_test.go -+++ b/virtcontainers/vm_test.go -@@ -125,7 +125,7 @@ func TestVMConfigGrpc(t *testing.T) { - HypervisorType: QemuHypervisor, - HypervisorConfig: newQemuConfig(), - AgentType: KataContainersAgent, -- AgentConfig: KataAgentConfig{false, true, false, false, 0, "", "", []string{}}, -+ AgentConfig: KataAgentConfig{false, true, false, false, 0, "", "", []string{}, false }, - ProxyType: NoopProxyType, - } - --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0036-mount-limit-the-maximum-number-of-virtio-scsi-bus-sl.patch b/runtime/patches/0036-mount-limit-the-maximum-number-of-virtio-scsi-bus-sl.patch deleted file mode 100644 index 51431e8..0000000 --- a/runtime/patches/0036-mount-limit-the-maximum-number-of-virtio-scsi-bus-sl.patch +++ /dev/null @@ -1,131 +0,0 @@ -From 448bb661d6759c1e20c18084604b150bff08ada4 Mon Sep 17 00:00:00 2001 -From: holyfei -Date: Tue, 18 Aug 2020 11:49:26 +0800 -Subject: [PATCH 36/50] mount: limit the maximum number of virtio-scsi bus - slots - -reason: -1. add SCSIBus functions for add or remove device about ScsiBus -2. limit the maximum number of virtio-scsi bus slots to 25 - -Signed-off-by: yangfeiyu ---- - virtcontainers/qemu.go | 20 +++++++++++++++++ - virtcontainers/types/scsi.go | 53 ++++++++++++++++++++++++++++++++++++++++++++ - 2 files changed, 73 insertions(+) - create mode 100644 virtcontainers/types/scsi.go - -diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go -index bb83b1bb..c2b65376 100644 ---- a/virtcontainers/qemu.go -+++ b/virtcontainers/qemu.go -@@ -65,6 +65,7 @@ type CPUDevice struct { - // QemuState keeps Qemu's state - type QemuState struct { - Bridges []types.Bridge -+ ScsiBus *types.SCSIBus - // HotpluggedCPUs is the list of CPUs that were hot-added - HotpluggedVCPUs []CPUDevice - HotpluggedMemory int -@@ -411,6 +412,13 @@ func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.I - - var ioThread *govmmQemu.IOThread - if q.config.BlockDeviceDriver == config.VirtioSCSI { -+ if q.state.ScsiBus == nil { -+ // only use `scsi0.0` -+ q.state.ScsiBus = &types.SCSIBus{ -+ Address: make(map[uint32]int), -+ ID: fmt.Sprintf("%s.0", scsiControllerID), -+ } -+ } - return q.arch.appendSCSIController(devices, q.config.EnableIOThreads) - } - -@@ -1168,6 +1176,16 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev - - // Bus exposed by the SCSI Controller - bus := scsiControllerID + ".0" -+ var err error -+ if err = q.state.ScsiBus.AddDevToScsiBus(drive.Index); err != nil { -+ return err -+ } -+ -+ defer func() { -+ if err != nil { -+ q.state.ScsiBus.RemoveDevFromScsiBus(drive.Index) -+ } -+ }() - - // Get SCSI-id and LUN based on the order of attaching drives. - scsiID, lun, err := utils.GetSCSIIdLun(drive.Index) -@@ -1234,6 +1252,8 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error - if err := q.arch.removeDeviceFromBridge(drive.ID); err != nil { - return err - } -+ } else if q.config.BlockDeviceDriver == config.VirtioSCSI { -+ q.state.ScsiBus.RemoveDevFromScsiBus(drive.Index) - } - - if err := q.qmpMonitorCh.qmp.ExecuteDeviceDel(q.qmpMonitorCh.ctx, devID); err != nil { -diff --git a/virtcontainers/types/scsi.go b/virtcontainers/types/scsi.go -new file mode 100644 -index 00000000..f4d489f0 ---- /dev/null -+++ b/virtcontainers/types/scsi.go -@@ -0,0 +1,53 @@ -+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. -+// SPDX-License-Identifier: Apache-2.0 -+// Description: common functions -+// Author: leizhongkai -+// Create: 2019-06-30 -+ -+package types -+ -+import ( -+ "fmt" -+) -+ -+const ( -+ scsiMaxCapacity = 25 -+) -+ -+type SCSIBus struct { -+ // Address contains information about devices plugged and number limit -+ Address map[uint32]int -+ -+ // ID is used to identify the bus exposed by the SCSI Controller -+ ID string -+} -+ -+func (sb *SCSIBus) AddDevToScsiBus(index int) error { -+ var addr uint32 -+ -+ // looking for the first available address -+ for i := uint32(1); i <= scsiMaxCapacity; i++ { -+ if _, ok := sb.Address[i]; !ok { -+ addr = i -+ break -+ } -+ } -+ -+ if addr == 0 { -+ return fmt.Errorf("Scsi bus capacity limited.") -+ } -+ -+ // save address and device -+ sb.Address[addr] = index -+ -+ return nil -+} -+ -+func (sb *SCSIBus) RemoveDevFromScsiBus(index int) { -+ for addr, i := range sb.Address { -+ if i == index { -+ delete(sb.Address, addr) -+ return -+ } -+ } -+} --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0037-runtime-add-IPVS-test.patch b/runtime/patches/0037-runtime-add-IPVS-test.patch deleted file mode 100644 index 9c43970..0000000 --- a/runtime/patches/0037-runtime-add-IPVS-test.patch +++ /dev/null @@ -1,875 +0,0 @@ -From d083f0e0247fbded92a0ae2a0e71da4176baed95 Mon Sep 17 00:00:00 2001 -From: xiadanni -Date: Tue, 18 Aug 2020 17:08:23 +0800 -Subject: [PATCH 37/50] runtime: add IPVS test - -Signed-off-by: xiadanni ---- - cli/ipvsadm_test.go | 775 +++++++++++++++++++++++++++++++++++++ - virtcontainers/api_test.go | 36 ++ - virtcontainers/kata_agent_test.go | 4 + - virtcontainers/pkg/vcmock/types.go | 2 +- - 4 files changed, 816 insertions(+), 1 deletion(-) - create mode 100644 cli/ipvsadm_test.go - -diff --git a/cli/ipvsadm_test.go b/cli/ipvsadm_test.go -new file mode 100644 -index 00000000..92958aee ---- /dev/null -+++ b/cli/ipvsadm_test.go -@@ -0,0 +1,775 @@ -+// Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. -+// SPDX-License-Identifier: Apache-2.0 -+// Description: IPVS common functions test -+// Author: xiadanni -+// Create: 2020-08-01 -+ -+package main -+ -+import ( -+ "context" -+ "flag" -+ "fmt" -+ "os" -+ "strings" -+ "testing" -+ -+ "github.com/kata-containers/agent/protocols/grpc" -+ vc "github.com/kata-containers/runtime/virtcontainers" -+ vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" -+ "github.com/kata-containers/runtime/virtcontainers/pkg/vcmock" -+ "github.com/kata-containers/runtime/virtcontainers/types" -+ "github.com/stretchr/testify/assert" -+ "github.com/urfave/cli" -+) -+ -+func TestKataIPVSCliAction(t *testing.T) { -+ assert := assert.New(t) -+ -+ actionFunc, ok := kataIPVSCLICommand.Action.(func(ctx *cli.Context) error) -+ assert.True(ok) -+ -+ flagSet := flag.NewFlagSet("kata-IPVS", flag.ContinueOnError) -+ ctx := createCLIContext(flagSet) -+ -+ err := actionFunc(ctx) -+ assert.NoError(err) -+} -+ -+func TestIPVSadmCliAction(t *testing.T) { -+ assert := assert.New(t) -+ -+ actionFunc, ok := IPVSadmCommand.Action.(func(ctx *cli.Context) error) -+ assert.True(ok) -+ -+ flagSet := flag.NewFlagSet("IPVSadm", flag.ContinueOnError) -+ ctx := createCLIContext(flagSet) -+ -+ err := actionFunc(ctx) -+ assert.Error(err, "Missing container ID") -+} -+ -+func TestIPVSadmCLISuccessful(t *testing.T) { -+ assert := assert.New(t) -+ -+ sandbox := &vcmock.Sandbox{ -+ MockID: testContainerID, -+ } -+ -+ sandbox.MockContainers = []*vcmock.Container{ -+ { -+ MockID: sandbox.ID(), -+ MockSandbox: sandbox, -+ }, -+ } -+ -+ testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) { -+ return vc.ContainerStatus{ -+ ID: sandbox.ID(), -+ Annotations: map[string]string{ -+ vcAnnotations.ContainerTypeKey: string(vc.PodContainer), -+ }, -+ State: types.ContainerState{ -+ State: types.StateRunning, -+ }, -+ }, nil -+ } -+ -+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) { -+ return &grpc.IPVSResponse{}, nil -+ } -+ -+ defer func() { -+ testingImpl.StatusContainerFunc = nil -+ testingImpl.UpdateIPVSRuleFunc = nil -+ }() -+ -+ path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID()) -+ assert.NoError(err) -+ defer os.RemoveAll(path) -+ -+ actionFunc, ok := IPVSadmCommand.Action.(func(ctx *cli.Context) error) -+ assert.True(ok) -+ -+ flagSet := flag.NewFlagSet("IPVSadm", flag.ContinueOnError) -+ flagSet.Parse([]string{testContainerID}) -+ flagSet.String("parameters", "-a -t 192.168.0.7:80 -r 192.168.0.4:80", "") -+ ctx := createCLIContext(flagSet) -+ err = actionFunc(ctx) -+ assert.NoError(err) -+ -+ // result not nil -+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) { -+ return &grpc.IPVSResponse{IPVSRes: "IPVS rule updating success"}, nil -+ } -+ -+ err = actionFunc(ctx) -+ assert.NoError(err) -+} -+ -+func TestIPVSadmCLIError(t *testing.T) { -+ assert := assert.New(t) -+ -+ sandbox := &vcmock.Sandbox{ -+ MockID: testContainerID, -+ } -+ -+ sandbox.MockContainers = []*vcmock.Container{ -+ { -+ MockID: sandbox.ID(), -+ MockSandbox: sandbox, -+ }, -+ } -+ -+ testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) { -+ return vc.ContainerStatus{ -+ ID: sandbox.ID(), -+ Annotations: map[string]string{ -+ vcAnnotations.ContainerTypeKey: string(vc.PodContainer), -+ }, -+ State: types.ContainerState{ -+ State: types.StateRunning, -+ }, -+ }, nil -+ } -+ -+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) { -+ return &grpc.IPVSResponse{}, nil -+ } -+ -+ defer func() { -+ testingImpl.StatusContainerFunc = nil -+ testingImpl.UpdateIPVSRuleFunc = nil -+ }() -+ -+ path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID()) -+ assert.NoError(err) -+ defer os.RemoveAll(path) -+ -+ actionFunc, ok := IPVSadmCommand.Action.(func(ctx *cli.Context) error) -+ assert.True(ok) -+ -+ // no stdin rule file -+ flagSet := flag.NewFlagSet("IPVSadm", flag.ContinueOnError) -+ flagSet.Parse([]string{testContainerID}) -+ flagSet.String("restore", "-", "") -+ ctx := createCLIContext(flagSet) -+ err = actionFunc(ctx) -+ assert.Error(err) -+ -+ // restore parameter error -+ flagSet = flag.NewFlagSet("IPVSadm", flag.ContinueOnError) -+ flagSet.Parse([]string{testContainerID}) -+ flagSet.String("restore", "abc", "") -+ ctx = createCLIContext(flagSet) -+ err = actionFunc(ctx) -+ assert.Error(err) -+ -+ // checkrule returns error -+ flagSet = flag.NewFlagSet("IPVSadm", flag.ContinueOnError) -+ flagSet.Parse([]string{testContainerID}) -+ flagSet.String("parameters", "-A -t 192.168.0.7:80 -s rr -p -3000", "") -+ ctx = createCLIContext(flagSet) -+ err = actionFunc(ctx) -+ assert.Error(err) -+ -+ // updatefunction returns error -+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) { -+ return &grpc.IPVSResponse{}, fmt.Errorf("IPVSadm test error") -+ } -+ -+ flagSet = flag.NewFlagSet("IPVSadm", flag.ContinueOnError) -+ flagSet.Parse([]string{testContainerID}) -+ flagSet.String("parameters", "-A -t 192.168.0.7:80 -s rr -p 3000", "") -+ ctx = createCLIContext(flagSet) -+ err = actionFunc(ctx) -+ assert.Error(err) -+} -+ -+func TestCleanupCliAction(t *testing.T) { -+ assert := assert.New(t) -+ -+ actionFunc, ok := CleanupCommand.Action.(func(ctx *cli.Context) error) -+ assert.True(ok) -+ -+ flagSet := flag.NewFlagSet("cleanup", flag.ContinueOnError) -+ ctx := createCLIContext(flagSet) -+ -+ err := actionFunc(ctx) -+ assert.Error(err, "Missing container ID") -+} -+ -+func TestCleanupCLISuccessful(t *testing.T) { -+ assert := assert.New(t) -+ -+ sandbox := &vcmock.Sandbox{ -+ MockID: testContainerID, -+ } -+ -+ sandbox.MockContainers = []*vcmock.Container{ -+ { -+ MockID: sandbox.ID(), -+ MockSandbox: sandbox, -+ }, -+ } -+ -+ testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) { -+ return vc.ContainerStatus{ -+ ID: sandbox.ID(), -+ Annotations: map[string]string{ -+ vcAnnotations.ContainerTypeKey: string(vc.PodContainer), -+ }, -+ State: types.ContainerState{ -+ State: types.StateRunning, -+ }, -+ }, nil -+ } -+ -+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) { -+ return &grpc.IPVSResponse{}, nil -+ } -+ -+ defer func() { -+ testingImpl.StatusContainerFunc = nil -+ testingImpl.UpdateIPVSRuleFunc = nil -+ }() -+ -+ path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID()) -+ assert.NoError(err) -+ defer os.RemoveAll(path) -+ -+ actionFunc, ok := CleanupCommand.Action.(func(ctx *cli.Context) error) -+ assert.True(ok) -+ -+ flagSet := flag.NewFlagSet("cleanup", flag.ContinueOnError) -+ flagSet.Parse([]string{testContainerID}) -+ flagSet.String("parameters", "-d 192.168.0.4 -p tcp", "") -+ ctx := createCLIContext(flagSet) -+ err = actionFunc(ctx) -+ assert.NoError(err) -+} -+ -+func TestCleanupCLIError(t *testing.T) { -+ assert := assert.New(t) -+ -+ sandbox := &vcmock.Sandbox{ -+ MockID: testContainerID, -+ } -+ -+ sandbox.MockContainers = []*vcmock.Container{ -+ { -+ MockID: sandbox.ID(), -+ MockSandbox: sandbox, -+ }, -+ } -+ -+ testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) { -+ return vc.ContainerStatus{ -+ ID: sandbox.ID(), -+ Annotations: map[string]string{ -+ vcAnnotations.ContainerTypeKey: string(vc.PodContainer), -+ }, -+ State: types.ContainerState{ -+ State: types.StateRunning, -+ }, -+ }, nil -+ } -+ -+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) { -+ return &grpc.IPVSResponse{}, nil -+ } -+ -+ defer func() { -+ testingImpl.StatusContainerFunc = nil -+ testingImpl.UpdateIPVSRuleFunc = nil -+ }() -+ -+ path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID()) -+ assert.NoError(err) -+ defer os.RemoveAll(path) -+ -+ actionFunc, ok := CleanupCommand.Action.(func(ctx *cli.Context) error) -+ assert.True(ok) -+ -+ // checkrule returns error -+ flagSet := flag.NewFlagSet("cleanup", flag.ContinueOnError) -+ flagSet.Parse([]string{testContainerID}) -+ flagSet.String("parameters", "-d 192.168.0.4", "") -+ ctx := createCLIContext(flagSet) -+ err = actionFunc(ctx) -+ assert.Error(err) -+ -+ // updatefunction returns error -+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) { -+ return &grpc.IPVSResponse{}, fmt.Errorf("IPVSadm cleanup test error") -+ } -+ -+ flagSet = flag.NewFlagSet("cleanup", flag.ContinueOnError) -+ flagSet.Parse([]string{testContainerID}) -+ flagSet.String("parameters", "-d 192.168.0.4 -p tcp", "") -+ ctx = createCLIContext(flagSet) -+ err = actionFunc(ctx) -+ assert.Error(err) -+} -+ -+func TestAddServiceSuccessfully(t *testing.T) { -+ assert := assert.New(t) -+ addServiceCommandTCP := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --scheduler rr --persistent 3000" -+ _, err := checkIPVSRule(addServiceCommandTCP) -+ assert.NoError(err) -+ addServiceCommandUDP := "IPVSadm --add-service --udp-service 192.168.0.7:80 --scheduler rr --persistent 3000" -+ _, err = checkIPVSRule(addServiceCommandUDP) -+ assert.NoError(err) -+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rr -p 3000" -+ _, err = checkIPVSRule(addServiceCmd) -+ assert.NoError(err) -+} -+ -+func TestAddServiceNoService(t *testing.T) { -+ assert := assert.New(t) -+ addServiceCommand := "IPVSadm --add-service --scheduler rr --persistent 3000" -+ _, err := checkIPVSRule(addServiceCommand) -+ assert.Error(err) -+ addServiceCmd := "IPVSadm -A -s rr -p 3000" -+ _, err = checkIPVSRule(addServiceCmd) -+ assert.Error(err) -+} -+ -+func TestAddServiceNoScheduler(t *testing.T) { -+ assert := assert.New(t) -+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --persistent 3000" -+ _, err := checkIPVSRule(addServiceCommand) -+ assert.Error(err) -+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -p 3000" -+ _, err = checkIPVSRule(addServiceCmd) -+ assert.Error(err) -+} -+ -+func TestAddServiceNoPersistent(t *testing.T) { -+ assert := assert.New(t) -+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --scheduler rr" -+ _, err := checkIPVSRule(addServiceCommand) -+ assert.Error(err) -+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rr" -+ _, err = checkIPVSRule(addServiceCmd) -+ assert.Error(err) -+} -+ -+func TestAddServiceIpError(t *testing.T) { -+ assert := assert.New(t) -+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.2.2.7:80 --scheduler rr --persistent 3000" -+ _, err := checkIPVSRule(addServiceCommand) -+ assert.Error(err) -+ addServiceCmd := "IPVSadm -A -t 192.168.2.2.7:80 -s rr -p 3000" -+ _, err = checkIPVSRule(addServiceCmd) -+ assert.Error(err) -+} -+ -+func TestAddServicePortError(t *testing.T) { -+ assert := assert.New(t) -+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:9999999 --scheduler rr --persistent 3000" -+ _, err := checkIPVSRule(addServiceCommand) -+ assert.Error(err) -+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:9999999 -s rr -p 3000" -+ _, err = checkIPVSRule(addServiceCmd) -+ assert.Error(err) -+} -+ -+func TestAddServiceSchedulerError(t *testing.T) { -+ assert := assert.New(t) -+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --scheduler rrr --persistent 3000" -+ _, err := checkIPVSRule(addServiceCommand) -+ assert.Error(err) -+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rrr -p 3000" -+ _, err = checkIPVSRule(addServiceCmd) -+ assert.Error(err) -+} -+ -+func TestAddServicePersistentError(t *testing.T) { -+ assert := assert.New(t) -+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --scheduler rr --persistent 99999999999" -+ _, err := checkIPVSRule(addServiceCommand) -+ assert.Error(err) -+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rr -p 99999999999" -+ _, err = checkIPVSRule(addServiceCmd) -+ assert.Error(err) -+} -+ -+func TestEditServiceSuccessfully(t *testing.T) { -+ assert := assert.New(t) -+ editServiceCommandTCP := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --scheduler rr --persistent 3000" -+ _, err := checkIPVSRule(editServiceCommandTCP) -+ assert.NoError(err) -+ editServiceCommandUcp := "IPVSadm --edit-service --udp-service 192.168.0.7:80 --scheduler rr --persistent 3000" -+ _, err = checkIPVSRule(editServiceCommandUcp) -+ assert.NoError(err) -+ editServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rr -p 3000" -+ _, err = checkIPVSRule(editServiceCmd) -+ assert.NoError(err) -+} -+ -+func TestEditServiceNoService(t *testing.T) { -+ assert := assert.New(t) -+ editServiceCommand := "IPVSadm --edit-service --scheduler rr --persistent 3000" -+ _, err := checkIPVSRule(editServiceCommand) -+ assert.Error(err) -+ editServiceCmd := "IPVSadm -E -s rr -p 3000" -+ _, err = checkIPVSRule(editServiceCmd) -+ assert.Error(err) -+} -+ -+func TestEditServiceNoScheduler(t *testing.T) { -+ assert := assert.New(t) -+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --persistent 3000" -+ _, err := checkIPVSRule(editServiceCommand) -+ assert.Error(err) -+ editServiceCmd := "IPVSadm -E -t 192.168.0.7:80 -p 3000" -+ _, err = checkIPVSRule(editServiceCmd) -+ assert.Error(err) -+} -+ -+func TestEditServiceNoPersistent(t *testing.T) { -+ assert := assert.New(t) -+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --scheduler rr" -+ _, err := checkIPVSRule(editServiceCommand) -+ assert.Error(err) -+ editServiceCmd := "IPVSadm -E -t 192.168.0.7:80 -s rr" -+ _, err = checkIPVSRule(editServiceCmd) -+ assert.Error(err) -+} -+ -+func TestEditServiceIpError(t *testing.T) { -+ assert := assert.New(t) -+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.2.2.7:80 --scheduler rr --persistent 3000" -+ _, err := checkIPVSRule(editServiceCommand) -+ assert.Error(err) -+ editServiceCmd := "IPVSadm -E -t 192.168.2.2.7:80 -s rr -p 3000" -+ _, err = checkIPVSRule(editServiceCmd) -+ assert.Error(err) -+} -+ -+func TestEditServicePortError(t *testing.T) { -+ assert := assert.New(t) -+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:9999999 --scheduler rr --persistent 3000" -+ _, err := checkIPVSRule(editServiceCommand) -+ assert.Error(err) -+ editServiceCmd := "IPVSadm -E -t 192.168.0.7:9999999 -s rr -p 3000" -+ _, err = checkIPVSRule(editServiceCmd) -+ assert.Error(err) -+} -+ -+func TestEditServiceSchedulerError(t *testing.T) { -+ assert := assert.New(t) -+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --scheduler rrr --persistent 3000" -+ _, err := checkIPVSRule(editServiceCommand) -+ assert.Error(err) -+ editServiceCmd := "IPVSadm -E -t 192.168.0.7:80 -s rrr -p 3000" -+ _, err = checkIPVSRule(editServiceCmd) -+ assert.Error(err) -+} -+ -+func TestEditServicePersistentError(t *testing.T) { -+ assert := assert.New(t) -+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --scheduler rr --persistent 99999999999" -+ _, err := checkIPVSRule(editServiceCommand) -+ assert.Error(err) -+ editServiceCmd := "IPVSadm -E -t 192.168.0.7:80 -s rr -p 99999999999" -+ _, err = checkIPVSRule(editServiceCmd) -+ assert.Error(err) -+} -+ -+func TestDeleteServiceSuccessfully(t *testing.T) { -+ assert := assert.New(t) -+ deleteServiceCommandTCP := "IPVSadm --delete-service --tcp-service 192.168.0.7:80" -+ _, err := checkIPVSRule(deleteServiceCommandTCP) -+ assert.NoError(err) -+ deleteServiceCommandUDP := "IPVSadm --delete-service --udp-service 192.168.0.7:80" -+ _, err = checkIPVSRule(deleteServiceCommandUDP) -+ assert.NoError(err) -+ deleteServiceCmd := "IPVSadm -D -t 192.168.0.7:80" -+ _, err = checkIPVSRule(deleteServiceCmd) -+ assert.NoError(err) -+} -+ -+func TestDeleteServiceNoService(t *testing.T) { -+ assert := assert.New(t) -+ deleteServiceCommand := "IPVSadm --delete-service" -+ _, err := checkIPVSRule(deleteServiceCommand) -+ assert.Error(err) -+ deleteServiceCmd := "IPVSadm -D" -+ _, err = checkIPVSRule(deleteServiceCmd) -+ assert.Error(err) -+} -+ -+func TestDeleteServiceIpError(t *testing.T) { -+ assert := assert.New(t) -+ deleteServiceCommand := "IPVSadm --delete-service --tcp-service 192.168.2.2.7:80" -+ _, err := checkIPVSRule(deleteServiceCommand) -+ assert.Error(err) -+ deleteServiceCmd := "IPVSadm -D -t 192.168.2.2.7:80" -+ _, err = checkIPVSRule(deleteServiceCmd) -+ assert.Error(err) -+} -+ -+func TestDeleteServicePortError(t *testing.T) { -+ assert := assert.New(t) -+ deleteServiceCommand := "IPVSadm --delete-service --tcp-service 192.168.0.7:9999999" -+ _, err := checkIPVSRule(deleteServiceCommand) -+ assert.Error(err) -+ deleteServiceCmd := "IPVSadm -D -t 192.168.0.7:9999999" -+ _, err = checkIPVSRule(deleteServiceCmd) -+ assert.Error(err) -+} -+ -+func TestAddServerSuccessfully(t *testing.T) { -+ assert := assert.New(t) -+ addServerCommandTCP := "IPVSadm --add-server --tcp-service 192.168.0.7:80 --real-server 192.168.0.4:80 --weight 100" -+ _, err := checkIPVSRule(addServerCommandTCP) -+ assert.NoError(err) -+ addServerCommandUDP := "IPVSadm --add-server --udp-service 192.168.0.7:80 --real-server 192.168.0.4:80 --weight 100" -+ _, err = checkIPVSRule(addServerCommandUDP) -+ assert.NoError(err) -+ addServerCmd := "IPVSadm -a -t 192.168.0.7:80 -r 192.168.0.4:80 -w 100" -+ _, err = checkIPVSRule(addServerCmd) -+ assert.NoError(err) -+} -+ -+func TestAddServerNoService(t *testing.T) { -+ assert := assert.New(t) -+ addServerCommand := "IPVSadm --add-server --real-server 192.168.0.4:80" -+ _, err := checkIPVSRule(addServerCommand) -+ assert.Error(err) -+ addServerCmd := "IPVSadm -a -r 192.168.0.4:80" -+ _, err = checkIPVSRule(addServerCmd) -+ assert.Error(err) -+} -+ -+func TestAddServerNoServer(t *testing.T) { -+ assert := assert.New(t) -+ addServerCommand := "IPVSadm --add-server --tcp-service 192.168.0.7:80" -+ _, err := checkIPVSRule(addServerCommand) -+ assert.Error(err) -+ addServerCmd := "IPVSadm -a -t 192.168.0.7:80" -+ _, err = checkIPVSRule(addServerCmd) -+ assert.Error(err) -+} -+ -+func TestAddServerIpError(t *testing.T) { -+ assert := assert.New(t) -+ addServerCommand := "IPVSadm --add-server --tcp-service 192.168.2.0.7:80 --real-server 192.168.0.4:80" -+ _, err := checkIPVSRule(addServerCommand) -+ assert.Error(err) -+ addServerCmd := "IPVSadm -a -t 192.168.2.0.7:80 -r 192.168.0.4:80" -+ _, err = checkIPVSRule(addServerCmd) -+ assert.Error(err) -+} -+ -+func TestAddServerPortError(t *testing.T) { -+ assert := assert.New(t) -+ addServerCommand := "IPVSadm --add-server --tcp-service 192.168.0.7:99999 --real-server 192.168.0.4:80" -+ _, err := checkIPVSRule(addServerCommand) -+ assert.Error(err) -+ addServerCmd := "IPVSadm -a -t 192.168.0.7:99999 -r 192.168.0.4:80" -+ _, err = checkIPVSRule(addServerCmd) -+ assert.Error(err) -+} -+ -+func TestEditServerSuccessfully(t *testing.T) { -+ assert := assert.New(t) -+ editServerCommandTCP := "IPVSadm --edit-server --tcp-service 192.168.0.7:80 --real-server 192.168.0.4:80 --weight 100" -+ _, err := checkIPVSRule(editServerCommandTCP) -+ assert.NoError(err) -+ editServerCommandUDP := "IPVSadm --edit-server --udp-service 192.168.0.7:80 --real-server 192.168.0.4:80 --weight 100" -+ _, err = checkIPVSRule(editServerCommandUDP) -+ assert.NoError(err) -+ editServerCmd := "IPVSadm -e -t 192.168.0.7:80 -r 192.168.0.4:80 -w 100" -+ _, err = checkIPVSRule(editServerCmd) -+ assert.NoError(err) -+} -+ -+func TestEditServerNoService(t *testing.T) { -+ assert := assert.New(t) -+ editServerCommand := "IPVSadm --edit-server --real-server 192.168.0.4:80" -+ _, err := checkIPVSRule(editServerCommand) -+ assert.Error(err) -+ editServerCmd := "IPVSadm -e -r 192.168.0.4:80" -+ _, err = checkIPVSRule(editServerCmd) -+ assert.Error(err) -+} -+ -+func TestEditServerNoServer(t *testing.T) { -+ assert := assert.New(t) -+ editServerCommand := "IPVSadm --edit-server --tcp-service 192.168.0.7:80" -+ _, err := checkIPVSRule(editServerCommand) -+ assert.Error(err) -+ editServerCmd := "IPVSadm -e -t 192.168.0.7:80" -+ _, err = checkIPVSRule(editServerCmd) -+ assert.Error(err) -+} -+ -+func TestEditServerIpError(t *testing.T) { -+ assert := assert.New(t) -+ editServerCommand := "IPVSadm --edit-server --tcp-service 192.168.2.0.7:80 --real-server 192.168.0.4:80" -+ _, err := checkIPVSRule(editServerCommand) -+ assert.Error(err) -+ editServerCmd := "IPVSadm -e -t 192.168.2.0.7:80 -r 192.168.0.4:80" -+ _, err = checkIPVSRule(editServerCmd) -+ assert.Error(err) -+} -+ -+func TestEditServerPortError(t *testing.T) { -+ assert := assert.New(t) -+ editServerCommand := "IPVSadm --edit-server --tcp-service 192.168.0.7:99999 --real-server 192.168.0.4:80" -+ _, err := checkIPVSRule(editServerCommand) -+ assert.Error(err) -+ editServerCmd := "IPVSadm -e -t 192.168.0.7:99999 -r 192.168.0.4:80" -+ _, err = checkIPVSRule(editServerCmd) -+ assert.Error(err) -+} -+ -+func TestDeleteServerSuccessfully(t *testing.T) { -+ assert := assert.New(t) -+ deleteServerCommandTCP := "IPVSadm --delete-server --tcp-service 192.168.0.7:80 --real-server 192.168.0.4:80" -+ _, err := checkIPVSRule(deleteServerCommandTCP) -+ assert.NoError(err) -+ deleteServerCommandUDP := "IPVSadm --delete-server --udp-service 192.168.0.7:80 --real-server 192.168.0.4:80" -+ _, err = checkIPVSRule(deleteServerCommandUDP) -+ assert.NoError(err) -+ deleteServerCmd := "IPVSadm -d -t 192.168.0.7:80 -r 192.168.0.4:80" -+ _, err = checkIPVSRule(deleteServerCmd) -+ assert.NoError(err) -+} -+ -+func TestDeleteServerNoService(t *testing.T) { -+ assert := assert.New(t) -+ deleteServerCommand := "IPVSadm --delete-server --real-server 192.168.0.4:80" -+ _, err := checkIPVSRule(deleteServerCommand) -+ assert.Error(err) -+ deleteServerCmd := "IPVSadm -d -r 192.168.0.4:80" -+ _, err = checkIPVSRule(deleteServerCmd) -+ assert.Error(err) -+} -+ -+func TestDeleteServerNoServer(t *testing.T) { -+ assert := assert.New(t) -+ deleteServerCommand := "IPVSadm --delete-server --tcp-service 192.168.0.7:80" -+ _, err := checkIPVSRule(deleteServerCommand) -+ assert.Error(err) -+ deleteServerCmd := "IPVSadm -d -t 192.168.0.7:80" -+ _, err = checkIPVSRule(deleteServerCmd) -+ assert.Error(err) -+} -+ -+func TestDeleteServerIpError(t *testing.T) { -+ assert := assert.New(t) -+ deleteServerCommand := "IPVSadm --delete-server --tcp-service 192.168.2.0.7:80 --real-server 192.168.0.4:80" -+ _, err := checkIPVSRule(deleteServerCommand) -+ assert.Error(err) -+ deleteServerCmd := "IPVSadm -d -t 192.168.2.0.7:80 -r 192.168.0.4:80" -+ _, err = checkIPVSRule(deleteServerCmd) -+ assert.Error(err) -+} -+ -+func TestDeleteServerPortError(t *testing.T) { -+ assert := assert.New(t) -+ deleteServerCommand := "IPVSadm --delete-server --tcp-service 192.168.0.7:99999 --real-server 192.168.0.4:80" -+ _, err := checkIPVSRule(deleteServerCommand) -+ assert.Error(err) -+ deleteServerCmd := "IPVSadm -d -t 192.168.0.7:99999 -r 192.168.0.4:80" -+ _, err = checkIPVSRule(deleteServerCmd) -+ assert.Error(err) -+} -+ -+func TestListSuccessfully(t *testing.T) { -+ assert := assert.New(t) -+ listCommand := "IPVSadm --list" -+ _, err := checkIPVSRule(listCommand) -+ assert.NoError(err) -+ listCmd := "IPVSadm -L" -+ _, err = checkIPVSRule(listCmd) -+ assert.NoError(err) -+} -+ -+func TestSetSuccessfully(t *testing.T) { -+ assert := assert.New(t) -+ setCommand := "IPVSadm --set" -+ _, err := checkIPVSRule(setCommand) -+ assert.NoError(err) -+} -+ -+func TestCleanupSuccessfully(t *testing.T) { -+ assert := assert.New(t) -+ cleanupCommand := "conntrack -D --orig-dst 192.168.0.4 --protonum tcp" -+ _, err := checkIPVSRule(cleanupCommand) -+ assert.NoError(err) -+ cleanupCmd := "conntrack -D -d 192.168.0.4 -p tcp" -+ _, err = checkIPVSRule(cleanupCmd) -+ assert.NoError(err) -+} -+ -+func TestCleanupNoDestination(t *testing.T) { -+ assert := assert.New(t) -+ cleanupCommand := "conntrack -D --protonum tcp" -+ _, err := checkIPVSRule(cleanupCommand) -+ assert.Error(err) -+ cleanupCmd := "conntrack -D -p tcp" -+ _, err = checkIPVSRule(cleanupCmd) -+ assert.Error(err) -+} -+ -+func TestCleanupNoProtocol(t *testing.T) { -+ assert := assert.New(t) -+ cleanupCommand := "conntrack -D --orig-dst 192.168.0.4" -+ _, err := checkIPVSRule(cleanupCommand) -+ assert.Error(err) -+ cleanupCmd := "conntrack -D -d 192.168.0.4" -+ _, err = checkIPVSRule(cleanupCmd) -+ assert.Error(err) -+} -+ -+func TestCleanupIpError(t *testing.T) { -+ assert := assert.New(t) -+ cleanupCommand := "conntrack -D --orig-dst 192.168.0.2.4 --protonum tcp" -+ _, err := checkIPVSRule(cleanupCommand) -+ assert.Error(err) -+ cleanupCmd := "conntrack -D -d 192.168.0.2.4 -p tcp" -+ _, err = checkIPVSRule(cleanupCmd) -+ assert.Error(err) -+} -+ -+func TestCleanupProtocolError(t *testing.T) { -+ assert := assert.New(t) -+ cleanupCommand := "conntrack -D --orig-dst 192.168.0.4 --protonum ttcp" -+ _, err := checkIPVSRule(cleanupCommand) -+ assert.Error(err) -+ cleanupCmd := "conntrack -D -d 192.168.0.4 -p ttcp" -+ _, err = checkIPVSRule(cleanupCmd) -+ assert.Error(err) -+} -+ -+func TestRestoreFileNil(t *testing.T) { -+ assert := assert.New(t) -+ stdin := strings.NewReader("") -+ fileContext, cnt, err := getRestoreFile(stdin) -+ assert.Equal(cnt, 0) -+ assert.Equal(fileContext, "") -+ assert.Error(err) -+} -+ -+func TestRestoreFileError(t *testing.T) { -+ assert := assert.New(t) -+ stdin := strings.NewReader("aaa") -+ fileContext, cnt, err := getRestoreFile(stdin) -+ assert.Equal(cnt, 0) -+ assert.Equal(fileContext, "") -+ assert.Error(err) -+} -+ -+func TestRestoreFile(t *testing.T) { -+ assert := assert.New(t) -+ stdin := strings.NewReader("-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m") -+ fileContext, cnt, err := getRestoreFile(stdin) -+ assert.Equal(cnt, 2) -+ assert.Equal(fileContext, "-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m") -+ assert.NoError(err) -+} -diff --git a/virtcontainers/api_test.go b/virtcontainers/api_test.go -index c01d47b8..2a5488f9 100644 ---- a/virtcontainers/api_test.go -+++ b/virtcontainers/api_test.go -@@ -15,6 +15,7 @@ import ( - "syscall" - "testing" - -+ "github.com/kata-containers/agent/protocols/grpc" - ktu "github.com/kata-containers/runtime/pkg/katatestutils" - "github.com/kata-containers/runtime/virtcontainers/persist" - "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" -@@ -1745,3 +1746,38 @@ func TestCleanupContainer(t *testing.T) { - t.Fatal(err) - } - } -+ -+func TestUpdateIPVSRule(t *testing.T) { -+ defer cleanUp() -+ assert := assert.New(t) -+ -+ contID := "abc" -+ ctx := context.Background() -+ var ipvs = &grpc.UpdateIPVSRequest{ -+ IPVSReq: "ipvsadm -A -t 192.168.0.7:80 -s rr -p -3000", -+ } -+ -+ config := newTestSandboxConfigNoop() -+ p, _, err := createAndStartSandbox(ctx, config) -+ if p == nil || err != nil { -+ t.Fatal(err) -+ } -+ -+ s, ok := p.(*Sandbox) -+ assert.True(ok) -+ -+ contConfig := newTestContainerConfigNoop(contID) -+ c, err := p.CreateContainer(contConfig) -+ if c == nil || err != nil { -+ t.Fatal(err) -+ } -+ -+ _, err = UpdateIPVSRule(ctx, s.id, ipvs) -+ assert.NoError(err) -+ -+ _, err = UpdateIPVSRule(ctx, "aaa", ipvs) -+ assert.Error(err) -+ -+ _, err = UpdateIPVSRule(ctx, "", ipvs) -+ assert.Error(err) -+} -diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go -index 18a5a0a6..4f9409a0 100644 ---- a/virtcontainers/kata_agent_test.go -+++ b/virtcontainers/kata_agent_test.go -@@ -246,6 +246,10 @@ func (p *gRPCProxy) MemHotplugByProbe(ctx context.Context, req *pb.MemHotplugByP - return &gpb.Empty{}, nil - } - -+func (p *gRPCProxy) UpdateIPVSRule(ctx context.Context, req *pb.UpdateIPVSRequest) (*pb.IPVSResponse, error) { -+ return &pb.IPVSResponse{}, nil -+} -+ - func gRPCRegister(s *grpc.Server, srv interface{}) { - switch g := srv.(type) { - case *gRPCProxy: -diff --git a/virtcontainers/pkg/vcmock/types.go b/virtcontainers/pkg/vcmock/types.go -index 43247ef9..610b4602 100644 ---- a/virtcontainers/pkg/vcmock/types.go -+++ b/virtcontainers/pkg/vcmock/types.go -@@ -76,5 +76,5 @@ type VCMock struct { - UpdateRoutesFunc func(ctx context.Context, sandboxID string, routes []*vcTypes.Route) ([]*vcTypes.Route, error) - ListRoutesFunc func(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error) - CleanupContainerFunc func(ctx context.Context, sandboxID, containerID string, force bool) error -- UpdateIPVSRuleFunc func(ctx context.Context, sandboxID string, ipvs *grpc.UpdateIpvsRequest) (*grpc.IpvsResponse, error) -+ UpdateIPVSRuleFunc func(ctx context.Context, sandboxID string, ipvs *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) - } --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0038-pcie-using-pcie-root-port-driver-to-hotplug-device-i.patch b/runtime/patches/0038-pcie-using-pcie-root-port-driver-to-hotplug-device-i.patch deleted file mode 100644 index 8f3584f..0000000 --- a/runtime/patches/0038-pcie-using-pcie-root-port-driver-to-hotplug-device-i.patch +++ /dev/null @@ -1,501 +0,0 @@ -From 5a220e9be1cfb03316a62aa00d2040638ba1a855 Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Tue, 18 Aug 2020 15:15:52 +0800 -Subject: [PATCH 38/50] pcie: using pcie-root-port driver to hotplug device in - aarch64 - -reason: Since qemu with "virt" machine type doesn't support hotplug -device in the pcie.0 root bus, so need to use add root port devices -to support hotplug pci device in aarch64. - -we reuse the pcie_root_port config in the configuration.toml file to -set pcie_root_port device number when qemu process start. - -Signed-off-by: jiangpengfei ---- - vendor/github.com/intel/govmm/qemu/qemu.go | 22 ++--- - virtcontainers/persist/api/hypervisor.go | 5 +- - virtcontainers/qemu.go | 130 +++++++++++++++++++++++------ - virtcontainers/qemu_arch_base.go | 9 ++ - virtcontainers/types/pcie.go | 103 +++++++++++++++++++++++ - 5 files changed, 230 insertions(+), 39 deletions(-) - create mode 100644 virtcontainers/types/pcie.go - -diff --git a/vendor/github.com/intel/govmm/qemu/qemu.go b/vendor/github.com/intel/govmm/qemu/qemu.go -index a5e5dfaf..3e7720b4 100644 ---- a/vendor/github.com/intel/govmm/qemu/qemu.go -+++ b/vendor/github.com/intel/govmm/qemu/qemu.go -@@ -1252,6 +1252,8 @@ type PCIeRootPortDevice struct { - Chassis string // (slot, chassis) pair is mandatory and must be unique for each pcie-root-port, >=0, default is 0x00 - Slot string // >=0, default is 0x00 - -+ Port string // Port number is the device index -+ - Multifunction bool // true => "on", false => "off", default is off - Addr string // >=0, default is 0x00 - -@@ -1277,6 +1279,10 @@ func (b PCIeRootPortDevice) QemuParams(config *Config) []string { - - deviceParams = append(deviceParams, fmt.Sprintf("%s,id=%s", driver, b.ID)) - -+ if b.Port != "" { -+ deviceParams = append(deviceParams, fmt.Sprintf("port=%s", b.Port)) -+ } -+ - if b.Bus == "" { - b.Bus = "pcie.0" - } -@@ -1287,20 +1293,10 @@ func (b PCIeRootPortDevice) QemuParams(config *Config) []string { - } - deviceParams = append(deviceParams, fmt.Sprintf("chassis=%s", b.Chassis)) - -- if b.Slot == "" { -- b.Slot = "0x00" -- } -- deviceParams = append(deviceParams, fmt.Sprintf("slot=%s", b.Slot)) -- -- multifunction := "off" - if b.Multifunction { -- multifunction = "on" -- if b.Addr == "" { -- b.Addr = "0x00" -- } -- deviceParams = append(deviceParams, fmt.Sprintf("addr=%s", b.Addr)) -+ deviceParams = append(deviceParams, "multifunction=on") - } -- deviceParams = append(deviceParams, fmt.Sprintf("multifunction=%v", multifunction)) -+ deviceParams = append(deviceParams, fmt.Sprintf("addr=%s", b.Addr)) - - if b.BusReserve != "" { - deviceParams = append(deviceParams, fmt.Sprintf("bus-reserve=%s", b.BusReserve)) -@@ -1337,7 +1333,7 @@ func (b PCIeRootPortDevice) Valid() bool { - if b.Pref64Reserve != "" && b.Pref32Reserve != "" { - return false - } -- if b.ID == "" { -+ if b.ID == "" || b.Port == "" || b.Bus == "" || b.Addr == ""{ - return false - } - return true -diff --git a/virtcontainers/persist/api/hypervisor.go b/virtcontainers/persist/api/hypervisor.go -index 375fd56b..fd61b3c2 100644 ---- a/virtcontainers/persist/api/hypervisor.go -+++ b/virtcontainers/persist/api/hypervisor.go -@@ -5,6 +5,8 @@ - - package persistapi - -+import "github.com/kata-containers/runtime/virtcontainers/types" -+ - // Bridge is a bridge where devices can be hot plugged - type Bridge struct { - // DeviceAddr contains information about devices plugged and its address in the bridge -@@ -35,7 +37,8 @@ type HypervisorState struct { - - // Belows are qemu specific - // Refs: virtcontainers/qemu.go:QemuState -- Bridges []Bridge -+ Bridges []Bridge -+ PCIeRootPortsPool *types.PCIeRootPortPool - // HotpluggedCPUs is the list of CPUs that were hot-added - HotpluggedVCPUs []CPUDevice - HotpluggedMemory int -diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go -index c2b65376..a10c66fb 100644 ---- a/virtcontainers/qemu.go -+++ b/virtcontainers/qemu.go -@@ -64,8 +64,9 @@ type CPUDevice struct { - - // QemuState keeps Qemu's state - type QemuState struct { -- Bridges []types.Bridge -- ScsiBus *types.SCSIBus -+ Bridges []types.Bridge -+ ScsiBus *types.SCSIBus -+ PCIeRootPortsPool *types.PCIeRootPortPool - // HotpluggedCPUs is the list of CPUs that were hot-added - HotpluggedVCPUs []CPUDevice - HotpluggedMemory int -@@ -271,6 +272,9 @@ func (q *qemu) setup(id string, hypervisorConfig *HypervisorConfig) error { - - q.state.HotplugVFIOOnRootBus = q.config.HotplugVFIOOnRootBus - q.state.PCIeRootPort = int(q.config.PCIeRootPort) -+ // init the PCIeRootPortsPool with pcie_root_port config value -+ q.state.PCIeRootPortsPool = &types.PCIeRootPortPool{} -+ q.state.PCIeRootPortsPool.Init(q.state.PCIeRootPort) - - // The path might already exist, but in case of VM templating, - // we have to create it since the sandbox has not created it yet. -@@ -394,9 +398,18 @@ func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.I - return nil, nil, err - } - -- // Add bridges before any other devices. This way we make sure that -- // bridge gets the first available PCI address i.e bridgePCIStartAddr -- devices = q.arch.appendBridges(devices) -+ machine, err := q.getQemuMachine() -+ if err != nil { -+ return nil, nil, err -+ } -+ switch machine.Type { -+ case QemuVirt: -+ devices = q.arch.appendRootPorts(devices, q.state.PCIeRootPortsPool) -+ default: -+ // Add bridges before any other devices. This way we make sure that -+ // bridge gets the first available PCI address i.e bridgePCIStartAddr -+ devices = q.arch.appendBridges(devices) -+ } - - devices, err = q.arch.appendConsole(devices, console) - if err != nil { -@@ -608,7 +621,7 @@ func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNa - // Add PCIe Root Port devices to hypervisor - // The pcie.0 bus do not support hot-plug, but PCIe device can be hot-plugged into PCIe Root Port. - // For more details, please see https://github.com/qemu/qemu/blob/master/docs/pcie.txt -- if hypervisorConfig.PCIeRootPort > 0 { -+ if hypervisorConfig.PCIeRootPort > 0 && hypervisorConfig.HypervisorMachineType == QemuQ35 { - qemuConfig.Devices = q.arch.appendPCIeRootPortDevice(qemuConfig.Devices, hypervisorConfig.PCIeRootPort) - } - -@@ -1154,21 +1167,19 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev - } - case q.config.BlockDeviceDriver == config.VirtioBlock: - driver := "virtio-blk-pci" -- addr, bridge, err := q.arch.addDeviceToBridge(drive.ID, types.PCI) -+ -+ addr, bus, pciAddr, err := q.getPciAddress(drive.ID, types.PCI) - if err != nil { - return err - } -- - defer func() { - if err != nil { -- q.arch.removeDeviceFromBridge(drive.ID) -+ q.putPciAddress(drive.ID) - } - }() -+ drive.PCIAddr = pciAddr - -- // PCI address is in the format bridge-addr/device-addr eg. "03/02" -- drive.PCIAddr = fmt.Sprintf("%02x", bridge.Addr) + "/" + addr -- -- if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID, romFile, 0, true, defaultDisableModern); err != nil { -+ if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bus, romFile, 0, true, defaultDisableModern); err != nil { - return err - } - case q.config.BlockDeviceDriver == config.VirtioSCSI: -@@ -1249,7 +1260,7 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error - err = q.hotplugAddBlockDevice(drive, op, devID) - } else { - if q.config.BlockDeviceDriver == config.VirtioBlock { -- if err := q.arch.removeDeviceFromBridge(drive.ID); err != nil { -+ if err := q.putPciAddress(drive.ID); err != nil { - return err - } - } else if q.config.BlockDeviceDriver == config.VirtioSCSI { -@@ -1345,22 +1356,22 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro - } - } - -- addr, bridge, err := q.arch.addDeviceToBridge(devID, types.PCI) -+ addr, bus, _, err := q.getPciAddress(devID, types.PCI) - if err != nil { - return err - } - - defer func() { - if err != nil { -- q.arch.removeDeviceFromBridge(devID) -+ q.putPciAddress(devID) - } - }() - - switch device.Type { - case config.VFIODeviceNormalType: -- return q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bridge.ID, romFile) -+ return q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bus, romFile) - case config.VFIODeviceMediatedType: -- return q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, addr, bridge.ID, romFile) -+ return q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, addr, bus, romFile) - default: - return fmt.Errorf("Incorrect VFIO device type found") - } -@@ -1368,7 +1379,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro - q.Logger().WithField("dev-id", devID).Info("Start hot-unplug VFIO device") - - if !q.state.HotplugVFIOOnRootBus { -- if err := q.arch.removeDeviceFromBridge(devID); err != nil { -+ if err := q.putPciAddress(devID); err != nil { - return err - } - } -@@ -1439,18 +1450,17 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) { - } - }() - -- addr, bridge, err := q.arch.addDeviceToBridge(tap.ID, types.PCI) -+ addr, bus, pciAddr, err := q.getPciAddress(tap.ID, types.PCI) - if err != nil { - return err - } - - defer func() { - if err != nil { -- q.arch.removeDeviceFromBridge(tap.ID) -+ q.putPciAddress(tap.ID) - } - }() - -- pciAddr := fmt.Sprintf("%02x/%s", bridge.Addr, addr) - endpoint.SetPciAddr(pciAddr) - - var machine govmmQemu.Machine -@@ -1459,14 +1469,14 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) { - return err - } - if machine.Type == QemuCCWVirtio { -- devNoHotplug := fmt.Sprintf("fe.%x.%x", bridge.Addr, addr) -+ devNoHotplug := fmt.Sprintf("fe.%x.%x", bus, addr) - return q.qmpMonitorCh.qmp.ExecuteNetCCWDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), devNoHotplug, int(q.config.NumVCPUs)) - } -- return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bridge.ID, romFile, int(q.config.NumVCPUs), defaultDisableModern) -+ return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bus, romFile, int(q.config.NumVCPUs), defaultDisableModern) - - } - -- if err := q.arch.removeDeviceFromBridge(tap.ID); err != nil { -+ if err := q.putPciAddress(tap.ID); err != nil { - return err - } - -@@ -2042,6 +2052,21 @@ func genericMemoryTopology(memoryMb, hostMemoryMb uint64, slots uint8, memoryOff - return memory - } - -+func genericAppendRootPorts(devices []govmmQemu.Device, rootPorts *types.PCIeRootPortPool) []govmmQemu.Device { -+ for _, rp := range rootPorts.Items { -+ devices = append(devices, govmmQemu.PCIeRootPortDevice{ -+ Port: rp.Port, -+ Bus: rp.Bus, -+ ID: rp.ID, -+ Chassis: strconv.Itoa(int(rp.Chassis)), -+ Multifunction: rp.Multifunction, -+ Addr: fmt.Sprintf("0x%s.0x%s", rp.Slot, rp.Function), -+ }) -+ } -+ -+ return devices -+} -+ - // genericAppendPCIeRootPort appends to devices the given pcie-root-port - func genericAppendPCIeRootPort(devices []govmmQemu.Device, number uint32, machineType string) []govmmQemu.Device { - var ( -@@ -2241,6 +2266,7 @@ func (q *qemu) save() (s persistapi.HypervisorState) { - s.HotpluggedMemory = q.state.HotpluggedMemory - s.HotplugVFIOOnRootBus = q.state.HotplugVFIOOnRootBus - s.PCIeRootPort = q.state.PCIeRootPort -+ s.PCIeRootPortsPool = q.state.PCIeRootPortsPool - - for _, bridge := range q.arch.getBridges() { - s.Bridges = append(s.Bridges, persistapi.Bridge{ -@@ -2265,6 +2291,7 @@ func (q *qemu) load(s persistapi.HypervisorState) { - q.state.HotplugVFIOOnRootBus = s.HotplugVFIOOnRootBus - q.state.VirtiofsdPid = s.VirtiofsdPid - q.state.PCIeRootPort = s.PCIeRootPort -+ q.state.PCIeRootPortsPool = s.PCIeRootPortsPool - - for _, bridge := range s.Bridges { - q.state.Bridges = append(q.state.Bridges, types.NewBridge(types.Type(bridge.Type), bridge.ID, bridge.DeviceAddr, bridge.Addr)) -@@ -2302,3 +2329,56 @@ func (q *qemu) generateSocket(id string, useVsock bool) (interface{}, error) { - func (q *qemu) getMemorySize() uint32 { - return q.config.MemorySize + uint32(q.state.HotpluggedMemory) - } -+ -+// getPciAddress allocate the pci slot to hotplugged device and -+// return the pci slot address -+func (q *qemu) getPciAddress(devID string, t types.Type) (slot, bus, pciAddr string, err error) { -+ machine, err := q.getQemuMachine() -+ if err != nil { -+ return "", "", "", err -+ } -+ -+ switch machine.Type { -+ case QemuVirt: -+ rp, err := q.state.PCIeRootPortsPool.AddDevice(devID) -+ if err != nil { -+ return "", "", "", err -+ } -+ // PCIe Root Port only have one slot -+ slot = "0x0" -+ // pciAddr specifies the slot and function of the Root Port and the slot of the device -+ pciAddr = fmt.Sprintf("%s.%s/00", rp.Slot, rp.Function) -+ bus = rp.ID -+ default: -+ var bridge types.Bridge -+ slot, bridge, err = q.arch.addDeviceToBridge(devID, t) -+ if err != nil { -+ return "", "", "", err -+ } -+ bus = bridge.ID -+ // PCI address is in the format bridge-addr.0/device-addr eg. "03.0/02" -+ pciAddr = fmt.Sprintf("%02x.0", bridge.Addr) + "/" + slot -+ } -+ return slot, bus, pciAddr, nil -+} -+ -+func (q *qemu) putPciAddress(devID string) error { -+ machine, err := q.getQemuMachine() -+ if err != nil { -+ return err -+ } -+ -+ switch machine.Type { -+ case QemuVirt: -+ err := q.state.PCIeRootPortsPool.RemoveDevice(devID) -+ if err != nil { -+ return err -+ } -+ default: -+ if err := q.arch.removeDeviceFromBridge(devID); err != nil { -+ return err -+ } -+ } -+ -+ return nil -+} -diff --git a/virtcontainers/qemu_arch_base.go b/virtcontainers/qemu_arch_base.go -index 9d72dd09..cb045530 100644 ---- a/virtcontainers/qemu_arch_base.go -+++ b/virtcontainers/qemu_arch_base.go -@@ -130,6 +130,9 @@ type qemuArch interface { - - // appendPCIeRootPortDevice appends a pcie-root-port device to pcie.0 bus - appendPCIeRootPortDevice(devices []govmmQemu.Device, number uint32) []govmmQemu.Device -+ -+ // appendRootPorts appends a pcie-root-port device to devices when qemu machine type is "virt" -+ appendRootPorts(devices []govmmQemu.Device, rootPorts *types.PCIeRootPortPool) []govmmQemu.Device - } - - type qemuArchBase struct { -@@ -766,3 +769,9 @@ func (q *qemuArchBase) addBridge(b types.Bridge) { - func (q *qemuArchBase) appendPCIeRootPortDevice(devices []govmmQemu.Device, number uint32) []govmmQemu.Device { - return genericAppendPCIeRootPort(devices, number, q.machineType) - } -+ -+// appendRootPorts appends a pcie-root-port device to devices when qemu machine type is "virt" -+// which is different appendPCIeRootPortDevice function -+func (q *qemuArchBase) appendRootPorts(devices []govmmQemu.Device, rootPorts *types.PCIeRootPortPool) []govmmQemu.Device { -+ return genericAppendRootPorts(devices, rootPorts) -+} -diff --git a/virtcontainers/types/pcie.go b/virtcontainers/types/pcie.go -new file mode 100644 -index 00000000..83eb6944 ---- /dev/null -+++ b/virtcontainers/types/pcie.go -@@ -0,0 +1,103 @@ -+package types -+ -+import ( -+ "fmt" -+) -+ -+const ( -+ maxRootPortsCapacity = 25 -+ slotPerDevice = 7 -+ -+ // PCIeRootBus is "pcie.0" -+ PCIeRootBus = "pcie.0" -+ -+ // startPort specifies the start port of pcie-root-port -+ // for the first slot of "pcie.0" is reserved, so the 1~7 ports are also reserved -+ startPort = 8 -+ funcNumPerSlot = 8 -+) -+ -+// PCIeRootPort describe the PCIe Root Port -+type PCIeRootPort struct { -+ // DeviceID specify the device hotplug on the Root Port -+ DeviceID string -+ -+ // Port number is the Root Port index -+ Port string -+ -+ // Bus number where the Root Port is plugged, typically pcie.0 -+ Bus string -+ -+ // ID is used to identify the pcie-root-port in qemu -+ ID string -+ -+ // Slot specifies slot address of Root Port -+ Slot string -+ -+ // Function specifies function of Root Port -+ Function string -+ -+ // Chassis number -+ Chassis uint32 -+ -+ // Multifunction is used to specify the pcie-root-port is multifunction supported -+ Multifunction bool -+} -+ -+// PCIeRootPortPool describe a set of PCIe Root Ports -+type PCIeRootPortPool struct { -+ // Items contains information about devices plugged and number limit -+ Items []*PCIeRootPort -+} -+ -+// Init Initialized the PCIeRootPortPool instance -+func (rp *PCIeRootPortPool) Init(number int) { -+ if number == 0 || number > maxRootPortsCapacity { -+ number = maxRootPortsCapacity -+ } -+ -+ for i := 0; i < number; i++ { -+ dev := &PCIeRootPort{ -+ DeviceID: "", -+ Port: fmt.Sprintf("0x%x", startPort+i), -+ Bus: PCIeRootBus, -+ ID: fmt.Sprintf("pci.%d", i+1), -+ Chassis: uint32(i + 1), -+ } -+ -+ major := i / funcNumPerSlot -+ minor := i % funcNumPerSlot -+ dev.Multifunction = false -+ if minor == 0 { -+ dev.Multifunction = true -+ } -+ dev.Slot = fmt.Sprintf("%02x", major+1) -+ dev.Function = fmt.Sprintf("%x", minor) -+ -+ rp.Items = append(rp.Items, dev) -+ } -+} -+ -+// AddDevice add a device to the PCIeRootPortPool -+func (rp *PCIeRootPortPool) AddDevice(devID string) (*PCIeRootPort, error) { -+ for _, it := range rp.Items { -+ if it.DeviceID == "" { -+ it.DeviceID = devID -+ return it, nil -+ } -+ } -+ return nil, fmt.Errorf("Unable to hot plug device on Root Ports: there are not empty slots") -+} -+ -+// RemoveDevice remove a device from the PCIeRootPortPool -+func (rp *PCIeRootPortPool) RemoveDevice(devID string) error { -+ for _, it := range rp.Items { -+ if it.DeviceID == devID { -+ // free address to re-use the same slot with other devices -+ it.DeviceID = "" -+ return nil -+ } -+ } -+ -+ return fmt.Errorf("Unable to hot unplug device %s: not present on Root Port", devID) -+} --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0039-storage-add-storage-common-functions-and-structs.patch b/runtime/patches/0039-storage-add-storage-common-functions-and-structs.patch deleted file mode 100644 index 701b072..0000000 --- a/runtime/patches/0039-storage-add-storage-common-functions-and-structs.patch +++ /dev/null @@ -1,245 +0,0 @@ -From a36c9857447aaf22628af1ef01406a916436133b Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Tue, 18 Aug 2020 22:00:04 +0800 -Subject: [PATCH 39/50] storage: add storage common functions and structs - -reason: -1. add storage.go and storage_spec.go -2. provide funcs for mount nfs and gpath - -Signed-off-by: jiangpengfei ---- - virtcontainers/storage/storage.go | 115 +++++++++++++++++++++++++++++++++ - virtcontainers/storage/storage_spec.go | 98 ++++++++++++++++++++++++++++ - 2 files changed, 213 insertions(+) - create mode 100644 virtcontainers/storage/storage.go - create mode 100644 virtcontainers/storage/storage_spec.go - -diff --git a/virtcontainers/storage/storage.go b/virtcontainers/storage/storage.go -new file mode 100644 -index 00000000..77ef994f ---- /dev/null -+++ b/virtcontainers/storage/storage.go -@@ -0,0 +1,115 @@ -+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. -+// SPDX-License-Identifier: Apache-2.0 -+// Description: common functions -+// Author: licuifang -+// Create: 2019-07-14 -+ -+// Package storage provides functions for mounting -+package storage -+ -+import ( -+ "fmt" -+ "path/filepath" -+ "regexp" -+ "strings" -+ -+ "github.com/kata-containers/agent/protocols/grpc" -+ "github.com/opencontainers/runtime-spec/specs-go" -+) -+ -+var ( -+ // simple regular expressions for "nfs-server.com:/aaa/bbb/ccc/" or "192.168.1.1:/remote/path" -+ regUrl = regexp.MustCompile(`^((([a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,6})|((?:(?:25[0-5]|2[0-4]\d|[01]?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d?\d))):/[A-Za-z0-9_./-]{0,1000}$`) -+ -+ defaultMountOption = "nolock" -+ storageMountOptionKey = "mount_op" -+ storageCustomOptionKey = "custom_op" -+) -+ -+type RemoteStorage struct { -+ Source string -+ Dest string -+ Options map[string][]string -+} -+ -+func (n *RemoteStorage) Validate(volumeType string) error { -+ if n.Source == "" || n.Dest == "" { -+ return fmt.Errorf("the source and dest of storage cannot be empty") -+ } -+ switch volumeType { -+ case NFS: -+ chk := regUrl.FindAllString(n.Source, -1) -+ if chk == nil { -+ return fmt.Errorf("invalid url for nfs") -+ } -+ case GPATHFS: -+ if !filepath.IsAbs(n.Source) { -+ return fmt.Errorf("invalid gpath") -+ } -+ } -+ -+ return nil -+} -+ -+func (n *RemoteStorage) GetGrpcStorageAndAppendMount(volumeType string, vmBasePath string, spec *specs.Spec, sandboxId string) *grpc.Storage { -+ var ( -+ grpcStorag *grpc.Storage -+ vmPath string -+ ) -+ switch volumeType { -+ case NFS: -+ // source of sfs like remote-nfs.com:/share-53ee51a1 -+ // source of sfs with subpath like remote-nfs.com:/share-53ee51a1/aaa/bbb -+ // source of sfs-turbo like ip:/ -+ // source of sfs-turbo with subpath like ip://aaa or ip://aaa/bbb -+ // here we only get the origin source such as remote-nfs.com:/share-53ee51a1 or ip:/ as the Source of grpcStorage -+ item := strings.Split(n.Source, ":") -+ if len(item) != 2 { -+ return nil -+ } -+ vmPath = filepath.Join(vmBasePath, sandboxId, item[0], item[1]) -+ parts := strings.SplitAfter(item[1], "/") -+ if len(parts) > 2 { -+ n.Source = item[0] + ":" + parts[0] + parts[1] -+ } -+ -+ grpcStorag = &grpc.Storage{ -+ Driver: NFS, -+ Source: n.Source, -+ MountPoint: vmPath, -+ } -+ grpcStorag.Options = append(grpcStorag.Options, defaultMountOption) -+ case GPATHFS: -+ vmPath = n.Source -+ grpcStorag = &grpc.Storage{ -+ Driver: GPATHFS, -+ Source: vmPath, -+ MountPoint: n.Dest, -+ } -+ } -+ if custumOpts, ok := n.Options[storageCustomOptionKey]; ok { -+ for _, custumOpt := range custumOpts { -+ grpcStorag.Options = append(grpcStorag.Options, custumOpt) -+ } -+ } -+ -+ mnt := specs.Mount{ -+ Source: vmPath, -+ Destination: n.Dest, -+ } -+ if mountOpts, ok := n.Options[storageMountOptionKey]; ok { -+ for _, mountOpt := range mountOpts { -+ if mountOpt == "shared" { -+ // in order to support "shared" propagation for guestpath mounting to container directory -+ // we should set rootfspropagation shared -+ spec.Linux.RootfsPropagation = "shared" -+ } -+ mnt.Options = append(mnt.Options, mountOpt) -+ } -+ } -+ mnt.Options = append(mnt.Options, "rbind") -+ mnt.Type = "bind" -+ spec.Mounts = append(spec.Mounts, mnt) -+ -+ return grpcStorag -+} -diff --git a/virtcontainers/storage/storage_spec.go b/virtcontainers/storage/storage_spec.go -new file mode 100644 -index 00000000..8e866b8d ---- /dev/null -+++ b/virtcontainers/storage/storage_spec.go -@@ -0,0 +1,98 @@ -+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. -+// Description: common functions -+// Author: caihaomin c00416947 -+// Create: 2019-05-05 -+ -+package storage -+ -+import ( -+ "encoding/json" -+ "fmt" -+ -+ "github.com/kata-containers/agent/protocols/grpc" -+ "github.com/opencontainers/runtime-spec/specs-go" -+) -+ -+const ( -+ NFS = "nfs" -+ GPATHFS = "gpath" -+) -+ -+// STORAGES specifies the storage type and is supported -+var STORAGES = map[string]bool{ -+ NFS: true, -+ GPATHFS: true, -+} -+ -+type StorageSpec struct { -+ // StorageType specifies the type of storage passing down -+ StorageType string `json:"storage_type,omitempty"` -+ -+ Source string `json:"source,omitempty"` -+ Destination string `json:"dest,omitempty"` -+ -+ // Options specifies the options of storage type -+ Options map[string][]string `json:"options,omitempty"` -+} -+ -+func (s *StorageSpec) generateInstances() StorageOperation { -+ var storageOpt StorageOperation -+ storageOpt = &RemoteStorage{ -+ Source: s.Source, -+ Dest: s.Destination, -+ Options: s.Options, -+ } -+ -+ return storageOpt -+} -+ -+func ValidateStorageValue(value string) error { -+ var storageSpecs []StorageSpec -+ if err := json.Unmarshal([]byte(value), &storageSpecs); err != nil { -+ return err -+ } -+ -+ var storageOpt []StorageOperation -+ for _, item := range storageSpecs { -+ if supported, ok := STORAGES[item.StorageType]; !ok || !supported { -+ return fmt.Errorf("type %s of storage is not supported", item.StorageType) -+ } -+ storageItem := item.generateInstances() -+ if err := storageItem.Validate(item.StorageType); err != nil { -+ return err -+ } -+ storageOpt = append(storageOpt, storageItem) -+ } -+ -+ return nil -+} -+ -+func GetGrpcStorageAndAppendMount(vmBasePath, value string, spec *specs.Spec, sandboxId string) []*grpc.Storage { -+ var ( -+ grpcStorages []*grpc.Storage -+ storageSpec []StorageSpec -+ ) -+ -+ // the value has been validated before, and there is no need to judge the return value here -+ json.Unmarshal([]byte(value), &storageSpec) -+ for _, item := range storageSpec { -+ storageItem := item.generateInstances() -+ grpcStorages = append(grpcStorages, storageItem.GetGrpcStorageAndAppendMount(item.StorageType, vmBasePath, spec, sandboxId)) -+ } -+ return grpcStorages -+} -+ -+type StorageOperation interface { -+ Validate(volumeType string) error -+ GetGrpcStorageAndAppendMount(volumeType string, vmBasePath string, spec *specs.Spec, sandboxId string) *grpc.Storage -+} -+ -+type FakeStorage struct{} -+ -+func (f *FakeStorage) Validate() error { -+ return nil -+} -+ -+func (f *FakeStorage) GetGrpcStorageAndAppendMount(vmBasePath string, spec *specs.Spec) *grpc.Storage { -+ return &grpc.Storage{} -+} --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0040-storage-add-go-tests-for-storage.patch b/runtime/patches/0040-storage-add-go-tests-for-storage.patch deleted file mode 100644 index e08f71d..0000000 --- a/runtime/patches/0040-storage-add-go-tests-for-storage.patch +++ /dev/null @@ -1,221 +0,0 @@ -From 2e32e2c156a134605de42c53ef77366ac73f2614 Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Tue, 18 Aug 2020 22:01:58 +0800 -Subject: [PATCH 40/50] storage: add go tests for storage - -reason: add go tests file storage_test.go and storage_spec_test.go - -Signed-off-by: jiangpengfei ---- - virtcontainers/storage/storage_spec_test.go | 151 ++++++++++++++++++++++++++++ - virtcontainers/storage/storage_test.go | 40 ++++++++ - 2 files changed, 191 insertions(+) - create mode 100644 virtcontainers/storage/storage_spec_test.go - create mode 100644 virtcontainers/storage/storage_test.go - -diff --git a/virtcontainers/storage/storage_spec_test.go b/virtcontainers/storage/storage_spec_test.go -new file mode 100644 -index 00000000..f638245a ---- /dev/null -+++ b/virtcontainers/storage/storage_spec_test.go -@@ -0,0 +1,151 @@ -+/* -+Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. -+Description: common functions test -+Author: yangfeiyu -+Create: 2019-07-24 -+*/ -+ -+package storage -+ -+import ( -+ "encoding/json" -+ "testing" -+ -+ "github.com/stretchr/testify/assert" -+ "github.com/opencontainers/runtime-spec/specs-go" -+) -+ -+func TestValidateStorageValue(t *testing.T) { -+ assert := assert.New(t) -+ //invalid storage type,not NFS or GPATHS -+ spec := &StorageSpec{ -+ StorageType: "invalidStorage", -+ Source: "sfs.com:/remote/path", -+ Destination: "/opt/", -+ } -+ data, err := json.Marshal(spec) -+ assert.NoError(err) -+ -+ err = ValidateStorageValue(string(data)) -+ assert.Error(err, "StorageType is not NFS or GPATHS,it should return error") -+ -+ //case 1: NFS -+ specNFS := []StorageSpec{ -+ { -+ StorageType: NFS, -+ Source: "sfs.com:/remote/path", -+ Destination: "/opt/"}, -+ } -+ data, err = json.Marshal(specNFS) -+ assert.NoError(err) -+ -+ err = ValidateStorageValue(string(data)) -+ assert.NoError(err) -+ -+ // case2: invalid source domain address -+ specNFS = []StorageSpec{ -+ { -+ StorageType: NFS, -+ Source: "sfs..com:/remote/path", -+ Destination: "/opt/"}, -+ } -+ data, err = json.Marshal(specNFS) -+ assert.NoError(err) -+ -+ err = ValidateStorageValue(string(data)) -+ assert.Error(err) -+ -+ // case 3: nfs source is valid ip address -+ ipSpecNFS := []StorageSpec{ -+ { -+ StorageType: NFS, -+ Source: "192.168.18.147:/remote/path", -+ Destination: "/tmp/sfsturbo0", -+ }, -+ } -+ -+ data, err = json.Marshal(ipSpecNFS) -+ assert.NoError(err) -+ -+ err = ValidateStorageValue(string(data)) -+ assert.NoError(err) -+ -+ // case 4: invalid ip address for nfs source -+ ipSpecNFS = []StorageSpec{ -+ { -+ StorageType: NFS, -+ Source: "192.168.18.300:/remote/path", -+ Destination: "/tmp/sfsturbo0", -+ }, -+ } -+ -+ data, err = json.Marshal(ipSpecNFS) -+ assert.NoError(err) -+ -+ err = ValidateStorageValue(string(data)) -+ assert.Error(err) -+ -+ // case 5: validate ip address and source remote path is / -+ ipSpecNFS = []StorageSpec{ -+ { -+ StorageType: NFS, -+ Source: "192.168.18.3:/", -+ Destination: "/tmp/sfsturbo0", -+ }, -+ } -+ -+ data, err = json.Marshal(ipSpecNFS) -+ assert.NoError(err) -+ -+ err = ValidateStorageValue(string(data)) -+ assert.NoError(err) -+ -+ // case 6: invalid ip address 192.168.18.147.11 -+ ipSpecNFS = []StorageSpec{ -+ { -+ StorageType: NFS, -+ Source: "192.168.18.147.11:/", -+ Destination: "/tmp/sfsturbo0", -+ }, -+ } -+ -+ data, err = json.Marshal(ipSpecNFS) -+ assert.NoError(err) -+ -+ err = ValidateStorageValue(string(data)) -+ assert.Error(err) -+ -+ //GPATHS -+ specGPATHFS := []StorageSpec{ -+ { -+ StorageType: GPATHFS, -+ Source: "/remote/path", -+ Destination: "/opt/"}, -+ } -+ data, err = json.Marshal(specGPATHFS) -+ assert.NoError(err) -+ -+ err = ValidateStorageValue(string(data)) -+ if STORAGES[GPATHFS] == true { -+ assert.NoError(err, "StorageType GPATHFS is valid,it should return no error") -+ } else { -+ assert.Error(err, "StorageType GPATHFS is invalid,it should return error") -+ } -+} -+ -+func TestGetGrpcStorageAndAppendMount(t *testing.T) { -+ assert := assert.New(t) -+ var spec specs.Spec -+ sandboxId := "7f3b5e32cc7fe757e1b69f7a005ca1971def5011e730c82535783630fe24b318" -+ vmBasePath := "/tmp" -+ -+ // storageType nfs with legal source and dest will succeed -+ storageSpec := "[{\"storage_type\":\"nfs\",\"source\":\"sfs.com:/remote/path\",\"dest\":\"/opt/nfs\"}]" -+ grpcStorages := GetGrpcStorageAndAppendMount(vmBasePath,storageSpec,&spec,sandboxId) -+ assert.NotNil(grpcStorages[0]) -+ -+ // storageType nfs with illegal source and dest will fail -+ storageSpec = "[{\"storage_type\":\"nfs\",\"source\":\"sfs.com\",\"dest\":\"/opt/nfs\"}]" -+ grpcStorages = GetGrpcStorageAndAppendMount(vmBasePath,storageSpec,&spec,sandboxId) -+ assert.Nil(grpcStorages[0]) -+} -diff --git a/virtcontainers/storage/storage_test.go b/virtcontainers/storage/storage_test.go -new file mode 100644 -index 00000000..c9ca4926 ---- /dev/null -+++ b/virtcontainers/storage/storage_test.go -@@ -0,0 +1,40 @@ -+/* -+Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. -+Description: common functions test -+Author: yangfeiyu -+Create: 2019-07-24 -+*/ -+ -+package storage -+ -+import ( -+ "testing" -+ -+ "github.com/stretchr/testify/assert" -+) -+ -+func TestValidate(t *testing.T) { -+ //NFS valid -+ assert := assert.New(t) -+ remoteStorage := &RemoteStorage{ -+ Source: "sfs.com:/remote/path", -+ Dest: "/opt/", -+ } -+ err := remoteStorage.Validate(NFS) -+ assert.NoError(err, "NFS url is valid,it should return no error") -+ -+ //NFS invalid -+ remoteStorage.Source = "sfs.com/../remote/path" -+ err = remoteStorage.Validate(NFS) -+ assert.Error(err, "NFS url is invalid,it should return error") -+ -+ //GPATHS valid -+ remoteStorage.Source = "/path/in/vm" -+ err = remoteStorage.Validate(GPATHFS) -+ assert.NoError(err, "GPATHFS is valid,it should return no error") -+ -+ //GPATHS invalid -+ remoteStorage.Source = "./../path/in/../vm" -+ err = remoteStorage.Validate(GPATHFS) -+ assert.Error(err, "GPATHFS is invalid,it should return error") -+} --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0041-storage-mount-nfs-and-gpath-with-given-annotation.patch b/runtime/patches/0041-storage-mount-nfs-and-gpath-with-given-annotation.patch deleted file mode 100644 index b63c2d7..0000000 --- a/runtime/patches/0041-storage-mount-nfs-and-gpath-with-given-annotation.patch +++ /dev/null @@ -1,115 +0,0 @@ -From 31e07f1b6cbf361783c4d7adf9e4b8da30c67384 Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Tue, 18 Aug 2020 22:05:25 +0800 -Subject: [PATCH 41/50] storage: mount nfs and gpath with given annotation - -reason: when run container with annotation about storage spec, -prepare basic info in kata-runtime - -Signed-off-by: jiangpengfei ---- - virtcontainers/kata_agent.go | 14 +++++++++++++- - virtcontainers/pkg/annotations/annotations.go | 3 +++ - virtcontainers/pkg/oci/utils.go | 16 ++++++++++++++++ - 3 files changed, 32 insertions(+), 1 deletion(-) - -diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go -index d82a7f2d..ac64817a 100644 ---- a/virtcontainers/kata_agent.go -+++ b/virtcontainers/kata_agent.go -@@ -30,10 +30,11 @@ import ( - ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter" - "github.com/kata-containers/runtime/virtcontainers/pkg/rootless" - vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" -- "github.com/kata-containers/runtime/virtcontainers/utils" - "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" -+ "github.com/kata-containers/runtime/virtcontainers/storage" - "github.com/kata-containers/runtime/virtcontainers/store" - "github.com/kata-containers/runtime/virtcontainers/types" -+ "github.com/kata-containers/runtime/virtcontainers/utils" - "github.com/opencontainers/runtime-spec/specs-go" - opentracing "github.com/opentracing/opentracing-go" - "github.com/sirupsen/logrus" -@@ -1427,6 +1428,9 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process, - localStorages := k.handleLocalStorage(ociSpec.Mounts, sandbox.id, c.rootfsSuffix) - ctrStorages = append(ctrStorages, localStorages...) - -+ remoteStoragtes := k.handleRemoteStorage(ociSpec, sandbox.id) -+ ctrStorages = append(ctrStorages, remoteStoragtes...) -+ - // We replace all OCI mount sources that match our container mount - // with the right source path (The guest one). - if err = k.replaceOCIMountSource(ociSpec, newMounts); err != nil { -@@ -1510,6 +1514,14 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process, - k.state.URL, consoleURL, c.config.Cmd, createNSList, enterNSList) - } - -+func (k *kataAgent) handleRemoteStorage(spec *specs.Spec, sandboxId string) []*grpc.Storage { -+ if value, ok := spec.Annotations[vcAnnotations.StorageSpecTypeKey]; ok { -+ return storage.GetGrpcStorageAndAppendMount(kataGuestStorageDir, value, spec, sandboxId) -+ } -+ -+ return []*grpc.Storage{} -+} -+ - // handleEphemeralStorage handles ephemeral storages by - // creating a Storage from corresponding source of the mount point - func (k *kataAgent) handleEphemeralStorage(mounts []specs.Mount) []*grpc.Storage { -diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go -index 903c7f03..e50a697c 100644 ---- a/virtcontainers/pkg/annotations/annotations.go -+++ b/virtcontainers/pkg/annotations/annotations.go -@@ -68,6 +68,9 @@ const ( - // AssetHashType is the hash type used for assets verification - AssetHashType = kataAnnotationsPrefix + "asset_hash_type" - -+ // StorageSpecTypeKey is the annotation key to fetch storage_spec -+ StorageSpecTypeKey = kataAnnotationsPrefix + "storage_spec" -+ - // - // Generic annotations - // -diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go -index 948bd3cb..d032227e 100644 ---- a/virtcontainers/pkg/oci/utils.go -+++ b/virtcontainers/pkg/oci/utils.go -@@ -22,6 +22,7 @@ import ( - exp "github.com/kata-containers/runtime/virtcontainers/experimental" - vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" - dockershimAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations/dockershim" -+ "github.com/kata-containers/runtime/virtcontainers/storage" - "github.com/kata-containers/runtime/virtcontainers/types" - "github.com/kata-containers/runtime/virtcontainers/utils" - specs "github.com/opencontainers/runtime-spec/specs-go" -@@ -340,6 +341,17 @@ func SandboxID(spec specs.Spec) (string, error) { - return "", fmt.Errorf("Could not find sandbox ID") - } - -+func validateStorageSpec(spec specs.Spec) error { -+ if storageSpec, ok := spec.Annotations[vcAnnotations.StorageSpecTypeKey]; ok { -+ err := storage.ValidateStorageValue(storageSpec) -+ if err != nil { -+ return err -+ } -+ } -+ -+ return nil -+} -+ - func addAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) error { - addAssetAnnotations(ocispec, config) - if err := addHypervisorConfigOverrides(ocispec, config); err != nil { -@@ -873,6 +885,10 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid, c - // ContainerConfig converts an OCI compatible runtime configuration - // file to a virtcontainers container configuration structure. - func ContainerConfig(ocispec specs.Spec, bundlePath, cid, console string, detach bool) (vc.ContainerConfig, error) { -+ err := validateStorageSpec(ocispec) -+ if err != nil { -+ return vc.ContainerConfig{}, err -+ } - rootfs := vc.RootFs{Target: ocispec.Root.Path, Mounted: true} - if !filepath.IsAbs(rootfs.Target) { - rootfs.Target = filepath.Join(bundlePath, ocispec.Root.Path) --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0042-kata-runtime-do-not-ignore-updateInterface-return-er.patch b/runtime/patches/0042-kata-runtime-do-not-ignore-updateInterface-return-er.patch deleted file mode 100644 index e5bc286..0000000 --- a/runtime/patches/0042-kata-runtime-do-not-ignore-updateInterface-return-er.patch +++ /dev/null @@ -1,38 +0,0 @@ -From 37372be961528471d418bbe066d88a010ad0513c Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Tue, 18 Aug 2020 22:12:59 +0800 -Subject: [PATCH 42/50] kata-runtime: do not ignore updateInterface return - error - -reason: If send UpdateInterfaceRequest to kata-agent and return the -error back, we should process it correctly, should not ignore the -error, which may casue deference nil pointer problem. - -Signed-off-by: jiangpengfei ---- - virtcontainers/kata_agent.go | 8 ++++++++ - 1 file changed, 8 insertions(+) - -diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go -index ac64817a..fee4215f 100644 ---- a/virtcontainers/kata_agent.go -+++ b/virtcontainers/kata_agent.go -@@ -604,7 +604,15 @@ func (k *kataAgent) updateInterface(ifc *vcTypes.Interface) (*vcTypes.Interface, - "interface-requested": fmt.Sprintf("%+v", ifc), - "resulting-interface": fmt.Sprintf("%+v", resultingInterface), - }).WithError(err).Error("update interface request failed") -+ return nil, err -+ } -+ -+ // need to judege resultingInterface is not not, otherwise may cause -+ // deference nil pointer panic problem -+ if resultingInterface == nil { -+ return nil, fmt.Errorf("resultingInterface should not be nil") - } -+ - if resultInterface, ok := resultingInterface.(*aTypes.Interface); ok { - iface := &vcTypes.Interface{ - Device: resultInterface.Device, --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0043-kata-runtime-support-add-hypervisor-global-parameter.patch b/runtime/patches/0043-kata-runtime-support-add-hypervisor-global-parameter.patch deleted file mode 100644 index 6543ee9..0000000 --- a/runtime/patches/0043-kata-runtime-support-add-hypervisor-global-parameter.patch +++ /dev/null @@ -1,162 +0,0 @@ -From d9738cfd6500ced30efde9e747eb73e5076e73ed Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Tue, 18 Aug 2020 22:47:13 +0800 -Subject: [PATCH 43/50] kata-runtime: support add hypervisor global parameters - in config file - -reason: support add hypervisor global parameters in config file -with defaultHypervisorParams config option. - -Signed-off-by: jiangpengfei ---- - Makefile | 2 ++ - cli/config/configuration-qemu.toml.in | 7 +++++++ - pkg/katautils/config-settings.go.in | 1 + - pkg/katautils/config.go | 13 ++++++++++++- - vendor/github.com/intel/govmm/qemu/qemu.go | 6 +++--- - virtcontainers/qemu.go | 2 +- - 6 files changed, 26 insertions(+), 5 deletions(-) - -diff --git a/Makefile b/Makefile -index d5e4bbe1..b62e64b0 100644 ---- a/Makefile -+++ b/Makefile -@@ -400,6 +400,7 @@ USER_VARS += FIRMWAREPATH - USER_VARS += MACHINEACCELERATORS - USER_VARS += DEFMACHINETYPE_CLH - USER_VARS += KERNELPARAMS -+USER_VARS += HYPERVISORPARAMS - USER_VARS += LIBEXECDIR - USER_VARS += LOCALSTATEDIR - USER_VARS += PKGDATADIR -@@ -607,6 +608,7 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit - -e "s|@FIRMWAREPATH_CLH@|$(FIRMWAREPATH_CLH)|g" \ - -e "s|@DEFMACHINETYPE_CLH@|$(DEFMACHINETYPE_CLH)|g" \ - -e "s|@KERNELPARAMS@|$(KERNELPARAMS)|g" \ -+ -e "s|@HYPERVISORPARAMS@|$(HYPERVISORPARAMS)|g" \ - -e "s|@LOCALSTATEDIR@|$(LOCALSTATEDIR)|g" \ - -e "s|@PKGLIBEXECDIR@|$(PKGLIBEXECDIR)|g" \ - -e "s|@PKGRUNDIR@|$(PKGRUNDIR)|g" \ -diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in -index aa11b38f..e57a954c 100644 ---- a/cli/config/configuration-qemu.toml.in -+++ b/cli/config/configuration-qemu.toml.in -@@ -29,6 +29,13 @@ machine_type = "@MACHINETYPE@" - # container and look for 'default-kernel-parameters' log entries. - kernel_params = "@KERNELPARAMS@" - -+# Optional space-separated list of options to pass to the hypervisor. -+# For example, use `hypervisor_params = "kvm-pit.lost_tick_policy=discard"` -+# -+# WARNING: - any parameter specified here will take priority over the default -+# parameter value of the same name used to start the hypervisor. -+hypervisor_params = "@HYPERVISORPARAMS@" -+ - # Path to the firmware. - # If you want that qemu uses the default firmware leave this option empty - firmware = "@FIRMWAREPATH@" -diff --git a/pkg/katautils/config-settings.go.in b/pkg/katautils/config-settings.go.in -index aaf78cc3..b2dfdfa1 100644 ---- a/pkg/katautils/config-settings.go.in -+++ b/pkg/katautils/config-settings.go.in -@@ -20,6 +20,7 @@ var defaultShimPath = "/usr/libexec/kata-containers/kata-shim" - var systemdUnitName = "kata-containers.target" - - const defaultKernelParams = "" -+const defaultHypervisorParams = "kvm-pit.lost_tick_policy=discard" - const defaultMachineType = "pc" - - const defaultVCPUCount uint32 = 1 -diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go -index 51120311..3365b3f5 100644 ---- a/pkg/katautils/config.go -+++ b/pkg/katautils/config.go -@@ -130,6 +130,7 @@ type hypervisor struct { - HotplugVFIOOnRootBus bool `toml:"hotplug_vfio_on_root_bus"` - DisableVhostNet bool `toml:"disable_vhost_net"` - GuestHookPath string `toml:"guest_hook_path"` -+ HypervisorParams string `toml:"hypervisor_params"` - } - - type proxy struct { -@@ -270,6 +271,14 @@ func (h hypervisor) kernelParams() string { - return h.KernelParams - } - -+func (h hypervisor) hypervisorParams() string { -+ if h.HypervisorParams == "" { -+ return defaultHypervisorParams -+ } -+ -+ return h.HypervisorParams -+} -+ - func (h hypervisor) machineType() string { - if h.MachineType == "" { - return defaultMachineType -@@ -632,6 +641,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { - - machineAccelerators := h.machineAccelerators() - kernelParams := h.kernelParams() -+ hypervisorParams := h.hypervisorParams() - machineType := h.machineType() - - blockDriver, err := h.blockDeviceDriver() -@@ -675,6 +685,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { - FirmwarePath: firmware, - MachineAccelerators: machineAccelerators, - KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), -+ HypervisorParams: vc.DeserializeParams(strings.Fields(hypervisorParams)), - HypervisorMachineType: machineType, - NumVCPUs: h.defaultVCPUs(), - DefaultMaxVCPUs: h.defaultMaxVCPUs(), -@@ -983,7 +994,7 @@ func updateRuntimeConfigAgent(configPath string, tomlConf tomlConfig, config *oc - TraceMode: agent.traceMode(), - TraceType: agent.traceType(), - KernelModules: agent.kernelModules(), -- MountBlkInVM: agent.mountBlkDevInVM(), -+ MountBlkInVM: agent.mountBlkDevInVM(), - } - default: - return fmt.Errorf("%s agent type is not supported", k) -diff --git a/vendor/github.com/intel/govmm/qemu/qemu.go b/vendor/github.com/intel/govmm/qemu/qemu.go -index 3e7720b4..68f8d2b0 100644 ---- a/vendor/github.com/intel/govmm/qemu/qemu.go -+++ b/vendor/github.com/intel/govmm/qemu/qemu.go -@@ -2096,7 +2096,7 @@ type Config struct { - SMP SMP - - // GlobalParam is the -global parameter. -- GlobalParam string -+ GlobalParam []string - - // Knobs is a set of qemu boolean settings. - Knobs Knobs -@@ -2285,9 +2285,9 @@ func (config *Config) appendRTC() { - } - - func (config *Config) appendGlobalParam() { -- if config.GlobalParam != "" { -+ for _, param := range config.GlobalParam { - config.qemuParams = append(config.qemuParams, "-global") -- config.qemuParams = append(config.qemuParams, config.GlobalParam) -+ config.qemuParams = append(config.qemuParams, param) - } - } - -diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go -index a10c66fb..657b6be7 100644 ---- a/virtcontainers/qemu.go -+++ b/virtcontainers/qemu.go -@@ -600,7 +600,7 @@ func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNa - Knobs: knobs, - Incoming: incoming, - VGA: "none", -- GlobalParam: "kvm-pit.lost_tick_policy=discard", -+ GlobalParam: SerializeParams(hypervisorConfig.HypervisorParams, "="), - Bios: firmwarePath, - PidFile: filepath.Join(q.store.RunVMStoragePath(), q.id, "pid"), - } --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0044-network-support-dpdk-vhost_user-net-device.patch b/runtime/patches/0044-network-support-dpdk-vhost_user-net-device.patch deleted file mode 100644 index cf800a7..0000000 --- a/runtime/patches/0044-network-support-dpdk-vhost_user-net-device.patch +++ /dev/null @@ -1,375 +0,0 @@ -From dbbf8c5deb14d8033b2e863fb0eb731523af2a47 Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Wed, 19 Aug 2020 09:58:07 +0800 -Subject: [PATCH 44/50] network: support dpdk vhost_user net device - -reason: support dpdk vhost_user net device - -Signed-off-by: jiangpengfei ---- - vendor/github.com/intel/govmm/qemu/qmp.go | 3 +- - virtcontainers/hypervisor.go | 3 ++ - virtcontainers/network.go | 49 ++++++++++++++------ - virtcontainers/pkg/types/types.go | 2 + - virtcontainers/qemu.go | 75 +++++++++++++++++++++++++++++++ - virtcontainers/sandbox.go | 7 ++- - virtcontainers/vhostuser_endpoint.go | 18 ++++++-- - virtcontainers/vhostuser_endpoint_test.go | 7 ++- - 8 files changed, 143 insertions(+), 21 deletions(-) - -diff --git a/vendor/github.com/intel/govmm/qemu/qmp.go b/vendor/github.com/intel/govmm/qemu/qmp.go -index a64039de..0cb82ffa 100644 ---- a/vendor/github.com/intel/govmm/qemu/qmp.go -+++ b/vendor/github.com/intel/govmm/qemu/qmp.go -@@ -970,11 +970,12 @@ func (q *QMP) ExecuteNetdevAdd(ctx context.Context, netdevType, netdevID, ifname - // ExecuteNetdevChardevAdd adds a Net device to a QEMU instance - // using the netdev_add command. netdevID is the id of the device to add. - // Must be valid QMP identifier. --func (q *QMP) ExecuteNetdevChardevAdd(ctx context.Context, netdevType, netdevID, chardev string, queues int) error { -+func (q *QMP) ExecuteNetdevChardevAdd(ctx context.Context, netdevType, netdevID, chardev string, vhostforce bool, queues int) error { - args := map[string]interface{}{ - "type": netdevType, - "id": netdevID, - "chardev": chardev, -+ "vhostforce": vhostforce, - } - if queues > 1 { - args["queues"] = queues -diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go -index c0723daa..60f1d190 100644 ---- a/virtcontainers/hypervisor.go -+++ b/virtcontainers/hypervisor.go -@@ -132,6 +132,9 @@ const ( - // vhostuserDev is a Vhost-user device type - vhostuserDev - -+ // vhostUserNetDev is a Vhost-user net device type -+ vhostUserNetDev -+ - // CPUDevice is CPU device type - cpuDev - -diff --git a/virtcontainers/network.go b/virtcontainers/network.go -index 488bd00c..a0c64356 100644 ---- a/virtcontainers/network.go -+++ b/virtcontainers/network.go -@@ -13,6 +13,7 @@ import ( - "math/rand" - "net" - "os" -+ "path/filepath" - "regexp" - "runtime" - "sort" -@@ -65,7 +66,8 @@ const ( - ) - - var ( -- regInfName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_\-.]*$`) -+ regInfName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_\-.]*$`) -+ regVhostName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9\-.]{0,127}$`) - ) - - //IsValid checks if a model is valid -@@ -132,11 +134,12 @@ type NetlinkIface struct { - // NetworkInfo gathers all information related to a network interface. - // It can be used to store the description of the underlying network. - type NetworkInfo struct { -- Device string -- Iface NetlinkIface -- Addrs []netlink.Addr -- Routes []netlink.Route -- DNS DNSInfo -+ Device string -+ Iface NetlinkIface -+ Addrs []netlink.Addr -+ Routes []netlink.Route -+ DNS DNSInfo -+ VhostUserSocket string - } - - // NetworkInterface defines a network interface. -@@ -1198,15 +1201,11 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li - return endpoint, err - } - -- // Check if this is a dummy interface which has a vhost-user socket associated with it -- socketPath, err := vhostUserSocketPath(netInfo) -- if err != nil { -- return nil, err -- } -- -- if socketPath != "" { -+ // Currently, we only accept the vhost-user socket paas from user input -+ // interface info json file -+ if netInfo.VhostUserSocket != "" { - networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found") -- endpoint, err = createVhostUserEndpoint(netInfo, socketPath) -+ endpoint, err = createVhostUserEndpoint(netInfo, netInfo.VhostUserSocket) - return endpoint, err - } - -@@ -1481,6 +1480,24 @@ func verifyIP(ip string) (*net.IP, error) { - return &netIP, nil - } - -+// verifyVhostSocket verifies the vhost socket is valid or not -+func verifyVhostSocket(vhostSocket string) error { -+ if vhostSocket == "" { -+ networkLogger().Debug("no dpdk network") -+ return nil -+ } -+ if regVhostName.FindAllString(filepath.Base(vhostSocket), -1) == nil { -+ return fmt.Errorf("invalid input of vhostSocket name, please check the rules") -+ } -+ info, err := os.Stat(vhostSocket) -+ if err != nil || info.IsDir() || info.Mode()&os.ModeSocket == 0 { -+ return fmt.Errorf("invalid vhost socket: %v", vhostSocket) -+ } -+ -+ networkLogger().WithField("vhostSocket", vhostSocket).Infof("using dpdk network, socket file is: %s ", vhostSocket) -+ return nil -+} -+ - // validInterface check the input interface valid or not - func validInterface(inf *vcTypes.Interface, enableCompatOldCNI bool) error { - if enableCompatOldCNI && verifyInterfaceName(inf.Device) != nil { -@@ -1503,6 +1520,10 @@ func validInterface(inf *vcTypes.Interface, enableCompatOldCNI bool) error { - return err - } - -+ if err := verifyVhostSocket(inf.VhostUserSocket); err != nil { -+ return err -+ } -+ - // Currently, only one IP address can be passed, which reduces the test entry and fault injection. - if len(inf.IPAddresses) > 0 { - if len(inf.IPAddresses) != 1 { -diff --git a/virtcontainers/pkg/types/types.go b/virtcontainers/pkg/types/types.go -index b41b0c75..dccd92f8 100644 ---- a/virtcontainers/pkg/types/types.go -+++ b/virtcontainers/pkg/types/types.go -@@ -29,6 +29,8 @@ type Interface struct { - // library, regarding each type of link. Here is a non exhaustive - // list: "veth", "macvtap", "vlan", "macvlan", "tap", ... - LinkType string `json:"linkType,omitempty"` -+ // VhostUserSocket is DPDK-backed vHost user ports. -+ VhostUserSocket string `json:"vhostUserSocket,omitempty"` - } - - // Route describes a network route. -diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go -index 657b6be7..84797e0d 100644 ---- a/virtcontainers/qemu.go -+++ b/virtcontainers/qemu.go -@@ -1279,6 +1279,78 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error - return err - } - -+func (q *qemu) hotplugVhostUserNetDevice(endpoint Endpoint, op operation) error { -+ err := q.qmpSetup() -+ if err != nil { -+ return err -+ } -+ -+ device := endpoint.(*VhostUserEndpoint) -+ -+ charDevID := utils.MakeNameID("char", device.ID, maxDevIDSize) -+ devID := utils.MakeNameID("virtio", device.ID, maxDevIDSize) -+ -+ if op == addDevice { -+ // Add chardev specifying the appropriate socket. -+ if err = q.qmpMonitorCh.qmp.ExecuteCharDevUnixSocketAdd(q.qmpMonitorCh.ctx, charDevID, device.SocketPath, true, false); err != nil { -+ return err -+ } -+ -+ defer func() { -+ if err != nil { -+ errRb := q.qmpMonitorCh.qmp.ExecuteChardevDel(q.qmpMonitorCh.ctx, charDevID) -+ if errRb != nil { -+ q.Logger().Errorf("deletes a net device roll back failed. charDev id:%s", charDevID) -+ } -+ } -+ }() -+ -+ netDevType := "vhost-user" -+ // Add netdev of type vhost-user. -+ if err = q.qmpMonitorCh.qmp.ExecuteNetdevChardevAdd(q.qmpMonitorCh.ctx, netDevType, devID, charDevID, true, 0); err != nil { -+ return err -+ } -+ -+ defer func() { -+ if err != nil { -+ errRb := q.qmpMonitorCh.qmp.ExecuteNetdevDel(q.qmpMonitorCh.ctx, devID) -+ if errRb != nil { -+ q.Logger().Errorf("deletes a net device roll back failed. netDev id:%s", devID) -+ } -+ } -+ }() -+ -+ var addr, bus, pciAddr string -+ addr, bus, pciAddr, err = q.getPciAddress(device.ID, types.PCI) -+ if err != nil { -+ return nil -+ } -+ defer func() { -+ if err != nil { -+ q.putPciAddress(device.ID) -+ } -+ }() -+ -+ endpoint.SetPciAddr(pciAddr) -+ -+ // Add virtio-net-pci device. -+ err = q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, devID, devID, device.HardAddr, addr, bus, romFile, 0, q.arch.runNested()) -+ return err -+ } -+ -+ if err = q.putPciAddress(device.ID); err != nil { -+ return err -+ } -+ if err = q.qmpMonitorCh.qmp.ExecuteDeviceDel(q.qmpMonitorCh.ctx, devID); err != nil { -+ return err -+ } -+ if err = q.qmpMonitorCh.qmp.ExecuteNetdevDel(q.qmpMonitorCh.ctx, devID); err != nil { -+ return err -+ } -+ -+ return nil -+} -+ - func (q *qemu) hotplugVhostUserDevice(vAttr *config.VhostUserDeviceAttrs, op operation) error { - err := q.qmpSetup() - if err != nil { -@@ -1510,6 +1582,9 @@ func (q *qemu) hotplugDevice(devInfo interface{}, devType deviceType, op operati - case vhostuserDev: - vAttr := devInfo.(*config.VhostUserDeviceAttrs) - return nil, q.hotplugVhostUserDevice(vAttr, op) -+ case vhostUserNetDev: -+ device := devInfo.(Endpoint) -+ return nil, q.hotplugVhostUserNetDevice(device, op) - default: - return nil, fmt.Errorf("cannot hotplug device: unsupported device type '%v'", devType) - } -diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go -index 0d599267..e6f155a3 100644 ---- a/virtcontainers/sandbox.go -+++ b/virtcontainers/sandbox.go -@@ -946,7 +946,8 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) { - }, - Type: inf.LinkType, - }, -- Addrs: addrs, -+ Addrs: addrs, -+ VhostUserSocket: inf.VhostUserSocket, - }, nil - } - -@@ -961,6 +962,10 @@ func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (grpcIf *vcTypes.Interfac - if ep.Name() == inf.Name { - return nil, fmt.Errorf("interface %s is already exist", inf.Name) - } -+ -+ if ep.Properties().VhostUserSocket != "" && inf.VhostUserSocket != "" { -+ return nil, fmt.Errorf("sandbox %s only support one dpdk socket", s.ID()) -+ } - } - - netInfo, err := s.generateNetInfo(inf) -diff --git a/virtcontainers/vhostuser_endpoint.go b/virtcontainers/vhostuser_endpoint.go -index bb4a67be..2fc3d837 100644 ---- a/virtcontainers/vhostuser_endpoint.go -+++ b/virtcontainers/vhostuser_endpoint.go -@@ -12,6 +12,7 @@ import ( - - "github.com/kata-containers/runtime/virtcontainers/device/config" - persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" -+ "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" - "github.com/kata-containers/runtime/virtcontainers/utils" - ) - -@@ -23,6 +24,7 @@ const hostSocketSearchPath = "/tmp/vhostuser_%s/vhu.sock" - - // VhostUserEndpoint represents a vhost-user socket based network interface - type VhostUserEndpoint struct { -+ ID string - // Path to the vhost-user socket on the host system - SocketPath string - // MAC address of the interface -@@ -99,18 +101,28 @@ func (endpoint *VhostUserEndpoint) Detach(netNsCreated bool, netNsPath string) e - - // HotAttach for vhostuser endpoint not supported yet - func (endpoint *VhostUserEndpoint) HotAttach(h hypervisor) error { -- return fmt.Errorf("VhostUserEndpoint does not support Hot attach") -+ networkLogger().Debug("Hot attaching vhost-user endpoint") -+ if _, err := h.hotplugAddDevice(endpoint, vhostUserNetDev); err != nil { -+ return err -+ } -+ return nil - } - - // HotDetach for vhostuser endpoint not supported yet - func (endpoint *VhostUserEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { -- return fmt.Errorf("VhostUserEndpoint does not support Hot detach") -+ networkLogger().Debug("Hot detaching vhost-user endpoint") -+ if _, err := h.hotplugRemoveDevice(endpoint, vhostUserNetDev); err != nil { -+ networkLogger().WithError(err).Errorf("Error detach vhostUserSocket") -+ return err -+ } -+ return nil - } - - // Create a vhostuser endpoint - func createVhostUserEndpoint(netInfo NetworkInfo, socket string) (*VhostUserEndpoint, error) { -- -+ uniqueID := uuid.Generate().String() - vhostUserEndpoint := &VhostUserEndpoint{ -+ ID: uniqueID, - SocketPath: socket, - HardAddr: netInfo.Iface.HardwareAddr.String(), - IfaceName: netInfo.Iface.Name, -diff --git a/virtcontainers/vhostuser_endpoint_test.go b/virtcontainers/vhostuser_endpoint_test.go -index ad013e12..584490cc 100644 ---- a/virtcontainers/vhostuser_endpoint_test.go -+++ b/virtcontainers/vhostuser_endpoint_test.go -@@ -11,6 +11,7 @@ import ( - "os" - "testing" - -+ "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" - "github.com/stretchr/testify/assert" - "github.com/vishvananda/netlink" - ) -@@ -95,7 +96,7 @@ func TestVhostUserEndpoint_HotAttach(t *testing.T) { - h := &mockHypervisor{} - - err := v.HotAttach(h) -- assert.Error(err) -+ assert.NoError(err) - } - - func TestVhostUserEndpoint_HotDetach(t *testing.T) { -@@ -109,10 +110,11 @@ func TestVhostUserEndpoint_HotDetach(t *testing.T) { - h := &mockHypervisor{} - - err := v.HotDetach(h, true, "") -- assert.Error(err) -+ assert.NoError(err) - } - - func TestCreateVhostUserEndpoint(t *testing.T) { -+ uniqueID := uuid.Generate().String() - macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x48} - ifcName := "vhost-deadbeef" - socket := "/tmp/vhu_192.168.0.1" -@@ -128,6 +130,7 @@ func TestCreateVhostUserEndpoint(t *testing.T) { - } - - expected := &VhostUserEndpoint{ -+ ID: uniqueID, - SocketPath: socket, - HardAddr: macAddr.String(), - IfaceName: ifcName, --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0045-network-support-set-dns.patch b/runtime/patches/0045-network-support-set-dns.patch deleted file mode 100644 index 2d039ff..0000000 --- a/runtime/patches/0045-network-support-set-dns.patch +++ /dev/null @@ -1,115 +0,0 @@ -From e104f084ed4f10049f45d9473faed229371a1c6c Mon Sep 17 00:00:00 2001 -From: holyfei -Date: Wed, 19 Aug 2020 09:59:52 +0800 -Subject: [PATCH 45/50] network: support set dns - -reason: when running a sandbox with annotation about -dns spec, overload k.getDNS(sandbox) - -Signed-off-by: yangfeiyu ---- - virtcontainers/kata_agent.go | 11 +++++++++++ - virtcontainers/network.go | 8 ++++---- - virtcontainers/pkg/annotations/annotations.go | 3 +++ - virtcontainers/pkg/oci/utils.go | 19 +++++++++++++++++-- - 4 files changed, 35 insertions(+), 6 deletions(-) - -diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go -index fee4215f..d28d8cce 100644 ---- a/virtcontainers/kata_agent.go -+++ b/virtcontainers/kata_agent.go -@@ -890,6 +890,17 @@ func (k *kataAgent) startSandbox(sandbox *Sandbox) error { - KernelModules: kmodules, - } - -+ if value, ok := sandbox.config.Annotations[vcAnnotations.SandboxDNSTypeKey]; ok { -+ var sandboxDns *DNSInfo -+ if err := json.Unmarshal([]byte(value), &sandboxDns); err != nil { -+ return fmt.Errorf("get sandbox dns failed %v", err) -+ } -+ -+ if sandboxDns != nil { -+ req.Dns = sandboxDns.Servers -+ } -+ } -+ - _, err = k.sendReq(req) - if err != nil { - return err -diff --git a/virtcontainers/network.go b/virtcontainers/network.go -index 488bd00c..68eda4a6 100644 ---- a/virtcontainers/network.go -+++ b/virtcontainers/network.go -@@ -117,10 +117,10 @@ const ( - - // DNSInfo describes the DNS setup related to a network interface. - type DNSInfo struct { -- Servers []string -- Domain string -- Searches []string -- Options []string -+ Servers []string `json:"Servers"` -+ Domain string `json:"Domain,omitempty"` -+ Searches []string `json:"Searches,omitempty"` -+ Options []string `json:"Options,omitempty"` - } - - // NetlinkIface describes fully a network interface. -diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go -index e50a697c..528dfa66 100644 ---- a/virtcontainers/pkg/annotations/annotations.go -+++ b/virtcontainers/pkg/annotations/annotations.go -@@ -71,6 +71,9 @@ const ( - // StorageSpecTypeKey is the annotation key to fetch storage_spec - StorageSpecTypeKey = kataAnnotationsPrefix + "storage_spec" - -+ // SandboxDNSTypeKey is the annotation key to fetch sandbox dns options -+ SandboxDNSTypeKey = kataAnnotationsPrefix + "sandbox_dns" -+ - // - // Generic annotations - // -diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go -index d032227e..3b2af753 100644 ---- a/virtcontainers/pkg/oci/utils.go -+++ b/virtcontainers/pkg/oci/utils.go -@@ -7,6 +7,7 @@ package oci - - import ( - "context" -+ "encoding/json" - "errors" - "fmt" - "path/filepath" -@@ -37,8 +38,9 @@ type annotationContainerType struct { - type annotationHandler func(value string) error - - var annotationHandlerList = map[string]annotationHandler{ -- vcAnnotations.StaticCPUTypeKey: validateSandboxCPU, -- vcAnnotations.StaticMemTypeKey: validateSandboxMem, -+ vcAnnotations.StaticCPUTypeKey: validateSandboxCPU, -+ vcAnnotations.StaticMemTypeKey: validateSandboxMem, -+ vcAnnotations.SandboxDNSTypeKey: validateSandboxDNS, - } - - var ( -@@ -1108,3 +1110,16 @@ func validateSandboxMem(value string) error { - - return nil - } -+ -+func validateSandboxDNS(value string) error { -+ if value == "" { -+ return fmt.Errorf("--annotation %s value should not be empty", vcAnnotations.SandboxDNSTypeKey) -+ } -+ -+ var sandboxDns *vc.DNSInfo -+ if err := json.Unmarshal([]byte(value), &sandboxDns); err != nil { -+ return fmt.Errorf("invalid value passed by --annotation %s, error: %v", vcAnnotations.SandboxDNSTypeKey, err) -+ } -+ -+ return nil -+} --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0046-network-support-multiqueue-when-inserting-interface-.patch b/runtime/patches/0046-network-support-multiqueue-when-inserting-interface-.patch deleted file mode 100644 index ac07aec..0000000 --- a/runtime/patches/0046-network-support-multiqueue-when-inserting-interface-.patch +++ /dev/null @@ -1,489 +0,0 @@ -From e9c1fb235d0e69d5db72497a8779df707c499d3b Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Wed, 19 Aug 2020 20:36:40 +0800 -Subject: [PATCH 46/50] network: support multiqueue when inserting interface - dynamically - -reason: support multiqueue when inserting interface dynamically - -Signed-off-by: jiangpengfei ---- - .../kata-containers/agent/pkg/types/types.pb.go | 82 ++++++++++++++++------ - virtcontainers/bridgedmacvlan_endpoint.go | 2 +- - virtcontainers/ipvlan_endpoint.go | 2 +- - virtcontainers/kata_agent.go | 2 + - virtcontainers/network.go | 13 ++-- - virtcontainers/network_test.go | 2 +- - virtcontainers/pkg/types/types.go | 2 + - virtcontainers/qemu.go | 10 +-- - virtcontainers/sandbox.go | 1 + - virtcontainers/tap_endpoint.go | 6 +- - virtcontainers/tuntap_endpoint.go | 4 +- - virtcontainers/veth_endpoint.go | 4 +- - virtcontainers/veth_endpoint_test.go | 6 +- - virtcontainers/vhostuser_endpoint.go | 4 +- - virtcontainers/vhostuser_endpoint_test.go | 2 +- - 15 files changed, 96 insertions(+), 46 deletions(-) - -diff --git a/vendor/github.com/kata-containers/agent/pkg/types/types.pb.go b/vendor/github.com/kata-containers/agent/pkg/types/types.pb.go -index 7ea63e3c..8b7e2a5d 100644 ---- a/vendor/github.com/kata-containers/agent/pkg/types/types.pb.go -+++ b/vendor/github.com/kata-containers/agent/pkg/types/types.pb.go -@@ -100,6 +100,7 @@ type Interface struct { - // list: "veth", "macvtap", "vlan", "macvlan", "tap", ... - Type string `protobuf:"bytes,7,opt,name=type,proto3" json:"type,omitempty"` - RawFlags uint32 `protobuf:"varint,8,opt,name=raw_flags,json=rawFlags,proto3" json:"raw_flags,omitempty"` -+ Queues uint32 `protobuf:"varint,9,opt,name=Queues,proto3" json:"Queues,omitempty"` - } - - func (m *Interface) Reset() { *m = Interface{} } -@@ -163,6 +164,13 @@ func (m *Interface) GetRawFlags() uint32 { - return 0 - } - -+func (m *Interface) GetQueues() uint32 { -+ if m != nil { -+ return m.Queues -+ } -+ return 0 -+} -+ - type Route struct { - Dest string `protobuf:"bytes,1,opt,name=dest,proto3" json:"dest,omitempty"` - Gateway string `protobuf:"bytes,2,opt,name=gateway,proto3" json:"gateway,omitempty"` -@@ -319,6 +327,11 @@ func (m *Interface) MarshalTo(dAtA []byte) (int, error) { - i++ - i = encodeVarintTypes(dAtA, i, uint64(m.RawFlags)) - } -+ if m.Queues != 0 { -+ dAtA[i] = 0x48 -+ i++ -+ i = encodeVarintTypes(dAtA, i, uint64(m.Queues)) -+ } - return i, nil - } - -@@ -430,6 +443,9 @@ func (m *Interface) Size() (n int) { - if m.RawFlags != 0 { - n += 1 + sovTypes(uint64(m.RawFlags)) - } -+ if m.Queues != 0 { -+ n += 1 + sovTypes(uint64(m.Queues)) -+ } - return n - } - -@@ -841,6 +857,25 @@ func (m *Interface) Unmarshal(dAtA []byte) error { - break - } - } -+ case 9: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field Queues", wireType) -+ } -+ m.Queues = 0 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowTypes -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ m.Queues |= (uint32(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } - default: - iNdEx = preIndex - skippy, err := skipTypes(dAtA[iNdEx:]) -@@ -1155,27 +1190,28 @@ var ( - func init() { proto.RegisterFile("pkg/types/types.proto", fileDescriptorTypes) } - - var fileDescriptorTypes = []byte{ -- // 345 bytes of a gzipped FileDescriptorProto -- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x91, 0xd1, 0x6a, 0xfa, 0x30, -- 0x14, 0xc6, 0xff, 0xb1, 0xb6, 0xda, 0xe3, 0xdf, 0xad, 0x84, 0x4d, 0xc2, 0x06, 0x52, 0xbc, 0x59, -- 0xd9, 0x85, 0x03, 0x37, 0x76, 0xbf, 0x5d, 0x08, 0xde, 0x49, 0x5e, 0x60, 0xc4, 0x36, 0x3a, 0xd1, -- 0xda, 0xd2, 0x44, 0x8b, 0xec, 0x05, 0x77, 0xb9, 0x47, 0x10, 0x9f, 0x64, 0xe4, 0x24, 0x8a, 0xbb, -- 0xd1, 0xef, 0x97, 0x93, 0xd3, 0xef, 0x7c, 0x27, 0x70, 0x5b, 0xae, 0x16, 0x4f, 0x7a, 0x5f, 0x4a, -- 0x65, 0x7f, 0x87, 0x65, 0x55, 0xe8, 0x82, 0xfa, 0x08, 0x83, 0x19, 0x84, 0x93, 0xe9, 0x5b, 0x96, -- 0x55, 0x52, 0x29, 0xfa, 0x00, 0xc1, 0x5c, 0xe4, 0xcb, 0xf5, 0x9e, 0x91, 0x98, 0x24, 0x57, 0xa3, -- 0xeb, 0xa1, 0xed, 0x98, 0x4c, 0xc7, 0x78, 0xcc, 0x5d, 0x99, 0x32, 0x68, 0x09, 0xdb, 0xc3, 0x1a, -- 0x31, 0x49, 0x42, 0x7e, 0x42, 0x4a, 0xa1, 0x99, 0x0b, 0xb5, 0x62, 0x1e, 0x1e, 0xa3, 0x1e, 0x1c, -- 0x08, 0x84, 0x93, 0x8d, 0x96, 0xd5, 0x5c, 0xa4, 0x92, 0xf6, 0x20, 0xc8, 0xe4, 0x6e, 0x99, 0x4a, -- 0x34, 0x09, 0xb9, 0x23, 0xd3, 0xb9, 0x11, 0xb9, 0x74, 0x1f, 0x44, 0x4d, 0x47, 0xd0, 0x39, 0x4f, -- 0x27, 0x15, 0xf3, 0x62, 0x2f, 0xe9, 0x8c, 0xa2, 0xf3, 0x54, 0xae, 0xc2, 0x2f, 0x2f, 0xd1, 0x08, -- 0xbc, 0x5c, 0x6f, 0x59, 0x33, 0x26, 0x49, 0x93, 0x1b, 0x69, 0x1c, 0x3f, 0x6b, 0x73, 0x81, 0xf9, -- 0xd6, 0xd1, 0x92, 0x49, 0x51, 0xa6, 0x4b, 0x2c, 0x04, 0x36, 0x85, 0x43, 0x33, 0x8b, 0xf1, 0x60, -- 0x2d, 0x3b, 0x8b, 0xd1, 0xf4, 0x1e, 0xc2, 0x4a, 0xd4, 0x1f, 0xf3, 0xb5, 0x58, 0x28, 0xd6, 0x8e, -- 0x49, 0xd2, 0xe5, 0xed, 0x4a, 0xd4, 0x63, 0xc3, 0x83, 0x2f, 0xf0, 0x79, 0xb1, 0xd5, 0x98, 0x22, -- 0x93, 0x4a, 0xbb, 0x6c, 0xa8, 0x8d, 0xcf, 0x42, 0x68, 0x59, 0x8b, 0xfd, 0x69, 0x5b, 0x0e, 0x2f, -- 0x76, 0xe1, 0xfd, 0xd9, 0x45, 0x0f, 0x02, 0x55, 0x6c, 0xab, 0x54, 0x62, 0x8c, 0x90, 0x3b, 0xa2, -- 0x37, 0xe0, 0xab, 0xb4, 0x28, 0x25, 0x06, 0xe9, 0x72, 0x0b, 0x8f, 0x77, 0xd0, 0x3e, 0xbd, 0x10, -- 0x0d, 0xa0, 0xb1, 0x7b, 0x89, 0xfe, 0xe1, 0xff, 0x6b, 0x44, 0xde, 0xff, 0x7f, 0x1f, 0xfb, 0xe4, -- 0xe7, 0xd8, 0x27, 0x87, 0x63, 0x9f, 0xcc, 0x02, 0x7c, 0xfb, 0xe7, 0xdf, 0x00, 0x00, 0x00, 0xff, -- 0xff, 0xb5, 0x52, 0x37, 0xf2, 0x14, 0x02, 0x00, 0x00, -+ // 356 bytes of a gzipped FileDescriptorProto -+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x92, 0xc1, 0x4e, 0xc2, 0x40, -+ 0x10, 0x86, 0x5d, 0x4a, 0x0b, 0x1d, 0x44, 0x9b, 0x8d, 0x92, 0x8d, 0x26, 0xa4, 0xe1, 0x62, 0xe3, -+ 0x01, 0x13, 0x34, 0xde, 0xf5, 0x40, 0xc2, 0x0d, 0xf7, 0x05, 0xcc, 0xd2, 0x2e, 0x48, 0xa0, 0xb4, -+ 0xe9, 0xb6, 0x34, 0xc4, 0x17, 0xf4, 0xe8, 0x23, 0x18, 0x9e, 0xc2, 0xa3, 0xd9, 0xd9, 0x85, 0xe0, -+ 0x05, 0xfe, 0x6f, 0x67, 0xa7, 0xff, 0xfc, 0xd3, 0xc2, 0x75, 0xbe, 0x5a, 0x3c, 0x94, 0xbb, 0x5c, -+ 0x2a, 0xf3, 0x3b, 0xcc, 0x8b, 0xac, 0xcc, 0xa8, 0x8b, 0x30, 0x98, 0x81, 0x3f, 0x99, 0xbe, 0x24, -+ 0x49, 0x21, 0x95, 0xa2, 0x77, 0xe0, 0xcd, 0x45, 0xba, 0x5c, 0xef, 0x18, 0x09, 0x49, 0x74, 0x31, -+ 0xba, 0x1c, 0x9a, 0x8e, 0xc9, 0x74, 0x8c, 0xc7, 0xdc, 0x96, 0x29, 0x83, 0x96, 0x30, 0x3d, 0xac, -+ 0x11, 0x92, 0xc8, 0xe7, 0x07, 0xa4, 0x14, 0x9a, 0xa9, 0x50, 0x2b, 0xe6, 0xe0, 0x31, 0xea, 0xc1, -+ 0x2f, 0x01, 0x7f, 0xb2, 0x29, 0x65, 0x31, 0x17, 0xb1, 0xa4, 0x3d, 0xf0, 0x12, 0xb9, 0x5d, 0xc6, -+ 0x12, 0x4d, 0x7c, 0x6e, 0x49, 0x77, 0x6e, 0x44, 0x2a, 0xed, 0x03, 0x51, 0xd3, 0x11, 0x74, 0x8e, -+ 0xd3, 0x49, 0xc5, 0x9c, 0xd0, 0x89, 0x3a, 0xa3, 0xe0, 0x38, 0x95, 0xad, 0xf0, 0xd3, 0x4b, 0x34, -+ 0x00, 0x27, 0x2d, 0x2b, 0xd6, 0x0c, 0x49, 0xd4, 0xe4, 0x5a, 0x6a, 0xc7, 0x8f, 0x5a, 0x5f, 0x60, -+ 0xae, 0x71, 0x34, 0xa4, 0x53, 0xe4, 0xf1, 0x12, 0x0b, 0x9e, 0x49, 0x61, 0x51, 0xcf, 0xa2, 0x3d, -+ 0x58, 0xcb, 0xcc, 0xa2, 0x35, 0xbd, 0x05, 0xbf, 0x10, 0xf5, 0xfb, 0x7c, 0x2d, 0x16, 0x8a, 0xb5, -+ 0x43, 0x12, 0x75, 0x79, 0xbb, 0x10, 0xf5, 0x58, 0xb3, 0xb6, 0x78, 0xab, 0x64, 0x25, 0x15, 0xf3, -+ 0xb1, 0x62, 0x69, 0xf0, 0x09, 0x2e, 0xcf, 0xaa, 0x12, 0xd3, 0x25, 0x52, 0x95, 0x36, 0x33, 0x6a, -+ 0xed, 0xbf, 0x10, 0xa5, 0xac, 0xc5, 0xee, 0xb0, 0x45, 0x8b, 0x27, 0x3b, 0x72, 0xfe, 0xed, 0xa8, -+ 0x07, 0x9e, 0xca, 0xaa, 0x22, 0x96, 0x18, 0xcf, 0xe7, 0x96, 0xe8, 0x15, 0xb8, 0x2a, 0xce, 0x72, -+ 0x89, 0x01, 0xbb, 0xdc, 0xc0, 0xfd, 0x0d, 0xb4, 0x0f, 0x6f, 0x8e, 0x7a, 0xd0, 0xd8, 0x3e, 0x05, -+ 0x67, 0xf8, 0xff, 0x1c, 0x90, 0xd7, 0xf3, 0xaf, 0x7d, 0x9f, 0x7c, 0xef, 0xfb, 0xe4, 0x67, 0xdf, -+ 0x27, 0x33, 0x0f, 0xbf, 0x89, 0xc7, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa5, 0x57, 0x70, 0xc8, -+ 0x2c, 0x02, 0x00, 0x00, - } -diff --git a/virtcontainers/bridgedmacvlan_endpoint.go b/virtcontainers/bridgedmacvlan_endpoint.go -index 700aea34..9179bbbe 100644 ---- a/virtcontainers/bridgedmacvlan_endpoint.go -+++ b/virtcontainers/bridgedmacvlan_endpoint.go -@@ -25,7 +25,7 @@ func createBridgedMacvlanNetworkEndpoint(idx int, ifName string, interworkingMod - return &BridgedMacvlanEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx) - } - -- netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel) -+ netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel, 0) - if err != nil { - return nil, err - } -diff --git a/virtcontainers/ipvlan_endpoint.go b/virtcontainers/ipvlan_endpoint.go -index 6e924a74..a6ef8179 100644 ---- a/virtcontainers/ipvlan_endpoint.go -+++ b/virtcontainers/ipvlan_endpoint.go -@@ -28,7 +28,7 @@ func createIPVlanNetworkEndpoint(idx int, ifName string) (*IPVlanEndpoint, error - // Use tc filtering for ipvlan, since the other inter networking models will - // not work for ipvlan. - interworkingModel := NetXConnectTCFilterModel -- netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel) -+ netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel, 0) - if err != nil { - return nil, err - } -diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go -index d28d8cce..38e9a204 100644 ---- a/virtcontainers/kata_agent.go -+++ b/virtcontainers/kata_agent.go -@@ -621,6 +621,7 @@ func (k *kataAgent) updateInterface(ifc *vcTypes.Interface) (*vcTypes.Interface, - Mtu: resultInterface.Mtu, - HwAddr: resultInterface.HwAddr, - PciAddr: resultInterface.PciAddr, -+ Queues: resultInterface.Queues, - } - return iface, err - } -@@ -2295,6 +2296,7 @@ func (k *kataAgent) convertToKataAgentInterface(iface *vcTypes.Interface) *aType - RawFlags: iface.RawFlags, - HwAddr: iface.HwAddr, - PciAddr: iface.PciAddr, -+ Queues: iface.Queues, - } - } - -diff --git a/virtcontainers/network.go b/virtcontainers/network.go -index 46703656..15eb7906 100644 ---- a/virtcontainers/network.go -+++ b/virtcontainers/network.go -@@ -140,6 +140,7 @@ type NetworkInfo struct { - Routes []netlink.Route - DNS DNSInfo - VhostUserSocket string -+ Queues uint32 - } - - // NetworkInterface defines a network interface. -@@ -156,6 +157,7 @@ type TapInterface struct { - TAPIface NetworkInterface - VMFds []*os.File - VhostFds []*os.File -+ Queues uint32 - } - - // TuntapInterface defines a tap interface -@@ -1047,7 +1049,7 @@ func generateInterfacesAndRoutes(networkNS NetworkNamespace) ([]*vcTypes.Interfa - return ifaces, routes, nil - } - --func createNetworkInterfacePair(idx int, ifName string, interworkingModel NetInterworkingModel) (NetworkInterfacePair, error) { -+func createNetworkInterfacePair(idx int, ifName string, interworkingModel NetInterworkingModel, queues uint32) (NetworkInterfacePair, error) { - uniqueID := uuid.Generate().String() - - randomMacAddr, err := generateRandomPrivateMacAddr() -@@ -1062,6 +1064,7 @@ func createNetworkInterfacePair(idx int, ifName string, interworkingModel NetInt - TAPIface: NetworkInterface{ - Name: fmt.Sprintf("tap%d_kata", idx), - }, -+ Queues: queues, - }, - VirtIface: NetworkInterface{ - Name: fmt.Sprintf("eth%d", idx), -@@ -1205,7 +1208,7 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li - // interface info json file - if netInfo.VhostUserSocket != "" { - networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found") -- endpoint, err = createVhostUserEndpoint(netInfo, netInfo.VhostUserSocket) -+ endpoint, err = createVhostUserEndpoint(netInfo, netInfo.VhostUserSocket, netInfo.Queues) - return endpoint, err - } - -@@ -1220,7 +1223,7 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li - endpoint, err = createMacvtapNetworkEndpoint(netInfo) - case "tap": - networkLogger().Info("tap interface found") -- endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Device) -+ endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Device, netInfo.Queues) - case "tuntap": - if link != nil { - switch link.(*netlink.Tuntap).Mode { -@@ -1231,13 +1234,13 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li - return nil, fmt.Errorf("tun networking device not yet supported") - case 2: - networkLogger().Info("tuntap tap interface found") -- endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, model) -+ endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, model, netInfo.Queues) - default: - return nil, fmt.Errorf("tuntap network %v mode unsupported", link.(*netlink.Tuntap).Mode) - } - } - case "veth": -- endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model) -+ endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model, netInfo.Queues) - case "ipvlan": - endpoint, err = createIPVlanNetworkEndpoint(idx, netInfo.Iface.Name) - default: -diff --git a/virtcontainers/network_test.go b/virtcontainers/network_test.go -index b86f679f..c52c7452 100644 ---- a/virtcontainers/network_test.go -+++ b/virtcontainers/network_test.go -@@ -265,7 +265,7 @@ func TestTcRedirectNetwork(t *testing.T) { - err = netlink.LinkAdd(veth) - assert.NoError(err) - -- endpoint, err := createVethNetworkEndpoint(1, vethName, NetXConnectTCFilterModel) -+ endpoint, err := createVethNetworkEndpoint(1, vethName, NetXConnectTCFilterModel, 0) - assert.NoError(err) - - link, err := netlink.LinkByName(vethName) -diff --git a/virtcontainers/pkg/types/types.go b/virtcontainers/pkg/types/types.go -index dccd92f8..6502259b 100644 ---- a/virtcontainers/pkg/types/types.go -+++ b/virtcontainers/pkg/types/types.go -@@ -31,6 +31,8 @@ type Interface struct { - LinkType string `json:"linkType,omitempty"` - // VhostUserSocket is DPDK-backed vHost user ports. - VhostUserSocket string `json:"vhostUserSocket,omitempty"` -+ // Queues specifies the interface queue number -+ Queues uint32 `json:"queues,omitempty"` - } - - // Route describes a network route. -diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go -index 84797e0d..be6e33b9 100644 ---- a/virtcontainers/qemu.go -+++ b/virtcontainers/qemu.go -@@ -1464,7 +1464,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro - return nil - } - --func (q *qemu) hotAddNetDevice(deviceName, name, hardAddr string, VMFds, VhostFds []*os.File) error { -+func (q *qemu) hotAddNetDevice(deviceName, name, hardAddr string, VMFds, VhostFds []*os.File, queues uint32) error { - var ( - VMFdNames []string - VhostFdNames []string -@@ -1489,7 +1489,7 @@ func (q *qemu) hotAddNetDevice(deviceName, name, hardAddr string, VMFds, VhostFd - return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", name, VMFdNames, VhostFdNames) - } - -- return q.qmpMonitorCh.qmp.ExecuteNetdevAdd(q.qmpMonitorCh.ctx, "tap", name, deviceName, "no", "no", 0) -+ return q.qmpMonitorCh.qmp.ExecuteNetdevAdd(q.qmpMonitorCh.ctx, "tap", name, deviceName, "no", "no", int(queues)) - } - - func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) { -@@ -1512,7 +1512,7 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) { - - devID := "virtio-" + tap.ID - if op == addDevice { -- if err = q.hotAddNetDevice(tap.TAPIface.Name, tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds); err != nil { -+ if err = q.hotAddNetDevice(tap.TAPIface.Name, tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds, tap.Queues); err != nil { - return err - } - -@@ -1542,9 +1542,9 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) { - } - if machine.Type == QemuCCWVirtio { - devNoHotplug := fmt.Sprintf("fe.%x.%x", bus, addr) -- return q.qmpMonitorCh.qmp.ExecuteNetCCWDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), devNoHotplug, int(q.config.NumVCPUs)) -+ return q.qmpMonitorCh.qmp.ExecuteNetCCWDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), devNoHotplug, int(tap.Queues)) - } -- return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bus, romFile, int(q.config.NumVCPUs), defaultDisableModern) -+ return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bus, romFile, int(tap.Queues), defaultDisableModern) - - } - -diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go -index e6f155a3..d8ab6c1a 100644 ---- a/virtcontainers/sandbox.go -+++ b/virtcontainers/sandbox.go -@@ -948,6 +948,7 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) { - }, - Addrs: addrs, - VhostUserSocket: inf.VhostUserSocket, -+ Queues: inf.Queues, - }, nil - } - -diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go -index 2cf70dce..5a3e7f7e 100644 ---- a/virtcontainers/tap_endpoint.go -+++ b/virtcontainers/tap_endpoint.go -@@ -115,7 +115,7 @@ func (endpoint *TapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPat - return nil - } - --func createTapNetworkEndpoint(idx int, ifName string, tapIfName string) (*TapEndpoint, error) { -+func createTapNetworkEndpoint(idx int, ifName string, tapIfName string, queues uint32) (*TapEndpoint, error) { - if idx < 0 { - return &TapEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx) - } -@@ -139,6 +139,10 @@ func createTapNetworkEndpoint(idx int, ifName string, tapIfName string) (*TapEnd - endpoint.TapInterface.TAPIface.Name = tapIfName - } - -+ if queues > 0 { -+ endpoint.TapInterface.Queues = queues -+ } -+ - return endpoint, nil - } - -diff --git a/virtcontainers/tuntap_endpoint.go b/virtcontainers/tuntap_endpoint.go -index b076d694..827d8852 100644 ---- a/virtcontainers/tuntap_endpoint.go -+++ b/virtcontainers/tuntap_endpoint.go -@@ -117,12 +117,12 @@ func (endpoint *TuntapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNs - return nil - } - --func createTuntapNetworkEndpoint(idx int, ifName string, hwName net.HardwareAddr, internetworkingModel NetInterworkingModel) (*TuntapEndpoint, error) { -+func createTuntapNetworkEndpoint(idx int, ifName string, hwName net.HardwareAddr, internetworkingModel NetInterworkingModel, queues uint32) (*TuntapEndpoint, error) { - if idx < 0 { - return &TuntapEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx) - } - -- netPair, err := createNetworkInterfacePair(idx, ifName, internetworkingModel) -+ netPair, err := createNetworkInterfacePair(idx, ifName, internetworkingModel, queues) - if err != nil { - return nil, err - } -diff --git a/virtcontainers/veth_endpoint.go b/virtcontainers/veth_endpoint.go -index 0f2ec9ba..554b9b22 100644 ---- a/virtcontainers/veth_endpoint.go -+++ b/virtcontainers/veth_endpoint.go -@@ -20,12 +20,12 @@ type VethEndpoint struct { - PCIAddr string - } - --func createVethNetworkEndpoint(idx int, ifName string, interworkingModel NetInterworkingModel) (*VethEndpoint, error) { -+func createVethNetworkEndpoint(idx int, ifName string, interworkingModel NetInterworkingModel, queues uint32) (*VethEndpoint, error) { - if idx < 0 { - return &VethEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx) - } - -- netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel) -+ netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel, queues) - if err != nil { - return nil, err - } -diff --git a/virtcontainers/veth_endpoint_test.go b/virtcontainers/veth_endpoint_test.go -index 9649b82e..23f1876d 100644 ---- a/virtcontainers/veth_endpoint_test.go -+++ b/virtcontainers/veth_endpoint_test.go -@@ -34,7 +34,7 @@ func TestCreateVethNetworkEndpoint(t *testing.T) { - EndpointType: VethEndpointType, - } - -- result, err := createVethNetworkEndpoint(4, "", DefaultNetInterworkingModel) -+ result, err := createVethNetworkEndpoint(4, "", DefaultNetInterworkingModel, 0) - assert.NoError(err) - - // the resulting ID will be random - so let's overwrite to test the rest of the flow -@@ -68,7 +68,7 @@ func TestCreateVethNetworkEndpointChooseIfaceName(t *testing.T) { - EndpointType: VethEndpointType, - } - -- result, err := createVethNetworkEndpoint(4, "eth1", DefaultNetInterworkingModel) -+ result, err := createVethNetworkEndpoint(4, "eth1", DefaultNetInterworkingModel, 0) - assert.NoError(err) - - // the resulting ID will be random - so let's overwrite to test the rest of the flow -@@ -95,7 +95,7 @@ func TestCreateVethNetworkEndpointInvalidArgs(t *testing.T) { - } - - for _, d := range failingValues { -- _, err := createVethNetworkEndpoint(d.idx, d.ifName, DefaultNetInterworkingModel) -+ _, err := createVethNetworkEndpoint(d.idx, d.ifName, DefaultNetInterworkingModel, 0) - assert.Error(err) - } - } -diff --git a/virtcontainers/vhostuser_endpoint.go b/virtcontainers/vhostuser_endpoint.go -index 2fc3d837..85ef67b4 100644 ---- a/virtcontainers/vhostuser_endpoint.go -+++ b/virtcontainers/vhostuser_endpoint.go -@@ -33,6 +33,7 @@ type VhostUserEndpoint struct { - EndpointProperties NetworkInfo - EndpointType EndpointType - PCIAddr string -+ Queues uint32 - } - - // Properties returns the properties of the interface. -@@ -119,7 +120,7 @@ func (endpoint *VhostUserEndpoint) HotDetach(h hypervisor, netNsCreated bool, ne - } - - // Create a vhostuser endpoint --func createVhostUserEndpoint(netInfo NetworkInfo, socket string) (*VhostUserEndpoint, error) { -+func createVhostUserEndpoint(netInfo NetworkInfo, socket string, queues uint32) (*VhostUserEndpoint, error) { - uniqueID := uuid.Generate().String() - vhostUserEndpoint := &VhostUserEndpoint{ - ID: uniqueID, -@@ -127,6 +128,7 @@ func createVhostUserEndpoint(netInfo NetworkInfo, socket string) (*VhostUserEndp - HardAddr: netInfo.Iface.HardwareAddr.String(), - IfaceName: netInfo.Iface.Name, - EndpointType: VhostUserEndpointType, -+ Queues: queues, - } - return vhostUserEndpoint, nil - } -diff --git a/virtcontainers/vhostuser_endpoint_test.go b/virtcontainers/vhostuser_endpoint_test.go -index 584490cc..046ca229 100644 ---- a/virtcontainers/vhostuser_endpoint_test.go -+++ b/virtcontainers/vhostuser_endpoint_test.go -@@ -137,7 +137,7 @@ func TestCreateVhostUserEndpoint(t *testing.T) { - EndpointType: VhostUserEndpointType, - } - -- result, err := createVhostUserEndpoint(netinfo, socket) -+ result, err := createVhostUserEndpoint(netinfo, socket, 0) - assert.NoError(err) - assert.Exactly(result, expected) - } --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0047-network-add-support-to-get-network-stats-of-vm-throu.patch b/runtime/patches/0047-network-add-support-to-get-network-stats-of-vm-throu.patch deleted file mode 100644 index bbb179f..0000000 --- a/runtime/patches/0047-network-add-support-to-get-network-stats-of-vm-throu.patch +++ /dev/null @@ -1,2511 +0,0 @@ -From 292df0bbfd4cb529eb7febdd8216859682b27bf2 Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Wed, 19 Aug 2020 22:06:31 +0800 -Subject: [PATCH 47/50] network: add support to get network stats of vm through - events interface - -reason: add support to get network stats of vm through events interface - -Signed-off-by: jiangpengfei ---- - cli/events.go | 29 +- - .../agent/protocols/grpc/agent.pb.go | 1749 ++++++++++++++++---- - virtcontainers/container.go | 89 +- - virtcontainers/kata_agent.go | 18 +- - 4 files changed, 1516 insertions(+), 369 deletions(-) - -diff --git a/cli/events.go b/cli/events.go -index 9e9200a6..75554665 100644 ---- a/cli/events.go -+++ b/cli/events.go -@@ -13,10 +13,9 @@ import ( - "sync" - "time" - -+ "github.com/kata-containers/runtime/pkg/katautils" - vc "github.com/kata-containers/runtime/virtcontainers" - "github.com/kata-containers/runtime/virtcontainers/types" -- -- "github.com/kata-containers/runtime/pkg/katautils" - "github.com/sirupsen/logrus" - "github.com/urfave/cli" - ) -@@ -29,12 +28,17 @@ type event struct { - - // stats is the runc specific stats structure for stability when encoding and decoding stats. - type stats struct { -- CPU cpu `json:"cpu"` -- Memory memory `json:"memory"` -- Pids pids `json:"pids"` -- Blkio blkio `json:"blkio"` -- Hugetlb map[string]hugetlb `json:"hugetlb"` -- IntelRdt intelRdt `json:"intel_rdt"` -+ CPU cpu `json:"cpu"` -+ Memory memory `json:"memory"` -+ Pids pids `json:"pids"` -+ Blkio blkio `json:"blkio"` -+ Hugetlb map[string]hugetlb `json:"hugetlb"` -+ IntelRdt intelRdt `json:"intel_rdt"` -+ Interfaces []vc.InterfaceStats `json:"interfaces"` -+ Tcp vc.TcpStat `json:"tcp"` -+ Tcp6 vc.TcpStat `json:"tcp6"` -+ Udp vc.UdpStat `json:"udp"` -+ Udp6 vc.UdpStat `json:"udp6"` - } - - type hugetlb struct { -@@ -225,7 +229,8 @@ information is displayed once every 5 seconds.`, - - func convertVirtcontainerStats(containerStats *vc.ContainerStats) *stats { - cg := containerStats.CgroupStats -- if cg == nil { -+ net := containerStats.NetworkStats -+ if cg == nil && net == nil { - return nil - } - var s stats -@@ -261,6 +266,12 @@ func convertVirtcontainerStats(containerStats *vc.ContainerStats) *stats { - s.Hugetlb[k] = convertHugtlb(v) - } - -+ s.Interfaces = net.Interfaces -+ s.Tcp = net.Tcp -+ s.Tcp6 = net.Tcp6 -+ s.Udp = net.Udp -+ s.Udp6 = net.Udp6 -+ - return &s - } - -diff --git a/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go b/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go -index 04d0ee52..c50ecb58 100644 ---- a/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go -+++ b/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go -@@ -33,6 +33,9 @@ - BlkioStats - HugetlbStats - CgroupStats -+ InterfaceStats -+ TcpStat -+ UdpStat - NetworkStats - StatsContainerResponse - WriteStreamRequest -@@ -883,7 +886,7 @@ func (m *CgroupStats) GetHugetlbStats() map[string]*HugetlbStats { - return nil - } - --type NetworkStats struct { -+type InterfaceStats struct { - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - RxBytes uint64 `protobuf:"varint,2,opt,name=rx_bytes,json=rxBytes,proto3" json:"rx_bytes,omitempty"` - RxPackets uint64 `protobuf:"varint,3,opt,name=rx_packets,json=rxPackets,proto3" json:"rx_packets,omitempty"` -@@ -895,83 +898,267 @@ type NetworkStats struct { - TxDropped uint64 `protobuf:"varint,9,opt,name=tx_dropped,json=txDropped,proto3" json:"tx_dropped,omitempty"` - } - --func (m *NetworkStats) Reset() { *m = NetworkStats{} } --func (m *NetworkStats) String() string { return proto.CompactTextString(m) } --func (*NetworkStats) ProtoMessage() {} --func (*NetworkStats) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{23} } -+func (m *InterfaceStats) Reset() { *m = InterfaceStats{} } -+func (m *InterfaceStats) String() string { return proto.CompactTextString(m) } -+func (*InterfaceStats) ProtoMessage() {} -+func (*InterfaceStats) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{23} } - --func (m *NetworkStats) GetName() string { -+func (m *InterfaceStats) GetName() string { - if m != nil { - return m.Name - } - return "" - } - --func (m *NetworkStats) GetRxBytes() uint64 { -+func (m *InterfaceStats) GetRxBytes() uint64 { - if m != nil { - return m.RxBytes - } - return 0 - } - --func (m *NetworkStats) GetRxPackets() uint64 { -+func (m *InterfaceStats) GetRxPackets() uint64 { - if m != nil { - return m.RxPackets - } - return 0 - } - --func (m *NetworkStats) GetRxErrors() uint64 { -+func (m *InterfaceStats) GetRxErrors() uint64 { - if m != nil { - return m.RxErrors - } - return 0 - } - --func (m *NetworkStats) GetRxDropped() uint64 { -+func (m *InterfaceStats) GetRxDropped() uint64 { - if m != nil { - return m.RxDropped - } - return 0 - } - --func (m *NetworkStats) GetTxBytes() uint64 { -+func (m *InterfaceStats) GetTxBytes() uint64 { - if m != nil { - return m.TxBytes - } - return 0 - } - --func (m *NetworkStats) GetTxPackets() uint64 { -+func (m *InterfaceStats) GetTxPackets() uint64 { - if m != nil { - return m.TxPackets - } - return 0 - } - --func (m *NetworkStats) GetTxErrors() uint64 { -+func (m *InterfaceStats) GetTxErrors() uint64 { - if m != nil { - return m.TxErrors - } - return 0 - } - --func (m *NetworkStats) GetTxDropped() uint64 { -+func (m *InterfaceStats) GetTxDropped() uint64 { - if m != nil { - return m.TxDropped - } - return 0 - } - -+type TcpStat struct { -+ Established uint64 `protobuf:"varint,1,opt,name=established,proto3" json:"established,omitempty"` -+ SynSent uint64 `protobuf:"varint,2,opt,name=syn_sent,json=synSent,proto3" json:"syn_sent,omitempty"` -+ SynRecv uint64 `protobuf:"varint,3,opt,name=syn_recv,json=synRecv,proto3" json:"syn_recv,omitempty"` -+ FinWait1 uint64 `protobuf:"varint,4,opt,name=fin_wait1,json=finWait1,proto3" json:"fin_wait1,omitempty"` -+ FinWait2 uint64 `protobuf:"varint,5,opt,name=fin_wait2,json=finWait2,proto3" json:"fin_wait2,omitempty"` -+ TimeWait uint64 `protobuf:"varint,6,opt,name=time_wait,json=timeWait,proto3" json:"time_wait,omitempty"` -+ Close uint64 `protobuf:"varint,7,opt,name=close,proto3" json:"close,omitempty"` -+ CloseWait uint64 `protobuf:"varint,8,opt,name=close_wait,json=closeWait,proto3" json:"close_wait,omitempty"` -+ LastAck uint64 `protobuf:"varint,9,opt,name=last_ack,json=lastAck,proto3" json:"last_ack,omitempty"` -+ Listen uint64 `protobuf:"varint,10,opt,name=listen,proto3" json:"listen,omitempty"` -+ Closing uint64 `protobuf:"varint,11,opt,name=closing,proto3" json:"closing,omitempty"` -+} -+ -+func (m *TcpStat) Reset() { *m = TcpStat{} } -+func (m *TcpStat) String() string { return proto.CompactTextString(m) } -+func (*TcpStat) ProtoMessage() {} -+func (*TcpStat) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{24} } -+ -+func (m *TcpStat) GetEstablished() uint64 { -+ if m != nil { -+ return m.Established -+ } -+ return 0 -+} -+ -+func (m *TcpStat) GetSynSent() uint64 { -+ if m != nil { -+ return m.SynSent -+ } -+ return 0 -+} -+ -+func (m *TcpStat) GetSynRecv() uint64 { -+ if m != nil { -+ return m.SynRecv -+ } -+ return 0 -+} -+ -+func (m *TcpStat) GetFinWait1() uint64 { -+ if m != nil { -+ return m.FinWait1 -+ } -+ return 0 -+} -+ -+func (m *TcpStat) GetFinWait2() uint64 { -+ if m != nil { -+ return m.FinWait2 -+ } -+ return 0 -+} -+ -+func (m *TcpStat) GetTimeWait() uint64 { -+ if m != nil { -+ return m.TimeWait -+ } -+ return 0 -+} -+ -+func (m *TcpStat) GetClose() uint64 { -+ if m != nil { -+ return m.Close -+ } -+ return 0 -+} -+ -+func (m *TcpStat) GetCloseWait() uint64 { -+ if m != nil { -+ return m.CloseWait -+ } -+ return 0 -+} -+ -+func (m *TcpStat) GetLastAck() uint64 { -+ if m != nil { -+ return m.LastAck -+ } -+ return 0 -+} -+ -+func (m *TcpStat) GetListen() uint64 { -+ if m != nil { -+ return m.Listen -+ } -+ return 0 -+} -+ -+func (m *TcpStat) GetClosing() uint64 { -+ if m != nil { -+ return m.Closing -+ } -+ return 0 -+} -+ -+type UdpStat struct { -+ Listen uint64 `protobuf:"varint,1,opt,name=listen,proto3" json:"listen,omitempty"` -+ Dropped uint64 `protobuf:"varint,2,opt,name=dropped,proto3" json:"dropped,omitempty"` -+ RxQueued uint64 `protobuf:"varint,3,opt,name=rx_queued,json=rxQueued,proto3" json:"rx_queued,omitempty"` -+ TxQueued uint64 `protobuf:"varint,4,opt,name=tx_queued,json=txQueued,proto3" json:"tx_queued,omitempty"` -+} -+ -+func (m *UdpStat) Reset() { *m = UdpStat{} } -+func (m *UdpStat) String() string { return proto.CompactTextString(m) } -+func (*UdpStat) ProtoMessage() {} -+func (*UdpStat) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{25} } -+ -+func (m *UdpStat) GetListen() uint64 { -+ if m != nil { -+ return m.Listen -+ } -+ return 0 -+} -+ -+func (m *UdpStat) GetDropped() uint64 { -+ if m != nil { -+ return m.Dropped -+ } -+ return 0 -+} -+ -+func (m *UdpStat) GetRxQueued() uint64 { -+ if m != nil { -+ return m.RxQueued -+ } -+ return 0 -+} -+ -+func (m *UdpStat) GetTxQueued() uint64 { -+ if m != nil { -+ return m.TxQueued -+ } -+ return 0 -+} -+ -+type NetworkStats struct { -+ Interfaces []*InterfaceStats `protobuf:"bytes,1,rep,name=interfaces" json:"interfaces,omitempty"` -+ Tcp *TcpStat `protobuf:"bytes,2,opt,name=tcp" json:"tcp,omitempty"` -+ Tcp6 *TcpStat `protobuf:"bytes,3,opt,name=tcp6" json:"tcp6,omitempty"` -+ Udp *UdpStat `protobuf:"bytes,4,opt,name=udp" json:"udp,omitempty"` -+ Udp6 *UdpStat `protobuf:"bytes,5,opt,name=udp6" json:"udp6,omitempty"` -+} -+ -+func (m *NetworkStats) Reset() { *m = NetworkStats{} } -+func (m *NetworkStats) String() string { return proto.CompactTextString(m) } -+func (*NetworkStats) ProtoMessage() {} -+func (*NetworkStats) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{26} } -+ -+func (m *NetworkStats) GetInterfaces() []*InterfaceStats { -+ if m != nil { -+ return m.Interfaces -+ } -+ return nil -+} -+ -+func (m *NetworkStats) GetTcp() *TcpStat { -+ if m != nil { -+ return m.Tcp -+ } -+ return nil -+} -+ -+func (m *NetworkStats) GetTcp6() *TcpStat { -+ if m != nil { -+ return m.Tcp6 -+ } -+ return nil -+} -+ -+func (m *NetworkStats) GetUdp() *UdpStat { -+ if m != nil { -+ return m.Udp -+ } -+ return nil -+} -+ -+func (m *NetworkStats) GetUdp6() *UdpStat { -+ if m != nil { -+ return m.Udp6 -+ } -+ return nil -+} -+ - type StatsContainerResponse struct { -- CgroupStats *CgroupStats `protobuf:"bytes,1,opt,name=cgroup_stats,json=cgroupStats" json:"cgroup_stats,omitempty"` -- NetworkStats []*NetworkStats `protobuf:"bytes,2,rep,name=network_stats,json=networkStats" json:"network_stats,omitempty"` -+ CgroupStats *CgroupStats `protobuf:"bytes,1,opt,name=cgroup_stats,json=cgroupStats" json:"cgroup_stats,omitempty"` -+ NetworkStats *NetworkStats `protobuf:"bytes,2,opt,name=network_stats,json=networkStats" json:"network_stats,omitempty"` - } - - func (m *StatsContainerResponse) Reset() { *m = StatsContainerResponse{} } - func (m *StatsContainerResponse) String() string { return proto.CompactTextString(m) } - func (*StatsContainerResponse) ProtoMessage() {} --func (*StatsContainerResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{24} } -+func (*StatsContainerResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{27} } - - func (m *StatsContainerResponse) GetCgroupStats() *CgroupStats { - if m != nil { -@@ -980,7 +1167,7 @@ func (m *StatsContainerResponse) GetCgroupStats() *CgroupStats { - return nil - } - --func (m *StatsContainerResponse) GetNetworkStats() []*NetworkStats { -+func (m *StatsContainerResponse) GetNetworkStats() *NetworkStats { - if m != nil { - return m.NetworkStats - } -@@ -996,7 +1183,7 @@ type WriteStreamRequest struct { - func (m *WriteStreamRequest) Reset() { *m = WriteStreamRequest{} } - func (m *WriteStreamRequest) String() string { return proto.CompactTextString(m) } - func (*WriteStreamRequest) ProtoMessage() {} --func (*WriteStreamRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{25} } -+func (*WriteStreamRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{28} } - - func (m *WriteStreamRequest) GetContainerId() string { - if m != nil { -@@ -1026,7 +1213,7 @@ type WriteStreamResponse struct { - func (m *WriteStreamResponse) Reset() { *m = WriteStreamResponse{} } - func (m *WriteStreamResponse) String() string { return proto.CompactTextString(m) } - func (*WriteStreamResponse) ProtoMessage() {} --func (*WriteStreamResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{26} } -+func (*WriteStreamResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{29} } - - func (m *WriteStreamResponse) GetLen() uint32 { - if m != nil { -@@ -1044,7 +1231,7 @@ type ReadStreamRequest struct { - func (m *ReadStreamRequest) Reset() { *m = ReadStreamRequest{} } - func (m *ReadStreamRequest) String() string { return proto.CompactTextString(m) } - func (*ReadStreamRequest) ProtoMessage() {} --func (*ReadStreamRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{27} } -+func (*ReadStreamRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{30} } - - func (m *ReadStreamRequest) GetContainerId() string { - if m != nil { -@@ -1074,7 +1261,7 @@ type ReadStreamResponse struct { - func (m *ReadStreamResponse) Reset() { *m = ReadStreamResponse{} } - func (m *ReadStreamResponse) String() string { return proto.CompactTextString(m) } - func (*ReadStreamResponse) ProtoMessage() {} --func (*ReadStreamResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{28} } -+func (*ReadStreamResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{31} } - - func (m *ReadStreamResponse) GetData() []byte { - if m != nil { -@@ -1091,7 +1278,7 @@ type CloseStdinRequest struct { - func (m *CloseStdinRequest) Reset() { *m = CloseStdinRequest{} } - func (m *CloseStdinRequest) String() string { return proto.CompactTextString(m) } - func (*CloseStdinRequest) ProtoMessage() {} --func (*CloseStdinRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{29} } -+func (*CloseStdinRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{32} } - - func (m *CloseStdinRequest) GetContainerId() string { - if m != nil { -@@ -1117,7 +1304,7 @@ type TtyWinResizeRequest struct { - func (m *TtyWinResizeRequest) Reset() { *m = TtyWinResizeRequest{} } - func (m *TtyWinResizeRequest) String() string { return proto.CompactTextString(m) } - func (*TtyWinResizeRequest) ProtoMessage() {} --func (*TtyWinResizeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{30} } -+func (*TtyWinResizeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{33} } - - func (m *TtyWinResizeRequest) GetContainerId() string { - if m != nil { -@@ -1158,7 +1345,7 @@ type KernelModule struct { - func (m *KernelModule) Reset() { *m = KernelModule{} } - func (m *KernelModule) String() string { return proto.CompactTextString(m) } - func (*KernelModule) ProtoMessage() {} --func (*KernelModule) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{31} } -+func (*KernelModule) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{34} } - - func (m *KernelModule) GetName() string { - if m != nil { -@@ -1197,7 +1384,7 @@ type CreateSandboxRequest struct { - func (m *CreateSandboxRequest) Reset() { *m = CreateSandboxRequest{} } - func (m *CreateSandboxRequest) String() string { return proto.CompactTextString(m) } - func (*CreateSandboxRequest) ProtoMessage() {} --func (*CreateSandboxRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{32} } -+func (*CreateSandboxRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{35} } - - func (m *CreateSandboxRequest) GetHostname() string { - if m != nil { -@@ -1254,7 +1441,7 @@ type DestroySandboxRequest struct { - func (m *DestroySandboxRequest) Reset() { *m = DestroySandboxRequest{} } - func (m *DestroySandboxRequest) String() string { return proto.CompactTextString(m) } - func (*DestroySandboxRequest) ProtoMessage() {} --func (*DestroySandboxRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{33} } -+func (*DestroySandboxRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{36} } - - type Interfaces struct { - Interfaces []*types.Interface `protobuf:"bytes,1,rep,name=Interfaces" json:"Interfaces,omitempty"` -@@ -1263,7 +1450,7 @@ type Interfaces struct { - func (m *Interfaces) Reset() { *m = Interfaces{} } - func (m *Interfaces) String() string { return proto.CompactTextString(m) } - func (*Interfaces) ProtoMessage() {} --func (*Interfaces) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{34} } -+func (*Interfaces) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{37} } - - func (m *Interfaces) GetInterfaces() []*types.Interface { - if m != nil { -@@ -1279,7 +1466,7 @@ type Routes struct { - func (m *Routes) Reset() { *m = Routes{} } - func (m *Routes) String() string { return proto.CompactTextString(m) } - func (*Routes) ProtoMessage() {} --func (*Routes) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{35} } -+func (*Routes) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{38} } - - func (m *Routes) GetRoutes() []*types.Route { - if m != nil { -@@ -1295,7 +1482,7 @@ type UpdateInterfaceRequest struct { - func (m *UpdateInterfaceRequest) Reset() { *m = UpdateInterfaceRequest{} } - func (m *UpdateInterfaceRequest) String() string { return proto.CompactTextString(m) } - func (*UpdateInterfaceRequest) ProtoMessage() {} --func (*UpdateInterfaceRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{36} } -+func (*UpdateInterfaceRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{39} } - - func (m *UpdateInterfaceRequest) GetInterface() *types.Interface { - if m != nil { -@@ -1312,7 +1499,7 @@ type UpdateRoutesRequest struct { - func (m *UpdateRoutesRequest) Reset() { *m = UpdateRoutesRequest{} } - func (m *UpdateRoutesRequest) String() string { return proto.CompactTextString(m) } - func (*UpdateRoutesRequest) ProtoMessage() {} --func (*UpdateRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{37} } -+func (*UpdateRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{40} } - - func (m *UpdateRoutesRequest) GetRoutes() *Routes { - if m != nil { -@@ -1334,7 +1521,7 @@ type ListInterfacesRequest struct { - func (m *ListInterfacesRequest) Reset() { *m = ListInterfacesRequest{} } - func (m *ListInterfacesRequest) String() string { return proto.CompactTextString(m) } - func (*ListInterfacesRequest) ProtoMessage() {} --func (*ListInterfacesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{38} } -+func (*ListInterfacesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{41} } - - type ListRoutesRequest struct { - } -@@ -1342,7 +1529,7 @@ type ListRoutesRequest struct { - func (m *ListRoutesRequest) Reset() { *m = ListRoutesRequest{} } - func (m *ListRoutesRequest) String() string { return proto.CompactTextString(m) } - func (*ListRoutesRequest) ProtoMessage() {} --func (*ListRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{39} } -+func (*ListRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{42} } - - type OnlineCPUMemRequest struct { - // Wait specifies if the caller waits for the agent to online all resources. -@@ -1358,7 +1545,7 @@ type OnlineCPUMemRequest struct { - func (m *OnlineCPUMemRequest) Reset() { *m = OnlineCPUMemRequest{} } - func (m *OnlineCPUMemRequest) String() string { return proto.CompactTextString(m) } - func (*OnlineCPUMemRequest) ProtoMessage() {} --func (*OnlineCPUMemRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{40} } -+func (*OnlineCPUMemRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{43} } - - func (m *OnlineCPUMemRequest) GetWait() bool { - if m != nil { -@@ -1389,7 +1576,7 @@ type ReseedRandomDevRequest struct { - func (m *ReseedRandomDevRequest) Reset() { *m = ReseedRandomDevRequest{} } - func (m *ReseedRandomDevRequest) String() string { return proto.CompactTextString(m) } - func (*ReseedRandomDevRequest) ProtoMessage() {} --func (*ReseedRandomDevRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{41} } -+func (*ReseedRandomDevRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{44} } - - func (m *ReseedRandomDevRequest) GetData() []byte { - if m != nil { -@@ -1416,7 +1603,7 @@ type AgentDetails struct { - func (m *AgentDetails) Reset() { *m = AgentDetails{} } - func (m *AgentDetails) String() string { return proto.CompactTextString(m) } - func (*AgentDetails) ProtoMessage() {} --func (*AgentDetails) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{42} } -+func (*AgentDetails) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{45} } - - func (m *AgentDetails) GetVersion() string { - if m != nil { -@@ -1467,7 +1654,7 @@ type GuestDetailsRequest struct { - func (m *GuestDetailsRequest) Reset() { *m = GuestDetailsRequest{} } - func (m *GuestDetailsRequest) String() string { return proto.CompactTextString(m) } - func (*GuestDetailsRequest) ProtoMessage() {} --func (*GuestDetailsRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{43} } -+func (*GuestDetailsRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{46} } - - func (m *GuestDetailsRequest) GetMemBlockSize() bool { - if m != nil { -@@ -1493,7 +1680,7 @@ type GuestDetailsResponse struct { - func (m *GuestDetailsResponse) Reset() { *m = GuestDetailsResponse{} } - func (m *GuestDetailsResponse) String() string { return proto.CompactTextString(m) } - func (*GuestDetailsResponse) ProtoMessage() {} --func (*GuestDetailsResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{44} } -+func (*GuestDetailsResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{47} } - - func (m *GuestDetailsResponse) GetMemBlockSizeBytes() uint64 { - if m != nil { -@@ -1525,7 +1712,7 @@ type MemHotplugByProbeRequest struct { - func (m *MemHotplugByProbeRequest) Reset() { *m = MemHotplugByProbeRequest{} } - func (m *MemHotplugByProbeRequest) String() string { return proto.CompactTextString(m) } - func (*MemHotplugByProbeRequest) ProtoMessage() {} --func (*MemHotplugByProbeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{45} } -+func (*MemHotplugByProbeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{48} } - - func (m *MemHotplugByProbeRequest) GetMemHotplugProbeAddr() []uint64 { - if m != nil { -@@ -1544,7 +1731,7 @@ type SetGuestDateTimeRequest struct { - func (m *SetGuestDateTimeRequest) Reset() { *m = SetGuestDateTimeRequest{} } - func (m *SetGuestDateTimeRequest) String() string { return proto.CompactTextString(m) } - func (*SetGuestDateTimeRequest) ProtoMessage() {} --func (*SetGuestDateTimeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{46} } -+func (*SetGuestDateTimeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{49} } - - func (m *SetGuestDateTimeRequest) GetSec() int64 { - if m != nil { -@@ -1593,7 +1780,7 @@ type Storage struct { - func (m *Storage) Reset() { *m = Storage{} } - func (m *Storage) String() string { return proto.CompactTextString(m) } - func (*Storage) ProtoMessage() {} --func (*Storage) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{47} } -+func (*Storage) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{50} } - - func (m *Storage) GetDriver() string { - if m != nil { -@@ -1676,7 +1863,7 @@ type Device struct { - func (m *Device) Reset() { *m = Device{} } - func (m *Device) String() string { return proto.CompactTextString(m) } - func (*Device) ProtoMessage() {} --func (*Device) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{48} } -+func (*Device) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{51} } - - func (m *Device) GetId() string { - if m != nil { -@@ -1722,7 +1909,7 @@ type StringUser struct { - func (m *StringUser) Reset() { *m = StringUser{} } - func (m *StringUser) String() string { return proto.CompactTextString(m) } - func (*StringUser) ProtoMessage() {} --func (*StringUser) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{49} } -+func (*StringUser) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{52} } - - func (m *StringUser) GetUid() string { - if m != nil { -@@ -1770,7 +1957,7 @@ type CopyFileRequest struct { - func (m *CopyFileRequest) Reset() { *m = CopyFileRequest{} } - func (m *CopyFileRequest) String() string { return proto.CompactTextString(m) } - func (*CopyFileRequest) ProtoMessage() {} --func (*CopyFileRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{50} } -+func (*CopyFileRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{53} } - - func (m *CopyFileRequest) GetPath() string { - if m != nil { -@@ -1834,7 +2021,7 @@ type StartTracingRequest struct { - func (m *StartTracingRequest) Reset() { *m = StartTracingRequest{} } - func (m *StartTracingRequest) String() string { return proto.CompactTextString(m) } - func (*StartTracingRequest) ProtoMessage() {} --func (*StartTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{51} } -+func (*StartTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{54} } - - type StopTracingRequest struct { - } -@@ -1842,7 +2029,7 @@ type StopTracingRequest struct { - func (m *StopTracingRequest) Reset() { *m = StopTracingRequest{} } - func (m *StopTracingRequest) String() string { return proto.CompactTextString(m) } - func (*StopTracingRequest) ProtoMessage() {} --func (*StopTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{52} } -+func (*StopTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{55} } - - type UpdateIPVSRequest struct { - // IPVS_req is the IPVS rule message needed to update -@@ -1852,7 +2039,7 @@ type UpdateIPVSRequest struct { - func (m *UpdateIPVSRequest) Reset() { *m = UpdateIPVSRequest{} } - func (m *UpdateIPVSRequest) String() string { return proto.CompactTextString(m) } - func (*UpdateIPVSRequest) ProtoMessage() {} --func (*UpdateIPVSRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{53} } -+func (*UpdateIPVSRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{56} } - - func (m *UpdateIPVSRequest) GetIPVSReq() string { - if m != nil { -@@ -1869,7 +2056,7 @@ type IPVSResponse struct { - func (m *IPVSResponse) Reset() { *m = IPVSResponse{} } - func (m *IPVSResponse) String() string { return proto.CompactTextString(m) } - func (*IPVSResponse) ProtoMessage() {} --func (*IPVSResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{54} } -+func (*IPVSResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{57} } - - func (m *IPVSResponse) GetIPVSRes() string { - if m != nil { -@@ -1902,6 +2089,9 @@ func init() { - proto.RegisterType((*BlkioStats)(nil), "grpc.BlkioStats") - proto.RegisterType((*HugetlbStats)(nil), "grpc.HugetlbStats") - proto.RegisterType((*CgroupStats)(nil), "grpc.CgroupStats") -+ proto.RegisterType((*InterfaceStats)(nil), "grpc.InterfaceStats") -+ proto.RegisterType((*TcpStat)(nil), "grpc.TcpStat") -+ proto.RegisterType((*UdpStat)(nil), "grpc.UdpStat") - proto.RegisterType((*NetworkStats)(nil), "grpc.NetworkStats") - proto.RegisterType((*StatsContainerResponse)(nil), "grpc.StatsContainerResponse") - proto.RegisterType((*WriteStreamRequest)(nil), "grpc.WriteStreamRequest") -@@ -4008,7 +4198,7 @@ func (m *CgroupStats) MarshalTo(dAtA []byte) (int, error) { - return i, nil - } - --func (m *NetworkStats) Marshal() (dAtA []byte, err error) { -+func (m *InterfaceStats) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) -@@ -4018,7 +4208,7 @@ func (m *NetworkStats) Marshal() (dAtA []byte, err error) { - return dAtA[:n], nil - } - --func (m *NetworkStats) MarshalTo(dAtA []byte) (int, error) { -+func (m *InterfaceStats) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int -@@ -4072,7 +4262,7 @@ func (m *NetworkStats) MarshalTo(dAtA []byte) (int, error) { - return i, nil - } - --func (m *StatsContainerResponse) Marshal() (dAtA []byte, err error) { -+func (m *TcpStat) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) -@@ -4082,37 +4272,70 @@ func (m *StatsContainerResponse) Marshal() (dAtA []byte, err error) { - return dAtA[:n], nil - } - --func (m *StatsContainerResponse) MarshalTo(dAtA []byte) (int, error) { -+func (m *TcpStat) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l -- if m.CgroupStats != nil { -- dAtA[i] = 0xa -+ if m.Established != 0 { -+ dAtA[i] = 0x8 - i++ -- i = encodeVarintAgent(dAtA, i, uint64(m.CgroupStats.Size())) -- n18, err := m.CgroupStats.MarshalTo(dAtA[i:]) -- if err != nil { -- return 0, err -- } -- i += n18 -+ i = encodeVarintAgent(dAtA, i, uint64(m.Established)) - } -- if len(m.NetworkStats) > 0 { -- for _, msg := range m.NetworkStats { -- dAtA[i] = 0x12 -- i++ -- i = encodeVarintAgent(dAtA, i, uint64(msg.Size())) -- n, err := msg.MarshalTo(dAtA[i:]) -- if err != nil { -- return 0, err -- } -- i += n -- } -+ if m.SynSent != 0 { -+ dAtA[i] = 0x10 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.SynSent)) -+ } -+ if m.SynRecv != 0 { -+ dAtA[i] = 0x18 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.SynRecv)) -+ } -+ if m.FinWait1 != 0 { -+ dAtA[i] = 0x20 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.FinWait1)) -+ } -+ if m.FinWait2 != 0 { -+ dAtA[i] = 0x28 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.FinWait2)) -+ } -+ if m.TimeWait != 0 { -+ dAtA[i] = 0x30 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.TimeWait)) -+ } -+ if m.Close != 0 { -+ dAtA[i] = 0x38 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.Close)) -+ } -+ if m.CloseWait != 0 { -+ dAtA[i] = 0x40 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.CloseWait)) -+ } -+ if m.LastAck != 0 { -+ dAtA[i] = 0x48 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.LastAck)) -+ } -+ if m.Listen != 0 { -+ dAtA[i] = 0x50 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.Listen)) -+ } -+ if m.Closing != 0 { -+ dAtA[i] = 0x58 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.Closing)) - } - return i, nil - } - --func (m *WriteStreamRequest) Marshal() (dAtA []byte, err error) { -+func (m *UdpStat) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) -@@ -4122,33 +4345,35 @@ func (m *WriteStreamRequest) Marshal() (dAtA []byte, err error) { - return dAtA[:n], nil - } - --func (m *WriteStreamRequest) MarshalTo(dAtA []byte) (int, error) { -+func (m *UdpStat) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l -- if len(m.ContainerId) > 0 { -- dAtA[i] = 0xa -+ if m.Listen != 0 { -+ dAtA[i] = 0x8 - i++ -- i = encodeVarintAgent(dAtA, i, uint64(len(m.ContainerId))) -- i += copy(dAtA[i:], m.ContainerId) -+ i = encodeVarintAgent(dAtA, i, uint64(m.Listen)) - } -- if len(m.ExecId) > 0 { -- dAtA[i] = 0x12 -+ if m.Dropped != 0 { -+ dAtA[i] = 0x10 - i++ -- i = encodeVarintAgent(dAtA, i, uint64(len(m.ExecId))) -- i += copy(dAtA[i:], m.ExecId) -+ i = encodeVarintAgent(dAtA, i, uint64(m.Dropped)) - } -- if len(m.Data) > 0 { -- dAtA[i] = 0x1a -+ if m.RxQueued != 0 { -+ dAtA[i] = 0x18 - i++ -- i = encodeVarintAgent(dAtA, i, uint64(len(m.Data))) -- i += copy(dAtA[i:], m.Data) -+ i = encodeVarintAgent(dAtA, i, uint64(m.RxQueued)) -+ } -+ if m.TxQueued != 0 { -+ dAtA[i] = 0x20 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.TxQueued)) - } - return i, nil - } - --func (m *WriteStreamResponse) Marshal() (dAtA []byte, err error) { -+func (m *NetworkStats) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) -@@ -4158,20 +4383,67 @@ func (m *WriteStreamResponse) Marshal() (dAtA []byte, err error) { - return dAtA[:n], nil - } - --func (m *WriteStreamResponse) MarshalTo(dAtA []byte) (int, error) { -+func (m *NetworkStats) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l -- if m.Len != 0 { -- dAtA[i] = 0x8 -+ if len(m.Interfaces) > 0 { -+ for _, msg := range m.Interfaces { -+ dAtA[i] = 0xa -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(msg.Size())) -+ n, err := msg.MarshalTo(dAtA[i:]) -+ if err != nil { -+ return 0, err -+ } -+ i += n -+ } -+ } -+ if m.Tcp != nil { -+ dAtA[i] = 0x12 - i++ -- i = encodeVarintAgent(dAtA, i, uint64(m.Len)) -+ i = encodeVarintAgent(dAtA, i, uint64(m.Tcp.Size())) -+ n18, err := m.Tcp.MarshalTo(dAtA[i:]) -+ if err != nil { -+ return 0, err -+ } -+ i += n18 -+ } -+ if m.Tcp6 != nil { -+ dAtA[i] = 0x1a -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.Tcp6.Size())) -+ n19, err := m.Tcp6.MarshalTo(dAtA[i:]) -+ if err != nil { -+ return 0, err -+ } -+ i += n19 -+ } -+ if m.Udp != nil { -+ dAtA[i] = 0x22 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.Udp.Size())) -+ n20, err := m.Udp.MarshalTo(dAtA[i:]) -+ if err != nil { -+ return 0, err -+ } -+ i += n20 -+ } -+ if m.Udp6 != nil { -+ dAtA[i] = 0x2a -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.Udp6.Size())) -+ n21, err := m.Udp6.MarshalTo(dAtA[i:]) -+ if err != nil { -+ return 0, err -+ } -+ i += n21 - } - return i, nil - } - --func (m *ReadStreamRequest) Marshal() (dAtA []byte, err error) { -+func (m *StatsContainerResponse) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) -@@ -4181,32 +4453,129 @@ func (m *ReadStreamRequest) Marshal() (dAtA []byte, err error) { - return dAtA[:n], nil - } - --func (m *ReadStreamRequest) MarshalTo(dAtA []byte) (int, error) { -+func (m *StatsContainerResponse) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l -- if len(m.ContainerId) > 0 { -+ if m.CgroupStats != nil { - dAtA[i] = 0xa - i++ -- i = encodeVarintAgent(dAtA, i, uint64(len(m.ContainerId))) -- i += copy(dAtA[i:], m.ContainerId) -+ i = encodeVarintAgent(dAtA, i, uint64(m.CgroupStats.Size())) -+ n22, err := m.CgroupStats.MarshalTo(dAtA[i:]) -+ if err != nil { -+ return 0, err -+ } -+ i += n22 - } -- if len(m.ExecId) > 0 { -+ if m.NetworkStats != nil { - dAtA[i] = 0x12 - i++ -- i = encodeVarintAgent(dAtA, i, uint64(len(m.ExecId))) -- i += copy(dAtA[i:], m.ExecId) -- } -- if m.Len != 0 { -- dAtA[i] = 0x18 -- i++ -- i = encodeVarintAgent(dAtA, i, uint64(m.Len)) -+ i = encodeVarintAgent(dAtA, i, uint64(m.NetworkStats.Size())) -+ n23, err := m.NetworkStats.MarshalTo(dAtA[i:]) -+ if err != nil { -+ return 0, err -+ } -+ i += n23 - } - return i, nil - } - --func (m *ReadStreamResponse) Marshal() (dAtA []byte, err error) { -+func (m *WriteStreamRequest) Marshal() (dAtA []byte, err error) { -+ size := m.Size() -+ dAtA = make([]byte, size) -+ n, err := m.MarshalTo(dAtA) -+ if err != nil { -+ return nil, err -+ } -+ return dAtA[:n], nil -+} -+ -+func (m *WriteStreamRequest) MarshalTo(dAtA []byte) (int, error) { -+ var i int -+ _ = i -+ var l int -+ _ = l -+ if len(m.ContainerId) > 0 { -+ dAtA[i] = 0xa -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(len(m.ContainerId))) -+ i += copy(dAtA[i:], m.ContainerId) -+ } -+ if len(m.ExecId) > 0 { -+ dAtA[i] = 0x12 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(len(m.ExecId))) -+ i += copy(dAtA[i:], m.ExecId) -+ } -+ if len(m.Data) > 0 { -+ dAtA[i] = 0x1a -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(len(m.Data))) -+ i += copy(dAtA[i:], m.Data) -+ } -+ return i, nil -+} -+ -+func (m *WriteStreamResponse) Marshal() (dAtA []byte, err error) { -+ size := m.Size() -+ dAtA = make([]byte, size) -+ n, err := m.MarshalTo(dAtA) -+ if err != nil { -+ return nil, err -+ } -+ return dAtA[:n], nil -+} -+ -+func (m *WriteStreamResponse) MarshalTo(dAtA []byte) (int, error) { -+ var i int -+ _ = i -+ var l int -+ _ = l -+ if m.Len != 0 { -+ dAtA[i] = 0x8 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.Len)) -+ } -+ return i, nil -+} -+ -+func (m *ReadStreamRequest) Marshal() (dAtA []byte, err error) { -+ size := m.Size() -+ dAtA = make([]byte, size) -+ n, err := m.MarshalTo(dAtA) -+ if err != nil { -+ return nil, err -+ } -+ return dAtA[:n], nil -+} -+ -+func (m *ReadStreamRequest) MarshalTo(dAtA []byte) (int, error) { -+ var i int -+ _ = i -+ var l int -+ _ = l -+ if len(m.ContainerId) > 0 { -+ dAtA[i] = 0xa -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(len(m.ContainerId))) -+ i += copy(dAtA[i:], m.ContainerId) -+ } -+ if len(m.ExecId) > 0 { -+ dAtA[i] = 0x12 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(len(m.ExecId))) -+ i += copy(dAtA[i:], m.ExecId) -+ } -+ if m.Len != 0 { -+ dAtA[i] = 0x18 -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.Len)) -+ } -+ return i, nil -+} -+ -+func (m *ReadStreamResponse) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) -@@ -4521,11 +4890,11 @@ func (m *UpdateInterfaceRequest) MarshalTo(dAtA []byte) (int, error) { - dAtA[i] = 0xa - i++ - i = encodeVarintAgent(dAtA, i, uint64(m.Interface.Size())) -- n19, err := m.Interface.MarshalTo(dAtA[i:]) -+ n24, err := m.Interface.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } -- i += n19 -+ i += n24 - } - return i, nil - } -@@ -4549,11 +4918,11 @@ func (m *UpdateRoutesRequest) MarshalTo(dAtA []byte) (int, error) { - dAtA[i] = 0xa - i++ - i = encodeVarintAgent(dAtA, i, uint64(m.Routes.Size())) -- n20, err := m.Routes.MarshalTo(dAtA[i:]) -+ n25, err := m.Routes.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } -- i += n20 -+ i += n25 - } - if m.Increment { - dAtA[i] = 0x10 -@@ -4807,11 +5176,11 @@ func (m *GuestDetailsResponse) MarshalTo(dAtA []byte) (int, error) { - dAtA[i] = 0x12 - i++ - i = encodeVarintAgent(dAtA, i, uint64(m.AgentDetails.Size())) -- n21, err := m.AgentDetails.MarshalTo(dAtA[i:]) -+ n26, err := m.AgentDetails.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } -- i += n21 -+ i += n26 - } - if m.SupportMemHotplugProbe { - dAtA[i] = 0x18 -@@ -4842,21 +5211,21 @@ func (m *MemHotplugByProbeRequest) MarshalTo(dAtA []byte) (int, error) { - var l int - _ = l - if len(m.MemHotplugProbeAddr) > 0 { -- dAtA23 := make([]byte, len(m.MemHotplugProbeAddr)*10) -- var j22 int -+ dAtA28 := make([]byte, len(m.MemHotplugProbeAddr)*10) -+ var j27 int - for _, num := range m.MemHotplugProbeAddr { - for num >= 1<<7 { -- dAtA23[j22] = uint8(uint64(num)&0x7f | 0x80) -+ dAtA28[j27] = uint8(uint64(num)&0x7f | 0x80) - num >>= 7 -- j22++ -+ j27++ - } -- dAtA23[j22] = uint8(num) -- j22++ -+ dAtA28[j27] = uint8(num) -+ j27++ - } - dAtA[i] = 0xa - i++ -- i = encodeVarintAgent(dAtA, i, uint64(j22)) -- i += copy(dAtA[i:], dAtA23[:j22]) -+ i = encodeVarintAgent(dAtA, i, uint64(j27)) -+ i += copy(dAtA[i:], dAtA28[:j27]) - } - return i, nil - } -@@ -5648,7 +6017,7 @@ func (m *CgroupStats) Size() (n int) { - return n - } - --func (m *NetworkStats) Size() (n int) { -+func (m *InterfaceStats) Size() (n int) { - var l int - _ = l - l = len(m.Name) -@@ -5682,6 +6051,91 @@ func (m *NetworkStats) Size() (n int) { - return n - } - -+func (m *TcpStat) Size() (n int) { -+ var l int -+ _ = l -+ if m.Established != 0 { -+ n += 1 + sovAgent(uint64(m.Established)) -+ } -+ if m.SynSent != 0 { -+ n += 1 + sovAgent(uint64(m.SynSent)) -+ } -+ if m.SynRecv != 0 { -+ n += 1 + sovAgent(uint64(m.SynRecv)) -+ } -+ if m.FinWait1 != 0 { -+ n += 1 + sovAgent(uint64(m.FinWait1)) -+ } -+ if m.FinWait2 != 0 { -+ n += 1 + sovAgent(uint64(m.FinWait2)) -+ } -+ if m.TimeWait != 0 { -+ n += 1 + sovAgent(uint64(m.TimeWait)) -+ } -+ if m.Close != 0 { -+ n += 1 + sovAgent(uint64(m.Close)) -+ } -+ if m.CloseWait != 0 { -+ n += 1 + sovAgent(uint64(m.CloseWait)) -+ } -+ if m.LastAck != 0 { -+ n += 1 + sovAgent(uint64(m.LastAck)) -+ } -+ if m.Listen != 0 { -+ n += 1 + sovAgent(uint64(m.Listen)) -+ } -+ if m.Closing != 0 { -+ n += 1 + sovAgent(uint64(m.Closing)) -+ } -+ return n -+} -+ -+func (m *UdpStat) Size() (n int) { -+ var l int -+ _ = l -+ if m.Listen != 0 { -+ n += 1 + sovAgent(uint64(m.Listen)) -+ } -+ if m.Dropped != 0 { -+ n += 1 + sovAgent(uint64(m.Dropped)) -+ } -+ if m.RxQueued != 0 { -+ n += 1 + sovAgent(uint64(m.RxQueued)) -+ } -+ if m.TxQueued != 0 { -+ n += 1 + sovAgent(uint64(m.TxQueued)) -+ } -+ return n -+} -+ -+func (m *NetworkStats) Size() (n int) { -+ var l int -+ _ = l -+ if len(m.Interfaces) > 0 { -+ for _, e := range m.Interfaces { -+ l = e.Size() -+ n += 1 + l + sovAgent(uint64(l)) -+ } -+ } -+ if m.Tcp != nil { -+ l = m.Tcp.Size() -+ n += 1 + l + sovAgent(uint64(l)) -+ } -+ if m.Tcp6 != nil { -+ l = m.Tcp6.Size() -+ n += 1 + l + sovAgent(uint64(l)) -+ } -+ if m.Udp != nil { -+ l = m.Udp.Size() -+ n += 1 + l + sovAgent(uint64(l)) -+ } -+ if m.Udp6 != nil { -+ l = m.Udp6.Size() -+ n += 1 + l + sovAgent(uint64(l)) -+ } -+ return n -+} -+ - func (m *StatsContainerResponse) Size() (n int) { - var l int - _ = l -@@ -5689,11 +6143,9 @@ func (m *StatsContainerResponse) Size() (n int) { - l = m.CgroupStats.Size() - n += 1 + l + sovAgent(uint64(l)) - } -- if len(m.NetworkStats) > 0 { -- for _, e := range m.NetworkStats { -- l = e.Size() -- n += 1 + l + sovAgent(uint64(l)) -- } -+ if m.NetworkStats != nil { -+ l = m.NetworkStats.Size() -+ n += 1 + l + sovAgent(uint64(l)) - } - return n - } -@@ -9396,7 +9848,7 @@ func (m *CgroupStats) Unmarshal(dAtA []byte) error { - } - return nil - } --func (m *NetworkStats) Unmarshal(dAtA []byte) error { -+func (m *InterfaceStats) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { -@@ -9419,10 +9871,10 @@ func (m *NetworkStats) Unmarshal(dAtA []byte) error { - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { -- return fmt.Errorf("proto: NetworkStats: wiretype end group for non-group") -+ return fmt.Errorf("proto: InterfaceStats: wiretype end group for non-group") - } - if fieldNum <= 0 { -- return fmt.Errorf("proto: NetworkStats: illegal tag %d (wire type %d)", fieldNum, wire) -+ return fmt.Errorf("proto: InterfaceStats: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: -@@ -9627,7 +10079,7 @@ func (m *NetworkStats) Unmarshal(dAtA []byte) error { - } - return nil - } --func (m *StatsContainerResponse) Unmarshal(dAtA []byte) error { -+func (m *TcpStat) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { -@@ -9650,17 +10102,17 @@ func (m *StatsContainerResponse) Unmarshal(dAtA []byte) error { - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { -- return fmt.Errorf("proto: StatsContainerResponse: wiretype end group for non-group") -+ return fmt.Errorf("proto: TcpStat: wiretype end group for non-group") - } - if fieldNum <= 0 { -- return fmt.Errorf("proto: StatsContainerResponse: illegal tag %d (wire type %d)", fieldNum, wire) -+ return fmt.Errorf("proto: TcpStat: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field CgroupStats", wireType) -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field Established", wireType) - } -- var msglen int -+ m.Established = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowAgent -@@ -9670,30 +10122,54 @@ func (m *StatsContainerResponse) Unmarshal(dAtA []byte) error { - } - b := dAtA[iNdEx] - iNdEx++ -- msglen |= (int(b) & 0x7F) << shift -+ m.Established |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } -- if msglen < 0 { -- return ErrInvalidLengthAgent -+ case 2: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field SynSent", wireType) - } -- postIndex := iNdEx + msglen -- if postIndex > l { -- return io.ErrUnexpectedEOF -+ m.SynSent = 0 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ m.SynSent |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } - } -- if m.CgroupStats == nil { -- m.CgroupStats = &CgroupStats{} -+ case 3: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field SynRecv", wireType) - } -- if err := m.CgroupStats.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -- return err -+ m.SynRecv = 0 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ m.SynRecv |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } - } -- iNdEx = postIndex -- case 2: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field NetworkStats", wireType) -+ case 4: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field FinWait1", wireType) - } -- var msglen int -+ m.FinWait1 = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowAgent -@@ -9703,20 +10179,596 @@ func (m *StatsContainerResponse) Unmarshal(dAtA []byte) error { - } - b := dAtA[iNdEx] - iNdEx++ -- msglen |= (int(b) & 0x7F) << shift -+ m.FinWait1 |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } -- if msglen < 0 { -- return ErrInvalidLengthAgent -+ case 5: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field FinWait2", wireType) - } -- postIndex := iNdEx + msglen -- if postIndex > l { -- return io.ErrUnexpectedEOF -+ m.FinWait2 = 0 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ m.FinWait2 |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ case 6: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field TimeWait", wireType) -+ } -+ m.TimeWait = 0 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ m.TimeWait |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ case 7: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field Close", wireType) -+ } -+ m.Close = 0 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ m.Close |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ case 8: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field CloseWait", wireType) -+ } -+ m.CloseWait = 0 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ m.CloseWait |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ case 9: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field LastAck", wireType) -+ } -+ m.LastAck = 0 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ m.LastAck |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ case 10: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field Listen", wireType) -+ } -+ m.Listen = 0 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ m.Listen |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ case 11: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field Closing", wireType) -+ } -+ m.Closing = 0 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ m.Closing |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ default: -+ iNdEx = preIndex -+ skippy, err := skipAgent(dAtA[iNdEx:]) -+ if err != nil { -+ return err -+ } -+ if skippy < 0 { -+ return ErrInvalidLengthAgent -+ } -+ if (iNdEx + skippy) > l { -+ return io.ErrUnexpectedEOF -+ } -+ iNdEx += skippy -+ } -+ } -+ -+ if iNdEx > l { -+ return io.ErrUnexpectedEOF -+ } -+ return nil -+} -+func (m *UdpStat) Unmarshal(dAtA []byte) error { -+ l := len(dAtA) -+ iNdEx := 0 -+ for iNdEx < l { -+ preIndex := iNdEx -+ var wire uint64 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ wire |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ fieldNum := int32(wire >> 3) -+ wireType := int(wire & 0x7) -+ if wireType == 4 { -+ return fmt.Errorf("proto: UdpStat: wiretype end group for non-group") -+ } -+ if fieldNum <= 0 { -+ return fmt.Errorf("proto: UdpStat: illegal tag %d (wire type %d)", fieldNum, wire) -+ } -+ switch fieldNum { -+ case 1: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field Listen", wireType) -+ } -+ m.Listen = 0 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ m.Listen |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ case 2: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field Dropped", wireType) -+ } -+ m.Dropped = 0 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ m.Dropped |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ case 3: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field RxQueued", wireType) -+ } -+ m.RxQueued = 0 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ m.RxQueued |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ case 4: -+ if wireType != 0 { -+ return fmt.Errorf("proto: wrong wireType = %d for field TxQueued", wireType) -+ } -+ m.TxQueued = 0 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ m.TxQueued |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ default: -+ iNdEx = preIndex -+ skippy, err := skipAgent(dAtA[iNdEx:]) -+ if err != nil { -+ return err -+ } -+ if skippy < 0 { -+ return ErrInvalidLengthAgent -+ } -+ if (iNdEx + skippy) > l { -+ return io.ErrUnexpectedEOF -+ } -+ iNdEx += skippy -+ } -+ } -+ -+ if iNdEx > l { -+ return io.ErrUnexpectedEOF -+ } -+ return nil -+} -+func (m *NetworkStats) Unmarshal(dAtA []byte) error { -+ l := len(dAtA) -+ iNdEx := 0 -+ for iNdEx < l { -+ preIndex := iNdEx -+ var wire uint64 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ wire |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ fieldNum := int32(wire >> 3) -+ wireType := int(wire & 0x7) -+ if wireType == 4 { -+ return fmt.Errorf("proto: NetworkStats: wiretype end group for non-group") -+ } -+ if fieldNum <= 0 { -+ return fmt.Errorf("proto: NetworkStats: illegal tag %d (wire type %d)", fieldNum, wire) -+ } -+ switch fieldNum { -+ case 1: -+ if wireType != 2 { -+ return fmt.Errorf("proto: wrong wireType = %d for field Interfaces", wireType) -+ } -+ var msglen int -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ msglen |= (int(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ if msglen < 0 { -+ return ErrInvalidLengthAgent -+ } -+ postIndex := iNdEx + msglen -+ if postIndex > l { -+ return io.ErrUnexpectedEOF -+ } -+ m.Interfaces = append(m.Interfaces, &InterfaceStats{}) -+ if err := m.Interfaces[len(m.Interfaces)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -+ return err -+ } -+ iNdEx = postIndex -+ case 2: -+ if wireType != 2 { -+ return fmt.Errorf("proto: wrong wireType = %d for field Tcp", wireType) -+ } -+ var msglen int -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ msglen |= (int(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ if msglen < 0 { -+ return ErrInvalidLengthAgent -+ } -+ postIndex := iNdEx + msglen -+ if postIndex > l { -+ return io.ErrUnexpectedEOF -+ } -+ if m.Tcp == nil { -+ m.Tcp = &TcpStat{} -+ } -+ if err := m.Tcp.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -+ return err -+ } -+ iNdEx = postIndex -+ case 3: -+ if wireType != 2 { -+ return fmt.Errorf("proto: wrong wireType = %d for field Tcp6", wireType) -+ } -+ var msglen int -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ msglen |= (int(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ if msglen < 0 { -+ return ErrInvalidLengthAgent -+ } -+ postIndex := iNdEx + msglen -+ if postIndex > l { -+ return io.ErrUnexpectedEOF -+ } -+ if m.Tcp6 == nil { -+ m.Tcp6 = &TcpStat{} -+ } -+ if err := m.Tcp6.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -+ return err -+ } -+ iNdEx = postIndex -+ case 4: -+ if wireType != 2 { -+ return fmt.Errorf("proto: wrong wireType = %d for field Udp", wireType) -+ } -+ var msglen int -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ msglen |= (int(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ if msglen < 0 { -+ return ErrInvalidLengthAgent -+ } -+ postIndex := iNdEx + msglen -+ if postIndex > l { -+ return io.ErrUnexpectedEOF -+ } -+ if m.Udp == nil { -+ m.Udp = &UdpStat{} -+ } -+ if err := m.Udp.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -+ return err -+ } -+ iNdEx = postIndex -+ case 5: -+ if wireType != 2 { -+ return fmt.Errorf("proto: wrong wireType = %d for field Udp6", wireType) -+ } -+ var msglen int -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ msglen |= (int(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ if msglen < 0 { -+ return ErrInvalidLengthAgent -+ } -+ postIndex := iNdEx + msglen -+ if postIndex > l { -+ return io.ErrUnexpectedEOF -+ } -+ if m.Udp6 == nil { -+ m.Udp6 = &UdpStat{} -+ } -+ if err := m.Udp6.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -+ return err -+ } -+ iNdEx = postIndex -+ default: -+ iNdEx = preIndex -+ skippy, err := skipAgent(dAtA[iNdEx:]) -+ if err != nil { -+ return err -+ } -+ if skippy < 0 { -+ return ErrInvalidLengthAgent -+ } -+ if (iNdEx + skippy) > l { -+ return io.ErrUnexpectedEOF -+ } -+ iNdEx += skippy -+ } -+ } -+ -+ if iNdEx > l { -+ return io.ErrUnexpectedEOF -+ } -+ return nil -+} -+func (m *StatsContainerResponse) Unmarshal(dAtA []byte) error { -+ l := len(dAtA) -+ iNdEx := 0 -+ for iNdEx < l { -+ preIndex := iNdEx -+ var wire uint64 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ wire |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ fieldNum := int32(wire >> 3) -+ wireType := int(wire & 0x7) -+ if wireType == 4 { -+ return fmt.Errorf("proto: StatsContainerResponse: wiretype end group for non-group") -+ } -+ if fieldNum <= 0 { -+ return fmt.Errorf("proto: StatsContainerResponse: illegal tag %d (wire type %d)", fieldNum, wire) -+ } -+ switch fieldNum { -+ case 1: -+ if wireType != 2 { -+ return fmt.Errorf("proto: wrong wireType = %d for field CgroupStats", wireType) -+ } -+ var msglen int -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ msglen |= (int(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ if msglen < 0 { -+ return ErrInvalidLengthAgent -+ } -+ postIndex := iNdEx + msglen -+ if postIndex > l { -+ return io.ErrUnexpectedEOF -+ } -+ if m.CgroupStats == nil { -+ m.CgroupStats = &CgroupStats{} -+ } -+ if err := m.CgroupStats.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -+ return err -+ } -+ iNdEx = postIndex -+ case 2: -+ if wireType != 2 { -+ return fmt.Errorf("proto: wrong wireType = %d for field NetworkStats", wireType) -+ } -+ var msglen int -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ msglen |= (int(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ if msglen < 0 { -+ return ErrInvalidLengthAgent -+ } -+ postIndex := iNdEx + msglen -+ if postIndex > l { -+ return io.ErrUnexpectedEOF -+ } -+ if m.NetworkStats == nil { -+ m.NetworkStats = &NetworkStats{} - } -- m.NetworkStats = append(m.NetworkStats, &NetworkStats{}) -- if err := m.NetworkStats[len(m.NetworkStats)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -+ if err := m.NetworkStats.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex -@@ -13190,189 +14242,204 @@ var ( - func init() { proto.RegisterFile("agent.proto", fileDescriptorAgent) } - - var fileDescriptorAgent = []byte{ -- // 2931 bytes of a gzipped FileDescriptorProto -- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0xcb, 0x6e, 0x1c, 0xc7, -- 0xb5, 0x98, 0x07, 0x87, 0x33, 0x67, 0x5e, 0x9c, 0x22, 0x45, 0x8d, 0x46, 0xb2, 0xae, 0xdc, 0xb6, -- 0x65, 0xfa, 0xfa, 0x7a, 0x68, 0xcb, 0xc6, 0xf5, 0x0b, 0xbe, 0x82, 0x48, 0xe9, 0x8a, 0x8c, 0xad, -- 0x88, 0xe9, 0x91, 0xe2, 0x04, 0x41, 0xd0, 0x68, 0x76, 0x97, 0x86, 0x65, 0x4e, 0x77, 0xb5, 0xab, -- 0xaa, 0x29, 0x8e, 0x03, 0x64, 0x99, 0xec, 0xb2, 0xcc, 0x2e, 0x3f, 0x10, 0x64, 0x97, 0x65, 0xb6, -- 0x59, 0x18, 0x59, 0x05, 0xf9, 0x80, 0x20, 0xf0, 0x27, 0xe4, 0x0b, 0x82, 0x7a, 0xf5, 0x63, 0x66, -- 0x48, 0x23, 0x82, 0x80, 0x6c, 0x1a, 0x75, 0x4e, 0x9d, 0x3a, 0xaf, 0xaa, 0x3a, 0x75, 0xce, 0x69, -- 0x68, 0xfb, 0x53, 0x1c, 0x8b, 0x71, 0xc2, 0xa8, 0xa0, 0xa8, 0x3e, 0x65, 0x49, 0x30, 0x6a, 0xd1, -- 0x80, 0x68, 0xc4, 0xe8, 0x7f, 0xa7, 0x44, 0x9c, 0xa4, 0xc7, 0xe3, 0x80, 0x46, 0xbb, 0xa7, 0xbe, -- 0xf0, 0xdf, 0x09, 0x68, 0x2c, 0x7c, 0x12, 0x63, 0xc6, 0x77, 0xd5, 0xc2, 0xdd, 0xe4, 0x74, 0xba, -- 0x2b, 0xe6, 0x09, 0xe6, 0xfa, 0x6b, 0xd6, 0x5d, 0x9f, 0x52, 0x3a, 0x9d, 0xe1, 0x5d, 0x05, 0x1d, -- 0xa7, 0xcf, 0x76, 0x71, 0x94, 0x88, 0xb9, 0x9e, 0x74, 0x7e, 0x57, 0x85, 0xed, 0x7d, 0x86, 0x7d, -- 0x81, 0xf7, 0x2d, 0x37, 0x17, 0x7f, 0x9d, 0x62, 0x2e, 0xd0, 0xab, 0xd0, 0xc9, 0x24, 0x78, 0x24, -- 0x1c, 0x56, 0x6e, 0x55, 0x76, 0x5a, 0x6e, 0x3b, 0xc3, 0x1d, 0x86, 0xe8, 0x2a, 0xac, 0xe3, 0x73, -- 0x1c, 0xc8, 0xd9, 0xaa, 0x9a, 0x6d, 0x48, 0xf0, 0x30, 0x44, 0xef, 0x41, 0x9b, 0x0b, 0x46, 0xe2, -- 0xa9, 0x97, 0x72, 0xcc, 0x86, 0xb5, 0x5b, 0x95, 0x9d, 0xf6, 0x9d, 0x8d, 0xb1, 0x34, 0x69, 0x3c, -- 0x51, 0x13, 0x4f, 0x39, 0x66, 0x2e, 0xf0, 0x6c, 0x8c, 0x6e, 0xc3, 0x7a, 0x88, 0xcf, 0x48, 0x80, -- 0xf9, 0xb0, 0x7e, 0xab, 0xb6, 0xd3, 0xbe, 0xd3, 0xd1, 0xe4, 0xf7, 0x15, 0xd2, 0xb5, 0x93, 0xe8, -- 0x2d, 0x68, 0x72, 0x41, 0x99, 0x3f, 0xc5, 0x7c, 0xb8, 0xa6, 0x08, 0xbb, 0x96, 0xaf, 0xc2, 0xba, -- 0xd9, 0x34, 0xba, 0x01, 0xb5, 0xc7, 0xfb, 0x87, 0xc3, 0x86, 0x92, 0x0e, 0x86, 0x2a, 0xc1, 0x81, -- 0x2b, 0xd1, 0xe8, 0x35, 0xe8, 0x72, 0x3f, 0x0e, 0x8f, 0xe9, 0xb9, 0x97, 0x90, 0x30, 0xe6, 0xc3, -- 0xf5, 0x5b, 0x95, 0x9d, 0xa6, 0xdb, 0x31, 0xc8, 0x23, 0x89, 0x73, 0x3e, 0x81, 0x2b, 0x13, 0xe1, -- 0x33, 0xf1, 0x02, 0xde, 0x71, 0x9e, 0xc2, 0xb6, 0x8b, 0x23, 0x7a, 0xf6, 0x42, 0xae, 0x1d, 0xc2, -- 0xba, 0x20, 0x11, 0xa6, 0xa9, 0x50, 0xae, 0xed, 0xba, 0x16, 0x74, 0xfe, 0x50, 0x01, 0xf4, 0xe0, -- 0x1c, 0x07, 0x47, 0x8c, 0x06, 0x98, 0xf3, 0xff, 0xd0, 0x76, 0xbd, 0x09, 0xeb, 0x89, 0x56, 0x60, -- 0x58, 0x57, 0xe4, 0x66, 0x17, 0xac, 0x56, 0x76, 0xd6, 0xf9, 0x0a, 0xb6, 0x26, 0x64, 0x1a, 0xfb, -- 0xb3, 0x97, 0xa8, 0xef, 0x36, 0x34, 0xb8, 0xe2, 0xa9, 0x54, 0xed, 0xba, 0x06, 0x72, 0x8e, 0x00, -- 0x7d, 0xe9, 0x13, 0xf1, 0xf2, 0x24, 0x39, 0xef, 0xc0, 0x66, 0x89, 0x23, 0x4f, 0x68, 0xcc, 0xb1, -- 0x52, 0x40, 0xf8, 0x22, 0xe5, 0x8a, 0xd9, 0x9a, 0x6b, 0x20, 0x07, 0xc3, 0xd6, 0x17, 0x84, 0x5b, -- 0x72, 0xfc, 0xef, 0xa8, 0xb0, 0x0d, 0x8d, 0x67, 0x94, 0x45, 0xbe, 0xb0, 0x1a, 0x68, 0x08, 0x21, -- 0xa8, 0xfb, 0x6c, 0xca, 0x87, 0xb5, 0x5b, 0xb5, 0x9d, 0x96, 0xab, 0xc6, 0xf2, 0x54, 0x2e, 0x88, -- 0x31, 0x7a, 0xbd, 0x0a, 0x1d, 0xe3, 0x77, 0x6f, 0x46, 0xb8, 0x50, 0x72, 0x3a, 0x6e, 0xdb, 0xe0, -- 0xe4, 0x1a, 0x87, 0xc2, 0xf6, 0xd3, 0x24, 0x7c, 0xc1, 0x0b, 0x7f, 0x07, 0x5a, 0x0c, 0x73, 0x9a, -- 0x32, 0x79, 0x4d, 0xab, 0x6a, 0xdf, 0xb7, 0xf4, 0xbe, 0x7f, 0x41, 0xe2, 0xf4, 0xdc, 0xb5, 0x73, -- 0x6e, 0x4e, 0x66, 0xae, 0x90, 0xe0, 0x2f, 0x72, 0x85, 0x3e, 0x81, 0x2b, 0x47, 0x7e, 0xca, 0x5f, -- 0x44, 0x57, 0xe7, 0x53, 0x79, 0xfd, 0x78, 0x1a, 0xbd, 0xd0, 0xe2, 0xdf, 0x57, 0xa0, 0xb9, 0x9f, -- 0xa4, 0x4f, 0xb9, 0x3f, 0xc5, 0xe8, 0xbf, 0xa0, 0x2d, 0xa8, 0xf0, 0x67, 0x5e, 0x2a, 0x41, 0x45, -- 0x5e, 0x77, 0x41, 0xa1, 0x34, 0x81, 0x74, 0x3b, 0x66, 0x41, 0x92, 0x1a, 0x8a, 0xea, 0xad, 0xda, -- 0x4e, 0xdd, 0x6d, 0x6b, 0x9c, 0x26, 0x19, 0xc3, 0xa6, 0x9a, 0xf3, 0x48, 0xec, 0x9d, 0x62, 0x16, -- 0xe3, 0x59, 0x44, 0x43, 0xac, 0xce, 0x6f, 0xdd, 0x1d, 0xa8, 0xa9, 0xc3, 0xf8, 0xf3, 0x6c, 0x02, -- 0xfd, 0x37, 0x0c, 0x32, 0x7a, 0x79, 0x29, 0x15, 0x75, 0x5d, 0x51, 0xf7, 0x0d, 0xf5, 0x53, 0x83, -- 0x76, 0x7e, 0x09, 0xbd, 0x27, 0x27, 0x8c, 0x0a, 0x31, 0x23, 0xf1, 0xf4, 0xbe, 0x2f, 0x7c, 0x19, -- 0x3d, 0x12, 0xcc, 0x08, 0x0d, 0xb9, 0xd1, 0xd6, 0x82, 0xe8, 0x6d, 0x18, 0x08, 0x4d, 0x8b, 0x43, -- 0xcf, 0xd2, 0x54, 0x15, 0xcd, 0x46, 0x36, 0x71, 0x64, 0x88, 0xdf, 0x80, 0x5e, 0x4e, 0x2c, 0xe3, -- 0x8f, 0xd1, 0xb7, 0x9b, 0x61, 0x9f, 0x90, 0x08, 0x3b, 0x67, 0xca, 0x57, 0x6a, 0x93, 0xd1, 0xdb, -- 0xd0, 0xca, 0xfd, 0x50, 0x51, 0x27, 0xa4, 0xa7, 0x4f, 0x88, 0x75, 0xa7, 0xdb, 0xcc, 0x9c, 0xf2, -- 0x19, 0xf4, 0x45, 0xa6, 0xb8, 0x17, 0xfa, 0xc2, 0x2f, 0x1f, 0xaa, 0xb2, 0x55, 0x6e, 0x4f, 0x94, -- 0x60, 0xe7, 0x53, 0x68, 0x1d, 0x91, 0x90, 0x6b, 0xc1, 0x43, 0x58, 0x0f, 0x52, 0xc6, 0x70, 0x2c, -- 0xac, 0xc9, 0x06, 0x44, 0x5b, 0xb0, 0x36, 0x23, 0x11, 0x11, 0xc6, 0x4c, 0x0d, 0x38, 0x14, 0xe0, -- 0x11, 0x8e, 0x28, 0x9b, 0x2b, 0x87, 0x6d, 0xc1, 0x5a, 0x71, 0x73, 0x35, 0x80, 0xae, 0x43, 0x2b, -- 0xf2, 0xcf, 0xb3, 0x4d, 0x95, 0x33, 0xcd, 0xc8, 0x3f, 0xd7, 0xca, 0x0f, 0x61, 0xfd, 0x99, 0x4f, -- 0x66, 0x41, 0x2c, 0x8c, 0x57, 0x2c, 0x98, 0x0b, 0xac, 0x17, 0x05, 0xfe, 0xb9, 0x0a, 0x6d, 0x2d, -- 0x51, 0x2b, 0xbc, 0x05, 0x6b, 0x81, 0x1f, 0x9c, 0x64, 0x22, 0x15, 0x80, 0x6e, 0x5b, 0x45, 0xaa, -- 0xc5, 0x20, 0x9c, 0x6b, 0x6a, 0x55, 0xdb, 0x05, 0xe0, 0xcf, 0xfd, 0xc4, 0xe8, 0x56, 0xbb, 0x80, -- 0xb8, 0x25, 0x69, 0xb4, 0xba, 0xef, 0x43, 0x47, 0x9f, 0x3b, 0xb3, 0xa4, 0x7e, 0xc1, 0x92, 0xb6, -- 0xa6, 0xd2, 0x8b, 0x5e, 0x83, 0x6e, 0xca, 0xb1, 0x77, 0x42, 0x30, 0xf3, 0x59, 0x70, 0x32, 0x1f, -- 0xae, 0xe9, 0x37, 0x32, 0xe5, 0xf8, 0xc0, 0xe2, 0xd0, 0x1d, 0x58, 0x93, 0xe1, 0x8f, 0x0f, 0x1b, -- 0xea, 0x39, 0xbe, 0x51, 0x64, 0xa9, 0x4c, 0x1d, 0xab, 0xef, 0x83, 0x58, 0xb0, 0xb9, 0xab, 0x49, -- 0x47, 0x1f, 0x01, 0xe4, 0x48, 0xb4, 0x01, 0xb5, 0x53, 0x3c, 0x37, 0xf7, 0x50, 0x0e, 0xa5, 0x73, -- 0xce, 0xfc, 0x59, 0x6a, 0xbd, 0xae, 0x81, 0x4f, 0xaa, 0x1f, 0x55, 0x9c, 0x00, 0xfa, 0x7b, 0xb3, -- 0x53, 0x42, 0x0b, 0xcb, 0xb7, 0x60, 0x2d, 0xf2, 0xbf, 0xa2, 0xcc, 0x7a, 0x52, 0x01, 0x0a, 0x4b, -- 0x62, 0xca, 0x2c, 0x0b, 0x05, 0xa0, 0x1e, 0x54, 0x69, 0xa2, 0xfc, 0xd5, 0x72, 0xab, 0x34, 0xc9, -- 0x05, 0xd5, 0x0b, 0x82, 0x9c, 0xbf, 0xd7, 0x01, 0x72, 0x29, 0xc8, 0x85, 0x11, 0xa1, 0x1e, 0xc7, -- 0x4c, 0xa6, 0x20, 0xde, 0xf1, 0x5c, 0x60, 0xee, 0x31, 0x1c, 0xa4, 0x8c, 0x93, 0x33, 0xb9, 0x7f, -- 0xd2, 0xec, 0x2b, 0xda, 0xec, 0x05, 0xdd, 0xdc, 0xab, 0x84, 0x4e, 0xf4, 0xba, 0x3d, 0xb9, 0xcc, -- 0xb5, 0xab, 0xd0, 0x21, 0x5c, 0xc9, 0x79, 0x86, 0x05, 0x76, 0xd5, 0xcb, 0xd8, 0x6d, 0x66, 0xec, -- 0xc2, 0x9c, 0xd5, 0x03, 0xd8, 0x24, 0xd4, 0xfb, 0x3a, 0xc5, 0x69, 0x89, 0x51, 0xed, 0x32, 0x46, -- 0x03, 0x42, 0x7f, 0xa4, 0x16, 0xe4, 0x6c, 0x8e, 0xe0, 0x5a, 0xc1, 0x4a, 0x79, 0xdd, 0x0b, 0xcc, -- 0xea, 0x97, 0x31, 0xdb, 0xce, 0xb4, 0x92, 0xf1, 0x20, 0xe7, 0xf8, 0x03, 0xd8, 0x26, 0xd4, 0x7b, -- 0xee, 0x13, 0xb1, 0xc8, 0x6e, 0xed, 0x7b, 0x8c, 0x94, 0x8f, 0x6e, 0x99, 0x97, 0x36, 0x32, 0xc2, -- 0x6c, 0x5a, 0x32, 0xb2, 0xf1, 0x3d, 0x46, 0x3e, 0x52, 0x0b, 0x72, 0x36, 0xf7, 0x60, 0x40, 0xe8, -- 0xa2, 0x36, 0xeb, 0x97, 0x31, 0xe9, 0x13, 0x5a, 0xd6, 0x64, 0x0f, 0x06, 0x1c, 0x07, 0x82, 0xb2, -- 0xe2, 0x21, 0x68, 0x5e, 0xc6, 0x62, 0xc3, 0xd0, 0x67, 0x3c, 0x9c, 0x9f, 0x41, 0xe7, 0x20, 0x9d, -- 0x62, 0x31, 0x3b, 0xce, 0x82, 0xc1, 0x4b, 0x8b, 0x3f, 0xce, 0x3f, 0xab, 0xd0, 0xde, 0x9f, 0x32, -- 0x9a, 0x26, 0xa5, 0x98, 0xac, 0x2f, 0xe9, 0x62, 0x4c, 0x56, 0x24, 0x2a, 0x26, 0x6b, 0xe2, 0x0f, -- 0xa0, 0x13, 0xa9, 0xab, 0x6b, 0xe8, 0x75, 0x1c, 0x1a, 0x2c, 0x5d, 0x6a, 0xb7, 0x1d, 0x15, 0x82, -- 0xd9, 0x18, 0x20, 0x21, 0x21, 0x37, 0x6b, 0x74, 0x38, 0xea, 0x9b, 0x8c, 0xd0, 0x86, 0x68, 0xb7, -- 0x95, 0x64, 0xd1, 0xfa, 0x3d, 0x68, 0x1f, 0x4b, 0x27, 0x99, 0x05, 0xa5, 0x60, 0x94, 0x7b, 0xcf, -- 0x85, 0xe3, 0xfc, 0x12, 0x1e, 0x40, 0xf7, 0x44, 0xbb, 0xcc, 0x2c, 0xd2, 0x67, 0xe8, 0x35, 0x63, -- 0x49, 0x6e, 0xef, 0xb8, 0xe8, 0x59, 0xbd, 0x01, 0x9d, 0x93, 0x02, 0x6a, 0x34, 0x81, 0xc1, 0x12, -- 0xc9, 0x8a, 0x18, 0xb4, 0x53, 0x8c, 0x41, 0xed, 0x3b, 0x48, 0x0b, 0x2a, 0xae, 0x2c, 0xc6, 0xa5, -- 0xdf, 0x54, 0xa1, 0xf3, 0x43, 0x2c, 0x9e, 0x53, 0x76, 0xaa, 0xf5, 0x45, 0x50, 0x8f, 0xfd, 0x08, -- 0x1b, 0x8e, 0x6a, 0x8c, 0xae, 0x41, 0x93, 0x9d, 0xeb, 0x00, 0x62, 0xf6, 0x73, 0x9d, 0x9d, 0xab, -- 0xc0, 0x80, 0x5e, 0x01, 0x60, 0xe7, 0x5e, 0xe2, 0x07, 0xa7, 0xd8, 0x78, 0xb0, 0xee, 0xb6, 0xd8, -- 0xf9, 0x91, 0x46, 0xc8, 0xa3, 0xc0, 0xce, 0x3d, 0xcc, 0x18, 0x65, 0xdc, 0xc4, 0xaa, 0x26, 0x3b, -- 0x7f, 0xa0, 0x60, 0xb3, 0x36, 0x64, 0x34, 0x49, 0x70, 0xa8, 0x62, 0xb4, 0x5a, 0x7b, 0x5f, 0x23, -- 0xa4, 0x54, 0x61, 0xa5, 0x36, 0xb4, 0x54, 0x91, 0x4b, 0x15, 0xb9, 0xd4, 0x75, 0xbd, 0x52, 0x14, -- 0xa5, 0x8a, 0x4c, 0x6a, 0x53, 0x4b, 0x15, 0x05, 0xa9, 0x22, 0x97, 0xda, 0xb2, 0x6b, 0x8d, 0x54, -- 0xe7, 0xd7, 0x15, 0xd8, 0x5e, 0x4c, 0xfc, 0x4c, 0x9a, 0xfa, 0x01, 0x74, 0x02, 0xb5, 0x5f, 0xa5, -- 0x33, 0x39, 0x58, 0xda, 0x49, 0xb7, 0x1d, 0x14, 0x8e, 0xf1, 0x87, 0xd0, 0x8d, 0xb5, 0x83, 0xb3, -- 0xa3, 0x59, 0xcb, 0xf7, 0xa5, 0xe8, 0x7b, 0xb7, 0x13, 0x17, 0x20, 0x27, 0x04, 0xf4, 0x25, 0x23, -- 0x02, 0x4f, 0x04, 0xc3, 0x7e, 0xf4, 0x32, 0x0a, 0x10, 0x04, 0x75, 0x95, 0xad, 0xd4, 0x54, 0x7e, -- 0xad, 0xc6, 0xce, 0x9b, 0xb0, 0x59, 0x92, 0x62, 0x6c, 0xdd, 0x80, 0xda, 0x0c, 0xc7, 0x8a, 0x7b, -- 0xd7, 0x95, 0x43, 0xc7, 0x87, 0x81, 0x8b, 0xfd, 0xf0, 0xe5, 0x69, 0x63, 0x44, 0xd4, 0x72, 0x11, -- 0x3b, 0x80, 0x8a, 0x22, 0x8c, 0x2a, 0x56, 0xeb, 0x4a, 0x41, 0xeb, 0xc7, 0x30, 0xd8, 0x9f, 0x51, -- 0x8e, 0x27, 0x22, 0x24, 0xf1, 0xcb, 0xa8, 0x98, 0x7e, 0x01, 0x9b, 0x4f, 0xc4, 0xfc, 0x4b, 0xc9, -- 0x8c, 0x93, 0x6f, 0xf0, 0x4b, 0xb2, 0x8f, 0xd1, 0xe7, 0xd6, 0x3e, 0x46, 0x9f, 0xcb, 0x62, 0x29, -- 0xa0, 0xb3, 0x34, 0x8a, 0xd5, 0x55, 0xe8, 0xba, 0x06, 0x72, 0xf6, 0xa0, 0xa3, 0x73, 0xe8, 0x47, -- 0x34, 0x4c, 0x67, 0x78, 0xe5, 0x1d, 0xbc, 0x09, 0x90, 0xf8, 0xcc, 0x8f, 0xb0, 0xc0, 0x4c, 0x9f, -- 0xa1, 0x96, 0x5b, 0xc0, 0x38, 0xbf, 0xad, 0xc2, 0x96, 0x6e, 0x89, 0x4c, 0x74, 0x27, 0xc0, 0x9a, -- 0x30, 0x82, 0xe6, 0x09, 0xe5, 0xa2, 0xc0, 0x30, 0x83, 0xa5, 0x8a, 0x61, 0x6c, 0xb9, 0xc9, 0x61, -- 0xa9, 0x4f, 0x51, 0xbb, 0xbc, 0x4f, 0xb1, 0xd4, 0x89, 0xa8, 0x2f, 0x77, 0x22, 0xe4, 0x6d, 0xb3, -- 0x44, 0x44, 0xdf, 0xf1, 0x96, 0xdb, 0x32, 0x98, 0xc3, 0x10, 0xdd, 0x86, 0xfe, 0x54, 0x6a, 0xe9, -- 0x9d, 0x50, 0x7a, 0xea, 0x25, 0xbe, 0x38, 0x51, 0x57, 0xbd, 0xe5, 0x76, 0x15, 0xfa, 0x80, 0xd2, -- 0xd3, 0x23, 0x5f, 0x9c, 0xa0, 0x8f, 0xa1, 0x67, 0xd2, 0xc0, 0x48, 0xb9, 0x88, 0x9b, 0xc7, 0xcf, -- 0xdc, 0xa2, 0xa2, 0xf7, 0xdc, 0xee, 0x69, 0x01, 0xe2, 0xce, 0x55, 0xb8, 0x72, 0x1f, 0x73, 0xc1, -- 0xe8, 0xbc, 0xec, 0x18, 0xe7, 0xff, 0x00, 0x0e, 0x63, 0x81, 0xd9, 0x33, 0x3f, 0xc0, 0x1c, 0xbd, -- 0x5b, 0x84, 0x4c, 0x72, 0xb4, 0x31, 0xd6, 0x1d, 0xa9, 0x6c, 0xc2, 0x2d, 0xd0, 0x38, 0x63, 0x68, -- 0xb8, 0x34, 0x95, 0xe1, 0xe8, 0x75, 0x3b, 0x32, 0xeb, 0x3a, 0x66, 0x9d, 0x42, 0xba, 0x66, 0xce, -- 0x39, 0xb0, 0x25, 0x6c, 0xce, 0xce, 0x6c, 0xd1, 0x18, 0x5a, 0xc4, 0xe2, 0x4c, 0x54, 0x59, 0x16, -- 0x9d, 0x93, 0x38, 0x3f, 0x85, 0x4d, 0xcd, 0x49, 0x73, 0xb6, 0x6c, 0x5e, 0x87, 0x06, 0xb3, 0x6a, -- 0x54, 0xf2, 0x56, 0x94, 0x21, 0x32, 0x73, 0xe8, 0x86, 0x14, 0x16, 0x30, 0x1c, 0xc9, 0x9a, 0xa3, -- 0xaa, 0xb6, 0x2c, 0x47, 0x48, 0x6f, 0xc9, 0x7a, 0x3b, 0x37, 0xd3, 0x7a, 0x6b, 0x13, 0x06, 0x72, -- 0xa2, 0x24, 0xd1, 0xf9, 0x39, 0x6c, 0x3e, 0x8e, 0x67, 0x24, 0xc6, 0xfb, 0x47, 0x4f, 0x1f, 0xe1, -- 0x2c, 0x2a, 0x20, 0xa8, 0xcb, 0xec, 0x49, 0xa9, 0xd1, 0x74, 0xd5, 0x58, 0x5e, 0x93, 0xf8, 0xd8, -- 0x0b, 0x92, 0x94, 0x9b, 0xce, 0x50, 0x23, 0x3e, 0xde, 0x4f, 0x52, 0x2e, 0xc3, 0xbc, 0x7c, 0xe6, -- 0x69, 0x3c, 0x9b, 0xab, 0xbb, 0xd2, 0x74, 0xd7, 0x83, 0x24, 0x7d, 0x1c, 0xcf, 0xe6, 0xce, 0xff, -- 0xa8, 0x5a, 0x18, 0xe3, 0xd0, 0xf5, 0xe3, 0x90, 0x46, 0xf7, 0xf1, 0x59, 0x41, 0x42, 0x56, 0x77, -- 0xd9, 0x98, 0xf0, 0x6d, 0x05, 0x3a, 0xf7, 0xa6, 0x38, 0x16, 0xf7, 0xb1, 0xf0, 0xc9, 0x4c, 0xd5, -- 0x56, 0x67, 0x98, 0x71, 0x42, 0x63, 0x73, 0xf0, 0x2d, 0x28, 0x4b, 0x63, 0x12, 0x13, 0xe1, 0x85, -- 0x3e, 0x8e, 0x68, 0x6c, 0xbc, 0x00, 0x12, 0x75, 0x5f, 0x61, 0xd0, 0x9b, 0xd0, 0xd7, 0x9d, 0x3b, -- 0xef, 0xc4, 0x8f, 0xc3, 0x99, 0xbc, 0x72, 0xba, 0x93, 0xd1, 0xd3, 0xe8, 0x03, 0x83, 0x45, 0x6f, -- 0xc1, 0x86, 0xb9, 0x10, 0x39, 0x65, 0x5d, 0x51, 0xf6, 0x0d, 0xbe, 0x44, 0x9a, 0x26, 0x09, 0x65, -- 0x82, 0x7b, 0x1c, 0x07, 0x01, 0x8d, 0x12, 0x53, 0x98, 0xf4, 0x2d, 0x7e, 0xa2, 0xd1, 0xce, 0x14, -- 0x36, 0x1f, 0x4a, 0x3b, 0x8d, 0x25, 0xf9, 0x06, 0xf7, 0x22, 0x1c, 0x79, 0xc7, 0x33, 0x1a, 0x9c, -- 0x7a, 0x32, 0x4c, 0x19, 0x0f, 0xcb, 0xd4, 0x67, 0x4f, 0x22, 0x27, 0xe4, 0x1b, 0x55, 0x83, 0x4b, -- 0xaa, 0x13, 0x2a, 0x92, 0x59, 0x3a, 0xf5, 0x12, 0x46, 0x8f, 0xb1, 0x31, 0xb1, 0x1f, 0xe1, 0xe8, -- 0x40, 0xe3, 0x8f, 0x24, 0xda, 0xf9, 0x53, 0x05, 0xb6, 0xca, 0x92, 0x4c, 0xd0, 0xdd, 0x85, 0xad, -- 0xb2, 0x28, 0xf3, 0x10, 0xeb, 0x44, 0x6f, 0x50, 0x14, 0xa8, 0x9f, 0xe4, 0x0f, 0xa1, 0xab, 0xda, -- 0xb9, 0x5e, 0xa8, 0x39, 0x95, 0xd3, 0x8f, 0xe2, 0xbe, 0xb8, 0x1d, 0xbf, 0xb8, 0x4b, 0x1f, 0xc3, -- 0x35, 0x63, 0xbe, 0xb7, 0xac, 0xb6, 0x3e, 0x10, 0xdb, 0x86, 0xe0, 0xd1, 0x82, 0xf6, 0x5f, 0xc0, -- 0x30, 0x47, 0xed, 0xcd, 0x15, 0xd2, 0xfa, 0xea, 0x5d, 0xd8, 0x5c, 0x30, 0xf6, 0x5e, 0x18, 0x32, -- 0x75, 0x41, 0xeb, 0xee, 0xaa, 0x29, 0xe7, 0x2e, 0x5c, 0x9d, 0x60, 0xa1, 0xbd, 0xe1, 0x0b, 0x53, -- 0x13, 0x68, 0x66, 0x1b, 0x50, 0x9b, 0xe0, 0x40, 0x19, 0x5f, 0x73, 0xe5, 0x50, 0x1e, 0xc0, 0xa7, -- 0x1c, 0x07, 0xca, 0xca, 0x9a, 0xab, 0xc6, 0xce, 0x1f, 0x2b, 0xb0, 0x6e, 0xc2, 0xa4, 0x0c, 0xf5, -- 0x21, 0x23, 0x67, 0x98, 0x99, 0xa3, 0x67, 0x20, 0xf4, 0x06, 0xf4, 0xf4, 0xc8, 0xa3, 0x89, 0x20, -- 0x34, 0x0b, 0xbe, 0x5d, 0x8d, 0x7d, 0xac, 0x91, 0xaa, 0x53, 0xa7, 0x1a, 0x51, 0xa6, 0xe6, 0x33, -- 0x90, 0x6a, 0xb7, 0x71, 0x19, 0x19, 0x54, 0xb0, 0x6d, 0xb9, 0x06, 0x92, 0x47, 0xdd, 0xf2, 0x5b, -- 0x53, 0xfc, 0x2c, 0x28, 0x8f, 0x7a, 0x44, 0xd3, 0x58, 0x78, 0x09, 0x25, 0xb1, 0x30, 0xd1, 0x15, -- 0x14, 0xea, 0x48, 0x62, 0x9c, 0x5f, 0x55, 0xa0, 0xa1, 0xbb, 0xd5, 0xb2, 0xca, 0xcc, 0xde, 0xb8, -- 0x2a, 0x51, 0xf9, 0x82, 0x92, 0xa5, 0xdf, 0x35, 0x35, 0x96, 0xf7, 0xf8, 0x2c, 0xd2, 0x91, 0xda, -- 0xa8, 0x76, 0x16, 0xa9, 0x10, 0xfd, 0x06, 0xf4, 0xf2, 0xa7, 0x52, 0xcd, 0x6b, 0x15, 0xbb, 0x19, -- 0x56, 0x91, 0x5d, 0xa8, 0xa9, 0xf3, 0x13, 0x59, 0x5c, 0x67, 0x9d, 0xda, 0x0d, 0xa8, 0xa5, 0x99, -- 0x32, 0x72, 0x28, 0x31, 0xd3, 0xec, 0x91, 0x95, 0x43, 0x74, 0x1b, 0x7a, 0x7e, 0x18, 0x12, 0xb9, -- 0xdc, 0x9f, 0x3d, 0x24, 0x61, 0x76, 0x49, 0xcb, 0x58, 0xe7, 0x2f, 0x15, 0xe8, 0xef, 0xd3, 0x64, -- 0xfe, 0xff, 0x64, 0x86, 0x0b, 0x11, 0x44, 0x29, 0x69, 0xde, 0x58, 0x39, 0x96, 0x79, 0xe3, 0x33, -- 0x32, 0xc3, 0xfa, 0x6a, 0xe9, 0x9d, 0x6d, 0x4a, 0x84, 0xba, 0x56, 0x76, 0x32, 0x6b, 0x80, 0x75, -- 0xf5, 0xe4, 0x23, 0x1a, 0xaa, 0x0c, 0x39, 0x24, 0xcc, 0xcb, 0xda, 0x5d, 0x5d, 0x77, 0x3d, 0x24, -- 0x4c, 0x4d, 0x19, 0x43, 0xd6, 0x54, 0xc7, 0xb5, 0x68, 0x48, 0x43, 0x63, 0xa4, 0x21, 0xdb, 0xd0, -- 0xa0, 0xcf, 0x9e, 0x71, 0x2c, 0x54, 0x2e, 0x5b, 0x73, 0x0d, 0x94, 0x85, 0xb9, 0x66, 0x21, 0xcc, -- 0x5d, 0x81, 0x4d, 0xd5, 0xdb, 0x7f, 0xc2, 0xfc, 0x80, 0xc4, 0x53, 0x1b, 0x8a, 0xb7, 0x00, 0x4d, -- 0x04, 0x4d, 0x16, 0xb0, 0x63, 0x18, 0x98, 0x37, 0xe7, 0xe8, 0xc7, 0x13, 0x6b, 0xfa, 0x35, 0x68, -- 0x4a, 0xd0, 0x63, 0xf8, 0x6b, 0x1b, 0x18, 0xcd, 0xb4, 0xf3, 0x16, 0x74, 0xf4, 0xd0, 0x84, 0x81, -- 0x9c, 0x94, 0x97, 0x49, 0xf9, 0x9d, 0xbf, 0x6d, 0x98, 0x70, 0x6b, 0x6a, 0x68, 0xf4, 0x10, 0xfa, -- 0x0b, 0xff, 0x64, 0x90, 0x69, 0xaa, 0xac, 0xfe, 0x55, 0x33, 0xda, 0x1e, 0xeb, 0x7f, 0x3c, 0x63, -- 0xfb, 0x8f, 0x67, 0xfc, 0x20, 0x4a, 0xc4, 0x1c, 0x3d, 0x80, 0x5e, 0xf9, 0xef, 0x05, 0xba, 0x6e, -- 0x73, 0x90, 0x15, 0xff, 0x34, 0x2e, 0x64, 0xf3, 0x10, 0xfa, 0x0b, 0x3f, 0x32, 0xac, 0x3e, 0xab, -- 0xff, 0x6f, 0x5c, 0xc8, 0xe8, 0x2e, 0xb4, 0x0b, 0x7f, 0x2e, 0xd0, 0x50, 0x33, 0x59, 0xfe, 0x99, -- 0x71, 0x21, 0x83, 0x7d, 0xe8, 0x96, 0x7e, 0x26, 0xa0, 0x91, 0xb1, 0x67, 0xc5, 0x1f, 0x86, 0x0b, -- 0x99, 0xec, 0x41, 0xbb, 0xd0, 0xd3, 0xb7, 0x5a, 0x2c, 0xff, 0x38, 0x18, 0x5d, 0x5b, 0x31, 0x63, -- 0xb6, 0xf3, 0x00, 0xba, 0xa5, 0x0e, 0xbc, 0x55, 0x64, 0x55, 0xf7, 0x7f, 0x74, 0x7d, 0xe5, 0x9c, -- 0xe1, 0xf4, 0x10, 0xfa, 0x0b, 0xfd, 0x78, 0xeb, 0xdc, 0xd5, 0x6d, 0xfa, 0x0b, 0xcd, 0xfa, 0x5c, -- 0x6d, 0x76, 0xa1, 0xdc, 0x2a, 0x6c, 0xf6, 0x72, 0xf7, 0x7d, 0x74, 0x63, 0xf5, 0xa4, 0xd1, 0xea, -- 0x01, 0xf4, 0xca, 0x8d, 0x77, 0xcb, 0x6c, 0x65, 0x3b, 0xfe, 0xf2, 0x93, 0x53, 0xea, 0xc1, 0xe7, -- 0x27, 0x67, 0x55, 0x6b, 0xfe, 0x42, 0x46, 0xf7, 0x00, 0x4c, 0x71, 0x15, 0x92, 0x38, 0xdb, 0xb2, -- 0xa5, 0xa2, 0x2e, 0xdb, 0xb2, 0x15, 0x85, 0xd8, 0x5d, 0x00, 0x5d, 0x13, 0x85, 0x34, 0x15, 0xe8, -- 0xaa, 0x55, 0x63, 0xa1, 0x10, 0x1b, 0x0d, 0x97, 0x27, 0x96, 0x18, 0x60, 0xc6, 0x5e, 0x84, 0xc1, -- 0x67, 0x00, 0x79, 0xad, 0x65, 0x19, 0x2c, 0x55, 0x5f, 0x97, 0xf8, 0xa0, 0x53, 0xac, 0xac, 0x90, -- 0xb1, 0x75, 0x45, 0xb5, 0x75, 0x09, 0x8b, 0xfe, 0x42, 0xe6, 0x5c, 0x3e, 0x6c, 0x8b, 0x09, 0xf5, -- 0x68, 0x29, 0x7b, 0x46, 0x1f, 0x42, 0xa7, 0x98, 0x32, 0x5b, 0x2d, 0x56, 0xa4, 0xd1, 0xa3, 0x52, -- 0xda, 0x8c, 0xee, 0x42, 0xaf, 0x9c, 0x10, 0xa3, 0xc2, 0xbd, 0x58, 0x4a, 0x93, 0x47, 0xa6, 0x19, -- 0x54, 0x20, 0x7f, 0x1f, 0x20, 0x4f, 0x9c, 0xad, 0xfb, 0x96, 0x52, 0xe9, 0x05, 0xa9, 0x9f, 0x41, -- 0xaf, 0x10, 0xb7, 0x65, 0x4d, 0x78, 0xb5, 0x64, 0x70, 0x1e, 0xcd, 0x47, 0x26, 0xc3, 0x2a, 0x85, -- 0xed, 0x7b, 0xd0, 0x29, 0xbe, 0x11, 0xd6, 0xda, 0x15, 0xef, 0xc6, 0x65, 0x41, 0xaf, 0xf0, 0x9e, -- 0xd8, 0xb3, 0xbb, 0xfc, 0xc4, 0x5c, 0x16, 0xf4, 0x4a, 0xf5, 0xa8, 0x8d, 0x35, 0xab, 0x8a, 0xd4, -- 0xcb, 0x9e, 0x82, 0x72, 0xf1, 0x66, 0xbd, 0xbf, 0xb2, 0xa4, 0xbb, 0xec, 0x0c, 0x16, 0xeb, 0x14, -- 0xeb, 0x8f, 0x15, 0xb5, 0xcb, 0xf7, 0xc4, 0x84, 0x62, 0x2d, 0x52, 0x88, 0x09, 0x2b, 0x4a, 0x94, -- 0x0b, 0x19, 0x1d, 0x40, 0xff, 0xa1, 0x4d, 0x33, 0x4d, 0x0a, 0x6c, 0xd4, 0x59, 0x91, 0xf2, 0x8f, -- 0x46, 0xab, 0xa6, 0xcc, 0x2e, 0x7f, 0x0e, 0x83, 0xa5, 0xf4, 0x17, 0xdd, 0xcc, 0x5a, 0x9e, 0x2b, -- 0xf3, 0xe2, 0x0b, 0xd5, 0x3a, 0x84, 0x8d, 0xc5, 0xec, 0x17, 0xbd, 0x62, 0x36, 0x7d, 0x75, 0x56, -- 0x7c, 0x21, 0xab, 0x8f, 0xa1, 0x69, 0xb3, 0x2d, 0x64, 0x5a, 0xcb, 0x0b, 0xd9, 0xd7, 0x45, 0x4b, -- 0xf7, 0x3a, 0xdf, 0x7e, 0x77, 0xb3, 0xf2, 0xd7, 0xef, 0x6e, 0x56, 0xfe, 0xf1, 0xdd, 0xcd, 0xca, -- 0x71, 0x43, 0xcd, 0xbe, 0xff, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x45, 0xa8, 0x91, 0xab, 0x62, -- 0x22, 0x00, 0x00, -+ // 3183 bytes of a gzipped FileDescriptorProto -+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0x4b, 0x6f, 0x1c, 0xc7, -+ 0x99, 0x98, 0x07, 0x39, 0x33, 0xdf, 0xbc, 0xc8, 0x22, 0x45, 0x8d, 0x46, 0xb2, 0x2c, 0xb7, 0x6d, -+ 0x99, 0x5e, 0xaf, 0x49, 0x8b, 0x36, 0xfc, 0x84, 0x57, 0x10, 0x29, 0xad, 0xc8, 0xb5, 0xb4, 0xe2, -+ 0xf6, 0x88, 0xeb, 0x5d, 0x2c, 0x16, 0x8d, 0x66, 0x77, 0x71, 0x58, 0xe6, 0x74, 0x57, 0xbb, 0xaa, -+ 0x9a, 0xe2, 0x78, 0x81, 0x3d, 0x26, 0xd7, 0x9c, 0x72, 0xcb, 0x1f, 0x08, 0x72, 0xcb, 0x2d, 0xb9, -+ 0xe6, 0x60, 0x04, 0x39, 0x04, 0xf9, 0x01, 0x41, 0xe0, 0x9f, 0x90, 0x5f, 0x10, 0xd4, 0xab, 0x1f, -+ 0x33, 0x43, 0x1a, 0x11, 0x04, 0xe4, 0xd2, 0xe8, 0xef, 0x51, 0xdf, 0xab, 0xaa, 0xbe, 0xfa, 0xbe, -+ 0x2a, 0x68, 0xfb, 0x63, 0x1c, 0x8b, 0xad, 0x84, 0x51, 0x41, 0x51, 0x7d, 0xcc, 0x92, 0x60, 0xd8, -+ 0xa2, 0x01, 0xd1, 0x88, 0xe1, 0xc7, 0x63, 0x22, 0x4e, 0xd3, 0xe3, 0xad, 0x80, 0x46, 0xdb, 0x67, -+ 0xbe, 0xf0, 0xdf, 0x0f, 0x68, 0x2c, 0x7c, 0x12, 0x63, 0xc6, 0xb7, 0xd5, 0xc0, 0xed, 0xe4, 0x6c, -+ 0xbc, 0x2d, 0xa6, 0x09, 0xe6, 0xfa, 0x6b, 0xc6, 0xdd, 0x1c, 0x53, 0x3a, 0x9e, 0xe0, 0x6d, 0x05, -+ 0x1d, 0xa7, 0x27, 0xdb, 0x38, 0x4a, 0xc4, 0x54, 0x13, 0x9d, 0x5f, 0x54, 0x61, 0x63, 0x8f, 0x61, -+ 0x5f, 0xe0, 0x3d, 0x2b, 0xcd, 0xc5, 0xdf, 0xa6, 0x98, 0x0b, 0xf4, 0x06, 0x74, 0x32, 0x0d, 0x1e, -+ 0x09, 0x07, 0x95, 0x3b, 0x95, 0xcd, 0x96, 0xdb, 0xce, 0x70, 0x07, 0x21, 0xba, 0x0e, 0x0d, 0x7c, -+ 0x81, 0x03, 0x49, 0xad, 0x2a, 0xea, 0xb2, 0x04, 0x0f, 0x42, 0x74, 0x0f, 0xda, 0x5c, 0x30, 0x12, -+ 0x8f, 0xbd, 0x94, 0x63, 0x36, 0xa8, 0xdd, 0xa9, 0x6c, 0xb6, 0x77, 0x56, 0xb6, 0xa4, 0x4b, 0x5b, -+ 0x23, 0x45, 0x38, 0xe2, 0x98, 0xb9, 0xc0, 0xb3, 0x7f, 0x74, 0x17, 0x1a, 0x21, 0x3e, 0x27, 0x01, -+ 0xe6, 0x83, 0xfa, 0x9d, 0xda, 0x66, 0x7b, 0xa7, 0xa3, 0xd9, 0x1f, 0x2a, 0xa4, 0x6b, 0x89, 0xe8, -+ 0x5d, 0x68, 0x72, 0x41, 0x99, 0x3f, 0xc6, 0x7c, 0xb0, 0xa4, 0x18, 0xbb, 0x56, 0xae, 0xc2, 0xba, -+ 0x19, 0x19, 0xdd, 0x82, 0xda, 0xb3, 0xbd, 0x83, 0xc1, 0xb2, 0xd2, 0x0e, 0x86, 0x2b, 0xc1, 0x81, -+ 0x2b, 0xd1, 0xe8, 0x4d, 0xe8, 0x72, 0x3f, 0x0e, 0x8f, 0xe9, 0x85, 0x97, 0x90, 0x30, 0xe6, 0x83, -+ 0xc6, 0x9d, 0xca, 0x66, 0xd3, 0xed, 0x18, 0xe4, 0xa1, 0xc4, 0x39, 0x9f, 0xc3, 0xb5, 0x91, 0xf0, -+ 0x99, 0x78, 0x89, 0xe8, 0x38, 0x47, 0xb0, 0xe1, 0xe2, 0x88, 0x9e, 0xbf, 0x54, 0x68, 0x07, 0xd0, -+ 0x10, 0x24, 0xc2, 0x34, 0x15, 0x2a, 0xb4, 0x5d, 0xd7, 0x82, 0xce, 0xaf, 0x2a, 0x80, 0x1e, 0x5d, -+ 0xe0, 0xe0, 0x90, 0xd1, 0x00, 0x73, 0xfe, 0x0f, 0x9a, 0xae, 0x77, 0xa0, 0x91, 0x68, 0x03, 0x06, -+ 0x75, 0xc5, 0x6e, 0x66, 0xc1, 0x5a, 0x65, 0xa9, 0xce, 0x37, 0xb0, 0x3e, 0x22, 0xe3, 0xd8, 0x9f, -+ 0xbc, 0x42, 0x7b, 0x37, 0x60, 0x99, 0x2b, 0x99, 0xca, 0xd4, 0xae, 0x6b, 0x20, 0xe7, 0x10, 0xd0, -+ 0xd7, 0x3e, 0x11, 0xaf, 0x4e, 0x93, 0xf3, 0x3e, 0xac, 0x95, 0x24, 0xf2, 0x84, 0xc6, 0x1c, 0x2b, -+ 0x03, 0x84, 0x2f, 0x52, 0xae, 0x84, 0x2d, 0xb9, 0x06, 0x72, 0x30, 0xac, 0x3f, 0x21, 0xdc, 0xb2, -+ 0xe3, 0xbf, 0xc7, 0x84, 0x0d, 0x58, 0x3e, 0xa1, 0x2c, 0xf2, 0x85, 0xb5, 0x40, 0x43, 0x08, 0x41, -+ 0xdd, 0x67, 0x63, 0x3e, 0xa8, 0xdd, 0xa9, 0x6d, 0xb6, 0x5c, 0xf5, 0x2f, 0x57, 0xe5, 0x8c, 0x1a, -+ 0x63, 0xd7, 0x1b, 0xd0, 0x31, 0x71, 0xf7, 0x26, 0x84, 0x0b, 0xa5, 0xa7, 0xe3, 0xb6, 0x0d, 0x4e, -+ 0x8e, 0x71, 0x28, 0x6c, 0x1c, 0x25, 0xe1, 0x4b, 0x6e, 0xf8, 0x1d, 0x68, 0x31, 0xcc, 0x69, 0xca, -+ 0xe4, 0x36, 0xad, 0xaa, 0x79, 0x5f, 0xd7, 0xf3, 0xfe, 0x84, 0xc4, 0xe9, 0x85, 0x6b, 0x69, 0x6e, -+ 0xce, 0x66, 0xb6, 0x90, 0xe0, 0x2f, 0xb3, 0x85, 0x3e, 0x87, 0x6b, 0x87, 0x7e, 0xca, 0x5f, 0xc6, -+ 0x56, 0xe7, 0x0b, 0xb9, 0xfd, 0x78, 0x1a, 0xbd, 0xd4, 0xe0, 0x5f, 0x56, 0xa0, 0xb9, 0x97, 0xa4, -+ 0x47, 0xdc, 0x1f, 0x63, 0xf4, 0x3a, 0xb4, 0x05, 0x15, 0xfe, 0xc4, 0x4b, 0x25, 0xa8, 0xd8, 0xeb, -+ 0x2e, 0x28, 0x94, 0x66, 0x90, 0x61, 0xc7, 0x2c, 0x48, 0x52, 0xc3, 0x51, 0xbd, 0x53, 0xdb, 0xac, -+ 0xbb, 0x6d, 0x8d, 0xd3, 0x2c, 0x5b, 0xb0, 0xa6, 0x68, 0x1e, 0x89, 0xbd, 0x33, 0xcc, 0x62, 0x3c, -+ 0x89, 0x68, 0x88, 0xd5, 0xfa, 0xad, 0xbb, 0xab, 0x8a, 0x74, 0x10, 0x7f, 0x95, 0x11, 0xd0, 0x3f, -+ 0xc1, 0x6a, 0xc6, 0x2f, 0x37, 0xa5, 0xe2, 0xae, 0x2b, 0xee, 0xbe, 0xe1, 0x3e, 0x32, 0x68, 0xe7, -+ 0xff, 0xa1, 0xf7, 0xfc, 0x94, 0x51, 0x21, 0x26, 0x24, 0x1e, 0x3f, 0xf4, 0x85, 0x2f, 0xb3, 0x47, -+ 0x82, 0x19, 0xa1, 0x21, 0x37, 0xd6, 0x5a, 0x10, 0xbd, 0x07, 0xab, 0x42, 0xf3, 0xe2, 0xd0, 0xb3, -+ 0x3c, 0x55, 0xc5, 0xb3, 0x92, 0x11, 0x0e, 0x0d, 0xf3, 0xdb, 0xd0, 0xcb, 0x99, 0x65, 0xfe, 0x31, -+ 0xf6, 0x76, 0x33, 0xec, 0x73, 0x12, 0x61, 0xe7, 0x5c, 0xc5, 0x4a, 0x4d, 0x32, 0x7a, 0x0f, 0x5a, -+ 0x79, 0x1c, 0x2a, 0x6a, 0x85, 0xf4, 0xf4, 0x0a, 0xb1, 0xe1, 0x74, 0x9b, 0x59, 0x50, 0xbe, 0x84, -+ 0xbe, 0xc8, 0x0c, 0xf7, 0x42, 0x5f, 0xf8, 0xe5, 0x45, 0x55, 0xf6, 0xca, 0xed, 0x89, 0x12, 0xec, -+ 0x7c, 0x01, 0xad, 0x43, 0x12, 0x72, 0xad, 0x78, 0x00, 0x8d, 0x20, 0x65, 0x0c, 0xc7, 0xc2, 0xba, -+ 0x6c, 0x40, 0xb4, 0x0e, 0x4b, 0x13, 0x12, 0x11, 0x61, 0xdc, 0xd4, 0x80, 0x43, 0x01, 0x9e, 0xe2, -+ 0x88, 0xb2, 0xa9, 0x0a, 0xd8, 0x3a, 0x2c, 0x15, 0x27, 0x57, 0x03, 0xe8, 0x26, 0xb4, 0x22, 0xff, -+ 0x22, 0x9b, 0x54, 0x49, 0x69, 0x46, 0xfe, 0x85, 0x36, 0x7e, 0x00, 0x8d, 0x13, 0x9f, 0x4c, 0x82, -+ 0x58, 0x98, 0xa8, 0x58, 0x30, 0x57, 0x58, 0x2f, 0x2a, 0xfc, 0x5d, 0x15, 0xda, 0x5a, 0xa3, 0x36, -+ 0x78, 0x1d, 0x96, 0x02, 0x3f, 0x38, 0xcd, 0x54, 0x2a, 0x00, 0xdd, 0xb5, 0x86, 0x54, 0x8b, 0x49, -+ 0x38, 0xb7, 0xd4, 0x9a, 0xb6, 0x0d, 0xc0, 0x5f, 0xf8, 0x89, 0xb1, 0xad, 0x76, 0x09, 0x73, 0x4b, -+ 0xf2, 0x68, 0x73, 0x3f, 0x84, 0x8e, 0x5e, 0x77, 0x66, 0x48, 0xfd, 0x92, 0x21, 0x6d, 0xcd, 0xa5, -+ 0x07, 0xbd, 0x09, 0xdd, 0x94, 0x63, 0xef, 0x94, 0x60, 0xe6, 0xb3, 0xe0, 0x74, 0x3a, 0x58, 0xd2, -+ 0x67, 0x64, 0xca, 0xf1, 0xbe, 0xc5, 0xa1, 0x1d, 0x58, 0x92, 0xe9, 0x8f, 0x0f, 0x96, 0xd5, 0x71, -+ 0x7c, 0xab, 0x28, 0x52, 0xb9, 0xba, 0xa5, 0xbe, 0x8f, 0x62, 0xc1, 0xa6, 0xae, 0x66, 0x1d, 0x7e, -+ 0x0a, 0x90, 0x23, 0xd1, 0x0a, 0xd4, 0xce, 0xf0, 0xd4, 0xec, 0x43, 0xf9, 0x2b, 0x83, 0x73, 0xee, -+ 0x4f, 0x52, 0x1b, 0x75, 0x0d, 0x7c, 0x5e, 0xfd, 0xb4, 0xe2, 0x04, 0xd0, 0xdf, 0x9d, 0x9c, 0x11, -+ 0x5a, 0x18, 0xbe, 0x0e, 0x4b, 0x91, 0xff, 0x0d, 0x65, 0x36, 0x92, 0x0a, 0x50, 0x58, 0x12, 0x53, -+ 0x66, 0x45, 0x28, 0x00, 0xf5, 0xa0, 0x4a, 0x13, 0x15, 0xaf, 0x96, 0x5b, 0xa5, 0x49, 0xae, 0xa8, -+ 0x5e, 0x50, 0xe4, 0xfc, 0xb9, 0x0e, 0x90, 0x6b, 0x41, 0x2e, 0x0c, 0x09, 0xf5, 0x38, 0x66, 0xb2, -+ 0x04, 0xf1, 0x8e, 0xa7, 0x02, 0x73, 0x8f, 0xe1, 0x20, 0x65, 0x9c, 0x9c, 0xcb, 0xf9, 0x93, 0x6e, -+ 0x5f, 0xd3, 0x6e, 0xcf, 0xd8, 0xe6, 0x5e, 0x27, 0x74, 0xa4, 0xc7, 0xed, 0xca, 0x61, 0xae, 0x1d, -+ 0x85, 0x0e, 0xe0, 0x5a, 0x2e, 0x33, 0x2c, 0x88, 0xab, 0x5e, 0x25, 0x6e, 0x2d, 0x13, 0x17, 0xe6, -+ 0xa2, 0x1e, 0xc1, 0x1a, 0xa1, 0xde, 0xb7, 0x29, 0x4e, 0x4b, 0x82, 0x6a, 0x57, 0x09, 0x5a, 0x25, -+ 0xf4, 0x3f, 0xd4, 0x80, 0x5c, 0xcc, 0x21, 0xdc, 0x28, 0x78, 0x29, 0xb7, 0x7b, 0x41, 0x58, 0xfd, -+ 0x2a, 0x61, 0x1b, 0x99, 0x55, 0x32, 0x1f, 0xe4, 0x12, 0xff, 0x0d, 0x36, 0x08, 0xf5, 0x5e, 0xf8, -+ 0x44, 0xcc, 0x8a, 0x5b, 0xfa, 0x11, 0x27, 0xe5, 0xa1, 0x5b, 0x96, 0xa5, 0x9d, 0x8c, 0x30, 0x1b, -+ 0x97, 0x9c, 0x5c, 0xfe, 0x11, 0x27, 0x9f, 0xaa, 0x01, 0xb9, 0x98, 0x07, 0xb0, 0x4a, 0xe8, 0xac, -+ 0x35, 0x8d, 0xab, 0x84, 0xf4, 0x09, 0x2d, 0x5b, 0xb2, 0x0b, 0xab, 0x1c, 0x07, 0x82, 0xb2, 0xe2, -+ 0x22, 0x68, 0x5e, 0x25, 0x62, 0xc5, 0xf0, 0x67, 0x32, 0x9c, 0xff, 0x81, 0xce, 0x7e, 0x3a, 0xc6, -+ 0x62, 0x72, 0x9c, 0x25, 0x83, 0x57, 0x96, 0x7f, 0x9c, 0xbf, 0x56, 0xa1, 0xbd, 0x37, 0x66, 0x34, -+ 0x4d, 0x4a, 0x39, 0x59, 0x6f, 0xd2, 0xd9, 0x9c, 0xac, 0x58, 0x54, 0x4e, 0xd6, 0xcc, 0x1f, 0x41, -+ 0x27, 0x52, 0x5b, 0xd7, 0xf0, 0xeb, 0x3c, 0xb4, 0x3a, 0xb7, 0xa9, 0xdd, 0x76, 0x54, 0x48, 0x66, -+ 0x5b, 0x00, 0x09, 0x09, 0xb9, 0x19, 0xa3, 0xd3, 0x51, 0xdf, 0x54, 0x84, 0x36, 0x45, 0xbb, 0xad, -+ 0x24, 0xcb, 0xd6, 0xf7, 0xa0, 0x7d, 0x2c, 0x83, 0x64, 0x06, 0x94, 0x92, 0x51, 0x1e, 0x3d, 0x17, -+ 0x8e, 0xf3, 0x4d, 0xb8, 0x0f, 0xdd, 0x53, 0x1d, 0x32, 0x33, 0x48, 0xaf, 0xa1, 0x37, 0x8d, 0x27, -+ 0xb9, 0xbf, 0x5b, 0xc5, 0xc8, 0xea, 0x09, 0xe8, 0x9c, 0x16, 0x50, 0xc3, 0x11, 0xac, 0xce, 0xb1, -+ 0x2c, 0xc8, 0x41, 0x9b, 0xc5, 0x1c, 0xd4, 0xde, 0x41, 0x5a, 0x51, 0x71, 0x64, 0x31, 0x2f, 0xfd, -+ 0xac, 0x0a, 0xbd, 0x83, 0x58, 0x60, 0x76, 0xe2, 0x07, 0x58, 0x5b, 0x8c, 0xa0, 0x1e, 0xfb, 0x11, -+ 0x36, 0x32, 0xd5, 0x3f, 0xba, 0x01, 0x4d, 0x76, 0xa1, 0x53, 0x88, 0x99, 0xd1, 0x06, 0xbb, 0x50, -+ 0xa9, 0x01, 0xbd, 0x06, 0xc0, 0x2e, 0xbc, 0xc4, 0x0f, 0xce, 0xb0, 0x89, 0x61, 0xdd, 0x6d, 0xb1, -+ 0x8b, 0x43, 0x8d, 0x90, 0x8b, 0x81, 0x5d, 0x78, 0x98, 0x31, 0xca, 0xb8, 0xc9, 0x56, 0x4d, 0x76, -+ 0xf1, 0x48, 0xc1, 0x66, 0x6c, 0xc8, 0x68, 0x92, 0xe0, 0x50, 0x65, 0x69, 0x35, 0xf6, 0xa1, 0x46, -+ 0x48, 0xad, 0xc2, 0x6a, 0x5d, 0xd6, 0x5a, 0x45, 0xae, 0x55, 0xe4, 0x5a, 0x1b, 0x7a, 0xa4, 0x28, -+ 0x6a, 0x15, 0x99, 0xd6, 0xa6, 0xd6, 0x2a, 0x0a, 0x5a, 0x45, 0xae, 0xb5, 0x65, 0xc7, 0x1a, 0xad, -+ 0xce, 0x6f, 0xaa, 0xd0, 0x78, 0x1e, 0xa8, 0x49, 0x41, 0x77, 0xa0, 0x8d, 0xb9, 0xf0, 0x8f, 0x27, -+ 0x84, 0x9f, 0xe2, 0xd0, 0x2c, 0xf3, 0x22, 0x4a, 0xda, 0xc8, 0xa7, 0xb1, 0xc7, 0xe5, 0x09, 0x6e, -+ 0x22, 0xc3, 0xa7, 0xf1, 0x48, 0x9e, 0xe0, 0x86, 0xc4, 0x70, 0x70, 0x6e, 0xd7, 0x3a, 0x9f, 0xc6, -+ 0x2e, 0x0e, 0xce, 0xa5, 0x7d, 0x27, 0x24, 0x56, 0x39, 0xe6, 0x9e, 0x8d, 0xca, 0x09, 0x89, 0x65, -+ 0xfe, 0xb8, 0x57, 0x24, 0xee, 0x98, 0xa0, 0x58, 0xe2, 0x8e, 0xf2, 0x4c, 0xa6, 0x01, 0x49, 0x35, -+ 0x41, 0x69, 0x4a, 0x84, 0xa4, 0xaa, 0xc3, 0x79, 0x42, 0x39, 0x36, 0x01, 0xd1, 0x80, 0xf4, 0x57, -+ 0xfd, 0xe8, 0x31, 0x3a, 0x1a, 0x2d, 0x85, 0x51, 0x83, 0x6e, 0x40, 0x73, 0xe2, 0x73, 0xe1, 0xf9, -+ 0xc1, 0x99, 0x09, 0x46, 0x43, 0xc2, 0x0f, 0x82, 0x33, 0x59, 0xdd, 0xcb, 0x82, 0x1c, 0xc7, 0x03, -+ 0x50, 0x04, 0x03, 0xa9, 0xaa, 0x65, 0x42, 0x39, 0x89, 0xc7, 0x83, 0xb6, 0xa9, 0x5a, 0x34, 0xe8, -+ 0xa4, 0xd0, 0x38, 0x0a, 0x75, 0xec, 0xf2, 0xc1, 0x95, 0xd9, 0xc1, 0x36, 0xf6, 0x26, 0x60, 0x06, -+ 0x34, 0x6b, 0x45, 0x9f, 0x08, 0x26, 0x62, 0x4d, 0x76, 0xa1, 0x13, 0xbe, 0x99, 0x52, 0x43, 0xac, -+ 0xdb, 0x29, 0xd5, 0x44, 0xe7, 0x0f, 0x15, 0xe8, 0xfc, 0x3b, 0x16, 0x2f, 0x28, 0x3b, 0xb3, 0xf9, -+ 0x00, 0x88, 0x5d, 0xd6, 0xdc, 0x9c, 0x75, 0xa6, 0x3c, 0x2b, 0x2f, 0x77, 0xb7, 0xc0, 0x87, 0x5e, -+ 0x87, 0x9a, 0x08, 0x12, 0xb3, 0x73, 0x4c, 0x6b, 0x68, 0x96, 0x82, 0x2b, 0x29, 0xe8, 0x0d, 0xa8, -+ 0x8b, 0x20, 0xf9, 0xd8, 0xa4, 0x8a, 0x19, 0x0e, 0x45, 0x92, 0x32, 0xd2, 0x30, 0x29, 0xb7, 0x97, -+ 0x26, 0x24, 0xae, 0xa4, 0x48, 0x19, 0x69, 0x98, 0x7c, 0xac, 0x66, 0x76, 0x8e, 0x43, 0x91, 0x9c, -+ 0x9f, 0x56, 0x60, 0x63, 0xb6, 0xfb, 0x30, 0xbd, 0xd2, 0x47, 0xd0, 0x09, 0x54, 0xd2, 0x28, 0x25, -+ 0xc6, 0xd5, 0xb9, 0x74, 0xe2, 0xb6, 0x83, 0x42, 0x2e, 0xfd, 0x04, 0xba, 0xb1, 0x0e, 0x4f, 0x29, -+ 0x3f, 0x9a, 0xe4, 0x50, 0x8c, 0x9c, 0xdb, 0x89, 0x0b, 0x90, 0x13, 0x02, 0xfa, 0x9a, 0x11, 0x81, -+ 0x47, 0x82, 0x61, 0x3f, 0x7a, 0x15, 0x5d, 0x30, 0x82, 0xba, 0x2a, 0x99, 0x6b, 0xaa, 0xc9, 0x53, -+ 0xff, 0xce, 0x3b, 0xb0, 0x56, 0xd2, 0x62, 0x7c, 0x5d, 0x81, 0xda, 0xc4, 0x2c, 0x9f, 0xae, 0x2b, -+ 0x7f, 0x1d, 0x1f, 0x56, 0x5d, 0xec, 0x87, 0xaf, 0xce, 0x1a, 0xa3, 0xa2, 0x96, 0xab, 0xd8, 0x04, -+ 0x54, 0x54, 0x61, 0x4c, 0xb1, 0x56, 0x57, 0x0a, 0x56, 0x3f, 0x83, 0xd5, 0x3d, 0xb9, 0x8b, 0x46, -+ 0x22, 0x24, 0xf1, 0xab, 0x68, 0xdb, 0xff, 0x0f, 0xd6, 0x9e, 0x8b, 0xe9, 0xd7, 0x52, 0x18, 0x27, -+ 0xdf, 0xe1, 0x57, 0xe4, 0x1f, 0xa3, 0x2f, 0xac, 0x7f, 0x8c, 0xbe, 0x90, 0xdb, 0x32, 0xa0, 0x93, -+ 0x34, 0x8a, 0xd5, 0x12, 0xed, 0xba, 0x06, 0x72, 0x76, 0xa1, 0xa3, 0x1b, 0xb9, 0xa7, 0x34, 0x4c, -+ 0x27, 0x78, 0xe1, 0x31, 0x70, 0x1b, 0x20, 0xf1, 0x99, 0x1f, 0x61, 0x81, 0x19, 0x57, 0x25, 0x5f, -+ 0xcb, 0x2d, 0x60, 0x9c, 0x9f, 0x57, 0x61, 0x5d, 0xdf, 0xcb, 0x8d, 0xf4, 0x75, 0x94, 0x75, 0x61, -+ 0x08, 0xcd, 0x53, 0xca, 0x45, 0x41, 0x60, 0x06, 0x4b, 0x13, 0xc3, 0xd8, 0x4a, 0x93, 0xbf, 0xa5, -+ 0xcb, 0xb2, 0xda, 0xd5, 0x97, 0x65, 0x73, 0xd7, 0x61, 0xf5, 0xf9, 0xeb, 0x30, 0x99, 0x00, 0x2d, -+ 0x13, 0xd1, 0xc7, 0x4c, 0xcb, 0x6d, 0x19, 0xcc, 0x41, 0x88, 0xee, 0x42, 0x7f, 0x2c, 0xad, 0xf4, -+ 0x4e, 0x29, 0x3d, 0xf3, 0x12, 0x5f, 0x9c, 0xaa, 0xc4, 0xda, 0x72, 0xbb, 0x0a, 0xbd, 0x4f, 0xe9, -+ 0xd9, 0xa1, 0x2f, 0x4e, 0xd1, 0x67, 0xd0, 0x33, 0xbd, 0x48, 0xa4, 0x42, 0xc4, 0x4d, 0x05, 0x66, -+ 0x76, 0x51, 0x31, 0x7a, 0x6e, 0xf7, 0xac, 0x00, 0x71, 0xe7, 0x3a, 0x5c, 0x7b, 0x88, 0xb9, 0x60, -+ 0x74, 0x5a, 0x0e, 0x8c, 0xf3, 0x2f, 0x00, 0x07, 0x79, 0xfe, 0xf9, 0xa0, 0x08, 0x99, 0xac, 0xb5, -+ 0xb2, 0xa5, 0xaf, 0x45, 0x33, 0x82, 0x5b, 0xe0, 0x71, 0xb6, 0x60, 0xd9, 0xa5, 0xa9, 0x3c, 0x11, -+ 0xdf, 0xb2, 0x7f, 0x66, 0x5c, 0xc7, 0x8c, 0x53, 0x48, 0xd7, 0xd0, 0x9c, 0x7d, 0x7b, 0x8f, 0x92, -+ 0x8b, 0x33, 0x53, 0xb4, 0x05, 0xad, 0x2c, 0x13, 0x9a, 0xac, 0x32, 0xaf, 0x3a, 0x67, 0x71, 0xfe, -+ 0x1b, 0xd6, 0xb4, 0x24, 0x2d, 0xd9, 0x8a, 0x79, 0x0b, 0x96, 0x99, 0x35, 0xa3, 0x92, 0xdf, 0x87, -+ 0x1a, 0x26, 0x43, 0x43, 0xb7, 0xa4, 0xb2, 0x80, 0xe1, 0xc8, 0x1e, 0x9b, 0x4d, 0x37, 0x47, 0xc8, -+ 0x68, 0x3d, 0x21, 0x5c, 0xe4, 0x6e, 0xda, 0x68, 0xad, 0xc1, 0xaa, 0x24, 0x94, 0x34, 0x3a, 0xff, -+ 0x0b, 0x6b, 0xcf, 0xe2, 0x09, 0x89, 0xf1, 0xde, 0xe1, 0xd1, 0x53, 0x9c, 0x65, 0x05, 0x04, 0x75, -+ 0x75, 0xde, 0x55, 0x94, 0x74, 0xf5, 0x2f, 0xb7, 0x49, 0x7c, 0xec, 0x05, 0x49, 0xca, 0xcd, 0xf5, -+ 0xe4, 0x72, 0x7c, 0xbc, 0x97, 0xa4, 0x5c, 0x9e, 0x81, 0xb2, 0xd6, 0xa4, 0xf1, 0x64, 0xaa, 0xf6, -+ 0x4a, 0xd3, 0x6d, 0x04, 0x49, 0xfa, 0x2c, 0x9e, 0x4c, 0x9d, 0x7f, 0x56, 0x17, 0x32, 0x18, 0x87, -+ 0xae, 0x1f, 0x87, 0x34, 0x7a, 0x88, 0xcf, 0x0b, 0x1a, 0xb2, 0xe6, 0xdf, 0xe6, 0x84, 0xef, 0x2b, -+ 0xd0, 0x79, 0x30, 0xc6, 0xb1, 0x78, 0x88, 0x85, 0x4f, 0x26, 0xaa, 0xc1, 0x3f, 0xc7, 0x8c, 0x13, -+ 0x1a, 0x9b, 0x85, 0x6f, 0x41, 0xf4, 0x3a, 0xb4, 0x49, 0x4c, 0x84, 0x17, 0xfa, 0x38, 0xa2, 0xb1, -+ 0x89, 0x02, 0x48, 0xd4, 0x43, 0x85, 0x41, 0xef, 0x40, 0x5f, 0x5f, 0x1f, 0x7b, 0xa7, 0x7e, 0x1c, -+ 0x4e, 0xe4, 0x96, 0xd3, 0xd7, 0x69, 0x3d, 0x8d, 0xde, 0x37, 0x58, 0xf4, 0x2e, 0xac, 0x98, 0x0d, -+ 0x91, 0x73, 0xd6, 0x15, 0x67, 0xdf, 0xe0, 0x4b, 0xac, 0x69, 0x92, 0x50, 0x26, 0xb8, 0xc7, 0x71, -+ 0x10, 0xd0, 0x28, 0x31, 0xdd, 0x71, 0xdf, 0xe2, 0x47, 0x1a, 0xed, 0x8c, 0x61, 0xed, 0xb1, 0xf4, -+ 0xd3, 0x78, 0x92, 0x4f, 0x70, 0x2f, 0xc2, 0x91, 0x77, 0x3c, 0xa1, 0xc1, 0x99, 0x27, 0xd3, 0x94, -+ 0x89, 0xb0, 0xac, 0xbf, 0x77, 0x25, 0x72, 0x44, 0xbe, 0x53, 0x17, 0x41, 0x92, 0xeb, 0x94, 0x8a, -+ 0x64, 0x92, 0x8e, 0xbd, 0x84, 0xd1, 0x63, 0x6c, 0x5c, 0xec, 0x47, 0x38, 0xda, 0xd7, 0xf8, 0x43, -+ 0x89, 0x76, 0x7e, 0x5b, 0x81, 0xf5, 0xb2, 0x26, 0x93, 0x74, 0xb7, 0x61, 0xbd, 0xac, 0xca, 0xd4, -+ 0x82, 0xba, 0x9e, 0x58, 0x2d, 0x2a, 0xd4, 0x55, 0xe1, 0x27, 0xd0, 0x55, 0x6f, 0x0a, 0x5e, 0xa8, -+ 0x25, 0x95, 0x8f, 0xb9, 0xe2, 0xbc, 0xb8, 0x1d, 0xbf, 0x38, 0x4b, 0x9f, 0xc1, 0x0d, 0xe3, 0xbe, -+ 0x37, 0x6f, 0xb6, 0x5e, 0x10, 0x1b, 0x86, 0xe1, 0xe9, 0x8c, 0xf5, 0x4f, 0x60, 0x90, 0xa3, 0x76, -+ 0xa7, 0x0a, 0x69, 0x63, 0xf5, 0x01, 0xac, 0xcd, 0x38, 0xfb, 0x20, 0x0c, 0x99, 0xda, 0xa0, 0x75, -+ 0x77, 0x11, 0xc9, 0xb9, 0x0f, 0xd7, 0x47, 0x58, 0xe8, 0x68, 0xf8, 0xc2, 0x34, 0xa6, 0x5a, 0xd8, -+ 0x0a, 0xd4, 0x46, 0x38, 0x50, 0xce, 0xd7, 0x5c, 0xf9, 0x2b, 0x17, 0xe0, 0x11, 0xc7, 0x81, 0xf2, -+ 0xb2, 0xe6, 0xaa, 0x7f, 0xe7, 0xd7, 0x15, 0x68, 0x98, 0x34, 0x29, 0x53, 0x7d, 0xc8, 0xc8, 0x39, -+ 0x66, 0x66, 0xe9, 0x19, 0x08, 0xbd, 0x0d, 0x3d, 0xfd, 0xe7, 0xd1, 0x44, 0x10, 0x9a, 0x25, 0xdf, -+ 0xae, 0xc6, 0x3e, 0xd3, 0x48, 0x75, 0x5d, 0xac, 0x6e, 0x43, 0xcd, 0xc5, 0x83, 0x81, 0xd4, 0x9d, -+ 0x2f, 0x97, 0x99, 0x41, 0x25, 0xdb, 0x96, 0x6b, 0x20, 0xb9, 0xd4, 0xad, 0xbc, 0x25, 0x25, 0xcf, -+ 0x82, 0x72, 0xa9, 0x47, 0x34, 0x8d, 0x85, 0x97, 0x50, 0x12, 0x0b, 0x93, 0x5d, 0x41, 0xa1, 0x0e, -+ 0x25, 0xc6, 0xf9, 0x49, 0x05, 0x96, 0xf5, 0x93, 0x09, 0xea, 0x41, 0x35, 0x3b, 0xe3, 0xaa, 0x44, -+ 0xd5, 0x0b, 0x4a, 0x97, 0x3e, 0xd7, 0xd4, 0xbf, 0xdc, 0xc7, 0xe7, 0x91, 0xce, 0xd4, 0xc6, 0xb4, -+ 0xf3, 0x48, 0xa5, 0xe8, 0xb7, 0xa1, 0x97, 0x1f, 0x95, 0x8a, 0xae, 0x4d, 0xec, 0x66, 0x58, 0xc5, -+ 0x76, 0xa9, 0xa5, 0xce, 0x7f, 0x01, 0xe4, 0x4f, 0x07, 0x32, 0xe4, 0x69, 0x66, 0x8c, 0xfc, 0x95, -+ 0x98, 0x71, 0x76, 0xc8, 0xca, 0x5f, 0x74, 0x17, 0x7a, 0x7e, 0x18, 0x12, 0x39, 0xdc, 0x9f, 0x3c, -+ 0x26, 0x61, 0xb6, 0x49, 0xcb, 0x58, 0xe7, 0xf7, 0x15, 0xe8, 0xef, 0xd1, 0x64, 0xfa, 0xaf, 0x64, -+ 0x82, 0x0b, 0x19, 0x44, 0x19, 0x69, 0xce, 0x58, 0xf9, 0xaf, 0xab, 0xff, 0x09, 0xd6, 0x5b, 0x4b, -+ 0xcf, 0x6c, 0x53, 0x22, 0xd4, 0xb6, 0xb2, 0xc4, 0xec, 0x16, 0xb6, 0xab, 0x89, 0x4f, 0x69, 0xa8, -+ 0x9a, 0xb4, 0x90, 0x30, 0x2f, 0xbb, 0x73, 0xed, 0xba, 0x8d, 0x90, 0x30, 0x45, 0x32, 0x8e, 0x2c, -+ 0xa9, 0x6b, 0xff, 0xa2, 0x23, 0xcb, 0x1a, 0x23, 0x1d, 0xd9, 0x80, 0x65, 0x7a, 0x72, 0xc2, 0xb1, -+ 0x50, 0xdd, 0x43, 0xcd, 0x35, 0x50, 0x96, 0xe6, 0x9a, 0x85, 0x34, 0x77, 0x0d, 0xd6, 0xd4, 0x03, -+ 0xd3, 0x73, 0xe6, 0x07, 0x24, 0x1e, 0xdb, 0x54, 0xbc, 0x0e, 0x68, 0x24, 0x68, 0x32, 0x83, 0xdd, -+ 0x82, 0x55, 0x73, 0xe6, 0x1c, 0xfe, 0xe7, 0xc8, 0xba, 0x7e, 0x03, 0x9a, 0x12, 0xf4, 0x18, 0xfe, -+ 0xd6, 0x26, 0x46, 0x43, 0x76, 0xde, 0x85, 0x8e, 0xfe, 0x35, 0x69, 0x20, 0x67, 0xe5, 0x65, 0x56, -+ 0xbe, 0xf3, 0xa7, 0x15, 0x93, 0x6e, 0xcd, 0x45, 0x0e, 0x7a, 0x0c, 0xfd, 0x99, 0x87, 0x41, 0x64, -+ 0x6e, 0xf6, 0x16, 0xbf, 0x17, 0x0e, 0x37, 0xb6, 0xf4, 0x43, 0xe3, 0x96, 0x7d, 0x68, 0xdc, 0x7a, -+ 0x14, 0x25, 0x62, 0x8a, 0x1e, 0x41, 0xaf, 0xfc, 0x84, 0x86, 0x6e, 0xda, 0x1a, 0x64, 0xc1, 0xc3, -+ 0xda, 0xa5, 0x62, 0x1e, 0x43, 0x7f, 0xe6, 0x35, 0xcd, 0xda, 0xb3, 0xf8, 0x91, 0xed, 0x52, 0x41, -+ 0xf7, 0xa1, 0x5d, 0x78, 0x3e, 0x43, 0x03, 0x2d, 0x64, 0xfe, 0x45, 0xed, 0x52, 0x01, 0x7b, 0xd0, -+ 0x2d, 0xbd, 0x68, 0xa1, 0xa1, 0xf1, 0x67, 0xc1, 0x33, 0xd7, 0xa5, 0x42, 0x76, 0xa1, 0x5d, 0x78, -+ 0x58, 0xb2, 0x56, 0xcc, 0xbf, 0x5e, 0x0d, 0x6f, 0x2c, 0xa0, 0x98, 0xe9, 0xdc, 0x87, 0x6e, 0xe9, -+ 0x19, 0xc8, 0x1a, 0xb2, 0xe8, 0x09, 0x6a, 0x78, 0x73, 0x21, 0xcd, 0x48, 0x7a, 0x0c, 0xfd, 0x99, -+ 0x47, 0x21, 0x1b, 0xdc, 0xc5, 0x6f, 0x45, 0x97, 0xba, 0xf5, 0x95, 0x9a, 0xec, 0x42, 0xbb, 0x55, -+ 0x98, 0xec, 0xf9, 0x27, 0xa0, 0xe1, 0xad, 0xc5, 0x44, 0x63, 0xd5, 0x23, 0xe8, 0x95, 0x5f, 0x7f, -+ 0xac, 0xb0, 0x85, 0x6f, 0x42, 0x57, 0xaf, 0x9c, 0xd2, 0x43, 0x50, 0xbe, 0x72, 0x16, 0xbd, 0x0f, -+ 0x5d, 0x2a, 0xe8, 0x01, 0x80, 0x69, 0xae, 0x42, 0x12, 0x67, 0x53, 0x36, 0xd7, 0xd4, 0x65, 0x53, -+ 0xb6, 0xa0, 0x11, 0xbb, 0x0f, 0xa0, 0x7b, 0xa2, 0x90, 0xa6, 0x02, 0x5d, 0xb7, 0x66, 0xcc, 0x34, -+ 0x62, 0xc3, 0xc1, 0x3c, 0x61, 0x4e, 0x00, 0x66, 0xec, 0x65, 0x04, 0x7c, 0x09, 0x90, 0xf7, 0x5a, -+ 0x56, 0xc0, 0x5c, 0xf7, 0x75, 0x45, 0x0c, 0x3a, 0xc5, 0xce, 0x0a, 0x19, 0x5f, 0x17, 0x74, 0x5b, -+ 0x57, 0x88, 0xe8, 0xcf, 0x54, 0xce, 0xe5, 0xc5, 0x36, 0x5b, 0x50, 0x0f, 0xe7, 0xaa, 0x67, 0xf4, -+ 0x09, 0x74, 0x8a, 0x25, 0xb3, 0xb5, 0x62, 0x41, 0x19, 0x3d, 0x2c, 0x95, 0xcd, 0xe8, 0x3e, 0xf4, -+ 0xca, 0x05, 0x31, 0x2a, 0xec, 0x8b, 0xb9, 0x32, 0x79, 0xb8, 0x32, 0x73, 0xd1, 0xc1, 0xd1, 0x87, -+ 0x00, 0x79, 0xe1, 0x6c, 0xc3, 0x37, 0x57, 0x4a, 0xcf, 0x68, 0xfd, 0x12, 0x7a, 0x85, 0xbc, 0x2d, -+ 0x7b, 0xc2, 0xeb, 0x25, 0x87, 0xf3, 0x6c, 0x3e, 0x34, 0x15, 0x56, 0x29, 0x6d, 0x3f, 0x80, 0x4e, -+ 0xf1, 0x8c, 0xb0, 0xde, 0x2e, 0x38, 0x37, 0xae, 0x4a, 0x7a, 0x85, 0xf3, 0xc4, 0xae, 0xdd, 0xf9, -+ 0x23, 0xe6, 0xaa, 0xa4, 0x57, 0xea, 0x47, 0x6d, 0xae, 0x59, 0xd4, 0xa4, 0x5e, 0x75, 0x14, 0x94, -+ 0x9b, 0x37, 0x1b, 0xfd, 0x85, 0x2d, 0xdd, 0x55, 0x6b, 0xb0, 0xd8, 0xa7, 0xd8, 0x78, 0x2c, 0xe8, -+ 0x5d, 0x7e, 0x24, 0x27, 0x14, 0x7b, 0x91, 0x42, 0x4e, 0x58, 0xd0, 0xa2, 0x5c, 0x2a, 0x68, 0x1f, -+ 0xfa, 0x8f, 0x6d, 0x99, 0x69, 0x4a, 0x60, 0x63, 0xce, 0x82, 0x92, 0x7f, 0x38, 0x5c, 0x44, 0x32, -+ 0xb3, 0xfc, 0x15, 0xac, 0xce, 0x95, 0xbf, 0xe8, 0x76, 0x76, 0xef, 0xbe, 0xb0, 0x2e, 0xbe, 0xd4, -+ 0xac, 0x03, 0x58, 0x99, 0xad, 0x7e, 0xd1, 0x6b, 0x66, 0xd2, 0x17, 0x57, 0xc5, 0x97, 0x8a, 0xfa, -+ 0x0c, 0x9a, 0xb6, 0xda, 0x42, 0xe6, 0x7d, 0x63, 0xa6, 0xfa, 0xba, 0x6c, 0xe8, 0x6e, 0xe7, 0xfb, -+ 0x1f, 0x6e, 0x57, 0xfe, 0xf8, 0xc3, 0xed, 0xca, 0x5f, 0x7e, 0xb8, 0x5d, 0x39, 0x5e, 0x56, 0xd4, -+ 0x0f, 0xff, 0x16, 0x00, 0x00, 0xff, 0xff, 0x9a, 0x16, 0x45, 0x76, 0xe7, 0x24, 0x00, 0x00, - } -diff --git a/virtcontainers/container.go b/virtcontainers/container.go -index 1b89f6ac..29e17bab 100644 ---- a/virtcontainers/container.go -+++ b/virtcontainers/container.go -@@ -195,25 +195,82 @@ type CgroupStats struct { - HugetlbStats map[string]HugetlbStats `json:"hugetlb_stats,omitempty"` - } - --// NetworkStats describe all network stats. --type NetworkStats struct { -- // Name is the name of the network interface. -- Name string `json:"name,omitempty"` -- -- RxBytes uint64 `json:"rx_bytes,omitempty"` -- RxPackets uint64 `json:"rx_packets,omitempty"` -- RxErrors uint64 `json:"rx_errors,omitempty"` -- RxDropped uint64 `json:"rx_dropped,omitempty"` -- TxBytes uint64 `json:"tx_bytes,omitempty"` -- TxPackets uint64 `json:"tx_packets,omitempty"` -- TxErrors uint64 `json:"tx_errors,omitempty"` -- TxDropped uint64 `json:"tx_dropped,omitempty"` --} -- - // ContainerStats describes a container stats. - type ContainerStats struct { - CgroupStats *CgroupStats -- NetworkStats []*NetworkStats -+ NetworkStats *NetworkStats -+} -+ -+type InterfaceStats struct { -+ // The name of the interface. -+ Name string `json:"name"` -+ // Cumulative count of bytes received. -+ RxBytes uint64 `json:"rx_bytes"` -+ // Cumulative count of packets received. -+ RxPackets uint64 `json:"rx_packets"` -+ // Cumulative count of receive errors encountered. -+ RxErrors uint64 `json:"rx_errors"` -+ // Cumulative count of packets dropped while receiving. -+ RxDropped uint64 `json:"rx_dropped"` -+ // Cumulative count of bytes transmitted. -+ TxBytes uint64 `json:"tx_bytes"` -+ // Cumulative count of packets transmitted. -+ TxPackets uint64 `json:"tx_packets"` -+ // Cumulative count of transmit errors encountered. -+ TxErrors uint64 `json:"tx_errors"` -+ // Cumulative count of packets dropped while transmitting. -+ TxDropped uint64 `json:"tx_dropped"` -+} -+ -+type NetworkStats struct { -+ Interfaces []InterfaceStats `json:"interfaces,omitempty"` -+ // TCP connection stats (Established, Listen...) -+ Tcp TcpStat `json:"tcp,omitempty"` -+ // TCP6 connection stats (Established, Listen...) -+ Tcp6 TcpStat `json:"tcp6,omitempty"` -+ // UDP connection stats -+ Udp UdpStat `json:"udp,omitempty"` -+ // UDP6 connection stats -+ Udp6 UdpStat `json:"udp6,omitempty"` -+} -+ -+type TcpStat struct { -+ // Count of TCP connections in state "Established" -+ Established uint64 `json:"established,omitempty"` -+ // Count of TCP connections in state "Syn_Sent" -+ SynSent uint64 `json:"syn_sent,omitempty"` -+ // Count of TCP connections in state "Syn_Recv" -+ SynRecv uint64 `json:"syn_recv,omitempty"` -+ // Count of TCP connections in state "Fin_Wait1" -+ FinWait1 uint64 `json:"fin_wait1,omitempty"` -+ // Count of TCP connections in state "Fin_Wait2" -+ FinWait2 uint64 `json:"fin_wait2,omitempty"` -+ // Count of TCP connections in state "Time_Wait -+ TimeWait uint64 `json:"time_wait,omitempty"` -+ // Count of TCP connections in state "Close" -+ Close uint64 `json:"close,omitempty"` -+ // Count of TCP connections in state "Close_Wait" -+ CloseWait uint64 `json:"close_wait,omitempty"` -+ // Count of TCP connections in state "Listen_Ack" -+ LastAck uint64 `json:"last_ack,omitempty"` -+ // Count of TCP connections in state "Listen" -+ Listen uint64 `json:"listen,omitempty"` -+ // Count of TCP connections in state "Closing" -+ Closing uint64 `json:"closing,omitempty"` -+} -+ -+type UdpStat struct { -+ // Count of UDP sockets in state "Listen" -+ Listen uint64 `json:"listen,omitempty"` -+ -+ // Count of UDP packets dropped by the IP stack -+ Dropped uint64 `json:"dropped,omitempty"` -+ -+ // Count of packets Queued for Receieve -+ RxQueued uint64 `json:"rx_queued,omitempty"` -+ -+ // Count of packets Queued for Transmit -+ TxQueued uint64 `json:"tx_queued,omitempty"` - } - - // ContainerResources describes container resources -diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go -index 38e9a204..0f03c9d9 100644 ---- a/virtcontainers/kata_agent.go -+++ b/virtcontainers/kata_agent.go -@@ -1894,18 +1894,30 @@ func (k *kataAgent) statsContainer(sandbox *Sandbox, c Container) (*ContainerSta - return nil, fmt.Errorf("irregular response container stats") - } - -- data, err := json.Marshal(stats.CgroupStats) -+ cgroupData, err := json.Marshal(stats.CgroupStats) - if err != nil { - return nil, err - } - - var cgroupStats CgroupStats -- err = json.Unmarshal(data, &cgroupStats) -+ err = json.Unmarshal(cgroupData, &cgroupStats) -+ if err != nil { -+ return nil, err -+ } -+ -+ networkData, err := json.Marshal(stats.NetworkStats) -+ if err != nil { -+ return nil, err -+ } -+ -+ var networkStats NetworkStats -+ err = json.Unmarshal(networkData, &networkStats) - if err != nil { - return nil, err - } - containerStats := &ContainerStats{ -- CgroupStats: &cgroupStats, -+ CgroupStats: &cgroupStats, -+ NetworkStats: &networkStats, - } - return containerStats, nil - } --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0048-console-fix-the-file-resource-leak.patch b/runtime/patches/0048-console-fix-the-file-resource-leak.patch deleted file mode 100644 index 8cc751c..0000000 --- a/runtime/patches/0048-console-fix-the-file-resource-leak.patch +++ /dev/null @@ -1,34 +0,0 @@ -From d3b8e21829bd671a1717fed8ab645ad2d447899a Mon Sep 17 00:00:00 2001 -From: holyfei -Date: Wed, 19 Aug 2020 22:18:05 +0800 -Subject: [PATCH 48/50] console: fix the file resource leak - -reason: newConsole will open a file, if error occurs in the -middle proccess, the file resource may leak, use defer close -to prevent it - -Signed-off-by: yangfeiyu ---- - cli/console.go | 6 ++++++ - 1 file changed, 6 insertions(+) - -diff --git a/cli/console.go b/cli/console.go -index c1555c9f..97dfee99 100644 ---- a/cli/console.go -+++ b/cli/console.go -@@ -47,6 +47,12 @@ func newConsole() (*Console, error) { - if err != nil { - return nil, err - } -+ defer func() { -+ if err != nil { -+ master.Close() -+ } -+ }() -+ - if err := saneTerminal(master); err != nil { - return nil, err - } --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0049-container-fix-the-write-operation-transparently-tran.patch b/runtime/patches/0049-container-fix-the-write-operation-transparently-tran.patch deleted file mode 100644 index 4d8892e..0000000 --- a/runtime/patches/0049-container-fix-the-write-operation-transparently-tran.patch +++ /dev/null @@ -1,58 +0,0 @@ -From a6fab7014922d85b1105b44fdbb98239b22d3e00 Mon Sep 17 00:00:00 2001 -From: holyfei -Date: Wed, 19 Aug 2020 22:31:57 +0800 -Subject: [PATCH 49/50] container: fix the write operation transparently - transmitted to the host - -reason:fix the write operation transparently transmitted to the host -when we fullfill the "/etc/hosts","/etc/resolv.conf","/etc/hostname" file in the container, -for example: -```bash -$ docker exec -ti 63 bash - -Signed-off-by: yangfeiyu ---- - virtcontainers/container.go | 16 ++++++++++++++-- - 1 file changed, 14 insertions(+), 2 deletions(-) - -diff --git a/virtcontainers/container.go b/virtcontainers/container.go -index 1b89f6ac..6edcb3f2 100644 ---- a/virtcontainers/container.go -+++ b/virtcontainers/container.go -@@ -51,6 +51,12 @@ var cdromMajors = map[int64]string{ - 32: "CM206_CDROM_MAJOR", - } - -+var safeCopyFiles = map[string]struct{}{ -+ "resolv.conf": {}, -+ "hostname": {}, -+ "hosts": {}, -+} -+ - // https://github.com/torvalds/linux/blob/master/include/uapi/linux/major.h - // #define FLOPPY_MAJOR 2 - const floppyMajor = int64(2) -@@ -452,12 +458,18 @@ func (c *Container) shareFiles(m Mount, idx int, hostSharedDir, guestSharedDir s - } - - filename := fmt.Sprintf("%s-%s-%s", c.id, hex.EncodeToString(randBytes), filepath.Base(m.Destination)) -- guestDest := filepath.Join(guestSharedDir, filename) -+ var guestDest string -+ _, needCopy := safeCopyFiles[filepath.Base(m.Destination)] -+ if needCopy { -+ guestDest = filepath.Join(kataGuestStorageDir, filename) -+ } else { -+ guestDest = filepath.Join(guestSharedDir, filename) -+ } - - // copy file to contaier's rootfs if filesystem sharing is not supported, otherwise - // bind mount it in the shared directory. - caps := c.sandbox.hypervisor.capabilities() -- if !caps.IsFsSharingSupported() { -+ if !caps.IsFsSharingSupported() || needCopy { - c.Logger().Debug("filesystem sharing is not supported, files will be copied") - - fileInfo, err := os.Stat(m.Source) --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0050-runtime-add-kata-network-upate-iface-subcommand.patch b/runtime/patches/0050-runtime-add-kata-network-upate-iface-subcommand.patch deleted file mode 100644 index cf16d25..0000000 --- a/runtime/patches/0050-runtime-add-kata-network-upate-iface-subcommand.patch +++ /dev/null @@ -1,358 +0,0 @@ -From f943e5d91f6922c599b46727e2dfd5e8e7e5bea8 Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Wed, 19 Aug 2020 22:43:54 +0800 -Subject: [PATCH 50/50] runtime: add kata-network upate-iface subcommand - -reason: add kata-network update-iface subcommand to -update the exist interface in the Kata VM. - -Signed-off-by: jiangpengfei ---- - cli/network.go | 37 +++++++++- - virtcontainers/api.go | 25 +++++-- - virtcontainers/implementation.go | 5 ++ - virtcontainers/interfaces.go | 1 + - virtcontainers/pkg/vcmock/mock.go | 8 +++ - virtcontainers/pkg/vcmock/types.go | 1 + - virtcontainers/sandbox.go | 137 +++++++++++++++++++++++++++++++++++++ - 7 files changed, 207 insertions(+), 7 deletions(-) - -diff --git a/cli/network.go b/cli/network.go -index 3dd0971e..7dce0528 100644 ---- a/cli/network.go -+++ b/cli/network.go -@@ -65,8 +65,9 @@ var addIfaceCommand = cli.Command{ - "hwAddr":"", - "linkType":"tap", - "vhostUserSocket":"" -+ "queues": - } -- device,name,mtu,hwAddr are required, IPAddresses and vhostUserSocket are optional. -+ device,name,mtu,hwAddr are required, IPAddresses, queues and vhostUserSocket are optional. - `, - Flags: []cli.Flag{}, - Action: func(context *cli.Context) error { -@@ -104,6 +105,34 @@ var delIfaceCommand = cli.Command{ - }, - } - -+var updateIfaceCommand = cli.Command{ -+ Name: "update-iface", -+ Usage: "update the interface in container", -+ ArgsUsage: `update-iface file or - for stdin -+ file or stdin for example: -+ { -+ "device":"", -+ "name":"", -+ "IPAddresses":[{"address":"","mask":""}], -+ "mtu":0, -+ "hwAddr":"", -+ "linkType":"", -+ "vhostUserSocket":"", -+ "queues":0 -+ } -+ name is required,IPAddresses is optional,device,mtu,hwAddr,linkType,queues and vhostUserSocket should be 0 or empty. -+ `, -+ Flags: []cli.Flag{}, -+ Action: func(context *cli.Context) error { -+ ctx, err := cliContextToContext(context) -+ if err != nil { -+ return err -+ } -+ -+ return networkModifyCommand(ctx, context.Args().First(), context.Args().Get(1), interfaceType, vcTypes.NetworkOpUpdate) -+ }, -+} -+ - var listIfacesCommand = cli.Command{ - Name: "list-ifaces", - Usage: "list network interfaces in a container", -@@ -262,6 +291,12 @@ func networkModifyCommand(ctx context.Context, containerID, input string, opType - kataLog.WithField("resulting-interface", fmt.Sprintf("%+v", resultingInf)). - WithError(err).Error("delete interface failed") - } -+ case vcTypes.NetworkOpUpdate: -+ resultingInf, err = vci.UpdateInterface(ctx, sandboxID, inf) -+ if err != nil { -+ kataLog.WithField("resulting-interface", fmt.Sprintf("%+v", resultingInf)). -+ WithError(err).Error("update interface failed") -+ } - } - - json.NewEncoder(output).Encode(resultingInf) -diff --git a/virtcontainers/api.go b/virtcontainers/api.go -index 7036df8c..ca5412a9 100644 ---- a/virtcontainers/api.go -+++ b/virtcontainers/api.go -@@ -889,7 +889,7 @@ func AddDevice(ctx context.Context, sandboxID string, info deviceConfig.DeviceIn - return s.AddDevice(info) - } - --func toggleInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface, add bool) (*vcTypes.Interface, error) { -+func toggleInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface, op vcTypes.NetworkOp) (*vcTypes.Interface, error) { - if sandboxID == "" { - return nil, vcTypes.ErrNeedSandboxID - } -@@ -906,11 +906,16 @@ func toggleInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interfa - } - defer s.releaseStatelessSandbox() - -- if add { -+ switch op { -+ case vcTypes.NetworkOpAdd: - return s.AddInterface(inf) -+ case vcTypes.NetworkOpRemove: -+ return s.RemoveInterface(inf) -+ case vcTypes.NetworkOpUpdate: -+ return s.UpdateInterface(inf) -+ default: -+ return nil, fmt.Errorf("operation is not found") - } -- -- return s.RemoveInterface(inf) - } - - // AddInterface is the virtcontainers add interface entry point. -@@ -918,7 +923,7 @@ func AddInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) - span, ctx := trace(ctx, "AddInterface") - defer span.Finish() - -- return toggleInterface(ctx, sandboxID, inf, true) -+ return toggleInterface(ctx, sandboxID, inf, vcTypes.NetworkOpAdd) - } - - // RemoveInterface is the virtcontainers remove interface entry point. -@@ -926,7 +931,15 @@ func RemoveInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interfa - span, ctx := trace(ctx, "RemoveInterface") - defer span.Finish() - -- return toggleInterface(ctx, sandboxID, inf, false) -+ return toggleInterface(ctx, sandboxID, inf, vcTypes.NetworkOpRemove) -+} -+ -+// UpdateInterface updates interface entry point in the virtcontainers. -+func UpdateInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) { -+ span, ctx := trace(ctx, "UpdateInterface") -+ defer span.Finish() -+ -+ return toggleInterface(ctx, sandboxID, inf, vcTypes.NetworkOpUpdate) - } - - // ListInterfaces is the virtcontainers list interfaces entry point. -diff --git a/virtcontainers/implementation.go b/virtcontainers/implementation.go -index 0265d1ed..e4bc4ae6 100644 ---- a/virtcontainers/implementation.go -+++ b/virtcontainers/implementation.go -@@ -163,6 +163,11 @@ func (impl *VCImpl) ListInterfaces(ctx context.Context, sandboxID string) ([]*vc - return ListInterfaces(ctx, sandboxID) - } - -+// ListInterfaces implements the VC function of the same name. -+func (impl *VCImpl) UpdateInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) { -+ return UpdateInterface(ctx, sandboxID, inf) -+} -+ - // UpdateRoutes implements the VC function of the same name. - func (impl *VCImpl) UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) { - return UpdateRoutes(ctx, sandboxID, routes, op) -diff --git a/virtcontainers/interfaces.go b/virtcontainers/interfaces.go -index d8f0be69..0fd12d84 100644 ---- a/virtcontainers/interfaces.go -+++ b/virtcontainers/interfaces.go -@@ -51,6 +51,7 @@ type VC interface { - - AddInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) - RemoveInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) -+ UpdateInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) - ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTypes.Interface, error) - UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) - ListRoutes(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error) -diff --git a/virtcontainers/pkg/vcmock/mock.go b/virtcontainers/pkg/vcmock/mock.go -index aa9bae9a..d3bf201f 100644 ---- a/virtcontainers/pkg/vcmock/mock.go -+++ b/virtcontainers/pkg/vcmock/mock.go -@@ -273,6 +273,14 @@ func (m *VCMock) ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTyp - return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID) - } - -+// UpdateInterface implements the VC function of the same name -+func (m *VCMock) UpdateInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) { -+ if m.UpdateRoutesFunc != nil { -+ return m.UpdateInterfaceFunc(ctx, sandboxID, inf) -+ } -+ return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID) -+} -+ - // UpdateRoutes implements the VC function of the same name. - func (m *VCMock) UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) { - if m.UpdateRoutesFunc != nil { -diff --git a/virtcontainers/pkg/vcmock/types.go b/virtcontainers/pkg/vcmock/types.go -index 610b4602..152e66b7 100644 ---- a/virtcontainers/pkg/vcmock/types.go -+++ b/virtcontainers/pkg/vcmock/types.go -@@ -73,6 +73,7 @@ type VCMock struct { - AddInterfaceFunc func(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) - RemoveInterfaceFunc func(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) - ListInterfacesFunc func(ctx context.Context, sandboxID string) ([]*vcTypes.Interface, error) -+ UpdateInterfaceFunc func(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) - UpdateRoutesFunc func(ctx context.Context, sandboxID string, routes []*vcTypes.Route) ([]*vcTypes.Route, error) - ListRoutesFunc func(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error) - CleanupContainerFunc func(ctx context.Context, sandboxID, containerID string, force bool) error -diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go -index d8ab6c1a..174e6cb6 100644 ---- a/virtcontainers/sandbox.go -+++ b/virtcontainers/sandbox.go -@@ -1031,6 +1031,54 @@ func (s *Sandbox) RemoveInterface(inf *vcTypes.Interface) (*vcTypes.Interface, e - return nil, nil - } - -+// UpdateInterface updates the nic. -+func (s *Sandbox) UpdateInterface(inf *vcTypes.Interface) (*vcTypes.Interface, error) { -+ err := checkUpdateInterfaceInfo(inf) -+ if err != nil { -+ return nil, err -+ } -+ -+ nicInGuest := false -+ -+ endpoint := s.getEndpointByName(inf.Name) -+ // nic is not in local endpoints -+ if endpoint == nil { -+ interfaceInGuestInf := s.getInterfaceInfByName(inf.Name) -+ if interfaceInGuestInf == nil { -+ return nil, fmt.Errorf("can not find the network interface") -+ } -+ -+ nicInGuest = true -+ endpoint, err = s.generateEndpointForStore(interfaceInGuestInf) -+ if err != nil { -+ return nil, err -+ } -+ } -+ -+ // update endpoint ip -+ err = s.updateEndpointIP(endpoint, inf) -+ if err != nil { -+ return nil, err -+ } -+ -+ s.Logger().WithField("endpoint-type", endpoint.Type()).Info("Update endpoint") -+ grpcIf, err := s.agent.updateInterface(inf) -+ if err != nil { -+ return nil, err -+ } -+ -+ if nicInGuest { -+ s.networkNS.Endpoints = append(s.networkNS.Endpoints, endpoint) -+ } -+ -+ if err = s.Save(); err != nil { -+ s.Logger().WithError(err).Error("failed to store the network information") -+ return nil, err -+ } -+ -+ return grpcIf, nil -+} -+ - // ListInterfaces lists all nics and their configurations in the sandbox. - func (s *Sandbox) ListInterfaces() ([]*vcTypes.Interface, error) { - return s.agent.listInterfaces() -@@ -2431,6 +2479,95 @@ func (s *Sandbox) IsCompatOldCNI() bool { - return s.config.NetworkConfig.EnableCompatOldCNI - } - -+func checkUpdateInterfaceInfo(inf *vcTypes.Interface) error { -+ if inf.Mtu > 0 || inf.Queues > 0 || inf.Device != "" || inf.VhostUserSocket != "" || inf.LinkType != "" || inf.HwAddr != "" { -+ return fmt.Errorf("device,mtu,hwAddr,linkType,queues and vhostUserSocket should be 0 or empty") -+ } -+ -+ return nil -+} -+ -+// getEndpointByName returns the endpoint by name. -+func (s *Sandbox) getEndpointByName(name string) Endpoint { -+ if name == "" { -+ return nil -+ } -+ -+ for i, endpoint := range s.networkNS.Endpoints { -+ if endpoint.Name() == name { -+ return s.networkNS.Endpoints[i] -+ } -+ } -+ -+ return nil -+} -+ -+// getInterfaceInfByName returns the interface information. -+func (s *Sandbox) getInterfaceInfByName(name string) *vcTypes.Interface { -+ if name == "" { -+ return nil -+ } -+ -+ nics, err := s.agent.listInterfaces() -+ if err != nil { -+ return nil -+ } -+ -+ for _, i := range nics { -+ if i.Name == name { -+ return i -+ } -+ } -+ -+ return nil -+} -+ -+// generateEndpointForStore only generates endpoint information for store,it does not create a endpoint -+func (s *Sandbox) generateEndpointForStore(inf *vcTypes.Interface) (*TapEndpoint, error) { -+ netInfo, err := s.generateNetInfo(inf) -+ if err != nil { -+ return nil, err -+ } -+ -+ endpoint := &TapEndpoint{ -+ TapInterface: TapInterface{ -+ Name: inf.Name, -+ TAPIface: NetworkInterface{ -+ HardAddr: inf.HwAddr, -+ }, -+ }, -+ EndpointType: TapEndpointType, -+ } -+ -+ endpoint.SetProperties(netInfo) -+ -+ return endpoint, nil -+} -+ -+func (s *Sandbox) updateEndpointIP(endpoint Endpoint, inf *vcTypes.Interface) error { -+ if endpoint == nil { -+ return fmt.Errorf("endpoint is nil") -+ } -+ -+ netInfo := endpoint.Properties() -+ -+ var addrs []netlink.Addr -+ for _, addr := range inf.IPAddresses { -+ netlinkAddrStr := fmt.Sprintf("%s/%s", addr.Address, addr.Mask) -+ netlinkAddr, err := netlink.ParseAddr(netlinkAddrStr) -+ if err != nil { -+ return fmt.Errorf("could not parse %q: %v", netlinkAddrStr, err) -+ } -+ -+ addrs = append(addrs, *netlinkAddr) -+ } -+ -+ netInfo.Addrs = addrs -+ endpoint.SetProperties(netInfo) -+ -+ return nil -+} -+ - // updateStaticSandboxResources update sandbox's cpu and memory resource passed by - // sandbox_cpu and sandbox_mem annotations - func updateStaticSandboxResources(sandboxConfig *SandboxConfig) error { --- -2.14.3 (Apple Git-98) - diff --git a/runtime/patches/0051-network-fix-del-iface-doesn-t-delete-the-tap-interfa.patch b/runtime/patches/0051-network-fix-del-iface-doesn-t-delete-the-tap-interfa.patch deleted file mode 100644 index 385a376..0000000 --- a/runtime/patches/0051-network-fix-del-iface-doesn-t-delete-the-tap-interfa.patch +++ /dev/null @@ -1,78 +0,0 @@ -From be0d60f5fa88267afb26125681a217ef9476e133 Mon Sep 17 00:00:00 2001 -From: holyfei -Date: Sat, 19 Sep 2020 12:47:14 +0800 -Subject: [PATCH] runtime: fix del-iface doesn't delete the tap interface in - the host problem - -reason: kata-runtime add-iface support add exist tap device in the host or -new created tap device by kata-network add-iface process. If add interface -is exist tap device, del-iface will not delete the tap device, so we need -to judge the deleted interface is exist device or new created, which is judged -by the VMFds of the tap Link and del-iface will load this info from the -network.json in 1.7.0 version. However, 1.11.1 version don't write the VMFds -into the persiste.json file, del-iface can not figure out which tap is exist -already or new created, so cause the this problem. ---- - virtcontainers/endpoint.go | 2 ++ - virtcontainers/persist/api/network.go | 5 +++++ - virtcontainers/tap_endpoint.go | 1 + - 3 files changed, 8 insertions(+) - -diff --git a/virtcontainers/endpoint.go b/virtcontainers/endpoint.go -index 7efcf49c..3618792e 100644 ---- a/virtcontainers/endpoint.go -+++ b/virtcontainers/endpoint.go -@@ -129,6 +129,7 @@ func saveTapIf(tapif *TapInterface) *persistapi.TapInterface { - HardAddr: tapif.TAPIface.HardAddr, - Addrs: tapif.TAPIface.Addrs, - }, -+ VMFds: tapif.VMFds, - } - } - -@@ -167,6 +168,7 @@ func loadTapIf(tapif *persistapi.TapInterface) *TapInterface { - HardAddr: tapif.TAPIface.HardAddr, - Addrs: tapif.TAPIface.Addrs, - }, -+ VMFds: tapif.VMFds, - } - } - -diff --git a/virtcontainers/persist/api/network.go b/virtcontainers/persist/api/network.go -index 53c6de44..c5611767 100644 ---- a/virtcontainers/persist/api/network.go -+++ b/virtcontainers/persist/api/network.go -@@ -7,6 +7,8 @@ - package persistapi - - import ( -+ "os" -+ - "github.com/vishvananda/netlink" - ) - -@@ -45,6 +47,9 @@ type TapInterface struct { - Name string - TAPIface NetworkInterface - // remove VMFds and VhostFds -+ // add VMFds back to judge a tap interface is exist before add-iface -+ // or new created by kata-network add-iface -+ VMFds []*os.File - } - - // TuntapInterface defines a tap interface -diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go -index 5a3e7f7e..0b6002aa 100644 ---- a/virtcontainers/tap_endpoint.go -+++ b/virtcontainers/tap_endpoint.go -@@ -199,6 +199,7 @@ func unTapNetwork(endpoint *TapEndpoint) error { - return nil - } - -+ networkLogger().Debug("untap the new created tap interface") - name := endpoint.TapInterface.TAPIface.Name - netHandle, err := netlink.NewHandle() - if err != nil { --- -2.11.0 - diff --git a/runtime/patches/0052-runtime-add-support-of-new-sandbox-StratoVirt.patch b/runtime/patches/0052-runtime-add-support-of-new-sandbox-StratoVirt.patch deleted file mode 100644 index 3456c9f..0000000 --- a/runtime/patches/0052-runtime-add-support-of-new-sandbox-StratoVirt.patch +++ /dev/null @@ -1,1720 +0,0 @@ -From 60b6a9c63a116db6d60a91b3dc875de3f33aa57c Mon Sep 17 00:00:00 2001 -From: LiangZhang -Date: Mon, 21 Sep 2020 15:33:17 +0800 -Subject: [PATCH] runtime: add support of new sandbox StratoVirt - -Signed-off-by: LiangZhang ---- - pkg/katautils/config.go | 5 +- - .../agent/protocols/grpc/agent.pb.go | 636 +++++++++++++-------- - virtcontainers/agent.go | 3 + - virtcontainers/hypervisor.go | 12 + - virtcontainers/kata_agent.go | 73 ++- - virtcontainers/noop_agent.go | 5 + - virtcontainers/stratovirt.go | 617 ++++++++++++++++++++ - 7 files changed, 1090 insertions(+), 261 deletions(-) - create mode 100644 virtcontainers/stratovirt.go - -diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go -index 3365b3f..ed6f03c 100644 ---- a/pkg/katautils/config.go -+++ b/pkg/katautils/config.go -@@ -52,7 +52,7 @@ const ( - clhHypervisorTableType = "clh" - qemuHypervisorTableType = "qemu" - acrnHypervisorTableType = "acrn" -- -+ stratovirtHypervisorTableType = "stratovirt" - // supported proxy component types - kataProxyTableType = "kata" - -@@ -921,6 +921,9 @@ func updateRuntimeConfigHypervisor(configPath string, tomlConf tomlConfig, confi - case clhHypervisorTableType: - config.HypervisorType = vc.ClhHypervisor - hConfig, err = newClhHypervisorConfig(hypervisor) -+ case stratovirtHypervisorTableType: -+ config.HypervisorType = vc.StratovirtHypervisor -+ hConfig, err = newQemuHypervisorConfig(hypervisor) - } - - if err != nil { -diff --git a/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go b/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go -index c50ecb5..181fcb6 100644 ---- a/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go -+++ b/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go -@@ -50,6 +50,7 @@ - Interfaces - Routes - UpdateInterfaceRequest -+ UpdateInterfaceHwAddrByNameRequest - UpdateRoutesRequest - ListInterfacesRequest - ListRoutesRequest -@@ -1491,6 +1492,24 @@ func (m *UpdateInterfaceRequest) GetInterface() *types.Interface { - return nil - } - -+type UpdateInterfaceHwAddrByNameRequest struct { -+ Interface *types.Interface `protobuf:"bytes,1,opt,name=interface" json:"interface,omitempty"` -+} -+ -+func (m *UpdateInterfaceHwAddrByNameRequest) Reset() { *m = UpdateInterfaceHwAddrByNameRequest{} } -+func (m *UpdateInterfaceHwAddrByNameRequest) String() string { return proto.CompactTextString(m) } -+func (*UpdateInterfaceHwAddrByNameRequest) ProtoMessage() {} -+func (*UpdateInterfaceHwAddrByNameRequest) Descriptor() ([]byte, []int) { -+ return fileDescriptorAgent, []int{40} -+} -+ -+func (m *UpdateInterfaceHwAddrByNameRequest) GetInterface() *types.Interface { -+ if m != nil { -+ return m.Interface -+ } -+ return nil -+} -+ - type UpdateRoutesRequest struct { - Routes *Routes `protobuf:"bytes,1,opt,name=routes" json:"routes,omitempty"` - Increment bool `protobuf:"varint,2,opt,name=increment,proto3" json:"increment,omitempty"` -@@ -1499,7 +1518,7 @@ type UpdateRoutesRequest struct { - func (m *UpdateRoutesRequest) Reset() { *m = UpdateRoutesRequest{} } - func (m *UpdateRoutesRequest) String() string { return proto.CompactTextString(m) } - func (*UpdateRoutesRequest) ProtoMessage() {} --func (*UpdateRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{40} } -+func (*UpdateRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{41} } - - func (m *UpdateRoutesRequest) GetRoutes() *Routes { - if m != nil { -@@ -1521,7 +1540,7 @@ type ListInterfacesRequest struct { - func (m *ListInterfacesRequest) Reset() { *m = ListInterfacesRequest{} } - func (m *ListInterfacesRequest) String() string { return proto.CompactTextString(m) } - func (*ListInterfacesRequest) ProtoMessage() {} --func (*ListInterfacesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{41} } -+func (*ListInterfacesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{42} } - - type ListRoutesRequest struct { - } -@@ -1529,7 +1548,7 @@ type ListRoutesRequest struct { - func (m *ListRoutesRequest) Reset() { *m = ListRoutesRequest{} } - func (m *ListRoutesRequest) String() string { return proto.CompactTextString(m) } - func (*ListRoutesRequest) ProtoMessage() {} --func (*ListRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{42} } -+func (*ListRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{43} } - - type OnlineCPUMemRequest struct { - // Wait specifies if the caller waits for the agent to online all resources. -@@ -1545,7 +1564,7 @@ type OnlineCPUMemRequest struct { - func (m *OnlineCPUMemRequest) Reset() { *m = OnlineCPUMemRequest{} } - func (m *OnlineCPUMemRequest) String() string { return proto.CompactTextString(m) } - func (*OnlineCPUMemRequest) ProtoMessage() {} --func (*OnlineCPUMemRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{43} } -+func (*OnlineCPUMemRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{44} } - - func (m *OnlineCPUMemRequest) GetWait() bool { - if m != nil { -@@ -1576,7 +1595,7 @@ type ReseedRandomDevRequest struct { - func (m *ReseedRandomDevRequest) Reset() { *m = ReseedRandomDevRequest{} } - func (m *ReseedRandomDevRequest) String() string { return proto.CompactTextString(m) } - func (*ReseedRandomDevRequest) ProtoMessage() {} --func (*ReseedRandomDevRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{44} } -+func (*ReseedRandomDevRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{45} } - - func (m *ReseedRandomDevRequest) GetData() []byte { - if m != nil { -@@ -1603,7 +1622,7 @@ type AgentDetails struct { - func (m *AgentDetails) Reset() { *m = AgentDetails{} } - func (m *AgentDetails) String() string { return proto.CompactTextString(m) } - func (*AgentDetails) ProtoMessage() {} --func (*AgentDetails) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{45} } -+func (*AgentDetails) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{46} } - - func (m *AgentDetails) GetVersion() string { - if m != nil { -@@ -1654,7 +1673,7 @@ type GuestDetailsRequest struct { - func (m *GuestDetailsRequest) Reset() { *m = GuestDetailsRequest{} } - func (m *GuestDetailsRequest) String() string { return proto.CompactTextString(m) } - func (*GuestDetailsRequest) ProtoMessage() {} --func (*GuestDetailsRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{46} } -+func (*GuestDetailsRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{47} } - - func (m *GuestDetailsRequest) GetMemBlockSize() bool { - if m != nil { -@@ -1680,7 +1699,7 @@ type GuestDetailsResponse struct { - func (m *GuestDetailsResponse) Reset() { *m = GuestDetailsResponse{} } - func (m *GuestDetailsResponse) String() string { return proto.CompactTextString(m) } - func (*GuestDetailsResponse) ProtoMessage() {} --func (*GuestDetailsResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{47} } -+func (*GuestDetailsResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{48} } - - func (m *GuestDetailsResponse) GetMemBlockSizeBytes() uint64 { - if m != nil { -@@ -1712,7 +1731,7 @@ type MemHotplugByProbeRequest struct { - func (m *MemHotplugByProbeRequest) Reset() { *m = MemHotplugByProbeRequest{} } - func (m *MemHotplugByProbeRequest) String() string { return proto.CompactTextString(m) } - func (*MemHotplugByProbeRequest) ProtoMessage() {} --func (*MemHotplugByProbeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{48} } -+func (*MemHotplugByProbeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{49} } - - func (m *MemHotplugByProbeRequest) GetMemHotplugProbeAddr() []uint64 { - if m != nil { -@@ -1731,7 +1750,7 @@ type SetGuestDateTimeRequest struct { - func (m *SetGuestDateTimeRequest) Reset() { *m = SetGuestDateTimeRequest{} } - func (m *SetGuestDateTimeRequest) String() string { return proto.CompactTextString(m) } - func (*SetGuestDateTimeRequest) ProtoMessage() {} --func (*SetGuestDateTimeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{49} } -+func (*SetGuestDateTimeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{50} } - - func (m *SetGuestDateTimeRequest) GetSec() int64 { - if m != nil { -@@ -1780,7 +1799,7 @@ type Storage struct { - func (m *Storage) Reset() { *m = Storage{} } - func (m *Storage) String() string { return proto.CompactTextString(m) } - func (*Storage) ProtoMessage() {} --func (*Storage) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{50} } -+func (*Storage) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{51} } - - func (m *Storage) GetDriver() string { - if m != nil { -@@ -1863,7 +1882,7 @@ type Device struct { - func (m *Device) Reset() { *m = Device{} } - func (m *Device) String() string { return proto.CompactTextString(m) } - func (*Device) ProtoMessage() {} --func (*Device) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{51} } -+func (*Device) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{52} } - - func (m *Device) GetId() string { - if m != nil { -@@ -1909,7 +1928,7 @@ type StringUser struct { - func (m *StringUser) Reset() { *m = StringUser{} } - func (m *StringUser) String() string { return proto.CompactTextString(m) } - func (*StringUser) ProtoMessage() {} --func (*StringUser) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{52} } -+func (*StringUser) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{53} } - - func (m *StringUser) GetUid() string { - if m != nil { -@@ -1957,7 +1976,7 @@ type CopyFileRequest struct { - func (m *CopyFileRequest) Reset() { *m = CopyFileRequest{} } - func (m *CopyFileRequest) String() string { return proto.CompactTextString(m) } - func (*CopyFileRequest) ProtoMessage() {} --func (*CopyFileRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{53} } -+func (*CopyFileRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{54} } - - func (m *CopyFileRequest) GetPath() string { - if m != nil { -@@ -2021,7 +2040,7 @@ type StartTracingRequest struct { - func (m *StartTracingRequest) Reset() { *m = StartTracingRequest{} } - func (m *StartTracingRequest) String() string { return proto.CompactTextString(m) } - func (*StartTracingRequest) ProtoMessage() {} --func (*StartTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{54} } -+func (*StartTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{55} } - - type StopTracingRequest struct { - } -@@ -2029,7 +2048,7 @@ type StopTracingRequest struct { - func (m *StopTracingRequest) Reset() { *m = StopTracingRequest{} } - func (m *StopTracingRequest) String() string { return proto.CompactTextString(m) } - func (*StopTracingRequest) ProtoMessage() {} --func (*StopTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{55} } -+func (*StopTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{56} } - - type UpdateIPVSRequest struct { - // IPVS_req is the IPVS rule message needed to update -@@ -2039,7 +2058,7 @@ type UpdateIPVSRequest struct { - func (m *UpdateIPVSRequest) Reset() { *m = UpdateIPVSRequest{} } - func (m *UpdateIPVSRequest) String() string { return proto.CompactTextString(m) } - func (*UpdateIPVSRequest) ProtoMessage() {} --func (*UpdateIPVSRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{56} } -+func (*UpdateIPVSRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{57} } - - func (m *UpdateIPVSRequest) GetIPVSReq() string { - if m != nil { -@@ -2056,7 +2075,7 @@ type IPVSResponse struct { - func (m *IPVSResponse) Reset() { *m = IPVSResponse{} } - func (m *IPVSResponse) String() string { return proto.CompactTextString(m) } - func (*IPVSResponse) ProtoMessage() {} --func (*IPVSResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{57} } -+func (*IPVSResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{58} } - - func (m *IPVSResponse) GetIPVSRes() string { - if m != nil { -@@ -2106,6 +2125,7 @@ func init() { - proto.RegisterType((*Interfaces)(nil), "grpc.Interfaces") - proto.RegisterType((*Routes)(nil), "grpc.Routes") - proto.RegisterType((*UpdateInterfaceRequest)(nil), "grpc.UpdateInterfaceRequest") -+ proto.RegisterType((*UpdateInterfaceHwAddrByNameRequest)(nil), "grpc.UpdateInterfaceHwAddrByNameRequest") - proto.RegisterType((*UpdateRoutesRequest)(nil), "grpc.UpdateRoutesRequest") - proto.RegisterType((*ListInterfacesRequest)(nil), "grpc.ListInterfacesRequest") - proto.RegisterType((*ListRoutesRequest)(nil), "grpc.ListRoutesRequest") -@@ -2163,6 +2183,7 @@ type AgentServiceClient interface { - TtyWinResize(ctx context.Context, in *TtyWinResizeRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error) - // networking - UpdateInterface(ctx context.Context, in *UpdateInterfaceRequest, opts ...grpc1.CallOption) (*types.Interface, error) -+ UpdateInterfaceHwAddrByName(ctx context.Context, in *UpdateInterfaceHwAddrByNameRequest, opts ...grpc1.CallOption) (*types.Interface, error) - UpdateRoutes(ctx context.Context, in *UpdateRoutesRequest, opts ...grpc1.CallOption) (*Routes, error) - ListInterfaces(ctx context.Context, in *ListInterfacesRequest, opts ...grpc1.CallOption) (*Interfaces, error) - ListRoutes(ctx context.Context, in *ListRoutesRequest, opts ...grpc1.CallOption) (*Routes, error) -@@ -2342,6 +2363,15 @@ func (c *agentServiceClient) UpdateInterface(ctx context.Context, in *UpdateInte - return out, nil - } - -+func (c *agentServiceClient) UpdateInterfaceHwAddrByName(ctx context.Context, in *UpdateInterfaceHwAddrByNameRequest, opts ...grpc1.CallOption) (*types.Interface, error) { -+ out := new(types.Interface) -+ err := grpc1.Invoke(ctx, "/grpc.AgentService/UpdateInterfaceHwAddrByName", in, out, c.cc, opts...) -+ if err != nil { -+ return nil, err -+ } -+ return out, nil -+} -+ - func (c *agentServiceClient) UpdateRoutes(ctx context.Context, in *UpdateRoutesRequest, opts ...grpc1.CallOption) (*Routes, error) { - out := new(Routes) - err := grpc1.Invoke(ctx, "/grpc.AgentService/UpdateRoutes", in, out, c.cc, opts...) -@@ -2497,6 +2527,7 @@ type AgentServiceServer interface { - TtyWinResize(context.Context, *TtyWinResizeRequest) (*google_protobuf2.Empty, error) - // networking - UpdateInterface(context.Context, *UpdateInterfaceRequest) (*types.Interface, error) -+ UpdateInterfaceHwAddrByName(context.Context, *UpdateInterfaceHwAddrByNameRequest) (*types.Interface, error) - UpdateRoutes(context.Context, *UpdateRoutesRequest) (*Routes, error) - ListInterfaces(context.Context, *ListInterfacesRequest) (*Interfaces, error) - ListRoutes(context.Context, *ListRoutesRequest) (*Routes, error) -@@ -2825,6 +2856,24 @@ func _AgentService_UpdateInterface_Handler(srv interface{}, ctx context.Context, - return interceptor(ctx, in, info, handler) - } - -+func _AgentService_UpdateInterfaceHwAddrByName_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) { -+ in := new(UpdateInterfaceHwAddrByNameRequest) -+ if err := dec(in); err != nil { -+ return nil, err -+ } -+ if interceptor == nil { -+ return srv.(AgentServiceServer).UpdateInterfaceHwAddrByName(ctx, in) -+ } -+ info := &grpc1.UnaryServerInfo{ -+ Server: srv, -+ FullMethod: "/grpc.AgentService/UpdateInterfaceHwAddrByName", -+ } -+ handler := func(ctx context.Context, req interface{}) (interface{}, error) { -+ return srv.(AgentServiceServer).UpdateInterfaceHwAddrByName(ctx, req.(*UpdateInterfaceHwAddrByNameRequest)) -+ } -+ return interceptor(ctx, in, info, handler) -+} -+ - func _AgentService_UpdateRoutes_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) { - in := new(UpdateRoutesRequest) - if err := dec(in); err != nil { -@@ -3150,6 +3199,10 @@ var _AgentService_serviceDesc = grpc1.ServiceDesc{ - Handler: _AgentService_UpdateInterface_Handler, - }, - { -+ MethodName: "UpdateInterfaceHwAddrByName", -+ Handler: _AgentService_UpdateInterfaceHwAddrByName_Handler, -+ }, -+ { - MethodName: "UpdateRoutes", - Handler: _AgentService_UpdateRoutes_Handler, - }, -@@ -4899,6 +4952,34 @@ func (m *UpdateInterfaceRequest) MarshalTo(dAtA []byte) (int, error) { - return i, nil - } - -+func (m *UpdateInterfaceHwAddrByNameRequest) Marshal() (dAtA []byte, err error) { -+ size := m.Size() -+ dAtA = make([]byte, size) -+ n, err := m.MarshalTo(dAtA) -+ if err != nil { -+ return nil, err -+ } -+ return dAtA[:n], nil -+} -+ -+func (m *UpdateInterfaceHwAddrByNameRequest) MarshalTo(dAtA []byte) (int, error) { -+ var i int -+ _ = i -+ var l int -+ _ = l -+ if m.Interface != nil { -+ dAtA[i] = 0xa -+ i++ -+ i = encodeVarintAgent(dAtA, i, uint64(m.Interface.Size())) -+ n25, err := m.Interface.MarshalTo(dAtA[i:]) -+ if err != nil { -+ return 0, err -+ } -+ i += n25 -+ } -+ return i, nil -+} -+ - func (m *UpdateRoutesRequest) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) -@@ -4918,11 +4999,11 @@ func (m *UpdateRoutesRequest) MarshalTo(dAtA []byte) (int, error) { - dAtA[i] = 0xa - i++ - i = encodeVarintAgent(dAtA, i, uint64(m.Routes.Size())) -- n25, err := m.Routes.MarshalTo(dAtA[i:]) -+ n26, err := m.Routes.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } -- i += n25 -+ i += n26 - } - if m.Increment { - dAtA[i] = 0x10 -@@ -5176,11 +5257,11 @@ func (m *GuestDetailsResponse) MarshalTo(dAtA []byte) (int, error) { - dAtA[i] = 0x12 - i++ - i = encodeVarintAgent(dAtA, i, uint64(m.AgentDetails.Size())) -- n26, err := m.AgentDetails.MarshalTo(dAtA[i:]) -+ n27, err := m.AgentDetails.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } -- i += n26 -+ i += n27 - } - if m.SupportMemHotplugProbe { - dAtA[i] = 0x18 -@@ -5211,21 +5292,21 @@ func (m *MemHotplugByProbeRequest) MarshalTo(dAtA []byte) (int, error) { - var l int - _ = l - if len(m.MemHotplugProbeAddr) > 0 { -- dAtA28 := make([]byte, len(m.MemHotplugProbeAddr)*10) -- var j27 int -+ dAtA29 := make([]byte, len(m.MemHotplugProbeAddr)*10) -+ var j28 int - for _, num := range m.MemHotplugProbeAddr { - for num >= 1<<7 { -- dAtA28[j27] = uint8(uint64(num)&0x7f | 0x80) -+ dAtA29[j28] = uint8(uint64(num)&0x7f | 0x80) - num >>= 7 -- j27++ -+ j28++ - } -- dAtA28[j27] = uint8(num) -- j27++ -+ dAtA29[j28] = uint8(num) -+ j28++ - } - dAtA[i] = 0xa - i++ -- i = encodeVarintAgent(dAtA, i, uint64(j27)) -- i += copy(dAtA[i:], dAtA28[:j27]) -+ i = encodeVarintAgent(dAtA, i, uint64(j28)) -+ i += copy(dAtA[i:], dAtA29[:j28]) - } - return i, nil - } -@@ -6333,6 +6414,16 @@ func (m *UpdateInterfaceRequest) Size() (n int) { - return n - } - -+func (m *UpdateInterfaceHwAddrByNameRequest) Size() (n int) { -+ var l int -+ _ = l -+ if m.Interface != nil { -+ l = m.Interface.Size() -+ n += 1 + l + sovAgent(uint64(l)) -+ } -+ return n -+} -+ - func (m *UpdateRoutesRequest) Size() (n int) { - var l int - _ = l -@@ -12114,6 +12205,89 @@ func (m *UpdateInterfaceRequest) Unmarshal(dAtA []byte) error { - } - return nil - } -+func (m *UpdateInterfaceHwAddrByNameRequest) Unmarshal(dAtA []byte) error { -+ l := len(dAtA) -+ iNdEx := 0 -+ for iNdEx < l { -+ preIndex := iNdEx -+ var wire uint64 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ wire |= (uint64(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ fieldNum := int32(wire >> 3) -+ wireType := int(wire & 0x7) -+ if wireType == 4 { -+ return fmt.Errorf("proto: UpdateInterfaceHwAddrByNameRequest: wiretype end group for non-group") -+ } -+ if fieldNum <= 0 { -+ return fmt.Errorf("proto: UpdateInterfaceHwAddrByNameRequest: illegal tag %d (wire type %d)", fieldNum, wire) -+ } -+ switch fieldNum { -+ case 1: -+ if wireType != 2 { -+ return fmt.Errorf("proto: wrong wireType = %d for field Interface", wireType) -+ } -+ var msglen int -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ msglen |= (int(b) & 0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ if msglen < 0 { -+ return ErrInvalidLengthAgent -+ } -+ postIndex := iNdEx + msglen -+ if postIndex > l { -+ return io.ErrUnexpectedEOF -+ } -+ if m.Interface == nil { -+ m.Interface = &types.Interface{} -+ } -+ if err := m.Interface.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -+ return err -+ } -+ iNdEx = postIndex -+ default: -+ iNdEx = preIndex -+ skippy, err := skipAgent(dAtA[iNdEx:]) -+ if err != nil { -+ return err -+ } -+ if skippy < 0 { -+ return ErrInvalidLengthAgent -+ } -+ if (iNdEx + skippy) > l { -+ return io.ErrUnexpectedEOF -+ } -+ iNdEx += skippy -+ } -+ } -+ -+ if iNdEx > l { -+ return io.ErrUnexpectedEOF -+ } -+ return nil -+} - func (m *UpdateRoutesRequest) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 -@@ -14242,204 +14416,206 @@ var ( - func init() { proto.RegisterFile("agent.proto", fileDescriptorAgent) } - - var fileDescriptorAgent = []byte{ -- // 3183 bytes of a gzipped FileDescriptorProto -- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0x4b, 0x6f, 0x1c, 0xc7, -- 0x99, 0x98, 0x07, 0x39, 0x33, 0xdf, 0xbc, 0xc8, 0x22, 0x45, 0x8d, 0x46, 0xb2, 0x2c, 0xb7, 0x6d, -- 0x99, 0x5e, 0xaf, 0x49, 0x8b, 0x36, 0xfc, 0x84, 0x57, 0x10, 0x29, 0xad, 0xc8, 0xb5, 0xb4, 0xe2, -- 0xf6, 0x88, 0xeb, 0x5d, 0x2c, 0x16, 0x8d, 0x66, 0x77, 0x71, 0x58, 0xe6, 0x74, 0x57, 0xbb, 0xaa, -- 0x9a, 0xe2, 0x78, 0x81, 0x3d, 0x26, 0xd7, 0x9c, 0x72, 0xcb, 0x1f, 0x08, 0x72, 0xcb, 0x2d, 0xb9, -- 0xe6, 0x60, 0x04, 0x39, 0x04, 0xf9, 0x01, 0x41, 0xe0, 0x9f, 0x90, 0x5f, 0x10, 0xd4, 0xab, 0x1f, -- 0x33, 0x43, 0x1a, 0x11, 0x04, 0xe4, 0xd2, 0xe8, 0xef, 0x51, 0xdf, 0xab, 0xaa, 0xbe, 0xfa, 0xbe, -- 0x2a, 0x68, 0xfb, 0x63, 0x1c, 0x8b, 0xad, 0x84, 0x51, 0x41, 0x51, 0x7d, 0xcc, 0x92, 0x60, 0xd8, -- 0xa2, 0x01, 0xd1, 0x88, 0xe1, 0xc7, 0x63, 0x22, 0x4e, 0xd3, 0xe3, 0xad, 0x80, 0x46, 0xdb, 0x67, -- 0xbe, 0xf0, 0xdf, 0x0f, 0x68, 0x2c, 0x7c, 0x12, 0x63, 0xc6, 0xb7, 0xd5, 0xc0, 0xed, 0xe4, 0x6c, -- 0xbc, 0x2d, 0xa6, 0x09, 0xe6, 0xfa, 0x6b, 0xc6, 0xdd, 0x1c, 0x53, 0x3a, 0x9e, 0xe0, 0x6d, 0x05, -- 0x1d, 0xa7, 0x27, 0xdb, 0x38, 0x4a, 0xc4, 0x54, 0x13, 0x9d, 0x5f, 0x54, 0x61, 0x63, 0x8f, 0x61, -- 0x5f, 0xe0, 0x3d, 0x2b, 0xcd, 0xc5, 0xdf, 0xa6, 0x98, 0x0b, 0xf4, 0x06, 0x74, 0x32, 0x0d, 0x1e, -- 0x09, 0x07, 0x95, 0x3b, 0x95, 0xcd, 0x96, 0xdb, 0xce, 0x70, 0x07, 0x21, 0xba, 0x0e, 0x0d, 0x7c, -- 0x81, 0x03, 0x49, 0xad, 0x2a, 0xea, 0xb2, 0x04, 0x0f, 0x42, 0x74, 0x0f, 0xda, 0x5c, 0x30, 0x12, -- 0x8f, 0xbd, 0x94, 0x63, 0x36, 0xa8, 0xdd, 0xa9, 0x6c, 0xb6, 0x77, 0x56, 0xb6, 0xa4, 0x4b, 0x5b, -- 0x23, 0x45, 0x38, 0xe2, 0x98, 0xb9, 0xc0, 0xb3, 0x7f, 0x74, 0x17, 0x1a, 0x21, 0x3e, 0x27, 0x01, -- 0xe6, 0x83, 0xfa, 0x9d, 0xda, 0x66, 0x7b, 0xa7, 0xa3, 0xd9, 0x1f, 0x2a, 0xa4, 0x6b, 0x89, 0xe8, -- 0x5d, 0x68, 0x72, 0x41, 0x99, 0x3f, 0xc6, 0x7c, 0xb0, 0xa4, 0x18, 0xbb, 0x56, 0xae, 0xc2, 0xba, -- 0x19, 0x19, 0xdd, 0x82, 0xda, 0xb3, 0xbd, 0x83, 0xc1, 0xb2, 0xd2, 0x0e, 0x86, 0x2b, 0xc1, 0x81, -- 0x2b, 0xd1, 0xe8, 0x4d, 0xe8, 0x72, 0x3f, 0x0e, 0x8f, 0xe9, 0x85, 0x97, 0x90, 0x30, 0xe6, 0x83, -- 0xc6, 0x9d, 0xca, 0x66, 0xd3, 0xed, 0x18, 0xe4, 0xa1, 0xc4, 0x39, 0x9f, 0xc3, 0xb5, 0x91, 0xf0, -- 0x99, 0x78, 0x89, 0xe8, 0x38, 0x47, 0xb0, 0xe1, 0xe2, 0x88, 0x9e, 0xbf, 0x54, 0x68, 0x07, 0xd0, -- 0x10, 0x24, 0xc2, 0x34, 0x15, 0x2a, 0xb4, 0x5d, 0xd7, 0x82, 0xce, 0xaf, 0x2a, 0x80, 0x1e, 0x5d, -- 0xe0, 0xe0, 0x90, 0xd1, 0x00, 0x73, 0xfe, 0x0f, 0x9a, 0xae, 0x77, 0xa0, 0x91, 0x68, 0x03, 0x06, -- 0x75, 0xc5, 0x6e, 0x66, 0xc1, 0x5a, 0x65, 0xa9, 0xce, 0x37, 0xb0, 0x3e, 0x22, 0xe3, 0xd8, 0x9f, -- 0xbc, 0x42, 0x7b, 0x37, 0x60, 0x99, 0x2b, 0x99, 0xca, 0xd4, 0xae, 0x6b, 0x20, 0xe7, 0x10, 0xd0, -- 0xd7, 0x3e, 0x11, 0xaf, 0x4e, 0x93, 0xf3, 0x3e, 0xac, 0x95, 0x24, 0xf2, 0x84, 0xc6, 0x1c, 0x2b, -- 0x03, 0x84, 0x2f, 0x52, 0xae, 0x84, 0x2d, 0xb9, 0x06, 0x72, 0x30, 0xac, 0x3f, 0x21, 0xdc, 0xb2, -- 0xe3, 0xbf, 0xc7, 0x84, 0x0d, 0x58, 0x3e, 0xa1, 0x2c, 0xf2, 0x85, 0xb5, 0x40, 0x43, 0x08, 0x41, -- 0xdd, 0x67, 0x63, 0x3e, 0xa8, 0xdd, 0xa9, 0x6d, 0xb6, 0x5c, 0xf5, 0x2f, 0x57, 0xe5, 0x8c, 0x1a, -- 0x63, 0xd7, 0x1b, 0xd0, 0x31, 0x71, 0xf7, 0x26, 0x84, 0x0b, 0xa5, 0xa7, 0xe3, 0xb6, 0x0d, 0x4e, -- 0x8e, 0x71, 0x28, 0x6c, 0x1c, 0x25, 0xe1, 0x4b, 0x6e, 0xf8, 0x1d, 0x68, 0x31, 0xcc, 0x69, 0xca, -- 0xe4, 0x36, 0xad, 0xaa, 0x79, 0x5f, 0xd7, 0xf3, 0xfe, 0x84, 0xc4, 0xe9, 0x85, 0x6b, 0x69, 0x6e, -- 0xce, 0x66, 0xb6, 0x90, 0xe0, 0x2f, 0xb3, 0x85, 0x3e, 0x87, 0x6b, 0x87, 0x7e, 0xca, 0x5f, 0xc6, -- 0x56, 0xe7, 0x0b, 0xb9, 0xfd, 0x78, 0x1a, 0xbd, 0xd4, 0xe0, 0x5f, 0x56, 0xa0, 0xb9, 0x97, 0xa4, -- 0x47, 0xdc, 0x1f, 0x63, 0xf4, 0x3a, 0xb4, 0x05, 0x15, 0xfe, 0xc4, 0x4b, 0x25, 0xa8, 0xd8, 0xeb, -- 0x2e, 0x28, 0x94, 0x66, 0x90, 0x61, 0xc7, 0x2c, 0x48, 0x52, 0xc3, 0x51, 0xbd, 0x53, 0xdb, 0xac, -- 0xbb, 0x6d, 0x8d, 0xd3, 0x2c, 0x5b, 0xb0, 0xa6, 0x68, 0x1e, 0x89, 0xbd, 0x33, 0xcc, 0x62, 0x3c, -- 0x89, 0x68, 0x88, 0xd5, 0xfa, 0xad, 0xbb, 0xab, 0x8a, 0x74, 0x10, 0x7f, 0x95, 0x11, 0xd0, 0x3f, -- 0xc1, 0x6a, 0xc6, 0x2f, 0x37, 0xa5, 0xe2, 0xae, 0x2b, 0xee, 0xbe, 0xe1, 0x3e, 0x32, 0x68, 0xe7, -- 0xff, 0xa1, 0xf7, 0xfc, 0x94, 0x51, 0x21, 0x26, 0x24, 0x1e, 0x3f, 0xf4, 0x85, 0x2f, 0xb3, 0x47, -- 0x82, 0x19, 0xa1, 0x21, 0x37, 0xd6, 0x5a, 0x10, 0xbd, 0x07, 0xab, 0x42, 0xf3, 0xe2, 0xd0, 0xb3, -- 0x3c, 0x55, 0xc5, 0xb3, 0x92, 0x11, 0x0e, 0x0d, 0xf3, 0xdb, 0xd0, 0xcb, 0x99, 0x65, 0xfe, 0x31, -- 0xf6, 0x76, 0x33, 0xec, 0x73, 0x12, 0x61, 0xe7, 0x5c, 0xc5, 0x4a, 0x4d, 0x32, 0x7a, 0x0f, 0x5a, -- 0x79, 0x1c, 0x2a, 0x6a, 0x85, 0xf4, 0xf4, 0x0a, 0xb1, 0xe1, 0x74, 0x9b, 0x59, 0x50, 0xbe, 0x84, -- 0xbe, 0xc8, 0x0c, 0xf7, 0x42, 0x5f, 0xf8, 0xe5, 0x45, 0x55, 0xf6, 0xca, 0xed, 0x89, 0x12, 0xec, -- 0x7c, 0x01, 0xad, 0x43, 0x12, 0x72, 0xad, 0x78, 0x00, 0x8d, 0x20, 0x65, 0x0c, 0xc7, 0xc2, 0xba, -- 0x6c, 0x40, 0xb4, 0x0e, 0x4b, 0x13, 0x12, 0x11, 0x61, 0xdc, 0xd4, 0x80, 0x43, 0x01, 0x9e, 0xe2, -- 0x88, 0xb2, 0xa9, 0x0a, 0xd8, 0x3a, 0x2c, 0x15, 0x27, 0x57, 0x03, 0xe8, 0x26, 0xb4, 0x22, 0xff, -- 0x22, 0x9b, 0x54, 0x49, 0x69, 0x46, 0xfe, 0x85, 0x36, 0x7e, 0x00, 0x8d, 0x13, 0x9f, 0x4c, 0x82, -- 0x58, 0x98, 0xa8, 0x58, 0x30, 0x57, 0x58, 0x2f, 0x2a, 0xfc, 0x5d, 0x15, 0xda, 0x5a, 0xa3, 0x36, -- 0x78, 0x1d, 0x96, 0x02, 0x3f, 0x38, 0xcd, 0x54, 0x2a, 0x00, 0xdd, 0xb5, 0x86, 0x54, 0x8b, 0x49, -- 0x38, 0xb7, 0xd4, 0x9a, 0xb6, 0x0d, 0xc0, 0x5f, 0xf8, 0x89, 0xb1, 0xad, 0x76, 0x09, 0x73, 0x4b, -- 0xf2, 0x68, 0x73, 0x3f, 0x84, 0x8e, 0x5e, 0x77, 0x66, 0x48, 0xfd, 0x92, 0x21, 0x6d, 0xcd, 0xa5, -- 0x07, 0xbd, 0x09, 0xdd, 0x94, 0x63, 0xef, 0x94, 0x60, 0xe6, 0xb3, 0xe0, 0x74, 0x3a, 0x58, 0xd2, -- 0x67, 0x64, 0xca, 0xf1, 0xbe, 0xc5, 0xa1, 0x1d, 0x58, 0x92, 0xe9, 0x8f, 0x0f, 0x96, 0xd5, 0x71, -- 0x7c, 0xab, 0x28, 0x52, 0xb9, 0xba, 0xa5, 0xbe, 0x8f, 0x62, 0xc1, 0xa6, 0xae, 0x66, 0x1d, 0x7e, -- 0x0a, 0x90, 0x23, 0xd1, 0x0a, 0xd4, 0xce, 0xf0, 0xd4, 0xec, 0x43, 0xf9, 0x2b, 0x83, 0x73, 0xee, -- 0x4f, 0x52, 0x1b, 0x75, 0x0d, 0x7c, 0x5e, 0xfd, 0xb4, 0xe2, 0x04, 0xd0, 0xdf, 0x9d, 0x9c, 0x11, -- 0x5a, 0x18, 0xbe, 0x0e, 0x4b, 0x91, 0xff, 0x0d, 0x65, 0x36, 0x92, 0x0a, 0x50, 0x58, 0x12, 0x53, -- 0x66, 0x45, 0x28, 0x00, 0xf5, 0xa0, 0x4a, 0x13, 0x15, 0xaf, 0x96, 0x5b, 0xa5, 0x49, 0xae, 0xa8, -- 0x5e, 0x50, 0xe4, 0xfc, 0xb9, 0x0e, 0x90, 0x6b, 0x41, 0x2e, 0x0c, 0x09, 0xf5, 0x38, 0x66, 0xb2, -- 0x04, 0xf1, 0x8e, 0xa7, 0x02, 0x73, 0x8f, 0xe1, 0x20, 0x65, 0x9c, 0x9c, 0xcb, 0xf9, 0x93, 0x6e, -- 0x5f, 0xd3, 0x6e, 0xcf, 0xd8, 0xe6, 0x5e, 0x27, 0x74, 0xa4, 0xc7, 0xed, 0xca, 0x61, 0xae, 0x1d, -- 0x85, 0x0e, 0xe0, 0x5a, 0x2e, 0x33, 0x2c, 0x88, 0xab, 0x5e, 0x25, 0x6e, 0x2d, 0x13, 0x17, 0xe6, -- 0xa2, 0x1e, 0xc1, 0x1a, 0xa1, 0xde, 0xb7, 0x29, 0x4e, 0x4b, 0x82, 0x6a, 0x57, 0x09, 0x5a, 0x25, -- 0xf4, 0x3f, 0xd4, 0x80, 0x5c, 0xcc, 0x21, 0xdc, 0x28, 0x78, 0x29, 0xb7, 0x7b, 0x41, 0x58, 0xfd, -- 0x2a, 0x61, 0x1b, 0x99, 0x55, 0x32, 0x1f, 0xe4, 0x12, 0xff, 0x0d, 0x36, 0x08, 0xf5, 0x5e, 0xf8, -- 0x44, 0xcc, 0x8a, 0x5b, 0xfa, 0x11, 0x27, 0xe5, 0xa1, 0x5b, 0x96, 0xa5, 0x9d, 0x8c, 0x30, 0x1b, -- 0x97, 0x9c, 0x5c, 0xfe, 0x11, 0x27, 0x9f, 0xaa, 0x01, 0xb9, 0x98, 0x07, 0xb0, 0x4a, 0xe8, 0xac, -- 0x35, 0x8d, 0xab, 0x84, 0xf4, 0x09, 0x2d, 0x5b, 0xb2, 0x0b, 0xab, 0x1c, 0x07, 0x82, 0xb2, 0xe2, -- 0x22, 0x68, 0x5e, 0x25, 0x62, 0xc5, 0xf0, 0x67, 0x32, 0x9c, 0xff, 0x81, 0xce, 0x7e, 0x3a, 0xc6, -- 0x62, 0x72, 0x9c, 0x25, 0x83, 0x57, 0x96, 0x7f, 0x9c, 0xbf, 0x56, 0xa1, 0xbd, 0x37, 0x66, 0x34, -- 0x4d, 0x4a, 0x39, 0x59, 0x6f, 0xd2, 0xd9, 0x9c, 0xac, 0x58, 0x54, 0x4e, 0xd6, 0xcc, 0x1f, 0x41, -- 0x27, 0x52, 0x5b, 0xd7, 0xf0, 0xeb, 0x3c, 0xb4, 0x3a, 0xb7, 0xa9, 0xdd, 0x76, 0x54, 0x48, 0x66, -- 0x5b, 0x00, 0x09, 0x09, 0xb9, 0x19, 0xa3, 0xd3, 0x51, 0xdf, 0x54, 0x84, 0x36, 0x45, 0xbb, 0xad, -- 0x24, 0xcb, 0xd6, 0xf7, 0xa0, 0x7d, 0x2c, 0x83, 0x64, 0x06, 0x94, 0x92, 0x51, 0x1e, 0x3d, 0x17, -- 0x8e, 0xf3, 0x4d, 0xb8, 0x0f, 0xdd, 0x53, 0x1d, 0x32, 0x33, 0x48, 0xaf, 0xa1, 0x37, 0x8d, 0x27, -- 0xb9, 0xbf, 0x5b, 0xc5, 0xc8, 0xea, 0x09, 0xe8, 0x9c, 0x16, 0x50, 0xc3, 0x11, 0xac, 0xce, 0xb1, -- 0x2c, 0xc8, 0x41, 0x9b, 0xc5, 0x1c, 0xd4, 0xde, 0x41, 0x5a, 0x51, 0x71, 0x64, 0x31, 0x2f, 0xfd, -- 0xac, 0x0a, 0xbd, 0x83, 0x58, 0x60, 0x76, 0xe2, 0x07, 0x58, 0x5b, 0x8c, 0xa0, 0x1e, 0xfb, 0x11, -- 0x36, 0x32, 0xd5, 0x3f, 0xba, 0x01, 0x4d, 0x76, 0xa1, 0x53, 0x88, 0x99, 0xd1, 0x06, 0xbb, 0x50, -- 0xa9, 0x01, 0xbd, 0x06, 0xc0, 0x2e, 0xbc, 0xc4, 0x0f, 0xce, 0xb0, 0x89, 0x61, 0xdd, 0x6d, 0xb1, -- 0x8b, 0x43, 0x8d, 0x90, 0x8b, 0x81, 0x5d, 0x78, 0x98, 0x31, 0xca, 0xb8, 0xc9, 0x56, 0x4d, 0x76, -- 0xf1, 0x48, 0xc1, 0x66, 0x6c, 0xc8, 0x68, 0x92, 0xe0, 0x50, 0x65, 0x69, 0x35, 0xf6, 0xa1, 0x46, -- 0x48, 0xad, 0xc2, 0x6a, 0x5d, 0xd6, 0x5a, 0x45, 0xae, 0x55, 0xe4, 0x5a, 0x1b, 0x7a, 0xa4, 0x28, -- 0x6a, 0x15, 0x99, 0xd6, 0xa6, 0xd6, 0x2a, 0x0a, 0x5a, 0x45, 0xae, 0xb5, 0x65, 0xc7, 0x1a, 0xad, -- 0xce, 0x6f, 0xaa, 0xd0, 0x78, 0x1e, 0xa8, 0x49, 0x41, 0x77, 0xa0, 0x8d, 0xb9, 0xf0, 0x8f, 0x27, -- 0x84, 0x9f, 0xe2, 0xd0, 0x2c, 0xf3, 0x22, 0x4a, 0xda, 0xc8, 0xa7, 0xb1, 0xc7, 0xe5, 0x09, 0x6e, -- 0x22, 0xc3, 0xa7, 0xf1, 0x48, 0x9e, 0xe0, 0x86, 0xc4, 0x70, 0x70, 0x6e, 0xd7, 0x3a, 0x9f, 0xc6, -- 0x2e, 0x0e, 0xce, 0xa5, 0x7d, 0x27, 0x24, 0x56, 0x39, 0xe6, 0x9e, 0x8d, 0xca, 0x09, 0x89, 0x65, -- 0xfe, 0xb8, 0x57, 0x24, 0xee, 0x98, 0xa0, 0x58, 0xe2, 0x8e, 0xf2, 0x4c, 0xa6, 0x01, 0x49, 0x35, -- 0x41, 0x69, 0x4a, 0x84, 0xa4, 0xaa, 0xc3, 0x79, 0x42, 0x39, 0x36, 0x01, 0xd1, 0x80, 0xf4, 0x57, -- 0xfd, 0xe8, 0x31, 0x3a, 0x1a, 0x2d, 0x85, 0x51, 0x83, 0x6e, 0x40, 0x73, 0xe2, 0x73, 0xe1, 0xf9, -- 0xc1, 0x99, 0x09, 0x46, 0x43, 0xc2, 0x0f, 0x82, 0x33, 0x59, 0xdd, 0xcb, 0x82, 0x1c, 0xc7, 0x03, -- 0x50, 0x04, 0x03, 0xa9, 0xaa, 0x65, 0x42, 0x39, 0x89, 0xc7, 0x83, 0xb6, 0xa9, 0x5a, 0x34, 0xe8, -- 0xa4, 0xd0, 0x38, 0x0a, 0x75, 0xec, 0xf2, 0xc1, 0x95, 0xd9, 0xc1, 0x36, 0xf6, 0x26, 0x60, 0x06, -- 0x34, 0x6b, 0x45, 0x9f, 0x08, 0x26, 0x62, 0x4d, 0x76, 0xa1, 0x13, 0xbe, 0x99, 0x52, 0x43, 0xac, -- 0xdb, 0x29, 0xd5, 0x44, 0xe7, 0x0f, 0x15, 0xe8, 0xfc, 0x3b, 0x16, 0x2f, 0x28, 0x3b, 0xb3, 0xf9, -- 0x00, 0x88, 0x5d, 0xd6, 0xdc, 0x9c, 0x75, 0xa6, 0x3c, 0x2b, 0x2f, 0x77, 0xb7, 0xc0, 0x87, 0x5e, -- 0x87, 0x9a, 0x08, 0x12, 0xb3, 0x73, 0x4c, 0x6b, 0x68, 0x96, 0x82, 0x2b, 0x29, 0xe8, 0x0d, 0xa8, -- 0x8b, 0x20, 0xf9, 0xd8, 0xa4, 0x8a, 0x19, 0x0e, 0x45, 0x92, 0x32, 0xd2, 0x30, 0x29, 0xb7, 0x97, -- 0x26, 0x24, 0xae, 0xa4, 0x48, 0x19, 0x69, 0x98, 0x7c, 0xac, 0x66, 0x76, 0x8e, 0x43, 0x91, 0x9c, -- 0x9f, 0x56, 0x60, 0x63, 0xb6, 0xfb, 0x30, 0xbd, 0xd2, 0x47, 0xd0, 0x09, 0x54, 0xd2, 0x28, 0x25, -- 0xc6, 0xd5, 0xb9, 0x74, 0xe2, 0xb6, 0x83, 0x42, 0x2e, 0xfd, 0x04, 0xba, 0xb1, 0x0e, 0x4f, 0x29, -- 0x3f, 0x9a, 0xe4, 0x50, 0x8c, 0x9c, 0xdb, 0x89, 0x0b, 0x90, 0x13, 0x02, 0xfa, 0x9a, 0x11, 0x81, -- 0x47, 0x82, 0x61, 0x3f, 0x7a, 0x15, 0x5d, 0x30, 0x82, 0xba, 0x2a, 0x99, 0x6b, 0xaa, 0xc9, 0x53, -- 0xff, 0xce, 0x3b, 0xb0, 0x56, 0xd2, 0x62, 0x7c, 0x5d, 0x81, 0xda, 0xc4, 0x2c, 0x9f, 0xae, 0x2b, -- 0x7f, 0x1d, 0x1f, 0x56, 0x5d, 0xec, 0x87, 0xaf, 0xce, 0x1a, 0xa3, 0xa2, 0x96, 0xab, 0xd8, 0x04, -- 0x54, 0x54, 0x61, 0x4c, 0xb1, 0x56, 0x57, 0x0a, 0x56, 0x3f, 0x83, 0xd5, 0x3d, 0xb9, 0x8b, 0x46, -- 0x22, 0x24, 0xf1, 0xab, 0x68, 0xdb, 0xff, 0x0f, 0xd6, 0x9e, 0x8b, 0xe9, 0xd7, 0x52, 0x18, 0x27, -- 0xdf, 0xe1, 0x57, 0xe4, 0x1f, 0xa3, 0x2f, 0xac, 0x7f, 0x8c, 0xbe, 0x90, 0xdb, 0x32, 0xa0, 0x93, -- 0x34, 0x8a, 0xd5, 0x12, 0xed, 0xba, 0x06, 0x72, 0x76, 0xa1, 0xa3, 0x1b, 0xb9, 0xa7, 0x34, 0x4c, -- 0x27, 0x78, 0xe1, 0x31, 0x70, 0x1b, 0x20, 0xf1, 0x99, 0x1f, 0x61, 0x81, 0x19, 0x57, 0x25, 0x5f, -- 0xcb, 0x2d, 0x60, 0x9c, 0x9f, 0x57, 0x61, 0x5d, 0xdf, 0xcb, 0x8d, 0xf4, 0x75, 0x94, 0x75, 0x61, -- 0x08, 0xcd, 0x53, 0xca, 0x45, 0x41, 0x60, 0x06, 0x4b, 0x13, 0xc3, 0xd8, 0x4a, 0x93, 0xbf, 0xa5, -- 0xcb, 0xb2, 0xda, 0xd5, 0x97, 0x65, 0x73, 0xd7, 0x61, 0xf5, 0xf9, 0xeb, 0x30, 0x99, 0x00, 0x2d, -- 0x13, 0xd1, 0xc7, 0x4c, 0xcb, 0x6d, 0x19, 0xcc, 0x41, 0x88, 0xee, 0x42, 0x7f, 0x2c, 0xad, 0xf4, -- 0x4e, 0x29, 0x3d, 0xf3, 0x12, 0x5f, 0x9c, 0xaa, 0xc4, 0xda, 0x72, 0xbb, 0x0a, 0xbd, 0x4f, 0xe9, -- 0xd9, 0xa1, 0x2f, 0x4e, 0xd1, 0x67, 0xd0, 0x33, 0xbd, 0x48, 0xa4, 0x42, 0xc4, 0x4d, 0x05, 0x66, -- 0x76, 0x51, 0x31, 0x7a, 0x6e, 0xf7, 0xac, 0x00, 0x71, 0xe7, 0x3a, 0x5c, 0x7b, 0x88, 0xb9, 0x60, -- 0x74, 0x5a, 0x0e, 0x8c, 0xf3, 0x2f, 0x00, 0x07, 0x79, 0xfe, 0xf9, 0xa0, 0x08, 0x99, 0xac, 0xb5, -- 0xb2, 0xa5, 0xaf, 0x45, 0x33, 0x82, 0x5b, 0xe0, 0x71, 0xb6, 0x60, 0xd9, 0xa5, 0xa9, 0x3c, 0x11, -- 0xdf, 0xb2, 0x7f, 0x66, 0x5c, 0xc7, 0x8c, 0x53, 0x48, 0xd7, 0xd0, 0x9c, 0x7d, 0x7b, 0x8f, 0x92, -- 0x8b, 0x33, 0x53, 0xb4, 0x05, 0xad, 0x2c, 0x13, 0x9a, 0xac, 0x32, 0xaf, 0x3a, 0x67, 0x71, 0xfe, -- 0x1b, 0xd6, 0xb4, 0x24, 0x2d, 0xd9, 0x8a, 0x79, 0x0b, 0x96, 0x99, 0x35, 0xa3, 0x92, 0xdf, 0x87, -- 0x1a, 0x26, 0x43, 0x43, 0xb7, 0xa4, 0xb2, 0x80, 0xe1, 0xc8, 0x1e, 0x9b, 0x4d, 0x37, 0x47, 0xc8, -- 0x68, 0x3d, 0x21, 0x5c, 0xe4, 0x6e, 0xda, 0x68, 0xad, 0xc1, 0xaa, 0x24, 0x94, 0x34, 0x3a, 0xff, -- 0x0b, 0x6b, 0xcf, 0xe2, 0x09, 0x89, 0xf1, 0xde, 0xe1, 0xd1, 0x53, 0x9c, 0x65, 0x05, 0x04, 0x75, -- 0x75, 0xde, 0x55, 0x94, 0x74, 0xf5, 0x2f, 0xb7, 0x49, 0x7c, 0xec, 0x05, 0x49, 0xca, 0xcd, 0xf5, -- 0xe4, 0x72, 0x7c, 0xbc, 0x97, 0xa4, 0x5c, 0x9e, 0x81, 0xb2, 0xd6, 0xa4, 0xf1, 0x64, 0xaa, 0xf6, -- 0x4a, 0xd3, 0x6d, 0x04, 0x49, 0xfa, 0x2c, 0x9e, 0x4c, 0x9d, 0x7f, 0x56, 0x17, 0x32, 0x18, 0x87, -- 0xae, 0x1f, 0x87, 0x34, 0x7a, 0x88, 0xcf, 0x0b, 0x1a, 0xb2, 0xe6, 0xdf, 0xe6, 0x84, 0xef, 0x2b, -- 0xd0, 0x79, 0x30, 0xc6, 0xb1, 0x78, 0x88, 0x85, 0x4f, 0x26, 0xaa, 0xc1, 0x3f, 0xc7, 0x8c, 0x13, -- 0x1a, 0x9b, 0x85, 0x6f, 0x41, 0xf4, 0x3a, 0xb4, 0x49, 0x4c, 0x84, 0x17, 0xfa, 0x38, 0xa2, 0xb1, -- 0x89, 0x02, 0x48, 0xd4, 0x43, 0x85, 0x41, 0xef, 0x40, 0x5f, 0x5f, 0x1f, 0x7b, 0xa7, 0x7e, 0x1c, -- 0x4e, 0xe4, 0x96, 0xd3, 0xd7, 0x69, 0x3d, 0x8d, 0xde, 0x37, 0x58, 0xf4, 0x2e, 0xac, 0x98, 0x0d, -- 0x91, 0x73, 0xd6, 0x15, 0x67, 0xdf, 0xe0, 0x4b, 0xac, 0x69, 0x92, 0x50, 0x26, 0xb8, 0xc7, 0x71, -- 0x10, 0xd0, 0x28, 0x31, 0xdd, 0x71, 0xdf, 0xe2, 0x47, 0x1a, 0xed, 0x8c, 0x61, 0xed, 0xb1, 0xf4, -- 0xd3, 0x78, 0x92, 0x4f, 0x70, 0x2f, 0xc2, 0x91, 0x77, 0x3c, 0xa1, 0xc1, 0x99, 0x27, 0xd3, 0x94, -- 0x89, 0xb0, 0xac, 0xbf, 0x77, 0x25, 0x72, 0x44, 0xbe, 0x53, 0x17, 0x41, 0x92, 0xeb, 0x94, 0x8a, -- 0x64, 0x92, 0x8e, 0xbd, 0x84, 0xd1, 0x63, 0x6c, 0x5c, 0xec, 0x47, 0x38, 0xda, 0xd7, 0xf8, 0x43, -- 0x89, 0x76, 0x7e, 0x5b, 0x81, 0xf5, 0xb2, 0x26, 0x93, 0x74, 0xb7, 0x61, 0xbd, 0xac, 0xca, 0xd4, -- 0x82, 0xba, 0x9e, 0x58, 0x2d, 0x2a, 0xd4, 0x55, 0xe1, 0x27, 0xd0, 0x55, 0x6f, 0x0a, 0x5e, 0xa8, -- 0x25, 0x95, 0x8f, 0xb9, 0xe2, 0xbc, 0xb8, 0x1d, 0xbf, 0x38, 0x4b, 0x9f, 0xc1, 0x0d, 0xe3, 0xbe, -- 0x37, 0x6f, 0xb6, 0x5e, 0x10, 0x1b, 0x86, 0xe1, 0xe9, 0x8c, 0xf5, 0x4f, 0x60, 0x90, 0xa3, 0x76, -- 0xa7, 0x0a, 0x69, 0x63, 0xf5, 0x01, 0xac, 0xcd, 0x38, 0xfb, 0x20, 0x0c, 0x99, 0xda, 0xa0, 0x75, -- 0x77, 0x11, 0xc9, 0xb9, 0x0f, 0xd7, 0x47, 0x58, 0xe8, 0x68, 0xf8, 0xc2, 0x34, 0xa6, 0x5a, 0xd8, -- 0x0a, 0xd4, 0x46, 0x38, 0x50, 0xce, 0xd7, 0x5c, 0xf9, 0x2b, 0x17, 0xe0, 0x11, 0xc7, 0x81, 0xf2, -- 0xb2, 0xe6, 0xaa, 0x7f, 0xe7, 0xd7, 0x15, 0x68, 0x98, 0x34, 0x29, 0x53, 0x7d, 0xc8, 0xc8, 0x39, -- 0x66, 0x66, 0xe9, 0x19, 0x08, 0xbd, 0x0d, 0x3d, 0xfd, 0xe7, 0xd1, 0x44, 0x10, 0x9a, 0x25, 0xdf, -- 0xae, 0xc6, 0x3e, 0xd3, 0x48, 0x75, 0x5d, 0xac, 0x6e, 0x43, 0xcd, 0xc5, 0x83, 0x81, 0xd4, 0x9d, -- 0x2f, 0x97, 0x99, 0x41, 0x25, 0xdb, 0x96, 0x6b, 0x20, 0xb9, 0xd4, 0xad, 0xbc, 0x25, 0x25, 0xcf, -- 0x82, 0x72, 0xa9, 0x47, 0x34, 0x8d, 0x85, 0x97, 0x50, 0x12, 0x0b, 0x93, 0x5d, 0x41, 0xa1, 0x0e, -- 0x25, 0xc6, 0xf9, 0x49, 0x05, 0x96, 0xf5, 0x93, 0x09, 0xea, 0x41, 0x35, 0x3b, 0xe3, 0xaa, 0x44, -- 0xd5, 0x0b, 0x4a, 0x97, 0x3e, 0xd7, 0xd4, 0xbf, 0xdc, 0xc7, 0xe7, 0x91, 0xce, 0xd4, 0xc6, 0xb4, -- 0xf3, 0x48, 0xa5, 0xe8, 0xb7, 0xa1, 0x97, 0x1f, 0x95, 0x8a, 0xae, 0x4d, 0xec, 0x66, 0x58, 0xc5, -- 0x76, 0xa9, 0xa5, 0xce, 0x7f, 0x01, 0xe4, 0x4f, 0x07, 0x32, 0xe4, 0x69, 0x66, 0x8c, 0xfc, 0x95, -- 0x98, 0x71, 0x76, 0xc8, 0xca, 0x5f, 0x74, 0x17, 0x7a, 0x7e, 0x18, 0x12, 0x39, 0xdc, 0x9f, 0x3c, -- 0x26, 0x61, 0xb6, 0x49, 0xcb, 0x58, 0xe7, 0xf7, 0x15, 0xe8, 0xef, 0xd1, 0x64, 0xfa, 0xaf, 0x64, -- 0x82, 0x0b, 0x19, 0x44, 0x19, 0x69, 0xce, 0x58, 0xf9, 0xaf, 0xab, 0xff, 0x09, 0xd6, 0x5b, 0x4b, -- 0xcf, 0x6c, 0x53, 0x22, 0xd4, 0xb6, 0xb2, 0xc4, 0xec, 0x16, 0xb6, 0xab, 0x89, 0x4f, 0x69, 0xa8, -- 0x9a, 0xb4, 0x90, 0x30, 0x2f, 0xbb, 0x73, 0xed, 0xba, 0x8d, 0x90, 0x30, 0x45, 0x32, 0x8e, 0x2c, -- 0xa9, 0x6b, 0xff, 0xa2, 0x23, 0xcb, 0x1a, 0x23, 0x1d, 0xd9, 0x80, 0x65, 0x7a, 0x72, 0xc2, 0xb1, -- 0x50, 0xdd, 0x43, 0xcd, 0x35, 0x50, 0x96, 0xe6, 0x9a, 0x85, 0x34, 0x77, 0x0d, 0xd6, 0xd4, 0x03, -- 0xd3, 0x73, 0xe6, 0x07, 0x24, 0x1e, 0xdb, 0x54, 0xbc, 0x0e, 0x68, 0x24, 0x68, 0x32, 0x83, 0xdd, -- 0x82, 0x55, 0x73, 0xe6, 0x1c, 0xfe, 0xe7, 0xc8, 0xba, 0x7e, 0x03, 0x9a, 0x12, 0xf4, 0x18, 0xfe, -- 0xd6, 0x26, 0x46, 0x43, 0x76, 0xde, 0x85, 0x8e, 0xfe, 0x35, 0x69, 0x20, 0x67, 0xe5, 0x65, 0x56, -- 0xbe, 0xf3, 0xa7, 0x15, 0x93, 0x6e, 0xcd, 0x45, 0x0e, 0x7a, 0x0c, 0xfd, 0x99, 0x87, 0x41, 0x64, -- 0x6e, 0xf6, 0x16, 0xbf, 0x17, 0x0e, 0x37, 0xb6, 0xf4, 0x43, 0xe3, 0x96, 0x7d, 0x68, 0xdc, 0x7a, -- 0x14, 0x25, 0x62, 0x8a, 0x1e, 0x41, 0xaf, 0xfc, 0x84, 0x86, 0x6e, 0xda, 0x1a, 0x64, 0xc1, 0xc3, -- 0xda, 0xa5, 0x62, 0x1e, 0x43, 0x7f, 0xe6, 0x35, 0xcd, 0xda, 0xb3, 0xf8, 0x91, 0xed, 0x52, 0x41, -- 0xf7, 0xa1, 0x5d, 0x78, 0x3e, 0x43, 0x03, 0x2d, 0x64, 0xfe, 0x45, 0xed, 0x52, 0x01, 0x7b, 0xd0, -- 0x2d, 0xbd, 0x68, 0xa1, 0xa1, 0xf1, 0x67, 0xc1, 0x33, 0xd7, 0xa5, 0x42, 0x76, 0xa1, 0x5d, 0x78, -- 0x58, 0xb2, 0x56, 0xcc, 0xbf, 0x5e, 0x0d, 0x6f, 0x2c, 0xa0, 0x98, 0xe9, 0xdc, 0x87, 0x6e, 0xe9, -- 0x19, 0xc8, 0x1a, 0xb2, 0xe8, 0x09, 0x6a, 0x78, 0x73, 0x21, 0xcd, 0x48, 0x7a, 0x0c, 0xfd, 0x99, -- 0x47, 0x21, 0x1b, 0xdc, 0xc5, 0x6f, 0x45, 0x97, 0xba, 0xf5, 0x95, 0x9a, 0xec, 0x42, 0xbb, 0x55, -- 0x98, 0xec, 0xf9, 0x27, 0xa0, 0xe1, 0xad, 0xc5, 0x44, 0x63, 0xd5, 0x23, 0xe8, 0x95, 0x5f, 0x7f, -- 0xac, 0xb0, 0x85, 0x6f, 0x42, 0x57, 0xaf, 0x9c, 0xd2, 0x43, 0x50, 0xbe, 0x72, 0x16, 0xbd, 0x0f, -- 0x5d, 0x2a, 0xe8, 0x01, 0x80, 0x69, 0xae, 0x42, 0x12, 0x67, 0x53, 0x36, 0xd7, 0xd4, 0x65, 0x53, -- 0xb6, 0xa0, 0x11, 0xbb, 0x0f, 0xa0, 0x7b, 0xa2, 0x90, 0xa6, 0x02, 0x5d, 0xb7, 0x66, 0xcc, 0x34, -- 0x62, 0xc3, 0xc1, 0x3c, 0x61, 0x4e, 0x00, 0x66, 0xec, 0x65, 0x04, 0x7c, 0x09, 0x90, 0xf7, 0x5a, -- 0x56, 0xc0, 0x5c, 0xf7, 0x75, 0x45, 0x0c, 0x3a, 0xc5, 0xce, 0x0a, 0x19, 0x5f, 0x17, 0x74, 0x5b, -- 0x57, 0x88, 0xe8, 0xcf, 0x54, 0xce, 0xe5, 0xc5, 0x36, 0x5b, 0x50, 0x0f, 0xe7, 0xaa, 0x67, 0xf4, -- 0x09, 0x74, 0x8a, 0x25, 0xb3, 0xb5, 0x62, 0x41, 0x19, 0x3d, 0x2c, 0x95, 0xcd, 0xe8, 0x3e, 0xf4, -- 0xca, 0x05, 0x31, 0x2a, 0xec, 0x8b, 0xb9, 0x32, 0x79, 0xb8, 0x32, 0x73, 0xd1, 0xc1, 0xd1, 0x87, -- 0x00, 0x79, 0xe1, 0x6c, 0xc3, 0x37, 0x57, 0x4a, 0xcf, 0x68, 0xfd, 0x12, 0x7a, 0x85, 0xbc, 0x2d, -- 0x7b, 0xc2, 0xeb, 0x25, 0x87, 0xf3, 0x6c, 0x3e, 0x34, 0x15, 0x56, 0x29, 0x6d, 0x3f, 0x80, 0x4e, -- 0xf1, 0x8c, 0xb0, 0xde, 0x2e, 0x38, 0x37, 0xae, 0x4a, 0x7a, 0x85, 0xf3, 0xc4, 0xae, 0xdd, 0xf9, -- 0x23, 0xe6, 0xaa, 0xa4, 0x57, 0xea, 0x47, 0x6d, 0xae, 0x59, 0xd4, 0xa4, 0x5e, 0x75, 0x14, 0x94, -- 0x9b, 0x37, 0x1b, 0xfd, 0x85, 0x2d, 0xdd, 0x55, 0x6b, 0xb0, 0xd8, 0xa7, 0xd8, 0x78, 0x2c, 0xe8, -- 0x5d, 0x7e, 0x24, 0x27, 0x14, 0x7b, 0x91, 0x42, 0x4e, 0x58, 0xd0, 0xa2, 0x5c, 0x2a, 0x68, 0x1f, -- 0xfa, 0x8f, 0x6d, 0x99, 0x69, 0x4a, 0x60, 0x63, 0xce, 0x82, 0x92, 0x7f, 0x38, 0x5c, 0x44, 0x32, -- 0xb3, 0xfc, 0x15, 0xac, 0xce, 0x95, 0xbf, 0xe8, 0x76, 0x76, 0xef, 0xbe, 0xb0, 0x2e, 0xbe, 0xd4, -- 0xac, 0x03, 0x58, 0x99, 0xad, 0x7e, 0xd1, 0x6b, 0x66, 0xd2, 0x17, 0x57, 0xc5, 0x97, 0x8a, 0xfa, -- 0x0c, 0x9a, 0xb6, 0xda, 0x42, 0xe6, 0x7d, 0x63, 0xa6, 0xfa, 0xba, 0x6c, 0xe8, 0x6e, 0xe7, 0xfb, -- 0x1f, 0x6e, 0x57, 0xfe, 0xf8, 0xc3, 0xed, 0xca, 0x5f, 0x7e, 0xb8, 0x5d, 0x39, 0x5e, 0x56, 0xd4, -- 0x0f, 0xff, 0x16, 0x00, 0x00, 0xff, 0xff, 0x9a, 0x16, 0x45, 0x76, 0xe7, 0x24, 0x00, 0x00, -+ // 3214 bytes of a gzipped FileDescriptorProto -+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0xcb, 0x6e, 0x1c, 0xc7, -+ 0x76, 0x98, 0x07, 0x39, 0x33, 0x67, 0x5e, 0x9c, 0x22, 0x45, 0x8d, 0x46, 0xb2, 0x2c, 0xb7, 0x6d, -+ 0x99, 0x8e, 0x63, 0xd2, 0xa2, 0x0d, 0x3f, 0xe1, 0x08, 0x22, 0xa5, 0x88, 0x8c, 0x25, 0x8b, 0x69, -+ 0x8a, 0x71, 0x8c, 0x20, 0x68, 0x34, 0xbb, 0x8b, 0xc3, 0x32, 0xa7, 0xbb, 0xda, 0x55, 0xd5, 0x14, -+ 0xc7, 0x01, 0xb2, 0x4c, 0xb6, 0x59, 0x65, 0x97, 0x1f, 0x08, 0x92, 0x55, 0x76, 0xc9, 0x36, 0x0b, -+ 0x23, 0xc8, 0xe2, 0x7e, 0xc1, 0xc5, 0x85, 0x3f, 0xe1, 0x7e, 0xc1, 0x45, 0xbd, 0xfa, 0x31, 0x33, -+ 0xa4, 0x71, 0x09, 0x01, 0x77, 0xd3, 0xe8, 0xf3, 0xa8, 0xf3, 0xaa, 0xaa, 0x53, 0xe7, 0x54, 0x41, -+ 0xdb, 0x1f, 0xe3, 0x58, 0x6c, 0x26, 0x8c, 0x0a, 0x8a, 0xea, 0x63, 0x96, 0x04, 0xa3, 0x16, 0x0d, -+ 0x88, 0x46, 0x8c, 0x3e, 0x1d, 0x13, 0x71, 0x9a, 0x1e, 0x6f, 0x06, 0x34, 0xda, 0x3a, 0xf3, 0x85, -+ 0xff, 0x61, 0x40, 0x63, 0xe1, 0x93, 0x18, 0x33, 0xbe, 0xa5, 0x06, 0x6e, 0x25, 0x67, 0xe3, 0x2d, -+ 0x31, 0x4d, 0x30, 0xd7, 0x5f, 0x33, 0xee, 0xf6, 0x98, 0xd2, 0xf1, 0x04, 0x6f, 0x29, 0xe8, 0x38, -+ 0x3d, 0xd9, 0xc2, 0x51, 0x22, 0xa6, 0x9a, 0xe8, 0xfc, 0x5b, 0x15, 0xd6, 0x77, 0x19, 0xf6, 0x05, -+ 0xde, 0xb5, 0xd2, 0x5c, 0xfc, 0x63, 0x8a, 0xb9, 0x40, 0x6f, 0x41, 0x27, 0xd3, 0xe0, 0x91, 0x70, -+ 0x58, 0xb9, 0x57, 0xd9, 0x68, 0xb9, 0xed, 0x0c, 0xb7, 0x1f, 0xa2, 0x9b, 0xd0, 0xc0, 0x17, 0x38, -+ 0x90, 0xd4, 0xaa, 0xa2, 0x2e, 0x4b, 0x70, 0x3f, 0x44, 0x0f, 0xa0, 0xcd, 0x05, 0x23, 0xf1, 0xd8, -+ 0x4b, 0x39, 0x66, 0xc3, 0xda, 0xbd, 0xca, 0x46, 0x7b, 0x7b, 0x65, 0x53, 0xba, 0xb4, 0x79, 0xa8, -+ 0x08, 0x47, 0x1c, 0x33, 0x17, 0x78, 0xf6, 0x8f, 0xee, 0x43, 0x23, 0xc4, 0xe7, 0x24, 0xc0, 0x7c, -+ 0x58, 0xbf, 0x57, 0xdb, 0x68, 0x6f, 0x77, 0x34, 0xfb, 0x63, 0x85, 0x74, 0x2d, 0x11, 0xbd, 0x0f, -+ 0x4d, 0x2e, 0x28, 0xf3, 0xc7, 0x98, 0x0f, 0x97, 0x14, 0x63, 0xd7, 0xca, 0x55, 0x58, 0x37, 0x23, -+ 0xa3, 0x3b, 0x50, 0x7b, 0xb1, 0xbb, 0x3f, 0x5c, 0x56, 0xda, 0xc1, 0x70, 0x25, 0x38, 0x70, 0x25, -+ 0x1a, 0xbd, 0x0d, 0x5d, 0xee, 0xc7, 0xe1, 0x31, 0xbd, 0xf0, 0x12, 0x12, 0xc6, 0x7c, 0xd8, 0xb8, -+ 0x57, 0xd9, 0x68, 0xba, 0x1d, 0x83, 0x3c, 0x90, 0x38, 0xe7, 0x4b, 0xb8, 0x71, 0x28, 0x7c, 0x26, -+ 0xae, 0x11, 0x1d, 0xe7, 0x08, 0xd6, 0x5d, 0x1c, 0xd1, 0xf3, 0x6b, 0x85, 0x76, 0x08, 0x0d, 0x41, -+ 0x22, 0x4c, 0x53, 0xa1, 0x42, 0xdb, 0x75, 0x2d, 0xe8, 0xfc, 0x47, 0x05, 0xd0, 0x93, 0x0b, 0x1c, -+ 0x1c, 0x30, 0x1a, 0x60, 0xce, 0xff, 0x44, 0xd3, 0xf5, 0x1e, 0x34, 0x12, 0x6d, 0xc0, 0xb0, 0xae, -+ 0xd8, 0xcd, 0x2c, 0x58, 0xab, 0x2c, 0xd5, 0xf9, 0x01, 0xd6, 0x0e, 0xc9, 0x38, 0xf6, 0x27, 0xaf, -+ 0xd1, 0xde, 0x75, 0x58, 0xe6, 0x4a, 0xa6, 0x32, 0xb5, 0xeb, 0x1a, 0xc8, 0x39, 0x00, 0xf4, 0x9d, -+ 0x4f, 0xc4, 0xeb, 0xd3, 0xe4, 0x7c, 0x08, 0xab, 0x25, 0x89, 0x3c, 0xa1, 0x31, 0xc7, 0xca, 0x00, -+ 0xe1, 0x8b, 0x94, 0x2b, 0x61, 0x4b, 0xae, 0x81, 0x1c, 0x0c, 0x6b, 0xcf, 0x08, 0xb7, 0xec, 0xf8, -+ 0x8f, 0x31, 0x61, 0x1d, 0x96, 0x4f, 0x28, 0x8b, 0x7c, 0x61, 0x2d, 0xd0, 0x10, 0x42, 0x50, 0xf7, -+ 0xd9, 0x98, 0x0f, 0x6b, 0xf7, 0x6a, 0x1b, 0x2d, 0x57, 0xfd, 0xcb, 0x55, 0x39, 0xa3, 0xc6, 0xd8, -+ 0xf5, 0x16, 0x74, 0x4c, 0xdc, 0xbd, 0x09, 0xe1, 0x42, 0xe9, 0xe9, 0xb8, 0x6d, 0x83, 0x93, 0x63, -+ 0x1c, 0x0a, 0xeb, 0x47, 0x49, 0x78, 0xcd, 0x0d, 0xbf, 0x0d, 0x2d, 0x86, 0x39, 0x4d, 0x99, 0xdc, -+ 0xa6, 0x55, 0x35, 0xef, 0x6b, 0x7a, 0xde, 0x9f, 0x91, 0x38, 0xbd, 0x70, 0x2d, 0xcd, 0xcd, 0xd9, -+ 0xcc, 0x16, 0x12, 0xfc, 0x3a, 0x5b, 0xe8, 0x4b, 0xb8, 0x71, 0xe0, 0xa7, 0xfc, 0x3a, 0xb6, 0x3a, -+ 0x5f, 0xc9, 0xed, 0xc7, 0xd3, 0xe8, 0x5a, 0x83, 0xff, 0xbd, 0x02, 0xcd, 0xdd, 0x24, 0x3d, 0xe2, -+ 0xfe, 0x18, 0xa3, 0x37, 0xa1, 0x2d, 0xa8, 0xf0, 0x27, 0x5e, 0x2a, 0x41, 0xc5, 0x5e, 0x77, 0x41, -+ 0xa1, 0x34, 0x83, 0x0c, 0x3b, 0x66, 0x41, 0x92, 0x1a, 0x8e, 0xea, 0xbd, 0xda, 0x46, 0xdd, 0x6d, -+ 0x6b, 0x9c, 0x66, 0xd9, 0x84, 0x55, 0x45, 0xf3, 0x48, 0xec, 0x9d, 0x61, 0x16, 0xe3, 0x49, 0x44, -+ 0x43, 0xac, 0xd6, 0x6f, 0xdd, 0x1d, 0x28, 0xd2, 0x7e, 0xfc, 0x4d, 0x46, 0x40, 0x7f, 0x06, 0x83, -+ 0x8c, 0x5f, 0x6e, 0x4a, 0xc5, 0x5d, 0x57, 0xdc, 0x7d, 0xc3, 0x7d, 0x64, 0xd0, 0xce, 0x3f, 0x42, -+ 0xef, 0xe5, 0x29, 0xa3, 0x42, 0x4c, 0x48, 0x3c, 0x7e, 0xec, 0x0b, 0x5f, 0x66, 0x8f, 0x04, 0x33, -+ 0x42, 0x43, 0x6e, 0xac, 0xb5, 0x20, 0xfa, 0x00, 0x06, 0x42, 0xf3, 0xe2, 0xd0, 0xb3, 0x3c, 0x55, -+ 0xc5, 0xb3, 0x92, 0x11, 0x0e, 0x0c, 0xf3, 0xbb, 0xd0, 0xcb, 0x99, 0x65, 0xfe, 0x31, 0xf6, 0x76, -+ 0x33, 0xec, 0x4b, 0x12, 0x61, 0xe7, 0x5c, 0xc5, 0x4a, 0x4d, 0x32, 0xfa, 0x00, 0x5a, 0x79, 0x1c, -+ 0x2a, 0x6a, 0x85, 0xf4, 0xf4, 0x0a, 0xb1, 0xe1, 0x74, 0x9b, 0x59, 0x50, 0xbe, 0x86, 0xbe, 0xc8, -+ 0x0c, 0xf7, 0x42, 0x5f, 0xf8, 0xe5, 0x45, 0x55, 0xf6, 0xca, 0xed, 0x89, 0x12, 0xec, 0x7c, 0x05, -+ 0xad, 0x03, 0x12, 0x72, 0xad, 0x78, 0x08, 0x8d, 0x20, 0x65, 0x0c, 0xc7, 0xc2, 0xba, 0x6c, 0x40, -+ 0xb4, 0x06, 0x4b, 0x13, 0x12, 0x11, 0x61, 0xdc, 0xd4, 0x80, 0x43, 0x01, 0x9e, 0xe3, 0x88, 0xb2, -+ 0xa9, 0x0a, 0xd8, 0x1a, 0x2c, 0x15, 0x27, 0x57, 0x03, 0xe8, 0x36, 0xb4, 0x22, 0xff, 0x22, 0x9b, -+ 0x54, 0x49, 0x69, 0x46, 0xfe, 0x85, 0x36, 0x7e, 0x08, 0x8d, 0x13, 0x9f, 0x4c, 0x82, 0x58, 0x98, -+ 0xa8, 0x58, 0x30, 0x57, 0x58, 0x2f, 0x2a, 0xfc, 0xdf, 0x2a, 0xb4, 0xb5, 0x46, 0x6d, 0xf0, 0x1a, -+ 0x2c, 0x05, 0x7e, 0x70, 0x9a, 0xa9, 0x54, 0x00, 0xba, 0x6f, 0x0d, 0xa9, 0x16, 0x93, 0x70, 0x6e, -+ 0xa9, 0x35, 0x6d, 0x0b, 0x80, 0xbf, 0xf2, 0x13, 0x63, 0x5b, 0xed, 0x12, 0xe6, 0x96, 0xe4, 0xd1, -+ 0xe6, 0x7e, 0x0c, 0x1d, 0xbd, 0xee, 0xcc, 0x90, 0xfa, 0x25, 0x43, 0xda, 0x9a, 0x4b, 0x0f, 0x7a, -+ 0x1b, 0xba, 0x29, 0xc7, 0xde, 0x29, 0xc1, 0xcc, 0x67, 0xc1, 0xe9, 0x74, 0xb8, 0xa4, 0xcf, 0xc8, -+ 0x94, 0xe3, 0x3d, 0x8b, 0x43, 0xdb, 0xb0, 0x24, 0xd3, 0x1f, 0x1f, 0x2e, 0xab, 0xe3, 0xf8, 0x4e, -+ 0x51, 0xa4, 0x72, 0x75, 0x53, 0x7d, 0x9f, 0xc4, 0x82, 0x4d, 0x5d, 0xcd, 0x3a, 0xfa, 0x1c, 0x20, -+ 0x47, 0xa2, 0x15, 0xa8, 0x9d, 0xe1, 0xa9, 0xd9, 0x87, 0xf2, 0x57, 0x06, 0xe7, 0xdc, 0x9f, 0xa4, -+ 0x36, 0xea, 0x1a, 0xf8, 0xb2, 0xfa, 0x79, 0xc5, 0x09, 0xa0, 0xbf, 0x33, 0x39, 0x23, 0xb4, 0x30, -+ 0x7c, 0x0d, 0x96, 0x22, 0xff, 0x07, 0xca, 0x6c, 0x24, 0x15, 0xa0, 0xb0, 0x24, 0xa6, 0xcc, 0x8a, -+ 0x50, 0x00, 0xea, 0x41, 0x95, 0x26, 0x2a, 0x5e, 0x2d, 0xb7, 0x4a, 0x93, 0x5c, 0x51, 0xbd, 0xa0, -+ 0xc8, 0xf9, 0x6d, 0x1d, 0x20, 0xd7, 0x82, 0x5c, 0x18, 0x11, 0xea, 0x71, 0xcc, 0x64, 0x09, 0xe2, -+ 0x1d, 0x4f, 0x05, 0xe6, 0x1e, 0xc3, 0x41, 0xca, 0x38, 0x39, 0x97, 0xf3, 0x27, 0xdd, 0xbe, 0xa1, -+ 0xdd, 0x9e, 0xb1, 0xcd, 0xbd, 0x49, 0xe8, 0xa1, 0x1e, 0xb7, 0x23, 0x87, 0xb9, 0x76, 0x14, 0xda, -+ 0x87, 0x1b, 0xb9, 0xcc, 0xb0, 0x20, 0xae, 0x7a, 0x95, 0xb8, 0xd5, 0x4c, 0x5c, 0x98, 0x8b, 0x7a, -+ 0x02, 0xab, 0x84, 0x7a, 0x3f, 0xa6, 0x38, 0x2d, 0x09, 0xaa, 0x5d, 0x25, 0x68, 0x40, 0xe8, 0x5f, -+ 0xab, 0x01, 0xb9, 0x98, 0x03, 0xb8, 0x55, 0xf0, 0x52, 0x6e, 0xf7, 0x82, 0xb0, 0xfa, 0x55, 0xc2, -+ 0xd6, 0x33, 0xab, 0x64, 0x3e, 0xc8, 0x25, 0xfe, 0x15, 0xac, 0x13, 0xea, 0xbd, 0xf2, 0x89, 0x98, -+ 0x15, 0xb7, 0xf4, 0x2b, 0x4e, 0xca, 0x43, 0xb7, 0x2c, 0x4b, 0x3b, 0x19, 0x61, 0x36, 0x2e, 0x39, -+ 0xb9, 0xfc, 0x2b, 0x4e, 0x3e, 0x57, 0x03, 0x72, 0x31, 0x8f, 0x60, 0x40, 0xe8, 0xac, 0x35, 0x8d, -+ 0xab, 0x84, 0xf4, 0x09, 0x2d, 0x5b, 0xb2, 0x03, 0x03, 0x8e, 0x03, 0x41, 0x59, 0x71, 0x11, 0x34, -+ 0xaf, 0x12, 0xb1, 0x62, 0xf8, 0x33, 0x19, 0xce, 0xdf, 0x41, 0x67, 0x2f, 0x1d, 0x63, 0x31, 0x39, -+ 0xce, 0x92, 0xc1, 0x6b, 0xcb, 0x3f, 0xce, 0xef, 0xab, 0xd0, 0xde, 0x1d, 0x33, 0x9a, 0x26, 0xa5, -+ 0x9c, 0xac, 0x37, 0xe9, 0x6c, 0x4e, 0x56, 0x2c, 0x2a, 0x27, 0x6b, 0xe6, 0x4f, 0xa0, 0x13, 0xa9, -+ 0xad, 0x6b, 0xf8, 0x75, 0x1e, 0x1a, 0xcc, 0x6d, 0x6a, 0xb7, 0x1d, 0x15, 0x92, 0xd9, 0x26, 0x40, -+ 0x42, 0x42, 0x6e, 0xc6, 0xe8, 0x74, 0xd4, 0x37, 0x15, 0xa1, 0x4d, 0xd1, 0x6e, 0x2b, 0xc9, 0xb2, -+ 0xf5, 0x03, 0x68, 0x1f, 0xcb, 0x20, 0x99, 0x01, 0xa5, 0x64, 0x94, 0x47, 0xcf, 0x85, 0xe3, 0x7c, -+ 0x13, 0xee, 0x41, 0xf7, 0x54, 0x87, 0xcc, 0x0c, 0xd2, 0x6b, 0xe8, 0x6d, 0xe3, 0x49, 0xee, 0xef, -+ 0x66, 0x31, 0xb2, 0x7a, 0x02, 0x3a, 0xa7, 0x05, 0xd4, 0xe8, 0x10, 0x06, 0x73, 0x2c, 0x0b, 0x72, -+ 0xd0, 0x46, 0x31, 0x07, 0xb5, 0xb7, 0x91, 0x56, 0x54, 0x1c, 0x59, 0xcc, 0x4b, 0xff, 0x52, 0x85, -+ 0xde, 0x7e, 0x2c, 0x30, 0x3b, 0xf1, 0x03, 0xac, 0x2d, 0x46, 0x50, 0x8f, 0xfd, 0x08, 0x1b, 0x99, -+ 0xea, 0x1f, 0xdd, 0x82, 0x26, 0xbb, 0xd0, 0x29, 0xc4, 0xcc, 0x68, 0x83, 0x5d, 0xa8, 0xd4, 0x80, -+ 0xde, 0x00, 0x60, 0x17, 0x5e, 0xe2, 0x07, 0x67, 0xd8, 0xc4, 0xb0, 0xee, 0xb6, 0xd8, 0xc5, 0x81, -+ 0x46, 0xc8, 0xc5, 0xc0, 0x2e, 0x3c, 0xcc, 0x18, 0x65, 0xdc, 0x64, 0xab, 0x26, 0xbb, 0x78, 0xa2, -+ 0x60, 0x33, 0x36, 0x64, 0x34, 0x49, 0x70, 0xa8, 0xb2, 0xb4, 0x1a, 0xfb, 0x58, 0x23, 0xa4, 0x56, -+ 0x61, 0xb5, 0x2e, 0x6b, 0xad, 0x22, 0xd7, 0x2a, 0x72, 0xad, 0x0d, 0x3d, 0x52, 0x14, 0xb5, 0x8a, -+ 0x4c, 0x6b, 0x53, 0x6b, 0x15, 0x05, 0xad, 0x22, 0xd7, 0xda, 0xb2, 0x63, 0x8d, 0x56, 0xe7, 0xbf, -+ 0xab, 0xd0, 0x78, 0x19, 0xa8, 0x49, 0x41, 0xf7, 0xa0, 0x8d, 0xb9, 0xf0, 0x8f, 0x27, 0x84, 0x9f, -+ 0xe2, 0xd0, 0x2c, 0xf3, 0x22, 0x4a, 0xda, 0xc8, 0xa7, 0xb1, 0xc7, 0xe5, 0x09, 0x6e, 0x22, 0xc3, -+ 0xa7, 0xf1, 0xa1, 0x3c, 0xc1, 0x0d, 0x89, 0xe1, 0xe0, 0xdc, 0xae, 0x75, 0x3e, 0x8d, 0x5d, 0x1c, -+ 0x9c, 0x4b, 0xfb, 0x4e, 0x48, 0xac, 0x72, 0xcc, 0x03, 0x1b, 0x95, 0x13, 0x12, 0xcb, 0xfc, 0xf1, -+ 0xa0, 0x48, 0xdc, 0x36, 0x41, 0xb1, 0xc4, 0x6d, 0xe5, 0x99, 0x4c, 0x03, 0x92, 0x6a, 0x82, 0xd2, -+ 0x94, 0x08, 0x49, 0x55, 0x87, 0xf3, 0x84, 0x72, 0x6c, 0x02, 0xa2, 0x01, 0xe9, 0xaf, 0xfa, 0xd1, -+ 0x63, 0x74, 0x34, 0x5a, 0x0a, 0xa3, 0x06, 0xdd, 0x82, 0xe6, 0xc4, 0xe7, 0xc2, 0xf3, 0x83, 0x33, -+ 0x13, 0x8c, 0x86, 0x84, 0x1f, 0x05, 0x67, 0xb2, 0xba, 0x97, 0x05, 0x39, 0x8e, 0x87, 0xa0, 0x08, -+ 0x06, 0x52, 0x55, 0xcb, 0x84, 0x72, 0x12, 0x8f, 0x87, 0x6d, 0x53, 0xb5, 0x68, 0xd0, 0x49, 0xa1, -+ 0x71, 0x14, 0xea, 0xd8, 0xe5, 0x83, 0x2b, 0xb3, 0x83, 0x6d, 0xec, 0x4d, 0xc0, 0x0c, 0x68, 0xd6, -+ 0x8a, 0x3e, 0x11, 0x4c, 0xc4, 0x9a, 0xec, 0x42, 0x27, 0x7c, 0x33, 0xa5, 0x86, 0x58, 0xb7, 0x53, -+ 0xaa, 0x89, 0xce, 0xff, 0x57, 0xa0, 0xf3, 0x2d, 0x16, 0xaf, 0x28, 0x3b, 0xb3, 0xf9, 0x00, 0x88, -+ 0x5d, 0xd6, 0xdc, 0x9c, 0x75, 0xa6, 0x3c, 0x2b, 0x2f, 0x77, 0xb7, 0xc0, 0x87, 0xde, 0x84, 0x9a, -+ 0x08, 0x12, 0xb3, 0x73, 0x4c, 0x6b, 0x68, 0x96, 0x82, 0x2b, 0x29, 0xe8, 0x2d, 0xa8, 0x8b, 0x20, -+ 0xf9, 0xd4, 0xa4, 0x8a, 0x19, 0x0e, 0x45, 0x92, 0x32, 0xd2, 0x30, 0x29, 0xb7, 0x97, 0x26, 0x24, -+ 0xae, 0xa4, 0x48, 0x19, 0x69, 0x98, 0x7c, 0xaa, 0x66, 0x76, 0x8e, 0x43, 0x91, 0x9c, 0x7f, 0xae, -+ 0xc0, 0xfa, 0x6c, 0xf7, 0x61, 0x7a, 0xa5, 0x4f, 0xa0, 0x13, 0xa8, 0xa4, 0x51, 0x4a, 0x8c, 0x83, -+ 0xb9, 0x74, 0xe2, 0xb6, 0x83, 0x42, 0x2e, 0xfd, 0x0c, 0xba, 0xb1, 0x0e, 0x4f, 0x29, 0x3f, 0x9a, -+ 0xe4, 0x50, 0x8c, 0x9c, 0xdb, 0x89, 0x0b, 0x90, 0x13, 0x02, 0xfa, 0x8e, 0x11, 0x81, 0x0f, 0x05, -+ 0xc3, 0x7e, 0xf4, 0x3a, 0xba, 0x60, 0x04, 0x75, 0x55, 0x32, 0xd7, 0x54, 0x93, 0xa7, 0xfe, 0x9d, -+ 0xf7, 0x60, 0xb5, 0xa4, 0xc5, 0xf8, 0xba, 0x02, 0xb5, 0x89, 0x59, 0x3e, 0x5d, 0x57, 0xfe, 0x3a, -+ 0x3e, 0x0c, 0x5c, 0xec, 0x87, 0xaf, 0xcf, 0x1a, 0xa3, 0xa2, 0x96, 0xab, 0xd8, 0x00, 0x54, 0x54, -+ 0x61, 0x4c, 0xb1, 0x56, 0x57, 0x0a, 0x56, 0xbf, 0x80, 0xc1, 0xae, 0xdc, 0x45, 0x87, 0x22, 0x24, -+ 0xf1, 0xeb, 0x68, 0xdb, 0xff, 0x01, 0x56, 0x5f, 0x8a, 0xe9, 0x77, 0x52, 0x18, 0x27, 0x3f, 0xe1, -+ 0xd7, 0xe4, 0x1f, 0xa3, 0xaf, 0xac, 0x7f, 0x8c, 0xbe, 0x92, 0xdb, 0x32, 0xa0, 0x93, 0x34, 0x8a, -+ 0xd5, 0x12, 0xed, 0xba, 0x06, 0x72, 0x76, 0xa0, 0xa3, 0x1b, 0xb9, 0xe7, 0x34, 0x4c, 0x27, 0x78, -+ 0xe1, 0x31, 0x70, 0x17, 0x20, 0xf1, 0x99, 0x1f, 0x61, 0x81, 0x19, 0x57, 0x25, 0x5f, 0xcb, 0x2d, -+ 0x60, 0x9c, 0x7f, 0xad, 0xc2, 0x9a, 0xbe, 0x97, 0x3b, 0xd4, 0xd7, 0x51, 0xd6, 0x85, 0x11, 0x34, -+ 0x4f, 0x29, 0x17, 0x05, 0x81, 0x19, 0x2c, 0x4d, 0x0c, 0x63, 0x2b, 0x4d, 0xfe, 0x96, 0x2e, 0xcb, -+ 0x6a, 0x57, 0x5f, 0x96, 0xcd, 0x5d, 0x87, 0xd5, 0xe7, 0xaf, 0xc3, 0x64, 0x02, 0xb4, 0x4c, 0x44, -+ 0x1f, 0x33, 0x2d, 0xb7, 0x65, 0x30, 0xfb, 0x21, 0xba, 0x0f, 0xfd, 0xb1, 0xb4, 0xd2, 0x3b, 0xa5, -+ 0xf4, 0xcc, 0x4b, 0x7c, 0x71, 0xaa, 0x12, 0x6b, 0xcb, 0xed, 0x2a, 0xf4, 0x1e, 0xa5, 0x67, 0x07, -+ 0xbe, 0x38, 0x45, 0x5f, 0x40, 0xcf, 0xf4, 0x22, 0x91, 0x0a, 0x11, 0x37, 0x15, 0x98, 0xd9, 0x45, -+ 0xc5, 0xe8, 0xb9, 0xdd, 0xb3, 0x02, 0xc4, 0x9d, 0x9b, 0x70, 0xe3, 0x31, 0xe6, 0x82, 0xd1, 0x69, -+ 0x39, 0x30, 0xce, 0x5f, 0x00, 0xec, 0xe7, 0xf9, 0xe7, 0xa3, 0x22, 0x64, 0xb2, 0xd6, 0xca, 0xa6, -+ 0xbe, 0x16, 0xcd, 0x08, 0x6e, 0x81, 0xc7, 0xd9, 0x84, 0x65, 0x97, 0xa6, 0xf2, 0x44, 0x7c, 0xc7, -+ 0xfe, 0x99, 0x71, 0x1d, 0x33, 0x4e, 0x21, 0x5d, 0x43, 0x73, 0xf6, 0xec, 0x3d, 0x4a, 0x2e, 0xce, -+ 0x4c, 0xd1, 0x26, 0xb4, 0xb2, 0x4c, 0x68, 0xb2, 0xca, 0xbc, 0xea, 0x9c, 0xc5, 0x79, 0x09, 0xce, -+ 0x8c, 0xa4, 0xbd, 0x57, 0x8f, 0xc2, 0x90, 0xed, 0x4c, 0xbf, 0xf5, 0xa3, 0x6b, 0x4b, 0xfd, 0x1e, -+ 0x56, 0xb5, 0x54, 0x6d, 0xaf, 0x15, 0xf3, 0x0e, 0x2c, 0x33, 0xeb, 0x5c, 0x25, 0xbf, 0x65, 0x35, -+ 0x4c, 0x86, 0x86, 0xee, 0x48, 0x65, 0x01, 0xc3, 0x91, 0x3d, 0x8c, 0x9b, 0x6e, 0x8e, 0x90, 0x73, -+ 0xf0, 0x8c, 0x70, 0x91, 0x07, 0xcf, 0xce, 0xc1, 0x2a, 0x0c, 0x24, 0xa1, 0xa4, 0xd1, 0xf9, 0x7b, -+ 0x58, 0x7d, 0x11, 0x4f, 0x48, 0x8c, 0x77, 0x0f, 0x8e, 0x9e, 0xe3, 0x2c, 0xd7, 0x20, 0xa8, 0xab, -+ 0x53, 0xb4, 0xa2, 0xa4, 0xab, 0x7f, 0xb9, 0xf9, 0xe2, 0x63, 0x2f, 0x48, 0x52, 0x6e, 0x2e, 0x3d, -+ 0x97, 0xe3, 0xe3, 0xdd, 0x24, 0xe5, 0xf2, 0x64, 0x95, 0x15, 0x2c, 0x8d, 0x27, 0x53, 0xb5, 0x03, -+ 0x9b, 0x6e, 0x23, 0x48, 0xd2, 0x17, 0xf1, 0x64, 0xea, 0xfc, 0xb9, 0xba, 0xe6, 0xc1, 0x38, 0x74, -+ 0xfd, 0x38, 0xa4, 0xd1, 0x63, 0x7c, 0x5e, 0xd0, 0x90, 0x5d, 0x29, 0xd8, 0x4c, 0xf3, 0x73, 0x05, -+ 0x3a, 0x8f, 0xc6, 0x38, 0x16, 0x8f, 0xb1, 0xf0, 0xc9, 0x44, 0x5d, 0x1b, 0x9c, 0x63, 0xc6, 0x09, -+ 0x8d, 0xcd, 0x76, 0xb2, 0x20, 0x7a, 0x13, 0xda, 0x24, 0x26, 0xc2, 0x0b, 0x7d, 0x1c, 0xd1, 0xd8, -+ 0x44, 0x01, 0x24, 0xea, 0xb1, 0xc2, 0xa0, 0xf7, 0xa0, 0xaf, 0x2f, 0xa5, 0xbd, 0x53, 0x3f, 0x0e, -+ 0x27, 0x72, 0x23, 0xeb, 0x4b, 0xba, 0x9e, 0x46, 0xef, 0x19, 0x2c, 0x7a, 0x1f, 0x56, 0xcc, 0x36, -+ 0xcb, 0x39, 0xeb, 0x8a, 0xb3, 0x6f, 0xf0, 0x25, 0xd6, 0x34, 0x49, 0x28, 0x13, 0xdc, 0xe3, 0x38, -+ 0x08, 0x68, 0x94, 0x98, 0x9e, 0xbb, 0x6f, 0xf1, 0x87, 0x1a, 0xed, 0x8c, 0x61, 0xf5, 0xa9, 0xf4, -+ 0xd3, 0x78, 0x92, 0x4f, 0x70, 0x2f, 0xc2, 0x91, 0x77, 0x3c, 0xa1, 0xc1, 0x99, 0x27, 0x93, 0x9f, -+ 0x89, 0xb0, 0xac, 0xea, 0x77, 0x24, 0xf2, 0x90, 0xfc, 0xa4, 0xae, 0x97, 0x24, 0xd7, 0x29, 0x15, -+ 0xc9, 0x24, 0x1d, 0x7b, 0x09, 0xa3, 0xc7, 0xd8, 0xb8, 0xd8, 0x8f, 0x70, 0xb4, 0xa7, 0xf1, 0x07, -+ 0x12, 0xed, 0xfc, 0x4f, 0x05, 0xd6, 0xca, 0x9a, 0x4c, 0x2a, 0xdf, 0x82, 0xb5, 0xb2, 0x2a, 0x53, -+ 0x61, 0xea, 0x2a, 0x65, 0x50, 0x54, 0xa8, 0x6b, 0xcd, 0xcf, 0xa0, 0xab, 0x5e, 0x2a, 0xbc, 0x50, -+ 0x4b, 0x2a, 0x1f, 0x9e, 0xc5, 0x79, 0x71, 0x3b, 0x7e, 0x71, 0x96, 0xbe, 0x80, 0x5b, 0xc6, 0x7d, -+ 0x6f, 0xde, 0x6c, 0xbd, 0x20, 0xd6, 0x0d, 0xc3, 0xf3, 0x19, 0xeb, 0x9f, 0xc1, 0x30, 0x47, 0xed, -+ 0x4c, 0x15, 0xd2, 0xc6, 0xea, 0x23, 0x58, 0x9d, 0x71, 0x56, 0xee, 0x3b, 0xb5, 0xed, 0xeb, 0xee, -+ 0x22, 0x92, 0xf3, 0x10, 0x6e, 0x1e, 0x62, 0xa1, 0xa3, 0xe1, 0x0b, 0xd3, 0xee, 0x6a, 0x61, 0x2b, -+ 0x50, 0x3b, 0xc4, 0x81, 0x72, 0xbe, 0xe6, 0xca, 0x5f, 0xb9, 0x00, 0x8f, 0x38, 0x0e, 0x94, 0x97, -+ 0x35, 0x57, 0xfd, 0x3b, 0xff, 0x55, 0x81, 0x86, 0x49, 0xbe, 0xf2, 0x00, 0x09, 0x19, 0x39, 0xc7, -+ 0xcc, 0x2c, 0x3d, 0x03, 0xa1, 0x77, 0xa1, 0xa7, 0xff, 0x3c, 0x9a, 0x08, 0x42, 0xb3, 0x94, 0xde, -+ 0xd5, 0xd8, 0x17, 0x1a, 0xa9, 0x2e, 0xa1, 0xd5, 0x1d, 0xab, 0xb9, 0xce, 0x30, 0x90, 0xba, 0x49, -+ 0xe6, 0x32, 0x33, 0xa8, 0x14, 0xde, 0x72, 0x0d, 0x24, 0x97, 0xba, 0x95, 0xb7, 0xa4, 0xe4, 0x59, -+ 0x50, 0x2e, 0xf5, 0x88, 0xa6, 0xb1, 0xf0, 0x12, 0x4a, 0x62, 0x61, 0x72, 0x36, 0x28, 0xd4, 0x81, -+ 0xc4, 0x38, 0xff, 0x54, 0x81, 0x65, 0xfd, 0x10, 0x83, 0x7a, 0x50, 0xcd, 0x4e, 0xce, 0x2a, 0x51, -+ 0x55, 0x88, 0xd2, 0xa5, 0x4f, 0x4b, 0xf5, 0x2f, 0xf7, 0xf1, 0x79, 0xa4, 0xf3, 0xbf, 0x31, 0xed, -+ 0x3c, 0x52, 0x89, 0xff, 0x5d, 0xe8, 0xe5, 0x07, 0xb0, 0xa2, 0x6b, 0x13, 0xbb, 0x19, 0x56, 0xb1, -+ 0x5d, 0x6a, 0xa9, 0xf3, 0xb7, 0x00, 0xf9, 0x83, 0x84, 0x0c, 0x79, 0x9a, 0x19, 0x23, 0x7f, 0x25, -+ 0x66, 0x9c, 0x1d, 0xdd, 0xf2, 0x17, 0xdd, 0x87, 0x9e, 0x1f, 0x86, 0x44, 0x0e, 0xf7, 0x27, 0x4f, -+ 0x49, 0x98, 0x6d, 0xd2, 0x32, 0xd6, 0xf9, 0xbf, 0x0a, 0xf4, 0x77, 0x69, 0x32, 0xfd, 0x4b, 0x32, -+ 0xc1, 0x85, 0x0c, 0xa2, 0x8c, 0x34, 0x27, 0xb7, 0xfc, 0xd7, 0x3d, 0xc5, 0x04, 0xeb, 0xad, 0xa5, -+ 0x67, 0xb6, 0x29, 0x11, 0x6a, 0x5b, 0x59, 0x62, 0x76, 0xb7, 0xdb, 0xd5, 0xc4, 0xe7, 0x34, 0x54, -+ 0xad, 0x5f, 0x48, 0x98, 0x97, 0xdd, 0xe4, 0x76, 0xdd, 0x46, 0x48, 0x98, 0x22, 0x19, 0x47, 0x96, -+ 0xd4, 0x63, 0x42, 0xd1, 0x91, 0x65, 0x8d, 0x91, 0x8e, 0xac, 0xc3, 0x32, 0x3d, 0x39, 0xe1, 0x58, -+ 0xa8, 0x9e, 0xa4, 0xe6, 0x1a, 0x28, 0x4b, 0x73, 0xcd, 0x42, 0x9a, 0xbb, 0x01, 0xab, 0xea, 0xd9, -+ 0xea, 0x25, 0xf3, 0x03, 0x12, 0x8f, 0x6d, 0x2a, 0x5e, 0x03, 0x74, 0x28, 0x68, 0x32, 0x83, 0xdd, -+ 0x84, 0x81, 0x39, 0x7f, 0x0e, 0xfe, 0xe6, 0xd0, 0xba, 0x7e, 0x0b, 0x9a, 0x12, 0xf4, 0x18, 0xfe, -+ 0xd1, 0x26, 0x46, 0x43, 0x76, 0xde, 0x87, 0x8e, 0xfe, 0x35, 0x69, 0x20, 0x67, 0xe5, 0x65, 0x56, -+ 0xbe, 0xfd, 0x9f, 0x03, 0x93, 0x6e, 0xcd, 0xf5, 0x10, 0x7a, 0x0a, 0xfd, 0x99, 0xe7, 0x46, 0x64, -+ 0xee, 0x0b, 0x17, 0xbf, 0x42, 0x8e, 0xd6, 0x37, 0xf5, 0xf3, 0xe5, 0xa6, 0x7d, 0xbe, 0xdc, 0x7c, -+ 0x12, 0x25, 0x62, 0x8a, 0x9e, 0x40, 0xaf, 0xfc, 0x30, 0x87, 0x6e, 0xdb, 0xca, 0x66, 0xc1, 0x73, -+ 0xdd, 0xa5, 0x62, 0x9e, 0x42, 0x7f, 0xe6, 0x8d, 0xce, 0xda, 0xb3, 0xf8, 0xe9, 0xee, 0x52, 0x41, -+ 0x0f, 0xa1, 0x5d, 0x78, 0x94, 0x43, 0x43, 0x2d, 0x64, 0xfe, 0x9d, 0xee, 0x52, 0x01, 0xbb, 0xd0, -+ 0x2d, 0xbd, 0x93, 0xa1, 0x91, 0xf1, 0x67, 0xc1, 0xe3, 0xd9, 0xa5, 0x42, 0x76, 0xa0, 0x5d, 0x78, -+ 0xae, 0xb2, 0x56, 0xcc, 0xbf, 0x89, 0x8d, 0x6e, 0x2d, 0xa0, 0x98, 0xe9, 0xdc, 0x83, 0x6e, 0xe9, -+ 0x71, 0xc9, 0x1a, 0xb2, 0xe8, 0x61, 0x6b, 0x74, 0x7b, 0x21, 0xcd, 0x48, 0x7a, 0x0a, 0xfd, 0x99, -+ 0xa7, 0x26, 0x1b, 0xdc, 0xc5, 0x2f, 0x50, 0x97, 0xba, 0xf5, 0x8d, 0x9a, 0xec, 0x42, 0x13, 0x57, -+ 0x98, 0xec, 0xf9, 0x87, 0xa5, 0xd1, 0x9d, 0xc5, 0x44, 0x63, 0xd5, 0x13, 0xe8, 0x95, 0xdf, 0x94, -+ 0xac, 0xb0, 0x85, 0x2f, 0x4d, 0x57, 0xaf, 0x9c, 0xd2, 0xf3, 0x52, 0xbe, 0x72, 0x16, 0xbd, 0x3a, -+ 0x5d, 0x2a, 0xe8, 0x11, 0x80, 0x69, 0xd9, 0x42, 0x12, 0x67, 0x53, 0x36, 0xd7, 0x2a, 0x66, 0x53, -+ 0xb6, 0xa0, 0xbd, 0x7b, 0x08, 0xa0, 0x3b, 0xad, 0x90, 0xa6, 0x02, 0xdd, 0xb4, 0x66, 0xcc, 0xb4, -+ 0x77, 0xa3, 0xe1, 0x3c, 0x61, 0x4e, 0x00, 0x66, 0xec, 0x3a, 0x02, 0xbe, 0x06, 0xc8, 0x3b, 0x38, -+ 0x2b, 0x60, 0xae, 0xa7, 0xbb, 0x22, 0x06, 0x9d, 0x62, 0xbf, 0x86, 0x8c, 0xaf, 0x0b, 0x7a, 0xb8, -+ 0x2b, 0x44, 0xf4, 0x67, 0xaa, 0xe8, 0xf2, 0x62, 0x9b, 0x2d, 0xd3, 0x47, 0x73, 0xd5, 0x33, 0xfa, -+ 0x1e, 0x6e, 0x5f, 0x51, 0x88, 0xa3, 0x8d, 0x85, 0xe2, 0x16, 0xd4, 0xea, 0x0b, 0x44, 0x7f, 0x06, -+ 0x9d, 0x62, 0x35, 0x6e, 0x1d, 0x5c, 0x50, 0xa1, 0x8f, 0x4a, 0x15, 0x39, 0x7a, 0x08, 0xbd, 0x72, -+ 0xad, 0x8d, 0x0a, 0x5b, 0x6e, 0xae, 0x02, 0x1f, 0xad, 0xcc, 0xdc, 0xcc, 0x70, 0xf4, 0x31, 0x40, -+ 0x5e, 0x93, 0xdb, 0x99, 0x99, 0xab, 0xd2, 0x67, 0xb4, 0x7e, 0x0d, 0xbd, 0xc2, 0x91, 0x20, 0x9b, -+ 0xd8, 0x9b, 0x25, 0xe7, 0xf3, 0x83, 0x62, 0x64, 0x8a, 0xb7, 0xd2, 0x89, 0xf0, 0x08, 0x3a, 0xc5, -+ 0xe3, 0xc7, 0x7a, 0xbb, 0xe0, 0x48, 0xba, 0x2a, 0x9f, 0x16, 0x8e, 0x2a, 0xbb, 0x2d, 0xe6, 0x4f, -+ 0xaf, 0xab, 0xf2, 0x69, 0xa9, 0x81, 0xb6, 0x69, 0x6c, 0x51, 0x57, 0x7d, 0xd5, 0x29, 0x53, 0xee, -+ 0x36, 0x6d, 0xf4, 0x17, 0xf6, 0xa0, 0x57, 0x2d, 0xef, 0x62, 0x0b, 0x64, 0xe3, 0xb1, 0xa0, 0x2d, -+ 0xfa, 0x95, 0x74, 0x53, 0x6c, 0x73, 0x0a, 0xe9, 0x66, 0x41, 0xf7, 0x73, 0xa9, 0xa0, 0x3d, 0xe8, -+ 0x3f, 0xb5, 0x15, 0xac, 0xa9, 0xae, 0x8d, 0x39, 0x0b, 0xba, 0x89, 0xd1, 0x68, 0x11, 0xc9, 0xcc, -+ 0xf2, 0x37, 0x30, 0x98, 0xab, 0xac, 0xd1, 0xdd, 0xec, 0xa1, 0x60, 0x61, 0xc9, 0x7d, 0xa9, 0x59, -+ 0xfb, 0xb0, 0x32, 0x5b, 0x58, 0xa3, 0x37, 0xcc, 0xa4, 0x2f, 0x2e, 0xb8, 0x2f, 0x15, 0xf5, 0x05, -+ 0x34, 0x6d, 0x21, 0x87, 0xcc, 0x83, 0xcc, 0x4c, 0x61, 0x77, 0xd9, 0xd0, 0x9d, 0xce, 0xcf, 0xbf, -+ 0xdc, 0xad, 0xfc, 0xe6, 0x97, 0xbb, 0x95, 0xdf, 0xfd, 0x72, 0xb7, 0x72, 0xbc, 0xac, 0xa8, 0x1f, -+ 0xff, 0x21, 0x00, 0x00, 0xff, 0xff, 0x1b, 0x62, 0x55, 0x85, 0x98, 0x25, 0x00, 0x00, - } -diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go -index 3f22d9f..5e3ace9 100644 ---- a/virtcontainers/agent.go -+++ b/virtcontainers/agent.go -@@ -231,6 +231,9 @@ type agent interface { - // updateInterface will tell the agent to update a nic for an existed Sandbox. - updateInterface(inf *vcTypes.Interface) (*vcTypes.Interface, error) - -+ // updateInterfaceHwAddrByName will tell the agent to update a nic for an existed Sandbox by nic name. -+ updateInterfaceHwAddrByName(inf *vcTypes.Interface) (*vcTypes.Interface, error) -+ - // listInterfaces will tell the agent to list interfaces of an existed Sandbox - listInterfaces() ([]*vcTypes.Interface, error) - -diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go -index 60f1d19..3ca874e 100644 ---- a/virtcontainers/hypervisor.go -+++ b/virtcontainers/hypervisor.go -@@ -47,6 +47,9 @@ const ( - - // MockHypervisor is a mock hypervisor for testing purposes - MockHypervisor HypervisorType = "mock" -+ -+ // StratovirtHypervisor is stratovirt hypervisor -+ StratovirtHypervisor HypervisorType = "stratovirt" - ) - - const ( -@@ -171,6 +174,9 @@ func (hType *HypervisorType) Set(value string) error { - case "mock": - *hType = MockHypervisor - return nil -+ case "stratovirt": -+ *hType = StratovirtHypervisor -+ return nil - default: - return fmt.Errorf("Unknown hypervisor type %s", value) - } -@@ -189,6 +195,8 @@ func (hType *HypervisorType) String() string { - return string(ClhHypervisor) - case MockHypervisor: - return string(MockHypervisor) -+ case StratovirtHypervisor: -+ return string(StratovirtHypervisor) - default: - return "" - } -@@ -218,6 +226,10 @@ func newHypervisor(hType HypervisorType) (hypervisor, error) { - }, nil - case MockHypervisor: - return &mockHypervisor{}, nil -+ case StratovirtHypervisor: -+ return &stratovirt{ -+ store: store, -+ }, nil - default: - return nil, fmt.Errorf("Unknown hypervisor type %s", hType) - } -diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go -index 0f03c9d..f8cad39 100644 ---- a/virtcontainers/kata_agent.go -+++ b/virtcontainers/kata_agent.go -@@ -102,36 +102,37 @@ const ( - ) - - const ( -- grpcCheckRequest = "grpc.CheckRequest" -- grpcExecProcessRequest = "grpc.ExecProcessRequest" -- grpcCreateSandboxRequest = "grpc.CreateSandboxRequest" -- grpcDestroySandboxRequest = "grpc.DestroySandboxRequest" -- grpcCreateContainerRequest = "grpc.CreateContainerRequest" -- grpcStartContainerRequest = "grpc.StartContainerRequest" -- grpcRemoveContainerRequest = "grpc.RemoveContainerRequest" -- grpcSignalProcessRequest = "grpc.SignalProcessRequest" -- grpcUpdateRoutesRequest = "grpc.UpdateRoutesRequest" -- grpcUpdateInterfaceRequest = "grpc.UpdateInterfaceRequest" -- grpcListInterfacesRequest = "grpc.ListInterfacesRequest" -- grpcListRoutesRequest = "grpc.ListRoutesRequest" -- grpcOnlineCPUMemRequest = "grpc.OnlineCPUMemRequest" -- grpcListProcessesRequest = "grpc.ListProcessesRequest" -- grpcUpdateContainerRequest = "grpc.UpdateContainerRequest" -- grpcWaitProcessRequest = "grpc.WaitProcessRequest" -- grpcTtyWinResizeRequest = "grpc.TtyWinResizeRequest" -- grpcWriteStreamRequest = "grpc.WriteStreamRequest" -- grpcCloseStdinRequest = "grpc.CloseStdinRequest" -- grpcStatsContainerRequest = "grpc.StatsContainerRequest" -- grpcPauseContainerRequest = "grpc.PauseContainerRequest" -- grpcResumeContainerRequest = "grpc.ResumeContainerRequest" -- grpcReseedRandomDevRequest = "grpc.ReseedRandomDevRequest" -- grpcGuestDetailsRequest = "grpc.GuestDetailsRequest" -- grpcMemHotplugByProbeRequest = "grpc.MemHotplugByProbeRequest" -- grpcCopyFileRequest = "grpc.CopyFileRequest" -- grpcSetGuestDateTimeRequest = "grpc.SetGuestDateTimeRequest" -- grpcStartTracingRequest = "grpc.StartTracingRequest" -- grpcStopTracingRequest = "grpc.StopTracingRequest" -- grpcUpdateIPVSRequest = "grpc.UpdateIPVSRequest" -+ grpcCheckRequest = "grpc.CheckRequest" -+ grpcExecProcessRequest = "grpc.ExecProcessRequest" -+ grpcCreateSandboxRequest = "grpc.CreateSandboxRequest" -+ grpcDestroySandboxRequest = "grpc.DestroySandboxRequest" -+ grpcCreateContainerRequest = "grpc.CreateContainerRequest" -+ grpcStartContainerRequest = "grpc.StartContainerRequest" -+ grpcRemoveContainerRequest = "grpc.RemoveContainerRequest" -+ grpcSignalProcessRequest = "grpc.SignalProcessRequest" -+ grpcUpdateRoutesRequest = "grpc.UpdateRoutesRequest" -+ grpcUpdateInterfaceRequest = "grpc.UpdateInterfaceRequest" -+ grpcUpdateInterfaceHwAddrByNameRequest = "grpc.UpdateInterfaceHwAddrByNameRequest" -+ grpcListInterfacesRequest = "grpc.ListInterfacesRequest" -+ grpcListRoutesRequest = "grpc.ListRoutesRequest" -+ grpcOnlineCPUMemRequest = "grpc.OnlineCPUMemRequest" -+ grpcListProcessesRequest = "grpc.ListProcessesRequest" -+ grpcUpdateContainerRequest = "grpc.UpdateContainerRequest" -+ grpcWaitProcessRequest = "grpc.WaitProcessRequest" -+ grpcTtyWinResizeRequest = "grpc.TtyWinResizeRequest" -+ grpcWriteStreamRequest = "grpc.WriteStreamRequest" -+ grpcCloseStdinRequest = "grpc.CloseStdinRequest" -+ grpcStatsContainerRequest = "grpc.StatsContainerRequest" -+ grpcPauseContainerRequest = "grpc.PauseContainerRequest" -+ grpcResumeContainerRequest = "grpc.ResumeContainerRequest" -+ grpcReseedRandomDevRequest = "grpc.ReseedRandomDevRequest" -+ grpcGuestDetailsRequest = "grpc.GuestDetailsRequest" -+ grpcMemHotplugByProbeRequest = "grpc.MemHotplugByProbeRequest" -+ grpcCopyFileRequest = "grpc.CopyFileRequest" -+ grpcSetGuestDateTimeRequest = "grpc.SetGuestDateTimeRequest" -+ grpcStartTracingRequest = "grpc.StartTracingRequest" -+ grpcStopTracingRequest = "grpc.StopTracingRequest" -+ grpcUpdateIPVSRequest = "grpc.UpdateIPVSRequest" - ) - - // The function is declared this way for mocking in unit tests -@@ -628,6 +629,15 @@ func (k *kataAgent) updateInterface(ifc *vcTypes.Interface) (*vcTypes.Interface, - return nil, err - } - -+func (k *kataAgent) updateInterfaceHwAddrByName(ifc *vcTypes.Interface) (*vcTypes.Interface, error) { -+ ifcReq := &grpc.UpdateInterfaceHwAddrByNameRequest{ -+ Interface: k.convertToKataAgentInterface(ifc), -+ } -+ _, err := k.sendReq(ifcReq) -+ -+ return nil, err -+} -+ - func (k *kataAgent) updateInterfaces(interfaces []*vcTypes.Interface) error { - for _, ifc := range interfaces { - if _, err := k.updateInterface(ifc); err != nil { -@@ -2074,6 +2084,9 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) { - k.reqHandlers[grpcUpdateInterfaceRequest] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) { - return k.client.UpdateInterface(ctx, req.(*grpc.UpdateInterfaceRequest), opts...) - } -+ k.reqHandlers[grpcUpdateInterfaceHwAddrByNameRequest] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) { -+ return k.client.UpdateInterfaceHwAddrByName(ctx, req.(*grpc.UpdateInterfaceHwAddrByNameRequest), opts...) -+ } - k.reqHandlers[grpcListInterfacesRequest] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) { - return k.client.ListInterfaces(ctx, req.(*grpc.ListInterfacesRequest), opts...) - } -diff --git a/virtcontainers/noop_agent.go b/virtcontainers/noop_agent.go -index d174623..bac34be 100644 ---- a/virtcontainers/noop_agent.go -+++ b/virtcontainers/noop_agent.go -@@ -111,6 +111,11 @@ func (n *noopAgent) updateInterface(inf *vcTypes.Interface) (*vcTypes.Interface, - return nil, nil - } - -+// updateInterfaceHwAddrByName is the Noop agent Interface update implementation. It does nothing. -+func (n *noopAgent) updateInterfaceHwAddrByName(inf *vcTypes.Interface) (*vcTypes.Interface, error) { -+ return nil, nil -+} -+ - // listInterfaces is the Noop agent Interfaces list implementation. It does nothing. - func (n *noopAgent) listInterfaces() ([]*vcTypes.Interface, error) { - return nil, nil -diff --git a/virtcontainers/stratovirt.go b/virtcontainers/stratovirt.go -new file mode 100644 -index 0000000..7c156d5 ---- /dev/null -+++ b/virtcontainers/stratovirt.go -@@ -0,0 +1,617 @@ -+package virtcontainers -+ -+import ( -+ "context" -+ "fmt" -+ "os" -+ "os/exec" -+ "path/filepath" -+ "strconv" -+ "strings" -+ "syscall" -+ "time" -+ -+ govmmQemu "github.com/intel/govmm/qemu" -+ "github.com/opentracing/opentracing-go" -+ "github.com/pkg/errors" -+ "github.com/sirupsen/logrus" -+ -+ "github.com/kata-containers/runtime/virtcontainers/device/config" -+ persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" -+ vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" -+ "github.com/kata-containers/runtime/virtcontainers/store" -+ "github.com/kata-containers/runtime/virtcontainers/types" -+) -+ -+const defaultDummyMac = "22:33:44:aa:bb:" -+const mmioBlkCount = 6 -+const mmioNetCount = 2 -+ -+type stratovirtDev struct { -+ dev interface{} -+ devType deviceType -+} -+ -+type stratovirt struct { -+ id string -+ ctx context.Context -+ sandbox *Sandbox -+ store persistapi.PersistDriver -+ config HypervisorConfig -+ pid int -+ socketPath string -+ consolePath string -+ qmpMonitorCh qmpChannel -+ devices []stratovirtDev -+ HotpluggedVCPUs []CPUDevice -+ mmioBlkSlots [mmioBlkCount]bool -+ mmioNetSlots [mmioNetCount]bool -+} -+ -+func (s *stratovirt) Logger() *logrus.Entry { -+ return virtLog.WithField("subsystem", "stratovirt") -+} -+ -+func (s *stratovirt) trace(name string) (opentracing.Span, context.Context) { -+ span, ctx := opentracing.StartSpanFromContext(s.ctx, name) -+ -+ span.SetTag("subsystem", "hypervisor") -+ span.SetTag("type", "stratovirt") -+ -+ return span, ctx -+} -+ -+func (s *stratovirt) getKernelCmdLine() string { -+ var params []string -+ -+ params = append(params, "pci=off") -+ params = append(params, "reboot=k") -+ params = append(params, "panic=1") -+ params = append(params, "iommu=off") -+ params = append(params, "acpi=off") -+ params = append(params, "quiet") -+ params = append(params, "agent.use_vsock="+strconv.FormatBool(s.config.UseVSock)) -+ params = append(params, SerializeParams(s.config.KernelParams, "=")...) -+ -+ return strings.Join(params, " ") -+} -+ -+func (s *stratovirt) hypervisorConfig() HypervisorConfig { -+ return s.config -+} -+ -+func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, stateful bool) error { -+ s.ctx = ctx -+ -+ span, _ := s.trace("createSandbox") -+ defer span.Finish() -+ -+ sandbox, err := globalSandboxList.lookupSandbox(id) -+ if err != nil { -+ s.Logger().Error("Get sandbox failed") -+ return err -+ } -+ -+ s.id = id -+ s.sandbox = sandbox -+ s.config = *hypervisorConfig -+ s.socketPath = filepath.Join(s.store.RunVMStoragePath(), id, "qmp.sock") -+ s.consolePath = filepath.Join(s.store.RunVMStoragePath(), id, "console.sock") -+ s.qmpMonitorCh = qmpChannel{ -+ ctx: s.ctx, -+ path: s.socketPath, -+ } -+ -+ return err -+} -+ -+func (s *stratovirt) waitSandBoxStarted(timeout int) error { -+ timeStart := time.Now() -+ for { -+ err := s.qmpSetup() -+ if err == nil { -+ break -+ } -+ -+ if int(time.Since(timeStart).Seconds()) > timeout { -+ return fmt.Errorf("Failed to connect to StratoVirt instance (timeout %ds): %v", timeout, err) -+ } -+ -+ time.Sleep(time.Duration(50) * time.Millisecond) -+ } -+ -+ if err := s.qmpMonitorCh.qmp.ExecuteQMPCapabilities(s.qmpMonitorCh.ctx); err != nil { -+ s.Logger().WithError(err).Error(qmpCapErrMsg) -+ return err -+ } -+ -+ return nil -+} -+ -+func (s *stratovirt) startSandbox(timeout int) error { -+ span, _ := s.trace("startSandbox") -+ defer span.Finish() -+ -+ var params []string -+ params = append(params, "-name "+fmt.Sprintf("sandbox-%s", s.id)) -+ params = append(params, "-api-channel unix:"+s.socketPath) -+ -+ if kernelPath, err := s.config.KernelAssetPath(); err == nil { -+ params = append(params, "-kernel "+kernelPath) -+ } -+ -+ if initrdPath, err := s.config.InitrdAssetPath(); err == nil { -+ params = append(params, "-initrd "+initrdPath) -+ } -+ -+ params = append(params, "-append "+s.getKernelCmdLine()) -+ params = append(params, fmt.Sprintf("-smp %d", s.config.NumVCPUs)) -+ params = append(params, fmt.Sprintf("-m %d", s.config.MemorySize*1024*1024)) -+ params = append(params, fmt.Sprintf("-chardev id=charconsole0,path=%s", s.consolePath)) -+ -+ // add devices to cmdline -+ for _, d := range s.devices { -+ switch v := d.dev.(type) { -+ case Endpoint: -+ name := v.Name() -+ mac := v.HardwareAddr() -+ tapName := v.NetworkPair().TapInterface.TAPIface.Name -+ params = append(params, fmt.Sprintf("-net id=%s,mac=%s,host_dev_name=%s", name, mac, tapName)) -+ case config.BlockDrive: -+ id := v.ID -+ path := v.File -+ params = append(params, fmt.Sprintf("-drive id=%s,file=%s", id, path)) -+ case types.VSock: -+ v.VhostFd.Close() -+ params = append(params, fmt.Sprintf("-device vsock,id=vsock-id,guest-cid=%d", v.ContextID)) -+ default: -+ s.Logger().Error("Adding device type is unsupported") -+ } -+ } -+ -+ // daemonize -+ params = append(params, "-daemonize") -+ -+ // append logfile only on debug -+ if s.config.Debug { -+ dir := filepath.Join(store.RunVMStoragePath(), s.id) -+ params = append(params, fmt.Sprintf("-D %s/stratovirt.log", dir)) -+ } -+ -+ s.Logger().Info("StratoVirt start with params: ", strings.Join(params, " ")) -+ -+ dir := filepath.Join(store.RunVMStoragePath(), s.id) -+ err := os.MkdirAll(dir, store.DirMode) -+ if err != nil { -+ return err -+ } -+ defer func() { -+ if err != nil { -+ if err := os.RemoveAll(dir); err != nil { -+ s.Logger().WithError(err).Error("Fail to clean up vm dir %s", dir) -+ } -+ } -+ }() -+ -+ binPath, err := s.config.HypervisorAssetPath() -+ if err != nil { -+ s.Logger().WithField("Fail to get hypervisor bin path", err).Error() -+ return err -+ } -+ -+ cmd := exec.CommandContext(s.ctx, binPath, params...) -+ if err := cmd.Run(); err != nil { -+ s.Logger().WithField("Error starting hypervisor, please check the params", err).Error() -+ return err -+ } -+ -+ if err = s.waitSandBoxStarted(timeout); err != nil { -+ return err -+ } -+ -+ return nil -+} -+ -+func (s *stratovirt) stopSandbox(force bool) error { -+ span, _ := s.trace("stopSandbox") -+ defer span.Finish() -+ -+ defer func() { -+ dir := filepath.Join(store.RunVMStoragePath(), s.id) -+ link, _ := filepath.EvalSymlinks(dir) -+ -+ if err := os.RemoveAll(dir); err != nil { -+ s.Logger().WithError(err).Warnf("Failed to clean up vm dir %s", dir) -+ } -+ -+ if link != dir && link != "" { -+ if err := os.RemoveAll(link); err != nil { -+ s.Logger().WithError(err).WithField("link", link).Warn("Failed to remove vm path link %s", link) -+ } -+ } -+ }() -+ -+ if !force { -+ err := s.qmpSetup() -+ if err != nil { -+ return err -+ } -+ -+ err = s.qmpMonitorCh.qmp.ExecuteQuit(s.qmpMonitorCh.ctx) -+ if err != nil { -+ s.Logger().WithError(err).Error("Fail to execute qmp: QUIT") -+ return err -+ } -+ } else { -+ if s.pid > 0 { -+ syscall.Kill(s.pid, syscall.SIGKILL) -+ } -+ } -+ return nil -+} -+ -+func (s *stratovirt) pauseSandbox() error { -+ return nil -+} -+ -+func (s *stratovirt) saveSandbox() error { -+ return nil -+} -+ -+func (s *stratovirt) resumeSandbox() error { -+ return nil -+} -+ -+func (s *stratovirt) addDevice(devInfo interface{}, devType deviceType) error { -+ dev := stratovirtDev{ -+ dev: devInfo, -+ devType: devType, -+ } -+ s.devices = append(s.devices, dev) -+ -+ return nil -+} -+ -+func (s *stratovirt) updateVMInterfaceHwAddr(Name string, HwAddr string) error { -+ inf := &vcTypes.Interface{ -+ Name: Name, -+ HwAddr: HwAddr, -+ IPAddresses: []*vcTypes.IPAddress{}, -+ PciAddr: "", -+ } -+ -+ if _, err := s.sandbox.agent.updateInterfaceHwAddrByName(inf); err != nil { -+ s.Logger().WithError(err).Error("Agent updateInterfaceHwAddrByName failed") -+ return err -+ } -+ -+ return nil -+} -+ -+func (s *stratovirt) getDevSlot(Name string, isPut bool) (slot int, err error) { -+ Name = filepath.Base(strings.ToLower(Name)) -+ -+ if strings.HasPrefix(Name, "eth") { -+ idxStr := strings.TrimPrefix(Name, "eth") -+ if idxStr == Name { -+ return 0, fmt.Errorf("Could not parse idx from Name %q", Name) -+ } -+ -+ idx, err := strconv.Atoi(idxStr) -+ if err != nil { -+ return 0, fmt.Errorf("Could not convert to int from Str %q", idxStr) -+ } -+ -+ if !isPut && s.mmioNetSlots[idx] { -+ return 0, fmt.Errorf("GetDevSlot failed, slot is being used %q", idxStr) -+ } -+ s.mmioNetSlots[idx] = !isPut -+ -+ return idx, nil -+ } else if strings.HasPrefix(Name, "vd") { -+ charStr := strings.TrimPrefix(Name, "vd") -+ if charStr == Name { -+ return 0, fmt.Errorf("Could not parse idx from Name %q", Name) -+ } -+ -+ char := []rune(charStr) -+ idx := int(char[0] - 'a') -+ -+ if !isPut && s.mmioBlkSlots[idx] { -+ return 0, fmt.Errorf("GetDevSlot failed, slot is being used %q", charStr) -+ } -+ s.mmioBlkSlots[idx] = !isPut -+ -+ return idx, nil -+ } -+ -+ return 0, fmt.Errorf("GetDevSlot failed, Name is invalid %q", Name) -+} -+ -+func (s *stratovirt) hotplugNet(endpoint Endpoint, op operation) (err error) { -+ err = s.qmpSetup() -+ if err != nil { -+ return err -+ } -+ var tap TapInterface -+ -+ switch endpoint.Type() { -+ case VethEndpointType: -+ drive := endpoint.(*VethEndpoint) -+ tap = drive.NetPair.TapInterface -+ case TapEndpointType: -+ drive := endpoint.(*TapEndpoint) -+ tap = drive.TapInterface -+ default: -+ return fmt.Errorf("Endpoint is not supported") -+ } -+ -+ switch op { -+ case addDevice: -+ var ( -+ VMFdNames []string -+ VhostFdNames []string -+ ) -+ for i, VMFd := range tap.VMFds { -+ fdName := fmt.Sprintf("fd%d", i) -+ if err := s.qmpMonitorCh.qmp.ExecuteGetFD(s.qmpMonitorCh.ctx, fdName, VMFd); err != nil { -+ return err -+ } -+ VMFdNames = append(VMFdNames, fdName) -+ } -+ for i, VhostFd := range tap.VhostFds { -+ fdName := fmt.Sprintf("vhostfd%d", i) -+ if err := s.qmpMonitorCh.qmp.ExecuteGetFD(s.qmpMonitorCh.ctx, fdName, VhostFd); err != nil { -+ return err -+ } -+ VhostFd.Close() -+ VhostFdNames = append(VhostFdNames, fdName) -+ } -+ -+ slot, err := s.getDevSlot(endpoint.Name(), false) -+ if err != nil { -+ return fmt.Errorf("Could not get unused slot for %q", endpoint.Name()) -+ } -+ -+ if len(VMFdNames) != 0 || len(VhostFdNames) != 0 { -+ if err := s.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(s.qmpMonitorCh.ctx, "tap", tap.ID, VMFdNames, VhostFdNames); err != nil { -+ s.getDevSlot(endpoint.Name(), true) -+ return err -+ } -+ } else { -+ if err := s.qmpMonitorCh.qmp.ExecuteNetdevAdd(s.qmpMonitorCh.ctx, "tap", tap.ID, tap.TAPIface.Name, "no", "no", 0); err != nil { -+ s.getDevSlot(endpoint.Name(), true) -+ return err -+ } -+ } -+ if err := s.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(s.qmpMonitorCh.ctx, tap.Name, tap.ID, endpoint.HardwareAddr(), fmt.Sprintf("%d", slot), "", "", 0, false); err != nil { -+ s.getDevSlot(endpoint.Name(), true) -+ return err -+ } -+ -+ // update vm interface -+ if err := s.updateVMInterfaceHwAddr(endpoint.Name(), endpoint.HardwareAddr()); err != nil { -+ s.getDevSlot(endpoint.Name(), true) -+ return err -+ } -+ case removeDevice: -+ if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, tap.ID); err != nil { -+ return err -+ } -+ if err := s.qmpMonitorCh.qmp.ExecuteNetdevDel(s.qmpMonitorCh.ctx, tap.ID); err != nil { -+ return err -+ } -+ -+ slot, err := s.getDevSlot(endpoint.Name(), true) -+ if err != nil { -+ slot = 0 -+ } -+ -+ // clear vm interface -+ if err := s.updateVMInterfaceHwAddr(endpoint.Name(), defaultDummyMac+fmt.Sprintf("%02x", slot)); err != nil { -+ s.getDevSlot(endpoint.Name(), false) -+ return err -+ } -+ default: -+ return fmt.Errorf("Operation is not supported") -+ } -+ -+ return nil -+} -+ -+func (s *stratovirt) hotplugBlk(drive *config.BlockDrive, op operation) (err error) { -+ err = s.qmpSetup() -+ if err != nil { -+ return err -+ } -+ -+ switch op { -+ case addDevice: -+ driver := "virtio-blk-pci" -+ slot, err := s.getDevSlot(drive.VirtPath, false) -+ if err != nil { -+ return fmt.Errorf("Could not get unused slot for %q", drive.VirtPath) -+ } -+ -+ if err := s.qmpMonitorCh.qmp.ExecuteBlockdevAdd(s.qmpMonitorCh.ctx, drive.File, drive.ID); err != nil { -+ s.getDevSlot(drive.VirtPath, true) -+ return err -+ } -+ -+ if err := s.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(s.qmpMonitorCh.ctx, drive.ID, drive.ID, driver, fmt.Sprintf("%d", slot), "", "", 0, true, false); err != nil { -+ s.getDevSlot(drive.VirtPath, true) -+ return err -+ } -+ case removeDevice: -+ if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, drive.ID); err != nil { -+ return err -+ } -+ if err := s.qmpMonitorCh.qmp.ExecuteBlockdevDel(s.qmpMonitorCh.ctx, drive.ID); err != nil { -+ return err -+ } -+ -+ s.getDevSlot(drive.VirtPath, true) -+ default: -+ return fmt.Errorf("Operation is not supported") -+ } -+ -+ return nil -+} -+ -+func (s *stratovirt) hotplugAddDevice(devInfo interface{}, devType deviceType) (interface{}, error) { -+ span, _ := s.trace("hotplugAddDevice") -+ defer span.Finish() -+ -+ switch devType { -+ case netDev: -+ return nil, s.hotplugNet(devInfo.(Endpoint), addDevice) -+ case blockDev: -+ return nil, s.hotplugBlk(devInfo.(*config.BlockDrive), addDevice) -+ default: -+ return nil, fmt.Errorf("Hotplug add device failed: unsupported device type '%v'", devType) -+ } -+} -+ -+func (s *stratovirt) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) { -+ span, _ := s.trace("hotplugRemoveDevice") -+ defer span.Finish() -+ -+ switch devType { -+ case netDev: -+ return nil, s.hotplugNet(devInfo.(Endpoint), removeDevice) -+ case blockDev: -+ return nil, s.hotplugBlk(devInfo.(*config.BlockDrive), removeDevice) -+ default: -+ return nil, fmt.Errorf("Hotplug remove device: unsupported device type '%v'", devType) -+ } -+} -+ -+func (s *stratovirt) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error) { -+ return 0, memoryDevice{}, nil -+} -+ -+func (s *stratovirt) resizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint32, err error) { -+ return 0, 0, nil -+} -+ -+func (s *stratovirt) getSandboxConsole(id string) (string, error) { -+ return s.consolePath, nil -+} -+ -+func (s *stratovirt) getMemorySize() uint32 { -+ return 0 -+} -+ -+func (s *stratovirt) disconnect() { -+ span, _ := s.trace("disconnect") -+ defer span.Finish() -+ -+ s.qmpTeardown() -+} -+ -+func (s *stratovirt) capabilities() types.Capabilities { -+ span, _ := s.trace("capabilities") -+ defer span.Finish() -+ -+ var caps types.Capabilities -+ caps.SetBlockDeviceHotplugSupport() -+ -+ return caps -+} -+ -+func (s *stratovirt) qmpTeardown() { -+ if s.qmpMonitorCh.qmp != nil { -+ s.qmpMonitorCh.qmp.Shutdown() -+ <-s.qmpMonitorCh.disconn -+ s.qmpMonitorCh.qmp = nil -+ s.qmpMonitorCh.disconn = nil -+ } -+} -+ -+func (s *stratovirt) qmpSetup() error { -+ s.qmpTeardown() -+ -+ cfg := govmmQemu.QMPConfig{Logger: newQMPLogger()} -+ -+ // Auto-closed by QMPStart(). -+ disconnectCh := make(chan struct{}) -+ -+ qmp, _, err := govmmQemu.QMPStart(s.qmpMonitorCh.ctx, s.qmpMonitorCh.path, cfg, disconnectCh) -+ if err != nil { -+ s.Logger().WithError(err).Error("Failed to connect to StratoVirt instance") -+ return err -+ } -+ -+ s.qmpMonitorCh.qmp = qmp -+ s.qmpMonitorCh.disconn = disconnectCh -+ -+ return nil -+} -+ -+func (s *stratovirt) getThreadIDs() (vcpuThreadIDs, error) { -+ span, _ := s.trace("getThreadIDs") -+ defer span.Finish() -+ -+ tid := vcpuThreadIDs{} -+ if err := s.qmpSetup(); err != nil { -+ return tid, err -+ } -+ -+ cpuInfos, err := s.qmpMonitorCh.qmp.ExecQueryCpus(s.qmpMonitorCh.ctx) -+ if err != nil { -+ s.Logger().WithError(err).Error("Failed to query cpu infos") -+ return tid, err -+ } -+ -+ tid.vcpus = make(map[int]int, len(cpuInfos)) -+ for _, i := range cpuInfos { -+ if i.ThreadID > 0 { -+ tid.vcpus[i.CPU] = i.ThreadID -+ } -+ } -+ return tid, nil -+} -+ -+func (s *stratovirt) cleanup() error { -+ span, _ := s.trace("cleanup") -+ defer span.Finish() -+ -+ s.qmpTeardown() -+ -+ return nil -+} -+ -+func (s *stratovirt) getPids() []int { -+ return []int{s.pid} -+} -+ -+func (s *stratovirt) fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, j []byte) error { -+ return nil -+} -+ -+func (s *stratovirt) toGrpc() ([]byte, error) { -+ return nil, nil -+} -+ -+func (s *stratovirt) check() error { -+ if err := syscall.Kill(s.pid, syscall.Signal(0)); err != nil { -+ return errors.Wrapf(err, "Failed to ping StratoVirt process") -+ } -+ -+ return nil -+} -+ -+func (s *stratovirt) generateSocket(id string, useVsock bool) (interface{}, error) { -+ return generateVMSocket(id, useVsock, s.store.RunVMStoragePath()) -+} -+ -+func (s *stratovirt) save() (p persistapi.HypervisorState) { -+ p.Pid = s.pid -+ p.Type = string(StratovirtHypervisor) -+ return -+} -+ -+func (s *stratovirt) load(p persistapi.HypervisorState) { -+ s.pid = p.Pid -+ return -+} --- -1.8.3.1 - diff --git a/runtime/patches/0053-kata-runtime-add-interface-for-host-cgroup.patch b/runtime/patches/0053-kata-runtime-add-interface-for-host-cgroup.patch deleted file mode 100644 index 419d5aa..0000000 --- a/runtime/patches/0053-kata-runtime-add-interface-for-host-cgroup.patch +++ /dev/null @@ -1,199 +0,0 @@ -From 31bbb64a0682e326b354ac54c6596402fc20a977 Mon Sep 17 00:00:00 2001 -From: holyfei -Date: Tue, 8 Sep 2020 20:03:41 +0800 -Subject: [PATCH 1/5] kata-runtime: add interface for host cgroup - -reason: add interface for host cgroup, including -CreateSandboxCgroup, DestroySandboxCgroup and -AddPidToSandboxCgroup, GetSandboxCgroupPath - -Signed-off-by: yangfeiyu ---- - virtcontainers/cgroups.go | 111 +++++++++++++++++++++++++++++++++++++++ - virtcontainers/implementation.go | 20 +++++++ - virtcontainers/interfaces.go | 5 ++ - 3 files changed, 136 insertions(+) - -diff --git a/virtcontainers/cgroups.go b/virtcontainers/cgroups.go -index 4459df5..df0ec30 100644 ---- a/virtcontainers/cgroups.go -+++ b/virtcontainers/cgroups.go -@@ -8,13 +8,20 @@ package virtcontainers - - import ( - "bufio" -+ "context" -+ "encoding/json" - "fmt" -+ "io/ioutil" - "os" - "path/filepath" - "strings" - - "github.com/containerd/cgroups" - specs "github.com/opencontainers/runtime-spec/specs-go" -+ "github.com/sirupsen/logrus" -+ -+ "github.com/kata-containers/runtime/virtcontainers/store" -+ "github.com/kata-containers/runtime/virtcontainers/types" - ) - - type cgroupPather interface { -@@ -30,6 +37,110 @@ const cgroupKataPath = "/kata/" - var cgroupsLoadFunc = cgroups.Load - var cgroupsNewFunc = cgroups.New - -+// CreateSandboxCgroup create cgroup based on the first container's cgroupPath of sandbox -+func CreateSandboxCgroup(ctx context.Context, path string) error { -+ if path == "" { -+ return fmt.Errorf("sandbox cgroupPath shouldn't be empty!") -+ } -+ -+ var cgroupPath string -+ vcpuCgroupPath := filepath.Join(path, "vcpu") -+ if filepath.IsAbs(vcpuCgroupPath) { -+ cgroupPath = filepath.Clean(vcpuCgroupPath) -+ } else { -+ cgroupPath = filepath.Join(filepath.Clean("/" + vcpuCgroupPath)) -+ } -+ -+ resources := specs.LinuxResources{} -+ if _, err := cgroupsNewFunc(cgroups.V1, cgroups.StaticPath(cgroupPath), &resources); err != nil { -+ return fmt.Errorf("Could not create cgroup for %v: %v", cgroupPath, err) -+ } -+ -+ return nil -+} -+ -+// DestroySandboxCgroup destroy the cgroup dir created for sandbox -+func DestroySandboxCgroup(ctx context.Context, cgroupPath string) error { -+ return deleteCgroup(cgroups.V1, cgroupPath) -+} -+ -+func deleteCgroup(hierarchy cgroups.Hierarchy, cgroupPath string) error { -+ if cgroupPath == "" { -+ logrus.Warn("delete cgroupPath shouldn't be empty!") -+ return nil -+ } -+ -+ cgroup, err := cgroupsLoadFunc(hierarchy, -+ cgroups.StaticPath(cgroupPath)) -+ -+ if err == cgroups.ErrCgroupDeleted { -+ // cgroup already deleted -+ return nil -+ } -+ -+ if err != nil { -+ return fmt.Errorf("Could not load cgroup %v: %v", cgroupPath, err) -+ } -+ -+ // move running process here, that way cgroup can be removed -+ parent, err := parentCgroup(hierarchy, cgroupPath) -+ if err != nil { -+ // parent cgroup doesn't exist, that means there are no process running -+ // and the container cgroup was removed. -+ logrus.Warn(err) -+ return nil -+ } -+ -+ if err := cgroup.MoveTo(parent); err != nil { -+ // Don't fail, cgroup can be deleted -+ logrus.Warnf("Could not move container process into parent cgroup: %v", err) -+ } -+ -+ if err := cgroup.Delete(); err != nil { -+ return fmt.Errorf("Could not delete cgroup %v: %v", cgroupPath, err) -+ } -+ -+ return nil -+} -+ -+// GetSandboxCgroupPath return the cgroup path of specified sandbox -+func GetSandboxCgroupPath(ctx context.Context, sandboxID string) (string, error) { -+ stateFilePath := filepath.Join(store.RunStoragePath(), sandboxID, store.StateFile) -+ -+ fileData, err := ioutil.ReadFile(stateFilePath) -+ if err != nil { -+ return "", err -+ } -+ -+ state := types.SandboxState{} -+ -+ if err := json.Unmarshal(fileData, &state); err != nil { -+ return "", err -+ } -+ -+ if state.CgroupPath == "" { -+ return "", fmt.Errorf("get sandbox cgroup path error: cgroupPath is empty") -+ } -+ -+ return state.CgroupPath, nil -+} -+ -+// AddPidToSandboxCgroup add kata-runtime create process to cgroup -+// the pid will be added to the cgroup of "/vcpu" -+func AddPidToSandboxCgroup(ctx context.Context, pid int, cgroupPath string) error { -+ cgroup, err := cgroupsLoadFunc(cgroups.V1, cgroups.StaticPath(cgroupPath)) -+ if err != nil { -+ return err -+ } -+ -+ err = cgroup.Add(cgroups.Process{Pid: pid}) -+ if err != nil { -+ return err -+ } -+ -+ return nil -+} -+ - // V1Constraints returns the cgroups that are compatible with the VC architecture - // and hypervisor, constraints can be applied to these cgroups. - func V1Constraints() ([]cgroups.Subsystem, error) { -diff --git a/virtcontainers/implementation.go b/virtcontainers/implementation.go -index e4bc4ae..fedc51f 100644 ---- a/virtcontainers/implementation.go -+++ b/virtcontainers/implementation.go -@@ -188,3 +188,23 @@ func (impl *VCImpl) CleanupContainer(ctx context.Context, sandboxID, containerID - func (impl *VCImpl) UpdateIPVSRule(ctx context.Context, sandboxID string, IPVSRule *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) { - return UpdateIPVSRule(ctx, sandboxID, IPVSRule) - } -+ -+// CreateSandboxCgroup implements the VC function of the same name. -+func (impl *VCImpl) CreateSandboxCgroup(ctx context.Context, sandboxCgroupPath string) error { -+ return CreateSandboxCgroup(ctx, sandboxCgroupPath) -+} -+ -+// DestroySandboxCgroup implements the VC function of the same name. -+func (impl *VCImpl) DestroySandboxCgroup(ctx context.Context, sandboxCgroupPath string) error { -+ return DestroySandboxCgroup(ctx, sandboxCgroupPath) -+} -+ -+// AddPidToSandboxCgroup implements the VC function of the same name. -+func (impl *VCImpl) AddPidToSandboxCgroup(ctx context.Context, pid int, sandboxCgroupPath string) error { -+ return AddPidToSandboxCgroup(ctx, pid, sandboxCgroupPath) -+} -+ -+// GetSandboxCgroupPath implements the VC function of the same name. -+func (impl *VCImpl) GetSandboxCgroupPath(ctx context.Context, sandboxID string) (string, error) { -+ return GetSandboxCgroupPath(ctx, sandboxID) -+} -diff --git a/virtcontainers/interfaces.go b/virtcontainers/interfaces.go -index 0fd12d8..4d166e0 100644 ---- a/virtcontainers/interfaces.go -+++ b/virtcontainers/interfaces.go -@@ -24,6 +24,11 @@ type VC interface { - SetLogger(ctx context.Context, logger *logrus.Entry) - SetFactory(ctx context.Context, factory Factory) - -+ CreateSandboxCgroup(ctx context.Context, sandboxCgroupPath string) error -+ DestroySandboxCgroup(ctx context.Context, sandboxCgroupPath string) error -+ AddPidToSandboxCgroup(ctx context.Context, pid int, sandboxCgroupPath string) error -+ GetSandboxCgroupPath(ctx context.Context, sandboxID string) (string, error) -+ - CreateSandbox(ctx context.Context, sandboxConfig SandboxConfig) (VCSandbox, error) - DeleteSandbox(ctx context.Context, sandboxID string) (VCSandbox, error) - FetchSandbox(ctx context.Context, sandboxID string) (VCSandbox, error) --- -1.8.3.1 - diff --git a/runtime/patches/0054-kata-runtime-add-sandbox-cgroup-with-vcpu-and-emulat.patch b/runtime/patches/0054-kata-runtime-add-sandbox-cgroup-with-vcpu-and-emulat.patch deleted file mode 100644 index 1c28525..0000000 --- a/runtime/patches/0054-kata-runtime-add-sandbox-cgroup-with-vcpu-and-emulat.patch +++ /dev/null @@ -1,376 +0,0 @@ -From 98a3c4677261e1c0364015f36928cddfb0af253e Mon Sep 17 00:00:00 2001 -From: holyfei -Date: Wed, 9 Sep 2020 16:45:24 +0800 -Subject: [PATCH 2/5] kata-runtime: add sandbox cgroup with vcpu and emulator - switch - -reason: add sandbox cgroup with vcpu and emulator switch, if -sandbox_cgroup_with_emulator is true, it will overload the feature -of sandbox_cgroup_only, there will be two cgroups, vcpu and emulator - -Signed-off-by: yangfeiyu ---- - cli/config/configuration-qemu.toml.in | 12 +++++++++ - cli/kata-env.go | 38 ++++++++++++++------------- - pkg/katautils/config.go | 18 +++++++------ - virtcontainers/api.go | 4 ++- - virtcontainers/container.go | 6 ++--- - virtcontainers/persist.go | 30 +++++++++++---------- - virtcontainers/persist/api/config.go | 2 ++ - virtcontainers/pkg/annotations/annotations.go | 2 ++ - virtcontainers/pkg/oci/utils.go | 13 +++++++++ - virtcontainers/sandbox.go | 23 +++++++++++----- - 10 files changed, 97 insertions(+), 51 deletions(-) - -diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in -index e57a954..fae88f9 100644 ---- a/cli/config/configuration-qemu.toml.in -+++ b/cli/config/configuration-qemu.toml.in -@@ -477,6 +477,18 @@ enable_compat_old_cni = true - # See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType - sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@ - -+# It is a new host cgroup solution to limit the kata resouce in the host different from the -+# community original solution.If sandbox_cgroup_with_emulator is enabled, it will override -+# the config of sandbox_cgroup_only. Each Pod corresponds to a pod level cgroup directory -+# which is named with sandboxID. In each pod level cgroup, it contains two sub cgroup -+# directory: vcpu and emulator, these two sub cgroup only valid in the CPU cgroup subsystem, -+# because we just want to distinguish the emulator main thread and vcpu thread in the CPU -+# cgroup subsystem.And with this config enabled, kata-runtime and related sub processes will -+# added into the vcpu cgroup directory with resource limited, and qemu main thread and other -+# non-vcpu threads will be moved into the emulator cgroup without resource limit, which will -+# improve the IO throughput for kata-containers. -+sandbox_cgroup_with_emulator = true -+ - # Enabled experimental feature list, format: ["a", "b"]. - # Experimental features are features not stable enough for production, - # they may break compatibility, and are prepared for a big version bump. -diff --git a/cli/kata-env.go b/cli/kata-env.go -index d8a6068..48026fe 100644 ---- a/cli/kata-env.go -+++ b/cli/kata-env.go -@@ -63,15 +63,16 @@ type RuntimeConfigInfo struct { - - // RuntimeInfo stores runtime details. - type RuntimeInfo struct { -- Version RuntimeVersionInfo -- Config RuntimeConfigInfo -- Debug bool -- Trace bool -- DisableGuestSeccomp bool -- DisableNewNetNs bool -- SandboxCgroupOnly bool -- Experimental []exp.Feature -- Path string -+ Version RuntimeVersionInfo -+ Config RuntimeConfigInfo -+ Debug bool -+ Trace bool -+ DisableGuestSeccomp bool -+ DisableNewNetNs bool -+ SandboxCgroupOnly bool -+ SandboxCgroupWithEmulator bool -+ Experimental []exp.Feature -+ Path string - } - - type VersionInfo struct { -@@ -194,15 +195,16 @@ func getRuntimeInfo(configFile string, config oci.RuntimeConfig) RuntimeInfo { - runtimePath, _ := os.Executable() - - return RuntimeInfo{ -- Debug: config.Debug, -- Trace: config.Trace, -- Version: runtimeVersion, -- Config: runtimeConfig, -- Path: runtimePath, -- DisableNewNetNs: config.DisableNewNetNs, -- SandboxCgroupOnly: config.SandboxCgroupOnly, -- Experimental: config.Experimental, -- DisableGuestSeccomp: config.DisableGuestSeccomp, -+ Debug: config.Debug, -+ Trace: config.Trace, -+ Version: runtimeVersion, -+ Config: runtimeConfig, -+ Path: runtimePath, -+ DisableNewNetNs: config.DisableNewNetNs, -+ SandboxCgroupOnly: config.SandboxCgroupOnly, -+ SandboxCgroupWithEmulator: config.SandboxCgroupWithEmulator, -+ Experimental: config.Experimental, -+ DisableGuestSeccomp: config.DisableGuestSeccomp, - } - } - -diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go -index 3365b3f..89e46f6 100644 ---- a/pkg/katautils/config.go -+++ b/pkg/katautils/config.go -@@ -139,14 +139,15 @@ type proxy struct { - } - - type runtime struct { -- Debug bool `toml:"enable_debug"` -- Tracing bool `toml:"enable_tracing"` -- DisableNewNetNs bool `toml:"disable_new_netns"` -- EnableCompatOldCNI bool `toml:"enable_compat_old_cni"` -- DisableGuestSeccomp bool `toml:"disable_guest_seccomp"` -- SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"` -- Experimental []string `toml:"experimental"` -- InterNetworkModel string `toml:"internetworking_model"` -+ Debug bool `toml:"enable_debug"` -+ Tracing bool `toml:"enable_tracing"` -+ DisableNewNetNs bool `toml:"disable_new_netns"` -+ EnableCompatOldCNI bool `toml:"enable_compat_old_cni"` -+ DisableGuestSeccomp bool `toml:"disable_guest_seccomp"` -+ SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"` -+ SandboxCgroupWithEmulator bool `toml:"sandbox_cgroup_with_emulator"` -+ Experimental []string `toml:"experimental"` -+ InterNetworkModel string `toml:"internetworking_model"` - } - - type shim struct { -@@ -1252,6 +1253,7 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool, debugFlag - } - - config.SandboxCgroupOnly = tomlConf.Runtime.SandboxCgroupOnly -+ config.SandboxCgroupWithEmulator = tomlConf.Runtime.SandboxCgroupWithEmulator - config.DisableNewNetNs = tomlConf.Runtime.DisableNewNetNs - config.EnableCompatOldCNI = tomlConf.Runtime.EnableCompatOldCNI - for _, f := range tomlConf.Runtime.Experimental { -diff --git a/virtcontainers/api.go b/virtcontainers/api.go -index ca5412a..08bcbb5 100644 ---- a/virtcontainers/api.go -+++ b/virtcontainers/api.go -@@ -103,7 +103,9 @@ func createSandboxFromConfig(ctx context.Context, sandboxConfig SandboxConfig, f - }() - - // Move runtime to sandbox cgroup so all process are created there. -- if s.config.SandboxCgroupOnly { -+ if s.config.SandboxCgroupWithEmulator{ -+ // emulator -+ } else if s.config.SandboxCgroupOnly { - if err := s.setupSandboxCgroup(); err != nil { - return nil, err - } -diff --git a/virtcontainers/container.go b/virtcontainers/container.go -index 4060ebb..1b70382 100644 ---- a/virtcontainers/container.go -+++ b/virtcontainers/container.go -@@ -1009,7 +1009,7 @@ func (c *Container) create() (err error) { - } - } - -- if !rootless.IsRootless() && !c.sandbox.config.SandboxCgroupOnly { -+ if !rootless.IsRootless() && !c.sandbox.config.SandboxCgroupOnly && !c.sandbox.config.SandboxCgroupWithEmulator { - if err = c.cgroupsCreate(); err != nil { - return - } -@@ -1034,7 +1034,7 @@ func (c *Container) delete() error { - } - - // If running rootless, there are no cgroups to remove -- if !c.sandbox.config.SandboxCgroupOnly || !rootless.IsRootless() { -+ if !c.sandbox.config.SandboxCgroupWithEmulator && (!c.sandbox.config.SandboxCgroupOnly || !rootless.IsRootless()) { - if err := c.cgroupsDelete(); err != nil { - return err - } -@@ -1348,7 +1348,7 @@ func (c *Container) update(resources specs.LinuxResources) error { - } - } - -- if !c.sandbox.config.SandboxCgroupOnly { -+ if !c.sandbox.config.SandboxCgroupWithEmulator && !c.sandbox.config.SandboxCgroupOnly { - if err := c.cgroupsUpdate(resources); err != nil { - return err - } -diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go -index fe00bf9..efa4506 100644 ---- a/virtcontainers/persist.go -+++ b/virtcontainers/persist.go -@@ -194,13 +194,14 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) { - InterworkingModel: int(sconfig.NetworkConfig.InterworkingModel), - }, - -- ShmSize: sconfig.ShmSize, -- SharePidNs: sconfig.SharePidNs, -- Stateful: sconfig.Stateful, -- SystemdCgroup: sconfig.SystemdCgroup, -- SandboxCgroupOnly: sconfig.SandboxCgroupOnly, -- DisableGuestSeccomp: sconfig.DisableGuestSeccomp, -- Cgroups: sconfig.Cgroups, -+ ShmSize: sconfig.ShmSize, -+ SharePidNs: sconfig.SharePidNs, -+ Stateful: sconfig.Stateful, -+ SystemdCgroup: sconfig.SystemdCgroup, -+ SandboxCgroupOnly: sconfig.SandboxCgroupOnly, -+ SandboxCgroupWithEmulator: sconfig.SandboxCgroupWithEmulator, -+ DisableGuestSeccomp: sconfig.DisableGuestSeccomp, -+ Cgroups: sconfig.Cgroups, - } - - for _, e := range sconfig.Experimental { -@@ -485,13 +486,14 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) { - InterworkingModel: NetInterworkingModel(savedConf.NetworkConfig.InterworkingModel), - }, - -- ShmSize: savedConf.ShmSize, -- SharePidNs: savedConf.SharePidNs, -- Stateful: savedConf.Stateful, -- SystemdCgroup: savedConf.SystemdCgroup, -- SandboxCgroupOnly: savedConf.SandboxCgroupOnly, -- DisableGuestSeccomp: savedConf.DisableGuestSeccomp, -- Cgroups: savedConf.Cgroups, -+ ShmSize: savedConf.ShmSize, -+ SharePidNs: savedConf.SharePidNs, -+ Stateful: savedConf.Stateful, -+ SystemdCgroup: savedConf.SystemdCgroup, -+ SandboxCgroupOnly: savedConf.SandboxCgroupOnly, -+ SandboxCgroupWithEmulator: savedConf.SandboxCgroupWithEmulator, -+ DisableGuestSeccomp: savedConf.DisableGuestSeccomp, -+ Cgroups: savedConf.Cgroups, - } - - for _, name := range savedConf.Experimental { -diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go -index 3a2df32..28204fc 100644 ---- a/virtcontainers/persist/api/config.go -+++ b/virtcontainers/persist/api/config.go -@@ -258,6 +258,8 @@ type SandboxConfig struct { - // SandboxCgroupOnly enables cgroup only at podlevel in the host - SandboxCgroupOnly bool - -+ SandboxCgroupWithEmulator bool -+ - DisableGuestSeccomp bool - - // Experimental enables experimental features -diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go -index 528dfa6..96c4ef2 100644 ---- a/virtcontainers/pkg/annotations/annotations.go -+++ b/virtcontainers/pkg/annotations/annotations.go -@@ -215,6 +215,8 @@ const ( - // SandboxCgroupOnly is a sandbox annotation that determines if kata processes are managed only in sandbox cgroup. - SandboxCgroupOnly = kataAnnotRuntimePrefix + "sandbox_cgroup_only" - -+ SandboxCgroupWithEmulator = kataAnnotRuntimePrefix + "sandbox_cgroup_with_emulator" -+ - // Experimental is a sandbox annotation that determines if experimental features enabled. - Experimental = kataAnnotRuntimePrefix + "experimental" - -diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go -index 3b2af75..91067fb 100644 ---- a/virtcontainers/pkg/oci/utils.go -+++ b/virtcontainers/pkg/oci/utils.go -@@ -139,6 +139,8 @@ type RuntimeConfig struct { - //Determines kata processes are managed only in sandbox cgroup - SandboxCgroupOnly bool - -+ SandboxCgroupWithEmulator bool -+ - //Experimental features enabled - Experimental []exp.Feature - } -@@ -746,6 +748,15 @@ func addRuntimeConfigOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e - sbConfig.SandboxCgroupOnly = sandboxCgroupOnly - } - -+ if value, ok := ocispec.Annotations[vcAnnotations.SandboxCgroupWithEmulator]; ok { -+ sandboxCgroupWithEmulator, err := strconv.ParseBool(value) -+ if err != nil { -+ return fmt.Errorf("error parsing annotation for sandbox_cgroup_with_emulator : Please specify boolean value 'true|false'") -+ } -+ -+ sbConfig.SandboxCgroupWithEmulator = sandboxCgroupWithEmulator -+ } -+ - if value, ok := ocispec.Annotations[vcAnnotations.Experimental]; ok { - features := strings.Split(value, " ") - sbConfig.Experimental = []exp.Feature{} -@@ -869,6 +880,8 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid, c - - SandboxCgroupOnly: runtime.SandboxCgroupOnly, - -+ SandboxCgroupWithEmulator: runtime.SandboxCgroupWithEmulator, -+ - DisableGuestSeccomp: runtime.DisableGuestSeccomp, - - // Q: Is this really necessary? @weizhang555 -diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go -index 174e6cb..b479cf5 100644 ---- a/virtcontainers/sandbox.go -+++ b/virtcontainers/sandbox.go -@@ -126,6 +126,8 @@ type SandboxConfig struct { - // SandboxCgroupOnly enables cgroup only at podlevel in the host - SandboxCgroupOnly bool - -+ SandboxCgroupWithEmulator bool -+ - DisableGuestSeccomp bool - - // Experimental features enabled -@@ -1532,8 +1534,9 @@ func (s *Sandbox) Stats() (SandboxStats, error) { - - var path string - var cgroupSubsystems cgroups.Hierarchy -- -- if s.config.SandboxCgroupOnly { -+ if !s.config.SandboxCgroupWithEmulator { -+ // vcpu and emulator -+ } else if s.config.SandboxCgroupOnly { - cgroupSubsystems = cgroups.V1 - path = s.state.CgroupPath - } else { -@@ -1793,7 +1796,9 @@ func (s *Sandbox) HotplugAddDevice(device api.Device, devType config.DeviceType) - span, _ := s.trace("HotplugAddDevice") - defer span.Finish() - -- if s.config.SandboxCgroupOnly { -+ if s.config.SandboxCgroupWithEmulator { -+ // emulator -+ } else if s.config.SandboxCgroupOnly { - // We are about to add a device to the hypervisor, - // the device cgroup MUST be updated since the hypervisor - // will need access to such device -@@ -1849,7 +1854,9 @@ func (s *Sandbox) HotplugAddDevice(device api.Device, devType config.DeviceType) - // Sandbox implement DeviceReceiver interface from device/api/interface.go - func (s *Sandbox) HotplugRemoveDevice(device api.Device, devType config.DeviceType) error { - defer func() { -- if s.config.SandboxCgroupOnly { -+ if s.config.SandboxCgroupWithEmulator { -+ -+ } else if s.config.SandboxCgroupOnly { - // Remove device from cgroup, the hypervisor - // should not have access to such device anymore. - hdev := device.GetHostPath() -@@ -2107,7 +2114,7 @@ func (s *Sandbox) cgroupsUpdate() error { - // If Kata is configured for SandboxCgroupOnly, the VMM and its processes are already - // in the Kata sandbox cgroup (inherited). No need to move threads/processes, and we should - // rely on parent's cgroup CPU/memory values -- if s.config.SandboxCgroupOnly { -+ if s.config.SandboxCgroupWithEmulator || s.config.SandboxCgroupOnly { - return nil - } - -@@ -2154,7 +2161,9 @@ func (s *Sandbox) cgroupsDelete() error { - var path string - var cgroupSubsystems cgroups.Hierarchy - -- if s.config.SandboxCgroupOnly { -+ if s.config.SandboxCgroupWithEmulator { -+ // emulator -+ } else if s.config.SandboxCgroupOnly { - return s.cgroupMgr.Destroy() - } - -@@ -2197,7 +2206,7 @@ func (s *Sandbox) constrainHypervisor(cgroup cgroups.Cgroup) error { - // Kata/VMM into account, Kata may fail to boot due to being overconstrained. - // If !SandboxCgroupOnly, place the VMM into an unconstrained cgroup, and the vCPU threads into constrained - // cgroup -- if s.config.SandboxCgroupOnly { -+ if s.config.SandboxCgroupOnly || s.config.SandboxCgroupWithEmulator { - // Kata components were moved into the sandbox-cgroup already, so VMM - // will already land there as well. No need to take action - return nil --- -1.8.3.1 - diff --git a/runtime/patches/0055-kata_runtime-support-host-cgroup-with-emulator-polic.patch b/runtime/patches/0055-kata_runtime-support-host-cgroup-with-emulator-polic.patch deleted file mode 100644 index 6d63e83..0000000 --- a/runtime/patches/0055-kata_runtime-support-host-cgroup-with-emulator-polic.patch +++ /dev/null @@ -1,428 +0,0 @@ -From ce7523dfe1bb60cf54254e16a103fd3fc9503618 Mon Sep 17 00:00:00 2001 -From: yangfeiyu -Date: Thu, 17 Sep 2020 10:38:38 +0800 -Subject: [PATCH 3/5] kata_runtime: support host cgroup with emulator policy - -reason: support host cgroup with emulator policy when -sandbox_cgroup_with_emulator is set true - -Signed-off-by: yangfeiyu ---- - cli/create.go | 38 ++++++++++++ - virtcontainers/api.go | 10 ++- - virtcontainers/cgroups.go | 132 ++++++++++++++++++++++++++++++++++------ - virtcontainers/persist/fs/fs.go | 8 +++ - virtcontainers/pkg/oci/utils.go | 14 +++++ - virtcontainers/sandbox.go | 70 ++++++++++++++++++++- - 6 files changed, 250 insertions(+), 22 deletions(-) - -diff --git a/cli/create.go b/cli/create.go -index 02cb2c5..b14434b 100644 ---- a/cli/create.go -+++ b/cli/create.go -@@ -11,6 +11,7 @@ import ( - "errors" - "fmt" - "os" -+ "path/filepath" - - "github.com/kata-containers/runtime/pkg/katautils" - vc "github.com/kata-containers/runtime/virtcontainers" -@@ -134,11 +135,48 @@ func create(ctx context.Context, containerID, bundlePath, console, pidFilePath s - var process vc.Process - switch containerType { - case vc.PodSandbox: -+ if runtimeConfig.SandboxCgroupWithEmulator { -+ // create the sandbox level cgroup -+ cgroupPath := ociSpec.Linux.CgroupsPath -+ if err = vci.CreateSandboxCgroup(ctx, cgroupPath); err != nil { -+ return err -+ } -+ -+ defer func() { -+ if err != nil { -+ _ = vci.DestroySandboxCgroup(ctx, cgroupPath) -+ } -+ }() -+ -+ // add kata-runtime create process into /vcpu cgroup -+ vcpuCgroupPath := filepath.Join(cgroupPath, "vcpu") -+ if err = vci.AddPidToSandboxCgroup(ctx, os.Getpid(), vcpuCgroupPath); err != nil { -+ return err -+ } -+ } -+ - _, process, err = katautils.CreateSandbox(ctx, vci, ociSpec, runtimeConfig, rootFs, containerID, bundlePath, console, disableOutput, systemdCgroup, false) - if err != nil { - return err - } - case vc.PodContainer: -+ if runtimeConfig.SandboxCgroupWithEmulator { -+ sandboxID, err := oci.GetSandboxIDFromAnnotations(&ociSpec) -+ if err != nil { -+ return fmt.Errorf("container annotation doesn't contain sandboxID") -+ } -+ -+ sandboxCgroupPath, err := vci.GetSandboxCgroupPath(ctx, sandboxID) -+ if err != nil { -+ return err -+ } -+ -+ // add kata-runtime create process into /vcpu cgroup -+ vcpuCgroupPath := filepath.Join(sandboxCgroupPath, "vcpu") -+ if err = vci.AddPidToSandboxCgroup(ctx, os.Getpid(), vcpuCgroupPath); err != nil { -+ return err -+ } -+ } - process, err = katautils.CreateContainer(ctx, vci, nil, ociSpec, rootFs, containerID, bundlePath, console, disableOutput, false) - if err != nil { - return err -diff --git a/virtcontainers/api.go b/virtcontainers/api.go -index 08bcbb5..38c8235 100644 ---- a/virtcontainers/api.go -+++ b/virtcontainers/api.go -@@ -103,9 +103,7 @@ func createSandboxFromConfig(ctx context.Context, sandboxConfig SandboxConfig, f - }() - - // Move runtime to sandbox cgroup so all process are created there. -- if s.config.SandboxCgroupWithEmulator{ -- // emulator -- } else if s.config.SandboxCgroupOnly { -+ if !s.config.SandboxCgroupWithEmulator && s.config.SandboxCgroupOnly { - if err := s.setupSandboxCgroup(); err != nil { - return nil, err - } -@@ -129,6 +127,12 @@ func createSandboxFromConfig(ctx context.Context, sandboxConfig SandboxConfig, f - return nil, err - } - -+ if s.config.SandboxCgroupWithEmulator { -+ if err := s.setupHostCgroupsWithEmulator(); err != nil { -+ return nil, err -+ } -+ } -+ - // Create Containers - if err = s.createContainers(); err != nil { - return nil, err -diff --git a/virtcontainers/cgroups.go b/virtcontainers/cgroups.go -index df0ec30..65d2001 100644 ---- a/virtcontainers/cgroups.go -+++ b/virtcontainers/cgroups.go -@@ -9,19 +9,15 @@ package virtcontainers - import ( - "bufio" - "context" -- "encoding/json" - "fmt" -- "io/ioutil" - "os" - "path/filepath" -+ "strconv" - "strings" - - "github.com/containerd/cgroups" - specs "github.com/opencontainers/runtime-spec/specs-go" - "github.com/sirupsen/logrus" -- -- "github.com/kata-containers/runtime/virtcontainers/store" -- "github.com/kata-containers/runtime/virtcontainers/types" - ) - - type cgroupPather interface { -@@ -32,7 +28,11 @@ type cgroupPather interface { - // unconstrained cgroups are placed here. - // for example /sys/fs/cgroup/memory/kata/$CGPATH - // where path is defined by the containers manager --const cgroupKataPath = "/kata/" -+const ( -+ cgroupKataPath = "/kata/" -+ vcpuCgroupName = "vcpu" -+ emulatorCgroupName = "emulator" -+) - - var cgroupsLoadFunc = cgroups.Load - var cgroupsNewFunc = cgroups.New -@@ -105,24 +105,16 @@ func deleteCgroup(hierarchy cgroups.Hierarchy, cgroupPath string) error { - - // GetSandboxCgroupPath return the cgroup path of specified sandbox - func GetSandboxCgroupPath(ctx context.Context, sandboxID string) (string, error) { -- stateFilePath := filepath.Join(store.RunStoragePath(), sandboxID, store.StateFile) -- -- fileData, err := ioutil.ReadFile(stateFilePath) -+ config, err := loadSandboxConfig(sandboxID) - if err != nil { - return "", err - } - -- state := types.SandboxState{} -- -- if err := json.Unmarshal(fileData, &state); err != nil { -- return "", err -- } -- -- if state.CgroupPath == "" { -- return "", fmt.Errorf("get sandbox cgroup path error: cgroupPath is empty") -+ if config.Cgroups == nil { -+ return "", fmt.Errorf("the cgroups of sandbox %s is nil", sandboxID) - } - -- return state.CgroupPath, nil -+ return config.Cgroups.Path, nil - } - - // AddPidToSandboxCgroup add kata-runtime create process to cgroup -@@ -276,3 +268,107 @@ func validCPUResources(cpuSpec *specs.LinuxCPU) *specs.LinuxCPU { - - return &cpu - } -+ -+// getQemuTaskWithoutVcpu filter out tasks under /proc/{qemu pid}/task, to find out the task of not VCPU, -+// VCPU task is filtered by "query-cpus" qmp command -+func getQemuTaskWithoutVcpu(sandbox *Sandbox, vmPid int) []int { -+ procPath := fmt.Sprintf("/proc/%d/task", vmPid) -+ -+ dirReader, err := os.Open(procPath) -+ if err != nil { -+ logrus.Warningf("cannot open %s: %s", procPath, err) -+ return nil -+ } -+ -+ defer dirReader.Close() -+ -+ dirs, err := dirReader.Readdirnames(0) -+ if err != nil { -+ logrus.Warningf("walking dirs in %s failed: %s", procPath, err) -+ return nil -+ } -+ -+ vcpuThreadInfo, err := sandbox.hypervisor.getThreadIDs() -+ if err != nil { -+ logrus.Warnf("get hypervisor Thread ID failed: %v", err) -+ return nil -+ } -+ -+ var vcpuThreadIDs []int -+ for _, value := range vcpuThreadInfo.vcpus { -+ vcpuThreadIDs = append(vcpuThreadIDs, value) -+ } -+ -+ var allThreadIDs []int -+ for _, dir := range dirs { -+ p, err := strconv.Atoi(dir) -+ if err != nil { -+ logrus.Warnf("can not change string dir: %s to int type", dir) -+ return nil -+ } -+ -+ allThreadIDs = append(allThreadIDs, p) -+ } -+ -+ nonVCPUThreads := diffSlice(allThreadIDs, vcpuThreadIDs) -+ -+ return nonVCPUThreads -+} -+ -+func pulloutQemuThread(sandbox *Sandbox, vmPid int, path string) error { -+ control, err := cgroups.New(cgroups.SingleSubsystem(cgroups.V1, cgroups.Cpu), -+ cgroups.StaticPath(path), -+ &specs.LinuxResources{}) -+ if err != nil { -+ return err -+ } -+ taskIds := getQemuTaskWithoutVcpu(sandbox, vmPid) -+ if len(taskIds) == 0 { -+ logrus.Warnf("no taskId id in qemu other than vcpu found of pid %d", vmPid) -+ return nil -+ } -+ for _, taskId := range taskIds { -+ if err := control.AddTask(cgroups.Process{ -+ Pid: taskId, -+ }); err != nil { -+ logrus.Errorf("failed to add task %d to cgroup of %s", taskId, path) -+ return err -+ } -+ } -+ -+ return nil -+} -+ -+// checkCgroupExist check cgroup exist or not -+func checkCgroupExist(hierarchy cgroups.Hierarchy, path string) bool { -+ subSystems, _ := hierarchy() -+ for _, s := range cgroupPathers(subSystems) { -+ if _, err := os.Lstat(s.Path(path)); err != nil { -+ if os.IsNotExist(err) { -+ return false -+ } -+ } -+ } -+ -+ return true -+} -+ -+// diffSlice return the s1 - s2 -+func diffSlice(s1, s2 []int) []int { -+ var diffSlice []int -+ for _, p := range s1 { -+ if !isInSlice(p, s2) { -+ diffSlice = append(diffSlice, p) -+ } -+ } -+ return diffSlice -+} -+ -+func isInSlice(i int, s []int) bool { -+ for _, v := range s { -+ if i == v { -+ return true -+ } -+ } -+ return false -+} -diff --git a/virtcontainers/persist/fs/fs.go b/virtcontainers/persist/fs/fs.go -index 38efdba..641d64e 100644 ---- a/virtcontainers/persist/fs/fs.go -+++ b/virtcontainers/persist/fs/fs.go -@@ -14,6 +14,8 @@ import ( - "path/filepath" - "syscall" - -+ "github.com/opencontainers/runc/libcontainer/configs" -+ - persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" - "github.com/sirupsen/logrus" - ) -@@ -78,6 +80,12 @@ func (fs *FS) ToDisk(ss persistapi.SandboxState, cs map[string]persistapi.Contai - return fmt.Errorf("sandbox container id required") - } - -+ if ss.Config.Cgroups == nil { -+ ss.Config.Cgroups = &configs.Cgroup{ -+ Path: ss.CgroupPath, -+ } -+ } -+ - fs.sandboxState = &ss - fs.containerState = cs - -diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go -index 91067fb..e8ef41b 100644 ---- a/virtcontainers/pkg/oci/utils.go -+++ b/virtcontainers/pkg/oci/utils.go -@@ -1136,3 +1136,17 @@ func validateSandboxDNS(value string) error { - - return nil - } -+ -+func GetSandboxIDFromAnnotations(s *specs.Spec) (string, error) { -+ if s == nil { -+ return "", fmt.Errorf("spec is nil") -+ } -+ -+ for _, v := range CRISandboxNameKeyList { -+ if sandboxID, ok := s.Annotations[v]; ok { -+ return sandboxID, nil -+ } -+ } -+ -+ return "", fmt.Errorf("failed to find the sandbox ID") -+} -diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go -index b479cf5..ca4e700 100644 ---- a/virtcontainers/sandbox.go -+++ b/virtcontainers/sandbox.go -@@ -2162,7 +2162,9 @@ func (s *Sandbox) cgroupsDelete() error { - var cgroupSubsystems cgroups.Hierarchy - - if s.config.SandboxCgroupWithEmulator { -- // emulator -+ if err := deleteCgroup(cgroups.V1, s.state.CgroupPath); err != nil { -+ return err -+ } - } else if s.config.SandboxCgroupOnly { - return s.cgroupMgr.Destroy() - } -@@ -2381,6 +2383,68 @@ func (s *Sandbox) setupSandboxCgroup() error { - return nil - } - -+func (s *Sandbox) setupHostCgroupsWithEmulator() error { -+ if len(s.config.Containers) == 0 { -+ return nil -+ } -+ -+ sandboxContainerSpec := s.GetPatchedOCISpec() -+ if sandboxContainerSpec == nil { -+ return fmt.Errorf("sandbox container should not be empty") -+ } -+ -+ // Set sandbox's cgroup path -+ s.state.CgroupPath = sandboxContainerSpec.Linux.CgroupsPath -+ -+ if !checkCgroupExist(cgroups.V1, s.state.CgroupPath) { -+ return fmt.Errorf("sandbox's cgroup %s doesn't exist", s.state.CgroupPath) -+ } -+ -+ // pull out qemu threads other than vcpu to the cgroup of "/emulator" -+ if s.config.HypervisorType == QemuHypervisor { -+ emulatorCgroupPath := filepath.Join(s.state.CgroupPath, emulatorCgroupName) -+ hypervisorPids := s.hypervisor.getPids() -+ if len(hypervisorPids) == 0 || hypervisorPids[0] == 0 { -+ return fmt.Errorf("hypervisor pid: %v invalid", hypervisorPids) -+ } -+ if err := pulloutQemuThread(s, hypervisorPids[0], emulatorCgroupPath); err != nil { -+ return err -+ } -+ } -+ -+ // limit cpu to "/vcpu" -+ vcpuCgroupPath := filepath.Join(s.state.CgroupPath, vcpuCgroupName) -+ vcpuResources := specs.LinuxResources{ -+ CPU: s.cpuResources(), -+ } -+ if err := applyResourceLimit(&vcpuResources, vcpuCgroupPath); err != nil { -+ return err -+ } -+ -+ // limit blkio resource to "" -+ -+ // limit files resource -+ -+ return nil -+} -+ -+func applyResourceLimit(resources *specs.LinuxResources, cgroupPath string) error { -+ if resources == nil { -+ return nil -+ } -+ -+ control, err := cgroupsLoadFunc(cgroups.V1, cgroups.StaticPath(cgroupPath)) -+ if err != nil { -+ return fmt.Errorf("could not load cgroup %v: %v", cgroupPath, err) -+ } -+ -+ if err = control.Update(resources); err != nil { -+ return fmt.Errorf("could not update cgroup %v: %v", cgroupPath, err) -+ } -+ -+ return nil -+} -+ - // GetPatchedOCISpec returns sandbox's OCI specification - // This OCI specification was patched when the sandbox was created - // by containerCapabilities(), SetEphemeralStorageType() and others -@@ -2452,6 +2516,10 @@ func (s *Sandbox) forceDeleteSandbox() { - c.forceDeleteContainer() - } - -+ if err := deleteCgroup(cgroups.V1, s.state.CgroupPath); err != nil { -+ s.Logger().Warnf("sandbox forceDelete cgroups failed: %v", err) -+ } -+ - globalSandboxList.removeSandbox(s.id) - - if s.monitor != nil { --- -1.8.3.1 - diff --git a/runtime/patches/0056-kata_runtime-support-the-blkio-in-host-cgroups.patch b/runtime/patches/0056-kata_runtime-support-the-blkio-in-host-cgroups.patch deleted file mode 100644 index dc8be5d..0000000 --- a/runtime/patches/0056-kata_runtime-support-the-blkio-in-host-cgroups.patch +++ /dev/null @@ -1,318 +0,0 @@ -From f4b899b933a3a30fc378ceb4d8855778cd783d9a Mon Sep 17 00:00:00 2001 -From: holyfei -Date: Fri, 18 Sep 2020 16:54:11 +0800 -Subject: [PATCH 4/5] kata_runtime: support the blkio in host cgroups - -reason: support the blkio in host cgroups, run with paras ---annotation io.kubernetes.docker.type=podsandbox --annotation io.katacontainers.blkio_cgroup= -'{"blkiocgroup":[{"path":"/dev/sda","limits":[{"type":"throttle_read_bps","value":400}]}]}' - -Signed-off-by: yangfeiyu ---- - vendor/github.com/containerd/cgroups/blkio.go | 6 ++ - virtcontainers/cgroups.go | 102 ++++++++++++++++++++++++++ - virtcontainers/pkg/annotations/annotations.go | 3 + - virtcontainers/pkg/oci/utils.go | 50 ++++++++++++- - virtcontainers/sandbox.go | 6 ++ - virtcontainers/utils/utils.go | 22 ++++++ - 6 files changed, 186 insertions(+), 3 deletions(-) - -diff --git a/vendor/github.com/containerd/cgroups/blkio.go b/vendor/github.com/containerd/cgroups/blkio.go -index 7c498de..485ff7b 100644 ---- a/vendor/github.com/containerd/cgroups/blkio.go -+++ b/vendor/github.com/containerd/cgroups/blkio.go -@@ -23,6 +23,7 @@ import ( - "io/ioutil" - "os" - "path/filepath" -+ "reflect" - "strconv" - "strings" - -@@ -55,6 +56,11 @@ func (b *blkioController) Create(path string, resources *specs.LinuxResources) e - return nil - } - for _, t := range createBlkioSettings(resources.BlockIO) { -+ ptr := reflect.ValueOf(t.value) -+ if ptr.Kind() == reflect.Ptr && ptr.IsNil() { -+ continue -+ } -+ - if t.value != nil { - if err := ioutil.WriteFile( - filepath.Join(b.Path(path), fmt.Sprintf("blkio.%s", t.name)), -diff --git a/virtcontainers/cgroups.go b/virtcontainers/cgroups.go -index 65d2001..e8c5a7b 100644 ---- a/virtcontainers/cgroups.go -+++ b/virtcontainers/cgroups.go -@@ -9,6 +9,7 @@ package virtcontainers - import ( - "bufio" - "context" -+ "encoding/json" - "fmt" - "os" - "path/filepath" -@@ -16,6 +17,8 @@ import ( - "strings" - - "github.com/containerd/cgroups" -+ "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" -+ "github.com/kata-containers/runtime/virtcontainers/utils" - specs "github.com/opencontainers/runtime-spec/specs-go" - "github.com/sirupsen/logrus" - ) -@@ -32,8 +35,46 @@ const ( - cgroupKataPath = "/kata/" - vcpuCgroupName = "vcpu" - emulatorCgroupName = "emulator" -+ -+ // BlkioThrottleReadBps is the key to fetch throttle_read_bps -+ BlkioThrottleReadBps = "throttle_read_bps" -+ -+ // BlkioThrottleWriteBps is the key to fetch throttle_write_bps -+ BlkioThrottleWriteBps = "throttle_write_bps" -+ -+ // BlkioThrottleReadIOPS is the key to fetch throttle_read_iops -+ BlkioThrottleReadIOPS = "throttle_read_iops" -+ -+ // BlkioThrottleWriteIOPS is the key to fetch throttle_write_iops -+ BlkioThrottleWriteIOPS = "throttle_write_iops" -+ -+ // BlkioWeight is the key to fetch blkio_weight -+ BlkioWeight = "blkio_weight" -+ -+ // BlkioLeafWeight is the key to fetch blkio_leaf_weight -+ BlkioLeafWeight = "blkio_leaf_weight" - ) - -+// BlkioCgroup for Linux cgroup 'blkio' data exchange -+type BlkioCgroup struct { -+ // Items specifies per cgroup values -+ Items []BlockIOCgroupItem `json:"blkiocgroup,omitempty"` -+} -+ -+type BlockIOCgroupItem struct { -+ // Path represent path of blkio device -+ Path string `json:"path,omitempty"` -+ // Limits specifies the blkio type and value -+ Limits []IOLimit `json:"limits,omitempty"` -+} -+ -+type IOLimit struct { -+ // Type specifies IO type -+ Type string `json:"type,omitempty"` -+ // Value specifies rate or weight value -+ Value uint64 `json:"value,omitempty"` -+} -+ - var cgroupsLoadFunc = cgroups.Load - var cgroupsNewFunc = cgroups.New - -@@ -372,3 +413,64 @@ func isInSlice(i int, s []int) bool { - } - return false - } -+ -+func (s *Sandbox) blockIOResource() *specs.LinuxBlockIO { -+ value, ok := s.config.Annotations[annotations.BlkioCgroupTypeKey] -+ if !ok { -+ return nil -+ } -+ -+ var blkioCgroupParse BlkioCgroup -+ var linuxBlkio specs.LinuxBlockIO -+ if err := json.Unmarshal([]byte(value), &blkioCgroupParse); err != nil { -+ s.Logger().Errorf("blkio_cgroup Unmarshal error:%v", err) -+ return nil -+ } -+ -+ for _, item := range blkioCgroupParse.Items { -+ if item.Limits == nil { -+ s.Logger().Errorf("%v:limits have none data", item) -+ return nil -+ } -+ for _, limit := range item.Limits { -+ if item.Path != "" { -+ major, minor, err := utils.GetDeviceByPath(item.Path) -+ if err != nil { -+ s.Logger().Errorf("failed to find major and minor of device %s: %v", item.Path, err) -+ return nil -+ } -+ td := specs.LinuxThrottleDevice{ -+ Rate: limit.Value, -+ } -+ td.Major = major -+ td.Minor = minor -+ switch limit.Type { -+ case BlkioThrottleReadBps: -+ linuxBlkio.ThrottleReadBpsDevice = append(linuxBlkio.ThrottleReadBpsDevice, td) -+ case BlkioThrottleWriteBps: -+ linuxBlkio.ThrottleWriteBpsDevice = append(linuxBlkio.ThrottleWriteBpsDevice, td) -+ case BlkioThrottleReadIOPS: -+ linuxBlkio.ThrottleReadIOPSDevice = append(linuxBlkio.ThrottleReadIOPSDevice, td) -+ case BlkioThrottleWriteIOPS: -+ linuxBlkio.ThrottleWriteIOPSDevice = append(linuxBlkio.ThrottleWriteIOPSDevice, td) -+ default: -+ s.Logger().Errorf("the type of throtlle device：%s is not surpport", limit.Type) -+ return nil -+ } -+ } else { -+ switch limit.Type { -+ case BlkioWeight: -+ weightUint16 := uint16(limit.Value) -+ linuxBlkio.Weight = &weightUint16 -+ case BlkioLeafWeight: -+ weightLeafUint16 := uint16(limit.Value) -+ linuxBlkio.LeafWeight = &weightLeafUint16 -+ default: -+ s.Logger().Errorf("the type:%s is not one of the supported types of blkio_weight and blkio_leaf_weight", limit.Type) -+ return nil -+ } -+ } -+ } -+ } -+ return &linuxBlkio -+} -diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go -index 96c4ef2..c35993e 100644 ---- a/virtcontainers/pkg/annotations/annotations.go -+++ b/virtcontainers/pkg/annotations/annotations.go -@@ -21,6 +21,9 @@ const ( - ContainerTypeKey = kataAnnotationsPrefix + "pkg.oci.container_type" - - SandboxConfigPathKey = kataAnnotationsPrefix + "config_path" -+ -+ // BlkioCgroupTypeKey is the annotation key to fetch sandbox blkio cgroup values -+ BlkioCgroupTypeKey = kataAnnotationsPrefix + "blkio_cgroup" - ) - - // Annotations related to Hypervisor configuration -diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go -index e8ef41b..643753b 100644 ---- a/virtcontainers/pkg/oci/utils.go -+++ b/virtcontainers/pkg/oci/utils.go -@@ -38,9 +38,10 @@ type annotationContainerType struct { - type annotationHandler func(value string) error - - var annotationHandlerList = map[string]annotationHandler{ -- vcAnnotations.StaticCPUTypeKey: validateSandboxCPU, -- vcAnnotations.StaticMemTypeKey: validateSandboxMem, -- vcAnnotations.SandboxDNSTypeKey: validateSandboxDNS, -+ vcAnnotations.StaticCPUTypeKey: validateSandboxCPU, -+ vcAnnotations.StaticMemTypeKey: validateSandboxMem, -+ vcAnnotations.SandboxDNSTypeKey: validateSandboxDNS, -+ vcAnnotations.BlkioCgroupTypeKey: validateBlkioCgroup, - } - - var ( -@@ -1076,6 +1077,7 @@ func validateOtherSandboxAnnotations(annotation, value string) error { - // addOtherSandboxAnnotation add self defined annotation for sandbox - func addOtherSandboxAnnotation(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error { - otherSandboxAnnotationsKey := []string{ -+ vcAnnotations.BlkioCgroupTypeKey, - vcAnnotations.StaticCPUTypeKey, - vcAnnotations.StaticMemTypeKey, - } -@@ -1137,6 +1139,48 @@ func validateSandboxDNS(value string) error { - return nil - } - -+func validateBlkioCgroup(value string) error { -+ var linuxBlkioCgroup vc.BlkioCgroup -+ if err := json.Unmarshal([]byte(value), &linuxBlkioCgroup); err != nil { -+ return fmt.Errorf("blkio_cgroup Unmarshal error:%v, value is %s", err, value) -+ } -+ -+ if linuxBlkioCgroup.Items == nil { -+ return fmt.Errorf("BlkioCgroup:%v fetch none data", linuxBlkioCgroup) -+ } -+ -+ // pathNull used to judge path for blkio.weight/blkio.leaf_weight only once -+ pathNull := false -+ for _, item := range linuxBlkioCgroup.Items { -+ if pathNull { -+ return fmt.Errorf("too many paths for blkio.weight/blkio.leaf_weight") -+ } -+ if item.Path == "" && !pathNull { -+ pathNull = true -+ if len(item.Limits) > 2 || len(item.Limits) == 0 { -+ return fmt.Errorf("the format of values for blkio.weight or blkio.leaf_weight is wrong") -+ } -+ -+ for _, ioLimit := range item.Limits { -+ switch ioLimit.Type { -+ case vc.BlkioWeight: -+ // Blkio.weight, between 10 and 1000 -+ if ioLimit.Value < 10 || ioLimit.Value > 1000 { -+ return fmt.Errorf("blkio.weight:%v must be between 10 and 1000", ioLimit.Value) -+ } -+ case vc.BlkioLeafWeight: -+ return fmt.Errorf("the blkio.leaf_weight is not supported now") -+ default: -+ return fmt.Errorf("the type of blkio device：%s is not surpport", ioLimit.Type) -+ } -+ } -+ } else if _, _, err := utils.GetDeviceByPath(item.Path); err != nil { -+ return fmt.Errorf("failed to find major and minor of device %s: %v", item.Path, err) -+ } -+ } -+ return nil -+} -+ - func GetSandboxIDFromAnnotations(s *specs.Spec) (string, error) { - if s == nil { - return "", fmt.Errorf("spec is nil") -diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go -index ca4e700..9284f99 100644 ---- a/virtcontainers/sandbox.go -+++ b/virtcontainers/sandbox.go -@@ -2422,6 +2422,12 @@ func (s *Sandbox) setupHostCgroupsWithEmulator() error { - } - - // limit blkio resource to "" -+ blkioResources := specs.LinuxResources{ -+ BlockIO: s.blockIOResource(), -+ } -+ if err := applyResourceLimit(&blkioResources, vcpuCgroupPath); err != nil { -+ return err -+ } - - // limit files resource - -diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go -index 36ac67a..d4dad40 100644 ---- a/virtcontainers/utils/utils.go -+++ b/virtcontainers/utils/utils.go -@@ -370,6 +370,28 @@ func GetPhysicalCPUNumber() int { - return cpuNum - } - -+// GetDeviceByPath returns the device No. -+func GetDeviceByPath(path string) (int64, int64, error) { -+ if path == "" { -+ return -1, -1, fmt.Errorf("Path cannot be empty") -+ } -+ -+ path, err := filepath.EvalSymlinks(path) -+ if err != nil { -+ return -1, -1, err -+ } -+ -+ stat := syscall.Stat_t{} -+ err = syscall.Stat(path, &stat) -+ if err != nil { -+ return -1, -1, err -+ } -+ -+ major := int64(stat.Rdev / 256) -+ minor := int64(stat.Rdev % 256) -+ return major, minor, nil -+} -+ - func RoundVCPUNumber(value string) (int, error) { - cpuNum, err := strconv.ParseFloat(value, 64) - if err != nil || cpuNum < minCPUs { --- -1.8.3.1 - diff --git a/runtime/patches/0057-kata-runtime-support-files-limit-in-host-cgroups.patch b/runtime/patches/0057-kata-runtime-support-files-limit-in-host-cgroups.patch deleted file mode 100644 index ba425ba..0000000 --- a/runtime/patches/0057-kata-runtime-support-files-limit-in-host-cgroups.patch +++ /dev/null @@ -1,5834 +0,0 @@ -From 04011b8a08bb69aa01ca7045bdcb477c14d5feac Mon Sep 17 00:00:00 2001 -From: holyfei -Date: Mon, 21 Sep 2020 19:27:11 +0800 -Subject: [PATCH 5/5] kata-runtime: support --files-limit in host cgroups - -reason: support --files-limit in host cgroups, like --files-limit 2000 - -Signed-off-by: yangfeiyu ---- - vendor/github.com/containerd/cgroups/files.go | 142 + - vendor/github.com/containerd/cgroups/metrics.pb.go | 5050 +++----------------- - vendor/github.com/containerd/cgroups/subsystem.go | 2 + - vendor/github.com/containerd/cgroups/utils.go | 2 +- - .../opencontainers/runtime-spec/specs-go/config.go | 6 + - virtcontainers/cgroups.go | 82 + - virtcontainers/fileslimit/fileslimit.c | 23 + - virtcontainers/fileslimit/fileslimit.go | 23 + - virtcontainers/fileslimit/fileslimit.h | 14 + - virtcontainers/sandbox.go | 38 +- - 10 files changed, 1012 insertions(+), 4370 deletions(-) - create mode 100644 vendor/github.com/containerd/cgroups/files.go - create mode 100644 virtcontainers/fileslimit/fileslimit.c - create mode 100644 virtcontainers/fileslimit/fileslimit.go - create mode 100644 virtcontainers/fileslimit/fileslimit.h - -diff --git a/vendor/github.com/containerd/cgroups/files.go b/vendor/github.com/containerd/cgroups/files.go -new file mode 100644 -index 0000000..c8374c1 ---- /dev/null -+++ b/vendor/github.com/containerd/cgroups/files.go -@@ -0,0 +1,142 @@ -+/* -+Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. -+SPDX-License-Identifier: Apache-2.0 -+Description: common functions -+Author: jiangpengfei -+Create: 2019-06-12 -+*/ -+ -+package cgroups -+ -+import ( -+ "fmt" -+ "io/ioutil" -+ "os" -+ "path/filepath" -+ "strconv" -+ "strings" -+ -+ specs "github.com/opencontainers/runtime-spec/specs-go" -+) -+ -+func NewFiles(root string) *filesController { -+ return &filesController{ -+ root: filepath.Join(root, string(Files)), -+ } -+} -+ -+type filesController struct { -+ root string -+} -+ -+type filesSettings struct { -+ name string -+ value *uint64 -+} -+ -+func (f *filesController) Name() Name { -+ return Files -+} -+ -+func (f *filesController) Path(path string) string { -+ return filepath.Join(f.root, path) -+} -+ -+func (f *filesController) Create(path string, resources *specs.LinuxResources) error { -+ if err := os.MkdirAll(f.Path(path), defaultDirPerm); err != nil { -+ return err -+ } -+ -+ if resources.Files == nil { -+ return nil -+ } -+ -+ settings := getFilesSettings(resources) -+ for _, t := range settings { -+ if t.value != nil { -+ // If files.limit value equal 0, which regard as no limit -+ if *(t.value) == 0 { -+ if err := ioutil.WriteFile( -+ filepath.Join(f.Path(path), fmt.Sprintf("files.%s", t.name)), -+ []byte("max"), -+ defaultFilePerm, -+ ); err != nil { -+ return err -+ } -+ } else { -+ if err := ioutil.WriteFile( -+ filepath.Join(f.Path(path), fmt.Sprintf("files.%s", t.name)), -+ []byte(strconv.FormatUint(*t.value, 10)), -+ defaultFilePerm, -+ ); err != nil { -+ return err -+ } -+ } -+ } -+ } -+ -+ return nil -+} -+ -+func (f *filesController) Update(path string, resources *specs.LinuxResources) error { -+ return f.Create(path, resources) -+} -+ -+func (f *filesController) Stat(path string, stats *Metrics) error { -+ usage, err := getCgroupParamUint(path, "files.usage") -+ if err != nil { -+ return fmt.Errorf("failed to parse files.usage - %s", err) -+ } -+ -+ maxString, err := getCgroupParamString(path, "files.limit") -+ if err != nil { -+ return fmt.Errorf("failed to parse files.limit - %s", err) -+ } -+ -+ // Default if files.limit == "max" is 0 -- which represents "no limit". -+ var max uint64 -+ if maxString != "max" { -+ max, err = parseUint(maxString, 10, 64) -+ if err != nil { -+ return fmt.Errorf("failed to parse files.limit -- unable to parse %q as a uint from Cgroup file %q", maxString, filepath.Join(path, "file.limits")) -+ } -+ } -+ -+ stats.Files.Usage = usage -+ stats.Files.Limit = max -+ return nil -+} -+ -+func getFilesSettings(resources *specs.LinuxResources) []filesSettings { -+ return []filesSettings{ -+ { -+ name: "limit", -+ value: resources.Files.Limit, -+ }, -+ } -+} -+ -+// Gets a single uint64 value from the specified cgroup file. -+func getCgroupParamUint(cgroupPath, cgroupFile string) (uint64, error) { -+ fileName := filepath.Join(cgroupPath, cgroupFile) -+ contents, err := ioutil.ReadFile(fileName) -+ if err != nil { -+ return 0, err -+ } -+ -+ res, err := parseUint(strings.TrimSpace(string(contents)), 10, 64) -+ if err != nil { -+ return res, fmt.Errorf("unable to parse %q as a uint from Cgroup file %q", string(contents), fileName) -+ } -+ return res, nil -+} -+ -+// Gets a string value from the specified cgroup file -+func getCgroupParamString(cgroupPath, cgroupFile string) (string, error) { -+ contents, err := ioutil.ReadFile(filepath.Join(cgroupPath, cgroupFile)) -+ if err != nil { -+ return "", err -+ } -+ -+ return strings.TrimSpace(string(contents)), nil -+} -diff --git a/vendor/github.com/containerd/cgroups/metrics.pb.go b/vendor/github.com/containerd/cgroups/metrics.pb.go -index 7dd7f6f..652744d 100644 ---- a/vendor/github.com/containerd/cgroups/metrics.pb.go -+++ b/vendor/github.com/containerd/cgroups/metrics.pb.go -@@ -2,37 +2,38 @@ - // source: github.com/containerd/cgroups/metrics.proto - - /* -- Package cgroups is a generated protocol buffer package. -- -- It is generated from these files: -- github.com/containerd/cgroups/metrics.proto -- -- It has these top-level messages: -- Metrics -- HugetlbStat -- PidsStat -- CPUStat -- CPUUsage -- Throttle -- MemoryStat -- MemoryEntry -- BlkIOStat -- BlkIOEntry -- RdmaStat -- RdmaEntry -- NetworkStat -+Package cgroups is a generated protocol buffer package. -+ -+It is generated from these files: -+ github.com/containerd/cgroups/metrics.proto -+ -+It has these top-level messages: -+ Metrics -+ HugetlbStat -+ PidsStat -+ CPUStat -+ CPUUsage -+ Throttle -+ MemoryStat -+ MemoryEntry -+ BlkIOStat -+ BlkIOEntry -+ RdmaStat -+ RdmaEntry -+ NetworkStat -+ FilesStat - */ - package cgroups - --import proto "github.com/gogo/protobuf/proto" --import fmt "fmt" --import math "math" --import _ "github.com/gogo/protobuf/gogoproto" -+import ( -+ fmt "fmt" - --import strings "strings" --import reflect "reflect" -+ proto "github.com/gogo/protobuf/proto" - --import io "io" -+ math "math" -+ -+ _ "github.com/gogo/protobuf/gogoproto" -+) - - // Reference imports to suppress errors if they are not otherwise used. - var _ = proto.Marshal -@@ -53,12 +54,70 @@ type Metrics struct { - Blkio *BlkIOStat `protobuf:"bytes,5,opt,name=blkio" json:"blkio,omitempty"` - Rdma *RdmaStat `protobuf:"bytes,6,opt,name=rdma" json:"rdma,omitempty"` - Network []*NetworkStat `protobuf:"bytes,7,rep,name=network" json:"network,omitempty"` -+ Files *FilesStat `protobuf:"bytes,8,opt,name=files" json:"files,omitempty"` - } - - func (m *Metrics) Reset() { *m = Metrics{} } -+func (m *Metrics) String() string { return proto.CompactTextString(m) } - func (*Metrics) ProtoMessage() {} - func (*Metrics) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{0} } - -+func (m *Metrics) GetHugetlb() []*HugetlbStat { -+ if m != nil { -+ return m.Hugetlb -+ } -+ return nil -+} -+ -+func (m *Metrics) GetPids() *PidsStat { -+ if m != nil { -+ return m.Pids -+ } -+ return nil -+} -+ -+func (m *Metrics) GetCPU() *CPUStat { -+ if m != nil { -+ return m.CPU -+ } -+ return nil -+} -+ -+func (m *Metrics) GetMemory() *MemoryStat { -+ if m != nil { -+ return m.Memory -+ } -+ return nil -+} -+ -+func (m *Metrics) GetBlkio() *BlkIOStat { -+ if m != nil { -+ return m.Blkio -+ } -+ return nil -+} -+ -+func (m *Metrics) GetRdma() *RdmaStat { -+ if m != nil { -+ return m.Rdma -+ } -+ return nil -+} -+ -+func (m *Metrics) GetNetwork() []*NetworkStat { -+ if m != nil { -+ return m.Network -+ } -+ return nil -+} -+ -+func (m *Metrics) GetFiles() *FilesStat { -+ if m != nil { -+ return m.Files -+ } -+ return nil -+} -+ - type HugetlbStat struct { - Usage uint64 `protobuf:"varint,1,opt,name=usage,proto3" json:"usage,omitempty"` - Max uint64 `protobuf:"varint,2,opt,name=max,proto3" json:"max,omitempty"` -@@ -67,27 +126,86 @@ type HugetlbStat struct { - } - - func (m *HugetlbStat) Reset() { *m = HugetlbStat{} } -+func (m *HugetlbStat) String() string { return proto.CompactTextString(m) } - func (*HugetlbStat) ProtoMessage() {} - func (*HugetlbStat) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{1} } - -+func (m *HugetlbStat) GetUsage() uint64 { -+ if m != nil { -+ return m.Usage -+ } -+ return 0 -+} -+ -+func (m *HugetlbStat) GetMax() uint64 { -+ if m != nil { -+ return m.Max -+ } -+ return 0 -+} -+ -+func (m *HugetlbStat) GetFailcnt() uint64 { -+ if m != nil { -+ return m.Failcnt -+ } -+ return 0 -+} -+ -+func (m *HugetlbStat) GetPagesize() string { -+ if m != nil { -+ return m.Pagesize -+ } -+ return "" -+} -+ - type PidsStat struct { - Current uint64 `protobuf:"varint,1,opt,name=current,proto3" json:"current,omitempty"` - Limit uint64 `protobuf:"varint,2,opt,name=limit,proto3" json:"limit,omitempty"` - } - - func (m *PidsStat) Reset() { *m = PidsStat{} } -+func (m *PidsStat) String() string { return proto.CompactTextString(m) } - func (*PidsStat) ProtoMessage() {} - func (*PidsStat) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{2} } - -+func (m *PidsStat) GetCurrent() uint64 { -+ if m != nil { -+ return m.Current -+ } -+ return 0 -+} -+ -+func (m *PidsStat) GetLimit() uint64 { -+ if m != nil { -+ return m.Limit -+ } -+ return 0 -+} -+ - type CPUStat struct { - Usage *CPUUsage `protobuf:"bytes,1,opt,name=usage" json:"usage,omitempty"` - Throttling *Throttle `protobuf:"bytes,2,opt,name=throttling" json:"throttling,omitempty"` - } - - func (m *CPUStat) Reset() { *m = CPUStat{} } -+func (m *CPUStat) String() string { return proto.CompactTextString(m) } - func (*CPUStat) ProtoMessage() {} - func (*CPUStat) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{3} } - -+func (m *CPUStat) GetUsage() *CPUUsage { -+ if m != nil { -+ return m.Usage -+ } -+ return nil -+} -+ -+func (m *CPUStat) GetThrottling() *Throttle { -+ if m != nil { -+ return m.Throttling -+ } -+ return nil -+} -+ - type CPUUsage struct { - // values in nanoseconds - Total uint64 `protobuf:"varint,1,opt,name=total,proto3" json:"total,omitempty"` -@@ -97,9 +215,38 @@ type CPUUsage struct { - } - - func (m *CPUUsage) Reset() { *m = CPUUsage{} } -+func (m *CPUUsage) String() string { return proto.CompactTextString(m) } - func (*CPUUsage) ProtoMessage() {} - func (*CPUUsage) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{4} } - -+func (m *CPUUsage) GetTotal() uint64 { -+ if m != nil { -+ return m.Total -+ } -+ return 0 -+} -+ -+func (m *CPUUsage) GetKernel() uint64 { -+ if m != nil { -+ return m.Kernel -+ } -+ return 0 -+} -+ -+func (m *CPUUsage) GetUser() uint64 { -+ if m != nil { -+ return m.User -+ } -+ return 0 -+} -+ -+func (m *CPUUsage) GetPerCPU() []uint64 { -+ if m != nil { -+ return m.PerCPU -+ } -+ return nil -+} -+ - type Throttle struct { - Periods uint64 `protobuf:"varint,1,opt,name=periods,proto3" json:"periods,omitempty"` - ThrottledPeriods uint64 `protobuf:"varint,2,opt,name=throttled_periods,json=throttledPeriods,proto3" json:"throttled_periods,omitempty"` -@@ -107,9 +254,31 @@ type Throttle struct { - } - - func (m *Throttle) Reset() { *m = Throttle{} } -+func (m *Throttle) String() string { return proto.CompactTextString(m) } - func (*Throttle) ProtoMessage() {} - func (*Throttle) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{5} } - -+func (m *Throttle) GetPeriods() uint64 { -+ if m != nil { -+ return m.Periods -+ } -+ return 0 -+} -+ -+func (m *Throttle) GetThrottledPeriods() uint64 { -+ if m != nil { -+ return m.ThrottledPeriods -+ } -+ return 0 -+} -+ -+func (m *Throttle) GetThrottledTime() uint64 { -+ if m != nil { -+ return m.ThrottledTime -+ } -+ return 0 -+} -+ - type MemoryStat struct { - Cache uint64 `protobuf:"varint,1,opt,name=cache,proto3" json:"cache,omitempty"` - RSS uint64 `protobuf:"varint,2,opt,name=rss,proto3" json:"rss,omitempty"` -@@ -150,4547 +319,698 @@ type MemoryStat struct { - } - - func (m *MemoryStat) Reset() { *m = MemoryStat{} } -+func (m *MemoryStat) String() string { return proto.CompactTextString(m) } - func (*MemoryStat) ProtoMessage() {} - func (*MemoryStat) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{6} } - --type MemoryEntry struct { -- Limit uint64 `protobuf:"varint,1,opt,name=limit,proto3" json:"limit,omitempty"` -- Usage uint64 `protobuf:"varint,2,opt,name=usage,proto3" json:"usage,omitempty"` -- Max uint64 `protobuf:"varint,3,opt,name=max,proto3" json:"max,omitempty"` -- Failcnt uint64 `protobuf:"varint,4,opt,name=failcnt,proto3" json:"failcnt,omitempty"` -+func (m *MemoryStat) GetCache() uint64 { -+ if m != nil { -+ return m.Cache -+ } -+ return 0 - } - --func (m *MemoryEntry) Reset() { *m = MemoryEntry{} } --func (*MemoryEntry) ProtoMessage() {} --func (*MemoryEntry) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{7} } -- --type BlkIOStat struct { -- IoServiceBytesRecursive []*BlkIOEntry `protobuf:"bytes,1,rep,name=io_service_bytes_recursive,json=ioServiceBytesRecursive" json:"io_service_bytes_recursive,omitempty"` -- IoServicedRecursive []*BlkIOEntry `protobuf:"bytes,2,rep,name=io_serviced_recursive,json=ioServicedRecursive" json:"io_serviced_recursive,omitempty"` -- IoQueuedRecursive []*BlkIOEntry `protobuf:"bytes,3,rep,name=io_queued_recursive,json=ioQueuedRecursive" json:"io_queued_recursive,omitempty"` -- IoServiceTimeRecursive []*BlkIOEntry `protobuf:"bytes,4,rep,name=io_service_time_recursive,json=ioServiceTimeRecursive" json:"io_service_time_recursive,omitempty"` -- IoWaitTimeRecursive []*BlkIOEntry `protobuf:"bytes,5,rep,name=io_wait_time_recursive,json=ioWaitTimeRecursive" json:"io_wait_time_recursive,omitempty"` -- IoMergedRecursive []*BlkIOEntry `protobuf:"bytes,6,rep,name=io_merged_recursive,json=ioMergedRecursive" json:"io_merged_recursive,omitempty"` -- IoTimeRecursive []*BlkIOEntry `protobuf:"bytes,7,rep,name=io_time_recursive,json=ioTimeRecursive" json:"io_time_recursive,omitempty"` -- SectorsRecursive []*BlkIOEntry `protobuf:"bytes,8,rep,name=sectors_recursive,json=sectorsRecursive" json:"sectors_recursive,omitempty"` -+func (m *MemoryStat) GetRSS() uint64 { -+ if m != nil { -+ return m.RSS -+ } -+ return 0 - } - --func (m *BlkIOStat) Reset() { *m = BlkIOStat{} } --func (*BlkIOStat) ProtoMessage() {} --func (*BlkIOStat) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{8} } -- --type BlkIOEntry struct { -- Op string `protobuf:"bytes,1,opt,name=op,proto3" json:"op,omitempty"` -- Device string `protobuf:"bytes,2,opt,name=device,proto3" json:"device,omitempty"` -- Major uint64 `protobuf:"varint,3,opt,name=major,proto3" json:"major,omitempty"` -- Minor uint64 `protobuf:"varint,4,opt,name=minor,proto3" json:"minor,omitempty"` -- Value uint64 `protobuf:"varint,5,opt,name=value,proto3" json:"value,omitempty"` -+func (m *MemoryStat) GetRSSHuge() uint64 { -+ if m != nil { -+ return m.RSSHuge -+ } -+ return 0 - } - --func (m *BlkIOEntry) Reset() { *m = BlkIOEntry{} } --func (*BlkIOEntry) ProtoMessage() {} --func (*BlkIOEntry) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{9} } -- --type RdmaStat struct { -- Current []*RdmaEntry `protobuf:"bytes,1,rep,name=current" json:"current,omitempty"` -- Limit []*RdmaEntry `protobuf:"bytes,2,rep,name=limit" json:"limit,omitempty"` -+func (m *MemoryStat) GetMappedFile() uint64 { -+ if m != nil { -+ return m.MappedFile -+ } -+ return 0 - } - --func (m *RdmaStat) Reset() { *m = RdmaStat{} } --func (*RdmaStat) ProtoMessage() {} --func (*RdmaStat) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{10} } -- --type RdmaEntry struct { -- Device string `protobuf:"bytes,1,opt,name=device,proto3" json:"device,omitempty"` -- HcaHandles uint32 `protobuf:"varint,2,opt,name=hca_handles,json=hcaHandles,proto3" json:"hca_handles,omitempty"` -- HcaObjects uint32 `protobuf:"varint,3,opt,name=hca_objects,json=hcaObjects,proto3" json:"hca_objects,omitempty"` -+func (m *MemoryStat) GetDirty() uint64 { -+ if m != nil { -+ return m.Dirty -+ } -+ return 0 - } - --func (m *RdmaEntry) Reset() { *m = RdmaEntry{} } --func (*RdmaEntry) ProtoMessage() {} --func (*RdmaEntry) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{11} } -- --type NetworkStat struct { -- Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` -- RxBytes uint64 `protobuf:"varint,2,opt,name=rx_bytes,json=rxBytes,proto3" json:"rx_bytes,omitempty"` -- RxPackets uint64 `protobuf:"varint,3,opt,name=rx_packets,json=rxPackets,proto3" json:"rx_packets,omitempty"` -- RxErrors uint64 `protobuf:"varint,4,opt,name=rx_errors,json=rxErrors,proto3" json:"rx_errors,omitempty"` -- RxDropped uint64 `protobuf:"varint,5,opt,name=rx_dropped,json=rxDropped,proto3" json:"rx_dropped,omitempty"` -- TxBytes uint64 `protobuf:"varint,6,opt,name=tx_bytes,json=txBytes,proto3" json:"tx_bytes,omitempty"` -- TxPackets uint64 `protobuf:"varint,7,opt,name=tx_packets,json=txPackets,proto3" json:"tx_packets,omitempty"` -- TxErrors uint64 `protobuf:"varint,8,opt,name=tx_errors,json=txErrors,proto3" json:"tx_errors,omitempty"` -- TxDropped uint64 `protobuf:"varint,9,opt,name=tx_dropped,json=txDropped,proto3" json:"tx_dropped,omitempty"` -+func (m *MemoryStat) GetWriteback() uint64 { -+ if m != nil { -+ return m.Writeback -+ } -+ return 0 - } - --func (m *NetworkStat) Reset() { *m = NetworkStat{} } --func (*NetworkStat) ProtoMessage() {} --func (*NetworkStat) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{12} } -- --func init() { -- proto.RegisterType((*Metrics)(nil), "io.containerd.cgroups.v1.Metrics") -- proto.RegisterType((*HugetlbStat)(nil), "io.containerd.cgroups.v1.HugetlbStat") -- proto.RegisterType((*PidsStat)(nil), "io.containerd.cgroups.v1.PidsStat") -- proto.RegisterType((*CPUStat)(nil), "io.containerd.cgroups.v1.CPUStat") -- proto.RegisterType((*CPUUsage)(nil), "io.containerd.cgroups.v1.CPUUsage") -- proto.RegisterType((*Throttle)(nil), "io.containerd.cgroups.v1.Throttle") -- proto.RegisterType((*MemoryStat)(nil), "io.containerd.cgroups.v1.MemoryStat") -- proto.RegisterType((*MemoryEntry)(nil), "io.containerd.cgroups.v1.MemoryEntry") -- proto.RegisterType((*BlkIOStat)(nil), "io.containerd.cgroups.v1.BlkIOStat") -- proto.RegisterType((*BlkIOEntry)(nil), "io.containerd.cgroups.v1.BlkIOEntry") -- proto.RegisterType((*RdmaStat)(nil), "io.containerd.cgroups.v1.RdmaStat") -- proto.RegisterType((*RdmaEntry)(nil), "io.containerd.cgroups.v1.RdmaEntry") -- proto.RegisterType((*NetworkStat)(nil), "io.containerd.cgroups.v1.NetworkStat") --} --func (m *Metrics) Marshal() (dAtA []byte, err error) { -- size := m.Size() -- dAtA = make([]byte, size) -- n, err := m.MarshalTo(dAtA) -- if err != nil { -- return nil, err -+func (m *MemoryStat) GetPgPgIn() uint64 { -+ if m != nil { -+ return m.PgPgIn - } -- return dAtA[:n], nil -+ return 0 - } - --func (m *Metrics) MarshalTo(dAtA []byte) (int, error) { -- var i int -- _ = i -- var l int -- _ = l -- if len(m.Hugetlb) > 0 { -- for _, msg := range m.Hugetlb { -- dAtA[i] = 0xa -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) -- n, err := msg.MarshalTo(dAtA[i:]) -- if err != nil { -- return 0, err -- } -- i += n -- } -- } -- if m.Pids != nil { -- dAtA[i] = 0x12 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.Pids.Size())) -- n1, err := m.Pids.MarshalTo(dAtA[i:]) -- if err != nil { -- return 0, err -- } -- i += n1 -- } -- if m.CPU != nil { -- dAtA[i] = 0x1a -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.CPU.Size())) -- n2, err := m.CPU.MarshalTo(dAtA[i:]) -- if err != nil { -- return 0, err -- } -- i += n2 -+func (m *MemoryStat) GetPgPgOut() uint64 { -+ if m != nil { -+ return m.PgPgOut - } -- if m.Memory != nil { -- dAtA[i] = 0x22 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.Memory.Size())) -- n3, err := m.Memory.MarshalTo(dAtA[i:]) -- if err != nil { -- return 0, err -- } -- i += n3 -- } -- if m.Blkio != nil { -- dAtA[i] = 0x2a -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.Blkio.Size())) -- n4, err := m.Blkio.MarshalTo(dAtA[i:]) -- if err != nil { -- return 0, err -- } -- i += n4 -- } -- if m.Rdma != nil { -- dAtA[i] = 0x32 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.Rdma.Size())) -- n5, err := m.Rdma.MarshalTo(dAtA[i:]) -- if err != nil { -- return 0, err -- } -- i += n5 -- } -- if len(m.Network) > 0 { -- for _, msg := range m.Network { -- dAtA[i] = 0x3a -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) -- n, err := msg.MarshalTo(dAtA[i:]) -- if err != nil { -- return 0, err -- } -- i += n -- } -- } -- return i, nil -+ return 0 - } - --func (m *HugetlbStat) Marshal() (dAtA []byte, err error) { -- size := m.Size() -- dAtA = make([]byte, size) -- n, err := m.MarshalTo(dAtA) -- if err != nil { -- return nil, err -+func (m *MemoryStat) GetPgFault() uint64 { -+ if m != nil { -+ return m.PgFault - } -- return dAtA[:n], nil -+ return 0 - } - --func (m *HugetlbStat) MarshalTo(dAtA []byte) (int, error) { -- var i int -- _ = i -- var l int -- _ = l -- if m.Usage != 0 { -- dAtA[i] = 0x8 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.Usage)) -- } -- if m.Max != 0 { -- dAtA[i] = 0x10 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.Max)) -- } -- if m.Failcnt != 0 { -- dAtA[i] = 0x18 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.Failcnt)) -- } -- if len(m.Pagesize) > 0 { -- dAtA[i] = 0x22 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(len(m.Pagesize))) -- i += copy(dAtA[i:], m.Pagesize) -+func (m *MemoryStat) GetPgMajFault() uint64 { -+ if m != nil { -+ return m.PgMajFault - } -- return i, nil -+ return 0 - } - --func (m *PidsStat) Marshal() (dAtA []byte, err error) { -- size := m.Size() -- dAtA = make([]byte, size) -- n, err := m.MarshalTo(dAtA) -- if err != nil { -- return nil, err -+func (m *MemoryStat) GetInactiveAnon() uint64 { -+ if m != nil { -+ return m.InactiveAnon - } -- return dAtA[:n], nil -+ return 0 - } - --func (m *PidsStat) MarshalTo(dAtA []byte) (int, error) { -- var i int -- _ = i -- var l int -- _ = l -- if m.Current != 0 { -- dAtA[i] = 0x8 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.Current)) -+func (m *MemoryStat) GetActiveAnon() uint64 { -+ if m != nil { -+ return m.ActiveAnon - } -- if m.Limit != 0 { -- dAtA[i] = 0x10 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.Limit)) -- } -- return i, nil -+ return 0 - } - --func (m *CPUStat) Marshal() (dAtA []byte, err error) { -- size := m.Size() -- dAtA = make([]byte, size) -- n, err := m.MarshalTo(dAtA) -- if err != nil { -- return nil, err -+func (m *MemoryStat) GetInactiveFile() uint64 { -+ if m != nil { -+ return m.InactiveFile - } -- return dAtA[:n], nil -+ return 0 - } - --func (m *CPUStat) MarshalTo(dAtA []byte) (int, error) { -- var i int -- _ = i -- var l int -- _ = l -- if m.Usage != nil { -- dAtA[i] = 0xa -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.Usage.Size())) -- n6, err := m.Usage.MarshalTo(dAtA[i:]) -- if err != nil { -- return 0, err -- } -- i += n6 -- } -- if m.Throttling != nil { -- dAtA[i] = 0x12 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.Throttling.Size())) -- n7, err := m.Throttling.MarshalTo(dAtA[i:]) -- if err != nil { -- return 0, err -- } -- i += n7 -+func (m *MemoryStat) GetActiveFile() uint64 { -+ if m != nil { -+ return m.ActiveFile - } -- return i, nil -+ return 0 - } - --func (m *CPUUsage) Marshal() (dAtA []byte, err error) { -- size := m.Size() -- dAtA = make([]byte, size) -- n, err := m.MarshalTo(dAtA) -- if err != nil { -- return nil, err -+func (m *MemoryStat) GetUnevictable() uint64 { -+ if m != nil { -+ return m.Unevictable - } -- return dAtA[:n], nil -+ return 0 - } - --func (m *CPUUsage) MarshalTo(dAtA []byte) (int, error) { -- var i int -- _ = i -- var l int -- _ = l -- if m.Total != 0 { -- dAtA[i] = 0x8 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.Total)) -- } -- if m.Kernel != 0 { -- dAtA[i] = 0x10 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.Kernel)) -- } -- if m.User != 0 { -- dAtA[i] = 0x18 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.User)) -+func (m *MemoryStat) GetHierarchicalMemoryLimit() uint64 { -+ if m != nil { -+ return m.HierarchicalMemoryLimit - } -- if len(m.PerCPU) > 0 { -- dAtA9 := make([]byte, len(m.PerCPU)*10) -- var j8 int -- for _, num := range m.PerCPU { -- for num >= 1<<7 { -- dAtA9[j8] = uint8(uint64(num)&0x7f | 0x80) -- num >>= 7 -- j8++ -- } -- dAtA9[j8] = uint8(num) -- j8++ -- } -- dAtA[i] = 0x22 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(j8)) -- i += copy(dAtA[i:], dAtA9[:j8]) -- } -- return i, nil -+ return 0 - } - --func (m *Throttle) Marshal() (dAtA []byte, err error) { -- size := m.Size() -- dAtA = make([]byte, size) -- n, err := m.MarshalTo(dAtA) -- if err != nil { -- return nil, err -+func (m *MemoryStat) GetHierarchicalSwapLimit() uint64 { -+ if m != nil { -+ return m.HierarchicalSwapLimit - } -- return dAtA[:n], nil -+ return 0 - } - --func (m *Throttle) MarshalTo(dAtA []byte) (int, error) { -- var i int -- _ = i -- var l int -- _ = l -- if m.Periods != 0 { -- dAtA[i] = 0x8 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.Periods)) -- } -- if m.ThrottledPeriods != 0 { -- dAtA[i] = 0x10 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.ThrottledPeriods)) -+func (m *MemoryStat) GetTotalCache() uint64 { -+ if m != nil { -+ return m.TotalCache - } -- if m.ThrottledTime != 0 { -- dAtA[i] = 0x18 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.ThrottledTime)) -- } -- return i, nil -+ return 0 - } - --func (m *MemoryStat) Marshal() (dAtA []byte, err error) { -- size := m.Size() -- dAtA = make([]byte, size) -- n, err := m.MarshalTo(dAtA) -- if err != nil { -- return nil, err -+func (m *MemoryStat) GetTotalRSS() uint64 { -+ if m != nil { -+ return m.TotalRSS - } -- return dAtA[:n], nil -+ return 0 - } - --func (m *MemoryStat) MarshalTo(dAtA []byte) (int, error) { -- var i int -- _ = i -- var l int -- _ = l -- if m.Cache != 0 { -- dAtA[i] = 0x8 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.Cache)) -- } -- if m.RSS != 0 { -- dAtA[i] = 0x10 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.RSS)) -- } -- if m.RSSHuge != 0 { -- dAtA[i] = 0x18 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.RSSHuge)) -- } -- if m.MappedFile != 0 { -- dAtA[i] = 0x20 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.MappedFile)) -- } -- if m.Dirty != 0 { -- dAtA[i] = 0x28 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.Dirty)) -- } -- if m.Writeback != 0 { -- dAtA[i] = 0x30 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.Writeback)) -- } -- if m.PgPgIn != 0 { -- dAtA[i] = 0x38 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.PgPgIn)) -- } -- if m.PgPgOut != 0 { -- dAtA[i] = 0x40 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.PgPgOut)) -- } -- if m.PgFault != 0 { -- dAtA[i] = 0x48 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.PgFault)) -- } -- if m.PgMajFault != 0 { -- dAtA[i] = 0x50 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.PgMajFault)) -- } -- if m.InactiveAnon != 0 { -- dAtA[i] = 0x58 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.InactiveAnon)) -- } -- if m.ActiveAnon != 0 { -- dAtA[i] = 0x60 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.ActiveAnon)) -- } -- if m.InactiveFile != 0 { -- dAtA[i] = 0x68 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.InactiveFile)) -- } -- if m.ActiveFile != 0 { -- dAtA[i] = 0x70 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.ActiveFile)) -- } -- if m.Unevictable != 0 { -- dAtA[i] = 0x78 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.Unevictable)) -- } -- if m.HierarchicalMemoryLimit != 0 { -- dAtA[i] = 0x80 -- i++ -- dAtA[i] = 0x1 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.HierarchicalMemoryLimit)) -- } -- if m.HierarchicalSwapLimit != 0 { -- dAtA[i] = 0x88 -- i++ -- dAtA[i] = 0x1 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.HierarchicalSwapLimit)) -- } -- if m.TotalCache != 0 { -- dAtA[i] = 0x90 -- i++ -- dAtA[i] = 0x1 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.TotalCache)) -- } -- if m.TotalRSS != 0 { -- dAtA[i] = 0x98 -- i++ -- dAtA[i] = 0x1 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.TotalRSS)) -- } -- if m.TotalRSSHuge != 0 { -- dAtA[i] = 0xa0 -- i++ -- dAtA[i] = 0x1 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.TotalRSSHuge)) -- } -- if m.TotalMappedFile != 0 { -- dAtA[i] = 0xa8 -- i++ -- dAtA[i] = 0x1 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.TotalMappedFile)) -- } -- if m.TotalDirty != 0 { -- dAtA[i] = 0xb0 -- i++ -- dAtA[i] = 0x1 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.TotalDirty)) -- } -- if m.TotalWriteback != 0 { -- dAtA[i] = 0xb8 -- i++ -- dAtA[i] = 0x1 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.TotalWriteback)) -+func (m *MemoryStat) GetTotalRSSHuge() uint64 { -+ if m != nil { -+ return m.TotalRSSHuge - } -- if m.TotalPgPgIn != 0 { -- dAtA[i] = 0xc0 -- i++ -- dAtA[i] = 0x1 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.TotalPgPgIn)) -- } -- if m.TotalPgPgOut != 0 { -- dAtA[i] = 0xc8 -- i++ -- dAtA[i] = 0x1 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.TotalPgPgOut)) -- } -- if m.TotalPgFault != 0 { -- dAtA[i] = 0xd0 -- i++ -- dAtA[i] = 0x1 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.TotalPgFault)) -- } -- if m.TotalPgMajFault != 0 { -- dAtA[i] = 0xd8 -- i++ -- dAtA[i] = 0x1 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.TotalPgMajFault)) -- } -- if m.TotalInactiveAnon != 0 { -- dAtA[i] = 0xe0 -- i++ -- dAtA[i] = 0x1 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.TotalInactiveAnon)) -- } -- if m.TotalActiveAnon != 0 { -- dAtA[i] = 0xe8 -- i++ -- dAtA[i] = 0x1 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.TotalActiveAnon)) -- } -- if m.TotalInactiveFile != 0 { -- dAtA[i] = 0xf0 -- i++ -- dAtA[i] = 0x1 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.TotalInactiveFile)) -- } -- if m.TotalActiveFile != 0 { -- dAtA[i] = 0xf8 -- i++ -- dAtA[i] = 0x1 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.TotalActiveFile)) -- } -- if m.TotalUnevictable != 0 { -- dAtA[i] = 0x80 -- i++ -- dAtA[i] = 0x2 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.TotalUnevictable)) -- } -- if m.Usage != nil { -- dAtA[i] = 0x8a -- i++ -- dAtA[i] = 0x2 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.Usage.Size())) -- n10, err := m.Usage.MarshalTo(dAtA[i:]) -- if err != nil { -- return 0, err -- } -- i += n10 -- } -- if m.Swap != nil { -- dAtA[i] = 0x92 -- i++ -- dAtA[i] = 0x2 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.Swap.Size())) -- n11, err := m.Swap.MarshalTo(dAtA[i:]) -- if err != nil { -- return 0, err -- } -- i += n11 -- } -- if m.Kernel != nil { -- dAtA[i] = 0x9a -- i++ -- dAtA[i] = 0x2 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.Kernel.Size())) -- n12, err := m.Kernel.MarshalTo(dAtA[i:]) -- if err != nil { -- return 0, err -- } -- i += n12 -- } -- if m.KernelTCP != nil { -- dAtA[i] = 0xa2 -- i++ -- dAtA[i] = 0x2 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.KernelTCP.Size())) -- n13, err := m.KernelTCP.MarshalTo(dAtA[i:]) -- if err != nil { -- return 0, err -- } -- i += n13 -- } -- return i, nil -+ return 0 - } - --func (m *MemoryEntry) Marshal() (dAtA []byte, err error) { -- size := m.Size() -- dAtA = make([]byte, size) -- n, err := m.MarshalTo(dAtA) -- if err != nil { -- return nil, err -+func (m *MemoryStat) GetTotalMappedFile() uint64 { -+ if m != nil { -+ return m.TotalMappedFile - } -- return dAtA[:n], nil -+ return 0 - } - --func (m *MemoryEntry) MarshalTo(dAtA []byte) (int, error) { -- var i int -- _ = i -- var l int -- _ = l -- if m.Limit != 0 { -- dAtA[i] = 0x8 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.Limit)) -- } -- if m.Usage != 0 { -- dAtA[i] = 0x10 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.Usage)) -+func (m *MemoryStat) GetTotalDirty() uint64 { -+ if m != nil { -+ return m.TotalDirty - } -- if m.Max != 0 { -- dAtA[i] = 0x18 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.Max)) -- } -- if m.Failcnt != 0 { -- dAtA[i] = 0x20 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.Failcnt)) -- } -- return i, nil -+ return 0 - } - --func (m *BlkIOStat) Marshal() (dAtA []byte, err error) { -- size := m.Size() -- dAtA = make([]byte, size) -- n, err := m.MarshalTo(dAtA) -- if err != nil { -- return nil, err -+func (m *MemoryStat) GetTotalWriteback() uint64 { -+ if m != nil { -+ return m.TotalWriteback - } -- return dAtA[:n], nil -+ return 0 - } - --func (m *BlkIOStat) MarshalTo(dAtA []byte) (int, error) { -- var i int -- _ = i -- var l int -- _ = l -- if len(m.IoServiceBytesRecursive) > 0 { -- for _, msg := range m.IoServiceBytesRecursive { -- dAtA[i] = 0xa -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) -- n, err := msg.MarshalTo(dAtA[i:]) -- if err != nil { -- return 0, err -- } -- i += n -- } -- } -- if len(m.IoServicedRecursive) > 0 { -- for _, msg := range m.IoServicedRecursive { -- dAtA[i] = 0x12 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) -- n, err := msg.MarshalTo(dAtA[i:]) -- if err != nil { -- return 0, err -- } -- i += n -- } -- } -- if len(m.IoQueuedRecursive) > 0 { -- for _, msg := range m.IoQueuedRecursive { -- dAtA[i] = 0x1a -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) -- n, err := msg.MarshalTo(dAtA[i:]) -- if err != nil { -- return 0, err -- } -- i += n -- } -- } -- if len(m.IoServiceTimeRecursive) > 0 { -- for _, msg := range m.IoServiceTimeRecursive { -- dAtA[i] = 0x22 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) -- n, err := msg.MarshalTo(dAtA[i:]) -- if err != nil { -- return 0, err -- } -- i += n -- } -- } -- if len(m.IoWaitTimeRecursive) > 0 { -- for _, msg := range m.IoWaitTimeRecursive { -- dAtA[i] = 0x2a -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) -- n, err := msg.MarshalTo(dAtA[i:]) -- if err != nil { -- return 0, err -- } -- i += n -- } -- } -- if len(m.IoMergedRecursive) > 0 { -- for _, msg := range m.IoMergedRecursive { -- dAtA[i] = 0x32 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) -- n, err := msg.MarshalTo(dAtA[i:]) -- if err != nil { -- return 0, err -- } -- i += n -- } -+func (m *MemoryStat) GetTotalPgPgIn() uint64 { -+ if m != nil { -+ return m.TotalPgPgIn - } -- if len(m.IoTimeRecursive) > 0 { -- for _, msg := range m.IoTimeRecursive { -- dAtA[i] = 0x3a -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) -- n, err := msg.MarshalTo(dAtA[i:]) -- if err != nil { -- return 0, err -- } -- i += n -- } -- } -- if len(m.SectorsRecursive) > 0 { -- for _, msg := range m.SectorsRecursive { -- dAtA[i] = 0x42 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) -- n, err := msg.MarshalTo(dAtA[i:]) -- if err != nil { -- return 0, err -- } -- i += n -- } -- } -- return i, nil -+ return 0 - } - --func (m *BlkIOEntry) Marshal() (dAtA []byte, err error) { -- size := m.Size() -- dAtA = make([]byte, size) -- n, err := m.MarshalTo(dAtA) -- if err != nil { -- return nil, err -+func (m *MemoryStat) GetTotalPgPgOut() uint64 { -+ if m != nil { -+ return m.TotalPgPgOut - } -- return dAtA[:n], nil -+ return 0 - } - --func (m *BlkIOEntry) MarshalTo(dAtA []byte) (int, error) { -- var i int -- _ = i -- var l int -- _ = l -- if len(m.Op) > 0 { -- dAtA[i] = 0xa -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(len(m.Op))) -- i += copy(dAtA[i:], m.Op) -- } -- if len(m.Device) > 0 { -- dAtA[i] = 0x12 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(len(m.Device))) -- i += copy(dAtA[i:], m.Device) -- } -- if m.Major != 0 { -- dAtA[i] = 0x18 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.Major)) -+func (m *MemoryStat) GetTotalPgFault() uint64 { -+ if m != nil { -+ return m.TotalPgFault - } -- if m.Minor != 0 { -- dAtA[i] = 0x20 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.Minor)) -- } -- if m.Value != 0 { -- dAtA[i] = 0x28 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.Value)) -- } -- return i, nil -+ return 0 - } - --func (m *RdmaStat) Marshal() (dAtA []byte, err error) { -- size := m.Size() -- dAtA = make([]byte, size) -- n, err := m.MarshalTo(dAtA) -- if err != nil { -- return nil, err -+func (m *MemoryStat) GetTotalPgMajFault() uint64 { -+ if m != nil { -+ return m.TotalPgMajFault - } -- return dAtA[:n], nil -+ return 0 - } - --func (m *RdmaStat) MarshalTo(dAtA []byte) (int, error) { -- var i int -- _ = i -- var l int -- _ = l -- if len(m.Current) > 0 { -- for _, msg := range m.Current { -- dAtA[i] = 0xa -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) -- n, err := msg.MarshalTo(dAtA[i:]) -- if err != nil { -- return 0, err -- } -- i += n -- } -- } -- if len(m.Limit) > 0 { -- for _, msg := range m.Limit { -- dAtA[i] = 0x12 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) -- n, err := msg.MarshalTo(dAtA[i:]) -- if err != nil { -- return 0, err -- } -- i += n -- } -+func (m *MemoryStat) GetTotalInactiveAnon() uint64 { -+ if m != nil { -+ return m.TotalInactiveAnon - } -- return i, nil -+ return 0 - } - --func (m *RdmaEntry) Marshal() (dAtA []byte, err error) { -- size := m.Size() -- dAtA = make([]byte, size) -- n, err := m.MarshalTo(dAtA) -- if err != nil { -- return nil, err -+func (m *MemoryStat) GetTotalActiveAnon() uint64 { -+ if m != nil { -+ return m.TotalActiveAnon - } -- return dAtA[:n], nil -+ return 0 - } - --func (m *RdmaEntry) MarshalTo(dAtA []byte) (int, error) { -- var i int -- _ = i -- var l int -- _ = l -- if len(m.Device) > 0 { -- dAtA[i] = 0xa -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(len(m.Device))) -- i += copy(dAtA[i:], m.Device) -+func (m *MemoryStat) GetTotalInactiveFile() uint64 { -+ if m != nil { -+ return m.TotalInactiveFile - } -- if m.HcaHandles != 0 { -- dAtA[i] = 0x10 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.HcaHandles)) -- } -- if m.HcaObjects != 0 { -- dAtA[i] = 0x18 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.HcaObjects)) -- } -- return i, nil -+ return 0 - } - --func (m *NetworkStat) Marshal() (dAtA []byte, err error) { -- size := m.Size() -- dAtA = make([]byte, size) -- n, err := m.MarshalTo(dAtA) -- if err != nil { -- return nil, err -+func (m *MemoryStat) GetTotalActiveFile() uint64 { -+ if m != nil { -+ return m.TotalActiveFile - } -- return dAtA[:n], nil -+ return 0 - } - --func (m *NetworkStat) MarshalTo(dAtA []byte) (int, error) { -- var i int -- _ = i -- var l int -- _ = l -- if len(m.Name) > 0 { -- dAtA[i] = 0xa -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(len(m.Name))) -- i += copy(dAtA[i:], m.Name) -- } -- if m.RxBytes != 0 { -- dAtA[i] = 0x10 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.RxBytes)) -- } -- if m.RxPackets != 0 { -- dAtA[i] = 0x18 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.RxPackets)) -- } -- if m.RxErrors != 0 { -- dAtA[i] = 0x20 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.RxErrors)) -- } -- if m.RxDropped != 0 { -- dAtA[i] = 0x28 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.RxDropped)) -- } -- if m.TxBytes != 0 { -- dAtA[i] = 0x30 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.TxBytes)) -- } -- if m.TxPackets != 0 { -- dAtA[i] = 0x38 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.TxPackets)) -+func (m *MemoryStat) GetTotalUnevictable() uint64 { -+ if m != nil { -+ return m.TotalUnevictable - } -- if m.TxErrors != 0 { -- dAtA[i] = 0x40 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.TxErrors)) -- } -- if m.TxDropped != 0 { -- dAtA[i] = 0x48 -- i++ -- i = encodeVarintMetrics(dAtA, i, uint64(m.TxDropped)) -- } -- return i, nil -+ return 0 - } - --func encodeVarintMetrics(dAtA []byte, offset int, v uint64) int { -- for v >= 1<<7 { -- dAtA[offset] = uint8(v&0x7f | 0x80) -- v >>= 7 -- offset++ -+func (m *MemoryStat) GetUsage() *MemoryEntry { -+ if m != nil { -+ return m.Usage - } -- dAtA[offset] = uint8(v) -- return offset + 1 -+ return nil - } --func (m *Metrics) Size() (n int) { -- var l int -- _ = l -- if len(m.Hugetlb) > 0 { -- for _, e := range m.Hugetlb { -- l = e.Size() -- n += 1 + l + sovMetrics(uint64(l)) -- } -- } -- if m.Pids != nil { -- l = m.Pids.Size() -- n += 1 + l + sovMetrics(uint64(l)) -- } -- if m.CPU != nil { -- l = m.CPU.Size() -- n += 1 + l + sovMetrics(uint64(l)) -- } -- if m.Memory != nil { -- l = m.Memory.Size() -- n += 1 + l + sovMetrics(uint64(l)) -- } -- if m.Blkio != nil { -- l = m.Blkio.Size() -- n += 1 + l + sovMetrics(uint64(l)) -- } -- if m.Rdma != nil { -- l = m.Rdma.Size() -- n += 1 + l + sovMetrics(uint64(l)) -- } -- if len(m.Network) > 0 { -- for _, e := range m.Network { -- l = e.Size() -- n += 1 + l + sovMetrics(uint64(l)) -- } -+ -+func (m *MemoryStat) GetSwap() *MemoryEntry { -+ if m != nil { -+ return m.Swap - } -- return n -+ return nil - } - --func (m *HugetlbStat) Size() (n int) { -- var l int -- _ = l -- if m.Usage != 0 { -- n += 1 + sovMetrics(uint64(m.Usage)) -- } -- if m.Max != 0 { -- n += 1 + sovMetrics(uint64(m.Max)) -- } -- if m.Failcnt != 0 { -- n += 1 + sovMetrics(uint64(m.Failcnt)) -- } -- l = len(m.Pagesize) -- if l > 0 { -- n += 1 + l + sovMetrics(uint64(l)) -+func (m *MemoryStat) GetKernel() *MemoryEntry { -+ if m != nil { -+ return m.Kernel - } -- return n -+ return nil - } - --func (m *PidsStat) Size() (n int) { -- var l int -- _ = l -- if m.Current != 0 { -- n += 1 + sovMetrics(uint64(m.Current)) -- } -- if m.Limit != 0 { -- n += 1 + sovMetrics(uint64(m.Limit)) -+func (m *MemoryStat) GetKernelTCP() *MemoryEntry { -+ if m != nil { -+ return m.KernelTCP - } -- return n -+ return nil - } - --func (m *CPUStat) Size() (n int) { -- var l int -- _ = l -- if m.Usage != nil { -- l = m.Usage.Size() -- n += 1 + l + sovMetrics(uint64(l)) -- } -- if m.Throttling != nil { -- l = m.Throttling.Size() -- n += 1 + l + sovMetrics(uint64(l)) -- } -- return n -+type MemoryEntry struct { -+ Limit uint64 `protobuf:"varint,1,opt,name=limit,proto3" json:"limit,omitempty"` -+ Usage uint64 `protobuf:"varint,2,opt,name=usage,proto3" json:"usage,omitempty"` -+ Max uint64 `protobuf:"varint,3,opt,name=max,proto3" json:"max,omitempty"` -+ Failcnt uint64 `protobuf:"varint,4,opt,name=failcnt,proto3" json:"failcnt,omitempty"` - } - --func (m *CPUUsage) Size() (n int) { -- var l int -- _ = l -- if m.Total != 0 { -- n += 1 + sovMetrics(uint64(m.Total)) -- } -- if m.Kernel != 0 { -- n += 1 + sovMetrics(uint64(m.Kernel)) -- } -- if m.User != 0 { -- n += 1 + sovMetrics(uint64(m.User)) -- } -- if len(m.PerCPU) > 0 { -- l = 0 -- for _, e := range m.PerCPU { -- l += sovMetrics(uint64(e)) -- } -- n += 1 + sovMetrics(uint64(l)) + l -+func (m *MemoryEntry) Reset() { *m = MemoryEntry{} } -+func (m *MemoryEntry) String() string { return proto.CompactTextString(m) } -+func (*MemoryEntry) ProtoMessage() {} -+func (*MemoryEntry) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{7} } -+ -+func (m *MemoryEntry) GetLimit() uint64 { -+ if m != nil { -+ return m.Limit - } -- return n -+ return 0 - } - --func (m *Throttle) Size() (n int) { -- var l int -- _ = l -- if m.Periods != 0 { -- n += 1 + sovMetrics(uint64(m.Periods)) -- } -- if m.ThrottledPeriods != 0 { -- n += 1 + sovMetrics(uint64(m.ThrottledPeriods)) -- } -- if m.ThrottledTime != 0 { -- n += 1 + sovMetrics(uint64(m.ThrottledTime)) -+func (m *MemoryEntry) GetUsage() uint64 { -+ if m != nil { -+ return m.Usage - } -- return n -+ return 0 - } - --func (m *MemoryStat) Size() (n int) { -- var l int -- _ = l -- if m.Cache != 0 { -- n += 1 + sovMetrics(uint64(m.Cache)) -- } -- if m.RSS != 0 { -- n += 1 + sovMetrics(uint64(m.RSS)) -- } -- if m.RSSHuge != 0 { -- n += 1 + sovMetrics(uint64(m.RSSHuge)) -- } -- if m.MappedFile != 0 { -- n += 1 + sovMetrics(uint64(m.MappedFile)) -- } -- if m.Dirty != 0 { -- n += 1 + sovMetrics(uint64(m.Dirty)) -- } -- if m.Writeback != 0 { -- n += 1 + sovMetrics(uint64(m.Writeback)) -- } -- if m.PgPgIn != 0 { -- n += 1 + sovMetrics(uint64(m.PgPgIn)) -- } -- if m.PgPgOut != 0 { -- n += 1 + sovMetrics(uint64(m.PgPgOut)) -- } -- if m.PgFault != 0 { -- n += 1 + sovMetrics(uint64(m.PgFault)) -- } -- if m.PgMajFault != 0 { -- n += 1 + sovMetrics(uint64(m.PgMajFault)) -- } -- if m.InactiveAnon != 0 { -- n += 1 + sovMetrics(uint64(m.InactiveAnon)) -- } -- if m.ActiveAnon != 0 { -- n += 1 + sovMetrics(uint64(m.ActiveAnon)) -- } -- if m.InactiveFile != 0 { -- n += 1 + sovMetrics(uint64(m.InactiveFile)) -- } -- if m.ActiveFile != 0 { -- n += 1 + sovMetrics(uint64(m.ActiveFile)) -- } -- if m.Unevictable != 0 { -- n += 1 + sovMetrics(uint64(m.Unevictable)) -- } -- if m.HierarchicalMemoryLimit != 0 { -- n += 2 + sovMetrics(uint64(m.HierarchicalMemoryLimit)) -- } -- if m.HierarchicalSwapLimit != 0 { -- n += 2 + sovMetrics(uint64(m.HierarchicalSwapLimit)) -- } -- if m.TotalCache != 0 { -- n += 2 + sovMetrics(uint64(m.TotalCache)) -+func (m *MemoryEntry) GetMax() uint64 { -+ if m != nil { -+ return m.Max - } -- if m.TotalRSS != 0 { -- n += 2 + sovMetrics(uint64(m.TotalRSS)) -- } -- if m.TotalRSSHuge != 0 { -- n += 2 + sovMetrics(uint64(m.TotalRSSHuge)) -- } -- if m.TotalMappedFile != 0 { -- n += 2 + sovMetrics(uint64(m.TotalMappedFile)) -- } -- if m.TotalDirty != 0 { -- n += 2 + sovMetrics(uint64(m.TotalDirty)) -- } -- if m.TotalWriteback != 0 { -- n += 2 + sovMetrics(uint64(m.TotalWriteback)) -- } -- if m.TotalPgPgIn != 0 { -- n += 2 + sovMetrics(uint64(m.TotalPgPgIn)) -- } -- if m.TotalPgPgOut != 0 { -- n += 2 + sovMetrics(uint64(m.TotalPgPgOut)) -- } -- if m.TotalPgFault != 0 { -- n += 2 + sovMetrics(uint64(m.TotalPgFault)) -- } -- if m.TotalPgMajFault != 0 { -- n += 2 + sovMetrics(uint64(m.TotalPgMajFault)) -- } -- if m.TotalInactiveAnon != 0 { -- n += 2 + sovMetrics(uint64(m.TotalInactiveAnon)) -- } -- if m.TotalActiveAnon != 0 { -- n += 2 + sovMetrics(uint64(m.TotalActiveAnon)) -- } -- if m.TotalInactiveFile != 0 { -- n += 2 + sovMetrics(uint64(m.TotalInactiveFile)) -- } -- if m.TotalActiveFile != 0 { -- n += 2 + sovMetrics(uint64(m.TotalActiveFile)) -- } -- if m.TotalUnevictable != 0 { -- n += 2 + sovMetrics(uint64(m.TotalUnevictable)) -- } -- if m.Usage != nil { -- l = m.Usage.Size() -- n += 2 + l + sovMetrics(uint64(l)) -- } -- if m.Swap != nil { -- l = m.Swap.Size() -- n += 2 + l + sovMetrics(uint64(l)) -- } -- if m.Kernel != nil { -- l = m.Kernel.Size() -- n += 2 + l + sovMetrics(uint64(l)) -- } -- if m.KernelTCP != nil { -- l = m.KernelTCP.Size() -- n += 2 + l + sovMetrics(uint64(l)) -- } -- return n -+ return 0 - } - --func (m *MemoryEntry) Size() (n int) { -- var l int -- _ = l -- if m.Limit != 0 { -- n += 1 + sovMetrics(uint64(m.Limit)) -- } -- if m.Usage != 0 { -- n += 1 + sovMetrics(uint64(m.Usage)) -+func (m *MemoryEntry) GetFailcnt() uint64 { -+ if m != nil { -+ return m.Failcnt - } -- if m.Max != 0 { -- n += 1 + sovMetrics(uint64(m.Max)) -- } -- if m.Failcnt != 0 { -- n += 1 + sovMetrics(uint64(m.Failcnt)) -- } -- return n -+ return 0 - } - --func (m *BlkIOStat) Size() (n int) { -- var l int -- _ = l -- if len(m.IoServiceBytesRecursive) > 0 { -- for _, e := range m.IoServiceBytesRecursive { -- l = e.Size() -- n += 1 + l + sovMetrics(uint64(l)) -- } -- } -- if len(m.IoServicedRecursive) > 0 { -- for _, e := range m.IoServicedRecursive { -- l = e.Size() -- n += 1 + l + sovMetrics(uint64(l)) -- } -- } -- if len(m.IoQueuedRecursive) > 0 { -- for _, e := range m.IoQueuedRecursive { -- l = e.Size() -- n += 1 + l + sovMetrics(uint64(l)) -- } -- } -- if len(m.IoServiceTimeRecursive) > 0 { -- for _, e := range m.IoServiceTimeRecursive { -- l = e.Size() -- n += 1 + l + sovMetrics(uint64(l)) -- } -- } -- if len(m.IoWaitTimeRecursive) > 0 { -- for _, e := range m.IoWaitTimeRecursive { -- l = e.Size() -- n += 1 + l + sovMetrics(uint64(l)) -- } -- } -- if len(m.IoMergedRecursive) > 0 { -- for _, e := range m.IoMergedRecursive { -- l = e.Size() -- n += 1 + l + sovMetrics(uint64(l)) -- } -- } -- if len(m.IoTimeRecursive) > 0 { -- for _, e := range m.IoTimeRecursive { -- l = e.Size() -- n += 1 + l + sovMetrics(uint64(l)) -- } -- } -- if len(m.SectorsRecursive) > 0 { -- for _, e := range m.SectorsRecursive { -- l = e.Size() -- n += 1 + l + sovMetrics(uint64(l)) -- } -- } -- return n -+type BlkIOStat struct { -+ IoServiceBytesRecursive []*BlkIOEntry `protobuf:"bytes,1,rep,name=io_service_bytes_recursive,json=ioServiceBytesRecursive" json:"io_service_bytes_recursive,omitempty"` -+ IoServicedRecursive []*BlkIOEntry `protobuf:"bytes,2,rep,name=io_serviced_recursive,json=ioServicedRecursive" json:"io_serviced_recursive,omitempty"` -+ IoQueuedRecursive []*BlkIOEntry `protobuf:"bytes,3,rep,name=io_queued_recursive,json=ioQueuedRecursive" json:"io_queued_recursive,omitempty"` -+ IoServiceTimeRecursive []*BlkIOEntry `protobuf:"bytes,4,rep,name=io_service_time_recursive,json=ioServiceTimeRecursive" json:"io_service_time_recursive,omitempty"` -+ IoWaitTimeRecursive []*BlkIOEntry `protobuf:"bytes,5,rep,name=io_wait_time_recursive,json=ioWaitTimeRecursive" json:"io_wait_time_recursive,omitempty"` -+ IoMergedRecursive []*BlkIOEntry `protobuf:"bytes,6,rep,name=io_merged_recursive,json=ioMergedRecursive" json:"io_merged_recursive,omitempty"` -+ IoTimeRecursive []*BlkIOEntry `protobuf:"bytes,7,rep,name=io_time_recursive,json=ioTimeRecursive" json:"io_time_recursive,omitempty"` -+ SectorsRecursive []*BlkIOEntry `protobuf:"bytes,8,rep,name=sectors_recursive,json=sectorsRecursive" json:"sectors_recursive,omitempty"` - } - --func (m *BlkIOEntry) Size() (n int) { -- var l int -- _ = l -- l = len(m.Op) -- if l > 0 { -- n += 1 + l + sovMetrics(uint64(l)) -- } -- l = len(m.Device) -- if l > 0 { -- n += 1 + l + sovMetrics(uint64(l)) -- } -- if m.Major != 0 { -- n += 1 + sovMetrics(uint64(m.Major)) -- } -- if m.Minor != 0 { -- n += 1 + sovMetrics(uint64(m.Minor)) -- } -- if m.Value != 0 { -- n += 1 + sovMetrics(uint64(m.Value)) -+func (m *BlkIOStat) Reset() { *m = BlkIOStat{} } -+func (m *BlkIOStat) String() string { return proto.CompactTextString(m) } -+func (*BlkIOStat) ProtoMessage() {} -+func (*BlkIOStat) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{8} } -+ -+func (m *BlkIOStat) GetIoServiceBytesRecursive() []*BlkIOEntry { -+ if m != nil { -+ return m.IoServiceBytesRecursive - } -- return n -+ return nil - } - --func (m *RdmaStat) Size() (n int) { -- var l int -- _ = l -- if len(m.Current) > 0 { -- for _, e := range m.Current { -- l = e.Size() -- n += 1 + l + sovMetrics(uint64(l)) -- } -+func (m *BlkIOStat) GetIoServicedRecursive() []*BlkIOEntry { -+ if m != nil { -+ return m.IoServicedRecursive - } -- if len(m.Limit) > 0 { -- for _, e := range m.Limit { -- l = e.Size() -- n += 1 + l + sovMetrics(uint64(l)) -- } -- } -- return n -+ return nil - } - --func (m *RdmaEntry) Size() (n int) { -- var l int -- _ = l -- l = len(m.Device) -- if l > 0 { -- n += 1 + l + sovMetrics(uint64(l)) -- } -- if m.HcaHandles != 0 { -- n += 1 + sovMetrics(uint64(m.HcaHandles)) -+func (m *BlkIOStat) GetIoQueuedRecursive() []*BlkIOEntry { -+ if m != nil { -+ return m.IoQueuedRecursive - } -- if m.HcaObjects != 0 { -- n += 1 + sovMetrics(uint64(m.HcaObjects)) -- } -- return n -+ return nil - } - --func (m *NetworkStat) Size() (n int) { -- var l int -- _ = l -- l = len(m.Name) -- if l > 0 { -- n += 1 + l + sovMetrics(uint64(l)) -- } -- if m.RxBytes != 0 { -- n += 1 + sovMetrics(uint64(m.RxBytes)) -- } -- if m.RxPackets != 0 { -- n += 1 + sovMetrics(uint64(m.RxPackets)) -- } -- if m.RxErrors != 0 { -- n += 1 + sovMetrics(uint64(m.RxErrors)) -- } -- if m.RxDropped != 0 { -- n += 1 + sovMetrics(uint64(m.RxDropped)) -- } -- if m.TxBytes != 0 { -- n += 1 + sovMetrics(uint64(m.TxBytes)) -+func (m *BlkIOStat) GetIoServiceTimeRecursive() []*BlkIOEntry { -+ if m != nil { -+ return m.IoServiceTimeRecursive - } -- if m.TxPackets != 0 { -- n += 1 + sovMetrics(uint64(m.TxPackets)) -- } -- if m.TxErrors != 0 { -- n += 1 + sovMetrics(uint64(m.TxErrors)) -- } -- if m.TxDropped != 0 { -- n += 1 + sovMetrics(uint64(m.TxDropped)) -- } -- return n -+ return nil - } - --func sovMetrics(x uint64) (n int) { -- for { -- n++ -- x >>= 7 -- if x == 0 { -- break -- } -+func (m *BlkIOStat) GetIoWaitTimeRecursive() []*BlkIOEntry { -+ if m != nil { -+ return m.IoWaitTimeRecursive - } -- return n --} --func sozMetrics(x uint64) (n int) { -- return sovMetrics(uint64((x << 1) ^ uint64((int64(x) >> 63)))) -+ return nil - } --func (this *Metrics) String() string { -- if this == nil { -- return "nil" -+ -+func (m *BlkIOStat) GetIoMergedRecursive() []*BlkIOEntry { -+ if m != nil { -+ return m.IoMergedRecursive - } -- s := strings.Join([]string{`&Metrics{`, -- `Hugetlb:` + strings.Replace(fmt.Sprintf("%v", this.Hugetlb), "HugetlbStat", "HugetlbStat", 1) + `,`, -- `Pids:` + strings.Replace(fmt.Sprintf("%v", this.Pids), "PidsStat", "PidsStat", 1) + `,`, -- `CPU:` + strings.Replace(fmt.Sprintf("%v", this.CPU), "CPUStat", "CPUStat", 1) + `,`, -- `Memory:` + strings.Replace(fmt.Sprintf("%v", this.Memory), "MemoryStat", "MemoryStat", 1) + `,`, -- `Blkio:` + strings.Replace(fmt.Sprintf("%v", this.Blkio), "BlkIOStat", "BlkIOStat", 1) + `,`, -- `Rdma:` + strings.Replace(fmt.Sprintf("%v", this.Rdma), "RdmaStat", "RdmaStat", 1) + `,`, -- `Network:` + strings.Replace(fmt.Sprintf("%v", this.Network), "NetworkStat", "NetworkStat", 1) + `,`, -- `}`, -- }, "") -- return s -+ return nil - } --func (this *HugetlbStat) String() string { -- if this == nil { -- return "nil" -+ -+func (m *BlkIOStat) GetIoTimeRecursive() []*BlkIOEntry { -+ if m != nil { -+ return m.IoTimeRecursive - } -- s := strings.Join([]string{`&HugetlbStat{`, -- `Usage:` + fmt.Sprintf("%v", this.Usage) + `,`, -- `Max:` + fmt.Sprintf("%v", this.Max) + `,`, -- `Failcnt:` + fmt.Sprintf("%v", this.Failcnt) + `,`, -- `Pagesize:` + fmt.Sprintf("%v", this.Pagesize) + `,`, -- `}`, -- }, "") -- return s -+ return nil - } --func (this *PidsStat) String() string { -- if this == nil { -- return "nil" -+ -+func (m *BlkIOStat) GetSectorsRecursive() []*BlkIOEntry { -+ if m != nil { -+ return m.SectorsRecursive - } -- s := strings.Join([]string{`&PidsStat{`, -- `Current:` + fmt.Sprintf("%v", this.Current) + `,`, -- `Limit:` + fmt.Sprintf("%v", this.Limit) + `,`, -- `}`, -- }, "") -- return s -+ return nil - } --func (this *CPUStat) String() string { -- if this == nil { -- return "nil" -- } -- s := strings.Join([]string{`&CPUStat{`, -- `Usage:` + strings.Replace(fmt.Sprintf("%v", this.Usage), "CPUUsage", "CPUUsage", 1) + `,`, -- `Throttling:` + strings.Replace(fmt.Sprintf("%v", this.Throttling), "Throttle", "Throttle", 1) + `,`, -- `}`, -- }, "") -- return s -+ -+type BlkIOEntry struct { -+ Op string `protobuf:"bytes,1,opt,name=op,proto3" json:"op,omitempty"` -+ Device string `protobuf:"bytes,2,opt,name=device,proto3" json:"device,omitempty"` -+ Major uint64 `protobuf:"varint,3,opt,name=major,proto3" json:"major,omitempty"` -+ Minor uint64 `protobuf:"varint,4,opt,name=minor,proto3" json:"minor,omitempty"` -+ Value uint64 `protobuf:"varint,5,opt,name=value,proto3" json:"value,omitempty"` - } --func (this *CPUUsage) String() string { -- if this == nil { -- return "nil" -+ -+func (m *BlkIOEntry) Reset() { *m = BlkIOEntry{} } -+func (m *BlkIOEntry) String() string { return proto.CompactTextString(m) } -+func (*BlkIOEntry) ProtoMessage() {} -+func (*BlkIOEntry) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{9} } -+ -+func (m *BlkIOEntry) GetOp() string { -+ if m != nil { -+ return m.Op - } -- s := strings.Join([]string{`&CPUUsage{`, -- `Total:` + fmt.Sprintf("%v", this.Total) + `,`, -- `Kernel:` + fmt.Sprintf("%v", this.Kernel) + `,`, -- `User:` + fmt.Sprintf("%v", this.User) + `,`, -- `PerCPU:` + fmt.Sprintf("%v", this.PerCPU) + `,`, -- `}`, -- }, "") -- return s -+ return "" - } --func (this *Throttle) String() string { -- if this == nil { -- return "nil" -+ -+func (m *BlkIOEntry) GetDevice() string { -+ if m != nil { -+ return m.Device - } -- s := strings.Join([]string{`&Throttle{`, -- `Periods:` + fmt.Sprintf("%v", this.Periods) + `,`, -- `ThrottledPeriods:` + fmt.Sprintf("%v", this.ThrottledPeriods) + `,`, -- `ThrottledTime:` + fmt.Sprintf("%v", this.ThrottledTime) + `,`, -- `}`, -- }, "") -- return s -+ return "" - } --func (this *MemoryStat) String() string { -- if this == nil { -- return "nil" -+ -+func (m *BlkIOEntry) GetMajor() uint64 { -+ if m != nil { -+ return m.Major - } -- s := strings.Join([]string{`&MemoryStat{`, -- `Cache:` + fmt.Sprintf("%v", this.Cache) + `,`, -- `RSS:` + fmt.Sprintf("%v", this.RSS) + `,`, -- `RSSHuge:` + fmt.Sprintf("%v", this.RSSHuge) + `,`, -- `MappedFile:` + fmt.Sprintf("%v", this.MappedFile) + `,`, -- `Dirty:` + fmt.Sprintf("%v", this.Dirty) + `,`, -- `Writeback:` + fmt.Sprintf("%v", this.Writeback) + `,`, -- `PgPgIn:` + fmt.Sprintf("%v", this.PgPgIn) + `,`, -- `PgPgOut:` + fmt.Sprintf("%v", this.PgPgOut) + `,`, -- `PgFault:` + fmt.Sprintf("%v", this.PgFault) + `,`, -- `PgMajFault:` + fmt.Sprintf("%v", this.PgMajFault) + `,`, -- `InactiveAnon:` + fmt.Sprintf("%v", this.InactiveAnon) + `,`, -- `ActiveAnon:` + fmt.Sprintf("%v", this.ActiveAnon) + `,`, -- `InactiveFile:` + fmt.Sprintf("%v", this.InactiveFile) + `,`, -- `ActiveFile:` + fmt.Sprintf("%v", this.ActiveFile) + `,`, -- `Unevictable:` + fmt.Sprintf("%v", this.Unevictable) + `,`, -- `HierarchicalMemoryLimit:` + fmt.Sprintf("%v", this.HierarchicalMemoryLimit) + `,`, -- `HierarchicalSwapLimit:` + fmt.Sprintf("%v", this.HierarchicalSwapLimit) + `,`, -- `TotalCache:` + fmt.Sprintf("%v", this.TotalCache) + `,`, -- `TotalRSS:` + fmt.Sprintf("%v", this.TotalRSS) + `,`, -- `TotalRSSHuge:` + fmt.Sprintf("%v", this.TotalRSSHuge) + `,`, -- `TotalMappedFile:` + fmt.Sprintf("%v", this.TotalMappedFile) + `,`, -- `TotalDirty:` + fmt.Sprintf("%v", this.TotalDirty) + `,`, -- `TotalWriteback:` + fmt.Sprintf("%v", this.TotalWriteback) + `,`, -- `TotalPgPgIn:` + fmt.Sprintf("%v", this.TotalPgPgIn) + `,`, -- `TotalPgPgOut:` + fmt.Sprintf("%v", this.TotalPgPgOut) + `,`, -- `TotalPgFault:` + fmt.Sprintf("%v", this.TotalPgFault) + `,`, -- `TotalPgMajFault:` + fmt.Sprintf("%v", this.TotalPgMajFault) + `,`, -- `TotalInactiveAnon:` + fmt.Sprintf("%v", this.TotalInactiveAnon) + `,`, -- `TotalActiveAnon:` + fmt.Sprintf("%v", this.TotalActiveAnon) + `,`, -- `TotalInactiveFile:` + fmt.Sprintf("%v", this.TotalInactiveFile) + `,`, -- `TotalActiveFile:` + fmt.Sprintf("%v", this.TotalActiveFile) + `,`, -- `TotalUnevictable:` + fmt.Sprintf("%v", this.TotalUnevictable) + `,`, -- `Usage:` + strings.Replace(fmt.Sprintf("%v", this.Usage), "MemoryEntry", "MemoryEntry", 1) + `,`, -- `Swap:` + strings.Replace(fmt.Sprintf("%v", this.Swap), "MemoryEntry", "MemoryEntry", 1) + `,`, -- `Kernel:` + strings.Replace(fmt.Sprintf("%v", this.Kernel), "MemoryEntry", "MemoryEntry", 1) + `,`, -- `KernelTCP:` + strings.Replace(fmt.Sprintf("%v", this.KernelTCP), "MemoryEntry", "MemoryEntry", 1) + `,`, -- `}`, -- }, "") -- return s -+ return 0 - } --func (this *MemoryEntry) String() string { -- if this == nil { -- return "nil" -+ -+func (m *BlkIOEntry) GetMinor() uint64 { -+ if m != nil { -+ return m.Minor - } -- s := strings.Join([]string{`&MemoryEntry{`, -- `Limit:` + fmt.Sprintf("%v", this.Limit) + `,`, -- `Usage:` + fmt.Sprintf("%v", this.Usage) + `,`, -- `Max:` + fmt.Sprintf("%v", this.Max) + `,`, -- `Failcnt:` + fmt.Sprintf("%v", this.Failcnt) + `,`, -- `}`, -- }, "") -- return s -+ return 0 - } --func (this *BlkIOStat) String() string { -- if this == nil { -- return "nil" -+ -+func (m *BlkIOEntry) GetValue() uint64 { -+ if m != nil { -+ return m.Value - } -- s := strings.Join([]string{`&BlkIOStat{`, -- `IoServiceBytesRecursive:` + strings.Replace(fmt.Sprintf("%v", this.IoServiceBytesRecursive), "BlkIOEntry", "BlkIOEntry", 1) + `,`, -- `IoServicedRecursive:` + strings.Replace(fmt.Sprintf("%v", this.IoServicedRecursive), "BlkIOEntry", "BlkIOEntry", 1) + `,`, -- `IoQueuedRecursive:` + strings.Replace(fmt.Sprintf("%v", this.IoQueuedRecursive), "BlkIOEntry", "BlkIOEntry", 1) + `,`, -- `IoServiceTimeRecursive:` + strings.Replace(fmt.Sprintf("%v", this.IoServiceTimeRecursive), "BlkIOEntry", "BlkIOEntry", 1) + `,`, -- `IoWaitTimeRecursive:` + strings.Replace(fmt.Sprintf("%v", this.IoWaitTimeRecursive), "BlkIOEntry", "BlkIOEntry", 1) + `,`, -- `IoMergedRecursive:` + strings.Replace(fmt.Sprintf("%v", this.IoMergedRecursive), "BlkIOEntry", "BlkIOEntry", 1) + `,`, -- `IoTimeRecursive:` + strings.Replace(fmt.Sprintf("%v", this.IoTimeRecursive), "BlkIOEntry", "BlkIOEntry", 1) + `,`, -- `SectorsRecursive:` + strings.Replace(fmt.Sprintf("%v", this.SectorsRecursive), "BlkIOEntry", "BlkIOEntry", 1) + `,`, -- `}`, -- }, "") -- return s -+ return 0 - } --func (this *BlkIOEntry) String() string { -- if this == nil { -- return "nil" -- } -- s := strings.Join([]string{`&BlkIOEntry{`, -- `Op:` + fmt.Sprintf("%v", this.Op) + `,`, -- `Device:` + fmt.Sprintf("%v", this.Device) + `,`, -- `Major:` + fmt.Sprintf("%v", this.Major) + `,`, -- `Minor:` + fmt.Sprintf("%v", this.Minor) + `,`, -- `Value:` + fmt.Sprintf("%v", this.Value) + `,`, -- `}`, -- }, "") -- return s -+ -+type RdmaStat struct { -+ Current []*RdmaEntry `protobuf:"bytes,1,rep,name=current" json:"current,omitempty"` -+ Limit []*RdmaEntry `protobuf:"bytes,2,rep,name=limit" json:"limit,omitempty"` - } --func (this *RdmaStat) String() string { -- if this == nil { -- return "nil" -+ -+func (m *RdmaStat) Reset() { *m = RdmaStat{} } -+func (m *RdmaStat) String() string { return proto.CompactTextString(m) } -+func (*RdmaStat) ProtoMessage() {} -+func (*RdmaStat) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{10} } -+ -+func (m *RdmaStat) GetCurrent() []*RdmaEntry { -+ if m != nil { -+ return m.Current - } -- s := strings.Join([]string{`&RdmaStat{`, -- `Current:` + strings.Replace(fmt.Sprintf("%v", this.Current), "RdmaEntry", "RdmaEntry", 1) + `,`, -- `Limit:` + strings.Replace(fmt.Sprintf("%v", this.Limit), "RdmaEntry", "RdmaEntry", 1) + `,`, -- `}`, -- }, "") -- return s -+ return nil - } --func (this *RdmaEntry) String() string { -- if this == nil { -- return "nil" -+ -+func (m *RdmaStat) GetLimit() []*RdmaEntry { -+ if m != nil { -+ return m.Limit - } -- s := strings.Join([]string{`&RdmaEntry{`, -- `Device:` + fmt.Sprintf("%v", this.Device) + `,`, -- `HcaHandles:` + fmt.Sprintf("%v", this.HcaHandles) + `,`, -- `HcaObjects:` + fmt.Sprintf("%v", this.HcaObjects) + `,`, -- `}`, -- }, "") -- return s -+ return nil - } --func (this *NetworkStat) String() string { -- if this == nil { -- return "nil" -- } -- s := strings.Join([]string{`&NetworkStat{`, -- `Name:` + fmt.Sprintf("%v", this.Name) + `,`, -- `RxBytes:` + fmt.Sprintf("%v", this.RxBytes) + `,`, -- `RxPackets:` + fmt.Sprintf("%v", this.RxPackets) + `,`, -- `RxErrors:` + fmt.Sprintf("%v", this.RxErrors) + `,`, -- `RxDropped:` + fmt.Sprintf("%v", this.RxDropped) + `,`, -- `TxBytes:` + fmt.Sprintf("%v", this.TxBytes) + `,`, -- `TxPackets:` + fmt.Sprintf("%v", this.TxPackets) + `,`, -- `TxErrors:` + fmt.Sprintf("%v", this.TxErrors) + `,`, -- `TxDropped:` + fmt.Sprintf("%v", this.TxDropped) + `,`, -- `}`, -- }, "") -- return s -+ -+type RdmaEntry struct { -+ Device string `protobuf:"bytes,1,opt,name=device,proto3" json:"device,omitempty"` -+ HcaHandles uint32 `protobuf:"varint,2,opt,name=hca_handles,json=hcaHandles,proto3" json:"hca_handles,omitempty"` -+ HcaObjects uint32 `protobuf:"varint,3,opt,name=hca_objects,json=hcaObjects,proto3" json:"hca_objects,omitempty"` - } --func valueToStringMetrics(v interface{}) string { -- rv := reflect.ValueOf(v) -- if rv.IsNil() { -- return "nil" -+ -+func (m *RdmaEntry) Reset() { *m = RdmaEntry{} } -+func (m *RdmaEntry) String() string { return proto.CompactTextString(m) } -+func (*RdmaEntry) ProtoMessage() {} -+func (*RdmaEntry) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{11} } -+ -+func (m *RdmaEntry) GetDevice() string { -+ if m != nil { -+ return m.Device - } -- pv := reflect.Indirect(rv).Interface() -- return fmt.Sprintf("*%v", pv) -+ return "" - } --func (m *Metrics) Unmarshal(dAtA []byte) error { -- l := len(dAtA) -- iNdEx := 0 -- for iNdEx < l { -- preIndex := iNdEx -- var wire uint64 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- wire |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- fieldNum := int32(wire >> 3) -- wireType := int(wire & 0x7) -- if wireType == 4 { -- return fmt.Errorf("proto: Metrics: wiretype end group for non-group") -- } -- if fieldNum <= 0 { -- return fmt.Errorf("proto: Metrics: illegal tag %d (wire type %d)", fieldNum, wire) -- } -- switch fieldNum { -- case 1: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field Hugetlb", wireType) -- } -- var msglen int -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- msglen |= (int(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- if msglen < 0 { -- return ErrInvalidLengthMetrics -- } -- postIndex := iNdEx + msglen -- if postIndex > l { -- return io.ErrUnexpectedEOF -- } -- m.Hugetlb = append(m.Hugetlb, &HugetlbStat{}) -- if err := m.Hugetlb[len(m.Hugetlb)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -- return err -- } -- iNdEx = postIndex -- case 2: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field Pids", wireType) -- } -- var msglen int -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- msglen |= (int(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- if msglen < 0 { -- return ErrInvalidLengthMetrics -- } -- postIndex := iNdEx + msglen -- if postIndex > l { -- return io.ErrUnexpectedEOF -- } -- if m.Pids == nil { -- m.Pids = &PidsStat{} -- } -- if err := m.Pids.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -- return err -- } -- iNdEx = postIndex -- case 3: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field CPU", wireType) -- } -- var msglen int -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- msglen |= (int(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- if msglen < 0 { -- return ErrInvalidLengthMetrics -- } -- postIndex := iNdEx + msglen -- if postIndex > l { -- return io.ErrUnexpectedEOF -- } -- if m.CPU == nil { -- m.CPU = &CPUStat{} -- } -- if err := m.CPU.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -- return err -- } -- iNdEx = postIndex -- case 4: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field Memory", wireType) -- } -- var msglen int -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- msglen |= (int(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- if msglen < 0 { -- return ErrInvalidLengthMetrics -- } -- postIndex := iNdEx + msglen -- if postIndex > l { -- return io.ErrUnexpectedEOF -- } -- if m.Memory == nil { -- m.Memory = &MemoryStat{} -- } -- if err := m.Memory.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -- return err -- } -- iNdEx = postIndex -- case 5: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field Blkio", wireType) -- } -- var msglen int -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- msglen |= (int(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- if msglen < 0 { -- return ErrInvalidLengthMetrics -- } -- postIndex := iNdEx + msglen -- if postIndex > l { -- return io.ErrUnexpectedEOF -- } -- if m.Blkio == nil { -- m.Blkio = &BlkIOStat{} -- } -- if err := m.Blkio.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -- return err -- } -- iNdEx = postIndex -- case 6: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field Rdma", wireType) -- } -- var msglen int -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- msglen |= (int(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- if msglen < 0 { -- return ErrInvalidLengthMetrics -- } -- postIndex := iNdEx + msglen -- if postIndex > l { -- return io.ErrUnexpectedEOF -- } -- if m.Rdma == nil { -- m.Rdma = &RdmaStat{} -- } -- if err := m.Rdma.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -- return err -- } -- iNdEx = postIndex -- case 7: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field Network", wireType) -- } -- var msglen int -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- msglen |= (int(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- if msglen < 0 { -- return ErrInvalidLengthMetrics -- } -- postIndex := iNdEx + msglen -- if postIndex > l { -- return io.ErrUnexpectedEOF -- } -- m.Network = append(m.Network, &NetworkStat{}) -- if err := m.Network[len(m.Network)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -- return err -- } -- iNdEx = postIndex -- default: -- iNdEx = preIndex -- skippy, err := skipMetrics(dAtA[iNdEx:]) -- if err != nil { -- return err -- } -- if skippy < 0 { -- return ErrInvalidLengthMetrics -- } -- if (iNdEx + skippy) > l { -- return io.ErrUnexpectedEOF -- } -- iNdEx += skippy -- } -- } - -- if iNdEx > l { -- return io.ErrUnexpectedEOF -+func (m *RdmaEntry) GetHcaHandles() uint32 { -+ if m != nil { -+ return m.HcaHandles - } -- return nil -+ return 0 - } --func (m *HugetlbStat) Unmarshal(dAtA []byte) error { -- l := len(dAtA) -- iNdEx := 0 -- for iNdEx < l { -- preIndex := iNdEx -- var wire uint64 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- wire |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- fieldNum := int32(wire >> 3) -- wireType := int(wire & 0x7) -- if wireType == 4 { -- return fmt.Errorf("proto: HugetlbStat: wiretype end group for non-group") -- } -- if fieldNum <= 0 { -- return fmt.Errorf("proto: HugetlbStat: illegal tag %d (wire type %d)", fieldNum, wire) -- } -- switch fieldNum { -- case 1: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field Usage", wireType) -- } -- m.Usage = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.Usage |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 2: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field Max", wireType) -- } -- m.Max = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.Max |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 3: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field Failcnt", wireType) -- } -- m.Failcnt = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.Failcnt |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 4: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field Pagesize", wireType) -- } -- var stringLen uint64 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- stringLen |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- intStringLen := int(stringLen) -- if intStringLen < 0 { -- return ErrInvalidLengthMetrics -- } -- postIndex := iNdEx + intStringLen -- if postIndex > l { -- return io.ErrUnexpectedEOF -- } -- m.Pagesize = string(dAtA[iNdEx:postIndex]) -- iNdEx = postIndex -- default: -- iNdEx = preIndex -- skippy, err := skipMetrics(dAtA[iNdEx:]) -- if err != nil { -- return err -- } -- if skippy < 0 { -- return ErrInvalidLengthMetrics -- } -- if (iNdEx + skippy) > l { -- return io.ErrUnexpectedEOF -- } -- iNdEx += skippy -- } -- } - -- if iNdEx > l { -- return io.ErrUnexpectedEOF -+func (m *RdmaEntry) GetHcaObjects() uint32 { -+ if m != nil { -+ return m.HcaObjects - } -- return nil -+ return 0 - } --func (m *PidsStat) Unmarshal(dAtA []byte) error { -- l := len(dAtA) -- iNdEx := 0 -- for iNdEx < l { -- preIndex := iNdEx -- var wire uint64 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- wire |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- fieldNum := int32(wire >> 3) -- wireType := int(wire & 0x7) -- if wireType == 4 { -- return fmt.Errorf("proto: PidsStat: wiretype end group for non-group") -- } -- if fieldNum <= 0 { -- return fmt.Errorf("proto: PidsStat: illegal tag %d (wire type %d)", fieldNum, wire) -- } -- switch fieldNum { -- case 1: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field Current", wireType) -- } -- m.Current = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.Current |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 2: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field Limit", wireType) -- } -- m.Limit = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.Limit |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- default: -- iNdEx = preIndex -- skippy, err := skipMetrics(dAtA[iNdEx:]) -- if err != nil { -- return err -- } -- if skippy < 0 { -- return ErrInvalidLengthMetrics -- } -- if (iNdEx + skippy) > l { -- return io.ErrUnexpectedEOF -- } -- iNdEx += skippy -- } -- } - -- if iNdEx > l { -- return io.ErrUnexpectedEOF -- } -- return nil -+type NetworkStat struct { -+ Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` -+ RxBytes uint64 `protobuf:"varint,2,opt,name=rx_bytes,json=rxBytes,proto3" json:"rx_bytes,omitempty"` -+ RxPackets uint64 `protobuf:"varint,3,opt,name=rx_packets,json=rxPackets,proto3" json:"rx_packets,omitempty"` -+ RxErrors uint64 `protobuf:"varint,4,opt,name=rx_errors,json=rxErrors,proto3" json:"rx_errors,omitempty"` -+ RxDropped uint64 `protobuf:"varint,5,opt,name=rx_dropped,json=rxDropped,proto3" json:"rx_dropped,omitempty"` -+ TxBytes uint64 `protobuf:"varint,6,opt,name=tx_bytes,json=txBytes,proto3" json:"tx_bytes,omitempty"` -+ TxPackets uint64 `protobuf:"varint,7,opt,name=tx_packets,json=txPackets,proto3" json:"tx_packets,omitempty"` -+ TxErrors uint64 `protobuf:"varint,8,opt,name=tx_errors,json=txErrors,proto3" json:"tx_errors,omitempty"` -+ TxDropped uint64 `protobuf:"varint,9,opt,name=tx_dropped,json=txDropped,proto3" json:"tx_dropped,omitempty"` - } --func (m *CPUStat) Unmarshal(dAtA []byte) error { -- l := len(dAtA) -- iNdEx := 0 -- for iNdEx < l { -- preIndex := iNdEx -- var wire uint64 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- wire |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- fieldNum := int32(wire >> 3) -- wireType := int(wire & 0x7) -- if wireType == 4 { -- return fmt.Errorf("proto: CPUStat: wiretype end group for non-group") -- } -- if fieldNum <= 0 { -- return fmt.Errorf("proto: CPUStat: illegal tag %d (wire type %d)", fieldNum, wire) -- } -- switch fieldNum { -- case 1: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field Usage", wireType) -- } -- var msglen int -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- msglen |= (int(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- if msglen < 0 { -- return ErrInvalidLengthMetrics -- } -- postIndex := iNdEx + msglen -- if postIndex > l { -- return io.ErrUnexpectedEOF -- } -- if m.Usage == nil { -- m.Usage = &CPUUsage{} -- } -- if err := m.Usage.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -- return err -- } -- iNdEx = postIndex -- case 2: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field Throttling", wireType) -- } -- var msglen int -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- msglen |= (int(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- if msglen < 0 { -- return ErrInvalidLengthMetrics -- } -- postIndex := iNdEx + msglen -- if postIndex > l { -- return io.ErrUnexpectedEOF -- } -- if m.Throttling == nil { -- m.Throttling = &Throttle{} -- } -- if err := m.Throttling.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -- return err -- } -- iNdEx = postIndex -- default: -- iNdEx = preIndex -- skippy, err := skipMetrics(dAtA[iNdEx:]) -- if err != nil { -- return err -- } -- if skippy < 0 { -- return ErrInvalidLengthMetrics -- } -- if (iNdEx + skippy) > l { -- return io.ErrUnexpectedEOF -- } -- iNdEx += skippy -- } -- } - -- if iNdEx > l { -- return io.ErrUnexpectedEOF -+func (m *NetworkStat) Reset() { *m = NetworkStat{} } -+func (m *NetworkStat) String() string { return proto.CompactTextString(m) } -+func (*NetworkStat) ProtoMessage() {} -+func (*NetworkStat) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{12} } -+ -+func (m *NetworkStat) GetName() string { -+ if m != nil { -+ return m.Name - } -- return nil -+ return "" - } --func (m *CPUUsage) Unmarshal(dAtA []byte) error { -- l := len(dAtA) -- iNdEx := 0 -- for iNdEx < l { -- preIndex := iNdEx -- var wire uint64 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- wire |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- fieldNum := int32(wire >> 3) -- wireType := int(wire & 0x7) -- if wireType == 4 { -- return fmt.Errorf("proto: CPUUsage: wiretype end group for non-group") -- } -- if fieldNum <= 0 { -- return fmt.Errorf("proto: CPUUsage: illegal tag %d (wire type %d)", fieldNum, wire) -- } -- switch fieldNum { -- case 1: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field Total", wireType) -- } -- m.Total = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.Total |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 2: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field Kernel", wireType) -- } -- m.Kernel = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.Kernel |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 3: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field User", wireType) -- } -- m.User = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.User |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 4: -- if wireType == 0 { -- var v uint64 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- v |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- m.PerCPU = append(m.PerCPU, v) -- } else if wireType == 2 { -- var packedLen int -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- packedLen |= (int(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- if packedLen < 0 { -- return ErrInvalidLengthMetrics -- } -- postIndex := iNdEx + packedLen -- if postIndex > l { -- return io.ErrUnexpectedEOF -- } -- for iNdEx < postIndex { -- var v uint64 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- v |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- m.PerCPU = append(m.PerCPU, v) -- } -- } else { -- return fmt.Errorf("proto: wrong wireType = %d for field PerCPU", wireType) -- } -- default: -- iNdEx = preIndex -- skippy, err := skipMetrics(dAtA[iNdEx:]) -- if err != nil { -- return err -- } -- if skippy < 0 { -- return ErrInvalidLengthMetrics -- } -- if (iNdEx + skippy) > l { -- return io.ErrUnexpectedEOF -- } -- iNdEx += skippy -- } -- } - -- if iNdEx > l { -- return io.ErrUnexpectedEOF -+func (m *NetworkStat) GetRxBytes() uint64 { -+ if m != nil { -+ return m.RxBytes - } -- return nil -+ return 0 - } --func (m *Throttle) Unmarshal(dAtA []byte) error { -- l := len(dAtA) -- iNdEx := 0 -- for iNdEx < l { -- preIndex := iNdEx -- var wire uint64 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- wire |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- fieldNum := int32(wire >> 3) -- wireType := int(wire & 0x7) -- if wireType == 4 { -- return fmt.Errorf("proto: Throttle: wiretype end group for non-group") -- } -- if fieldNum <= 0 { -- return fmt.Errorf("proto: Throttle: illegal tag %d (wire type %d)", fieldNum, wire) -- } -- switch fieldNum { -- case 1: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field Periods", wireType) -- } -- m.Periods = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.Periods |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 2: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field ThrottledPeriods", wireType) -- } -- m.ThrottledPeriods = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.ThrottledPeriods |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 3: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field ThrottledTime", wireType) -- } -- m.ThrottledTime = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.ThrottledTime |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- default: -- iNdEx = preIndex -- skippy, err := skipMetrics(dAtA[iNdEx:]) -- if err != nil { -- return err -- } -- if skippy < 0 { -- return ErrInvalidLengthMetrics -- } -- if (iNdEx + skippy) > l { -- return io.ErrUnexpectedEOF -- } -- iNdEx += skippy -- } -- } - -- if iNdEx > l { -- return io.ErrUnexpectedEOF -+func (m *NetworkStat) GetRxPackets() uint64 { -+ if m != nil { -+ return m.RxPackets - } -- return nil -+ return 0 - } --func (m *MemoryStat) Unmarshal(dAtA []byte) error { -- l := len(dAtA) -- iNdEx := 0 -- for iNdEx < l { -- preIndex := iNdEx -- var wire uint64 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- wire |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- fieldNum := int32(wire >> 3) -- wireType := int(wire & 0x7) -- if wireType == 4 { -- return fmt.Errorf("proto: MemoryStat: wiretype end group for non-group") -- } -- if fieldNum <= 0 { -- return fmt.Errorf("proto: MemoryStat: illegal tag %d (wire type %d)", fieldNum, wire) -- } -- switch fieldNum { -- case 1: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field Cache", wireType) -- } -- m.Cache = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.Cache |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 2: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field RSS", wireType) -- } -- m.RSS = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.RSS |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 3: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field RSSHuge", wireType) -- } -- m.RSSHuge = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.RSSHuge |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 4: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field MappedFile", wireType) -- } -- m.MappedFile = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.MappedFile |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 5: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field Dirty", wireType) -- } -- m.Dirty = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.Dirty |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 6: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field Writeback", wireType) -- } -- m.Writeback = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.Writeback |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 7: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field PgPgIn", wireType) -- } -- m.PgPgIn = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.PgPgIn |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 8: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field PgPgOut", wireType) -- } -- m.PgPgOut = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.PgPgOut |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 9: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field PgFault", wireType) -- } -- m.PgFault = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.PgFault |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 10: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field PgMajFault", wireType) -- } -- m.PgMajFault = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.PgMajFault |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 11: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field InactiveAnon", wireType) -- } -- m.InactiveAnon = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.InactiveAnon |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 12: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field ActiveAnon", wireType) -- } -- m.ActiveAnon = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.ActiveAnon |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 13: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field InactiveFile", wireType) -- } -- m.InactiveFile = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.InactiveFile |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 14: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field ActiveFile", wireType) -- } -- m.ActiveFile = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.ActiveFile |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 15: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field Unevictable", wireType) -- } -- m.Unevictable = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.Unevictable |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 16: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field HierarchicalMemoryLimit", wireType) -- } -- m.HierarchicalMemoryLimit = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.HierarchicalMemoryLimit |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 17: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field HierarchicalSwapLimit", wireType) -- } -- m.HierarchicalSwapLimit = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.HierarchicalSwapLimit |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 18: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field TotalCache", wireType) -- } -- m.TotalCache = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.TotalCache |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 19: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field TotalRSS", wireType) -- } -- m.TotalRSS = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.TotalRSS |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 20: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field TotalRSSHuge", wireType) -- } -- m.TotalRSSHuge = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.TotalRSSHuge |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 21: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field TotalMappedFile", wireType) -- } -- m.TotalMappedFile = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.TotalMappedFile |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 22: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field TotalDirty", wireType) -- } -- m.TotalDirty = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.TotalDirty |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 23: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field TotalWriteback", wireType) -- } -- m.TotalWriteback = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.TotalWriteback |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 24: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field TotalPgPgIn", wireType) -- } -- m.TotalPgPgIn = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.TotalPgPgIn |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 25: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field TotalPgPgOut", wireType) -- } -- m.TotalPgPgOut = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.TotalPgPgOut |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 26: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field TotalPgFault", wireType) -- } -- m.TotalPgFault = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.TotalPgFault |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 27: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field TotalPgMajFault", wireType) -- } -- m.TotalPgMajFault = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.TotalPgMajFault |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 28: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field TotalInactiveAnon", wireType) -- } -- m.TotalInactiveAnon = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.TotalInactiveAnon |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 29: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field TotalActiveAnon", wireType) -- } -- m.TotalActiveAnon = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.TotalActiveAnon |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 30: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field TotalInactiveFile", wireType) -- } -- m.TotalInactiveFile = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.TotalInactiveFile |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 31: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field TotalActiveFile", wireType) -- } -- m.TotalActiveFile = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.TotalActiveFile |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 32: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field TotalUnevictable", wireType) -- } -- m.TotalUnevictable = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.TotalUnevictable |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 33: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field Usage", wireType) -- } -- var msglen int -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- msglen |= (int(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- if msglen < 0 { -- return ErrInvalidLengthMetrics -- } -- postIndex := iNdEx + msglen -- if postIndex > l { -- return io.ErrUnexpectedEOF -- } -- if m.Usage == nil { -- m.Usage = &MemoryEntry{} -- } -- if err := m.Usage.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -- return err -- } -- iNdEx = postIndex -- case 34: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field Swap", wireType) -- } -- var msglen int -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- msglen |= (int(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- if msglen < 0 { -- return ErrInvalidLengthMetrics -- } -- postIndex := iNdEx + msglen -- if postIndex > l { -- return io.ErrUnexpectedEOF -- } -- if m.Swap == nil { -- m.Swap = &MemoryEntry{} -- } -- if err := m.Swap.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -- return err -- } -- iNdEx = postIndex -- case 35: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field Kernel", wireType) -- } -- var msglen int -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- msglen |= (int(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- if msglen < 0 { -- return ErrInvalidLengthMetrics -- } -- postIndex := iNdEx + msglen -- if postIndex > l { -- return io.ErrUnexpectedEOF -- } -- if m.Kernel == nil { -- m.Kernel = &MemoryEntry{} -- } -- if err := m.Kernel.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -- return err -- } -- iNdEx = postIndex -- case 36: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field KernelTCP", wireType) -- } -- var msglen int -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- msglen |= (int(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- if msglen < 0 { -- return ErrInvalidLengthMetrics -- } -- postIndex := iNdEx + msglen -- if postIndex > l { -- return io.ErrUnexpectedEOF -- } -- if m.KernelTCP == nil { -- m.KernelTCP = &MemoryEntry{} -- } -- if err := m.KernelTCP.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -- return err -- } -- iNdEx = postIndex -- default: -- iNdEx = preIndex -- skippy, err := skipMetrics(dAtA[iNdEx:]) -- if err != nil { -- return err -- } -- if skippy < 0 { -- return ErrInvalidLengthMetrics -- } -- if (iNdEx + skippy) > l { -- return io.ErrUnexpectedEOF -- } -- iNdEx += skippy -- } -- } - -- if iNdEx > l { -- return io.ErrUnexpectedEOF -+func (m *NetworkStat) GetRxErrors() uint64 { -+ if m != nil { -+ return m.RxErrors - } -- return nil -+ return 0 - } --func (m *MemoryEntry) Unmarshal(dAtA []byte) error { -- l := len(dAtA) -- iNdEx := 0 -- for iNdEx < l { -- preIndex := iNdEx -- var wire uint64 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- wire |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- fieldNum := int32(wire >> 3) -- wireType := int(wire & 0x7) -- if wireType == 4 { -- return fmt.Errorf("proto: MemoryEntry: wiretype end group for non-group") -- } -- if fieldNum <= 0 { -- return fmt.Errorf("proto: MemoryEntry: illegal tag %d (wire type %d)", fieldNum, wire) -- } -- switch fieldNum { -- case 1: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field Limit", wireType) -- } -- m.Limit = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.Limit |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 2: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field Usage", wireType) -- } -- m.Usage = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.Usage |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 3: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field Max", wireType) -- } -- m.Max = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.Max |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 4: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field Failcnt", wireType) -- } -- m.Failcnt = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.Failcnt |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- default: -- iNdEx = preIndex -- skippy, err := skipMetrics(dAtA[iNdEx:]) -- if err != nil { -- return err -- } -- if skippy < 0 { -- return ErrInvalidLengthMetrics -- } -- if (iNdEx + skippy) > l { -- return io.ErrUnexpectedEOF -- } -- iNdEx += skippy -- } -- } - -- if iNdEx > l { -- return io.ErrUnexpectedEOF -+func (m *NetworkStat) GetRxDropped() uint64 { -+ if m != nil { -+ return m.RxDropped - } -- return nil -+ return 0 - } --func (m *BlkIOStat) Unmarshal(dAtA []byte) error { -- l := len(dAtA) -- iNdEx := 0 -- for iNdEx < l { -- preIndex := iNdEx -- var wire uint64 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- wire |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- fieldNum := int32(wire >> 3) -- wireType := int(wire & 0x7) -- if wireType == 4 { -- return fmt.Errorf("proto: BlkIOStat: wiretype end group for non-group") -- } -- if fieldNum <= 0 { -- return fmt.Errorf("proto: BlkIOStat: illegal tag %d (wire type %d)", fieldNum, wire) -- } -- switch fieldNum { -- case 1: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field IoServiceBytesRecursive", wireType) -- } -- var msglen int -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- msglen |= (int(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- if msglen < 0 { -- return ErrInvalidLengthMetrics -- } -- postIndex := iNdEx + msglen -- if postIndex > l { -- return io.ErrUnexpectedEOF -- } -- m.IoServiceBytesRecursive = append(m.IoServiceBytesRecursive, &BlkIOEntry{}) -- if err := m.IoServiceBytesRecursive[len(m.IoServiceBytesRecursive)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -- return err -- } -- iNdEx = postIndex -- case 2: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field IoServicedRecursive", wireType) -- } -- var msglen int -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- msglen |= (int(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- if msglen < 0 { -- return ErrInvalidLengthMetrics -- } -- postIndex := iNdEx + msglen -- if postIndex > l { -- return io.ErrUnexpectedEOF -- } -- m.IoServicedRecursive = append(m.IoServicedRecursive, &BlkIOEntry{}) -- if err := m.IoServicedRecursive[len(m.IoServicedRecursive)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -- return err -- } -- iNdEx = postIndex -- case 3: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field IoQueuedRecursive", wireType) -- } -- var msglen int -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- msglen |= (int(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- if msglen < 0 { -- return ErrInvalidLengthMetrics -- } -- postIndex := iNdEx + msglen -- if postIndex > l { -- return io.ErrUnexpectedEOF -- } -- m.IoQueuedRecursive = append(m.IoQueuedRecursive, &BlkIOEntry{}) -- if err := m.IoQueuedRecursive[len(m.IoQueuedRecursive)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -- return err -- } -- iNdEx = postIndex -- case 4: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field IoServiceTimeRecursive", wireType) -- } -- var msglen int -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- msglen |= (int(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- if msglen < 0 { -- return ErrInvalidLengthMetrics -- } -- postIndex := iNdEx + msglen -- if postIndex > l { -- return io.ErrUnexpectedEOF -- } -- m.IoServiceTimeRecursive = append(m.IoServiceTimeRecursive, &BlkIOEntry{}) -- if err := m.IoServiceTimeRecursive[len(m.IoServiceTimeRecursive)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -- return err -- } -- iNdEx = postIndex -- case 5: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field IoWaitTimeRecursive", wireType) -- } -- var msglen int -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- msglen |= (int(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- if msglen < 0 { -- return ErrInvalidLengthMetrics -- } -- postIndex := iNdEx + msglen -- if postIndex > l { -- return io.ErrUnexpectedEOF -- } -- m.IoWaitTimeRecursive = append(m.IoWaitTimeRecursive, &BlkIOEntry{}) -- if err := m.IoWaitTimeRecursive[len(m.IoWaitTimeRecursive)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -- return err -- } -- iNdEx = postIndex -- case 6: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field IoMergedRecursive", wireType) -- } -- var msglen int -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- msglen |= (int(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- if msglen < 0 { -- return ErrInvalidLengthMetrics -- } -- postIndex := iNdEx + msglen -- if postIndex > l { -- return io.ErrUnexpectedEOF -- } -- m.IoMergedRecursive = append(m.IoMergedRecursive, &BlkIOEntry{}) -- if err := m.IoMergedRecursive[len(m.IoMergedRecursive)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -- return err -- } -- iNdEx = postIndex -- case 7: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field IoTimeRecursive", wireType) -- } -- var msglen int -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- msglen |= (int(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- if msglen < 0 { -- return ErrInvalidLengthMetrics -- } -- postIndex := iNdEx + msglen -- if postIndex > l { -- return io.ErrUnexpectedEOF -- } -- m.IoTimeRecursive = append(m.IoTimeRecursive, &BlkIOEntry{}) -- if err := m.IoTimeRecursive[len(m.IoTimeRecursive)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -- return err -- } -- iNdEx = postIndex -- case 8: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field SectorsRecursive", wireType) -- } -- var msglen int -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- msglen |= (int(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- if msglen < 0 { -- return ErrInvalidLengthMetrics -- } -- postIndex := iNdEx + msglen -- if postIndex > l { -- return io.ErrUnexpectedEOF -- } -- m.SectorsRecursive = append(m.SectorsRecursive, &BlkIOEntry{}) -- if err := m.SectorsRecursive[len(m.SectorsRecursive)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -- return err -- } -- iNdEx = postIndex -- default: -- iNdEx = preIndex -- skippy, err := skipMetrics(dAtA[iNdEx:]) -- if err != nil { -- return err -- } -- if skippy < 0 { -- return ErrInvalidLengthMetrics -- } -- if (iNdEx + skippy) > l { -- return io.ErrUnexpectedEOF -- } -- iNdEx += skippy -- } -- } - -- if iNdEx > l { -- return io.ErrUnexpectedEOF -+func (m *NetworkStat) GetTxBytes() uint64 { -+ if m != nil { -+ return m.TxBytes - } -- return nil -+ return 0 - } --func (m *BlkIOEntry) Unmarshal(dAtA []byte) error { -- l := len(dAtA) -- iNdEx := 0 -- for iNdEx < l { -- preIndex := iNdEx -- var wire uint64 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- wire |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- fieldNum := int32(wire >> 3) -- wireType := int(wire & 0x7) -- if wireType == 4 { -- return fmt.Errorf("proto: BlkIOEntry: wiretype end group for non-group") -- } -- if fieldNum <= 0 { -- return fmt.Errorf("proto: BlkIOEntry: illegal tag %d (wire type %d)", fieldNum, wire) -- } -- switch fieldNum { -- case 1: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field Op", wireType) -- } -- var stringLen uint64 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- stringLen |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- intStringLen := int(stringLen) -- if intStringLen < 0 { -- return ErrInvalidLengthMetrics -- } -- postIndex := iNdEx + intStringLen -- if postIndex > l { -- return io.ErrUnexpectedEOF -- } -- m.Op = string(dAtA[iNdEx:postIndex]) -- iNdEx = postIndex -- case 2: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field Device", wireType) -- } -- var stringLen uint64 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- stringLen |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- intStringLen := int(stringLen) -- if intStringLen < 0 { -- return ErrInvalidLengthMetrics -- } -- postIndex := iNdEx + intStringLen -- if postIndex > l { -- return io.ErrUnexpectedEOF -- } -- m.Device = string(dAtA[iNdEx:postIndex]) -- iNdEx = postIndex -- case 3: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field Major", wireType) -- } -- m.Major = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.Major |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 4: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field Minor", wireType) -- } -- m.Minor = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.Minor |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 5: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field Value", wireType) -- } -- m.Value = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.Value |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- default: -- iNdEx = preIndex -- skippy, err := skipMetrics(dAtA[iNdEx:]) -- if err != nil { -- return err -- } -- if skippy < 0 { -- return ErrInvalidLengthMetrics -- } -- if (iNdEx + skippy) > l { -- return io.ErrUnexpectedEOF -- } -- iNdEx += skippy -- } -- } - -- if iNdEx > l { -- return io.ErrUnexpectedEOF -+func (m *NetworkStat) GetTxPackets() uint64 { -+ if m != nil { -+ return m.TxPackets - } -- return nil -+ return 0 - } --func (m *RdmaStat) Unmarshal(dAtA []byte) error { -- l := len(dAtA) -- iNdEx := 0 -- for iNdEx < l { -- preIndex := iNdEx -- var wire uint64 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- wire |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- fieldNum := int32(wire >> 3) -- wireType := int(wire & 0x7) -- if wireType == 4 { -- return fmt.Errorf("proto: RdmaStat: wiretype end group for non-group") -- } -- if fieldNum <= 0 { -- return fmt.Errorf("proto: RdmaStat: illegal tag %d (wire type %d)", fieldNum, wire) -- } -- switch fieldNum { -- case 1: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field Current", wireType) -- } -- var msglen int -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- msglen |= (int(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- if msglen < 0 { -- return ErrInvalidLengthMetrics -- } -- postIndex := iNdEx + msglen -- if postIndex > l { -- return io.ErrUnexpectedEOF -- } -- m.Current = append(m.Current, &RdmaEntry{}) -- if err := m.Current[len(m.Current)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -- return err -- } -- iNdEx = postIndex -- case 2: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field Limit", wireType) -- } -- var msglen int -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- msglen |= (int(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- if msglen < 0 { -- return ErrInvalidLengthMetrics -- } -- postIndex := iNdEx + msglen -- if postIndex > l { -- return io.ErrUnexpectedEOF -- } -- m.Limit = append(m.Limit, &RdmaEntry{}) -- if err := m.Limit[len(m.Limit)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -- return err -- } -- iNdEx = postIndex -- default: -- iNdEx = preIndex -- skippy, err := skipMetrics(dAtA[iNdEx:]) -- if err != nil { -- return err -- } -- if skippy < 0 { -- return ErrInvalidLengthMetrics -- } -- if (iNdEx + skippy) > l { -- return io.ErrUnexpectedEOF -- } -- iNdEx += skippy -- } -- } - -- if iNdEx > l { -- return io.ErrUnexpectedEOF -+func (m *NetworkStat) GetTxErrors() uint64 { -+ if m != nil { -+ return m.TxErrors - } -- return nil -+ return 0 - } --func (m *RdmaEntry) Unmarshal(dAtA []byte) error { -- l := len(dAtA) -- iNdEx := 0 -- for iNdEx < l { -- preIndex := iNdEx -- var wire uint64 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- wire |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- fieldNum := int32(wire >> 3) -- wireType := int(wire & 0x7) -- if wireType == 4 { -- return fmt.Errorf("proto: RdmaEntry: wiretype end group for non-group") -- } -- if fieldNum <= 0 { -- return fmt.Errorf("proto: RdmaEntry: illegal tag %d (wire type %d)", fieldNum, wire) -- } -- switch fieldNum { -- case 1: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field Device", wireType) -- } -- var stringLen uint64 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- stringLen |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- intStringLen := int(stringLen) -- if intStringLen < 0 { -- return ErrInvalidLengthMetrics -- } -- postIndex := iNdEx + intStringLen -- if postIndex > l { -- return io.ErrUnexpectedEOF -- } -- m.Device = string(dAtA[iNdEx:postIndex]) -- iNdEx = postIndex -- case 2: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field HcaHandles", wireType) -- } -- m.HcaHandles = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.HcaHandles |= (uint32(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 3: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field HcaObjects", wireType) -- } -- m.HcaObjects = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.HcaObjects |= (uint32(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- default: -- iNdEx = preIndex -- skippy, err := skipMetrics(dAtA[iNdEx:]) -- if err != nil { -- return err -- } -- if skippy < 0 { -- return ErrInvalidLengthMetrics -- } -- if (iNdEx + skippy) > l { -- return io.ErrUnexpectedEOF -- } -- iNdEx += skippy -- } -- } - -- if iNdEx > l { -- return io.ErrUnexpectedEOF -+func (m *NetworkStat) GetTxDropped() uint64 { -+ if m != nil { -+ return m.TxDropped - } -- return nil -+ return 0 - } --func (m *NetworkStat) Unmarshal(dAtA []byte) error { -- l := len(dAtA) -- iNdEx := 0 -- for iNdEx < l { -- preIndex := iNdEx -- var wire uint64 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- wire |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- fieldNum := int32(wire >> 3) -- wireType := int(wire & 0x7) -- if wireType == 4 { -- return fmt.Errorf("proto: NetworkStat: wiretype end group for non-group") -- } -- if fieldNum <= 0 { -- return fmt.Errorf("proto: NetworkStat: illegal tag %d (wire type %d)", fieldNum, wire) -- } -- switch fieldNum { -- case 1: -- if wireType != 2 { -- return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) -- } -- var stringLen uint64 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- stringLen |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- intStringLen := int(stringLen) -- if intStringLen < 0 { -- return ErrInvalidLengthMetrics -- } -- postIndex := iNdEx + intStringLen -- if postIndex > l { -- return io.ErrUnexpectedEOF -- } -- m.Name = string(dAtA[iNdEx:postIndex]) -- iNdEx = postIndex -- case 2: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field RxBytes", wireType) -- } -- m.RxBytes = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.RxBytes |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 3: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field RxPackets", wireType) -- } -- m.RxPackets = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.RxPackets |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 4: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field RxErrors", wireType) -- } -- m.RxErrors = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.RxErrors |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 5: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field RxDropped", wireType) -- } -- m.RxDropped = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.RxDropped |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 6: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field TxBytes", wireType) -- } -- m.TxBytes = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.TxBytes |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 7: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field TxPackets", wireType) -- } -- m.TxPackets = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.TxPackets |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 8: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field TxErrors", wireType) -- } -- m.TxErrors = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.TxErrors |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- case 9: -- if wireType != 0 { -- return fmt.Errorf("proto: wrong wireType = %d for field TxDropped", wireType) -- } -- m.TxDropped = 0 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- m.TxDropped |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- default: -- iNdEx = preIndex -- skippy, err := skipMetrics(dAtA[iNdEx:]) -- if err != nil { -- return err -- } -- if skippy < 0 { -- return ErrInvalidLengthMetrics -- } -- if (iNdEx + skippy) > l { -- return io.ErrUnexpectedEOF -- } -- iNdEx += skippy -- } -- } - -- if iNdEx > l { -- return io.ErrUnexpectedEOF -+type FilesStat struct { -+ Usage uint64 `protobuf:"varint,1,opt,name=usage,proto3" json:"usage,omitempty"` -+ Limit uint64 `protobuf:"varint,2,opt,name=limit,proto3" json:"limit,omitempty"` -+} -+ -+func (m *FilesStat) Reset() { *m = FilesStat{} } -+func (m *FilesStat) String() string { return proto.CompactTextString(m) } -+func (*FilesStat) ProtoMessage() {} -+func (*FilesStat) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{13} } -+ -+func (m *FilesStat) GetUsage() uint64 { -+ if m != nil { -+ return m.Usage - } -- return nil -+ return 0 - } --func skipMetrics(dAtA []byte) (n int, err error) { -- l := len(dAtA) -- iNdEx := 0 -- for iNdEx < l { -- var wire uint64 -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return 0, ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return 0, io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- wire |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- wireType := int(wire & 0x7) -- switch wireType { -- case 0: -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return 0, ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return 0, io.ErrUnexpectedEOF -- } -- iNdEx++ -- if dAtA[iNdEx-1] < 0x80 { -- break -- } -- } -- return iNdEx, nil -- case 1: -- iNdEx += 8 -- return iNdEx, nil -- case 2: -- var length int -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return 0, ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return 0, io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- length |= (int(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- iNdEx += length -- if length < 0 { -- return 0, ErrInvalidLengthMetrics -- } -- return iNdEx, nil -- case 3: -- for { -- var innerWire uint64 -- var start int = iNdEx -- for shift := uint(0); ; shift += 7 { -- if shift >= 64 { -- return 0, ErrIntOverflowMetrics -- } -- if iNdEx >= l { -- return 0, io.ErrUnexpectedEOF -- } -- b := dAtA[iNdEx] -- iNdEx++ -- innerWire |= (uint64(b) & 0x7F) << shift -- if b < 0x80 { -- break -- } -- } -- innerWireType := int(innerWire & 0x7) -- if innerWireType == 4 { -- break -- } -- next, err := skipMetrics(dAtA[start:]) -- if err != nil { -- return 0, err -- } -- iNdEx = start + next -- } -- return iNdEx, nil -- case 4: -- return iNdEx, nil -- case 5: -- iNdEx += 4 -- return iNdEx, nil -- default: -- return 0, fmt.Errorf("proto: illegal wireType %d", wireType) -- } -+ -+func (m *FilesStat) GetLimit() uint64 { -+ if m != nil { -+ return m.Limit - } -- panic("unreachable") -+ return 0 - } - --var ( -- ErrInvalidLengthMetrics = fmt.Errorf("proto: negative length found during unmarshaling") -- ErrIntOverflowMetrics = fmt.Errorf("proto: integer overflow") --) -+func init() { -+ proto.RegisterType((*Metrics)(nil), "io.containerd.cgroups.v1.Metrics") -+ proto.RegisterType((*HugetlbStat)(nil), "io.containerd.cgroups.v1.HugetlbStat") -+ proto.RegisterType((*PidsStat)(nil), "io.containerd.cgroups.v1.PidsStat") -+ proto.RegisterType((*CPUStat)(nil), "io.containerd.cgroups.v1.CPUStat") -+ proto.RegisterType((*CPUUsage)(nil), "io.containerd.cgroups.v1.CPUUsage") -+ proto.RegisterType((*Throttle)(nil), "io.containerd.cgroups.v1.Throttle") -+ proto.RegisterType((*MemoryStat)(nil), "io.containerd.cgroups.v1.MemoryStat") -+ proto.RegisterType((*MemoryEntry)(nil), "io.containerd.cgroups.v1.MemoryEntry") -+ proto.RegisterType((*BlkIOStat)(nil), "io.containerd.cgroups.v1.BlkIOStat") -+ proto.RegisterType((*BlkIOEntry)(nil), "io.containerd.cgroups.v1.BlkIOEntry") -+ proto.RegisterType((*RdmaStat)(nil), "io.containerd.cgroups.v1.RdmaStat") -+ proto.RegisterType((*RdmaEntry)(nil), "io.containerd.cgroups.v1.RdmaEntry") -+ proto.RegisterType((*NetworkStat)(nil), "io.containerd.cgroups.v1.NetworkStat") -+ proto.RegisterType((*FilesStat)(nil), "io.containerd.cgroups.v1.FilesStat") -+} - - func init() { proto.RegisterFile("github.com/containerd/cgroups/metrics.proto", fileDescriptorMetrics) } - - var fileDescriptorMetrics = []byte{ -- // 1549 bytes of a gzipped FileDescriptorProto -- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x57, 0x4d, 0x6f, 0x1b, 0xb7, -- 0x16, 0x8d, 0x2c, 0xd9, 0xd2, 0x5c, 0xd9, 0x8e, 0x4d, 0x27, 0xce, 0xd8, 0x49, 0x2c, 0x47, 0xb6, -- 0xdf, 0xf3, 0x7b, 0x06, 0x64, 0xbc, 0x3c, 0x20, 0x68, 0xd2, 0x04, 0x45, 0xe4, 0x24, 0x48, 0xd0, -- 0xba, 0x51, 0x46, 0x36, 0xd2, 0xae, 0x06, 0xd4, 0x88, 0x19, 0xd1, 0x96, 0x86, 0x13, 0x0e, 0xc7, -- 0x96, 0xbb, 0xea, 0xa2, 0x40, 0x57, 0xfd, 0x33, 0xfd, 0x15, 0x59, 0x76, 0x53, 0xa0, 0xdd, 0x18, -- 0x8d, 0x7e, 0x49, 0x41, 0x72, 0x3e, 0xa8, 0x24, 0x8e, 0xab, 0xdd, 0x90, 0x3c, 0xe7, 0xdc, 0xcb, -- 0x3b, 0x87, 0xc3, 0x3b, 0xb0, 0xe3, 0x53, 0xd1, 0x8b, 0x3b, 0x0d, 0x8f, 0x0d, 0x76, 0x3d, 0x16, -- 0x08, 0x4c, 0x03, 0xc2, 0xbb, 0xbb, 0x9e, 0xcf, 0x59, 0x1c, 0x46, 0xbb, 0x03, 0x22, 0x38, 0xf5, -- 0xa2, 0x46, 0xc8, 0x99, 0x60, 0xc8, 0xa6, 0xac, 0x91, 0x83, 0x1a, 0x09, 0xa8, 0x71, 0xf2, 0xbf, -- 0xd5, 0x6b, 0x3e, 0xf3, 0x99, 0x02, 0xed, 0xca, 0x27, 0x8d, 0xaf, 0xff, 0x5a, 0x84, 0xf2, 0xbe, -- 0x56, 0x40, 0x5f, 0x41, 0xb9, 0x17, 0xfb, 0x44, 0xf4, 0x3b, 0x76, 0x61, 0xbd, 0xb8, 0x5d, 0xbd, -- 0xbb, 0xd5, 0xb8, 0x48, 0xad, 0xf1, 0x5c, 0x03, 0xdb, 0x02, 0x0b, 0x27, 0x65, 0xa1, 0x7b, 0x50, -- 0x0a, 0x69, 0x37, 0xb2, 0xa7, 0xd6, 0x0b, 0xdb, 0xd5, 0xbb, 0xf5, 0x8b, 0xd9, 0x2d, 0xda, 0x8d, -- 0x14, 0x55, 0xe1, 0xd1, 0x43, 0x28, 0x7a, 0x61, 0x6c, 0x17, 0x15, 0xed, 0xce, 0xc5, 0xb4, 0xbd, -- 0xd6, 0xa1, 0x64, 0x35, 0xcb, 0xa3, 0xf3, 0x5a, 0x71, 0xaf, 0x75, 0xe8, 0x48, 0x1a, 0x7a, 0x08, -- 0x33, 0x03, 0x32, 0x60, 0xfc, 0xcc, 0x2e, 0x29, 0x81, 0xcd, 0x8b, 0x05, 0xf6, 0x15, 0x4e, 0x45, -- 0x4e, 0x38, 0xe8, 0x3e, 0x4c, 0x77, 0xfa, 0xc7, 0x94, 0xd9, 0xd3, 0x8a, 0xbc, 0x71, 0x31, 0xb9, -- 0xd9, 0x3f, 0x7e, 0xf1, 0x52, 0x71, 0x35, 0x43, 0x6e, 0x97, 0x77, 0x07, 0xd8, 0x9e, 0xb9, 0x6c, -- 0xbb, 0x4e, 0x77, 0x80, 0xf5, 0x76, 0x25, 0x5e, 0xd6, 0x39, 0x20, 0xe2, 0x94, 0xf1, 0x63, 0xbb, -- 0x7c, 0x59, 0x9d, 0xbf, 0xd5, 0x40, 0x5d, 0xe7, 0x84, 0x55, 0x3f, 0x86, 0xaa, 0x51, 0x7f, 0x74, -- 0x0d, 0xa6, 0xe3, 0x08, 0xfb, 0xc4, 0x2e, 0xac, 0x17, 0xb6, 0x4b, 0x8e, 0x1e, 0xa0, 0x05, 0x28, -- 0x0e, 0xf0, 0x50, 0xbd, 0x8b, 0x92, 0x23, 0x1f, 0x91, 0x0d, 0xe5, 0x37, 0x98, 0xf6, 0xbd, 0x40, -- 0xa8, 0x52, 0x97, 0x9c, 0x74, 0x88, 0x56, 0xa1, 0x12, 0x62, 0x9f, 0x44, 0xf4, 0x07, 0xa2, 0x8a, -- 0x68, 0x39, 0xd9, 0xb8, 0xfe, 0x00, 0x2a, 0xe9, 0xeb, 0x92, 0x0a, 0x5e, 0xcc, 0x39, 0x09, 0x44, -- 0x12, 0x2b, 0x1d, 0xca, 0x1c, 0xfa, 0x74, 0x40, 0x45, 0x12, 0x4f, 0x0f, 0xea, 0x3f, 0x17, 0xa0, -- 0x9c, 0xbc, 0x34, 0xf4, 0x85, 0x99, 0xe5, 0x67, 0xcb, 0xb5, 0xd7, 0x3a, 0x3c, 0x94, 0xc8, 0x74, -- 0x27, 0x4d, 0x00, 0xd1, 0xe3, 0x4c, 0x88, 0x3e, 0x0d, 0xfc, 0xcb, 0xcd, 0x75, 0xa0, 0xb1, 0xc4, -- 0x31, 0x58, 0xf5, 0xb7, 0x50, 0x49, 0x65, 0x65, 0xae, 0x82, 0x09, 0xdc, 0x4f, 0xeb, 0xa5, 0x06, -- 0x68, 0x19, 0x66, 0x8e, 0x09, 0x0f, 0x48, 0x3f, 0xd9, 0x42, 0x32, 0x42, 0x08, 0x4a, 0x71, 0x44, -- 0x78, 0x52, 0x32, 0xf5, 0x8c, 0x36, 0xa0, 0x1c, 0x12, 0xee, 0x4a, 0xd3, 0x96, 0xd6, 0x8b, 0xdb, -- 0xa5, 0x26, 0x8c, 0xce, 0x6b, 0x33, 0x2d, 0xc2, 0xa5, 0x29, 0x67, 0x42, 0xc2, 0xf7, 0xc2, 0xb8, -- 0x3e, 0x84, 0x4a, 0x9a, 0x8a, 0x2c, 0x5c, 0x48, 0x38, 0x65, 0xdd, 0x28, 0x2d, 0x5c, 0x32, 0x44, -- 0x3b, 0xb0, 0x98, 0xa4, 0x49, 0xba, 0x6e, 0x8a, 0xd1, 0x19, 0x2c, 0x64, 0x0b, 0xad, 0x04, 0xbc, -- 0x05, 0xf3, 0x39, 0x58, 0xd0, 0x01, 0x49, 0xb2, 0x9a, 0xcb, 0x66, 0x0f, 0xe8, 0x80, 0xd4, 0xff, -- 0xac, 0x02, 0xe4, 0x56, 0x97, 0xfb, 0xf5, 0xb0, 0xd7, 0xcb, 0xfc, 0xa1, 0x06, 0x68, 0x05, 0x8a, -- 0x3c, 0x4a, 0x42, 0xe9, 0x13, 0xe5, 0xb4, 0xdb, 0x8e, 0x9c, 0x43, 0xff, 0x82, 0x0a, 0x8f, 0x22, -- 0x57, 0x1e, 0x6b, 0x1d, 0xa0, 0x59, 0x1d, 0x9d, 0xd7, 0xca, 0x4e, 0xbb, 0x2d, 0x6d, 0xe7, 0x94, -- 0x79, 0x14, 0xc9, 0x07, 0x54, 0x83, 0xea, 0x00, 0x87, 0x21, 0xe9, 0xba, 0x6f, 0x68, 0x5f, 0x3b, -- 0xa7, 0xe4, 0x80, 0x9e, 0x7a, 0x46, 0xfb, 0xaa, 0xd2, 0x5d, 0xca, 0xc5, 0x99, 0x3a, 0x5c, 0x25, -- 0x47, 0x0f, 0xd0, 0x2d, 0xb0, 0x4e, 0x39, 0x15, 0xa4, 0x83, 0xbd, 0x63, 0x75, 0x78, 0x4a, 0x4e, -- 0x3e, 0x81, 0x6c, 0xa8, 0x84, 0xbe, 0x1b, 0xfa, 0x2e, 0x0d, 0xec, 0xb2, 0x7e, 0x13, 0xa1, 0xdf, -- 0xf2, 0x5f, 0x04, 0x68, 0x15, 0x2c, 0xbd, 0xc2, 0x62, 0x61, 0x57, 0x92, 0x32, 0xfa, 0x2d, 0xff, -- 0x65, 0x2c, 0xd0, 0x8a, 0x62, 0xbd, 0xc1, 0x71, 0x5f, 0xd8, 0x56, 0xba, 0xf4, 0x4c, 0x0e, 0xd1, -- 0x3a, 0xcc, 0x86, 0xbe, 0x3b, 0xc0, 0x47, 0xc9, 0x32, 0xe8, 0x34, 0x43, 0x7f, 0x1f, 0x1f, 0x69, -- 0xc4, 0x06, 0xcc, 0xd1, 0x00, 0x7b, 0x82, 0x9e, 0x10, 0x17, 0x07, 0x2c, 0xb0, 0xab, 0x0a, 0x32, -- 0x9b, 0x4e, 0x3e, 0x0e, 0x58, 0x20, 0x37, 0x6b, 0x42, 0x66, 0xb5, 0x8a, 0x01, 0x30, 0x55, 0x54, -- 0x3d, 0xe6, 0xc6, 0x55, 0x54, 0x45, 0x72, 0x15, 0x05, 0x99, 0x37, 0x55, 0x14, 0x60, 0x1d, 0xaa, -- 0x71, 0x40, 0x4e, 0xa8, 0x27, 0x70, 0xa7, 0x4f, 0xec, 0xab, 0x0a, 0x60, 0x4e, 0xa1, 0x07, 0xb0, -- 0xd2, 0xa3, 0x84, 0x63, 0xee, 0xf5, 0xa8, 0x87, 0xfb, 0xae, 0xfe, 0x90, 0xb9, 0xfa, 0xf8, 0x2d, -- 0x28, 0xfc, 0x0d, 0x13, 0xa0, 0x9d, 0xf0, 0x8d, 0x5c, 0x46, 0xf7, 0x60, 0x6c, 0xc9, 0x8d, 0x4e, -- 0x71, 0x98, 0x30, 0x17, 0x15, 0xf3, 0xba, 0xb9, 0xdc, 0x3e, 0xc5, 0xa1, 0xe6, 0xd5, 0xa0, 0xaa, -- 0x4e, 0x89, 0xab, 0x8d, 0x84, 0x74, 0xda, 0x6a, 0x6a, 0x4f, 0xb9, 0xe9, 0x3f, 0x60, 0x69, 0x80, -- 0xf4, 0xd4, 0x92, 0xf2, 0xcc, 0xec, 0xe8, 0xbc, 0x56, 0x39, 0x90, 0x93, 0xd2, 0x58, 0x15, 0xb5, -- 0xec, 0x44, 0x11, 0xba, 0x07, 0xf3, 0x19, 0x54, 0x7b, 0xec, 0x9a, 0xc2, 0x2f, 0x8c, 0xce, 0x6b, -- 0xb3, 0x29, 0x5e, 0x19, 0x6d, 0x36, 0xe5, 0x28, 0xb7, 0xfd, 0x17, 0x16, 0x35, 0xcf, 0xf4, 0xdc, -- 0x75, 0x95, 0xc9, 0x55, 0xb5, 0xb0, 0x9f, 0x1b, 0x2f, 0xcb, 0x57, 0xdb, 0x6f, 0xd9, 0xc8, 0xf7, -- 0x89, 0xf2, 0xe0, 0xbf, 0x41, 0x73, 0xdc, 0xdc, 0x89, 0x37, 0x14, 0x48, 0xe7, 0xf6, 0x3a, 0xb3, -- 0xe3, 0x46, 0x9a, 0x6d, 0x66, 0x4a, 0x5b, 0xbf, 0x12, 0x35, 0xdb, 0xd2, 0xce, 0xdc, 0x4a, 0xd5, -- 0x72, 0x7f, 0xae, 0xe8, 0x97, 0x9f, 0xa1, 0xa4, 0x49, 0x37, 0x0d, 0x2d, 0xed, 0xc5, 0xd5, 0x31, -- 0x94, 0x76, 0xe3, 0x0e, 0xa0, 0x0c, 0x95, 0xbb, 0xf6, 0xa6, 0xb1, 0xd1, 0x56, 0x6e, 0xdd, 0x06, -- 0x2c, 0x69, 0xf0, 0xb8, 0x81, 0x6f, 0x29, 0xb4, 0xae, 0xd7, 0x0b, 0xd3, 0xc5, 0x59, 0x11, 0x4d, -- 0xf4, 0x6d, 0x43, 0xfb, 0x71, 0x8e, 0xfd, 0x58, 0x5b, 0x95, 0x7c, 0xed, 0x13, 0xda, 0xaa, 0xe8, -- 0x1f, 0x6a, 0x2b, 0x74, 0xed, 0x23, 0x6d, 0x85, 0xdd, 0x49, 0xb1, 0xa6, 0xd9, 0xd7, 0x93, 0xcf, -- 0x9e, 0x5c, 0x38, 0x34, 0x1c, 0xff, 0x65, 0x7a, 0x75, 0xdc, 0x51, 0xdf, 0xfe, 0xad, 0xcb, 0x2e, -- 0xf8, 0xa7, 0x81, 0xe0, 0x67, 0xe9, 0xed, 0x71, 0x1f, 0x4a, 0xd2, 0xe5, 0x76, 0x7d, 0x12, 0xae, -- 0xa2, 0xa0, 0x47, 0xd9, 0x95, 0xb0, 0x31, 0x09, 0x39, 0xbd, 0x39, 0xda, 0x00, 0xfa, 0xc9, 0x15, -- 0x5e, 0x68, 0x6f, 0x4e, 0x20, 0xd1, 0x9c, 0x1b, 0x9d, 0xd7, 0xac, 0xaf, 0x15, 0xf9, 0x60, 0xaf, -- 0xe5, 0x58, 0x5a, 0xe7, 0xc0, 0x0b, 0xeb, 0x04, 0xaa, 0x06, 0x30, 0xbf, 0x77, 0x0b, 0xc6, 0xbd, -- 0x9b, 0x77, 0x04, 0x53, 0x9f, 0xe8, 0x08, 0x8a, 0x9f, 0xec, 0x08, 0x4a, 0x63, 0x1d, 0x41, 0xfd, -- 0xf7, 0x69, 0xb0, 0xb2, 0x86, 0x07, 0x61, 0x58, 0xa5, 0xcc, 0x8d, 0x08, 0x3f, 0xa1, 0x1e, 0x71, -- 0x3b, 0x67, 0x82, 0x44, 0x2e, 0x27, 0x5e, 0xcc, 0x23, 0x7a, 0x42, 0x92, 0x66, 0x71, 0xf3, 0x92, -- 0xce, 0x49, 0xd7, 0xe6, 0x06, 0x65, 0x6d, 0x2d, 0xd3, 0x94, 0x2a, 0x4e, 0x2a, 0x82, 0xbe, 0x83, -- 0xeb, 0x79, 0x88, 0xae, 0xa1, 0x3e, 0x35, 0x81, 0xfa, 0x52, 0xa6, 0xde, 0xcd, 0x95, 0x0f, 0x60, -- 0x89, 0x32, 0xf7, 0x6d, 0x4c, 0xe2, 0x31, 0xdd, 0xe2, 0x04, 0xba, 0x8b, 0x94, 0xbd, 0x52, 0xfc, -- 0x5c, 0xd5, 0x85, 0x15, 0xa3, 0x24, 0xf2, 0x2e, 0x36, 0xb4, 0x4b, 0x13, 0x68, 0x2f, 0x67, 0x39, -- 0xcb, 0xbb, 0x3b, 0x0f, 0xf0, 0x3d, 0x2c, 0x53, 0xe6, 0x9e, 0x62, 0x2a, 0x3e, 0x54, 0x9f, 0x9e, -- 0xac, 0x22, 0xaf, 0x31, 0x15, 0xe3, 0xd2, 0xba, 0x22, 0x03, 0xc2, 0xfd, 0xb1, 0x8a, 0xcc, 0x4c, -- 0x56, 0x91, 0x7d, 0xc5, 0xcf, 0x55, 0x5b, 0xb0, 0x48, 0xd9, 0x87, 0xb9, 0x96, 0x27, 0xd0, 0xbc, -- 0x4a, 0xd9, 0x78, 0x9e, 0xaf, 0x60, 0x31, 0x22, 0x9e, 0x60, 0xdc, 0x74, 0x5b, 0x65, 0x02, 0xc5, -- 0x85, 0x84, 0x9e, 0x49, 0xd6, 0x4f, 0x00, 0xf2, 0x75, 0x34, 0x0f, 0x53, 0x2c, 0x54, 0x47, 0xc7, -- 0x72, 0xa6, 0x58, 0x28, 0x7b, 0xc0, 0xae, 0xfc, 0xec, 0xe8, 0x83, 0x63, 0x39, 0xc9, 0x48, 0x9e, -- 0xa7, 0x01, 0x3e, 0x62, 0x69, 0x13, 0xa8, 0x07, 0x6a, 0x96, 0x06, 0x8c, 0x27, 0x67, 0x47, 0x0f, -- 0xe4, 0xec, 0x09, 0xee, 0xc7, 0x24, 0xed, 0x79, 0xd4, 0xa0, 0xfe, 0x53, 0x01, 0x2a, 0xe9, 0x6f, -- 0x00, 0x7a, 0x64, 0xb6, 0xd1, 0xc5, 0xcf, 0xff, 0x75, 0x48, 0x92, 0xde, 0x4c, 0xd6, 0x6b, 0xdf, -- 0xcf, 0x7b, 0xed, 0x7f, 0x4c, 0x4e, 0x1a, 0x72, 0x02, 0x56, 0x36, 0x67, 0xec, 0xb6, 0x30, 0xb6, -- 0xdb, 0x1a, 0x54, 0x7b, 0x1e, 0x76, 0x7b, 0x38, 0xe8, 0xf6, 0x89, 0xee, 0x10, 0xe7, 0x1c, 0xe8, -- 0x79, 0xf8, 0xb9, 0x9e, 0x49, 0x01, 0xac, 0x73, 0x44, 0x3c, 0x11, 0xa9, 0xa2, 0x68, 0xc0, 0x4b, -- 0x3d, 0x53, 0xff, 0x65, 0x0a, 0xaa, 0xc6, 0x9f, 0x8b, 0xec, 0xa1, 0x03, 0x3c, 0x48, 0xe3, 0xa8, -- 0x67, 0xd9, 0xb1, 0xf1, 0xa1, 0xfe, 0x96, 0x24, 0x9f, 0xa9, 0x32, 0x1f, 0xaa, 0x8f, 0x02, 0xba, -- 0x0d, 0xc0, 0x87, 0x6e, 0x88, 0xbd, 0x63, 0x92, 0xc8, 0x97, 0x1c, 0x8b, 0x0f, 0x5b, 0x7a, 0x02, -- 0xdd, 0x04, 0x8b, 0x0f, 0x5d, 0xc2, 0x39, 0xe3, 0x51, 0x52, 0xfb, 0x0a, 0x1f, 0x3e, 0x55, 0xe3, -- 0x84, 0xdb, 0xe5, 0x4c, 0xf6, 0x02, 0xc9, 0x3b, 0xb0, 0xf8, 0xf0, 0x89, 0x9e, 0x90, 0x51, 0x45, -- 0x1a, 0x55, 0xb7, 0x9e, 0x65, 0x91, 0x47, 0x15, 0x79, 0x54, 0xdd, 0x7a, 0x5a, 0xc2, 0x8c, 0x2a, -- 0xb2, 0xa8, 0xba, 0xfb, 0xac, 0x08, 0x23, 0xaa, 0xc8, 0xa3, 0x5a, 0x29, 0x37, 0x89, 0xda, 0xb4, -- 0xdf, 0xbd, 0x5f, 0xbb, 0xf2, 0xc7, 0xfb, 0xb5, 0x2b, 0x3f, 0x8e, 0xd6, 0x0a, 0xef, 0x46, 0x6b, -- 0x85, 0xdf, 0x46, 0x6b, 0x85, 0xbf, 0x46, 0x6b, 0x85, 0xce, 0x8c, 0xfa, 0x0d, 0xff, 0xff, 0xdf, -- 0x01, 0x00, 0x00, 0xff, 0xff, 0x19, 0x9d, 0xe2, 0xd3, 0xe5, 0x0f, 0x00, 0x00, -+ // 1542 bytes of a gzipped FileDescriptorProto -+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x57, 0x5f, 0x53, 0xdb, 0xc6, -+ 0x17, 0x1d, 0x63, 0x83, 0xad, 0x6b, 0x20, 0xb0, 0x24, 0x44, 0x90, 0xe4, 0x67, 0x62, 0xe0, 0x57, -+ 0x5a, 0x66, 0xcc, 0x34, 0x9d, 0x49, 0x9b, 0x34, 0x99, 0x4e, 0x20, 0xc9, 0x24, 0xd3, 0xd2, 0x38, -+ 0x32, 0x4c, 0xda, 0x27, 0xcd, 0x5a, 0xde, 0xc8, 0x0b, 0xb6, 0x56, 0x59, 0xad, 0xc0, 0xf4, 0xb9, -+ 0x33, 0x7d, 0xea, 0xc7, 0xeb, 0x43, 0x5f, 0x78, 0xe0, 0xbd, 0xdf, 0xa1, 0xb3, 0x77, 0xf5, 0xcf, -+ 0x09, 0x84, 0xfa, 0x4d, 0xbb, 0x7b, 0xce, 0xb9, 0x77, 0xef, 0x9e, 0x95, 0xae, 0x60, 0xdb, 0xe7, -+ 0xaa, 0x1f, 0x77, 0x5b, 0x9e, 0x18, 0xee, 0x78, 0x22, 0x50, 0x94, 0x07, 0x4c, 0xf6, 0x76, 0x3c, -+ 0x5f, 0x8a, 0x38, 0x8c, 0x76, 0x86, 0x4c, 0x49, 0xee, 0x45, 0xad, 0x50, 0x0a, 0x25, 0x88, 0xcd, -+ 0x45, 0x2b, 0x07, 0xb5, 0x12, 0x50, 0xeb, 0xe4, 0xeb, 0xd5, 0x9b, 0xbe, 0xf0, 0x05, 0x82, 0x76, -+ 0xf4, 0x93, 0xc1, 0x37, 0xff, 0x29, 0x43, 0x75, 0xdf, 0x28, 0x90, 0x1f, 0xa0, 0xda, 0x8f, 0x7d, -+ 0xa6, 0x06, 0x5d, 0xbb, 0xb4, 0x56, 0xde, 0xaa, 0x3f, 0xd8, 0x6c, 0x5d, 0xa5, 0xd6, 0x7a, 0x65, -+ 0x80, 0x1d, 0x45, 0x95, 0x93, 0xb2, 0xc8, 0x43, 0xa8, 0x84, 0xbc, 0x17, 0xd9, 0x53, 0x6b, 0xa5, -+ 0xad, 0xfa, 0x83, 0xe6, 0xd5, 0xec, 0x36, 0xef, 0x45, 0x48, 0x45, 0x3c, 0x79, 0x02, 0x65, 0x2f, -+ 0x8c, 0xed, 0x32, 0xd2, 0xee, 0x5f, 0x4d, 0xdb, 0x6b, 0x1f, 0x6a, 0xd6, 0x6e, 0xf5, 0xe2, 0xbc, -+ 0x51, 0xde, 0x6b, 0x1f, 0x3a, 0x9a, 0x46, 0x9e, 0xc0, 0xcc, 0x90, 0x0d, 0x85, 0x3c, 0xb3, 0x2b, -+ 0x28, 0xb0, 0x71, 0xb5, 0xc0, 0x3e, 0xe2, 0x30, 0x72, 0xc2, 0x21, 0x8f, 0x60, 0xba, 0x3b, 0x38, -+ 0xe6, 0xc2, 0x9e, 0x46, 0xf2, 0xfa, 0xd5, 0xe4, 0xdd, 0xc1, 0xf1, 0xeb, 0x37, 0xc8, 0x35, 0x0c, -+ 0xbd, 0x5d, 0xd9, 0x1b, 0x52, 0x7b, 0xe6, 0xba, 0xed, 0x3a, 0xbd, 0x21, 0x35, 0xdb, 0xd5, 0x78, -+ 0x5d, 0xe7, 0x80, 0xa9, 0x53, 0x21, 0x8f, 0xed, 0xea, 0x75, 0x75, 0xfe, 0xd9, 0x00, 0x4d, 0x9d, -+ 0x13, 0x96, 0xce, 0xf9, 0x3d, 0x1f, 0xb0, 0xc8, 0xae, 0x5d, 0x97, 0xf3, 0x4b, 0x0d, 0x33, 0x39, -+ 0x23, 0xa3, 0x79, 0x0c, 0xf5, 0xc2, 0xd1, 0x91, 0x9b, 0x30, 0x1d, 0x47, 0xd4, 0x67, 0x76, 0x69, -+ 0xad, 0xb4, 0x55, 0x71, 0xcc, 0x80, 0x2c, 0x40, 0x79, 0x48, 0x47, 0x78, 0x8c, 0x15, 0x47, 0x3f, -+ 0x12, 0x1b, 0xaa, 0xef, 0x29, 0x1f, 0x78, 0x81, 0xc2, 0x53, 0xaa, 0x38, 0xe9, 0x90, 0xac, 0x42, -+ 0x2d, 0xa4, 0x3e, 0x8b, 0xf8, 0x6f, 0x0c, 0xeb, 0x6f, 0x39, 0xd9, 0xb8, 0xf9, 0x18, 0x6a, 0xe9, -+ 0x49, 0x6b, 0x05, 0x2f, 0x96, 0x92, 0x05, 0x2a, 0x89, 0x95, 0x0e, 0x75, 0x0e, 0x03, 0x3e, 0xe4, -+ 0x2a, 0x89, 0x67, 0x06, 0xcd, 0x3f, 0x4a, 0x50, 0x4d, 0xce, 0x9b, 0x7c, 0x57, 0xcc, 0xf2, 0xb3, -+ 0x95, 0xde, 0x6b, 0x1f, 0x1e, 0x6a, 0x64, 0xba, 0x93, 0x5d, 0x00, 0xd5, 0x97, 0x42, 0xa9, 0x01, -+ 0x0f, 0xfc, 0xeb, 0x7d, 0x79, 0x60, 0xb0, 0xcc, 0x29, 0xb0, 0x9a, 0x1f, 0xa0, 0x96, 0xca, 0xea, -+ 0x5c, 0x95, 0x50, 0x74, 0x90, 0xd6, 0x0b, 0x07, 0x64, 0x19, 0x66, 0x8e, 0x99, 0x0c, 0xd8, 0x20, -+ 0xd9, 0x42, 0x32, 0x22, 0x04, 0x2a, 0x71, 0xc4, 0x64, 0x52, 0x32, 0x7c, 0x26, 0xeb, 0x50, 0x0d, -+ 0x99, 0x74, 0xb5, 0xdf, 0x2b, 0x6b, 0xe5, 0xad, 0xca, 0x2e, 0x5c, 0x9c, 0x37, 0x66, 0xda, 0x4c, -+ 0x6a, 0x3f, 0xcf, 0x84, 0x4c, 0xee, 0x85, 0x71, 0x73, 0x04, 0xb5, 0x34, 0x15, 0x5d, 0xb8, 0x90, -+ 0x49, 0x2e, 0x7a, 0x51, 0x5a, 0xb8, 0x64, 0x48, 0xb6, 0x61, 0x31, 0x49, 0x93, 0xf5, 0xdc, 0x14, -+ 0x63, 0x32, 0x58, 0xc8, 0x16, 0xda, 0x09, 0x78, 0x13, 0xe6, 0x73, 0xb0, 0xe2, 0x43, 0x96, 0x64, -+ 0x35, 0x97, 0xcd, 0x1e, 0xf0, 0x21, 0x6b, 0xfe, 0x5d, 0x07, 0xc8, 0x6f, 0x89, 0xde, 0xaf, 0x47, -+ 0xbd, 0x7e, 0xe6, 0x0f, 0x1c, 0x90, 0x15, 0x28, 0xcb, 0x28, 0x09, 0x65, 0x2e, 0xa3, 0xd3, 0xe9, -+ 0x38, 0x7a, 0x8e, 0xfc, 0x1f, 0x6a, 0x32, 0x8a, 0x5c, 0xfd, 0x46, 0x30, 0x01, 0x76, 0xeb, 0x17, -+ 0xe7, 0x8d, 0xaa, 0xd3, 0xe9, 0x68, 0xdb, 0x39, 0x55, 0x19, 0x45, 0xfa, 0x81, 0x34, 0xa0, 0x3e, -+ 0xa4, 0x61, 0xc8, 0x7a, 0xae, 0xf6, 0x25, 0x3a, 0xa7, 0xe2, 0x80, 0x99, 0xd2, 0xa6, 0xd5, 0x91, -+ 0x7b, 0x5c, 0xaa, 0x33, 0xbc, 0x97, 0x15, 0xc7, 0x0c, 0xc8, 0x5d, 0xb0, 0x4e, 0x25, 0x57, 0xac, -+ 0x4b, 0xbd, 0x63, 0xbc, 0x77, 0x15, 0x27, 0x9f, 0x20, 0x36, 0xd4, 0x42, 0xdf, 0x0d, 0x7d, 0x97, -+ 0x07, 0x76, 0xd5, 0x9c, 0x44, 0xe8, 0xb7, 0xfd, 0xd7, 0x01, 0x59, 0x05, 0xcb, 0xac, 0x88, 0x58, -+ 0xe1, 0xad, 0xd1, 0x65, 0xf4, 0xdb, 0xfe, 0x9b, 0x58, 0x91, 0x15, 0x64, 0xbd, 0xa7, 0xf1, 0x40, -+ 0xd9, 0x56, 0xba, 0xf4, 0x52, 0x0f, 0xc9, 0x1a, 0xcc, 0x86, 0xbe, 0x3b, 0xa4, 0x47, 0xc9, 0x32, -+ 0x98, 0x34, 0x43, 0x7f, 0x9f, 0x1e, 0x19, 0xc4, 0x3a, 0xcc, 0xf1, 0x80, 0x7a, 0x8a, 0x9f, 0x30, -+ 0x97, 0x06, 0x22, 0xb0, 0xeb, 0x08, 0x99, 0x4d, 0x27, 0x9f, 0x05, 0x22, 0xd0, 0x9b, 0x2d, 0x42, -+ 0x66, 0x8d, 0x4a, 0x01, 0x50, 0x54, 0xc1, 0x7a, 0xcc, 0x8d, 0xab, 0x60, 0x45, 0x72, 0x15, 0x84, -+ 0xcc, 0x17, 0x55, 0x10, 0xb0, 0x06, 0xf5, 0x38, 0x60, 0x27, 0xdc, 0x53, 0xb4, 0x3b, 0x60, 0xf6, -+ 0x0d, 0x04, 0x14, 0xa7, 0xc8, 0x63, 0x58, 0xe9, 0x73, 0x26, 0xa9, 0xf4, 0xfa, 0xdc, 0xa3, 0x03, -+ 0xd7, 0xbc, 0x03, 0x5d, 0x73, 0xfd, 0x16, 0x10, 0x7f, 0xbb, 0x08, 0x30, 0x4e, 0xf8, 0x49, 0x2f, -+ 0x93, 0x87, 0x30, 0xb6, 0xe4, 0x46, 0xa7, 0x34, 0x4c, 0x98, 0x8b, 0xc8, 0xbc, 0x55, 0x5c, 0xee, -+ 0x9c, 0xd2, 0xd0, 0xf0, 0x1a, 0x50, 0xc7, 0x5b, 0xe2, 0x1a, 0x23, 0x11, 0x93, 0x36, 0x4e, 0xed, -+ 0xa1, 0x9b, 0xbe, 0x04, 0xcb, 0x00, 0xb4, 0xa7, 0x96, 0xd0, 0x33, 0xb3, 0x17, 0xe7, 0x8d, 0xda, -+ 0x81, 0x9e, 0xd4, 0xc6, 0xaa, 0xe1, 0xb2, 0x13, 0x45, 0xe4, 0x21, 0xcc, 0x67, 0x50, 0xe3, 0xb1, -+ 0x9b, 0x88, 0x5f, 0xb8, 0x38, 0x6f, 0xcc, 0xa6, 0x78, 0x34, 0xda, 0x6c, 0xca, 0x41, 0xb7, 0x7d, -+ 0x05, 0x8b, 0x86, 0x57, 0xf4, 0xdc, 0x2d, 0xcc, 0xe4, 0x06, 0x2e, 0xec, 0xe7, 0xc6, 0xcb, 0xf2, -+ 0x35, 0xf6, 0x5b, 0x2e, 0xe4, 0xfb, 0x1c, 0x3d, 0xf8, 0x05, 0x18, 0x8e, 0x9b, 0x3b, 0xf1, 0x36, -+ 0x82, 0x4c, 0x6e, 0xef, 0x32, 0x3b, 0xae, 0xa7, 0xd9, 0x66, 0xa6, 0xb4, 0xcd, 0x91, 0xe0, 0x6c, -+ 0xdb, 0x38, 0x73, 0x33, 0x55, 0xcb, 0xfd, 0xb9, 0x62, 0x0e, 0x3f, 0x43, 0x69, 0x93, 0x6e, 0x14, -+ 0xb4, 0x8c, 0x17, 0x57, 0xc7, 0x50, 0xc6, 0x8d, 0xdb, 0x40, 0x32, 0x54, 0xee, 0xda, 0x3b, 0x85, -+ 0x8d, 0xb6, 0x73, 0xeb, 0xb6, 0x60, 0xc9, 0x80, 0xc7, 0x0d, 0x7c, 0x17, 0xd1, 0xa6, 0x5e, 0xaf, -+ 0x8b, 0x2e, 0xce, 0x8a, 0x58, 0x44, 0xdf, 0x2b, 0x68, 0x3f, 0xcb, 0xb1, 0x9f, 0x6a, 0x63, 0xc9, -+ 0xff, 0x77, 0x89, 0x36, 0x16, 0xfd, 0x63, 0x6d, 0x44, 0x37, 0x3e, 0xd1, 0x46, 0xec, 0x76, 0x8a, -+ 0x2d, 0x9a, 0x7d, 0x2d, 0x79, 0xed, 0xe9, 0x85, 0xc3, 0x82, 0xe3, 0xbf, 0x4f, 0x3f, 0x1d, 0xf7, -+ 0xf1, 0xdd, 0xbf, 0x79, 0x5d, 0x6f, 0xf0, 0x22, 0x50, 0xf2, 0x2c, 0xfd, 0x7a, 0x3c, 0x82, 0x8a, -+ 0x76, 0xb9, 0xdd, 0x9c, 0x84, 0x8b, 0x14, 0xf2, 0x34, 0xfb, 0x24, 0xac, 0x4f, 0x42, 0x4e, 0xbf, -+ 0x1c, 0x1d, 0x00, 0xf3, 0xe4, 0x2a, 0x2f, 0xb4, 0x37, 0x26, 0x90, 0xd8, 0x9d, 0xbb, 0x38, 0x6f, -+ 0x58, 0x3f, 0x22, 0xf9, 0x60, 0xaf, 0xed, 0x58, 0x46, 0xe7, 0xc0, 0x0b, 0x9b, 0x0c, 0xea, 0x05, -+ 0x60, 0xfe, 0xdd, 0x2d, 0x15, 0xbe, 0xbb, 0x79, 0x47, 0x30, 0x75, 0x49, 0x47, 0x50, 0xbe, 0xb4, -+ 0x23, 0xa8, 0x8c, 0x75, 0x04, 0xcd, 0xbf, 0xa6, 0xc1, 0xca, 0x7a, 0x25, 0x42, 0x61, 0x95, 0x0b, -+ 0x37, 0x62, 0xf2, 0x84, 0x7b, 0xcc, 0xed, 0x9e, 0x29, 0x16, 0xb9, 0x92, 0x79, 0xb1, 0x8c, 0xf8, -+ 0x09, 0x4b, 0xfa, 0xcc, 0x8d, 0x6b, 0x9a, 0x2e, 0x53, 0x9b, 0xdb, 0x5c, 0x74, 0x8c, 0xcc, 0xae, -+ 0x56, 0x71, 0x52, 0x11, 0xf2, 0x0b, 0xdc, 0xca, 0x43, 0xf4, 0x0a, 0xea, 0x53, 0x13, 0xa8, 0x2f, -+ 0x65, 0xea, 0xbd, 0x5c, 0xf9, 0x00, 0x96, 0xb8, 0x70, 0x3f, 0xc4, 0x2c, 0x1e, 0xd3, 0x2d, 0x4f, -+ 0xa0, 0xbb, 0xc8, 0xc5, 0x5b, 0xe4, 0xe7, 0xaa, 0x2e, 0xac, 0x14, 0x4a, 0xa2, 0xbf, 0xc5, 0x05, -+ 0xed, 0xca, 0x04, 0xda, 0xcb, 0x59, 0xce, 0xfa, 0xdb, 0x9d, 0x07, 0xf8, 0x15, 0x96, 0xb9, 0x70, -+ 0x4f, 0x29, 0x57, 0x1f, 0xab, 0x4f, 0x4f, 0x56, 0x91, 0x77, 0x94, 0xab, 0x71, 0x69, 0x53, 0x91, -+ 0x21, 0x93, 0xfe, 0x58, 0x45, 0x66, 0x26, 0xab, 0xc8, 0x3e, 0xf2, 0x73, 0xd5, 0x36, 0x2c, 0x72, -+ 0xf1, 0x71, 0xae, 0xd5, 0x09, 0x34, 0x6f, 0x70, 0x31, 0x9e, 0xe7, 0x5b, 0x58, 0x8c, 0x98, 0xa7, -+ 0x84, 0x2c, 0xba, 0xad, 0x36, 0x81, 0xe2, 0x42, 0x42, 0xcf, 0x24, 0x9b, 0x27, 0x00, 0xf9, 0x3a, -+ 0x99, 0x87, 0x29, 0x11, 0xe2, 0xd5, 0xb1, 0x9c, 0x29, 0x11, 0xea, 0x1e, 0xb0, 0xa7, 0x5f, 0x3b, -+ 0xe6, 0xe2, 0x58, 0x4e, 0x32, 0xd2, 0xf7, 0x69, 0x48, 0x8f, 0x44, 0xda, 0x04, 0x9a, 0x01, 0xce, -+ 0xf2, 0x40, 0xc8, 0xe4, 0xee, 0x98, 0x81, 0x9e, 0x3d, 0xa1, 0x83, 0x98, 0xa5, 0x3d, 0x0f, 0x0e, -+ 0x9a, 0xbf, 0x97, 0xa0, 0x96, 0xfe, 0x41, 0x90, 0xa7, 0xc5, 0x36, 0xba, 0xfc, 0xf9, 0xe6, 0x5f, -+ 0x93, 0xcc, 0x66, 0xb2, 0x5e, 0xfb, 0x51, 0xde, 0x6b, 0xff, 0x67, 0x72, 0xd2, 0x90, 0x33, 0xb0, -+ 0xb2, 0xb9, 0xc2, 0x6e, 0x4b, 0x63, 0xbb, 0x6d, 0x40, 0xbd, 0xef, 0x51, 0xb7, 0x4f, 0x83, 0x9e, -+ 0xfe, 0x3f, 0xd1, 0xa5, 0x98, 0x73, 0xa0, 0xef, 0xd1, 0x57, 0x66, 0x26, 0x05, 0x88, 0xee, 0x11, -+ 0xf3, 0x54, 0x84, 0x45, 0x31, 0x80, 0x37, 0x66, 0xa6, 0xf9, 0xe7, 0x14, 0xd4, 0x0b, 0x3f, 0x3d, -+ 0xba, 0x87, 0x0e, 0xe8, 0x30, 0x8d, 0x83, 0xcf, 0xba, 0x63, 0x93, 0x23, 0xf3, 0x2e, 0x49, 0x5e, -+ 0x53, 0x55, 0x39, 0xc2, 0x97, 0x02, 0xb9, 0x07, 0x20, 0x47, 0x6e, 0x48, 0xbd, 0x63, 0x96, 0xc8, -+ 0x57, 0x1c, 0x4b, 0x8e, 0xda, 0x66, 0x82, 0xdc, 0x01, 0x4b, 0x8e, 0x5c, 0x26, 0xa5, 0x90, 0x51, -+ 0x52, 0xfb, 0x9a, 0x1c, 0xbd, 0xc0, 0x71, 0xc2, 0xed, 0x49, 0xa1, 0x7b, 0x81, 0xe4, 0x0c, 0x2c, -+ 0x39, 0x7a, 0x6e, 0x26, 0x74, 0x54, 0x95, 0x46, 0x35, 0xad, 0x67, 0x55, 0xe5, 0x51, 0x55, 0x1e, -+ 0xd5, 0xb4, 0x9e, 0x96, 0x2a, 0x46, 0x55, 0x59, 0x54, 0xd3, 0x7d, 0xd6, 0x54, 0x21, 0xaa, 0xca, -+ 0xa3, 0x5a, 0x29, 0x37, 0x89, 0xda, 0xfc, 0x16, 0xac, 0xec, 0x27, 0xee, 0x8a, 0xdf, 0xb5, 0x4b, -+ 0x7f, 0xa0, 0xba, 0x33, 0xf8, 0x83, 0xff, 0xcd, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x8a, 0x90, -+ 0xfa, 0xea, 0x3f, 0x10, 0x00, 0x00, - } -diff --git a/vendor/github.com/containerd/cgroups/subsystem.go b/vendor/github.com/containerd/cgroups/subsystem.go -index 23de04d..237fc4c 100644 ---- a/vendor/github.com/containerd/cgroups/subsystem.go -+++ b/vendor/github.com/containerd/cgroups/subsystem.go -@@ -39,6 +39,7 @@ const ( - Memory Name = "memory" - Blkio Name = "blkio" - Rdma Name = "rdma" -+ Files Name = "files" - ) - - // Subsystems returns a complete list of the default cgroups -@@ -57,6 +58,7 @@ func Subsystems() []Name { - Memory, - Blkio, - Rdma, -+ Files, - } - if !isUserNS { - n = append(n, Devices) -diff --git a/vendor/github.com/containerd/cgroups/utils.go b/vendor/github.com/containerd/cgroups/utils.go -index 82dbe2d..820b516 100644 ---- a/vendor/github.com/containerd/cgroups/utils.go -+++ b/vendor/github.com/containerd/cgroups/utils.go -@@ -69,7 +69,6 @@ func defaults(root string) ([]Subsystem, error) { - return nil, err - } - s := []Subsystem{ -- NewNamed(root, "systemd"), - NewFreezer(root), - NewPids(root), - NewNetCls(root), -@@ -81,6 +80,7 @@ func defaults(root string) ([]Subsystem, error) { - NewMemory(root), - NewBlkio(root), - NewRdma(root), -+ NewFiles(root), - } - // only add the devices cgroup if we are not in a user namespace - // because modifications are not allowed -diff --git a/vendor/github.com/opencontainers/runtime-spec/specs-go/config.go b/vendor/github.com/opencontainers/runtime-spec/specs-go/config.go -index 27268f9..7a1acf2 100644 ---- a/vendor/github.com/opencontainers/runtime-spec/specs-go/config.go -+++ b/vendor/github.com/opencontainers/runtime-spec/specs-go/config.go -@@ -332,6 +332,10 @@ type LinuxRdma struct { - HcaObjects *uint32 `json:"hcaObjects,omitempty"` - } - -+type Files struct { -+ Limit *uint64 `json:"limit,omitempty"` -+} -+ - // LinuxResources has container runtime resource constraints - type LinuxResources struct { - // Devices configures the device whitelist. -@@ -352,6 +356,8 @@ type LinuxResources struct { - // Limits are a set of key value pairs that define RDMA resource limits, - // where the key is device name and value is resource limits. - Rdma map[string]LinuxRdma `json:"rdma,omitempty"` -+ // Files resource restriction configuration. -+ Files *Files `json:"files,omitempty"` - } - - // LinuxDevice represents the mknod information for a Linux special device file -diff --git a/virtcontainers/cgroups.go b/virtcontainers/cgroups.go -index e8c5a7b..1b6b04d 100644 ---- a/virtcontainers/cgroups.go -+++ b/virtcontainers/cgroups.go -@@ -11,6 +11,7 @@ import ( - "context" - "encoding/json" - "fmt" -+ "io/ioutil" - "os" - "path/filepath" - "strconv" -@@ -35,6 +36,12 @@ const ( - cgroupKataPath = "/kata/" - vcpuCgroupName = "vcpu" - emulatorCgroupName = "emulator" -+ fileCgroupName = "files" -+ -+ defaultMinFilesLimit uint64 = 1024 -+ defaultMaxContainers uint64 = 200 -+ -+ procFileMaxPath = "/proc/sys/fs/file-max" - - // BlkioThrottleReadBps is the key to fetch throttle_read_bps - BlkioThrottleReadBps = "throttle_read_bps" -@@ -414,6 +421,81 @@ func isInSlice(i int, s []int) bool { - return false - } - -+// supportFileCgroup check current host OS support files cgroup or not -+func supportFileCgroup() bool { -+ root, err := cgroupV1MountPoint() -+ if err != nil { -+ return false -+ } -+ -+ filesCgroupPath := root + "/" + fileCgroupName -+ _, err = os.Stat(filesCgroupPath) -+ if err != nil { -+ return false -+ } -+ -+ return true -+} -+ -+func (s *Sandbox) filesResource(fileMaxPath string) *specs.Files { -+ var filesLimit uint64 = 0 -+ var err error -+ -+ // get fileslimit from sandbox container spec -+ sandboxContainerSpec := s.GetPatchedOCISpec() -+ if sandboxContainerSpec == nil || sandboxContainerSpec.Linux == nil { -+ return nil -+ } -+ -+ var configFl uint64 = 0 -+ sandboxResources := sandboxContainerSpec.Linux.Resources -+ if sandboxResources != nil && sandboxResources.Files != nil && sandboxResources.Files.Limit != nil { -+ configFl = *sandboxResources.Files.Limit -+ } -+ -+ if configFl > 0 { -+ if configFl < defaultMinFilesLimit { -+ logrus.Warnf("At least %d fds are needed to support vm running", defaultMinFilesLimit) -+ filesLimit = defaultMinFilesLimit -+ } else { -+ filesLimit = configFl -+ } -+ } else { -+ filesLimit, err = getDefFilesLimitFromHost(fileMaxPath) -+ if err != nil { -+ // do nothing but print the warning log -+ logrus.Errorf("get default files limit failed : %#v", err) -+ return nil -+ } -+ } -+ -+ return &specs.Files{ -+ Limit: &filesLimit, -+ } -+} -+ -+// getDefFilesLimitFromHost read default file-max files limit value from host -+func getDefFilesLimitFromHost(fileMaxPath string) (uint64, error) { -+ data, err := ioutil.ReadFile(fileMaxPath) -+ if err != nil { -+ return 0, fmt.Errorf("read file-max failed : %#v", err) -+ } -+ -+ tmpFilesLimit, err := strconv.Atoi(strings.Replace(string(data), "\n", "", -1)) -+ if err != nil { -+ return 0, fmt.Errorf("convert file-max failed : %#v", err) -+ } -+ -+ var filesLimit uint64 -+ // Distribute the maximum value of each container evenly, and take 50% as the default value -+ filesLimit = (uint64(tmpFilesLimit) / defaultMaxContainers) / 2 -+ if filesLimit < defaultMinFilesLimit { -+ return 0, fmt.Errorf("file describe resource is shortage, require : %d, get : %d ", defaultMinFilesLimit, filesLimit) -+ } -+ -+ return filesLimit, nil -+} -+ - func (s *Sandbox) blockIOResource() *specs.LinuxBlockIO { - value, ok := s.config.Annotations[annotations.BlkioCgroupTypeKey] - if !ok { -diff --git a/virtcontainers/fileslimit/fileslimit.c b/virtcontainers/fileslimit/fileslimit.c -new file mode 100644 -index 0000000..0e9ae72 ---- /dev/null -+++ b/virtcontainers/fileslimit/fileslimit.c -@@ -0,0 +1,23 @@ -+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. -+// SPDX-License-Identifier: Apache-2.0 -+// Description: common functions -+// Author: jiangpengfei -+// Create: 2019-06-12 -+ -+#define _GNU_SOURCE -+#include -+#include -+#include -+#include "fileslimit.h" -+ -+int setProcessFilesLimit(int pid, int soft, int hard){ -+ struct rlimit new; -+ new.rlim_cur = soft; -+ new.rlim_max = hard; -+ -+ if (prlimit(pid, RLIMIT_NOFILE, &new, NULL) == -1) { -+ return errno; -+ } -+ -+ return 0; -+} -diff --git a/virtcontainers/fileslimit/fileslimit.go b/virtcontainers/fileslimit/fileslimit.go -new file mode 100644 -index 0000000..a0d2e2a ---- /dev/null -+++ b/virtcontainers/fileslimit/fileslimit.go -@@ -0,0 +1,23 @@ -+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. -+// SPDX-License-Identifier: Apache-2.0 -+// Description: common functions -+// Author: jiangpengfei -+// Create: 2019-06-12 -+ -+package fileslimit -+ -+/* -+#cgo CFLAGS: -Wall -+extern int setProcessFilesLimit(int pid, int soft, int hard); -+*/ -+import "C" -+import "fmt" -+ -+func SetProcessFilesLimit(pid, soft, hard int) error { -+ ret := C.setProcessFilesLimit(C.int(pid), C.int(soft), C.int(hard)) -+ if ret != 0 { -+ return fmt.Errorf("failed to set fileslimit of vm process ,errno:%d", ret) -+ } -+ -+ return nil -+} -diff --git a/virtcontainers/fileslimit/fileslimit.h b/virtcontainers/fileslimit/fileslimit.h -new file mode 100644 -index 0000000..4249d7a ---- /dev/null -+++ b/virtcontainers/fileslimit/fileslimit.h -@@ -0,0 +1,14 @@ -+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. -+// SPDX-License-Identifier: Apache-2.0 -+// Description: common functions -+// Author: jiangpengfei -+// Create: 2019-06-12 -+ -+#ifndef FILES_LIMIT_H -+#define FILES_LIMIT_H -+ -+#ifndef _GNU_SOURCE -+#define _GNU_SOURCE -+#endif -+ -+#endif /* FILES_LIMIT_H */ -diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go -index 9284f99..de8652f 100644 ---- a/virtcontainers/sandbox.go -+++ b/virtcontainers/sandbox.go -@@ -27,6 +27,7 @@ import ( - "github.com/kata-containers/runtime/virtcontainers/device/drivers" - deviceManager "github.com/kata-containers/runtime/virtcontainers/device/manager" - exp "github.com/kata-containers/runtime/virtcontainers/experimental" -+ "github.com/kata-containers/runtime/virtcontainers/fileslimit" - "github.com/kata-containers/runtime/virtcontainers/persist" - persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" - "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" -@@ -2400,13 +2401,14 @@ func (s *Sandbox) setupHostCgroupsWithEmulator() error { - return fmt.Errorf("sandbox's cgroup %s doesn't exist", s.state.CgroupPath) - } - -+ hypervisorPids := s.hypervisor.getPids() -+ if len(hypervisorPids) == 0 || hypervisorPids[0] == 0 { -+ return fmt.Errorf("hypervisor pid: %v invalid", hypervisorPids) -+ } -+ - // pull out qemu threads other than vcpu to the cgroup of "/emulator" - if s.config.HypervisorType == QemuHypervisor { - emulatorCgroupPath := filepath.Join(s.state.CgroupPath, emulatorCgroupName) -- hypervisorPids := s.hypervisor.getPids() -- if len(hypervisorPids) == 0 || hypervisorPids[0] == 0 { -- return fmt.Errorf("hypervisor pid: %v invalid", hypervisorPids) -- } - if err := pulloutQemuThread(s, hypervisorPids[0], emulatorCgroupPath); err != nil { - return err - } -@@ -2430,6 +2432,18 @@ func (s *Sandbox) setupHostCgroupsWithEmulator() error { - } - - // limit files resource -+ filesResources := specs.LinuxResources{ -+ Files: s.filesResource(procFileMaxPath), -+ } -+ if supportFileCgroup() { -+ if err := applyResourceLimit(&filesResources, vcpuCgroupPath); err != nil { -+ return err -+ } -+ } else { -+ if err := applyFilelimit(hypervisorPids[0], &filesResources); err != nil { -+ return err -+ } -+ } - - return nil - } -@@ -2451,6 +2465,22 @@ func applyResourceLimit(resources *specs.LinuxResources, cgroupPath string) erro - return nil - } - -+// apply files limit by pr_limit -+func applyFilelimit(vmPid int, resources *specs.LinuxResources) error { -+ if resources == nil { -+ return nil -+ } -+ -+ filesLimit := *(resources.Files.Limit) -+ -+ err := fileslimit.SetProcessFilesLimit(vmPid, int(filesLimit), int(filesLimit)) -+ if err != nil { -+ return fmt.Errorf("set vm fileslimit failed.vmPid:%d, limit:%d, err:%v", vmPid, filesLimit, err.Error()) -+ } -+ -+ return nil -+} -+ - // GetPatchedOCISpec returns sandbox's OCI specification - // This OCI specification was patched when the sandbox was created - // by containerCapabilities(), SetEphemeralStorageType() and others --- -1.8.3.1 - diff --git a/runtime/patches/0058-runtime-fix-sandboxRuntimeRootPath-left-problem.patch b/runtime/patches/0058-runtime-fix-sandboxRuntimeRootPath-left-problem.patch deleted file mode 100644 index 74f9bc6..0000000 --- a/runtime/patches/0058-runtime-fix-sandboxRuntimeRootPath-left-problem.patch +++ /dev/null @@ -1,69 +0,0 @@ -From faffb26c307556e1d84399060d7aef1753e9f41a Mon Sep 17 00:00:00 2001 -From: jiangpengfei -Date: Mon, 21 Sep 2020 21:52:48 -0400 -Subject: [PATCH] runtime: fix sandboxRuntimeRootPath left problem - -reason: If pod_sandbox type container deleted before pod_container -type containers in the sandbox, which leads to kata-runtime delete -pod_container process create a new sandboxRuntimeRootPath in the -/run/vc/sbs/ dir. - -So, we fix this problem by check sandboxRuntimeRootPath is exist -before fetchSandbox function is called. - -Signed-off-by: jiangpengfei ---- - cli/delete.go | 2 +- - virtcontainers/sandbox.go | 18 ++++++++++++++++++ - 2 files changed, 19 insertions(+), 1 deletion(-) - -diff --git a/cli/delete.go b/cli/delete.go -index 871ac40d..09552b9a 100644 ---- a/cli/delete.go -+++ b/cli/delete.go -@@ -87,7 +87,7 @@ func delete(ctx context.Context, containerID string, force bool) error { - // container is deleted before pod_container type container, just return nil - // and let containerd delete container operations continue - if strings.Contains(err.Error(), "no such file or directory") { -- kataLog.Warnf("pod_sandbox deleted before pod_container: %v", err) -+ kataLog.Warnf("pod_sandbox container deleted before pod_container in the sandbox: %v", err) - return nil - } - -diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go -index de8652fc..9f5d3c08 100644 ---- a/virtcontainers/sandbox.go -+++ b/virtcontainers/sandbox.go -@@ -729,6 +729,13 @@ func fetchSandbox(ctx context.Context, sandboxID string) (sandbox *Sandbox, err - - var config SandboxConfig - -+ // check sandbox runtime root path(for example: /run/vc/sbs/) -+ // exist or not before fetch sandbox config file -+ if !checkSandboxRuntimeRootPathExist(sandboxID) { -+ sandboxRuntimeRootPath := store.SandboxRuntimeRootPath(sandboxID) -+ return nil, fmt.Errorf("sandbox %s does not exist, %s no such file or directory", sandboxID,sandboxRuntimeRootPath) -+ } -+ - // Try to load sandbox config from old store at first. - c, ctx, err := loadSandboxConfigFromOldStore(ctx, sandboxID) - if err != nil { -@@ -2707,3 +2714,14 @@ func updateStaticSandboxResources(sandboxConfig *SandboxConfig) error { - - return nil - } -+ -+// checkSandboxRuntimeRootPathExist check /run/vc/sbs// dir exist or not -+func checkSandboxRuntimeRootPathExist(sandboxID string) bool { -+ sandboxRuntimeRootPath := store.SandboxRuntimeRootPath(sandboxID) -+ _, err := os.Stat(sandboxRuntimeRootPath) -+ if os.IsNotExist(err) { -+ return false -+ } -+ -+ return true -+} -\ No newline at end of file --- -2.11.0 - diff --git a/runtime/patches/0059-runtime-fix-invalid-cmdline-when-start-sandbox-stratovirt.patch b/runtime/patches/0059-runtime-fix-invalid-cmdline-when-start-sandbox-stratovirt.patch deleted file mode 100644 index e969f7d..0000000 --- a/runtime/patches/0059-runtime-fix-invalid-cmdline-when-start-sandbox-stratovirt.patch +++ /dev/null @@ -1,42 +0,0 @@ -From e10fe6ecf2d895fe009a080bb34334ff63739f3e Mon Sep 17 00:00:00 2001 -From: LiangZhang -Date: Thu, 24 Sep 2020 14:38:04 +0800 -Subject: [PATCH] runtime: fix invalid cmdline when start sandbox stratovirt - -reason: - 1. MemorySize is in type of uint32, value larger than 4096 will trigger overflow. - 2. cannot disable seccomp now. - -Signed-off-by: LiangZhang ---- - virtcontainers/stratovirt.go | 7 ++++++- - 1 file changed, 6 insertions(+), 1 deletion(-) - -diff --git a/virtcontainers/stratovirt.go b/virtcontainers/stratovirt.go -index 7c156d5..6db8dc0 100644 ---- a/virtcontainers/stratovirt.go -+++ b/virtcontainers/stratovirt.go -@@ -146,7 +146,7 @@ func (s *stratovirt) startSandbox(timeout int) error { - - params = append(params, "-append "+s.getKernelCmdLine()) - params = append(params, fmt.Sprintf("-smp %d", s.config.NumVCPUs)) -- params = append(params, fmt.Sprintf("-m %d", s.config.MemorySize*1024*1024)) -+ params = append(params, fmt.Sprintf("-m %d", uint64(s.config.MemorySize)*1024*1024)) - params = append(params, fmt.Sprintf("-chardev id=charconsole0,path=%s", s.consolePath)) - - // add devices to cmdline -@@ -178,6 +178,11 @@ func (s *stratovirt) startSandbox(timeout int) error { - params = append(params, fmt.Sprintf("-D %s/stratovirt.log", dir)) - } - -+ // disable Seccomp -+ if s.sandbox.config.DisableGuestSeccomp { -+ params = append(params, "-disable-seccomp") -+ } -+ - s.Logger().Info("StratoVirt start with params: ", strings.Join(params, " ")) - - dir := filepath.Join(store.RunVMStoragePath(), s.id) --- -1.8.3.1 - diff --git a/runtime/patches/0060-runtime-fix-cmd-params-of-direct-use-stratovirt-bina.patch b/runtime/patches/0060-runtime-fix-cmd-params-of-direct-use-stratovirt-bina.patch deleted file mode 100644 index 89d473f..0000000 --- a/runtime/patches/0060-runtime-fix-cmd-params-of-direct-use-stratovirt-bina.patch +++ /dev/null @@ -1,94 +0,0 @@ -From b2b11cc8fb2144c13f4f9138b4420248ea200fb6 Mon Sep 17 00:00:00 2001 -From: LiangZhang -Date: Sun, 27 Sep 2020 18:10:18 +0800 -Subject: [PATCH] runtime: fix cmd params of direct use stratovirt binary - -reason: when directly use stratovirt binary, not through bash, it will cause problem. - -Signed-off-by: LiangZhang ---- - virtcontainers/stratovirt.go | 28 ++++++++++++++-------------- - 1 file changed, 14 insertions(+), 14 deletions(-) - -diff --git a/virtcontainers/stratovirt.go b/virtcontainers/stratovirt.go -index 6db8dc0..020135e 100644 ---- a/virtcontainers/stratovirt.go -+++ b/virtcontainers/stratovirt.go -@@ -133,21 +133,21 @@ func (s *stratovirt) startSandbox(timeout int) error { - defer span.Finish() - - var params []string -- params = append(params, "-name "+fmt.Sprintf("sandbox-%s", s.id)) -- params = append(params, "-api-channel unix:"+s.socketPath) -+ params = append(params, "-name", fmt.Sprintf("sandbox-%s", s.id)) -+ params = append(params, "-api-channel", fmt.Sprintf("unix:%s", s.socketPath)) - - if kernelPath, err := s.config.KernelAssetPath(); err == nil { -- params = append(params, "-kernel "+kernelPath) -+ params = append(params, "-kernel", kernelPath) - } - - if initrdPath, err := s.config.InitrdAssetPath(); err == nil { -- params = append(params, "-initrd "+initrdPath) -+ params = append(params, "-initrd", initrdPath) - } - -- params = append(params, "-append "+s.getKernelCmdLine()) -- params = append(params, fmt.Sprintf("-smp %d", s.config.NumVCPUs)) -- params = append(params, fmt.Sprintf("-m %d", uint64(s.config.MemorySize)*1024*1024)) -- params = append(params, fmt.Sprintf("-chardev id=charconsole0,path=%s", s.consolePath)) -+ params = append(params, "-append", s.getKernelCmdLine()) -+ params = append(params, "-smp", fmt.Sprintf("%d", s.config.NumVCPUs)) -+ params = append(params, "-m", fmt.Sprintf("%d", uint64(s.config.MemorySize)*1024*1024)) -+ params = append(params, "-chardev", fmt.Sprintf("id=charconsole0,path=%s", s.consolePath)) - - // add devices to cmdline - for _, d := range s.devices { -@@ -156,14 +156,14 @@ func (s *stratovirt) startSandbox(timeout int) error { - name := v.Name() - mac := v.HardwareAddr() - tapName := v.NetworkPair().TapInterface.TAPIface.Name -- params = append(params, fmt.Sprintf("-net id=%s,mac=%s,host_dev_name=%s", name, mac, tapName)) -+ params = append(params, "-net", fmt.Sprintf("id=%s,mac=%s,host_dev_name=%s", name, mac, tapName)) - case config.BlockDrive: - id := v.ID - path := v.File -- params = append(params, fmt.Sprintf("-drive id=%s,file=%s", id, path)) -+ params = append(params, "-drive", fmt.Sprintf("id=%s,file=%s", id, path)) - case types.VSock: - v.VhostFd.Close() -- params = append(params, fmt.Sprintf("-device vsock,id=vsock-id,guest-cid=%d", v.ContextID)) -+ params = append(params, "-device", fmt.Sprintf("vsock,id=vsock-id,guest-cid=%d", v.ContextID)) - default: - s.Logger().Error("Adding device type is unsupported") - } -@@ -175,7 +175,7 @@ func (s *stratovirt) startSandbox(timeout int) error { - // append logfile only on debug - if s.config.Debug { - dir := filepath.Join(store.RunVMStoragePath(), s.id) -- params = append(params, fmt.Sprintf("-D %s/stratovirt.log", dir)) -+ params = append(params, "-D", fmt.Sprintf("%s/stratovirt.log", dir)) - } - - // disable Seccomp -@@ -183,8 +183,6 @@ func (s *stratovirt) startSandbox(timeout int) error { - params = append(params, "-disable-seccomp") - } - -- s.Logger().Info("StratoVirt start with params: ", strings.Join(params, " ")) -- - dir := filepath.Join(store.RunVMStoragePath(), s.id) - err := os.MkdirAll(dir, store.DirMode) - if err != nil { -@@ -205,6 +203,8 @@ func (s *stratovirt) startSandbox(timeout int) error { - } - - cmd := exec.CommandContext(s.ctx, binPath, params...) -+ s.Logger().Info("StratoVirt start with params: ", cmd) -+ - if err := cmd.Run(); err != nil { - s.Logger().WithField("Error starting hypervisor, please check the params", err).Error() - return err --- -1.8.3.1 - diff --git a/runtime/patches/0061-kata-runtime-retry-inserting-of-CNI-interface.patch b/runtime/patches/0061-kata-runtime-retry-inserting-of-CNI-interface.patch deleted file mode 100644 index f61d224..0000000 --- a/runtime/patches/0061-kata-runtime-retry-inserting-of-CNI-interface.patch +++ /dev/null @@ -1,223 +0,0 @@ -From babe7b3028d601a9b00aedda673e6a472f29ad07 Mon Sep 17 00:00:00 2001 -From: yangfeiyu -Date: Mon, 28 Sep 2020 21:01:00 +0800 -Subject: [PATCH] kata-runtime: retry inserting of CNI interface - -reason: when netmon is enable, a interface is inserted into the -netns created by the sandbox, it should wait for the generating of -IP of the new interface, and get the ipv4 address - -Signed-off-by: yangfeiyu ---- - netmon/netmon.go | 128 +++++++++++++++++++++++++++++++++++++++++------ - 1 file changed, 112 insertions(+), 16 deletions(-) - -diff --git a/netmon/netmon.go b/netmon/netmon.go -index 94305ce2..57beacfb 100644 ---- a/netmon/netmon.go -+++ b/netmon/netmon.go -@@ -52,7 +52,7 @@ var ( - version = "unknown" - - // For simplicity the code will only focus on IPv4 addresses for now. -- netlinkFamily = netlink.FAMILY_ALL -+ netlinkFamily = netlink.FAMILY_V4 - - storageParentPath = "/var/run/kata-containers/netmon/sbs" - ) -@@ -70,7 +70,9 @@ type netmon struct { - storagePath string - sharedFile string - -- netIfaces map[int]vcTypes.Interface -+ netIfaces map[int]vcTypes.Interface -+ plugedIfaces map[string]vcTypes.Interface -+ pendingRoutes map[string][]vcTypes.Route - - linkUpdateCh chan netlink.LinkUpdate - linkDoneCh chan struct{} -@@ -148,15 +150,17 @@ func newNetmon(params netmonParams) (*netmon, error) { - } - - n := &netmon{ -- netmonParams: params, -- storagePath: filepath.Join(storageParentPath, params.sandboxID), -- sharedFile: filepath.Join(storageParentPath, params.sandboxID, sharedFile), -- netIfaces: make(map[int]vcTypes.Interface), -- linkUpdateCh: make(chan netlink.LinkUpdate), -- linkDoneCh: make(chan struct{}), -- rtUpdateCh: make(chan netlink.RouteUpdate), -- rtDoneCh: make(chan struct{}), -- netHandler: handler, -+ netmonParams: params, -+ storagePath: filepath.Join(storageParentPath, params.sandboxID), -+ sharedFile: filepath.Join(storageParentPath, params.sandboxID, sharedFile), -+ netIfaces: make(map[int]vcTypes.Interface), -+ plugedIfaces: make(map[string]vcTypes.Interface), -+ pendingRoutes: make(map[string][]vcTypes.Route), -+ linkUpdateCh: make(chan netlink.LinkUpdate), -+ linkDoneCh: make(chan struct{}), -+ rtUpdateCh: make(chan netlink.RouteUpdate), -+ rtDoneCh: make(chan struct{}), -+ netHandler: handler, - } - - if err := os.MkdirAll(n.storagePath, storageDirPerm); err != nil { -@@ -266,7 +270,6 @@ func convertInterface(linkAttrs *netlink.LinkAttrs, linkType string, addrs []net - } - - var ipAddrs []*vcTypes.IPAddress -- - for _, addr := range addrs { - if addr.IPNet == nil { - continue -@@ -450,10 +453,24 @@ func (n *netmon) updateRoutes() error { - if err != nil { - return err - } -+ if len(netlinkRoutes) == 0 { -+ n.logger().Debug("get 0 routes") -+ return nil -+ } - - // Translate them into Route structures. - routes := convertRoutes(netlinkRoutes) - -+ // if the device of the routes have not be hotplug to guest, -+ // the update operation will be failed, pending them. -+ // For all routes are belong the same device, so we just need -+ // judge the device of the first route -+ if _, pluged := n.plugedIfaces[routes[0].Device]; !pluged { -+ n.pendingRoutes[routes[0].Device] = routes -+ n.logger().Infof("dev %s have not been added, pending:%v", routes[0].Device, routes) -+ return nil -+ } -+ - // Update the routes through the Kata CLI. - return n.updateRoutesCLI(routes) - } -@@ -489,6 +506,9 @@ func (n *netmon) handleRTMNewLink(ev netlink.LinkUpdate) error { - return nil - } - -+ // the link is usually not ready, and `sleep 3` is an empirical value. -+ time.Sleep(3 * time.Second) -+ - // Check if the interface exist in the internal list. - if _, exist := n.netIfaces[int(ev.Index)]; exist { - n.logger().Debugf("Ignoring interface %s because already exist", -@@ -504,10 +524,26 @@ func (n *netmon) handleRTMNewLink(ev netlink.LinkUpdate) error { - return nil - } - -- // Get the list of IP addresses associated with this interface. -- addrs, err := n.netHandler.AddrList(ev.Link, netlinkFamily) -- if err != nil { -- return err -+ // In some scenarios, the ip have not prepared,so we should do some retries. -+ var addrs []netlink.Addr -+ var err error -+ for i := 0; i < 5; i++ { -+ // Get the list of IP addresses associated with this interface. -+ addrs, err = n.netHandler.AddrList(ev.Link, netlinkFamily) -+ if err != nil { -+ return err -+ } -+ if len(addrs) > 0 { -+ break -+ } -+ time.Sleep(500 * time.Millisecond) -+ } -+ -+ // In some scenarios, the link reported by event can not found, do extras check here. -+ if n.checkLinkByHw(linkAttrs.HardwareAddr.String()) != true { -+ n.logger().Infof("Ignore %v because can not find link by HW %s", -+ linkAttrs.Name, linkAttrs.HardwareAddr.String()) -+ return nil - } - - // Convert the interfaces in the appropriate structure format. -@@ -520,6 +556,17 @@ func (n *netmon) handleRTMNewLink(ev netlink.LinkUpdate) error { - - // Add the interface to the internal list. - n.netIfaces[linkAttrs.Index] = iface -+ n.plugedIfaces[iface.Name] = iface -+ -+ // The pending routes is preferentially selected. -+ if routes, ok := n.pendingRoutes[iface.Name]; ok { -+ n.logger().Infof("dev %s find pending routes:%v", iface.Name, routes) -+ err = n.updateRoutesCLI(routes) -+ if err != nil { -+ return err -+ } -+ delete(n.pendingRoutes, iface.Name) -+ } - - // Complete by updating the routes. - return n.updateRoutes() -@@ -556,6 +603,8 @@ func (n *netmon) handleRTMDelLink(ev netlink.LinkUpdate) error { - - // Delete the interface from the internal list. - delete(n.netIfaces, linkAttrs.Index) -+ delete(n.plugedIfaces, iface.Name) -+ delete(n.pendingRoutes, iface.Name) - - // Complete by updating the routes. - return n.updateRoutes() -@@ -640,6 +689,53 @@ func (n *netmon) handleEvents() (err error) { - } - } - -+func linkByHwAddr(netHandle *netlink.Handle, hwAddr string) (netlink.Link, error) { -+ if netHandle == nil { -+ return nil, fmt.Errorf("no handler ") -+ } -+ -+ links, err := netHandle.LinkList() -+ if err != nil { -+ return nil, err -+ } -+ -+ for _, link := range links { -+ if link == nil { -+ continue -+ } -+ -+ lAttrs := link.Attrs() -+ if lAttrs == nil { -+ continue -+ } -+ -+ if lAttrs.HardwareAddr == nil { -+ continue -+ } -+ -+ if lAttrs.HardwareAddr.String() == hwAddr { -+ return link, nil -+ } -+ } -+ -+ return nil, fmt.Errorf("could not find the link corresponding to HwAddr %s", hwAddr) -+} -+ -+func (n *netmon) checkLinkByHw(hw string) bool { -+ netHandle, err := netlink.NewHandle(unix.NETLINK_ROUTE) -+ if err != nil { -+ return false -+ } -+ defer netHandle.Delete() -+ -+ link, err := linkByHwAddr(netHandle, hw) -+ if err != nil || link == nil { -+ return false -+ } -+ -+ return true -+} -+ - func main() { - // Parse parameters. - params := parseOptions() --- -2.23.0 - diff --git a/runtime/patches/0062-kata-runtime-support-using-CNI-plugin-to-insert-muti.patch b/runtime/patches/0062-kata-runtime-support-using-CNI-plugin-to-insert-muti.patch deleted file mode 100644 index d62395c..0000000 --- a/runtime/patches/0062-kata-runtime-support-using-CNI-plugin-to-insert-muti.patch +++ /dev/null @@ -1,43 +0,0 @@ -From f0d2f8a19956045b4b53ac5f2c4b59940016ca41 Mon Sep 17 00:00:00 2001 -From: yangfeiyu -Date: Fri, 9 Oct 2020 16:02:27 +0800 -Subject: [PATCH] kata-runtime: support using CNI plugin to insert mutiple - network interfaces at the same time - -reason: support using CNI plugin to insert mutiple network interfaces at the same time - -Signed-off-by: yangfeiyu ---- - netmon/netmon.go | 16 +++++++++++----- - 1 file changed, 11 insertions(+), 5 deletions(-) - -diff --git a/netmon/netmon.go b/netmon/netmon.go -index 57beacfb..a519e5ba 100644 ---- a/netmon/netmon.go -+++ b/netmon/netmon.go -@@ -463,11 +463,17 @@ func (n *netmon) updateRoutes() error { - - // if the device of the routes have not be hotplug to guest, - // the update operation will be failed, pending them. -- // For all routes are belong the same device, so we just need -- // judge the device of the first route -- if _, pluged := n.plugedIfaces[routes[0].Device]; !pluged { -- n.pendingRoutes[routes[0].Device] = routes -- n.logger().Infof("dev %s have not been added, pending:%v", routes[0].Device, routes) -+ var pendingFlag bool -+ for _, route := range routes{ -+ if _, pluged := n.plugedIfaces[route.Device]; !pluged { -+ pendingFlag = true -+ n.pendingRoutes[route.Device] = append(n.pendingRoutes[route.Device],route) -+ n.logger().Infof("dev %s have not been added, pending:%v", route.Device, n.pendingRoutes[route.Device]) -+ } -+ } -+ -+ // find pending route -+ if pendingFlag { - return nil - } - --- -2.23.0 - diff --git a/runtime/patches/0063-kata-runtime-fix-get-sandbox-cpu-resources-problem.patch b/runtime/patches/0063-kata-runtime-fix-get-sandbox-cpu-resources-problem.patch deleted file mode 100644 index bb2dfe0..0000000 --- a/runtime/patches/0063-kata-runtime-fix-get-sandbox-cpu-resources-problem.patch +++ /dev/null @@ -1,90 +0,0 @@ -From bac206ee5b4ccb90fd8c06c0b6244ba163ac82b9 Mon Sep 17 00:00:00 2001 -From: holyfei -Date: Sun, 15 Nov 2020 21:31:49 +0800 -Subject: [PATCH] kata-runtime: fix get sandbox cpu resources problem - -reason: If sandox_cgroup_with_emulator config is enabled, -kata-runtime should get cpu resources by cpuResourcesWithEmulator -func instead of the original cpuResource func. - -Signed-off-by: holyfei ---- - virtcontainers/cgroups.go | 2 ++ - virtcontainers/sandbox.go | 34 +++++++++++++++++++++++++++++++++- - 2 files changed, 35 insertions(+), 1 deletion(-) - -diff --git a/virtcontainers/cgroups.go b/virtcontainers/cgroups.go -index 1b6b04da..21708ebf 100644 ---- a/virtcontainers/cgroups.go -+++ b/virtcontainers/cgroups.go -@@ -40,6 +40,8 @@ const ( - - defaultMinFilesLimit uint64 = 1024 - defaultMaxContainers uint64 = 200 -+ defaultCPUPeriod uint64 = 1000000 -+ defaultCPUShares uint64 = 1024 - - procFileMaxPath = "/proc/sys/fs/file-max" - -diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go -index 9f5d3c08..d3c64b4f 100644 ---- a/virtcontainers/sandbox.go -+++ b/virtcontainers/sandbox.go -@@ -14,6 +14,7 @@ import ( - "net" - "os" - "path/filepath" -+ "strconv" - "strings" - "sync" - "syscall" -@@ -2351,6 +2352,37 @@ func (s *Sandbox) cpuResources() *specs.LinuxCPU { - return validCPUResources(cpu) - } - -+func (s *Sandbox) cpuResourcesWithEmulator() *specs.LinuxCPU { -+ // Use default period and quota if they are not specified. -+ // Container will inherit the constraints from its parent. -+ quota := int64(0) -+ period := uint64(0) -+ shares := uint64(0) -+ -+ cpu := &specs.LinuxCPU{ -+ Quota: "a, -+ Period: &period, -+ Shares: &shares, -+ } -+ -+ period = defaultCPUPeriod -+ -+ cpuValue, exist := s.config.Annotations[annotations.StaticCPUTypeKey] -+ if exist { -+ // If sandbox_cpu is set, we have validate at first, so we don't need to check it again -+ cpuNum, _ := strconv.ParseFloat(cpuValue, 64) -+ quota = int64(cpuNum * float64(period)) -+ shares = uint64(cpuNum * float64(defaultCPUShares)) -+ } else { -+ // If sandbox_cpu is not set, we use the hypervisor's cpu number as cpu limit -+ hypervisorConfig := s.hypervisor.hypervisorConfig() -+ quota = int64(uint64(hypervisorConfig.NumVCPUs) * period) -+ shares = uint64(hypervisorConfig.NumVCPUs) * defaultCPUShares -+ } -+ -+ return cpu -+} -+ - // setupSandboxCgroup creates and joins sandbox cgroups for the sandbox config - func (s *Sandbox) setupSandboxCgroup() error { - var err error -@@ -2424,7 +2456,7 @@ func (s *Sandbox) setupHostCgroupsWithEmulator() error { - // limit cpu to "/vcpu" - vcpuCgroupPath := filepath.Join(s.state.CgroupPath, vcpuCgroupName) - vcpuResources := specs.LinuxResources{ -- CPU: s.cpuResources(), -+ CPU: s.cpuResourcesWithEmulator(), - } - if err := applyResourceLimit(&vcpuResources, vcpuCgroupPath); err != nil { - return err --- -2.11.0 - diff --git a/runtime/patches/0064-runtime-add-support-for-stratovirt-of-kata-check-cli.patch b/runtime/patches/0064-runtime-add-support-for-stratovirt-of-kata-check-cli.patch deleted file mode 100644 index d020b11..0000000 --- a/runtime/patches/0064-runtime-add-support-for-stratovirt-of-kata-check-cli.patch +++ /dev/null @@ -1,38 +0,0 @@ -From 5af764ee8aad86f57df64ad78b44611c47067cc9 Mon Sep 17 00:00:00 2001 -From: LiangZhang -Date: Tue, 15 Dec 2020 10:14:10 +0800 -Subject: [PATCH] runtime: add support for stratovirt of kata-check cli - -reason: The current version of kata-check lacks support of stratovirt - -Signed-off-by: LiangZhang ---- - cli/kata-check_amd64.go | 4 +++- - 1 file changed, 3 insertions(+), 1 deletion(-) - -diff --git a/cli/kata-check_amd64.go b/cli/kata-check_amd64.go -index 8376293..ae62746 100644 ---- a/cli/kata-check_amd64.go -+++ b/cli/kata-check_amd64.go -@@ -107,7 +107,7 @@ func setCPUtype(hypervisorType vc.HypervisorType) error { - fallthrough - case "clh": - fallthrough -- case "qemu": -+ case "qemu", "stratovirt": - archRequiredCPUFlags = map[string]string{ - cpuFlagVMX: "Virtualization support", - cpuFlagLM: "64Bit CPU", -@@ -286,6 +286,8 @@ func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error { - switch hypervisorType { - case "qemu": - fallthrough -+ case "stratovirt": -+ fallthrough - case "clh": - fallthrough - case "firecracker": --- -2.25.1 - - diff --git a/runtime/patches/0065-runtime-fixup-that-the-getPids-function-returns-pid-.patch b/runtime/patches/0065-runtime-fixup-that-the-getPids-function-returns-pid-.patch deleted file mode 100644 index 661608b..0000000 --- a/runtime/patches/0065-runtime-fixup-that-the-getPids-function-returns-pid-.patch +++ /dev/null @@ -1,83 +0,0 @@ -From 899164391d2ca57f84716a5b2f34500883bc3690 Mon Sep 17 00:00:00 2001 -From: LiangZhang -Date: Mon, 11 Jan 2021 11:03:39 +0800 -Subject: [PATCH] runtime: fixup that the getPids function returns pid 0 - -Use pidfile to record the real pid of stratovirt, getPids then read the content of pidfile to get pid value. - -Signed-off-by: LiangZhang ---- - virtcontainers/stratovirt.go | 31 ++++++++++++++++++++++++++-- - 1 file changed, 29 insertions(+), 2 deletions(-) - -diff --git a/virtcontainers/stratovirt.go b/virtcontainers/stratovirt.go -index 020135e..a8151de 100644 ---- a/virtcontainers/stratovirt.go -+++ b/virtcontainers/stratovirt.go -@@ -3,6 +3,7 @@ package virtcontainers - import ( - "context" - "fmt" -+ "io/ioutil" - "os" - "os/exec" - "path/filepath" -@@ -148,6 +149,7 @@ func (s *stratovirt) startSandbox(timeout int) error { - params = append(params, "-smp", fmt.Sprintf("%d", s.config.NumVCPUs)) - params = append(params, "-m", fmt.Sprintf("%d", uint64(s.config.MemorySize)*1024*1024)) - params = append(params, "-chardev", fmt.Sprintf("id=charconsole0,path=%s", s.consolePath)) -+ params = append(params, "-pidfile", filepath.Join(s.store.RunVMStoragePath(), s.id, "pid")) - - // add devices to cmdline - for _, d := range s.devices { -@@ -587,7 +589,27 @@ func (s *stratovirt) cleanup() error { - } - - func (s *stratovirt) getPids() []int { -- return []int{s.pid} -+ var pids []int -+ if s.pid != 0 { -+ pids = append(pids, s.pid) -+ } else { -+ pid, err := ioutil.ReadFile(filepath.Join(s.store.RunVMStoragePath(), s.id, "pid")) -+ if err != nil { -+ s.Logger().WithError(err).Error("Read pid file failed.") -+ return []int{0} -+ } -+ -+ p, err := strconv.Atoi(strings.Trim(string(pid), "\n\t ")) -+ if err != nil { -+ s.Logger().WithError(err).Error("Get pid from pid file failed.") -+ return []int{0} -+ } -+ -+ pids = append(pids, p) -+ s.pid = p -+ } -+ -+ return pids - } - - func (s *stratovirt) fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, j []byte) error { -@@ -611,12 +633,17 @@ func (s *stratovirt) generateSocket(id string, useVsock bool) (interface{}, erro - } - - func (s *stratovirt) save() (p persistapi.HypervisorState) { -- p.Pid = s.pid -+ pids := s.getPids() -+ p.Pid = pids[0] - p.Type = string(StratovirtHypervisor) - return - } - - func (s *stratovirt) load(p persistapi.HypervisorState) { - s.pid = p.Pid -+ if sandbox, err := globalSandboxList.lookupSandbox(s.id); err == nil { -+ s.sandbox = sandbox -+ } -+ - return - } --- -2.27.0 - diff --git a/runtime/patches/0066-CVE-2020-28914-1.patch b/runtime/patches/0066-CVE-2020-28914-1.patch deleted file mode 100644 index 910d1ee..0000000 --- a/runtime/patches/0066-CVE-2020-28914-1.patch +++ /dev/null @@ -1,78 +0,0 @@ -From 228e6eb4b9c000fb105e3bf1401ac3938588fae2 Mon Sep 17 00:00:00 2001 -From: Peng Tao -Date: Fri, 30 Oct 2020 14:54:49 +0800 -Subject: [PATCH] runtime: readonly mounts should be readonly bindmount on the - host - -So that we get protected at the VM boundary not just the guest kernel. - -Signed-off-by: Peng Tao -Reference: https://github.com/kata-containers/runtime/commit/228e6eb4b9c000fb105e3bf1401ac3938588fae2 - https://github.com/kata-containers/community/blob/master/VMT/KCSA/KCSA-CVE-2020-28914.md -(cherry picked from commit 509eb6f850c0ceb60eb91a6095cceb8e4c7150f5) ---- - virtcontainers/container.go | 14 ++------------ - virtcontainers/pkg/oci/utils.go | 8 ++++++++ - 2 files changed, 10 insertions(+), 12 deletions(-) - -diff --git a/virtcontainers/container.go b/virtcontainers/container.go -index 88863ec42..6973c8328 100644 ---- a/virtcontainers/container.go -+++ b/virtcontainers/container.go -@@ -481,7 +481,7 @@ func (c *Container) shareFiles(m Mount, idx int, hostSharedDir, guestSharedDir s - } else { - // These mounts are created in the shared dir - mountDest := filepath.Join(hostSharedDir, filename) -- if err := bindMount(c.ctx, m.Source, mountDest, false, "private"); err != nil { -+ if err := bindMount(c.ctx, m.Source, mountDest, m.ReadOnly, "private"); err != nil { - return "", false, err - } - // Save HostPath mount value into the mount list of the container. -@@ -557,22 +557,12 @@ func (c *Container) mountSharedDirMounts(hostSharedDir, guestSharedDir string) ( - continue - } - -- // Check if mount is readonly, let the agent handle the readonly mount -- // within the VM. -- readonly := false -- for _, flag := range m.Options { -- if flag == "ro" { -- readonly = true -- break -- } -- } -- - sharedDirMount := Mount{ - Source: guestDest, - Destination: m.Destination, - Type: m.Type, - Options: m.Options, -- ReadOnly: readonly, -+ ReadOnly: m.ReadOnly, - } - - sharedDirMounts[sharedDirMount.Destination] = sharedDirMount -diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go -index 0832a757c..9701df3d5 100644 ---- a/virtcontainers/pkg/oci/utils.go -+++ b/virtcontainers/pkg/oci/utils.go -@@ -162,11 +162,19 @@ func cmdEnvs(spec specs.Spec, envs []types.EnvVar) []types.EnvVar { - } - - func newMount(m specs.Mount) vc.Mount { -+ readonly := false -+ for _, flag := range m.Options { -+ if flag == "ro" { -+ readonly = true -+ break -+ } -+ } - return vc.Mount{ - Source: m.Source, - Destination: m.Destination, - Type: m.Type, - Options: m.Options, -+ ReadOnly: readonly, - } - } - diff --git a/runtime/patches/0067-CVE-2020-28914-2.patch b/runtime/patches/0067-CVE-2020-28914-2.patch deleted file mode 100644 index f61670d..0000000 --- a/runtime/patches/0067-CVE-2020-28914-2.patch +++ /dev/null @@ -1,141 +0,0 @@ -From 3f0e61c010556846e2f96811059d86cb10309748 Mon Sep 17 00:00:00 2001 -From: Peng Tao -Date: Fri, 30 Oct 2020 17:40:12 +0800 -Subject: [PATCH] runtime: mount shared mountpoint readonly - -bindmount remount events are not propagated through mount subtrees, -so we have to remount the shared dir mountpoint directly. - -E.g., -``` -mkdir -p source dest foo source/foo - -mount -o bind --make-shared source dest - -mount -o bind foo source/foo -echo bind mount rw -mount | grep foo -echo remount ro -mount -o remount,bind,ro source/foo -mount | grep foo -``` -would result in: -``` -bind mount rw -/dev/xvda1 on /home/ubuntu/source/foo type ext4 (rw,relatime,discard,data=ordered) -/dev/xvda1 on /home/ubuntu/dest/foo type ext4 (rw,relatime,discard,data=ordered) -remount ro -/dev/xvda1 on /home/ubuntu/source/foo type ext4 (ro,relatime,discard,data=ordered) -/dev/xvda1 on /home/ubuntu/dest/foo type ext4 (rw,relatime,discard,data=ordered) -``` - -The reason is that bind mount creats new mount structs and attaches them to different mount subtrees. -However, MS_REMOUNT only looks for existing mount structs to modify and does not try to propagate the -change to mount structs in other subtrees. - -Fixes: #3041 -Signed-off-by: Peng Tao -Reference: https://github.com/kata-containers/runtime/commit/3f0e61c010556846e2f96811059d86cb10309748 - https://github.com/kata-containers/community/blob/master/VMT/KCSA/KCSA-CVE-2020-28914.md -(cherry picked from commit 77399058bf3ded60aaeb08978073d000f178478f) ---- - virtcontainers/container.go | 15 +++++++++++---- - virtcontainers/kata_agent.go | 2 +- - virtcontainers/mount.go | 5 +++++ - virtcontainers/sandbox_test.go | 2 +- - 4 files changed, 18 insertions(+), 6 deletions(-) - -diff --git a/virtcontainers/container.go b/virtcontainers/container.go -index 6973c8328..37117d7e3 100644 ---- a/virtcontainers/container.go -+++ b/virtcontainers/container.go -@@ -446,7 +446,7 @@ func (c *Container) setContainerState(state types.StateString) error { - return nil - } - --func (c *Container) shareFiles(m Mount, idx int, hostSharedDir, guestSharedDir string) (string, bool, error) { -+func (c *Container) shareFiles(m Mount, idx int, hostSharedDir, hostMountDir, guestSharedDir string) (string, bool, error) { - randBytes, err := utils.GenerateRandomBytes(8) - if err != nil { - return "", false, err -@@ -480,12 +480,19 @@ func (c *Container) shareFiles(m Mount, idx int, hostSharedDir, guestSharedDir s - } - } else { - // These mounts are created in the shared dir -- mountDest := filepath.Join(hostSharedDir, filename) -+ mountDest := filepath.Join(hostMountDir, filename) - if err := bindMount(c.ctx, m.Source, mountDest, m.ReadOnly, "private"); err != nil { - return "", false, err - } - // Save HostPath mount value into the mount list of the container. - c.mounts[idx].HostPath = mountDest -+ // bindmount remount event is not propagated to mount subtrees, so we have to remount the shared dir mountpoint directly. -+ if m.ReadOnly { -+ mountDest = filepath.Join(hostSharedDir, filename) -+ if err := remountRo(c.ctx, mountDest); err != nil { -+ return "", false, err -+ } -+ } - } - - return guestDest, false, nil -@@ -496,7 +503,7 @@ func (c *Container) shareFiles(m Mount, idx int, hostSharedDir, guestSharedDir s - // It also updates the container mount list with the HostPath info, and store - // container mounts to the storage. This way, we will have the HostPath info - // available when we will need to unmount those mounts. --func (c *Container) mountSharedDirMounts(hostSharedDir, guestSharedDir string) (sharedDirMounts map[string]Mount, ignoredMounts map[string]Mount, err error) { -+func (c *Container) mountSharedDirMounts(hostSharedDir, hostMountDir, guestSharedDir string) (sharedDirMounts map[string]Mount, ignoredMounts map[string]Mount, err error) { - sharedDirMounts = make(map[string]Mount) - ignoredMounts = make(map[string]Mount) - var devicesToDetach []string -@@ -546,7 +553,7 @@ func (c *Container) mountSharedDirMounts(hostSharedDir, guestSharedDir string) ( - - var ignore bool - var guestDest string -- guestDest, ignore, err = c.shareFiles(m, idx, hostSharedDir, guestSharedDir) -+ guestDest, ignore, err = c.shareFiles(m, idx, hostSharedDir, hostMountDir, guestSharedDir) - if err != nil { - return nil, nil, err - } -diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go -index a308952a8..a26e0411b 100644 ---- a/virtcontainers/kata_agent.go -+++ b/virtcontainers/kata_agent.go -@@ -1387,7 +1387,7 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process, - } - - // Handle container mounts -- newMounts, ignoredMounts, err := c.mountSharedDirMounts(getMountPath(sandbox.id), kataGuestSharedDir()) -+ newMounts, ignoredMounts, err := c.mountSharedDirMounts(getSharePath(sandbox.id), getMountPath(sandbox.id), kataGuestSharedDir()) - if err != nil { - return nil, err - } -diff --git a/virtcontainers/mount.go b/virtcontainers/mount.go -index ab9e9f8d0..2f9846e80 100644 ---- a/virtcontainers/mount.go -+++ b/virtcontainers/mount.go -@@ -273,6 +273,11 @@ func remount(ctx context.Context, mountflags uintptr, src string) error { - return nil - } - -+// remount a mount point as readonly -+func remountRo(ctx context.Context, src string) error { -+ return remount(ctx, syscall.MS_BIND|syscall.MS_RDONLY, src) -+} -+ - // bindMountContainerRootfs bind mounts a container rootfs into a 9pfs shared - // directory between the guest and the host. - func bindMountContainerRootfs(ctx context.Context, shareDir, cid, cRootFs string, readonly bool) error { -diff --git a/virtcontainers/sandbox_test.go b/virtcontainers/sandbox_test.go -index 55e0ffea0..9f9d50ac1 100644 ---- a/virtcontainers/sandbox_test.go -+++ b/virtcontainers/sandbox_test.go -@@ -1377,7 +1377,7 @@ func TestPreAddDevice(t *testing.T) { - }, - } - -- mounts, ignoreMounts, err := container.mountSharedDirMounts("", "") -+ mounts, ignoreMounts, err := container.mountSharedDirMounts("", "", "") - assert.Nil(t, err) - assert.Equal(t, len(mounts), 0, - "mounts should contain nothing because it only contains a block device") diff --git a/runtime/runtime-1.11.1.tar.gz b/runtime/runtime-1.11.1.tar.gz deleted file mode 100644 index b286238..0000000 Binary files a/runtime/runtime-1.11.1.tar.gz and /dev/null differ diff --git a/runtime/series.conf b/runtime/series.conf deleted file mode 100644 index 462da99..0000000 --- a/runtime/series.conf +++ /dev/null @@ -1,65 +0,0 @@ -0001-qmp-fix-kata-runtime-hungs-when-qemu-process-is-D-T-.patch -0002-kata-runtime-fix-kata-runtime-skip-read-lines-in-pro.patch -0003-kata-runtime-fix-kata-proxy-process-left-problem.patch -0004-kata-runtime-keep-the-process-name-of-qemu-same-as-c.patch -0005-cgroups-increase-delete-cgroup-retry-times.patch -0006-kata-runtime-fix-umount-container-rootfs-dir-return-.patch -0007-kata-runtime-enhance-reliability-when-kata-related-p.patch -0008-kata-runtime-fix-kata-runtime-resource-left-problem.patch -0009-kata-runtime-add-kata-runtime-global-flag-debug.patch -0010-kata-runtime-fix-kata-shim-pid-reused-problem.patch -0011-kata-runtime-check-the-process-info-before-send-SIGK.patch -0012-kata-runtime-truncate-the-log.json-file-before-kata-.patch -0013-kata-runtime-get-container-info-by-containerID-prefi.patch -0014-kata-runtime-add-self-defined-annotations-framework.patch -0015-kata-runtime-add-reuse-hypervisor-cpu-and-memory-fea.patch -0016-virtcontainers-fix-hotplug-huge-size-memory-cause-ag.patch -0017-kata-runtime-validate-sandbox-cpu-and-memory-size.patch -0018-sandbox-Stop-and-clean-up-containers-that-fail-to-cr.patch -0019-virtcontainers-fix-delete-sandbox-failed-problem.patch -0020-virtcontainers-add-enable_cpu_memory_hotplug-config-.patch -0021-kata-runtime-add-sandbox_cpu-and-sandbox_mem-annotat.patch -0022-kata-runtime-skip-go-version-check-and-do-not-build-.patch -0023-kata-runtime-set-PCIBridgeMaxCapacity-limit-to-25.patch -0024-kata-runtime-support-hotplug-tap-interface-into-kata.patch -0025-network-keep-list-ifaces-result-compatible-with-cni.patch -0026-network-add-enable_compat_old_cni-config.patch -0027-network-add-more-strict-check-for-input-network-inte.patch -0028-network-add-kata-network-add-route-subcommand.patch -0029-network-add-kata-network-del-route-subcommand.patch -0030-network-kata-network-list-routes-support-display-com.patch -0031-device_mangaer-check-VFIO-when-create-device.patch -0032-network-add-more-detail-usage-for-update-routes-subc.patch -0033-network-do-not-delete-the-exist-tap-device-in-the-ho.patch -0034-kata-runtime-add-kata-ipvs-command.patch -0035-device-mount-blockdevices-in-the-guest-VM.patch -0036-mount-limit-the-maximum-number-of-virtio-scsi-bus-sl.patch -0037-runtime-add-IPVS-test.patch -0038-pcie-using-pcie-root-port-driver-to-hotplug-device-i.patch -0039-storage-add-storage-common-functions-and-structs.patch -0040-storage-add-go-tests-for-storage.patch -0041-storage-mount-nfs-and-gpath-with-given-annotation.patch -0042-kata-runtime-do-not-ignore-updateInterface-return-er.patch -0043-kata-runtime-support-add-hypervisor-global-parameter.patch -0044-network-support-dpdk-vhost_user-net-device.patch -0045-network-support-set-dns.patch -0046-network-support-multiqueue-when-inserting-interface-.patch -0047-network-add-support-to-get-network-stats-of-vm-throu.patch -0048-console-fix-the-file-resource-leak.patch -0049-container-fix-the-write-operation-transparently-tran.patch -0050-runtime-add-kata-network-upate-iface-subcommand.patch -0051-network-fix-del-iface-doesn-t-delete-the-tap-interfa.patch -0052-runtime-add-support-of-new-sandbox-StratoVirt.patch -0053-kata-runtime-add-interface-for-host-cgroup.patch -0054-kata-runtime-add-sandbox-cgroup-with-vcpu-and-emulat.patch -0055-kata_runtime-support-host-cgroup-with-emulator-polic.patch -0056-kata_runtime-support-the-blkio-in-host-cgroups.patch -0057-kata-runtime-support-files-limit-in-host-cgroups.patch -0058-runtime-fix-sandboxRuntimeRootPath-left-problem.patch -0059-runtime-fix-invalid-cmdline-when-start-sandbox-stratovirt.patch -0060-runtime-fix-cmd-params-of-direct-use-stratovirt-bina.patch -0061-kata-runtime-retry-inserting-of-CNI-interface.patch -0062-kata-runtime-support-using-CNI-plugin-to-insert-muti.patch -0063-kata-runtime-fix-get-sandbox-cpu-resources-problem.patch -0064-runtime-add-support-for-stratovirt-of-kata-check-cli.patch -0065-runtime-fixup-that-the-getPids-function-returns-pid-.patch diff --git a/series.conf b/series.conf new file mode 100644 index 0000000..e69de29 diff --git a/shim/patches/0001-kata-shim-fix-kata-shim-process-wait-long-tim.patch b/shim/patches/0001-kata-shim-fix-kata-shim-process-wait-long-tim.patch deleted file mode 100644 index b75e064..0000000 --- a/shim/patches/0001-kata-shim-fix-kata-shim-process-wait-long-tim.patch +++ /dev/null @@ -1,38 +0,0 @@ -From f5d031ef7d3437a05fe41f80aaf0e59ac54a575b Mon Sep 17 00:00:00 2001 -From: jiangpengfei9 -Date: Mon, 1 Apr 2019 12:35:59 -0400 -Subject: [PATCH 2/2] kata-shim: fix kata-shim process wait long time - to exit problem - -reason: If containerd-shim process is killed and call kata-runtime delete -f command -to delete container, it will wait 10s for kata-shim process exit in the waitForShim -function, however 10s wait time is too long in the case of containerd-shim and containerd -process is killed.So when containerd-shim process is killed, kata-shim process will -receive a SIGHUP signal, we can handle this signal as exit signal which makes kata-shim -process exit. - -Signed-off-by: jiangpengfei9 ---- - shim.go | 6 ++++++ - 1 file changed, 6 insertions(+) - -diff --git a/shim.go b/shim.go -index a93a00d..90172fe 100644 ---- a/shim.go -+++ b/shim.go -@@ -134,6 +134,12 @@ func (s *shim) handleSignals(ctx context.Context, tty *os.File) chan os.Signal { - backtrace() - } - -+ // if containerd-shim process is killed, kata-shim will receive SIGHUP signal -+ if sysSig == syscall.SIGHUP { -+ logger().WithField("signal", sig).Warn("received SIGHUP signal") -+ os.Exit(1) -+ } -+ - // forward this signal to container - _, err := s.agent.SignalProcess(s.ctx, &pb.SignalProcessRequest{ - ContainerId: s.containerID, --- -1.8.3.1 - diff --git a/shim/series.conf b/shim/series.conf deleted file mode 100644 index ce1ab40..0000000 --- a/shim/series.conf +++ /dev/null @@ -1 +0,0 @@ -0001-kata-shim-fix-kata-shim-process-wait-long-tim.patch diff --git a/vendor.tar.gz b/vendor.tar.gz new file mode 100644 index 0000000..448b4f5 Binary files /dev/null and b/vendor.tar.gz differ