From 77ed6fefe70edde63b01d797b76f389bc82bb1a0 Mon Sep 17 00:00:00 2001 From: Wei Gao Date: Mon, 9 Aug 2021 14:57:06 +0800 Subject: [PATCH 6/6] factory: add the template factory support for hypervisor type stratovirt. Signed-off-by: Wei Gao --- src/runtime/pkg/katautils/config.go | 2 +- .../factory/template/template.go | 21 +++-- src/runtime/virtcontainers/kata_agent.go | 7 +- src/runtime/virtcontainers/stratovirt.go | 89 +++++++++++++++++-- src/runtime/virtcontainers/vm.go | 28 ++++-- 5 files changed, 125 insertions(+), 22 deletions(-) diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index 828c2a43..718677b4 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -1363,7 +1363,7 @@ func checkNetNsConfig(config oci.RuntimeConfig) error { // checkFactoryConfig ensures the VM factory configuration is valid. func checkFactoryConfig(config oci.RuntimeConfig) error { if config.FactoryConfig.Template { - if config.HypervisorConfig.InitrdPath == "" { + if config.HypervisorConfig.InitrdPath == "" && (config.HypervisorType != vc.StratovirtHypervisor) { return errors.New("Factory option enable_template requires an initrd image") } } diff --git a/src/runtime/virtcontainers/factory/template/template.go b/src/runtime/virtcontainers/factory/template/template.go index 66070126..02497097 100644 --- a/src/runtime/virtcontainers/factory/template/template.go +++ b/src/runtime/virtcontainers/factory/template/template.go @@ -96,11 +96,15 @@ func (t *template) prepareTemplateFiles() error { if err != nil { return err } - flags := uintptr(syscall.MS_NOSUID | syscall.MS_NODEV) - opts := fmt.Sprintf("size=%dM", t.config.HypervisorConfig.MemorySize+templateDeviceStateSize) - if err = syscall.Mount("tmpfs", t.statePath, "tmpfs", flags, opts); err != nil { - t.close() - return err + + // If use hypervisor stratovirt, no need to create template path with ramdisk. + if t.config.HypervisorType != vc.StratovirtHypervisor { + flags := uintptr(syscall.MS_NOSUID | syscall.MS_NODEV) + opts := fmt.Sprintf("size=%dM", t.config.HypervisorConfig.MemorySize+templateDeviceStateSize) + if err = syscall.Mount("tmpfs", t.statePath, "tmpfs", flags, opts); err != nil { + t.close() + return err + } } f, err := os.Create(t.statePath + "/memory") if err != nil { @@ -126,8 +130,11 @@ func (t *template) createTemplateVM(ctx context.Context) error { } defer vm.Stop(ctx) - if err = vm.Disconnect(ctx); err != nil { - return err + // Create template on hypervisor stratovirt, don't have connection with agent. + if config.HypervisorType != vc.StratovirtHypervisor { + if err = vm.Disconnect(ctx); err != nil { + return err + } } // Sleep a bit to let the agent grpc server clean up diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 13d31658..bc882c70 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -1306,8 +1306,11 @@ func (k *kataAgent) buildContainerRootfs(ctx context.Context, sandbox *Sandbox, // TODO: remove dependency on shared fs path. shared fs is just one kind of storage source. // we should not always use shared fs path for all kinds of storage. Instead, all storage // should be bind mounted to a tmpfs path for containers to use. - if err := os.MkdirAll(filepath.Join(getMountPath(c.sandbox.id), c.id, c.rootfsSuffix), DirMode); err != nil { - return nil, err + // If boot from template on stratovirt, no need to mkdir mount path. + if !((sandbox.config.HypervisorType == StratovirtHypervisor) && sandbox.config.HypervisorConfig.BootFromTemplate) { + if err := os.MkdirAll(filepath.Join(getMountPath(c.sandbox.id), c.id, c.rootfsSuffix), DirMode); err != nil { + return nil, err + } } return rootfs, nil } diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go index 47daa817..e9b2ba85 100644 --- a/src/runtime/virtcontainers/stratovirt.go +++ b/src/runtime/virtcontainers/stratovirt.go @@ -48,6 +48,7 @@ type stratovirt struct { config HypervisorConfig rootfsPath string kernelPath string + templatePath string pid int consolePath string socketPath string @@ -115,7 +116,7 @@ func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS Net s.id = id s.config = *hypervisorConfig - if s.config.OzonePath == "" { + if (s.config.OzonePath == "") || s.config.BootToBeTemplate { s.useOzone = false s.pidfile = filepath.Join(s.store.RunVMStoragePath(), s.id, "pid") s.logfile = filepath.Join(s.store.RunVMStoragePath(), s.id, "/stratovirt.log") @@ -129,6 +130,20 @@ func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS Net s.socketPath = filepath.Join(s.ozoneRoot, apiSocket) s.consolePath = filepath.Join(s.ozoneRoot, debugSocket) } + + if s.config.VMid != "" && s.useOzone { + // Make sure the symlinks do not exist + os.RemoveAll(s.ozoneRoot) + ozoneVmRoot := filepath.Join(ozoneBaseDir, s.config.VMid) + if err := os.Symlink(ozoneVmRoot, s.ozoneRoot); err != nil { + return err + } + } + + if s.config.BootFromTemplate || s.config.BootToBeTemplate { + s.templatePath = strings.Replace(s.config.DevicesStatePath, "/state", "", -1) + } + s.netNSPath = networkNS.NetNsPath s.qmpMonitorCh = qmpChannel{ ctx: s.ctx, @@ -221,6 +236,12 @@ func (s *stratovirt) createOzoneParams(params []string) ([]string, error) { params = append(params, "-initrd", filepath.Base(s.rootfsPath)) } + // handle boot from template + if s.config.BootFromTemplate { + s.ozoneRes = append(s.ozoneRes, s.templatePath) + params = append(params, "-incoming", fmt.Sprintf("file:%s", filepath.Base(s.templatePath))) + } + // add devices to cmdline for _, d := range s.devices { switch v := d.dev.(type) { @@ -266,6 +287,11 @@ func (s *stratovirt) createParams(params []string) ([]string, error) { params = append(params, "-initrd", s.rootfsPath) } + // handle boot from template + if s.config.BootFromTemplate { + params = append(params, "-incoming", fmt.Sprintf("file:%s", s.templatePath)) + } + // add devices to cmdline for _, d := range s.devices { switch v := d.dev.(type) { @@ -410,14 +436,55 @@ func (s *stratovirt) stopSandbox(ctx context.Context, force bool) error { } func (s *stratovirt) pauseSandbox(ctx context.Context) error { - return nil + span, _ := s.trace(ctx, "pauseSandbox") + defer span.End() + + return s.togglePauseSandbox(ctx, true) } func (s *stratovirt) saveSandbox() error { + s.Logger().Info("save sandbox") + + err := s.qmpSetup() + if err != nil { + return err + } + + // BootToBeTemplate sets the VM to be a template that other VMs can can clone from. + // We would want to bypass shared memory when saving VM to local file through migrate. + if s.config.BootToBeTemplate { + err = s.qmpMonitorCh.qmp.ExecSetMigrateArguments(s.qmpMonitorCh.ctx, fmt.Sprintf("file:%s", s.templatePath)) + if err != nil { + s.Logger().WithError(err).Error("exec migration") + return err + } + } + return nil } func (s *stratovirt) resumeSandbox(ctx context.Context) error { + span, _ := s.trace(ctx, "resumeSandbox") + defer span.End() + + return s.togglePauseSandbox(ctx, false) +} + +func (s *stratovirt) togglePauseSandbox(ctx context.Context, pause bool) error { + span, _ := s.trace(ctx, "togglePauseSandbox") + defer span.End() + + err := s.qmpSetup() + if err != nil { + return err + } + + if pause { + s.qmpMonitorCh.qmp.ExecuteStop(s.qmpMonitorCh.ctx) + } else { + s.qmpMonitorCh.qmp.ExecuteCont(s.qmpMonitorCh.ctx) + } + return nil } @@ -734,11 +801,23 @@ func (s *stratovirt) updateOzoneRes(src string, add bool) (string, error) { } func (s *stratovirt) cleanOzoneRes() { - s.updateOzoneRes(s.rootfsPath, false) - s.updateOzoneRes(s.kernelPath, false) + // Umount all resource in ozoneRoot + if dir, err := ioutil.ReadDir(s.ozoneRoot); err == nil { + for _, file := range dir { + syscall.Unmount(filepath.Join(s.ozoneRoot, file.Name()), syscall.MNT_DETACH) + } + } if err := os.RemoveAll(s.ozoneRoot); err != nil { - s.Logger().WithField("cleanupOzone failed", err).Error() + s.Logger().WithField("cleanup Ozone failed", err).Error() + } + + // If have VMid, the VM is boot from template. ozoneVmRoot also need clean. + if s.config.VMid != "" { + ozoneVmRoot := filepath.Join(ozoneBaseDir, s.config.VMid) + if err := os.RemoveAll(ozoneVmRoot); err != nil { + s.Logger().WithField("cleanup Ozone failed", err).Error() + } } } diff --git a/src/runtime/virtcontainers/vm.go b/src/runtime/virtcontainers/vm.go index e6f02b6e..c4f9df73 100644 --- a/src/runtime/virtcontainers/vm.go +++ b/src/runtime/virtcontainers/vm.go @@ -142,13 +142,19 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) { }() // 4. check agent aliveness - // VMs booted from template are paused, do not check - if !config.HypervisorConfig.BootFromTemplate { + // On hypervisor StratoVirt, VMs booted from template are running, check agent + // On other hypervisors, VMs booted from template are paused, do not check + if config.HypervisorType == StratovirtHypervisor { + if !config.HypervisorConfig.BootToBeTemplate { + virtLog.WithField("vm", id).Info("check agent status") + err = agent.check(ctx) + } + } else if !config.HypervisorConfig.BootFromTemplate { virtLog.WithField("vm", id).Info("check agent status") err = agent.check(ctx) - if err != nil { - return nil, err - } + } + if err != nil { + return nil, err } return &VM{ @@ -329,9 +335,16 @@ func (v *VM) assignSandbox(s *Sandbox) error { // - link 9pfs share path from sandbox dir (/run/kata-containers/shared/sandboxes/sbid/) to vm dir (/run/vc/vm/vmid/shared/) vmSharePath := buildVMSharePath(v.id, v.store.RunVMStoragePath()) - vmSockDir := filepath.Join(v.store.RunVMStoragePath(), v.id) sbSharePath := getMountPath(s.id) - sbSockDir := filepath.Join(v.store.RunVMStoragePath(), s.id) + var vmSockDir string + var sbSockDir string + if v.hypervisor.hypervisorConfig().OzonePath != "" { + vmSockDir = filepath.Join(ozoneBaseDir, v.id) + sbSockDir = filepath.Join(ozoneBaseDir, s.id) + } else { + vmSockDir = filepath.Join(v.store.RunVMStoragePath(), v.id) + sbSockDir = filepath.Join(v.store.RunVMStoragePath(), s.id) + } v.logger().WithFields(logrus.Fields{ "vmSharePath": vmSharePath, @@ -359,6 +372,7 @@ func (v *VM) assignSandbox(s *Sandbox) error { s.hypervisor = v.hypervisor s.config.HypervisorConfig.VMid = v.id + s.config.HypervisorConfig.BootFromTemplate = true return nil } -- 2.21.1 (Apple Git-122.3)