e299de34ba
reason: adapt with iSulad and support for new hypervisor type "stratovirt" Signed-off-by: holyfei <yangfeiyu20092010@163.com>
296 lines
11 KiB
Diff
296 lines
11 KiB
Diff
From 77ed6fefe70edde63b01d797b76f389bc82bb1a0 Mon Sep 17 00:00:00 2001
|
|
From: Wei Gao <gaowei66@huawei.com>
|
|
Date: Mon, 9 Aug 2021 14:57:06 +0800
|
|
Subject: [PATCH 6/6] factory: add the template factory support for hypervisor
|
|
type stratovirt.
|
|
|
|
Signed-off-by: Wei Gao <gaowei66@huawei.com>
|
|
---
|
|
src/runtime/pkg/katautils/config.go | 2 +-
|
|
.../factory/template/template.go | 21 +++--
|
|
src/runtime/virtcontainers/kata_agent.go | 7 +-
|
|
src/runtime/virtcontainers/stratovirt.go | 89 +++++++++++++++++--
|
|
src/runtime/virtcontainers/vm.go | 28 ++++--
|
|
5 files changed, 125 insertions(+), 22 deletions(-)
|
|
|
|
diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go
|
|
index 828c2a43..718677b4 100644
|
|
--- a/src/runtime/pkg/katautils/config.go
|
|
+++ b/src/runtime/pkg/katautils/config.go
|
|
@@ -1363,7 +1363,7 @@ func checkNetNsConfig(config oci.RuntimeConfig) error {
|
|
// checkFactoryConfig ensures the VM factory configuration is valid.
|
|
func checkFactoryConfig(config oci.RuntimeConfig) error {
|
|
if config.FactoryConfig.Template {
|
|
- if config.HypervisorConfig.InitrdPath == "" {
|
|
+ if config.HypervisorConfig.InitrdPath == "" && (config.HypervisorType != vc.StratovirtHypervisor) {
|
|
return errors.New("Factory option enable_template requires an initrd image")
|
|
}
|
|
}
|
|
diff --git a/src/runtime/virtcontainers/factory/template/template.go b/src/runtime/virtcontainers/factory/template/template.go
|
|
index 66070126..02497097 100644
|
|
--- a/src/runtime/virtcontainers/factory/template/template.go
|
|
+++ b/src/runtime/virtcontainers/factory/template/template.go
|
|
@@ -96,11 +96,15 @@ func (t *template) prepareTemplateFiles() error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
- flags := uintptr(syscall.MS_NOSUID | syscall.MS_NODEV)
|
|
- opts := fmt.Sprintf("size=%dM", t.config.HypervisorConfig.MemorySize+templateDeviceStateSize)
|
|
- if err = syscall.Mount("tmpfs", t.statePath, "tmpfs", flags, opts); err != nil {
|
|
- t.close()
|
|
- return err
|
|
+
|
|
+ // If use hypervisor stratovirt, no need to create template path with ramdisk.
|
|
+ if t.config.HypervisorType != vc.StratovirtHypervisor {
|
|
+ flags := uintptr(syscall.MS_NOSUID | syscall.MS_NODEV)
|
|
+ opts := fmt.Sprintf("size=%dM", t.config.HypervisorConfig.MemorySize+templateDeviceStateSize)
|
|
+ if err = syscall.Mount("tmpfs", t.statePath, "tmpfs", flags, opts); err != nil {
|
|
+ t.close()
|
|
+ return err
|
|
+ }
|
|
}
|
|
f, err := os.Create(t.statePath + "/memory")
|
|
if err != nil {
|
|
@@ -126,8 +130,11 @@ func (t *template) createTemplateVM(ctx context.Context) error {
|
|
}
|
|
defer vm.Stop(ctx)
|
|
|
|
- if err = vm.Disconnect(ctx); err != nil {
|
|
- return err
|
|
+ // Create template on hypervisor stratovirt, don't have connection with agent.
|
|
+ if config.HypervisorType != vc.StratovirtHypervisor {
|
|
+ if err = vm.Disconnect(ctx); err != nil {
|
|
+ return err
|
|
+ }
|
|
}
|
|
|
|
// Sleep a bit to let the agent grpc server clean up
|
|
diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go
|
|
index 13d31658..bc882c70 100644
|
|
--- a/src/runtime/virtcontainers/kata_agent.go
|
|
+++ b/src/runtime/virtcontainers/kata_agent.go
|
|
@@ -1306,8 +1306,11 @@ func (k *kataAgent) buildContainerRootfs(ctx context.Context, sandbox *Sandbox,
|
|
// TODO: remove dependency on shared fs path. shared fs is just one kind of storage source.
|
|
// we should not always use shared fs path for all kinds of storage. Instead, all storage
|
|
// should be bind mounted to a tmpfs path for containers to use.
|
|
- if err := os.MkdirAll(filepath.Join(getMountPath(c.sandbox.id), c.id, c.rootfsSuffix), DirMode); err != nil {
|
|
- return nil, err
|
|
+ // If boot from template on stratovirt, no need to mkdir mount path.
|
|
+ if !((sandbox.config.HypervisorType == StratovirtHypervisor) && sandbox.config.HypervisorConfig.BootFromTemplate) {
|
|
+ if err := os.MkdirAll(filepath.Join(getMountPath(c.sandbox.id), c.id, c.rootfsSuffix), DirMode); err != nil {
|
|
+ return nil, err
|
|
+ }
|
|
}
|
|
return rootfs, nil
|
|
}
|
|
diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go
|
|
index 47daa817..e9b2ba85 100644
|
|
--- a/src/runtime/virtcontainers/stratovirt.go
|
|
+++ b/src/runtime/virtcontainers/stratovirt.go
|
|
@@ -48,6 +48,7 @@ type stratovirt struct {
|
|
config HypervisorConfig
|
|
rootfsPath string
|
|
kernelPath string
|
|
+ templatePath string
|
|
pid int
|
|
consolePath string
|
|
socketPath string
|
|
@@ -115,7 +116,7 @@ func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS Net
|
|
|
|
s.id = id
|
|
s.config = *hypervisorConfig
|
|
- if s.config.OzonePath == "" {
|
|
+ if (s.config.OzonePath == "") || s.config.BootToBeTemplate {
|
|
s.useOzone = false
|
|
s.pidfile = filepath.Join(s.store.RunVMStoragePath(), s.id, "pid")
|
|
s.logfile = filepath.Join(s.store.RunVMStoragePath(), s.id, "/stratovirt.log")
|
|
@@ -129,6 +130,20 @@ func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS Net
|
|
s.socketPath = filepath.Join(s.ozoneRoot, apiSocket)
|
|
s.consolePath = filepath.Join(s.ozoneRoot, debugSocket)
|
|
}
|
|
+
|
|
+ if s.config.VMid != "" && s.useOzone {
|
|
+ // Make sure the symlinks do not exist
|
|
+ os.RemoveAll(s.ozoneRoot)
|
|
+ ozoneVmRoot := filepath.Join(ozoneBaseDir, s.config.VMid)
|
|
+ if err := os.Symlink(ozoneVmRoot, s.ozoneRoot); err != nil {
|
|
+ return err
|
|
+ }
|
|
+ }
|
|
+
|
|
+ if s.config.BootFromTemplate || s.config.BootToBeTemplate {
|
|
+ s.templatePath = strings.Replace(s.config.DevicesStatePath, "/state", "", -1)
|
|
+ }
|
|
+
|
|
s.netNSPath = networkNS.NetNsPath
|
|
s.qmpMonitorCh = qmpChannel{
|
|
ctx: s.ctx,
|
|
@@ -221,6 +236,12 @@ func (s *stratovirt) createOzoneParams(params []string) ([]string, error) {
|
|
params = append(params, "-initrd", filepath.Base(s.rootfsPath))
|
|
}
|
|
|
|
+ // handle boot from template
|
|
+ if s.config.BootFromTemplate {
|
|
+ s.ozoneRes = append(s.ozoneRes, s.templatePath)
|
|
+ params = append(params, "-incoming", fmt.Sprintf("file:%s", filepath.Base(s.templatePath)))
|
|
+ }
|
|
+
|
|
// add devices to cmdline
|
|
for _, d := range s.devices {
|
|
switch v := d.dev.(type) {
|
|
@@ -266,6 +287,11 @@ func (s *stratovirt) createParams(params []string) ([]string, error) {
|
|
params = append(params, "-initrd", s.rootfsPath)
|
|
}
|
|
|
|
+ // handle boot from template
|
|
+ if s.config.BootFromTemplate {
|
|
+ params = append(params, "-incoming", fmt.Sprintf("file:%s", s.templatePath))
|
|
+ }
|
|
+
|
|
// add devices to cmdline
|
|
for _, d := range s.devices {
|
|
switch v := d.dev.(type) {
|
|
@@ -410,14 +436,55 @@ func (s *stratovirt) stopSandbox(ctx context.Context, force bool) error {
|
|
}
|
|
|
|
func (s *stratovirt) pauseSandbox(ctx context.Context) error {
|
|
- return nil
|
|
+ span, _ := s.trace(ctx, "pauseSandbox")
|
|
+ defer span.End()
|
|
+
|
|
+ return s.togglePauseSandbox(ctx, true)
|
|
}
|
|
|
|
func (s *stratovirt) saveSandbox() error {
|
|
+ s.Logger().Info("save sandbox")
|
|
+
|
|
+ err := s.qmpSetup()
|
|
+ if err != nil {
|
|
+ return err
|
|
+ }
|
|
+
|
|
+ // BootToBeTemplate sets the VM to be a template that other VMs can can clone from.
|
|
+ // We would want to bypass shared memory when saving VM to local file through migrate.
|
|
+ if s.config.BootToBeTemplate {
|
|
+ err = s.qmpMonitorCh.qmp.ExecSetMigrateArguments(s.qmpMonitorCh.ctx, fmt.Sprintf("file:%s", s.templatePath))
|
|
+ if err != nil {
|
|
+ s.Logger().WithError(err).Error("exec migration")
|
|
+ return err
|
|
+ }
|
|
+ }
|
|
+
|
|
return nil
|
|
}
|
|
|
|
func (s *stratovirt) resumeSandbox(ctx context.Context) error {
|
|
+ span, _ := s.trace(ctx, "resumeSandbox")
|
|
+ defer span.End()
|
|
+
|
|
+ return s.togglePauseSandbox(ctx, false)
|
|
+}
|
|
+
|
|
+func (s *stratovirt) togglePauseSandbox(ctx context.Context, pause bool) error {
|
|
+ span, _ := s.trace(ctx, "togglePauseSandbox")
|
|
+ defer span.End()
|
|
+
|
|
+ err := s.qmpSetup()
|
|
+ if err != nil {
|
|
+ return err
|
|
+ }
|
|
+
|
|
+ if pause {
|
|
+ s.qmpMonitorCh.qmp.ExecuteStop(s.qmpMonitorCh.ctx)
|
|
+ } else {
|
|
+ s.qmpMonitorCh.qmp.ExecuteCont(s.qmpMonitorCh.ctx)
|
|
+ }
|
|
+
|
|
return nil
|
|
}
|
|
|
|
@@ -734,11 +801,23 @@ func (s *stratovirt) updateOzoneRes(src string, add bool) (string, error) {
|
|
}
|
|
|
|
func (s *stratovirt) cleanOzoneRes() {
|
|
- s.updateOzoneRes(s.rootfsPath, false)
|
|
- s.updateOzoneRes(s.kernelPath, false)
|
|
+ // Umount all resource in ozoneRoot
|
|
+ if dir, err := ioutil.ReadDir(s.ozoneRoot); err == nil {
|
|
+ for _, file := range dir {
|
|
+ syscall.Unmount(filepath.Join(s.ozoneRoot, file.Name()), syscall.MNT_DETACH)
|
|
+ }
|
|
+ }
|
|
|
|
if err := os.RemoveAll(s.ozoneRoot); err != nil {
|
|
- s.Logger().WithField("cleanupOzone failed", err).Error()
|
|
+ s.Logger().WithField("cleanup Ozone failed", err).Error()
|
|
+ }
|
|
+
|
|
+ // If have VMid, the VM is boot from template. ozoneVmRoot also need clean.
|
|
+ if s.config.VMid != "" {
|
|
+ ozoneVmRoot := filepath.Join(ozoneBaseDir, s.config.VMid)
|
|
+ if err := os.RemoveAll(ozoneVmRoot); err != nil {
|
|
+ s.Logger().WithField("cleanup Ozone failed", err).Error()
|
|
+ }
|
|
}
|
|
}
|
|
|
|
diff --git a/src/runtime/virtcontainers/vm.go b/src/runtime/virtcontainers/vm.go
|
|
index e6f02b6e..c4f9df73 100644
|
|
--- a/src/runtime/virtcontainers/vm.go
|
|
+++ b/src/runtime/virtcontainers/vm.go
|
|
@@ -142,13 +142,19 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) {
|
|
}()
|
|
|
|
// 4. check agent aliveness
|
|
- // VMs booted from template are paused, do not check
|
|
- if !config.HypervisorConfig.BootFromTemplate {
|
|
+ // On hypervisor StratoVirt, VMs booted from template are running, check agent
|
|
+ // On other hypervisors, VMs booted from template are paused, do not check
|
|
+ if config.HypervisorType == StratovirtHypervisor {
|
|
+ if !config.HypervisorConfig.BootToBeTemplate {
|
|
+ virtLog.WithField("vm", id).Info("check agent status")
|
|
+ err = agent.check(ctx)
|
|
+ }
|
|
+ } else if !config.HypervisorConfig.BootFromTemplate {
|
|
virtLog.WithField("vm", id).Info("check agent status")
|
|
err = agent.check(ctx)
|
|
- if err != nil {
|
|
- return nil, err
|
|
- }
|
|
+ }
|
|
+ if err != nil {
|
|
+ return nil, err
|
|
}
|
|
|
|
return &VM{
|
|
@@ -329,9 +335,16 @@ func (v *VM) assignSandbox(s *Sandbox) error {
|
|
// - link 9pfs share path from sandbox dir (/run/kata-containers/shared/sandboxes/sbid/) to vm dir (/run/vc/vm/vmid/shared/)
|
|
|
|
vmSharePath := buildVMSharePath(v.id, v.store.RunVMStoragePath())
|
|
- vmSockDir := filepath.Join(v.store.RunVMStoragePath(), v.id)
|
|
sbSharePath := getMountPath(s.id)
|
|
- sbSockDir := filepath.Join(v.store.RunVMStoragePath(), s.id)
|
|
+ var vmSockDir string
|
|
+ var sbSockDir string
|
|
+ if v.hypervisor.hypervisorConfig().OzonePath != "" {
|
|
+ vmSockDir = filepath.Join(ozoneBaseDir, v.id)
|
|
+ sbSockDir = filepath.Join(ozoneBaseDir, s.id)
|
|
+ } else {
|
|
+ vmSockDir = filepath.Join(v.store.RunVMStoragePath(), v.id)
|
|
+ sbSockDir = filepath.Join(v.store.RunVMStoragePath(), s.id)
|
|
+ }
|
|
|
|
v.logger().WithFields(logrus.Fields{
|
|
"vmSharePath": vmSharePath,
|
|
@@ -359,6 +372,7 @@ func (v *VM) assignSandbox(s *Sandbox) error {
|
|
|
|
s.hypervisor = v.hypervisor
|
|
s.config.HypervisorConfig.VMid = v.id
|
|
+ s.config.HypervisorConfig.BootFromTemplate = true
|
|
|
|
return nil
|
|
}
|
|
--
|
|
2.21.1 (Apple Git-122.3)
|
|
|