From cf2b34f477cba88641de3719bf5c8f933b919bcc Mon Sep 17 00:00:00 2001 From: holyfei Date: Mon, 17 Aug 2020 21:44:57 +0800 Subject: [PATCH 35/50] device: mount blockdevices in the guest VM reason: support mount blockdevices in the guest VM like "-v /dev/blockdevice:/home/test" Signed-off-by: yangfeiyu --- cli/config/configuration-qemu.toml.in | 5 ++++ pkg/katautils/config.go | 6 +++++ virtcontainers/kata_agent.go | 27 ++++++++++++++++--- virtcontainers/kata_agent_test.go | 2 +- virtcontainers/utils/utils.go | 50 +++++++++++++++++++++++++++++++++++ virtcontainers/vm_test.go | 2 +- 6 files changed, 86 insertions(+), 6 deletions(-) diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in index 46f8b632..aa11b38f 100644 --- a/cli/config/configuration-qemu.toml.in +++ b/cli/config/configuration-qemu.toml.in @@ -391,6 +391,11 @@ path = "@SHIMPATH@" # kernel_modules=[] +# If enabled, when we pass a block device to the guest VM +# through -v, such as `docker run -v /dev/loop100:/foo/bar`, +# agent will create a directory in guest VM and mount the +# file system on `/dev/loop100` inside the container +enable_blk_mount = true [netmon] # If enabled, the network monitoring process gets started when the diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go index 94c916a0..51120311 100644 --- a/pkg/katautils/config.go +++ b/pkg/katautils/config.go @@ -160,6 +160,7 @@ type agent struct { TraceMode string `toml:"trace_mode"` TraceType string `toml:"trace_type"` KernelModules []string `toml:"kernel_modules"` + MountBlkInVM bool `toml:"enable_blk_mount"` } type netmon struct { @@ -463,6 +464,10 @@ func (h hypervisor) getInitrdAndImage() (initrd string, image string, err error) return } +func (a agent) mountBlkDevInVM() bool { + return a.MountBlkInVM +} + func (p proxy) path() (string, error) { path := p.Path if path == "" { @@ -978,6 +983,7 @@ func updateRuntimeConfigAgent(configPath string, tomlConf tomlConfig, config *oc TraceMode: agent.traceMode(), TraceType: agent.traceType(), KernelModules: agent.kernelModules(), + MountBlkInVM: agent.mountBlkDevInVM(), } default: return fmt.Errorf("%s agent type is not supported", k) diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index 16662949..b0f88c15 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -30,6 +30,7 @@ import ( ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter" "github.com/kata-containers/runtime/virtcontainers/pkg/rootless" vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" + "github.com/kata-containers/runtime/virtcontainers/utils" "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" @@ -64,6 +65,7 @@ var ( errorMissingOCISpec = errors.New("Missing OCI specification") defaultKataHostSharedDir = "/run/kata-containers/shared/sandboxes/" defaultKataGuestSharedDir = "/run/kata-containers/shared/containers/" + kataGuestStorageDir = "/run/kata-containers/storage/containers/" mountGuestTag = "kataShared" defaultKataGuestSandboxDir = "/run/kata-containers/sandbox/" type9pFs = "9p" @@ -199,6 +201,7 @@ type KataAgentConfig struct { TraceMode string TraceType string KernelModules []string + MountBlkInVM bool } // KataAgentState is the structure describing the data stored from this @@ -1061,7 +1064,12 @@ func (k *kataAgent) replaceOCIMountsForStorages(spec *specs.Spec, volumeStorages // Create a temporary location to mount the Storage. Mounting to the correct location // will be handled by the OCI mount structure. filename := fmt.Sprintf("%s-%s", uuid.Generate().String(), filepath.Base(m.Destination)) - path := filepath.Join(kataGuestSandboxStorageDir(), filename) + var path string + if v.Driver == kataBlkDevType || v.Driver == kataSCSIDevType { + path = filepath.Join(kataGuestStorageDir, filename) + } else { + path = filepath.Join(kataGuestSandboxStorageDir(), filename) + } k.Logger().Debugf("Replacing OCI mount source (%s) with %s", m.Source, path) ociMounts[index].Source = path @@ -1436,7 +1444,7 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process, // Note this call modifies the list of container devices to make sure // all hotplugged devices are unplugged, so this needs be done // after devices passed with --device are handled. - volumeStorages, err := k.handleBlockVolumes(c) + volumeStorages, err := k.handleBlockVolumes(sandbox, c) if err != nil { return nil, err } @@ -1609,7 +1617,7 @@ func (k *kataAgent) handleVhostUserBlkVolume(c *Container, device api.Device) (* // handleBlockVolumes handles volumes that are block devices files // by passing the block devices as Storage to the agent. -func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) { +func (k *kataAgent) handleBlockVolumes(sandbox *Sandbox, c *Container) ([]*grpc.Storage, error) { var volumeStorages []*grpc.Storage @@ -1648,9 +1656,20 @@ func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) { } vol.MountPoint = m.Destination - if vol.Fstype == "" { + + ac, _ := sandbox.config.AgentConfig.(KataAgentConfig) + if ac.MountBlkInVM { + // Ensure the block device is formatted, for the devices here are specified as volumes + fsType, gerr := utils.GetDevFormat(m.Source) + if gerr != nil || fsType == "" { + k.Logger().WithField("device", id).WithError(gerr).Error("get device format failed") + return nil, gerr + } + vol.Fstype = fmt.Sprintf("bind-%s", fsType) + } else { vol.Fstype = "bind" } + if len(vol.Options) == 0 { vol.Options = []string{"bind"} } diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go index 18a5a0a6..68caaab2 100644 --- a/virtcontainers/kata_agent_test.go +++ b/virtcontainers/kata_agent_test.go @@ -446,7 +446,7 @@ func TestHandleBlockVolume(t *testing.T) { containers[c.id].sandbox = &sandbox containers[c.id].mounts = mounts - volumeStorages, err := k.handleBlockVolumes(c) + volumeStorages, err := k.handleBlockVolumes(&sandbox, c) assert.Nil(t, err, "Error while handling block volumes") vStorage := &pb.Storage{ diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go index 9490faa1..36ac67a7 100644 --- a/virtcontainers/utils/utils.go +++ b/virtcontainers/utils/utils.go @@ -379,3 +379,53 @@ func RoundVCPUNumber(value string) (int, error) { cpus := int(math.Ceil(cpuNum)) return cpus, nil } + +// GetDevFormat get the formated filesystem of input disk use `blkid`, Such as `ext4`,`xfs`,etc. +func GetDevFormat(disk string) (string, error) { + // refer to https://github.com/kubernetes/kubernetes/blob/v1.12.2/pkg/util/mount/mount_linux.go#L512 + args := []string{"-p", "-s", "TYPE", "-s", "PTTYPE", "-o", "export", disk} + dataOut, err := exec.Command("blkid", args...).Output() + output := string(dataOut) + + if err != nil { + if strings.Contains(err.Error(), "exit status 2") { + // Disk device is unformatted. + // For `blkid`, if the specified token (TYPE/PTTYPE, etc) was + // not found, or no (specified) devices could be identified, an + // exit code of 2 is returned. + return "", nil + } + return "", err + } + + var fstype string + + lines := strings.Split(output, "\n") + for _, l := range lines { + if len(l) <= 0 { + // Ignore empty line. + continue + } + // if we use busybox as rootfs,the output of command + // `busybox blkid` is different with original`blkid`, + // so we should make a compatible parse + subLine := strings.Split(l, " ") + for _, sl := range subLine { + if len(subLine) <= 0 { + continue + } + cs := strings.Split(sl, "=") + if len(cs) != 2 { + continue + } + if cs[0] == "TYPE" { + fstype = cs[1] + if strings.Contains(fstype, `"`) { + fstype = strings.Replace(fstype, `"`, "", -1) + } + } + } + } + + return fstype, nil +} diff --git a/virtcontainers/vm_test.go b/virtcontainers/vm_test.go index 36fd5c2f..d2414232 100644 --- a/virtcontainers/vm_test.go +++ b/virtcontainers/vm_test.go @@ -125,7 +125,7 @@ func TestVMConfigGrpc(t *testing.T) { HypervisorType: QemuHypervisor, HypervisorConfig: newQemuConfig(), AgentType: KataContainersAgent, - AgentConfig: KataAgentConfig{false, true, false, false, 0, "", "", []string{}}, + AgentConfig: KataAgentConfig{false, true, false, false, 0, "", "", []string{}, false }, ProxyType: NoopProxyType, } -- 2.14.3 (Apple Git-98)