From 459e53cedad524b83a9772f20ebdaa65ba1d17f1 Mon Sep 17 00:00:00 2001 From: DCCooper <1866858@gmail.com> Date: Wed, 3 Nov 2021 15:17:10 +0800 Subject: [PATCH] isula-build: sync upstream to fix loaded images cover existing images name and tag Signed-off-by: DCCooper <1866858@gmail.com> (cherry picked from commit 69a110a5d4162290bfee7e3e93ca46670b51fdfb) --- VERSION-openeuler | 2 +- git-commit | 2 +- isula-build.spec | 8 +- ...ages-cover-existing-images-name-and-.patch | 434 ++++++++++++++++++ series.conf | 1 + 5 files changed, 444 insertions(+), 3 deletions(-) create mode 100644 patch/0088-bugfix-loaded-images-cover-existing-images-name-and-.patch diff --git a/VERSION-openeuler b/VERSION-openeuler index 6080c78..3b6ec87 100644 --- a/VERSION-openeuler +++ b/VERSION-openeuler @@ -1 +1 @@ -0.9.5-17 +0.9.5-18 diff --git a/git-commit b/git-commit index 53aa02f..317f3e6 100644 --- a/git-commit +++ b/git-commit @@ -1 +1 @@ -b9391035f9e5c9ae2729b7be920d3c72284d60c0 +f7609ebf9b086beeb7ef204cb89a70c4a34fde84 diff --git a/isula-build.spec b/isula-build.spec index 0a28853..8e6c3a3 100644 --- a/isula-build.spec +++ b/isula-build.spec @@ -2,7 +2,7 @@ Name: isula-build Version: 0.9.5 -Release: 17 +Release: 18 Summary: A tool to build container images License: Mulan PSL V2 URL: https://gitee.com/openeuler/isula-build @@ -85,6 +85,12 @@ fi /usr/share/bash-completion/completions/isula-build %changelog +* Wed Nov 03 2021 lixiang - 0.9.5-18 +- Type:bugfix +- CVE:NA +- SUG:restart +- DESC:fix loaded images cover existing images name and tag + * Wed Nov 03 2021 DCCooper <1866858@gmail.com> - 0.9.5-17 - Type:bugfix - CVE:NA diff --git a/patch/0088-bugfix-loaded-images-cover-existing-images-name-and-.patch b/patch/0088-bugfix-loaded-images-cover-existing-images-name-and-.patch new file mode 100644 index 0000000..ad75be5 --- /dev/null +++ b/patch/0088-bugfix-loaded-images-cover-existing-images-name-and-.patch @@ -0,0 +1,434 @@ +From 6efcb2e785e452505894b0e1e589e72487439e17 Mon Sep 17 00:00:00 2001 +From: xingweizheng +Date: Wed, 27 Oct 2021 18:27:11 +0800 +Subject: [PATCH] bugfix: loaded images cover existing images name and tag + +--- + daemon/load.go | 114 +++++++++++++++++++++++++++++++------------- + daemon/load_test.go | 26 +++++----- + image/image.go | 66 ++++++++++++------------- + 3 files changed, 129 insertions(+), 77 deletions(-) + +diff --git a/daemon/load.go b/daemon/load.go +index 41690ab..b6d675b 100644 +--- a/daemon/load.go ++++ b/daemon/load.go +@@ -17,6 +17,8 @@ import ( + "io/ioutil" + "os" + "path/filepath" ++ "context" ++ "strings" + + "github.com/containers/image/v5/docker/tarfile" + ociarchive "github.com/containers/image/v5/oci/archive" +@@ -25,6 +27,7 @@ import ( + "github.com/containers/storage" + "github.com/containers/storage/pkg/archive" + securejoin "github.com/cyphar/filepath-securejoin" ++ digest "github.com/opencontainers/go-digest" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +@@ -72,6 +75,12 @@ type loadOptions struct { + sep separatorLoad + } + ++type singleImage struct { ++ index int ++ id string ++ nameTag []string ++} ++ + func (b *Backend) getLoadOptions(req *pb.LoadRequest) (loadOptions, error) { + var opt = loadOptions{ + path: req.GetPath(), +@@ -119,10 +128,8 @@ func (b *Backend) Load(req *pb.LoadRequest, stream pb.Control_LoadServer) error + "LoadID": req.GetLoadID(), + }).Info("LoadRequest received") + +- var ( +- si *storage.Image +- repoTags [][]string +- ) ++ var si *storage.Image ++ + opts, err := b.getLoadOptions(req) + if err != nil { + return errors.Wrap(err, "process load options failed") +@@ -142,7 +149,7 @@ func (b *Backend) Load(req *pb.LoadRequest, stream pb.Control_LoadServer) error + } + } + +- repoTags, err = tryToParseImageFormatFromTarball(b.daemon.opts.DataRoot, &opts) ++ imagesInTar, err := tryToParseImageFormatFromTarball(b.daemon.opts.DataRoot, &opts) + if err != nil { + return err + } +@@ -163,24 +170,30 @@ func (b *Backend) Load(req *pb.LoadRequest, stream pb.Control_LoadServer) error + eg.Go(func() error { + defer log.CloseContent() + +- for index, nameAndTag := range repoTags { ++ for _, singleImage := range imagesInTar { + _, si, err = image.ResolveFromImage(&image.PrepareImageOptions{ + Ctx: ctx, + FromImage: exporter.FormatTransport(opts.format, opts.path), ++ ToImage: singleImage.id, + SystemContext: image.GetSystemContext(), + Store: b.daemon.localStore, + Reporter: log, +- ManifestIndex: index, ++ ManifestIndex: singleImage.index, + }) + if err != nil { + return err + } + +- if sErr := b.daemon.localStore.SetNames(si.ID, nameAndTag); sErr != nil { +- return sErr ++ originalNames, err := b.daemon.localStore.Names(si.ID) ++ if err != nil { ++ return err ++ } ++ if err = b.daemon.localStore.SetNames(si.ID, append(originalNames, singleImage.nameTag...)); err != nil { ++ return err + } + + log.Print("Loaded image as %s\n", si.ID) ++ logrus.Infof("Loaded image as %s", si.ID) + } + + return nil +@@ -189,17 +202,11 @@ func (b *Backend) Load(req *pb.LoadRequest, stream pb.Control_LoadServer) error + if wErr := eg.Wait(); wErr != nil { + return wErr + } +- logrus.Infof("Loaded image as %s", si.ID) + + return nil + } + +-func tryToParseImageFormatFromTarball(dataRoot string, opts *loadOptions) ([][]string, error) { +- var ( +- allRepoTags [][]string +- err error +- ) +- ++func tryToParseImageFormatFromTarball(dataRoot string, opts *loadOptions) ([]singleImage, error) { + // tmp dir will be removed after NewSourceFromFileWithContext + tmpDir, err := securejoin.SecureJoin(dataRoot, dataRootTmpDirPrefix) + if err != nil { +@@ -208,19 +215,21 @@ func tryToParseImageFormatFromTarball(dataRoot string, opts *loadOptions) ([][]s + systemContext := image.GetSystemContext() + systemContext.BigFilesTemporaryDir = tmpDir + +- allRepoTags, err = getDockerRepoTagFromImageTar(systemContext, opts.path) ++ // try docker format loading ++ imagesInTar, err := getDockerRepoTagFromImageTar(systemContext, opts.path) + if err == nil { + logrus.Infof("Parse image successful with %q format", constant.DockerTransport) + opts.format = constant.DockerArchiveTransport +- return allRepoTags, nil ++ return imagesInTar, nil + } + logrus.Warnf("Try to Parse image of docker format failed with error: %v", err) + +- allRepoTags, err = getOCIRepoTagFromImageTar(systemContext, opts.path) ++ // try oci format loading ++ imagesInTar, err = getOCIRepoTagFromImageTar(systemContext, opts.path) + if err == nil { + logrus.Infof("Parse image successful with %q format", constant.OCITransport) + opts.format = constant.OCIArchiveTransport +- return allRepoTags, nil ++ return imagesInTar, nil + } + logrus.Warnf("Try to parse image of oci format failed with error: %v", err) + +@@ -228,7 +237,7 @@ func tryToParseImageFormatFromTarball(dataRoot string, opts *loadOptions) ([][]s + return nil, errors.Wrap(err, "wrong image format detected from local tarball") + } + +-func getDockerRepoTagFromImageTar(systemContext *types.SystemContext, path string) ([][]string, error) { ++func getDockerRepoTagFromImageTar(systemContext *types.SystemContext, path string) ([]singleImage, error) { + // tmp dir will be removed after NewSourceFromFileWithContext + tarfileSource, err := tarfile.NewSourceFromFileWithContext(systemContext, path) + if err != nil { +@@ -245,35 +254,74 @@ func getDockerRepoTagFromImageTar(systemContext *types.SystemContext, path strin + return nil, errors.Errorf("failed to get the top level image manifest: %v", err) + } + +- var allRepoTags [][]string +- for _, manifestItem := range topLevelImageManifest { +- allRepoTags = append(allRepoTags, manifestItem.RepoTags) ++ imagesInTar := make([]singleImage, 0, len(topLevelImageManifest)) ++ for i, manifestItem := range topLevelImageManifest { ++ imageID, err := parseConfigID(manifestItem.Config) ++ if err != nil { ++ return nil, err ++ } ++ imagesInTar = append(imagesInTar, singleImage{index: i, id: imageID, nameTag: manifestItem.RepoTags}) + } + +- return allRepoTags, nil ++ return imagesInTar, nil + } + +-func getOCIRepoTagFromImageTar(systemContext *types.SystemContext, path string) ([][]string, error) { +- var ( +- err error +- ) +- ++func getOCIRepoTagFromImageTar(systemContext *types.SystemContext, path string) ([]singleImage, error) { + srcRef, err := alltransports.ParseImageName(exporter.FormatTransport(constant.OCIArchiveTransport, path)) + if err != nil { + return nil, errors.Wrap(err, "failed to parse image name of oci image format") + } + ++ imageID, err := getLoadedImageID(srcRef, systemContext) ++ if err != nil { ++ return nil, err ++ } + tarManifest, err := ociarchive.LoadManifestDescriptorWithContext(systemContext, srcRef) + if err != nil { + return nil, errors.Wrap(err, "failed to load manifest descriptor of oci image format") + } + +- // For now, we only support load single image in archive file ++ // For now, we only support loading oci-archive file with one single image + if _, ok := tarManifest.Annotations[imgspecv1.AnnotationRefName]; ok { +- return [][]string{{tarManifest.Annotations[imgspecv1.AnnotationRefName]}}, nil ++ return []singleImage{{0, imageID, []string{tarManifest.Annotations[imgspecv1.AnnotationRefName]}}}, nil ++ } ++ return []singleImage{{0, imageID, []string{}}}, nil ++} ++ ++func parseConfigID(configID string) (string, error) { ++ parts := strings.SplitN(configID, ".", 2) ++ if len(parts) != 2 { ++ return "", errors.New("wrong config info of manifest.json") ++ } ++ ++ configDigest := "sha256:" + digest.Digest(parts[0]) ++ if err := configDigest.Validate(); err != nil { ++ return "", errors.Wrapf(err, "failed to get config info") ++ } ++ ++ return "@" + configDigest.Encoded(), nil ++} ++ ++func getLoadedImageID(imageRef types.ImageReference, systemContext *types.SystemContext) (string, error) { ++ if imageRef == nil || systemContext == nil { ++ return "", errors.New("nil image reference or system context when loading image") ++ } ++ ++ newImage, err := imageRef.NewImage(context.TODO(), systemContext) ++ if err != nil { ++ return "", err ++ } ++ defer func() { ++ if err = newImage.Close(); err != nil { ++ logrus.Errorf("failed to close image: %v", err) ++ } ++ }() ++ imageDigest := newImage.ConfigInfo().Digest ++ if err = imageDigest.Validate(); err != nil { ++ return "", errors.Wrap(err, "failed to get config info") + } + +- return [][]string{{}}, nil ++ return "@" + imageDigest.Encoded(), nil + } + + func loadSeparatedImage(opt *loadOptions) error { +diff --git a/daemon/load_test.go b/daemon/load_test.go +index cbcb5d8..860b897 100644 +--- a/daemon/load_test.go ++++ b/daemon/load_test.go +@@ -31,9 +31,9 @@ import ( + ) + + const ( +- loadedTarFile = "load.tar" ++ loadedTarFile = "load.tar" + manifestJSONFile = "manifest.json" +- indexJSONFile = "index.json" ++ indexJSONFile = "index.json" + ) + + var ( +@@ -167,8 +167,10 @@ func TestLoadSingleImage(t *testing.T) { + } + ] + }`, +- format: "oci", +- withTag: true, ++ format: "oci", ++ withTag: true, ++ wantErr: true, ++ errString: "no such file or directory", + }, + { + name: "TC3 normal case load docker tar with no RepoTags", +@@ -197,8 +199,10 @@ func TestLoadSingleImage(t *testing.T) { + } + ] + }`, +- format: "oci", +- withTag: false, ++ format: "oci", ++ withTag: false, ++ wantErr: true, ++ errString: "no such file or directory", + }, + { + name: "TC5 abnormal case load docker tar with wrong manifestJSON", +@@ -217,7 +221,7 @@ func TestLoadSingleImage(t *testing.T) { + format: "docker", + withTag: true, + wantErr: true, +- errString: "error loading index", ++ errString: "no such file or directory", + }, + { + name: "TC6 abnormal case with wrong tar path", +@@ -312,10 +316,10 @@ func TestLoadMultipleImages(t *testing.T) { + path := dir.Join(loadedTarFile) + repoTags, err := tryToParseImageFormatFromTarball(daemon.opts.DataRoot, &loadOptions{path: path}) + assert.NilError(t, err) +- assert.Equal(t, repoTags[0][0], "registry.example.com/sayhello:first") +- assert.Equal(t, repoTags[1][0], "registry.example.com/sayhello:second") +- assert.Equal(t, repoTags[1][1], "registry.example.com/sayhello:third") +- assert.Equal(t, len(repoTags[2]), 0) ++ assert.Equal(t, repoTags[0].nameTag[0], "registry.example.com/sayhello:first") ++ assert.Equal(t, repoTags[1].nameTag[0], "registry.example.com/sayhello:second") ++ assert.Equal(t, repoTags[1].nameTag[1], "registry.example.com/sayhello:third") ++ assert.Equal(t, len(repoTags[2].nameTag), 0) + + req := &pb.LoadRequest{Path: path} + stream := &controlLoadServer{} +diff --git a/image/image.go b/image/image.go +index 5dda185..b24cb41 100644 +--- a/image/image.go ++++ b/image/image.go +@@ -37,7 +37,6 @@ import ( + "github.com/containers/image/v5/transports/alltransports" + "github.com/containers/image/v5/types" + "github.com/containers/storage" +- "github.com/containers/storage/pkg/stringid" + "github.com/opencontainers/go-digest" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +@@ -55,6 +54,7 @@ type PrepareImageOptions struct { + SystemContext *types.SystemContext + Ctx context.Context + FromImage string ++ ToImage string + Store *store.Store + Reporter io.Writer + ManifestIndex int +@@ -125,20 +125,17 @@ func PullAndGetImageInfo(opt *PrepareImageOptions) (types.ImageReference, *stora + } + + if transport == "" { +- // if the image can be obtained from the local storage by image id, +- // then only one image can be obtained. ++ // if the image can be obtained from the local storage by image id, then only one image can be obtained. + if len(candidates) != 1 { + return nil, nil, errors.New("transport is empty and multi or no image be found") + } +- img, err := opt.Store.Image(candidates[0]) +- if err != nil { +- pLog.Errorf("Failed to find the image %q in local store: %v", candidates[0], err) +- return nil, nil, err +- } +- ref, err := is.Transport.ParseStoreReference(opt.Store, img.ID) +- if err != nil { +- return nil, nil, errors.Wrapf(err, "failed to get the ref in store by %q", candidates[0]) ++ ++ ref, img, fErr := FindImage(opt.Store, candidates[0]) ++ if fErr != nil { ++ pLog.Errorf("Failed to find the image %q in local store: %v", candidates[0], fErr) ++ return nil, nil, fErr + } ++ + pLog.Infof("Get image from local store first search by %q", opt.FromImage) + return ref, img, nil + } +@@ -147,25 +144,33 @@ func PullAndGetImageInfo(opt *PrepareImageOptions) (types.ImageReference, *stora + var errPull error + for _, strImage := range candidates { + var ( +- srcRef types.ImageReference +- pErr error ++ srcRef types.ImageReference ++ destImage string + ) + + imageName := exporter.FormatTransport(transport, strImage) +- if transport == constant.DockerArchiveTransport { +- srcRef, pErr = alltransports.ParseImageName(imageName + ":@" + strconv.Itoa(opt.ManifestIndex)) +- } else { +- srcRef, pErr = alltransports.ParseImageName(imageName) +- } +- if pErr != nil { +- pLog.Debugf("Failed to parse the image %q: %v", imageName, pErr) +- continue +- } +- +- destImage, err := getLocalImageNameFromRef(opt.Store, srcRef) +- if err != nil { +- pLog.Debugf("Failed to get local image name for %q: %v", imageName, err) +- continue ++ switch transport { ++ case constant.DockerArchiveTransport: ++ if srcRef, err = alltransports.ParseImageName(imageName + ":@" + strconv.Itoa(opt.ManifestIndex)); err != nil { ++ pLog.Debugf("Failed to parse the image %q with %q transport: %v", imageName, constant.DockerArchiveTransport, err) ++ continue ++ } ++ destImage = opt.ToImage ++ case constant.OCIArchiveTransport: ++ if srcRef, err = alltransports.ParseImageName(imageName); err != nil { ++ pLog.Debugf("Failed to parse the image %q with %q transport: %v", imageName, constant.OCIArchiveTransport, err) ++ continue ++ } ++ destImage = opt.ToImage ++ default: ++ if srcRef, err = alltransports.ParseImageName(imageName); err != nil { ++ pLog.Debugf("Failed to get local image name for %q: %v", imageName, err) ++ continue ++ } ++ if destImage, err = getLocalImageNameFromRef(srcRef); err != nil { ++ pLog.Debugf("Failed to parse store reference for %q: %v", destImage, err) ++ continue ++ } + } + + destRef, err := is.Transport.ParseStoreReference(opt.Store, destImage) +@@ -173,7 +178,6 @@ func PullAndGetImageInfo(opt *PrepareImageOptions) (types.ImageReference, *stora + pLog.Debugf("Failed to parse store reference for %q: %v", destImage, err) + continue + } +- + img, err := is.Transport.GetStoreImage(opt.Store, destRef) + if err == nil { + // find the unique image in local store by name or digest +@@ -246,14 +250,10 @@ func instantiatingImage(ctx context.Context, sc *types.SystemContext, ref types. + return baseImg, nil + } + +-func getLocalImageNameFromRef(store storage.Store, srcRef types.ImageReference) (string, error) { ++func getLocalImageNameFromRef(srcRef types.ImageReference) (string, error) { + if srcRef == nil { + return "", errors.Errorf("reference to image is empty") + } +- +- if err := exporter.CheckArchiveFormat(srcRef.Transport().Name()); err == nil { +- return stringid.GenerateRandomID() + ":" + stringid.GenerateRandomID(), nil +- } + if srcRef.Transport().Name() != constant.DockerTransport { + return "", errors.Errorf("the %s transport is not supported yet", srcRef.Transport().Name()) + } +-- +2.27.0 + diff --git a/series.conf b/series.conf index 92168bb..a66abfa 100644 --- a/series.conf +++ b/series.conf @@ -50,3 +50,4 @@ patch/0084-test-add-integration-tests-for-saving-and-loading-se.patch patch/0085-util-add-unit-test-for-increment-util-functions.patch patch/0086-bugfix-fix-random-sequence-for-saving-separated-imag.patch patch/0087-bugfix-optimize-function-IsExist.patch +patch/0088-bugfix-loaded-images-cover-existing-images-name-and-.patch