Add --keep-manifest-list support to oc adm release * commands

contrib/completions/bash/oc

@@ -5708,6 +5708,8 @@ _oc_adm_release_mirror()
     local_nonpersistent_flags+=("--from-dir=")
     flags+=("--insecure")
     local_nonpersistent_flags+=("--insecure")
+    flags+=("--keep-manifest-list")
+    local_nonpersistent_flags+=("--keep-manifest-list")
     flags+=("--max-per-registry=")
     two_word_flags+=("--max-per-registry")
     local_nonpersistent_flags+=("--max-per-registry")
@@ -5866,6 +5868,8 @@ _oc_adm_release_new()
     local_nonpersistent_flags+=("--include=")
     flags+=("--insecure")
     local_nonpersistent_flags+=("--insecure")
+    flags+=("--keep-manifest-list")
+    local_nonpersistent_flags+=("--keep-manifest-list")
     flags+=("--mapping-file=")
     two_word_flags+=("--mapping-file")
     local_nonpersistent_flags+=("--mapping-file")

pkg/cli/admin/release/mirror.go

@@ -158,6 +158,7 @@ func NewMirror(f kcmdutil.Factory, streams genericclioptions.IOStreams) *cobra.C
 	flags.StringVar(&o.ToDir, "to-dir", o.ToDir, "A directory to export images to.")
 	flags.BoolVar(&o.ToMirror, "to-mirror", o.ToMirror, "Output the mirror mappings instead of mirroring.")
 	flags.BoolVar(&o.DryRun, "dry-run", o.DryRun, "Display information about the mirror without actually executing it.")
+	flags.BoolVar(&o.KeepManifestList, "keep-manifest-list", o.KeepManifestList, "If an image is part of a manifest list, always mirror the list even if only one image is found.")
 	flags.BoolVar(&o.ApplyReleaseImageSignature, "apply-release-image-signature", o.ApplyReleaseImageSignature, "Apply release image signature to connected cluster.")
 	flags.StringVar(&o.ReleaseImageSignatureToDir, "release-image-signature-to-dir", o.ReleaseImageSignatureToDir, "A directory to export release image signature to.")
 
@@ -186,6 +187,8 @@ type MirrorOptions struct {
 	ToMirror bool
 	ToDir    string
 
+	KeepManifestList bool
+
 	ApplyReleaseImageSignature bool
 	ReleaseImageSignatureToDir string
 	Overwrite                  bool
@@ -754,6 +757,7 @@ func (o *MirrorOptions) Run() error {
 	opts.FromFileDir = o.FromDir
 	opts.FileDir = o.ToDir
 	opts.DryRun = o.DryRun
+	opts.KeepManifestList = o.KeepManifestList
 	opts.ManifestUpdateCallback = func(registry string, manifests map[digest.Digest]digest.Digest) error {
 		lock.Lock()
 		defer lock.Unlock()

pkg/cli/admin/release/new.go

@@ -134,6 +134,7 @@ func NewRelease(f kcmdutil.Factory, streams genericclioptions.IOStreams) *cobra.
 	flags.StringSliceVar(&o.PreviousVersions, "previous", o.PreviousVersions, "A list of semantic versions that should precede this version in the release manifest.")
 	flags.StringVar(&o.ReleaseMetadata, "metadata", o.ReleaseMetadata, "A JSON object to attach as the metadata for the release manifest.")
 	flags.BoolVar(&o.ForceManifest, "release-manifest", o.ForceManifest, "If true, a release manifest will be created using --name as the semantic version.")
+	flags.BoolVar(&o.KeepManifestList, "keep-manifest-list", o.KeepManifestList, "If an image is part of a manifest list, always mirror the list even if only one image is found.")
 
 	// validation
 	flags.BoolVar(&o.AllowMissingImages, "allow-missing-images", o.AllowMissingImages, "Ignore errors when an operator references a release image that is not included.")
@@ -188,6 +189,7 @@ type NewOptions struct {
 	ForceManifest    bool
 	ReleaseMetadata  string
 	PreviousVersions []string
+	KeepManifestList bool
 
 	DryRun bool
 
@@ -1042,6 +1044,7 @@ func (o *NewOptions) mirrorImages(is *imageapi.ImageStream) error {
 	opts.SkipRelease = true
 	opts.ParallelOptions = o.ParallelOptions
 	opts.SecurityOptions = o.SecurityOptions
+	opts.KeepManifestList = o.KeepManifestList
 
 	if err := opts.Run(); err != nil {
 		return err
@@ -1175,6 +1178,7 @@ func (o *NewOptions) write(r io.Reader, is *imageapi.ImageStream, now time.Time)
 		options.SecurityOptions = o.SecurityOptions
 		options.DryRun = o.DryRun
 		options.From = toImageBase
+		options.KeepManifestList = o.KeepManifestList
 		options.ConfigurationCallback = func(dgst, contentDigest digest.Digest, config *dockerv1client.DockerImageConfig) error {
 			verifier.Verify(dgst, contentDigest)
 			// reset any base image info

pkg/cli/image/append/append.go

@@ -7,19 +7,19 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"net/http"
 	"os"
 	"strconv"
 	"time"
 
-	units "github.com/docker/go-units"
 	"github.com/spf13/cobra"
 	"k8s.io/klog/v2"
 
 	"github.com/docker/distribution"
+	"github.com/docker/distribution/manifest/manifestlist"
 	"github.com/docker/distribution/manifest/schema2"
 	"github.com/docker/distribution/reference"
 	"github.com/docker/distribution/registry/client"
+	units "github.com/docker/go-units"
 	digest "github.com/opencontainers/go-digest"
 
 	"k8s.io/cli-runtime/pkg/genericclioptions"
@@ -105,6 +105,9 @@ type AppendImageOptions struct {
 	DropHistory bool
 	CreatedAt   string
 
+	// exposed only to be used by the `oc adm release`
+	KeepManifestList bool
+
 	SecurityOptions imagemanifest.SecurityOptions
 	FilterOptions   imagemanifest.FilterOptions
 	ParallelOptions imagemanifest.ParallelOptions
@@ -260,26 +263,86 @@ func (o *AppendImageOptions) Run() error {
 	}
 
 	var (
-		base              *dockerv1client.DockerImageConfig
-		baseDigest        digest.Digest
-		baseContentDigest digest.Digest
-		layers            []distribution.Descriptor
-		fromRepo          distribution.Repository
+		repo             distribution.Repository
+		manifestLocation imagemanifest.ManifestLocation
+		srcManifest      distribution.Manifest
 	)
 	if from != nil {
-		repo, err := fromOptions.Repository(ctx, *from)
+		repo, err = fromOptions.Repository(ctx, *from)
 		if err != nil {
 			return err
 		}
-		fromRepo = repo
 
-		srcManifest, manifestLocation, err := imagemanifest.FirstManifest(ctx, from.Ref, repo, o.FilterOptions.Include)
+		srcManifest, manifestLocation, err = imagemanifest.FirstManifest(ctx, from.Ref, repo, o.FilterOptions.Include)
 		if err != nil {
 			return fmt.Errorf("unable to read image %s: %v", from, err)
 		}
+
+		if o.KeepManifestList {
+			return o.appendManifestList(ctx, createdAt, from, to, repo, srcManifest, manifestLocation, toRepo, toManifests)
+		}
+	}
+
+	return o.append(ctx, createdAt, from, to, false, repo, srcManifest, manifestLocation, toRepo, toManifests)
+}
+
+func (o *AppendImageOptions) appendManifestList(ctx context.Context, createdAt *time.Time,
+	from *imagesource.TypedImageReference, to imagesource.TypedImageReference,
+	repo distribution.Repository, srcManifest distribution.Manifest, manifestLocation imagemanifest.ManifestLocation,
+	toRepo distribution.Repository, toManifests distribution.ManifestService) error {
+	// process manifestlist
+	// oldDigest:newDigest mapping so that we can create a new manifestlist
+	newDigests := make(map[digest.Digest]digest.Digest)
+	manifestMap, oldList, _, err := imagemanifest.AllManifests(ctx, from.Ref, repo)
+	for digest, srcManifest := range manifestMap {
+		err = o.append(ctx, createdAt, from, to, true, repo, srcManifest, manifestLocation, toRepo, toManifests)
+		if err != nil {
+			return fmt.Errorf("error appending image %s: %w", digest, err)
+		}
+		newDigests[digest] = o.ToDigest
+	}
+	// create new manifestlist from the old one swapping digest with the new ones
+	newDescriptors := make([]manifestlist.ManifestDescriptor, 0, len(oldList.Manifests))
+	for _, manifest := range oldList.Manifests {
+		manifest.Digest = newDigests[manifest.Digest]
+		newDescriptors = append(newDescriptors, manifest)
+
+	}
+	forPush, err := manifestlist.FromDescriptors(newDescriptors)
+	if err != nil {
+		return fmt.Errorf("error creating new manifestlist: %#v", err)
+	}
+	// push new manifestlist to registry
+	toDigest, err := imagemanifest.PutManifestInCompatibleSchema(ctx, forPush, to.Ref.Tag, toManifests, toRepo.Named(), nil, nil)
+	if err != nil {
+		return fmt.Errorf("unable to push manifestlist: %#v", err)
+	}
+	o.ToDigest = toDigest
+	if !o.DryRun {
+		fmt.Fprintf(o.Out, "Pushed %s to %s\n", toDigest, to)
+	}
+
+	return nil
+}
+
+func (o *AppendImageOptions) append(ctx context.Context, createdAt *time.Time,
+	from *imagesource.TypedImageReference, to imagesource.TypedImageReference, skipTagging bool,
+	repo distribution.Repository, srcManifest distribution.Manifest, manifestLocation imagemanifest.ManifestLocation,
+	toRepo distribution.Repository, toManifests distribution.ManifestService) error {
+	var (
+		base              *dockerv1client.DockerImageConfig
+		baseDigest        digest.Digest
+		baseContentDigest digest.Digest
+		err               error
+		layers            []distribution.Descriptor
+		fromRepo          distribution.Repository
+	)
+	if repo != nil || srcManifest != nil {
+		fromRepo = repo
+
 		base, layers, err = imagemanifest.ManifestToImageConfig(ctx, srcManifest, repo.Blobs(ctx), manifestLocation)
 		if err != nil {
-			return fmt.Errorf("unable to parse image %s: %v", from, err)
+			return err
 		}
 
 		contentDigest, err := registryclient.ContentDigestForManifest(srcManifest, manifestLocation.Manifest.Algorithm())
@@ -454,13 +517,21 @@ func (o *AppendImageOptions) Run() error {
 		return fmt.Errorf("unable to upload the new image manifest: %v", err)
 	}
 	klog.V(4).Infof("Created config JSON:\n%s", configJSON)
-	toDigest, err := imagemanifest.PutManifestInCompatibleSchema(ctx, manifest, to.Ref.Tag, toManifests, toRepo.Named(), fromRepo.Blobs(ctx), configJSON)
+	tag := to.Ref.Tag
+	if skipTagging {
+		tag = ""
+	}
+	toDigest, err := imagemanifest.PutManifestInCompatibleSchema(ctx, manifest, tag, toManifests, toRepo.Named(), fromRepo.Blobs(ctx), configJSON)
 	if err != nil {
 		return fmt.Errorf("unable to convert the image to a compatible schema version: %v", err)
 	}
 	o.ToDigest = toDigest
 	if !o.DryRun {
-		fmt.Fprintf(o.Out, "Pushed %s to %s\n", toDigest, to)
+		toString := to.String()
+		if skipTagging {
+			toString = to.Ref.AsRepository().String()
+		}
+		fmt.Fprintf(o.Out, "Pushed %s to %s\n", toDigest, toString)
 	}
 	return nil
 }
@@ -611,141 +682,3 @@ func calculateLayerDigest(blobs distribution.BlobService, dgst digest.Digest, re
 	layerDigest, _, _, _, err := add.DigestCopy(readerFrom, r)
 	return layerDigest, err
 }
-
-// scratchRepo can serve the scratch image blob.
-type scratchRepo struct{}
-
-var _ distribution.Repository = scratchRepo{}
-
-func (_ scratchRepo) Named() reference.Named { panic("not implemented") }
-func (_ scratchRepo) Tags(ctx context.Context) distribution.TagService {
-	panic("not implemented")
-}
-func (_ scratchRepo) Manifests(ctx context.Context, options ...distribution.ManifestServiceOption) (distribution.ManifestService, error) {
-	panic("not implemented")
-}
-
-func (r scratchRepo) Blobs(ctx context.Context) distribution.BlobStore { return r }
-
-func (_ scratchRepo) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) {
-	if dgst != dockerlayer.GzippedEmptyLayerDigest {
-		return distribution.Descriptor{}, distribution.ErrBlobUnknown
-	}
-	return distribution.Descriptor{
-		MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip",
-		Digest:    digest.Digest(dockerlayer.GzippedEmptyLayerDigest),
-		Size:      int64(len(dockerlayer.GzippedEmptyLayer)),
-	}, nil
-}
-
-func (_ scratchRepo) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) {
-	if dgst != dockerlayer.GzippedEmptyLayerDigest {
-		return nil, distribution.ErrBlobUnknown
-	}
-	return dockerlayer.GzippedEmptyLayer, nil
-}
-
-type nopCloseBuffer struct {
-	*bytes.Buffer
-}
-
-func (_ nopCloseBuffer) Seek(offset int64, whence int) (int64, error) {
-	return 0, nil
-}
-
-func (_ nopCloseBuffer) Close() error {
-	return nil
-}
-
-func (_ scratchRepo) Open(ctx context.Context, dgst digest.Digest) (distribution.ReadSeekCloser, error) {
-	if dgst != dockerlayer.GzippedEmptyLayerDigest {
-		return nil, distribution.ErrBlobUnknown
-	}
-	return nopCloseBuffer{bytes.NewBuffer(dockerlayer.GzippedEmptyLayer)}, nil
-}
-
-func (_ scratchRepo) Put(ctx context.Context, mediaType string, p []byte) (distribution.Descriptor, error) {
-	panic("not implemented")
-}
-
-func (_ scratchRepo) Create(ctx context.Context, options ...distribution.BlobCreateOption) (distribution.BlobWriter, error) {
-	panic("not implemented")
-}
-
-func (_ scratchRepo) Resume(ctx context.Context, id string) (distribution.BlobWriter, error) {
-	panic("not implemented")
-}
-
-func (_ scratchRepo) ServeBlob(ctx context.Context, w http.ResponseWriter, r *http.Request, dgst digest.Digest) error {
-	panic("not implemented")
-}
-
-func (_ scratchRepo) Delete(ctx context.Context, dgst digest.Digest) error {
-	panic("not implemented")
-}
-
-// dryRunManifestService emulates a remote registry for dry run behavior
-type dryRunManifestService struct{}
-
-func (s *dryRunManifestService) Exists(ctx context.Context, dgst digest.Digest) (bool, error) {
-	panic("not implemented")
-}
-
-func (s *dryRunManifestService) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) {
-	panic("not implemented")
-}
-
-func (s *dryRunManifestService) Put(ctx context.Context, manifest distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) {
-	klog.V(4).Infof("Manifest: %#v", manifest.References())
-	return registryclient.ContentDigestForManifest(manifest, digest.SHA256)
-}
-
-func (s *dryRunManifestService) Delete(ctx context.Context, dgst digest.Digest) error {
-	panic("not implemented")
-}
-
-// dryRunBlobStore emulates a remote registry for dry run behavior
-type dryRunBlobStore struct {
-	layers []distribution.Descriptor
-}
-
-func (s *dryRunBlobStore) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) {
-	for _, layer := range s.layers {
-		if layer.Digest == dgst {
-			return layer, nil
-		}
-	}
-	return distribution.Descriptor{}, distribution.ErrBlobUnknown
-}
-
-func (s *dryRunBlobStore) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) {
-	panic("not implemented")
-}
-
-func (s *dryRunBlobStore) Open(ctx context.Context, dgst digest.Digest) (distribution.ReadSeekCloser, error) {
-	panic("not implemented")
-}
-
-func (s *dryRunBlobStore) Put(ctx context.Context, mediaType string, p []byte) (distribution.Descriptor, error) {
-	return distribution.Descriptor{
-		MediaType: mediaType,
-		Size:      int64(len(p)),
-		Digest:    digest.SHA256.FromBytes(p),
-	}, nil
-}
-
-func (s *dryRunBlobStore) Create(ctx context.Context, options ...distribution.BlobCreateOption) (distribution.BlobWriter, error) {
-	panic("not implemented")
-}
-
-func (s *dryRunBlobStore) Resume(ctx context.Context, id string) (distribution.BlobWriter, error) {
-	panic("not implemented")
-}
-
-func (s *dryRunBlobStore) ServeBlob(ctx context.Context, w http.ResponseWriter, r *http.Request, dgst digest.Digest) error {
-	panic("not implemented")
-}
-
-func (s *dryRunBlobStore) Delete(ctx context.Context, dgst digest.Digest) error {
-	panic("not implemented")
-}