diff --git a/contrib/completions/bash/oc b/contrib/completions/bash/oc index 544f077062..cd4540656b 100644 --- a/contrib/completions/bash/oc +++ b/contrib/completions/bash/oc @@ -5708,6 +5708,8 @@ _oc_adm_release_mirror() local_nonpersistent_flags+=("--from-dir=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--keep-manifest-list") + local_nonpersistent_flags+=("--keep-manifest-list") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry") @@ -5866,6 +5868,8 @@ _oc_adm_release_new() local_nonpersistent_flags+=("--include=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--keep-manifest-list") + local_nonpersistent_flags+=("--keep-manifest-list") flags+=("--mapping-file=") two_word_flags+=("--mapping-file") local_nonpersistent_flags+=("--mapping-file") diff --git a/pkg/cli/admin/release/mirror.go b/pkg/cli/admin/release/mirror.go index e57a9bd6a6..0d701fc5b8 100644 --- a/pkg/cli/admin/release/mirror.go +++ b/pkg/cli/admin/release/mirror.go @@ -158,6 +158,7 @@ func NewMirror(f kcmdutil.Factory, streams genericclioptions.IOStreams) *cobra.C flags.StringVar(&o.ToDir, "to-dir", o.ToDir, "A directory to export images to.") flags.BoolVar(&o.ToMirror, "to-mirror", o.ToMirror, "Output the mirror mappings instead of mirroring.") flags.BoolVar(&o.DryRun, "dry-run", o.DryRun, "Display information about the mirror without actually executing it.") + flags.BoolVar(&o.KeepManifestList, "keep-manifest-list", o.KeepManifestList, "If an image is part of a manifest list, always mirror the list even if only one image is found.") flags.BoolVar(&o.ApplyReleaseImageSignature, "apply-release-image-signature", o.ApplyReleaseImageSignature, "Apply release image signature to connected cluster.") flags.StringVar(&o.ReleaseImageSignatureToDir, "release-image-signature-to-dir", o.ReleaseImageSignatureToDir, "A directory to export release image signature to.") @@ -186,6 +187,8 @@ type MirrorOptions struct { ToMirror bool ToDir string + KeepManifestList bool + ApplyReleaseImageSignature bool ReleaseImageSignatureToDir string Overwrite bool @@ -754,6 +757,7 @@ func (o *MirrorOptions) Run() error { opts.FromFileDir = o.FromDir opts.FileDir = o.ToDir opts.DryRun = o.DryRun + opts.KeepManifestList = o.KeepManifestList opts.ManifestUpdateCallback = func(registry string, manifests map[digest.Digest]digest.Digest) error { lock.Lock() defer lock.Unlock() diff --git a/pkg/cli/admin/release/new.go b/pkg/cli/admin/release/new.go index 0fc52c5f71..dcab34aec0 100644 --- a/pkg/cli/admin/release/new.go +++ b/pkg/cli/admin/release/new.go @@ -134,6 +134,7 @@ func NewRelease(f kcmdutil.Factory, streams genericclioptions.IOStreams) *cobra. flags.StringSliceVar(&o.PreviousVersions, "previous", o.PreviousVersions, "A list of semantic versions that should precede this version in the release manifest.") flags.StringVar(&o.ReleaseMetadata, "metadata", o.ReleaseMetadata, "A JSON object to attach as the metadata for the release manifest.") flags.BoolVar(&o.ForceManifest, "release-manifest", o.ForceManifest, "If true, a release manifest will be created using --name as the semantic version.") + flags.BoolVar(&o.KeepManifestList, "keep-manifest-list", o.KeepManifestList, "If an image is part of a manifest list, always mirror the list even if only one image is found.") // validation flags.BoolVar(&o.AllowMissingImages, "allow-missing-images", o.AllowMissingImages, "Ignore errors when an operator references a release image that is not included.") @@ -188,6 +189,7 @@ type NewOptions struct { ForceManifest bool ReleaseMetadata string PreviousVersions []string + KeepManifestList bool DryRun bool @@ -1042,6 +1044,7 @@ func (o *NewOptions) mirrorImages(is *imageapi.ImageStream) error { opts.SkipRelease = true opts.ParallelOptions = o.ParallelOptions opts.SecurityOptions = o.SecurityOptions + opts.KeepManifestList = o.KeepManifestList if err := opts.Run(); err != nil { return err @@ -1175,6 +1178,7 @@ func (o *NewOptions) write(r io.Reader, is *imageapi.ImageStream, now time.Time) options.SecurityOptions = o.SecurityOptions options.DryRun = o.DryRun options.From = toImageBase + options.KeepManifestList = o.KeepManifestList options.ConfigurationCallback = func(dgst, contentDigest digest.Digest, config *dockerv1client.DockerImageConfig) error { verifier.Verify(dgst, contentDigest) // reset any base image info diff --git a/pkg/cli/image/append/append.go b/pkg/cli/image/append/append.go index 752f1bd585..5f7fbe28ab 100644 --- a/pkg/cli/image/append/append.go +++ b/pkg/cli/image/append/append.go @@ -7,19 +7,19 @@ import ( "fmt" "io" "io/ioutil" - "net/http" "os" "strconv" "time" - units "github.com/docker/go-units" "github.com/spf13/cobra" "k8s.io/klog/v2" "github.com/docker/distribution" + "github.com/docker/distribution/manifest/manifestlist" "github.com/docker/distribution/manifest/schema2" "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/client" + units "github.com/docker/go-units" digest "github.com/opencontainers/go-digest" "k8s.io/cli-runtime/pkg/genericclioptions" @@ -105,6 +105,9 @@ type AppendImageOptions struct { DropHistory bool CreatedAt string + // exposed only to be used by the `oc adm release` + KeepManifestList bool + SecurityOptions imagemanifest.SecurityOptions FilterOptions imagemanifest.FilterOptions ParallelOptions imagemanifest.ParallelOptions @@ -260,26 +263,86 @@ func (o *AppendImageOptions) Run() error { } var ( - base *dockerv1client.DockerImageConfig - baseDigest digest.Digest - baseContentDigest digest.Digest - layers []distribution.Descriptor - fromRepo distribution.Repository + repo distribution.Repository + manifestLocation imagemanifest.ManifestLocation + srcManifest distribution.Manifest ) if from != nil { - repo, err := fromOptions.Repository(ctx, *from) + repo, err = fromOptions.Repository(ctx, *from) if err != nil { return err } - fromRepo = repo - srcManifest, manifestLocation, err := imagemanifest.FirstManifest(ctx, from.Ref, repo, o.FilterOptions.Include) + srcManifest, manifestLocation, err = imagemanifest.FirstManifest(ctx, from.Ref, repo, o.FilterOptions.Include) if err != nil { return fmt.Errorf("unable to read image %s: %v", from, err) } + + if o.KeepManifestList { + return o.appendManifestList(ctx, createdAt, from, to, repo, srcManifest, manifestLocation, toRepo, toManifests) + } + } + + return o.append(ctx, createdAt, from, to, false, repo, srcManifest, manifestLocation, toRepo, toManifests) +} + +func (o *AppendImageOptions) appendManifestList(ctx context.Context, createdAt *time.Time, + from *imagesource.TypedImageReference, to imagesource.TypedImageReference, + repo distribution.Repository, srcManifest distribution.Manifest, manifestLocation imagemanifest.ManifestLocation, + toRepo distribution.Repository, toManifests distribution.ManifestService) error { + // process manifestlist + // oldDigest:newDigest mapping so that we can create a new manifestlist + newDigests := make(map[digest.Digest]digest.Digest) + manifestMap, oldList, _, err := imagemanifest.AllManifests(ctx, from.Ref, repo) + for digest, srcManifest := range manifestMap { + err = o.append(ctx, createdAt, from, to, true, repo, srcManifest, manifestLocation, toRepo, toManifests) + if err != nil { + return fmt.Errorf("error appending image %s: %w", digest, err) + } + newDigests[digest] = o.ToDigest + } + // create new manifestlist from the old one swapping digest with the new ones + newDescriptors := make([]manifestlist.ManifestDescriptor, 0, len(oldList.Manifests)) + for _, manifest := range oldList.Manifests { + manifest.Digest = newDigests[manifest.Digest] + newDescriptors = append(newDescriptors, manifest) + + } + forPush, err := manifestlist.FromDescriptors(newDescriptors) + if err != nil { + return fmt.Errorf("error creating new manifestlist: %#v", err) + } + // push new manifestlist to registry + toDigest, err := imagemanifest.PutManifestInCompatibleSchema(ctx, forPush, to.Ref.Tag, toManifests, toRepo.Named(), nil, nil) + if err != nil { + return fmt.Errorf("unable to push manifestlist: %#v", err) + } + o.ToDigest = toDigest + if !o.DryRun { + fmt.Fprintf(o.Out, "Pushed %s to %s\n", toDigest, to) + } + + return nil +} + +func (o *AppendImageOptions) append(ctx context.Context, createdAt *time.Time, + from *imagesource.TypedImageReference, to imagesource.TypedImageReference, skipTagging bool, + repo distribution.Repository, srcManifest distribution.Manifest, manifestLocation imagemanifest.ManifestLocation, + toRepo distribution.Repository, toManifests distribution.ManifestService) error { + var ( + base *dockerv1client.DockerImageConfig + baseDigest digest.Digest + baseContentDigest digest.Digest + err error + layers []distribution.Descriptor + fromRepo distribution.Repository + ) + if repo != nil || srcManifest != nil { + fromRepo = repo + base, layers, err = imagemanifest.ManifestToImageConfig(ctx, srcManifest, repo.Blobs(ctx), manifestLocation) if err != nil { - return fmt.Errorf("unable to parse image %s: %v", from, err) + return err } contentDigest, err := registryclient.ContentDigestForManifest(srcManifest, manifestLocation.Manifest.Algorithm()) @@ -454,13 +517,21 @@ func (o *AppendImageOptions) Run() error { return fmt.Errorf("unable to upload the new image manifest: %v", err) } klog.V(4).Infof("Created config JSON:\n%s", configJSON) - toDigest, err := imagemanifest.PutManifestInCompatibleSchema(ctx, manifest, to.Ref.Tag, toManifests, toRepo.Named(), fromRepo.Blobs(ctx), configJSON) + tag := to.Ref.Tag + if skipTagging { + tag = "" + } + toDigest, err := imagemanifest.PutManifestInCompatibleSchema(ctx, manifest, tag, toManifests, toRepo.Named(), fromRepo.Blobs(ctx), configJSON) if err != nil { return fmt.Errorf("unable to convert the image to a compatible schema version: %v", err) } o.ToDigest = toDigest if !o.DryRun { - fmt.Fprintf(o.Out, "Pushed %s to %s\n", toDigest, to) + toString := to.String() + if skipTagging { + toString = to.Ref.AsRepository().String() + } + fmt.Fprintf(o.Out, "Pushed %s to %s\n", toDigest, toString) } return nil } @@ -611,141 +682,3 @@ func calculateLayerDigest(blobs distribution.BlobService, dgst digest.Digest, re layerDigest, _, _, _, err := add.DigestCopy(readerFrom, r) return layerDigest, err } - -// scratchRepo can serve the scratch image blob. -type scratchRepo struct{} - -var _ distribution.Repository = scratchRepo{} - -func (_ scratchRepo) Named() reference.Named { panic("not implemented") } -func (_ scratchRepo) Tags(ctx context.Context) distribution.TagService { - panic("not implemented") -} -func (_ scratchRepo) Manifests(ctx context.Context, options ...distribution.ManifestServiceOption) (distribution.ManifestService, error) { - panic("not implemented") -} - -func (r scratchRepo) Blobs(ctx context.Context) distribution.BlobStore { return r } - -func (_ scratchRepo) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) { - if dgst != dockerlayer.GzippedEmptyLayerDigest { - return distribution.Descriptor{}, distribution.ErrBlobUnknown - } - return distribution.Descriptor{ - MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip", - Digest: digest.Digest(dockerlayer.GzippedEmptyLayerDigest), - Size: int64(len(dockerlayer.GzippedEmptyLayer)), - }, nil -} - -func (_ scratchRepo) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) { - if dgst != dockerlayer.GzippedEmptyLayerDigest { - return nil, distribution.ErrBlobUnknown - } - return dockerlayer.GzippedEmptyLayer, nil -} - -type nopCloseBuffer struct { - *bytes.Buffer -} - -func (_ nopCloseBuffer) Seek(offset int64, whence int) (int64, error) { - return 0, nil -} - -func (_ nopCloseBuffer) Close() error { - return nil -} - -func (_ scratchRepo) Open(ctx context.Context, dgst digest.Digest) (distribution.ReadSeekCloser, error) { - if dgst != dockerlayer.GzippedEmptyLayerDigest { - return nil, distribution.ErrBlobUnknown - } - return nopCloseBuffer{bytes.NewBuffer(dockerlayer.GzippedEmptyLayer)}, nil -} - -func (_ scratchRepo) Put(ctx context.Context, mediaType string, p []byte) (distribution.Descriptor, error) { - panic("not implemented") -} - -func (_ scratchRepo) Create(ctx context.Context, options ...distribution.BlobCreateOption) (distribution.BlobWriter, error) { - panic("not implemented") -} - -func (_ scratchRepo) Resume(ctx context.Context, id string) (distribution.BlobWriter, error) { - panic("not implemented") -} - -func (_ scratchRepo) ServeBlob(ctx context.Context, w http.ResponseWriter, r *http.Request, dgst digest.Digest) error { - panic("not implemented") -} - -func (_ scratchRepo) Delete(ctx context.Context, dgst digest.Digest) error { - panic("not implemented") -} - -// dryRunManifestService emulates a remote registry for dry run behavior -type dryRunManifestService struct{} - -func (s *dryRunManifestService) Exists(ctx context.Context, dgst digest.Digest) (bool, error) { - panic("not implemented") -} - -func (s *dryRunManifestService) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) { - panic("not implemented") -} - -func (s *dryRunManifestService) Put(ctx context.Context, manifest distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) { - klog.V(4).Infof("Manifest: %#v", manifest.References()) - return registryclient.ContentDigestForManifest(manifest, digest.SHA256) -} - -func (s *dryRunManifestService) Delete(ctx context.Context, dgst digest.Digest) error { - panic("not implemented") -} - -// dryRunBlobStore emulates a remote registry for dry run behavior -type dryRunBlobStore struct { - layers []distribution.Descriptor -} - -func (s *dryRunBlobStore) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) { - for _, layer := range s.layers { - if layer.Digest == dgst { - return layer, nil - } - } - return distribution.Descriptor{}, distribution.ErrBlobUnknown -} - -func (s *dryRunBlobStore) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) { - panic("not implemented") -} - -func (s *dryRunBlobStore) Open(ctx context.Context, dgst digest.Digest) (distribution.ReadSeekCloser, error) { - panic("not implemented") -} - -func (s *dryRunBlobStore) Put(ctx context.Context, mediaType string, p []byte) (distribution.Descriptor, error) { - return distribution.Descriptor{ - MediaType: mediaType, - Size: int64(len(p)), - Digest: digest.SHA256.FromBytes(p), - }, nil -} - -func (s *dryRunBlobStore) Create(ctx context.Context, options ...distribution.BlobCreateOption) (distribution.BlobWriter, error) { - panic("not implemented") -} - -func (s *dryRunBlobStore) Resume(ctx context.Context, id string) (distribution.BlobWriter, error) { - panic("not implemented") -} - -func (s *dryRunBlobStore) ServeBlob(ctx context.Context, w http.ResponseWriter, r *http.Request, dgst digest.Digest) error { - panic("not implemented") -} - -func (s *dryRunBlobStore) Delete(ctx context.Context, dgst digest.Digest) error { - panic("not implemented") -} diff --git a/pkg/cli/image/append/dryrun.go b/pkg/cli/image/append/dryrun.go new file mode 100644 index 0000000000..0fecf484e5 --- /dev/null +++ b/pkg/cli/image/append/dryrun.go @@ -0,0 +1,79 @@ +package append + +import ( + "context" + "net/http" + + "k8s.io/klog/v2" + + "github.com/docker/distribution" + digest "github.com/opencontainers/go-digest" + + "github.com/openshift/library-go/pkg/image/registryclient" +) + +// dryRunManifestService emulates a remote registry for dry run behavior +type dryRunManifestService struct{} + +func (s *dryRunManifestService) Exists(ctx context.Context, dgst digest.Digest) (bool, error) { + panic("not implemented") +} + +func (s *dryRunManifestService) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) { + panic("not implemented") +} + +func (s *dryRunManifestService) Put(ctx context.Context, manifest distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) { + klog.V(4).Infof("Manifest: %#v", manifest.References()) + return registryclient.ContentDigestForManifest(manifest, digest.SHA256) +} + +func (s *dryRunManifestService) Delete(ctx context.Context, dgst digest.Digest) error { + panic("not implemented") +} + +// dryRunBlobStore emulates a remote registry for dry run behavior +type dryRunBlobStore struct { + layers []distribution.Descriptor +} + +func (s *dryRunBlobStore) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) { + for _, layer := range s.layers { + if layer.Digest == dgst { + return layer, nil + } + } + return distribution.Descriptor{}, distribution.ErrBlobUnknown +} + +func (s *dryRunBlobStore) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) { + panic("not implemented") +} + +func (s *dryRunBlobStore) Open(ctx context.Context, dgst digest.Digest) (distribution.ReadSeekCloser, error) { + panic("not implemented") +} + +func (s *dryRunBlobStore) Put(ctx context.Context, mediaType string, p []byte) (distribution.Descriptor, error) { + return distribution.Descriptor{ + MediaType: mediaType, + Size: int64(len(p)), + Digest: digest.SHA256.FromBytes(p), + }, nil +} + +func (s *dryRunBlobStore) Create(ctx context.Context, options ...distribution.BlobCreateOption) (distribution.BlobWriter, error) { + panic("not implemented") +} + +func (s *dryRunBlobStore) Resume(ctx context.Context, id string) (distribution.BlobWriter, error) { + panic("not implemented") +} + +func (s *dryRunBlobStore) ServeBlob(ctx context.Context, w http.ResponseWriter, r *http.Request, dgst digest.Digest) error { + panic("not implemented") +} + +func (s *dryRunBlobStore) Delete(ctx context.Context, dgst digest.Digest) error { + panic("not implemented") +} diff --git a/pkg/cli/image/append/scratch.go b/pkg/cli/image/append/scratch.go new file mode 100644 index 0000000000..dd8fe31dbf --- /dev/null +++ b/pkg/cli/image/append/scratch.go @@ -0,0 +1,85 @@ +package append + +import ( + "bytes" + "context" + "net/http" + + "github.com/docker/distribution" + "github.com/docker/distribution/reference" + digest "github.com/opencontainers/go-digest" + + "github.com/openshift/oc/pkg/helpers/image/dockerlayer" +) + +// scratchRepo can serve the scratch image blob. +type scratchRepo struct{} + +var _ distribution.Repository = scratchRepo{} + +func (_ scratchRepo) Named() reference.Named { panic("not implemented") } +func (_ scratchRepo) Tags(ctx context.Context) distribution.TagService { + panic("not implemented") +} +func (_ scratchRepo) Manifests(ctx context.Context, options ...distribution.ManifestServiceOption) (distribution.ManifestService, error) { + panic("not implemented") +} + +func (r scratchRepo) Blobs(ctx context.Context) distribution.BlobStore { return r } + +func (_ scratchRepo) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) { + if dgst != dockerlayer.GzippedEmptyLayerDigest { + return distribution.Descriptor{}, distribution.ErrBlobUnknown + } + return distribution.Descriptor{ + MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip", + Digest: digest.Digest(dockerlayer.GzippedEmptyLayerDigest), + Size: int64(len(dockerlayer.GzippedEmptyLayer)), + }, nil +} + +func (_ scratchRepo) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) { + if dgst != dockerlayer.GzippedEmptyLayerDigest { + return nil, distribution.ErrBlobUnknown + } + return dockerlayer.GzippedEmptyLayer, nil +} + +type nopCloseBuffer struct { + *bytes.Buffer +} + +func (_ nopCloseBuffer) Seek(offset int64, whence int) (int64, error) { + return 0, nil +} + +func (_ nopCloseBuffer) Close() error { + return nil +} + +func (_ scratchRepo) Open(ctx context.Context, dgst digest.Digest) (distribution.ReadSeekCloser, error) { + if dgst != dockerlayer.GzippedEmptyLayerDigest { + return nil, distribution.ErrBlobUnknown + } + return nopCloseBuffer{bytes.NewBuffer(dockerlayer.GzippedEmptyLayer)}, nil +} + +func (_ scratchRepo) Put(ctx context.Context, mediaType string, p []byte) (distribution.Descriptor, error) { + panic("not implemented") +} + +func (_ scratchRepo) Create(ctx context.Context, options ...distribution.BlobCreateOption) (distribution.BlobWriter, error) { + panic("not implemented") +} + +func (_ scratchRepo) Resume(ctx context.Context, id string) (distribution.BlobWriter, error) { + panic("not implemented") +} + +func (_ scratchRepo) ServeBlob(ctx context.Context, w http.ResponseWriter, r *http.Request, dgst digest.Digest) error { + panic("not implemented") +} + +func (_ scratchRepo) Delete(ctx context.Context, dgst digest.Digest) error { + panic("not implemented") +} diff --git a/pkg/cli/image/extract/extract.go b/pkg/cli/image/extract/extract.go index 4397ab2d5a..db106bf8c2 100644 --- a/pkg/cli/image/extract/extract.go +++ b/pkg/cli/image/extract/extract.go @@ -234,7 +234,7 @@ func parseMappings(images, paths, files []string, requireEmpty bool) ([]Mapping, case 2: mapping = Mapping{Image: image, From: parts[0], To: parts[1]} default: - return nil, fmt.Errorf("--paths must be of the form SRC:DST") + return nil, fmt.Errorf("--path must be of the form SRC:DST") } if len(mapping.From) > 0 { mapping.From = strings.TrimPrefix(mapping.From, "/")