Updated openshift-tests images to utilize mapped images from external binary

Currently, openshift-tests images cmd uses vendored GetMappedImageConfigs()
function to map upstream images to mirrored ones. After this change, images
cmd will parsed mapped images directly from outputs from the k8s-tests-ext
binary. This prevents unexpected issues from imcompatible upstreams changes
with vendored funcs.

Signed-off-by: Vu Dinh <[email protected]>

Author: dinhxuanvu

pkg/cmd/openshift-tests/images/images_command.go

@@ -9,8 +9,8 @@ import (
 	"time"
 
 	"golang.org/x/exp/slices"
-	k8simage "k8s.io/kubernetes/test/utils/image"
 
+	"github.com/openshift-eng/openshift-tests-extension/pkg/extension"
 	"github.com/openshift/library-go/pkg/image/reference"
 	"github.com/openshift/origin/pkg/clioptions/imagesetup"
 	"github.com/openshift/origin/pkg/cmd"
@@ -19,6 +19,7 @@ import (
 	"github.com/spf13/cobra"
 	"k8s.io/kube-openapi/pkg/util/sets"
 	"k8s.io/kubectl/pkg/util/templates"
+	k8simage "k8s.io/kubernetes/test/utils/image"
 )
 
 func NewImagesCommand() *cobra.Command {
@@ -112,14 +113,13 @@ type imagesOptions struct {
 // TAG is the hash described above.
 func createImageMirrorForInternalImages(prefix string, ref reference.DockerImageReference, mirrored bool) ([]string, error) {
 	source := ref.Exact()
-
+	externalImageSets := []extension.Image{}
 	initialImageSets := []extensions.ImageSet{
 		k8simage.GetOriginalImageConfigs(),
 	}
 
 	// If ENV is not set, the list of images should come from external binaries
 	if len(os.Getenv("OPENSHIFT_SKIP_EXTERNAL_TESTS")) == 0 {
-		// Extract all test binaries
 		extractionContext, extractionContextCancel := context.WithTimeout(context.Background(), 30*time.Minute)
 		defer extractionContextCancel()
 		cleanUpFn, externalBinaries, err := extensions.ExtractAllTestBinaries(extractionContext, 10)
@@ -138,11 +138,31 @@ func createImageMirrorForInternalImages(prefix string, ref reference.DockerImage
 		if len(imageSetsFromBinaries) == 0 {
 			return nil, fmt.Errorf("no test images were reported by external binaries")
 		}
-		initialImageSets = imageSetsFromBinaries
+		externalImageSets = imageSetsFromBinaries
 	}
 
-	// Take the initial images coming from external binaries and remove any exceptions that might exist.
+	// Convert external images to initial and updated image sets
+	// Add mapped images to updated image set if they exist
 	exceptions := image.Exceptions.List()
+	updatedImageSets := []extensions.ImageSet{}
+	initial := extensions.ImageSet{}
+	updated := extensions.ImageSet{}
+	for _, image := range externalImageSets {
+		imageConfig := covertMappedImageToImageConfig(image)
+		if !slices.ContainsFunc(exceptions, func(e string) bool {
+			return strings.Contains(imageConfig.GetE2EImage(), e)
+		}) {
+			initial[k8simage.ImageID(image.Index)] = imageConfig
+			if image.Mapped != nil {
+				updated[k8simage.ImageID(image.Index)] = covertMappedImageToImageConfig(*image.Mapped)
+			}
+		}
+	}
+	if len(initial) > 0 {
+		initialImageSets = []extensions.ImageSet{initial}
+	}
+
+	// Take the initial images coming from external binaries and remove any exceptions that might exist.
 	defaultImageSets := []extensions.ImageSet{}
 	for i := range initialImageSets {
 		filtered := extensions.ImageSet{}
@@ -158,12 +178,31 @@ func createImageMirrorForInternalImages(prefix string, ref reference.DockerImage
 		}
 	}
 
-	// Created a new slice with the updatedImageSets addresses for the images
-	updatedImageSets := []extensions.ImageSet{}
-	for i := range defaultImageSets {
-		updatedImageSets = append(updatedImageSets, k8simage.GetMappedImageConfigs(defaultImageSets[i], ref.Exact()))
+	// Map initial images to the target repository
+	for _, img := range initialImageSets {
+		for imageID, imageConfig := range img {
+			// If the imageID is in the updated image set, skip it
+			if _, ok := updated[imageID]; ok {
+				continue
+			}
+			m := map[string]k8simage.ImageID{
+				imageConfig.GetE2EImage(): k8simage.ImageID(imageID),
+			}
+			mappedImage := image.GetMappedImages(m, source)
+			ref, err := reference.Parse(mappedImage[imageConfig.GetE2EImage()])
+			if err != nil {
+				continue
+			}
+			config := k8simage.Config{}
+			config.SetRegistry(ref.Registry)
+			config.SetName(ref.RepositoryName())
+			config.SetVersion(ref.Tag)
+			updated[k8simage.ImageID(imageID)] = config
+		}
 	}
 
+	updatedImageSets = []extensions.ImageSet{updated}
+
 	openshiftDefaults := image.OriginalImages()
 	openshiftUpdated := image.GetMappedImages(openshiftDefaults, imagesetup.DefaultTestImageMirrorLocation)
 
@@ -178,9 +217,9 @@ func createImageMirrorForInternalImages(prefix string, ref reference.DockerImage
 		covered := sets.NewString()
 		for i := range updatedImageSets {
 			for imageID, imageConfig := range updatedImageSets[i] {
-				defaultConfig := defaultImageSets[i][imageID]
+				originalConfig := defaultImageSets[i][imageID]
 				pullSpec := imageConfig.GetE2EImage()
-				if pullSpec == defaultConfig.GetE2EImage() {
+				if pullSpec == originalConfig.GetE2EImage() {
 					continue
 				}
 				if covered.Has(pullSpec) {
@@ -250,3 +289,12 @@ func createImageMirrorForInternalImages(prefix string, ref reference.DockerImage
 	sort.Strings(lines)
 	return lines, nil
 }
+
+func covertMappedImageToImageConfig(image extension.Image) k8simage.Config {
+	imageConfig := k8simage.Config{}
+	imageConfig.SetName(image.Name)
+	imageConfig.SetVersion(image.Version)
+	imageConfig.SetRegistry(image.Registry)
+
+	return imageConfig
+}

pkg/test/extensions/binary.go

@@ -397,7 +397,7 @@ func (b *TestBinary) RunTests(ctx context.Context, timeout time.Duration, env []
 	return results
 }
 
-func (b *TestBinary) ListImages(ctx context.Context) (ImageSet, error) {
+func (b *TestBinary) ListImages(ctx context.Context) ([]extension.Image, error) {
 	start := time.Now()
 	binName := filepath.Base(b.binaryPath)
 
@@ -408,23 +408,14 @@ func (b *TestBinary) ListImages(ctx context.Context) (ImageSet, error) {
 		return nil, fmt.Errorf("failed running '%s list': %w\nOutput: %s", b.binaryPath, err, output)
 	}
 
-	var images []Image
+	var images []extension.Image
 	err = json.Unmarshal(output, &images)
 	if err != nil {
 		return nil, err
 	}
 
-	result := make(ImageSet, len(images))
-	for _, image := range images {
-		imageConfig := k8simage.Config{}
-		imageConfig.SetName(image.Name)
-		imageConfig.SetVersion(image.Version)
-		imageConfig.SetRegistry(image.Registry)
-		result[k8simage.ImageID(image.Index)] = imageConfig
-	}
-
 	logrus.Infof("Listed %d test images for %q in %v", len(images), binName, time.Since(start))
-	return result, nil
+	return images, nil
 }
 
 // ExtractAllTestBinaries determines the optimal release payload to use, and extracts all the external
@@ -613,9 +604,9 @@ func (binaries TestBinaries) Info(ctx context.Context, parallelism int) ([]*Exte
 	return infos, nil
 }
 
-func (binaries TestBinaries) ListImages(ctx context.Context, parallelism int) ([]ImageSet, error) {
+func (binaries TestBinaries) ListImages(ctx context.Context, parallelism int) ([]extension.Image, error) {
 	var (
-		allImages []ImageSet
+		allImages []extension.Image
 		mu        sync.Mutex
 		wg        sync.WaitGroup
 		errCh     = make(chan error, len(binaries))
@@ -651,12 +642,12 @@ func (binaries TestBinaries) ListImages(ctx context.Context, parallelism int) ([
 						continue // Skip self - only external binaries need to be queried for images
 					}
 
-					imageConfig, err := binary.ListImages(ctx)
+					image, err := binary.ListImages(ctx)
 					if err != nil {
 						errCh <- err
 					}
 					mu.Lock()
-					allImages = append(allImages, imageConfig)
+					allImages = append(allImages, image...)
 					mu.Unlock()
 				}
 			}

test/extended/util/image/zz_generated.txt

@@ -1,5 +1,10 @@
 # This file is generated by hack/update-generated.sh
+/: quay.io/openshift/community-e2e-images:e2e-25-registry-k8s-io-pause-3-10-b3MYAwZ_MelO9baY
 docker.io/library/registry:2.8.0-beta.1 quay.io/openshift/community-e2e-images:e2e-docker-io-library-registry-2-8-0-beta-1-8x_YFKSuz9Xw6lZD
+gcr.io/authenticated-image-pulling/alpine:3.7 quay.io/openshift/community-e2e-images:e2e-5-gcr-io-authenticated-image-pulling-alpine-3-7-k0vnpjJdFU4B_885
+gcr.io/authenticated-image-pulling/windows-nanoserver:v1 quay.io/openshift/community-e2e-images:e2e-6-gcr-io-authenticated-image-pulling-windows-nanoserver-v1-JL_XVPCftjC9psmR
+gcr.io/k8s-authenticated-test/agnhost:2.6 quay.io/openshift/community-e2e-images:e2e-2-gcr-io-k8s-authenticated-test-agnhost-2-6-JK0iEHnrlg2SLZKc
+invalid.registry.k8s.io/invalid/alpine:3.1 quay.io/openshift/community-e2e-images:e2e-12-invalid-registry-k8s-io-invalid-alpine-3-1-0j4KrpB4tVH3QMjb
 quay.io/keycloak/keycloak:25.0 quay.io/openshift/community-e2e-images:e2e-quay-io-keycloak-keycloak-25-0-rEIw9B2Zcc3L1M6k
 quay.io/kubevirt/fedora-with-test-tooling-container-disk:20241024_891122a6fc quay.io/openshift/community-e2e-images:e2e-quay-io-kubevirt-fedora-with-test-tooling-container-disk-20241024_891122a6fc-IycYTh-87XrXse4E
 quay.io/openshifttest/ldap:1.2 quay.io/openshift/community-e2e-images:e2e-quay-io-openshifttest-ldap-1-2-O3f5zPgtmWzEtasv