multiarch: use normal Dockerfiles and build command

This removes code duplication:
- the same Go environment is used for multiarch and for testing
- BUILD_PLATFORMS determines what is getting build instead
  of hard-coding that in the make target
- no separate Dockerfile.multiarch

We still need separate Dockerfiles for Linux and Windows because
Windows has special requirements.

Indention is changed to use tabs.

Author: pohly

Dockerfile

@@ -1,6 +1,7 @@
 FROM gcr.io/distroless/static:latest
 LABEL maintainers="Kubernetes Authors"
 LABEL description="CSI Node driver registrar"
+ARG binary=./bin/csi-node-driver-registrar
 
-COPY ./bin/csi-node-driver-registrar csi-node-driver-registrar
+COPY ${binary} csi-node-driver-registrar
 ENTRYPOINT ["/csi-node-driver-registrar"]

Dockerfile.multiarch

@@ -19,38 +19,62 @@ all: build
 
 include release-tools/build.make
 
-# This target builds multiarch images using Moby BuildKit builder toolkit.
-# Docker Buildx is included in Docker 19.03 and needs DOCKER_CLI_EXPERIMENTAL enabled to run corresponding commands.
-# Currently amd, s390x and Windows manifest is pushed for canary, release branch and released tags.
-# Images generated from Prow build are pushed to staging area on gcr
-push-multiarch-%:
-	make BUILD_PLATFORMS="windows amd64 .exe"
+# Additional parameters are needed when pushing to a local registry,
+# see https://github.com/docker/buildx/issues/94.
+# However, that then runs into https://github.com/docker/cli/issues/2396.
+#
+# What works for local testing is:
+# make push-multiarch PULL_BASE_REF=master REGISTRY_NAME=<your account on dockerhub.io> BUILD_PLATFORMS="linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x"
+DOCKER_BUILDX_CREATE_ARGS ?=
+
+# This target builds a multiarch image for one command using Moby BuildKit builder toolkit.
+# Docker Buildx is included in Docker 19.03.
+#
+# ./cmd/<command>/Dockerfile[.Windows] is used if found, otherwise Dockerfile[.Windows].
+# BUILD_PLATFORMS determines which individual images are included in the multiarch image.
+# PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name, and determines
+# the tag for the resulting multiarch image.
+push-multiarch-%: build-%
+	if ! [ "$(PULL_BASE_REF)" ]; then \
+		echo >&2 "ERROR: PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name."; \
+		exit 1; \
+	fi
 	set -ex; \
 	DOCKER_CLI_EXPERIMENTAL=enabled; \
 	export DOCKER_CLI_EXPERIMENTAL; \
-	docker buildx create --use --name multiarchimage-buildertest; \
+	docker buildx create $(DOCKER_BUILDX_CREATE_ARGS) --use --name multiarchimage-buildertest; \
+	trap "docker buildx rm multiarchimage-buildertest" EXIT; \
+	dockerfile_linux=$$(if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi); \
+	dockerfile_windows=$$(if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \
+	build_platforms='$(BUILD_PLATFORMS)'; \
+	if ! [ "$$build_platforms" ]; then build_platforms="linux amd64"; fi; \
 	pushMultiArch () { \
-                tag=$$1; \
-                docker buildx build --push -t $(IMAGE_NAME):amd64-linux-$$tag --platform=linux/amd64 -f $(shell if [ -e ./cmd/$*/Dockerfile.multiarch ]; then echo ./cmd/$*/Dockerfile.multiarch; else echo Dockerfile.multiarch; fi) --label revision=$(REV) .; \
-                docker buildx build --push -t $(IMAGE_NAME):s390x-linux-$$tag --platform=linux/s390x -f $(shell if [ -e ./cmd/$*/Dockerfile.multiarch ]; then echo ./cmd/$*/Dockerfile.multiarch; else echo Dockerfile.multiarch; fi) --label revision=$(REV) .; \
-                docker buildx build --push -t $(IMAGE_NAME):amd64-windows-$$tag --platform=windows -f $(shell if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi) --label revision=$(REV) .; \
-                docker manifest create --amend $(IMAGE_NAME):$$tag $(IMAGE_NAME):amd64-linux-$$tag \
-                        $(IMAGE_NAME):s390x-linux-$$tag \
-                        $(IMAGE_NAME):amd64-windows-$$tag; \
-                docker manifest push -p $(IMAGE_NAME):$$tag; \
+		tag=$$1; \
+		echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do \
+			docker buildx build --push \
+				--tag $(IMAGE_NAME):$$arch-$$os-$$tag \
+				--platform=$$os/$$arch \
+				--file $$(eval echo \$${dockerfile_$$os}) \
+				--build-arg binary=./bin/$*$$suffix \
+				--label revision=$(REV) \
+				.; \
+		done; \
+		images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do echo $(IMAGE_NAME):$$arch-$$os-$$tag; done); \
+		docker manifest create --amend $(IMAGE_NAME):$$tag $$images; \
+		docker manifest push -p $(IMAGE_NAME):$$tag; \
 	}; \
 	if [ $(PULL_BASE_REF) = "master" ]; then \
-                       : "creating or overwriting canary image"; \
-                       pushMultiArch canary; \
+		       : "creating or overwriting canary image"; \
+		       pushMultiArch canary; \
 	elif echo $(PULL_BASE_REF) | grep -q -e 'release-*' ; then \
-                       : "creating or overwriting canary image for release branch"; \
-                        release_canary_tag=$$(echo $(PULL_BASE_REF) | cut -f2 -d '-')-canary; \
-                        pushMultiArch $$release_canary_tag; \
+		       : "creating or overwriting canary image for release branch"; \
+			release_canary_tag=$$(echo $(PULL_BASE_REF) | cut -f2 -d '-')-canary; \
+			pushMultiArch $$release_canary_tag; \
 	elif docker pull $(IMAGE_NAME):$(PULL_BASE_REF) 2>&1 | tee /dev/stderr | grep -q "manifest for $(IMAGE_NAME):$(PULL_BASE_REF) not found"; then \
-                       : "creating release image"; \
-                       pushMultiArch $(PULL_BASE_REF); \
+		       : "creating release image"; \
+		       pushMultiArch $(PULL_BASE_REF); \
 	else \
-                       : "release image $(IMAGE_NAME):$(PULL_BASE_REF) already exists, skipping push"; \
+		       : "release image $(IMAGE_NAME):$(PULL_BASE_REF) already exists, skipping push"; \
 	fi; \
 
 push-multiarch: $(CMDS:%=push-multiarch-%)