[SPARK-43365][FOLLWUP] Refactor publish workflow based on base image

.github/workflows/main.yml

@@ -107,6 +107,9 @@ jobs:
             TEST_REPO=${{ inputs.repository }}
             UNIQUE_IMAGE_TAG=${{ inputs.image-tag }}
           fi
+
+          # We can't use the real image for build because we haven't publish the image yet.
+          # The base image for build, it's something like localhost:5000/$REPO_OWNER/spark-docker/spark:3.3.0-scala2.12-java11-ubuntu
           BASE_IMAGE_URL=$TEST_REPO/$IMAGE_NAME:$BASE_IMGAE_TAG
           IMAGE_URL=$TEST_REPO/$IMAGE_NAME:$UNIQUE_IMAGE_TAG
 
@@ -157,19 +160,33 @@ jobs:
           driver-opts: network=host
 
       - name: Build - Build the base image
-        if: ${{ inputs.build }}
+        # Don't need to build the base image when publish
+        if: ${{ inputs.build && !inputs.publish }}
         uses: docker/build-push-action@v3
         with:
           context: ${{ env.BASE_IMAGE_PATH }}
           tags: ${{ env.BASE_IMAGE_URL }}
           platforms: linux/amd64,linux/arm64
           push: true
 
+      - name: Build - Use the test image repo when build
+        # Don't need to build the base image when publish
+        if: ${{ inputs.build && !inputs.publish }}
+        working-directory: ${{ env.IMAGE_PATH }}
+        run: |
+          sed -i "s@FROM spark@FROM $TEST_REPO/spark@g" ./Dockerfile
+
+      - name: Build - Use real image repo when publish
+        # Don't need to build the base image when publish
+        if: ${{ inputs.publish }}
+        working-directory: ${{ env.IMAGE_PATH }}
+        run: |
+          sed -i "s@FROM spark@FROM $PUBLISH_REPO/spark@g" ./Dockerfile
+
       - name: Build - Build and push test image
         if: ${{ inputs.build }}
         uses: docker/build-push-action@v3
         with:
-          build-args: BASE_IMAGE=${{ env.BASE_IMAGE_URL }}
           context: ${{ env.IMAGE_PATH }}
           tags: ${{ env.IMAGE_URL }}
           platforms: linux/amd64,linux/arm64

.github/workflows/publish.yml

@@ -49,13 +49,36 @@ on:
         - apache
 
 jobs:
+  # We first build and publish the base image
+  run-base-build:
+    # if: startsWith(inputs.spark, '3.3')
+    strategy:
+      matrix:
+        scala: [2.12]
+        java: [11]
+        image-type: ["scala"]
+    permissions:
+      packages: write
+    name: Run Base
+    secrets: inherit
+    uses: ./.github/workflows/main.yml
+    with:
+      spark: ${{ inputs.spark }}
+      scala: ${{ matrix.scala }}
+      java: ${{ matrix.java }}
+      publish: ${{ inputs.publish }}
+      repository: ${{ inputs.repository }}
+      image-type: ${{ matrix.image-type }}
+
+  # Then publish the all / python / r images
   run-build:
+    needs: run-base-build
     # if: startsWith(inputs.spark, '3.3')
     strategy:
       matrix:
         scala: [2.12]
         java: [11]
-        image-type: ["all", "python", "scala", "r"]
+        image-type: ["all", "python", "r"]
     permissions:
       packages: write
     name: Run

3.4.0/scala2.12-java11-python3-r-ubuntu/Dockerfile

@@ -14,8 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-ARG BASE_IMAGE=spark:3.4.0-scala2.12-java11-ubuntu
-FROM $BASE_IMAGE
+FROM spark:3.4.0-scala2.12-java11-ubuntu
 
 RUN set -ex && \
     apt-get update && \

3.4.0/scala2.12-java11-python3-ubuntu/Dockerfile

@@ -14,8 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-ARG BASE_IMAGE=spark:3.4.0-scala2.12-java11-ubuntu
-FROM $BASE_IMAGE
+FROM spark:3.4.0-scala2.12-java11-ubuntu
 
 RUN set -ex && \
     apt-get update && \

3.4.0/scala2.12-java11-r-ubuntu/Dockerfile

@@ -14,8 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-ARG BASE_IMAGE=spark:3.4.0-scala2.12-java11-ubuntu
-FROM $BASE_IMAGE
+FROM spark:3.4.0-scala2.12-java11-ubuntu
 
 RUN set -ex && \
     apt-get update && \

r-python.template

@@ -14,8 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-ARG BASE_IMAGE=spark:{{ SPARK_VERSION }}-scala{{ SCALA_VERSION }}-java{{ JAVA_VERSION }}-ubuntu
-FROM $BASE_IMAGE
+FROM spark:{{ SPARK_VERSION }}-scala{{ SCALA_VERSION }}-java{{ JAVA_VERSION }}-ubuntu
 
 RUN set -ex && \
     apt-get update && \