[SPARK-48664] Add Apache Spark 4.0.0-preview1 Dockerfiles

.github/workflows/build_4.0.0-preview1.yaml

@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: "Build and Test (4.0.0-preview1)"
+
+on:
+  pull_request:
+    branches:
+      - 'master'
+    paths:
+      - '4.0.0-preview1/**'
+
+jobs:
+  run-build:
+    strategy:
+      matrix:
+        image-type: ["all", "python", "scala", "r"]
+        java: [17]
+    name: Run
+    secrets: inherit
+    uses: ./.github/workflows/main.yml
+    with:
+      spark: 4.0.0-preview1
+      scala: 2.13
+      java: ${{ matrix.java }}
+      image-type: ${{ matrix.image-type }}
+

.github/workflows/main.yml

@@ -262,7 +262,7 @@ jobs:
           # See more in "Installation" https://minikube.sigs.k8s.io/docs/start/
           # curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64
           # TODO(SPARK-44495): Resume to use the latest minikube for k8s-integration-tests.
-          curl -LO https://storage.googleapis.com/minikube/releases/v1.30.1/minikube-linux-amd64
+          curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64
           sudo install minikube-linux-amd64 /usr/local/bin/minikube
           rm minikube-linux-amd64
           # Github Action limit cpu:2, memory: 6947MB, limit to 2U6G for better resource statistic

.github/workflows/publish-java17.yaml

@@ -0,0 +1,88 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: "Publish"
+
+on:
+  workflow_dispatch:
+    inputs:
+      spark:
+        description: 'The Spark version of Spark image.'
+        required: true
+        default: '4.0.0-preview1'
+        type: choice
+        options:
+        - 4.0.0-preview1
+      publish:
+        description: 'Publish the image or not.'
+        default: false
+        type: boolean
+        required: true
+      repository:
+        description: The registry to be published (Available only when publish is true).
+        required: false
+        default: ghcr.io/apache/spark-docker
+        type: choice
+        options:
+        # GHCR: This required the write permission of apache/spark-docker (Spark Committer)
+        - ghcr.io/apache/spark-docker
+        # Dockerhub: This required the DOCKERHUB_TOKEN and DOCKERHUB_USER (Spark Committer)
+        - apache
+
+jobs:
+  # We first build and publish the base image
+  run-base-build:
+    strategy:
+      matrix:
+        scala: [2.13]
+        java: [17]
+        image-type: ["scala"]
+    permissions:
+      packages: write
+    name: Run Base
+    secrets: inherit
+    uses: ./.github/workflows/main.yml
+    with:
+      spark: ${{ inputs.spark }}
+      scala: ${{ matrix.scala }}
+      java: ${{ matrix.java }}
+      publish: ${{ inputs.publish }}
+      repository: ${{ inputs.repository }}
+      image-type: ${{ matrix.image-type }}
+
+  # Then publish the all / python / r images
+  run-build:
+    needs: run-base-build
+    strategy:
+      matrix:
+        scala: [2.13]
+        java: [17]
+        image-type: ["all", "python", "r"]
+    permissions:
+      packages: write
+    name: Run
+    secrets: inherit
+    uses: ./.github/workflows/main.yml
+    with:
+      spark: ${{ inputs.spark }}
+      scala: ${{ matrix.scala }}
+      java: ${{ matrix.java }}
+      publish: ${{ inputs.publish }}
+      repository: ${{ inputs.repository }}
+      image-type: ${{ matrix.image-type }}

.github/workflows/test.yml

@@ -28,6 +28,7 @@ on:
         default: '3.5.1'
         type: choice
         options:
+        - 4.0.0-preview1
         - 3.5.1
         - 3.5.0
         - 3.4.2
@@ -50,6 +51,9 @@ on:
         default: 2.12
         type: string
         required: true
+        options:
+        - 2.12
+        - 2.13
       image-type:
         description: 'The image type of Spark image.'
         required: true

4.0.0-preview1/scala2.13-java17-python3-r-ubuntu/Dockerfile

@@ -0,0 +1,29 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+FROM spark:4.0.0-preview1-scala2.13-java17-ubuntu
+
+USER root
+
+RUN set -ex; \
+    apt-get update; \
+    apt-get install -y python3 python3-pip; \
+    apt-get install -y r-base r-base-dev; \
+    rm -rf /var/lib/apt/lists/*
+
+ENV R_HOME /usr/lib/R
+
+USER spark

4.0.0-preview1/scala2.13-java17-python3-ubuntu/Dockerfile

@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+FROM spark:4.0.0-preview1-scala2.13-java17-ubuntu
+
+USER root
+
+RUN set -ex; \
+    apt-get update; \
+    apt-get install -y python3 python3-pip; \
+    rm -rf /var/lib/apt/lists/*
+
+USER spark

4.0.0-preview1/scala2.13-java17-r-ubuntu/Dockerfile

@@ -0,0 +1,28 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+FROM spark:4.0.0-preview1-scala2.13-java17-ubuntu
+
+USER root
+
+RUN set -ex; \
+    apt-get update; \
+    apt-get install -y r-base r-base-dev; \
+    rm -rf /var/lib/apt/lists/*
+
+ENV R_HOME /usr/lib/R
+
+USER spark

4.0.0-preview1/scala2.13-java17-ubuntu/Dockerfile

@@ -0,0 +1,81 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+FROM eclipse-temurin:17-jre-jammy
+
+ARG spark_uid=185
+
+RUN groupadd --system --gid=${spark_uid} spark && \
+    useradd --system --uid=${spark_uid} --gid=spark spark
+
+RUN set -ex; \
+    apt-get update; \
+    apt-get install -y gnupg2 wget bash tini libc6 libpam-modules krb5-user libnss3 procps net-tools gosu libnss-wrapper; \
+    mkdir -p /opt/spark; \
+    mkdir /opt/spark/python; \
+    mkdir -p /opt/spark/examples; \
+    mkdir -p /opt/spark/work-dir; \
+    chmod g+w /opt/spark/work-dir; \
+    touch /opt/spark/RELEASE; \
+    chown -R spark:spark /opt/spark; \
+    echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su; \
+    rm -rf /var/lib/apt/lists/*
+
+# Install Apache Spark
+# https://downloads.apache.org/spark/KEYS
+ENV SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-4.0.0-preview1/spark-4.0.0-preview1-bin-hadoop3.tgz \
+    SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-4.0.0-preview1/spark-4.0.0-preview1-bin-hadoop3.tgz.asc \
+    GPG_KEY=4DC9676CEF9A83E98FCA02784D6620843CD87F5A
+
+RUN set -ex; \
+    export SPARK_TMP="$(mktemp -d)"; \
+    cd $SPARK_TMP; \
+    wget -nv -O spark.tgz "$SPARK_TGZ_URL"; \
+    wget -nv -O spark.tgz.asc "$SPARK_TGZ_ASC_URL"; \
+    export GNUPGHOME="$(mktemp -d)"; \
+    wget -nv -O KEYS https://downloads.apache.org/spark/KEYS; \
+    gpg --import KEYS; \
+    gpg --batch --keyserver hkps://keys.openpgp.org --recv-key "$GPG_KEY" || \
+    gpg --batch --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY"; \
+    gpg --batch --verify spark.tgz.asc spark.tgz; \
+    gpgconf --kill all; \
+    rm -rf "$GNUPGHOME" spark.tgz.asc; \
+    \
+    tar -xf spark.tgz --strip-components=1; \
+    chown -R spark:spark .; \
+    mv jars /opt/spark/; \
+    mv bin /opt/spark/; \
+    mv sbin /opt/spark/; \
+    mv kubernetes/dockerfiles/spark/decom.sh /opt/; \
+    mv examples /opt/spark/; \
+    mv kubernetes/tests /opt/spark/; \
+    mv data /opt/spark/; \
+    mv python/pyspark /opt/spark/python/pyspark/; \
+    mv python/lib /opt/spark/python/lib/; \
+    mv R /opt/spark/; \
+    chmod a+x /opt/decom.sh; \
+    cd ..; \
+    rm -rf "$SPARK_TMP";
+
+COPY entrypoint.sh /opt/
+
+ENV SPARK_HOME /opt/spark
+
+WORKDIR /opt/spark/work-dir
+
+USER spark
+
+ENTRYPOINT [ "/opt/entrypoint.sh" ]