From 5fefd5243f7408efe32ae1ded0fe8abe83b07d97 Mon Sep 17 00:00:00 2001
From: Yikun Jiang <yikunkero@gmail.com>
Date: Tue, 12 Sep 2023 14:44:31 +0800
Subject: [PATCH 1/4] Add Apache Spark 3.5.0 dockerfiles

Signed-off-by: Yikun Jiang <yikunkero@gmail.com>
---
 .github/workflows/build_3.5.0.yaml            |  41 ++++++
 .github/workflows/publish.yml                 |   3 +-
 .github/workflows/test.yml                    |   3 +-
 .../Dockerfile                                |  29 ++++
 .../Dockerfile                                |  26 ++++
 3.5.0/scala2.12-java11-r-ubuntu/Dockerfile    |  28 ++++
 3.5.0/scala2.12-java11-ubuntu/Dockerfile      |  79 +++++++++++
 3.5.0/scala2.12-java11-ubuntu/entrypoint.sh   | 126 ++++++++++++++++++
 tools/template.py                             |   4 +-
 versions.json                                 |  42 +++++-
 10 files changed, 371 insertions(+), 10 deletions(-)
 create mode 100644 .github/workflows/build_3.5.0.yaml
 create mode 100644 3.5.0/scala2.12-java11-python3-r-ubuntu/Dockerfile
 create mode 100644 3.5.0/scala2.12-java11-python3-ubuntu/Dockerfile
 create mode 100644 3.5.0/scala2.12-java11-r-ubuntu/Dockerfile
 create mode 100644 3.5.0/scala2.12-java11-ubuntu/Dockerfile
 create mode 100755 3.5.0/scala2.12-java11-ubuntu/entrypoint.sh

diff --git a/.github/workflows/build_3.5.0.yaml b/.github/workflows/build_3.5.0.yaml
new file mode 100644
index 0000000..6eb3ad6
--- /dev/null
+++ b/.github/workflows/build_3.5.0.yaml
@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: "Build and Test (3.5.0)"
+
+on:
+  pull_request:
+    branches:
+      - 'master'
+    paths:
+      - '3.5.0/**'
+
+jobs:
+  run-build:
+    strategy:
+      matrix:
+        image-type: ["all", "python", "scala", "r"]
+    name: Run
+    secrets: inherit
+    uses: ./.github/workflows/main.yml
+    with:
+      spark: 3.5.0
+      scala: 2.12
+      java: 11
+      image-type: ${{ matrix.image-type }}
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index d213ada..8cfa95d 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -25,9 +25,10 @@ on:
       spark:
         description: 'The Spark version of Spark image.'
         required: true
-        default: '3.4.1'
+        default: '3.5.0'
         type: choice
         options:
+        - 3.5.0
         - 3.4.1
         - 3.4.0
         - 3.3.3
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 4f0f741..47dac20 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -25,9 +25,10 @@ on:
       spark:
         description: 'The Spark version of Spark image.'
         required: true
-        default: '3.4.1'
+        default: '3.5.0'
         type: choice
         options:
+        - 3.5.0
         - 3.4.1
         - 3.4.0
         - 3.3.3
diff --git a/3.5.0/scala2.12-java11-python3-r-ubuntu/Dockerfile b/3.5.0/scala2.12-java11-python3-r-ubuntu/Dockerfile
new file mode 100644
index 0000000..d6faaa7
--- /dev/null
+++ b/3.5.0/scala2.12-java11-python3-r-ubuntu/Dockerfile
@@ -0,0 +1,29 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+FROM spark:3.5.0-scala2.12-java11-ubuntu
+
+USER root
+
+RUN set -ex; \
+    apt-get update; \
+    apt-get install -y python3 python3-pip; \
+    apt-get install -y r-base r-base-dev; \
+    rm -rf /var/lib/apt/lists/*
+
+ENV R_HOME /usr/lib/R
+
+USER spark
diff --git a/3.5.0/scala2.12-java11-python3-ubuntu/Dockerfile b/3.5.0/scala2.12-java11-python3-ubuntu/Dockerfile
new file mode 100644
index 0000000..78db218
--- /dev/null
+++ b/3.5.0/scala2.12-java11-python3-ubuntu/Dockerfile
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+FROM spark:3.5.0-scala2.12-java11-ubuntu
+
+USER root
+
+RUN set -ex; \
+    apt-get update; \
+    apt-get install -y python3 python3-pip; \
+    rm -rf /var/lib/apt/lists/*
+
+USER spark
diff --git a/3.5.0/scala2.12-java11-r-ubuntu/Dockerfile b/3.5.0/scala2.12-java11-r-ubuntu/Dockerfile
new file mode 100644
index 0000000..b5fa221
--- /dev/null
+++ b/3.5.0/scala2.12-java11-r-ubuntu/Dockerfile
@@ -0,0 +1,28 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+FROM spark:3.5.0-scala2.12-java11-ubuntu
+
+USER root
+
+RUN set -ex; \
+    apt-get update; \
+    apt-get install -y r-base r-base-dev; \
+    rm -rf /var/lib/apt/lists/*
+
+ENV R_HOME /usr/lib/R
+
+USER spark
diff --git a/3.5.0/scala2.12-java11-ubuntu/Dockerfile b/3.5.0/scala2.12-java11-ubuntu/Dockerfile
new file mode 100644
index 0000000..15f4b31
--- /dev/null
+++ b/3.5.0/scala2.12-java11-ubuntu/Dockerfile
@@ -0,0 +1,79 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+FROM eclipse-temurin:11-jre-focal
+
+ARG spark_uid=185
+
+RUN groupadd --system --gid=${spark_uid} spark && \
+    useradd --system --uid=${spark_uid} --gid=spark spark
+
+RUN set -ex; \
+    apt-get update; \
+    apt-get install -y gnupg2 wget bash tini libc6 libpam-modules krb5-user libnss3 procps net-tools gosu libnss-wrapper; \
+    mkdir -p /opt/spark; \
+    mkdir /opt/spark/python; \
+    mkdir -p /opt/spark/examples; \
+    mkdir -p /opt/spark/work-dir; \
+    chmod g+w /opt/spark/work-dir; \
+    touch /opt/spark/RELEASE; \
+    chown -R spark:spark /opt/spark; \
+    echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su; \
+    rm -rf /var/lib/apt/lists/*
+
+# Install Apache Spark
+# https://downloads.apache.org/spark/KEYS
+ENV SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-3.5.0/spark-3.5.0-bin-hadoop3.tgz \
+    SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-3.5.0/spark-3.5.0-bin-hadoop3.tgz.asc \
+    GPG_KEY=FC3AE3A7EAA1BAC98770840E7E1ABCC53AAA2216
+
+RUN set -ex; \
+    export SPARK_TMP="$(mktemp -d)"; \
+    cd $SPARK_TMP; \
+    wget -nv -O spark.tgz "$SPARK_TGZ_URL"; \
+    wget -nv -O spark.tgz.asc "$SPARK_TGZ_ASC_URL"; \
+    export GNUPGHOME="$(mktemp -d)"; \
+    gpg --batch --keyserver hkps://keys.openpgp.org --recv-key "$GPG_KEY" || \
+    gpg --batch --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY"; \
+    gpg --batch --verify spark.tgz.asc spark.tgz; \
+    gpgconf --kill all; \
+    rm -rf "$GNUPGHOME" spark.tgz.asc; \
+    \
+    tar -xf spark.tgz --strip-components=1; \
+    chown -R spark:spark .; \
+    mv jars /opt/spark/; \
+    mv bin /opt/spark/; \
+    mv sbin /opt/spark/; \
+    mv kubernetes/dockerfiles/spark/decom.sh /opt/; \
+    mv examples /opt/spark/; \
+    mv kubernetes/tests /opt/spark/; \
+    mv data /opt/spark/; \
+    mv python/pyspark /opt/spark/python/pyspark/; \
+    mv python/lib /opt/spark/python/lib/; \
+    mv R /opt/spark/; \
+    chmod a+x /opt/decom.sh; \
+    cd ..; \
+    rm -rf "$SPARK_TMP";
+
+COPY entrypoint.sh /opt/
+
+ENV SPARK_HOME /opt/spark
+
+WORKDIR /opt/spark/work-dir
+
+USER spark
+
+ENTRYPOINT [ "/opt/entrypoint.sh" ]
diff --git a/3.5.0/scala2.12-java11-ubuntu/entrypoint.sh b/3.5.0/scala2.12-java11-ubuntu/entrypoint.sh
new file mode 100755
index 0000000..2e3d2a8
--- /dev/null
+++ b/3.5.0/scala2.12-java11-ubuntu/entrypoint.sh
@@ -0,0 +1,126 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Prevent any errors from being silently ignored
+set -eo pipefail
+
+attempt_setup_fake_passwd_entry() {
+  # Check whether there is a passwd entry for the container UID
+  local myuid; myuid="$(id -u)"
+  # If there is no passwd entry for the container UID, attempt to fake one
+  # You can also refer to the https://github.com/docker-library/official-images/pull/13089#issuecomment-1534706523
+  # It's to resolve OpenShift random UID case.
+  # See also: https://github.com/docker-library/postgres/pull/448
+  if ! getent passwd "$myuid" &> /dev/null; then
+      local wrapper
+      for wrapper in {/usr,}/lib{/*,}/libnss_wrapper.so; do
+        if [ -s "$wrapper" ]; then
+          NSS_WRAPPER_PASSWD="$(mktemp)"
+          NSS_WRAPPER_GROUP="$(mktemp)"
+          export LD_PRELOAD="$wrapper" NSS_WRAPPER_PASSWD NSS_WRAPPER_GROUP
+          local mygid; mygid="$(id -g)"
+          printf 'spark:x:%s:%s:${SPARK_USER_NAME:-anonymous uid}:%s:/bin/false\n' "$myuid" "$mygid" "$SPARK_HOME" > "$NSS_WRAPPER_PASSWD"
+          printf 'spark:x:%s:\n' "$mygid" > "$NSS_WRAPPER_GROUP"
+          break
+        fi
+      done
+  fi
+}
+
+if [ -z "$JAVA_HOME" ]; then
+  JAVA_HOME=$(java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' | awk '{print $3}')
+fi
+
+SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*"
+for v in "${!SPARK_JAVA_OPT_@}"; do
+    SPARK_EXECUTOR_JAVA_OPTS+=( "${!v}" )
+done
+
+if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then
+  SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"
+fi
+
+if ! [ -z "${PYSPARK_PYTHON+x}" ]; then
+    export PYSPARK_PYTHON
+fi
+if ! [ -z "${PYSPARK_DRIVER_PYTHON+x}" ]; then
+    export PYSPARK_DRIVER_PYTHON
+fi
+
+# If HADOOP_HOME is set and SPARK_DIST_CLASSPATH is not set, set it here so Hadoop jars are available to the executor.
+# It does not set SPARK_DIST_CLASSPATH if already set, to avoid overriding customizations of this value from elsewhere e.g. Docker/K8s.
+if [ -n "${HADOOP_HOME}"  ] && [ -z "${SPARK_DIST_CLASSPATH}"  ]; then
+  export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)"
+fi
+
+if ! [ -z "${HADOOP_CONF_DIR+x}" ]; then
+  SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH";
+fi
+
+if ! [ -z "${SPARK_CONF_DIR+x}" ]; then
+  SPARK_CLASSPATH="$SPARK_CONF_DIR:$SPARK_CLASSPATH";
+elif ! [ -z "${SPARK_HOME+x}" ]; then
+  SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";
+fi
+
+# Switch to spark if no USER specified (root by default) otherwise use USER directly
+switch_spark_if_root() {
+  if [ $(id -u) -eq 0 ]; then
+    echo gosu spark
+  fi
+}
+
+case "$1" in
+  driver)
+    shift 1
+    CMD=(
+      "$SPARK_HOME/bin/spark-submit"
+      --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
+      --deploy-mode client
+      "$@"
+    )
+    attempt_setup_fake_passwd_entry
+    # Execute the container CMD under tini for better hygiene
+    exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
+    ;;
+  executor)
+    shift 1
+    CMD=(
+      ${JAVA_HOME}/bin/java
+      "${SPARK_EXECUTOR_JAVA_OPTS[@]}"
+      -Xms"$SPARK_EXECUTOR_MEMORY"
+      -Xmx"$SPARK_EXECUTOR_MEMORY"
+      -cp "$SPARK_CLASSPATH:$SPARK_DIST_CLASSPATH"
+      org.apache.spark.scheduler.cluster.k8s.KubernetesExecutorBackend
+      --driver-url "$SPARK_DRIVER_URL"
+      --executor-id "$SPARK_EXECUTOR_ID"
+      --cores "$SPARK_EXECUTOR_CORES"
+      --app-id "$SPARK_APPLICATION_ID"
+      --hostname "$SPARK_EXECUTOR_POD_IP"
+      --resourceProfileId "$SPARK_RESOURCE_PROFILE_ID"
+      --podName "$SPARK_EXECUTOR_POD_NAME"
+    )
+    attempt_setup_fake_passwd_entry
+    # Execute the container CMD under tini for better hygiene
+    exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
+    ;;
+
+  *)
+    # Non-spark-on-k8s command provided, proceeding in pass-through mode...
+    exec "$@"
+    ;;
+esac
diff --git a/tools/template.py b/tools/template.py
index d305e62..7bd56d2 100755
--- a/tools/template.py
+++ b/tools/template.py
@@ -33,7 +33,9 @@
     # issuer "xinrong@apache.org"
     "3.4.0": "CC68B3D16FE33A766705160BA7E57908C7A4E1B1",
     # issuer "dongjoon@apache.org"
-    "3.4.1": "F28C9C925C188C35E345614DEDA00CE834F0FC5C"
+    "3.4.1": "F28C9C925C188C35E345614DEDA00CE834F0FC5C",
+    # issuer "liyuanjian@apache.org"
+    "3.5.0": "FC3AE3A7EAA1BAC98770840E7E1ABCC53AAA2216"
 }
 
 
diff --git a/versions.json b/versions.json
index 1442bfe..80d35b9 100644
--- a/versions.json
+++ b/versions.json
@@ -1,29 +1,57 @@
 {
   "versions": [
+    {
+      "path": "3.5.0/scala2.12-java11-python3-ubuntu",
+      "tags": [
+        "3.5.0-scala2.12-java11-python3-ubuntu",
+        "3.5.0-python3",
+        "3.5.0",
+        "python3",
+        "latest"
+      ]
+    },
+    {
+      "path": "3.5.0/scala2.12-java11-r-ubuntu",
+      "tags": [
+        "3.5.0-scala2.12-java11-r-ubuntu",
+        "3.5.0-r",
+        "r"
+      ]
+    },
+    {
+      "path": "3.5.0/scala2.12-java11-ubuntu",
+      "tags": [
+        "3.5.0-scala2.12-java11-ubuntu",
+        "3.5.0-scala",
+        "scala"
+      ]
+    },
+    {
+      "path": "3.5.0/scala2.12-java11-python3-r-ubuntu",
+      "tags": [
+        "3.5.0-scala2.12-java11-python3-r-ubuntu"
+      ]
+    },
     {
       "path": "3.4.1/scala2.12-java11-python3-ubuntu",
       "tags": [
         "3.4.1-scala2.12-java11-python3-ubuntu",
         "3.4.1-python3",
-        "3.4.1",
-        "python3",
-        "latest"
+        "3.4.1"
       ]
     },
     {
       "path": "3.4.1/scala2.12-java11-r-ubuntu",
       "tags": [
         "3.4.1-scala2.12-java11-r-ubuntu",
-        "3.4.1-r",
-        "r"
+        "3.4.1-r"
       ]
     },
     {
       "path": "3.4.1/scala2.12-java11-ubuntu",
       "tags": [
         "3.4.1-scala2.12-java11-ubuntu",
-        "3.4.1-scala",
-        "scala"
+        "3.4.1-scala"
       ]
     },
     {

From f451ce2436d354660e12111dbbe6842d9162a1a0 Mon Sep 17 00:00:00 2001
From: Yikun Jiang <yikunkero@gmail.com>
Date: Tue, 12 Sep 2023 14:49:12 +0800
Subject: [PATCH 2/4] RC5

Signed-off-by: Yikun Jiang <yikunkero@gmail.com>
---
 .github/workflows/build_3.3.0.yaml       | 2 --
 .github/workflows/build_3.3.1.yaml       | 2 --
 .github/workflows/build_3.3.2.yaml       | 2 --
 .github/workflows/build_3.3.3.yaml       | 2 --
 .github/workflows/build_3.4.0.yaml       | 2 --
 .github/workflows/main.yml               | 2 +-
 3.5.0/scala2.12-java11-ubuntu/Dockerfile | 4 ++--
 7 files changed, 3 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/build_3.3.0.yaml b/.github/workflows/build_3.3.0.yaml
index a4f8224..95370a6 100644
--- a/.github/workflows/build_3.3.0.yaml
+++ b/.github/workflows/build_3.3.0.yaml
@@ -25,8 +25,6 @@ on:
       - 'master'
     paths:
       - '3.3.0/**'
-      - '.github/workflows/build_3.3.0.yaml'
-      - '.github/workflows/main.yml'
 
 jobs:
   run-build:
diff --git a/.github/workflows/build_3.3.1.yaml b/.github/workflows/build_3.3.1.yaml
index 9e5c082..73094d3 100644
--- a/.github/workflows/build_3.3.1.yaml
+++ b/.github/workflows/build_3.3.1.yaml
@@ -25,8 +25,6 @@ on:
       - 'master'
     paths:
       - '3.3.1/**'
-      - '.github/workflows/build_3.3.1.yaml'
-      - '.github/workflows/main.yml'
 
 jobs:
   run-build:
diff --git a/.github/workflows/build_3.3.2.yaml b/.github/workflows/build_3.3.2.yaml
index 9ae1a13..e7bb57a 100644
--- a/.github/workflows/build_3.3.2.yaml
+++ b/.github/workflows/build_3.3.2.yaml
@@ -25,8 +25,6 @@ on:
       - 'master'
     paths:
       - '3.3.2/**'
-      - '.github/workflows/build_3.3.2.yaml'
-      - '.github/workflows/main.yml'
 
 jobs:
   run-build:
diff --git a/.github/workflows/build_3.3.3.yaml b/.github/workflows/build_3.3.3.yaml
index 75d3948..11b404a 100644
--- a/.github/workflows/build_3.3.3.yaml
+++ b/.github/workflows/build_3.3.3.yaml
@@ -25,8 +25,6 @@ on:
       - 'master'
     paths:
       - '3.3.3/**'
-      - '.github/workflows/build_3.3.3.yaml'
-      - '.github/workflows/main.yml'
 
 jobs:
   run-build:
diff --git a/.github/workflows/build_3.4.0.yaml b/.github/workflows/build_3.4.0.yaml
index 8dd4e1e..a7b99b1 100644
--- a/.github/workflows/build_3.4.0.yaml
+++ b/.github/workflows/build_3.4.0.yaml
@@ -25,8 +25,6 @@ on:
       - 'master'
     paths:
       - '3.4.0/**'
-      - '.github/workflows/build_3.4.0.yaml'
-      - '.github/workflows/main.yml'
 
 jobs:
   run-build:
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index fe755ed..3e27434 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -200,7 +200,7 @@ jobs:
         with:
           fetch-depth: 0
           repository: apache/spark
-          ref: v${{ inputs.spark }}
+          ref: v3.5.0-rc5
           path: ${{ github.workspace }}/spark
 
       - name: Test - Cherry pick commits
diff --git a/3.5.0/scala2.12-java11-ubuntu/Dockerfile b/3.5.0/scala2.12-java11-ubuntu/Dockerfile
index 15f4b31..b40a9b7 100644
--- a/3.5.0/scala2.12-java11-ubuntu/Dockerfile
+++ b/3.5.0/scala2.12-java11-ubuntu/Dockerfile
@@ -36,8 +36,8 @@ RUN set -ex; \
 
 # Install Apache Spark
 # https://downloads.apache.org/spark/KEYS
-ENV SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-3.5.0/spark-3.5.0-bin-hadoop3.tgz \
-    SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-3.5.0/spark-3.5.0-bin-hadoop3.tgz.asc \
+ENV SPARK_TGZ_URL=https://dist.apache.org/repos/dist/dev/spark/v3.5.0-rc5-bin/spark-3.5.0-bin-hadoop3.tgz \
+    SPARK_TGZ_ASC_URL=https://dist.apache.org/repos/dist/dev/spark/v3.5.0-rc5-bin/spark-3.5.0-bin-hadoop3.tgz.asc \
     GPG_KEY=FC3AE3A7EAA1BAC98770840E7E1ABCC53AAA2216
 
 RUN set -ex; \

From 997413e44da5a38fd2539f679b228c019ff04b84 Mon Sep 17 00:00:00 2001
From: Yikun Jiang <yikunkero@gmail.com>
Date: Thu, 14 Sep 2023 14:07:42 +0800
Subject: [PATCH 3/4] Revert "RC5"

This reverts commit f451ce2436d354660e12111dbbe6842d9162a1a0.
---
 .github/workflows/build_3.3.0.yaml       | 2 ++
 .github/workflows/build_3.3.1.yaml       | 2 ++
 .github/workflows/build_3.3.2.yaml       | 2 ++
 .github/workflows/build_3.3.3.yaml       | 2 ++
 .github/workflows/build_3.4.0.yaml       | 2 ++
 .github/workflows/main.yml               | 2 +-
 3.5.0/scala2.12-java11-ubuntu/Dockerfile | 4 ++--
 7 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build_3.3.0.yaml b/.github/workflows/build_3.3.0.yaml
index 95370a6..a4f8224 100644
--- a/.github/workflows/build_3.3.0.yaml
+++ b/.github/workflows/build_3.3.0.yaml
@@ -25,6 +25,8 @@ on:
       - 'master'
     paths:
       - '3.3.0/**'
+      - '.github/workflows/build_3.3.0.yaml'
+      - '.github/workflows/main.yml'
 
 jobs:
   run-build:
diff --git a/.github/workflows/build_3.3.1.yaml b/.github/workflows/build_3.3.1.yaml
index 73094d3..9e5c082 100644
--- a/.github/workflows/build_3.3.1.yaml
+++ b/.github/workflows/build_3.3.1.yaml
@@ -25,6 +25,8 @@ on:
       - 'master'
     paths:
       - '3.3.1/**'
+      - '.github/workflows/build_3.3.1.yaml'
+      - '.github/workflows/main.yml'
 
 jobs:
   run-build:
diff --git a/.github/workflows/build_3.3.2.yaml b/.github/workflows/build_3.3.2.yaml
index e7bb57a..9ae1a13 100644
--- a/.github/workflows/build_3.3.2.yaml
+++ b/.github/workflows/build_3.3.2.yaml
@@ -25,6 +25,8 @@ on:
       - 'master'
     paths:
       - '3.3.2/**'
+      - '.github/workflows/build_3.3.2.yaml'
+      - '.github/workflows/main.yml'
 
 jobs:
   run-build:
diff --git a/.github/workflows/build_3.3.3.yaml b/.github/workflows/build_3.3.3.yaml
index 11b404a..75d3948 100644
--- a/.github/workflows/build_3.3.3.yaml
+++ b/.github/workflows/build_3.3.3.yaml
@@ -25,6 +25,8 @@ on:
       - 'master'
     paths:
       - '3.3.3/**'
+      - '.github/workflows/build_3.3.3.yaml'
+      - '.github/workflows/main.yml'
 
 jobs:
   run-build:
diff --git a/.github/workflows/build_3.4.0.yaml b/.github/workflows/build_3.4.0.yaml
index a7b99b1..8dd4e1e 100644
--- a/.github/workflows/build_3.4.0.yaml
+++ b/.github/workflows/build_3.4.0.yaml
@@ -25,6 +25,8 @@ on:
       - 'master'
     paths:
       - '3.4.0/**'
+      - '.github/workflows/build_3.4.0.yaml'
+      - '.github/workflows/main.yml'
 
 jobs:
   run-build:
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 3e27434..fe755ed 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -200,7 +200,7 @@ jobs:
         with:
           fetch-depth: 0
           repository: apache/spark
-          ref: v3.5.0-rc5
+          ref: v${{ inputs.spark }}
           path: ${{ github.workspace }}/spark
 
       - name: Test - Cherry pick commits
diff --git a/3.5.0/scala2.12-java11-ubuntu/Dockerfile b/3.5.0/scala2.12-java11-ubuntu/Dockerfile
index b40a9b7..15f4b31 100644
--- a/3.5.0/scala2.12-java11-ubuntu/Dockerfile
+++ b/3.5.0/scala2.12-java11-ubuntu/Dockerfile
@@ -36,8 +36,8 @@ RUN set -ex; \
 
 # Install Apache Spark
 # https://downloads.apache.org/spark/KEYS
-ENV SPARK_TGZ_URL=https://dist.apache.org/repos/dist/dev/spark/v3.5.0-rc5-bin/spark-3.5.0-bin-hadoop3.tgz \
-    SPARK_TGZ_ASC_URL=https://dist.apache.org/repos/dist/dev/spark/v3.5.0-rc5-bin/spark-3.5.0-bin-hadoop3.tgz.asc \
+ENV SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-3.5.0/spark-3.5.0-bin-hadoop3.tgz \
+    SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-3.5.0/spark-3.5.0-bin-hadoop3.tgz.asc \
     GPG_KEY=FC3AE3A7EAA1BAC98770840E7E1ABCC53AAA2216
 
 RUN set -ex; \

From 24cbf40abdc252fdcf48303efa33ba7f84adefaf Mon Sep 17 00:00:00 2001
From: Yikun Jiang <yikunkero@gmail.com>
Date: Thu, 14 Sep 2023 19:04:50 +0800
Subject: [PATCH 4/4] Backport https://github.com/apache/spark/pull/40392 and
 https://github.com/apache/spark/pull/41201

Signed-off-by: Yikun Jiang <yikunkero@gmail.com>
---
 3.5.0/scala2.12-java11-ubuntu/entrypoint.sh | 4 ++++
 entrypoint.sh.template                      | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/3.5.0/scala2.12-java11-ubuntu/entrypoint.sh b/3.5.0/scala2.12-java11-ubuntu/entrypoint.sh
index 2e3d2a8..c576d8f 100755
--- a/3.5.0/scala2.12-java11-ubuntu/entrypoint.sh
+++ b/3.5.0/scala2.12-java11-ubuntu/entrypoint.sh
@@ -77,6 +77,9 @@ elif ! [ -z "${SPARK_HOME+x}" ]; then
   SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";
 fi
 
+# SPARK-43540: add current working directory into executor classpath
+SPARK_CLASSPATH="$SPARK_CLASSPATH:$PWD"
+
 # Switch to spark if no USER specified (root by default) otherwise use USER directly
 switch_spark_if_root() {
   if [ $(id -u) -eq 0 ]; then
@@ -90,6 +93,7 @@ case "$1" in
     CMD=(
       "$SPARK_HOME/bin/spark-submit"
       --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
+      --conf "spark.executorEnv.SPARK_DRIVER_POD_IP=$SPARK_DRIVER_BIND_ADDRESS"
       --deploy-mode client
       "$@"
     )
diff --git a/entrypoint.sh.template b/entrypoint.sh.template
index 2e3d2a8..c576d8f 100644
--- a/entrypoint.sh.template
+++ b/entrypoint.sh.template
@@ -77,6 +77,9 @@ elif ! [ -z "${SPARK_HOME+x}" ]; then
   SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";
 fi
 
+# SPARK-43540: add current working directory into executor classpath
+SPARK_CLASSPATH="$SPARK_CLASSPATH:$PWD"
+
 # Switch to spark if no USER specified (root by default) otherwise use USER directly
 switch_spark_if_root() {
   if [ $(id -u) -eq 0 ]; then
@@ -90,6 +93,7 @@ case "$1" in
     CMD=(
       "$SPARK_HOME/bin/spark-submit"
       --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
+      --conf "spark.executorEnv.SPARK_DRIVER_POD_IP=$SPARK_DRIVER_BIND_ADDRESS"
       --deploy-mode client
       "$@"
     )