Backport apache/spark#40392 and apache/spark#41201

Signed-off-by: Yikun Jiang <yikunkero@gmail.com>

Author: Yikun

3.5.0/scala2.12-java11-ubuntu/entrypoint.sh

@@ -77,6 +77,9 @@ elif ! [ -z "${SPARK_HOME+x}" ]; then
   SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";
 fi
 
+# SPARK-43540: add current working directory into executor classpath
+SPARK_CLASSPATH="$SPARK_CLASSPATH:$PWD"
+
 # Switch to spark if no USER specified (root by default) otherwise use USER directly
 switch_spark_if_root() {
   if [ $(id -u) -eq 0 ]; then
@@ -90,6 +93,7 @@ case "$1" in
     CMD=(
       "$SPARK_HOME/bin/spark-submit"
       --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
+      --conf "spark.executorEnv.SPARK_DRIVER_POD_IP=$SPARK_DRIVER_BIND_ADDRESS"
       --deploy-mode client
       "$@"
     )

entrypoint.sh.template

@@ -77,6 +77,9 @@ elif ! [ -z "${SPARK_HOME+x}" ]; then
   SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";
 fi
 
+# SPARK-43540: add current working directory into executor classpath
+SPARK_CLASSPATH="$SPARK_CLASSPATH:$PWD"
+
 # Switch to spark if no USER specified (root by default) otherwise use USER directly
 switch_spark_if_root() {
   if [ $(id -u) -eq 0 ]; then
@@ -90,6 +93,7 @@ case "$1" in
     CMD=(
       "$SPARK_HOME/bin/spark-submit"
       --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
+      --conf "spark.executorEnv.SPARK_DRIVER_POD_IP=$SPARK_DRIVER_BIND_ADDRESS"
       --deploy-mode client
       "$@"
     )