Changes mostly targeted at using the script for 2.1 builds.

- Allow mounting a local JDK (2.1 wants 1.7).
- Tweak profiles and builds to 2.1
- Restore support for Scala 2.10
- set up JAVA_HOME in the docker container

I also expanded "dry run" mode a bit to build one of the binary
packages, and also the documentation. The build documentation is
now also kept in the output directory along with the other
artifacts.

dev/create-release/do-release-docker.sh

@@ -23,11 +23,13 @@ SELF=$(cd $(dirname $0) && pwd)
 
 WORKDIR=
 IMGTAG=latest
-while getopts "d:n:t:" opt; do
+JAVA=
+while getopts "d:j:nt:" opt; do
   case $opt in
     d) WORKDIR="$OPTARG" ;;
     n) DRY_RUN=1 ;;
     t) IMGTAG="$OPTARG" ;;
+    j) JAVA="$OPTARG" ;;
     ?) error "Invalid option: $OPTARG" ;;
   esac
 done
@@ -93,8 +95,15 @@ ASF_PASSWORD=$ASF_PASSWORD
 GPG_PASSPHRASE=$GPG_PASSPHRASE
 EOF
 
+JAVA_VOL=
+if [ -n "$JAVA" ]; then
+  echo "JAVA_HOME=/opt/spark-java" >> $ENVFILE
+  JAVA_VOL="--volume $JAVA:/opt/spark-java"
+fi
+
 echo "Building $RELEASE_TAG; output will be at $WORKDIR/output"
-docker run \
+docker run -ti \
   --env-file "$ENVFILE" \
-  --volume "$WORKDIR":/opt/spark-rm \
+  --volume "$WORKDIR:/opt/spark-rm" \
+  $JAVA_VOL \
   "spark-rm:$IMGTAG"

dev/create-release/do-release.sh

@@ -28,18 +28,24 @@ while getopts "bn" opt; do
   esac
 done
 
-set -e
-
 if [ "$RUNNING_IN_DOCKER" = "1" ]; then
   # Inside docker, need to import the GPG key stored in the current directory.
-  echo $GPG_PASSPHRASE | $GPG --passphrase-fd 0 "$SELF/gpg.key"
+  echo $GPG_PASSPHRASE | $GPG --passphrase-fd 0 --import "$SELF/gpg.key"
+
+  # We may need to adjust the path since JAVA_HOME may be overridden by the driver script.
+  if [ -n "$JAVA_HOME" ]; then
+    export PATH="$JAVA_HOME/bin:$PATH"
+  else
+    # JAVA_HOME for the openjdk package.
+    export JAVA_HOME=/usr
+  fi
 else
   # Outside docker, need to ask for information about the release.
   get_release_info
 fi
 
 if [ $SKIP_TAG = 0 ]; then
-  run_silent "Creating release tag $RELEASE_TAG..." "tag.log" \
+  maybe_run "Creating release tag $RELEASE_TAG..." "tag.log" \
     "$SELF/release-tag.sh"
 else
   echo "Skipping tag creation for $RELEASE_TAG."
@@ -49,5 +55,5 @@ run_silent "Building Spark..." "build.log" \
   "$SELF/release-build.sh" package
 run_silent "Building documentation..." "docs.log" \
   "$SELF/release-build.sh" docs
-run_silent "Publishing release" "publish.log" \
+maybe_run "Publishing release" "publish.log" \
   "$SELF/release-build.sh" publish-release

dev/create-release/release-build.sh

@@ -17,6 +17,9 @@
 # limitations under the License.
 #
 
+SELF=$(cd $(dirname $0) && pwd)
+. "$SELF/release-util.sh"
+
 function exit_with_usage {
   cat << EOF
 usage: release-build.sh <package|docs|publish-snapshot|publish-release>
@@ -89,17 +92,6 @@ BASE_DIR=$(pwd)
 
 MVN="build/mvn --force"
 
-# Hive-specific profiles for some builds
-HIVE_PROFILES="-Phive -Phive-thriftserver"
-# Profiles for publishing snapshots and release to Maven Central
-PUBLISH_PROFILES="-Pmesos -Pyarn -Pkubernetes -Pflume $HIVE_PROFILES -Pspark-ganglia-lgpl -Pkinesis-asl"
-# Profiles for building binary releases
-BASE_RELEASE_PROFILES="-Pmesos -Pyarn -Pkubernetes -Pflume -Psparkr"
-# Scala 2.11 only profiles for some builds
-SCALA_2_11_PROFILES="-Pkafka-0-8"
-# Scala 2.12 only profiles for some builds
-SCALA_2_12_PROFILES="-Pscala-2.12"
-
 rm -rf spark
 git clone https://git-wip-us.apache.org/repos/asf/spark.git
 cd spark
@@ -112,6 +104,28 @@ if [ -z "$SPARK_VERSION" ]; then
     | grep -v INFO | grep -v WARNING | grep -v Download)
 fi
 
+# Depending on the version being built, certain extra profiles need to be activated, and
+# different versions of Scala are supported.
+BASE_PROFILES="-Pmesos -Pyarn"
+PUBLISH_SCALA_2_10=0
+SCALA_2_10_PROFILES="-Pscala-2.10"
+SCALA_2_11_PROFILES=
+SCALA_2_12_PROFILES="-Pscala-2.12"
+
+if [[ $SPARK_VERSION > "2.3" ]]; then
+  BASE_PROFILES="$BASE_PROFILES -Pkubernetes -Pflume"
+  SCALA_2_11_PROFILES="-Pkafka-0-8"
+else
+  PUBLISH_SCALA_2_10=1
+fi
+
+# Hive-specific profiles for some builds
+HIVE_PROFILES="-Phive -Phive-thriftserver"
+# Profiles for publishing snapshots and release to Maven Central
+PUBLISH_PROFILES="$BASE_PROFILES $HIVE_PROFILES -Pspark-ganglia-lgpl -Pkinesis-asl"
+# Profiles for building binary releases
+BASE_RELEASE_PROFILES="$BASE_PROFILES -Psparkr"
+
 # Verify we have the right java version set
 if [ -z "$JAVA_HOME" ]; then
   echo "Please set JAVA_HOME."
@@ -127,7 +141,7 @@ if [[ ! $SPARK_VERSION < "2.2." ]]; then
     exit 1
   fi
 else
-  if [[ $java_version > "1.7." ]]; then
+  if ! [[ $java_version =~ 1\.7\..* ]]; then
     if [ -z "$JAVA_7_HOME" ]; then
       echo "Java version $java_version is higher than required 1.7 for pre-2.2"
       echo "Please set JAVA_HOME correctly."
@@ -174,8 +188,9 @@ if [[ "$1" == "package" ]]; then
     FLAGS=$2
     ZINC_PORT=$3
     BUILD_PACKAGE=$4
-    cp -r spark spark-$SPARK_VERSION-bin-$NAME
 
+    echo "Building binary dist $NAME"
+    cp -r spark spark-$SPARK_VERSION-bin-$NAME
     cd spark-$SPARK_VERSION-bin-$NAME
 
     # TODO There should probably be a flag to make-distribution to allow 2.12 support
@@ -250,25 +265,32 @@ if [[ "$1" == "package" ]]; then
   # We increment the Zinc port each time to avoid OOM's and other craziness if multiple builds
   # share the same Zinc server.
   make_binary_release "hadoop2.6" "-Phadoop-2.6 $HIVE_PROFILES $SCALA_2_11_PROFILES $BASE_RELEASE_PROFILES" "3035" "withr" &
-  make_binary_release "hadoop2.7" "-Phadoop-2.7 $HIVE_PROFILES $SCALA_2_11_PROFILES $BASE_RELEASE_PROFILES" "3036" "withpip" &
-  make_binary_release "without-hadoop" "-Phadoop-provided $SCALA_2_11_PROFILES $BASE_RELEASE_PROFILES" "3038" &
+
+  if ! is_dry_run; then
+    make_binary_release "hadoop2.7" "-Phadoop-2.7 $HIVE_PROFILES $SCALA_2_11_PROFILES $BASE_RELEASE_PROFILES" "3036" "withpip" &
+    make_binary_release "without-hadoop" "-Phadoop-provided $SCALA_2_11_PROFILES $BASE_RELEASE_PROFILES" "3038" &
+  fi
+
   wait
   rm -rf spark-$SPARK_VERSION-bin-*/
 
-  svn co --depth=empty $RELEASE_STAGING_LOCATION svn-spark
-  rm -rf "svn-spark/${DEST_DIR_NAME}-bin"
-  mkdir -p "svn-spark/${DEST_DIR_NAME}-bin"
-
-  echo "Copying release tarballs"
-  cp spark-* "svn-spark/${DEST_DIR_NAME}-bin/"
-  cp pyspark-* "svn-spark/${DEST_DIR_NAME}-bin/"
-  cp SparkR_* "svn-spark/${DEST_DIR_NAME}-bin/"
-  svn add "svn-spark/${DEST_DIR_NAME}-bin"
+  if ! is_dry_run; then
+    svn co --depth=empty $RELEASE_STAGING_LOCATION svn-spark
+    rm -rf "svn-spark/${DEST_DIR_NAME}-bin"
+    mkdir -p "svn-spark/${DEST_DIR_NAME}-bin"
+
+    echo "Copying release tarballs"
+    cp spark-* "svn-spark/${DEST_DIR_NAME}-bin/"
+    cp pyspark-* "svn-spark/${DEST_DIR_NAME}-bin/"
+    cp SparkR_* "svn-spark/${DEST_DIR_NAME}-bin/"
+    svn add "svn-spark/${DEST_DIR_NAME}-bin"
+
+    cd svn-spark
+    svn ci --username $ASF_USERNAME --password "$ASF_PASSWORD" -m"Apache Spark $SPARK_PACKAGE_VERSION"
+    cd ..
+    rm -rf svn-spark
+  fi
 
-  cd svn-spark
-  svn ci --username $ASF_USERNAME --password "$ASF_PASSWORD" -m"Apache Spark $SPARK_PACKAGE_VERSION"
-  cd ..
-  rm -rf svn-spark
   exit 0
 fi
 
@@ -282,18 +304,22 @@ if [[ "$1" == "docs" ]]; then
   cd ..
   cd ..
 
-  svn co --depth=empty $RELEASE_STAGING_LOCATION svn-spark
-  rm -rf "svn-spark/${DEST_DIR_NAME}-docs"
-  mkdir -p "svn-spark/${DEST_DIR_NAME}-docs"
+  if ! is_dry_run; then
+    svn co --depth=empty $RELEASE_STAGING_LOCATION svn-spark
+    rm -rf "svn-spark/${DEST_DIR_NAME}-docs"
+    mkdir -p "svn-spark/${DEST_DIR_NAME}-docs"
 
-  echo "Copying release documentation"
-  cp -R "spark/docs/_site" "svn-spark/${DEST_DIR_NAME}-docs/"
-  svn add "svn-spark/${DEST_DIR_NAME}-docs"
+    echo "Copying release documentation"
+    cp -R "spark/docs/_site" "svn-spark/${DEST_DIR_NAME}-docs/"
+    svn add "svn-spark/${DEST_DIR_NAME}-docs"
 
-  cd svn-spark
-  svn ci --username $ASF_USERNAME --password "$ASF_PASSWORD" -m"Apache Spark $SPARK_PACKAGE_VERSION docs"
-  cd ..
-  rm -rf svn-spark
+    cd svn-spark
+    svn ci --username $ASF_USERNAME --password "$ASF_PASSWORD" -m"Apache Spark $SPARK_PACKAGE_VERSION docs"
+    cd ..
+    rm -rf svn-spark
+  fi
+
+  mv "spark/docs/_site" docs/
   exit 0
 fi
 
@@ -356,6 +382,12 @@ if [[ "$1" == "publish-release" ]]; then
 
   $MVN -DzincPort=$ZINC_PORT -Dmaven.repo.local=$tmp_repo -DskipTests $SCALA_2_11_PROFILES $PUBLISH_PROFILES clean install
 
+  if [[ $PUBLISH_SCALA_2_10 = 1 ]]; then
+    ./dev/change-scala-version.sh 2.10
+    $MVN -DzincPort=$ZINC_PORT -Dmaven.repo.local=$tmp_repo -Dscala-2.10 \
+      -DskipTests $PUBLISH_PROFILES $SCALA_2_10_PROFILES clean install
+  fi
+
   #./dev/change-scala-version.sh 2.12
   #$MVN -DzincPort=$ZINC_PORT -Dmaven.repo.local=$tmp_repo \
   #  -DskipTests $SCALA_2_12_PROFILES §$PUBLISH_PROFILES clean install

dev/create-release/release-util.sh

@@ -47,27 +47,35 @@ function parse_version {
 
 function run_silent {
   local BANNER="$1"
-  shift
-  local LOG_FILE="$1"
-  shift
+  local LOG_FILE="$2"
+  shift 2
 
-  echo "=================================="
+  echo "========================"
   echo "= $BANNER"
   echo "Command: $@"
   echo "Log file: $LOG_FILE"
 
-  if [ $DRY_RUN = 0 ]; then
-    "$@" 1>"$LOG_FILE" 2>&1
-  else
-    echo "$@"
-    touch "$LOG_FILE"
-  fi
+  "$@" 1>"$LOG_FILE" 2>&1
 
   local EC=$?
   if [ $EC != 0 ]; then
     echo "Command FAILED. Check full logs for details."
     tail "$LOG_FILE"
-    return $EC
+    exit $EC
+  fi
+}
+
+function maybe_run {
+  if is_dry_run; then
+    local BANNER="$1"
+    local LOG_FILE="$2"
+    shift 2
+    echo "======= DRY RUN ======="
+    echo "= $BANNER"
+    echo "Command: $@"
+    echo "Log file: $LOG_FILE"
+  else
+    run_silent "$@"
   fi
 }
 
@@ -149,8 +157,15 @@ function get_release_info {
     SKIP_TAG=1
   fi
 
+
   export RELEASE_TAG
-  export GIT_REF="$RELEASE_TAG"
+
+  GIT_REF="$RELEASE_TAG"
+  if is_dry_run; then
+    echo "This is a dry run. Please confirm the ref that will be built for testing."
+    GIT_REF=$(read_config "Ref" "$GIT_REF")
+  fi
+  export GIT_REF
   export SPARK_PACKAGE_VERSION="$RELEASE_TAG"
 
   # Gather some user information.
@@ -183,13 +198,22 @@ EOF
     exit 1
   fi
 
-  if [ -z "$ASF_PASSWORD" ]; then
-    stty -echo && printf "ASF password: " && read ASF_PASSWORD && printf '\n' && stty echo
-    export ASF_PASSWORD
+  if ! is_dry_run; then
+    if [ -z "$ASF_PASSWORD" ]; then
+      stty -echo && printf "ASF password: " && read ASF_PASSWORD && printf '\n' && stty echo
+    fi
+  else
+    ASF_PASSWORD="***INVALID***"
   fi
 
   if [ -z "$GPG_PASSPHRASE" ]; then
     stty -echo && printf "GPG passphrase: " && read GPG_PASSPHRASE && printf '\n' && stty echo
-    export GPG_PASSPHRASE
   fi
+
+  export ASF_PASSWORD
+  export GPG_PASSPHRASE
+}
+
+function is_dry_run {
+  [[ $DRY_RUN = 1 ]]
 }