More fixes.

- Override SBT_OPTS for Java 7
- Allow executins individual steps in docker
- Fix docker image so that changing what to install also forces
  an "apt update" to happen.

dev/create-release/do-release-docker.sh

@@ -46,18 +46,21 @@ Options are:
   -t [tag]    : tag for the spark-rm docker image to use for building (default: "latest").
   -j [path]   : path to local JDK installation to use for building. By default the script will
                 use openjdk8 installed in the docker image.
+  -s [step]   : runs a single step of the process; valid steps are: tag, build, docs, publish
 EOF
 }
 
 WORKDIR=
 IMGTAG=latest
 JAVA=
-while getopts "d:hj:nt:" opt; do
+RELEASE_STEP=
+while getopts "d:hj:ns:t:" opt; do
   case $opt in
     d) WORKDIR="$OPTARG" ;;
     n) DRY_RUN=1 ;;
     t) IMGTAG="$OPTARG" ;;
     j) JAVA="$OPTARG" ;;
+    s) RELEASE_STEP="$OPTARG" ;;
     h) usage ;;
     ?) error "Invalid option. Run with -h for help." ;;
   esac
@@ -123,6 +126,7 @@ GIT_EMAIL=$GIT_EMAIL
 GPG_KEY=$GPG_KEY
 ASF_PASSWORD=$ASF_PASSWORD
 GPG_PASSPHRASE=$GPG_PASSPHRASE
+RELEASE_STEP=$RELEASE_STEP
 EOF
 
 JAVA_VOL=

dev/create-release/do-release.sh

@@ -44,7 +44,12 @@ else
   get_release_info
 fi
 
-if [ $SKIP_TAG = 0 ]; then
+function should_build {
+  local WHAT=$1
+  [ -n "$RELEASE_STEP" ] && [ "$WHAT" = "$RELEASE_STEP" ]
+}
+
+if should_build "docs" && [ $SKIP_TAG = 0 ]; then
   maybe_run "Creating release tag $RELEASE_TAG..." "tag.log" \
     "$SELF/release-tag.sh"
   echo "It may take some time for the tag to be synchronized to github."
@@ -54,9 +59,23 @@ else
   echo "Skipping tag creation for $RELEASE_TAG."
 fi
 
-run_silent "Building Spark..." "build.log" \
-  "$SELF/release-build.sh" package
-run_silent "Building documentation..." "docs.log" \
-  "$SELF/release-build.sh" docs
-maybe_run "Publishing release" "publish.log" \
-  "$SELF/release-build.sh" publish-release
+if should_build "build"; then
+  run_silent "Building Spark..." "build.log" \
+    "$SELF/release-build.sh" package
+else
+  echo "Skipping build step."
+fi
+
+if should_build "docs"; then
+  run_silent "Building documentation..." "docs.log" \
+    "$SELF/release-build.sh" docs
+else
+  echo "Skipping docs step."
+fi
+
+if should_build "publish"; then
+  maybe_run "Publishing release" "publish.log" \
+    "$SELF/release-build.sh" publish-release
+else
+  echo "Skipping publish step."
+fi

dev/create-release/release-build.sh

@@ -96,6 +96,7 @@ java_version=$("${JAVA_HOME}"/bin/javac -version 2>&1 | cut -d " " -f 2)
 MVN_EXTRA_OPTS=
 if [[ $java_version < "1.8." ]]; then
   # Needed for maven central when using Java 7.
+  export SBT_OPTS="-Dhttps.protocols=TLSv1.1,TLSv1.2"
   MVN_EXTRA_OPTS="-Dhttps.protocols=TLSv1.1,TLSv1.2"
   MVN="$MVN $MVN_EXTRA_OPTS"
 fi

dev/create-release/spark-rm/Dockerfile

@@ -29,9 +29,15 @@ FROM ubuntu:16.04
 # These arguments are just for reuse and not really meant to be customized.
 ARG APT_INSTALL="apt-get install --no-install-recommends -y"
 
+ARG BASE_PIP_PKGS="setuptools wheel virtualenv"
+ARG PIP_PKGS="pyopenssl pypandoc numpy pygments sphinx"
+
 # Install extra needed repos and refresh.
 # - CRAN repo
 # - Ruby repo (for doc generation)
+#
+# This is all in a single "RUN" command so that if anything changes, "apt update" is run to fetch
+# the most current package versions (instead of potentially using old versions cached by docker).
 RUN echo 'deb http://cran.cnr.Berkeley.edu/bin/linux/ubuntu xenial/' >> /etc/apt/sources.list && \
   gpg --keyserver keyserver.ubuntu.com --recv-key E084DAB9 && \
   gpg -a --export E084DAB9 | apt-key add - && \
@@ -41,41 +47,33 @@ RUN echo 'deb http://cran.cnr.Berkeley.edu/bin/linux/ubuntu xenial/' >> /etc/apt
   apt-get update && \
   $APT_INSTALL software-properties-common && \
   apt-add-repository -y ppa:brightbox/ruby-ng && \
-  apt-get update
-
-# Install openjdk 8.
-RUN $APT_INSTALL openjdk-8-jdk && \
-  update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java
-
-# Install build / source control tools
-RUN $APT_INSTALL curl wget git maven ivy subversion make gcc libffi-dev \
+  apt-get update && \
+  # Install openjdk 8.
+  $APT_INSTALL openjdk-8-jdk && \
+  update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java && \
+  # Install build / source control tools
+  $APT_INSTALL curl wget git maven ivy subversion make gcc lsof libffi-dev \
     pandoc pandoc-citeproc libssl-dev libcurl4-openssl-dev libxml2-dev && \
   ln -s -T /usr/share/java/ivy.jar /usr/share/ant/lib/ivy.jar && \
   curl -sL https://deb.nodesource.com/setup_4.x | bash && \
-  $APT_INSTALL nodejs
-
-# Install needed python packages. Use pip for installing packages (for consistency).
-ARG BASE_PIP_PKGS="setuptools wheel virtualenv"
-ARG PIP_PKGS="pyopenssl pypandoc numpy pygments sphinx"
-
-RUN $APT_INSTALL libpython2.7-dev libpython3-dev python-pip python3-pip && \
+  $APT_INSTALL nodejs && \
+  # Install needed python packages. Use pip for installing packages (for consistency).
+  $APT_INSTALL libpython2.7-dev libpython3-dev python-pip python3-pip && \
   pip install $BASE_PIP_PKGS && \
   pip install $PIP_PKGS && \
   cd && \
   virtualenv -p python3 p35 && \
   . p35/bin/activate && \
   pip install $BASE_PIP_PKGS && \
-  pip install $PIP_PKGS
-
-# Install R packages and dependencies used when building.
-# R depends on pandoc*, libssl (which are installed above).
-RUN $APT_INSTALL r-base r-base-dev && \
+  pip install $PIP_PKGS && \
+  # Install R packages and dependencies used when building.
+  # R depends on pandoc*, libssl (which are installed above).
+  $APT_INSTALL r-base r-base-dev && \
   $APT_INSTALL texlive-latex-base texlive texlive-fonts-extra texinfo qpdf && \
   Rscript -e "install.packages(c('curl', 'xml2', 'httr', 'devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2', 'e1071', 'survival'), repos='http://cran.us.r-project.org/')" && \
-  Rscript -e "devtools::install_github('jimhester/lintr')"
-
-# Install tools needed to build the documentation.
-RUN $APT_INSTALL ruby2.3 ruby2.3-dev && \
+  Rscript -e "devtools::install_github('jimhester/lintr')" && \
+  # Install tools needed to build the documentation.
+  $APT_INSTALL ruby2.3 ruby2.3-dev && \
   gem install jekyll --no-rdoc --no-ri && \
   gem install jekyll-redirect-from && \
   gem install pygments.rb