Merge pull request tensorflow#5208 from yifeif/r0.11

Cherry-picks for 0.11.0rc2

Author: gunan

tensorflow/tools/ci_build/builds/test_installation.sh

@@ -47,9 +47,8 @@
 # TF_BUILD_BAZEL_CLEAN, if set to any non-empty and non-0 value, directs the
 # script to perform bazel clean prior to main build and test steps.
 #
-# TF_BUILD_SERIAL_INSTALL_TESTS, if set to any non-empty and non-0 value,
-# will force the Python install tests to run serially, overriding than the
-# concurrent testing behavior.
+# TF_BUILD_USE_GPU, if set to 1, limits the number of concurrent tests to
+# the number stored in TF_GPU_COUNT and assigns each test to a different GPU.
 #
 # TF_GPU_COUNT, Set the number of GPUs in the system. We run only this many
 # concurrent tests when running GPU tests.
@@ -411,21 +410,21 @@ SKIP_COUNTER=0
 FAILED_TESTS=""
 FAILED_TEST_LOGS=""
 
-N_JOBS=$(grep -c ^processor /proc/cpuinfo)
-if [[ -z ${N_JOBS} ]]; then
-  # Try the Mac way of getting number of CPUs
-  N_JOBS=$(sysctl -n hw.ncpu)
-fi
-
-if [[ -z ${N_JOBS} ]]; then
-  N_JOBS=8
-  echo "Cannot determine the number of processors"
-  echo "Using default concurrent job counter ${N_JOBS}"
-fi
-
-if [[ ! -z "${TF_BUILD_SERIAL_INSTALL_TESTS}" ]] &&
-   [[ "${TF_BUILD_SERIAL_INSTALL_TESTS}" != "0" ]]; then
+if [[ "${TF_BUILD_USE_GPU}" == "1" ]]; then
   N_JOBS=$TF_GPU_COUNT
+else
+  N_JOBS=$(grep -c ^processor /proc/cpuinfo)
+  if [[ -z ${N_JOBS} ]]; then
+    # Try the Mac way of getting number of CPUs
+    N_JOBS=$(sysctl -n hw.ncpu)
+  fi
+
+  # If still cannot determine the number of CPUs, pick 8.
+  if [[ -z ${N_JOBS} ]]; then
+    N_JOBS=8
+    echo "Cannot determine the number of processors"
+    echo "Using default concurrent job counter ${N_JOBS}"
+  fi
 fi
 
 echo "Running Python tests-on-install with ${N_JOBS} concurrent jobs..."
@@ -485,9 +484,14 @@ while true; do
     TEST_LOGS="${TEST_LOGS} ${TEST_LOG}"
 
     # Launch test asynchronously
-    "${SCRIPT_DIR}/../gpu_build/parallel_gpu_execute.sh" \
+    if [[ "${TF_BUILD_USE_GPU}" == "1" ]]; then
+      "${SCRIPT_DIR}/../gpu_build/parallel_gpu_execute.sh" \
+        "${SCRIPT_DIR}/py_test_delegate.sh" \
+        "${PYTHON_BIN_PATH}" "${PY_TEST_DIR}/${TEST_BASENAME}" "${TEST_LOG}" &
+    else
       "${SCRIPT_DIR}/py_test_delegate.sh" \
-      "${PYTHON_BIN_PATH}" "${PY_TEST_DIR}/${TEST_BASENAME}" "${TEST_LOG}" &
+        "${PYTHON_BIN_PATH}" "${PY_TEST_DIR}/${TEST_BASENAME}" "${TEST_LOG}" &
+    fi
 
     if [[ "${TEST_COUNTER}" -ge "${N_PAR_TESTS}" ]]; then
       # Run in exclusive mode

tensorflow/tools/ci_build/ci_build.sh

@@ -105,7 +105,7 @@ BUILD_TAG="${BUILD_TAG:-tf_ci}"
 # Add extra params for cuda devices and libraries for GPU container.
 if [ "${CONTAINER_TYPE}" == "gpu" ]; then
   # GPU pip tests-on-install concurrency is limited to the number of GPUs.
-  GPU_EXTRA_PARAMS="${GPU_EXTRA_PARAMS} -e TF_BUILD_SERIAL_INSTALL_TESTS=1"
+  GPU_EXTRA_PARAMS="${GPU_EXTRA_PARAMS} -e TF_BUILD_USE_GPU=1"
 else
   GPU_EXTRA_PARAMS=""
 fi

tensorflow/tools/dist_test/build_server.sh

@@ -16,7 +16,14 @@
 #
 # Builds the test server for distributed (GRPC) TensorFlow
 #
-# Usage: build_server.sh <docker_image_name> [--test]
+# Usage: build_server.sh <docker_image_name> <whl_url> [--test]
+#
+# Arguments:
+#   docker_image_name: Name of the docker image to build.
+#     E.g.: tensorflow/tf_grpc_test_server:0.11.0rc1
+#
+#   whl_url: URL from which the TensorFlow whl file will be downloaded.
+#     E.g.: https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc1-cp27-none-linux_x86_64.whl
 #
 # The optional flag --test lets the script to use the Dockerfile for the
 # testing GRPC server. Without the flag, the script will build the non-test
@@ -33,28 +40,44 @@ die() {
 }
 
 # Check arguments
-if [[ $# != 1 ]] && [[ $# != 2 ]]; then
-  die "Usage: $0 <docker_image_name> [--test]"
+if [[ $# -lt 2 ]]; then
+  die "Usage: $0 <docker_image_name> <whl_url> [--test]"
 fi
 
 DOCKER_IMG_NAME=$1
-shift
+WHL_URL=$2
+shift 2
 
 # Current script directory
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
-DOCKER_FILE="${DIR}/server/Dockerfile"
+BUILD_DIR=$(mktemp -d)
+echo ""
+echo "Using whl file URL: ${WHL_URL}"
+echo "Building in temporary directory: ${BUILD_DIR}"
+
+cp -r ${DIR}/* "${BUILD_DIR}"/ || \
+    die "Failed to copy files to ${BUILD_DIR}"
+
+DOCKER_FILE="${BUILD_DIR}/server/Dockerfile"
 if [[ $1 == "--test" ]]; then
-  DOCKER_FILE="${DIR}/server/Dockerfile.test"
+  DOCKER_FILE="${BUILD_DIR}/server/Dockerfile.test"
 fi
 echo "Using Docker file: ${DOCKER_FILE}"
 
+# Download whl file into the build context directory.
+wget -P "${BUILD_DIR}" ${WHL_URL} || \
+    die "Failed to download tensorflow whl file from URL: ${WHL_URL}"
+
 if [[ ! -f "${DOCKER_FILE}" ]]; then
   die "ERROR: Unable to find dockerfile: ${DOCKER_FILE}"
 fi
 echo "Dockerfile: ${DOCKER_FILE}"
 
 # Call docker build
 docker build --no-cache -t "${DOCKER_IMG_NAME}" \
-   -f "${DOCKER_FILE}" \
-   "${DIR}"
+   -f "${DOCKER_FILE}" "${BUILD_DIR}" || \
+   die "Failed to build docker image: ${DOCKER_IMG_NAME}"
+
+# Clean up docker build context directory.
+rm -rf "${BUILD_DIR}"

tensorflow/tools/dist_test/server/Dockerfile

@@ -34,9 +34,10 @@ RUN curl -O https://bootstrap.pypa.io/get-pip.py && \
     python get-pip.py && \
     rm get-pip.py
 
-# Install TensorFlow CPU version from nightly build
-RUN pip --no-cache-dir install \
-    https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc2-cp27-none-linux_x86_64.whl
+# Install TensorFlow wheel
+COPY tensorflow-*.whl /
+RUN pip install /tensorflow-*.whl && \
+    rm -f /tensorflow-*.whl
 
 # Copy files, including the GRPC server binary at
 # server/grpc_tensorflow_server.py

tensorflow/tools/dist_test/server/Dockerfile.test

@@ -40,9 +40,10 @@ RUN curl -O https://bootstrap.pypa.io/get-pip.py && \
 # Install python panda for the census wide&deep test
 RUN pip install --upgrade pandas==0.18.1
 
-# Install TensorFlow CPU version.
-RUN pip --no-cache-dir install \
-    https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc2-cp27-none-linux_x86_64.whl
+# Install TensorFlow wheel
+COPY tensorflow-*.whl /
+RUN pip install /tensorflow-*.whl && \
+    rm -f /tensorflow-*.whl
 
 # Copy files, including the GRPC server binary at
 # server/grpc_tensorflow_server.py