Merge branches 'cache' and 'py2'

Author: edomora97

.bazelrc

@@ -10,3 +10,4 @@ build --verbose_failures
 # in a container this fails to run tests.
 build --spawn_strategy=standalone --genrule_strategy=standalone
 test --test_strategy=standalone --test_output=streamed
+build --python_path=python3

.circleci/config.yml

@@ -1,5 +1,30 @@
 version: 2
 
+SETUP_UBUNTU: &SETUP_UBUNTU
+    name: Setup system
+    command: |
+      apt update
+      apt install -y openjdk-8-jdk curl python python3 python3-wheel xz-utils
+      echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list
+      curl https://bazel.build/bazel-release.pub.gpg | apt-key add -
+      apt update
+      apt install -y bazel
+      cat > .bazelrc <<EOF
+        # This is from Bazel's former travis setup, to avoid blowing up the RAM usage.
+        startup --host_jvm_args=-Xmx2500m
+        startup --host_jvm_args=-Xms2500m
+        startup --batch
+        # This is so we understand failures better
+        build --verbose_failures
+        # This is so we don't use sandboxed execution. Sandboxed execution
+        # runs stuff in a container, and since GitLabCI already runs its script
+        # in a container this fails to run tests.
+        build --spawn_strategy=standalone --genrule_strategy=standalone
+        test --test_strategy=standalone --test_output=streamed --nocache_test_results
+        build --show_progress_rate_limit 3
+        build --local_resources=4000,2,1.0
+      EOF
+
 jobs:
   build:
     docker:
@@ -9,6 +34,11 @@ jobs:
       - run:
           name: Install dependencies
           command: |
+            cat > /etc/pacman.d/mirrorlist <<EOF
+            Server = https://archlinux.surlyjake.com/archlinux/\$repo/os/\$arch
+            Server = http://mirrors.evowise.com/archlinux/\$repo/os/\$arch
+            Server = http://mirror.rackspace.com/archlinux/\$repo/os/\$arch
+            EOF
             pacman -Syu --noconfirm jdk8-openjdk bazel gcc make python-virtualenv python-wheel
       - run:
           name: Configure bazel
@@ -44,14 +74,14 @@ jobs:
 
   release-ubuntu-14.04:
     docker:
-      - image: ubuntu:14.04.4
+      - image: ubuntu:14.04.5
     steps:
       - checkout
       - run:
           name: Setup system
           command: |
             apt-get update
-            apt-get -y install software-properties-common curl build-essential python3-wheel python-virtualenv python3-dev python3-setuptools zip
+            apt-get -y install software-properties-common curl build-essential python-wheel python-dev python-setuptools zip
             add-apt-repository -y ppa:webupd8team/java
             add-apt-repository -y ppa:ubuntu-toolchain-r/test
             apt-get update
@@ -82,11 +112,53 @@ jobs:
           environment:
             - CC: gcc-4.9
             - CXX: g++-4.9
-          command: ./release.sh
+          command: .circleci/release.sh
       - store_artifacts:
           path: ./release
           destination: ubuntu-14.04
 
+  release-ubuntu-16.04:
+    docker:
+      - image: ubuntu:16.04
+    steps:
+      - checkout
+      - run:
+          <<: *SETUP_UBUNTU
+      - run:
+          name: Build release
+          command: bazel build //release:release-deb -c opt --python_path=python3
+      - store_artifacts:
+          path: ./bazel-bin/release
+          destination: ubuntu-16.04
+
+  release-ubuntu-17.10:
+    docker:
+      - image: ubuntu:17.10
+    steps:
+      - checkout
+      - run:
+          <<: *SETUP_UBUNTU
+      - run:
+          name: Build release
+          command: bazel build //release:release-deb -c opt --python_path=python3
+      - store_artifacts:
+          path: ./bazel-bin/release
+          destination: ubuntu-17.10
+
+  release-ubuntu-18.04:
+    docker:
+      - image: ubuntu:18.04
+    steps:
+      - checkout
+      - run:
+          <<: *SETUP_UBUNTU
+      - run:
+          name: Build release
+          command: bazel build //release:release-deb -c opt --python_path=python3
+      - store_artifacts:
+          path: ./bazel-bin/release
+          destination: ubuntu-18.04
+
   release-archlinux:
       docker:
         - image: base/archlinux
@@ -113,7 +185,7 @@ jobs:
               EOF
         - run:
             name: Build release
-            command: ./release.sh
+            command: .circleci/release.sh
         - store_artifacts:
             path: ./release
             destination: archlinux
@@ -135,11 +207,35 @@ workflows:
               ignore: /.*/
             tags:
               only: /^v.*/
+      - release-ubuntu-16.04:
+          requires:
+            - build
+          filters:
+            branches:
+              ignore: /.*/
+            tags:
+              only: /^v.*/
+      - release-ubuntu-17.10:
+          requires:
+            - build
+          filters:
+            branches:
+              ignore: /.*/
+            tags:
+              only: /^v.*/
+      - release-ubuntu-18.04:
+          requires:
+            - build
+          filters:
+            branches:
+              ignore: /.*/
+            tags:
+              only: /^v.*/
       - release-archlinux:
           requires:
             - build
           filters:
             branches:
               ignore: /.*/
             tags:
-              only: /^v.*/
+              only: /^v.*/

release.sh renamed to .circleci/release.sh

@@ -1,29 +1,26 @@
 #!/bin/bash -e
 set -x
 
-cd $(dirname $0)
+TEMPDIR=/tmp/task_maker
+ROOT=$(echo "$(cd "$(dirname "$0")"; pwd)")/..
+CIRCLE_TAG=${CIRCLE_TAG:-unknown}
+OUT_FILE=$ROOT/release/task-maker-$CIRCLE_TAG.zip
 
-TEMPDIR=/tmp
+cd $ROOT
 
 bazel build -c opt //python:task_maker
-mkdir -p $TEMPDIR/task_maker
-cp -r bazel-bin/python/task_maker.runfiles bazel-bin/python/task_maker -L $TEMPDIR/task_maker
+mkdir -p $TEMPDIR
+cp -r bazel-bin/python/task_maker.runfiles bazel-bin/python/task_maker -L $TEMPDIR
 
-pushd $TEMPDIR/task_maker
+pushd $TEMPDIR
 
 chmod u+w -R .
-for dep in $(cd task_maker.runfiles/; ls -d pypi*)
-do
-  rm -rf task_maker.runfiles/$dep
-  ln -s oii_task_maker/external/$dep task_maker.runfiles
-done
 find . -type f -name '*.pyc' -delete
 rm -rf task_maker.runfiles/oii_task_maker/_solib_k8
 find . -type f -name '*.so' | xargs strip -s || true
 find . -type f -name 'manager' | xargs strip -s || true
 find . -type f -print0 | xargs -0 chmod u-w
 
-popd
+(cd .. && zip -r $OUT_FILE task_maker)
 
-mkdir release
-zip -r release/task-maker-$CIRCLE_TAG.zip $TEMPDIR/task_maker
+popd

.gitignore

@@ -6,4 +6,5 @@ __pycache__/
 .mypy_cache/
 *.pyc
 *pyo
-.idea
+.idea
+release/*.zip

WORKSPACE

@@ -36,15 +36,15 @@ new_git_repository(
 # Rules for Python
 git_repository(
     name = "io_bazel_rules_python",
-    commit = "63cdc8f29b6e6ff517744756cc67cf05577ae724",
-    remote = "https://github.com/bazelbuild/rules_python",
+    commit = "3d703d11769c7e400696f44ca1c3e9cefe7312a7",
+    remote = "https://github.com/joshclimacell/rules_python",
 )
 
-load("@io_bazel_rules_python//python:pip.bzl", "pip_repositories", "pip_import")
+load("@io_bazel_rules_python//python:pip.bzl", "pip_repositories", "pip3_import")
 
 pip_repositories()
 
-pip_import(
+pip3_import(
     name = "python_deps",
     requirements = "//tools:requirements.txt",
 )

core/execution.hpp

@@ -61,6 +61,8 @@ class Execution {
   int32_t StatusCode() const { return response_.status_code(); }
   int32_t Signal() const { return response_.signal(); }
 
+  bool Cached() const { return cached_; }
+
   Execution(const Execution&) = delete;
   Execution& operator=(const Execution&) = delete;
   Execution(Execution&&) = delete;

manager/BUILD

@@ -23,6 +23,7 @@ cc_library(
         "//core",
         "//core:execution",
         "//external:glog",
+        "//proto:manager_cc",
         "//proto:task_cc",
         "//util:which",
         "@com_google_absl//absl/types:optional",