Automatic Example Collator

.dockerignore

@@ -0,0 +1,8 @@
+.git
+**/target/
+**/*.txt
+**/*.md
+/docker/
+
+# dotfiles in the repo root
+/.*

README.md

@@ -20,7 +20,7 @@ A planned Polkadot collator for the parachain.
 
 ## Running a collator
 
-1. Checkout polkadot at `cumulus-branch`.
+1. Checkout polkadot at `96f5dc510ef770fd5c5ab57a90565bb5819bbbea`.
 
 2. Run `Alice` and `Bob`:
 
@@ -56,3 +56,39 @@ A planned Polkadot collator for the parachain.
 	them to the relay chain.
 
 6. Now the `collator` should build blocks and the relay-chain should include them. You can check that the `parachain-header` for parachain `100` is changing.
+
+### Running the collator automatically
+
+To simplify the above process, you can run steps 1-5 above automatically:
+
+```sh
+export BRANCH=96f5dc510ef770fd5c5ab57a90565bb5819bbbea
+scripts/build_polkadot.sh
+scripts/run_collator.sh
+```
+
+This will churn for several minutes, but should end with docker reporting that several containers have successfully been brought up.
+
+To run step 6, first set up an alias which gives you quick access to the polkadot-js CLI:
+
+```sh
+docker build -f docker/parachain-registrar.dockerfile --target pjs -t parachain-registrar:pjs .
+alias pjs='docker run --rm --net cumulus_testing_net parachain-registrar:pjs --ws ws://172.28.1.1:9944'
+```
+
+Those steps should complete very quickly. At that point, you can do things like:
+
+```sh
+$ pjs query.parachains.heads 100
+{
+  "heads": "0xe1efbf8cc2e1304da927986f4cd6964ce0888ce3995948bf71fe427b1a9d39b02101d2dac9c5342d7e8c4f4de2f5277ef860b3a518c1cd823b9a8cee175dce11bf7f57c5016e8a60a6cec16244b2cbf81a67a1dc7a825c288fc694997bc70e2d456400"
+}
+```
+
+The collator includes its own health check, which you can inspect with
+
+```sh
+docker inspect --format='{{json .State.Health}}' cumulus_collator_1
+```
+
+The check runs every 5 minutes, and takes about a minute to complete each time. Most of that time is spent sleeping; it remains a very lightweight process.

docker/docker-compose.yml

@@ -0,0 +1,129 @@
+version: '3.7'
+services:
+  node_alice:
+    image: "polkadot:${BRANCH:-cumulus-branch}"
+    ports:
+      - "30333:30333"
+      - "9933:9933"
+      - "9944:9944"
+    volumes:
+      - "polkadot-data-alice:/data"
+      - type: bind
+        source: ./test/parachain/res/polkadot_chainspec.json
+        target: /chainspec.json
+        read_only: true
+    command: >
+      polkadot
+      --chain=/chainspec.json
+      --base-path=/data
+      --port 30333
+      --rpc-port 9933
+      --ws-port 9944
+      --rpc-external
+      --rpc-cors all
+      --ws-external
+      --alice
+    networks:
+      testing_net:
+        ipv4_address: 172.28.1.1
+        aliases:
+          - alice
+
+  node_bob:
+    image: "polkadot:${BRANCH:-cumulus-branch}"
+    ports:
+      - "30344:30333"
+      - "9935:9933"
+      - "9945:9944"
+    volumes:
+      - "polkadot-data-bob:/data"
+      - type: bind
+        source: ./test/parachain/res/polkadot_chainspec.json
+        target: /chainspec.json
+        read_only: true
+    command: >
+      polkadot
+      --chain=/chainspec.json
+      --base-path=/data
+      --port 30333
+      --rpc-port 9933
+      --ws-port 9944
+      --rpc-external
+      --ws-external
+      --rpc-cors all
+      --bob
+    networks:
+      testing_net:
+        ipv4_address: 172.28.1.2
+        aliases:
+          - bob
+
+  genesis_state:
+    build:
+      context: .
+      dockerfile: ./docker/test-parachain-collator.dockerfile
+    image: "ctpc:latest"
+    volumes:
+      - "genesis-state:/data"
+    command: >
+      cumulus-test-parachain-collator
+      export-genesis-state
+      /data/genesis-state
+
+  collator:
+    build:
+      context: .
+      dockerfile: ./docker/test-parachain-collator.dockerfile
+      target: collator
+    image: "ctpc:collator"
+    volumes:
+      - "collator-data:/data"
+    depends_on:
+      - node_alice
+      - node_bob
+    command: >
+      inject_bootnodes.sh
+      --base-path=/data
+    networks:
+      testing_net:
+
+  runtime:
+    build:
+      context: .
+      dockerfile: ./docker/test-parachain-collator.dockerfile
+      target: runtime
+    image: "ctpc:runtime"
+    volumes:
+      - "parachain-runtime:/runtime"
+
+
+  registrar:
+    build:
+      context: .
+      dockerfile: ./docker/parachain-registrar.dockerfile
+    image: para-reg:latest
+    volumes:
+      - "genesis-state:/genesis"
+      - "parachain-runtime:/runtime"
+    depends_on:
+      - node_alice
+      - runtime
+      - genesis_state
+    networks:
+      testing_net:
+
+
+volumes:
+  polkadot-data-alice:
+  polkadot-data-bob:
+  collator-data:
+  genesis-state:
+  parachain-runtime:
+
+
+networks:
+  testing_net:
+    ipam:
+      driver: default
+      config:
+        - subnet: 172.28.0.0/16

docker/parachain-registrar.dockerfile

@@ -0,0 +1,27 @@
+FROM node:latest AS pjs
+
+# It would be great to depend on a more stable tag, but we need some
+# as-yet-unreleased features.
+RUN yarn global add @polkadot/[email protected]
+
+ENTRYPOINT [ "polkadot-js-api" ]
+CMD [ "--version" ]
+
+# To use the pjs build stage to access the blockchain from the host machine:
+#
+#   docker build -f docker/parachain-registrar.dockerfile --target pjs -t parachain-registrar:pjs .
+#   alias pjs='docker run --rm --net cumulus_testing_net parachain-registrar:pjs --ws ws://172.28.1.1:9944'
+#
+# Then, as long as the chain is running, you can use the polkadot-js-api CLI like:
+#
+#   pjs query.sudo.key
+
+FROM pjs
+RUN apt-get update && apt-get install curl netcat -y && \
+    curl -sSo /wait-for-it.sh https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh && \
+    chmod +x /wait-for-it.sh
+# the only thing left to do is to actually run the transaction.
+COPY ./scripts/register_para.sh /usr/bin
+# unset the previous stage's entrypoint
+ENTRYPOINT []
+CMD [ "/usr/bin/register_para.sh" ]

docker/test-parachain-collator.dockerfile

@@ -0,0 +1,61 @@
+FROM rust:buster as builder
+
+RUN apt-get update && apt-get install time clang libclang-dev llvm -y
+RUN rustup toolchain install nightly
+RUN rustup target add wasm32-unknown-unknown --toolchain nightly
+RUN command -v wasm-gc || cargo +nightly install --git https://github.com/alexcrichton/wasm-gc --force
+
+WORKDIR /paritytech/cumulus
+
+# Ideally, we could just do something like `COPY . .`, but that doesn't work:
+# it busts the cache every time non-source files like inject_bootnodes.sh change,
+# as well as when non-`.dockerignore`'d transient files (*.log and friends)
+# show up. There is no way to exclude particular files, or write a negative
+# rule, using Docker's COPY syntax, which derives from go's filepath.Match rules.
+#
+# We can't combine these into a single big COPY operation like
+# `COPY collator consensus network runtime test Cargo.* .`, because in that case
+# docker will copy the _contents_ of each directory into the image workdir,
+# not the actual directory. We're stuck just enumerating them.
+COPY . .
+
+RUN cargo build --release -p cumulus-test-parachain-collator
+
+# the collator stage is normally built once, cached, and then ignored, but can
+# be specified with the --target build flag. This adds some extra tooling to the
+# image, which is required for a launcher script. The script simply adds two
+# arguments to the list passed in:
+#
+#   --bootnodes /ip4/127.0.0.1/tcp/30333/p2p/PEER_ID
+#
+# with the appropriate ip and ID for both Alice and Bob
+FROM debian:buster-slim as collator
+RUN apt-get update && apt-get install jq curl bash -y && \
+    curl -sSo /wait-for-it.sh https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh && \
+    chmod +x /wait-for-it.sh && \
+    curl -sL https://deb.nodesource.com/setup_12.x | bash - && \
+    apt-get install -y nodejs && \
+    npm install --global yarn && \
+    yarn global add @polkadot/[email protected]
+COPY --from=builder \
+    /paritytech/cumulus/target/release/cumulus-test-parachain-collator /usr/bin
+COPY ./scripts/inject_bootnodes.sh /usr/bin
+CMD ["/usr/bin/inject_bootnodes.sh"]
+COPY ./scripts/healthcheck.sh /usr/bin/
+HEALTHCHECK --interval=300s --timeout=75s --start-period=30s --retries=3 \
+    CMD ["/usr/bin/healthcheck.sh"]
+
+# the runtime stage is normally built once, cached, and ignored, but can be
+# specified with the --target build flag. This just preserves one of the builder's
+# outputs, which can then be moved into a volume at runtime
+FROM debian:buster-slim as runtime
+COPY --from=builder \
+    /paritytech/cumulus/target/release/wbuild/cumulus-test-parachain-runtime/cumulus_test_parachain_runtime.compact.wasm \
+    /var/opt/
+CMD ["cp", "-v", "/var/opt/cumulus_test_parachain_runtime.compact.wasm", "/runtime/"]
+
+FROM debian:buster-slim
+COPY --from=builder \
+    /paritytech/cumulus/target/release/cumulus-test-parachain-collator /usr/bin
+
+CMD ["/usr/bin/cumulus-test-parachain-collator"]

scripts/build_docker.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -e
+
+cd "$(cd "$(dirname "$0")" && git rev-parse --show-toplevel)"
+
+dockerfile="$1"
+if [ -z "$dockerfile" ]; then
+    dockerfile="./docker/test-parachain-collator.dockerfile"
+else
+    shift 1
+fi
+image_name="$(basename "$dockerfile" | rev | cut -d. -f2- | rev)"
+
+echo "building $dockerfile as $image_name..."
+
+time docker build \
+    -f "$dockerfile" \
+    -t "$image_name":latest \
+    "$@" \
+    .

scripts/build_polkadot.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -e
+
+cumulus_repo=$(cd "$(dirname "$0")" && git rev-parse --show-toplevel)
+polkadot_repo=$(dirname "$cumulus_repo")/polkadot
+if [ ! -d "$polkadot_repo/.git" ]; then
+    echo "please clone polkadot in parallel to this repo:"
+    echo "  (cd .. && git clone [email protected]:paritytech/polkadot.git)"
+    exit 1
+fi
+
+if [ -z "$BRANCH" ]; then
+    BRANCH=cumulus-branch
+fi
+
+cd "$polkadot_repo"
+git fetch
+git checkout "$BRANCH"
+time docker build \
+    -f ./docker/Dockerfile \
+    --build-arg PROFILE=release \
+    -t polkadot:"$BRANCH" .

scripts/dc.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+# helper function to run docker-compose using the docker/docker-compose.yml file while
+# retaining a context from the root of the repository
+
+set -e
+
+dc () {
+    cd "$(cd "$(dirname "$0")" && git rev-parse --show-toplevel)"
+    docker-compose -f - "$@" < docker/docker-compose.yml
+}

scripts/healthcheck.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -e
+
+head () {
+    polkadot-js-api --ws ws://172.28.1.1:9944 query.parachains.heads 100 |\
+        jq -r .heads
+}
+
+start=$(head)
+sleep 60
+end=$(head)
+
+[ "$start" != "$end" ]