synthetic-network for nightly-pipeline (#804)

* as_ref in aleph-client for Connection types

* rust API for synthetic-network's json API

* refactored synthetic-link

* new e2e-tests: high out-latency

* refactored synthetic-network bash scripts

* added git submodule for synthetic-network

* added nightly e2e test for synthetic-network

* refactored synthetic-link library

* added load_config for synthetic-network e2e test

* more refactoring of synthetic-link

* missing synthetic-link in Cargo.toml

* e2e-tests/config.rs cleaned

* Into -> From in synthetic-link

* slightly cleaned code for synthetic-network

* added RUST_SRC_PATH in shell.nix for better support for IDEs

* slightly cleaned synthetic-link

* refactored synthetic-link: more types for ranged parameters

* refactored synthetic-link lib

* added code-docs to synthetic-link

* using try_from for PortRange in synthetic-lib for input validation

* missing deps after merge for synthetic-network

* REVERT ME testing nightly pipeline

* fix after merge

* say no to install-protoc in github pipelines

* Revert "say no to install-protoc in github pipelines"

This reverts commit b991f75.

* GITHUB_TOKEN for "Install Protoc" action

* reverted changed file permissions

* reverted rust-toolchain file

* typo in description for PortRange

* e2e-tests: config.validator_names() uses validators_count

* e2e-tests: added description for the `latency` tests

* shell.nix: added description for new ENV var RUST_SRC_PATH - it's for rust-analyzer

* BACKUP: e2e-tests/config.rs with iterators

* new verion of the synthetic-network e2e-test that supports non-dockerized runs (urls)

* fix for e2e-test `no_quorum_without_high_latency` - missing call for urls instead of names

* extended README.md for the synthetic-network: how to run e2e-tests with synthetic-network locally

* renamed e2e-tests for high-out-latency with synthetic-network

* renamed one of the e2e-tests pipelines in nightly for high-latency (synthetic-network)

* Revert "REVERT ME testing nightly pipeline"

This reverts commit 8eb1410.

Author: fixxxedpoint

.github/actions/run-e2e-test/action.yml

@@ -30,6 +30,18 @@ inputs:
     description: 'Whether to deploy the adder sample contract to the node.'
     required: false
     default: 'false'
+  image-path:
+    description: 'Custom path to docker image for aleph-node'
+    required: false
+    default: aleph-test-docker
+  node-image:
+    description: 'Custom name of aleph-node image'
+    required: false
+    default: aleph-node:latest
+  compose-file:
+    description: 'Custom docker-compose configuration'
+    required: false
+    default: ''
 
 runs:
   using: 'composite'
@@ -38,15 +50,15 @@ runs:
     - name: Download artifact with docker image
       uses: actions/download-artifact@v2
       with:
-        name: aleph-test-docker
+        name: ${{ inputs.image-path }}
 
     - name: Load node docker image
       shell: bash
       run: docker load -i aleph-node.tar
 
     - name: Run consensus party
       shell: bash
-      run: ./.github/scripts/run_consensus.sh -m ${{ inputs.min-validator-count }} -n ${{ inputs.node-count }}
+      run: NODE_IMAGE=${{ inputs.node-image }} DOCKER_COMPOSE=${{ inputs.compose-file }} ./.github/scripts/run_consensus.sh -m ${{ inputs.min-validator-count }} -n ${{ inputs.node-count }}
 
     - name: Sleep
       shell: bash

.github/scripts/run_consensus.sh

@@ -8,11 +8,12 @@ NODE_COUNT=5
 MIN_VALIDATOR_COUNT=4
 DOCKER_COMPOSE=${DOCKER_COMPOSE:-"docker/docker-compose.yml"}
 OVERRIDE_DOCKER_COMPOSE=${OVERRIDE_DOCKER_COMPOSE:-""}
+NODE_IMAGE=${NODE_IMAGE:-aleph-node:latest}
 
 # default minimum validator count
 MIN_VALIDATOR_COUNT=4
 
-export NODE_IMAGE=aleph-node:latest
+export NODE_IMAGE
 
 mkdir -p docker/data/
 

.github/scripts/run_e2e_test.sh

@@ -116,6 +116,10 @@ if [[ -n "${ADDER:-}" ]]; then
     ARGS+=(-e "ADDER_METADATA=/contracts/adder/target/ink/metadata.json")
 fi
 
+if [[ -n "${OUT_LATENCY:-}" ]]; then
+    ARGS+=(-e OUT_LATENCY)
+fi
+
 docker run -v "$(pwd)/contracts:/contracts" -v "$(pwd)/docker/data:/data" "${ARGS[@]}" aleph-e2e-client:latest
 
 exit $?

.github/workflows/build-and-push-cliain.yaml

@@ -21,6 +21,7 @@ jobs:
         uses: arduino/setup-protoc@v1
         with:
           version: '3.6.1'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Cargo | Build release binary
         run: |
@@ -29,18 +30,18 @@ jobs:
       - name: GIT | Get branch name and commit SHA
         id: get_branch
         uses: ./.github/actions/get-branch
-      
+
       - name: Login to ECR
         uses: docker/login-action@v1
         with:
           registry: public.ecr.aws
           username: ${{ secrets.AWS_MAINNET_ACCESS_KEY_ID }}
           password: ${{ secrets.AWS_MAINNET_SECRET_ACCESS_KEY }}
-      
+
       - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@master
-      
+
       - name: Build and push latest docker image
         id: build-image
         env:

.github/workflows/build-node-and-runtime.yml

@@ -37,6 +37,7 @@ jobs:
         uses: arduino/setup-protoc@v1
         with:
           version: '3.6.1'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Install WASM target
         run: rustup target add wasm32-unknown-unknown

.github/workflows/build-send-postsync-hook-runtime-image.yml

@@ -36,7 +36,8 @@ jobs:
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
         with:
-          version: '3.6.1'        
+          version: '3.6.1'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build binary
         run: |
@@ -76,7 +77,7 @@ jobs:
 
           docker tag ${{ env.ECR_PUSH_IMAGE }} ${{ env.ECR_LATEST }}
           docker push ${{ env.ECR_LATEST }}
-      
+
       - name: GIT | Checkout aleph-apps repo
         uses: actions/checkout@master
         with:
@@ -89,8 +90,8 @@ jobs:
         with:
           kustomize-version: "3.8.6"
 
-      - name: Update postsync hook docker image                                                                     
-        env:                                                                                                          
+      - name: Update postsync hook docker image
+        env:
           RELEASE_IMAGE: public.ecr.aws/p6e8q1z1/runtime-update-hook:${{ steps.vars.outputs.sha_short }}
           REGIONS_AWS: 'eu-central-1'
         run: |

.github/workflows/check-excluded-packages.yml

@@ -31,6 +31,7 @@ jobs:
         uses: arduino/setup-protoc@v1
         with:
           version: '3.6.1'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Read excluded packages from Cargo.toml
         id: read_excluded

.github/workflows/e2e-tests-main-devnet.yml

@@ -68,7 +68,8 @@ jobs:
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
         with:
-          version: '3.6.1'        
+          version: '3.6.1'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Restore cache
         uses: ./.github/actions/restore-cache
@@ -115,7 +116,8 @@ jobs:
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
         with:
-          version: '3.6.1'        
+          version: '3.6.1'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Install WASM target
         run: rustup target add wasm32-unknown-unknown
@@ -153,7 +155,8 @@ jobs:
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
         with:
-          version: '3.6.1'        
+          version: '3.6.1'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Restore cache
         uses: ./.github/actions/restore-cache
@@ -839,7 +842,8 @@ jobs:
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
         with:
-          version: '3.6.1'        
+          version: '3.6.1'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Install WASM target
         run: rustup target add wasm32-unknown-unknown

.github/workflows/nightly-pipeline.yaml

@@ -43,6 +43,37 @@ jobs:
           if-no-files-found: error
           retention-days: 7
 
+  build-synthetic-network-docker:
+    needs: [build-test-docker]
+    name: Build docker image with the test node artifact and support for synthetic-network
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v2
+
+      - name: Download artifact with docker image for aleph-node
+        uses: actions/download-artifact@v2
+        with:
+          name: aleph-test-docker
+
+      - name: Load node docker image
+        shell: bash
+        run: docker load -i aleph-node.tar
+
+      - name: Build test docker image
+        id: build-image
+        run: |
+          scripts/synthetic-network/build_synthetic-network.sh
+          docker save -o aleph-node.tar aleph-node:syntheticnet
+
+      - name: Upload test docker image
+        uses: actions/upload-artifact@v2
+        with:
+          name: aleph-test-synthetic-docker
+          path: aleph-node.tar
+          if-no-files-found: error
+          retention-days: 7
+
   check-determinism:
     needs: [build-new-node]
     name: Verify runtime build determinism
@@ -59,7 +90,8 @@ jobs:
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
         with:
-          version: '3.6.1'        
+          version: '3.6.1'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Install WASM target
         run: rustup target add wasm32-unknown-unknown
@@ -96,7 +128,8 @@ jobs:
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
         with:
-          version: '3.6.1'        
+          version: '3.6.1'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Restore cache
         uses: ./.github/actions/restore-cache
@@ -140,9 +173,45 @@ jobs:
           randomized: true
         timeout-minutes: 60
 
+  run-e2e-high-out-latency:
+    needs: [build-synthetic-network-docker, build-test-client]
+    name: Run high out-latency test
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v2
+
+      - name: Run e2e test
+        uses: ./.github/actions/run-e2e-test
+        with:
+          test-case: high_out_latency_for_all
+          image-path: aleph-test-synthetic-docker
+          node-image: aleph-node:syntheticnet
+          compose-file: docker/docker-compose.synthetic-network.yml
+        timeout-minutes: 30
+
+  run-e2e-no-quorum-without-high-out-latency:
+    needs: [build-synthetic-network-docker, build-test-client]
+    name: Run high out-latency for every quorum
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v2
+
+      - name: Run e2e test
+        uses: ./.github/actions/run-e2e-test
+        with:
+          test-case: high_out_latency_for_each_quorum
+          image-path: aleph-test-synthetic-docker
+          node-image: aleph-node:syntheticnet
+          compose-file: docker/docker-compose.synthetic-network.yml
+        timeout-minutes: 15
+
   check-e2e-test-suite-completion:
     needs: [
       run-e2e-authorities-are-staking,
+      run-e2e-high-out-latency,
+      run-e2e-no-quorum-without-high-out-latency,
     ]
     name: Check e2e test suite completion
     runs-on: ubuntu-20.04

.github/workflows/unit_tests.yml

@@ -30,7 +30,8 @@ jobs:
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
         with:
-          version: '3.6.1'        
+          version: '3.6.1'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Install clippy and fmt
         run: rustup component add clippy rustfmt