Merge branch 'main' into A0-1614-hide-subxt-types

Author: krzysztofziobro

.github/actions/run-e2e-test/action.yml

@@ -91,7 +91,7 @@ runs:
 
         RESERVED_SEATS="${{ inputs.reserved-seats }}"
         NON_RESERVED_SEATS="${{ inputs.non-reserved-seats }}"
-
+        
         if [[ -n "${RANDOMIZED}" ]]; then
           ARGS+=(-r "${RANDOMIZED}")
         fi

.github/scripts/run_consensus.sh

@@ -3,8 +3,10 @@
 set -euo pipefail
 
 # default node count
-# change when increasing the number of node containers
 NODE_COUNT=5
+# max node count that will not crash current GH machines
+MAX_NODE_COUNT=6
+# default minimum validator count
 MIN_VALIDATOR_COUNT=4
 DOCKER_COMPOSE=${DOCKER_COMPOSE:-"docker/docker-compose.yml"}
 OVERRIDE_DOCKER_COMPOSE=${OVERRIDE_DOCKER_COMPOSE:-""}
@@ -51,6 +53,11 @@ done
 
 export NODE_COUNT
 
+if [[ ${NODE_COUNT} -gt ${MAX_NODE_COUNT} ]]; then
+  echo "Tried to run ${NODE_COUNT} nodes. Max node count allowed: ${MAX_NODE_COUNT}."
+  exit 1
+fi
+
 function generate_authorities {
   local authorities_count="$1"
 

.github/workflows/deploy-testnet.yml

@@ -79,22 +79,13 @@ jobs:
           docker push ${{ env.DOCKERHUB_TESTNET_IMAGE }}
           docker push ${{ env.DOCKERHUB_TESTNET_LATEST_IMAGE }}
 
-      - name: S3 CI | Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v1
-        env:
-          AWS_REGION: us-east-1
-        with:
-          aws-access-key-id: ${{ secrets.AWS_DEVNET_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_DEVNET_SECRET_ACCESS_KEY }}
-          aws-region: ${{ env.AWS_REGION }}
-
       - name: S3 CI | Download release runtime from S3 bucket
         shell: bash
         env:
           S3BUCKET_URL: s3://${{ secrets.CI_MAINNET_S3BUCKET_NAME }}/builds/aleph-node/commits/${{ steps.get_branch.outputs.sha_short }}/aleph-runtime
           S3BUCKET_FILE: aleph-runtime-${{ steps.get_branch.outputs.sha_short }}.tar.gz
         run: |
-          aws s3 cp ${{ env.S3BUCKET_URL }}/${{ S3BUCKET_FILE }} ${{ env.S3BUCKET_FILE }}
+          aws s3 cp ${{ env.S3BUCKET_URL }}/${{ env.S3BUCKET_FILE }} ${{ env.S3BUCKET_FILE }}
 
       - name: RELEASE ASSET | Add runtime to the release
         uses: softprops/action-gh-release@v1

.github/workflows/deploy-to-devnet.yml

@@ -1,5 +1,11 @@
 name: Deploy to Devnet
 
+# This workflow performs automatic deployment of aleph-node to the Devnet environment
+# It does it from the scratch, ie it
+#   1) syncs the validators keys from S3,
+#   2) generates raw chainspec from the deployed aleph-node binary,
+#   3) restart nodes with cleaned db
+
 on:
   workflow_dispatch:
 
@@ -22,8 +28,7 @@ jobs:
         id: vars
         shell: bash
         run: |
-          echo "##[set-output name=branch;]$(echo ${GITHUB_REF##*/})"
-          echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+          echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1
@@ -49,58 +54,50 @@ jobs:
         with:
           version: 'v1.23.6'
 
-      - name: Run fork-off update
-        env:
-          RELEASE_TAG: ${{ steps.vars.outputs.sha_short }}
-
+      - name: Sync all validator's keystores from S3
         run: |
           #!/bin/bash
-
-          COMMIT_ID=$(curl -s -H "Content-Type: application/json" -d '{"id":1, "jsonrpc":"2.0", "method": "system_version"}' https://rpc.test.azero.dev | jq -r '.result' | cut -d "-" -f 2 | head -c 7)
-          echo $COMMIT_ID
-
-          # sync all validator's keystores from S3
           aws s3 cp s3://alephzero-devnet-eu-central-1-keys-bucket/data data --recursive
-
           # rename validator paths
           declare -A NAMES=([aleph-node-validator-0]=5D34dL5prEUaGNQtPPZ3yN5Y6BnkfXunKXXz6fo7ZJbLwRRH [aleph-node-validator-1]=5GBNeWRhZc2jXu7D55rBimKYDk8PGk8itRYFTPfC8RJLKG5o [aleph-node-validator-2]=5Dfis6XL8J2P6JHUnUtArnFWndn62SydeP8ee8sG2ky9nfm9 [aleph-node-validator-3]=5F4H97f7nQovyrbiq4ZetaaviNwThSVcFobcA5aGab6167dK [aleph-node-validator-4]=5DiDShBWa1fQx6gLzpf3SFBhMinCoyvHM1BWjPNsmXS8hkrW [aleph-node-validator-5]=5EFb84yH9tpcFuiKUcsmdoF7xeeY3ajG1ZLQimxQoFt9HMKR [aleph-node-validator-6]=5DZLHESsfGrJ5YzT3HuRPXsSNb589xQ4Unubh1mYLodzKdVY [aleph-node-validator-7]=5GHJzqvG6tXnngCpG7B12qjUvbo5e4e9z8Xjidk3CQZHxTPZ [aleph-node-validator-8]=5CUnSsgAyLND3bxxnfNhgWXSe9Wn676JzLpGLgyJv858qhoX [aleph-node-validator-9]=5CVKn7HAZW1Ky4r7Vkgsr7VEW88C2sHgUNDiwHY9Ct2hjU8q)
-
           for NAME in "${!NAMES[@]}"; do
             mv -v data/$NAME data/${NAMES[$NAME]}
           done
 
-          # generate chainspec, it will reuse keys from the synced keystore
-          docker run -i -v $(pwd)/data:/data --env RUST_BACKTRACE=1 --entrypoint "/usr/local/bin/aleph-node" public.ecr.aws/p6e8q1z1/aleph-node:${COMMIT_ID} bootstrap-chain --raw --base-path /data --chain-id a0dnet1 --account-ids 5D34dL5prEUaGNQtPPZ3yN5Y6BnkfXunKXXz6fo7ZJbLwRRH,5GBNeWRhZc2jXu7D55rBimKYDk8PGk8itRYFTPfC8RJLKG5o,5Dfis6XL8J2P6JHUnUtArnFWndn62SydeP8ee8sG2ky9nfm9,5F4H97f7nQovyrbiq4ZetaaviNwThSVcFobcA5aGab6167dK,5DiDShBWa1fQx6gLzpf3SFBhMinCoyvHM1BWjPNsmXS8hkrW,5EFb84yH9tpcFuiKUcsmdoF7xeeY3ajG1ZLQimxQoFt9HMKR,5DZLHESsfGrJ5YzT3HuRPXsSNb589xQ4Unubh1mYLodzKdVY,5GHJzqvG6tXnngCpG7B12qjUvbo5e4e9z8Xjidk3CQZHxTPZ,5CUnSsgAyLND3bxxnfNhgWXSe9Wn676JzLpGLgyJv858qhoX,5CVKn7HAZW1Ky4r7Vkgsr7VEW88C2sHgUNDiwHY9Ct2hjU8q --sudo-account-id 5F4SvwaUEQubiqkPF8YnRfcN77cLsT2DfG4vFeQmSXNjR7hD > chainspec.skeleton.json
-
-          docker run -i -v $(pwd):/app public.ecr.aws/p6e8q1z1/fork-off:latest --ws-rpc-endpoint=wss://ws.test.azero.dev --initial-spec-path=chainspec.skeleton.json --combined-spec-path=chainspec.json
+      - name: Generate chainspec
+        env:
+          RELEASE_TAG: ${{ steps.vars.outputs.sha_short }}
 
+        run: |
+          #!/bin/bash
+          aws s3 cp s3://alephzero-devnet-eu-central-1-keys-bucket/data data --recursive
+          docker run -i -v $(pwd)/data:/data --env RUST_BACKTRACE=1 --entrypoint "/usr/local/bin/aleph-node" public.ecr.aws/p6e8q1z1/aleph-node:${RELEASE_TAG} bootstrap-chain --raw --base-path /data --chain-id a0dnet1 --account-ids 5D34dL5prEUaGNQtPPZ3yN5Y6BnkfXunKXXz6fo7ZJbLwRRH,5GBNeWRhZc2jXu7D55rBimKYDk8PGk8itRYFTPfC8RJLKG5o,5Dfis6XL8J2P6JHUnUtArnFWndn62SydeP8ee8sG2ky9nfm9,5F4H97f7nQovyrbiq4ZetaaviNwThSVcFobcA5aGab6167dK,5DiDShBWa1fQx6gLzpf3SFBhMinCoyvHM1BWjPNsmXS8hkrW,5EFb84yH9tpcFuiKUcsmdoF7xeeY3ajG1ZLQimxQoFt9HMKR,5DZLHESsfGrJ5YzT3HuRPXsSNb589xQ4Unubh1mYLodzKdVY,5GHJzqvG6tXnngCpG7B12qjUvbo5e4e9z8Xjidk3CQZHxTPZ,5CUnSsgAyLND3bxxnfNhgWXSe9Wn676JzLpGLgyJv858qhoX,5CVKn7HAZW1Ky4r7Vkgsr7VEW88C2sHgUNDiwHY9Ct2hjU8q --sudo-account-id 5F4SvwaUEQubiqkPF8YnRfcN77cLsT2DfG4vFeQmSXNjR7hD > chainspec.json
           aws s3 cp chainspec.json s3://alephzero-devnet-eu-central-1-keys-bucket/chainspec.json
 
-          # stop and clean devnet
-
+      - name: Stop and purge db Devnet
+        run: |
+          #!/bin/bash
           aws eks --region eu-central-1 update-kubeconfig --name alephzero-devnet-eu-central-1-eks
 
           kubectl delete sts aleph-node-validator -n devnet --ignore-not-found=true
           kubectl delete pvc -l app=aleph-node-validator -n devnet --ignore-not-found=true
-          kubectl delete job send-runtime-hook -n devnet --ignore-not-found=true
-
-          cd aleph-apps/aleph-node-validators/overlays/devnet/eu-central-1
-          kustomize edit set image "aleph-node-validator-image-placeholder=public.ecr.aws/p6e8q1z1/aleph-node:${COMMIT_ID}"
-          kustomize edit remove resource send-runtime-hook.yaml
-          kustomize build . | kubectl apply -f -
-          sleep 2
-          kubectl rollout status --watch --timeout=3600s statefulset/aleph-node-validator -n devnet
 
-          echo "Waiting 15 minutes"
-          sleep 900
+      - name: Start Devnet
+        env:
+          RELEASE_TAG: ${{ steps.vars.outputs.sha_short }}
+        run: |
+          #!/bin/bash
 
+          cd aleph-apps/aleph-node-validators/overlays/devnet/eu-central-1
           kustomize edit set image "aleph-node-validator-image-placeholder=public.ecr.aws/p6e8q1z1/aleph-node:${RELEASE_TAG}"
           kustomize build . | kubectl apply -f -
           sleep 2
           kubectl rollout status --watch --timeout=3600s statefulset/aleph-node-validator -n devnet
 
-          kustomize edit add resource send-runtime-hook.yaml
-          kustomize build . | kubectl apply -f -
+      - name: Waiting 5 minutes for validators to boot
+        run: |
+          #!/bin/bash
+          sleep 300
 
       - name: GIT | Commit changes to aleph-apps repository.
         uses: EndBug/add-and-commit@v5.1.0

.github/workflows/e2e-tests-main-devnet.yml

@@ -188,6 +188,9 @@ jobs:
   run-aleph-client-subxt-codegen-check:
     needs: [build-test-docker]
     name: Checks if runtime file in aleph-client is up-to-date
+    # disabling this check as it causes troubles
+    # this should be reworked to sth like https://github.com/paritytech/subxt/blob/master/examples/examples/metadata_compatibility.rs
+    if: false
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout source code

.github/workflows/nightly-pipeline.yaml

@@ -170,7 +170,8 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: authorities_are_staking
-          randomized: true
+          node-count: 6
+          follow-up-finalization-check: true
         timeout-minutes: 60
 
   run-e2e-high-out-latency:

BUILD.md

@@ -53,7 +53,7 @@ is stored at `/lib64/ld-linux-x86-64.so.2`, you can execute `patchelf --set-inte
 aleph-node>`. Alternatively, you can use our nix-build script (used by docker based approach), i.q. `nix/nix-build.sh`.
 
 Note: we recommend using `direnv` together with `nix-direnv` for setting up nix-shell. This way you can use your preferred shell,
-instead of one provided by nix-shell.  
+instead of one provided by nix-shell.
 Example configuration for `direnv`. Copy it into `.envrc` file and then run `direnv-allow`:
 ```
 # installs nix-direnv, https://github.com/nix-community/nix-direnv
@@ -88,7 +88,7 @@ It might influence some of the build scripts of our build dependencies and it mi
 the environment flags related with the build process, like `CXXFLAGS` etc.
 Example build procedure using Ubuntu 20.04 LTS and bash shell:
 ```
-sudo apt install build-essential curl git clang libclang-dev pkg-config libssl-dev
+sudo apt install build-essential curl git clang libclang-dev pkg-config libssl-dev protobuf-compiler
 curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
 source "$HOME/.cargo/env"
 git clone https://github.com/Cardinal-Cryptography/aleph-node.git