diff --git a/.github/ISSUE_TEMPLATE/release.md b/.github/ISSUE_TEMPLATE/release.md index 709ae0d64..f14af724a 100644 --- a/.github/ISSUE_TEMPLATE/release.md +++ b/.github/ISSUE_TEMPLATE/release.md @@ -116,18 +116,15 @@ as long as the indexes did not change. ### Benchmarks -There are three benchmarking machines reserved for updating the weights at -release-time. To initialize a benchmark run for each production runtime -(calamari, manta): -* Go to [Calamari Benchmarking Github Action](https://github.com/Manta-Network/Manta/actions/workflows/generate_calamari_weights_files.yml) - and [Manta Benchmarking Github Action](https://github.com/Manta-Network/Manta/actions/workflows/generate_manta_weights_files.yml) +There is a manually deployed github action that runs all benchmarks on a bare-metal AWS machine. In order to use go to : +* Go to [Run All Benchmarks Github Action](https://github.com/Manta-Network/Manta/actions/workflows/run_all_benchmarks.yml) * Open `Run workflow` drop-down menu. -* Choose your branch and run the workflow. -* When these jobs have completed (it takes a few hours), custom weights files will - be available to download as artifacts. +* Choose your branch. +* Choose a chain-spec. You'll have to run the workflow multiple times usually with `dolphin-dev`, `calamari-dev`, `manta-dev`. +* Optionally choose whether you want to provide a chain database snapshot to benchmark storage performance. +* When these jobs have completed (it takes a few hours), all the benchmarks outputs files will be available to download as workflow artifacts. * Commit the changes to your branch and push to the remote branch for review. -* The weights should be (Currently manually) checked to make sure there are no - big outliers (i.e., twice or half the weight). +* The weights should be (Currently manually) checked to make sure there are no big outliers (i.e., twice or half the weight). ### Security Audit diff --git a/.github/workflows/generate_calamari_weights_files.yml b/.github/workflows/generate_calamari_weights_files.yml deleted file mode 100644 index 30a71a996..000000000 --- a/.github/workflows/generate_calamari_weights_files.yml +++ /dev/null @@ -1,271 +0,0 @@ -name: Benchmark Calamari Runtime & Generate Weights Files -on: - workflow_dispatch: -env: - AWS_REGION: us-east-1 - AWS_INSTANCE_TYPE: c5d.metal - AWS_INSTANCE_ROOT_VOLUME_SIZE: 32 - AWS_IMAGE_SEARCH_PATTERN: ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-* - AWS_IMAGE_SEARCH_OWNERS: '["099720109477"]' # canonical -jobs: - build-benchmark: - needs: start-node-builder-current - runs-on: ${{ needs.start-node-builder-current.outputs.runner-label }} - steps: - - uses: actions/checkout@v2 - - name: install sccache - env: - SCCACHE_RELEASE_URL: https://github.com/mozilla/sccache/releases/download - SCCACHE_VERSION: v0.2.15 - run: | - SCCACHE_FILE=sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl - mkdir -p $HOME/.local/bin - curl -L "$SCCACHE_RELEASE_URL/$SCCACHE_VERSION/$SCCACHE_FILE.tar.gz" | tar xz - mv -f $SCCACHE_FILE/sccache $HOME/.local/bin/sccache - chmod +x $HOME/.local/bin/sccache - echo "$HOME/.local/bin" >> $GITHUB_PATH - echo "$HOME/.cargo/bin" >> $GITHUB_PATH - - name: cache cargo registry - uses: actions/cache@v2 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - key: cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - cargo- - - name: cache sccache - uses: actions/cache@v2 - continue-on-error: false - with: - path: /home/runner/.cache/sccache - key: sccache-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - sccache- - - name: start sccache server - run: sccache --start-server - - name: init - run: | - curl -s https://sh.rustup.rs -sSf | sh -s -- -y - source ${HOME}/.cargo/env - rustup toolchain install stable - rustup default stable - rustup update - rustup target add wasm32-unknown-unknown - - name: build - env: - RUST_BACKTRACE: full - RUSTC_WRAPPER: sccache - SCCACHE_CACHE_SIZE: 2G - SCCACHE_DIR: /home/runner/.cache/sccache - CARGO_TERM_COLOR: always - run: | - source ${HOME}/.cargo/env - RUSTC_BOOTSTRAP=1 cargo build --profile production --verbose --features=runtime-benchmarks - - name: stop sccache server - run: sccache --stop-server || true - - name: upload - uses: actions/upload-artifact@v2 - with: - name: manta - path: target/production/manta - run-benchmark: - name: benchmark (${{ matrix.benchmark.pallet.name }} ${{ matrix.benchmark.extrinsic.name }}) - needs: [start-node-builder-current, build-benchmark] - runs-on: ${{ needs.start-node-builder-current.outputs.runner-label }} - strategy: - matrix: - benchmark: - - extrinsic: - id: '*' - name: calamari_vesting - pallet: - id: calamari_vesting - name: calamari_vesting - iterations: 20 - - extrinsic: - id: '*' - name: frame_system - pallet: - id: frame_system - name: frame_system - iterations: 20 - - extrinsic: - id: '*' - name: pallet_author_inherent - pallet: - id: pallet_author_inherent - name: pallet_author_inherent - iterations: 20 - - extrinsic: - id: '*' - name: pallet_balances - pallet: - id: pallet_balances - name: pallet_balances - iterations: 20 - - extrinsic: - id: '*' - name: pallet_collective - pallet: - id: pallet_collective - name: pallet_collective - iterations: 20 - - extrinsic: - id: '*' - name: pallet_democracy - pallet: - id: pallet_democracy - name: pallet_democracy - iterations: 20 - - extrinsic: - id: '*' - name: pallet_membership - pallet: - id: pallet_membership - name: pallet_membership - iterations: 20 - - extrinsic: - id: '*' - name: pallet_multisig - pallet: - id: pallet_multisig - name: pallet_multisig - iterations: 20 - - extrinsic: - id: '*' - name: pallet_scheduler - pallet: - id: pallet_scheduler - name: pallet_scheduler - iterations: 20 - - extrinsic: - id: '*' - name: pallet_session - pallet: - id: pallet_session - name: pallet_session - iterations: 20 - - extrinsic: - id: '*' - name: pallet_tx_pause - pallet: - id: pallet_tx_pause - name: pallet_tx_pause - iterations: 20 - - extrinsic: - id: '*' - name: pallet_utility - pallet: - id: pallet_utility - name: pallet_utility - iterations: 20 - - extrinsic: - id: '*' - name: pallet_timestamp - pallet: - id: pallet_timestamp - name: pallet_timestamp - iterations: 20 - - extrinsic: - id: '*' - name: pallet_treasury - pallet: - id: pallet_treasury - name: pallet_treasury - iterations: 20 - - extrinsic: - id: '*' - name: manta_collator_selection - pallet: - id: manta_collator_selection - name: manta_collator_selection - iterations: 20 - - extrinsic: - id: '*' - name: pallet_preimage - pallet: - id: pallet_preimage - name: pallet_preimage - iterations: 20 - - extrinsic: - id: '*' - name: pallet_assets - pallet: - id: pallet_assets - name: pallet_assets - iterations: 20 - - extrinsic: - id: '*' - name: pallet_asset_manager - pallet: - id: pallet_asset_manager - name: pallet_asset_manager - iterations: 20 - - extrinsic: - id: '*' - name: cumulus_pallet_xcmp_queue - pallet: - id: cumulus_pallet_xcmp_queue - name: cumulus_pallet_xcmp_queue - iterations: 20 - steps: - - uses: actions/download-artifact@v2 - with: - name: manta - - run: | - mv manta $HOME/.local/bin/ - chmod +x $HOME/.local/bin/manta - echo "$HOME/.local/bin" >> $GITHUB_PATH - - name: run benchmark - run: | - manta benchmark \ - pallet \ - --chain=calamari-dev \ - --pallet=${{ matrix.benchmark.pallet.id }} \ - --extrinsic=${{ matrix.benchmark.extrinsic.id }} \ - --execution=Wasm \ - --wasm-execution=Compiled \ - --heap-pages=4096 \ - --repeat=${{ matrix.benchmark.iterations }} \ - --steps=50 \ - --template=.github/resources/frame-weight-template.hbs \ - --output=${{ matrix.benchmark.pallet.name }}.rs - - uses: actions/upload-artifact@v2 - with: - name: ${{ matrix.benchmark.pallet.id }}-${{ matrix.benchmark.pallet.name }}.rs - path: ${{ github.workspace }}/${{ matrix.benchmark.pallet.name }}.rs - start-node-builder-current: - runs-on: ubuntu-20.04 - outputs: - runner-label: ${{ steps.start-self-hosted-runner.outputs.runner-label }} - aws-region: ${{ steps.start-self-hosted-runner.outputs.aws-region }} - aws-instance-id: ${{ steps.start-self-hosted-runner.outputs.aws-instance-id }} - steps: - - id: start-self-hosted-runner - uses: audacious-network/aws-github-runner@v1.0.33 - with: - mode: start - github-token: ${{ secrets.SELF_HOSTED_RUNNER_TOKEN }} - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-instance-ssh-public-key: ${{ env.AWS_INSTANCE_SSH_PUBLIC_KEY }} - aws-region: ${{ env.AWS_REGION }} - aws-instance-type: ${{ env.AWS_INSTANCE_TYPE }} - aws-instance-root-volume-size: ${{ env.AWS_INSTANCE_ROOT_VOLUME_SIZE }} - aws-image-search-pattern: ${{ env.AWS_IMAGE_SEARCH_PATTERN }} - aws-image-search-owners: ${{ env.AWS_IMAGE_SEARCH_OWNERS }} # canonical - stop-node-builder-current: - needs: [start-node-builder-current, run-benchmark] - runs-on: ubuntu-20.04 - if: ${{ always() }} - steps: - - uses: audacious-network/aws-github-runner@v1.0.33 - with: - mode: stop - github-token: ${{ secrets.SELF_HOSTED_RUNNER_TOKEN }} - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ needs.start-node-builder-current.outputs.aws-region }} - runner-label: ${{ needs.start-node-builder-current.outputs.runner-label }} - aws-instance-id: ${{ needs.start-node-builder-current.outputs.aws-instance-id }} diff --git a/.github/workflows/generate_dolphin_weights_files.yml b/.github/workflows/generate_dolphin_weights_files.yml deleted file mode 100644 index 00eb5d94b..000000000 --- a/.github/workflows/generate_dolphin_weights_files.yml +++ /dev/null @@ -1,271 +0,0 @@ -name: Benchmark Dolphin Runtime & Generate Weights Files -on: - workflow_dispatch: -env: - AWS_REGION: us-east-1 - AWS_INSTANCE_TYPE: c5d.metal - AWS_INSTANCE_ROOT_VOLUME_SIZE: 32 - AWS_IMAGE_SEARCH_PATTERN: ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-* - AWS_IMAGE_SEARCH_OWNERS: '["099720109477"]' # canonical -jobs: - build-benchmark: - needs: start-node-builder-current - runs-on: ${{ needs.start-node-builder-current.outputs.runner-label }} - steps: - - uses: actions/checkout@v2 - - name: install sccache - env: - SCCACHE_RELEASE_URL: https://github.com/mozilla/sccache/releases/download - SCCACHE_VERSION: v0.2.15 - run: | - SCCACHE_FILE=sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl - mkdir -p $HOME/.local/bin - curl -L "$SCCACHE_RELEASE_URL/$SCCACHE_VERSION/$SCCACHE_FILE.tar.gz" | tar xz - mv -f $SCCACHE_FILE/sccache $HOME/.local/bin/sccache - chmod +x $HOME/.local/bin/sccache - echo "$HOME/.local/bin" >> $GITHUB_PATH - echo "$HOME/.cargo/bin" >> $GITHUB_PATH - - name: cache cargo registry - uses: actions/cache@v2 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - key: cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - cargo- - - name: cache sccache - uses: actions/cache@v2 - continue-on-error: false - with: - path: /home/runner/.cache/sccache - key: sccache-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - sccache- - - name: start sccache server - run: sccache --start-server - - name: init - run: | - curl -s https://sh.rustup.rs -sSf | sh -s -- -y - source ${HOME}/.cargo/env - rustup toolchain install stable - rustup default stable - rustup update - rustup target add wasm32-unknown-unknown - - name: build - env: - RUST_BACKTRACE: full - RUSTC_WRAPPER: sccache - SCCACHE_CACHE_SIZE: 2G - SCCACHE_DIR: /home/runner/.cache/sccache - CARGO_TERM_COLOR: always - run: | - source ${HOME}/.cargo/env - RUSTC_BOOTSTRAP=1 cargo build --profile production --verbose --features=runtime-benchmarks - - name: stop sccache server - run: sccache --stop-server || true - - name: upload - uses: actions/upload-artifact@v2 - with: - name: manta - path: target/production/manta - run-benchmark: - name: benchmark (${{ matrix.benchmark.pallet.name }} ${{ matrix.benchmark.extrinsic.name }}) - needs: [start-node-builder-current, build-benchmark] - runs-on: ${{ needs.start-node-builder-current.outputs.runner-label }} - strategy: - matrix: - benchmark: - - extrinsic: - id: '*' - name: frame_system - pallet: - id: frame_system - name: frame_system - iterations: 20 - - extrinsic: - id: '*' - name: pallet_author_inherent - pallet: - id: pallet_author_inherent - name: pallet_author_inherent - iterations: 20 - - extrinsic: - id: '*' - name: pallet_balances - pallet: - id: pallet_balances - name: pallet_balances - iterations: 20 - - extrinsic: - id: '*' - name: pallet_collective - pallet: - id: pallet_collective - name: pallet_collective - iterations: 20 - - extrinsic: - id: '*' - name: pallet_democracy - pallet: - id: pallet_democracy - name: pallet_democracy - iterations: 20 - - extrinsic: - id: '*' - name: pallet_membership - pallet: - id: pallet_membership - name: pallet_membership - iterations: 20 - - extrinsic: - id: '*' - name: pallet_multisig - pallet: - id: pallet_multisig - name: pallet_multisig - iterations: 20 - - extrinsic: - id: '*' - name: pallet_scheduler - pallet: - id: pallet_scheduler - name: pallet_scheduler - iterations: 20 - - extrinsic: - id: '*' - name: pallet_session - pallet: - id: pallet_session - name: pallet_session - iterations: 20 - - extrinsic: - id: '*' - name: pallet_tx_pause - pallet: - id: pallet_tx_pause - name: pallet_tx_pause - iterations: 20 - - extrinsic: - id: '*' - name: pallet_utility - pallet: - id: pallet_utility - name: pallet_utility - iterations: 20 - - extrinsic: - id: '*' - name: pallet_timestamp - pallet: - id: pallet_timestamp - name: pallet_timestamp - iterations: 20 - - extrinsic: - id: '*' - name: pallet_treasury - pallet: - id: pallet_treasury - name: pallet_treasury - iterations: 20 - - extrinsic: - id: '*' - name: manta_collator_selection - pallet: - id: manta_collator_selection - name: manta_collator_selection - iterations: 20 - - extrinsic: - id: '*' - name: pallet_preimage - pallet: - id: pallet_preimage - name: pallet_preimage - iterations: 20 - - extrinsic: - id: '*' - name: pallet_assets - pallet: - id: pallet_assets - name: pallet_assets - iterations: 20 - - extrinsic: - id: '*' - name: pallet_asset_manager - pallet: - id: pallet_asset_manager - name: pallet_asset_manager - iterations: 20 - - extrinsic: - id: '*' - name: pallet_manta_pay - pallet: - id: pallet_manta_pay - name: pallet_manta_pay - iterations: 20 - - extrinsic: - id: '*' - name: cumulus_pallet_xcmp_queue - pallet: - id: cumulus_pallet_xcmp_queue - name: cumulus_pallet_xcmp_queue - iterations: 20 - steps: - - uses: actions/download-artifact@v2 - with: - name: manta - - run: | - mv manta $HOME/.local/bin/ - chmod +x $HOME/.local/bin/manta - echo "$HOME/.local/bin" >> $GITHUB_PATH - - name: run benchmark - run: | - manta benchmark \ - pallet \ - --chain=dolphin-dev \ - --pallet=${{ matrix.benchmark.pallet.id }} \ - --extrinsic=${{ matrix.benchmark.extrinsic.id }} \ - --execution=Wasm \ - --wasm-execution=Compiled \ - --heap-pages=4096 \ - --repeat=${{ matrix.benchmark.iterations }} \ - --steps=50 \ - --template=.github/resources/frame-weight-template.hbs \ - --output=${{ matrix.benchmark.pallet.name }}.rs - - uses: actions/upload-artifact@v2 - with: - name: ${{ matrix.benchmark.pallet.id }}-${{ matrix.benchmark.pallet.name }}.rs - path: ${{ github.workspace }}/${{ matrix.benchmark.pallet.name }}.rs - start-node-builder-current: - runs-on: ubuntu-20.04 - outputs: - runner-label: ${{ steps.start-self-hosted-runner.outputs.runner-label }} - aws-region: ${{ steps.start-self-hosted-runner.outputs.aws-region }} - aws-instance-id: ${{ steps.start-self-hosted-runner.outputs.aws-instance-id }} - steps: - - id: start-self-hosted-runner - uses: audacious-network/aws-github-runner@v1.0.33 - with: - mode: start - github-token: ${{ secrets.SELF_HOSTED_RUNNER_TOKEN }} - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-instance-ssh-public-key: ${{ env.AWS_INSTANCE_SSH_PUBLIC_KEY }} - aws-region: ${{ env.AWS_REGION }} - aws-instance-type: ${{ env.AWS_INSTANCE_TYPE }} - aws-instance-root-volume-size: ${{ env.AWS_INSTANCE_ROOT_VOLUME_SIZE }} - aws-image-search-pattern: ${{ env.AWS_IMAGE_SEARCH_PATTERN }} - aws-image-search-owners: ${{ env.AWS_IMAGE_SEARCH_OWNERS }} # canonical - stop-node-builder-current: - needs: [start-node-builder-current, run-benchmark] - runs-on: ubuntu-20.04 - if: ${{ always() }} - steps: - - uses: audacious-network/aws-github-runner@v1.0.33 - with: - mode: stop - github-token: ${{ secrets.SELF_HOSTED_RUNNER_TOKEN }} - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ needs.start-node-builder-current.outputs.aws-region }} - runner-label: ${{ needs.start-node-builder-current.outputs.runner-label }} - aws-instance-id: ${{ needs.start-node-builder-current.outputs.aws-instance-id }} diff --git a/.github/workflows/metadata_diff.yml b/.github/workflows/metadata_diff.yml index 8cc1ba14d..ef0cc73c8 100644 --- a/.github/workflows/metadata_diff.yml +++ b/.github/workflows/metadata_diff.yml @@ -129,7 +129,8 @@ jobs: cat output.txt - name: Stop our local node run: pkill manta - - name: Save output as artifact + - if: always() + name: Save output as artifact uses: actions/upload-artifact@v2 with: name: ${{ env.CHAIN }} diff --git a/.github/workflows/publish_draft_releases.yml b/.github/workflows/publish_draft_releases.yml index 5d1f3a5f1..336144386 100644 --- a/.github/workflows/publish_draft_releases.yml +++ b/.github/workflows/publish_draft_releases.yml @@ -63,12 +63,14 @@ jobs: run: > echo '${{ steps.srtool-build.outputs.json }}' | jq > ${{ matrix.runtime.name }}-srtool-output.json - - name: upload srtool json + - if: always() + name: upload srtool json uses: actions/upload-artifact@v2 with: name: ${{ matrix.runtime.name }}-srtool-json path: ${{ matrix.runtime.name }}-srtool-output.json - - name: upload runtime + - if: always() + name: upload runtime uses: actions/upload-artifact@v2 with: name: ${{ matrix.runtime.name }}-runtime @@ -134,17 +136,20 @@ jobs: RUSTC_BOOTSTRAP=1 cargo build --profile production --verbose - name: stop sccache server run: sccache --stop-server || true - - name: upload + - if: always() + name: upload uses: actions/upload-artifact@v2 with: name: manta path: target/production/manta - - name: upload + - if: always() + name: upload uses: actions/upload-artifact@v2 with: name: config-for-integration-test path: .github/resources/config-for-integration-test.json - - name: upload + - if: always() + name: upload uses: actions/upload-artifact@v2 with: name: config-for-runtime-upgrade-test @@ -404,15 +409,18 @@ jobs: ls -ahl $HOME/.local/share/calamari-pc/ echo "::set-output name=short-sha::${GITHUB_SHA:0:7}" manta export-state --chain $HOME/.local/share/calamari-pc/${{ matrix.chain-spec.id }}-${GITHUB_SHA:0:7}-spec.json > $HOME/.local/share/calamari-pc/${{ matrix.chain-spec.id }}-state.json || true - - uses: actions/upload-artifact@v2 + - if: always() + uses: actions/upload-artifact@v2 with: name: ${{ matrix.chain-spec.id }}-state.json path: /home/runner/.local/share/calamari-pc/${{ matrix.chain-spec.id }}-state.json - - uses: actions/upload-artifact@v2 + - if: always() + uses: actions/upload-artifact@v2 with: name: ${{ matrix.chain-spec.id }}-spec.json path: /home/runner/.local/share/calamari-pc/${{ matrix.chain-spec.id }}-spec.json - - uses: actions/upload-artifact@v2 + - if: always() + uses: actions/upload-artifact@v2 with: name: ${{ matrix.chain-spec.id }}-${{ steps.create-chainspec.outputs.short-sha }}-spec.json path: /home/runner/.local/share/calamari-pc/${{ matrix.chain-spec.id }}-${{ steps.create-chainspec.outputs.short-sha }}-spec.json @@ -438,7 +446,8 @@ jobs: ' > $HOME/.local/share/calamari-pc/${{ matrix.chain-spec.id }}-${GITHUB_SHA:0:7}-launch-config.json jq . $HOME/.local/share/calamari-pc/${{ matrix.chain-spec.id }}-${GITHUB_SHA:0:7}-launch-config.json ls -ahl $HOME/.local/share/calamari-pc/ - - uses: actions/upload-artifact@v2 + - if: always() + uses: actions/upload-artifact@v2 with: name: ${{ matrix.chain-spec.id }}-${{ steps.create-chainspec.outputs.short-sha }}-launch-config.json path: /home/runner/.local/share/calamari-pc/${{ matrix.chain-spec.id }}-${{ steps.create-chainspec.outputs.short-sha }}-launch-config.json @@ -505,43 +514,53 @@ jobs: pm2 stop measure-block-time-rococo-relay-for-${{ matrix.chain-spec.id }} pm2 stop measure-block-time-${{ matrix.chain-spec.id }} pm2 stop polkadot-launch - - uses: actions/upload-artifact@v2 + - if: always() + uses: actions/upload-artifact@v2 with: name: polkadot-launch-for-${{ matrix.chain-spec.id }}-stdout.log path: ${{ github.workspace }}/polkadot-launch-for-${{ matrix.chain-spec.id }}-stdout.log - - uses: actions/upload-artifact@v2 + - if: always() + uses: actions/upload-artifact@v2 with: name: polkadot-launch-for-${{ matrix.chain-spec.id }}-stderr.log path: ${{ github.workspace }}/polkadot-launch-for-${{ matrix.chain-spec.id }}-stderr.log - - uses: actions/upload-artifact@v2 + - if: always() + uses: actions/upload-artifact@v2 with: name: measure-block-time-rococo-relay-for-${{ matrix.chain-spec.id }}-stdout.log path: ${{ github.workspace }}/measure-block-time-rococo-relay-for-${{ matrix.chain-spec.id }}-stdout.log - - uses: actions/upload-artifact@v2 + - if: always() + uses: actions/upload-artifact@v2 with: name: measure-block-time-rococo-relay-for-${{ matrix.chain-spec.id }}-stderr.log path: ${{ github.workspace }}/measure-block-time-rococo-relay-for-${{ matrix.chain-spec.id }}-stderr.log - - uses: actions/upload-artifact@v2 + - if: always() + uses: actions/upload-artifact@v2 with: name: measure-block-time-${{ matrix.chain-spec.id }}-stdout.log path: ${{ github.workspace }}/measure-block-time-${{ matrix.chain-spec.id }}-stdout.log - - uses: actions/upload-artifact@v2 + - if: always() + uses: actions/upload-artifact@v2 with: name: measure-block-time-${{ matrix.chain-spec.id }}-stderr.log path: ${{ github.workspace }}/measure-block-time-${{ matrix.chain-spec.id }}-stderr.log - - uses: actions/upload-artifact@v2 + - if: always() + uses: actions/upload-artifact@v2 with: name: rococo-alice-relay-for-${{ matrix.chain-spec.id }}.log path: ${{ github.workspace }}/polkadot-launch/alice.log - - uses: actions/upload-artifact@v2 + - if: always() + uses: actions/upload-artifact@v2 with: name: rococo-bob-relay-for-${{ matrix.chain-spec.id }}.log path: ${{ github.workspace }}/polkadot-launch/bob.log - - uses: actions/upload-artifact@v2 + - if: always() + uses: actions/upload-artifact@v2 with: name: ${{ matrix.chain-spec.id }}-alice.log path: ${{ github.workspace }}/polkadot-launch/9921.log - - uses: actions/upload-artifact@v2 + - if: always() + uses: actions/upload-artifact@v2 with: name: ${{ matrix.chain-spec.id }}-bob.log path: ${{ github.workspace }}/polkadot-launch/9922.log @@ -550,26 +569,32 @@ jobs: grep '#.*' ${{ github.workspace }}/measure-block-time-rococo-relay-for-${{ matrix.chain-spec.id }}-stdout.log | while read -r line; do words=($line); echo ${words[6]},${words[10]} | tee ${{ github.workspace }}/block-time-rococo.csv; done if [ ! -f ${{ github.workspace }}/block-time-rococo.csv ]; then echo "block times not detected"; exit 1; fi jq -s -R '[split("\n") | .[] | select(length > 0) | split(",") | {block:.[0]|tonumber, time:.[1]|tonumber} ]' ${{ github.workspace }}/block-time-rococo.csv > ${{ github.workspace }}/block-time-rococo.json - - uses: actions/upload-artifact@v2 + - if: always() + uses: actions/upload-artifact@v2 with: name: block-time-rococo-relay-for-${{ matrix.chain-spec.id }}.json path: ${{ github.workspace }}/block-time-rococo.json -# - name: parse calamari block times -# run: | -# grep '#.*' ${{ github.workspace }}/measure-block-time-${{ matrix.chain-spec.id }}-stdout.log | while read -r line; do words=($line); echo ${words[6]},${words[10]} | tee ${{ github.workspace }}/block-time-${{ matrix.chain-spec.id }}.csv; done -# if [ ! -f ${{ github.workspace }}/block-time-${{ matrix.chain-spec.id }}.csv ]; then echo "block times not detected"; exit 1; fi -# jq -s -R '[split("\n") | .[] | select(length > 0) | split(",") | {block:.[0]|tonumber, time:.[1]|tonumber} ]' ${{ github.workspace }}/block-time-${{ matrix.chain-spec.id }}.csv > ${{ github.workspace }}/block-time-${{ matrix.chain-spec.id }}.json - - uses: actions/upload-artifact@v2 + # - if: always() + # name: parse parachain block times + # run: | + # grep '#.*' ${{ github.workspace }}/measure-block-time-${{ matrix.chain-spec.id }}-stdout.log | while read -r line; do words=($line); echo ${words[6]},${words[10]} | tee ${{ github.workspace }}/block-time-${{ matrix.chain-spec.id }}.csv; done + # if [ ! -f ${{ github.workspace }}/block-time-${{ matrix.chain-spec.id }}.csv ]; then echo "block times not detected"; exit 1; fi + # jq -s -R '[split("\n") | .[] | select(length > 0) | split(",") | {block:.[0]|tonumber, time:.[1]|tonumber} ]' ${{ github.workspace }}/block-time-${{ matrix.chain-spec.id }}.csv > ${{ github.workspace }}/block-time-${{ matrix.chain-spec.id }}.json + - if: always() + uses: actions/upload-artifact@v2 with: name: block-time-${{ matrix.chain-spec.id }}.json path: ${{ github.workspace }}/block-time-${{ matrix.chain-spec.id }}.json - - name: test - rococo alice peered successfully + - if: always() + name: test - rococo alice peered successfully run: | grep '💤 Idle (${{ matrix.chain-spec.expected.peer-count.relay }} peers)' ${{ github.workspace }}/polkadot-launch/alice.log - - name: test - rococo alice imported block ${{ matrix.chain-spec.expected.block-count.relay }} + - if: always() + name: test - rococo alice imported block ${{ matrix.chain-spec.expected.block-count.relay }} run: | grep ' Imported #${{ matrix.chain-spec.expected.block-count.relay }} ' ${{ github.workspace }}/polkadot-launch/alice.log - - name: test - calamari alice peered successfully + - if: always() + name: test - parachain alice peered successfully run: | grep '\[Parachain\] 💤 Idle (${{ matrix.chain-spec.expected.peer-count.para }} peers)' ${{ github.workspace }}/polkadot-launch/9921.log dolphin-integration-test: @@ -634,15 +659,18 @@ jobs: ls -ahl $HOME/.local/share/calamari-pc/ echo "::set-output name=short-sha::${GITHUB_SHA:0:7}" manta export-state --chain $HOME/.local/share/calamari-pc/${{ matrix.chain-spec.id }}-${GITHUB_SHA:0:7}-spec.json > $HOME/.local/share/calamari-pc/${{ matrix.chain-spec.id }}-state.json || true - - uses: actions/upload-artifact@v2 + - if: always() + uses: actions/upload-artifact@v2 with: name: ${{ matrix.chain-spec.id }}-state.json path: /home/runner/.local/share/calamari-pc/${{ matrix.chain-spec.id }}-state.json - - uses: actions/upload-artifact@v2 + - if: always() + uses: actions/upload-artifact@v2 with: name: ${{ matrix.chain-spec.id }}-spec.json path: /home/runner/.local/share/calamari-pc/${{ matrix.chain-spec.id }}-spec.json - - uses: actions/upload-artifact@v2 + - if: always() + uses: actions/upload-artifact@v2 with: name: ${{ matrix.chain-spec.id }}-${{ steps.create-chainspec.outputs.short-sha }}-spec.json path: /home/runner/.local/share/calamari-pc/${{ matrix.chain-spec.id }}-${{ steps.create-chainspec.outputs.short-sha }}-spec.json @@ -668,7 +696,8 @@ jobs: ' > $HOME/.local/share/calamari-pc/${{ matrix.chain-spec.id }}-${GITHUB_SHA:0:7}-launch-config.json jq . $HOME/.local/share/calamari-pc/${{ matrix.chain-spec.id }}-${GITHUB_SHA:0:7}-launch-config.json ls -ahl $HOME/.local/share/calamari-pc/ - - uses: actions/upload-artifact@v2 + - if: always() + uses: actions/upload-artifact@v2 with: name: ${{ matrix.chain-spec.id }}-${{ steps.create-chainspec.outputs.short-sha }}-launch-config.json path: /home/runner/.local/share/calamari-pc/${{ matrix.chain-spec.id }}-${{ steps.create-chainspec.outputs.short-sha }}-launch-config.json @@ -748,71 +777,88 @@ jobs: pm2 stop measure-block-time-rococo-relay-for-${{ matrix.chain-spec.id }} pm2 stop measure-block-time-${{ matrix.chain-spec.id }} pm2 stop polkadot-launch - - uses: actions/upload-artifact@v2 + - if: always() + uses: actions/upload-artifact@v2 with: name: polkadot-launch-for-${{ matrix.chain-spec.id }}-stdout.log path: ${{ github.workspace }}/polkadot-launch-for-${{ matrix.chain-spec.id }}-stdout.log - - uses: actions/upload-artifact@v2 + - if: always() + uses: actions/upload-artifact@v2 with: name: polkadot-launch-for-${{ matrix.chain-spec.id }}-stderr.log path: ${{ github.workspace }}/polkadot-launch-for-${{ matrix.chain-spec.id }}-stderr.log - - uses: actions/upload-artifact@v2 + - if: always() + uses: actions/upload-artifact@v2 with: name: measure-block-time-rococo-relay-for-${{ matrix.chain-spec.id }}-stdout.log path: ${{ github.workspace }}/measure-block-time-rococo-relay-for-${{ matrix.chain-spec.id }}-stdout.log - - uses: actions/upload-artifact@v2 + - if: always() + uses: actions/upload-artifact@v2 with: name: measure-block-time-rococo-relay-for-${{ matrix.chain-spec.id }}-stderr.log path: ${{ github.workspace }}/measure-block-time-rococo-relay-for-${{ matrix.chain-spec.id }}-stderr.log - - uses: actions/upload-artifact@v2 + - if: always() + uses: actions/upload-artifact@v2 with: name: measure-block-time-${{ matrix.chain-spec.id }}-stdout.log path: ${{ github.workspace }}/measure-block-time-${{ matrix.chain-spec.id }}-stdout.log - - uses: actions/upload-artifact@v2 + - if: always() + uses: actions/upload-artifact@v2 with: name: measure-block-time-${{ matrix.chain-spec.id }}-stderr.log path: ${{ github.workspace }}/measure-block-time-${{ matrix.chain-spec.id }}-stderr.log - - uses: actions/upload-artifact@v2 + - if: always() + uses: actions/upload-artifact@v2 with: name: rococo-alice-relay-for-${{ matrix.chain-spec.id }}.log path: ${{ github.workspace }}/polkadot-launch/alice.log - - uses: actions/upload-artifact@v2 + - if: always() + uses: actions/upload-artifact@v2 with: name: rococo-bob-relay-for-${{ matrix.chain-spec.id }}.log path: ${{ github.workspace }}/polkadot-launch/bob.log - - uses: actions/upload-artifact@v2 + - if: always() + uses: actions/upload-artifact@v2 with: name: ${{ matrix.chain-spec.id }}-alice.log path: ${{ github.workspace }}/polkadot-launch/9921.log - - uses: actions/upload-artifact@v2 + - if: always() + uses: actions/upload-artifact@v2 with: name: ${{ matrix.chain-spec.id }}-bob.log path: ${{ github.workspace }}/polkadot-launch/9922.log - - name: parse rococo block times + - if: always() + name: parse rococo block times run: | grep '#.*' ${{ github.workspace }}/measure-block-time-rococo-relay-for-${{ matrix.chain-spec.id }}-stdout.log | while read -r line; do words=($line); echo ${words[6]},${words[10]} | tee ${{ github.workspace }}/block-time-rococo.csv; done if [ ! -f ${{ github.workspace }}/block-time-rococo.csv ]; then echo "block times not detected"; exit 1; fi jq -s -R '[split("\n") | .[] | select(length > 0) | split(",") | {block:.[0]|tonumber, time:.[1]|tonumber} ]' ${{ github.workspace }}/block-time-rococo.csv > ${{ github.workspace }}/block-time-rococo.json - - uses: actions/upload-artifact@v2 + - if: always() + uses: actions/upload-artifact@v2 with: name: block-time-rococo-relay-for-${{ matrix.chain-spec.id }}.json path: ${{ github.workspace }}/block-time-rococo.json -# - name: parse calamari block times -# run: | -# grep '#.*' ${{ github.workspace }}/measure-block-time-${{ matrix.chain-spec.id }}-stdout.log | while read -r line; do words=($line); echo ${words[6]},${words[10]} | tee ${{ github.workspace }}/block-time-${{ matrix.chain-spec.id }}.csv; done -# if [ ! -f ${{ github.workspace }}/block-time-${{ matrix.chain-spec.id }}.csv ]; then echo "block times not detected"; exit 1; fi -# jq -s -R '[split("\n") | .[] | select(length > 0) | split(",") | {block:.[0]|tonumber, time:.[1]|tonumber} ]' ${{ github.workspace }}/block-time-${{ matrix.chain-spec.id }}.csv > ${{ github.workspace }}/block-time-${{ matrix.chain-spec.id }}.json - - uses: actions/upload-artifact@v2 + # - if: always() + # name: parse parachain block times + # run: | + # grep '#.*' ${{ github.workspace }}/measure-block-time-${{ matrix.chain-spec.id }}-stdout.log | while read -r line; do words=($line); echo ${words[6]},${words[10]} | tee ${{ github.workspace }}/block-time-${{ matrix.chain-spec.id }}.csv; done + # if [ ! -f ${{ github.workspace }}/block-time-${{ matrix.chain-spec.id }}.csv ]; then echo "block times not detected"; exit 1; fi + # jq -s -R '[split("\n") | .[] | select(length > 0) | split(",") | {block:.[0]|tonumber, time:.[1]|tonumber} ]' ${{ github.workspace }}/block-time-${{ matrix.chain-spec.id }}.csv > ${{ github.workspace }}/block-time-${{ matrix.chain-spec.id }}.json + - if: always() + uses: actions/upload-artifact@v2 with: name: block-time-${{ matrix.chain-spec.id }}.json path: ${{ github.workspace }}/block-time-${{ matrix.chain-spec.id }}.json - - name: test - rococo alice peered successfully + - if: always() + name: test - rococo alice peered successfully run: | grep '💤 Idle (${{ matrix.chain-spec.expected.peer-count.relay }} peers)' ${{ github.workspace }}/polkadot-launch/alice.log - - name: test - rococo alice imported block ${{ matrix.chain-spec.expected.block-count.relay }} + - if: always() + name: test - rococo alice imported block ${{ matrix.chain-spec.expected.block-count.relay }} run: | grep ' Imported #${{ matrix.chain-spec.expected.block-count.relay }} ' ${{ github.workspace }}/polkadot-launch/alice.log - - name: test - calamari alice peered successfully + - if: always() + name: test - parachain alice peered successfully run: | grep '\[Parachain\] 💤 Idle (${{ matrix.chain-spec.expected.peer-count.para }} peers)' ${{ github.workspace }}/polkadot-launch/9921.log create-draft-release: @@ -948,10 +994,10 @@ jobs: aws-region: ${{ env.AWS_REGION }} aws-subnet-id: ${{ env.AWS_SUBNET_ID }} aws-security-group-id: ${{ env.AWS_SECURITY_GROUP_ID }} - aws-instance-type: ${{ env.AWS_INSTANCE_TYPE }} # 32 vcpu, 64gb ram, $1.392 hourly + aws-instance-type: ${{ env.AWS_INSTANCE_TYPE }} aws-instance-root-volume-size: 32 aws-image-search-pattern: ${{ env.AWS_IMAGE_SEARCH_PATTERN }} - aws-image-search-owners: ${{ env.AWS_IMAGE_SEARCH_OWNERS }} # canonical + aws-image-search-owners: ${{ env.AWS_IMAGE_SEARCH_OWNERS }} stop-node-builder-current: needs: [start-node-builder-current, build-node-current] runs-on: ubuntu-20.04 @@ -985,10 +1031,10 @@ jobs: aws-region: ${{ env.AWS_REGION }} aws-subnet-id: ${{ env.AWS_SUBNET_ID }} aws-security-group-id: ${{ env.AWS_SECURITY_GROUP_ID }} - aws-instance-type: ${{ env.AWS_INSTANCE_TYPE }} # 32 vcpu, 64gb ram, $1.392 hourly + aws-instance-type: ${{ env.AWS_INSTANCE_TYPE }} aws-instance-root-volume-size: 64 aws-image-search-pattern: ${{ env.AWS_IMAGE_SEARCH_PATTERN }} - aws-image-search-owners: ${{ env.AWS_IMAGE_SEARCH_OWNERS }} # canonical + aws-image-search-owners: ${{ env.AWS_IMAGE_SEARCH_OWNERS }} stop-calamari-integration-tester: needs: [start-calamari-integration-tester, calamari-integration-test] runs-on: ubuntu-20.04 diff --git a/.github/workflows/generate_manta_weights_files.yml b/.github/workflows/run_all_benchmarks.yml similarity index 52% rename from .github/workflows/generate_manta_weights_files.yml rename to .github/workflows/run_all_benchmarks.yml index e66188e13..5c3be5976 100644 --- a/.github/workflows/generate_manta_weights_files.yml +++ b/.github/workflows/run_all_benchmarks.yml @@ -1,14 +1,26 @@ -name: Benchmark Manta Runtime & Generate Weights Files +name: Run All Benchmarks on: workflow_dispatch: + inputs: + chain_spec: + description: The chain spec to be used for all benchmarks. Required. + default: calamari-dev + required: true + snapshot_url: + description: The chain snapshot to use for storage benchmarks. Leaving it blank will skip the storage benchmark. + default: "" + required: false env: AWS_REGION: us-east-1 AWS_INSTANCE_TYPE: c5d.metal - AWS_INSTANCE_ROOT_VOLUME_SIZE: 32 + AWS_INSTANCE_ROOT_VOLUME_SIZE: 128 AWS_IMAGE_SEARCH_PATTERN: ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-* - AWS_IMAGE_SEARCH_OWNERS: '["099720109477"]' # canonical + AWS_IMAGE_SEARCH_OWNERS: '["099720109477"]' + CHAIN_SPEC: ${{github.event.inputs.chain_spec}} + SNAPSHOT_URL: ${{github.event.inputs.snapshot_url}} + FULL_DB_FOLDER: full-db jobs: - build-benchmark: + run-benchmarks: needs: start-node-builder-current runs-on: ${{ needs.start-node-builder-current.outputs.runner-label }} steps: @@ -52,7 +64,7 @@ jobs: rustup default stable rustup update rustup target add wasm32-unknown-unknown - - name: build + - name: build benchmarking binary env: RUST_BACKTRACE: full RUSTC_WRAPPER: sccache @@ -61,103 +73,50 @@ jobs: CARGO_TERM_COLOR: always run: | source ${HOME}/.cargo/env - RUSTC_BOOTSTRAP=1 cargo build --profile production --verbose --features=runtime-benchmarks + RUSTC_BOOTSTRAP=1 cargo build --profile production --features runtime-benchmarks --verbose - name: stop sccache server run: sccache --stop-server || true - - name: upload - uses: actions/upload-artifact@v2 + - if: ${{ env.SNAPSHOT_URL == '' }} + name: run all benchmarks script without storage benchmark + run: | + ./scripts/benchmarking/run_all_benchmarks.sh -b -c $CHAIN_SPEC + - if: ${{ env.SNAPSHOT_URL != '' }} + name: run all benchmarks script with storage benchmark + run: | + mkdir $FULL_DB_FOLDER + wget -q $SNAPSHOT_URL + tar -xf calamari.tar.gz --directory ./$FULL_DB_FOLDER + ./scripts/benchmarking/run_all_benchmarks.sh -b -c $CHAIN_SPEC -s ./$FULL_DB_FOLDER + - if: always() + name: upload benchmarking binary + uses: actions/upload-artifact@v3 with: name: manta - path: target/production/manta - run-benchmark: - name: benchmark (${{ matrix.benchmark.pallet.name }} ${{ matrix.benchmark.extrinsic.name }}) - needs: [start-node-builder-current, build-benchmark] - runs-on: ${{ needs.start-node-builder-current.outputs.runner-label }} - strategy: - matrix: - benchmark: - - extrinsic: - id: '*' - name: pallet_scheduler - pallet: - id: pallet_scheduler - name: pallet_scheduler - iterations: 20 - - extrinsic: - id: '*' - name: pallet_balances - pallet: - id: pallet_balances - name: pallet_balances - iterations: 20 - - extrinsic: - id: '*' - name: frame_system - pallet: - id: frame_system - name: frame_system - iterations: 20 - - extrinsic: - id: '*' - name: pallet_session - pallet: - id: pallet_session - name: pallet_session - iterations: 20 - - extrinsic: - id: '*' - name: pallet_tx_pause - pallet: - id: pallet_tx_pause - name: pallet_tx_pause - iterations: 20 - - extrinsic: - id: '*' - name: pallet_multisig - pallet: - id: pallet_multisig - name: pallet_multisig - iterations: 20 - - extrinsic: - id: '*' - name: pallet_timestamp - pallet: - id: pallet_timestamp - name: pallet_timestamp - iterations: 20 - - extrinsic: - id: '*' - name: pallet_preimage - pallet: - id: pallet_preimage - name: pallet_preimage - iterations: 20 - steps: - - uses: actions/download-artifact@v2 + path: ./target/production/manta + - if: always() + name: upload pallets' weights + uses: actions/upload-artifact@v3 with: - name: manta - - run: | - mv manta $HOME/.local/bin/ - chmod +x $HOME/.local/bin/manta - echo "$HOME/.local/bin" >> $GITHUB_PATH - - name: run benchmark - run: | - manta benchmark \ - pallet \ - --chain=manta-local \ - --pallet=${{ matrix.benchmark.pallet.id }} \ - --extrinsic=${{ matrix.benchmark.extrinsic.id }} \ - --execution=Wasm \ - --wasm-execution=Compiled \ - --heap-pages=4096 \ - --repeat=${{ matrix.benchmark.iterations }} \ - --steps=50 \ - --template=.github/resources/frame-weight-template.hbs \ - --output=${{ matrix.benchmark.pallet.name }}.rs - - uses: actions/upload-artifact@v2 + name: weights-output + path: ./scripts/benchmarking/weights-output/ + - if: always() + name: upload benchmarking errors + uses: actions/upload-artifact@v3 + with: + name: benchmarks-errors + path: ./scripts/benchmarking/benchmark_errors.txt + - if: always() + name: upload machine benchmark result + uses: actions/upload-artifact@v3 + with: + name: machine-benchmark + path: ./scripts/benchmarking/machine_benchmark_result.txt + - if: always() + name: upload storage weights + uses: actions/upload-artifact@v3 with: - name: ${{ matrix.benchmark.pallet.id }}-${{ matrix.benchmark.pallet.name }}.rs - path: ${{ github.workspace }}/${{ matrix.benchmark.pallet.name }}.rs + name: rocksdb-weights + path: ./scripts/benchmarking/rocksdb_weights.rs start-node-builder-current: runs-on: ubuntu-20.04 outputs: @@ -169,7 +128,7 @@ jobs: uses: audacious-network/aws-github-runner@v1.0.33 with: mode: start - github-token: ${{ secrets.SELF_HOSTED_RUNNER_TOKEN }} + github-token: ${{ secrets.SELF_HOSTED_RUNNER_TOKEN }} aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-instance-ssh-public-key: ${{ env.AWS_INSTANCE_SSH_PUBLIC_KEY }} @@ -177,16 +136,16 @@ jobs: aws-instance-type: ${{ env.AWS_INSTANCE_TYPE }} aws-instance-root-volume-size: ${{ env.AWS_INSTANCE_ROOT_VOLUME_SIZE }} aws-image-search-pattern: ${{ env.AWS_IMAGE_SEARCH_PATTERN }} - aws-image-search-owners: ${{ env.AWS_IMAGE_SEARCH_OWNERS }} # canonical + aws-image-search-owners: ${{ env.AWS_IMAGE_SEARCH_OWNERS }} stop-node-builder-current: - needs: [start-node-builder-current, run-benchmark] + needs: [run-benchmarks, start-node-builder-current] runs-on: ubuntu-20.04 if: ${{ always() }} steps: - uses: audacious-network/aws-github-runner@v1.0.33 with: mode: stop - github-token: ${{ secrets.SELF_HOSTED_RUNNER_TOKEN }} + github-token: ${{ secrets.SELF_HOSTED_RUNNER_TOKEN }} aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ needs.start-node-builder-current.outputs.aws-region }} diff --git a/.gitignore b/.gitignore index a90070faf..62d7814fb 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,7 @@ runtime/**/target .vscode .idea /tests/node_modules/ +/scripts/benchmarking/benchmarking_errors.txt +/scripts/benchmarking/machine_benchmark_result.txt +/scripts/benchmarking/rocksdb_weights.rs +/scripts/benchmarking/weights-output/ diff --git a/scripts/benchmarking/run_all_benchmarks.sh b/scripts/benchmarking/run_all_benchmarks.sh new file mode 100755 index 000000000..3b241ba5d --- /dev/null +++ b/scripts/benchmarking/run_all_benchmarks.sh @@ -0,0 +1,180 @@ +#!/usr/bin/env bash + +# This file is part of Substrate. +# Copyright (C) 2022 Parity Technologies (UK) Ltd. +# SPDX-License-Identifier: Apache-2.0 +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script has three parts which all use the Substrate runtime: +# - Pallet benchmarking to update the pallet weights +# - Overhead benchmarking for the Extrinsic and Block weights +# - Machine benchmarking +# +# Should be run on a reference machine to gain accurate benchmarks +# current reference machine: https://github.com/paritytech/substrate/pull/5848 +# +# Should be run from the root of the repo. + +while getopts 'bfps:c:v' flag; do + case "${flag}" in + b) + # Skip build. + skip_build='true' + ;; + c) + # Which chain spec to use. + chain_spec="${OPTARG}" + ;; + f) + # Fail if any sub-command in a pipe fails, not just the last one. + set -o pipefail + # Fail on undeclared variables. + set -u + # Fail if any sub-command fails. + set -e + # Fail on traps. + set -E + ;; + p) + # Start at pallet + start_pallet="${OPTARG}" + ;; + s) + # Storage snapshot url + storage_folder="${OPTARG}" + ;; + v) + # Echo all executed commands. + set -x + ;; + *) + # Exit early. + echo "Bad options. Check Script." + exit 1 + ;; + esac +done + + +if [ "$skip_build" != true ] +then + echo "[+] Compiling Manta benchmarks..." + cargo build --profile=production --locked --features=runtime-benchmarks +fi + +# The executable to use. +MANTA=./target/production/manta + +# Manually exclude some pallets. +EXCLUDED_PALLETS=( +) + +# Load all pallet names in an array. +ALL_PALLETS=($( + $MANTA benchmark pallet --list --chain=$chain_spec |\ + tail -n+2 |\ + cut -d',' -f1 |\ + sort |\ + uniq +)) + +# Filter out the excluded pallets by concatenating the arrays and discarding duplicates. +PALLETS=($({ printf '%s\n' "${ALL_PALLETS[@]}" "${EXCLUDED_PALLETS[@]}"; } | sort | uniq -u)) + +echo "[+] Benchmarking ${#PALLETS[@]} Manta pallets by excluding ${#EXCLUDED_PALLETS[@]} from ${#ALL_PALLETS[@]}." + +# Define the error file. +ERR_FILE="scripts/benchmarking/benchmarking_errors.txt" +# Delete the error file before each run. +rm -f $ERR_FILE + +WEIGHTS_OUTPUT="scripts/benchmarking/weights-output" +# Delete the weights output folders before each run. +rm -R ${WEIGHTS_OUTPUT} +# Create the weights output folders. +mkdir ${WEIGHTS_OUTPUT} + +STORAGE_OUTPUT="scripts/benchmarking/rocksdb_weights.rs" +rm -f ${STORAGE_OUTPUT} + +MACHINE_OUTPUT="scripts/benchmarking/machine_benchmark_result.txt" +rm -f $MACHINE_OUTPUT + +# Benchmark each pallet. +for PALLET in "${PALLETS[@]}"; do + # If `-p` is used, skip benchmarks until the start pallet. + if [ ! -z "$start_pallet" ] && [ "$start_pallet" != "$PALLET" ] + then + echo "[+] Skipping ${PALLET}..." + continue + else + unset start_pallet + fi + + FOLDER="$(echo "${PALLET#*_}" | tr '_' '-')"; + WEIGHT_FILE="./${WEIGHTS_OUTPUT}/${PALLET}.rs" + echo "[+] Benchmarking $PALLET with weight file $WEIGHT_FILE"; + + OUTPUT=$( + $MANTA benchmark pallet \ + --chain=$chain_spec \ + --steps=50 \ + --repeat=20 \ + --pallet="$PALLET" \ + --extrinsic="*" \ + --execution=wasm \ + --wasm-execution=compiled \ + --heap-pages=4096 \ + --output="$WEIGHT_FILE" \ + --template=.github/resources/frame-weight-template.hbs 2>&1 + ) + if [ $? -ne 0 ]; then + echo "$OUTPUT" >> "$ERR_FILE" + echo "[-] Failed to benchmark $PALLET. Error written to $ERR_FILE; continuing..." + fi +done + +echo "[+] Benchmarking the machine..." +OUTPUT=$( + $MANTA benchmark machine --chain=$chain_spec --allow-fail 2>&1 +) +# In any case don't write errors to the error file since they're not benchmarking errors. +echo "[x] Machine benchmark:\n$OUTPUT" +echo $OUTPUT >> $MACHINE_OUTPUT + +# If `-s` is used, run the storage benchmark. +if [ ! -z "$storage_folder" ]; then + OUTPUT=$( + $MANTA benchmark storage \ + --chain=$chain_spec \ + --state-version=1 \ + --warmups=10 \ + --base-path=$storage_folder \ + --weight-path=./$STORAGE_OUTPUT 2>&1 + ) + if [ $? -ne 0 ]; then + echo "$OUTPUT" >> "$ERR_FILE" + echo "[-] Failed the storage benchmark. Error written to $ERR_FILE; continuing..." + fi +else + unset storage_folder +fi + +# Check if the error file exists. +if [ -f "$ERR_FILE" ]; then + echo "[-] Some benchmarks failed. See: $ERR_FILE" + exit 1 +else + echo "[+] All benchmarks passed." + exit 0 +fi \ No newline at end of file