[ci] Add short benchmarks to the pipeline

.gitlab-ci.yml

@@ -577,20 +577,6 @@ generate-impl-guide:
   script:
     - mdbook build roadmap/implementers-guide
 
-check-runtime-benchmarks:
-  stage:                           stage3
-  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
-  needs:
-    - job:                         test-node-metrics
-      artifacts:                   false
-  <<:                              *test-refs
-  <<:                              *docker-env
-  <<:                              *compiler-info
-  script:
-    # Check that everything compiles with `runtime-benchmarks` feature flag.
-    - cargo check --features runtime-benchmarks --all
-    - sccache -s
-
 check-try-runtime:
   stage:                           stage3
   # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
@@ -620,6 +606,19 @@ check-no-default-features:
     - pushd cli && cargo check --no-default-features --features "service" && popd
     - sccache -s
 
+build-short-benchmark:
+  stage:                           stage3
+  <<:                              *test-refs
+  <<:                              *docker-env
+  <<:                              *collect-artifacts
+  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
+  needs:
+    - job:                         test-node-metrics
+      artifacts:                   false
+  script:
+    - cargo +nightly build --profile release --locked --features=runtime-benchmarks
+    - mkdir artifacts
+    - cp ./target/release/polkadot ./artifacts/
 
 deploy-parity-testnet:
   stage:                           stage3
@@ -804,6 +803,30 @@ publish-rustdoc:
   after_script:
     - rm -rf .git/ ./*
 
+# Run all pallet benchmarks only once to check if there are any errors
+short-benchmark-polkadot:          &short-bench
+  stage:                           stage4
+  <<:                              *test-refs
+  <<:                              *docker-env
+  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
+  needs:
+    - job:                         build-short-benchmark
+      artifacts:                   true
+  variables:
+    RUNTIME:                       polkadot
+  script:
+    - ./scripts/run_short_benches_for_runtime.sh $RUNTIME
+
+short-benchmark-kusama:
+  <<:                              *short-bench
+  variables:
+    RUNTIME:                       kusama
+
+short-benchmark-westend:
+  <<:                              *short-bench
+  variables:
+    RUNTIME:                       westend
+
 #### stage:                        .post
 
 # This job cancels the whole pipeline if any of provided jobs fail.
@@ -814,8 +837,6 @@ cancel-pipeline:
   needs:
     - job:                         test-linux-stable
       artifacts:                   false
-    - job:                         check-runtime-benchmarks
-      artifacts:                   false
     - job:                         check-try-runtime
       artifacts:                   false
   rules:
@@ -825,3 +846,5 @@ cancel-pipeline:
     PROJECT_ID:                    "${CI_PROJECT_ID}"
     PIPELINE_ID:                   "${CI_PIPELINE_ID}"
   trigger:                         "parity/infrastructure/ci_cd/pipeline-stopper"
+
+

scripts/run_short_benches_for_runtime.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -e
+
+# Runs all benchmarks for all pallets, for a given runtime, provided by $1
+# Should be run on a reference machine to gain accurate benchmarks
+# current reference machine: https://github.com/paritytech/substrate/pull/5848
+
+runtime="$1"
+
+echo "[+] Running all benchmarks for $runtime"
+
+./artifacts/polkadot benchmark \
+    --chain "${runtime}-dev" \
+    --list |\
+  tail -n+2 |\
+  cut -d',' -f1 |\
+  uniq | \
+  grep -v frame_system > "${runtime}_pallets"
+
+# For each pallet found in the previous command, run benches on each function
+while read -r line; do
+  pallet="$(echo "$line" | cut -d' ' -f1)";
+  echo "Runtime: $runtime. Pallet: $pallet";
+  # '!' has the side effect of bypassing errexit / set -e
+  ! ./artifacts/polkadot benchmark \
+    --chain="${runtime}-dev" \
+    --steps=1 \
+    --repeat=1 \
+    --pallet="$pallet" \
+    --extrinsic="*" \
+    --execution=wasm \
+    --wasm-execution=compiled \
+    --heap-pages=4096 \
+    --header=./file_header.txt \
+    --output="./runtime/${runtime}/src/weights/${pallet/::/_}.rs"
+done < "${runtime}_pallets"
+rm "${runtime}_pallets"