Feature/update to polkadot v0 8 29

.github/workflows/check-labels.yml

@@ -0,0 +1,16 @@
+name: Check labels
+
+on:
+  pull_request:
+    types: [labeled, opened, synchronize, unlabeled]
+
+jobs:
+  check-labels:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Check labels
+        run: bash ${{ github.workspace }}/.maintain/github/check_labels.sh
+        env:
+          GITHUB_PR: ${{ github.event.pull_request.number }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -25,3 +25,4 @@ rls*.log
 frame/ddc-metrics-offchain-worker/src/tests/test_data/ddc.wasm
 frame/ddc-metrics-offchain-worker/src/tests/test_data/metadata.json
 .cargo-remote.toml
+*.bin

.gitlab-ci.yml

@@ -39,6 +39,7 @@ variables:                         &default-vars
   CARGO_INCREMENTAL:               0
   DOCKER_OS:                       "debian:stretch"
   ARCH:                            "x86_64"
+  CI_IMAGE:                        "paritytech/ci-linux:production"
   # FIXME set to release
   CARGO_UNLEASH_INSTALL_PARAMS:    "--version 1.0.0-alpha.11"
   CARGO_UNLEASH_PKG_DEF:           "--skip node node-* pallet-template pallet-example pallet-example-* subkey chain-spec-builder"
@@ -60,7 +61,7 @@ default:
   interruptible:                   true
 
 .docker-env:                       &docker-env
-  image:                           paritytech/ci-linux:production
+  image:                           "${CI_IMAGE}"
   before_script:
     - rustup show
     - cargo --version
@@ -84,9 +85,36 @@ default:
     - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
     - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
 
+.test-refs-no-trigger:             &test-refs-no-trigger
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "pipeline"
+      when: never
+    - if: $CI_PIPELINE_SOURCE == "web"
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_COMMIT_REF_NAME == "master"
+    - if: $CI_COMMIT_REF_NAME == "tags"
+    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
+    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
+
 .build-refs:                       &build-refs
   rules:
+    # .publish-refs with manual on PRs
+    - if: $CI_PIPELINE_SOURCE == "pipeline"
+      when: never
+    - if: $CI_PIPELINE_SOURCE == "web"
+    - if: $CI_COMMIT_REF_NAME == "master"
+    - if: $CI_COMMIT_REF_NAME == "tags"
+    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
+    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
+      when: manual
+      allow_failure: true
+
+.publish-refs:                     &publish-refs
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "pipeline"
+      when: never
     - if: $CI_PIPELINE_SOURCE == "web"
+    - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_REF_NAME == "master"
     - if: $CI_COMMIT_REF_NAME == "tags"
     - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
@@ -145,6 +173,8 @@ test-dependency-rules:
   stage:                           check
   image:                           paritytech/tools:latest
   <<:                              *kubernetes-build
+  rules:
+    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
   script:
     - .maintain/ensure-deps.sh
 
@@ -153,36 +183,28 @@ test-prometheus-alerting-rules:
   image:                           paritytech/tools:latest
   <<:                              *kubernetes-build
   rules:
+    - if: $CI_PIPELINE_SOURCE == "pipeline"
+      when: never
     - if: $CI_COMMIT_BRANCH
       changes:
         - .gitlab-ci.yml
         - .maintain/monitoring/**/*
   script:
     - promtool check rules .maintain/monitoring/alerting-rules/alerting-rules.yaml
-    - cat .maintain/monitoring/alerting-rules/alerting-rules.yaml | promtool test rules .maintain/monitoring/alerting-rules/alerting-rule-tests.yaml
+    - cat .maintain/monitoring/alerting-rules/alerting-rules.yaml |
+        promtool test rules .maintain/monitoring/alerting-rules/alerting-rule-tests.yaml
 
 #### stage:                        test
 
-cargo-audit:
-  stage:                           test
-  <<:                              *docker-env
-  rules:
-    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
-      when: never
-    - if: $CI_PIPELINE_SOURCE == "web"
-    - if: $CI_PIPELINE_SOURCE == "schedule"
-    - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
-  script:
-    - cargo audit
-  allow_failure:                   true
-
 cargo-deny:
   stage:                           test
   <<:                              *docker-env
   rules:
-    - if: $CI_COMMIT_MESSAGE =~ /skip-checks/
+    - if: $CI_PIPELINE_SOURCE == "pipeline"
       when: never
+    - changes:
+      - "Cargo.lock"
+      - "**/Cargo.toml"
     - if: $CI_PIPELINE_SOURCE == "web"
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_REF_NAME == "master"
@@ -199,11 +221,13 @@ cargo-deny:
     when:                          always
     paths:
       - deny.log
+  # FIXME: Temorarily allow to fail.
+  allow_failure:                   true
 
 cargo-check-benches:
   stage:                           test
   <<:                              *docker-env
-  <<:                              *test-refs
+  <<:                              *test-refs-no-trigger
   script:
     - SKIP_WASM_BUILD=1 time cargo +nightly check --benches --all
     - cargo run --release -p node-bench -- ::node::import::native::sr25519::transfer_keep_alive::paritydb::small
@@ -260,9 +284,8 @@ unleash-check:
   stage:                           test
   <<:                              *docker-env
   rules:
-    - if: $CI_COMMIT_MESSAGE =~ /skip-checks/
+    - if: $CI_PIPELINE_SOURCE == "pipeline"
       when: never
-      # .test-refs
     - if: $CI_COMMIT_REF_NAME == "master"
     - if: $CI_COMMIT_REF_NAME == "tags"
     - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
@@ -279,7 +302,7 @@ test-frame-examples-compile-to-wasm:
     <<:                            *default-vars
     # Enable debug assertions since we are running optimized builds for testing
     # but still want to have debug assertions.
-    RUSTFLAGS: -Cdebug-assertions=y
+    RUSTFLAGS:                     "-Cdebug-assertions=y"
     RUST_BACKTRACE: 1
   script:
     - cd frame/example-offchain-worker/
@@ -331,7 +354,7 @@ test-full-crypto-feature:
     <<:                            *default-vars
     # Enable debug assertions since we are running optimized builds for testing
     # but still want to have debug assertions.
-    RUSTFLAGS: -Cdebug-assertions=y
+    RUSTFLAGS:                     "-Cdebug-assertions=y"
     RUST_BACKTRACE: 1
   script:
     - cd primitives/core/
@@ -344,7 +367,7 @@ cargo-check-macos:
   stage:                           test
   # shell runner on mac ignores the image set in *docker-env
   <<:                              *docker-env
-  <<:                              *test-refs
+  <<:                              *test-refs-no-trigger
   script:
     - SKIP_WASM_BUILD=1 time cargo check --release
     - sccache -s
@@ -365,7 +388,7 @@ check-polkadot-companion-status:
 check-polkadot-companion-build:
   stage:                           build
   <<:                              *docker-env
-  <<:                              *test-refs
+  <<:                              *test-refs-no-trigger
   needs:
     - job:                         test-linux-stable-int
       artifacts:                   false
@@ -394,15 +417,7 @@ build-linux-substrate:             &build-binary
   stage:                           build
   <<:                              *collect-artifacts
   <<:                              *docker-env
-  rules:
-    # .build-refs with manual on PRs
-    - if: $CI_PIPELINE_SOURCE == "web"
-    - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME == "tags"
-    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
-    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
-      when: manual
-      allow_failure: true
+  <<:                              *build-refs
   needs:
     - job:                         test-linux-stable
       artifacts:                   false
@@ -429,15 +444,7 @@ build-linux-subkey:                &build-subkey
   stage:                           build
   <<:                              *collect-artifacts
   <<:                              *docker-env
-  rules:
-    # .build-refs with manual on PRs
-    - if: $CI_PIPELINE_SOURCE == "web"
-    - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME == "tags"
-    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
-    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
-      when: manual
-      allow_failure: true
+  <<:                              *build-refs
   needs:
     - job:                         cargo-check-subkey
       artifacts:                   false
@@ -461,34 +468,10 @@ build-macos-subkey:
   tags:
     - osx
 
-build-rust-doc:
-  stage:                           build
-  <<:                              *docker-env
-  <<:                              *test-refs
-  needs:
-    - job:                         test-linux-stable
-      artifacts:                   false
-  variables:
-    <<:                            *default-vars
-    RUSTFLAGS:                     -Dwarnings
-  artifacts:
-    name:                          "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc"
-    when:                          on_success
-    expire_in:                     7 days
-    paths:
-    - ./crate-docs/
-  script:
-    - rm -f ./crate-docs/index.html # use it as an indicator if the job succeeds
-    - SKIP_WASM_BUILD=1 RUSTDOCFLAGS="--html-in-header $(pwd)/.maintain/rustdoc-header.html"
-        time cargo +nightly doc --no-deps --workspace --all-features --verbose
-    - mv ./target/doc ./crate-docs
-    - echo "<meta http-equiv=refresh content=0;url=sc_service/index.html>" > ./crate-docs/index.html
-    - sccache -s
-
 #### stage:                        publish
 
 .build-push-docker-image:          &build-push-docker-image
-  <<:                              *build-refs
+  <<:                              *publish-refs
   <<:                              *kubernetes-build
   image:                           quay.io/buildah/stable
   variables:                       &docker-build-vars
@@ -521,17 +504,19 @@ build-rust-doc:
 publish-docker-substrate:
   stage:                           publish
   <<:                              *build-push-docker-image
-  # collect VERSION artifact here to pass it on to kubernetes
-  <<:                              *collect-artifacts
   needs:
     - job:                         build-linux-substrate
       artifacts:                   true
   variables:
     <<:                            *docker-build-vars
     PRODUCT:                       substrate
   after_script:
-    # only VERSION information is needed for the deployment
-    - find ./artifacts/ -depth -not -name VERSION -type f -delete
+    - echo "VERSION=${VERSION}" >> build.env
+  artifacts:
+    reports:
+      # this artifact is used in trigger-simnet job
+      # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance
+      dotenv: artifacts/substrate/build.env
 
 publish-docker-subkey:
   stage:                           publish
@@ -545,7 +530,7 @@ publish-docker-subkey:
 
 publish-s3-release:
   stage:                           publish
-  <<:                              *build-refs
+  <<:                              *publish-refs
   <<:                              *kubernetes-build
   needs:
     - job:                         build-linux-substrate
@@ -565,32 +550,6 @@ publish-s3-release:
     - aws s3 ls s3://${BUCKET}/${PREFIX}/latest/
         --recursive --human-readable --summarize
 
-publish-s3-doc:
-  stage:                           publish
-  image:                           paritytech/awscli:latest
-  allow_failure:                   true
-  needs:
-    - job:                         build-rust-doc
-      artifacts:                   true
-    - job:                         build-linux-substrate
-      artifacts:                   false
-  <<:                              *build-refs
-  <<:                              *kubernetes-build
-  variables:
-    GIT_STRATEGY:                  none
-    BUCKET:                        "releases.parity.io"
-    PREFIX:                        "substrate-rustdoc"
-  script:
-    - test -r ./crate-docs/index.html || (
-        echo "./crate-docs/index.html not present, build:rust:doc:release job not complete";
-        exit 1
-      )
-    - aws s3 sync --delete --size-only --only-show-errors
-        ./crate-docs/ s3://${BUCKET}/${PREFIX}/
-  after_script:
-    - aws s3 ls s3://${BUCKET}/${PREFIX}/
-        --human-readable --summarize
-
 publish-draft-release:
   stage:                           publish
   image:                           paritytech/tools:latest
@@ -635,6 +594,8 @@ deploy-prometheus-alerting-rules:
     - kubectl -n ${NAMESPACE} patch prometheusrule ${PROMETHEUSRULE}
         --type=merge --patch "$(sed 's/^/  /;1s/^/spec:\n/' ${RULES})"
   rules:
+    - if: $CI_PIPELINE_SOURCE == "pipeline"
+      when: never
     - if: $CI_COMMIT_REF_NAME == "master"
       changes:
         - .gitlab-ci.yml
@@ -647,19 +608,7 @@ trigger-simnet:
     - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "nightly"
   needs:
     - job:                         publish-docker-substrate
-      artifacts:                   false
   trigger:
     project:                       parity/simnet
     branch:                        master
     strategy:                      depend
-
-#### stage:                        .post
-
-check-labels:
-  stage:                           .post
-  image:                           paritytech/tools:latest
-  <<:                              *kubernetes-build
-  rules:
-    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
-  script:
-    - ./.maintain/gitlab/check_labels.sh

.maintain/gitlab/lib.sh → .maintain/common/lib.sh

@@ -66,11 +66,17 @@ has_label(){
   repo="$1"
   pr_id="$2"
   label="$3"
+
+  # These will exist if the function is called in Gitlab.
+  # If the function's called in Github, we should have GITHUB_ACCESS_TOKEN set
+  # already.
   if [ -n "$GITHUB_RELEASE_TOKEN" ]; then
-    out=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$api_base/$repo/pulls/$pr_id")
-  else
-    out=$(curl -H "Authorization: token $GITHUB_PR_TOKEN" -s "$api_base/$repo/pulls/$pr_id")
+    GITHUB_TOKEN="$GITHUB_RELEASE_TOKEN"
+  elif [ -n "$GITHUB_PR_TOKEN" ]; then
+    GITHUB_TOKEN="$GITHUB_PR_TOKEN"
   fi
+
+  out=$(curl -H "Authorization: token $GITHUB_TOKEN" -s "$api_base/$repo/pulls/$pr_id")
   [ -n "$(echo "$out" | tr -d '\r\n' | jq ".labels | .[] | select(.name==\"$label\")")" ]
 }