Merge branch 'master' into andre/report-grandpa-equivocations

.dockerignore

@@ -3,3 +3,4 @@ doc
 .idea/
 Dockerfile
 .dockerignore
+.local

.editorconfig

@@ -6,7 +6,7 @@ tab_width=4
 end_of_line=lf
 charset=utf-8
 trim_trailing_whitespace=true
-max_line_length=120
+max_line_length=100
 insert_final_newline=true
 
 [*.yml]

.gitignore

@@ -21,3 +21,4 @@ rls*.log
 *.orig
 *.rej
 **/wip/*.stderr
+.local

.gitlab-ci.yml

@@ -49,6 +49,12 @@ variables:
     - schedules
     - web
     - /^[0-9]+$/                   # PRs
+  retry:
+    max: 2
+    when:
+      - runner_system_failure
+      - unknown_failure
+      - api_failure
   tags:
     - linux-docker
 
@@ -72,7 +78,7 @@ check-runtime:
     GITLAB_API:                    "https://gitlab.parity.io/api/v4"
     GITHUB_API_PROJECT:            "parity%2Finfrastructure%2Fgithub-api"
   script:
-    - ./scripts/gitlab/check_runtime.sh
+    - ./.maintain/gitlab/check_runtime.sh
   allow_failure:                   true
 
 
@@ -83,7 +89,7 @@ check-line-width:
   only:
     - /^[0-9]+$/
   script:
-    - ./scripts/gitlab/check_line_width.sh
+    - ./.maintain/gitlab/check_line_width.sh
   allow_failure:                   true
 
 
@@ -111,7 +117,7 @@ cargo-check-subkey:
   except:
     - /^v[0-9]+\.[0-9]+.*$/        # i.e. v1.0, v2.1rc1
   script:
-    - cd ./subkey
+    - cd ./bin/subkey
     - BUILD_DUMMY_WASM_BINARY=1 time cargo check --release
     - sccache -s
 
@@ -127,7 +133,7 @@ test-linux-stable:                 &test-linux
     variables:
       - $DEPLOY_TAG
   script:
-    - time cargo test --all --release --verbose --locked |
+    - WASM_BUILD_NO_COLOR=1 time cargo test --all --release --verbose --locked |
         tee output.log
     - sccache -s
   after_script:
@@ -139,8 +145,16 @@ test-linux-stable:                 &test-linux
     paths:
       - ${CI_COMMIT_SHORT_SHA}_warnings.log
 
+test-dependency-rules:             &test-linux
+  stage:                           test
+  <<:                              *docker-env
+  except:
+    variables:
+      - $DEPLOY_TAG
+  script:
+    - .maintain/ensure-deps.sh
 
-test-srml-staking:                 &test-srml-staking
+test-frame-staking:                 &test-frame-staking
   stage:                           test
   <<:                              *docker-env
   variables:
@@ -152,14 +166,25 @@ test-srml-staking:                 &test-srml-staking
     variables:
       - $DEPLOY_TAG
   script:
-    - cd srml/staking/
-    - time cargo test --release --verbose --no-default-features --features std
+    - cd frame/staking/
+    - WASM_BUILD_NO_COLOR=1 time cargo test --release --verbose --no-default-features --features std
     - sccache -s
 
-
-
-
-
+test-wasmtime:                 &test-wasmtime
+  stage:                           test
+  <<:                              *docker-env
+  variables:
+    # Enable debug assertions since we are running optimized builds for testing
+    # but still want to have debug assertions.
+    RUSTFLAGS: -Cdebug-assertions=y
+    RUST_BACKTRACE: 1
+  except:
+    variables:
+      - $DEPLOY_TAG
+  script:
+    - cd client/executor
+    - WASM_BUILD_NO_COLOR=1 time cargo test --release --verbose --features wasmtime
+    - sccache -s
 
 test-linux-stable-int:
   <<:                              *test-linux
@@ -171,7 +196,7 @@ test-linux-stable-int:
   script:
     - echo "___Logs will be partly shown at the end in case of failure.___"
     - echo "___Full log will be saved to the job artifacts only in case of failure.___"
-    - RUST_LOG=sync=trace,consensus=trace,client=trace,state-db=trace,db=trace,forks=trace,state_db=trace,storage_cache=trace
+    - WASM_BUILD_NO_COLOR=1 RUST_LOG=sync=trace,consensus=trace,client=trace,state-db=trace,db=trace,forks=trace,state_db=trace,storage_cache=trace
         time cargo test -p node-cli --release --verbose --locked -- --ignored --test-threads=1
         &> ${CI_COMMIT_SHORT_SHA}_int_failure.log
     - sccache -s
@@ -193,27 +218,16 @@ check-web-wasm:
   script:
     # WASM support is in progress. As more and more crates support WASM, we
     # should add entries here. See https://github.com/paritytech/substrate/issues/2416
-    - time cargo web build -p sr-io
-    - time cargo web build -p sr-primitives
-    - time cargo web build -p sr-std
-    - time cargo web build -p substrate-client
-    - time cargo web build -p substrate-consensus-aura
-    - time cargo web build -p substrate-consensus-babe
-    - time cargo web build -p substrate-consensus-common
-    - time cargo web build -p substrate-keyring
-    - time cargo web build -p substrate-keystore
-    - time cargo web build -p substrate-executor
-    - time cargo web build -p substrate-network
-    - time cargo web build -p substrate-panic-handler
-    - time cargo web build -p substrate-peerset
-    - time cargo web build -p substrate-primitives
-    # TODO: we can't use cargo web until https://github.com/paritytech/jsonrpc/pull/436 is deployed
-    - time cargo build -p substrate-rpc-servers --target wasm32-unknown-unknown
-    - time cargo web build -p substrate-serializer
-    - time cargo web build -p substrate-state-db
-    - time cargo web build -p substrate-state-machine
-    - time cargo web build -p substrate-telemetry
-    - time cargo web build -p substrate-trie
+    - time cargo build --target=wasm32-unknown-unknown -p sr-io
+    - time cargo build --target=wasm32-unknown-unknown -p sr-primitives
+    - time cargo build --target=wasm32-unknown-unknown -p sr-std
+    - time cargo build --target=wasm32-unknown-unknown -p substrate-client
+    - time cargo build --target=wasm32-unknown-unknown -p substrate-consensus-aura
+    - time cargo build --target=wasm32-unknown-unknown -p substrate-consensus-babe
+    - time cargo build --target=wasm32-unknown-unknown -p substrate-consensus-common
+    - time cargo build --target=wasm32-unknown-unknown -p substrate-telemetry
+    # Note: the command below is a bit weird because several Cargo issues prevent us from compiling the node in a more straight-forward way.
+    - time cargo build --manifest-path=bin/node/cli/Cargo.toml --no-default-features --features "browser" --target=wasm32-unknown-unknown
     - sccache -s
 
 node-exits:
@@ -222,7 +236,27 @@ node-exits:
   except:
     - /^v[0-9]+\.[0-9]+.*$/        # i.e. v1.0, v2.1rc1
   script:
-    - ./ci/check_for_exit.sh
+    - ./.maintain/check_for_exit.sh
+
+
+test-full-crypto-feature:          &test-full-crypto-feature
+  stage:                           test
+  <<:                              *docker-env
+  variables:
+    # Enable debug assertions since we are running optimized builds for testing
+    # but still want to have debug assertions.
+    RUSTFLAGS: -Cdebug-assertions=y
+    RUST_BACKTRACE: 1
+  except:
+    variables:
+      - $DEPLOY_TAG
+  script:
+    - cd primitives/core/
+    - time cargo +nightly build --verbose --no-default-features --features full_crypto
+    - cd ../application-crypto
+    - time cargo +nightly build --verbose --no-default-features --features full_crypto
+    - sccache -s
+
 
 #### stage:                        build
 
@@ -235,7 +269,7 @@ build-linux-substrate:
     variables:
       - $DEPLOY_TAG
   script:
-    - time cargo build --release --verbose
+    - WASM_BUILD_NO_COLOR=1 time cargo build --release --verbose
     - mkdir -p ./artifacts/substrate/
     - mv ./target/release/substrate ./artifacts/substrate/.
     - echo -n "Substrate version = "
@@ -248,8 +282,8 @@ build-linux-substrate:
       fi
     - sha256sum ./artifacts/substrate/substrate | tee ./artifacts/substrate/substrate.sha256
     - printf '\n# building node-template\n\n'
-    - ./scripts/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz
-    - cp -r scripts/docker/substrate.Dockerfile ./artifacts/substrate/
+    - ./.maintain/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz
+    - cp -r .maintain/docker/substrate.Dockerfile ./artifacts/substrate/
     - sccache -s
 
 build-linux-subkey:
@@ -261,9 +295,9 @@ build-linux-subkey:
     variables:
       - $DEPLOY_TAG
   script:
-    - cd ./subkey
+    - cd ./bin/subkey
     - BUILD_DUMMY_WASM_BINARY=1 time cargo build --release --verbose
-    - cd ..
+    - cd -
     - sccache -s
     - mkdir -p ./artifacts/subkey
     - mv ./target/release/subkey ./artifacts/subkey/.
@@ -272,7 +306,7 @@ build-linux-subkey:
         sed -n -r 's/^subkey ([0-9.]+.*)/\1/p' |
           tee ./artifacts/subkey/VERSION;
     - sha256sum ./artifacts/subkey/subkey | tee ./artifacts/subkey/subkey.sha256
-    - cp -r scripts/docker/subkey.Dockerfile ./artifacts/subkey/
+    - cp -r .maintain/docker/subkey.Dockerfile ./artifacts/subkey/
     - sccache -s
 
 build-rust-doc-release:
@@ -288,7 +322,7 @@ build-rust-doc-release:
   <<:                              *build-only
   script:
     - rm -f ./crate-docs/index.html # use it as an indicator if the job succeeds
-    - BUILD_DUMMY_WASM_BINARY=1 RUSTDOCFLAGS="--html-in-header $(pwd)/rustdoc-header.html" time cargo +nightly doc --release --all --verbose
+    - BUILD_DUMMY_WASM_BINARY=1 RUSTDOCFLAGS="--html-in-header $(pwd)/.maintain/rustdoc-header.html" time cargo +nightly doc --release --all --verbose
     - cp -R ./target/doc ./crate-docs
     - echo "<meta http-equiv=refresh content=0;url=substrate_service/index.html>" > ./crate-docs/index.html
     - sccache -s
@@ -312,6 +346,35 @@ check_warnings:
       fi
   allow_failure:                   true
 
+# Check whether Polkadot 'master' branch builds using this Substrate commit.
+check_polkadot:
+  stage:                           build
+  <<:                              *docker-env
+  allow_failure:                   true
+  dependencies:
+    - test-linux-stable
+  script:
+    - COMMIT_HASH=$(git rev-parse HEAD)
+    - SUBSTRATE_PATH=$(pwd)
+    # Clone the current Polkadot master branch into ./polkadot.
+    - git clone --depth 1 https://gitlab.parity.io/parity/polkadot.git
+    - cd polkadot
+    # Within Polkadot 'master' alter each Cargo.toml that references the
+    # Substrate 'polkadot-master' branch:
+    # 1. Replace the 'branch = "polkadot-master"' statements with the rev of our
+    #    commit.
+    # 2. Replace 'git = "https://.*"' with 'git = "file://.*"' (the local
+    #    checked out Substrate repository one folder above).
+    # 3. Remove any trailing commas.
+    - git grep -l "polkadot-master" | grep toml | xargs sed -i "s/branch.*=.*\"polkadot-master\"/rev = \"$COMMIT_HASH\"/; s~https://github.com/paritytech/substrate~file://$SUBSTRATE_PATH~; s/,\s*}/ }/"
+    # Make sure 'Cargo.lock' matches 'Cargo.toml'. It's enough to update one
+    # package, others are updated along the way.
+    - cargo update -p sr-io
+    # Check whether Polkadot 'master' branch builds with this Substrate commit.
+    - time cargo check
+    - cd -
+    - sccache -s
+
 #### stage:                        publish
 
 .publish-docker-release:           &publish-docker-release
@@ -478,10 +541,10 @@ publish-gh-doc:
     - echo "Substrate version = ${DEPLOY_TAG}"
     # or use helm to render the template
     - helm template
-      --values ./scripts/kubernetes/values.yaml
+      --values ./.maintain/kubernetes/values.yaml
       --set image.tag=${DEPLOY_TAG}
       --set validator.keys=${VALIDATOR_KEYS}
-      ./scripts/kubernetes | kubectl apply -f - --dry-run=false
+      ./.maintain/kubernetes | kubectl apply -f - --dry-run=false
     - echo "# substrate namespace ${KUBE_NAMESPACE}"
     - kubectl -n ${KUBE_NAMESPACE} get all
     - echo "# substrate's nodes' external ip addresses:"
@@ -543,16 +606,16 @@ deploy-ue1-tag:
 validator 1 4:
   <<:                              *validator-deploy
   script:
-    - ./scripts/flamingfir-deploy.sh flamingfir-validator1
+    - ./.maintain/flamingfir-deploy.sh flamingfir-validator1
 validator 2 4:
   <<:                              *validator-deploy
   script:
-    - ./scripts/flamingfir-deploy.sh flamingfir-validator2
+    - ./.maintain/flamingfir-deploy.sh flamingfir-validator2
 validator 3 4:
   <<:                              *validator-deploy
   script:
-    - ./scripts/flamingfir-deploy.sh flamingfir-validator3
+    - ./.maintain/flamingfir-deploy.sh flamingfir-validator3
 validator 4 4:
   <<:                              *validator-deploy
   script:
-    - ./scripts/flamingfir-deploy.sh flamingfir-validator4
+    - ./.maintain/flamingfir-deploy.sh flamingfir-validator4

Dockerfile → .maintain/Dockerfile

@@ -20,7 +20,6 @@ RUN curl https://sh.rustup.rs -sSf | sh -s -- -y && \
 	export PATH="$PATH:$HOME/.cargo/bin" && \
 	rustup toolchain install nightly && \
 	rustup target add wasm32-unknown-unknown --toolchain nightly && \
-	cargo install --git https://github.com/alexcrichton/wasm-gc && \
 	rustup default nightly && \
 	rustup default stable && \
 	cargo build "--$PROFILE"

scripts/docker/subkey.Dockerfile → .maintain/docker/subkey.Dockerfile

@@ -8,7 +8,7 @@ LABEL io.parity.image.authors="[email protected]" \
 	io.parity.image.vendor="Parity Technologies" \
 	io.parity.image.title="parity/subkey" \
 	io.parity.image.description="Subkey: key generating utility for Substrate." \
-	io.parity.image.source="https://github.com/paritytech/substrate/blob/${VCS_REF}/scripts/docker/subkey.Dockerfile" \
+	io.parity.image.source="https://github.com/paritytech/substrate/blob/${VCS_REF}/.maintain/docker/subkey.Dockerfile" \
 	io.parity.image.revision="${VCS_REF}" \
 	io.parity.image.created="${BUILD_DATE}" \
 	io.parity.image.documentation="https://github.com/paritytech/substrate/tree/${VCS_REF}/subkey"

scripts/docker/substrate.Dockerfile → .maintain/docker/substrate.Dockerfile

@@ -8,7 +8,7 @@ LABEL io.parity.image.authors="[email protected]" \
 	io.parity.image.vendor="Parity Technologies" \
 	io.parity.image.title="parity/substrate" \
 	io.parity.image.description="Substrate: The platform for blockchain innovators." \
-	io.parity.image.source="https://github.com/paritytech/substrate/blob/${VCS_REF}/scripts/docker/Dockerfile" \
+	io.parity.image.source="https://github.com/paritytech/substrate/blob/${VCS_REF}/.maintain/docker/Dockerfile" \
 	io.parity.image.revision="${VCS_REF}" \
 	io.parity.image.created="${BUILD_DATE}" \
 	io.parity.image.documentation="https://wiki.parity.io/Parity-Substrate"