diff --git a/.editorconfig b/.editorconfig index f511aad46079..33db021c8a47 100644 --- a/.editorconfig +++ b/.editorconfig @@ -14,3 +14,9 @@ indent_style=space indent_size=2 tab_width=8 end_of_line=lf + +[*.sh] +indent_style=space +indent_size=2 +tab_width=2 +end_of_line=lf diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index feb65ace7972..9b6162208882 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -110,17 +110,10 @@ test-deterministic-wasm: <<: *docker-env except: script: - # build runtime - - WASM_BUILD_NO_COLOR=1 cargo build --verbose --release -p kusama-runtime -p polkadot-runtime -p westend-runtime - # make checksum - - sha256sum target/release/wbuild/target/wasm32-unknown-unknown/release/*.wasm > checksum.sha256 - # clean up – FIXME: can we reuse some of the artifacts? - - cargo clean - # build again - - WASM_BUILD_NO_COLOR=1 cargo build --verbose --release -p kusama-runtime -p polkadot-runtime -p westend-runtime - # confirm checksum - - sha256sum -c checksum.sha256 - + - ./scripts/gitlab/test_deterministic_wasm.sh + - sccache -s + after_script: + - cd substrate && git rev-parse --abbrev-ref HEAD test-linux-stable: &test stage: test @@ -134,8 +127,10 @@ test-linux-stable: &test RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" TARGET: native script: - - time cargo test --all --release --verbose --locked --features runtime-benchmarks + - ./scripts/gitlab/test_linux_stable.sh - sccache -s + after_script: + - cd substrate && git rev-parse --abbrev-ref HEAD check-web-wasm: &test stage: test @@ -143,17 +138,10 @@ check-web-wasm: &test <<: *docker-env <<: *compiler_info script: - # WASM support is in progress. As more and more crates support WASM, we - # should add entries here. See https://github.com/paritytech/polkadot/issues/625 - - time cargo build --locked --target=wasm32-unknown-unknown --manifest-path runtime/polkadot/Cargo.toml - - time cargo build --locked --target=wasm32-unknown-unknown --manifest-path runtime/kusama/Cargo.toml - - time cargo build --locked --target=wasm32-unknown-unknown --manifest-path erasure-coding/Cargo.toml - - time cargo build --locked --target=wasm32-unknown-unknown --manifest-path parachain/Cargo.toml - - time cargo build --locked --target=wasm32-unknown-unknown --manifest-path primitives/Cargo.toml - - time cargo build --locked --target=wasm32-unknown-unknown --manifest-path rpc/Cargo.toml - - time cargo build --locked --target=wasm32-unknown-unknown --manifest-path statement-table/Cargo.toml - - time cargo build --locked --target=wasm32-unknown-unknown --manifest-path cli/Cargo.toml --no-default-features --features browser + - ./scripts/gitlab/check_web_wasm.sh - sccache -s + after_script: + - cd substrate && git rev-parse --abbrev-ref HEAD check-runtime-benchmarks: &test stage: test @@ -161,9 +149,10 @@ check-runtime-benchmarks: &test <<: *docker-env <<: *compiler_info script: - # Check that the node will compile with `runtime-benchmarks` feature flag. - - time cargo check --features runtime-benchmarks + - ./scripts/gitlab/check_runtime_benchmarks.sh - sccache -s + after_script: + - cd substrate && git rev-parse --abbrev-ref HEAD build-wasm-release: stage: build diff --git a/scripts/gitlab/check_runtime.sh b/scripts/gitlab/check_runtime.sh index 30fdcd455c1d..0c635c88246c 100755 --- a/scripts/gitlab/check_runtime.sh +++ b/scripts/gitlab/check_runtime.sh @@ -34,13 +34,13 @@ boldprint "make sure the master branch is available in shallow clones" git fetch --depth="${GIT_DEPTH:-100}" origin master runtimes=( - "kusama" - "polkadot" - "westend" + "kusama" + "polkadot" + "westend" ) common_dirs=( - "common" + "common" ) # Helper function to join elements in an array with a multi-char delimiter @@ -52,176 +52,178 @@ runtime_regex="^runtime/$(join_by '|^runtime/' "${runtimes[@]}" "${common_dirs[@ boldprint "check if the wasm sources changed since ${LATEST_TAG}" if ! git diff --name-only "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" \ - | grep -E -q -e "$runtime_regex" + | grep -E -q -e "$runtime_regex" then - boldprint "no changes to any runtime source code detected" - # continue checking if Cargo.lock was updated with a new substrate reference - # and if that change includes a {spec|impl}_version update. - - SUBSTRATE_REFS_CHANGED="$( - git diff "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" Cargo.lock \ - | sed -n -r "s~^[\+\-]source = \"${SUBSTRATE_REPO_CARGO}#([a-f0-9]+)\".*$~\1~p" | sort -u | wc -l - )" - - # check Cargo.lock for substrate ref change - case "${SUBSTRATE_REFS_CHANGED}" in - (0) - boldprint "substrate refs not changed in Cargo.lock" - exit 0 - ;; - (2) - boldprint "substrate refs updated since ${LATEST_TAG}" - ;; - (*) - boldprint "check unsupported: more than one commit targeted in repo ${SUBSTRATE_REPO_CARGO}" - exit 1 - esac - - - SUBSTRATE_PREV_REF="$( - git diff "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" Cargo.lock \ - | sed -n -r "s~^\-source = \"${SUBSTRATE_REPO_CARGO}#([a-f0-9]+)\".*$~\1~p" | sort -u | head -n 1 - )" - - SUBSTRATE_NEW_REF="$( - git diff "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" Cargo.lock \ - | sed -n -r "s~^\+source = \"${SUBSTRATE_REPO_CARGO}#([a-f0-9]+)\".*$~\1~p" | sort -u | head -n 1 - )" - - - boldcat <<-EOT - previous substrate commit id ${SUBSTRATE_PREV_REF} - new substrate commit id ${SUBSTRATE_NEW_REF} - EOT - - # okay so now need to fetch the substrate repository and check whether spec_version or impl_version has changed there - SUBSTRATE_CLONE_DIR="$(mktemp -t -d substrate-XXXXXX)" - trap 'rm -rf "${SUBSTRATE_CLONE_DIR}"' INT QUIT TERM ABRT EXIT - - git clone --depth="${GIT_DEPTH:-100}" --no-tags \ - "${SUBSTRATE_REPO}" "${SUBSTRATE_CLONE_DIR}" - - # check if there are changes to the spec|impl versions - git -C "${SUBSTRATE_CLONE_DIR}" diff \ - "${SUBSTRATE_PREV_REF}..${SUBSTRATE_NEW_REF}" "${SUBSTRATE_VERSIONS_FILE}" \ - | grep -E '^[\+\-][[:space:]]+(spec|impl)_version: +([0-9]+),$' || exit 0 - - boldcat <<-EOT - spec_version or or impl_version have changed in substrate after updating Cargo.lock - please make sure versions are bumped in polkadot accordingly - EOT - - # Now check if any of the substrate changes have been tagged D2-breaksapi - ( - cd "${SUBSTRATE_CLONE_DIR}" - substrate_changes="$(sanitised_git_logs "${SUBSTRATE_PREV_REF}" "${SUBSTRATE_NEW_REF}")" - echo "$substrate_changes" | while read -r line; do - pr_id=$(echo "$line" | sed -E 's/.*#([0-9]+)\)$/\1/') - - if has_label 'paritytech/substrate' "$pr_id" 'D2-breaksapi'; then - boldprint "Substrate change labelled with D2-breaksapi. Labelling..." - github_label "D2-breaksapi" - exit 1 - fi - done - ) + boldprint "no changes to any runtime source code detected" + # continue checking if Cargo.lock was updated with a new substrate reference + # and if that change includes a {spec|impl}_version update. + + SUBSTRATE_REFS_CHANGED="$( + git diff "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" Cargo.lock \ + | sed -n -r "s~^[\+\-]source = \"${SUBSTRATE_REPO_CARGO}#([a-f0-9]+)\".*$~\1~p" | sort -u | wc -l + )" + + # check Cargo.lock for substrate ref change + case "${SUBSTRATE_REFS_CHANGED}" in + (0) + boldprint "substrate refs not changed in Cargo.lock" + exit 0 + ;; + (2) + boldprint "substrate refs updated since ${LATEST_TAG}" + ;; + (*) + boldprint "check unsupported: more than one commit targeted in repo ${SUBSTRATE_REPO_CARGO}" + exit 1 + esac + + + SUBSTRATE_PREV_REF="$( + git diff "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" Cargo.lock \ + | sed -n -r "s~^\-source = \"${SUBSTRATE_REPO_CARGO}#([a-f0-9]+)\".*$~\1~p" | sort -u | head -n 1 + )" + + SUBSTRATE_NEW_REF="$( + git diff "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" Cargo.lock \ + | sed -n -r "s~^\+source = \"${SUBSTRATE_REPO_CARGO}#([a-f0-9]+)\".*$~\1~p" | sort -u | head -n 1 + )" + + + boldcat < ${add_spec_version} +changes to the ${RUNTIME} runtime sources and changes in the spec version. - EOT - continue +spec_version: ${sub_spec_version} -> ${add_spec_version} - else - # check for impl_version updates: if only the impl versions changed, we assume - # there is no consensus-critical logic that has changed. +EOT + continue - add_impl_version="$( - git diff refs/tags/"${LATEST_TAG}...${CI_COMMIT_SHA}" "runtime/${RUNTIME}/src/lib.rs" \ - | sed -n -r 's/^\+[[:space:]]+impl_version: +([0-9]+),$/\1/p' - )" - sub_impl_version="$( - git diff refs/tags/"${LATEST_TAG}...${CI_COMMIT_SHA}" "runtime/${RUNTIME}/src/lib.rs" \ - | sed -n -r 's/^\-[[:space:]]+impl_version: +([0-9]+),$/\1/p' - )" + else + # check for impl_version updates: if only the impl versions changed, we assume + # there is no consensus-critical logic that has changed. + add_impl_version="$( + git diff refs/tags/"${LATEST_TAG}...${CI_COMMIT_SHA}" "runtime/${RUNTIME}/src/lib.rs" \ + | sed -n -r 's/^\+[[:space:]]+impl_version: +([0-9]+),$/\1/p' + )" + sub_impl_version="$( + git diff refs/tags/"${LATEST_TAG}...${CI_COMMIT_SHA}" "runtime/${RUNTIME}/src/lib.rs" \ + | sed -n -r 's/^\-[[:space:]]+impl_version: +([0-9]+),$/\1/p' + )" - # see if the impl version changed - if [ "${add_impl_version}" != "${sub_impl_version}" ] - then - boldcat <<-EOT - ## RUNTIME: ${RUNTIME} ## + # see if the impl version changed + if [ "${add_impl_version}" != "${sub_impl_version}" ] + then + boldcat < ${add_impl_version} +changes to the ${RUNTIME} runtime sources and changes in the impl version. - EOT - continue - fi +impl_version: ${sub_impl_version} -> ${add_impl_version} +EOT + continue + fi - boldcat <<-EOT - wasm source files changed or the spec version in the substrate reference in - the Cargo.lock but not the spec/impl version. If changes made do not alter - logic, just bump 'impl_version'. If they do change logic, bump - 'spec_version'. + failed_runtime_checks+=("$RUNTIME") + fi +done - source file directories: - - runtime +if [ ${#failed_runtime_checks} -gt 0 ]; then + boldcat </dev/null 2>&1 && pwd )/lib.sh" + +pull_companion_substrate + +set -e + +# Check that the node will compile with `runtime-benchmarks` feature flag. +time cargo check --features runtime-benchmarks diff --git a/scripts/gitlab/check_web_wasm.sh b/scripts/gitlab/check_web_wasm.sh new file mode 100755 index 000000000000..4f0bfcc1db15 --- /dev/null +++ b/scripts/gitlab/check_web_wasm.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +#shellcheck source=lib.sh +source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/lib.sh" + +pull_companion_substrate + +set -e + +# WASM support is in progress. As more and more crates support WASM, we +# should add entries here. See https://github.com/paritytech/polkadot/issues/625 +time cargo build --locked --target=wasm32-unknown-unknown --manifest-path runtime/polkadot/Cargo.toml +time cargo build --locked --target=wasm32-unknown-unknown --manifest-path runtime/kusama/Cargo.toml +time cargo build --locked --target=wasm32-unknown-unknown --manifest-path erasure-coding/Cargo.toml +time cargo build --locked --target=wasm32-unknown-unknown --manifest-path parachain/Cargo.toml +time cargo build --locked --target=wasm32-unknown-unknown --manifest-path primitives/Cargo.toml +time cargo build --locked --target=wasm32-unknown-unknown --manifest-path rpc/Cargo.toml +time cargo build --locked --target=wasm32-unknown-unknown --manifest-path statement-table/Cargo.toml +time cargo build --locked --target=wasm32-unknown-unknown --manifest-path cli/Cargo.toml --no-default-features --features browser diff --git a/scripts/gitlab/lib.sh b/scripts/gitlab/lib.sh index ab2e7c86f676..262944786f5c 100755 --- a/scripts/gitlab/lib.sh +++ b/scripts/gitlab/lib.sh @@ -96,3 +96,79 @@ curl -XPOST -d "$1" "https://matrix.parity.io/_matrix/client/r0/rooms/$2/send/m. # Pretty-printing functions boldprint () { printf "|\n| \033[1m%s\033[0m\n|\n" "${@}"; } boldcat () { printf "|\n"; while read -r l; do printf "| \033[1m%s\033[0m\n" "${l}"; done; printf "|\n" ; } + +prepare_git() { + # Set the user name and email to make merging work + git config --global user.name 'CI system' + git config --global user.email '<>' +} + +prepare_substrate() { + pr_companion=$1 + boldprint "companion pr specified/detected: #${pr_companion}" + + # Clone the current Substrate master branch into ./substrate. + # NOTE: we need to pull enough commits to be able to find a common + # ancestor for successfully performing merges below. + git clone --depth 20 https://github.com/paritytech/substrate.git + previous_path=$(pwd) + cd substrate || exit 1 + SUBSTRATE_PATH=$(pwd) + + git fetch origin "refs/pull/${pr_companion}/head:pr/${pr_companion}" + git checkout "pr/${pr_companion}" + git merge origin/master + + cd "$previous_path" || exit 1 + + # Merge master into our branch before building Polkadot to make sure we don't miss + # any commits that are required by Polkadot. + git merge origin/master + + # Make sure we override the crates in native and wasm build + # patching the git path as described in the link below did not test correctly + # https://doc.rust-lang.org/cargo/reference/overriding-dependencies.html + mkdir .cargo + echo "paths = [ \"$SUBSTRATE_PATH\" ]" > .cargo/config + + mkdir -p target/debug/wbuild/.cargo + cp .cargo/config target/debug/wbuild/.cargo/config +} + +pull_companion_substrate() { + github_api_polkadot_pull_url="https://api.github.com/repos/paritytech/polkadot/pulls" + # use github api v3 in order to access the data without authentication + github_header="Authorization: token ${GITHUB_PR_TOKEN}" + + + # either it's a pull request then check for a companion otherwise use + # substrate:master + if expr "${CI_COMMIT_REF_NAME}" : '^[0-9]\+$' >/dev/null + then + boldprint "this is pull request no ${CI_COMMIT_REF_NAME}" + + pr_data_file="$(mktemp)" + # get the last reference to a pr in substrate + curl -sSL -H "${github_header}" -o "${pr_data_file}" \ + "${github_api_polkadot_pull_url}/${CI_COMMIT_REF_NAME}" + + pr_body="$(jq '.body' < "${pr_data_file}")" + + pr_companion="$(echo "${pr_body}" | sed -n -r \ + -e 's;^.*substrate companion: paritytech/substrate#([0-9]+).*$;\1;p' \ + -e 's;^.*substrate companion: https://github.com/paritytech/substrate/pull/([0-9]+).*$;\1;p' \ + | tail -n 1)" + + if [ "${pr_companion}" ] + then + echo "Substrate path: $SUBSTRATE_PATH" + prepare_git + prepare_substrate "$pr_companion" + else + boldprint "no companion branch found - building your Polkadot branch" + fi + rm -f "${pr_data_file}" + else + boldprint "this is not a pull request - building your Polkadot branch" + fi +} diff --git a/scripts/gitlab/test_deterministic_wasm.sh b/scripts/gitlab/test_deterministic_wasm.sh new file mode 100755 index 000000000000..a7e6a23532c4 --- /dev/null +++ b/scripts/gitlab/test_deterministic_wasm.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +#shellcheck source=lib.sh +source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/lib.sh" + +pull_companion_substrate + +set -e + +# build runtime +WASM_BUILD_NO_COLOR=1 cargo build --verbose --release -p kusama-runtime -p polkadot-runtime -p westend-runtime +# make checksum +sha256sum target/release/wbuild/target/wasm32-unknown-unknown/release/*.wasm > checksum.sha256 +# clean up – FIXME: can we reuse some of the artifacts? +cargo clean +# build again +WASM_BUILD_NO_COLOR=1 cargo build --verbose --release -p kusama-runtime -p polkadot-runtime -p westend-runtime +# confirm checksum +sha256sum -c checksum.sha256 diff --git a/scripts/gitlab/test_linux_stable.sh b/scripts/gitlab/test_linux_stable.sh new file mode 100755 index 000000000000..894443cb8566 --- /dev/null +++ b/scripts/gitlab/test_linux_stable.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +#shellcheck source=lib.sh +source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/lib.sh" + +pull_companion_substrate + +# Test Polkadot pr or master branch with this Substrate commit. +time cargo test --all --release --verbose --locked --features runtime-benchmarks