[CI] Add cumulus + beefy companions

.gitlab-ci.yml

@@ -265,7 +265,7 @@ node-bench-regression-guard:
     CI_IMAGE:                      "paritytech/node-bench-regression-guard:latest"
   before_script: [""]
   script:
-    - 'node-bench-regression-guard --reference artifacts/benches/master-* 
+    - 'node-bench-regression-guard --reference artifacts/benches/master-*
        --compare-with artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA'
 
 cargo-check-subkey:
@@ -428,19 +428,41 @@ check-polkadot-companion-status:
   script:
     - ./.maintain/gitlab/check_polkadot_companion_status.sh
 
-check-polkadot-companion-build:
-  stage:                           build
+.check-companion-build:            &check-companion-build
+  stage:                           test
   <<:                              *docker-env
   <<:                              *test-refs-no-trigger
-  needs:
-    - job:                         test-linux-stable-int
-      artifacts:                   false
   script:
-    - ./.maintain/gitlab/check_polkadot_companion_build.sh
+    - ./.maintain/gitlab/check_companion_build.sh "$COMPANION_ORG" "$COMPANION_REPO" "$COMPANION_BUILD" "$COMPANION_DEPENDENCY"
   after_script:
-    - cd polkadot && git rev-parse --abbrev-ref HEAD
+    - cd "$COMPANION_REPO" && git rev-parse --abbrev-ref HEAD
   allow_failure:                   true
 
+check-polkadot-companion-build:
+  <<: *check-companion-build
+  variables:
+    COMPANION_ORG: paritytech
+    COMPANION_REPO: polkadot
+    COMPANION_DEPENDENCY: paritytech/grandpa-bridge-gadget
+  allow_failure: true
+
+check-cumulus-companion-build:
+  <<: *check-companion-build
+  variables:
+    COMPANION_ORG: paritytech
+    COMPANION_REPO: cumulus
+    COMPANION_BUILD: "cargo check"
+    COMPANION_DEPENDENCY: paritytech/polkadot
+  allow_failure: true
+
+check-beefy-companion-build:
+  <<: *check-companion-build
+  variables:
+    COMPANION_ORG: paritytech
+    COMPANION_REPO: grandpa-bridge-gadget
+    COMPANION_BUILD: "cargo check"
+  allow_failure: true
+
 test-browser-node:
   stage:                           build
   <<:                              *docker-env
@@ -568,7 +590,7 @@ build-rust-doc:
     - buildah push --format=v2s2 "$IMAGE_NAME:latest"
   after_script:
     - buildah logout "$IMAGE_NAME"
-    # pass artifacts to the trigger-simnet job 
+    # pass artifacts to the trigger-simnet job
     - echo "IMAGE_NAME=${IMAGE_NAME}" | tee -a ./artifacts/$PRODUCT/build.env
     - IMAGE_TAG="$(cat ./artifacts/$PRODUCT/VERSION)"
     - echo "IMAGE_TAG=${IMAGE_TAG}"   | tee -a ./artifacts/$PRODUCT/build.env
@@ -702,7 +724,7 @@ trigger-simnet:
     - if: $CI_COMMIT_REF_NAME == "master"
   needs:
     - job:                         publish-docker-substrate
-  # `build.env` brings here `$IMAGE_NAME` and `$IMAGE_TAG` (`$VERSION` here, 
+  # `build.env` brings here `$IMAGE_NAME` and `$IMAGE_TAG` (`$VERSION` here,
   # i.e. `2643-0.8.29-5f689e0a-6b24dc54`).
   variables:
     TRGR_PROJECT:                  ${CI_PROJECT_NAME}

.maintain/common/lib.sh

@@ -2,6 +2,18 @@
 
 api_base="https://api.github.com/repos"
 
+
+set_github_token(){
+  # These will exist if the function is called in Gitlab.
+  # If the function's called in Github, we should have GITHUB_ACCESS_TOKEN set
+  # already.
+  if [ -n "$GITHUB_RELEASE_TOKEN" ]; then
+    GITHUB_TOKEN="$GITHUB_RELEASE_TOKEN"
+  elif [ -n "$GITHUB_PR_TOKEN" ]; then
+    GITHUB_TOKEN="$GITHUB_PR_TOKEN"
+  fi
+}
+
 # Function to take 2 git tags/commits and get any lines from commit messages
 # that contain something that looks like a PR reference: e.g., (#1234)
 sanitised_git_logs(){
@@ -66,15 +78,7 @@ has_label(){
   repo="$1"
   pr_id="$2"
   label="$3"
-
-  # These will exist if the function is called in Gitlab.
-  # If the function's called in Github, we should have GITHUB_ACCESS_TOKEN set
-  # already.
-  if [ -n "$GITHUB_RELEASE_TOKEN" ]; then
-    GITHUB_TOKEN="$GITHUB_RELEASE_TOKEN"
-  elif [ -n "$GITHUB_PR_TOKEN" ]; then
-    GITHUB_TOKEN="$GITHUB_PR_TOKEN"
-  fi
+  set_github_token
 
   out=$(curl -H "Authorization: token $GITHUB_TOKEN" -s "$api_base/$repo/pulls/$pr_id")
   [ -n "$(echo "$out" | tr -d '\r\n' | jq ".labels | .[] | select(.name==\"$label\")")" ]
@@ -115,3 +119,25 @@ has_runtime_changes() {
     return 1
   fi
 }
+
+# Given an origin repo & PR,and a repository, will check if the given PR has a
+# companion PR in the target repo
+get_companion() {
+  origin_repo="$1"
+  pr_num="$2"
+  companion_repo="$3"
+  pr_data_file="$(mktemp)"
+  set_github_token
+  github_header="Authorization: token ${GITHUB_TOKEN}"
+  # get the last reference to a pr in the target repo
+  curl -sSL -H "${github_header}" -o "${pr_data_file}" \
+    "${api_base}/$origin_repo/pulls/$pr_num"
+
+  pr_body="$(sed -n -r 's/^[[:space:]]+"body": (".*")[^"]+$/\1/p' "${pr_data_file}")"
+
+  echo "${pr_body}" | sed -n -r \
+      -e "s;^.*[Cc]ompanion.*$companion_repo#([0-9]+).*$;\1;p" \
+      -e "s;^.*[Cc]ompanion.*https://github.com/$companion_repo/pull/([0-9]+).*$;\1;p" \
+    | tail -n 1
+  rm -f "${pr_data_file}"
+}

.maintain/gitlab/check_companion_build.sh

@@ -0,0 +1,123 @@
+#!/usr/bin/env bash
+#
+# check if a pr is compatible with companion pr or master if not available
+#
+# to override one that was just mentioned mark companion pr in the body of the
+# pr like
+#
+# $REPO companion: $ORGANISATION/$REPO#567
+#
+# $ORGANISATION and $REPO are set using $1 and $2. You can also specify a custom
+# build command with $3
+# Every argument after $3 is for specifying *additional* dependencies this
+# project has that depend on substrate, which might also have companion PRs.
+
+# Example: Cumulus relies on both substrate and polkadot. If this substrate PR
+# requires a companion build on polkadot, when we are testing that cumulus builds
+# with this commit of substrate, we *also* need to clone & patch polkadot, and tell
+# cumulus to use this cloned+patched version, else the build is guaranteed to fail
+# (since it doesn't have the changes to polkadot that were required in the polkadot
+# companion PR)
+
+# So invoke this script like (arguments in [] indicate optional arguments)
+# ./check_companion_build.sh paritytech cumulus ['cargo test --release'] [paritytech/polkadot]
+
+set -e
+
+# Include the common functions library
+#shellcheck source=../common/lib.sh
+. "$(dirname "${0}")/../common/lib.sh"
+
+ORGANISATION=$1
+REPO=$2
+BUILDSTRING=${3:-cargo test --workspace --release}
+DEPS=("${@:4}")
+SUBSTRATE_DIR="$(pwd)"
+
+boldprint () { printf "|\n| \033[1m%s\033[0m\n|\n" "${@}"; }
+boldcat () { printf "|\n"; while read -r l; do printf "| \033[1m%s\033[0m\n" "${l}"; done; printf "|\n" ; }
+
+boldcat <<-EOT
+
+
+check_${REPO}_companion_build
+==============================
+
+this job checks if there is a string in the description of the pr like
+
+$REPO companion: $ORGANISATION/$REPO#567
+
+
+it will then run cargo check from this ${REPO}'s branch with substrate code
+from this pull request. otherwise, it will uses master instead
+
+
+EOT
+
+# Set the user name and email to make merging work
+git config --global user.name 'CI system'
+git config --global user.email '<>'
+
+# Merge master into our branch before building to make sure we don't miss
+# any commits that are required.
+git fetch --depth 100 origin
+git merge origin/master
+
+# Clone the current master branch into a local directory
+# NOTE: we need to pull enough commits to be able to find a common
+# ancestor for successfully performing merges below.
+git clone --depth 20 "https://github.com/${ORGANISATION}/${REPO}.git"
+
+cd "$REPO"
+
+# either it's a pull request then check for a companion otherwise use
+# master
+# shellcheck disable=SC2003
+if expr match "${CI_COMMIT_REF_NAME}" '^[0-9]\+$' >/dev/null
+then
+  boldprint "this is pull request no ${CI_COMMIT_REF_NAME}"
+  pr_companion="$(get_companion "paritytech/substrate" "$CI_COMMIT_REF_NAME" "$ORGANISATION/$REPO")"
+  if [ "$pr_companion" ]
+  then
+    boldprint "companion pr specified/detected: #${pr_companion}"
+    git fetch origin "refs/pull/${pr_companion}/head:pr/${pr_companion}"
+    git checkout "pr/${pr_companion}"
+    git merge origin/master
+  else
+    boldprint "no companion branch found - building ${REPO}:master"
+  fi
+
+  # If this repo has any additional dependencies, we should check whether they
+  # are mentioned as companions as well. If they are, we patch this repo to
+  # use that companion build as well. See example at top of this script
+  # Note: Will only work with repos supported by diener
+  declare -A diener_commands
+  diener_commands=()
+  diener_commands["paritytech/polkadot"]='--polkadot'
+  diener_commands["paritytech/substrate"]='--substrate'
+  diener_commands["paritytech/grandpa-bridge-gadget"]='--beefy'
+
+  for dep in "${DEPS[@]}"; do
+    dep_companion="$(get_companion "paritytech/substrate" "$CI_COMMIT_REF_NAME" "$dep")"
+    if [ "$dep_companion" ]; then
+      echo "Companion PR found for $dep, need to patch $REPO to use that"
+      git clone --depth 20 "https://github.com/$dep.git" "$dep"
+      git -C "$dep" fetch origin "refs/pull/${dep_companion}/head:pr/${dep_companion}"
+      git -C "$dep" checkout "pr/${dep_companion}"
+      git -C "$dep" merge origin/master
+      # Tell this dependency to use the version of substrate in this PR
+      diener patch --crates-to-patch "$SUBSTRATE_DIR" --substrate --path "$dep/Cargo.toml"
+      # then we tell this repository to point at our locally cloned dependency
+      diener patch --crates-to-patch "$dep" "${diener_commands[$dep]}" --path "Cargo.toml"
+    fi
+
+  done
+
+else
+  boldprint "this is not a pull request - building ${REPO}:master"
+fi
+
+# Test pr or master branch with this Substrate commit.
+diener patch --crates-to-patch ".." --substrate --path "Cargo.toml"
+
+eval "$BUILDSTRING"