From 62f4f66c231006545ebf7ff43d924c70ca8a22ed Mon Sep 17 00:00:00 2001 From: Sergejs Kostjucenko Date: Wed, 25 Aug 2021 14:23:12 +0300 Subject: [PATCH 1/4] Change pipeline to use Vault --- .gitlab-ci.yml | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 07f8fc5a558d..63f96c4e21ad 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,6 +28,9 @@ variables: CI_SERVER_NAME: "GitLab CI" DOCKER_OS: "debian:stretch" ARCH: "x86_64" + VAULT_SERVER_URL: "https://vault.parity-mgmt-vault.parity.io" + VAULT_AUTH_PATH: "gitlab-parity-io-jwt" + VAULT_AUTH_ROLE: "cicd_gitlab_parity_${CI_PROJECT_NAME}" default: cache: {} @@ -84,6 +87,40 @@ default: when: never - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs +#### Vault secrets +.vault-secrets: &vault-secrets + secrets: + GITHUB_USER: + vault: cicd/gitlab/$CI_PROJECT_PATH/GITHUB_USER@kv + file: false + GITHUB_RELEASE_TOKEN: + vault: cicd/gitlab/$CI_PROJECT_PATH/GITHUB_RELEASE_TOKEN@kv + file: false + GITHUB_TOKEN: + vault: cicd/gitlab/$CI_PROJECT_PATH/GITHUB_TOKEN@kv + file: false + MATRIX_ACCESS_TOKEN: + vault: cicd/gitlab/$CI_PROJECT_PATH/MATRIX_ACCESS_TOKEN@kv + file: false + MATRIX_ROOM_ID: + vault: cicd/gitlab/$CI_PROJECT_PATH/MATRIX_ROOM_ID@kv + file: false + PARITYPR_USER: + vault: cicd/gitlab/$CI_PROJECT_PATH/PARITYPR_USER@kv + file: false + PARITYPR_PASS: + vault: cicd/gitlab/$CI_PROJECT_PATH/PARITYPR_PASS@kv + file: false + PIPELINE_TOKEN: + vault: cicd/gitlab/$CI_PROJECT_PATH/PIPELINE_TOKEN@kv + file: false + REL_MAN_ROOM_ID: + vault: cicd/gitlab/$CI_PROJECT_PATH/REL_MAN_ROOM_ID@kv + file: false + SSH_PRIVATE_KEY: + vault: cicd/gitlab/$CI_PROJECT_PATH/SSH_PRIVATE_KEY@kv + file: false + #### stage: test check-runtime: @@ -91,6 +128,7 @@ check-runtime: image: paritytech/tools:latest <<: *kubernetes-env <<: *rules-pr-only + <<: *vault-secrets variables: GITLAB_API: "https://gitlab.parity.io/api/v4" GITHUB_API_PROJECT: "parity%2Finfrastructure%2Fgithub-api" @@ -120,6 +158,7 @@ test-deterministic-wasm: <<: *rules-test <<: *docker-env <<: *compiler-info + <<: *vault-secrets script: - ./scripts/gitlab/test_deterministic_wasm.sh @@ -128,6 +167,7 @@ test-build-linux-stable: <<: *docker-env <<: *compiler-info <<: *collect-artifacts + <<: *vault-secrets variables: RUST_TOOLCHAIN: stable # Enable debug assertions since we are running optimized builds for testing @@ -162,6 +202,7 @@ check-runtime-benchmarks: <<: *rules-test <<: *docker-env <<: *compiler-info + <<: *vault-secrets script: # Check that the node will compile with `runtime-benchmarks` feature flag. - ./scripts/gitlab/check_runtime_benchmarks.sh @@ -207,6 +248,7 @@ check-transaction-versions: stage: build <<: *rules-test <<: *docker-env + <<: *vault-secrets needs: - job: test-build-linux-stable artifacts: true @@ -251,6 +293,7 @@ build-rustdoc: .build-push-image: &build-push-image <<: *kubernetes-env + <<: *vault-secrets image: quay.io/buildah/stable variables: &image-variables GIT_STRATEGY: none @@ -384,6 +427,13 @@ publish-s3-release: &publish-s3 variables: GIT_STRATEGY: none PREFIX: "builds/polkadot/${ARCH}-${DOCKER_OS}" + secrets: + AWS_ACCESS_KEY_ID: + vault: cicd/gitlab/$CI_PROJECT_PATH/AWS_ACCESS_KEY_ID@kv + file: false + AWS_SECRET_ACCESS_KEY: + vault: cicd/gitlab/$CI_PROJECT_PATH/AWS_SECRET_ACCESS_KEY@kv + file: false rules: # publishing binaries nightly - if: $CI_PIPELINE_SOURCE == "schedule" From ac8fa3668728ab10a86bd94b2cee631a768cf183 Mon Sep 17 00:00:00 2001 From: Sergejs Kostjucenko Date: Tue, 31 Aug 2021 13:55:39 +0300 Subject: [PATCH 2/4] Add Parity Docker credentials aquisition from Vault --- .gitlab-ci.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 63f96c4e21ad..4931c2a00412 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -90,6 +90,12 @@ default: #### Vault secrets .vault-secrets: &vault-secrets secrets: + DOCKER_HUB_USER: + vault: cicd/gitlab/parity/DOCKER_HUB_USER@kv + file: false + DOCKER_HUB_PASS: + vault: cicd/gitlab/parity/DOCKER_HUB_PASS@kv + file: false GITHUB_USER: vault: cicd/gitlab/$CI_PROJECT_PATH/GITHUB_USER@kv file: false @@ -346,8 +352,8 @@ publish-polkadot-image: variables: <<: *image-variables IMAGE_NAME: docker.io/parity/rococo - DOCKER_USER: ${Docker_Hub_User_Parity} - DOCKER_PASS: ${Docker_Hub_Pass_Parity} + DOCKER_USER: ${DOCKER_HUB_USER} + DOCKER_PASS: ${DOCKER_HUB_PASS} needs: - job: test-build-linux-stable artifacts: true From 1f13751f351282d5a0ee42a8cf9153232dfa0358 Mon Sep 17 00:00:00 2001 From: Sergejs Kostjucenko Date: Tue, 31 Aug 2021 14:25:33 +0300 Subject: [PATCH 3/4] Add GITHUB_PR_TOKEN aquisition from Vault --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4931c2a00412..1900d7eaf7a1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -96,6 +96,9 @@ default: DOCKER_HUB_PASS: vault: cicd/gitlab/parity/DOCKER_HUB_PASS@kv file: false + GITHUB_PR_TOKEN: + vault: cicd/gitlab/parity/GITHUB_PR_TOKEN@kv + file: false GITHUB_USER: vault: cicd/gitlab/$CI_PROJECT_PATH/GITHUB_USER@kv file: false From 471b1707a792dd1960f7c172b05bcbd69a95a0a2 Mon Sep 17 00:00:00 2001 From: Sergejs Kostjucenko Date: Wed, 8 Sep 2021 17:51:04 +0300 Subject: [PATCH 4/4] Move s3 secrets to common group --- .gitlab-ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9a7487659e7d..97dc29a58557 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -90,6 +90,12 @@ default: #### Vault secrets .vault-secrets: &vault-secrets secrets: + AWS_ACCESS_KEY_ID: + vault: cicd/gitlab/$CI_PROJECT_PATH/AWS_ACCESS_KEY_ID@kv + file: false + AWS_SECRET_ACCESS_KEY: + vault: cicd/gitlab/$CI_PROJECT_PATH/AWS_SECRET_ACCESS_KEY@kv + file: false DOCKER_HUB_USER: vault: cicd/gitlab/parity/DOCKER_HUB_USER@kv file: false @@ -432,17 +438,11 @@ publish-s3-release: &publish-s3 - job: test-build-linux-stable artifacts: true <<: *kubernetes-env + <<: *vault-secrets image: paritytech/awscli:latest variables: GIT_STRATEGY: none PREFIX: "builds/polkadot/${ARCH}-${DOCKER_OS}" - secrets: - AWS_ACCESS_KEY_ID: - vault: cicd/gitlab/$CI_PROJECT_PATH/AWS_ACCESS_KEY_ID@kv - file: false - AWS_SECRET_ACCESS_KEY: - vault: cicd/gitlab/$CI_PROJECT_PATH/AWS_SECRET_ACCESS_KEY@kv - file: false rules: # publishing binaries nightly - if: $CI_PIPELINE_SOURCE == "schedule"