Skip to content

Commit 474a3c9

Browse files
alvicsamTriplEight
alvicsam
and
TriplEight
authored
Publish polkadot-parachain docker images on PRs (#1311)
* [WIP] Dockerize polkadot-parachain * fix build job * disable test job for debug * disable all tests for debug * add publish docker * fix docker publish * uncomment tests; add dag * fix docker naming * add DAG to build job * small fixes * combine test and build * fix typo * divide test and build back * Update .gitlab-ci.yml Co-authored-by: Denis Pisarev <[email protected]> * rename docker image * add needs publish-s3 * remove collect artifacts from test Co-authored-by: Denis Pisarev <[email protected]>
1 parent faf3563 commit 474a3c9

File tree

2 files changed

+151
-50
lines changed

2 files changed

+151
-50
lines changed

‎.gitlab-ci.yml‎

Lines changed: 102 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
stages:
99
- test
10+
- build
1011
- publish
1112
- benchmarks-build
1213
- benchmarks-run
@@ -43,6 +44,15 @@ variables:
4344
- cargo +nightly --version
4445
- bash --version
4546

47+
.common-refs: &common-refs
48+
# these jobs run always*
49+
rules:
50+
- if: $CI_PIPELINE_SOURCE == "web"
51+
- if: $CI_PIPELINE_SOURCE == "schedule"
52+
- if: $CI_COMMIT_REF_NAME == "master"
53+
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
54+
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
55+
4656
.publish-refs: &publish-refs
4757
rules:
4858
- if: $CI_PIPELINE_SOURCE == "web" &&
@@ -80,107 +90,130 @@ variables:
8090
tags:
8191
- kubernetes-parity-build
8292

83-
.collect-artifacts: &collect-artifacts
84-
artifacts:
85-
name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
86-
when: on_success
87-
expire_in: 28 days
88-
paths:
89-
- ./artifacts/
90-
9193
#### stage: test
9294

9395
test-linux-stable:
9496
stage: test
9597
<<: *docker-env
96-
rules:
97-
- if: $CI_COMMIT_REF_NAME == "master"
98-
- if: $CI_COMMIT_REF_NAME == "tags"
99-
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
100-
# It doesn't make sense to build on every commit, so we build on tags
101-
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
102-
variables:
103-
ARE_WE_RELEASING_YET: maybe!
104-
# web and schedule triggers can be provided with the non-empty variable ARE_WE_RELEASING_YET
105-
# to run building and publishing the binary.
106-
- if: $CI_PIPELINE_SOURCE == "web"
107-
- if: $CI_PIPELINE_SOURCE == "schedule"
108-
<<: *collect-artifacts
98+
<<: *common-refs
10999
variables:
110100
# Enable debug assertions since we are running optimized builds for testing
111101
# but still want to have debug assertions.
112102
RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
113103
script:
114104
- time cargo test --all --release --locked -- --include-ignored
115-
# It's almost free to produce a binary here, please refrain from using it in production since
116-
# it goes with the debug assertions.
117-
- if [ "${ARE_WE_RELEASING_YET}" ]; then
118-
echo "___Building a binary___";
119-
time cargo build --release --locked --bin polkadot-parachain;
120-
echo "___Packing the artifacts___";
121-
mkdir -p ./artifacts;
122-
mv ${CARGO_TARGET_DIR}/release/polkadot-parachain ./artifacts/.;
123-
echo "___The VERSION is either a tag name or the curent branch if triggered not by a tag___";
124-
echo ${CI_COMMIT_REF_NAME} | tee ./artifacts/VERSION;
125-
else
126-
exit 0;
127-
fi
128-
- sccache -s
129105

130106
check-runtime-benchmarks:
131107
stage: test
132108
<<: *docker-env
109+
<<: *common-refs
133110
script:
134111
# Check that the node will compile with `runtime-benchmarks` feature flag.
135112
- time cargo check --all --features runtime-benchmarks
136113
# Check that parachain-template will compile with `runtime-benchmarks` feature flag.
137114
- time cargo check -p parachain-template-node --features runtime-benchmarks
138-
- sccache -s
139115

140116
cargo-check-try-runtime:
141117
stage: test
142118
<<: *docker-env
119+
<<: *common-refs
120+
# this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
121+
needs:
122+
- job: check-runtime-benchmarks
123+
artifacts: false
143124
script:
144125
# Check that the node will compile with `try-runtime` feature flag.
145126
- time cargo check --all --features try-runtime
146127
# Check that parachain-template will compile with `try-runtime` feature flag.
147128
- time cargo check -p parachain-template-node --features try-runtime
148-
- sccache -s
129+
130+
check-rustdoc:
131+
stage: test
132+
<<: *docker-env
133+
<<: *common-refs
134+
variables:
135+
SKIP_WASM_BUILD: 1
136+
RUSTDOCFLAGS: "-Dwarnings"
137+
script:
138+
- time cargo +nightly doc --workspace --all-features --verbose --no-deps
149139

150140
cargo-check-benches:
151141
stage: test
152142
<<: *docker-env
143+
<<: *common-refs
144+
# this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
145+
needs:
146+
- job: check-rustdoc
147+
artifacts: false
153148
script:
154149
- time cargo check --all --benches
155-
- sccache -s
156150

157-
check-rustdoc:
158-
stage: test
151+
#### stage: build
152+
153+
build-linux-stable:
154+
stage: build
159155
<<: *docker-env
156+
<<: *collect-artifacts
160157
variables:
161-
SKIP_WASM_BUILD: 1
162-
RUSTDOCFLAGS: "-Dwarnings"
158+
# Enable debug assertions since we are running optimized builds for testing
159+
# but still want to have debug assertions.
160+
RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
161+
# this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
162+
needs:
163+
- job: check-rustdoc
164+
artifacts: false
163165
script:
164-
- time cargo +nightly doc --workspace --all-features --verbose --no-deps
165-
- sccache -s
166+
- echo "___Building a binary, please refrain from using it in production since it goes with the debug assertions.___"
167+
- time cargo build --release --locked --bin polkadot-parachain
168+
- echo "___Packing the artifacts___"
169+
- mkdir -p ./artifacts
170+
- mv ./target/release/polkadot-parachain ./artifacts/.
171+
- echo "___The VERSION is either a tag name or the curent branch if triggered not by a tag___"
172+
- echo ${CI_COMMIT_REF_NAME} | tee ./artifacts/VERSION
166173

167174
#### stage: publish
168175

176+
build-push-image:
177+
stage: publish
178+
<<: *kubernetes-env
179+
<<: *common-refs
180+
image: quay.io/buildah/stable
181+
needs:
182+
- job: build-linux-stable
183+
artifacts: true
184+
variables:
185+
DOCKERFILE: "docker/polkadot-parachain-debug_unsigned_injected.Dockerfile"
186+
IMAGE_NAME: docker.io/paritypr/polkadot-parachain-debug
187+
VERSION: "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}"
188+
script:
189+
- test "$PARITYPR_USER" -a "$PARITYPR_PASS" ||
190+
( echo "no docker credentials provided"; exit 1 )
191+
- buildah bud
192+
--format=docker
193+
--build-arg VCS_REF="${CI_COMMIT_SHA}"
194+
--build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
195+
--build-arg IMAGE_NAME="${IMAGE_NAME}"
196+
--tag "$IMAGE_NAME:$VERSION"
197+
--file ${DOCKERFILE} .
198+
- echo "$PARITYPR_PASS" |
199+
buildah login --username "$PARITYPR_USER" --password-stdin docker.io
200+
- buildah info
201+
- buildah push --format=v2s2 "$IMAGE_NAME:$VERSION"
202+
after_script:
203+
- buildah logout --all
204+
169205
publish-s3:
170206
stage: publish
171207
<<: *kubernetes-env
172208
image: paritytech/awscli:latest
173209
<<: *publish-refs
210+
needs:
211+
- job: build-linux-stable
212+
artifacts: true
174213
variables:
175214
GIT_STRATEGY: none
176215
BUCKET: "releases.parity.io"
177216
PREFIX: "cumulus/${ARCH}-${DOCKER_OS}"
178-
before_script:
179-
# Job will fail if no artifacts were provided by test-linux-stable job. It's only possible for
180-
# this test to fail if the pipeline was triggered by web or schedule trigger without supplying
181-
# a nono-empty ARE_WE_RELEASING_YET variable.
182-
- test -e ./artifacts/polkadot-parachain ||
183-
( echo "___No artifacts were provided by the previous job, please check the build there___"; exit 1 )
184217
script:
185218
- echo "___Publishing a binary with debug assertions!___"
186219
- echo "___VERSION = $(cat ./artifacts/VERSION) ___"
@@ -235,3 +268,22 @@ benchmarks:
235268
- rm -rf .git/config
236269
tags:
237270
- weights
271+
272+
273+
#### stage: .post
274+
275+
# This job cancels the whole pipeline if any of provided jobs fail.
276+
# In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests
277+
# to fail the pipeline as soon as possible to shorten the feedback loop.
278+
cancel-pipeline:
279+
stage: .post
280+
needs:
281+
- job: test-linux-stable
282+
artifacts: false
283+
rules:
284+
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
285+
when: on_failure
286+
variables:
287+
PROJECT_ID: "${CI_PROJECT_ID}"
288+
PIPELINE_ID: "${CI_PIPELINE_ID}"
289+
trigger: "parity/infrastructure/ci_cd/pipeline-stopper"

‎docker/polkadot-parachain-debug_unsigned_injected.Dockerfile‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
FROM docker.io/library/ubuntu:20.04
2+
3+
# metadata
4+
ARG VCS_REF
5+
ARG BUILD_DATE
6+
ARG IMAGE_NAME
7+
8+
LABEL io.parity.image.authors="[email protected]" \
9+
io.parity.image.vendor="Parity Technologies" \
10+
io.parity.image.title="${IMAGE_NAME}" \
11+
io.parity.image.description="Cumulus, the Polkadot collator." \
12+
io.parity.image.source="https://github.com/paritytech/cumulus/blob/${VCS_REF}/scripts/docker/polkadot-parachain-debug_unsigned_injected.Dockerfile" \
13+
io.parity.image.revision="${VCS_REF}" \
14+
io.parity.image.created="${BUILD_DATE}" \
15+
io.parity.image.documentation="https://github.com/paritytech/cumulus/"
16+
17+
# show backtraces
18+
ENV RUST_BACKTRACE 1
19+
20+
# install tools and dependencies
21+
RUN apt-get update && \
22+
DEBIAN_FRONTEND=noninteractive apt-get install -y \
23+
libssl1.1 \
24+
ca-certificates \
25+
curl && \
26+
# apt cleanup
27+
apt-get autoremove -y && \
28+
apt-get clean && \
29+
find /var/lib/apt/lists/ -type f -not -name lock -delete; \
30+
# add user and link ~/.local/share/polkadot-parachain to /data
31+
useradd -m -u 10000 -U -s /bin/sh -d /polkadot-parachain polkadot-parachain && \
32+
mkdir -p /data /polkadot-parachain/.local/share && \
33+
chown -R polkadot-parachain:polkadot-parachain /data && \
34+
ln -s /data /polkadot-parachain/.local/share/polkadot-parachain && \
35+
mkdir -p /specs
36+
37+
# add polkadot-parachain binary to the docker image
38+
COPY ./artifacts/polkadot-parachain /usr/local/bin
39+
COPY ./parachains/chain-specs/*.json /specs/
40+
41+
USER polkadot-parachain
42+
43+
# check if executable works in this container
44+
RUN /usr/local/bin/polkadot-parachain --version
45+
46+
EXPOSE 30333 9933 9944
47+
VOLUME ["/polkadot-parachain"]
48+
49+
ENTRYPOINT ["/usr/local/bin/polkadot-parachain"]

0 commit comments

Comments
 (0)