Skip to content

Commit 7932d52

Browse files
tianontianon
committed
Add new "update.sh/versions-pipeline.groovy" script
For new-style jq-template repos, like: - docker-library/php#1052 - docker-library/golang#339 - docker-library/buildpack-deps#113
1 parent fb7a0df commit 7932d52

File tree

2 files changed

+281
-3
lines changed

2 files changed

+281
-3
lines changed

‎update.sh/vars.groovy‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ def rawReposData = [
3333
],
3434
]],
3535
['golang', [
36-
'env': 'GOLANG_VERSION',
36+
'pipeline-script': 'update.sh/versions-pipeline.groovy',
3737
]],
3838
['haproxy', [
3939
'env': 'HAPROXY_VERSION',
@@ -70,7 +70,7 @@ def rawReposData = [
7070
],
7171
]],
7272
['php', [
73-
'env': 'PHP_VERSION',
73+
'pipeline-script': 'update.sh/versions-pipeline.groovy',
7474
]],
7575
['postgres', [
7676
'env': 'PG_VERSION',
@@ -141,7 +141,7 @@ def rawReposData = [
141141

142142
// versionless
143143
['buildpack-deps', [
144-
'update-script': 'true', // TODO determine if there's more we could do here (auto add/remove of suites?)
144+
'pipeline-script': 'update.sh/versions-pipeline.groovy',
145145
]],
146146
['hello-world', [
147147
'update-script': 'true',

‎update.sh/versions-pipeline.groovy‎

Lines changed: 278 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,278 @@
1+
// properties are set via "generate-pipeline.groovy" (jobDsl)
2+
3+
// This file is intended for use by repositories that have adopted the newer "versions.json" templating (https://github.com/docker-library/php/pull/1052, etc).
4+
5+
// we can't use "load()" here because we don't have a file context (or a real checkout of "oi-janky-groovy" -- the pipeline plugin hides that checkout from the actual pipeline execution)
6+
def vars = fileLoader.fromGit(
7+
'update.sh/vars.groovy', // script
8+
'https://github.com/docker-library/oi-janky-groovy.git', // repo
9+
'master', // branch
10+
null, // credentialsId
11+
'master', // node/label
12+
)
13+
def repo = env.JOB_BASE_NAME
14+
def repoMeta = vars.repoMeta(repo)
15+
16+
node {
17+
env.repo = repo
18+
env.BRANCH_BASE = repoMeta['branch-base']
19+
env.BRANCH_PUSH = repoMeta['branch-push']
20+
21+
env.BASHBREW_CACHE = workspace + '/bashbrew-cache'
22+
env.BASHBREW_LIBRARY = workspace + '/oi/library'
23+
env.BASHBREW_NAMESPACE = 'update.sh'
24+
dir(env.BASHBREW_CACHE) { deleteDir() }
25+
sh 'mkdir -p "$BASHBREW_CACHE"'
26+
27+
env.TEST_RUN_SH = workspace + '/oi/test/run.sh'
28+
29+
stage('Checkout') {
30+
checkout(
31+
poll: false,
32+
changelog: false,
33+
scm: [
34+
$class: 'GitSCM',
35+
userRemoteConfigs: [[
36+
url: 'https://github.com/docker-library/official-images.git',
37+
]],
38+
branches: [[name: '*/master']],
39+
extensions: [
40+
[
41+
$class: 'CleanCheckout',
42+
],
43+
[
44+
$class: 'RelativeTargetDirectory',
45+
relativeTargetDir: 'oi',
46+
],
47+
],
48+
doGenerateSubmoduleConfigurations: false,
49+
submoduleCfg: [],
50+
],
51+
)
52+
checkout([
53+
$class: 'GitSCM',
54+
userRemoteConfigs: [[
55+
name: 'origin',
56+
url: repoMeta['url'],
57+
credentialsId: 'docker-library-bot',
58+
]],
59+
branches: [
60+
[name: '*/' + repoMeta['branch-push']],
61+
[name: '*/' + repoMeta['branch-base']],
62+
],
63+
extensions: [
64+
[
65+
$class: 'CleanCheckout',
66+
],
67+
[
68+
$class: 'RelativeTargetDirectory',
69+
relativeTargetDir: 'repo',
70+
],
71+
],
72+
doGenerateSubmoduleConfigurations: false,
73+
submoduleCfg: [],
74+
])
75+
sh '''
76+
git -C oi config user.name 'Docker Library Bot'
77+
git -C oi config user.email '[email protected]'
78+
git -C repo config user.name 'Docker Library Bot'
79+
git -C repo config user.email '[email protected]'
80+
'''
81+
82+
if (repoMeta['branch-base'] != repoMeta['branch-push']) {
83+
sshagent(['docker-library-bot']) {
84+
sh '''
85+
git -C repo pull --rebase origin "$BRANCH_BASE"
86+
'''
87+
}
88+
}
89+
90+
sh '''
91+
# prefill the bashbrew cache
92+
cd repo
93+
./generate-stackbrew-library.sh \\
94+
| bashbrew from --apply-constraints /dev/stdin > /dev/null
95+
'''
96+
}
97+
98+
ansiColor('xterm') { dir('repo') {
99+
def initialCommit = sh(script: 'git rev-parse HEAD', returnStdout: true).trim()
100+
101+
def versions = sh(
102+
script: '''
103+
jq -r 'keys[]' versions.json
104+
''',
105+
returnStdout: true,
106+
).tokenize()
107+
108+
for (def version in versions) { withEnv(['version=' + version]) {
109+
def commit = sh(script: 'git rev-parse HEAD', returnStdout: true).trim()
110+
111+
try {
112+
stage('Update ' + version) {
113+
sh '''#!/usr/bin/env bash
114+
set -Eeuo pipefail -x
115+
116+
# gather a list of this version's components first so we can diff it later and generate a useful commit message ("Update 1.2 to 1.2.4", "Update 3.4 to openssl 1.1.1g", etc)
117+
version_components() {
118+
jq -r '
119+
.[env.version] | [
120+
.version // empty,
121+
(
122+
to_entries[]
123+
| select(.value | type == "object" and has("version"))
124+
| .key + " " + .value.version
125+
)
126+
] | join("\n")
127+
' versions.json
128+
}
129+
componentsBefore="$(version_components)"
130+
131+
./update.sh "$version"
132+
133+
componentsAfter="$(version_components)"
134+
componentsChanged="$(comm -13 <(echo "$componentsBefore") <(echo "$componentsAfter"))"
135+
136+
# Example generated commit messages:
137+
# Update 3.7
138+
# Update 3.7 to 3.7.14
139+
# Update 3.7 to 3.7.14, openssl 1.1.1g
140+
# Update 3.7 to openssl 1.1.1g
141+
# etc.
142+
commitMessage="Update $version"
143+
if [ -n "$componentsChanged" ]; then
144+
components="$(jq <<<"$componentsChanged" -rRs 'rtrimstr("\n") | split("\n") | join(", ")')"
145+
commitMessage+=" to $components"
146+
fi
147+
git add -A . || :
148+
git commit -m "$commitMessage" || :
149+
'''
150+
}
151+
152+
def newCommit = sh(script: 'git rev-parse HEAD', returnStdout: true).trim()
153+
def didChange = (newCommit != commit)
154+
155+
// Jenkins is silly about stages, so we want to create all the same ones every time as often as we can (even if they end up being fully empty).
156+
157+
stage('Diff ' + version) { if (didChange) {
158+
sh 'git log -1 --stat -p'
159+
} }
160+
161+
stage('Build ' + version) { if (didChange) {
162+
timeout(time: 6, unit: 'HOURS') {
163+
retry(3) {
164+
sh '''
165+
# force our new commits into bashbrew
166+
./generate-stackbrew-library.sh "$version" > "$BASHBREW_LIBRARY/$repo"
167+
git -C "$BASHBREW_CACHE/git" fetch "$PWD" HEAD:
168+
169+
bashbrew cat -f '{{ range .Entries }}{{ $.DockerFroms . | join "\\n" }}{{ "\\n" }}{{ end }}' "$repo" \\
170+
| sort -u \\
171+
| grep -vE '^(scratch|mcr.microsoft.com/windows/(nanoserver|servercore):.*)$' \\
172+
| xargs -rtn1 docker pull \\
173+
|| :
174+
175+
bashbrew build "$repo"
176+
'''
177+
}
178+
}
179+
} }
180+
181+
stage('Test ' + version) { if (didChange) {
182+
timeout(time: 1, unit: 'HOURS') {
183+
retry(3) {
184+
sh 'bashbrew list --apply-constraints --uniq "$repo" | xargs -rt "$TEST_RUN_SH"'
185+
}
186+
}
187+
} }
188+
}
189+
catch (err) {
190+
// if there's an error updating this version, make it like we were never here (and set the build as "unstable")
191+
withEnv(['commit=' + commit]) {
192+
sh '''
193+
git reset --hard "$commit"
194+
git clean -dffx
195+
'''
196+
}
197+
echo "ERROR while updating ${version}: ${err}"
198+
// "catchError" is the only way to set "stageResult" :(
199+
catchError(message: "ERROR while updating ${version}: ${err}", buildResult: 'UNSTABLE', stageResult: 'FAILURE') { error() }
200+
}
201+
} }
202+
203+
// smoke test to ensure the final result is fully valid (and we didn't cause a glitch in "generate-stackbrew-library.sh" like causing two things to share the "buildpack-deps:stable" alias, for example)
204+
stage('Validate') {
205+
sh '''
206+
./generate-stackbrew-library.sh > "$BASHBREW_LIBRARY/$repo"
207+
git -C "$BASHBREW_CACHE/git" fetch "$PWD" HEAD:
208+
bashbrew from --all --uniq "$repo"
209+
210+
../oi/naughty-from.sh "$repo"
211+
../oi/naughty-constraints.sh "$repo"
212+
'''
213+
}
214+
215+
def newCommit = sh(script: 'git rev-parse HEAD', returnStdout: true).trim()
216+
stage('Push') { if (newCommit != initialCommit) {
217+
sshagent(['docker-library-bot']) {
218+
sh 'git push $([ "$BRANCH_BASE" = "$BRANCH_PUSH" ] || echo --force) origin "HEAD:$BRANCH_PUSH"'
219+
}
220+
} }
221+
} }
222+
223+
stage('Stage PR') {
224+
sshagent(['docker-library-bot']) {
225+
sh '''#!/usr/bin/env bash
226+
set -Eeuo pipefail
227+
set -x
228+
''' + """
229+
repo='${repo}'
230+
url='${repoMeta['url'].replaceFirst(':', '/').replaceFirst('git@', 'https://').replaceFirst('[.]git$', '')}'
231+
""" + '''
232+
git -C oi reset HEAD
233+
git -C oi clean -dfx
234+
git -C oi checkout -- .
235+
236+
prevDate="$(git -C oi log -1 --format='format:%at' "$BASHBREW_LIBRARY/$repo")"
237+
(( prevDate++ )) || :
238+
prevDate="$(date --date "@$prevDate" --rfc-2822)"
239+
changesFormat='- %h: %s'
240+
prSed=''
241+
case "$url" in
242+
*github.com*)
243+
changesFormat="- $url/commit/%h: %s"
244+
# look for "#NNN" so we can explicitly link to any PRs too
245+
prSed="s!#([0-9]+)!$url/pull/\\\\1!g"
246+
;;
247+
esac
248+
changes="$(git -C repo log --after="$prevDate" --format="$changesFormat" || :)"
249+
if [ -n "$prSed" ]; then
250+
changes="$(sed -r "$prSed" <<<"$changes")"
251+
fi
252+
253+
commitArgs=( -m "Update $repo" )
254+
if [ -n "$changes" ]; then
255+
# might be something like just "Architectures:" changes for which there's no commits
256+
commitArgs+=( -m 'Changes:' -m "$changes" )
257+
fi
258+
259+
( cd repo && ./generate-stackbrew-library.sh > "$BASHBREW_LIBRARY/$repo" )
260+
261+
oi/naughty-from.sh "$repo"
262+
oi/naughty-constraints.sh "$repo"
263+
264+
date="$(git -C repo log -1 --format='format:%aD')"
265+
export GIT_AUTHOR_DATE="$date" GIT_COMMITTER_DATE="$date"
266+
if [ "$BRANCH_BASE" = "$BRANCH_PUSH" ] && git -C oi add "$BASHBREW_LIBRARY/$repo" && git -C oi commit "${commitArgs[@]}"; then
267+
if diff "$BASHBREW_LIBRARY/$repo" <(wget -qO- "https://github.com/docker-library-bot/official-images/raw/$repo/library/$repo") &> /dev/null; then
268+
# if this exact file content is already pushed to a bot branch, don't force push it again
269+
exit
270+
fi
271+
git -C oi push -f [email protected]:docker-library-bot/official-images.git "HEAD:refs/heads/$repo"
272+
else
273+
git -C oi push [email protected]:docker-library-bot/official-images.git --delete "refs/heads/$repo"
274+
fi
275+
'''
276+
}
277+
}
278+
}

0 commit comments

Comments
 (0)