From 54071cfea53fb005978805ad2927c243914f35f3 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 2 Mar 2020 10:53:22 +1300 Subject: [PATCH 1/3] Use the latest base image automatically --- .azurePipeline/templateDockerBuildPush.yml | 33 ++++++++++++++-------- azure-pipelines.yml | 5 ++-- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/.azurePipeline/templateDockerBuildPush.yml b/.azurePipeline/templateDockerBuildPush.yml index 9c199727..1f18ac50 100644 --- a/.azurePipeline/templateDockerBuildPush.yml +++ b/.azurePipeline/templateDockerBuildPush.yml @@ -9,7 +9,7 @@ # - `imageTag` is optional - if not provided the templateSetVariableDockerTag will be used to determine the tag. # - `jobName` - should only include alphanumeric values and '_'. # - `skip` - don't bother building if set to anything other than 'false' -# - `dockerBuildArgs` - extra arguments passed to docker build. +# - `dockerBuildVersion` - VERSION argument passed to docker build. # - `extraJobs` can contain a list of jobs to do in the same stage which are dependent on the built and pushed docker image. parameters: @@ -19,7 +19,7 @@ parameters: imageTag: '' jobName: '' skip: 'False' - dockerBuildArgs: '' + dockerBuildVersion: '' extraJobs: [] dependsOn: '' @@ -33,7 +33,7 @@ jobs: variables: DOCKER_TAG: ${{ parameters.imageTag }} SKIP: ${{ parameters.skip }} - DOCKER_BUILD_ARGS: ${{ parameters.dockerBuildArgs }} + DOCKER_BUILD_VERSION: ${{ parameters.dockerBuildVersion }} steps: - bash: | if [ -z "$DOCKER_TAG" ]; then @@ -44,7 +44,7 @@ jobs: echo "Variable DOCKER_TAG = \"$DOCKER_TAG\"" echo "Variable SKIP = \"$SKIP\"" - echo "Variable DOCKER_BUILD_ARGS = \"$DOCKER_BUILD_ARGS\"" + echo "Variable DOCKER_BUILD_VERSION = \"$DOCKER_BUILD_VERSION\"" env: DOCKER_TAG: $(DOCKER_TAG) displayName: Echo tag parameter and variables @@ -54,15 +54,26 @@ jobs: docker login -u $(dockerHubId) -p $(dockerHubPassword) displayName: 'Dockerhub login' - script: | + + if [[ -z "${{ parameters.dockerBuildVersion }}" ]]; then + DOCKER_BUILD_ARGS="" + else + DOCKER_BUILD_ARGS="--build-arg VERSION=${{ parameters.dockerBuildVersion }}" + fi + echo "Variable DOCKER_BUILD_ARGS = \"$DOCKER_BUILD_ARGS\"" + + if [[ -z "${{ parameters.dockerFilePath }}" ]]; then + DOCKER_PATH="" + else + DOCKER_PATH="-f ${{ parameters.dockerFilePath }}" + fi + echo "Variable DOCKER_PATH = \"$DOCKER_PATH\"" + if [[ "$SKIP" == "False" ]]; then - if [[ -z "${{ parameters.dockerFilePath }}" ]]; then - docker build -t ${{ parameters.imageName }}:$(DOCKER_TAG) $(DOCKER_BUILD_ARGS) ${{ parameters.folder }} - else - docker build -t ${{ parameters.imageName }}:$(DOCKER_TAG) -f ${{ parameters.dockerFilePath }} $(DOCKER_BUILD_ARGS) ${{ parameters.folder }} - fi - docker push ${{ parameters.imageName }}:$(DOCKER_TAG) + docker build -t ${{ parameters.imageName }}:$(DOCKER_TAG) $DOCKER_PATH $DOCKER_BUILD_ARGS ${{ parameters.folder }} + docker push ${{ parameters.imageName }}:$(DOCKER_TAG) else - echo "Skipping build" + echo "Skipping build" fi displayName: 'Build and push docker image' diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 08896125..2f434b24 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -88,8 +88,9 @@ stages: dependsOn: HashBaseDependencies imageName: data61/anonlink-app # this part is still todo automated - dockerBuildArgs: "--build-arg VERSION=9e015cb563319b280895f786a9f3647c5e5e43a81966e85d0a8c0a2424f94fde" - #dockerBuildArgs: "--build-arg VERSION=$[dependencies.HashBaseDependencies.outputs['SetDockerBaseTag.DOCKER_BASE_TAG']]" + #dockerBuildVersion: "9e015cb563319b280895f786a9f3647c5e5e43a81966e85d0a8c0a2424f94fde" + dockerBuildVersion: "$[dependencies.HashBaseDependencies.outputs['SetDockerBaseTag.DOCKER_BASE_TAG']]" + - stage: stage_docker_nginx_image_build displayName: Nginx Docker build From f912c649a7a0c58991da7fd47e89e4a098c924b0 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 2 Mar 2020 11:23:48 +1300 Subject: [PATCH 2/3] Hello azure my old friend --- .azurePipeline/templateDockerBuildPush.yml | 5 ++--- azure-pipelines.yml | 2 -- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.azurePipeline/templateDockerBuildPush.yml b/.azurePipeline/templateDockerBuildPush.yml index 1f18ac50..940182b1 100644 --- a/.azurePipeline/templateDockerBuildPush.yml +++ b/.azurePipeline/templateDockerBuildPush.yml @@ -54,11 +54,10 @@ jobs: docker login -u $(dockerHubId) -p $(dockerHubPassword) displayName: 'Dockerhub login' - script: | - - if [[ -z "${{ parameters.dockerBuildVersion }}" ]]; then + if [[ -z "$DOCKER_BUILD_VERSION" ]]; then DOCKER_BUILD_ARGS="" else - DOCKER_BUILD_ARGS="--build-arg VERSION=${{ parameters.dockerBuildVersion }}" + DOCKER_BUILD_ARGS="--build-arg VERSION=$DOCKER_BUILD_VERSION" fi echo "Variable DOCKER_BUILD_ARGS = \"$DOCKER_BUILD_ARGS\"" diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 2f434b24..fbfc8942 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -87,8 +87,6 @@ stages: jobName: 'anonlink_app' dependsOn: HashBaseDependencies imageName: data61/anonlink-app - # this part is still todo automated - #dockerBuildVersion: "9e015cb563319b280895f786a9f3647c5e5e43a81966e85d0a8c0a2424f94fde" dockerBuildVersion: "$[dependencies.HashBaseDependencies.outputs['SetDockerBaseTag.DOCKER_BASE_TAG']]" From f9f3d8a37409690365c22c80c2fbadfa3b68a29e Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 2 Mar 2020 11:35:37 +1300 Subject: [PATCH 3/3] Remove some azure debugging and update devops docs --- .azurePipeline/templateDockerBuildPush.yml | 15 --------------- azure-pipelines.yml | 1 - docs/development.rst | 10 ++++++---- docs/devops.rst | 9 +++++---- 4 files changed, 11 insertions(+), 24 deletions(-) diff --git a/.azurePipeline/templateDockerBuildPush.yml b/.azurePipeline/templateDockerBuildPush.yml index 940182b1..c7cfd058 100644 --- a/.azurePipeline/templateDockerBuildPush.yml +++ b/.azurePipeline/templateDockerBuildPush.yml @@ -35,19 +35,6 @@ jobs: SKIP: ${{ parameters.skip }} DOCKER_BUILD_VERSION: ${{ parameters.dockerBuildVersion }} steps: - - bash: | - if [ -z "$DOCKER_TAG" ]; then - echo "Empty docker tag" - else - echo "Non empty docker tag $DOCKER_TAG" - fi - - echo "Variable DOCKER_TAG = \"$DOCKER_TAG\"" - echo "Variable SKIP = \"$SKIP\"" - echo "Variable DOCKER_BUILD_VERSION = \"$DOCKER_BUILD_VERSION\"" - env: - DOCKER_TAG: $(DOCKER_TAG) - displayName: Echo tag parameter and variables - ${{ if eq(parameters.imageTag, '') }}: - template: templateSetVariableDockerTag.yml # Set DOCKER_TAG if it wasn't provided in parameters as imageTag - script: | @@ -59,14 +46,12 @@ jobs: else DOCKER_BUILD_ARGS="--build-arg VERSION=$DOCKER_BUILD_VERSION" fi - echo "Variable DOCKER_BUILD_ARGS = \"$DOCKER_BUILD_ARGS\"" if [[ -z "${{ parameters.dockerFilePath }}" ]]; then DOCKER_PATH="" else DOCKER_PATH="-f ${{ parameters.dockerFilePath }}" fi - echo "Variable DOCKER_PATH = \"$DOCKER_PATH\"" if [[ "$SKIP" == "False" ]]; then docker build -t ${{ parameters.imageName }}:$(DOCKER_TAG) $DOCKER_PATH $DOCKER_BUILD_ARGS ${{ parameters.folder }} diff --git a/azure-pipelines.yml b/azure-pipelines.yml index fbfc8942..d04cef0f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -89,7 +89,6 @@ stages: imageName: data61/anonlink-app dockerBuildVersion: "$[dependencies.HashBaseDependencies.outputs['SetDockerBaseTag.DOCKER_BASE_TAG']]" - - stage: stage_docker_nginx_image_build displayName: Nginx Docker build dependsOn: [] diff --git a/docs/development.rst b/docs/development.rst index 7182fb25..f002dd6f 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -47,10 +47,11 @@ manually installed using ``pip``:: pip install -r base/requirements.txt -Docker is used for packaging the application, we rely on a base image that includes the Python -dependencies. To update a dependency commit a change to the pinned version in ``base/requirements.txt``. -Our CI system will bake the base image and tag it with a digest. You could generate the digest yourself -with bash (example digest shown):: +Docker is used for packaging the application, we rely on a base image that includes the operating system +level and Python level dependencies. To update a dependency change the pinned version in ``base/requirements.txt`` +or ``base/Dockerfile``. Our CI system will bake the base image and tag it with a digest. + +If you were so inclined you could generate the digest yourself with bash (example digest shown):: $ cd base $ sha256sum requirements.txt Dockerfile | sha256sum | cut -f 1 -d " " | tr [:upper:] [:lower:] @@ -69,6 +70,7 @@ argument:: --build-arg VERSION=3814723844e4b359f0b07e86a57093ad4f88aa434c42ced9c72c611bbcf9819a +Note the CI system automatically uses the current base image when building the application images. Redis ----- diff --git a/docs/devops.rst b/docs/devops.rst index 111c954e..6f63f52a 100644 --- a/docs/devops.rst +++ b/docs/devops.rst @@ -50,7 +50,8 @@ It also requires a service connection to a ``k8s`` cluster to be configured. Base Image ~~~~~~~~~~ -The CI system builds and pushes the base image, before building downstream images. Ideally the CI -system should automatically build the application using the most recent base ``VERSION``, -however this is currently manual. In the ``azure-pipelines.yml`` file we pass the desired base -version. See :ref:`dev-dependencies`. +The CI system builds and pushes the base image, before building downstream images. The CI +system builds the application images using the current base ``VERSION``. If a base image with the given +digest is already present on Docker Hub the base image won't be rebuilt. + +For additional details see :ref:`dev-dependencies`.