Convert to new split pipeline approach for CI/CD (microsoft#278)

* new split pipelines

* specify channel

* fix pip jq install

* fixes and cleanup

* add new lines

* add docs and clean up naming

* use shared image

* rename

* strip quotes from location

* fix env var

* remove succeeded condition

* remove extra deploy yml

* no pr trigger

* remove unused template

* remove unused pipeline

* add more docs and add to bootstrap

* linting

* fix model package to show logs

* Squashed commit of the following:

commit 01af8da
Author: j-so <[email protected]>
Date:   Fri Jun 12 14:15:35 2020 -0700

    fixed failure handling

commit 1e6f906
Author: j-so <[email protected]>
Date:   Fri Jun 12 14:03:09 2020 -0700

    test failed conda dep

commit a8030d7
Author: j-so <[email protected]>
Date:   Fri Jun 12 13:55:00 2020 -0700

    test package fail

commit c7845aa
Author: j-so <[email protected]>
Date:   Fri Jun 12 13:46:27 2020 -0700

    fail on deploy error'

* Squashed commit of the following:

commit 5af6eeb
Author: j-so <[email protected]>
Date:   Mon Jun 15 18:31:15 2020 -0700

    fix bootstrap

commit f61e103
Merge: 2796b40 08bb6f4
Author: j-so <[email protected]>
Date:   Mon Jun 15 18:30:21 2020 -0700

    Merge branch 'master' into jenns/splitpipeline_docfix

commit 2796b40
Author: j-so <[email protected]>
Date:   Mon Jun 15 18:30:00 2020 -0700

    remove old pipeline

commit 08bb6f4
Author: David Tesar <[email protected]>
Date:   Mon Jun 15 14:09:12 2020 -0700

    Simplify docs flow (microsoft#297)

commit cd762ec
Author: jotaylo <[email protected]>
Date:   Mon Jun 15 12:28:23 2020 -0700

    Move instruction to install AML extension to Azure Devops setup instructions (microsoft#298)

* remove need for model build id

* fix batch scoring

* use model version for batch scoring

* linting

* Squashed commit of the following:

commit 493ed3e
Author: j-so <[email protected]>
Date:   Mon Jun 22 16:42:07 2020 -0700

    mark as output

commit 1ca7a59
Author: j-so <[email protected]>
Date:   Mon Jun 22 16:12:10 2020 -0700

    fix import

commit 743e301
Author: j-so <[email protected]>
Date:   Mon Jun 22 15:59:43 2020 -0700

    more fixes

commit 44abcac
Author: j-so <[email protected]>
Date:   Mon Jun 22 15:50:49 2020 -0700

    fix batch scoring

* improve the docs

* fix secret access

* pass to cli task and impove naming

Author: j-so

.pipelines/diabetes_regression-batchscoring-ci.yml

@@ -1,12 +1,27 @@
 # Continuous Integration (CI) pipeline that orchestrates the batch scoring of the diabetes_regression model.
 
+# Runtime parameters to select artifacts
+parameters:
+- name : artifactBuildId
+  displayName: Model Train CI Build ID. Default is 'latest'.
+  type: string
+  default: latest
+
+pr: none
+
+# Trigger this pipeline on model-train pipeline completion
 resources:
   containers:
   - container: mlops
     image: mcr.microsoft.com/mlops/python:latest
- 
+  pipelines:
+  - pipeline: model-train-ci
+    source: Model-Train-Register-CI # Name of the triggering pipeline
+    trigger:
+      branches:
+        include:
+        - master
 
-pr: none
 trigger:
   branches:
     include:
@@ -34,7 +49,13 @@ stages:
     timeoutInMinutes: 0
     steps:
     - template: code-quality-template.yml
+    - template: diabetes_regression-get-model-id-artifact-template.yml
+      parameters:
+        projectId: '$(resources.pipeline.model-train-ci.projectID)'
+        pipelineId: '$(resources.pipeline.model-train-ci.pipelineID)'
+        artifactBuildId: ${{ parameters.artifactBuildId }}
     - task: AzureCLI@1
+      displayName: "Publish Batch Scoring Pipeline"
       name: publish_batchscore
       inputs:
         azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'        
@@ -45,20 +66,24 @@ stages:
           export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
           # Invoke the Python building and publishing a training pipeline
           python -m ml_service.pipelines.diabetes_regression_build_parallel_batchscore_pipeline
- 
+      env:
+        SCORING_DATASTORE_ACCESS_KEY: $(SCORING_DATASTORE_ACCESS_KEY)
+
   - job: "Run_Batch_Score_Pipeline"
     displayName: "Run Batch Scoring Pipeline"
-    dependsOn: "Build_Batch_Scoring_Pipeline"
+    dependsOn: ["Build_Batch_Scoring_Pipeline"]
     timeoutInMinutes: 240
     pool: server
     variables:
       pipeline_id: $[ dependencies.Build_Batch_Scoring_Pipeline.outputs['publish_batchscore.pipeline_id']]
+      model_name: $[ dependencies.Build_Batch_Scoring_Pipeline.outputs['get_model.MODEL_NAME']]
+      model_version: $[ dependencies.Build_Batch_Scoring_Pipeline.outputs['get_model.MODEL_VERSION']]
     steps:
     - task: ms-air-aiagility.vss-services-azureml.azureml-restApi-task.MLPublishedPipelineRestAPITask@0
       displayName: 'Invoke Batch Scoring pipeline'
       inputs:
         azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
         PipelineId: '$(pipeline_id)'
         ExperimentName: '$(EXPERIMENT_NAME)'
-        PipelineParameters: '"ParameterAssignments": {"model_name": "$(MODEL_NAME)"}'
+        PipelineParameters: '"ParameterAssignments": {"model_name": "$(model_name)", "model_version": "$(model_version)"}'
 

.pipelines/diabetes_regression-cd.yml

@@ -0,0 +1,161 @@
+# Continuous Integration (CI) pipeline that orchestrates the deployment of the diabetes_regression model.
+
+# Runtime parameters to select artifacts
+parameters:
+- name : artifactBuildId
+  displayName: Model Train CI Build ID. Default is 'latest'.
+  type: string
+  default: latest
+
+pr: none
+
+# Trigger this pipeline on model-train pipeline completion
+trigger: none
+resources:
+  containers:
+  - container: mlops
+    image: mcr.microsoft.com/mlops/python:latest
+  pipelines:
+  - pipeline: model-train-ci
+    source: Model-Train-Register-CI # Name of the triggering pipeline
+    trigger:
+      branches:
+        include:
+        - master
+
+variables:
+- template: diabetes_regression-variables-template.yml
+- group: devopsforai-aml-vg
+
+stages:
+- stage: 'Deploy_ACI'
+  displayName: 'Deploy to ACI'
+  condition: variables['ACI_DEPLOYMENT_NAME']
+  jobs:
+  - job: "Deploy_ACI"
+    displayName: "Deploy to ACI"
+    container: mlops
+    timeoutInMinutes: 0
+    steps:
+    - download: none
+    - template: diabetes_regression-get-model-id-artifact-template.yml
+      parameters:
+        projectId: '$(resources.pipeline.model-train-ci.projectID)'
+        pipelineId: '$(resources.pipeline.model-train-ci.pipelineID)'
+        artifactBuildId: ${{ parameters.artifactBuildId }}
+    - task: AzureCLI@1
+      displayName: 'Install AzureML CLI'
+      inputs:
+        azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+        scriptLocation: inlineScript
+        workingDirectory: $(Build.SourcesDirectory)
+        inlineScript: 'az extension add -n azure-cli-ml'
+    - task: AzureCLI@1
+      displayName: "Deploy to ACI (CLI)"
+      inputs:
+        azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+        scriptLocation: inlineScript
+        workingDirectory: $(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring
+        inlineScript: |
+          set -e # fail on error
+          
+          az ml model deploy --name $(ACI_DEPLOYMENT_NAME) --model '$(MODEL_NAME):$(MODEL_VERSION)' \
+          --ic inference_config.yml \
+          --dc deployment_config_aci.yml \
+          -g $(RESOURCE_GROUP) --workspace-name $(WORKSPACE_NAME) \
+          --overwrite -v
+    - task: AzureCLI@1
+      displayName: 'Smoke test'
+      inputs:
+        azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+        scriptLocation: inlineScript
+        inlineScript: |
+          set -e # fail on error
+          export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+          python -m ml_service.util.smoke_test_scoring_service --type ACI --service "$(ACI_DEPLOYMENT_NAME)"
+
+- stage: 'Deploy_AKS'
+  displayName: 'Deploy to AKS'
+  dependsOn: Deploy_ACI
+  condition: and(succeeded(), variables['AKS_DEPLOYMENT_NAME'])
+  jobs:
+  - job: "Deploy_AKS"
+    displayName: "Deploy to AKS"
+    container: mlops
+    timeoutInMinutes: 0
+    steps:
+    - template: diabetes_regression-get-model-id-artifact-template.yml
+      parameters:
+        projectId: '$(resources.pipeline.model-train-ci.projectID)'
+        pipelineId: '$(resources.pipeline.model-train-ci.pipelineID)'
+        artifactBuildId: ${{ parameters.artifactBuildId }}
+    - task: AzureCLI@1
+      displayName: 'Install AzureML CLI'
+      inputs:
+        azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+        scriptLocation: inlineScript
+        workingDirectory: $(Build.SourcesDirectory)
+        inlineScript: 'az extension add -n azure-cli-ml'
+    - task: AzureCLI@1
+      displayName: "Deploy to AKS (CLI)"
+      inputs:
+        azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+        scriptLocation: inlineScript
+        workingDirectory: $(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring
+        inlineScript: |
+          set -e # fail on error
+          
+          az ml model deploy --name $(AKS_DEPLOYMENT_NAME) --model '$(MODEL_NAME):$(MODEL_VERSION)' \
+          --compute-target $(AKS_COMPUTE_NAME) \
+          --ic inference_config.yml \
+          --dc deployment_config_aks.yml \
+          -g $(RESOURCE_GROUP) --workspace-name $(WORKSPACE_NAME) \
+          --overwrite -v
+    - task: AzureCLI@1
+      displayName: 'Smoke test'
+      inputs:
+        azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+        scriptLocation: inlineScript
+        inlineScript: |
+          set -e # fail on error
+          export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+          python -m ml_service.util.smoke_test_scoring_service --type AKS --service "$(AKS_DEPLOYMENT_NAME)"
+
+- stage: 'Deploy_Webapp'
+  displayName: 'Deploy to Webapp'
+  condition: variables['WEBAPP_DEPLOYMENT_NAME']
+  jobs:
+  - job: "Deploy_Webapp"
+    displayName: "Package and deploy model"
+    container: mlops
+    timeoutInMinutes: 0
+    steps:
+    - template: diabetes_regression-get-model-id-artifact-template.yml
+      parameters:
+        projectId: '$(resources.pipeline.model-train-ci.projectID)'
+        pipelineId: '$(resources.pipeline.model-train-ci.pipelineID)'
+        artifactBuildId: ${{ parameters.artifactBuildId }}
+    - template: diabetes_regression-package-model-template.yml
+      parameters:
+        modelId: $(MODEL_NAME):$(MODEL_VERSION)
+        scoringScriptPath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring/score.py'
+        condaFilePath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/conda_dependencies.yml'
+    - script: echo $(IMAGE_LOCATION) >image_location.txt
+      displayName: "Write image location file"
+    - task: AzureWebAppContainer@1
+      name: WebAppDeploy
+      displayName: 'Azure Web App on Container Deploy'
+      inputs:
+        azureSubscription: '$(AZURE_RM_SVC_CONNECTION)'
+        appName: '$(WEBAPP_DEPLOYMENT_NAME)'
+        resourceGroupName: '$(RESOURCE_GROUP)'
+        imageName: '$(IMAGE_LOCATION)'
+    - task: AzureCLI@1
+      displayName: 'Smoke test'
+      inputs:
+        azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+        scriptLocation: inlineScript
+        inlineScript: |
+          set -e # fail on error
+          export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+          python -m ml_service.util.smoke_test_scoring_service --type Webapp --service "$(WebAppDeploy.AppServiceApplicationUrl)/score"

.pipelines/diabetes_regression-ci-image.yml

@@ -30,14 +30,9 @@ variables:
   value: 'scoring/scoreB.py'
 
 steps:
-- task: AzureCLI@1
-  inputs:
-    azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
-    scriptLocation: inlineScript
-    workingDirectory: $(Build.SourcesDirectory)
-    inlineScript: |
-      set -e
-      export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
-      python3 -m ml_service.util.create_scoring_image
-  displayName: 'Create Scoring Image'
+- template: diabetes_regression-package-model-template.yml
+  parameters:
+    modelId: $(MODEL_NAME):$(MODEL_VERSION)
+    scoringScriptPath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/$(SCORE_SCRIPT)'
+    condaFilePath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/conda_dependencies.yml'
 

.pipelines/diabetes_regression-ci.yml

@@ -1,4 +1,4 @@
-# Continuous Integration (CI) pipeline that orchestrates the training, evaluation, registration, deployment, and testing of the diabetes_regression model.
+# Continuous Integration (CI) pipeline that orchestrates the training, evaluation, and registration of the diabetes_regression model.
 
 resources:
   containers:
@@ -27,7 +27,6 @@ pool:
 stages:
 - stage: 'Model_CI'
   displayName: 'Model CI'
-  condition: not(variables['MODEL_BUILD_ID'])
   jobs:
   - job: "Model_CI_Pipeline"
     displayName: "Model CI Pipeline"
@@ -48,8 +47,8 @@ stages:
       displayName: 'Publish Azure Machine Learning Pipeline'
 
 - stage: 'Trigger_AML_Pipeline'
-  displayName: 'Train model'
-  condition: and(succeeded(), not(variables['MODEL_BUILD_ID']))
+  displayName: 'Train and evaluate model'
+  condition: succeeded()
   variables:
     BUILD_URI: '$(SYSTEM.COLLECTIONURI)$(SYSTEM.TEAMPROJECT)/_build/results?buildId=$(BUILD.BUILDID)'
   jobs:
@@ -91,116 +90,8 @@ stages:
   - job: "Training_Run_Report"
     dependsOn: "Run_ML_Pipeline"
     condition: always()
-    displayName: "Determine if evaluation succeeded and new model is registered"
+    displayName: "Publish artifact if new model was registered"
     container: mlops
     timeoutInMinutes: 0
     steps:
-    - template: diabetes_regression-get-model-version-template.yml
-
-- stage: 'Deploy_ACI'
-  displayName: 'Deploy to ACI'
-  dependsOn: Trigger_AML_Pipeline
-  condition: and(or(succeeded(), variables['MODEL_BUILD_ID']), variables['ACI_DEPLOYMENT_NAME'])
-  jobs:
-  - job: "Deploy_ACI"
-    displayName: "Deploy to ACI"
-    container: mlops
-    timeoutInMinutes: 0
-    steps:
-    - template: diabetes_regression-get-model-version-template.yml
-    - task: ms-air-aiagility.vss-services-azureml.azureml-model-deploy-task.AMLModelDeploy@0
-      displayName: 'Azure ML Model Deploy'
-      inputs:
-        azureSubscription: $(WORKSPACE_SVC_CONNECTION)
-        modelSourceType: manualSpec
-        modelName: '$(MODEL_NAME)'
-        modelVersion: $(MODEL_VERSION)
-        inferencePath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring/inference_config.yml'
-        deploymentTarget: ACI
-        deploymentName: $(ACI_DEPLOYMENT_NAME)
-        deployConfig: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring/deployment_config_aci.yml'
-        overwriteExistingDeployment: true
-    - task: AzureCLI@1
-      displayName: 'Smoke test'
-      inputs:
-        azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
-        scriptLocation: inlineScript
-        inlineScript: |
-          set -e # fail on error
-          export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
-          python -m ml_service.util.smoke_test_scoring_service --type ACI --service "$(ACI_DEPLOYMENT_NAME)"
-
-- stage: 'Deploy_AKS'
-  displayName: 'Deploy to AKS'
-  dependsOn: Deploy_ACI
-  condition: and(succeeded(), variables['AKS_DEPLOYMENT_NAME'])
-  jobs:
-  - job: "Deploy_AKS"
-    displayName: "Deploy to AKS"
-    container: mlops
-    timeoutInMinutes: 0
-    steps:
-    - template: diabetes_regression-get-model-version-template.yml
-    - task: ms-air-aiagility.vss-services-azureml.azureml-model-deploy-task.AMLModelDeploy@0
-      displayName: 'Azure ML Model Deploy'
-      inputs:
-        azureSubscription: $(WORKSPACE_SVC_CONNECTION)
-        modelSourceType: manualSpec
-        modelName: '$(MODEL_NAME)'
-        modelVersion: $(MODEL_VERSION)
-        inferencePath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring/inference_config.yml'
-        deploymentTarget: AKS
-        aksCluster: $(AKS_COMPUTE_NAME)
-        deploymentName: $(AKS_DEPLOYMENT_NAME)
-        deployConfig: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring/deployment_config_aks.yml'
-        overwriteExistingDeployment: true
-    - task: AzureCLI@1
-      displayName: 'Smoke test'
-      inputs:
-        azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
-        scriptLocation: inlineScript
-        inlineScript: |
-          set -e # fail on error
-          export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
-          python -m ml_service.util.smoke_test_scoring_service --type AKS --service "$(AKS_DEPLOYMENT_NAME)"
-
-- stage: 'Deploy_Webapp'
-  displayName: 'Deploy to Webapp'
-  dependsOn: Trigger_AML_Pipeline
-  condition: and(or(succeeded(), variables['MODEL_BUILD_ID']), variables['WEBAPP_DEPLOYMENT_NAME'])
-  jobs:
-  - job: "Deploy_Webapp"
-    displayName: "Deploy to Webapp"
-    container: mlops
-    timeoutInMinutes: 0
-    steps:
-    - template: diabetes_regression-get-model-version-template.yml
-    - task: AzureCLI@1
-      displayName: 'Create scoring image and set IMAGE_LOCATION variable'
-      inputs:
-        azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
-        scriptLocation: inlineScript
-        inlineScript: |
-          set -e # fail on error
-          export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
-          python -m ml_service.util.create_scoring_image --output_image_location_file image_location.txt
-          # Output image location to Azure DevOps job
-          IMAGE_LOCATION="$(cat image_location.txt)"
-          echo "##vso[task.setvariable variable=IMAGE_LOCATION]$IMAGE_LOCATION"
-    - task: AzureWebAppContainer@1
-      name: WebAppDeploy
-      displayName: 'Azure Web App on Container Deploy'
-      inputs:
-        azureSubscription: '$(AZURE_RM_SVC_CONNECTION)'
-        appName: '$(WEBAPP_DEPLOYMENT_NAME)'
-        resourceGroupName: '$(RESOURCE_GROUP)'
-        imageName: '$(IMAGE_LOCATION)'
-    - task: AzureCLI@1
-      displayName: 'Smoke test'
-      inputs:
-        azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
-        scriptLocation: inlineScript
-        inlineScript: |
-          set -e # fail on error
-          export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
-          python -m ml_service.util.smoke_test_scoring_service --type Webapp --service "$(WebAppDeploy.AppServiceApplicationUrl)/score"
+    - template: diabetes_regression-publish-model-artifact-template.yml