[CI] Onboard pytest and deprecate nose for Automation Test (#13153)

.azure-pipelines/templates/automation_test.yml

@@ -0,0 +1,34 @@
+parameters:
+  pythonVersion: ''
+  profile: ''
+
+steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: ${{ parameters.pythonVersion }}
+      displayName: "Use Python ${{ parameters.pythonVersion }}"
+  - bash: |
+      set -ev
+
+      python -m venv env
+      chmod +x env/bin/activate
+      source env/bin/activate
+
+      pip install -q azdev
+      azdev --version
+
+      azdev setup -c ./
+    displayName: Setup Env
+  - bash: |
+      set -ev
+
+      source env/bin/activate
+
+      if [[ "$(System.PullRequest.TargetBranch)" != "" ]]; then
+        azdev test --series --repo=./ --src=HEAD --tgt=origin/$(System.PullRequest.TargetBranch) --cli-ci --profile ${{ parameters.profile }}
+      else
+        azdev test --series
+      fi
+    displayName: "Test on Profile ${{ parameters.profile }}"
+    env:
+      ADO_PULL_REQUEST_LATEST_COMMIT: $(System.PullRequest.TargetBranch)

azure-pipelines.yml

@@ -353,7 +353,7 @@ jobs:
        echo "== Testing pip install on Python 3.8 =="
        docker run \
          --rm -v $PYPI_FILES:/mnt/pypi python:3.8 \
-         /bin/bash -c "ls /mnt/pypi && pip install -f /mnt/pypi -q azure-cli==$CLI_VERSION && az self-test && az --version && sleep 5"  
+         /bin/bash -c "ls /mnt/pypi && pip install -f /mnt/pypi -q azure-cli==$CLI_VERSION && az self-test && az --version && sleep 5"
 
     displayName: 'Test pip Install'
 
@@ -407,146 +407,76 @@ jobs:
     - bash: ./scripts/ci/test_profile_integration.sh
       displayName: 'Run Integration Test against Profiles'
 
-- job: RunAutomationPython3_6
-  displayName: Run Automation Python 3.6
-  dependsOn: BuildPythonWheel
-  timeoutInMinutes: 90
-
+- job: AutomationTest
+  displayName: Automation Test (Profile Latest)
+  timeoutInMinutes: 120
   pool:
     vmImage: 'ubuntu-16.04'
+  strategy:
+    matrix:
+      Python36:
+        python.version: '3.6'
+      Python38:
+        python.version: '3.8'
   steps:
-  - task: UsePythonVersion@0
-    displayName: 'Use Python 3.6'
-    inputs:
-      versionSpec: 3.6
-  - task: DownloadPipelineArtifact@1
-    displayName: 'Download Python Wheels'
-    inputs:
-      TargetPath: '$(Build.ArtifactStagingDirectory)/pypi'
-      artifactName: pypi
-  - bash: |
-      pip install --force-reinstall wheel
-  - task: Bash@3
-    displayName: 'Run Test Automation'
-    inputs:
-      targetType: 'filePath'
-      filePath: ./scripts/ci/test_automation.sh
-    env:
-      ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch)
-
-- job: RunAutomationReduced20190301
-  displayName: Run Automation Reduced Python 3.6, Profile 2019-03-01
-  dependsOn: BuildPythonWheel
-
+    - template: .azure-pipelines/templates/automation_test.yml
+      parameters:
+        pythonVersion: '$(python.version)'
+        profile: 'latest'
+
+- job: AutomationTest20190301
+  displayName: Automation Test (Profile 2019-03-01)
+  timeoutInMinutes: 30
   pool:
     vmImage: 'ubuntu-16.04'
+  strategy:
+    matrix:
+      Python36:
+        python.version: '3.6'
+      Python38:
+        python.version: '3.8'
   steps:
-  - task: UsePythonVersion@0
-    displayName: 'Use Python 3.6'
-    inputs:
-      versionSpec: 3.6
-  - task: DownloadPipelineArtifact@1
-    displayName: 'Download Python Wheels'
-    inputs:
-      TargetPath: '$(Build.ArtifactStagingDirectory)/pypi'
-      artifactName: pypi
-  - bash: |
-      pip install --force-reinstall wheel
-  - task: Bash@3
-    displayName: 'Run Test Automation'
-    inputs:
-      targetType: 'filePath'
-      filePath: ./scripts/ci/test_automation.sh
-    env:
-      REDUCE_SDK: 'True'
-      AZURE_CLI_TEST_TARGET_PROFILE: '2019-03-01'
-      ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch)
-
-- job: RunAutomation20180301
-  displayName: Run Automation, Profile 2018-03-01
-  dependsOn: BuildPythonWheel
-
-  pool:
-    vmImage: 'ubuntu-16.04'
-  steps:
-  - task: UsePythonVersion@0
-    displayName: 'Use Python 3.6'
-    inputs:
-      versionSpec: 3.6
-  - task: DownloadPipelineArtifact@1
-    displayName: 'Download Python Wheels'
-    inputs:
-      TargetPath: '$(Build.ArtifactStagingDirectory)/pypi'
-      artifactName: pypi
-  - bash: |
-      pip install --force-reinstall wheel
-  - task: Bash@3
-    displayName: 'Run Test Automation'
-    inputs:
-      targetType: 'filePath'
-      filePath: ./scripts/ci/test_automation.sh
-    env:
-      AZURE_CLI_TEST_TARGET_PROFILE: '2018-03-01'
-      ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch)
-
-- job: RunAutomationPython3_8
-  displayName: Run Automation Python 3.8
-  dependsOn: BuildPythonWheel
-  timeoutInMinutes: 90
-
+    - template: .azure-pipelines/templates/automation_test.yml
+      parameters:
+        pythonVersion: '$(python.version)'
+        profile: '2019-03-01-hybrid'
+
+- job: AutomationTest20180301
+  displayName: Automation Test (Profile 2018-03-01)
+  timeoutInMinutes: 30
   pool:
     vmImage: 'ubuntu-16.04'
+  strategy:
+    matrix:
+      Python36:
+        python.version: '3.6'
+      Python38:
+        python.version: '3.8'
   steps:
-  - task: UsePythonVersion@0
-    displayName: 'Use Python 3.8'
-    inputs:
-      versionSpec: 3.8
-  - task: DownloadPipelineArtifact@1
-    displayName: 'Download Python Wheels'
-    inputs:
-      TargetPath: '$(Build.ArtifactStagingDirectory)/pypi'
-      artifactName: pypi
-  - bash: |
-      pip install --force-reinstall wheel
-  - task: Bash@3
-    displayName: 'Run Test Automation'
-    inputs:
-      targetType: 'filePath'
-      filePath: ./scripts/ci/test_automation.sh
-    env:
-      ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch)
-
-- job: RunAutomationReduced20190301Python3_8
-  displayName: Run Automation Reduced Python 3.8, Profile 2019-03-01
-  dependsOn: BuildPythonWheel
-
+    - template: .azure-pipelines/templates/automation_test.yml
+      parameters:
+        pythonVersion: '$(python.version)'
+        profile: '2018-03-01-hybrid'
+
+- job: AutomationTest20170309
+  displayName: Automation Test (Profile 2017-03-09)
+  timeoutInMinutes: 30
   pool:
     vmImage: 'ubuntu-16.04'
+  strategy:
+    matrix:
+      Python36:
+        python.version: '3.6'
+      Python38:
+        python.version: '3.8'
   steps:
-  - task: UsePythonVersion@0
-    displayName: 'Use Python 3.8'
-    inputs:
-      versionSpec: 3.8
-  - task: DownloadPipelineArtifact@1
-    displayName: 'Download Python Wheels'
-    inputs:
-      TargetPath: '$(Build.ArtifactStagingDirectory)/pypi'
-      artifactName: pypi
-  - bash: |
-      pip install --force-reinstall wheel
-  - task: Bash@3
-    displayName: 'Run Test Automation'
-    inputs:
-      targetType: 'filePath'
-      filePath: ./scripts/ci/test_automation.sh
-    env:
-      REDUCE_SDK: 'True'
-      AZURE_CLI_TEST_TARGET_PROFILE: '2019-03-01'
-      ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch)
+    - template: .azure-pipelines/templates/automation_test.yml
+      parameters:
+        pythonVersion: '$(python.version)'
+        profile: '2017-03-09-profile'
 
 - job: TestExtensionsLoading
   displayName: Test Extensions Loading
-  dependsOn: BuildPythonWheel
   condition: succeeded()
   timeoutInMinutes: 10
 
@@ -566,33 +496,6 @@ jobs:
     - bash: ./scripts/ci/test_extensions.sh
       displayName: 'Load extensions'
 
-- job: RunAutomation20180301Python3_8
-  displayName: Run Automation Python 3.8, Profile 2018-03-01
-  dependsOn: BuildPythonWheel
-
-  pool:
-    vmImage: 'ubuntu-16.04'
-  steps:
-  - task: UsePythonVersion@0
-    displayName: 'Use Python 3.8'
-    inputs:
-      versionSpec: 3.8
-  - task: DownloadPipelineArtifact@1
-    displayName: 'Download Python Wheels'
-    inputs:
-      TargetPath: '$(Build.ArtifactStagingDirectory)/pypi'
-      artifactName: pypi
-  - bash: |
-      pip install --force-reinstall wheel
-  - task: Bash@3
-    displayName: 'Run Test Automation'
-    inputs:
-      targetType: 'filePath'
-      filePath: ./scripts/ci/test_automation.sh
-    env:
-      AZURE_CLI_TEST_TARGET_PROFILE: '2018-03-01'
-      ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch)
-
 - job: BuildHomebrewFormula
   displayName: Build Homebrew Formula
 
@@ -1020,7 +923,7 @@ jobs:
     displayName: 'Download Ubuntu:Eoan Builds'
     inputs:
       TargetPath: '$(Build.ArtifactStagingDirectory)/debian'
-      artifactName: ubuntu-eoan    
+      artifactName: ubuntu-eoan
 
 
   - bash: |