Cosmos Pipeline Refactor (#7227)

* add emulator to ci.yml
* mark globaldb tests, try removing emulator from live tests
* use the accepted gating cosmos emulator url
* live tests
* remove redundant BuildTargetingString
* make coverage arg into a variable because of build-test.yml's construction
* remove extra comment, extra templates don't add value here given that this file is cosmos-specific already
* remove old cosmos pipelines
* remove extra comment

eng/pipelines/templates/jobs/archetype-sdk-tests.yml

@@ -4,6 +4,7 @@ parameters:
   MaxParallel: 0
   BeforeTestSteps: []
   BuildTargetingString: 'azure-*'
+  TestMarkArgument: ''
   Matrix:
     Linux_Python35:
       OSName: 'Linux'
@@ -22,6 +23,7 @@ jobs:
   - job: 'Test'
     variables:
       skipComponentGovernanceDetection: true
+      CoverageArg: --disablecov
 
     timeoutInMinutes: 120
     strategy:
@@ -37,9 +39,10 @@ jobs:
         parameters:
           BuildTargetingString: ${{ parameters.BuildTargetingString }}
           ServiceDirectory: ${{ parameters.ServiceDirectory }}
-          CoverageArg: '--disablecov'
+          CoverageArg: $(CoverageArg)
           EnvVars: ${{ parameters.EnvVars }}
           BeforeTestSteps: ${{ parameters.BeforeTestSteps }}
           PythonVersion: $(PythonVersion)
           OSName: $(OSName)
           ToxTestEnv: "whl"
+          TestMarkArgument: ${{ parameters.TestMarkArgument }}

eng/pipelines/templates/stages/cosmos-sdk-client.yml

@@ -0,0 +1,44 @@
+parameters:
+  Artifacts: []
+  ServiceDirectory: not-specified
+  EmulatorMsiUrl: 'https://aka.ms/cosmosdb-emulator'
+  EmulatorStartParameters: ''
+
+stages:
+  - stage: Build
+    jobs:
+      - template: ../jobs/archetype-sdk-client.yml
+        parameters:
+          ServiceDirectory: ${{ parameters.ServiceDirectory }}
+          # Skip emulator tests in the CI run
+          TestMarkArgument: not cosmosEmulator
+
+      - job: Emulator
+        strategy:
+          matrix:
+            Windows_Python35:
+              OSVmImage: 'windows-2019'
+              OSName: Windows
+              PythonVersion: '3.5'
+            Windows_Python27:
+              OSVmImage: 'windows-2019'
+              OSName: Windows
+              PythonVersion: '2.7'
+        pool:
+          vmImage: $(OSVmImage)
+
+        steps:
+          - template: eng/pipelines/templates/steps/cosmos-emulator.yml@azure-sdk-tools
+            parameters:
+              EmulatorMsiUrl: ${{ parameters.EmulatorMsiUrl }}
+              StartParameters: ${{ parameters.EmulatorStartParameters }}
+
+          - template: ../steps/build-test.yml
+            parameters:
+              TestMarkArgument: not globaldb
+              EnvVars:
+                ACCOUNT_HOST: https://localhost:8081/
+              ServiceDirectory: ${{ parameters.ServiceDirectory }}
+              PythonVersion: $(PythonVersion)
+              OSName: $(OSName)
+              ToxTestEnv: 'whl,sdist'

sdk/cosmos/azure-cosmos/test/globaldb_mock_tests.py

@@ -32,7 +32,7 @@
 from azure.cosmos import _retry_utility
 import test_config
 
-pytestmark = pytest.mark.cosmosEmulator
+pytestmark = [pytest.mark.cosmosEmulator, pytest.mark.globaldb]
 
 location_changed = False
 

sdk/cosmos/azure-cosmos/test/globaldb_tests.py

@@ -35,7 +35,7 @@
 from azure.cosmos.http_constants import HttpHeaders, StatusCodes, SubStatusCodes
 import test_config
 
-pytestmark = pytest.mark.cosmosEmulator
+pytestmark = [pytest.mark.cosmosEmulator, pytest.mark.globaldb]
 
 #IMPORTANT NOTES: 
 

sdk/cosmos/ci.yml

@@ -1,6 +1,13 @@
 # DO NOT EDIT THIS FILE
 # This file is generated automatically and any changes will be lost.
 
+resources:
+  repositories:
+    - repository: azure-sdk-tools
+      type: github
+      name: Azure/azure-sdk-tools
+      endpoint: azure
+
 trigger:
   branches:
     include:
@@ -21,8 +28,7 @@ pr:
     include:
     - sdk/cosmos/
 
-jobs:
-- template: ../../eng/pipelines/templates/jobs/archetype-sdk-client.yml
+stages:
+- template: ../../eng/pipelines/templates/stages/cosmos-sdk-client.yml
   parameters:
     ServiceDirectory: cosmos
-    TestMarkArgument: not cosmosEmulator

sdk/cosmos/tests.yml

@@ -0,0 +1,28 @@
+trigger: none
+
+jobs:
+  - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml
+    parameters:
+      MaxParallel: 1
+      Matrix:
+        Linux_Python35:
+          OSName: 'Linux'
+          OSVmImage: 'ubuntu-16.04'
+          PythonVersion: '3.5'
+        MacOs_Python37:
+          OSName: 'MacOS'
+          OSVmImage: 'macOS-10.14'
+          PythonVersion: '3.7'
+        Windows_Python27:
+          OSName: 'Windows'
+          OSVmImage: 'windows-2019'
+          PythonVersion: '2.7'
+        Linux_Pypy3:
+          OSName: 'Linux'
+          OSVmImage: 'ubuntu-16.04'
+          PythonVersion: 'pypy3'
+      ServiceDirectory: cosmos
+      TestMarkArgument: not globaldb
+      EnvVars:
+        ACCOUNT_KEY: $(python-cosmos-live-account-key)
+        ACCOUNT_HOST: $(python-cosmos-live-account-host)