Merge branch 'main' into legacy-typing

Author: MJoshuaB

.github/CODEOWNERS

@@ -69,3 +69,9 @@
 /src/swift                               @tjprescott
 /src/java/apiview-java-processor         @JonathanGiles
 /src/go                                  @jhendrixMSFT @chlowell @RickWinter
+
+
+###########
+# Pylint checkers
+###########
+/tools/pylint-extensions/azure-pylint-guidelines-checker @l0lawrence

eng/common/TestResources/SubConfig-Helpers.ps1

@@ -82,7 +82,8 @@ function ShouldMarkValueAsSecret([string]$serviceName, [string]$key, [string]$va
         "TenantId",
         "TestApplicationId",
         "TestApplicationOid",
-        "ProvisionerApplicationId"
+        "ProvisionerApplicationId",
+        "ProvisionerApplicationOid"
     )
 
     $serviceDirectoryPrefix = BuildServiceDirectoryPrefix $serviceName
@@ -129,7 +130,7 @@ function SetSubscriptionConfiguration([object]$subscriptionConfiguration)
     return $subscriptionConfiguration
 }
 
-function UpdateSubscriptionConfiguration([object]$subscriptionConfigurationBase, [object]$subscriptionConfiguration)
+function UpdateSubscriptionConfiguration([object]$subscriptionConfigurationBase, [object]$subscriptionConfiguration, [array]$allowedValues)
 {
     foreach ($pair in $subscriptionConfiguration.GetEnumerator()) {
         if ($pair.Value -is [Hashtable]) {
@@ -140,13 +141,13 @@ function UpdateSubscriptionConfiguration([object]$subscriptionConfigurationBase,
                 # Mark values as secret so we don't print json blobs containing secrets in the logs.
                 # Prepend underscore to the variable name, so we can still access the variable names via environment
                 # variables if they get set subsequently.
-                if (ShouldMarkValueAsSecret "AZURE_" $nestedPair.Name $nestedPair.Value) {
+                if (ShouldMarkValueAsSecret "AZURE_" $nestedPair.Name $nestedPair.Value $allowedValues) {
                     Write-Host "##vso[task.setvariable variable=_$($nestedPair.Name);issecret=true;]$($nestedPair.Value)"
                 }
                 $subscriptionConfigurationBase[$pair.Name][$nestedPair.Name] = $nestedPair.Value
             }
         } else {
-            if (ShouldMarkValueAsSecret "AZURE_" $pair.Name $pair.Value) {
+            if (ShouldMarkValueAsSecret "AZURE_" $pair.Name $pair.Value $allowedValues) {
                 Write-Host "##vso[task.setvariable variable=_$($pair.Name);issecret=true;]$($pair.Value)"
             }
             $subscriptionConfigurationBase[$pair.Name] = $pair.Value
@@ -155,3 +156,66 @@ function UpdateSubscriptionConfiguration([object]$subscriptionConfigurationBase,
 
     return $subscriptionConfigurationBase
 }
+
+# Helper function for processing sub config files from a pipeline file list yaml parameter
+function UpdateSubscriptionConfigurationWithFiles([object]$baseSubConfig, [string]$fileListJson) {
+  if (!$fileListJson) {
+    return $baseSubConfig
+  }
+
+  $finalConfig = $baseSubConfig
+
+  $subConfigFiles = $fileListJson | ConvertFrom-Json -AsHashtable
+  foreach ($file in $subConfigFiles) {
+    # In some cases, $file could be an empty string. Get-Content will fail
+    # if $file is an empty string, so skip those cases.
+    if (!$file) {
+      continue
+    }
+
+    Write-Host "Merging sub config from file: $file"
+    $subConfig = Get-Content $file | ConvertFrom-Json -AsHashtable
+    $allowedValues = @()
+    # Since the keys are all coming from a file in github, we know every key should not be marked
+    # as a secret. Set up these exclusions here to make pipeline log debugging easier.
+    foreach ($pair in $subConfig.GetEnumerator()) {
+      if ($pair.Value -is [Hashtable]) {
+        foreach($nestedPair in $pair.Value.GetEnumerator()) {
+          $allowedValues += $nestedPair.Name
+        }
+      } else {
+        $allowedValues += $pair.Name
+      }
+    }
+    $finalConfig = UpdateSubscriptionConfiguration $finalConfig $subConfig $allowedValues
+  }
+
+  return $finalConfig
+}
+
+# Helper function for processing stringified json sub configs from pipeline parameter data
+function BuildAndSetSubscriptionConfig([string]$baseSubConfigJson, [string]$additionalSubConfigsJson, [string]$subConfigFilesJson) {
+  $finalConfig = @{}
+  if ($baseSubConfigJson) {
+    $baseSubConfig = $baseSubConfigJson | ConvertFrom-Json -AsHashtable
+
+    Write-Host "Setting base sub config"
+    $finalConfig = SetSubscriptionConfiguration $baseSubConfig
+  }
+
+  if ($additionalSubConfigsJson) {
+    $subConfigs = $additionalSubConfigsJson | ConvertFrom-Json -AsHashtable
+
+    foreach ($subConfig in $subConfigs) {
+      Write-Host "Merging sub config from list"
+      $finalConfig = UpdateSubscriptionConfiguration $finalConfig $subConfig
+    }
+  }
+
+  Write-Host "Merging sub config from files"
+  $finalConfig = UpdateSubscriptionConfigurationWithFiles $finalConfig $subConfigFilesJson
+
+  Write-Host ($finalConfig | ConvertTo-Json)
+  $serialized = $finalConfig | ConvertTo-Json -Compress
+  Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]$serialized"
+}

eng/common/TestResources/build-test-resource-config.yml

@@ -37,50 +37,17 @@ steps:
   - pwsh: |
       . ./eng/common/TestResources/SubConfig-Helpers.ps1
 
-      $finalConfig = @{}
-      $baseSubConfigRaw = @'
+      $baseSubConfigJson = @'
         ${{ parameters.SubscriptionConfiguration }}
       '@.Trim()
-      if ($baseSubConfigRaw) {
-        $baseSubConfig = $baseSubConfigRaw | ConvertFrom-Json -AsHashtable
 
-        Write-Host "Setting base sub config"
-        $finalConfig = SetSubscriptionConfiguration $baseSubConfig
-      }
-
-      $subConfigJsonsRaw = @'
+      $additionalSubConfigsJson = @'
         ${{ convertToJson(parameters.SubscriptionConfigurations) }}
       '@.Trim() -replace '"{', '{' -replace '}"', '}'
 
-      if ($subConfigJsonsRaw) {
-        $subConfigs = $subConfigJsonsRaw | ConvertFrom-Json -AsHashtable
-
-        foreach ($subConfig in $subConfigs) {
-          Write-Host "Merging sub config from list"
-          $finalConfig = UpdateSubscriptionConfiguration $finalConfig $subConfig
-        }
-      }
-
-      $subConfigFilesRaw = @'
+      $subConfigFilesJson = @'
         ${{ convertToJson(parameters.SubscriptionConfigurationFilePaths) }}
       '@.Trim()
 
-      if ($subConfigFilesRaw) {
-        $subConfigFiles = $subConfigFilesRaw | ConvertFrom-Json -AsHashtable
-        foreach ($file in $subConfigFiles) {
-          # In some cases, $file could be an empty string. Get-Content will fail
-          # if $file is an empty string, so skip those cases.
-          if (!$file) {
-            continue
-          }
-
-          Write-Host "Merging sub config from file: $file"
-          $subConfig = Get-Content $file | ConvertFrom-Json -AsHashtable
-          $finalConfig = UpdateSubscriptionConfiguration $finalConfig $subConfig
-        }
-      }
-
-      Write-Host ($finalConfig | ConvertTo-Json)
-      $serialized = $finalConfig | ConvertTo-Json -Compress
-      Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]$serialized"
+      BuildAndSetSubscriptionConfig $baseSubConfigJson $additionalSubConfigsJson $subConfigFilesJson
     displayName: Merge subscription configurations

eng/common/TestResources/sub-config/AzureChinaMsft.json

@@ -0,0 +1,12 @@
+{
+    "SubscriptionId": "44b2f395-b47b-462a-9a40-6ec719b6b910",
+    "TenantId": "3d0a72e2-8b06-4528-98df-1391c6f12c11",
+    "TestApplicationId": "5296eef3-d187-4e3e-bd80-7a05c38c389b",
+    "TestApplicationSecret": "",
+    "TestApplicationOid": "0d7672d5-e1db-42d4-a85e-add73010cfe5",
+    "ProvisionerApplicationId": "5296eef3-d187-4e3e-bd80-7a05c38c389b",
+    "ProvisionerApplicationSecret": "",
+    "ProvisionerApplicationOid": "0d7672d5-e1db-42d4-a85e-add73010cfe5",
+    "Environment": "AzureChinaCloud",
+    "AzureSubscription": "azuresdkchina"
+}

eng/common/TestResources/sub-config/AzurePreviewMsft.json

@@ -0,0 +1,12 @@
+{
+    "SubscriptionId": "23fddbc8-cb64-4b59-ba97-4c9f77c212e4",
+    "TenantId": "72f988bf-86f1-41af-91ab-2d7cd011db47",
+    "TestApplicationId": "f850650c-1fcf-4489-b46f-71af2e30d360",
+    "TestApplicationSecret": "",
+    "TestApplicationOid": "30511c9d-ba1a-4c7b-b422-5b543da11b3f",
+    "ProvisionerApplicationId": "f850650c-1fcf-4489-b46f-71af2e30d360",
+    "ProvisionerApplicationSecret": "",
+    "ProvisionerApplicationOid": "30511c9d-ba1a-4c7b-b422-5b543da11b3f",
+    "Environment": "AzureCloud",
+    "AzureSubscription": "Azure SDK Test Resources - Preview"
+}

eng/common/TestResources/sub-config/AzureUsGovMsft.json

@@ -0,0 +1,12 @@
+{
+    "SubscriptionId": "4ad9eeaf-9573-4288-8357-1445436f34cd",
+    "TenantId": "63296244-ce2c-46d8-bc36-3e558792fbee",
+    "TestApplicationId": "48104cb1-6c74-4857-a8b6-f44d5cf43438",
+    "TestApplicationSecret": "",
+    "TestApplicationOid": "431d66d1-2713-4135-a0b6-28ea5abd3411",
+    "ProvisionerApplicationId": "48104cb1-6c74-4857-a8b6-f44d5cf43438",
+    "ProvisionerApplicationSecret": "",
+    "ProvisionerApplicationOid": "431d66d1-2713-4135-a0b6-28ea5abd3411",
+    "Environment": "AzureUSGovernment",
+    "AzureSubscription": "azuresdkgov"
+}

eng/common/pipelines/templates/steps/set-daily-docs-branch-name.yml

@@ -6,8 +6,8 @@ parameters:
 steps:
   - pwsh: |
       $branchName = $env:DAILYDOCSBRANCHNAMEOVERRIDE
-      if (!$branchName) { 
-        $branchName = "daily/$(Get-Date -Format 'yyyy-MM-dd')"
+      if (!$branchName) {
+        $branchName = "daily/$(Get-Date -Format 'yyyy-MM-dd')-ignore-build"
       }
       Write-Host "Daily Branch Name: $branchName"
       Write-Host "##vso[task.setvariable variable=${{ parameters.DailyBranchVariableName }};]$branchName"

eng/common/testproxy/dotnet-devcert.crt

@@ -1,21 +1,21 @@
 -----BEGIN CERTIFICATE-----
-MIIDZzCCAk+gAwIBAgIUXjY6UxqL53TvxH8dtPNZm6/getIwDQYJKoZIhvcNAQEL
-BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTIzMDcyODIxMDM1MloXDTI0MDcy
-NzIxMDM1MlowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF
-AAOCAQ8AMIIBCgKCAQEAsTPe57bim6NTmBHS1ldLAr7dnMT7AMtfttfjqPzggDgR
-kNKj/mi5Xd86AOR6QfLMMipdL2NpPTZP557t4V1oVgODv+M3SiKRriY01TNbL9K5
-zjYlPDik1BzKJgHiLmHPmuKsWslTTMO86nau5YNzKfaOIFbiV5uTUCLTZh3NspDo
-OaIeJ4Efud/6bHQkbIXggAt2TFjVum+jMXLYeyA8ZjwFgW1ENAlOOV5Gm8eFjkIt
-OhQSZLlLc9BnJkOAhT0v6Xq0oRwCm0YW42+JFzmIvjK0cU/sFmjDqzKAxhtWexz7
-WT7KDiJU+GNsZmm8KjeU0EaQpzpK8q/MEbNRX1OqrQIDAQABo4GwMIGtMA8GA1Ud
+MIIDZzCCAk+gAwIBAgIUUGsnTw0cjASpYrNVxo7S41oMGtUwDQYJKoZIhvcNAQEL
+BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTI0MDcyODA2NTAzOFoXDTI1MDcy
+ODA2NTAzOFowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF
+AAOCAQ8AMIIBCgKCAQEAs9w2heE1VvnZcD3aR3jLgI/tiwjJScf+ljOMor9lxcIA
+80NvcnfH3T7eH4Gsk0xcQ+J94FpFC5sDcsumcuEV1W6flvqj9k8vASoBpvpkoUnA
+tr6aChLEp5hpwz59NAhdLrzd1eU+PNtB2CS0Blb6OEKUsVvoscmP5BwOWe+fuAdW
+rQisHYrYzT9K5oWQA+AntrRJPbow7LCGQtT1viIlqxKaI8mTor+aRB0AmrAMJlPZ
+4scmrpNs7+ertRVTz+uZ6HWHN5CJsVKnFKgTTuYYustRV2Oz9tF4W0+j1fBKCkYM
+WQdga3md/9t51mG0naezJNDsMlT45IFOadqOozqsswIDAQABo4GwMIGtMA8GA1Ud
 EwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGmMBYGA1UdJQEB/wQMMAoGCCsGAQUF
 BwMBMBcGA1UdEQEB/wQNMAuCCWxvY2FsaG9zdDA6BgorBgEEAYI3VAEBBCwMKkFT
 UC5ORVQgQ29yZSBIVFRQUyBkZXZlbG9wbWVudCBjZXJ0aWZpY2F0ZTAdBgNVHQ4E
-FgQU+9BDdfBwXH38ti7IGpL4Kn7IDVgwDQYJKoZIhvcNAQELBQADggEBAKoy2dp8
-gsKk1U51164IIhOYisaJ1egOI0++STd8j94SX9XlCAgwLaPr/mU/U438xKI1inSA
-Miaboqtt2tqvfP5nceSYL3FPJ3K0ADw8UFgwApKiRYpRevAIspG+OaqHwHUFRhyG
-bxkUZ4w96IEpVtDOGoy12sCmChZgdVk44+y8uurSza18Vj1LfkrN6ppZLt4FII5e
-p8BFKtqCRToFRJIIjMePOdTUbeRwUCjBPyYv/h5jcJUfFXQJpPXvJs4LFcUivqA3
-sAut3tut6CDzToTMtAD5ebxSPh3DDM6JOsWhrGoT0if5qoio75tG6yV40gi3Tocr
-KCwDu8B2O2HlL5Q=
+FgQUhYGoJsA6zisErkL5EcbKAO3LoQ8wDQYJKoZIhvcNAQELBQADggEBAHOBgMVZ
+a27y41edktHaLC4kayKHe3bLBtiXUd1vjWEZaPrcvQH6FItJdB5Z3xbi/4LA6QuU
+dZ33z44Lt4AzOiyvR7AGojytM2UYimtBBOFvi9CZuyrMC6ls4SLy6c/Rq1ATmyet
+QfkT1a7awC1cJUn7jeuHfly7hnYqrw+67HFY7Ajugv+Uwu6wCessKloWM127AZj/
+V0Gw0rblOmX+uX7fvH3BrYOkrNiHQdzCYZ1P1WU7eWm55JgBfaCrheNZ4L5TVeB2
+uTZPnlsBKQ23qgZ0dtcJzL0scC7zSuYI+qm1QNNyz13K+1ORXPQGQJXULlt7UeHB
+ZYN060pO6IO4Ti8=
 -----END CERTIFICATE-----

eng/common/testproxy/dotnet-devcert.pfx

@@ -14,7 +14,7 @@ In that case, the assets.json should live alongside the ci.yml in the sdk/<Servi
 
 Generated assets.json file contents
 - AssetsRepo: "Azure/azure-sdk-assets" - This is the assets repository, aka where your recordings will live after this script runs.
-- AssetsRepoPrefixPath: "<language>" - this is will be computed from repository it's being run in. 
+- AssetsRepoPrefixPath: "<language>" - this is will be computed from repository it's being run in.
 - TagPrefix: "<language>/<ServiceDirectory>" or "<language>/<ServiceDirectory>/<library>" or deeper if things
               are nested in such a manner. All tags created for this assets.json will start with this name.
 - Tag: "" - Initially empty, as nothing has yet been pushed.
@@ -81,6 +81,7 @@ $LangRecordingDirs = @{"cpp" = "recordings";
 . (Join-Path $PSScriptRoot "common-asset-functions.ps1")
 
 Test-Exe-In-Path -ExeToLookFor $GitExe
+
 $language = Get-Repo-Language
 
 # If the initial push is being performed, ensure that test-proxy is
@@ -89,7 +90,7 @@ $language = Get-Repo-Language
 if ($InitialPush) {
   $proxyPresent = Test-Exe-In-Path -ExeToLookFor $TestProxyExe -ExitOnError $false
 
-  # try to fall back 
+  # try to fall back
   if (-not $proxyPresent) {
     $StandaloneTestProxyExe = "Azure.Sdk.Tools.TestProxy"
 
@@ -107,6 +108,17 @@ if ($InitialPush) {
       Write-Error "The user has selected option InitialPush to push their assets, neither $TestProxyExe nor standalone executable $StandaloneTestProxyExe are installed on this machine."
       exit 1
     }
+
+    # if we're pushing, we also need to confirm that the necessary git configuration items are set
+    $result = git config --get user.name
+    if ($LASTEXITCODE -ne 0 -or !$result){
+      Write-Error "The git config setting `"user.name`" is unset. Set it to your git user name via 'git config --global user.name `"<setting>`'"
+    }
+
+    $result = git config --get user.email
+    if ($LASTEXITCODE -ne 0 -or !$result){
+      Write-Error "The git config setting `"user.email`" is unset. Set it to your git email via 'git config --global user.email `"<setting>`'"
+    }
   }
 
   if ($TestProxyExe -eq "test-proxy" -or $TestProxyExe.StartsWith("Azure.Sdk.Tools.TestProxy")) {