Joharrington/Workspace Managed Network

.vscode/cspell.json

@@ -396,6 +396,13 @@
         "Phong"
       ]
     },
+    {
+      "filename": "sdk/core/azure-core/tests/test_serialization.py",
+      "words": [
+        "Rlcw",
+        "Jwcmlud"
+      ]
+    },
     {
       "filename": "sdk/tables/azure-data-tables/tests/**/*.py",
       "words": [

eng/black-pyproject.toml

@@ -0,0 +1,8 @@
+[tool.black]
+line-length = 120
+extend-exclude = '''
+# Exclude some directories from formatting (note the slashes surrounding the pattern)
+/(
+  azure/ai/ml/_restclient
+)/
+'''

eng/common/pipelines/templates/jobs/perf.yml

@@ -2,18 +2,15 @@ parameters:
 - name: JobName
   type: string
   default: 'Perf'
+- name: TimeoutInMinutes
+  type: number
+  default: '360'
 - name: LinuxPool
   type: string
-  default: 'azsdk-pool-mms-ubuntu-2004-perf'
-- name: LinuxVmImage
-  type: string
-  default: 'MMSUbuntu20.04'
+  default: 'azsdk-pool-mms-ubuntu-2204-perf'
 - name: WindowsPool
   type: string
-  default: 'azsdk-pool-mms-win-2019-perf'
-- name: WindowsVmImage
-  type: string
-  default: 'MMS2019'
+  default: 'azsdk-pool-mms-win-2022-perf'
 - name: Language
   type: string
   default: ''
@@ -65,18 +62,16 @@ parameters:
 
 jobs:
 - job: ${{ parameters.JobName }}
-  timeoutInMinutes: 360
+  timeoutInMinutes: ${{ parameters.TimeoutInMinutes }}
   strategy:
     matrix:
       ${{ if contains(parameters.OperatingSystems, 'Linux') }}:
         Linux:
           Pool: ${{ parameters.LinuxPool }}
-          OsVmImage: ${{ parameters.LinuxVmImage }}
           MatrixName: 'Linux'
       ${{ if contains(parameters.OperatingSystems, 'Windows') }}:
         Windows:
           Pool: ${{ parameters.WindowsPool }}
-          OsVmImage: ${{ parameters.WindowsVmImage }}
           MatrixName: 'Windows'
   variables:
   - ${{ parameters.Variables }}
@@ -87,7 +82,6 @@ jobs:
       value: ''
   pool:
     name: $(Pool)
-    vmImage: $(OSVmImage)
   steps:
 
   - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
@@ -106,10 +100,6 @@ jobs:
         Commitish: ${{ parameters.ToolsRepoCommitish }}
         WorkingDirectory: $(System.DefaultWorkingDirectory)/azure-sdk-tools
 
-  - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml
-    parameters:
-      AgentImage: $(OSVmImage)
-
   - ${{ parameters.InstallLanguageSteps }}
 
   - template: /eng/common/TestResources/deploy-test-resources.yml

eng/common/scripts/get-codeowners.ps1

@@ -1,72 +1,151 @@
+<#
+.SYNOPSIS
+A script that given as input $TargetPath param, returns the owners
+of that path, as determined by CODEOWNERS file passed in $CodeOwnersFileLocation
+param.
+
+.PARAMETER TargetPath
+Required*. Path to file or directory whose owners are to be determined from a 
+CODEOWNERS file. e.g. sdk/core/azure-amqp/ or sdk/core/foo.txt.
+
+*for backward compatibility, you might provide $TargetDirectory instead.
+
+.PARAMETER TargetDirectory
+Obsolete. Replaced by $TargetPath. Kept for backward-compatibility.
+If both $TargetPath and $TargetDirectory are provided, $TargetDirectory is
+ignored.
+
+.PARAMETER CodeOwnerFileLocation
+Optional. An absolute path to the CODEOWNERS file against which the $TargetPath param
+will be checked to determine its owners.
+
+.PARAMETER ToolVersion
+Optional. The NuGet package version of the package containing the "retrieve-codeowners" 
+tool, around which this script is a wrapper.
+
+.PARAMETER ToolPath
+Optional. The place to check the "retrieve-codeowners" tool existence.
+
+.PARAMETER DevOpsFeed
+Optional. The NuGet package feed from which the "retrieve-codeowners" tool is to be installed.
+
+NuGet feed:
+https://dev.azure.com/azure-sdk/public/_artifacts/feed/azure-sdk-for-net/NuGet/Azure.Sdk.Tools.RetrieveCodeOwners
+
+Pipeline publishing the NuGet package to the feed, "tools - code-owners-parser":
+https://dev.azure.com/azure-sdk/internal/_build?definitionId=3188
+
+.PARAMETER VsoVariable 
+Optional. If provided, the determined owners, based on $TargetPath matched against CODEOWNERS file at $CodeOwnerFileLocation, 
+will be output to Azure DevOps pipeline log as variable named $VsoVariable.
+
+Reference:
+https://learn.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch
+https://learn.microsoft.com/en-us/azure/devops/pipelines/scripts/logging-commands?view=azure-devops&tabs=bash#logging-command-format
+
+.PARAMETER IncludeNonUserAliases
+Optional. Whether to include in the returned owners list aliases that are team aliases, e.g. Azure/azure-sdk-team
+
+.PARAMETER Test
+Optional. Whether to run the script against hard-coded tests.
+
+#>
 param (
-  [string]$TargetDirectory = "", # Code path to code owners. e.g sdk/core/azure-amqp
-  [string]$CodeOwnerFileLocation = (Resolve-Path $PSScriptRoot/../../../.github/CODEOWNERS), # The absolute path of CODEOWNERS file. 
-  [string]$ToolVersion = "1.0.0-dev.20230108.6", 
-  [string]$ToolPath = (Join-Path ([System.IO.Path]::GetTempPath()) "codeowners-tool-path"), # The place to check the tool existence. Put temp path as default
-  [string]$DevOpsFeed = "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json", # DevOp tool feeds.
-  [string]$VsoVariable = "", # Option of write code owners into devop variable
-  [switch]$IncludeNonUserAliases, # Option to filter out the team alias in code owner list. e.g. Azure/azure-sdk-team
-  [switch]$Test  #Run test functions against the script logic
+  [string]$TargetPath = "",
+  [string]$TargetDirectory = "",
+  # The path used assumes the script is located in azure-sdk-tools/eng/common/scripts/get-codeowners.ps1
+  [string]$CodeOwnerFileLocation = (Resolve-Path $PSScriptRoot/../../../.github/CODEOWNERS),
+  # The $ToolVersion 1.0.0-dev.20230214.3 includes following PR:
+  #    Use CodeownersFile.UseRegexMatcherDefault everywhere where applicable + remove obsolete tests
+  #    https://github.com/Azure/azure-sdk-tools/pull/5437
+  # 
+  # but not this one:
+  #    Remove the obsolete, prefix-based CODEOWNERS matcher & related tests
+  #    https://github.com/Azure/azure-sdk-tools/pull/5431
+  [string]$ToolVersion = "1.0.0-dev.20230214.3",
+  [string]$ToolPath = (Join-Path ([System.IO.Path]::GetTempPath()) "codeowners-tool-path"),
+  [string]$DevOpsFeed = "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json",
+  [string]$VsoVariable = "",
+  [switch]$IncludeNonUserAliases,
+  [switch]$Test
 )
 
-function Get-CodeOwnersTool()
+function Get-CodeownersTool()
 {
-  $command = Join-Path $ToolPath "retrieve-codeowners"
-  # Check if the retrieve-codeowners tool exsits or not.
-  if (Get-Command $command -errorAction SilentlyContinue) {
-    return $command
+  $codeownersToolCommand = Join-Path $ToolPath "retrieve-codeowners"
+  # Check if the retrieve-codeowners tool exists or not.
+  if (Get-Command $codeownersToolCommand -errorAction SilentlyContinue) {
+    return $codeownersToolCommand
   }
   if (!(Test-Path $ToolPath)) {
     New-Item -ItemType Directory -Path $ToolPath | Out-Null
   }
-  Write-Host "Installing the retrieve-codeowners tool under $ToolPath... "
+  Write-Host "Installing the retrieve-codeowners tool under tool path: $ToolPath ..."
 
   # Run command under tool path to avoid dotnet tool install command checking .csproj files. 
   # This is a bug for dotnet tool command. Issue: https://github.com/dotnet/sdk/issues/9623
   Push-Location $ToolPath
   dotnet tool install --tool-path $ToolPath --add-source $DevOpsFeed --version $ToolVersion "Azure.Sdk.Tools.RetrieveCodeOwners" | Out-Null
   Pop-Location
   # Test to see if the tool properly installed.
-  if (!(Get-Command $command -errorAction SilentlyContinue)) {
-    Write-Error "The retrieve-codeowners tool is not properly installed. Please check your tool path. $ToolPath"
+  if (!(Get-Command $codeownersToolCommand -errorAction SilentlyContinue)) {
+    Write-Error "The retrieve-codeowners tool is not properly installed. Please check your tool path: $ToolPath"
     return 
   }
-  return $command
+  return $codeownersToolCommand
 }
 
-function Get-CodeOwners ([string]$targetDirectory, [string]$codeOwnerFileLocation, [bool]$includeNonUserAliases = $false)
+function Get-Codeowners(
+  [string]$targetPath,
+  [string]$targetDirectory,
+  [string]$codeownersFileLocation,
+  [bool]$includeNonUserAliases = $false)
 {
-  $command = Get-CodeOwnersTool
-  # Filter out the non user alias from code owner list.
-  if($includeNonUserAliases) {
-    $codeOwnersString = & $command --target-directory $targetDirectory --code-owner-file-path $codeOwnerFileLocation 2>&1
-  }
-  else {
-    $codeOwnersString = & $command --target-directory $targetDirectory --code-owner-file-path $codeOwnerFileLocation --filter-out-non-user-aliases 2>&1
+  # Backward compaitiblity: if $targetPath is not provided, fall-back to the legacy $targetDirectory
+  if ([string]::IsNullOrWhiteSpace($targetPath)) {
+    $targetPath = $targetDirectory
   }
-  # Failed at the command of fetching code owners.
-  if ($LASTEXITCODE -ne 0) {
-    Write-Host $codeOwnersString
+  if ([string]::IsNullOrWhiteSpace($targetPath)) {
+    Write-Error "TargetPath (or TargetDirectory) parameter must be neither null nor whitespace."
     return ,@()
   }
-
-  $codeOwnersJson = $codeOwnersString | ConvertFrom-Json
-  if (!$codeOwnersJson) {
-    Write-Host "No code owners returned from the path: $targetDirectory"
+
+  $codeownersToolCommand = Get-CodeownersTool
+  Write-Host "Executing: & $codeownersToolCommand --target-path $targetPath --codeowners-file-path-or-url $codeownersFileLocation --exclude-non-user-aliases:$(!$includeNonUserAliases)"
+  $commandOutput = & $codeownersToolCommand `
+      --target-path $targetPath `
+      --codeowners-file-path-or-url $codeownersFileLocation `
+      --exclude-non-user-aliases:$(!$includeNonUserAliases) `
+      2>&1
+
+  if ($LASTEXITCODE -ne 0) {
+    Write-Host "Command $codeownersToolCommand execution failed (exit code = $LASTEXITCODE). Output string: $commandOutput"
     return ,@()
+  } else
+  {
+    Write-Host "Command $codeownersToolCommand executed successfully (exit code = 0). Output string length: $($commandOutput.length)"
   }
+
+# Assert: $commandOutput is a valid JSON representing:
+# - a single CodeownersEntry, if the $targetPath was a single path
+# - or a dictionary of CodeownerEntries, keyes by each path resolved from a $targetPath glob path.
+#
+# For implementation details, see Azure.Sdk.Tools.RetrieveCodeOwners.Program.Main
+
+$codeownersJson = $commandOutput | ConvertFrom-Json
 
   if ($VsoVariable) {
-    $codeOwners = $codeOwnersJson.Owners -join ","
-    Write-Host "##vso[task.setvariable variable=$VsoVariable;]$codeOwners"
+    $codeowners = $codeownersJson.Owners -join ","
+    Write-Host "##vso[task.setvariable variable=$VsoVariable;]$codeowners"
   }
 
-  return ,@($codeOwnersJson.Owners)
+  return ,@($codeownersJson.Owners)
 }
 
-function TestGetCodeOwner([string]$targetDirectory, [string]$codeOwnerFileLocation, [bool]$includeNonUserAliases = $false, [string[]]$expectReturn) {
-  Write-Host "Testing on $targetDirectory..."
-  $actualReturn = Get-CodeOwners -targetDirectory $targetDirectory -codeOwnerFileLocation $codeOwnerFileLocation -includeNonUserAliases $IncludeNonUserAliases
+function TestGetCodeowners([string]$targetPath, [string]$codeownersFileLocation, [bool]$includeNonUserAliases = $false, [string[]]$expectReturn) {
+  Write-Host "Test: find owners matching '$targetPath' ..."
+
+  $actualReturn = Get-Codeowners -targetPath $targetPath -codeownersFileLocation $codeownersFileLocation -includeNonUserAliases $IncludeNonUserAliases
 
   if ($actualReturn.Count -ne $expectReturn.Count) {
     Write-Error "The length of actual result is not as expected. Expected length: $($expectReturn.Count), Actual length: $($actualReturn.Count)."
@@ -80,16 +159,19 @@ function TestGetCodeOwner([string]$targetDirectory, [string]$codeOwnerFileLocati
   }
 }
 
-if($Test) {
-  $testFile = (Resolve-Path $PSScriptRoot/../../../tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/simple_path_CODEOWNERS)
-  TestGetCodeOwner -targetDirectory "sdk" -codeOwnerFileLocation $testFile -includeNonUserAliases $true -expectReturn @("person1", "person2")
-  TestGetCodeOwner -targetDirectory "sdk/noPath" -codeOwnerFileLocation $testFile -includeNonUserAliases $true -expectReturn @("person1", "person2")
-  TestGetCodeOwner -targetDirectory "/sdk/azconfig" -codeOwnerFileLocation $testFile -includeNonUserAliases $true -expectReturn @("person3", "person4")
-  TestGetCodeOwner -targetDirectory "/sdk/azconfig/package" -codeOwnerFileLocation $testFile -includeNonUserAliases $true  $testFile -expectReturn @("person3", "person4")
-  TestGetCodeOwner -targetDirectory "/sd" -codeOwnerFileLocation $testFile -includeNonUserAliases $true  -expectReturn @()
-  TestGetCodeOwner -targetDirectory "/sdk/testUser/" -codeOwnerFileLocation $testFile -expectReturn @("azure-sdk") 
+if ($Test) {
+  # Most of tests here have been removed; now instead we should run tests from RetrieveCodeOwnersProgramTests, and in a way as explained in:
+  # https://github.com/Azure/azure-sdk-tools/issues/5434
+  # https://github.com/Azure/azure-sdk-tools/pull/5103#discussion_r1068680818
+  Write-Host "Running reduced test suite at `$PSScriptRoot of $PSSCriptRoot. Please see https://github.com/Azure/azure-sdk-tools/issues/5434 for more."
+
+  $azSdkToolsCodeowners = (Resolve-Path "$PSScriptRoot/../../../.github/CODEOWNERS")
+  TestGetCodeowners -targetPath "eng/common/scripts/get-codeowners.ps1" -codeownersFileLocation $azSdkToolsCodeowners -includeNonUserAliases $true -expectReturn @("konrad-jamrozik", "weshaggard", "benbp")
+
+  $testCodeowners = (Resolve-Path "$PSScriptRoot/../../../tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/glob_path_CODEOWNERS")
+  TestGetCodeowners -targetPath "tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/a.txt" -codeownersFileLocation $testCodeowners -includeNonUserAliases $true -expectReturn @("2star")
   exit 0
 }
 else {
-  return Get-CodeOwners -targetDirectory $TargetDirectory -codeOwnerFileLocation $CodeOwnerFileLocation -includeNonUserAliases $IncludeNonUserAliases
+  return Get-Codeowners -targetPath $TargetPath -targetDirectory $TargetDirectory -codeownersFileLocation $CodeOwnerFileLocation -includeNonUserAliases $IncludeNonUserAliases
 }

eng/pipelines/templates/jobs/ci.yml

@@ -66,7 +66,9 @@ jobs:
   - job: 'Build'
     timeoutInMinutes: 90
     variables:
-    - template: ../variables/globals.yml
+      Codeql.Enabled: true
+      Codeql.BuildIdentifier: ${{ parameters.ServiceDirectory }}
+      Codeql.SkipTaskAutoInjection: false
 
     pool:
       name: azsdk-pool-mms-ubuntu-2004-general
@@ -96,9 +98,6 @@ jobs:
 
     timeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }}
 
-    variables:
-    - template: ../variables/globals.yml
-
     dependsOn:
       - 'Build'
 

eng/pipelines/templates/jobs/perf.yml

@@ -1,4 +1,7 @@
 parameters:
+- name: TimeoutInMinutes
+  type: number
+  default: '360'
 - name: LanguageVersion
   type: string
   default: '3.11'
@@ -27,12 +30,13 @@ parameters:
 extends:
   template: /eng/common/pipelines/templates/jobs/perf.yml
   parameters:
+    TimeoutInMinutes: ${{ parameters.TimeoutInMinutes }}
+    Language: Python
+    LanguageVersion: ${{ parameters.LanguageVersion }}
     Variables:
     - template: /eng/pipelines/templates/variables/globals.yml
     - name: PythonVersion
       value: ${{ parameters.LanguageVersion }}
-    Language: Python
-    LanguageVersion: ${{ parameters.LanguageVersion }}
     InstallLanguageSteps:
     - task: UsePythonVersion@0
       displayName: "Use Python $(PythonVersion)"

eng/pipelines/templates/stages/archetype-sdk-client.yml

@@ -73,6 +73,9 @@ parameters:
   type: boolean
   default: false
 
+variables:
+  - template: /eng/pipelines/templates/variables/globals.yml
+
 stages:
   - stage: Build
     jobs:

eng/pipelines/templates/stages/archetype-sdk-tests.yml

@@ -143,13 +143,13 @@ stages:
               Cloud: ${{ cloud.key }}
 
 - ${{ if contains(variables['Build.DefinitionName'], 'tests-weekly')  }}:
-  - stage: Analysis
+  - stage:
+    displayName: 'Analyze_${{ parameters.JobName }}'
     dependsOn: []
     jobs:
       - job: 'Analyze'
         timeoutInMinutes: 90
 
-
         variables:
           PythonVersion: '3.9'
 

eng/pipelines/templates/stages/cosmos-sdk-client.yml

@@ -18,14 +18,14 @@ parameters:
   type: boolean
   default: false
 
-stages:
-  - template: /eng/pipelines/templates/stages/archetype-sdk-client.yml
-    parameters:
-      ServiceDirectory: ${{ parameters.ServiceDirectory }}
-      # Skip emulator tests in the CI run
-      TestMarkArgument: not cosmosEmulator
-      InjectedPackages: ${{parameters.InjectedPackages}}
-      Artifacts: ${{parameters.Artifacts}}
+extends:
+  template: /eng/pipelines/templates/stages/archetype-sdk-client.yml
+  parameters:
+    ServiceDirectory: ${{ parameters.ServiceDirectory }}
+    # Skip emulator tests in the CI run
+    TestMarkArgument: not cosmosEmulator
+    InjectedPackages: ${{parameters.InjectedPackages}}
+    Artifacts: ${{parameters.Artifacts}}
 
 #  - stage: Test_Emulator
 #    dependsOn: []