Merge azure-sdk-tools main into main

.github/workflows/tsp-client-tests.yml

@@ -0,0 +1,57 @@
+name: tsp-client - Test
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths:
+      - .github/workflows/tsp-client-tests.yml
+      - tools/tsp-client/**
+
+jobs:
+  tsp-client:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        node-version: [18, 20]
+        exclude:
+          - os: ubuntu-latest
+            node-version: 20
+          - os: windows-latest
+            node-version: 18
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - if: runner.os == 'Windows'
+        run: git config --global core.longpaths true
+        shell: pwsh
+
+      - uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            .github/workflows
+            tools/tsp-client
+
+      - name: Use Node ${{ matrix.node-version }}.x
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node-version }}.x
+
+      - run: npm ci
+        shell: pwsh
+        working-directory: tools/tsp-client
+
+      - run: npm ls -a
+        shell: pwsh
+        continue-on-error: true
+        working-directory: tools/tsp-client
+
+      - run: npm run build
+        shell: pwsh
+        working-directory: tools/tsp-client
+
+      - run: npm run test
+        shell: pwsh
+        working-directory: tools/tsp-client

eng/common/pipelines/templates/steps/eng-common-workflow-enforcer.yml

@@ -21,7 +21,7 @@ steps:
         }
         if ((!"$(System.PullRequest.SourceBranch)".StartsWith("sync-.github/workflows")) -and "$(System.PullRequest.TargetBranch)" -match "^(refs/heads/)?$(DefaultBranch)$")
         {
-          $filesInCommonDir  = & "eng/common/scripts/get-changedfiles.ps1" -DiffPath '.github/workflows/*' -DiffFilterType ""
+          $filesInCommonDir  = & "eng/common/scripts/get-changedfiles.ps1" -DiffPath '.github/workflows/*event*' -DiffFilterType ""
           if (($LASTEXITCODE -eq 0) -and ($filesInCommonDir.Count -gt 0))
           {
             Write-Host "##vso[task.LogIssue type=error;]Changes to files under '.github/workflows' directory should not be made in this Repo`n${filesInCommonDir}"

eng/common/pipelines/templates/steps/publish-blobs.yml

@@ -1,20 +1,14 @@
 parameters:
   FolderForUpload: ''
   TargetLanguage: ''
-  BlobName: ''
-  ScriptPath: ''
+  BlobName: '' # To be removed once all usages are removed in the languages
+  BlobAccountName: 'https://azuresdkdocs.blob.core.windows.net'
+  ScriptPath: 'eng/common/scripts/copy-docs-to-blobstorage.ps1'
   ArtifactLocation: ''
   RepoId: $(Build.Repository.Name)
 
 steps:
 - template: /eng/common/pipelines/templates/steps/set-default-branch.yml
-- pwsh: |
-    if (!(Test-Path '$(Build.BinariesDirectory)/azcopy/azcopy_windows_amd64_*/azcopy.exe')) {
-      Invoke-WebRequest -MaximumRetryCount 10 -Uri "https://aka.ms/downloadazcopy-v10-windows" -OutFile "azcopy.zip" | Wait-Process;
-      Expand-Archive -Path "azcopy.zip" -DestinationPath "$(Build.BinariesDirectory)/azcopy/" -Force
-    }
-  workingDirectory: $(Build.BinariesDirectory)
-  displayName: Download and Extract azcopy Zip
 
 - task: AzurePowerShell@5
   displayName: 'Copy Docs to Blob Storage'
@@ -24,9 +18,8 @@ steps:
     ScriptType: 'FilePath'
     ScriptPath: ${{ parameters.ScriptPath }}
     ScriptArguments: >
-      -AzCopy $(Resolve-Path "$(Build.BinariesDirectory)/azcopy/azcopy_windows_amd64_*/azcopy.exe")[0]
       -DocLocation "${{ parameters.FolderForUpload }}"
-      -BlobName "${{ parameters.BlobName }}"
+      -BlobAccountName "${{ parameters.BlobAccountName }}"
       -PublicArtifactLocation "${{ parameters.ArtifactLocation }}"
       -RepoReplaceRegex "(https://github.com/${{ parameters.RepoId }}/(?:blob|tree)/)$(DefaultBranch)"
     azurePowerShellVersion: latestVersion

eng/common/pipelines/templates/steps/save-package-properties.yml

@@ -0,0 +1,46 @@
+parameters:
+  - name: ServiceDirectory
+    type: string
+    default: ""
+  - name: DiffDirectory
+    type: string
+    default: $(Build.ArtifactStagingDirectory)/diff
+  - name: PackageInfoDirectory
+    type: string
+    default: $(Build.ArtifactStagingDirectory)/PackageInfo
+  - name: TargetPath
+    type: string
+    default: $(Build.SourcesDirectory)
+  - name: ScriptDirectory
+    type: string
+    default: eng/common/scripts
+
+steps:
+  - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
+      - task: Powershell@2
+        displayName: Generate PR Diff
+        inputs:
+          filePath: ${{ parameters.ScriptDirectory }}/Generate-PR-Diff.ps1
+          arguments: >
+            -TargetPath '${{ parameters.TargetPath }}'
+            -ArtifactPath '${{ parameters.DiffDirectory }}'
+          pwsh: true
+
+      - task: Powershell@2
+        displayName: Save package properties filtered for PR
+        inputs:
+          filePath: ${{ parameters.ScriptDirectory }}/Save-Package-Properties.ps1
+          arguments: >
+            -PrDiff '${{ parameters.DiffDirectory }}/diff.json'
+            -OutDirectory '${{ parameters.PackageInfoDirectory }}'
+          pwsh: true
+  - ${{ else }}:
+      - task: Powershell@2
+        displayName: Save package properties
+        inputs:
+          filePath: ${{ parameters.ScriptDirectory }}/Save-Package-Properties.ps1
+          arguments: >
+            -ServiceDirectory '${{parameters.ServiceDirectory}}'
+            -OutDirectory '${{ parameters.PackageInfoDirectory }}'
+            -AddDevVersion:$${{ eq(variables['SetDevVersion'],'true') }}
+          pwsh: true

eng/common/scripts/Delete-RemoteBranches.ps1

@@ -15,7 +15,6 @@ param(
   $AuthToken
 )
 Set-StrictMode -version 3
-
 . (Join-Path $PSScriptRoot common.ps1)
 
 function Get-AllBranchesAndPullRequestInfo($owner, $repo) {
@@ -66,67 +65,124 @@ if ($AuthToken) {
 }
 
 $owner, $repo = $RepoId -split "/"
-$branches = Get-AllBranchesAndPullRequestInfo $owner $repo
 
-foreach ($branch in $branches)
-{
-  $branchName = $branch.Name
-  if ($branchName -notmatch $BranchRegex) {
-    continue
+# These will always be output at the end of the script. Their only purpose is for information gathering
+# Total number returned from query
+$totalBranchesFromQuery = 0
+# reasons why a branch was skipped
+$skippedBranchNotMatchRegex = 0
+$skippedForCommitDate = 0
+$skippedForOpenPRs = 0
+$skippedForPRNotInBranch = 0
+$skippedForPRNotInRepo = 0
+# gh call counters
+$ghPRViewCalls = 0
+$ghBranchDeleteCalls = 0
+
+try {
+  # Output the core rate limit at the start of processing. There's no real need
+  # to output this at the end because the GH call counts are being output
+  $coreRateLimit = Get-RateLimit core
+  Write-RateLimit $coreRateLimit
+  # Output the GraphQL rate limit before and after the call
+  $graphqlRateLimit = Get-RateLimit graphql
+  Write-RateLimit $graphqlRateLimit "Before GraphQL Call"
+  $branches = Get-AllBranchesAndPullRequestInfo $owner $repo
+  $graphqlRateLimit = Get-RateLimit graphql
+  Write-RateLimit $graphqlRateLimit "After GraphQL Call"
+
+  if ($branches) {
+    $totalBranchesFromQuery = $branches.Count
   }
-  $openPullRequests = @($branch.pullRequests | Where-Object { !$_.Closed })
 
-  # If we have a central PR that created this branch still open don't delete the branch
-  if ($CentralRepoId)
+  foreach ($branch in $branches)
   {
-    $pullRequestNumber = $matches["PrNumber"]
-    # If central PR number is not found, then skip
-    if (!$pullRequestNumber) {
-      LogError "No PR number found in the branch name. Please check the branch name '$branchName'. Skipping..."
+    $branchName = $branch.Name
+    if ($branchName -notmatch $BranchRegex) {
+      $skippedBranchNotMatchRegex++
       continue
     }
+    $openPullRequests = @($branch.pullRequests | Where-Object { !$_.Closed })
 
-    $centralPR = gh pr view --json 'url,closed' --repo $CentralRepoId $pullRequestNumber | ConvertFrom-Json
-    if ($LASTEXITCODE) {
-      LogError "PR '$pullRequestNumber' not found in repo '$CentralRepoId'. Skipping..."
-      continue;
-    } else {
-      LogDebug "Found central PR $($centralPR.url) and Closed=$($centralPR.closed)"
-      if (!$centralPR.Closed) {
-        # Skipping if there is an open central PR open for the branch.
-        LogDebug "Central PR is still open so skipping the deletion of branch '$branchName'. Skipping..."
-        continue;
+    # If we have a central PR that created this branch still open don't delete the branch
+    if ($CentralRepoId)
+    {
+      $pullRequestNumber = $matches["PrNumber"]
+      # If central PR number is not found, then skip
+      if (!$pullRequestNumber) {
+        LogError "No PR number found in the branch name. Please check the branch name '$branchName'. Skipping..."
+        $skippedForPRNotInBranch++
+        continue
+      }
+
+      $ghPRViewCalls++
+      $centralPR = gh pr view --json 'url,closed' --repo $CentralRepoId $pullRequestNumber | ConvertFrom-Json
+      if ($LASTEXITCODE) {
+        LogError "PR '$pullRequestNumber' not found in repo '$CentralRepoId'. Skipping..."
+        $skippedForPRNotInRepo++
+        continue
+      } else {
+        LogDebug "Found central PR $($centralPR.url) and Closed=$($centralPR.closed)"
+        if (!$centralPR.Closed) {
+          $skippedForOpenPRs++
+          # Skipping if there is an open central PR open for the branch.
+          LogDebug "Central PR is still open so skipping the deletion of branch '$branchName'. Skipping..."
+          continue
+        }
       }
     }
-  }
-  else {
-    # Not CentralRepoId - not associated with a central repo PR
-    if ($openPullRequests.Count -gt 0 -and !$DeleteBranchesEvenIfThereIsOpenPR) {
-      LogDebug "Found open PRs associate with branch '$branchName'. Skipping..."
-      continue
+    else {
+      # Not CentralRepoId - not associated with a central repo PR
+      if ($openPullRequests.Count -gt 0 -and !$DeleteBranchesEvenIfThereIsOpenPR) {
+        $skippedForOpenPRs++
+        LogDebug "Found open PRs associate with branch '$branchName'. Skipping..."
+        continue
+      }
     }
-  }
 
-  # If there is date filter, then check if branch last commit is older than the date.
-  if ($LastCommitOlderThan)
-  {
-    $commitDate = $branch.committedDate
-    if ($commitDate -gt $LastCommitOlderThan) {
-      LogDebug "The branch $branch last commit date '$commitDate' is newer than the date '$LastCommitOlderThan'. Skipping..."
-      continue
+    # If there is date filter, then check if branch last commit is older than the date.
+    if ($LastCommitOlderThan)
+    {
+      $commitDate = $branch.committedDate
+      if ($commitDate -gt $LastCommitOlderThan) {
+        $skippedForCommitDate++
+        LogDebug "The branch $branch last commit date '$commitDate' is newer than the date '$LastCommitOlderThan'. Skipping..."
+        continue
+      }
     }
-  }
 
-  foreach ($openPullRequest in $openPullRequests) {
-    Write-Host "Note: Open pull Request '$($openPullRequest.url)' will be closed after branch deletion, given the central PR is closed."
-  }
+    foreach ($openPullRequest in $openPullRequests) {
+      LogDebug "Note: Open pull Request '$($openPullRequest.url)' will be closed after branch deletion, given the central PR is closed."
+    }
 
-  $commitUrl = $branch.commitUrl
-  if ($PSCmdlet.ShouldProcess("'$branchName' in '$RepoId'", "Deleting branch on cleanup script")) {
-    gh api "repos/${RepoId}/git/refs/heads/${branchName}" -X DELETE
-    if ($LASTEXITCODE) {
-      LogError "Deletion of branch '$branchName` failed"
+    $commitUrl = $branch.commitUrl
+    if ($PSCmdlet.ShouldProcess("'$branchName' in '$RepoId'", "Deleting branch on cleanup script")) {
+      $ghBranchDeleteCalls++
+      gh api "repos/${RepoId}/git/refs/heads/${branchName}" -X DELETE
+      if ($LASTEXITCODE) {
+        LogError "Deletion of branch '$branchName` failed, see command output above"
+        exit $LASTEXITCODE
+      }
+      LogDebug "The branch '$branchName' at commit '$commitUrl' in '$RepoId' has been deleted."
     }
-    Write-Host "The branch '$branchName' at commit '$commitUrl' in '$RepoId' has been deleted."
+  }
+}
+finally {
+
+
+  Write-Host "Number of branches returned from graphql query: $totalBranchesFromQuery"
+  # The $BranchRegex seems to be always set
+  if ($BranchRegex) {
+    Write-Host "Number of branches that didn't match the BranchRegex: $skippedBranchNotMatchRegex"
+  }
+  Write-Host "Number of branches skipped for newer last commit date: $skippedForCommitDate"
+  Write-Host "Number of branches skipped for open PRs: $skippedForOpenPRs"
+  Write-Host "Number of gh api calls to delete branches: $ghBranchDeleteCalls"
+  # The following are only applicable when $CentralRepoId is passed in
+  if ($CentralRepoId) {
+    Write-Host "The following are applicable because CentralRepoId was passed in:"
+    Write-Host "  Number of gh pr view calls: $ghPRViewCalls"
+    Write-Host "  Number of branches skipped due to PR not in the repository: $skippedForPRNotInRepo "
+    Write-Host "  Number of branches skipped due to PR not in the branch name: $skippedForPRNotInBranch"
   }
 }

eng/common/scripts/Generate-PR-Diff.ps1

@@ -13,27 +13,29 @@ The path under which changes will be detected.
 #>
 [CmdletBinding()]
 Param (
-  [Parameter(Mandatory=$True)]
+  [Parameter(Mandatory = $True)]
   [string] $ArtifactPath,
-  [Parameter(Mandatory=$True)]
+  [Parameter(Mandatory = $True)]
   [string] $TargetPath
 )
 
 . (Join-Path $PSScriptRoot "Helpers" "git-helpers.ps1")
 
-function Get-ChangedServices {
-    Param (
-        [Parameter(Mandatory=$True)]
-        [string[]] $ChangedFiles
-    )
+function Get-ChangedServices
+{
+  Param (
+    [Parameter(Mandatory = $True)]
+    [string[]] $ChangedFiles
+  )
 
-    $changedServices = $ChangedFiles | Foreach-Object { if ($_ -match "sdk/([^/]+)") { $matches[1] } } | Sort-Object -Unique
+  $changedServices = $ChangedFiles | Foreach-Object { if ($_ -match "sdk/([^/]+)") { $matches[1] } } | Sort-Object -Unique
 
-    return $changedServices
+  return $changedServices
 }
 
-if (!(Test-Path $ArtifactPath)) {
-    New-Item -ItemType Directory -Path $ArtifactPath | Out-Null
+if (!(Test-Path $ArtifactPath))
+{
+  New-Item -ItemType Directory -Path $ArtifactPath | Out-Null
 }
 
 $ArtifactPath = Resolve-Path $ArtifactPath
@@ -43,9 +45,13 @@ $changedFiles = Get-ChangedFiles -DiffPath $TargetPath
 $changedServices = Get-ChangedServices -ChangedFiles $changedFiles
 
 $result = [PSCustomObject]@{
-    "ChangedFiles" = $changedFiles
-    "ChangedServices" = $changedServices
-    "PRNumber" = if ($env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER) { $env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER } else { "-1" }
+  "ChangedFiles"    = $changedFiles
+  "ChangedServices" = $changedServices
+  "PRNumber"        = if ($env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER) { $env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER } else { "-1" }
 }
 
-$result | ConvertTo-Json | Out-File $ArtifactName
+$json = $result | ConvertTo-Json
+$json | Out-File $ArtifactName
+
+Write-Host "`nGenerated diff.json file at $ArtifactName"
+Write-Host "  $($json -replace "`n", "`n  ")"