From 03e9f7cd37763045ebcd575e5f3bd7f969f21787 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 2 Oct 2024 13:27:42 -0700
Subject: [PATCH 01/90] preliminary changes to build

---
 eng/pipelines/templates/steps/build.yml | 37 +++++++++++++------------
 sdk/pullrequest.yml                     | 25 +++++++++++++++++
 2 files changed, 45 insertions(+), 17 deletions(-)
 create mode 100644 sdk/pullrequest.yml

diff --git a/eng/pipelines/templates/steps/build.yml b/eng/pipelines/templates/steps/build.yml
index 804b85784a4b..35d07d088032 100644
--- a/eng/pipelines/templates/steps/build.yml
+++ b/eng/pipelines/templates/steps/build.yml
@@ -13,14 +13,15 @@ steps:
 
   - pwsh: |
       $folder = "${{parameters.ServiceDirectory}}"
-      if ($folder -eq "*") { $folder = "" }
+      if ($folder -eq "*" -or $folder -eq "auto") { $folder = "" }
       echo "##vso[task.setvariable variable=folder]$folder"
     displayName: "Set folder variable for readme links"
 
+  # we are not passing service directory, so we only ever set dev build to true
   - template: /eng/common/pipelines/templates/steps/daily-dev-build-variable.yml
-    parameters:
-      ServiceDirectory: ${{ parameters.ServiceDirectory }}
 
+  # todo: update set-dev.js to handle multiple service directories based on the ChangedServices value of the diff
+  # currently defaulting to nothing, so we set the entire repo to dev build (we do this for python)
   - script: |
       npm install ./eng/tools/versioning
       node eng/tools/versioning/set-dev.js --build-id "$(Build.BuildNumber)" --repo-root "$(Build.SourcesDirectory)" --service "$(folder)"
@@ -28,17 +29,9 @@ steps:
     condition: and(succeeded(),eq(variables['SetDevVersion'],'true'))
     displayName: "Update package versions for dev build"
 
-  - task: Powershell@2
-    inputs:
-      filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1
-      arguments: >
-        -ServiceDirectory ${{parameters.ServiceDirectory}}
-        -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo
-        -AddDevVersion
-      pwsh: true
-      workingDirectory: $(Pipeline.Workspace)
-    displayName: Update package properties with dev version
-    condition: and(succeeded(),eq(variables['SetDevVersion'],'true'))
+  - template: /eng/common/pipelines/templates/steps/save-package-properties.yml
+    parameters:
+      ServiceDirectory: ${{parameters.ServiceDirectory}}
 
   - script: |
       node common/scripts/install-run-rush.js install
@@ -47,6 +40,7 @@ steps:
 
   - template: /eng/pipelines/templates/steps/set-artifact-packages.yml
     parameters:
+      ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
       Artifacts: ${{ parameters.Artifacts }}
   - ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) }}:
     - task: AzureCLI@2
@@ -92,16 +86,25 @@ steps:
     parameters:
       ServiceDirectory: ${{parameters.ServiceDirectory}}
 
+  # todo, handle direct/indirect packages here
+  # todo, add the parse of the documentation status to eng/common so that we don't have to look at the ci.yml properties again
   - pwsh: |
       $artifacts = '${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json
+
+      $artifacts = Get-ChildItem -Recurse -Force "$(Build.ArtifactStagingDirectory/PackageInfo/*.json" `
+          | ForEach-Object { $_.Name.Replace(".json", "") }
+
       foreach ($artifact in $artifacts)
       {
         $artifactName = $artifact.name
+        $artifactDetails = Get-Content $(Build.ArtifactStagingDirectory)/PackageInfo/$artifactName.json | ConvertFrom-Json
+
         Write-Host "Copying $artifactName artifacts to $(Build.ArtifactStagingDirectory)/$artifactName"
         New-Item -Type Directory -Name $artifactName -Path $(Build.ArtifactStagingDirectory) > $null
-        Copy-Item sdk/${{parameters.ServiceDirectory}}/**/$artifactName-[0-9]*.[0-9]*.[0-9]*.tgz $(Build.ArtifactStagingDirectory)/$artifactName
-        Copy-Item sdk/${{parameters.ServiceDirectory}}/**/browser/$artifactName-[0-9]*.[0-9]*.[0-9]*.zip $(Build.ArtifactStagingDirectory)/$artifactName
-        if ($${{ parameters.IncludeRelease }} -eq $true -and $artifact.skipPublishDocMs -ne $true)
+        Copy-Item sdk/$($artifactDetails.Service)/**/$artifactName-[0-9]*.[0-9]*.[0-9]*.tgz $(Build.ArtifactStagingDirectory)/$artifactName
+        Copy-Item sdk/$($artifactDetails.Service)/**/browser/$artifactName-[0-9]*.[0-9]*.[0-9]*.zip $(Build.ArtifactStagingDirectory)/$artifactName
+
+        if ($true)
         {
           New-Item -Type Directory -Name documentation -Path $(Build.ArtifactStagingDirectory)/$artifactName > $null
           Copy-Item $(Build.SourcesDirectory)/docGen/$artifactName.zip $(Build.ArtifactStagingDirectory)/$artifactName/documentation
diff --git a/sdk/pullrequest.yml b/sdk/pullrequest.yml
new file mode 100644
index 000000000000..1665db38607a
--- /dev/null
+++ b/sdk/pullrequest.yml
@@ -0,0 +1,25 @@
+pr:
+  branches:
+    include:
+    - main
+    - feature/*
+    - hotfix/*
+    - release/*
+    - restapi*
+    - pipelinev3*
+  paths:
+    include:
+    - "*"
+
+    exclude:
+    - sdk/cosmos
+
+parameters:
+  - name: Service
+    type: string
+    default: auto
+
+extends:
+  template: ../../eng/pipelines/templates/stages/archetype-sdk-client.yml
+  parameters:
+    ServiceDirectory: ${{ parameters.Service }}

From f6352dcf7460ca6884b45b401bf58538aa4b8a0e Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 2 Oct 2024 13:36:41 -0700
Subject: [PATCH 02/90] pulling across the improvements from original
 pipelinev3 branch

---
 eng/pipelines/templates/steps/build.yml       | 18 +++++-------
 .../templates/steps/set-artifact-packages.yml | 28 +++++++++++++------
 eng/tools/rush-runner.js                      |  6 ++--
 3 files changed, 29 insertions(+), 23 deletions(-)

diff --git a/eng/pipelines/templates/steps/build.yml b/eng/pipelines/templates/steps/build.yml
index 35d07d088032..5226a0f07dc9 100644
--- a/eng/pipelines/templates/steps/build.yml
+++ b/eng/pipelines/templates/steps/build.yml
@@ -40,8 +40,7 @@ steps:
 
   - template: /eng/pipelines/templates/steps/set-artifact-packages.yml
     parameters:
-      ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
-      Artifacts: ${{ parameters.Artifacts }}
+      PackageInfo: $(Build.ArtifactStagingDirectory)/PackageInfo
   - ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) }}:
     - task: AzureCLI@2
       inputs:
@@ -89,22 +88,19 @@ steps:
   # todo, handle direct/indirect packages here
   # todo, add the parse of the documentation status to eng/common so that we don't have to look at the ci.yml properties again
   - pwsh: |
-      $artifacts = '${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json
-
       $artifacts = Get-ChildItem -Recurse -Force "$(Build.ArtifactStagingDirectory/PackageInfo/*.json" `
           | ForEach-Object { $_.Name.Replace(".json", "") }
 
       foreach ($artifact in $artifacts)
       {
-        $artifactName = $artifact.name
-        $artifactDetails = Get-Content $(Build.ArtifactStagingDirectory)/PackageInfo/$artifactName.json | ConvertFrom-Json
+        $artifactDetails = Get-Content $(Build.ArtifactStagingDirectory)/PackageInfo/$artifact.json | ConvertFrom-Json
 
-        Write-Host "Copying $artifactName artifacts to $(Build.ArtifactStagingDirectory)/$artifactName"
-        New-Item -Type Directory -Name $artifactName -Path $(Build.ArtifactStagingDirectory) > $null
-        Copy-Item sdk/$($artifactDetails.Service)/**/$artifactName-[0-9]*.[0-9]*.[0-9]*.tgz $(Build.ArtifactStagingDirectory)/$artifactName
-        Copy-Item sdk/$($artifactDetails.Service)/**/browser/$artifactName-[0-9]*.[0-9]*.[0-9]*.zip $(Build.ArtifactStagingDirectory)/$artifactName
+        Write-Host "Copying $artifact artifacts to $(Build.ArtifactStagingDirectory)/$artifact"
+        New-Item -Type Directory -Name $artifact -Path $(Build.ArtifactStagingDirectory) > $null
+        Copy-Item sdk/$($artifactDetails.Service)/**/$artifact-[0-9]*.[0-9]*.[0-9]*.tgz $(Build.ArtifactStagingDirectory)/$artifact
+        Copy-Item sdk/$($artifactDetails.Service)/**/browser/$artifact-[0-9]*.[0-9]*.[0-9]*.zip $(Build.ArtifactStagingDirectory)/$artifact
 
-        if ($true)
+        if ($${{ parameters.IncludeRelease }} -eq $true)
         {
           New-Item -Type Directory -Name documentation -Path $(Build.ArtifactStagingDirectory)/$artifactName > $null
           Copy-Item $(Build.SourcesDirectory)/docGen/$artifactName.zip $(Build.ArtifactStagingDirectory)/$artifactName/documentation
diff --git a/eng/pipelines/templates/steps/set-artifact-packages.yml b/eng/pipelines/templates/steps/set-artifact-packages.yml
index 50090c8258d9..6b37e003d0df 100644
--- a/eng/pipelines/templates/steps/set-artifact-packages.yml
+++ b/eng/pipelines/templates/steps/set-artifact-packages.yml
@@ -1,12 +1,22 @@
 parameters:
-  Artifacts: []
+  PackageInfo: ''
 steps:
+  # Package-Properties folder now contains the package properties for the packages that should be built or tested.
+  # if targeting has been set by matrix, this value will already be populated, can't override it. Handle these cases here.
   - pwsh: |
-      $artifacts = '${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json
-      $packages = ""
-      foreach ($artifact in $artifacts)
-      {
-        $packages += "$($artifact.name),"
-      }
-      echo "##vso[task.setvariable variable=ArtifactPackageNames]$packages"
-    displayName: "Find Packages to build"
+      Get-ChildItem -R "${{ parameters.PackageInfo }}"
+
+      $packageProperties = Get-ChildItem -Recurse "${{ parameters.PackageInfo }}" *.json `
+        | Foreach-Object { $_.Name } `
+        | ForEach-Object { $_.Replace(".json", "") }
+      $setting = $packageProperties -join ","
+
+      Write-Host "Setting ArtifactPackageNames to: `n$setting"
+      Write-Host "##vso[task.setvariable variable=ArtifactPackageNames;]$setting"
+    displayName: Resolve Targeted Packages
+    condition: eq(variables['ArtifactPackageNames'], '')
+
+  - pwsh: |
+      Write-Host "This run is targeting: `n$(ArtifactPackageNames)"
+    displayName: Resolve Targeted Packages
+    condition: ne(variables['ArtifactPackageNames'], '')
diff --git a/eng/tools/rush-runner.js b/eng/tools/rush-runner.js
index 68560ca1a514..c1a9097aee50 100644
--- a/eng/tools/rush-runner.js
+++ b/eng/tools/rush-runner.js
@@ -150,8 +150,8 @@ function rushRunAll(direction, packages) {
 /**
  * Helper function to get the relative path of a package directory from an absolute
  * one
- * 
- * @param {string} absolutePath absolute path to a package 
+ *
+ * @param {string} absolutePath absolute path to a package
  * @returns either the relative path of the package starting from the "sdk" directory
  *          or the just the absolute path itself if "sdk" if not found
  */
@@ -177,7 +177,7 @@ if (serviceDirs.length === 0) {
       // If service is configured to run only a set of downstream projects then build all projects leading to them to support testing
       // if this is build:test for any non-configured package service then all impacted projects downstream and it's dependents should be built
       var rushCommandFlag = "--impacted-by";
-      if (isReducedTestScopeEnabled) {
+      if (isReducedTestScopeEnabled || serviceDirs.length < 1) {
         // reduced preconfigured set of projects and it's required projects
         rushCommandFlag = "--to";
       }

From dccd0f99ad53c52e36bfbc125db883a57cf51b35 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 2 Oct 2024 13:44:57 -0700
Subject: [PATCH 03/90] fix the pullrequest caller for proper relative path

---
 sdk/pullrequest.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdk/pullrequest.yml b/sdk/pullrequest.yml
index 1665db38607a..d123bf24be3b 100644
--- a/sdk/pullrequest.yml
+++ b/sdk/pullrequest.yml
@@ -20,6 +20,6 @@ parameters:
     default: auto
 
 extends:
-  template: ../../eng/pipelines/templates/stages/archetype-sdk-client.yml
+  template: ../eng/pipelines/templates/stages/archetype-sdk-client.yml
   parameters:
     ServiceDirectory: ${{ parameters.Service }}

From cf03790f33372d757d867122933a8a75d25c8dbc Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 2 Oct 2024 13:45:57 -0700
Subject: [PATCH 04/90] is this better?

---
 sdk/pullrequest.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdk/pullrequest.yml b/sdk/pullrequest.yml
index d123bf24be3b..951dcee8d339 100644
--- a/sdk/pullrequest.yml
+++ b/sdk/pullrequest.yml
@@ -20,6 +20,6 @@ parameters:
     default: auto
 
 extends:
-  template: ../eng/pipelines/templates/stages/archetype-sdk-client.yml
+  template: /eng/pipelines/templates/stages/archetype-sdk-client.yml
   parameters:
     ServiceDirectory: ${{ parameters.Service }}

From 4c683df6bb6b9891c6cdca5153e5263bb5420202 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 2 Oct 2024 14:31:58 -0700
Subject: [PATCH 05/90] handle nonexisting service when invoking for PR

---
 eng/pipelines/templates/steps/build.yml | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/eng/pipelines/templates/steps/build.yml b/eng/pipelines/templates/steps/build.yml
index 5226a0f07dc9..280e8272ebad 100644
--- a/eng/pipelines/templates/steps/build.yml
+++ b/eng/pipelines/templates/steps/build.yml
@@ -53,11 +53,28 @@ steps:
           sasToken=$(az storage container generate-sas --account-name azuresdkartifacts --name azure-sdk-for-js-rush-cache --permissions dlrw --auth-mode login --as-user --expiry $expiry --https-only -o tsv)
           echo "##vso[task.setvariable variable=rushBuildCacheCred;issecret=true;]$sasToken"
 
+  - script: |
+      $prDiff = Get-Content $(Build.ArtifactStagingDirectory)/diff/diff.json | ConvertFrom-Json -AsHashTable
+
+      $services = "auto"
+      if ($prDiff["ChangedServices"]) {
+        $services = $prDiff["ChangedServices"] -join " "
+      }
+
+      node eng/tools/rush-runner.js build $services -packages "$(ArtifactPackageNames)" --verbose -p max
+    displayName: "Build libraries for PR"
+    condition: and(succeeded(),eq(variables['Build.Reason'],'PullRequest'))
+    env:
+      ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) }}:
+        RUSH_BUILD_CACHE_CREDENTIAL: $(rushBuildCacheCred)
+        RUSH_BUILD_CACHE_WRITE_ALLOWED: 1
+
   # Option "-p max" ensures parallelism is set to the number of cores on all platforms, which improves build times.
   # The default on Windows is "cores - 1" (microsoft/rushstack#436).
   - script: |
       node eng/tools/rush-runner.js build "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)" --verbose -p max
-    displayName: "Build libraries"
+    displayName: "Build libraries for Service"
+    condition: and(succeeded(),ne(variables['Build.Reason'],'PullRequest'))
     env:
       ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) }}:
         RUSH_BUILD_CACHE_CREDENTIAL: $(rushBuildCacheCred)

From 2cb2a7b08b71da0659bd790713f939fd85caface Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 2 Oct 2024 15:25:59 -0700
Subject: [PATCH 06/90] script -> pwsh

---
 eng/pipelines/templates/steps/build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eng/pipelines/templates/steps/build.yml b/eng/pipelines/templates/steps/build.yml
index 280e8272ebad..44a79e4704ec 100644
--- a/eng/pipelines/templates/steps/build.yml
+++ b/eng/pipelines/templates/steps/build.yml
@@ -53,7 +53,7 @@ steps:
           sasToken=$(az storage container generate-sas --account-name azuresdkartifacts --name azure-sdk-for-js-rush-cache --permissions dlrw --auth-mode login --as-user --expiry $expiry --https-only -o tsv)
           echo "##vso[task.setvariable variable=rushBuildCacheCred;issecret=true;]$sasToken"
 
-  - script: |
+  - pwsh: |
       $prDiff = Get-Content $(Build.ArtifactStagingDirectory)/diff/diff.json | ConvertFrom-Json -AsHashTable
 
       $services = "auto"

From 62958258e2b2eececfbeb14db2365c094b7a08dd Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 2 Oct 2024 16:06:11 -0700
Subject: [PATCH 07/90] fix rush runner

---
 eng/tools/rush-runner.js | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/eng/tools/rush-runner.js b/eng/tools/rush-runner.js
index c1a9097aee50..777095b00df4 100644
--- a/eng/tools/rush-runner.js
+++ b/eng/tools/rush-runner.js
@@ -48,6 +48,7 @@ const parseArgs = () => {
   const baseDir = path.resolve(`${path.dirname(scriptPath)}/../..`);
 
   for (const arg of givenArgs) {
+    console.log(`Examining ${arg}`);
     if (arg === "-packages") {
       isPackageFilter = true;
       continue;
@@ -66,7 +67,8 @@ const parseArgs = () => {
     else {
       if (arg && arg !== "*") {
         // exclude empty value and special value "*" meaning all libraries
-        services.push(arg);
+        arg.split(" ").forEach(serviceDirectory => services.push(serviceDirectory));
+        console.log(`Services is ${services}`)
       }
     }
   }
@@ -99,6 +101,7 @@ const getServicePackages = (baseDir, serviceDirs, artifactNames) => {
   console.log(`Packages to build: ${artifactNames}`);
   const artifacts = artifactNames.split(",");
   for (const serviceDir of serviceDirs) {
+    console.log(`Looking at ${serviceDir}`);
     const searchDir = path.resolve(path.join(baseDir, "sdk", serviceDir));
     const packageJsons = getPackageJsons(searchDir);
     for (const filePath of packageJsons) {

From dd7365382299c67107fed932c89ea155d50a8a43 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 2 Oct 2024 16:12:14 -0700
Subject: [PATCH 08/90] swap from from to to when multiple services are
 provided

---
 eng/tools/rush-runner.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eng/tools/rush-runner.js b/eng/tools/rush-runner.js
index 777095b00df4..1128f86b0636 100644
--- a/eng/tools/rush-runner.js
+++ b/eng/tools/rush-runner.js
@@ -180,7 +180,7 @@ if (serviceDirs.length === 0) {
       // If service is configured to run only a set of downstream projects then build all projects leading to them to support testing
       // if this is build:test for any non-configured package service then all impacted projects downstream and it's dependents should be built
       var rushCommandFlag = "--impacted-by";
-      if (isReducedTestScopeEnabled || serviceDirs.length < 1) {
+      if (isReducedTestScopeEnabled || serviceDirs.length > 1) {
         // reduced preconfigured set of projects and it's required projects
         rushCommandFlag = "--to";
       }

From e7504882f617ee9eddc1e8127c0b484f76c86033 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 2 Oct 2024 16:27:21 -0700
Subject: [PATCH 09/90] iterating across all the artifacts for a prdiff

---
 eng/pipelines/templates/steps/analyze.yml | 15 ++++++-
 eng/scripts/spell-check-public-apis.ps1   | 49 +++++++++++++++++++++++
 2 files changed, 62 insertions(+), 2 deletions(-)
 create mode 100644 eng/scripts/spell-check-public-apis.ps1

diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml
index 2bd57782144d..b1e07d176339 100644
--- a/eng/pipelines/templates/steps/analyze.yml
+++ b/eng/pipelines/templates/steps/analyze.yml
@@ -4,15 +4,27 @@ parameters:
   TestPipeline: false
 
 steps:
+  - template: /eng/common/pipelines/templates/steps/save-package-properties.yml
+
   - template: /eng/common/pipelines/templates/steps/check-spelling.yml
 
+  - task: PowerShell@2
+    inputs:
+      targetType: 'filePath'
+      filePath: eng/scripts/spell-check-public-apis.ps1
+      arguments: -DiffJsonFile $(Build.ArtifactStagingDirectory)/diff/diff.json
+      pwsh: true
+    displayName: Spell check public API
+    condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))
+
   - task: PowerShell@2
     inputs:
       targetType: 'filePath'
       filePath: eng/scripts/spell-check-public-api.ps1
       arguments: -ServiceDirectory ${{ parameters.ServiceDirectory }}
       pwsh: true
-    displayName: Spell check public API
+    displayName: Spell check public APIs
+    condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
 
   - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml
     parameters:
@@ -26,7 +38,6 @@ steps:
       ArtifactName: 'package-diffs'
       SbomEnabled: false
 
-
   - template: /eng/common/pipelines/templates/steps/verify-readme.yml
     parameters:
       ScanPath: $(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}
diff --git a/eng/scripts/spell-check-public-apis.ps1 b/eng/scripts/spell-check-public-apis.ps1
new file mode 100644
index 000000000000..59d80c2f41c8
--- /dev/null
+++ b/eng/scripts/spell-check-public-apis.ps1
@@ -0,0 +1,49 @@
+<#
+.SYNOPSIS
+Spell checks JS public API surface as exported to review/**/*.md. This script invokes
+against multiple service directories.
+
+.DESCRIPTION
+Checks spelling of package's public API. Some packages may be excluded by
+criteria in the cspell.json config. The precise list of files to scan is
+determined by cspell. If a pacakge is opted out in the cspell.json a command
+will still be issued to scan that folder but cspell will report 0 files checked.
+Given a diff json file, this script will check the spelling of any service directories
+that have been changed.
+
+.PARAMETER DiffJson
+Scopes scanning to one or multiple service directories contained within `diff.json->ChangedServices`.
+
+.EXAMPLE
+./spell-check-public-apis.ps1 -PRDiff diff.json
+
+Spell check all public API specs for all services under `sdk` that have been changed in the PR.
+
+#>
+[CmdletBinding()]
+param (
+  [Parameter(mandatory = $true)]
+  $DiffJsonFile
+)
+
+Set-StrictMode -Version 3.0
+
+$prDiff = Get-Content $DiffJsonFile | ConvertFrom-Json -AsHashTable
+
+
+$allSuccess = $true
+if ($prDiff["ChangedServices"]) {
+  foreach($service in $prDiff["ChangedServices"]) {
+
+    &"$PSScriptRoot/spell-check-public-api.ps1" -ServiceDirectory $service
+
+    if($LASTEXITCODE -ne 0) {
+      $allSuccess = $false
+    }
+  }
+}
+
+if (-not $allSuccess) {
+  Write-Error "One or more service directories failed spell check, check above output for details."
+  exit 1
+}

From 725c6a75894017fa70d7ebf9e565af78197eafe3 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 2 Oct 2024 17:04:05 -0700
Subject: [PATCH 10/90] working my way through the crashing build

---
 eng/pipelines/templates/steps/analyze.yml |  6 +++---
 eng/pipelines/templates/steps/build.yml   | 12 ++++++++++--
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml
index b1e07d176339..8f52b4fbf4c8 100644
--- a/eng/pipelines/templates/steps/analyze.yml
+++ b/eng/pipelines/templates/steps/analyze.yml
@@ -14,7 +14,7 @@ steps:
       filePath: eng/scripts/spell-check-public-apis.ps1
       arguments: -DiffJsonFile $(Build.ArtifactStagingDirectory)/diff/diff.json
       pwsh: true
-    displayName: Spell check public API
+    displayName: Spell check public API for PR
     condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))
 
   - task: PowerShell@2
@@ -38,9 +38,9 @@ steps:
       ArtifactName: 'package-diffs'
       SbomEnabled: false
 
-  - template: /eng/common/pipelines/templates/steps/verify-readme.yml
+  - template: /eng/common/pipelines/templates/steps/verify-readmes.yml
     parameters:
-      ScanPath: $(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}
+      PackageInfoFolder: $(Build.ArtifactStagingDirectory)/PackageInfo
 
   - template: /eng/common/pipelines/templates/steps/verify-path-length.yml
     parameters:
diff --git a/eng/pipelines/templates/steps/build.yml b/eng/pipelines/templates/steps/build.yml
index 44a79e4704ec..2fc588b50b41 100644
--- a/eng/pipelines/templates/steps/build.yml
+++ b/eng/pipelines/templates/steps/build.yml
@@ -61,6 +61,8 @@ steps:
         $services = $prDiff["ChangedServices"] -join " "
       }
 
+      Write-Host "##vso[task.setvariable variable=ChangedServices;]$services"
+
       node eng/tools/rush-runner.js build $services -packages "$(ArtifactPackageNames)" --verbose -p max
     displayName: "Build libraries for PR"
     condition: and(succeeded(),eq(variables['Build.Reason'],'PullRequest'))
@@ -80,16 +82,22 @@ steps:
         RUSH_BUILD_CACHE_CREDENTIAL: $(rushBuildCacheCred)
         RUSH_BUILD_CACHE_WRITE_ALLOWED: 1
 
+  - script: |
+      node eng/tools/rush-runner.js build:samples "$(ChangedServices)" -packages "$(ArtifactPackageNames)" --verbose
+    displayName: "Build samples for PR"
+    condition: and(succeeded(),eq(variables['Build.Reason'],'PullRequest'))
+
   - script: |
       node eng/tools/rush-runner.js build:samples "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)" --verbose
-    displayName: "Build samples"
+    displayName: "Build samples for Service"
+    condition: and(succeeded(),ne(variables['Build.Reason'],'PullRequest'))
 
   - pwsh: |
       eng/tools/check-api-warning.ps1
     displayName: "Check api extractor output changes"
 
   - script: |
-      node eng/tools/rush-runner.js pack "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)" --verbose
+      node eng/tools/rush-runner.js pack "$(ChangedServices)" -packages "$(ArtifactPackageNames)" --verbose
     displayName: "Pack libraries"
 
   # Unlink node_modules folders to significantly improve performance of subsequent tasks

From 733b0201c55334ceccbf10ab24ea1dd148272660 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 2 Oct 2024 17:07:35 -0700
Subject: [PATCH 11/90] we need to pass PackagePropertiesFolder not
 PackageInfoFolder

---
 eng/pipelines/templates/steps/analyze.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml
index 8f52b4fbf4c8..c36ef0ffa655 100644
--- a/eng/pipelines/templates/steps/analyze.yml
+++ b/eng/pipelines/templates/steps/analyze.yml
@@ -40,7 +40,7 @@ steps:
 
   - template: /eng/common/pipelines/templates/steps/verify-readmes.yml
     parameters:
-      PackageInfoFolder: $(Build.ArtifactStagingDirectory)/PackageInfo
+      PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo
 
   - template: /eng/common/pipelines/templates/steps/verify-path-length.yml
     parameters:

From 3bf5519e17fcb13e419a1d5f1943c1734d5dd72c Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 2 Oct 2024 17:59:32 -0700
Subject: [PATCH 12/90] fix documentation run

---
 eng/pipelines/templates/steps/generate-doc.yml | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/eng/pipelines/templates/steps/generate-doc.yml b/eng/pipelines/templates/steps/generate-doc.yml
index 828ffb0519ef..c7ea2691a5b1 100644
--- a/eng/pipelines/templates/steps/generate-doc.yml
+++ b/eng/pipelines/templates/steps/generate-doc.yml
@@ -15,6 +15,17 @@ steps:
   - pwsh: |
       node $(Build.SourcesDirectory)/eng/tools/generate-doc/dist/index.js --serviceDir "${{parameters.ServiceDirectory}}"
     displayName: "Run Typedoc Docs"
+    condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
+
+  # todo, possibly make this a wrapper node script instead of wrapper powershell.
+  - pwsh: |
+      if ("$(ChangedServices)") {
+        foreach($service in "$(ChangedServices)".Split(" ")) {
+          node $(Build.SourcesDirectory)/eng/tools/generate-doc/dist/index.js --serviceDir "${{parameters.ServiceDirectory}}"
+        }
+      }
+    displayName: "Run Typedocs for PR"
+    condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))
 
   - pwsh: |
       $(Build.SourcesDirectory)/eng/tools/compress-subfolders.ps1 "$(Build.SourcesDirectory)/docGen" "$(Build.SourcesDirectory)/docGen"

From 61aee6daa7dcb1d96b9bc62040d20f3c8f1635a2 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 2 Oct 2024 18:08:41 -0700
Subject: [PATCH 13/90] need to pass the service to each one

---
 eng/pipelines/templates/steps/generate-doc.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eng/pipelines/templates/steps/generate-doc.yml b/eng/pipelines/templates/steps/generate-doc.yml
index c7ea2691a5b1..445ac72c61a8 100644
--- a/eng/pipelines/templates/steps/generate-doc.yml
+++ b/eng/pipelines/templates/steps/generate-doc.yml
@@ -21,7 +21,7 @@ steps:
   - pwsh: |
       if ("$(ChangedServices)") {
         foreach($service in "$(ChangedServices)".Split(" ")) {
-          node $(Build.SourcesDirectory)/eng/tools/generate-doc/dist/index.js --serviceDir "${{parameters.ServiceDirectory}}"
+          node $(Build.SourcesDirectory)/eng/tools/generate-doc/dist/index.js --serviceDir "$service"
         }
       }
     displayName: "Run Typedocs for PR"

From cd78beaa6d173ac1fd758efdf79a049b6460e883 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 2 Oct 2024 18:16:01 -0700
Subject: [PATCH 14/90] $(Build.ArtifactStagingDirectory ->
 $(Build.ArtifactStagingDirectory)

---
 eng/pipelines/templates/steps/build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eng/pipelines/templates/steps/build.yml b/eng/pipelines/templates/steps/build.yml
index 2fc588b50b41..c90ce9103d6f 100644
--- a/eng/pipelines/templates/steps/build.yml
+++ b/eng/pipelines/templates/steps/build.yml
@@ -113,7 +113,7 @@ steps:
   # todo, handle direct/indirect packages here
   # todo, add the parse of the documentation status to eng/common so that we don't have to look at the ci.yml properties again
   - pwsh: |
-      $artifacts = Get-ChildItem -Recurse -Force "$(Build.ArtifactStagingDirectory/PackageInfo/*.json" `
+      $artifacts = Get-ChildItem -Recurse -Force "$(Build.ArtifactStagingDirectory)/PackageInfo/*.json" `
           | ForEach-Object { $_.Name.Replace(".json", "") }
 
       foreach ($artifact in $artifacts)

From b6d50deba3730fbc44084d35c27a86216c4570ff Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 2 Oct 2024 18:27:30 -0700
Subject: [PATCH 15/90] artifactName -> artifact

---
 eng/pipelines/templates/steps/build.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/eng/pipelines/templates/steps/build.yml b/eng/pipelines/templates/steps/build.yml
index c90ce9103d6f..78bfe6b7213a 100644
--- a/eng/pipelines/templates/steps/build.yml
+++ b/eng/pipelines/templates/steps/build.yml
@@ -127,8 +127,8 @@ steps:
 
         if ($${{ parameters.IncludeRelease }} -eq $true)
         {
-          New-Item -Type Directory -Name documentation -Path $(Build.ArtifactStagingDirectory)/$artifactName > $null
-          Copy-Item $(Build.SourcesDirectory)/docGen/$artifactName.zip $(Build.ArtifactStagingDirectory)/$artifactName/documentation
+          New-Item -Type Directory -Name documentation -Path $(Build.ArtifactStagingDirectory)/$artifact > $null
+          Copy-Item $(Build.SourcesDirectory)/docGen/$artifact.zip $(Build.ArtifactStagingDirectory)/$artifact/documentation
         }
       }
     displayName: 'Copy Packages'

From b913f2e0795703faa596442e08526e6730c80ee7 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Thu, 10 Oct 2024 11:04:20 -0700
Subject: [PATCH 16/90] pull across the artifact changes from the tools PR. use
 in analyze phase

---
 .../scripts/Helpers/Package-Helpers.ps1       | 48 +++++++++++++-
 eng/common/scripts/Package-Properties.ps1     | 62 ++++++++++++++++++-
 eng/scripts/Language-Settings.ps1             |  4 ++
 3 files changed, 111 insertions(+), 3 deletions(-)

diff --git a/eng/common/scripts/Helpers/Package-Helpers.ps1 b/eng/common/scripts/Helpers/Package-Helpers.ps1
index 3cc8adbb7e3d..325207b5cf08 100644
--- a/eng/common/scripts/Helpers/Package-Helpers.ps1
+++ b/eng/common/scripts/Helpers/Package-Helpers.ps1
@@ -12,7 +12,7 @@ function GetPackageKey($pkg) {
 
   return $pkgKey
 }
-  
+
 # Different language needs a different way to index the package. Build a map in convienice to lookup the package.
 # E.g. <groupId>:<packageName> is the package key in java.
 function GetPackageLookup($packageList) {
@@ -44,4 +44,50 @@ function GetDocsTocDisplayName($pkg) {
     $displayName += " (deprecated)"
   }
   return $displayName
+}
+
+
+<#
+.SYNOPSIS
+This function is a safe wrapper around `yq` and `ConvertFrom-Yaml` to convert YAML content to a PowerShell HashTable object
+
+.DESCRIPTION
+This function wraps `yq` and `ConvertFrom-Yaml` to convert YAML content to a PowerShell HashTable object. The reason this function exists is
+because while on a local user's machine, installing a module from the powershell gallery is an easy task, in pipelines we often have failures
+to install modules from the gallery. This function will attempt to use the `yq` command if it is available on the machine, and only will install
+the yaml module if `yq` is not available. This means that for the majority of runs on CI machines, the yaml module will not be installed.
+
+.PARAMETER Content
+The content to convert from YAML to a PowerShell HashTable object. Accepted as named argument or from the pipeline.
+
+.EXAMPLE
+CompatibleConvertFrom-Yaml -Content (Get-Content -Raw path/to/file.yml)
+
+.EXAMPLE
+Get-Content -Raw path/to/file.yml | CompatibleConvertFrom-Yaml
+#>
+function CompatibleConvertFrom-Yaml {
+  param(
+    [Parameter(Mandatory=$true, ValueFromPipeline=$true)]
+    [string]$Content
+  )
+
+  if (!($Content)) {
+    throw "Content to parse is a required input."
+  }
+
+  # Initialize any variables or checks that need to be done once
+  $yqPresent = Get-Command 'yq' -ErrorAction SilentlyContinue
+  if (-not $yqPresent) {
+    . (Join-Path $PSScriptRoot "../../../../eng/common/scripts/Helpers" PSModule-Helpers.ps1)
+    Install-ModuleIfNotInstalled -WhatIf:$false "powershell-yaml" "0.4.1" | Import-Module
+  }
+
+  # Process the content (for example, you could convert from YAML here)
+  if ($yqPresent) {
+      return ($content | yq -o=json | ConvertFrom-Json -AsHashTable)
+  }
+  else {
+      return ConvertFrom-Yaml $content
+  }
 }
\ No newline at end of file
diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1
index 486eeaca4b15..05bbb25ded1b 100644
--- a/eng/common/scripts/Package-Properties.ps1
+++ b/eng/common/scripts/Package-Properties.ps1
@@ -1,6 +1,6 @@
-# Helper functions for retireving useful information from azure-sdk-for-* repo
+# Helper functions for retrieving useful information from azure-sdk-for-* repo
 . "${PSScriptRoot}\logging.ps1"
-
+. "${PSScriptRoot}\Helpers\Package-Helpers.ps1"
 class PackageProps
 {
     [string]$Name
@@ -19,6 +19,7 @@ class PackageProps
     [boolean]$IncludedForValidation
     # does this package include other packages that we should trigger validation for?
     [string[]]$AdditionalValidationPackages
+    [HashTable]$ArtifactDetails
 
     PackageProps([string]$name, [string]$version, [string]$directoryPath, [string]$serviceDirectory)
     {
@@ -66,6 +67,8 @@ class PackageProps
         {
             $this.ChangeLogPath = $null
         }
+
+        $this.InitializeCIArtifacts()
     }
 
     hidden [void]Initialize(
@@ -79,6 +82,61 @@ class PackageProps
         $this.Initialize($name, $version, $directoryPath, $serviceDirectory)
         $this.Group = $group
     }
+
+    hidden [object]GetValueSafely($hashtable, [string[]]$keys) {
+        $current = $hashtable
+        foreach ($key in $keys) {
+            if ($current.ContainsKey($key) -or $current[$key]) {
+                $current = $current[$key]
+            }
+            else {
+                return $null
+            }
+        }
+
+        return $current
+    }
+
+    hidden [HashTable]ParseYmlForArtifact([string]$ymlPath) {
+        if (Test-Path -Path $ymlPath) {
+            try {
+                $content = Get-Content -Raw -Path $ymlPath | CompatibleConvertFrom-Yaml
+                if ($content) {
+                    $artifacts = $this.GetValueSafely($content, @("extends", "parameters", "Artifacts"))
+
+                    $artifactForCurrentPackage = $artifacts | Where-Object { $_["name"] -eq $this.ArtifactName -or $_["name"] -eq $this.Name }
+
+                    if ($artifactForCurrentPackage) {
+                        return [HashTable]$artifactForCurrentPackage
+                    }
+                }
+            }
+            catch {
+              Write-Host "Exception while parsing yml file $($ymlPath): $_"
+            }
+        }
+
+        return $null
+    }
+
+    [void]InitializeCIArtifacts(){
+        $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot ".." ".." "..")
+        # $ciFilePath = Join-Path -Path $RepoRoot -ChildPath (Join-Path "sdk" $this.ServiceDirectory "ci.yml")
+        # $ciMgmtYmlFilePath = Join-Path -Path $RepoRoot -ChildPath (Join-Path "sdk" $this.ServiceDirectory "ci.mgmt.yml")
+
+        $ciFolderPath = Join-Path -Path $RepoRoot -ChildPath (Join-Path "sdk" $this.ServiceDirectory)
+        $ciFiles = Get-ChildItem -Path $ciFolderPath -Filter "ci*.yml" -File
+
+        if (-not $this.ArtifactDetails) {
+            foreach($ciFile in $ciFiles) {
+                $ciArtifactResult = $this.ParseYmlForArtifact($ciFile.FullName)
+                if ($ciArtifactResult) {
+                    $this.ArtifactDetails = [Hashtable]$ciArtifactResult
+                    break
+                }
+            }
+        }
+    }
 }
 
 # Takes package name and service Name
diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1
index 35a21e6fbd3e..21262e4ca7f1 100644
--- a/eng/scripts/Language-Settings.ps1
+++ b/eng/scripts/Language-Settings.ps1
@@ -40,6 +40,10 @@ function Get-javascript-PackageInfoFromRepo ($pkgPath, $serviceDirectory) {
     }
     $pkgProp.IsNewSdk = ($pkgProp.SdkType -eq "client") -or ($pkgProp.SdkType -eq "mgmt")
     $pkgProp.ArtifactName = $jsStylePkgName
+    # necessary because we find the artifact configuration from the ci.yml file based upon the artifactname, which is
+    # not populated at time of constructor for the package properties default population of ArtifactDetails
+    $pkgProp.InitializeCIArtifacts()
+
     return $pkgProp
   }
   return $null

From 6daa9b0a9ea0a3e443d42c31b42a00bca01dc5e5 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Thu, 10 Oct 2024 13:06:42 -0700
Subject: [PATCH 17/90] pass the package info folder in analyze

---
 eng/pipelines/templates/steps/analyze.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml
index c36ef0ffa655..6ca4584e7e8f 100644
--- a/eng/pipelines/templates/steps/analyze.yml
+++ b/eng/pipelines/templates/steps/analyze.yml
@@ -75,7 +75,7 @@ steps:
 
   - template: /eng/pipelines/templates/steps/set-artifact-packages.yml
     parameters:
-      Artifacts: ${{ parameters.Artifacts }}
+      PackageInfo: $(Build.ArtifactStagingDirectory)/PackageInfo
 
   - script: |
       node eng/tools/rush-runner.js build "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)" --verbose -p max

From 741a3c72522751e0144a23238e238c51aaeecf9a Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Thu, 10 Oct 2024 13:10:15 -0700
Subject: [PATCH 18/90] repair the calls to resolve-package-artifacts

---
 eng/pipelines/templates/steps/analyze.yml | 2 ++
 eng/pipelines/templates/steps/test.yml    | 8 ++++++--
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml
index 6ca4584e7e8f..7c7641462b3c 100644
--- a/eng/pipelines/templates/steps/analyze.yml
+++ b/eng/pipelines/templates/steps/analyze.yml
@@ -5,6 +5,8 @@ parameters:
 
 steps:
   - template: /eng/common/pipelines/templates/steps/save-package-properties.yml
+    parameters:
+      ServiceDirectory: ${{parameters.ServiceDirectory}}
 
   - template: /eng/common/pipelines/templates/steps/check-spelling.yml
 
diff --git a/eng/pipelines/templates/steps/test.yml b/eng/pipelines/templates/steps/test.yml
index a5e1c57d6ac6..72130fb28c9c 100644
--- a/eng/pipelines/templates/steps/test.yml
+++ b/eng/pipelines/templates/steps/test.yml
@@ -9,6 +9,10 @@ steps:
     parameters:
       AgentImage: ${{ parameters.OSName}}
 
+  - template: /eng/common/pipelines/templates/steps/save-package-properties.yml
+    parameters:
+      ServiceDirectory: ${{parameters.ServiceDirectory}}
+
   - script: |
       node common/scripts/install-run-rush.js install
     displayName: "Install dependencies"
@@ -33,7 +37,7 @@ steps:
 
   - ${{ if eq(parameters.TestProxy, true) }}:
     - template: /eng/common/testproxy/test-proxy-standalone-tool.yml
-  
+
   # Option "-p max" ensures parallelism is set to the number of cores on all platforms, which improves build times.
   # The default on Windows is "cores - 1" (microsoft/rushstack#436).
   - script: |
@@ -53,7 +57,7 @@ steps:
         copy $(Build.SourcesDirectory)/test-proxy.log $(Build.ArtifactStagingDirectory)
       displayName: 'Dump Test Proxy logs'
       condition: succeededOrFailed()
-      
+
   # Unlink node_modules folders to significantly improve performance of subsequent tasks
   # which need to walk the directory tree (and are hardcoded to follow symlinks).
   # Retry for 30 seconds, since this command may fail with error "Another rush command is already

From f651fd105486b4f363c27f7caf38f382a851fe0d Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Thu, 10 Oct 2024 13:25:52 -0700
Subject: [PATCH 19/90] update everywhere that we utilize ServiceDirectory with
 changedServices in ci.tests.yml

---
 eng/pipelines/templates/steps/build.yml             | 11 +----------
 .../templates/steps/set-artifact-packages.yml       | 13 ++++++++++++-
 eng/pipelines/templates/steps/test.yml              |  8 ++++----
 3 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/eng/pipelines/templates/steps/build.yml b/eng/pipelines/templates/steps/build.yml
index 78bfe6b7213a..5c1e060f01d5 100644
--- a/eng/pipelines/templates/steps/build.yml
+++ b/eng/pipelines/templates/steps/build.yml
@@ -54,16 +54,7 @@ steps:
           echo "##vso[task.setvariable variable=rushBuildCacheCred;issecret=true;]$sasToken"
 
   - pwsh: |
-      $prDiff = Get-Content $(Build.ArtifactStagingDirectory)/diff/diff.json | ConvertFrom-Json -AsHashTable
-
-      $services = "auto"
-      if ($prDiff["ChangedServices"]) {
-        $services = $prDiff["ChangedServices"] -join " "
-      }
-
-      Write-Host "##vso[task.setvariable variable=ChangedServices;]$services"
-
-      node eng/tools/rush-runner.js build $services -packages "$(ArtifactPackageNames)" --verbose -p max
+      node eng/tools/rush-runner.js build $(ChangedServices) -packages "$(ArtifactPackageNames)" --verbose -p max
     displayName: "Build libraries for PR"
     condition: and(succeeded(),eq(variables['Build.Reason'],'PullRequest'))
     env:
diff --git a/eng/pipelines/templates/steps/set-artifact-packages.yml b/eng/pipelines/templates/steps/set-artifact-packages.yml
index 6b37e003d0df..4533b3fd7c8b 100644
--- a/eng/pipelines/templates/steps/set-artifact-packages.yml
+++ b/eng/pipelines/templates/steps/set-artifact-packages.yml
@@ -4,6 +4,17 @@ steps:
   # Package-Properties folder now contains the package properties for the packages that should be built or tested.
   # if targeting has been set by matrix, this value will already be populated, can't override it. Handle these cases here.
   - pwsh: |
+      # fix the parameter here don't just keep using the diff file directly, we need to rationalize between using this or the service directory
+      # depending if we're in a PR or not. this will work for some smoke testing though.
+      $prDiff = Get-Content $(Build.ArtifactStagingDirectory)/diff/diff.json | ConvertFrom-Json -AsHashTable
+
+      $services = "auto"
+      if ($prDiff["ChangedServices"]) {
+        $services = $prDiff["ChangedServices"] -join " "
+      }
+
+      Write-Host "##vso[task.setvariable variable=ChangedServices;]$services"
+
       Get-ChildItem -R "${{ parameters.PackageInfo }}"
 
       $packageProperties = Get-ChildItem -Recurse "${{ parameters.PackageInfo }}" *.json `
@@ -17,6 +28,6 @@ steps:
     condition: eq(variables['ArtifactPackageNames'], '')
 
   - pwsh: |
-      Write-Host "This run is targeting: `n$(ArtifactPackageNames)"
+      Write-Host "This run is targeting: `n$(ArtifactPackageNames) in [$(ChangedServices)]"
     displayName: Resolve Targeted Packages
     condition: ne(variables['ArtifactPackageNames'], '')
diff --git a/eng/pipelines/templates/steps/test.yml b/eng/pipelines/templates/steps/test.yml
index 72130fb28c9c..e1987a2e8fe3 100644
--- a/eng/pipelines/templates/steps/test.yml
+++ b/eng/pipelines/templates/steps/test.yml
@@ -24,13 +24,13 @@ steps:
   # Option "-p max" ensures parallelism is set to the number of cores on all platforms, which improves build times.
   # The default on Windows is "cores - 1" (microsoft/rushstack#436).
   - script: |
-      node eng/tools/rush-runner.js build "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)" --verbose -p max
+      node eng/tools/rush-runner.js build $(ChangedServices) -packages "$(ArtifactPackageNames)" --verbose -p max
     displayName: "Build libraries"
 
   # Option "-p max" ensures parallelism is set to the number of cores on all platforms, which improves build times.
   # The default on Windows is "cores - 1" (microsoft/rushstack#436).
   - script: |
-      node eng/tools/rush-runner.js build:test "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)" --verbose -p max
+      node eng/tools/rush-runner.js build:test $(ChangedServices) -packages "$(ArtifactPackageNames)" --verbose -p max
     displayName: "Build test assets"
 
   - template: ../steps/use-node-test-version.yml
@@ -41,14 +41,14 @@ steps:
   # Option "-p max" ensures parallelism is set to the number of cores on all platforms, which improves build times.
   # The default on Windows is "cores - 1" (microsoft/rushstack#436).
   - script: |
-      node eng/tools/rush-runner.js unit-test:node "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)" --verbose -p max
+      node eng/tools/rush-runner.js unit-test:node $(ChangedServices) -packages "$(ArtifactPackageNames)" --verbose -p max
     displayName: "Test libraries"
     condition: and(succeeded(),eq(variables['TestType'], 'node'))
 
   # Option "-p max" ensures parallelism is set to the number of cores on all platforms, which improves build times.
   # The default on Windows is "cores - 1" (microsoft/rushstack#436).
   - script: |
-      node eng/tools/rush-runner.js unit-test:browser "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)" --verbose -p max
+      node eng/tools/rush-runner.js unit-test:browser $(ChangedServices) -packages "$(ArtifactPackageNames)" --verbose -p max
     displayName: "Test libraries"
     condition: and(succeeded(),eq(variables['TestType'], 'browser'))
 

From 5cb6786182a2452c8f51235089412ace95f59885 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Thu, 10 Oct 2024 13:43:23 -0700
Subject: [PATCH 20/90] update analyze usage to pass changed services

---
 eng/pipelines/templates/steps/analyze.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml
index 7c7641462b3c..ab17c6f26208 100644
--- a/eng/pipelines/templates/steps/analyze.yml
+++ b/eng/pipelines/templates/steps/analyze.yml
@@ -23,7 +23,7 @@ steps:
     inputs:
       targetType: 'filePath'
       filePath: eng/scripts/spell-check-public-api.ps1
-      arguments: -ServiceDirectory ${{ parameters.ServiceDirectory }}
+      arguments: -ServiceDirectory $(ChangedServices)
       pwsh: true
     displayName: Spell check public APIs
     condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
@@ -80,19 +80,19 @@ steps:
       PackageInfo: $(Build.ArtifactStagingDirectory)/PackageInfo
 
   - script: |
-      node eng/tools/rush-runner.js build "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)" --verbose -p max
+      node eng/tools/rush-runner.js build $(ChangedServices) -packages "$(ArtifactPackageNames)" --verbose -p max
     displayName: "Build libraries"
 
   - template: /eng/pipelines/templates/steps/run-eslint.yml
     parameters:
-      ServiceDirectory: ${{ parameters.ServiceDirectory }}
+      ServiceDirectory: $(ChangedServices)
 
   - pwsh: |
-      node eng/tools/rush-runner.js check-format "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)" --verbose
+      node eng/tools/rush-runner.js check-format $(ChangedServices) -packages "$(ArtifactPackageNames)" --verbose
     displayName: "Check Format in Libraries"
 
   - script: |
-      node eng/tools/rush-runner.js audit "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)"
+      node eng/tools/rush-runner.js audit $(ChangedServices) -packages "$(ArtifactPackageNames)"
     condition: and(succeeded(), eq(variables['RunNpmAudit'], 'true'))
     displayName: "Audit libraries"
 

From d0f58a5e425567318705348b2b6ce1315a585c1f Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Thu, 10 Oct 2024 15:30:07 -0700
Subject: [PATCH 21/90] update to consume packageinfo instead of artifact list
 for set-artifact-packages

---
 eng/pipelines/templates/steps/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eng/pipelines/templates/steps/test.yml b/eng/pipelines/templates/steps/test.yml
index e1987a2e8fe3..81af90c88acc 100644
--- a/eng/pipelines/templates/steps/test.yml
+++ b/eng/pipelines/templates/steps/test.yml
@@ -19,7 +19,7 @@ steps:
 
   - template: /eng/pipelines/templates/steps/set-artifact-packages.yml
     parameters:
-      Artifacts: ${{ parameters.Artifacts }}
+      PackageInfo: $(Build.ArtifactStagingDirectory)/PackageInfo
 
   # Option "-p max" ensures parallelism is set to the number of cores on all platforms, which improves build times.
   # The default on Windows is "cores - 1" (microsoft/rushstack#436).

From e52aba6ec9c261e7cc7f7a05730bcb4c693a05fd Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Thu, 10 Oct 2024 17:44:10 -0700
Subject: [PATCH 22/90] actually, pass it --only?

---
 eng/tools/rush-runner.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eng/tools/rush-runner.js b/eng/tools/rush-runner.js
index 1128f86b0636..c912e23f39bc 100644
--- a/eng/tools/rush-runner.js
+++ b/eng/tools/rush-runner.js
@@ -195,7 +195,7 @@ if (serviceDirs.length === 0) {
     case "unit-test":
     case "integration-test":
       var rushCommandFlag = "--impacted-by";
-      if (isReducedTestScopeEnabled) {
+      if (isReducedTestScopeEnabled || serviceDirs.length > 1) {
         // If a service is configured to have reduced test matrix then run rush test only for those projects
         rushCommandFlag = "--only";
       }

From 0d0034cea82dd1f9913220111f7cc97786f42b8e Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Tue, 15 Oct 2024 16:13:12 -0700
Subject: [PATCH 23/90] upate rush-runner to invoke --to when handling core
 changes, otherwise falling back on --from

---
 eng/scripts/Language-Settings.ps1 | 10 ++++++++++
 eng/tools/rush-runner.js          | 10 ++++++++--
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1
index 21262e4ca7f1..7a04e2796436 100644
--- a/eng/scripts/Language-Settings.ps1
+++ b/eng/scripts/Language-Settings.ps1
@@ -40,6 +40,16 @@ function Get-javascript-PackageInfoFromRepo ($pkgPath, $serviceDirectory) {
     }
     $pkgProp.IsNewSdk = ($pkgProp.SdkType -eq "client") -or ($pkgProp.SdkType -eq "mgmt")
     $pkgProp.ArtifactName = $jsStylePkgName
+
+    $lookup = @{
+      'core' = @('@azure-rest/synapse-access-control', '@azure/arm-resources', '@azure/identity', '@azure/service-bus', '@azure/template')
+      'test-utils' = @('@azure-tests/perf-storage-blob', '@azure/arm-eventgrid', '@azure/ai-text-analytics', '@azure/identity', '@azure/template')
+      'identity' = @('@azure-tests/perf-storage-blob', '@azure/ai-text-analytics', '@azure/arm-resources', '@azure/identity-cache-persistence', '@azure/identity-vscode', '@azure/storage-blob', '@azure/template')
+    }
+    if ($lookup.ContainsKey($pkgProp.ServiceDirectory))) {
+      $pkgProp.AdditionalPackages = $lookup[$pkgProp.ServiceDirectory]
+    }
+
     # necessary because we find the artifact configuration from the ci.yml file based upon the artifactname, which is
     # not populated at time of constructor for the package properties default population of ArtifactDetails
     $pkgProp.InitializeCIArtifacts()
diff --git a/eng/tools/rush-runner.js b/eng/tools/rush-runner.js
index c912e23f39bc..d4c6860429f4 100644
--- a/eng/tools/rush-runner.js
+++ b/eng/tools/rush-runner.js
@@ -181,8 +181,14 @@ if (serviceDirs.length === 0) {
       // if this is build:test for any non-configured package service then all impacted projects downstream and it's dependents should be built
       var rushCommandFlag = "--impacted-by";
       if (isReducedTestScopeEnabled || serviceDirs.length > 1) {
-        // reduced preconfigured set of projects and it's required projects
-        rushCommandFlag = "--to";
+        // if we include a core package, then we should only target the packages that are passed in
+        if (serviceDirs.indexOf("core") !== -1) {
+          rushCommandFlag = "--to";
+        }
+        // otherwise we pull other packages in by dependency
+        else {
+          rushCommandFlag = "--from";
+        }
       }
       else if (actionComponents.length == 1) {
         rushCommandFlag = "--from";

From c5a6059bf1f0369dfc4db6a9acb459c824665e1f Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Tue, 15 Oct 2024 16:38:19 -0700
Subject: [PATCH 24/90] ensure that our new logic doesn't get activated in
 other service directories that are running as normal

---
 .../pipelines/templates/steps/save-package-properties.yml       | 2 +-
 eng/pipelines/templates/steps/analyze.yml                       | 1 +
 eng/pipelines/templates/steps/build.yml                         | 1 +
 eng/pipelines/templates/steps/test.yml                          | 1 +
 4 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/eng/common/pipelines/templates/steps/save-package-properties.yml b/eng/common/pipelines/templates/steps/save-package-properties.yml
index 1808dc2a00f3..cf7dc479b882 100644
--- a/eng/common/pipelines/templates/steps/save-package-properties.yml
+++ b/eng/common/pipelines/templates/steps/save-package-properties.yml
@@ -16,7 +16,7 @@ parameters:
     default: eng/common/scripts
 
 steps:
-  - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
+  - ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), or(eq(parameters.ServiceDirectoryFilter, ''), eq(parameters.ServiceDirectory, parameters.ServiceDirectoryFilter))) }}:
       - task: Powershell@2
         displayName: Generate PR Diff
         inputs:
diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml
index ab17c6f26208..8080073a0fe1 100644
--- a/eng/pipelines/templates/steps/analyze.yml
+++ b/eng/pipelines/templates/steps/analyze.yml
@@ -7,6 +7,7 @@ steps:
   - template: /eng/common/pipelines/templates/steps/save-package-properties.yml
     parameters:
       ServiceDirectory: ${{parameters.ServiceDirectory}}
+      ServiceDirectoryFilter: 'auto'
 
   - template: /eng/common/pipelines/templates/steps/check-spelling.yml
 
diff --git a/eng/pipelines/templates/steps/build.yml b/eng/pipelines/templates/steps/build.yml
index 5c1e060f01d5..97afec34811d 100644
--- a/eng/pipelines/templates/steps/build.yml
+++ b/eng/pipelines/templates/steps/build.yml
@@ -32,6 +32,7 @@ steps:
   - template: /eng/common/pipelines/templates/steps/save-package-properties.yml
     parameters:
       ServiceDirectory: ${{parameters.ServiceDirectory}}
+      ServiceDirectoryFilter: 'auto'
 
   - script: |
       node common/scripts/install-run-rush.js install
diff --git a/eng/pipelines/templates/steps/test.yml b/eng/pipelines/templates/steps/test.yml
index 81af90c88acc..f693be25d716 100644
--- a/eng/pipelines/templates/steps/test.yml
+++ b/eng/pipelines/templates/steps/test.yml
@@ -12,6 +12,7 @@ steps:
   - template: /eng/common/pipelines/templates/steps/save-package-properties.yml
     parameters:
       ServiceDirectory: ${{parameters.ServiceDirectory}}
+      ServiceDirectoryFilter: 'auto'
 
   - script: |
       node common/scripts/install-run-rush.js install

From b5a6a65a942b6483fcdaf71be04987168b0cd281 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Tue, 15 Oct 2024 17:01:48 -0700
Subject: [PATCH 25/90] add servicedirectoryfilter param

---
 .../pipelines/templates/steps/save-package-properties.yml      | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/eng/common/pipelines/templates/steps/save-package-properties.yml b/eng/common/pipelines/templates/steps/save-package-properties.yml
index cf7dc479b882..7e2bfa2445af 100644
--- a/eng/common/pipelines/templates/steps/save-package-properties.yml
+++ b/eng/common/pipelines/templates/steps/save-package-properties.yml
@@ -14,6 +14,9 @@ parameters:
   - name: ScriptDirectory
     type: string
     default: eng/common/scripts
+  - name: ServiceDirectoryFilter
+    type: string
+    default: ""
 
 steps:
   - ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), or(eq(parameters.ServiceDirectoryFilter, ''), eq(parameters.ServiceDirectory, parameters.ServiceDirectoryFilter))) }}:

From 16fd9e1710409db984103a63a84dec562e1662fc Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Tue, 15 Oct 2024 17:12:18 -0700
Subject: [PATCH 26/90] fix issue with language-settings

---
 eng/scripts/Language-Settings.ps1 | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1
index 7a04e2796436..55015748c50f 100644
--- a/eng/scripts/Language-Settings.ps1
+++ b/eng/scripts/Language-Settings.ps1
@@ -46,7 +46,8 @@ function Get-javascript-PackageInfoFromRepo ($pkgPath, $serviceDirectory) {
       'test-utils' = @('@azure-tests/perf-storage-blob', '@azure/arm-eventgrid', '@azure/ai-text-analytics', '@azure/identity', '@azure/template')
       'identity' = @('@azure-tests/perf-storage-blob', '@azure/ai-text-analytics', '@azure/arm-resources', '@azure/identity-cache-persistence', '@azure/identity-vscode', '@azure/storage-blob', '@azure/template')
     }
-    if ($lookup.ContainsKey($pkgProp.ServiceDirectory))) {
+
+    if ($lookup.ContainsKey($pkgProp.ServiceDirectory)) {
       $pkgProp.AdditionalPackages = $lookup[$pkgProp.ServiceDirectory]
     }
 

From ece0dbf79118b7d729b4e9e35ba3e7a9d899a302 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Tue, 15 Oct 2024 17:19:52 -0700
Subject: [PATCH 27/90] we need to set the property of package-props.
 AdditionalPackages doesn't exist

---
 eng/scripts/Language-Settings.ps1 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1
index 08040e630e16..68bb99c4599e 100644
--- a/eng/scripts/Language-Settings.ps1
+++ b/eng/scripts/Language-Settings.ps1
@@ -48,7 +48,7 @@ function Get-javascript-PackageInfoFromRepo ($pkgPath, $serviceDirectory) {
     }
 
     if ($lookup.ContainsKey($pkgProp.ServiceDirectory)) {
-      $pkgProp.AdditionalPackages = $lookup[$pkgProp.ServiceDirectory]
+      $pkgProp.AdditionalValidationPackages = $lookup[$pkgProp.ServiceDirectory]
     }
 
     # the constructor for the package properties object attempts to initialize CI artifacts on instantiation

From 1b2386778e739acdd8d24e48bb61eb4e032de09b Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 16 Oct 2024 11:58:56 -0700
Subject: [PATCH 28/90] save ChangedServices dynamically

---
 .../templates/steps/set-artifact-packages.yml | 22 +++++++------------
 1 file changed, 8 insertions(+), 14 deletions(-)

diff --git a/eng/pipelines/templates/steps/set-artifact-packages.yml b/eng/pipelines/templates/steps/set-artifact-packages.yml
index 4533b3fd7c8b..0127efa80675 100644
--- a/eng/pipelines/templates/steps/set-artifact-packages.yml
+++ b/eng/pipelines/templates/steps/set-artifact-packages.yml
@@ -4,24 +4,18 @@ steps:
   # Package-Properties folder now contains the package properties for the packages that should be built or tested.
   # if targeting has been set by matrix, this value will already be populated, can't override it. Handle these cases here.
   - pwsh: |
-      # fix the parameter here don't just keep using the diff file directly, we need to rationalize between using this or the service directory
-      # depending if we're in a PR or not. this will work for some smoke testing though.
-      $prDiff = Get-Content $(Build.ArtifactStagingDirectory)/diff/diff.json | ConvertFrom-Json -AsHashTable
+      $packageProperties = Get-ChildItem -Recurse "${{ parameters.PackageInfo }}" *.json
 
-      $services = "auto"
-      if ($prDiff["ChangedServices"]) {
-        $services = $prDiff["ChangedServices"] -join " "
-      }
-
-      Write-Host "##vso[task.setvariable variable=ChangedServices;]$services"
-
-      Get-ChildItem -R "${{ parameters.PackageInfo }}"
-
-      $packageProperties = Get-ChildItem -Recurse "${{ parameters.PackageInfo }}" *.json `
-        | Foreach-Object { $_.Name } `
+      $pkgNames = $packageProperties | Foreach-Object { $_.Name } `
         | ForEach-Object { $_.Replace(".json", "") }
       $setting = $packageProperties -join ","
 
+      $changedServicesArray = $packageProperties | Foreach-Object { Get-Content -Raw -Path $_.FullName | ConvertFrom-Json } `
+        | ForEach-Object { $_.ServiceDirectory } | Get-Unique
+      $changedServices = $changedServicesArray -join " "
+      Write-Host "Setting ChangedServices to : `n$changedServices"
+      Write-Host "##vso[task.setvariable variable=ChangedServices;]$changedServices"
+
       Write-Host "Setting ArtifactPackageNames to: `n$setting"
       Write-Host "##vso[task.setvariable variable=ArtifactPackageNames;]$setting"
     displayName: Resolve Targeted Packages

From 262617281da040c224c1763f8e1a9f5fcf947e13 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 16 Oct 2024 12:44:54 -0700
Subject: [PATCH 29/90] resolve changedServiced from the packageProperties
 files, instead of the PRDiff. this will allow the same code to run in pr vs
 internal build

---
 eng/pipelines/templates/steps/analyze.yml | 11 +----------
 eng/scripts/spell-check-public-apis.ps1   | 21 +++++++++------------
 2 files changed, 10 insertions(+), 22 deletions(-)

diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml
index 8080073a0fe1..d2a6b7acf3e3 100644
--- a/eng/pipelines/templates/steps/analyze.yml
+++ b/eng/pipelines/templates/steps/analyze.yml
@@ -15,20 +15,11 @@ steps:
     inputs:
       targetType: 'filePath'
       filePath: eng/scripts/spell-check-public-apis.ps1
-      arguments: -DiffJsonFile $(Build.ArtifactStagingDirectory)/diff/diff.json
+      arguments: -ChangedServices $(ChangedServices)
       pwsh: true
     displayName: Spell check public API for PR
     condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))
 
-  - task: PowerShell@2
-    inputs:
-      targetType: 'filePath'
-      filePath: eng/scripts/spell-check-public-api.ps1
-      arguments: -ServiceDirectory $(ChangedServices)
-      pwsh: true
-    displayName: Spell check public APIs
-    condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
-
   - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml
     parameters:
       PackageName: "@azure/template"
diff --git a/eng/scripts/spell-check-public-apis.ps1 b/eng/scripts/spell-check-public-apis.ps1
index 59d80c2f41c8..f11768fb9f25 100644
--- a/eng/scripts/spell-check-public-apis.ps1
+++ b/eng/scripts/spell-check-public-apis.ps1
@@ -8,14 +8,13 @@ Checks spelling of package's public API. Some packages may be excluded by
 criteria in the cspell.json config. The precise list of files to scan is
 determined by cspell. If a pacakge is opted out in the cspell.json a command
 will still be issued to scan that folder but cspell will report 0 files checked.
-Given a diff json file, this script will check the spelling of any service directories
-that have been changed.
 
-.PARAMETER DiffJson
-Scopes scanning to one or multiple service directories contained within `diff.json->ChangedServices`.
+.PARAMETER ChangedServices
+The list of service directories that have been changed in the PR. Space separated list of service directories.
 
 .EXAMPLE
-./spell-check-public-apis.ps1 -PRDiff diff.json
+$(ChangedServices) is set in set-artifact-packages.yml
+./spell-check-public-apis.ps1 -ChangedServices $(ChangedServices)
 
 Spell check all public API specs for all services under `sdk` that have been changed in the PR.
 
@@ -23,18 +22,16 @@ Spell check all public API specs for all services under `sdk` that have been cha
 [CmdletBinding()]
 param (
   [Parameter(mandatory = $true)]
-  $DiffJsonFile
+  [string]$ChangedServices
 )
 
 Set-StrictMode -Version 3.0
-
-$prDiff = Get-Content $DiffJsonFile | ConvertFrom-Json -AsHashTable
-
-
 $allSuccess = $true
-if ($prDiff["ChangedServices"]) {
-  foreach($service in $prDiff["ChangedServices"]) {
 
+if ($ChangedServices) {
+  $changed = $ChangedServices.Split(" ")
+
+  foreach ($service in $changed) {
     &"$PSScriptRoot/spell-check-public-api.ps1" -ServiceDirectory $service
 
     if($LASTEXITCODE -ne 0) {

From 48cb9cfd12744e678d6f76c8b035cd5c84248b6b Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 16 Oct 2024 12:52:17 -0700
Subject: [PATCH 30/90] resolve all the arguments here

---
 eng/pipelines/templates/steps/analyze.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml
index d2a6b7acf3e3..7fc161cd09ee 100644
--- a/eng/pipelines/templates/steps/analyze.yml
+++ b/eng/pipelines/templates/steps/analyze.yml
@@ -9,6 +9,10 @@ steps:
       ServiceDirectory: ${{parameters.ServiceDirectory}}
       ServiceDirectoryFilter: 'auto'
 
+  - template: /eng/pipelines/templates/steps/set-artifact-packages.yml
+    parameters:
+      PackageInfo: $(Build.ArtifactStagingDirectory)/PackageInfo
+
   - template: /eng/common/pipelines/templates/steps/check-spelling.yml
 
   - task: PowerShell@2

From ddba88448ecaad3851cb680428fbd9aeca1c2fe5 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 16 Oct 2024 13:36:44 -0700
Subject: [PATCH 31/90] standardizing some of the code, we don't need two
 separate handles when we set ChangedServices regardless

---
 eng/pipelines/templates/steps/analyze.yml |  7 +------
 eng/pipelines/templates/steps/build.yml   | 25 +++--------------------
 2 files changed, 4 insertions(+), 28 deletions(-)

diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml
index 7fc161cd09ee..bd94a593c3da 100644
--- a/eng/pipelines/templates/steps/analyze.yml
+++ b/eng/pipelines/templates/steps/analyze.yml
@@ -21,8 +21,7 @@ steps:
       filePath: eng/scripts/spell-check-public-apis.ps1
       arguments: -ChangedServices $(ChangedServices)
       pwsh: true
-    displayName: Spell check public API for PR
-    condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))
+    displayName: Spell check public API
 
   - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml
     parameters:
@@ -71,10 +70,6 @@ steps:
       node common/scripts/install-run-rush.js install
     displayName: "Install dependencies"
 
-  - template: /eng/pipelines/templates/steps/set-artifact-packages.yml
-    parameters:
-      PackageInfo: $(Build.ArtifactStagingDirectory)/PackageInfo
-
   - script: |
       node eng/tools/rush-runner.js build $(ChangedServices) -packages "$(ArtifactPackageNames)" --verbose -p max
     displayName: "Build libraries"
diff --git a/eng/pipelines/templates/steps/build.yml b/eng/pipelines/templates/steps/build.yml
index 97afec34811d..797cdedab300 100644
--- a/eng/pipelines/templates/steps/build.yml
+++ b/eng/pipelines/templates/steps/build.yml
@@ -56,19 +56,7 @@ steps:
 
   - pwsh: |
       node eng/tools/rush-runner.js build $(ChangedServices) -packages "$(ArtifactPackageNames)" --verbose -p max
-    displayName: "Build libraries for PR"
-    condition: and(succeeded(),eq(variables['Build.Reason'],'PullRequest'))
-    env:
-      ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) }}:
-        RUSH_BUILD_CACHE_CREDENTIAL: $(rushBuildCacheCred)
-        RUSH_BUILD_CACHE_WRITE_ALLOWED: 1
-
-  # Option "-p max" ensures parallelism is set to the number of cores on all platforms, which improves build times.
-  # The default on Windows is "cores - 1" (microsoft/rushstack#436).
-  - script: |
-      node eng/tools/rush-runner.js build "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)" --verbose -p max
-    displayName: "Build libraries for Service"
-    condition: and(succeeded(),ne(variables['Build.Reason'],'PullRequest'))
+    displayName: "Build libraries"
     env:
       ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) }}:
         RUSH_BUILD_CACHE_CREDENTIAL: $(rushBuildCacheCred)
@@ -77,12 +65,6 @@ steps:
   - script: |
       node eng/tools/rush-runner.js build:samples "$(ChangedServices)" -packages "$(ArtifactPackageNames)" --verbose
     displayName: "Build samples for PR"
-    condition: and(succeeded(),eq(variables['Build.Reason'],'PullRequest'))
-
-  - script: |
-      node eng/tools/rush-runner.js build:samples "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)" --verbose
-    displayName: "Build samples for Service"
-    condition: and(succeeded(),ne(variables['Build.Reason'],'PullRequest'))
 
   - pwsh: |
       eng/tools/check-api-warning.ps1
@@ -98,12 +80,11 @@ steps:
       node eng/tools/rush-runner.js unlink
     displayName: "Unlink dependencies"
 
+  # todo, modify this to walk the service directories
   - template: ../steps/generate-doc.yml
     parameters:
       ServiceDirectory: ${{parameters.ServiceDirectory}}
 
-  # todo, handle direct/indirect packages here
-  # todo, add the parse of the documentation status to eng/common so that we don't have to look at the ci.yml properties again
   - pwsh: |
       $artifacts = Get-ChildItem -Recurse -Force "$(Build.ArtifactStagingDirectory)/PackageInfo/*.json" `
           | ForEach-Object { $_.Name.Replace(".json", "") }
@@ -117,7 +98,7 @@ steps:
         Copy-Item sdk/$($artifactDetails.Service)/**/$artifact-[0-9]*.[0-9]*.[0-9]*.tgz $(Build.ArtifactStagingDirectory)/$artifact
         Copy-Item sdk/$($artifactDetails.Service)/**/browser/$artifact-[0-9]*.[0-9]*.[0-9]*.zip $(Build.ArtifactStagingDirectory)/$artifact
 
-        if ($${{ parameters.IncludeRelease }} -eq $true)
+        if ($${{ parameters.IncludeRelease }} -eq $true -and $artifactDetails.ArtifactDetails.skipPublishDocMs -ne $true)
         {
           New-Item -Type Directory -Name documentation -Path $(Build.ArtifactStagingDirectory)/$artifact > $null
           Copy-Item $(Build.SourcesDirectory)/docGen/$artifact.zip $(Build.ArtifactStagingDirectory)/$artifact/documentation

From 83464d4ce13be961257cae19980039b3707e0ae2 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 16 Oct 2024 14:18:25 -0700
Subject: [PATCH 32/90] update all of analyze for non-service-specific check

---
 .../pipelines/templates/steps/verify-samples.yml    |  4 ++--
 eng/pipelines/templates/steps/analyze.yml           | 13 +++++--------
 eng/pipelines/templates/steps/run-eslint.yml        |  4 ++--
 eng/scripts/spell-check-public-apis.ps1             |  2 +-
 4 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/eng/common/pipelines/templates/steps/verify-samples.yml b/eng/common/pipelines/templates/steps/verify-samples.yml
index d722cdcfd0bd..56fa36185936 100644
--- a/eng/common/pipelines/templates/steps/verify-samples.yml
+++ b/eng/common/pipelines/templates/steps/verify-samples.yml
@@ -12,10 +12,10 @@ parameters:
 steps:
   - pwsh: |
       # Take whichever parameter has been set. If set, ServiceDirectory will be a single path or
-      # ServiceDirectories will be a comma separated list.
+      # ServiceDirectories will be a space separated list.
       $ServiceDirectories = '${{ coalesce(parameters.ServiceDirectory, parameters.ServiceDirectories) }}'
       $ScanPaths = @()
-      foreach ($ServiceDirectory in $ServiceDirectories.Split(',')) {
+      foreach ($ServiceDirectory in $ServiceDirectories.Split(' ')) {
         $ScanPath = [System.IO.Path]::Combine('$(Build.SourcesDirectory)', 'sdk', $ServiceDirectory)
         Write-Host "Adding $ScanPath to the scanned paths"
         $ScanPaths += $ScanPath
diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml
index bd94a593c3da..db71fb94b248 100644
--- a/eng/pipelines/templates/steps/analyze.yml
+++ b/eng/pipelines/templates/steps/analyze.yml
@@ -54,7 +54,7 @@ steps:
 
   - template: /eng/common/pipelines/templates/steps/verify-samples.yml
     parameters:
-      ServiceDirectory: ${{ parameters.ServiceDirectory }}
+      ServiceDirectories: $(ChangedServices)
 
   - script: |
       npm ci
@@ -76,7 +76,7 @@ steps:
 
   - template: /eng/pipelines/templates/steps/run-eslint.yml
     parameters:
-      ServiceDirectory: $(ChangedServices)
+      ServiceDirectories: $(ChangedServices)
 
   - pwsh: |
       node eng/tools/rush-runner.js check-format $(ChangedServices) -packages "$(ArtifactPackageNames)" --verbose
@@ -87,12 +87,9 @@ steps:
     condition: and(succeeded(), eq(variables['RunNpmAudit'], 'true'))
     displayName: "Audit libraries"
 
-  - ${{ each artifact in parameters.Artifacts }}:
-      - template: /eng/common/pipelines/templates/steps/verify-changelog.yml
-        parameters:
-          PackageName: ${{artifact.name}}
-          ServiceName: ${{parameters.ServiceDirectory}}
-          ForRelease: false
+  - template: /eng/common/pipelines/templates/steps/verify-changelogs.yml
+    parameters:
+      PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo
 
   - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
     # ComponentGovernance is currently unable to run on pull requests of public projects. Running on non-PR
diff --git a/eng/pipelines/templates/steps/run-eslint.yml b/eng/pipelines/templates/steps/run-eslint.yml
index d12f49e20dfe..06e714f0f9dd 100644
--- a/eng/pipelines/templates/steps/run-eslint.yml
+++ b/eng/pipelines/templates/steps/run-eslint.yml
@@ -1,5 +1,5 @@
 parameters:
-  ServiceDirectory: ''
+  ServiceDirectories: ''
 
 steps:
   - script: |
@@ -8,5 +8,5 @@ steps:
 
   - pwsh: |
       node common/scripts/install-run-rush.js build -t @azure/eslint-plugin-azure-sdk -t @azure/monitor-opentelemetry-exporter
-      node eng/tools/rush-runner.js lint "${{parameters.ServiceDirectory}}" -p max
+      node eng/tools/rush-runner.js lint "${{parameters.ServiceDirectories}}" -p max
     displayName: "Build ESLint Plugin and Lint Libraries"
diff --git a/eng/scripts/spell-check-public-apis.ps1 b/eng/scripts/spell-check-public-apis.ps1
index f11768fb9f25..3e818ef0e895 100644
--- a/eng/scripts/spell-check-public-apis.ps1
+++ b/eng/scripts/spell-check-public-apis.ps1
@@ -6,7 +6,7 @@ against multiple service directories.
 .DESCRIPTION
 Checks spelling of package's public API. Some packages may be excluded by
 criteria in the cspell.json config. The precise list of files to scan is
-determined by cspell. If a pacakge is opted out in the cspell.json a command
+determined by cspell. If a package is opted out in the cspell.json a command
 will still be issued to scan that folder but cspell will report 0 files checked.
 
 .PARAMETER ChangedServices

From a3f2366d1c72b7fc9b37355d9d1d472692eb4e36 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 16 Oct 2024 14:42:17 -0700
Subject: [PATCH 33/90] update this thing to allow non-sdk changes to still
 trigger properly

---
 eng/scripts/Language-Settings.ps1 | 89 ++++++++++++++++++++++++++++---
 1 file changed, 82 insertions(+), 7 deletions(-)

diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1
index 68bb99c4599e..66b76383095a 100644
--- a/eng/scripts/Language-Settings.ps1
+++ b/eng/scripts/Language-Settings.ps1
@@ -6,6 +6,11 @@ $packagePattern = "*.tgz"
 $MetadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/main/_data/releases/latest/js-packages.csv"
 $GithubUri = "https://github.com/Azure/azure-sdk-for-js"
 $PackageRepositoryUri = "https://www.npmjs.com/package"
+$ReducedDependencyLookup = @{
+  'core' = @('@azure-rest/synapse-access-control', '@azure/arm-resources', '@azure/identity', '@azure/service-bus', '@azure/template')
+  'test-utils' = @('@azure-tests/perf-storage-blob', '@azure/arm-eventgrid', '@azure/ai-text-analytics', '@azure/identity', '@azure/template')
+  'identity' = @('@azure-tests/perf-storage-blob', '@azure/ai-text-analytics', '@azure/arm-resources', '@azure/identity-cache-persistence', '@azure/identity-vscode', '@azure/storage-blob', '@azure/template')
+}
 
 . "$PSScriptRoot/docs/Docs-ToC.ps1"
 . "$PSScriptRoot/docs/Docs-Onboarding.ps1"
@@ -25,6 +30,81 @@ function Get-javascript-EmitterAdditionalOptions([string]$projectDirectory) {
   return "--option @azure-tools/typespec-ts.emitter-output-dir=$projectDirectory/"
 }
 
+function Get-javascript-AdditionalValidationPackagesFromPackageSet {
+  param(
+    [Parameter(Mandatory=$true)]
+    $LocatedPackages,
+    [Parameter(Mandatory=$true)]
+    $diffObj,
+    [Parameter(Mandatory=$true)]
+    $AllPkgProps
+  )
+  $additionalValidationPackages = @()
+
+  function isOther($fileName) {
+    $startsWithPrefixes = @(".config", ".devcontainer", ".github", ".scripts", ".vscode", "common", "design", "documentation", "eng", "samples")
+
+    $startsWith = $false
+    foreach ($prefix in $startsWithPrefixes) {
+      if ($fileName.StartsWith($prefix)) {
+        $startsWith = $true
+      }
+    }
+
+    return $startsWith
+  }
+
+  $changedServices = @()
+  foreach($file in $diffObj.ChangedFiles) {
+    $pathComponents = $file -split "/"
+    # handle changes only in sdk/<service>/<file>/<extension>
+    if ($pathComponents.Length -eq 3 -and $pathComponents[0] -eq "sdk") {
+      $changedServices += $pathComponents[1]
+    }
+
+    # handle any changes under sdk/<file>.<extension>
+    if ($pathComponents.Length -eq 2 -and $pathComponents[0] -eq "sdk") {
+      changedServices += "template"
+    }
+  }
+
+  $engChanged = $diffObj.ChangedFiles | Where-Object { $_.StartsWith("eng")}
+  $othersChanged = $diffObj.ChangedFiles | Where-Object { isOther($_) }
+  $changedServices = $changedServices | Get-Unique
+
+  if ($engChanged -or $othersChanged) {
+    $additionalPackages = $ReducedDependencyLookup["core"] | ForEach-Object { $me=$_; $AllPkgProps | Where-Object { $_.Name -eq $me } | Select-Object -First 1 }
+    $additionalValidationPackages += $additionalPackages
+  }
+
+  foreach ($changedService in $changedServices) {
+    if ($ReducedDependencyLookup.ContainsKey($changedService)) {
+      $additionalPackages = $ReducedDependencyLookup[$changedService] | ForEach-Object { $me=$_; $AllPkgProps | Where-Object { $_.Name -eq $me } | Select-Object -First 1 }
+      $additionalValidationPackages += $additionalPackages
+    }
+    else {
+      $additionalPackages = $AllPkgProps | Where-Object { $_.ServiceDirectory -eq $changedService }
+      $additionalValidationPackages += $additionalPackages
+    }
+  }
+
+  $uniqueResultSet = @()
+  foreach ($pkg in $additionalValidationPackages) {
+    if ($uniqueResultSet -notcontains $pkg -and $LocatedPackages -notcontains $pkg) {
+      $pkg.IncludedForValidation = $true
+      $uniqueResultSet += $pkg
+    }
+  }
+
+  Write-Host "Returning additional packages for validation: $($uniqueResultSet.Count)"
+  foreach ($pkg in $uniqueResultSet) {
+    Write-Host "  - $($pkg.Name)"
+  }
+
+  return $uniqueResultSet
+}
+
+
 function Get-javascript-PackageInfoFromRepo ($pkgPath, $serviceDirectory) {
   $projectPath = Join-Path $pkgPath "package.json"
   if (Test-Path $projectPath) {
@@ -41,14 +121,9 @@ function Get-javascript-PackageInfoFromRepo ($pkgPath, $serviceDirectory) {
     $pkgProp.IsNewSdk = ($pkgProp.SdkType -eq "client") -or ($pkgProp.SdkType -eq "mgmt")
     $pkgProp.ArtifactName = $jsStylePkgName
 
-    $lookup = @{
-      'core' = @('@azure-rest/synapse-access-control', '@azure/arm-resources', '@azure/identity', '@azure/service-bus', '@azure/template')
-      'test-utils' = @('@azure-tests/perf-storage-blob', '@azure/arm-eventgrid', '@azure/ai-text-analytics', '@azure/identity', '@azure/template')
-      'identity' = @('@azure-tests/perf-storage-blob', '@azure/ai-text-analytics', '@azure/arm-resources', '@azure/identity-cache-persistence', '@azure/identity-vscode', '@azure/storage-blob', '@azure/template')
-    }
 
-    if ($lookup.ContainsKey($pkgProp.ServiceDirectory)) {
-      $pkgProp.AdditionalValidationPackages = $lookup[$pkgProp.ServiceDirectory]
+    if ($ReducedDependencyLookup.ContainsKey($pkgProp.ServiceDirectory)) {
+      $pkgProp.AdditionalValidationPackages = $ReducedDependencyLookup[$pkgProp.ServiceDirectory]
     }
 
     # the constructor for the package properties object attempts to initialize CI artifacts on instantiation

From 4722a54770099cc151bfdd65f65dfa4e56258773 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 16 Oct 2024 14:56:49 -0700
Subject: [PATCH 34/90] changedServices -> $changedServices

---
 eng/scripts/Language-Settings.ps1 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1
index 66b76383095a..6badef1497ec 100644
--- a/eng/scripts/Language-Settings.ps1
+++ b/eng/scripts/Language-Settings.ps1
@@ -64,7 +64,7 @@ function Get-javascript-AdditionalValidationPackagesFromPackageSet {
 
     # handle any changes under sdk/<file>.<extension>
     if ($pathComponents.Length -eq 2 -and $pathComponents[0] -eq "sdk") {
-      changedServices += "template"
+      $changedServices += "template"
     }
   }
 

From 4a7d646c37f18053afdf30a545f5cc71aa836ba5 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 16 Oct 2024 15:23:49 -0700
Subject: [PATCH 35/90] ensure the argument is surrounded in quotes

---
 eng/pipelines/templates/steps/analyze.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml
index db71fb94b248..7fd0c2950942 100644
--- a/eng/pipelines/templates/steps/analyze.yml
+++ b/eng/pipelines/templates/steps/analyze.yml
@@ -19,7 +19,7 @@ steps:
     inputs:
       targetType: 'filePath'
       filePath: eng/scripts/spell-check-public-apis.ps1
-      arguments: -ChangedServices $(ChangedServices)
+      arguments: -ChangedServices "$(ChangedServices)"
       pwsh: true
     displayName: Spell check public API
 

From b0727940897a86f8436b0774655d69529908859a Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 16 Oct 2024 15:41:53 -0700
Subject: [PATCH 36/90] fix spelling warnings

---
 .vscode/cspell.json | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.vscode/cspell.json b/.vscode/cspell.json
index 5da0892cef7b..7b74502d6c2f 100644
--- a/.vscode/cspell.json
+++ b/.vscode/cspell.json
@@ -167,7 +167,11 @@
   "overrides": [
     {
       "filename": "eng/pipelines",
-      "words": ["azuresdkartifacts", "gdnbaselines", "policheck"]
+      "words": ["azuresdkartifacts", "gdnbaselines", "policheck", "issecret", "dlrw"]
+    },
+    {
+      "filename": "eng/tools/rush-runner.js",
+      "words": ["Jsons"]
     },
     {
       "filename": "sdk/apimanagement/api-management-custom-widgets-scaffolder/review/api-management-custom-widgets-scaffolder.api.md",

From aa7643f0fc63ce69054ce70abef9580f38db8d1d Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 16 Oct 2024 15:43:36 -0700
Subject: [PATCH 37/90] update step name

---
 eng/pipelines/templates/steps/generate-doc.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eng/pipelines/templates/steps/generate-doc.yml b/eng/pipelines/templates/steps/generate-doc.yml
index 445ac72c61a8..eaf1e653b317 100644
--- a/eng/pipelines/templates/steps/generate-doc.yml
+++ b/eng/pipelines/templates/steps/generate-doc.yml
@@ -29,4 +29,4 @@ steps:
 
   - pwsh: |
       $(Build.SourcesDirectory)/eng/tools/compress-subfolders.ps1 "$(Build.SourcesDirectory)/docGen" "$(Build.SourcesDirectory)/docGen"
-    displayName: "Generate Typedoc Docs"
+    displayName: "Compress generated docs"

From d0d2f1f6fbc4888d09e2239ff6d05ca631e67f6b Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 16 Oct 2024 16:27:39 -0700
Subject: [PATCH 38/90] we weren't passing the correct package names

---
 eng/pipelines/templates/steps/set-artifact-packages.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eng/pipelines/templates/steps/set-artifact-packages.yml b/eng/pipelines/templates/steps/set-artifact-packages.yml
index 0127efa80675..9178e776bbc9 100644
--- a/eng/pipelines/templates/steps/set-artifact-packages.yml
+++ b/eng/pipelines/templates/steps/set-artifact-packages.yml
@@ -8,7 +8,7 @@ steps:
 
       $pkgNames = $packageProperties | Foreach-Object { $_.Name } `
         | ForEach-Object { $_.Replace(".json", "") }
-      $setting = $packageProperties -join ","
+      $setting = $pkgNames -join ","
 
       $changedServicesArray = $packageProperties | Foreach-Object { Get-Content -Raw -Path $_.FullName | ConvertFrom-Json } `
         | ForEach-Object { $_.ServiceDirectory } | Get-Unique

From 0ce2151c4a61f68390cf3e0abc39e6560ce94a66 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Fri, 18 Oct 2024 14:46:35 -0700
Subject: [PATCH 39/90] updating to handle the split between mgmt and not

---
 eng/pipelines/templates/steps/analyze.yml     |  1 +
 eng/pipelines/templates/steps/build.yml       |  2 ++
 .../templates/steps/set-artifact-packages.yml | 25 +++++++++++++++----
 eng/scripts/Language-Settings.ps1             |  2 +-
 4 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml
index 7fd0c2950942..612aec95bd2b 100644
--- a/eng/pipelines/templates/steps/analyze.yml
+++ b/eng/pipelines/templates/steps/analyze.yml
@@ -12,6 +12,7 @@ steps:
   - template: /eng/pipelines/templates/steps/set-artifact-packages.yml
     parameters:
       PackageInfo: $(Build.ArtifactStagingDirectory)/PackageInfo
+      Artifacts: ${{ parameters.Artifacts }}
 
   - template: /eng/common/pipelines/templates/steps/check-spelling.yml
 
diff --git a/eng/pipelines/templates/steps/build.yml b/eng/pipelines/templates/steps/build.yml
index 797cdedab300..a3f5db47d66c 100644
--- a/eng/pipelines/templates/steps/build.yml
+++ b/eng/pipelines/templates/steps/build.yml
@@ -42,6 +42,8 @@ steps:
   - template: /eng/pipelines/templates/steps/set-artifact-packages.yml
     parameters:
       PackageInfo: $(Build.ArtifactStagingDirectory)/PackageInfo
+      Artifacts: ${{ parameters.Artifacts }}
+
   - ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) }}:
     - task: AzureCLI@2
       inputs:
diff --git a/eng/pipelines/templates/steps/set-artifact-packages.yml b/eng/pipelines/templates/steps/set-artifact-packages.yml
index 9178e776bbc9..69655e2edf05 100644
--- a/eng/pipelines/templates/steps/set-artifact-packages.yml
+++ b/eng/pipelines/templates/steps/set-artifact-packages.yml
@@ -1,21 +1,36 @@
 parameters:
   PackageInfo: ''
+  Artifacts: []
 steps:
-  # Package-Properties folder now contains the package properties for the packages that should be built or tested.
-  # if targeting has been set by matrix, this value will already be populated, can't override it. Handle these cases here.
+  # Package-Properties folder contains the package properties for all discovered packages that were either A) affected by the PR or
+  # B) explicitly present in the service directory. This repo splits the builds into two categories: "mgmt" and "dataplane". While
+  # a given directory may contain both, there will be a separate build definition to release the management packages. Due to this
+  # we need to artificially filter the packages to ensure that only the packages targeted for THAT ci.yml are built.
+  # When we merge the PR adding js - pullrequest, this code will also merge, meaning the public - <service> - ci builds will still operate
+  # EXACTLY the same as they did before the pipelinev3 change. This is important to ensure that we don't accidentally build the wrong packages
+  # while in the integration period. After we disable all the public `js - <service> - ci` builds, only the `js - pullrequest` build WONT provide
+  # an artifact list, which will allow the expand/contract. Meanwhile the internal builds will continue to provide the artifact list from their
+  # individual ci.yml and ci.mgmt.yml files.
   - pwsh: |
-      $packageProperties = Get-ChildItem -Recurse "${{ parameters.PackageInfo }}" *.json
+      $artifacts = '${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json
+      $knownArtifacts = @()
+      if ($artifacts) {
+        $knownArtifacts = $artifacts | ForEach-Object { $knownArtifacts += $_.name }
+      }
+
+      $packageProperties = Get-ChildItem -Recurse "${{ parameters.PackageInfo }}" *.json `
+        | Where-Object { if($knownArtifacts) { $knownArtifacts -contains $_.Name.Replace(".json", "") } else { $true } }
 
       $pkgNames = $packageProperties | Foreach-Object { $_.Name } `
         | ForEach-Object { $_.Replace(".json", "") }
-      $setting = $pkgNames -join ","
 
       $changedServicesArray = $packageProperties | Foreach-Object { Get-Content -Raw -Path $_.FullName | ConvertFrom-Json } `
         | ForEach-Object { $_.ServiceDirectory } | Get-Unique
+
+      $setting = $pkgNames -join ","
       $changedServices = $changedServicesArray -join " "
       Write-Host "Setting ChangedServices to : `n$changedServices"
       Write-Host "##vso[task.setvariable variable=ChangedServices;]$changedServices"
-
       Write-Host "Setting ArtifactPackageNames to: `n$setting"
       Write-Host "##vso[task.setvariable variable=ArtifactPackageNames;]$setting"
     displayName: Resolve Targeted Packages
diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1
index 6badef1497ec..8e3c2877b714 100644
--- a/eng/scripts/Language-Settings.ps1
+++ b/eng/scripts/Language-Settings.ps1
@@ -73,7 +73,7 @@ function Get-javascript-AdditionalValidationPackagesFromPackageSet {
   $changedServices = $changedServices | Get-Unique
 
   if ($engChanged -or $othersChanged) {
-    $additionalPackages = $ReducedDependencyLookup["core"] | ForEach-Object { $me=$_; $AllPkgProps | Where-Object { $_.Name -eq $me } | Select-Object -First 1 }
+    $additionalPackages = $ReducedDependencyLookup["test-utils"] | ForEach-Object { $me=$_; $AllPkgProps | Where-Object { $_.Name -eq $me } | Select-Object -First 1 }
     $additionalValidationPackages += $additionalPackages
   }
 

From 1c6218b3b0d1b13cbacbf449c47849fe607f77a7 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Fri, 18 Oct 2024 15:02:12 -0700
Subject: [PATCH 40/90] flatten to what's in main

---
 .../pipelines/templates/steps/save-package-properties.yml     | 3 ---
 eng/common/pipelines/templates/steps/verify-samples.yml       | 4 ++--
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/eng/common/pipelines/templates/steps/save-package-properties.yml b/eng/common/pipelines/templates/steps/save-package-properties.yml
index af3536cc56f5..3714c0264388 100644
--- a/eng/common/pipelines/templates/steps/save-package-properties.yml
+++ b/eng/common/pipelines/templates/steps/save-package-properties.yml
@@ -14,9 +14,6 @@ parameters:
   - name: ScriptDirectory
     type: string
     default: eng/common/scripts
-  - name: ServiceDirectoryFilter
-    type: string
-    default: ""
 
 steps:
   # There will be transitory period for every language repo where the <language> - pullrequest build definition will run
diff --git a/eng/common/pipelines/templates/steps/verify-samples.yml b/eng/common/pipelines/templates/steps/verify-samples.yml
index 56fa36185936..d722cdcfd0bd 100644
--- a/eng/common/pipelines/templates/steps/verify-samples.yml
+++ b/eng/common/pipelines/templates/steps/verify-samples.yml
@@ -12,10 +12,10 @@ parameters:
 steps:
   - pwsh: |
       # Take whichever parameter has been set. If set, ServiceDirectory will be a single path or
-      # ServiceDirectories will be a space separated list.
+      # ServiceDirectories will be a comma separated list.
       $ServiceDirectories = '${{ coalesce(parameters.ServiceDirectory, parameters.ServiceDirectories) }}'
       $ScanPaths = @()
-      foreach ($ServiceDirectory in $ServiceDirectories.Split(' ')) {
+      foreach ($ServiceDirectory in $ServiceDirectories.Split(',')) {
         $ScanPath = [System.IO.Path]::Combine('$(Build.SourcesDirectory)', 'sdk', $ServiceDirectory)
         Write-Host "Adding $ScanPath to the scanned paths"
         $ScanPaths += $ScanPath

From 73989265a9f888320def0dbdf781b3e9fb7f855a Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Fri, 18 Oct 2024 15:02:48 -0700
Subject: [PATCH 41/90] remove trailing space

---
 eng/common/scripts/Helpers/Package-Helpers.ps1 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eng/common/scripts/Helpers/Package-Helpers.ps1 b/eng/common/scripts/Helpers/Package-Helpers.ps1
index a04dc47022be..985ebcc4ee3b 100644
--- a/eng/common/scripts/Helpers/Package-Helpers.ps1
+++ b/eng/common/scripts/Helpers/Package-Helpers.ps1
@@ -90,4 +90,4 @@ function CompatibleConvertFrom-Yaml {
   else {
       return ConvertFrom-Yaml $content
   }
-}
+}
\ No newline at end of file

From d503e4b13d7d2ca14bc8548eafa679d72c5d1886 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Fri, 18 Oct 2024 15:04:45 -0700
Subject: [PATCH 42/90] fix ChangedServices on the fly

---
 eng/pipelines/templates/steps/analyze.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml
index 612aec95bd2b..9df01430e6d0 100644
--- a/eng/pipelines/templates/steps/analyze.yml
+++ b/eng/pipelines/templates/steps/analyze.yml
@@ -55,7 +55,7 @@ steps:
 
   - template: /eng/common/pipelines/templates/steps/verify-samples.yml
     parameters:
-      ServiceDirectories: $(ChangedServices)
+      ServiceDirectories: $[replace(variables['ChangedServices'], ' ', ',')]
 
   - script: |
       npm ci

From 36140a41e42a8200918ca2ada81d3c1a7982d587 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Fri, 18 Oct 2024 15:05:33 -0700
Subject: [PATCH 43/90] remove unnecessary parameter ServiceDirectoryFilter it
 is hardcoded

---
 eng/pipelines/templates/steps/analyze.yml | 1 -
 eng/pipelines/templates/steps/build.yml   | 1 -
 eng/pipelines/templates/steps/test.yml    | 1 -
 3 files changed, 3 deletions(-)

diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml
index 9df01430e6d0..e5374d22e889 100644
--- a/eng/pipelines/templates/steps/analyze.yml
+++ b/eng/pipelines/templates/steps/analyze.yml
@@ -7,7 +7,6 @@ steps:
   - template: /eng/common/pipelines/templates/steps/save-package-properties.yml
     parameters:
       ServiceDirectory: ${{parameters.ServiceDirectory}}
-      ServiceDirectoryFilter: 'auto'
 
   - template: /eng/pipelines/templates/steps/set-artifact-packages.yml
     parameters:
diff --git a/eng/pipelines/templates/steps/build.yml b/eng/pipelines/templates/steps/build.yml
index a3f5db47d66c..5bc230d53fbb 100644
--- a/eng/pipelines/templates/steps/build.yml
+++ b/eng/pipelines/templates/steps/build.yml
@@ -32,7 +32,6 @@ steps:
   - template: /eng/common/pipelines/templates/steps/save-package-properties.yml
     parameters:
       ServiceDirectory: ${{parameters.ServiceDirectory}}
-      ServiceDirectoryFilter: 'auto'
 
   - script: |
       node common/scripts/install-run-rush.js install
diff --git a/eng/pipelines/templates/steps/test.yml b/eng/pipelines/templates/steps/test.yml
index f693be25d716..81af90c88acc 100644
--- a/eng/pipelines/templates/steps/test.yml
+++ b/eng/pipelines/templates/steps/test.yml
@@ -12,7 +12,6 @@ steps:
   - template: /eng/common/pipelines/templates/steps/save-package-properties.yml
     parameters:
       ServiceDirectory: ${{parameters.ServiceDirectory}}
-      ServiceDirectoryFilter: 'auto'
 
   - script: |
       node common/scripts/install-run-rush.js install

From 2166d3b241960aaa87a788b67f7bb815fe870416 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Tue, 22 Oct 2024 11:22:45 -0700
Subject: [PATCH 44/90] try using double quotes instead of single, does that
 work?

---
 eng/pipelines/templates/steps/analyze.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml
index e5374d22e889..05d1e9781dc9 100644
--- a/eng/pipelines/templates/steps/analyze.yml
+++ b/eng/pipelines/templates/steps/analyze.yml
@@ -54,7 +54,7 @@ steps:
 
   - template: /eng/common/pipelines/templates/steps/verify-samples.yml
     parameters:
-      ServiceDirectories: $[replace(variables['ChangedServices'], ' ', ',')]
+      ServiceDirectories: $[replace(variables["ChangedServices"], " ", ",")]
 
   - script: |
       npm ci

From 01b582274617321eac99dae562e9e50a6b45b720 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Tue, 22 Oct 2024 11:35:01 -0700
Subject: [PATCH 45/90] update to the runtime expression

---
 eng/pipelines/templates/steps/analyze.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml
index 05d1e9781dc9..675a37c1896c 100644
--- a/eng/pipelines/templates/steps/analyze.yml
+++ b/eng/pipelines/templates/steps/analyze.yml
@@ -54,7 +54,7 @@ steps:
 
   - template: /eng/common/pipelines/templates/steps/verify-samples.yml
     parameters:
-      ServiceDirectories: $[replace(variables["ChangedServices"], " ", ",")]
+      ServiceDirectories: $[replace(variables["ChangedServices"], ' ', ',')]
 
   - script: |
       npm ci

From 37c108477b6a017d88043073dcdbae1aea5df3b1 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Tue, 22 Oct 2024 11:53:29 -0700
Subject: [PATCH 46/90] adjust the issue with verify-samples

---
 eng/common/pipelines/templates/steps/verify-samples.yml | 2 +-
 eng/pipelines/templates/steps/analyze.yml               | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/eng/common/pipelines/templates/steps/verify-samples.yml b/eng/common/pipelines/templates/steps/verify-samples.yml
index d722cdcfd0bd..5161db205538 100644
--- a/eng/common/pipelines/templates/steps/verify-samples.yml
+++ b/eng/common/pipelines/templates/steps/verify-samples.yml
@@ -15,7 +15,7 @@ steps:
       # ServiceDirectories will be a comma separated list.
       $ServiceDirectories = '${{ coalesce(parameters.ServiceDirectory, parameters.ServiceDirectories) }}'
       $ScanPaths = @()
-      foreach ($ServiceDirectory in $ServiceDirectories.Split(',')) {
+      foreach ($ServiceDirectory in $ServiceDirectories.Split(' ')) {
         $ScanPath = [System.IO.Path]::Combine('$(Build.SourcesDirectory)', 'sdk', $ServiceDirectory)
         Write-Host "Adding $ScanPath to the scanned paths"
         $ScanPaths += $ScanPath
diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml
index 675a37c1896c..84c85726452b 100644
--- a/eng/pipelines/templates/steps/analyze.yml
+++ b/eng/pipelines/templates/steps/analyze.yml
@@ -54,7 +54,7 @@ steps:
 
   - template: /eng/common/pipelines/templates/steps/verify-samples.yml
     parameters:
-      ServiceDirectories: $[replace(variables["ChangedServices"], ' ', ',')]
+      ServiceDirectories: $(ChangedServices)
 
   - script: |
       npm ci

From 599bf6e8c78c7bda2400dd092035e0d6e84d6ddc Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Tue, 22 Oct 2024 12:19:45 -0700
Subject: [PATCH 47/90] CommaChangedServices def

---
 eng/common/pipelines/templates/steps/verify-samples.yml | 2 +-
 eng/pipelines/templates/steps/analyze.yml               | 2 +-
 eng/pipelines/templates/steps/set-artifact-packages.yml | 2 ++
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/eng/common/pipelines/templates/steps/verify-samples.yml b/eng/common/pipelines/templates/steps/verify-samples.yml
index 5161db205538..d722cdcfd0bd 100644
--- a/eng/common/pipelines/templates/steps/verify-samples.yml
+++ b/eng/common/pipelines/templates/steps/verify-samples.yml
@@ -15,7 +15,7 @@ steps:
       # ServiceDirectories will be a comma separated list.
       $ServiceDirectories = '${{ coalesce(parameters.ServiceDirectory, parameters.ServiceDirectories) }}'
       $ScanPaths = @()
-      foreach ($ServiceDirectory in $ServiceDirectories.Split(' ')) {
+      foreach ($ServiceDirectory in $ServiceDirectories.Split(',')) {
         $ScanPath = [System.IO.Path]::Combine('$(Build.SourcesDirectory)', 'sdk', $ServiceDirectory)
         Write-Host "Adding $ScanPath to the scanned paths"
         $ScanPaths += $ScanPath
diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml
index 84c85726452b..66938ffcccfe 100644
--- a/eng/pipelines/templates/steps/analyze.yml
+++ b/eng/pipelines/templates/steps/analyze.yml
@@ -54,7 +54,7 @@ steps:
 
   - template: /eng/common/pipelines/templates/steps/verify-samples.yml
     parameters:
-      ServiceDirectories: $(ChangedServices)
+      ServiceDirectories: $(CChangedServices)
 
   - script: |
       npm ci
diff --git a/eng/pipelines/templates/steps/set-artifact-packages.yml b/eng/pipelines/templates/steps/set-artifact-packages.yml
index 69655e2edf05..c520208a9ddf 100644
--- a/eng/pipelines/templates/steps/set-artifact-packages.yml
+++ b/eng/pipelines/templates/steps/set-artifact-packages.yml
@@ -29,8 +29,10 @@ steps:
 
       $setting = $pkgNames -join ","
       $changedServices = $changedServicesArray -join " "
+      $commaChangedServices = $changedServicesArray -join ","
       Write-Host "Setting ChangedServices to : `n$changedServices"
       Write-Host "##vso[task.setvariable variable=ChangedServices;]$changedServices"
+      Write-Host "##vso[task.setvariable variable=CChangedServices;]$commaChangedServices"
       Write-Host "Setting ArtifactPackageNames to: `n$setting"
       Write-Host "##vso[task.setvariable variable=ArtifactPackageNames;]$setting"
     displayName: Resolve Targeted Packages

From 53f5ef8320dfd7c5ee892b74b6f81f3e26810c9c Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Tue, 22 Oct 2024 14:00:23 -0700
Subject: [PATCH 48/90] handle the specific --to vs --from requests

---
 eng/scripts/Language-Settings.ps1 |  2 +-
 eng/tools/rush-runner.js          | 88 ++++++++++++++++++++++++-------
 2 files changed, 69 insertions(+), 21 deletions(-)

diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1
index 8e3c2877b714..6badef1497ec 100644
--- a/eng/scripts/Language-Settings.ps1
+++ b/eng/scripts/Language-Settings.ps1
@@ -73,7 +73,7 @@ function Get-javascript-AdditionalValidationPackagesFromPackageSet {
   $changedServices = $changedServices | Get-Unique
 
   if ($engChanged -or $othersChanged) {
-    $additionalPackages = $ReducedDependencyLookup["test-utils"] | ForEach-Object { $me=$_; $AllPkgProps | Where-Object { $_.Name -eq $me } | Select-Object -First 1 }
+    $additionalPackages = $ReducedDependencyLookup["core"] | ForEach-Object { $me=$_; $AllPkgProps | Where-Object { $_.Name -eq $me } | Select-Object -First 1 }
     $additionalValidationPackages += $additionalPackages
   }
 
diff --git a/eng/tools/rush-runner.js b/eng/tools/rush-runner.js
index d4c6860429f4..221add761141 100644
--- a/eng/tools/rush-runner.js
+++ b/eng/tools/rush-runner.js
@@ -94,6 +94,51 @@ const getPackageJsons = (searchDir) => {
   return sdkDirectories.concat(perfTestDirectories).filter((f) => fs.existsSync(f)); // only keep paths for files that actually exist
 };
 
+const restrictedToPackages = [
+  "@azure/abort-controller",
+  "@azure/core-amqp",
+  "@azure/core-auth",
+  "@azure/core-client",
+  "@azure/core-http-compat",
+  "@azure/core-lro",
+  "@azure/core-paging",
+  "@azure/core-rest-pipeline",
+  "@azure/core-sse",
+  "@azure/core-tracing",
+  "@azure/core-util",
+  "@azure/core-xml",
+  "@azure/logger",
+  "@azure-rest/core-client",
+  "@typespec/ts-http-runtime",
+  "@azure/identity",
+  "@azure/arm-resources"
+];
+const getDirectionMappedPackages = (serviceDirs, packageNames) => {
+  const mappedPackages = [];
+
+  for (const packageName of packageNames) {
+    // Build command without any additional option should build the project and downstream
+    // If service is configured to run only a set of downstream projects then build all projects leading to them to support testing
+    // If the package is a core package, azure-identity or arm-resources then build only the package,
+    // otherwise build the package and all its dependents
+    var rushCommandFlag = "--impacted-by";
+
+    if (restrictedToPackages.includes(packageName)) {
+      // if this is one of our restricted packages with a ton of deps, make it targeted
+      // as including all dependents will be too much
+      rushCommandFlag = "--to";
+    }
+    else if (actionComponents.length == 1) {
+      // else we are building the project and its dependents
+      rushCommandFlag = "--from";
+    }
+
+    mappedPackages.push([rushCommandFlag, packageName]);
+  }
+
+  return mappedPackages;
+};
+
 const getServicePackages = (baseDir, serviceDirs, artifactNames) => {
   const packageNames = [];
   const packageDirs = [];
@@ -136,9 +181,12 @@ const flatMap = (arr, f) => {
 };
 
 const [baseDir, action, serviceDirs, rushParams, artifactNames] = parseArgs();
+const actionComponents = action.toLowerCase().split(":");
 
 const [packageNames, packageDirs] = getServicePackages(baseDir, serviceDirs, artifactNames);
 
+const packagesWithDirection = getDirectionMappedPackages(serviceDirs, packageNames);
+
 /**
  * Helper function to provide the rush logic that is used frequently below
  *
@@ -150,6 +198,15 @@ function rushRunAll(direction, packages) {
   spawnNode(baseDir, "common/scripts/install-run-rush.js", action, ...params, ...rushParams);
 }
 
+/**
+ * Helper function to invoke a bunch of combined rush commands.
+ *
+ * @param packagesWithDirection string[][] Any array of tuples containing [[direction, packageName]...]
+ */
+function rushRunAll(packagesWithDirection) {
+  spawnNode(baseDir, "common/scripts/install-run-rush.js", action, ...packagesWithDirection, ...rushParams);
+}
+
 /**
  * Helper function to get the relative path of a package directory from an absolute
  * one
@@ -173,34 +230,25 @@ const rushx_runner_path = path.join(baseDir, "common/scripts/install-run-rushx.j
 if (serviceDirs.length === 0) {
   spawnNode(baseDir, "common/scripts/install-run-rush.js", action, ...rushParams);
 } else {
-  const actionComponents = action.toLowerCase().split(":");
+
+  // if the targeted package is one of:
+  // - a core package
+  // - azure-identity
+  // - arm-resources
+  // use the --to clause with the package
+
+  // if the targeted package is NOT one of those, then run it with the
+  // --from clause to resolve the package and all of its dependents
   switch (actionComponents[0]) {
     case "build":
-      // Build command without any additional option should build the project and downstream
-      // If service is configured to run only a set of downstream projects then build all projects leading to them to support testing
-      // if this is build:test for any non-configured package service then all impacted projects downstream and it's dependents should be built
-      var rushCommandFlag = "--impacted-by";
-      if (isReducedTestScopeEnabled || serviceDirs.length > 1) {
-        // if we include a core package, then we should only target the packages that are passed in
-        if (serviceDirs.indexOf("core") !== -1) {
-          rushCommandFlag = "--to";
-        }
-        // otherwise we pull other packages in by dependency
-        else {
-          rushCommandFlag = "--from";
-        }
-      }
-      else if (actionComponents.length == 1) {
-        rushCommandFlag = "--from";
-      }
-
-      rushRunAll(rushCommandFlag, packageNames);
+      rushRunAll(packagesWithDirection);
       break;
 
     case "test":
     case "unit-test":
     case "integration-test":
       var rushCommandFlag = "--impacted-by";
+
       if (isReducedTestScopeEnabled || serviceDirs.length > 1) {
         // If a service is configured to have reduced test matrix then run rush test only for those projects
         rushCommandFlag = "--only";

From eb37ef5a972237f4c0f979006f8cd90948a262e1 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Tue, 22 Oct 2024 14:30:48 -0700
Subject: [PATCH 49/90] ensure that we flatten to an actual array

---
 eng/tools/rush-runner.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eng/tools/rush-runner.js b/eng/tools/rush-runner.js
index 221add761141..172d0d0a78f3 100644
--- a/eng/tools/rush-runner.js
+++ b/eng/tools/rush-runner.js
@@ -204,7 +204,7 @@ function rushRunAll(direction, packages) {
  * @param packagesWithDirection string[][] Any array of tuples containing [[direction, packageName]...]
  */
 function rushRunAll(packagesWithDirection) {
-  spawnNode(baseDir, "common/scripts/install-run-rush.js", action, ...packagesWithDirection, ...rushParams);
+  spawnNode(baseDir, "common/scripts/install-run-rush.js", action, ...(packagesWithDirection.flat()), ...rushParams);
 }
 
 /**

From 8ecf06141c482d3f0f05f214adb6e27039c6a44b Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Tue, 22 Oct 2024 14:52:59 -0700
Subject: [PATCH 50/90] get rid of usage of flat() in favor of simpler
 getPackagesWithDirection

---
 eng/tools/rush-runner.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/eng/tools/rush-runner.js b/eng/tools/rush-runner.js
index 172d0d0a78f3..6fb3164f1f8a 100644
--- a/eng/tools/rush-runner.js
+++ b/eng/tools/rush-runner.js
@@ -133,7 +133,7 @@ const getDirectionMappedPackages = (serviceDirs, packageNames) => {
       rushCommandFlag = "--from";
     }
 
-    mappedPackages.push([rushCommandFlag, packageName]);
+    mappedPackages.push(`${rushCommandFlag} ${packageName}`);
   }
 
   return mappedPackages;
@@ -201,10 +201,10 @@ function rushRunAll(direction, packages) {
 /**
  * Helper function to invoke a bunch of combined rush commands.
  *
- * @param packagesWithDirection string[][] Any array of tuples containing [[direction, packageName]...]
+ * @param packagesWithDirection string[] Any array of strings containing ["direction packageName"...]
  */
 function rushRunAll(packagesWithDirection) {
-  spawnNode(baseDir, "common/scripts/install-run-rush.js", action, ...(packagesWithDirection.flat()), ...rushParams);
+  spawnNode(baseDir, "common/scripts/install-run-rush.js", action, ...packagesWithDirection, ...rushParams);
 }
 
 /**

From 2dc41120e32eff94f2cf7187eaf2e4894a759c86 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Tue, 22 Oct 2024 15:10:48 -0700
Subject: [PATCH 51/90] we now split up --to vs --from

---
 eng/tools/rush-runner.js | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/eng/tools/rush-runner.js b/eng/tools/rush-runner.js
index 6fb3164f1f8a..51945c92c5b5 100644
--- a/eng/tools/rush-runner.js
+++ b/eng/tools/rush-runner.js
@@ -133,7 +133,7 @@ const getDirectionMappedPackages = (serviceDirs, packageNames) => {
       rushCommandFlag = "--from";
     }
 
-    mappedPackages.push(`${rushCommandFlag} ${packageName}`);
+    mappedPackages.push([rushCommandFlag, packageName]);
   }
 
   return mappedPackages;
@@ -204,7 +204,21 @@ function rushRunAll(direction, packages) {
  * @param packagesWithDirection string[] Any array of strings containing ["direction packageName"...]
  */
 function rushRunAll(packagesWithDirection) {
-  spawnNode(baseDir, "common/scripts/install-run-rush.js", action, ...packagesWithDirection, ...rushParams);
+  // we HAVE to split --from and --to into separate commands otherwise rush will crash on startup
+  toPackages = [];
+  fromPackages = [];
+
+  for (const [direction, packageName] of packagesWithDirection) {
+    if (direction === "--to") {
+      toPackages.push(`--to ${packageName}`);
+    } else {
+      fromPackages.push(`--from ${packageName}`);
+    }
+  }
+
+
+  spawnNode(baseDir, "common/scripts/install-run-rush.js", action, ...toPackages, ...rushParams);
+  spawnNode(baseDir, "common/scripts/install-run-rush.js", action, ...fromPackages, ...rushParams);
 }
 
 /**

From 0805b6051165cd488480d780c099d3933e2548ec Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Tue, 22 Oct 2024 15:32:07 -0700
Subject: [PATCH 52/90] directional updates

---
 eng/tools/rush-runner.js | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/eng/tools/rush-runner.js b/eng/tools/rush-runner.js
index 51945c92c5b5..f8dcefad708a 100644
--- a/eng/tools/rush-runner.js
+++ b/eng/tools/rush-runner.js
@@ -195,6 +195,8 @@ const packagesWithDirection = getDirectionMappedPackages(serviceDirs, packageNam
  */
 function rushRunAll(direction, packages) {
   const params = flatMap(packages, (p) => [direction, p]);
+
+  console.log(params);
   spawnNode(baseDir, "common/scripts/install-run-rush.js", action, ...params, ...rushParams);
 }
 
@@ -203,20 +205,19 @@ function rushRunAll(direction, packages) {
  *
  * @param packagesWithDirection string[] Any array of strings containing ["direction packageName"...]
  */
-function rushRunAll(packagesWithDirection) {
+function rushRunAllWithDirection(packagesWithDirection) {
   // we HAVE to split --from and --to into separate commands otherwise rush will crash on startup
   toPackages = [];
   fromPackages = [];
 
   for (const [direction, packageName] of packagesWithDirection) {
     if (direction === "--to") {
-      toPackages.push(`--to ${packageName}`);
+      toPackages.push("--to", packageName);
     } else {
-      fromPackages.push(`--from ${packageName}`);
+      fromPackages.push("--from", packageName);
     }
   }
 
-
   spawnNode(baseDir, "common/scripts/install-run-rush.js", action, ...toPackages, ...rushParams);
   spawnNode(baseDir, "common/scripts/install-run-rush.js", action, ...fromPackages, ...rushParams);
 }
@@ -255,7 +256,7 @@ if (serviceDirs.length === 0) {
   // --from clause to resolve the package and all of its dependents
   switch (actionComponents[0]) {
     case "build":
-      rushRunAll(packagesWithDirection);
+      rushRunAllWithDirection(packagesWithDirection);
       break;
 
     case "test":

From 64f581e6cbb643c5cfb8d9cc7bb33ff51a13a41b Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Tue, 22 Oct 2024 15:46:04 -0700
Subject: [PATCH 53/90] clean up some comments and console logs from
 rush-runner.js

---
 eng/tools/rush-runner.js | 14 --------------
 1 file changed, 14 deletions(-)

diff --git a/eng/tools/rush-runner.js b/eng/tools/rush-runner.js
index f8dcefad708a..5fc41c05859c 100644
--- a/eng/tools/rush-runner.js
+++ b/eng/tools/rush-runner.js
@@ -48,7 +48,6 @@ const parseArgs = () => {
   const baseDir = path.resolve(`${path.dirname(scriptPath)}/../..`);
 
   for (const arg of givenArgs) {
-    console.log(`Examining ${arg}`);
     if (arg === "-packages") {
       isPackageFilter = true;
       continue;
@@ -68,7 +67,6 @@ const parseArgs = () => {
       if (arg && arg !== "*") {
         // exclude empty value and special value "*" meaning all libraries
         arg.split(" ").forEach(serviceDirectory => services.push(serviceDirectory));
-        console.log(`Services is ${services}`)
       }
     }
   }
@@ -146,7 +144,6 @@ const getServicePackages = (baseDir, serviceDirs, artifactNames) => {
   console.log(`Packages to build: ${artifactNames}`);
   const artifacts = artifactNames.split(",");
   for (const serviceDir of serviceDirs) {
-    console.log(`Looking at ${serviceDir}`);
     const searchDir = path.resolve(path.join(baseDir, "sdk", serviceDir));
     const packageJsons = getPackageJsons(searchDir);
     for (const filePath of packageJsons) {
@@ -195,8 +192,6 @@ const packagesWithDirection = getDirectionMappedPackages(serviceDirs, packageNam
  */
 function rushRunAll(direction, packages) {
   const params = flatMap(packages, (p) => [direction, p]);
-
-  console.log(params);
   spawnNode(baseDir, "common/scripts/install-run-rush.js", action, ...params, ...rushParams);
 }
 
@@ -245,15 +240,6 @@ const rushx_runner_path = path.join(baseDir, "common/scripts/install-run-rushx.j
 if (serviceDirs.length === 0) {
   spawnNode(baseDir, "common/scripts/install-run-rush.js", action, ...rushParams);
 } else {
-
-  // if the targeted package is one of:
-  // - a core package
-  // - azure-identity
-  // - arm-resources
-  // use the --to clause with the package
-
-  // if the targeted package is NOT one of those, then run it with the
-  // --from clause to resolve the package and all of its dependents
   switch (actionComponents[0]) {
     case "build":
       rushRunAllWithDirection(packagesWithDirection);

From 86f4dd64dcef157d586bba81df9f210a04821070 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Tue, 22 Oct 2024 15:52:19 -0700
Subject: [PATCH 54/90] clean up rushRunAllWithDirection

---
 eng/tools/rush-runner.js | 40 +++++++++++++++++++++++++++-------------
 1 file changed, 27 insertions(+), 13 deletions(-)

diff --git a/eng/tools/rush-runner.js b/eng/tools/rush-runner.js
index 5fc41c05859c..1ff29a2d55e6 100644
--- a/eng/tools/rush-runner.js
+++ b/eng/tools/rush-runner.js
@@ -111,7 +111,15 @@ const restrictedToPackages = [
   "@azure/identity",
   "@azure/arm-resources"
 ];
-const getDirectionMappedPackages = (serviceDirs, packageNames) => {
+
+/**
+ * Helper function that determines the rush command flag to use based on each individual package name for the 'build' check.
+ *
+ * If the targeted package is one of the restricted packages with a ton of dependents, we only want to run that package
+ * and not all of its dependents.
+ * @param packageNames string[] An array of strings containing the packages names to run the action on.
+ */
+const getDirectionMappedPackages = (packageNames) => {
   const mappedPackages = [];
 
   for (const packageName of packageNames) {
@@ -182,7 +190,7 @@ const actionComponents = action.toLowerCase().split(":");
 
 const [packageNames, packageDirs] = getServicePackages(baseDir, serviceDirs, artifactNames);
 
-const packagesWithDirection = getDirectionMappedPackages(serviceDirs, packageNames);
+const packagesWithDirection = getDirectionMappedPackages(packageNames);
 
 /**
  * Helper function to provide the rush logic that is used frequently below
@@ -196,25 +204,31 @@ function rushRunAll(direction, packages) {
 }
 
 /**
- * Helper function to invoke a bunch of combined rush commands.
+ * Helper function to invoke the rush logic split up by direction.
  *
  * @param packagesWithDirection string[] Any array of strings containing ["direction packageName"...]
  */
 function rushRunAllWithDirection(packagesWithDirection) {
-  // we HAVE to split --from and --to into separate commands otherwise rush will crash on startup
-  toPackages = [];
-  fromPackages = [];
+  // we HAVE to split --from and --to into separate commands otherwise rush will crash on startup...maybe?
+  // lets confirm this one last time with correct spacing
+  // toPackages = [];
+  // fromPackages = [];
+
+  // for (const [direction, packageName] of packagesWithDirection) {
+  //   if (direction === "--to") {
+  //     toPackages.push("--to", packageName);
+  //   } else {
+  //     fromPackages.push("--from", packageName);
+  //   }
+  // }
+
+  invocation = [];
 
   for (const [direction, packageName] of packagesWithDirection) {
-    if (direction === "--to") {
-      toPackages.push("--to", packageName);
-    } else {
-      fromPackages.push("--from", packageName);
-    }
+    invocation.push(direction, packageName);
   }
 
-  spawnNode(baseDir, "common/scripts/install-run-rush.js", action, ...toPackages, ...rushParams);
-  spawnNode(baseDir, "common/scripts/install-run-rush.js", action, ...fromPackages, ...rushParams);
+  spawnNode(baseDir, "common/scripts/install-run-rush.js", action, ...invocation, ...rushParams);
 }
 
 /**

From b70cd1f1a3dd88a4e2973145f386f918c7b7500d Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Tue, 22 Oct 2024 16:33:36 -0700
Subject: [PATCH 55/90] cleaning up

---
 eng/pipelines/templates/steps/build.yml       |  5 +----
 .../templates/steps/generate-doc.yml          |  1 -
 eng/tools/rush-runner.js                      | 20 +------------------
 3 files changed, 2 insertions(+), 24 deletions(-)

diff --git a/eng/pipelines/templates/steps/build.yml b/eng/pipelines/templates/steps/build.yml
index 5bc230d53fbb..fa6ee227f73a 100644
--- a/eng/pipelines/templates/steps/build.yml
+++ b/eng/pipelines/templates/steps/build.yml
@@ -20,11 +20,9 @@ steps:
   # we are not passing service directory, so we only ever set dev build to true
   - template: /eng/common/pipelines/templates/steps/daily-dev-build-variable.yml
 
-  # todo: update set-dev.js to handle multiple service directories based on the ChangedServices value of the diff
-  # currently defaulting to nothing, so we set the entire repo to dev build (we do this for python)
   - script: |
       npm install ./eng/tools/versioning
-      node eng/tools/versioning/set-dev.js --build-id "$(Build.BuildNumber)" --repo-root "$(Build.SourcesDirectory)" --service "$(folder)"
+      node eng/tools/versioning/set-dev.js --build-id "$(Build.BuildNumber)" --repo-root "$(Build.SourcesDirectory)"
       node common/scripts/install-run-rush.js update
     condition: and(succeeded(),eq(variables['SetDevVersion'],'true'))
     displayName: "Update package versions for dev build"
@@ -81,7 +79,6 @@ steps:
       node eng/tools/rush-runner.js unlink
     displayName: "Unlink dependencies"
 
-  # todo, modify this to walk the service directories
   - template: ../steps/generate-doc.yml
     parameters:
       ServiceDirectory: ${{parameters.ServiceDirectory}}
diff --git a/eng/pipelines/templates/steps/generate-doc.yml b/eng/pipelines/templates/steps/generate-doc.yml
index eaf1e653b317..bbdd75f1f47a 100644
--- a/eng/pipelines/templates/steps/generate-doc.yml
+++ b/eng/pipelines/templates/steps/generate-doc.yml
@@ -17,7 +17,6 @@ steps:
     displayName: "Run Typedoc Docs"
     condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
 
-  # todo, possibly make this a wrapper node script instead of wrapper powershell.
   - pwsh: |
       if ("$(ChangedServices)") {
         foreach($service in "$(ChangedServices)".Split(" ")) {
diff --git a/eng/tools/rush-runner.js b/eng/tools/rush-runner.js
index 1ff29a2d55e6..28d2494cd31a 100644
--- a/eng/tools/rush-runner.js
+++ b/eng/tools/rush-runner.js
@@ -209,25 +209,7 @@ function rushRunAll(direction, packages) {
  * @param packagesWithDirection string[] Any array of strings containing ["direction packageName"...]
  */
 function rushRunAllWithDirection(packagesWithDirection) {
-  // we HAVE to split --from and --to into separate commands otherwise rush will crash on startup...maybe?
-  // lets confirm this one last time with correct spacing
-  // toPackages = [];
-  // fromPackages = [];
-
-  // for (const [direction, packageName] of packagesWithDirection) {
-  //   if (direction === "--to") {
-  //     toPackages.push("--to", packageName);
-  //   } else {
-  //     fromPackages.push("--from", packageName);
-  //   }
-  // }
-
-  invocation = [];
-
-  for (const [direction, packageName] of packagesWithDirection) {
-    invocation.push(direction, packageName);
-  }
-
+  const invocation = packagesWithDirection.flatMap(([direction, packageName]) => [direction, packageName]);
   spawnNode(baseDir, "common/scripts/install-run-rush.js", action, ...invocation, ...rushParams);
 }
 

From c69371bcfb21a2f2765550d90dac50a05447aa59 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Tue, 22 Oct 2024 17:16:25 -0700
Subject: [PATCH 56/90] we were snapping the targeted packages too soon!

---
 eng/tools/rush-runner.js | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/eng/tools/rush-runner.js b/eng/tools/rush-runner.js
index 28d2494cd31a..480546bbf2aa 100644
--- a/eng/tools/rush-runner.js
+++ b/eng/tools/rush-runner.js
@@ -190,8 +190,6 @@ const actionComponents = action.toLowerCase().split(":");
 
 const [packageNames, packageDirs] = getServicePackages(baseDir, serviceDirs, artifactNames);
 
-const packagesWithDirection = getDirectionMappedPackages(packageNames);
-
 /**
  * Helper function to provide the rush logic that is used frequently below
  *
@@ -232,6 +230,7 @@ if (isReducedTestScopeEnabled) {
   console.log(`Found reduced test matrix configured for ${serviceDirs}.`);
   packageNames.push(...reducedDependencyTestMatrix[serviceDirs]);
 }
+const packagesWithDirection = getDirectionMappedPackages(packageNames);
 const rushx_runner_path = path.join(baseDir, "common/scripts/install-run-rushx.js");
 if (serviceDirs.length === 0) {
   spawnNode(baseDir, "common/scripts/install-run-rush.js", action, ...rushParams);

From e2ceb92e07879d4f7350a50b001d62539e56d180 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 23 Oct 2024 13:37:06 -0700
Subject: [PATCH 57/90] initial version of distributing packages to the matrix.
 still buggy for include

---
 eng/pipelines/templates/jobs/ci.yml           |  19 ++
 eng/scripts/distribute-packages-to-matrix.ps1 | 244 ++++++++++++++++++
 2 files changed, 263 insertions(+)
 create mode 100644 eng/scripts/distribute-packages-to-matrix.ps1

diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml
index 2d91faa403f0..aa4e84e5a26f 100644
--- a/eng/pipelines/templates/jobs/ci.yml
+++ b/eng/pipelines/templates/jobs/ci.yml
@@ -74,6 +74,25 @@ jobs:
           MatrixConfigs: ${{ parameters.MatrixConfigs }}
           MatrixFilters: ${{ parameters.MatrixFilters }}
           MatrixReplace: ${{ parameters.MatrixReplace }}
+          PreGenerationSteps:
+            - template: /eng/common/pipelines/templates/steps/save-package-properties.yml
+              parameters:
+                ServiceDirectory: ${{parameters.ServiceDirectory}}
+
+            - template: /eng/pipelines/templates/steps/set-artifact-packages.yml
+              parameters:
+                PackageInfo: $(Build.ArtifactStagingDirectory)/PackageInfo
+                Artifacts: ${{ parameters.Artifacts }}
+
+            - task: Powershell@2
+              inputs:
+                pwsh: true
+                filePath: eng/scripts/distribute-packages-to-matrix.ps1
+                arguments: >-
+                  -PackageInfoFolder "$(Build.ArtifactStagingDirectory)/PackageInfo"
+                  -PlatformMatrix "${{ parameters.MatrixConfigs[0].Path }}"
+              displayName: 'Distribute Packages to Matrix'
+              condition: and(eq(variables['Build.Reason'], 'PullRequest'), eq('${{ parameters.ServiceDirectory }}','auto'))
           CloudConfig:
             Cloud: Public
           AdditionalParameters:
diff --git a/eng/scripts/distribute-packages-to-matrix.ps1 b/eng/scripts/distribute-packages-to-matrix.ps1
new file mode 100644
index 000000000000..4d5daeebdb8f
--- /dev/null
+++ b/eng/scripts/distribute-packages-to-matrix.ps1
@@ -0,0 +1,244 @@
+<#
+.SYNOPSIS
+Distributes a set of packages to a platform matrix file by adding computing and adding a ArtifactPackageNames property
+dynamically. If an unexpected situation is encountered, the script will make no changes to the input file.
+
+.DESCRIPTION
+Because of the way the platform matrix file is structured, we need to distribute the packages in a way that
+honors the Include packages. We have these included this way because want to ensure that these python versions
+run on that platform. This script is aware of these additional configurations and will set ArtifactPackageNames for
+them as well.
+#>
+param(
+    [parameter(Mandatory=$true)]
+    [string]$PackageInfoFolder,
+    [parameter(Mandatory=$true)]
+    [string]$PlatformMatrix
+)
+
+$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot ".." ".."))
+Set-StrictMode -Version 4
+$BATCHSIZE = 10
+
+if (!(Test-Path $PackageInfoFolder)) {
+    Write-Error "PackageInfo folder file not found: $PackageInfoFolder"
+    exit 1
+}
+
+if (!(Test-Path $PlatformMatrix)) {
+    Write-Error "Platform matrix file not found: $PlatformMatrix"
+    exit 1
+}
+
+function Extract-MatrixMultiplier {
+    param (
+        [Parameter(Mandatory=$true)]
+        [PSCustomObject]$Matrix
+    )
+
+    $highestCount = 1
+
+    foreach ($property in $Matrix.PSObject.Properties) {
+        $type = $property.Value.GetType().Name
+        switch ($type) {
+            "PSCustomObject" {
+                $itemCount = 0
+
+                # this looks very strange to loop over the properties of a PSCustomObject,
+                # but this is the only way to actually count. The Count/Length property is
+                # NOT available
+                foreach($innerProperty in $property.Value.PSObject.Properties) {
+                    $itemCount++
+                }
+
+                if ($itemCount -gt $highestCount) {
+                    $highestCount = $itemCount
+                }
+            }
+            "Object[]" {
+                $count = $property.Value.Length
+                if ($count -gt $highestCount) {
+                    $highestCount = $count
+                }
+            }
+        }
+    }
+
+    return $highestCount
+}
+
+function Split-ArrayIntoBatches {
+    param (
+        [Parameter(Mandatory=$true)]
+        [object[]]$InputArray,
+
+        [Parameter(Mandatory=$true)]
+        [int]$BatchSize
+    )
+
+    $batches = @()
+
+    for ($i = 0; $i -lt $InputArray.Count; $i += $BatchSize) {
+        $batch = $InputArray[$i..[math]::Min($i + $BatchSize - 1, $InputArray.Count - 1)]
+
+        $batches += ,$batch
+    }
+
+    return ,$batches
+}
+
+function Update-Include {
+    param (
+        [Parameter(Mandatory=$true)]
+        $Matrix,
+        [Parameter(Mandatory=$true)]
+        $IncludeConfig,
+        [Parameter(Mandatory=$true)]
+        $TargetingString
+    )
+
+    foreach ($configElement in $IncludeConfig) {
+        Write-Host "Examining $configElement of include object"
+
+        if (-not $configElement.PSObject.Properties["ArtifactPackageNames"]) {
+            Write-Host "Attempting to add ArtifactPackageNames to $configElement"
+            $configElement | Add-Member -Force -MemberType NoteProperty -Name ArtifactPackageNames -Value ""
+        }
+
+        Write-Host "I somehow got to here, what is in this object? $configElement"
+        $configElement.ArtifactPackageNames = $TargetingString
+    }
+
+    $Matrix.include += $configElement
+}
+
+function Update-Matrix {
+    param (
+        [Parameter(Mandatory=$true)]
+        $Matrix,
+
+        [Parameter(Mandatory=$true)]
+        $DirectBatches,
+
+        [Parameter(Mandatory=$true)]
+        $IndirectBatches,
+
+        [Parameter(Mandatory=$true)]
+        $MatrixMultiplier,
+
+        [Parameter(Mandatory=$true)]
+        $IncludeCount,
+
+        [Parameter(Mandatory=$false)]
+        $OriginalIncludeObject
+    )
+
+    $matrixUpdate = $true
+    if ($Matrix.matrix.PSObject.Properties["`$IMPORT"]) {
+        $matrixUpdate = $false
+    }
+
+    # we need to ensure the presence of TargetingString in the matrix object
+    if ($matrixUpdate) {
+        if ($directBatches -or $indirectBatches) {
+            if (-not $Matrix.matrix.PSObject.Properties["ArtifactPackageNames"]) {
+                $Matrix.matrix | Add-Member -Force -MemberType NoteProperty -Name ArtifactPackageNames -Value @()
+            }
+            else {
+                $Matrix.matrix.ArtifactPackageNames = @()
+            }
+        }
+    }
+
+    # a basic platform matrix has the following:
+    # "matrix" -> "PythonVersion" = ["3.6", "3.7", "3.8", "3.9", "3.10"]
+    # "include" -> "ConfigName" -> "ActualJobName" -> PythonVersion=3.9
+    # so what we need to do is for each batch
+        # 1. assign the batch * matrix size to the matrix as "ArtifactPackagesNames", this will evenly distribute the packages to EACH node version non-sparsely
+            # So direct packages add their duplicated targetingstring [ "azure-core,azure-storage-blob", "azure-core,azure-storage-blob", "azure-core,azure-storage-blob"]
+            # a number of times indicated by the highest matrix multiplier. This forces a non-sparse coverage of this targetingstring batch
+            # even in a sparse matrix generation.
+        # 2. assign the batch to each include with the batch being the value for TargetingString for the include
+            # the easiest way to do this is to take a copy of the `Include` object at the beginning of this script,
+            # for each batch, assign the batch to the include object, and then add the include object to the matrix object
+            # this will have the result of multiplexing our includes, so we will need to update the name there as well
+    # any other packages will be added to the matrix as additional targetingstring batches. it'll be sparse in that there won't be full matrix coverage
+
+    foreach($batch in $DirectBatches) {
+        $targetingString = ($batch | Select-Object -ExpandProperty ArtifactName) -join ","
+
+        # we need to equal the largest multiplier in the matrix. That is USUALLY python version,
+        # but in some cases it could be something else (like sdk/cosmos/cosmos-emulator-matrix.json) the multiplier
+        # is a different property. We do this because we want to ensure that the matrix is fully covered, even if
+        # we are generating the matrix sparsely, we still want full coverage of these targetingStrings
+        $targetingStringArray = @($targetingString) * $MatrixMultiplier
+
+        if ($matrixUpdate) {
+            $matrix.matrix.ArtifactPackageNames += $targetingStringArray
+        }
+
+        # if there were any include objects, we need to duplicate them exactly and add the targeting string to each
+        # this means that the number of includes at the end of this operation will be incoming # of includes * the number of batches
+        if ($includeCount -gt 0) {
+            $includeCopy = $OriginalIncludeObject | ConvertTo-Json -Depth 100 | ConvertFrom-Json
+
+            Update-Include -Matrix $matrix -IncludeConfig $includeCopy -TargetingString $targetingString
+        }
+    }
+
+    foreach($batch in $IndirectBatches) {
+        $targetingString = ($batch | Select-Object -ExpandProperty Name) -join ","
+        if ($matrixUpdate) {
+            $matrix.matrix.ArtifactPackageNames += @($targetingString)
+        }
+    }
+
+}
+
+# calculate general targeting information and create our batches prior to updating any matrix
+$packageProperties = Get-ChildItem -Recurse "$PackageInfoFolder" *.json `
+    | % { Get-Content -Path $_.FullName | ConvertFrom-Json }
+
+# I will assign all the direct included packages first. our goal is to get full coverage of the direct included packages
+# then, for the indirect packages, we will add them as sparse ArtifactPackageNames bundles to the matrix
+$batches = Split-ArrayIntoBatches -InputArray $packageProperties -BatchSize $BATCHSIZE
+
+$matrix = Get-Content -Path $PlatformMatrix | ConvertFrom-Json
+
+$matrixLocation = $PlatformMatrix
+
+# handle the case where we discover an import, we need to climb the dependency in that case
+if ($matrix.matrix.PSObject.Properties["`$IMPORT"]) {
+    $originalMatrix = $matrix
+    $originalMatrixLocation = $PlatformMatrix
+
+    # rest of the update will happen to that matrix object
+    $matrixLocation = (Join-Path $RepoRoot $matrix.matrix."`$IMPORT")
+    $matrix = Get-Content -Path $matrixLocation | ConvertFrom-Json
+
+    # we just need to walk the include objects and update the targeting string for the batches we have before
+    # updating in place, then continuing on to update the actual matrix object that we imported
+    # there is a clear and present edge case here with nested import. We will not handle that case
+
+    $originalInclude = $originalMatrix.include
+    $originalMatrix.include = @()
+
+    # update the include objects for the original matrix that was importing
+    Update-Matrix -Matrix $originalMatrix -DirectBatches $directBatches -IndirectBatches @() -MatrixMultiplier 1 -IncludeCount $matrix.include.Count -OriginalIncludeObject $originalInclude
+    $originalMatrix | ConvertTo-Json -Depth 100 | Set-Content -Path $originalMatrixLocation
+}
+
+$matrixMultiplier = Extract-MatrixMultiplier -Matrix $matrix.matrix
+
+$includeCount = 0
+$includeObject = $null
+$originalInclude = $null
+if ($matrix.PSObject.Properties.Name -contains "include") {
+    $includeCount = $matrix.include.Count
+    $originalInclude = $matrix.include
+    $matrix.include = @()
+}
+
+Update-Matrix -Matrix $matrix -DirectBatches $batches -IndirectBatches @() -MatrixMultiplier $matrixMultiplier -IncludeCount $includeCount -OriginalIncludeObject $originalInclude
+
+$matrix | ConvertTo-Json -Depth 100 | Set-Content -Path $matrixLocation

From abbff35a352e93fdadac8a25073e66a3bf9f2ba5 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 23 Oct 2024 13:41:44 -0700
Subject: [PATCH 58/90] add package distribution to the matrix

---
 .../templates/stages/platform-matrix.json     | 24 +++++++++++++++----
 eng/scripts/distribute-packages-to-matrix.ps1 |  8 +------
 2 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/eng/pipelines/templates/stages/platform-matrix.json b/eng/pipelines/templates/stages/platform-matrix.json
index b8fda65b7211..8f3ae6502718 100644
--- a/eng/pipelines/templates/stages/platform-matrix.json
+++ b/eng/pipelines/templates/stages/platform-matrix.json
@@ -17,9 +17,18 @@
         "Pool": "env:MACPOOL"
       }
     },
-    "NodeTestVersion": ["18.x", "20.x", "22.x"],
+    "NodeTestVersion": [
+      "18.x",
+      "20.x",
+      "22.x"
+    ],
     "TestType": "node",
-    "TestResultsFiles": "**/test-results.xml"
+    "TestResultsFiles": "**/test-results.xml",
+    "ArtifactPackageNames": [
+      "azure-arm-resources,azure-identity,azure-rest-synapse-access-control,azure-service-bus,azure-template-dpg,azure-template",
+      "azure-arm-resources,azure-identity,azure-rest-synapse-access-control,azure-service-bus,azure-template-dpg,azure-template",
+      "azure-arm-resources,azure-identity,azure-rest-synapse-access-control,azure-service-bus,azure-template-dpg,azure-template"
+    ]
   },
   "include": [
     {
@@ -44,7 +53,8 @@
           "TestResultsFiles": "**/test-results.browser.xml"
         }
       },
-      "NodeTestVersion": "18.x"
+      "NodeTestVersion": "18.x",
+      "ArtifactPackageNames": "azure-arm-resources,azure-identity,azure-rest-synapse-access-control,azure-service-bus,azure-template-dpg,azure-template"
     },
     {
       "Agent": {
@@ -55,8 +65,12 @@
       },
       "TestType": "node",
       "NodeTestVersion": "18.x",
-      "DependencyVersion": ["max", "min"],
-      "TestResultsFiles": "**/test-results.xml"
+      "DependencyVersion": [
+        "max",
+        "min"
+      ],
+      "TestResultsFiles": "**/test-results.xml",
+      "ArtifactPackageNames": "azure-arm-resources,azure-identity,azure-rest-synapse-access-control,azure-service-bus,azure-template-dpg,azure-template"
     }
   ]
 }
diff --git a/eng/scripts/distribute-packages-to-matrix.ps1 b/eng/scripts/distribute-packages-to-matrix.ps1
index 4d5daeebdb8f..bf1d90f4394b 100644
--- a/eng/scripts/distribute-packages-to-matrix.ps1
+++ b/eng/scripts/distribute-packages-to-matrix.ps1
@@ -98,18 +98,12 @@ function Update-Include {
     )
 
     foreach ($configElement in $IncludeConfig) {
-        Write-Host "Examining $configElement of include object"
-
         if (-not $configElement.PSObject.Properties["ArtifactPackageNames"]) {
-            Write-Host "Attempting to add ArtifactPackageNames to $configElement"
             $configElement | Add-Member -Force -MemberType NoteProperty -Name ArtifactPackageNames -Value ""
         }
-
-        Write-Host "I somehow got to here, what is in this object? $configElement"
         $configElement.ArtifactPackageNames = $TargetingString
+        $Matrix.include += $configElement
     }
-
-    $Matrix.include += $configElement
 }
 
 function Update-Matrix {

From 7dc60ba8fad299b3a4b5ba0d8c7df5c78d1425a1 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 23 Oct 2024 13:45:47 -0700
Subject: [PATCH 59/90] splatting should be working. just need to ensure
 multiple batches are rendering properly

---
 eng/scripts/distribute-packages-to-matrix.ps1 | 15 +++++----------
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/eng/scripts/distribute-packages-to-matrix.ps1 b/eng/scripts/distribute-packages-to-matrix.ps1
index bf1d90f4394b..b43ed6134204 100644
--- a/eng/scripts/distribute-packages-to-matrix.ps1
+++ b/eng/scripts/distribute-packages-to-matrix.ps1
@@ -5,7 +5,7 @@ dynamically. If an unexpected situation is encountered, the script will make no
 
 .DESCRIPTION
 Because of the way the platform matrix file is structured, we need to distribute the packages in a way that
-honors the Include packages. We have these included this way because want to ensure that these python versions
+honors the Include packages. We have these included this way because want to ensure that these node versions
 run on that platform. This script is aware of these additional configurations and will set ArtifactPackageNames for
 them as well.
 #>
@@ -145,23 +145,18 @@ function Update-Matrix {
     }
 
     # a basic platform matrix has the following:
-    # "matrix" -> "PythonVersion" = ["3.6", "3.7", "3.8", "3.9", "3.10"]
-    # "include" -> "ConfigName" -> "ActualJobName" -> PythonVersion=3.9
+    # "matrix" -> "NodeVersion" = []
+    # "include" -> "ConfigName" -> NodeVersion=18.X
     # so what we need to do is for each batch
         # 1. assign the batch * matrix size to the matrix as "ArtifactPackagesNames", this will evenly distribute the packages to EACH node version non-sparsely
-            # So direct packages add their duplicated targetingstring [ "azure-core,azure-storage-blob", "azure-core,azure-storage-blob", "azure-core,azure-storage-blob"]
-            # a number of times indicated by the highest matrix multiplier. This forces a non-sparse coverage of this targetingstring batch
-            # even in a sparse matrix generation.
-        # 2. assign the batch to each include with the batch being the value for TargetingString for the include
+        # 2. assign the batch to each include with the batch being the value for ArtifactPackageNames for the include
             # the easiest way to do this is to take a copy of the `Include` object at the beginning of this script,
             # for each batch, assign the batch to the include object, and then add the include object to the matrix object
             # this will have the result of multiplexing our includes, so we will need to update the name there as well
-    # any other packages will be added to the matrix as additional targetingstring batches. it'll be sparse in that there won't be full matrix coverage
-
     foreach($batch in $DirectBatches) {
         $targetingString = ($batch | Select-Object -ExpandProperty ArtifactName) -join ","
 
-        # we need to equal the largest multiplier in the matrix. That is USUALLY python version,
+        # we need to equal the largest multiplier in the matrix. That is USUALLY nodeVersion version,
         # but in some cases it could be something else (like sdk/cosmos/cosmos-emulator-matrix.json) the multiplier
         # is a different property. We do this because we want to ensure that the matrix is fully covered, even if
         # we are generating the matrix sparsely, we still want full coverage of these targetingStrings

From 6adb684b2c5b6bf65c533fabe8ba8cdf967f0d13 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 23 Oct 2024 14:35:47 -0700
Subject: [PATCH 60/90] ensure that we are able to scan for the packages

---
 eng/pipelines/templates/jobs/ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml
index aa4e84e5a26f..ca4320529070 100644
--- a/eng/pipelines/templates/jobs/ci.yml
+++ b/eng/pipelines/templates/jobs/ci.yml
@@ -74,6 +74,7 @@ jobs:
           MatrixConfigs: ${{ parameters.MatrixConfigs }}
           MatrixFilters: ${{ parameters.MatrixFilters }}
           MatrixReplace: ${{ parameters.MatrixReplace }}
+          SparseCheckoutPaths: [ "sdk/" ]
           PreGenerationSteps:
             - template: /eng/common/pipelines/templates/steps/save-package-properties.yml
               parameters:

From 795a98a834eca3b2d3974f83a5b4a1aaae121b88 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 23 Oct 2024 14:36:33 -0700
Subject: [PATCH 61/90] restore accidentally changed matrix

---
 eng/pipelines/templates/stages/platform-matrix.json | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

diff --git a/eng/pipelines/templates/stages/platform-matrix.json b/eng/pipelines/templates/stages/platform-matrix.json
index 8f3ae6502718..6680f8cd40fc 100644
--- a/eng/pipelines/templates/stages/platform-matrix.json
+++ b/eng/pipelines/templates/stages/platform-matrix.json
@@ -23,12 +23,7 @@
       "22.x"
     ],
     "TestType": "node",
-    "TestResultsFiles": "**/test-results.xml",
-    "ArtifactPackageNames": [
-      "azure-arm-resources,azure-identity,azure-rest-synapse-access-control,azure-service-bus,azure-template-dpg,azure-template",
-      "azure-arm-resources,azure-identity,azure-rest-synapse-access-control,azure-service-bus,azure-template-dpg,azure-template",
-      "azure-arm-resources,azure-identity,azure-rest-synapse-access-control,azure-service-bus,azure-template-dpg,azure-template"
-    ]
+    "TestResultsFiles": "**/test-results.xml"
   },
   "include": [
     {
@@ -53,8 +48,7 @@
           "TestResultsFiles": "**/test-results.browser.xml"
         }
       },
-      "NodeTestVersion": "18.x",
-      "ArtifactPackageNames": "azure-arm-resources,azure-identity,azure-rest-synapse-access-control,azure-service-bus,azure-template-dpg,azure-template"
+      "NodeTestVersion": "18.x"
     },
     {
       "Agent": {
@@ -69,8 +63,7 @@
         "max",
         "min"
       ],
-      "TestResultsFiles": "**/test-results.xml",
-      "ArtifactPackageNames": "azure-arm-resources,azure-identity,azure-rest-synapse-access-control,azure-service-bus,azure-template-dpg,azure-template"
+      "TestResultsFiles": "**/test-results.xml"
     }
   ]
 }

From 3a975b6742e646182a8578e2f1be9aaf76cf5650 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 23 Oct 2024 14:41:25 -0700
Subject: [PATCH 62/90] update our service resolution logic to handle the
 ArtifactPackageNames being set through variable

---
 .../templates/steps/set-artifact-packages.yml | 24 +++++++++++++------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/eng/pipelines/templates/steps/set-artifact-packages.yml b/eng/pipelines/templates/steps/set-artifact-packages.yml
index c520208a9ddf..1a46e1d9a3b0 100644
--- a/eng/pipelines/templates/steps/set-artifact-packages.yml
+++ b/eng/pipelines/templates/steps/set-artifact-packages.yml
@@ -24,21 +24,31 @@ steps:
       $pkgNames = $packageProperties | Foreach-Object { $_.Name } `
         | ForEach-Object { $_.Replace(".json", "") }
 
+      $setting = $pkgNames -join ","
+      Write-Host "Setting ArtifactPackageNames to: `n$setting"
+      Write-Host "##vso[task.setvariable variable=ArtifactPackageNames;]$setting"
+    displayName: Resolve Targeted Packages
+    condition: eq(variables['ArtifactPackageNames'], '')
+
+  - pwsh: |
+      # set changed services given the set of changed packages, this will mean that
+      # ChangedServices will be appropriate for the batched set of packages if that is indeed how
+      # we set the targeted artifacts
+      $packageProperties = Get-ChildItem -Recurse "${{ parameters.PackageInfo }}" *.json `
+        | Foreach-Object { Get-Content -Raw -Path $_.FullName | ConvertFrom-Json }
+
+      $packageSet = "$(ArtifactPackagesNames)" -split ","
+
       $changedServicesArray = $packageProperties | Foreach-Object { Get-Content -Raw -Path $_.FullName | ConvertFrom-Json } `
+        | Where-Object { $packageSet -contains $_.ArtifactName }
         | ForEach-Object { $_.ServiceDirectory } | Get-Unique
 
-      $setting = $pkgNames -join ","
       $changedServices = $changedServicesArray -join " "
       $commaChangedServices = $changedServicesArray -join ","
       Write-Host "Setting ChangedServices to : `n$changedServices"
       Write-Host "##vso[task.setvariable variable=ChangedServices;]$changedServices"
       Write-Host "##vso[task.setvariable variable=CChangedServices;]$commaChangedServices"
-      Write-Host "Setting ArtifactPackageNames to: `n$setting"
-      Write-Host "##vso[task.setvariable variable=ArtifactPackageNames;]$setting"
-    displayName: Resolve Targeted Packages
-    condition: eq(variables['ArtifactPackageNames'], '')
 
-  - pwsh: |
-      Write-Host "This run is targeting: `n$(ArtifactPackageNames) in [$(ChangedServices)]"
+      Write-Host "This run is targeting: `n$(ArtifactPackageNames) in [$changedServices]"
     displayName: Resolve Targeted Packages
     condition: ne(variables['ArtifactPackageNames'], '')

From 37d47de8439ba6d678fee0842ce57dbd15e19256 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 23 Oct 2024 14:42:42 -0700
Subject: [PATCH 63/90] ensure the cspell config is pulled across

---
 eng/pipelines/templates/jobs/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml
index ca4320529070..a477ca5626b4 100644
--- a/eng/pipelines/templates/jobs/ci.yml
+++ b/eng/pipelines/templates/jobs/ci.yml
@@ -74,7 +74,7 @@ jobs:
           MatrixConfigs: ${{ parameters.MatrixConfigs }}
           MatrixFilters: ${{ parameters.MatrixFilters }}
           MatrixReplace: ${{ parameters.MatrixReplace }}
-          SparseCheckoutPaths: [ "sdk/" ]
+          SparseCheckoutPaths: [ "sdk/", ".vscode"]
           PreGenerationSteps:
             - template: /eng/common/pipelines/templates/steps/save-package-properties.yml
               parameters:

From f02bcf238329ad3f9f5b1dad43232c72c8c23452 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 23 Oct 2024 15:00:16 -0700
Subject: [PATCH 64/90] remove useless set-artifact-packages call. update
 variable reference

---
 eng/pipelines/templates/jobs/ci.yml                     | 5 -----
 eng/pipelines/templates/steps/set-artifact-packages.yml | 2 +-
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml
index a477ca5626b4..2ba1eb86557c 100644
--- a/eng/pipelines/templates/jobs/ci.yml
+++ b/eng/pipelines/templates/jobs/ci.yml
@@ -80,11 +80,6 @@ jobs:
               parameters:
                 ServiceDirectory: ${{parameters.ServiceDirectory}}
 
-            - template: /eng/pipelines/templates/steps/set-artifact-packages.yml
-              parameters:
-                PackageInfo: $(Build.ArtifactStagingDirectory)/PackageInfo
-                Artifacts: ${{ parameters.Artifacts }}
-
             - task: Powershell@2
               inputs:
                 pwsh: true
diff --git a/eng/pipelines/templates/steps/set-artifact-packages.yml b/eng/pipelines/templates/steps/set-artifact-packages.yml
index 1a46e1d9a3b0..ae0028a463a0 100644
--- a/eng/pipelines/templates/steps/set-artifact-packages.yml
+++ b/eng/pipelines/templates/steps/set-artifact-packages.yml
@@ -37,7 +37,7 @@ steps:
       $packageProperties = Get-ChildItem -Recurse "${{ parameters.PackageInfo }}" *.json `
         | Foreach-Object { Get-Content -Raw -Path $_.FullName | ConvertFrom-Json }
 
-      $packageSet = "$(ArtifactPackagesNames)" -split ","
+      $packageSet = "$(ArtifactPackageNames)" -split ","
 
       $changedServicesArray = $packageProperties | Foreach-Object { Get-Content -Raw -Path $_.FullName | ConvertFrom-Json } `
         | Where-Object { $packageSet -contains $_.ArtifactName }

From 8a4fea9d4a26d6d13178aa5fe25ebe5dac67c509 Mon Sep 17 00:00:00 2001
From: Scott Beddall <45376673+scbedd@users.noreply.github.com>
Date: Wed, 23 Oct 2024 15:13:39 -0700
Subject: [PATCH 65/90] Apply suggestions from code review

Co-authored-by: Ben Broderick Phillips <bebroder@microsoft.com>
---
 eng/pipelines/templates/steps/analyze.yml               | 2 +-
 eng/pipelines/templates/steps/build.yml                 | 6 +++---
 eng/pipelines/templates/steps/generate-doc.yml          | 4 ++--
 eng/pipelines/templates/steps/set-artifact-packages.yml | 7 +++----
 eng/scripts/Language-Settings.ps1                       | 7 +++----
 5 files changed, 12 insertions(+), 14 deletions(-)

diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml
index 66938ffcccfe..09f899441371 100644
--- a/eng/pipelines/templates/steps/analyze.yml
+++ b/eng/pipelines/templates/steps/analyze.yml
@@ -54,7 +54,7 @@ steps:
 
   - template: /eng/common/pipelines/templates/steps/verify-samples.yml
     parameters:
-      ServiceDirectories: $(CChangedServices)
+      ServiceDirectories: $(ChangedServicesCsv)
 
   - script: |
       npm ci
diff --git a/eng/pipelines/templates/steps/build.yml b/eng/pipelines/templates/steps/build.yml
index fa6ee227f73a..d9f547ef28c6 100644
--- a/eng/pipelines/templates/steps/build.yml
+++ b/eng/pipelines/templates/steps/build.yml
@@ -89,16 +89,16 @@ steps:
 
       foreach ($artifact in $artifacts)
       {
-        $artifactDetails = Get-Content $(Build.ArtifactStagingDirectory)/PackageInfo/$artifact.json | ConvertFrom-Json
+        $artifactDetails = Get-Content -Raw $(Build.ArtifactStagingDirectory)/PackageInfo/$artifact.json | ConvertFrom-Json
 
         Write-Host "Copying $artifact artifacts to $(Build.ArtifactStagingDirectory)/$artifact"
-        New-Item -Type Directory -Name $artifact -Path $(Build.ArtifactStagingDirectory) > $null
+        New-Item -Type Directory -Force -Name $artifact -Path $(Build.ArtifactStagingDirectory) > $null
         Copy-Item sdk/$($artifactDetails.Service)/**/$artifact-[0-9]*.[0-9]*.[0-9]*.tgz $(Build.ArtifactStagingDirectory)/$artifact
         Copy-Item sdk/$($artifactDetails.Service)/**/browser/$artifact-[0-9]*.[0-9]*.[0-9]*.zip $(Build.ArtifactStagingDirectory)/$artifact
 
         if ($${{ parameters.IncludeRelease }} -eq $true -and $artifactDetails.ArtifactDetails.skipPublishDocMs -ne $true)
         {
-          New-Item -Type Directory -Name documentation -Path $(Build.ArtifactStagingDirectory)/$artifact > $null
+          New-Item -Type Directory -Force -Name documentation -Path $(Build.ArtifactStagingDirectory)/$artifact > $null
           Copy-Item $(Build.SourcesDirectory)/docGen/$artifact.zip $(Build.ArtifactStagingDirectory)/$artifact/documentation
         }
       }
diff --git a/eng/pipelines/templates/steps/generate-doc.yml b/eng/pipelines/templates/steps/generate-doc.yml
index bbdd75f1f47a..72af43d7dbd3 100644
--- a/eng/pipelines/templates/steps/generate-doc.yml
+++ b/eng/pipelines/templates/steps/generate-doc.yml
@@ -18,8 +18,8 @@ steps:
     condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
 
   - pwsh: |
-      if ("$(ChangedServices)") {
-        foreach($service in "$(ChangedServices)".Split(" ")) {
+      if ('$(ChangedServices)' -and '$(ChangedServices)' -notlike '*ChangedServices*') {
+        foreach($service in '$(ChangedServices)'.Split(" ")) {
           node $(Build.SourcesDirectory)/eng/tools/generate-doc/dist/index.js --serviceDir "$service"
         }
       }
diff --git a/eng/pipelines/templates/steps/set-artifact-packages.yml b/eng/pipelines/templates/steps/set-artifact-packages.yml
index ae0028a463a0..2303fdb1fe7c 100644
--- a/eng/pipelines/templates/steps/set-artifact-packages.yml
+++ b/eng/pipelines/templates/steps/set-artifact-packages.yml
@@ -15,14 +15,13 @@ steps:
       $artifacts = '${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json
       $knownArtifacts = @()
       if ($artifacts) {
-        $knownArtifacts = $artifacts | ForEach-Object { $knownArtifacts += $_.name }
+        $knownArtifacts = $artifacts | ForEach-Object { $_.name }
       }
 
       $packageProperties = Get-ChildItem -Recurse "${{ parameters.PackageInfo }}" *.json `
         | Where-Object { if($knownArtifacts) { $knownArtifacts -contains $_.Name.Replace(".json", "") } else { $true } }
 
-      $pkgNames = $packageProperties | Foreach-Object { $_.Name } `
-        | ForEach-Object { $_.Replace(".json", "") }
+      $pkgNames = $packageProperties | ForEach-Object { $_.Name.Replace(".json", "") }
 
       $setting = $pkgNames -join ","
       Write-Host "Setting ArtifactPackageNames to: `n$setting"
@@ -47,7 +46,7 @@ steps:
       $commaChangedServices = $changedServicesArray -join ","
       Write-Host "Setting ChangedServices to : `n$changedServices"
       Write-Host "##vso[task.setvariable variable=ChangedServices;]$changedServices"
-      Write-Host "##vso[task.setvariable variable=CChangedServices;]$commaChangedServices"
+      Write-Host "##vso[task.setvariable variable=ChangedServicesCsv;]$commaChangedServices"
 
       Write-Host "This run is targeting: `n$(ArtifactPackageNames) in [$changedServices]"
     displayName: Resolve Targeted Packages
diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1
index 6badef1497ec..dd54f53cbb97 100644
--- a/eng/scripts/Language-Settings.ps1
+++ b/eng/scripts/Language-Settings.ps1
@@ -44,14 +44,13 @@ function Get-javascript-AdditionalValidationPackagesFromPackageSet {
   function isOther($fileName) {
     $startsWithPrefixes = @(".config", ".devcontainer", ".github", ".scripts", ".vscode", "common", "design", "documentation", "eng", "samples")
 
-    $startsWith = $false
     foreach ($prefix in $startsWithPrefixes) {
       if ($fileName.StartsWith($prefix)) {
-        $startsWith = $true
+        return $true
       }
     }
 
-    return $startsWith
+    return $false
   }
 
   $changedServices = @()
@@ -69,7 +68,7 @@ function Get-javascript-AdditionalValidationPackagesFromPackageSet {
   }
 
   $engChanged = $diffObj.ChangedFiles | Where-Object { $_.StartsWith("eng")}
-  $othersChanged = $diffObj.ChangedFiles | Where-Object { isOther($_) }
+  $othersChanged = $diffObj.ChangedFiles | Where-Object { isOther $_ }
   $changedServices = $changedServices | Get-Unique
 
   if ($engChanged -or $othersChanged) {

From 25c20ae9bd70150ac1787af504cb34b7272c0d15 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 23 Oct 2024 15:18:08 -0700
Subject: [PATCH 66/90] ensure we pass servicedirectory to the test job.
 resolve the issue with resolving the changed services

---
 eng/pipelines/templates/steps/set-artifact-packages.yml | 3 +--
 eng/pipelines/templates/steps/test.yml                  | 1 +
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/eng/pipelines/templates/steps/set-artifact-packages.yml b/eng/pipelines/templates/steps/set-artifact-packages.yml
index 2303fdb1fe7c..9ab0bcdd12cd 100644
--- a/eng/pipelines/templates/steps/set-artifact-packages.yml
+++ b/eng/pipelines/templates/steps/set-artifact-packages.yml
@@ -38,8 +38,7 @@ steps:
 
       $packageSet = "$(ArtifactPackageNames)" -split ","
 
-      $changedServicesArray = $packageProperties | Foreach-Object { Get-Content -Raw -Path $_.FullName | ConvertFrom-Json } `
-        | Where-Object { $packageSet -contains $_.ArtifactName }
+      $changedServicesArray = $packageProperties | Where-Object { $packageSet -contains $_.ArtifactName }
         | ForEach-Object { $_.ServiceDirectory } | Get-Unique
 
       $changedServices = $changedServicesArray -join " "
diff --git a/eng/pipelines/templates/steps/test.yml b/eng/pipelines/templates/steps/test.yml
index 81af90c88acc..ef90c99d2729 100644
--- a/eng/pipelines/templates/steps/test.yml
+++ b/eng/pipelines/templates/steps/test.yml
@@ -20,6 +20,7 @@ steps:
   - template: /eng/pipelines/templates/steps/set-artifact-packages.yml
     parameters:
       PackageInfo: $(Build.ArtifactStagingDirectory)/PackageInfo
+      Artifacts: ${{ parameters.Artifacts }}
 
   # Option "-p max" ensures parallelism is set to the number of cores on all platforms, which improves build times.
   # The default on Windows is "cores - 1" (microsoft/rushstack#436).

From ea53757ad57f358db268eecaac0d6261438beeee Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 23 Oct 2024 15:26:04 -0700
Subject: [PATCH 67/90] clean up changed service output

---
 eng/pipelines/templates/steps/set-artifact-packages.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/eng/pipelines/templates/steps/set-artifact-packages.yml b/eng/pipelines/templates/steps/set-artifact-packages.yml
index 9ab0bcdd12cd..8164c81bc668 100644
--- a/eng/pipelines/templates/steps/set-artifact-packages.yml
+++ b/eng/pipelines/templates/steps/set-artifact-packages.yml
@@ -43,7 +43,6 @@ steps:
 
       $changedServices = $changedServicesArray -join " "
       $commaChangedServices = $changedServicesArray -join ","
-      Write-Host "Setting ChangedServices to : `n$changedServices"
       Write-Host "##vso[task.setvariable variable=ChangedServices;]$changedServices"
       Write-Host "##vso[task.setvariable variable=ChangedServicesCsv;]$commaChangedServices"
 

From 72b74013d8f2fab93b9c6de1e7a5cf276b8030e9 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Fri, 25 Oct 2024 12:42:32 -0700
Subject: [PATCH 68/90] actually remove the branch ref

---
 sdk/pullrequest.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/sdk/pullrequest.yml b/sdk/pullrequest.yml
index 951dcee8d339..076b5d29654b 100644
--- a/sdk/pullrequest.yml
+++ b/sdk/pullrequest.yml
@@ -5,7 +5,6 @@ pr:
     - feature/*
     - hotfix/*
     - release/*
-    - restapi*
     - pipelinev3*
   paths:
     include:

From 191c7493f23ffe4a058d5c29925a53b8e0633d80 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Fri, 25 Oct 2024 14:30:47 -0700
Subject: [PATCH 69/90] rely on some heavier usage of common parlance

---
 .../templates/jobs/generate-job-matrix.yml    |   4 +
 .../scripts/job-matrix/Create-JobMatrix.ps1   |   5 +
 eng/pipelines/templates/jobs/ci.yml           |  24 +--
 eng/scripts/distribute-packages-to-matrix.ps1 | 202 +-----------------
 4 files changed, 30 insertions(+), 205 deletions(-)

diff --git a/eng/common/pipelines/templates/jobs/generate-job-matrix.yml b/eng/common/pipelines/templates/jobs/generate-job-matrix.yml
index a7459e6b5db5..45b73fd4a47e 100644
--- a/eng/common/pipelines/templates/jobs/generate-job-matrix.yml
+++ b/eng/common/pipelines/templates/jobs/generate-job-matrix.yml
@@ -42,6 +42,8 @@ parameters:
 - name: PreGenerationSteps
   type: stepList
   default: []
+- name: PostGenerationScript
+  type: string
 # Mappings to OS name required at template compile time by 1es pipeline templates
 - name: Pools
   type: object
@@ -98,6 +100,7 @@ jobs:
                 -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}'
                 -Replace '${{ join(''',''', parameters.MatrixReplace) }}'
                 -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
+                -PostGenerationScript '${{ parameters.PostGenerationScript }}'
             displayName: Create ${{ pool.name }} Matrix ${{ config.Name }}
             name: vm_job_matrix_${{ config.Name }}_${{ pool.name }}
 
@@ -112,6 +115,7 @@ jobs:
                 -DisplayNameFilter '$(displayNameFilter)'
                 -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}'
                 -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
+                -PostGenerationScript '${{ parameters.PostGenerationScript }}'
             displayName: Create ${{ pool.name }} Container Matrix ${{ config.Name }}
             name: container_job_matrix_${{ config.Name }}_${{ pool.name }}
 
diff --git a/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-JobMatrix.ps1
index fe98ca172167..25f3c24badda 100644
--- a/eng/common/scripts/job-matrix/Create-JobMatrix.ps1
+++ b/eng/common/scripts/job-matrix/Create-JobMatrix.ps1
@@ -15,6 +15,7 @@ param (
     [Parameter(Mandatory=$False)][array] $Filters,
     [Parameter(Mandatory=$False)][array] $Replace,
     [Parameter(Mandatory=$False)][array] $NonSparseParameters,
+    [Parameter(Mandatory=$False)][array] $PostGenerationScript,
     [Parameter()][switch] $CI = ($null -ne $env:SYSTEM_TEAMPROJECTID)
 )
 
@@ -36,6 +37,10 @@ $Filters = $Filters | Where-Object { $_ }
     -replace $Replace `
     -nonSparseParameters $NonSparseParameters
 
+if (Test-Path $PostGenerationScript) {
+    $matrix = &$PostGenerationScript -Matrix $matrix
+}
+
 $serialized = SerializePipelineMatrix $matrix
 
 Write-Output $serialized.pretty
diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml
index 2ba1eb86557c..ee47e4c227e4 100644
--- a/eng/pipelines/templates/jobs/ci.yml
+++ b/eng/pipelines/templates/jobs/ci.yml
@@ -75,23 +75,19 @@ jobs:
           MatrixFilters: ${{ parameters.MatrixFilters }}
           MatrixReplace: ${{ parameters.MatrixReplace }}
           SparseCheckoutPaths: [ "sdk/", ".vscode"]
-          PreGenerationSteps:
-            - template: /eng/common/pipelines/templates/steps/save-package-properties.yml
-              parameters:
-                ServiceDirectory: ${{parameters.ServiceDirectory}}
-
-            - task: Powershell@2
-              inputs:
-                pwsh: true
-                filePath: eng/scripts/distribute-packages-to-matrix.ps1
-                arguments: >-
-                  -PackageInfoFolder "$(Build.ArtifactStagingDirectory)/PackageInfo"
-                  -PlatformMatrix "${{ parameters.MatrixConfigs[0].Path }}"
-              displayName: 'Distribute Packages to Matrix'
-              condition: and(eq(variables['Build.Reason'], 'PullRequest'), eq('${{ parameters.ServiceDirectory }}','auto'))
           CloudConfig:
             Cloud: Public
           AdditionalParameters:
             ServiceDirectory: ${{ parameters.ServiceDirectory }}
             Artifacts: ${{ parameters.Artifacts }}
             TestProxy: ${{ parameters.TestProxy }}
+          ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.ServiceDirectory, 'auto')) }}:
+            PreGenerationSteps:
+              - template: /eng/common/pipelines/templates/steps/save-package-properties.yml
+                parameters:
+                  ServiceDirectory: ${{parameters.ServiceDirectory}}
+              - template: /eng/pipelines/templates/steps/set-artifact-packages.yml
+                parameters:
+                  PackageInfo: $(Build.ArtifactStagingDirectory)/PackageInfo
+                  Artifacts: ${{ parameters.Artifacts }}
+            PostGenerationScript: $(Build.SourcesDirectory)/eng/scripts/distribute-packages-to-matrix.ps1
diff --git a/eng/scripts/distribute-packages-to-matrix.ps1 b/eng/scripts/distribute-packages-to-matrix.ps1
index b43ed6134204..1533d7edd0fc 100644
--- a/eng/scripts/distribute-packages-to-matrix.ps1
+++ b/eng/scripts/distribute-packages-to-matrix.ps1
@@ -1,72 +1,21 @@
 <#
 .SYNOPSIS
-Distributes a set of packages to a platform matrix file by adding computing and adding a ArtifactPackageNames property
-dynamically. If an unexpected situation is encountered, the script will make no changes to the input file.
-
-.DESCRIPTION
-Because of the way the platform matrix file is structured, we need to distribute the packages in a way that
-honors the Include packages. We have these included this way because want to ensure that these node versions
-run on that platform. This script is aware of these additional configurations and will set ArtifactPackageNames for
-them as well.
+Used to update a generated matrix with targeted information for packages that are being built.
 #>
 param(
     [parameter(Mandatory=$true)]
-    [string]$PackageInfoFolder,
-    [parameter(Mandatory=$true)]
-    [string]$PlatformMatrix
+    [array]$Matrix
 )
 
 $RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot ".." ".."))
 Set-StrictMode -Version 4
 $BATCHSIZE = 10
 
-if (!(Test-Path $PackageInfoFolder)) {
-    Write-Error "PackageInfo folder file not found: $PackageInfoFolder"
-    exit 1
-}
-
-if (!(Test-Path $PlatformMatrix)) {
-    Write-Error "Platform matrix file not found: $PlatformMatrix"
+if (!(Test-Path $Matrix)) {
+    Write-Error "Matrix input not found: $Matrix"
     exit 1
 }
 
-function Extract-MatrixMultiplier {
-    param (
-        [Parameter(Mandatory=$true)]
-        [PSCustomObject]$Matrix
-    )
-
-    $highestCount = 1
-
-    foreach ($property in $Matrix.PSObject.Properties) {
-        $type = $property.Value.GetType().Name
-        switch ($type) {
-            "PSCustomObject" {
-                $itemCount = 0
-
-                # this looks very strange to loop over the properties of a PSCustomObject,
-                # but this is the only way to actually count. The Count/Length property is
-                # NOT available
-                foreach($innerProperty in $property.Value.PSObject.Properties) {
-                    $itemCount++
-                }
-
-                if ($itemCount -gt $highestCount) {
-                    $highestCount = $itemCount
-                }
-            }
-            "Object[]" {
-                $count = $property.Value.Length
-                if ($count -gt $highestCount) {
-                    $highestCount = $count
-                }
-            }
-        }
-    }
-
-    return $highestCount
-}
-
 function Split-ArrayIntoBatches {
     param (
         [Parameter(Mandatory=$true)]
@@ -87,147 +36,18 @@ function Split-ArrayIntoBatches {
     return ,$batches
 }
 
-function Update-Include {
-    param (
-        [Parameter(Mandatory=$true)]
-        $Matrix,
-        [Parameter(Mandatory=$true)]
-        $IncludeConfig,
-        [Parameter(Mandatory=$true)]
-        $TargetingString
-    )
-
-    foreach ($configElement in $IncludeConfig) {
-        if (-not $configElement.PSObject.Properties["ArtifactPackageNames"]) {
-            $configElement | Add-Member -Force -MemberType NoteProperty -Name ArtifactPackageNames -Value ""
-        }
-        $configElement.ArtifactPackageNames = $TargetingString
-        $Matrix.include += $configElement
-    }
-}
-
-function Update-Matrix {
-    param (
-        [Parameter(Mandatory=$true)]
-        $Matrix,
-
-        [Parameter(Mandatory=$true)]
-        $DirectBatches,
-
-        [Parameter(Mandatory=$true)]
-        $IndirectBatches,
-
-        [Parameter(Mandatory=$true)]
-        $MatrixMultiplier,
-
-        [Parameter(Mandatory=$true)]
-        $IncludeCount,
-
-        [Parameter(Mandatory=$false)]
-        $OriginalIncludeObject
-    )
-
-    $matrixUpdate = $true
-    if ($Matrix.matrix.PSObject.Properties["`$IMPORT"]) {
-        $matrixUpdate = $false
-    }
-
-    # we need to ensure the presence of TargetingString in the matrix object
-    if ($matrixUpdate) {
-        if ($directBatches -or $indirectBatches) {
-            if (-not $Matrix.matrix.PSObject.Properties["ArtifactPackageNames"]) {
-                $Matrix.matrix | Add-Member -Force -MemberType NoteProperty -Name ArtifactPackageNames -Value @()
-            }
-            else {
-                $Matrix.matrix.ArtifactPackageNames = @()
-            }
-        }
-    }
-
-    # a basic platform matrix has the following:
-    # "matrix" -> "NodeVersion" = []
-    # "include" -> "ConfigName" -> NodeVersion=18.X
-    # so what we need to do is for each batch
-        # 1. assign the batch * matrix size to the matrix as "ArtifactPackagesNames", this will evenly distribute the packages to EACH node version non-sparsely
-        # 2. assign the batch to each include with the batch being the value for ArtifactPackageNames for the include
-            # the easiest way to do this is to take a copy of the `Include` object at the beginning of this script,
-            # for each batch, assign the batch to the include object, and then add the include object to the matrix object
-            # this will have the result of multiplexing our includes, so we will need to update the name there as well
-    foreach($batch in $DirectBatches) {
-        $targetingString = ($batch | Select-Object -ExpandProperty ArtifactName) -join ","
-
-        # we need to equal the largest multiplier in the matrix. That is USUALLY nodeVersion version,
-        # but in some cases it could be something else (like sdk/cosmos/cosmos-emulator-matrix.json) the multiplier
-        # is a different property. We do this because we want to ensure that the matrix is fully covered, even if
-        # we are generating the matrix sparsely, we still want full coverage of these targetingStrings
-        $targetingStringArray = @($targetingString) * $MatrixMultiplier
-
-        if ($matrixUpdate) {
-            $matrix.matrix.ArtifactPackageNames += $targetingStringArray
-        }
-
-        # if there were any include objects, we need to duplicate them exactly and add the targeting string to each
-        # this means that the number of includes at the end of this operation will be incoming # of includes * the number of batches
-        if ($includeCount -gt 0) {
-            $includeCopy = $OriginalIncludeObject | ConvertTo-Json -Depth 100 | ConvertFrom-Json
-
-            Update-Include -Matrix $matrix -IncludeConfig $includeCopy -TargetingString $targetingString
-        }
-    }
-
-    foreach($batch in $IndirectBatches) {
-        $targetingString = ($batch | Select-Object -ExpandProperty Name) -join ","
-        if ($matrixUpdate) {
-            $matrix.matrix.ArtifactPackageNames += @($targetingString)
-        }
-    }
-
-}
-
 # calculate general targeting information and create our batches prior to updating any matrix
-$packageProperties = Get-ChildItem -Recurse "$PackageInfoFolder" *.json `
-    | % { Get-Content -Path $_.FullName | ConvertFrom-Json }
+$packageProperties = $env:ArtifactPackageNames.Split(",")
 
-# I will assign all the direct included packages first. our goal is to get full coverage of the direct included packages
-# then, for the indirect packages, we will add them as sparse ArtifactPackageNames bundles to the matrix
 $batches = Split-ArrayIntoBatches -InputArray $packageProperties -BatchSize $BATCHSIZE
 
-$matrix = Get-Content -Path $PlatformMatrix | ConvertFrom-Json
-
-$matrixLocation = $PlatformMatrix
+# we just smear the batches across the matrix now
 
-# handle the case where we discover an import, we need to climb the dependency in that case
-if ($matrix.matrix.PSObject.Properties["`$IMPORT"]) {
-    $originalMatrix = $matrix
-    $originalMatrixLocation = $PlatformMatrix
+Write-Host "Ok I see the following batches: "
 
-    # rest of the update will happen to that matrix object
-    $matrixLocation = (Join-Path $RepoRoot $matrix.matrix."`$IMPORT")
-    $matrix = Get-Content -Path $matrixLocation | ConvertFrom-Json
-
-    # we just need to walk the include objects and update the targeting string for the batches we have before
-    # updating in place, then continuing on to update the actual matrix object that we imported
-    # there is a clear and present edge case here with nested import. We will not handle that case
-
-    $originalInclude = $originalMatrix.include
-    $originalMatrix.include = @()
-
-    # update the include objects for the original matrix that was importing
-    Update-Matrix -Matrix $originalMatrix -DirectBatches $directBatches -IndirectBatches @() -MatrixMultiplier 1 -IncludeCount $matrix.include.Count -OriginalIncludeObject $originalInclude
-    $originalMatrix | ConvertTo-Json -Depth 100 | Set-Content -Path $originalMatrixLocation
+foreach ($batch in $batches) {
+    Write-Host "-> " $batch
 }
+Write-Host "Returning the exact same matrix that was passed in"
 
-$matrixMultiplier = Extract-MatrixMultiplier -Matrix $matrix.matrix
-
-$includeCount = 0
-$includeObject = $null
-$originalInclude = $null
-if ($matrix.PSObject.Properties.Name -contains "include") {
-    $includeCount = $matrix.include.Count
-    $originalInclude = $matrix.include
-    $matrix.include = @()
-}
-
-Update-Matrix -Matrix $matrix -DirectBatches $batches -IndirectBatches @() -MatrixMultiplier $matrixMultiplier -IncludeCount $includeCount -OriginalIncludeObject $originalInclude
-
-$matrix | ConvertTo-Json -Depth 100 | Set-Content -Path $matrixLocation
+return $Matrix

From 0f20711b6d8dc4c3bbfe9550195ad48681d3f956 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Fri, 25 Oct 2024 14:37:43 -0700
Subject: [PATCH 70/90] we need PostGenerationScript to be optional

---
 eng/common/pipelines/templates/jobs/generate-job-matrix.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/eng/common/pipelines/templates/jobs/generate-job-matrix.yml b/eng/common/pipelines/templates/jobs/generate-job-matrix.yml
index 45b73fd4a47e..43eaa892f900 100644
--- a/eng/common/pipelines/templates/jobs/generate-job-matrix.yml
+++ b/eng/common/pipelines/templates/jobs/generate-job-matrix.yml
@@ -44,6 +44,7 @@ parameters:
   default: []
 - name: PostGenerationScript
   type: string
+  default: ''
 # Mappings to OS name required at template compile time by 1es pipeline templates
 - name: Pools
   type: object

From 72db8de7bf5a18bd02a0044421cccf1a16fc6319 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Fri, 25 Oct 2024 15:16:57 -0700
Subject: [PATCH 71/90] simplify the matrix update

---
 eng/scripts/distribute-packages-to-matrix.ps1 | 31 ++++++++++++++-----
 1 file changed, 24 insertions(+), 7 deletions(-)

diff --git a/eng/scripts/distribute-packages-to-matrix.ps1 b/eng/scripts/distribute-packages-to-matrix.ps1
index 1533d7edd0fc..42412e9b53ec 100644
--- a/eng/scripts/distribute-packages-to-matrix.ps1
+++ b/eng/scripts/distribute-packages-to-matrix.ps1
@@ -11,7 +11,7 @@ $RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot ".." ".."))
 Set-StrictMode -Version 4
 $BATCHSIZE = 10
 
-if (!(Test-Path $Matrix)) {
+if (!$Matrix) {
     Write-Error "Matrix input not found: $Matrix"
     exit 1
 }
@@ -36,18 +36,35 @@ function Split-ArrayIntoBatches {
     return ,$batches
 }
 
-# calculate general targeting information and create our batches prior to updating any matrix
+# calculate the batches, then duplicate the entire matrix if need be
 $packageProperties = $env:ArtifactPackageNames.Split(",")
-
 $batches = Split-ArrayIntoBatches -InputArray $packageProperties -BatchSize $BATCHSIZE
 
-# we just smear the batches across the matrix now
-
-Write-Host "Ok I see the following batches: "
+Write-Host "I will invoke for the following batches: "
 
 foreach ($batch in $batches) {
     Write-Host "-> " $batch
 }
-Write-Host "Returning the exact same matrix that was passed in"
+
+Write-Host "This is the input: "
+Write-Host $Matrix
+Write-Host "End input`n`n`n"
+
+
+$ModifiedMatrix = @()
+# to understand this iteration, one must understand that the matrix is a list of hashtables, each with a couple keys:
+# [
+#  { "name": "jobname", "parameters": { matrixSetting1: matrixValue1, ...} },
+# ]
+if($batches.Length -gt 1) {
+  throw "This script is not prepared to handle more than one batch. We will need to duplicate the input objects."
+}
+else {
+  foreach($config in $Matrix) {
+      # we just need to iterate across them, grab the parameters hashtable, and add the new key
+      # if there is more than one batch, we will need to add a suffix including the batch name to the job name
+      $config["parameters"]["ArtifactPackageNames"] = $batch
+  }
+}
 
 return $Matrix

From 94ba6b598d82f06abfa27e18951ebbcb26ccca8b Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Fri, 25 Oct 2024 15:29:24 -0700
Subject: [PATCH 72/90] add some additional output

---
 eng/common/scripts/job-matrix/Create-JobMatrix.ps1 | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-JobMatrix.ps1
index 25f3c24badda..3d2879731058 100644
--- a/eng/common/scripts/job-matrix/Create-JobMatrix.ps1
+++ b/eng/common/scripts/job-matrix/Create-JobMatrix.ps1
@@ -38,7 +38,8 @@ $Filters = $Filters | Where-Object { $_ }
     -nonSparseParameters $NonSparseParameters
 
 if (Test-Path $PostGenerationScript) {
-    $matrix = &$PostGenerationScript -Matrix $matrix
+    Write-Host "Invoking post-generation script: $PostGenerationScript"
+    $matrix = &"$PostGenerationScript" -Matrix $matrix
 }
 
 $serialized = SerializePipelineMatrix $matrix

From 9aa8facf77921da1e8a914f9adb8e7151bd5cb3e Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Fri, 25 Oct 2024 15:45:32 -0700
Subject: [PATCH 73/90] dump the output right before using it

---
 eng/common/scripts/job-matrix/Create-JobMatrix.ps1 | 1 +
 1 file changed, 1 insertion(+)

diff --git a/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-JobMatrix.ps1
index 3d2879731058..6e4b6beb5dc1 100644
--- a/eng/common/scripts/job-matrix/Create-JobMatrix.ps1
+++ b/eng/common/scripts/job-matrix/Create-JobMatrix.ps1
@@ -39,6 +39,7 @@ $Filters = $Filters | Where-Object { $_ }
 
 if (Test-Path $PostGenerationScript) {
     Write-Host "Invoking post-generation script: $PostGenerationScript"
+    cat $PostGenerationScript
     $matrix = &"$PostGenerationScript" -Matrix $matrix
 }
 

From 4f67fe6847cdafaa9c6f92d4a31039e0f5743d80 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Fri, 25 Oct 2024 15:49:34 -0700
Subject: [PATCH 74/90] remove useless check for engChanged when it is
 encapsulated by 'other' changes

---
 eng/common/scripts/job-matrix/Create-JobMatrix.ps1 | 2 +-
 eng/scripts/Language-Settings.ps1                  | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-JobMatrix.ps1
index 6e4b6beb5dc1..a5a89544ed77 100644
--- a/eng/common/scripts/job-matrix/Create-JobMatrix.ps1
+++ b/eng/common/scripts/job-matrix/Create-JobMatrix.ps1
@@ -15,7 +15,7 @@ param (
     [Parameter(Mandatory=$False)][array] $Filters,
     [Parameter(Mandatory=$False)][array] $Replace,
     [Parameter(Mandatory=$False)][array] $NonSparseParameters,
-    [Parameter(Mandatory=$False)][array] $PostGenerationScript,
+    [Parameter(Mandatory=$False)][string] $PostGenerationScript,
     [Parameter()][switch] $CI = ($null -ne $env:SYSTEM_TEAMPROJECTID)
 )
 
diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1
index dd54f53cbb97..e639c6fd4ede 100644
--- a/eng/scripts/Language-Settings.ps1
+++ b/eng/scripts/Language-Settings.ps1
@@ -67,11 +67,10 @@ function Get-javascript-AdditionalValidationPackagesFromPackageSet {
     }
   }
 
-  $engChanged = $diffObj.ChangedFiles | Where-Object { $_.StartsWith("eng")}
   $othersChanged = $diffObj.ChangedFiles | Where-Object { isOther $_ }
   $changedServices = $changedServices | Get-Unique
 
-  if ($engChanged -or $othersChanged) {
+  if ($othersChanged) {
     $additionalPackages = $ReducedDependencyLookup["core"] | ForEach-Object { $me=$_; $AllPkgProps | Where-Object { $_.Name -eq $me } | Select-Object -First 1 }
     $additionalValidationPackages += $additionalPackages
   }

From 2a0391e8c7ca3b20081bce43ceea514e33627550 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Fri, 25 Oct 2024 15:56:12 -0700
Subject: [PATCH 75/90] add cmdlet binding

---
 eng/scripts/distribute-packages-to-matrix.ps1 | 1 +
 1 file changed, 1 insertion(+)

diff --git a/eng/scripts/distribute-packages-to-matrix.ps1 b/eng/scripts/distribute-packages-to-matrix.ps1
index 42412e9b53ec..ee3b5552f09f 100644
--- a/eng/scripts/distribute-packages-to-matrix.ps1
+++ b/eng/scripts/distribute-packages-to-matrix.ps1
@@ -2,6 +2,7 @@
 .SYNOPSIS
 Used to update a generated matrix with targeted information for packages that are being built.
 #>
+[CmdletBinding()]
 param(
     [parameter(Mandatory=$true)]
     [array]$Matrix

From 2996fdfe98323bc69d207b68246db934eb0f68d9 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Fri, 25 Oct 2024 16:00:27 -0700
Subject: [PATCH 76/90] dramatically simplify the call to distribute packages
 to matrix

---
 .../scripts/job-matrix/Create-JobMatrix.ps1   |  2 +-
 eng/scripts/distribute-packages-to-matrix.ps1 | 92 +++++++++----------
 2 files changed, 47 insertions(+), 47 deletions(-)

diff --git a/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-JobMatrix.ps1
index a5a89544ed77..3d288ae6380f 100644
--- a/eng/common/scripts/job-matrix/Create-JobMatrix.ps1
+++ b/eng/common/scripts/job-matrix/Create-JobMatrix.ps1
@@ -40,7 +40,7 @@ $Filters = $Filters | Where-Object { $_ }
 if (Test-Path $PostGenerationScript) {
     Write-Host "Invoking post-generation script: $PostGenerationScript"
     cat $PostGenerationScript
-    $matrix = &"$PostGenerationScript" -Matrix $matrix
+    $matrix = & $PostGenerationScript -Matrix $matrix
 }
 
 $serialized = SerializePipelineMatrix $matrix
diff --git a/eng/scripts/distribute-packages-to-matrix.ps1 b/eng/scripts/distribute-packages-to-matrix.ps1
index ee3b5552f09f..ac88891014c5 100644
--- a/eng/scripts/distribute-packages-to-matrix.ps1
+++ b/eng/scripts/distribute-packages-to-matrix.ps1
@@ -8,64 +8,64 @@ param(
     [array]$Matrix
 )
 
-$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot ".." ".."))
-Set-StrictMode -Version 4
-$BATCHSIZE = 10
+# $RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot ".." ".."))
+# Set-StrictMode -Version 4
+# $BATCHSIZE = 10
 
-if (!$Matrix) {
-    Write-Error "Matrix input not found: $Matrix"
-    exit 1
-}
+# if (!$Matrix) {
+#     Write-Error "Matrix input not found: $Matrix"
+#     exit 1
+# }
 
-function Split-ArrayIntoBatches {
-    param (
-        [Parameter(Mandatory=$true)]
-        [object[]]$InputArray,
+# function Split-ArrayIntoBatches {
+#     param (
+#         [Parameter(Mandatory=$true)]
+#         [object[]]$InputArray,
 
-        [Parameter(Mandatory=$true)]
-        [int]$BatchSize
-    )
+#         [Parameter(Mandatory=$true)]
+#         [int]$BatchSize
+#     )
 
-    $batches = @()
+#     $batches = @()
 
-    for ($i = 0; $i -lt $InputArray.Count; $i += $BatchSize) {
-        $batch = $InputArray[$i..[math]::Min($i + $BatchSize - 1, $InputArray.Count - 1)]
+#     for ($i = 0; $i -lt $InputArray.Count; $i += $BatchSize) {
+#         $batch = $InputArray[$i..[math]::Min($i + $BatchSize - 1, $InputArray.Count - 1)]
 
-        $batches += ,$batch
-    }
+#         $batches += ,$batch
+#     }
 
-    return ,$batches
-}
+#     return ,$batches
+# }
 
-# calculate the batches, then duplicate the entire matrix if need be
-$packageProperties = $env:ArtifactPackageNames.Split(",")
-$batches = Split-ArrayIntoBatches -InputArray $packageProperties -BatchSize $BATCHSIZE
+# # calculate the batches, then duplicate the entire matrix if need be
+# $packageProperties = $env:ArtifactPackageNames.Split(",")
+# $batches = Split-ArrayIntoBatches -InputArray $packageProperties -BatchSize $BATCHSIZE
 
-Write-Host "I will invoke for the following batches: "
+# Write-Host "I will invoke for the following batches: "
 
-foreach ($batch in $batches) {
-    Write-Host "-> " $batch
-}
+# foreach ($batch in $batches) {
+#     Write-Host "-> " $batch
+# }
 
-Write-Host "This is the input: "
-Write-Host $Matrix
-Write-Host "End input`n`n`n"
+# Write-Host "This is the input: "
+# Write-Host $Matrix
+# Write-Host "End input`n`n`n"
 
 
-$ModifiedMatrix = @()
-# to understand this iteration, one must understand that the matrix is a list of hashtables, each with a couple keys:
-# [
-#  { "name": "jobname", "parameters": { matrixSetting1: matrixValue1, ...} },
-# ]
-if($batches.Length -gt 1) {
-  throw "This script is not prepared to handle more than one batch. We will need to duplicate the input objects."
-}
-else {
-  foreach($config in $Matrix) {
-      # we just need to iterate across them, grab the parameters hashtable, and add the new key
-      # if there is more than one batch, we will need to add a suffix including the batch name to the job name
-      $config["parameters"]["ArtifactPackageNames"] = $batch
-  }
-}
+# $ModifiedMatrix = @()
+# # to understand this iteration, one must understand that the matrix is a list of hashtables, each with a couple keys:
+# # [
+# #  { "name": "jobname", "parameters": { matrixSetting1: matrixValue1, ...} },
+# # ]
+# if($batches.Length -gt 1) {
+#   throw "This script is not prepared to handle more than one batch. We will need to duplicate the input objects."
+# }
+# else {
+#   foreach($config in $Matrix) {
+#       # we just need to iterate across them, grab the parameters hashtable, and add the new key
+#       # if there is more than one batch, we will need to add a suffix including the batch name to the job name
+#       $config["parameters"]["ArtifactPackageNames"] = $batch
+#   }
+# }
 
 return $Matrix

From e75d15672bc404f5280fdac8493e55d9e37faed4 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Fri, 25 Oct 2024 16:08:12 -0700
Subject: [PATCH 77/90] and now that it will actually invoke the script, add
 back in the call logic

---
 .../scripts/job-matrix/Create-JobMatrix.ps1   |  2 -
 eng/scripts/distribute-packages-to-matrix.ps1 | 92 +++++++++----------
 2 files changed, 46 insertions(+), 48 deletions(-)

diff --git a/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-JobMatrix.ps1
index 3d288ae6380f..6e76f94f6c8e 100644
--- a/eng/common/scripts/job-matrix/Create-JobMatrix.ps1
+++ b/eng/common/scripts/job-matrix/Create-JobMatrix.ps1
@@ -38,8 +38,6 @@ $Filters = $Filters | Where-Object { $_ }
     -nonSparseParameters $NonSparseParameters
 
 if (Test-Path $PostGenerationScript) {
-    Write-Host "Invoking post-generation script: $PostGenerationScript"
-    cat $PostGenerationScript
     $matrix = & $PostGenerationScript -Matrix $matrix
 }
 
diff --git a/eng/scripts/distribute-packages-to-matrix.ps1 b/eng/scripts/distribute-packages-to-matrix.ps1
index ac88891014c5..ee3b5552f09f 100644
--- a/eng/scripts/distribute-packages-to-matrix.ps1
+++ b/eng/scripts/distribute-packages-to-matrix.ps1
@@ -8,64 +8,64 @@ param(
     [array]$Matrix
 )
 
-# $RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot ".." ".."))
-# Set-StrictMode -Version 4
-# $BATCHSIZE = 10
+$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot ".." ".."))
+Set-StrictMode -Version 4
+$BATCHSIZE = 10
 
-# if (!$Matrix) {
-#     Write-Error "Matrix input not found: $Matrix"
-#     exit 1
-# }
+if (!$Matrix) {
+    Write-Error "Matrix input not found: $Matrix"
+    exit 1
+}
 
-# function Split-ArrayIntoBatches {
-#     param (
-#         [Parameter(Mandatory=$true)]
-#         [object[]]$InputArray,
+function Split-ArrayIntoBatches {
+    param (
+        [Parameter(Mandatory=$true)]
+        [object[]]$InputArray,
 
-#         [Parameter(Mandatory=$true)]
-#         [int]$BatchSize
-#     )
+        [Parameter(Mandatory=$true)]
+        [int]$BatchSize
+    )
 
-#     $batches = @()
+    $batches = @()
 
-#     for ($i = 0; $i -lt $InputArray.Count; $i += $BatchSize) {
-#         $batch = $InputArray[$i..[math]::Min($i + $BatchSize - 1, $InputArray.Count - 1)]
+    for ($i = 0; $i -lt $InputArray.Count; $i += $BatchSize) {
+        $batch = $InputArray[$i..[math]::Min($i + $BatchSize - 1, $InputArray.Count - 1)]
 
-#         $batches += ,$batch
-#     }
+        $batches += ,$batch
+    }
 
-#     return ,$batches
-# }
+    return ,$batches
+}
 
-# # calculate the batches, then duplicate the entire matrix if need be
-# $packageProperties = $env:ArtifactPackageNames.Split(",")
-# $batches = Split-ArrayIntoBatches -InputArray $packageProperties -BatchSize $BATCHSIZE
+# calculate the batches, then duplicate the entire matrix if need be
+$packageProperties = $env:ArtifactPackageNames.Split(",")
+$batches = Split-ArrayIntoBatches -InputArray $packageProperties -BatchSize $BATCHSIZE
 
-# Write-Host "I will invoke for the following batches: "
+Write-Host "I will invoke for the following batches: "
 
-# foreach ($batch in $batches) {
-#     Write-Host "-> " $batch
-# }
+foreach ($batch in $batches) {
+    Write-Host "-> " $batch
+}
 
-# Write-Host "This is the input: "
-# Write-Host $Matrix
-# Write-Host "End input`n`n`n"
+Write-Host "This is the input: "
+Write-Host $Matrix
+Write-Host "End input`n`n`n"
 
 
-# $ModifiedMatrix = @()
-# # to understand this iteration, one must understand that the matrix is a list of hashtables, each with a couple keys:
-# # [
-# #  { "name": "jobname", "parameters": { matrixSetting1: matrixValue1, ...} },
-# # ]
-# if($batches.Length -gt 1) {
-#   throw "This script is not prepared to handle more than one batch. We will need to duplicate the input objects."
-# }
-# else {
-#   foreach($config in $Matrix) {
-#       # we just need to iterate across them, grab the parameters hashtable, and add the new key
-#       # if there is more than one batch, we will need to add a suffix including the batch name to the job name
-#       $config["parameters"]["ArtifactPackageNames"] = $batch
-#   }
-# }
+$ModifiedMatrix = @()
+# to understand this iteration, one must understand that the matrix is a list of hashtables, each with a couple keys:
+# [
+#  { "name": "jobname", "parameters": { matrixSetting1: matrixValue1, ...} },
+# ]
+if($batches.Length -gt 1) {
+  throw "This script is not prepared to handle more than one batch. We will need to duplicate the input objects."
+}
+else {
+  foreach($config in $Matrix) {
+      # we just need to iterate across them, grab the parameters hashtable, and add the new key
+      # if there is more than one batch, we will need to add a suffix including the batch name to the job name
+      $config["parameters"]["ArtifactPackageNames"] = $batch
+  }
+}
 
 return $Matrix

From 1248bca2d7c1d4befd13e9790e81a268a59640ad Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Fri, 25 Oct 2024 16:18:42 -0700
Subject: [PATCH 78/90] one attempt using no write-hosts or strict mode

---
 eng/scripts/distribute-packages-to-matrix.ps1 | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/eng/scripts/distribute-packages-to-matrix.ps1 b/eng/scripts/distribute-packages-to-matrix.ps1
index ee3b5552f09f..2590b9dd8601 100644
--- a/eng/scripts/distribute-packages-to-matrix.ps1
+++ b/eng/scripts/distribute-packages-to-matrix.ps1
@@ -9,7 +9,6 @@ param(
 )
 
 $RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot ".." ".."))
-Set-StrictMode -Version 4
 $BATCHSIZE = 10
 
 if (!$Matrix) {
@@ -40,18 +39,6 @@ function Split-ArrayIntoBatches {
 # calculate the batches, then duplicate the entire matrix if need be
 $packageProperties = $env:ArtifactPackageNames.Split(",")
 $batches = Split-ArrayIntoBatches -InputArray $packageProperties -BatchSize $BATCHSIZE
-
-Write-Host "I will invoke for the following batches: "
-
-foreach ($batch in $batches) {
-    Write-Host "-> " $batch
-}
-
-Write-Host "This is the input: "
-Write-Host $Matrix
-Write-Host "End input`n`n`n"
-
-
 $ModifiedMatrix = @()
 # to understand this iteration, one must understand that the matrix is a list of hashtables, each with a couple keys:
 # [

From 5d831c963df7b183d6481884272aede617bb6750 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Fri, 25 Oct 2024 16:23:44 -0700
Subject: [PATCH 79/90] remove weirdness...I think this was breaking somehow
 behind the scenes

---
 eng/scripts/distribute-packages-to-matrix.ps1 | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/eng/scripts/distribute-packages-to-matrix.ps1 b/eng/scripts/distribute-packages-to-matrix.ps1
index 2590b9dd8601..147daf08baa2 100644
--- a/eng/scripts/distribute-packages-to-matrix.ps1
+++ b/eng/scripts/distribute-packages-to-matrix.ps1
@@ -39,6 +39,7 @@ function Split-ArrayIntoBatches {
 # calculate the batches, then duplicate the entire matrix if need be
 $packageProperties = $env:ArtifactPackageNames.Split(",")
 $batches = Split-ArrayIntoBatches -InputArray $packageProperties -BatchSize $BATCHSIZE
+
 $ModifiedMatrix = @()
 # to understand this iteration, one must understand that the matrix is a list of hashtables, each with a couple keys:
 # [
@@ -51,7 +52,7 @@ else {
   foreach($config in $Matrix) {
       # we just need to iterate across them, grab the parameters hashtable, and add the new key
       # if there is more than one batch, we will need to add a suffix including the batch name to the job name
-      $config["parameters"]["ArtifactPackageNames"] = $batch
+      $config["parameters"]["ArtifactPackageNames"] = ($batches -join ",")
   }
 }
 

From 3a094729c0687134e653737cb05ef32b38c43180 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Fri, 25 Oct 2024 16:32:14 -0700
Subject: [PATCH 80/90] disable everything except a single write host

---
 eng/scripts/distribute-packages-to-matrix.ps1 | 96 ++++++++++---------
 1 file changed, 49 insertions(+), 47 deletions(-)

diff --git a/eng/scripts/distribute-packages-to-matrix.ps1 b/eng/scripts/distribute-packages-to-matrix.ps1
index 147daf08baa2..7a16ec5efd55 100644
--- a/eng/scripts/distribute-packages-to-matrix.ps1
+++ b/eng/scripts/distribute-packages-to-matrix.ps1
@@ -8,52 +8,54 @@ param(
     [array]$Matrix
 )
 
-$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot ".." ".."))
-$BATCHSIZE = 10
-
-if (!$Matrix) {
-    Write-Error "Matrix input not found: $Matrix"
-    exit 1
-}
-
-function Split-ArrayIntoBatches {
-    param (
-        [Parameter(Mandatory=$true)]
-        [object[]]$InputArray,
-
-        [Parameter(Mandatory=$true)]
-        [int]$BatchSize
-    )
-
-    $batches = @()
-
-    for ($i = 0; $i -lt $InputArray.Count; $i += $BatchSize) {
-        $batch = $InputArray[$i..[math]::Min($i + $BatchSize - 1, $InputArray.Count - 1)]
-
-        $batches += ,$batch
-    }
-
-    return ,$batches
-}
-
-# calculate the batches, then duplicate the entire matrix if need be
-$packageProperties = $env:ArtifactPackageNames.Split(",")
-$batches = Split-ArrayIntoBatches -InputArray $packageProperties -BatchSize $BATCHSIZE
-
-$ModifiedMatrix = @()
-# to understand this iteration, one must understand that the matrix is a list of hashtables, each with a couple keys:
-# [
-#  { "name": "jobname", "parameters": { matrixSetting1: matrixValue1, ...} },
-# ]
-if($batches.Length -gt 1) {
-  throw "This script is not prepared to handle more than one batch. We will need to duplicate the input objects."
-}
-else {
-  foreach($config in $Matrix) {
-      # we just need to iterate across them, grab the parameters hashtable, and add the new key
-      # if there is more than one batch, we will need to add a suffix including the batch name to the job name
-      $config["parameters"]["ArtifactPackageNames"] = ($batches -join ",")
-  }
-}
+# $RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot ".." ".."))
+# $BATCHSIZE = 10
+
+# if (!$Matrix) {
+#     Write-Error "Matrix input not found: $Matrix"
+#     exit 1
+# }
+
+# function Split-ArrayIntoBatches {
+#     param (
+#         [Parameter(Mandatory=$true)]
+#         [object[]]$InputArray,
+
+#         [Parameter(Mandatory=$true)]
+#         [int]$BatchSize
+#     )
+
+#     $batches = @()
+
+#     for ($i = 0; $i -lt $InputArray.Count; $i += $BatchSize) {
+#         $batch = $InputArray[$i..[math]::Min($i + $BatchSize - 1, $InputArray.Count - 1)]
+
+#         $batches += ,$batch
+#     }
+
+#     return ,$batches
+# }
+
+Write-Host $env:ArtifactPackageNames
+
+# # calculate the batches, then duplicate the entire matrix if need be
+# $packageProperties = $env:ArtifactPackageNames.Split(",")
+# $batches = Split-ArrayIntoBatches -InputArray $packageProperties -BatchSize $BATCHSIZE
+
+# $ModifiedMatrix = @()
+# # to understand this iteration, one must understand that the matrix is a list of hashtables, each with a couple keys:
+# # [
+# #  { "name": "jobname", "parameters": { matrixSetting1: matrixValue1, ...} },
+# # ]
+# if($batches.Length -gt 1) {
+#   throw "This script is not prepared to handle more than one batch. We will need to duplicate the input objects."
+# }
+# else {
+#   foreach($config in $Matrix) {
+#       # we just need to iterate across them, grab the parameters hashtable, and add the new key
+#       # if there is more than one batch, we will need to add a suffix including the batch name to the job name
+#       $config["parameters"]["ArtifactPackageNames"] = ($batches -join ",")
+#   }
+# }
 
 return $Matrix

From 7b64344df9e59b773422b5219cfd9f65be06774c Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Fri, 25 Oct 2024 16:34:45 -0700
Subject: [PATCH 81/90] now remove the Write-Host

---
 eng/scripts/distribute-packages-to-matrix.ps1 | 2 --
 1 file changed, 2 deletions(-)

diff --git a/eng/scripts/distribute-packages-to-matrix.ps1 b/eng/scripts/distribute-packages-to-matrix.ps1
index 7a16ec5efd55..d9dde18a414d 100644
--- a/eng/scripts/distribute-packages-to-matrix.ps1
+++ b/eng/scripts/distribute-packages-to-matrix.ps1
@@ -36,8 +36,6 @@ param(
 #     return ,$batches
 # }
 
-Write-Host $env:ArtifactPackageNames
-
 # # calculate the batches, then duplicate the entire matrix if need be
 # $packageProperties = $env:ArtifactPackageNames.Split(",")
 # $batches = Split-ArrayIntoBatches -InputArray $packageProperties -BatchSize $BATCHSIZE

From 16cd6b30a4e08a0ba61ed14f4b7cbfd1d984ce49 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Fri, 25 Oct 2024 16:42:41 -0700
Subject: [PATCH 82/90] removing the write-host solved the problem. time to
 dump the variable names. what is going on!?

---
 eng/pipelines/templates/jobs/ci.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml
index ee47e4c227e4..33e45a71875c 100644
--- a/eng/pipelines/templates/jobs/ci.yml
+++ b/eng/pipelines/templates/jobs/ci.yml
@@ -90,4 +90,7 @@ jobs:
                 parameters:
                   PackageInfo: $(Build.ArtifactStagingDirectory)/PackageInfo
                   Artifacts: ${{ parameters.Artifacts }}
+              - pwsh: |
+                  Write-Host "This is what I see in environment variables"
+                  Get-ChildItem env:
             PostGenerationScript: $(Build.SourcesDirectory)/eng/scripts/distribute-packages-to-matrix.ps1

From cd50e715137b4f3f60c3045e3b7abeb8786a56ff Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Fri, 25 Oct 2024 16:50:46 -0700
Subject: [PATCH 83/90] try re-enabling and referencing a real environment
 setting

---
 eng/scripts/distribute-packages-to-matrix.ps1 | 94 +++++++++----------
 1 file changed, 47 insertions(+), 47 deletions(-)

diff --git a/eng/scripts/distribute-packages-to-matrix.ps1 b/eng/scripts/distribute-packages-to-matrix.ps1
index d9dde18a414d..cf920dcdc27a 100644
--- a/eng/scripts/distribute-packages-to-matrix.ps1
+++ b/eng/scripts/distribute-packages-to-matrix.ps1
@@ -8,52 +8,52 @@ param(
     [array]$Matrix
 )
 
-# $RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot ".." ".."))
-# $BATCHSIZE = 10
-
-# if (!$Matrix) {
-#     Write-Error "Matrix input not found: $Matrix"
-#     exit 1
-# }
-
-# function Split-ArrayIntoBatches {
-#     param (
-#         [Parameter(Mandatory=$true)]
-#         [object[]]$InputArray,
-
-#         [Parameter(Mandatory=$true)]
-#         [int]$BatchSize
-#     )
-
-#     $batches = @()
-
-#     for ($i = 0; $i -lt $InputArray.Count; $i += $BatchSize) {
-#         $batch = $InputArray[$i..[math]::Min($i + $BatchSize - 1, $InputArray.Count - 1)]
-
-#         $batches += ,$batch
-#     }
-
-#     return ,$batches
-# }
-
-# # calculate the batches, then duplicate the entire matrix if need be
-# $packageProperties = $env:ArtifactPackageNames.Split(",")
-# $batches = Split-ArrayIntoBatches -InputArray $packageProperties -BatchSize $BATCHSIZE
-
-# $ModifiedMatrix = @()
-# # to understand this iteration, one must understand that the matrix is a list of hashtables, each with a couple keys:
-# # [
-# #  { "name": "jobname", "parameters": { matrixSetting1: matrixValue1, ...} },
-# # ]
-# if($batches.Length -gt 1) {
-#   throw "This script is not prepared to handle more than one batch. We will need to duplicate the input objects."
-# }
-# else {
-#   foreach($config in $Matrix) {
-#       # we just need to iterate across them, grab the parameters hashtable, and add the new key
-#       # if there is more than one batch, we will need to add a suffix including the batch name to the job name
-#       $config["parameters"]["ArtifactPackageNames"] = ($batches -join ",")
-#   }
-# }
+$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot ".." ".."))
+$BATCHSIZE = 10
+
+if (!$Matrix) {
+    Write-Error "Matrix input not found: $Matrix"
+    exit 1
+}
+
+function Split-ArrayIntoBatches {
+    param (
+        [Parameter(Mandatory=$true)]
+        [object[]]$InputArray,
+
+        [Parameter(Mandatory=$true)]
+        [int]$BatchSize
+    )
+
+    $batches = @()
+
+    for ($i = 0; $i -lt $InputArray.Count; $i += $BatchSize) {
+        $batch = $InputArray[$i..[math]::Min($i + $BatchSize - 1, $InputArray.Count - 1)]
+
+        $batches += ,$batch
+    }
+
+    return ,$batches
+}
+
+# calculate the batches, then duplicate the entire matrix if need be
+$packageProperties = $env:ARTIFACTPACKAGENAMES.Split(",")
+$batches = Split-ArrayIntoBatches -InputArray $packageProperties -BatchSize $BATCHSIZE
+
+$ModifiedMatrix = @()
+# to understand this iteration, one must understand that the matrix is a list of hashtables, each with a couple keys:
+# [
+#  { "name": "jobname", "parameters": { matrixSetting1: matrixValue1, ...} },
+# ]
+if($batches.Length -gt 1) {
+  throw "This script is not prepared to handle more than one batch. We will need to duplicate the input objects."
+}
+else {
+  foreach($config in $Matrix) {
+      # we just need to iterate across them, grab the parameters hashtable, and add the new key
+      # if there is more than one batch, we will need to add a suffix including the batch name to the job name
+      $config["parameters"]["ArtifactPackageNames"] = ($batches -join ",")
+  }
+}
 
 return $Matrix

From 909c5b7da72e675e2d7355affd92a80a63249a15 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Fri, 25 Oct 2024 17:00:52 -0700
Subject: [PATCH 84/90] fix the batch save

---
 eng/scripts/distribute-packages-to-matrix.ps1 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eng/scripts/distribute-packages-to-matrix.ps1 b/eng/scripts/distribute-packages-to-matrix.ps1
index cf920dcdc27a..6ebecf18d260 100644
--- a/eng/scripts/distribute-packages-to-matrix.ps1
+++ b/eng/scripts/distribute-packages-to-matrix.ps1
@@ -52,7 +52,7 @@ else {
   foreach($config in $Matrix) {
       # we just need to iterate across them, grab the parameters hashtable, and add the new key
       # if there is more than one batch, we will need to add a suffix including the batch name to the job name
-      $config["parameters"]["ArtifactPackageNames"] = ($batches -join ",")
+      $config["parameters"]["ArtifactPackageNames"] = $batches
   }
 }
 

From e9e0e827bd1ede030ea77e04938f4704669d0ecf Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Fri, 25 Oct 2024 17:42:50 -0700
Subject: [PATCH 85/90] the parameter input MUST be a string, not an array[]
 type

---
 eng/scripts/distribute-packages-to-matrix.ps1 | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/eng/scripts/distribute-packages-to-matrix.ps1 b/eng/scripts/distribute-packages-to-matrix.ps1
index 6ebecf18d260..a74b02cc1cee 100644
--- a/eng/scripts/distribute-packages-to-matrix.ps1
+++ b/eng/scripts/distribute-packages-to-matrix.ps1
@@ -19,7 +19,7 @@ if (!$Matrix) {
 function Split-ArrayIntoBatches {
     param (
         [Parameter(Mandatory=$true)]
-        [object[]]$InputArray,
+        [string[]]$InputArray,
 
         [Parameter(Mandatory=$true)]
         [int]$BatchSize
@@ -38,7 +38,7 @@ function Split-ArrayIntoBatches {
 
 # calculate the batches, then duplicate the entire matrix if need be
 $packageProperties = $env:ARTIFACTPACKAGENAMES.Split(",")
-$batches = Split-ArrayIntoBatches -InputArray $packageProperties -BatchSize $BATCHSIZE
+[array]$batches = Split-ArrayIntoBatches -InputArray $packageProperties -BatchSize $BATCHSIZE
 
 $ModifiedMatrix = @()
 # to understand this iteration, one must understand that the matrix is a list of hashtables, each with a couple keys:
@@ -52,7 +52,7 @@ else {
   foreach($config in $Matrix) {
       # we just need to iterate across them, grab the parameters hashtable, and add the new key
       # if there is more than one batch, we will need to add a suffix including the batch name to the job name
-      $config["parameters"]["ArtifactPackageNames"] = $batches
+      $config["parameters"]["ArtifactPackageNames"] = $batches[0] -join ","
   }
 }
 

From cddda7f301aab8d9f505a7b889deb7cecb316095 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 30 Oct 2024 17:07:31 -0700
Subject: [PATCH 86/90] now remove the distribute-packages-to-matrix

---
 eng/pipelines/templates/jobs/ci.yml           | 11 +---
 eng/scripts/distribute-packages-to-matrix.ps1 | 59 -------------------
 2 files changed, 2 insertions(+), 68 deletions(-)
 delete mode 100644 eng/scripts/distribute-packages-to-matrix.ps1

diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml
index 33e45a71875c..d6f5c38b0ebb 100644
--- a/eng/pipelines/templates/jobs/ci.yml
+++ b/eng/pipelines/templates/jobs/ci.yml
@@ -81,16 +81,9 @@ jobs:
             ServiceDirectory: ${{ parameters.ServiceDirectory }}
             Artifacts: ${{ parameters.Artifacts }}
             TestProxy: ${{ parameters.TestProxy }}
-          ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.ServiceDirectory, 'auto')) }}:
+          ${{ if eq(parameters.ServiceDirectory, 'auto') }}:
+            EnablePRGeneration: true
             PreGenerationSteps:
               - template: /eng/common/pipelines/templates/steps/save-package-properties.yml
                 parameters:
                   ServiceDirectory: ${{parameters.ServiceDirectory}}
-              - template: /eng/pipelines/templates/steps/set-artifact-packages.yml
-                parameters:
-                  PackageInfo: $(Build.ArtifactStagingDirectory)/PackageInfo
-                  Artifacts: ${{ parameters.Artifacts }}
-              - pwsh: |
-                  Write-Host "This is what I see in environment variables"
-                  Get-ChildItem env:
-            PostGenerationScript: $(Build.SourcesDirectory)/eng/scripts/distribute-packages-to-matrix.ps1
diff --git a/eng/scripts/distribute-packages-to-matrix.ps1 b/eng/scripts/distribute-packages-to-matrix.ps1
deleted file mode 100644
index a74b02cc1cee..000000000000
--- a/eng/scripts/distribute-packages-to-matrix.ps1
+++ /dev/null
@@ -1,59 +0,0 @@
-<#
-.SYNOPSIS
-Used to update a generated matrix with targeted information for packages that are being built.
-#>
-[CmdletBinding()]
-param(
-    [parameter(Mandatory=$true)]
-    [array]$Matrix
-)
-
-$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot ".." ".."))
-$BATCHSIZE = 10
-
-if (!$Matrix) {
-    Write-Error "Matrix input not found: $Matrix"
-    exit 1
-}
-
-function Split-ArrayIntoBatches {
-    param (
-        [Parameter(Mandatory=$true)]
-        [string[]]$InputArray,
-
-        [Parameter(Mandatory=$true)]
-        [int]$BatchSize
-    )
-
-    $batches = @()
-
-    for ($i = 0; $i -lt $InputArray.Count; $i += $BatchSize) {
-        $batch = $InputArray[$i..[math]::Min($i + $BatchSize - 1, $InputArray.Count - 1)]
-
-        $batches += ,$batch
-    }
-
-    return ,$batches
-}
-
-# calculate the batches, then duplicate the entire matrix if need be
-$packageProperties = $env:ARTIFACTPACKAGENAMES.Split(",")
-[array]$batches = Split-ArrayIntoBatches -InputArray $packageProperties -BatchSize $BATCHSIZE
-
-$ModifiedMatrix = @()
-# to understand this iteration, one must understand that the matrix is a list of hashtables, each with a couple keys:
-# [
-#  { "name": "jobname", "parameters": { matrixSetting1: matrixValue1, ...} },
-# ]
-if($batches.Length -gt 1) {
-  throw "This script is not prepared to handle more than one batch. We will need to duplicate the input objects."
-}
-else {
-  foreach($config in $Matrix) {
-      # we just need to iterate across them, grab the parameters hashtable, and add the new key
-      # if there is more than one batch, we will need to add a suffix including the batch name to the job name
-      $config["parameters"]["ArtifactPackageNames"] = $batches[0] -join ","
-  }
-}
-
-return $Matrix

From f0172ed10a35aafe42ddd3bf8d43ca0d08162074 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 30 Oct 2024 17:50:16 -0700
Subject: [PATCH 87/90] restore eng/common to main

---
 eng/common/scripts/job-matrix/Create-JobMatrix.ps1  |  5 -----
 eng/pipelines/templates/stages/platform-matrix.json | 13 +++----------
 2 files changed, 3 insertions(+), 15 deletions(-)

diff --git a/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-JobMatrix.ps1
index 6e76f94f6c8e..fe98ca172167 100644
--- a/eng/common/scripts/job-matrix/Create-JobMatrix.ps1
+++ b/eng/common/scripts/job-matrix/Create-JobMatrix.ps1
@@ -15,7 +15,6 @@ param (
     [Parameter(Mandatory=$False)][array] $Filters,
     [Parameter(Mandatory=$False)][array] $Replace,
     [Parameter(Mandatory=$False)][array] $NonSparseParameters,
-    [Parameter(Mandatory=$False)][string] $PostGenerationScript,
     [Parameter()][switch] $CI = ($null -ne $env:SYSTEM_TEAMPROJECTID)
 )
 
@@ -37,10 +36,6 @@ $Filters = $Filters | Where-Object { $_ }
     -replace $Replace `
     -nonSparseParameters $NonSparseParameters
 
-if (Test-Path $PostGenerationScript) {
-    $matrix = & $PostGenerationScript -Matrix $matrix
-}
-
 $serialized = SerializePipelineMatrix $matrix
 
 Write-Output $serialized.pretty
diff --git a/eng/pipelines/templates/stages/platform-matrix.json b/eng/pipelines/templates/stages/platform-matrix.json
index 6680f8cd40fc..ba03aa2b3691 100644
--- a/eng/pipelines/templates/stages/platform-matrix.json
+++ b/eng/pipelines/templates/stages/platform-matrix.json
@@ -17,11 +17,7 @@
         "Pool": "env:MACPOOL"
       }
     },
-    "NodeTestVersion": [
-      "18.x",
-      "20.x",
-      "22.x"
-    ],
+    "NodeTestVersion": ["18.x", "20.x", "22.x"],
     "TestType": "node",
     "TestResultsFiles": "**/test-results.xml"
   },
@@ -59,11 +55,8 @@
       },
       "TestType": "node",
       "NodeTestVersion": "18.x",
-      "DependencyVersion": [
-        "max",
-        "min"
-      ],
+      "DependencyVersion": ["max", "min"],
       "TestResultsFiles": "**/test-results.xml"
     }
   ]
-}
+}
\ No newline at end of file

From 0799806ec1003b1aecb3e8cc47bfd13b6184dfb4 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Wed, 30 Oct 2024 18:27:54 -0700
Subject: [PATCH 88/90] fix the platform matrix back to main version

---
 eng/pipelines/templates/stages/platform-matrix.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eng/pipelines/templates/stages/platform-matrix.json b/eng/pipelines/templates/stages/platform-matrix.json
index ba03aa2b3691..b8fda65b7211 100644
--- a/eng/pipelines/templates/stages/platform-matrix.json
+++ b/eng/pipelines/templates/stages/platform-matrix.json
@@ -59,4 +59,4 @@
       "TestResultsFiles": "**/test-results.xml"
     }
   ]
-}
\ No newline at end of file
+}

From 6dda23d30fc2ffc35e744d27b9c320f06763b2ec Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Thu, 31 Oct 2024 13:40:57 -0700
Subject: [PATCH 89/90] resolve the issue with the artifactpackagenames

---
 eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1
index 5129d757f99c..7c52ce335876 100644
--- a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1
+++ b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1
@@ -93,6 +93,7 @@ foreach ($matrixBatchKey in $matrixBatchesByConfig.Keys) {
     # we only need to modify the generated job name if there is more than one matrix config or batch in the matrix
     $matrixSuffixNecessary = $matrixConfigs.Count -gt 1
     $batchSuffixNecessary = $packageBatches.Length -gt 1
+    $batchCounter = 1
 
     foreach ($batch in $packageBatches) {
       # to understand this iteration, one must understand that the matrix is a list of hashtables, each with a couple keys:
@@ -110,11 +111,12 @@ foreach ($matrixBatchKey in $matrixBatchesByConfig.Keys) {
         }
 
         if ($batchSuffixNecessary) {
-          $matrixOutputItem["name"] = $matrixOutputItem["name"] + $namesForBatch
+          $matrixOutputItem["name"] = $matrixOutputItem["name"] + "b$batchCounter"
         }
 
         $OverallResult += $matrixOutputItem
       }
+      $batchCounter += 1
     }
   }
 }

From d67e2f9b4d21e515199a99db8015085001e9f8f4 Mon Sep 17 00:00:00 2001
From: Scott Beddall <scbedd@microsoft.com>
Date: Thu, 31 Oct 2024 13:59:54 -0700
Subject: [PATCH 90/90] remove erroneously added audit step

---
 eng/pipelines/templates/steps/analyze.yml | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml
index 09f899441371..41e21b077c91 100644
--- a/eng/pipelines/templates/steps/analyze.yml
+++ b/eng/pipelines/templates/steps/analyze.yml
@@ -82,11 +82,6 @@ steps:
       node eng/tools/rush-runner.js check-format $(ChangedServices) -packages "$(ArtifactPackageNames)" --verbose
     displayName: "Check Format in Libraries"
 
-  - script: |
-      node eng/tools/rush-runner.js audit $(ChangedServices) -packages "$(ArtifactPackageNames)"
-    condition: and(succeeded(), eq(variables['RunNpmAudit'], 'true'))
-    displayName: "Audit libraries"
-
   - template: /eng/common/pipelines/templates/steps/verify-changelogs.yml
     parameters:
       PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo