From 30a3364cca5d645e6cbb162cd2e4dcf872645026 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 7 Jan 2025 12:44:26 -0800 Subject: [PATCH 1/2] updates to dynamic matrix generation. moves indirectly included packages to sparse usage. adds additional parameters to GeneratePRMatrix that enable further filtering of the matrix generated for the indirect packages adjustments from feedback. currently running in an sdk-for-net branch Update eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 Co-authored-by: Wes Haggard --- .../templates/jobs/generate-job-matrix.yml | 11 +- .../templates/steps/verify-readmes.yml | 9 +- .../scripts/job-matrix/Create-PrJobMatrix.ps1 | 221 +++++++++++++----- 3 files changed, 185 insertions(+), 56 deletions(-) diff --git a/eng/common/pipelines/templates/jobs/generate-job-matrix.yml b/eng/common/pipelines/templates/jobs/generate-job-matrix.yml index ab67e915de85..d1db47b30b22 100644 --- a/eng/common/pipelines/templates/jobs/generate-job-matrix.yml +++ b/eng/common/pipelines/templates/jobs/generate-job-matrix.yml @@ -48,6 +48,12 @@ parameters: - name: PRMatrixSetting type: string default: 'ArtifactPackageNames' +- name: PRJobBatchSize + type: number + default: 10 +- name: PRMatrixIndirectFilters + type: object + default: [] # Mappings to OS name required at template compile time by 1es pipeline templates - name: Pools type: object @@ -126,7 +132,6 @@ jobs: - ${{ else }}: - ${{ each pool in parameters.Pools }}: - pwsh: | - # dump the conglomerated CI matrix '${{ convertToJson(parameters.MatrixConfigs) }}' | Set-Content matrix.json ./eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 ` @@ -135,7 +140,9 @@ jobs: -PRMatrixSetting ${{ parameters.PRMatrixSetting }} ` -DisplayNameFilter '$(displayNameFilter)' ` -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}' ` - -Replace '${{ join(''',''', parameters.MatrixReplace) }}' + -IndirectFilters '${{ join(''',''', parameters.PRMatrixIndirectFilters) }}' ` + -Replace '${{ join(''',''', parameters.MatrixReplace) }}' ` + -PackagesPerPRJob ${{ parameters.PRJobBatchSize }} displayName: Create ${{ pool.name }} PR Matrix name: vm_job_matrix_pr_${{ pool.name }} diff --git a/eng/common/pipelines/templates/steps/verify-readmes.yml b/eng/common/pipelines/templates/steps/verify-readmes.yml index 09628b373897..40994767e8cd 100644 --- a/eng/common/pipelines/templates/steps/verify-readmes.yml +++ b/eng/common/pipelines/templates/steps/verify-readmes.yml @@ -13,13 +13,20 @@ parameters: - name: Condition type: string default: succeeded() +- name: IncludeIndirect + type: boolean + default: true steps: - pwsh: | + $includeIndirect = $${{ parameters.IncludeIndirect }} $packageProperties = Get-ChildItem -Recurse "${{ parameters.PackagePropertiesFolder }}" *.json - $paths = @() + if (-not $includeIndirect) { + $packageProperties = $packageProperties | Where-Object { (Get-Content -Raw $_ | ConvertFrom-Json).IncludedForValidation -eq $false } + } + foreach($propertiesFile in $packageProperties) { $PackageProp = Get-Content -Path $propertiesFile | ConvertFrom-Json diff --git a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 index 1d86cfcd79db..9120f6c2c1ca 100644 --- a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 +++ b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 @@ -23,6 +23,12 @@ It generates the matrix by the following algorithm: - add the combined property name to the parameters of the matrix item - add the matrix item to the overall result +.PARAMETER IndirectFilters +Any array of strings representing filters that will only be applied to the matrix generation for indirect packages. This is useful for +filtering out OTHER parameter settings othan than PRMatrixSetting that are only relevant to direct packages. + +For .NET, this value will be AdditionalTestArguments=/p:UseProjectReferenceToAzureClients=true + .EXAMPLE ./eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 ` -PackagePropertiesFolder "path/to/populated/PackageInfo" ` @@ -36,13 +42,158 @@ param ( [Parameter(Mandatory = $true)][string] $PRMatrixSetting, [Parameter(Mandatory = $False)][string] $DisplayNameFilter, [Parameter(Mandatory = $False)][array] $Filters, + [Parameter(Mandatory = $False)][array] $IndirectFilters, [Parameter(Mandatory = $False)][array] $Replace, + [Parameter(Mandatory = $False)][int] $PackagesPerPRJob = 10, [Parameter()][switch] $CI = ($null -ne $env:SYSTEM_TEAMPROJECTID) ) +Set-StrictMode -Version 4 . $PSScriptRoot/job-matrix-functions.ps1 . $PSScriptRoot/../Helpers/Package-Helpers.ps1 -$BATCHSIZE = 10 +. $PSScriptRoot/../Package-Properties.ps1 +$BATCHSIZE = $PackagesPerPRJob + +# this function takes an array of objects, takes a copy of the first item, and moves that item to the back of the array +function QueuePop([ref]$queue) { + + if ($queue.Value.Length -eq 1) { + return ($queue.Value[0] | ConvertTo-Json -Depth 100 | ConvertFrom-Json -AsHashtable) + } + + # otherwise we can rotate stuff + $first = $queue.Value[0] + $rest = $queue.Value[1..($queue.Value.Length - 1)] + + $queue.Value = $rest + $first + + return ($first | ConvertTo-Json -Depth 100 | ConvertFrom-Json -AsHashtable) +} + +function GeneratePRMatrixForBatch { + param ( + [Parameter(Mandatory = $true)][array] $Packages + ) + + $OverallResult = @() + if (!$Packages) { + Write-Host "Unable to generate matrix for empty package list" + return ,$OverallResult + } + + # this check assumes that we have properly separated the direct and indirect package lists + $directBatch = $Packages[0].IncludedForValidation -eq $false + Write-Host "Generating matrix for $($directBatch ? 'direct' : 'indirect') packages" + + # The key here is that after we group the packages by the matrix config objects, we can use the first item's MatrixConfig + # to generate the matrix for the group, no reason to have to parse the key value backwards to get the matrix config. + $matrixBatchesByConfig = Group-ByObjectKey $Packages "CIMatrixConfigs" + + foreach ($matrixBatchKey in $matrixBatchesByConfig.Keys) { + $matrixBatch = $matrixBatchesByConfig[$matrixBatchKey] + $matrixConfigs = $matrixBatch | Select-Object -First 1 -ExpandProperty CIMatrixConfigs + + $matrixResults = @() + foreach ($matrixConfig in $matrixConfigs) { + Write-Host "Generating config for $($matrixConfig.Path)" + + $matrixResults = @() + if ($directBatch) { + $matrixResults = GenerateMatrixForConfig ` + -ConfigPath $matrixConfig.Path ` + -Selection $matrixConfig.Selection ` + -DisplayNameFilter $DisplayNameFilter ` + -Filters $Filters ` + -Replace $Replace + + if ($matrixResults) { + Write-Host "We have the following direct matrix results: " + Write-Host ($matrixResults | Out-String) + } + } + else { + $matrixResults = GenerateMatrixForConfig ` + -ConfigPath $matrixConfig.Path ` + -Selection $matrixConfig.Selection ` + -DisplayNameFilter $DisplayNameFilter ` + -Filters ($Filters + $IndirectFilters) ` + -Replace $Replace + + if ($matrixResults) { + Write-Host "We have the following indirect matrix results: " + Write-Host ($matrixResults | Out-String) + } + else { + Write-Host "No indirect matrix results found for $($matrixConfig.Path)" + continue + } + } + + $packageBatches = Split-ArrayIntoBatches -InputArray $matrixBatch -BatchSize $BATCHSIZE + + # we only need to modify the generated job name if there is more than one matrix config + batch + $matrixSuffixNecessary = $matrixBatchesByConfig.Keys.Count -gt 1 + + # if we are doing direct packages, we need to walk the batches and duplicate the matrix config for each batch, fully assigning + # the each batch's packages to the matrix config. This will generate a _non-sparse_ matrix for the incoming packages + if ($directBatch) { + $batchSuffixNecessary = $packageBatches.Length -gt 1 + $batchCounter = 1 + + foreach ($batch in $packageBatches) { + $namesForBatch = ($batch | ForEach-Object { $_.ArtifactName }) -join "," + + foreach ($matrixOutputItem in $matrixResults) { + # we need to clone this, as each item is an object with possible children + $outputItem = $matrixOutputItem | ConvertTo-Json -Depth 100 | ConvertFrom-Json -AsHashtable + # we just need to iterate across them, grab the parameters hashtable, and add the new key + # if there is more than one batch, we will need to add a suffix including the batch name to the job name + $outputItem["parameters"]["$PRMatrixSetting"] = $namesForBatch + + if ($matrixSuffixNecessary) { + $outputItem["name"] = $outputItem["name"] + "_" + $matrixConfig.Name + } + + if ($batchSuffixNecessary) { + $outputItem["name"] = $outputItem["name"] + "_b$batchCounter" + } + + $OverallResult += $outputItem + } + $batchCounter += 1 + } + } + # in the case of indirect packages, instead of walking the batches and duplicating their matrix config entirely, + # we instead will walk each each matrix, create a parameter named for the PRMatrixSetting, and add the targeted packages + # as an array. This will generate a _sparse_ matrix for for whatever the incoming packages are + else { + $batchSuffixNecessary = $packageBatches.Length -gt 1 + $batchCounter = 1 + foreach ($batch in $packageBatches) { + $namesForBatch = ($batch | ForEach-Object { $_.ArtifactName }) -join "," + $outputItem = QueuePop -queue ([ref]$matrixResults) + + $outputItem["parameters"]["$PRMatrixSetting"] = $namesForBatch + + if ($matrixSuffixNecessary) { + $outputItem["name"] = $outputItem["name"] + "_" + $matrixConfig.Name + } + + if ($batchSuffixNecessary) { + $outputItem["name"] = $outputItem["name"] + "_ib$batchCounter" + } + # now we need to take an item from the front of the matrix results, clone it, and add it to the back of the matrix results + # we will add the cloned version to OverallResult + $OverallResult += $outputItem + $batchCounter += 1 + } + } + } + } + + + return ,$OverallResult +} if (!(Test-Path $PackagePropertiesFolder)) { Write-Error "Package Properties folder doesn't exist" @@ -58,71 +209,35 @@ Write-Host "Generating PR job matrix for $PackagePropertiesFolder" $configs = Get-Content -Raw $PRMatrixFile | ConvertFrom-Json -# calculate general targeting information and create our batches prior to generating any matrix +# get all the package property objects loaded $packageProperties = Get-ChildItem -Recurse "$PackagePropertiesFolder" *.json ` | ForEach-Object { Get-Content -Path $_.FullName | ConvertFrom-Json } -# set default matrix config for each package if there isn't an override +# enhance the package props with a default matrix config if one isn't present $packageProperties | ForEach-Object { if (-not $_.CIMatrixConfigs) { $_.CIMatrixConfigs = $configs } } -# The key here is that after we group the packages by the matrix config objects, we can use the first item's MatrixConfig -# to generate the matrix for the group, no reason to have to parse the key value backwards to get the matrix config. -$matrixBatchesByConfig = Group-ByObjectKey $packageProperties "CIMatrixConfigs" +$directPackages = $packageProperties | Where-Object { $_.IncludedForValidation -eq $false } +$indirectPackages = $packageProperties | Where-Object { $_.IncludedForValidation -eq $true } $OverallResult = @() -foreach ($matrixBatchKey in $matrixBatchesByConfig.Keys) { - $matrixBatch = $matrixBatchesByConfig[$matrixBatchKey] - $matrixConfigs = $matrixBatch | Select-Object -First 1 -ExpandProperty CIMatrixConfigs - - $matrixResults = @() - foreach ($matrixConfig in $matrixConfigs) { - Write-Host "Generating config for $($matrixConfig.Path)" - $matrixResults = GenerateMatrixForConfig ` - -ConfigPath $matrixConfig.Path ` - -Selection $matrixConfig.Selection ` - -DisplayNameFilter $DisplayNameFilter ` - -Filters $Filters ` - -Replace $Replace - - $packageBatches = Split-ArrayIntoBatches -InputArray $matrixBatch -BatchSize $BATCHSIZE - - # we only need to modify the generated job name if there is more than one matrix config + batch - $matrixSuffixNecessary = $matrixBatchesByConfig.Keys.Count -gt 1 - $batchSuffixNecessary = $packageBatches.Length -gt 1 - $batchCounter = 1 - - foreach ($batch in $packageBatches) { - $namesForBatch = ($batch | ForEach-Object { $_.ArtifactName }) -join "," - # to understand this iteration, one must understand that the matrix is a list of hashtables, each with a couple keys: - # [ - # { "name": "jobname", "parameters": { matrixSetting1: matrixValue1, ...} }, - # ] - foreach ($matrixOutputItem in $matrixResults) { - # we need to clone this, as each item is an object with possible children - $outputItem = $matrixOutputItem | ConvertTo-Json -Depth 100 | ConvertFrom-Json -AsHashtable - # we just need to iterate across them, grab the parameters hashtable, and add the new key - # if there is more than one batch, we will need to add a suffix including the batch name to the job name - $outputItem["parameters"]["$PRMatrixSetting"] = $namesForBatch - - if ($matrixSuffixNecessary) { - $outputItem["name"] = $outputItem["name"] + "_" + $matrixConfig.Name - } - - if ($batchSuffixNecessary) { - $outputItem["name"] = $outputItem["name"] + "_b$batchCounter" - } - - $OverallResult += $outputItem - } - $batchCounter += 1 - } +if ($directPackages) { + Write-Host "Discovered $($directPackages.Length) direct packages" + foreach($artifact in $directPackages) { + Write-Host "-> $($artifact.ArtifactName)" } + $OverallResult += GeneratePRMatrixForBatch -Packages $directPackages +} +if ($indirectPackages) { + Write-Host "Discovered $($indirectPackages.Length) indirect packages" + foreach($artifact in $indirectPackages) { + Write-Host "-> $($artifact.ArtifactName)" + } + $OverallResult += GeneratePRMatrixForBatch -Packages $indirectPackages } - $serialized = SerializePipelineMatrix $OverallResult Write-Output $serialized.pretty From ea9d74d720442c40367354daffec81f7587c4e9d Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 16 Jan 2025 14:56:10 -0800 Subject: [PATCH 2/2] in the case where it's all the same matrix (so no batches), not adding a suffix for the indirect batch causes duplicate config names, so the indirect batch is ignored entirely --- eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 index 9120f6c2c1ca..8e18e2479ca4 100644 --- a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 +++ b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 @@ -167,7 +167,7 @@ function GeneratePRMatrixForBatch { # we instead will walk each each matrix, create a parameter named for the PRMatrixSetting, and add the targeted packages # as an array. This will generate a _sparse_ matrix for for whatever the incoming packages are else { - $batchSuffixNecessary = $packageBatches.Length -gt 1 + $batchSuffixNecessary = $packageBatches.Length -gt 0 $batchCounter = 1 foreach ($batch in $packageBatches) { $namesForBatch = ($batch | ForEach-Object { $_.ArtifactName }) -join ","