From 4fcfe1d896b1a1f9fd3f914c8b3e6bca0473fda2 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Sun, 27 Oct 2024 16:51:54 -0700 Subject: [PATCH 01/15] well it's invoking without breaking --- .../templates/jobs/generate-job-matrix.yml | 123 ++++++---- .../scripts/Helpers/Package-Helpers.ps1 | 22 +- eng/common/scripts/Package-Properties.ps1 | 24 +- .../scripts/job-matrix/Create-PrJobMatrix.ps1 | 109 +++++++++ eng/pipelines/templates/jobs/ci.yml | 24 +- eng/scripts/distribute-packages-to-matrix.ps1 | 220 ++---------------- 6 files changed, 263 insertions(+), 259 deletions(-) create mode 100644 eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 diff --git a/eng/common/pipelines/templates/jobs/generate-job-matrix.yml b/eng/common/pipelines/templates/jobs/generate-job-matrix.yml index a7459e6b5db5..7bee03f8d235 100644 --- a/eng/common/pipelines/templates/jobs/generate-job-matrix.yml +++ b/eng/common/pipelines/templates/jobs/generate-job-matrix.yml @@ -42,6 +42,11 @@ parameters: - name: PreGenerationSteps type: stepList default: [] +- name: EnablePRGeneration + type: boolean + default: false +- name: PRMatrix + type: object # Mappings to OS name required at template compile time by 1es pipeline templates - name: Pools type: object @@ -84,57 +89,87 @@ jobs: - ${{ parameters.PreGenerationSteps }} - - ${{ each config in parameters.MatrixConfigs }}: - - ${{ each pool in parameters.Pools }}: - - ${{ if eq(config.GenerateVMJobs, 'true') }}: + - ${{ if eq(parameters.EnablePRGeneration, false) }}: + - ${{ each config in parameters.MatrixConfigs }}: + - ${{ each pool in parameters.Pools }}: + - ${{ if eq(config.GenerateVMJobs, 'true') }}: + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1 + arguments: > + -ConfigPath ${{ config.Path }} + -Selection ${{ config.Selection }} + -DisplayNameFilter '$(displayNameFilter)' + -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}' + -Replace '${{ join(''',''', parameters.MatrixReplace) }}' + -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}' + displayName: Create ${{ pool.name }} Matrix ${{ config.Name }} + name: vm_job_matrix_${{ config.Name }}_${{ pool.name }} + - ${{ if eq(config.GenerateContainerJobs, 'true') }}: + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1 + arguments: > + -ConfigPath ${{ config.Path }} + -Selection ${{ config.Selection }} + -DisplayNameFilter '$(displayNameFilter)' + -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}' + -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}' + displayName: Create ${{ pool.name }} Container Matrix ${{ config.Name }} + name: container_job_matrix_${{ config.Name }}_${{ pool.name }} + + # This else being set also currently assumes that the $(Build.ArtifactStagingDirectory)/PackageInfo folder is populated by PreGenerationSteps. + # Not currently not hardcoded, so not doing the needful and populating this folder before we hit this step will result in generation errors. + - ${{ else }}: + - ${{ each pool in parameters.Pools }}: - task: Powershell@2 inputs: pwsh: true - filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1 + filePath: eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 arguments: > - -ConfigPath ${{ config.Path }} - -Selection ${{ config.Selection }} + -PackagePropertiesFolder $(Build.ArtifactStagingDirectory)/PackageInfo + -PRMatrixFile ${{ parameters.PRMatrix.Path }} + -PRMatrixSetting ${{ parameters.PRMatrix.Setting }} -DisplayNameFilter '$(displayNameFilter)' -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}' -Replace '${{ join(''',''', parameters.MatrixReplace) }}' - -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}' - displayName: Create ${{ pool.name }} Matrix ${{ config.Name }} - name: vm_job_matrix_${{ config.Name }}_${{ pool.name }} + displayName: Create ${{ pool.name }} PR Matrix + name: vm_job_matrix_pr_${{ pool.name }} - - ${{ if eq(config.GenerateContainerJobs, 'true') }}: - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1 - arguments: > - -ConfigPath ${{ config.Path }} - -Selection ${{ config.Selection }} - -DisplayNameFilter '$(displayNameFilter)' - -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}' - -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}' - displayName: Create ${{ pool.name }} Container Matrix ${{ config.Name }} - name: container_job_matrix_${{ config.Name }}_${{ pool.name }} +- ${{ if eq(parameters.EnablePRGeneration, false) }}: + - ${{ each config in parameters.MatrixConfigs }}: + - ${{ each pool in parameters.Pools }}: + - ${{ if eq(config.GenerateVMJobs, 'true') }}: + - template: ${{ parameters.JobTemplatePath }} + parameters: + UsePlatformContainer: false + OSName: ${{ pool.os }} + Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix'] + DependsOn: ${{ parameters.GenerateJobName }} + CloudConfig: ${{ parameters.CloudConfig }} + ${{ each param in parameters.AdditionalParameters }}: + ${{ param.key }}: ${{ param.value }} -- ${{ each config in parameters.MatrixConfigs }}: + - ${{ if eq(config.GenerateContainerJobs, 'true') }}: + - template: ${{ parameters.JobTemplatePath }} + parameters: + UsePlatformContainer: true + OSName: ${{ pool.os }} + Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix'] + DependsOn: ${{ parameters.GenerateJobName }} + CloudConfig: ${{ parameters.CloudConfig }} + ${{ each param in parameters.AdditionalParameters }}: + ${{ param.key }}: ${{ param.value }} +- ${{ else }}: - ${{ each pool in parameters.Pools }}: - - ${{ if eq(config.GenerateVMJobs, 'true') }}: - - template: ${{ parameters.JobTemplatePath }} - parameters: - UsePlatformContainer: false - OSName: ${{ pool.os }} - Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix'] - DependsOn: ${{ parameters.GenerateJobName }} - CloudConfig: ${{ parameters.CloudConfig }} - ${{ each param in parameters.AdditionalParameters }}: - ${{ param.key }}: ${{ param.value }} - - - ${{ if eq(config.GenerateContainerJobs, 'true') }}: - - template: ${{ parameters.JobTemplatePath }} - parameters: - UsePlatformContainer: true - OSName: ${{ pool.os }} - Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix'] - DependsOn: ${{ parameters.GenerateJobName }} - CloudConfig: ${{ parameters.CloudConfig }} - ${{ each param in parameters.AdditionalParameters }}: - ${{ param.key }}: ${{ param.value }} + - template: ${{ parameters.JobTemplatePath }} + parameters: + UsePlatformContainer: false + OSName: ${{ pool.os }} + Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_pr_${{ pool.name }}.matrix'] + DependsOn: ${{ parameters.GenerateJobName }} + CloudConfig: ${{ parameters.CloudConfig }} + ${{ each param in parameters.AdditionalParameters }}: + ${{ param.key }}: ${{ param.value }} diff --git a/eng/common/scripts/Helpers/Package-Helpers.ps1 b/eng/common/scripts/Helpers/Package-Helpers.ps1 index 3c882b31111b..bb492cc6f109 100644 --- a/eng/common/scripts/Helpers/Package-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Package-Helpers.ps1 @@ -176,4 +176,24 @@ function GetValueSafelyFrom-Yaml { } return [object]$current -} \ No newline at end of file +} + +function Split-ArrayIntoBatches { + param ( + [Parameter(Mandatory=$true)] + [string[]]$InputArray, + + [Parameter(Mandatory=$true)] + [int]$BatchSize + ) + + $batches = @() + + for ($i = 0; $i -lt $InputArray.Count; $i += $BatchSize) { + $batch = $InputArray[$i..[math]::Min($i + $BatchSize - 1, $InputArray.Count - 1)] + + $batches += ,$batch + } + + return ,$batches +} diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1 index 61054f10f22a..572ecbe449c9 100644 --- a/eng/common/scripts/Package-Properties.ps1 +++ b/eng/common/scripts/Package-Properties.ps1 @@ -21,6 +21,7 @@ class PackageProps # additional packages required for validation of this one [string[]]$AdditionalValidationPackages [HashTable]$ArtifactDetails + [HashTable[]]$CIMatrixConfigs PackageProps([string]$name, [string]$version, [string]$directoryPath, [string]$serviceDirectory) { @@ -84,8 +85,7 @@ class PackageProps $this.Group = $group } - hidden [HashTable]ParseYmlForArtifact([string]$ymlPath) { - + hidden [PSCustomObject]ParseYmlForArtifact([string]$ymlPath) { $content = LoadFrom-Yaml $ymlPath if ($content) { $artifacts = GetValueSafelyFrom-Yaml $content @("extends", "parameters", "Artifacts") @@ -95,8 +95,21 @@ class PackageProps $artifactForCurrentPackage = $artifacts | Where-Object { $_["name"] -eq $this.ArtifactName -or $_["name"] -eq $this.Name } } + # if we found an artifact for the current package, we should count this ci file as the source of the matrix for this package if ($artifactForCurrentPackage) { - return [HashTable]$artifactForCurrentPackage + $result = [PSCustomObject]@{ + ArtifactConfig = [HashTable]$artifactForCurrentPackage + MatrixConfigs = @() + } + + # if we know this is the matrix for our file, we should now see if there is a custom matrix config for the package + $matrixConfigList = GetValueSafelyFrom-Yaml $content @("extends", "parameters", "MatrixConfigs") + + if ($matrixConfigList) { + $result.MatrixConfigs = matrixConfigList + } + + return $result } } return $null @@ -112,7 +125,10 @@ class PackageProps foreach($ciFile in $ciFiles) { $ciArtifactResult = $this.ParseYmlForArtifact($ciFile.FullName) if ($ciArtifactResult) { - $this.ArtifactDetails = [Hashtable]$ciArtifactResult + $this.ArtifactDetails = [Hashtable]$ciArtifactResult.ArtifactConfig + $this.CIMatrixConfigs = $ciArtifactResult.MatrixConfigs + # if this package appeared in this ci file, then we should + # treat this CI file as the source of the Matrix for this package break } } diff --git a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 new file mode 100644 index 000000000000..003a34faa341 --- /dev/null +++ b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 @@ -0,0 +1,109 @@ +<# +.SYNOPSIS +Generates a combined PR job matrix from a package properties folder. It is effectively a combination of +Create-JobMatrix and distribute-packages-to-matrix + +This script is intended to be used within an Azure DevOps pipeline to generate a job matrix for a PR. + +.EXAMPLE +./eng/common/scripts/Create-PRJobMatrix $(Build.ArtifactStagingDirectory)/PackageProperties +#> + +[CmdletBinding()] +param ( + [Parameter(Mandatory=$true)][string] $PackagePropertiesFolder, + [Parameter(Mandatory=$true)][string] $PRMatrixFile, + [Parameter(Mandatory=$true)][string] $PRMatrixSetting, + [Parameter(Mandatory=$False)][string] $DisplayNameFilter, + [Parameter(Mandatory=$False)][array] $Filters, + [Parameter(Mandatory=$False)][array] $Replace, + [Parameter()][switch] $CI = ($null -ne $env:SYSTEM_TEAMPROJECTID) +) + +. $PSScriptRoot/job-matrix-functions.ps1 +. $PSScriptRoot/../Helpers/Package-Helpers.ps1 + +if (!(Test-Path $PackagePropertiesFolder)) { + Write-Error "Package Properties folder doesn't exist" + exit 1 +} + +function GenerateMatrixForConfig { + param ( + [Parameter(Mandatory=$true)][string] $ConfigPath, + [Parameter(Mandatory=$True)][string] $Selection, + [Parameter(Mandatory=$false)][string] $DisplayNameFilter, + [Parameter(Mandatory=$false)][array] $Filters, + [Parameter(Mandatory=$false)][array] $Replace + ) + + $config = GetMatrixConfigFromFile (Get-Content $ConfigPath -Raw) + # Strip empty string filters in order to be able to use azure pipelines yaml join() + $Filters = $Filters | Where-Object { $_ } + + [array]$matrix = GenerateMatrix ` + -config $config ` + -selectFromMatrixType $Selection ` + -displayNameFilter $DisplayNameFilter ` + -filters $Filters ` + -replace $Replace + + return ,$matrix +} + +# calculate general targeting information and create our batches prior to generating any matrix +$packageProperties = Get-ChildItem -Recurse "$PackagePropertiesFolder" *.json ` + | % { Get-Content -Path $_.FullName | ConvertFrom-Json } + +# in the full proto, we will compress the CIMatrixConfig value into a single string, then use it as a key +# to group the packages +# then we will simply iterate over the groups and generate the matrix for each group +# at the very end, will distribute the packages to the matrix based on the group key (batched over course) +$configs = @( + [PsCustomObject]@{ + ConfigPath = $PRMatrixFile + Selection = "sparse" + } +) + +$OverallResult = @() +# this prototype doesn't handle direct and indirect, it just batches for simplicity of the proto +foreach($matrixConfig in $configs) { + Write-Host "Generating config for $($matrixConfig.ConfigPath)" + $generatedMatrix = GenerateMatrixForConfig -ConfigPath $matrixConfig.ConfigPath -Selection $matrixConfig.Selection -DisplayNameFilter $DisplayNameFilter -Filters $Filters -Replace $Replace + + # also note here that we should be using a filtered set of package properties + # (remember we figured out which matrix config is associated with each package) + # $batches = Split-ArrayIntoBatches -InputArray $packageProperties -BatchSize $BATCHSIZE + $batches = @() + $everything = ($packageProperties | ForEach-Object { $_.ArtifactName }) + $batches += ,$everything + + foreach($batch in $batches) { + $ModifiedMatrix = @() + # to understand this iteration, one must understand that the matrix is a list of hashtables, each with a couple keys: + # [ + # { "name": "jobname", "parameters": { matrixSetting1: matrixValue1, ...} }, + # ] + if($batches.Length -gt 1) { + throw "This script is not prepared to handle more than one batch. We will need to duplicate the input objects." + } + else { + foreach($config in $generatedMatrix) { + $namesForBatch = $batches[0] -join "," + # we just need to iterate across them, grab the parameters hashtable, and add the new key + # if there is more than one batch, we will need to add a suffix including the batch name to the job name + $config["parameters"]["$PRMatrixSetting"] = $namesForBatch + $OverallResult += $config + } + } + } +} + +$serialized = SerializePipelineMatrix $OverallResult + +Write-Output $serialized.pretty + +if ($CI) { + Write-Output "##vso[task.setVariable variable=matrix;isOutput=true]$($serialized.compressed)" +} diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 2ba1eb86557c..e1ea47a36d79 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -75,20 +75,18 @@ jobs: MatrixFilters: ${{ parameters.MatrixFilters }} MatrixReplace: ${{ parameters.MatrixReplace }} SparseCheckoutPaths: [ "sdk/", ".vscode"] - PreGenerationSteps: - - template: /eng/common/pipelines/templates/steps/save-package-properties.yml - parameters: - ServiceDirectory: ${{parameters.ServiceDirectory}} + ${{ if eq(parameters.ServiceDirectory, 'auto') }}: + EnablePRGeneration: true + PRMatrix: + Name: pr_test_base + Path: eng/pipelines/templates/stages/platform-matrix.json + Setting: ArtifactPackageNames + + PreGenerationSteps: + - template: /eng/common/pipelines/templates/steps/save-package-properties.yml + parameters: + ServiceDirectory: ${{parameters.ServiceDirectory}} - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/scripts/distribute-packages-to-matrix.ps1 - arguments: >- - -PackageInfoFolder "$(Build.ArtifactStagingDirectory)/PackageInfo" - -PlatformMatrix "${{ parameters.MatrixConfigs[0].Path }}" - displayName: 'Distribute Packages to Matrix' - condition: and(eq(variables['Build.Reason'], 'PullRequest'), eq('${{ parameters.ServiceDirectory }}','auto')) CloudConfig: Cloud: Public AdditionalParameters: diff --git a/eng/scripts/distribute-packages-to-matrix.ps1 b/eng/scripts/distribute-packages-to-matrix.ps1 index b43ed6134204..a74b02cc1cee 100644 --- a/eng/scripts/distribute-packages-to-matrix.ps1 +++ b/eng/scripts/distribute-packages-to-matrix.ps1 @@ -1,76 +1,25 @@ <# .SYNOPSIS -Distributes a set of packages to a platform matrix file by adding computing and adding a ArtifactPackageNames property -dynamically. If an unexpected situation is encountered, the script will make no changes to the input file. - -.DESCRIPTION -Because of the way the platform matrix file is structured, we need to distribute the packages in a way that -honors the Include packages. We have these included this way because want to ensure that these node versions -run on that platform. This script is aware of these additional configurations and will set ArtifactPackageNames for -them as well. +Used to update a generated matrix with targeted information for packages that are being built. #> +[CmdletBinding()] param( [parameter(Mandatory=$true)] - [string]$PackageInfoFolder, - [parameter(Mandatory=$true)] - [string]$PlatformMatrix + [array]$Matrix ) $RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot ".." "..")) -Set-StrictMode -Version 4 $BATCHSIZE = 10 -if (!(Test-Path $PackageInfoFolder)) { - Write-Error "PackageInfo folder file not found: $PackageInfoFolder" - exit 1 -} - -if (!(Test-Path $PlatformMatrix)) { - Write-Error "Platform matrix file not found: $PlatformMatrix" +if (!$Matrix) { + Write-Error "Matrix input not found: $Matrix" exit 1 } -function Extract-MatrixMultiplier { - param ( - [Parameter(Mandatory=$true)] - [PSCustomObject]$Matrix - ) - - $highestCount = 1 - - foreach ($property in $Matrix.PSObject.Properties) { - $type = $property.Value.GetType().Name - switch ($type) { - "PSCustomObject" { - $itemCount = 0 - - # this looks very strange to loop over the properties of a PSCustomObject, - # but this is the only way to actually count. The Count/Length property is - # NOT available - foreach($innerProperty in $property.Value.PSObject.Properties) { - $itemCount++ - } - - if ($itemCount -gt $highestCount) { - $highestCount = $itemCount - } - } - "Object[]" { - $count = $property.Value.Length - if ($count -gt $highestCount) { - $highestCount = $count - } - } - } - } - - return $highestCount -} - function Split-ArrayIntoBatches { param ( [Parameter(Mandatory=$true)] - [object[]]$InputArray, + [string[]]$InputArray, [Parameter(Mandatory=$true)] [int]$BatchSize @@ -87,147 +36,24 @@ function Split-ArrayIntoBatches { return ,$batches } -function Update-Include { - param ( - [Parameter(Mandatory=$true)] - $Matrix, - [Parameter(Mandatory=$true)] - $IncludeConfig, - [Parameter(Mandatory=$true)] - $TargetingString - ) - - foreach ($configElement in $IncludeConfig) { - if (-not $configElement.PSObject.Properties["ArtifactPackageNames"]) { - $configElement | Add-Member -Force -MemberType NoteProperty -Name ArtifactPackageNames -Value "" - } - $configElement.ArtifactPackageNames = $TargetingString - $Matrix.include += $configElement - } -} - -function Update-Matrix { - param ( - [Parameter(Mandatory=$true)] - $Matrix, - - [Parameter(Mandatory=$true)] - $DirectBatches, - - [Parameter(Mandatory=$true)] - $IndirectBatches, - - [Parameter(Mandatory=$true)] - $MatrixMultiplier, - - [Parameter(Mandatory=$true)] - $IncludeCount, - - [Parameter(Mandatory=$false)] - $OriginalIncludeObject - ) - - $matrixUpdate = $true - if ($Matrix.matrix.PSObject.Properties["`$IMPORT"]) { - $matrixUpdate = $false - } - - # we need to ensure the presence of TargetingString in the matrix object - if ($matrixUpdate) { - if ($directBatches -or $indirectBatches) { - if (-not $Matrix.matrix.PSObject.Properties["ArtifactPackageNames"]) { - $Matrix.matrix | Add-Member -Force -MemberType NoteProperty -Name ArtifactPackageNames -Value @() - } - else { - $Matrix.matrix.ArtifactPackageNames = @() - } - } - } - - # a basic platform matrix has the following: - # "matrix" -> "NodeVersion" = [] - # "include" -> "ConfigName" -> NodeVersion=18.X - # so what we need to do is for each batch - # 1. assign the batch * matrix size to the matrix as "ArtifactPackagesNames", this will evenly distribute the packages to EACH node version non-sparsely - # 2. assign the batch to each include with the batch being the value for ArtifactPackageNames for the include - # the easiest way to do this is to take a copy of the `Include` object at the beginning of this script, - # for each batch, assign the batch to the include object, and then add the include object to the matrix object - # this will have the result of multiplexing our includes, so we will need to update the name there as well - foreach($batch in $DirectBatches) { - $targetingString = ($batch | Select-Object -ExpandProperty ArtifactName) -join "," - - # we need to equal the largest multiplier in the matrix. That is USUALLY nodeVersion version, - # but in some cases it could be something else (like sdk/cosmos/cosmos-emulator-matrix.json) the multiplier - # is a different property. We do this because we want to ensure that the matrix is fully covered, even if - # we are generating the matrix sparsely, we still want full coverage of these targetingStrings - $targetingStringArray = @($targetingString) * $MatrixMultiplier - - if ($matrixUpdate) { - $matrix.matrix.ArtifactPackageNames += $targetingStringArray - } - - # if there were any include objects, we need to duplicate them exactly and add the targeting string to each - # this means that the number of includes at the end of this operation will be incoming # of includes * the number of batches - if ($includeCount -gt 0) { - $includeCopy = $OriginalIncludeObject | ConvertTo-Json -Depth 100 | ConvertFrom-Json - - Update-Include -Matrix $matrix -IncludeConfig $includeCopy -TargetingString $targetingString - } - } - - foreach($batch in $IndirectBatches) { - $targetingString = ($batch | Select-Object -ExpandProperty Name) -join "," - if ($matrixUpdate) { - $matrix.matrix.ArtifactPackageNames += @($targetingString) - } - } - -} - -# calculate general targeting information and create our batches prior to updating any matrix -$packageProperties = Get-ChildItem -Recurse "$PackageInfoFolder" *.json ` - | % { Get-Content -Path $_.FullName | ConvertFrom-Json } - -# I will assign all the direct included packages first. our goal is to get full coverage of the direct included packages -# then, for the indirect packages, we will add them as sparse ArtifactPackageNames bundles to the matrix -$batches = Split-ArrayIntoBatches -InputArray $packageProperties -BatchSize $BATCHSIZE - -$matrix = Get-Content -Path $PlatformMatrix | ConvertFrom-Json - -$matrixLocation = $PlatformMatrix +# calculate the batches, then duplicate the entire matrix if need be +$packageProperties = $env:ARTIFACTPACKAGENAMES.Split(",") +[array]$batches = Split-ArrayIntoBatches -InputArray $packageProperties -BatchSize $BATCHSIZE -# handle the case where we discover an import, we need to climb the dependency in that case -if ($matrix.matrix.PSObject.Properties["`$IMPORT"]) { - $originalMatrix = $matrix - $originalMatrixLocation = $PlatformMatrix - - # rest of the update will happen to that matrix object - $matrixLocation = (Join-Path $RepoRoot $matrix.matrix."`$IMPORT") - $matrix = Get-Content -Path $matrixLocation | ConvertFrom-Json - - # we just need to walk the include objects and update the targeting string for the batches we have before - # updating in place, then continuing on to update the actual matrix object that we imported - # there is a clear and present edge case here with nested import. We will not handle that case - - $originalInclude = $originalMatrix.include - $originalMatrix.include = @() - - # update the include objects for the original matrix that was importing - Update-Matrix -Matrix $originalMatrix -DirectBatches $directBatches -IndirectBatches @() -MatrixMultiplier 1 -IncludeCount $matrix.include.Count -OriginalIncludeObject $originalInclude - $originalMatrix | ConvertTo-Json -Depth 100 | Set-Content -Path $originalMatrixLocation +$ModifiedMatrix = @() +# to understand this iteration, one must understand that the matrix is a list of hashtables, each with a couple keys: +# [ +# { "name": "jobname", "parameters": { matrixSetting1: matrixValue1, ...} }, +# ] +if($batches.Length -gt 1) { + throw "This script is not prepared to handle more than one batch. We will need to duplicate the input objects." } - -$matrixMultiplier = Extract-MatrixMultiplier -Matrix $matrix.matrix - -$includeCount = 0 -$includeObject = $null -$originalInclude = $null -if ($matrix.PSObject.Properties.Name -contains "include") { - $includeCount = $matrix.include.Count - $originalInclude = $matrix.include - $matrix.include = @() +else { + foreach($config in $Matrix) { + # we just need to iterate across them, grab the parameters hashtable, and add the new key + # if there is more than one batch, we will need to add a suffix including the batch name to the job name + $config["parameters"]["ArtifactPackageNames"] = $batches[0] -join "," + } } -Update-Matrix -Matrix $matrix -DirectBatches $batches -IndirectBatches @() -MatrixMultiplier $matrixMultiplier -IncludeCount $includeCount -OriginalIncludeObject $originalInclude - -$matrix | ConvertTo-Json -Depth 100 | Set-Content -Path $matrixLocation +return $Matrix From ab67ac5cadff91eaf30960bb29c1e977e98d70dc Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Sun, 27 Oct 2024 17:03:51 -0700 Subject: [PATCH 02/15] update the pr trigger --- sdk/pullrequest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/pullrequest.yml b/sdk/pullrequest.yml index 951dcee8d339..a66d6680b60c 100644 --- a/sdk/pullrequest.yml +++ b/sdk/pullrequest.yml @@ -6,7 +6,7 @@ pr: - hotfix/* - release/* - restapi* - - pipelinev3* + - pipelinev* paths: include: - "*" From 061f80f010523d26ee8947cad34bab37436f8948 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Sun, 27 Oct 2024 17:12:46 -0700 Subject: [PATCH 03/15] correct a comment --- eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 index 003a34faa341..72607a2273f5 100644 --- a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 +++ b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 @@ -52,6 +52,7 @@ function GenerateMatrixForConfig { } # calculate general targeting information and create our batches prior to generating any matrix +# this prototype doesn't handle direct and indirect, it just batches for simplicity of the proto $packageProperties = Get-ChildItem -Recurse "$PackagePropertiesFolder" *.json ` | % { Get-Content -Path $_.FullName | ConvertFrom-Json } @@ -67,7 +68,6 @@ $configs = @( ) $OverallResult = @() -# this prototype doesn't handle direct and indirect, it just batches for simplicity of the proto foreach($matrixConfig in $configs) { Write-Host "Generating config for $($matrixConfig.ConfigPath)" $generatedMatrix = GenerateMatrixForConfig -ConfigPath $matrixConfig.ConfigPath -Selection $matrixConfig.Selection -DisplayNameFilter $DisplayNameFilter -Filters $Filters -Replace $Replace From 839b444c4778cacb2526255e91672a99ac209f30 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Sun, 27 Oct 2024 19:21:17 -0700 Subject: [PATCH 04/15] disable the protection to only enable on pr. I want to test this crap and no matter what I do AZDO is refusing to queue the build --- .../pipelines/templates/steps/save-package-properties.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/pipelines/templates/steps/save-package-properties.yml b/eng/common/pipelines/templates/steps/save-package-properties.yml index 3714c0264388..4f0ad20d2fed 100644 --- a/eng/common/pipelines/templates/steps/save-package-properties.yml +++ b/eng/common/pipelines/templates/steps/save-package-properties.yml @@ -22,7 +22,7 @@ steps: # The other public CI builds will pass a real service directory, which will not activate the PR diff logic and as such will operate # as before this change. - - ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.ServiceDirectory, 'auto')) }}: + - ${{ if eq(parameters.ServiceDirectory, 'auto') }}: - task: Powershell@2 displayName: Generate PR Diff inputs: From f24289f5141277877ee26c212bc1c515b42a933d Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Sun, 27 Oct 2024 19:42:02 -0700 Subject: [PATCH 05/15] add newline --- eng/pipelines/templates/jobs/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index e1ea47a36d79..1e4f6d68e6a1 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -77,6 +77,7 @@ jobs: SparseCheckoutPaths: [ "sdk/", ".vscode"] ${{ if eq(parameters.ServiceDirectory, 'auto') }}: EnablePRGeneration: true + PRMatrix: Name: pr_test_base Path: eng/pipelines/templates/stages/platform-matrix.json From ec8288c628e78345aef188c01971d398c66598e7 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 29 Oct 2024 11:28:08 -0700 Subject: [PATCH 06/15] group by random key now working. --- .../scripts/Helpers/Package-Helpers.ps1 | 115 ++++++++++++++---- .../scripts/job-matrix/Create-PrJobMatrix.ps1 | 70 ++++++----- .../job-matrix/job-matrix-functions.ps1 | 22 ++++ 3 files changed, 154 insertions(+), 53 deletions(-) diff --git a/eng/common/scripts/Helpers/Package-Helpers.ps1 b/eng/common/scripts/Helpers/Package-Helpers.ps1 index bb492cc6f109..51f2434e141a 100644 --- a/eng/common/scripts/Helpers/Package-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Package-Helpers.ps1 @@ -68,7 +68,7 @@ Get-Content -Raw path/to/file.yml | CompatibleConvertFrom-Yaml #> function CompatibleConvertFrom-Yaml { param( - [Parameter(Mandatory=$true, ValueFromPipeline=$true)] + [Parameter(Mandatory = $true, ValueFromPipeline = $true)] [string]$Content ) @@ -85,10 +85,10 @@ function CompatibleConvertFrom-Yaml { # Process the content (for example, you could convert from YAML here) if ($yqPresent) { - return ($content | yq -o=json | ConvertFrom-Json -AsHashTable) + return ($content | yq -o=json | ConvertFrom-Json -AsHashTable) } else { - return ConvertFrom-Yaml $content + return ConvertFrom-Yaml $content } } @@ -114,7 +114,7 @@ LoadFrom-Yaml -YmlFile path/to/file.yml #> function LoadFrom-Yaml { param( - [Parameter(Mandatory=$true)] + [Parameter(Mandatory = $true)] [string]$YmlFile ) if (Test-Path -Path $YmlFile) { @@ -159,41 +159,106 @@ GetValueSafelyFrom-Yaml -YamlContentAsHashtable $YmlFileContent -Keys @("extends #> function GetValueSafelyFrom-Yaml { param( - [Parameter(Mandatory=$true)] + [Parameter(Mandatory = $true)] $YamlContentAsHashtable, - [Parameter(Mandatory=$true)] + [Parameter(Mandatory = $true)] [string[]]$Keys ) $current = $YamlContentAsHashtable foreach ($key in $Keys) { - if ($current.ContainsKey($key) -or $current[$key]) { - $current = $current[$key] - } - else { - Write-Host "The '$key' part of the path $($Keys -join "/") doesn't exist or is null." - return $null - } + if ($current.ContainsKey($key) -or $current[$key]) { + $current = $current[$key] + } + else { + return $null + } } return [object]$current } -function Split-ArrayIntoBatches { - param ( - [Parameter(Mandatory=$true)] - [string[]]$InputArray, +function Get-ObjectKey { + param ( + [Parameter(Mandatory = $true)] + [object]$Object + ) + + Write-Host "In Get-ObjectKey with $Object" + + if (-not $Object) { + return "unset" + } + + if ($Object -is [hashtable] -or $Object -is [System.Collections.Specialized.OrderedDictionary]) { + $sortedEntries = $Object.GetEnumerator() | Sort-Object Name + $hashString = ($sortedEntries | ForEach-Object { "$($_.Key)=$($_.Value)" }) -join ";" + return $hashString.GetHashCode() + } + + elseif ($Object -is [PSCustomObject]) { + $sortedProperties = $Object.PSObject.Properties | Sort-Object Name + $propertyString = ($sortedProperties | ForEach-Object { "$($_.Name)=$($_.Value)" }) -join ";" + return $propertyString.GetHashCode() + } + + elseif ($Object -is [array]) { + $arrayString = ($Object | ForEach-Object { Get-ObjectKey $_ }) -join "," + return $arrayString.GetHashCode() + } + + else { + return $Object.ToString() + } +} + +function Group-ByObjectKey { + param ( + [Parameter(Mandatory)] + [array]$Items, - [Parameter(Mandatory=$true)] - [int]$BatchSize - ) + [Parameter(Mandatory)] + [string]$GroupByProperty + ) - $batches = @() + $groupedDictionary = @{} - for ($i = 0; $i -lt $InputArray.Count; $i += $BatchSize) { - $batch = $InputArray[$i..[math]::Min($i + $BatchSize - 1, $InputArray.Count - 1)] + foreach ($item in $Items) { + Write-Host "Getting object key for $($item.Name)" + $key = Get-ObjectKey $item."$GroupByProperty" + + if ($key) { + if (-not $groupedDictionary.ContainsKey($key)) { + $groupedDictionary[$key] = @() + } - $batches += ,$batch + # Add the current item to the array for this key + $groupedDictionary[$key] += $item } + else { + Write-Host "Apparently don't have a key for $($item.Name)" + } + } + + return $groupedDictionary +} + + +function Split-ArrayIntoBatches { + param ( + [Parameter(Mandatory = $true)] + [string[]]$InputArray, + + [Parameter(Mandatory = $true)] + [int]$BatchSize + ) + + $batches = @() + + for ($i = 0; $i -lt $InputArray.Count; $i += $BatchSize) { + $batch = $InputArray[$i..[math]::Min($i + $BatchSize - 1, $InputArray.Count - 1)] + + $batches += , $batch + } - return ,$batches + return , $batches } diff --git a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 index 72607a2273f5..9772953266c2 100644 --- a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 +++ b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 @@ -28,44 +28,58 @@ if (!(Test-Path $PackagePropertiesFolder)) { exit 1 } -function GenerateMatrixForConfig { - param ( - [Parameter(Mandatory=$true)][string] $ConfigPath, - [Parameter(Mandatory=$True)][string] $Selection, - [Parameter(Mandatory=$false)][string] $DisplayNameFilter, - [Parameter(Mandatory=$false)][array] $Filters, - [Parameter(Mandatory=$false)][array] $Replace - ) - - $config = GetMatrixConfigFromFile (Get-Content $ConfigPath -Raw) - # Strip empty string filters in order to be able to use azure pipelines yaml join() - $Filters = $Filters | Where-Object { $_ } - - [array]$matrix = GenerateMatrix ` - -config $config ` - -selectFromMatrixType $Selection ` - -displayNameFilter $DisplayNameFilter ` - -filters $Filters ` - -replace $Replace - - return ,$matrix -} +# create batching by grouping packages based on the matrix config +Write-Host "Generating PR job matrix for $PackagePropertiesFolder" + +# this will become a parameter in the future, we will pass MatrixConfigs as a paremeter +# these will be the "default" matrices that we will use to generate the PR matrix in lieu of +# a matrix file +$configs = @( + [PsCustomObject]@{ + Name = "default_platform_matrix" + ConfigPath = $PRMatrixFile + Selection = "sparse" + } +) + # calculate general targeting information and create our batches prior to generating any matrix # this prototype doesn't handle direct and indirect, it just batches for simplicity of the proto $packageProperties = Get-ChildItem -Recurse "$PackagePropertiesFolder" *.json ` | % { Get-Content -Path $_.FullName | ConvertFrom-Json } +$packageProperties | ForEach-Object { + if (-not $_.CIMatrixConfigs) { + $_.CIMatrixConfigs = $configs + } +} + +foreach ($package in $packageProperties) { + Write-Host "$($package.ArtifactName) has the following matrix configs: $($package.CIMatrixConfigs)" +} + +# this will group the packages, but the values of the grouping itself won't be super useful given how they were +# constructed +$batchesByConfig = Group-ByObjectKey $packageProperties "CIMatrixConfigs" + +foreach($key in $batchesByConfig.Keys) { + Write-Host $key + foreach($package in $batchesByConfig[$key]) { + Write-Host $package.ArtifactName + } +} + +exit 0 + +# the key here is that after we group the packages by the matrix config objects, we can use the first item's MatrixConfig +# to generate the matrix for the group, no reason to have to parse the key value backwards to get the matrix config + + # in the full proto, we will compress the CIMatrixConfig value into a single string, then use it as a key # to group the packages # then we will simply iterate over the groups and generate the matrix for each group # at the very end, will distribute the packages to the matrix based on the group key (batched over course) -$configs = @( - [PsCustomObject]@{ - ConfigPath = $PRMatrixFile - Selection = "sparse" - } -) + $OverallResult = @() foreach($matrixConfig in $configs) { diff --git a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 index f20dbe5281b0..0c175408fc48 100644 --- a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 +++ b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 @@ -740,3 +740,25 @@ function Get4dMatrixIndex([int]$index, [Array]$dimensions) { return @($page3, $page2, $page1, $remainder) } +function GenerateMatrixForConfig { + param ( + [Parameter(Mandatory=$true)][string] $ConfigPath, + [Parameter(Mandatory=$True)][string] $Selection, + [Parameter(Mandatory=$false)][string] $DisplayNameFilter, + [Parameter(Mandatory=$false)][array] $Filters, + [Parameter(Mandatory=$false)][array] $Replace + ) + + $config = GetMatrixConfigFromFile (Get-Content $ConfigPath -Raw) + # Strip empty string filters in order to be able to use azure pipelines yaml join() + $Filters = $Filters | Where-Object { $_ } + + [array]$matrix = GenerateMatrix ` + -config $config ` + -selectFromMatrixType $Selection ` + -displayNameFilter $DisplayNameFilter ` + -filters $Filters ` + -replace $Replace + + return ,$matrix +} From 29c12409d4404bd25a613bbff009de978d7779a7 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 29 Oct 2024 13:48:14 -0700 Subject: [PATCH 07/15] new broken methodology. I somehow broke GenerateSparseMatrix and it makes 0 sense that it's returning an error. I hate powershell so much --- .../scripts/job-matrix/Create-PrJobMatrix.ps1 | 143 +++++++++--------- .../job-matrix/job-matrix-functions.ps1 | 23 --- 2 files changed, 73 insertions(+), 93 deletions(-) diff --git a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 index 9772953266c2..dd477851eca6 100644 --- a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 +++ b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 @@ -11,106 +11,109 @@ This script is intended to be used within an Azure DevOps pipeline to generate a [CmdletBinding()] param ( - [Parameter(Mandatory=$true)][string] $PackagePropertiesFolder, - [Parameter(Mandatory=$true)][string] $PRMatrixFile, - [Parameter(Mandatory=$true)][string] $PRMatrixSetting, - [Parameter(Mandatory=$False)][string] $DisplayNameFilter, - [Parameter(Mandatory=$False)][array] $Filters, - [Parameter(Mandatory=$False)][array] $Replace, - [Parameter()][switch] $CI = ($null -ne $env:SYSTEM_TEAMPROJECTID) + [Parameter(Mandatory = $true)][string] $PackagePropertiesFolder, + [Parameter(Mandatory = $true)][string] $PRMatrixFile, + [Parameter(Mandatory = $true)][string] $PRMatrixSetting, + [Parameter(Mandatory = $False)][string] $DisplayNameFilter, + [Parameter(Mandatory = $False)][array] $Filters, + [Parameter(Mandatory = $False)][array] $Replace, + [Parameter()][switch] $CI = ($null -ne $env:SYSTEM_TEAMPROJECTID) ) . $PSScriptRoot/job-matrix-functions.ps1 . $PSScriptRoot/../Helpers/Package-Helpers.ps1 +$BATCHSIZE = 10 + +function GenerateMatrixForConfig { + param ( + [Parameter(Mandatory = $true)][string] $ConfigPath, + [Parameter(Mandatory = $True)][string] $Selection, + [Parameter(Mandatory = $false)][string] $DisplayNameFilter, + [Parameter(Mandatory = $false)][array] $Filters, + [Parameter(Mandatory = $false)][array] $Replace + ) + + $config = GetMatrixConfigFromFile (Get-Content $ConfigPath -Raw) + # Strip empty string filters in order to be able to use azure pipelines yaml join() + $Filters = $Filters | Where-Object { $_ } + + [array]$matrix = GenerateMatrix ` + -config $config ` + -selectFromMatrixType $Selection ` + -displayNameFilter $DisplayNameFilter ` + -filters $Filters ` + -replace $Replace + + return , $matrix +} + if (!(Test-Path $PackagePropertiesFolder)) { - Write-Error "Package Properties folder doesn't exist" - exit 1 + Write-Error "Package Properties folder doesn't exist" + exit 1 } -# create batching by grouping packages based on the matrix config Write-Host "Generating PR job matrix for $PackagePropertiesFolder" -# this will become a parameter in the future, we will pass MatrixConfigs as a paremeter +# this will become a parameter in the future, we will pass MatrixConfigs as a parameter # these will be the "default" matrices that we will use to generate the PR matrix in lieu of # a matrix file $configs = @( [PsCustomObject]@{ - Name = "default_platform_matrix" - ConfigPath = $PRMatrixFile + Name = "default_platform_matrix" + Path = $PRMatrixFile Selection = "sparse" } ) - # calculate general targeting information and create our batches prior to generating any matrix # this prototype doesn't handle direct and indirect, it just batches for simplicity of the proto $packageProperties = Get-ChildItem -Recurse "$PackagePropertiesFolder" *.json ` - | % { Get-Content -Path $_.FullName | ConvertFrom-Json } +| ForEach-Object { Get-Content -Path $_.FullName | ConvertFrom-Json } +# set default matrix config for each package if there isn't an override $packageProperties | ForEach-Object { if (-not $_.CIMatrixConfigs) { $_.CIMatrixConfigs = $configs } } -foreach ($package in $packageProperties) { - Write-Host "$($package.ArtifactName) has the following matrix configs: $($package.CIMatrixConfigs)" -} +# The key here is that after we group the packages by the matrix config objects, we can use the first item's MatrixConfig +# to generate the matrix for the group, no reason to have to parse the key value backwards to get the matrix config. +$matrixBatchesByConfig = Group-ByObjectKey $packageProperties "CIMatrixConfigs" -# this will group the packages, but the values of the grouping itself won't be super useful given how they were -# constructed -$batchesByConfig = Group-ByObjectKey $packageProperties "CIMatrixConfigs" - -foreach($key in $batchesByConfig.Keys) { - Write-Host $key - foreach($package in $batchesByConfig[$key]) { - Write-Host $package.ArtifactName +$OverallResult = @() +foreach ($matrixBatchKey in $matrixBatchesByConfig.Keys) { + $matrixBatch = $matrixBatchesByConfig[$matrixBatchKey] + $matrixConfig = $matrixBatch | Select-Object -First 1 -ExpandProperty CIMatrixConfigs + Write-Host "Generating config for $($matrixConfig.Path)" + + $results = @() + foreach ($matrixConfig in $matrixConfigs) { + $results = GenerateMatrixForConfig -ConfigPath $matrixConfig.Path -Selection $matrixConfig.Selection -DisplayNameFilter $DisplayNameFilter -Filters $Filters -Replace $Replace } -} -exit 0 - -# the key here is that after we group the packages by the matrix config objects, we can use the first item's MatrixConfig -# to generate the matrix for the group, no reason to have to parse the key value backwards to get the matrix config - - -# in the full proto, we will compress the CIMatrixConfig value into a single string, then use it as a key -# to group the packages -# then we will simply iterate over the groups and generate the matrix for each group -# at the very end, will distribute the packages to the matrix based on the group key (batched over course) - - -$OverallResult = @() -foreach($matrixConfig in $configs) { - Write-Host "Generating config for $($matrixConfig.ConfigPath)" - $generatedMatrix = GenerateMatrixForConfig -ConfigPath $matrixConfig.ConfigPath -Selection $matrixConfig.Selection -DisplayNameFilter $DisplayNameFilter -Filters $Filters -Replace $Replace - - # also note here that we should be using a filtered set of package properties - # (remember we figured out which matrix config is associated with each package) - # $batches = Split-ArrayIntoBatches -InputArray $packageProperties -BatchSize $BATCHSIZE - $batches = @() - $everything = ($packageProperties | ForEach-Object { $_.ArtifactName }) - $batches += ,$everything - - foreach($batch in $batches) { - $ModifiedMatrix = @() - # to understand this iteration, one must understand that the matrix is a list of hashtables, each with a couple keys: - # [ - # { "name": "jobname", "parameters": { matrixSetting1: matrixValue1, ...} }, - # ] - if($batches.Length -gt 1) { - throw "This script is not prepared to handle more than one batch. We will need to duplicate the input objects." - } - else { - foreach($config in $generatedMatrix) { - $namesForBatch = $batches[0] -join "," - # we just need to iterate across them, grab the parameters hashtable, and add the new key - # if there is more than one batch, we will need to add a suffix including the batch name to the job name - $config["parameters"]["$PRMatrixSetting"] = $namesForBatch - $OverallResult += $config - } - } + $packageBatches = Split-ArrayIntoBatches -InputArray $results -BatchSize $BATCHSIZE + + foreach ($batch in $packageBatches) { + Write-Host "Ok walking through $batch" + # $ModifiedMatrix = @() + # # to understand this iteration, one must understand that the matrix is a list of hashtables, each with a couple keys: + # # [ + # # { "name": "jobname", "parameters": { matrixSetting1: matrixValue1, ...} }, + # # ] + # if ($batches.Length -gt 1) { + # throw "This script is not prepared to handle more than one batch. We will need to duplicate the input objects." + # } + # else { + # foreach ($config in $generatedMatrix) { + # $namesForBatch = $batches[0] -join "," + # # we just need to iterate across them, grab the parameters hashtable, and add the new key + # # if there is more than one batch, we will need to add a suffix including the batch name to the job name + # $config["parameters"]["$PRMatrixSetting"] = $namesForBatch + # $OverallResult += $config + # } + # } } } @@ -119,5 +122,5 @@ $serialized = SerializePipelineMatrix $OverallResult Write-Output $serialized.pretty if ($CI) { - Write-Output "##vso[task.setVariable variable=matrix;isOutput=true]$($serialized.compressed)" + Write-Output "##vso[task.setVariable variable=matrix;isOutput=true]$($serialized.compressed)" } diff --git a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 index 0c175408fc48..6ff429e153e6 100644 --- a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 +++ b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 @@ -739,26 +739,3 @@ function Get4dMatrixIndex([int]$index, [Array]$dimensions) { return @($page3, $page2, $page1, $remainder) } - -function GenerateMatrixForConfig { - param ( - [Parameter(Mandatory=$true)][string] $ConfigPath, - [Parameter(Mandatory=$True)][string] $Selection, - [Parameter(Mandatory=$false)][string] $DisplayNameFilter, - [Parameter(Mandatory=$false)][array] $Filters, - [Parameter(Mandatory=$false)][array] $Replace - ) - - $config = GetMatrixConfigFromFile (Get-Content $ConfigPath -Raw) - # Strip empty string filters in order to be able to use azure pipelines yaml join() - $Filters = $Filters | Where-Object { $_ } - - [array]$matrix = GenerateMatrix ` - -config $config ` - -selectFromMatrixType $Selection ` - -displayNameFilter $DisplayNameFilter ` - -filters $Filters ` - -replace $Replace - - return ,$matrix -} From ebe17c6eccfe968766d8f33f4a1a18052e10c746 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 29 Oct 2024 15:02:06 -0700 Subject: [PATCH 08/15] fixing the broken assumptions --- eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 index dd477851eca6..5a7b07deef96 100644 --- a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 +++ b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 @@ -32,8 +32,11 @@ function GenerateMatrixForConfig { [Parameter(Mandatory = $false)][array] $Filters, [Parameter(Mandatory = $false)][array] $Replace ) + $matrixFile = Join-Path $PSScriptRoot ".." ".." ".." ".." $ConfigPath - $config = GetMatrixConfigFromFile (Get-Content $ConfigPath -Raw) + $resolvedMatrixFile = Resolve-Path $matrixFile + + $config = GetMatrixConfigFromFile (Get-Content $resolvedMatrixFile -Raw) # Strip empty string filters in order to be able to use azure pipelines yaml join() $Filters = $Filters | Where-Object { $_ } @@ -85,7 +88,7 @@ $matrixBatchesByConfig = Group-ByObjectKey $packageProperties "CIMatrixConfigs" $OverallResult = @() foreach ($matrixBatchKey in $matrixBatchesByConfig.Keys) { $matrixBatch = $matrixBatchesByConfig[$matrixBatchKey] - $matrixConfig = $matrixBatch | Select-Object -First 1 -ExpandProperty CIMatrixConfigs + $matrixConfigs = $matrixBatch | Select-Object -First 1 -ExpandProperty CIMatrixConfigs Write-Host "Generating config for $($matrixConfig.Path)" $results = @() From 63614b1ef46ccbc8f7325a10f4b104478d01b11e Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 29 Oct 2024 16:07:12 -0700 Subject: [PATCH 09/15] unrolling across the batches --- .../scripts/Helpers/Package-Helpers.ps1 | 2 +- .../scripts/job-matrix/Create-PrJobMatrix.ps1 | 93 +- .../job-matrix/job-matrix-functions.ps1 | 1116 +++++++++-------- 3 files changed, 611 insertions(+), 600 deletions(-) diff --git a/eng/common/scripts/Helpers/Package-Helpers.ps1 b/eng/common/scripts/Helpers/Package-Helpers.ps1 index 51f2434e141a..ac94bd997d77 100644 --- a/eng/common/scripts/Helpers/Package-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Package-Helpers.ps1 @@ -246,7 +246,7 @@ function Group-ByObjectKey { function Split-ArrayIntoBatches { param ( [Parameter(Mandatory = $true)] - [string[]]$InputArray, + [Object[]]$InputArray, [Parameter(Mandatory = $true)] [int]$BatchSize diff --git a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 index 5a7b07deef96..6accc37f0e3b 100644 --- a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 +++ b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 @@ -24,33 +24,6 @@ param ( . $PSScriptRoot/../Helpers/Package-Helpers.ps1 $BATCHSIZE = 10 -function GenerateMatrixForConfig { - param ( - [Parameter(Mandatory = $true)][string] $ConfigPath, - [Parameter(Mandatory = $True)][string] $Selection, - [Parameter(Mandatory = $false)][string] $DisplayNameFilter, - [Parameter(Mandatory = $false)][array] $Filters, - [Parameter(Mandatory = $false)][array] $Replace - ) - $matrixFile = Join-Path $PSScriptRoot ".." ".." ".." ".." $ConfigPath - - $resolvedMatrixFile = Resolve-Path $matrixFile - - $config = GetMatrixConfigFromFile (Get-Content $resolvedMatrixFile -Raw) - # Strip empty string filters in order to be able to use azure pipelines yaml join() - $Filters = $Filters | Where-Object { $_ } - - [array]$matrix = GenerateMatrix ` - -config $config ` - -selectFromMatrixType $Selection ` - -displayNameFilter $DisplayNameFilter ` - -filters $Filters ` - -replace $Replace - - return , $matrix -} - - if (!(Test-Path $PackagePropertiesFolder)) { Write-Error "Package Properties folder doesn't exist" exit 1 @@ -72,7 +45,8 @@ $configs = @( # calculate general targeting information and create our batches prior to generating any matrix # this prototype doesn't handle direct and indirect, it just batches for simplicity of the proto $packageProperties = Get-ChildItem -Recurse "$PackagePropertiesFolder" *.json ` -| ForEach-Object { Get-Content -Path $_.FullName | ConvertFrom-Json } +| ForEach-Object { Get-Content -Path $_.FullName | ConvertFrom-Json } ` +| ForEach-Object { [PSCustomObject]$_ } # set default matrix config for each package if there isn't an override $packageProperties | ForEach-Object { @@ -87,36 +61,47 @@ $matrixBatchesByConfig = Group-ByObjectKey $packageProperties "CIMatrixConfigs" $OverallResult = @() foreach ($matrixBatchKey in $matrixBatchesByConfig.Keys) { + Write-Host "Generating config for $($matrixConfig.Path)" $matrixBatch = $matrixBatchesByConfig[$matrixBatchKey] $matrixConfigs = $matrixBatch | Select-Object -First 1 -ExpandProperty CIMatrixConfigs - Write-Host "Generating config for $($matrixConfig.Path)" - $results = @() + $matrixResults = @() foreach ($matrixConfig in $matrixConfigs) { - $results = GenerateMatrixForConfig -ConfigPath $matrixConfig.Path -Selection $matrixConfig.Selection -DisplayNameFilter $DisplayNameFilter -Filters $Filters -Replace $Replace - } - - $packageBatches = Split-ArrayIntoBatches -InputArray $results -BatchSize $BATCHSIZE - - foreach ($batch in $packageBatches) { - Write-Host "Ok walking through $batch" - # $ModifiedMatrix = @() - # # to understand this iteration, one must understand that the matrix is a list of hashtables, each with a couple keys: - # # [ - # # { "name": "jobname", "parameters": { matrixSetting1: matrixValue1, ...} }, - # # ] - # if ($batches.Length -gt 1) { - # throw "This script is not prepared to handle more than one batch. We will need to duplicate the input objects." - # } - # else { - # foreach ($config in $generatedMatrix) { - # $namesForBatch = $batches[0] -join "," - # # we just need to iterate across them, grab the parameters hashtable, and add the new key - # # if there is more than one batch, we will need to add a suffix including the batch name to the job name - # $config["parameters"]["$PRMatrixSetting"] = $namesForBatch - # $OverallResult += $config - # } - # } + $matrixResults = GenerateMatrixForConfig ` + -ConfigPath $matrixConfig.Path ` + -Selection $matrixConfig.Selection ` + -DisplayNameFilter $DisplayNameFilter ` + -Filters $Filters ` + -Replace $Replace + + $packageBatches = Split-ArrayIntoBatches -InputArray $matrixBatch -BatchSize $BATCHSIZE + + # we only need to modify the generated job name if there is more than one matrix config or batch in the matrix + $matrixSuffixNecessary = $matrixConfigs.Count -gt 1 + $batchSuffixNecessary = $packageBatches.Length -gt 1 + + foreach ($batch in $packageBatches) { + # to understand this iteration, one must understand that the matrix is a list of hashtables, each with a couple keys: + # [ + # { "name": "jobname", "parameters": { matrixSetting1: matrixValue1, ...} }, + # ] + foreach ($matrixOutputItem in $matrixResults) { + $namesForBatch = ($batch | ForEach-Object { $_.ArtifactName }) -join "-" + # we just need to iterate across them, grab the parameters hashtable, and add the new key + # if there is more than one batch, we will need to add a suffix including the batch name to the job name + $matrixOutputItem["parameters"]["$PRMatrixSetting"] = $namesForBatch + + if ($matrixSuffixNecessary) { + $matrixOutputItem["name"] = $matrixOutputItem["name"] + $matrixConfig.Name + } + + if ($batchSuffixNecessary) { + $matrixOutputItem["name"] = $matrixOutputItem["name"] + $namesForBatch + } + + $OverallResult += $matrixOutputItem + } + } } } diff --git a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 index 6ff429e153e6..64b5144392c7 100644 --- a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 +++ b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 @@ -1,718 +1,718 @@ Set-StrictMode -Version "4.0" class MatrixConfig { - [PSCustomObject]$displayNames - [Hashtable]$displayNamesLookup - [PSCustomObject]$matrix - [MatrixParameter[]]$matrixParameters - [Array]$include - [Array]$exclude + [PSCustomObject]$displayNames + [Hashtable]$displayNamesLookup + [PSCustomObject]$matrix + [MatrixParameter[]]$matrixParameters + [Array]$include + [Array]$exclude } class MatrixParameter { - MatrixParameter([String]$name, [System.Object]$value) { - $this.Value = $value - $this.Name = $name - } - - [System.Object]$Value - [System.Object]$Name - - Set($value, [String]$keyRegex = '') { - if ($this.Value -is [PSCustomObject]) { - $set = $false - foreach ($prop in $this.Value.PSObject.Properties) { - if ($prop.Name -match $keyRegex) { - $prop.Value = $value - $set = $true - break - } - } - if (!$set) { - throw "Property `"$keyRegex`" does not exist for MatrixParameter." - } - } - else { - $this.Value = $value - } + MatrixParameter([String]$name, [System.Object]$value) { + $this.Value = $value + $this.Name = $name + } + + [System.Object]$Value + [System.Object]$Name + + Set($value, [String]$keyRegex = '') { + if ($this.Value -is [PSCustomObject]) { + $set = $false + foreach ($prop in $this.Value.PSObject.Properties) { + if ($prop.Name -match $keyRegex) { + $prop.Value = $value + $set = $true + break + } + } + if (!$set) { + throw "Property `"$keyRegex`" does not exist for MatrixParameter." + } } - - [System.Object]Flatten() { - if ($this.Value -is [PSCustomObject]) { - return $this.Value.PSObject.Properties | ForEach-Object { - [MatrixParameter]::new($_.Name, $_.Value) - } - } - elseif ($this.Value -is [Array]) { - return $this.Value | ForEach-Object { - [MatrixParameter]::new($this.Name, $_) - } - } - else { - return $this - } + else { + $this.Value = $value } + } - [Int]Length() { - if ($this.Value -is [PSCustomObject]) { - return ($this.Value.PSObject.Properties | Measure-Object).Count - } - elseif ($this.Value -is [Array]) { - return $this.Value.Length - } - else { - return 1 - } + [System.Object]Flatten() { + if ($this.Value -is [PSCustomObject]) { + return $this.Value.PSObject.Properties | ForEach-Object { + [MatrixParameter]::new($_.Name, $_.Value) + } } - - [String]CreateDisplayName([Hashtable]$displayNamesLookup) { - if ($null -eq $this.Value) { - $displayName = "" - } - elseif ($this.Value -is [PSCustomObject]) { - $displayName = $this.Name - } - else { - $displayName = $this.Value.ToString() - } - - if ($displayNamesLookup -and $displayNamesLookup.ContainsKey($displayName)) { - $displayName = $displayNamesLookup[$displayName] - } - - # Matrix naming restrictions: - # https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#multi-job-configuration - $displayName = $displayName -replace "[^A-Za-z0-9_]", "" - return $displayName + elseif ($this.Value -is [Array]) { + return $this.Value | ForEach-Object { + [MatrixParameter]::new($this.Name, $_) + } } -} - -. (Join-Path $PSScriptRoot "../Helpers" PSModule-Helpers.ps1) -$IMPORT_KEYWORD = '$IMPORT' + else { + return $this + } + } -function GenerateMatrix( - [MatrixConfig]$config, - [String]$selectFromMatrixType, - [String]$displayNameFilter = ".*", - [Array]$filters = @(), - [Array]$replace = @(), - [Array]$nonSparseParameters = @(), - [Switch]$skipEnvironmentVariables -) { - $matrixParameters, $importedMatrix, $combinedDisplayNameLookup = ` - ProcessImport $config.matrixParameters $selectFromMatrixType $nonSparseParameters $config.displayNamesLookup - if ($selectFromMatrixType -eq "sparse") { - $matrix = GenerateSparseMatrix $matrixParameters $config.displayNamesLookup $nonSparseParameters + [Int]Length() { + if ($this.Value -is [PSCustomObject]) { + return ($this.Value.PSObject.Properties | Measure-Object).Count } - elseif ($selectFromMatrixType -eq "all") { - $matrix = GenerateFullMatrix $matrixParameters $config.displayNamesLookup + elseif ($this.Value -is [Array]) { + return $this.Value.Length } else { - throw "Matrix generator not implemented for selectFromMatrixType: '$selectFromMatrixType'" + return 1 } + } - # Combine with imported after matrix generation, since a sparse selection should result in a full combination of the - # top level and imported sparse matrices (as opposed to a sparse selection of both matrices). - if ($importedMatrix) { - $matrix = CombineMatrices $matrix $importedMatrix $combinedDisplayNameLookup + [String]CreateDisplayName([Hashtable]$displayNamesLookup) { + if ($null -eq $this.Value) { + $displayName = "" } - if ($config.exclude) { - $matrix = ProcessExcludes $matrix $config.exclude + elseif ($this.Value -is [PSCustomObject]) { + $displayName = $this.Name } - if ($config.include) { - $matrix = ProcessIncludes $config $matrix $selectFromMatrixType + else { + $displayName = $this.Value.ToString() } - $matrix = FilterMatrix $matrix $filters - $matrix = ProcessReplace $matrix $replace $combinedDisplayNameLookup - if (!$skipEnvironmentVariables) { - $matrix = ProcessEnvironmentVariableReferences $matrix $combinedDisplayNameLookup + if ($displayNamesLookup -and $displayNamesLookup.ContainsKey($displayName)) { + $displayName = $displayNamesLookup[$displayName] } - $matrix = FilterMatrixDisplayName $matrix $displayNameFilter - return $matrix + + # Matrix naming restrictions: + # https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#multi-job-configuration + $displayName = $displayName -replace "[^A-Za-z0-9_]", "" + return $displayName + } +} + +. (Join-Path $PSScriptRoot "../Helpers" PSModule-Helpers.ps1) +$IMPORT_KEYWORD = '$IMPORT' + +function GenerateMatrix( + [MatrixConfig]$config, + [String]$selectFromMatrixType, + [String]$displayNameFilter = ".*", + [Array]$filters = @(), + [Array]$replace = @(), + [Array]$nonSparseParameters = @(), + [Switch]$skipEnvironmentVariables +) { + $matrixParameters, $importedMatrix, $combinedDisplayNameLookup = ` + ProcessImport $config.matrixParameters $selectFromMatrixType $nonSparseParameters $config.displayNamesLookup + if ($selectFromMatrixType -eq "sparse") { + $matrix = GenerateSparseMatrix $matrixParameters $config.displayNamesLookup $nonSparseParameters + } + elseif ($selectFromMatrixType -eq "all") { + $matrix = GenerateFullMatrix $matrixParameters $config.displayNamesLookup + } + else { + throw "Matrix generator not implemented for selectFromMatrixType: '$selectFromMatrixType'" + } + + # Combine with imported after matrix generation, since a sparse selection should result in a full combination of the + # top level and imported sparse matrices (as opposed to a sparse selection of both matrices). + if ($importedMatrix) { + $matrix = CombineMatrices $matrix $importedMatrix $combinedDisplayNameLookup + } + if ($config.exclude) { + $matrix = ProcessExcludes $matrix $config.exclude + } + if ($config.include) { + $matrix = ProcessIncludes $config $matrix $selectFromMatrixType + } + + $matrix = FilterMatrix $matrix $filters + $matrix = ProcessReplace $matrix $replace $combinedDisplayNameLookup + if (!$skipEnvironmentVariables) { + $matrix = ProcessEnvironmentVariableReferences $matrix $combinedDisplayNameLookup + } + $matrix = FilterMatrixDisplayName $matrix $displayNameFilter + return $matrix } function ProcessNonSparseParameters( - [MatrixParameter[]]$parameters, - [Array]$nonSparseParameters + [MatrixParameter[]]$parameters, + [Array]$nonSparseParameters ) { - if (!$nonSparseParameters) { - return $parameters, $null - } + if (!$nonSparseParameters) { + return $parameters, $null + } - $sparse = [MatrixParameter[]]@() - $nonSparse = [MatrixParameter[]]@() + $sparse = [MatrixParameter[]]@() + $nonSparse = [MatrixParameter[]]@() - foreach ($param in $parameters) { - if ($param.Name -in $nonSparseParameters) { - $nonSparse += $param - } - else { - $sparse += $param - } + foreach ($param in $parameters) { + if ($param.Name -in $nonSparseParameters) { + $nonSparse += $param } + else { + $sparse += $param + } + } - return $sparse, $nonSparse + return $sparse, $nonSparse } function FilterMatrixDisplayName([array]$matrix, [string]$filter) { - return $matrix | Where-Object { $_ } | ForEach-Object { - if ($_.ContainsKey("Name") -and $_.Name -match $filter) { - return $_ - } + return $matrix | Where-Object { $_ } | ForEach-Object { + if ($_.ContainsKey("Name") -and $_.Name -match $filter) { + return $_ } + } } # Filters take the format of key=valueregex,key2=valueregex2 function FilterMatrix([array]$matrix, [array]$filters) { - $matrix = $matrix | ForEach-Object { - if (MatchesFilters $_ $filters) { - return $_ - } + $matrix = $matrix | ForEach-Object { + if (MatchesFilters $_ $filters) { + return $_ } - return $matrix + } + return $matrix } function MatchesFilters([hashtable]$entry, [array]$filters) { - foreach ($filter in $filters) { - $key, $regex = ParseFilter $filter - # Default all regex checks to go against empty string when keys are missing. - # This simplifies the filter syntax/interface to be regex only. - $value = "" - if ($null -ne $entry -and $entry.ContainsKey("parameters") -and $entry.parameters.Contains($key)) { - $value = $entry.parameters[$key] - } - if ($value -notmatch $regex) { - return $false - } + foreach ($filter in $filters) { + $key, $regex = ParseFilter $filter + # Default all regex checks to go against empty string when keys are missing. + # This simplifies the filter syntax/interface to be regex only. + $value = "" + if ($null -ne $entry -and $entry.ContainsKey("parameters") -and $entry.parameters.Contains($key)) { + $value = $entry.parameters[$key] } + if ($value -notmatch $regex) { + return $false + } + } - return $true + return $true } function ParseFilter([string]$filter) { - # Lazy match key in case value contains '=' - if ($filter -match "(.+?)=(.+)") { - $key = $matches[1] - $regex = $matches[2] - return $key, $regex - } - else { - throw "Invalid filter: `"${filter}`", expected = format" - } + # Lazy match key in case value contains '=' + if ($filter -match "(.+?)=(.+)") { + $key = $matches[1] + $regex = $matches[2] + return $key, $regex + } + else { + throw "Invalid filter: `"${filter}`", expected = format" + } } function GetMatrixConfigFromFile([String] $config) { - [MatrixConfig]$config = try { - GetMatrixConfigFromJson $config - } - catch { - GetMatrixConfigFromYaml $config - } - return $config + [MatrixConfig]$config = try { + GetMatrixConfigFromJson $config + } + catch { + GetMatrixConfigFromYaml $config + } + return $config } function GetMatrixConfigFromYaml([String] $yamlConfig) { - Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module - # ConvertTo then from json is to make sure the nested values are in PSCustomObject - [MatrixConfig]$config = ConvertFrom-Yaml $yamlConfig -Ordered | ConvertTo-Json -Depth 100 | ConvertFrom-Json - return GetMatrixConfig $config + Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module + # ConvertTo then from json is to make sure the nested values are in PSCustomObject + [MatrixConfig]$config = ConvertFrom-Yaml $yamlConfig -Ordered | ConvertTo-Json -Depth 100 | ConvertFrom-Json + return GetMatrixConfig $config } function GetMatrixConfigFromJson([String]$jsonConfig) { - [MatrixConfig]$config = $jsonConfig | ConvertFrom-Json - return GetMatrixConfig $config + [MatrixConfig]$config = $jsonConfig | ConvertFrom-Json + return GetMatrixConfig $config } # Importing the JSON as PSCustomObject preserves key ordering, # whereas ConvertFrom-Json -AsHashtable does not function GetMatrixConfig([MatrixConfig]$config) { - $config.matrixParameters = @() - $config.displayNamesLookup = @{} - $include = [MatrixParameter[]]@() - $exclude = [MatrixParameter[]]@() - - if ($null -ne $config.displayNames) { - $config.displayNames.PSObject.Properties | ForEach-Object { - $config.displayNamesLookup.Add($_.Name, $_.Value) - } - } - if ($null -ne $config.matrix) { - $config.matrixParameters = PsObjectToMatrixParameterArray $config.matrix - } - foreach ($includeMatrix in $config.include) { - $include += , @(PsObjectToMatrixParameterArray $includeMatrix) - } - foreach ($excludeMatrix in $config.exclude) { - $exclude += , @(PsObjectToMatrixParameterArray $excludeMatrix) + $config.matrixParameters = @() + $config.displayNamesLookup = @{} + $include = [MatrixParameter[]]@() + $exclude = [MatrixParameter[]]@() + + if ($null -ne $config.displayNames) { + $config.displayNames.PSObject.Properties | ForEach-Object { + $config.displayNamesLookup.Add($_.Name, $_.Value) } + } + if ($null -ne $config.matrix) { + $config.matrixParameters = PsObjectToMatrixParameterArray $config.matrix + } + foreach ($includeMatrix in $config.include) { + $include += , @(PsObjectToMatrixParameterArray $includeMatrix) + } + foreach ($excludeMatrix in $config.exclude) { + $exclude += , @(PsObjectToMatrixParameterArray $excludeMatrix) + } - $config.include = $include - $config.exclude = $exclude + $config.include = $include + $config.exclude = $exclude - return $config + return $config } function PsObjectToMatrixParameterArray([PSCustomObject]$obj) { - if ($obj -eq $null) { - return $null - } - return $obj.PSObject.Properties | ForEach-Object { - [MatrixParameter]::new($_.Name, $_.Value) - } + if ($obj -eq $null) { + return $null + } + return $obj.PSObject.Properties | ForEach-Object { + [MatrixParameter]::new($_.Name, $_.Value) + } } function ProcessExcludes([Array]$matrix, [Array]$excludes) { - $deleteKey = "%DELETE%" - $exclusionMatrix = @() + $deleteKey = "%DELETE%" + $exclusionMatrix = @() - foreach ($exclusion in $excludes) { - $full = GenerateFullMatrix $exclusion - $exclusionMatrix += $full - } + foreach ($exclusion in $excludes) { + $full = GenerateFullMatrix $exclusion + $exclusionMatrix += $full + } - foreach ($element in $matrix) { - foreach ($exclusion in $exclusionMatrix) { - $match = MatrixElementMatch $element.parameters $exclusion.parameters - if ($match) { - $element.parameters[$deleteKey] = $true - } - } + foreach ($element in $matrix) { + foreach ($exclusion in $exclusionMatrix) { + $match = MatrixElementMatch $element.parameters $exclusion.parameters + if ($match) { + $element.parameters[$deleteKey] = $true + } } + } - return $matrix | Where-Object { !$_.parameters.Contains($deleteKey) } + return $matrix | Where-Object { !$_.parameters.Contains($deleteKey) } } function ProcessIncludes([MatrixConfig]$config, [Array]$matrix) { - $inclusionMatrix = @() - foreach ($inclusion in $config.include) { - $full = GenerateFullMatrix $inclusion $config.displayNamesLookup - $inclusionMatrix += $full - } + $inclusionMatrix = @() + foreach ($inclusion in $config.include) { + $full = GenerateFullMatrix $inclusion $config.displayNamesLookup + $inclusionMatrix += $full + } - return $matrix + $inclusionMatrix + return $matrix + $inclusionMatrix } function ParseReplacement([String]$replacement) { - $parsed = '', '', '' - $idx = 0 - $escaped = $false - $operators = '=', '/' - $err = "Invalid replacement syntax, expecting =/" - - foreach ($c in $replacement -split '') { - if ($idx -ge $parsed.Length) { - throw $err - } - if (!$escaped -and $c -in $operators) { - $idx++ - } - else { - $parsed[$idx] += $c - } - $escaped = $c -eq '\' - } + $parsed = '', '', '' + $idx = 0 + $escaped = $false + $operators = '=', '/' + $err = "Invalid replacement syntax, expecting =/" - if ($idx -lt $parsed.Length - 1) { - throw $err + foreach ($c in $replacement -split '') { + if ($idx -ge $parsed.Length) { + throw $err } + if (!$escaped -and $c -in $operators) { + $idx++ + } + else { + $parsed[$idx] += $c + } + $escaped = $c -eq '\' + } - $replace = $parsed[2] -replace "\\([$($operators -join '')])", '$1' + if ($idx -lt $parsed.Length - 1) { + throw $err + } - return @{ - "key" = '^' + $parsed[0] + '$' - # Force full matches only. - "value" = '^' + $parsed[1] + '$' - "replace" = $replace - } + $replace = $parsed[2] -replace "\\([$($operators -join '')])", '$1' + + return @{ + "key" = '^' + $parsed[0] + '$' + # Force full matches only. + "value" = '^' + $parsed[1] + '$' + "replace" = $replace + } } function ProcessReplace { - param( - [Array]$matrix, - [Array]$replacements, - [Hashtable]$displayNamesLookup - ) + param( + [Array]$matrix, + [Array]$replacements, + [Hashtable]$displayNamesLookup + ) - if (!$replacements) { - return $matrix - } + if (!$replacements) { + return $matrix + } - $replaceMatrix = @() + $replaceMatrix = @() - foreach ($element in $matrix) { - if (!$element -or $element.Count -eq 0) { - continue - } - $replacement = [MatrixParameter[]]@() - if (!$element -or $element.Count -eq 0) { - continue - } + foreach ($element in $matrix) { + if (!$element -or $element.Count -eq 0) { + continue + } + $replacement = [MatrixParameter[]]@() + if (!$element -or $element.Count -eq 0) { + continue + } - foreach ($perm in $element._permutation) { - $replace = $perm - - # Iterate nested permutations or run once for singular values (int, string, bool) - foreach ($flattened in $perm.Flatten()) { - foreach ($query in $replacements) { - $parsed = ParseReplacement $query - if ($flattened.Name -match $parsed.key -and $flattened.Value -match $parsed.value) { - # In most cases, this will just swap one value for another, however -replace - # is used here in order to support replace values which may use regex capture groups - # e.g. 'foo-1' -replace '(foo)-1', '$1-replaced' - $replaceValue = $flattened.Value -replace $parsed.value, $parsed.replace - $perm.Set($replaceValue, $parsed.key) - break - } - } - } - - $replacement += $perm + foreach ($perm in $element._permutation) { + $replace = $perm + + # Iterate nested permutations or run once for singular values (int, string, bool) + foreach ($flattened in $perm.Flatten()) { + foreach ($query in $replacements) { + $parsed = ParseReplacement $query + if ($flattened.Name -match $parsed.key -and $flattened.Value -match $parsed.value) { + # In most cases, this will just swap one value for another, however -replace + # is used here in order to support replace values which may use regex capture groups + # e.g. 'foo-1' -replace '(foo)-1', '$1-replaced' + $replaceValue = $flattened.Value -replace $parsed.value, $parsed.replace + $perm.Set($replaceValue, $parsed.key) + break + } } + } - $replaceMatrix += CreateMatrixCombinationScalar $replacement $displayNamesLookup + $replacement += $perm } - return $replaceMatrix + $replaceMatrix += CreateMatrixCombinationScalar $replacement $displayNamesLookup + } + + return $replaceMatrix } function ProcessEnvironmentVariableReferences([array]$matrix, $displayNamesLookup) { - $updatedMatrix = @() - $missingEnvVars = @{} + $updatedMatrix = @() + $missingEnvVars = @{} - foreach ($element in $matrix) { - $updated = [MatrixParameter[]]@() - if (!$element -or $element.Count -eq 0) { - continue - } + foreach ($element in $matrix) { + $updated = [MatrixParameter[]]@() + if (!$element -or $element.Count -eq 0) { + continue + } - foreach ($perm in $element._permutation) { - # Iterate nested permutations or run once for singular values (int, string, bool) - foreach ($flattened in $perm.Flatten()) { - if ($flattened.Value -is [string] -and $flattened.Value.StartsWith("env:")) { - $envKey = $flattened.Value.Replace("env:", "") - $value = [System.Environment]::GetEnvironmentVariable($envKey) - if (!$value) { - $missingEnvVars[$envKey] = $true - } - $perm.Set($value, $flattened.Name) - } - } - - $updated += $perm + foreach ($perm in $element._permutation) { + # Iterate nested permutations or run once for singular values (int, string, bool) + foreach ($flattened in $perm.Flatten()) { + if ($flattened.Value -is [string] -and $flattened.Value.StartsWith("env:")) { + $envKey = $flattened.Value.Replace("env:", "") + $value = [System.Environment]::GetEnvironmentVariable($envKey) + if (!$value) { + $missingEnvVars[$envKey] = $true + } + $perm.Set($value, $flattened.Name) } + } - $updatedMatrix += CreateMatrixCombinationScalar $updated $displayNamesLookup + $updated += $perm } - if ($missingEnvVars.Count -gt 0) { - throw "Environment variables '$($missingEnvVars.Keys -join ", ")' were empty or not found." - } - return $updatedMatrix + $updatedMatrix += CreateMatrixCombinationScalar $updated $displayNamesLookup + } + + if ($missingEnvVars.Count -gt 0) { + throw "Environment variables '$($missingEnvVars.Keys -join ", ")' were empty or not found." + } + return $updatedMatrix } function ProcessImport([MatrixParameter[]]$matrix, [String]$selection, [Array]$nonSparseParameters, [Hashtable]$displayNamesLookup) { - $importPath = "" - $matrix = $matrix | ForEach-Object { - if ($_.Name -ne $IMPORT_KEYWORD) { - return $_ - } - else { - $importPath = $_.Value - } - } - if ((!$matrix -and !$importPath) -or !$importPath) { - return $matrix, @(), $displayNamesLookup - } - - if (!(Test-Path $importPath)) { - Write-Error "`$IMPORT path '$importPath' does not exist. Current dir: $(Get-Location)" - exit 1 - } - $importedMatrixConfig = GetMatrixConfigFromFile (Get-Content -Raw $importPath) - # Add skipEnvironmentVariables so we don't process environment variables on import - # because we want top level filters to work against the the env key, not the value. - # The environment variables will get resolved after the import. - $importedMatrix = GenerateMatrix ` - -config $importedMatrixConfig ` - -selectFromMatrixType $selection ` - -nonSparseParameters $nonSparseParameters ` - -skipEnvironmentVariables - - $combinedDisplayNameLookup = $importedMatrixConfig.displayNamesLookup - foreach ($lookup in $displayNamesLookup.GetEnumerator()) { - $combinedDisplayNameLookup[$lookup.Name] = $lookup.Value + $importPath = "" + $matrix = $matrix | ForEach-Object { + if ($_.Name -ne $IMPORT_KEYWORD) { + return $_ } - - return $matrix, $importedMatrix, $combinedDisplayNameLookup + else { + $importPath = $_.Value + } + } + if ((!$matrix -and !$importPath) -or !$importPath) { + return $matrix, @(), $displayNamesLookup + } + + if (!(Test-Path $importPath)) { + Write-Error "`$IMPORT path '$importPath' does not exist. Current dir: $(Get-Location)" + exit 1 + } + $importedMatrixConfig = GetMatrixConfigFromFile (Get-Content -Raw $importPath) + # Add skipEnvironmentVariables so we don't process environment variables on import + # because we want top level filters to work against the the env key, not the value. + # The environment variables will get resolved after the import. + $importedMatrix = GenerateMatrix ` + -config $importedMatrixConfig ` + -selectFromMatrixType $selection ` + -nonSparseParameters $nonSparseParameters ` + -skipEnvironmentVariables + + $combinedDisplayNameLookup = $importedMatrixConfig.displayNamesLookup + foreach ($lookup in $displayNamesLookup.GetEnumerator()) { + $combinedDisplayNameLookup[$lookup.Name] = $lookup.Value + } + + return $matrix, $importedMatrix, $combinedDisplayNameLookup } function CombineMatrices([Array]$matrix1, [Array]$matrix2, [Hashtable]$displayNamesLookup = @{}) { - $combined = @() - if (!$matrix1) { - return $matrix2 - } - if (!$matrix2) { - return $matrix1 - } + $combined = @() + if (!$matrix1) { + return $matrix2 + } + if (!$matrix2) { + return $matrix1 + } - foreach ($entry1 in $matrix1) { - foreach ($entry2 in $matrix2) { - $combined += CreateMatrixCombinationScalar ($entry1._permutation + $entry2._permutation) $displayNamesLookup - } + foreach ($entry1 in $matrix1) { + foreach ($entry2 in $matrix2) { + $combined += CreateMatrixCombinationScalar ($entry1._permutation + $entry2._permutation) $displayNamesLookup } + } - return $combined + return $combined } function MatrixElementMatch([System.Collections.Specialized.OrderedDictionary]$source, [System.Collections.Specialized.OrderedDictionary]$target) { - if ($target.Count -eq 0) { - return $false - } + if ($target.Count -eq 0) { + return $false + } - foreach ($key in $target.Keys) { - if (!$source.Contains($key) -or $source[$key] -ne $target[$key]) { - return $false - } + foreach ($key in $target.Keys) { + if (!$source.Contains($key) -or $source[$key] -ne $target[$key]) { + return $false } + } - return $true + return $true } function CloneOrderedDictionary([System.Collections.Specialized.OrderedDictionary]$dictionary) { - $newDictionary = [Ordered]@{} - foreach ($element in $dictionary.GetEnumerator()) { - $newDictionary[$element.Name] = $element.Value - } - return $newDictionary + $newDictionary = [Ordered]@{} + foreach ($element in $dictionary.GetEnumerator()) { + $newDictionary[$element.Name] = $element.Value + } + return $newDictionary } function SerializePipelineMatrix([Array]$matrix) { - $pipelineMatrix = [Ordered]@{} - foreach ($entry in $matrix) { - if ($pipelineMatrix.Contains($entry.Name)) { - Write-Warning "Found duplicate configurations for job `"$($entry.name)`". Multiple values may have been replaced with the same value." - continue - } - $pipelineMatrix.Add($entry.name, [Ordered]@{}) - foreach ($key in $entry.parameters.Keys) { - $pipelineMatrix[$entry.name].Add($key, $entry.parameters[$key]) - } + $pipelineMatrix = [Ordered]@{} + foreach ($entry in $matrix) { + if ($pipelineMatrix.Contains($entry.Name)) { + Write-Warning "Found duplicate configurations for job `"$($entry.name)`". Multiple values may have been replaced with the same value." + continue } - - return @{ - compressed = $pipelineMatrix | ConvertTo-Json -Compress ; - pretty = $pipelineMatrix | ConvertTo-Json; + $pipelineMatrix.Add($entry.name, [Ordered]@{}) + foreach ($key in $entry.parameters.Keys) { + $pipelineMatrix[$entry.name].Add($key, $entry.parameters[$key]) } + } + + return @{ + compressed = $pipelineMatrix | ConvertTo-Json -Compress ; + pretty = $pipelineMatrix | ConvertTo-Json; + } } function GenerateSparseMatrix( - [MatrixParameter[]]$parameters, - [Hashtable]$displayNamesLookup, - [Array]$nonSparseParameters = @() + [MatrixParameter[]]$parameters, + [Hashtable]$displayNamesLookup, + [Array]$nonSparseParameters = @() ) { - $parameters, $nonSparse = ProcessNonSparseParameters $parameters $nonSparseParameters - $dimensions = GetMatrixDimensions $parameters - $matrix = GenerateFullMatrix $parameters $displayNamesLookup + $parameters, $nonSparse = ProcessNonSparseParameters $parameters $nonSparseParameters + $dimensions = GetMatrixDimensions $parameters + $matrix = GenerateFullMatrix $parameters $displayNamesLookup - $sparseMatrix = @() - [array]$indexes = GetSparseMatrixIndexes $dimensions - foreach ($idx in $indexes) { - $sparseMatrix += GetNdMatrixElement $idx $matrix $dimensions - } + $sparseMatrix = @() + [array]$indexes = GetSparseMatrixIndexes $dimensions + foreach ($idx in $indexes) { + $sparseMatrix += GetNdMatrixElement $idx $matrix $dimensions + } - if ($nonSparse) { - $allOfMatrix = GenerateFullMatrix $nonSparse $displayNamesLookup - return CombineMatrices $allOfMatrix $sparseMatrix $displayNamesLookup - } + if ($nonSparse) { + $allOfMatrix = GenerateFullMatrix $nonSparse $displayNamesLookup + return CombineMatrices $allOfMatrix $sparseMatrix $displayNamesLookup + } - return $sparseMatrix + return $sparseMatrix } function GetSparseMatrixIndexes([Array]$dimensions) { - $size = ($dimensions | Measure-Object -Maximum).Maximum - $indexes = @() - - # With full matrix, retrieve items by doing diagonal lookups across the matrix N times. - # For example, given a matrix with dimensions 3, 2, 2: - # 0, 0, 0 - # 1, 1, 1 - # 2, 2, 2 - # 3, 0, 0 <- 3, 3, 3 wraps to 3, 0, 0 given the dimensions - for ($i = 0; $i -lt $size; $i++) { - $idx = @() - for ($j = 0; $j -lt $dimensions.Length; $j++) { - $idx += $i % $dimensions[$j] - } - $indexes += , $idx + $size = ($dimensions | Measure-Object -Maximum).Maximum + $indexes = @() + + # With full matrix, retrieve items by doing diagonal lookups across the matrix N times. + # For example, given a matrix with dimensions 3, 2, 2: + # 0, 0, 0 + # 1, 1, 1 + # 2, 2, 2 + # 3, 0, 0 <- 3, 3, 3 wraps to 3, 0, 0 given the dimensions + for ($i = 0; $i -lt $size; $i++) { + $idx = @() + for ($j = 0; $j -lt $dimensions.Length; $j++) { + $idx += $i % $dimensions[$j] } + $indexes += , $idx + } - return , $indexes + return , $indexes } function GenerateFullMatrix( - [MatrixParameter[]] $parameters, - [Hashtable]$displayNamesLookup = @{} + [MatrixParameter[]] $parameters, + [Hashtable]$displayNamesLookup = @{} ) { - # Handle when the config does not have a matrix specified (e.g. only the include field is specified) - if (!$parameters) { - return @() - } + # Handle when the config does not have a matrix specified (e.g. only the include field is specified) + if (!$parameters) { + return @() + } - $matrix = [System.Collections.ArrayList]::new() - InitializeMatrix $parameters $displayNamesLookup $matrix + $matrix = [System.Collections.ArrayList]::new() + InitializeMatrix $parameters $displayNamesLookup $matrix - return $matrix + return $matrix } function CreateMatrixCombinationScalar([MatrixParameter[]]$permutation, [Hashtable]$displayNamesLookup = @{}) { - $names = @() - $flattenedParameters = [Ordered]@{} - - foreach ($entry in $permutation) { - $nameSegment = "" - - # Unwind nested permutations or run once for singular values (int, string, bool) - foreach ($param in $entry.Flatten()) { - if ($flattenedParameters.Contains($param.Name)) { - throw "Found duplicate parameter `"$($param.Name)`" when creating matrix combination." - } - $flattenedParameters.Add($param.Name, $param.Value) - } + $names = @() + $flattenedParameters = [Ordered]@{} - $nameSegment = $entry.CreateDisplayName($displayNamesLookup) - if ($nameSegment) { - $names += $nameSegment - } - } + foreach ($entry in $permutation) { + $nameSegment = "" - # The maximum allowed matrix name length is 100 characters - $name = $names -join "_" - if ($name -and $name[0] -match "^[0-9]") { - $name = "job_" + $name # Azure Pipelines only supports job names starting with letters - } - if ($name.Length -gt 100) { - $name = $name[0..99] -join "" + # Unwind nested permutations or run once for singular values (int, string, bool) + foreach ($param in $entry.Flatten()) { + if ($flattenedParameters.Contains($param.Name)) { + throw "Found duplicate parameter `"$($param.Name)`" when creating matrix combination." + } + $flattenedParameters.Add($param.Name, $param.Value) } - return @{ - name = $name - parameters = $flattenedParameters - # Keep the original permutation around in case we need to re-process this entry when transforming the matrix - _permutation = $permutation + $nameSegment = $entry.CreateDisplayName($displayNamesLookup) + if ($nameSegment) { + $names += $nameSegment } + } + + # The maximum allowed matrix name length is 100 characters + $name = $names -join "_" + if ($name -and $name[0] -match "^[0-9]") { + $name = "job_" + $name # Azure Pipelines only supports job names starting with letters + } + if ($name.Length -gt 100) { + $name = $name[0..99] -join "" + } + + return @{ + name = $name + parameters = $flattenedParameters + # Keep the original permutation around in case we need to re-process this entry when transforming the matrix + _permutation = $permutation + } } function InitializeMatrix { - param( - [MatrixParameter[]]$parameters, - [Hashtable]$displayNamesLookup, - [System.Collections.ArrayList]$permutations, - $permutation = [MatrixParameter[]]@() - ) - $head, $tail = $parameters + param( + [MatrixParameter[]]$parameters, + [Hashtable]$displayNamesLookup, + [System.Collections.ArrayList]$permutations, + $permutation = [MatrixParameter[]]@() + ) + $head, $tail = $parameters - if (!$head) { - $entry = CreateMatrixCombinationScalar $permutation $displayNamesLookup - $permutations.Add($entry) | Out-Null - return - } + if (!$head) { + $entry = CreateMatrixCombinationScalar $permutation $displayNamesLookup + $permutations.Add($entry) | Out-Null + return + } - # This behavior implicitly treats non-array values as single elements - foreach ($param in $head.Flatten()) { - $newPermutation = $permutation + $param - InitializeMatrix $tail $displayNamesLookup $permutations $newPermutation - } + # This behavior implicitly treats non-array values as single elements + foreach ($param in $head.Flatten()) { + $newPermutation = $permutation + $param + InitializeMatrix $tail $displayNamesLookup $permutations $newPermutation + } } function GetMatrixDimensions([MatrixParameter[]]$parameters) { - $dimensions = @() - foreach ($param in $parameters) { - $dimensions += $param.Length() - } + $dimensions = @() + foreach ($param in $parameters) { + $dimensions += $param.Length() + } - return $dimensions + return $dimensions } function SetNdMatrixElement { - param( - $element, - [ValidateNotNullOrEmpty()] - [Array]$idx, - [ValidateNotNullOrEmpty()] - [Array]$matrix, - [ValidateNotNullOrEmpty()] - [Array]$dimensions - ) + param( + $element, + [ValidateNotNullOrEmpty()] + [Array]$idx, + [ValidateNotNullOrEmpty()] + [Array]$matrix, + [ValidateNotNullOrEmpty()] + [Array]$dimensions + ) - if ($idx.Length -ne $dimensions.Length) { - throw "Matrix index query $($idx.Length) must be the same length as its dimensions $($dimensions.Length)" - } + if ($idx.Length -ne $dimensions.Length) { + throw "Matrix index query $($idx.Length) must be the same length as its dimensions $($dimensions.Length)" + } - $arrayIndex = GetNdMatrixArrayIndex $idx $dimensions - $matrix[$arrayIndex] = $element + $arrayIndex = GetNdMatrixArrayIndex $idx $dimensions + $matrix[$arrayIndex] = $element } function GetNdMatrixArrayIndex { - param( - [ValidateNotNullOrEmpty()] - [Array]$idx, - [ValidateNotNullOrEmpty()] - [Array]$dimensions - ) - - if ($idx.Length -ne $dimensions.Length) { - throw "Matrix index query length ($($idx.Length)) must be the same as dimension length ($($dimensions.Length))" - } - - $stride = 1 + param( + [ValidateNotNullOrEmpty()] + [Array]$idx, + [ValidateNotNullOrEmpty()] + [Array]$dimensions + ) + + if ($idx.Length -ne $dimensions.Length) { + throw "Matrix index query length ($($idx.Length)) must be the same as dimension length ($($dimensions.Length))" + } + + $stride = 1 + # Commented out does lookup with wrap handling + # $index = $idx[$idx.Length-1] % $dimensions[$idx.Length-1] + $index = $idx[$idx.Length - 1] + + for ($i = $dimensions.Length - 1; $i -ge 1; $i--) { + $stride *= $dimensions[$i] # Commented out does lookup with wrap handling - # $index = $idx[$idx.Length-1] % $dimensions[$idx.Length-1] - $index = $idx[$idx.Length - 1] - - for ($i = $dimensions.Length - 1; $i -ge 1; $i--) { - $stride *= $dimensions[$i] - # Commented out does lookup with wrap handling - # $index += ($idx[$i-1] % $dimensions[$i-1]) * $stride - $index += $idx[$i - 1] * $stride - } + # $index += ($idx[$i-1] % $dimensions[$i-1]) * $stride + $index += $idx[$i - 1] * $stride + } - return $index + return $index } function GetNdMatrixElement { - param( - [ValidateNotNullOrEmpty()] - [Array]$idx, - [ValidateNotNullOrEmpty()] - [Array]$matrix, - [ValidateNotNullOrEmpty()] - [Array]$dimensions - ) + param( + [ValidateNotNullOrEmpty()] + [Array]$idx, + [ValidateNotNullOrEmpty()] + [Array]$matrix, + [ValidateNotNullOrEmpty()] + [Array]$dimensions + ) - $arrayIndex = GetNdMatrixArrayIndex $idx $dimensions - return $matrix[$arrayIndex] + $arrayIndex = GetNdMatrixArrayIndex $idx $dimensions + return $matrix[$arrayIndex] } function GetNdMatrixIndex { - param( - [int]$index, - [ValidateNotNullOrEmpty()] - [Array]$dimensions - ) + param( + [int]$index, + [ValidateNotNullOrEmpty()] + [Array]$dimensions + ) - $matrixIndex = @() - $stride = 1 + $matrixIndex = @() + $stride = 1 - for ($i = $dimensions.Length - 1; $i -ge 1; $i--) { - $stride *= $dimensions[$i] - $page = [math]::floor($index / $stride) % $dimensions[$i - 1] - $matrixIndex = , $page + $matrixIndex - } - $col = $index % $dimensions[$dimensions.Length - 1] - $matrixIndex += $col + for ($i = $dimensions.Length - 1; $i -ge 1; $i--) { + $stride *= $dimensions[$i] + $page = [math]::floor($index / $stride) % $dimensions[$i - 1] + $matrixIndex = , $page + $matrixIndex + } + $col = $index % $dimensions[$dimensions.Length - 1] + $matrixIndex += $col - return $matrixIndex + return $matrixIndex } # # # # # # # # # # # # # # # # # # # # # # # # # # # # @@ -720,22 +720,48 @@ function GetNdMatrixIndex { # help explain the above N-dimensional algorithm # # # # # # # # # # # # # # # # # # # # # # # # # # # # # function Get4dMatrixElement([Array]$idx, [Array]$matrix, [Array]$dimensions) { - $stride1 = $idx[0] * $dimensions[1] * $dimensions[2] * $dimensions[3] - $stride2 = $idx[1] * $dimensions[2] * $dimensions[3] - $stride3 = $idx[2] * $dimensions[3] - $stride4 = $idx[3] + $stride1 = $idx[0] * $dimensions[1] * $dimensions[2] * $dimensions[3] + $stride2 = $idx[1] * $dimensions[2] * $dimensions[3] + $stride3 = $idx[2] * $dimensions[3] + $stride4 = $idx[3] - return $matrix[$stride1 + $stride2 + $stride3 + $stride4] + return $matrix[$stride1 + $stride2 + $stride3 + $stride4] } function Get4dMatrixIndex([int]$index, [Array]$dimensions) { - $stride1 = $dimensions[3] - $stride2 = $dimensions[2] - $stride3 = $dimensions[1] - $page1 = [math]::floor($index / $stride1) % $dimensions[2] - $page2 = [math]::floor($index / ($stride1 * $stride2)) % $dimensions[1] - $page3 = [math]::floor($index / ($stride1 * $stride2 * $stride3)) % $dimensions[0] - $remainder = $index % $dimensions[3] - - return @($page3, $page2, $page1, $remainder) + $stride1 = $dimensions[3] + $stride2 = $dimensions[2] + $stride3 = $dimensions[1] + $page1 = [math]::floor($index / $stride1) % $dimensions[2] + $page2 = [math]::floor($index / ($stride1 * $stride2)) % $dimensions[1] + $page3 = [math]::floor($index / ($stride1 * $stride2 * $stride3)) % $dimensions[0] + $remainder = $index % $dimensions[3] + + return @($page3, $page2, $page1, $remainder) +} + +function GenerateMatrixForConfig { + param ( + [Parameter(Mandatory = $true)][string] $ConfigPath, + [Parameter(Mandatory = $True)][string] $Selection, + [Parameter(Mandatory = $false)][string] $DisplayNameFilter, + [Parameter(Mandatory = $false)][array] $Filters, + [Parameter(Mandatory = $false)][array] $Replace + ) + $matrixFile = Join-Path $PSScriptRoot ".." ".." ".." ".." $ConfigPath + + $resolvedMatrixFile = Resolve-Path $matrixFile + + $config = GetMatrixConfigFromFile (Get-Content $resolvedMatrixFile -Raw) + # Strip empty string filters in order to be able to use azure pipelines yaml join() + $Filters = $Filters | Where-Object { $_ } + + [array]$matrix = GenerateMatrix ` + -config $config ` + -selectFromMatrixType $Selection ` + -displayNameFilter $DisplayNameFilter ` + -filters $Filters ` + -replace $Replace + + return , $matrix } From 6ac2ad8a0d942066424ce7cf20e869a8d609d269 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 29 Oct 2024 16:30:24 -0700 Subject: [PATCH 10/15] indentation of the create-prjobmatrix was incorrect, so we weren't actually invoking the script --- .../templates/jobs/generate-job-matrix.yml | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/eng/common/pipelines/templates/jobs/generate-job-matrix.yml b/eng/common/pipelines/templates/jobs/generate-job-matrix.yml index 7bee03f8d235..6e29162e83ca 100644 --- a/eng/common/pipelines/templates/jobs/generate-job-matrix.yml +++ b/eng/common/pipelines/templates/jobs/generate-job-matrix.yml @@ -122,21 +122,21 @@ jobs: # This else being set also currently assumes that the $(Build.ArtifactStagingDirectory)/PackageInfo folder is populated by PreGenerationSteps. # Not currently not hardcoded, so not doing the needful and populating this folder before we hit this step will result in generation errors. - - ${{ else }}: - - ${{ each pool in parameters.Pools }}: - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 - arguments: > - -PackagePropertiesFolder $(Build.ArtifactStagingDirectory)/PackageInfo - -PRMatrixFile ${{ parameters.PRMatrix.Path }} - -PRMatrixSetting ${{ parameters.PRMatrix.Setting }} - -DisplayNameFilter '$(displayNameFilter)' - -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}' - -Replace '${{ join(''',''', parameters.MatrixReplace) }}' - displayName: Create ${{ pool.name }} PR Matrix - name: vm_job_matrix_pr_${{ pool.name }} + - ${{ else }}: + - ${{ each pool in parameters.Pools }}: + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 + arguments: > + -PackagePropertiesFolder $(Build.ArtifactStagingDirectory)/PackageInfo + -PRMatrixFile ${{ parameters.PRMatrix.Path }} + -PRMatrixSetting ${{ parameters.PRMatrix.Setting }} + -DisplayNameFilter '$(displayNameFilter)' + -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}' + -Replace '${{ join(''',''', parameters.MatrixReplace) }}' + displayName: Create ${{ pool.name }} PR Matrix + name: vm_job_matrix_pr_${{ pool.name }} - ${{ if eq(parameters.EnablePRGeneration, false) }}: - ${{ each config in parameters.MatrixConfigs }}: From e48009b15da1c89ae03045fc947b0f659497f1a3 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 29 Oct 2024 16:51:46 -0700 Subject: [PATCH 11/15] remove some debug statements, ensure our ordering is correct --- eng/common/scripts/Helpers/Package-Helpers.ps1 | 2 -- eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/eng/common/scripts/Helpers/Package-Helpers.ps1 b/eng/common/scripts/Helpers/Package-Helpers.ps1 index ac94bd997d77..5ec91ce82b0f 100644 --- a/eng/common/scripts/Helpers/Package-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Package-Helpers.ps1 @@ -183,8 +183,6 @@ function Get-ObjectKey { [object]$Object ) - Write-Host "In Get-ObjectKey with $Object" - if (-not $Object) { return "unset" } diff --git a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 index 6accc37f0e3b..f903e8eef6d0 100644 --- a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 +++ b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 @@ -61,12 +61,12 @@ $matrixBatchesByConfig = Group-ByObjectKey $packageProperties "CIMatrixConfigs" $OverallResult = @() foreach ($matrixBatchKey in $matrixBatchesByConfig.Keys) { - Write-Host "Generating config for $($matrixConfig.Path)" $matrixBatch = $matrixBatchesByConfig[$matrixBatchKey] $matrixConfigs = $matrixBatch | Select-Object -First 1 -ExpandProperty CIMatrixConfigs $matrixResults = @() foreach ($matrixConfig in $matrixConfigs) { + Write-Host "Generating config for $($matrixConfig.Path)" $matrixResults = GenerateMatrixForConfig ` -ConfigPath $matrixConfig.Path ` -Selection $matrixConfig.Selection ` From 9b42c8051eff28657f953b633d401c55f23b58b2 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 30 Oct 2024 12:18:57 -0700 Subject: [PATCH 12/15] commit the change to see how it handles it --- .../templates/jobs/generate-job-matrix.yml | 27 +++++++++++-------- .../scripts/Helpers/Package-Helpers.ps1 | 17 +++++------- .../scripts/job-matrix/Create-PrJobMatrix.ps1 | 27 ++++++++++++++++--- eng/scripts/try-package-config.ps1 | 13 +++++++++ 4 files changed, 58 insertions(+), 26 deletions(-) create mode 100644 eng/scripts/try-package-config.ps1 diff --git a/eng/common/pipelines/templates/jobs/generate-job-matrix.yml b/eng/common/pipelines/templates/jobs/generate-job-matrix.yml index 6e29162e83ca..8376fa5aed03 100644 --- a/eng/common/pipelines/templates/jobs/generate-job-matrix.yml +++ b/eng/common/pipelines/templates/jobs/generate-job-matrix.yml @@ -124,17 +124,22 @@ jobs: # Not currently not hardcoded, so not doing the needful and populating this folder before we hit this step will result in generation errors. - ${{ else }}: - ${{ each pool in parameters.Pools }}: - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 - arguments: > - -PackagePropertiesFolder $(Build.ArtifactStagingDirectory)/PackageInfo - -PRMatrixFile ${{ parameters.PRMatrix.Path }} - -PRMatrixSetting ${{ parameters.PRMatrix.Setting }} - -DisplayNameFilter '$(displayNameFilter)' - -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}' - -Replace '${{ join(''',''', parameters.MatrixReplace) }}' + + - pwsh: | + $defaultMatrixes = '${{ convertToJson(parameters.MatrixConfigs) }}' + $defaultMatrixes >> matrix.json + + cat matrix.json + + ./eng/scripts/try-package-config.ps1 -MatrixConfigFile matrix.json + + # ./eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 ` + # -PackagePropertiesFolder $(Build.ArtifactStagingDirectory)/PackageInfo ` + # -PRMatrixFile ${{ parameters.PRMatrix.Path }} ` + # -PRMatrixSetting ${{ parameters.PRMatrix.Setting }} ` + # -DisplayNameFilter '$(displayNameFilter)' ` + # -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}' ` + # -Replace '${{ join(''',''', parameters.MatrixReplace) }}' displayName: Create ${{ pool.name }} PR Matrix name: vm_job_matrix_pr_${{ pool.name }} diff --git a/eng/common/scripts/Helpers/Package-Helpers.ps1 b/eng/common/scripts/Helpers/Package-Helpers.ps1 index 5ec91ce82b0f..d72ae84fbff4 100644 --- a/eng/common/scripts/Helpers/Package-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Package-Helpers.ps1 @@ -209,6 +209,7 @@ function Get-ObjectKey { } } + function Group-ByObjectKey { param ( [Parameter(Mandatory)] @@ -221,20 +222,14 @@ function Group-ByObjectKey { $groupedDictionary = @{} foreach ($item in $Items) { - Write-Host "Getting object key for $($item.Name)" $key = Get-ObjectKey $item."$GroupByProperty" - if ($key) { - if (-not $groupedDictionary.ContainsKey($key)) { - $groupedDictionary[$key] = @() - } - - # Add the current item to the array for this key - $groupedDictionary[$key] += $item - } - else { - Write-Host "Apparently don't have a key for $($item.Name)" + if (-not $groupedDictionary.ContainsKey($key)) { + $groupedDictionary[$key] = @() } + + # Add the current item to the array for this key + $groupedDictionary[$key] += $item } return $groupedDictionary diff --git a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 index f903e8eef6d0..3d6efce92842 100644 --- a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 +++ b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 @@ -1,12 +1,31 @@ <# .SYNOPSIS Generates a combined PR job matrix from a package properties folder. It is effectively a combination of -Create-JobMatrix and distribute-packages-to-matrix - -This script is intended to be used within an Azure DevOps pipeline to generate a job matrix for a PR. +Create-JobMatrix and distribute-packages-to-matrix. + +.DESCRIPTION +Create-JobMatrix has a limitation in that it accepts one or multiple matrix files, but it doesn't allow runtime +selection of the matrix file based on what is being built. Due to this, this script exists to provide exactly +that mapping. + +It should be called from a PR build only. + +It generates the matrix by the following algorithm: + - load all package properties files + - group the package properties by their targeted CI Matrix Configs + - for each package group, generate the matrix for each matrix config in the group (remember MatrixConfigs is a list not a single object) + - for each matrix config, generate the matrix + - calculate if batching is necessary for this matrix config + - for each batch + - create combined property name for the batch + - walk each matrix item + - add suffixes for batch and matrix config if nececessary to the job name + - add the combined property name to the parameters of the matrix item .EXAMPLE -./eng/common/scripts/Create-PRJobMatrix $(Build.ArtifactStagingDirectory)/PackageProperties +./eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 ` + -PackagePropertiesFolder "path/to/populated/PackageInfo" ` + -PrMatrixSetting "" #> [CmdletBinding()] diff --git a/eng/scripts/try-package-config.ps1 b/eng/scripts/try-package-config.ps1 new file mode 100644 index 000000000000..1604add82baf --- /dev/null +++ b/eng/scripts/try-package-config.ps1 @@ -0,0 +1,13 @@ +param( + $MatrixConfigFile +) + +if ( -not (Test-Path $MatrixConfigFile)) { + Write-Error "Matrix Config file doesn't exist" + exit 1 +} + +$content = Get-Content $MatrixConfigFile -Raw | ConvertFrom-Json +Write-Host $content + +Write-Host ($content | ConvertTo-Json -Depth 100) From c00dad06fde8f1da959ae5c0685b21ed9ba41dfb Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 30 Oct 2024 13:44:01 -0700 Subject: [PATCH 13/15] pass the dynamic config to our script --- .../templates/jobs/generate-job-matrix.yml | 28 ++++++++----------- .../scripts/job-matrix/Create-PrJobMatrix.ps1 | 16 ++++------- eng/pipelines/templates/jobs/ci.yml | 6 ---- eng/scripts/try-package-config.ps1 | 13 --------- 4 files changed, 18 insertions(+), 45 deletions(-) delete mode 100644 eng/scripts/try-package-config.ps1 diff --git a/eng/common/pipelines/templates/jobs/generate-job-matrix.yml b/eng/common/pipelines/templates/jobs/generate-job-matrix.yml index 8376fa5aed03..7ced6d580d40 100644 --- a/eng/common/pipelines/templates/jobs/generate-job-matrix.yml +++ b/eng/common/pipelines/templates/jobs/generate-job-matrix.yml @@ -45,9 +45,9 @@ parameters: - name: EnablePRGeneration type: boolean default: false -- name: PRMatrix - type: object -# Mappings to OS name required at template compile time by 1es pipeline templates +- name: PRMatrixSetting + type: string + default: 'ArtifactPackageNames' - name: Pools type: object default: @@ -126,20 +126,16 @@ jobs: - ${{ each pool in parameters.Pools }}: - pwsh: | - $defaultMatrixes = '${{ convertToJson(parameters.MatrixConfigs) }}' - $defaultMatrixes >> matrix.json - - cat matrix.json - - ./eng/scripts/try-package-config.ps1 -MatrixConfigFile matrix.json + # dump the conglomerated CI matrix + '${{ convertToJson(parameters.MatrixConfigs) }}' | Set-Content matrix.json - # ./eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 ` - # -PackagePropertiesFolder $(Build.ArtifactStagingDirectory)/PackageInfo ` - # -PRMatrixFile ${{ parameters.PRMatrix.Path }} ` - # -PRMatrixSetting ${{ parameters.PRMatrix.Setting }} ` - # -DisplayNameFilter '$(displayNameFilter)' ` - # -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}' ` - # -Replace '${{ join(''',''', parameters.MatrixReplace) }}' + ./eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 ` + -PackagePropertiesFolder $(Build.ArtifactStagingDirectory)/PackageInfo ` + -PRMatrixFile matrix.json ` + -PRMatrixSetting ${{ parameters.PRMatrixSetting }} ` + -DisplayNameFilter '$(displayNameFilter)' ` + -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}' ` + -Replace '${{ join(''',''', parameters.MatrixReplace) }}' displayName: Create ${{ pool.name }} PR Matrix name: vm_job_matrix_pr_${{ pool.name }} diff --git a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 index 3d6efce92842..c087885ae4b8 100644 --- a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 +++ b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 @@ -48,18 +48,14 @@ if (!(Test-Path $PackagePropertiesFolder)) { exit 1 } +if (!(Test-Path $PRMatrixFile)) { + Write-Error "PR Matrix file doesn't exist" + exit 1 +} + Write-Host "Generating PR job matrix for $PackagePropertiesFolder" -# this will become a parameter in the future, we will pass MatrixConfigs as a parameter -# these will be the "default" matrices that we will use to generate the PR matrix in lieu of -# a matrix file -$configs = @( - [PsCustomObject]@{ - Name = "default_platform_matrix" - Path = $PRMatrixFile - Selection = "sparse" - } -) +$configs = Get-Content -Raw $PRMatrixFile | ConvertFrom-Json | ForEach-Object { [PSCustomObject]$_ } # calculate general targeting information and create our batches prior to generating any matrix # this prototype doesn't handle direct and indirect, it just batches for simplicity of the proto diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 1e4f6d68e6a1..1c16b68bc71e 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -77,12 +77,6 @@ jobs: SparseCheckoutPaths: [ "sdk/", ".vscode"] ${{ if eq(parameters.ServiceDirectory, 'auto') }}: EnablePRGeneration: true - - PRMatrix: - Name: pr_test_base - Path: eng/pipelines/templates/stages/platform-matrix.json - Setting: ArtifactPackageNames - PreGenerationSteps: - template: /eng/common/pipelines/templates/steps/save-package-properties.yml parameters: diff --git a/eng/scripts/try-package-config.ps1 b/eng/scripts/try-package-config.ps1 deleted file mode 100644 index 1604add82baf..000000000000 --- a/eng/scripts/try-package-config.ps1 +++ /dev/null @@ -1,13 +0,0 @@ -param( - $MatrixConfigFile -) - -if ( -not (Test-Path $MatrixConfigFile)) { - Write-Error "Matrix Config file doesn't exist" - exit 1 -} - -$content = Get-Content $MatrixConfigFile -Raw | ConvertFrom-Json -Write-Host $content - -Write-Host ($content | ConvertTo-Json -Depth 100) From e3b7f52b680b0cbf6a290de5e0332fed940e4191 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 30 Oct 2024 15:33:36 -0700 Subject: [PATCH 14/15] remove useless ForEach-Object { [PSCustomObject] } --- eng/common/pipelines/templates/jobs/generate-job-matrix.yml | 2 +- eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/eng/common/pipelines/templates/jobs/generate-job-matrix.yml b/eng/common/pipelines/templates/jobs/generate-job-matrix.yml index 7ced6d580d40..ab67e915de85 100644 --- a/eng/common/pipelines/templates/jobs/generate-job-matrix.yml +++ b/eng/common/pipelines/templates/jobs/generate-job-matrix.yml @@ -48,6 +48,7 @@ parameters: - name: PRMatrixSetting type: string default: 'ArtifactPackageNames' +# Mappings to OS name required at template compile time by 1es pipeline templates - name: Pools type: object default: @@ -124,7 +125,6 @@ jobs: # Not currently not hardcoded, so not doing the needful and populating this folder before we hit this step will result in generation errors. - ${{ else }}: - ${{ each pool in parameters.Pools }}: - - pwsh: | # dump the conglomerated CI matrix '${{ convertToJson(parameters.MatrixConfigs) }}' | Set-Content matrix.json diff --git a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 index c087885ae4b8..530b6c33ac6a 100644 --- a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 +++ b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 @@ -55,13 +55,11 @@ if (!(Test-Path $PRMatrixFile)) { Write-Host "Generating PR job matrix for $PackagePropertiesFolder" -$configs = Get-Content -Raw $PRMatrixFile | ConvertFrom-Json | ForEach-Object { [PSCustomObject]$_ } +$configs = Get-Content -Raw $PRMatrixFile | ConvertFrom-Json # calculate general targeting information and create our batches prior to generating any matrix -# this prototype doesn't handle direct and indirect, it just batches for simplicity of the proto $packageProperties = Get-ChildItem -Recurse "$PackagePropertiesFolder" *.json ` -| ForEach-Object { Get-Content -Path $_.FullName | ConvertFrom-Json } ` -| ForEach-Object { [PSCustomObject]$_ } +| ForEach-Object { Get-Content -Path $_.FullName | ConvertFrom-Json } # set default matrix config for each package if there isn't an override $packageProperties | ForEach-Object { From 7d1b75623456b915d3b99d3ac516cc367981e84c Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 30 Oct 2024 15:45:09 -0700 Subject: [PATCH 15/15] new resolution logic --- eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 index 530b6c33ac6a..b7c3b9d3ae09 100644 --- a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 +++ b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 @@ -21,6 +21,7 @@ It generates the matrix by the following algorithm: - walk each matrix item - add suffixes for batch and matrix config if nececessary to the job name - add the combined property name to the parameters of the matrix item + - add the matrix item to the overall result .EXAMPLE ./eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 ` @@ -63,9 +64,7 @@ $packageProperties = Get-ChildItem -Recurse "$PackagePropertiesFolder" *.json ` # set default matrix config for each package if there isn't an override $packageProperties | ForEach-Object { - if (-not $_.CIMatrixConfigs) { - $_.CIMatrixConfigs = $configs - } + $_.CIMatrixConfigs = $_.CIMatrixConfigs ?? $configs } # The key here is that after we group the packages by the matrix config objects, we can use the first item's MatrixConfig