Using Dynamic Matrix Config #31542
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -21,6 +21,7 @@ class PackageProps | |
| # additional packages required for validation of this one | ||
| [string[]]$AdditionalValidationPackages | ||
| [HashTable]$ArtifactDetails | ||
| [HashTable[]]$CIMatrixConfigs | ||
|
|
||
| PackageProps([string]$name, [string]$version, [string]$directoryPath, [string]$serviceDirectory) | ||
| { | ||
|
|
@@ -84,8 +85,7 @@ class PackageProps | |
| $this.Group = $group | ||
| } | ||
|
|
||
| hidden [PSCustomObject]ParseYmlForArtifact([string]$ymlPath) { | ||
| $content = LoadFrom-Yaml $ymlPath | ||
| if ($content) { | ||
| $artifacts = GetValueSafelyFrom-Yaml $content @("extends", "parameters", "Artifacts") | ||
|
|
@@ -95,8 +95,21 @@ class PackageProps | |
| $artifactForCurrentPackage = $artifacts | Where-Object { $_["name"] -eq $this.ArtifactName -or $_["name"] -eq $this.Name } | ||
| } | ||
|
|
||
| # if we found an artifact for the current package, we should count this ci file as the source of the matrix for this package | ||
| if ($artifactForCurrentPackage) { | ||
| $result = [PSCustomObject]@{ | ||
| ArtifactConfig = [HashTable]$artifactForCurrentPackage | ||
| MatrixConfigs = @() | ||
| } | ||
|
|
||
| # if we know this is the matrix for our file, we should now see if there is a custom matrix config for the package | ||
| $matrixConfigList = GetValueSafelyFrom-Yaml $content @("extends", "parameters", "MatrixConfigs") | ||
|
Member
There was a problem hiding this comment. Does this also handle values behind devops template conditionals? e.g. https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/identity/tests.yml#L33 |
||
|
|
||
| if ($matrixConfigList) { | ||
| $result.MatrixConfigs = matrixConfigList | ||
| } | ||
|
|
||
| return $result | ||
| } | ||
| } | ||
| return $null | ||
|
|
@@ -112,7 +125,10 @@ class PackageProps | |
| foreach($ciFile in $ciFiles) { | ||
| $ciArtifactResult = $this.ParseYmlForArtifact($ciFile.FullName) | ||
| if ($ciArtifactResult) { | ||
| $this.ArtifactDetails = [Hashtable]$ciArtifactResult.ArtifactConfig | ||
| $this.CIMatrixConfigs = $ciArtifactResult.MatrixConfigs | ||
| # if this package appeared in this ci file, then we should | ||
| # treat this CI file as the source of the Matrix for this package | ||
| break | ||
| } | ||
| } | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,109 @@ | ||
| <# | ||
| .SYNOPSIS | ||
| Generates a combined PR job matrix from a package properties folder. It is effectively a combination of | ||
| Create-JobMatrix and distribute-packages-to-matrix | ||
| This script is intended to be used within an Azure DevOps pipeline to generate a job matrix for a PR. | ||
| .EXAMPLE | ||
| ./eng/common/scripts/Create-PRJobMatrix $(Build.ArtifactStagingDirectory)/PackageProperties | ||
| #> | ||
|
|
||
| [CmdletBinding()] | ||
| param ( | ||
| [Parameter(Mandatory=$true)][string] $PackagePropertiesFolder, | ||
| [Parameter(Mandatory=$true)][string] $PRMatrixFile, | ||
| [Parameter(Mandatory=$true)][string] $PRMatrixSetting, | ||
| [Parameter(Mandatory=$False)][string] $DisplayNameFilter, | ||
| [Parameter(Mandatory=$False)][array] $Filters, | ||
| [Parameter(Mandatory=$False)][array] $Replace, | ||
| [Parameter()][switch] $CI = ($null -ne $env:SYSTEM_TEAMPROJECTID) | ||
| ) | ||
|
|
||
| . $PSScriptRoot/job-matrix-functions.ps1 | ||
| . $PSScriptRoot/../Helpers/Package-Helpers.ps1 | ||
|
|
||
| if (!(Test-Path $PackagePropertiesFolder)) { | ||
| Write-Error "Package Properties folder doesn't exist" | ||
| exit 1 | ||
| } | ||
|
|
||
| function GenerateMatrixForConfig { | ||
| param ( | ||
| [Parameter(Mandatory=$true)][string] $ConfigPath, | ||
| [Parameter(Mandatory=$True)][string] $Selection, | ||
| [Parameter(Mandatory=$false)][string] $DisplayNameFilter, | ||
| [Parameter(Mandatory=$false)][array] $Filters, | ||
| [Parameter(Mandatory=$false)][array] $Replace | ||
| ) | ||
|
|
||
| $config = GetMatrixConfigFromFile (Get-Content $ConfigPath -Raw) | ||
| # Strip empty string filters in order to be able to use azure pipelines yaml join() | ||
| $Filters = $Filters | Where-Object { $_ } | ||
|
|
||
| [array]$matrix = GenerateMatrix ` | ||
| -config $config ` | ||
| -selectFromMatrixType $Selection ` | ||
| -displayNameFilter $DisplayNameFilter ` | ||
| -filters $Filters ` | ||
| -replace $Replace | ||
|
|
||
| return ,$matrix | ||
| } | ||
|
|
||
| # calculate general targeting information and create our batches prior to generating any matrix | ||
| # this prototype doesn't handle direct and indirect, it just batches for simplicity of the proto | ||
| $packageProperties = Get-ChildItem -Recurse "$PackagePropertiesFolder" *.json ` | ||
| | % { Get-Content -Path $_.FullName | ConvertFrom-Json } | ||
|
|
||
| # in the full proto, we will compress the CIMatrixConfig value into a single string, then use it as a key | ||
| # to group the packages | ||
| # then we will simply iterate over the groups and generate the matrix for each group | ||
| # at the very end, will distribute the packages to the matrix based on the group key (batched over course) | ||
| $configs = @( | ||
| [PsCustomObject]@{ | ||
| ConfigPath = $PRMatrixFile | ||
| Selection = "sparse" | ||
| } | ||
| ) | ||
|
|
||
| $OverallResult = @() | ||
| foreach($matrixConfig in $configs) { | ||
| Write-Host "Generating config for $($matrixConfig.ConfigPath)" | ||
| $generatedMatrix = GenerateMatrixForConfig -ConfigPath $matrixConfig.ConfigPath -Selection $matrixConfig.Selection -DisplayNameFilter $DisplayNameFilter -Filters $Filters -Replace $Replace | ||
|
|
||
| # also note here that we should be using a filtered set of package properties | ||
| # (remember we figured out which matrix config is associated with each package) | ||
| # $batches = Split-ArrayIntoBatches -InputArray $packageProperties -BatchSize $BATCHSIZE | ||
| $batches = @() | ||
| $everything = ($packageProperties | ForEach-Object { $_.ArtifactName }) | ||
| $batches += ,$everything | ||
|
|
||
| foreach($batch in $batches) { | ||
| $ModifiedMatrix = @() | ||
| # to understand this iteration, one must understand that the matrix is a list of hashtables, each with a couple keys: | ||
| # [ | ||
| # { "name": "jobname", "parameters": { matrixSetting1: matrixValue1, ...} }, | ||
| # ] | ||
| if($batches.Length -gt 1) { | ||
benbp marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
| throw "This script is not prepared to handle more than one batch. We will need to duplicate the input objects." | ||
| } | ||
| else { | ||
| foreach($config in $generatedMatrix) { | ||
| $namesForBatch = $batches[0] -join "," | ||
benbp marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
| # we just need to iterate across them, grab the parameters hashtable, and add the new key | ||
| # if there is more than one batch, we will need to add a suffix including the batch name to the job name | ||
| $config["parameters"]["$PRMatrixSetting"] = $namesForBatch | ||
| $OverallResult += $config | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| $serialized = SerializePipelineMatrix $OverallResult | ||
|
|
||
| Write-Output $serialized.pretty | ||
|
|
||
| if ($CI) { | ||
| Write-Output "##vso[task.setVariable variable=matrix;isOutput=true]$($serialized.compressed)" | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.