From 447bb36d9f38b708df671c0717425dee9fd94f84 Mon Sep 17 00:00:00 2001 From: Patrick Hallisey Date: Wed, 28 Aug 2024 13:07:04 -0700 Subject: [PATCH 1/4] Add save-package-properties yaml --- .../steps/save-package-properties.yml | 40 +++++++++++++++++++ eng/common/scripts/Generate-PR-Diff.ps1 | 7 +++- 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 eng/common/pipelines/templates/steps/save-package-properties.yml diff --git a/eng/common/pipelines/templates/steps/save-package-properties.yml b/eng/common/pipelines/templates/steps/save-package-properties.yml new file mode 100644 index 0000000000..1d4c456137 --- /dev/null +++ b/eng/common/pipelines/templates/steps/save-package-properties.yml @@ -0,0 +1,40 @@ +parameters: + - name: ServiceDirectory + type: string + default: '' + - name: DiffDirectory + type: string + default: '$(Build.ArtifactStagingDirectory)/diff' + - name: PackageInfoDirectory + type: string + default: '$(Build.ArtifactStagingDirectory)/PackageInfo' + +steps: + - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - task: Powershell@2 + displayName: Generate PR Diff + inputs: + filePath: ${{ parameters.ScriptDirectory }}/Generate-PR-Diff.ps1 + arguments: > + -TargetPath '$(Build.SourcesDirectory)' + -ArtifactPath '${{ parameters.DiffDirectory }}' + pwsh: true + + - task: Powershell@2 + displayName: Save package properties filtered for PR + inputs: + filePath: ${{ parameters.ScriptDirectory }}/Save-Package-Properties.ps1 + arguments: > + -PrDiff $(Build.ArtifactStagingDirectory)/diff/diff.json + -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo + pwsh: true + - ${{ else }}: + - task: Powershell@2 + displayName: Save package properties + inputs: + filePath: ${{ parameters.ScriptDirectory }}/Save-Package-Properties.ps1 + arguments: > + -ServiceDirectory ${{parameters.ServiceDirectory}} + -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo + -AddDevVersion:$${{ eq(variables['SetDevVersion'],'true') }} + pwsh: true \ No newline at end of file diff --git a/eng/common/scripts/Generate-PR-Diff.ps1 b/eng/common/scripts/Generate-PR-Diff.ps1 index 7a31456b36..64d07313eb 100644 --- a/eng/common/scripts/Generate-PR-Diff.ps1 +++ b/eng/common/scripts/Generate-PR-Diff.ps1 @@ -48,4 +48,9 @@ $result = [PSCustomObject]@{ "PRNumber" = if ($env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER) { $env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER } else { "-1" } } -$result | ConvertTo-Json | Out-File $ArtifactName +$json = $result | ConvertTo-Json +$json | Out-File $ArtifactName + +Write-Host "Generated diff.json file at $ArtifactName" +" " + ($json -replace '\n', '\n ') | Out-Host + From 37fcd5a2db3b435739e2742367a9fbd622313dd5 Mon Sep 17 00:00:00 2001 From: Patrick Hallisey Date: Wed, 28 Aug 2024 13:49:38 -0700 Subject: [PATCH 2/4] Encapsulate Save-Package-Properties.ps1 invocation into yaml --- .../templates/steps/save-package-properties.yml | 15 +++++++++------ eng/common/scripts/Generate-PR-Diff.ps1 | 5 ++--- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/eng/common/pipelines/templates/steps/save-package-properties.yml b/eng/common/pipelines/templates/steps/save-package-properties.yml index 1d4c456137..7a3005eae7 100644 --- a/eng/common/pipelines/templates/steps/save-package-properties.yml +++ b/eng/common/pipelines/templates/steps/save-package-properties.yml @@ -4,10 +4,13 @@ parameters: default: '' - name: DiffDirectory type: string - default: '$(Build.ArtifactStagingDirectory)/diff' + default: $(Build.ArtifactStagingDirectory)/diff - name: PackageInfoDirectory type: string - default: '$(Build.ArtifactStagingDirectory)/PackageInfo' + default: $(Build.ArtifactStagingDirectory)/PackageInfo + - name: TargetPath + type: string + default: $(Build.SourcesDirectory) steps: - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: @@ -16,7 +19,7 @@ steps: inputs: filePath: ${{ parameters.ScriptDirectory }}/Generate-PR-Diff.ps1 arguments: > - -TargetPath '$(Build.SourcesDirectory)' + -TargetPath '${{ parameters.TargetPath }}' -ArtifactPath '${{ parameters.DiffDirectory }}' pwsh: true @@ -25,8 +28,8 @@ steps: inputs: filePath: ${{ parameters.ScriptDirectory }}/Save-Package-Properties.ps1 arguments: > - -PrDiff $(Build.ArtifactStagingDirectory)/diff/diff.json - -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo + -PrDiff ${{ parameters.DiffDirectory }}/diff.json + -OutDirectory ${{ parameters.PackageInfoDirectory }} pwsh: true - ${{ else }}: - task: Powershell@2 @@ -35,6 +38,6 @@ steps: filePath: ${{ parameters.ScriptDirectory }}/Save-Package-Properties.ps1 arguments: > -ServiceDirectory ${{parameters.ServiceDirectory}} - -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo + -OutDirectory ${{ parameters.PackageInfoDirectory }} -AddDevVersion:$${{ eq(variables['SetDevVersion'],'true') }} pwsh: true \ No newline at end of file diff --git a/eng/common/scripts/Generate-PR-Diff.ps1 b/eng/common/scripts/Generate-PR-Diff.ps1 index 64d07313eb..a93d2ac85c 100644 --- a/eng/common/scripts/Generate-PR-Diff.ps1 +++ b/eng/common/scripts/Generate-PR-Diff.ps1 @@ -51,6 +51,5 @@ $result = [PSCustomObject]@{ $json = $result | ConvertTo-Json $json | Out-File $ArtifactName -Write-Host "Generated diff.json file at $ArtifactName" -" " + ($json -replace '\n', '\n ') | Out-Host - +Write-Host "`nGenerated $ArtifactName`:" +' ' + ($json -replace "`n", "`n ") | Out-Host From ef98878b03f8d500facc84022100f6f64d96084a Mon Sep 17 00:00:00 2001 From: Patrick Hallisey Date: Thu, 29 Aug 2024 13:13:42 -0700 Subject: [PATCH 3/4] Fix output formatting --- .../steps/save-package-properties.yml | 7 +++- eng/common/scripts/Generate-PR-Diff.ps1 | 28 ++++++------- .../scripts/Save-Package-Properties.ps1 | 42 +++++++++---------- 3 files changed, 38 insertions(+), 39 deletions(-) diff --git a/eng/common/pipelines/templates/steps/save-package-properties.yml b/eng/common/pipelines/templates/steps/save-package-properties.yml index 7a3005eae7..7560f69e32 100644 --- a/eng/common/pipelines/templates/steps/save-package-properties.yml +++ b/eng/common/pipelines/templates/steps/save-package-properties.yml @@ -1,7 +1,7 @@ parameters: - name: ServiceDirectory type: string - default: '' + default: "" - name: DiffDirectory type: string default: $(Build.ArtifactStagingDirectory)/diff @@ -11,6 +11,9 @@ parameters: - name: TargetPath type: string default: $(Build.SourcesDirectory) + - name: ScriptDirectory + type: string + default: eng/common/scripts steps: - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: @@ -40,4 +43,4 @@ steps: -ServiceDirectory ${{parameters.ServiceDirectory}} -OutDirectory ${{ parameters.PackageInfoDirectory }} -AddDevVersion:$${{ eq(variables['SetDevVersion'],'true') }} - pwsh: true \ No newline at end of file + pwsh: true diff --git a/eng/common/scripts/Generate-PR-Diff.ps1 b/eng/common/scripts/Generate-PR-Diff.ps1 index a93d2ac85c..d250025f0c 100644 --- a/eng/common/scripts/Generate-PR-Diff.ps1 +++ b/eng/common/scripts/Generate-PR-Diff.ps1 @@ -13,27 +13,27 @@ The path under which changes will be detected. #> [CmdletBinding()] Param ( - [Parameter(Mandatory=$True)] + [Parameter(Mandatory = $True)] [string] $ArtifactPath, - [Parameter(Mandatory=$True)] + [Parameter(Mandatory = $True)] [string] $TargetPath ) . (Join-Path $PSScriptRoot "Helpers" "git-helpers.ps1") function Get-ChangedServices { - Param ( - [Parameter(Mandatory=$True)] - [string[]] $ChangedFiles - ) + Param ( + [Parameter(Mandatory = $True)] + [string[]] $ChangedFiles + ) - $changedServices = $ChangedFiles | Foreach-Object { if ($_ -match "sdk/([^/]+)") { $matches[1] } } | Sort-Object -Unique + $changedServices = $ChangedFiles | Foreach-Object { if ($_ -match "sdk/([^/]+)") { $matches[1] } } | Sort-Object -Unique - return $changedServices + return $changedServices } if (!(Test-Path $ArtifactPath)) { - New-Item -ItemType Directory -Path $ArtifactPath | Out-Null + New-Item -ItemType Directory -Path $ArtifactPath | Out-Null } $ArtifactPath = Resolve-Path $ArtifactPath @@ -43,13 +43,13 @@ $changedFiles = Get-ChangedFiles -DiffPath $TargetPath $changedServices = Get-ChangedServices -ChangedFiles $changedFiles $result = [PSCustomObject]@{ - "ChangedFiles" = $changedFiles - "ChangedServices" = $changedServices - "PRNumber" = if ($env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER) { $env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER } else { "-1" } + "ChangedFiles" = $changedFiles + "ChangedServices" = $changedServices + "PRNumber" = if ($env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER) { $env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER } else { "-1" } } $json = $result | ConvertTo-Json $json | Out-File $ArtifactName -Write-Host "`nGenerated $ArtifactName`:" -' ' + ($json -replace "`n", "`n ") | Out-Host +Write-Host "`nGenerated diff.json file at $ArtifactName" +Write-Host " $($json -replace "`n", "`n ")" diff --git a/eng/common/scripts/Save-Package-Properties.ps1 b/eng/common/scripts/Save-Package-Properties.ps1 index 1922da5adf..5d40e839da 100644 --- a/eng/common/scripts/Save-Package-Properties.ps1 +++ b/eng/common/scripts/Save-Package-Properties.ps1 @@ -7,10 +7,10 @@ Saves package properties in source of a given service directory to JSON files. JSON files are named in the form .json or .json if an artifact name property is available in the package properties. -Can optionally add a dev version property which can be used logic for daily +Can optionally add a dev version property which can be used logic for daily builds. -In cases of collisions where track 2 packages (IsNewSdk = true) have the same +In cases of collisions where track 2 packages (IsNewSdk = true) have the same filename as track 1 packages (e.g. same artifact name or package name), the track 2 package properties will be written. @@ -21,22 +21,22 @@ Service directory in which to search for packages. A file path leading to a file generated from Generate-PR-Diff.json. This parameter takes precedence over serviceDirectory, do not provide both. .PARAMETER outDirectory -Output location (generally a package artifact directory in DevOps) for JSON +Output location (generally a package artifact directory in DevOps) for JSON files .PARAMETER addDevVersion -Reads the version out of the source and adds a DevVersion property to the -package properties JSON file. If the package properties JSON file already +Reads the version out of the source and adds a DevVersion property to the +package properties JSON file. If the package properties JSON file already exists, read the Version property from the existing package properties JSON file and set that as the Version property for the new output. This has the effect of -"adding" a DevVersion property to the file which could be different from the +"adding" a DevVersion property to the file which could be different from the Verison property in that file. #> [CmdletBinding()] Param ( [string] $serviceDirectory, - [Parameter(Mandatory=$True)] + [Parameter(Mandatory = $True)] [string] $outDirectory, [string] $prDiff, [switch] $addDevVersion @@ -47,19 +47,16 @@ Param ( function SetOutput($outputPath, $incomingPackageSpec) { # If there is an exsiting package info json file read that and set that as output object which gets properties updated here. - if (Test-Path $outputPath) - { + if (Test-Path $outputPath) { Write-Host "Found existing package info json." $outputObject = ConvertFrom-Json (Get-Content $outputPath -Raw) } - else - { + else { $outputObject = $incomingPackageSpec } - - if ($addDevVersion) - { + + if ($addDevVersion) { # Use the "Version" property which was provided by the incoming package spec # as the DevVersion. This may be overridden later. $outputObject.DevVersion = $incomingPackageSpec.Version @@ -119,29 +116,26 @@ else { } } -if (-not (Test-Path -Path $outDirectory)) -{ +if (-not (Test-Path -Path $outDirectory)) { New-Item -ItemType Directory -Force -Path $outDirectory | Out-Null } -foreach($pkg in $allPackageProperties) -{ +foreach ($pkg in $allPackageProperties) { if ($pkg.Name) { + Write-Host "" Write-Host "Package Name: $($pkg.Name)" Write-Host "Package Version: $($pkg.Version)" Write-Host "Package SDK Type: $($pkg.SdkType)" Write-Host "Artifact Name: $($pkg.ArtifactName)" Write-Host "Release date: $($pkg.ReleaseStatus)" $configFilePrefix = $pkg.Name - if ($pkg.ArtifactName) - { + if ($pkg.ArtifactName) { $configFilePrefix = $pkg.ArtifactName } $outputPath = Join-Path -Path $outDirectory "$configFilePrefix.json" Write-Host "Output path of json file: $outputPath" $outDir = Split-Path $outputPath -parent - if (-not (Test-Path -path $outDir)) - { + if (-not (Test-Path -path $outDir)) { Write-Host "Creating directory $($outDir) for json property file" New-Item -ItemType Directory -Path $outDir | Out-Null } @@ -158,4 +152,6 @@ foreach($pkg in $allPackageProperties) } } -Get-ChildItem -Path $outDirectory +$fileNames = (Get-ChildItem -Path $outDirectory).Name +Write-Host "`nFiles written to $outDirectory`:" +Write-Host " $($fileNames -join "`n ")" From e9d26f95f9bf37ac8b652c398dd5cadc7fb7e2ed Mon Sep 17 00:00:00 2001 From: Patrick Hallisey Date: Thu, 29 Aug 2024 13:31:15 -0700 Subject: [PATCH 4/4] Fix code style --- eng/common/scripts/Generate-PR-Diff.ps1 | 6 +- .../scripts/Save-Package-Properties.ps1 | 56 +++++++++++++------ 2 files changed, 42 insertions(+), 20 deletions(-) diff --git a/eng/common/scripts/Generate-PR-Diff.ps1 b/eng/common/scripts/Generate-PR-Diff.ps1 index d250025f0c..3b508b1cb9 100644 --- a/eng/common/scripts/Generate-PR-Diff.ps1 +++ b/eng/common/scripts/Generate-PR-Diff.ps1 @@ -21,7 +21,8 @@ Param ( . (Join-Path $PSScriptRoot "Helpers" "git-helpers.ps1") -function Get-ChangedServices { +function Get-ChangedServices +{ Param ( [Parameter(Mandatory = $True)] [string[]] $ChangedFiles @@ -32,7 +33,8 @@ function Get-ChangedServices { return $changedServices } -if (!(Test-Path $ArtifactPath)) { +if (!(Test-Path $ArtifactPath)) +{ New-Item -ItemType Directory -Path $ArtifactPath | Out-Null } diff --git a/eng/common/scripts/Save-Package-Properties.ps1 b/eng/common/scripts/Save-Package-Properties.ps1 index 5d40e839da..0257e73a15 100644 --- a/eng/common/scripts/Save-Package-Properties.ps1 +++ b/eng/common/scripts/Save-Package-Properties.ps1 @@ -44,19 +44,23 @@ Param ( . (Join-Path $PSScriptRoot common.ps1) -function SetOutput($outputPath, $incomingPackageSpec) { +function SetOutput($outputPath, $incomingPackageSpec) +{ # If there is an exsiting package info json file read that and set that as output object which gets properties updated here. - if (Test-Path $outputPath) { + if (Test-Path $outputPath) + { Write-Host "Found existing package info json." $outputObject = ConvertFrom-Json (Get-Content $outputPath -Raw) } - else { + else + { $outputObject = $incomingPackageSpec } - if ($addDevVersion) { + if ($addDevVersion) + { # Use the "Version" property which was provided by the incoming package spec # as the DevVersion. This may be overridden later. $outputObject.DevVersion = $incomingPackageSpec.Version @@ -72,16 +76,19 @@ function SetOutput($outputPath, $incomingPackageSpec) { -Value (ConvertTo-Json -InputObject $outputObject -Depth 100) } -function GetRelativePath($path) { +function GetRelativePath($path) +{ # If the path is empty return an empty string - if (!$path) { + if (!$path) + { return '' } # If the path is already relative return the path. Calling `GetRelativePath` # on a relative path converts the relative path to an absolute path based on # the current working directory which can result in unexpected outputs. - if (![IO.Path]::IsPathRooted($path)) { + if (![IO.Path]::IsPathRooted($path)) + { return $path } @@ -95,33 +102,40 @@ $exportedPaths = @{} $allPackageProperties = @() -if ($prDiff) { +if ($prDiff) +{ Write-Host "Getting package properties for PR diff file: $prDiff" $allPackageProperties = Get-PrPkgProperties $prDiff - if (!$allPackageProperties) { + if (!$allPackageProperties) + { Write-Host "No packages found matching PR diff file $prDiff" Write-Host "Setting NoPackagesChanged variable to true" Write-Host "##vso[task.setvariable variable=NoPackagesChanged]true" exit 0 } } -else { +else +{ Write-Host "Getting package properties for service directory: $serviceDirectory" $allPackageProperties = Get-AllPkgProperties $serviceDirectory - if (!$allPackageProperties) { + if (!$allPackageProperties) + { Write-Error "Package properties are not available for service directory $serviceDirectory" exit 1 } } -if (-not (Test-Path -Path $outDirectory)) { +if (-not (Test-Path -Path $outDirectory)) +{ New-Item -ItemType Directory -Force -Path $outDirectory | Out-Null } -foreach ($pkg in $allPackageProperties) { - if ($pkg.Name) { +foreach ($pkg in $allPackageProperties) +{ + if ($pkg.Name) + { Write-Host "" Write-Host "Package Name: $($pkg.Name)" Write-Host "Package Version: $($pkg.Version)" @@ -129,25 +143,31 @@ foreach ($pkg in $allPackageProperties) { Write-Host "Artifact Name: $($pkg.ArtifactName)" Write-Host "Release date: $($pkg.ReleaseStatus)" $configFilePrefix = $pkg.Name - if ($pkg.ArtifactName) { + + if ($pkg.ArtifactName) + { $configFilePrefix = $pkg.ArtifactName } + $outputPath = Join-Path -Path $outDirectory "$configFilePrefix.json" Write-Host "Output path of json file: $outputPath" + $outDir = Split-Path $outputPath -parent - if (-not (Test-Path -path $outDir)) { + if (-not (Test-Path -path $outDir)) + { Write-Host "Creating directory $($outDir) for json property file" New-Item -ItemType Directory -Path $outDir | Out-Null } # If package properties for a track 2 (IsNewSdk = true) package has # already been written, skip writing to that same path. - if ($exportedPaths.ContainsKey($outputPath) -and $exportedPaths[$outputPath].IsNewSdk -eq $true) { + if ($exportedPaths.ContainsKey($outputPath) -and $exportedPaths[$outputPath].IsNewSdk -eq $true) + { Write-Host "Track 2 package info with file name $($outputPath) already exported. Skipping export." continue } - $exportedPaths[$outputPath] = $pkg + $exportedPaths[$outputPath] = $pkg SetOutput $outputPath $pkg } }