From 320b944453708fd62ac27b50ba6990e27fe6439b Mon Sep 17 00:00:00 2001 From: Patrick Hallisey Date: Wed, 28 Aug 2024 13:07:04 -0700 Subject: [PATCH 1/4] Add save-package-properties yaml --- .../steps/save-package-properties.yml | 40 +++++++++++++++++++ eng/common/scripts/Generate-PR-Diff.ps1 | 7 +++- 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 eng/common/pipelines/templates/steps/save-package-properties.yml diff --git a/eng/common/pipelines/templates/steps/save-package-properties.yml b/eng/common/pipelines/templates/steps/save-package-properties.yml new file mode 100644 index 000000000000..1d4c45613733 --- /dev/null +++ b/eng/common/pipelines/templates/steps/save-package-properties.yml @@ -0,0 +1,40 @@ +parameters: + - name: ServiceDirectory + type: string + default: '' + - name: DiffDirectory + type: string + default: '$(Build.ArtifactStagingDirectory)/diff' + - name: PackageInfoDirectory + type: string + default: '$(Build.ArtifactStagingDirectory)/PackageInfo' + +steps: + - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - task: Powershell@2 + displayName: Generate PR Diff + inputs: + filePath: ${{ parameters.ScriptDirectory }}/Generate-PR-Diff.ps1 + arguments: > + -TargetPath '$(Build.SourcesDirectory)' + -ArtifactPath '${{ parameters.DiffDirectory }}' + pwsh: true + + - task: Powershell@2 + displayName: Save package properties filtered for PR + inputs: + filePath: ${{ parameters.ScriptDirectory }}/Save-Package-Properties.ps1 + arguments: > + -PrDiff $(Build.ArtifactStagingDirectory)/diff/diff.json + -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo + pwsh: true + - ${{ else }}: + - task: Powershell@2 + displayName: Save package properties + inputs: + filePath: ${{ parameters.ScriptDirectory }}/Save-Package-Properties.ps1 + arguments: > + -ServiceDirectory ${{parameters.ServiceDirectory}} + -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo + -AddDevVersion:$${{ eq(variables['SetDevVersion'],'true') }} + pwsh: true \ No newline at end of file diff --git a/eng/common/scripts/Generate-PR-Diff.ps1 b/eng/common/scripts/Generate-PR-Diff.ps1 index 7a31456b36ca..64d07313eb3d 100644 --- a/eng/common/scripts/Generate-PR-Diff.ps1 +++ b/eng/common/scripts/Generate-PR-Diff.ps1 @@ -48,4 +48,9 @@ $result = [PSCustomObject]@{ "PRNumber" = if ($env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER) { $env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER } else { "-1" } } -$result | ConvertTo-Json | Out-File $ArtifactName +$json = $result | ConvertTo-Json +$json | Out-File $ArtifactName + +Write-Host "Generated diff.json file at $ArtifactName" +" " + ($json -replace '\n', '\n ') | Out-Host + From 9dde04ca9b0b4c0494552dc8c08da909ad498165 Mon Sep 17 00:00:00 2001 From: Patrick Hallisey Date: Wed, 28 Aug 2024 13:49:38 -0700 Subject: [PATCH 2/4] Encapsulate Save-Package-Properties.ps1 invocation into yaml --- .../templates/steps/save-package-properties.yml | 15 +++++++++------ eng/common/scripts/Generate-PR-Diff.ps1 | 5 ++--- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/eng/common/pipelines/templates/steps/save-package-properties.yml b/eng/common/pipelines/templates/steps/save-package-properties.yml index 1d4c45613733..7a3005eae78d 100644 --- a/eng/common/pipelines/templates/steps/save-package-properties.yml +++ b/eng/common/pipelines/templates/steps/save-package-properties.yml @@ -4,10 +4,13 @@ parameters: default: '' - name: DiffDirectory type: string - default: '$(Build.ArtifactStagingDirectory)/diff' + default: $(Build.ArtifactStagingDirectory)/diff - name: PackageInfoDirectory type: string - default: '$(Build.ArtifactStagingDirectory)/PackageInfo' + default: $(Build.ArtifactStagingDirectory)/PackageInfo + - name: TargetPath + type: string + default: $(Build.SourcesDirectory) steps: - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: @@ -16,7 +19,7 @@ steps: inputs: filePath: ${{ parameters.ScriptDirectory }}/Generate-PR-Diff.ps1 arguments: > - -TargetPath '$(Build.SourcesDirectory)' + -TargetPath '${{ parameters.TargetPath }}' -ArtifactPath '${{ parameters.DiffDirectory }}' pwsh: true @@ -25,8 +28,8 @@ steps: inputs: filePath: ${{ parameters.ScriptDirectory }}/Save-Package-Properties.ps1 arguments: > - -PrDiff $(Build.ArtifactStagingDirectory)/diff/diff.json - -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo + -PrDiff ${{ parameters.DiffDirectory }}/diff.json + -OutDirectory ${{ parameters.PackageInfoDirectory }} pwsh: true - ${{ else }}: - task: Powershell@2 @@ -35,6 +38,6 @@ steps: filePath: ${{ parameters.ScriptDirectory }}/Save-Package-Properties.ps1 arguments: > -ServiceDirectory ${{parameters.ServiceDirectory}} - -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo + -OutDirectory ${{ parameters.PackageInfoDirectory }} -AddDevVersion:$${{ eq(variables['SetDevVersion'],'true') }} pwsh: true \ No newline at end of file diff --git a/eng/common/scripts/Generate-PR-Diff.ps1 b/eng/common/scripts/Generate-PR-Diff.ps1 index 64d07313eb3d..a93d2ac85caf 100644 --- a/eng/common/scripts/Generate-PR-Diff.ps1 +++ b/eng/common/scripts/Generate-PR-Diff.ps1 @@ -51,6 +51,5 @@ $result = [PSCustomObject]@{ $json = $result | ConvertTo-Json $json | Out-File $ArtifactName -Write-Host "Generated diff.json file at $ArtifactName" -" " + ($json -replace '\n', '\n ') | Out-Host - +Write-Host "`nGenerated $ArtifactName`:" +' ' + ($json -replace "`n", "`n ") | Out-Host From 54a509f46c68a068c144a465ce2af8406118c6f2 Mon Sep 17 00:00:00 2001 From: Patrick Hallisey Date: Thu, 29 Aug 2024 13:13:42 -0700 Subject: [PATCH 3/4] Fix output formatting --- .../steps/save-package-properties.yml | 7 +++- eng/common/scripts/Generate-PR-Diff.ps1 | 28 ++++++------- .../scripts/Save-Package-Properties.ps1 | 42 +++++++++---------- 3 files changed, 38 insertions(+), 39 deletions(-) diff --git a/eng/common/pipelines/templates/steps/save-package-properties.yml b/eng/common/pipelines/templates/steps/save-package-properties.yml index 7a3005eae78d..7560f69e3213 100644 --- a/eng/common/pipelines/templates/steps/save-package-properties.yml +++ b/eng/common/pipelines/templates/steps/save-package-properties.yml @@ -1,7 +1,7 @@ parameters: - name: ServiceDirectory type: string - default: '' + default: "" - name: DiffDirectory type: string default: $(Build.ArtifactStagingDirectory)/diff @@ -11,6 +11,9 @@ parameters: - name: TargetPath type: string default: $(Build.SourcesDirectory) + - name: ScriptDirectory + type: string + default: eng/common/scripts steps: - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: @@ -40,4 +43,4 @@ steps: -ServiceDirectory ${{parameters.ServiceDirectory}} -OutDirectory ${{ parameters.PackageInfoDirectory }} -AddDevVersion:$${{ eq(variables['SetDevVersion'],'true') }} - pwsh: true \ No newline at end of file + pwsh: true diff --git a/eng/common/scripts/Generate-PR-Diff.ps1 b/eng/common/scripts/Generate-PR-Diff.ps1 index a93d2ac85caf..d250025f0c41 100644 --- a/eng/common/scripts/Generate-PR-Diff.ps1 +++ b/eng/common/scripts/Generate-PR-Diff.ps1 @@ -13,27 +13,27 @@ The path under which changes will be detected. #> [CmdletBinding()] Param ( - [Parameter(Mandatory=$True)] + [Parameter(Mandatory = $True)] [string] $ArtifactPath, - [Parameter(Mandatory=$True)] + [Parameter(Mandatory = $True)] [string] $TargetPath ) . (Join-Path $PSScriptRoot "Helpers" "git-helpers.ps1") function Get-ChangedServices { - Param ( - [Parameter(Mandatory=$True)] - [string[]] $ChangedFiles - ) + Param ( + [Parameter(Mandatory = $True)] + [string[]] $ChangedFiles + ) - $changedServices = $ChangedFiles | Foreach-Object { if ($_ -match "sdk/([^/]+)") { $matches[1] } } | Sort-Object -Unique + $changedServices = $ChangedFiles | Foreach-Object { if ($_ -match "sdk/([^/]+)") { $matches[1] } } | Sort-Object -Unique - return $changedServices + return $changedServices } if (!(Test-Path $ArtifactPath)) { - New-Item -ItemType Directory -Path $ArtifactPath | Out-Null + New-Item -ItemType Directory -Path $ArtifactPath | Out-Null } $ArtifactPath = Resolve-Path $ArtifactPath @@ -43,13 +43,13 @@ $changedFiles = Get-ChangedFiles -DiffPath $TargetPath $changedServices = Get-ChangedServices -ChangedFiles $changedFiles $result = [PSCustomObject]@{ - "ChangedFiles" = $changedFiles - "ChangedServices" = $changedServices - "PRNumber" = if ($env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER) { $env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER } else { "-1" } + "ChangedFiles" = $changedFiles + "ChangedServices" = $changedServices + "PRNumber" = if ($env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER) { $env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER } else { "-1" } } $json = $result | ConvertTo-Json $json | Out-File $ArtifactName -Write-Host "`nGenerated $ArtifactName`:" -' ' + ($json -replace "`n", "`n ") | Out-Host +Write-Host "`nGenerated diff.json file at $ArtifactName" +Write-Host " $($json -replace "`n", "`n ")" diff --git a/eng/common/scripts/Save-Package-Properties.ps1 b/eng/common/scripts/Save-Package-Properties.ps1 index 1922da5adff4..5d40e839da82 100644 --- a/eng/common/scripts/Save-Package-Properties.ps1 +++ b/eng/common/scripts/Save-Package-Properties.ps1 @@ -7,10 +7,10 @@ Saves package properties in source of a given service directory to JSON files. JSON files are named in the form .json or .json if an artifact name property is available in the package properties. -Can optionally add a dev version property which can be used logic for daily +Can optionally add a dev version property which can be used logic for daily builds. -In cases of collisions where track 2 packages (IsNewSdk = true) have the same +In cases of collisions where track 2 packages (IsNewSdk = true) have the same filename as track 1 packages (e.g. same artifact name or package name), the track 2 package properties will be written. @@ -21,22 +21,22 @@ Service directory in which to search for packages. A file path leading to a file generated from Generate-PR-Diff.json. This parameter takes precedence over serviceDirectory, do not provide both. .PARAMETER outDirectory -Output location (generally a package artifact directory in DevOps) for JSON +Output location (generally a package artifact directory in DevOps) for JSON files .PARAMETER addDevVersion -Reads the version out of the source and adds a DevVersion property to the -package properties JSON file. If the package properties JSON file already +Reads the version out of the source and adds a DevVersion property to the +package properties JSON file. If the package properties JSON file already exists, read the Version property from the existing package properties JSON file and set that as the Version property for the new output. This has the effect of -"adding" a DevVersion property to the file which could be different from the +"adding" a DevVersion property to the file which could be different from the Verison property in that file. #> [CmdletBinding()] Param ( [string] $serviceDirectory, - [Parameter(Mandatory=$True)] + [Parameter(Mandatory = $True)] [string] $outDirectory, [string] $prDiff, [switch] $addDevVersion @@ -47,19 +47,16 @@ Param ( function SetOutput($outputPath, $incomingPackageSpec) { # If there is an exsiting package info json file read that and set that as output object which gets properties updated here. - if (Test-Path $outputPath) - { + if (Test-Path $outputPath) { Write-Host "Found existing package info json." $outputObject = ConvertFrom-Json (Get-Content $outputPath -Raw) } - else - { + else { $outputObject = $incomingPackageSpec } - - if ($addDevVersion) - { + + if ($addDevVersion) { # Use the "Version" property which was provided by the incoming package spec # as the DevVersion. This may be overridden later. $outputObject.DevVersion = $incomingPackageSpec.Version @@ -119,29 +116,26 @@ else { } } -if (-not (Test-Path -Path $outDirectory)) -{ +if (-not (Test-Path -Path $outDirectory)) { New-Item -ItemType Directory -Force -Path $outDirectory | Out-Null } -foreach($pkg in $allPackageProperties) -{ +foreach ($pkg in $allPackageProperties) { if ($pkg.Name) { + Write-Host "" Write-Host "Package Name: $($pkg.Name)" Write-Host "Package Version: $($pkg.Version)" Write-Host "Package SDK Type: $($pkg.SdkType)" Write-Host "Artifact Name: $($pkg.ArtifactName)" Write-Host "Release date: $($pkg.ReleaseStatus)" $configFilePrefix = $pkg.Name - if ($pkg.ArtifactName) - { + if ($pkg.ArtifactName) { $configFilePrefix = $pkg.ArtifactName } $outputPath = Join-Path -Path $outDirectory "$configFilePrefix.json" Write-Host "Output path of json file: $outputPath" $outDir = Split-Path $outputPath -parent - if (-not (Test-Path -path $outDir)) - { + if (-not (Test-Path -path $outDir)) { Write-Host "Creating directory $($outDir) for json property file" New-Item -ItemType Directory -Path $outDir | Out-Null } @@ -158,4 +152,6 @@ foreach($pkg in $allPackageProperties) } } -Get-ChildItem -Path $outDirectory +$fileNames = (Get-ChildItem -Path $outDirectory).Name +Write-Host "`nFiles written to $outDirectory`:" +Write-Host " $($fileNames -join "`n ")" From d74ba1261596416ab1e627720bd95de99567c5ef Mon Sep 17 00:00:00 2001 From: Patrick Hallisey Date: Thu, 29 Aug 2024 13:31:15 -0700 Subject: [PATCH 4/4] Fix code style --- eng/common/scripts/Generate-PR-Diff.ps1 | 6 +- .../scripts/Save-Package-Properties.ps1 | 56 +++++++++++++------ 2 files changed, 42 insertions(+), 20 deletions(-) diff --git a/eng/common/scripts/Generate-PR-Diff.ps1 b/eng/common/scripts/Generate-PR-Diff.ps1 index d250025f0c41..3b508b1cb994 100644 --- a/eng/common/scripts/Generate-PR-Diff.ps1 +++ b/eng/common/scripts/Generate-PR-Diff.ps1 @@ -21,7 +21,8 @@ Param ( . (Join-Path $PSScriptRoot "Helpers" "git-helpers.ps1") -function Get-ChangedServices { +function Get-ChangedServices +{ Param ( [Parameter(Mandatory = $True)] [string[]] $ChangedFiles @@ -32,7 +33,8 @@ function Get-ChangedServices { return $changedServices } -if (!(Test-Path $ArtifactPath)) { +if (!(Test-Path $ArtifactPath)) +{ New-Item -ItemType Directory -Path $ArtifactPath | Out-Null } diff --git a/eng/common/scripts/Save-Package-Properties.ps1 b/eng/common/scripts/Save-Package-Properties.ps1 index 5d40e839da82..0257e73a1545 100644 --- a/eng/common/scripts/Save-Package-Properties.ps1 +++ b/eng/common/scripts/Save-Package-Properties.ps1 @@ -44,19 +44,23 @@ Param ( . (Join-Path $PSScriptRoot common.ps1) -function SetOutput($outputPath, $incomingPackageSpec) { +function SetOutput($outputPath, $incomingPackageSpec) +{ # If there is an exsiting package info json file read that and set that as output object which gets properties updated here. - if (Test-Path $outputPath) { + if (Test-Path $outputPath) + { Write-Host "Found existing package info json." $outputObject = ConvertFrom-Json (Get-Content $outputPath -Raw) } - else { + else + { $outputObject = $incomingPackageSpec } - if ($addDevVersion) { + if ($addDevVersion) + { # Use the "Version" property which was provided by the incoming package spec # as the DevVersion. This may be overridden later. $outputObject.DevVersion = $incomingPackageSpec.Version @@ -72,16 +76,19 @@ function SetOutput($outputPath, $incomingPackageSpec) { -Value (ConvertTo-Json -InputObject $outputObject -Depth 100) } -function GetRelativePath($path) { +function GetRelativePath($path) +{ # If the path is empty return an empty string - if (!$path) { + if (!$path) + { return '' } # If the path is already relative return the path. Calling `GetRelativePath` # on a relative path converts the relative path to an absolute path based on # the current working directory which can result in unexpected outputs. - if (![IO.Path]::IsPathRooted($path)) { + if (![IO.Path]::IsPathRooted($path)) + { return $path } @@ -95,33 +102,40 @@ $exportedPaths = @{} $allPackageProperties = @() -if ($prDiff) { +if ($prDiff) +{ Write-Host "Getting package properties for PR diff file: $prDiff" $allPackageProperties = Get-PrPkgProperties $prDiff - if (!$allPackageProperties) { + if (!$allPackageProperties) + { Write-Host "No packages found matching PR diff file $prDiff" Write-Host "Setting NoPackagesChanged variable to true" Write-Host "##vso[task.setvariable variable=NoPackagesChanged]true" exit 0 } } -else { +else +{ Write-Host "Getting package properties for service directory: $serviceDirectory" $allPackageProperties = Get-AllPkgProperties $serviceDirectory - if (!$allPackageProperties) { + if (!$allPackageProperties) + { Write-Error "Package properties are not available for service directory $serviceDirectory" exit 1 } } -if (-not (Test-Path -Path $outDirectory)) { +if (-not (Test-Path -Path $outDirectory)) +{ New-Item -ItemType Directory -Force -Path $outDirectory | Out-Null } -foreach ($pkg in $allPackageProperties) { - if ($pkg.Name) { +foreach ($pkg in $allPackageProperties) +{ + if ($pkg.Name) + { Write-Host "" Write-Host "Package Name: $($pkg.Name)" Write-Host "Package Version: $($pkg.Version)" @@ -129,25 +143,31 @@ foreach ($pkg in $allPackageProperties) { Write-Host "Artifact Name: $($pkg.ArtifactName)" Write-Host "Release date: $($pkg.ReleaseStatus)" $configFilePrefix = $pkg.Name - if ($pkg.ArtifactName) { + + if ($pkg.ArtifactName) + { $configFilePrefix = $pkg.ArtifactName } + $outputPath = Join-Path -Path $outDirectory "$configFilePrefix.json" Write-Host "Output path of json file: $outputPath" + $outDir = Split-Path $outputPath -parent - if (-not (Test-Path -path $outDir)) { + if (-not (Test-Path -path $outDir)) + { Write-Host "Creating directory $($outDir) for json property file" New-Item -ItemType Directory -Path $outDir | Out-Null } # If package properties for a track 2 (IsNewSdk = true) package has # already been written, skip writing to that same path. - if ($exportedPaths.ContainsKey($outputPath) -and $exportedPaths[$outputPath].IsNewSdk -eq $true) { + if ($exportedPaths.ContainsKey($outputPath) -and $exportedPaths[$outputPath].IsNewSdk -eq $true) + { Write-Host "Track 2 package info with file name $($outputPath) already exported. Skipping export." continue } - $exportedPaths[$outputPath] = $pkg + $exportedPaths[$outputPath] = $pkg SetOutput $outputPath $pkg } }