Sync eng/common directory with azure-sdk-tools for PR 1725 (Azure#16079)

* Bring changes from JS docs metadata

* Move business logic inside Update-DocsMsMetadata.ps1

* Update with the latest changes in JS PR

* Update from latest PR feedback

* Add check for empty path to prevent crashes when creating relative paths

Co-authored-by: Daniel Jurek <djurek@microsoft.com>

eng/common/pipelines/templates/steps/enable-long-path-support.yml

@@ -0,0 +1,10 @@
+steps:
+- pwsh: |
+    if ($IsWindows) {
+      REG ADD HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem /f /v LongPathsEnabled /t REG_DWORD /d 1
+      git config --system core.longpaths true
+    }
+    else {
+      Write-Host "This script is not executing on Windows, skipping registry modification."
+    }
+  displayName: Enable long path support if necessary

eng/common/pipelines/templates/steps/set-daily-docs-branch-name.yml

@@ -0,0 +1,14 @@
+parameters:
+  - name: DailyBranchVariableName
+    type: string
+    default: TargetBranchName
+
+steps:
+  - pwsh: |
+      $branchName = $env:DAILYDOCSBRANCHNAMEOVERRIDE
+      if (!$branchName) { 
+        $branchName = "daily/$(Get-Date -Format 'yyyy-MM-dd')"
+      }
+      Write-Host "Daily Branch Name: $branchName"
+      Write-Host "##vso[task.setvariable variable=${{ parameters.DailyBranchVariableName }};]$branchName"
+    displayName: Set daily docs branch name in $(${{ parameters.DailyBranchVariableName }})

eng/common/pipelines/templates/steps/set-default-branch.yml

@@ -1,15 +1,16 @@
 parameters:
   WorkingDirectory: '$(System.DefaultWorkingDirectory)'
   RemoteRepo: 'origin'
+  DefaultBranchVariableName: DefaultBranch
 steps:
 - pwsh: |
     $setDefaultBranch = (git remote show ${{ parameters.RemoteRepo }} | Out-String) -replace "(?ms).*HEAD branch: (\w+).*", '$1'
     if ($LASTEXITCODE -ne 0) {
       Write-Host "Not able to fetch the default branch from git command. Set to main."
       $setDefaultBranch = 'main'
     }
-    Write-Host "Setting DefaultBranch=$setDefaultBranch"
-    Write-Host "##vso[task.setvariable variable=DefaultBranch]$setDefaultBranch"
+    Write-Host "Setting ${{ parameters.DefaultBranchVariableName }}=$setDefaultBranch"
+    Write-Host "##vso[task.setvariable variable=${{ parameters.DefaultBranchVariableName }}]$setDefaultBranch"
   displayName: "Setup Default Branch"
   workingDirectory: ${{ parameters.workingDirectory }}
   ignoreLASTEXITCODE: true

eng/common/pipelines/templates/steps/update-docsms-metadata.yml

@@ -0,0 +1,95 @@
+parameters:
+  - name: PackageInfoLocations
+    type: object
+    default: []
+  - name: RepoId
+    type: string
+    default: $(Build.Repository.Name)
+  - name: WorkingDirectory
+    type: string
+    default: ''
+  - name: ScriptDirectory
+    type: string
+    default: eng/common/scripts
+  - name: TargetDocRepoName
+    type: string
+    default: ''
+  - name: TargetDocRepoOwner
+    type: string
+  - name: Language
+    type: string
+    default: ''
+  - name: DailyDocsBuild
+    type: boolean
+    default: false
+  - name: SparseCheckoutPaths
+    type: object
+    default:
+      - '**'
+
+steps:
+- template: /eng/common/pipelines/templates/steps/enable-long-path-support.yml
+
+- pwsh: |
+    Write-Host "###vso[task.setvariable variable=DocRepoLocation]${{ parameters.WorkingDirectory }}/doc"
+  displayName: Set $(DocRepoLocation)
+
+- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+  parameters:
+    SkipDefaultCheckout: true
+    Repositories:
+      - Name: ${{ parameters.TargetDocRepoOwner }}/${{ parameters.TargetDocRepoName }}
+        WorkingDirectory: $(DocRepoLocation)
+    Paths: ${{ parameters.SparseCheckoutPaths }}
+
+# If performing a daily docs build set the $(TargetBranchName) to a daily branch
+# name and attempt to checkout the daily docs branch. If the branch doesn't
+# exist, create it
+- ${{ if eq(parameters.DailyDocsBuild, 'true') }}:
+  - template: /eng/common/pipelines/templates/steps/set-daily-docs-branch-name.yml
+
+  - pwsh: |
+      $ErrorActionPreference = "Continue"
+      $RemoteName = "origin"
+      $BranchName = "$(TargetBranchName)"
+      # Fetch and checkout remote branch if it already exists otherwise create a new branch.
+      git ls-remote --exit-code --heads $RemoteName $BranchName
+      if ($LASTEXITCODE -eq 0) {
+        Write-Host "git fetch $RemoteName $BranchName"
+        git fetch $RemoteName $BranchName
+        Write-Host "git checkout $BranchName."
+        git checkout $BranchName
+      } else {
+        Write-Host "git checkout -b $BranchName."
+        git checkout -b $BranchName
+      }
+    displayName: Checkout daily docs branch if it exists
+    workingDirectory: $(DocRepoLocation)
+
+# If NOT performing a daily docs build, set the $(TargetBranchName) to the 
+# default branch of the documentation repository.
+- ${{ if ne(parameters.DailyDocsBuild, 'true') }}:
+  - template: /eng/common/pipelines/templates/steps/set-default-branch.yml
+    parameters:
+      WorkingDirectory: $(DocRepoLocation)
+      DefaultBranchVariableName: TargetBranchName
+
+- pwsh: |
+    $packageInfoJson = '${{ convertToJson(parameters.PackageInfoLocations) }}'.Trim('"')
+    $packageInfoLocations = ConvertFrom-Json $packageInfoJson
+    ${{ parameters.ScriptDirectory }}/Update-DocsMsMetadata.ps1 `
+      -PackageInfoJsonLocations $packageInfoLocations `
+      -DocRepoLocation "$(DocRepoLocation)" `
+      -Language '${{parameters.Language}}' `
+      -RepoId '${{ parameters.RepoId }}'
+  displayName: Apply Documentation Updates
+
+- template: /eng/common/pipelines/templates/steps/git-push-changes.yml
+  parameters:
+    BaseRepoBranch: $(TargetBranchName)
+    BaseRepoOwner: ${{ parameters.TargetDocRepoOwner }}
+    CommitMsg: "Update docs metadata"
+    TargetRepoName: ${{ parameters.TargetDocRepoName }}
+    TargetRepoOwner: ${{ parameters.TargetDocRepoOwner }}
+    WorkingDirectory: $(DocRepoLocation)
+    ScriptDirectory: ${{ parameters.WorkingDirectory }}/${{ parameters.ScriptDirectory }}

eng/common/scripts/Invoke-DevOpsAPI.ps1

@@ -19,14 +19,17 @@ function Start-DevOpsBuild {
     $DefinitionId,
     [ValidateNotNullOrEmpty()]
     [Parameter(Mandatory = $true)]
-    $Base64EncodedAuthToken
+    $Base64EncodedAuthToken,
+    [Parameter(Mandatory = $false)]
+    [string]$BuildParametersJson
   )
 
   $uri = "$DevOpsAPIBaseURI" -F $Organization, $Project , "build" , "builds", ""
 
   $parameters = @{
     sourceBranch = $SourceBranch
     definition = @{ id = $DefinitionId }
+    parameters = $BuildParametersJson
   }
 
   return Invoke-RestMethod `
@@ -157,4 +160,4 @@ function Add-RetentionLease {
           -MaximumRetryCount 3 `
           -ContentType "application/json"
 
-}
+}

eng/common/scripts/Package-Properties.ps1

@@ -5,6 +5,7 @@ class PackageProps
 {
     [string]$Name
     [string]$Version
+    [string]$DevVersion
     [string]$DirectoryPath
     [string]$ServiceDirectory
     [string]$ReadMePath

eng/common/scripts/Queue-Pipeline.ps1

@@ -18,7 +18,10 @@ param(
   [string]$VsoQueuedPipelines,
 
   [Parameter(Mandatory = $true)]
-  [string]$Base64EncodedAuthToken
+  [string]$Base64EncodedAuthToken,
+
+  [Parameter(Mandatory = $false)]
+  [string]$BuildParametersJson
 )
 
 . (Join-Path $PSScriptRoot common.ps1)
@@ -46,7 +49,13 @@ if ($CancelPreviousBuilds)
 }
 
 try {
-  $resp = Start-DevOpsBuild -SourceBranch $SourceBranch -DefinitionId $DefinitionId -Base64EncodedAuthToken $Base64EncodedAuthToken
+  $resp = Start-DevOpsBuild `
+    -Organization $Organization `
+    -Project $Project `
+    -SourceBranch $SourceBranch `
+    -DefinitionId $DefinitionId `
+    -Base64EncodedAuthToken $Base64EncodedAuthToken `
+    -BuildParametersJson $BuildParametersJson
 }
 catch {
   LogError "Start-DevOpsBuild failed with exception:`n$_"
@@ -64,4 +73,4 @@ if ($VsoQueuedPipelines) {
   }
   $QueuedPipelineLinks
   Write-Host "##vso[task.setvariable variable=$VsoQueuedPipelines]$QueuedPipelineLinks"
-}
+}

eng/common/scripts/Save-Package-Properties.ps1

@@ -1,12 +1,79 @@
+<#
+.SYNOPSIS
+Saves package properties from source into JSON files
+
+.DESCRIPTION
+Saves package properties in source of a given service directory to JSON files.
+JSON files are named in the form <package name>.json or <artifact name>.json if
+an artifact name property is available in the package properties.
+
+Can optionally add a dev version property which can be used logic for daily 
+builds.
+
+.PARAMETER serviceDirectory
+Service directory in which to search for packages
+
+.PARAMETER outDirectory
+Output location (generally a package artifact directory in DevOps) for JSON 
+files
+
+.PARAMETER addDevVersion
+Reads the version out of the source and adds a DevVersion property to the 
+package properties JSON file. If the package properties JSON file already 
+exists, read the Version property from the existing package properties JSON file
+and set that as the Version property for the new output. This has the effect of
+"adding" a DevVersion property to the file which could be different from the 
+Verison property in that file.
+#>
+
 [CmdletBinding()]
 Param (
   [Parameter(Mandatory=$True)]
   [string] $serviceDirectory,
   [Parameter(Mandatory=$True)]
-  [string] $outDirectory
+  [string] $outDirectory,
+  [switch] $addDevVersion
 )
 
 . (Join-Path $PSScriptRoot common.ps1)
+
+function SetOutput($outputPath, $incomingPackageSpec) { 
+  $outputObject = $incomingPackageSpec
+
+  if ($addDevVersion) {
+    # Use the "Version" property which was provided by the incoming package spec
+    # as the DevVersion. This may be overridden later.
+    $outputObject.DevVersion = $incomingPackageSpec.Version
+
+    # If there is an exsiting package info json file read that and set the 
+    # Version property from that JSON file.
+    if (Test-Path $outputPath) { 
+      $originalObject = ConvertFrom-Json (Get-Content $outputPath -Raw)
+      $outputObject.Version = $originalObject.Version
+    }
+  }
+
+  # Set file paths to relative paths
+  $outputObject.DirectoryPath = GetRelativePath $outputObject.DirectoryPath
+  $outputObject.ReadMePath = GetRelativePath $outputObject.ReadMePath
+  $outputObject.ChangeLogPath = GetRelativePath $outputObject.ChangeLogPath
+
+  Set-Content `
+    -Path $outputPath `
+    -Value (ConvertTo-Json -InputObject $outputObject -Depth 100)
+}
+
+function GetRelativePath($path) {
+  # If the path is empty return an empty string
+  if (!$path) {
+    return ''
+  }
+  $relativeTo = Resolve-Path $PSScriptRoot/../../../
+  # Replace "\" with "/" so the path is valid across other platforms and tools
+  $relativePath = [IO.Path]::GetRelativePath($relativeTo, $path) -replace "\\", '/'
+  return $relativePath
+}
+
 $allPackageProperties = Get-AllPkgProperties $serviceDirectory
 if ($allPackageProperties)
 {
@@ -29,9 +96,8 @@ if ($allPackageProperties)
               $configFilePrefix = $pkg.ArtifactName
             }
             $outputPath = Join-Path -Path $outDirectory "$configFilePrefix.json"
-            $outputObject = $pkg | ConvertTo-Json
-            Set-Content -Path $outputPath -Value $outputObject
-        }        
+            SetOutput $outputPath $pkg
+        }
     }
 
     Get-ChildItem -Path $outDirectory