From 5e30f1bfd4e7959a31094bce557af36f172fe4f4 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Fri, 22 Nov 2024 09:05:04 -0800 Subject: [PATCH 1/6] Update the docIndex default for manual runs --- eng/pipelines/docindex.yml | 301 ++++++++++++++++++++----------------- 1 file changed, 160 insertions(+), 141 deletions(-) diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml index 8a23b814f407..86e9a3461714 100644 --- a/eng/pipelines/docindex.yml +++ b/eng/pipelines/docindex.yml @@ -44,144 +44,163 @@ jobs: - template: /eng/common/pipelines/templates/steps/set-default-branch.yml - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Update-DocsMsPackageMonikers.ps1 - arguments: -DocRepoLocation $(DocRepoLocation) - displayName: Move deprecated packages to legacy moniker - condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.MainUpdate'], 'true'))) - - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Update-DocsMsPackages.ps1 - arguments: -DocRepoLocation $(DocRepoLocation) - displayName: Update Docs Onboarding for main branch - condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.MainUpdate'], 'true'))) - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Service-Level-Readme-Automation.ps1 - arguments: >- - -DocRepoLocation $(DocRepoLocation) - -ReadmeFolderRoot 'api/overview/azure' - displayName: Generate Service Level Readme for main branch - condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.MainUpdate'], 'true'))) - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Update-DocsMsToc.ps1 - arguments: >- - -DocRepoLocation $(DocRepoLocation) - -OutputLocation $(DocRepoLocation)/docs-ref-toc/reference-unified.yml - -ReadmeFolderRoot "api/overview/azure" - displayName: Generate ToC for main branch - condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.MainUpdate'], 'true'))) - - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Verify-RequiredDocsJsonMembers.ps1 - arguments: >- - -DocRepoLocation $(DocRepoLocation) - displayName: Verify Required Docs Json Members - - - template: /eng/common/pipelines/templates/steps/git-push-changes.yml - parameters: - BaseRepoBranch: $(DefaultBranch) - BaseRepoOwner: $(DocRepoOwner) - CommitMsg: "Update docs CI configuration" - TargetRepoName: $(DocRepoName) - TargetRepoOwner: $(DocRepoOwner) - WorkingDirectory: $(DocRepoLocation) - - # Prepare daily docs CI - - template: /eng/common/pipelines/templates/steps/set-daily-docs-branch-name.yml - parameters: - DailyBranchVariableName: DailyDocsBranchName - - pwsh: | - $ErrorActionPreference = "Continue" - git checkout "origin/$(DailyDocsBranchName)" 2>&1 | Out-Null - $LASTEXITCODE = 0 # This ignores any error from git checkout - git status - displayName: Checkout daily branch if it exists - workingDirectory: $(DocRepoLocation) - - - pwsh: | - $publicFeedUrl = "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json" - Write-Host "##vso[task.setvariable variable=PackageSourceOverride]$publicFeedUrl" - displayName: Set package source variable - workingDirectory: $(DocRepoLocation) - - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Update-DocsMsPackageMonikers.ps1 - arguments: -DocRepoLocation $(DocRepoLocation) - displayName: Move deprecated packages to legacy moniker - condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.MainUpdate'], 'true'))) - - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Update-DocsMsPackages.ps1 - # Use the dotnet public daily package preview feed as the source for - # updated packages. - arguments: > - -DocRepoLocation $(DocRepoLocation) - -PackageSourceOverride $(PackageSourceOverride) - displayName: Update Docs Onboarding for Daily docs - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Service-Level-Readme-Automation.ps1 - arguments: >- - -DocRepoLocation $(DocRepoLocation) - -ReadmeFolderRoot 'api/overview/azure' - displayName: Generate Service Level Readme for Daily docs - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Update-DocsMsToc.ps1 - arguments: >- - -DocRepoLocation $(DocRepoLocation) - -OutputLocation $(DocRepoLocation)/docs-ref-toc/reference-unified.yml - -ReadmeFolderRoot "api/overview/azure" - -PackageSourceOverride $(PackageSourceOverride) - displayName: Generate ToC for Daily docs - - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Verify-RequiredDocsJsonMembers.ps1 - arguments: >- - -DocRepoLocation $(DocRepoLocation) - displayName: Verify Required Docs Json Members - - - template: /eng/common/pipelines/templates/steps/git-push-changes.yml - parameters: - BaseRepoBranch: $(DailyDocsBranchName) - BaseRepoOwner: $(DocRepoOwner) - CommitMsg: "Update targeting packages based on release metadata. (Daily docs)" - TargetRepoName: $(DocRepoName) - TargetRepoOwner: $(DocRepoOwner) - WorkingDirectory: $(DocRepoLocation) - ScriptDirectory: $(Build.SourcesDirectory)/eng/common/scripts - PushArgs: -f - - - task: AzureCLI@2 - displayName: Queue Docs CI build - inputs: - azureSubscription: msdocs-apidrop-connection - scriptType: pscore - scriptLocation: inlineScript - inlineScript: | - $accessToken = az account get-access-token --resource "499b84ac-1321-427f-aa17-267ca6975798" --query "accessToken" --output tsv - $buildParamJson = (@{ params = (Get-Content ./eng/dailydocsconfig.json -Raw) -replace '%%DailyDocsBranchName%%', "$(DailyDocsBranchName)" } | ConvertTo-Json) - eng/common/scripts/Queue-Pipeline.ps1 ` - -Organization "apidrop" ` - -Project "Content%20CI" ` - -DefinitionId 397 ` - -BuildParametersJson $buildParamJson ` - -BearerToken $accessToken + # Updating main is the default reason that manual builds of the docIndex run are kicked off. + # Manual builds of DocIndex are typically done to update Main when certain updates are made. + # These updates include new libraries and CSV updates in azure-sdk which, depending on the + # column(s) updated can cause anything from package deprecation to service level readme or + # ToC updates. + - ${{if or(eq(variables['Build.Reason'], 'Schedule'), ne(variables['Skip.MainUpdate'], 'true'))}}: + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Verify-RequiredDocsJsonMembers.ps1 + arguments: >- + -DocRepoLocation $(DocRepoLocation) + displayName: Verify Required Docs Json Members + condition: succeeded() + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Update-DocsMsPackageMonikers.ps1 + arguments: -DocRepoLocation $(DocRepoLocation) + displayName: Move deprecated packages to legacy moniker + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Update-DocsMsPackages.ps1 + arguments: -DocRepoLocation $(DocRepoLocation) + displayName: Update Docs Onboarding for main branch + condition: succeeded() + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Service-Level-Readme-Automation.ps1 + arguments: >- + -DocRepoLocation $(DocRepoLocation) + -ReadmeFolderRoot 'api/overview/azure' + displayName: Generate Service Level Readme for main branch + condition: succeeded() + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Update-DocsMsToc.ps1 + arguments: >- + -DocRepoLocation $(DocRepoLocation) + -OutputLocation $(DocRepoLocation)/docs-ref-toc/reference-unified.yml + -ReadmeFolderRoot "api/overview/azure" + displayName: Generate ToC for main branch + condition: succeeded() + + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoBranch: $(DefaultBranch) + BaseRepoOwner: $(DocRepoOwner) + CommitMsg: "Update docs CI configuration" + TargetRepoName: $(DocRepoName) + TargetRepoOwner: $(DocRepoOwner) + WorkingDirectory: $(DocRepoLocation) + + # The scenario for running a Manual build is normally only for the main updates. The daily build + # should really only get kicked off for scheduled runs. + - ${{if or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.DailyUpdate'], 'true'))}}: + + # Prepare daily docs CI + - template: /eng/common/pipelines/templates/steps/set-daily-docs-branch-name.yml + parameters: + DailyBranchVariableName: DailyDocsBranchName + - pwsh: | + $ErrorActionPreference = "Continue" + git checkout "origin/$(DailyDocsBranchName)" 2>&1 | Out-Null + $LASTEXITCODE = 0 # This ignores any error from git checkout + git status + displayName: Checkout daily branch if it exists + workingDirectory: $(DocRepoLocation) + + - pwsh: | + $publicFeedUrl = "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json" + Write-Host "##vso[task.setvariable variable=PackageSourceOverride]$publicFeedUrl" + displayName: Set package source variable + workingDirectory: $(DocRepoLocation) + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Verify-RequiredDocsJsonMembers.ps1 + arguments: >- + -DocRepoLocation $(DocRepoLocation) + displayName: Verify Required Docs Json Members + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Update-DocsMsPackageMonikers.ps1 + arguments: -DocRepoLocation $(DocRepoLocation) + displayName: Move deprecated packages to legacy moniker + condition: succeeded() + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Update-DocsMsPackages.ps1 + # Use the dotnet public daily package preview feed as the source for + # updated packages. + arguments: > + -DocRepoLocation $(DocRepoLocation) + -PackageSourceOverride $(PackageSourceOverride) + displayName: Update Docs Onboarding for Daily docs + condition: succeeded() + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Service-Level-Readme-Automation.ps1 + arguments: >- + -DocRepoLocation $(DocRepoLocation) + -ReadmeFolderRoot 'api/overview/azure' + displayName: Generate Service Level Readme for Daily docs + condition: succeeded() + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Update-DocsMsToc.ps1 + arguments: >- + -DocRepoLocation $(DocRepoLocation) + -OutputLocation $(DocRepoLocation)/docs-ref-toc/reference-unified.yml + -ReadmeFolderRoot "api/overview/azure" + -PackageSourceOverride $(PackageSourceOverride) + displayName: Generate ToC for Daily docs + condition: succeeded() + + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoBranch: $(DailyDocsBranchName) + BaseRepoOwner: $(DocRepoOwner) + CommitMsg: "Update targeting packages based on release metadata. (Daily docs)" + TargetRepoName: $(DocRepoName) + TargetRepoOwner: $(DocRepoOwner) + WorkingDirectory: $(DocRepoLocation) + ScriptDirectory: $(Build.SourcesDirectory)/eng/common/scripts + PushArgs: -f + + - task: AzureCLI@2 + displayName: Queue Docs CI build + inputs: + azureSubscription: msdocs-apidrop-connection + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + $accessToken = az account get-access-token --resource "499b84ac-1321-427f-aa17-267ca6975798" --query "accessToken" --output tsv + $buildParamJson = (@{ params = (Get-Content ./eng/dailydocsconfig.json -Raw) -replace '%%DailyDocsBranchName%%', "$(DailyDocsBranchName)" } | ConvertTo-Json) + eng/common/scripts/Queue-Pipeline.ps1 ` + -Organization "apidrop" ` + -Project "Content%20CI" ` + -DefinitionId 397 ` + -BuildParametersJson $buildParamJson ` + -BearerToken $accessToken + condition: succeeded() From 92028e49aae85b6a0c95e3acd80048df89f1e034 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Fri, 22 Nov 2024 09:10:24 -0800 Subject: [PATCH 2/6] update the comment in the yml --- eng/pipelines/docindex.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml index 86e9a3461714..f82e3203d739 100644 --- a/eng/pipelines/docindex.yml +++ b/eng/pipelines/docindex.yml @@ -44,11 +44,11 @@ jobs: - template: /eng/common/pipelines/templates/steps/set-default-branch.yml - # Updating main is the default reason that manual builds of the docIndex run are kicked off. - # Manual builds of DocIndex are typically done to update Main when certain updates are made. - # These updates include new libraries and CSV updates in azure-sdk which, depending on the - # column(s) updated can cause anything from package deprecation to service level readme or - # ToC updates. + # Updating main is the default reason to kick off manual builds of the docIndex run. + # Manual builds of docIndex are typically done to update Main when certain updates are + # made. These updates include new libraries and CSV updates in azure-sdk which, depending + # on the column(s) updated can cause anything from package deprecation to service level + # readme or ToC updates. - ${{if or(eq(variables['Build.Reason'], 'Schedule'), ne(variables['Skip.MainUpdate'], 'true'))}}: - task: Powershell@2 From c226a7aa4dd93b669a31d28e11623cb34faa0ec0 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Fri, 22 Nov 2024 09:54:09 -0800 Subject: [PATCH 3/6] fix some spacing --- eng/pipelines/docindex.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml index f82e3203d739..2774d7d671d8 100644 --- a/eng/pipelines/docindex.yml +++ b/eng/pipelines/docindex.yml @@ -49,8 +49,7 @@ jobs: # made. These updates include new libraries and CSV updates in azure-sdk which, depending # on the column(s) updated can cause anything from package deprecation to service level # readme or ToC updates. - - ${{if or(eq(variables['Build.Reason'], 'Schedule'), ne(variables['Skip.MainUpdate'], 'true'))}}: - + - ${{ if or(eq(variables['Build.Reason'], 'Schedule'), ne(variables['Skip.MainUpdate'], 'true')) }}: - task: Powershell@2 inputs: pwsh: true @@ -105,11 +104,10 @@ jobs: TargetRepoOwner: $(DocRepoOwner) WorkingDirectory: $(DocRepoLocation) + # The scenario for running a Manual build is normally only for the main updates. The daily build # should really only get kicked off for scheduled runs. - - ${{if or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.DailyUpdate'], 'true'))}}: - - # Prepare daily docs CI + - ${{ if or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.DailyUpdate'], 'true')) }}: - template: /eng/common/pipelines/templates/steps/set-daily-docs-branch-name.yml parameters: DailyBranchVariableName: DailyDocsBranchName From 8f5f8f508a09051b0785bf243bc4fa6a57ea39c2 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Mon, 25 Nov 2024 11:50:27 -0800 Subject: [PATCH 4/6] make the variables paremeters --- eng/pipelines/docindex.yml | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml index 2774d7d671d8..329ef5187c13 100644 --- a/eng/pipelines/docindex.yml +++ b/eng/pipelines/docindex.yml @@ -1,5 +1,19 @@ trigger: none +parameters: +- name: SkipMainUpdate + displayName: | + Skip updating main. + type: boolean + default: false + +- name: ForceDailyUpdate + displayName: | + Force the daily branch update (includes starting daily branch run). + type: boolean + default: false + + variables: - template: /eng/pipelines/templates/variables/globals.yml - template: /eng/pipelines/templates/variables/image.yml @@ -49,7 +63,7 @@ jobs: # made. These updates include new libraries and CSV updates in azure-sdk which, depending # on the column(s) updated can cause anything from package deprecation to service level # readme or ToC updates. - - ${{ if or(eq(variables['Build.Reason'], 'Schedule'), ne(variables['Skip.MainUpdate'], 'true')) }}: + - ${{ if or(eq(variables['Build.Reason'], 'Schedule'), ne(parameters.SkipMainUpdate, 'true')) }}: - task: Powershell@2 inputs: pwsh: true @@ -57,7 +71,6 @@ jobs: arguments: >- -DocRepoLocation $(DocRepoLocation) displayName: Verify Required Docs Json Members - condition: succeeded() - task: Powershell@2 inputs: @@ -72,7 +85,6 @@ jobs: filePath: eng/common/scripts/Update-DocsMsPackages.ps1 arguments: -DocRepoLocation $(DocRepoLocation) displayName: Update Docs Onboarding for main branch - condition: succeeded() - task: Powershell@2 inputs: @@ -82,7 +94,6 @@ jobs: -DocRepoLocation $(DocRepoLocation) -ReadmeFolderRoot 'api/overview/azure' displayName: Generate Service Level Readme for main branch - condition: succeeded() - task: Powershell@2 inputs: @@ -93,7 +104,6 @@ jobs: -OutputLocation $(DocRepoLocation)/docs-ref-toc/reference-unified.yml -ReadmeFolderRoot "api/overview/azure" displayName: Generate ToC for main branch - condition: succeeded() - template: /eng/common/pipelines/templates/steps/git-push-changes.yml parameters: @@ -107,7 +117,7 @@ jobs: # The scenario for running a Manual build is normally only for the main updates. The daily build # should really only get kicked off for scheduled runs. - - ${{ if or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.DailyUpdate'], 'true')) }}: + - ${{ if or(eq(variables['Build.Reason'], 'Schedule'), eq(parameters.ForceDailyUpdate, 'true')) }}: - template: /eng/common/pipelines/templates/steps/set-daily-docs-branch-name.yml parameters: DailyBranchVariableName: DailyDocsBranchName @@ -139,7 +149,6 @@ jobs: filePath: eng/common/scripts/Update-DocsMsPackageMonikers.ps1 arguments: -DocRepoLocation $(DocRepoLocation) displayName: Move deprecated packages to legacy moniker - condition: succeeded() - task: Powershell@2 inputs: @@ -151,7 +160,6 @@ jobs: -DocRepoLocation $(DocRepoLocation) -PackageSourceOverride $(PackageSourceOverride) displayName: Update Docs Onboarding for Daily docs - condition: succeeded() - task: Powershell@2 inputs: @@ -161,7 +169,6 @@ jobs: -DocRepoLocation $(DocRepoLocation) -ReadmeFolderRoot 'api/overview/azure' displayName: Generate Service Level Readme for Daily docs - condition: succeeded() - task: Powershell@2 inputs: @@ -173,7 +180,6 @@ jobs: -ReadmeFolderRoot "api/overview/azure" -PackageSourceOverride $(PackageSourceOverride) displayName: Generate ToC for Daily docs - condition: succeeded() - template: /eng/common/pipelines/templates/steps/git-push-changes.yml parameters: @@ -201,4 +207,3 @@ jobs: -DefinitionId 397 ` -BuildParametersJson $buildParamJson ` -BearerToken $accessToken - condition: succeeded() From 79610c97781a4e77cebef59699b68375770596ca Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Mon, 25 Nov 2024 12:02:57 -0800 Subject: [PATCH 5/6] Change to paremeters --- eng/pipelines/docindex.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml index 329ef5187c13..a7578873e3a0 100644 --- a/eng/pipelines/docindex.yml +++ b/eng/pipelines/docindex.yml @@ -1,11 +1,11 @@ trigger: none parameters: -- name: SkipMainUpdate +- name: UpdateMain displayName: | - Skip updating main. + Updating main branch. type: boolean - default: false + default: true - name: ForceDailyUpdate displayName: | @@ -63,7 +63,7 @@ jobs: # made. These updates include new libraries and CSV updates in azure-sdk which, depending # on the column(s) updated can cause anything from package deprecation to service level # readme or ToC updates. - - ${{ if or(eq(variables['Build.Reason'], 'Schedule'), ne(parameters.SkipMainUpdate, 'true')) }}: + - ${{ if or(eq(variables['Build.Reason'], 'Schedule'), parameters.UpdateMain) }}: - task: Powershell@2 inputs: pwsh: true @@ -117,7 +117,7 @@ jobs: # The scenario for running a Manual build is normally only for the main updates. The daily build # should really only get kicked off for scheduled runs. - - ${{ if or(eq(variables['Build.Reason'], 'Schedule'), eq(parameters.ForceDailyUpdate, 'true')) }}: + - ${{ if or(eq(variables['Build.Reason'], 'Schedule'), parameters.ForceDailyUpdate) }}: - template: /eng/common/pipelines/templates/steps/set-daily-docs-branch-name.yml parameters: DailyBranchVariableName: DailyDocsBranchName From b8a2121c588b37342f6e96b8042b31c697133417 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Mon, 25 Nov 2024 12:04:02 -0800 Subject: [PATCH 6/6] grammar better --- eng/pipelines/docindex.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml index a7578873e3a0..841f5ff13947 100644 --- a/eng/pipelines/docindex.yml +++ b/eng/pipelines/docindex.yml @@ -3,7 +3,7 @@ trigger: none parameters: - name: UpdateMain displayName: | - Updating main branch. + Update main branch. type: boolean default: true