From d47cf26316339e06b575205bcd475670d8aeefcf Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Thu, 24 Oct 2024 11:26:17 -0700 Subject: [PATCH 1/9] Enable py2docfx docs gen tool, remove the dockerimage docs validation --- eng/pipelines/docindex.yml | 16 +- .../stages/archetype-python-release.yml | 20 +- .../steps/install-rex-validation-tool.yml | 9 + eng/scripts/Language-Settings.ps1 | 30 -- eng/scripts/docs/Docs-Onboarding.ps1 | 270 ++++++++---------- eng/scripts/docs/py2docfx.version.txt | 1 + 6 files changed, 158 insertions(+), 188 deletions(-) create mode 100644 eng/pipelines/templates/steps/install-rex-validation-tool.yml create mode 100644 eng/scripts/docs/py2docfx.version.txt diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml index 59aee791b36e..12b0026ccd26 100644 --- a/eng/pipelines/docindex.yml +++ b/eng/pipelines/docindex.yml @@ -14,20 +14,16 @@ jobs: DocRepoName: azure-docs-sdk-python DocValidationImageId: azuresdkimages.azurecr.io/pyrefautocr:latest steps: - # Docs CI uses Python 3.9.13 + # py2docfx requires Python >= 3.11 - task: UsePythonVersion@0 - displayName: 'Use Python 3.9.13' + displayName: 'Use Python 3.11' inputs: - versionSpec: '3.9.13' + versionSpec: '3.11' # Docs CI upgrades pip, wheel, and setuptools - pwsh: python -m pip install --upgrade pip wheel setuptools displayName: Update python tools for package verification - # Pull and build the docker image. - - template: /eng/common/pipelines/templates/steps/docker-pull-image.yml - parameters: - ImageId: "$(DocValidationImageId)" # Sync docs repo onboarding files/folders - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml parameters: @@ -41,6 +37,8 @@ jobs: - Name: $(DocRepoOwner)/$(DocRepoName) WorkingDirectory: $(DocRepoLocation) + - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml + - task: Powershell@2 inputs: pwsh: true @@ -53,7 +51,7 @@ jobs: inputs: pwsh: true filePath: eng/common/scripts/Update-DocsMsPackages.ps1 - arguments: -DocRepoLocation $(DocRepoLocation) -ImageId '$(DocValidationImageId)' + arguments: -DocRepoLocation $(DocRepoLocation) displayName: Update Docs Onboarding for main branch condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.MainUpdate'], 'true'))) @@ -92,7 +90,7 @@ jobs: parameters: BaseRepoBranch: $(DefaultBranch) BaseRepoOwner: $(DocRepoOwner) - CommitMsg: "Update docs CI configuration" + CommitMsg: "Update docs CI configuration Build: $(System.CollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)" TargetRepoName: $(DocRepoName) TargetRepoOwner: $(DocRepoOwner) WorkingDirectory: $(DocRepoLocation) diff --git a/eng/pipelines/templates/stages/archetype-python-release.yml b/eng/pipelines/templates/stages/archetype-python-release.yml index fece5adabba3..8bdcfd24b8fa 100644 --- a/eng/pipelines/templates/stages/archetype-python-release.yml +++ b/eng/pipelines/templates/stages/archetype-python-release.yml @@ -9,7 +9,6 @@ parameters: TargetDocRepoOwner: '' TargetDocRepoName: '' PackageSourceOverride: "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" - DocValidationImageId: "azuresdkimages.azurecr.io/pyrefautocr:latest" stages: - ${{if and(in(variables['Build.Reason'], 'Manual', ''), eq(variables['System.TeamProject'], 'internal'))}}: @@ -232,6 +231,15 @@ stages: - sdk/**/*.md - .github/CODEOWNERS - download: current + + # py2docfx requires Python >= 3.11 + - task: UsePythonVersion@0 + displayName: 'Use Python 3.11' + inputs: + versionSpec: '3.11' + + - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml + - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml parameters: PackageInfoLocations: @@ -244,7 +252,6 @@ stages: - docs-ref-services/ - metadata/ PackageSourceOverride: ${{parameters.PackageSourceOverride}} - DocValidationImageId: ${{parameters.DocValidationImageId}} - deployment: UpdatePackageVersion displayName: "Update Package Version" @@ -351,6 +358,14 @@ stages: Get-ChildItem -Recurse $(Pipeline.Workspace)/${{parameters.ArtifactName}}/ displayName: Show visible artifacts + # py2docfx requires Python >= 3.11 + - task: UsePythonVersion@0 + displayName: 'Use Python 3.11' + inputs: + versionSpec: '3.11' + + - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml + - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml parameters: PackageInfoLocations: @@ -366,6 +381,5 @@ stages: - docs-ref-services/ - metadata/ PackageSourceOverride: ${{parameters.PackageSourceOverride}} - DocValidationImageId: ${{parameters.DocValidationImageId}} - template: /eng/common/pipelines/templates/steps/docsms-ensure-validation.yml diff --git a/eng/pipelines/templates/steps/install-rex-validation-tool.yml b/eng/pipelines/templates/steps/install-rex-validation-tool.yml new file mode 100644 index 000000000000..57b9871d370c --- /dev/null +++ b/eng/pipelines/templates/steps/install-rex-validation-tool.yml @@ -0,0 +1,9 @@ +steps: + - pwsh: | + $py2docfxVer = Get-Content eng/scripts/docs/py2docfx.version.txt + Write-Host "python -m pip install py2docfx==$py2docfxVer" + python -m pip install py2docfx==$py2docfxVer + # After install py2docfx, + Write-Host "Testing the install. Running python -m py2docfx -h to display the help" + python -m py2docfx -h + displayName: Install py2docfx for package validation diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index d264a1a1b39e..40142dd4ddd2 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -427,36 +427,6 @@ function Import-Dev-Cert-python "The variables SSL_CERT_DIR, SSL_CERT_FILE, and REQUESTS_CA_BUNDLE are now dynamically set in proxy_startup.py" } -# Defined in common.ps1 as: -# $ValidateDocsMsPackagesFn = "Validate-${Language}-DocMsPackages" -function Validate-Python-DocMsPackages ($PackageInfo, $PackageInfos, $PackageSourceOverride, $DocValidationImageId) -{ - # While eng/common/scripts/Update-DocsMsMetadata.ps1 is still passing a single packageInfo, process as a batch - if (!$PackageInfos) { - $PackageInfos = @($PackageInfo) - } - - $allSucceeded = $true - foreach ($item in $PackageInfos) { - # If the Version is IGNORE that means it's a source install and those aren't run through ValidatePackage - if ($item.Version -eq 'IGNORE') { - continue - } - - $result = ValidatePackage ` - -packageName $item.Name ` - -packageVersion "==$($item.Version)" ` - -PackageSourceOverride $PackageSourceOverride ` - -DocValidationImageId $DocValidationImageId - - if (!$result) { - $allSucceeded = $false - } - } - - return $allSucceeded -} - function Get-python-EmitterName() { return "@azure-tools/typespec-python" } diff --git a/eng/scripts/docs/Docs-Onboarding.ps1 b/eng/scripts/docs/Docs-Onboarding.ps1 index 23f14cf20edb..9be956239078 100644 --- a/eng/scripts/docs/Docs-Onboarding.ps1 +++ b/eng/scripts/docs/Docs-Onboarding.ps1 @@ -1,7 +1,7 @@ . "$PSScriptRoot/Docs-ToC.ps1" # $SetDocsPackageOnboarding = "Set-${Language}-DocsPackageOnboarding" -function Set-python-DocsPackageOnboarding($moniker, $metadata, $docRepoLocation, $packageSourceOverride) { +function Set-python-DocsPackageOnboarding($moniker, $metadata, $docRepoLocation, $packageSourceOverride) { $onboardingFile = GetOnboardingFileForMoniker $docRepoLocation $moniker $onboardingSpec = Get-Content $onboardingFile -Raw | ConvertFrom-Json -AsHashtable @@ -30,7 +30,7 @@ function Set-python-DocsPackageOnboarding($moniker, $metadata, $docRepoLocation, if ($package.ContainsKey('DocsCiConfigProperties')) { $overrides = $package['DocsCiConfigProperties'] - + # Merge properties from package_info object (duplicate values will) # be overwritten if ($overrides.ContainsKey('package_info')) { @@ -40,7 +40,7 @@ function Set-python-DocsPackageOnboarding($moniker, $metadata, $docRepoLocation, } # Directly override other keys like exlcude_path - foreach ($key in $overrides.Keys) { + foreach ($key in $overrides.Keys) { if ($key -in @('package_info')) { # Skip over keys that have already been processed continue @@ -67,153 +67,131 @@ function Get-python-DocsPackagesAlreadyOnboarded($docRepoLocation, $moniker) { -moniker $moniker } - - -function ValidatePackage { - Param( - [Parameter(Mandatory = $true)] - [string]$packageName, - [Parameter(Mandatory = $true)] - [string]$packageVersion, - [Parameter(Mandatory = $false)] - [string]$PackageSourceOverride, - [Parameter(Mandatory = $false)] - [string]$DocValidationImageId - ) - $installValidationFolder = Join-Path ([System.IO.Path]::GetTempPath()) "validation" - if (!(Test-Path $installValidationFolder)) { - New-Item -ItemType Directory -Force -Path $installValidationFolder | Out-Null - } - # Add more validation by replicating as much of the docs CI process as - # possible - # https://github.com/Azure/azure-sdk-for-python/issues/20109 - $result = $true - if (!$DocValidationImageId) { - Write-Host "Validating using pip command directly on $packageName." - $result = FallbackValidation -packageName "$packageName" -packageVersion "$packageVersion" -workingDirectory $installValidationFolder -PackageSourceOverride $PackageSourceOverride - } - else { - Write-Host "Validating using $DocValidationImageId on $packageName." - $result = DockerValidation -packageName "$packageName" -packageVersion "$packageVersion" ` - -PackageSourceOverride $PackageSourceOverride -DocValidationImageId $DocValidationImageId -workingDirectory $installValidationFolder - } - - return $result -} -function DockerValidation { - Param( - [Parameter(Mandatory = $true)] - [string]$packageName, - [Parameter(Mandatory = $true)] - [string]$packageVersion, - [Parameter(Mandatory = $false)] - [string]$PackageSourceOverride, - [Parameter(Mandatory = $false)] - [string]$DocValidationImageId, - [Parameter(Mandatory = $false)] - [string]$workingDirectory - ) - if ($PackageSourceOverride) { - Write-Host "docker run -v ${workingDirectory}:/workdir/out -e TARGET_PACKAGE=$packageName -e TARGET_VERSION=$packageVersion -e EXTRA_INDEX_URL=$PackageSourceOverride -t $DocValidationImageId" - $commandLine = docker run -v "${workingDirectory}:/workdir/out" -e TARGET_PACKAGE=$packageName -e TARGET_VERSION=$packageVersion ` - -e EXTRA_INDEX_URL=$PackageSourceOverride -t $DocValidationImageId 2>&1 - } - else { - Write-Host "docker run -v ${workingDirectory}:/workdir/out -e TARGET_PACKAGE=$packageName -e TARGET_VERSION=$packageVersion -t $DocValidationImageId" - $commandLine = docker run -v "${workingDirectory}:/workdir/out" ` - -e TARGET_PACKAGE=$packageName -e TARGET_VERSION=$packageVersion -t $DocValidationImageId 2>&1 - } - # The docker exit codes: https://docs.docker.com/engine/reference/run/#exit-status - # If the docker failed because of docker itself instead of the application, - # we should skip the validation and keep the packages. - - if ($LASTEXITCODE -eq 125 -Or $LASTEXITCODE -eq 126 -Or $LASTEXITCODE -eq 127) { - $commandLine | ForEach-Object { Write-Debug $_ } - LogWarning "The `docker` command does not work with exit code $LASTEXITCODE. Fall back to npm install $packageName directly." - FallbackValidation -packageName "$packageName" -packageVersion "$packageVersion" -workingDirectory $workingDirectory -PackageSourceOverride $PackageSourceOverride - } - elseif ($LASTEXITCODE -ne 0) { - $commandLine | ForEach-Object { Write-Debug $_ } - LogWarning "Package $packageName ref docs validation failed." - return $false - } - return $true -} - -function FallbackValidation { - Param( - [Parameter(Mandatory = $true)] - [string]$packageName, - [Parameter(Mandatory = $true)] - [string]$packageVersion, - [Parameter(Mandatory = $true)] - [string]$workingDirectory, - [Parameter(Mandatory = $false)] - [string]$PackageSourceOverride - ) - $installTargetFolder = Join-Path $workingDirectory $packageName - New-Item -ItemType Directory -Force -Path $installTargetFolder | Out-Null - $packageExpression = "$packageName$packageVersion" - try { - $pipInstallOutput = "" - if ($PackageSourceOverride) { - Write-Host "python -m pip install $packageExpression --no-cache-dir --target $installTargetFolder --extra-index-url=$PackageSourceOverride" - $pipInstallOutput = python -m pip ` - install ` - $packageExpression ` - --no-cache-dir ` - --target $installTargetFolder ` - --extra-index-url=$PackageSourceOverride 2>&1 - } - else { - Write-Host "python -m pip install $packageExpression --no-cache-dir --target $installTargetFolder" - $pipInstallOutput = python -m pip ` - install ` - $packageExpression ` - --no-cache-dir ` - --target $installTargetFolder 2>&1 - } - if ($LASTEXITCODE -ne 0) { - LogWarning "python -m pip install failed for $packageExpression" - Write-Host $pipInstallOutput - return $false - } - } - catch { - LogWarning "python -m pip install failed for $packageExpression with exception" - LogWarning $_.Exception - LogWarning $_.Exception.StackTrace - return $false - } - - return $true +# The package info format required by py2docfx to verify a single library. +# The format can be found at https://github.com/MicrosoftDocs/azure-docs-sdk-python/blob/main/ci-configs/packages-latest.json +# which is a file that contains all of package_info objects. For verifying a single library +# the same format is used but there's only a single package_info object in the packages list. +# This is an example: +# { +# "packages": [ +# { +# "package_info": { +# "name": "azure-core", +# "install_type": "pypi", +# "prefer_source_distribution": "true", +# "version": "==1.29.6a20231207001", +# "extra_index_url": "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" +# }, +# "exclude_path": [ +# "test*", +# "example*", +# "sample*", +# "doc*" +# ] +# } +# ] +# } +# The above example contains the common settings for Python track 2 libraries. +# There are a few things of note: +# 1. install_type can be something other than pypi. source_code or dist_file are two examples of this +# but those are for track 1 or libraries released by other teams and not through our engineering system. +# 2. extra_index_url only needs to exist on the object if PackageSourceOverride is set +# 3. The reason this needs to be done using a json file instead of just a command line is because py2docfx +# doesn't handle the autodoc_default_options on the +function Get-SinglePackageJsonForDocsValidation($PackageInfo, $PackageSourceOverride) +{ + + $packageArr = @() + $packageSpec = [ordered]@{ + package_info = [ordered]@{ + name = $PackageInfo.Name + install_type = 'pypi' + prefer_source_distribution = 'true' + version = "==$($PackageInfo.Version)" + } + exclude_path = @("test*","example*","sample*","doc*") + } + if ($PackageSourceOverride) { + $packageSpec['package_info']['extra_index_url'] = $PackageSourceOverride + } + # Data-plane packages (not mgmt packages, and not manually added '00`packages) + # should document inherited members + if ($PackageInfo.Name -notlike 'azure-mgmt-*' -and $PackageInfo.Name -notlike '*-00-*') { + $packageSpec['extension_config'] = @{ 'autodoc_default_options' = @{ 'inherited-members' = 1 } } + } + $packageArr += $packageSpec + + # "packages" must be an array of packages even if there's only a single package in it. + # There are other top level elements, required_packages, target_repo etc. that aren't + # required for validation of a single package. + $docsConfigPackage = [ordered]@{ + packages = $packageArr + } + + # Return the JSon string + return $docsConfigPackage | ConvertTo-Json -Depth 10 } # Defined in common.ps1 as: # $ValidateDocsMsPackagesFn = "Validate-${Language}-DocMsPackages" -function Validate-Python-DocMsPackages ($PackageInfo, $PackageInfos, $PackageSourceOverride, $DocValidationImageId) { - # While eng/common/scripts/Update-DocsMsMetadata.ps1 is still passing a single packageInfo, process as a batch - if (!$PackageInfos) { - $PackageInfos = @($PackageInfo) +function Validate-Python-DocMsPackages($PackageInfo, $PackageInfos, $PackageSourceOverride, $DocValidationImageId) +{ + # While eng/common/scripts/Update-DocsMsMetadata.ps1 is still passing a single packageInfo, process as a batch + if (!$PackageInfos) { + $PackageInfos = @($PackageInfo) + } + # + $tempDirs = @() + $allSucceeded = $true + try { + foreach ($packageInfo in $PackageInfos) { + # Some packages won't have a version and this is the case when they're being onboarded manually + # and there's no version, only a repository and a SHA. In that case package we skip traditional + # package validation since the library doesn't exist yet outside of source and there's nothing + # the verification tools can do with this. + if ($packageInfo.Version -eq 'IGNORE') { + continue + } + + # Create a temporary directory. The json file being passed to py2docfx will be in the root and + # the docs will be generated to a docsOutput subdirectory. + $outputRoot = New-Item ` + -ItemType Directory ` + -Path (Join-Path ([System.IO.Path]::GetTempPath()) ([System.IO.Path]::GetRandomFileName())) + + $tempDirs += $outputRoot + + # Create the JSON file + $outputJsonFile = New-Item ` + -ItemType File ` + -Path (Join-Path $outputRoot ($packageInfo.Name + ".json")) + + ## Write out the json file and echo the contents + $JsonString = Get-SinglePackageJsonForDocsValidation $packageInfo $PackageSourceOverride + $JsonString | Out-File $outputJsonFile + Write-Host "$JsonString" + + # Create the docs output subdirectory. This is where the tool will generate its docs + $outputDocsDir = New-Item ` + -ItemType Directory ` + -Path (Join-Path $outputRoot "docsOutput") + + # Because this would be running on a lab image, a virtual environment shouldn't be required + Write-Host "Executing: python -m py2docfx --param-file-path $outputJsonFile -o $outputDocsDir --no-venv-required" + #$py2docfxOutput = python -m py2docfx --param-file-path $outputJsonFile -o $outputDocsDir --no-venv-required 2>&1 + python -m py2docfx --param-file-path $outputJsonFile -o $outputDocsDir --no-venv-required + if ($LASTEXITCODE -ne 0) { + LogWarning "py2docfx command failed, see output above." + $allSucceeded = $false + } } - - $allSucceeded = $true - foreach ($item in $PackageInfos) { - # If the Version is IGNORE that means it's a source install and those aren't run through ValidatePackage - if ($item.Version -eq 'IGNORE') { - continue - } - - $result = ValidatePackage ` - -packageName $item.Name ` - -packageVersion "==$($item.Version)" ` - -PackageSourceOverride $PackageSourceOverride ` - -DocValidationImageId $DocValidationImageId - - if (!$result) { - $allSucceeded = $false - } + } + finally { + # Clean up any temp directories + foreach ($tempDir in $tempDirs) + { + Remove-Item -Force -Recurse $tempDir | Out-Null } - - return $allSucceeded + } + return $allSucceeded } diff --git a/eng/scripts/docs/py2docfx.version.txt b/eng/scripts/docs/py2docfx.version.txt new file mode 100644 index 000000000000..20f49513e15f --- /dev/null +++ b/eng/scripts/docs/py2docfx.version.txt @@ -0,0 +1 @@ +0.1.11 From 1018df997f837e20a272cecc123d40c3e556c0bd Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Thu, 24 Oct 2024 13:20:59 -0700 Subject: [PATCH 2/9] Remove the --no-venv-required flag from the command line which was a leftover copy/paste from the original PR --- eng/scripts/docs/Docs-Onboarding.ps1 | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/eng/scripts/docs/Docs-Onboarding.ps1 b/eng/scripts/docs/Docs-Onboarding.ps1 index 9be956239078..7c501d3c6124 100644 --- a/eng/scripts/docs/Docs-Onboarding.ps1 +++ b/eng/scripts/docs/Docs-Onboarding.ps1 @@ -177,9 +177,8 @@ function Validate-Python-DocMsPackages($PackageInfo, $PackageInfos, $PackageSour -Path (Join-Path $outputRoot "docsOutput") # Because this would be running on a lab image, a virtual environment shouldn't be required - Write-Host "Executing: python -m py2docfx --param-file-path $outputJsonFile -o $outputDocsDir --no-venv-required" - #$py2docfxOutput = python -m py2docfx --param-file-path $outputJsonFile -o $outputDocsDir --no-venv-required 2>&1 - python -m py2docfx --param-file-path $outputJsonFile -o $outputDocsDir --no-venv-required + Write-Host "Executing: python -m py2docfx --param-file-path $outputJsonFile -o $outputDocsDir" + python -m py2docfx --param-file-path $outputJsonFile -o $outputDocsDir if ($LASTEXITCODE -ne 0) { LogWarning "py2docfx command failed, see output above." $allSucceeded = $false From ede3b2edfb9291bc5280d0bb44564c738a42ae8a Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Thu, 24 Oct 2024 15:50:39 -0700 Subject: [PATCH 3/9] Change the version to a pre-release for testing as the released version doesn't appear to have these changes --- eng/pipelines/templates/steps/install-rex-validation-tool.yml | 4 ++-- eng/scripts/docs/py2docfx.version.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/templates/steps/install-rex-validation-tool.yml b/eng/pipelines/templates/steps/install-rex-validation-tool.yml index 57b9871d370c..9f575c94d7a1 100644 --- a/eng/pipelines/templates/steps/install-rex-validation-tool.yml +++ b/eng/pipelines/templates/steps/install-rex-validation-tool.yml @@ -1,8 +1,8 @@ steps: - pwsh: | $py2docfxVer = Get-Content eng/scripts/docs/py2docfx.version.txt - Write-Host "python -m pip install py2docfx==$py2docfxVer" - python -m pip install py2docfx==$py2docfxVer + Write-Host "python -m pip install py2docfx==$py2docfxVer --pre" + python -m pip install py2docfx==$py2docfxVer --pre # After install py2docfx, Write-Host "Testing the install. Running python -m py2docfx -h to display the help" python -m py2docfx -h diff --git a/eng/scripts/docs/py2docfx.version.txt b/eng/scripts/docs/py2docfx.version.txt index 20f49513e15f..dd9944c181f5 100644 --- a/eng/scripts/docs/py2docfx.version.txt +++ b/eng/scripts/docs/py2docfx.version.txt @@ -1 +1 @@ -0.1.11 +0.1.11.dev1827111 From 2bf3990be7b74adf2c48c3f441c90457827dca27 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Mon, 28 Oct 2024 08:44:15 -0700 Subject: [PATCH 4/9] Update py2docfx version since previous version had problems on Linux --- eng/scripts/docs/py2docfx.version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/scripts/docs/py2docfx.version.txt b/eng/scripts/docs/py2docfx.version.txt index dd9944c181f5..4b6c2b392350 100644 --- a/eng/scripts/docs/py2docfx.version.txt +++ b/eng/scripts/docs/py2docfx.version.txt @@ -1 +1 @@ -0.1.11.dev1827111 +0.1.11.dev1902551 From 3f6c87e696c41b1b2180408a0ef20a12e90e1c9b Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Thu, 31 Oct 2024 09:50:59 -0700 Subject: [PATCH 5/9] Change the python command to add -u so the sdtout and stderr streams are unbuffered and we see the full output --- eng/scripts/docs/Docs-Onboarding.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/scripts/docs/Docs-Onboarding.ps1 b/eng/scripts/docs/Docs-Onboarding.ps1 index 7c501d3c6124..de3c384dc90e 100644 --- a/eng/scripts/docs/Docs-Onboarding.ps1 +++ b/eng/scripts/docs/Docs-Onboarding.ps1 @@ -176,9 +176,9 @@ function Validate-Python-DocMsPackages($PackageInfo, $PackageInfos, $PackageSour -ItemType Directory ` -Path (Join-Path $outputRoot "docsOutput") - # Because this would be running on a lab image, a virtual environment shouldn't be required - Write-Host "Executing: python -m py2docfx --param-file-path $outputJsonFile -o $outputDocsDir" - python -m py2docfx --param-file-path $outputJsonFile -o $outputDocsDir + # Force the python output to be unbuffered so we see more than just the warnings. + Write-Host "Executing: python -u -m py2docfx --param-file-path $outputJsonFile -o $outputDocsDir" + python -u -m py2docfx --param-file-path $outputJsonFile -o $outputDocsDir if ($LASTEXITCODE -ne 0) { LogWarning "py2docfx command failed, see output above." $allSucceeded = $false From ed9a53ffcf7899f2c998795b13d0d9d9adacf86a Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Thu, 31 Oct 2024 11:33:22 -0700 Subject: [PATCH 6/9] Updates for feedback and to see what's going on with the output --- .../templates/steps/install-rex-validation-tool.yml | 4 ++-- eng/scripts/docs/Docs-Onboarding.ps1 | 9 ++++++++- eng/scripts/docs/py2docfx.version.txt | 1 - eng/scripts/docs/py2docfx_requirements.txt | 1 + 4 files changed, 11 insertions(+), 4 deletions(-) delete mode 100644 eng/scripts/docs/py2docfx.version.txt create mode 100644 eng/scripts/docs/py2docfx_requirements.txt diff --git a/eng/pipelines/templates/steps/install-rex-validation-tool.yml b/eng/pipelines/templates/steps/install-rex-validation-tool.yml index 9f575c94d7a1..50aa5a4044f3 100644 --- a/eng/pipelines/templates/steps/install-rex-validation-tool.yml +++ b/eng/pipelines/templates/steps/install-rex-validation-tool.yml @@ -1,8 +1,8 @@ steps: - pwsh: | $py2docfxVer = Get-Content eng/scripts/docs/py2docfx.version.txt - Write-Host "python -m pip install py2docfx==$py2docfxVer --pre" - python -m pip install py2docfx==$py2docfxVer --pre + Write-Host "python -m pip install -r eng/scripts/docs/py2docfx_requirements.txt" + python -m pip install -r eng/scripts/docs/py2docfx_requirements.txt # After install py2docfx, Write-Host "Testing the install. Running python -m py2docfx -h to display the help" python -m py2docfx -h diff --git a/eng/scripts/docs/Docs-Onboarding.ps1 b/eng/scripts/docs/Docs-Onboarding.ps1 index de3c384dc90e..4e2b50248f75 100644 --- a/eng/scripts/docs/Docs-Onboarding.ps1 +++ b/eng/scripts/docs/Docs-Onboarding.ps1 @@ -176,9 +176,16 @@ function Validate-Python-DocMsPackages($PackageInfo, $PackageInfos, $PackageSour -ItemType Directory ` -Path (Join-Path $outputRoot "docsOutput") + # Adding these for diagnostics purposes so we know which version of python is being + # executed and dumping all the pip packages for this install of python + which python + python -m pip freeze --all + # Force the python output to be unbuffered so we see more than just the warnings. Write-Host "Executing: python -u -m py2docfx --param-file-path $outputJsonFile -o $outputDocsDir" - python -u -m py2docfx --param-file-path $outputJsonFile -o $outputDocsDir + $pyOutput = python -u -m py2docfx --param-file-path $outputJsonFile -o $outputDocsDir 2>&1 + Write-Host "pyOutput" + Write-Host $pyOutput if ($LASTEXITCODE -ne 0) { LogWarning "py2docfx command failed, see output above." $allSucceeded = $false diff --git a/eng/scripts/docs/py2docfx.version.txt b/eng/scripts/docs/py2docfx.version.txt deleted file mode 100644 index 4b6c2b392350..000000000000 --- a/eng/scripts/docs/py2docfx.version.txt +++ /dev/null @@ -1 +0,0 @@ -0.1.11.dev1902551 diff --git a/eng/scripts/docs/py2docfx_requirements.txt b/eng/scripts/docs/py2docfx_requirements.txt new file mode 100644 index 000000000000..22113936dea0 --- /dev/null +++ b/eng/scripts/docs/py2docfx_requirements.txt @@ -0,0 +1 @@ +py2docfx==0.1.11.dev1902551 From 224df931553c0088379c366a7f6b83d126c7e97f Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Thu, 31 Oct 2024 12:36:49 -0700 Subject: [PATCH 7/9] Remove the Get-Content line which should have been removed in the last commit --- eng/pipelines/templates/steps/install-rex-validation-tool.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/pipelines/templates/steps/install-rex-validation-tool.yml b/eng/pipelines/templates/steps/install-rex-validation-tool.yml index 50aa5a4044f3..f447b4dfca48 100644 --- a/eng/pipelines/templates/steps/install-rex-validation-tool.yml +++ b/eng/pipelines/templates/steps/install-rex-validation-tool.yml @@ -1,6 +1,5 @@ steps: - pwsh: | - $py2docfxVer = Get-Content eng/scripts/docs/py2docfx.version.txt Write-Host "python -m pip install -r eng/scripts/docs/py2docfx_requirements.txt" python -m pip install -r eng/scripts/docs/py2docfx_requirements.txt # After install py2docfx, From bfc64dba9357c3ee7ad9fcba9dd3ec8c925c2ba3 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Thu, 31 Oct 2024 13:47:24 -0700 Subject: [PATCH 8/9] Print the command output on actual different lines --- eng/scripts/docs/Docs-Onboarding.ps1 | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/eng/scripts/docs/Docs-Onboarding.ps1 b/eng/scripts/docs/Docs-Onboarding.ps1 index 4e2b50248f75..30d8782cc38f 100644 --- a/eng/scripts/docs/Docs-Onboarding.ps1 +++ b/eng/scripts/docs/Docs-Onboarding.ps1 @@ -140,7 +140,19 @@ function Validate-Python-DocMsPackages($PackageInfo, $PackageInfos, $PackageSour if (!$PackageInfos) { $PackageInfos = @($PackageInfo) } - # + + # Adding these for diagnostics purposes so we know which version of python is being + # executed and dumping all the pip packages for this install of python + Write-Host "Executing: which python" + $whichOutput = which python 2>&1 + $whichOutput | ForEach-Object { Write-Host $_ } + Write-Host "`n" + + Write-Host "Executing: python -m pip freeze --all" + $pipFreezeOutput = python -m pip freeze --all 2>&1 + $pipFreezeOutput | ForEach-Object { Write-Host $_ } + Write-Host "`n" + $tempDirs = @() $allSucceeded = $true try { @@ -176,16 +188,11 @@ function Validate-Python-DocMsPackages($PackageInfo, $PackageInfos, $PackageSour -ItemType Directory ` -Path (Join-Path $outputRoot "docsOutput") - # Adding these for diagnostics purposes so we know which version of python is being - # executed and dumping all the pip packages for this install of python - which python - python -m pip freeze --all - # Force the python output to be unbuffered so we see more than just the warnings. Write-Host "Executing: python -u -m py2docfx --param-file-path $outputJsonFile -o $outputDocsDir" $pyOutput = python -u -m py2docfx --param-file-path $outputJsonFile -o $outputDocsDir 2>&1 - Write-Host "pyOutput" - Write-Host $pyOutput + $pyOutput | ForEach-Object { Write-Host $_ } + Write-Host "`n" if ($LASTEXITCODE -ne 0) { LogWarning "py2docfx command failed, see output above." $allSucceeded = $false From 63d682692191289f1d09a051bd7830a991f999ef Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Thu, 31 Oct 2024 15:41:58 -0700 Subject: [PATCH 9/9] Finish the sentence in the comment --- eng/scripts/docs/Docs-Onboarding.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/scripts/docs/Docs-Onboarding.ps1 b/eng/scripts/docs/Docs-Onboarding.ps1 index 30d8782cc38f..70f8a7a68c0a 100644 --- a/eng/scripts/docs/Docs-Onboarding.ps1 +++ b/eng/scripts/docs/Docs-Onboarding.ps1 @@ -97,7 +97,7 @@ function Get-python-DocsPackagesAlreadyOnboarded($docRepoLocation, $moniker) { # but those are for track 1 or libraries released by other teams and not through our engineering system. # 2. extra_index_url only needs to exist on the object if PackageSourceOverride is set # 3. The reason this needs to be done using a json file instead of just a command line is because py2docfx -# doesn't handle the autodoc_default_options on the +# doesn't handle the autodoc_default_options on the command line. function Get-SinglePackageJsonForDocsValidation($PackageInfo, $PackageSourceOverride) {