From c500c88ef219aa4519cbaa4e0070701d81a76956 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Mon, 5 Feb 2024 09:51:01 -0800 Subject: [PATCH 1/7] Enable py2docfx docs gen tool, remove the dockerimage docs validation --- eng/pipelines/docindex.yml | 17 +- .../stages/archetype-python-release.yml | 51 +- .../steps/install-rex-validation-tool.yml | 6 + eng/scripts/Language-Settings.ps1 | 462 +++++------------- eng/scripts/docs/py2docfx.version.txt | 1 + 5 files changed, 163 insertions(+), 374 deletions(-) create mode 100644 eng/pipelines/templates/steps/install-rex-validation-tool.yml create mode 100644 eng/scripts/docs/py2docfx.version.txt diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml index 3b8be4f47740..4a4d93df7489 100644 --- a/eng/pipelines/docindex.yml +++ b/eng/pipelines/docindex.yml @@ -12,22 +12,17 @@ jobs: DocRepoLocation: $(Pipeline.Workspace)/docs DocRepoOwner: MicrosoftDocs DocRepoName: azure-docs-sdk-python - DocValidationImageId: azuresdkimages.azurecr.io/pyrefautocr:latest steps: - # Docs CI uses Python 3.9.13 + # py2docfx requires python >= 3.11 - task: UsePythonVersion@0 - displayName: 'Use Python 3.9.13' + displayName: 'Use Python 3.11' inputs: - versionSpec: '3.9.13' + versionSpec: '3.11' # Docs CI upgrades pip, wheel, and setuptools - pwsh: python -m pip install --upgrade pip wheel setuptools displayName: Update python tools for package verification - # Pull and build the docker image. - - template: /eng/common/pipelines/templates/steps/docker-pull-image.yml - parameters: - ImageId: "$(DocValidationImageId)" # Sync docs repo onboarding files/folders - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml parameters: @@ -41,6 +36,8 @@ jobs: - Name: $(DocRepoOwner)/$(DocRepoName) WorkingDirectory: $(DocRepoLocation) + - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml + - task: Powershell@2 inputs: pwsh: true @@ -53,7 +50,7 @@ jobs: inputs: pwsh: true filePath: eng/common/scripts/Update-DocsMsPackages.ps1 - arguments: -DocRepoLocation $(DocRepoLocation) -ImageId '$(DocValidationImageId)' + arguments: -DocRepoLocation $(DocRepoLocation) displayName: Update Docs Onboarding for main branch condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.MainUpdate'], 'true'))) @@ -92,7 +89,7 @@ jobs: parameters: BaseRepoBranch: $(DefaultBranch) BaseRepoOwner: $(DocRepoOwner) - CommitMsg: "Update docs CI configuration" + CommitMsg: "Update docs CI configuration Build: $(System.CollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)" TargetRepoName: $(DocRepoName) TargetRepoOwner: $(DocRepoOwner) WorkingDirectory: $(DocRepoLocation) diff --git a/eng/pipelines/templates/stages/archetype-python-release.yml b/eng/pipelines/templates/stages/archetype-python-release.yml index 50bd2962ae32..d63b4ff26081 100644 --- a/eng/pipelines/templates/stages/archetype-python-release.yml +++ b/eng/pipelines/templates/stages/archetype-python-release.yml @@ -9,7 +9,6 @@ parameters: TargetDocRepoOwner: '' TargetDocRepoName: '' PackageSourceOverride: "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" - DocValidationImageId: "azuresdkimages.azurecr.io/pyrefautocr:latest" stages: - ${{if and(in(variables['Build.Reason'], 'Manual', ''), eq(variables['System.TeamProject'], 'internal'))}}: @@ -207,6 +206,15 @@ stages: - sdk/**/*.md - .github/CODEOWNERS - download: current + + # py2docfx requires python >= 3.11 + - task: UsePythonVersion@0 + displayName: 'Use Python 3.11' + inputs: + versionSpec: '3.11' + + - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml + - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml parameters: PackageInfoLocations: @@ -219,7 +227,7 @@ stages: - docs-ref-services/ - metadata/ PackageSourceOverride: ${{parameters.PackageSourceOverride}} - DocValidationImageId: ${{parameters.DocValidationImageId}} + - ${{if ne(artifact.skipPublishDocGithubIo, 'true')}}: - deployment: UpdatePackageVersion displayName: "Update Package Version" @@ -320,21 +328,28 @@ stages: Get-ChildItem -Recurse $(Pipeline.Workspace)/${{parameters.ArtifactName}}/ displayName: Show visible artifacts - - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml - parameters: - PackageInfoLocations: - - ${{ each artifact in parameters.Artifacts }}: - - ${{if ne(artifact.skipPublishDocMs, 'true')}}: - - $(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{artifact.name}}.json - WorkingDirectory: $(System.DefaultWorkingDirectory) - TargetDocRepoOwner: ${{parameters.TargetDocRepoOwner}} - TargetDocRepoName: ${{parameters.TargetDocRepoName}} - Language: 'python' - DailyDocsBuild: true - SparseCheckoutPaths: - - docs-ref-services/ - - metadata/ - PackageSourceOverride: ${{parameters.PackageSourceOverride}} - DocValidationImageId: ${{parameters.DocValidationImageId}} + # py2docfx requires python >= 3.11 + - task: UsePythonVersion@0 + displayName: 'Use Python 3.11' + inputs: + versionSpec: '3.11' + + - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml + + - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml + parameters: + PackageInfoLocations: + - ${{ each artifact in parameters.Artifacts }}: + - ${{if ne(artifact.skipPublishDocMs, 'true')}}: + - $(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{artifact.name}}.json + WorkingDirectory: $(System.DefaultWorkingDirectory) + TargetDocRepoOwner: ${{parameters.TargetDocRepoOwner}} + TargetDocRepoName: ${{parameters.TargetDocRepoName}} + Language: 'python' + DailyDocsBuild: true + SparseCheckoutPaths: + - docs-ref-services/ + - metadata/ + PackageSourceOverride: ${{parameters.PackageSourceOverride}} - template: /eng/common/pipelines/templates/steps/docsms-ensure-validation.yml diff --git a/eng/pipelines/templates/steps/install-rex-validation-tool.yml b/eng/pipelines/templates/steps/install-rex-validation-tool.yml new file mode 100644 index 000000000000..e45aa317d290 --- /dev/null +++ b/eng/pipelines/templates/steps/install-rex-validation-tool.yml @@ -0,0 +1,6 @@ +steps: + - pwsh: | + $py2docfxVer = Get-Content eng/scripts/docs/py2docfx.version.txt + Write-Host "python -m pip install py2docfx==$py2docfxVer" + python -m pip install py2docfx==$py2docfxVer + displayName: Install py2docfx for package validation diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 57654c04f911..9e911d740980 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -209,336 +209,6 @@ function Get-python-GithubIoDocIndex() GenerateDocfxTocContent -tocContent $tocContent -lang "Python" -campaignId "UA-62780441-36" } -function ValidatePackage -{ - Param( - [Parameter(Mandatory=$true)] - [string]$packageName, - [Parameter(Mandatory=$true)] - [string]$packageVersion, - [Parameter(Mandatory=$false)] - [string]$PackageSourceOverride, - [Parameter(Mandatory=$false)] - [string]$DocValidationImageId - ) - $installValidationFolder = Join-Path ([System.IO.Path]::GetTempPath()) "validation" - if (!(Test-Path $installValidationFolder)) { - New-Item -ItemType Directory -Force -Path $installValidationFolder | Out-Null - } - # Add more validation by replicating as much of the docs CI process as - # possible - # https://github.com/Azure/azure-sdk-for-python/issues/20109 - $result = $true - if (!$DocValidationImageId) { - Write-Host "Validating using pip command directly on $packageName." - $result = FallbackValidation -packageName "$packageName" -packageVersion "$packageVersion" -workingDirectory $installValidationFolder -PackageSourceOverride $PackageSourceOverride - } else { - Write-Host "Validating using $DocValidationImageId on $packageName." - $result = DockerValidation -packageName "$packageName" -packageVersion "$packageVersion" ` - -PackageSourceOverride $PackageSourceOverride -DocValidationImageId $DocValidationImageId -workingDirectory $installValidationFolder - } - - return $result -} -function DockerValidation{ - Param( - [Parameter(Mandatory=$true)] - [string]$packageName, - [Parameter(Mandatory=$true)] - [string]$packageVersion, - [Parameter(Mandatory=$false)] - [string]$PackageSourceOverride, - [Parameter(Mandatory=$false)] - [string]$DocValidationImageId, - [Parameter(Mandatory=$false)] - [string]$workingDirectory - ) - if ($PackageSourceOverride) { - Write-Host "docker run -v ${workingDirectory}:/workdir/out -e TARGET_PACKAGE=$packageName -e TARGET_VERSION=$packageVersion -e EXTRA_INDEX_URL=$PackageSourceOverride -t $DocValidationImageId" - $commandLine = docker run -v "${workingDirectory}:/workdir/out" -e TARGET_PACKAGE=$packageName -e TARGET_VERSION=$packageVersion ` - -e EXTRA_INDEX_URL=$PackageSourceOverride -t $DocValidationImageId 2>&1 - } - else { - Write-Host "docker run -v ${workingDirectory}:/workdir/out -e TARGET_PACKAGE=$packageName -e TARGET_VERSION=$packageVersion -t $DocValidationImageId" - $commandLine = docker run -v "${workingDirectory}:/workdir/out" ` - -e TARGET_PACKAGE=$packageName -e TARGET_VERSION=$packageVersion -t $DocValidationImageId 2>&1 - } - # The docker exit codes: https://docs.docker.com/engine/reference/run/#exit-status - # If the docker failed because of docker itself instead of the application, - # we should skip the validation and keep the packages. - - if ($LASTEXITCODE -eq 125 -Or $LASTEXITCODE -eq 126 -Or $LASTEXITCODE -eq 127) { - $commandLine | ForEach-Object { Write-Debug $_ } - LogWarning "The `docker` command does not work with exit code $LASTEXITCODE. Fall back to npm install $packageName directly." - FallbackValidation -packageName "$packageName" -packageVersion "$packageVersion" -workingDirectory $workingDirectory -PackageSourceOverride $PackageSourceOverride - } - elseif ($LASTEXITCODE -ne 0) { - $commandLine | ForEach-Object { Write-Debug $_ } - LogWarning "Package $packageName ref docs validation failed." - return $false - } - return $true -} - -function FallbackValidation -{ - Param( - [Parameter(Mandatory=$true)] - [string]$packageName, - [Parameter(Mandatory=$true)] - [string]$packageVersion, - [Parameter(Mandatory=$true)] - [string]$workingDirectory, - [Parameter(Mandatory=$false)] - [string]$PackageSourceOverride - ) - $installTargetFolder = Join-Path $workingDirectory $packageName - New-Item -ItemType Directory -Force -Path $installTargetFolder | Out-Null - $packageExpression = "$packageName$packageVersion" - try { - $pipInstallOutput = "" - if ($PackageSourceOverride) { - Write-Host "python -m pip install $packageExpression --no-cache-dir --target $installTargetFolder --extra-index-url=$PackageSourceOverride" - $pipInstallOutput = python -m pip ` - install ` - $packageExpression ` - --no-cache-dir ` - --target $installTargetFolder ` - --extra-index-url=$PackageSourceOverride 2>&1 - } - else { - Write-Host "python -m pip install $packageExpression --no-cache-dir --target $installTargetFolder" - $pipInstallOutput = python -m pip ` - install ` - $packageExpression ` - --no-cache-dir ` - --target $installTargetFolder 2>&1 - } - if ($LASTEXITCODE -ne 0) { - LogWarning "python -m pip install failed for $packageExpression" - Write-Host $pipInstallOutput - return $false - } - } catch { - LogWarning "python -m pip install failed for $packageExpression with exception" - LogWarning $_.Exception - LogWarning $_.Exception.StackTrace - return $false - } - - return $true -} - -$PackageExclusions = @{ - 'azure-mgmt-videoanalyzer' = 'Unsupported doc directives: https://github.com/Azure/azure-sdk-for-python/issues/21563'; - 'azure-mgmt-quota' = 'Unsupported doc directives: https://github.com/Azure/azure-sdk-for-python/issues/21366'; - 'azure-mgmt-apimanagement' = 'Unsupported doc directives https://github.com/Azure/azure-sdk-for-python/issues/18084'; - 'azure-mgmt-reservations' = 'Unsupported doc directives https://github.com/Azure/azure-sdk-for-python/issues/18077'; - 'azure-mgmt-signalr' = 'Unsupported doc directives https://github.com/Azure/azure-sdk-for-python/issues/18085'; - 'azure-mgmt-mixedreality' = 'Missing version info https://github.com/Azure/azure-sdk-for-python/issues/18457'; - 'azure-mgmt-network' = 'Manual process used to build'; - 'azureml-fsspec' = 'Build issues related to Python requirements: https://github.com/Azure/azure-sdk-for-python/issues/30565'; - 'mltable' = 'Build issues related to Python requirements: https://github.com/Azure/azure-sdk-for-python/issues/30565'; - - 'azure-mgmt-compute' = 'Latest package requires Python >= 3.7 and this breaks docs build. https://github.com/Azure/azure-sdk-for-python/issues/22492'; - 'azure-mgmt-consumption' = 'Latest package requires Python >= 3.7 and this breaks docs build. https://github.com/Azure/azure-sdk-for-python/issues/22492'; - 'azure-mgmt-notificationhubs' = 'Latest package requires Python >= 3.7 and this breaks docs build. https://github.com/Azure/azure-sdk-for-python/issues/22492'; - 'azure-servicebus' = 'Latest package requires Python >= 3.7 and this breaks docs build. https://github.com/Azure/azure-sdk-for-python/issues/22492'; - 'azure-mgmt-web' = 'Latest package requires Python >= 3.7 and this breaks docs build. https://github.com/Azure/azure-sdk-for-python/issues/22492'; - 'azure-mgmt-netapp' = 'Latest package requires Python >= 3.7 and this breaks docs build. https://github.com/Azure/azure-sdk-for-python/issues/22492'; - 'azure-synapse-artifacts' = 'Latest package requires Python >= 3.7 and this breaks docs build. https://github.com/Azure/azure-sdk-for-python/issues/22492'; - 'azure-mgmt-streamanalytics' = 'Latest package requires Python >= 3.7 and this breaks docs build. https://github.com/Azure/azure-sdk-for-python/issues/22492'; - 'azure-ai-ml' = 'Docs CI build issues https://github.com/Azure/azure-sdk-for-python/issues/30774'; - - 'azure-keyvault' = 'Metapackages should not be documented'; -} - -function Update-python-DocsMsPackages($DocsRepoLocation, $DocsMetadata, $PackageSourceOverride, $DocValidationImageId) { - Write-Host "Excluded packages:" - foreach ($excludedPackage in $PackageExclusions.Keys) { - Write-Host " $excludedPackage - $($PackageExclusions[$excludedPackage])" - } - - $FilteredMetadata = $DocsMetadata.Where({ !($PackageExclusions.ContainsKey($_.Package)) }) - - UpdateDocsMsPackages ` - (Join-Path $DocsRepoLocation 'ci-configs/packages-preview.json') ` - 'preview' ` - $FilteredMetadata ` - $PackageSourceOverride ` - $DocValidationImageId - - UpdateDocsMsPackages ` - (Join-Path $DocsRepoLocation 'ci-configs/packages-latest.json') ` - 'latest' ` - $FilteredMetadata ` - $PackageSourceOverride ` - $DocValidationImageId -} - -function UpdateDocsMsPackages($DocConfigFile, $Mode, $DocsMetadata, $PackageSourceOverride, $DocValidationImageId) { - Write-Host "Updating configuration: $DocConfigFile with mode: $Mode" - $packageConfig = Get-Content $DocConfigFile -Raw | ConvertFrom-Json - - $outputPackages = @() - foreach ($package in $packageConfig.packages) { - $packageName = $package.package_info.name - if (!$packageName) { - Write-Host "Keeping package with no name: $($package.package_info)" - $outputPackages += $package - continue - } - - if ($package.package_info.install_type -ne 'pypi') { - Write-Host "Keeping package with install_type not 'pypi': $($package.package_info.name)" - $outputPackages += $package - continue - } - - if ($package.package_info.name.EndsWith("-nspkg")) { - Write-Host "Skipping $($package.package_info.name) because it's a namespace package." - continue - } - - # Do not filter by GA/Preview status because we want differentiate between - # tracked and non-tracked packages - $matchingPublishedPackageArray = $DocsMetadata.Where( { $_.Package -eq $packageName }) - - # If this package does not match any published packages keep it in the list. - # This handles packages which are not tracked in metadata but still need to - # be built in Docs CI. - if ($matchingPublishedPackageArray.Count -eq 0) { - Write-Host "Keep non-tracked package: $packageName" - $outputPackages += $package - continue - } - - if ($matchingPublishedPackageArray.Count -gt 1) { - LogWarning "Found more than one matching published package in metadata for $packageName; only updating first entry" - } - $matchingPublishedPackage = $matchingPublishedPackageArray[0] - - if ($Mode -eq 'preview' -and !$matchingPublishedPackage.VersionPreview.Trim()) { - # If we are in preview mode and the package does not have a superseding - # preview version, remove the package from the list. - Write-Host "Remove superseded preview package: $packageName" - continue - } - - if ($Mode -eq 'latest' -and !$matchingPublishedPackage.VersionGA.Trim()) { - LogWarning "Metadata is missing GA version for GA package $packageName. Keeping existing package." - $outputPackages += $package - continue - } - - $packageVersion = "==$($matchingPublishedPackage.VersionGA)" - if ($Mode -eq 'preview') { - if (!$matchingPublishedPackage.VersionPreview.Trim()) { - LogWarning "Metadata is missing preview version for preview package $packageName. Keeping existing package." - $outputPackages += $package - continue - } - $packageVersion = "==$($matchingPublishedPackage.VersionPreview)" - } - - # If upgrading the package, run basic sanity checks against the package - if ($package.package_info.version -ne $packageVersion) { - Write-Host "New version detected for $packageName ($packageVersion)" - if (!(ValidatePackage -packageName $packageName -packageVersion $packageVersion -PackageSourceOverride $PackageSourceOverride -DocValidationImageId $DocValidationImageId)) { - LogWarning "Package is not valid: $packageName. Keeping old version." - $outputPackages += $package - continue - } - - $package.package_info = Add-Member ` - -InputObject $package.package_info ` - -MemberType NoteProperty ` - -Name 'version' ` - -Value $packageVersion ` - -PassThru ` - -Force - if ($PackageSourceOverride) { - $package.package_info = Add-Member ` - -InputObject $package.package_info ` - -MemberType NoteProperty ` - -Name 'extra_index_url' ` - -Value $PackageSourceOverride ` - -PassThru ` - -Force - } - } - - Write-Host "Keeping tracked package: $packageName." - $outputPackages += $package - } - - $outputPackagesHash = @{} - foreach ($package in $outputPackages) { - # In some cases there is no $package.package_info.name, only hash if the - # name is set. - if ($package.package_info.name) { - $outputPackagesHash[$package.package_info.name] = $true - } - } - - $remainingPackages = @() - if ($Mode -eq 'preview') { - $remainingPackages = $DocsMetadata.Where({ - $_.VersionPreview.Trim() ` - -and !$outputPackagesHash.ContainsKey($_.Package) ` - -and !$_.Package.EndsWith("-nspkg") - }) - } else { - $remainingPackages = $DocsMetadata.Where({ - $_.VersionGA.Trim() ` - -and !$outputPackagesHash.ContainsKey($_.Package) ` - -and !$_.Package.EndsWith("-nspkg") - }) - } - - # Add packages that exist in the metadata but are not onboarded in docs config - foreach ($package in $remainingPackages) { - $packageName = $package.Package - $packageVersion = "==$($package.VersionGA)" - if ($Mode -eq 'preview') { - $packageVersion = "==$($package.VersionPreview)" - } - if (!(ValidatePackage -packageName $packageName -packageVersion $packageVersion -PackageSourceOverride $PackageSourceOverride -DocValidationImageId $DocValidationImageId)) { - LogWarning "Package is not valid: $packageName. Cannot onboard." - continue - } - - Write-Host "Add new package from metadata: $packageName" - if ($PackageSourceOverride) { - $package = [ordered]@{ - package_info = [ordered]@{ - name = $packageName; - install_type = 'pypi'; - prefer_source_distribution = 'true'; - version = $packageVersion; - extra_index_url = $PackageSourceOverride - }; - exclude_path = @("test*","example*","sample*","doc*"); - } - } else { - $package = [ordered]@{ - package_info = [ordered]@{ - name = $packageName; - install_type = 'pypi'; - prefer_source_distribution = 'true'; - version = $packageVersion; - }; - exclude_path = @("test*","example*","sample*","doc*"); - } - } - $outputPackages += $package - } - - $packageConfig.packages = $outputPackages - $packageConfig | ConvertTo-Json -Depth 100 | Set-Content $DocConfigFile - Write-Host "Onboarding configuration written to: $DocConfigFile" -} - # function is used to auto generate API View function Find-python-Artifacts-For-Apireview($artifactDir, $artifactName) { @@ -651,33 +321,133 @@ function Import-Dev-Cert-python python $pathToScript } +# The package info format required by py2docfx to verify a single library. +# The format can be found at https://github.com/MicrosoftDocs/azure-docs-sdk-python/blob/main/ci-configs/packages-latest.json +# which is a file that contains all of package_info objects. For verifying a single library +# the same format is used but there's only a single package_info object in the packages list. +# This is an example: +# { +# "packages": [ +# { +# "package_info": { +# "name": "azure-core", +# "install_type": "pypi", +# "prefer_source_distribution": "true", +# "version": "==1.29.6a20231207001", +# "extra_index_url": "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" +# }, +# "exclude_path": [ +# "test*", +# "example*", +# "sample*", +# "doc*" +# ] +# } +# ] +# } +# The above example contains the common settings for Python track 2 libraries. +# There are a few things of note: +# 1. install_type can be something other than pypi. source_code or dist_file are two examples of this +# but those are for track 1 or libraries released by other teams and not through our engineering system. +# 2. extra_index_url only needs to exist on the object if PackageSourceOverride is set +# 3. The reason this needs to be done using a json file instead of just a command line is because py2docfx +# doesn't handle the autodoc_default_options on the +function Get-SinglePackageJsonForDocsValidation($PackageInfo, $PackageSourceOverride) +{ + + $packageArr = @() + $packageSpec = [ordered]@{ + package_info = [ordered]@{ + name = $PackageInfo.Name + install_type = 'pypi' + prefer_source_distribution = 'true' + version = "==$($PackageInfo.Version)" + } + exclude_path = @("test*","example*","sample*","doc*") + } + if ($PackageSourceOverride) { + $packageSpec['package_info']['extra_index_url'] = $PackageSourceOverride + } + # Data-plane packages (not mgmt packages, and not manually added '00`packages) + # should document inherited members + if ($PackageInfo.Name -notlike 'azure-mgmt-*' -and $PackageInfo.Name -notlike '*-00-*') { + $packageSpec['extension_config'] = @{ 'autodoc_default_options' = @{ 'inherited-members' = 1 } } + } + $packageArr += $packageSpec + + # "packages" must be an array of packages even if there's only a single package in it. + # There are other top level elements, required_packages, target_repo etc. that aren't + # required for validation of a single package. + $docsConfigPackage = [ordered]@{ + packages = $packageArr + } + + # Return the JSon string + return $docsConfigPackage | ConvertTo-Json -Depth 10 +} + # Defined in common.ps1 as: # $ValidateDocsMsPackagesFn = "Validate-${Language}-DocMsPackages" -function Validate-Python-DocMsPackages ($PackageInfo, $PackageInfos, $PackageSourceOverride, $DocValidationImageId) +function Validate-Python-DocMsPackages($PackageInfo, $PackageInfos, $PackageSourceOverride, $DocValidationImageId) { # While eng/common/scripts/Update-DocsMsMetadata.ps1 is still passing a single packageInfo, process as a batch if (!$PackageInfos) { $PackageInfos = @($PackageInfo) } - + # + $tempDirs = @() $allSucceeded = $true - foreach ($item in $PackageInfos) { - # If the Version is IGNORE that means it's a source install and those aren't run through ValidatePackage - if ($item.Version -eq 'IGNORE') { - continue - } - - $result = ValidatePackage ` - -packageName $item.Name ` - -packageVersion "==$($item.Version)" ` - -PackageSourceOverride $PackageSourceOverride ` - -DocValidationImageId $DocValidationImageId + try { + foreach ($packageInfo in $PackageInfos) { + # Some packages won't have a version and this is the case when they're being onboarded manually + # and there's no version, only a repository and a SHA. In that case package we skip traditional + # package validation since the library doesn't exist yet outside of source and there's nothing + # the verification tools can do with this. + if ($packageInfo.Version -eq 'IGNORE') { + continue + } - if (!$result) { - $allSucceeded = $false + # Create a temporary directory. The json file being passed to py2docfx will be in the root and + # the docs will be generated to a docsOutput subdirectory. + $outputRoot = New-Item ` + -ItemType Directory ` + -Path (Join-Path ([System.IO.Path]::GetTempPath()) ([System.IO.Path]::GetRandomFileName())) + + $tempDirs += $outputRoot + + # Create the JSON file + $outputJsonFile = New-Item ` + -ItemType File ` + -Path (Join-Path $outputRoot ($packageInfo.Name + ".json")) + + ## Write out the json file and echo the contents + $JsonString = Get-SinglePackageJsonForDocsValidation $packageInfo $PackageSourceOverride + $JsonString | Out-File $outputJsonFile + Write-Host "$JsonString" + + # Create the docs output subdirectory. This is where the tool will generate its docs + $outputDocsDir = New-Item ` + -ItemType Directory ` + -Path (Join-Path $outputRoot "docsOutput") + + # Because this would be running on a lab image, a virtual environment shouldn't be required + Write-Host "Executing: python -m py2docfx --param-file-path $outputJsonFile -o $outputDocsDir --no-venv-required" + $py2docfxOutput = python -m py2docfx --param-file-path $outputJsonFile -o $outputDocsDir --no-venv-required 2>&1 + + Write-Host $py2docfxOutput + if ($LASTEXITCODE -ne 0) { + LogWarning "py2docfx command failed, see output below." + $allSucceeded = $false + } + } + } + finally { + # Clean up any temp directories + foreach ($tempDir in $tempDirs) + { + Remove-Item -Force -Recurse $tempDir | Out-Null } } - return $allSucceeded } diff --git a/eng/scripts/docs/py2docfx.version.txt b/eng/scripts/docs/py2docfx.version.txt new file mode 100644 index 000000000000..6da28dde76d6 --- /dev/null +++ b/eng/scripts/docs/py2docfx.version.txt @@ -0,0 +1 @@ +0.1.1 \ No newline at end of file From 99e1a41e8f17a1e8bcbeb6a4d3b0b2d0eab63e81 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Mon, 5 Feb 2024 11:10:50 -0800 Subject: [PATCH 2/7] install rex tool before update-docsms-metadata.yml --- eng/pipelines/templates/stages/archetype-python-release.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/eng/pipelines/templates/stages/archetype-python-release.yml b/eng/pipelines/templates/stages/archetype-python-release.yml index d63b4ff26081..3dfa55b2692d 100644 --- a/eng/pipelines/templates/stages/archetype-python-release.yml +++ b/eng/pipelines/templates/stages/archetype-python-release.yml @@ -215,6 +215,9 @@ stages: - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml + + - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml + - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml parameters: PackageInfoLocations: @@ -336,6 +339,8 @@ stages: - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml + - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml + - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml parameters: PackageInfoLocations: From 59a17c731a9f35c27de384cf57643a500aca81ab Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Mon, 5 Feb 2024 13:55:14 -0800 Subject: [PATCH 3/7] Need to UsePythonVersion before rex install to ensure the correct version of python for the tool --- .../templates/stages/archetype-python-release.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-python-release.yml b/eng/pipelines/templates/stages/archetype-python-release.yml index 3dfa55b2692d..868e226e7a75 100644 --- a/eng/pipelines/templates/stages/archetype-python-release.yml +++ b/eng/pipelines/templates/stages/archetype-python-release.yml @@ -216,6 +216,12 @@ stages: - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml + # py2docfx requires python >= 3.11 + - task: UsePythonVersion@0 + displayName: 'Use Python 3.11' + inputs: + versionSpec: '3.11' + - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml @@ -339,8 +345,6 @@ stages: - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml - - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml - - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml parameters: PackageInfoLocations: From eaa132db94a3f501660b9925fc1ceb1a28a437b3 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Mon, 5 Feb 2024 14:37:47 -0800 Subject: [PATCH 4/7] Remove the py2doxfx output redirection --- eng/scripts/Language-Settings.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 9e911d740980..20eb2ff08556 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -432,11 +432,11 @@ function Validate-Python-DocMsPackages($PackageInfo, $PackageInfos, $PackageSour # Because this would be running on a lab image, a virtual environment shouldn't be required Write-Host "Executing: python -m py2docfx --param-file-path $outputJsonFile -o $outputDocsDir --no-venv-required" - $py2docfxOutput = python -m py2docfx --param-file-path $outputJsonFile -o $outputDocsDir --no-venv-required 2>&1 - - Write-Host $py2docfxOutput + #$py2docfxOutput = python -m py2docfx --param-file-path $outputJsonFile -o $outputDocsDir --no-venv-required 2>&1 + python -m py2docfx --param-file-path $outputJsonFile -o $outputDocsDir --no-venv-required if ($LASTEXITCODE -ne 0) { - LogWarning "py2docfx command failed, see output below." + #Write-Host $py2docfxOutput + LogWarning "py2docfx command failed, see output above." $allSucceeded = $false } } From 59d8baabcedb12fc6e950411bbd18df82123926b Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Fri, 5 Apr 2024 09:05:17 -0700 Subject: [PATCH 5/7] remove commented out print line --- eng/scripts/Language-Settings.ps1 | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 20eb2ff08556..9f1415c942eb 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -435,7 +435,6 @@ function Validate-Python-DocMsPackages($PackageInfo, $PackageInfos, $PackageSour #$py2docfxOutput = python -m py2docfx --param-file-path $outputJsonFile -o $outputDocsDir --no-venv-required 2>&1 python -m py2docfx --param-file-path $outputJsonFile -o $outputDocsDir --no-venv-required if ($LASTEXITCODE -ne 0) { - #Write-Host $py2docfxOutput LogWarning "py2docfx command failed, see output above." $allSucceeded = $false } From ef73ac77dee884820cbbef815482c85fc22bb265 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Fri, 5 Apr 2024 09:50:24 -0700 Subject: [PATCH 6/7] stupid merge tool is a piece of trash --- .../templates/stages/archetype-python-release.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-python-release.yml b/eng/pipelines/templates/stages/archetype-python-release.yml index 868e226e7a75..d63b4ff26081 100644 --- a/eng/pipelines/templates/stages/archetype-python-release.yml +++ b/eng/pipelines/templates/stages/archetype-python-release.yml @@ -207,15 +207,6 @@ stages: - .github/CODEOWNERS - download: current - # py2docfx requires python >= 3.11 - - task: UsePythonVersion@0 - displayName: 'Use Python 3.11' - inputs: - versionSpec: '3.11' - - - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml - - # py2docfx requires python >= 3.11 - task: UsePythonVersion@0 displayName: 'Use Python 3.11' From adc36ad5edbef43d8e20ed09710af7145baf952a Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Fri, 19 Apr 2024 08:30:58 -0700 Subject: [PATCH 7/7] Test the install of py2docfx by executing it to just display the help --- eng/pipelines/templates/steps/install-rex-validation-tool.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eng/pipelines/templates/steps/install-rex-validation-tool.yml b/eng/pipelines/templates/steps/install-rex-validation-tool.yml index e45aa317d290..57b9871d370c 100644 --- a/eng/pipelines/templates/steps/install-rex-validation-tool.yml +++ b/eng/pipelines/templates/steps/install-rex-validation-tool.yml @@ -3,4 +3,7 @@ steps: $py2docfxVer = Get-Content eng/scripts/docs/py2docfx.version.txt Write-Host "python -m pip install py2docfx==$py2docfxVer" python -m pip install py2docfx==$py2docfxVer + # After install py2docfx, + Write-Host "Testing the install. Running python -m py2docfx -h to display the help" + python -m py2docfx -h displayName: Install py2docfx for package validation