Enable py2docfx docs gen tool, remove the dockerimage docs validation

eng/pipelines/docindex.yml

@@ -12,22 +12,17 @@ jobs:
       DocRepoLocation: $(Pipeline.Workspace)/docs
       DocRepoOwner: MicrosoftDocs
       DocRepoName: azure-docs-sdk-python
-      DocValidationImageId: azuresdkimages.azurecr.io/pyrefautocr:latest
     steps:
-      # Docs CI uses Python 3.9.13
+      # py2docfx requires python >= 3.11
       - task: UsePythonVersion@0
-        displayName: 'Use Python 3.9.13'
+        displayName: 'Use Python 3.11'
         inputs:
-          versionSpec: '3.9.13'
+          versionSpec: '3.11'
 
       # Docs CI upgrades pip, wheel, and setuptools
       - pwsh: python -m pip install --upgrade pip wheel setuptools
         displayName: Update python tools for package verification
 
-      # Pull and build the docker image.
-      - template: /eng/common/pipelines/templates/steps/docker-pull-image.yml
-        parameters:
-          ImageId: "$(DocValidationImageId)"
       # Sync docs repo onboarding files/folders
       - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
         parameters:
@@ -41,6 +36,8 @@ jobs:
             - Name: $(DocRepoOwner)/$(DocRepoName)
               WorkingDirectory: $(DocRepoLocation)
 
+      - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml
+
       - task: Powershell@2
         inputs:
           pwsh: true
@@ -53,7 +50,7 @@ jobs:
         inputs:
           pwsh: true
           filePath: eng/common/scripts/Update-DocsMsPackages.ps1
-          arguments: -DocRepoLocation $(DocRepoLocation) -ImageId '$(DocValidationImageId)'
+          arguments: -DocRepoLocation $(DocRepoLocation)
         displayName: Update Docs Onboarding for main branch
         condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.MainUpdate'], 'true')))
 
@@ -92,7 +89,7 @@ jobs:
         parameters:
           BaseRepoBranch: $(DefaultBranch)
           BaseRepoOwner: $(DocRepoOwner)
-          CommitMsg: "Update docs CI configuration"
+          CommitMsg: "Update docs CI configuration Build: $(System.CollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)"
           TargetRepoName: $(DocRepoName)
           TargetRepoOwner: $(DocRepoOwner)
           WorkingDirectory: $(DocRepoLocation)

eng/pipelines/templates/stages/archetype-python-release.yml

@@ -9,7 +9,6 @@ parameters:
   TargetDocRepoOwner: ''
   TargetDocRepoName: ''
   PackageSourceOverride: "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/"
-  DocValidationImageId: "azuresdkimages.azurecr.io/pyrefautocr:latest"
 
 stages:
   - ${{if and(in(variables['Build.Reason'], 'Manual', ''), eq(variables['System.TeamProject'], 'internal'))}}:
@@ -207,6 +206,15 @@ stages:
                                 - sdk/**/*.md
                                 - .github/CODEOWNERS
                           - download: current
+
+                          # py2docfx requires python >= 3.11
+                          - task: UsePythonVersion@0
+                            displayName: 'Use Python 3.11'
+                            inputs:
+                              versionSpec: '3.11'
+
+                          - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml
+
                           - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml
                             parameters:
                               PackageInfoLocations:
@@ -219,7 +227,7 @@ stages:
                                 - docs-ref-services/
                                 - metadata/
                               PackageSourceOverride: ${{parameters.PackageSourceOverride}}
-                              DocValidationImageId: ${{parameters.DocValidationImageId}}
+
           - ${{if ne(artifact.skipPublishDocGithubIo, 'true')}}:
             - deployment: UpdatePackageVersion
               displayName: "Update Package Version"
@@ -320,21 +328,28 @@ stages:
               Get-ChildItem -Recurse $(Pipeline.Workspace)/${{parameters.ArtifactName}}/
             displayName: Show visible artifacts
 
-          - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml
-            parameters:
-              PackageInfoLocations:
-                - ${{ each artifact in parameters.Artifacts }}:
-                  - ${{if ne(artifact.skipPublishDocMs, 'true')}}:
-                    - $(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{artifact.name}}.json
-              WorkingDirectory: $(System.DefaultWorkingDirectory)
-              TargetDocRepoOwner: ${{parameters.TargetDocRepoOwner}}
-              TargetDocRepoName: ${{parameters.TargetDocRepoName}}
-              Language: 'python'
-              DailyDocsBuild: true
-              SparseCheckoutPaths:
-                - docs-ref-services/
-                - metadata/
-              PackageSourceOverride: ${{parameters.PackageSourceOverride}}
-              DocValidationImageId: ${{parameters.DocValidationImageId}}
+        # py2docfx requires python >= 3.11
+        - task: UsePythonVersion@0
+          displayName: 'Use Python 3.11'
+          inputs:
+            versionSpec: '3.11'
+
+        - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml
+
+        - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml
+          parameters:
+            PackageInfoLocations:
+              - ${{ each artifact in parameters.Artifacts }}:
+                - ${{if ne(artifact.skipPublishDocMs, 'true')}}:
+                  - $(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{artifact.name}}.json
+            WorkingDirectory: $(System.DefaultWorkingDirectory)
+            TargetDocRepoOwner: ${{parameters.TargetDocRepoOwner}}
+            TargetDocRepoName: ${{parameters.TargetDocRepoName}}
+            Language: 'python'
+            DailyDocsBuild: true
+            SparseCheckoutPaths:
+              - docs-ref-services/
+              - metadata/
+            PackageSourceOverride: ${{parameters.PackageSourceOverride}}
 
           - template: /eng/common/pipelines/templates/steps/docsms-ensure-validation.yml

eng/pipelines/templates/steps/install-rex-validation-tool.yml

@@ -0,0 +1,6 @@
+steps:
+  - pwsh: |
+      $py2docfxVer = Get-Content eng/scripts/docs/py2docfx.version.txt
+      Write-Host "python -m pip install py2docfx==$py2docfxVer"
+      python -m pip install py2docfx==$py2docfxVer
+    displayName: Install py2docfx for package validation