Move from using the docker image to java2docfx for docs validation (#…

…39744)

* Move from using the docker image to java2docfx for docs validation

* Temporarily turn on docs processing for template libraries for testing

* Actually install the rex validation tool

* Fix the if not Test-Path statement

* Update java2docfx version and add a couple of diagnostics output lines

* Add missing close paren

* Ensure that Sort-Object always returns an array even if there's only one item

* add another piece of diagnostics output

* trying one more thing

* remove some diag, add other

* Remove the additional diagnostics, add permanent output message

* Invoke java -jar on java2docfx to show the help command to ensure the install is okay

* fiddling with the java -jar command

* Set the working directory to the java2docfx directory before executing the mvn dependency download

* Actually create the directory before trying to set location...oops

* Update rex validation to verify MAVEN_HOME is set

* Updates for Java PR 39875 which had changes from this PR that were more immediate

* Update java2docfx version

* remove check for MAVEN_HOME which was only for testing

* Update the version of java2docfx to test a fix

* Update version of java2docfx to 1.0.4

* revert template's ci.yml changes that were only necessary to test java2docfx

eng/pipelines/docindex.yml

@@ -13,7 +13,6 @@ jobs:
       DailyDocRepoLocation: $(Pipeline.Workspace)/daily
       DocRepoOwner: Azure
       DocRepoName: azure-docs-sdk-java
-      DocValidationImageId: azuresdkimages.azurecr.io/javarefautocr:latest
     steps:
       # Sync docs repo onboarding files/folders
       - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
@@ -29,10 +28,6 @@ jobs:
               WorkingDirectory: $(DocRepoLocation)
             - Name: azure-sdk/$(DocRepoName)
               WorkingDirectory: $(DailyDocRepoLocation)
-      # Pull and build the docker image.
-      - template: /eng/common/pipelines/templates/steps/docker-pull-image.yml
-        parameters:
-          ImageId: "$(DocValidationImageId)"
 
       - task: Powershell@2
         inputs:
@@ -46,7 +41,7 @@ jobs:
         inputs:
           pwsh: true
           filePath: eng/common/scripts/Update-DocsMsPackages.ps1
-          arguments: -DocRepoLocation $(DocRepoLocation) -ImageId '$(DocValidationImageId)'
+          arguments: -DocRepoLocation $(DocRepoLocation)
         displayName: Update Docs Onboarding for main branch
         condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.MainUpdate'], 'true')))
 
@@ -85,7 +80,7 @@ jobs:
         parameters:
           BaseRepoBranch: $(DefaultBranch)
           BaseRepoOwner: $(DocRepoOwner)
-          CommitMsg: "Update docs CI configuration"
+          CommitMsg: "Update docs CI configuration Build: $(System.CollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)"
           TargetRepoName: $(DocRepoName)
           TargetRepoOwner: $(DocRepoOwner)
           WorkingDirectory: $(DocRepoLocation)
@@ -95,20 +90,7 @@ jobs:
         parameters:
           DailyBranchVariableName: DailyDocsBranchName
 
-      # Docs daily updates is supposed to download packages from public feed repository, so we have to specify additional repositories in a POM or the profile.
-      # Here is maven documentation: https://maven.apache.org/guides/mini/guide-multiple-repositories.html
-      - powershell: |
-          # Linux mvn `setting.xml` is sitting under path `~/.m2/setting.xml`
-          Get-Command mvn
-          if (!(Test-Path '~/.m2/')) {
-            mkdir ~/.m2/
-          }
-          if (Test-Path '~/.m2/setting.xml') {
-            Write-Host "'setting.xml' exists. Overwriting the file to support multiple repositories."
-          }
-          Copy-Item "./eng/repo-docs/docms/daily.update.setting.xml" -Destination "~/.m2/settings.xml"
-        displayName: 'Configure mvn'
-        workingDirectory: $(Build.SourcesDirectory)
+      - template: /eng/pipelines/templates/steps/mvn-linux-settings-for-docs.yml
 
       - task: Powershell@2
         inputs:

eng/pipelines/templates/stages/archetype-java-release-batch.yml

@@ -319,10 +319,6 @@ stages:
             image: azsdk-pool-mms-ubuntu-2004-1espt
             os: linux
 
-          variables:
-            - name: DocValidationImageId
-              value: azuresdkimages.azurecr.io/javarefautocr:latest
-
           strategy:
             runOnce:
               deploy:
@@ -341,6 +337,8 @@ stages:
 
                   - template: /eng/pipelines/templates/steps/mvn-linux-repository-settings.yml
 
+                  - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml
+
                   - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml
                     parameters:
                       PackageInfoLocations:
@@ -354,7 +352,6 @@ stages:
                       SparseCheckoutPaths:
                         - docs-ref-services/
                         - metadata/
-                      DocValidationImageId: "$(DocValidationImageId)"
 
         - deployment: PublishDocs
           displayName: Publish Docs to GitHubIO Blob Storage
@@ -464,9 +461,6 @@ stages:
           name: azsdk-pool-mms-ubuntu-2004-general
           image: azsdk-pool-mms-ubuntu-2004-1espt
           os: linux
-        variables:
-          - name: DocValidationImageId
-            value: azuresdkimages.azurecr.io/javarefautocr:latest
         steps:
           - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
             parameters:
@@ -485,6 +479,8 @@ stages:
 
           - template: /eng/pipelines/templates/steps/mvn-linux-repository-settings.yml
 
+          - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml
+
           - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml
             parameters:
               PackageInfoLocations:
@@ -499,6 +495,5 @@ stages:
               SparseCheckoutPaths:
                 - docs-ref-services/
                 - metadata/
-              DocValidationImageId: "$(DocValidationImageId)"
 
           - template: /eng/common/pipelines/templates/steps/docsms-ensure-validation.yml

eng/pipelines/templates/stages/archetype-java-release-patch.yml

@@ -239,8 +239,6 @@ stages:
 
           variables:
             - template: /eng/pipelines/templates/variables/globals.yml
-            - name: DocValidationImageId
-              value: azuresdkimages.azurecr.io/javarefautocr:latest
 
           strategy:
             runOnce:
@@ -261,6 +259,8 @@ stages:
 
                   - template: /eng/pipelines/templates/steps/mvn-linux-repository-settings.yml
 
+                  - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml
+
                   - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml
                     parameters:
                       PackageInfoLocations:
@@ -274,7 +274,6 @@ stages:
                       SparseCheckoutPaths:
                         - docs-ref-services/
                         - metadata/
-                      DocValidationImageId: "$(DocValidationImageId)"
 
         - deployment: PublishDocs
           displayName: Publish Docs to GitHubIO Blob Storage

eng/pipelines/templates/stages/archetype-java-release.yml

@@ -257,8 +257,6 @@ stages:
 
               variables:
                 - template: /eng/pipelines/templates/variables/globals.yml
-                - name: DocValidationImageId
-                  value: azuresdkimages.azurecr.io/javarefautocr:latest
 
               strategy:
                 runOnce:
@@ -275,10 +273,11 @@ stages:
                       - download: current
                         displayName: 'Download Artifact: ${{parameters.ArtifactName}}'
                         artifact: ${{parameters.ArtifactName}}
-                      # Pull and build the docker image.
 
                       - template: /eng/pipelines/templates/steps/mvn-linux-repository-settings.yml
 
+                      - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml
+
                       - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml
                         parameters:
                           PackageInfoLocations:
@@ -290,7 +289,6 @@ stages:
                           SparseCheckoutPaths:
                             - docs-ref-services/
                             - metadata/
-                          DocValidationImageId: "$(DocValidationImageId)"
 
           - ${{if ne(artifact.skipPublishDocGithubIo, 'true')}}:
             - deployment: PublishDocs
@@ -446,8 +444,6 @@ stages:
           os: linux
         variables:
           - template: /eng/pipelines/templates/variables/globals.yml
-          - name: DocValidationImageId
-            value: azuresdkimages.azurecr.io/javarefautocr:latest
         steps:
           - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
             parameters:
@@ -466,6 +462,8 @@ stages:
 
           - template: /eng/pipelines/templates/steps/mvn-linux-repository-settings.yml
 
+          - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml
+
           - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml
             parameters:
               PackageInfoLocations:
@@ -480,6 +478,5 @@ stages:
               SparseCheckoutPaths:
                 - docs-ref-services/
                 - metadata/
-              DocValidationImageId: "$(DocValidationImageId)"
 
           - template: /eng/common/pipelines/templates/steps/docsms-ensure-validation.yml

eng/pipelines/templates/steps/install-rex-validation-tool.yml

@@ -0,0 +1,22 @@
+steps:
+  # Create a java2docfx subdirectory in the $(Build.BinariesDirectory) and install the java2docfx there
+  # This way, the jar file is in its own subdirectory and isolated.
+  - pwsh: |
+      $java2docfxVer = Get-Content eng/scripts/docs/java2docfx.version.txt
+      $java2docfxDir = Join-Path $(Build.BinariesDirectory) "java2docfx"
+      New-Item $java2docfxDir -ItemType Directory | Out-Null
+      $originLocation = Get-Location
+      try {
+        # Set the location into the created java2docfx directory . Running mvn from the repository root
+        # picks up the root POM and processes that while doing the dependency:copy in spite of
+        # the fact it has nothing to do with this download.
+        Set-Location $java2docfxDir
+        Write-Host "mvn dependency:copy -Dartifact=""com.microsoft:java2docfx:$java2docfxVer"" -DoutputDirectory=""$java2docfxDir"""
+        mvn dependency:copy -Dartifact="com.microsoft:java2docfx:$java2docfxVer" -DoutputDirectory="$java2docfxDir"
+        $java2docfxJarLoc = Join-Path $java2docfxDir -ChildPath "java2docfx-$java2docfxVer.jar"
+        Write-Host "Testing the install, running java -jar $java2docfxJarLoc -h"
+        java -jar $java2docfxJarLoc -h
+      } finally {
+        Set-Location $originLocation
+      }
+    displayName: Install java2docfx for package validation

eng/repo-docs/docms/daily.update.setting.xml

@@ -18,6 +18,16 @@
                         <enabled>true</enabled>
                     </snapshots>
                 </repository>
+                <repository>
+                    <id>docs-public-packages</id>
+                    <url>https://docfx.pkgs.visualstudio.com/docfx/_packaging/docs-public-packages/maven/v1</url>
+                    <releases>
+                        <enabled>true</enabled>
+                    </releases>
+                    <snapshots>
+                        <enabled>true</enabled>
+                    </snapshots>
+                </repository>
             </repositories>
         </profile>
     </profiles>