diff --git a/.vscode/cspell.json b/.vscode/cspell.json index 5da0892cef7b..7b74502d6c2f 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -167,7 +167,11 @@ "overrides": [ { "filename": "eng/pipelines", - "words": ["azuresdkartifacts", "gdnbaselines", "policheck"] + "words": ["azuresdkartifacts", "gdnbaselines", "policheck", "issecret", "dlrw"] + }, + { + "filename": "eng/tools/rush-runner.js", + "words": ["Jsons"] }, { "filename": "sdk/apimanagement/api-management-custom-widgets-scaffolder/review/api-management-custom-widgets-scaffolder.api.md", diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 2d91faa403f0..d6f5c38b0ebb 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -74,9 +74,16 @@ jobs: MatrixConfigs: ${{ parameters.MatrixConfigs }} MatrixFilters: ${{ parameters.MatrixFilters }} MatrixReplace: ${{ parameters.MatrixReplace }} + SparseCheckoutPaths: [ "sdk/", ".vscode"] CloudConfig: Cloud: Public AdditionalParameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} Artifacts: ${{ parameters.Artifacts }} TestProxy: ${{ parameters.TestProxy }} + ${{ if eq(parameters.ServiceDirectory, 'auto') }}: + EnablePRGeneration: true + PreGenerationSteps: + - template: /eng/common/pipelines/templates/steps/save-package-properties.yml + parameters: + ServiceDirectory: ${{parameters.ServiceDirectory}} diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml index dfe7a8054d88..41e21b077c91 100644 --- a/eng/pipelines/templates/steps/analyze.yml +++ b/eng/pipelines/templates/steps/analyze.yml @@ -4,13 +4,22 @@ parameters: TestPipeline: false steps: + - template: /eng/common/pipelines/templates/steps/save-package-properties.yml + parameters: + ServiceDirectory: ${{parameters.ServiceDirectory}} + + - template: /eng/pipelines/templates/steps/set-artifact-packages.yml + parameters: + PackageInfo: $(Build.ArtifactStagingDirectory)/PackageInfo + Artifacts: ${{ parameters.Artifacts }} + - template: /eng/common/pipelines/templates/steps/check-spelling.yml - task: PowerShell@2 inputs: targetType: 'filePath' - filePath: eng/scripts/spell-check-public-api.ps1 - arguments: -ServiceDirectory ${{ parameters.ServiceDirectory }} + filePath: eng/scripts/spell-check-public-apis.ps1 + arguments: -ChangedServices "$(ChangedServices)" pwsh: true displayName: Spell check public API @@ -26,10 +35,9 @@ steps: ArtifactName: 'package-diffs' SbomEnabled: false - - - template: /eng/common/pipelines/templates/steps/verify-readme.yml + - template: /eng/common/pipelines/templates/steps/verify-readmes.yml parameters: - ScanPath: $(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }} + PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo - template: /eng/common/pipelines/templates/steps/verify-path-length.yml parameters: @@ -46,7 +54,7 @@ steps: - template: /eng/common/pipelines/templates/steps/verify-samples.yml parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} + ServiceDirectories: $(ChangedServicesCsv) - script: | npm ci @@ -62,28 +70,21 @@ steps: node common/scripts/install-run-rush.js install displayName: "Install dependencies" - - template: /eng/pipelines/templates/steps/set-artifact-packages.yml - parameters: - Artifacts: ${{ parameters.Artifacts }} - - script: | - node eng/tools/rush-runner.js build "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)" --verbose -p max + node eng/tools/rush-runner.js build $(ChangedServices) -packages "$(ArtifactPackageNames)" --verbose -p max displayName: "Build libraries" - template: /eng/pipelines/templates/steps/run-eslint.yml parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} + ServiceDirectories: $(ChangedServices) - pwsh: | - node eng/tools/rush-runner.js check-format "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)" --verbose + node eng/tools/rush-runner.js check-format $(ChangedServices) -packages "$(ArtifactPackageNames)" --verbose displayName: "Check Format in Libraries" - - ${{ each artifact in parameters.Artifacts }}: - - template: /eng/common/pipelines/templates/steps/verify-changelog.yml - parameters: - PackageName: ${{artifact.name}} - ServiceName: ${{parameters.ServiceDirectory}} - ForRelease: false + - template: /eng/common/pipelines/templates/steps/verify-changelogs.yml + parameters: + PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 # ComponentGovernance is currently unable to run on pull requests of public projects. Running on non-PR diff --git a/eng/pipelines/templates/steps/build.yml b/eng/pipelines/templates/steps/build.yml index 804b85784a4b..d9f547ef28c6 100644 --- a/eng/pipelines/templates/steps/build.yml +++ b/eng/pipelines/templates/steps/build.yml @@ -13,32 +13,23 @@ steps: - pwsh: | $folder = "${{parameters.ServiceDirectory}}" - if ($folder -eq "*") { $folder = "" } + if ($folder -eq "*" -or $folder -eq "auto") { $folder = "" } echo "##vso[task.setvariable variable=folder]$folder" displayName: "Set folder variable for readme links" + # we are not passing service directory, so we only ever set dev build to true - template: /eng/common/pipelines/templates/steps/daily-dev-build-variable.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - script: | npm install ./eng/tools/versioning - node eng/tools/versioning/set-dev.js --build-id "$(Build.BuildNumber)" --repo-root "$(Build.SourcesDirectory)" --service "$(folder)" + node eng/tools/versioning/set-dev.js --build-id "$(Build.BuildNumber)" --repo-root "$(Build.SourcesDirectory)" node common/scripts/install-run-rush.js update condition: and(succeeded(),eq(variables['SetDevVersion'],'true')) displayName: "Update package versions for dev build" - - task: Powershell@2 - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1 - arguments: > - -ServiceDirectory ${{parameters.ServiceDirectory}} - -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo - -AddDevVersion - pwsh: true - workingDirectory: $(Pipeline.Workspace) - displayName: Update package properties with dev version - condition: and(succeeded(),eq(variables['SetDevVersion'],'true')) + - template: /eng/common/pipelines/templates/steps/save-package-properties.yml + parameters: + ServiceDirectory: ${{parameters.ServiceDirectory}} - script: | node common/scripts/install-run-rush.js install @@ -47,7 +38,9 @@ steps: - template: /eng/pipelines/templates/steps/set-artifact-packages.yml parameters: + PackageInfo: $(Build.ArtifactStagingDirectory)/PackageInfo Artifacts: ${{ parameters.Artifacts }} + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) }}: - task: AzureCLI@2 inputs: @@ -60,10 +53,8 @@ steps: sasToken=$(az storage container generate-sas --account-name azuresdkartifacts --name azure-sdk-for-js-rush-cache --permissions dlrw --auth-mode login --as-user --expiry $expiry --https-only -o tsv) echo "##vso[task.setvariable variable=rushBuildCacheCred;issecret=true;]$sasToken" - # Option "-p max" ensures parallelism is set to the number of cores on all platforms, which improves build times. - # The default on Windows is "cores - 1" (microsoft/rushstack#436). - - script: | - node eng/tools/rush-runner.js build "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)" --verbose -p max + - pwsh: | + node eng/tools/rush-runner.js build $(ChangedServices) -packages "$(ArtifactPackageNames)" --verbose -p max displayName: "Build libraries" env: ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) }}: @@ -71,15 +62,15 @@ steps: RUSH_BUILD_CACHE_WRITE_ALLOWED: 1 - script: | - node eng/tools/rush-runner.js build:samples "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)" --verbose - displayName: "Build samples" + node eng/tools/rush-runner.js build:samples "$(ChangedServices)" -packages "$(ArtifactPackageNames)" --verbose + displayName: "Build samples for PR" - pwsh: | eng/tools/check-api-warning.ps1 displayName: "Check api extractor output changes" - script: | - node eng/tools/rush-runner.js pack "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)" --verbose + node eng/tools/rush-runner.js pack "$(ChangedServices)" -packages "$(ArtifactPackageNames)" --verbose displayName: "Pack libraries" # Unlink node_modules folders to significantly improve performance of subsequent tasks @@ -93,18 +84,22 @@ steps: ServiceDirectory: ${{parameters.ServiceDirectory}} - pwsh: | - $artifacts = '${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json + $artifacts = Get-ChildItem -Recurse -Force "$(Build.ArtifactStagingDirectory)/PackageInfo/*.json" ` + | ForEach-Object { $_.Name.Replace(".json", "") } + foreach ($artifact in $artifacts) { - $artifactName = $artifact.name - Write-Host "Copying $artifactName artifacts to $(Build.ArtifactStagingDirectory)/$artifactName" - New-Item -Type Directory -Name $artifactName -Path $(Build.ArtifactStagingDirectory) > $null - Copy-Item sdk/${{parameters.ServiceDirectory}}/**/$artifactName-[0-9]*.[0-9]*.[0-9]*.tgz $(Build.ArtifactStagingDirectory)/$artifactName - Copy-Item sdk/${{parameters.ServiceDirectory}}/**/browser/$artifactName-[0-9]*.[0-9]*.[0-9]*.zip $(Build.ArtifactStagingDirectory)/$artifactName - if ($${{ parameters.IncludeRelease }} -eq $true -and $artifact.skipPublishDocMs -ne $true) + $artifactDetails = Get-Content -Raw $(Build.ArtifactStagingDirectory)/PackageInfo/$artifact.json | ConvertFrom-Json + + Write-Host "Copying $artifact artifacts to $(Build.ArtifactStagingDirectory)/$artifact" + New-Item -Type Directory -Force -Name $artifact -Path $(Build.ArtifactStagingDirectory) > $null + Copy-Item sdk/$($artifactDetails.Service)/**/$artifact-[0-9]*.[0-9]*.[0-9]*.tgz $(Build.ArtifactStagingDirectory)/$artifact + Copy-Item sdk/$($artifactDetails.Service)/**/browser/$artifact-[0-9]*.[0-9]*.[0-9]*.zip $(Build.ArtifactStagingDirectory)/$artifact + + if ($${{ parameters.IncludeRelease }} -eq $true -and $artifactDetails.ArtifactDetails.skipPublishDocMs -ne $true) { - New-Item -Type Directory -Name documentation -Path $(Build.ArtifactStagingDirectory)/$artifactName > $null - Copy-Item $(Build.SourcesDirectory)/docGen/$artifactName.zip $(Build.ArtifactStagingDirectory)/$artifactName/documentation + New-Item -Type Directory -Force -Name documentation -Path $(Build.ArtifactStagingDirectory)/$artifact > $null + Copy-Item $(Build.SourcesDirectory)/docGen/$artifact.zip $(Build.ArtifactStagingDirectory)/$artifact/documentation } } displayName: 'Copy Packages' diff --git a/eng/pipelines/templates/steps/generate-doc.yml b/eng/pipelines/templates/steps/generate-doc.yml index 828ffb0519ef..72af43d7dbd3 100644 --- a/eng/pipelines/templates/steps/generate-doc.yml +++ b/eng/pipelines/templates/steps/generate-doc.yml @@ -15,7 +15,17 @@ steps: - pwsh: | node $(Build.SourcesDirectory)/eng/tools/generate-doc/dist/index.js --serviceDir "${{parameters.ServiceDirectory}}" displayName: "Run Typedoc Docs" + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) + + - pwsh: | + if ('$(ChangedServices)' -and '$(ChangedServices)' -notlike '*ChangedServices*') { + foreach($service in '$(ChangedServices)'.Split(" ")) { + node $(Build.SourcesDirectory)/eng/tools/generate-doc/dist/index.js --serviceDir "$service" + } + } + displayName: "Run Typedocs for PR" + condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) - pwsh: | $(Build.SourcesDirectory)/eng/tools/compress-subfolders.ps1 "$(Build.SourcesDirectory)/docGen" "$(Build.SourcesDirectory)/docGen" - displayName: "Generate Typedoc Docs" + displayName: "Compress generated docs" diff --git a/eng/pipelines/templates/steps/run-eslint.yml b/eng/pipelines/templates/steps/run-eslint.yml index d12f49e20dfe..06e714f0f9dd 100644 --- a/eng/pipelines/templates/steps/run-eslint.yml +++ b/eng/pipelines/templates/steps/run-eslint.yml @@ -1,5 +1,5 @@ parameters: - ServiceDirectory: '' + ServiceDirectories: '' steps: - script: | @@ -8,5 +8,5 @@ steps: - pwsh: | node common/scripts/install-run-rush.js build -t @azure/eslint-plugin-azure-sdk -t @azure/monitor-opentelemetry-exporter - node eng/tools/rush-runner.js lint "${{parameters.ServiceDirectory}}" -p max + node eng/tools/rush-runner.js lint "${{parameters.ServiceDirectories}}" -p max displayName: "Build ESLint Plugin and Lint Libraries" diff --git a/eng/pipelines/templates/steps/set-artifact-packages.yml b/eng/pipelines/templates/steps/set-artifact-packages.yml index 50090c8258d9..8164c81bc668 100644 --- a/eng/pipelines/templates/steps/set-artifact-packages.yml +++ b/eng/pipelines/templates/steps/set-artifact-packages.yml @@ -1,12 +1,51 @@ parameters: + PackageInfo: '' Artifacts: [] steps: + # Package-Properties folder contains the package properties for all discovered packages that were either A) affected by the PR or + # B) explicitly present in the service directory. This repo splits the builds into two categories: "mgmt" and "dataplane". While + # a given directory may contain both, there will be a separate build definition to release the management packages. Due to this + # we need to artificially filter the packages to ensure that only the packages targeted for THAT ci.yml are built. + # When we merge the PR adding js - pullrequest, this code will also merge, meaning the public - - ci builds will still operate + # EXACTLY the same as they did before the pipelinev3 change. This is important to ensure that we don't accidentally build the wrong packages + # while in the integration period. After we disable all the public `js - - ci` builds, only the `js - pullrequest` build WONT provide + # an artifact list, which will allow the expand/contract. Meanwhile the internal builds will continue to provide the artifact list from their + # individual ci.yml and ci.mgmt.yml files. - pwsh: | $artifacts = '${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json - $packages = "" - foreach ($artifact in $artifacts) - { - $packages += "$($artifact.name)," + $knownArtifacts = @() + if ($artifacts) { + $knownArtifacts = $artifacts | ForEach-Object { $_.name } } - echo "##vso[task.setvariable variable=ArtifactPackageNames]$packages" - displayName: "Find Packages to build" + + $packageProperties = Get-ChildItem -Recurse "${{ parameters.PackageInfo }}" *.json ` + | Where-Object { if($knownArtifacts) { $knownArtifacts -contains $_.Name.Replace(".json", "") } else { $true } } + + $pkgNames = $packageProperties | ForEach-Object { $_.Name.Replace(".json", "") } + + $setting = $pkgNames -join "," + Write-Host "Setting ArtifactPackageNames to: `n$setting" + Write-Host "##vso[task.setvariable variable=ArtifactPackageNames;]$setting" + displayName: Resolve Targeted Packages + condition: eq(variables['ArtifactPackageNames'], '') + + - pwsh: | + # set changed services given the set of changed packages, this will mean that + # ChangedServices will be appropriate for the batched set of packages if that is indeed how + # we set the targeted artifacts + $packageProperties = Get-ChildItem -Recurse "${{ parameters.PackageInfo }}" *.json ` + | Foreach-Object { Get-Content -Raw -Path $_.FullName | ConvertFrom-Json } + + $packageSet = "$(ArtifactPackageNames)" -split "," + + $changedServicesArray = $packageProperties | Where-Object { $packageSet -contains $_.ArtifactName } + | ForEach-Object { $_.ServiceDirectory } | Get-Unique + + $changedServices = $changedServicesArray -join " " + $commaChangedServices = $changedServicesArray -join "," + Write-Host "##vso[task.setvariable variable=ChangedServices;]$changedServices" + Write-Host "##vso[task.setvariable variable=ChangedServicesCsv;]$commaChangedServices" + + Write-Host "This run is targeting: `n$(ArtifactPackageNames) in [$changedServices]" + displayName: Resolve Targeted Packages + condition: ne(variables['ArtifactPackageNames'], '') diff --git a/eng/pipelines/templates/steps/test.yml b/eng/pipelines/templates/steps/test.yml index a5e1c57d6ac6..ef90c99d2729 100644 --- a/eng/pipelines/templates/steps/test.yml +++ b/eng/pipelines/templates/steps/test.yml @@ -9,42 +9,47 @@ steps: parameters: AgentImage: ${{ parameters.OSName}} + - template: /eng/common/pipelines/templates/steps/save-package-properties.yml + parameters: + ServiceDirectory: ${{parameters.ServiceDirectory}} + - script: | node common/scripts/install-run-rush.js install displayName: "Install dependencies" - template: /eng/pipelines/templates/steps/set-artifact-packages.yml parameters: + PackageInfo: $(Build.ArtifactStagingDirectory)/PackageInfo Artifacts: ${{ parameters.Artifacts }} # Option "-p max" ensures parallelism is set to the number of cores on all platforms, which improves build times. # The default on Windows is "cores - 1" (microsoft/rushstack#436). - script: | - node eng/tools/rush-runner.js build "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)" --verbose -p max + node eng/tools/rush-runner.js build $(ChangedServices) -packages "$(ArtifactPackageNames)" --verbose -p max displayName: "Build libraries" # Option "-p max" ensures parallelism is set to the number of cores on all platforms, which improves build times. # The default on Windows is "cores - 1" (microsoft/rushstack#436). - script: | - node eng/tools/rush-runner.js build:test "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)" --verbose -p max + node eng/tools/rush-runner.js build:test $(ChangedServices) -packages "$(ArtifactPackageNames)" --verbose -p max displayName: "Build test assets" - template: ../steps/use-node-test-version.yml - ${{ if eq(parameters.TestProxy, true) }}: - template: /eng/common/testproxy/test-proxy-standalone-tool.yml - + # Option "-p max" ensures parallelism is set to the number of cores on all platforms, which improves build times. # The default on Windows is "cores - 1" (microsoft/rushstack#436). - script: | - node eng/tools/rush-runner.js unit-test:node "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)" --verbose -p max + node eng/tools/rush-runner.js unit-test:node $(ChangedServices) -packages "$(ArtifactPackageNames)" --verbose -p max displayName: "Test libraries" condition: and(succeeded(),eq(variables['TestType'], 'node')) # Option "-p max" ensures parallelism is set to the number of cores on all platforms, which improves build times. # The default on Windows is "cores - 1" (microsoft/rushstack#436). - script: | - node eng/tools/rush-runner.js unit-test:browser "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)" --verbose -p max + node eng/tools/rush-runner.js unit-test:browser $(ChangedServices) -packages "$(ArtifactPackageNames)" --verbose -p max displayName: "Test libraries" condition: and(succeeded(),eq(variables['TestType'], 'browser')) @@ -53,7 +58,7 @@ steps: copy $(Build.SourcesDirectory)/test-proxy.log $(Build.ArtifactStagingDirectory) displayName: 'Dump Test Proxy logs' condition: succeededOrFailed() - + # Unlink node_modules folders to significantly improve performance of subsequent tasks # which need to walk the directory tree (and are hardcoded to follow symlinks). # Retry for 30 seconds, since this command may fail with error "Another rush command is already diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 6358d7795f13..e639c6fd4ede 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -6,6 +6,11 @@ $packagePattern = "*.tgz" $MetadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/main/_data/releases/latest/js-packages.csv" $GithubUri = "https://github.com/Azure/azure-sdk-for-js" $PackageRepositoryUri = "https://www.npmjs.com/package" +$ReducedDependencyLookup = @{ + 'core' = @('@azure-rest/synapse-access-control', '@azure/arm-resources', '@azure/identity', '@azure/service-bus', '@azure/template') + 'test-utils' = @('@azure-tests/perf-storage-blob', '@azure/arm-eventgrid', '@azure/ai-text-analytics', '@azure/identity', '@azure/template') + 'identity' = @('@azure-tests/perf-storage-blob', '@azure/ai-text-analytics', '@azure/arm-resources', '@azure/identity-cache-persistence', '@azure/identity-vscode', '@azure/storage-blob', '@azure/template') +} . "$PSScriptRoot/docs/Docs-ToC.ps1" . "$PSScriptRoot/docs/Docs-Onboarding.ps1" @@ -25,6 +30,79 @@ function Get-javascript-EmitterAdditionalOptions([string]$projectDirectory) { return "--option @azure-tools/typespec-ts.emitter-output-dir=$projectDirectory/" } +function Get-javascript-AdditionalValidationPackagesFromPackageSet { + param( + [Parameter(Mandatory=$true)] + $LocatedPackages, + [Parameter(Mandatory=$true)] + $diffObj, + [Parameter(Mandatory=$true)] + $AllPkgProps + ) + $additionalValidationPackages = @() + + function isOther($fileName) { + $startsWithPrefixes = @(".config", ".devcontainer", ".github", ".scripts", ".vscode", "common", "design", "documentation", "eng", "samples") + + foreach ($prefix in $startsWithPrefixes) { + if ($fileName.StartsWith($prefix)) { + return $true + } + } + + return $false + } + + $changedServices = @() + foreach($file in $diffObj.ChangedFiles) { + $pathComponents = $file -split "/" + # handle changes only in sdk/// + if ($pathComponents.Length -eq 3 -and $pathComponents[0] -eq "sdk") { + $changedServices += $pathComponents[1] + } + + # handle any changes under sdk/. + if ($pathComponents.Length -eq 2 -and $pathComponents[0] -eq "sdk") { + $changedServices += "template" + } + } + + $othersChanged = $diffObj.ChangedFiles | Where-Object { isOther $_ } + $changedServices = $changedServices | Get-Unique + + if ($othersChanged) { + $additionalPackages = $ReducedDependencyLookup["core"] | ForEach-Object { $me=$_; $AllPkgProps | Where-Object { $_.Name -eq $me } | Select-Object -First 1 } + $additionalValidationPackages += $additionalPackages + } + + foreach ($changedService in $changedServices) { + if ($ReducedDependencyLookup.ContainsKey($changedService)) { + $additionalPackages = $ReducedDependencyLookup[$changedService] | ForEach-Object { $me=$_; $AllPkgProps | Where-Object { $_.Name -eq $me } | Select-Object -First 1 } + $additionalValidationPackages += $additionalPackages + } + else { + $additionalPackages = $AllPkgProps | Where-Object { $_.ServiceDirectory -eq $changedService } + $additionalValidationPackages += $additionalPackages + } + } + + $uniqueResultSet = @() + foreach ($pkg in $additionalValidationPackages) { + if ($uniqueResultSet -notcontains $pkg -and $LocatedPackages -notcontains $pkg) { + $pkg.IncludedForValidation = $true + $uniqueResultSet += $pkg + } + } + + Write-Host "Returning additional packages for validation: $($uniqueResultSet.Count)" + foreach ($pkg in $uniqueResultSet) { + Write-Host " - $($pkg.Name)" + } + + return $uniqueResultSet +} + + function Get-javascript-PackageInfoFromRepo ($pkgPath, $serviceDirectory) { $projectPath = Join-Path $pkgPath "package.json" if (Test-Path $projectPath) { @@ -40,10 +118,17 @@ function Get-javascript-PackageInfoFromRepo ($pkgPath, $serviceDirectory) { } $pkgProp.IsNewSdk = ($pkgProp.SdkType -eq "client") -or ($pkgProp.SdkType -eq "mgmt") $pkgProp.ArtifactName = $jsStylePkgName + + + if ($ReducedDependencyLookup.ContainsKey($pkgProp.ServiceDirectory)) { + $pkgProp.AdditionalValidationPackages = $ReducedDependencyLookup[$pkgProp.ServiceDirectory] + } + # the constructor for the package properties object attempts to initialize CI artifacts on instantiation # of the class. however, due to the fact that we set the ArtifactName _after_ the constructor is called, # we need to call it again here to ensure the CI artifacts are properly initialized $pkgProp.InitializeCIArtifacts() + return $pkgProp } return $null diff --git a/eng/scripts/spell-check-public-apis.ps1 b/eng/scripts/spell-check-public-apis.ps1 new file mode 100644 index 000000000000..3e818ef0e895 --- /dev/null +++ b/eng/scripts/spell-check-public-apis.ps1 @@ -0,0 +1,46 @@ +<# +.SYNOPSIS +Spell checks JS public API surface as exported to review/**/*.md. This script invokes +against multiple service directories. + +.DESCRIPTION +Checks spelling of package's public API. Some packages may be excluded by +criteria in the cspell.json config. The precise list of files to scan is +determined by cspell. If a package is opted out in the cspell.json a command +will still be issued to scan that folder but cspell will report 0 files checked. + +.PARAMETER ChangedServices +The list of service directories that have been changed in the PR. Space separated list of service directories. + +.EXAMPLE +$(ChangedServices) is set in set-artifact-packages.yml +./spell-check-public-apis.ps1 -ChangedServices $(ChangedServices) + +Spell check all public API specs for all services under `sdk` that have been changed in the PR. + +#> +[CmdletBinding()] +param ( + [Parameter(mandatory = $true)] + [string]$ChangedServices +) + +Set-StrictMode -Version 3.0 +$allSuccess = $true + +if ($ChangedServices) { + $changed = $ChangedServices.Split(" ") + + foreach ($service in $changed) { + &"$PSScriptRoot/spell-check-public-api.ps1" -ServiceDirectory $service + + if($LASTEXITCODE -ne 0) { + $allSuccess = $false + } + } +} + +if (-not $allSuccess) { + Write-Error "One or more service directories failed spell check, check above output for details." + exit 1 +} diff --git a/eng/tools/rush-runner.js b/eng/tools/rush-runner.js index 68560ca1a514..480546bbf2aa 100644 --- a/eng/tools/rush-runner.js +++ b/eng/tools/rush-runner.js @@ -66,7 +66,7 @@ const parseArgs = () => { else { if (arg && arg !== "*") { // exclude empty value and special value "*" meaning all libraries - services.push(arg); + arg.split(" ").forEach(serviceDirectory => services.push(serviceDirectory)); } } } @@ -92,6 +92,59 @@ const getPackageJsons = (searchDir) => { return sdkDirectories.concat(perfTestDirectories).filter((f) => fs.existsSync(f)); // only keep paths for files that actually exist }; +const restrictedToPackages = [ + "@azure/abort-controller", + "@azure/core-amqp", + "@azure/core-auth", + "@azure/core-client", + "@azure/core-http-compat", + "@azure/core-lro", + "@azure/core-paging", + "@azure/core-rest-pipeline", + "@azure/core-sse", + "@azure/core-tracing", + "@azure/core-util", + "@azure/core-xml", + "@azure/logger", + "@azure-rest/core-client", + "@typespec/ts-http-runtime", + "@azure/identity", + "@azure/arm-resources" +]; + +/** + * Helper function that determines the rush command flag to use based on each individual package name for the 'build' check. + * + * If the targeted package is one of the restricted packages with a ton of dependents, we only want to run that package + * and not all of its dependents. + * @param packageNames string[] An array of strings containing the packages names to run the action on. + */ +const getDirectionMappedPackages = (packageNames) => { + const mappedPackages = []; + + for (const packageName of packageNames) { + // Build command without any additional option should build the project and downstream + // If service is configured to run only a set of downstream projects then build all projects leading to them to support testing + // If the package is a core package, azure-identity or arm-resources then build only the package, + // otherwise build the package and all its dependents + var rushCommandFlag = "--impacted-by"; + + if (restrictedToPackages.includes(packageName)) { + // if this is one of our restricted packages with a ton of deps, make it targeted + // as including all dependents will be too much + rushCommandFlag = "--to"; + } + else if (actionComponents.length == 1) { + // else we are building the project and its dependents + rushCommandFlag = "--from"; + } + + mappedPackages.push([rushCommandFlag, packageName]); + } + + return mappedPackages; +}; + const getServicePackages = (baseDir, serviceDirs, artifactNames) => { const packageNames = []; const packageDirs = []; @@ -133,6 +186,7 @@ const flatMap = (arr, f) => { }; const [baseDir, action, serviceDirs, rushParams, artifactNames] = parseArgs(); +const actionComponents = action.toLowerCase().split(":"); const [packageNames, packageDirs] = getServicePackages(baseDir, serviceDirs, artifactNames); @@ -147,11 +201,21 @@ function rushRunAll(direction, packages) { spawnNode(baseDir, "common/scripts/install-run-rush.js", action, ...params, ...rushParams); } +/** + * Helper function to invoke the rush logic split up by direction. + * + * @param packagesWithDirection string[] Any array of strings containing ["direction packageName"...] + */ +function rushRunAllWithDirection(packagesWithDirection) { + const invocation = packagesWithDirection.flatMap(([direction, packageName]) => [direction, packageName]); + spawnNode(baseDir, "common/scripts/install-run-rush.js", action, ...invocation, ...rushParams); +} + /** * Helper function to get the relative path of a package directory from an absolute * one - * - * @param {string} absolutePath absolute path to a package + * + * @param {string} absolutePath absolute path to a package * @returns either the relative path of the package starting from the "sdk" directory * or the just the absolute path itself if "sdk" if not found */ @@ -166,33 +230,22 @@ if (isReducedTestScopeEnabled) { console.log(`Found reduced test matrix configured for ${serviceDirs}.`); packageNames.push(...reducedDependencyTestMatrix[serviceDirs]); } +const packagesWithDirection = getDirectionMappedPackages(packageNames); const rushx_runner_path = path.join(baseDir, "common/scripts/install-run-rushx.js"); if (serviceDirs.length === 0) { spawnNode(baseDir, "common/scripts/install-run-rush.js", action, ...rushParams); } else { - const actionComponents = action.toLowerCase().split(":"); switch (actionComponents[0]) { case "build": - // Build command without any additional option should build the project and downstream - // If service is configured to run only a set of downstream projects then build all projects leading to them to support testing - // if this is build:test for any non-configured package service then all impacted projects downstream and it's dependents should be built - var rushCommandFlag = "--impacted-by"; - if (isReducedTestScopeEnabled) { - // reduced preconfigured set of projects and it's required projects - rushCommandFlag = "--to"; - } - else if (actionComponents.length == 1) { - rushCommandFlag = "--from"; - } - - rushRunAll(rushCommandFlag, packageNames); + rushRunAllWithDirection(packagesWithDirection); break; case "test": case "unit-test": case "integration-test": var rushCommandFlag = "--impacted-by"; - if (isReducedTestScopeEnabled) { + + if (isReducedTestScopeEnabled || serviceDirs.length > 1) { // If a service is configured to have reduced test matrix then run rush test only for those projects rushCommandFlag = "--only"; } diff --git a/sdk/pullrequest.yml b/sdk/pullrequest.yml new file mode 100644 index 000000000000..076b5d29654b --- /dev/null +++ b/sdk/pullrequest.yml @@ -0,0 +1,24 @@ +pr: + branches: + include: + - main + - feature/* + - hotfix/* + - release/* + - pipelinev3* + paths: + include: + - "*" + + exclude: + - sdk/cosmos + +parameters: + - name: Service + type: string + default: auto + +extends: + template: /eng/pipelines/templates/stages/archetype-sdk-client.yml + parameters: + ServiceDirectory: ${{ parameters.Service }}