ci: distinguish pipeline pnpm store caches (#27577)

Use lockfile paths as literal strings to avoid unintended
cross-pollination when there is a cache miss. Recently a less than 2GB
cache was bumped to 37GB when there was a miss for Kusto upload job and
the E2E test cache was used as fallback.

Avoid secondary pnpm install for client jobs to avoid downgrading (or
upgrading) pnpm versions.
- In perf-benchmarks, swap the install order; so, that FF client pnpm
install is first.

Note: This change will cause all pipelines to reset their caches. (Which
can be good time to time anyway.)

Author: jason-ha

tools/pipelines/templates/build-npm-client-package.yml

@@ -331,9 +331,17 @@ extends:
 
               - template: /tools/pipelines/templates/include-install.yml@self
                 parameters:
-                  packageManager: '${{ parameters.packageManager }}'
                   buildDirectory: '${{ parameters.buildDirectory }}'
+                  installPnpm: ${{ eq(parameters.packageManager, 'pnpm') }}
                   packageManagerInstallCommand: '${{ parameters.packageManagerInstallCommand }}'
+                  # The compat-workspaces lockfile is committed to the FluidFramework repo and changes
+                  # whenever legacy package versions are added or removed. Including it in the pnpm store
+                  # cache key ensures a cache miss whenever the set of compat packages changes.
+                  ${{ if and(eq(parameters.isBundleSizeArtifactsPipeline, false), eq(parameters.buildToolsVersionToInstall, 'repo')) }}:
+                    # Currently repo build-tools also installs sharing the store - ideally that is removed by #73628 and the `| ...` can be removed.
+                    additionalLockFilePaths: '${{ parameters.buildDirectory }}/packages/test/test-version-utils/compat-workspaces/full/pnpm-lock.yaml | ${{ parameters.buildDirectory }}/build-tools/pnpm-lock.yaml'
+                  ${{ else }}:
+                    additionalLockFilePaths: '${{ parameters.buildDirectory }}/packages/test/test-version-utils/compat-workspaces/full/pnpm-lock.yaml'
 
               # The bundle-size-artifacts pipeline runs a client build but doesn't publish packages,
               # so we skip version setting for it.
@@ -343,6 +351,11 @@ extends:
                       buildDirectory: '${{ parameters.buildDirectory }}'
                       buildNumberInPatch: ${{ parameters.buildNumberInPatch }}
                       buildToolsVersionToInstall: '${{ parameters.buildToolsVersionToInstall }}'
+                      # Note that skipping pnpm install means client pnpm version must suffice for build-tools.
+                      # Ideally, #73628 removes include-install-build-tools template use in include-set-package-version
+                      # making it a non-issue. For now, second pnpm install is skipped to preserve the client version
+                      # for all remaining steps.
+                      installPnpmForRepoBuildTools: ${{ ne(parameters.packageManager, 'pnpm') }}
                       tagName: '${{ parameters.tagName }}'
                       interdependencyRange: '${{ parameters.interdependencyRange }}'
                       packageTypesOverride: '${{ parameters.packageTypesOverride }}'
@@ -600,15 +613,25 @@ extends:
 
               - template: /tools/pipelines/templates/include-install.yml@self
                 parameters:
-                  packageManager: '${{ parameters.packageManager }}'
                   buildDirectory: '${{ parameters.buildDirectory }}'
+                  installPnpm: ${{ eq(parameters.packageManager, 'pnpm') }}
                   packageManagerInstallCommand: '${{ parameters.packageManagerInstallCommand }}'
+                  # The compat-workspaces lockfile is committed to the FluidFramework repo and changes
+                  # whenever legacy package versions are added or removed. Including it in the pnpm store
+                  # cache key ensures a cache miss whenever the set of compat packages changes.
+                  # Currently repo build-tools also installs sharing the store - ideally that is removed by #73628 and the `| ...` can be removed.
+                  additionalLockFilePaths: '${{ parameters.buildDirectory }}/packages/test/test-version-utils/compat-workspaces/full/pnpm-lock.yaml | ${{ parameters.buildDirectory }}/build-tools/pnpm-lock.yaml'
 
               # We need it in order to run flub where the code coverage comparison logic calls for it
               - template: /tools/pipelines/templates/include-install-build-tools.yml@self
                 parameters:
                   buildDirectory: ${{ parameters.buildDirectory }}
                   buildToolsVersionToInstall: repo
+                  # Note that skipping pnpm install means client pnpm version must suffice for build-tools.
+                  # Ideally, #73628 removes this whole template use making it a non-issue. For now, second
+                  # pnpm install is skipped to preserve the client version for any remaining steps.
+                  installPnpmForRepoBuildTools: ${{ ne(parameters.packageManager, 'pnpm') }}
+                  # This shares the store with the client installs (above).
                   pnpmStorePath: $(Pipeline.Workspace)/.pnpm-store
 
               - task: DownloadPipelineArtifact@2
@@ -764,9 +787,13 @@ extends:
 
                 - template: /tools/pipelines/templates/include-install.yml@self
                   parameters:
-                    packageManager: '${{ parameters.packageManager }}'
                     buildDirectory: '${{ parameters.buildDirectory }}'
+                    installPnpm: ${{ eq(parameters.packageManager, 'pnpm') }}
                     packageManagerInstallCommand: '${{ parameters.packageManagerInstallCommand }}'
+                    # The compat-workspaces lockfile is committed to the FluidFramework repo and changes
+                    # whenever legacy package versions are added or removed. Including it in the pnpm store
+                    # cache key ensures a cache miss whenever the set of compat packages changes.
+                    additionalLockFilePaths: '${{ parameters.buildDirectory }}/packages/test/test-version-utils/compat-workspaces/full/pnpm-lock.yaml'
 
                 - task: DownloadPipelineArtifact@2
                   inputs:

tools/pipelines/templates/build-npm-package.yml

@@ -314,8 +314,8 @@ extends:
 
               - template: /tools/pipelines/templates/include-install.yml@self
                 parameters:
-                  packageManager: '${{ parameters.packageManager }}'
                   buildDirectory: '${{ parameters.buildDirectory }}'
+                  installPnpm: ${{ eq(parameters.packageManager, 'pnpm') }}
                   packageManagerInstallCommand: '${{ parameters.packageManagerInstallCommand }}'
 
               # The bundle-size-artifacts pipeline runs a client build but doesn't publish packages,

tools/pipelines/templates/include-install-build-tools.yml

@@ -23,6 +23,10 @@ parameters:
   type: string
   default: repo
 
+- name: installPnpmForRepoBuildTools
+  type: boolean
+  default: true
+
 # The path to the pnpm store.
 - name: pnpmStorePath
   type: string
@@ -32,11 +36,12 @@ steps:
   # These steps should ONLY run if we're using the repo version of the build tools. These steps are mutually exclusive
   # with the next group of steps.
   - ${{ if eq(parameters.buildToolsVersionToInstall, 'repo') }}:
-    - template: /tools/pipelines/templates/include-install-pnpm.yml@self
-      parameters:
-        buildDirectory: $(FluidFrameworkDirectory)/build-tools
-        pnpmStorePath: ${{ parameters.pnpmStorePath }}
-        enableCache: false
+    - ${{ if parameters.installPnpmForRepoBuildTools }}:
+      - template: /tools/pipelines/templates/include-install-pnpm.yml@self
+        parameters:
+          buildDirectory: $(FluidFrameworkDirectory)/build-tools
+          pnpmStorePath: ${{ parameters.pnpmStorePath }}
+          enableCache: false
 
     - task: Bash@3
       name: InstallBuildTools

tools/pipelines/templates/include-install-pnpm.yml

@@ -7,6 +7,8 @@
 
 parameters:
 # The path containing the project(s) to build.
+# A package.json is expected under this folder and will be used to determine
+# the version of pnpm to install.
 - name: buildDirectory
   type: string
 
@@ -16,6 +18,8 @@ parameters:
   default: true
 
 # The path to the pnpm store. The contents here will be cached and restored when using pnpm in a pipeline.
+# If a job uses this template multiple times with enableCache=true, then each pnpmStorePath should be
+# unique AND all use of pnpm to install should be done before another use of this template.
 - name: pnpmStorePath
   type: string
   default: $(Pipeline.Workspace)/.pnpm-store
@@ -34,33 +38,50 @@ parameters:
 # An optional extra segment appended to the pnpm store cache key (separated by |).
 # Note that Azure pipelines interpret unquoted cache keys as files rather than literal strings. E.g. the following specification:
 #
-# additionalCacheKey: 'foo | "bar"'
+# additionalOptionalCacheKey: 'foo | "bar"'
 #
 # will use the hashed contents of the file "foo" (hashing nothing if that file does not exist) as well as the string "bar".
 # Use this parameter if the set of dependencies to be installed may be be better distinguished using files other than the root pnpm-lock.
-- name: additionalCacheKey
+# And specify quoted files paths of any files to requiredCacheKey.
+- name: additionalOptionalCacheKey
   type: string
   default: ''
 
+# An optional extra segment appended to the pnpm store cache key (separated by |) and restore key.
+# Note that Azure pipelines interpret unquoted cache keys as files rather than literal strings. E.g. the following specification:
+#
+# requiredCacheKey: 'foo | "bar"'
+#
+# will use the hashed contents of the file "foo" (hashing nothing if that file does not exist) as well as the string "bar".
+# Unquoted files are *not recommended* as requiredCacheKey as it limits restore possibilities. However the quoted file path
+# of any files specified in additionalOptionalCacheKey is recommended to avoid cross-pollination from caches of other jobs.
+# There is a "default" default value to avoid unintended restore contamination. That can be overridden by setting empty string.
+- name: requiredCacheKey
+  type: string
+  default: '"default"'
+
 steps:
 - ${{ if eq(parameters.enableCache, true) }}:
   - task: Cache@2
     displayName: Cache pnpm store
     # The timeout applies to both the pre-job restore and post-job save operations.
     # 3 minutes was insufficient for the post-job cache upload of the pnpm store, causing
     # intermittent "The task has timed out" errors that mark builds as Partially Succeeded.
+    # In 2026-06, 5 minutes has been insufficient for pre-job as the pnpm store has grown
+    # with additional past versions of FF client packages for compatibility testing. But
+    # it may also just be polluted. This will either remain at 5 or be adjusted after
+    # caches are reset.
     timeoutInMinutes: 5
     continueOnError: true
     inputs:
-      # Caches are already scoped to individual pipelines, so no need to include the release group name or tag
-      # in the cache key
-      ${{ if ne(parameters.additionalCacheKey, '') }}:
-        key: '"pnpm-store" | "$(Agent.OS)" | $(Pipeline.Workspace)/${{ parameters.buildDirectory }}/pnpm-lock.yaml | ${{ parameters.additionalCacheKey }}'
-      ${{ else }}:
-        key: '"pnpm-store" | "$(Agent.OS)" | $(Pipeline.Workspace)/${{ parameters.buildDirectory }}/pnpm-lock.yaml'
       path: ${{ parameters.pnpmStorePath }}
+      # Caches are already scoped to individual pipelines, so no need to include the release group name or tag
+      # in the cache key.
+      # The pnpm lockfile path is kept as an explicit key to avoid job cross-pollination
+      # when using multiple workspaces in a pipeline.
+      key: '"pnpm-store" | "$(Agent.OS)" | "${{ parameters.buildDirectory }}/pnpm-lock.yaml" | ${{ parameters.requiredCacheKey }} | $(Pipeline.Workspace)/${{ parameters.buildDirectory }}/pnpm-lock.yaml | ${{ parameters.additionalOptionalCacheKey }}'
       restoreKeys: |
-        "pnpm-store" | "$(Agent.OS)"
+        "pnpm-store" | "$(Agent.OS)" | "${{ parameters.buildDirectory }}/pnpm-lock.yaml" | ${{ parameters.requiredCacheKey }}
 
 # Seed the userconfig .npmrc with the primary registry. This propagates to subsequent tasks so
 # that npmAuthenticate (below) and any later pnpm/npm invocation in the job reuse the same file.

tools/pipelines/templates/include-install.yml

@@ -4,12 +4,16 @@
 # include-install template for the install step in client build and test stability pipeline
 
 parameters:
-- name: packageManager
+- name: buildDirectory
   type: string
-  default: pnpm
 
-- name: buildDirectory
+- name: installDependenciesDescription
   type: string
+  default: Install dependencies
+
+- name: installPnpm
+  type: boolean
+  default: true
 
 - name: packageManagerInstallCommand
   type: string
@@ -23,16 +27,24 @@ parameters:
   type: string
   default: $(Agent.TempDirectory)/.npmrc
 
+# string with ` | ` separated list of lockfiles (note that the spacing is meaningful)
+- name: additionalLockFilePaths
+  type: string
+  default: ''
+
 steps:
-  - ${{ if eq(parameters.packageManager, 'pnpm') }}:
+  - ${{ if parameters.installPnpm }}:
     - template: /tools/pipelines/templates/include-install-pnpm.yml@self
       parameters:
         buildDirectory: ${{ parameters.buildDirectory }}
         primaryRegistry: ${{ parameters.primaryRegistry }}
         userNpmrcPath: ${{ parameters.userNpmrcPath }}
+        ${{ if ne(parameters.additionalLockFilePaths, '') }}:
+          additionalOptionalCacheKey: '$(Pipeline.Workspace)/${{ replace(parameters.additionalLockFilePaths, '' | '', '' | $(Pipeline.Workspace)/'') }}'
+          requiredCacheKey: '"${{ replace(parameters.additionalLockFilePaths, '' | '', ''" | "'') }}"'
 
   - task: Bash@3
-    displayName: Install dependencies
+    displayName: ${{ parameters.installDependenciesDescription }}
     retryCountOnTaskFailure: 4
     inputs:
       targetType: 'inline'

tools/pipelines/templates/include-set-package-version.yml

@@ -15,6 +15,9 @@ parameters:
 - name: buildToolsVersionToInstall
   type: string
   default: repo
+- name: installPnpmForRepoBuildTools
+  type: boolean
+  default: true
 
 # The path to the pnpm store.
 - name: pnpmStorePath
@@ -53,6 +56,7 @@ steps:
   parameters:
     buildDirectory: ${{ parameters.buildDirectory }}
     buildToolsVersionToInstall: ${{ parameters.buildToolsVersionToInstall }}
+    installPnpmForRepoBuildTools: ${{ parameters.installPnpmForRepoBuildTools }}
     pnpmStorePath: ${{ parameters.pnpmStorePath }}
 
 - task: Bash@3

tools/pipelines/templates/include-test-real-service.yml

@@ -246,13 +246,18 @@ stages:
 
         - template: /tools/pipelines/templates/include-install-pnpm.yml@self
           parameters:
+            # Install pnpm based on the FF repo settings as it has an engine requirement whereas
+            # ff_pipeline_host does not yet. So long a FF repo's pnpm version is compatible with
+            # ff_pipeline_host this single install and ordering of steps will suffice. If the
+            # needs diverge, then additional coordination may be required.
             buildDirectory: $(FluidFrameworkDirectory)
             # The compat-workspaces lockfile is committed to the FluidFramework repo and changes
             # whenever legacy package versions are added or removed. Including it in the pnpm store
             # cache key ensures a cache miss whenever the set of compat packages changes.
             # The ff_pipeline_host lockfile is included so the cache key also invalidates when the
             # internal tooling dependencies change.
-            additionalCacheKey: '$(Pipeline.Workspace)/$(FluidFrameworkDirectory)/packages/test/test-version-utils/compat-workspaces/full/pnpm-lock.yaml | $(Pipeline.Workspace)/$(FFPipelineHostDirectory)/pnpm-lock.yaml'
+            additionalOptionalCacheKey: '$(Pipeline.Workspace)/$(FluidFrameworkDirectory)/packages/test/test-version-utils/compat-workspaces/full/pnpm-lock.yaml | $(Pipeline.Workspace)/$(FFPipelineHostDirectory)/pnpm-lock.yaml'
+            requiredCacheKey: '"$(Pipeline.Workspace)/$(FluidFrameworkDirectory)/packages/test/test-version-utils/compat-workspaces/full/pnpm-lock.yaml" | "$(Pipeline.Workspace)/$(FFPipelineHostDirectory)/pnpm-lock.yaml"'
 
         - template: /tools/pipelines/templates/include-setup-npmrc-for-download.yml@self
 

tools/pipelines/test-perf-benchmarks.yml

@@ -142,16 +142,30 @@ stages:
 
         - template: /tools/pipelines/templates/include-use-node-version.yml@self
 
-        # Install ff_pipeline_host dependencies (provides telemetry-generator and trips-setup/cleanup)
+        # Install FluidFramework dependencies
         - template: /tools/pipelines/templates/include-install.yml@self
           parameters:
-            buildDirectory: $(FFPipelineHostDirectory)
+            buildDirectory: $(FluidFrameworkDirectory)
+            installDependenciesDescription: Install FluidFramework dependencies
+            installPnpm: true
             primaryRegistry: $(ado-feeds-ff-download-only)
+            # The compat-workspaces lockfile is committed to the FluidFramework repo and changes
+            # whenever legacy package versions are added or removed. Including it in the pnpm store
+            # cache key ensures a cache miss whenever the set of compat packages changes.
+            # The ff_pipeline_host lockfile is included so the cache key also invalidates when the
+            # internal tooling dependencies change.
+            additionalLockFilePaths: '$(FluidFrameworkDirectory)/packages/test/test-version-utils/compat-workspaces/full/pnpm-lock.yaml | $(FFPipelineHostDirectory)/pnpm-lock.yaml'
 
-        # Install FluidFramework dependencies
+        # Install ff_pipeline_host dependencies (provides telemetry-generator and trips-setup/cleanup)
         - template: /tools/pipelines/templates/include-install.yml@self
           parameters:
-            buildDirectory: $(FluidFrameworkDirectory)
+            buildDirectory: $(FFPipelineHostDirectory)
+            installDependenciesDescription: Install ff_pipeline_host dependencies
+            # Note that skipping pnpm install means client pnpm version must suffice for ff_pipeline_host.
+            # TODO #74732: consider separating pnpm stores and caches if later ff_pipeline_host
+            # steps can remain version agnostic and then this install can be restored to take
+            # place before FluidFramework install.
+            installPnpm: false
             primaryRegistry: $(ado-feeds-ff-download-only)
 
         # Build the workspace so compiled test code is available for benchmarks