Implements --check-resolutions

.yarn/versions/1f93b69c.yml

@@ -0,0 +1,33 @@
+releases:
+  "@yarnpkg/cli": major
+  "@yarnpkg/core": major
+  "@yarnpkg/plugin-essentials": major
+  "@yarnpkg/plugin-exec": major
+  "@yarnpkg/plugin-file": major
+  "@yarnpkg/plugin-git": major
+  "@yarnpkg/plugin-http": major
+  "@yarnpkg/plugin-link": major
+  "@yarnpkg/plugin-npm": major
+  "@yarnpkg/plugin-patch": major
+
+declined:
+  - "@yarnpkg/plugin-compat"
+  - "@yarnpkg/plugin-constraints"
+  - "@yarnpkg/plugin-dlx"
+  - "@yarnpkg/plugin-github"
+  - "@yarnpkg/plugin-init"
+  - "@yarnpkg/plugin-interactive-tools"
+  - "@yarnpkg/plugin-nm"
+  - "@yarnpkg/plugin-npm-cli"
+  - "@yarnpkg/plugin-pack"
+  - "@yarnpkg/plugin-pnp"
+  - "@yarnpkg/plugin-pnpm"
+  - "@yarnpkg/plugin-stage"
+  - "@yarnpkg/plugin-typescript"
+  - "@yarnpkg/plugin-version"
+  - "@yarnpkg/plugin-workspace-tools"
+  - "@yarnpkg/builder"
+  - "@yarnpkg/doctor"
+  - "@yarnpkg/nm"
+  - "@yarnpkg/pnpify"
+  - "@yarnpkg/sdks"

packages/acceptance-tests/pkg-tests-specs/sources/features/checkResolutions.test.ts

@@ -0,0 +1,60 @@
+import {structUtils}              from '@yarnpkg/core';
+import {Filename, ppath, xfs}     from '@yarnpkg/fslib';
+import {parseSyml, stringifySyml} from '@yarnpkg/parsers';
+
+const tests: Array<[initial: string, replacement: string, valid: boolean]> = [
+  [`no-deps@npm:^1.0.0`, `no-deps@npm:2.0.0`, false],
+  [`no-deps@npm:^1.0.0`, `no-deps@npm:1.0.0`, true],
+
+  [`no-deps@npm:^1.0.0`, `no-deps-bins@npm:1.0.0`, false],
+  [`no-deps@npm:no-deps-bins@^1.0.0`, `no-deps-bins@npm:1.0.0`, true],
+
+  [`util-deprecate@https://github.com/yarnpkg/util-deprecate.git#commit=4bcc600d20e3a53ea27fa52c4d1fc49cc2d0eabb`, `util-deprecate@https://github.com/yarnpkg/util-deprecate.git`, false],
+  [`util-deprecate@https://github.com/yarnpkg/util-deprecate.git#commit=4bcc600d20e3a53ea27fa52c4d1fc49cc2d0eabb`, `util-deprecate@https://github.com/yarnpkg/util-deprecate.git#commit=475fb6857cd23fafff20c1be846c1350abf8e6d4`, false],
+  [`util-deprecate@https://github.com/yarnpkg/util-deprecate.git#commit=4bcc600d20e3a53ea27fa52c4d1fc49cc2d0eabb`, `util-deprecate@https://github.com/yarnpkg/util-deprecate.git#commit=4bcc600d20e3a53ea27fa52c4d1fc49cc2d0eabb`, true],
+];
+
+describe(`Features`, () => {
+  for (const [initial, replacement, valid] of tests) {
+    it(
+      `should ${valid ? `allow` : `prevent`} resolving "${initial}" with "${replacement}"`,
+      makeTemporaryEnv({}, {
+        // We don't care about this flag; in an actual attack,
+        // the hash would be correct
+        checksumBehavior: `ignore`,
+      }, async ({path, run, source}) => {
+        await run(`add`, replacement);
+
+        const lockfilePath = ppath.join(path, Filename.lockfile);
+        const lockfileContent = await xfs.readFilePromise(lockfilePath, `utf8`);
+        const lockfileData = parseSyml(lockfileContent);
+
+        lockfileData[initial] = {
+          version: lockfileData[replacement].version,
+          resolution: replacement,
+          languageName: lockfileData[replacement].languageName,
+          linkType: lockfileData[replacement].linkType,
+        };
+
+        await xfs.writeFilePromise(lockfilePath, stringifySyml(lockfileData));
+
+        const manifestPath = ppath.join(path, Filename.manifest);
+        const manifestData = await xfs.readJsonPromise(manifestPath);
+
+        const descriptor = structUtils.parseDescriptor(initial);
+        manifestData.dependencies = {
+          [structUtils.stringifyIdent(descriptor)]: descriptor.range,
+        };
+
+        await xfs.writeJsonPromise(manifestPath, manifestData);
+
+        const check = run(`install`, `--check-resolutions`);
+        if (valid) {
+          await check;
+        } else {
+          await expect(check).rejects.toThrow(/YN0078/);
+        }
+      }),
+    );
+  }
+});

packages/gatsby/content/advanced/error-codes.md

@@ -368,3 +368,19 @@ A package is specified in its manifest (through the [`os`](/configuration/manife
 Some native packages may be excluded from the install if they signal they don't support the systems the project is intended for. This detection is typically based on your current system parameters, but it can be configured using the [`supportedArchitectures` config option](/configuration/yarnrc#supportedArchitectures). If your os or cpu are missing from this list, Yarn will skip the packages and raise a warning.
 
 Note that all fields from `supportedArchitectures` default to `current`, which is a dynamic value depending on your local parameters. For instance, if you wish to support "my current os, whatever it is, plus linux", you can set `supportedArchitectures.os` to `["current", "linux"]`.
+
+## YN0078 - `RESOLUTION_MISMATCH`
+
+Starting from Yarn 4, Yarn will automatically enable the `--check-resolutions` flag on CI when it detects the current environment is a pull request. Under this mode, Yarn will check that the lockfile resolutions are consistent with what the initial range is. For example, given an initial dependency of `foo@npm:^1.0.0`:
+
+- `foo@npm:1.2.0` is a valid resolution
+- `foo@npm:2.0.0` isn't a valid resolution, because it doesn't match the expected semver range
+- `bar@npm:1.2.0` isn't a valid resolution either, because the name doesn't match
+
+This error should never trigger under normal circumstances, as Yarn should always generate satisfying resolutions given a dependency. If you hit it nonetheless, it may be either of two things:
+
+- Yarn has a bug. It may happen! Review the mismatch to be sure and, in case you have a doubt, ping us on Discord and we'll tell you whether it's something to worry about (before doing that, take a quick look at our [repository issues](https://github.com/yarnpkg/berry/issues?q=is%3Aissue+is%3Aopen+YN0078) in case someone reported the same behaviour).
+
+- Or you might have someone doing strange things on your lockfile. It might be a mistake (for example someone manually modifying a lockfile for debug but forgetting to revert the changes), or a problem (for example a malicious users trying to perform some sort of [supply chain attack](https://en.wikipedia.org/wiki/Supply_chain_attack)).
+
+If the use case appears legit (for example if the bug comes from Yarn), you can bypass the check on PRs by adding a `--no-check-resolutions` flag to your `yarn install` command. But be careful: this is a security feature; disabling it may have consequences.

packages/plugin-essentials/sources/commands/install.ts

@@ -70,6 +70,10 @@ export default class YarnCommand extends BaseCommand {
     description: `Always refetch the packages and ensure that their checksums are consistent`,
   });
 
+  checkResolutions = Option.Boolean(`--check-resolutions`, {
+    description: `Validates that the package resolutions are coherent`,
+  });
+
   inlineBuilds = Option.Boolean(`--inline-builds`, {
     description: `Verbosely print the output of the build steps of dependencies`,
   });
@@ -313,13 +317,15 @@ export default class YarnCommand extends BaseCommand {
     // the Configuration and Install classes). Feel free to open an issue
     // in order to ask for design feedback before writing features.
 
+    const checkResolutions = this.checkResolutions ?? CI.isPR ?? false;
+
     const report = await StreamReport.start({
       configuration,
       json: this.json,
       stdout: this.context.stdout,
       includeLogs: true,
     }, async (report: StreamReport) => {
-      await project.install({cache, report, immutable, mode: this.mode});
+      await project.install({cache, report, immutable, checkResolutions, mode: this.mode});
     });
 
     return report.exitCode();

packages/plugin-essentials/sources/dedupeUtils.ts

@@ -1,17 +1,20 @@
-import {Project, ResolveOptions, ThrowReport, Resolver, miscUtils, Descriptor, Package, Report, Cache} from '@yarnpkg/core';
-import {formatUtils, structUtils, IdentHash, LocatorHash, MessageName, Fetcher, FetchOptions}          from '@yarnpkg/core';
-import micromatch                                                                                      from 'micromatch';
+import {Project, ResolveOptions, ThrowReport, Resolver, miscUtils, Descriptor, Package, Report, Cache, DescriptorHash} from '@yarnpkg/core';
+import {formatUtils, structUtils, IdentHash, LocatorHash, MessageName, Fetcher, FetchOptions}                          from '@yarnpkg/core';
+import micromatch                                                                                                      from 'micromatch';
+
+export type PackageUpdate = {
+  descriptor: Descriptor;
+  currentPackage: Package;
+  updatedPackage: Package;
+  resolvedPackage: Package;
+};
 
 export type Algorithm = (project: Project, patterns: Array<string>, opts: {
   resolver: Resolver;
   resolveOptions: ResolveOptions;
   fetcher: Fetcher;
   fetchOptions: FetchOptions;
-}) => Promise<Array<Promise<{
-  descriptor: Descriptor;
-  currentPackage: Package;
-  updatedPackage: Package;
-} | null>>>;
+}) => Promise<Array<Promise<PackageUpdate>>>;
 
 export enum Strategy {
   /**
@@ -37,57 +40,100 @@ const DEDUPE_ALGORITHMS: Record<Strategy, Algorithm> = {
       miscUtils.getSetWithDefault(locatorsByIdent, descriptor.identHash).add(locatorHash);
     }
 
-    return Array.from(project.storedDescriptors.values(), async descriptor => {
-      if (patterns.length && !micromatch.isMatch(structUtils.stringifyIdent(descriptor), patterns))
-        return null;
+    const deferredMap = new Map<DescriptorHash, miscUtils.Deferred<PackageUpdate>>(
+      miscUtils.mapAndFilter(project.storedDescriptors.values(), descriptor => {
+        // We only care about resolutions that are stored in the lockfile
+        // (we shouldn't accidentally try deduping virtual packages)
+        if (structUtils.isVirtualDescriptor(descriptor))
+          return miscUtils.mapAndFilter.skip;
+
+        return [descriptor.descriptorHash, miscUtils.makeDeferred()];
+      }),
+    );
+
+    for (const descriptor of project.storedDescriptors.values()) {
+      const deferred = deferredMap.get(descriptor.descriptorHash);
+      if (typeof deferred === `undefined`)
+        throw new Error(`Assertion failed: The descriptor (${descriptor.descriptorHash}) should have been registered`);
 
       const currentResolution = project.storedResolutions.get(descriptor.descriptorHash);
       if (typeof currentResolution === `undefined`)
         throw new Error(`Assertion failed: The resolution (${descriptor.descriptorHash}) should have been registered`);
 
-      // We only care about resolutions that are stored in the lockfile
-      // (we shouldn't accidentally try deduping virtual packages)
       const currentPackage = project.originalPackages.get(currentResolution);
       if (typeof currentPackage === `undefined`)
-        return null;
-
-      // No need to try deduping packages that are not persisted,
-      // they will be resolved again anyways
-      if (!resolver.shouldPersistResolution(currentPackage, resolveOptions))
-        return null;
-
-      const locators = locatorsByIdent.get(descriptor.identHash);
-      if (typeof locators === `undefined`)
-        throw new Error(`Assertion failed: The resolutions (${descriptor.identHash}) should have been registered`);
-
-      // No need to choose when there's only one possibility
-      if (locators.size === 1)
-        return null;
-
-      const references = [...locators].map(locatorHash => {
-        const pkg = project.originalPackages.get(locatorHash);
-        if (typeof pkg === `undefined`)
-          throw new Error(`Assertion failed: The package (${locatorHash}) should have been registered`);
-
-        return pkg.reference;
+        throw new Error(`Assertion failed: The package (${currentResolution}) should have been registered`);
+
+      Promise.resolve().then(async () => {
+        const dependencies = resolver.getResolutionDependencies(descriptor, resolveOptions);
+
+        const resolvedDependencies = Object.fromEntries(
+          await miscUtils.allSettledSafe(
+            Object.entries(dependencies).map(async ([dependencyName, dependency]) => {
+              const dependencyDeferred = deferredMap.get(dependency.descriptorHash);
+              if (typeof dependencyDeferred === `undefined`)
+                throw new Error(`Assertion failed: The descriptor (${dependency.descriptorHash}) should have been registered`);
+
+              const dedupeResult = await dependencyDeferred.promise;
+              if (!dedupeResult)
+                throw new Error(`Assertion failed: Expected the dependency to have been through the dedupe process itself`);
+
+              return [dependencyName, dedupeResult.updatedPackage];
+            }),
+          ),
+        );
+
+        if (patterns.length && !micromatch.isMatch(structUtils.stringifyIdent(descriptor), patterns))
+          return currentPackage;
+
+        // No need to try deduping packages that are not persisted,
+        // they will be resolved again anyways
+        if (!resolver.shouldPersistResolution(currentPackage, resolveOptions))
+          return currentPackage;
+
+        const candidateHashes = locatorsByIdent.get(descriptor.identHash);
+        if (typeof candidateHashes === `undefined`)
+          throw new Error(`Assertion failed: The resolutions (${descriptor.identHash}) should have been registered`);
+
+        // No need to choose when there's only one possibility
+        if (candidateHashes.size === 1)
+          return currentPackage;
+
+        const candidates = [...candidateHashes].map(locatorHash => {
+          const pkg = project.originalPackages.get(locatorHash);
+          if (typeof pkg === `undefined`)
+            throw new Error(`Assertion failed: The package (${locatorHash}) should have been registered`);
+
+          return pkg;
+        });
+
+        const satisfying = await resolver.getSatisfying(descriptor, resolvedDependencies, candidates, resolveOptions);
+
+        const bestLocator = satisfying.locators?.[0];
+        if (typeof bestLocator === `undefined` || !satisfying.sorted)
+          return currentPackage;
+
+        const updatedPackage = project.originalPackages.get(bestLocator.locatorHash);
+        if (typeof updatedPackage === `undefined`)
+          throw new Error(`Assertion failed: The package (${bestLocator.locatorHash}) should have been registered`);
+
+        return updatedPackage;
+      }).then(async updatedPackage => {
+        const resolvedPackage = await project.preparePackage(updatedPackage, {resolver, resolveOptions});
+
+        deferred.resolve({
+          descriptor,
+          currentPackage,
+          updatedPackage,
+          resolvedPackage,
+        });
+      }).catch(error => {
+        deferred.reject(error);
       });
+    }
 
-      const candidates = await resolver.getSatisfying(descriptor, references, resolveOptions);
-
-      const bestCandidate = candidates?.[0];
-      if (typeof bestCandidate === `undefined`)
-        return null;
-
-      const updatedResolution = bestCandidate.locatorHash;
-
-      const updatedPackage = project.originalPackages.get(updatedResolution);
-      if (typeof updatedPackage === `undefined`)
-        throw new Error(`Assertion failed: The package (${updatedResolution}) should have been registered`);
-
-      if (updatedResolution === currentResolution)
-        return null;
-
-      return {descriptor, currentPackage, updatedPackage};
+    return [...deferredMap.values()].map(deferred => {
+      return deferred.promise;
     });
   },
 };
@@ -137,7 +183,7 @@ export async function dedupe(project: Project, {strategy, patterns, cache, repor
       dedupePromises.map(dedupePromise =>
         dedupePromise
           .then(dedupe => {
-            if (dedupe === null)
+            if (dedupe === null || dedupe.currentPackage.locatorHash === dedupe.updatedPackage.locatorHash)
               return;
 
             dedupedPackageCount++;

packages/plugin-exec/sources/ExecResolver.ts

@@ -60,8 +60,13 @@ export class ExecResolver implements Resolver {
     return [execUtils.makeLocator(descriptor, {parentLocator, path, generatorHash, protocol: PROTOCOL})];
   }
 
-  async getSatisfying(descriptor: Descriptor, references: Array<string>, opts: ResolveOptions) {
-    return null;
+  async getSatisfying(descriptor: Descriptor, dependencies: Record<string, Package>, locators: Array<Locator>, opts: ResolveOptions) {
+    const [locator] = await this.getCandidates(descriptor, dependencies, opts);
+
+    return {
+      locators: locators.filter(candidate => candidate.locatorHash === locator.locatorHash),
+      sorted: false,
+    };
   }
 
   async resolve(locator: Locator, opts: ResolveOptions) {

packages/plugin-file/sources/FileResolver.ts

@@ -1,4 +1,4 @@
-import {miscUtils, structUtils, hashUtils}               from '@yarnpkg/core';
+import {miscUtils, structUtils, hashUtils, Package}      from '@yarnpkg/core';
 import {LinkType}                                        from '@yarnpkg/core';
 import {Descriptor, Locator, Manifest}                   from '@yarnpkg/core';
 import {Resolver, ResolveOptions, MinimalResolveOptions} from '@yarnpkg/core';
@@ -72,8 +72,13 @@ export class FileResolver implements Resolver {
     return [fileUtils.makeLocator(descriptor, {parentLocator, path, folderHash, protocol: PROTOCOL})];
   }
 
-  async getSatisfying(descriptor: Descriptor, references: Array<string>, opts: ResolveOptions) {
-    return null;
+  async getSatisfying(descriptor: Descriptor, dependencies: Record<string, Package>, locators: Array<Locator>, opts: ResolveOptions) {
+    const [locator] = await this.getCandidates(descriptor, dependencies, opts);
+
+    return {
+      locators: locators.filter(candidate => candidate.locatorHash === locator.locatorHash),
+      sorted: false,
+    };
   }
 
   async resolve(locator: Locator, opts: ResolveOptions) {

packages/plugin-file/sources/TarballFileResolver.ts

@@ -1,10 +1,10 @@
-import {Resolver, ResolveOptions, MinimalResolveOptions} from '@yarnpkg/core';
-import {Descriptor, Locator, Manifest}                   from '@yarnpkg/core';
-import {LinkType}                                        from '@yarnpkg/core';
-import {miscUtils, structUtils}                          from '@yarnpkg/core';
-import {npath}                                           from '@yarnpkg/fslib';
+import {Resolver, ResolveOptions, MinimalResolveOptions, Package} from '@yarnpkg/core';
+import {Descriptor, Locator, Manifest}                            from '@yarnpkg/core';
+import {LinkType}                                                 from '@yarnpkg/core';
+import {miscUtils, structUtils}                                   from '@yarnpkg/core';
+import {npath}                                                    from '@yarnpkg/fslib';
 
-import {FILE_REGEXP, TARBALL_REGEXP, PROTOCOL}           from './constants';
+import {FILE_REGEXP, TARBALL_REGEXP, PROTOCOL}                    from './constants';
 
 export class TarballFileResolver implements Resolver {
   supportsDescriptor(descriptor: Descriptor, opts: MinimalResolveOptions) {
@@ -55,8 +55,13 @@ export class TarballFileResolver implements Resolver {
     return [structUtils.makeLocator(descriptor, `${PROTOCOL}${npath.toPortablePath(path)}`)];
   }
 
-  async getSatisfying(descriptor: Descriptor, references: Array<string>, opts: ResolveOptions) {
-    return null;
+  async getSatisfying(descriptor: Descriptor, dependencies: Record<string, Package>, locators: Array<Locator>, opts: ResolveOptions) {
+    const [locator] = await this.getCandidates(descriptor, dependencies, opts);
+
+    return {
+      locators: locators.filter(candidate => candidate.locatorHash === locator.locatorHash),
+      sorted: false,
+    };
   }
 
   async resolve(locator: Locator, opts: ResolveOptions) {