Normalizes dependencies

.yarn/versions/b3d80a1d.yml

@@ -0,0 +1,33 @@
+releases:
+  "@yarnpkg/cli": major
+  "@yarnpkg/core": major
+  "@yarnpkg/doctor": major
+  "@yarnpkg/plugin-essentials": major
+  "@yarnpkg/plugin-exec": major
+  "@yarnpkg/plugin-file": major
+  "@yarnpkg/plugin-git": major
+  "@yarnpkg/plugin-http": major
+  "@yarnpkg/plugin-link": major
+  "@yarnpkg/plugin-npm": major
+  "@yarnpkg/plugin-patch": major
+  "@yarnpkg/plugin-typescript": major
+
+declined:
+  - "@yarnpkg/plugin-compat"
+  - "@yarnpkg/plugin-constraints"
+  - "@yarnpkg/plugin-dlx"
+  - "@yarnpkg/plugin-github"
+  - "@yarnpkg/plugin-init"
+  - "@yarnpkg/plugin-interactive-tools"
+  - "@yarnpkg/plugin-nm"
+  - "@yarnpkg/plugin-npm-cli"
+  - "@yarnpkg/plugin-pack"
+  - "@yarnpkg/plugin-pnp"
+  - "@yarnpkg/plugin-pnpm"
+  - "@yarnpkg/plugin-stage"
+  - "@yarnpkg/plugin-version"
+  - "@yarnpkg/plugin-workspace-tools"
+  - "@yarnpkg/builder"
+  - "@yarnpkg/nm"
+  - "@yarnpkg/pnpify"
+  - "@yarnpkg/sdks"

CHANGELOG.md

@@ -17,6 +17,18 @@ Yarn now accepts sponsorships! Please give a look at our [OpenCollective](https:
   - Plugins cannot access the Clipanion 2 APIs anymore (upgrade to [Clipanion 3](https://github.com/arcanis/clipanion))
   - Plugins cannot access the internal copy of Yup anymore (use [Typanion](https://github.com/arcanis/typanion) instead)
 
+### **API Changes**
+
+The following changes only affect people writing Yarn plugins:
+
+- The `dependencies` field sent returned by `Resolver#resolve` must now be the result of a `Configuration#normalizeDependencyMap` call. This change is prompted by a refactoring of how default protocols (ie `npm:`) are injected into descriptors. The previous implementation caused various descriptors to never be normalized, which made it difficult to know what were the descriptors each function should expect.
+
+  - Similarly, the descriptors returned by `Resolve#getResolutionDependencies` are now expected to be the result of `Configuration#normalizeDependency` calls.
+
+  - Note that this only applies to the `dependencies` field; the `peerDependencies` field is unchanged, as it must only contains semver ranges without any protocol (with an exception for `workspace:`, but that's not relevant here).
+
+- The `Resolve#getResolutionDependencies` function must now return an object of arbitrary string keys and descriptor values (instead of a map with `DescriptorHash` keys). Those descriptors will be resolved and assigned to the same keys as the initial object. This change allows resolvers to wrap resolution dependencies from other resolvers, which wasn't possible before since it'd have caused the key to change.
+
 ### Installs
 
 - The pnpm linker no longer tries to remove `node_modules` directory, when `node-modules` linker is active

packages/plugin-compat/tests/patches.test.ts

@@ -27,7 +27,7 @@ async function getDescriptorCandidates(descriptor: Descriptor) {
     const resolver = configuration.makeResolver();
     const resolveOptions: ResolveOptions = {project, resolver, report: new ThrowReport()};
 
-    const candidates = await resolver.getCandidates(descriptor, new Map(), resolveOptions);
+    const candidates = await resolver.getCandidates(descriptor, {}, resolveOptions);
 
     return candidates;
   });

packages/plugin-essentials/sources/suggestUtils.ts

@@ -367,7 +367,7 @@ export async function fetchDescriptorFrom(ident: Ident, range: string, {project,
   // If we didn't bind it, `yarn add ./folder` wouldn't work.
   const boundDescriptor = resolver.bindDescriptor(latestDescriptor, workspace.anchoredLocator, resolveOptions);
 
-  const candidateLocators = await resolver.getCandidates(boundDescriptor, new Map(), resolveOptions);
+  const candidateLocators = await resolver.getCandidates(boundDescriptor, {}, resolveOptions);
 
   if (candidateLocators.length === 0)
     return null;

packages/plugin-exec/sources/ExecResolver.ts

@@ -35,10 +35,10 @@ export class ExecResolver implements Resolver {
   }
 
   getResolutionDependencies(descriptor: Descriptor, opts: MinimalResolveOptions) {
-    return [];
+    return {};
   }
 
-  async getCandidates(descriptor: Descriptor, dependencies: Map<DescriptorHash, Package>, opts: ResolveOptions) {
+  async getCandidates(descriptor: Descriptor, dependencies: Record<string, Package>, opts: ResolveOptions) {
     if (!opts.fetchOptions)
       throw new Error(`Assertion failed: This resolver cannot be used unless a fetcher is configured`);
 
@@ -84,7 +84,7 @@ export class ExecResolver implements Resolver {
 
       conditions: manifest.getConditions(),
 
-      dependencies: manifest.dependencies,
+      dependencies: opts.project.configuration.normalizeDependencyMap(manifest.dependencies),
       peerDependencies: manifest.peerDependencies,
 
       dependenciesMeta: manifest.dependenciesMeta,

packages/plugin-file/sources/FileResolver.ts

@@ -41,7 +41,7 @@ export class FileResolver implements Resolver {
   }
 
   getResolutionDependencies(descriptor: Descriptor, opts: MinimalResolveOptions) {
-    return [];
+    return {};
   }
 
   async getCandidates(descriptor: Descriptor, dependencies: unknown, opts: ResolveOptions) {
@@ -96,7 +96,7 @@ export class FileResolver implements Resolver {
 
       conditions: manifest.getConditions(),
 
-      dependencies: manifest.dependencies,
+      dependencies: opts.project.configuration.normalizeDependencyMap(manifest.dependencies),
       peerDependencies: manifest.peerDependencies,
 
       dependenciesMeta: manifest.dependenciesMeta,

packages/plugin-file/sources/TarballFileResolver.ts

@@ -44,7 +44,7 @@ export class TarballFileResolver implements Resolver {
   }
 
   getResolutionDependencies(descriptor: Descriptor, opts: MinimalResolveOptions) {
-    return [];
+    return {};
   }
 
   async getCandidates(descriptor: Descriptor, dependencies: unknown, opts: ResolveOptions) {
@@ -79,7 +79,7 @@ export class TarballFileResolver implements Resolver {
 
       conditions: manifest.getConditions(),
 
-      dependencies: manifest.dependencies,
+      dependencies: opts.project.configuration.normalizeDependencyMap(manifest.dependencies),
       peerDependencies: manifest.peerDependencies,
 
       dependenciesMeta: manifest.dependenciesMeta,

packages/plugin-git/sources/GitResolver.ts

@@ -23,7 +23,7 @@ export class GitResolver implements Resolver {
   }
 
   getResolutionDependencies(descriptor: Descriptor, opts: MinimalResolveOptions) {
-    return [];
+    return {};
   }
 
   async getCandidates(descriptor: Descriptor, dependencies: unknown, opts: ResolveOptions) {
@@ -57,7 +57,7 @@ export class GitResolver implements Resolver {
 
       conditions: manifest.getConditions(),
 
-      dependencies: manifest.dependencies,
+      dependencies: opts.project.configuration.normalizeDependencyMap(manifest.dependencies),
       peerDependencies: manifest.peerDependencies,
 
       dependenciesMeta: manifest.dependenciesMeta,

packages/plugin-http/sources/TarballHttpResolver.ts

@@ -35,7 +35,7 @@ export class TarballHttpResolver implements Resolver {
   }
 
   getResolutionDependencies(descriptor: Descriptor, opts: MinimalResolveOptions) {
-    return [];
+    return {};
   }
 
   async getCandidates(descriptor: Descriptor, dependencies: unknown, opts: ResolveOptions) {
@@ -66,7 +66,7 @@ export class TarballHttpResolver implements Resolver {
 
       conditions: manifest.getConditions(),
 
-      dependencies: manifest.dependencies,
+      dependencies: opts.project.configuration.normalizeDependencyMap(manifest.dependencies),
       peerDependencies: manifest.peerDependencies,
 
       dependenciesMeta: manifest.dependenciesMeta,

packages/plugin-link/sources/LinkResolver.ts

@@ -32,7 +32,7 @@ export class LinkResolver implements Resolver {
   }
 
   getResolutionDependencies(descriptor: Descriptor, opts: MinimalResolveOptions) {
-    return [];
+    return {};
   }
 
   async getCandidates(descriptor: Descriptor, dependencies: unknown, opts: ResolveOptions) {
@@ -65,7 +65,7 @@ export class LinkResolver implements Resolver {
 
       conditions: manifest.getConditions(),
 
-      dependencies: new Map([...manifest.dependencies]),
+      dependencies: opts.project.configuration.normalizeDependencyMap(manifest.dependencies),
       peerDependencies: manifest.peerDependencies,
 
       dependenciesMeta: manifest.dependenciesMeta,

packages/plugin-link/sources/RawLinkResolver.ts

@@ -32,7 +32,7 @@ export class RawLinkResolver implements Resolver {
   }
 
   getResolutionDependencies(descriptor: Descriptor, opts: MinimalResolveOptions) {
-    return [];
+    return {};
   }
 
   async getCandidates(descriptor: Descriptor, dependencies: unknown, opts: ResolveOptions) {

packages/plugin-npm/sources/NpmRemapResolver.ts

@@ -1,7 +1,7 @@
-import {Descriptor, Locator, MinimalResolveOptions, ResolveOptions, Resolver, DescriptorHash, Package} from '@yarnpkg/core';
-import {structUtils}                                                                                   from '@yarnpkg/core';
+import {Descriptor, Locator, MinimalResolveOptions, ResolveOptions, Resolver, Package} from '@yarnpkg/core';
+import {structUtils}                                                                   from '@yarnpkg/core';
 
-import {PROTOCOL}                                                                                      from './constants';
+import {PROTOCOL}                                                                      from './constants';
 
 export class NpmRemapResolver implements Resolver {
   supportsDescriptor(descriptor: Descriptor, opts: MinimalResolveOptions) {
@@ -29,19 +29,25 @@ export class NpmRemapResolver implements Resolver {
   }
 
   getResolutionDependencies(descriptor: Descriptor, opts: MinimalResolveOptions) {
-    const nextDescriptor = structUtils.parseDescriptor(descriptor.range.slice(PROTOCOL.length), true);
+    const nextDescriptor = opts.project.configuration.normalizeDependency(
+      structUtils.parseDescriptor(descriptor.range.slice(PROTOCOL.length), true),
+    );
 
     return opts.resolver.getResolutionDependencies(nextDescriptor, opts);
   }
 
-  async getCandidates(descriptor: Descriptor, dependencies: Map<DescriptorHash, Package>, opts: ResolveOptions) {
-    const nextDescriptor = structUtils.parseDescriptor(descriptor.range.slice(PROTOCOL.length), true);
+  async getCandidates(descriptor: Descriptor, dependencies: Record<string, Package>, opts: ResolveOptions) {
+    const nextDescriptor = opts.project.configuration.normalizeDependency(
+      structUtils.parseDescriptor(descriptor.range.slice(PROTOCOL.length), true),
+    );
 
     return await opts.resolver.getCandidates(nextDescriptor, dependencies, opts);
   }
 
   async getSatisfying(descriptor: Descriptor, references: Array<string>, opts: ResolveOptions) {
-    const nextDescriptor = structUtils.parseDescriptor(descriptor.range.slice(PROTOCOL.length), true);
+    const nextDescriptor = opts.project.configuration.normalizeDependency(
+      structUtils.parseDescriptor(descriptor.range.slice(PROTOCOL.length), true),
+    );
 
     return opts.resolver.getSatisfying(nextDescriptor, references, opts);
   }

packages/plugin-npm/sources/NpmSemverResolver.ts

@@ -39,10 +39,10 @@ export class NpmSemverResolver implements Resolver {
   }
 
   getResolutionDependencies(descriptor: Descriptor, opts: MinimalResolveOptions) {
-    return [];
+    return {};
   }
 
-  async getCandidates(descriptor: Descriptor, dependencies: Map<DescriptorHash, Package>, opts: ResolveOptions) {
+  async getCandidates(descriptor: Descriptor, dependencies: Record<string, Package>, opts: ResolveOptions) {
     const range = semverUtils.validRange(descriptor.range.slice(PROTOCOL.length));
     if (range === null)
       throw new Error(`Expected a valid range, got ${descriptor.range.slice(PROTOCOL.length)}`);
@@ -174,7 +174,7 @@ export class NpmSemverResolver implements Resolver {
 
       conditions: manifest.getConditions(),
 
-      dependencies: manifest.dependencies,
+      dependencies: opts.project.configuration.normalizeDependencyMap(manifest.dependencies),
       peerDependencies: manifest.peerDependencies,
 
       dependenciesMeta: manifest.dependenciesMeta,

packages/plugin-npm/sources/NpmTagResolver.ts

@@ -32,7 +32,7 @@ export class NpmTagResolver implements Resolver {
   }
 
   getResolutionDependencies(descriptor: Descriptor, opts: MinimalResolveOptions) {
-    return [];
+    return {};
   }
 
   async getCandidates(descriptor: Descriptor, dependencies: unknown, opts: ResolveOptions) {

packages/plugin-patch/sources/PatchResolver.ts

@@ -42,17 +42,19 @@ export class PatchResolver implements Resolver {
   getResolutionDependencies(descriptor: Descriptor, opts: MinimalResolveOptions) {
     const {sourceDescriptor} = patchUtils.parseDescriptor(descriptor);
 
-    return [sourceDescriptor];
+    return {
+      sourceDescriptor: opts.project.configuration.normalizeDependency(sourceDescriptor),
+    };
   }
 
-  async getCandidates(descriptor: Descriptor, dependencies: Map<DescriptorHash, Package>, opts: ResolveOptions) {
+  async getCandidates(descriptor: Descriptor, dependencies: Record<string, Package>, opts: ResolveOptions) {
     if (!opts.fetchOptions)
       throw new Error(`Assertion failed: This resolver cannot be used unless a fetcher is configured`);
 
-    const {parentLocator, sourceDescriptor, patchPaths} = patchUtils.parseDescriptor(descriptor);
+    const {parentLocator, patchPaths} = patchUtils.parseDescriptor(descriptor);
     const patchFiles = await patchUtils.loadPatchFiles(parentLocator, patchPaths, opts.fetchOptions);
 
-    const sourcePackage = dependencies.get(sourceDescriptor.descriptorHash);
+    const sourcePackage = dependencies.sourceDescriptor;
     if (typeof sourcePackage === `undefined`)
       throw new Error(`Assertion failed: The dependency should have been resolved`);
 

packages/plugin-typescript/sources/index.ts

@@ -49,7 +49,7 @@ const afterWorkspaceDependencyAddition = async (
   let range = structUtils.parseRange(descriptor.range).selector;
   // If the range is a tag, we have to resolve it into a semver version
   if (!semverUtils.validRange(range)) {
-    const originalCandidates = await resolver.getCandidates(descriptor, new Map<DescriptorHash, Package>(), resolveOptions);
+    const originalCandidates = await resolver.getCandidates(descriptor, {}, resolveOptions);
     range = structUtils.parseRange(originalCandidates[0].reference).selector;
   }
 
@@ -92,7 +92,7 @@ const afterWorkspaceDependencyAddition = async (
   } else {
     // Return if the atTypes descriptor can't be resolved
     try {
-      const atTypesCandidates = await resolver.getCandidates(atTypesDescriptor, new Map<DescriptorHash, Package>(), resolveOptions);
+      const atTypesCandidates = await resolver.getCandidates(atTypesDescriptor, {}, resolveOptions);
       if (atTypesCandidates.length === 0) {
         return;
       }

packages/yarnpkg-core/sources/Configuration.ts

@@ -5,14 +5,14 @@ import camelcase
 import {isCI}                                                                                           from 'ci-info';
 import {UsageError}                                                                                     from 'clipanion';
 import pLimit, {Limit}                                                                                  from 'p-limit';
+import semver                                                                                           from 'semver';
 import {PassThrough, Writable}                                                                          from 'stream';
 
 import {CorePlugin}                                                                                     from './CorePlugin';
 import {Manifest, PeerDependencyMeta}                                                                   from './Manifest';
 import {MultiFetcher}                                                                                   from './MultiFetcher';
 import {MultiResolver}                                                                                  from './MultiResolver';
 import {Plugin, Hooks}                                                                                  from './Plugin';
-import {ProtocolResolver}                                                                               from './ProtocolResolver';
 import {Report}                                                                                         from './Report';
 import {TelemetryManager}                                                                               from './TelemetryManager';
 import {VirtualFetcher}                                                                                 from './VirtualFetcher';
@@ -58,6 +58,8 @@ const IGNORED_ENV_VARIABLES = new Set([
   `confDir`,
 ]);
 
+export const TAG_REGEXP = /^(?!v)[a-z0-9._-]+$/i;
+
 export const ENVIRONMENT_PREFIX = `yarn_`;
 export const DEFAULT_RC_FILENAME = `.yarnrc.yml` as Filename;
 export const DEFAULT_LOCK_FILENAME = `yarn.lock` as Filename;
@@ -1449,13 +1451,14 @@ export class Configuration {
       for (const resolver of plugin.resolvers || [])
         pluginResolvers.push(new resolver());
 
-    return new MultiResolver([
-      new VirtualResolver(),
-      new WorkspaceResolver(),
-      new ProtocolResolver(),
+    return (
+      new MultiResolver([
+        new VirtualResolver(),
+        new WorkspaceResolver(),
 
-      ...pluginResolvers,
-    ]);
+        ...pluginResolvers,
+      ])
+    );
   }
 
   makeFetcher() {
@@ -1545,6 +1548,22 @@ export class Configuration {
     }
   }
 
+  normalizeDependency(dependency: Descriptor) {
+    if (semver.validRange(dependency.range))
+      return structUtils.makeDescriptor(dependency, `${this.get(`defaultProtocol`)}${dependency.range}`);
+
+    if (TAG_REGEXP.test(dependency.range))
+      return structUtils.makeDescriptor(dependency, `${this.get(`defaultProtocol`)}${dependency.range}`);
+
+    return dependency;
+  }
+
+  normalizeDependencyMap<TKey>(dependencyMap: Map<TKey, Descriptor>) {
+    return new Map([...dependencyMap].map(([key, dependency]) => {
+      return [key, this.normalizeDependency(dependency)];
+    }));
+  }
+
   normalizePackage(original: Package) {
     const pkg = structUtils.copyPackage(original);
 
@@ -1574,15 +1593,15 @@ export class Configuration {
                 const currentDependency = pkg.dependencies.get(extension.descriptor.identHash);
                 if (typeof currentDependency === `undefined`) {
                   extension.status = PackageExtensionStatus.Active;
-                  pkg.dependencies.set(extension.descriptor.identHash, extension.descriptor);
+                  pkg.dependencies.set(extension.descriptor.identHash, this.normalizeDependency(extension.descriptor));
                 }
               } break;
 
               case PackageExtensionType.PeerDependency: {
                 const currentPeerDependency = pkg.peerDependencies.get(extension.descriptor.identHash);
                 if (typeof currentPeerDependency === `undefined`) {
                   extension.status = PackageExtensionStatus.Active;
-                  pkg.peerDependencies.set(extension.descriptor.identHash, extension.descriptor);
+                  pkg.peerDependencies.set(extension.descriptor.identHash, this.normalizeDependency(extension.descriptor));
                 }
               } break;