Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
79f42d3
feat(plugin-essentials): yarn deduplicate
paul-soporan Jul 8, 2020
f1ce7d2
refactor: improve implementation
paul-soporan Jul 8, 2020
05781f1
Merge branch 'master' into paul/feat/deduplicate
paul-soporan Aug 17, 2020
4303e0d
refactor: shorten command name to `yarn dedupe`
paul-soporan Aug 17, 2020
408acbf
refactor: rewrite algorithm
paul-soporan Aug 17, 2020
7063670
refactor: more refactoring
paul-soporan Aug 17, 2020
d5a2291
refactor: even more refactoring
paul-soporan Aug 17, 2020
925112a
refactor: more refactoring + improved docs
paul-soporan Aug 19, 2020
ebfd0c1
docs: better dedupe docs
paul-soporan Aug 19, 2020
e8df62b
test: add tests
paul-soporan Aug 19, 2020
658055c
test: add tests for selective dedupe
paul-soporan Aug 19, 2020
ea4cc55
Merge remote-tracking branch 'yarnpkg/master' into paul/feat/deduplicate
paul-soporan Aug 19, 2020
bb60e25
refactor: refactor everything yet again
paul-soporan Aug 20, 2020
cc91662
refactor: fix cyclic dependency
paul-soporan Aug 20, 2020
51b342e
Update packages/plugin-essentials/sources/commands/dedupe.ts
paul-soporan Aug 22, 2020
4379dbe
Update packages/plugin-essentials/sources/commands/dedupe.ts
paul-soporan Aug 22, 2020
d637d8e
refactor: dedupeSkip -> null
paul-soporan Aug 22, 2020
1a705f5
refactor: skip -> assertion
paul-soporan Aug 22, 2020
cab447e
refactor: add note about virtual packages
paul-soporan Aug 22, 2020
d6b9a58
refactor: skip ->assertion failed
paul-soporan Aug 22, 2020
58ac353
Merge branch 'master' into paul/feat/deduplicate
paul-soporan Aug 22, 2020
9eb67d8
refactor: disable getSatisfying for some resolvers
paul-soporan Aug 23, 2020
aaf702f
refactor: remove resolutionDependencies
paul-soporan Aug 23, 2020
2465986
chore: lint
paul-soporan Aug 23, 2020
1fc26fa
refactor: skip -> assertion failed
paul-soporan Aug 23, 2020
d05b284
Apply suggestions from code review
paul-soporan Aug 23, 2020
50f0c51
refactor: use semver.SemVer instances
paul-soporan Aug 23, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
refactor: disable getSatisfying for some resolvers
  • Loading branch information
paul-soporan committed Aug 23, 2020
commit 9eb67d8ca6c189a26eef6c53b2173a32d24f4240
48 changes: 19 additions & 29 deletions packages/plugin-exec/sources/ExecResolver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,17 +39,29 @@ export class ExecResolver implements Resolver {
}

async getCandidates(descriptor: Descriptor, dependencies: Map<DescriptorHash, Package>, opts: ResolveOptions) {
const locator = await this.getCandidateForDescriptor(descriptor, opts);
if (!opts.fetchOptions)
throw new Error(`Assertion failed: This resolver cannot be used unless a fetcher is configured`);

const {path, parentLocator} = execUtils.parseSpec(descriptor.range);

if (parentLocator === null)
throw new Error(`Assertion failed: The descriptor should have been bound`);

return [locator];
const generatorFile = await execUtils.loadGeneratorFile(structUtils.makeRange({
protocol: PROTOCOL,
source: path,
selector: path,
params: {
locator: structUtils.stringifyLocator(parentLocator),
},
}), PROTOCOL, opts.fetchOptions);
const generatorHash = hashUtils.makeHash(`${CACHE_VERSION}`, generatorFile).slice(0, 6);

return [execUtils.makeLocator(descriptor, {parentLocator, path, generatorHash, protocol: PROTOCOL})];
}

async getSatisfying(descriptor: Descriptor, references: Array<string>, dependencies: Map<DescriptorHash, Package>, opts: ResolveOptions) {
const {reference: execReference} = await this.getCandidateForDescriptor(descriptor, opts);

return references
.filter(reference => reference === execReference)
.map(reference => structUtils.makeLocator(descriptor, reference));
return null;
}

async resolve(locator: Locator, opts: ResolveOptions) {
Expand Down Expand Up @@ -79,26 +91,4 @@ export class ExecResolver implements Resolver {
bin: manifest.bin,
};
}

private async getCandidateForDescriptor(descriptor: Descriptor, opts: ResolveOptions) {
if (!opts.fetchOptions)
throw new Error(`Assertion failed: This resolver cannot be used unless a fetcher is configured`);

const {path, parentLocator} = execUtils.parseSpec(descriptor.range);

if (parentLocator === null)
throw new Error(`Assertion failed: The descriptor should have been bound`);

const generatorFile = await execUtils.loadGeneratorFile(structUtils.makeRange({
protocol: PROTOCOL,
source: path,
selector: path,
params: {
locator: structUtils.stringifyLocator(parentLocator),
},
}), PROTOCOL, opts.fetchOptions);
const generatorHash = hashUtils.makeHash(`${CACHE_VERSION}`, generatorFile).slice(0, 6);

return execUtils.makeLocator(descriptor, {parentLocator, path, generatorHash, protocol: PROTOCOL});
}
}
60 changes: 25 additions & 35 deletions packages/plugin-file/sources/FileResolver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,17 +45,35 @@ export class FileResolver implements Resolver {
}

async getCandidates(descriptor: Descriptor, dependencies: unknown, opts: ResolveOptions) {
const locator = await this.getCandidateForDescriptor(descriptor, opts);
if (!opts.fetchOptions)
throw new Error(`Assertion failed: This resolver cannot be used unless a fetcher is configured`);

const {path, parentLocator} = fileUtils.parseSpec(descriptor.range);

if (parentLocator === null)
throw new Error(`Assertion failed: The descriptor should have been bound`);

return [locator];
const archiveBuffer = await fileUtils.makeBufferFromLocator(
structUtils.makeLocator(descriptor,
structUtils.makeRange({
protocol: PROTOCOL,
source: path,
selector: path,
params: {
locator: structUtils.stringifyLocator(parentLocator),
},
})
),
{protocol: PROTOCOL, fetchOptions: opts.fetchOptions}
);

const folderHash = hashUtils.makeHash(`${CACHE_VERSION}`, archiveBuffer).slice(0, 6);

return [fileUtils.makeLocator(descriptor, {parentLocator, path, folderHash, protocol: PROTOCOL})];
}

async getSatisfying(descriptor: Descriptor, references: Array<string>, dependencies: Map<DescriptorHash, Package>, opts: ResolveOptions) {
const {reference: fileReference} = await this.getCandidateForDescriptor(descriptor, opts);

return references
.filter(reference => reference === fileReference)
.map(reference => structUtils.makeLocator(descriptor, reference));
return null;
}

async resolve(locator: Locator, opts: ResolveOptions) {
Expand Down Expand Up @@ -85,32 +103,4 @@ export class FileResolver implements Resolver {
bin: manifest.bin,
};
}

private async getCandidateForDescriptor(descriptor: Descriptor, opts: ResolveOptions) {
if (!opts.fetchOptions)
throw new Error(`Assertion failed: This resolver cannot be used unless a fetcher is configured`);

const {path, parentLocator} = fileUtils.parseSpec(descriptor.range);

if (parentLocator === null)
throw new Error(`Assertion failed: The descriptor should have been bound`);

const archiveBuffer = await fileUtils.makeBufferFromLocator(
structUtils.makeLocator(descriptor,
structUtils.makeRange({
protocol: PROTOCOL,
source: path,
selector: path,
params: {
locator: structUtils.stringifyLocator(parentLocator),
},
})
),
{protocol: PROTOCOL, fetchOptions: opts.fetchOptions}
);

const folderHash = hashUtils.makeHash(`${CACHE_VERSION}`, archiveBuffer).slice(0, 6);

return fileUtils.makeLocator(descriptor, {parentLocator, path, folderHash, protocol: PROTOCOL});
}
}
22 changes: 6 additions & 16 deletions packages/plugin-file/sources/TarballFileResolver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,17 +48,16 @@ export class TarballFileResolver implements Resolver {
}

async getCandidates(descriptor: Descriptor, dependencies: unknown, opts: ResolveOptions) {
const locator = await this.getCandidateForDescriptor(descriptor, opts);
let path = descriptor.range;

if (path.startsWith(PROTOCOL))
path = path.slice(PROTOCOL.length);

return [locator];
return [structUtils.makeLocator(descriptor, `${PROTOCOL}${npath.toPortablePath(path)}`)];
}

async getSatisfying(descriptor: Descriptor, references: Array<string>, dependencies: Map<DescriptorHash, Package>, opts: ResolveOptions) {
const {reference: tarballFileReference} = await this.getCandidateForDescriptor(descriptor, opts);

return references
.filter(reference => reference === tarballFileReference)
.map(reference => structUtils.makeLocator(descriptor, reference));
return null;
}

async resolve(locator: Locator, opts: ResolveOptions) {
Expand Down Expand Up @@ -88,13 +87,4 @@ export class TarballFileResolver implements Resolver {
bin: manifest.bin,
};
}

private async getCandidateForDescriptor(descriptor: Descriptor, opts: ResolveOptions) {
let path = descriptor.range;

if (path.startsWith(PROTOCOL))
path = path.slice(PROTOCOL.length);

return structUtils.makeLocator(descriptor, `${PROTOCOL}${npath.toPortablePath(path)}`);
}
}
16 changes: 3 additions & 13 deletions packages/plugin-git/sources/GitResolver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,17 +27,14 @@ export class GitResolver implements Resolver {
}

async getCandidates(descriptor: Descriptor, dependencies: unknown, opts: ResolveOptions) {
const locator = await this.getCandidateForDescriptor(descriptor, opts);
const reference = await gitUtils.resolveUrl(descriptor.range, opts.project.configuration);
const locator = structUtils.makeLocator(descriptor, reference);

return [locator];
}

async getSatisfying(descriptor: Descriptor, references: Array<string>, dependencies: Map<DescriptorHash, Package>, opts: ResolveOptions) {
const {reference: gitReference} = await this.getCandidateForDescriptor(descriptor, opts);

return references
.filter(reference => reference === gitReference)
.map(reference => structUtils.makeLocator(descriptor, reference));
return null;
}

async resolve(locator: Locator, opts: ResolveOptions) {
Expand Down Expand Up @@ -67,11 +64,4 @@ export class GitResolver implements Resolver {
bin: manifest.bin,
};
}

private async getCandidateForDescriptor(descriptor: Descriptor, opts: ResolveOptions) {
const reference = await gitUtils.resolveUrl(descriptor.range, opts.project.configuration);
const locator = structUtils.makeLocator(descriptor, reference);

return locator;
}
}
6 changes: 1 addition & 5 deletions packages/plugin-http/sources/TarballHttpResolver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,7 @@ export class TarballHttpResolver implements Resolver {
}

async getSatisfying(descriptor: Descriptor, references: Array<string>, dependencies: Map<DescriptorHash, Package>, opts: ResolveOptions) {
const tarballReference = structUtils.convertDescriptorToLocator(descriptor).reference;

return references
.filter(reference => reference === tarballReference)
.map(reference => structUtils.makeLocator(descriptor, reference));
return null;
}

async resolve(locator: Locator, opts: ResolveOptions) {
Expand Down
16 changes: 3 additions & 13 deletions packages/plugin-link/sources/LinkResolver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,17 +36,13 @@ export class LinkResolver implements Resolver {
}

async getCandidates(descriptor: Descriptor, dependencies: unknown, opts: ResolveOptions) {
const locator = await this.getCandidateForDescriptor(descriptor, opts);
const path = descriptor.range.slice(LINK_PROTOCOL.length);

return [locator];
return [structUtils.makeLocator(descriptor, `${LINK_PROTOCOL}${npath.toPortablePath(path)}`)];
}

async getSatisfying(descriptor: Descriptor, references: Array<string>, dependencies: Map<DescriptorHash, Package>, opts: ResolveOptions) {
const {reference: linkReference} = await this.getCandidateForDescriptor(descriptor, opts);

return references
.filter(reference => reference === linkReference)
.map(reference => structUtils.makeLocator(descriptor, reference));
return null;
}

async resolve(locator: Locator, opts: ResolveOptions): Promise<Package> {
Expand Down Expand Up @@ -76,10 +72,4 @@ export class LinkResolver implements Resolver {
bin: manifest.bin,
};
}

private async getCandidateForDescriptor(descriptor: Descriptor, opts: ResolveOptions) {
const path = descriptor.range.slice(LINK_PROTOCOL.length);

return structUtils.makeLocator(descriptor, `${LINK_PROTOCOL}${npath.toPortablePath(path)}`);
}
}
16 changes: 3 additions & 13 deletions packages/plugin-link/sources/RawLinkResolver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,17 +36,13 @@ export class RawLinkResolver implements Resolver {
}

async getCandidates(descriptor: Descriptor, dependencies: unknown, opts: ResolveOptions) {
const locator = await this.getCandidateForDescriptor(descriptor, opts);
const path = descriptor.range.slice(RAW_LINK_PROTOCOL.length);

return [locator];
return [structUtils.makeLocator(descriptor, `${RAW_LINK_PROTOCOL}${npath.toPortablePath(path)}`)];
}

async getSatisfying(descriptor: Descriptor, references: Array<string>, dependencies: Map<DescriptorHash, Package>, opts: ResolveOptions) {
const {reference: rawLinkReference} = await this.getCandidateForDescriptor(descriptor, opts);

return references
.filter(reference => reference === rawLinkReference)
.map(reference => structUtils.makeLocator(descriptor, reference));
return null;
}

async resolve(locator: Locator, opts: ResolveOptions) {
Expand All @@ -67,10 +63,4 @@ export class RawLinkResolver implements Resolver {
bin: new Map(),
};
}

private async getCandidateForDescriptor(descriptor: Descriptor, opts: ResolveOptions) {
const path = descriptor.range.slice(RAW_LINK_PROTOCOL.length);

return structUtils.makeLocator(descriptor, `${RAW_LINK_PROTOCOL}${npath.toPortablePath(path)}`);
}
}
2 changes: 1 addition & 1 deletion packages/plugin-npm/sources/NpmTagResolver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ export class NpmTagResolver implements Resolver {

async getSatisfying(descriptor: Descriptor, references: Array<string>, dependencies: Map<DescriptorHash, Package>, opts: ResolveOptions) {
// We can't statically know if a tag resolves to a specific version without using the network
return [];
return null;
}

async resolve(locator: Locator, opts: ResolveOptions): Promise<Package> {
Expand Down
34 changes: 12 additions & 22 deletions packages/plugin-patch/sources/PatchResolver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,27 +46,6 @@ export class PatchResolver implements Resolver {
}

async getCandidates(descriptor: Descriptor, dependencies: Map<DescriptorHash, Package>, opts: ResolveOptions) {
const locator = await this.getCandidateForDescriptor(descriptor, dependencies, opts);

return [locator];
}

async getSatisfying(descriptor: Descriptor, references: Array<string>, dependencies: Map<DescriptorHash, Package>, opts: ResolveOptions) {
const {reference: patchReference} = await this.getCandidateForDescriptor(descriptor, dependencies, opts);

return references
.filter(reference => reference === patchReference)
.map(reference => structUtils.makeLocator(descriptor, reference));
}

async resolve(locator: Locator, opts: ResolveOptions): Promise<Package> {
const {sourceLocator} = patchUtils.parseLocator(locator);
const sourcePkg = await opts.resolver.resolve(sourceLocator, opts);

return {...sourcePkg, ...locator};
}

private async getCandidateForDescriptor(descriptor: Descriptor, dependencies: Map<DescriptorHash, Package>, opts: ResolveOptions) {
if (!opts.fetchOptions)
throw new Error(`Assertion failed: This resolver cannot be used unless a fetcher is configured`);

Expand All @@ -79,6 +58,17 @@ export class PatchResolver implements Resolver {

const patchHash = hashUtils.makeHash(`${CACHE_VERSION}`, ...patchFiles).slice(0, 6);

return patchUtils.makeLocator(descriptor, {parentLocator, sourcePackage, patchPaths, patchHash});
return [patchUtils.makeLocator(descriptor, {parentLocator, sourcePackage, patchPaths, patchHash})];
}

async getSatisfying(descriptor: Descriptor, references: Array<string>, dependencies: Map<DescriptorHash, Package>, opts: ResolveOptions) {
return null;
}

async resolve(locator: Locator, opts: ResolveOptions): Promise<Package> {
const {sourceLocator} = patchUtils.parseLocator(locator);
const sourcePkg = await opts.resolver.resolve(sourceLocator, opts);

return {...sourcePkg, ...locator};
}
}
11 changes: 1 addition & 10 deletions packages/yarnpkg-core/sources/LegacyMigrationResolver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -116,16 +116,7 @@ export class LegacyMigrationResolver implements Resolver {
}

async getSatisfying(descriptor: Descriptor, references: Array<string>, dependencies: Map<DescriptorHash, Package>, opts: ResolveOptions) {
if (!this.resolutions)
throw new Error(`Assertion failed: The resolution store should have been setup`);

const resolution = this.resolutions.get(descriptor.descriptorHash);
if (!resolution)
throw new Error(`Assertion failed: The resolution should have been registered`);

return references
.filter(reference => reference === resolution.reference)
.map(reference => structUtils.makeLocator(descriptor, reference));
return null;
}

async resolve(locator: Locator, opts: ResolveOptions): Promise<never> {
Expand Down
14 changes: 1 addition & 13 deletions packages/yarnpkg-core/sources/LockfileResolver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,19 +52,7 @@ export class LockfileResolver implements Resolver {
}

async getSatisfying(descriptor: Descriptor, references: Array<string>, dependencies: Map<DescriptorHash, Package>, opts: ResolveOptions) {
const convertedPkg = opts.project.originalPackages.get(structUtils.convertDescriptorToLocator(descriptor).locatorHash);

const resolution = opts.project.storedResolutions.get(descriptor.descriptorHash);
const resolutionPkg = typeof resolution !== `undefined`
? opts.project.originalPackages.get(resolution)
: undefined;

return references
.filter(reference => [
convertedPkg?.reference,
resolutionPkg?.reference,
].includes(reference))
.map(reference => structUtils.makeLocator(descriptor, reference));
return null;
}

async resolve(locator: Locator, opts: ResolveOptions) {
Expand Down
5 changes: 4 additions & 1 deletion packages/yarnpkg-core/sources/Resolver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -138,12 +138,15 @@ export interface Resolver {
* locators are first. This will cause the resolution algorithm to prioritize
* them if possible (it doesn't guarantee that they'll end up being used).
*
* If the operation is unsupported by the resolver (i.e. if it can't be statically
* determined which references satisfy the target descriptor), `null` should be returned.
*
* @param descriptor The target descriptor.
* @param references The candidate references.
* @param dependencies The resolution dependencies and their resolutions.
* @param opts The resolution options.
*/
getSatisfying(descriptor: Descriptor, references: Array<string>, dependencies: Map<DescriptorHash, Package>, opts: ResolveOptions): Promise<Array<Locator>>;
getSatisfying(descriptor: Descriptor, references: Array<string>, dependencies: Map<DescriptorHash, Package>, opts: ResolveOptions): Promise<Array<Locator> | null>;

/**
* This function will, given a locator, return the full package definition
Expand Down
Loading