Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
79f42d3
feat(plugin-essentials): yarn deduplicate
paul-soporan Jul 8, 2020
f1ce7d2
refactor: improve implementation
paul-soporan Jul 8, 2020
05781f1
Merge branch 'master' into paul/feat/deduplicate
paul-soporan Aug 17, 2020
4303e0d
refactor: shorten command name to `yarn dedupe`
paul-soporan Aug 17, 2020
408acbf
refactor: rewrite algorithm
paul-soporan Aug 17, 2020
7063670
refactor: more refactoring
paul-soporan Aug 17, 2020
d5a2291
refactor: even more refactoring
paul-soporan Aug 17, 2020
925112a
refactor: more refactoring + improved docs
paul-soporan Aug 19, 2020
ebfd0c1
docs: better dedupe docs
paul-soporan Aug 19, 2020
e8df62b
test: add tests
paul-soporan Aug 19, 2020
658055c
test: add tests for selective dedupe
paul-soporan Aug 19, 2020
ea4cc55
Merge remote-tracking branch 'yarnpkg/master' into paul/feat/deduplicate
paul-soporan Aug 19, 2020
bb60e25
refactor: refactor everything yet again
paul-soporan Aug 20, 2020
cc91662
refactor: fix cyclic dependency
paul-soporan Aug 20, 2020
51b342e
Update packages/plugin-essentials/sources/commands/dedupe.ts
paul-soporan Aug 22, 2020
4379dbe
Update packages/plugin-essentials/sources/commands/dedupe.ts
paul-soporan Aug 22, 2020
d637d8e
refactor: dedupeSkip -> null
paul-soporan Aug 22, 2020
1a705f5
refactor: skip -> assertion
paul-soporan Aug 22, 2020
cab447e
refactor: add note about virtual packages
paul-soporan Aug 22, 2020
d6b9a58
refactor: skip ->assertion failed
paul-soporan Aug 22, 2020
58ac353
Merge branch 'master' into paul/feat/deduplicate
paul-soporan Aug 22, 2020
9eb67d8
refactor: disable getSatisfying for some resolvers
paul-soporan Aug 23, 2020
aaf702f
refactor: remove resolutionDependencies
paul-soporan Aug 23, 2020
2465986
chore: lint
paul-soporan Aug 23, 2020
1fc26fa
refactor: skip -> assertion failed
paul-soporan Aug 23, 2020
d05b284
Apply suggestions from code review
paul-soporan Aug 23, 2020
50f0c51
refactor: use semver.SemVer instances
paul-soporan Aug 23, 2020
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
refactor: more refactoring
  • Loading branch information
paul-soporan committed Aug 17, 2020
commit 7063670e051302653562b47268af5ec3166c08ff
3 changes: 3 additions & 0 deletions .pnp.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 2 additions & 10 deletions .yarn/versions/e9b298c7.yml
Original file line number Diff line number Diff line change
@@ -1,22 +1,14 @@
releases:
"@yarnpkg/cli": prerelease
"@yarnpkg/core": prerelease
"@yarnpkg/plugin-essentials": prerelease
"@yarnpkg/plugin-essentials": minor

declined:
- "@yarnpkg/plugin-compat"
- "@yarnpkg/plugin-constraints"
- "@yarnpkg/plugin-dlx"
- "@yarnpkg/plugin-exec"
- "@yarnpkg/plugin-file"
- "@yarnpkg/plugin-git"
- "@yarnpkg/plugin-github"
- "@yarnpkg/plugin-http"
- "@yarnpkg/plugin-init"
- "@yarnpkg/plugin-interactive-tools"
- "@yarnpkg/plugin-link"
- "@yarnpkg/plugin-node-modules"
- "@yarnpkg/plugin-npm"
- "@yarnpkg/plugin-npm-cli"
- "@yarnpkg/plugin-pack"
- "@yarnpkg/plugin-patch"
Expand All @@ -26,5 +18,5 @@ declined:
- "@yarnpkg/plugin-version"
- "@yarnpkg/plugin-workspace-tools"
- "@yarnpkg/builder"
- "@yarnpkg/core"
- "@yarnpkg/doctor"
- "@yarnpkg/pnpify"
1 change: 1 addition & 0 deletions packages/plugin-essentials/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
"@types/micromatch": "^4.0.1",
"@types/semver": "^7.1.0",
"@types/treeify": "^1.0.0",
"@types/yup": "0.26.12",
"@yarnpkg/cli": "workspace:^2.1.1",
"@yarnpkg/core": "workspace:^2.1.1"
},
Expand Down
83 changes: 65 additions & 18 deletions packages/plugin-essentials/sources/commands/dedupe.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,12 @@
* - https://github.com/eps1lon/yarn-plugin-deduplicate
*/

import {BaseCommand} from '@yarnpkg/cli';
import {Configuration, Project, ResolveOptions, ThrowReport, Cache, StreamReport, Resolver, miscUtils, Descriptor, Package} from '@yarnpkg/core';
import {structUtils, IdentHash, LocatorHash, MessageName, Report, Fetcher, FetchOptions} from '@yarnpkg/core';
import {Command} from 'clipanion';
import micromatch from 'micromatch';
import {BaseCommand} from '@yarnpkg/cli';
import {Configuration, Project, ResolveOptions, ThrowReport, Cache, StreamReport, Resolver, miscUtils, Descriptor, Package, FormatType} from '@yarnpkg/core';
import {structUtils, IdentHash, LocatorHash, MessageName, Report, Fetcher, FetchOptions} from '@yarnpkg/core';
import {Command} from 'clipanion';
import micromatch from 'micromatch';
import * as yup from 'yup';

export const dedupeSkip = Symbol(`dedupeSkip`);

Expand All @@ -23,19 +24,20 @@ export type DedupeAlgorithm = (project: Project, patterns: Array<string>, opts:
resolveOptions: ResolveOptions,
fetcher: Fetcher,
fetchOptions: FetchOptions,
report: Report,
}) => Promise<Array<DedupePromise>>;

export enum Strategy {
Highest = `highest`,
HIGHEST = `highest`,
}

const acceptedStrategies = new Set(Object.values(Strategy));

export const DEDUPE_ALGORITHMS: Record<Strategy, DedupeAlgorithm> = {
highest: async (project, patterns, {resolver, fetcher, resolveOptions, fetchOptions, report}) => {
highest: async (project, patterns, {resolver, fetcher, resolveOptions, fetchOptions}) => {
const locatorsByIdent = new Map<IdentHash, Set<LocatorHash>>();
for (const [descriptorHash, locatorHash] of project.storedResolutions) {
const descriptor = project.storedDescriptors.get(descriptorHash);
if (!descriptor)
if (typeof descriptor === `undefined`)
throw new Error(`Assertion failed: The descriptor (${descriptorHash}) should have been registered`);

miscUtils.getSetWithDefault(locatorsByIdent, descriptor.identHash).add(locatorHash);
Expand Down Expand Up @@ -112,14 +114,25 @@ export default class DedupeCommand extends BaseCommand {
patterns: Array<string> = [];

@Command.String(`-s,--strategy`)
strategy: Strategy = Strategy.Highest;
strategy: Strategy = Strategy.HIGHEST;

@Command.Boolean(`--check`)
check: boolean = false;

@Command.Boolean(`--json`)
json: boolean = false;

static schema = yup.object().shape({
strategy: yup.string().test({
name: `strategy`,
message: `\${path} must be one of \${strategies}`,
params: {strategies: [...acceptedStrategies].join(`, `)},
test: (strategy: string) => {
return acceptedStrategies.has(strategy as Strategy);
},
}),
});

static usage = Command.Usage({
description: `deduplicate dependencies with overlapping ranges`,
details: `
Expand Down Expand Up @@ -154,17 +167,18 @@ export default class DedupeCommand extends BaseCommand {
const {project} = await Project.find(configuration, this.context.cwd);
const cache = await Cache.find(configuration);

const deduplicateReport = await StreamReport.start({
let dedupedPackageCount: number = 0;
await StreamReport.start({
configuration,
includeFooter: false,
stdout: this.context.stdout,
json: this.json,
}, async report => {
await deduplicate(this.strategy, project, this.patterns, {cache, report});
dedupedPackageCount = await dedupe({project, strategy: this.strategy, patterns: this.patterns, cache, report});
});

if (this.check) {
return deduplicateReport.hasWarnings() ? 1 : 0;
return dedupedPackageCount ? 1 : 0;
} else {
const installReport = await StreamReport.start({
configuration,
Expand All @@ -179,7 +193,15 @@ export default class DedupeCommand extends BaseCommand {
}
}

export async function deduplicate(strategy: Strategy, project: Project, patterns: Array<string>, {cache, report}: {cache: Cache, report: Report}) {
export type DedupeSpec = {
strategy: Strategy,
project: Project,
patterns: Array<string>,
cache: Cache,
report: Report,
};

export async function dedupe({strategy, project, patterns, cache, report}: DedupeSpec) {
const {configuration} = project;
const throwReport = new ThrowReport();

Expand All @@ -201,27 +223,31 @@ export async function deduplicate(strategy: Strategy, project: Project, patterns
fetchOptions,
};

await report.startTimerPromise(`Deduplication step`, async () => {
return await report.startTimerPromise(`Deduplication step`, async () => {
const algorithm = DEDUPE_ALGORITHMS[strategy];
const dedupePromises = await algorithm(project, patterns, {resolver, resolveOptions, fetcher, fetchOptions, report});
const dedupePromises = await algorithm(project, patterns, {resolver, resolveOptions, fetcher, fetchOptions});

const progress = StreamReport.progressViaCounter(dedupePromises.length);
report.reportProgress(progress);

let dedupedPackageCount = 0;

await Promise.all(
dedupePromises.map(dedupePromise =>
dedupePromise
.then(dedupe => {
if (dedupe === dedupeSkip)
return;

dedupedPackageCount++;

const {descriptor, currentPackage, updatedPackage} = dedupe;

report.reportWarning(
report.reportInfo(
MessageName.UNNAMED,
`${
structUtils.prettyDescriptor(configuration, descriptor)
} can be deduplicated from ${
} can be deduped from ${
structUtils.prettyLocator(configuration, currentPackage)
} to ${
structUtils.prettyLocator(configuration, updatedPackage)
Expand All @@ -239,5 +265,26 @@ export async function deduplicate(strategy: Strategy, project: Project, patterns
.finally(() => progress.tick())
)
);

const prettyStrategy = configuration.format(strategy, FormatType.CODE);

let packages: string;
switch (dedupedPackageCount) {
case 0: {
packages = `No packages`;
} break;

case 1: {
packages = `One package`;
} break;

default: {
packages = `${dedupedPackageCount} packages`;
}
}

report.reportInfo(MessageName.UNNAMED, `${packages} can be deduped using the strategy ${prettyStrategy}`);

return dedupedPackageCount;
});
}
4 changes: 0 additions & 4 deletions packages/yarnpkg-core/sources/StreamReport.ts
Original file line number Diff line number Diff line change
Expand Up @@ -192,10 +192,6 @@ export class StreamReport extends Report {
this.stdout = stdout;
}

hasWarnings() {
return this.warningCount > 0;
}

hasErrors() {
return this.errorCount > 0;
}
Expand Down
1 change: 1 addition & 0 deletions yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -5715,6 +5715,7 @@ __metadata:
"@types/micromatch": ^4.0.1
"@types/semver": ^7.1.0
"@types/treeify": ^1.0.0
"@types/yup": 0.26.12
"@yarnpkg/cli": "workspace:^2.1.1"
"@yarnpkg/core": "workspace:^2.1.1"
"@yarnpkg/fslib": "workspace:^2.1.0"
Expand Down