Implements code splitting

packages/acceptance-tests/pkg-tests-core/sources/utils/fs.ts

@@ -196,7 +196,7 @@ export const readSyml = async (source: PortablePath): Promise<any> => {
   const fileContent = await exports.readFile(source, `utf8`);
 
   try {
-    return parseSyml(fileContent);
+    return await parseSyml(fileContent);
   } catch (error) {
     throw new Error(`Invalid syml file (${source})`);
   }

packages/acceptance-tests/pkg-tests-specs/sources/commands/add.test.ts

@@ -281,7 +281,7 @@ describe(`Commands`, () => {
         await run(`add`, `inject-node-gyp`);
 
         const content = await xfs.readFilePromise(`${path}/yarn.lock` as PortablePath, `utf8`);
-        const lock = parseSyml(content);
+        const lock = await parseSyml(content);
 
         await expect(lock).toMatchObject({
           [`inject-node-gyp@npm:^1.0.0`]: {

packages/plugin-compat/sources/index.ts

@@ -2,14 +2,11 @@ import {Hooks as CoreHooks, Plugin, structUtils} from '@yarnpkg/core';
 import {Hooks as PatchHooks}                     from '@yarnpkg/plugin-patch';
 
 import {packageExtensions}                       from './extensions';
-import {getPatch as getFseventsPatch}            from './patches/fsevents.patch';
-import {getPatch as getResolvePatch}             from './patches/resolve.patch';
-import {getPatch as getTypescriptPatch}          from './patches/typescript.patch';
 
 const PATCHES = new Map([
-  [structUtils.makeIdent(null, `fsevents`).identHash, getFseventsPatch],
-  [structUtils.makeIdent(null, `resolve`).identHash, getResolvePatch],
-  [structUtils.makeIdent(null, `typescript`).identHash, getTypescriptPatch],
+  [structUtils.makeIdent(null, `fsevents`).identHash, () => import(`./patches/fsevents.patch`)],
+  [structUtils.makeIdent(null, `resolve`).identHash, () => import(`./patches/resolve.patch`)],
+  [structUtils.makeIdent(null, `typescript`).identHash, () => import(`./patches/typescript.patch`)],
 ]);
 
 const plugin: Plugin<CoreHooks & PatchHooks> = {
@@ -26,7 +23,7 @@ const plugin: Plugin<CoreHooks & PatchHooks> = {
         return undefined;
 
       const ident = structUtils.parseIdent(name.slice(TAG.length));
-      const patch = PATCHES.get(ident.identHash)?.();
+      const patch = (await PATCHES.get(ident.identHash)?.())?.getPatch();
 
       return typeof patch !== `undefined` ? patch : null;
     },

packages/plugin-essentials/sources/commands/install.ts

@@ -351,8 +351,8 @@ async function autofixMergeConflicts(configuration: Configuration, immutable: bo
   let parsedLeft;
   let parsedRight;
   try {
-    parsedLeft = parseSyml(left);
-    parsedRight = parseSyml(right);
+    parsedLeft = await parseSyml(left);
+    parsedRight = await parseSyml(right);
   } catch (error) {
     throw new ReportError(MessageName.AUTOMERGE_FAILED_TO_PARSE, `The individual variants of the lockfile failed to parse`);
   }

packages/plugin-essentials/sources/commands/plugin/list.ts

@@ -7,7 +7,7 @@ const REMOTE_REGISTRY = `https://raw.githubusercontent.com/yarnpkg/berry/master/
 
 export async function getAvailablePlugins(configuration: Configuration) {
   const raw = await httpUtils.get(REMOTE_REGISTRY, {configuration});
-  const data = parseSyml(raw.toString());
+  const data = await parseSyml(raw.toString());
 
   return data;
 }

packages/plugin-nm/sources/NodeModulesLinker.ts

@@ -457,7 +457,7 @@ async function findInstallState(project: Project, {unrollAliases = false}: {unro
   if (!xfs.existsSync(installStatePath))
     return null;
 
-  const locatorState = parseSyml(await xfs.readFilePromise(installStatePath, `utf8`));
+  const locatorState = await parseSyml(await xfs.readFilePromise(installStatePath, `utf8`));
 
   // If we have a higher serialized version than we can handle, ignore the state alltogether
   if (locatorState.__metadata.version > STATE_FILE_VERSION)

packages/plugin-nm/sources/PnpLooseLinker.ts

@@ -3,7 +3,7 @@ import {VirtualFS, ZipOpenFS, ppath, Filename}         from '@yarnpkg/fslib';
 import {getLibzipPromise}                              from '@yarnpkg/libzip';
 import {NodeModulesPackageNode, buildNodeModulesTree}  from '@yarnpkg/nm';
 import {PnpInstaller, PnpLinker}                       from '@yarnpkg/plugin-pnp';
-import {PnpSettings, makeRuntimeApi, DependencyTarget} from '@yarnpkg/pnp';
+import {PnpSettings, DependencyTarget} from '@yarnpkg/pnp';
 
 export class PnpLooseLinker extends PnpLinker {
   protected mode = `loose`;
@@ -25,7 +25,9 @@ class PnpLooseInstaller extends PnpInstaller {
       }),
     });
 
+    const {makeRuntimeApi} = await import(`@yarnpkg/pnp`);
     const pnp = makeRuntimeApi(pnpSettings, this.opts.project.cwd, defaultFsLayer);
+
     const {tree, errors} = buildNodeModulesTree(pnp, {pnpifyFs: false, project: this.opts.project});
     if (!tree) {
       for (const {messageName, text} of errors)

packages/plugin-pnp/sources/PnpLinker.ts

@@ -2,7 +2,7 @@ import {miscUtils, structUtils, formatUtils, Descriptor, LocatorHash, InstallPac
 import {FetchResult, Locator, Package}                                                                          from '@yarnpkg/core';
 import {Linker, LinkOptions, MinimalLinkOptions, Manifest, MessageName, DependencyMeta, LinkType, Installer}    from '@yarnpkg/core';
 import {AliasFS, CwdFS, PortablePath, VirtualFS, npath, ppath, xfs, Filename}                                   from '@yarnpkg/fslib';
-import {generateInlinedScript, generateSplitScript, PackageRegistry, PnpApi, PnpSettings, getESMLoaderTemplate} from '@yarnpkg/pnp';
+import {PackageRegistry, PnpApi, PnpSettings} from '@yarnpkg/pnp';
 import {UsageError}                                                                                             from 'clipanion';
 
 import {getPnpPath}                                                                                             from './index';
@@ -337,6 +337,12 @@ export class PnpInstaller implements Installer {
 
     await this.transformPnpSettings(pnpSettings);
 
+    const {
+      generateInlinedScript,
+      generateSplitScript,
+      getESMLoaderTemplate
+    } = await import('@yarnpkg/pnp');
+
     if (this.opts.project.configuration.get(`pnpEnableInlining`)) {
       const loaderFile = generateInlinedScript(pnpSettings);
 

packages/plugin-version/sources/versionUtils.ts

@@ -82,7 +82,7 @@ export async function resolveVersionFiles(project: Project, {prerelease = null}:
 
     const versionPath = ppath.join(deferredVersionFolder, entry);
     const versionContent = await xfs.readFilePromise(versionPath, `utf8`);
-    const versionData = parseSyml(versionContent);
+    const versionData = await parseSyml(versionContent);
 
     for (const [identStr, decision] of Object.entries(versionData.releases || {})) {
       if (decision === Decision.DECLINE)
@@ -146,7 +146,7 @@ export async function updateVersionFiles(project: Project) {
 
     const versionPath = ppath.join(deferredVersionFolder, entry);
     const versionContent = await xfs.readFilePromise(versionPath, `utf8`);
-    const versionData = parseSyml(versionContent);
+    const versionData = await parseSyml(versionContent);
 
     const releases = versionData?.releases;
     if (!releases)
@@ -211,7 +211,7 @@ export async function openVersionFile(project: Project, {allowEmpty = false}: {a
     ? await xfs.readFilePromise(versionPath, `utf8`)
     : `{}`;
 
-  const versionData = parseSyml(versionContent);
+  const versionData = await parseSyml(versionContent);
   const releaseStore: Releases = new Map();
 
   for (const identStr of versionData.declined || []) {

packages/yarnpkg-builder/package.json

@@ -12,7 +12,8 @@
     "clipanion": "^3.2.0-rc.4",
     "esbuild": "npm:esbuild-wasm@^0.11.20",
     "semver": "^7.1.2",
-    "tslib": "^1.13.0"
+    "tslib": "^1.13.0",
+    "webpack": "^5.70.0"
   },
   "devDependencies": {
     "@types/semver": "^7.1.0",

packages/yarnpkg-builder/sources/commands/build/bundle.ts

@@ -1,7 +1,7 @@
 import {getDynamicLibs}                                                     from '@yarnpkg/cli';
-import {StreamReport, MessageName, Configuration, formatUtils, structUtils} from '@yarnpkg/core';
+import {StreamReport, MessageName, Configuration, formatUtils, structUtils, hashUtils} from '@yarnpkg/core';
 import {pnpPlugin}                                                          from '@yarnpkg/esbuild-plugin-pnp';
-import {npath}                                                              from '@yarnpkg/fslib';
+import {Filename, npath, PortablePath, ppath, xfs}                                                         from '@yarnpkg/fslib';
 import chalk                                                                from 'chalk';
 import cp                                                                   from 'child_process';
 import {Command, Option, Usage}                                             from 'clipanion';
@@ -11,6 +11,7 @@ import {createRequire}                                                      from
 import path                                                                 from 'path';
 import semver                                                               from 'semver';
 import {promisify}                                                          from 'util';
+import {webpack}                                                            from 'webpack';
 
 import {findPlugins}                                                        from '../../tools/findPlugins';
 
@@ -59,6 +60,10 @@ export default class BuildBundleCommand extends Command {
     description: `An array of plugins that should be included besides the ones specified in the profile`,
   });
 
+  split = Option.Boolean(`--split`, false, {
+    description: `Build a bundle that splits the application then unifies it into a self-extracting packaged app`,
+  });
+
   noGitHash = Option.Boolean(`--no-git-hash`, false, {
     description: `Don't include the git hash of the current commit in bundle version`,
   });
@@ -114,38 +119,128 @@ export default class BuildBundleCommand extends Command {
           },
         };
 
-        const res = await build({
-          banner: {
-            js: `#!/usr/bin/env node\n/* eslint-disable */\n//prettier-ignore`,
-          },
-          entryPoints: [path.join(basedir, `sources/cli.ts`)],
-          bundle: true,
-          define: {YARN_VERSION: JSON.stringify(version)},
-          outfile: output,
-          logLevel: `silent`,
-          format: `iife`,
-          platform: `node`,
-          plugins: [valLoader, pnpPlugin()],
-          minify: !this.noMinify,
-          sourcemap: this.sourceMap ? `inline` : false,
-          target: `node14`,
+        await xfs.mktempPromise(async p => {
+          xfs.detachTemp(p);
+
+          const esmDir = ppath.join(p, `esm` as Filename);
+          const cjsDir = ppath.join(p, `cjs` as Filename);
+
+          console.log(p);
+
+          const res = await build({
+            banner: {
+              js: `/* eslint-disable */\n//prettier-ignore\nimport {createRequire} from 'module';\nconst require = createRequire(import.meta.url);`,
+            },
+            entryPoints: [path.join(basedir, `sources/cli.ts`)],
+            bundle: true,
+            splitting: this.split,
+            define: {YARN_VERSION: JSON.stringify(version)},
+            logLevel: `silent`,
+            format: this.split ? `esm` : `iife`,
+            platform: `node`,
+            plugins: [valLoader, pnpPlugin()],
+            minify: !this.noMinify,
+            sourcemap: this.sourceMap ? `inline` : false,
+            target: `node14`,
+            outExtension: {[`.js`]: '.mjs'},
+            ...this.split ? {
+              outdir: npath.fromPortablePath(esmDir),
+            } : {
+              outfile: output,
+            }
+          });
+
+          if (this.split) {
+            /*
+            await new Promise((resolve, reject) => {
+              webpack({
+                mode: 'production',
+                target: 'node',
+                bail: true,
+                cache: false,
+                entry: {
+                  cli: ppath.join(esmDir, `cli.js` as Filename),
+                },
+                output: {
+                  path: npath.fromPortablePath(cjsDir),
+                },
+                optimization: {
+                  splitChunks: {
+                    chunks: 'async',
+                    minSize: 1,
+                    minRemainingSize: 0,
+                    minChunks: 1,
+                    maxAsyncRequests: 4000,
+                    maxInitialRequests: 4000,
+                    enforceSizeThreshold: 10000
+                  },
+                },
+              }, (err, stats) => {
+                if (err) {
+                    reject(err);
+                } else {
+                    resolve(stats);
+                }
+              });
+            });
+            */
+
+            const chunks = await xfs.readdirPromise(esmDir);
+
+            let payload = ``;
+
+            const packFiles = async () => {
+              return await Promise.all(chunks.map(async name => {
+                return [name, await xfs.readFilePromise(ppath.join(esmDir, name), `base64`)];
+              }));
+            };
+
+            payload += `const fs = require('fs');\n`;
+            payload += `const os = require('os');\n`;
+            payload += `const path = require('path');\n`;
+            payload += `\n`;
+            payload += `const unpackDir = path.join(os.homedir(), '.yarn/berry/chunks', hash);\n`;
+            payload += `const flagFile = path.join(unpackDir, '.ready');\n`;
+            payload += `const entryFile = path.join(unpackDir, 'cli.mjs');\n`;
+            payload += `\n`;
+            payload += `if (!fs.existsSync(flagFile)) {\n`;
+            payload += `  const files = JSON.parse(${JSON.stringify(JSON.stringify(await packFiles()))});\n`;
+            payload += `\n`;
+            payload += `  fs.rmSync(unpackDir, {force: true, recursive: true});\n`;
+            payload += `  fs.mkdirSync(unpackDir, {recursive: true});\n`;
+            payload += `\n`;
+            payload += `  Promise.all(files.map(i => fs.promises.writeFile(path.join(unpackDir, i[0]), i[1], 'base64'))).then(() => {\n`;
+            payload += `    fs.writeFileSync(flagFile, '');\n`;
+            payload += `    import(entryFile);\n`;
+            payload += `  });\n`;
+            payload += `} else {\n`;
+            payload += `  import(entryFile);\n`;
+            payload += `}\n`;
+
+            const hash = hashUtils.makeHash(payload);
+            payload = `const hash = ${JSON.stringify(hash)};\n\n${payload}`;
+
+            await xfs.writeFilePromise(output as PortablePath, payload);
+
+            report.reportWarning(MessageName.UNNAMED, `Note: The split bundle hash is ${hash}`);
+          }
+
+          for (const warning of res.warnings) {
+            if (warning.location !== null)
+              continue;
+
+            report.reportWarning(MessageName.UNNAMED, warning.text);
+          }
+
+
+          for (const warning of res.warnings) {
+            if (warning.location === null)
+              continue;
+
+            report.reportWarning(MessageName.UNNAMED, `${warning.location.file}:${warning.location.line}:${warning.location.column}`);
+            report.reportWarning(MessageName.UNNAMED, `   ↳ ${warning.text}`);
+          }
         });
-
-        for (const warning of res.warnings) {
-          if (warning.location !== null)
-            continue;
-
-          report.reportWarning(MessageName.UNNAMED, warning.text);
-        }
-
-
-        for (const warning of res.warnings) {
-          if (warning.location === null)
-            continue;
-
-          report.reportWarning(MessageName.UNNAMED, `${warning.location.file}:${warning.location.line}:${warning.location.column}`);
-          report.reportWarning(MessageName.UNNAMED, `   ↳ ${warning.text}`);
-        }
       });
     });
 

packages/yarnpkg-core/sources/Configuration.ts

@@ -1159,7 +1159,7 @@ export class Configuration {
 
         let data;
         try {
-          data = parseSyml(content) as any;
+          data = await parseSyml(content) as any;
         } catch (error) {
           let tip = ``;
 
@@ -1186,7 +1186,7 @@ export class Configuration {
 
     if (xfs.existsSync(homeRcFilePath)) {
       const content = await xfs.readFilePromise(homeRcFilePath, `utf8`);
-      const data = parseSyml(content) as any;
+      const data = await parseSyml(content) as any;
 
       return {path: homeRcFilePath, cwd: homeFolder, data};
     }
@@ -1228,7 +1228,7 @@ export class Configuration {
     const configurationPath =  ppath.join(cwd, rcFilename as PortablePath);
 
     const current = xfs.existsSync(configurationPath)
-      ? parseSyml(await xfs.readFilePromise(configurationPath, `utf8`)) as any
+      ? await parseSyml(await xfs.readFilePromise(configurationPath, `utf8`)) as any
       : {};
 
     let patched = false;

packages/yarnpkg-core/sources/LegacyMigrationResolver.ts

@@ -44,7 +44,7 @@ export class LegacyMigrationResolver implements Resolver {
       return;
 
     const content = await xfs.readFilePromise(lockfilePath, `utf8`);
-    const parsed = parseSyml(content);
+    const parsed = await parseSyml(content);
 
     // No need to enable it either if the lockfile is modern
     if (Object.prototype.hasOwnProperty.call(parsed, `__metadata`))

packages/yarnpkg-core/sources/Project.ts

@@ -306,7 +306,7 @@ export class Project {
       // We store the salted checksum of the lockfile in order to invalidate the install state when needed
       this.lockFileChecksum = makeLockfileChecksum(content);
 
-      const parsed: any = parseSyml(content);
+      const parsed: any = await parseSyml(content);
 
       // Protects against v1 lockfiles
       if (parsed.__metadata) {