esm: populate separate cache for require(esm) in imported CJS

Otherwise if the ESM happens to be cached separately by the ESM loader
before it gets loaded with `require(esm)` from within an imported
CJS file (which uses a re-invented require() with a couple of quirks,
including a separate cache), it won't be able to load the esm properly
from the cache.

PR-URL: #59679
Refs: #59666
Refs: #52697
Reviewed-By: Benjamin Gruenbaum <[email protected]>
Reviewed-By: Chengzhong Wu <[email protected]>

Author: joyeecheung

lib/internal/modules/cjs/loader.js

@@ -69,7 +69,8 @@ const {
     module_export_names_private_symbol,
     module_circular_visited_private_symbol,
     module_export_private_symbol,
-    module_parent_private_symbol,
+    module_first_parent_private_symbol,
+    module_last_parent_private_symbol,
   },
   isInsideNodeModules,
 } = internalBinding('util');
@@ -94,9 +95,13 @@ const kModuleCircularVisited = module_circular_visited_private_symbol;
  */
 const kModuleExport = module_export_private_symbol;
 /**
- * {@link Module} parent module.
+ * {@link Module} The first parent module that loads a module with require().
  */
-const kModuleParent = module_parent_private_symbol;
+const kFirstModuleParent = module_first_parent_private_symbol;
+/**
+ * {@link Module} The last parent module that loads a module with require().
+ */
+const kLastModuleParent = module_last_parent_private_symbol;
 
 const kIsMainSymbol = Symbol('kIsMainSymbol');
 const kIsCachedByESMLoader = Symbol('kIsCachedByESMLoader');
@@ -117,6 +122,7 @@ module.exports = {
   findLongestRegisteredExtension,
   resolveForCJSWithHooks,
   loadSourceForCJSWithHooks: loadSource,
+  populateCJSExportsFromESM,
   wrapSafe,
   wrapModuleLoad,
   kIsMainSymbol,
@@ -326,7 +332,8 @@ function Module(id = '', parent) {
   this.id = id;
   this.path = path.dirname(id);
   setOwnProperty(this, 'exports', {});
-  this[kModuleParent] = parent;
+  this[kFirstModuleParent] = parent;
+  this[kLastModuleParent] = parent;
   updateChildren(parent, this, false);
   this.filename = null;
   this.loaded = false;
@@ -408,7 +415,7 @@ ObjectDefineProperty(BuiltinModule.prototype, 'isPreloading', isPreloadingDesc);
  * @returns {object}
  */
 function getModuleParent() {
-  return this[kModuleParent];
+  return this[kFirstModuleParent];
 }
 
 /**
@@ -418,7 +425,7 @@ function getModuleParent() {
  * @returns {void}
  */
 function setModuleParent(value) {
-  this[kModuleParent] = value;
+  this[kFirstModuleParent] = value;
 }
 
 let debug = debuglog('module', (fn) => {
@@ -998,7 +1005,7 @@ function getExportsForCircularRequire(module) {
   const requiredESM = module[kRequiredModuleSymbol];
   if (requiredESM && requiredESM.getStatus() !== kEvaluated) {
     let message = `Cannot require() ES Module ${module.id} in a cycle.`;
-    const parent = module[kModuleParent];
+    const parent = module[kLastModuleParent];
     if (parent) {
       message += ` (from ${parent.filename})`;
     }
@@ -1279,6 +1286,8 @@ Module._load = function(request, parent, isMain) {
     // load hooks for the module keyed by the (potentially customized) filename.
     module[kURL] = url;
     module[kFormat] = format;
+  } else {
+    module[kLastModuleParent] = parent;
   }
 
   if (parent !== undefined) {
@@ -1398,7 +1407,8 @@ Module._resolveFilename = function(request, parent, isMain, options) {
   const requireStack = [];
   for (let cursor = parent;
     cursor;
-    cursor = cursor[kModuleParent]) {
+    // TODO(joyeecheung): it makes more sense to use kLastModuleParent here.
+    cursor = cursor[kFirstModuleParent]) {
     ArrayPrototypePush(requireStack, cursor.filename || cursor.id);
   }
   let message = `Cannot find module '${request}'`;
@@ -1515,7 +1525,7 @@ function loadESMFromCJS(mod, filename, format, source) {
     // ESM won't be accessible via process.mainModule.
     setOwnProperty(process, 'mainModule', undefined);
   } else {
-    const parent = mod[kModuleParent];
+    const parent = mod[kLastModuleParent];
 
     requireModuleWarningMode ??= getOptionValue('--trace-require-module');
     if (requireModuleWarningMode) {
@@ -1565,54 +1575,66 @@ function loadESMFromCJS(mod, filename, format, source) {
       wrap,
       namespace,
     } = cascadedLoader.importSyncForRequire(mod, filename, source, isMain, parent);
-    // Tooling in the ecosystem have been using the __esModule property to recognize
-    // transpiled ESM in consuming code. For example, a 'log' package written in ESM:
-    //
-    // export default function log(val) { console.log(val); }
-    //
-    // Can be transpiled as:
-    //
-    // exports.__esModule = true;
-    // exports.default = function log(val) { console.log(val); }
-    //
-    // The consuming code may be written like this in ESM:
-    //
-    // import log from 'log'
-    //
-    // Which gets transpiled to:
-    //
-    // const _mod = require('log');
-    // const log = _mod.__esModule ? _mod.default : _mod;
-    //
-    // So to allow transpiled consuming code to recognize require()'d real ESM
-    // as ESM and pick up the default exports, we add a __esModule property by
-    // building a source text module facade for any module that has a default
-    // export and add .__esModule = true to the exports. This maintains the
-    // enumerability of the re-exported names and the live binding of the exports,
-    // without incurring a non-trivial per-access overhead on the exports.
-    //
-    // The source of the facade is defined as a constant per-isolate property
-    // required_module_default_facade_source_string, which looks like this
-    //
-    // export * from 'original';
-    // export { default } from 'original';
-    // export const __esModule = true;
-    //
-    // And the 'original' module request is always resolved by
-    // createRequiredModuleFacade() to `wrap` which is a ModuleWrap wrapping
-    // over the original module.
-
-    // We don't do this to modules that are marked as CJS ESM or that
-    // don't have default exports to avoid the unnecessary overhead.
-    // If __esModule is already defined, we will also skip the extension
-    // to allow users to override it.
-    if (ObjectHasOwn(namespace, 'module.exports')) {
-      mod.exports = namespace['module.exports'];
-    } else if (!ObjectHasOwn(namespace, 'default') || ObjectHasOwn(namespace, '__esModule')) {
-      mod.exports = namespace;
-    } else {
-      mod.exports = createRequiredModuleFacade(wrap);
-    }
+
+    populateCJSExportsFromESM(mod, wrap, namespace);
+  }
+}
+
+/**
+ * Populate the exports of a CJS module entry from an ESM module's namespace object for
+ * require(esm).
+ * @param {Module} mod CJS module instance
+ * @param {ModuleWrap} wrap ESM ModuleWrap instance.
+ * @param {object} namespace The ESM namespace object.
+ */
+function populateCJSExportsFromESM(mod, wrap, namespace) {
+  // Tooling in the ecosystem have been using the __esModule property to recognize
+  // transpiled ESM in consuming code. For example, a 'log' package written in ESM:
+  //
+  // export default function log(val) { console.log(val); }
+  //
+  // Can be transpiled as:
+  //
+  // exports.__esModule = true;
+  // exports.default = function log(val) { console.log(val); }
+  //
+  // The consuming code may be written like this in ESM:
+  //
+  // import log from 'log'
+  //
+  // Which gets transpiled to:
+  //
+  // const _mod = require('log');
+  // const log = _mod.__esModule ? _mod.default : _mod;
+  //
+  // So to allow transpiled consuming code to recognize require()'d real ESM
+  // as ESM and pick up the default exports, we add a __esModule property by
+  // building a source text module facade for any module that has a default
+  // export and add .__esModule = true to the exports. This maintains the
+  // enumerability of the re-exported names and the live binding of the exports,
+  // without incurring a non-trivial per-access overhead on the exports.
+  //
+  // The source of the facade is defined as a constant per-isolate property
+  // required_module_default_facade_source_string, which looks like this
+  //
+  // export * from 'original';
+  // export { default } from 'original';
+  // export const __esModule = true;
+  //
+  // And the 'original' module request is always resolved by
+  // createRequiredModuleFacade() to `wrap` which is a ModuleWrap wrapping
+  // over the original module.
+
+  // We don't do this to modules that are marked as CJS ESM or that
+  // don't have default exports to avoid the unnecessary overhead.
+  // If __esModule is already defined, we will also skip the extension
+  // to allow users to override it.
+  if (ObjectHasOwn(namespace, 'module.exports')) {
+    mod.exports = namespace['module.exports'];
+  } else if (!ObjectHasOwn(namespace, 'default') || ObjectHasOwn(namespace, '__esModule')) {
+    mod.exports = namespace;
+  } else {
+    mod.exports = createRequiredModuleFacade(wrap);
   }
 }
 
@@ -1805,7 +1827,7 @@ function reconstructErrorStack(err, parentPath, parentSource) {
  */
 function getRequireESMError(mod, pkg, content, filename) {
   // This is an error path because `require` of a `.js` file in a `"type": "module"` scope is not allowed.
-  const parent = mod[kModuleParent];
+  const parent = mod[kFirstModuleParent];
   const parentPath = parent?.filename;
   const packageJsonPath = pkg?.path;
   const usesEsm = containsModuleSyntax(content, filename);

lib/internal/modules/esm/loader.js

@@ -17,6 +17,7 @@ const {
 const {
   kIsExecuting,
   kRequiredModuleSymbol,
+  Module: CJSModule,
 } = require('internal/modules/cjs/loader');
 const { imported_cjs_symbol } = internalBinding('symbols');
 
@@ -91,13 +92,18 @@ function newLoadCache() {
   return new LoadCache();
 }
 
+let _translators;
+function lazyLoadTranslators() {
+  _translators ??= require('internal/modules/esm/translators');
+  return _translators;
+}
+
 /**
  * Lazy-load translators to avoid potentially unnecessary work at startup (ex if ESM is not used).
  * @returns {import('./translators.js').Translators}
  */
 function getTranslators() {
-  const { translators } = require('internal/modules/esm/translators');
-  return translators;
+  return lazyLoadTranslators().translators;
 }
 
 /**
@@ -506,7 +512,7 @@ class ModuleLoader {
 
     const { source } = loadResult;
     const isMain = (parentURL === undefined);
-    const wrap = this.#translate(url, finalFormat, source, isMain);
+    const wrap = this.#translate(url, finalFormat, source, parentURL);
     assert(wrap instanceof ModuleWrap, `Translator used for require(${url}) should not be async`);
 
     if (process.env.WATCH_REPORT_DEPENDENCIES && process.send) {
@@ -542,18 +548,31 @@ class ModuleLoader {
    * @param {string} format Format of the module to be translated. This is used to find
    *   matching translators.
    * @param {ModuleSource} source Source of the module to be translated.
-   * @param {boolean} isMain Whether the module to be translated is the entry point.
+   * @param {string|undefined} parentURL URL of the parent module. Undefined if it's the entry point.
    * @returns {ModuleWrap}
    */
-  #translate(url, format, source, isMain) {
+  #translate(url, format, source, parentURL) {
     this.validateLoadResult(url, format);
     const translator = getTranslators().get(format);
 
     if (!translator) {
       throw new ERR_UNKNOWN_MODULE_FORMAT(format, url);
     }
 
-    const result = FunctionPrototypeCall(translator, this, url, source, isMain);
+    // Populate the CJS cache with a facade for ESM in case subsequent require(esm) is
+    // looking it up from the cache. The parent module of the CJS cache entry would be the
+    // first CJS module that loads it with require(). This is an approximation, because
+    // ESM caches more and it does not get re-loaded and updated every time an `import` is
+    // encountered, unlike CJS require(), and we only use the parent entry to provide
+    // more information in error messages.
+    if (format === 'module') {
+      const parentFilename = urlToFilename(parentURL);
+      const parent = parentFilename ? CJSModule._cache[parentFilename] : undefined;
+      const cjsModule = lazyLoadTranslators().cjsEmplaceModuleCacheEntryForURL(url, parent);
+      debug('cjsEmplaceModuleCacheEntryForURL', url, parent, cjsModule);
+    }
+
+    const result = FunctionPrototypeCall(translator, this, url, source, parentURL === undefined);
     assert(result instanceof ModuleWrap);
     return result;
   }
@@ -563,10 +582,10 @@ class ModuleLoader {
    * This is run synchronously, and the translator always return a ModuleWrap synchronously.
    * @param {string} url URL of the module to be translated.
    * @param {object} loadContext See {@link load}
-   * @param {boolean} isMain Whether the module to be translated is the entry point.
+   * @param {string|undefined} parentURL URL of the parent module. Undefined if it's the entry point.
    * @returns {ModuleWrap}
    */
-  loadAndTranslateForRequireInImportedCJS(url, loadContext, isMain) {
+  loadAndTranslateForRequireInImportedCJS(url, loadContext, parentURL) {
     const { format: formatFromLoad, source } = this.#loadSync(url, loadContext);
 
     if (formatFromLoad === 'wasm') {  // require(wasm) is not supported.
@@ -587,7 +606,7 @@ class ModuleLoader {
       finalFormat = 'require-commonjs-typescript';
     }
 
-    const wrap = this.#translate(url, finalFormat, source, isMain);
+    const wrap = this.#translate(url, finalFormat, source, parentURL);
     assert(wrap instanceof ModuleWrap, `Translator used for require(${url}) should not be async`);
     return wrap;
   }
@@ -597,13 +616,13 @@ class ModuleLoader {
    * This may be run asynchronously if there are asynchronous module loader hooks registered.
    * @param {string} url URL of the module to be translated.
    * @param {object} loadContext See {@link load}
-   * @param {boolean} isMain Whether the module to be translated is the entry point.
+   * @param {string|undefined} parentURL URL of the parent module. Undefined if it's the entry point.
    * @returns {Promise<ModuleWrap>|ModuleWrap}
    */
-  loadAndTranslate(url, loadContext, isMain) {
+  loadAndTranslate(url, loadContext, parentURL) {
     const maybePromise = this.load(url, loadContext);
     const afterLoad = ({ format, source }) => {
-      return this.#translate(url, format, source, isMain);
+      return this.#translate(url, format, source, parentURL);
     };
     if (isPromise(maybePromise)) {
       return maybePromise.then(afterLoad);
@@ -630,9 +649,9 @@ class ModuleLoader {
     const isMain = parentURL === undefined;
     let moduleOrModulePromise;
     if (isForRequireInImportedCJS) {
-      moduleOrModulePromise = this.loadAndTranslateForRequireInImportedCJS(url, context, isMain);
+      moduleOrModulePromise = this.loadAndTranslateForRequireInImportedCJS(url, context, parentURL);
     } else {
-      moduleOrModulePromise = this.loadAndTranslate(url, context, isMain);
+      moduleOrModulePromise = this.loadAndTranslate(url, context, parentURL);
     }
 
     const inspectBrk = (