perf(ngcc): reduce directory traversing (angular#35756)

This reduces the time that `findEntryPoints` takes from 9701.143ms to 4177.278ms, by reducing the file operations done.

Reference: angular#35717

PR Close angular#35756

Author: alan-agius4

packages/compiler-cli/ngcc/src/entry_point_finder/directory_walker_entry_point_finder.ts

@@ -9,8 +9,9 @@ import {AbsoluteFsPath, FileSystem, join, resolve} from '../../../src/ngtsc/file
 import {DependencyResolver, SortedEntryPointsInfo} from '../dependencies/dependency_resolver';
 import {Logger} from '../logging/logger';
 import {NgccConfiguration} from '../packages/configuration';
-import {EntryPoint, getEntryPointInfo} from '../packages/entry_point';
+import {EntryPoint, INVALID_ENTRY_POINT, NO_ENTRY_POINT, getEntryPointInfo} from '../packages/entry_point';
 import {PathMappings} from '../utils';
+import {NGCC_DIRECTORY} from '../writing/new_entry_point_file_writer';
 import {EntryPointFinder} from './interface';
 import {getBasePaths} from './utils';
 
@@ -40,10 +41,24 @@ export class DirectoryWalkerEntryPointFinder implements EntryPointFinder {
    * The function will recurse into directories that start with `@...`, e.g. `@angular/...`.
    * @param sourceDirectory An absolute path to the root directory where searching begins.
    */
-  private walkDirectoryForEntryPoints(sourceDirectory: AbsoluteFsPath): EntryPoint[] {
+  walkDirectoryForEntryPoints(sourceDirectory: AbsoluteFsPath): EntryPoint[] {
     const entryPoints = this.getEntryPointsForPackage(sourceDirectory);
+    if (entryPoints === null) {
+      return [];
+    }
+
     if (entryPoints.length > 0) {
-      // The `sourceDirectory` is an entry-point itself so no need to search its sub-directories.
+      // The `sourceDirectory` is an entry point itself so no need to search its sub-directories.
+      // Also check for any nested node_modules in this package but only if it was compiled by
+      // Angular.
+      // It is unlikely that a non Angular entry point has a dependency on an Angular library.
+      if (entryPoints.some(e => e.compiledByAngular)) {
+        const nestedNodeModulesPath = this.fs.join(sourceDirectory, 'node_modules');
+        if (this.fs.exists(nestedNodeModulesPath)) {
+          entryPoints.push(...this.walkDirectoryForEntryPoints(nestedNodeModulesPath));
+        }
+      }
+
       return entryPoints;
     }
 
@@ -52,54 +67,57 @@ export class DirectoryWalkerEntryPointFinder implements EntryPointFinder {
         // Not interested in hidden files
         .filter(p => !p.startsWith('.'))
         // Ignore node_modules
-        .filter(p => p !== 'node_modules')
+        .filter(p => p !== 'node_modules' && p !== NGCC_DIRECTORY)
         // Only interested in directories (and only those that are not symlinks)
         .filter(p => {
           const stat = this.fs.lstat(resolve(sourceDirectory, p));
           return stat.isDirectory() && !stat.isSymbolicLink();
         })
         .forEach(p => {
-          // Either the directory is a potential package or a namespace containing packages (e.g
-          // `@angular`).
+          // Package is a potential namespace containing packages (e.g `@angular`).
           const packagePath = join(sourceDirectory, p);
           entryPoints.push(...this.walkDirectoryForEntryPoints(packagePath));
-
-          // Also check for any nested node_modules in this package
-          const nestedNodeModulesPath = join(packagePath, 'node_modules');
-          if (this.fs.exists(nestedNodeModulesPath)) {
-            entryPoints.push(...this.walkDirectoryForEntryPoints(nestedNodeModulesPath));
-          }
         });
     return entryPoints;
   }
 
   /**
    * Recurse the folder structure looking for all the entry points
    * @param packagePath The absolute path to an npm package that may contain entry points
-   * @returns An array of entry points that were discovered.
+   * @returns An array of entry points that were discovered or null when it's not a valid entrypoint
    */
-  private getEntryPointsForPackage(packagePath: AbsoluteFsPath): EntryPoint[] {
+  private getEntryPointsForPackage(packagePath: AbsoluteFsPath): EntryPoint[]|null {
     const entryPoints: EntryPoint[] = [];
 
     // Try to get an entry point from the top level package directory
     const topLevelEntryPoint =
         getEntryPointInfo(this.fs, this.config, this.logger, packagePath, packagePath);
 
     // If there is no primary entry-point then exit
-    if (topLevelEntryPoint === null) {
+    if (topLevelEntryPoint === NO_ENTRY_POINT) {
       return [];
     }
 
+    if (topLevelEntryPoint === INVALID_ENTRY_POINT) {
+      return null;
+    }
+
     // Otherwise store it and search for secondary entry-points
     entryPoints.push(topLevelEntryPoint);
     this.walkDirectory(packagePath, packagePath, (path, isDirectory) => {
+      if (!path.endsWith('.js') && !isDirectory) {
+        return false;
+      }
+
       // If the path is a JS file then strip its extension and see if we can match an entry-point.
       const possibleEntryPointPath = isDirectory ? path : stripJsExtension(path);
       const subEntryPoint =
           getEntryPointInfo(this.fs, this.config, this.logger, packagePath, possibleEntryPointPath);
-      if (subEntryPoint !== null) {
-        entryPoints.push(subEntryPoint);
+      if (subEntryPoint === NO_ENTRY_POINT || subEntryPoint === INVALID_ENTRY_POINT) {
+        return false;
       }
+      entryPoints.push(subEntryPoint);
+      return true;
     });
 
     return entryPoints;
@@ -113,26 +131,25 @@ export class DirectoryWalkerEntryPointFinder implements EntryPointFinder {
    */
   private walkDirectory(
       packagePath: AbsoluteFsPath, dir: AbsoluteFsPath,
-      fn: (path: AbsoluteFsPath, isDirectory: boolean) => void) {
+      fn: (path: AbsoluteFsPath, isDirectory: boolean) => boolean) {
     return this.fs
         .readdir(dir)
         // Not interested in hidden files
         .filter(path => !path.startsWith('.'))
         // Ignore node_modules
-        .filter(path => path !== 'node_modules')
-        .map(path => resolve(dir, path))
+        .filter(path => path !== 'node_modules' && path !== NGCC_DIRECTORY)
         .forEach(path => {
-          const stat = this.fs.lstat(path);
+          const absolutePath = resolve(dir, path);
+          const stat = this.fs.lstat(absolutePath);
 
           if (stat.isSymbolicLink()) {
             // We are not interested in symbolic links
             return;
           }
 
-          fn(path, stat.isDirectory());
-
-          if (stat.isDirectory()) {
-            this.walkDirectory(packagePath, path, fn);
+          const containsEntryPoint = fn(absolutePath, stat.isDirectory());
+          if (containsEntryPoint) {
+            this.walkDirectory(packagePath, absolutePath, fn);
           }
         });
   }

packages/compiler-cli/ngcc/src/entry_point_finder/targeted_entry_point_finder.ts

@@ -10,7 +10,7 @@ import {DependencyResolver, SortedEntryPointsInfo} from '../dependencies/depende
 import {Logger} from '../logging/logger';
 import {hasBeenProcessed} from '../packages/build_marker';
 import {NgccConfiguration} from '../packages/configuration';
-import {EntryPoint, EntryPointJsonProperty, getEntryPointInfo} from '../packages/entry_point';
+import {EntryPoint, EntryPointJsonProperty, INVALID_ENTRY_POINT, NO_ENTRY_POINT, getEntryPointInfo} from '../packages/entry_point';
 import {PathMappings} from '../utils';
 import {EntryPointFinder} from './interface';
 import {getBasePaths} from './utils';
@@ -78,20 +78,26 @@ export class TargetedEntryPointFinder implements EntryPointFinder {
   private processNextPath(): void {
     const path = this.unprocessedPaths.shift() !;
     const entryPoint = this.getEntryPoint(path);
-    if (entryPoint !== null && entryPoint.compiledByAngular) {
-      this.unsortedEntryPoints.set(entryPoint.path, entryPoint);
-      const deps = this.resolver.getEntryPointDependencies(entryPoint);
-      deps.dependencies.forEach(dep => {
-        if (!this.unsortedEntryPoints.has(dep)) {
-          this.unprocessedPaths.push(dep);
-        }
-      });
+    if (entryPoint === null || !entryPoint.compiledByAngular) {
+      return;
     }
+    this.unsortedEntryPoints.set(entryPoint.path, entryPoint);
+    const deps = this.resolver.getEntryPointDependencies(entryPoint);
+    deps.dependencies.forEach(dep => {
+      if (!this.unsortedEntryPoints.has(dep)) {
+        this.unprocessedPaths.push(dep);
+      }
+    });
   }
 
   private getEntryPoint(entryPointPath: AbsoluteFsPath): EntryPoint|null {
     const packagePath = this.computePackagePath(entryPointPath);
-    return getEntryPointInfo(this.fs, this.config, this.logger, packagePath, entryPointPath);
+    const entryPoint =
+        getEntryPointInfo(this.fs, this.config, this.logger, packagePath, entryPointPath);
+    if (entryPoint === NO_ENTRY_POINT || entryPoint === INVALID_ENTRY_POINT) {
+      return null;
+    }
+    return entryPoint;
   }
 
   /**

packages/compiler-cli/ngcc/src/packages/entry_point.ts

@@ -75,41 +75,77 @@ export type EntryPointJsonProperty = Exclude<PackageJsonFormatProperties, 'types
 export const SUPPORTED_FORMAT_PROPERTIES: EntryPointJsonProperty[] =
     ['fesm2015', 'fesm5', 'es2015', 'esm2015', 'esm5', 'main', 'module'];
 
+
+/**
+ * The path does not represent an entry-point:
+ * * there is no package.json at the path and there is no config to force an entry-point
+ * * or the entrypoint is `ignored` by a config.
+ */
+export const NO_ENTRY_POINT = 'no-entry-point';
+
+/**
+ * The path has a package.json, but it is not a valid entry-point for ngcc processing.
+ */
+export const INVALID_ENTRY_POINT = 'invalid-entry-point';
+
+/**
+ * The result of calling `getEntryPointInfo()`.
+ *
+ * This will be an `EntryPoint` object if an Angular entry-point was identified;
+ * Otherwise it will be a flag indicating one of:
+ * * NO_ENTRY_POINT - the path is not an entry-point or ngcc is configured to ignore it
+ * * INVALID_ENTRY_POINT - the path was a non-processable entry-point that should be searched
+ * for sub-entry-points
+ */
+export type GetEntryPointResult = EntryPoint | typeof INVALID_ENTRY_POINT | typeof NO_ENTRY_POINT;
+
+
 /**
  * Try to create an entry-point from the given paths and properties.
  *
  * @param packagePath the absolute path to the containing npm package
  * @param entryPointPath the absolute path to the potential entry-point.
- * @returns An entry-point if it is valid, `null` otherwise.
+ * @returns
+ * - An entry-point if it is valid.
+ * - `undefined` when there is no package.json at the path and there is no config to force an
+ * entry-point or the entrypoint is `ignored`.
+ * - `null` there is a package.json but it is not a valid Angular compiled entry-point.
  */
 export function getEntryPointInfo(
     fs: FileSystem, config: NgccConfiguration, logger: Logger, packagePath: AbsoluteFsPath,
-    entryPointPath: AbsoluteFsPath): EntryPoint|null {
+    entryPointPath: AbsoluteFsPath): GetEntryPointResult {
   const packageJsonPath = resolve(entryPointPath, 'package.json');
   const packageVersion = getPackageVersion(fs, packageJsonPath);
   const entryPointConfig =
       config.getConfig(packagePath, packageVersion).entryPoints[entryPointPath];
-  if (entryPointConfig === undefined && !fs.exists(packageJsonPath)) {
-    return null;
+  const hasConfig = entryPointConfig !== undefined;
+
+  if (!hasConfig && !fs.exists(packageJsonPath)) {
+    // No package.json and no config
+    return NO_ENTRY_POINT;
   }
 
-  if (entryPointConfig !== undefined && entryPointConfig.ignore === true) {
-    return null;
+  if (hasConfig && entryPointConfig.ignore === true) {
+    // Explicitly ignored
+    return NO_ENTRY_POINT;
   }
 
-  const loadedEntryPointPackageJson =
-      loadEntryPointPackage(fs, logger, packageJsonPath, entryPointConfig !== undefined);
-  const entryPointPackageJson = mergeConfigAndPackageJson(
-      loadedEntryPointPackageJson, entryPointConfig, packagePath, entryPointPath);
+  const loadedEntryPointPackageJson = loadEntryPointPackage(fs, logger, packageJsonPath, hasConfig);
+  const entryPointPackageJson = hasConfig ?
+      mergeConfigAndPackageJson(
+          loadedEntryPointPackageJson, entryPointConfig, packagePath, entryPointPath) :
+      loadedEntryPointPackageJson;
+
   if (entryPointPackageJson === null) {
-    return null;
+    // package.json exists but could not be parsed and there was no redeeming config
+    return INVALID_ENTRY_POINT;
   }
 
-  // We must have a typings property
   const typings = entryPointPackageJson.typings || entryPointPackageJson.types ||
       guessTypingsFromPackageJson(fs, entryPointPath, entryPointPackageJson);
   if (typeof typings !== 'string') {
-    return null;
+    // Missing the required `typings` property
+    return INVALID_ENTRY_POINT;
   }
 
   // An entry-point is assumed to be compiled by Angular if there is either:
@@ -198,22 +234,13 @@ function isUmdModule(fs: FileSystem, sourceFilePath: AbsoluteFsPath): boolean {
 }
 
 function mergeConfigAndPackageJson(
-    entryPointPackageJson: EntryPointPackageJson | null,
-    entryPointConfig: NgccEntryPointConfig | undefined, packagePath: AbsoluteFsPath,
-    entryPointPath: AbsoluteFsPath): EntryPointPackageJson|null {
+    entryPointPackageJson: EntryPointPackageJson | null, entryPointConfig: NgccEntryPointConfig,
+    packagePath: AbsoluteFsPath, entryPointPath: AbsoluteFsPath): EntryPointPackageJson {
   if (entryPointPackageJson !== null) {
-    if (entryPointConfig === undefined) {
-      return entryPointPackageJson;
-    } else {
-      return {...entryPointPackageJson, ...entryPointConfig.override};
-    }
+    return {...entryPointPackageJson, ...entryPointConfig.override};
   } else {
-    if (entryPointConfig === undefined) {
-      return null;
-    } else {
-      const name = `${basename(packagePath)}/${relative(packagePath, entryPointPath)}`;
-      return {name, ...entryPointConfig.override};
-    }
+    const name = `${basename(packagePath)}/${relative(packagePath, entryPointPath)}`;
+    return {name, ...entryPointConfig.override};
   }
 }