Parallel linting

Gulpfile.ts

@@ -34,7 +34,6 @@ import through2 = require("through2");
 import merge2 = require("merge2");
 import intoStream = require("into-stream");
 import * as os from "os";
-import Linter = require("tslint");
 import fold = require("travis-fold");
 const gulp = helpMaker(originalGulp);
 const mochaParallel = require("./scripts/mocha-parallel.js");
@@ -929,26 +928,6 @@ gulp.task("build-rules", "Compiles tslint rules to js", () => {
         .pipe(gulp.dest(dest));
 });
 
-function getLinterOptions() {
-    return {
-        configuration: require("./tslint.json"),
-        formatter: "prose",
-        formattersDirectory: undefined,
-        rulesDirectory: "built/local/tslint"
-    };
-}
-
-function lintFileContents(options, path, contents) {
-    const ll = new Linter(path, contents, options);
-    console.log("Linting '" + path + "'.");
-    return ll.lint();
-}
-
-function lintFile(options, path) {
-    const contents = fs.readFileSync(path, "utf8");
-    return lintFileContents(options, path, contents);
-}
-
 const lintTargets = [
     "Gulpfile.ts",
     "src/compiler/**/*.ts",
@@ -960,29 +939,72 @@ const lintTargets = [
     "tests/*.ts", "tests/webhost/*.ts" // Note: does *not* descend recursively
 ];
 
+function sendNextFile(files, child, callback, failures) {
+    const file = files.pop();
+    if (file) {
+        console.log(`Linting '${file.path}'.`);
+        child.send({ kind: "file", name: file.path });
+    }
+    else {
+        child.send({ kind: "close" });
+        callback(failures);
+    }
+}
+
+function spawnLintWorker(files, callback) {
+    const child = cp.fork("./scripts/parallel-lint");
+    let failures = 0;
+    child.on("message", function(data) {
+        switch (data.kind) {
+            case "result":
+                if (data.failures > 0) {
+                    failures += data.failures;
+                    console.log(data.output);
+                }
+                sendNextFile(files, child, callback, failures);
+                break;
+            case "error":
+                console.error(data.error);
+                failures++;
+                sendNextFile(files, child, callback, failures);
+                break;
+        }
+    });
+    sendNextFile(files, child, callback, failures);
+}
 
 gulp.task("lint", "Runs tslint on the compiler sources. Optional arguments are: --f[iles]=regex", ["build-rules"], () => {
     const fileMatcher = RegExp(cmdLineOptions["files"]);
-    const lintOptions = getLinterOptions();
-    let failed = 0;
     if (fold.isTravis()) console.log(fold.start("lint"));
-    return gulp.src(lintTargets)
-        .pipe(insert.transform((contents, file) => {
-            if (!fileMatcher.test(file.path)) return contents;
-            const result = lintFile(lintOptions, file.path);
-            if (result.failureCount > 0) {
-                console.log(result.output);
-                failed += result.failureCount;
+
+    const files = [];
+    return gulp.src(lintTargets, { read: false })
+        .pipe(through2.obj((chunk, enc, cb) => {
+            files.push(chunk);
+            cb();
+        }, (cb) => {
+            files.sort((filea, fileb) => filea.stat.size - fileb.stat.size).filter(file =>  fileMatcher.test(file.path));
+            const workerCount = (process.env.workerCount && +process.env.workerCount) || os.cpus().length;
+            for (let i = 0; i < workerCount; i++) {
+                spawnLintWorker(files, finished);
             }
-            return contents; // TODO (weswig): Automatically apply fixes? :3
-        }))
-        .on("end", () => {
-            if (fold.isTravis()) console.log(fold.end("lint"));
-            if (failed > 0) {
-                console.error("Linter errors.");
-                process.exit(1);
+
+            let completed = 0;
+            let failures = 0;
+            function finished(fails) {
+                completed++;
+                failures += fails;
+                if (completed === workerCount) {
+                    if (fold.isTravis()) console.log(fold.end("lint"));
+                    if (failures > 0) {
+                        throw new Error(`Linter errors: ${failures}`);
+                    }
+                    else {
+                        cb();
+                    }
+                }
             }
-        });
+        }));
 });
 
 

Jakefile.js

@@ -4,7 +4,6 @@ var fs = require("fs");
 var os = require("os");
 var path = require("path");
 var child_process = require("child_process");
-var Linter = require("tslint");
 var fold = require("travis-fold");
 var runTestsInParallel = require("./scripts/mocha-parallel").runTestsInParallel;
 
@@ -1054,36 +1053,6 @@ task("build-rules-end", [] , function() {
     if (fold.isTravis()) console.log(fold.end("build-rules"));
 });
 
-function getLinterOptions() {
-    return {
-        configuration: require("./tslint.json"),
-        formatter: "prose",
-        formattersDirectory: undefined,
-        rulesDirectory: "built/local/tslint"
-    };
-}
-
-function lintFileContents(options, path, contents) {
-    var ll = new Linter(path, contents, options);
-    console.log("Linting '" + path + "'.");
-    return ll.lint();
-}
-
-function lintFile(options, path) {
-    var contents = fs.readFileSync(path, "utf8");
-    return lintFileContents(options, path, contents);
-}
-
-function lintFileAsync(options, path, cb) {
-    fs.readFile(path, "utf8", function(err, contents) {
-        if (err) {
-            return cb(err);
-        }
-        var result = lintFileContents(options, path, contents);
-        cb(undefined, result);
-    });
-}
-
 var lintTargets = compilerSources
     .concat(harnessSources)
     // Other harness sources
@@ -1094,75 +1063,78 @@ var lintTargets = compilerSources
     .concat(["Gulpfile.ts"])
     .concat([nodeServerInFile, perftscPath, "tests/perfsys.ts", webhostPath]);
 
+function sendNextFile(files, child, callback, failures) {
+    var file = files.pop();
+    if (file) {
+        console.log("Linting '" + file + "'.");
+        child.send({kind: "file", name: file});
+    }
+    else {
+        child.send({kind: "close"});
+        callback(failures);
+    }
+}
+
+function spawnLintWorker(files, callback) {
+    var child = child_process.fork("./scripts/parallel-lint");
+    var failures = 0;
+    child.on("message", function(data) {
+        switch (data.kind) {
+            case "result":
+                if (data.failures > 0) {
+                    failures += data.failures;
+                    console.log(data.output);
+                }
+                sendNextFile(files, child, callback, failures);
+                break;
+            case "error":
+                console.error(data.error);
+                failures++;
+                sendNextFile(files, child, callback, failures);
+                break;
+        }
+    });
+    sendNextFile(files, child, callback, failures);
+}
 
 desc("Runs tslint on the compiler sources. Optional arguments are: f[iles]=regex");
 task("lint", ["build-rules"], function() {
     if (fold.isTravis()) console.log(fold.start("lint"));
     var startTime = mark();
-    var lintOptions = getLinterOptions();
     var failed = 0;
     var fileMatcher = RegExp(process.env.f || process.env.file || process.env.files || "");
     var done = {};
     for (var i in lintTargets) {
         var target = lintTargets[i];
         if (!done[target] && fileMatcher.test(target)) {
-            var result = lintFile(lintOptions, target);
-            if (result.failureCount > 0) {
-                console.log(result.output);
-                failed += result.failureCount;
-            }
-            done[target] = true;
+            done[target] = fs.statSync(target).size;
         }
     }
-    measure(startTime);
-    if (fold.isTravis()) console.log(fold.end("lint"));
-    if (failed > 0) {
-        fail('Linter errors.', failed);
-    }
-});
 
-/**
- * This is required because file watches on Windows get fires _twice_
- * when a file changes on some node/windows version configuations
- * (node v4 and win 10, for example). By not running a lint for a file
- * which already has a pending lint, we avoid duplicating our work.
- * (And avoid printing duplicate results!)
- */
-var lintSemaphores = {};
-
-function lintWatchFile(filename) {
-    fs.watch(filename, {persistent: true}, function(event) {
-        if (event !== "change") {
-            return;
-        }
+    var workerCount = (process.env.workerCount && +process.env.workerCount) || os.cpus().length;
 
-        if (!lintSemaphores[filename]) {
-            lintSemaphores[filename] = true;
-            lintFileAsync(getLinterOptions(), filename, function(err, result) {
-                delete lintSemaphores[filename];
-                if (err) {
-                    console.log(err);
-                    return;
-                }
-                if (result.failureCount > 0) {
-                    console.log("***Lint failure***");
-                    for (var i = 0; i < result.failures.length; i++) {
-                        var failure = result.failures[i];
-                        var start = failure.startPosition.lineAndCharacter;
-                        var end = failure.endPosition.lineAndCharacter;
-                        console.log("warning " + filename + " (" + (start.line + 1) + "," + (start.character + 1) + "," + (end.line + 1) + "," + (end.character + 1) + "): " + failure.failure);
-                    }
-                    console.log("*** Total " + result.failureCount + " failures.");
-                }
-            });
-        }
+    var names = Object.keys(done).sort(function(namea, nameb) {
+        return done[namea] - done[nameb];
     });
-}
 
-desc("Watches files for changes to rerun a lint pass");
-task("lint-server", ["build-rules"], function() {
-    console.log("Watching ./src for changes to linted files");
-    for (var i = 0; i < lintTargets.length; i++) {
-        lintWatchFile(lintTargets[i]);
+    for (var i = 0; i < workerCount; i++) {
+        spawnLintWorker(names, finished);
     }
-});
+
+    var completed = 0;
+    var failures = 0;
+    function finished(fails) {
+        completed++;
+        failures += fails;
+        if (completed === workerCount) {
+            measure(startTime);
+            if (fold.isTravis()) console.log(fold.end("lint"));
+            if (failures > 0) {
+                fail('Linter errors.', failed);
+            }
+            else {
+                complete();
+            }
+        }
+    }
+}, {async: true});

scripts/parallel-lint.js

@@ -0,0 +1,45 @@
+var Linter = require("tslint");
+var fs = require("fs");
+
+function getLinterOptions() {
+    return {
+        configuration: require("../tslint.json"),
+        formatter: "prose",
+        formattersDirectory: undefined,
+        rulesDirectory: "built/local/tslint"
+    };
+}
+
+function lintFileContents(options, path, contents) {
+    var ll = new Linter(path, contents, options);
+    return ll.lint();
+}
+
+function lintFileAsync(options, path, cb) {
+    fs.readFile(path, "utf8", function(err, contents) {
+        if (err) {
+            return cb(err);
+        }
+        var result = lintFileContents(options, path, contents);
+        cb(undefined, result);
+    });
+}
+
+process.on("message", function(data) {
+    switch (data.kind) {
+        case "file":
+        var target = data.name;
+        var lintOptions = getLinterOptions();
+        lintFileAsync(lintOptions, target, function(err, result) {
+            if (err) {
+                process.send({kind: "error", error: err.toString()});
+                return;
+            }
+            process.send({kind: "result", failures: result.failureCount, output: result.output});
+        });
+        break;
+        case "close":
+        process.exit(0);
+        break;
+    }
+});