Skip to content

Commit 0b7fdd6

Browse files
author
Kanchalai Tanglertsampan
committed
Merge branch 'master' into fix10625
2 parents 5f88566 + 7ee05ba commit 0b7fdd6

File tree

62 files changed

+1672
-778
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

62 files changed

+1672
-778
lines changed

‎Gulpfile.ts‎

Lines changed: 60 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@ import through2 = require("through2");
3434
import merge2 = require("merge2");
3535
import intoStream = require("into-stream");
3636
import * as os from "os";
37-
import Linter = require("tslint");
3837
import fold = require("travis-fold");
3938
const gulp = helpMaker(originalGulp);
4039
const mochaParallel = require("./scripts/mocha-parallel.js");
@@ -929,26 +928,6 @@ gulp.task("build-rules", "Compiles tslint rules to js", () => {
929928
.pipe(gulp.dest(dest));
930929
});
931930

932-
function getLinterOptions() {
933-
return {
934-
configuration: require("./tslint.json"),
935-
formatter: "prose",
936-
formattersDirectory: undefined,
937-
rulesDirectory: "built/local/tslint"
938-
};
939-
}
940-
941-
function lintFileContents(options, path, contents) {
942-
const ll = new Linter(path, contents, options);
943-
console.log("Linting '" + path + "'.");
944-
return ll.lint();
945-
}
946-
947-
function lintFile(options, path) {
948-
const contents = fs.readFileSync(path, "utf8");
949-
return lintFileContents(options, path, contents);
950-
}
951-
952931
const lintTargets = [
953932
"Gulpfile.ts",
954933
"src/compiler/**/*.ts",
@@ -960,29 +939,72 @@ const lintTargets = [
960939
"tests/*.ts", "tests/webhost/*.ts" // Note: does *not* descend recursively
961940
];
962941

942+
function sendNextFile(files: {path: string}[], child: cp.ChildProcess, callback: (failures: number) => void, failures: number) {
943+
const file = files.pop();
944+
if (file) {
945+
console.log(`Linting '${file.path}'.`);
946+
child.send({ kind: "file", name: file.path });
947+
}
948+
else {
949+
child.send({ kind: "close" });
950+
callback(failures);
951+
}
952+
}
953+
954+
function spawnLintWorker(files: {path: string}[], callback: (failures: number) => void) {
955+
const child = cp.fork("./scripts/parallel-lint");
956+
let failures = 0;
957+
child.on("message", function(data) {
958+
switch (data.kind) {
959+
case "result":
960+
if (data.failures > 0) {
961+
failures += data.failures;
962+
console.log(data.output);
963+
}
964+
sendNextFile(files, child, callback, failures);
965+
break;
966+
case "error":
967+
console.error(data.error);
968+
failures++;
969+
sendNextFile(files, child, callback, failures);
970+
break;
971+
}
972+
});
973+
sendNextFile(files, child, callback, failures);
974+
}
963975

964976
gulp.task("lint", "Runs tslint on the compiler sources. Optional arguments are: --f[iles]=regex", ["build-rules"], () => {
965977
const fileMatcher = RegExp(cmdLineOptions["files"]);
966-
const lintOptions = getLinterOptions();
967-
let failed = 0;
968978
if (fold.isTravis()) console.log(fold.start("lint"));
969-
return gulp.src(lintTargets)
970-
.pipe(insert.transform((contents, file) => {
971-
if (!fileMatcher.test(file.path)) return contents;
972-
const result = lintFile(lintOptions, file.path);
973-
if (result.failureCount > 0) {
974-
console.log(result.output);
975-
failed += result.failureCount;
979+
980+
let files: {stat: fs.Stats, path: string}[] = [];
981+
return gulp.src(lintTargets, { read: false })
982+
.pipe(through2.obj((chunk, enc, cb) => {
983+
files.push(chunk);
984+
cb();
985+
}, (cb) => {
986+
files = files.filter(file => fileMatcher.test(file.path)).sort((filea, fileb) => filea.stat.size - fileb.stat.size);
987+
const workerCount = (process.env.workerCount && +process.env.workerCount) || os.cpus().length;
988+
for (let i = 0; i < workerCount; i++) {
989+
spawnLintWorker(files, finished);
976990
}
977-
return contents; // TODO (weswig): Automatically apply fixes? :3
978-
}))
979-
.on("end", () => {
980-
if (fold.isTravis()) console.log(fold.end("lint"));
981-
if (failed > 0) {
982-
console.error("Linter errors.");
983-
process.exit(1);
991+
992+
let completed = 0;
993+
let failures = 0;
994+
function finished(fails) {
995+
completed++;
996+
failures += fails;
997+
if (completed === workerCount) {
998+
if (fold.isTravis()) console.log(fold.end("lint"));
999+
if (failures > 0) {
1000+
throw new Error(`Linter errors: ${failures}`);
1001+
}
1002+
else {
1003+
cb();
1004+
}
1005+
}
9841006
}
985-
});
1007+
}));
9861008
});
9871009

9881010

‎Jakefile.js‎

Lines changed: 88 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ var fs = require("fs");
44
var os = require("os");
55
var path = require("path");
66
var child_process = require("child_process");
7-
var Linter = require("tslint");
87
var fold = require("travis-fold");
98
var runTestsInParallel = require("./scripts/mocha-parallel").runTestsInParallel;
109

@@ -33,6 +32,28 @@ if (process.env.path !== undefined) {
3332
process.env.PATH = nodeModulesPathPrefix + process.env.PATH;
3433
}
3534

35+
function toNs(diff) {
36+
return diff[0] * 1e9 + diff[1];
37+
}
38+
39+
function mark() {
40+
if (!fold.isTravis()) return;
41+
var stamp = process.hrtime();
42+
var id = Math.floor(Math.random() * 0xFFFFFFFF).toString(16);
43+
console.log("travis_time:start:" + id + "\r");
44+
return {
45+
stamp: stamp,
46+
id: id
47+
};
48+
}
49+
50+
function measure(marker) {
51+
if (!fold.isTravis()) return;
52+
var diff = process.hrtime(marker.stamp);
53+
var total = [marker.stamp[0] + diff[0], marker.stamp[1] + diff[1]];
54+
console.log("travis_time:end:" + marker.id + ":start=" + toNs(marker.stamp) + ",finish=" + toNs(total) + ",duration=" + toNs(diff) + "\r");
55+
}
56+
3657
var compilerSources = [
3758
"core.ts",
3859
"performance.ts",
@@ -286,6 +307,7 @@ var builtLocalCompiler = path.join(builtLocalDirectory, compilerFilename);
286307
*/
287308
function compileFile(outFile, sources, prereqs, prefixes, useBuiltCompiler, opts, callback) {
288309
file(outFile, prereqs, function() {
310+
var startCompileTime = mark();
289311
opts = opts || {};
290312
var compilerPath = useBuiltCompiler ? builtLocalCompiler : LKGCompiler;
291313
var options = "--noImplicitAny --noImplicitThis --noEmitOnError --types "
@@ -362,11 +384,13 @@ function compileFile(outFile, sources, prereqs, prefixes, useBuiltCompiler, opts
362384
callback();
363385
}
364386

387+
measure(startCompileTime);
365388
complete();
366389
});
367390
ex.addListener("error", function() {
368391
fs.unlinkSync(outFile);
369392
fail("Compilation of " + outFile + " unsuccessful");
393+
measure(startCompileTime);
370394
});
371395
ex.run();
372396
}, {async: true});
@@ -769,6 +793,7 @@ function runConsoleTests(defaultReporter, runInParallel) {
769793
// timeout normally isn't necessary but Travis-CI has been timing out on compiler baselines occasionally
770794
// default timeout is 2sec which really should be enough, but maybe we just need a small amount longer
771795
if(!runInParallel) {
796+
var startTime = mark();
772797
tests = tests ? ' -g "' + tests + '"' : '';
773798
var cmd = "mocha" + (debug ? " --debug-brk" : "") + " -R " + reporter + tests + colors + bail + ' -t ' + testTimeout + ' ' + run;
774799
console.log(cmd);
@@ -777,20 +802,23 @@ function runConsoleTests(defaultReporter, runInParallel) {
777802
process.env.NODE_ENV = "development";
778803
exec(cmd, function () {
779804
process.env.NODE_ENV = savedNodeEnv;
805+
measure(startTime);
780806
runLinter();
781807
finish();
782808
}, function(e, status) {
783809
process.env.NODE_ENV = savedNodeEnv;
810+
measure(startTime);
784811
finish(status);
785812
});
786813

787814
}
788815
else {
789816
var savedNodeEnv = process.env.NODE_ENV;
790817
process.env.NODE_ENV = "development";
818+
var startTime = mark();
791819
runTestsInParallel(taskConfigsFolder, run, { testTimeout: testTimeout, noColors: colors === " --no-colors " }, function (err) {
792820
process.env.NODE_ENV = savedNodeEnv;
793-
821+
measure(startTime);
794822
// last worker clean everything and runs linter in case if there were no errors
795823
deleteTemporaryProjectOutput();
796824
jake.rmRf(taskConfigsFolder);
@@ -1025,36 +1053,6 @@ task("build-rules-end", [] , function() {
10251053
if (fold.isTravis()) console.log(fold.end("build-rules"));
10261054
});
10271055

1028-
function getLinterOptions() {
1029-
return {
1030-
configuration: require("./tslint.json"),
1031-
formatter: "prose",
1032-
formattersDirectory: undefined,
1033-
rulesDirectory: "built/local/tslint"
1034-
};
1035-
}
1036-
1037-
function lintFileContents(options, path, contents) {
1038-
var ll = new Linter(path, contents, options);
1039-
console.log("Linting '" + path + "'.");
1040-
return ll.lint();
1041-
}
1042-
1043-
function lintFile(options, path) {
1044-
var contents = fs.readFileSync(path, "utf8");
1045-
return lintFileContents(options, path, contents);
1046-
}
1047-
1048-
function lintFileAsync(options, path, cb) {
1049-
fs.readFile(path, "utf8", function(err, contents) {
1050-
if (err) {
1051-
return cb(err);
1052-
}
1053-
var result = lintFileContents(options, path, contents);
1054-
cb(undefined, result);
1055-
});
1056-
}
1057-
10581056
var lintTargets = compilerSources
10591057
.concat(harnessSources)
10601058
// Other harness sources
@@ -1065,73 +1063,78 @@ var lintTargets = compilerSources
10651063
.concat(["Gulpfile.ts"])
10661064
.concat([nodeServerInFile, perftscPath, "tests/perfsys.ts", webhostPath]);
10671065

1066+
function sendNextFile(files, child, callback, failures) {
1067+
var file = files.pop();
1068+
if (file) {
1069+
console.log("Linting '" + file + "'.");
1070+
child.send({kind: "file", name: file});
1071+
}
1072+
else {
1073+
child.send({kind: "close"});
1074+
callback(failures);
1075+
}
1076+
}
1077+
1078+
function spawnLintWorker(files, callback) {
1079+
var child = child_process.fork("./scripts/parallel-lint");
1080+
var failures = 0;
1081+
child.on("message", function(data) {
1082+
switch (data.kind) {
1083+
case "result":
1084+
if (data.failures > 0) {
1085+
failures += data.failures;
1086+
console.log(data.output);
1087+
}
1088+
sendNextFile(files, child, callback, failures);
1089+
break;
1090+
case "error":
1091+
console.error(data.error);
1092+
failures++;
1093+
sendNextFile(files, child, callback, failures);
1094+
break;
1095+
}
1096+
});
1097+
sendNextFile(files, child, callback, failures);
1098+
}
10681099

10691100
desc("Runs tslint on the compiler sources. Optional arguments are: f[iles]=regex");
10701101
task("lint", ["build-rules"], function() {
10711102
if (fold.isTravis()) console.log(fold.start("lint"));
1072-
var lintOptions = getLinterOptions();
1103+
var startTime = mark();
10731104
var failed = 0;
10741105
var fileMatcher = RegExp(process.env.f || process.env.file || process.env.files || "");
10751106
var done = {};
10761107
for (var i in lintTargets) {
10771108
var target = lintTargets[i];
10781109
if (!done[target] && fileMatcher.test(target)) {
1079-
var result = lintFile(lintOptions, target);
1080-
if (result.failureCount > 0) {
1081-
console.log(result.output);
1082-
failed += result.failureCount;
1083-
}
1084-
done[target] = true;
1110+
done[target] = fs.statSync(target).size;
10851111
}
10861112
}
1087-
if (fold.isTravis()) console.log(fold.end("lint"));
1088-
if (failed > 0) {
1089-
fail('Linter errors.', failed);
1090-
}
1091-
});
10921113

1093-
/**
1094-
* This is required because file watches on Windows get fires _twice_
1095-
* when a file changes on some node/windows version configuations
1096-
* (node v4 and win 10, for example). By not running a lint for a file
1097-
* which already has a pending lint, we avoid duplicating our work.
1098-
* (And avoid printing duplicate results!)
1099-
*/
1100-
var lintSemaphores = {};
1101-
1102-
function lintWatchFile(filename) {
1103-
fs.watch(filename, {persistent: true}, function(event) {
1104-
if (event !== "change") {
1105-
return;
1106-
}
1114+
var workerCount = (process.env.workerCount && +process.env.workerCount) || os.cpus().length;
11071115

1108-
if (!lintSemaphores[filename]) {
1109-
lintSemaphores[filename] = true;
1110-
lintFileAsync(getLinterOptions(), filename, function(err, result) {
1111-
delete lintSemaphores[filename];
1112-
if (err) {
1113-
console.log(err);
1114-
return;
1115-
}
1116-
if (result.failureCount > 0) {
1117-
console.log("***Lint failure***");
1118-
for (var i = 0; i < result.failures.length; i++) {
1119-
var failure = result.failures[i];
1120-
var start = failure.startPosition.lineAndCharacter;
1121-
var end = failure.endPosition.lineAndCharacter;
1122-
console.log("warning " + filename + " (" + (start.line + 1) + "," + (start.character + 1) + "," + (end.line + 1) + "," + (end.character + 1) + "): " + failure.failure);
1123-
}
1124-
console.log("*** Total " + result.failureCount + " failures.");
1125-
}
1126-
});
1127-
}
1116+
var names = Object.keys(done).sort(function(namea, nameb) {
1117+
return done[namea] - done[nameb];
11281118
});
1129-
}
11301119

1131-
desc("Watches files for changes to rerun a lint pass");
1132-
task("lint-server", ["build-rules"], function() {
1133-
console.log("Watching ./src for changes to linted files");
1134-
for (var i = 0; i < lintTargets.length; i++) {
1135-
lintWatchFile(lintTargets[i]);
1120+
for (var i = 0; i < workerCount; i++) {
1121+
spawnLintWorker(names, finished);
11361122
}
1137-
});
1123+
1124+
var completed = 0;
1125+
var failures = 0;
1126+
function finished(fails) {
1127+
completed++;
1128+
failures += fails;
1129+
if (completed === workerCount) {
1130+
measure(startTime);
1131+
if (fold.isTravis()) console.log(fold.end("lint"));
1132+
if (failures > 0) {
1133+
fail('Linter errors.', failed);
1134+
}
1135+
else {
1136+
complete();
1137+
}
1138+
}
1139+
}
1140+
}, {async: true});

0 commit comments

Comments
 (0)