Skip to content

Commit 77aa3ed

Browse files
tboschtbosch
committed
feat(benchpress): show more metrics and make the run mode configurable
Shows the metrics: script, render, gcAmount, gcAmountInScript, gcTime Run modes: - detect: auto detect whether to force gc - forceGc: forces a gc before every run and ignores no runs - noGcInScript: ignore runs that have gc while a script was executing - plain: does not force gc nor ignore runs Closes angular#368
1 parent 82b1601 commit 77aa3ed

File tree

11 files changed

+138
-115
lines changed

11 files changed

+138
-115
lines changed

‎modules/benchmarks/e2e_test/change_detection_perf.es6‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -29,14 +29,9 @@ function runClickBenchmark(config) {
2929
var buttons = config.buttons.map(function(selector) {
3030
return $(selector);
3131
});
32-
var timeParams = browser.params.benchmark;
33-
benchpress.runBenchmark({
34-
sampleSize: timeParams.sampleSize,
35-
targetCoefficientOfVariation: timeParams.targetCoefficientOfVariation,
36-
timeout: timeParams.timeout,
37-
metrics: timeParams.metrics,
38-
logId: browser.params.lang+'.'+config.logId
39-
}, function() {
32+
var params = Object.create(browser.params.benchmark);
33+
params.logId = browser.params.lang+'.'+config.logId;
34+
benchpress.runBenchmark(params, function() {
4035
buttons.forEach(function(button) {
4136
button.click();
4237
});

‎modules/benchmarks/e2e_test/compiler_perf.es6‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -29,14 +29,9 @@ function runClickBenchmark(config) {
2929
var buttons = config.buttons.map(function(selector) {
3030
return $(selector);
3131
});
32-
var timeParams = browser.params.benchmark;
33-
benchpress.runBenchmark({
34-
sampleSize: timeParams.sampleSize,
35-
targetCoefficientOfVariation: timeParams.targetCoefficientOfVariation,
36-
timeout: timeParams.timeout,
37-
metrics: timeParams.metrics,
38-
logId: browser.params.lang+'.'+config.logId
39-
}, function() {
32+
var params = Object.create(browser.params.benchmark);
33+
params.logId = browser.params.lang+'.'+config.logId;
34+
benchpress.runBenchmark(params, function() {
4035
buttons.forEach(function(button) {
4136
button.click();
4237
});

‎modules/benchmarks/e2e_test/di_perf.es6‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -45,14 +45,9 @@ function runClickBenchmark(config) {
4545
var buttons = config.buttons.map(function(selector) {
4646
return $(selector);
4747
});
48-
var timeParams = browser.params.benchmark;
49-
benchpress.runBenchmark({
50-
sampleSize: timeParams.sampleSize,
51-
targetCoefficientOfVariation: timeParams.targetCoefficientOfVariation,
52-
timeout: timeParams.timeout,
53-
metrics: timeParams.metrics,
54-
logId: browser.params.lang+'.'+config.logId
55-
}, function() {
48+
var params = Object.create(browser.params.benchmark);
49+
params.logId = browser.params.lang+'.'+config.logId;
50+
benchpress.runBenchmark(params, function() {
5651
buttons.forEach(function(button) {
5752
button.click();
5853
});

‎modules/benchmarks/e2e_test/element_injector_perf.es6‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -29,14 +29,9 @@ function runClickBenchmark(config) {
2929
var buttons = config.buttons.map(function(selector) {
3030
return $(selector);
3131
});
32-
var timeParams = browser.params.benchmark;
33-
benchpress.runBenchmark({
34-
sampleSize: timeParams.sampleSize,
35-
targetCoefficientOfVariation: timeParams.targetCoefficientOfVariation,
36-
timeout: timeParams.timeout,
37-
metrics: timeParams.metrics,
38-
logId: browser.params.lang+'.'+config.logId
39-
}, function() {
32+
var params = Object.create(browser.params.benchmark);
33+
params.logId = browser.params.lang+'.'+config.logId;
34+
benchpress.runBenchmark(params, function() {
4035
buttons.forEach(function(button) {
4136
button.click();
4237
});

‎modules/benchmarks/e2e_test/tree_perf.es6‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -29,14 +29,9 @@ function runClickBenchmark(config) {
2929
var buttons = config.buttons.map(function(selector) {
3030
return $(selector);
3131
});
32-
var timeParams = browser.params.benchmark;
33-
benchpress.runBenchmark({
34-
sampleSize: timeParams.sampleSize,
35-
targetCoefficientOfVariation: timeParams.targetCoefficientOfVariation,
36-
timeout: timeParams.timeout,
37-
metrics: timeParams.metrics,
38-
logId: browser.params.lang+'.'+config.logId
39-
}, function() {
32+
var params = Object.create(browser.params.benchmark);
33+
params.logId = browser.params.lang+'.'+config.logId;
34+
benchpress.runBenchmark(params, function() {
4035
buttons.forEach(function(button) {
4136
button.click();
4237
});

‎modules/benchmarks_external/e2e_test/compiler_perf.es6‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -29,14 +29,9 @@ function runClickBenchmark(config) {
2929
var buttons = config.buttons.map(function(selector) {
3030
return $(selector);
3131
});
32-
var timeParams = browser.params.benchmark;
33-
benchpress.runBenchmark({
34-
sampleSize: timeParams.sampleSize,
35-
targetCoefficientOfVariation: timeParams.targetCoefficientOfVariation,
36-
timeout: timeParams.timeout,
37-
metrics: timeParams.metrics,
38-
logId: browser.params.lang+'.'+config.logId
39-
}, function() {
32+
var params = Object.create(browser.params.benchmark);
33+
params.logId = browser.params.lang+'.'+config.logId;
34+
benchpress.runBenchmark(params, function() {
4035
buttons.forEach(function(button) {
4136
button.click();
4237
});

‎modules/benchmarks_external/e2e_test/tree_perf.es6‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,9 @@ function runClickBenchmark(config) {
2121
var buttons = config.buttons.map(function(selector) {
2222
return $(selector);
2323
});
24-
var timeParams = browser.params.benchmark;
25-
benchpress.runBenchmark({
26-
sampleSize: timeParams.sampleSize,
27-
targetCoefficientOfVariation: timeParams.targetCoefficientOfVariation,
28-
timeout: timeParams.timeout,
29-
metrics: timeParams.metrics,
30-
logId: browser.params.lang+'.'+config.logId
31-
}, function() {
24+
var params = Object.create(browser.params.benchmark);
25+
params.logId = browser.params.lang+'.'+config.logId;
26+
benchpress.runBenchmark(params, function() {
3227
buttons.forEach(function(button) {
3328
button.click();
3429
});

‎protractor-perf-shared.js‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,13 @@ var config = exports.config = {
1111
sampleSize: 10,
1212
targetCoefficientOfVariation: 4,
1313
timeout: 20000,
14-
metrics: ['script', 'render']
14+
metrics: ['script', 'render', 'gcAmount', 'gcAmountInScript', 'gcTime'],
15+
// run mode of the benchmark:
16+
// - detect: auto detect whether to force gc
17+
// - forceGc: forces a gc before every run and ignores no runs
18+
// - noGcInScript: ignore runs that have gc while a script was executing
19+
// - plain: does not force nor ignore runs
20+
mode: 'detect'
1521
}
1622
},
1723

‎tools/benchpress/src/benchmark.es6‎

Lines changed: 93 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,57 @@ var SUPPORTED_METRICS = {
66
script: true,
77
gcTime: true,
88
gcAmount: true,
9-
gcTimeDuringScript: true,
10-
gcAmountDuringScript: true,
9+
gcTimeInScript: true,
10+
gcAmountInScript: true,
1111
gcAmountPerMs: true,
1212
render: true
1313
};
14-
var DETERMINE_FORCE_GC_MODE_ITERATIONS = 5;
1514

16-
var MODE_FORCE_GC = 'forceGc';
17-
var MODE_IGNORE_RUNS_WITH_GC = 'ignoreRunsWithGc';
18-
var MODE_INDETERMINATE = 'indeterminate';
15+
var RUN_MODE = {
16+
detect: function(prevState, benchmarkData, iterationIndex) {
17+
var gcInScriptCount = prevState.gcInScriptCount || 0;
18+
if (benchmarkData.gcAmountInScript) {
19+
gcInScriptCount++;
20+
}
21+
var ignoreRun = !!benchmarkData.gcAmountInScript;
22+
var nextMode = RUN_MODE.detect;
23+
if (iterationIndex > 10) {
24+
if (gcInScriptCount / iterationIndex > 0.7) {
25+
nextMode = RUN_MODE.forceGc;
26+
} else {
27+
nextMode = RUN_MODE.noGcInScript;
28+
}
29+
}
30+
return {
31+
forceGc: false,
32+
ignoreRun: ignoreRun,
33+
gcInScriptCount: gcInScriptCount,
34+
nextMode: nextMode
35+
};
36+
},
37+
forceGc: function() {
38+
return {
39+
forceGc: true,
40+
ignoreRun: false,
41+
nextMode: RUN_MODE.forceGc
42+
}
43+
},
44+
noGcInScript: function(prevState, benchmarkData) {
45+
var ignoreRun = !!benchmarkData.gcAmountInScript;
46+
return {
47+
forceGc: false,
48+
ignoreRun: ignoreRun,
49+
nextMode: RUN_MODE.noGcInScript
50+
}
51+
},
52+
plain: function() {
53+
return {
54+
forceGc: false,
55+
ignoreRun: false,
56+
nextMode: RUN_MODE.plain
57+
}
58+
}
59+
};
1960

2061
var nextTimestampId = 0;
2162

@@ -30,66 +71,66 @@ function runBenchmark(config, workCallback) {
3071
throw new Error('Metric '+metric+' is not suported by benchpress right now');
3172
}
3273
});
33-
var ROW_FORMAT = ['%-40s'].concat(config.metrics.map(function() {
74+
var ROW_FORMAT = ['%-40s', '%12s'].concat(config.metrics.map(function() {
3475
return '%12s';
3576
})).join(' | ');
3677

37-
var benchmarkStatsAggregator = stats.createObjectStatsAggregator(config.sampleSize);
78+
var benchmarkStatsAggregator = stats.createObjectStatsAggregator(config.metrics, config.sampleSize);
3879

3980
var startTime = Date.now();
4081
startLoop().then(endLoop);
4182

42-
var gcDuringScriptCount = 0;
43-
4483
function startLoop(gcData) {
4584
reporter.printHeading('SCRIPT DATA: sampling size '+config.sampleSize);
46-
reporter.printTableHeader(ROW_FORMAT, ['name'].concat(config.metrics));
47-
return loop(0, MODE_INDETERMINATE);
85+
reporter.printTableHeader(ROW_FORMAT, ['name', 'action'].concat(config.metrics));
86+
if (!(config.mode in RUN_MODE)) {
87+
throw new Error('Unknown mode '+config.mode);
88+
}
89+
return loop(0, {
90+
forceGc: false,
91+
ignoreRun: false,
92+
nextMode: RUN_MODE[config.mode]
93+
});
4894
}
4995

5096
function endLoop(stats) {
51-
reporter.printTableFooter(ROW_FORMAT, [config.logId]
97+
reporter.printTableFooter(ROW_FORMAT, [config.logId, '']
5298
.concat(formatObjectStats(stats, config.metrics))
5399
);
54100
return config.metrics.map(function(metric) {
55101
return stats[metric];
56102
});
57103
}
58104

59-
function loop(iterationIndex, mode) {
60-
// For fast tests that don't create a lot of garbage,
61-
// we don't want to force gc before every run as that
62-
// can slow down the script execution time (even when we subtract
63-
// the gc time)!
64-
if (mode === MODE_FORCE_GC) {
65-
commands.gc();
66-
}
67-
return measureTime(workCallback).then(function(benchmarkData) {
68-
var hasGcDuringScript = !!benchmarkData.gcTimeDuringScript;
69-
var ignoreBenchmarkRun = false;
70-
if (hasGcDuringScript) {
71-
gcDuringScriptCount ++;
72-
ignoreBenchmarkRun = (mode === MODE_INDETERMINATE || mode === MODE_IGNORE_RUNS_WITH_GC);
73-
}
74-
if (mode === MODE_INDETERMINATE && iterationIndex >= DETERMINE_FORCE_GC_MODE_ITERATIONS) {
75-
mode = (gcDuringScriptCount / iterationIndex > 0.5) ? MODE_FORCE_GC : MODE_IGNORE_RUNS_WITH_GC;
105+
function loop(iterationIndex, modeState) {
106+
return measureTime(function() {
107+
workCallback();
108+
if (modeState.forceGc) {
109+
// For fast tests that don't create a lot of garbage,
110+
// we don't want to force gc before every run as that
111+
// can slow down the script execution time (even when we subtract
112+
// the gc time)!
113+
// Note: we need to call gc AFTER the actual test so the
114+
// gc amount is added to the current test run!
115+
commands.gc();
76116
}
77-
78-
var rowTitle;
79-
if (ignoreBenchmarkRun) {
80-
rowTitle = '(ignored: gc in script)';
81-
} else {
82-
rowTitle = config.logId + '#' + iterationIndex;
117+
}).then(function(benchmarkData) {
118+
modeState = modeState.nextMode(modeState, benchmarkData, iterationIndex);
119+
var action = '';
120+
if (modeState.ignoreRun) {
121+
action = 'ignore';
122+
} else if (modeState.forceGc) {
123+
action = 'forceGc';
83124
}
84-
reporter.printRow(ROW_FORMAT, [rowTitle]
125+
reporter.printRow(ROW_FORMAT, [config.logId + '#' + iterationIndex, action]
85126
.concat(formatObjectData(benchmarkData, config.metrics))
86127
);
87128

88129
var benchmarkStats;
89-
if (!ignoreBenchmarkRun) {
90-
benchmarkStats = benchmarkStatsAggregator(benchmarkData);
91-
} else {
130+
if (modeState.ignoreRun) {
92131
benchmarkStats = benchmarkStatsAggregator.current;
132+
} else {
133+
benchmarkStats = benchmarkStatsAggregator(benchmarkData);
93134
}
94135

95136
if (Date.now() - startTime > config.timeout) {
@@ -102,22 +143,26 @@ function runBenchmark(config, workCallback) {
102143
) {
103144
return benchmarkStats
104145
}
105-
return loop(iterationIndex+1, mode);
146+
return loop(iterationIndex+1, modeState);
106147
});
107148
}
108149
}
109150

110-
111151
function formatObjectData(data, props) {
112152
return props.map(function(prop) {
113-
return data[prop].toFixed(2);
153+
var val = data[prop];
154+
if (typeof val === 'number') {
155+
return val.toFixed(2);
156+
} else {
157+
return val;
158+
}
114159
});
115160
}
116161

117162
function formatObjectStats(stats, props) {
118163
return props.map(function(prop) {
119164
var entry = stats[prop];
120-
return entry.mean.toFixed(2) + '\u00B1' + entry.coefficientOfVariation.toFixed(2);
165+
return entry.mean.toFixed(2) + '\u00B1' + entry.coefficientOfVariation.toFixed(0)+ '%';
121166
});
122167
}
123168

@@ -156,8 +201,8 @@ function sumTimelineRecords(records, startTimeStampId, endTimeStampId) {
156201
script: 0,
157202
gcTime: 0,
158203
gcAmount: 0,
159-
gcTimeDuringScript: 0,
160-
gcAmountDuringScript: 0,
204+
gcTimeInScript: 0,
205+
gcAmountInScript: 0,
161206
render: 0,
162207
timeStamps: []
163208
};
@@ -205,8 +250,8 @@ function sumTimelineRecords(records, startTimeStampId, endTimeStampId) {
205250
recordStats.gcTime += recordDuration;
206251
recordStats.gcAmount += record.data.usedHeapSizeDelta;
207252
if (parentIsFunctionCall) {
208-
recordStats.gcTimeDuringScript += recordDuration;
209-
recordStats.gcAmountDuringScript += record.data.usedHeapSizeDelta;
253+
recordStats.gcTimeInScript += recordDuration;
254+
recordStats.gcAmountInScript += record.data.usedHeapSizeDelta;
210255
}
211256
recordUsed = true;
212257
} else if (record.type === 'RecalculateStyles' ||

‎tools/benchpress/src/reporter.es6‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
var vsprintf = require("sprintf-js").vsprintf;
22

3-
var HEADER_SEPARATORS = ['----', '----', '----', '----'];
4-
var FOOTER_SEPARATORS = ['====', '====', '====', '===='];
3+
var HEADER_SEPARATORS = ['----', '----', '----', '----', '----', '----', '----'];
4+
var FOOTER_SEPARATORS = ['====', '====', '====', '====', '====', '====', '===='];
55

66
module.exports = {
77
printHeading: printHeading,

0 commit comments

Comments
 (0)