feat(bench press): add microIterations option

tbosch · tbosch · commit 043b8c6d2ebb · 2015-03-03T14:41:03.000-08:00
diff --git a/modules/angular2/src/test_lib/perf_util.es6 b/modules/angular2/src/test_lib/perf_util.es6
@@ -23,7 +23,14 @@ function runBenchmark(config) {
   return getScaleFactor(browser.params.benchmark.scaling).then(function(scaleFactor) {
     var description = {};
     var urlParams = [];
-    config.params.forEach(function(param) {
+    var microIterations = config.microIterations || 0;
+    var params = config.params || [];
+    if (microIterations) {
+      params = params.concat([{
+        name: 'iterations', value: microIterations, scale: 'linear'
+      }]);
+    }
+    params.forEach(function(param) {
       var name = param.name;
       var value = applyScaleFactor(param.value, scaleFactor, param.scale);
       urlParams.push(name + '=' + value);
@@ -35,6 +42,7 @@ function runBenchmark(config) {
       id: config.id,
       execute: config.work,
       prepare: config.prepare,
+      microIterations: microIterations,
       bindings: [
         benchpress.bind(benchpress.Options.SAMPLE_DESCRIPTION).toValue(description)
       ]
diff --git a/modules/benchmarks/e2e_test/change_detection_perf.es6 b/modules/benchmarks/e2e_test/change_detection_perf.es6
@@ -12,8 +12,9 @@ describe('ng2 change detection benchmark', function () {
       buttons: ['#ng2ChangeDetectionDynamic'],
       id: 'ng2.changeDetection.dynamic',
       params: [{
-        name: 'numberOfChecks', value: 900000, scale: 'linear'
-      }]
+        name: 'numberOfChecks', value: 900000
+      }],
+      microIterations: 20
     }).then(done, done.fail);
   });
 
@@ -23,8 +24,9 @@ describe('ng2 change detection benchmark', function () {
       buttons: ['#ng2ChangeDetectionJit'],
       id: 'ng2.changeDetection.jit',
       params: [{
-        name: 'numberOfChecks', value: 900000, scale: 'linear'
-      }]
+        name: 'numberOfChecks', value: 900000
+      }],
+      microIterations: 20
     }).then(done, done.fail);
   });
 
@@ -34,8 +36,9 @@ describe('ng2 change detection benchmark', function () {
       buttons: ['#baselineChangeDetection'],
       id: 'baseline.changeDetection',
       params: [{
-        name: 'numberOfChecks', value: 900000, scale: 'linear'
-      }]
+        name: 'numberOfChecks', value: 900000
+      }],
+      microIterations: 20
     }).then(done, done.fail);
   });
 
diff --git a/modules/benchmarks/e2e_test/di_perf.es6 b/modules/benchmarks/e2e_test/di_perf.es6
@@ -13,7 +13,8 @@ describe('ng2 di benchmark', function () {
       id: 'ng2.di.getByToken',
       params: [{
         name: 'iterations', value: 20000, scale: 'linear'
-      }]
+      }],
+      microIterations: 20000
     }).then(done, done.fail);
   });
 
@@ -22,9 +23,7 @@ describe('ng2 di benchmark', function () {
       url: URL,
       buttons: ['#getByKey'],
       id: 'ng2.di.getByKey',
-      params: [{
-        name: 'iterations', value: 20000, scale: 'linear'
-      }]
+      microIterations: 20000
     }).then(done, done.fail);
   });
 
@@ -33,9 +32,7 @@ describe('ng2 di benchmark', function () {
       url: URL,
       buttons: ['#getChild'],
       id: 'ng2.di.getChild',
-      params: [{
-        name: 'iterations', value: 20000, scale: 'linear'
-      }]
+      microIterations: 20000
     }).then(done, done.fail);
   });
 
@@ -44,9 +41,7 @@ describe('ng2 di benchmark', function () {
       url: URL,
       buttons: ['#instantiate'],
       id: 'ng2.di.instantiate',
-      params: [{
-        name: 'iterations', value: 10000, scale: 'linear'
-      }]
+      microIterations: 10000
     }).then(done, done.fail);
   });
 
diff --git a/modules/benchmarks/e2e_test/element_injector_perf.es6 b/modules/benchmarks/e2e_test/element_injector_perf.es6
@@ -11,9 +11,7 @@ describe('ng2 element injector benchmark', function () {
       url: URL,
       buttons: ['#instantiate'],
       id: 'ng2.elementInjector.instantiate',
-      params: [{
-        name: 'iterations', value: 20000, scale: 'linear'
-      }]
+      microIterations: 20000
     }).then(done, done.fail);
   });
 
@@ -22,9 +20,7 @@ describe('ng2 element injector benchmark', function () {
       url: URL,
       buttons: ['#instantiateDirectives'],
       id: 'ng2.elementInjector.instantiateDirectives',
-      params: [{
-        name: 'iterations', value: 20000, scale: 'linear'
-      }]
+      microIterations: 20000
     }).then(done, done.fail);
   });
 
diff --git a/modules/benchmarks/src/change_detection/change_detection_benchmark.html b/modules/benchmarks/src/change_detection/change_detection_benchmark.html
@@ -5,6 +5,8 @@
 <h2>Params</h2>
 <form>
   Iterations:
+  <input type="number" name="iterations" placeholder="iterations" value="20">
+  Number of checks:
   <input type="number" name="numberOfChecks" placeholder="numberOfChecks" value="900000">
   <br>
   <button>Apply</button>
diff --git a/modules/benchmarks/src/change_detection/change_detection_benchmark.js b/modules/benchmarks/src/change_detection/change_detection_benchmark.js
@@ -138,9 +138,9 @@ function setUpChangeDetection(changeDetection:ChangeDetection, iterations) {
 export function main () {
   BrowserDomAdapter.makeCurrent();
   var numberOfChecks = getIntParameter('numberOfChecks');
+  var numberOfRuns = getIntParameter('iterations');
 
   var numberOfChecksPerDetector = 10;
-  var numberOfRuns = 20;
   var numberOfDetectors = numberOfChecks / numberOfChecksPerDetector / numberOfRuns;
 
   setUpReflector();
diff --git a/modules/benchpress/src/metric/perflog_metric.js b/modules/benchpress/src/metric/perflog_metric.js
@@ -5,6 +5,7 @@ import { bind, OpaqueToken } from 'angular2/di';
 
 import { WebDriverExtension } from '../web_driver_extension';
 import { Metric } from '../metric';
+import { Options } from '../sample_options';
 
 /**
  * A metric that reads out the performance log
@@ -19,24 +20,36 @@ export class PerflogMetric extends Metric {
   _remainingEvents:List;
   _measureCount:int;
   _setTimeout:Function;
+  _microIterations:int;
 
-  constructor(driverExtension:WebDriverExtension, setTimeout:Function) {
+  /**
+   * @param driverExtension
+   * @param setTimeout
+   * @param microIterations Number of iterations that run inside the browser by user code.
+   *                        Used for micro benchmarks.
+   **/
+  constructor(driverExtension:WebDriverExtension, setTimeout:Function, microIterations:int) {
     super();
     this._driverExtension = driverExtension;
     this._remainingEvents = [];
     this._measureCount = 0;
     this._setTimeout = setTimeout;
+    this._microIterations = microIterations;
   }
 
   describe():StringMap {
-    return {
+    var res = {
       'script': 'script execution time in ms',
       'render': 'render time in ms',
       'gcTime': 'gc time in ms',
       'gcAmount': 'gc amount in kbytes',
       'gcTimeInScript': 'gc time during script execution in ms',
       'gcAmountInScript': 'gc amount during script execution in kbytes'
     };
+    if (this._microIterations > 0) {
+      res['scriptMicroAvg'] = 'average script time for a micro iteration';
+    }
+    return res;
   }
 
   beginMeasure():Promise {
@@ -148,6 +161,9 @@ export class PerflogMetric extends Metric {
       }
     });
     result['script'] -= result['gcTimeInScript'];
+    if (this._microIterations > 0) {
+      result['scriptMicroAvg'] = result['script'] / this._microIterations;
+    }
     return isPresent(markStartEvent) && isPresent(markEndEvent) ? result : null;
   }
 
@@ -161,8 +177,9 @@ var _MARK_NAME_PREFIX = 'benchpress';
 var _SET_TIMEOUT = new OpaqueToken('PerflogMetric.setTimeout');
 var _BINDINGS = [
   bind(PerflogMetric).toFactory(
-    (driverExtension, setTimeout) => new PerflogMetric(driverExtension, setTimeout),
-    [WebDriverExtension, _SET_TIMEOUT]
+    (driverExtension, setTimeout, microIterations) => new PerflogMetric(driverExtension, setTimeout, microIterations),
+    [WebDriverExtension, _SET_TIMEOUT, Options.MICRO_ITERATIONS]
   ),
-  bind(_SET_TIMEOUT).toValue( (fn, millis) => PromiseWrapper.setTimeout(fn, millis) )
+  bind(_SET_TIMEOUT).toValue( (fn, millis) => PromiseWrapper.setTimeout(fn, millis) ),
+  bind(Options.MICRO_ITERATIONS).toValue(0)
 ];
diff --git a/modules/benchpress/src/runner.js b/modules/benchpress/src/runner.js
@@ -34,7 +34,7 @@ export class Runner {
     this._defaultBindings = defaultBindings;
   }
 
-  sample({id, execute, prepare, bindings}):Promise<SampleState> {
+  sample({id, execute, prepare, microIterations, bindings}):Promise<SampleState> {
     var sampleBindings = [
       _DEFAULT_BINDINGS,
       this._defaultBindings,
@@ -44,6 +44,9 @@ export class Runner {
     if (isPresent(prepare)) {
       ListWrapper.push(sampleBindings, bind(Options.PREPARE).toValue(prepare));
     }
+    if (isPresent(microIterations)) {
+      ListWrapper.push(sampleBindings, bind(Options.MICRO_ITERATIONS).toValue(microIterations));
+    }
     if (isPresent(bindings)) {
       ListWrapper.push(sampleBindings, bindings);
     }
diff --git a/modules/benchpress/src/sample_options.js b/modules/benchpress/src/sample_options.js
@@ -17,6 +17,12 @@ export class Options {
   static get CAPABILITIES() { return _CAPABILITIES; }
   // TODO(tbosch): use static initializer when our transpiler supports it
   static get USER_AGENT() { return _USER_AGENT; }
+  // TODO(tbosch): use static initializer when our transpiler supports it
+  /**
+   * Number of iterations that run inside the browser by user code.
+   * Used for micro benchmarks.
+   **/
+  static get MICRO_ITERATIONS() { return _MICRO_ITERATIONS; }
 }
 
 var _SAMPLE_ID = new OpaqueToken('Options.sampleId');
@@ -27,3 +33,4 @@ var _PREPARE = new OpaqueToken('Options.prepare');
 var _EXECUTE = new OpaqueToken('Options.execute');
 var _CAPABILITIES = new OpaqueToken('Options.capabilities');
 var _USER_AGENT = new OpaqueToken('Options.userAgent');
+var _MICRO_ITERATIONS = new OpaqueToken('Options.microIterations');
diff --git a/modules/benchpress/src/sampler.js b/modules/benchpress/src/sampler.js
@@ -136,7 +136,10 @@ var _BINDINGS = [
       execute: execute,
       time: time
     }),
-    [WebDriverAdapter, WebDriverExtension, Metric, Reporter, Validator, Options.FORCE_GC, Options.PREPARE, Options.EXECUTE, _TIME]
+    [
+      WebDriverAdapter, WebDriverExtension, Metric, Reporter, Validator,
+      Options.FORCE_GC, Options.PREPARE, Options.EXECUTE, _TIME
+    ]
   ),
   bind(Options.FORCE_GC).toValue(false),
   bind(Options.PREPARE).toValue(false),
diff --git a/modules/benchpress/test/metric/perflog_metric_spec.js b/modules/benchpress/test/metric/perflog_metric_spec.js
@@ -2,25 +2,28 @@ import {ddescribe, describe, it, iit, xit, expect, beforeEach, afterEach} from '
 
 import { List, ListWrapper } from 'angular2/src/facade/collection';
 import { PromiseWrapper, Promise } from 'angular2/src/facade/async';
+import { isPresent } from 'angular2/src/facade/lang';
 
-import { Metric, PerflogMetric, WebDriverExtension, bind, Injector } from 'benchpress/common';
+import { Metric, PerflogMetric, WebDriverExtension, bind, Injector, Options } from 'benchpress/common';
 
 import { TraceEventFactory } from '../trace_event_factory';
 
 export function main() {
   var commandLog;
   var eventFactory = new TraceEventFactory('timeline', 'pid0');
 
-  function createMetric(perfLogs) {
+  function createMetric(perfLogs, microIterations = 0) {
     commandLog = [];
-    return new Injector([
+    var bindings = [
       PerflogMetric.BINDINGS,
       bind(PerflogMetric.SET_TIMEOUT).toValue( (fn, millis) => {
         ListWrapper.push(commandLog, ['setTimeout', millis]);
         fn();
       }),
-      bind(WebDriverExtension).toValue(new MockDriverExtension(perfLogs, commandLog))
-    ]).get(PerflogMetric);
+      bind(WebDriverExtension).toValue(new MockDriverExtension(perfLogs, commandLog)),
+      bind(Options.MICRO_ITERATIONS).toValue(microIterations)
+    ];
+    return new Injector(bindings).get(PerflogMetric);
   }
 
   describe('perflog metric', () => {
@@ -151,10 +154,10 @@ export function main() {
 
     describe('aggregation', () => {
 
-      function aggregate(events) {
+      function aggregate(events, microIterations = 0) {
         ListWrapper.insert(events, 0, eventFactory.markStart('benchpress0', 0));
         ListWrapper.push(events, eventFactory.markEnd('benchpress0', 10));
-        var metric = createMetric([events]);
+        var metric = createMetric([events], microIterations);
         return metric
           .beginMeasure().then( (_) => metric.endMeasure(false) );
       }
@@ -252,6 +255,31 @@ export function main() {
         });
       });
 
+      describe('microIterations', () => {
+
+        it('should not report scriptMicroAvg if microIterations = 0', (done) => {
+          aggregate([
+            eventFactory.start('script', 0),
+            eventFactory.end('script', 5)
+          ], 0).then((data) => {
+            expect(isPresent(data['scriptMicroAvg'])).toBe(false);
+            done();
+          });
+        });
+
+        it('should report scriptMicroAvg', (done) => {
+          aggregate([
+            eventFactory.start('script', 0),
+            eventFactory.end('script', 5)
+          ], 4).then((data) => {
+            expect(data['script']).toBe(5);
+            expect(data['scriptMicroAvg']).toBe(5/4);
+            done();
+          });
+        });
+
+      });
+
       describe('gcTimeInScript / gcAmountInScript', () => {
 
         it('should detect gc during script execution with begin/end events', (done) => {
diff --git a/modules/benchpress/test/runner_spec.js b/modules/benchpress/test/runner_spec.js
@@ -85,6 +85,13 @@ export function main() {
       });
     });
 
+    it('should bind Options.MICRO_ITERATIONS', (done) => {
+      createRunner().sample({id: 'someId', microIterations: 23}).then( (_) => {
+        expect(injector.get(Options.MICRO_ITERATIONS)).toEqual(23);
+        done();
+      });
+    });
+
     it('should overwrite bindings per sample call', (done) => {
       createRunner([
         bind(Options.DEFAULT_DESCRIPTION).toValue({'a': 1}),

Original file line number	Diff line number	Diff line change
`@@ -34,7 +34,7 @@ export class Runner {`
`34`	`34`	`this._defaultBindings = defaultBindings;`
`35`	`35`	`}`
`36`	`36`
`37`		`- sample({id, execute, prepare, bindings}):Promise<SampleState> {`
	`37`	`+ sample({id, execute, prepare, microIterations, bindings}):Promise<SampleState> {`
`38`	`38`	`var sampleBindings = [`
`39`	`39`	`_DEFAULT_BINDINGS,`
`40`	`40`	`this._defaultBindings,`
`@@ -44,6 +44,9 @@ export class Runner {`
`44`	`44`	`if (isPresent(prepare)) {`
`45`	`45`	`ListWrapper.push(sampleBindings, bind(Options.PREPARE).toValue(prepare));`
`46`	`46`	`}`
	`47`	`+ if (isPresent(microIterations)) {`
	`48`	`+ ListWrapper.push(sampleBindings, bind(Options.MICRO_ITERATIONS).toValue(microIterations));`
	`49`	`+ }`
`47`	`50`	`if (isPresent(bindings)) {`
`48`	`51`	`ListWrapper.push(sampleBindings, bindings);`
`49`	`52`	`}`