[web_benchmarks] Add support for analyzing benchmark results (flutter#5630)

kenzieschmoll · web-flow · commit 42396ee81548 · 2023-12-11T16:20:55.000-08:00
This PR adds a new top level library `analysis.dart` that exposes methods: * `computeDelta` * `computeAverage` These were already added to DevTools in flutter/devtools#6918 and flutter/devtools#6920, respectively, but I think it makes more sense to upstream this logic into `package:web_benchmarks` so that other users of this package can take advantage of it. As part of this PR, I have updated the README to give an example of how to use the analysis features.
diff --git a/packages/web_benchmarks/CHANGELOG.md b/packages/web_benchmarks/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 1.1.0
+
+* Adds `computeAverage` and `computeDelta` methods to support analysis of benchmark results.
+
 ## 1.0.1
 
 * Adds `parse` constructors for the `BenchmarkResults` and `BenchmarkScore` classes.
diff --git a/packages/web_benchmarks/README.md b/packages/web_benchmarks/README.md
@@ -13,3 +13,45 @@ app's code and assets. Additionally, the server communicates with the browser to
 extract the performance traces.
 
 [1]: https://github.com/flutter/packages/blob/master/packages/web_benchmarks/testing/web_benchmarks_test.dart
+
+# Analyzing benchmark results
+
+After running web benchmarks, you may want to analyze the results or compare
+with the results from other benchmark runs. The `web_benchmarks` package
+supports the following analysis operations:
+
+* compute the delta between two benchmark results
+* compute the average of a set of benchmark results
+
+<?code-excerpt "example/analyze_example.dart (analyze)"?>
+```dart
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:web_benchmarks/analysis.dart';
+
+void main() {
+  final BenchmarkResults baselineResults =
+      _benchmarkResultsFromFile('/path/to/benchmark_baseline.json');
+  final BenchmarkResults testResults1 =
+      _benchmarkResultsFromFile('/path/to/benchmark_test_1.json');
+  final BenchmarkResults testResults2 =
+      _benchmarkResultsFromFile('/path/to/benchmark_test_2.json');
+
+  // Compute the delta between [baselineResults] and [testResults1].
+  final BenchmarkResults delta = computeDelta(baselineResults, testResults1);
+  stdout.writeln(delta.toJson());
+
+  // Compute the average of [testResults] and [testResults2].
+  final BenchmarkResults average =
+      computeAverage(<BenchmarkResults>[testResults1, testResults2]);
+  stdout.writeln(average.toJson());
+}
+
+BenchmarkResults _benchmarkResultsFromFile(String path) {
+  final File file = File.fromUri(Uri.parse(path));
+  final Map<String, Object?> fileContentAsJson =
+      jsonDecode(file.readAsStringSync()) as Map<String, Object?>;
+  return BenchmarkResults.parse(fileContentAsJson);
+}
+```
diff --git a/packages/web_benchmarks/example/analyze_example.dart b/packages/web_benchmarks/example/analyze_example.dart
@@ -0,0 +1,35 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// #docregion analyze
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:web_benchmarks/analysis.dart';
+
+void main() {
+  final BenchmarkResults baselineResults =
+      _benchmarkResultsFromFile('/path/to/benchmark_baseline.json');
+  final BenchmarkResults testResults1 =
+      _benchmarkResultsFromFile('/path/to/benchmark_test_1.json');
+  final BenchmarkResults testResults2 =
+      _benchmarkResultsFromFile('/path/to/benchmark_test_2.json');
+
+  // Compute the delta between [baselineResults] and [testResults1].
+  final BenchmarkResults delta = computeDelta(baselineResults, testResults1);
+  stdout.writeln(delta.toJson());
+
+  // Compute the average of [testResults] and [testResults2].
+  final BenchmarkResults average =
+      computeAverage(<BenchmarkResults>[testResults1, testResults2]);
+  stdout.writeln(average.toJson());
+}
+
+BenchmarkResults _benchmarkResultsFromFile(String path) {
+  final File file = File.fromUri(Uri.parse(path));
+  final Map<String, Object?> fileContentAsJson =
+      jsonDecode(file.readAsStringSync()) as Map<String, Object?>;
+  return BenchmarkResults.parse(fileContentAsJson);
+}
+// #enddocregion analyze
diff --git a/packages/web_benchmarks/lib/analysis.dart b/packages/web_benchmarks/lib/analysis.dart
@@ -0,0 +1,121 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:collection/collection.dart';
+import 'server.dart';
+
+export 'src/benchmark_result.dart';
+
+/// Returns the average of the benchmark results in [results].
+///
+/// Each element in [results] is expected to have identical benchmark names and
+/// metrics; otherwise, an [Exception] will be thrown.
+BenchmarkResults computeAverage(List<BenchmarkResults> results) {
+  if (results.isEmpty) {
+    throw ArgumentError('Cannot take average of empty list.');
+  }
+
+  final BenchmarkResults totalSum = results.reduce(
+    (BenchmarkResults sum, BenchmarkResults next) => sum._sumWith(next),
+  );
+
+  final BenchmarkResults average = totalSum;
+  for (final String benchmark in totalSum.scores.keys) {
+    final List<BenchmarkScore> scoresForBenchmark = totalSum.scores[benchmark]!;
+    for (int i = 0; i < scoresForBenchmark.length; i++) {
+      final BenchmarkScore score = scoresForBenchmark[i];
+      final double averageValue = score.value / results.length;
+      average.scores[benchmark]![i] =
+          BenchmarkScore(metric: score.metric, value: averageValue);
+    }
+  }
+  return average;
+}
+
+/// Computes the delta for each matching metric in [test] and [baseline], and
+/// returns a new [BenchmarkResults] object where each [BenchmarkScore] contains
+/// a [delta] value.
+BenchmarkResults computeDelta(
+  BenchmarkResults baseline,
+  BenchmarkResults test,
+) {
+  final Map<String, List<BenchmarkScore>> delta =
+      <String, List<BenchmarkScore>>{};
+  for (final String benchmarkName in test.scores.keys) {
+    final List<BenchmarkScore> testScores = test.scores[benchmarkName]!;
+    final List<BenchmarkScore>? baselineScores = baseline.scores[benchmarkName];
+    delta[benchmarkName] = testScores.map<BenchmarkScore>(
+      (BenchmarkScore testScore) {
+        final BenchmarkScore? baselineScore = baselineScores?.firstWhereOrNull(
+            (BenchmarkScore s) => s.metric == testScore.metric);
+        return testScore._copyWith(
+          delta: baselineScore == null
+              ? null
+              : (testScore.value - baselineScore.value).toDouble(),
+        );
+      },
+    ).toList();
+  }
+  return BenchmarkResults(delta);
+}
+
+extension _AnalysisExtension on BenchmarkResults {
+  /// Sums this [BenchmarkResults] instance with [other] by adding the values
+  /// of each matching benchmark score.
+  ///
+  /// Returns a [BenchmarkResults] object with the summed values.
+  ///
+  /// When [throwExceptionOnMismatch] is true (default), the set of benchmark
+  /// names and metric names in [other] are expected to be identical to those in
+  /// [scores], or else an [Exception] will be thrown.
+  BenchmarkResults _sumWith(
+    BenchmarkResults other, {
+    bool throwExceptionOnMismatch = true,
+  }) {
+    final Map<String, List<BenchmarkScore>> sum =
+        <String, List<BenchmarkScore>>{};
+    for (final String benchmark in scores.keys) {
+      // Look up this benchmark in [other].
+      final List<BenchmarkScore>? matchingBenchmark = other.scores[benchmark];
+      if (matchingBenchmark == null) {
+        if (throwExceptionOnMismatch) {
+          throw Exception(
+            'Cannot sum benchmarks because [other] is missing an entry for '
+            'benchmark "$benchmark".',
+          );
+        }
+        continue;
+      }
+
+      final List<BenchmarkScore> scoresForBenchmark = scores[benchmark]!;
+      sum[benchmark] =
+          scoresForBenchmark.map<BenchmarkScore>((BenchmarkScore score) {
+        // Look up this score in the [matchingBenchmark] from [other].
+        final BenchmarkScore? matchingScore = matchingBenchmark
+            .firstWhereOrNull((BenchmarkScore s) => s.metric == score.metric);
+        if (matchingScore == null && throwExceptionOnMismatch) {
+          throw Exception(
+            'Cannot sum benchmarks because benchmark "$benchmark" is missing '
+            'a score for metric ${score.metric}.',
+          );
+        }
+        return score._copyWith(
+          value: matchingScore == null
+              ? score.value
+              : score.value + matchingScore.value,
+        );
+      }).toList();
+    }
+    return BenchmarkResults(sum);
+  }
+}
+
+extension _CopyExtension on BenchmarkScore {
+  BenchmarkScore _copyWith({String? metric, num? value, num? delta}) =>
+      BenchmarkScore(
+        metric: metric ?? this.metric,
+        value: value ?? this.value,
+        delta: delta ?? this.delta,
+      );
+}
diff --git a/packages/web_benchmarks/lib/src/benchmark_result.dart b/packages/web_benchmarks/lib/src/benchmark_result.dart
@@ -10,17 +10,26 @@ class BenchmarkScore {
   BenchmarkScore({
     required this.metric,
     required this.value,
+    this.delta,
   });
 
   /// Deserializes a JSON object to create a [BenchmarkScore] object.
   factory BenchmarkScore.parse(Map<String, Object?> json) {
-    final String metric = json[_metricKey]! as String;
-    final double value = (json[_valueKey]! as num).toDouble();
-    return BenchmarkScore(metric: metric, value: value);
+    final String metric = json[metricKey]! as String;
+    final double value = (json[valueKey]! as num).toDouble();
+    final num? delta = json[deltaKey] as num?;
+    return BenchmarkScore(metric: metric, value: value, delta: delta);
   }
 
-  static const String _metricKey = 'metric';
-  static const String _valueKey = 'value';
+  /// The key for the value [metric] in the [BenchmarkScore] JSON
+  /// representation.
+  static const String metricKey = 'metric';
+
+  /// The key for the value [value] in the [BenchmarkScore] JSON representation.
+  static const String valueKey = 'value';
+
+  /// The key for the value [delta] in the [BenchmarkScore] JSON representation.
+  static const String deltaKey = 'delta';
 
   /// The name of the metric that this score is categorized under.
   ///
@@ -31,11 +40,18 @@ class BenchmarkScore {
   /// The result of measuring a particular metric in this benchmark run.
   final num value;
 
+  /// Optional delta value describing the difference between this metric's score
+  /// and the score of a matching metric from another [BenchmarkResults].
+  ///
+  /// This value may be assigned by the [computeDelta] analysis method.
+  final num? delta;
+
   /// Serializes the benchmark metric to a JSON object.
   Map<String, Object?> toJson() {
     return <String, Object?>{
-      _metricKey: metric,
-      _valueKey: value,
+      metricKey: metric,
+      valueKey: value,
+      if (delta != null) deltaKey: delta,
     };
   }
 }
@@ -53,7 +69,7 @@ class BenchmarkResults {
       final List<BenchmarkScore> scores = (json[key]! as List<Object?>)
           .cast<Map<String, Object?>>()
           .map(BenchmarkScore.parse)
-          .toList();
+          .toList(growable: false);
       results[key] = scores;
     }
     return BenchmarkResults(results);
diff --git a/packages/web_benchmarks/pubspec.yaml b/packages/web_benchmarks/pubspec.yaml
@@ -2,13 +2,14 @@ name: web_benchmarks
 description: A benchmark harness for performance-testing Flutter apps in Chrome.
 repository: https://github.com/flutter/packages/tree/main/packages/web_benchmarks
 issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+web_benchmarks%22
-version: 1.0.1
+version: 1.1.0
 
 environment:
   sdk: ">=3.2.0 <4.0.0"
   flutter: ">=3.16.0"
 
 dependencies:
+  collection: ^1.18.0
   flutter:
     sdk: flutter
   flutter_test:
diff --git a/packages/web_benchmarks/test/src/analysis_test.dart b/packages/web_benchmarks/test/src/analysis_test.dart
diff --git a/packages/web_benchmarks/test/src/benchmark_result_test.dart b/packages/web_benchmarks/test/src/benchmark_result_test.dart
