misc: update minify trace logic to date

lighthouse-core/lib/minify-devtoolslog.js

@@ -0,0 +1,87 @@
+/**
+ * @license Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/* eslint-disable no-console */
+
+/**
+ * @fileoverview Minifies a devtools log by removing noisy header values, eliminating data URIs, etc.
+ */
+
+const headersToKeep = new Set([
+  // Request headers
+  'accept',
+  'accept-encoding',
+  'accept-ranges',
+  // Response headers
+  'status',
+  'content-length',
+  'content-type',
+  'content-encoding',
+  'content-range',
+  'etag',
+  'cache-control',
+  'last-modified',
+  'link',
+  'x-robots-tag',
+]);
+
+/** @param {LH.Crdp.Network.Headers} [headers] */
+function cleanHeaders(headers) {
+  if (!headers) return;
+
+  for (const key of Object.keys(headers)) {
+    if (!headersToKeep.has(key.toLowerCase())) delete headers[key];
+  }
+}
+
+/** @param {{url: string}} obj */
+function cleanDataURI(obj) {
+  obj.url = obj.url.replace(/^(data:.*?base64,).*/, '$1FILLER');
+}
+
+/** @param {LH.Crdp.Network.Response} [response] */
+function cleanResponse(response) {
+  if (!response) return;
+  cleanDataURI(response);
+  cleanHeaders(response.requestHeaders);
+  cleanHeaders(response.headers);
+  response.securityDetails = undefined;
+  response.headersText = undefined;
+  response.requestHeadersText = undefined;
+
+  /** @type {any} */
+  const timing = response.timing || {};
+  for (const [k, v] of Object.entries(timing)) {
+    if (v === -1) timing[k] = undefined;
+  }
+}
+
+/**
+ * @param {LH.DevtoolsLog} log
+ * @return {LH.DevtoolsLog}
+ */
+function minifyDevtoolsLog(log) {
+  return log.map(original => {
+    /** @type {LH.Protocol.RawEventMessage} */
+    const entry = JSON.parse(JSON.stringify(original));
+
+    switch (entry.method) {
+      case 'Network.requestWillBeSent':
+        cleanDataURI(entry.params.request);
+        cleanHeaders(entry.params.request.headers);
+        cleanResponse(entry.params.redirectResponse);
+        break;
+      case 'Network.responseReceived':
+        cleanResponse(entry.params.response);
+        break;
+    }
+
+    return entry;
+  });
+}
+
+module.exports = {minifyDevtoolsLog};

lighthouse-core/lib/minify-trace.js

@@ -0,0 +1,149 @@
+/**
+ * @license Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/* eslint-disable no-console */
+
+/**
+ * @fileoverview Minifies a trace by removing unnecessary events, throttling screenshots, etc.
+ *  See the following files for necessary events:
+ *    - lighthouse-core/computed/trace-of-tab.js
+ *    - lighthouse-core/computed/page-dependency-graph.js
+ *    - lighthouse-core/lib/dependency-graph/cpu-node.js
+ *    - lighthouse-core/lib/traces/tracing-processor.js
+ */
+
+const TracingProcessor = require('./traces/tracing-processor.js');
+
+const toplevelTaskNames = new Set([
+  'RunTask', // m71+
+  'ThreadControllerImpl::RunTask', // m69-70
+  'ThreadControllerImpl::DoWork', // m66-68
+  'TaskQueueManager::ProcessTaskFromWorkQueue', // m65 and below
+]);
+
+const traceEventsToAlwaysKeep = new Set([
+  'Screenshot',
+  'TracingStartedInBrowser',
+  'TracingStartedInPage',
+  'navigationStart',
+  'ParseAuthorStyleSheet',
+  'ParseHTML',
+  'PlatformResourceSendRequest',
+  'ResourceSendRequest',
+  'ResourceReceiveResponse',
+  'ResourceFinish',
+  'ResourceReceivedData',
+  'EventDispatch',
+]);
+
+const traceEventsToKeepInToplevelTask = new Set([
+  // Needed for CPU node timing simulations
+  'Layout',
+  // All of these are needed to create graph relationships
+  'TimerInstall',
+  'TimerFire',
+  'InvalidateLayout',
+  'ScheduleStyleRecalculation',
+  'EvaluateScript',
+  'XHRReadyStateChange',
+  'FunctionCall',
+  'v8.compile',
+  'ParseAuthorStyleSheet',
+  'ResourceSendRequest',
+]);
+
+const traceEventsToKeepInProcess = new Set([
+  ...toplevelTaskNames,
+  ...traceEventsToKeepInToplevelTask,
+  'firstPaint',
+  'firstContentfulPaint',
+  'firstMeaningfulPaint',
+  'firstMeaningfulPaintCandidate',
+  'loadEventEnd',
+  'domContentLoadedEventEnd',
+]);
+
+/**
+ * @param {LH.TraceEvent[]} events
+ */
+function filterOutUnnecessaryTasksByNameAndDuration(events) {
+  const {pid} = TracingProcessor.findMainFrameIds(events);
+
+  return events.filter(evt => {
+    if (toplevelTaskNames.has(evt.name) && evt.dur < 1000) return false;
+    if (evt.pid === pid && traceEventsToKeepInProcess.has(evt.name)) return true;
+    return traceEventsToAlwaysKeep.has(evt.name);
+  });
+}
+
+/**
+ * Filters out tasks that are not within a toplevel task.
+ * @param {LH.TraceEvent[]} events
+ */
+function filterOutOrphanedTasks(events) {
+  const toplevelRanges = events
+    .filter(evt => toplevelTaskNames.has(evt.name))
+    .map(evt => [evt.ts, evt.ts + evt.dur]);
+
+  /** @param {LH.TraceEvent} e */
+  const isInToplevelTask = e => toplevelRanges.some(([start, end]) => e.ts >= start && e.ts <= end);
+
+  return events.filter((evt, index) => {
+    if (!traceEventsToKeepInToplevelTask.has(evt.name)) return true;
+    if (!isInToplevelTask(evt)) return false;
+
+    if (evt.ph === 'B') {
+      const endEvent = events.slice(index).find(e => e.name === evt.name && e.ph === 'E');
+      return endEvent && isInToplevelTask(endEvent);
+    } else {
+      return true;
+    }
+  });
+}
+
+/**
+ * Throttles screenshot events in the trace to 2fps.
+ * @param {LH.TraceEvent[]} events
+ */
+function filterOutExcessiveScreenshots(events) {
+  const screenshotTimestamps = events.filter(evt => evt.name === 'Screenshot').map(evt => evt.ts);
+
+  let lastScreenshotTs = -Infinity;
+  return events.filter(evt => {
+    if (evt.name !== 'Screenshot') return true;
+    const timeSinceLastScreenshot = evt.ts - lastScreenshotTs;
+    const nextScreenshotTs = screenshotTimestamps.find(ts => ts > evt.ts);
+    const timeUntilNextScreenshot = nextScreenshotTs ? nextScreenshotTs - evt.ts : Infinity;
+    const threshold = 500 * 1000; // Throttle to ~2fps
+    // Keep the frame if it's been more than 500ms since the last frame we kept or the next frame won't happen for at least 500ms
+    const shouldKeep = timeUntilNextScreenshot > threshold || timeSinceLastScreenshot > threshold;
+    if (shouldKeep) lastScreenshotTs = evt.ts;
+    return shouldKeep;
+  });
+}
+
+/**
+ * @param {LH.TraceEvent[]} events
+ */
+function filterTraceEvents(events) {
+  // Filter out event names we don't care about and tasks <1ms
+  let filtered = filterOutUnnecessaryTasksByNameAndDuration(events);
+  // Filter out events not inside a toplevel task
+  filtered = filterOutOrphanedTasks(filtered);
+  // Filter down the screenshots to key moments + 2fps animations
+  return filterOutExcessiveScreenshots(filtered);
+}
+
+/**
+ * @param {LH.Trace} inputTrace
+ * @return {LH.Trace}
+ */
+function minifyTrace(inputTrace) {
+  return {traceEvents: filterTraceEvents(inputTrace.traceEvents)};
+}
+
+module.exports = {minifyTrace};

lighthouse-core/scripts/lantern/minify-devtoolslog.js

@@ -14,6 +14,7 @@
 
 const fs = require('fs');
 const path = require('path');
+const {minifyDevtoolsLog} = require('../../lib/minify-devtoolslog.js');
 
 if (process.argv.length !== 4) {
   console.error('Usage $0: <input file> <output file>');
@@ -26,79 +27,9 @@ const inputDevtoolsLogRaw = fs.readFileSync(inputDevtoolsLogPath, 'utf8');
 /** @type {LH.DevtoolsLog} */
 const inputDevtoolsLog = JSON.parse(inputDevtoolsLogRaw);
 
-const headersToKeep = new Set([
-  // Request headers
-  'accept',
-  'accept-encoding',
-  'accept-ranges',
-  // Response headers
-  'status',
-  'content-length',
-  'content-type',
-  'content-encoding',
-  'content-range',
-  'etag',
-  'cache-control',
-  'last-modified',
-  'link',
-  'x-robots-tag',
-]);
-
-/** @param {Partial<LH.Crdp.Network.Headers>} [headers] */
-function cleanHeaders(headers) {
-  if (!headers) return;
-
-  for (const key of Object.keys(headers)) {
-    if (!headersToKeep.has(key.toLowerCase())) headers[key] = undefined;
-  }
-}
-
-/** @param {{url: string}} obj */
-function cleanDataURI(obj) {
-  obj.url = obj.url.replace(/^(data:.*?base64,).*/, '$1FILLER');
-}
-
-/** @param {LH.Crdp.Network.Response} [response] */
-function cleanResponse(response) {
-  if (!response) return;
-  cleanDataURI(response);
-  cleanHeaders(response.requestHeaders);
-  cleanHeaders(response.headers);
-  response.securityDetails = undefined;
-  response.headersText = undefined;
-  response.requestHeadersText = undefined;
-
-  /** @type {any} */
-  const timing = response.timing || {};
-  for (const [k, v] of Object.entries(timing)) {
-    if (v === -1) timing[k] = undefined;
-  }
-}
-
-/** @param {LH.DevtoolsLog} log */
-function filterDevtoolsLogEvents(log) {
-  return log.map(original => {
-    /** @type {LH.Protocol.RawEventMessage} */
-    const entry = JSON.parse(JSON.stringify(original));
-
-    switch (entry.method) {
-      case 'Network.requestWillBeSent':
-        cleanDataURI(entry.params.request);
-        cleanHeaders(entry.params.request.headers);
-        cleanResponse(entry.params.redirectResponse);
-        break;
-      case 'Network.responseReceived':
-        cleanResponse(entry.params.response);
-        break;
-    }
-
-    return entry;
-  });
-}
-
-const filteredLog = filterDevtoolsLogEvents(inputDevtoolsLog);
+const outputDevtoolsLog = minifyDevtoolsLog(inputDevtoolsLog);
 const output = `[
-${filteredLog.map(e => '  ' + JSON.stringify(e)).join(',\n')}
+${outputDevtoolsLog.map(e => '  ' + JSON.stringify(e)).join(',\n')}
 ]`;
 
 /** @param {string} s */