diff --git a/lighthouse-core/lib/minify-devtoolslog.js b/lighthouse-core/lib/minify-devtoolslog.js new file mode 100644 index 000000000000..4f6e9451e8a8 --- /dev/null +++ b/lighthouse-core/lib/minify-devtoolslog.js @@ -0,0 +1,87 @@ +/** + * @license Copyright 2019 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +/* eslint-disable no-console */ + +/** + * @fileoverview Minifies a devtools log by removing noisy header values, eliminating data URIs, etc. + */ + +const headersToKeep = new Set([ + // Request headers + 'accept', + 'accept-encoding', + 'accept-ranges', + // Response headers + 'status', + 'content-length', + 'content-type', + 'content-encoding', + 'content-range', + 'etag', + 'cache-control', + 'last-modified', + 'link', + 'x-robots-tag', +]); + +/** @param {LH.Crdp.Network.Headers} [headers] */ +function cleanHeaders(headers) { + if (!headers) return; + + for (const key of Object.keys(headers)) { + if (!headersToKeep.has(key.toLowerCase())) delete headers[key]; + } +} + +/** @param {{url: string}} obj */ +function cleanDataURI(obj) { + obj.url = obj.url.replace(/^(data:.*?base64,).*/, '$1FILLER'); +} + +/** @param {LH.Crdp.Network.Response} [response] */ +function cleanResponse(response) { + if (!response) return; + cleanDataURI(response); + cleanHeaders(response.requestHeaders); + cleanHeaders(response.headers); + response.securityDetails = undefined; + response.headersText = undefined; + response.requestHeadersText = undefined; + + /** @type {any} */ + const timing = response.timing || {}; + for (const [k, v] of Object.entries(timing)) { + if (v === -1) timing[k] = undefined; + } +} + +/** + * @param {LH.DevtoolsLog} log + * @return {LH.DevtoolsLog} + */ +function minifyDevtoolsLog(log) { + return log.map(original => { + /** @type {LH.Protocol.RawEventMessage} */ + const entry = JSON.parse(JSON.stringify(original)); + + switch (entry.method) { + case 'Network.requestWillBeSent': + cleanDataURI(entry.params.request); + cleanHeaders(entry.params.request.headers); + cleanResponse(entry.params.redirectResponse); + break; + case 'Network.responseReceived': + cleanResponse(entry.params.response); + break; + } + + return entry; + }); +} + +module.exports = {minifyDevtoolsLog}; diff --git a/lighthouse-core/lib/minify-trace.js b/lighthouse-core/lib/minify-trace.js new file mode 100644 index 000000000000..c6610a760a9f --- /dev/null +++ b/lighthouse-core/lib/minify-trace.js @@ -0,0 +1,149 @@ +/** + * @license Copyright 2019 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +/* eslint-disable no-console */ + +/** + * @fileoverview Minifies a trace by removing unnecessary events, throttling screenshots, etc. + * See the following files for necessary events: + * - lighthouse-core/computed/trace-of-tab.js + * - lighthouse-core/computed/page-dependency-graph.js + * - lighthouse-core/lib/dependency-graph/cpu-node.js + * - lighthouse-core/lib/traces/tracing-processor.js + */ + +const TracingProcessor = require('./traces/tracing-processor.js'); + +const toplevelTaskNames = new Set([ + 'RunTask', // m71+ + 'ThreadControllerImpl::RunTask', // m69-70 + 'ThreadControllerImpl::DoWork', // m66-68 + 'TaskQueueManager::ProcessTaskFromWorkQueue', // m65 and below +]); + +const traceEventsToAlwaysKeep = new Set([ + 'Screenshot', + 'TracingStartedInBrowser', + 'TracingStartedInPage', + 'navigationStart', + 'ParseAuthorStyleSheet', + 'ParseHTML', + 'PlatformResourceSendRequest', + 'ResourceSendRequest', + 'ResourceReceiveResponse', + 'ResourceFinish', + 'ResourceReceivedData', + 'EventDispatch', +]); + +const traceEventsToKeepInToplevelTask = new Set([ + // Needed for CPU node timing simulations + 'Layout', + // All of these are needed to create graph relationships + 'TimerInstall', + 'TimerFire', + 'InvalidateLayout', + 'ScheduleStyleRecalculation', + 'EvaluateScript', + 'XHRReadyStateChange', + 'FunctionCall', + 'v8.compile', + 'ParseAuthorStyleSheet', + 'ResourceSendRequest', +]); + +const traceEventsToKeepInProcess = new Set([ + ...toplevelTaskNames, + ...traceEventsToKeepInToplevelTask, + 'firstPaint', + 'firstContentfulPaint', + 'firstMeaningfulPaint', + 'firstMeaningfulPaintCandidate', + 'loadEventEnd', + 'domContentLoadedEventEnd', +]); + +/** + * @param {LH.TraceEvent[]} events + */ +function filterOutUnnecessaryTasksByNameAndDuration(events) { + const {pid} = TracingProcessor.findMainFrameIds(events); + + return events.filter(evt => { + if (toplevelTaskNames.has(evt.name) && evt.dur < 1000) return false; + if (evt.pid === pid && traceEventsToKeepInProcess.has(evt.name)) return true; + return traceEventsToAlwaysKeep.has(evt.name); + }); +} + +/** + * Filters out tasks that are not within a toplevel task. + * @param {LH.TraceEvent[]} events + */ +function filterOutOrphanedTasks(events) { + const toplevelRanges = events + .filter(evt => toplevelTaskNames.has(evt.name)) + .map(evt => [evt.ts, evt.ts + evt.dur]); + + /** @param {LH.TraceEvent} e */ + const isInToplevelTask = e => toplevelRanges.some(([start, end]) => e.ts >= start && e.ts <= end); + + return events.filter((evt, index) => { + if (!traceEventsToKeepInToplevelTask.has(evt.name)) return true; + if (!isInToplevelTask(evt)) return false; + + if (evt.ph === 'B') { + const endEvent = events.slice(index).find(e => e.name === evt.name && e.ph === 'E'); + return endEvent && isInToplevelTask(endEvent); + } else { + return true; + } + }); +} + +/** + * Throttles screenshot events in the trace to 2fps. + * @param {LH.TraceEvent[]} events + */ +function filterOutExcessiveScreenshots(events) { + const screenshotTimestamps = events.filter(evt => evt.name === 'Screenshot').map(evt => evt.ts); + + let lastScreenshotTs = -Infinity; + return events.filter(evt => { + if (evt.name !== 'Screenshot') return true; + const timeSinceLastScreenshot = evt.ts - lastScreenshotTs; + const nextScreenshotTs = screenshotTimestamps.find(ts => ts > evt.ts); + const timeUntilNextScreenshot = nextScreenshotTs ? nextScreenshotTs - evt.ts : Infinity; + const threshold = 500 * 1000; // Throttle to ~2fps + // Keep the frame if it's been more than 500ms since the last frame we kept or the next frame won't happen for at least 500ms + const shouldKeep = timeUntilNextScreenshot > threshold || timeSinceLastScreenshot > threshold; + if (shouldKeep) lastScreenshotTs = evt.ts; + return shouldKeep; + }); +} + +/** + * @param {LH.TraceEvent[]} events + */ +function filterTraceEvents(events) { + // Filter out event names we don't care about and tasks <1ms + let filtered = filterOutUnnecessaryTasksByNameAndDuration(events); + // Filter out events not inside a toplevel task + filtered = filterOutOrphanedTasks(filtered); + // Filter down the screenshots to key moments + 2fps animations + return filterOutExcessiveScreenshots(filtered); +} + +/** + * @param {LH.Trace} inputTrace + * @return {LH.Trace} + */ +function minifyTrace(inputTrace) { + return {traceEvents: filterTraceEvents(inputTrace.traceEvents)}; +} + +module.exports = {minifyTrace}; diff --git a/lighthouse-core/scripts/lantern/minify-devtoolslog.js b/lighthouse-core/scripts/lantern/minify-devtoolslog.js index 878e5243e6bd..d1e6b3a1b7fc 100755 --- a/lighthouse-core/scripts/lantern/minify-devtoolslog.js +++ b/lighthouse-core/scripts/lantern/minify-devtoolslog.js @@ -14,6 +14,7 @@ const fs = require('fs'); const path = require('path'); +const {minifyDevtoolsLog} = require('../../lib/minify-devtoolslog.js'); if (process.argv.length !== 4) { console.error('Usage $0: '); @@ -26,79 +27,9 @@ const inputDevtoolsLogRaw = fs.readFileSync(inputDevtoolsLogPath, 'utf8'); /** @type {LH.DevtoolsLog} */ const inputDevtoolsLog = JSON.parse(inputDevtoolsLogRaw); -const headersToKeep = new Set([ - // Request headers - 'accept', - 'accept-encoding', - 'accept-ranges', - // Response headers - 'status', - 'content-length', - 'content-type', - 'content-encoding', - 'content-range', - 'etag', - 'cache-control', - 'last-modified', - 'link', - 'x-robots-tag', -]); - -/** @param {Partial} [headers] */ -function cleanHeaders(headers) { - if (!headers) return; - - for (const key of Object.keys(headers)) { - if (!headersToKeep.has(key.toLowerCase())) headers[key] = undefined; - } -} - -/** @param {{url: string}} obj */ -function cleanDataURI(obj) { - obj.url = obj.url.replace(/^(data:.*?base64,).*/, '$1FILLER'); -} - -/** @param {LH.Crdp.Network.Response} [response] */ -function cleanResponse(response) { - if (!response) return; - cleanDataURI(response); - cleanHeaders(response.requestHeaders); - cleanHeaders(response.headers); - response.securityDetails = undefined; - response.headersText = undefined; - response.requestHeadersText = undefined; - - /** @type {any} */ - const timing = response.timing || {}; - for (const [k, v] of Object.entries(timing)) { - if (v === -1) timing[k] = undefined; - } -} - -/** @param {LH.DevtoolsLog} log */ -function filterDevtoolsLogEvents(log) { - return log.map(original => { - /** @type {LH.Protocol.RawEventMessage} */ - const entry = JSON.parse(JSON.stringify(original)); - - switch (entry.method) { - case 'Network.requestWillBeSent': - cleanDataURI(entry.params.request); - cleanHeaders(entry.params.request.headers); - cleanResponse(entry.params.redirectResponse); - break; - case 'Network.responseReceived': - cleanResponse(entry.params.response); - break; - } - - return entry; - }); -} - -const filteredLog = filterDevtoolsLogEvents(inputDevtoolsLog); +const outputDevtoolsLog = minifyDevtoolsLog(inputDevtoolsLog); const output = `[ -${filteredLog.map(e => ' ' + JSON.stringify(e)).join(',\n')} +${outputDevtoolsLog.map(e => ' ' + JSON.stringify(e)).join(',\n')} ]`; /** @param {string} s */ diff --git a/lighthouse-core/scripts/lantern/minify-trace.js b/lighthouse-core/scripts/lantern/minify-trace.js index c41f5d3c9f51..4f930158dbd1 100755 --- a/lighthouse-core/scripts/lantern/minify-trace.js +++ b/lighthouse-core/scripts/lantern/minify-trace.js @@ -8,17 +8,9 @@ /* eslint-disable no-console */ -/** - * @fileoverview Minifies a trace by removing unnecessary events, throttling screenshots, etc. - * See the following files for necessary events: - * - lighthouse-core/computed/trace-of-tab.js - * - lighthouse-core/computed/page-dependency-graph.js - * - lighthouse-core/lib/traces/tracing-processor.js - */ - const fs = require('fs'); const path = require('path'); -const TracingProcessor = require('../../lib/traces/tracing-processor'); +const {minifyTrace} = require('../../lib/minify-trace.js'); if (process.argv.length !== 4) { console.error('Usage $0: '); @@ -31,129 +23,16 @@ const inputTraceRaw = fs.readFileSync(inputTracePath, 'utf8'); /** @type {LH.Trace} */ const inputTrace = JSON.parse(inputTraceRaw); -const toplevelTaskNames = new Set([ - 'TaskQueueManager::ProcessTaskFromWorkQueue', - 'ThreadControllerImpl::DoWork', - 'ThreadControllerImpl::RunTask', -]); - -const traceEventsToAlwaysKeep = new Set([ - 'Screenshot', - 'TracingStartedInBrowser', - 'TracingStartedInPage', - 'navigationStart', - 'ParseAuthorStyleSheet', - 'ParseHTML', - 'PlatformResourceSendRequest', - 'ResourceSendRequest', - 'ResourceReceiveResponse', - 'ResourceFinish', - 'ResourceReceivedData', - 'EventDispatch', -]); - -const traceEventsToKeepInToplevelTask = new Set([ - 'TimerInstall', - 'TimerFire', - 'InvalidateLayout', - 'ScheduleStyleRecalculation', - 'EvaluateScript', - 'XHRReadyStateChange', - 'FunctionCall', - 'v8.compile', -]); - -const traceEventsToKeepInProcess = new Set([ - ...toplevelTaskNames, - ...traceEventsToKeepInToplevelTask, - 'firstPaint', - 'firstContentfulPaint', - 'firstMeaningfulPaint', - 'firstMeaningfulPaintCandidate', - 'loadEventEnd', - 'domContentLoadedEventEnd', -]); - -/** - * @param {LH.TraceEvent[]} events - */ -function filterOutUnnecessaryTasksByNameAndDuration(events) { - const {pid} = TracingProcessor.findMainFrameIds(events); - - return events.filter(evt => { - if (toplevelTaskNames.has(evt.name) && evt.dur < 1000) return false; - if (evt.pid === pid && traceEventsToKeepInProcess.has(evt.name)) return true; - return traceEventsToAlwaysKeep.has(evt.name); - }); -} - -/** - * Filters out tasks that are not within a toplevel task. - * @param {LH.TraceEvent[]} events - */ -function filterOutOrphanedTasks(events) { - const toplevelRanges = events - .filter(evt => toplevelTaskNames.has(evt.name)) - .map(evt => [evt.ts, evt.ts + evt.dur]); - - /** @param {LH.TraceEvent} e */ - const isInToplevelTask = e => toplevelRanges.some(([start, end]) => e.ts >= start && e.ts <= end); - - return events.filter((evt, index) => { - if (!traceEventsToKeepInToplevelTask.has(evt.name)) return true; - if (!isInToplevelTask(evt)) return false; - - if (evt.ph === 'B') { - const endEvent = events.slice(index).find(e => e.name === evt.name && e.ph === 'E'); - return endEvent && isInToplevelTask(endEvent); - } else { - return true; - } - }); -} - -/** - * Throttles screenshot events in the trace to 2fps. - * @param {LH.TraceEvent[]} events - */ -function filterOutExcessiveScreenshots(events) { - const screenshotTimestamps = events.filter(evt => evt.name === 'Screenshot').map(evt => evt.ts); - - let lastScreenshotTs = -Infinity; - return events.filter(evt => { - if (evt.name !== 'Screenshot') return true; - const timeSinceLastScreenshot = evt.ts - lastScreenshotTs; - const nextScreenshotTs = screenshotTimestamps.find(ts => ts > evt.ts); - const timeUntilNextScreenshot = nextScreenshotTs ? nextScreenshotTs - evt.ts : Infinity; - const threshold = 500 * 1000; // Throttle to ~2fps - // Keep the frame if it's been more than 500ms since the last frame we kept or the next frame won't happen for at least 500ms - const shouldKeep = timeUntilNextScreenshot > threshold || timeSinceLastScreenshot > threshold; - if (shouldKeep) lastScreenshotTs = evt.ts; - return shouldKeep; - }); -} - -/** - * @param {LH.TraceEvent[]} events - */ -function filterTraceEvents(events) { - // Filter out event names we don't care about and tasks <1ms - let filtered = filterOutUnnecessaryTasksByNameAndDuration(events); - // Filter out events not inside a toplevel task - filtered = filterOutOrphanedTasks(filtered); - // Filter down the screenshots to key moments + 2fps animations - return filterOutExcessiveScreenshots(filtered); -} - -const filteredEvents = filterTraceEvents(inputTrace.traceEvents); +const outputTrace = minifyTrace(inputTrace); const output = `{ "traceEvents": [ -${filteredEvents.map(e => ' ' + JSON.stringify(e)).join(',\n')} +${outputTrace.traceEvents.map(e => ' ' + JSON.stringify(e)).join(',\n')} ] }`; /** @param {string} s */ const size = s => Math.round(s.length / 1024) + 'kb'; +const eventDelta = inputTrace.traceEvents.length - outputTrace.traceEvents.length; console.log(`Reduced trace from ${size(inputTraceRaw)} to ${size(output)}`); -console.log(`Filtered out ${inputTrace.traceEvents.length - filteredEvents.length} trace events`); +console.log(`Filtered out ${eventDelta} trace events`); fs.writeFileSync(outputTracePath, output); diff --git a/lighthouse-core/test/lib/minify-devtoolslog-test.js b/lighthouse-core/test/lib/minify-devtoolslog-test.js new file mode 100644 index 000000000000..dbd2f3a6afef --- /dev/null +++ b/lighthouse-core/test/lib/minify-devtoolslog-test.js @@ -0,0 +1,33 @@ +/** + * @license Copyright 2019 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +const {minifyDevtoolsLog} = require('../../lib/minify-devtoolslog.js'); +const trace = require('../fixtures/traces/progressive-app-m60.json'); +const devtoolsLog = require('../fixtures/traces/progressive-app-m60.devtools.log.json'); +const MetricsAudit = require('../../audits/metrics.js'); + +/* eslint-env jest */ + +describe('minify-devtoolslog', () => { + it('has identical metrics to unminified', async () => { + const artifacts = {traces: {defaultPass: trace}, devtoolsLogs: {defaultPass: devtoolsLog}}; + const context = {settings: {throttlingMethod: 'simulate'}, computedCache: new Map()}; + const {details: {items: [before]}} = await MetricsAudit.audit(artifacts, context); + const beforeSize = JSON.stringify(devtoolsLog).length; + + const minifiedDevtoolsLog = minifyDevtoolsLog(devtoolsLog); + artifacts.devtoolsLogs.defaultPass = minifiedDevtoolsLog; + context.computedCache.clear(); // not strictly necessary, but we'll be safe + const {details: {items: [after]}} = await MetricsAudit.audit(artifacts, context); + const afterSize = JSON.stringify(minifiedDevtoolsLog).length; + + // It should reduce the size of the log. + expect(afterSize).toBeLessThan(beforeSize * 0.5); + // And not affect the metrics. + expect(after).toEqual(before); + }); +}); diff --git a/lighthouse-core/test/lib/minify-trace-test.js b/lighthouse-core/test/lib/minify-trace-test.js new file mode 100644 index 000000000000..6bc1f1abfc9b --- /dev/null +++ b/lighthouse-core/test/lib/minify-trace-test.js @@ -0,0 +1,43 @@ +/** + * @license Copyright 2019 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +const minifyTrace = require('../../lib/minify-trace.js').minifyTrace; +const trace = require('../fixtures/traces/progressive-app-m60.json'); +const devtoolsLog = require('../fixtures/traces/progressive-app-m60.devtools.log.json'); +const MetricsAudit = require('../../audits/metrics.js'); + +/* eslint-env jest */ + +describe('minify-trace', () => { + it('has identical metrics to unminified', async () => { + const artifacts = {traces: {defaultPass: trace}, devtoolsLogs: {defaultPass: devtoolsLog}}; + const context = {settings: {throttlingMethod: 'simulate'}, computedCache: new Map()}; + const {details: {items: [before]}} = await MetricsAudit.audit(artifacts, context); + const beforeSize = JSON.stringify(trace).length; + + const minifiedTrace = minifyTrace(trace); + artifacts.traces.defaultPass = minifiedTrace; + context.computedCache.clear(); + const {details: {items: [after]}} = await MetricsAudit.audit(artifacts, context); + const afterSize = JSON.stringify(minifiedTrace).length; + + for (const key of Object.keys(after)) { + // Speed Index is expected to be different because of screenshot throttling. + // Trace End can also be different if the last event was unimportant. + // There are many different `observed*` and `*Ts` versions, so remove with a regex. + if (/speedIndex|traceEnd/i.test(key)) { + delete before[key]; + delete after[key]; + } + } + + // It should greatly reduce the size of the trace. + expect(afterSize).toBeLessThan(beforeSize * 0.2); + // And not affect the metrics. + expect(after).toEqual(before); + }); +});