Telemetry: Fix storybook version and add test run events

code/addons/vitest/src/preset.ts

@@ -180,11 +180,6 @@ export const experimental_serverChannel = async (channel: Channel, options: Opti
 
   if (!core.disableTelemetry) {
     const enableCrashReports = core.enableCrashReports || options.enableCrashReports;
-    const packageJsonPath = require.resolve('@storybook/addon-vitest/package.json');
-
-    const { version: addonVersion } = JSON.parse(
-      readFileSync(packageJsonPath, { encoding: 'utf-8' })
-    );
 
     channel.on(STORYBOOK_ADDON_TEST_CHANNEL, (event: Event) => {
       telemetry('addon-test', {
@@ -193,14 +188,12 @@ export const experimental_serverChannel = async (channel: Channel, options: Opti
           ...event.payload,
           storyId: oneWayHash(event.payload.storyId),
         },
-        addonVersion,
       });
     });
 
     store.subscribe('TOGGLE_WATCHING', async (event) => {
       await telemetry('addon-test', {
         watchMode: event.payload.to,
-        addonVersion,
       });
     });
     store.subscribe('TEST_RUN_COMPLETED', async (event) => {
@@ -216,15 +209,13 @@ export const experimental_serverChannel = async (channel: Channel, options: Opti
               return sanitizeError(errorWithoutStacks);
             }),
           }),
-        addonVersion,
       });
     });
 
     if (enableCrashReports) {
       store.subscribe('FATAL_ERROR', async (event) => {
         await telemetry('addon-test', {
           fatalError: cleanPaths(event.payload.error.message),
-          addonVersion,
         });
       });
     }

code/addons/vitest/src/vitest-plugin/index.ts

@@ -18,6 +18,7 @@ import type {
 } from 'storybook/internal/core-server';
 import { readConfig, vitestTransform } from 'storybook/internal/csf-tools';
 import { MainFileMissingError } from 'storybook/internal/server-errors';
+import { telemetry } from 'storybook/internal/telemetry';
 import { oneWayHash } from 'storybook/internal/telemetry';
 import type { Presets } from 'storybook/internal/types';
 
@@ -144,13 +145,15 @@ export const storybookTest = async (options?: UserOptions): Promise<Plugin[]> =>
     viteConfigFromStorybook,
     staticDirs,
     previewLevelTags,
+    core,
   ] = await Promise.all([
     getStoryGlobsAndFiles(presets, directories),
     presets.apply('framework', undefined),
     presets.apply('env', {}),
     presets.apply<{ plugins?: Plugin[] }>('viteFinal', {}),
     presets.apply('staticDirs', []),
     extractTagsFromPreview(finalOptions.configDir),
+    presets.apply('core'),
   ]);
 
   const pluginsToIgnore = [
@@ -361,6 +364,23 @@ export const storybookTest = async (options?: UserOptions): Promise<Plugin[]> =>
     },
     configureVitest(context) {
       context.vitest.config.coverage.exclude.push('storybook-static');
+
+      const disableTelemetryVar =
+        process.env.STORYBOOK_DISABLE_TELEMETRY &&
+        process.env.STORYBOOK_DISABLE_TELEMETRY !== 'false';
+      if (!core?.disableTelemetry && !disableTelemetryVar) {
+        // NOTE: we start telemetry immediately but do not wait on it. Typically it should complete
+        // before the tests do. If not we may miss the event, we are OK with that.
+        telemetry(
+          'test-run',
+          {
+            runner: 'vitest',
+            watch: context.vitest.config.watch,
+            coverage: !!context.vitest.config.coverage?.enabled,
+          },
+          { configDir: finalOptions.configDir }
+        );
+      }
     },
     async configureServer(server) {
       if (staticDirs) {

code/core/src/core-server/build-static.ts

@@ -105,6 +105,13 @@ export async function buildStaticStandalone(options: BuildStaticStandaloneOption
     presets.apply('docs'),
   ]);
 
+  const invokedBy = process.env.STORYBOOK_INVOKED_BY;
+  if (!core?.disableTelemetry && invokedBy) {
+    // NOTE: we don't await this event to avoid slowing things down.
+    // This could result in telemetry events being lost.
+    telemetry('test-run', { runner: invokedBy, watch: false }, { configDir: options.configDir });
+  }
+
   const fullOptions: Options = {
     ...options,
     presets,
@@ -205,7 +212,8 @@ export async function buildStaticStandalone(options: BuildStaticStandaloneOption
   ]);
 
   // Now the code has successfully built, we can count this as a 'dev' event.
-  if (!core?.disableTelemetry) {
+  // NOTE: we don't send the 'build' event for test runs as we want to be as fast as possible
+  if (!core?.disableTelemetry && !options.test) {
     effects.push(
       initializedStoryIndexGenerator.then(async (generator) => {
         const storyIndex = await generator?.getIndex();
@@ -217,6 +225,7 @@ export async function buildStaticStandalone(options: BuildStaticStandaloneOption
             storyIndex: summarizeIndex(storyIndex),
           });
         }
+
         await telemetry('build', payload, { configDir: options.configDir });
       })
     );

code/core/src/core-server/presets/common-preset.ts

@@ -205,7 +205,7 @@ export const experimental_serverAPI = (extension: Record<string, Function>, opti
  */
 export const core = async (existing: CoreConfig, options: Options): Promise<CoreConfig> => ({
   ...existing,
-  disableTelemetry: options.disableTelemetry === true || options.test === true,
+  disableTelemetry: options.disableTelemetry === true,
   enableCrashReports:
     options.enableCrashReports || optionalEnvToBoolean(process.env.STORYBOOK_ENABLE_CRASH_REPORTS),
 });

code/core/src/telemetry/storybook-metadata.ts

@@ -100,6 +100,7 @@ export const computeStorybookMetadata = async ({
     'msw',
     'miragejs',
     'sinon',
+    'chromatic',
   ];
   const testPackageDeps = Object.keys(allDependencies).filter((dep) =>
     testPackages.find((pkg) => dep.includes(pkg))

code/core/src/telemetry/types.ts

@@ -26,7 +26,8 @@ export type EventType =
   | 'testing-module-watch-mode'
   | 'testing-module-completed-report'
   | 'testing-module-crash-report'
-  | 'addon-test';
+  | 'addon-test'
+  | 'test-run';
 
 export interface Dependency {
   version: string | undefined;

scripts/event-log-checker.ts

@@ -8,8 +8,14 @@ import { esMain } from './utils/esmain';
 
 const PORT = process.env.PORT || 6007;
 
-const eventTypeExpectations = {
+type EventType = 'build' | 'test-run';
+type EventDefinition = {
+  noBoot?: boolean;
+};
+
+const eventTypeDefinitions: Record<EventType, EventDefinition> = {
   build: {},
+  'test-run': { noBoot: true },
 };
 
 async function run() {
@@ -29,9 +35,9 @@ async function run() {
       );
     }
 
-    const expectation = eventTypeExpectations[eventType as keyof typeof eventTypeExpectations];
+    const definition = eventTypeDefinitions[eventType as EventType];
 
-    if (!expectation) {
+    if (!definition) {
       throw new Error(`Unexpected eventType '${eventType}'`);
     }
 
@@ -43,23 +49,38 @@ async function run() {
 
     const events: any = await (await fetch(`http://localhost:${PORT}/event-log`)).json();
 
-    test('Should log 2 events', () => {
-      assert.equal(
-        events.length,
-        2,
-        `Expected 2 events but received ${
-          events.length
-        } instead. The following events were logged: ${JSON.stringify(events)}`
-      );
-    });
+    if (definition.noBoot) {
+      test('Should log 1 event', () => {
+        assert.equal(
+          events.length,
+          1,
+          `Expected 1 event but received ${
+            events.length
+          } instead. The following events were logged: ${JSON.stringify(events)}`
+        );
+      });
+    } else {
+      test('Should log 2 events', () => {
+        assert.equal(
+          events.length,
+          2,
+          `Expected 2 events but received ${
+            events.length
+          } instead. The following events were logged: ${JSON.stringify(events)}`
+        );
+      });
+    }
 
-    const [bootEvent, mainEvent] = events;
+    if (events.length === 0) throw new Error('No events were logged');
+    const [bootEvent, mainEvent] = definition.noBoot ? [null, events[0]] : events;
 
     const storybookVersion = versions.storybook;
-    test('boot event should have cliVersion and storybookVersion in context', () => {
-      assert.equal(bootEvent.context.cliVersion, storybookVersion);
-      assert.equal(bootEvent.context.storybookVersion, storybookVersion);
-    });
+    if (bootEvent) {
+      test('boot event should have cliVersion and storybookVersion in context', () => {
+        assert.equal(bootEvent.context.cliVersion, storybookVersion);
+        assert.equal(bootEvent.context.storybookVersion, storybookVersion);
+      });
+    }
 
     test(`main event should have storybookVersion in context`, () => {
       assert.equal(mainEvent.context.storybookVersion, storybookVersion);
@@ -69,15 +90,21 @@ async function run() {
       assert.equal(mainEvent.metadata.storybookVersion, storybookVersion);
     });
 
-    test(`Should log a boot event with a payload of type ${eventType}`, () => {
-      assert.equal(bootEvent.eventType, 'boot');
-      assert.equal(bootEvent.payload?.eventType, eventType);
-    });
+    if (bootEvent) {
+      test(`Should log a boot event with a payload of type ${eventType}`, () => {
+        assert.equal(bootEvent.eventType, 'boot');
+        assert.equal(bootEvent.payload?.eventType, eventType);
+      });
+    }
 
     test(`main event should be ${eventType} and contain correct id and session id`, () => {
       assert.equal(mainEvent.eventType, eventType);
-      assert.notEqual(mainEvent.eventId, bootEvent.eventId);
-      assert.equal(mainEvent.sessionId, bootEvent.sessionId);
+      assert.ok(typeof mainEvent.eventId === 'string');
+      assert.ok(typeof mainEvent.sessionId === 'string');
+      if (bootEvent) {
+        assert.notEqual(mainEvent.eventId, bootEvent.eventId);
+        assert.equal(mainEvent.sessionId, bootEvent.sessionId);
+      }
     });
 
     test(`main event should contain anonymousId properly hashed`, () => {

scripts/tasks/sandbox-parts.ts

@@ -599,7 +599,8 @@ export async function addExtraDependencies({
   debug: boolean;
   extraDeps?: string[];
 }) {
-  const extraDevDeps = ['@storybook/test-runner@next'];
+  // FIXME: revert back to `next` once https://github.com/storybookjs/test-runner/pull/560 is merged
+  const extraDevDeps = ['@storybook/[email protected]'];
 
   if (debug) {
     logger.log('\uD83C\uDF81 Adding extra dev deps', extraDevDeps);