refactor: Simplify template metadata system by consolidating template definitions in extractor.json

.changeset/template-metadata-system-refactor.md

@@ -0,0 +1,14 @@
+---
+'@baseplate-dev/sync': patch
+'@baseplate-dev/core-generators': patch
+'@baseplate-dev/project-builder-server': patch
+---
+
+Simplify template metadata system by consolidating template definitions in extractor.json
+
+- Consolidate template definitions in extractor.json using template names as keys instead of file paths
+- Rename .template-metadata.json to .templates-info.json with simplified instance tracking
+- Remove file-id-map.json dependency and related file ID mapping logic
+- Update TemplateExtractorConfigLookup to work without file ID mapping
+- Update all template extractors and tests to use new metadata format
+- Add migration script to convert existing extractor.json files to new format

.changeset/templates-generate-cli-command.md

@@ -0,0 +1,13 @@
+---
+'@baseplate-dev/project-builder-cli': patch
+'@baseplate-dev/sync': patch
+'@baseplate-dev/project-builder-server': patch
+---
+
+Add templates generate CLI command for regenerating template files without extraction
+
+- Add `templates generate <directory> <app>` CLI command to regenerate template files from existing extractor.json configurations
+- Add `--skip-clean` option to skip cleaning output directories
+- Add `generateTemplateFiles` function in sync package that initializes plugins and writes generated files without running extraction
+- Add `generateTemplateFilesForProject` wrapper function in project-builder-server
+- Command allows manual modification of extractor.json followed by regeneration without full extraction process

package.json

@@ -11,7 +11,6 @@
     "changesets:check": "node --experimental-strip-types ./scripts/check-changesets.ts",
     "clean": "turbo run clean",
     "dev:serve": "pnpm --color --reporter-hide-prefix --filter @baseplate-dev/project-builder-cli --filter @baseplate-dev/project-builder-web --parallel dev:serve",
-    "extract:templates": "pnpm --filter @baseplate-dev/project-builder-cli extract:templates",
     "preinstall": "npx only-allow pnpm",
     "knip": "knip",
     "lint": "turbo run lint --continue",
@@ -22,6 +21,7 @@
     "metadata:check": "workspace-meta check",
     "metadata:generate": "workspace-meta generate",
     "metadata:sync": "workspace-meta sync",
+    "migrate:extractor-templates": "node migrate-extractor-templates.js",
     "precheck": "turbo run lint prettier:check typecheck --output-logs=errors-only --affected --continue && pnpm run knip",
     "prepare": "husky",
     "prettier:check": "turbo run prettier:check --continue",
@@ -33,6 +33,8 @@
     "run:morpher": "pnpm --filter @baseplate-dev/code-morph run:morpher",
     "serve": "turbo run @baseplate-dev/project-builder-cli:start serve",
     "storybook:build": "turbo run storybook:build --filter=@baseplate-dev/ui-components",
+    "templates:extract": "pnpm --filter @baseplate-dev/project-builder-cli templates:extract",
+    "templates:generate": "pnpm --filter @baseplate-dev/project-builder-cli templates:generate",
     "test": "turbo run test --continue",
     "test:affected": "turbo run test --affected --continue",
     "test:e2e": "turbo run test:e2e --continue",

packages/core-generators/src/generators/node/eslint/extractor.json

@@ -2,8 +2,7 @@
   "name": "node/eslint",
   "plugins": { "template-paths": { "skipTaskGeneration": true } },
   "templates": {
-    "eslint.config.js": {
-      "name": "eslint-config",
+    "eslint-config": {
       "type": "ts",
       "fileOptions": {
         "generatorTemplatePath": "eslint.config.js",
@@ -13,6 +12,7 @@
       "generator": "@baseplate-dev/core-generators#node/eslint",
       "importMapProviders": {},
       "pathRootRelativePath": "{package-root}/eslint.config.js",
+      "sourceFile": "eslint.config.js",
       "variables": {
         "TPL_DEFAULT_PROJECT_FILES": {},
         "TPL_DEV_DEPENDENCIES": {},

packages/core-generators/src/generators/node/ts-utils/extractor.json

@@ -1,44 +1,44 @@
 {
   "name": "node/ts-utils",
   "templates": {
-    "src/utils/arrays.ts": {
-      "name": "arrays",
+    "arrays": {
       "type": "ts",
       "fileOptions": { "kind": "singleton" },
       "generator": "@baseplate-dev/core-generators#node/ts-utils",
       "importMapProviders": {},
       "pathRootRelativePath": "{src-root}/utils/arrays.ts",
       "projectExports": { "notEmpty": {} },
+      "sourceFile": "src/utils/arrays.ts",
       "variables": {}
     },
-    "src/utils/normalize-types.ts": {
-      "name": "normalize-types",
+    "normalize-types": {
       "type": "ts",
       "fileOptions": { "kind": "singleton" },
       "generator": "@baseplate-dev/core-generators#node/ts-utils",
       "importMapProviders": {},
       "pathRootRelativePath": "{src-root}/utils/normalize-types.ts",
       "projectExports": { "NormalizeTypes": { "isTypeOnly": true } },
+      "sourceFile": "src/utils/normalize-types.ts",
       "variables": {}
     },
-    "src/utils/nulls.ts": {
-      "name": "nulls",
+    "nulls": {
       "type": "ts",
       "fileOptions": { "kind": "singleton" },
       "generator": "@baseplate-dev/core-generators#node/ts-utils",
       "importMapProviders": {},
       "pathRootRelativePath": "{src-root}/utils/nulls.ts",
       "projectExports": { "restrictObjectNulls": {} },
+      "sourceFile": "src/utils/nulls.ts",
       "variables": {}
     },
-    "src/utils/string.ts": {
-      "name": "string",
+    "string": {
       "type": "ts",
       "fileOptions": { "kind": "singleton" },
       "generator": "@baseplate-dev/core-generators#node/ts-utils",
       "importMapProviders": {},
       "pathRootRelativePath": "{src-root}/utils/string.ts",
       "projectExports": { "capitalizeString": {} },
+      "sourceFile": "src/utils/string.ts",
       "variables": {}
     }
   }

packages/core-generators/src/generators/node/typescript/typescript.generator.unit.test.ts

@@ -142,7 +142,7 @@ describe('typescriptGenerator', () => {
       expect(utilsFileOutput?.contents).toEqual(
         'export const helper = () => {};',
       );
-      expect(utilsFileOutput?.options?.templateMetadata?.generator).toEqual(
+      expect(utilsFileOutput?.options?.templateInfo?.generator).toEqual(
         'lazy-generator',
       );
 

packages/core-generators/src/generators/node/vitest/extractor.json

@@ -2,17 +2,16 @@
   "name": "node/vitest",
   "plugins": { "template-paths": { "skipTaskGeneration": true } },
   "templates": {
-    "src/tests/scripts/global-setup.ts": {
-      "name": "global-setup",
+    "global-setup": {
       "type": "ts",
       "fileOptions": { "kind": "singleton" },
       "generator": "@baseplate-dev/core-generators#node/vitest",
       "importMapProviders": {},
       "pathRootRelativePath": "{src-root}/tests/scripts/global-setup.ts",
+      "sourceFile": "src/tests/scripts/global-setup.ts",
       "variables": { "TPL_OPERATIONS": {} }
     },
-    "vitest.config.ts": {
-      "name": "vitest-config",
+    "vitest-config": {
       "type": "ts",
       "fileOptions": {
         "generatorTemplatePath": "vitest.config.ts",
@@ -22,6 +21,7 @@
       "generator": "@baseplate-dev/core-generators#node/vitest",
       "importMapProviders": {},
       "pathRootRelativePath": "{package-root}/vitest.config.ts",
+      "sourceFile": "vitest.config.ts",
       "variables": { "TPL_CONFIG": {} }
     }
   }

packages/core-generators/src/renderers/extractor/utils/deduplicate-templates.ts

@@ -18,11 +18,12 @@ export function deduplicateTemplateFileExtractorSourceFiles<
   for (const file of files.toSorted(
     (a, b) => b.modifiedTime.getTime() - a.modifiedTime.getTime(),
   )) {
-    const fileKey = `${file.metadata.generator}__${file.metadata.name}`;
+    const fileKey = `${file.generatorName}__${file.templateName}`;
     if (addedTemplates.has(fileKey)) {
       if (
-        'fileOptions' in file.metadata &&
-        (file.metadata.fileOptions as TemplateFileOptions).kind === 'singleton'
+        'fileOptions' in file.existingMetadata &&
+        (file.existingMetadata.fileOptions as TemplateFileOptions).kind ===
+          'singleton'
       ) {
         throw new Error(`Duplicate singleton template found: ${fileKey}`);
       } else {

packages/core-generators/src/renderers/raw/raw-template-file-extractor.ts

@@ -1,59 +1,61 @@
+import type { TemplateFileExtractorMetadataEntry } from '@baseplate-dev/sync';
+
 import {
   createTemplateFileExtractor,
   getGenerationConcurrencyLimit,
 } from '@baseplate-dev/sync';
 import { camelCase } from 'change-case';
 import pLimit from 'p-limit';
 
+import type { RawTemplateMetadata } from './types.js';
+
 import { templatePathsPlugin } from '../extractor/plugins/template-paths/template-paths.plugin.js';
 import { typedTemplatesFilePlugin } from '../extractor/plugins/typed-templates-file.js';
 import { deduplicateTemplateFileExtractorSourceFiles } from '../extractor/utils/deduplicate-templates.js';
 import { resolvePackagePathSpecifier } from '../extractor/utils/package-path-specifier.js';
 import { TsCodeUtils, tsImportBuilder } from '../typescript/index.js';
-import {
-  rawTemplateGeneratorTemplateMetadataSchema,
-  rawTemplateOutputTemplateMetadataSchema,
-} from './types.js';
+import { rawTemplateMetadataSchema } from './types.js';
 
 const limit = pLimit(getGenerationConcurrencyLimit());
 
 export const RawTemplateFileExtractor = createTemplateFileExtractor({
   name: 'raw',
   pluginDependencies: [templatePathsPlugin, typedTemplatesFilePlugin],
-  outputTemplateMetadataSchema: rawTemplateOutputTemplateMetadataSchema,
-  generatorTemplateMetadataSchema: rawTemplateGeneratorTemplateMetadataSchema,
+  templateMetadataSchema: rawTemplateMetadataSchema,
   extractTemplateMetadataEntries: (files, context) => {
     const deduplicatedFiles =
       deduplicateTemplateFileExtractorSourceFiles(files);
     const templatePathPlugin = context.getPlugin('template-paths');
-    return deduplicatedFiles.map(({ metadata, absolutePath }) => {
-      try {
-        const { pathRootRelativePath, generatorTemplatePath } =
-          templatePathPlugin.resolveTemplatePaths(
-            metadata.fileOptions,
-            absolutePath,
-            metadata.name,
-            metadata.generator,
-          );
+    return deduplicatedFiles.map(
+      ({ templateName, generatorName, existingMetadata, absolutePath }) => {
+        try {
+          const { pathRootRelativePath, generatorTemplatePath } =
+            templatePathPlugin.resolveTemplatePaths(
+              existingMetadata.fileOptions,
+              absolutePath,
+              templateName,
+              generatorName,
+            );
 
-        return {
-          generator: metadata.generator,
-          generatorTemplatePath,
-          sourceAbsolutePath: absolutePath,
-          metadata: {
-            name: metadata.name,
-            type: metadata.type,
-            fileOptions: metadata.fileOptions,
-            pathRootRelativePath,
-          },
-        };
-      } catch (error) {
-        throw new Error(
-          `Error extracting template metadata for ${absolutePath}: ${error instanceof Error ? error.message : String(error)}`,
-          { cause: error },
-        );
-      }
-    });
+          return {
+            generator: generatorName,
+            sourceAbsolutePath: absolutePath,
+            templateName,
+            metadata: {
+              ...existingMetadata,
+              sourceFile: generatorTemplatePath,
+              pathRootRelativePath,
+            },
+            instanceData: {},
+          } satisfies TemplateFileExtractorMetadataEntry<RawTemplateMetadata>;
+        } catch (error) {
+          throw new Error(
+            `Error extracting template metadata for ${absolutePath}: ${error instanceof Error ? error.message : String(error)}`,
+            { cause: error },
+          );
+        }
+      },
+    );
   },
   writeTemplateFiles: async (files, _context, api) => {
     await Promise.all(
@@ -65,7 +67,7 @@ export const RawTemplateFileExtractor = createTemplateFileExtractor({
             );
             await api.writeTemplateFile(
               file.generator,
-              file.generatorTemplatePath,
+              file.metadata.sourceFile,
               contents,
             );
           } catch (error) {
@@ -87,11 +89,12 @@ export const RawTemplateFileExtractor = createTemplateFileExtractor({
         context.configLookup.getExtractorConfigOrThrow(generatorName);
       const templates = context.configLookup.getTemplatesForGenerator(
         generatorName,
-        rawTemplateGeneratorTemplateMetadataSchema,
+        rawTemplateMetadataSchema,
         'raw',
       );
-      for (const { path, config } of templates) {
-        const exportName = camelCase(config.name);
+      for (const { name, config } of templates) {
+        const sourceFilePath = config.sourceFile;
+        const exportName = camelCase(name);
         const fragment = TsCodeUtils.templateWithImports([
           tsImportBuilder(['createRawTemplateFile']).from(
             resolvePackagePathSpecifier(
@@ -101,9 +104,9 @@ export const RawTemplateFileExtractor = createTemplateFileExtractor({
           ),
           tsImportBuilder().default('path').from('node:path'),
         ])`const ${exportName} = createRawTemplateFile({
-          name: '${config.name}',
+          name: '${name}',
           source: {
-            path: path.join(import.meta.dirname, '../templates/${path}'),
+            path: path.join(import.meta.dirname, '../templates/${sourceFilePath}'),
           },
           fileOptions: ${JSON.stringify(config.fileOptions)},
         });`;
@@ -116,7 +119,7 @@ export const RawTemplateFileExtractor = createTemplateFileExtractor({
         if (config.pathRootRelativePath) {
           templatePathsPlugin.registerTemplatePathEntry(
             generatorName,
-            config.name,
+            name,
             config.pathRootRelativePath,
           );
         }

packages/core-generators/src/renderers/raw/render-raw-template-action.ts

@@ -2,19 +2,14 @@ import type { BuilderAction, WriteFileOptions } from '@baseplate-dev/sync';
 
 import { readTemplateFileSourceBuffer } from '@baseplate-dev/sync';
 
-import type {
-  RawTemplateFile,
-  RawTemplateOutputTemplateMetadata,
-} from './types.js';
-
-import { RAW_TEMPLATE_TYPE } from './types.js';
+import type { RawTemplateFile } from './types.js';
 
 interface RenderRawTemplateFileActionInputBase<
   TemplateFile extends RawTemplateFile,
 > {
   template: TemplateFile;
   destination: string;
-  options?: Omit<WriteFileOptions, 'templateMetadata'>;
+  options?: Omit<WriteFileOptions, 'templateInfo'>;
 }
 
 type RenderRawTemplateFileActionInput<TTemplateFile extends RawTemplateFile> =
@@ -50,16 +45,6 @@ export function renderRawTemplateFileAction<
           isInstance: template.fileOptions.kind === 'instance',
         });
 
-      const templateMetadata: RawTemplateOutputTemplateMetadata | undefined =
-        'path' in template.source
-          ? {
-              generator: builder.generatorInfo.name,
-              type: RAW_TEMPLATE_TYPE,
-              name: template.name,
-              fileOptions: template.fileOptions,
-            }
-          : undefined;
-
       builder.writeFile({
         id: fileId,
         destination,
@@ -68,7 +53,11 @@ export function renderRawTemplateFileAction<
           skipFormatting: true,
           ...options,
         },
-        templateMetadata: shouldWriteMetadata ? templateMetadata : undefined,
+        templateInfo: {
+          template: template.name,
+          generator: builder.generatorInfo.name,
+          instanceData: shouldWriteMetadata ? {} : undefined,
+        },
       });
     },
   };