diff --git a/.changeset/deep-states-talk.md b/.changeset/deep-states-talk.md new file mode 100644 index 000000000000..ee4760061141 --- /dev/null +++ b/.changeset/deep-states-talk.md @@ -0,0 +1,5 @@ +--- +'astro': major +--- + +Updates how schema types are inferred for content loaders with schemas (Loader API) - ([v6 upgrade guidance](https://deploy-preview-12322--astro-docs-2.netlify.app/en/guides/upgrade-to/v6/TODO:)) diff --git a/.changeset/fresh-rocks-sing.md b/.changeset/fresh-rocks-sing.md new file mode 100644 index 000000000000..072b42123a22 --- /dev/null +++ b/.changeset/fresh-rocks-sing.md @@ -0,0 +1,5 @@ +--- +'astro': major +--- + +Removes the option to define dynamic schemas in content loaders as functions and adds a new equivalent `createSchema()` property (Loader API) - ([v6 upgrade guidance](https://deploy-preview-12322--astro-docs-2.netlify.app/en/guides/upgrade-to/v6/TODO:)) diff --git a/packages/astro/package.json b/packages/astro/package.json index 1f57a43facdc..39f00435a119 100644 --- a/packages/astro/package.json +++ b/packages/astro/package.json @@ -172,8 +172,7 @@ "yargs-parser": "^21.1.1", "yocto-spinner": "^0.2.3", "zod": "^3.25.76", - "zod-to-json-schema": "^3.24.6", - "zod-to-ts": "^1.2.0" + "zod-to-json-schema": "^3.24.6" }, "optionalDependencies": { "sharp": "^0.34.0" diff --git a/packages/astro/src/content/config.ts b/packages/astro/src/content/config.ts index e6424b5b7b10..a148d8ced405 100644 --- a/packages/astro/src/content/config.ts +++ b/packages/astro/src/content/config.ts @@ -71,29 +71,6 @@ export type BaseSchema = ZodType; export type SchemaContext = { image: ImageFunction }; -type ContentLayerConfig = { - type?: 'content_layer'; - schema?: S | ((context: SchemaContext) => S); - loader: - | Loader - | (() => - | Array - | Promise> - | Record & { id?: string }> - | Promise & { id?: string }>>); -}; - -type DataCollectionConfig = { - type: 'data'; - schema?: S | ((context: SchemaContext) => S); -}; - -type ContentCollectionConfig = { - type?: 'content'; - schema?: S | ((context: SchemaContext) => S); - loader?: never; -}; - export type LiveCollectionConfig< L extends LiveLoader, S extends BaseSchema | undefined = undefined, @@ -103,10 +80,22 @@ export type LiveCollectionConfig< loader: L; }; -export type CollectionConfig = - | ContentCollectionConfig - | DataCollectionConfig - | ContentLayerConfig; +type LoaderConstraint = + | Loader + | (() => + | Array + | Promise> + | Record & { id?: string }> + | Promise & { id?: string }>>); + +export type CollectionConfig< + TSchema extends BaseSchema, + TLoader extends LoaderConstraint<{ id: string }>, +> = { + type?: 'content_layer'; + schema?: TSchema | ((context: SchemaContext) => TSchema); + loader: TLoader; +}; export function defineLiveCollection< L extends LiveLoader, @@ -167,9 +156,10 @@ export function defineLiveCollection< return config; } -export function defineCollection( - config: CollectionConfig, -): CollectionConfig { +export function defineCollection< + TSchema extends BaseSchema, + TLoader extends LoaderConstraint<{ id: string }>, +>(config: CollectionConfig): CollectionConfig { const importerFilename = getImporterFilename(); if (importerFilename?.includes('live.config')) { diff --git a/packages/astro/src/content/content-layer.ts b/packages/astro/src/content/content-layer.ts index 17335f54f1e5..583f104e8e4c 100644 --- a/packages/astro/src/content/content-layer.ts +++ b/packages/astro/src/content/content-layer.ts @@ -253,8 +253,8 @@ class ContentLayer { if (!schema && typeof collection.loader === 'object') { schema = collection.loader.schema; - if (typeof schema === 'function') { - schema = await schema(); + if (!schema && collection.loader.createSchema) { + ({ schema } = await collection.loader.createSchema()); } } diff --git a/packages/astro/src/content/loaders/types.ts b/packages/astro/src/content/loaders/types.ts index 6939357a5620..8b7449751f66 100644 --- a/packages/astro/src/content/loaders/types.ts +++ b/packages/astro/src/content/loaders/types.ts @@ -49,14 +49,24 @@ export interface LoaderContext { entryTypes: Map; } -export interface Loader { +export type Loader = { /** Unique name of the loader, e.g. the npm package name */ name: string; /** Do the actual loading of the data */ load: (context: LoaderContext) => Promise; - /** Optionally, define the schema of the data. Will be overridden by user-defined schema */ - schema?: ZodSchema | Promise | (() => ZodSchema | Promise); -} +} & ( + | { + /** Optionally, define the schema of the data. Will be overridden by user-defined schema */ + schema?: ZodSchema; + } + | { + /** Optionally, provide a function to dynamically provide a schema. Will be overridden by user-defined schema */ + createSchema?: () => Promise<{ + schema: ZodSchema; + types: string; + }>; + } +); export interface LoadEntryContext { filter: TEntryFilter extends never ? { id: string } : TEntryFilter; diff --git a/packages/astro/src/content/types-generator.ts b/packages/astro/src/content/types-generator.ts index aae315255232..28124f9cfcac 100644 --- a/packages/astro/src/content/types-generator.ts +++ b/packages/astro/src/content/types-generator.ts @@ -17,6 +17,7 @@ import type { Logger } from '../core/logger/core.js'; import { isRelativePath } from '../core/path.js'; import type { AstroSettings } from '../types/astro.js'; import type { ContentEntryType } from '../types/public/content.js'; +import type { InjectedType } from '../types/public/integrations.js'; import { COLLECTIONS_DIR, CONTENT_LAYER_TYPE, @@ -355,13 +356,13 @@ function normalizeConfigPath(from: string, to: string) { return `"${isRelativePath(configPath) ? '' : './'}${normalizedPath}"` as const; } -const schemaCache = new Map(); +const createSchemaResultCache = new Map(); -async function getContentLayerSchema( +async function getCreateSchemaResult( collection: ContentConfig['collections'][T], collectionKey: T, -): Promise { - const cached = schemaCache.get(collectionKey); +) { + const cached = createSchemaResultCache.get(collectionKey); if (cached) { return cached; } @@ -369,44 +370,54 @@ async function getContentLayerSchema( + collection: ContentConfig['collections'][T], + collectionKey: T, +): Promise { + if (collection?.type !== CONTENT_LAYER_TYPE || typeof collection.loader === 'function') { + return; + } + if (collection.loader.schema) { + return collection.loader.schema; + } + const result = await getCreateSchemaResult(collection, collectionKey); + return result?.schema; +} + async function typeForCollection( collection: ContentConfig['collections'][T] | undefined, collectionKey: T, -): Promise { +): Promise<{ type: string; injectedType?: InjectedType }> { if (collection?.schema) { - return `InferEntrySchema<${collectionKey}>`; + return { type: `InferEntrySchema<${collectionKey}>` }; } - if (!collection?.type) { - return 'any'; + if (!collection?.type || typeof collection.loader === 'function') { + return { type: 'any' }; } - const schema = await getContentLayerSchema(collection, collectionKey); - if (!schema) { - return 'any'; + if (collection.loader.schema) { + return { type: `InferLoaderSchema<${collectionKey}>` }; } - try { - const zodToTs = await import('zod-to-ts'); - const ast = zodToTs.zodToTs(schema); - return zodToTs.printNode(ast.node); - } catch (err: any) { - // zod-to-ts is sad if we don't have TypeScript installed, but that's fine as we won't be needing types in that case - if (err.message.includes("Cannot find package 'typescript'")) { - return 'any'; - } - throw err; + const result = await getCreateSchemaResult(collection, collectionKey); + if (!result) { + return { type: 'any' }; } + const base = `loaders/${collectionKey.slice(1, -1)}`; + return { + type: `import("./${base}.js").Entry`, + injectedType: { + filename: `${base}.ts`, + content: result.types, + }, + }; } async function writeContentFiles({ @@ -473,7 +484,21 @@ async function writeContentFiles({ return; } - const dataType = await typeForCollection(collectionConfig, collectionKey); + const { type: dataType, injectedType } = await typeForCollection( + collectionConfig, + collectionKey, + ); + + if (injectedType) { + if (settings.injectedTypes.some((t) => t.filename === CONTENT_TYPES_FILE)) { + // If it's the first time, we inject types the usual way. sync() will handle creating files and references. If it's not the first time, we just override the dts content + const url = new URL(injectedType.filename, settings.dotAstroDir); + await fs.promises.mkdir(path.dirname(fileURLToPath(url)), { recursive: true }); + await fs.promises.writeFile(url, injectedType.content, 'utf-8'); + } else { + settings.injectedTypes.push(injectedType); + } + } dataTypesStr += `${collectionKey}: Record;\n`; @@ -500,7 +525,16 @@ async function writeContentFiles({ const key = JSON.parse(collectionKey); contentCollectionManifest.collections.push({ - hasSchema: Boolean(collectionConfig?.schema || schemaCache.has(collectionKey)), + hasSchema: Boolean( + // Is there a user provided schema or + collectionConfig?.schema || + // Is it a loader object and + (typeof collectionConfig?.loader !== 'function' && + // Is it a loader static schema or + (collectionConfig?.loader.schema || + // is it a loader dynamic schema + createSchemaResultCache.has(collectionKey))), + ), name: key, }); diff --git a/packages/astro/src/content/utils.ts b/packages/astro/src/content/utils.ts index cd5c0e477e1c..0cc4bf33d713 100644 --- a/packages/astro/src/content/utils.ts +++ b/packages/astro/src/content/utils.ts @@ -7,7 +7,7 @@ import colors from 'piccolore'; import type { PluginContext } from 'rollup'; import type { RunnableDevEnvironment } from 'vite'; import xxhash from 'xxhash-wasm'; -import { z } from 'zod'; +import { type ZodSchema, z } from 'zod'; import { AstroError, AstroErrorData, errorMap, MarkdownError } from '../core/errors/index.js'; import { isYAMLException } from '../core/errors/utils.js'; import type { Logger } from '../core/logger/core.js'; @@ -60,14 +60,39 @@ const collectionConfigParser = z.union([ z.function(), z.object({ name: z.string(), - load: z.function().args(z.custom()).returns( - z.custom<{ - schema?: any; - types?: string; - } | void>(), - ), - schema: z.any().optional(), - render: z.function(z.tuple([z.any()], z.unknown())).optional(), + load: z.function().args(z.custom()).returns(z.promise(z.void())), + schema: z + .any() + .transform((v) => { + if (typeof v === 'function') { + console.warn( + `Your loader's schema is defined using a function. This is no longer supported and the schema will be ignored. Please update your loader to use the \`createSchema()\` utility instead, or report this to the loader author. In a future major version, this will cause the loader to break entirely.`, + ); + return undefined; + } + return v; + }) + .superRefine((v, ctx) => { + if (v !== undefined && !('_def' in v)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Invalid Zod schema', + }); + return z.NEVER; + } + }) + .optional(), + createSchema: z + .function() + .returns( + z.promise( + z.object({ + schema: z.custom((v) => '_def' in v), + types: z.string(), + }), + ), + ) + .optional(), }), ]), }), diff --git a/packages/astro/templates/content/types.d.ts b/packages/astro/templates/content/types.d.ts index 12d008e3d6e3..c89b7748a075 100644 --- a/packages/astro/templates/content/types.d.ts +++ b/packages/astro/templates/content/types.d.ts @@ -104,6 +104,12 @@ declare module 'astro:content' { type InferEntrySchema = import('astro/zod').infer< ReturnTypeOrOriginal['schema']> >; + type InferLoaderSchema< + C extends keyof DataEntryMap, + L = Required['loader'], + > = L extends { schema: import('astro/zod').ZodSchema } + ? import('astro/zod').infer['loader']['schema']> + : any; type DataEntryMap = { // @@DATA_ENTRY_MAP@@ diff --git a/packages/astro/test/astro-sync.test.js b/packages/astro/test/astro-sync.test.js index 43a00b355fa2..56e4a766c2b8 100644 --- a/packages/astro/test/astro-sync.test.js +++ b/packages/astro/test/astro-sync.test.js @@ -11,6 +11,8 @@ const createFixture = () => { let astroFixture; /** @type {Record} */ const writtenFiles = {}; + /** @type {Array} */ + const warnLogs = []; /** * @param {string} path @@ -50,13 +52,23 @@ const createFixture = () => { }, }; - await astroFixture.sync( - { root: fileURLToPath(astroFixture.config.root) }, - { - // @ts-ignore - fs: fsMock, - }, - ); + const originalWarn = console.warn; + console.warn = (message) => { + originalWarn(message); + warnLogs.push(message); + }; + + try { + await astroFixture.sync( + { root: fileURLToPath(astroFixture.config.root) }, + { + // @ts-ignore + fs: fsMock, + }, + ); + } finally { + console.error = originalWarn; + } }, /** @param {string} path */ thenFileShouldExist(path) { @@ -102,6 +114,16 @@ const createFixture = () => { assert.fail(`${path} is not valid TypeScript. Error: ${error.message}`); } }, + /** + * @param {string} message + */ + thenWarnLogsInclude(message) { + if (warnLogs.length === 0) { + assert.fail('No error log'); + } + const index = warnLogs.findIndex((log) => log.includes(message)); + assert.equal(index !== -1, true, 'No error log found'); + }, }; }; @@ -170,6 +192,13 @@ describe('astro sync', () => { 'Types file does not include empty collection type', ); }); + + it('fails when using a loader schema function', async () => { + await fixture.load('./fixtures/content-layer-loader-schema-function/'); + fixture.clean(); + await fixture.whenSyncing(); + fixture.thenWarnLogsInclude("Your loader's schema is defined using a function."); + }); }); describe('astro:env', () => { diff --git a/packages/astro/test/fixtures/content-layer-loader-schema-function/astro.config.mjs b/packages/astro/test/fixtures/content-layer-loader-schema-function/astro.config.mjs new file mode 100644 index 000000000000..86dbfb924824 --- /dev/null +++ b/packages/astro/test/fixtures/content-layer-loader-schema-function/astro.config.mjs @@ -0,0 +1,3 @@ +import { defineConfig } from 'astro/config'; + +export default defineConfig({}); diff --git a/packages/astro/test/fixtures/content-layer-loader-schema-function/package.json b/packages/astro/test/fixtures/content-layer-loader-schema-function/package.json new file mode 100644 index 000000000000..4467e4797a5b --- /dev/null +++ b/packages/astro/test/fixtures/content-layer-loader-schema-function/package.json @@ -0,0 +1,8 @@ +{ + "name": "@test/content-layer-loader-schema-function", + "version": "0.0.0", + "private": true, + "dependencies": { + "astro": "workspace:*" + } +} diff --git a/packages/astro/test/fixtures/content-layer-loader-schema-function/src/content.config.ts b/packages/astro/test/fixtures/content-layer-loader-schema-function/src/content.config.ts new file mode 100644 index 000000000000..f94f7cd93f53 --- /dev/null +++ b/packages/astro/test/fixtures/content-layer-loader-schema-function/src/content.config.ts @@ -0,0 +1,14 @@ +import { defineCollection, z } from 'astro:content'; +import { glob } from 'astro/loaders'; + +const blog = defineCollection({ + loader: { + name: 'test', + load: async () => {}, + schema: () => z.object() + } +}); + +export const collections = { + blog, +}; diff --git a/packages/astro/test/fixtures/content-layer/src/content.config.ts b/packages/astro/test/fixtures/content-layer/src/content.config.ts index 242dee6baced..4ec376d3d31a 100644 --- a/packages/astro/test/fixtures/content-layer/src/content.config.ts +++ b/packages/astro/test/fixtures/content-layer/src/content.config.ts @@ -227,14 +227,22 @@ const increment = defineCollection({ rendered: await renderMarkdown(markdownContent) }); }, - // Example of a loader that returns an async schema function - schema: async () => - z.object({ + createSchema: async () => { + return { + schema: z.object({ lastValue: z.number(), lastUpdated: z.date(), refreshContextData: z.record(z.unknown()).optional(), slug: z.string().optional(), }), + types: /* ts */`export interface Entry { + lastValue: number; + lastUpdated: Date; + refreshContextData?: Record | undefined; + slug?: string | undefined; +}` + } + } }, }); diff --git a/packages/astro/test/fixtures/content-layer/src/loaders/post-loader.ts b/packages/astro/test/fixtures/content-layer/src/loaders/post-loader.ts index 232819ce2585..c9047d961aac 100644 --- a/packages/astro/test/fixtures/content-layer/src/loaders/post-loader.ts +++ b/packages/astro/test/fixtures/content-layer/src/loaders/post-loader.ts @@ -31,15 +31,23 @@ export function loader(config:PostLoaderConfig): Loader { } meta.set('lastSynced', String(Date.now())); }, - schema: async () => { + createSchema: async () => { // Simulate a delay await new Promise((resolve) => setTimeout(resolve, 1000)); - return z.object({ + return { + schema: z.object({ title: z.string(), body: z.string(), userId: z.number(), id: z.number(), - }); + }), + types: /* ts */`export interface Entry { + title: string; + body: string; + userId: number; + id: number; +}` + } } }; } diff --git a/packages/astro/types/content.d.ts b/packages/astro/types/content.d.ts index ec0f5596974c..ec034806788f 100644 --- a/packages/astro/types/content.d.ts +++ b/packages/astro/types/content.d.ts @@ -8,6 +8,7 @@ declare module 'astro:content' { BaseSchema, SchemaContext, } from 'astro/content/config'; + export { defineLiveCollection, defineCollection } from 'astro/content/config'; // TODO: remove in Astro 7 /** @@ -17,17 +18,6 @@ declare module 'astro:content' { */ export const z = zod.z; - export function defineLiveCollection< - L extends import('astro/loaders').LiveLoader, - S extends import('astro/content/config').BaseSchema | undefined = undefined, - >( - config: import('astro/content/config').LiveCollectionConfig, - ): import('astro/content/config').LiveCollectionConfig; - - export function defineCollection( - config: import('astro/content/config').CollectionConfig, - ): import('astro/content/config').CollectionConfig; - /** Run `astro dev` or `astro sync` to generate high fidelity types */ export const getEntryBySlug: (...args: any[]) => any; /** Run `astro dev` or `astro sync` to generate high fidelity types */ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a6b8b00dca73..a13fdbe8db81 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -643,9 +643,6 @@ importers: zod-to-json-schema: specifier: ^3.24.6 version: 3.24.6(zod@3.25.76) - zod-to-ts: - specifier: ^1.2.0 - version: 1.2.0(typescript@5.9.3)(zod@3.25.76) devDependencies: '@astrojs/check': specifier: workspace:* @@ -2730,6 +2727,12 @@ importers: specifier: workspace:* version: link:../../.. + packages/astro/test/fixtures/content-layer-loader-schema-function: + dependencies: + astro: + specifier: workspace:* + version: link:../../.. + packages/astro/test/fixtures/content-layer-markdoc: dependencies: '@astrojs/markdoc':