feat: loader.createSchema()

.changeset/deep-states-talk.md

@@ -0,0 +1,5 @@
+---
+'astro': major
+---
+
+Updates how schema types are inferred for content loaders with schemas (Loader API) - ([v6 upgrade guidance](https://deploy-preview-12322--astro-docs-2.netlify.app/en/guides/upgrade-to/v6/TODO:))

.changeset/fresh-rocks-sing.md

@@ -0,0 +1,5 @@
+---
+'astro': major
+---
+
+Removes the option to define dynamic schemas in content loaders as functions and adds a new equivalent `createSchema()` property (Loader API) - ([v6 upgrade guidance](https://deploy-preview-12322--astro-docs-2.netlify.app/en/guides/upgrade-to/v6/TODO:))

packages/astro/package.json

@@ -168,8 +168,7 @@
     "yargs-parser": "^21.1.1",
     "yocto-spinner": "^0.2.3",
     "zod": "^3.25.76",
-    "zod-to-json-schema": "^3.24.6",
-    "zod-to-ts": "^1.2.0"
+    "zod-to-json-schema": "^3.24.6"
   },
   "optionalDependencies": {
     "sharp": "^0.34.0"

packages/astro/src/content/config.ts

@@ -71,29 +71,6 @@ export type BaseSchema = ZodType;
 
 export type SchemaContext = { image: ImageFunction };
 
-type ContentLayerConfig<S extends BaseSchema, TData extends { id: string } = { id: string }> = {
-	type?: 'content_layer';
-	schema?: S | ((context: SchemaContext) => S);
-	loader:
-		| Loader
-		| (() =>
-				| Array<TData>
-				| Promise<Array<TData>>
-				| Record<string, Omit<TData, 'id'> & { id?: string }>
-				| Promise<Record<string, Omit<TData, 'id'> & { id?: string }>>);
-};
-
-type DataCollectionConfig<S extends BaseSchema> = {
-	type: 'data';
-	schema?: S | ((context: SchemaContext) => S);
-};
-
-type ContentCollectionConfig<S extends BaseSchema> = {
-	type?: 'content';
-	schema?: S | ((context: SchemaContext) => S);
-	loader?: never;
-};
-
 export type LiveCollectionConfig<
 	L extends LiveLoader,
 	S extends BaseSchema | undefined = undefined,
@@ -103,10 +80,22 @@ export type LiveCollectionConfig<
 	loader: L;
 };
 
-export type CollectionConfig<S extends BaseSchema> =
-	| ContentCollectionConfig<S>
-	| DataCollectionConfig<S>
-	| ContentLayerConfig<S>;
+type LoaderConstraint<TData extends { id: string }> =
+	| Loader
+	| (() =>
+			| Array<TData>
+			| Promise<Array<TData>>
+			| Record<string, Omit<TData, 'id'> & { id?: string }>
+			| Promise<Record<string, Omit<TData, 'id'> & { id?: string }>>);
+
+export type CollectionConfig<
+	TSchema extends BaseSchema,
+	TLoader extends LoaderConstraint<{ id: string }>,
+> = {
+	type?: 'content_layer';
+	schema?: TSchema | ((context: SchemaContext) => TSchema);
+	loader: TLoader;
+};
 
 export function defineLiveCollection<
 	L extends LiveLoader,
@@ -167,9 +156,10 @@ export function defineLiveCollection<
 	return config;
 }
 
-export function defineCollection<S extends BaseSchema>(
-	config: CollectionConfig<S>,
-): CollectionConfig<S> {
+export function defineCollection<
+	TSchema extends BaseSchema,
+	TLoader extends LoaderConstraint<{ id: string }>,
+>(config: CollectionConfig<TSchema, TLoader>): CollectionConfig<TSchema, TLoader> {
 	const importerFilename = getImporterFilename();
 
 	if (importerFilename?.includes('live.config')) {

packages/astro/src/content/content-layer.ts

@@ -253,8 +253,8 @@ class ContentLayer {
 
 				if (!schema && typeof collection.loader === 'object') {
 					schema = collection.loader.schema;
-					if (typeof schema === 'function') {
-						schema = await schema();
+					if (!schema && collection.loader.createSchema) {
+						({ schema } = await collection.loader.createSchema());
 					}
 				}
 

packages/astro/src/content/loaders/types.ts

@@ -49,14 +49,24 @@ export interface LoaderContext {
 	entryTypes: Map<string, ContentEntryType>;
 }
 
-export interface Loader {
+export type Loader = {
 	/** Unique name of the loader, e.g. the npm package name */
 	name: string;
 	/** Do the actual loading of the data */
 	load: (context: LoaderContext) => Promise<void>;
-	/** Optionally, define the schema of the data. Will be overridden by user-defined schema */
-	schema?: ZodSchema | Promise<ZodSchema> | (() => ZodSchema | Promise<ZodSchema>);
-}
+} & (
+	| {
+			/** Optionally, define the schema of the data. Will be overridden by user-defined schema */
+			schema?: ZodSchema;
+	  }
+	| {
+			/** Optionally, provide a function to dynamically provide a schema. Will be overridden by user-defined schema */
+			createSchema?: () => Promise<{
+				schema: ZodSchema;
+				types: string;
+			}>;
+	  }
+);
 
 export interface LoadEntryContext<TEntryFilter = never> {
 	filter: TEntryFilter extends never ? { id: string } : TEntryFilter;

packages/astro/src/content/types-generator.ts

@@ -11,6 +11,7 @@ import type { Logger } from '../core/logger/core.js';
 import { isRelativePath } from '../core/path.js';
 import type { AstroSettings } from '../types/astro.js';
 import type { ContentEntryType } from '../types/public/content.js';
+import type { InjectedType } from '../types/public/integrations.js';
 import {
 	COLLECTIONS_DIR,
 	CONTENT_LAYER_TYPE,
@@ -342,58 +343,68 @@ function normalizeConfigPath(from: string, to: string) {
 	return `"${isRelativePath(configPath) ? '' : './'}${normalizedPath}"` as const;
 }
 
-const schemaCache = new Map<string, ZodSchema>();
+const createSchemaResultCache = new Map<string, { schema: ZodSchema; types: string }>();
 
-async function getContentLayerSchema<T extends keyof ContentConfig['collections']>(
+async function getCreateSchemaResult<T extends keyof ContentConfig['collections']>(
 	collection: ContentConfig['collections'][T],
 	collectionKey: T,
-): Promise<ZodSchema | undefined> {
-	const cached = schemaCache.get(collectionKey);
+) {
+	const cached = createSchemaResultCache.get(collectionKey);
 	if (cached) {
 		return cached;
 	}
 
 	if (
 		collection?.type === CONTENT_LAYER_TYPE &&
 		typeof collection.loader === 'object' &&
-		collection.loader.schema
+		!collection.loader.schema &&
+		collection.loader.createSchema
 	) {
-		let schema = collection.loader.schema;
-		if (typeof schema === 'function') {
-			schema = await schema();
-		}
-		if (schema) {
-			schemaCache.set(collectionKey, await schema);
-			return schema;
-		}
+		const result = await collection.loader.createSchema();
+		createSchemaResultCache.set(collectionKey, result);
+		return result;
 	}
 }
 
+async function getContentLayerSchema<T extends keyof ContentConfig['collections']>(
+	collection: ContentConfig['collections'][T],
+	collectionKey: T,
+): Promise<ZodSchema | undefined> {
+	if (collection?.type !== CONTENT_LAYER_TYPE || typeof collection.loader === 'function') {
+		return;
+	}
+	if (collection.loader.schema) {
+		return collection.loader.schema;
+	}
+	const result = await getCreateSchemaResult(collection, collectionKey);
+	return result?.schema;
+}
+
 async function typeForCollection<T extends keyof ContentConfig['collections']>(
 	collection: ContentConfig['collections'][T] | undefined,
 	collectionKey: T,
-): Promise<string> {
+): Promise<{ type: string; injectedType?: InjectedType }> {
 	if (collection?.schema) {
-		return `InferEntrySchema<${collectionKey}>`;
+		return { type: `InferEntrySchema<${collectionKey}>` };
 	}
-	if (!collection?.type) {
-		return 'any';
+	if (!collection?.type || typeof collection.loader === 'function') {
+		return { type: 'any' };
 	}
-	const schema = await getContentLayerSchema(collection, collectionKey);
-	if (!schema) {
-		return 'any';
+	if (collection.loader.schema) {
+		return { type: `InferLoaderSchema<${collectionKey}>` };
 	}
-	try {
-		const zodToTs = await import('zod-to-ts');
-		const ast = zodToTs.zodToTs(schema);
-		return zodToTs.printNode(ast.node);
-	} catch (err: any) {
-		// zod-to-ts is sad if we don't have TypeScript installed, but that's fine as we won't be needing types in that case
-		if (err.message.includes("Cannot find package 'typescript'")) {
-			return 'any';
-		}
-		throw err;
+	const result = await getCreateSchemaResult(collection, collectionKey);
+	if (!result) {
+		return { type: 'any' };
 	}
+	const base = `loaders/${collectionKey.slice(1, -1)}`;
+	return {
+		type: `import("./${base}.js").Entry`,
+		injectedType: {
+			filename: `${base}.ts`,
+			content: result.types,
+		},
+	};
 }
 
 async function writeContentFiles({
@@ -460,7 +471,21 @@ async function writeContentFiles({
 			return;
 		}
 
-		const dataType = await typeForCollection(collectionConfig, collectionKey);
+		const { type: dataType, injectedType } = await typeForCollection(
+			collectionConfig,
+			collectionKey,
+		);
+
+		if (injectedType) {
+			if (settings.injectedTypes.some((t) => t.filename === CONTENT_TYPES_FILE)) {
+				// If it's the first time, we inject types the usual way. sync() will handle creating files and references. If it's not the first time, we just override the dts content
+				const url = new URL(injectedType.filename, settings.dotAstroDir);
+				await fs.promises.mkdir(path.dirname(fileURLToPath(url)), { recursive: true });
+				await fs.promises.writeFile(url, injectedType.content, 'utf-8');
+			} else {
+				settings.injectedTypes.push(injectedType);
+			}
+		}
 
 		dataTypesStr += `${collectionKey}: Record<string, {\n  id: string;\n  body?: string;\n  collection: ${collectionKey};\n  data: ${dataType};\n  rendered?: RenderedContent;\n  filePath?: string;\n}>;\n`;
 
@@ -487,7 +512,16 @@ async function writeContentFiles({
 			const key = JSON.parse(collectionKey);
 
 			contentCollectionManifest.collections.push({
-				hasSchema: Boolean(collectionConfig?.schema || schemaCache.has(collectionKey)),
+				hasSchema: Boolean(
+					// Is there a user provided schema or
+					collectionConfig?.schema ||
+						// Is it a loader object and
+						(typeof collectionConfig?.loader !== 'function' &&
+							// Is it a loader static schema or
+							(collectionConfig?.loader.schema ||
+								// is it a loader dynamic schema
+								createSchemaResultCache.has(collectionKey))),
+				),
 				name: key,
 			});
 

packages/astro/src/content/utils.ts

@@ -7,7 +7,7 @@ import colors from 'piccolore';
 import type { PluginContext } from 'rollup';
 import type { ViteDevServer } from 'vite';
 import xxhash from 'xxhash-wasm';
-import { z } from 'zod';
+import { type ZodSchema, z } from 'zod';
 import { AstroError, AstroErrorData, errorMap, MarkdownError } from '../core/errors/index.js';
 import { isYAMLException } from '../core/errors/utils.js';
 import type { Logger } from '../core/logger/core.js';
@@ -60,14 +60,39 @@ const collectionConfigParser = z.union([
 			z.function(),
 			z.object({
 				name: z.string(),
-				load: z.function().args(z.custom<LoaderContext>()).returns(
-					z.custom<{
-						schema?: any;
-						types?: string;
-					} | void>(),
-				),
-				schema: z.any().optional(),
-				render: z.function(z.tuple([z.any()], z.unknown())).optional(),
+				load: z.function().args(z.custom<LoaderContext>()).returns(z.promise(z.void())),
+				schema: z
+					.any()
+					.transform((v) => {
+						if (typeof v === 'function') {
+							console.warn(
+								`Your loader's schema is defined using a function. This is no longer supported and the schema will be ignored. Please update your loader to use the `createSchema()` utility instead, or report this to the loader author. In a future major version, this will cause the loader to break entirely.`,
+							);
+							return undefined;
+						}
+						return v;
+					})
+					.superRefine((v, ctx) => {
+						if (v !== undefined && !('_def' in v)) {
+							ctx.addIssue({
+								code: z.ZodIssueCode.custom,
+								message: 'Invalid Zod schema',
+							});
+							return z.NEVER;
+						}
+					})
+					.optional(),
+				createSchema: z
+					.function()
+					.returns(
+						z.promise(
+							z.object({
+								schema: z.custom<ZodSchema>((v) => '_def' in v),
+								types: z.string(),
+							}),
+						),
+					)
+					.optional(),
 			}),
 		]),
 	}),

packages/astro/templates/content/types.d.ts

@@ -104,6 +104,12 @@ declare module 'astro:content' {
 	type InferEntrySchema<C extends keyof DataEntryMap> = import('astro/zod').infer<
 		ReturnTypeOrOriginal<Required<ContentConfig['collections'][C]>['schema']>
 	>;
+	type InferLoaderSchema<
+		C extends keyof DataEntryMap,
+		L = Required<ContentConfig['collections'][C]>['loader'],
+	> = L extends { schema: import('astro/zod').ZodSchema }
+		? import('astro/zod').infer<Required<ContentConfig['collections'][C]>['loader']['schema']>
+		: any;
 
 	type DataEntryMap = {
 		// @@DATA_ENTRY_MAP@@