feat: live content collections

.changeset/pretty-doodles-wash.md

@@ -0,0 +1,78 @@
+---
+'astro': minor
+---
+
+Adds experimental support for live content collections
+
+Live content collections are a new type of [content collection](https://docs.astro.build/en/guides/content-collections/) that fetch their data at runtime rather than build time. This allows you to access frequently-updated data from CMSs, APIs, databases, or other sources using a unified API, without needing to rebuild your site when the data changes.
+
+In Astro 5.0, the content layer API added support for adding diverse content sources to content collections. You can create loaders that fetch data from any source at build time, and then access it inside a page via `getEntry` and `getCollection`. The data is cached between builds, giving fast access and updates. 
+
+However there is no method for updating the data store between builds, meaning any updates to the data need a full site deploy, even if the pages are rendered on-demand. This means that content collections are not suitable for pages that update frequently. Instead, today these pages tend to access the APIs directly in the frontmatter. This works, but leads to a lot of boilerplate, and means users don't benefit from the simple, unified API that content loaders offer. In most cases users tend to individually create loader libraries that they share between pages.
+
+Live content collections solve this problem by allowing you to create loaders that fetch data at runtime, rather than build time. This means that the data is always up-to-date, without needing to rebuild the site. 
+
+## How to use
+
+To enable live collections add the `experimental.liveContentCollections` flag to your `astro.config.mjs` file:
+
+```js title="astro.config.mjs"
+{
+  experimental: {
+    liveContentCollections: true,
+  },
+}
+```
+
+Then create a new `src/live.config.ts` file (alongside your `src/content.config.ts` if you have one) to define your live collections with a [live loader](https://docs.astro.build/en/reference/experimental-flags/live-content-collections/#creating-a-live-loader) and optionally a [schema](https://docs.astro.build/en/reference/experimental-flags/live-content-collections/#using-zod-schemas) using the new `defineLiveCollection()` function from the `astro:content` module.
+
+```ts title="src/live.config.ts"
+import { defineLiveCollection } from 'astro:content';
+import { storeLoader } from '@mystore/astro-loader';
+
+const products = defineLiveCollection({
+  type: 'live',
+  loader: storeLoader({
+    apiKey: process.env.STORE_API_KEY,
+    endpoint: 'https://api.mystore.com/v1',
+  }),
+});
+
+export const collections = { products };
+```
+
+You can then use the dedicated `getLiveCollection()` and `getLiveEntry()` functions to access your live data:
+
+```astro
+---
+import { getLiveCollection, getLiveEntry, render } from 'astro:content';
+
+// Get all products
+const { entries: allProducts, error } = await getLiveCollection('products');
+if (error) {
+  // Handle error appropriately
+  console.error(error.message);
+}
+
+// Get products with a filter (if supported by your loader)
+const { entries: electronics } = await getLiveCollection('products', { category: 'electronics' });
+
+// Get a single product by ID (string syntax)
+const { entry: product, error: productError } = await getLiveEntry('products', Astro.params.id);
+if (productError) {
+  return Astro.redirect('/404');
+}
+
+// Get a single product with a custom query (if supported by your loader) using a filter object
+const { entry: productBySlug } = await getLiveEntry('products', { slug: Astro.params.slug });
+
+const { Content } = await render(product);
+
+---
+
+<h1>{product.title}</h1>
+<Content />
+
+```
+
+See [the docs for the experimental live content collections feature](https://docs.astro.build/en/reference/experimental-flags/live-content-collections/) for more details on how to use this feature, including how to create a live loader. Please give feedback on [the RFC PR](https://github.com/withastro/roadmap/pull/1164) if you have any suggestions or issues.

examples/with-markdoc/src/content.config.ts

@@ -1,5 +1,5 @@
 import { defineCollection } from 'astro:content';
 
 export const collections = {
-	docs: defineCollection({})
+	docs: defineCollection({}),
 };

packages/astro/package.json

@@ -64,6 +64,7 @@
     "./assets/services/noop": "./dist/assets/services/noop.js",
     "./assets/fonts/providers/*": "./dist/assets/fonts/providers/entrypoints/*.js",
     "./loaders": "./dist/content/loaders/index.js",
+    "./content/config": "./dist/content/config.js",
     "./content/runtime": "./dist/content/runtime.js",
     "./content/runtime-assets": "./dist/content/runtime-assets.js",
     "./debug": "./components/Debug.astro",

packages/astro/src/config/entrypoint.ts

@@ -2,7 +2,6 @@
 
 import type { SharpImageServiceConfig } from '../assets/services/sharp.js';
 import type { ImageServiceConfig } from '../types/public/index.js';
-
 export { defineConfig, getViteConfig } from './index.js';
 export { envField } from '../env/config.js';
 export { mergeConfig } from '../core/config/merge.js';

packages/astro/src/content/config.ts

@@ -0,0 +1,178 @@
+import type { ZodLiteral, ZodNumber, ZodObject, ZodString, ZodType, ZodUnion } from 'zod';
+import { CONTENT_LAYER_TYPE, LIVE_CONTENT_TYPE } from './consts.js';
+import type { LiveLoader, Loader } from './loaders/types.js';
+import { AstroError, AstroErrorData, AstroUserError } from '../core/errors/index.js';
+
+function getImporterFilename() {
+	// Find the first line in the stack trace that doesn't include 'defineCollection' or 'getImporterFilename'
+	const stackLine = new Error().stack
+		?.split('\n')
+		.find(
+			(line) =>
+				!line.includes('defineCollection') &&
+				!line.includes('defineLiveCollection') &&
+				!line.includes('getImporterFilename') &&
+				line !== 'Error',
+		);
+	if (!stackLine) {
+		return undefined;
+	}
+	// Extract the relative path from the stack line
+	const match = /\/((?:src|chunks)\/.*?):\d+:\d+/.exec(stackLine);
+
+	return match?.[1] ?? undefined;
+}
+
+// This needs to be in sync with ImageMetadata
+export type ImageFunction = () => ZodObject<{
+	src: ZodString;
+	width: ZodNumber;
+	height: ZodNumber;
+	format: ZodUnion<
+		[
+			ZodLiteral<'png'>,
+			ZodLiteral<'jpg'>,
+			ZodLiteral<'jpeg'>,
+			ZodLiteral<'tiff'>,
+			ZodLiteral<'webp'>,
+			ZodLiteral<'gif'>,
+			ZodLiteral<'svg'>,
+			ZodLiteral<'avif'>,
+		]
+	>;
+}>;
+
+export interface DataEntry {
+	id: string;
+	data: Record<string, unknown>;
+	filePath?: string;
+	body?: string;
+}
+
+export interface DataStore {
+	get: (key: string) => DataEntry;
+	entries: () => Array<[id: string, DataEntry]>;
+	set: (key: string, data: Record<string, unknown>, body?: string, filePath?: string) => void;
+	values: () => Array<DataEntry>;
+	keys: () => Array<string>;
+	delete: (key: string) => void;
+	clear: () => void;
+	has: (key: string) => boolean;
+}
+
+export interface MetaStore {
+	get: (key: string) => string | undefined;
+	set: (key: string, value: string) => void;
+	delete: (key: string) => void;
+	has: (key: string) => boolean;
+}
+
+export type BaseSchema = ZodType;
+
+export type SchemaContext = { image: ImageFunction };
+
+type ContentLayerConfig<S extends BaseSchema, TData extends { id: string } = { id: string }> = {
+	type?: 'content_layer';
+	schema?: S | ((context: SchemaContext) => S);
+	loader:
+		| Loader
+		| (() =>
+				| Array<TData>
+				| Promise<Array<TData>>
+				| Record<string, Omit<TData, 'id'> & { id?: string }>
+				| Promise<Record<string, Omit<TData, 'id'> & { id?: string }>>);
+};
+
+type DataCollectionConfig<S extends BaseSchema> = {
+	type: 'data';
+	schema?: S | ((context: SchemaContext) => S);
+};
+
+type ContentCollectionConfig<S extends BaseSchema> = {
+	type?: 'content';
+	schema?: S | ((context: SchemaContext) => S);
+	loader?: never;
+};
+
+export type LiveCollectionConfig<L extends LiveLoader, S extends BaseSchema | undefined = undefined> = {
+	type: 'live';
+	schema?: S;
+	loader: L;
+};
+
+export type CollectionConfig<S extends BaseSchema> =
+	| ContentCollectionConfig<S>
+	| DataCollectionConfig<S>
+	| ContentLayerConfig<S>;
+
+export function defineLiveCollection<
+L extends LiveLoader,
+S extends BaseSchema | undefined = undefined,
+>(config: LiveCollectionConfig<L, S>): LiveCollectionConfig<L, S> {
+	const importerFilename = getImporterFilename();
+	if (!importerFilename?.includes('live.config')) {
+		throw new AstroError({
+			...AstroErrorData.LiveContentConfigError,
+			message: AstroErrorData.LiveContentConfigError.message(
+				'Live collections must be defined in a `src/live.config.ts` file.',
+				importerFilename ?? 'your content config file',
+			),
+		});
+	}
+	if (config.type !== LIVE_CONTENT_TYPE) {
+		throw new AstroError({
+			...AstroErrorData.LiveContentConfigError,
+			message: AstroErrorData.LiveContentConfigError.message(
+				'Collections in a live config file must have a type of `live`.',
+				importerFilename,
+			),
+		});
+	}
+
+	if (!config.loader) {
+		throw new AstroError({
+			...AstroErrorData.LiveContentConfigError,
+			message: AstroErrorData.LiveContentConfigError.message(
+				'Live collections must have a `loader` defined.',
+				importerFilename,
+			),
+		});
+	}
+	if (typeof config.schema === 'function') {
+		throw new AstroError({
+			...AstroErrorData.LiveContentConfigError,
+			message: AstroErrorData.LiveContentConfigError.message(
+				'The schema cannot be a function for live collections. Please use a schema object instead.',
+				importerFilename,
+			),
+		});
+	}
+	return config;
+}
+
+export function defineCollection<S extends BaseSchema>(
+	config: CollectionConfig<S>,
+): CollectionConfig<S> {
+	const importerFilename = getImporterFilename();
+
+	if (importerFilename?.includes('live.config')) {
+		throw new AstroError({
+			...AstroErrorData.LiveContentConfigError,
+			message: AstroErrorData.LiveContentConfigError.message(
+				'Collections in a live config file must use `defineLiveCollection`.',
+				importerFilename,
+			),
+		});
+	}
+
+	if ('loader' in config) {
+		if (config.type && config.type !== CONTENT_LAYER_TYPE) {
+			throw new AstroUserError(
+				`Collections that use the Content Layer API must have a \`loader\` defined and no \`type\` set. Check your collection definitions in ${importerFilename ?? 'your content config file'}.`,
+			);
+		}
+		config.type = CONTENT_LAYER_TYPE;
+	}
+	if (!config.type) config.type = 'content';
+	return config;
+}

packages/astro/src/content/consts.ts

@@ -41,3 +41,4 @@ export const COLLECTIONS_MANIFEST_FILE = 'collections/collections.json';
 export const COLLECTIONS_DIR = 'collections/';
 
 export const CONTENT_LAYER_TYPE = 'content_layer';
+export const LIVE_CONTENT_TYPE = 'live';

packages/astro/src/content/loaders/errors.ts

@@ -0,0 +1,62 @@
+import type { ZodError } from "zod";
+
+export class LiveCollectionError extends Error {
+	constructor(
+		public readonly collection: string,
+		public readonly message: string,
+		public readonly cause?: Error,
+	) {
+		super(message);
+		this.name = 'LiveCollectionError';
+	}
+	static is(error: unknown): error is LiveCollectionError {
+		return error instanceof LiveCollectionError;
+	}
+}
+
+export class LiveEntryNotFoundError extends LiveCollectionError {
+	constructor(collection: string, entryFilter: string | Record<string, unknown>) {
+		super(
+			collection,
+			`Entry ${collection} → ${typeof entryFilter === 'string' ? entryFilter : JSON.stringify(entryFilter)} was not found.`,
+		);
+		this.name = 'LiveEntryNotFoundError';
+	}
+	static is(error: unknown): error is LiveEntryNotFoundError {
+		return (error as any)?.name === 'LiveEntryNotFoundError';
+	}
+}
+
+export class LiveCollectionValidationError extends LiveCollectionError {
+	constructor(collection: string, entryId: string, error: ZodError) {
+		super(
+			collection,
+			[
+				`**${collection} → ${entryId}** data does not match the collection schema.\n`,
+				...error.errors.map((zodError) => `  **${zodError.path.join('.')}**: ${zodError.message}`),
+				'',
+			].join('\n'),
+		);
+		this.name = 'LiveCollectionValidationError';
+	}
+	static is(error: unknown): error is LiveCollectionValidationError {
+		return (error as any)?.name === 'LiveCollectionValidationError';
+	}
+}
+
+export class LiveCollectionCacheHintError extends LiveCollectionError {
+	constructor(collection: string, entryId: string | undefined, error: ZodError) {
+		super(
+			collection,
+			[
+				`**${String(collection)}${entryId ? ` → ${String(entryId)}` : ''}** returned an invalid cache hint.\n`,
+				...error.errors.map((zodError) => `  **${zodError.path.join('.')}**: ${zodError.message}`),
+				'',
+			].join('\n'),
+		);
+		this.name = 'LiveCollectionCacheHintError';
+	}
+	static is(error: unknown): error is LiveCollectionCacheHintError {
+		return (error as any)?.name === 'LiveCollectionCacheHintError';
+	}
+}

packages/astro/src/content/loaders/types.ts

@@ -2,7 +2,11 @@ import type { FSWatcher } from 'vite';
 import type { ZodSchema } from 'zod';
 import type { AstroIntegrationLogger } from '../../core/logger/core.js';
 import type { AstroConfig } from '../../types/public/config.js';
-import type { ContentEntryType } from '../../types/public/content.js';
+import type {
+	ContentEntryType,
+	LiveDataCollection,
+	LiveDataEntry,
+} from '../../types/public/content.js';
 import type { RenderedContent } from '../data-store.js';
 import type { DataStore, MetaStore } from '../mutable-data-store.js';
 
@@ -53,3 +57,29 @@ export interface Loader {
 	/** Optionally, define the schema of the data. Will be overridden by user-defined schema */
 	schema?: ZodSchema | Promise<ZodSchema> | (() => ZodSchema | Promise<ZodSchema>);
 }
+
+export interface LoadEntryContext<TEntryFilter = never> {
+	filter: TEntryFilter extends never ? { id: string } : TEntryFilter;
+}
+
+export interface LoadCollectionContext<TCollectionFilter = unknown> {
+	filter?: TCollectionFilter;
+}
+
+export interface LiveLoader<
+	TData extends Record<string, any> = Record<string, unknown>,
+	TEntryFilter extends Record<string, any> | never = never,
+	TCollectionFilter extends Record<string, any> | never = never,
+	TError extends Error = Error,
+> {
+	/** Unique name of the loader, e.g. the npm package name */
+	name: string;
+	/** Load a single entry */
+	loadEntry: (
+		context: LoadEntryContext<TEntryFilter>,
+	) => Promise<LiveDataEntry<TData> | undefined | { error: TError }>;
+	/** Load a collection of entries */
+	loadCollection: (
+		context: LoadCollectionContext<TCollectionFilter>,
+	) => Promise<LiveDataCollection<TData> | { error: TError }>;
+}