feat: initial live loader implementation

examples/with-markdoc/src/content.config.ts

@@ -1,5 +1,5 @@
 import { defineCollection } from 'astro:content';
 
 export const collections = {
-	docs: defineCollection({})
+	docs: defineCollection({}),
 };

packages/astro/src/content/consts.ts

@@ -41,3 +41,4 @@ export const COLLECTIONS_MANIFEST_FILE = 'collections/collections.json';
 export const COLLECTIONS_DIR = 'collections/';
 
 export const CONTENT_LAYER_TYPE = 'content_layer';
+export const LIVE_CONTENT_TYPE = 'live';

packages/astro/src/content/loaders/types.ts

@@ -2,7 +2,11 @@ import type { FSWatcher } from 'vite';
 import type { ZodSchema } from 'zod';
 import type { AstroIntegrationLogger } from '../../core/logger/core.js';
 import type { AstroConfig } from '../../types/public/config.js';
-import type { ContentEntryType } from '../../types/public/content.js';
+import type {
+	ContentEntryType,
+	LiveDataCollection,
+	LiveDataEntry,
+} from '../../types/public/content.js';
 import type { DataStore, MetaStore } from '../mutable-data-store.js';
 
 export type { DataStore, MetaStore };
@@ -49,3 +53,26 @@ export interface Loader {
 	/** Optionally, define the schema of the data. Will be overridden by user-defined schema */
 	schema?: ZodSchema | Promise<ZodSchema> | (() => ZodSchema | Promise<ZodSchema>);
 }
+
+export interface LoadEntryContext<TEntryFilter = never> {
+	filter: TEntryFilter extends never ? { id: string } : TEntryFilter;
+}
+
+export interface LoadCollectionContext<TCollectionFilter = unknown> {
+	filter?: TCollectionFilter;
+}
+
+export interface LiveLoader<
+	TData extends Record<string, unknown> = Record<string, unknown>,
+	TEntryFilter extends Record<string, unknown> | never = never,
+	TCollectionFilter extends Record<string, unknown> | never = never,
+> {
+	/** Unique name of the loader, e.g. the npm package name */
+	name: string;
+	/** Load a single entry */
+	loadEntry: (context: LoadEntryContext<TEntryFilter>) => Promise<LiveDataEntry<TData> | undefined>;
+	/** Load a collection of entries */
+	loadCollection: (
+		context: LoadCollectionContext<TCollectionFilter>,
+	) => Promise<LiveDataCollection<TData>>;
+}

packages/astro/src/content/runtime.ts

@@ -19,14 +19,19 @@ import {
 	render as serverRender,
 	unescapeHTML,
 } from '../runtime/server/index.js';
-import { CONTENT_LAYER_TYPE, IMAGE_IMPORT_PREFIX } from './consts.js';
+import { CONTENT_LAYER_TYPE, IMAGE_IMPORT_PREFIX, LIVE_CONTENT_TYPE } from './consts.js';
 import { type DataEntry, globalDataStore } from './data-store.js';
+import type { LiveLoader } from './loaders/types.js';
 import type { ContentLookupMap } from './utils.js';
 
 type LazyImport = () => Promise<any>;
 type GlobResult = Record<string, LazyImport>;
 type CollectionToEntryMap = Record<string, GlobResult>;
 type GetEntryImport = (collection: string, lookupId: string) => Promise<LazyImport>;
+type LiveCollectionConfigMap = Record<
+	string,
+	{ loader: LiveLoader; type: typeof LIVE_CONTENT_TYPE }
+>;
 
 export function getImporterFilename() {
 	// The 4th line in the stack trace should be the importer filename
@@ -40,6 +45,27 @@ export function getImporterFilename() {
 }
 
 export function defineCollection(config: any) {
+	const isInLiveConfig = getImporterFilename()?.endsWith('/live-content.config.ts');
+
+	if (config.type === LIVE_CONTENT_TYPE) {
+		if (!isInLiveConfig) {
+			throw new AstroUserError(
+				`Collections with type "live" must be defined in a \`src/live-content.config.ts\` file. Check your collection definitions in ${getImporterFilename() ?? 'your content config file'}.`,
+			);
+		}
+		if (!config.loader) {
+			throw new AstroUserError(
+				`Collections that use the Live Content API must have a \`loader\` defined. Check your collection definitions in ${getImporterFilename() ?? 'your live content config file'}.`,
+			);
+		}
+		return config;
+	}
+	if (isInLiveConfig) {
+		throw new AstroUserError(
+			`Collections in a \`live-content.config.ts\` file must be defined with the type "live". Check your collection definitions.`,
+		);
+	}
+
 	if ('loader' in config) {
 		if (config.type && config.type !== CONTENT_LAYER_TYPE) {
 			throw new AstroUserError(
@@ -76,13 +102,40 @@ export function createGetCollection({
 	dataCollectionToEntryMap,
 	getRenderEntryImport,
 	cacheEntriesByCollection,
+	liveCollections,
 }: {
 	contentCollectionToEntryMap: CollectionToEntryMap;
 	dataCollectionToEntryMap: CollectionToEntryMap;
 	getRenderEntryImport: GetEntryImport;
 	cacheEntriesByCollection: Map<string, any[]>;
+	liveCollections: LiveCollectionConfigMap;
 }) {
-	return async function getCollection(collection: string, filter?: (entry: any) => unknown) {
+	return async function getCollection(
+		collection: string,
+		filter?: ((entry: any) => unknown) | Record<string, unknown>,
+	) {
+		if (collection in liveCollections) {
+			if (typeof filter === 'function') {
+				throw new AstroError({
+					...AstroErrorData.UnknownContentCollectionError,
+					message: `The filter function is not supported for live collections. Please use a filter object instead.`,
+				});
+			}
+
+			const context = {
+				filter,
+			};
+
+			const response = await (
+				liveCollections[collection].loader as LiveLoader<any, any, Record<string, unknown>>
+			)?.loadCollection?.(context);
+
+			return {
+				...response,
+				collection,
+			};
+		}
+
 		const hasFilter = typeof filter === 'function';
 		const store = await globalDataStore.get();
 		let type: 'content' | 'data';
@@ -297,27 +350,29 @@ export function createGetEntry({
 	getEntryImport,
 	getRenderEntryImport,
 	collectionNames,
+	liveCollections,
 }: {
 	getEntryImport: GetEntryImport;
 	getRenderEntryImport: GetEntryImport;
 	collectionNames: Set<string>;
+	liveCollections: LiveCollectionConfigMap;
 }) {
 	return async function getEntry(
 		// Can either pass collection and identifier as 2 positional args,
 		// Or pass a single object with the collection and identifier as properties.
 		// This means the first positional arg can have different shapes.
 		collectionOrLookupObject: string | EntryLookupObject,
-		_lookupId?: string,
+		lookup?: string | Record<string, unknown>,
 	): Promise<ContentEntryResult | DataEntryResult | undefined> {
-		let collection: string, lookupId: string;
+		let collection: string, lookupId: string | Record<string, unknown>;
 		if (typeof collectionOrLookupObject === 'string') {
 			collection = collectionOrLookupObject;
-			if (!_lookupId)
+			if (!lookup)
 				throw new AstroError({
 					...AstroErrorData.UnknownContentCollectionError,
 					message: '`getEntry()` requires an entry identifier as the second argument.',
 				});
-			lookupId = _lookupId;
+			lookupId = lookup;
 		} else {
 			collection = collectionOrLookupObject.collection;
 			// Identifier could be `slug` for content entries, or `id` for data entries
@@ -327,6 +382,39 @@ export function createGetEntry({
 					: collectionOrLookupObject.slug;
 		}
 
+		if (collection in liveCollections) {
+			if (!lookup) {
+				throw new AstroError({
+					...AstroErrorData.UnknownContentCollectionError,
+					message: '`getEntry()` requires an entry identifier as the second argument.',
+				});
+			}
+
+			const lookupObject = {
+				filter: typeof lookup === 'string' ? { id: lookup } : lookup,
+			};
+
+			const entry = await (
+				liveCollections[collection].loader as LiveLoader<
+					Record<string, unknown>,
+					Record<string, unknown>
+				>
+			)?.loadEntry?.(lookupObject);
+
+			if (!entry) {
+				return;
+			}
+			return {
+				...entry,
+				collection,
+			};
+		}
+		if (typeof lookupId === 'object') {
+			throw new AstroError({
+				...AstroErrorData.UnknownContentCollectionError,
+				message: `The entry identifier must be a string. Received object.`,
+			});
+		}
 		const store = await globalDataStore.get();
 
 		if (store.hasCollection(collection)) {

packages/astro/src/content/types-generator.ts

@@ -7,7 +7,7 @@ import { type ViteDevServer, normalizePath } from 'vite';
 import { type ZodSchema, z } from 'zod';
 import { zodToJsonSchema } from 'zod-to-json-schema';
 import { AstroError } from '../core/errors/errors.js';
-import { AstroErrorData } from '../core/errors/index.js';
+import { AstroErrorData, AstroUserError } from '../core/errors/index.js';
 import type { Logger } from '../core/logger/core.js';
 import { isRelativePath } from '../core/path.js';
 import type { AstroSettings } from '../types/astro.js';
@@ -16,6 +16,7 @@ import {
 	COLLECTIONS_DIR,
 	CONTENT_LAYER_TYPE,
 	CONTENT_TYPES_FILE,
+	LIVE_CONTENT_TYPE,
 	VIRTUAL_MODULE_ID,
 } from './consts.js';
 import {
@@ -53,6 +54,10 @@ type CollectionEntryMap = {
 		| {
 				type: 'data' | typeof CONTENT_LAYER_TYPE;
 				entries: Record<string, DataEntryMetadata>;
+		  }
+		| {
+				type: typeof LIVE_CONTENT_TYPE;
+				entries: Record<string, never>;
 		  };
 };
 
@@ -493,6 +498,11 @@ async function writeContentFiles({
 		const collectionEntryKeys = Object.keys(collection.entries).sort();
 		const dataType = await typeForCollection(collectionConfig, collectionKey);
 		switch (resolvedType) {
+			case LIVE_CONTENT_TYPE:
+				// This error should never be thrown, as it should have been caught earlier in the process
+				throw new AstroUserError(
+					`Invalid definition for collection ${collectionKey}: Live content collections must be defined in "src/live-content.config.ts"`,
+				);
 			case 'content':
 				if (collectionEntryKeys.length === 0) {
 					contentTypesStr += `${collectionKey}: Record<string, {\n  id: string;\n  slug: string;\n  body: string;\n  collection: ${collectionKey};\n  data: ${dataType};\n  render(): Render[".md"];\n}>;\n`;
@@ -579,17 +589,28 @@ async function writeContentFiles({
 		contentPaths.config.url.pathname,
 	);
 
+	const liveConfigPathRelativeToCacheDir = contentPaths.liveConfig?.exists
+		? normalizeConfigPath(settings.dotAstroDir.pathname, contentPaths.liveConfig.url.pathname)
+		: undefined;
+
 	for (const contentEntryType of contentEntryTypes) {
 		if (contentEntryType.contentModuleTypes) {
 			typeTemplateContent = contentEntryType.contentModuleTypes + '\n' + typeTemplateContent;
 		}
 	}
-	typeTemplateContent = typeTemplateContent.replace('// @@CONTENT_ENTRY_MAP@@', contentTypesStr);
-	typeTemplateContent = typeTemplateContent.replace('// @@DATA_ENTRY_MAP@@', dataTypesStr);
-	typeTemplateContent = typeTemplateContent.replace(
-		"'@@CONTENT_CONFIG_TYPE@@'",
-		contentConfig ? `typeof import(${configPathRelativeToCacheDir})` : 'never',
-	);
+	typeTemplateContent = typeTemplateContent
+		.replace('// @@CONTENT_ENTRY_MAP@@', contentTypesStr)
+		.replace('// @@DATA_ENTRY_MAP@@', dataTypesStr)
+		.replace(
+			"'@@CONTENT_CONFIG_TYPE@@'",
+			contentConfig ? `typeof import(${configPathRelativeToCacheDir})` : 'never',
+		)
+		.replace(
+			"'@@LIVE_CONTENT_CONFIG_TYPE@@'",
+			liveConfigPathRelativeToCacheDir
+				? `typeof import(${liveConfigPathRelativeToCacheDir})`
+				: 'never',
+		);
 
 	// If it's the first time, we inject types the usual way. sync() will handle creating files and references. If it's not the first time, we just override the dts content
 	if (settings.injectedTypes.some((t) => t.filename === CONTENT_TYPES_FILE)) {