Refactor AgentPond architecture and CLI workflows

Author: marcusschiesser

.changeset/add-files-sdk-storage.md

@@ -2,6 +2,8 @@
 "@agentpond/files-sdk": minor
 "@agentpond/core": minor
 "agentpond": minor
+"@agentpond/firebase": minor
+"@agentpond/ingest-service": minor
 ---
 
-Add Files SDK span export and persistent bucket-backed CLI sync environments.
+Consolidate manual remote storage and container ingestion on Files SDK, remove legacy local, S3, and GCS environment formats, and keep platform-native Firebase, Supabase, and Vercel storage.

README.md

@@ -14,7 +14,7 @@
   <a href="https://www.npmjs.com/package/agentpond"><img src="https://img.shields.io/node/v/agentpond.svg" alt="Node.js version"></a>
 </p>
 
-AgentPond is a lightweight trace backend and CLI for AI applications. It keeps raw traces in object storage you control and syncs them into a local DuckDB cache for fast analysis by your coding agent. Use it with Firebase Storage, Supabase Storage, Amazon S3, Google Cloud Storage, Vercel Blob, or custom infrastructure. Firebase, Supabase, and Vercel projects include automated setup; other deployments use a manual setup path.
+AgentPond is a lightweight trace backend and CLI for AI applications. It keeps raw traces in object storage you control and syncs them into a local DuckDB cache for fast analysis by your coding agent. It supports all bucket providers from [Files SDK](https://github.com/haydenbleasel/files-sdk) such as Amazon S3, Google Cloud Storage, R2, and MinIO; For platforms like Firebase, Supabase, and Vercel there are explicit one-command quick starts.
 
 ## How it works
 
@@ -24,21 +24,15 @@ Object storage is the durable source of truth. The local DuckDB database is a re
 
 ## Getting started
 
-Start with the [Manual deployment setup](./docs/getting-started/manual-setup.md) to configure object storage, instrument the application, and sync its traces into AgentPond.
+Start with the [Manual deployment setup](./docs/getting-started/manual-setup.md) to configure object storage, instrument your application, and sync its traces into AgentPond.
 
-For Firebase, Supabase, and Vercel projects, AgentPond provides automated quick starts:
-
-- [Firebase quick start](./docs/getting-started/firebase.md)
-- [Supabase quick start](./docs/getting-started/supabase.md)
-- [Vercel quick start](./docs/getting-started/vercel.md)
-
-All require Node.js 22 or newer for the CLI. From your project folder, run:
+For [Firebase](./docs/getting-started/firebase.md), [Supabase](./docs/getting-started/supabase.md), and [Vercel](./docs/getting-started/vercel.md) projects, to get started, just run:
 
 ```sh
 npx agentpond init
 ```
 
-The command detects the platform, installs AgentPond's instrumentation and analytics skills, and prints a install prompt for your coding agent. Your agent inspects then the application, proposes a setup, implements it after confirmation, and verifies a real trace end to end.
+The command installs AgentPond's instrumentation and analytics skills, and prints a install prompt for your coding agent. Your coding agent inspects then the application, proposes a setup, implements it after confirmation, and verifies a real trace end to end.
 
 Once the application has emitted a trace:
 
@@ -47,26 +41,9 @@ npx agentpond sync
 npx agentpond traces list --limit 10
 ```
 
-## Analyze traces
-
-Use focused commands for individual traces and sessions:
-
-```sh
-npx agentpond traces get <trace-id>
-npx agentpond observations list --traceId <trace-id>
-npx agentpond sessions get <session-id>
-npx agentpond scores list --traceId <trace-id>
-```
-
-Use DuckDB SQL for aggregation, joins, time windows, or cost analysis:
-
-```sh
-npx agentpond sql "select id, name, session_id, total_cost from traces order by start_time desc limit 10"
-```
-
 ## Features
 
-- Direct OpenTelemetry export to Firebase Storage, Supabase Storage, S3, GCS, and Vercel Blob
+- Direct OpenTelemetry export through Files SDK, Firebase Storage, Supabase Storage, and Vercel Blob
 - Langfuse-compatible and OTLP HTTP ingestion adapters
 - Incremental object-store synchronization
 - Local DuckDB projections for traces, observations, sessions, and scores

apps/cli/package.json

@@ -25,20 +25,25 @@
 		"directory": "apps/cli"
 	},
 	"dependencies": {
-		"@agentpond/aws": "workspace:*",
 		"@agentpond/core": "workspace:*",
 		"@agentpond/duckdb": "workspace:*",
 		"@agentpond/fastify-ingest": "workspace:*",
 		"@agentpond/firebase": "workspace:*",
 		"@agentpond/files-sdk": "workspace:*",
-		"@agentpond/google": "workspace:*",
 		"@agentpond/supabase": "workspace:*",
 		"@agentpond/vercel": "workspace:*",
+		"@aws-sdk/client-s3": "3.1068.0",
+		"@aws-sdk/s3-presigned-post": "3.1068.0",
+		"@aws-sdk/s3-request-presigner": "3.1068.0",
 		"@duckdb/node-api": "^1.5.4-r.1",
+		"@google-cloud/storage": "^7.21.0",
 		"@inquirer/prompts": "8.5.2",
+		"@supabase/storage-js": "2.110.6",
 		"commander": "15.0.0",
+		"disk": "^0.8.18",
 		"fastify": "^5.6.2",
 		"files-sdk": "2.2.1",
+		"node-appwrite": "^26.0.0",
 		"protobufjs": "^7.6.5",
 		"skills": "1.5.9",
 		"zod": "^4.3.6"

apps/cli/src/commands/env.ts

@@ -1,6 +1,5 @@
 import { existsSync } from "node:fs";
 import {
-	type AgentPondStoreType,
 	type FilesSdkEnvironmentConfig,
 	initAgentPondEnvironment,
 	listAgentPondEnvironments,
@@ -23,6 +22,7 @@ import {
 import {
 	environmentContextForCommand,
 	manualEnvironmentContextForCommand,
+	validateManualEnvironment,
 } from "../environment-context.js";
 import { providerForCommand } from "../providers.js";
 
@@ -32,8 +32,6 @@ export type SelectPrompt<T extends string> = (config: {
 }) => Promise<T>;
 
 export type SelectEnvironmentPrompt = SelectPrompt<string>;
-export type AgentPondInitStore = AgentPondStoreType;
-export type SelectStorePrompt = SelectPrompt<AgentPondInitStore>;
 export type SelectFilesProviderPrompt = SelectPrompt<string>;
 export type InputPrompt = (config: {
 	message: string;
@@ -42,25 +40,28 @@ export type InputPrompt = (config: {
 
 type EnvOptions = GlobalOptions & {
 	bucket?: string;
+	endpoint?: string;
 	langfuse?: boolean;
 	otel?: boolean;
 	provider?: string;
-	store?: string;
+	region?: string;
 };
 
 export function registerEnvCommand(
 	program: Command,
 	options: {
 		selectEnvironment?: SelectEnvironmentPrompt;
 		selectFilesProvider?: SelectFilesProviderPrompt;
-		selectStore?: SelectStorePrompt;
 		inputBucket?: InputPrompt;
+		inputEndpoint?: InputPrompt;
+		inputRegion?: InputPrompt;
 	} = {},
 ): void {
 	const promptSelect = options.selectEnvironment ?? select<string>;
-	const promptStore = options.selectStore ?? select<AgentPondInitStore>;
 	const promptFilesProvider = options.selectFilesProvider ?? select<string>;
 	const promptBucket = options.inputBucket ?? input;
+	const promptEndpoint = options.inputEndpoint ?? input;
+	const promptRegion = options.inputRegion ?? input;
 	const env = addGlobalOptions(
 		program.command("env").description("select and manage environments"),
 	);
@@ -108,44 +109,51 @@ export function registerEnvCommand(
 				name,
 				selected: name === selected,
 			}));
+			for (const name of new Set([selected, ...rows.map(({ name }) => name)])) {
+				validateManualEnvironment(cwd, name);
+			}
 			return print(rows, Boolean(globalOptions.json));
 		});
 
 	addGlobalOptions(env.command("init <name>"))
 		.description("initialize a manual environment")
-		.option("--store <store>", "object store: files-sdk, s3, gcs, or local")
 		.option("--provider <provider>", "Files SDK bucket provider")
 		.option("--bucket <bucket>", "Files SDK bucket name")
+		.option("--endpoint <endpoint>", "Files SDK provider endpoint")
+		.option("--region <region>", "Files SDK provider region")
 		.action(
 			async (name: string, commandOptions: EnvOptions, command: Command) => {
+				if (name === "dev") {
+					throw new CliError(
+						'The "dev" environment is managed by `npx agentpond dev`',
+					);
+				}
 				const globalOptions = command.optsWithGlobals<GlobalOptions>();
 				const context = manualEnvironmentContextForCommand("init", {
 					envName: name,
 					platform: globalOptions.platform,
 				});
-				const store =
-					storeFromValue(commandOptions.store) ??
-					(await promptForStore(promptStore));
-				const filesSdk = await filesSdkConfigForStore(
-					store,
+				const filesSdk = await filesSdkConfigForOptions(
 					commandOptions,
 					promptFilesProvider,
 					promptBucket,
+					promptEndpoint,
+					promptRegion,
 				);
 				const environment = initAgentPondEnvironment(name, {
 					cwd: context.rootDir,
-					storeType: store,
-					filesSdk: filesSdk?.config,
+					filesSdk,
 				});
+				validateManualEnvironment(context.rootDir, environment.name);
 				return print(
 					{
-						bucket: filesSdk?.config.bucket,
+						bucket: filesSdk.bucket,
 						name: environment.name,
 						envFile: environment.envFilePath,
 						dbPath: environment.dbPath,
-						peerDependencies: filesSdk?.peerDependencies,
-						provider: filesSdk?.config.provider,
-						store,
+						provider: filesSdk.provider,
+						...(filesSdk.endpoint ? { endpoint: filesSdk.endpoint } : {}),
+						...(filesSdk.region ? { region: filesSdk.region } : {}),
 					},
 					Boolean(globalOptions.json),
 				);
@@ -194,64 +202,19 @@ async function selectEnvironmentForCommand(
 	});
 	const selectedName =
 		name ?? (await promptForEnvironmentName(promptSelect, context.rootDir));
+	validateManualEnvironment(context.rootDir, selectedName);
 	return selectAgentPondEnvironment(selectedName, {
 		cwd: context.rootDir,
 	}).name;
 }
 
-function storeFromValue(
-	value: string | undefined,
-): AgentPondInitStore | undefined {
-	if (value === undefined) return undefined;
-	if (
-		value === "files-sdk" ||
-		value === "s3" ||
-		value === "gcs" ||
-		value === "local"
-	) {
-		return value;
-	}
-	throw new CliError(
-		`--store must be files-sdk, s3, gcs, or local, got "${value}"`,
-	);
-}
-
-async function promptForStore(
-	promptSelect: SelectStorePrompt,
-): Promise<AgentPondInitStore> {
-	if (!process.stdin.isTTY || !process.stdout.isTTY) {
-		throw new CliError("Missing --store");
-	}
-	return promptSelect({
-		message: "Select AgentPond object store",
-		choices: [
-			{ name: "Files SDK bucket provider", value: "files-sdk" },
-			{ name: "AWS S3 (or compatible)", value: "s3" },
-			{ name: "Google Cloud Storage (GCS)", value: "gcs" },
-			{ name: "Local filesystem", value: "local" },
-		],
-	});
-}
-
-async function filesSdkConfigForStore(
-	store: AgentPondInitStore,
-	options: Pick<EnvOptions, "bucket" | "provider">,
+async function filesSdkConfigForOptions(
+	options: Pick<EnvOptions, "bucket" | "endpoint" | "provider" | "region">,
 	promptProvider: SelectFilesProviderPrompt,
 	promptBucket: InputPrompt,
-): Promise<
-	| {
-			config: FilesSdkEnvironmentConfig;
-			peerDependencies: readonly string[];
-	  }
-	| undefined
-> {
-	if (store !== "files-sdk") {
-		if (options.provider !== undefined || options.bucket !== undefined) {
-			throw new CliError("--provider and --bucket require --store files-sdk");
-		}
-		return undefined;
-	}
-
+	promptEndpoint: InputPrompt,
+	promptRegion: InputPrompt,
+): Promise<FilesSdkEnvironmentConfig> {
 	const provider =
 		options.provider ?? (await promptForFilesProvider(promptProvider));
 	const definition = bucketProvider(provider);
@@ -260,9 +223,23 @@ async function filesSdkConfigForStore(
 	).trim();
 	if (!bucket) throw new CliError("Missing --bucket");
 
+	const endpoint = await configuredProviderValue(
+		"endpoint",
+		definition.env.config?.includes("endpoint") ?? false,
+		options.endpoint,
+		promptEndpoint,
+	);
+	const region = await configuredProviderValue(
+		"region",
+		definition.env.config?.includes("region") ?? false,
+		options.region,
+		promptRegion,
+	);
 	return {
-		config: { provider, bucket },
-		peerDependencies: definition.peerDeps,
+		provider,
+		bucket,
+		...(endpoint ? { endpoint } : {}),
+		...(region ? { region } : {}),
 	};
 }
 
@@ -305,16 +282,54 @@ function bucketProvider(provider: string) {
 			`Files SDK provider "${provider}" is not bucket-backed and is not supported by AgentPond`,
 		);
 	}
+	if (provider === "bun-s3") {
+		throw new CliError(
+			'Files SDK provider "bun-s3" is not supported by AgentPond\'s Node.js runtime',
+		);
+	}
+	for (const field of definition.env.config) {
+		if (field !== "bucket" && field !== "endpoint" && field !== "region") {
+			throw new CliError(
+				`Files SDK provider "${provider}" requires unsupported configuration field "${field}"`,
+			);
+		}
+	}
 	return definition;
 }
 
 function bucketProviders() {
 	return PROVIDER_NAMES.flatMap((provider) => {
 		const definition = getProvider(provider);
-		return definition?.env.config?.includes("bucket") ? [definition] : [];
+		return provider !== "bun-s3" &&
+			definition?.env.config?.includes("bucket") &&
+			definition.env.config.every(
+				(field) =>
+					field === "bucket" || field === "endpoint" || field === "region",
+			)
+			? [definition]
+			: [];
 	});
 }
 
+async function configuredProviderValue(
+	field: "endpoint" | "region",
+	required: boolean,
+	value: string | undefined,
+	promptInput: InputPrompt,
+): Promise<string | undefined> {
+	const configured = value?.trim();
+	if (configured) return configured;
+	if (!required) return undefined;
+	if (!process.stdin.isTTY || !process.stdout.isTTY) {
+		throw new CliError(`Missing --${field}`);
+	}
+	const prompted = (
+		await promptInput({ message: `Files SDK ${field}` })
+	).trim();
+	if (!prompted) throw new CliError(`Missing --${field}`);
+	return prompted;
+}
+
 async function promptForEnvironmentName(
 	promptSelect: SelectEnvironmentPrompt,
 	cwd: string,

apps/cli/src/commands/scores.ts

@@ -140,7 +140,7 @@ export async function createScore(
 		},
 	};
 
-	if (context.usesAgentPondDevServer) {
+	if (context.kind === "dev") {
 		const result = await new DuckDbIngestionSink(config.dbPath).writeEvents({
 			projectId: config.projectId,
 			events: [event],

apps/cli/src/commands/sync.ts

@@ -14,7 +14,7 @@ export function registerSyncCommand(program: Command): void {
 			const { context, json } = commandContext(
 				command.optsWithGlobals<GlobalOptions>(),
 			);
-			if (context.usesAgentPondDevServer) {
+			if (context.kind === "dev") {
 				return print(
 					{
 						skipped: true,

apps/cli/src/commands/traces.ts

@@ -113,7 +113,7 @@ export async function createTrace(
 	const now = new Date().toISOString();
 	const traceId = stringFlag(options, "id") ?? createOtelTraceId();
 	const resourceSpans = manualTraceResourceSpans(options, traceId, now);
-	if (context.usesAgentPondDevServer) {
+	if (context.kind === "dev") {
 		const result = await new DuckDbIngestionSink(
 			config.dbPath,
 		).writeOtelResourceSpans({