Evaluate Files SDK for AgentPond object storage

.changeset/add-files-sdk-storage.md

@@ -0,0 +1,9 @@
+---
+"@agentpond/files-sdk": minor
+"@agentpond/core": minor
+"agentpond": minor
+"@agentpond/firebase": minor
+"@agentpond/ingest-service": minor
+---
+
+Consolidate manual remote storage and container ingestion on Files SDK, install only the integrations needed by supported bucket providers, remove legacy local, S3, and GCS environment formats, and keep platform-native Firebase, Supabase, and Vercel storage.

AGENTS.md

@@ -38,7 +38,7 @@ This repository uses Changesets to generate package changelogs.
 <!-- gitnexus:start -->
 # GitNexus — Code Intelligence
 
-This project is indexed by GitNexus as **agentpond** (1333 symbols, 3159 relationships, 106 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely.
+This project is indexed by GitNexus as **agentpond** (1524 symbols, 3645 relationships, 121 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely.
 
 > Index stale? Run `node .gitnexus/run.cjs analyze` from the project root — it auto-selects an available runner. No `.gitnexus/run.cjs` yet? `npx gitnexus analyze` (npm 11 crash → `npm i -g gitnexus`; #1939).
 

CLAUDE.md

@@ -1,7 +1,7 @@
 <!-- gitnexus:start -->
 # GitNexus — Code Intelligence
 
-This project is indexed by GitNexus as **agentpond** (1333 symbols, 3159 relationships, 106 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely.
+This project is indexed by GitNexus as **agentpond** (1524 symbols, 3645 relationships, 121 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely.
 
 > Index stale? Run `node .gitnexus/run.cjs analyze` from the project root — it auto-selects an available runner. No `.gitnexus/run.cjs` yet? `npx gitnexus analyze` (npm 11 crash → `npm i -g gitnexus`; #1939).
 

README.md

@@ -14,7 +14,7 @@
   <a href="https://www.npmjs.com/package/agentpond"><img src="https://img.shields.io/node/v/agentpond.svg" alt="Node.js version"></a>
 </p>
 
-AgentPond is a lightweight trace backend and CLI for AI applications. It keeps raw traces in object storage you control and syncs them into a local DuckDB cache for fast analysis by your coding agent. Use it with Firebase Storage, Supabase Storage, Amazon S3, Google Cloud Storage, Vercel Blob, or custom infrastructure. Firebase, Supabase, and Vercel projects include automated setup; other deployments use a manual setup path.
+AgentPond is a lightweight trace backend and CLI for AI applications. It keeps raw traces in object storage you control and syncs them into a local DuckDB cache for fast analysis by your coding agent. It supports all bucket providers from [Files SDK](https://github.com/haydenbleasel/files-sdk) such as Amazon S3, Google Cloud Storage, R2, and MinIO; For platforms like Firebase, Supabase, and Vercel there are explicit one-command quick starts.
 
 ## How it works
 
@@ -24,21 +24,15 @@ Object storage is the durable source of truth. The local DuckDB database is a re
 
 ## Getting started
 
-Start with the [Manual deployment setup](./docs/getting-started/manual-setup.md) to configure object storage, instrument the application, and sync its traces into AgentPond.
+Start with the [Manual deployment setup](./docs/getting-started/manual-setup.md) to configure object storage, instrument your application, and sync its traces into AgentPond.
 
-For Firebase, Supabase, and Vercel projects, AgentPond provides automated quick starts:
-
-- [Firebase quick start](./docs/getting-started/firebase.md)
-- [Supabase quick start](./docs/getting-started/supabase.md)
-- [Vercel quick start](./docs/getting-started/vercel.md)
-
-All require Node.js 22 or newer for the CLI. From your project folder, run:
+For [Firebase](./docs/getting-started/firebase.md), [Supabase](./docs/getting-started/supabase.md), and [Vercel](./docs/getting-started/vercel.md) projects, to get started, just run:
 
 ```sh
 npx agentpond init
 ```
 
-The command detects the platform, installs AgentPond's instrumentation and analytics skills, and prints a install prompt for your coding agent. Your agent inspects then the application, proposes a setup, implements it after confirmation, and verifies a real trace end to end.
+The command installs AgentPond's instrumentation and analytics skills, and prints a install prompt for your coding agent. Your coding agent inspects then the application, proposes a setup, implements it after confirmation, and verifies a real trace end to end.
 
 Once the application has emitted a trace:
 
@@ -47,26 +41,9 @@ npx agentpond sync
 npx agentpond traces list --limit 10
 ```
 
-## Analyze traces
-
-Use focused commands for individual traces and sessions:
-
-```sh
-npx agentpond traces get <trace-id>
-npx agentpond observations list --traceId <trace-id>
-npx agentpond sessions get <session-id>
-npx agentpond scores list --traceId <trace-id>
-```
-
-Use DuckDB SQL for aggregation, joins, time windows, or cost analysis:
-
-```sh
-npx agentpond sql "select id, name, session_id, total_cost from traces order by start_time desc limit 10"
-```
-
 ## Features
 
-- Direct OpenTelemetry export to Firebase Storage, Supabase Storage, S3, GCS, and Vercel Blob
+- Direct OpenTelemetry export through Files SDK, Firebase Storage, Supabase Storage, and Vercel Blob
 - Langfuse-compatible and OTLP HTTP ingestion adapters
 - Incremental object-store synchronization
 - Local DuckDB projections for traces, observations, sessions, and scores

apps/cli/package.json

@@ -25,18 +25,25 @@
 		"directory": "apps/cli"
 	},
 	"dependencies": {
-		"@agentpond/aws": "workspace:*",
 		"@agentpond/core": "workspace:*",
 		"@agentpond/duckdb": "workspace:*",
 		"@agentpond/fastify-ingest": "workspace:*",
 		"@agentpond/firebase": "workspace:*",
-		"@agentpond/google": "workspace:*",
+		"@agentpond/files-sdk": "workspace:*",
 		"@agentpond/supabase": "workspace:*",
 		"@agentpond/vercel": "workspace:*",
+		"@aws-sdk/client-s3": "3.1068.0",
+		"@aws-sdk/s3-presigned-post": "3.1068.0",
+		"@aws-sdk/s3-request-presigner": "3.1068.0",
 		"@duckdb/node-api": "^1.5.4-r.1",
+		"@google-cloud/storage": "^7.21.0",
 		"@inquirer/prompts": "8.5.2",
+		"@supabase/storage-js": "2.110.6",
 		"commander": "15.0.0",
+		"disk": "^0.8.18",
 		"fastify": "^5.6.2",
+		"files-sdk": "2.2.1",
+		"node-appwrite": "^26.0.0",
 		"protobufjs": "^7.6.5",
 		"skills": "1.5.9",
 		"zod": "^4.3.6"

apps/cli/src/commands/env.ts

@@ -1,15 +1,16 @@
 import { existsSync } from "node:fs";
 import {
-	type AgentPondStoreType,
+	type FilesSdkEnvironmentConfig,
 	initAgentPondEnvironment,
 	listAgentPondEnvironments,
 	parseEnvFileEntries,
 	readDevServerLock,
 	resolveAgentPondEnvironment,
 	selectAgentPondEnvironment,
 } from "@agentpond/core";
-import { select } from "@inquirer/prompts";
+import { input, select } from "@inquirer/prompts";
 import type { Command } from "commander";
+import { getProvider, PROVIDER_NAMES } from "files-sdk/providers";
 import { CliError, print } from "../cli-support.js";
 import { addGlobalOptions, type GlobalOptions } from "../command-support.js";
 import {
@@ -21,6 +22,7 @@ import {
 import {
 	environmentContextForCommand,
 	manualEnvironmentContextForCommand,
+	validateManualEnvironment,
 } from "../environment-context.js";
 import { providerForCommand } from "../providers.js";
 
@@ -30,24 +32,36 @@ export type SelectPrompt<T extends string> = (config: {
 }) => Promise<T>;
 
 export type SelectEnvironmentPrompt = SelectPrompt<string>;
-export type AgentPondInitStore = AgentPondStoreType;
-export type SelectStorePrompt = SelectPrompt<AgentPondInitStore>;
+export type SelectFilesProviderPrompt = SelectPrompt<string>;
+export type InputPrompt = (config: {
+	message: string;
+	default?: string;
+}) => Promise<string>;
 
 type EnvOptions = GlobalOptions & {
+	bucket?: string;
+	endpoint?: string;
 	langfuse?: boolean;
 	otel?: boolean;
-	store?: string;
+	provider?: string;
+	region?: string;
 };
 
 export function registerEnvCommand(
 	program: Command,
 	options: {
 		selectEnvironment?: SelectEnvironmentPrompt;
-		selectStore?: SelectStorePrompt;
+		selectFilesProvider?: SelectFilesProviderPrompt;
+		inputBucket?: InputPrompt;
+		inputEndpoint?: InputPrompt;
+		inputRegion?: InputPrompt;
 	} = {},
 ): void {
 	const promptSelect = options.selectEnvironment ?? select<string>;
-	const promptStore = options.selectStore ?? select<AgentPondInitStore>;
+	const promptFilesProvider = options.selectFilesProvider ?? select<string>;
+	const promptBucket = options.inputBucket ?? input;
+	const promptEndpoint = options.inputEndpoint ?? input;
+	const promptRegion = options.inputRegion ?? input;
 	const env = addGlobalOptions(
 		program.command("env").description("select and manage environments"),
 	);
@@ -95,32 +109,51 @@ export function registerEnvCommand(
 				name,
 				selected: name === selected,
 			}));
+			for (const name of new Set([selected, ...rows.map(({ name }) => name)])) {
+				validateManualEnvironment(cwd, name);
+			}
 			return print(rows, Boolean(globalOptions.json));
 		});
 
 	addGlobalOptions(env.command("init <name>"))
 		.description("initialize a manual environment")
-		.option("--store <store>", "object store: s3, gcs, or local")
+		.option("--provider <provider>", "Files SDK bucket provider")
+		.option("--bucket <bucket>", "Files SDK bucket name")
+		.option("--endpoint <endpoint>", "Files SDK provider endpoint")
+		.option("--region <region>", "Files SDK provider region")
 		.action(
 			async (name: string, commandOptions: EnvOptions, command: Command) => {
+				if (name === "dev") {
+					throw new CliError(
+						'The "dev" environment is managed by `npx agentpond dev`',
+					);
+				}
 				const globalOptions = command.optsWithGlobals<GlobalOptions>();
 				const context = manualEnvironmentContextForCommand("init", {
 					envName: name,
 					platform: globalOptions.platform,
 				});
-				const store =
-					storeFromValue(commandOptions.store) ??
-					(await promptForStore(promptStore));
+				const filesSdk = await filesSdkConfigForOptions(
+					commandOptions,
+					promptFilesProvider,
+					promptBucket,
+					promptEndpoint,
+					promptRegion,
+				);
 				const environment = initAgentPondEnvironment(name, {
 					cwd: context.rootDir,
-					storeType: store,
+					filesSdk,
 				});
+				validateManualEnvironment(context.rootDir, environment.name);
 				return print(
 					{
+						bucket: filesSdk.bucket,
 						name: environment.name,
 						envFile: environment.envFilePath,
 						dbPath: environment.dbPath,
-						store,
+						provider: filesSdk.provider,
+						...(filesSdk.endpoint ? { endpoint: filesSdk.endpoint } : {}),
+						...(filesSdk.region ? { region: filesSdk.region } : {}),
 					},
 					Boolean(globalOptions.json),
 				);
@@ -169,37 +202,134 @@ async function selectEnvironmentForCommand(
 	});
 	const selectedName =
 		name ?? (await promptForEnvironmentName(promptSelect, context.rootDir));
+	validateManualEnvironment(context.rootDir, selectedName);
 	return selectAgentPondEnvironment(selectedName, {
 		cwd: context.rootDir,
 	}).name;
 }
 
-function storeFromValue(
-	value: string | undefined,
-): AgentPondInitStore | undefined {
-	if (value === undefined) return undefined;
-	if (value === "s3" || value === "gcs" || value === "local") {
-		return value;
-	}
-	throw new CliError(`--store must be s3, gcs, or local, got "${value}"`);
+async function filesSdkConfigForOptions(
+	options: Pick<EnvOptions, "bucket" | "endpoint" | "provider" | "region">,
+	promptProvider: SelectFilesProviderPrompt,
+	promptBucket: InputPrompt,
+	promptEndpoint: InputPrompt,
+	promptRegion: InputPrompt,
+): Promise<FilesSdkEnvironmentConfig> {
+	const provider =
+		options.provider ?? (await promptForFilesProvider(promptProvider));
+	const definition = bucketProvider(provider);
+	const bucket = (
+		options.bucket ?? (await promptForFilesBucket(promptBucket))
+	).trim();
+	if (!bucket) throw new CliError("Missing --bucket");
+
+	const endpoint = await configuredProviderValue(
+		"endpoint",
+		definition.env.config?.includes("endpoint") ?? false,
+		options.endpoint,
+		promptEndpoint,
+	);
+	const region = await configuredProviderValue(
+		"region",
+		definition.env.config?.includes("region") ?? false,
+		options.region,
+		promptRegion,
+	);
+	return {
+		provider,
+		bucket,
+		...(endpoint ? { endpoint } : {}),
+		...(region ? { region } : {}),
+	};
 }
 
-async function promptForStore(
-	promptSelect: SelectStorePrompt,
-): Promise<AgentPondInitStore> {
+async function promptForFilesProvider(
+	promptSelect: SelectFilesProviderPrompt,
+): Promise<string> {
 	if (!process.stdin.isTTY || !process.stdout.isTTY) {
-		throw new CliError("Missing --store");
+		throw new CliError("Missing --provider");
 	}
 	return promptSelect({
-		message: "Select AgentPond object store",
-		choices: [
-			{ name: "AWS S3 (or compatible)", value: "s3" },
-			{ name: "Google Cloud Storage (GCS)", value: "gcs" },
-			{ name: "Local filesystem", value: "local" },
-		],
+		message: "Select Files SDK bucket provider",
+		choices: bucketProviders().map(({ name, slug }) => ({
+			name: `${name} (${slug})`,
+			value: slug,
+		})),
 	});
 }
 
+async function promptForFilesBucket(promptInput: InputPrompt): Promise<string> {
+	if (!process.stdin.isTTY || !process.stdout.isTTY) {
+		throw new CliError("Missing --bucket");
+	}
+	return promptInput({
+		message: "Files SDK bucket",
+		default: "agentpond",
+	});
+}
+
+function bucketProvider(provider: string) {
+	const definition = getProvider(provider);
+	if (!definition) {
+		throw new CliError(
+			`Unknown Files SDK provider "${provider}". Bucket providers: ${bucketProviders()
+				.map(({ slug }) => slug)
+				.join(", ")}`,
+		);
+	}
+	if (!definition.env.config?.includes("bucket")) {
+		throw new CliError(
+			`Files SDK provider "${provider}" is not bucket-backed and is not supported by AgentPond`,
+		);
+	}
+	if (provider === "bun-s3") {
+		throw new CliError(
+			'Files SDK provider "bun-s3" is not supported by AgentPond\'s Node.js runtime',
+		);
+	}
+	for (const field of definition.env.config) {
+		if (field !== "bucket" && field !== "endpoint" && field !== "region") {
+			throw new CliError(
+				`Files SDK provider "${provider}" requires unsupported configuration field "${field}"`,
+			);
+		}
+	}
+	return definition;
+}
+
+function bucketProviders() {
+	return PROVIDER_NAMES.flatMap((provider) => {
+		const definition = getProvider(provider);
+		return provider !== "bun-s3" &&
+			definition?.env.config?.includes("bucket") &&
+			definition.env.config.every(
+				(field) =>
+					field === "bucket" || field === "endpoint" || field === "region",
+			)
+			? [definition]
+			: [];
+	});
+}
+
+async function configuredProviderValue(
+	field: "endpoint" | "region",
+	required: boolean,
+	value: string | undefined,
+	promptInput: InputPrompt,
+): Promise<string | undefined> {
+	const configured = value?.trim();
+	if (configured) return configured;
+	if (!required) return undefined;
+	if (!process.stdin.isTTY || !process.stdout.isTTY) {
+		throw new CliError(`Missing --${field}`);
+	}
+	const prompted = (
+		await promptInput({ message: `Files SDK ${field}` })
+	).trim();
+	if (!prompted) throw new CliError(`Missing --${field}`);
+	return prompted;
+}
+
 async function promptForEnvironmentName(
 	promptSelect: SelectEnvironmentPrompt,
 	cwd: string,