diff --git a/packages/core/src/agent-os.ts b/packages/core/src/agent-os.ts index eae7c6cb03..9e6b6533c6 100644 --- a/packages/core/src/agent-os.ts +++ b/packages/core/src/agent-os.ts @@ -699,7 +699,10 @@ export class AgentOs { await kernel.mount( createWasmVmRuntime( processed.commandDirs.length > 0 - ? { commandDirs: processed.commandDirs } + ? { + commandDirs: processed.commandDirs, + permissions: processed.commandPermissions, + } : undefined, ), ); diff --git a/packages/core/src/packages.ts b/packages/core/src/packages.ts index e20338b781..2ec8e88e4a 100644 --- a/packages/core/src/packages.ts +++ b/packages/core/src/packages.ts @@ -1,5 +1,6 @@ import { readFileSync, realpathSync, existsSync } from "node:fs"; import { join, dirname } from "node:path"; +import type { PermissionTier } from "@rivet-dev/agent-os-posix"; /** * Resolve a package directory by walking up the directory tree. @@ -99,6 +100,7 @@ export interface WasmCommandSoftwareDescriptor extends SoftwareDescriptor { full?: string[]; readWrite?: string[]; readOnly?: string[] | "*"; + isolated?: string[]; }; } @@ -228,6 +230,8 @@ export function defineSoftware(desc: T): T { export interface ProcessedSoftware { /** WASM command directories to pass to the WasmVM driver. */ commandDirs: string[]; + /** Per-command permission tiers propagated into the WasmVM runtime. */ + commandPermissions: Record; /** Host-to-VM path mappings for ModuleAccessFileSystem. */ softwareRoots: SoftwareRoot[]; /** Agent configs registered by agent software. */ @@ -239,6 +243,73 @@ function isTypedDescriptor(desc: AnySoftwareDescriptor): desc is AgentSoftwareDe return "type" in desc && typeof (desc as SoftwareDescriptor).type === "string"; } +const VALID_PERMISSION_TIERS = new Set([ + "full", + "read-write", + "read-only", + "isolated", +]); + +function isPermissionTier(value: unknown): value is PermissionTier { + return typeof value === "string" && VALID_PERMISSION_TIERS.has(value as PermissionTier); +} + +function registerPermission( + commandPermissions: Record, + commandName: string, + tier: PermissionTier, +): void { + if (commandName in commandPermissions) return; + commandPermissions[commandName] = tier; +} + +function collectRegistryPackagePermissions( + commandPermissions: Record, + pkg: WasmCommandDirDescriptor, +): void { + const rawCommands = (pkg as { commands?: unknown }).commands; + if (!Array.isArray(rawCommands)) return; + + for (const rawCommand of rawCommands) { + if ( + typeof rawCommand !== "object" || + rawCommand === null || + !Object.hasOwn(rawCommand, "name") || + !Object.hasOwn(rawCommand, "permissionTier") + ) { + continue; + } + + const name = (rawCommand as { name: unknown }).name; + const permissionTier = (rawCommand as { permissionTier: unknown }).permissionTier; + if (typeof name !== "string" || !isPermissionTier(permissionTier)) continue; + registerPermission(commandPermissions, name, permissionTier); + } +} + +function collectTypedDescriptorPermissions( + commandPermissions: Record, + pkg: WasmCommandSoftwareDescriptor, +): void { + const permissions = pkg.permissions; + if (!permissions) return; + + for (const commandName of permissions.full ?? []) { + registerPermission(commandPermissions, commandName, "full"); + } + for (const commandName of permissions.readWrite ?? []) { + registerPermission(commandPermissions, commandName, "read-write"); + } + if (Array.isArray(permissions.readOnly)) { + for (const commandName of permissions.readOnly) { + registerPermission(commandPermissions, commandName, "read-only"); + } + } + for (const commandName of permissions.isolated ?? []) { + registerPermission(commandPermissions, commandName, "isolated"); + } +} + /** * Process an array of software descriptors at boot time. * Collects WASM command dirs, module access roots, and agent configurations. @@ -251,6 +322,7 @@ export function processSoftware( software: SoftwareInput[], ): ProcessedSoftware { const commandDirs: string[] = []; + const commandPermissions: Record = {}; const softwareRoots: SoftwareRoot[] = []; const agentConfigs = new Map(); @@ -261,12 +333,14 @@ export function processSoftware( if (!isTypedDescriptor(pkg)) { // Duck-typed: any object with commandDir is a WASM command source. commandDirs.push(pkg.commandDir); + collectRegistryPackagePermissions(commandPermissions, pkg); continue; } switch (pkg.type) { case "wasm-commands": { commandDirs.push(pkg.commandDir); + collectTypedDescriptorPermissions(commandPermissions, pkg); break; } @@ -313,5 +387,5 @@ export function processSoftware( } } - return { commandDirs, softwareRoots, agentConfigs }; + return { commandDirs, commandPermissions, softwareRoots, agentConfigs }; } diff --git a/packages/core/tests/duckdb-package.test.ts b/packages/core/tests/duckdb-package.test.ts new file mode 100644 index 0000000000..5ad86307db --- /dev/null +++ b/packages/core/tests/duckdb-package.test.ts @@ -0,0 +1,145 @@ +import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; +import { existsSync } from "node:fs"; +import coreutils from "@rivet-dev/agent-os-coreutils"; +import duckdb from "../../../registry/software/duckdb/dist/index.js"; +import httpGet from "../../../registry/software/http-get/dist/index.js"; +import { AgentOs } from "../src/index.js"; + +const hasDuckdbPackage = existsSync(`${duckdb.commandDir}/duckdb`); +const hasHttpGetPackage = existsSync(`${httpGet.commandDir}/http_get`); +const hasCoreutilsPackage = existsSync(`${coreutils.commandDir}/sh`); + +function closeServer(server: Server) { + return new Promise((resolve, reject) => { + server.close((err) => { + if (err) reject(err); + else resolve(); + }); + }); +} + +describe.skipIf(!hasDuckdbPackage || !hasHttpGetPackage || !hasCoreutilsPackage)( + "duckdb registry package", + () => { + let vm: AgentOs; + + beforeEach(async () => { + vm = await AgentOs.create({ software: [coreutils, httpGet, duckdb] }); + }); + + afterEach(async () => { + await vm.dispose(); + }); + + test("runs file-backed DuckDB DML through the registry package path", async () => { + let result = await vm.exec( + `duckdb -csv /tmp/app.duckdb -c "CREATE TABLE items(id INTEGER, value INTEGER); INSERT INTO items VALUES (1, 10), (2, 20); UPDATE items SET value = value + 1 WHERE id = 2;"`, + ); + expect(result.exitCode).toBe(0); + + result = await vm.exec( + `duckdb -csv /tmp/app.duckdb -c "SELECT id, value FROM items ORDER BY id;"`, + ); + expect(result.exitCode).toBe(0); + expect(result.stdout.trim()).toBe("id,value\n1,10\n2,21"); + expect(await vm.exists("/tmp/app.duckdb")).toBe(true); + }); + + test("fetches remote CSV data into the VFS and queries it from DuckDB", async () => { + const server = createServer((req: IncomingMessage, res: ServerResponse) => { + if (req.url === "/remote.csv") { + res.writeHead(200, { "Content-Type": "text/csv" }); + res.end("city,value\nsf,3\nla,5\n"); + return; + } + + res.writeHead(404, { "Content-Type": "text/plain" }); + res.end("not found"); + }); + + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("failed to bind test HTTP server"); + } + + let result = await vm.exec( + `http_get ${address.port} /remote.csv /tmp/remote.csv`, + ); + expect(result.exitCode).toBe(0); + + result = await vm.exec( + `duckdb -csv -c "SELECT SUM(value) AS total FROM read_csv_auto('/tmp/remote.csv');"`, + ); + expect(result.exitCode).toBe(0); + expect(result.stdout.trim()).toBe("total\n8"); + } finally { + await closeServer(server); + } + }); + + test("keeps DuckDB itself file-scoped while the network helper handles remote fetches", async () => { + let requests = 0; + const server = createServer((req: IncomingMessage, res: ServerResponse) => { + requests += 1; + if (req.url === "/remote.csv") { + res.writeHead(200, { "Content-Type": "text/csv" }); + res.end("city,value\nsf,3\nla,5\n"); + return; + } + + res.writeHead(404, { "Content-Type": "text/plain" }); + res.end("not found"); + }); + + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("failed to bind test HTTP server"); + } + + const result = await vm.exec( + `duckdb -csv -c "SELECT SUM(value) AS total FROM read_csv_auto('http://127.0.0.1:${address.port}/remote.csv');"`, + ); + expect(result.exitCode).not.toBe(0); + expect(requests).toBe(0); + } finally { + await closeServer(server); + } + }); + + test("propagates registry package command permission tiers into the runtime", async () => { + await vm.dispose(); + + const httpGetReadOnly = { + ...httpGet, + commands: [{ name: "http_get", permissionTier: "read-only" as const }], + }; + vm = await AgentOs.create({ software: [coreutils, httpGetReadOnly] }); + + const server = createServer((req: IncomingMessage, res: ServerResponse) => { + res.writeHead(200, { "Content-Type": "text/plain" }); + res.end("ok"); + }); + + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("failed to bind test HTTP server"); + } + + const result = await vm.exec(`http_get ${address.port} /blocked`); + expect(result.exitCode).not.toBe(0); + } finally { + await closeServer(server); + } + }); + }, +); diff --git a/packages/posix/src/driver.ts b/packages/posix/src/driver.ts index bb5a1b8237..3128f3308a 100644 --- a/packages/posix/src/driver.ts +++ b/packages/posix/src/driver.ts @@ -1042,6 +1042,13 @@ class WasmVmRuntimeDriver implements RuntimeDriver { await kernel.vfs.mkdir(scopedProcPath(pid, msg.args.path as string)); break; } + case 'vfsTruncate': { + await kernel.vfs.truncate( + scopedProcPath(pid, msg.args.path as string), + msg.args.length as number, + ); + break; + } case 'vfsUnlink': { await kernel.vfs.removeFile(scopedProcPath(pid, msg.args.path as string)); break; diff --git a/packages/posix/src/kernel-worker.ts b/packages/posix/src/kernel-worker.ts index d9f14c1546..025f623b2b 100644 --- a/packages/posix/src/kernel-worker.ts +++ b/packages/posix/src/kernel-worker.ts @@ -27,6 +27,9 @@ import { ERRNO_EBADF, FDFLAG_NONBLOCK, RIGHT_FD_FDSTAT_SET_FLAGS, + ERRNO_ENOENT, + RIGHT_FD_READ, + RIGHT_FD_WRITE, } from './wasi-constants.js'; import { VfsError } from './wasi-types.js'; import type { WasiVFS, WasiInode, VfsStat, VfsSnapshotEntry } from './wasi-types.js'; @@ -251,6 +254,7 @@ function createKernelFileIO(): WasiFileIO { return { errno: res.errno, written: res.intResult }; }, fdOpen(path, dirflags, oflags, fdflags, rightsBase, rightsInheriting) { + const createIfMissing = !!(oflags & 0x1); // OFLAG_CREAT const wantDirectory = !!(oflags & 0x2); // OFLAG_DIRECTORY // Permission check: isolated tier restricts reads to cwd subtree @@ -259,7 +263,9 @@ function createKernelFileIO(): WasiFileIO { } // Permission check: block write flags for read-only/isolated tiers - const hasWriteIntent = !!(oflags & 0x1) || !!(oflags & 0x8) || !!(fdflags & 0x1) || !!(rightsBase & 2n); + const wantsRead = !!(rightsBase & RIGHT_FD_READ); + const wantsWrite = !!(rightsBase & RIGHT_FD_WRITE); + const hasWriteIntent = !!(oflags & 0x1) || !!(oflags & 0x8) || !!(fdflags & 0x1) || wantsWrite; if (isWriteBlocked() && hasWriteIntent) { return { errno: ERRNO_EACCES, fd: -1, filetype: 0 }; } @@ -293,13 +299,24 @@ function createKernelFileIO(): WasiFileIO { return { errno: 0, fd: localFd, filetype: FILETYPE_DIRECTORY }; } + // The kernel FD layer only materializes missing paths for O_CREAT/O_EXCL/O_TRUNC + // via prepareOpenSync. For a plain open on a nonexistent file, reject here so + // callers observe POSIX/WASI ENOENT instead of receiving an unusable descriptor. + if (!createIfMissing) { + const statRes = rpcCall('vfsStat', { path }); + if (statRes.errno !== 0) { + return { errno: ERRNO_ENOENT, fd: -1, filetype: 0 }; + } + } + // Map WASI oflags to POSIX open flags for kernel let flags = 0; if (oflags & 0x1) flags |= 0o100; // O_CREAT if (oflags & 0x4) flags |= 0o200; // O_EXCL if (oflags & 0x8) flags |= 0o1000; // O_TRUNC if (fdflags & 0x1) flags |= 0o2000; // O_APPEND - if (rightsBase & 2n) flags |= 1; // O_WRONLY + if (wantsRead && wantsWrite) flags |= 2; // O_RDWR + else if (wantsWrite) flags |= 1; // O_WRONLY const res = rpcCall('fdOpen', { path, flags, mode: 0o666 }); if (res.errno !== 0) return { errno: res.errno, fd: -1, filetype: 0 }; @@ -452,6 +469,33 @@ function createKernelVfs(): WasiVFS { return ino; } + function refreshCachedInode(ino: number): void { + const path = inoToPath.get(ino); + const node = inoCache.get(ino); + if (!path || !node) return; + + if (permissionTier === 'isolated' && !isPathInCwd(path)) return; + + const res = rpcCall('vfsStat', { path }); + if (res.errno !== 0) return; + + const raw = JSON.parse(decoder.decode(res.data)) as Record; + const nodeType = (raw.type as string) ?? 'file'; + node.type = nodeType; + node.mode = (raw.mode as number) ?? node.mode; + node.uid = (raw.uid as number) ?? node.uid; + node.gid = (raw.gid as number) ?? node.gid; + node.nlink = (raw.nlink as number) ?? node.nlink; + node.atime = (raw.atime as number) ?? node.atime; + node.mtime = (raw.mtime as number) ?? node.mtime; + node.ctime = (raw.ctime as number) ?? node.ctime; + (node as WasiInode & { size: number }).size = (raw.size as number) ?? node.size; + + if (nodeType === 'dir') { + node.entries ??= new Map(); + } + } + /** Lazy-populate directory entries from kernel VFS readdir. */ function populateDirEntries(ino: number, node: WasiInode): void { if (populatedDirs.has(ino)) return; @@ -504,6 +548,22 @@ function createKernelVfs(): WasiVFS { const bytes = typeof data === 'string' ? new TextEncoder().encode(data) : data; rpcCall('vfsWriteFile', { path, data: Array.from(bytes) }); }, + truncate(path: string, length: number): void { + if (isWriteBlocked()) throw new VfsError('EACCES', path); + const res = rpcCall('vfsTruncate', { path, length }); + if (res.errno !== 0) throw new VfsError('EINVAL', path); + + const cachedIno = pathToIno.get(path); + if (cachedIno !== undefined) { + const node = inoCache.get(cachedIno); + if (node) { + const mutableNode = node as WasiInode & { size: number }; + mutableNode.size = length; + mutableNode.mtime = Date.now(); + mutableNode.ctime = Date.now(); + } + } + }, readFile(path: string): Uint8Array { // Isolated tier: restrict reads to cwd subtree if (permissionTier === 'isolated' && !isPathInCwd(path)) { @@ -583,6 +643,7 @@ function createKernelVfs(): WasiVFS { getInodeByIno(ino: number): WasiInode | null { const node = inoCache.get(ino); if (!node) return null; + refreshCachedInode(ino); // Lazy-populate directory entries from kernel VFS if (node.type === 'dir' && node.entries) { populateDirEntries(ino, node); diff --git a/packages/posix/src/wasi-polyfill.ts b/packages/posix/src/wasi-polyfill.ts index 210d4dde56..c1231637eb 100644 --- a/packages/posix/src/wasi-polyfill.ts +++ b/packages/posix/src/wasi-polyfill.ts @@ -981,17 +981,15 @@ export class WasiPolyfill { if (!(entry.rightsBase & RIGHT_FD_FILESTAT_SET_SIZE)) return ERRNO_EBADF; if (entry.resource.type !== 'vfsFile') return ERRNO_EINVAL; - const node = this.vfs.getInodeByIno(entry.resource.ino); - if (!node || node.type !== 'file') return ERRNO_EINVAL; - const newSize = Number(typeof size === 'bigint' ? size : BigInt(size)); - if (newSize === node.data!.length) return ERRNO_SUCCESS; + const ino = entry.resource.ino !== 0 + ? entry.resource.ino + : this.vfs.getIno(entry.resource.path, true); + if (ino === null) return ERRNO_EBADF; - const newData = new Uint8Array(newSize); - newData.set(node.data!.subarray(0, Math.min(node.data!.length, newSize))); - node.data = newData; - node.mtime = Date.now(); - node.ctime = Date.now(); + const node = this.vfs.getInodeByIno(ino); + if (!node || node.type !== 'file') return ERRNO_EINVAL; + this.vfs.truncate(entry.resource.path, newSize); return ERRNO_SUCCESS; } @@ -1004,7 +1002,11 @@ export class WasiPolyfill { if (!(entry.rightsBase & RIGHT_FD_FILESTAT_SET_TIMES)) return ERRNO_EBADF; if (entry.resource.type === 'vfsFile') { - const node = this.vfs.getInodeByIno(entry.resource.ino); + const ino = entry.resource.ino !== 0 + ? entry.resource.ino + : this.vfs.getIno(entry.resource.path, true); + if (ino === null) return ERRNO_EBADF; + const node = this.vfs.getInodeByIno(ino); if (!node) return ERRNO_EBADF; this._applyTimestamps(node, atim, mtim, fst_flags); } else if (entry.resource.type === 'preopen') { diff --git a/packages/posix/src/wasi-types.ts b/packages/posix/src/wasi-types.ts index 4f1f3a1d0c..000ca78d04 100644 --- a/packages/posix/src/wasi-types.ts +++ b/packages/posix/src/wasi-types.ts @@ -101,6 +101,7 @@ export interface WasiVFS { mkdir(path: string): void; mkdirp(path: string): void; writeFile(path: string, content: Uint8Array | string): void; + truncate(path: string, length: number): void; readFile(path: string): Uint8Array; readdir(path: string): string[]; stat(path: string): VfsStat; diff --git a/packages/posix/test/helpers/test-vfs.ts b/packages/posix/test/helpers/test-vfs.ts index bafe28edb9..1beb3f4d9a 100644 --- a/packages/posix/test/helpers/test-vfs.ts +++ b/packages/posix/test/helpers/test-vfs.ts @@ -404,6 +404,28 @@ export class VFS implements WasiVFS { dir.mtime = Date.now(); } + /** + * Resize a file, extending with zero bytes or truncating in place. + * @throws If the path does not exist or is not a regular file + */ + truncate(path: string, length: number): void { + const ino = this._resolve(path, true); + if (ino === null) throw new VfsError('ENOENT', `no such file: ${path}`); + + const node = this._getInode(ino)!; + if (node.type !== INODE_FILE) { + throw new VfsError(node.type === INODE_DIR ? 'EISDIR' : 'EINVAL', `not a regular file: ${path}`); + } + if (length < 0) throw new VfsError('EINVAL', `invalid length: ${length}`); + if (node.data!.length === length) return; + + const next = new Uint8Array(length); + next.set(node.data!.subarray(0, Math.min(node.data!.length, length))); + node.data = next; + node.mtime = Date.now(); + node.ctime = Date.now(); + } + /** * Read a file's contents. * @throws If file doesn't exist or is a directory diff --git a/packages/posix/test/wasi-path-ops.test.ts b/packages/posix/test/wasi-path-ops.test.ts index b7220edfbf..72c05f138f 100644 --- a/packages/posix/test/wasi-path-ops.test.ts +++ b/packages/posix/test/wasi-path-ops.test.ts @@ -545,6 +545,20 @@ describe('WasiPolyfill - Path Operations (US-008)', () => { expect(data[4]).toBe(0); }); + it('resolves path-backed file descriptors whose inode is synthesized later', () => { + const { wasi, vfs, fdTable } = createTestSetup(); + vfs.writeFile('/tmp/kernel-opened.txt', 'abcdef'); + const fd = fdTable.open( + { type: 'vfsFile', ino: 0, path: '/tmp/kernel-opened.txt' }, + { filetype: FILETYPE_REGULAR_FILE } + ); + + const errno = wasi.fd_filestat_set_size(fd, 3n); + expect(errno).toBe(ERRNO_SUCCESS); + const content = new TextDecoder().decode(vfs.readFile('/tmp/kernel-opened.txt')); + expect(content).toBe('abc'); + }); + it('returns EBADF for invalid fd', () => { const { wasi } = createTestSetup(); const errno = wasi.fd_filestat_set_size(99, 0n); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0c210b9095..264bb74784 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -677,6 +677,18 @@ importers: specifier: ^5.9.2 version: 5.9.3 + registry/software/duckdb: + devDependencies: + '@rivet-dev/agent-os-registry-types': + specifier: link:../../../packages/registry-types + version: link:../../../packages/registry-types + '@types/node': + specifier: ^22.15.17 + version: 22.19.15 + typescript: + specifier: ^5.9.2 + version: 5.9.3 + registry/software/everything: dependencies: '@rivet-dev/agent-os-codex': diff --git a/registry/Makefile b/registry/Makefile index 41f607215e..2f90ee9386 100644 --- a/registry/Makefile +++ b/registry/Makefile @@ -14,7 +14,7 @@ TS_MARKER := $(MARKER_DIR)/ts-built # Command packages (excludes _types and meta-packages) CMD_PACKAGES := coreutils sed grep gawk findutils diffutils tar gzip \ - curl wget zip unzip jq ripgrep fd tree file sqlite3 yq codex git + curl http-get wget zip unzip jq ripgrep fd tree file sqlite3 duckdb yq codex git # Meta-packages META_PACKAGES := common build-essential everything @@ -129,6 +129,10 @@ $(COPY_MARKER): $(RUST_MARKER) $(C_MARKER) @mkdir -p software/curl/wasm @[ -f "$(COMMANDS_DIR)/curl" ] && cp -f "$(COMMANDS_DIR)/curl" software/curl/wasm/ || echo " WARN: curl not found (needs patched sysroot)" + @# --- http-get --- + @mkdir -p software/http-get/wasm + @[ -f "$(COMMANDS_DIR)/http_get" ] && cp -f "$(COMMANDS_DIR)/http_get" software/http-get/wasm/ || echo " WARN: http_get not found" + @# --- wget (C build, needs patched sysroot) --- @mkdir -p software/wget/wasm @[ -f "$(COMMANDS_DIR)/wget" ] && cp -f "$(COMMANDS_DIR)/wget" software/wget/wasm/ || echo " WARN: wget not found (needs patched sysroot)" @@ -165,6 +169,10 @@ $(COPY_MARKER): $(RUST_MARKER) $(C_MARKER) @mkdir -p software/sqlite3/wasm @[ -f "$(COMMANDS_DIR)/sqlite3" ] && cp -f "$(COMMANDS_DIR)/sqlite3" software/sqlite3/wasm/ || echo " WARN: sqlite3 not found (needs patched sysroot)" + @# --- duckdb (upstream C++ build, needs patched sysroot) --- + @mkdir -p software/duckdb/wasm + @[ -f "$(COMMANDS_DIR)/duckdb" ] && cp -f "$(COMMANDS_DIR)/duckdb" software/duckdb/wasm/ || echo " WARN: duckdb not found (needs patched sysroot)" + @# --- yq --- @mkdir -p software/yq/wasm @[ -f "$(COMMANDS_DIR)/yq" ] && cp -f "$(COMMANDS_DIR)/yq" software/yq/wasm/ || echo " WARN: yq not found" diff --git a/registry/native/c/Makefile b/registry/native/c/Makefile index 7cf8527e01..2d2c90b7ad 100644 --- a/registry/native/c/Makefile +++ b/registry/native/c/Makefile @@ -63,7 +63,7 @@ NATIVE_CFLAGS := -O0 -g -I include/ COMMANDS_DIR ?= ../target/wasm32-wasip1/release/commands # Real commands to install (add command names as they are opted in) -COMMANDS := zip unzip envsubst sqlite3 curl wget +COMMANDS := zip unzip envsubst sqlite3 curl wget duckdb # Programs requiring patched sysroot (Tier 2+ custom host imports) PATCHED_PROGRAMS := isatty_test getpid_test getppid_test getppid_verify userinfo pipe_test dup_test spawn_child spawn_exit_code pipeline kill_child waitpid_return waitpid_edge syscall_coverage getpwuid_test signal_tests sigaction_behavior delayed_tcp_echo delayed_kill pipe_edge tcp_echo tcp_server udp_echo unix_socket signal_handler http_get dns_lookup sqlite3_cli curl wget @@ -88,9 +88,11 @@ endif WASM_PROG_NAMES := $(sort $(basename $(notdir $(SOURCES))) $(CUSTOM_WASM_PROG_NAMES)) NATIVE_PROG_NAMES := $(basename $(notdir $(SOURCES))) +PROGRAM_REPORT_NAMES := $(WASM_PROG_NAMES) duckdb # Default WASM output targets -WASM_OUTPUTS := $(addprefix $(BUILD_DIR)/,$(WASM_PROG_NAMES)) +EXTRA_WASM_OUTPUTS := $(BUILD_DIR)/duckdb +WASM_OUTPUTS := $(addprefix $(BUILD_DIR)/,$(WASM_PROG_NAMES)) $(EXTRA_WASM_OUTPUTS) # Native output targets NATIVE_OUTPUTS := $(addprefix $(NATIVE_DIR)/,$(NATIVE_PROG_NAMES)) @@ -137,6 +139,11 @@ CURL_UPSTREAM_BUILD_DIR := $(BUILD_DIR)/curl-upstream CURL_UPSTREAM_OVERLAY_DIR := curl-upstream-overlay CURL_UPSTREAM_OVERLAY_FILES := $(wildcard $(CURL_UPSTREAM_OVERLAY_DIR)/lib/*.c $(CURL_UPSTREAM_OVERLAY_DIR)/lib/*.h $(CURL_UPSTREAM_OVERLAY_DIR)/lib/vtls/*.c $(CURL_UPSTREAM_OVERLAY_DIR)/lib/vtls/*.h) MINIZIP_URL := https://github.com/madler/zlib/archive/refs/tags/v1.3.1.zip +# duckdb-wasm currently documents DuckDB v1.5.0 in its README. We build the +# matching upstream DuckDB tag ourselves with our patched WASI/POSIX sysroot. +DUCKDB_VERSION := v1.5.0 +DUCKDB_GIT_DESCRIBE := $(DUCKDB_VERSION)-0-g0123456789 +DUCKDB_URL := https://github.com/duckdb/duckdb/archive/refs/tags/$(DUCKDB_VERSION).tar.gz LIBS_DIR := libs LIBS_CACHE := .cache/libs @@ -183,8 +190,16 @@ libs/curl/lib/easy.c: @rm -rf $(LIBS_DIR)/curl @mv $(LIBS_CACHE)/secure-exec-curl-* $(LIBS_DIR)/curl +libs/duckdb/CMakeLists.txt: + @echo "Fetching DuckDB ($(DUCKDB_VERSION))..." + @mkdir -p $(LIBS_CACHE) + @curl -fSL "$(DUCKDB_URL)" -o "$(LIBS_CACHE)/duckdb.tar.gz" + @rm -rf $(LIBS_DIR)/duckdb + @mkdir -p $(LIBS_DIR)/duckdb + @tar -xzf "$(LIBS_CACHE)/duckdb.tar.gz" --strip-components=1 -C $(LIBS_DIR)/duckdb + .PHONY: fetch-libs clean-libs -fetch-libs: libs/sqlite3/sqlite3.c libs/zlib/zutil.c libs/minizip/ioapi.c libs/cjson/cJSON.c libs/curl/lib/easy.c +fetch-libs: libs/sqlite3/sqlite3.c libs/zlib/zutil.c libs/minizip/ioapi.c libs/cjson/cJSON.c libs/curl/lib/easy.c libs/duckdb/CMakeLists.txt clean-libs: rm -rf $(LIBS_DIR) $(LIBS_CACHE) @@ -383,9 +398,15 @@ libc-test-native: fetch-libc-test # --- Patched sysroot (delegates to patch-wasi-libc.sh) --- -sysroot: wasi-sdk +WASI_LIBC_PATCHES := $(wildcard ../patches/wasi-libc/*.patch) +WASI_LIBC_OVERRIDES := $(wildcard ../patches/wasi-libc-overrides/*.c) +LLVM_RUNTIME_PATCHES := $(wildcard patches/llvm-project/*.patch) + +$(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a: wasi-sdk ../scripts/patch-wasi-libc.sh scripts/build-llvm-runtimes.sh $(WASI_LIBC_PATCHES) $(WASI_LIBC_OVERRIDES) $(LLVM_RUNTIME_PATCHES) ../scripts/patch-wasi-libc.sh +sysroot: $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a + # --- wasm-opt check --- wasm-opt-check: @@ -403,7 +424,7 @@ wasm-opt-check: programs: wasi-sdk fetch-libs wasm-opt-check $(WASM_OUTPUTS) @echo "" @echo "=== C WASM Build Report ===" - @echo "Programs: $(words $(WASM_PROG_NAMES)) compiled" + @echo "Programs: $(words $(PROGRAM_REPORT_NAMES)) compiled" @echo "Sysroot: $(SYSROOT)" @echo "Output: $(BUILD_DIR)/" @echo "=== Build complete ===" @@ -492,11 +513,16 @@ $(NATIVE_DIR)/unzip: programs/unzip.c $(ZLIB_SRCS) $(MINIZIP_UNZIP_SRCS) @mkdir -p $(NATIVE_DIR) $(NATIVE_CC) $(NATIVE_CFLAGS) $(ZIP_INCLUDES) -o $@ programs/unzip.c $(ZLIB_SRCS) $(MINIZIP_UNZIP_SRCS) -lz +# curl_test: links libcurl (HTTP/HTTPS build for WASM via host_net + host_tls) +# Curl still expects stable local/peer socket-name queries after connect, so +# keep its WASI compatibility shim but route curl's own references onto private +# wrapper symbols to avoid colliding with our patched sysroot exports. CURL_SRCS := $(wildcard libs/curl/lib/*.c) $(wildcard libs/curl/lib/vauth/*.c) \ $(wildcard libs/curl/lib/vtls/*.c) $(wildcard libs/curl/lib/vquic/*.c) \ $(wildcard libs/curl/lib/vssh/*.c) CURL_INCLUDES := -Ilibs/curl/include -Ilibs/curl/lib -include libs/curl/lib/curl_setup.h -include libs/curl/lib/curl_printf.h -CURL_LIB_DEFS := -DHAVE_CONFIG_H -DBUILDING_LIBCURL -D_WASI_EMULATED_SIGNAL -DHAVE_BASENAME -DHAVE_LIBGEN_H +CURL_LIB_DEFS := -DHAVE_CONFIG_H -DBUILDING_LIBCURL -D_WASI_EMULATED_SIGNAL -DHAVE_BASENAME -DHAVE_LIBGEN_H \ + -Dgetsockname=curl_wasi_getsockname -Dgetpeername=curl_wasi_getpeername $(BUILD_DIR)/curl: scripts/build-curl-upstream.sh $(CURL_UPSTREAM_OVERLAY_FILES) $(WASI_SDK_DIR)/bin/clang @mkdir -p $(BUILD_DIR) @@ -525,6 +551,19 @@ $(NATIVE_DIR)/wget: programs/wget.c @mkdir -p $(NATIVE_DIR) $(NATIVE_CC) $(NATIVE_CFLAGS) -o $@ programs/wget.c -lcurl +# duckdb: upstream DuckDB CLI built from source with our patched WASI/POSIX sysroot +$(BUILD_DIR)/duckdb: libs/duckdb/CMakeLists.txt $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang $(WASI_SDK_DIR)/bin/clang++ cmake/FindThreads.cmake scripts/build-duckdb.sh include/fcntl.h include/ifaddrs.h include/net/if.h include/sys/ioctl.h + @mkdir -p $(BUILD_DIR) + DUCKDB_SRC_DIR="$(abspath $(LIBS_DIR)/duckdb)" \ + DUCKDB_BUILD_DIR="$(abspath $(BUILD_DIR)/duckdb-cmake)" \ + DUCKDB_OUTPUT="$(abspath $(BUILD_DIR)/duckdb)" \ + WASI_SDK_DIR="$(abspath $(WASI_SDK_DIR))" \ + SYSROOT_DIR="$(abspath $(PATCHED_SYSROOT))" \ + MODULE_PATH="$(abspath cmake)" \ + OVERLAY_INCLUDE_DIR="$(abspath include)" \ + DUCKDB_GIT_DESCRIBE="$(DUCKDB_GIT_DESCRIBE)" \ + bash scripts/build-duckdb.sh + # --- Compile all C programs to native binaries (for parity testing) --- native: $(NATIVE_OUTPUTS) diff --git a/registry/native/c/cmake/FindThreads.cmake b/registry/native/c/cmake/FindThreads.cmake new file mode 100644 index 0000000000..992033a665 --- /dev/null +++ b/registry/native/c/cmake/FindThreads.cmake @@ -0,0 +1,15 @@ +if(NOT TARGET Threads::Threads) + add_library(Threads::Threads INTERFACE IMPORTED) + set_target_properties( + Threads::Threads + PROPERTIES + INTERFACE_COMPILE_DEFINITIONS "_WASI_EMULATED_PTHREAD" + INTERFACE_LINK_LIBRARIES "wasi-emulated-pthread" + ) +endif() + +set(CMAKE_THREAD_LIBS_INIT "wasi-emulated-pthread" CACHE STRING "" FORCE) +set(CMAKE_USE_PTHREADS_INIT 0 CACHE BOOL "" FORCE) +set(CMAKE_USE_WIN32_THREADS_INIT 0 CACHE BOOL "" FORCE) +set(Threads_FOUND TRUE) +set(THREADS_FOUND TRUE) diff --git a/registry/native/c/include/fcntl.h b/registry/native/c/include/fcntl.h new file mode 100644 index 0000000000..cb426954d2 --- /dev/null +++ b/registry/native/c/include/fcntl.h @@ -0,0 +1,38 @@ +#ifndef REGISTRY_NATIVE_C_INCLUDE_FCNTL_H +#define REGISTRY_NATIVE_C_INCLUDE_FCNTL_H + +#include_next + +#ifndef F_DUPFD_CLOEXEC +#define F_DUPFD_CLOEXEC 1030 +#endif + +#ifndef F_RDLCK +#define F_RDLCK 0 +#endif +#ifndef F_WRLCK +#define F_WRLCK 1 +#endif +#ifndef F_UNLCK +#define F_UNLCK 2 +#endif + +#ifndef F_GETLK +#if __LONG_MAX == 0x7fffffffL +#define F_GETLK 12 +#define F_SETLK 13 +#define F_SETLKW 14 +#else +#define F_GETLK 5 +#define F_SETLK 6 +#define F_SETLKW 7 +#endif +#endif + +#ifndef F_GETLK64 +#define F_GETLK64 F_GETLK +#define F_SETLK64 F_SETLK +#define F_SETLKW64 F_SETLKW +#endif + +#endif diff --git a/registry/native/c/include/ifaddrs.h b/registry/native/c/include/ifaddrs.h new file mode 100644 index 0000000000..0c1b7320ca --- /dev/null +++ b/registry/native/c/include/ifaddrs.h @@ -0,0 +1,33 @@ +#ifndef REGISTRY_NATIVE_C_INCLUDE_IFADDRS_H +#define REGISTRY_NATIVE_C_INCLUDE_IFADDRS_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct ifaddrs { + struct ifaddrs *ifa_next; + char *ifa_name; + unsigned int ifa_flags; + struct sockaddr *ifa_addr; + struct sockaddr *ifa_netmask; + union { + struct sockaddr *ifu_broadaddr; + struct sockaddr *ifu_dstaddr; + } ifa_ifu; + void *ifa_data; +}; + +#define ifa_broadaddr ifa_ifu.ifu_broadaddr +#define ifa_dstaddr ifa_ifu.ifu_dstaddr + +int getifaddrs(struct ifaddrs **ifap); +void freeifaddrs(struct ifaddrs *ifa); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/registry/native/c/include/net/if.h b/registry/native/c/include/net/if.h new file mode 100644 index 0000000000..0a03c72f2d --- /dev/null +++ b/registry/native/c/include/net/if.h @@ -0,0 +1,12 @@ +#ifndef REGISTRY_NATIVE_C_INCLUDE_NET_IF_H +#define REGISTRY_NATIVE_C_INCLUDE_NET_IF_H + +#ifndef IFNAMSIZ +#define IFNAMSIZ 16 +#endif + +#ifndef IF_NAMESIZE +#define IF_NAMESIZE IFNAMSIZ +#endif + +#endif diff --git a/registry/native/c/include/sched.h b/registry/native/c/include/sched.h new file mode 100644 index 0000000000..81722bc010 --- /dev/null +++ b/registry/native/c/include/sched.h @@ -0,0 +1,10 @@ +#ifndef REGISTRY_NATIVE_C_INCLUDE_SCHED_H +#define REGISTRY_NATIVE_C_INCLUDE_SCHED_H + +#include_next + +#ifdef _GNU_SOURCE +int sched_getcpu(void); +#endif + +#endif diff --git a/registry/native/c/include/sys/ioctl.h b/registry/native/c/include/sys/ioctl.h new file mode 100644 index 0000000000..391b1f589d --- /dev/null +++ b/registry/native/c/include/sys/ioctl.h @@ -0,0 +1,20 @@ +#ifndef REGISTRY_NATIVE_C_INCLUDE_SYS_IOCTL_H +#define REGISTRY_NATIVE_C_INCLUDE_SYS_IOCTL_H + +#include_next + +#ifndef __DEFINED_struct_winsize +struct winsize { + unsigned short ws_row; + unsigned short ws_col; + unsigned short ws_xpixel; + unsigned short ws_ypixel; +}; +#define __DEFINED_struct_winsize +#endif + +#ifndef TIOCGWINSZ +#define TIOCGWINSZ 0x5413 +#endif + +#endif diff --git a/registry/native/c/patches/duckdb/0001-skip-http-object-when-extension-loading-is-disabled.patch b/registry/native/c/patches/duckdb/0001-skip-http-object-when-extension-loading-is-disabled.patch new file mode 100644 index 0000000000..ee844d1e05 --- /dev/null +++ b/registry/native/c/patches/duckdb/0001-skip-http-object-when-extension-loading-is-disabled.patch @@ -0,0 +1,14 @@ +diff --git a/src/main/http/CMakeLists.txt b/src/main/http/CMakeLists.txt +index 2dccc81..a58bd58 100644 +--- a/src/main/http/CMakeLists.txt ++++ b/src/main/http/CMakeLists.txt +@@ -1,3 +1,4 @@ ++if(NOT DISABLE_EXTENSION_LOAD) + include_directories(../../../third_party/httplib) + + # work-around for httplib +@@ -10,3 +11,4 @@ add_library_unity(duckdb_common_http OBJECT http_util.cpp) + set(ALL_OBJECT_FILES + ${ALL_OBJECT_FILES} $ + PARENT_SCOPE) ++endif() diff --git a/registry/native/c/patches/duckdb/0002-disable-linenoise-on-wasi.patch b/registry/native/c/patches/duckdb/0002-disable-linenoise-on-wasi.patch new file mode 100644 index 0000000000..25eb94ab26 --- /dev/null +++ b/registry/native/c/patches/duckdb/0002-disable-linenoise-on-wasi.patch @@ -0,0 +1,18 @@ +diff --git a/tools/shell/CMakeLists.txt b/tools/shell/CMakeLists.txt +index 2cc7e13..37853f9 100644 +--- a/tools/shell/CMakeLists.txt ++++ b/tools/shell/CMakeLists.txt +@@ -1,8 +1,10 @@ + include_directories(include) +-add_subdirectory(linenoise) +-add_definitions(-DHAVE_LINENOISE=1) ++if(NOT CMAKE_SYSTEM_NAME STREQUAL "WASI") ++ add_subdirectory(linenoise) ++ add_definitions(-DHAVE_LINENOISE=1) ++ include_directories(linenoise/include) ++endif() + include_directories(../../third_party/utf8proc/include) +-include_directories(linenoise/include) + add_definitions(-DSQLITE_SHELL_IS_UTF8) + add_definitions(-DUSE_DUCKDB_SHELL_WRAPPER) + diff --git a/registry/native/c/patches/duckdb/0003-disable-shell-system-command-on-wasi.patch b/registry/native/c/patches/duckdb/0003-disable-shell-system-command-on-wasi.patch new file mode 100644 index 0000000000..d1cef08688 --- /dev/null +++ b/registry/native/c/patches/duckdb/0003-disable-shell-system-command-on-wasi.patch @@ -0,0 +1,24 @@ +diff --git a/tools/shell/shell_metadata_command.cpp b/tools/shell/shell_metadata_command.cpp +index 09d81cf..53c5357 100644 +--- a/tools/shell/shell_metadata_command.cpp ++++ b/tools/shell/shell_metadata_command.cpp +@@ -406,6 +406,11 @@ MetadataResult DisplaySchemas(ShellState &state, const vector &args) { + } + + MetadataResult RunShellCommand(ShellState &state, const vector &args) { ++#ifdef SQLITE_NOHAVE_SYSTEM ++ (void)args; ++ state.Print(PrintOutput::STDERR, ".sh/.system is not supported in this WASI DuckDB build\n"); ++ return MetadataResult::FAIL; ++#else + if (state.safe_mode) { + state.Print(PrintOutput::STDERR, ".sh/.system cannot be used in -safe mode\n"); + return MetadataResult::FAIL; +@@ -422,6 +427,7 @@ MetadataResult RunShellCommand(ShellState &state, const vector &args) { + state.PrintF(PrintOutput::STDERR, "System command returns %d\n", x); + } + return MetadataResult::SUCCESS; ++#endif + } + + MetadataResult ShowConfiguration(ShellState &state, const vector &args) { diff --git a/registry/native/c/patches/duckdb/0004-skip-http-util-init-when-extension-loading-is-disabled.patch b/registry/native/c/patches/duckdb/0004-skip-http-util-init-when-extension-loading-is-disabled.patch new file mode 100644 index 0000000000..5e20ff3a85 --- /dev/null +++ b/registry/native/c/patches/duckdb/0004-skip-http-util-init-when-extension-loading-is-disabled.patch @@ -0,0 +1,15 @@ +diff --git a/src/main/database.cpp b/src/main/database.cpp +index a2aab0d..7c6ea97 100644 +--- a/src/main/database.cpp ++++ b/src/main/database.cpp +@@ -54,7 +54,9 @@ DBConfig::DBConfig() { + collation_bindings = make_uniq(); + index_types = make_uniq(); + error_manager = make_uniq(); + secret_manager = make_uniq(); ++#ifndef DUCKDB_DISABLE_EXTENSION_LOAD + http_util = make_shared_ptr(); ++#endif + callback_manager = make_uniq(); + callback_manager->Register("__open_file__", OpenFileStorageExtension::Create()); + } diff --git a/registry/native/c/patches/duckdb/0005-disable-shell-pager-popen-on-wasi.patch b/registry/native/c/patches/duckdb/0005-disable-shell-pager-popen-on-wasi.patch new file mode 100644 index 0000000000..d50a6cbd74 --- /dev/null +++ b/registry/native/c/patches/duckdb/0005-disable-shell-pager-popen-on-wasi.patch @@ -0,0 +1,13 @@ +diff --git a/tools/shell/shell.cpp b/tools/shell/shell.cpp +--- a/tools/shell/shell.cpp ++++ b/tools/shell/shell.cpp +@@ -1416,6 +1416,10 @@ unique_ptr ShellState::SetupPager() { + SetConsoleCP(CP_UTF8); + } + #endif ++#ifdef SQLITE_OMIT_POPEN ++ Print(PrintOutput::STDERR, "Pager pipes are not supported in this WASI DuckDB build.\n"); ++ return nullptr; ++#endif + StartPagerDisplay(); + auto pager_out = popen(pager_command.c_str(), "w"); diff --git a/registry/native/c/patches/llvm-project/0001-libunwind-wasm-exception-support.patch b/registry/native/c/patches/llvm-project/0001-libunwind-wasm-exception-support.patch new file mode 100644 index 0000000000..767f47a846 --- /dev/null +++ b/registry/native/c/patches/llvm-project/0001-libunwind-wasm-exception-support.patch @@ -0,0 +1,196 @@ +diff --git a/libunwind/include/libunwind.h b/libunwind/include/libunwind.h +index b2dae8f..63e147a 100644 +--- a/libunwind/include/libunwind.h ++++ b/libunwind/include/libunwind.h +@@ -15,6 +15,7 @@ + + #include <__libunwind_config.h> + ++#ifndef __wasm__ + #include + #include + +@@ -1299,5 +1300,6 @@ enum { + UNW_LOONGARCH_F30 = 62, + UNW_LOONGARCH_F31 = 63, + }; ++#endif + + #endif +diff --git a/libunwind/src/Unwind-wasm.c b/libunwind/src/Unwind-wasm.c +index b18b32c..d2f61a3 100644 +--- a/libunwind/src/Unwind-wasm.c ++++ b/libunwind/src/Unwind-wasm.c +@@ -11,14 +11,10 @@ + //===----------------------------------------------------------------------===// + + #include +- + #include "config.h" +- +-#ifdef __WASM_EXCEPTIONS__ +- + #include "unwind.h" +-#include + ++#ifdef __wasm__ + _Unwind_Reason_Code __gxx_personality_wasm0(int version, _Unwind_Action actions, + uint64_t exceptionClass, + _Unwind_Exception *unwind_exception, +@@ -35,7 +31,7 @@ struct _Unwind_LandingPadContext { + + // Communication channel between compiler-generated user code and personality + // function +-thread_local struct _Unwind_LandingPadContext __wasm_lpad_context; ++_Thread_local struct _Unwind_LandingPadContext __wasm_lpad_context; + + /// Calls to this function is in landing pads in compiler-generated user code. + /// In other EH schemes, stack unwinding is done by libunwind library, which +@@ -120,4 +116,4 @@ _Unwind_GetRegionStart(struct _Unwind_Context *context) { + return 0; + } + +-#endif // defined(__WASM_EXCEPTIONS__) ++#endif +diff --git a/libunwind/src/UnwindRegistersRestore.S b/libunwind/src/UnwindRegistersRestore.S +index 9d34c79..91b3691 100644 +--- a/libunwind/src/UnwindRegistersRestore.S ++++ b/libunwind/src/UnwindRegistersRestore.S +@@ -6,6 +6,7 @@ + // + //===----------------------------------------------------------------------===// + ++#if !defined(__wasm__) + #include "assembly.h" + + #define FROM_0_TO_15 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 +@@ -1249,4 +1250,4 @@ DEFINE_LIBUNWIND_FUNCTION(_ZN9libunwind19Registers_loongarch6jumptoEv) + #endif /* !defined(__USING_SJLJ_EXCEPTIONS__) && !defined(__wasm__) */ + + NO_EXEC_STACK_DIRECTIVE +- ++#endif +diff --git a/libunwind/src/UnwindRegistersSave.S b/libunwind/src/UnwindRegistersSave.S +index 5bf6055..4397295 100644 +--- a/libunwind/src/UnwindRegistersSave.S ++++ b/libunwind/src/UnwindRegistersSave.S +@@ -6,6 +6,7 @@ + // + //===----------------------------------------------------------------------===// + ++#ifndef __wasm__ + #include "assembly.h" + + #define FROM_0_TO_15 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 +@@ -1180,3 +1181,5 @@ DEFINE_LIBUNWIND_FUNCTION(__unw_getcontext) + #endif /* !defined(__USING_SJLJ_EXCEPTIONS__) && !defined(__wasm__) */ + + NO_EXEC_STACK_DIRECTIVE ++ ++#endif +diff --git a/libunwind/src/assembly.h b/libunwind/src/assembly.h +index f8e83e1..21b20a8 100644 +--- a/libunwind/src/assembly.h ++++ b/libunwind/src/assembly.h +@@ -15,6 +15,7 @@ + #ifndef UNWIND_ASSEMBLY_H + #define UNWIND_ASSEMBLY_H + ++#ifndef __wasm__ + #if defined(__linux__) && defined(__CET__) + #include + #define _LIBUNWIND_CET_ENDBR _CET_ENDBR +@@ -299,5 +300,6 @@ aliasname: \ + #if defined(__powerpc__) + #define PPC_LEFT_SHIFT(index) << (index) + #endif ++#endif + + #endif /* UNWIND_ASSEMBLY_H */ +diff --git a/libunwind/src/cet_unwind.h b/libunwind/src/cet_unwind.h +index 47d7616..dd9bfec 100644 +--- a/libunwind/src/cet_unwind.h ++++ b/libunwind/src/cet_unwind.h +@@ -10,6 +10,7 @@ + #ifndef LIBUNWIND_CET_UNWIND_H + #define LIBUNWIND_CET_UNWIND_H + ++#ifndef __wasm__ + #include "libunwind.h" + + // Currently, CET is implemented on Linux x86 platforms. +@@ -61,3 +62,4 @@ extern void *__libunwind_cet_get_registers(unw_cursor_t *); + extern void *__libunwind_cet_get_jump_target(void); + + #endif ++#endif +diff --git a/libunwind/src/config.h b/libunwind/src/config.h +index deb5a4d..2a13969 100644 +--- a/libunwind/src/config.h ++++ b/libunwind/src/config.h +@@ -66,13 +66,14 @@ + #define _LIBUNWIND_EXPORT + #define _LIBUNWIND_HIDDEN + #else +- #if !defined(__ELF__) && !defined(__MACH__) && !defined(_AIX) +- #define _LIBUNWIND_EXPORT __declspec(dllexport) +- #define _LIBUNWIND_HIDDEN +- #else +- #define _LIBUNWIND_EXPORT __attribute__((visibility("default"))) +- #define _LIBUNWIND_HIDDEN __attribute__((visibility("hidden"))) +- #endif ++#if !defined(__ELF__) && !defined(__MACH__) && !defined(_AIX) && \ ++ !defined(__wasm__) ++#define _LIBUNWIND_EXPORT __declspec(dllexport) ++#define _LIBUNWIND_HIDDEN ++#else ++#define _LIBUNWIND_EXPORT __attribute__((visibility("default"))) ++#define _LIBUNWIND_HIDDEN __attribute__((visibility("hidden"))) ++#endif + #endif + + #define STR(a) #a +diff --git a/libunwind/src/libunwind.cpp b/libunwind/src/libunwind.cpp +index cf39ec5..efade46 100644 +--- a/libunwind/src/libunwind.cpp ++++ b/libunwind/src/libunwind.cpp +@@ -12,6 +12,7 @@ + #include + + #include "config.h" ++#ifndef __wasm__ + #include "libunwind_ext.h" + + #include +@@ -430,6 +431,7 @@ int __unw_remove_find_dynamic_unwind_sections( + } + + #endif // __APPLE__ ++#endif + + // Add logging hooks in Debug builds only + #ifndef NDEBUG +@@ -472,4 +474,3 @@ bool logDWARF() { + } + + #endif // NDEBUG +- +diff --git a/libunwind/src/libunwind_ext.h b/libunwind/src/libunwind_ext.h +index 28db43a..3a7ca7c 100644 +--- a/libunwind/src/libunwind_ext.h ++++ b/libunwind/src/libunwind_ext.h +@@ -12,6 +12,7 @@ + #ifndef __LIBUNWIND_EXT__ + #define __LIBUNWIND_EXT__ + ++#ifndef __wasm__ + #include "config.h" + #include + #include +@@ -133,5 +134,6 @@ extern _Unwind_Reason_Code _Unwind_VRS_Interpret(_Unwind_Context *context, + #ifdef __cplusplus + } + #endif ++#endif + + #endif // __LIBUNWIND_EXT__ diff --git a/registry/native/c/programs/http_get.c b/registry/native/c/programs/http_get.c index a435ccb00c..0e7dc466ea 100644 --- a/registry/native/c/programs/http_get.c +++ b/registry/native/c/programs/http_get.c @@ -9,11 +9,13 @@ int main(int argc, char *argv[]) { if (argc < 2) { - fprintf(stderr, "usage: http_get \n"); + fprintf(stderr, "usage: http_get [path] [output_file]\n"); return 1; } int port = atoi(argv[1]); + const char *path = argc >= 3 ? argv[2] : "/"; + const char *output_file = argc >= 4 ? argv[3] : NULL; int fd = socket(AF_INET, SOCK_STREAM, 0); if (fd < 0) { @@ -33,8 +35,20 @@ int main(int argc, char *argv[]) { return 1; } - const char *request = "GET / HTTP/1.0\r\nHost: localhost\r\n\r\n"; - ssize_t sent = send(fd, request, strlen(request), 0); + char request[1024]; + int request_len = snprintf( + request, + sizeof(request), + "GET %s HTTP/1.0\r\nHost: localhost\r\n\r\n", + path + ); + if (request_len < 0 || (size_t)request_len >= sizeof(request)) { + fprintf(stderr, "request path too long\n"); + close(fd); + return 1; + } + + ssize_t sent = send(fd, request, (size_t)request_len, 0); if (sent < 0) { perror("send"); close(fd); @@ -56,7 +70,22 @@ int main(int argc, char *argv[]) { const char *body = strstr(response, "\r\n\r\n"); if (body) { body += 4; - printf("body: %s\n", body); + if (output_file) { + FILE *out = fopen(output_file, "wb"); + if (!out) { + perror("fopen"); + return 1; + } + size_t body_len = total - (size_t)(body - response); + if (fwrite(body, 1, body_len, out) != body_len) { + perror("fwrite"); + fclose(out); + return 1; + } + fclose(out); + } else { + printf("body: %s\n", body); + } } else { printf("body: (no separator found)\n"); return 1; diff --git a/registry/native/c/scripts/build-duckdb.sh b/registry/native/c/scripts/build-duckdb.sh new file mode 100644 index 0000000000..145822b433 --- /dev/null +++ b/registry/native/c/scripts/build-duckdb.sh @@ -0,0 +1,76 @@ +#!/bin/bash +set -euo pipefail + +# Reference only: +# - https://github.com/duckdb/duckdb-wasm#readme +# - https://github.com/duckdb/duckdb-wasm/blob/main/Makefile +# - https://github.com/duckdb/duckdb-wasm/blob/main/extension_config_wasm.cmake +# +# Unlike duckdb-wasm, we do not use their prebuilt WebAssembly bundles or +# Emscripten runtime shims. This script builds upstream DuckDB directly against +# our patched WASI/POSIX sysroot so file and network operations flow through the +# existing registry host bindings. + +: "${DUCKDB_SRC_DIR:?DUCKDB_SRC_DIR is required}" +: "${DUCKDB_BUILD_DIR:?DUCKDB_BUILD_DIR is required}" +: "${DUCKDB_OUTPUT:?DUCKDB_OUTPUT is required}" +: "${WASI_SDK_DIR:?WASI_SDK_DIR is required}" +: "${SYSROOT_DIR:?SYSROOT_DIR is required}" +: "${MODULE_PATH:?MODULE_PATH is required}" +: "${OVERLAY_INCLUDE_DIR:?OVERLAY_INCLUDE_DIR is required}" +: "${DUCKDB_GIT_DESCRIBE:?DUCKDB_GIT_DESCRIBE is required}" + +TOOLCHAIN_FILE="$WASI_SDK_DIR/share/cmake/wasi-sdk.cmake" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PATCH_DIR="$SCRIPT_DIR/../patches/duckdb" +COMMON_FLAGS="-I$OVERLAY_INCLUDE_DIR -D_WASI_EMULATED_PTHREAD -D_WASI_EMULATED_MMAN -D_WASI_EMULATED_SIGNAL -D_WASI_EMULATED_PROCESS_CLOCKS" +COMMON_CXX_FLAGS="$COMMON_FLAGS -DDUCKDB_DISABLE_EXTENSION_LOAD -DSQLITE_NOHAVE_SYSTEM -DSQLITE_OMIT_POPEN -fwasm-exceptions -DWEBDB_FAST_EXCEPTIONS=1" +CXX_STDLIB_INCLUDE="$SYSROOT_DIR/include/wasm32-wasi/c++/v1" + +if [ ! -d "$CXX_STDLIB_INCLUDE" ]; then + echo "missing libc++ headers at $CXX_STDLIB_INCLUDE" >&2 + exit 1 +fi + +if [ -d "$PATCH_DIR" ]; then + while IFS= read -r patch_file; do + if patch --dry-run -p1 -d "$DUCKDB_SRC_DIR" < "$patch_file" >/dev/null 2>&1; then + patch --no-backup-if-mismatch -p1 -d "$DUCKDB_SRC_DIR" < "$patch_file" >/dev/null + elif patch --dry-run -R -p1 -d "$DUCKDB_SRC_DIR" < "$patch_file" >/dev/null 2>&1; then + : + else + echo "failed to apply DuckDB patch: $patch_file" >&2 + exit 1 + fi + done < <(find "$PATCH_DIR" -name '*.patch' -type f | sort) +fi + +mkdir -p "$DUCKDB_BUILD_DIR" + +cmake \ + -S "$DUCKDB_SRC_DIR" \ + -B "$DUCKDB_BUILD_DIR" \ + -G "Unix Makefiles" \ + -DCMAKE_TOOLCHAIN_FILE="$TOOLCHAIN_FILE" \ + -DWASI_SDK_PREFIX="$WASI_SDK_DIR" \ + -DCMAKE_SYSROOT="$SYSROOT_DIR" \ + -DCMAKE_MODULE_PATH="$MODULE_PATH" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_FLAGS="$COMMON_FLAGS" \ + -DCMAKE_CXX_FLAGS="$COMMON_CXX_FLAGS -isystem $CXX_STDLIB_INCLUDE" \ + -DCMAKE_EXE_LINKER_FLAGS="-lwasi-emulated-mman -lwasi-emulated-signal -lwasi-emulated-process-clocks" \ + -DBUILD_UNITTESTS=0 \ + -DENABLE_UNITTEST_CPP_TESTS=0 \ + -DBUILD_BENCHMARKS=0 \ + -DENABLE_SANITIZER=0 \ + -DENABLE_UBSAN=0 \ + -DDISABLE_THREADS=1 \ + -DSMALLER_BINARY=1 \ + -DDISABLE_EXTENSION_LOAD=1 \ + -DBUILD_EXTENSIONS=core_functions \ + -DSKIP_EXTENSIONS="parquet;jemalloc" \ + -DDUCKDB_EXPLICIT_PLATFORM=wasm32-wasip1-posix \ + -DOVERRIDE_GIT_DESCRIBE="$DUCKDB_GIT_DESCRIBE" + +cmake --build "$DUCKDB_BUILD_DIR" --target shell -j"$(nproc 2>/dev/null || echo 4)" +cp "$DUCKDB_BUILD_DIR/duckdb" "$DUCKDB_OUTPUT" diff --git a/registry/native/c/scripts/build-llvm-runtimes.sh b/registry/native/c/scripts/build-llvm-runtimes.sh new file mode 100644 index 0000000000..918b26d55e --- /dev/null +++ b/registry/native/c/scripts/build-llvm-runtimes.sh @@ -0,0 +1,114 @@ +#!/bin/bash +set -euo pipefail + +# Build the libc++/libc++abi/libunwind runtime set ourselves so C++ exception +# handling matches the same patched WASI/POSIX sysroot DuckDB uses. +# +# Reference: +# - https://github.com/llvm/llvm-project/pull/79667 + +: "${LLVM_PROJECT_SRC_DIR:?LLVM_PROJECT_SRC_DIR is required}" +: "${LLVM_RUNTIME_BUILD_DIR:?LLVM_RUNTIME_BUILD_DIR is required}" +: "${LLVM_RUNTIME_INSTALL_DIR:?LLVM_RUNTIME_INSTALL_DIR is required}" +: "${WASI_SDK_DIR:?WASI_SDK_DIR is required}" +: "${SYSROOT_DIR:?SYSROOT_DIR is required}" + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PATCH_DIR="$SCRIPT_DIR/../patches/llvm-project" +WASI_NM="$WASI_SDK_DIR/bin/llvm-nm" + +if [ -d "$PATCH_DIR" ]; then + while IFS= read -r patch_file; do + if patch --dry-run -p1 -d "$LLVM_PROJECT_SRC_DIR" < "$patch_file" >/dev/null 2>&1; then + patch --no-backup-if-mismatch -p1 -d "$LLVM_PROJECT_SRC_DIR" < "$patch_file" >/dev/null + elif patch --dry-run -R -p1 -d "$LLVM_PROJECT_SRC_DIR" < "$patch_file" >/dev/null 2>&1; then + : + else + echo "failed to apply llvm-project patch: $patch_file" >&2 + exit 1 + fi + done < <(find "$PATCH_DIR" -name '*.patch' -type f | sort) +fi + +rm -rf "$LLVM_RUNTIME_BUILD_DIR" "$LLVM_RUNTIME_INSTALL_DIR" + +cmake \ + -S "$LLVM_PROJECT_SRC_DIR/runtimes" \ + -B "$LLVM_RUNTIME_BUILD_DIR" \ + -G "Unix Makefiles" \ + -DUNIX=1 \ + -DCMAKE_TOOLCHAIN_FILE="$WASI_SDK_DIR/share/cmake/wasi-sdk.cmake" \ + -DCMAKE_MODULE_PATH="$SCRIPT_DIR/../cmake" \ + -DWASI_SDK_PREFIX="$WASI_SDK_DIR" \ + -DCMAKE_SYSROOT="$SYSROOT_DIR" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_FLAGS="-fwasm-exceptions -D_WASI_EMULATED_PTHREAD" \ + -DCMAKE_CXX_FLAGS="-fwasm-exceptions -D_WASI_EMULATED_PTHREAD" \ + -DCMAKE_EXE_LINKER_FLAGS="-lwasi-emulated-pthread" \ + -DCMAKE_SHARED_LINKER_FLAGS="-lwasi-emulated-pthread" \ + -DCMAKE_REQUIRED_FLAGS="-D_WASI_EMULATED_PTHREAD" \ + -DCMAKE_REQUIRED_LIBRARIES="wasi-emulated-pthread" \ + -DLLVM_ENABLE_RUNTIMES="libunwind;libcxxabi;libcxx" \ + -DLLVM_INCLUDE_TESTS=OFF \ + -DLLVM_INCLUDE_DOCS=OFF \ + -DLIBUNWIND_INCLUDE_TESTS=OFF \ + -DLIBCXXABI_INCLUDE_TESTS=OFF \ + -DLIBCXX_INCLUDE_TESTS=OFF \ + -DLIBCXX_INCLUDE_BENCHMARKS=OFF \ + -DLIBUNWIND_ENABLE_SHARED=OFF \ + -DLIBCXXABI_ENABLE_SHARED=OFF \ + -DLIBCXX_ENABLE_SHARED=OFF \ + -DLIBUNWIND_ENABLE_THREADS=OFF \ + -DLIBCXXABI_ENABLE_THREADS=ON \ + -DLIBCXX_ENABLE_THREADS=ON \ + -DLIBCXX_USE_COMPILER_RT=ON \ + -DLIBCXXABI_USE_COMPILER_RT=ON \ + -DLIBCXXABI_USE_LLVM_UNWINDER=ON \ + -DLIBCXXABI_ENABLE_STATIC_UNWINDER=ON \ + -DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=ON \ + -DLIBCXX_CXX_ABI=libcxxabi \ + -DLIBCXX_ENABLE_STATIC=ON \ + -DLIBCXXABI_ENABLE_STATIC=ON \ + -DLIBUNWIND_ENABLE_STATIC=ON \ + -DCMAKE_INSTALL_PREFIX="$LLVM_RUNTIME_INSTALL_DIR" + +cmake --build "$LLVM_RUNTIME_BUILD_DIR" --target install -j"$(nproc 2>/dev/null || echo 4)" + +# llvm-nm returns a non-zero status for archive members with no symbols, which +# is expected here for several libunwind objects. Capture output explicitly so +# pipefail does not turn a valid archive into a false negative. +libcxxabi_symbols="$("$WASI_NM" "$LLVM_RUNTIME_INSTALL_DIR/lib/libc++abi.a" 2>/dev/null || true)" + +if ! grep -q ' T __cxa_throw$' <<<"$libcxxabi_symbols"; then + echo "rebuilt libc++abi.a does not export __cxa_throw" >&2 + exit 1 +fi +if ! grep -q ' T __cxa_allocate_exception$' <<<"$libcxxabi_symbols"; then + echo "rebuilt libc++abi.a does not export __cxa_allocate_exception" >&2 + exit 1 +fi + +SYSROOT_LIB="$SYSROOT_DIR/lib/wasm32-wasi" +SYSROOT_INCLUDE="$SYSROOT_DIR/include" +SYSROOT_CXX_INCLUDE="$SYSROOT_DIR/include/wasm32-wasi/c++/v1" +LLVM_CXX_INCLUDE="$LLVM_RUNTIME_INSTALL_DIR/include/c++/v1" + +mkdir -p "$SYSROOT_LIB" "$SYSROOT_INCLUDE" "$SYSROOT_CXX_INCLUDE" + +for header in __libunwind_config.h libunwind.h libunwind.modulemap unwind.h unwind_itanium.h unwind_arm_ehabi.h; do + if [ -f "$LLVM_RUNTIME_INSTALL_DIR/include/$header" ]; then + cp "$LLVM_RUNTIME_INSTALL_DIR/include/$header" "$SYSROOT_INCLUDE/$header" + fi +done + +if [ -d "$LLVM_CXX_INCLUDE" ]; then + rm -rf "$SYSROOT_CXX_INCLUDE" + mkdir -p "$SYSROOT_CXX_INCLUDE" + cp -R "$LLVM_CXX_INCLUDE/." "$SYSROOT_CXX_INCLUDE/" +fi + +for runtime in libunwind.a libc++abi.a libc++.a libc++experimental.a; do + if [ -f "$LLVM_RUNTIME_INSTALL_DIR/lib/$runtime" ]; then + cp "$LLVM_RUNTIME_INSTALL_DIR/lib/$runtime" "$SYSROOT_LIB/$runtime" + fi +done diff --git a/registry/native/patches/wasi-libc-overrides/fcntl.c b/registry/native/patches/wasi-libc-overrides/fcntl.c index f34388e204..ad938ca653 100644 --- a/registry/native/patches/wasi-libc-overrides/fcntl.c +++ b/registry/native/patches/wasi-libc-overrides/fcntl.c @@ -143,6 +143,27 @@ int fcntl(int fd, int cmd, ...) { break; } + case F_GETLK: { + struct flock *lock = va_arg(ap, struct flock *); + if (!lock) { + errno = EINVAL; + result = -1; + } else { + lock->l_type = F_UNLCK; + lock->l_pid = 0; + result = 0; + } + break; + } + + case F_SETLK: + case F_SETLKW: + // WASI has no kernel-level advisory locking. Treat locks as a + // successful no-op so single-process workloads like DuckDB can open + // writable database files on the VFS-backed filesystem. + result = 0; + break; + default: errno = EINVAL; result = -1; diff --git a/registry/native/patches/wasi-libc-overrides/ifaddrs.c b/registry/native/patches/wasi-libc-overrides/ifaddrs.c new file mode 100644 index 0000000000..2f1abe78b5 --- /dev/null +++ b/registry/native/patches/wasi-libc-overrides/ifaddrs.c @@ -0,0 +1,25 @@ +/** + * Minimal getifaddrs/freeifaddrs shim for runtimes without host network + * interface enumeration. + * + * cpp-httplib includes ifaddrs support on POSIX targets, but DuckDB's embedded + * HTTP client only uses it when callers opt into binding a request to a named + * interface. Our WASI runtime does not expose interface enumeration today, so + * return an empty list instead of failing the build or forcing DuckDB-specific + * source patches. + */ + +#include +#include + +int getifaddrs(struct ifaddrs **ifap) { + if (ifap) { + *ifap = 0; + } + errno = ENOSYS; + return -1; +} + +void freeifaddrs(struct ifaddrs *ifa) { + (void)ifa; +} diff --git a/registry/native/patches/wasi-libc-overrides/mlock.c b/registry/native/patches/wasi-libc-overrides/mlock.c new file mode 100644 index 0000000000..a9a445713c --- /dev/null +++ b/registry/native/patches/wasi-libc-overrides/mlock.c @@ -0,0 +1,40 @@ +/** + * Minimal mlock/munlock shim for WASI/POSIX builds without page pinning. + * + * DuckDB uses mlock/munlock as a best-effort hardening step for encryption + * keys. Our runtime does not currently expose memory locking primitives, so + * treat these calls as successful no-ops instead of failing the link. + * + * Installed into the patched sysroot so upstream code can keep its existing + * POSIX calls without carrying a WASI-specific source patch. + */ + +#include + +int mlock(const void *addr, size_t len) { + (void)addr; + (void)len; + return 0; +} + +int munlock(const void *addr, size_t len) { + (void)addr; + (void)len; + return 0; +} + +int mlockall(int flags) { + (void)flags; + return 0; +} + +int munlockall(void) { + return 0; +} + +int madvise(void *addr, size_t len, int advice) { + (void)addr; + (void)len; + (void)advice; + return 0; +} diff --git a/registry/native/patches/wasi-libc-overrides/sched.c b/registry/native/patches/wasi-libc-overrides/sched.c new file mode 100644 index 0000000000..b87e4dacb3 --- /dev/null +++ b/registry/native/patches/wasi-libc-overrides/sched.c @@ -0,0 +1,7 @@ +#include +#include + +int sched_getcpu(void) { + errno = ENOSYS; + return -1; +} diff --git a/registry/native/patches/wasi-libc/0008-sockets.patch b/registry/native/patches/wasi-libc/0008-sockets.patch index 6b9bb8e3e3..aced37f357 100644 --- a/registry/native/patches/wasi-libc/0008-sockets.patch +++ b/registry/native/patches/wasi-libc/0008-sockets.patch @@ -1,5 +1,72 @@ -diff --git a/libc-bottom-half/headers/public/__struct_sockaddr_un.h b/libc-bottom-half/headers/public/__struct_sockaddr_un.h -index 6371194..60634cf 100644 +Implement socket(), connect(), bind(), listen(), accept(), send(), +recv(), sendto(), recvfrom(), getaddrinfo(), freeaddrinfo(), +gai_strerror(), gethostname(), setsockopt(), poll(), and select() +via host_net WASM imports. + +Replaces the wasi-libc stubs (which return -ENOSYS or are #ifdef'd out) +with implementations that call our host_net.net_socket, net_connect, +net_bind, net_listen, net_accept, net_send, net_recv, net_sendto, +net_recvfrom, net_getaddrinfo, net_close, net_setsockopt, and net_poll +WASM imports. + +Un-omits netdb.h from the sysroot headers so C programs can use +getaddrinfo/freeaddrinfo/gai_strerror. Un-gates bind() and listen() +declarations from the wasip2-only guard. + +Supports AF_INET, AF_INET6, and AF_UNIX address families in +sockaddr serialization (sockaddr_to_string / string_to_sockaddr). + +Import signatures match wasmvm/crates/wasi-ext/src/lib.rs exactly. + +--- a/scripts/install-include-headers.sh 2026-03-20 04:05:48.609869966 -0700 ++++ b/scripts/install-include-headers.sh 2026-03-20 04:05:57.781880813 -0700 +@@ -69,10 +69,11 @@ + "net/ethernet.h" "net/route.h" "netinet/if_ether.h" "netinet/ether.h" \ + "sys/timerfd.h" "libintl.h" "sys/sysmacros.h" "aio.h") + # Exclude `netdb.h` from all of the p1 targets. +-if [[ $TARGET_TRIPLE == *"wasi" || $TARGET_TRIPLE == *"wasi-threads" || \ +- $TARGET_TRIPLE == *"wasip1" || $TARGET_TRIPLE == *"wasip1-threads" ]]; then +- MUSL_OMIT_HEADERS+=("netdb.h") +-fi ++# NOTE: commented out by secureexec 0008-sockets patch — we provide getaddrinfo via host_net ++#if [[ $TARGET_TRIPLE == *"wasi" || $TARGET_TRIPLE == *"wasi-threads" || \ ++# $TARGET_TRIPLE == *"wasip1" || $TARGET_TRIPLE == *"wasip1-threads" ]]; then ++# MUSL_OMIT_HEADERS+=("netdb.h") ++#fi + + # Remove all the `MUSL_OMIT_HEADERS` previously copied over. + for OMIT_HEADER in "${MUSL_OMIT_HEADERS[@]}"; do + +--- a/libc-top-half/musl/include/sys/socket.h 2026-03-20 04:05:48.609869966 -0700 ++++ b/libc-top-half/musl/include/sys/socket.h 2026-03-20 04:06:04.989889334 -0700 +@@ -401,3 +401 @@ +-#if (defined __wasilibc_unmodified_upstream) || (defined __wasilibc_use_wasip2) + int socket (int, int, int); +-#endif +@@ -411,5 +409,3 @@ +-#if (defined __wasilibc_unmodified_upstream) || (defined __wasilibc_use_wasip2) + int connect (int, const struct sockaddr *, socklen_t); + int bind (int, const struct sockaddr *, socklen_t); + int listen (int, int); +-#endif +@@ -420,4 +416,2 @@ +-#if (defined __wasilibc_unmodified_upstream) || (defined __wasilibc_use_wasip2) + int getsockname (int, struct sockaddr *__restrict, socklen_t *__restrict); + int getpeername (int, struct sockaddr *__restrict, socklen_t *__restrict); +-#endif +@@ -427,4 +421,2 @@ +-#if (defined __wasilibc_unmodified_upstream) || (defined __wasilibc_use_wasip2) + ssize_t sendto (int, const void *, size_t, int, const struct sockaddr *, socklen_t); + ssize_t recvfrom (int, void *__restrict, size_t, int, struct sockaddr *__restrict, socklen_t *__restrict); +-#endif +@@ -437,3 +429 @@ +-#if (defined __wasilibc_unmodified_upstream) || (defined __wasilibc_use_wasip2) + int setsockopt (int, int, int, const void *, socklen_t); +-#endif + + #ifdef __wasilibc_unmodified_upstream /* WASI has no sockatmark */ + int sockatmark (int); + --- a/libc-bottom-half/headers/public/__struct_sockaddr_un.h +++ b/libc-bottom-half/headers/public/__struct_sockaddr_un.h @@ -5,6 +5,7 @@ @@ -15,7 +82,7 @@ new file mode 100644 index 0000000..975e62a --- /dev/null +++ b/libc-bottom-half/sources/host_socket.c -@@ -0,0 +1,696 @@ +@@ -0,0 +1,779 @@ +// Socket API via wasmVM host_net imports. +// +// Replaces wasi-libc's ENOSYS stubs with calls to our custom WASM imports: diff --git a/registry/native/patches/wasi-libc/0012-posix-spawn-cwd.patch b/registry/native/patches/wasi-libc/0012-posix-spawn-cwd.patch index 12d6d6b575..c8e8ccc9e3 100644 --- a/registry/native/patches/wasi-libc/0012-posix-spawn-cwd.patch +++ b/registry/native/patches/wasi-libc/0012-posix-spawn-cwd.patch @@ -20,7 +20,7 @@ to spawned commands. @@ -101,6 +101,16 @@ static int __addfdop(posix_spawn_file_actions_t *fa, struct __fdop *op) { return 0; } - + +static const char *find_pwd_in_env(char *const envp[]) { + if (!envp) return NULL; + for (int i = 0; envp[i]; i++) { diff --git a/registry/native/scripts/patch-wasi-libc.sh b/registry/native/scripts/patch-wasi-libc.sh index f6119a136a..deda7709c1 100755 --- a/registry/native/scripts/patch-wasi-libc.sh +++ b/registry/native/scripts/patch-wasi-libc.sh @@ -21,10 +21,13 @@ PATCHES_DIR="$WASMCORE_DIR/patches/wasi-libc" # wasi-libc commit pinned by wasi-sdk-25's git submodule WASI_LIBC_COMMIT="574b88da481569b65a237cb80daf9a2d5aeaf82d" WASI_LIBC_REPO="https://github.com/WebAssembly/wasi-libc.git" +LLVM_PROJECT_TAG="llvmorg-19.1.5" +LLVM_PROJECT_URL="https://github.com/llvm/llvm-project/archive/refs/tags/${LLVM_PROJECT_TAG}.tar.gz" # Directories VENDOR_DIR="$WASMCORE_DIR/c/vendor" WASI_LIBC_DIR="$VENDOR_DIR/wasi-libc" +LLVM_PROJECT_DIR="$VENDOR_DIR/llvm-project" WASI_SDK_DIR="$VENDOR_DIR/wasi-sdk" SYSROOT_DIR="$WASMCORE_DIR/c/sysroot" WASI_LIBC_SRC_DIR="$WASI_LIBC_DIR" @@ -80,6 +83,31 @@ else fi fi +# Fetch llvm-project sources used to rebuild the exception-capable C++ runtime. +if [ ! -d "$LLVM_PROJECT_DIR/runtimes" ]; then + if [ "$MODE" = "check" ]; then + echo "ERROR: llvm-project not vendored at $LLVM_PROJECT_DIR" + echo "Run '$0' (without --check) to fetch the runtime sources." + exit 1 + fi + + echo "=== Fetching llvm-project at $LLVM_PROJECT_TAG ===" + mkdir -p "$VENDOR_DIR" + LLVM_TARBALL="$VENDOR_DIR/${LLVM_PROJECT_TAG}.tar.gz" + if command -v curl >/dev/null 2>&1; then + curl -fSL "$LLVM_PROJECT_URL" -o "$LLVM_TARBALL" + elif command -v wget >/dev/null 2>&1; then + wget -q "$LLVM_PROJECT_URL" -O "$LLVM_TARBALL" + else + echo "ERROR: neither curl nor wget found" + exit 1 + fi + rm -rf "$LLVM_PROJECT_DIR" + mkdir -p "$LLVM_PROJECT_DIR" + tar -xzf "$LLVM_TARBALL" --strip-components=1 -C "$LLVM_PROJECT_DIR" + echo "" +fi + cleanup() { if [ -n "$WORKTREE_DIR" ] && [ -d "$WORKTREE_DIR" ]; then git -C "$WASI_LIBC_DIR" worktree remove --force "$WORKTREE_DIR" >/dev/null 2>&1 || true @@ -217,6 +245,45 @@ for crt in "$VANILLA_LIB"/crt*.o; do [ -f "$crt" ] && cp "$crt" "$SYSROOT_LIB/" done +# Install the wasi-sdk libc++ runtime into the patched sysroot so upstream C++ +# projects can target the same sysroot we use for libc. We overlay the +# thread-capable headers/libs from wasm32-wasi-threads because libc++'s mutex +# support expects those definitions even when we satisfy pthread calls through +# wasi-emulated-pthread. +VANILLA_INCLUDE="$WASI_SDK_DIR/share/wasi-sysroot/include/wasm32-wasi" +THREADS_INCLUDE="$WASI_SDK_DIR/share/wasi-sysroot/include/wasm32-wasi-threads" +SYSROOT_INCLUDE="$SYSROOT_DIR/include/wasm32-wasi" +mkdir -p "$SYSROOT_INCLUDE/c++/v1" +if [ -d "$VANILLA_INCLUDE/c++/v1" ]; then + cp -R "$VANILLA_INCLUDE/c++/v1/." "$SYSROOT_INCLUDE/c++/v1/" +fi +if [ -d "$THREADS_INCLUDE/c++/v1" ]; then + cp -R "$THREADS_INCLUDE/c++/v1/." "$SYSROOT_INCLUDE/c++/v1/" +fi + +for runtime in libc++.a libc++.modules.json libc++.so libc++abi.a libc++abi.so libc++experimental.a; do + [ -f "$VANILLA_LIB/$runtime" ] && cp "$VANILLA_LIB/$runtime" "$SYSROOT_LIB/" +done +THREADS_LIB="$WASI_SDK_DIR/share/wasi-sysroot/lib/wasm32-wasi-threads" +for runtime in libc++.a libc++abi.a libc++experimental.a; do + [ -f "$THREADS_LIB/$runtime" ] && cp "$THREADS_LIB/$runtime" "$SYSROOT_LIB/" +done + +# Rebuild the C++ runtime with Wasm EH enabled so upstream C++ projects can use +# exceptions against the same patched WASI/POSIX sysroot. We also replace the +# libc++ headers with the rebuilt install so the header ABI namespace matches +# the custom libc++/libc++abi archives we overlay into the sysroot. +LLVM_RUNTIME_BUILD_SCRIPT="$WASMCORE_DIR/c/scripts/build-llvm-runtimes.sh" +LLVM_RUNTIME_BUILD_DIR="$WASMCORE_DIR/c/build/llvm-runtimes" +LLVM_RUNTIME_INSTALL_DIR="$WASMCORE_DIR/c/build/llvm-runtimes-install" +echo "Rebuilding libc++/libc++abi/libunwind with -fwasm-exceptions..." +LLVM_PROJECT_SRC_DIR="$LLVM_PROJECT_DIR" \ +LLVM_RUNTIME_BUILD_DIR="$LLVM_RUNTIME_BUILD_DIR" \ +LLVM_RUNTIME_INSTALL_DIR="$LLVM_RUNTIME_INSTALL_DIR" \ +WASI_SDK_DIR="$WASI_SDK_DIR" \ +SYSROOT_DIR="$SYSROOT_DIR" \ +bash "$LLVM_RUNTIME_BUILD_SCRIPT" + # Create empty dummy libraries (libm, librt, libpthread, etc.) for lib in m rt pthread crypt util xnet resolv; do "$WASI_AR" crs "$SYSROOT_LIB/lib${lib}.a" 2>/dev/null || true @@ -254,14 +321,16 @@ done # === Install sysroot overrides === # Override files in patches/wasi-libc-overrides/ fix broken libc behavior -# (fcntl, strfmon, open_wmemstream, swprintf, inet_ntop, pthread_attr, pthread_mutex, pthread_key, fmtmsg). +# (fcntl, sched_getcpu, strfmon, open_wmemstream, swprintf, inet_ntop, +# pthread_attr, pthread_mutex, pthread_key, fmtmsg). # The patched sysroot also provides host_sigaction.o, which must replace musl's # original sigaction.o / signal.o so cooperative signal registration flows # through the host_process import instead of the upstream rt_sigaction stub. # realloc is handled by 0009-realloc-glibc-semantics.patch directly. # Overrides are compiled and added to libc.a so ALL WASM programs get the fixes. OVERRIDES_DIR="$WASMCORE_DIR/patches/wasi-libc-overrides" -OVERRIDE_CFLAGS="--target=wasm32-wasip1 --sysroot=$SYSROOT_DIR -O2 -D_GNU_SOURCE" +OVERRIDE_INCLUDE_DIR="$WASMCORE_DIR/c/include" +OVERRIDE_CFLAGS="--target=wasm32-wasip1 --sysroot=$SYSROOT_DIR -O2 -D_GNU_SOURCE -I$OVERRIDE_INCLUDE_DIR" # Extra flags for overrides that need musl internal headers (struct __pthread, etc.) MUSL_INTERNAL_DIR="$WASI_LIBC_SRC_DIR/libc-top-half/musl/src/internal" diff --git a/registry/software/duckdb/agent-os-package.json b/registry/software/duckdb/agent-os-package.json new file mode 100644 index 0000000000..40b5ab6d5d --- /dev/null +++ b/registry/software/duckdb/agent-os-package.json @@ -0,0 +1,7 @@ +{ + "name": "@rivet-dev/agent-os-duckdb", + "type": "wasm", + "description": "DuckDB command-line interface", + "aptName": "duckdb", + "source": "c" +} diff --git a/registry/software/duckdb/package.json b/registry/software/duckdb/package.json new file mode 100644 index 0000000000..1ee8527b84 --- /dev/null +++ b/registry/software/duckdb/package.json @@ -0,0 +1,28 @@ +{ + "name": "@rivet-dev/agent-os-duckdb", + "version": "0.0.260331072558", + "type": "module", + "license": "Apache-2.0", + "description": "DuckDB CLI for agentOS", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "wasm" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "tsc", + "check-types": "tsc --noEmit" + }, + "devDependencies": { + "@rivet-dev/agent-os-registry-types": "link:../../../packages/registry-types", + "@types/node": "^22.10.2", + "typescript": "^5.9.2" + } +} diff --git a/registry/software/duckdb/src/index.ts b/registry/software/duckdb/src/index.ts new file mode 100644 index 0000000000..191d58f1c0 --- /dev/null +++ b/registry/software/duckdb/src/index.ts @@ -0,0 +1,18 @@ +import type { WasmCommandPackage } from "@rivet-dev/agent-os-registry-types"; +import { resolve, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +const pkg = { + name: "duckdb", + aptName: "duckdb", + description: "DuckDB command-line interface", + source: "c" as const, + commands: [{ name: "duckdb", permissionTier: "read-write" as const }], + get commandDir() { + return resolve(__dirname, "..", "wasm"); + }, +} satisfies WasmCommandPackage; + +export default pkg; diff --git a/registry/software/duckdb/tsconfig.json b/registry/software/duckdb/tsconfig.json new file mode 100644 index 0000000000..8f24167afd --- /dev/null +++ b/registry/software/duckdb/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/registry/software/http-get/agent-os-package.json b/registry/software/http-get/agent-os-package.json new file mode 100644 index 0000000000..ef46559f31 --- /dev/null +++ b/registry/software/http-get/agent-os-package.json @@ -0,0 +1,7 @@ +{ + "name": "@rivet-dev/agent-os-http-get", + "type": "wasm", + "description": "Minimal HTTP GET fetch helper", + "aptName": "http-get", + "source": "c" +} diff --git a/registry/software/http-get/package.json b/registry/software/http-get/package.json new file mode 100644 index 0000000000..220198c1a3 --- /dev/null +++ b/registry/software/http-get/package.json @@ -0,0 +1,28 @@ +{ + "name": "@rivet-dev/agent-os-http-get", + "version": "0.0.260331072558", + "type": "module", + "license": "Apache-2.0", + "description": "Minimal HTTP GET fetch helper for agentOS", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "wasm" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "tsc", + "check-types": "tsc --noEmit" + }, + "devDependencies": { + "@rivet-dev/agent-os-registry-types": "link:../../../packages/registry-types", + "@types/node": "^22.10.2", + "typescript": "^5.9.2" + } +} diff --git a/registry/software/http-get/src/index.ts b/registry/software/http-get/src/index.ts new file mode 100644 index 0000000000..278232e646 --- /dev/null +++ b/registry/software/http-get/src/index.ts @@ -0,0 +1,18 @@ +import type { WasmCommandPackage } from "@rivet-dev/agent-os-registry-types"; +import { resolve, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +const pkg = { + name: "http-get", + aptName: "http-get", + description: "Minimal HTTP GET fetch helper", + source: "c" as const, + commands: [{ name: "http_get", permissionTier: "full" as const }], + get commandDir() { + return resolve(__dirname, "..", "wasm"); + }, +} satisfies WasmCommandPackage; + +export default pkg; diff --git a/registry/software/http-get/tsconfig.json b/registry/software/http-get/tsconfig.json new file mode 100644 index 0000000000..8f24167afd --- /dev/null +++ b/registry/software/http-get/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/registry/tests/wasmvm/duckdb.test.ts b/registry/tests/wasmvm/duckdb.test.ts new file mode 100644 index 0000000000..e5d1bf2c05 --- /dev/null +++ b/registry/tests/wasmvm/duckdb.test.ts @@ -0,0 +1,266 @@ +/** + * Integration tests for the upstream DuckDB CLI build. + * + * Verifies that the registry's source-built DuckDB binary works end-to-end with + * the shared WASI/POSIX runtime: + * - basic in-memory SQL execution + * - joins, indexes, and temp tables + * - persistent database files on the kernel VFS + * - crash recovery for uncommitted transactions + * - spill-to-disk via temp files + * - CSV ingestion from the kernel filesystem + * - remote fetch via the shared WASI/POSIX network stack followed by DuckDB query + */ + +import { describe, it, expect, afterEach } from 'vitest'; +import { createInMemoryFileSystem } from '@secure-exec/core'; +import type { Kernel } from '@secure-exec/core'; +import { + COMMANDS_DIR, + C_BUILD_DIR, + allowAll, + createKernel, + createNodeHostNetworkAdapter, + createWasmVmRuntime, +} from '../helpers.js'; +import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http'; +import { existsSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { setTimeout as sleep } from 'node:timers/promises'; + +const hasWasmDuckDB = existsSync(resolve(C_BUILD_DIR, 'duckdb')); +const hasWasmCurl = + (existsSync(resolve(COMMANDS_DIR, 'curl')) || existsSync(resolve(C_BUILD_DIR, 'curl'))); +const hasWasmHttpGet = existsSync(resolve(C_BUILD_DIR, 'http_get')); + +async function mountKernel( + filesystem: ReturnType, +) { + const kernel = createKernel({ + filesystem, + cwd: '/tmp', + permissions: allowAll, + hostNetworkAdapter: createNodeHostNetworkAdapter(), + }); + const commandDirs = existsSync(COMMANDS_DIR) ? [C_BUILD_DIR, COMMANDS_DIR] : [C_BUILD_DIR]; + await kernel.mount( + createWasmVmRuntime({ + commandDirs, + permissions: { + duckdb: 'read-write', + http_get: 'full', + }, + }) + ); + return kernel; +} + +function closeServer(server: Server) { + return new Promise((resolve, reject) => { + server.close((err) => { + if (err) reject(err); + else resolve(); + }); + }); +} + +async function waitForText( + getText: () => string, + expected: string, + timeoutMs = 5_000, +) { + const start = Date.now(); + while (!getText().includes(expected)) { + if (Date.now() - start >= timeoutMs) { + throw new Error(`timed out waiting for output: ${expected}\n\n${getText()}`); + } + await sleep(25); + } +} + +describe.skipIf(!hasWasmDuckDB)('duckdb command', { timeout: 120_000 }, () => { + let kernel: Kernel | undefined; + + afterEach(async () => { + await kernel?.dispose(); + kernel = undefined; + }); + + it('executes basic SQL against an in-memory database', async () => { + const filesystem = createInMemoryFileSystem(); + await filesystem.mkdir('/tmp'); + kernel = await mountKernel(filesystem); + + const result = await kernel.exec('duckdb -csv -c "SELECT 41 + 1 AS answer"'); + expect(result.exitCode).toBe(0); + expect(result.stdout.trim()).toBe('answer\n42'); + }); + + it('persists database files on the shared VFS and reopens them in a new process', async () => { + const filesystem = createInMemoryFileSystem(); + await filesystem.mkdir('/tmp'); + await filesystem.writeFile('/tmp/input.csv', 'name,value\nalpha,1\nbeta,2\n'); + + kernel = await mountKernel(filesystem); + let result = await kernel.exec( + `duckdb -csv /tmp/app.duckdb -c "CREATE TABLE items AS SELECT * FROM read_csv_auto('/tmp/input.csv');"` + ); + expect(result.exitCode).toBe(0); + await kernel.dispose(); + kernel = undefined; + + expect(await filesystem.exists('/tmp/app.duckdb')).toBe(true); + expect((await filesystem.stat('/tmp/app.duckdb')).size).toBeGreaterThan(0); + + kernel = await mountKernel(filesystem); + result = await kernel.exec( + `duckdb -csv /tmp/app.duckdb -c "SELECT name, value FROM items ORDER BY value;"` + ); + expect(result.exitCode).toBe(0); + expect(result.stdout.trim()).toBe('name,value\nalpha,1\nbeta,2'); + }); + + it('persists inserted and updated rows across process reopens', async () => { + const filesystem = createInMemoryFileSystem(); + await filesystem.mkdir('/tmp'); + kernel = await mountKernel(filesystem); + + let result = await kernel.exec( + `duckdb -csv /tmp/dml.duckdb -c "CREATE TABLE items(id INTEGER, value INTEGER); INSERT INTO items VALUES (1, 10), (2, 20); UPDATE items SET value = value + 1 WHERE id = 2;"` + ); + expect(result.exitCode).toBe(0); + + result = await kernel.exec( + `duckdb -csv /tmp/dml.duckdb -c "SELECT id, value FROM items ORDER BY id;"` + ); + expect(result.exitCode).toBe(0); + expect(result.stdout.trim()).toBe('id,value\n1,10\n2,21'); + }); + + it('supports joins and indexes on file-backed tables', async () => { + const filesystem = createInMemoryFileSystem(); + await filesystem.mkdir('/tmp'); + kernel = await mountKernel(filesystem); + + const result = await kernel.exec( + `duckdb -csv /tmp/analytics.duckdb -c "CREATE TABLE numbers AS SELECT i AS id, i * 10 AS score FROM range(1, 6) tbl(i); CREATE TABLE labels AS SELECT i AS id, concat('n', CAST(i AS VARCHAR)) AS name FROM range(1, 6) tbl(i); CREATE INDEX idx_numbers_id ON numbers(id); SELECT name, score FROM numbers JOIN labels USING (id) WHERE id >= 3 ORDER BY id;"` + ); + expect(result.exitCode).toBe(0); + expect(result.stdout.trim()).toBe('name,score\nn3,30\nn4,40\nn5,50'); + }); + + it('keeps temp tables scoped to a single DuckDB process', async () => { + const filesystem = createInMemoryFileSystem(); + await filesystem.mkdir('/tmp'); + kernel = await mountKernel(filesystem); + + let result = await kernel.exec( + `duckdb -csv /tmp/session.duckdb -c "CREATE TEMP TABLE session_values AS SELECT 7 AS value; SELECT value FROM session_values;"` + ); + expect(result.exitCode).toBe(0); + expect(result.stdout.trim()).toBe('value\n7'); + + result = await kernel.exec( + `duckdb -csv /tmp/session.duckdb -c "SELECT value FROM session_values;"` + ); + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toContain('session_values'); + }); + + it('drops uncommitted rows after a hard-killed process is reopened', async () => { + const filesystem = createInMemoryFileSystem(); + await filesystem.mkdir('/tmp'); + kernel = await mountKernel(filesystem); + + let result = await kernel.exec( + `duckdb -csv /tmp/recover.duckdb -c "CREATE TABLE items(value INTEGER); INSERT INTO items VALUES (1);"` + ); + expect(result.exitCode).toBe(0); + + let stdout = ''; + const proc = kernel.spawn('duckdb', ['-csv', '/tmp/recover.duckdb'], { + streamStdin: true, + onStdout: (chunk) => { + stdout += new TextDecoder().decode(chunk); + }, + }); + + await sleep(300); + proc.writeStdin('BEGIN;\nINSERT INTO items VALUES (42);\nSELECT COUNT(*) AS rows_in_tx FROM items;\n'); + await waitForText(() => stdout, 'rows_in_tx\n2'); + + proc.kill(9); + await proc.wait().catch(() => undefined); + + result = await kernel.exec( + `duckdb -csv /tmp/recover.duckdb -c "SELECT COUNT(*) AS rows, SUM(value) AS total FROM items;"` + ); + expect(result.exitCode).toBe(0); + expect(result.stdout.trim()).toBe('rows,total\n1,1'); + }); + + it('handles large sorted exports with a configured temp directory under constrained memory', async () => { + const filesystem = createInMemoryFileSystem(); + await filesystem.mkdir('/tmp'); + kernel = await mountKernel(filesystem); + + const result = await kernel.exec( + `duckdb -csv /tmp/spill.duckdb -c "PRAGMA temp_directory='/tmp/duckdb-spill'; SET threads=1; SET preserve_insertion_order=false; SET memory_limit='64MB'; COPY (SELECT i, repeat('x', 256) AS payload FROM range(300000) tbl(i) ORDER BY i DESC) TO '/tmp/spilled.csv' (HEADER, DELIMITER ',');"` + ); + expect(result.exitCode).toBe(0); + expect(await filesystem.exists('/tmp/spilled.csv')).toBe(true); + expect((await filesystem.stat('/tmp/spilled.csv')).size).toBeGreaterThan(50_000_000); + }); + + it.skipIf(!hasWasmCurl && !hasWasmHttpGet)( + 'queries data fetched over the network through the shared VFS', + async () => { + const filesystem = createInMemoryFileSystem(); + await filesystem.mkdir('/tmp'); + kernel = await mountKernel(filesystem); + + const server = createServer((req: IncomingMessage, res: ServerResponse) => { + if (req.url === '/' || req.url === '/remote.csv') { + res.writeHead(200, { 'Content-Type': 'text/csv' }); + res.end('city,value\nsf,3\nla,5\n'); + return; + } + + res.writeHead(404, { 'Content-Type': 'text/plain' }); + res.end('not found'); + }); + + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + + try { + const address = server.address(); + if (!address || typeof address === 'string') { + throw new Error('failed to bind test HTTP server'); + } + + let result; + if (hasWasmCurl) { + result = await kernel.exec( + `curl -fsS -o /tmp/remote.csv http://127.0.0.1:${address.port}/remote.csv` + ); + expect(result.exitCode).toBe(0); + } else { + result = await kernel.exec( + `http_get ${address.port} /remote.csv /tmp/remote.csv` + ); + expect(result.exitCode).toBe(0); + } + + expect(await filesystem.readTextFile('/tmp/remote.csv')).toContain('city,value'); + + result = await kernel.exec( + `duckdb -csv -c "SELECT SUM(value) AS total FROM read_csv_auto('/tmp/remote.csv');"` + ); + expect(result.exitCode).toBe(0); + expect(result.stdout.trim()).toBe('total\n8'); + } finally { + await closeServer(server); + } + } + ); +});