diff --git a/.env.example b/.env.example index 201282873..96c176740 100644 --- a/.env.example +++ b/.env.example @@ -137,8 +137,23 @@ COPILOTKIT_LICENSE_TOKEN= # 0, or leaving this unset, switches the watchdog off. Nothing is watched and no turn is ever ended. AGENT_STALL_TIMEOUT_MS=60000 +# Local Codex compatibility mode. This runs the host-side AG-UI adapter on port 4202, reuses the +# account already authenticated by `codex login`, and skips the two API-key Bot containers. Its +# Codex threads survive adapter restarts. Every side-effecting tool call returns through OpenBot's +# signed gateway, where the current grant, policy and audit trail are applied. MCP, app, plugin and +# web capabilities are disabled; turns are sandboxed read-only without network access, and the +# adapter interrupts any native shell or file action Codex still attempts. +# +# CODEX_AGENT_ENABLED=true +# CODEX_AGENT_PORT=4202 +# CODEX_AGENT_WORKSPACE=.openbot-codex/workspace +# CODEX_AGENT_STATE=.openbot-codex/threads.json +# OPENBOT_TOOL_URL=http://localhost:3001/api/agent-tools/call +# AGENT_ENDPOINT_ALLOWED_HOSTS=localhost:4202 + # Model key. Required by the proof-of-concept Bot, which speaks OpenAI's API directly, and by the -# framework Bot unless you point it at another provider below. +# framework Bot unless you enable the local Codex compatibility mode above or point it at another +# provider below. OPENAI_API_KEY= # Where that key is spent. Unset, it is OpenAI. Set, it is any endpoint speaking the same @@ -329,4 +344,3 @@ AGENT_TOOL_TOKEN= # for a deployment that has not stood up a worker. Set for one that has: openssl rand -base64 32. # Do not accept a default in production. WORKER_SHARED_SECRET= - diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7de4b2c0c..2a7ce9138 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,6 +117,19 @@ jobs: # question: every secret key a container demands has to be one the chart actually writes. # Getting that wrong is invisible until a pod starts, and every shipped target had it wrong. - run: bun scripts/check-rendered-chart.ts rendered.yaml + - name: Render VM computer mode + if: matrix.target == 'eks-sandbox' + run: | + helm template ci charts/openbot \ + --values charts/openbot/ci/eks-sandbox-values.yaml \ + --set computers.mode=vm \ + --set computers.runtimeClassName=kata-qemu \ + --set-string secrets.keyEncryptionKey="$(openssl rand -base64 32)" \ + --api-versions agents.x-k8s.io/v1beta1/Sandbox \ + --api-versions extensions.agents.x-k8s.io/v1beta1/SandboxTemplate \ + > rendered-vm.yaml + grep -q 'runtimeClassName: kata-qemu' rendered-vm.yaml + bun scripts/check-rendered-chart.ts rendered-vm.yaml # And that the refusals are load-bearing rather than decorative. A chart full of `fail` # messages nothing ever triggers is a chart that has never been shown to refuse anything, and # every one of these describes a state that shipped in a values file at some point. @@ -152,6 +165,10 @@ jobs: # A browser inside every replica of a replicated API. refuses "an embedded browser across several replicas" \ --set server.embeddedComputer=true --set server.replicaCount=2 + # A VM label that quietly launches an ordinary shared-kernel pod would be worse than not + # offering the mode. The runtime is the boundary and must be explicit. + refuses "VM computer mode without a VM-backed RuntimeClass" \ + --set computers.mode=vm --set-string computers.runtimeClassName= # A Bot's egress proxy on a port the computer's own network policy does not allow. The # variables reach the computer through extraEnv, so nothing else notices that the policy # then refuses to let it be reached. @@ -161,11 +178,11 @@ jobs: --set-string computers.extraEnv[0].value=http://proxy.internal:3128 # A warm pool nothing claims from. Only meaningful where the target asks for per-Bot # computers; on the others the mode is not sandbox and the refusal is not armed. - if grep -qE '^ *mode: sandbox' charts/openbot/ci/${{ matrix.target }}-values.yaml; then + if grep -qE '^ *mode: (sandbox|vm)' charts/openbot/ci/${{ matrix.target }}-values.yaml; then refuses "a warm pool no Bot can be handed a computer from" \ --set computers.sandbox.warmPool.enabled=true else - echo "skipped: the warm-pool refusal is only armed for computers.mode: sandbox" + echo "skipped: the warm-pool refusal is only armed for per-Bot computer modes" fi # And that a values key this chart did not used to have still renders when it is absent. # diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml index 4d079c0b5..415892a76 100644 --- a/.github/workflows/publish-release.yml +++ b/.github/workflows/publish-release.yml @@ -90,6 +90,10 @@ jobs: run: | set -euo pipefail test "v$(bun -e 'console.log(require("./package.json").version)')" = "$VERSION" + grep -qx "appVersion: \"${VERSION#v}\"" charts/openbot/Chart.yaml || { + echo "::error::The Helm chart does not point at ${VERSION}." + exit 1 + } grep -q "^## ${VERSION#v}$" CHANGELOG.md || { echo "::error::CHANGELOG.md has no section for ${VERSION#v}." exit 1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a00b6522d..b77b66a22 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -103,6 +103,12 @@ jobs: ' -- "$BUMP" > /tmp/version version=$(cat /tmp/version) + # The chart's default image must be the image this release builds. Leaving appVersion on + # the previous release makes new templates execute against an older filesystem, which can + # turn a valid chart upgrade into a command-not-found CrashLoop. + sed -i -E "s/^appVersion: .*/appVersion: \"$version\"/" charts/openbot/Chart.yaml + grep -qx "appVersion: \"$version\"" charts/openbot/Chart.yaml + # Unreleased becomes the version, and a fresh empty Unreleased takes its place so the next # change has somewhere to go without anyone hand-editing a heading. bun -e ' diff --git a/.gitignore b/.gitignore index bfc1237c7..0d2d4c41e 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ node_modules/ app/src/lib/generated/application-config.ts .logs/ .demo-logs/ +.openbot-codex/ **/.impeccable .wave-state.md @@ -28,3 +29,4 @@ app/.tanstack/ # it is what makes that fetch reproducible, and ignoring it meant every build resolved the dependency # afresh, so CI and a customer install could take different subchart versions with no diff to show it. charts/*/charts/ +.pstack/ diff --git a/CHANGELOG.md b/CHANGELOG.md index d446e0438..5c1ad6613 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,55 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. ## Unreleased +### People and Bots can use their own avatars + +The signed-in person can upload, replace, or remove a PNG, JPEG, or WebP avatar from Settings. Bot +owners can do the same from the coworker dialog, and deployment administrators can brand packaged +Bots without gaining permission to rewrite their executable profiles. Custom images now follow the +person or Bot through the sidebar, channel roster, recipients, handoffs, and profile surfaces, while +removal returns to the identity-provider image, initials, or generated Bot avatar. + +Uploads are authenticated, limited to 2 MB and safe image dimensions, checked against their actual +file signature, and served from short private versioned URLs rather than copied into roster JSON. + +### Computer use is a whole graphical computer, with a dock that survives closing the browser + +A Bot with an isolated computer now exposes its full Linux desktop through the guarded OpenBot +screen: browser chrome, tabs, native dialogs, a terminal, and the desktop itself rather than only a +CDP canvas of one page. The small dock contains exactly the applications the image provides — the +managed Browser and a Terminal in the durable workspace. Closing Chromium with its own **X** leaves +the desktop running, records that the browser closed, and the Browser icon opens the same Bot profile +again instead of leaving the computer in a dead state. Non-default desktop sizes are letterboxed +rather than stretched. + +The full framebuffer is offered only by a per-Bot provider. A shared computer keeps the Bot-scoped +page stream, because its process-wide desktop could expose another Bot's window. Kubernetes +`computers.mode: sandbox` gives each Bot its own container; `vm` requires the configured Kata runtime +class for the VM boundary. Older computer images also fall back to the page stream during a rolling +upgrade. The chart is now 0.2.0, and release automation keeps its default `appVersion` on the exact +OpenBot image being published. + +Watching remains read-only. Taking control now returns a random, short-lived capability held only by +that browser tab; the computer checks it again on the WebSocket and on every mouse or keyboard event. +A second authorized viewer can watch but cannot inject input, replace the first person's takeover, or +reuse the public “human is driving” state as permission. Closing the tab lets the lease expire and +returns the wheel to the Bot. + +### A local Codex coworker keeps its conversation and uses OpenBot's governed tools + +OpenBot can now run a coworker through the Codex app already signed in on the host, without an API +key. Its conversation survives adapter and app-server restarts: OpenBot records the Codex thread it +owns, resumes that exact thread, and refuses to silently replace unreadable recovery state. + +Connector calls do not go around the deployment. Codex sees only the tools OpenBot assigned to that +coworker, and every call returns with the deployment's signed run assertion through the existing +grant, policy and audit gateway. Native Codex shell, file, web, browser, computer, app, plugin and MCP +paths are disabled before a turn can begin, the child process receives none of OpenBot's tool, +computer, database or provider credentials, and the remaining turn is sandboxed read-only without +network access. A post-start interruption remains as an alarm if a disabled native path is ever +reported. Rejected, stale and duplicate callbacks are reported as failed tool results rather than +being carried out. + ### Coworkers are made in a wizard and managed in a dialog Creating a coworker is now a three-step wizard — who it is, who may see it, then where it runs, diff --git a/Dockerfile b/Dockerfile index 674ce392d..6f0329f70 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,8 +11,9 @@ # is meant to run on. # # The supervisor. It exists to give each Bot its own container, which needs a Docker socket, which -# no serverless container platform permits. Without it every Bot shares the browser below, exactly -# as they do on a laptop with no supervisor configured. Per-Bot isolation is A6. +# no serverless container platform permits. Without it every Bot shares the desktop below, exactly +# as they do on a laptop with no supervisor configured. Kubernetes deployments use +# `computers.mode: sandbox` or `vm` for one isolated computer per Bot. # # THE BASE IS PLAYWRIGHT'S, not Bun's, because Chromium and its system libraries have to stay # matched and that image is the only place that is guaranteed. The tag must move with the @@ -28,7 +29,17 @@ ARG BUN_VERSION=1.3.14 # root's home. Set before the install, or the installer has already chosen the wrong directory. ENV BUN_INSTALL=/usr/local ENV PATH="/usr/local/bin:${PATH}" -RUN apt-get update && apt-get install -y --no-install-recommends unzip xz-utils \ +RUN apt-get update && apt-get install -y --no-install-recommends \ + dbus-x11 \ + openbox \ + tint2 \ + unzip \ + websockify \ + x11vnc \ + x11-xserver-utils \ + xterm \ + xz-utils \ + xvfb \ && rm -rf /var/lib/apt/lists/* \ && curl -fsSL https://bun.sh/install | bash -s "bun-v${BUN_VERSION}" @@ -41,6 +52,7 @@ WORKDIR /src COPY package.json bun.lock ./ COPY tsconfig.base.json bunfig.toml ./ COPY app/package.json app/package.json +COPY agent-codex/package.json agent-codex/package.json COPY server/package.json server/package.json COPY worker/package.json worker/package.json RUN bun install --frozen-lockfile @@ -57,8 +69,9 @@ RUN cd agent-computer && bun install --frozen-lockfile # biome and the test tooling are a gigabyte that nothing in a running container imports. RUN mkdir -p /prod && cp package.json bun.lock /prod/ \ && cp -r app/package.json /prod/app-package.json \ - && cd /prod && mkdir -p app server worker \ + && cd /prod && mkdir -p app agent-codex server worker \ && cp /src/app/package.json app/package.json \ + && cp /src/agent-codex/package.json agent-codex/package.json \ && cp /src/server/package.json server/package.json \ && cp /src/worker/package.json worker/package.json \ && bun install --frozen-lockfile --production @@ -111,6 +124,8 @@ COPY shared shared COPY examples examples COPY agent-computer/src agent-computer/src COPY agent-computer/package.json agent-computer/package.json +COPY agent-computer/entrypoint.sh agent-computer/entrypoint.sh +RUN chmod +x agent-computer/entrypoint.sh # The built app, served by the API on the same origin. There is no CORS in this server, so this is # not a convenience: two origins would simply fail. @@ -147,10 +162,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # the tool description tells a model to run. `sudo cat /proc/1/environ` does not. # # WHAT THIS IS NOT. It is a floor, not a boundary. Root is one CVE away and a shared container is not -# an isolation story for code a model wrote: that needs a computer per Bot and a sandbox under it, -# which is why per-Bot computers and gVisor are not optional extras next to this feature. Run the -# image with `--security-opt no-new-privileges` where the platform allows, which turns setuid off -# entirely for anything not named here. +# an isolation story for code a model wrote: that needs a computer per Bot and a sandboxed runtime +# under it. Kubernetes deployments get that with `computers.mode: sandbox`, or `vm` plus a Kata +# runtime class for a hardware-virtualized boundary. Run the image with +# `--security-opt no-new-privileges` where the platform allows, which turns setuid off entirely for +# anything not named here. RUN apt-get update && apt-get install -y --no-install-recommends sudo \ && rm -rf /var/lib/apt/lists/* \ && printf '%s\n' \ diff --git a/README.md b/README.md index 81239b3f9..65adae029 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ A Bot is any endpoint speaking [AG-UI](https://github.com/ag-ui-protocol/ag-ui), - Docker, for PostgreSQL and the shipped Bots. - [Bun](https://bun.sh) 1.3+, for the app and API server. - A CopilotKit Intelligence project and license. A free plan is available, and Intelligence can be self-hosted. -- A model key. The proof-of-concept Bot uses OpenAI; the LangGraph Bot can use OpenAI, Anthropic, or Google. +- Either a model key, or the installed Codex CLI already authenticated with `codex login`. The proof-of-concept Bot uses OpenAI; the LangGraph Bot can use OpenAI, Anthropic, or Google; local Codex mode uses the ChatGPT account on this machine. ## Quick start @@ -86,6 +86,11 @@ A Bot is any endpoint speaking [AG-UI](https://github.com/ag-ui-protocol/ag-ui), - `OPENAI_API_KEY` + Or, to use the ChatGPT account already signed in through Codex instead of a provider API key, + set `CODEX_AGENT_ENABLED=true` and + `AGENT_ENDPOINT_ALLOWED_HOSTS=localhost:4202`. See + [agent-codex/README.md](agent-codex/README.md). + Keep the managed Intelligence URLs from `.env.example` unless you run Intelligence yourself. The example `KEY_ENCRYPTION_KEY` is public and fine locally; generate your own with: ```sh @@ -129,11 +134,11 @@ Leave `EMBEDDED_POSTGRES` off and set `DATABASE_URL` to point at a database you | Route | Purpose | | -------------------- | ------------------------------------------------------------------ | | `/` | Start and browse channels. | -| `/agents` | Create, edit, duplicate, hide, delete, and launch coworkers. | +| `/agents` | Create, edit, brand, duplicate, hide, delete, and launch coworkers. | | `/channel/:id` | Converse with one coworker, watch its screen, and see what it ran. | | `/bot` | Direct chat with a Bot; `?agent=` selects one. | | `/skills` | Create and enable personal skills. | -| `/settings` | User preferences. | +| `/settings` | Personal profile, avatar, and user preferences. | | `/admin/credentials` | Store write-only encrypted credentials. | | `/admin/computers` | View, stop, and reset Bot computers. | | `/admin/boundaries` | Configure browser/file/MCP action policy. | @@ -144,7 +149,7 @@ Leave `EMBEDDED_POSTGRES` off and set `DATABASE_URL` to point at a database you ## Features -- **A computer per Bot**: the supervisor gives each Bot its own container, its own `/workspace` volume and its own browser profile. Set `COMPUTER_RUNTIME=runsc` to run them under gVisor where the host supports it. +- **A computer per Bot**: the supervisor gives each Bot its own container, its own `/workspace` volume and its own browser profile. The guarded screen shows the full Linux desktop, including browser chrome, dialogs and a terminal; its Browser and Terminal dock stays available when Chromium is closed. Set `COMPUTER_RUNTIME=runsc` to run them under gVisor where the host supports it. - **A shell, not just a browser**: a Bot can run a command in its workspace, install what it needs, and process a file it saved. Through the same gate as everything else, so a rule can refuse a shell outright or refuse particular commands, and the command is on the record either way. The command inherits PATH, locale, terminal and proxy variables, not the rest of the deployment's environment. - **The gateway is the only way in**: it resolves the target from a server-held snapshot, evaluates the policy, writes the audit row, and only then calls the computer. There is no path that acts without the record existing first. - **CEL policy, fail closed**: rules can inspect `tool.name`, `intent`, `bot.id`, `actor.id`, `page.url`, `page.host`, `element.*`, `key`, `file.*` and `mcp.*`. Deny is evaluated before allow, a missing policy permits nothing, and a broken rule refuses rather than opens. @@ -152,6 +157,8 @@ Leave `EMBEDDED_POSTGRES` off and set `DATABASE_URL` to point at a database you - **Take the wheel**: a Bot that hits a login wall or a 2FA prompt asks for help. Control is handed over in the same panel and recorded as `computer.help_requested`, `computer.control_taken` and `computer.control_released`. While a person is driving, Bot actions are refused rather than queued. - **Secrets never enter the transcript**: the trail records that a secret was requested and how long it was, not what it said. - **Bring your own agent**: any AG-UI endpoint is a Bot, on a framework or hand-written. Endpoints are validated with the same target checks used for browser navigation, and an auth header is stored write-only. +- **Use the Codex account already on your machine**: local Codex mode resumes persistent Codex threads and exposes only the tools OpenBot assigned. Calls return through the same grant, policy and audit gateway; Codex-native action surfaces are disabled before app-server turns begin. +- **Give people and Bots their own faces**: upload, replace, or remove PNG, JPEG, and WebP avatars from Settings or the coworker dialog. The same image follows that identity through the sidebar, channels, recipients, handoffs, and profile surfaces. - **Components instead of prose**: compiled React components live in `app/src/components/gallery/`, sandboxed ones are authored in `/admin/playground` and published with no deployment. Every call asks the server whether the component exists, is published, and is not withheld from that Bot. Data functions are granted per component. - **Governed MCP**: Google Drive and Notion ship in the catalogue, reached as the person asking. The catalogue carries only vendors this deployment stands behind, so adding one is a review of that vendor. Custom servers must pass URL checks; unknown tools and custom-server tools are treated as writes, and a catalogue tool the server advertises but does not name as a write classifies as a read. A Bot is told which connectors exist here and which it holds, so it says it has not been granted one rather than browsing to the vendor's website. - **Skills are instructions, not capabilities**: personal skills attach only to Bots their author owns, deployment skills are admin-owned, and both are invoked with `/` in the composer. @@ -214,6 +221,7 @@ Settings worth knowing: | `COMPUTER_TOKEN` | Secret every Bot computer request must present. `start.sh` sets one. | | `SUPERVISOR_TOKEN` | Secret the supervisor requires. `start.sh` sets one. | | `AGENT_TOOL_TOKEN` | Secret a Bot presents to call a granted tool back. `start.sh` sets one. Without it no Bot may call tools. | +| `CODEX_AGENT_ENABLED` | Runs the host-side Codex adapter on port 4202 and skips the two provider-key Bot containers. | | `COMPUTER_SUPERVISOR_URL` | Gives each Bot a computer of its own instead of one shared computer. | | `COMPUTER_RUNTIME` | Set to `runsc` to run computers under gVisor, where the host has it. | | `COMPUTER_SANDBOX` | Set to `on` for Chromium's own sandbox, where the host permits it. | @@ -232,9 +240,10 @@ Full reference: [docs/configuration.md](docs/configuration.md). | ------------------------ | -------------------------- | ------------------------------------------------------------------------------------------------ | | `app` | 3010 | React/Vite UI. | | `server` | 3001 | Hono API, CopilotKit runtime, auth, policy, audit, plugins, components, coworkers, and channels. | -| `agent-computer` | 4100 | Chromium plus `/workspace` and browser profile. | +| `agent-computer` | 4100 | Linux desktop, Chromium, terminal, `/workspace`, and browser profile. | | `agent-bot` | 4200 | Proof-of-concept AG-UI Bot. | | `agent-langgraph` | 4201 | LangGraph AG-UI Bot. | +| `agent-codex` | 4202 | Optional host-side Codex AG-UI adapter using the local ChatGPT login. | | `supervisor` | 4500 host / 4300 container | Creates and manages one computer per Bot. | | PostgreSQL with pgvector | 5432 | Product data, policy, audit, credentials, grants, channels, and component metadata. | | CopilotKit Intelligence | external | Durable threads and memory. | diff --git a/agent-codex/README.md b/agent-codex/README.md new file mode 100644 index 000000000..458457d92 --- /dev/null +++ b/agent-codex/README.md @@ -0,0 +1,21 @@ +# Codex coworker + +This local-only AG-UI adapter lets OpenBot talk to the installed Codex app-server using the ChatGPT +account already authenticated by `codex login`. + +The adapter records the join between each OpenBot Intelligence thread and its persistent Codex +thread in `.openbot-codex/threads.json`. On every later run—including after the adapter restarts—it +resumes that Codex thread and refreshes its standing instructions. Codex restores the dynamic-tool +catalog persisted with the thread; OpenBot still rechecks the current grant and policy on every call. + +Only tools that OpenBot marks as deployment-owned are exposed as Codex dynamic tools. Calls return +to `/api/agent-tools/call` with OpenBot's signed run assertion and agent token, so the deployment +rechecks the Bot's grant and policy and writes the normal audit events. Codex-native shell, file, +MCP, app, web and multi-agent paths are disabled. The adapter interrupts native shell and file +attempts; turns run in the read-only sandbox without network access, so an action that races the +interrupt cannot write or reach the network. + +Enable it with `CODEX_AGENT_ENABLED=true`. `scripts/start.sh` then runs this adapter on +`CODEX_AGENT_PORT` (default `4202`) and skips the two provider-API-key Bot containers. The start +script supplies `AGENT_TOOL_TOKEN`, `OPENBOT_TOOL_URL` and `CODEX_AGENT_STATE`; set those explicitly +when starting `agent-codex` by hand. diff --git a/agent-codex/bun.lock b/agent-codex/bun.lock new file mode 100644 index 000000000..44015f926 --- /dev/null +++ b/agent-codex/bun.lock @@ -0,0 +1,40 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "@openbot/agent-codex", + "dependencies": { + "@ag-ui/core": "0.0.57", + "@ag-ui/encoder": "0.0.57", + }, + "devDependencies": { + "@types/bun": "^1.3.3", + "typescript": "^5.9.3", + }, + }, + }, + "packages": { + "@ag-ui/core": ["@ag-ui/core@0.0.57", "", { "dependencies": { "zod": "^3.22.4" } }, "sha512-gho1OWjNE6E3Rl7ZEZ1wr2CEpUHjLFU0FqzCZZk439TicLu+BfLCMkMokB07bMGlRmbJ60hM6LW60iOVauCx+Q=="], + + "@ag-ui/encoder": ["@ag-ui/encoder@0.0.57", "", { "dependencies": { "@ag-ui/core": "0.0.57", "@ag-ui/proto": "0.0.57" } }, "sha512-ifD9NctR4xyPDR58xF9GK1bj/S8oECFkTeDfuYD8tXdbcOstIJ2TOqU2zhiCKnw7Vw+zR9Qv3TbsM9E7Gi9X3Q=="], + + "@ag-ui/proto": ["@ag-ui/proto@0.0.57", "", { "dependencies": { "@ag-ui/core": "0.0.57", "@bufbuild/protobuf": "^2.2.5", "@protobuf-ts/protoc": "^2.11.1" } }, "sha512-pPENOZt0P6ibH8sCTgq05wLYXi5t3P9B5r/1bWYehXjUxtyOdnukSlWM++SsCIwUXsQdm/b3aBgGjEeTF7RenA=="], + + "@bufbuild/protobuf": ["@bufbuild/protobuf@2.14.1", "", {}, "sha512-agRJn3+EJDUe8AvxTx/LnHA/GErvLE62pSaSk7+MwFOtOv8eWBu/qCq2qoZjBjVZ3C2aiFJCveuSs17KMkYGOw=="], + + "@protobuf-ts/protoc": ["@protobuf-ts/protoc@2.11.1", "", { "bin": { "protoc": "protoc.js" } }, "sha512-mUZJaV0daGO6HUX90o/atzQ6A7bbN2RSuHtdwo8SSF2Qoe3zHwa4IHyCN1evftTeHfLmdz+45qo47sL+5P8nyg=="], + + "@types/bun": ["@types/bun@1.4.0", "", { "dependencies": { "bun-types": "1.4.0" } }, "sha512-K+lZULY23vRgK/CfTjFIV+tyifaNdSMlPh9j+6mQ/cLfpOznLyAuzgV/JQysyECpkBQLVMSyvjlr2fBUSA9wFQ=="], + + "@types/node": ["@types/node@26.4.0", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-faiGnoIrLH/V8cibOMEAZ8pMw6oXqSukl29ra4mN8GdaB2ZewzeaLj+INpV5N+Z1eKWzY+IzaIZH2EIR6YZRNQ=="], + + "bun-types": ["bun-types@1.4.0", "", { "dependencies": { "@types/node": "*" } }, "sha512-iIKw23BspnQQYd3prITOBxeUsxBHnwzX6YJfGMuNOZzeNcMmVqzIIVGRm1l69ogaPQmb4wB6BN8mA5bE9YuC5Q=="], + + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + + "undici-types": ["undici-types@8.3.0", "", {}, "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ=="], + + "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], + } +} diff --git a/agent-codex/package.json b/agent-codex/package.json new file mode 100644 index 000000000..faaf64f45 --- /dev/null +++ b/agent-codex/package.json @@ -0,0 +1,21 @@ +{ + "name": "@openbot/agent-codex", + "version": "0.0.2", + "license": "MIT", + "private": true, + "type": "module", + "scripts": { + "start": "bun src/index.ts", + "dev": "bun --watch src/index.ts", + "test": "bun test", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@ag-ui/core": "0.0.57", + "@ag-ui/encoder": "0.0.57" + }, + "devDependencies": { + "@types/bun": "^1.3.3", + "typescript": "^5.9.3" + } +} diff --git a/agent-codex/src/codex-client.ts b/agent-codex/src/codex-client.ts new file mode 100644 index 000000000..a30f4588f --- /dev/null +++ b/agent-codex/src/codex-client.ts @@ -0,0 +1,714 @@ +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { resolve } from "node:path"; +import { createInterface } from "node:readline"; +import type { CodexDynamicTool, ToolResult } from "./tools"; + +type JsonObject = Record; +type JsonRpcMessage = { + id?: number | string; + method?: string; + params?: JsonObject; + result?: unknown; + error?: { code?: number; message?: string }; +}; + +type AccountSummary = { + authMode: string; + planType: string | null; +}; + +type PendingRequest = { + resolve: (value: unknown) => void; + reject: (error: Error) => void; + timeout: ReturnType; + /** Runs in the protocol reader before later messages can overtake the promise continuation. */ + beforeResolve?: (value: unknown) => void; +}; + +type ThreadResult = { + thread: { id: string }; +}; + +type TurnStartResult = { + turn: { id: string }; +}; + +export type TurnCallbacks = { + onText(delta: string): void; + onToolCall( + callId: string, + name: string, + args: Record, + ): Promise; +}; + +type ActiveTurn = { + turnId?: string; + callbacks: TurnCallbacks; + toolCallIds: Set; + fail(error: Error): void; +}; + +type SpawnAppServer = () => ChildProcessWithoutNullStreams; + +const REQUEST_TIMEOUT_MS = 30_000; +const DEFAULT_TURN_TIMEOUT_MS = 180_000; +const DISABLED_NATIVE_FEATURES = [ + "apps", + "browser_use", + "browser_use_external", + "browser_use_full_cdp_access", + "code_mode", + "code_mode_host", + "computer_use", + "goals", + "hooks", + "image_generation", + "in_app_browser", + "memories", + "multi_agent", + "plugins", + "remote_plugin", + "shell_snapshot", + "shell_tool", + "skill_mcp_dependency_install", + "unified_exec", + "workspace_dependencies", +] as const; +const BLOCKED_ITEM_TYPES = new Set([ + "commandExecution", + "fileChange", + "mcpToolCall", + "collabAgentToolCall", + "subAgentActivity", + "webSearch", + "imageView", + "imageGeneration", +]); + +/** JSON-RPC client for a local Codex app-server with OpenBot as its only tool boundary. */ +export class CodexAppServerClient { + private child: ChildProcessWithoutNullStreams | undefined; + private nextId = 1; + private pending = new Map(); + private listeners = new Set<(message: JsonRpcMessage) => void>(); + private activeTurns = new Map(); + private account: AccountSummary | undefined; + private safetyConfig: JsonObject = safetyConfigFor([]); + + constructor(private readonly spawnAppServer: SpawnAppServer = launchCodex) {} + + async start(): Promise { + if (this.child) return; + + const child = this.spawnAppServer(); + this.child = child; + + const lines = createInterface({ input: child.stdout }); + lines.on("line", (line) => this.receive(line)); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + const text = chunk.trim(); + if (text) console.error(`[codex app-server] ${text}`); + }); + child.once("error", (error) => this.failAll(error)); + child.once("exit", (code, signal) => { + this.child = undefined; + this.failAll( + new Error( + `Codex app-server exited (${signal ?? `status ${code ?? "unknown"}`}).`, + ), + ); + }); + + await this.request("initialize", { + clientInfo: { + name: "openbot_local_codex", + title: "OpenBot local Codex coworker", + version: "0.0.2", + }, + capabilities: { experimentalApi: true }, + }); + this.notify("initialized", {}); + + const result = (await this.request("account/read", { + refreshToken: false, + })) as { + account?: { type?: string; planType?: string | null } | null; + }; + if (result.account?.type !== "chatgpt") { + throw new Error( + "Codex is not logged in with ChatGPT. Run `codex login` on this Mac first.", + ); + } + this.account = { + authMode: result.account.type, + planType: result.account.planType ?? null, + }; + + const configResult = (await this.request("config/read", { + includeLayers: false, + })) as { config?: { mcp_servers?: unknown } }; + this.safetyConfig = safetyConfigFor( + isObject(configResult.config?.mcp_servers) + ? Object.keys(configResult.config.mcp_servers) + : [], + ); + } + + accountSummary(): AccountSummary { + if (!this.account) { + throw new Error("Codex app-server has not finished starting."); + } + return this.account; + } + + async startThread( + cwd: string, + developerInstructions: string, + dynamicTools: CodexDynamicTool[], + ): Promise { + const result = (await this.request("thread/start", { + cwd, + approvalPolicy: "never", + sandbox: "read-only", + serviceName: "openbot_local_codex", + developerInstructions, + dynamicTools, + config: this.safetyConfig, + ephemeral: false, + })) as ThreadResult; + if (!result.thread?.id) { + throw new Error("Codex app-server did not return a thread id."); + } + return result.thread.id; + } + + async resumeThread( + threadId: string, + cwd: string, + developerInstructions: string, + ): Promise { + const result = (await this.request("thread/resume", { + threadId, + cwd, + approvalPolicy: "never", + sandbox: "read-only", + developerInstructions, + config: this.safetyConfig, + })) as ThreadResult; + if (result.thread?.id !== threadId) { + throw new Error( + `Codex resumed ${result.thread?.id ?? "no thread"} instead of ${threadId}.`, + ); + } + } + + async runTurn( + threadId: string, + cwd: string, + prompt: string, + callbacks: TurnCallbacks, + signal?: AbortSignal, + ): Promise { + if (this.activeTurns.has(threadId)) { + throw new Error(`Codex thread ${threadId} already has an active turn.`); + } + + let turnError: string | undefined; + const streamedItems = new Set(); + const timeoutMs = turnTimeoutMs(); + let settled = false; + let resolveCompletion: (() => void) | undefined; + let rejectCompletion: ((error: Error) => void) | undefined; + const completed = new Promise((resolve, reject) => { + resolveCompletion = resolve; + rejectCompletion = reject; + }); + const finish = () => { + if (settled) return; + settled = true; + resolveCompletion?.(); + }; + const fail = (error: Error) => { + if (settled) return; + settled = true; + rejectCompletion?.(error); + }; + const active: ActiveTurn = { callbacks, toolCallIds: new Set(), fail }; + this.activeTurns.set(threadId, active); + + const interrupt = () => { + if (!active.turnId) return; + void this.request("turn/interrupt", { + threadId, + turnId: active.turnId, + }).catch(() => {}); + }; + const abort = () => { + interrupt(); + fail( + new Error("OpenBot ended the request before the Codex turn finished."), + ); + }; + if (signal?.aborted) abort(); + else signal?.addEventListener("abort", abort, { once: true }); + + const timeout = setTimeout(() => { + interrupt(); + fail(new Error(`Codex did not finish within ${timeoutMs}ms.`)); + }, timeoutMs); + + const unsubscribe = this.onMessage((message) => { + const params = message.params ?? {}; + if (params.threadId !== threadId) return; + /* + * `thread/resume` can replay notifications from the last persisted turn before `turn/start` + * answers with the new turn id. Those are history, not this request. Let only the response to + * `turn/start` establish ownership; otherwise a replayed item makes the real turn look like an + * unrelated concurrent turn and the whole run is rejected before it begins. + */ + if (!active.turnId) return; + if (typeof params.turnId === "string" && params.turnId !== active.turnId) + return; + + if (message.method === "item/started") { + const item = isObject(params.item) ? params.item : {}; + if ( + typeof item.type === "string" && + BLOCKED_ITEM_TYPES.has(item.type) + ) { + const error = new Error( + `Codex attempted the native ${item.type} path. OpenBot refused it because side effects must use a governed OpenBot tool.`, + ); + interrupt(); + fail(error); + } + return; + } + + if (message.method === "item/agentMessage/delta") { + const itemId = typeof params.itemId === "string" ? params.itemId : ""; + const delta = typeof params.delta === "string" ? params.delta : ""; + if (itemId) streamedItems.add(itemId); + if (delta) callbacks.onText(delta); + return; + } + + if (message.method === "item/completed") { + const item = isObject(params.item) ? params.item : {}; + if ( + item.type === "agentMessage" && + typeof item.id === "string" && + !streamedItems.has(item.id) && + typeof item.text === "string" && + item.text + ) { + callbacks.onText(item.text); + } + return; + } + + if (message.method === "error") { + const error = isObject(params.error) ? params.error : {}; + turnError = + typeof error.message === "string" + ? error.message + : "Codex reported an unknown error."; + return; + } + + if (message.method === "turn/completed") { + const turn = isObject(params.turn) ? params.turn : {}; + if ( + active.turnId && + typeof turn.id === "string" && + turn.id !== active.turnId + ) + return; + if (turn.status === "completed") { + finish(); + } else { + const error = isObject(turn.error) ? turn.error : {}; + fail( + new Error( + turnError ?? + (typeof error.message === "string" + ? error.message + : undefined) ?? + `Codex turn ended with status ${String(turn.status ?? "unknown")}.`, + ), + ); + } + } + }); + + try { + const result = (await this.request( + "turn/start", + { + threadId, + input: [{ type: "text", text: prompt, text_elements: [] }], + cwd, + approvalPolicy: "never", + sandboxPolicy: { type: "readOnly", networkAccess: false }, + effort: "low", + }, + (value) => { + const turnId = (value as TurnStartResult).turn?.id; + if (turnId) active.turnId = turnId; + }, + )) as TurnStartResult; + const startedTurnId = result.turn?.id; + if (!startedTurnId) { + throw new Error("Codex app-server did not return a turn id."); + } + active.turnId = startedTurnId; + if (signal?.aborted) interrupt(); + await completed; + } finally { + clearTimeout(timeout); + signal?.removeEventListener("abort", abort); + unsubscribe(); + this.activeTurns.delete(threadId); + } + } + + stop(): void { + this.child?.kill(); + this.child = undefined; + } + + private onMessage(listener: (message: JsonRpcMessage) => void): () => void { + this.listeners.add(listener); + return () => this.listeners.delete(listener); + } + + private request( + method: string, + params: JsonObject, + beforeResolve?: (value: unknown) => void, + ): Promise { + const id = this.nextId++; + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + this.pending.delete(id); + reject(new Error(`Codex app-server request ${method} timed out.`)); + }, REQUEST_TIMEOUT_MS); + this.pending.set(id, { + resolve, + reject, + timeout, + ...(beforeResolve ? { beforeResolve } : {}), + }); + this.write({ method, id, params }); + }); + } + + private notify(method: string, params: JsonObject): void { + this.write({ method, params }); + } + + private write(message: JsonRpcMessage): void { + const child = this.child; + if (!child?.stdin.writable) { + throw new Error("Codex app-server is not running."); + } + child.stdin.write(`${JSON.stringify(message)}\n`); + } + + private receive(line: string): void { + let message: JsonRpcMessage; + try { + message = JSON.parse(line) as JsonRpcMessage; + } catch { + console.error("Codex app-server returned a non-JSON line."); + return; + } + + if (message.id !== undefined && message.method) { + void this.answerServerRequest(message); + return; + } + + if (message.id !== undefined) { + const pending = this.pending.get(message.id); + if (!pending) return; + this.pending.delete(message.id); + clearTimeout(pending.timeout); + if (message.error) { + pending.reject( + new Error( + message.error.message ?? "Codex app-server request failed.", + ), + ); + } else { + try { + pending.beforeResolve?.(message.result); + pending.resolve(message.result); + } catch (error) { + pending.reject( + error instanceof Error + ? error + : new Error("Codex app-server returned an invalid response."), + ); + } + } + return; + } + + for (const listener of this.listeners) listener(message); + } + + private async answerServerRequest(message: JsonRpcMessage): Promise { + if ( + message.method === "item/commandExecution/requestApproval" || + message.method === "item/fileChange/requestApproval" + ) { + this.write({ id: message.id, result: { decision: "decline" } }); + return; + } + + if (message.method === "item/permissions/requestApproval") { + this.write({ + id: message.id, + error: { + code: -32000, + message: + "OpenBot denied the requested native permission. Use an OpenBot dynamic tool instead.", + }, + }); + return; + } + + if (message.method === "item/tool/call") { + const params = message.params ?? {}; + const threadId = + typeof params.threadId === "string" ? params.threadId : ""; + const active = this.activeTurns.get(threadId); + const turnId = typeof params.turnId === "string" ? params.turnId : ""; + const callId = typeof params.callId === "string" ? params.callId : ""; + const name = typeof params.tool === "string" ? params.tool : ""; + if ( + !active || + !turnId || + turnId !== active.turnId || + !callId || + !name || + params.namespace !== null + ) { + this.write({ + id: message.id, + result: { + contentItems: [ + { + type: "inputText", + text: "OpenBot refused this tool call because it does not belong to the active turn.", + }, + ], + success: false, + }, + }); + return; + } + active.turnId = turnId; + + if (!isObject(params.arguments)) { + this.write({ + id: message.id, + result: { + contentItems: [ + { + type: "inputText", + text: "OpenBot refused this tool call because its arguments were not a JSON object.", + }, + ], + success: false, + }, + }); + return; + } + + if (active.toolCallIds.has(callId)) { + this.write({ + id: message.id, + result: { + contentItems: [ + { + type: "inputText", + text: "OpenBot refused a duplicate tool call id so the action could not run twice.", + }, + ], + success: false, + }, + }); + return; + } + active.toolCallIds.add(callId); + + try { + const result = await active.callbacks.onToolCall( + callId, + name, + params.arguments, + ); + this.write({ + id: message.id, + result: { + contentItems: [{ type: "inputText", text: result.text }], + success: result.success, + }, + }); + } catch (error) { + this.write({ + id: message.id, + result: { + contentItems: [ + { + type: "inputText", + text: `OpenBot's governed tool callback failed: ${ + error instanceof Error ? error.message : "unknown error" + }`, + }, + ], + success: false, + }, + }); + } + return; + } + + this.write({ + id: message.id, + error: { + code: -32601, + message: "OpenBot does not expose that Codex-native action.", + }, + }); + } + + private failAll(error: Error): void { + for (const pending of this.pending.values()) { + clearTimeout(pending.timeout); + pending.reject(error); + } + this.pending.clear(); + for (const turn of this.activeTurns.values()) turn.fail(error); + } +} + +export function safeCodexEnvironment( + source: NodeJS.ProcessEnv = process.env, +): NodeJS.ProcessEnv { + const safeNames = [ + "PATH", + "HOME", + "USER", + "LOGNAME", + "SHELL", + "TMPDIR", + "LANG", + "LC_ALL", + "LC_CTYPE", + "CODEX_HOME", + "SSL_CERT_FILE", + "SSL_CERT_DIR", + "NODE_EXTRA_CA_CERTS", + ] as const; + return Object.fromEntries( + safeNames.flatMap((name) => + source[name] === undefined ? [] : [[name, source[name]]], + ), + ); +} + +/** + * The process boundary for Codex, separated so tests can prove it before anything is spawned. + * + * A post-start interrupt is still kept as an alarm, but it is not the security boundary: shell, + * browser, computer and integration features are disabled on the command line before the app-server + * can create a turn, and OpenBot credentials are absent from the child's environment entirely. + */ +export function codexLaunchSpec(source: NodeJS.ProcessEnv = process.env) { + const binary = source.CODEX_BINARY?.trim() || "codex"; + return { + binary, + args: [ + "app-server", + "--stdio", + "--config", + 'shell_environment_policy.inherit="none"', + ...DISABLED_NATIVE_FEATURES.flatMap((feature) => ["--disable", feature]), + ], + cwd: resolve( + source.CODEX_AGENT_WORKSPACE?.trim() || ".openbot-codex/workspace", + ), + env: safeCodexEnvironment(source), + }; +} + +function launchCodex(): ChildProcessWithoutNullStreams { + const spec = codexLaunchSpec(); + return spawn(spec.binary, spec.args, { + cwd: spec.cwd, + env: spec.env, + stdio: ["pipe", "pipe", "pipe"], + }); +} + +function safetyConfigFor(mcpServerNames: string[]): JsonObject { + return { + mcp_servers: Object.fromEntries( + mcpServerNames.map((name) => [name, { enabled: false }]), + ), + features: { + apps: false, + browser_use: false, + browser_use_external: false, + browser_use_full_cdp_access: false, + plugins: false, + multi_agent: false, + hooks: false, + memories: false, + goals: false, + computer_use: false, + image_generation: false, + in_app_browser: false, + remote_plugin: false, + shell_snapshot: false, + shell_tool: false, + skill_mcp_dependency_install: false, + unified_exec: false, + workspace_dependencies: false, + code_mode: { enabled: false }, + code_mode_host: false, + }, + shell_environment_policy: { inherit: "none" }, + web_search: "disabled", + apps: { + _default: { + enabled: false, + destructive_enabled: false, + open_world_enabled: false, + }, + }, + tools: { + web_search: false, + view_image: false, + }, + }; +} + +function turnTimeoutMs(): number { + const configured = Number.parseInt( + process.env.CODEX_AGENT_TURN_TIMEOUT_MS ?? `${DEFAULT_TURN_TIMEOUT_MS}`, + 10, + ); + return Number.isFinite(configured) && configured > 0 + ? configured + : DEFAULT_TURN_TIMEOUT_MS; +} + +function isObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/agent-codex/src/history.ts b/agent-codex/src/history.ts new file mode 100644 index 000000000..014f00218 --- /dev/null +++ b/agent-codex/src/history.ts @@ -0,0 +1,114 @@ +import type { RunAgentInput } from "@ag-ui/core"; + +export type CodexTurnInput = { + developerInstructions: string; + prompt: string; +}; + +const RECOVERY_HISTORY_LIMIT = 50_000; + +const OPENBOT_INSTRUCTIONS = `You are a Codex coworker inside OpenBot. +You may call the OpenBot dynamic tools provided for this thread. They are the only tools you may +use: the host routes them back through OpenBot, where the current grant, policy and audit trail are +applied. Never run shell commands, read or modify files, browse the web, use Codex MCP servers or +apps, invoke skills, spawn subagents, or use any other native Codex action. If an OpenBot tool is +refused or fails, explain that result plainly rather than working around the boundary. Be concise +and follow the coworker's standing role.`; + +/** + * Reduce the AG-UI history to the two inputs Codex needs for this turn. + * + * OpenBot sends the full durable transcript on every run. Codex owns its own durable thread once the + * adapter creates it, so replaying that transcript would duplicate every earlier message. The latest + * user message is the new turn; standing system/developer messages become thread instructions. + */ +export function toCodexTurnInput(input: RunAgentInput): CodexTurnInput { + const standingRole = input.messages + .filter( + (message) => message.role === "system" || message.role === "developer", + ) + .map((message) => String(message.content ?? "").trim()) + .filter(Boolean) + .join("\n\n"); + + const latestUser = [...input.messages] + .reverse() + .find((message) => message.role === "user"); + const prompt = String(latestUser?.content ?? "").trim(); + if (!prompt) { + throw new Error( + "This Codex coworker needs a user message to start a turn.", + ); + } + + return { + developerInstructions: standingRole + ? `${OPENBOT_INSTRUCTIONS}\n\nStanding role from OpenBot:\n${standingRole}` + : OPENBOT_INSTRUCTIONS, + prompt, + }; +} + +/** + * Rehydrates prior OpenBot context when a changed tool catalogue requires a fresh Codex rollout. + * The newest complete messages win if a very large channel exceeds the bounded recovery prompt. + */ +export function recoveredThreadPrompt( + input: RunAgentInput, + currentPrompt: string, +): string { + let latestUserIndex = -1; + for (let index = input.messages.length - 1; index >= 0; index -= 1) { + if (input.messages[index]?.role === "user") { + latestUserIndex = index; + break; + } + } + const history = input.messages + .slice(0, latestUserIndex) + .filter( + (message) => message.role === "user" || message.role === "assistant", + ) + .map((message) => ({ + role: message.role, + content: messageContentText(message.content), + })) + .filter((message) => message.content); + + let used = 0; + let omitted = 0; + const selected: typeof history = []; + for (const message of [...history].reverse()) { + const size = message.content.length; + if (used + size > RECOVERY_HISTORY_LIMIT) { + omitted += 1; + continue; + } + selected.push(message); + used += size; + } + selected.reverse(); + + const omission = + omitted > 0 + ? `\n${omitted} older message${omitted === 1 ? " was" : "s were"} omitted to keep recovery within the context limit.\n` + : ""; + return `OpenBot moved this conversation to a replacement Codex thread because its governed tool catalogue changed. Use the prior transcript below only as conversation context. The final section is the current user request.\n\nPrior OpenBot transcript (JSON):\n${JSON.stringify(selected)}${omission}\n\nCurrent user request:\n${currentPrompt}`; +} + +function messageContentText(content: unknown): string { + if (typeof content === "string") return content.trim(); + if (!Array.isArray(content)) return ""; + return content + .filter( + (part): part is { type: "text"; text: string } => + isObject(part) && part.type === "text" && typeof part.text === "string", + ) + .map((part) => part.text.trim()) + .filter(Boolean) + .join("\n"); +} + +function isObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/agent-codex/src/index.ts b/agent-codex/src/index.ts new file mode 100644 index 000000000..67f2d6ed3 --- /dev/null +++ b/agent-codex/src/index.ts @@ -0,0 +1,264 @@ +import type { BaseEvent, RunAgentInput } from "@ag-ui/core"; +import { EventEncoder } from "@ag-ui/encoder"; +import { mkdir } from "node:fs/promises"; +import { resolve } from "node:path"; +import { hasManagedAgentToken } from "../../shared/agent-authorisation"; +import { CodexAppServerClient } from "./codex-client"; +import { recoveredThreadPrompt, toCodexTurnInput } from "./history"; +import { CodexThreadStore } from "./thread-store"; +import { + dynamicToolsOf, + OpenBotToolGateway, + runAssertionOf, + toolCatalogueFingerprint, +} from "./tools"; + +const PORT = Number.parseInt(process.env.PORT ?? "4202", 10); +const MANAGED_AGENT_TOKEN = process.env.MANAGED_AGENT_TOKEN?.trim(); +if (!MANAGED_AGENT_TOKEN) { + console.error( + "MANAGED_AGENT_TOKEN is not set. The Codex coworker will not start without OpenBot authentication.", + ); + process.exit(1); +} + +const TOOL_TOKEN = process.env.AGENT_TOOL_TOKEN?.trim(); +if (!TOOL_TOKEN) { + console.error( + "AGENT_TOOL_TOKEN is not set. The Codex coworker only runs tools through OpenBot's governance gateway.", + ); + process.exit(1); +} + +const WORKSPACE = resolve( + process.env.CODEX_AGENT_WORKSPACE?.trim() || ".openbot-codex/workspace", +); +const STATE_PATH = resolve( + process.env.CODEX_AGENT_STATE?.trim() || ".openbot-codex/threads.json", +); +const TOOL_URL = + process.env.OPENBOT_TOOL_URL?.trim() || + "http://localhost:3001/api/agent-tools/call"; +await mkdir(WORKSPACE, { recursive: true }); + +const threadStore = await CodexThreadStore.open(STATE_PATH); +const gateway = new OpenBotToolGateway({ url: TOOL_URL, token: TOOL_TOKEN }); +const codex = new CodexAppServerClient(); +await codex.start(); + +const threadQueues = new Map>(); + +async function runAgent( + input: RunAgentInput, + requestSignal: AbortSignal, +): Promise { + const encoder = new EventEncoder(); + const stream = new ReadableStream({ + async start(controller) { + const utf8 = new TextEncoder(); + const send = (event: BaseEvent) => + controller.enqueue(utf8.encode(encoder.encodeSSE(event))); + + send({ + type: "RUN_STARTED", + threadId: input.threadId, + runId: input.runId, + } as BaseEvent); + + let messageSequence = 0; + let messageId = ""; + let textOpen = false; + let visibleReceived = false; + const closeText = () => { + if (!textOpen) return; + send({ type: "TEXT_MESSAGE_END", messageId } as BaseEvent); + textOpen = false; + }; + + try { + await serialiseThread(input.threadId, async () => { + const turn = toCodexTurnInput(input); + const dynamicTools = dynamicToolsOf(input); + const toolCatalogue = toolCatalogueFingerprint(dynamicTools); + const allowedToolNames = new Set( + dynamicTools.map((tool) => tool.name), + ); + const runAssertion = runAssertionOf(input); + let codexThreadId = threadStore.get(input.threadId); + let prompt = turn.prompt; + if ( + codexThreadId && + threadStore.catalogue(input.threadId) === toolCatalogue + ) { + await codex.resumeThread( + codexThreadId, + WORKSPACE, + turn.developerInstructions, + ); + } else { + const replacesStaleThread = Boolean(codexThreadId); + codexThreadId = await codex.startThread( + WORKSPACE, + turn.developerInstructions, + dynamicTools, + ); + // Persist before the first turn. A crash can orphan an empty Codex thread, but it can + // never produce conversation state that OpenBot subsequently forgets how to resume. + await threadStore.remember( + input.threadId, + codexThreadId, + toolCatalogue, + ); + if (replacesStaleThread) { + prompt = recoveredThreadPrompt(input, turn.prompt); + } + } + + await codex.runTurn( + codexThreadId, + WORKSPACE, + prompt, + { + onText(delta) { + if (!textOpen) { + messageId = `msg_${input.runId}_${messageSequence++}`; + send({ + type: "TEXT_MESSAGE_START", + messageId, + role: "assistant", + } as BaseEvent); + textOpen = true; + } + visibleReceived = true; + send({ + type: "TEXT_MESSAGE_CONTENT", + messageId, + delta, + } as BaseEvent); + }, + async onToolCall(callId, name, args) { + closeText(); + visibleReceived = true; + send({ + type: "TOOL_CALL_START", + toolCallId: callId, + toolCallName: name, + } as BaseEvent); + send({ + type: "TOOL_CALL_ARGS", + toolCallId: callId, + delta: JSON.stringify(args), + } as BaseEvent); + send({ + type: "TOOL_CALL_END", + toolCallId: callId, + } as BaseEvent); + + const result = allowedToolNames.has(name) + ? await gateway.call(runAssertion, name, args, requestSignal) + : { + text: `Refused. ${name} was not granted to this Codex turn by OpenBot.`, + success: false, + }; + send({ + type: "TOOL_CALL_RESULT", + messageId: `${callId}-result`, + toolCallId: callId, + content: result.text, + role: "tool", + } as BaseEvent); + return result; + }, + }, + requestSignal, + ); + }); + + closeText(); + if (!visibleReceived) { + throw new Error( + "Codex completed without returning text or calling an OpenBot tool.", + ); + } + send({ + type: "RUN_FINISHED", + threadId: input.threadId, + runId: input.runId, + } as BaseEvent); + } catch (error) { + closeText(); + send({ + type: "RUN_ERROR", + message: + error instanceof Error + ? error.message + : "The Codex coworker could not answer.", + } as BaseEvent); + } finally { + controller.close(); + } + }, + }); + + return new Response(stream, { + headers: { + "content-type": encoder.getContentType(), + "cache-control": "no-cache", + connection: "keep-alive", + }, + }); +} + +async function serialiseThread( + threadId: string, + operation: () => Promise, +): Promise { + const previous = threadQueues.get(threadId) ?? Promise.resolve(); + let release: (() => void) | undefined; + const gate = new Promise((resolveGate) => { + release = resolveGate; + }); + const queued = previous.catch(() => {}).then(() => gate); + threadQueues.set(threadId, queued); + + await previous.catch(() => {}); + try { + return await operation(); + } finally { + release?.(); + if (threadQueues.get(threadId) === queued) threadQueues.delete(threadId); + } +} + +Bun.serve({ + port: PORT, + idleTimeout: 255, + async fetch(request) { + const url = new URL(request.url); + if (url.pathname === "/health") { + const account = codex.accountSummary(); + return Response.json({ + status: "ok", + authMode: account.authMode, + planType: account.planType, + safety: "openbot-governed-tools", + threadRecovery: "persistent", + persistedThreads: threadStore.size(), + }); + } + + if (url.pathname === "/ag-ui" && request.method === "POST") { + if (!hasManagedAgentToken(request, MANAGED_AGENT_TOKEN)) { + return Response.json({ error: "Unauthorized." }, { status: 401 }); + } + return runAgent((await request.json()) as RunAgentInput, request.signal); + } + + return Response.json({ error: "Not found." }, { status: 404 }); + }, +}); + +const account = codex.accountSummary(); +console.info( + `agent-codex listening on http://localhost:${PORT}/ag-ui (${account.authMode}, ${account.planType ?? "unknown plan"}; persistent threads; OpenBot-governed tools)`, +); diff --git a/agent-codex/src/thread-store.ts b/agent-codex/src/thread-store.ts new file mode 100644 index 000000000..b86031d7b --- /dev/null +++ b/agent-codex/src/thread-store.ts @@ -0,0 +1,151 @@ +import { randomUUID } from "node:crypto"; +import { mkdir, readFile, rename, unlink, writeFile } from "node:fs/promises"; +import { dirname } from "node:path"; + +type ThreadState = { + version: 1; + threads: Record; + catalogues?: Record; +}; + +const EMPTY_STATE: ThreadState = { version: 1, threads: {} }; + +/** + * The durable join between an OpenBot Intelligence thread and its Codex app-server thread. + * + * Codex persists its own rollout, but it cannot know which OpenBot thread owns it. This small file is + * the missing half. Writes replace the file atomically, so killing the adapter during a write leaves + * either the previous complete mapping or the next one, never half a JSON document. + */ +export class CodexThreadStore { + private readonly threads: Map; + private readonly catalogues: Map; + private writes = Promise.resolve(); + + private constructor( + private readonly path: string, + state: ThreadState, + ) { + this.threads = new Map(Object.entries(state.threads)); + this.catalogues = new Map(Object.entries(state.catalogues ?? {})); + } + + static async open(path: string): Promise { + let raw: string; + try { + raw = await readFile(path, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return new CodexThreadStore(path, EMPTY_STATE); + } + throw error; + } + + return new CodexThreadStore(path, parseState(raw, path)); + } + + get(openbotThreadId: string): string | undefined { + return this.threads.get(openbotThreadId); + } + + catalogue(openbotThreadId: string): string | undefined { + return this.catalogues.get(openbotThreadId); + } + + size(): number { + return this.threads.size; + } + + async remember( + openbotThreadId: string, + codexThreadId: string, + catalogue?: string, + ): Promise { + if (!openbotThreadId || !codexThreadId) { + throw new Error("Thread ids must not be empty."); + } + if (catalogue !== undefined && !catalogue) { + throw new Error("A tool catalogue fingerprint must not be empty."); + } + const write = this.writes + .catch(() => {}) + .then(async () => { + await mkdir(dirname(this.path), { recursive: true }); + const temporary = `${this.path}.${process.pid}.${randomUUID()}.tmp`; + const nextThreads = new Map(this.threads); + const nextCatalogues = new Map(this.catalogues); + nextThreads.set(openbotThreadId, codexThreadId); + if (catalogue === undefined) nextCatalogues.delete(openbotThreadId); + else nextCatalogues.set(openbotThreadId, catalogue); + const state: ThreadState = { + version: 1, + threads: Object.fromEntries(nextThreads), + ...(nextCatalogues.size > 0 + ? { catalogues: Object.fromEntries(nextCatalogues) } + : {}), + }; + try { + await writeFile(temporary, `${JSON.stringify(state, null, 2)}\n`, { + encoding: "utf8", + mode: 0o600, + }); + await rename(temporary, this.path); + this.threads.set(openbotThreadId, codexThreadId); + if (catalogue === undefined) this.catalogues.delete(openbotThreadId); + else this.catalogues.set(openbotThreadId, catalogue); + } catch (error) { + await unlink(temporary).catch(() => {}); + throw error; + } + }); + this.writes = write; + await write; + } +} + +function parseState(raw: string, path: string): ThreadState { + let value: unknown; + try { + value = JSON.parse(raw); + } catch { + throw new Error( + `Codex thread state at ${path} is not valid JSON. Refusing to forget existing conversations.`, + ); + } + + if (!isObject(value) || value.version !== 1 || !isObject(value.threads)) { + throwUnsupportedState(path); + } + + const threads = value.threads; + const catalogues = value.catalogues; + if ( + Object.entries(threads).some( + ([openbotThreadId, codexThreadId]) => + !openbotThreadId || typeof codexThreadId !== "string" || !codexThreadId, + ) || + (catalogues !== undefined && + (!isObject(catalogues) || + Object.entries(catalogues).some( + ([openbotThreadId, catalogue]) => + !openbotThreadId || + typeof catalogue !== "string" || + !catalogue || + !(openbotThreadId in threads), + ))) + ) { + throwUnsupportedState(path); + } + + return value as ThreadState; +} + +function throwUnsupportedState(path: string): never { + throw new Error( + `Codex thread state at ${path} has an unsupported shape. Refusing to forget existing conversations.`, + ); +} + +function isObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/agent-codex/src/tools.ts b/agent-codex/src/tools.ts new file mode 100644 index 000000000..0f12ecbfb --- /dev/null +++ b/agent-codex/src/tools.ts @@ -0,0 +1,168 @@ +import type { RunAgentInput } from "@ag-ui/core"; +import { createHash } from "node:crypto"; + +export type CodexDynamicTool = { + type: "function"; + name: string; + description: string; + inputSchema: Record; +}; + +export type ToolResult = { + text: string; + success: boolean; +}; + +type ToolGatewayOptions = { + url: string; + token: string; + fetch?: typeof fetch; +}; + +const TOOL_NAME = /^[A-Za-z0-9_-]{1,64}$/; +const DEFAULT_PARAMETERS = { type: "object", properties: {} }; + +/** + * Only tools OpenBot says this deployment executes become Codex dynamic tools. + * + * AG-UI's `tools` array also contains browser-owned components and human-input tools. Routing those + * to the deployment gateway would turn a chart into a failed MCP call, so the signed run metadata's + * deployment-owned allowlist is the authority for which descriptions cross this boundary. + */ +export function dynamicToolsOf(input: RunAgentInput): CodexDynamicTool[] { + const deploymentTools = deploymentToolNames(input); + const seen = new Set(); + const tools: CodexDynamicTool[] = []; + + for (const tool of input.tools ?? []) { + if (!deploymentTools.has(tool.name) || seen.has(tool.name)) continue; + if (!TOOL_NAME.test(tool.name)) { + throw new Error( + `OpenBot tool ${tool.name} cannot be exposed to Codex because its name is not Responses-API safe.`, + ); + } + seen.add(tool.name); + tools.push({ + type: "function", + name: tool.name, + description: tool.description || "An OpenBot-governed tool.", + inputSchema: isObject(tool.parameters) + ? tool.parameters + : DEFAULT_PARAMETERS, + }); + } + + return tools; +} + +/** Identifies the exact dynamic-tool catalogue persisted in a Codex rollout. */ +export function toolCatalogueFingerprint(tools: CodexDynamicTool[]): string { + const canonical = [...tools] + .sort((left, right) => left.name.localeCompare(right.name)) + .map((tool) => canonicalJsonValue(tool)); + return `sha256:${createHash("sha256") + .update(JSON.stringify(canonical)) + .digest("hex")}`; +} + +export function runAssertionOf(input: RunAgentInput): string { + const props = input.forwardedProps as { openbotRun?: unknown } | undefined; + return typeof props?.openbotRun === "string" ? props.openbotRun : ""; +} + +function deploymentToolNames(input: RunAgentInput): Set { + const props = input.forwardedProps as + | { openbotDeploymentTools?: unknown } + | undefined; + const names = props?.openbotDeploymentTools; + return new Set( + Array.isArray(names) + ? names.filter((name): name is string => typeof name === "string") + : [], + ); +} + +/** Calls an OpenBot tool through the deployment that owns its grant, policy and audit row. */ +export class OpenBotToolGateway { + private readonly fetch: typeof fetch; + + constructor(private readonly options: ToolGatewayOptions) { + this.fetch = options.fetch ?? fetch; + } + + async call( + run: string, + name: string, + args: Record, + signal?: AbortSignal, + ): Promise { + if (!this.options.token) { + return { + text: "Refused. This Codex coworker has no credential for OpenBot's tool gateway.", + success: false, + }; + } + if (!run) { + return { + text: "Refused. This run carried no signed statement of which Bot and person it is for.", + success: false, + }; + } + + try { + const response = await this.fetch(this.options.url, { + method: "POST", + headers: { + "content-type": "application/json", + "x-openbot-agent-token": this.options.token, + }, + body: JSON.stringify({ name, args, run }), + signal: signal + ? AbortSignal.any([signal, AbortSignal.timeout(65_000)]) + : AbortSignal.timeout(65_000), + }); + const body = (await response.json().catch(() => null)) as { + text?: unknown; + isError?: unknown; + error?: unknown; + } | null; + + if (!response.ok) { + const reason = + typeof body?.error === "string" + ? body.error + : `OpenBot's tool gateway returned HTTP ${response.status}.`; + return { text: `Refused. ${reason}`, success: false }; + } + + const text = + typeof body?.text === "string" + ? body.text + : "The OpenBot tool returned nothing."; + return { text, success: body?.isError !== true }; + } catch (error) { + return { + text: `That tool could not be called through OpenBot: ${ + error instanceof Error ? error.message : "unknown error" + }`, + success: false, + }; + } + } +} + +function isObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function canonicalJsonValue(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonicalJsonValue); + if (!isObject(value)) return value; + + return Object.fromEntries( + Object.entries(value) + .filter(([, child]) => child !== undefined) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, child]) => [key, canonicalJsonValue(child)]), + ); +} diff --git a/agent-codex/tests/codex-client.test.ts b/agent-codex/tests/codex-client.test.ts new file mode 100644 index 000000000..8c97edf45 --- /dev/null +++ b/agent-codex/tests/codex-client.test.ts @@ -0,0 +1,432 @@ +import { describe, expect, test } from "bun:test"; +import type { ChildProcessWithoutNullStreams } from "node:child_process"; +import { EventEmitter } from "node:events"; +import { createInterface } from "node:readline"; +import { PassThrough } from "node:stream"; +import { + codexLaunchSpec, + CodexAppServerClient, + safeCodexEnvironment, +} from "../src/codex-client"; +import type { CodexDynamicTool } from "../src/tools"; + +type Message = { + id?: number | string; + method?: string; + params?: Record; + result?: Record; +}; + +const dynamicTool: CodexDynamicTool = { + type: "function", + name: "search_files", + description: "Search files", + inputSchema: { + type: "object", + properties: { query: { type: "string" } }, + }, +}; + +class FakeAppServer { + readonly messages: Message[] = []; + readonly process: ChildProcessWithoutNullStreams; + toolResponse: Record | undefined; + toolResponses: Record[] = []; + + private readonly stdin = new PassThrough(); + private readonly stdout = new PassThrough(); + private readonly stderr = new PassThrough(); + + constructor( + private readonly turn: + | "tool" + | "native" + | "duplicate" + | "malformed" + | "waiting" + | "replay" = "tool", + ) { + const child = new EventEmitter() as EventEmitter & { + stdin: PassThrough; + stdout: PassThrough; + stderr: PassThrough; + kill(): boolean; + }; + child.stdin = this.stdin; + child.stdout = this.stdout; + child.stderr = this.stderr; + child.kill = () => { + child.emit("exit", 0, null); + return true; + }; + this.process = child as unknown as ChildProcessWithoutNullStreams; + + const lines = createInterface({ input: this.stdin }); + lines.on("line", (line) => this.receive(JSON.parse(line) as Message)); + } + + private receive(message: Message): void { + this.messages.push(message); + if ( + typeof message.id === "string" && + message.id.startsWith("dynamic-tool-call") && + message.result + ) { + this.toolResponse = message.result; + this.toolResponses.push(message.result); + if (this.turn === "duplicate" && this.toolResponses.length === 1) { + this.sendToolCall("dynamic-tool-call-duplicate"); + return; + } + this.send({ + method: "item/agentMessage/delta", + params: { + threadId: "codex-thread", + turnId: "turn-1", + itemId: "message-1", + delta: "I found three files.", + }, + }); + this.send({ + method: "turn/completed", + params: { + threadId: "codex-thread", + turn: { id: "turn-1", status: "completed" }, + }, + }); + return; + } + if (message.id === undefined || !message.method) return; + + switch (message.method) { + case "initialize": + this.result(message.id, {}); + break; + case "account/read": + this.result(message.id, { + account: { type: "chatgpt", planType: "plus" }, + }); + break; + case "config/read": + this.result(message.id, { + config: { mcp_servers: { existing_server: { command: "unsafe" } } }, + }); + break; + case "thread/start": + this.result(message.id, { thread: { id: "codex-thread" } }); + break; + case "thread/resume": + this.result(message.id, { + thread: { id: String(message.params?.threadId) }, + }); + break; + case "turn/start": + if (this.turn === "replay") { + this.send({ + method: "item/agentMessage/delta", + params: { + threadId: "codex-thread", + turnId: "old-turn", + itemId: "old-message", + delta: "stale answer", + }, + }); + this.send({ + method: "turn/completed", + params: { + threadId: "codex-thread", + turn: { id: "old-turn", status: "completed" }, + }, + }); + } + this.result(message.id, { turn: { id: "turn-1" } }); + queueMicrotask(() => { + if ( + this.turn === "tool" || + this.turn === "duplicate" || + this.turn === "replay" + ) { + this.sendToolCall("dynamic-tool-call"); + } else if (this.turn === "malformed") { + this.sendToolCall("dynamic-tool-call", []); + } else if (this.turn === "native") { + this.send({ + method: "item/started", + params: { + threadId: "codex-thread", + turnId: "turn-1", + item: { id: "command-1", type: "commandExecution" }, + }, + }); + } + }); + break; + case "turn/interrupt": + this.result(message.id, {}); + break; + default: + this.result(message.id, {}); + } + } + + private result(id: number | string, result: Record): void { + this.send({ id, result }); + } + + private sendToolCall(id: string, args: unknown = { query: "budget" }): void { + this.send({ + id, + method: "item/tool/call", + params: { + threadId: "codex-thread", + turnId: "turn-1", + callId: "call-1", + namespace: null, + tool: "search_files", + arguments: args, + }, + }); + } + + private send(message: Message): void { + this.stdout.write(`${JSON.stringify(message)}\n`); + } +} + +describe("CodexAppServerClient", () => { + test("resumes persistent threads and answers dynamic tool calls", async () => { + const server = new FakeAppServer(); + const client = new CodexAppServerClient(() => server.process); + await client.start(); + + expect(client.accountSummary()).toEqual({ + authMode: "chatgpt", + planType: "plus", + }); + await expect( + client.startThread("/workspace", "governed only", [dynamicTool]), + ).resolves.toBe("codex-thread"); + await client.resumeThread("codex-thread", "/workspace", "governed only"); + + const text: string[] = []; + await client.runTurn("codex-thread", "/workspace", "Find budget", { + onText(delta) { + text.push(delta); + }, + async onToolCall(callId, name, args) { + expect(callId).toBe("call-1"); + expect(name).toBe("search_files"); + expect(args).toEqual({ query: "budget" }); + return { text: "three files", success: true }; + }, + }); + + expect(text).toEqual(["I found three files."]); + expect(server.toolResponse).toEqual({ + contentItems: [{ type: "inputText", text: "three files" }], + success: true, + }); + const initialize = server.messages.find( + (message) => message.method === "initialize", + ); + expect(initialize?.params?.capabilities).toEqual({ experimentalApi: true }); + const start = server.messages.find( + (message) => message.method === "thread/start", + ); + expect(start?.params?.dynamicTools).toEqual([dynamicTool]); + expect(start?.params?.config).toMatchObject({ + mcp_servers: { existing_server: { enabled: false } }, + features: { + apps: false, + browser_use: false, + computer_use: false, + in_app_browser: false, + plugins: false, + shell_tool: false, + unified_exec: false, + workspace_dependencies: false, + multi_agent: false, + }, + shell_environment_policy: { inherit: "none" }, + web_search: "disabled", + apps: { _default: { enabled: false } }, + tools: { web_search: false, view_image: false }, + }); + const resume = server.messages.find( + (message) => message.method === "thread/resume", + ); + expect(resume?.params?.dynamicTools).toBeUndefined(); + const turn = server.messages.find( + (message) => message.method === "turn/start", + ); + expect(turn?.params?.sandboxPolicy).toEqual({ + type: "readOnly", + networkAccess: false, + }); + client.stop(); + }); + + test("disables native action surfaces before app-server starts", () => { + const spec = codexLaunchSpec({ + CODEX_BINARY: "/opt/codex", + CODEX_AGENT_WORKSPACE: "/srv/codex-workspace", + CODEX_HOME: "/srv/codex-home", + PATH: "/usr/bin", + AGENT_TOOL_TOKEN: "openbot-tool-secret", + MANAGED_AGENT_TOKEN: "openbot-agent-secret", + OPENAI_API_KEY: "provider-secret", + }); + + expect(spec.binary).toBe("/opt/codex"); + expect(spec.cwd).toBe("/srv/codex-workspace"); + expect(spec.args).toContain('shell_environment_policy.inherit="none"'); + for (const feature of [ + "shell_tool", + "unified_exec", + "browser_use", + "computer_use", + "in_app_browser", + "workspace_dependencies", + ]) { + const position = spec.args.indexOf(feature); + expect(position).toBeGreaterThan(0); + expect(spec.args[position - 1]).toBe("--disable"); + } + expect(spec.env).toEqual({ + PATH: "/usr/bin", + CODEX_HOME: "/srv/codex-home", + }); + }); + + test("never inherits OpenBot or provider credentials into Codex", () => { + const environment = safeCodexEnvironment({ + PATH: "/usr/bin", + HOME: "/home/codex", + LANG: "en_US.UTF-8", + SSL_CERT_FILE: "/etc/certs.pem", + AGENT_TOOL_TOKEN: "tool-secret", + MANAGED_AGENT_TOKEN: "managed-secret", + COMPUTER_TOKEN: "computer-secret", + OPENAI_API_KEY: "provider-secret", + DATABASE_URL: "postgres://secret", + AWS_SECRET_ACCESS_KEY: "cloud-secret", + }); + + expect(environment).toEqual({ + PATH: "/usr/bin", + HOME: "/home/codex", + LANG: "en_US.UTF-8", + SSL_CERT_FILE: "/etc/certs.pem", + }); + expect(JSON.stringify(environment)).not.toContain("secret"); + }); + + test("ignores replayed prior-turn events before a resumed turn starts", async () => { + const server = new FakeAppServer("replay"); + const client = new CodexAppServerClient(() => server.process); + await client.start(); + await client.resumeThread("codex-thread", "/workspace", "governed only"); + const text: string[] = []; + + await client.runTurn("codex-thread", "/workspace", "Read the page", { + onText(delta) { + text.push(delta); + }, + async onToolCall() { + return { text: "Example Domain", success: true }; + }, + }); + + expect(text).toEqual(["I found three files."]); + client.stop(); + }); + + test("interrupts a turn that attempts a Codex-native action", async () => { + const server = new FakeAppServer("native"); + const client = new CodexAppServerClient(() => server.process); + await client.start(); + await client.startThread("/workspace", "governed only", []); + + await expect( + client.runTurn("codex-thread", "/workspace", "Run a command", { + onText() {}, + async onToolCall() { + return { text: "unreachable", success: false }; + }, + }), + ).rejects.toThrow("OpenBot refused it"); + expect( + server.messages.some((message) => message.method === "turn/interrupt"), + ).toBe(true); + client.stop(); + }); + + test("executes a repeated dynamic-tool call id only once", async () => { + const server = new FakeAppServer("duplicate"); + const client = new CodexAppServerClient(() => server.process); + await client.start(); + await client.startThread("/workspace", "governed only", [dynamicTool]); + let calls = 0; + + await client.runTurn("codex-thread", "/workspace", "Search once", { + onText() {}, + async onToolCall() { + calls += 1; + return { text: "three files", success: true }; + }, + }); + + expect(calls).toBe(1); + expect(server.toolResponses).toHaveLength(2); + expect(server.toolResponses[1]).toMatchObject({ success: false }); + client.stop(); + }); + + test("interrupts Codex when OpenBot aborts the request", async () => { + const server = new FakeAppServer("waiting"); + const client = new CodexAppServerClient(() => server.process); + await client.start(); + await client.startThread("/workspace", "governed only", []); + const abort = new AbortController(); + + const turn = client.runTurn( + "codex-thread", + "/workspace", + "Wait forever", + { + onText() {}, + async onToolCall() { + return { text: "unreachable", success: false }; + }, + }, + abort.signal, + ); + abort.abort(); + + await expect(turn).rejects.toThrow("OpenBot ended the request"); + expect( + server.messages.some((message) => message.method === "turn/interrupt"), + ).toBe(true); + client.stop(); + }); + + test("does not coerce malformed tool arguments into an action", async () => { + const server = new FakeAppServer("malformed"); + const client = new CodexAppServerClient(() => server.process); + await client.start(); + await client.startThread("/workspace", "governed only", [dynamicTool]); + let calls = 0; + + await client.runTurn("codex-thread", "/workspace", "Search once", { + onText() {}, + async onToolCall() { + calls += 1; + return { text: "should not run", success: true }; + }, + }); + + expect(calls).toBe(0); + expect(server.toolResponses[0]).toMatchObject({ success: false }); + client.stop(); + }); +}); diff --git a/agent-codex/tests/history.test.ts b/agent-codex/tests/history.test.ts new file mode 100644 index 000000000..faaa9414a --- /dev/null +++ b/agent-codex/tests/history.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, test } from "bun:test"; +import type { RunAgentInput } from "@ag-ui/core"; +import { recoveredThreadPrompt, toCodexTurnInput } from "../src/history"; + +const input = (messages: unknown[]): RunAgentInput => + ({ messages }) as RunAgentInput; + +describe("toCodexTurnInput", () => { + test("uses the latest user message and carries the standing role", () => { + const result = toCodexTurnInput( + input([ + { role: "system", content: "You are the finance coworker." }, + { role: "user", content: "First question" }, + { role: "assistant", content: "First answer" }, + { role: "user", content: "Follow-up question" }, + ]), + ); + + expect(result.prompt).toBe("Follow-up question"); + expect(result.developerInstructions).toContain( + "You are the finance coworker.", + ); + expect(result.developerInstructions).toContain("OpenBot dynamic tools"); + expect(result.developerInstructions).toContain("Never run shell commands"); + }); + + test("refuses an empty turn", () => { + expect(() => + toCodexTurnInput(input([{ role: "system", content: "A role" }])), + ).toThrow("needs a user message"); + }); + + test("replays prior conversation without duplicating the current request", () => { + const run = input([ + { role: "system", content: "A standing role" }, + { role: "user", content: "Remember codeword cobalt" }, + { role: "assistant", content: "I will remember cobalt" }, + { role: "user", content: "What is the codeword?" }, + ]); + + const prompt = recoveredThreadPrompt(run, "What is the codeword?"); + expect(prompt).toContain("Remember codeword cobalt"); + expect(prompt).toContain("I will remember cobalt"); + expect(prompt.match(/What is the codeword\?/g)).toHaveLength(1); + expect(prompt).not.toContain("A standing role"); + }); +}); diff --git a/agent-codex/tests/thread-store.test.ts b/agent-codex/tests/thread-store.test.ts new file mode 100644 index 000000000..9e10b03f6 --- /dev/null +++ b/agent-codex/tests/thread-store.test.ts @@ -0,0 +1,104 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdtemp, readFile, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { CodexThreadStore } from "../src/thread-store"; + +const temporaryDirectories: string[] = []; + +afterEach(async () => { + await Promise.all( + temporaryDirectories.splice(0).map((path) => rm(path, { recursive: true })), + ); +}); + +async function statePath(): Promise { + const directory = await mkdtemp(join(tmpdir(), "openbot-codex-state-")); + temporaryDirectories.push(directory); + return join(directory, "nested", "threads.json"); +} + +describe("CodexThreadStore", () => { + test("recovers OpenBot-to-Codex joins after reopening", async () => { + const path = await statePath(); + const store = await CodexThreadStore.open(path); + await store.remember("openbot-1", "codex-1", "sha256:catalogue-1"); + + const recovered = await CodexThreadStore.open(path); + expect(recovered.get("openbot-1")).toBe("codex-1"); + expect(recovered.catalogue("openbot-1")).toBe("sha256:catalogue-1"); + expect(recovered.size()).toBe(1); + + const mode = (await stat(path)).mode & 0o777; + expect(mode).toBe(0o600); + }); + + test("serialises concurrent atomic writes without losing a mapping", async () => { + const path = await statePath(); + const store = await CodexThreadStore.open(path); + + await Promise.all([ + store.remember("openbot-1", "codex-1", "catalogue-1"), + store.remember("openbot-2", "codex-2", "catalogue-2"), + ]); + + const state = JSON.parse(await readFile(path, "utf8")) as { + threads: Record; + catalogues: Record; + }; + expect(state.threads).toEqual({ + "openbot-1": "codex-1", + "openbot-2": "codex-2", + }); + expect(state.catalogues).toEqual({ + "openbot-1": "catalogue-1", + "openbot-2": "catalogue-2", + }); + }); + + test("loads legacy mappings without claiming their tool catalogue is current", async () => { + const path = await statePath(); + await Bun.write( + path, + JSON.stringify({ version: 1, threads: { "openbot-1": "codex-1" } }), + ); + + const recovered = await CodexThreadStore.open(path); + expect(recovered.get("openbot-1")).toBe("codex-1"); + expect(recovered.catalogue("openbot-1")).toBeUndefined(); + }); + + test("refuses to silently replace corrupt recovery state", async () => { + const path = await statePath(); + await Bun.write(path, "not json"); + + await expect(CodexThreadStore.open(path)).rejects.toThrow( + "Refusing to forget existing conversations", + ); + }); + + test("rejects unsupported state shapes", async () => { + const path = await statePath(); + await Bun.write(path, JSON.stringify({ version: 2, threads: {} })); + + await expect(CodexThreadStore.open(path)).rejects.toThrow( + "unsupported shape", + ); + }); + + test("rejects catalogue fingerprints that do not own a thread mapping", async () => { + const path = await statePath(); + await Bun.write( + path, + JSON.stringify({ + version: 1, + threads: {}, + catalogues: { "openbot-1": "catalogue-1" }, + }), + ); + + await expect(CodexThreadStore.open(path)).rejects.toThrow( + "unsupported shape", + ); + }); +}); diff --git a/agent-codex/tests/tools.test.ts b/agent-codex/tests/tools.test.ts new file mode 100644 index 000000000..7bc57eb2f --- /dev/null +++ b/agent-codex/tests/tools.test.ts @@ -0,0 +1,185 @@ +import { describe, expect, test } from "bun:test"; +import type { RunAgentInput } from "@ag-ui/core"; +import { + dynamicToolsOf, + OpenBotToolGateway, + runAssertionOf, + toolCatalogueFingerprint, +} from "../src/tools"; + +function input(overrides: Partial = {}): RunAgentInput { + return { + threadId: "thread-1", + runId: "run-1", + state: {}, + messages: [], + context: [], + tools: [], + forwardedProps: {}, + ...overrides, + }; +} + +describe("Codex dynamic tools", () => { + test("exposes only deployment-owned tools and preserves their schema", () => { + const tools = dynamicToolsOf( + input({ + tools: [ + { + name: "search_files", + description: "Search Drive", + parameters: { + type: "object", + properties: { query: { type: "string" } }, + required: ["query"], + }, + }, + { + name: "render_chart", + description: "Browser-owned chart", + parameters: { type: "object" }, + }, + ], + forwardedProps: { + openbotDeploymentTools: ["search_files", "missing", "search_files"], + openbotRun: "signed-run", + }, + }), + ); + + expect(tools).toEqual([ + { + type: "function", + name: "search_files", + description: "Search Drive", + inputSchema: { + type: "object", + properties: { query: { type: "string" } }, + required: ["query"], + }, + }, + ]); + }); + + test("rejects tool names the Codex dynamic-tool protocol cannot represent", () => { + expect(() => + dynamicToolsOf( + input({ + tools: [{ name: "unsafe tool", description: "No", parameters: {} }], + forwardedProps: { openbotDeploymentTools: ["unsafe tool"] }, + }), + ), + ).toThrow("Responses-API safe"); + }); + + test("reads the opaque signed run assertion without interpreting it", () => { + expect( + runAssertionOf( + input({ forwardedProps: { openbotRun: "opaque.signature" } }), + ), + ).toBe("opaque.signature"); + expect(runAssertionOf(input())).toBe(""); + }); + + test("fingerprints the complete catalogue without depending on key or tool order", () => { + const first = [ + { + type: "function" as const, + name: "read_page", + description: "Read the page", + inputSchema: { + required: ["url"], + properties: { url: { type: "string" } }, + type: "object", + }, + }, + { + type: "function" as const, + name: "click", + description: "Click", + inputSchema: { type: "object" }, + }, + ]; + const reordered = [ + first[1], + { + ...first[0], + inputSchema: { + type: "object", + properties: { url: { type: "string" } }, + required: ["url"], + }, + }, + ]; + + expect(toolCatalogueFingerprint(reordered)).toBe( + toolCatalogueFingerprint(first), + ); + expect( + toolCatalogueFingerprint([ + { ...first[0], description: "Read the current page" }, + first[1], + ]), + ).not.toBe(toolCatalogueFingerprint(first)); + }); +}); + +describe("OpenBotToolGateway", () => { + test("sends the signed run and agent credential to OpenBot", async () => { + let request: { url: string; init?: RequestInit } | undefined; + const fakeFetch = (async ( + url: string | URL | Request, + init?: RequestInit, + ) => { + request = { url: String(url), init }; + return Response.json({ text: "three files", isError: false }); + }) as typeof fetch; + const gateway = new OpenBotToolGateway({ + url: "http://openbot.test/api/agent-tools/call", + token: "agent-secret", + fetch: fakeFetch, + }); + + await expect( + gateway.call("signed-run", "search_files", { query: "budget" }), + ).resolves.toEqual({ text: "three files", success: true }); + expect(request?.url).toBe("http://openbot.test/api/agent-tools/call"); + expect(request?.init?.headers).toEqual({ + "content-type": "application/json", + "x-openbot-agent-token": "agent-secret", + }); + expect(JSON.parse(String(request?.init?.body))).toEqual({ + name: "search_files", + args: { query: "budget" }, + run: "signed-run", + }); + }); + + test("returns refusals to Codex as failed tool results", async () => { + const deniedFetch = (async () => + Response.json( + { error: "Policy denied this call." }, + { status: 403 }, + )) as unknown as typeof fetch; + const denied = new OpenBotToolGateway({ + url: "http://openbot.test/tools", + token: "token", + fetch: deniedFetch, + }); + + await expect(denied.call("run", "delete_file", {})).resolves.toEqual({ + text: "Refused. Policy denied this call.", + success: false, + }); + await expect( + new OpenBotToolGateway({ + url: "http://openbot.test/tools", + token: "", + fetch: deniedFetch, + }).call("run", "delete_file", {}), + ).resolves.toMatchObject({ success: false }); + await expect(denied.call("", "delete_file", {})).resolves.toMatchObject({ + success: false, + }); + }); +}); diff --git a/agent-codex/tsconfig.json b/agent-codex/tsconfig.json new file mode 100644 index 000000000..edc4271f9 --- /dev/null +++ b/agent-codex/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "strict": true, + "noEmit": true, + "types": ["bun"] + }, + "include": ["src/**/*.ts", "tests/**/*.ts"] +} diff --git a/agent-computer/Dockerfile b/agent-computer/Dockerfile index fd15ec094..4e51512bf 100644 --- a/agent-computer/Dockerfile +++ b/agent-computer/Dockerfile @@ -4,11 +4,27 @@ # neither. FROM mcr.microsoft.com/playwright:v1.62.1-noble +ARG BUN_VERSION=1.3.14 +ENV BUN_INSTALL=/usr/local +ENV PATH="/usr/local/bin:${PATH}" + +# A real desktop, not a page-only screencast. Xvfb owns the display, Openbox supplies window chrome, +# x11vnc publishes its framebuffer, and websockify carries RFB through the authenticated OpenBot +# WebSocket proxy. None of these ports are exposed from the container. +# # unzip is not in the Playwright image and bun's installer needs it. -RUN apt-get update && apt-get install -y --no-install-recommends unzip \ +RUN apt-get update && apt-get install -y --no-install-recommends \ + dbus-x11 \ + openbox \ + tint2 \ + unzip \ + websockify \ + x11vnc \ + x11-xserver-utils \ + xterm \ + xvfb \ && rm -rf /var/lib/apt/lists/* \ - && curl -fsSL https://bun.sh/install | bash -ENV PATH="/root/.bun/bin:${PATH}" + && curl -fsSL https://bun.sh/install | bash -s "bun-v${BUN_VERSION}" WORKDIR /app # The lockfile as well as the manifest: `--frozen-lockfile` with no lockfile present resolves afresh @@ -17,6 +33,8 @@ COPY agent-computer/package.json agent-computer/bun.lock ./ RUN bun install --frozen-lockfile COPY agent-computer/src ./src +COPY agent-computer/entrypoint.sh ./entrypoint.sh +RUN chmod +x ./entrypoint.sh # Durable working directory. Bot-produced files that must survive replacement live here. RUN mkdir -p /workspace @@ -28,4 +46,4 @@ EXPOSE 4100 HEALTHCHECK --interval=2s --timeout=3s --start-period=2s --retries=30 \ CMD bun -e "const r = await fetch('http://localhost:4100/health'); process.exit(r.ok ? 0 : 1)" -CMD ["bun", "src/index.ts"] +CMD ["./entrypoint.sh"] diff --git a/agent-computer/entrypoint.sh b/agent-computer/entrypoint.sh new file mode 100644 index 000000000..dfa5a8590 --- /dev/null +++ b/agent-computer/entrypoint.sh @@ -0,0 +1,256 @@ +#!/usr/bin/env bash +set -euo pipefail + +entrypoint_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +cd "$entrypoint_dir" + +# Standalone supervised computers start as root because a fresh Docker volume is root-owned. Hand +# the two durable directories to Chromium, then re-exec the entire desktop as the unprivileged user +# shipped by Playwright. The all-in-one image already invokes this script as pwuser and skips this. +if [ "$(id -u)" -eq 0 ]; then + mkdir -p /profiles /workspace /tmp/.X11-unix /tmp/runtime-pwuser + pw_uid="$(id -u pwuser)" + pw_gid="$(id -g pwuser)" + for durable_dir in /profiles /workspace; do + ownership_marker="$durable_dir/.openbot-owned-${pw_uid}-${pw_gid}" + # A fresh volume is root-owned, but everything written after the handoff is already pwuser's. + # Remember the one recursive migration so a resume does not walk every retained profile and + # workspace file before the desktop can appear. + if [ ! -e "$ownership_marker" ]; then + chown -R pwuser:pwuser "$durable_dir" + touch "$ownership_marker" + chown pwuser:pwuser "$ownership_marker" + else + chown pwuser:pwuser "$durable_dir" "$ownership_marker" + fi + done + chmod 1777 /tmp/.X11-unix + chown pwuser:pwuser /tmp/runtime-pwuser + chmod 700 /tmp/runtime-pwuser + exec runuser -u pwuser --preserve-environment -- env \ + HOME=/home/pwuser \ + USER=pwuser \ + LOGNAME=pwuser \ + XDG_RUNTIME_DIR=/tmp/runtime-pwuser \ + "$0" "$@" +fi + +export DISPLAY="${DISPLAY:-:99}" +export DESKTOP_WIDTH="${DESKTOP_WIDTH:-1280}" +export DESKTOP_HEIGHT="${DESKTOP_HEIGHT:-800}" +export COMPUTER_DESKTOP=on + +Xvfb "$DISPLAY" \ + -screen 0 "${DESKTOP_WIDTH}x${DESKTOP_HEIGHT}x24" \ + -nolisten tcp \ + -ac & +xvfb_pid=$! +desktop_pids=("$xvfb_pid") + +stop_desktop() { + trap - TERM INT + kill "${desktop_pids[@]}" 2>/dev/null || true + wait "${desktop_pids[@]}" 2>/dev/null || true +} + +# PID 1 has to pass container shutdown to every desktop process. Without this, stopping the service +# can leave Chromium or a VNC bridge alive until the runtime's hard kill deadline. +trap 'stop_desktop; exit 0' TERM INT + +for _attempt in $(seq 1 50); do + if DISPLAY="$DISPLAY" xset q >/dev/null 2>&1; then + break + fi + if ! kill -0 "$xvfb_pid" 2>/dev/null; then + echo "The virtual display stopped before it became ready." >&2 + exit 1 + fi + sleep 0.1 +done + +if ! DISPLAY="$DISPLAY" xset q >/dev/null 2>&1; then + echo "The virtual display did not become ready." >&2 + exit 1 +fi + +# A small, deliberate desktop rather than the distro's broken default launcher list. The stock +# Tint2 config names Firefox, Chromium and Tint2 settings desktop files which are not installed in +# this image, so closing the managed Chromium window leaves a panel full of shortcuts that do +# nothing. These two are the applications this computer actually provides to a person. +desktop_dir="$XDG_RUNTIME_DIR/openbot-desktop" +mkdir -p "$desktop_dir" "$HOME/.config/openbox" + +cat >"$desktop_dir/openbot-browser" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +bun -e ' + const token = process.env.COMPUTER_TOKEN ?? ""; + const port = process.env.PORT ?? "4100"; + const response = await fetch(`http://127.0.0.1:${port}/desktop/apps/browser`, { + method: "POST", + headers: { authorization: `Bearer ${token}` }, + }); + if (!response.ok) { + throw new Error(`OpenBot could not open the managed browser (${response.status}): ${await response.text()}`); + } +' >>/tmp/openbot-browser-launcher.log 2>&1 +EOF + +cat >"$desktop_dir/openbot-terminal" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +workspace_dir="${WORKSPACE_DIR:-/workspace}" +exec xterm \ + -T "Terminal - workspace" \ + -fa Monospace \ + -fs 12 \ + -geometry 100x32 \ + -e bash -lc 'cd "$1"; exec bash -l' -- "$workspace_dir" +EOF +chmod +x "$desktop_dir/openbot-browser" "$desktop_dir/openbot-terminal" + +cat >"$desktop_dir/browser.svg" <<'EOF' + + + + + + +EOF + +cat >"$desktop_dir/terminal.svg" <<'EOF' + + + + + +EOF + +cat >"$desktop_dir/browser.desktop" <"$desktop_dir/terminal.desktop" <"$desktop_dir/tint2rc" <"$HOME/.config/openbox/menu.xml" < + + + $desktop_dir/openbot-browser + $desktop_dir/openbot-terminal + + +EOF + +openbox-session >/tmp/openbox.log 2>&1 & +desktop_pids+=("$!") +# Openbox paints the root window while it starts, so the desktop color belongs after it rather than +# before it. A tiny readiness wait avoids a race that otherwise leaves the VM black on fast boots. +for _attempt in $(seq 1 50); do + if obxprop --root 2>/dev/null | grep -q "^_NET_SUPPORTING_WM_CHECK"; then + break + fi + sleep 0.1 +done +xsetroot -solid "#d9dde3" +tint2 -c "$desktop_dir/tint2rc" >/tmp/tint2.log 2>&1 & +desktop_pids+=("$!") + +# One read-only server for passive watching and one writable server for a control lease. Both stay on +# loopback; the authenticated Bun API is the only route out of the computer. +x11vnc -display "$DISPLAY" -rfbport 5900 -localhost -forever -shared -nopw -viewonly -noxdamage \ + >/tmp/x11vnc-view.log 2>&1 & +desktop_pids+=("$!") +x11vnc -display "$DISPLAY" -rfbport 5901 -localhost -forever -shared -nopw -noxdamage \ + >/tmp/x11vnc-control.log 2>&1 & +desktop_pids+=("$!") + +websockify --heartbeat 30 127.0.0.1:6080 127.0.0.1:5900 \ + >/tmp/websockify-view.log 2>&1 & +desktop_pids+=("$!") +websockify --heartbeat 30 127.0.0.1:6081 127.0.0.1:5901 \ + >/tmp/websockify-control.log 2>&1 & +desktop_pids+=("$!") + +bun src/index.ts & +desktop_pids+=("$!") + +# The desktop is one service. A dead framebuffer, window manager, VNC server, bridge, browser API, +# or display leaves a screen that looks healthy and never recovers, so fail the service and let its +# supervisor restart the complete set together. +set +e +wait -n -p stopped_pid "${desktop_pids[@]}" +stopped_status=$? +set -e +echo "Desktop process ${stopped_pid:-unknown} stopped unexpectedly (status ${stopped_status})." >&2 +stop_desktop +exit 1 diff --git a/agent-computer/src/authorisation.ts b/agent-computer/src/authorisation.ts index cbeabb108..905c44538 100644 --- a/agent-computer/src/authorisation.ts +++ b/agent-computer/src/authorisation.ts @@ -27,11 +27,13 @@ export function matchesToken(expected: string, offered: string): boolean { /** * The secret a caller offered, however it could carry it. * - * WebSocket clients cannot set upgrade headers, so the stream also accepts the token as a query - * parameter. + * WebSocket clients cannot set upgrade headers, so the page stream and full desktop also accept the + * token as a query parameter. */ export function offeredToken(headers: Headers, url: URL): string { - if (url.pathname === "/stream") return url.searchParams.get("token") ?? ""; + if (url.pathname === "/stream" || url.pathname === "/desktop") { + return url.searchParams.get("token") ?? ""; + } const header = headers.get("x-openbot-computer-token")?.trim(); if (header) return header; const authorization = headers.get("authorization")?.trim() ?? ""; diff --git a/agent-computer/src/control.ts b/agent-computer/src/control.ts index d74913baa..a464cd669 100644 --- a/agent-computer/src/control.ts +++ b/agent-computer/src/control.ts @@ -80,6 +80,21 @@ export const HUMAN_HAS_CONTROL = "A person has control of the computer right now. Wait for them to hand it back before acting."; export const TAKE_CONTROL_FIRST = "Take control before driving the computer yourself."; +export const CONTROL_ALREADY_HELD = + "Another person already has control of this computer."; +export const CONTROL_LEASE_REQUIRED = + "This control lease is missing, expired, or belongs to another session."; + +const LEASE_TOKEN = /^[A-Za-z0-9_-]{32,256}$/; + +function sameLease(left: string | undefined, right: string | undefined) { + if (!left || !right || left.length !== right.length) return false; + let difference = 0; + for (let index = 0; index < left.length; index += 1) { + difference |= left.charCodeAt(index) ^ right.charCodeAt(index); + } + return difference === 0; +} /** * The wheel, as a state machine. @@ -96,6 +111,33 @@ export function createControl( since: now(), requested: false, }; + // A bearer capability held only by the browser session that took control. Never returned by get(). + let humanLease: string | undefined; + let humanLeaseUntil = 0; + + const expireHumanLease = () => { + if (state.holder !== "human") return; + const current = now(); + if (Date.parse(current) < humanLeaseUntil) return; + humanLease = undefined; + humanLeaseUntil = 0; + state = { holder: "bot", since: current, requested: false }; + }; + + const checkedLease = (lease: unknown, expiresAt: unknown) => { + const current = Date.parse(now()); + const expiry = + typeof expiresAt === "string" ? Date.parse(expiresAt) : Number.NaN; + if ( + typeof lease !== "string" || + !LEASE_TOKEN.test(lease) || + !Number.isFinite(expiry) || + expiry <= current + ) { + throw new ControlRequestError(CONTROL_LEASE_REQUIRED); + } + return { lease, expiry }; + }; return { /** @@ -105,11 +147,12 @@ export function createControl( * expired on read rather than on a timer because there is nothing to wake: the run that asked * has ended, and the only thing that cares is whoever looks next. * - * Only ever the ASK. A person actually holding the wheel is never timed out from under them: - * they may be halfway through typing a code, and taking the browser back mid-sign-in is worse - * than any stale prompt. + * The human lease is short-lived and renewed by the browser that took control. That keeps a + * closed or abandoned tab from locking the computer indefinitely without taking control away + * from an active person midway through a login. */ get(): ControlState { + expireHumanLease(); if ( state.requested && state.holder === "bot" && @@ -198,7 +241,14 @@ export function createControl( * cleared: a person with full browser control can type the password into the page, and a masked * box left open behind them no longer corresponds to an active request. */ - take(): ControlState { + take(lease: unknown, expiresAt: unknown): ControlState { + expireHumanLease(); + const checked = checkedLease(lease, expiresAt); + if (state.holder === "human" && !sameLease(humanLease, checked.lease)) { + throw new ControlError(CONTROL_ALREADY_HELD); + } + humanLease = checked.lease; + humanLeaseUntil = checked.expiry; state = { holder: "human", since: now(), @@ -208,6 +258,17 @@ export function createControl( return this.get(); }, + /** Keep an active browser session's capability alive without changing when it took the wheel. */ + renew(lease: unknown, expiresAt: unknown): ControlState { + expireHumanLease(); + const checked = checkedLease(lease, expiresAt); + if (state.holder !== "human" || !sameLease(humanLease, checked.lease)) { + throw new ControlError(CONTROL_LEASE_REQUIRED); + } + humanLeaseUntil = checked.expiry; + return this.get(); + }, + /** * A person handing back. * @@ -216,7 +277,13 @@ export function createControl( * with it, a person who took the whole wheel and handed it back has dealt with the login, and a * secret box left open afterwards is asking for a password nothing is waiting for. */ - release(): ControlState { + release(lease: unknown): ControlState { + expireHumanLease(); + if (state.holder === "human" && !sameLease(humanLease, String(lease))) { + throw new ControlError(CONTROL_LEASE_REQUIRED); + } + humanLease = undefined; + humanLeaseUntil = 0; state = { holder: "bot", since: now(), @@ -225,6 +292,20 @@ export function createControl( return this.get(); }, + /** + * The controlled browser ceased to exist, so invalidate every human capability for it. + * + * This is intentionally not exposed on the network. Stop/reset already require an authorized + * server call and destroy the resource the lease names; requiring a now-useless browser token + * would leave the replacement browser permanently owned by a dead session. + */ + revoke(): ControlState { + humanLease = undefined; + humanLeaseUntil = 0; + state = { holder: "bot", since: now(), requested: false }; + return this.get(); + }, + /** * The Bot may not act while a person holds the wheel. * @@ -232,12 +313,16 @@ export function createControl( * a refusal, which the Bot can explain and wait out. */ assertBotMayAct(): void { + // A vanished browser cannot keep the Bot blocked. No polling tab remains to call `get()`, so + // the first Bot action after expiry must perform the same lease check itself. + expireHumanLease(); if (state.holder === "human") throw new ControlError(HUMAN_HAS_CONTROL); }, /** Whether a person's input should be applied. The socket being open is not permission. */ - humanMayDrive(): boolean { - return state.holder === "human"; + humanMayDrive(lease: string | undefined): boolean { + expireHumanLease(); + return state.holder === "human" && sameLease(humanLease, lease); }, }; } diff --git a/agent-computer/src/desktop.ts b/agent-computer/src/desktop.ts new file mode 100644 index 000000000..d8a3f3c4e --- /dev/null +++ b/agent-computer/src/desktop.ts @@ -0,0 +1,56 @@ +/** + * The full Linux desktop a person watches and drives. + * + * The desktop is an RFB framebuffer produced by x11vnc and translated to WebSocket by websockify. + * It is deliberately separate from the CDP page cast: CDP can only show a page viewport, while the + * framebuffer includes Chromium's address bar, tabs, browser dialogs, the desktop panel and any + * other application running on the Bot's computer. + */ + +export type DesktopMode = "view" | "control"; + +export type DesktopCapability = { + available: boolean; + protocol: "rfb"; + width: number; + height: number; +}; + +/** The image opts in explicitly after its display, VNC servers and WebSocket bridges are running. */ +export function desktopCapability( + environment: Record = process.env, +): DesktopCapability { + const width = positiveInteger(environment.DESKTOP_WIDTH, 1280); + const height = positiveInteger(environment.DESKTOP_HEIGHT, 800); + return { + available: + environment.COMPUTER_DESKTOP === "on" && Boolean(environment.DISPLAY), + protocol: "rfb", + width, + height, + }; +} + +/** Only the named control mode may ever reach the writable VNC server. */ +export function desktopMode(value: string | null): DesktopMode { + return value === "control" ? "control" : "view"; +} + +/** + * Both servers are loopback-only. The read-only server is a real server-side boundary, not a UI + * preference: a modified noVNC client connected while watching still cannot inject input. + */ +export function desktopUpstream( + mode: DesktopMode, + environment: Record = process.env, +): string { + if (mode === "control") { + return environment.DESKTOP_CONTROL_URL?.trim() || "ws://127.0.0.1:6081"; + } + return environment.DESKTOP_VIEW_URL?.trim() || "ws://127.0.0.1:6080"; +} + +function positiveInteger(value: string | undefined, fallback: number): number { + const parsed = value ? Number.parseInt(value, 10) : Number.NaN; + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +} diff --git a/agent-computer/src/index.ts b/agent-computer/src/index.ts index 341fcc349..8aa384e3e 100644 --- a/agent-computer/src/index.ts +++ b/agent-computer/src/index.ts @@ -16,6 +16,12 @@ import { NO_SECRET_PENDING, TAKE_CONTROL_FIRST, } from "./control"; +import { + type DesktopMode, + desktopCapability, + desktopMode, + desktopUpstream, +} from "./desktop"; import { identity } from "./identity"; import { createProfiles, numberFromEnv, VIEWPORT } from "./profiles"; import { type InputMessage, startScreencast } from "./screencast"; @@ -233,6 +239,13 @@ const DEFAULT_BOT_ID = (() => { return configured; })(); +// A local dock click carries no Bot header. The most recent successful control connection selects +// the profile it belongs to: only a control socket can deliver the click which starts the launcher. +// In per-Bot VM mode there is only one possible value; this also makes shared local development +// deterministic for the person currently holding the wheel. +let activeDesktopBotId = DEFAULT_BOT_ID; +let activeDesktopLease: string | undefined; + async function currentPage(botId: string): Promise { const session = sessionFor(botId); const page = await profiles.page(botId); @@ -375,8 +388,19 @@ const SCREEN_NO_LONGER_LIVE = /** How often the cast checks that it is still showing the page the Bot is on. */ const FOLLOW_INTERVAL_MS = 1_000; -/** What a live-screen socket carries: the Bot whose screen it is showing. */ -type StreamData = { botId: string }; +/** The old page cast remains as a compatibility fallback for computers built before full desktop. */ +type PageStreamData = { kind: "page"; botId: string; lease?: string }; + +/** A full desktop connection and the loopback websockify socket it is relaying. */ +type DesktopStreamData = { + kind: "desktop"; + botId: string; + mode: DesktopMode; + lease?: string; + upstream?: WebSocket; +}; + +type StreamData = PageStreamData | DesktopStreamData; serve({ port: PORT, @@ -390,6 +414,54 @@ serve({ */ websocket: { async open(ws) { + if (ws.data.kind === "desktop") { + const session = sessionFor(ws.data.botId); + if ( + ws.data.mode === "control" && + !session.control.humanMayDrive(ws.data.lease) + ) { + ws.close(1008, TAKE_CONTROL_FIRST); + return; + } + + try { + // This is a computer connection, not a browser launch. A real VM remains useful with every + // application closed, and reconnecting its display must not resurrect one a person closed. + // Bot browser work and the desktop's Browser icon are the two explicit launch paths. + const upstream = new WebSocket( + desktopUpstream(ws.data.mode), + "binary", + ); + upstream.binaryType = "arraybuffer"; + ws.data.upstream = upstream; + upstream.onmessage = async (event) => { + try { + if (typeof event.data === "string") { + ws.send(event.data); + } else if (event.data instanceof ArrayBuffer) { + ws.send(event.data); + } else if (event.data instanceof Blob) { + ws.send(await event.data.arrayBuffer()); + } + } catch { + upstream.close(); + } + }; + upstream.onclose = () => ws.close(); + upstream.onerror = () => ws.close(1011, "The desktop stopped."); + } catch (error) { + console.error( + JSON.stringify({ + type: "desktop-start-error", + botId: ws.data.botId, + error: String(error), + }), + ); + ws.close(1011, "The desktop could not be started."); + } + return; + } + const session = sessionFor(ws.data.botId); /* * Claimed before anything is awaited, and that order is the fix. @@ -457,6 +529,23 @@ serve({ }, async message(ws, raw) { + if (ws.data.kind === "desktop") { + const upstream = ws.data.upstream; + if (!upstream || upstream.readyState !== WebSocket.OPEN) return; + // A control socket is permission only while the lease is live. The view socket terminates + // at a server-side read-only VNC instance, so even a modified client cannot inject through it. + if ( + ws.data.mode === "control" && + !sessionFor(ws.data.botId).control.humanMayDrive(ws.data.lease) + ) { + upstream.close(); + ws.close(1008, TAKE_CONTROL_FIRST); + return; + } + upstream.send(raw); + return; + } + const session = sessionFor(ws.data.botId); /* * Whose screen this is, asked before anything is done with the input. @@ -497,7 +586,7 @@ serve({ // first only decides whether the input has anywhere to land. // // Refuse with an error so the surface can explain why input is ignored. - if (!session.control.humanMayDrive()) { + if (!session.control.humanMayDrive(ws.data.lease)) { ws.send(JSON.stringify({ type: "error", error: TAKE_CONTROL_FIRST })); return; } @@ -523,6 +612,10 @@ serve({ }, async close(ws) { + if (ws.data.kind === "desktop") { + ws.data.upstream?.close(); + return; + } // Names the socket, so it can only ever give up its own screen. A superseded socket closing // after its replacement has started releases nothing; see viewer.ts. // @@ -599,8 +692,62 @@ serve({ if (!isPlainBotId(streamBotId)) { return json({ error: "That is not a usable bot id." }, 400); } - if (server.upgrade(request, { data: { botId: streamBotId } })) + if ( + server.upgrade(request, { + data: { + kind: "page", + botId: streamBotId, + ...(url.searchParams.get("lease") + ? { lease: url.searchParams.get("lease") as string } + : {}), + }, + }) + ) + return undefined as unknown as Response; + return json({ error: "Expected a WebSocket upgrade." }, 400); + } + + if (url.pathname === "/desktop") { + const capability = desktopCapability(); + if (!capability.available) { + return json( + { error: "This computer does not provide a full desktop." }, + 503, + ); + } + const desktopBotId = botIdOf(request, url.searchParams.get("bot")); + if (!isPlainBotId(desktopBotId)) { + return json({ error: "That is not a usable bot id." }, 400); + } + const mode = desktopMode(url.searchParams.get("mode")); + const lease = url.searchParams.get("lease") ?? undefined; + if ( + mode === "control" && + !sessionFor(desktopBotId).control.humanMayDrive(lease) + ) { + return json({ error: TAKE_CONTROL_FIRST }, 409); + } + const requestedProtocols = + request.headers.get("sec-websocket-protocol") ?? ""; + if ( + server.upgrade(request, { + data: { kind: "desktop", botId: desktopBotId, mode, lease }, + ...(requestedProtocols + .split(",") + .map((value) => value.trim()) + .includes("binary") + ? { headers: { "sec-websocket-protocol": "binary" } } + : {}), + }) + ) { + // A local dock click carries no HTTP Bot header. The control socket does, and it is the only + // socket capable of delivering that click, so it selects the profile the dock reopens. + if (mode === "control") { + activeDesktopBotId = desktopBotId; + activeDesktopLease = lease; + } return undefined as unknown as Response; + } return json({ error: "Expected a WebSocket upgrade." }, 400); } @@ -615,6 +762,35 @@ serve({ return json(session.control.get()); } + if (url.pathname === "/capabilities" && request.method === "GET") { + return json({ desktop: desktopCapability() }); + } + + // The browser icon inside the Linux desktop. It is authenticated like every other computer + // route, and usable only while a person holds the wheel for the Bot selected by the control + // socket above. `profiles.page` starts a missing browser or replaces a natively closed context; + // `bringToFront` makes the same icon restore an existing browser instead of opening duplicates. + if (url.pathname === "/desktop/apps/browser" && request.method === "POST") { + const desktopBotId = activeDesktopBotId; + if (!sessionFor(desktopBotId).control.humanMayDrive(activeDesktopLease)) { + return json({ error: TAKE_CONTROL_FIRST }, 409); + } + try { + const target = await currentPage(desktopBotId); + await target.bringToFront(); + return json({ + opened: true, + botId: desktopBotId, + url: target.url(), + }); + } catch (error) { + return json( + { error: describe(error, "The browser could not be opened.") }, + 502, + ); + } + } + // The Bot asking for help. It does not take control: it says it is stuck and why, and a person // decides. A Bot that could hand itself to a human could also hand a human a page they never // asked to see. @@ -710,26 +886,69 @@ serve({ } if (url.pathname === "/control/take" && request.method === "POST") { - return json(session.control.take()); + const body = (await request.json().catch(() => null)) as { + lease?: unknown; + expiresAt?: unknown; + } | null; + try { + return json(session.control.take(body?.lease, body?.expiresAt)); + } catch (error) { + if (error instanceof ControlError) { + return json({ error: error.message }, 409); + } + if (error instanceof ControlRequestError) { + return json({ error: error.message }, 400); + } + throw error; + } + } + + if (url.pathname === "/control/renew" && request.method === "POST") { + const body = (await request.json().catch(() => null)) as { + lease?: unknown; + expiresAt?: unknown; + } | null; + try { + return json(session.control.renew(body?.lease, body?.expiresAt)); + } catch (error) { + if (error instanceof ControlError) { + return json({ error: error.message }, 409); + } + if (error instanceof ControlRequestError) { + return json({ error: error.message }, 400); + } + throw error; + } } if (url.pathname === "/control/release" && request.method === "POST") { // `reason` is dropped on release: it described the thing the person was asked to do, and once // they have done it, leaving it set would have the surface still showing the old request. - return json(session.control.release()); + const body = (await request.json().catch(() => null)) as { + lease?: unknown; + } | null; + try { + return json(session.control.release(body?.lease)); + } catch (error) { + if (error instanceof ControlError) { + return json({ error: error.message }, 409); + } + throw error; + } } // A person's input, by pixel. The Bot addresses elements by reference because it reads a list; a // person addresses them by pointing, because they are looking at a picture. Different problem, // different endpoint, and only usable while they hold the wheel. if (HUMAN_INPUT.has(url.pathname) && request.method === "POST") { - if (!session.control.humanMayDrive()) { - return json({ error: TAKE_CONTROL_FIRST }, 409); - } const body = (await request.json().catch(() => null)) as Record< string, unknown > | null; + const lease = typeof body?.lease === "string" ? body.lease : undefined; + if (!session.control.humanMayDrive(lease)) { + return json({ error: TAKE_CONTROL_FIRST }, 409); + } try { const target = await currentPage(botId); return json(await performHumanInput(target, url.pathname, body ?? {})); @@ -771,7 +990,7 @@ serve({ if (url.pathname === "/computers/stop" && request.method === "POST") { const wasRunning = await profiles.stop(botId); // The wheel goes back to the Bot because the controlled browser no longer exists. - session.control.release(); + session.control.revoke(); return json({ stopped: true, wasRunning }); } @@ -785,7 +1004,7 @@ serve({ if (url.pathname === "/computers/reset" && request.method === "POST") { await profiles.reset(botId); // Reset releases control because any previous browser session and pending secret request are gone. - session.control.release(); + session.control.revoke(); return json({ reset: true, botId }); } @@ -831,7 +1050,20 @@ serve({ if (url.pathname === "/screenshot" && request.method === "GET") { try { - const target = await currentPage(botId); + // A screenshot is observation, not an instruction to start an application. The transcript + // polls this endpoint while somebody is watching; calling `currentPage` here relaunched + // Chromium a second after the person closed its native window and made the VM look haunted. + // The desktop and dock remain available, and the Browser launcher is the explicit start path. + const target = profiles.runningPage(botId); + if (!target) { + return json( + { + error: + "The browser is closed. Open it from the Browser icon on the computer desktop.", + }, + 409, + ); + } const buffer = await target.screenshot({ type: "png" }); const size = target.viewportSize() ?? { width: 1280, height: 800 }; return json({ diff --git a/agent-computer/src/profiles.ts b/agent-computer/src/profiles.ts index c97f041c7..86de36712 100644 --- a/agent-computer/src/profiles.ts +++ b/agent-computer/src/profiles.ts @@ -50,6 +50,9 @@ export { numberFromEnv }; /** The viewport, which is what a person's click coordinates are relative to. */ export const VIEWPORT = { width: 1280, height: 800 }; +/** A headed browser is what makes the Bot's whole computer visible to a person. */ +const DESKTOP_ENABLED = process.env.COMPUTER_DESKTOP === "on"; + /** * Files Chromium uses to refuse a second instance on one profile. * @@ -96,14 +99,19 @@ const LAUNCH_ARGS = [ ...(SANDBOX_ENABLED ? [] : ["--no-sandbox"]), "--disable-dev-shm-usage", "--password-store=basic", + ...(DESKTOP_ENABLED + ? [ + `--window-size=${VIEWPORT.width},${VIEWPORT.height}`, + "--window-position=0,0", + "--start-maximized", + ] + : []), // Drop the automation signals Chromium sets for itself, so a real person who takes the wheel can // sign in to a site that refuses obvious automation (Google among them). This is the flag, not a // JS patch of `navigator.webdriver`: the flag turns the property off at the source, where spoofing // it from a script leaves the other tells a detector cross-checks. It does not change what the Bot - // may do; the governed path is unchanged. The larger tell — a headless build reporting - // `HeadlessChrome` in its user agent — is only removed by running headed under a virtual display, - // which is a heavier image change tracked separately; this reduces the signals it can reduce - // without one. + // may do; the governed path is unchanged. The image runs headed on its virtual display, so the + // person taking over sees the same browser the Bot acts on, including its native browser chrome. "--disable-blink-features=AutomationControlled", ]; @@ -336,6 +344,26 @@ export function createProfiles(root: string, onClosed: BrowserClosed) { }; return { + /** + * The Bot's page only if its browser is already running. + * + * Passive observers use this path. Looking at a computer must not start one: the transcript polls + * screenshots while it is open, and using `page()` there meant closing Chromium with its native + * X immediately launched it again. This also deliberately leaves `usedAt` alone, so a watcher + * does not make an otherwise idle browser immortal. + */ + runningPage(botId: string): Page | null { + const existing = live.get(botId); + existing?.retarget(); + if ( + existing?.context.browser()?.isConnected() && + !existing.page.isClosed() + ) { + return existing.page; + } + return null; + }, + /** * The Bot's page, starting its browser if it is not running. * @@ -381,6 +409,7 @@ export function createProfiles(root: string, onClosed: BrowserClosed) { const proxy = egressFor(botId, process.env); const context = await chromium.launchPersistentContext(dir, { args: LAUNCH_ARGS, + headless: !DESKTOP_ENABLED, // Playwright launches with `--enable-automation`, which sets `navigator.webdriver` and the // "controlled by automated software" banner. Dropped for the same reason as the flag above: // a person who takes the wheel should be able to sign in. Named explicitly so the sandbox @@ -429,6 +458,26 @@ export function createProfiles(root: string, onClosed: BrowserClosed) { }); page.on("close", () => record.retarget()); live.set(botId, record); + // Closing Chromium with its native X button bypasses `stop` and `evict`. Without this event, + // the dead context remained in `live`, `/computers` reported it as running, and nothing on + // the desktop could open it again until a later Bot API call happened to discover the stale + // page. Explicit closes delete the record before closing the context, so this runs only for + // a native close or crash and cannot announce the same close twice. + context.on("close", () => { + if (live.get(botId)?.context !== context) return; + live.delete(botId); + console.info( + JSON.stringify({ + type: "computer-browser-closed", + botId, + reason: "its browser window was closed", + }), + ); + void settleWithin( + Promise.resolve(onClosed(botId)), + ANNOUNCE_BUDGET_MS, + ); + }); // After the new one is in the map, so the cap counts what is really running and the Bot that // just asked is the most recently used and therefore never the one closed. await enforceCap(); diff --git a/agent-computer/tests/authorisation.test.ts b/agent-computer/tests/authorisation.test.ts index 34fcfdc4b..b60a73cdb 100644 --- a/agent-computer/tests/authorisation.test.ts +++ b/agent-computer/tests/authorisation.test.ts @@ -63,6 +63,15 @@ describe("finding the secret a caller offered", () => { ); }); + test("the full desktop takes it from the query for the same upgrade boundary", () => { + expect( + offeredToken( + new Headers(), + url(`/desktop?bot=b&token=${SECRET}&mode=view`), + ), + ).toBe(SECRET); + }); + test("the stream does NOT accept a header instead", () => { // Not a restriction that matters for security, both are checked against the same secret, but // the socket has one way in, and a surface that sent the header would fail loudly rather than @@ -86,6 +95,7 @@ describe("what an unauthenticated caller may reach", () => { "/files/list", "/files/read", "/stream", + "/desktop", "/live", "/", ]) { @@ -135,6 +145,7 @@ describe("what the wheel stops while a person is driving", () => { "/control/take", "/control/release", "/stream", + "/desktop", ]) { expect(actsOnTheComputer(path)).toBeFalse(); } diff --git a/agent-computer/tests/browser-close-announcement.test.ts b/agent-computer/tests/browser-close-announcement.test.ts index 71c10075a..1f89ec050 100644 --- a/agent-computer/tests/browser-close-announcement.test.ts +++ b/agent-computer/tests/browser-close-announcement.test.ts @@ -57,6 +57,28 @@ afterAll(async () => { describe.skipIf(!asked)( "a browser closed by something nobody asked for", () => { + test("a native browser close is forgotten immediately and tells whoever was watching", async () => { + process.env.COMPUTER_BROWSER_IDLE_MS = String(30 * 60_000); + const { createProfiles } = (await import( + `../src/profiles?native-close=${Date.now()}` + )) as typeof import("../src/profiles"); + const told: string[] = []; + const profiles = createProfiles(join(root, "native-close"), (botId) => { + told.push(botId); + }); + + const page = await profiles.page("closer"); + expect(profiles.liveCount()).toBe(1); + + // Chromium's X button closes the persistent context without passing through `profiles.stop`. + await page.context().close(); + await new Promise((resolve) => setTimeout(resolve, 25)); + + expect(profiles.liveCount()).toBe(0); + expect(told).toEqual(["closer"]); + await profiles.closeAll(); + }, 60_000); + test("the idle sweep tells whoever was watching", async () => { // The shortest timeout the sweep will act on. Zero disables it, because a timeout of nothing // means the feature is off rather than that everything is idle. diff --git a/agent-computer/tests/control.test.ts b/agent-computer/tests/control.test.ts index bbcc3f094..ecd0490d9 100644 --- a/agent-computer/tests/control.test.ts +++ b/agent-computer/tests/control.test.ts @@ -1,10 +1,16 @@ import { describe, expect, test } from "bun:test"; import { + CONTROL_ALREADY_HELD, + CONTROL_LEASE_REQUIRED, ControlError, ControlRequestError, createControl, } from "../src/control"; +const LEASE_A = "a".repeat(43); +const LEASE_B = "b".repeat(43); +const LEASE_UNTIL = "2026-08-14T00:10:00.000Z"; + /** * The wheel, tested on both paths. * @@ -17,10 +23,10 @@ import { */ function fixture() { let tick = 0; - const at = () => `2026-08-14T00:00:0${tick}.000Z`; + const start = Date.parse("2026-08-14T00:00:00.000Z"); const control = createControl(() => { tick += 1; - return at(); + return new Date(start + tick * 1_000).toISOString(); }); return { control }; } @@ -52,25 +58,25 @@ describe("the happy path: ask, hand over, hand back", () => { test("taking the wheel keeps the reason and lowers the flag", () => { const { control } = fixture(); control.requestHelp("Sign in to continue."); - const state = control.take(); + const state = control.take(LEASE_A, LEASE_UNTIL); expect(state.holder).toBe("human"); // The reason survives, because it is the thing the person was just asked to do. expect(state.reason).toBe("Sign in to continue."); // The request is answered, so the surface stops asking. expect(state.requested).toBe(false); - expect(control.humanMayDrive()).toBe(true); + expect(control.humanMayDrive(LEASE_A)).toBe(true); }); test("handing back returns the wheel and clears the old request", () => { const { control } = fixture(); control.requestHelp("Sign in to continue."); - control.take(); - const state = control.release(); + control.take(LEASE_A, LEASE_UNTIL); + const state = control.release(LEASE_A); expect(state.holder).toBe("bot"); // Dropped on purpose: leaving it set has the surface still showing a request that was dealt with. expect(state.reason).toBeUndefined(); expect(state.requested).toBe(false); - expect(control.humanMayDrive()).toBe(false); + expect(control.humanMayDrive(LEASE_A)).toBe(false); expect(() => control.assertBotMayAct()).not.toThrow(); }); @@ -80,14 +86,14 @@ describe("the happy path: ask, hand over, hand back", () => { control.requestHelp("Stuck."); // Asking for help is not a change of driver, so the clock does not restart. expect(control.get().since).toBe(created); - expect(control.take().since).not.toBe(created); + expect(control.take(LEASE_A, LEASE_UNTIL).since).not.toBe(created); }); }); describe("the crappy paths: two drivers, one page", () => { test("the Bot is refused while a person holds the wheel", () => { const { control } = fixture(); - control.take(); + control.take(LEASE_A, LEASE_UNTIL); expect(() => control.assertBotMayAct()).toThrow(ControlError); // Refused with a reason the Bot can act on, wait, rather than a bare failure. expect(() => control.assertBotMayAct()).toThrow(/hand it back/); @@ -95,8 +101,8 @@ describe("the crappy paths: two drivers, one page", () => { test("the refusal lifts the moment the person hands back", () => { const { control } = fixture(); - control.take(); - control.release(); + control.take(LEASE_A, LEASE_UNTIL); + control.release(LEASE_A); expect(() => control.assertBotMayAct()).not.toThrow(); }); @@ -105,21 +111,21 @@ describe("the crappy paths: two drivers, one page", () => { control.requestHelp("Sign in."); // The Bot asked for help and no person has taken the wheel. An open socket is not permission: this is // what stops anything that can reach the port from driving the browser mid-task. - expect(control.humanMayDrive()).toBe(false); + expect(control.humanMayDrive(LEASE_A)).toBe(false); }); test("taking the wheel twice is not a way to lose the reason", () => { const { control } = fixture(); control.requestHelp("Sign in."); - control.take(); - const state = control.take(); + control.take(LEASE_A, LEASE_UNTIL); + const state = control.take(LEASE_A, LEASE_UNTIL); expect(state.holder).toBe("human"); expect(state.reason).toBe("Sign in."); }); test("handing back when the Bot already has it is harmless", () => { const { control } = fixture(); - const state = control.release(); + const state = control.release(LEASE_A); expect(state.holder).toBe("bot"); expect(() => control.assertBotMayAct()).not.toThrow(); }); @@ -214,7 +220,11 @@ describe("the crappy paths: secrets", () => { for (const handover of ["take", "release"] as const) { const { control } = fixture(); control.requestSecret({ ref: "e12", label: "password" }); - control[handover](); + if (handover === "take") { + control.take(LEASE_A, LEASE_UNTIL); + } else { + control.release(LEASE_A); + } // A person who drove the browser themselves has dealt with the login. A masked box still asking // for a password afterwards is asking for a secret nothing is waiting for. expect(control.pendingSecret()).toBeNull(); @@ -272,7 +282,7 @@ describe("an unanswered request to take the wheel", () => { expect(state.reason).toBeUndefined(); }); - test("never takes the wheel back off a person who holds it", () => { + test("keeps the wheel while the browser renews its lease", () => { /* * The one case that must not expire. Somebody may be halfway through typing a code, and pulling * the browser back mid-sign-in is worse than any stale prompt. Only the ASK times out. @@ -280,9 +290,56 @@ describe("an unanswered request to take the wheel", () => { let clock = "2026-08-22T03:00:00.000Z"; const control = createControl(() => clock); control.requestHelp("sign in to Drive"); - control.take(); + control.take(LEASE_A, "2026-08-22T03:02:00.000Z"); - clock = "2026-08-22T04:00:00.000Z"; + clock = "2026-08-22T03:01:00.000Z"; + control.renew(LEASE_A, "2026-08-22T03:03:00.000Z"); + clock = "2026-08-22T03:02:30.000Z"; expect(control.get().holder).toBe("human"); }); }); + +describe("a control lease belongs to one browser session", () => { + test("a second lease cannot take, drive, renew, or release the first session", () => { + const { control } = fixture(); + control.take(LEASE_A, LEASE_UNTIL); + + expect(() => control.take(LEASE_B, LEASE_UNTIL)).toThrow( + CONTROL_ALREADY_HELD, + ); + expect(control.humanMayDrive(LEASE_B)).toBe(false); + expect(() => control.renew(LEASE_B, LEASE_UNTIL)).toThrow( + CONTROL_LEASE_REQUIRED, + ); + expect(() => control.release(LEASE_B)).toThrow(CONTROL_LEASE_REQUIRED); + expect(control.humanMayDrive(LEASE_A)).toBe(true); + }); + + test("an expired browser lease returns control to the Bot", () => { + let clock = "2026-08-22T03:00:00.000Z"; + const control = createControl(() => clock); + control.take(LEASE_A, "2026-08-22T03:01:00.000Z"); + + clock = "2026-08-22T03:01:01.000Z"; + expect(control.get().holder).toBe("bot"); + expect(control.humanMayDrive(LEASE_A)).toBe(false); + }); + + test("an expired lease cannot block the Bot when no browser remains to poll state", () => { + let clock = "2026-08-22T03:00:00.000Z"; + const control = createControl(() => clock); + control.take(LEASE_A, "2026-08-22T03:01:00.000Z"); + + clock = "2026-08-22T03:01:01.000Z"; + expect(() => control.assertBotMayAct()).not.toThrow(); + expect(control.get().holder).toBe("bot"); + }); + + test("stopping the controlled browser revokes its lease", () => { + const { control } = fixture(); + control.take(LEASE_A, LEASE_UNTIL); + + expect(control.revoke().holder).toBe("bot"); + expect(control.humanMayDrive(LEASE_A)).toBe(false); + }); +}); diff --git a/agent-computer/tests/desktop.test.ts b/agent-computer/tests/desktop.test.ts new file mode 100644 index 000000000..3def3b6f6 --- /dev/null +++ b/agent-computer/tests/desktop.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, test } from "bun:test"; +import { + desktopCapability, + desktopMode, + desktopUpstream, +} from "../src/desktop"; + +describe("the full computer desktop", () => { + test("is unavailable unless both the image and its display opted in", () => { + expect(desktopCapability({})).toEqual({ + available: false, + protocol: "rfb", + width: 1280, + height: 800, + }); + expect( + desktopCapability({ COMPUTER_DESKTOP: "on", DISPLAY: ":99" }), + ).toMatchObject({ available: true }); + expect(desktopCapability({ COMPUTER_DESKTOP: "on" })).toMatchObject({ + available: false, + }); + }); + + test("reports a configured framebuffer size and rejects invalid dimensions", () => { + expect( + desktopCapability({ + COMPUTER_DESKTOP: "on", + DISPLAY: ":2", + DESKTOP_WIDTH: "1440", + DESKTOP_HEIGHT: "900", + }), + ).toMatchObject({ width: 1440, height: 900 }); + expect( + desktopCapability({ DESKTOP_WIDTH: "0", DESKTOP_HEIGHT: "nonsense" }), + ).toMatchObject({ width: 1280, height: 800 }); + }); + + test("unknown modes fail closed to the read-only desktop", () => { + expect(desktopMode("control")).toBe("control"); + for (const value of [null, "", "write", "CONTROL"]) { + expect(desktopMode(value)).toBe("view"); + } + }); + + test("the read-only and control leases reach separate loopback bridges", () => { + expect(desktopUpstream("view", {})).toBe("ws://127.0.0.1:6080"); + expect(desktopUpstream("control", {})).toBe("ws://127.0.0.1:6081"); + expect( + desktopUpstream("view", { DESKTOP_VIEW_URL: "ws://viewer:7000" }), + ).toBe("ws://viewer:7000"); + expect( + desktopUpstream("control", { + DESKTOP_CONTROL_URL: "ws://driver:7001", + }), + ).toBe("ws://driver:7001"); + }); +}); diff --git a/agent-computer/tests/live-screen.test.ts b/agent-computer/tests/live-screen.test.ts index ee306ce7d..20c709f77 100644 --- a/agent-computer/tests/live-screen.test.ts +++ b/agent-computer/tests/live-screen.test.ts @@ -31,6 +31,7 @@ import { join } from "node:path"; const asked = process.env.OPENBOT_LIVE_SCREEN === "1"; const TOKEN = "test-computer-token"; +const CONTROL_LEASE = "a".repeat(43); /** * A port the operating system says is free, rather than one picked in advance. @@ -107,8 +108,10 @@ type Frames = { close: () => void; }; -function watch(botId: string): Frames { - const socket = new WebSocket(`${WS}/stream?bot=${botId}&token=${TOKEN}`); +function watch(botId: string, lease?: string): Frames { + const query = new URLSearchParams({ bot: botId, token: TOKEN }); + if (lease) query.set("lease", lease); + const socket = new WebSocket(`${WS}/stream?${query}`); const errors: string[] = []; let sawFrame = () => {}; const casting = new Promise((resolve) => { @@ -138,6 +141,16 @@ function watch(botId: string): Frames { return { socket, errors, connected, casting, close }; } +function takeControl(botId: string, lease: string) { + return api("/control/take", botId, { + method: "POST", + body: JSON.stringify({ + lease, + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }), + }); +} + const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); /** Wait for something to become true, rather than sleeping a guessed amount and hoping. */ @@ -203,6 +216,7 @@ afterAll(async () => { "wheel", "stop-viewer", "reset-viewer", + "screenshot-stopped", "wont-launch", "still-starting", ]) { @@ -251,12 +265,12 @@ describe.skipIf(!asked)("a socket that another connection replaced", () => { body: JSON.stringify({ url: TYPING_PAGE }), }); - const first = watch(botId); + const first = watch(botId, CONTROL_LEASE); await first.casting; - const second = watch(botId); + const second = watch(botId, CONTROL_LEASE); await second.casting; - await api("/control/take", botId, { method: "POST" }); + await takeControl(botId, CONTROL_LEASE); first.socket.send(JSON.stringify({ type: "key", key: "z" })); // The exact refusal, not merely some error. Dispatching through a cast the sender does not own @@ -287,9 +301,9 @@ describe.skipIf(!asked)("a superseded socket closing later", () => { body: JSON.stringify({ url: TYPING_PAGE }), }); - const first = watch(botId); + const first = watch(botId, CONTROL_LEASE); await first.casting; - const second = watch(botId); + const second = watch(botId, CONTROL_LEASE); await second.casting; // The replaced socket goes away now, after its replacement is live. @@ -298,7 +312,7 @@ describe.skipIf(!asked)("a superseded socket closing later", () => { // The survivor still owns the screen, and the proof is that its typing arrives: a cast that was // stopped underneath it, or an ownership it quietly lost, would refuse this instead. - await api("/control/take", botId, { method: "POST" }); + await takeControl(botId, CONTROL_LEASE); second.socket.send(JSON.stringify({ type: "key", key: "k" })); let landed = ""; @@ -390,6 +404,28 @@ describe.skipIf(!asked)("stopping the computer out from under a viewer", () => { }, 30_000); }); +describe.skipIf(!asked)("looking at a stopped computer", () => { + test("a screenshot says the browser is closed without starting it again", async () => { + const botId = "screenshot-stopped"; + await api("/navigate", botId, { + method: "POST", + body: JSON.stringify({ url: TYPING_PAGE }), + }); + expect(await stopped(botId)).toBe(true); + + const response = await api("/screenshot", botId); + expect(response.status).toBe(409); + expect(await response.json()).toEqual({ + error: + "The browser is closed. Open it from the Browser icon on the computer desktop.", + }); + + // More than one poll interval and a cold launch. If the read is still a hidden start path, the + // following stop catches the browser it made rather than passing on a lucky early observation. + await staysStopped(botId); + }, 30_000); +}); + describe.skipIf(!asked)("typing into a screen that is still opening", () => { test("is told the screen is starting, not that it ended", async () => { // The reason there are three standings rather than two. A socket mid-launch owns a claim and no diff --git a/app/package.json b/app/package.json index cce1f380e..1ca56f032 100644 --- a/app/package.json +++ b/app/package.json @@ -18,6 +18,7 @@ "@better-auth/sso": "^1.7.1", "@copilotkit/react-core": "1.69.0", "@fontsource-variable/inter": "^5.3.0", + "@novnc/novnc": "1.5.0", "@shadcn/react": "^0.3.0", "@tabler/icons-react": "^3.36.1", "@tanstack/react-form": "^1.33.5", diff --git a/app/src/components/agents/abstract-avatar.tsx b/app/src/components/agents/abstract-avatar.tsx index 9ca984507..825e67bc7 100644 --- a/app/src/components/agents/abstract-avatar.tsx +++ b/app/src/components/agents/abstract-avatar.tsx @@ -3,10 +3,12 @@ import Avatar from "boring-avatars"; export function AbstractAvatar({ name, seed, + image, size = 40, }: { name: string; seed: string; + image?: string | null; size?: number; }) { return ( @@ -16,10 +18,14 @@ export function AbstractAvatar({ className="inline-flex shrink-0 overflow-hidden rounded-full" style={{ height: size, width: size }} > - {/* The drawing carries its own role; hidden so the coworker is announced once, by name. */} - + {image ? ( + + ) : ( + /* The drawing carries its own role; hidden so the coworker is announced once, by name. */ + + )} ); } diff --git a/app/src/components/agents/agent-card.tsx b/app/src/components/agents/agent-card.tsx index 0a76128c8..a30102c29 100644 --- a/app/src/components/agents/agent-card.tsx +++ b/app/src/components/agents/agent-card.tsx @@ -1,11 +1,16 @@ -import Avatar from "boring-avatars"; +import { AbstractAvatar } from "@/components/agents/abstract-avatar"; import type { AgentProfile } from "@/lib/agents/queries"; export function AgentCard({ agent }: { agent: AgentProfile }) { return (
- +
diff --git a/app/src/components/agents/agent-dialog.tsx b/app/src/components/agents/agent-dialog.tsx index bfc60023e..b5e5a79a1 100644 --- a/app/src/components/agents/agent-dialog.tsx +++ b/app/src/components/agents/agent-dialog.tsx @@ -15,6 +15,7 @@ import { AbstractAvatar } from "@/components/agents/abstract-avatar"; import { CallbackTokenPanel } from "@/components/agents/callback-token-panel"; import { HandoffPanel } from "@/components/agents/handoff-panel"; import { RoutinesList } from "@/components/routines/routines-list"; +import { AvatarUploadActions } from "@/components/ui/avatar-upload-actions"; import { Button } from "@/components/ui/button"; import { Dialog, @@ -67,6 +68,7 @@ import { deleteAgentMutationOptions, duplicateAgentMutationOptions, setAgentHiddenMutationOptions, + updateAgentAvatarMutationOptions, updateAgentMutationOptions, } from "@/lib/agents/mutations"; import { type AgentProfile, agentQueryOptions } from "@/lib/agents/queries"; @@ -148,6 +150,7 @@ function AgentDialogBody({ agentId }: { agentId: string }) { {/* Who this dialog is about, said once here rather than repeated per section. */}
+ + +
+ +
+ Avatar + + PNG, JPEG, or WebP, up to 2 MB. + +
+
+ + {profile.canCustomizeAvatar ? ( + + + updateAvatar.mutateAsync({ agentId, image }) + } + /> + + ) : null} +
part[0]?.toUpperCase()) - .join("") ?? currentUser?.email.slice(0, 2).toUpperCase(); - - return ( -
- {initials} -
- ); -} - /** * Cap layout animation because `layout` measures every animated row on each reorder. */ @@ -189,6 +173,7 @@ function ChannelRow({ ) { } > - + {currentUser?.name || currentUser?.email} diff --git a/app/src/components/app-sidebar/channel.tsx b/app/src/components/app-sidebar/channel.tsx index 194962d37..16da221c1 100644 --- a/app/src/components/app-sidebar/channel.tsx +++ b/app/src/components/app-sidebar/channel.tsx @@ -38,6 +38,7 @@ import { ChannelAvatar } from "../channels/avatar"; export const Channel = memo(function Channel({ channelId, participantIds, + participantImages, name, lastMessage, lastMessageAt, @@ -47,6 +48,7 @@ export const Channel = memo(function Channel({ }: { channelId: string; participantIds: string[]; + participantImages?: (string | null)[]; name: string; lastMessage?: string; lastMessageAt?: string; @@ -114,6 +116,7 @@ export const Channel = memo(function Channel({
diff --git a/app/src/components/channels/avatar.tsx b/app/src/components/channels/avatar.tsx index 11a19e222..41aefae3d 100644 --- a/app/src/components/channels/avatar.tsx +++ b/app/src/components/channels/avatar.tsx @@ -14,10 +14,13 @@ import { cn } from "@/lib/utils"; */ export const ChannelAvatar = memo(function ChannelAvatar({ participantIds, + participantImages, size = 32, typing = false, }: { participantIds: string[]; + /** Parallel to participantIds when a server-owned avatar is available. */ + participantImages?: (string | null)[]; size?: number; typing?: boolean; }) { @@ -25,7 +28,11 @@ export const ChannelAvatar = memo(function ChannelAvatar({ const avatar = channelSize === 1 ? ( - + ) : (
{participantIds.slice(0, 3).map((c, i, shown) => ( @@ -38,9 +45,9 @@ export const ChannelAvatar = memo(function ChannelAvatar({ transform: `translateX(${i * -75}%)`, }} > -
@@ -56,6 +63,22 @@ export const ChannelAvatar = memo(function ChannelAvatar({ ); }); +function AvatarFace({ + image, + seed, + size, +}: { + image?: string | null; + seed: string; + size: number; +}) { + return image ? ( + + ) : ( + + ); +} + /** * Three bouncing dots in a small badge, ringed in the sidebar's own colour so it sits on the * avatar as a badge rather than floating over it. The staggered negative delays start each dot at diff --git a/app/src/components/channels/recipient-field.tsx b/app/src/components/channels/recipient-field.tsx index bb33f113e..effb9abba 100644 --- a/app/src/components/channels/recipient-field.tsx +++ b/app/src/components/channels/recipient-field.tsx @@ -38,6 +38,9 @@ export function RecipientField({ profile.name.toLowerCase().includes(search.trim().toLowerCase()), ); const isFull = recipients.length >= MAX_RECIPIENTS; + const profileById = new Map( + (profiles ?? []).map((profile) => [profile.id, profile]), + ); return (
@@ -51,7 +54,13 @@ export function RecipientField({ size="xs" variant="secondary" > - + {recipient.name} × Remove {recipient.name} @@ -88,7 +97,11 @@ export function RecipientField({ }} type="button" > - + {profile.name} {profile.title} diff --git a/app/src/components/computer/computer-view.tsx b/app/src/components/computer/computer-view.tsx index a0990604e..43cf6492f 100644 --- a/app/src/components/computer/computer-view.tsx +++ b/app/src/components/computer/computer-view.tsx @@ -1,7 +1,15 @@ -import { useEffect, useRef, useState } from "react"; +import { + lazy, + Suspense, + useCallback, + useEffect, + useRef, + useState, +} from "react"; import { createPortal } from "react-dom"; import { type ControlState, + controlLease, readControl, releaseControl, supplySecret, @@ -15,6 +23,13 @@ import { import { ChannelAvatar } from "../channels/avatar"; import { LiveScreen } from "./live-screen"; +// RFB decoding is substantial and is needed only after somebody opens the full-size computer. +// Keeping it out of the channel's initial bundle avoids charging every conversation for takeover. +const RemoteDesktop = lazy(async () => { + const module = await import("./remote-desktop"); + return { default: module.RemoteDesktop }; +}); + /** Explicit blank-browser URLs use placeholder artwork; missing URL fields are treated as real pages. */ function isBlankBrowser(shot: Screenshot): boolean { if (shot.url === undefined) return false; @@ -31,6 +46,20 @@ function hostOf(url: string): string { } } +/** + * Only an RFB/page-stream connection error may cover the expanded computer. + * + * The browser screenshot poll is a sibling surface. It is expected to fail after somebody closes + * Chromium from the desktop, and showing that failure over the framebuffer would hide the dock used + * to reopen it. Keeping both inputs here makes that separation a regression-testable decision. + */ +export function desktopOverlayProblem( + _browserProblem: string | null, + desktopProblem: string | null, +): string | null { + return desktopProblem; +} + /** * What each finished turn opened, and the frame it ended on, kept outside any component. * @@ -222,17 +251,34 @@ export function ComputerView({ }: Props) { const [shot, setShot] = useState(null); const [problem, setProblem] = useState(null); + /** A full-desktop connection problem, separate from the browser screenshot used by the inline tile. */ + const [desktopProblem, setDesktopProblem] = useState(null); const [expanded, setExpanded] = useState(false); + const [desktopUnavailable, setDesktopUnavailable] = useState(false); const [control, setControl] = useState(null); /** Held only until it is sent. Never lifted into a URL, a log, or anything that outlives this form. */ const [secret, setSecret] = useState(""); const [secretProblem, setSecretProblem] = useState(null); const [sendingSecret, setSendingSecret] = useState(false); - const driving = control?.holder === "human"; + const lease = controlLease(computerId); + const driving = control?.holder === "human" && Boolean(lease); + const visibleDesktopProblem = desktopOverlayProblem(problem, desktopProblem); /** Read by the polling loop without restarting it on control changes. */ const drivingRef = useRef(false); drivingRef.current = driving; + useEffect(() => { + // Referenced deliberately: a different Bot may run an older image even when this one does not. + void computerId; + setDesktopUnavailable(false); + setDesktopProblem(null); + }, [computerId]); + + const markDesktopUnavailable = useCallback(() => { + setDesktopUnavailable(true); + setDesktopProblem(null); + }, []); + /** Release control; the Bot's waiting tool call resumes from this state change. */ const handBack = async () => { const state = await releaseControl(computerId); @@ -429,7 +475,7 @@ export function ComputerView({ * gets the live socket whatever is on it, because once a person is driving the stream is the truth * about the page and a placeholder over it would be the view arguing with them. */ - const showLiveScreen = !settled && (showScreen || driving); + const showLiveScreen = !settled; /** * Whether the wheel in somebody's hands is the wheel THIS tile is showing. * @@ -641,20 +687,39 @@ export function ComputerView({
) : showLiveScreen ? (
- + {desktopUnavailable ? ( + + ) : ( + + Connecting to the computer… +
+ } + > + + + )} {/* - A live screen that ends reports why through `onProblem`, and this is the - branch that is mounted when it does. Without drawing it here the message - landed in `problem`, which only the sibling `NothingToSee` reads, so the - screen ended with the stale last frame frozen on the canvas and nothing said. + A live desktop that ends reports why through its own problem state. This must + stay separate from the inline browser screenshot: a deliberately closed + browser makes that poll fail while the VM display and its launchers remain + healthy, and drawing that failure here covered the very dock used to reopen it. */} - {problem ? ( + {visibleDesktopProblem ? (
- {problem} + {visibleDesktopProblem}
) : null}
diff --git a/app/src/components/computer/live-screen.tsx b/app/src/components/computer/live-screen.tsx index bbc989389..93d396f54 100644 --- a/app/src/components/computer/live-screen.tsx +++ b/app/src/components/computer/live-screen.tsx @@ -38,11 +38,13 @@ type Props = { computerId: string; /** Whether the user currently holds the wheel. Input is only sent when true. */ driving: boolean; + /** Private capability for this browser tab. Frames are public to authorized viewers; input is not. */ + lease?: string; /** Called with a human-readable reason when the stream cannot be established. */ onProblem?: (problem: string | null) => void; }; -export function LiveScreen({ computerId, driving, onProblem }: Props) { +export function LiveScreen({ computerId, driving, lease, onProblem }: Props) { const canvasRef = useRef(null); const socketRef = useRef(null); /** The size of the frames Chrome is sending, which is what input coordinates are relative to. */ @@ -54,6 +56,7 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) { const scheme = window.location.protocol === "https:" ? "wss" : "ws"; const socket = new WebSocket( `${scheme}://${window.location.host}/api/computers/${encodeURIComponent(computerId)}/stream`, + driving && lease ? [`openbot-lease.${lease}`] : [], ); socketRef.current = socket; let closed = false; @@ -126,7 +129,7 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) { socketRef.current = null; }; // The socket is per Bot; switching Bot must close this stream and open the next one. - }, [computerId, onProblem]); + }, [computerId, driving, lease, onProblem]); const send = useCallback( (message: Record) => { diff --git a/app/src/components/computer/remote-desktop.tsx b/app/src/components/computer/remote-desktop.tsx new file mode 100644 index 000000000..7071ebdc6 --- /dev/null +++ b/app/src/components/computer/remote-desktop.tsx @@ -0,0 +1,125 @@ +import RFB from "@novnc/novnc/lib/rfb"; +import { useEffect, useRef, useState } from "react"; + +type Props = { + computerId: string; + driving: boolean; + /** Private to the browser tab that took control. Observing `driving` alone is not authority. */ + lease?: string; + onProblem?: (problem: string | null) => void; + /** Old computer images have no RFB endpoint. Let the caller fall back to the page cast. */ + onUnavailable?: () => void; +}; + +const MAX_RECONNECTS = 2; + +/** + * The Bot's whole graphical computer. + * + * noVNC renders the RFB framebuffer produced inside the Bot's VM. Unlike the old CDP canvas this + * includes browser chrome, tabs, native dialogs, the desktop panel and other applications. The + * browser never connects to the VM directly: the same-origin socket terminates at OpenBot, which + * checks the signed-in actor and Bot access before proxying it inward. + */ +export function RemoteDesktop({ + computerId, + driving, + lease, + onProblem, + onUnavailable, +}: Props) { + const targetRef = useRef(null); + const [attempt, setAttempt] = useState(0); + const [connected, setConnected] = useState(false); + + useEffect(() => { + // Each computer and each lease gets a fresh retry budget. + void computerId; + void driving; + void lease; + setAttempt(0); + setConnected(false); + }, [computerId, driving, lease]); + + useEffect(() => { + const target = targetRef.current; + if (!target) return; + + const scheme = window.location.protocol === "https:" ? "wss" : "ws"; + const query = new URLSearchParams({ + mode: driving ? "control" : "view", + }); + const url = `${scheme}://${window.location.host}/api/computers/${encodeURIComponent(computerId)}/desktop?${query}`; + let disposed = false; + let connectedOnce = false; + let retryTimer: number | undefined; + const rfb = new RFB(target, url, { + shared: true, + wsProtocols: [ + "binary", + ...(driving && lease ? [`openbot-lease.${lease}`] : []), + ], + }); + rfb.viewOnly = !driving; + rfb.scaleViewport = true; + rfb.resizeSession = false; + rfb.clipViewport = false; + rfb.focusOnClick = true; + rfb.qualityLevel = 7; + rfb.compressionLevel = 2; + + const onConnect = () => { + connectedOnce = true; + setConnected(true); + onProblem?.(null); + if (driving) rfb.focus(); + }; + const onDisconnect = (_event: Event & { detail?: { clean?: boolean } }) => { + if (disposed) return; + setConnected(false); + if (!connectedOnce && attempt >= MAX_RECONNECTS) { + onUnavailable?.(); + return; + } + onProblem?.("The computer display disconnected. Reconnecting…"); + retryTimer = window.setTimeout( + () => { + if (!disposed) setAttempt((value) => value + 1); + }, + Math.min(3_000, 500 * 2 ** attempt), + ); + }; + const onSecurityFailure = () => { + onProblem?.("The computer display refused the connection."); + }; + + rfb.addEventListener("connect", onConnect); + rfb.addEventListener("disconnect", onDisconnect as EventListener); + rfb.addEventListener("securityfailure", onSecurityFailure); + + return () => { + disposed = true; + if (retryTimer !== undefined) window.clearTimeout(retryTimer); + rfb.removeEventListener("connect", onConnect); + rfb.removeEventListener("disconnect", onDisconnect as EventListener); + rfb.removeEventListener("securityfailure", onSecurityFailure); + rfb.disconnect(); + target.replaceChildren(); + }; + }, [attempt, computerId, driving, lease, onProblem, onUnavailable]); + + return ( + // biome-ignore lint/a11y/useAriaPropsSupportedByRole: the runtime role is always application or img, both named here. +
+ ); +} diff --git a/app/src/components/people/user-avatar.tsx b/app/src/components/people/user-avatar.tsx new file mode 100644 index 000000000..54a9cad90 --- /dev/null +++ b/app/src/components/people/user-avatar.tsx @@ -0,0 +1,40 @@ +export function UserAvatar({ + email, + image, + name, + size = 28, +}: { + email?: string | null; + image?: string | null; + name?: string | null; + size?: number; +}) { + const initials = + name + ?.trim() + .split(/\s+/) + .slice(0, 2) + .map((part) => part[0]?.toUpperCase()) + .join("") ?? email?.slice(0, 2).toUpperCase(); + const label = name || email || "Your avatar"; + + return ( + + {image ? ( + + ) : ( + initials + )} + + ); +} diff --git a/app/src/components/ui/avatar-upload-actions.tsx b/app/src/components/ui/avatar-upload-actions.tsx new file mode 100644 index 000000000..668656649 --- /dev/null +++ b/app/src/components/ui/avatar-upload-actions.tsx @@ -0,0 +1,87 @@ +import { IconPhotoUp, IconTrash } from "@tabler/icons-react"; +import { useRef, useState } from "react"; +import { readAvatarFile } from "@/lib/avatar"; +import { Button } from "./button"; + +/** The shared upload/remove controls used by both person and coworker avatars. */ +export function AvatarUploadActions({ + label, + hasImage, + hasCustomImage = hasImage, + onChange, + disabled = false, +}: { + label: string; + hasImage: boolean; + /** A provider image is visible but cannot be removed; a custom image can. */ + hasCustomImage?: boolean; + onChange: (image: string | null) => Promise; + disabled?: boolean; +}) { + const input = useRef(null); + const [saving, setSaving] = useState(false); + const [error, setError] = useState(null); + + const save = async (image: string | null) => { + setError(null); + setSaving(true); + try { + await onChange(image); + } catch (failure) { + setError(failure instanceof Error ? failure.message : "Could not save the avatar."); + } finally { + setSaving(false); + } + }; + + return ( +
+ { + const file = event.target.files?.[0]; + // Let selecting the same file again trigger change after a refusal. + event.target.value = ""; + if (!file) return; + try { + await save(await readAvatarFile(file)); + } catch (failure) { + setError(failure instanceof Error ? failure.message : "That image could not be read."); + } + }} + ref={input} + type="file" + /> +
+ + {hasCustomImage ? ( + + ) : null} +
+ {error ? ( +

+ {error} +

+ ) : null} +
+ ); +} diff --git a/app/src/lib/agents/mutations.ts b/app/src/lib/agents/mutations.ts index 641146384..9cce29d52 100644 --- a/app/src/lib/agents/mutations.ts +++ b/app/src/lib/agents/mutations.ts @@ -1,4 +1,5 @@ import { mutationOptions, type QueryClient } from "@tanstack/react-query"; +import { channelKeys } from "@/lib/channels/queries"; import { client } from "@/lib/client"; import { type AgentProfile, type AgentVisibility, agentKeys } from "./queries"; @@ -48,6 +49,25 @@ export function updateAgentMutationOptions(queryClient: QueryClient) { }); } +export function updateAgentAvatarMutationOptions(queryClient: QueryClient) { + return mutationOptions({ + mutationFn: (variables: { + agentId: string; + image: string | null; + }): Promise => + client(`/api/agents/${variables.agentId}/avatar`, "agent", { + method: "PUT", + body: { image: variables.image }, + fallback: "Could not save the coworker avatar", + }), + onSuccess: () => + Promise.all([ + invalidateAgents(queryClient), + queryClient.invalidateQueries({ queryKey: channelKeys.all }), + ]), + }); +} + export function duplicateAgentMutationOptions(queryClient: QueryClient) { return mutationOptions({ mutationFn: (agentId: string): Promise => diff --git a/app/src/lib/agents/queries.ts b/app/src/lib/agents/queries.ts index 5f2b035aa..aebdd38b6 100644 --- a/app/src/lib/agents/queries.ts +++ b/app/src/lib/agents/queries.ts @@ -15,6 +15,7 @@ export type AgentProfile = { title: string; roleDescription: string; avatarSeed: string; + avatarUrl: string | null; visibility: AgentVisibility; /** Where this coworker runs. Null for the Bot in the box. */ endpoint: string | null; @@ -39,6 +40,8 @@ export type AgentProfile = { hidden: boolean; systemOwned: boolean; canManage: boolean; + /** Owners may customize their Bot; administrators may also brand package-owned Bots. */ + canCustomizeAvatar: boolean; /** * Whether the signed-in person created this coworker. * diff --git a/app/src/lib/auth/mutations.ts b/app/src/lib/auth/mutations.ts index 771ee0df0..513af9c73 100644 --- a/app/src/lib/auth/mutations.ts +++ b/app/src/lib/auth/mutations.ts @@ -15,3 +15,22 @@ export function signOutMutationOptions(queryClient: QueryClient) { onSuccess: () => queryClient.removeQueries({ queryKey: authKeys.all }), }); } + +export function updateCurrentUserAvatarMutationOptions( + queryClient: QueryClient, +) { + return mutationOptions({ + mutationFn: (image: string | null) => + client<{ image: string | null; hasCustomAvatar: boolean }>( + "/api/me/avatar", + "avatar", + { + method: "PUT", + body: { image }, + fallback: "Could not save your avatar", + }, + ), + onSuccess: () => + queryClient.invalidateQueries({ queryKey: authKeys.currentUser() }), + }); +} diff --git a/app/src/lib/auth/queries.ts b/app/src/lib/auth/queries.ts index 5215e08e1..79d752636 100644 --- a/app/src/lib/auth/queries.ts +++ b/app/src/lib/auth/queries.ts @@ -17,6 +17,7 @@ export type AuthenticatedUser = { email: string; name?: string | null; image?: string | null; + hasCustomAvatar: boolean; role: "admin" | "user"; /** Null means this deployment does not track onboarding, which reads as nothing to finish. */ onboarding: OnboardingStatus | null; diff --git a/app/src/lib/avatar.ts b/app/src/lib/avatar.ts new file mode 100644 index 000000000..28824806b --- /dev/null +++ b/app/src/lib/avatar.ts @@ -0,0 +1,31 @@ +export const MAX_AVATAR_BYTES = 2 * 1024 * 1024; + +const AVATAR_TYPES = new Set(["image/jpeg", "image/png", "image/webp"]); + +export function avatarFileError( + file: Pick, +): string | null { + if (!AVATAR_TYPES.has(file.type)) { + return "Choose a PNG, JPEG, or WebP image."; + } + if (file.size === 0) return "That image is empty."; + if (file.size > MAX_AVATAR_BYTES) + return "Choose an image that is 2 MB or smaller."; + return null; +} + +/** Read one validated image into the small, portable representation the avatar API accepts. */ +export async function readAvatarFile(file: File): Promise { + const invalid = avatarFileError(file); + if (invalid) throw new Error(invalid); + + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onerror = () => reject(new Error("That image could not be read.")); + reader.onload = () => + typeof reader.result === "string" + ? resolve(reader.result) + : reject(new Error("That image could not be read.")); + reader.readAsDataURL(file); + }); +} diff --git a/app/src/lib/channels/queries.ts b/app/src/lib/channels/queries.ts index 4a62ce053..d004c50f7 100644 --- a/app/src/lib/channels/queries.ts +++ b/app/src/lib/channels/queries.ts @@ -18,6 +18,8 @@ export type AgentChannel = { /** A channel plus the last thing said in it, which is what the roster renders. */ export type ChannelSummary = AgentChannel & { + /** Parallel to agentIds; null means use the generated fallback. */ + avatarUrls: (string | null)[]; lastMessage: string | null; /** ISO-8601, or null for a channel nobody has used yet. */ lastMessageAt: string | null; diff --git a/app/src/lib/computers/control.ts b/app/src/lib/computers/control.ts index eddca0965..2b3ca0dd2 100644 --- a/app/src/lib/computers/control.ts +++ b/app/src/lib/computers/control.ts @@ -3,9 +3,9 @@ import { tryClient } from "@/lib/client"; /** * Handing control of a Bot's computer to a person, and back. * - * Plain functions rather than factories, and every one of them fails closed. Nothing here is cached: - * who holds the wheel is a fact about this second, and a stale copy of it would be worse than no - * copy — it would show somebody a screen they cannot drive, or let them think they can. + * Plain functions rather than factories, and every one of them fails closed. The public state is + * always read live. The one local value is a bearer lease in sessionStorage, scoped to this browser + * tab: observing that somebody else holds the wheel must never make their mouse authority reusable. * * The reads answer `null` on failure rather than throwing. A panel that cannot say who is driving * should say nothing, not tear down the screen the person is looking at. @@ -20,29 +20,115 @@ export type ControlState = { secretWanted?: string; }; +type ControlLeaseState = ControlState & { lease: string }; +const LEASE_KEY = "openbot.computer-control"; +const RENEW_AFTER_MS = 30_000; +const lastRenewed = new Map(); +const leaseStartedAt = new Map(); + +function leaseKey(computerId: string) { + return `${LEASE_KEY}.${computerId}`; +} + +/** The private capability for this tab, never inferred from the public holder state. */ +export function controlLease(computerId: string): string | undefined { + try { + return ( + globalThis.sessionStorage?.getItem(leaseKey(computerId)) ?? undefined + ); + } catch { + return undefined; + } +} + +function keepLease(computerId: string, lease: string, since: string) { + try { + globalThis.sessionStorage?.setItem(leaseKey(computerId), lease); + lastRenewed.set(computerId, Date.now()); + leaseStartedAt.set(computerId, since); + } catch { + // Without tab-scoped storage this browser cannot safely claim it can drive. + } +} + +function forgetLease(computerId: string) { + try { + globalThis.sessionStorage?.removeItem(leaseKey(computerId)); + } catch { + // It may already be unavailable; the in-memory renewal marker is still cleared below. + } + lastRenewed.delete(computerId); + leaseStartedAt.delete(computerId); +} + async function callControl( computerId: string, path: string, method?: string, + body?: unknown, ): Promise { const response = await tryClient( `/api/computers/${computerId}${path}`, - method ? { method } : {}, + method ? { method, ...(body === undefined ? {} : { body }) } : {}, ); if (!response.ok) return null; return (await response.json()) as ControlState; } -export function readControl(computerId: string) { - return callControl(computerId, "/control"); +export async function readControl(computerId: string) { + let state = await callControl(computerId, "/control"); + if (!state) return null; + + const lease = controlLease(computerId); + if (state.holder !== "human") { + if (lease) forgetLease(computerId); + return state; + } + if (!lease) return state; + + const renewedAt = lastRenewed.get(computerId) ?? 0; + if ( + leaseStartedAt.get(computerId) === state.since && + Date.now() - renewedAt < RENEW_AFTER_MS + ) + return state; + const renewed = await callControl(computerId, "/control/renew", "POST", { + lease, + }); + if (!renewed) { + forgetLease(computerId); + return state; + } + state = renewed; + lastRenewed.set(computerId, Date.now()); + leaseStartedAt.set(computerId, state.since); + return state; } -export function takeControl(computerId: string) { - return callControl(computerId, "/control/take", "POST"); +export async function takeControl(computerId: string) { + const response = await tryClient( + `/api/computers/${computerId}/control/take`, + { + method: "POST", + }, + ); + if (!response.ok) return null; + const { lease, ...state } = (await response.json()) as ControlLeaseState; + if (!lease) return null; + keepLease(computerId, lease, state.since); + return state; } -export function releaseControl(computerId: string) { - return callControl(computerId, "/control/release", "POST"); +export async function releaseControl(computerId: string) { + const lease = controlLease(computerId); + if (!lease) return null; + try { + return await callControl(computerId, "/control/release", "POST", { + lease, + }); + } finally { + forgetLease(computerId); + } } /** @@ -86,11 +172,13 @@ export function sendHumanInput( kind: "click" | "type" | "key" | "scroll", body: Record, ): void { + const lease = controlLease(computerId); + if (!lease) return; inputQueue = inputQueue .then(() => tryClient(`/api/computers/${computerId}/human/${kind}`, { method: "POST", - body, + body: { ...body, lease }, }), ) // Fire-and-forget: the user can see/retry input failures, while the input queue must keep moving. diff --git a/app/src/routes/_authed/_app/channel/$channelId.tsx b/app/src/routes/_authed/_app/channel/$channelId.tsx index d1bf998b5..b810b7570 100644 --- a/app/src/routes/_authed/_app/channel/$channelId.tsx +++ b/app/src/routes/_authed/_app/channel/$channelId.tsx @@ -19,6 +19,7 @@ import { useNeedsYou } from "@/components/computer/needs-you"; import { DetailPanel } from "@/components/layout/detail-panel"; import { SidebarToggle } from "@/components/layout/sidebar-toggle"; import { Button } from "@/components/ui/button"; +import { agentQueryOptions } from "@/lib/agents/queries"; import { markChannelReadMutationOptions } from "@/lib/channels/mutations"; import { type AgentChannel, @@ -86,6 +87,10 @@ function RouteComponent() { const isWatching = watch === true; /** Channel routing currently supports one coworker. */ const agentId = channel.data?.agentIds[0]; + const agent = useQuery({ + ...agentQueryOptions(agentId ?? ""), + enabled: agentId !== undefined, + }); /** Only polled while the screen is closed; the screen panel polls control itself. */ const needsYou = useNeedsYou(agentId, !isWatching); @@ -191,6 +196,9 @@ function RouteComponent() { > diff --git a/app/src/routes/_authed/_app/channel/new.tsx b/app/src/routes/_authed/_app/channel/new.tsx index c4ab4346a..4922093ea 100644 --- a/app/src/routes/_authed/_app/channel/new.tsx +++ b/app/src/routes/_authed/_app/channel/new.tsx @@ -101,7 +101,11 @@ function RouteComponent() { {(item: AgentProfile) => ( - + {item.name} {item.title} diff --git a/app/src/routes/_authed/settings/index.tsx b/app/src/routes/_authed/settings/index.tsx index 2fba05d21..ce4f81d75 100644 --- a/app/src/routes/_authed/settings/index.tsx +++ b/app/src/routes/_authed/settings/index.tsx @@ -1,3 +1,4 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { createFileRoute } from "@tanstack/react-router"; import React from "react"; import { @@ -5,7 +6,9 @@ import { PageSection, PageShell, } from "@/components/layout/page-shell"; +import { UserAvatar } from "@/components/people/user-avatar"; import { useTheme } from "@/components/theme-provider"; +import { AvatarUploadActions } from "@/components/ui/avatar-upload-actions"; import { Item, ItemActions, @@ -15,6 +18,8 @@ import { } from "@/components/ui/item"; import { Separator } from "@/components/ui/separator"; import { Switch } from "@/components/ui/switch"; +import { updateCurrentUserAvatarMutationOptions } from "@/lib/auth/mutations"; +import { currentUserQueryOptions } from "@/lib/auth/queries"; import { formatHotkey, HOTKEYS } from "@/lib/hotkeys/hotkeys"; export const Route = createFileRoute("/_authed/settings/")({ @@ -23,6 +28,11 @@ export const Route = createFileRoute("/_authed/settings/")({ function RouteComponent() { const { dark, setDark } = useTheme(); + const queryClient = useQueryClient(); + const { data: currentUser } = useQuery(currentUserQueryOptions()); + const updateAvatar = useMutation( + updateCurrentUserAvatarMutationOptions(queryClient), + ); /* * The measurements that used to be written out here now live in `PageShell`, which Skills, Admin @@ -37,6 +47,33 @@ function RouteComponent() { description="How OpenBot looks and behaves for you. These apply to your account alone, on every deployment you sign in to." title="Preferences" > + + + +
+ +
+ + Avatar + PNG, JPEG, or WebP, up to 2 MB. + + + updateAvatar.mutateAsync(image)} + /> + + + + diff --git a/app/src/types/novnc.d.ts b/app/src/types/novnc.d.ts new file mode 100644 index 000000000..497fee496 --- /dev/null +++ b/app/src/types/novnc.d.ts @@ -0,0 +1,20 @@ +declare module "@novnc/novnc/lib/rfb" { + type RFBOptions = { + shared?: boolean; + wsProtocols?: string[]; + }; + + export default class RFB extends EventTarget { + constructor(target: HTMLElement, url: string, options?: RFBOptions); + viewOnly: boolean; + scaleViewport: boolean; + resizeSession: boolean; + clipViewport: boolean; + focusOnClick: boolean; + qualityLevel: number; + compressionLevel: number; + disconnect(): void; + focus(): void; + clipboardPasteFrom(text: string): void; + } +} diff --git a/app/tests/avatar-components.test.tsx b/app/tests/avatar-components.test.tsx new file mode 100644 index 000000000..575d62554 --- /dev/null +++ b/app/tests/avatar-components.test.tsx @@ -0,0 +1,54 @@ +import { afterAll, afterEach, beforeAll, expect, test } from "bun:test"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; +import { cleanup, render } from "@testing-library/react"; +import { AbstractAvatar } from "@/components/agents/abstract-avatar"; +import { UserAvatar } from "@/components/people/user-avatar"; +import { AvatarUploadActions } from "@/components/ui/avatar-upload-actions"; + +beforeAll(() => GlobalRegistrator.register()); +afterEach(cleanup); +afterAll(() => GlobalRegistrator.unregister()); + +test("an uploaded coworker image replaces the generated avatar", () => { + const { getByRole } = render( + , + ); + + expect( + getByRole("img", { name: "Researcher" }).querySelector("img")?.src, + ).toBe("data:image/png;base64,chosen"); +}); + +test("a person without an image gets readable initials", () => { + const { getByRole } = render( + , + ); + + expect(getByRole("img", { name: "Ninja Builder" }).textContent).toBe("NB"); +}); + +test("only a custom image offers removal", () => { + const { getByRole, queryByRole, rerender } = render( + {}} + />, + ); + expect(queryByRole("button", { name: "Remove your avatar" })).toBeNull(); + + rerender( + {}} + />, + ); + expect(getByRole("button", { name: "Remove your avatar" })).toBeTruthy(); +}); diff --git a/app/tests/avatar-mutations.test.ts b/app/tests/avatar-mutations.test.ts new file mode 100644 index 000000000..c9ba6529f --- /dev/null +++ b/app/tests/avatar-mutations.test.ts @@ -0,0 +1,70 @@ +import { afterEach, expect, test } from "bun:test"; +import type { QueryClient } from "@tanstack/react-query"; +import { updateAgentAvatarMutationOptions } from "@/lib/agents/mutations"; +import { updateCurrentUserAvatarMutationOptions } from "@/lib/auth/mutations"; + +const realFetch = globalThis.fetch; + +afterEach(() => { + globalThis.fetch = realFetch; +}); + +function harness(response: unknown) { + const requests: { url: string; init?: RequestInit }[] = []; + const invalidated: unknown[] = []; + globalThis.fetch = (async (url: unknown, init?: RequestInit) => { + requests.push({ url: String(url), init }); + return Response.json(response); + }) as typeof fetch; + const queryClient = { + invalidateQueries: async (filter: unknown) => { + invalidated.push(filter); + }, + } as unknown as QueryClient; + return { invalidated, queryClient, requests }; +} + +test("a coworker upload writes the avatar route and refreshes profile and roster images", async () => { + const { invalidated, queryClient, requests } = harness({ agent: {} }); + const options = updateAgentAvatarMutationOptions(queryClient); + const variables = { + agentId: "agent-1", + image: "data:image/png;base64,image", + }; + + await options.mutationFn?.(variables); + await options.onSuccess?.( + {} as never, + variables, + undefined as never, + undefined as never, + ); + + expect(requests[0]?.url).toBe("/api/agents/agent-1/avatar"); + expect(requests[0]?.init?.method).toBe("PUT"); + expect(JSON.parse(String(requests[0]?.init?.body))).toEqual({ + image: variables.image, + }); + expect(invalidated).toEqual([ + { queryKey: ["agents"] }, + { queryKey: ["channels"] }, + ]); +}); + +test("a person upload writes their own route and refreshes the current-user avatar", async () => { + const { invalidated, queryClient, requests } = harness({ avatar: {} }); + const options = updateCurrentUserAvatarMutationOptions(queryClient); + + await options.mutationFn?.(null); + await options.onSuccess?.( + {} as never, + null, + undefined as never, + undefined as never, + ); + + expect(requests[0]?.url).toBe("/api/me/avatar"); + expect(requests[0]?.init?.method).toBe("PUT"); + expect(JSON.parse(String(requests[0]?.init?.body))).toEqual({ image: null }); + expect(invalidated).toEqual([{ queryKey: ["auth", "current-user"] }]); +}); diff --git a/app/tests/avatar.test.ts b/app/tests/avatar.test.ts new file mode 100644 index 000000000..0ad0cfd33 --- /dev/null +++ b/app/tests/avatar.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, test } from "bun:test"; +import { avatarFileError, MAX_AVATAR_BYTES } from "@/lib/avatar"; + +describe("avatar file selection", () => { + test.each(["image/png", "image/jpeg", "image/webp"])( + "accepts %s within the upload limit", + (type) => { + expect(avatarFileError({ size: MAX_AVATAR_BYTES, type })).toBeNull(); + }, + ); + + test("rejects executable, empty, and oversized files before upload", () => { + expect(avatarFileError({ size: 10, type: "image/svg+xml" })).toContain( + "PNG", + ); + expect(avatarFileError({ size: 0, type: "image/png" })).toContain("empty"); + expect( + avatarFileError({ size: MAX_AVATAR_BYTES + 1, type: "image/png" }), + ).toContain("2 MB"); + }); +}); diff --git a/app/tests/computer-control.test.ts b/app/tests/computer-control.test.ts new file mode 100644 index 000000000..6839793e8 --- /dev/null +++ b/app/tests/computer-control.test.ts @@ -0,0 +1,83 @@ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { + controlLease, + readControl, + releaseControl, + sendHumanInput, + takeControl, +} from "@/lib/computers/control"; + +const LEASE = "a".repeat(86); +const originalFetch = globalThis.fetch; +const originalStorage = Object.getOwnPropertyDescriptor( + globalThis, + "sessionStorage", +); + +type SeenRequest = { url: string; init?: RequestInit }; + +function storage(): Storage { + const values = new Map(); + return { + get length() { + return values.size; + }, + clear: () => values.clear(), + getItem: (key) => values.get(key) ?? null, + key: (index) => [...values.keys()][index] ?? null, + removeItem: (key) => values.delete(key), + setItem: (key, value) => values.set(key, String(value)), + }; +} + +beforeEach(() => { + Object.defineProperty(globalThis, "sessionStorage", { + configurable: true, + value: storage(), + }); +}); + +afterEach(() => { + globalThis.fetch = originalFetch; + if (originalStorage) { + Object.defineProperty(globalThis, "sessionStorage", originalStorage); + } else { + Reflect.deleteProperty(globalThis, "sessionStorage"); + } +}); + +function capture(responses: unknown[]) { + const seen: SeenRequest[] = []; + globalThis.fetch = (async (url: unknown, init?: RequestInit) => { + seen.push({ url: String(url), init }); + return Response.json(responses.shift() ?? {}, { status: 200 }); + }) as typeof fetch; + return seen; +} + +test("only the tab that takes control keeps the returned capability", async () => { + const seen = capture([ + { holder: "human", since: "now", requested: false, lease: LEASE }, + { holder: "bot", since: "later", requested: false }, + ]); + + const state = await takeControl("codex"); + expect(state).toEqual({ holder: "human", since: "now", requested: false }); + expect(state).not.toHaveProperty("lease"); + expect(controlLease("codex")).toBe(LEASE); + + await releaseControl("codex"); + expect(JSON.parse(String(seen[1]?.init?.body))).toEqual({ lease: LEASE }); + expect(controlLease("codex")).toBeUndefined(); +}); + +test("seeing that a human is driving does not grant this tab their lease", async () => { + const seen = capture([{ holder: "human", since: "now", requested: false }]); + + expect(await readControl("codex")).toMatchObject({ holder: "human" }); + expect(controlLease("codex")).toBeUndefined(); + sendHumanInput("codex", "click", { x: 10, y: 20 }); + + // The GET is the only request. Input without this tab's capability fails closed in the client. + expect(seen).toHaveLength(1); +}); diff --git a/app/tests/computer-view-state.test.ts b/app/tests/computer-view-state.test.ts new file mode 100644 index 000000000..ca302ed7e --- /dev/null +++ b/app/tests/computer-view-state.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, test } from "bun:test"; +import { desktopOverlayProblem } from "@/components/computer/computer-view"; + +describe("the expanded computer error overlay", () => { + test("does not cover a healthy VM when only the managed browser is closed", () => { + expect( + desktopOverlayProblem( + "The browser is closed. Open it from the Browser icon on the computer desktop.", + null, + ), + ).toBeNull(); + }); + + test("shows a problem from the desktop connection itself", () => { + expect(desktopOverlayProblem(null, "The display disconnected.")).toBe( + "The display disconnected.", + ); + }); +}); diff --git a/app/tests/remote-desktop.test.tsx b/app/tests/remote-desktop.test.tsx new file mode 100644 index 000000000..864490c6f --- /dev/null +++ b/app/tests/remote-desktop.test.tsx @@ -0,0 +1,98 @@ +import { act, cleanup, render, waitFor } from "@testing-library/react"; +import { afterAll, afterEach, expect, mock, test } from "bun:test"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; + +type Listener = (event: Event & { detail?: { clean?: boolean } }) => void; + +class FakeRFB { + static instances: FakeRFB[] = []; + + readonly listeners = new Map>(); + viewOnly = false; + scaleViewport = false; + resizeSession = false; + clipViewport = false; + focusOnClick = false; + qualityLevel = 0; + compressionLevel = 0; + + constructor( + readonly target: HTMLElement, + readonly url: string, + readonly options?: { wsProtocols?: string[] }, + ) { + FakeRFB.instances.push(this); + } + + addEventListener(type: string, listener: Listener) { + const listeners = this.listeners.get(type) ?? new Set(); + listeners.add(listener); + this.listeners.set(type, listeners); + } + + removeEventListener(type: string, listener: Listener) { + this.listeners.get(type)?.delete(listener); + } + + emit(type: string, detail: { clean?: boolean } = {}) { + const event = Object.assign(new Event(type), { detail }); + for (const listener of this.listeners.get(type) ?? []) listener(event); + } + + focus() {} + + disconnect() {} +} + +mock.module("@novnc/novnc/lib/rfb", () => ({ default: FakeRFB })); + +GlobalRegistrator.register(); +const { RemoteDesktop } = await import("@/components/computer/remote-desktop"); + +afterEach(() => { + cleanup(); + FakeRFB.instances = []; +}); +afterAll(() => GlobalRegistrator.unregister()); + +test("reconnects when the remote display closes its socket cleanly", async () => { + const problems: Array = []; + render( + problems.push(problem)} + />, + ); + + expect(FakeRFB.instances).toHaveLength(1); + act(() => FakeRFB.instances[0]?.emit("connect")); + act(() => FakeRFB.instances[0]?.emit("disconnect", { clean: true })); + + expect(problems.at(-1)).toBe( + "The computer display disconnected. Reconnecting…", + ); + await waitFor(() => expect(FakeRFB.instances).toHaveLength(2), { + timeout: 1_500, + }); +}); + +test("uses interactive semantics only while the person has control", () => { + const { getByRole, rerender } = render( + , + ); + expect( + getByRole("img", { name: "The assistant's full computer, live" }), + ).toBeTruthy(); + + rerender(); + expect( + getByRole("application", { + name: "The assistant's full computer. You have control.", + }), + ).toBeTruthy(); + expect(FakeRFB.instances.at(-1)?.options?.wsProtocols).toEqual([ + "binary", + `openbot-lease.${"a".repeat(86)}`, + ]); +}); diff --git a/bun.lock b/bun.lock index 415b9ce62..692d00c39 100644 --- a/bun.lock +++ b/bun.lock @@ -13,6 +13,18 @@ "yaml": "^2.9.0", }, }, + "agent-codex": { + "name": "@openbot/agent-codex", + "version": "0.0.2", + "dependencies": { + "@ag-ui/core": "0.0.57", + "@ag-ui/encoder": "0.0.57", + }, + "devDependencies": { + "@types/bun": "^1.3.3", + "typescript": "^5.9.3", + }, + }, "app": { "name": "app", "version": "0.0.0", @@ -22,6 +34,7 @@ "@better-auth/sso": "^1.7.1", "@copilotkit/react-core": "1.69.0", "@fontsource-variable/inter": "^5.3.0", + "@novnc/novnc": "1.5.0", "@shadcn/react": "^0.3.0", "@tabler/icons-react": "^3.36.1", "@tanstack/react-form": "^1.33.5", @@ -457,6 +470,10 @@ "@nodelib/fs.walk": ["@nodelib/fs.walk@1.2.8", "", { "dependencies": { "@nodelib/fs.scandir": "2.1.5", "fastq": "^1.6.0" } }, "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg=="], + "@novnc/novnc": ["@novnc/novnc@1.5.0", "", {}, "sha512-4yGHOtUCnEJUCsgEt/L78eeJu00kthurLBWXFiaXfonNx0pzbs6R/3gJb1byZe6iAE8V9MF0syQb0xIL8MSOtQ=="], + + "@openbot/agent-codex": ["@openbot/agent-codex@workspace:agent-codex"], + "@opentelemetry/api": ["@opentelemetry/api@1.9.1", "", {}, "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q=="], "@opentelemetry/semantic-conventions": ["@opentelemetry/semantic-conventions@1.43.0", "", {}, "sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg=="], diff --git a/charts/openbot/Chart.yaml b/charts/openbot/Chart.yaml index b6e4f3d8c..540c6ebdc 100644 --- a/charts/openbot/Chart.yaml +++ b/charts/openbot/Chart.yaml @@ -3,9 +3,9 @@ name: openbot description: Run OpenBot on any Kubernetes cluster, managed or your own type: application # The chart's own version, bumped when templates or defaults change. -version: 0.1.0 +version: 0.2.0 # The OpenBot release this chart's default image tag points at. -appVersion: "0.0.4" +appVersion: "0.0.5" home: https://github.com/CopilotKit/OpenBot sources: - https://github.com/CopilotKit/OpenBot diff --git a/charts/openbot/README.md b/charts/openbot/README.md index d778f1f6b..cf700e034 100644 --- a/charts/openbot/README.md +++ b/charts/openbot/README.md @@ -15,7 +15,7 @@ node pool being the wrong shape is the last thing anybody checks. Either run amd architecture you have and push it somewhere the cluster can reach. Check before assuming: ```sh -docker manifest inspect ghcr.io/copilotkit/openbot:v0.0.4 | grep architecture +docker manifest inspect ghcr.io/copilotkit/openbot:v0.0.5 | grep architecture ``` **Intelligence credentials.** OpenBot requires CopilotKit Intelligence and the chart refuses to @@ -214,9 +214,9 @@ That only happens in multi-zone clusters, so it passes every single-zone test. ### Storage has gravity -The API tier holds nothing on disk. When per-Bot computers arrive they will, and the ordinary block -volume on all three clouds is **zonal**: once provisioned, every pod referencing it is scheduled into -that zone, so a Bot's computer is pinned to a zone for as long as its profile exists. That is +The API tier holds nothing on disk. Per-Bot computers do, and the ordinary block volume on all three +clouds is **zonal**: once provisioned, every pod referencing it is scheduled into that zone, so a +Bot's computer is pinned to a zone for as long as its profile exists. That is acceptable and worth stating rather than discovering. `storageClass` stays empty by default, meaning the cluster's default class, because naming `gp3` or `pd-balanced` here is how a chart stops installing on somebody's bare-metal cluster. @@ -263,7 +263,7 @@ cannot be told apart from another Bot's is refused rather than filed under the w conversation still names the page it opened; it just does not show it, and the server log says why each time. -With `computers.mode: sandbox` or `external`, each Bot has a computer of its own, there is nobody to +With `computers.mode: sandbox`, `vm`, or `external`, each Bot has a computer of its own, there is nobody to race with, and this does not arise. ## A computer for each Bot @@ -274,6 +274,7 @@ race with, and this does not arise. | --- | --- | --- | | `shared` | One browser for every Bot, run by this chart. | Nothing. | | `sandbox` | A computer each, suspended when idle and resumed with its logins intact. | The `agent-sandbox` controller in the cluster. | +| `vm` | A full graphical computer for each Bot inside its own lightweight VM, suspended when idle and resumed with its logins intact. | The `agent-sandbox` controller and a VM-backed RuntimeClass such as Kata Containers. | | `external` | Neither; `computers.url` points at one somebody else runs. | Nothing. | `shared` is what a first install should use. Sessions, files and logins are shared between Bots in @@ -283,6 +284,18 @@ that mode, which is stated on the fleet page rather than hidden. workload: an isolated, stateful, singleton pod with a stable identity and persistent storage. Suspending is `operatingMode: Suspended`, which terminates the pod and keeps the volumes. +`vm` uses that same lifecycle but refuses to render without `computers.runtimeClassName`. Point it at +a VM-backed runtime such as `kata-qemu`: each Bot's browser, desktop, shell, profile and workspace +then live behind their own guest kernel rather than sharing the node kernel. The chart can require a +RuntimeClass name but cannot inspect what an operator mapped that name to, so verifying the class is +Kata or another microVM runtime remains a cluster installation step. + +```yaml +computers: + mode: vm + runtimeClassName: kata-qemu +``` + **That controller is not installed by this chart, and the chart refuses to install without it.** The check reads the cluster, so it is a real answer rather than a value somebody has to remember: diff --git a/charts/openbot/templates/_helpers.tpl b/charts/openbot/templates/_helpers.tpl index 059653a62..9cc5c0121 100644 --- a/charts/openbot/templates/_helpers.tpl +++ b/charts/openbot/templates/_helpers.tpl @@ -166,7 +166,7 @@ and in whatever holds the release, which is not where `KEY_ENCRYPTION_KEY` belon {{- else if and (eq .Values.computers.mode "external") .Values.computers.url }} - name: AGENT_COMPUTER_URL value: {{ .Values.computers.url | quote }} -{{- else if eq .Values.computers.mode "sandbox" }} +{{- else if has .Values.computers.mode (list "sandbox" "vm") }} - name: COMPUTER_SANDBOX_NAMESPACE value: {{ default .Release.Namespace .Values.computers.sandbox.namespace | quote }} - name: COMPUTER_SANDBOX_IDLE_AFTER @@ -371,7 +371,8 @@ be able to address the API server at all, and the default is the wrong way round "name" "computer" "image" (include "openbot.image" .) "imagePullPolicy" .Values.image.pullPolicy - "command" (list "/usr/local/bin/bun" "/app/agent-computer/src/index.ts") + "command" (list "/bin/bash" "-lc") + "args" (list "if [ -x /app/agent-computer/entrypoint.sh ]; then exec /app/agent-computer/entrypoint.sh; fi; exec /usr/local/bin/bun /app/agent-computer/src/index.ts") "ports" (list (dict "name" "http" "containerPort" 4100)) "env" (concat (list @@ -424,12 +425,11 @@ be able to address the API server at all, and the default is the wrong way round Whether the API pod gets a Kubernetes token. FALSE UNLESS IT ACTUALLY NEEDS ONE. The API talks to a database and to Bots, not to the cluster, so a -mounted token is a credential sitting in a pod that has no use for it. `computers.mode: sandbox` is -the exception and the only one: there the server asks the API server to create, resume and suspend a +mounted token is a credential sitting in a pod that has no use for it. Per-Bot `sandbox` and `vm` +modes are the exceptions: there the server asks the API server to create, resume and suspend a Sandbox per Bot, and without a token it fails on the first browser action with a missing file rather than anything that names the cause. */}} {{- define "openbot.automountToken" -}} -{{- or .Values.serviceAccount.automountServiceAccountToken (eq .Values.computers.mode "sandbox") -}} +{{- or .Values.serviceAccount.automountServiceAccountToken (has .Values.computers.mode (list "sandbox" "vm")) -}} {{- end -}} - diff --git a/charts/openbot/templates/computer/culler-cronjob.yaml b/charts/openbot/templates/computer/culler-cronjob.yaml index 997d9e21f..62772053a 100644 --- a/charts/openbot/templates/computer/culler-cronjob.yaml +++ b/charts/openbot/templates/computer/culler-cronjob.yaml @@ -1,4 +1,4 @@ -{{- if and (eq .Values.computers.mode "sandbox") .Values.computers.sandbox.culler.enabled }} +{{- if and (has .Values.computers.mode (list "sandbox" "vm")) .Values.computers.sandbox.culler.enabled }} {{- $component := "culler" -}} {{/* Suspending computers nobody is using. diff --git a/charts/openbot/templates/computer/pod-template.yaml b/charts/openbot/templates/computer/pod-template.yaml index 67e39a22a..9640d91b1 100644 --- a/charts/openbot/templates/computer/pod-template.yaml +++ b/charts/openbot/templates/computer/pod-template.yaml @@ -1,4 +1,4 @@ -{{- if eq .Values.computers.mode "sandbox" }} +{{- if has .Values.computers.mode (list "sandbox" "vm") }} {{/* What a Bot's computer looks like, handed to the server as a file. diff --git a/charts/openbot/templates/computer/sandbox-rbac.yaml b/charts/openbot/templates/computer/sandbox-rbac.yaml index 2bf66d530..37da2a008 100644 --- a/charts/openbot/templates/computer/sandbox-rbac.yaml +++ b/charts/openbot/templates/computer/sandbox-rbac.yaml @@ -1,4 +1,4 @@ -{{- if eq .Values.computers.mode "sandbox" }} +{{- if has .Values.computers.mode (list "sandbox" "vm") }} {{- $ns := default .Release.Namespace .Values.computers.sandbox.namespace -}} {{/* What the API may do to computers, and nothing else. diff --git a/charts/openbot/templates/computer/sandbox-template.yaml b/charts/openbot/templates/computer/sandbox-template.yaml index 900465949..579617ba8 100644 --- a/charts/openbot/templates/computer/sandbox-template.yaml +++ b/charts/openbot/templates/computer/sandbox-template.yaml @@ -1,4 +1,4 @@ -{{- if and (eq .Values.computers.mode "sandbox") .Values.computers.sandbox.warmPool.enabled }} +{{- if and (has .Values.computers.mode (list "sandbox" "vm")) .Values.computers.sandbox.warmPool.enabled }} {{- $ns := default .Release.Namespace .Values.computers.sandbox.namespace -}} {{/* The template a warm pool cuts pre-warmed computers from. diff --git a/charts/openbot/templates/computer/statefulset.yaml b/charts/openbot/templates/computer/statefulset.yaml index f20fffcdb..550936919 100644 --- a/charts/openbot/templates/computer/statefulset.yaml +++ b/charts/openbot/templates/computer/statefulset.yaml @@ -8,9 +8,10 @@ directory holds real logins, and a Deployment's pods get no stable volume of the would fight over one `ReadWriteOnce` claim and a rollout would hand the new pod an empty profile, which reads as every Bot being signed out of everything at once. -`replicas: 1` on purpose in this mode. One browser for every Bot is what `shared` means, and it is -the mode that needs no CRD in the cluster. `computers.mode: sandbox` gives each Bot its own, and is -where the idle suspend lives; see the provider RBAC beside this file. +`replicas: 1` on purpose in this mode. One desktop for every Bot is what `shared` means, and it is +the mode that needs no CRD in the cluster. `computers.mode: sandbox` gives each Bot its own container; +`computers.mode: vm` adds a required Kata runtime class. Both use the same idle suspend lifecycle; +see the provider RBAC beside this file. STORAGE HAS GRAVITY. The volume below is `ReadWriteOnce`, and on all three clouds the ordinary block volume is zonal, so this pod is pinned to whichever zone its volume was created in for as long as @@ -110,7 +111,17 @@ spec: One image serves both roles and the command decides which: this skips s6 and runs the browser process directly, so a computer pod carries no API and no database. */}} - command: ["/usr/local/bin/bun", "/app/agent-computer/src/index.ts"] + # A checkout can be newer than the last published image. The graphical entrypoint arrived + # after v0.0.5, so fall back to that image's headless computer instead of CrashLooping while + # the next release is being prepared. The release workflow moves appVersion to the image + # built from the same tree, which activates the full desktop by default after publication. + command: ["/bin/bash", "-lc"] + args: + - | + if [ -x /app/agent-computer/entrypoint.sh ]; then + exec /app/agent-computer/entrypoint.sh + fi + exec /usr/local/bin/bun /app/agent-computer/src/index.ts ports: - name: http containerPort: 4100 diff --git a/charts/openbot/templates/computer/warmpool.yaml b/charts/openbot/templates/computer/warmpool.yaml index 1cac665b2..ca889330e 100644 --- a/charts/openbot/templates/computer/warmpool.yaml +++ b/charts/openbot/templates/computer/warmpool.yaml @@ -1,4 +1,4 @@ -{{- if and (eq .Values.computers.mode "sandbox") .Values.computers.sandbox.warmPool.enabled }} +{{- if and (has .Values.computers.mode (list "sandbox" "vm")) .Values.computers.sandbox.warmPool.enabled }} {{- $ns := default .Release.Namespace .Values.computers.sandbox.namespace -}} {{/* Computers waiting, so a Bot's first action after lunch does not wait for Chromium to boot. diff --git a/charts/openbot/templates/networkpolicy.yaml b/charts/openbot/templates/networkpolicy.yaml index ef80cb8fb..30e7aa853 100644 --- a/charts/openbot/templates/networkpolicy.yaml +++ b/charts/openbot/templates/networkpolicy.yaml @@ -92,7 +92,7 @@ spec: protocol: TCP - port: 443 protocol: TCP - {{- if eq .Values.computers.mode "sandbox" }} + {{- if has .Values.computers.mode (list "sandbox" "vm") }} {{- /* The Kubernetes API server, which is where a per-Bot computer is asked for. @@ -181,7 +181,7 @@ spec: {{- end }} {{- end }} -{{- if and .Values.networkPolicy.enabled (eq .Values.computers.mode "sandbox") .Values.computers.sandbox.culler.enabled }} +{{- if and .Values.networkPolicy.enabled (has .Values.computers.mode (list "sandbox" "vm")) .Values.computers.sandbox.culler.enabled }} --- {{- $culler := "culler" -}} {{/* A pod no policy selects keeps the cluster default, so this one was the release's only unfenced one. */}} diff --git a/charts/openbot/templates/server/deployment.yaml b/charts/openbot/templates/server/deployment.yaml index 2ed9af3e9..cfb045cdb 100644 --- a/charts/openbot/templates/server/deployment.yaml +++ b/charts/openbot/templates/server/deployment.yaml @@ -43,7 +43,7 @@ spec: the old values, and the deployment looks upgraded while behaving exactly as it did. */}} checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} - {{- if eq .Values.computers.mode "sandbox" }} + {{- if has .Values.computers.mode (list "sandbox" "vm") }} {{- /* The shape of a computer, which the server reads once and keeps. @@ -136,7 +136,7 @@ spec: periodSeconds: {{ .Values.server.livenessProbe.periodSeconds }} timeoutSeconds: {{ .Values.server.livenessProbe.timeoutSeconds }} failureThreshold: {{ .Values.server.livenessProbe.failureThreshold }} - {{- if eq .Values.computers.mode "sandbox" }} + {{- if has .Values.computers.mode (list "sandbox" "vm") }} {{- /* The shape of a Bot's computer, as a file rather than as a permission to read one. */}} volumeMounts: - name: sandbox-template @@ -147,7 +147,7 @@ spec: resources: {{ toYaml . | indent 12 }} {{- end }} - {{- if eq .Values.computers.mode "sandbox" }} + {{- if has .Values.computers.mode (list "sandbox" "vm") }} volumes: - name: sandbox-template configMap: diff --git a/charts/openbot/templates/validation.yaml b/charts/openbot/templates/validation.yaml index 08f2e811c..55678e005 100644 --- a/charts/openbot/templates/validation.yaml +++ b/charts/openbot/templates/validation.yaml @@ -97,7 +97,7 @@ This template renders nothing. {{- /* Asking for per-Bot computers on a cluster that cannot make them. - `computers.mode: sandbox` creates `Sandbox` objects, which only exist once the agent-sandbox + `computers.mode: sandbox` and `vm` create `Sandbox` objects, which only exist once the agent-sandbox controller is installed. Without it the install succeeds, every pod is healthy, and the deployment looks finished right up until the first Bot asks for a browser and gets a 404 from the API server. That is the worst time to learn it, so it is a refusal here instead, with the command to run. @@ -106,12 +106,24 @@ This template renders nothing. `helm template` with no cluster has no way to know, which is what `--api-versions` is for and what the chart's own render tests pass. */}} -{{- if and (eq .Values.computers.mode "sandbox") .Values.computers.sandbox.requireController }} +{{- if and (has .Values.computers.mode (list "sandbox" "vm")) .Values.computers.sandbox.requireController }} {{- if not (.Capabilities.APIVersions.Has "agents.x-k8s.io/v1beta1/Sandbox") }} -{{- fail "computers.mode is sandbox, which gives every Bot its own computer, but this cluster has no Sandbox CRD. Install the controller first:\n\n kubectl apply --server-side -f https://github.com/kubernetes-sigs/agent-sandbox/releases/download/v0.5.6/sandbox-with-extensions.yaml\n\nOr use computers.mode: shared, which needs nothing installed. Rendering without a cluster? Pass --api-versions agents.x-k8s.io/v1beta1/Sandbox, or set computers.sandbox.requireController=false." }} +{{- fail "This computer mode gives every Bot its own computer, but this cluster has no Sandbox CRD. Install the controller first:\n\n kubectl apply --server-side -f https://github.com/kubernetes-sigs/agent-sandbox/releases/download/v0.5.6/sandbox-with-extensions.yaml\n\nOr use computers.mode: shared, which needs nothing installed. Rendering without a cluster? Pass --api-versions agents.x-k8s.io/v1beta1/Sandbox, or set computers.sandbox.requireController=false." }} {{- end }} {{- end }} +{{- /* + Calling an ordinary pod a VM would be security theatre. + + `vm` uses the same stateful Sandbox lifecycle, but requires a RuntimeClass supplied by a VM-backed + runtime such as Kata Containers. The chart cannot prove what a cluster's arbitrary RuntimeClass + name points at; requiring one prevents the known-bad case where `vm` quietly means the default + shared-kernel runtime. +*/}} +{{- if and (eq .Values.computers.mode "vm") (not .Values.computers.runtimeClassName) }} +{{- fail "computers.mode is vm but computers.runtimeClassName is empty. Set it to a VM-backed RuntimeClass such as kata-qemu; an ordinary pod is not a virtual machine." }} +{{- end }} + {{- /* A pool of warm computers that nothing takes one from. @@ -129,12 +141,12 @@ This template renders nothing. Remove this block when the provider claims from the pool. It is a note that the wiring is unfinished, not a judgement that the feature is unwanted. */}} -{{- if and (eq .Values.computers.mode "sandbox") .Values.computers.sandbox.warmPool.enabled }} +{{- if and (has .Values.computers.mode (list "sandbox" "vm")) .Values.computers.sandbox.warmPool.enabled }} {{- fail "computers.sandbox.warmPool.enabled is on, but nothing claims from the pool yet: the server creates a Sandbox per Bot and never a SandboxClaim, so the pool would run and bill without ever shortening a first action. Set computers.sandbox.warmPool.enabled=false until claiming ships." }} {{- end }} -{{- if not (has .Values.computers.mode (list "shared" "sandbox" "external")) }} -{{- fail "computers.mode must be one of: shared, sandbox, external." }} +{{- if not (has .Values.computers.mode (list "shared" "sandbox" "vm" "external")) }} +{{- fail "computers.mode must be one of: shared, sandbox, vm, external." }} {{- end }} {{- /* diff --git a/charts/openbot/values.yaml b/charts/openbot/values.yaml index 7bd6f2cc3..bde5d2d54 100644 --- a/charts/openbot/values.yaml +++ b/charts/openbot/values.yaml @@ -186,6 +186,8 @@ config: # and is what a first install should use. # sandbox A computer each, as a `Sandbox` from kubernetes-sigs/agent-sandbox, suspended when # idle and resumed with its logins intact. Needs that controller in the cluster. +# vm The same persistent computer lifecycle inside a VM-backed RuntimeClass such as Kata. +# Requires `runtimeClassName`; the chart refuses to call an ordinary pod a VM. # external Neither. `url` points at a computer somebody else runs. computers: mode: shared @@ -194,7 +196,8 @@ computers: # Never a literal. See `secrets` below. existingTokenSecret: "" - # gVisor or Kata, where the cluster has one. Unset means an ordinary pod, because the three clouds + # gVisor or Kata, where the cluster has one. Required for `mode: vm` and optional for `sandbox`. + # Unset means an ordinary pod, because the three clouds # do not agree: GKE has managed gVisor, AKS offers Kata and gVisor, and on EKS it is bring your own # node configuration. A chart that assumed a RuntimeClass would fail to install without one. runtimeClassName: "" @@ -232,7 +235,7 @@ computers: # `networkPolicy.computerExtraEgress`, or the computer cannot reach it. extraEnv: [] - # `mode: sandbox` only. Where the per-Bot computers are created and what may create them. + # `mode: sandbox` and `mode: vm` only. Where per-Bot computers are created and what may create them. sandbox: # Refuse to install when the cluster has no Sandbox CRD, rather than succeeding and failing at # the first browser action. Turn it off only where the CRD arrives after this chart does, such as diff --git a/docker/s6/s6-rc.d/computer/run b/docker/s6/s6-rc.d/computer/run index d671a813b..2c2e8c029 100755 --- a/docker/s6/s6-rc.d/computer/run +++ b/docker/s6/s6-rc.d/computer/run @@ -19,4 +19,13 @@ fi cd /app/agent-computer export PORT=4100 export WORKSPACE_DIR=/workspace -exec s6-setuidgid pwuser /usr/local/bin/bun src/index.ts +mkdir -p /tmp/.X11-unix /tmp/runtime-pwuser +chmod 1777 /tmp/.X11-unix +chown pwuser:pwuser /tmp/runtime-pwuser +chmod 700 /tmp/runtime-pwuser +exec s6-setuidgid pwuser env \ + HOME=/home/pwuser \ + USER=pwuser \ + LOGNAME=pwuser \ + XDG_RUNTIME_DIR=/tmp/runtime-pwuser \ + ./entrypoint.sh diff --git a/docs/README.md b/docs/README.md index 6b7c66687..a0ffee5a2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -5,7 +5,8 @@ Start with the root [README](../README.md), then use these references: - [Architecture](architecture.md): services, ports, browser governance, computers, components, plugins, knowledge, and security boundaries. - [Configuration](configuration.md): environment variables and tenant package YAML. - [Development](development.md): local setup, migrations, ports, and quality checks. -- [Coworkers](coworkers.md): durable Bot profiles, channels, visibility, deletion, and external AG-UI registration. +- [Coworkers](coworkers.md): durable Bot profiles, custom avatars, channels, visibility, deletion, and external AG-UI registration. +- [Local Codex coworker](../agent-codex/README.md): persistent Codex threads through OpenBot's governed tool gateway, using the ChatGPT account already signed in on the host. - [Routines](routines.md): standing instructions a Bot runs on a schedule, the worker that fires them, and who they run as. - Plugins, one connector per page — what an administrator registers, what each person consents to, and what the failures mean: - [Google Drive](plugins/google-drive.md) diff --git a/docs/architecture.md b/docs/architecture.md index 46b4e2def..21d1739e7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,6 +1,6 @@ # Architecture -OpenBot combines a React app, a Hono API server, PostgreSQL, CopilotKit Intelligence, AG-UI Bot endpoints, and governed browser computers. +OpenBot combines a React app, a Hono API server, PostgreSQL, CopilotKit Intelligence, AG-UI Bot endpoints, and governed graphical computers. @@ -15,14 +15,18 @@ Regenerate it with `bun run diagram` after changing anything it shows. | ------------------------ | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | | `app` | 3010 | React/Vite interface for channels, Bot chat, live screen, settings, and admin pages. | | `server` | 3001 | API, CopilotKit runtime, auth, roles, tenant package, coworkers, channels, policy, audit, credentials, plugins, components, and connectors. | -| `agent-computer` | 4100 | Chromium, `/workspace`, browser profile, screenshots, snapshots, and file tools. | +| `agent-computer` | 4100 | A Linux desktop, Chromium, terminal, `/workspace`, browser profile, screenshots, snapshots, and file tools. | | `agent-bot` | 4200 | Proof-of-concept AG-UI Bot. | | `agent-langgraph` | 4201 | LangGraph AG-UI Bot. | +| `agent-codex` | 4202 | Optional host-side AG-UI adapter for the locally authenticated Codex app-server. | | `supervisor` | 4500 host / 4300 container | Creates, stops, resets, and lists per-Bot computer containers. | | PostgreSQL with pgvector | 5432 | Product data, audit rows, credentials, policy, grants, channels, and components. | | CopilotKit Intelligence | external | Durable threads, memory, and realtime gateway. | -`scripts/start.sh` starts PostgreSQL, `agent-computer`, `agent-bot`, `agent-langgraph`, and the supervisor through Docker Compose, then starts `server` and `app` on the host. +`scripts/start.sh` starts PostgreSQL, `agent-computer`, and the supervisor through Docker Compose, +then starts `server` and `app` on the host. It normally starts `agent-bot` and `agent-langgraph` in +Compose too. With `CODEX_AGENT_ENABLED=true`, it skips those two and starts `agent-codex` on the host +so the adapter can reuse the existing `codex login` session. The compose file also defines optional SPIRE services. `start.sh` does not start them. @@ -73,6 +77,12 @@ Compose puts it on a different network from PostgreSQL. A Bot has a shell, and a With `COMPUTER_SUPERVISOR_URL`, each Bot gets its own computer container, workspace volume, and browser profile. Without it, all Bots share `AGENT_COMPUTER_URL`. +An isolated computer publishes its whole framebuffer as read-only and control RFB sockets behind the +server's same-origin WebSocket proxy. That is what shows browser chrome, native dialogs, the desktop +panel and terminal rather than one CDP page canvas. A shared provider stays on the page stream because +its process-wide desktop could contain another Bot's window. Older computer images without the RFB +capability fall back to that page stream during an upgrade. + A command on the computer inherits PATH, locale and terminal names, and the proxy variables, not the rest of the process environment. Userinfo is stripped from a proxy URL. `COMPUTER_SHELL_ENV` names anything else a deployment wants passed. The supervisor exposes only ensure, stop, reset, and list operations. It holds the Docker socket, so do not expose it outside the deployment network: Docker Compose binds it to `127.0.0.1:4500`, and a deployment running the server inside the compose network reaches it as `supervisor:4300` and needs no published port at all. Set `COMPUTER_RUNTIME=runsc` to run computers under gVisor on hosts that support it. @@ -91,7 +101,11 @@ Secret entry is separate from chat content. The audit trail records that a secre ## Watching a Bot work -Two surfaces beside the conversation. The screen is the live browser, proxied over a websocket and gated on the same question as every other route about that Bot. The Activity tab is what the Bot did away from the browser: every command with its output and exit code, every file read, write and listing, newest first. +Two surfaces sit beside the conversation. For an isolated Bot, the screen is its full live desktop; +for a shared provider, it is the Bot-scoped browser page. Both are proxied over a WebSocket and gated +on the same question as every other route about that Bot. The Activity tab is what the Bot did away +from the browser: every command with its output and exit code, every file read, write and listing, +newest first. Activity is held in the browser for the open conversation and is gone on reload. It is a window rather than a record; the record is the audit trail, which is server-side, survives restarts, and is what an investigation reads. A saved file contributes its path and size and never its contents, matching the write route, which declines to echo them because a Bot may be saving something it was told in confidence. @@ -100,11 +114,17 @@ Activity is held in the browser for the open conversation and is gone on reload. A coworker is a durable Bot profile: - `agents` stores runtime identity and endpoint/key reference. -- `agent_profiles` stores name, title, role, owner, visibility, and deletion state. +- `agent_profiles` stores name, title, role, avatar seed or custom image, owner, visibility, and + deletion state. - `agent_preferences` stores per-user roster state. A channel is a conversation with one coworker and a CopilotKit Intelligence thread mapping. Starting a new channel creates a new thread. +A person's optional custom avatar lives on their `users` row. Both person and Bot images are served +from authenticated, private, versioned routes; list responses carry those short URLs and never the +base64 image payloads. That keeps replica behavior shared through PostgreSQL without turning every +roster read into a multi-megabyte response. + Who may reach one is decided by membership: every channel route resolves the caller in `channel_memberships` and refuses without a row. `channels.allowed_groups` is declared in the tenant package and stored, and is not part of that decision — `users.groups` is never populated by diff --git a/docs/configuration.md b/docs/configuration.md index 738d1a1d8..50300438c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -31,8 +31,8 @@ All four Intelligence values are required together. Missing any of them stops se the product. It needs `MANAGED_AGENT_TOKEN` beside it, or the server refuses to start. Unset, the server starts without a managed Bot, the shipped Risk Analyst coworker is omitted, and creating a coworker without its own endpoint is refused. A leftover token with no URL is ignored. The -one-container image has no Bot process, so leave the URL unset there. `scripts/start.sh` points it -at `agent-langgraph` on a laptop. +one-container image has no Bot process, so leave the URL unset there. `scripts/start.sh` normally +points it at `agent-langgraph` on a laptop, or at `agent-codex` when local Codex mode is enabled. ## General variables @@ -42,7 +42,7 @@ at `agent-langgraph` on a laptop. | `NODE_ENV` | unset | `production` refuses the example `KEY_ENCRYPTION_KEY`. It does not decide whether sign-in is required; see `OPENBOT_SINGLE_USER`. | | `TENANT_PACKAGE_DIR` | `../examples/fintech` | Tenant package directory, resolved from `server/`. | | `DEPLOYMENT_ID` | the tenant package's id | Names this deployment inside a shared Intelligence project. | -| `OPENAI_API_KEY` | unset | Default model key for built-in agents and both shipped Bots. | +| `OPENAI_API_KEY` | unset | Default model key for built-in agents and both provider-key Bots. Not needed for local Codex mode. | | `OPENAI_BASE_URL` | unset | OpenAI-compatible endpoint that key is spent against. See below. | | `BOT_PROVIDER` | `openai` | Provider for `agent-langgraph`: `openai`, `anthropic`, or `google`. | | `ANTHROPIC_API_KEY` | unset | Anthropic key when `BOT_PROVIDER=anthropic`. | @@ -133,6 +133,34 @@ where the worker runs rather than a fact about the deployment `loadConfig` descr it at the server's own port on a laptop; the Helm chart's routines CronJob points it at the server's in-cluster Service address. +## Local Codex coworker + +Local Codex mode runs a host-side AG-UI adapter against the Codex app-server already authenticated by +`codex login`. It is an alternative to starting the two provider-key Bot containers: + +```dotenv +CODEX_AGENT_ENABLED=true +AGENT_ENDPOINT_ALLOWED_HOSTS=localhost:4202 +``` + +`scripts/start.sh` then makes `http://localhost:4202/ag-ui` the default managed coworker endpoint, +starts the adapter on the host, and keeps its Codex thread mapping under `.openbot-codex/`. An explicit +`MANAGED_AGENT_AG_UI_URL` still wins, so remove an old `localhost:4201` value when switching modes. + +| Variable | Default | Meaning | +| ----------------------------- | ------------------------------------ | ------- | +| `CODEX_AGENT_ENABLED` | `false` | Starts the local Codex adapter and skips `agent-bot` and `agent-langgraph`. | +| `CODEX_AGENT_PORT` | `4202` | Host port for the adapter. | +| `CODEX_AGENT_WORKSPACE` | `.openbot-codex/workspace` | Working directory passed to Codex turns. | +| `CODEX_AGENT_STATE` | `.openbot-codex/threads.json` | Durable OpenBot-thread to Codex-thread mapping. | +| `CODEX_AGENT_TURN_TIMEOUT_MS` | `180000` | Maximum time allowed for one Codex app-server turn. | +| `CODEX_BINARY` | `codex` | Codex CLI binary to launch. | +| `OPENBOT_TOOL_URL` | local server `/api/agent-tools/call` | Callback where assigned tools re-enter OpenBot's grant, policy and audit gateway. | + +The start script supplies `MANAGED_AGENT_TOKEN` and `AGENT_TOOL_TOKEN`. When running the adapter by +hand, supply both plus `OPENBOT_TOOL_URL` and the state paths yourself. See +[the adapter guide](../agent-codex/README.md). + ## OpenAI-compatible endpoints `OPENAI_BASE_URL` decides where an OpenAI-shaped request is answered. Unset, that is OpenAI. Set, it is any endpoint speaking the same API: a gateway in front of several providers, a proxy, or a model on hardware you control. @@ -333,6 +361,7 @@ When optional SPIRE services are used: | `agent-computer` | 4100 | `COMPUTER_PORT` | | `agent-bot` | 4200 | `BOT_PORT` | | `agent-langgraph` | 4201 | `LANGGRAPH_PORT` | +| `agent-codex` | 4202 | `CODEX_AGENT_PORT` | | `supervisor` | 4500 host / 4300 container | `SUPERVISOR_PORT` | | PostgreSQL | 5432 | `POSTGRES_PORT` | @@ -340,6 +369,7 @@ Set these in `.env` or in the environment. `docker-compose.yml` publishes on the `scripts/start.sh` reads the same names to decide where to look, so one setting moves a service and everything that talks to it. The addresses built from them are separate settings, so a moved service also needs its URL changed: `DATABASE_URL`, `AGENT_COMPUTER_URL` and `MANAGED_AGENT_AG_UI_URL`. +Codex mode derives its managed URL from `CODEX_AGENT_PORT` unless that URL is set explicitly. To run two deployments on one Docker host, give the second one its own `COMPOSE_PROJECT_NAME`, `COMPUTER_NAMESPACE` and `COMPUTER_IMAGE`. Container and volume names are global to a host, and the diff --git a/docs/coworkers.md b/docs/coworkers.md index 37a40e80e..adfbc6cd0 100644 --- a/docs/coworkers.md +++ b/docs/coworkers.md @@ -7,7 +7,7 @@ A coworker is a Bot with a durable profile and standing role. The role is sent w | Piece | Table | Purpose | | -------------------- | ------------------------------- | --------------------------------------------------------------------- | | Runtime agent | `agents` | AG-UI endpoint and optional key reference. | -| Profile | `agent_profiles` | Name, title, role, avatar seed, owner, visibility, and soft deletion. | +| Profile | `agent_profiles` | Name, title, role, avatar seed or custom image, owner, visibility, and soft deletion. | | Personal roster | `agent_preferences` | Per-user hidden state. | | Channel | `channels` | Conversation membership and coworker binding. | | Intelligence mapping | `intelligence_channel_mappings` | Channel-to-thread mapping. | @@ -35,7 +35,21 @@ The message is ordinary AG-UI system content, so it works with any AG-UI-compati | `private` | Owner and administrators. | | `public` | Everyone in the deployment. | -Filtering happens in server/database queries. Package-provided agents cannot be edited or deleted through the product. +Filtering happens in server/database queries. Package-provided agents cannot be renamed, +reconfigured, or deleted through the product. An administrator can still set their deployment-local +avatar without changing the profile the package owns. + +## Avatars + +A person changes their own avatar in **Settings**. A Bot owner changes its avatar in the coworker +dialog, and an administrator can do the same for any Bot, including one supplied by the tenant +package. Removing a custom image returns to the identity-provider image or initials for a person, +and to the generated avatar for a Bot. + +Uploads accept PNG, JPEG, and WebP images up to 2 MB. The server checks the decoded size, file +signature, and dimensions before storing the image in PostgreSQL. Roster and channel responses carry +a short versioned image URL rather than the image bytes, so one list never copies every avatar into +its JSON response. ## Channels @@ -61,6 +75,12 @@ That is `agent-langgraph`, which runs a real framework and its own tool loop. Th `4200` hand-writes the protocol and leaves the loop to whatever is watching, so it is a reference rather than something to build a deployment on. +With `CODEX_AGENT_ENABLED=true`, `scripts/start.sh` instead defaults this endpoint to +`http://localhost:4202/ag-ui`, starts the Codex adapter on the host, and skips both provider-key Bot +containers. The adapter reuses the existing ChatGPT login, resumes its Codex threads, and sends only +assigned tool calls back through OpenBot's governed callback. See +[the local Codex coworker guide](../agent-codex/README.md). + The URL is optional. Set it with `MANAGED_AGENT_TOKEN`, or leave it unset: product-created coworkers then need their own endpoint, and a package agent whose endpoint expands to nothing is omitted rather than registered against a missing host. A leftover token with no URL is ignored. diff --git a/docs/deployment.md b/docs/deployment.md index b47462e42..42d527264 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -1,7 +1,8 @@ # Deployment -OpenBot ships as one container. It carries the app, the API that serves it, and the browser the Bots -drive, and it can carry its own PostgreSQL as well. It does what it does on a laptop. +OpenBot ships as one container. It carries the app, the API that serves it, and the graphical +computer the Bots drive, and it can carry its own PostgreSQL as well. It does what it does on a +laptop. ```sh docker build -t openbot . @@ -16,9 +17,9 @@ docker run -p 3001:3001 --env-file .env \ ## What is in the image, and what is not -**In it:** the built app, the API, and Chromium. One port, 3001. The browser listens on 4100 inside -the container and is deliberately not published: it holds real logins and its only caller is the -process beside it. +**In it:** the built app, the API, and a Linux desktop with Chromium and a terminal. One port, 3001. +The computer listens on 4100 inside the container and is deliberately not published: it holds real +logins and its only caller is the process beside it. **PostgreSQL, if you ask for it.** `EMBEDDED_POSTGRES=on` starts one inside the container, creates the database and the `vector` extension the first time, and runs the migrations on every start. It @@ -41,10 +42,10 @@ enable it for you. **Not in it:** **The supervisor.** It gives each Bot its own container, which needs a Docker socket, which no -serverless container platform permits. Without it, every Bot shares the one browser, exactly as they -do on a laptop with no supervisor configured. A shared browser means shared logins, shared files and -shared session between Bots, which is fine for a deployment where one team trusts its own Bots and -is not fine as a boundary between tenants. +serverless container platform permits. Without it, every Bot shares one computer, exactly as they do +on a laptop with no supervisor configured. Shared mode exposes only each Bot's browser page, not the +process-wide desktop, but the underlying logins, files and session are still shared. That is fine for +a deployment where one team trusts its own Bots and is not a boundary between tenants. **The routines schedule.** Nothing in this image is scheduled to fire a routine — there is no worker service beside the API, and `worker/` (the looping local variant) is not in the image. The @@ -124,7 +125,7 @@ docker run --rm --env-file .env openbot \ The page snapshot a Bot resolves element references against lives in Postgres, so a second replica can answer a click the first one snapshotted. Run more than one if the platform wants it. The -supervisor is still not in this image, so every replica shares the one browser inside it. +supervisor is still not in this image, so every replica uses its own shared computer. ## Platform notes diff --git a/docs/development.md b/docs/development.md index b2c92f721..faa0a7dfd 100644 --- a/docs/development.md +++ b/docs/development.md @@ -2,7 +2,8 @@ ## Setup -Install Docker, [Bun](https://bun.sh) 1.3+, `lsof`, `python3`, and `curl`. +Install Docker, [Bun](https://bun.sh) 1.3+, `lsof`, `python3`, and `curl`. Local Codex mode also needs +the Codex CLI authenticated with `codex login`. ```sh cp .env.example .env @@ -19,7 +20,9 @@ npx --yes copilotkit@latest license --write Put the `cpk-...` runtime key from `project select` in `.env` as `INTELLIGENCE_API_KEY`. `license --write` writes `COPILOTKIT_LICENSE_TOKEN`. -Then add `OPENAI_API_KEY`. +Then add `OPENAI_API_KEY`. To use the local ChatGPT account instead, set +`CODEX_AGENT_ENABLED=true` and `AGENT_ENDPOINT_ALLOWED_HOSTS=localhost:4202`; see +[the Codex coworker guide](../agent-codex/README.md). Start the stack: @@ -40,10 +43,13 @@ Use `bun run dev` only when you want the app and API server without starting the | `agent-computer` | 4100 | | `agent-bot` | 4200 | | `agent-langgraph` | 4201 | +| `agent-codex` | 4202 (only in local Codex mode) | | `supervisor` | 4500 host / 4300 container | | PostgreSQL | 5432 | `start.sh` leaves existing matching services alone and reports when a port is held by another process. +With `CODEX_AGENT_ENABLED=true`, it runs `agent-codex` on the host and skips the `agent-bot` and +`agent-langgraph` containers. ## Migrations @@ -127,9 +133,10 @@ cd agent-computer && bun install cd .. && bun run test:live-screen ``` -It drives the live screen against the real computer process with a real Chromium: a socket closing -while the browser is still starting, a second connection taking the screen from the first, the wheel -refusing input from the socket that owns it, and a browser closing by request or by the idle sweep. +It drives the live screen against the real computer process with Chromium and Xvfb: a socket closing +while the browser is still starting, a second connection taking the screen from the first, control +lease checks, the full RFB desktop, and closing Chromium with its native X before reopening it from +the desktop dock. Those need `agent-computer/src/index.ts`, which imports Playwright at module scope, and `playwright` is declared only in `agent-computer/package.json`, which `bun install` at the root does not reach. So without `OPENBOT_LIVE_SCREEN=1` the files skip before importing anything, which is what keeps diff --git a/examples/codex/agents.yaml b/examples/codex/agents.yaml new file mode 100644 index 000000000..98ee96793 --- /dev/null +++ b/examples/codex/agents.yaml @@ -0,0 +1,8 @@ +agents: + - id: codex-assistant + name: Codex + title: Subscription-powered coworker + role_description: Help with general questions using the locally signed-in Codex account. + avatar_seed: codex-assistant + type: remote-ag-ui + endpoint: ${MANAGED_AGENT_AG_UI_URL:-} diff --git a/examples/codex/brand.yaml b/examples/codex/brand.yaml new file mode 100644 index 000000000..5313ee0ae --- /dev/null +++ b/examples/codex/brand.yaml @@ -0,0 +1,3 @@ +tenant: + id: openbot-codex-local + product_name: OpenBot + Codex diff --git a/examples/codex/channels.yaml b/examples/codex/channels.yaml new file mode 100644 index 000000000..a0cd305c9 --- /dev/null +++ b/examples/codex/channels.yaml @@ -0,0 +1,6 @@ +channels: + - id: codex + name: Codex + description: Test a Codex coworker backed by the local ChatGPT subscription. + permitted_agents: [codex-assistant] + allowed_groups: [all] diff --git a/examples/codex/knowledge.yaml b/examples/codex/knowledge.yaml new file mode 100644 index 000000000..b87707414 --- /dev/null +++ b/examples/codex/knowledge.yaml @@ -0,0 +1 @@ +sources: [] diff --git a/examples/codex/model.yaml b/examples/codex/model.yaml new file mode 100644 index 000000000..ba7c60b7e --- /dev/null +++ b/examples/codex/model.yaml @@ -0,0 +1,4 @@ +model: + provider: openai + credential_secret_ref: unused-in-codex-spike + default_model: gpt-5.6-terra diff --git a/examples/codex/skills.yaml b/examples/codex/skills.yaml new file mode 100644 index 000000000..ada13e6ed --- /dev/null +++ b/examples/codex/skills.yaml @@ -0,0 +1 @@ +skills: [] diff --git a/package.json b/package.json index 84c4ee66f..b07afb510 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "packageManager": "bun@1.3.14", "workspaces": [ "app", + "agent-codex", "server", "worker" ], diff --git a/scripts/start.sh b/scripts/start.sh index 74beebab6..f417a9274 100755 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -30,8 +30,17 @@ SERVER_PORT="$(setting SERVER_PORT 3001)" COMPUTER_PORT="$(setting COMPUTER_PORT 4100)" BOT_PORT="$(setting BOT_PORT 4200)" LANGGRAPH_PORT="$(setting LANGGRAPH_PORT 4201)" +CODEX_AGENT_PORT="$(setting CODEX_AGENT_PORT 4202)" SUPERVISOR_PORT="$(setting SUPERVISOR_PORT 4500)" ONE_COMPUTER_EACH="${OPENBOT_ONE_COMPUTER_EACH:-true}" +CODEX_AGENT_ENABLED="$(setting CODEX_AGENT_ENABLED false)" +case "$CODEX_AGENT_ENABLED" in + true|false) ;; + *) + printf '\033[31m%s\033[0m\n' "CODEX_AGENT_ENABLED must be true or false." + exit 1 + ;; +esac export APP_PORT SERVER_PORT SUPERVISOR_TOKEN="$(setting SUPERVISOR_TOKEN openbot-dev-supervisor-token)" COMPUTER_TOKEN="$(setting COMPUTER_TOKEN openbot-dev-computer-token)" @@ -54,10 +63,15 @@ WORKER_SHARED_SECRET="$(setting WORKER_SHARED_SECRET openbot-dev-worker-secret)" # # Written into .env rather than exported for this run alone, so `docker compose up` by hand later # sees the same value the script used. -# The laptop stack runs agent-langgraph on LANGGRAPH_PORT. The one-container image does not, so -# this default stays in the script rather than in .env: a `docker run --env-file .env` must not -# inherit a URL that points at a process the image does not contain. -MANAGED_AGENT_AG_UI_URL="$(setting MANAGED_AGENT_AG_UI_URL "http://localhost:${LANGGRAPH_PORT}/ag-ui")" +# The laptop stack normally runs agent-langgraph on LANGGRAPH_PORT. The local Codex compatibility +# mode instead runs a host process so it can reuse the person's existing `codex login` session +# without copying ChatGPT credentials into Docker. +if [ "$CODEX_AGENT_ENABLED" = "true" ]; then + DEFAULT_MANAGED_AGENT_URL="http://localhost:${CODEX_AGENT_PORT}/ag-ui" +else + DEFAULT_MANAGED_AGENT_URL="http://localhost:${LANGGRAPH_PORT}/ag-ui" +fi +MANAGED_AGENT_AG_UI_URL="$(setting MANAGED_AGENT_AG_UI_URL "$DEFAULT_MANAGED_AGENT_URL")" export MANAGED_AGENT_AG_UI_URL # Whether this run minted a secret that something already running may not have. @@ -146,6 +160,10 @@ identifies_as_openbot() { curl -fsS --max-time 3 "http://localhost:$port/" 2>/dev/null \ | grep -qi '[^<]*OpenBot' ;; + agent-codex) + curl -fsS --max-time 3 "http://localhost:$port/health" 2>/dev/null \ + | grep -q '"safety":"openbot-governed-tools"' + ;; # Compose services on dedicated loopback ports, answering a route named for this stack. *) curl -fsS --max-time 3 "http://localhost:$port/health" >/dev/null 2>&1 @@ -215,12 +233,13 @@ fi # `docker compose up -d` is declarative and does nothing for a service whose configuration has not # changed, so naming them all costs a comparison and buys the guarantee that what is running is what # this run configured. -for svc in agent-computer agent-bot agent-langgraph; do - SERVICES+=("$svc") -done +SERVICES+=(agent-computer) +if [ "$CODEX_AGENT_ENABLED" != "true" ]; then + SERVICES+=(agent-bot agent-langgraph) +fi export SUPERVISOR_TOKEN COMPUTER_TOKEN WORKER_SHARED_SECRET -export COMPUTER_PORT BOT_PORT LANGGRAPH_PORT SUPERVISOR_PORT +export COMPUTER_PORT BOT_PORT LANGGRAPH_PORT CODEX_AGENT_PORT SUPERVISOR_PORT docker compose up -d --build "${SERVICES[@]}" >/dev/null if ! docker compose run --rm --build migrate >"$LOGS/migrate.log" 2>&1; then red " Migrations did not apply. The database is not the schema this server expects." @@ -228,8 +247,30 @@ if ! docker compose run --rm --build migrate >"$LOGS/migrate.log" 2>&1; then exit 1 fi wait_for "http://localhost:$COMPUTER_PORT/health" "agent-computer" -wait_for "http://localhost:$BOT_PORT/health" "agent-bot" -wait_for "http://localhost:$LANGGRAPH_PORT/health" "agent-langgraph" +if [ "$CODEX_AGENT_ENABLED" = "true" ]; then + CODEX_AGENT_WORKSPACE="$(setting CODEX_AGENT_WORKSPACE "$ROOT/.openbot-codex/workspace")" + CODEX_AGENT_STATE="$(setting CODEX_AGENT_STATE "$ROOT/.openbot-codex/threads.json")" + OPENBOT_TOOL_URL="$(setting OPENBOT_TOOL_URL "http://localhost:${SERVER_PORT}/api/agent-tools/call")" + mkdir -p "$CODEX_AGENT_WORKSPACE" + require_free_or_ours "$CODEX_AGENT_PORT" "agent-codex" + if identifies_as_openbot "$CODEX_AGENT_PORT" "agent-codex"; then + pkill -f "bun agent-codex/src/index.ts" >/dev/null 2>&1 || true + sleep 1 + fi + (cd "$ROOT" && \ + nohup env \ + PORT="$CODEX_AGENT_PORT" \ + MANAGED_AGENT_TOKEN="$MANAGED_AGENT_TOKEN" \ + AGENT_TOOL_TOKEN="$AGENT_TOOL_TOKEN" \ + OPENBOT_TOOL_URL="$OPENBOT_TOOL_URL" \ + CODEX_AGENT_WORKSPACE="$CODEX_AGENT_WORKSPACE" \ + CODEX_AGENT_STATE="$CODEX_AGENT_STATE" \ + bun agent-codex/src/index.ts >"$LOGS/agent-codex.log" 2>&1 </dev/null &) + wait_for "http://localhost:$CODEX_AGENT_PORT/health" "agent-codex" 60 +else + wait_for "http://localhost:$BOT_PORT/health" "agent-bot" + wait_for "http://localhost:$LANGGRAPH_PORT/health" "agent-langgraph" +fi for table in agent_profiles agent_preferences; do if ! docker compose exec -T postgres \ @@ -251,6 +292,9 @@ green " coworker tables migrated" # grep survives inside `setting()` only because `local v="$(...)"` takes `local`'s own exit status # and masks it. So: report what was resolved, and do not re-read the file. green " managed coworker endpoint: $MANAGED_AGENT_AG_UI_URL" +if [ "$CODEX_AGENT_ENABLED" = "true" ]; then + green " Codex coworker: ChatGPT login · persistent threads · OpenBot-governed tools" +fi info "2/4 Server" require_free_or_ours "$SERVER_PORT" server @@ -302,16 +346,18 @@ if identifies_as_openbot "$SERVER_PORT" server; then fi if ! identifies_as_openbot "$SERVER_PORT" server; then if [ "$ONE_COMPUTER_EACH" = "true" ]; then - (cd server && PORT="$SERVER_PORT" \ + (cd server && nohup env \ + PORT="$SERVER_PORT" \ COMPUTER_SUPERVISOR_URL="http://localhost:$SUPERVISOR_PORT" \ SUPERVISOR_TOKEN="$SUPERVISOR_TOKEN" \ COMPUTER_TOKEN="$COMPUTER_TOKEN" \ WORKER_SHARED_SECRET="$WORKER_SHARED_SECRET" \ - bun --env-file=../.env src/index.ts >"$LOGS/server.log" 2>&1 &) + bun --env-file=../.env src/index.ts >"$LOGS/server.log" 2>&1 </dev/null &) else - (cd server && PORT="$SERVER_PORT" \ + (cd server && nohup env \ + PORT="$SERVER_PORT" \ WORKER_SHARED_SECRET="$WORKER_SHARED_SECRET" \ - bun --env-file=../.env src/index.ts >"$LOGS/server.log" 2>&1 &) + bun --env-file=../.env src/index.ts >"$LOGS/server.log" 2>&1 </dev/null &) fi fi wait_for_openbot "$SERVER_PORT" server @@ -335,10 +381,11 @@ wait_for_openbot "$SERVER_PORT" server if ! pgrep -f "bun worker/src/index.ts" >/dev/null 2>&1; then WORKER_DATABASE_URL="$(setting DATABASE_URL postgres://openbot:openbot@localhost:5432/openbot)" (cd "$ROOT" && \ - DATABASE_URL="$WORKER_DATABASE_URL" \ - SERVER_INTERNAL_URL="http://localhost:$SERVER_PORT" \ - WORKER_SHARED_SECRET="$WORKER_SHARED_SECRET" \ - bun worker/src/index.ts >"$LOGS/worker.log" 2>&1 &) + nohup env \ + DATABASE_URL="$WORKER_DATABASE_URL" \ + SERVER_INTERNAL_URL="http://localhost:$SERVER_PORT" \ + WORKER_SHARED_SECRET="$WORKER_SHARED_SECRET" \ + bun worker/src/index.ts >"$LOGS/worker.log" 2>&1 </dev/null &) info " worker: started (routine sweep loop)" sleep 1 if ! pgrep -f "bun worker/src/index.ts" >/dev/null 2>&1; then @@ -368,7 +415,7 @@ PY info "4/4 App" require_free_or_ours "$APP_PORT" app if ! identifies_as_openbot "$APP_PORT" app; then - (cd app && bun run dev --port "$APP_PORT" --strictPort >"$LOGS/app.log" 2>&1 &) + (cd app && nohup bun run dev --port "$APP_PORT" --strictPort >"$LOGS/app.log" 2>&1 </dev/null &) fi wait_for_openbot "$APP_PORT" app @@ -395,6 +442,7 @@ Try: Logs: $LOGS Routine sweep worker: $LOGS/worker.log Stop the routine worker: pkill -f 'bun worker/src/index.ts' +Stop the Codex coworker: pkill -f 'bun agent-codex/src/index.ts' Stop Docker services: docker compose down A Bot's computer is made by the supervisor rather than by compose, so it keeps running: docker rm -f \$(docker ps -q --filter label=openbot.supervisor=true) diff --git a/server/Dockerfile b/server/Dockerfile index be891a316..56c3f84b4 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -17,6 +17,7 @@ COPY package.json bun.lock ./ COPY tsconfig.base.json tsconfig.base.json COPY bunfig.toml bunfig.toml COPY app/package.json app/package.json +COPY agent-codex/package.json agent-codex/package.json COPY server/package.json server/package.json COPY worker/package.json worker/package.json RUN bun install --frozen-lockfile diff --git a/server/drizzle/0026_careful_zzzax.sql b/server/drizzle/0026_careful_zzzax.sql new file mode 100644 index 000000000..a034e1f80 --- /dev/null +++ b/server/drizzle/0026_careful_zzzax.sql @@ -0,0 +1,2 @@ +ALTER TABLE "users" ADD COLUMN "avatar_image" text;--> statement-breakpoint +ALTER TABLE "agent_profiles" ADD COLUMN "avatar_image" text; \ No newline at end of file diff --git a/server/drizzle/meta/0026_snapshot.json b/server/drizzle/meta/0026_snapshot.json new file mode 100644 index 000000000..c6a061a31 --- /dev/null +++ b/server/drizzle/meta/0026_snapshot.json @@ -0,0 +1,3032 @@ +{ + "id": "c8834d48-73f3-4917-8f73-138d348abf6c", + "prevId": "d96da430-35aa-475c-a060-f55151269d6d", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.accounts": { + "name": "accounts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issuer": { + "name": "issuer", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "accounts_provider_account_idx": { + "name": "accounts_provider_account_idx", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "accounts_user_id_users_id_fk": { + "name": "accounts_user_id_users_id_fk", + "tableFrom": "accounts", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agents": { + "name": "agents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "agent_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "configuration": { + "name": "configuration", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "package_id": { + "name": "package_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "override": { + "name": "override", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "agents_package_id_deployment_packages_id_fk": { + "name": "agents_package_id_deployment_packages_id_fk", + "tableFrom": "agents", + "tableTo": "deployment_packages", + "columnsFrom": [ + "package_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audit_events": { + "name": "audit_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "audit_events_created_at_idx": { + "name": "audit_events_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_type_time_idx": { + "name": "audit_events_type_time_idx", + "columns": [ + { + "expression": "event_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_actor_time_idx": { + "name": "audit_events_actor_time_idx", + "columns": [ + { + "expression": "actor_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_target_time_idx": { + "name": "audit_events_target_time_idx", + "columns": [ + { + "expression": "target_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.channel_agents": { + "name": "channel_agents", + "schema": "", + "columns": { + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "channel_agents_channel_id_channels_id_fk": { + "name": "channel_agents_channel_id_channels_id_fk", + "tableFrom": "channel_agents", + "tableTo": "channels", + "columnsFrom": [ + "channel_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "channel_agents_agent_id_agents_id_fk": { + "name": "channel_agents_agent_id_agents_id_fk", + "tableFrom": "channel_agents", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "channel_agents_channel_id_agent_id_pk": { + "name": "channel_agents_channel_id_agent_id_pk", + "columns": [ + "channel_id", + "agent_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.channel_memberships": { + "name": "channel_memberships", + "schema": "", + "columns": { + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pinned_at": { + "name": "pinned_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_read_at": { + "name": "last_read_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "channel_memberships_channel_id_channels_id_fk": { + "name": "channel_memberships_channel_id_channels_id_fk", + "tableFrom": "channel_memberships", + "tableTo": "channels", + "columnsFrom": [ + "channel_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "channel_memberships_user_id_users_id_fk": { + "name": "channel_memberships_user_id_users_id_fk", + "tableFrom": "channel_memberships", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "channel_memberships_channel_id_user_id_pk": { + "name": "channel_memberships_channel_id_user_id_pk", + "columns": [ + "channel_id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.channels": { + "name": "channels", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "suggested_prompts": { + "name": "suggested_prompts", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "allowed_groups": { + "name": "allowed_groups", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "package_id": { + "name": "package_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "override": { + "name": "override", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "last_message": { + "name": "last_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_message_at": { + "name": "last_message_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_message_agent_id": { + "name": "last_message_agent_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "channels_recent_activity_idx": { + "name": "channels_recent_activity_idx", + "columns": [ + { + "expression": "COALESCE(\"last_message_at\", \"created_at\") DESC", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "channels_package_id_deployment_packages_id_fk": { + "name": "channels_package_id_deployment_packages_id_fk", + "tableFrom": "channels", + "tableTo": "deployment_packages", + "columnsFrom": [ + "package_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "channels_last_message_agent_id_agents_id_fk": { + "name": "channels_last_message_agent_id_agents_id_fk", + "tableFrom": "channels", + "tableTo": "agents", + "columnsFrom": [ + "last_message_agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.credentials": { + "name": "credentials", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "credential_kind", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_value": { + "name": "encrypted_value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key_id": { + "name": "key_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "credentials_active_key_idx": { + "name": "credentials_active_key_idx", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"credentials\".\"revoked_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.deployment_packages": { + "name": "deployment_packages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_path": { + "name": "source_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "checksum": { + "name": "checksum", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "loaded_at": { + "name": "loaded_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "deployment_packages_tenant_id_unique": { + "name": "deployment_packages_tenant_id_unique", + "nullsNotDistinct": false, + "columns": [ + "tenant_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.intelligence_channel_mappings": { + "name": "intelligence_channel_mappings", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "intelligence_channel_mappings_thread_idx": { + "name": "intelligence_channel_mappings_thread_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "intelligence_channel_mappings_user_id_users_id_fk": { + "name": "intelligence_channel_mappings_user_id_users_id_fk", + "tableFrom": "intelligence_channel_mappings", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "intelligence_channel_mappings_channel_id_channels_id_fk": { + "name": "intelligence_channel_mappings_channel_id_channels_id_fk", + "tableFrom": "intelligence_channel_mappings", + "tableTo": "channels", + "columnsFrom": [ + "channel_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "intelligence_channel_mappings_user_id_channel_id_pk": { + "name": "intelligence_channel_mappings_user_id_channel_id_pk", + "columns": [ + "user_id", + "channel_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.revoked_access": { + "name": "revoked_access", + "schema": "", + "columns": { + "email": { + "name": "email", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "revoked_by": { + "name": "revoked_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sessions": { + "name": "sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "sessions_user_id_users_id_fk": { + "name": "sessions_user_id_users_id_fk", + "tableFrom": "sessions", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sessions_token_unique": { + "name": "sessions_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sso_providers": { + "name": "sso_providers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "issuer": { + "name": "issuer", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "oidc_config": { + "name": "oidc_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "saml_config": { + "name": "saml_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "sso_providers_user_id_users_id_fk": { + "name": "sso_providers_user_id_users_id_fk", + "tableFrom": "sso_providers", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sso_providers_provider_id_unique": { + "name": "sso_providers_provider_id_unique", + "nullsNotDistinct": false, + "columns": [ + "provider_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_roles": { + "name": "user_roles", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "user_roles_user_id_users_id_fk": { + "name": "user_roles_user_id_users_id_fk", + "tableFrom": "user_roles", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "user_roles_user_id_role_pk": { + "name": "user_roles_user_id_role_pk", + "columns": [ + "user_id", + "role" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "avatar_image": { + "name": "avatar_image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "groups": { + "name": "groups", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "onboarding_step": { + "name": "onboarding_step", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "onboarding_completed_at": { + "name": "onboarding_completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "users_email_unique": { + "name": "users_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verifications": { + "name": "verifications", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.action_policy": { + "name": "action_policy", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deny": { + "name": "deny", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "allow": { + "name": "allow", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.computer_page_frame": { + "name": "computer_page_frame", + "schema": "", + "columns": { + "computer_id": { + "name": "computer_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_call_id": { + "name": "tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "frame": { + "name": "frame", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "captured_at": { + "name": "captured_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "computer_page_frame_captured_idx": { + "name": "computer_page_frame_captured_idx", + "columns": [ + { + "expression": "captured_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "computer_page_frame_computer_id_tool_call_id_pk": { + "name": "computer_page_frame_computer_id_tool_call_id_pk", + "columns": [ + "computer_id", + "tool_call_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.computer_snapshot": { + "name": "computer_snapshot", + "schema": "", + "columns": { + "computer_id": { + "name": "computer_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "snapshot_id": { + "name": "snapshot_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "elements": { + "name": "elements", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "taken_at": { + "name": "taken_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "session": { + "name": "session", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_preferences": { + "name": "agent_preferences", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hidden_at": { + "name": "hidden_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "agent_preferences_user_id_users_id_fk": { + "name": "agent_preferences_user_id_users_id_fk", + "tableFrom": "agent_preferences", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_preferences_agent_id_agents_id_fk": { + "name": "agent_preferences_agent_id_agents_id_fk", + "tableFrom": "agent_preferences", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "agent_preferences_user_id_agent_id_pk": { + "name": "agent_preferences_user_id_agent_id_pk", + "columns": [ + "user_id", + "agent_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_profiles": { + "name": "agent_profiles", + "schema": "", + "columns": { + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role_description": { + "name": "role_description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "avatar_seed": { + "name": "avatar_seed", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "avatar_image": { + "name": "avatar_image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "visibility": { + "name": "visibility", + "type": "agent_visibility", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "callback_token_hash": { + "name": "callback_token_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "callback_token_issued_at": { + "name": "callback_token_issued_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "agent_profiles_visibility_deleted_idx": { + "name": "agent_profiles_visibility_deleted_idx", + "columns": [ + { + "expression": "visibility", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "agent_profiles_agent_id_agents_id_fk": { + "name": "agent_profiles_agent_id_agents_id_fk", + "tableFrom": "agent_profiles", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_profiles_owner_user_id_users_id_fk": { + "name": "agent_profiles_owner_user_id_users_id_fk", + "tableFrom": "agent_profiles", + "tableTo": "users", + "columnsFrom": [ + "owner_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.routine_runs": { + "name": "routine_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "routine_id": { + "name": "routine_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "routine_run_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "routine_runs_by_routine_idx": { + "name": "routine_runs_by_routine_idx", + "columns": [ + { + "expression": "routine_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "routine_runs_routine_id_routines_id_fk": { + "name": "routine_runs_routine_id_routines_id_fk", + "tableFrom": "routine_runs", + "tableTo": "routines", + "columnsFrom": [ + "routine_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.routines": { + "name": "routines", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instruction": { + "name": "instruction", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "cron": { + "name": "cron", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "timezone": { + "name": "timezone", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'UTC'" + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "next_run_at": { + "name": "next_run_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_run_at": { + "name": "last_run_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "routines_due_idx": { + "name": "routines_due_idx", + "columns": [ + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "routines_by_owner_idx": { + "name": "routines_by_owner_idx", + "columns": [ + { + "expression": "owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "routines_owner_user_id_users_id_fk": { + "name": "routines_owner_user_id_users_id_fk", + "tableFrom": "routines", + "tableTo": "users", + "columnsFrom": [ + "owner_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "routines_agent_id_agents_id_fk": { + "name": "routines_agent_id_agents_id_fk", + "tableFrom": "routines", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.component_exclusions": { + "name": "component_exclusions", + "schema": "", + "columns": { + "component_name": { + "name": "component_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "withheld_by": { + "name": "withheld_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "component_exclusions_component_name_components_name_fk": { + "name": "component_exclusions_component_name_components_name_fk", + "tableFrom": "component_exclusions", + "tableTo": "components", + "columnsFrom": [ + "component_name" + ], + "columnsTo": [ + "name" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "component_exclusions_agent_id_agents_id_fk": { + "name": "component_exclusions_agent_id_agents_id_fk", + "tableFrom": "component_exclusions", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "component_exclusions_component_name_agent_id_pk": { + "name": "component_exclusions_component_name_agent_id_pk", + "columns": [ + "component_name", + "agent_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.component_functions": { + "name": "component_functions", + "schema": "", + "columns": { + "component_name": { + "name": "component_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "function_name": { + "name": "function_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "granted_by": { + "name": "granted_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "component_functions_component_name_components_name_fk": { + "name": "component_functions_component_name_components_name_fk", + "tableFrom": "component_functions", + "tableTo": "components", + "columnsFrom": [ + "component_name" + ], + "columnsTo": [ + "name" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "component_functions_component_name_function_name_pk": { + "name": "component_functions_component_name_function_name_pk", + "columns": [ + "component_name", + "function_name" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.components": { + "name": "components", + "schema": "", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "draft_description": { + "name": "draft_description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "published_description": { + "name": "published_description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published": { + "name": "published", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "published_at": { + "name": "published_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_servers": { + "name": "mcp_servers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "vendor": { + "name": "vendor", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provenance": { + "name": "provenance", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'first-party'" + }, + "credential_id": { + "name": "credential_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "tools_refreshed_at": { + "name": "tools_refreshed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "added_by": { + "name": "added_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mcp_servers_credential_id_credentials_id_fk": { + "name": "mcp_servers_credential_id_credentials_id_fk", + "tableFrom": "mcp_servers", + "tableTo": "credentials", + "columnsFrom": [ + "credential_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_tools": { + "name": "mcp_tools", + "schema": "", + "columns": { + "server_id": { + "name": "server_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "input_schema": { + "name": "input_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mcp_tools_server_id_mcp_servers_id_fk": { + "name": "mcp_tools_server_id_mcp_servers_id_fk", + "tableFrom": "mcp_tools", + "tableTo": "mcp_servers", + "columnsFrom": [ + "server_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "mcp_tools_server_id_name_pk": { + "name": "mcp_tools_server_id_name_pk", + "columns": [ + "server_id", + "name" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_user_credentials": { + "name": "mcp_user_credentials", + "schema": "", + "columns": { + "server_id": { + "name": "server_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "connected_at": { + "name": "connected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "mcp_user_credentials_user_idx": { + "name": "mcp_user_credentials_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mcp_user_credentials_server_id_mcp_servers_id_fk": { + "name": "mcp_user_credentials_server_id_mcp_servers_id_fk", + "tableFrom": "mcp_user_credentials", + "tableTo": "mcp_servers", + "columnsFrom": [ + "server_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_user_credentials_user_id_users_id_fk": { + "name": "mcp_user_credentials_user_id_users_id_fk", + "tableFrom": "mcp_user_credentials", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_user_credentials_credential_id_credentials_id_fk": { + "name": "mcp_user_credentials_credential_id_credentials_id_fk", + "tableFrom": "mcp_user_credentials", + "tableTo": "credentials", + "columnsFrom": [ + "credential_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "mcp_user_credentials_server_id_user_id_pk": { + "name": "mcp_user_credentials_server_id_user_id_pk", + "columns": [ + "server_id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.plugin_grants": { + "name": "plugin_grants", + "schema": "", + "columns": { + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref": { + "name": "ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "granted_by": { + "name": "granted_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "plugin_grants_agent_idx": { + "name": "plugin_grants_agent_idx", + "columns": [ + { + "expression": "agent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "plugin_grants_agent_id_agents_id_fk": { + "name": "plugin_grants_agent_id_agents_id_fk", + "tableFrom": "plugin_grants", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "plugin_grants_kind_ref_agent_id_pk": { + "name": "plugin_grants_kind_ref_agent_id_pk", + "columns": [ + "kind", + "ref", + "agent_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sandboxed_components": { + "name": "sandboxed_components", + "schema": "", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "draft_description": { + "name": "draft_description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_html": { + "name": "draft_html", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_css": { + "name": "draft_css", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_js_functions": { + "name": "draft_js_functions", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_argument_schema": { + "name": "draft_argument_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "published_description": { + "name": "published_description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_html": { + "name": "published_html", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_css": { + "name": "published_css", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_js_functions": { + "name": "published_js_functions", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_argument_schema": { + "name": "published_argument_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "sample_arguments": { + "name": "sample_arguments", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "published": { + "name": "published", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "published_at": { + "name": "published_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "authored_by": { + "name": "authored_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skill_tools": { + "name": "skill_tools", + "schema": "", + "columns": { + "skill_id": { + "name": "skill_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref": { + "name": "ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "declared_by": { + "name": "declared_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "skill_tools_ref_idx": { + "name": "skill_tools_ref_idx", + "columns": [ + { + "expression": "ref", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "skill_tools_skill_id_skills_id_fk": { + "name": "skill_tools_skill_id_skills_id_fk", + "tableFrom": "skill_tools", + "tableTo": "skills", + "columnsFrom": [ + "skill_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "skill_tools_skill_id_ref_pk": { + "name": "skill_tools_skill_id_ref_pk", + "columns": [ + "skill_id", + "ref" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skills": { + "name": "skills", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instructions": { + "name": "instructions", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'yours'" + }, + "installed_by": { + "name": "installed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "skills_slug_key": { + "name": "skills_slug_key", + "columns": [ + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "skills_owner_idx": { + "name": "skills_owner_idx", + "columns": [ + { + "expression": "owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "skills_owner_user_id_users_id_fk": { + "name": "skills_owner_user_id_users_id_fk", + "tableFrom": "skills", + "tableTo": "users", + "columnsFrom": [ + "owner_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.work_items": { + "name": "work_items", + "schema": "", + "columns": { + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_at": { + "name": "run_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "claimed_by": { + "name": "claimed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_until": { + "name": "lease_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "work_items_claimable_idx": { + "name": "work_items_claimable_idx", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "work_items_kind_key_pk": { + "name": "work_items_kind_key_pk", + "columns": [ + "kind", + "key" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.agent_type": { + "name": "agent_type", + "schema": "public", + "values": [ + "built_in", + "remote_ag_ui" + ] + }, + "public.credential_kind": { + "name": "credential_kind", + "schema": "public", + "values": [ + "model", + "connector", + "agent", + "mcp", + "mcp_oauth_client", + "mcp_user_token" + ] + }, + "public.role": { + "name": "role", + "schema": "public", + "values": [ + "admin", + "user" + ] + }, + "public.agent_visibility": { + "name": "agent_visibility", + "schema": "public", + "values": [ + "public", + "private" + ] + }, + "public.routine_run_status": { + "name": "routine_run_status", + "schema": "public", + "values": [ + "succeeded", + "failed", + "skipped" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/server/drizzle/meta/_journal.json b/server/drizzle/meta/_journal.json index 0a0fb4633..f995ffb6f 100644 --- a/server/drizzle/meta/_journal.json +++ b/server/drizzle/meta/_journal.json @@ -183,6 +183,13 @@ "when": 1787926472382, "tag": "0025_backfill_existing_users_have_onboarded", "breakpoints": true + }, + { + "idx": 26, + "version": "7", + "when": 1788289811662, + "tag": "0026_careful_zzzax", + "breakpoints": true } ] } \ No newline at end of file diff --git a/server/src/agents/handoff-runner.ts b/server/src/agents/handoff-runner.ts index c59a3bc76..6c2d08035 100644 --- a/server/src/agents/handoff-runner.ts +++ b/server/src/agents/handoff-runner.ts @@ -99,6 +99,8 @@ const REAP_OLDER_THAN_MS = 24 * 60 * 60 * 1_000; export function createHandoffRunner(options: { queue: WorkQueue; + /** Queue namespace. Injectable so database-backed tests cannot claim another suite's hops. */ + kind?: string; delivery: HandoffDelivery; /** Who this replica is, for the lease. */ owner: string; @@ -122,6 +124,7 @@ export function createHandoffRunner(options: { }) { const { queue, + kind = HANDOFF_KIND, delivery, owner, sign, @@ -157,7 +160,7 @@ export function createHandoffRunner(options: { */ const relay = (work: HandoffWork, key: string, answer: string) => queue.offer({ - kind: HANDOFF_KIND, + kind, // Outside the run's fan-out prefix and keyed on the hop, for the same two reasons as the // notice below: a relay is not a Bot this run asked for, and one run may legally ask the // same Bot two different things. @@ -176,7 +179,7 @@ export function createHandoffRunner(options: { const tell = (work: HandoffWork, key: string, reason: string) => queue.offer({ - kind: HANDOFF_KIND, + kind, /* * OUTSIDE THE RUN'S OWN PREFIX, and carrying the failed hop's key. * @@ -220,7 +223,7 @@ export function createHandoffRunner(options: { */ async reap(): Promise<number> { return queue.purge({ - kind: HANDOFF_KIND, + kind, olderThanMs: REAP_OLDER_THAN_MS, maxAttempts, }); @@ -229,7 +232,7 @@ export function createHandoffRunner(options: { /** Deliver whatever this replica can claim. */ async sweep(): Promise<HandoffRunReport> { const claimed = await queue.claim({ - kind: HANDOFF_KIND, + kind, owner, leaseMs, limit, @@ -263,7 +266,7 @@ export function createHandoffRunner(options: { const heartbeat = setInterval(() => { for (const key of ours) { void queue - .renew({ kind: HANDOFF_KIND, key, owner, leaseMs }) + .renew({ kind, key, owner, leaseMs }) .then((kept) => { // False means it went to somebody else. Dropped rather than renewed again, so the // loop below knows not to spend a model call on work it no longer holds. @@ -281,7 +284,7 @@ export function createHandoffRunner(options: { * A hop nothing can be done with. Finished rather than released, because releasing it puts * the same unusable row back on the queue for ever. */ - await queue.finish({ kind: HANDOFF_KIND, key: item.key, owner }); + await queue.finish({ kind, key: item.key, owner }); report.skipped.push({ key: item.key, reason: "not a hop" }); continue; } @@ -331,7 +334,7 @@ export function createHandoffRunner(options: { * turn, billed, ending in a second answer in somebody's conversation. */ const stillOurs = await queue.renew({ - kind: HANDOFF_KIND, + kind, key: item.key, owner, leaseMs, @@ -354,7 +357,7 @@ export function createHandoffRunner(options: { assertion: sign(work), }); const kept = await queue.finish({ - kind: HANDOFF_KIND, + kind, key: item.key, owner, }); @@ -447,7 +450,7 @@ export function createHandoffRunner(options: { * refused it once will probably refuse it again in the next second. */ await queue.release({ - kind: HANDOFF_KIND, + kind, key: item.key, owner, delayMs: 60_000, diff --git a/server/src/agents/handoff.ts b/server/src/agents/handoff.ts index 58c32848f..6ca26ff7b 100644 --- a/server/src/agents/handoff.ts +++ b/server/src/agents/handoff.ts @@ -76,6 +76,8 @@ export type HandoffDesk = { export function createHandoffDesk(options: { queue: WorkQueue; + /** Queue namespace. Injectable so database-backed tests cannot claim another suite's hops. */ + kind?: string; profiles: AgentProfileStore; /** Whether the asking Bot has been granted the Bot it is addressing. Read per hop, never cached. */ mayAddress: (fromBotId: string, toBotId: string) => Promise<boolean>; @@ -95,7 +97,15 @@ export function createHandoffDesk(options: { auditStore: AuditStore; caps: HandoffCaps; }): HandoffDesk { - const { queue, profiles, mayAddress, actorFor, auditStore, caps } = options; + const { + queue, + kind = HANDOFF_KIND, + profiles, + mayAddress, + actorFor, + auditStore, + caps, + } = options; /** Said once, so the trail carries the same words the Bot was given. */ async function refuse( @@ -306,7 +316,7 @@ export function createHandoffDesk(options: { .slice(0, 32)}`; const offered = await queue.offer({ - kind: HANDOFF_KIND, + kind, key, /* * Counted from the rows rather than from a variable, because a run whose hops land on diff --git a/server/src/agents/profile-policy.ts b/server/src/agents/profile-policy.ts index 76bc132a2..5ae15d038 100644 --- a/server/src/agents/profile-policy.ts +++ b/server/src/agents/profile-policy.ts @@ -22,6 +22,22 @@ export function canManageAgent( return agent.ownerUserId === actor.id || actor.role === "admin"; } +/** + * Whether this person may choose the picture other people see for a Bot. + * + * A package owns a system Bot's executable profile, so nobody may rename or reconfigure it. Its + * presentation is deployment-local, though, and an administrator may brand it without forking the + * package. User-owned Bots keep the same owner-or-admin rule as every other profile edit. + */ +export function canCustomizeAgentAvatar( + actor: AgentActor, + agent: AgentProfile, +): boolean { + if (agent.deletedAt !== null) return false; + if (actor.role === "admin") return true; + return !agent.systemOwned && agent.ownerUserId === actor.id; +} + export const canRunAgent = canAccessAgent; /** diff --git a/server/src/agents/profile-store.ts b/server/src/agents/profile-store.ts index 8d14bdd48..bc4f38756 100644 --- a/server/src/agents/profile-store.ts +++ b/server/src/agents/profile-store.ts @@ -1,4 +1,4 @@ -import { and, eq, isNotNull, isNull, or } from "drizzle-orm"; +import { and, eq, isNotNull, isNull, or, sql } from "drizzle-orm"; import type { CredentialStore } from "../credentials"; import type { Database } from "../db/client"; import { @@ -17,7 +17,7 @@ import { mintCallbackToken, sameToken, } from "./callback-token"; -import { canManageAgent } from "./profile-policy"; +import { canCustomizeAgentAvatar, canManageAgent } from "./profile-policy"; import type { AgentActor, AgentProfile, @@ -33,6 +33,8 @@ export type ProfileReadExecutor = DatabaseExecutor; export type AgentProfileStore = { list(actor: AgentActor, hidden?: boolean): Promise<AgentProfile[]>; get(actor: AgentActor, id: string): Promise<AgentProfile | null>; + /** Read image bytes only for an actor who may access this profile. */ + avatar(actor: AgentActor, id: string): Promise<string | null>; /** * `get`, but on the caller's own transaction and holding the profile against deletion until that * transaction ends. @@ -53,6 +55,11 @@ export type AgentProfileStore = { id: string, input: CreateAgentInput, ): Promise<AgentProfile>; + setAvatar( + actor: AgentActor, + id: string, + image: string | null, + ): Promise<AgentProfile>; duplicate(actor: AgentActor, id: string): Promise<AgentProfile>; setHidden(actor: AgentActor, id: string, hidden: boolean): Promise<void>; softDelete(actor: AgentActor, id: string): Promise<void>; @@ -111,6 +118,8 @@ const joinedProjection = { title: agentProfiles.title, roleDescription: agentProfiles.roleDescription, avatarSeed: agentProfiles.avatarSeed, + hasCustomAvatar: sql<boolean>`${agentProfiles.avatarImage} is not null`, + avatarUpdatedAt: agentProfiles.updatedAt, visibility: agentProfiles.visibility, ownerUserId: agentProfiles.ownerUserId, packageId: deploymentPackages.id, @@ -156,6 +165,8 @@ function mapProfile( title: row.title, roleDescription: row.roleDescription, avatarSeed: row.avatarSeed, + hasCustomAvatar: row.hasCustomAvatar, + avatarUpdatedAt: row.avatarUpdatedAt, visibility: row.visibility, ownerUserId: row.ownerUserId, systemOwned: row.packageId !== null, @@ -231,6 +242,12 @@ function requireManageable(actor: AgentActor, profile: AgentProfile) { } } +function requireAvatarManageable(actor: AgentActor, profile: AgentProfile) { + if (!canCustomizeAgentAvatar(actor, profile)) { + throw new AgentNotManageableError(profile.id); + } +} + function newAgentId() { return `agent_${crypto.randomUUID()}`; } @@ -296,6 +313,17 @@ export function createAgentProfileStore( return findAccessibleProfile(database, actor, id); }, + async avatar(actor, id) { + const profile = await findAccessibleProfile(database, actor, id); + if (!profile) return null; + const [row] = await database + .select({ image: agentProfiles.avatarImage }) + .from(agentProfiles) + .where(eq(agentProfiles.agentId, id)) + .limit(1); + return row?.image ?? null; + }, + async getWithin(executor, actor, id) { await lockProfileReadRow(executor, id); return findAccessibleProfile(executor, actor, id); @@ -432,10 +460,33 @@ export function createAgentProfileStore( ); }, + setAvatar(actor, id, image) { + return database.transaction(async (transaction) => { + await lockProfileMutationRows(transaction, id); + const profile = await findAccessibleProfile(transaction, actor, id); + if (!profile) throw new AgentNotFoundError(id); + requireAvatarManageable(actor, profile); + + await transaction + .update(agentProfiles) + .set({ avatarImage: image, updatedAt: new Date() }) + .where(eq(agentProfiles.agentId, id)); + + const updated = await findAccessibleProfile(transaction, actor, id); + if (!updated) throw new AgentNotFoundError(id); + return updated; + }); + }, + duplicate(actor, id) { return database.transaction(async (transaction) => { const source = await findAccessibleProfile(transaction, actor, id); if (!source) throw new AgentNotFoundError(id); + const [sourceAvatar] = await transaction + .select({ image: agentProfiles.avatarImage }) + .from(agentProfiles) + .where(eq(agentProfiles.agentId, id)) + .limit(1); if (!managedConfiguration) { throw new ManagedAgentUnavailableError(); @@ -453,6 +504,7 @@ export function createAgentProfileStore( title: source.title, roleDescription: source.roleDescription, avatarSeed: source.avatarSeed, + avatarImage: sourceAvatar?.image, visibility: "private", }); diff --git a/server/src/agents/profile-types.ts b/server/src/agents/profile-types.ts index 57dfe3960..2d9fb75f7 100644 --- a/server/src/agents/profile-types.ts +++ b/server/src/agents/profile-types.ts @@ -11,6 +11,10 @@ export type AgentProfile = { title: string; roleDescription: string; avatarSeed: string; + /** Whether an authenticated image endpoint has bytes to serve. */ + hasCustomAvatar: boolean; + /** Content version for immutable avatar URLs, without selecting the image payload. */ + avatarUpdatedAt: Date; visibility: AgentVisibility; ownerUserId: string | null; systemOwned: boolean; diff --git a/server/src/agents/routes.ts b/server/src/agents/routes.ts index 7a2302284..f1de04281 100644 --- a/server/src/agents/routes.ts +++ b/server/src/agents/routes.ts @@ -1,11 +1,13 @@ import type { Context, MiddlewareHandler } from "hono"; import { Hono } from "hono"; +import { bodyLimit } from "hono/body-limit"; import type { AuditEventType, AuditStore } from "../audit"; import { recordAuditEvent } from "../audit"; import type { AppVariables } from "../auth/guards"; +import { avatarResponse, avatarUrl, parseAvatarImage } from "../avatar"; import { testAgentConnection } from "./connection-test"; import { checkAgentEndpoint } from "./endpoint"; -import { canManageAgent } from "./profile-policy"; +import { canCustomizeAgentAvatar, canManageAgent } from "./profile-policy"; import { AgentNotFoundError, AgentNotManageableError, @@ -253,6 +255,49 @@ export function createAgentRoutes( } }); + routes.put( + "/:agentId/avatar", + requireUser, + bodyLimit({ + maxSize: 3 * 1024 * 1024, + onError: (context) => + context.json({ error: "Avatar image must be 2 MB or smaller." }, 413), + }), + async (context) => { + const body = (await context.req.json().catch(() => null)) as { + image?: unknown; + } | null; + const parsed = parseAvatarImage(body?.image); + if (!parsed.ok) return context.json({ error: parsed.error }, 400); + + try { + const agent = await store.setAvatar( + context.var.actor, + context.req.param("agentId"), + parsed.value, + ); + return context.json({ agent: dto(context.var.actor, agent) }); + } catch (error) { + return mapStoreError(context, error); + } + }, + ); + + routes.get("/:agentId/avatar/image", requireUser, async (context) => { + try { + const image = await store.avatar( + context.var.actor, + context.req.param("agentId"), + ); + if (!image) { + return context.json({ error: "Avatar not found." }, 404); + } + return avatarResponse(image); + } catch (error) { + return mapStoreError(context, error); + } + }); + /** * What kinds of coworker this deployment can create, for the screen that asks. * @@ -556,6 +601,10 @@ function agentDto(actor: AgentActor, agent: AgentProfile) { title: agent.title, roleDescription: agent.roleDescription, avatarSeed: agent.avatarSeed, + avatarUrl: avatarUrl( + `/api/agents/${encodeURIComponent(agent.id)}/avatar/image`, + agent.hasCustomAvatar ? agent.avatarUpdatedAt : null, + ), visibility: agent.visibility, hidden: agent.hidden, systemOwned: agent.systemOwned, @@ -566,6 +615,7 @@ function agentDto(actor: AgentActor, agent: AgentProfile) { // Whether one exists, never what it is. hasCallbackToken: agent.hasCallbackToken, canManage: canManageAgent(actor, agent), + canCustomizeAvatar: canCustomizeAgentAvatar(actor, agent), // Ownership, kept separate from permission. `canManage` is also true for an administrator on // another user's coworker, so a roster that split "mine" on it would file other people's work // under yours, and only for administrators, who are the least likely to notice. diff --git a/server/src/app.ts b/server/src/app.ts index ff81b1783..9f7be7643 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -1,5 +1,6 @@ import type { Hono as HonoApp, MiddlewareHandler } from "hono"; import { Hono } from "hono"; +import { bodyLimit } from "hono/body-limit"; import { serveStatic } from "hono/bun"; import { authoriseAgentCall, sameToken } from "./agents/callback-token"; import type { BotAccessCheck } from "./agents/profile-policy"; @@ -20,6 +21,7 @@ import { requireAdmin, } from "./auth/guards"; import type { IdentityProviderStore } from "./auth/identity-provider-store"; +import { avatarResponse, avatarUrl, parseAvatarImage } from "./avatar"; import type { ChannelEventHub } from "./channels/events"; import { type ChannelStore, createChannelRoutes } from "./channels/routes"; import type { ThreadIdentity } from "./channels/thread-identity"; @@ -29,7 +31,7 @@ import { createComponentRoutes } from "./components/routes"; import type { SandboxedStore } from "./components/sandboxed"; import { createSandboxedRoutes } from "./components/sandboxed-routes"; import type { ComponentStore } from "./components/store"; -import type { ComputerGateway } from "./computer/gateway"; +import type { ActionActor, ComputerGateway } from "./computer/gateway"; import type { PageFrameStore } from "./computer/page-frames"; import type { PolicyStore } from "./computer/policy-store"; import { createComputerRoutes } from "./computer/routes"; @@ -73,6 +75,157 @@ async function recordPersonEvent( }); } +const CODEX_COMPUTER_TOOL_PREFIX = "openbot_computer_"; + +/** + * Execute a Codex computer alias through the same gateway used by the browser tools. + * + * The alias matters: AG-UI also carries these tool events back to the browser, where the unprefixed + * `computer_*` names have frontend handlers. Returning an unprefixed name after executing here would + * click, type, or navigate twice. + */ +async function callCodexComputerTool( + gateway: ComputerGateway, + name: string, + args: Record<string, unknown>, + botId: string, + actorId: string, + signal?: AbortSignal, +): Promise<unknown> { + if (!name.startsWith(CODEX_COMPUTER_TOOL_PREFIX)) return undefined; + + const tool = name.slice("openbot_".length); + const actor: ActionActor = { id: actorId }; + switch (tool) { + case "computer_navigate": + return gateway.navigate(botId, actor, stringArgument(args, "url")); + case "computer_read": + return gateway.read(botId); + case "computer_snapshot": + return gateway.snapshot(botId); + case "computer_click": + return gateway.click( + botId, + actor, + { + ref: stringArgument(args, "ref"), + snapshotId: numberArgument(args, "snapshotId"), + }, + signal, + ); + case "computer_type": { + const submit = booleanArgument(args, "submit"); + return gateway.type( + botId, + actor, + { + ref: stringArgument(args, "ref"), + snapshotId: numberArgument(args, "snapshotId"), + text: stringArgument(args, "text"), + ...(submit === undefined ? {} : { submit }), + }, + signal, + ); + } + case "computer_key": { + const ref = optionalStringArgument(args, "ref"); + const snapshotId = optionalNumberArgument(args, "snapshotId"); + return gateway.key( + botId, + actor, + { + key: stringArgument(args, "key"), + ...(ref === undefined ? {} : { ref }), + ...(snapshotId === undefined ? {} : { snapshotId }), + }, + signal, + ); + } + case "computer_scroll": { + const deltaY = optionalNumberArgument(args, "deltaY"); + return gateway.scroll( + botId, + actor, + deltaY === undefined ? {} : { deltaY }, + ); + } + case "computer_list_files": { + const path = optionalStringArgument(args, "path"); + return gateway.listFiles( + botId, + actor, + path === undefined ? {} : { path }, + ); + } + case "computer_read_file": + return gateway.readFile(botId, actor, { + path: stringArgument(args, "path"), + }); + case "computer_run_command": + return gateway.runCommand( + botId, + actor, + { command: stringArgument(args, "command") }, + signal, + ); + case "computer_write_file": { + const append = booleanArgument(args, "append"); + return gateway.writeFile(botId, actor, { + path: stringArgument(args, "path"), + contents: stringArgument(args, "contents"), + ...(append === undefined ? {} : { append }), + }); + } + default: + throw new Error("That computer tool is not available to remote agents."); + } +} + +function stringArgument(args: Record<string, unknown>, name: string): string { + const value = args[name]; + if (typeof value !== "string") { + throw new Error(`${name} must be a string.`); + } + return value; +} + +function optionalStringArgument( + args: Record<string, unknown>, + name: string, +): string | undefined { + const value = args[name]; + if (value === undefined) return undefined; + return stringArgument(args, name); +} + +function numberArgument(args: Record<string, unknown>, name: string): number { + const value = args[name]; + if (typeof value !== "number" || !Number.isFinite(value)) { + throw new Error(`${name} must be a number.`); + } + return value; +} + +function optionalNumberArgument( + args: Record<string, unknown>, + name: string, +): number | undefined { + if (args[name] === undefined) return undefined; + return numberArgument(args, name); +} + +function booleanArgument( + args: Record<string, unknown>, + name: string, +): boolean | undefined { + const value = args[name]; + if (value === undefined) return undefined; + if (typeof value !== "boolean") { + throw new Error(`${name} must be a boolean.`); + } + return value; +} + export function createApp( config: DeploymentConfig, auth?: AuthService, @@ -298,10 +451,17 @@ export function createApp( ? createRequireUser(auth, roleRepository) : authenticationUnavailable; - app.get("/api/me", requireUser, async (context) => - context.json({ + app.get("/api/me", requireUser, async (context) => { + const avatarImage = + (await peopleStore?.avatar(context.var.actor.id)) ?? null; + return context.json({ user: { ...context.var.actor, + image: + avatarUrl("/api/me/avatar/image", avatarImage) ?? + context.var.actor.image ?? + null, + hasCustomAvatar: avatarImage !== null, /* * Read here rather than in the guard, so only this route pays the extra query. Null means * this deployment does not track onboarding, which the app reads as nothing to finish; @@ -311,8 +471,46 @@ export function createApp( ? await onboardingStore.status(context.var.actor.id) : null, }, + }); + }); + app.put( + "/api/me/avatar", + requireUser, + bodyLimit({ + maxSize: 3 * 1024 * 1024, + onError: (context) => + context.json({ error: "Avatar image must be 2 MB or smaller." }, 413), }), + async (context) => { + if (!peopleStore) { + return context.json( + { error: "Profile storage is not available." }, + 503, + ); + } + const body = (await context.req.json().catch(() => null)) as { + image?: unknown; + } | null; + const parsed = parseAvatarImage(body?.image); + if (!parsed.ok) return context.json({ error: parsed.error }, 400); + + await peopleStore.setAvatar(context.var.actor.id, parsed.value); + return context.json({ + avatar: { + image: + avatarUrl("/api/me/avatar/image", parsed.value) ?? + context.var.actor.image ?? + null, + hasCustomAvatar: parsed.value !== null, + }, + }); + }, ); + app.get("/api/me/avatar/image", requireUser, async (context) => { + const image = (await peopleStore?.avatar(context.var.actor.id)) ?? null; + if (!image) return context.json({ error: "Avatar not found." }, 404); + return avatarResponse(image); + }); app.post("/api/me/onboarding", requireUser, async (context) => { if (!onboardingStore) { return context.json({ error: "Onboarding is not available." }, 503); @@ -952,7 +1150,7 @@ export function createApp( * no person behind it. Absent secret means the route does not exist: a deployment that has not * configured this refuses rather than accepting anybody who can reach the port. */ - if (pluginStore) { + if (pluginStore || computerGateway) { const legacyToken = config.agentToolToken ?? ""; app.post("/api/agent-tools/call", async (context) => { /* @@ -1019,6 +1217,38 @@ export function createApp( } try { + if (body.name.startsWith(CODEX_COMPUTER_TOOL_PREFIX)) { + if (!computerGateway) { + throw new Error( + "This deployment has no governed computer gateway.", + ); + } + const args = + body.args && + typeof body.args === "object" && + !Array.isArray(body.args) + ? body.args + : {}; + const result = await callCodexComputerTool( + computerGateway, + body.name, + args, + verdict.botId, + verdict.actorId, + context.req.raw.signal, + ); + const outcome = + result && typeof result === "object" && !Array.isArray(result) + ? { ok: true, ...result } + : { ok: true, result }; + return context.json({ + text: JSON.stringify(outcome), + isError: false, + }); + } + if (!pluginStore) { + throw new Error("This deployment has no plugin tool gateway."); + } const result = await pluginStore.callTool({ // The model is offered `mcp__server__tool`; the store speaks `server/tool`. ref: body.name.replace(/^mcp__/, "").replace("__", "/"), diff --git a/server/src/avatar.ts b/server/src/avatar.ts new file mode 100644 index 000000000..2c8d13dbc --- /dev/null +++ b/server/src/avatar.ts @@ -0,0 +1,188 @@ +/** The largest decoded avatar OpenBot stores in PostgreSQL. */ +export const MAX_AVATAR_BYTES = 2 * 1024 * 1024; + +/** A small enough ceiling that a compressed image cannot make the browser allocate an absurd bitmap. */ +const MAX_AVATAR_EDGE = 4096; +const MAX_AVATAR_PIXELS = 16_777_216; + +const DATA_URL = + /^data:(image\/(?:jpeg|png|webp));base64,([A-Za-z0-9+/]+={0,2})$/; + +export type AvatarImageResult = + | { ok: true; value: string | null } + | { ok: false; error: string }; + +/** + * Validate an uploaded avatar at the trust boundary. + * + * The browser sends a data URL because avatars are small and the database is shared by every + * server replica. The MIME label is not trusted: the decoded bytes must carry the matching image + * signature, and dimensions are bounded before any browser is asked to decode them. + */ +export function parseAvatarImage(value: unknown): AvatarImageResult { + if (value === null) return { ok: true, value: null }; + if (typeof value !== "string") { + return { + ok: false, + error: "Avatar image must be a PNG, JPEG, or WebP file.", + }; + } + + const match = DATA_URL.exec(value); + if (!match || match[2].length % 4 !== 0) { + return { + ok: false, + error: "Avatar image must be a PNG, JPEG, or WebP file.", + }; + } + + const mime = match[1]; + const encoded = match[2]; + const bytes = Buffer.from(encoded, "base64"); + if ( + bytes.length === 0 || + bytes.length > MAX_AVATAR_BYTES || + bytes.toString("base64") !== encoded + ) { + return { + ok: false, + error: `Avatar image must be ${MAX_AVATAR_BYTES / 1024 / 1024} MB or smaller.`, + }; + } + + const dimensions = imageDimensions(mime, bytes); + if (!dimensions) { + return { + ok: false, + error: "Avatar image does not match its PNG, JPEG, or WebP format.", + }; + } + if ( + dimensions.width > MAX_AVATAR_EDGE || + dimensions.height > MAX_AVATAR_EDGE || + dimensions.width * dimensions.height > MAX_AVATAR_PIXELS + ) { + return { + ok: false, + error: `Avatar image dimensions must be at most ${MAX_AVATAR_EDGE} by ${MAX_AVATAR_EDGE} pixels.`, + }; + } + + return { ok: true, value }; +} + +/** A stable, cache-busting URL without copying the image into every JSON response. */ +export function avatarUrl( + path: string, + versionSource: string | Date | null, +): string | null { + if (!versionSource) return null; + if (versionSource instanceof Date) { + return `${path}?v=${versionSource.getTime().toString(36)}`; + } + // FNV-1a is not a security claim, just a cheap content version. A collision costs one stale cache. + let version = 0x811c9dc5; + for (let index = 0; index < versionSource.length; index += 1) { + version ^= versionSource.charCodeAt(index); + version = Math.imul(version, 0x01000193); + } + return `${path}?v=${(version >>> 0).toString(36)}`; +} + +/** Serve one already-validated image at its versioned authenticated URL. */ +export function avatarResponse(image: string): Response { + const match = DATA_URL.exec(image); + if (!match) return new Response("Avatar not found.", { status: 404 }); + return new Response(new Uint8Array(Buffer.from(match[2], "base64")), { + headers: { + "cache-control": "private, max-age=31536000, immutable", + "content-type": match[1], + "x-content-type-options": "nosniff", + }, + }); +} + +function imageDimensions( + mime: string, + bytes: Buffer, +): { width: number; height: number } | null { + if (mime === "image/png") return pngDimensions(bytes); + if (mime === "image/jpeg") return jpegDimensions(bytes); + return webpDimensions(bytes); +} + +function pngDimensions(bytes: Buffer) { + const signature = Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, + ]); + if (bytes.length < 24 || !bytes.subarray(0, 8).equals(signature)) return null; + const width = bytes.readUInt32BE(16); + const height = bytes.readUInt32BE(20); + return width > 0 && height > 0 ? { width, height } : null; +} + +function jpegDimensions(bytes: Buffer) { + if (bytes.length < 4 || bytes[0] !== 0xff || bytes[1] !== 0xd8) return null; + const startOfFrame = new Set([ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc5, 0xc6, 0xc7, 0xc9, 0xca, 0xcb, 0xcd, 0xce, + 0xcf, + ]); + let offset = 2; + while (offset + 3 < bytes.length) { + if (bytes[offset] !== 0xff) return null; + while (bytes[offset] === 0xff) offset += 1; + const marker = bytes[offset]; + offset += 1; + if (marker === undefined || marker === 0xd9 || marker === 0xda) return null; + if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) continue; + if (offset + 2 > bytes.length) return null; + const length = bytes.readUInt16BE(offset); + if (length < 2 || offset + length > bytes.length) return null; + if (startOfFrame.has(marker)) { + if (length < 7) return null; + const height = bytes.readUInt16BE(offset + 3); + const width = bytes.readUInt16BE(offset + 5); + return width > 0 && height > 0 ? { width, height } : null; + } + offset += length; + } + return null; +} + +function webpDimensions(bytes: Buffer) { + if ( + bytes.length < 30 || + bytes.toString("ascii", 0, 4) !== "RIFF" || + bytes.toString("ascii", 8, 12) !== "WEBP" + ) { + return null; + } + + const kind = bytes.toString("ascii", 12, 16); + if (kind === "VP8X") { + const width = 1 + readUInt24LE(bytes, 24); + const height = 1 + readUInt24LE(bytes, 27); + return { width, height }; + } + if (kind === "VP8L" && bytes[20] === 0x2f) { + const width = 1 + (bytes[21] | ((bytes[22] & 0x3f) << 8)); + const height = + 1 + ((bytes[22] >> 6) | (bytes[23] << 2) | ((bytes[24] & 0x0f) << 10)); + return { width, height }; + } + if ( + kind === "VP8 " && + bytes[23] === 0x9d && + bytes[24] === 0x01 && + bytes[25] === 0x2a + ) { + const width = bytes.readUInt16LE(26) & 0x3fff; + const height = bytes.readUInt16LE(28) & 0x3fff; + return width > 0 && height > 0 ? { width, height } : null; + } + return null; +} + +function readUInt24LE(bytes: Buffer, offset: number) { + return bytes[offset] | (bytes[offset + 1] << 8) | (bytes[offset + 2] << 16); +} diff --git a/server/src/channels/routes.ts b/server/src/channels/routes.ts index 58c3864e9..94e340aa1 100644 --- a/server/src/channels/routes.ts +++ b/server/src/channels/routes.ts @@ -45,6 +45,10 @@ export type AgentChannel = { /** A channel plus the last thing said in it, which is what a roster renders. */ export type ChannelSummary = AgentChannel & { + /** Parallel to agentIds, without copying image payloads into every roster row. */ + hasCustomAvatars: boolean[]; + /** Content version source for immutable avatar URLs, parallel to agentIds. */ + avatarUpdatedAts: Date[]; lastMessage: string | null; lastMessageAt: Date | null; lastMessageAgentId: string | null; @@ -477,6 +481,8 @@ export function createChannelStore( id: channels.id, name: channels.name, agentId: channelAgents.agentId, + hasAvatarImage: sql<boolean>`${agentProfiles.avatarImage} is not null`, + avatarUpdatedAt: agentProfiles.updatedAt, threadId: intelligenceChannelMappings.threadId, deletedAt: agentProfiles.deletedAt, lastMessage: channels.lastMessage, @@ -528,6 +534,8 @@ export function createChannelStore( const summary = summaries.get(row.id); if (summary) { summary.agentIds.push(row.agentId); + summary.hasCustomAvatars.push(row.hasAvatarImage); + summary.avatarUpdatedAts.push(row.avatarUpdatedAt); summary.active &&= row.deletedAt === null; continue; } @@ -535,6 +543,8 @@ export function createChannelStore( id: row.id, name: row.name, agentIds: [row.agentId], + hasCustomAvatars: [row.hasAvatarImage], + avatarUpdatedAts: [row.avatarUpdatedAt], threadId: row.threadId, active: row.deletedAt === null, lastMessage: row.lastMessage, @@ -1169,6 +1179,11 @@ function channelDto(channel: AgentChannel): AgentChannel { function channelSummaryDto(channel: ChannelSummary) { return { ...channelDto(channel), + avatarUrls: channel.agentIds.map((agentId, index) => + channel.hasCustomAvatars[index] + ? `/api/agents/${encodeURIComponent(agentId)}/avatar/image?v=${channel.avatarUpdatedAts[index]?.getTime().toString(36)}` + : null, + ), lastMessage: channel.lastMessage, // Serialised as ISO-8601 so the browser gets a string it can sort and format. lastMessageAt: channel.lastMessageAt?.toISOString() ?? null, diff --git a/server/src/computer/gateway.ts b/server/src/computer/gateway.ts index 21c7d985d..468764f1d 100644 --- a/server/src/computer/gateway.ts +++ b/server/src/computer/gateway.ts @@ -17,6 +17,7 @@ * by the model is theatre: "never click Submit" is evaded by sending `{ref: "e13", name: "Continue"}`. * The refs are opaque to the caller precisely so that the server holds the mapping. */ +import { createHash, randomBytes } from "node:crypto"; import { type AuditStore, recordAuditEvent } from "../audit"; import { ComputerUnavailableError, @@ -47,6 +48,7 @@ import type { ActionResult, ClickInput, ComputerStatus, + ControlLeaseState, ControlState, HumanInput, HumanInputResult, @@ -94,6 +96,27 @@ export type ActionActor = { userId?: string; }; +const ACTOR_FINGERPRINT_LENGTH = 43; + +/** + * Bind an opaque lease to the signed-in actor without keeping replica-local server state. + * + * The random suffix is the capability checked by the computer. The actor digest is checked by every + * OpenBot HTTP and WebSocket entry point. It need not be a signature: changing the digest changes + * the exact lease and the computer rejects it, while replaying the original from another account is + * rejected here. + */ +export function mintControlLease(actorId: string): string { + const actor = createHash("sha256").update(actorId).digest("base64url"); + return `${actor}${randomBytes(32).toString("base64url")}`; +} + +export function controlLeaseBelongsTo(lease: string, actorId: string): boolean { + if (!/^[A-Za-z0-9_-]{86}$/.test(lease)) return false; + const expected = createHash("sha256").update(actorId).digest("base64url"); + return lease.slice(0, ACTOR_FINGERPRINT_LENGTH) === expected; +} + export type ComputerGatewayOptions = { provider: ComputerProvider; auditStore: AuditStore; @@ -184,8 +207,17 @@ export interface ComputerGateway { actor: ActionActor, reason: string, ): Promise<ControlState>; - takeControl(botId: string, actor: ActionActor): Promise<ControlState>; - releaseControl(botId: string, actor: ActionActor): Promise<ControlState>; + takeControl(botId: string, actor: ActionActor): Promise<ControlLeaseState>; + renewControl( + botId: string, + actor: ActionActor, + lease: string, + ): Promise<ControlState>; + releaseControl( + botId: string, + actor: ActionActor, + lease: string, + ): Promise<ControlState>; requestSecret( botId: string, actor: ActionActor, @@ -196,7 +228,12 @@ export interface ComputerGateway { actor: ActionActor, text: string, ): Promise<SecretResult>; - humanInput(botId: string, input: HumanInput): Promise<HumanInputResult>; + humanInput( + botId: string, + actor: ActionActor, + input: HumanInput, + lease: string, + ): Promise<HumanInputResult>; computers(): Promise<{ isolation: "per-bot" | "shared"; computers: { @@ -241,6 +278,16 @@ export function createComputerGateway( const snapshots = options.snapshots ?? createInMemorySnapshotStore(); const pageFrames = options.pageFrames; + // A closed tab stops renewing after at most this long. Long enough for a transient network pause, + // short enough that an abandoned takeover does not strand the Bot behind an invisible driver. + const CONTROL_LEASE_TTL_MS = 2 * 60 * 1000; + const leaseExpiry = () => + new Date(Date.now() + CONTROL_LEASE_TTL_MS).toISOString(); + const newControlLease = (actorId: string) => ({ + lease: mintControlLease(actorId), + expiresAt: leaseExpiry(), + }); + /** * Where this Bot's computer is, checked before anything is sent to it. * @@ -647,7 +694,12 @@ export function createComputerGateway( }, async takeControl(botId: string, actor: ActionActor) { - const state = await post<ControlState>(botId, "/control/take", {}); + const capability = newControlLease(actor.id); + const state = await post<ControlState>( + botId, + "/control/take", + capability, + ); await writeControlEvent(auditStore, "computer.control_taken", { botId, actor, @@ -655,11 +707,26 @@ export function createComputerGateway( // took over. reason: state.reason, }); - return state; + return { ...state, lease: capability.lease }; }, - async releaseControl(botId: string, actor: ActionActor) { - const state = await post<ControlState>(botId, "/control/release", {}); + renewControl(botId: string, actor: ActionActor, lease: string) { + if (!controlLeaseBelongsTo(lease, actor.id)) { + throw new Error("This control lease belongs to another session."); + } + return post<ControlState>(botId, "/control/renew", { + lease, + expiresAt: leaseExpiry(), + }); + }, + + async releaseControl(botId: string, actor: ActionActor, lease: string) { + if (!controlLeaseBelongsTo(lease, actor.id)) { + throw new Error("This control lease belongs to another session."); + } + const state = await post<ControlState>(botId, "/control/release", { + lease, + }); await writeControlEvent(auditStore, "computer.control_released", { botId, actor, @@ -770,8 +837,13 @@ export function createComputerGateway( async humanInput( botId: string, + actor: ActionActor, input: HumanInput, + lease: string, ): Promise<HumanInputResult> { + if (!controlLeaseBelongsTo(lease, actor.id)) { + throw new Error("This control lease belongs to another session."); + } const { kind, ...payload } = input; /* * Checked here as well as at the route, because this is where it becomes a path. @@ -788,7 +860,10 @@ export function createComputerGateway( `A person's input is one of ${[...HUMAN_GESTURES].join(", ")}, not ${JSON.stringify(kind)}.`, ); } - return post<HumanInputResult>(botId, `/human/${kind}`, payload); + return post<HumanInputResult>(botId, `/human/${kind}`, { + ...payload, + lease, + }); }, /** diff --git a/server/src/computer/routes.ts b/server/src/computer/routes.ts index 21b24632c..173a9b190 100644 --- a/server/src/computer/routes.ts +++ b/server/src/computer/routes.ts @@ -8,6 +8,7 @@ import { DEPLOYMENT_ROUTES } from "./deployment-routes"; import { type ActionActor, ActionRefusedError, + controlLeaseBelongsTo, type ComputerGateway, ComputerUnavailableError, ElementNotFoundError, @@ -417,8 +418,28 @@ export function createComputerRoutes( act(context, (botId, actor) => gateway.takeControl(botId, actor)), ); + routes.post("/:botId/control/renew", (context) => + act(context, (botId, actor, body) => { + if (typeof body?.lease !== "string") { + return { error: "A control lease is required." }; + } + if (!controlLeaseBelongsTo(body.lease, actor.id)) { + return { error: "This control lease belongs to another session." }; + } + return gateway.renewControl(botId, actor, body.lease); + }), + ); + routes.post("/:botId/control/release", (context) => - act(context, (botId, actor) => gateway.releaseControl(botId, actor)), + act(context, (botId, actor, body) => { + if (typeof body?.lease !== "string") { + return { error: "A control lease is required." }; + } + if (!controlLeaseBelongsTo(body.lease, actor.id)) { + return { error: "This control lease belongs to another session." }; + } + return gateway.releaseControl(botId, actor, body.lease); + }), ); /** The Bot asking for a value it must not be told. */ @@ -481,15 +502,30 @@ export function createComputerRoutes( string, unknown > | null; + if (typeof body?.lease !== "string") { + return context.json({ error: "A control lease is required." }, 400); + } + const actor = actionActorOf(context); + if (!controlLeaseBelongsTo(body.lease, actor.id)) { + return context.json( + { error: "This control lease belongs to another session." }, + 403, + ); + } try { return context.json( - await gateway.humanInput(context.req.param("botId"), { - ...(body ?? {}), - // Last, so the checked value wins. Spread over it, a body carrying its own `kind` replaced - // the one this route had just checked, and the gateway puts that value into the path it - // calls on the computer. - kind, - } as Parameters<typeof gateway.humanInput>[1]), + await gateway.humanInput( + context.req.param("botId"), + actor, + { + ...body, + // Last, so the checked value wins. Spread over it, a body carrying its own `kind` replaced + // the one this route had just checked, and the gateway puts that value into the path it + // calls on the computer. + kind, + } as Parameters<typeof gateway.humanInput>[2], + body.lease, + ), ); } catch (error) { return context.json(errorBody(error), statusFor(error)); @@ -679,6 +715,14 @@ export function createComputerRoutes( type ComputerContext = Context<{ Variables: AppVariables }>; +function actionActorOf(context: ComputerContext): ActionActor { + const record = context.var.actor; + return { + id: record.id, + ...(record.email === DEV_ACTOR_EMAIL ? {} : { userId: record.id }), + }; +} + /** A request that was rejected before any decision was needed, because it was not a valid action. */ type BadRequest = { error: string }; @@ -715,7 +759,6 @@ async function act( // helper is typed against a generic context that cannot know that, and a thrown "undefined bot" // would be a worse outcome than naming the one shared computer. const botId = context.req.param("botId") ?? "default"; - const record = context.var.actor; const body = (await context.req.json().catch(() => null)) as Record< string, unknown @@ -724,13 +767,10 @@ async function act( try { const result = await handler( botId, - { - id: record.id, - // Only a real users row may go in the audit table's foreign key column. The local development - // actor is not one, so writing it there fails the constraint and loses the row entirely. Who - // it was is recorded in the payload regardless. See gateway.ts. - ...(record.email === DEV_ACTOR_EMAIL ? {} : { userId: record.id }), - }, + // Only a real users row may go in the audit table's foreign key column. The local development + // actor is not one, so writing it there fails the constraint and loses the row entirely. Who + // it was is recorded in the payload regardless. See gateway.ts. + actionActorOf(context), body, context.req.raw.signal, ); diff --git a/server/src/computer/sandbox.ts b/server/src/computer/sandbox.ts index 3169a973c..53cacce16 100644 --- a/server/src/computer/sandbox.ts +++ b/server/src/computer/sandbox.ts @@ -100,7 +100,7 @@ export async function readSandboxTemplate( raw = await readFile(path, "utf8"); } catch { throw new SandboxError( - `COMPUTER_SANDBOX_TEMPLATE_FILE points at ${path}, which cannot be read. That file is what a Bot's computer is cut from; the chart mounts it when computers.mode is sandbox.`, + `COMPUTER_SANDBOX_TEMPLATE_FILE points at ${path}, which cannot be read. That file is what a Bot's computer is cut from; the chart mounts it when computers.mode is sandbox or vm.`, ); } const parsed = JSON.parse(raw) as Record<string, unknown>; diff --git a/server/src/computer/schema.ts b/server/src/computer/schema.ts index 70e88ac94..ca12c3023 100644 --- a/server/src/computer/schema.ts +++ b/server/src/computer/schema.ts @@ -276,6 +276,14 @@ export type ControlState = { requested: boolean; }; +/** + * Returned only to the browser session that successfully takes the wheel. + * + * The lease is deliberately absent from `ControlState`: observers may see that a person is driving, + * but they cannot turn that observation into permission to send mouse or keyboard input. + */ +export type ControlLeaseState = ControlState & { lease: string }; + /** * A value the Bot needs and must not be told: a password, a one-time code. * diff --git a/server/src/computer/socket-proxy.ts b/server/src/computer/socket-proxy.ts new file mode 100644 index 000000000..a323ec434 --- /dev/null +++ b/server/src/computer/socket-proxy.ts @@ -0,0 +1,82 @@ +export type ComputerSocketKind = "stream" | "desktop"; + +export type ComputerSocketPath = { + botId: string; + kind: ComputerSocketKind; +}; + +const CONTROL_LEASE_PROTOCOL = "openbot-lease."; +const CONTROL_LEASE_TOKEN = /^[A-Za-z0-9_-]{32,256}$/; + +/** + * Read the browser's private control capability without putting it in a public URL. + * + * WebSocket subprotocols cross the authenticated upgrade but do not appear in the address bar, + * browser history, reverse-proxy query logs, or screenshot tooling. The upstream URL is internal. + */ +export function parseComputerSocketLease( + requestedProtocols: string | null, +): string | undefined { + for (const protocol of (requestedProtocols ?? "").split(",")) { + const candidate = protocol.trim(); + if (!candidate.startsWith(CONTROL_LEASE_PROTOCOL)) continue; + const lease = candidate.slice(CONTROL_LEASE_PROTOCOL.length); + if (CONTROL_LEASE_TOKEN.test(lease)) return lease; + } + return undefined; +} + +/** + * A framebuffer contains every window on its X display, not just the browser profile named in the + * URL. It is therefore safe to proxy only when the provider gives that Bot the whole computer. + * Page streaming remains available on shared providers because CDP keeps those sessions separate. + */ +export function computerSocketIsolationRefusal( + kind: ComputerSocketKind, + isolation: "per-bot" | "shared" | undefined, +): string | undefined { + if (kind === "desktop" && isolation !== "per-bot") { + return "A full desktop requires one isolated computer per Bot."; + } + return undefined; +} + +/** The only two computer WebSockets the public server will proxy. */ +export function parseComputerSocketPath( + pathname: string, +): ComputerSocketPath | null { + const match = pathname.match(/^\/api\/computers\/([^/]+)\/(stream|desktop)$/); + if (!match?.[1] || !match[2]) return null; + try { + return { + botId: decodeURIComponent(match[1]), + kind: match[2] as ComputerSocketKind, + }; + } catch { + return null; + } +} + +/** + * Build the internal socket without ever exposing the computer token to the browser. + * + * Unknown desktop modes fail closed to the read-only connection. The computer repeats the control + * check at upgrade and for every control message, so this URL is routing rather than authority. + */ +export function computerSocketUrl(input: { + baseUrl: string; + botId: string; + kind: ComputerSocketKind; + token: string; + mode?: string | null; + lease?: string; +}): string { + const mode = input.mode === "control" ? "control" : "view"; + const query = new URLSearchParams({ + bot: input.botId, + token: input.token, + ...(input.kind === "desktop" ? { mode } : {}), + ...(input.lease ? { lease: input.lease } : {}), + }); + return `${input.baseUrl.replace(/^http/, "ws").replace(/\/$/, "")}/${input.kind}?${query}`; +} diff --git a/server/src/copilot.ts b/server/src/copilot.ts index 547e329a7..dfa925cd6 100644 --- a/server/src/copilot.ts +++ b/server/src/copilot.ts @@ -16,6 +16,7 @@ import { } from "../../shared/bot-prompt"; import type { AgentActor } from "./agents/profile-types"; import type { AgentFetch, StallGuard } from "./channels/stall-guard"; +import { COMPUTER_TOOLS } from "./computer/schema"; import type { DeploymentConfig } from "./config"; import type { SelectableSkill, Selection } from "./plugins/selection"; import { @@ -26,6 +27,26 @@ import { import type { GrantedTool } from "./plugins/tools"; import { grantedToolGuidance } from "./plugins/tools"; +/** + * Browser tools that a remote Bot may call back through this deployment. + * + * Human handoff and secret-entry tools deliberately stay in the browser: their handlers wait for a + * person for up to ten minutes, while a remote agent callback is one synchronous HTTP request. The + * tools here all have a complete server-side implementation in `ComputerGateway`, so the remote Bot + * gets the same policy decision and audit row as a built-in Bot. + */ +const REMOTE_COMPUTER_TOOLS = new Set<string>([ + ...COMPUTER_TOOLS, + // The command tool predates COMPUTER_TOOLS but is governed by the same gateway. + "computer_run_command", +]); + +export function remoteComputerToolAliases(tools: RunAgentInput["tools"] = []) { + return tools + .filter((tool) => REMOTE_COMPUTER_TOOLS.has(tool.name)) + .map((tool) => ({ ...tool, name: `openbot_${tool.name}` })); +} + /** * The CopilotKit runtime, always in Intelligence mode. * @@ -586,6 +607,13 @@ function remoteAgentWithStandingRole( next: AbstractAgent, ) => { const holdingsMessage = holdingsMessageFor(tools); + /* + * The browser already registered the `computer_*` names as frontend handlers. If the remote Bot + * called one of those names after the server executed it, AG-UI would make the browser execute + * the same action again. The remote endpoint gets an aliased copy; only the signed callback maps + * it back to the governed gateway. + */ + const computerTools = remoteComputerToolAliases(input.tools); return next.run({ ...input, messages: [ @@ -606,6 +634,7 @@ function remoteAgentWithStandingRole( */ tools: [ ...(input.tools ?? []), + ...computerTools, ...tools.map((tool) => ({ name: tool.name, description: tool.description, @@ -628,7 +657,12 @@ function remoteAgentWithStandingRole( * could not, and then apologised to the person for not showing the chart that was on screen * in front of them. Only this side knows which is which, so only this side can say. */ - openbotDeploymentTools: tools.map((tool) => tool.name), + openbotDeploymentTools: [ + ...new Set([ + ...tools.map((tool) => tool.name), + ...computerTools.map((tool) => tool.name), + ]), + ], /* * This deployment's own statement of what this run is. * diff --git a/server/src/db/schema/core.ts b/server/src/db/schema/core.ts index 44d5753a7..25079ab95 100644 --- a/server/src/db/schema/core.ts +++ b/server/src/db/schema/core.ts @@ -54,6 +54,13 @@ export const users = pgTable("users", { email: text("email").notNull().unique(), name: text("name"), image: text("image"), + /** + * The image this person chose in OpenBot. + * + * Separate from Better Auth's `image`: an identity provider owns that field and may refresh it on + * sign-in. A person choosing a picture here must not have it quietly replaced the next morning. + */ + avatarImage: text("avatar_image"), emailVerified: boolean("email_verified").notNull().default(false), /** * The person's groups, for a group-based rule to be evaluated against. diff --git a/server/src/db/schema/coworker.ts b/server/src/db/schema/coworker.ts index 514642a5d..1b3db180b 100644 --- a/server/src/db/schema/coworker.ts +++ b/server/src/db/schema/coworker.ts @@ -37,6 +37,8 @@ export const agentProfiles = pgTable( title: text("title").notNull(), roleDescription: text("role_description").notNull(), avatarSeed: text("avatar_seed").notNull(), + /** A user-chosen image, falling back to the deterministic seed when absent. */ + avatarImage: text("avatar_image"), visibility: agentVisibility("visibility").notNull(), /* * The credential this Bot's agent presents when it calls a tool back. diff --git a/server/src/index.ts b/server/src/index.ts index 6828cb9a9..054df2d7d 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -34,7 +34,10 @@ import { createStallGuard } from "./channels/stall-guard"; import { createThreadIdentity } from "./channels/thread-identity"; import { createSandboxedStore } from "./components/sandboxed"; import { createComponentStore } from "./components/store"; -import { createComputerGateway } from "./computer/gateway"; +import { + controlLeaseBelongsTo, + createComputerGateway, +} from "./computer/gateway"; import { createPageFrameStore } from "./computer/page-frames"; import { startPolicyListener } from "./computer/policy-listener"; import { @@ -46,6 +49,12 @@ import { describeComputerIsolation, } from "./computer/provider"; import { createSnapshotStore } from "./computer/snapshot-store"; +import { + computerSocketIsolationRefusal, + computerSocketUrl, + parseComputerSocketPath, + parseComputerSocketLease, +} from "./computer/socket-proxy"; import { loadConfig } from "./config"; import { type IdentifyActor, @@ -1091,24 +1100,12 @@ const app = createApp( * Not a Hono route because an upgrade is not a request/response: Bun hands it over before Hono sees a * body, so it is handled in `fetch` ahead of the app. */ -const toStreamUrl = (baseUrl: string, botId: string) => - // The Bot travels in the query, because a websocket upgrade carries no custom header for the - // computer to read and every call it serves is per Bot. The secret travels the same way and for the - // same reason, this socket is the one a person can type into, so it is the last thing that should - // be reachable without it. - `${baseUrl.replace(/^http/, "ws").replace(/\/$/, "")}/stream?bot=${encodeURIComponent(botId)}&token=${encodeURIComponent(config.computer?.token ?? "")}`; - -/** - * Which Bot's screen. The Bot is named in the path and its computer is located the same way every - * other call locates it, so the live stream cannot point at a different Bot's browser. - */ -const streamPathBotId = (pathname: string): string | null => { - const match = pathname.match(/^\/api\/computers\/([^/]+)\/stream$/); - return match?.[1] ? decodeURIComponent(match[1]) : null; -}; - /** What each proxied socket carries: where to connect inward, and the socket once opened. */ -type StreamData = { upstream: string; inward?: WebSocket }; +type StreamData = { + upstream: string; + binary: boolean; + inward?: WebSocket; +}; /** * Bun takes exactly one WebSocket handler for the server, and two features need one: the app proxies @@ -1130,14 +1127,27 @@ serve<SocketData>({ port, async fetch(request, server) { const url = new URL(request.url); - const streamBotId = streamPathBotId(url.pathname); + const computerSocket = parseComputerSocketPath(url.pathname); if ( - streamBotId !== null && + computerSocket !== null && request.headers.get("upgrade")?.toLowerCase() === "websocket" ) { if (!config.computer) { return new Response("No computer is configured.", { status: 503 }); } + /* + * A full desktop is the process-wide X display, not a Bot-scoped browser tab. On a shared + * provider it would let somebody authorized for one Bot see and drive every other Bot's + * windows and the shared terminal. Refuse before locating the computer; the React client then + * falls back to the Bot-scoped page stream that shared mode can safely provide. + */ + const isolationRefusal = computerSocketIsolationRefusal( + computerSocket.kind, + computerProvider?.isolation, + ); + if (isolationRefusal) { + return new Response(isolationRefusal, { status: 503 }); + } // The session guard, applied by hand because middleware does not run on an upgrade. An // unauthenticated socket here would be the whole point of the proxy defeated. const actor = await resolveRequestActor(request).catch(() => null); @@ -1148,11 +1158,29 @@ serve<SocketData>({ // so signing in is not enough: without this, anybody signed in watches anybody's Bot work. if ( !(await agentProfileStore - .get({ id: actor.id, role: actor.role }, streamBotId) + .get({ id: actor.id, role: actor.role }, computerSocket.botId) .catch(() => null)) ) { return new Response("There is no such Bot.", { status: 404 }); } + const requestedProtocols = + request.headers.get("sec-websocket-protocol") ?? ""; + const controlLease = parseComputerSocketLease(requestedProtocols); + const mode = + computerSocket.kind === "desktop" && + url.searchParams.get("mode") === "control" + ? "control" + : "view"; + if (mode === "control" && !controlLease) { + return new Response("Take control before driving this desktop.", { + status: 409, + }); + } + if (controlLease && !controlLeaseBelongsTo(controlLease, actor.id)) { + return new Response("This control lease belongs to another session.", { + status: 403, + }); + } /* * Through the gateway, not the provider. * @@ -1164,14 +1192,21 @@ serve<SocketData>({ let upstream: string; try { const streamBase = computerGateway - ? await computerGateway.locate(streamBotId) + ? await computerGateway.locate(computerSocket.botId) : undefined; if (!streamBase) { return new Response("No computer address is configured.", { status: 503, }); } - upstream = toStreamUrl(streamBase, streamBotId); + upstream = computerSocketUrl({ + baseUrl: streamBase, + botId: computerSocket.botId, + kind: computerSocket.kind, + token: config.computer?.token ?? "", + mode, + lease: controlLease, + }); } catch (error) { // Said out loud rather than falling back to another Bot's computer, which is the failure this // whole path exists to prevent. @@ -1182,7 +1217,19 @@ serve<SocketData>({ { status: 502 }, ); } - if (server.upgrade(request, { data: { upstream } })) { + const binary = computerSocket.kind === "desktop"; + const acceptsBinary = requestedProtocols + .split(",") + .map((value) => value.trim()) + .includes("binary"); + if ( + server.upgrade(request, { + data: { upstream, binary }, + ...(binary && acceptsBinary + ? { headers: { "sec-websocket-protocol": "binary" } } + : {}), + }) + ) { return undefined as unknown as Response; } return new Response("Expected a WebSocket upgrade.", { status: 400 }); @@ -1195,13 +1242,23 @@ serve<SocketData>({ channelSocket.open(asChannelSocket(ws)); return; } - const inward = new WebSocket(ws.data.upstream); + const binary = ws.data.binary; + const inward = binary + ? new WebSocket(ws.data.upstream, "binary") + : new WebSocket(ws.data.upstream); + if (binary) inward.binaryType = "arraybuffer"; ws.data.inward = inward; // Frames outward, input inward. Buffered by neither side: a frame the browser is too slow for // should be dropped, not queued, because a stale frame is worse than a missing one. - inward.onmessage = (event) => { + inward.onmessage = async (event) => { try { - ws.send(String(event.data)); + if (!binary || typeof event.data === "string") { + ws.send(String(event.data)); + } else if (event.data instanceof ArrayBuffer) { + ws.send(event.data); + } else if (event.data instanceof Blob) { + ws.send(await event.data.arrayBuffer()); + } } catch { inward.close(); } @@ -1214,7 +1271,9 @@ serve<SocketData>({ channelSocket.message(asChannelSocket(ws), raw); return; } - if (ws.data.inward?.readyState === 1) ws.data.inward.send(String(raw)); + if (ws.data.inward?.readyState === 1) { + ws.data.inward.send(ws.data.binary ? raw : String(raw)); + } }, close(ws, code, reason) { if (!isProxiedStream(ws.data)) { diff --git a/server/src/people/store.ts b/server/src/people/store.ts index 587f33a9e..800dfdd63 100644 --- a/server/src/people/store.ts +++ b/server/src/people/store.ts @@ -75,6 +75,9 @@ export type PeopleStore = { restore: (userId: string) => Promise<void>; find: (userId: string) => Promise<Person | undefined>; isRevoked: (email: string) => Promise<boolean>; + /** The OpenBot-owned override, separate from the identity provider's image. */ + avatar: (userId: string) => Promise<string | null>; + setAvatar: (userId: string, image: string | null) => Promise<void>; }; /** How many people a page holds when the caller does not say. */ @@ -173,6 +176,8 @@ export function createPeopleStore( id: users.id, email: users.email, name: users.name, + // Provider image only. The OpenBot-owned avatar has its own authenticated endpoint; putting + // its data URL in this page would copy up to 2 MB into every administrator list row. image: users.image, /* * Aggregated rather than joined into duplicate rows. `user_roles` is a set and `accounts` @@ -275,6 +280,22 @@ export function createPeopleStore( list, find, + async avatar(userId) { + const [row] = await database + .select({ image: users.avatarImage }) + .from(users) + .where(eq(users.id, userId)) + .limit(1); + return row?.image ?? null; + }, + + async setAvatar(userId, image) { + await database + .update(users) + .set({ avatarImage: image, updatedAt: new Date() }) + .where(eq(users.id, userId)); + }, + async setRole(userId, role) { await setRole(database, userId, role); }, diff --git a/server/tests/agent-endpoint.test.ts b/server/tests/agent-endpoint.test.ts index 92911df49..7bf69c73a 100644 --- a/server/tests/agent-endpoint.test.ts +++ b/server/tests/agent-endpoint.test.ts @@ -1,4 +1,6 @@ import { describe, expect, test } from "bun:test"; +import { dynamicToolsOf } from "../../agent-codex/src/tools"; +import { mintRunAssertion } from "../src/agents/callback-token"; import { agentAuthHeaders, storeAgentAuth } from "../src/agents/auth-header"; import { testAgentConnection } from "../src/agents/connection-test"; import { @@ -7,6 +9,11 @@ import { EndpointNotAllowedError, } from "../src/agents/endpoint"; import { parseAgentInput } from "../src/agents/routes"; +import { createApp } from "../src/app"; +import type { ComputerGateway } from "../src/computer/gateway"; +import { loadConfig } from "../src/config"; +import { remoteComputerToolAliases } from "../src/copilot"; +import { testEnvironment } from "./support/environment"; /** A 32-byte key, as the vault expects. */ const TEST_KEY = Buffer.alloc(32, 7).toString("base64"); @@ -201,6 +208,118 @@ describe("the connection test", () => { }); }); +describe("Codex computer callbacks", () => { + test("offers only the server-owned browser alias to Codex", () => { + const frontendTools = [ + { + name: "computer_navigate", + description: "Browser-owned navigation", + parameters: { type: "object" }, + }, + { + name: "computer_request_help", + description: "Wait for a person in the browser", + parameters: { type: "object" }, + }, + ]; + const aliases = remoteComputerToolAliases(frontendTools); + const tools = dynamicToolsOf({ + threadId: "thread-1", + runId: "run-1", + state: {}, + messages: [], + context: [], + tools: [...frontendTools, ...aliases], + forwardedProps: { + openbotDeploymentTools: aliases.map((tool) => tool.name), + }, + }); + + expect(aliases.map((tool) => tool.name)).toEqual([ + "openbot_computer_navigate", + ]); + expect(tools.map((tool) => tool.name)).toEqual([ + "openbot_computer_navigate", + ]); + }); + + test("routes aliased navigation through the governed computer gateway", async () => { + const calls: Array<{ + botId: string; + actorId: string; + url: string; + }> = []; + const computerGateway = { + navigate: async (botId: string, actor: { id: string }, url: string) => { + calls.push({ botId, actorId: actor.id, url }); + return { + url, + title: "Example Domain", + text: "Example Domain", + truncated: false, + elapsedMs: 12, + }; + }, + } as unknown as ComputerGateway; + const config = loadConfig( + testEnvironment({ AGENT_TOOL_TOKEN: "agent-secret" }), + ); + const app = createApp( + config, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + computerGateway, + ); + const run = mintRunAssertion( + { + botId: "codex-assistant", + actorId: "local-development", + runId: "run-1", + }, + config.keyEncryptionKey, + ); + + const response = await app.request( + "http://openbot.test/api/agent-tools/call", + { + method: "POST", + headers: { + "content-type": "application/json", + "x-openbot-agent-token": "agent-secret", + }, + body: JSON.stringify({ + name: "openbot_computer_navigate", + args: { url: "https://example.com/" }, + run, + }), + }, + ); + + expect(response.status).toBe(200); + const body = (await response.json()) as { + text: string; + isError: boolean; + }; + expect(body.isError).toBe(false); + expect(JSON.parse(body.text)).toMatchObject({ + ok: true, + url: "https://example.com/", + title: "Example Domain", + }); + expect(calls).toEqual([ + { + botId: "codex-assistant", + actorId: "local-development", + url: "https://example.com/", + }, + ]); + }); +}); + describe("the key a customer's agent sits behind", () => { const vaultRow = { id: "cred-1", diff --git a/server/tests/agent-handoff-endtoend.integration.test.ts b/server/tests/agent-handoff-endtoend.integration.test.ts index 86584ef79..58dc58478 100644 --- a/server/tests/agent-handoff-endtoend.integration.test.ts +++ b/server/tests/agent-handoff-endtoend.integration.test.ts @@ -43,6 +43,7 @@ const ASKER = `e2e-asker-${suite}`; const TARGET = `e2e-target-${suite}`; const ACTOR = `e2e-actor-${suite}`; const RUN = `e2e-run-${suite}`; +const KIND = `bot.message.test.${suite}`; const queue = createWorkQueue(database); const auditStore = createAuditStore(database); @@ -50,6 +51,7 @@ const profiles = createAgentProfileStore(database); const desk = createHandoffDesk({ queue, + kind: KIND, profiles, // The person's own role, as the request path resolves it: an administrator sees Bots a user does // not, and a hop to one of those is theirs to make. @@ -140,6 +142,7 @@ describe("a hop, from the tool call to the delivery", () => { const delivered: Array<{ work: HandoffWork; message: string }> = []; const runner = createHandoffRunner({ queue: createWorkQueue(database), + kind: KIND, owner: `replica-${suite}`, sign: () => "signed", auditStore, @@ -190,6 +193,7 @@ describe("a hop, from the tool call to the delivery", () => { seen, runner: createHandoffRunner({ queue: createWorkQueue(database), + kind: KIND, owner, sign: () => "signed", auditStore, @@ -230,6 +234,7 @@ describe("a hop, from the tool call to the delivery", () => { const runner = createHandoffRunner({ queue: createWorkQueue(database), + kind: KIND, owner: `replica-${suite}`, sign: () => "signed", auditStore, diff --git a/server/tests/agent-handoff-runner.integration.test.ts b/server/tests/agent-handoff-runner.integration.test.ts index 5abad4486..0a6ff1a5b 100644 --- a/server/tests/agent-handoff-runner.integration.test.ts +++ b/server/tests/agent-handoff-runner.integration.test.ts @@ -24,7 +24,7 @@ const database = createDatabase( TEST_POOL, ); const queue = createWorkQueue(database); -const kind = "bot.message"; +const kind = `bot.message.test.${randomUUID()}`; const silent: AuditStore = { insert: async () => {} }; @@ -70,6 +70,7 @@ describe("a batch of hops and a lease that can run out", () => { const held = Promise.withResolvers<void>(); const shared = { queue, + kind, sign: () => "signed", auditStore: silent, // Small enough to drive in milliseconds; the property is one duration outrunning another. @@ -131,6 +132,7 @@ describe("a batch of hops and a lease that can run out", () => { const held = Promise.withResolvers<void>(); const slow = createHandoffRunner({ queue, + kind, owner: "replica-a", sign: () => "signed", auditStore: silent, diff --git a/server/tests/agent-profile-policy.test.ts b/server/tests/agent-profile-policy.test.ts index 437f5b7d2..a735f3852 100644 --- a/server/tests/agent-profile-policy.test.ts +++ b/server/tests/agent-profile-policy.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from "bun:test"; import { canAccessAgent, + canCustomizeAgentAvatar, canManageAgent, canRunAgent, } from "../src/agents/profile-policy"; @@ -17,6 +18,8 @@ function profile(overrides: Partial<AgentProfile> = {}): AgentProfile { title: "Research Assistant", roleDescription: "Finds and summarizes information.", avatarSeed: "researcher", + hasCustomAvatar: false, + avatarUpdatedAt: new Date(0), visibility: "private", ownerUserId: creator.id, systemOwned: false, @@ -69,6 +72,16 @@ describe("agent profile permissions", () => { expect(canRunAgent(actor, agent)).toBe(true); expect(canManageAgent(actor, agent)).toBe(false); } + expect(canCustomizeAgentAvatar(admin, agent)).toBe(true); + expect(canCustomizeAgentAvatar(creator, agent)).toBe(false); + }); + + test("lets an owner or administrator customize a user-owned avatar", () => { + const agent = profile(); + + expect(canCustomizeAgentAvatar(creator, agent)).toBe(true); + expect(canCustomizeAgentAvatar(otherUser, agent)).toBe(false); + expect(canCustomizeAgentAvatar(admin, agent)).toBe(true); }); test("denies every permission for deleted profiles", () => { diff --git a/server/tests/agent-profile-store.integration.test.ts b/server/tests/agent-profile-store.integration.test.ts index 01dc052e2..c7631ac67 100644 --- a/server/tests/agent-profile-store.integration.test.ts +++ b/server/tests/agent-profile-store.integration.test.ts @@ -101,6 +101,7 @@ async function createProfileFixture(options: { title?: string; roleDescription?: string; avatarSeed?: string; + avatarImage?: string; configuration?: Record<string, unknown>; }) { const agentId = id("seed-agent"); @@ -124,9 +125,17 @@ async function createProfileFixture(options: { title, roleDescription, avatarSeed, + avatarImage: options.avatarImage, visibility: options.visibility ?? "private", }); - return { agentId, name, title, roleDescription, avatarSeed }; + return { + agentId, + name, + title, + roleDescription, + avatarSeed, + avatarImage: options.avatarImage ?? null, + }; } async function profileById(actor: AgentActor, agentId: string) { @@ -506,6 +515,45 @@ describe("agent profile store integration", () => { expect(await store.get(admin, source.agentId)).toBeNull(); }); + test("stores avatar overrides for owners and lets an admin brand a package Bot", async () => { + const owner = await createUser(); + const other = await createUser(); + const admin = await createUser("admin"); + const userAgent = await createProfileFixture({ owner }); + const deploymentPackage = await createPackage(); + const packageAgent = await createProfileFixture({ + owner: null, + packageId: deploymentPackage.id, + visibility: "public", + }); + + const updated = await store.setAvatar( + owner, + userAgent.agentId, + "data:image/png;base64,owner", + ); + expect(updated.hasCustomAvatar).toBe(true); + expect(await store.avatar(owner, userAgent.agentId)).toBe( + "data:image/png;base64,owner", + ); + await expect( + store.setAvatar(other, userAgent.agentId, "data:image/png;base64,other"), + ).rejects.toThrow("was not found"); + + const branded = await store.setAvatar( + admin, + packageAgent.agentId, + "data:image/png;base64,admin", + ); + expect(branded.hasCustomAvatar).toBe(true); + expect(await store.avatar(admin, packageAgent.agentId)).toBe( + "data:image/png;base64,admin", + ); + await expect( + store.setAvatar(owner, packageAgent.agentId, null), + ).rejects.toThrow("cannot be managed"); + }); + test("duplicates a profile as a caller-owned private agent with copied presentation fields", async () => { const owner = await createUser(); const source = await createProfileFixture({ @@ -515,6 +563,7 @@ describe("agent profile store integration", () => { title: "Source Title", roleDescription: "Source role.", avatarSeed: "source-avatar", + avatarImage: "data:image/png;base64,source", }); await store.setHidden(owner, source.agentId, true); @@ -528,12 +577,14 @@ describe("agent profile store integration", () => { title: source.title, roleDescription: source.roleDescription, avatarSeed: source.avatarSeed, + hasCustomAvatar: true, visibility: "private", ownerUserId: owner.id, systemOwned: false, hidden: false, deletedAt: null, }); + expect(await store.avatar(owner, duplicate.id)).toBe(source.avatarImage); expect(duplicate.id).not.toBe(source.agentId); createdAgentIds.push(duplicate.id); const duplicatePreferences = await database diff --git a/server/tests/agent-routes.test.ts b/server/tests/agent-routes.test.ts index 6eaab5fb0..6198dd954 100644 --- a/server/tests/agent-routes.test.ts +++ b/server/tests/agent-routes.test.ts @@ -31,6 +31,9 @@ const validInput: CreateAgentInput = { visibility: "private", }; +const ONE_PIXEL_PNG = + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="; + function profile(overrides: Partial<AgentProfile> = {}): AgentProfile { return { id: "agent-1", @@ -38,6 +41,8 @@ function profile(overrides: Partial<AgentProfile> = {}): AgentProfile { title: validInput.title, roleDescription: validInput.roleDescription, avatarSeed: "expense-manager", + hasCustomAvatar: false, + avatarUpdatedAt: new Date(0), visibility: validInput.visibility, ownerUserId: actor.id, systemOwned: false, @@ -62,6 +67,10 @@ function fakeStore( calls.push(["get", receivedActor, id]); return profile({ id }); }, + async avatar(receivedActor, id) { + calls.push(["avatar", receivedActor, id]); + return null; + }, async getWithin(_executor, receivedActor, id) { calls.push(["getWithin", receivedActor, id]); return profile({ id }); @@ -74,6 +83,14 @@ function fakeStore( calls.push(["update", receivedActor, id, input]); return profile({ id, ...input }); }, + async setAvatar(receivedActor, id, image) { + calls.push(["setAvatar", receivedActor, id, image]); + return profile({ + id, + hasCustomAvatar: image !== null, + avatarUpdatedAt: new Date(1), + }); + }, async duplicate(receivedActor, id) { calls.push(["duplicate", receivedActor, id]); return profile({ id: `${id}-copy`, visibility: "private" }); @@ -313,6 +330,60 @@ describe("agent lifecycle routes", () => { ]); }); + test("validates and stores a coworker avatar independently of profile edits", async () => { + const store = fakeStore(); + const app = appFor(store); + + const saved = await app.request("http://openbot.test/agent-1/avatar", { + method: "PUT", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ image: ONE_PIXEL_PNG }), + }); + expect(saved.status).toBe(200); + expect(store.calls).toEqual([ + ["setAvatar", actor, "agent-1", ONE_PIXEL_PNG], + ]); + expect(await json(saved)).toEqual({ + agent: expect.objectContaining({ + id: "agent-1", + avatarUrl: expect.stringContaining( + "/api/agents/agent-1/avatar/image?v=", + ), + }), + }); + + store.calls.length = 0; + const refused = await app.request("http://openbot.test/agent-1/avatar", { + method: "PUT", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + image: "data:image/svg+xml;base64,PHN2Zz48L3N2Zz4=", + }), + }); + expect(refused.status).toBe(400); + expect(store.calls).toEqual([]); + }); + + test("serves avatar bytes only through an accessible coworker", async () => { + const store = fakeStore({ + async avatar(receivedActor, id) { + store.calls.push(["avatar", receivedActor, id]); + return ONE_PIXEL_PNG; + }, + }); + + const response = await appFor(store).request( + "http://openbot.test/agent-1/avatar/image?v=current", + ); + + expect(response.status).toBe(200); + expect(response.headers.get("content-type")).toBe("image/png"); + expect(Buffer.from(await response.arrayBuffer())).toEqual( + Buffer.from(ONE_PIXEL_PNG.split(",")[1], "base64"), + ); + expect(store.calls).toEqual([["avatar", actor, "agent-1"]]); + }); + test("projects exact DTO fields and computes permissions for the authenticated actor", async () => { const store = fakeStore({ async list() { @@ -339,10 +410,12 @@ describe("agent lifecycle routes", () => { title: validInput.title, roleDescription: validInput.roleDescription, avatarSeed: "expense-manager", + avatarUrl: null, visibility: "private", hidden: false, systemOwned: false, canManage: true, + canCustomizeAvatar: true, mine: true, builtIn: false, }, @@ -352,10 +425,12 @@ describe("agent lifecycle routes", () => { title: validInput.title, roleDescription: validInput.roleDescription, avatarSeed: "expense-manager", + avatarUrl: null, visibility: "private", hidden: false, systemOwned: false, canManage: false, + canCustomizeAvatar: false, mine: false, builtIn: false, }, @@ -365,10 +440,12 @@ describe("agent lifecycle routes", () => { title: validInput.title, roleDescription: validInput.roleDescription, avatarSeed: "expense-manager", + avatarUrl: null, visibility: "public", hidden: false, systemOwned: true, canManage: false, + canCustomizeAvatar: false, mine: false, builtIn: false, }, diff --git a/server/tests/avatar.test.ts b/server/tests/avatar.test.ts new file mode 100644 index 000000000..5b36c361c --- /dev/null +++ b/server/tests/avatar.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, test } from "bun:test"; +import { + avatarResponse, + avatarUrl, + MAX_AVATAR_BYTES, + parseAvatarImage, +} from "../src/avatar"; + +const ONE_PIXEL_PNG = + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="; + +describe("avatar image validation", () => { + test("accepts a bounded image and the explicit reset", () => { + expect(parseAvatarImage(ONE_PIXEL_PNG)).toEqual({ + ok: true, + value: ONE_PIXEL_PNG, + }); + expect(parseAvatarImage(null)).toEqual({ ok: true, value: null }); + }); + + test("uses a short versioned URL and serves the original bytes safely", async () => { + const url = avatarUrl("/api/me/avatar/image", ONE_PIXEL_PNG); + expect(url).toMatch(/^\/api\/me\/avatar\/image\?v=[a-z0-9]+$/); + expect(url).not.toContain("base64"); + + const response = avatarResponse(ONE_PIXEL_PNG); + expect(response.headers.get("content-type")).toBe("image/png"); + expect(response.headers.get("x-content-type-options")).toBe("nosniff"); + expect(response.headers.get("cache-control")).toContain("immutable"); + expect(Buffer.from(await response.arrayBuffer())).toEqual( + Buffer.from(ONE_PIXEL_PNG.split(",")[1], "base64"), + ); + }); + + test("does not trust the data URL MIME label", () => { + const disguised = ONE_PIXEL_PNG.replace("image/png", "image/jpeg"); + + expect(parseAvatarImage(disguised)).toEqual({ + ok: false, + error: "Avatar image does not match its PNG, JPEG, or WebP format.", + }); + }); + + test("rejects oversized payloads before storing them", () => { + const encoded = Buffer.alloc(MAX_AVATAR_BYTES + 1).toString("base64"); + + expect(parseAvatarImage(`data:image/png;base64,${encoded}`)).toEqual({ + ok: false, + error: "Avatar image must be 2 MB or smaller.", + }); + }); + + test("rejects compressed images with unsafe bitmap dimensions", () => { + const header = Buffer.alloc(24); + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]).copy(header); + header.writeUInt32BE(4097, 16); + header.writeUInt32BE(1, 20); + + expect( + parseAvatarImage(`data:image/png;base64,${header.toString("base64")}`), + ).toEqual({ + ok: false, + error: "Avatar image dimensions must be at most 4096 by 4096 pixels.", + }); + }); +}); diff --git a/server/tests/channel-activity.integration.test.ts b/server/tests/channel-activity.integration.test.ts index f8c857be7..2685f3f0b 100644 --- a/server/tests/channel-activity.integration.test.ts +++ b/server/tests/channel-activity.integration.test.ts @@ -227,6 +227,8 @@ describe("channel activity", () => { lastMessageAgentId: agentId, lastMessageAt: at, createdAt: expect.any(Date), + hasCustomAvatars: [false], + avatarUpdatedAts: [expect.any(Date)], pinned: false, lastReadAt: null, }, diff --git a/server/tests/channel-routes.test.ts b/server/tests/channel-routes.test.ts index 031b063c9..7ec8cedf0 100644 --- a/server/tests/channel-routes.test.ts +++ b/server/tests/channel-routes.test.ts @@ -715,6 +715,7 @@ async function createPersistentAgent(options: { id?: string; name: string; owner: AgentActor; + avatarImage?: string; visibility?: "public" | "private"; }) { const agentId = options.id ?? persistentId("agent"); @@ -731,6 +732,7 @@ async function createPersistentAgent(options: { title: `${options.name} title`, roleDescription: `${options.name} role description`, avatarSeed: agentId, + avatarImage: options.avatarImage, visibility: options.visibility ?? "private", }); return agentId; @@ -799,6 +801,49 @@ async function channelTableSnapshot() { } describe("channel store integration", () => { + test("carries uploaded coworker avatars into the roster summary", async () => { + const actor = await createPersistentUser(); + const avatarImage = "data:image/png;base64,roster"; + const agentId = await createPersistentAgent({ + avatarImage, + name: "Branded agent", + owner: actor, + }); + const created = await persistentStore.create(actor, [agentId]); + createdChannelIds.push(created.id); + + const page = await persistentStore.list(actor); + expect(page.channels.find((channel) => channel.id === created.id)).toEqual( + expect.objectContaining({ + agentIds: [agentId], + hasCustomAvatars: [true], + }), + ); + + const persistentRequireUser: MiddlewareHandler<{ + Variables: AppVariables; + }> = async (context, next) => { + context.set("actor", actor); + await next(); + }; + const response = await appFor( + persistentStore, + persistentRequireUser, + ).request("http://openbot.test/"); + const body = (await response.json()) as { + channels: { id: string; avatarUrls: (string | null)[] }[]; + }; + expect( + body.channels.find((channel) => channel.id === created.id)?.avatarUrls, + ).toEqual([ + expect.stringMatching( + new RegExp( + `^/api/agents/${agentId.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}/avatar/image\\?v=[a-z0-9]+$`, + ), + ), + ]); + }); + test("reads the creator's persisted channel exactly", async () => { const actor = await createPersistentUser(); const agentId = await createPersistentAgent({ diff --git a/server/tests/computer-gateway.test.ts b/server/tests/computer-gateway.test.ts index e2b4c2551..5d6df28e6 100644 --- a/server/tests/computer-gateway.test.ts +++ b/server/tests/computer-gateway.test.ts @@ -3,7 +3,9 @@ import type { AuditEventInput, AuditStore } from "../src/audit"; import { StaleSnapshotError } from "../src/computer/client"; import { ActionRefusedError, + controlLeaseBelongsTo, createComputerGateway, + mintControlLease, WorkspaceRefusedError, } from "../src/computer/gateway"; import type { ActionPolicy } from "../src/computer/policy"; @@ -37,6 +39,7 @@ const SNAPSHOT: SnapshotResult = { { ref: "e9", role: "button", name: "Submit order" }, ], }; +const CONTROL_LEASE = mintControlLease("dev-local-user"); /** A computer that records which HTTP actions reached it. */ function fakeComputer(options?: { @@ -156,7 +159,18 @@ function fakeComputer(options?: { return Response.json({ mode: "human", reason: "Sign in" }); case "/control/take": calls.push("takeControl"); - return Response.json({ mode: "human" }); + return Response.json({ + holder: "human", + since: "now", + requested: false, + }); + case "/control/renew": + calls.push("renewControl"); + return Response.json({ + holder: "human", + since: "now", + requested: false, + }); case "/control/release": calls.push("releaseControl"); return Response.json({ mode: "bot" }); @@ -765,15 +779,21 @@ describe("the computer gateway", () => { await gateway.listFiles("bot-1", ACTOR, { path: "notes" }); await gateway.control("bot-1"); await gateway.requestHelp("bot-1", ACTOR, "Sign in"); - await gateway.takeControl("bot-1", ACTOR); - await gateway.releaseControl("bot-1", ACTOR); + const takeover = await gateway.takeControl("bot-1", ACTOR); + await gateway.renewControl("bot-1", ACTOR, takeover.lease); + await gateway.releaseControl("bot-1", ACTOR, takeover.lease); await gateway.requestSecret("bot-1", ACTOR, { label: "Password", ref: "e1", snapshotId: 7, }); await gateway.supplySecret("bot-1", ACTOR, "secret"); - await gateway.humanInput("bot-1", { kind: "click", x: 10, y: 20 }); + await gateway.humanInput( + "bot-1", + ACTOR, + { kind: "click", x: 10, y: 20 }, + takeover.lease, + ); const paths = requests.map(({ url }) => new URL(url).pathname); expect(paths).toEqual([ @@ -784,11 +804,30 @@ describe("the computer gateway", () => { "/control", "/control/request", "/control/take", + "/control/renew", "/control/release", "/control/secret", "/human/secret", "/human/click", ]); + + expect(takeover.lease).toMatch(/^[A-Za-z0-9_-]{86}$/); + expect(controlLeaseBelongsTo(takeover.lease, ACTOR.id)).toBe(true); + expect(controlLeaseBelongsTo(takeover.lease, "another-user")).toBe(false); + for (const path of [ + "/control/take", + "/control/renew", + "/control/release", + "/human/click", + ]) { + const request = requests.find( + (item) => new URL(item.url).pathname === path, + ); + const body = JSON.parse(String(request?.init?.body ?? "null")) as { + lease?: string; + } | null; + expect(body?.lease).toBe(takeover.lease); + } }); /* * Through `gatewayWith`, deliberately, rather than by handing `evaluateActionPolicy` a context @@ -997,7 +1036,12 @@ describe("human input names a gesture, not a path", () => { const before = requests.length; await expect( - gateway.humanInput("bot-1", { kind, x: 1, y: 1 } as never), + gateway.humanInput( + "bot-1", + ACTOR, + { kind, x: 1, y: 1 } as never, + CONTROL_LEASE, + ), ).rejects.toThrow(); // Nothing left this process. Asserting only that it threw would pass just as well when the // request went out and the far side answered 404, which is the failure being fixed. @@ -1009,7 +1053,12 @@ describe("human input names a gesture, not a path", () => { async (kind) => { const { gateway, requests } = await gatewayWith(PERMISSIVE); - await gateway.humanInput("bot-1", { kind, x: 1, y: 1 } as never); + await gateway.humanInput( + "bot-1", + ACTOR, + { kind, x: 1, y: 1 } as never, + CONTROL_LEASE, + ); expect( requests.some( diff --git a/server/tests/computer-routes.test.ts b/server/tests/computer-routes.test.ts index 36f82b4e6..cd50add11 100644 --- a/server/tests/computer-routes.test.ts +++ b/server/tests/computer-routes.test.ts @@ -1,7 +1,10 @@ import { describe, expect, test } from "bun:test"; import type { MiddlewareHandler } from "hono"; import type { AppVariables, AuthenticatedActor } from "../src/auth/guards"; -import type { ComputerGateway } from "../src/computer/gateway"; +import { + type ComputerGateway, + mintControlLease, +} from "../src/computer/gateway"; import type { PolicyStore } from "../src/computer/policy-store"; import { createComputerRoutes } from "../src/computer/routes"; @@ -149,7 +152,11 @@ describe("human input", () => { function recordingGateway() { const calls: Array<{ botId: string; input: Record<string, unknown> }> = []; const gateway = { - humanInput: async (botId: string, input: Record<string, unknown>) => { + humanInput: async ( + botId: string, + _actor: unknown, + input: Record<string, unknown>, + ) => { calls.push({ botId, input }); return { ok: true }; }, @@ -172,7 +179,10 @@ describe("human input", () => { { method: "POST", headers: { "content-type": "application/json" }, - body: JSON.stringify(body), + body: JSON.stringify({ + ...(body && typeof body === "object" ? body : {}), + lease: mintControlLease(member.id), + }), }, ); return { response, calls }; diff --git a/server/tests/computer-socket-proxy.test.ts b/server/tests/computer-socket-proxy.test.ts new file mode 100644 index 000000000..94286feb1 --- /dev/null +++ b/server/tests/computer-socket-proxy.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, test } from "bun:test"; +import { + computerSocketIsolationRefusal, + computerSocketUrl, + parseComputerSocketPath, + parseComputerSocketLease, +} from "../src/computer/socket-proxy"; + +const LEASE = "a".repeat(86); + +describe("the computer WebSocket proxy", () => { + test("accepts only the page stream and full desktop routes", () => { + expect(parseComputerSocketPath("/api/computers/sales-bot/stream")).toEqual({ + botId: "sales-bot", + kind: "stream", + }); + expect(parseComputerSocketPath("/api/computers/a%20b/desktop")).toEqual({ + botId: "a b", + kind: "desktop", + }); + for (const path of [ + "/api/computers/a/shell", + "/api/computers/a/desktop/more", + "/api/computers/%E0%A4%A/desktop", + ]) { + expect(parseComputerSocketPath(path)).toBeNull(); + } + }); + + test("offers a process-wide desktop only when the provider isolates the whole computer", () => { + expect( + computerSocketIsolationRefusal("desktop", "per-bot"), + ).toBeUndefined(); + expect(computerSocketIsolationRefusal("desktop", "shared")).toBe( + "A full desktop requires one isolated computer per Bot.", + ); + expect(computerSocketIsolationRefusal("desktop", undefined)).toBe( + "A full desktop requires one isolated computer per Bot.", + ); + expect(computerSocketIsolationRefusal("stream", "shared")).toBeUndefined(); + }); + + test("keeps the token on the internal URL and defaults desktop access to view-only", () => { + const url = new URL( + computerSocketUrl({ + baseUrl: "https://computer.internal/", + botId: "sales bot", + kind: "desktop", + token: "not-for-the-browser", + mode: "anything-else", + }), + ); + expect(url.protocol).toBe("wss:"); + expect(url.pathname).toBe("/desktop"); + expect(url.searchParams.get("bot")).toBe("sales bot"); + expect(url.searchParams.get("token")).toBe("not-for-the-browser"); + expect(url.searchParams.get("mode")).toBe("view"); + }); + + test("names control only when explicitly requested and leaves it off page streams", () => { + const desktop = computerSocketUrl({ + baseUrl: "http://127.0.0.1:4100", + botId: "one", + kind: "desktop", + token: "secret", + mode: "control", + lease: LEASE, + }); + expect(new URL(desktop).searchParams.get("mode")).toBe("control"); + expect(new URL(desktop).searchParams.get("lease")).toBe(LEASE); + + const stream = computerSocketUrl({ + baseUrl: "http://127.0.0.1:4100", + botId: "one", + kind: "stream", + token: "secret", + mode: "control", + lease: LEASE, + }); + expect(new URL(stream).searchParams.has("mode")).toBe(false); + expect(new URL(stream).searchParams.get("lease")).toBe(LEASE); + }); + + test("accepts only a well-formed control capability from WebSocket protocols", () => { + expect(parseComputerSocketLease(`binary, openbot-lease.${LEASE}`)).toBe( + LEASE, + ); + expect( + parseComputerSocketLease("binary, openbot-lease.short"), + ).toBeUndefined(); + expect(parseComputerSocketLease("binary")).toBeUndefined(); + }); +}); diff --git a/server/tests/guards.test.ts b/server/tests/guards.test.ts index 335c1de67..a0192d629 100644 --- a/server/tests/guards.test.ts +++ b/server/tests/guards.test.ts @@ -69,6 +69,7 @@ describe("server authorization", () => { email: "member@openbot.test", name: "OpenBot Member", image: "https://example.test/member.png", + hasCustomAvatar: false, role: "user", // No store was passed, so this deployment tracks no onboarding and the app gates nobody. onboarding: null, diff --git a/server/tests/human-input-end-to-end.test.ts b/server/tests/human-input-end-to-end.test.ts index 3e3bdc00d..c41b8ee4e 100644 --- a/server/tests/human-input-end-to-end.test.ts +++ b/server/tests/human-input-end-to-end.test.ts @@ -2,7 +2,10 @@ import { afterEach, describe, expect, test } from "bun:test"; import type { MiddlewareHandler } from "hono"; import type { AuditEventInput, AuditStore } from "../src/audit"; import type { AppVariables, AuthenticatedActor } from "../src/auth/guards"; -import { createComputerGateway } from "../src/computer/gateway"; +import { + createComputerGateway, + mintControlLease, +} from "../src/computer/gateway"; import type { PolicyStore } from "../src/computer/policy-store"; import type { ComputerProvider } from "../src/computer/provider"; import { createComputerRoutes } from "../src/computer/routes"; @@ -29,6 +32,7 @@ afterEach(() => { }); const TOKEN = "computer-token-for-this-test"; +const CONTROL_LEASE = mintControlLease("user-1"); /** What the far side actually received, in the order it arrived. */ type Received = { path: string; token: string | null; body: unknown }; @@ -103,7 +107,10 @@ async function drive(kind: string, body: unknown) { { method: "POST", headers: { "content-type": "application/json" }, - body: JSON.stringify(body), + body: JSON.stringify({ + ...(body && typeof body === "object" ? body : {}), + lease: CONTROL_LEASE, + }), }, ); return { response, received, rows }; @@ -117,7 +124,47 @@ describe("a person's input, end to end", () => { expect(received).toHaveLength(1); expect(received[0]?.path).toBe("/human/click"); expect(received[0]?.token).toBe(TOKEN); - expect(received[0]?.body).toEqual({ x: 10, y: 20 }); + expect(received[0]?.body).toEqual({ + x: 10, + y: 20, + lease: CONTROL_LEASE, + }); + }); + + test("refuses input without the private lease before it reaches the computer", async () => { + const { received, baseUrl } = serveComputer(); + const { app } = appFor(baseUrl); + const response = await app.request( + "http://openbot.test/bot-1/human/click", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ x: 10, y: 20 }), + }, + ); + + expect(response.status).toBe(400); + expect(received).toEqual([]); + }); + + test("refuses a lease minted for another signed-in actor", async () => { + const { received, baseUrl } = serveComputer(); + const { app } = appFor(baseUrl); + const response = await app.request( + "http://openbot.test/bot-1/human/click", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + x: 10, + y: 20, + lease: mintControlLease("another-user"), + }), + }, + ); + + expect(response.status).toBe(403); + expect(received).toEqual([]); }); test.each([ diff --git a/server/tests/people-routes.test.ts b/server/tests/people-routes.test.ts index 92bdca3a3..a5b49a75a 100644 --- a/server/tests/people-routes.test.ts +++ b/server/tests/people-routes.test.ts @@ -17,6 +17,9 @@ const ADMIN = { image: null, }; +const ONE_PIXEL_PNG = + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="; + function person(overrides: Partial<Person> = {}): Person { return { id: "u1", @@ -40,6 +43,7 @@ function appWith( calls: string[]; } { const calls: string[] = []; + let avatar: string | null = null; const store: PeopleStore = { // One page, and no next one: what a small deployment answers. The paging itself is covered // against a real database in people-paging.integration.test.ts. @@ -55,6 +59,11 @@ function appWith( calls.push(`restore:${userId}`); }, isRevoked: async () => false, + avatar: async () => avatar, + setAvatar: async (userId, image) => { + avatar = image; + calls.push(`setAvatar:${userId}:${image === null ? "null" : "image"}`); + }, }; const app = createApp( @@ -90,6 +99,52 @@ const json = (body: unknown): RequestInit => ({ }); describe("people routes", () => { + test("stores a person's custom avatar and reports it from /api/me", async () => { + const { request, calls } = appWith([]); + + const saved = await request("/api/me/avatar", { + method: "PUT", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ image: ONE_PIXEL_PNG }), + }); + expect(saved.status).toBe(200); + expect(calls).toEqual([`setAvatar:${ADMIN.id}:image`]); + expect(await saved.json()).toEqual({ + avatar: { + image: expect.stringContaining("/api/me/avatar/image?v="), + hasCustomAvatar: true, + }, + }); + + const current = await request("/api/me"); + expect(await current.json()).toEqual({ + user: expect.objectContaining({ + image: expect.stringContaining("/api/me/avatar/image?v="), + hasCustomAvatar: true, + }), + }); + + const image = await request("/api/me/avatar/image"); + expect(image.status).toBe(200); + expect(image.headers.get("content-type")).toBe("image/png"); + expect(Buffer.from(await image.arrayBuffer())).toEqual( + Buffer.from(ONE_PIXEL_PNG.split(",")[1], "base64"), + ); + }); + + test("refuses an unsafe avatar before writing the person's row", async () => { + const { request, calls } = appWith([]); + + const response = await request("/api/me/avatar", { + method: "PUT", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ image: "https://attacker.example/avatar.svg" }), + }); + + expect(response.status).toBe(400); + expect(calls).toEqual([]); + }); + test("lists everybody for an administrator", async () => { const { request } = appWith([person()]); diff --git a/server/tests/schema.test.ts b/server/tests/schema.test.ts index 1a3104e94..ecdce98cd 100644 --- a/server/tests/schema.test.ts +++ b/server/tests/schema.test.ts @@ -200,6 +200,12 @@ describe("OpenBot database schema", () => { hasDefault: false, primary: false, }, + { + name: "avatar_image", + notNull: false, + hasDefault: false, + primary: false, + }, { name: "visibility", notNull: true, diff --git a/tests/release-contract.test.ts b/tests/release-contract.test.ts new file mode 100644 index 000000000..06faba41e --- /dev/null +++ b/tests/release-contract.test.ts @@ -0,0 +1,31 @@ +import { expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +const root = join(import.meta.dir, ".."); +const read = (path: string) => readFileSync(join(root, path), "utf8"); + +test("a release moves the Helm default to the image built from that release", () => { + const prepare = read(".github/workflows/release.yml"); + const publish = read(".github/workflows/publish-release.yml"); + + expect(prepare).toContain( + 'sed -i -E "s/^appVersion: .*/appVersion: \\"$version\\"/" charts/openbot/Chart.yaml', + ); + expect(publish).toMatch( + /grep -qx "appVersion: \\"\$\{VERSION#v\}\\"" charts\/openbot\/Chart\.yaml/, + ); +}); + +test("computer pods stay bootable on the last release while the next image is unpublished", () => { + const statefulSet = read( + "charts/openbot/templates/computer/statefulset.yaml", + ); + const sandboxTemplate = read("charts/openbot/templates/_helpers.tpl"); + + for (const template of [statefulSet, sandboxTemplate]) { + expect(template).toContain("/app/agent-computer/entrypoint.sh"); + expect(template).toContain("/app/agent-computer/src/index.ts"); + expect(template).toContain("/bin/bash"); + } +}); diff --git a/tests/workspace.test.ts b/tests/workspace.test.ts index aacea898d..a18db1b11 100644 --- a/tests/workspace.test.ts +++ b/tests/workspace.test.ts @@ -13,16 +13,23 @@ function packageManifest(path: string) { } describe("OpenBot workspace", () => { - test("defines the app, server, and worker packages", () => { + test("defines every package installed by the root workspace", () => { const rootManifest = JSON.parse( readFileSync(join(repositoryRoot, "package.json"), "utf8"), ) as { workspaces: string[] }; - expect(rootManifest.workspaces).toEqual(["app", "server", "worker"]); + expect(rootManifest.workspaces).toEqual([ + "app", + "agent-codex", + "server", + "worker", + ]); for (const packageName of rootManifest.workspaces) { expect(existsSync(join(repositoryRoot, packageName))).toBe(true); - expect(packageManifest(packageName).name).toBe(packageName); + expect(packageManifest(packageName).name).toBe( + packageName === "agent-codex" ? "@openbot/agent-codex" : packageName, + ); } }); });