diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4c85ff..56ed81b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,9 +26,20 @@ jobs: - run: pnpm test e2e: - name: Actor dev-loop e2e (apify-cli + Docker) + name: e2e ${{ matrix.file }} (apify-cli + Docker) runs-on: ubuntu-latest timeout-minutes: 30 + strategy: + fail-fast: false + # One job per e2e file. Each file starts its own runtime container on the fixed host ports + # (3333/3000), so files cannot share a daemon; separate runners make them parallel instead. + matrix: + file: + - actor-dev-loop + - debug-mode + - dev-folder-bind-mount + - browser-view-ts + - browser-view-py steps: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v4 @@ -40,10 +51,13 @@ jobs: # The suite manages Docker itself against the runner's daemon: it pre-pulls the # Actor base images, builds the runtime image, starts the runtime container with # the host Docker socket, and drives it with stock apify-cli via npx. - - run: pnpm run test:e2e + - run: pnpm exec vitest run test/e2e/${{ matrix.file }}.test.ts # The e2e's runtime container is normally removed by the suite's afterAll; on # failure it is left running, so its server-side view of any failed request # (log-stream lifecycle included) is captured here for diagnosis. - name: Dump runtime container logs on failure if: failure() - run: docker logs actor-runtime-e2e --tail 300 || true + run: | + for c in $(docker ps -aq --filter name=actor-runtime-e2e); do + docker logs --tail 300 "$c" || true + done diff --git a/.gitignore b/.gitignore index b896597..a0f8cff 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,8 @@ dist/ *.tsbuildinfo sample_actor_ts/node_modules/ sample_actor_ts/dist/ +sample_actor_playwright/node_modules/ +sample_actor_playwright/dist/ sample_actor_py/__pycache__/ sample_actor_py/.venv/ .npm/ diff --git a/CLAUDE.MD b/CLAUDE.MD index cfa6cb0..a92d586 100644 --- a/CLAUDE.MD +++ b/CLAUDE.MD @@ -44,6 +44,12 @@ Local Actor runtime is an Actor development tool for developing, running, and de when you expect a slow attach. Clear the toggle with `apify api POST /actor-runtime/debug/ --body '{"enabled": false}'` to go back to running normally. +- To watch the browser of a Playwright/Puppeteer Actor while it runs, turn browser view on for it once: + `apify api POST /actor-runtime/browser-view/ --body '{"enabled": true}'` (`"interactive": true` also + sends mouse/keyboard input). Every subsequent run prints a viewer URL in its log, + `http://localhost:3000/runs//browser` - a live view of the display the Actor's browser draws on. The + browser must run headful to show anything (Apify's templates default to headless, which shows as a black + display); see `sample_actor_playwright` and `sample_actor_playwright_py`. Clear with `--body '{"enabled": false}'`. - To test how an Actor handles platform migrations: while a run is `RUNNING`, call `apify api POST /actor-runtime/migrate/`, or press the Migrate button on the run's console detail page. The run gets the platform migration experience: a `migrating` event, its container diff --git a/Dockerfile b/Dockerfile index c0f8c3a..9ca0c01 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,6 +29,22 @@ import debugpy._version as v; \ print(v.get_versions()['version'])" > /payload/debugpy-version.txt RUN tar -cf /payload/debugpy-payload.tar -C /payload/root . +# --- Browser-view sidecar: an Alpine rootfs with x11vnc, tarred so the runtime can `docker import` it at +# run time without a registry. Not pinned to $BUILDPLATFORM: it runs on the Actor containers' daemon, so it +# must be the target architecture's. +FROM alpine:3.21 AS browser-viewer-rootfs +RUN apk add --no-cache x11vnc +RUN mkdir -p /tmp/.X11-unix && chmod 1777 /tmp/.X11-unix +COPY docker/browser-viewer.sh /apify-browser-viewer.sh +RUN chmod 755 /apify-browser-viewer.sh + +# Tars the stage above and records its content hash, which the runtime uses as the imported image's tag. +FROM --platform=$BUILDPLATFORM alpine:3.21 AS browser-viewer-payload +COPY --from=browser-viewer-rootfs / /rootfs +RUN mkdir -p /payload \ + && tar -cf /payload/rootfs.tar -C /rootfs . \ + && sha256sum /payload/rootfs.tar | cut -c1-16 > /payload/version.txt + # Also architecture-independent: this stage only runs `tsc`, and the `dist/` it hands to the final # stage is plain JavaScript. The final stage does its own `pnpm install --prod`, so the target # architecture's native bindings still come from a native (emulated) install there. @@ -64,6 +80,10 @@ COPY --from=builder /usr/src/app/dist ./dist COPY --from=debugpy-payload /payload/debugpy-payload.tar /opt/apify-debug-payload/debugpy-payload.tar COPY --from=debugpy-payload /payload/debugpy-version.txt /opt/apify-debug-payload/debugpy-version.txt +# Matches config.ts's browserViewerPayloadDir() default. +COPY --from=browser-viewer-payload /payload/rootfs.tar /opt/apify-browser-viewer/rootfs.tar +COPY --from=browser-viewer-payload /payload/version.txt /opt/apify-browser-viewer/version.txt + # The runtime talks to the host Docker socket via dockerode (no docker CLI needed in-image) and # persists all storages under /data - mount both when running the container. VOLUME ["/data"] diff --git a/README.md b/README.md index bc92c40..a9dbdcc 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,27 @@ three-field form (`enabled`/`language`/`port`) on the Actor's page in the consol `requirements/actor-driver.md`'s "Debug mode" section; endpoint/console details: `requirements/api.md`'s `/actor-runtime/*` section and `requirements/console.md`. +## Watching an Actor's browser + +Turn **browser view** on for an Actor once, and every run of it gets a live view of the display its browser draws +on, served by the console: + +```bash +apify api POST /actor-runtime/browser-view/ --body '{"enabled": true}' +apify call +``` + +The run log prints the viewer URL (`http://localhost:3000/runs//browser`); the run's console page links to +it, and the Actor's console page has the same toggle as a form. `"interactive": true` also sends your mouse and +keyboard to the display; `{"enabled": false}` turns the view off. + +The view only reads the display's pixels. The Actor's container, command, environment, network and ports are +those of an ordinary run, so neither the browser nor the sites it visits can tell whether anyone is watching. +Two things follow: the browser must run **headful** (Apify's templates default to headless, which shows as a +black display - the bundled `sample_actor_playwright` and `sample_actor_playwright_py` set `headless: false` / +`headless=False`), and the image must provide an X display, which the Apify Playwright and Puppeteer base images +do. Like Python debug mode, this needs the runtime to run from its own built image. + ## Publishing the image Images go to [`apify/actor-runtime`](https://hub.docker.com/r/apify/actor-runtime) on Docker Hub by @@ -140,7 +161,7 @@ added by hand. pnpm install pnpm run build # tsc pnpm test # unit + integration (no Docker needed) -pnpm run test:e2e # full CLI-driven dev loop against a built image (requires Docker) +pnpm run test:e2e # full CLI-driven dev loop against a built image (requires Docker; the browser-view case pulls the ~2 GB Playwright base image) pnpm run dev # run the server directly against ./data with tsx ``` diff --git a/docker/browser-viewer.sh b/docker/browser-viewer.sh new file mode 100755 index 0000000..7ebb2ac --- /dev/null +++ b/docker/browser-viewer.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# actor-runtime's browser-view sidecar: waits for the Actor's X socket in the shared /tmp/.X11-unix, then +# mirrors that display with x11vnc. The display number is taken from the socket name (`xvfb-run -a` picks +# one at run time). When the Actor container restarts (migration), x11vnc exits and the loop waits again. +# Env names must match `src/driver/docker-driver.ts`. + +SOCKET_DIR=/tmp/.X11-unix +PORT="${APIFY_BROWSER_VIEWER_PORT:-5900}" +if [ "$APIFY_BROWSER_VIEWER_INTERACTIVE" = "1" ]; then + INPUT_FLAG="" + MODE=interactive +else + INPUT_FLAG="-viewonly" + MODE=view-only +fi + +log() { + echo "[actor-runtime browser view] $*" +} + +log "waiting for an X display socket in $SOCKET_DIR (created by the Actor's own Xvfb when the Actor starts)" +while :; do + socket="" + for candidate in "$SOCKET_DIR"/X*; do + if [ -S "$candidate" ]; then + socket="$candidate" + break + fi + done + if [ -z "$socket" ]; then + sleep 0.5 + continue + fi + + display=":${socket##*/X}" + log "mirroring display $display ($MODE) on port $PORT" + # -noshm: MIT-SHM cannot cross container IPC namespaces (x11vnc would die on X_ShmAttach). + # -nosel/-nobell/-noxrecord/-nowf/-noscr: only read pixels; no clipboard, bell, or X request recording. + # shellcheck disable=SC2086 # INPUT_FLAG is intentionally word-split. + x11vnc -display "$display" -rfbport "$PORT" -noshm -nosel -nobell -shared -forever -nopw -noipv6 -q \ + -noxrecord -nowf -noscr $INPUT_FLAG + log "x11vnc exited - display gone; waiting for a display again" + sleep 1 +done diff --git a/eslint.config.js b/eslint.config.js index 65a578c..3500df2 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -5,7 +5,14 @@ import prettier from 'eslint-config-prettier'; export default tseslint.config( { - ignores: ['dist/**', 'node_modules/**', 'sample_actor_ts/**', 'sample_actor_py/**', 'data/**'], + ignores: [ + 'dist/**', + 'node_modules/**', + 'sample_actor_ts/**', + 'sample_actor_py/**', + 'sample_actor_playwright/**', + 'data/**', + ], }, js.configs.recommended, ...tseslint.configs.recommended, diff --git a/package.json b/package.json index f3136e0..cb3fbc6 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "dependencies": { "@crawlee/core": "4.0.0-beta.145", "@crawlee/fs-storage": "4.0.0-beta.145", + "@novnc/novnc": "1.7.0", "dockerode": "^4.0.5", "express": "^5.1.0", "json5": "^2.2.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 79862ae..8127ff0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -17,6 +17,9 @@ importers: '@crawlee/fs-storage': specifier: 4.0.0-beta.145 version: 4.0.0-beta.145 + '@novnc/novnc': + specifier: 1.7.0 + version: 1.7.0 dockerode: specifier: ^4.0.5 version: 4.0.12 @@ -569,6 +572,9 @@ packages: os: [linux] libc: [glibc] + '@novnc/novnc@1.7.0': + resolution: {integrity: sha512-ucEJOx4T2avIRCleodk7YobZj5O2Ga2AeLfQ69A/yjG9HHba2+PDgwSkN3FttrmG+70ZGx21sElNFouK13RzyA==} + '@protobufjs/aspromise@1.1.2': resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} @@ -2599,6 +2605,8 @@ snapshots: '@napi-rs/lzma-linux-x64-gnu@1.5.1': optional: true + '@novnc/novnc@1.7.0': {} + '@protobufjs/aspromise@1.1.2': {} '@protobufjs/base64@1.1.2': {} diff --git a/requirements/actor-driver.md b/requirements/actor-driver.md index f713d64..09a68fd 100644 --- a/requirements/actor-driver.md +++ b/requirements/actor-driver.md @@ -121,6 +121,24 @@ start`, ...) is refused by name, naming both the `CMD` fix and how to clear debu apply to the same run when both are configured for an Actor, e.g. edit -> recompile -> `apify call` -> breakpoint, with no rebuild in between. +# Browser view + +- Browser view is a persistent per-Actor toggle. While it is on, every run of the Actor offers a live view + of the display its browser draws on, reachable from the run's console page and from a URL printed in the + run log. +- Two modes: **view-only** (the default) shows the display and sends nothing to it; **interactive** also + delivers the viewer's mouse and keyboard input to the display. Nothing else ever crosses in either + direction (no clipboard). +- Watching is not observable from inside the browser or by the sites it visits: the run's container, + command, environment, network and ports are those of an ordinary run, and whether the view is on, off, or + being watched changes nothing about the browser. +- The runtime never changes the browser's headless mode. A headless browser shows an empty display; an + Actor that wants to be watched runs its browser headful (the bundled `sample_actor_playwright` and + `sample_actor_playwright_py` do). The Actor image must provide an X display; the Apify Playwright and + Puppeteer base images do. +- The view lives exactly as long as the run, survives a migration/reboot of the run, and is gone once the run + ends. It composes with debug mode and the dev-folder bind mount. + # Networking - On startup, the runtime ensures a Docker network `apify-local` exists and joins it under the fixed diff --git a/requirements/api.md b/requirements/api.md index b7a2267..3419ab7 100644 --- a/requirements/api.md +++ b/requirements/api.md @@ -182,6 +182,12 @@ - The console's own debug-mode form (`console.md`) does **not** go through this endpoint - same console-local, unauthenticated split as the dev-folder form - but both surfaces accept and reject exactly the same inputs with the same outcomes. +- **`POST /actor-runtime/browser-view/:actorId`** - sets or clears the Actor's browser-view toggle + (`actor-driver.md`'s "Browser view" section). Authenticated and owner-scoped like every `/v2` route; no + build-first precondition. + - **Body**: `{ "enabled": boolean, "interactive"?: boolean }`, `interactive` defaulting to `false`. A call + fully replaces the prior state; `{"enabled": false}` clears it. Any other shape is `400 invalid-request`. + - **Response**: `{ data: { localBrowserView: { interactive } | null } }` - the read-back; there is no `GET`. - **`GET /actor-runtime/events/:runId`** - a websocket upgrade, reachable at exactly this one path on the fixed API port (`system.md`). It carries the run's platform events: `systemInfo` once a second (`actor-driver.md`), a one-off `aborting`-plus-`persistState` pair under `?gracefully=` (below), and a diff --git a/requirements/console.md b/requirements/console.md index c13c2ca..412a036 100644 --- a/requirements/console.md +++ b/requirements/console.md @@ -6,10 +6,10 @@ - The console has no login of its own, so with multiple users it lists and shows every user's objects rather than scoping to one - the API's own endpoints stay strictly scoped to the calling token's user (`storage.md`'s "Users" section). -- The console is unauthenticated. Every route is a read except the console's only four writes: the - dev-folder form, the debug-mode form, the run detail view's Migrate button, and the Settings form - (all below). -- All four of those writes reject a submission that identifies itself as cross-site (via the +- The console is unauthenticated. Every route is a read except the console's only five writes: the + dev-folder form, the debug-mode form, the browser-view form, the run detail view's Migrate button, and + the Settings form (all below). +- All five of those writes reject a submission that identifies itself as cross-site (via the `Sec-Fetch-Site` header) with a plain `403`; a submission that does not is unaffected. - There are three types of objects: key-value store, dataset, request queue. - For each object type there must be exactly one widget for inspection. @@ -40,6 +40,8 @@ - A run whose debug plan resolved (`actor-driver.md`'s "Debug mode" section) gets one extra row on its detail view: `debug` - `, attach at 127.0.0.1:`. Absent entirely for a non-debug run. This field is local-only and never appears in the emulated `/v2` run object (`api.md`). +- A run with browser view (`actor-driver.md`) gets one extra row on its detail view: `browser view` - a + link to its viewer page (below). Absent for other runs; never in the emulated `/v2` run object. - Log views render ANSI colors from actor output as HTML, while the `/v2/logs/:id` API keeps serving logs raw (unconverted) for the CLI to render itself. - The console accepts the real Apify Console's URL shapes (as printed by stock apify-cli, e.g. `/actors/:actorId/runs/:runId`, `/storage/datasets/:id`) via redirects to its own pages. @@ -66,6 +68,17 @@ - A submission that fails validation redirects back to the same detail page with the classified error message shown inline, never silently applied. +## Browser-view form (Actor detail view) + +- The Actor detail view shows the browser-view toggle status and a form with the API body's two fields, + `enabled` and `interactive`, as checkboxes. For any input, the form and the API produce the same outcome. + +## Browser view page (`/runs/:runId/browser`) + +- Shows the run's live display, view-only or interactive per the run's toggle, and says which. It reconnects + on its own while the run's browser is still starting. +- For a run that has ended, or never had browser view, the page says so instead. + ## Migrate button (run detail view) - The run detail view shows the run's `migrationCount` and `rebootCount`, and a "Migration" section: diff --git a/requirements/storage.md b/requirements/storage.md index 9a5bbaf..434fa3f 100644 --- a/requirements/storage.md +++ b/requirements/storage.md @@ -57,8 +57,10 @@ (`actor-driver.md`). When present: `{ language: "auto" | "node" | "python", port?: number }` - `port` absent means "resolve the language's own default port at run start", never a stored literal (`actor-driver.md`). - - Neither `localDevFolder`, `localDebug`, nor any build's `imageWorkingDirectory` is ever exposed - on the public `/v2` API. + - `localBrowserView` - **optional**, `{ interactive: boolean }`; absent means browser view is off. + Same rules as `localDebug`: set only through its endpoint or console form, never bumping `modifiedAt`. + - Neither `localDevFolder`, `localDebug`, `localBrowserView`, nor any build's + `imageWorkingDirectory` is ever exposed on the public `/v2` API. - The system stores Actor runs in dedicated key-value store called `__RUNS__`: - `key` is the id of the Actor run `runId` - `value` is the metadata of the Actor @@ -70,6 +72,8 @@ number }`, both already resolved (never `"auto"`, never absent-meaning-default). Absent for every non-debug run, and for a debug run that was refused before a plan could be resolved. Never exposed on the emulated `/v2` run object. + - `localBrowserView` - **optional**, specific to this one run: `{ interactive, vncHost, vncPort }`, + the run's browser view once it is up. Absent otherwise. Never exposed on the emulated `/v2` run object. - The system stores Actor builds in dedicated key-value store called `__BUILDS__`: - `key` is the id of the Actor build (`buildId`) - `value` is the metadata of the Actor diff --git a/requirements/system.md b/requirements/system.md index 0c625f2..097267b 100644 --- a/requirements/system.md +++ b/requirements/system.md @@ -30,6 +30,7 @@ port published on the host, bound to `127.0.0.1` (`5678` Python / `9229` Node by default, per-Actor overridable) - the runtime's own two ports above are unaffected, and no port is published for an Actor that never turned debug mode on. +- Browser view (`actor-driver.md`) publishes no port on the host; the view is served on the console's port 3000. - Required `docker run` flags: mount the host Docker socket read-write (`-v /var/run/docker.sock:/var/run/docker.sock`) so the runtime can build and run Actor containers, and mount a persistent data directory (`-v :/data`, e.g. `-v "$(pwd)/data:/data"`) so diff --git a/requirements/test.md b/requirements/test.md index 327d19c..70bdc4d 100644 --- a/requirements/test.md +++ b/requirements/test.md @@ -5,6 +5,7 @@ # Continuous integration - CI (GitHub Actions) runs on every pull request and on pushes to the main branches: build, lint, format check, and all test layers, with the mandatory CLI-only e2e suite below executing against a real Docker daemon. A missing daemon fails the CI job - the e2e suite never silently skips. +- CI runs each e2e file as its own job, in parallel; locally the files run one after another (each starts a runtime container on the fixed ports). # Mandatory end-to-end tests @@ -13,11 +14,14 @@ may connect directly to the published debug port to emulate an IDE attaching a debugger, since no `apify` command can express that. Every other assertion in that test (the pause, the attach log line, the abort) still goes through `apify` commands only, same as every other e2e case. +- **A second narrow exception of the same kind**: the browser-view e2e test may open the console's viewer + page and its websocket directly, to emulate a developer's browser opening the view. Everything else in it + goes through `apify` commands. - For asserting the test results, the tests must inspect the return values of the Apify cli commands. - The e2e suite requires a reachable Docker daemon (it builds and runs real Actor containers) and detects its absence, failing in such case. - The sample Actors crawl a live site (`https://crawlee.dev/` by default), so the e2e suite also requires outbound network access from Actor containers. This is separate from the runtime's own offline capability (see the offline notes in `system.md` and `cli.md`). -- CI must pre-pull the sample Actors' base images (`apify/actor-node:24`, `apify/actor-python:3.13`, and `python:3.11-slim` for `sample_actor_crawler`) before running the e2e suite, so push/call assertion timing is not dominated by first-time image pulls. +- CI must pre-pull the sample Actors' base images (`apify/actor-node:24`, `apify/actor-python:3.13`, and `python:3.11-slim` for `sample_actor_crawler`) before running the e2e suite, so push/call assertion timing is not dominated by first-time image pulls. The browser-view e2e test pre-pulls the two Playwright samples' base images itself. ## Actor full dev loop @@ -27,3 +31,9 @@ Test case must verify full Actor development flow: - Push and build Actor in local actor runtime `apify push` - Run each sample Actor in the local actor runtime with `apify call --input '{"maxPages":N}'` for at least two different values of `N`, waiting for each run to finish - Assert via `apify datasets info ` that the default dataset's `itemCount` tracks `N` - the assertion is input-dependent, not just "some items exist" + +## Browser view + +- For each Playwright sample Actor (`sample_actor_playwright`, `sample_actor_playwright_py`): push, turn browser view on, start a run +- Assert the run log names the viewer URL, the view is reachable while the run is going, the run finishes `SUCCEEDED` with an input-dependent `itemCount`, and the view is gone once the run has ended +- With the toggle cleared, a plain `apify call` of the same Actor runs with no browser-view line in its log diff --git a/sample_actor_playwright/.actor/actor.json b/sample_actor_playwright/.actor/actor.json new file mode 100644 index 0000000..ad03ad6 --- /dev/null +++ b/sample_actor_playwright/.actor/actor.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://apify.com/schemas/v1/actor.ide.json", + "actorSpecification": 1, + "name": "my-playwright-actor", + "title": "Playwright sample actor for actor-runtime", + "description": "Crawls up to maxRequestsPerCrawl same-hostname pages from startUrls with a headful Chrome driven by Playwright, pushing one dataset item per page.", + "version": "0.0", + "buildTag": "latest", + "meta": { + "templateId": "ts-crawlee-playwright-chrome" + }, + "inputSchema": "./input_schema.json", + "dockerfile": "../Dockerfile" +} diff --git a/sample_actor_playwright/.actor/input_schema.json b/sample_actor_playwright/.actor/input_schema.json new file mode 100644 index 0000000..fe27fb2 --- /dev/null +++ b/sample_actor_playwright/.actor/input_schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://apify.com/schemas/v1/input.ide.json", + "title": "Playwright sample actor input", + "type": "object", + "schemaVersion": 1, + "properties": { + "startUrls": { + "title": "Start URLs", + "type": "array", + "description": "URLs to start with.", + "editor": "requestListSources", + "prefill": [{ "url": "https://crawlee.dev/" }] + }, + "maxRequestsPerCrawl": { + "title": "Max requests per crawl", + "type": "integer", + "description": "Maximum number of pages the crawler opens, following same-hostname links from the start URLs. One dataset item is pushed per page.", + "default": 3, + "minimum": 1 + } + } +} diff --git a/sample_actor_playwright/.dockerignore b/sample_actor_playwright/.dockerignore new file mode 100644 index 0000000..3bbdbb7 --- /dev/null +++ b/sample_actor_playwright/.dockerignore @@ -0,0 +1,18 @@ +# configurations +.idea +.vscode +.zed + +# crawlee and apify storage folders +apify_storage +crawlee_storage +storage + +# installed files +node_modules + +# git folder +.git + +# dist folder +dist diff --git a/sample_actor_playwright/.gitignore b/sample_actor_playwright/.gitignore new file mode 100644 index 0000000..b0007f8 --- /dev/null +++ b/sample_actor_playwright/.gitignore @@ -0,0 +1,10 @@ +# This file tells Git which files shouldn't be added to source control + +.DS_Store +.idea +.vscode +.zed +dist +node_modules +apify_storage +storage diff --git a/sample_actor_playwright/Dockerfile b/sample_actor_playwright/Dockerfile new file mode 100644 index 0000000..3fa1c86 --- /dev/null +++ b/sample_actor_playwright/Dockerfile @@ -0,0 +1,58 @@ +# Ships Chrome, Playwright, and an Xvfb entrypoint, so a headful browser works with no display of its own. +# You can read more about the available images at https://crawlee.dev/docs/guides/docker-images +FROM apify/actor-node-playwright-chrome:24-1.61.1 AS builder + +# Check preinstalled packages +RUN npm ls @crawlee/core apify puppeteer playwright + +# Copy just package.json and package-lock.json +# to speed up the build using Docker layer cache. +COPY --chown=myuser:myuser package*.json Dockerfile ./ + +# Check Playwright version is the same as the one from base image. +RUN node check-playwright-version.mjs + +# Install all dependencies. Don't audit to speed up the installation. +RUN npm install --include=dev --audit=false + +# Next, copy the source files using the user set +# in the base image. +COPY --chown=myuser:myuser . ./ + +# Install all dependencies and build the project. +# Don't audit to speed up the installation. +RUN npm run build + +# Create final image +FROM apify/actor-node-playwright-chrome:24-1.61.1 + +# Check preinstalled packages +RUN npm ls @crawlee/core apify puppeteer playwright + +# Copy just package.json and package-lock.json +# to speed up the build using Docker layer cache. +COPY --chown=myuser:myuser package*.json ./ + +# Install NPM packages, skip optional and development dependencies to +# keep the image small. Avoid logging too much and print the dependency +# tree for debugging +RUN npm --quiet set progress=false \ + && npm install --omit=dev --omit=optional \ + && echo "Installed NPM packages:" \ + && (npm list --omit=dev --all || true) \ + && echo "Node.js version:" \ + && node --version \ + && echo "NPM version:" \ + && npm --version \ + && rm -r ~/.npm + +# Copy built JS files from builder image +COPY --from=builder --chown=myuser:myuser /home/myuser/dist ./dist + +# Next, copy the remaining files and directories with the source code. +# Since we do this after NPM install, quick build will be really fast +# for most source file changes. +COPY --chown=myuser:myuser . ./ + +# Run the image. `node` directly (not `npm start`), so actor-runtime's debug mode can attach. +CMD ["node", "dist/main.js"] diff --git a/sample_actor_playwright/README.md b/sample_actor_playwright/README.md new file mode 100644 index 0000000..f17a56b --- /dev/null +++ b/sample_actor_playwright/README.md @@ -0,0 +1,19 @@ +## Playwright sample Actor + +A `PlaywrightCrawler` Actor from Apify's +[`ts-crawlee-playwright-chrome`](https://github.com/apify/actor-templates/tree/master/templates/ts-crawlee-playwright-chrome) +template. It crawls `maxRequestsPerCrawl` same-hostname pages from `startUrls` with a real Chrome and pushes +`{ url, title }` per page. + +The one change from the template is `headless: false`: the browser draws on the base image's Xvfb display, so +actor-runtime's **browser view** can show it. Watching changes nothing about the run. + +```bash +cd sample_actor_playwright +apify push +apify api POST /actor-runtime/browser-view/ --body '{"enabled": true}' +apify call --input '{"maxRequestsPerCrawl": 5}' +``` + +The run log prints the viewer URL (`http://localhost:3000/runs//browser`); the run's console page links to +it too. See the repository README, "Watching an Actor's browser". diff --git a/sample_actor_playwright/package.json b/sample_actor_playwright/package.json new file mode 100644 index 0000000..1943a9c --- /dev/null +++ b/sample_actor_playwright/package.json @@ -0,0 +1,29 @@ +{ + "name": "my-playwright-actor", + "version": "0.0.1", + "type": "module", + "description": "Playwright + Crawlee sample Actor for actor-runtime, based on the ts-crawlee-playwright-chrome template.", + "engines": { + "node": ">=18.0.0" + }, + "dependencies": { + "apify": "^3.7.0", + "@crawlee/playwright": "^3.15.3", + "playwright": "1.61.1" + }, + "devDependencies": { + "@apify/tsconfig": "^0.2.0", + "@types/node": "^24.0.0", + "tsx": "^4.20.3", + "typescript": "^6.0.0" + }, + "scripts": { + "start": "npm run start:dev", + "start:prod": "node dist/main.js", + "start:dev": "tsx src/main.ts", + "build": "tsc", + "postinstall": "npx crawlee install-playwright-browsers" + }, + "author": "It's not you it's me", + "license": "ISC" +} diff --git a/sample_actor_playwright/src/main.ts b/sample_actor_playwright/src/main.ts new file mode 100644 index 0000000..331c943 --- /dev/null +++ b/sample_actor_playwright/src/main.ts @@ -0,0 +1,61 @@ +// Based on Apify's `ts-crawlee-playwright-chrome` template; the one deliberate change is `headless: false` +// (see README.md). + +// For more information, see https://crawlee.dev +import { PlaywrightCrawler } from '@crawlee/playwright'; +// For more information, see https://docs.apify.com/sdk/js +import { Actor, log } from 'apify'; + +// this is ESM project, and as such, it requires you to specify extensions in your relative imports +// read more about this here: https://nodejs.org/docs/latest-v18.x/api/esm.html#mandatory-file-extensions +// note that we need to use `.js` even when inside TS files +import { router } from './routes.js'; + +interface Input { + startUrls: { + url: string; + method?: 'GET' | 'HEAD' | 'POST' | 'PUT' | 'DELETE' | 'TRACE' | 'OPTIONS' | 'CONNECT' | 'PATCH'; + headers?: Record; + userData?: Record; + }[]; + maxRequestsPerCrawl: number; +} + +// Initialize the Apify SDK +await Actor.init(); + +// Structure of input is defined in .actor/input_schema.json +const { startUrls = [{ url: 'https://crawlee.dev/' }], maxRequestsPerCrawl = 3 } = + (await Actor.getInput()) ?? ({} as Input); + +// Without a proxy password (a plain local run) the crawler connects directly instead of failing the access check. +const proxyConfiguration = process.env.APIFY_PROXY_PASSWORD + ? await Actor.createProxyConfiguration({ checkAccess: true }) + : undefined; + +log.info( + `Crawling up to ${maxRequestsPerCrawl} page(s) with a headful Chrome, starting from ${startUrls.map((s) => s.url).join(', ')}.`, +); + +const crawler = new PlaywrightCrawler({ + proxyConfiguration, + maxRequestsPerCrawl, + requestHandler: router, + // Keeps the dataset item count exactly equal to maxRequestsPerCrawl. + maxConcurrency: 1, + // Headful always, so actor-runtime's browser view has something to show and watching changes nothing. + // The base image's Xvfb provides the display. + headless: false, + launchContext: { + launchOptions: { + args: [ + '--disable-gpu', // Mitigates the "crashing GPU process" issue in Docker containers + ], + }, + }, +}); + +await crawler.run(startUrls); + +// Exit successfully +await Actor.exit(); diff --git a/sample_actor_playwright/src/routes.ts b/sample_actor_playwright/src/routes.ts new file mode 100644 index 0000000..9fdd638 --- /dev/null +++ b/sample_actor_playwright/src/routes.ts @@ -0,0 +1,16 @@ +import { createPlaywrightRouter } from '@crawlee/playwright'; + +export const router = createPlaywrightRouter(); + +// One handler for every page: record it, then follow same-hostname links until maxRequestsPerCrawl is hit. +router.addDefaultHandler(async ({ request, page, log, pushData, enqueueLinks }) => { + const title = await page.title(); + log.info(`Processing ${request.loadedUrl} - ${title}`); + + await pushData({ + url: request.loadedUrl, + title, + }); + + await enqueueLinks({ strategy: 'same-hostname' }); +}); diff --git a/sample_actor_playwright/tsconfig.json b/sample_actor_playwright/tsconfig.json new file mode 100644 index 0000000..971ad9f --- /dev/null +++ b/sample_actor_playwright/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "@apify/tsconfig", + "compilerOptions": { + "module": "NodeNext", + "moduleResolution": "NodeNext", + "target": "ES2022", + "outDir": "dist", + "rootDir": "./src", + "incremental": false, + "noUnusedLocals": false, + "skipLibCheck": true, + "lib": ["DOM"] + }, + "include": ["./src/**/*"] +} diff --git a/sample_actor_playwright_py/.actor/actor.json b/sample_actor_playwright_py/.actor/actor.json new file mode 100644 index 0000000..a43052d --- /dev/null +++ b/sample_actor_playwright_py/.actor/actor.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://apify.com/schemas/v1/actor.ide.json", + "actorSpecification": 1, + "name": "my-playwright-actor-py", + "title": "Python Playwright sample actor for actor-runtime", + "description": "Crawls up to max_requests_per_crawl same-hostname pages from start_urls with a headful Chromium driven by Playwright (Crawlee for Python), pushing one dataset item per page.", + "version": "0.0", + "buildTag": "latest", + "meta": { + "templateId": "python-crawlee-playwright" + }, + "inputSchema": "./input_schema.json", + "dockerfile": "../Dockerfile" +} diff --git a/sample_actor_playwright_py/.actor/input_schema.json b/sample_actor_playwright_py/.actor/input_schema.json new file mode 100644 index 0000000..0f3040c --- /dev/null +++ b/sample_actor_playwright_py/.actor/input_schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://apify.com/schemas/v1/input.ide.json", + "title": "Python Playwright sample actor input", + "type": "object", + "schemaVersion": 1, + "properties": { + "start_urls": { + "title": "Start URLs", + "type": "array", + "description": "URLs to start with.", + "editor": "requestListSources", + "prefill": [{ "url": "https://crawlee.dev/" }] + }, + "max_requests_per_crawl": { + "title": "Max requests per crawl", + "type": "integer", + "description": "Maximum number of pages the crawler opens, following same-hostname links from the start URLs. One dataset item is pushed per page.", + "default": 3, + "minimum": 1 + } + } +} diff --git a/sample_actor_playwright_py/.dockerignore b/sample_actor_playwright_py/.dockerignore new file mode 100644 index 0000000..0d2f887 --- /dev/null +++ b/sample_actor_playwright_py/.dockerignore @@ -0,0 +1,11 @@ +.git +.mise.toml +.nvim.lua +storage +__pycache__/ +*.py[cod] +.venv +venv/ +.idea/ +.vscode +.zed diff --git a/sample_actor_playwright_py/.gitignore b/sample_actor_playwright_py/.gitignore new file mode 100644 index 0000000..cd51172 --- /dev/null +++ b/sample_actor_playwright_py/.gitignore @@ -0,0 +1,10 @@ +.mise.toml +.nvim.lua +storage +__pycache__/ +*.py[cod] +.venv +venv/ +.idea/ +.vscode +.zed diff --git a/sample_actor_playwright_py/Dockerfile b/sample_actor_playwright_py/Dockerfile new file mode 100644 index 0000000..5466b6f --- /dev/null +++ b/sample_actor_playwright_py/Dockerfile @@ -0,0 +1,33 @@ +# Ships Chromium, Playwright, and an Xvfb entrypoint, so a headful browser works with no display of its own. +# You can see the Docker images from Apify at https://hub.docker.com/r/apify/. +FROM apify/actor-python-playwright:3.14-1.61.0 + +USER myuser + +# Second, copy just requirements.txt into the Actor image, +# since it should be the only file that affects the dependency install in the next step, +# in order to speed up the build +COPY --chown=myuser:myuser requirements.txt ./ + +# Install the packages specified in requirements.txt, +# Print the installed Python version, pip version +# and all installed packages with their versions for debugging +RUN echo "Python version:" \ + && python --version \ + && echo "Pip version:" \ + && pip --version \ + && echo "Installing dependencies:" \ + && pip install -r requirements.txt \ + && echo "All installed Python packages:" \ + && pip freeze + +# Next, copy the remaining files and directories with the source code. +# Since we do this after installing the dependencies, quick build will be really fast +# for most source file changes. +COPY --chown=myuser:myuser . ./ + +# Use compileall to ensure the runnability of the Actor Python code. +RUN python -m compileall -q my_actor/ + +# Specify how to launch the source code of your Actor. `python` directly, so actor-runtime's debug mode can attach. +CMD ["python", "-m", "my_actor"] diff --git a/sample_actor_playwright_py/README.md b/sample_actor_playwright_py/README.md new file mode 100644 index 0000000..c77f29d --- /dev/null +++ b/sample_actor_playwright_py/README.md @@ -0,0 +1,20 @@ +## Python Playwright sample Actor + +A Crawlee for Python `PlaywrightCrawler` Actor from Apify's +[`python-crawlee-playwright`](https://github.com/apify/actor-templates/tree/master/templates/python-crawlee-playwright) +template. It crawls `max_requests_per_crawl` same-hostname pages from `start_urls` with a real Chromium and pushes +`{ url, title, h1s, h2s, h3s }` per page. + +The one change from the template is `headless=False` (the template hard-codes `headless=True`): the browser draws +on the base image's Xvfb display, so actor-runtime's **browser view** can show it. Watching changes nothing about +the run. + +```bash +cd sample_actor_playwright_py +apify push +apify api POST /actor-runtime/browser-view/ --body '{"enabled": true}' +apify call --input '{"max_requests_per_crawl": 5}' +``` + +The run log prints the viewer URL (`http://localhost:3000/runs//browser`); the run's console page links to +it too. See the repository README, "Watching an Actor's browser". diff --git a/sample_actor_playwright_py/my_actor/__init__.py b/sample_actor_playwright_py/my_actor/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sample_actor_playwright_py/my_actor/__main__.py b/sample_actor_playwright_py/my_actor/__main__.py new file mode 100644 index 0000000..8c4ab0b --- /dev/null +++ b/sample_actor_playwright_py/my_actor/__main__.py @@ -0,0 +1,6 @@ +import asyncio + +from .main import main + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/sample_actor_playwright_py/my_actor/main.py b/sample_actor_playwright_py/my_actor/main.py new file mode 100644 index 0000000..c3868a8 --- /dev/null +++ b/sample_actor_playwright_py/my_actor/main.py @@ -0,0 +1,55 @@ +"""Based on Apify's `python-crawlee-playwright` template; the one deliberate change is `headless=False` +(the template hard-codes `headless=True`) - see README.md. + +To build Apify Actors, utilize the Apify SDK toolkit, read more at the official documentation: +https://docs.apify.com/sdk/python +""" + +from __future__ import annotations + +from apify import Actor +from crawlee import ConcurrencySettings +from crawlee.crawlers import PlaywrightCrawler + +from .routes import router + + +async def main() -> None: + """Define a main entry point for the Apify Actor. + + This coroutine is executed using `asyncio.run()`, so it must remain an asynchronous function for proper execution. + Asynchronous execution is required for communication with Apify platform, and it also enhances performance in + the field of web scraping significantly. + """ + # Enter the context of the Actor. + async with Actor: + # Retrieve the Actor input, and use default values if not provided. + actor_input = await Actor.get_input() or {} + start_urls = [url.get('url') for url in actor_input.get('start_urls', [{'url': 'https://crawlee.dev/'}])] + max_requests_per_crawl = int(actor_input.get('max_requests_per_crawl', 3)) + + # Exit if no start URLs are provided. + if not start_urls: + Actor.log.info('No start URLs specified in Actor input, exiting...') + await Actor.exit() + + Actor.log.info( + f'Crawling up to {max_requests_per_crawl} page(s) with a headful Chromium, starting from {", ".join(start_urls)}.' + ) + + # Create a crawler. + crawler = PlaywrightCrawler( + # Limit the crawl to max requests. Remove or increase it for crawling all links. + max_requests_per_crawl=max_requests_per_crawl, + # Keeps the dataset item count exactly equal to max_requests_per_crawl. + concurrency_settings=ConcurrencySettings(desired_concurrency=1, max_concurrency=1), + # Headful always, so actor-runtime's browser view has something to show and watching changes + # nothing. The base image's Xvfb provides the display. + headless=False, + browser_launch_options={'args': ['--disable-gpu', '--no-sandbox']}, + # Set the request handler to the request router defined in routes.py. + request_handler=router, + ) + + # Run the crawler with the starting requests. + await crawler.run(start_urls) diff --git a/sample_actor_playwright_py/my_actor/routes.py b/sample_actor_playwright_py/my_actor/routes.py new file mode 100644 index 0000000..2f1d324 --- /dev/null +++ b/sample_actor_playwright_py/my_actor/routes.py @@ -0,0 +1,32 @@ +"""Module defines the router and request handlers for the crawler.""" + +from __future__ import annotations + +from apify import Actor +from crawlee.crawlers import PlaywrightCrawlingContext +from crawlee.router import Router + +router = Router[PlaywrightCrawlingContext]() + + +@router.default_handler +async def default_handler(context: PlaywrightCrawlingContext) -> None: + """Handle each request by extracting data and enqueueing links.""" + url = context.request.url + title = await context.page.title() + Actor.log.info(f'Processing {url} - {title}') + + # Extract the desired data. + data = { + 'url': url, + 'title': title, + 'h1s': [await h1.text_content() for h1 in await context.page.locator('h1').all()], + 'h2s': [await h2.text_content() for h2 in await context.page.locator('h2').all()], + 'h3s': [await h3.text_content() for h3 in await context.page.locator('h3').all()], + } + + # Store the extracted data to the default dataset. + await context.push_data(data) + + # Enqueue additional same-hostname links found on the current page. + await context.enqueue_links(strategy='same-hostname') diff --git a/sample_actor_playwright_py/requirements.txt b/sample_actor_playwright_py/requirements.txt new file mode 100644 index 0000000..fabc8d1 --- /dev/null +++ b/sample_actor_playwright_py/requirements.txt @@ -0,0 +1,5 @@ +# Feel free to add your Python dependencies below. For formatting guidelines, see: +# https://pip.pypa.io/en/latest/reference/requirements-file-format/ + +apify >= 4.0.0, < 5.0.0 +crawlee[playwright] >= 1.7.0 diff --git a/src/api/routes/browser-view.ts b/src/api/routes/browser-view.ts new file mode 100644 index 0000000..b4980d7 --- /dev/null +++ b/src/api/routes/browser-view.ts @@ -0,0 +1,26 @@ +/** `POST /actor-runtime/browser-view/:actorId` - the browser-view toggle, same shape and scoping as `debug-mode.ts`. */ +import type { Router } from 'express'; + +import { requireUser } from '../auth.js'; +import { sendData } from '../envelope.js'; +import { invalidRequest, recordNotFound } from '../errors.js'; +import { h, jsonBody } from '../handler.js'; +import { browserViewStatus, setBrowserView } from '../../services/browser-view.js'; +import { resolveOwnedActor } from '../../services/actors.js'; + +export function mountBrowserView(router: Router): void { + router.post( + '/browser-view/:actorId', + h(async (req, res) => { + const user = requireUser(req); + const actor = await resolveOwnedActor(user.id, req.params.actorId as string, user.username); + if (!actor) throw recordNotFound(); + + const raw = jsonBody(req); + const result = await setBrowserView(actor, raw); + if (result.kind !== 'ok') throw invalidRequest(result.message); + + sendData(res, browserViewStatus(result.actor)); + }), + ); +} diff --git a/src/api/server.ts b/src/api/server.ts index f8a6c64..e75a734 100644 --- a/src/api/server.ts +++ b/src/api/server.ts @@ -15,6 +15,7 @@ import { mountLogs } from './routes/logs.js'; import { mountRunStorageAliases } from './routes/run-storage-aliases.js'; import { mountDevFolder } from './routes/dev-folder.js'; import { mountDebugMode } from './routes/debug-mode.js'; +import { mountBrowserView } from './routes/browser-view.js'; import { mountMigrate } from './routes/migrate.js'; import { mountApiFallback } from './routes/api-fallback.js'; import { attemptFallback, type LocalError } from '../services/api-fallback.js'; @@ -51,6 +52,7 @@ export function createApiServer(deps: ApiServerDeps): Express { actorRuntime.use(auth()); mountDevFolder(actorRuntime, deps); mountDebugMode(actorRuntime); + mountBrowserView(actorRuntime); mountMigrate(actorRuntime, deps); mountApiFallback(actorRuntime); app.use('/actor-runtime', actorRuntime); diff --git a/src/config.ts b/src/config.ts index 8073011..766fb31 100644 --- a/src/config.ts +++ b/src/config.ts @@ -37,3 +37,18 @@ export function debugpyPayloadTarPath(): string { export function debugpyVersionFilePath(): string { return `${debugpyPayloadDir()}/debugpy-version.txt`; } + +/** Read fresh on every call, like `debugpyPayloadDir()`, so tests can point it at a fixture directory. */ +function browserViewerPayloadDir(): string { + return process.env.ACTOR_RUNTIME_BROWSER_VIEWER_PAYLOAD_DIR ?? '/opt/apify-browser-viewer'; +} + +/** The browser-view sidecar's root filesystem, `docker import`ed on first use. */ +export function browserViewerRootfsTarPath(): string { + return `${browserViewerPayloadDir()}/rootfs.tar`; +} + +/** Content hash of `rootfs.tar`, used as the imported image's tag. */ +export function browserViewerVersionFilePath(): string { + return `${browserViewerPayloadDir()}/version.txt`; +} diff --git a/src/console/browser-view-ws.ts b/src/console/browser-view-ws.ts new file mode 100644 index 0000000..ebd0cd1 --- /dev/null +++ b/src/console/browser-view-ws.ts @@ -0,0 +1,155 @@ +/** + * `GET /runs/:runId/browser/ws`: bridges the viewer page's noVNC client to the run's sidecar VNC server over + * `apify-local` (the runtime plays websockify). Upgraded on the console server directly, like + * `api/events-ws.ts` on the API server. Unauthenticated like the rest of the console. + */ +import type { IncomingMessage, Server } from 'node:http'; +import type { Duplex } from 'node:stream'; +import { connect, type Socket } from 'node:net'; +import { WebSocketServer, type RawData, type WebSocket } from 'ws'; + +import { getRunById } from '../services/runs.js'; +import { getActorById } from '../services/actors.js'; +import { isTerminalJobStatus } from '../services/job-status.js'; +import type { RunRecord } from '../storage/entities.js'; + +const BROWSER_VIEW_WS_PATH_PATTERN = /^\/runs\/([^/]+)\/browser\/ws$/; + +/** The sidecar's VNC server only listens once the Actor's X display exists; dial retries cover the gap. */ +const VNC_CONNECT_TIMEOUT_MS = 120_000; +const VNC_CONNECT_RETRY_MS = 500; + +export interface BrowserViewWebSocketServer { + /** Same contract as `EventsWebSocketServer.close()`. */ + close(): void; +} + +export function extractBrowserViewRunId(pathname: string): string | undefined { + return BROWSER_VIEW_WS_PATH_PATTERN.exec(pathname)?.[1]; +} + +function dialOnce(host: string, port: number): Promise { + return new Promise((resolve, reject) => { + const socket = connect({ host, port }); + socket.once('connect', () => { + socket.removeListener('error', reject); + resolve(socket); + }); + socket.once('error', reject); + }); +} + +/** Resolves `undefined` when cancelled or out of budget; never rejects. */ +async function dialWithRetry(host: string, port: number, isCancelled: () => boolean): Promise { + const deadline = Date.now() + VNC_CONNECT_TIMEOUT_MS; + for (;;) { + if (isCancelled()) return undefined; + try { + return await dialOnce(host, port); + } catch { + if (Date.now() >= deadline) return undefined; + await new Promise((resolve) => setTimeout(resolve, VNC_CONNECT_RETRY_MS)); + } + } +} + +function toBuffer(data: RawData): Buffer { + if (Buffer.isBuffer(data)) return data; + if (Array.isArray(data)) return Buffer.concat(data); + return Buffer.from(data); +} + +type MirroredRun = RunRecord & { localBrowserView: NonNullable }; + +/** The run's mirror address, or a `1008` reason. Waits while the mirror is still starting (see + * `isBrowserViewPending`). */ +async function awaitMirroredRun(runId: string, isCancelled: () => boolean): Promise { + const deadline = Date.now() + VNC_CONNECT_TIMEOUT_MS; + for (;;) { + const run = await getRunById(runId); + if (!run) return `Unknown run id: ${runId}`; + if (run.localBrowserView) { + if (isTerminalJobStatus(run.status)) return `Run ${runId} has already ended`; + return run as MirroredRun; + } + if (isTerminalJobStatus(run.status)) return `Browser view was not on for run ${runId}`; + if (!(await isBrowserViewPending(run))) return `Browser view is not on for run ${runId}`; + if (isCancelled() || Date.now() >= deadline) return `The display mirror of run ${runId} did not start in time`; + await new Promise((resolve) => setTimeout(resolve, VNC_CONNECT_RETRY_MS)); + } +} + +/** A live run with the toggle on but no mirror address yet: `services/runs.ts` writes it once the sidecar + * is up, a moment after the run id exists. */ +export async function isBrowserViewPending(run: RunRecord): Promise { + if (run.localBrowserView || isTerminalJobStatus(run.status)) return false; + const actor = await getActorById(run.actorId); + return actor?.localBrowserView !== undefined; +} + +async function handleConnection(ws: WebSocket, runId: string): Promise { + // An `'error'` with no listener crashes the process. + ws.on('error', () => undefined); + + let closed = false; + ws.once('close', () => { + closed = true; + }); + + const run = await awaitMirroredRun(runId, () => closed); + if (typeof run === 'string') { + if (!closed) ws.close(1008, run); + return; + } + + const socket = await dialWithRetry(run.localBrowserView.vncHost, run.localBrowserView.vncPort, () => closed); + if (!socket) { + if (!closed) ws.close(1011, `The display mirror of run ${runId} is not reachable`); + return; + } + if (closed) { + socket.destroy(); + return; + } + + ws.on('message', (data) => { + if (!socket.destroyed) socket.write(toBuffer(data)); + }); + socket.on('data', (chunk: Buffer) => { + if (ws.readyState === ws.OPEN) ws.send(chunk, { binary: true }); + }); + socket.on('error', () => undefined); + // The run ending removes the sidecar, which closes the TCP side. + socket.once('close', () => { + if (ws.readyState === ws.OPEN || ws.readyState === ws.CONNECTING) { + ws.close(1000, `The display mirror of run ${runId} has gone away`); + } + }); + ws.once('close', () => socket.destroy()); +} + +export function attachBrowserViewWebSocket(server: Server): BrowserViewWebSocketServer { + const wss = new WebSocketServer({ noServer: true }); + + server.on('upgrade', (req: IncomingMessage, socket: Duplex, head: Buffer) => { + const pathname = req.url ? new URL(req.url, 'http://localhost').pathname : undefined; + const runId = pathname ? extractBrowserViewRunId(pathname) : undefined; + if (!runId) { + socket.destroy(); + return; + } + + wss.handleUpgrade(req, socket, head, (ws) => { + void handleConnection(ws, runId).catch(() => { + ws.terminate(); + }); + }); + }); + + return { + close() { + for (const client of wss.clients) client.terminate(); + wss.close(); + }, + }; +} diff --git a/src/console/server.ts b/src/console/server.ts index dc06651..ce646bd 100644 --- a/src/console/server.ts +++ b/src/console/server.ts @@ -16,6 +16,8 @@ * form is runtime-global by nature (`api.md`'s "Upstream fallback" section), so ownership doesn't apply * to it at all. */ +import { createRequire } from 'node:module'; +import { dirname, join } from 'node:path'; import express, { type Express, type Request } from 'express'; import { getActorById, listAllActors } from '../services/actors.js'; @@ -27,9 +29,12 @@ import { type DevFolderStatus, } from '../services/dev-folder.js'; import { debugStatus, setDebugMode } from '../services/debug-mode.js'; +import { browserViewStatus, setBrowserView } from '../services/browser-view.js'; import { getBuildById, listAllBuilds } from '../services/builds.js'; import { getRunById, listAllRuns } from '../services/runs.js'; import { migrateRun } from '../services/migrations.js'; +import { isTerminalJobStatus } from '../services/job-status.js'; +import { isBrowserViewPending } from './browser-view-ws.js'; import { getFullLog } from '../services/logs.js'; import { getStorageById, listAllStorages } from '../services/storages.js'; import { listRequests } from '../services/request-queues.js'; @@ -40,6 +45,8 @@ import { ansiToHtml } from './ansi.js'; import { newestFirst } from './order.js'; import { apiFallbackWarning, + browserViewForm, + browserViewPage, debugModeForm, definitionList, devFolderForm, @@ -83,6 +90,9 @@ export interface ConsoleServerDeps { driver: Driver; } +/** `@novnc/novnc`'s `exports` points at `core/rfb.js`; the package root is two levels up from it. */ +const NOVNC_ROOT = dirname(dirname(createRequire(import.meta.url).resolve('@novnc/novnc'))); + /** The dev-folder registration form + its one read-only status row, rendered on the Actor detail view * (`console.md`'s "Local dev-folder registration form" section). Deliberately shows only the registered * folder, never a build's working directory or a "mount will apply" claim - whether a mount actually @@ -114,9 +124,32 @@ function debugModeSection(actorId: string, localDebug: ActorRecord['localDebug'] ); } +function browserViewSection( + actorId: string, + localBrowserView: ActorRecord['localBrowserView'], + errorMessage?: string, +): string { + const status = browserViewStatus({ localBrowserView }); + return ( + '

Browser view

' + + definitionList([ + [ + 'browser view', + status.localBrowserView + ? `on, ${status.localBrowserView.interactive ? 'interactive' : 'view-only'}` + : '(browser view is off)', + ], + ]) + + browserViewForm(actorId, localBrowserView ?? null, errorMessage) + ); +} + export function createConsoleServer(deps: ConsoleServerDeps): Express { const app = express(); app.disable('x-powered-by'); + // The noVNC client for the browser-view page, served straight from the installed package. + app.use('/vendor/novnc/core', express.static(join(NOVNC_ROOT, 'core'))); + app.use('/vendor/novnc/vendor', express.static(join(NOVNC_ROOT, 'vendor'))); // The dev-folder form, the debug-mode form, the run detail view's Migrate button, and the `/settings` // form below are the console's only four writes - every other route is a plain `GET` (`console.md`'s // "Every route is a read except..." list). @@ -149,6 +182,8 @@ export function createConsoleServer(deps: ConsoleServerDeps): Express { } const devFolderError = typeof req.query.devFolderError === 'string' ? req.query.devFolderError : undefined; const debugModeError = typeof req.query.debugModeError === 'string' ? req.query.debugModeError : undefined; + const browserViewError = + typeof req.query.browserViewError === 'string' ? req.query.browserViewError : undefined; const body = definitionList([ ['id', actor.id], @@ -171,10 +206,37 @@ export function createConsoleServer(deps: ConsoleServerDeps): Express { '/builds', ) + devFolderSection(actor.id, devFolderStatus(actor), devFolderError) + - debugModeSection(actor.id, actor.localDebug, debugModeError); + debugModeSection(actor.id, actor.localDebug, debugModeError) + + browserViewSection(actor.id, actor.localBrowserView, browserViewError); res.send(layout(`Actor ${actor.name}`, body)); }); + /** Same `setBrowserView` as the API endpoint, cross-user like the debug-mode form above. */ + app.post('/actors/:id/browser-view', async (req, res) => { + if (isCrossSiteWrite(req)) { + res.status(403).send('Cross-site form submissions are not allowed.'); + return; + } + const actor = await getActorById(req.params.id); + if (!actor) { + res.status(404).send(layout('Not found', '

Actor not found.

')); + return; + } + const body = req.body as Record | undefined; + const enabled = body?.enabled === 'on'; + const requestBody: Record = { enabled }; + if (enabled) requestBody.interactive = body?.interactive === 'on'; + + const result = await setBrowserView(actor, requestBody); + if (result.kind !== 'ok') { + res.redirect( + `/actors/${encodeURIComponent(actor.id)}?browserViewError=${encodeURIComponent(result.message)}`, + ); + return; + } + res.redirect(`/actors/${encodeURIComponent(actor.id)}`); + }); + /** One of the console's four mutations - funnels through the same `setDevFolder` the API endpoint uses, * resolving the Actor cross-user by the id already in the page URL (no token) rather than through * `resolveOwnedActor`. A failure redirects back with `describeDevFolderFailure`'s message in a query @@ -367,6 +429,15 @@ export function createConsoleServer(deps: ConsoleServerDeps): Express { if (run.localDebug) { rows.push(['debug', `${run.localDebug.language}, attach at 127.0.0.1:${run.localDebug.port}`]); } + if (run.localBrowserView) { + rows.push([ + 'browser view', + { + text: `${run.localBrowserView.interactive ? 'interactive' : 'view-only'} live mirror of the run's display`, + href: `/runs/${encodeURIComponent(run.id)}/browser`, + }, + ]); + } const body = definitionList(rows) + migrateSection + @@ -376,6 +447,61 @@ export function createConsoleServer(deps: ConsoleServerDeps): Express { res.send(layout(`Run ${run.id}`, body)); }); + /** The viewer page; its websocket is handled by `console/browser-view-ws.ts`, not Express. */ + app.get('/runs/:id/browser', async (req, res) => { + const run = await getRunById(req.params.id); + if (!run) { + res.status(404).send(layout('Not found', '

Run not found.

')); + return; + } + const backLink = `

Back to the run

`; + // Mirror still starting: render the client, which retries. + if (!run.localBrowserView && (await isBrowserViewPending(run))) { + const actor = await getActorById(run.actorId); + res.send( + layout( + `Browser view of run ${run.id}`, + browserViewPage({ + ...run, + localBrowserView: { + interactive: actor?.localBrowserView?.interactive ?? false, + vncHost: '', + vncPort: 0, + }, + }), + ), + ); + return; + } + if (!run.localBrowserView) { + res.status(404).send( + layout( + `Browser view of run ${run.id}`, + '

Browser view was not on for this run when it started, so there is no display ' + + 'mirror to show. Turn it on for the Actor and start a new run.

' + + backLink, + ), + ); + return; + } + if (isTerminalJobStatus(run.status)) { + res.send( + layout( + `Browser view of run ${run.id}`, + `

This run has ended (status: ${escapeHtml(run.status)}); its display mirror is gone.

` + + backLink, + ), + ); + return; + } + res.send( + layout( + `Browser view of run ${run.id}`, + browserViewPage(run as typeof run & { localBrowserView: NonNullable }), + ), + ); + }); + /** One of the console's four writes (`console.md`) - the same `migrateRun` as the API endpoint, cross-user * like the dev-folder form. */ app.post('/runs/:id/migrate', async (req, res) => { diff --git a/src/console/templates.ts b/src/console/templates.ts index 6132121..513189e 100644 --- a/src/console/templates.ts +++ b/src/console/templates.ts @@ -1,7 +1,7 @@ /** Minimal server-rendered HTML helpers. No SPA, no bundler, no build step (`console.md`). */ import { getApiFallbackState, type ApiFallbackState } from '../services/api-fallback.js'; -import type { ActorLocalDebug } from '../storage/entities.js'; +import type { ActorLocalBrowserView, ActorLocalDebug, RunRecord } from '../storage/entities.js'; export function escapeHtml(value: unknown): string { return String(value ?? '') @@ -58,6 +58,8 @@ export function layout(title: string, body: string): string { .warning { color: #94600b; } .wide-input { width: 28rem; } h1 { margin-top: 0; } + .browser-view-screen { width: 100%; height: 75vh; background: #222; } + .browser-view-screen canvas { outline: none; } @@ -155,6 +157,79 @@ export function debugModeForm( ); } +/** The browser-view toggle form; both API fields, submitted together like `debugModeForm`. */ +export function browserViewForm( + actorId: string, + current: ActorLocalBrowserView | null | undefined, + errorMessage?: string, +): string { + const errorHtml = errorMessage ? `

Error: ${escapeHtml(errorMessage)}

` : ''; + return ( + errorHtml + + `
` + + ` ` + + ` ' + + '' + + '
' + + '

When on, every run of this Actor gets a live mirror of its display, linked from the ' + + "run's page. The browser must run headful to show anything (Crawlee JS: headless: false, " + + 'Python: headless=False).

' + ); +} + +/** The viewer page: noVNC (served under `/vendor/novnc/`) connected to `console/browser-view-ws.ts`. The + * run id is embedded as a JSON literal with `<` escaped so it cannot break out of the script element. */ +export function browserViewPage( + run: RunRecord & { localBrowserView: NonNullable }, +): string { + const runIdLiteral = JSON.stringify(run.id).replace(/Live mirror of the X display of run ${escapeHtml(run.id)} ` + + `(${mode}). ` + + (run.localBrowserView.interactive + ? 'Your mouse and keyboard input is delivered to the display. Nothing else is: no clipboard, no data to the Actor. ' + : 'Nothing is sent to the display, the browser, or the Actor: no input, no clipboard. ') + + 'The picture is read from the display the browser draws on.

' + + '

Connecting…

' + + '
' + + `` + ); +} + /** Rendered only for a `RUNNING` run (`console.md`, "Migrate button"). */ export function migrateRunForm(runId: string): string { return ( diff --git a/src/driver/docker-driver.ts b/src/driver/docker-driver.ts index c263345..0e7f7c7 100644 --- a/src/driver/docker-driver.ts +++ b/src/driver/docker-driver.ts @@ -27,17 +27,26 @@ * The rest of the runtime (storages, actors-as-records, console) is unaffected. */ import { PassThrough } from 'node:stream'; +import { createReadStream } from 'node:fs'; import { readFile } from 'node:fs/promises'; import Docker from 'dockerode'; import * as tar from 'tar-stream'; -import { CONTAINER_API_ALIAS, debugpyPayloadTarPath, debugpyVersionFilePath } from '../config.js'; +import { + CONTAINER_API_ALIAS, + browserViewerRootfsTarPath, + browserViewerVersionFilePath, + debugpyPayloadTarPath, + debugpyVersionFilePath, +} from '../config.js'; import { CPU_PERIOD_US, cpuQuotaFor, dedicatedCpusFor } from '../resources.js'; import { normalizeEntryName } from './tar-entry-name.js'; import type { SourceFile } from '../storage/entities.js'; import { DebugPortInUseError, DriverTimedOutError, + type BrowserViewerHandle, + type BrowserViewerTarget, type BuildContext, type BuildOutcome, type DevFolderMount, @@ -58,6 +67,22 @@ const PROBE_LABEL = 'actor-runtime.devFolderProbe'; /** Target path for the probe container's mount - arbitrary, since the probe is never started and * nothing ever reads from it. */ const PROBE_MOUNT_TARGET = '/probe'; +/** On the browser-view sidecar container and its volume, so `reconcileOrphans` can sweep leftovers. */ +const BROWSER_VIEWER_LABEL = 'actor-runtime.browserViewer'; +/** Tagged with the payload's content hash, so a rebuilt runtime imports a fresh image. */ +const BROWSER_VIEWER_IMAGE_REPO = 'actor-runtime/browser-viewer'; +/** Shared between the Actor container and the sidecar through a tmpfs volume. */ +const X11_SOCKET_DIR = '/tmp/.X11-unix'; +/** Reachable only on `apify-local`; never published on the host. */ +const BROWSER_VIEWER_VNC_PORT = 5900; +const BROWSER_VIEWER_MEMORY_BYTES = 256 * 1024 * 1024; +const BROWSER_VIEWER_SCRIPT = '/apify-browser-viewer.sh'; +/** Names must match `docker/browser-viewer.sh`. */ +const BROWSER_VIEWER_INTERACTIVE_ENV = 'APIFY_BROWSER_VIEWER_INTERACTIVE'; +const BROWSER_VIEWER_PORT_ENV = 'APIFY_BROWSER_VIEWER_PORT'; +/** Removing a volume right after its last container can race the daemon ("volume is in use"). */ +const VOLUME_REMOVE_ATTEMPTS = 10; +const VOLUME_REMOVE_RETRY_MS = 200; /** Tag for `ensureProbeImage`'s own minimal image - built and owned by this driver, never an Actor's. * An explicit `:probe` suffix, deliberately never `latest` (Docker's own implicit default for an * untagged name) - this image has nothing to do with an Actor's `latest`-tagged build, and an untagged @@ -301,6 +326,10 @@ export class DockerDriver implements Driver { private probeImageBuild: Promise | undefined; /** Python debug payload tar + debugpy version, read from disk at most once and cached. */ private debugPayload: { tar: Buffer; debugpyVersion: string } | undefined; + private browserViewerImageId: string | undefined; + /** Shared by concurrent callers; cleared on failure so a later call retries (like `probeImageBuild`). */ + private browserViewerImport: Promise | undefined; + private readonly browserViewers = new Map(); available = false; unavailableReason: string | undefined; @@ -573,6 +602,14 @@ export class DockerDriver implements Driver { } } + // The X-socket volume is the only change a browser-view run makes to the Actor's container. + const mounts: Docker.MountSettings[] = [ + ...(ctx.devMount ? this.buildDevMounts(ctx.devMount) : []), + ...(ctx.x11SocketVolume + ? [{ Type: 'volume' as const, Source: ctx.x11SocketVolume, Target: X11_SOCKET_DIR }] + : []), + ]; + const container = await this.docker.createContainer({ Image: ctx.imageId, Env: env, @@ -587,7 +624,7 @@ export class DockerDriver implements Driver { CpuPeriod: CPU_PERIOD_US, CpuQuota: cpuQuotaFor(ctx.memoryMbytes), AutoRemove: false, - ...(ctx.devMount ? { Mounts: this.buildDevMounts(ctx.devMount) } : {}), + ...(mounts.length > 0 ? { Mounts: mounts } : {}), // Fixed 127.0.0.1-bound publish - lands on the developer's own host, not wherever the // runtime process itself runs. ...(ctx.debug @@ -882,6 +919,153 @@ export class DockerDriver implements Driver { await container.stop().catch(() => undefined); } + /** Imports the bundled sidecar rootfs (`docker import`, no network) once per process; an image already + * present under the content-hash tag is reused. */ + private async ensureBrowserViewerImage(): Promise { + if (this.browserViewerImageId) return this.browserViewerImageId; + this.browserViewerImport ??= this.importBrowserViewerImage().catch((error) => { + this.browserViewerImport = undefined; + throw error; + }); + const imageId = await this.browserViewerImport; + this.browserViewerImageId = imageId; + return imageId; + } + + private async importBrowserViewerImage(): Promise { + let version: string; + try { + version = (await readFile(browserViewerVersionFilePath(), 'utf8')).trim(); + } catch (error) { + throw new Error( + `the runtime's browser-view sidecar payload is missing (${(error as Error).message}). Browser view ` + + `needs the runtime to run from its own built image, not from source (e.g. \`pnpm dev\`).`, + ); + } + const tag = `${BROWSER_VIEWER_IMAGE_REPO}:${version}`; + + try { + await this.docker.getImage(tag).inspect(); + return tag; + } catch (error) { + if (!hasStatusCode(error) || error.statusCode !== 404) throw error; + } + + // A read error on the tar must reject the import, not surface as an unhandled stream error. + const rootfs = createReadStream(browserViewerRootfsTarPath()); + let rootfsError: Error | undefined; + rootfs.on('error', (error: Error) => { + rootfsError = error; + }); + const stream = await this.docker.importImage(rootfs, { + repo: BROWSER_VIEWER_IMAGE_REPO, + tag: version, + }); + await new Promise((resolve, reject) => { + if (rootfsError) { + reject(rootfsError); + return; + } + rootfs.once('error', reject); + this.docker.modem.followProgress(stream, (err: Error | null, res: Array<{ error?: string }>) => { + if (err) { + reject(err); + return; + } + const errorLine = res.find((line) => line.error); + if (errorLine) { + reject(new Error(errorLine.error)); + return; + } + resolve(); + }); + }); + return tag; + } + + /** The volume is created with mode 1777 up front: the Actor's Xvfb runs unprivileged and must be able + * to create its socket there. Anything created here is removed again if a later step fails. */ + async startBrowserViewer(target: BrowserViewerTarget): Promise { + if (!this.available) { + throw new Error(this.unavailableReason ?? 'Docker is not available'); + } + const imageId = await this.ensureBrowserViewerImage(); + + const volumeName = `actor-runtime-x11-${target.runId}`; + const containerName = `actor-runtime-browser-viewer-${target.runId}`; + const labels = { [RUN_LABEL]: target.runId, [BROWSER_VIEWER_LABEL]: 'true' }; + + await this.docker.createVolume({ + Name: volumeName, + Driver: 'local', + DriverOpts: { type: 'tmpfs', device: 'tmpfs', o: 'size=8m,mode=1777' }, + Labels: labels, + }); + + let container: Docker.Container | undefined; + try { + container = await this.docker.createContainer({ + Image: imageId, + name: containerName, + Cmd: ['/bin/sh', BROWSER_VIEWER_SCRIPT], + Env: [ + `${BROWSER_VIEWER_INTERACTIVE_ENV}=${target.interactive ? '1' : '0'}`, + `${BROWSER_VIEWER_PORT_ENV}=${BROWSER_VIEWER_VNC_PORT}`, + ], + Labels: labels, + HostConfig: { + NetworkMode: NETWORK_NAME, + Memory: BROWSER_VIEWER_MEMORY_BYTES, + AutoRemove: false, + Mounts: [{ Type: 'volume', Source: volumeName, Target: X11_SOCKET_DIR }], + }, + Tty: false, + }); + this.browserViewers.set(target.runId, { container, volumeName }); + await container.start(); + + const info = await container.inspect(); + const address = info.NetworkSettings?.Networks?.[NETWORK_NAME]?.IPAddress; + return { + // The IP also works from a runtime running outside Docker; the name only resolves from inside. + vncHost: address || containerName, + vncPort: BROWSER_VIEWER_VNC_PORT, + x11SocketVolume: volumeName, + }; + } catch (error) { + this.browserViewers.delete(target.runId); + if (container) await container.remove({ force: true }).catch(() => undefined); + await this.removeVolumeWithRetry(volumeName); + throw error; + } + } + + async stopBrowserViewer(runId: string): Promise { + const viewer = this.browserViewers.get(runId); + if (!viewer) return; + this.browserViewers.delete(runId); + await viewer.container.remove({ force: true }).catch(() => undefined); + await this.removeVolumeWithRetry(viewer.volumeName); + } + + /** Best-effort: a 404 is success, the last failure is logged, never thrown. */ + private async removeVolumeWithRetry(volumeName: string): Promise { + const volume = this.docker.getVolume(volumeName); + for (let attempt = 1; ; attempt++) { + try { + await volume.remove({ force: true }); + return; + } catch (error) { + if (hasStatusCode(error) && error.statusCode === 404) return; + if (attempt >= VOLUME_REMOVE_ATTEMPTS) { + console.warn(`Could not remove browser-view volume ${volumeName}: ${(error as Error).message}`); + return; + } + await new Promise((resolve) => setTimeout(resolve, VOLUME_REMOVE_RETRY_MS)); + } + } + } + /** * Cleans up two kinds of leftover containers from a previous process: orphaned *run* containers * (builds never create one of their own - orphaned build *records* are still marked `ABORTED` by @@ -902,22 +1086,33 @@ export class DockerDriver implements Driver { if (!this.available) return; const runIdSet = new Set(runIds); - const [runLabelled, probeLabelled] = await Promise.all([ + const [runLabelled, probeLabelled, viewerLabelled] = await Promise.all([ this.docker.listContainers({ all: true, filters: JSON.stringify({ label: [RUN_LABEL] }) }), this.docker.listContainers({ all: true, filters: JSON.stringify({ label: [PROBE_LABEL] }) }), + this.docker.listContainers({ all: true, filters: JSON.stringify({ label: [BROWSER_VIEWER_LABEL] }) }), ]); const byId = new Map(); - for (const info of [...runLabelled, ...probeLabelled]) byId.set(info.Id, info); + for (const info of [...runLabelled, ...probeLabelled, ...viewerLabelled]) byId.set(info.Id, info); for (const info of byId.values()) { const isOrphanedRun = runIdSet.has(info.Labels?.[RUN_LABEL] ?? ''); const isLeftoverProbe = info.Labels?.[PROBE_LABEL] !== undefined; - if (!isOrphanedRun && !isLeftoverProbe) continue; + // Swept unconditionally, like a probe. + const isLeftoverViewer = info.Labels?.[BROWSER_VIEWER_LABEL] !== undefined; + if (!isOrphanedRun && !isLeftoverProbe && !isLeftoverViewer) continue; const container = this.docker.getContainer(info.Id); // `{ v: true }` alongside `force: true`: an orphaned run's anonymous `node_modules` volume (if // it had a `devMount`) must not survive reconciliation either (mirrors `startRun`'s finally // block's identical fix). await container.remove({ force: true, v: true }).catch(() => undefined); } + + // Named volumes are not covered by `{ v: true }` above. + const { Volumes: viewerVolumes } = await this.docker.listVolumes({ + filters: JSON.stringify({ label: [BROWSER_VIEWER_LABEL] }), + }); + for (const volume of viewerVolumes ?? []) { + await this.removeVolumeWithRetry(volume.Name); + } } } diff --git a/src/driver/types.ts b/src/driver/types.ts index 0fe17ac..559f18c 100644 --- a/src/driver/types.ts +++ b/src/driver/types.ts @@ -35,6 +35,20 @@ export interface RunContext { timeoutSecs: number; devMount?: DevFolderMount; debug?: DebugRunTarget; + /** Volume from `BrowserViewerHandle`, mounted over the Actor container's `/tmp/.X11-unix`. */ + x11SocketVolume?: string; +} + +export interface BrowserViewerTarget { + runId: string; + interactive: boolean; +} + +/** A started sidecar: its VNC address on `apify-local`, and the X-socket volume the Actor container must mount. */ +export interface BrowserViewerHandle { + vncHost: string; + vncPort: number; + x11SocketVolume: string; } /** What `Driver.inspectDebugTarget` reads off a run's resolved build image, for @@ -178,4 +192,11 @@ export interface Driver { /** Reads back the image's `Config.Cmd`/`Config.Entrypoint` and env for * `services/debug-mode.ts: resolveDebugPlan`. Called only when the run's Actor has debug mode on. */ inspectDebugTarget(imageId: string): Promise; + + /** Starts the run's browser-view sidecar (called before the run's own container). Rejects when the + * runtime is not running from its own built image (no sidecar payload on disk). */ + startBrowserViewer(target: BrowserViewerTarget): Promise; + + /** Removes the run's sidecar and volume. Idempotent; never rejects. */ + stopBrowserViewer(runId: string): Promise; } diff --git a/src/index.ts b/src/index.ts index 2563d92..c9cc561 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,6 +5,7 @@ import { createDriver } from './driver/index.js'; import { createApiServer } from './api/server.js'; import { attachEventsWebSocket } from './api/events-ws.js'; import { createConsoleServer } from './console/server.js'; +import { attachBrowserViewWebSocket } from './console/browser-view-ws.js'; import { startLogFlusher } from './services/logs.js'; import { gracefulShutdown } from './shutdown.js'; import { API_PORT, CONSOLE_PORT, DEFAULT_DATA_DIR } from './config.js'; @@ -29,6 +30,7 @@ async function main(): Promise { // `api/events-ws.ts`'s own doc comment for why this attaches here rather than inside `createApiServer` // (Express never sees an `upgrade` event, so this needs the actual `http.Server` `listen()` returned). const eventsWebSocketServer = attachEventsWebSocket(apiServer); + const browserViewWebSocketServer = attachBrowserViewWebSocket(consoleServer); console.log(`actor-runtime API listening on port ${API_PORT}`); @@ -38,7 +40,7 @@ async function main(): Promise { } const shutdown = async () => { - await gracefulShutdown({ apiServer, consoleServer, eventsWebSocketServer }); + await gracefulShutdown({ apiServer, consoleServer, eventsWebSocketServer, browserViewWebSocketServer }); process.exit(0); }; diff --git a/src/services/browser-view.ts b/src/services/browser-view.ts new file mode 100644 index 0000000..2fb3a9e --- /dev/null +++ b/src/services/browser-view.ts @@ -0,0 +1,103 @@ +/** + * Per-Actor browser view: a live mirror of the X display a run's browser draws on. Like `debug-mode.ts`, + * `setBrowserView` is the single validate-and-persist entry point for the API route and the console form. + */ +import type { ActorLocalBrowserView, ActorRecord } from '../storage/entities.js'; +import { getRegistries } from '../storage/registries.js'; +import { CONSOLE_BASE_URL } from '../config.js'; + +const ALLOWED_FIELDS = ['enabled', 'interactive'] as const; + +function isPlainObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** Validation result; `message` is reused verbatim by the API's 400 response and the console's inline error. */ +export type ValidatedBrowserViewBody = + { kind: 'ok'; enabled: boolean; interactive: boolean } | { kind: 'invalid'; message: string }; + +/** Strict-object body validation only; never touches the registry - `setBrowserView` below does that. */ +export function validateBrowserViewBody(body: unknown): ValidatedBrowserViewBody { + if (!isPlainObject(body)) { + return { kind: 'invalid', message: 'Request body must be a JSON object' }; + } + + const unknownKey = Object.keys(body).find((key) => !(ALLOWED_FIELDS as readonly string[]).includes(key)); + if (unknownKey) { + return { + kind: 'invalid', + message: `Unknown field "${unknownKey}" - allowed fields are "enabled", "interactive".`, + }; + } + + if (typeof body.enabled !== 'boolean') { + return { kind: 'invalid', message: '"enabled" must be a boolean' }; + } + + if (body.interactive !== undefined && typeof body.interactive !== 'boolean') { + return { kind: 'invalid', message: '"interactive" must be a boolean' }; + } + + return { kind: 'ok', enabled: body.enabled, interactive: body.interactive ?? false }; +} + +async function writeLocalBrowserView( + actorId: string, + localBrowserView: ActorLocalBrowserView | undefined, +): Promise { + return getRegistries().actors.update(actorId, (current) => (current ? { ...current, localBrowserView } : current)); +} + +export type SetBrowserViewResult = { kind: 'ok'; actor: ActorRecord } | { kind: 'invalid'; message: string }; + +/** Enabling fully replaces `localBrowserView` (an omitted `interactive` resets to `false`). Writes bypass + * `updateActor`, so toggling never bumps `modifiedAt`. */ +export async function setBrowserView(actor: ActorRecord, rawBody: unknown): Promise { + const parsed = validateBrowserViewBody(rawBody); + if (parsed.kind === 'invalid') return parsed; + + if (!parsed.enabled) { + if (!actor.localBrowserView) return { kind: 'ok', actor }; + const updated = await writeLocalBrowserView(actor.id, undefined); + return { kind: 'ok', actor: updated ?? { ...actor, localBrowserView: undefined } }; + } + + const localBrowserView: ActorLocalBrowserView = { interactive: parsed.interactive }; + const updated = await writeLocalBrowserView(actor.id, localBrowserView); + return { kind: 'ok', actor: updated ?? { ...actor, localBrowserView } }; +} + +export interface BrowserViewStatus { + /** `null` when browser view is off (never toggled on, or explicitly cleared) for this Actor. */ + localBrowserView: { interactive: boolean } | null; +} + +/** Read-back for the API response and the console page (no separate `GET`). */ +export function browserViewStatus(actor: Pick): BrowserViewStatus { + if (!actor.localBrowserView) return { localBrowserView: null }; + return { localBrowserView: { interactive: actor.localBrowserView.interactive } }; +} + +export function browserViewPageUrl(runId: string): string { + return `${CONSOLE_BASE_URL}/runs/${encodeURIComponent(runId)}/browser`; +} + +/** The run log's browser-view line, written before the Actor's container is created. */ +export function browserViewLogLine(runId: string, interactive: boolean): string { + return ( + `Browser view: live mirror of this run's display at ${browserViewPageUrl(runId)} ` + + `(${interactive ? 'interactive' : 'view-only'}). Only the display's pixels are read; the Actor and its ` + + `browser are unaffected. A headless browser draws nothing - run it headful ` + + `(Crawlee JS: headless: false; Python: headless=False).\n` + ); +} + +/** No `Cannot start run: ` prefix - `services/runs.ts` adds it. */ +export function describeBrowserViewerStartFailure(actorId: string, error: unknown): string { + const reason = error instanceof Error ? error.message : String(error); + return ( + `browser view is on for this Actor, but its display mirror could not be started: ${reason} ` + + `Clear browser view with \`apify api POST /actor-runtime/browser-view/${actorId} --body '{"enabled": false}'\` ` + + `to run without it.` + ); +} diff --git a/src/services/runs.ts b/src/services/runs.ts index bf87007..9d7e594 100644 --- a/src/services/runs.ts +++ b/src/services/runs.ts @@ -3,7 +3,7 @@ import type { ActorRecord, ActorVersionRecord, BuildRecord, JobStatus, RunRecord import { getRegistries } from '../storage/registries.js'; import { createStorage } from './storages.js'; import { openKeyValueStore } from '../storage/open.js'; -import { DebugPortInUseError, type Driver } from '../driver/types.js'; +import { DebugPortInUseError, type BrowserViewerHandle, type Driver } from '../driver/types.js'; import { appendLog, flushLog, markLogTerminal } from './logs.js'; import { markEventsTerminal, publishAborting, publishPersistState, publishSystemInfo } from './events-channel.js'; import { clearRunRestartState, consumeRunRestart } from './migrations.js'; @@ -16,6 +16,7 @@ import { resolveDebugPlan, type DebugPlan, } from './debug-mode.js'; +import { browserViewLogLine, describeBrowserViewerStartFailure } from './browser-view.js'; import { dedicatedCpusFor } from '../resources.js'; import { CONTAINER_EVENTS_WS_BASE_URL } from '../config.js'; @@ -296,12 +297,32 @@ export async function runInBackground( ? { localDevFolder: actor.localDevFolder, imageWorkingDirectory: build.imageWorkingDirectory } : undefined; + // The sidecar comes up before the Actor's container. Started before the pre-start abort re-check below, + // so an abort landing during this (possibly slow) step is still caught by it. + let browserViewer: BrowserViewerHandle | undefined; + if (actor.localBrowserView) { + const { interactive } = actor.localBrowserView; + try { + browserViewer = await driver.startBrowserViewer({ runId: record.id, interactive }); + } catch (error) { + const message = `Cannot start run: ${describeBrowserViewerStartFailure(actor.id, error)}`; + await failBeforeContainer(record.id, message, message); + return; + } + const { vncHost, vncPort } = browserViewer; + await runs.update(record.id, (current) => + current ? { ...current, localBrowserView: { interactive, vncHost, vncPort } } : current, + ); + appendLog(record.id, browserViewLogLine(record.id, interactive)); + } + // Re-check right before creating the container: an abort issued while the registry/version lookups // above were in flight may have already moved the record to ABORTING. Closing this window is the fix // for the "abort races the pre-start window" finding - without it, an abort landing here would still // let `driver.startRun` create and start a container nothing will ever stop. const preStart = await runs.get(record.id); if (!preStart || preStart.status !== 'RUNNING') { + if (browserViewer) await driver.stopBrowserViewer(record.id); if (preStart?.status === 'ABORTING') { await driver.abortRun(record.id).catch(() => undefined); await transitionJobStatus(runs, record.id, 'ABORTED', { finishedAt: new Date().toISOString() }); @@ -322,6 +343,8 @@ export async function runInBackground( timeoutSecs: remainingTimeoutSecs(record), devMount, debug: debugPlan ? { language: debugPlan.language, port: debugPlan.port } : undefined, + // The sidecar outlives a migration/reboot restart; the new container mounts the same volume. + x11SocketVolume: browserViewer?.x11SocketVolume, }, (chunk) => appendLog(record.id, chunk), (sample) => publishSystemInfo(record.id, sample, record.options), @@ -374,6 +397,7 @@ export async function runInBackground( statusMessage, }); } finally { + if (browserViewer) await driver.stopBrowserViewer(record.id); // A run that ends for real must not leave an armed migration-stop timer behind. clearRunRestartState(record.id); markLogTerminal(record.id); diff --git a/src/shutdown.ts b/src/shutdown.ts index eaa95e5..33150bc 100644 --- a/src/shutdown.ts +++ b/src/shutdown.ts @@ -30,6 +30,8 @@ export interface ShutdownDeps { consoleServer: Server; /** Must be closed before `closeServer(apiServer)` is awaited - see `EventsWebSocketServer.close()`. */ eventsWebSocketServer?: { close(): void }; + /** Same, for the console server. */ + browserViewWebSocketServer?: { close(): void }; } /** @@ -46,12 +48,14 @@ export async function gracefulShutdown({ apiServer, consoleServer, eventsWebSocketServer, + browserViewWebSocketServer, }: ShutdownDeps): Promise { stopLogFlusher(); await flushAllLogs(); await releaseAllBuffersForShutdown(); eventsWebSocketServer?.close(); await closeServer(apiServer); + browserViewWebSocketServer?.close(); await closeServer(consoleServer); await shutdownStorage(); } diff --git a/src/storage/entities.ts b/src/storage/entities.ts index 88f26f7..c118936 100644 --- a/src/storage/entities.ts +++ b/src/storage/entities.ts @@ -64,6 +64,12 @@ export interface ActorLocalDebug { port?: number; } +/** The per-Actor browser-view toggle. Set or cleared only through `services/browser-view.ts: setBrowserView`. */ +export interface ActorLocalBrowserView { + /** `true` delivers the viewer's mouse/keyboard input to the display; `false` is view-only. */ + interactive: boolean; +} + export interface ActorRecord { id: string; userId: string; @@ -83,6 +89,8 @@ export interface ActorRecord { /** The per-Actor debug-mode toggle (`actor-driver.md`'s "Debug mode" section). Absent means off. * Same `modifiedAt`-preserving, never-`/v2`-exposed pattern as `localDevFolder` above. */ localDebug?: ActorLocalDebug; + /** Browser-view toggle; absent means off. Same `modifiedAt`/`/v2` rules as `localDevFolder`. */ + localBrowserView?: ActorLocalBrowserView; } export type JobStatus = 'READY' | 'RUNNING' | 'SUCCEEDED' | 'FAILED' | 'ABORTING' | 'ABORTED' | 'TIMED-OUT'; @@ -164,4 +172,7 @@ export interface RunRecord { * and for a debug run that failed before resolving. Top-level (not nested under `options`) to stay * out of the emulated `/v2` run object automatically. */ localDebug?: { language: DebugLanguage; port: number }; + /** Written once the run's sidecar is up: where its VNC server listens on `apify-local`. Absent when the + * toggle is off or the sidecar failed to start. Never on `/v2`, like `localDebug`. */ + localBrowserView?: { interactive: boolean; vncHost: string; vncPort: number }; } diff --git a/test/e2e/browser-view-py.test.ts b/test/e2e/browser-view-py.test.ts new file mode 100644 index 0000000..50c35eb --- /dev/null +++ b/test/e2e/browser-view-py.test.ts @@ -0,0 +1,10 @@ +import { PYTHON_PLAYWRIGHT_BASE_IMAGE } from './helpers/docker.js'; +import { describeBrowserViewSuite } from './helpers/browser-view-suite.js'; + +describeBrowserViewSuite({ + dir: 'sample_actor_playwright_py', + label: 'Python', + baseImage: PYTHON_PLAYWRIGHT_BASE_IMAGE, + input: (maxRequests) => ({ max_requests_per_crawl: maxRequests }), + withToggleClearedCase: false, +}); diff --git a/test/e2e/browser-view-ts.test.ts b/test/e2e/browser-view-ts.test.ts new file mode 100644 index 0000000..bd318c9 --- /dev/null +++ b/test/e2e/browser-view-ts.test.ts @@ -0,0 +1,10 @@ +import { PLAYWRIGHT_BASE_IMAGE } from './helpers/docker.js'; +import { describeBrowserViewSuite } from './helpers/browser-view-suite.js'; + +describeBrowserViewSuite({ + dir: 'sample_actor_playwright', + label: 'TypeScript', + baseImage: PLAYWRIGHT_BASE_IMAGE, + input: (maxRequests) => ({ maxRequestsPerCrawl: maxRequests }), + withToggleClearedCase: true, +}); diff --git a/test/e2e/helpers/browser-view-suite.ts b/test/e2e/helpers/browser-view-suite.ts new file mode 100644 index 0000000..ad5a614 --- /dev/null +++ b/test/e2e/helpers/browser-view-suite.ts @@ -0,0 +1,278 @@ +/** + * Shared body of the browser-view e2e (`actor-driver.md`'s "Browser view" section) against a real Docker + * daemon: push a Playwright sample Actor, turn browser view on, start a run, and reach its live view the way + * the console's viewer page does. Driven by `apify` commands per `requirements/test.md`'s CLI-only rule, with + * the one exception that rule documents for this test: the viewer websocket is opened directly. + * + * The run is started with `apify api POST actors//runs` rather than `apify call` (which blocks until the + * run ends), so the view can be probed while the run is live. + */ +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import WebSocket from 'ws'; + +import { + buildRuntimeImage, + isDockerAvailable, + pullImage, + startRuntimeContainer, + stopRuntimeContainer, + waitForHttpOk, +} from './docker.js'; +import { + apify, + apifyEnv, + createIsolatedApifyHome, + loginApifyCli, + removeIsolatedApifyHome, + type ApiEnvelope, + type CallResult, + type DatasetInfoResult, + type PushResult, +} from './apify-cli.js'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = join(__dirname, '..', '..', '..'); +const CONSOLE_URL = 'http://localhost:3000'; + +export interface BrowserViewSample { + /** Directory under the repository root. */ + dir: string; + label: string; + baseImage: string; + input: (maxRequests: number) => Record; + /** The "toggle cleared" case proves a runtime property; one sample is enough. */ + withToggleClearedCase: boolean; +} + +interface RunApi { + id: string; + status: string; + statusMessage?: string; +} + +/** The default 1024 MB grants 0.25 core, on which a headful browser is too slow for a tight e2e budget. */ +const RUN_MEMORY_MBYTES = 4096; + +function startRun(actorId: string, input: unknown, env: NodeJS.ProcessEnv): RunApi { + const params = JSON.stringify({ memory: RUN_MEMORY_MBYTES, timeout: 600 }); + const output = apify( + ['api', 'POST', `actors/${actorId}/runs`, '--params', params, '--body', JSON.stringify(input)], + { + cwd: REPO_ROOT, + env, + }, + ); + return (JSON.parse(output) as ApiEnvelope).data; +} + +function getRun(runId: string, env: NodeJS.ProcessEnv): RunApi { + const output = apify(['api', 'GET', `actor-runs/${runId}`], { cwd: REPO_ROOT, env }); + return (JSON.parse(output) as ApiEnvelope).data; +} + +/** Non-streaming log fetch (see `debug-mode.test.ts`'s `currentLog` for why not `apify runs log`). */ +function currentLog(runId: string, env: NodeJS.ProcessEnv): string { + return apify(['api', 'GET', `actor-runs/${runId}/log`], { cwd: REPO_ROOT, env }); +} + +async function waitFor( + check: () => T | undefined | Promise, + timeoutMs: number, + description: string, +): Promise { + const deadline = Date.now() + timeoutMs; + for (;;) { + let result: T | undefined; + try { + result = await check(); + } catch { + result = undefined; + } + if (result !== undefined) return result; + if (Date.now() >= deadline) throw new Error(`Timed out waiting for: ${description}`); + await new Promise((resolve) => setTimeout(resolve, 500)); + } +} + +/** + * Opens the console's viewer websocket for the run and resolves with the first bytes the mirror sends: an + * RFB server's `ProtocolVersion` greeting (`RFB 003.008\n`), which x11vnc sends the moment a client + * connects - proof that the bridge reached a live VNC server mirroring the run's display, before any + * handshake. The console bridge itself keeps re-dialing the sidecar until the Actor's Xvfb is up, so one + * connection attempt is enough; the timeout here just bounds that wait. + */ +function readMirrorGreeting(runId: string, timeoutMs: number): Promise { + return new Promise((resolve, reject) => { + const ws = new WebSocket(`${CONSOLE_URL.replace('http', 'ws')}/runs/${runId}/browser/ws`); + const timer = setTimeout(() => { + ws.terminate(); + reject(new Error('Timed out waiting for the RFB greeting over the viewer websocket')); + }, timeoutMs); + ws.once('message', (data) => { + clearTimeout(timer); + ws.close(); + resolve(Buffer.from(data as Buffer).toString('latin1')); + }); + ws.once('close', (code, reason) => { + clearTimeout(timer); + reject(new Error(`Viewer websocket closed before any data: ${code} ${reason.toString()}`)); + }); + ws.once('error', (error) => { + clearTimeout(timer); + reject(error); + }); + }); +} + +/** One e2e file per sample (`browser-view-ts.test.ts`, `browser-view-py.test.ts`), so CI runs them as separate + * jobs and each pulls only its own base image. */ +export function describeBrowserViewSuite(sample: BrowserViewSample): void { + const CONTAINER_NAME = `actor-runtime-e2e-browser-view-${sample.dir}`; + const IMAGE_TAG = `actor-runtime:e2e-browser-view-${sample.dir}`; + + describe(`per-Actor browser view: live mirror of the ${sample.label} Playwright sample Actor (requires Docker)`, () => { + let isolatedApifyHome: string; + /** Set by the first case and reused by the second: a repeated `apify push` of an unchanged Actor is + * refused by the CLI ("already exists ... newer changes than your local copy"). */ + let pushedActorId: string; + + beforeAll( + async () => { + if (!isDockerAvailable()) { + throw new Error( + 'Docker daemon is not reachable - this e2e case requires one (see requirements/test.md)', + ); + } + + pullImage(sample.baseImage); + buildRuntimeImage(REPO_ROOT, IMAGE_TAG); + startRuntimeContainer(IMAGE_TAG, CONTAINER_NAME); + await waitForHttpOk('http://localhost:3333/v2/users/me?token=x'); + + isolatedApifyHome = createIsolatedApifyHome(); + loginApifyCli(REPO_ROOT, isolatedApifyHome); + }, + 15 * 60 * 1000, + ); + + afterAll(() => { + stopRuntimeContainer(CONTAINER_NAME); + if (isolatedApifyHome) removeIsolatedApifyHome(isolatedApifyHome); + }); + + it( + `${sample.label} sample: push -> toggle on -> run: the log names the viewer URL, the viewer websocket reaches a live RFB server while the run crawls, the console links to the page, and the run finishes with the input-dependent item count`, + async () => { + const env = apifyEnv(isolatedApifyHome); + const actorDir = join(REPO_ROOT, sample.dir); + + const pushOutput = apify(['push', '--json'], { cwd: actorDir, env }); + const push = JSON.parse(pushOutput) as PushResult; + expect(push.build.status).toBe('SUCCEEDED'); + const actorId = push.actor.id; + pushedActorId = actorId; + + const toggle = apify( + ['api', 'POST', `/actor-runtime/browser-view/${actorId}`, '--body', '{"enabled": true}'], + { cwd: REPO_ROOT, env }, + ); + expect(JSON.parse(toggle).data.localBrowserView).toEqual({ interactive: false }); + + // Enough pages to keep the browser busy while the mirror is probed below. + const run = startRun(actorId, sample.input(4), env); + + const log = await waitFor( + () => { + const text = currentLog(run.id, env); + return text.includes('Browser view:') ? text : undefined; + }, + 60_000, + 'the browser-view line to appear in the run log', + ); + expect(log).toContain(`${CONSOLE_URL}/runs/${run.id}/browser`); + expect(log).toContain('view-only'); + + // The mirror: the console's own websocket bridge reaches the sidecar's x11vnc, which greets with the + // RFB protocol version once the Actor's Xvfb display exists. Chrome starting inside a fresh + // container can take a while, hence the generous bound. + const greeting = await readMirrorGreeting(run.id, 120_000); + expect(greeting.startsWith('RFB 003.')).toBe(true); + + // The console's run page links to the viewer, and the viewer page embeds the noVNC client. + const runPage = await fetch(`${CONSOLE_URL}/runs/${run.id}`); + expect(await runPage.text()).toContain(`href="/runs/${run.id}/browser"`); + const viewerPage = await fetch(`${CONSOLE_URL}/runs/${run.id}/browser`); + expect(viewerPage.status).toBe(200); + expect(await viewerPage.text()).toContain('/vendor/novnc/core/rfb.js'); + const client = await fetch(`${CONSOLE_URL}/vendor/novnc/core/rfb.js`); + expect(client.status).toBe(200); + + // Mirroring changed nothing about the crawl itself: the run finishes and the item count tracks input. + const finished = await waitFor( + () => { + const current = getRun(run.id, env); + return ['SUCCEEDED', 'FAILED', 'TIMED-OUT', 'ABORTED'].includes(current.status) + ? current + : undefined; + }, + 8 * 60 * 1000, + 'the browser-view run to finish', + ); + expect(finished.status).toBe('SUCCEEDED'); + const runDetail = JSON.parse( + apify(['api', 'GET', `actor-runs/${run.id}`], { cwd: REPO_ROOT, env }), + ) as ApiEnvelope<{ defaultDatasetId: string }>; + const info = JSON.parse( + apify(['datasets', 'info', runDetail.data.defaultDatasetId, '--json'], { cwd: actorDir, env }), + ) as DatasetInfoResult; + expect(info.itemCount).toBe(4); + + // Once the run is over its mirror is gone: the viewer page says so, and the websocket is refused. + const endedPage = await fetch(`${CONSOLE_URL}/runs/${run.id}/browser`); + expect(await endedPage.text()).toContain('This run has ended'); + await expect(readMirrorGreeting(run.id, 10_000)).rejects.toThrow(/1008/); + }, + 10 * 60 * 1000, + ); + + it.runIf(sample.withToggleClearedCase)( + 'with the toggle cleared, a plain `apify call` of the same Actor runs exactly as before (no mirror, same crawl)', + () => { + const env = apifyEnv(isolatedApifyHome); + const actorDir = join(REPO_ROOT, sample.dir); + expect(pushedActorId).toBeDefined(); + const actorId = pushedActorId; + apify(['api', 'POST', `/actor-runtime/browser-view/${actorId}`, '--body', '{"enabled": false}'], { + cwd: REPO_ROOT, + env, + }); + + const callOutput = apify( + [ + 'call', + '--input', + JSON.stringify(sample.input(2)), + '--memory', + String(RUN_MEMORY_MBYTES), + '--json', + ], + { + cwd: actorDir, + env, + }, + ); + const call = JSON.parse(callOutput) as CallResult; + expect(call.run.status).toBe('SUCCEEDED'); + expect(currentLog(call.run.id, env)).not.toContain('Browser view:'); + + const info = JSON.parse( + apify(['datasets', 'info', call.storage.defaultDatasetId, '--json'], { cwd: actorDir, env }), + ) as DatasetInfoResult; + expect(info.itemCount).toBe(2); + }, + 5 * 60 * 1000, + ); + }); +} diff --git a/test/e2e/helpers/docker.ts b/test/e2e/helpers/docker.ts index 7626c0f..f350024 100644 --- a/test/e2e/helpers/docker.ts +++ b/test/e2e/helpers/docker.ts @@ -23,6 +23,17 @@ export function pullBaseImages(): void { } } +/** `sample_actor_playwright/Dockerfile`'s base image - pulled only by its own e2e file, not by + * `pullBaseImages`, since it is large and no other file builds against it. */ +export const PLAYWRIGHT_BASE_IMAGE = 'apify/actor-node-playwright-chrome:24-1.61.1'; + +/** `sample_actor_playwright_py/Dockerfile`'s base image. */ +export const PYTHON_PLAYWRIGHT_BASE_IMAGE = 'apify/actor-python-playwright:3.14-1.61.0'; + +export function pullImage(image: string): void { + execFileSync('docker', ['pull', image], { stdio: 'inherit' }); +} + export function startRuntimeContainer(tag: string, containerName: string): void { execFileSync( 'docker', @@ -35,10 +46,10 @@ export function startRuntimeContainer(tag: string, containerName: string): void // be bound to 3333/3000 at a time. `package.json`'s `test:e2e` script therefore runs // `vitest run test/e2e --no-file-parallelism`: if a second e2e file's `beforeAll` ever raced // this one, the loser's `docker run` would fail with "port is already allocated" and take that - // file's whole suite down with it. Adding a third e2e file is safe as long as the suite stays - // serialized - do not drop `--no-file-parallelism` (and do not add a `vitest.workspace.ts` or - // per-file config that re-enables parallelism for this directory) without also parameterizing - // these two ports per container. + // file's whole suite down with it. Adding an e2e file is safe as long as the suite stays + // serialized on one daemon - do not drop `--no-file-parallelism` without also parameterizing + // these two ports per container. CI parallelizes by running each file in its own job instead + // (`.github/workflows/ci.yml`). '-p', '3333:3333', '-p', diff --git a/test/integration/actors-builds-runs.test.ts b/test/integration/actors-builds-runs.test.ts index afe4650..0903de8 100644 --- a/test/integration/actors-builds-runs.test.ts +++ b/test/integration/actors-builds-runs.test.ts @@ -27,6 +27,10 @@ function availableDriverWithNoImage(): Driver { async ensureProbeImage() { throw new Error('not used by this stub'); }, + async startBrowserViewer() { + throw new Error('not used by this stub'); + }, + async stopBrowserViewer() {}, async inspectDebugTarget() { throw new Error('not used by this stub'); }, diff --git a/test/integration/api-fallback.test.ts b/test/integration/api-fallback.test.ts index 266408b..d4962bb 100644 --- a/test/integration/api-fallback.test.ts +++ b/test/integration/api-fallback.test.ts @@ -1061,6 +1061,10 @@ describe('api-fallback: dev-folder-* and internal-error types never forward', () async ensureProbeImage() { return 'stub-probe-image:test'; }, + async startBrowserViewer() { + throw new Error('not used by this stub'); + }, + async stopBrowserViewer() {}, async inspectDebugTarget() { throw new Error('not used by this stub'); }, diff --git a/test/integration/browser-view.test.ts b/test/integration/browser-view.test.ts new file mode 100644 index 0000000..30f2160 --- /dev/null +++ b/test/integration/browser-view.test.ts @@ -0,0 +1,624 @@ +/** Integration coverage for the per-Actor browser-view toggle (actor-driver.md: "Browser view"): endpoint + * contract, `/v2` containment, persistence, console form parity, the run lifecycle through a stubbed + * driver (sidecar started before the container, its address persisted, torn down after), the console's + * viewer page and static noVNC client, and the console websocket bridge against a fake RFB server. The + * real x11vnc sidecar is only exercised by `test/e2e/browser-view.test.ts`. */ +import type { AddressInfo } from 'node:net'; +import { createServer, type Server as NetServer, type Socket } from 'node:net'; +import type { Server } from 'node:http'; +import { afterEach, describe, expect, it } from 'vitest'; +import axios from 'axios'; +import WebSocket from 'ws'; + +import { startTestServer, type TestServerHandle } from './helpers/test-server.js'; +import { createConsoleServer } from '../../src/console/server.js'; +import { attachBrowserViewWebSocket, type BrowserViewWebSocketServer } from '../../src/console/browser-view-ws.js'; +import { getRegistries } from '../../src/storage/registries.js'; +import type { BrowserViewerHandle, BrowserViewerTarget, Driver, RunContext } from '../../src/driver/types.js'; + +function post(baseUrl: string, actorId: string, body: unknown, token?: string) { + return axios.post(`${baseUrl}/actor-runtime/browser-view/${actorId}`, body, { + headers: token ? { Authorization: `Bearer ${token}` } : {}, + validateStatus: () => true, + }); +} + +/** An available driver whose `startBrowserViewer` returns a caller-supplied handle (or rejects), recording + * the order of every lifecycle call so the "sidecar up before the container, down after" contract can be + * asserted directly. */ +function viewerCapturingDriver( + handle: BrowserViewerHandle = { vncHost: '127.0.0.1', vncPort: 5901, x11SocketVolume: 'vol-test' }, + startError?: Error, +) { + const events: string[] = []; + const startRunContexts: RunContext[] = []; + const viewerTargets: BrowserViewerTarget[] = []; + const driver: Driver = { + available: true, + async init() {}, + async startBuild(_ctx, onLog) { + onLog('build ok\n'); + return { imageId: 'fake-image:test' }; + }, + async abortBuild() {}, + async startRun(ctx, onLog) { + events.push('startRun'); + startRunContexts.push(ctx); + onLog('done\n'); + return { exitCode: 0, timedOut: false }; + }, + async abortRun() {}, + async reconcileOrphans() {}, + async probeDevFolder() { + throw new Error('not used by this stub'); + }, + async ensureProbeImage() { + throw new Error('not used by this stub'); + }, + async startBrowserViewer(target) { + events.push('startBrowserViewer'); + viewerTargets.push(target); + if (startError) throw startError; + return handle; + }, + async stopBrowserViewer(runId) { + events.push(`stopBrowserViewer:${runId}`); + }, + async inspectDebugTarget() { + return { env: {} }; + }, + }; + return { driver, events, startRunContexts, viewerTargets }; +} + +async function pushAndBuild(server: TestServerHandle, name: string) { + const actor = await server.client.actors().create({ name }); + await server.client + .actor(actor.id) + .versions() + .create({ + versionNumber: '0.0', + buildTag: 'latest', + sourceType: 'SOURCE_FILES' as never, + sourceFiles: [], + } as never); + const build = await server.client.actor(actor.id).build('0.0', { waitForFinish: 5 }); + expect(build.status).toBe('SUCCEEDED'); + return actor; +} + +describe('POST /actor-runtime/browser-view/:actorId', () => { + let server: TestServerHandle; + + afterEach(async () => { + await server.close(); + }); + + it('401s with no auth token', async () => { + server = await startTestServer(); + const res = await post(server.baseUrl, 'whatever-id', { enabled: true }); + expect(res.status).toBe(401); + }); + + it("404s for an actor id that doesn't exist, and for another user's actor", async () => { + server = await startTestServer(); + expect((await post(server.baseUrl, 'made-up-id', { enabled: true }, server.token)).status).toBe(404); + const actor = await server.client.actors().create({ name: 'bv-other-users-actor' }); + expect((await post(server.baseUrl, actor.id, { enabled: true }, 'a-different-token')).status).toBe(404); + }); + + it('{"enabled": true} turns view-only browser view on and echoes it back; interactive is opt-in', async () => { + server = await startTestServer(); + const actor = await server.client.actors().create({ name: 'bv-default-actor' }); + + const res = await post(server.baseUrl, actor.id, { enabled: true }, server.token); + expect(res.status).toBe(200); + expect(res.data).toEqual({ data: { localBrowserView: { interactive: false } } }); + + const interactive = await post(server.baseUrl, actor.id, { enabled: true, interactive: true }, server.token); + expect(interactive.data).toEqual({ data: { localBrowserView: { interactive: true } } }); + expect((await getRegistries().actors.get(actor.id))?.localBrowserView).toEqual({ interactive: true }); + }); + + it('a later {"enabled": true} without interactive fully replaces the prior state (no merge)', async () => { + server = await startTestServer(); + const actor = await server.client.actors().create({ name: 'bv-replace-actor' }); + await post(server.baseUrl, actor.id, { enabled: true, interactive: true }, server.token); + + const res = await post(server.baseUrl, actor.id, { enabled: true }, server.token); + expect(res.data).toEqual({ data: { localBrowserView: { interactive: false } } }); + }); + + it('{"enabled": false} clears the toggle, returning null, whatever else the body names', async () => { + server = await startTestServer(); + const actor = await server.client.actors().create({ name: 'bv-clear-actor' }); + await post(server.baseUrl, actor.id, { enabled: true, interactive: true }, server.token); + + const res = await post(server.baseUrl, actor.id, { enabled: false, interactive: true }, server.token); + expect(res.data).toEqual({ data: { localBrowserView: null } }); + expect((await getRegistries().actors.get(actor.id))?.localBrowserView).toBeUndefined(); + }); + + it('400s for an unknown field, a non-boolean field, or a non-object body, leaving prior state untouched', async () => { + server = await startTestServer(); + const actor = await server.client.actors().create({ name: 'bv-invalid-actor' }); + await post(server.baseUrl, actor.id, { enabled: true, interactive: true }, server.token); + + for (const body of [ + { enabled: true, port: 5900 }, + { enabled: 'yes' }, + { enabled: true, interactive: 'yes' }, + 'true', + ]) { + const res = await post(server.baseUrl, actor.id, body, server.token); + expect(res.status).toBe(400); + expect(res.data.error.type).toBe('invalid-request'); + } + expect((await getRegistries().actors.get(actor.id))?.localBrowserView).toEqual({ interactive: true }); + }); + + it("never bumps the Actor's modifiedAt, on or off, and never appears on the /v2 actor response", async () => { + server = await startTestServer(); + const actor = await server.client.actors().create({ name: 'bv-modifiedat-actor' }); + const before = (await getRegistries().actors.get(actor.id))!.modifiedAt; + + await post(server.baseUrl, actor.id, { enabled: true, interactive: true }, server.token); + await post(server.baseUrl, actor.id, { enabled: false }, server.token); + expect((await getRegistries().actors.get(actor.id))!.modifiedAt).toBe(before); + + await post(server.baseUrl, actor.id, { enabled: true }, server.token); + const fetched = await server.client.actor(actor.id).get(); + expect(JSON.stringify(fetched)).not.toContain('localBrowserView'); + const listed = await server.client.actors().list(); + expect(JSON.stringify(listed)).not.toContain('localBrowserView'); + }); + + it('is reachable through the apify-api-hardcoded /v2 alias too', async () => { + server = await startTestServer(); + const actor = await server.client.actors().create({ name: 'bv-alias-actor' }); + const res = await axios.post( + `${server.baseUrl}/v2/actor-runtime/browser-view/${actor.id}`, + { enabled: true }, + { headers: { Authorization: `Bearer ${server.token}` } }, + ); + expect(res.data).toEqual({ data: { localBrowserView: { interactive: false } } }); + }); +}); + +describe('console: browser-view form on the Actor detail view', () => { + let server: TestServerHandle; + let consoleServer: Server; + let consoleBaseUrl: string; + + async function setUpConsole(): Promise { + server = await startTestServer(); + const app = createConsoleServer({ driver: server.driver }); + consoleServer = await new Promise((resolve) => { + const s = app.listen(0, () => resolve(s)); + }); + consoleBaseUrl = `http://127.0.0.1:${(consoleServer.address() as AddressInfo).port}`; + } + + afterEach(async () => { + await new Promise((resolve) => consoleServer.close(() => resolve())); + await server.close(); + }); + + const formHeaders = { 'Content-Type': 'application/x-www-form-urlencoded' }; + + it('renders "(browser view is off)" and the form for an Actor with no toggle set yet', async () => { + await setUpConsole(); + const actor = await server.client.actors().create({ name: 'bv-console-render-actor' }); + + const detail = await axios.get(`${consoleBaseUrl}/actors/${actor.id}`); + expect(detail.data).toContain('(browser view is off)'); + expect(detail.data).toContain(`
`); + }); + + it('submitting enabled+interactive persists the same state the API would, and shows it', async () => { + await setUpConsole(); + const actor = await server.client.actors().create({ name: 'bv-console-submit-actor' }); + + const submit = await axios.post( + `${consoleBaseUrl}/actors/${actor.id}/browser-view`, + 'enabled=on&interactive=on', + { + headers: formHeaders, + maxRedirects: 0, + validateStatus: () => true, + }, + ); + expect(submit.status).toBe(302); + expect(submit.headers.location).toBe(`/actors/${actor.id}`); + expect((await getRegistries().actors.get(actor.id))?.localBrowserView).toEqual({ interactive: true }); + + const detail = await axios.get(`${consoleBaseUrl}/actors/${actor.id}`); + expect(detail.data).toContain('on, interactive'); + }); + + it('submitting with "enabled" unchecked clears the toggle even with interactive checked', async () => { + await setUpConsole(); + const actor = await server.client.actors().create({ name: 'bv-console-clear-actor' }); + await post(server.baseUrl, actor.id, { enabled: true, interactive: true }, server.token); + + const submit = await axios.post(`${consoleBaseUrl}/actors/${actor.id}/browser-view`, 'interactive=on', { + headers: formHeaders, + maxRedirects: 0, + validateStatus: () => true, + }); + expect(submit.status).toBe(302); + expect((await getRegistries().actors.get(actor.id))?.localBrowserView).toBeUndefined(); + }); + + it('rejects a cross-site form submission with 403 and never touches the toggle', async () => { + await setUpConsole(); + const actor = await server.client.actors().create({ name: 'bv-console-cross-site-actor' }); + + const submit = await axios.post(`${consoleBaseUrl}/actors/${actor.id}/browser-view`, 'enabled=on', { + headers: { ...formHeaders, 'Sec-Fetch-Site': 'cross-site' }, + validateStatus: () => true, + }); + expect(submit.status).toBe(403); + expect((await getRegistries().actors.get(actor.id))?.localBrowserView).toBeUndefined(); + }); + + it('a 404 for a nonexistent Actor id renders Not found, not a 500', async () => { + await setUpConsole(); + const res = await axios.post(`${consoleBaseUrl}/actors/totally-made-up-id/browser-view`, 'enabled=on', { + headers: formHeaders, + validateStatus: () => true, + }); + expect(res.status).toBe(404); + }); +}); + +describe('run lifecycle with browser view (services/runs.ts, through the real startRun path)', () => { + let server: TestServerHandle; + + afterEach(async () => { + await server.close(); + }); + + it('an Actor without the toggle never touches the viewer: no startBrowserViewer, no x11SocketVolume, no localBrowserView on the record', async () => { + const capturing = viewerCapturingDriver(); + server = await startTestServer(capturing.driver); + const actor = await pushAndBuild(server, 'run-no-bv-actor'); + + const run = await server.client.actor(actor.id).start({}, { waitForFinish: 5 }); + expect(run.status).toBe('SUCCEEDED'); + expect(capturing.events).toEqual(['startRun']); + expect(capturing.startRunContexts[0]?.x11SocketVolume).toBeUndefined(); + expect((await getRegistries().runs.get(run.id))?.localBrowserView).toBeUndefined(); + }); + + it('a toggled Actor: sidecar starts before the container and stops after it, the run mounts the volume, the record persists the mirror address, the log carries the viewer URL, and the env is untouched', async () => { + const capturing = viewerCapturingDriver({ vncHost: '172.18.0.5', vncPort: 5900, x11SocketVolume: 'vol-x' }); + server = await startTestServer(capturing.driver); + const actor = await pushAndBuild(server, 'run-bv-actor'); + await post(server.baseUrl, actor.id, { enabled: true, interactive: true }, server.token); + + const run = await server.client.actor(actor.id).start({}, { waitForFinish: 5 }); + expect(run.status).toBe('SUCCEEDED'); + + expect(capturing.events).toEqual(['startBrowserViewer', 'startRun', `stopBrowserViewer:${run.id}`]); + expect(capturing.viewerTargets).toEqual([{ runId: run.id, interactive: true }]); + const ctx = capturing.startRunContexts[0]!; + expect(ctx.x11SocketVolume).toBe('vol-x'); + // The container's environment is exactly an ordinary run's - nothing browser-view-specific leaks in. + expect(Object.keys(ctx.env).some((key) => /BROWSER|VIEW|DISPLAY|HEADLESS/i.test(key))).toBe(false); + + const stored = await getRegistries().runs.get(run.id); + expect(stored?.localBrowserView).toEqual({ interactive: true, vncHost: '172.18.0.5', vncPort: 5900 }); + + const log = await server.client.run(run.id).log().get(); + expect(log).toContain( + `Browser view: live mirror of this run's display at http://localhost:3000/runs/${run.id}/browser`, + ); + expect(log).toContain('interactive'); + + // Never on the emulated /v2 run object. + const fetched = await server.client.run(run.id).get(); + expect(JSON.stringify(fetched)).not.toContain('localBrowserView'); + }); + + it('a sidecar that fails to start fails the run before any container exists, with the classified message naming the clear command', async () => { + const capturing = viewerCapturingDriver( + undefined, + new Error("the runtime's browser-view sidecar payload is missing (ENOENT)."), + ); + server = await startTestServer(capturing.driver); + const actor = await pushAndBuild(server, 'run-bv-fail-actor'); + await post(server.baseUrl, actor.id, { enabled: true }, server.token); + + const run = await server.client.actor(actor.id).start({}, { waitForFinish: 5 }); + expect(run.status).toBe('FAILED'); + expect(run.statusMessage).toContain('Cannot start run: browser view is on for this Actor'); + expect(run.statusMessage).toContain('sidecar payload is missing'); + expect(run.statusMessage).toContain( + `apify api POST /actor-runtime/browser-view/${actor.id} --body '{"enabled": false}'`, + ); + expect(capturing.events).toEqual(['startBrowserViewer']); + expect((await getRegistries().runs.get(run.id))?.localBrowserView).toBeUndefined(); + }); + + it('clearing the toggle between two runs makes the second run byte-identical to a never-toggled run', async () => { + const capturing = viewerCapturingDriver(); + server = await startTestServer(capturing.driver); + const actor = await pushAndBuild(server, 'run-bv-clear-actor'); + await post(server.baseUrl, actor.id, { enabled: true }, server.token); + await server.client.actor(actor.id).start({}, { waitForFinish: 5 }); + await post(server.baseUrl, actor.id, { enabled: false }, server.token); + + const second = await server.client.actor(actor.id).start({}, { waitForFinish: 5 }); + expect(capturing.startRunContexts[1]?.x11SocketVolume).toBeUndefined(); + expect((await getRegistries().runs.get(second.id))?.localBrowserView).toBeUndefined(); + }); +}); + +describe('console: viewer page, run row, noVNC client, and the websocket bridge', () => { + let server: TestServerHandle; + let consoleServer: Server; + let consoleBaseUrl: string; + let bridge: BrowserViewWebSocketServer; + let fakeVnc: NetServer | undefined; + + async function setUpConsole(driver: Driver): Promise { + server = await startTestServer(driver); + const app = createConsoleServer({ driver }); + consoleServer = await new Promise((resolve) => { + const s = app.listen(0, () => resolve(s)); + }); + bridge = attachBrowserViewWebSocket(consoleServer); + consoleBaseUrl = `http://127.0.0.1:${(consoleServer.address() as AddressInfo).port}`; + } + + /** A TCP server standing in for the sidecar's x11vnc: greets like an RFB server and echoes everything + * it receives, so both directions of the bridge can be asserted. */ + function startFakeVnc(): Promise<{ port: number; connections: Socket[] }> { + const connections: Socket[] = []; + fakeVnc = createServer((socket) => { + connections.push(socket); + socket.write('RFB 003.008\n'); + socket.on('data', (chunk) => socket.write(Buffer.concat([Buffer.from('echo:'), chunk]))); + }); + return new Promise((resolve) => { + fakeVnc!.listen(0, '127.0.0.1', () => { + resolve({ port: (fakeVnc!.address() as AddressInfo).port, connections }); + }); + }); + } + + afterEach(async () => { + bridge?.close(); + await new Promise((resolve) => consoleServer.close(() => resolve())); + if (fakeVnc) { + await new Promise((resolve) => fakeVnc!.close(() => resolve())); + fakeVnc = undefined; + } + await server.close(); + }); + + /** Seeds a RUNNING run record with a mirror pointing at `vncPort`, without going through the driver - + * the bridge only ever reads the record. */ + async function seedRunningRunWithView(vncPort: number, interactive = false) { + const actor = await server.client.actors().create({ name: `bv-ws-actor-${vncPort}` }); + const now = new Date().toISOString(); + const runId = `bv-run-${vncPort}-${interactive}`; + await getRegistries().runs.set(runId, { + id: runId, + userId: actor.userId, + actorId: actor.id, + buildId: 'build-x', + buildNumber: '0.0.1', + status: 'RUNNING', + startedAt: now, + defaultDatasetId: 'ds', + defaultKeyValueStoreId: 'kv', + defaultRequestQueueId: 'rq', + options: { memoryMbytes: 256, timeoutSecs: 300 }, + meta: { origin: 'API' }, + localBrowserView: { interactive, vncHost: '127.0.0.1', vncPort }, + }); + return runId; + } + + function openWs(runId: string): Promise { + const ws = new WebSocket(`${consoleBaseUrl.replace('http', 'ws')}/runs/${runId}/browser/ws`); + return new Promise((resolve, reject) => { + ws.once('open', () => resolve(ws)); + ws.once('error', reject); + }); + } + + function nextMessage(ws: WebSocket): Promise { + return new Promise((resolve) => ws.once('message', (data) => resolve(Buffer.from(data as Buffer)))); + } + + function closeCode(ws: WebSocket): Promise<{ code: number; reason: string }> { + return new Promise((resolve) => + ws.once('close', (code, reason) => resolve({ code, reason: reason.toString() })), + ); + } + + it('serves the noVNC ES-module client and its pako dependency from the installed package', async () => { + await setUpConsole(viewerCapturingDriver().driver); + const rfb = await axios.get(`${consoleBaseUrl}/vendor/novnc/core/rfb.js`); + expect(rfb.status).toBe(200); + expect(rfb.headers['content-type']).toMatch(/javascript/); + expect(rfb.data).toContain('class RFB'); + const pako = await axios.get(`${consoleBaseUrl}/vendor/novnc/vendor/pako/lib/zlib/inflate.js`); + expect(pako.status).toBe(200); + }); + + it('the run detail view links to the viewer page for a run with a mirror, and the page embeds the noVNC client for a live run', async () => { + await setUpConsole(viewerCapturingDriver().driver); + const runId = await seedRunningRunWithView(5999, true); + + const detail = await axios.get(`${consoleBaseUrl}/runs/${runId}`); + expect(detail.data).toContain(`href="/runs/${runId}/browser"`); + expect(detail.data).toContain('interactive live mirror'); + + const page = await axios.get(`${consoleBaseUrl}/runs/${runId}/browser`); + expect(page.status).toBe(200); + expect(page.data).toContain("import RFB from '/vendor/novnc/core/rfb.js'"); + expect(page.data).toContain(`const runId = "${runId}";`); + expect(page.data).toContain('const viewOnly = false;'); + expect(page.data).toContain('/browser/ws'); + // The one cause of a connected-but-black view, named on the page itself. + expect(page.data).toContain( + 'A black picture means nothing is drawn on the display: the browser is running headless.', + ); + }); + + it('the viewer page explains itself for a run without a mirror (404) and for an ended run (200, no client)', async () => { + const capturing = viewerCapturingDriver(); + await setUpConsole(capturing.driver); + const actor = await pushAndBuild(server, 'bv-page-ended-actor'); + + const plain = await server.client.actor(actor.id).start({}, { waitForFinish: 5 }); + const noView = await axios.get(`${consoleBaseUrl}/runs/${plain.id}/browser`, { validateStatus: () => true }); + expect(noView.status).toBe(404); + expect(noView.data).toContain('Browser view was not on for this run'); + + await post(server.baseUrl, actor.id, { enabled: true }, server.token); + const viewed = await server.client.actor(actor.id).start({}, { waitForFinish: 5 }); + expect(viewed.status).toBe('SUCCEEDED'); + const ended = await axios.get(`${consoleBaseUrl}/runs/${viewed.id}/browser`); + expect(ended.status).toBe(200); + expect(ended.data).toContain('This run has ended'); + expect(ended.data).not.toContain('rfb.js'); + // The run detail still links to it after the fact. + const detail = await axios.get(`${consoleBaseUrl}/runs/${viewed.id}`); + expect(detail.data).toContain(`href="/runs/${viewed.id}/browser"`); + }); + + it("bridges bytes both ways between the websocket and the mirror's TCP port, and closes 1000 when the TCP side goes away", async () => { + await setUpConsole(viewerCapturingDriver().driver); + const { port, connections } = await startFakeVnc(); + const runId = await seedRunningRunWithView(port); + + const ws = await openWs(runId); + const greeting = await nextMessage(ws); + expect(greeting.toString()).toBe('RFB 003.008\n'); + + ws.send(Buffer.from('RFB 003.008\n')); + const echoed = await nextMessage(ws); + expect(echoed.toString()).toBe('echo:RFB 003.008\n'); + + const closed = closeCode(ws); + for (const connection of connections) connection.destroy(); + expect((await closed).code).toBe(1000); + }); + + it('closes 1008 with a reason for an unknown run, a run without a mirror, and an ended run', async () => { + const capturing = viewerCapturingDriver(); + await setUpConsole(capturing.driver); + + const unknown = await openWs('no-such-run'); + expect((await closeCode(unknown)).reason).toContain('Unknown run id'); + + const actor = await pushAndBuild(server, 'bv-ws-1008-actor'); + const plain = await server.client.actor(actor.id).start({}, { waitForFinish: 5 }); + const noView = await openWs(plain.id); + expect(await closeCode(noView)).toEqual({ code: 1008, reason: `Browser view was not on for run ${plain.id}` }); + + await post(server.baseUrl, actor.id, { enabled: true }, server.token); + const viewed = await server.client.actor(actor.id).start({}, { waitForFinish: 5 }); + const ended = await openWs(viewed.id); + expect(await closeCode(ended)).toEqual({ code: 1008, reason: `Run ${viewed.id} has already ended` }); + }); + + it('keeps retrying the TCP dial while the mirror is not up yet, so a viewer that connects early still gets through', async () => { + await setUpConsole(viewerCapturingDriver().driver); + // Reserve a port, then release it so nothing listens there until the fake server starts below. + const probe = createServer(); + const port: number = await new Promise((resolve) => + probe.listen(0, '127.0.0.1', () => resolve((probe.address() as AddressInfo).port)), + ); + await new Promise((resolve) => probe.close(() => resolve())); + const runId = await seedRunningRunWithView(port); + + const ws = await openWs(runId); + const greeting = nextMessage(ws); + // Only now does the "sidecar" start listening. + await new Promise((resolve) => setTimeout(resolve, 700)); + fakeVnc = createServer((socket) => socket.write('RFB 003.008\n')); + await new Promise((resolve) => fakeVnc!.listen(port, '127.0.0.1', () => resolve())); + + expect((await greeting).toString()).toBe('RFB 003.008\n'); + ws.close(); + }); + + it('a live run of a toggled Actor whose sidecar is still starting: the page renders the client and the bridge waits for the mirror address instead of reporting "not on"', async () => { + await setUpConsole(viewerCapturingDriver().driver); + const { port } = await startFakeVnc(); + const actor = await server.client.actors().create({ name: 'bv-ws-pending-actor' }); + await post(server.baseUrl, actor.id, { enabled: true }, server.token); + const runId = 'bv-run-pending'; + // The window `services/runs.ts` leaves between creating the record and the sidecar coming up. + await getRegistries().runs.set(runId, { + id: runId, + userId: actor.userId, + actorId: actor.id, + buildId: 'build-x', + buildNumber: '0.0.1', + status: 'RUNNING', + startedAt: new Date().toISOString(), + defaultDatasetId: 'ds', + defaultKeyValueStoreId: 'kv', + defaultRequestQueueId: 'rq', + options: { memoryMbytes: 256, timeoutSecs: 300 }, + meta: { origin: 'API' }, + }); + + const page = await axios.get(`${consoleBaseUrl}/runs/${runId}/browser`); + expect(page.status).toBe(200); + expect(page.data).toContain("import RFB from '/vendor/novnc/core/rfb.js'"); + + const ws = await openWs(runId); + const greeting = nextMessage(ws); + await new Promise((resolve) => setTimeout(resolve, 800)); + // The sidecar comes up: the record gets its mirror address, and the waiting bridge dials it. + await getRegistries().runs.update(runId, (current) => + current + ? { ...current, localBrowserView: { interactive: false, vncHost: '127.0.0.1', vncPort: port } } + : current, + ); + expect((await greeting).toString()).toBe('RFB 003.008\n'); + ws.close(); + }); + + it('a live run without a mirror whose Actor has no toggle is refused at once with "not on", and its page is a 404', async () => { + await setUpConsole(viewerCapturingDriver().driver); + const actor = await server.client.actors().create({ name: 'bv-ws-untoggled-actor' }); + const runId = 'bv-run-untoggled'; + await getRegistries().runs.set(runId, { + id: runId, + userId: actor.userId, + actorId: actor.id, + buildId: 'build-x', + buildNumber: '0.0.1', + status: 'RUNNING', + startedAt: new Date().toISOString(), + defaultDatasetId: 'ds', + defaultKeyValueStoreId: 'kv', + defaultRequestQueueId: 'rq', + options: { memoryMbytes: 256, timeoutSecs: 300 }, + meta: { origin: 'API' }, + }); + const ws = await openWs(runId); + expect(await closeCode(ws)).toEqual({ code: 1008, reason: `Browser view is not on for run ${runId}` }); + const page = await axios.get(`${consoleBaseUrl}/runs/${runId}/browser`, { validateStatus: () => true }); + expect(page.status).toBe(404); + }); + + it('an upgrade on any other console path is refused outright', async () => { + await setUpConsole(viewerCapturingDriver().driver); + const ws = new WebSocket(`${consoleBaseUrl.replace('http', 'ws')}/runs/whatever/events`); + const failed = await new Promise((resolve) => { + ws.once('error', () => resolve(true)); + ws.once('open', () => resolve(false)); + }); + expect(failed).toBe(true); + }); +}); diff --git a/test/integration/cli-log-stream-race.test.ts b/test/integration/cli-log-stream-race.test.ts index 094bb0d..8c4c3cd 100644 --- a/test/integration/cli-log-stream-race.test.ts +++ b/test/integration/cli-log-stream-race.test.ts @@ -88,6 +88,10 @@ function timedDriver(resolveAfterMs: number): Driver { async ensureProbeImage() { throw new Error('not used by this stub'); }, + async startBrowserViewer() { + throw new Error('not used by this stub'); + }, + async stopBrowserViewer() {}, async inspectDebugTarget() { throw new Error('not used by this stub'); }, @@ -238,6 +242,10 @@ describe('CLI log-stream race: apify-cli outputJobLog must always settle (regres async ensureProbeImage() { throw new Error('not used by this stub'); }, + async startBrowserViewer() { + throw new Error('not used by this stub'); + }, + async stopBrowserViewer() {}, async inspectDebugTarget() { throw new Error('not used by this stub'); }, @@ -370,6 +378,10 @@ describe('CLI log-stream race: apify-cli outputJobLog must always settle (regres async ensureProbeImage() { throw new Error('not used by this stub'); }, + async startBrowserViewer() { + throw new Error('not used by this stub'); + }, + async stopBrowserViewer() {}, async inspectDebugTarget() { throw new Error('not used by this stub'); }, diff --git a/test/integration/debug-mode.test.ts b/test/integration/debug-mode.test.ts index 6c4f85b..1edaa5f 100644 --- a/test/integration/debug-mode.test.ts +++ b/test/integration/debug-mode.test.ts @@ -58,6 +58,10 @@ function debugCapturingDriver( async ensureProbeImage() { throw new Error('not used by this stub'); }, + async startBrowserViewer() { + throw new Error('not used by this stub'); + }, + async stopBrowserViewer() {}, async inspectDebugTarget() { return target; }, @@ -740,6 +744,10 @@ describe('run-start debug-plan resolution (services/runs.ts, through the real st async ensureProbeImage() { throw new Error('not used by this stub'); }, + async startBrowserViewer() { + throw new Error('not used by this stub'); + }, + async stopBrowserViewer() {}, async inspectDebugTarget() { return { cmd: ['node', 'dist/main.js'], env: {} }; }, diff --git a/test/integration/dev-folder.test.ts b/test/integration/dev-folder.test.ts index b438766..dc29849 100644 --- a/test/integration/dev-folder.test.ts +++ b/test/integration/dev-folder.test.ts @@ -61,6 +61,10 @@ function devFolderDriver( async ensureProbeImage() { return STUB_PROBE_IMAGE_ID; }, + async startBrowserViewer() { + throw new Error('not used by this stub'); + }, + async stopBrowserViewer() {}, async inspectDebugTarget() { throw new Error('not used by this stub'); }, @@ -794,6 +798,10 @@ function devMountCapturingDriver(): { async ensureProbeImage() { throw new Error('not used by this stub'); }, + async startBrowserViewer() { + throw new Error('not used by this stub'); + }, + async stopBrowserViewer() {}, async inspectDebugTarget() { throw new Error('not used by this stub'); }, diff --git a/test/integration/graceful-abort.test.ts b/test/integration/graceful-abort.test.ts index 05481c0..3e33677 100644 --- a/test/integration/graceful-abort.test.ts +++ b/test/integration/graceful-abort.test.ts @@ -100,6 +100,10 @@ function neverStartDriver(): Driver & { abortRunCalls: string[]; abortBuildCalls async ensureProbeImage() { throw new Error('not used by this stub'); }, + async startBrowserViewer() { + throw new Error('not used by this stub'); + }, + async stopBrowserViewer() {}, async inspectDebugTarget() { throw new Error('not used by this stub'); }, diff --git a/test/integration/helpers/test-server.ts b/test/integration/helpers/test-server.ts index 4546398..79bfe17 100644 --- a/test/integration/helpers/test-server.ts +++ b/test/integration/helpers/test-server.ts @@ -45,6 +45,10 @@ export function unavailableDriver(): Driver { async ensureProbeImage() { throw new Error('not used by this stub'); }, + async startBrowserViewer() { + throw new Error('not used by this stub'); + }, + async stopBrowserViewer() {}, async inspectDebugTarget() { throw new Error('not used by this stub'); }, @@ -75,6 +79,10 @@ export function fixedRunOutcomeDriver(outcome: RunOutcome): Driver { async ensureProbeImage() { throw new Error('not used by this stub'); }, + async startBrowserViewer() { + throw new Error('not used by this stub'); + }, + async stopBrowserViewer() {}, async inspectDebugTarget() { throw new Error('not used by this stub'); }, @@ -109,6 +117,10 @@ export function fixedBuildOutcomeDriver( async ensureProbeImage() { throw new Error('not used by this stub'); }, + async startBrowserViewer() { + throw new Error('not used by this stub'); + }, + async stopBrowserViewer() {}, async inspectDebugTarget() { throw new Error('not used by this stub'); }, @@ -172,6 +184,10 @@ export function deferredRunDriver(): DeferredRunDriver { async ensureProbeImage() { throw new Error('not used by this stub'); }, + async startBrowserViewer() { + throw new Error('not used by this stub'); + }, + async stopBrowserViewer() {}, async inspectDebugTarget() { throw new Error('not used by this stub'); }, @@ -228,6 +244,10 @@ export function deferredBuildDriver(): DeferredBuildDriver { async ensureProbeImage() { throw new Error('not used by this stub'); }, + async startBrowserViewer() { + throw new Error('not used by this stub'); + }, + async stopBrowserViewer() {}, async inspectDebugTarget() { throw new Error('not used by this stub'); }, @@ -311,6 +331,10 @@ export function multiRunDriver(): MultiRunDriver { async ensureProbeImage() { throw new Error('not used by this stub'); }, + async startBrowserViewer() { + throw new Error('not used by this stub'); + }, + async stopBrowserViewer() {}, async inspectDebugTarget() { throw new Error('not used by this stub'); }, diff --git a/test/integration/identity-resolution.test.ts b/test/integration/identity-resolution.test.ts index 398b130..d9b5799 100644 --- a/test/integration/identity-resolution.test.ts +++ b/test/integration/identity-resolution.test.ts @@ -205,6 +205,10 @@ function envCapturingDriver(): { driver: Driver; getCapturedEnv: () => Record { async ensureProbeImage() { throw new Error('not used by this stub'); }, + async startBrowserViewer() { + throw new Error('not used by this stub'); + }, + async stopBrowserViewer() {}, async inspectDebugTarget() { throw new Error('not used by this stub'); }, diff --git a/test/integration/run-env-vars.test.ts b/test/integration/run-env-vars.test.ts index 358d74f..a67041e 100644 --- a/test/integration/run-env-vars.test.ts +++ b/test/integration/run-env-vars.test.ts @@ -33,6 +33,10 @@ function envCapturingDriver(): { driver: Driver; getCapturedEnv: () => Record { + it('accepts {"enabled": true}, defaulting interactive to false', () => { + expect(validateBrowserViewBody({ enabled: true })).toEqual({ kind: 'ok', enabled: true, interactive: false }); + }); + + it('accepts an explicit interactive flag', () => { + expect(validateBrowserViewBody({ enabled: true, interactive: true })).toEqual({ + kind: 'ok', + enabled: true, + interactive: true, + }); + }); + + it('accepts {"enabled": false} (a clear), ignoring nothing else - interactive still defaults', () => { + expect(validateBrowserViewBody({ enabled: false })).toEqual({ kind: 'ok', enabled: false, interactive: false }); + }); + + it('rejects a non-object body', () => { + for (const body of [null, 'true', 42, [true], undefined]) { + const result = validateBrowserViewBody(body); + expect(result.kind).toBe('invalid'); + expect((result as { message: string }).message).toContain('JSON object'); + } + }); + + it('rejects an unknown field by name, listing the allowed ones', () => { + const result = validateBrowserViewBody({ enabled: true, interactve: true }); + expect(result).toEqual({ + kind: 'invalid', + message: 'Unknown field "interactve" - allowed fields are "enabled", "interactive".', + }); + }); + + it('rejects a missing or non-boolean enabled', () => { + expect(validateBrowserViewBody({})).toEqual({ kind: 'invalid', message: '"enabled" must be a boolean' }); + expect(validateBrowserViewBody({ enabled: 'yes' })).toEqual({ + kind: 'invalid', + message: '"enabled" must be a boolean', + }); + }); + + it('rejects a non-boolean interactive', () => { + expect(validateBrowserViewBody({ enabled: true, interactive: 1 })).toEqual({ + kind: 'invalid', + message: '"interactive" must be a boolean', + }); + }); +}); + +describe('browserViewStatus', () => { + it('is null when the toggle is off', () => { + expect(browserViewStatus({})).toEqual({ localBrowserView: null }); + expect(browserViewStatus({ localBrowserView: undefined })).toEqual({ localBrowserView: null }); + }); + + it('echoes the stored interactive flag when on', () => { + expect(browserViewStatus({ localBrowserView: { interactive: true } })).toEqual({ + localBrowserView: { interactive: true }, + }); + }); +}); + +describe('user-facing text', () => { + it('the page URL points at the console on its fixed port, run id encoded', () => { + expect(browserViewPageUrl('run a/b')).toBe('http://localhost:3000/runs/run%20a%2Fb/browser'); + }); + + it('the run-log line names the page URL, the mode, and that the Actor is unaffected', () => { + const viewOnly = browserViewLogLine('run-1', false); + expect(viewOnly).toContain('http://localhost:3000/runs/run-1/browser'); + expect(viewOnly).toContain('view-only'); + expect(viewOnly).toContain('unaffected'); + expect(viewOnly).toContain('headless: false'); + expect(viewOnly.endsWith('\n')).toBe(true); + + const interactive = browserViewLogLine('run-1', true); + expect(interactive).toContain('interactive'); + expect(interactive).not.toContain('view-only'); + }); + + it('the start-failure text carries the driver reason and the exact clear command for this Actor', () => { + const text = describeBrowserViewerStartFailure('actor-xyz', new Error('payload is missing.')); + expect(text).toContain('payload is missing.'); + expect(text).toContain(`apify api POST /actor-runtime/browser-view/actor-xyz --body '{"enabled": false}'`); + expect(text.startsWith('Cannot start run')).toBe(false); + }); +}); diff --git a/test/unit/docker-driver-browser-view.test.ts b/test/unit/docker-driver-browser-view.test.ts new file mode 100644 index 0000000..8a0238b --- /dev/null +++ b/test/unit/docker-driver-browser-view.test.ts @@ -0,0 +1,324 @@ +/** `DockerDriver`'s browser-view surface (actor-driver.md: "Browser view"): `startBrowserViewer` / + * `stopBrowserViewer` and `startRun`'s X-socket volume mount. Split out of `docker-driver.test.ts` (one + * file per area, like `docker-driver-debug.test.ts`). No daemon here - a stub `dockerode` records calls. */ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { PassThrough } from 'node:stream'; + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type Docker from 'dockerode'; + +import { DockerDriver } from '../../src/driver/docker-driver.js'; +import { stubDockerForRun } from './helpers/docker-stubs.js'; + +const PAYLOAD_ENV = 'ACTOR_RUNTIME_BROWSER_VIEWER_PAYLOAD_DIR'; + +/** A stub covering exactly what `startBrowserViewer`/`stopBrowserViewer` call. `imagePresent` controls + * whether `getImage(tag).inspect()` finds the imported sidecar image already (a previous process imported + * it) or 404s (first use on this daemon). */ +function stubDockerForViewer(options: { imagePresent?: boolean; ipAddress?: string } = {}) { + const calls: string[] = []; + const inspectImage = vi.fn(async () => { + calls.push('inspectImage'); + if (options.imagePresent) return { Id: 'sha256:present' }; + throw Object.assign(new Error('no such image'), { statusCode: 404 }); + }); + const getImage = vi.fn((_tag: string) => ({ inspect: inspectImage })); + const importImage = vi.fn(async (_file: unknown, _opts: { repo: string; tag: string }) => { + calls.push('importImage'); + return new PassThrough(); + }); + const followProgress = vi.fn((_stream: unknown, onFinished: (err: Error | null, res: unknown[]) => void) => { + onFinished(null, []); + }); + const createVolume = vi.fn(async (_opts: Docker.VolumeCreateOptions) => { + calls.push('createVolume'); + return {}; + }); + const container = { + id: 'viewer-container-id', + start: vi.fn(async () => { + calls.push('start'); + }), + inspect: vi.fn(async () => ({ + NetworkSettings: { + Networks: { 'apify-local': { IPAddress: options.ipAddress ?? '172.18.0.9' } }, + }, + })), + remove: vi.fn(async (_opts?: Record) => { + calls.push('removeContainer'); + }), + }; + const createContainer = vi.fn(async (_opts: Docker.ContainerCreateOptions) => { + calls.push('createContainer'); + return container; + }); + const volumeRemove = vi.fn(async (_opts?: Record) => { + calls.push('removeVolume'); + }); + const getVolume = vi.fn((_name: string) => ({ remove: volumeRemove })); + const docker = { + getImage, + importImage, + createVolume, + createContainer, + getVolume, + modem: { followProgress }, + } as unknown as Docker; + return { docker, calls, getImage, importImage, createVolume, createContainer, container, getVolume, volumeRemove }; +} + +describe('DockerDriver.startBrowserViewer / stopBrowserViewer', () => { + let payloadDir: string; + const originalEnv = process.env[PAYLOAD_ENV]; + + beforeEach(() => { + payloadDir = mkdtempSync(join(tmpdir(), 'actor-runtime-browser-viewer-payload-test-')); + writeFileSync(join(payloadDir, 'rootfs.tar'), 'fake-rootfs-tar'); + writeFileSync(join(payloadDir, 'version.txt'), 'abc123def456\n'); + process.env[PAYLOAD_ENV] = payloadDir; + }); + + afterEach(() => { + rmSync(payloadDir, { recursive: true, force: true }); + if (originalEnv === undefined) delete process.env[PAYLOAD_ENV]; + else process.env[PAYLOAD_ENV] = originalEnv; + }); + + it('imports the bundled rootfs under the payload version tag when the daemon has no such image yet, then creates the tmpfs 1777 volume and starts the labelled sidecar on apify-local', async () => { + const stub = stubDockerForViewer(); + const driver = new DockerDriver(stub.docker); + driver.available = true; + + const handle = await driver.startBrowserViewer({ runId: 'run-1', interactive: false }); + + expect(stub.calls).toEqual(['inspectImage', 'importImage', 'createVolume', 'createContainer', 'start']); + expect(stub.getImage).toHaveBeenCalledWith('actor-runtime/browser-viewer:abc123def456'); + expect(stub.importImage.mock.calls[0]![1]).toEqual({ + repo: 'actor-runtime/browser-viewer', + tag: 'abc123def456', + }); + + const [volumeOptions] = stub.createVolume.mock.calls[0]!; + expect(volumeOptions.Name).toBe('actor-runtime-x11-run-1'); + expect(volumeOptions.Driver).toBe('local'); + expect(volumeOptions.DriverOpts).toEqual({ type: 'tmpfs', device: 'tmpfs', o: 'size=8m,mode=1777' }); + expect(volumeOptions.Labels).toEqual({ 'actor-runtime.runId': 'run-1', 'actor-runtime.browserViewer': 'true' }); + + const [containerOptions] = stub.createContainer.mock.calls[0]!; + expect(containerOptions.Image).toBe('actor-runtime/browser-viewer:abc123def456'); + expect(containerOptions.name).toBe('actor-runtime-browser-viewer-run-1'); + expect(containerOptions.Cmd).toEqual(['/bin/sh', '/apify-browser-viewer.sh']); + expect(containerOptions.Env).toEqual(['APIFY_BROWSER_VIEWER_INTERACTIVE=0', 'APIFY_BROWSER_VIEWER_PORT=5900']); + expect(containerOptions.Labels).toEqual({ + 'actor-runtime.runId': 'run-1', + 'actor-runtime.browserViewer': 'true', + }); + expect(containerOptions.HostConfig?.NetworkMode).toBe('apify-local'); + expect(containerOptions.HostConfig?.Mounts).toEqual([ + { Type: 'volume', Source: 'actor-runtime-x11-run-1', Target: '/tmp/.X11-unix' }, + ]); + // Never published on the host (system.md): no port bindings of any kind. + expect(containerOptions.ExposedPorts).toBeUndefined(); + expect(containerOptions.HostConfig?.PortBindings).toBeUndefined(); + + expect(handle).toEqual({ vncHost: '172.18.0.9', vncPort: 5900, x11SocketVolume: 'actor-runtime-x11-run-1' }); + }); + + it('passes interactive=1 through to the sidecar env when asked for an interactive mirror', async () => { + const stub = stubDockerForViewer({ imagePresent: true }); + const driver = new DockerDriver(stub.docker); + driver.available = true; + + await driver.startBrowserViewer({ runId: 'run-2', interactive: true }); + + expect(stub.createContainer.mock.calls[0]![0].Env).toContain('APIFY_BROWSER_VIEWER_INTERACTIVE=1'); + }); + + it('skips the import when the image is already present, and imports at most once per process for two runs', async () => { + const stub = stubDockerForViewer({ imagePresent: true }); + const driver = new DockerDriver(stub.docker); + driver.available = true; + + await driver.startBrowserViewer({ runId: 'run-a', interactive: false }); + await driver.startBrowserViewer({ runId: 'run-b', interactive: false }); + + expect(stub.importImage).not.toHaveBeenCalled(); + expect(stub.getImage).toHaveBeenCalledTimes(1); + }); + + it('falls back to the sidecar container name as vncHost when the daemon reports no IP on apify-local', async () => { + const stub = stubDockerForViewer({ imagePresent: true, ipAddress: '' }); + const driver = new DockerDriver(stub.docker); + driver.available = true; + + const handle = await driver.startBrowserViewer({ runId: 'run-3', interactive: false }); + + expect(handle.vncHost).toBe('actor-runtime-browser-viewer-run-3'); + }); + + it('fails with a clear message, before any daemon call, when the sidecar payload is missing from disk (runtime running from source)', async () => { + rmSync(join(payloadDir, 'version.txt')); + const stub = stubDockerForViewer(); + const driver = new DockerDriver(stub.docker); + driver.available = true; + + await expect(driver.startBrowserViewer({ runId: 'run-4', interactive: false })).rejects.toThrow( + /browser-view sidecar payload is missing/, + ); + expect(stub.calls).toEqual([]); + }); + + it('fails fast when the driver is unavailable', async () => { + const stub = stubDockerForViewer(); + const driver = new DockerDriver(stub.docker); + + await expect(driver.startBrowserViewer({ runId: 'run-5', interactive: false })).rejects.toThrow( + /Docker is not available/, + ); + expect(stub.calls).toEqual([]); + }); + + it('tears the volume and container down again when the sidecar fails to start, then rethrows', async () => { + const stub = stubDockerForViewer({ imagePresent: true }); + stub.container.start.mockRejectedValueOnce(new Error('daemon said no')); + const driver = new DockerDriver(stub.docker); + driver.available = true; + + await expect(driver.startBrowserViewer({ runId: 'run-6', interactive: false })).rejects.toThrow( + 'daemon said no', + ); + + expect(stub.container.remove).toHaveBeenCalledWith({ force: true }); + expect(stub.getVolume).toHaveBeenCalledWith('actor-runtime-x11-run-6'); + expect(stub.volumeRemove).toHaveBeenCalledWith({ force: true }); + // Nothing is left tracked - a later stop for this run is a no-op. + stub.calls.length = 0; + await driver.stopBrowserViewer('run-6'); + expect(stub.calls).toEqual([]); + }); + + it('stopBrowserViewer force-removes the sidecar container and then the volume; a second call is a no-op', async () => { + const stub = stubDockerForViewer({ imagePresent: true }); + const driver = new DockerDriver(stub.docker); + driver.available = true; + await driver.startBrowserViewer({ runId: 'run-7', interactive: false }); + stub.calls.length = 0; + + await driver.stopBrowserViewer('run-7'); + expect(stub.calls).toEqual(['removeContainer', 'removeVolume']); + expect(stub.container.remove).toHaveBeenCalledWith({ force: true }); + + stub.calls.length = 0; + await driver.stopBrowserViewer('run-7'); + expect(stub.calls).toEqual([]); + }); + + it('stopBrowserViewer retries a "volume is in use" rejection and treats a 404 as already gone, never rejecting', async () => { + const stub = stubDockerForViewer({ imagePresent: true }); + stub.volumeRemove + .mockRejectedValueOnce(Object.assign(new Error('volume is in use'), { statusCode: 409 })) + .mockRejectedValueOnce(Object.assign(new Error('no such volume'), { statusCode: 404 })); + const driver = new DockerDriver(stub.docker); + driver.available = true; + await driver.startBrowserViewer({ runId: 'run-8', interactive: false }); + + await expect(driver.stopBrowserViewer('run-8')).resolves.toBeUndefined(); + expect(stub.volumeRemove).toHaveBeenCalledTimes(2); + }); + + it('stopBrowserViewer for a run that never started a viewer touches the daemon not at all', async () => { + const stub = stubDockerForViewer(); + const driver = new DockerDriver(stub.docker); + driver.available = true; + + await driver.stopBrowserViewer('never-started'); + expect(stub.calls).toEqual([]); + }); +}); + +describe('DockerDriver.startRun - the X-socket volume mount', () => { + it('mounts the given volume at /tmp/.X11-unix and changes nothing else about the container (no env, ports, or command)', async () => { + const stub = stubDockerForRun(); + const driver = new DockerDriver(stub.docker); + driver.available = true; + + const outcomePromise = driver.startRun( + { + runId: 'run-bv-1', + imageId: 'fake-image', + env: { A: 'b' }, + memoryMbytes: 128, + timeoutSecs: 60, + x11SocketVolume: 'actor-runtime-x11-run-bv-1', + }, + () => {}, + ); + await new Promise((resolve) => setImmediate(resolve)); + + const [options] = stub.createContainer.mock.calls[0]!; + expect(options.HostConfig?.Mounts).toEqual([ + { Type: 'volume', Source: 'actor-runtime-x11-run-bv-1', Target: '/tmp/.X11-unix' }, + ]); + expect(options.Env).toEqual(['A=b']); + expect(options.ExposedPorts).toBeUndefined(); + expect(options.HostConfig?.PortBindings).toBeUndefined(); + expect(options).not.toHaveProperty('Cmd'); + expect(options).not.toHaveProperty('Entrypoint'); + + stub.triggerContainerExit(0); + stub.endLogStream(); + await outcomePromise; + }); + + it('composes with a devMount: both mounts land in one Mounts array', async () => { + const stub = stubDockerForRun(); + const driver = new DockerDriver(stub.docker); + driver.available = true; + + const outcomePromise = driver.startRun( + { + runId: 'run-bv-2', + imageId: 'fake-image', + env: {}, + memoryMbytes: 128, + timeoutSecs: 60, + devMount: { localDevFolder: '/host/src', imageWorkingDirectory: '/usr/src/app' }, + x11SocketVolume: 'actor-runtime-x11-run-bv-2', + }, + () => {}, + ); + await new Promise((resolve) => setImmediate(resolve)); + + const [options] = stub.createContainer.mock.calls[0]!; + expect(options.HostConfig?.Mounts).toEqual([ + { Type: 'bind', Source: '/host/src', Target: '/usr/src/app' }, + { Type: 'volume', Source: '', Target: '/usr/src/app/node_modules' }, + { Type: 'volume', Source: 'actor-runtime-x11-run-bv-2', Target: '/tmp/.X11-unix' }, + ]); + + stub.triggerContainerExit(0); + stub.endLogStream(); + await outcomePromise; + }); + + it('a run without a browser view carries no Mounts key at all (regression: byte-identical to before)', async () => { + const stub = stubDockerForRun(); + const driver = new DockerDriver(stub.docker); + driver.available = true; + + const outcomePromise = driver.startRun( + { runId: 'run-bv-3', imageId: 'fake-image', env: {}, memoryMbytes: 128, timeoutSecs: 60 }, + () => {}, + ); + await new Promise((resolve) => setImmediate(resolve)); + + const [options] = stub.createContainer.mock.calls[0]!; + expect(options.HostConfig).not.toHaveProperty('Mounts'); + + stub.triggerContainerExit(0); + stub.endLogStream(); + await outcomePromise; + }); +}); diff --git a/test/unit/docker-driver.test.ts b/test/unit/docker-driver.test.ts index 137e8e9..ff82830 100644 --- a/test/unit/docker-driver.test.ts +++ b/test/unit/docker-driver.test.ts @@ -17,7 +17,10 @@ import { stubDockerForRun } from './helpers/docker-stubs.js'; * for the `PROBE_LABEL` call - exactly the daemon-side semantics a single combined call would get * wrong (it would require one container to carry both keys at once, matching nothing). */ -function stubDocker(containers: Array<{ Id: string; Labels: Record }>) { +function stubDocker( + containers: Array<{ Id: string; Labels: Record }>, + volumes: Array<{ Name: string; Labels: Record }> = [], +) { const removed: string[] = []; const removeCallOptions: Array | undefined> = []; const listContainers = vi.fn(async (options: { all: boolean; filters: string }) => { @@ -32,12 +35,26 @@ function stubDocker(containers: Array<{ Id: string; Labels: Record { + const filters = JSON.parse(options.filters) as { label?: string[] }; + const labelKeys = filters.label ?? []; + return { Volumes: volumes.filter((v) => labelKeys.every((key) => key in v.Labels)) }; + }); + const getVolume = vi.fn((name: string) => ({ + remove: vi.fn(async () => { + removedVolumes.push(name); + }), + })); return { - docker: { listContainers, getContainer } as unknown as Docker, + docker: { listContainers, getContainer, listVolumes, getVolume } as unknown as Docker, listContainers, getContainer, removed, removeCallOptions, + removedVolumes, }; } @@ -57,8 +74,12 @@ describe('DockerDriver.reconcileOrphans', () => { expect(filters.label?.length ?? 0).toBeLessThanOrEqual(1); if (filters.label) allLabelValues.push(...filters.label); } - // Both label keys are still queried, just never together in one call. - expect(allLabelValues.sort()).toEqual(['actor-runtime.devFolderProbe', 'actor-runtime.runId']); + // All three label keys are still queried, just never together in one call. + expect(allLabelValues.sort()).toEqual([ + 'actor-runtime.browserViewer', + 'actor-runtime.devFolderProbe', + 'actor-runtime.runId', + ]); }); it('removes both an orphaned run container and an unrelated leftover probe container from one reconcileOrphans call', async () => { @@ -122,8 +143,28 @@ describe('DockerDriver.reconcileOrphans', () => { // Unlike the "unavailable" case above, an empty `runIds` list must not short-circuit the daemon // calls entirely - a probe container that outlived its own removal (`probeDevFolder`) has no run - // id at all, so it can only ever be found by actually listing. Two calls now, one per label key. - expect(listContainers).toHaveBeenCalledTimes(2); + // id at all, so it can only ever be found by actually listing. Three calls now, one per label key. + expect(listContainers).toHaveBeenCalledTimes(3); + }); + + it('sweeps a leftover browser-view sidecar container and its labelled X-socket volume unconditionally, like a probe (actor-driver.md: "Browser view")', async () => { + const { docker, removed, removedVolumes } = stubDocker( + [ + { + Id: 'viewer-container', + Labels: { 'actor-runtime.runId': 'run-old', 'actor-runtime.browserViewer': 'true' }, + }, + ], + [{ Name: 'actor-runtime-x11-run-old', Labels: { 'actor-runtime.browserViewer': 'true' } }], + ); + const driver = new DockerDriver(docker); + driver.available = true; + + // `run-old` is not in the orphan list (its record is already terminal) - the sidecar goes anyway. + await driver.reconcileOrphans([]); + + expect(removed).toEqual(['viewer-container']); + expect(removedVolumes).toEqual(['actor-runtime-x11-run-old']); }); it('removes a leftover dev-folder probe container even when it matches no orphaned run id', async () => {