diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 56ed81b..6de8b58 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,14 +26,23 @@ jobs: - run: pnpm test e2e: - name: e2e ${{ matrix.file }} (apify-cli + Docker) - runs-on: ubuntu-latest + name: e2e ${{ matrix.file }} (apify-cli + ${{ matrix.engine }}) + # `podman-3.4` is the Podman 3.x leg: Ubuntu 22.04's stock Podman 3.4.4, which the ubuntu-22.04 + # runner image ships preinstalled - the CNI generation, with no usable user-defined networks. + runs-on: ${{ matrix.engine == 'podman-3.4' && 'ubuntu-22.04' || 'ubuntu-latest' }} timeout-minutes: 30 strategy: fail-fast: false - # One job per e2e file. Each file starts its own runtime container on the fixed host ports - # (3333/3000), so files cannot share a daemon; separate runners make them parallel instead. + # One job per e2e file and container engine. Each file starts its own runtime container on + # the fixed host ports (3333/3000), so files cannot share a daemon; separate runners make them + # parallel instead. The `podman` legs drive the runner's preinstalled Podman rootless, as the + # runner user - the same suite, unchanged, against the other supported engine (`test.md`): + # Podman 4 on the default runner, Podman 3.4 on the Ubuntu 22.04 one. matrix: + engine: + - docker + - podman + - podman-3.4 file: - actor-dev-loop - debug-mode @@ -48,9 +57,22 @@ jobs: node-version: 24 cache: pnpm - run: pnpm install --frozen-lockfile - # The suite manages Docker itself against the runner's daemon: it pre-pulls the + # Rootless Podman serves its Docker-compatible API on a socket only when asked to. The suite + # drives Podman through `CONTAINER_CLI=podman` and mounts the socket named by `DOCKER_HOST` + # into the runtime container. + - name: Serve the Podman API socket + if: startsWith(matrix.engine, 'podman') + run: | + podman --version + sock="$RUNNER_TEMP/podman.sock" + nohup podman system service --time=0 "unix://$sock" > "$RUNNER_TEMP/podman-service.log" 2>&1 & + for _ in $(seq 1 30); do [ -S "$sock" ] && break; sleep 1; done + podman --url "unix://$sock" info --format '{{.Host.Arch}}' >/dev/null + echo "CONTAINER_CLI=podman" >> "$GITHUB_ENV" + echo "DOCKER_HOST=unix://$sock" >> "$GITHUB_ENV" + # The suite manages the engine itself against the runner's daemon: it pre-pulls the # Actor base images, builds the runtime image, starts the runtime container with - # the host Docker socket, and drives it with stock apify-cli via npx. + # the host engine's socket, and drives it with stock apify-cli via npx. - run: pnpm exec vitest run test/e2e/${{ matrix.file }}.test.ts # The e2e's runtime container is normally removed by the suite's afterAll; on # failure it is left running, so its server-side view of any failed request @@ -58,6 +80,7 @@ jobs: - name: Dump runtime container logs on failure if: failure() run: | - for c in $(docker ps -aq --filter name=actor-runtime-e2e); do - docker logs --tail 300 "$c" || true + cli=${{ matrix.engine == 'docker' && 'docker' || 'podman' }} + for c in $($cli ps -aq --filter name=actor-runtime-e2e); do + $cli logs --tail 300 "$c" || true done diff --git a/CLAUDE.MD b/CLAUDE.MD index a92d586..9383042 100644 --- a/CLAUDE.MD +++ b/CLAUDE.MD @@ -8,6 +8,9 @@ Local Actor runtime is an Actor development tool for developing, running, and de - Build the docker image `docker build -t actor-runtime .` - Run the container `docker run --rm -p 3333:3333 -p 3000:3000 -v /var/run/docker.sock:/var/run/docker.sock -v "$(pwd)/data:/data" actor-runtime` - `-v "$(pwd)/data:/data"` shared volumes `data` is used to store internal actor runtime data. When exposed it can be directly inspected to determine internal state and storage backend (It is not recommended to manually edit those files. Any edit should be done through http API call). +- Podman (3.4 or newer) works the same way - mount Podman's Docker-compatible API socket where the runtime expects the Docker one: + `sudo systemctl enable --now podman.socket && podman build -t actor-runtime . && mkdir -p data && sudo podman run --rm -p 3333:3333 -p 3000:3000 -v /run/podman/podman.sock:/var/run/docker.sock -v "$(pwd)/data:/data" actor-runtime` + (rootless: mount `$XDG_RUNTIME_DIR/podman/podman.sock` instead and drop `sudo`). Unlike Docker, Podman does not create a missing `data` directory for the mount, hence the `mkdir -p`. See README.md's "Running with Podman" section for the details. ## Work through CLI diff --git a/Dockerfile b/Dockerfile index 9ca0c01..bfca9a9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ # `--platform=$BUILDPLATFORM`: this stage's whole output is architecture-independent (a pure-Python # wheel plus a .py file, tarred), so on a multi-arch build it runs once natively on the builder rather # than once per target under QEMU. Requires BuildKit, which is the default builder in Docker >= 23. -FROM --platform=$BUILDPLATFORM python:3.11-slim AS debugpy-payload +FROM --platform=$BUILDPLATFORM docker.io/library/python:3.11-slim AS debugpy-payload ARG DEBUGPY_VERSION=1.8.21 # Must match `services/debug-mode.ts`'s `PYTHON_DEBUG_PAYLOAD_DIR` - the in-Actor-container path the # tar is extracted to. @@ -32,14 +32,14 @@ RUN tar -cf /payload/debugpy-payload.tar -C /payload/root . # --- Browser-view sidecar: an Alpine rootfs with x11vnc, tarred so the runtime can `docker import` it at # run time without a registry. Not pinned to $BUILDPLATFORM: it runs on the Actor containers' daemon, so it # must be the target architecture's. -FROM alpine:3.21 AS browser-viewer-rootfs +FROM docker.io/library/alpine:3.21 AS browser-viewer-rootfs RUN apk add --no-cache x11vnc RUN mkdir -p /tmp/.X11-unix && chmod 1777 /tmp/.X11-unix COPY docker/browser-viewer.sh /apify-browser-viewer.sh RUN chmod 755 /apify-browser-viewer.sh # Tars the stage above and records its content hash, which the runtime uses as the imported image's tag. -FROM --platform=$BUILDPLATFORM alpine:3.21 AS browser-viewer-payload +FROM --platform=$BUILDPLATFORM docker.io/library/alpine:3.21 AS browser-viewer-payload COPY --from=browser-viewer-rootfs / /rootfs RUN mkdir -p /payload \ && tar -cf /payload/rootfs.tar -C /rootfs . \ @@ -48,7 +48,7 @@ RUN mkdir -p /payload \ # Also architecture-independent: this stage only runs `tsc`, and the `dist/` it hands to the final # stage is plain JavaScript. The final stage does its own `pnpm install --prod`, so the target # architecture's native bindings still come from a native (emulated) install there. -FROM --platform=$BUILDPLATFORM node:24-bookworm-slim AS builder +FROM --platform=$BUILDPLATFORM docker.io/library/node:24-bookworm-slim AS builder WORKDIR /usr/src/app @@ -62,7 +62,7 @@ COPY tsconfig.json ./ COPY src ./src RUN pnpm run build -FROM node:24-bookworm-slim +FROM docker.io/library/node:24-bookworm-slim WORKDIR /usr/src/app @@ -84,8 +84,9 @@ COPY --from=debugpy-payload /payload/debugpy-version.txt /opt/apify-debug-payloa COPY --from=browser-viewer-payload /payload/rootfs.tar /opt/apify-browser-viewer/rootfs.tar COPY --from=browser-viewer-payload /payload/version.txt /opt/apify-browser-viewer/version.txt -# The runtime talks to the host Docker socket via dockerode (no docker CLI needed in-image) and -# persists all storages under /data - mount both when running the container. +# The runtime talks to the host's Docker-Engine-API socket via dockerode (no docker CLI needed in-image; +# Podman's Docker-compatible socket works the same way) and persists all storages under /data - mount +# both when running the container. VOLUME ["/data"] ENV ACTOR_RUNTIME_DATA_DIR=/data diff --git a/README.md b/README.md index a9dbdcc..2da0de9 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ See `requirements/*.md` for the full behavioural spec (`system.md`, `api.md`, ```bash docker build -t actor-runtime . +mkdir -p data docker run --rm -p 3333:3333 -p 3000:3000 \ -v /var/run/docker.sock:/var/run/docker.sock \ -v "$(pwd)/data:/data" \ @@ -40,6 +41,64 @@ the real platform is reachable, the runtime also adopts that account's real user the first time it sees the token; fully offline (or with any other non-empty token) it just keeps using the single local user, with no error either way - see `requirements/cli.md`'s User bootstrap section. +## Running with Podman instead of Docker + +The runtime talks to the container engine only through its Docker-compatible API socket, and Podman +serves that same API. Everything works the same on Docker and Podman, rootful or rootless; the only +difference is which socket you mount. + +```bash +sudo systemctl enable --now podman.socket # one-time: serve Podman's API socket + +podman build -t actor-runtime . +mkdir -p data +sudo podman run --rm -p 3333:3333 -p 3000:3000 \ + -v /run/podman/podman.sock:/var/run/docker.sock \ + -v "$(pwd)/data:/data" \ + actor-runtime +``` + +Rootless Podman serves the socket at `$XDG_RUNTIME_DIR/podman/podman.sock` instead +(`systemctl --user enable --now podman.socket`); mount that path and drop the `sudo`. Rootless Docker +works the same way with its `$XDG_RUNTIME_DIR/docker.sock`. The socket can also be mounted at any other +path together with `-e DOCKER_HOST=unix:///that/path`. + +```bash +mkdir -p data +podman run --rm -p 3333:3333 -p 3000:3000 \ + -v "$XDG_RUNTIME_DIR/podman/podman.sock:/var/run/docker.sock" \ + -v "$(pwd)/data:/data" \ + actor-runtime +``` + +Good to know: + +- Podman 3.4 (Ubuntu 22.04's stock package) and newer work. On Podman 4 and newer, Actors run on the + runtime's own `apify-local` network; on Podman 3.x they run on the engine's default network instead + (its user-defined networks are unreliable: Ubuntu 22.04's CNI plugins reject the config Podman writes), + and the runtime says so at startup. Whenever the runtime's own container is not on `apify-local` + (Podman 3.x, or rootless Podman, which refuses to attach it), Actors reach the API through the published + port 3333, so keep `-p 3333:3333` published on all interfaces. Optionally, on Podman 4 and newer, create + the network first and add `--network apify-local` to `podman run` for the direct route. +- Podman does not create a missing host directory for a bind mount (Docker does), hence the + `mkdir -p data` before `podman run`. `apify runtime start` creates its data directory itself. +- Actors run on the engine whose socket you mount, so a dev folder registered for the bind-mount dev + loop below is a path on the machine that engine runs on (inside the VM for `podman machine`), and + under a rootless engine it must be readable by that user. +- A short image name in an Actor's `FROM` line (`apify/actor-node:20`, `python:3.11`) means Docker Hub, + as on the platform. The runtime qualifies it to `docker.io/...` before building, so Podman resolves it + without any `unqualified-search-registries` entry in `registries.conf`. The build log shows the + substitution. +- A rootless engine can only enforce the per-run limits whose cgroup controllers are delegated to your + user: on cgroups v1 none are, and Ubuntu 22.04 delegates `memory` and `pids` but not `cpu`. The runtime + asks Podman which controllers it has, leaves out the limits it cannot apply, and says so at startup; + runs still start. (To get CPU limits under rootless Podman on Ubuntu 22.04, delegate the controller: + `sudo mkdir -p /etc/systemd/system/user@.service.d && printf '[Service]\nDelegate=cpu cpuset io memory pids\n' | sudo tee /etc/systemd/system/user@.service.d/delegate.conf && sudo systemctl daemon-reload`, then log out and in.) +- If you restart a hand-started `podman system service`, the socket file mounted into the runtime goes + stale; restart the runtime container too. The `podman.socket` unit does not have this problem. +- `podman images` lists the images the runtime builds as `actor-runtime/:` under the + registry prefix Podman adds itself (`docker.io/` or `localhost/`, depending on the version). + ## Rapid dev loop: bind-mounting your local source (no rebuild per edit) After the one push+build above, register your Actor's local source folder so every future run picks up @@ -50,10 +109,9 @@ apify api POST /actor-runtime/dev-folder/ --body '"/abs/path/to/sample_ ``` `` is the id `apify push --json` printed (`.actor.id`); the path must be absolute and must -already exist on the **host** - the runtime verifies this by actually trying to mount it, and rejects -the call with a clear error if the Actor has no build tagged `latest` yet (a stock `apify push` always -tags its build `latest`, so this is normally just "build at least once first") or the path can't be -confirmed. +already exist on the **host** - the runtime checks this and rejects the call with a clear error if the +path can't be confirmed. The check runs again at every run start, so a folder deleted after +registration fails the run instead of running against an empty directory. The same thing is also a single-field form on the Actor's page in the console (`http://localhost:3000`). From then on: @@ -66,8 +124,10 @@ apify call --input '{"maxPages":3}' # picks up the new dist/, no rebuild Node doesn't hot-reload a running process, so a local recompile is picked up by the **next** run's container start, not by any run already in progress. `node_modules` inside the container still comes -from the built image - an anonymous volume preserves it underneath the bind mount - so a new dependency -in `package.json` still needs a real `apify push`/build; only source edits skip it. Clear the +from the built image - a per-run volume preserves it underneath the bind mount - so a new dependency +in `package.json` still needs a real `apify push`/build; only source edits skip it. An entrypoint script +the image keeps in its working directory (Apify's Playwright images start through an Xvfb wrapper there) +stays available too, unless your folder carries its own copy. Clear the registration with an empty body (`--body '""'`) to go back to running purely from the built image. Full mechanics: `requirements/actor-driver.md`'s "Bind mount volumes with Actor source code"; endpoint/console details: `requirements/api.md`'s `/actor-runtime/*` section and @@ -161,7 +221,7 @@ added by hand. pnpm install pnpm run build # tsc pnpm test # unit + integration (no Docker needed) -pnpm run test:e2e # full CLI-driven dev loop against a built image (requires Docker; the browser-view case pulls the ~2 GB Playwright base image) +pnpm run test:e2e # full CLI-driven dev loop against a built image (requires Docker, or Podman with CONTAINER_CLI=podman; the browser-view case pulls the ~2 GB Playwright base image) pnpm run dev # run the server directly against ./data with tsx ``` diff --git a/docker/browser-viewer.sh b/docker/browser-viewer.sh index 7ebb2ac..b6adf76 100755 --- a/docker/browser-viewer.sh +++ b/docker/browser-viewer.sh @@ -5,6 +5,10 @@ # Env names must match `src/driver/docker-driver.ts`. SOCKET_DIR=/tmp/.X11-unix +# The socket directory is a shared volume the Actor's unprivileged Xvfb must be able to write to. This +# sidecar mounts it first and runs as root, so it sets the mode itself rather than trusting the engine +# to copy it from the image. +chmod 1777 "$SOCKET_DIR" 2>/dev/null || true PORT="${APIFY_BROWSER_VIEWER_PORT:-5900}" if [ "$APIFY_BROWSER_VIEWER_INTERACTIVE" = "1" ]; then INPUT_FLAG="" diff --git a/requirements/actor-driver.md b/requirements/actor-driver.md index 09a68fd..2406247 100644 --- a/requirements/actor-driver.md +++ b/requirements/actor-driver.md @@ -43,6 +43,7 @@ surfaces. - Registration validates that the submitted value is an absolute POSIX path and that the path exists **on the host** and is a directory. + - Validation never creates anything on the host and follows symlinks. - Submitting the **empty string clears the registration** and skips validation. - Every non-success outcome is classified: unable to verify at all (e.g. Docker unreachable) reports "could not verify" - never "does not exist"; a path confirmed missing reports "path @@ -50,6 +51,8 @@ else unverifiable reports a generic "could not verify". - **Registration has no build-first precondition** - it requires no build of the Actor to exist, succeeded or otherwise. +- An image that starts through a file inside its working directory still starts under the mount: the + dev folder's copy of that file is used when it has one, the image's own copy otherwise. - The working directory the mount covers is recorded **per build**, never on the Actor (`storage.md`); the mount a run applies always uses the one from _that run's own resolved build_, never any other build the Actor happens to have. @@ -59,7 +62,8 @@ - The registration status the console and API report is the registered folder alone - never that a mount "will apply", since that depends on which build a given run resolves. - If the registered folder has since been deleted, moved, or made unreadable, the run must **fail - visibly** - never silently mount an empty directory in its place. + visibly** - never silently mount an empty directory in its place. The status message names the folder, + what is wrong with it, and how to clear the registration. - The Actor image's own installed dependencies (e.g. `node_modules`) must remain available to the Actor despite the mount covering the whole working directory. - **Registering or clearing a dev folder never bumps the Actor's `modifiedAt`.** @@ -141,9 +145,14 @@ start`, ...) is refused by name, naming both the `CMD` fix and how to clear debu # Networking -- On startup, the runtime ensures a Docker network `apify-local` exists and joins it under the fixed - DNS alias `apify-api`. Every Actor container is started on that network, so it can reach the - runtime's API at `http://apify-api:3333` regardless of the host's own networking. +- Every Actor container reaches the runtime's API at `http://apify-api:3333`, whatever the host's own + networking, whichever supported engine runs the containers, and however the runtime itself was started + (as a container or not). The runtime provides the `apify-local` network with the DNS alias `apify-api` + for this; when it has to reach the same goal another way, it says so at startup. +- A per-run resource limit the engine cannot enforce for the current user is left out rather than + failing the run; the runtime says so at startup. +- A run the engine refuses to start fails with the engine's reason in both the run's status message and + its log. # Actor run diff --git a/requirements/system.md b/requirements/system.md index 097267b..f9300f7 100644 --- a/requirements/system.md +++ b/requirements/system.md @@ -31,20 +31,26 @@ overridable) - the runtime's own two ports above are unaffected, and no port is published for an Actor that never turned debug mode on. - Browser view (`actor-driver.md`) publishes no port on the host; the view is served on the console's port 3000. -- Required `docker run` flags: mount the host Docker socket read-write +- Required `docker run` flags: mount the host's Docker-Engine-API socket read-write (`-v /var/run/docker.sock:/var/run/docker.sock`) so the runtime can build and run Actor containers, and mount a persistent data directory (`-v :/data`, e.g. `-v "$(pwd)/data:/data"`) so - storages survive a restart and are easy to inspect from the host. Publish both fixed ports + storages survive a restart and are easy to inspect from the host; the directory must exist before the + runtime starts. Publish both fixed ports (`-p 3333:3333 -p 3000:3000`). The canonical start command is: ```bash docker build -t actor-runtime . + mkdir -p data docker run --rm -p 3333:3333 -p 3000:3000 \ -v /var/run/docker.sock:/var/run/docker.sock \ -v "$(pwd)/data:/data" \ actor-runtime ``` +- **Docker and Podman are equally supported**, rootful or rootless: everything the system offers works + the same on either engine. The user picks the engine by mounting its Docker-compatible API socket in + place of the Docker one (e.g. `-v /run/podman/podman.sock:/var/run/docker.sock`). + - Optionally set `APIFY_PROXY_PASSWORD` in the runtime's own environment to have it forwarded into every Actor container (see `actor-driver.md`). diff --git a/requirements/test.md b/requirements/test.md index 70bdc4d..8a4cb50 100644 --- a/requirements/test.md +++ b/requirements/test.md @@ -20,6 +20,9 @@ - For asserting the test results, the tests must inspect the return values of the Apify cli commands. - The e2e suite requires a reachable Docker daemon (it builds and runs real Actor containers) and detects its absence, failing in such case. +- The same suite must pass unchanged against Podman, rootful or rootless, selected through + `CONTAINER_CLI` and `DOCKER_HOST`. CI runs every e2e file, browser view included, against Docker and + against both the oldest and the newest supported Podman. - The sample Actors crawl a live site (`https://crawlee.dev/` by default), so the e2e suite also requires outbound network access from Actor containers. This is separate from the runtime's own offline capability (see the offline notes in `system.md` and `cli.md`). - CI must pre-pull the sample Actors' base images (`apify/actor-node:24`, `apify/actor-python:3.13`, and `python:3.11-slim` for `sample_actor_crawler`) before running the e2e suite, so push/call assertion timing is not dominated by first-time image pulls. The browser-view e2e test pre-pulls the two Playwright samples' base images itself. diff --git a/src/driver/docker-driver.ts b/src/driver/docker-driver.ts index 0e7f7c7..d96c657 100644 --- a/src/driver/docker-driver.ts +++ b/src/driver/docker-driver.ts @@ -22,17 +22,30 @@ * so `abortBuild` can call `.abort()` on the live one. Runs are cancelled the same way as before - * `container.stop()` - since there is no HTTP request to abort there. * - * Unverified in this sandbox: there is no Docker socket here, so `init()` always finds - * `available: false` and every build/run fails fast with a clear status message instead of hanging. - * The rest of the runtime (storages, actors-as-records, console) is unaffected. + * Engine neutrality: everything here goes through the Docker Engine API, which Podman also serves + * (`podman system service` / the `podman.socket` unit), so the same driver runs Actors on Docker and on + * Podman 3.4 or newer, rootful or rootless. Where the engines genuinely differ, the difference is handled + * at the code that meets it: bind-mount sources Podman would auto-create (`probeDevFolder`, + * `assertDevFolderStillPresent`); Podman's `system_cpu_usage` scale (`cpuUsageSnapshotOf`); how Actor + * containers reach this API when this container is not on `apify-local` (`selfAttachToNetwork`, + * `chooseDefaultNetworkRoute` - Podman 3.x never uses the network at all); resource limits a rootless + * engine cannot apply (`detectResourceLimitSupport`); and the image names and volume options Podman 3.x + * needs (`LOCAL_IMAGE_PREFIX`, `ensureBrowserViewerImage`, `buildDevMounts`, `startBrowserViewer`). + * Without any reachable socket, `init()` finds `available: false` and every build/run fails fast with a + * clear status message instead of hanging; the rest of the runtime (storages, actors-as-records, + * console) is unaffected. */ import { PassThrough } from 'node:stream'; import { createReadStream } from 'node:fs'; import { readFile } from 'node:fs/promises'; +import * as os from 'node:os'; +import { createServer } from 'node:net'; +import * as path from 'node:path'; import Docker from 'dockerode'; import * as tar from 'tar-stream'; import { + API_PORT, CONTAINER_API_ALIAS, browserViewerRootfsTarPath, browserViewerVersionFilePath, @@ -60,6 +73,203 @@ import { } from './types.js'; const NETWORK_NAME = 'apify-local'; + +/** + * Prefix for the images this driver builds or imports for its own use and later looks up BY NAME (the + * browser-view sidecar, the dev-folder probe). A bare `actor-runtime/...` is a short name: Docker and + * Podman 4 resolve it to the local image, but Podman 3.x resolves short names only through its search + * registries and reports the locally stored `localhost/actor-runtime/...` as "image not known". Naming + * the image `localhost/...` outright is what every engine stores it as anyway. Actor images need no + * prefix: they are always referenced by image id. + */ +const LOCAL_IMAGE_PREFIX = 'localhost/'; + +/** The names an engine gives the host in every container's hosts file: Podman (3.3+) the first, Docker + * Desktop the second (Docker Engine adds neither). */ +const ENGINE_HOST_NAMES = ['host.containers.internal', 'host.docker.internal']; + +/** + * The address Actor containers can reach the host at, read from this process's own hosts file (this + * container got it from the same engine, on the same default network, as every Actor container will). + * Falls back to the engine keyword `host-gateway` (Docker 20.10+, Podman 4.1+) when the file has no such + * entry - on Docker Engine, or when this process runs outside a container. + */ +export async function hostAddressSeenFromContainers(hostsFile: string): Promise { + return (await engineHostEntry(hostsFile)) ?? 'host-gateway'; +} + +async function engineHostEntry(hostsFile: string): Promise { + const content = await readFile(hostsFile, 'utf8').catch(() => ''); + for (const line of content.split('\n')) { + const fields = line.split('#')[0]!.trim().split(/\s+/); + if (fields.length >= 2 && fields.slice(1).some((name) => ENGINE_HOST_NAMES.includes(name))) return fields[0]!; + } + return undefined; +} + +/** What this process can learn about its own container's network from the inside, with no engine call. */ +export interface OwnNetworkView { + /** Address the engine hands containers for the host (`hostAddressSeenFromContainers`), or undefined for `host-gateway`. */ + hostAddress: string | undefined; + /** This container's default gateway, from `/proc/net/route`. */ + gateway: string | undefined; + /** This container's first non-loopback IPv4 address and the interface it sits on. */ + own: { address: string; iface: string } | undefined; +} + +/** How an Actor container on the engine's DEFAULT network reaches this API (`actorsOnDefaultNetwork`). */ +export interface DefaultNetworkRoute { + /** `ExtraHosts` entry for `apify-api`. */ + extraHost: string; + /** Engine network mode the Actor container needs for that entry to work, if any. */ + networkMode?: string; +} + +const SLIRP4NETNS_GATEWAY = '10.0.2.2'; +const SLIRP4NETNS_INTERFACE = 'tap0'; +/** Podman's user-mode network with the host's loopback reachable at the gateway - Actors under rootless + * Podman before 4.0 need it, where the engine's own host entry IS that gateway and is otherwise blocked. */ +const SLIRP4NETNS_HOST_LOOPBACK_MODE = 'slirp4netns:allow_host_loopback=true'; + +/** + * Picks the route for Actor containers on the engine's default network. The engine's own host entry is + * the default. Two cases where that entry is this container's gateway need more: under slirp4netns the + * gateway only reaches the host's loopback when the Actor is started with `allow_host_loopback` + * (rootless Podman 3.x points `host.containers.internal` there); on a rootful bridge every Actor shares + * this container's bridge, so this container's own address is a direct route with no port-forward hop. + */ +export function chooseDefaultNetworkRoute(view: OwnNetworkView): DefaultNetworkRoute { + const { hostAddress, gateway, own } = view; + if (hostAddress && gateway && hostAddress === gateway) { + if (own?.iface === SLIRP4NETNS_INTERFACE || gateway === SLIRP4NETNS_GATEWAY) { + return { extraHost: `${CONTAINER_API_ALIAS}:${gateway}`, networkMode: SLIRP4NETNS_HOST_LOOPBACK_MODE }; + } + if (own) return { extraHost: `${CONTAINER_API_ALIAS}:${own.address}` }; + } + return { extraHost: `${CONTAINER_API_ALIAS}:${hostAddress ?? 'host-gateway'}` }; +} + +/** The default gateway from a Linux `/proc/net/route` (little-endian hex), or undefined. */ +export function defaultGatewayFromRouteTable(routeTable: string): string | undefined { + for (const line of routeTable.split('\n').slice(1)) { + const [, destination, gateway] = line.trim().split(/\s+/); + if (destination !== '00000000' || !gateway || gateway.length !== 8) continue; + const bytes = gateway.match(/../g)!.map((hex) => parseInt(hex, 16)); + return bytes.reverse().join('.'); + } + return undefined; +} + +function firstNonLoopbackIpv4(interfaces: NodeJS.Dict): OwnNetworkView['own'] { + for (const [iface, infos] of Object.entries(interfaces)) { + const info = infos?.find((i) => i.family === 'IPv4' && !i.internal); + if (info) return { address: info.address, iface }; + } + return undefined; +} + +/** Which per-container resource limits the engine can actually apply for the user it runs as. */ +export interface ResourceLimitSupport { + cpu: boolean; + memory: boolean; +} + +const ALL_LIMITS_SUPPORTED: ResourceLimitSupport = { cpu: true, memory: true }; + +/** + * Docker applies (or silently drops) whatever limits it is handed. Rootless Podman on cgroups v2 instead + * refuses to START a container whose limit needs a cgroup controller systemd did not delegate to the + * user - Ubuntu 22.04 delegates `memory` and `pids` but not `cpu`, so every run with a CPU quota died + * at start. Podman's own (non-Docker) info endpoint lists the controllers it can use; a limit whose + * controller is missing is left out, and `init` says so once. Anything unexpected keeps every limit. + */ +/** The engine behind the Docker API, as far as this driver needs to know: Podman's major version, or + * undefined for Docker (and for anything that does not identify itself). */ +export async function podmanMajorVersion(docker: Docker): Promise { + try { + const version = (await docker.version()) as { Components?: Array<{ Name: string; Version?: string }> }; + const podman = version.Components?.find((component) => component.Name === 'Podman Engine'); + const major = podman?.Version ? Number.parseInt(podman.Version, 10) : Number.NaN; + return Number.isNaN(major) ? undefined : major; + } catch { + return undefined; + } +} + +export async function detectResourceLimitSupport(docker: Docker): Promise { + try { + const version = (await docker.version()) as { Components?: Array<{ Name: string }> }; + if (!version.Components?.some((component) => component.Name === 'Podman Engine')) return ALL_LIMITS_SUPPORTED; + const info = await new Promise<{ host?: { cgroupControllers?: unknown } }>((resolve, reject) => { + docker.modem.dial( + { path: '/v4.0.0/libpod/info', method: 'GET', statusCodes: { 200: true, 500: 'server error' } }, + (error: Error | null, data: unknown) => + error ? reject(error) : resolve(data as { host?: { cgroupControllers?: unknown } }), + ); + }); + const controllers = info.host?.cgroupControllers; + if (!Array.isArray(controllers)) return ALL_LIMITS_SUPPORTED; + return { cpu: controllers.includes('cpu'), memory: controllers.includes('memory') }; + } catch { + return ALL_LIMITS_SUPPORTED; + } +} + +function devNodeModulesVolumeName(runId: string): string { + return `${DEV_NODE_MODULES_VOLUME_PREFIX}${runId}`; +} + +/** A command token the engine resolves against the working directory rather than `PATH`: `./x.sh`, + * `bin/x` - relative, with a slash. A bare `x.sh` goes through `PATH`, an absolute path is unaffected + * by what is mounted over the working directory. */ +function isWorkingDirectoryRelative(token: string): boolean { + return !token.startsWith('/') && token.includes('/'); +} + +function readStream(stream: NodeJS.ReadableStream): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + stream.on('data', (chunk: Buffer) => chunks.push(chunk)); + stream.once('error', reject); + stream.once('end', () => resolve(Buffer.concat(chunks))); + }); +} + +/** Re-packs the single-file archive `getArchive` returns so its entries land under `directory` when + * extracted at `/`, with an explicit directory entry so no engine has to invent the parent. */ +async function repackUnderDirectory(archive: Buffer, directory: string): Promise { + const dir = directory.replace(/^\/+/, ''); + const pack = tar.pack(); + const extract = tar.extract(); + const packed = readStream(pack); + pack.entry({ name: dir, type: 'directory', mode: 0o755 }); + await new Promise((resolve, reject) => { + extract.on('entry', (header, content, next) => { + const entry = pack.entry({ ...header, name: `${dir}/${header.name}` }, (error) => { + if (error) reject(error); + else next(); + }); + content.pipe(entry); + }); + extract.once('error', reject); + extract.once('finish', resolve); + extract.end(archive); + }); + pack.finalize(); + return packed; +} + +/** A container's address on `network`, else on whatever network it does have. */ +function containerAddress(info: Docker.ContainerInspectInfo, network: string | undefined): string | undefined { + const networks = info.NetworkSettings?.Networks ?? {}; + const preferred = network ? networks[network]?.IPAddress : undefined; + return ( + preferred || + Object.values(networks).find((n) => n.IPAddress)?.IPAddress || + info.NetworkSettings?.IPAddress || + undefined + ); +} const RUN_LABEL = 'actor-runtime.runId'; /** Marks a create-only dev-folder-probe container (`probeDevFolder` below) so `reconcileOrphans` can * sweep one that outlived its own removal call. */ @@ -70,9 +280,28 @@ const PROBE_MOUNT_TARGET = '/probe'; /** On the browser-view sidecar container and its volume, so `reconcileOrphans` can sweep leftovers. */ const BROWSER_VIEWER_LABEL = 'actor-runtime.browserViewer'; /** Tagged with the payload's content hash, so a rebuilt runtime imports a fresh image. */ -const BROWSER_VIEWER_IMAGE_REPO = 'actor-runtime/browser-viewer'; +const BROWSER_VIEWER_IMAGE_REPO = `${LOCAL_IMAGE_PREFIX}actor-runtime/browser-viewer`; /** Shared between the Actor container and the sidecar through a tmpfs volume. */ const X11_SOCKET_DIR = '/tmp/.X11-unix'; +/** Name prefix of the per-run `node_modules` volume of a `devMount` run (`buildDevMounts`); the run id + * follows. Named, not anonymous, because Podman 3.x refuses a volume mount without a source ("must set + * source volume"). Removed with the run, and swept by this prefix after a restart. */ +const DEV_NODE_MODULES_VOLUME_PREFIX = 'actor-runtime-node-modules-'; +/** Where a `devMount` run keeps the image's own copy of an entrypoint file the bind mount would hide + * (`preserveHiddenEntrypoint`). */ +const PRESERVED_ENTRYPOINT_DIR = '/apify-runtime-entrypoint'; + +/** The image command a `devMount` run starts through instead of its own, plus the tar that puts the + * preserved file in place before the container starts. */ +interface PreservedEntrypoint { + field: 'Entrypoint' | 'Cmd'; + command: string[]; + /** The image's own `Cmd`, restated whenever `Entrypoint` is overridden: an engine drops the image's + * `Cmd` from a create request that sets `Entrypoint` (Docker and Podman alike), which would have run + * the Xvfb wrapper with no program to wrap. */ + cmd?: string[]; + tar: Buffer; +} /** Reachable only on `apify-local`; never published on the host. */ const BROWSER_VIEWER_VNC_PORT = 5900; const BROWSER_VIEWER_MEMORY_BYTES = 256 * 1024 * 1024; @@ -87,27 +316,30 @@ const VOLUME_REMOVE_RETRY_MS = 200; * An explicit `:probe` suffix, deliberately never `latest` (Docker's own implicit default for an * untagged name) - this image has nothing to do with an Actor's `latest`-tagged build, and an untagged * name would silently print as `...probe:latest` and invite exactly that confusion. */ -const PROBE_IMAGE_TAG = 'actor-runtime/dev-folder-probe:probe'; +const PROBE_IMAGE_TAG = `${LOCAL_IMAGE_PREFIX}actor-runtime/dev-folder-probe:probe`; /** * `FROM scratch` with nothing else would build fine but fails every `createContainer` against it with * HTTP 400 "no command specified" (moby refuses to create a container for an image with no `Cmd`/ * `Entrypoint`) - which would look exactly like a bad candidate path if left undiagnosed. `CMD` fixes * that; the command itself is never exec'd, since `probeDevFolder`'s container is created but never - * started. Verified empirically against a real daemon: builds and creates with no network access. + * started. Verified empirically against real Docker and Podman daemons: builds and creates with no + * network access. */ const PROBE_DOCKERFILE = 'FROM scratch\nCMD ["/nonexistent"]\n'; -/** The daemon's own fixed error-message substring for a `Mounts`-type bind whose source is missing - * (moby's `daemon/volume/mounts/validate.go: errBindSourceDoesNotExist`) - the one rejection shape - * `classifyProbeError` reports as "does not exist" rather than a generic "could not verify". */ -const BIND_SOURCE_MISSING_SUBSTRING = 'bind source path does not exist'; -/** The daemon's own fixed error-message substring (moby's mount validation, `stat : not a - * directory`) for a bind source that exists but is a regular file, not a directory - reachable only - * because the probe below appends `/.` to the candidate path (see `probeDevFolder`'s doc comment): a - * trailing `/.` on a file path forces the stat that produces exactly this message, discriminating a file - * from a directory in the same create-only call that already discriminates missing from present. The one - * rejection shape `classifyProbeError` reports as "not a directory" rather than a generic "could not - * verify", and never as "does not exist". */ -const NOT_A_DIRECTORY_SUBSTRING = 'not a directory'; +/** The host path the probe container binds read-only at `PROBE_MOUNT_TARGET`: the host's root, so the + * mount source always exists and the daemon never has to validate (or, on Podman, auto-create - see + * `probeDevFolder`) the candidate path itself. */ +const PROBE_MOUNT_SOURCE = '/'; +/** Response header of `HEAD /containers/{id}/archive?path=...` - base64 JSON of moby's + * `ContainerPathStat` (`name`, `size`, `mode`, `mtime`, `linkTarget`), identical on Podman. */ +const PATH_STAT_HEADER = 'x-docker-container-path-stat'; +/** Go `os.FileMode` type bits, as serialized into the stat header's `mode`. `GO_MODE_DIR` is bit 31, so a + * JS bitwise AND against it yields a negative int32 for a directory - non-zero, which is all the checks + * below need. */ +const GO_MODE_DIR = 0x80000000; +const GO_MODE_SYMLINK = 0x08000000; +/** Bounds `probeDevFolder`'s manual symlink following - a longer chain is reported `unknown`. */ +const MAX_SYMLINK_HOPS = 16; /** Docker daemon rejection substrings for "host port already bound" - covers both the classic and * newer moby wording. */ const PORT_IN_USE_SUBSTRINGS = ['port is already allocated', 'address already in use']; @@ -140,18 +372,101 @@ function hasStatusCode(error: unknown): error is Error & { statusCode: number } ); } -/** Classifies a `createContainer` rejection from `probeDevFolder`, most specific first - see - * `DevFolderProbeFailureReason`'s doc comment in `driver/types.ts` for what each outcome means and why - * a permission error/Docker Desktop file-sharing denial must never be asserted as "does not exist" or - * "not a directory". */ -function classifyProbeError(error: unknown): DevFolderProbeFailureReason { +/** Classifies a `createContainer` rejection from `probeDevFolder` - see `DevFolderProbeFailureReason`'s + * doc comment in `driver/types.ts`. The probe's mount source is always `/`, so a create rejection is + * never about the candidate path: it is either the daemon being gone or the probe image being gone. */ +function classifyProbeCreateError(error: unknown): DevFolderProbeFailureReason { if (!hasStatusCode(error)) return 'unreachable'; if (error.statusCode === 404) return 'image-missing'; - if (error.message.includes(BIND_SOURCE_MISSING_SUBSTRING)) return 'not-found'; - if (error.message.includes(NOT_A_DIRECTORY_SUBSTRING)) return 'not-a-directory'; return 'unknown'; } +/** One parsed `PATH_STAT_HEADER`. `linkTarget` is only meaningful when `mode` carries `GO_MODE_SYMLINK`. */ +interface ProbeStat { + mode: number; + linkTarget: string; +} + +type ProbeStatOutcome = { ok: true; stat: ProbeStat } | { ok: false; reason: DevFolderProbeFailureReason }; + +/** Classifies an `infoArchive` (`HEAD .../archive`) rejection: the daemon answers 404 for a path that + * does not exist under the probe mount - the one case allowed to say "does not exist". Anything else the + * daemon answered is "could not verify" (a permission problem, a Docker Desktop file-sharing denial); no + * answer at all is `unreachable`. */ +function classifyProbeStatError(error: unknown): DevFolderProbeFailureReason { + if (!hasStatusCode(error)) return 'unreachable'; + if (error.statusCode === 404) return 'not-found'; + return 'unknown'; +} + +/** Reads `PATH_STAT_HEADER` off the response `container.infoArchive` resolves with (dockerode hands back + * the raw `http.IncomingMessage` for this `HEAD` call). A response without a parseable header is + * `unknown`, never a guess. */ +function parseProbeStatResponse(response: unknown): ProbeStatOutcome { + const headers = (response as { headers?: Record } | undefined)?.headers; + const raw = headers?.[PATH_STAT_HEADER]; + const encoded = Array.isArray(raw) ? raw[0] : raw; + if (!encoded) return { ok: false, reason: 'unknown' }; + try { + const parsed = JSON.parse(Buffer.from(encoded, 'base64').toString('utf8')) as { + mode?: unknown; + linkTarget?: unknown; + }; + if (typeof parsed.mode !== 'number') return { ok: false, reason: 'unknown' }; + return { + ok: true, + stat: { mode: parsed.mode, linkTarget: typeof parsed.linkTarget === 'string' ? parsed.linkTarget : '' }, + }; + } catch { + return { ok: false, reason: 'unknown' }; + } +} + +/** Stats one path inside the (never-started) probe container, through the daemon's own archive-stat + * endpoint - the same resolution `docker cp` uses, which both Docker and Podman perform on a stopped + * container's bind mounts too. */ +async function statInProbe(container: Docker.Container, containerPath: string): Promise { + let response: unknown; + try { + response = await container.infoArchive({ path: containerPath }); + } catch (error) { + return { ok: false, reason: classifyProbeStatError(error) }; + } + // A `HEAD` response has no body, but the socket is only released once the message is consumed. + (response as { resume?: () => void } | undefined)?.resume?.(); + return parseProbeStatResponse(response); +} + +/** A TCP port that is free in this process's own network namespace right now - bound on loopback and + * released again, for a sidecar about to share that namespace (`startBrowserViewer`). */ +async function allocateFreePort(): Promise { + return new Promise((resolve, reject) => { + const server = createServer(); + server.once('error', reject); + server.listen(0, '127.0.0.1', () => { + const address = server.address(); + server.close(() => { + if (address && typeof address === 'object') resolve(address.port); + else reject(new Error('Could not allocate a free port for the browser-view sidecar')); + }); + }); + }); +} + +/** + * Maps the `linkTarget` a symlink stat reports back to a host path. Docker resolves a link's target in + * the scope of the container's root filesystem and reports that absolute container path: a target that + * landed under the probe mount is reported with the `PROBE_MOUNT_TARGET` prefix (a relative link, or an + * absolute one pointing back inside), while a host-absolute target that escaped the mount is reported + * verbatim - which, since the mount source is the host's `/`, is already the host path. Podman follows + * symlinks itself before answering, so this never runs there. + */ +function hostPathOfLinkTarget(linkTarget: string): string { + if (linkTarget === PROBE_MOUNT_TARGET) return '/'; + if (linkTarget.startsWith(`${PROBE_MOUNT_TARGET}/`)) return linkTarget.slice(PROBE_MOUNT_TARGET.length); + return linkTarget; +} + /** True when a `container.start()` rejection means the host debug port is already bound. */ function isPortInUseError(error: unknown): boolean { const message = error instanceof Error ? error.message : String(error); @@ -181,10 +496,11 @@ function dockerfileTarball(contents: string): NodeJS.ReadableStream { return pack; } -/** The `cpu_stats` fields the sampler diffs between two of its own successive samples. */ +/** What the sampler diffs between two of its own successive samples: the container's cumulative CPU time + * (`cpu_stats.cpu_usage.total_usage`, nanoseconds on every daemon) and when the daemon read it. */ interface CpuUsageSnapshot { - totalUsage: number; - systemUsage: number; + totalUsageNs: number; + readAtMs: number; } /** @@ -202,15 +518,22 @@ function memoryUsageBytesExcludingCache(stats: Docker.ContainerStats): number | } /** - * Presence-and-finiteness guard for the two `cpu_stats` fields the delta reads. A missing field skips the - * tick instead of throwing or producing a `NaN`. `online_cpus` is excluded: it has a sane `|| 1` fallback. + * Presence-and-finiteness guard for the one `cpu_stats` field the delta reads. A missing or non-finite + * `total_usage` skips the tick instead of throwing or producing a `NaN`. The read time is the daemon's own + * `read` timestamp when it parses, else this process's clock - the two differ only by the request's + * latency, which a one-second cadence makes negligible. + * + * Deliberately NOT `docker stats`' `cpu_delta / system_cpu_delta * online_cpus` formula: that one is only + * right when `system_cpu_usage` is the sum over all CPUs of the host's `/proc/stat` time, which Docker + * reports but Podman's Docker-compatible API does not (measured against a real Podman 4.9 daemon: a run + * throttled to 0.25 core came out as ~66% of one core, not ~25%). CPU-time-over-wall-time needs no + * daemon-specific field and agrees with the Docker formula on Docker to within a fraction of a percent. */ function cpuUsageSnapshotOf(stats: Docker.ContainerStats): CpuUsageSnapshot | undefined { - const totalUsage = stats.cpu_stats?.cpu_usage?.total_usage; - const systemUsage = stats.cpu_stats?.system_cpu_usage; - if (typeof totalUsage !== 'number' || !Number.isFinite(totalUsage)) return undefined; - if (typeof systemUsage !== 'number' || !Number.isFinite(systemUsage)) return undefined; - return { totalUsage, systemUsage }; + const totalUsageNs = stats.cpu_stats?.cpu_usage?.total_usage; + if (typeof totalUsageNs !== 'number' || !Number.isFinite(totalUsageNs)) return undefined; + const daemonReadAtMs = typeof stats.read === 'string' ? Date.parse(stats.read) : Number.NaN; + return { totalUsageNs, readAtMs: Number.isFinite(daemonReadAtMs) ? daemonReadAtMs : Date.now() }; } /** @@ -253,13 +576,11 @@ function startResourceSampler( } if (emit && previous) { - const cpuDelta = current.totalUsage - previous.totalUsage; - const systemDelta = current.systemUsage - previous.systemUsage; - const onlineCpus = stats.cpu_stats.online_cpus || 1; - // `systemDelta` is 0 only in a degenerate case (no host-wide CPU time elapsed between two - // samples, e.g. two calls landing on the very same daemon tick) - reported as 0% rather than - // producing NaN/Infinity. - const cpuPercentOfOneCore = systemDelta > 0 ? (cpuDelta / systemDelta) * onlineCpus * 100 : 0; + const cpuDeltaNs = current.totalUsageNs - previous.totalUsageNs; + const wallDeltaNs = (current.readAtMs - previous.readAtMs) * 1_000_000; + // `wallDeltaNs` is 0 only in a degenerate case (two reads stamped with the very same instant) - + // reported as 0% rather than producing NaN/Infinity. + const cpuPercentOfOneCore = wallDeltaNs > 0 ? (cpuDeltaNs / wallDeltaNs) * 100 : 0; onSample({ cpuPercentOfOneCore, memoryBytes, @@ -326,6 +647,35 @@ export class DockerDriver implements Driver { private probeImageBuild: Promise | undefined; /** Python debug payload tar + debugpy version, read from disk at most once and cached. */ private debugPayload: { tar: Buffer; debugpyVersion: string } | undefined; + /** True once this process's own container is confirmed on the `apify-local` network + * (`selfAttachToNetwork`): joined by this process, or started there with `--network apify-local`. + * False when this process is not in a container at all (`pnpm dev`) or the engine refused the attach + * (rootless Podman runs the runtime container under slirp4netns/pasta, where joining a second network + * is unsupported). */ + private onActorNetwork = false; + /** The `ExtraHosts` entry every Actor container gets so `apify-api` reaches this API, or undefined when + * the network's own DNS resolves the alias (this process registered it when it joined). Set by + * `selfAttachToNetwork`: `apify-api:host-gateway` (the host's published port) when this process is + * off the network; `apify-api:` when it sits on the network without the alias - a + * container started with `--network apify-local` has no alias unless the user also passed one. */ + private apiHostEntry: string | undefined; + /** The `ExtraHosts` entry that routes `apify-api` to this API through the host's published port - + * for Actor containers off the `apify-local` network. The address is the one the engine itself hands + * every container for the host (`host.containers.internal` on Podman, `host.docker.internal` on Docker + * Desktop - read from this container's own hosts file in `init`), or the engine keyword `host-gateway` + * when the hosts file has no such entry (Docker Engine). Podman before 4.1 does not know the + * keyword, which is why the hosts file comes first. */ + private hostRouteEntry = `${CONTAINER_API_ALIAS}:host-gateway`; + /** True on Podman 3.x (`init`): Actor containers and browser-view sidecars never use `apify-local` + * and run on the engine's default network, reaching this API by `chooseDefaultNetworkRoute`. */ + private actorsOnDefaultNetwork = false; + /** `chooseDefaultNetworkRoute`'s answer, computed once on first use. */ + private defaultNetworkRoute: Promise | undefined; + /** `detectResourceLimitSupport`'s answer from `init`. */ + private resourceLimits: ResourceLimitSupport = ALL_LIMITS_SUPPORTED; + private readonly hostsFile: string; + private readonly routeFile: string; + private readonly networkInterfaces: () => NodeJS.Dict; private browserViewerImageId: string | undefined; /** Shared by concurrent callers; cleared on failure so a later call retries (like `probeImageBuild`). */ private browserViewerImport: Promise | undefined; @@ -336,8 +686,18 @@ export class DockerDriver implements Driver { /** `docker` is injectable (defaults to a real `Docker()` socket client) so tests can pass a stub * `dockerode`-shaped object - there is no Docker daemon in this sandbox to test against for real. */ - constructor(docker: Docker = new Docker()) { + constructor( + docker: Docker = new Docker(), + options: { + hostsFile?: string; + routeFile?: string; + networkInterfaces?: () => NodeJS.Dict; + } = {}, + ) { this.docker = docker; + this.hostsFile = options.hostsFile ?? '/etc/hosts'; + this.routeFile = options.routeFile ?? '/proc/net/route'; + this.networkInterfaces = options.networkInterfaces ?? (() => os.networkInterfaces()); } async init(): Promise { @@ -345,12 +705,43 @@ export class DockerDriver implements Driver { await this.docker.ping(); } catch (error) { this.available = false; - this.unavailableReason = `Docker socket is not reachable: ${(error as Error).message}`; + this.unavailableReason = + `Docker API socket is not reachable (mount your Docker or Podman socket at /var/run/docker.sock, ` + + `or point DOCKER_HOST at it): ${(error as Error).message}`; return; } // A `docker.info()` failure must not make an otherwise-reachable daemon look unavailable. await this.captureHostCapacity(); + this.resourceLimits = await detectResourceLimitSupport(this.docker); + const unenforced = (['cpu', 'memory'] as const).filter((limit) => !this.resourceLimits[limit]); + if (unenforced.length > 0) { + console.warn( + `Per-run ${unenforced.join(' and ')} limits are not enforced: the engine reports no ` + + `${unenforced.map((limit) => `'${limit}'`).join('/')} cgroup controller available to it (rootless ` + + `Podman on a host that does not delegate it, or cgroups v1). Runs start without ${unenforced.length > 1 ? 'those limits' : 'that limit'}.`, + ); + } + + this.hostRouteEntry = `${CONTAINER_API_ALIAS}:${await hostAddressSeenFromContainers(this.hostsFile)}`; + this.apiHostEntry = this.hostRouteEntry; + + // Podman 3.x (the CNI generation, e.g. Ubuntu 22.04's 3.4): its user-defined networks are not worth + // touching - the stock config is unusable on the most common install, rootless cannot attach a + // running container at all, and a failed attach can wreck this container's own networking. Actors + // run on the engine's default network instead. + if ((await podmanMajorVersion(this.docker)) === 3) { + this.actorsOnDefaultNetwork = true; + const route = await this.routeOnDefaultNetwork(); + console.warn( + `Podman 3.x: Actor containers run on the engine's default network (its user-defined networks are ` + + `not used) and reach this API as ${route.extraHost}` + + `${route.networkMode ? ` (network mode ${route.networkMode})` : ''}; keep -p ${API_PORT}:${API_PORT} ` + + `published on all interfaces.`, + ); + this.available = true; + return; + } try { await this.ensureNetwork(); @@ -415,23 +806,98 @@ export class DockerDriver implements Driver { private async selfAttachToNetwork(): Promise { // Docker sets the container hostname to its own short id by default; this is a best-effort // self-identification that only matters when this process itself runs inside a container - // (the shipped runtime image) - a bare `node dist/index.js` on the host skips it harmlessly. + // (the shipped runtime image) - a bare `node dist/index.js` on the host has nothing to attach. const selfId = process.env.HOSTNAME; - if (!selfId) return; + if (!selfId) { + this.onActorNetwork = false; + this.apiHostEntry = this.hostRouteEntry; + console.warn( + `Not running inside a container (no HOSTNAME): Actor containers will reach this API through ` + + `the host's port ${API_PORT} (${this.hostRouteEntry}) instead of the ${NETWORK_NAME} ` + + `network alias.`, + ); + return; + } const network = this.docker.getNetwork(NETWORK_NAME); const info = await network.inspect().catch(() => undefined); - if (info?.Containers?.[selfId]) return; // already attached + // `Containers` is keyed by full container id on both Docker and Podman, while `HOSTNAME` is the + // short (12-char) one - a prefix match is what "already attached" actually means here. Without it + // every restart of the runtime container re-attempted the connect and logged the daemon's + // "already connected" rejection as a warning. + if (Object.keys(info?.Containers ?? {}).some((id) => id.startsWith(selfId))) { + this.onActorNetwork = true; + this.apiHostEntry = await this.apiHostEntryWhenAlreadyAttached(selfId); + return; + } - await network - .connect({ Container: selfId, EndpointConfig: { Aliases: [CONTAINER_API_ALIAS] } }) - .catch((error: Error) => { - // Not fatal: most likely we are not actually running inside a container right now - // (local dev). Actor containers still get the network; only the alias resolution from - // inside those containers back to us would be affected. + try { + await network.connect({ Container: selfId, EndpointConfig: { Aliases: [CONTAINER_API_ALIAS] } }); + this.onActorNetwork = true; + this.apiHostEntry = undefined; + } catch (error) { + // Not fatal: `startRun` falls back to routing Actor containers to this API through the host's + // published port (`ExtraHosts: apify-api -> host-gateway`). The usual cause is rootless Podman, + // whose default slirp4netns/pasta network mode cannot join a second network at runtime. + this.onActorNetwork = false; + this.apiHostEntry = this.hostRouteEntry; + console.warn( + `Could not attach the runtime's own container to the ${NETWORK_NAME} network: ${(error as Error).message}. ` + + `Actor containers will reach this API through the host's published port ${API_PORT} instead ` + + `(${this.hostRouteEntry}), so keep -p ${API_PORT}:${API_PORT} published on all interfaces. ` + + `To use the network alias anyway, pre-create the network (\`podman network create ${NETWORK_NAME}\`) ` + + `and start the runtime container with \`--network ${NETWORK_NAME}\`.`, + ); + } + } - console.warn(`Could not self-attach to the ${NETWORK_NAME} network: ${error.message}`); - }); + /** For a runtime container that was put on the network by whoever started it (`--network apify-local`) + * rather than by `selfAttachToNetwork`: the alias is registered only if they also passed + * `--network-alias apify-api`. Otherwise the container's own address on the network stands in for it - + * a hosts-file entry needs no DNS at all, so this route also works on engines whose network has no + * name resolution. The host's published port is the last resort if even the address is unknown. */ + private async apiHostEntryWhenAlreadyAttached(selfId: string): Promise { + const self = await this.docker + .getContainer(selfId) + .inspect() + .catch(() => undefined); + const endpoint = self?.NetworkSettings?.Networks?.[NETWORK_NAME]; + if (endpoint?.Aliases?.includes(CONTAINER_API_ALIAS)) return undefined; + if (endpoint?.IPAddress) return `${CONTAINER_API_ALIAS}:${endpoint.IPAddress}`; + console.warn( + `The runtime's own container is on the ${NETWORK_NAME} network but its address there could not be ` + + `read; Actor containers will reach this API through the host's published port ${API_PORT} instead ` + + `(${this.hostRouteEntry}), so keep -p ${API_PORT}:${API_PORT} published on all interfaces.`, + ); + return this.hostRouteEntry; + } + + /** The `HostConfig` network fields for an Actor container: on the `apify-local` network with the alias + * route, or - Podman 3.x - on the engine's default network with `chooseDefaultNetworkRoute`'s. */ + private async actorNetworkHostConfig(): Promise> { + if (!this.actorsOnDefaultNetwork) { + return { + NetworkMode: NETWORK_NAME, + // Only when the alias cannot resolve through the network's own DNS (`apiHostEntry`'s doc + // comment). Never added when the alias works: a hosts-file entry would override the DNS alias + // and force every Actor through the host. + ...(this.apiHostEntry ? { ExtraHosts: [this.apiHostEntry] } : {}), + }; + } + const route = await this.routeOnDefaultNetwork(); + return { ...(route.networkMode ? { NetworkMode: route.networkMode } : {}), ExtraHosts: [route.extraHost] }; + } + + private routeOnDefaultNetwork(): Promise { + this.defaultNetworkRoute ??= (async () => { + const view: OwnNetworkView = { + hostAddress: await engineHostEntry(this.hostsFile), + gateway: defaultGatewayFromRouteTable(await readFile(this.routeFile, 'utf8').catch(() => '')), + own: firstNonLoopbackIpv4(this.networkInterfaces()), + }; + return chooseDefaultNetworkRoute(view); + })(); + return this.defaultNetworkRoute; } async startBuild(ctx: BuildContext, onLog: (chunk: string) => void): Promise { @@ -576,13 +1042,18 @@ export class DockerDriver implements Driver { const overCapacityWarning = this.buildOverCapacityWarning(ctx); if (overCapacityWarning) onLog(overCapacityWarning); - // A secondary diagnostic for the residual risk that a folder verified at registration later - // vanishes: written before `createContainer` so it lands even if that call is what fails. + // Re-verified on every dev-mount run, before any container exists: Docker would reject a `Mounts` + // bind whose source vanished since registration, but Podman's Docker-compatible API auto-creates the + // missing source instead - which would silently start the run against an empty directory, exactly + // what `actor-driver.md` forbids ("fail visibly - never silently mount an empty directory"). + let preservedEntrypoint: PreservedEntrypoint | undefined; if (ctx.devMount) { + await this.assertDevFolderStillPresent(ctx.devMount.localDevFolder); onLog( `Mounting local dev folder ${ctx.devMount.localDevFolder} over the image's working directory ` + - `${ctx.devMount.imageWorkingDirectory} (node_modules preserved via an anonymous volume).\n`, + `${ctx.devMount.imageWorkingDirectory} (node_modules preserved via a per-run volume).\n`, ); + preservedEntrypoint = await this.preserveHiddenEntrypoint(ctx.imageId, ctx.devMount, onLog); } // Loaded and logged before `createContainer` so a missing payload fails the run before any @@ -604,7 +1075,7 @@ export class DockerDriver implements Driver { // The X-socket volume is the only change a browser-view run makes to the Actor's container. const mounts: Docker.MountSettings[] = [ - ...(ctx.devMount ? this.buildDevMounts(ctx.devMount) : []), + ...(ctx.devMount ? this.buildDevMounts(ctx.devMount, ctx.runId) : []), ...(ctx.x11SocketVolume ? [{ Type: 'volume' as const, Source: ctx.x11SocketVolume, Target: X11_SOCKET_DIR }] : []), @@ -614,15 +1085,22 @@ export class DockerDriver implements Driver { Image: ctx.imageId, Env: env, Labels: { [RUN_LABEL]: ctx.runId }, + ...(preservedEntrypoint + ? { + [preservedEntrypoint.field]: preservedEntrypoint.command, + ...(preservedEntrypoint.cmd ? { Cmd: preservedEntrypoint.cmd } : {}), + } + : {}), ...(ctx.debug ? { ExposedPorts: { [`${ctx.debug.port}/tcp`]: {} } } : {}), HostConfig: { - NetworkMode: NETWORK_NAME, - Memory: ctx.memoryMbytes * 1024 * 1024, + ...(await this.actorNetworkHostConfig()), + ...(this.resourceLimits.memory ? { Memory: ctx.memoryMbytes * 1024 * 1024 } : {}), // A CFS quota, never `NanoCpus`: the daemon hard-rejects a `NanoCpus` above the host's own // CPU count, which would turn "warn, never clamp" into "cannot run at all". `CpuQuota` is // validated for range only, so an over-capacity request still starts. - CpuPeriod: CPU_PERIOD_US, - CpuQuota: cpuQuotaFor(ctx.memoryMbytes), + ...(this.resourceLimits.cpu + ? { CpuPeriod: CPU_PERIOD_US, CpuQuota: cpuQuotaFor(ctx.memoryMbytes) } + : {}), AutoRemove: false, ...(mounts.length > 0 ? { Mounts: mounts } : {}), // Fixed 127.0.0.1-bound publish - lands on the developer's own host, not wherever the @@ -648,6 +1126,7 @@ export class DockerDriver implements Driver { try { // Inside the try so a failed upload still reaches the finally below and removes the container. + if (preservedEntrypoint) await container.putArchive(preservedEntrypoint.tar, { path: '/' }); if (debugPayload) { await container.putArchive(debugPayload.tar, { path: '/' }); } @@ -751,22 +1230,118 @@ export class DockerDriver implements Driver { await sampler?.stop(); this.timedOutRuns.delete(ctx.runId); this.runContainers.delete(ctx.runId); - // `{ v: true }` also removes the container's anonymous volumes - without it, the anonymous - // `node_modules` volume `buildDevMounts` adds for a `devMount` run would leak one per run, - // forever. Harmless for a run with no `devMount`: no anonymous volumes to remove. + // `{ v: true }` also removes any anonymous volumes; the named per-run `node_modules` volume of a + // `devMount` run (`buildDevMounts`) is not covered by it and goes separately, after the container. await container.remove({ v: true }).catch(() => undefined); + if (ctx.devMount) await this.removeVolumeWithRetry(devNodeModulesVolumeName(ctx.runId)); + } + } + + /** `startRun`'s pre-container check that a registered dev folder is still a directory on the host - + * the same probe registration used (`probeDevFolder`), so the two can never disagree on what counts as + * present. Throws (failing the run, with this as its status message) on every non-ok outcome, including + * "could not verify": a run that cannot prove its folder exists must not start against whatever the + * daemon would put there instead. */ + private async assertDevFolderStillPresent(localDevFolder: string): Promise { + const outcome = await this.probeDevFolder(localDevFolder, await this.ensureProbeImage()); + if (outcome.ok) return; + const problem = + outcome.reason === 'not-found' + ? 'no longer exists on the host' + : outcome.reason === 'not-a-directory' + ? 'is no longer a directory on the host' + : `could not be verified on the host (${outcome.reason})`; + throw new Error( + `The registered local dev folder ${localDevFolder} ${problem} - the run was not started, so that a ` + + `missing folder is never silently replaced by an empty directory. Restore the folder, or clear the ` + + `registration (POST /actor-runtime/dev-folder/ with an empty string body) to run from the ` + + `built image alone.`, + ); + } + + /** + * A `devMount` run starts through the image's own `Entrypoint` (or `Cmd`); when that names a file + * inside the working directory - Apify's Playwright base images start through `./xvfb-entrypoint.sh` + * there - the bind mount hides it unless the dev folder happens to carry the same file, and the engine + * refuses to start ("executable file not found"). Unless the dev folder provides it, the file is + * taken from the image and the run starts through that copy, at a path no mount covers. Anything + * `PATH`-resolved or absolute is left alone: the mount cannot hide it. + */ + private async preserveHiddenEntrypoint( + imageId: string, + devMount: DevFolderMount, + onLog: (chunk: string) => void, + ): Promise { + const info = await this.docker.getImage(imageId).inspect(); + const entrypointRaw = info.Config?.Entrypoint; + const entrypoint = Array.isArray(entrypointRaw) ? entrypointRaw : entrypointRaw ? [entrypointRaw] : []; + const field: PreservedEntrypoint['field'] = entrypoint.length > 0 ? 'Entrypoint' : 'Cmd'; + const command = field === 'Entrypoint' ? entrypoint : (info.Config?.Cmd ?? []); + const first = command[0]; + if (!first || !isWorkingDirectoryRelative(first)) return undefined; + if (await this.devFolderHasEntry(devMount.localDevFolder, first)) return undefined; + + const inImage = path.posix.resolve(devMount.imageWorkingDirectory, first); + const archive = await this.extractFromImage(imageId, inImage); + const tarball = await repackUnderDirectory(archive, PRESERVED_ENTRYPOINT_DIR); + onLog( + `The image starts through ${first} in its working directory, which the dev folder does not contain; ` + + `using the image's own copy of it.\n`, + ); + return { + field, + command: [`${PRESERVED_ENTRYPOINT_DIR}/${path.posix.basename(inImage)}`, ...command.slice(1)], + ...(field === 'Entrypoint' && info.Config?.Cmd ? { cmd: info.Config.Cmd } : {}), + tar: tarball, + }; + } + + /** Whether `relativePath` exists inside the registered dev folder on the host - through the same probe + * container `probeDevFolder` uses, so the answer is the engine's own. Public for tests. */ + async devFolderHasEntry(localDevFolder: string, relativePath: string): Promise { + const container = await this.docker.createContainer({ + Image: await this.ensureProbeImage(), + Labels: { [PROBE_LABEL]: 'true' }, + HostConfig: { + Mounts: [{ Type: 'bind', Source: PROBE_MOUNT_SOURCE, Target: PROBE_MOUNT_TARGET, ReadOnly: true }], + }, + }); + try { + const outcome = await statInProbe( + container, + path.posix.join(PROBE_MOUNT_TARGET, localDevFolder, relativePath), + ); + return outcome.ok; + } finally { + await container.remove().catch((error: Error) => { + console.warn(`Could not remove dev-folder probe container ${container.id}: ${error.message}`); + }); + } + } + + /** The archive of one path from an image, read through a container created (never started) from it. */ + private async extractFromImage(imageId: string, pathInImage: string): Promise { + const container = await this.docker.createContainer({ Image: imageId, Labels: { [PROBE_LABEL]: 'true' } }); + try { + return await readStream((await container.getArchive({ path: pathInImage })) as NodeJS.ReadableStream); + } finally { + await container.remove().catch(() => undefined); } } /** The two `HostConfig.Mounts` entries for a `devMount` run: a read-write bind for the dev folder * itself (`Mounts`, not `Binds` - a `Mounts`-type bind errors on a missing source instead of silently - * auto-creating one), plus an anonymous volume (empty `Source`) over `node_modules` - Docker copies - * the image's existing contents into it before mounting, preserving the image's installed - * dependencies underneath the bind (a *named* volume would start empty; a plain bind would erase it). */ - private buildDevMounts(devMount: DevFolderMount): Docker.MountSettings[] { + * auto-creating one), plus a fresh per-run volume over `node_modules` - the engine creates it at + * container creation and copies the image's existing contents into it before mounting, preserving the + * image's installed dependencies underneath the bind (a plain bind would erase them). */ + private buildDevMounts(devMount: DevFolderMount, runId: string): Docker.MountSettings[] { return [ { Type: 'bind', Source: devMount.localDevFolder, Target: devMount.imageWorkingDirectory }, - { Type: 'volume', Source: '', Target: `${devMount.imageWorkingDirectory}/node_modules` }, + { + Type: 'volume', + Source: devNodeModulesVolumeName(runId), + Target: `${devMount.imageWorkingDirectory}/node_modules`, + }, ]; } @@ -864,26 +1439,30 @@ export class DockerDriver implements Driver { } /** - * Host-side existence-and-directory check for a candidate dev-folder path: a create-only probe - * container, never started. `fs.existsSync` would test this process's own filesystem, not the host's; - * the only Engine API surface that validates an arbitrary host path is the mount-validation moby runs - * inside `POST /containers/create`. `BindOptions.CreateMountpoint` (which would auto-create a missing - * source and defeat this check) is never set. `imageId` is always `ensureProbeImage`'s own image - * above - never an Actor's build (registration must work for an Actor with no build at all), a - * self-inspected runtime image (`HOSTNAME` is unset in bare local dev, per `selfAttachToNetwork` - * above), or a pulled one (would break offline-after-first-build). + * Host-side existence-and-directory check for a candidate dev-folder path: a probe container that + * binds the host's `/` read-only at `PROBE_MOUNT_TARGET`, is never started, and is stat'ed through + * `HEAD /containers/{id}/archive?path=/probe` (`container.infoArchive`) - the same + * resolution `docker cp` uses, which both Docker and Podman perform on a stopped container's bind + * mounts too. `fs.existsSync` would test this process's own filesystem, not the host's. `imageId` is + * always `ensureProbeImage`'s own image above - never an Actor's build (registration must work for an + * Actor with no build at all), a self-inspected runtime image (`HOSTNAME` is unset in bare local dev, + * per `selfAttachToNetwork` above), or a pulled one (would break offline-after-first-build). * - * The mount `Source` is the candidate path with a literal `/.` appended, never the bare path - - * verified empirically against a real daemon (a `FROM scratch` probe image, no network pull needed): - * appending `/.` forces the same `stat` moby already performs to also reject a regular file (`invalid - * mount config for type "bind": stat /.: not a directory`, classified below as - * `not-a-directory`) while leaving every other outcome unchanged - a real directory (or a symlink - * resolving to one) still succeeds, and a missing path still rejects with the same - * `BIND_SOURCE_MISSING_SUBSTRING` (now trailed by `/.`, which the substring match ignores). Since the - * daemon's rejection message and this call's own `Source` therefore always carry the `/.` suffix, the - * caller (`services/dev-folder.ts`) never echoes either back to the user - only this function's own - * classified `DevFolderProbeFailureReason` crosses that boundary, so the path stored and displayed - * anywhere is always exactly what the caller submitted. + * Why the host root rather than the candidate itself as the mount source: Docker rejects a `Mounts` + * bind whose source is missing, but Podman's Docker-compatible API instead auto-creates the missing + * source directory on the host (its `containers_create` compat handler `MkdirAll`s every bind source + * and ignores `BindOptions.CreateMountpoint`), so a create-only probe would report a typo'd path as + * present *and* leave a root-owned empty directory behind. Mounting `/` (which always exists) and + * stat'ing beneath it has no such side effect on either daemon, and one code path serves both. + * + * The candidate is walked component by component so a symlink anywhere in it still resolves to what + * it points at on the host: Docker reports a symlink component as such (with the daemon's + * container-scoped `linkTarget`, mapped back to a host path by `hostPathOfLinkTarget`) instead of + * following it, while Podman follows symlinks itself. A regular file, or a symlink to one, is + * `not-a-directory`; a missing component is `not-found`; anything the daemon would not confirm is + * `unknown`, never a guess. Only the classified `DevFolderProbeFailureReason` crosses back to the + * caller (`services/dev-folder.ts`), so the path stored and displayed anywhere is always exactly what + * the caller submitted. */ async probeDevFolder(candidatePath: string, imageId: string): Promise { if (!this.available) return { ok: false, reason: 'unreachable' }; @@ -894,23 +1473,63 @@ export class DockerDriver implements Driver { Image: imageId, Labels: { [PROBE_LABEL]: 'true' }, HostConfig: { - Mounts: [ - { Type: 'bind', Source: `${candidatePath}/.`, Target: PROBE_MOUNT_TARGET, ReadOnly: true }, - ], + Mounts: [{ Type: 'bind', Source: PROBE_MOUNT_SOURCE, Target: PROBE_MOUNT_TARGET, ReadOnly: true }], }, }); } catch (error) { // Creation itself failed, so there is nothing to clean up. - return { ok: false, reason: classifyProbeError(error) }; + return { ok: false, reason: classifyProbeCreateError(error) }; } - // Creation succeeded, so this container genuinely exists on the daemon now - unlike the rejected - // path above, a failed removal here would leak a real container. Logged rather than swallowed so - // the leak is discoverable; `PROBE_LABEL` also lets `reconcileOrphans` sweep it on next startup. - await container.remove().catch((error: Error) => { - console.warn(`Could not remove dev-folder probe container ${container.id}: ${error.message}`); - }); - return { ok: true }; + try { + return await this.resolveDirectoryInProbe(container, candidatePath); + } finally { + // This container genuinely exists on the daemon now - a failed removal here would leak a real + // container. Logged rather than swallowed so the leak is discoverable; `PROBE_LABEL` also lets + // `reconcileOrphans` sweep it on next startup. + await container.remove().catch((error: Error) => { + console.warn(`Could not remove dev-folder probe container ${container.id}: ${error.message}`); + }); + } + } + + /** The component walk `probeDevFolder`'s doc comment describes, against an already-created probe + * container. Restarts from the link's target whenever a component turns out to be a symlink, bounded + * by `MAX_SYMLINK_HOPS`. */ + private async resolveDirectoryInProbe( + container: Docker.Container, + candidatePath: string, + ): Promise { + let pending = candidatePath.split('/').filter((component) => component !== ''); + let resolved = ''; + let hops = 0; + let stat: ProbeStat | undefined; + while (pending.length > 0) { + const [component, ...rest] = pending; + const current = `${resolved}/${component}`; + const outcome = await statInProbe(container, `${PROBE_MOUNT_TARGET}${current}`); + if (!outcome.ok) return outcome; + stat = outcome.stat; + if ((stat.mode & GO_MODE_SYMLINK) !== 0) { + if (++hops > MAX_SYMLINK_HOPS || stat.linkTarget === '') return { ok: false, reason: 'unknown' }; + const target = hostPathOfLinkTarget(stat.linkTarget); + // A relative target is relative to the link's own directory; an absolute one restarts at `/`. + const base = target.startsWith('/') ? '' : resolved; + const normalized = path.posix.normalize(`${base}/${target}`); + pending = [...normalized.split('/').filter((part) => part !== ''), ...rest]; + resolved = ''; + continue; + } + resolved = current; + pending = rest; + } + // `/` itself (no components) stats the mount root, which is always a directory. + if (!stat) { + const outcome = await statInProbe(container, PROBE_MOUNT_TARGET); + if (!outcome.ok) return outcome; + stat = outcome.stat; + } + return (stat.mode & GO_MODE_DIR) !== 0 ? { ok: true } : { ok: false, reason: 'not-a-directory' }; } async abortRun(runId: string): Promise { @@ -957,10 +1576,9 @@ export class DockerDriver implements Driver { rootfs.on('error', (error: Error) => { rootfsError = error; }); - const stream = await this.docker.importImage(rootfs, { - repo: BROWSER_VIEWER_IMAGE_REPO, - tag: version, - }); + // The tag travels inside `repo` (`name:tag`, which the Docker API allows) rather than as the separate + // `tag` parameter: Podman 3.x ignores that parameter and would store the image as `:latest`. + const stream = await this.docker.importImage(rootfs, { repo: tag }); await new Promise((resolve, reject) => { if (rootfsError) { reject(rootfsError); @@ -983,8 +1601,13 @@ export class DockerDriver implements Driver { return tag; } - /** The volume is created with mode 1777 up front: the Actor's Xvfb runs unprivileged and must be able - * to create its socket there. Anything created here is removed again if a later step fails. */ + /** A plain local volume, no tmpfs options: rootless Podman 3.x cannot mount a tmpfs volume at all + * ("cannot mount volumes without root privileges"), and Podman 5 rejects a sized one on a filesystem + * without project quota. It only ever holds one Unix socket. The Actor's Xvfb runs unprivileged and + * must be able to create that socket, so the directory must end up mode 1777: the sidecar image + * carries `/tmp/.X11-unix` with that mode (copied onto the empty volume when the sidecar, which mounts + * it first, starts) and its script chmods it again as root to be sure. Anything created here is + * removed again if a later step fails. */ async startBrowserViewer(target: BrowserViewerTarget): Promise { if (!this.available) { throw new Error(this.unavailableReason ?? 'Docker is not available'); @@ -995,12 +1618,16 @@ export class DockerDriver implements Driver { const containerName = `actor-runtime-browser-viewer-${target.runId}`; const labels = { [RUN_LABEL]: target.runId, [BROWSER_VIEWER_LABEL]: 'true' }; - await this.docker.createVolume({ - Name: volumeName, - Driver: 'local', - DriverOpts: { type: 'tmpfs', device: 'tmpfs', o: 'size=8m,mode=1777' }, - Labels: labels, - }); + await this.docker.createVolume({ Name: volumeName, Driver: 'local', Labels: labels }); + + // How the console reaches the sidecar's VNC server. Normally the sidecar joins `apify-local` and is + // reached by its address there. When this process runs in a container that is not on that network + // (`onActorNetwork`'s doc comment - rootless Podman; or Podman 3.x, where Actors never use it), the + // sidecar shares this container's own network namespace instead, so the console reaches it on + // localhost; every sidecar then needs a port of its own in that shared namespace, allocated here. + const selfContainerId = process.env.HOSTNAME; + const sharesRuntimeNetns = (!this.onActorNetwork || this.actorsOnDefaultNetwork) && !!selfContainerId; + const vncPort = sharesRuntimeNetns ? await allocateFreePort() : BROWSER_VIEWER_VNC_PORT; let container: Docker.Container | undefined; try { @@ -1010,12 +1637,16 @@ export class DockerDriver implements Driver { Cmd: ['/bin/sh', BROWSER_VIEWER_SCRIPT], Env: [ `${BROWSER_VIEWER_INTERACTIVE_ENV}=${target.interactive ? '1' : '0'}`, - `${BROWSER_VIEWER_PORT_ENV}=${BROWSER_VIEWER_VNC_PORT}`, + `${BROWSER_VIEWER_PORT_ENV}=${vncPort}`, ], Labels: labels, HostConfig: { - NetworkMode: NETWORK_NAME, - Memory: BROWSER_VIEWER_MEMORY_BYTES, + ...(sharesRuntimeNetns + ? { NetworkMode: `container:${selfContainerId}` } + : this.actorsOnDefaultNetwork + ? {} + : { NetworkMode: NETWORK_NAME }), + ...(this.resourceLimits.memory ? { Memory: BROWSER_VIEWER_MEMORY_BYTES } : {}), AutoRemove: false, Mounts: [{ Type: 'volume', Source: volumeName, Target: X11_SOCKET_DIR }], }, @@ -1024,12 +1655,15 @@ export class DockerDriver implements Driver { this.browserViewers.set(target.runId, { container, volumeName }); await container.start(); + if (sharesRuntimeNetns) { + return { vncHost: '127.0.0.1', vncPort, x11SocketVolume: volumeName }; + } const info = await container.inspect(); - const address = info.NetworkSettings?.Networks?.[NETWORK_NAME]?.IPAddress; + const address = containerAddress(info, this.actorsOnDefaultNetwork ? undefined : NETWORK_NAME); return { // The IP also works from a runtime running outside Docker; the name only resolves from inside. vncHost: address || containerName, - vncPort: BROWSER_VIEWER_VNC_PORT, + vncPort, x11SocketVolume: volumeName, }; } catch (error) { @@ -1114,5 +1748,11 @@ export class DockerDriver implements Driver { for (const volume of viewerVolumes ?? []) { await this.removeVolumeWithRetry(volume.Name); } + const { Volumes: devVolumes } = await this.docker.listVolumes({ + filters: JSON.stringify({ name: [DEV_NODE_MODULES_VOLUME_PREFIX] }), + }); + for (const volume of devVolumes ?? []) { + if (volume.Name.startsWith(DEV_NODE_MODULES_VOLUME_PREFIX)) await this.removeVolumeWithRetry(volume.Name); + } } } diff --git a/src/services/builds.ts b/src/services/builds.ts index 1f93a77..402de0a 100644 --- a/src/services/builds.ts +++ b/src/services/builds.ts @@ -5,6 +5,8 @@ import type { ActorRecord, ActorVersionRecord } from '../storage/entities.js'; import { recordTaggedBuild, updateActor } from './actors.js'; import type { Driver } from '../driver/types.js'; import { DriverTimedOutError } from '../driver/types.js'; +import { normalizeEntryName } from '../driver/tar-entry-name.js'; +import { qualifyDockerfileImageReferences } from './dockerfile-image-refs.js'; import { resolveDockerfileLocation } from './dockerfile-location.js'; import { appendLog, flushLog, markLogTerminal } from './logs.js'; import { isTerminalJobStatus, transitionJobStatus } from './job-status.js'; @@ -78,6 +80,28 @@ async function nextBuildNumber(actorId: string, versionNumber: string): Promise< return `${versionNumber}.${count + 1}`; } +/** + * The build's Dockerfile with every short `FROM` image name qualified to Docker Hub + * (`services/dockerfile-image-refs.ts`), each rewrite stated in the build log. Only this build's copy + * of the file changes - the pushed source is never modified. The other files pass through untouched. + */ +function qualifyDockerfileImages( + sourceFiles: SourceFile[], + dockerfilePath: string, + log: (line: string) => void, +): SourceFile[] { + return sourceFiles.map((file) => { + if (normalizeEntryName(file.name) !== dockerfilePath) return file; + const text = file.format === 'BASE64' ? Buffer.from(file.content, 'base64').toString('utf8') : file.content; + const { dockerfile, qualified } = qualifyDockerfileImageReferences(text); + if (qualified.length === 0) return file; + for (const { from, to } of qualified) { + log(`Using "${to}" for FROM "${from}" - a short image name means Docker Hub, as on the platform.\n`); + } + return { ...file, format: 'TEXT', content: dockerfile }; + }); +} + export interface StartBuildOptions { tag: string; useCache: boolean; @@ -193,10 +217,13 @@ export async function runBuildInBackground( return; } for (const line of dockerfileResolution.logLines) appendLog(record.id, line); - const sourceFiles: SourceFile[] = + const sourceFiles: SourceFile[] = qualifyDockerfileImages( dockerfileResolution.outcome === 'default' ? [...version.sourceFiles, dockerfileResolution.extraSourceFile] - : version.sourceFiles; + : version.sourceFiles, + dockerfileResolution.dockerfilePath, + (line) => appendLog(record.id, line), + ); try { const outcome = await driver.startBuild( @@ -225,6 +252,19 @@ export async function runBuildInBackground( // `undefined` on an inspect failure or an empty/`/` working directory) rather than written as // `undefined` - `entities.ts`'s doc comment on the field: "never present on a non-SUCCEEDED build" // stays true for the value too, there is simply nothing to record for this build. + // The tag is recorded BEFORE the SUCCEEDED write lands: a client that polls the build to SUCCEEDED + // and immediately starts a run against the tag (apify-client's `build(..., { waitForFinish })` + // followed by `start()`) must never find the tag still missing - with the writes the other way + // round that was a real, if narrow, window. The tag is still never left pointing at an aborted + // build: if the SUCCEEDED write below is refused because an abort won the race (`RUNNING` is the + // only status `SUCCEEDED` is a legal next-state from - see `job-status.ts`), the tag is put back to + // whatever it pointed at before, so `apify call`/`POST .../runs` against it keep working exactly as + // they did. + let previousTag: ActorRecord['taggedBuilds'][string] | undefined; + await updateActor(actor.id, (current) => { + previousTag = current.taggedBuilds[options.tag]; + return recordTaggedBuild(current, options.tag, record.id, record.buildNumber); + }); const succeeded = await transitionJobStatus(builds, record.id, 'SUCCEEDED', { finishedAt: new Date().toISOString(), imageId: outcome.imageId, @@ -233,15 +273,14 @@ export async function runBuildInBackground( ? { imageWorkingDirectory: outcome.imageWorkingDirectory } : {}), }); - // Only tag the build against the actor if the SUCCEEDED write actually landed - if an abort won - // the race above, `succeeded.status` is `ABORTED` (or the record vanished) and tagging here would - // clobber `actor.taggedBuilds[]` with a build that has no image, breaking every future - // `apify call`/`POST .../runs` against that tag even though the build record itself correctly - // stayed ABORTED. - if (succeeded?.status === 'SUCCEEDED') { - await updateActor(actor.id, (current) => - recordTaggedBuild(current, options.tag, record.id, record.buildNumber), - ); + if (succeeded?.status !== 'SUCCEEDED') { + await updateActor(actor.id, (current) => { + if (current.taggedBuilds[options.tag]?.buildId !== record.id) return current; + const taggedBuilds = { ...current.taggedBuilds }; + if (previousTag) taggedBuilds[options.tag] = previousTag; + else delete taggedBuilds[options.tag]; + return { ...current, taggedBuilds }; + }); } } catch (error) { const status: JobStatus = error instanceof DriverTimedOutError ? 'TIMED-OUT' : 'FAILED'; diff --git a/src/services/dockerfile-image-refs.ts b/src/services/dockerfile-image-refs.ts new file mode 100644 index 0000000..36ff3dc --- /dev/null +++ b/src/services/dockerfile-image-refs.ts @@ -0,0 +1,85 @@ +/** + * Docker's own rule for a short image name - no registry host in front of it means Docker Hub + * (`docker.io`), and a single-segment name lives under `library/` - applied by the runtime to the + * `FROM` lines of every Actor Dockerfile it builds. Docker applies that rule implicitly; Podman leaves + * it to the host's `registries.conf`, which on a stock Debian/Ubuntu install names no search registry + * at all, so `FROM apify/actor-node:20` fails there with "short-name did not resolve to an alias". + * Qualifying the reference up front makes the Dockerfile mean the same thing on every engine, exactly + * as it does on the platform. + * + * Left alone: `scratch`, a reference to an earlier build stage (`FROM base`), anything containing a + * variable (`FROM ${BASE}`), and any reference that already names a registry (a first path component + * with a `.` or `:`, or `localhost`). + */ + +export interface QualifiedImageReference { + from: string; + to: string; +} + +export interface QualifyDockerfileResult { + dockerfile: string; + qualified: QualifiedImageReference[]; +} + +const DOCKER_HUB = 'docker.io'; + +/** Whether the first path component of a multi-component name is a registry host, per + * `distribution/reference`'s `splitDockerDomain`. */ +function hasRegistry(nameWithoutDigest: string): boolean { + const firstSlash = nameWithoutDigest.indexOf('/'); + if (firstSlash === -1) return false; + const first = nameWithoutDigest.slice(0, firstSlash); + return first.includes('.') || first.includes(':') || first === 'localhost'; +} + +/** The fully-qualified form of `ref`, or undefined when it must be left as written. */ +export function qualifyImageReference(ref: string, stageNames: ReadonlySet): string | undefined { + if (ref === '' || ref.includes('$')) return undefined; + if (ref.toLowerCase() === 'scratch' || stageNames.has(ref.toLowerCase())) return undefined; + const nameWithoutDigest = ref.split('@')[0]!; + if (hasRegistry(nameWithoutDigest)) return undefined; + return nameWithoutDigest.includes('/') ? `${DOCKER_HUB}/${ref}` : `${DOCKER_HUB}/library/${ref}`; +} + +/** Splits an instruction's arguments on whitespace, remembering where each token starts. */ +function tokenize(text: string): Array<{ token: string; start: number }> { + const tokens: Array<{ token: string; start: number }> = []; + const pattern = /\S+/g; + for (let match = pattern.exec(text); match; match = pattern.exec(text)) { + tokens.push({ token: match[0], start: match.index }); + } + return tokens; +} + +export function qualifyDockerfileImageReferences(dockerfile: string): QualifyDockerfileResult { + const stageNames = new Set(); + const qualified: QualifiedImageReference[] = []; + const lines = dockerfile.split('\n'); + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]!; + const instruction = /^(\s*)from\s+/i.exec(line); + if (!instruction) continue; + + const argsStart = instruction[0].length; + const tokens = tokenize(line.slice(argsStart)); + // Flags such as `--platform=...` come first; the image reference is the first non-flag token. + const imageIndex = tokens.findIndex(({ token }) => !token.startsWith('--')); + if (imageIndex === -1) continue; + const image = tokens[imageIndex]!; + + const asIndex = tokens.findIndex(({ token }, index) => index > imageIndex && token.toLowerCase() === 'as'); + const stageName = asIndex !== -1 ? tokens[asIndex + 1]?.token : undefined; + + const replacement = qualifyImageReference(image.token, stageNames); + if (replacement) { + const at = argsStart + image.start; + lines[i] = `${line.slice(0, at)}${replacement}${line.slice(at + image.token.length)}`; + qualified.push({ from: image.token, to: replacement }); + } + if (stageName) stageNames.add(stageName.toLowerCase()); + } + + return { dockerfile: lines.join('\n'), qualified }; +} diff --git a/src/services/runs.ts b/src/services/runs.ts index 9d7e594..e23f4d9 100644 --- a/src/services/runs.ts +++ b/src/services/runs.ts @@ -385,13 +385,16 @@ export async function runInBackground( return; } } catch (error) { - await flushLog(record.id); // This is the one place that knows both the Actor id and its stored language preference, so it // composes the port-conflict remediation from the driver's typed error. const statusMessage = error instanceof DebugPortInUseError && actor.localDebug ? describeDebugPortConflict(actor.id, actor.localDebug.language, error.port) : (error as Error).message; + // Into the run's own log too: the engine refusing the container (a network it cannot set up, an + // unusable mount) is what `apify call` streams, and the status message alone leaves it empty. + appendLog(record.id, `Cannot start run: ${statusMessage}\n`); + await flushLog(record.id); await transitionJobStatus(runs, record.id, 'FAILED', { finishedAt: new Date().toISOString(), statusMessage, diff --git a/test/e2e/dev-folder-bind-mount.test.ts b/test/e2e/dev-folder-bind-mount.test.ts index 979776c..defd90a 100644 --- a/test/e2e/dev-folder-bind-mount.test.ts +++ b/test/e2e/dev-folder-bind-mount.test.ts @@ -14,7 +14,7 @@ * Requires a reachable Docker daemon and fails loudly, never skips, mirroring `actor-dev-loop.test.ts`. */ import { execFileSync } from 'node:child_process'; -import { cpSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { fileURLToPath } from 'node:url'; import { dirname, join } from 'node:path'; @@ -298,6 +298,66 @@ describe('local dev-folder bind mount: edit-compile-call loop with no rebuild (r 5 * 60 * 1000, ); + it( + "an entrypoint the image keeps inside its working directory (Apify's Playwright images start through one) stays available under the mount when the dev folder lacks it", + () => { + const env = apifyEnv(isolatedApifyHome); + // A tiny Actor of its own: busybox, an entrypoint script at ./entry.sh in the working directory + // (what `apify/actor-*-playwright*` images do with their Xvfb wrapper), and a dev folder that has + // no such file - mounting it over /app would hide the script. + const entryActorDir = mkdtempSync(join(tmpdir(), 'actor-runtime-e2e-devfolder-entry-')); + const devFolder = mkdtempSync(join(tmpdir(), 'actor-runtime-e2e-devfolder-entry-src-')); + try { + mkdirSync(join(entryActorDir, '.actor')); + writeFileSync( + join(entryActorDir, '.actor', 'actor.json'), + JSON.stringify({ + actorSpecification: 1, + name: 'devfolder-entrypoint', + version: '0.0', + buildTag: 'latest', + }), + ); + writeFileSync( + join(entryActorDir, 'entry.sh'), + '#!/bin/sh\necho "entry.sh from the image: $0"\nexec "$@"\n', + ); + writeFileSync( + join(entryActorDir, 'Dockerfile'), + [ + 'FROM docker.io/library/busybox', + 'WORKDIR /app', + 'COPY entry.sh ./entry.sh', + 'RUN chmod 755 ./entry.sh', + 'ENTRYPOINT ["./entry.sh"]', + 'CMD ["sh", "-c", "ls /app; echo run-body-done"]', + '', + ].join('\n'), + ); + writeFileSync(join(devFolder, 'only-in-dev-folder.txt'), 'x'); + + const push = JSON.parse(apify(['push', '--json'], { cwd: entryActorDir, env })) as PushResult; + expect(push.build.status).toBe('SUCCEEDED'); + registerDevFolder(push.actor.id, devFolder, env); + + const call = JSON.parse(apify(['call', '--json'], { cwd: entryActorDir, env })) as CallResult; + expect(call.run.status).toBe('SUCCEEDED'); + // The stored log, not `apify call`'s streamed copy: for an Actor that exits within milliseconds + // the CLI's stream can close before its last lines are flushed, on any engine. + const log = apify(['api', 'GET', `actor-runs/${call.run.id}/log`], { cwd: REPO_ROOT, env }); + expect(log).toContain('starts through ./entry.sh in its working directory'); + expect(log).toContain('entry.sh from the image: /apify-runtime-entrypoint/entry.sh'); + // The dev folder, not the image's /app, is what the run sees in the working directory. + expect(log).toContain('only-in-dev-folder.txt'); + expect(log).toContain('run-body-done'); + } finally { + rmSync(entryActorDir, { recursive: true, force: true }); + rmSync(devFolder, { recursive: true, force: true }); + } + }, + 5 * 60 * 1000, + ); + it( 'anonymous node_modules volumes do not accumulate across runs ({ v: true } cleanup)', async () => { diff --git a/test/e2e/helpers/browser-view-suite.ts b/test/e2e/helpers/browser-view-suite.ts index ad5a614..7fd0f3f 100644 --- a/test/e2e/helpers/browser-view-suite.ts +++ b/test/e2e/helpers/browser-view-suite.ts @@ -134,8 +134,9 @@ export function describeBrowserViewSuite(sample: BrowserViewSample): void { describe(`per-Actor browser view: live mirror of the ${sample.label} Playwright sample Actor (requires Docker)`, () => { let isolatedApifyHome: string; - /** Set by the first case and reused by the second: a repeated `apify push` of an unchanged Actor is - * refused by the CLI ("already exists ... newer changes than your local copy"). */ + /** Pushed once for the whole suite: a repeated `apify push` of an unchanged Actor is refused by the CLI + * ("already exists ... newer changes than your local copy"), so neither a retried case nor the second + * case may push again. */ let pushedActorId: string; beforeAll( @@ -153,6 +154,16 @@ export function describeBrowserViewSuite(sample: BrowserViewSample): void { isolatedApifyHome = createIsolatedApifyHome(); loginApifyCli(REPO_ROOT, isolatedApifyHome); + + const pushOutput = apify(['push', '--json'], { + cwd: join(REPO_ROOT, sample.dir), + env: apifyEnv(isolatedApifyHome), + }); + const push = JSON.parse(pushOutput) as PushResult; + if (push.build.status !== 'SUCCEEDED') { + throw new Error(`apify push of ${sample.dir} ended with build status ${push.build.status}`); + } + pushedActorId = push.actor.id; }, 15 * 60 * 1000, ); @@ -163,16 +174,11 @@ export function describeBrowserViewSuite(sample: BrowserViewSample): void { }); it( - `${sample.label} sample: push -> toggle on -> run: the log names the viewer URL, the viewer websocket reaches a live RFB server while the run crawls, the console links to the page, and the run finishes with the input-dependent item count`, + `${sample.label} sample: toggle on -> run: the log names the viewer URL, the viewer websocket reaches a live RFB server while the run crawls, the console links to the page, and the run finishes with the input-dependent item count`, async () => { const env = apifyEnv(isolatedApifyHome); const actorDir = join(REPO_ROOT, sample.dir); - - const pushOutput = apify(['push', '--json'], { cwd: actorDir, env }); - const push = JSON.parse(pushOutput) as PushResult; - expect(push.build.status).toBe('SUCCEEDED'); - const actorId = push.actor.id; - pushedActorId = actorId; + const actorId = pushedActorId; const toggle = apify( ['api', 'POST', `/actor-runtime/browser-view/${actorId}`, '--body', '{"enabled": true}'], @@ -234,7 +240,9 @@ export function describeBrowserViewSuite(sample: BrowserViewSample): void { expect(await endedPage.text()).toContain('This run has ended'); await expect(readMirrorGreeting(run.id, 10_000)).rejects.toThrow(/1008/); }, - 10 * 60 * 1000, + // One retry: the sample crawls a real external site (deliberately - the route out of an Actor is part + // of what is tested), and CI runners occasionally see its navigations time out; a defect reproduces. + { timeout: 10 * 60 * 1000, retry: 1 }, ); it.runIf(sample.withToggleClearedCase)( @@ -242,7 +250,6 @@ export function describeBrowserViewSuite(sample: BrowserViewSample): void { () => { const env = apifyEnv(isolatedApifyHome); const actorDir = join(REPO_ROOT, sample.dir); - expect(pushedActorId).toBeDefined(); const actorId = pushedActorId; apify(['api', 'POST', `/actor-runtime/browser-view/${actorId}`, '--body', '{"enabled": false}'], { cwd: REPO_ROOT, @@ -272,7 +279,7 @@ export function describeBrowserViewSuite(sample: BrowserViewSample): void { ) as DatasetInfoResult; expect(info.itemCount).toBe(2); }, - 5 * 60 * 1000, + { timeout: 5 * 60 * 1000, retry: 1 }, ); }); } diff --git a/test/e2e/helpers/docker.ts b/test/e2e/helpers/docker.ts index f350024..390a07c 100644 --- a/test/e2e/helpers/docker.ts +++ b/test/e2e/helpers/docker.ts @@ -1,9 +1,34 @@ import { execFileSync } from 'node:child_process'; +import process from 'node:process'; -/** True when a Docker daemon is reachable from this process. Gates the whole e2e suite. */ +/** + * The container CLI the suite drives the host engine with - `docker` by default, `podman` via + * `CONTAINER_CLI=podman`. Both accept the exact `build`/`pull`/`run`/`logs`/`rm`/`volume rm` invocations + * below unchanged; the runtime container itself only ever sees the engine's Docker-compatible API socket. + */ +const CONTAINER_CLI = process.env.CONTAINER_CLI === 'podman' ? 'podman' : 'docker'; + +/** Default location of the engine's API socket on the host, per CLI: Docker's, or rootful Podman's + * (`podman.socket` / `podman system service`). A `unix://` `DOCKER_HOST` overrides both - the way to + * point the suite at a rootless Podman socket (`$XDG_RUNTIME_DIR/podman/podman.sock`). */ +const DEFAULT_SOCKET_PATH = CONTAINER_CLI === 'podman' ? '/run/podman/podman.sock' : '/var/run/docker.sock'; + +/** Where the runtime container looks for the socket - `docker-driver.ts`'s dockerode default. */ +const RUNTIME_SOCKET_PATH = '/var/run/docker.sock'; + +/** The host socket path the runtime container gets mounted, so builds and runs land on the same engine + * the suite itself is talking to. */ +export function hostEngineSocketPath(): string { + const dockerHost = process.env.DOCKER_HOST; + if (dockerHost?.startsWith('unix://')) return dockerHost.slice('unix://'.length); + return DEFAULT_SOCKET_PATH; +} + +/** True when a Docker-API-compatible engine (Docker, or Podman via `CONTAINER_CLI=podman`) is reachable + * from this process. Gates the whole e2e suite. */ export function isDockerAvailable(): boolean { try { - execFileSync('docker', ['info'], { stdio: 'ignore' }); + execFileSync(CONTAINER_CLI, ['info'], { stdio: 'ignore' }); return true; } catch { return false; @@ -11,32 +36,37 @@ export function isDockerAvailable(): boolean { } export function buildRuntimeImage(repoRoot: string, tag: string): void { - execFileSync('docker', ['build', '-t', tag, repoRoot], { stdio: 'inherit' }); + execFileSync(CONTAINER_CLI, ['build', '-t', tag, repoRoot], { stdio: 'inherit' }); } export function pullBaseImages(): void { // Pre-pulled here rather than left to the first build, per `test.md`'s documented CI requirement - // building an Actor image is the one step that still needs network, and doing it once up front - // keeps the timing of the actual push/call assertions predictable. - for (const image of ['apify/actor-node:24', 'apify/actor-python:3.13', 'python:3.11-slim']) { - execFileSync('docker', ['pull', image], { stdio: 'inherit' }); + // keeps the timing of the actual push/call assertions predictable. Fully-qualified names so Podman's + // short-name resolution never has to guess a registry. + for (const image of [ + 'docker.io/apify/actor-node:24', + 'docker.io/apify/actor-python:3.13', + 'docker.io/library/python:3.11-slim', + ]) { + execFileSync(CONTAINER_CLI, ['pull', image], { stdio: 'inherit' }); } } /** `sample_actor_playwright/Dockerfile`'s base image - pulled only by its own e2e file, not by * `pullBaseImages`, since it is large and no other file builds against it. */ -export const PLAYWRIGHT_BASE_IMAGE = 'apify/actor-node-playwright-chrome:24-1.61.1'; +export const PLAYWRIGHT_BASE_IMAGE = 'docker.io/apify/actor-node-playwright-chrome:24-1.61.1'; /** `sample_actor_playwright_py/Dockerfile`'s base image. */ -export const PYTHON_PLAYWRIGHT_BASE_IMAGE = 'apify/actor-python-playwright:3.14-1.61.0'; +export const PYTHON_PLAYWRIGHT_BASE_IMAGE = 'docker.io/apify/actor-python-playwright:3.14-1.61.0'; export function pullImage(image: string): void { - execFileSync('docker', ['pull', image], { stdio: 'inherit' }); + execFileSync(CONTAINER_CLI, ['pull', image], { stdio: 'inherit' }); } export function startRuntimeContainer(tag: string, containerName: string): void { execFileSync( - 'docker', + CONTAINER_CLI, [ 'run', '-d', @@ -55,7 +85,7 @@ export function startRuntimeContainer(tag: string, containerName: string): void '-p', '3000:3000', '-v', - '/var/run/docker.sock:/var/run/docker.sock', + `${hostEngineSocketPath()}:${RUNTIME_SOCKET_PATH}`, '-v', `${containerName}-data:/data`, tag, @@ -72,7 +102,7 @@ export function stopRuntimeContainer(containerName: string): void { // test stdout the failure shows up in, actually captures it; the workflow step stays as a harmless // backstop for cases where the process is killed before `afterAll` runs at all. try { - const logs = execFileSync('docker', ['logs', '--tail', '300', containerName], { + const logs = execFileSync(CONTAINER_CLI, ['logs', '--tail', '300', containerName], { stdio: ['ignore', 'pipe', 'pipe'], }); process.stdout.write(`\n--- ${containerName} container logs (last 300 lines) ---\n`); @@ -83,12 +113,12 @@ export function stopRuntimeContainer(containerName: string): void { // only, never a reason to skip the cleanup below. } try { - execFileSync('docker', ['rm', '-f', containerName], { stdio: 'ignore' }); + execFileSync(CONTAINER_CLI, ['rm', '-f', containerName], { stdio: 'ignore' }); } catch { // best-effort cleanup } try { - execFileSync('docker', ['volume', 'rm', '-f', `${containerName}-data`], { stdio: 'ignore' }); + execFileSync(CONTAINER_CLI, ['volume', 'rm', '-f', `${containerName}-data`], { stdio: 'ignore' }); } catch { // best-effort cleanup } diff --git a/test/integration/job-lifecycle.test.ts b/test/integration/job-lifecycle.test.ts index e2c5107..e3eab03 100644 --- a/test/integration/job-lifecycle.test.ts +++ b/test/integration/job-lifecycle.test.ts @@ -47,6 +47,7 @@ import type { SourceFile, } from '../../src/storage/entities.js'; import { DEFAULT_DOCKERFILE_CONTENT, DEFAULT_DOCKERFILE_NAME } from '../../src/services/default-dockerfile.js'; +import { getFullLog } from '../../src/services/logs.js'; /** Creates an Actor via the real client (so it has a genuine owner) and returns the underlying * `ActorRecord` for direct service-layer calls. */ @@ -240,6 +241,39 @@ describe('job lifecycle: TIMED-OUT mapping and abort/completion race guards', () expect(finalActor?.taggedBuilds.latest).toEqual({ buildId: 'previous-build-id', buildNumber: '0.0.1' }); }); + it('records the tag before the build record turns SUCCEEDED, so a client that polls the build to SUCCEEDED and immediately starts a run never finds the tag missing', async () => { + const driver = deferredBuildDriver(); + server = await startTestServer(driver); + const actor = await seedActor(server, 'tag-before-succeeded-actor'); + + const record: BuildRecord = { + id: generateId(), + userId: actor.userId, + actorId: actor.id, + versionNumber: '0.0', + buildNumber: '0.0.1', + tag: 'latest', + status: 'READY', + startedAt: new Date().toISOString(), + }; + await getRegistries().builds.set(record.id, record); + + const bg = runBuildInBackground(driver, actor, VERSION, record, { tag: 'latest', useCache: true }); + await driver.started; + driver.resolveBuild({ imageId: 'image:latest' }); + + // Poll the build record as tightly as a client can, and read the actor the instant it is + // SUCCEEDED - with the writes in the wrong order this observes the tag still missing. + let observed: BuildRecord | null = null; + while (observed?.status !== 'SUCCEEDED') { + observed = await getRegistries().builds.get(record.id); + } + const actorAtSuccess = await getRegistries().actors.get(actor.id); + expect(actorAtSuccess?.taggedBuilds.latest).toEqual({ buildId: record.id, buildNumber: '0.0.1' }); + + await bg; + }); + it('a normal (non-aborted) successful build does record itself against the tag', async () => { const driver = deferredBuildDriver(); server = await startTestServer(driver); @@ -400,7 +434,14 @@ describe('job lifecycle: TIMED-OUT mapping and abort/completion race guards', () expect(ctx.dockerfilePath).toBe(DEFAULT_DOCKERFILE_NAME); expect(ctx.sourceFiles).toEqual([ ...noDockerfileSourceFiles, - { name: 'Dockerfile', format: 'TEXT', content: DEFAULT_DOCKERFILE_CONTENT }, + { + name: 'Dockerfile', + format: 'TEXT', + content: DEFAULT_DOCKERFILE_CONTENT.replace( + 'FROM apify/actor-node:20', + 'FROM docker.io/apify/actor-node:20', + ), + }, ]); expect(version.sourceFiles).toEqual(noDockerfileSourceFiles); @@ -414,7 +455,7 @@ describe('job lifecycle: TIMED-OUT mapping and abort/completion race guards', () const actor = await seedActor(server, 'dockerfile-resolved-actor'); const resolvedSourceFiles: SourceFile[] = [ - { name: '.actor/Dockerfile', format: 'TEXT', content: 'FROM node:20\n' }, + { name: '.actor/Dockerfile', format: 'TEXT', content: 'FROM docker.io/library/node:20\n' }, { name: 'main.js', format: 'TEXT', content: 'console.log(1);\n' }, ]; const version: ActorVersionRecord = { ...VERSION, sourceFiles: resolvedSourceFiles }; @@ -441,6 +482,52 @@ describe('job lifecycle: TIMED-OUT mapping and abort/completion race guards', () const final = await getRegistries().builds.get(record.id); expect(final?.status).toBe('SUCCEEDED'); }); + + it("a short image name in the Dockerfile's FROM is qualified to Docker Hub before the driver sees it, and the build log says so", async () => { + const driver = fixedBuildOutcomeDriver({ imageId: 'x' }); + server = await startTestServer(driver); + const actor = await seedActor(server, 'dockerfile-short-name-actor'); + + const version: ActorVersionRecord = { + ...VERSION, + sourceFiles: [ + { + name: 'Dockerfile', + format: 'BASE64', + content: Buffer.from('FROM apify/actor-python-playwright:3.14-1.61.0\nCOPY . ./\n').toString( + 'base64', + ), + }, + ], + }; + + const record: BuildRecord = { + id: generateId(), + userId: actor.userId, + actorId: actor.id, + versionNumber: '0.0', + buildNumber: '0.0.1', + tag: 'latest', + status: 'READY', + startedAt: new Date().toISOString(), + }; + await getRegistries().builds.set(record.id, record); + + await runBuildInBackground(driver, actor, version, record, { tag: 'latest', useCache: true }); + + const ctx = driver.startBuildContexts[0]!; + expect(ctx.sourceFiles).toEqual([ + { + name: 'Dockerfile', + format: 'TEXT', + content: 'FROM docker.io/apify/actor-python-playwright:3.14-1.61.0\nCOPY . ./\n', + }, + ]); + const log = await getFullLog(record.id); + expect(log).toContain( + 'Using "docker.io/apify/actor-python-playwright:3.14-1.61.0" for FROM "apify/actor-python-playwright:3.14-1.61.0"', + ); + }); }); describe('imageWorkingDirectory is build-specific, not Actor-specific (human directive: "the workdir should be build specific, not actor specific")', () => { diff --git a/test/unit/docker-driver-browser-view.test.ts b/test/unit/docker-driver-browser-view.test.ts index 8a0238b..46eb6c1 100644 --- a/test/unit/docker-driver-browser-view.test.ts +++ b/test/unit/docker-driver-browser-view.test.ts @@ -81,6 +81,7 @@ describe('DockerDriver.startBrowserViewer / stopBrowserViewer', () => { }); afterEach(() => { + vi.unstubAllEnvs(); rmSync(payloadDir, { recursive: true, force: true }); if (originalEnv === undefined) delete process.env[PAYLOAD_ENV]; else process.env[PAYLOAD_ENV] = originalEnv; @@ -94,20 +95,21 @@ describe('DockerDriver.startBrowserViewer / stopBrowserViewer', () => { const handle = await driver.startBrowserViewer({ runId: 'run-1', interactive: false }); expect(stub.calls).toEqual(['inspectImage', 'importImage', 'createVolume', 'createContainer', 'start']); - expect(stub.getImage).toHaveBeenCalledWith('actor-runtime/browser-viewer:abc123def456'); + expect(stub.getImage).toHaveBeenCalledWith('localhost/actor-runtime/browser-viewer:abc123def456'); + // `name:tag` in `repo`, no separate `tag`: Podman 3.x ignores the `tag` parameter. expect(stub.importImage.mock.calls[0]![1]).toEqual({ - repo: 'actor-runtime/browser-viewer', - tag: 'abc123def456', + repo: 'localhost/actor-runtime/browser-viewer:abc123def456', }); const [volumeOptions] = stub.createVolume.mock.calls[0]!; expect(volumeOptions.Name).toBe('actor-runtime-x11-run-1'); expect(volumeOptions.Driver).toBe('local'); - expect(volumeOptions.DriverOpts).toEqual({ type: 'tmpfs', device: 'tmpfs', o: 'size=8m,mode=1777' }); + // A plain local volume: rootless Podman 3.x cannot mount tmpfs-backed volumes. + expect(volumeOptions.DriverOpts).toBeUndefined(); expect(volumeOptions.Labels).toEqual({ 'actor-runtime.runId': 'run-1', 'actor-runtime.browserViewer': 'true' }); const [containerOptions] = stub.createContainer.mock.calls[0]!; - expect(containerOptions.Image).toBe('actor-runtime/browser-viewer:abc123def456'); + expect(containerOptions.Image).toBe('localhost/actor-runtime/browser-viewer:abc123def456'); expect(containerOptions.name).toBe('actor-runtime-browser-viewer-run-1'); expect(containerOptions.Cmd).toEqual(['/bin/sh', '/apify-browser-viewer.sh']); expect(containerOptions.Env).toEqual(['APIFY_BROWSER_VIEWER_INTERACTIVE=0', 'APIFY_BROWSER_VIEWER_PORT=5900']); @@ -148,6 +150,42 @@ describe('DockerDriver.startBrowserViewer / stopBrowserViewer', () => { expect(stub.getImage).toHaveBeenCalledTimes(1); }); + it("when this process runs in a container that could not join apify-local (rootless Podman), the sidecar shares this container's network namespace on a port of its own and is reached on localhost", async () => { + vi.stubEnv('HOSTNAME', 'abc123def456'); + const stub = stubDockerForViewer({ imagePresent: true }); + const driver = new DockerDriver(stub.docker); + driver.available = true; // `onActorNetwork` stays false: `init()` never attached this container. + + const handle = await driver.startBrowserViewer({ runId: 'run-netns', interactive: false }); + + const [containerOptions] = stub.createContainer.mock.calls[0]!; + expect(containerOptions.HostConfig?.NetworkMode).toBe('container:abc123def456'); + expect(handle.vncHost).toBe('127.0.0.1'); + expect(handle.vncPort).toBeGreaterThan(0); + expect(handle.vncPort).not.toBe(5900); + expect(containerOptions.Env).toContain(`APIFY_BROWSER_VIEWER_PORT=${handle.vncPort}`); + // Nothing to look up on the network: the address is this container's own loopback. + expect(stub.container.inspect).not.toHaveBeenCalled(); + }); + + it('joins apify-local as usual when this container did attach to it, even though it runs in a container', async () => { + vi.stubEnv('HOSTNAME', 'abc123def456'); + const stub = stubDockerForViewer({ imagePresent: true }); + const driver = new DockerDriver(stub.docker); + driver.available = true; + (driver as unknown as { onActorNetwork: boolean }).onActorNetwork = true; + + const handle = await driver.startBrowserViewer({ runId: 'run-alias', interactive: false }); + + const [containerOptions] = stub.createContainer.mock.calls[0]!; + expect(containerOptions.HostConfig?.NetworkMode).toBe('apify-local'); + expect(handle).toEqual({ + vncHost: '172.18.0.9', + vncPort: 5900, + x11SocketVolume: 'actor-runtime-x11-run-alias', + }); + }); + it('falls back to the sidecar container name as vncHost when the daemon reports no IP on apify-local', async () => { const stub = stubDockerForViewer({ imagePresent: true, ipAddress: '' }); const driver = new DockerDriver(stub.docker); @@ -276,6 +314,9 @@ describe('DockerDriver.startRun - the X-socket volume mount', () => { const stub = stubDockerForRun(); const driver = new DockerDriver(stub.docker); driver.available = true; + // The run-start dev-folder re-check (`assertDevFolderStillPresent`) is not under test here. + vi.spyOn(driver, 'ensureProbeImage').mockResolvedValue('probe:image'); + vi.spyOn(driver, 'probeDevFolder').mockResolvedValue({ ok: true }); const outcomePromise = driver.startRun( { @@ -294,7 +335,11 @@ describe('DockerDriver.startRun - the X-socket volume mount', () => { const [options] = stub.createContainer.mock.calls[0]!; expect(options.HostConfig?.Mounts).toEqual([ { Type: 'bind', Source: '/host/src', Target: '/usr/src/app' }, - { Type: 'volume', Source: '', Target: '/usr/src/app/node_modules' }, + { + Type: 'volume', + Source: expect.stringMatching(/^actor-runtime-node-modules-/), + Target: '/usr/src/app/node_modules', + }, { Type: 'volume', Source: 'actor-runtime-x11-run-bv-2', Target: '/tmp/.X11-unix' }, ]); diff --git a/test/unit/docker-driver-debug.test.ts b/test/unit/docker-driver-debug.test.ts index f865af3..521668a 100644 --- a/test/unit/docker-driver-debug.test.ts +++ b/test/unit/docker-driver-debug.test.ts @@ -497,6 +497,9 @@ describe('DockerDriver.startRun - debug mode (actor-driver.md: "Debug mode")', ( const stub = stubDockerForRun(); const driver = new DockerDriver(stub.docker); driver.available = true; + // The run-start dev-folder re-check (`assertDevFolderStillPresent`) is not under test here. + vi.spyOn(driver, 'ensureProbeImage').mockResolvedValue('probe:image'); + vi.spyOn(driver, 'probeDevFolder').mockResolvedValue({ ok: true }); const outcomePromise = driver.startRun( { @@ -515,7 +518,11 @@ describe('DockerDriver.startRun - debug mode (actor-driver.md: "Debug mode")', ( const [options] = stub.createContainer.mock.calls[0]!; expect(options.HostConfig?.Mounts).toEqual([ { Type: 'bind', Source: '/host/src', Target: '/usr/src/app' }, - { Type: 'volume', Source: '', Target: '/usr/src/app/node_modules' }, + { + Type: 'volume', + Source: expect.stringMatching(/^actor-runtime-node-modules-/), + Target: '/usr/src/app/node_modules', + }, ]); expect(options.ExposedPorts).toEqual({ '9229/tcp': {} }); expect(options.HostConfig?.PortBindings).toEqual({ diff --git a/test/unit/docker-driver.test.ts b/test/unit/docker-driver.test.ts index ff82830..79cb10f 100644 --- a/test/unit/docker-driver.test.ts +++ b/test/unit/docker-driver.test.ts @@ -1,10 +1,20 @@ +import { mkdtemp, writeFile } from 'node:fs/promises'; +import * as os from 'node:os'; +import * as path from 'node:path'; import { PassThrough } from 'node:stream'; -import { describe, expect, it, vi } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import type Docker from 'dockerode'; import * as tar from 'tar-stream'; -import { DockerDriver } from '../../src/driver/docker-driver.js'; +import { + chooseDefaultNetworkRoute, + defaultGatewayFromRouteTable, + detectResourceLimitSupport, + DockerDriver, + hostAddressSeenFromContainers, + podmanMajorVersion, +} from '../../src/driver/docker-driver.js'; import { stubDockerForRun } from './helpers/docker-stubs.js'; /** @@ -284,6 +294,8 @@ describe('DockerDriver.startRun - dev-folder mount composition (actor-driver.md: const driver = new DockerDriver(stub.docker); driver.available = true; + allowDevMountRecheck(driver); + const outcomePromise = driver.startRun( { runId: 'run-mount-1', @@ -300,7 +312,11 @@ describe('DockerDriver.startRun - dev-folder mount composition (actor-driver.md: const [options] = stub.createContainer.mock.calls[0]!; expect(options.HostConfig?.Mounts).toEqual([ { Type: 'bind', Source: '/host/src', Target: '/usr/src/app' }, - { Type: 'volume', Source: '', Target: '/usr/src/app/node_modules' }, + { + Type: 'volume', + Source: expect.stringMatching(/^actor-runtime-node-modules-/), + Target: '/usr/src/app/node_modules', + }, ]); expect(options.HostConfig?.Binds).toBeUndefined(); @@ -335,6 +351,8 @@ describe('DockerDriver.startRun - dev-folder mount composition (actor-driver.md: driver.available = true; const chunks: string[] = []; + allowDevMountRecheck(driver); + const outcomePromise = driver.startRun( { runId: 'run-mount-3', @@ -712,38 +730,103 @@ describe('DockerDriver.ensureProbeImage (actor-driver.md: registration needs no }); }); +/** Go `os.FileMode` type bits as they appear in the stat header's `mode` (`docker-driver.ts`'s + * `GO_MODE_DIR`/`GO_MODE_SYMLINK`); `0o755`/`0o644` below are the permission bits real daemons add. */ +const GO_DIR = 0x80000000 + 0o755; +const GO_FILE = 0o644; +const GO_SYMLINK = 0x08000000 + 0o777; + +/** One stat answer per in-container path the probe may ask about: a mode (+ optional Docker-style + * `linkTarget`), or an Error to reject with. A path with no entry rejects 404 like a real daemon does. */ +type ProbeStatTable = Record; + +function statHeader(mode: number, linkTarget = ''): string { + const stat = { name: 'x', size: 0, mode, mtime: '2026-01-01T00:00:00Z', linkTarget }; + return Buffer.from(JSON.stringify(stat), 'utf8').toString('base64'); +} + +function http404(message: string): Error { + return Object.assign(new Error(`(HTTP code 404) no such container - ${message} `), { statusCode: 404 }); +} + +/** + * A stub `dockerode`-shaped object covering what `probeDevFolder` calls: `createContainer`, then + * `container.infoArchive({ path })` (the `HEAD .../archive` stat - answered from `table`, with the + * header a real daemon sets) and `container.remove()`. `infoArchive` resolves with the raw + * `http.IncomingMessage`-shaped `{ headers, resume }` dockerode hands back for that `HEAD` call. + */ +function stubDockerForProbe(table: ProbeStatTable, options: { createError?: Error; removeError?: Error } = {}) { + const start = vi.fn(); + const resume = vi.fn(); + const remove = vi.fn(async () => { + if (options.removeError) throw options.removeError; + }); + const infoArchive = vi.fn(async ({ path: containerPath }: { path: string }) => { + const entry = table[containerPath]; + if (!entry) throw http404(`Could not find the file ${containerPath} in container probe-id`); + if (entry instanceof Error) throw entry; + return { headers: { 'x-docker-container-path-stat': statHeader(entry.mode, entry.linkTarget) }, resume }; + }); + const createContainer = vi.fn(async (_options: Docker.ContainerCreateOptions) => { + if (options.createError) throw options.createError; + return { id: 'probe-id', remove, start, infoArchive }; + }); + return { + docker: { createContainer } as unknown as Docker, + createContainer, + infoArchive, + remove, + start, + resume, + statedPaths: () => infoArchive.mock.calls.map(([call]) => call.path), + }; +} + describe('DockerDriver.probeDevFolder (actor-driver.md: "A host-side existence-and-directory check")', () => { - it('returns ok and removes the (never-started) probe container on success, without ever calling .start()', async () => { - const start = vi.fn(); - const remove = vi.fn(async () => undefined); - // Typed with the real `dockerode` parameter shape so `mock.calls[0]` is genuinely a - // `[Docker.ContainerCreateOptions]` tuple below - no unsound cast needed to read it back. - const createContainer = vi.fn(async (_options: Docker.ContainerCreateOptions) => ({ remove, start })); - const driver = new DockerDriver({ createContainer } as unknown as Docker); + it('binds the host root read-only at /probe (never the candidate itself - Podman would auto-create a missing one), stats the candidate component by component, and removes the never-started container', async () => { + const stub = stubDockerForProbe({ '/probe/abs': { mode: GO_DIR }, '/probe/abs/path': { mode: GO_DIR } }); + const driver = new DockerDriver(stub.docker); driver.available = true; const outcome = await driver.probeDevFolder('/abs/path', 'image:tag'); expect(outcome).toEqual({ ok: true }); - expect(createContainer).toHaveBeenCalledTimes(1); - const [options] = createContainer.mock.calls[0]!; + expect(stub.createContainer).toHaveBeenCalledTimes(1); + const [options] = stub.createContainer.mock.calls[0]!; expect(options.Image).toBe('image:tag'); - // `/.` appended to the candidate path (directive: "the probe must accept ONLY directories") - the - // stored/returned path itself is never affected, only this internal probe `Source`. - expect(options.HostConfig?.Mounts).toEqual([ - { Type: 'bind', Source: '/abs/path/.', Target: '/probe', ReadOnly: true }, - ]); - expect(remove).toHaveBeenCalledTimes(1); - expect(start).not.toHaveBeenCalled(); + expect(options.HostConfig?.Mounts).toEqual([{ Type: 'bind', Source: '/', Target: '/probe', ReadOnly: true }]); expect(options.Labels).toEqual({ 'actor-runtime.devFolderProbe': 'true' }); + expect(stub.statedPaths()).toEqual(['/probe/abs', '/probe/abs/path']); + expect(stub.remove).toHaveBeenCalledTimes(1); + expect(stub.start).not.toHaveBeenCalled(); + // The bodiless HEAD response is still consumed so its socket is released. + expect(stub.resume).toHaveBeenCalled(); + }); + + it('normalizes a trailing slash and repeated separators away rather than stat-ing empty components', async () => { + const stub = stubDockerForProbe({ '/probe/abs': { mode: GO_DIR }, '/probe/abs/path': { mode: GO_DIR } }); + const driver = new DockerDriver(stub.docker); + driver.available = true; + + expect(await driver.probeDevFolder('/abs//path/', 'image:tag')).toEqual({ ok: true }); + expect(stub.statedPaths()).toEqual(['/probe/abs', '/probe/abs/path']); + }); + + it('accepts / itself by stat-ing the mount root', async () => { + const stub = stubDockerForProbe({ '/probe': { mode: GO_DIR } }); + const driver = new DockerDriver(stub.docker); + driver.available = true; + + expect(await driver.probeDevFolder('/', 'image:tag')).toEqual({ ok: true }); + expect(stub.statedPaths()).toEqual(['/probe']); }); it('still reports ok when the probe container was created but its removal fails, and logs the failure instead of swallowing it', async () => { - const remove = vi.fn(async () => { - throw new Error('removal failed: container already stopping'); - }); - const createContainer = vi.fn(async () => ({ id: 'probe-id', remove, start: vi.fn() })); - const driver = new DockerDriver({ createContainer } as unknown as Docker); + const stub = stubDockerForProbe( + { '/probe/abs': { mode: GO_DIR }, '/probe/abs/path': { mode: GO_DIR } }, + { removeError: new Error('removal failed: container already stopping') }, + ); + const driver = new DockerDriver(stub.docker); driver.available = true; const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); @@ -755,107 +838,365 @@ describe('DockerDriver.probeDevFolder (actor-driver.md: "A host-side existence-a warn.mockRestore(); }); - it('never even calls createContainer when the driver already knows Docker is unavailable - short-circuits to unreachable', async () => { - const createContainer = vi.fn(async () => ({ remove: vi.fn() })); - const driver = new DockerDriver({ createContainer } as unknown as Docker); + it('never even calls createContainer when the driver already knows the daemon is unavailable - short-circuits to unreachable', async () => { + const stub = stubDockerForProbe({}); + const driver = new DockerDriver(stub.docker); // driver.available defaults to false - init() never ran. - const outcome = await driver.probeDevFolder('/abs/path', 'image:tag'); + expect(await driver.probeDevFolder('/abs/path', 'image:tag')).toEqual({ ok: false, reason: 'unreachable' }); + expect(stub.createContainer).not.toHaveBeenCalled(); + }); - expect(outcome).toEqual({ ok: false, reason: 'unreachable' }); - expect(createContainer).not.toHaveBeenCalled(); + it('classifies a createContainer rejection with no .statusCode as unreachable (a raw transport failure), never as "does not exist"', async () => { + const stub = stubDockerForProbe({}, { createError: new Error('connect ECONNREFUSED /var/run/docker.sock') }); + const driver = new DockerDriver(stub.docker); + driver.available = true; + + expect(await driver.probeDevFolder('/abs/path', 'image:tag')).toEqual({ ok: false, reason: 'unreachable' }); }); - it('classifies a rejection with no .statusCode as unreachable (a raw transport failure), never as "does not exist"', async () => { - const createContainer = vi.fn(async () => { - throw new Error('connect ECONNREFUSED /var/run/docker.sock'); - }); - const driver = new DockerDriver({ createContainer } as unknown as Docker); + it("classifies a 404 createContainer rejection as image-missing (the probe's own image is gone, an operational fault) - the mount source is always /, so a create rejection is never about the candidate", async () => { + const stub = stubDockerForProbe({}, { createError: http404('no such image: image:tag') }); + const driver = new DockerDriver(stub.docker); driver.available = true; - const outcome = await driver.probeDevFolder('/abs/path', 'image:tag'); + expect(await driver.probeDevFolder('/abs/path', 'image:tag')).toEqual({ ok: false, reason: 'image-missing' }); + }); + + it('classifies any other answered createContainer rejection as unknown', async () => { + const stub = stubDockerForProbe( + {}, + { createError: Object.assign(new Error('(HTTP code 500) server error - boom'), { statusCode: 500 }) }, + ); + const driver = new DockerDriver(stub.docker); + driver.available = true; - expect(outcome).toEqual({ ok: false, reason: 'unreachable' }); + expect(await driver.probeDevFolder('/abs/path', 'image:tag')).toEqual({ ok: false, reason: 'unknown' }); }); - it("classifies a 404 rejection as image-missing (the probe's own image is gone, an operational fault)", async () => { - const createContainer = vi.fn(async () => { - throw Object.assign(new Error('(HTTP code 404) no such image: image:tag'), { statusCode: 404 }); - }); - const driver = new DockerDriver({ createContainer } as unknown as Docker); + it("classifies the daemon's 404 for the final component as not-found - the one case allowed to say so - and still removes the probe container", async () => { + const stub = stubDockerForProbe({ '/probe/abs': { mode: GO_DIR } }); + const driver = new DockerDriver(stub.docker); driver.available = true; - const outcome = await driver.probeDevFolder('/abs/path', 'image:tag'); + expect(await driver.probeDevFolder('/abs/path', 'image:tag')).toEqual({ ok: false, reason: 'not-found' }); + expect(stub.statedPaths()).toEqual(['/probe/abs', '/probe/abs/path']); + expect(stub.remove).toHaveBeenCalledTimes(1); + }); + + it('stops at the first missing intermediate component, also as not-found', async () => { + const stub = stubDockerForProbe({}); + const driver = new DockerDriver(stub.docker); + driver.available = true; - expect(outcome).toEqual({ ok: false, reason: 'image-missing' }); + expect(await driver.probeDevFolder('/abs/path', 'image:tag')).toEqual({ ok: false, reason: 'not-found' }); + expect(stub.statedPaths()).toEqual(['/probe/abs']); }); - it('classifies the exact "bind source path does not exist" substring as not-found - the one case allowed to say so', async () => { - const createContainer = vi.fn(async () => { - throw Object.assign( - new Error( - '(HTTP code 400) client error - invalid mount config for type "bind": bind source path does not exist: /abs/path ', - ), - { statusCode: 400 }, - ); + it('classifies a regular file candidate as not-a-directory, never as not-found', async () => { + const stub = stubDockerForProbe({ '/probe/abs': { mode: GO_DIR }, '/probe/abs/file.txt': { mode: GO_FILE } }); + const driver = new DockerDriver(stub.docker); + driver.available = true; + + expect(await driver.probeDevFolder('/abs/file.txt', 'image:tag')).toEqual({ + ok: false, + reason: 'not-a-directory', + }); + }); + + it('classifies a stat rejection with no .statusCode as unreachable, and any other answered non-404 rejection as unknown - never as "does not exist"', async () => { + const transport = stubDockerForProbe({ '/probe/abs': new Error('socket hang up') }); + const transportDriver = new DockerDriver(transport.docker); + transportDriver.available = true; + expect(await transportDriver.probeDevFolder('/abs/path', 'image:tag')).toEqual({ + ok: false, + reason: 'unreachable', + }); + + const denied = stubDockerForProbe({ + '/probe/abs': Object.assign(new Error('(HTTP code 500) server error - permission denied'), { + statusCode: 500, + }), }); - const driver = new DockerDriver({ createContainer } as unknown as Docker); + const deniedDriver = new DockerDriver(denied.docker); + deniedDriver.available = true; + expect(await deniedDriver.probeDevFolder('/abs/path', 'image:tag')).toEqual({ ok: false, reason: 'unknown' }); + }); + + it('classifies a stat response without a parseable X-Docker-Container-Path-Stat header as unknown', async () => { + const stub = stubDockerForProbe({}); + stub.infoArchive.mockResolvedValueOnce({ headers: {}, resume: vi.fn() }); + const driver = new DockerDriver(stub.docker); driver.available = true; - const outcome = await driver.probeDevFolder('/abs/path', 'image:tag'); + expect(await driver.probeDevFolder('/abs', 'image:tag')).toEqual({ ok: false, reason: 'unknown' }); + }); - expect(outcome).toEqual({ ok: false, reason: 'not-found' }); + describe('symlinks (Docker reports a symlink component as such, with a container-scoped linkTarget; Podman follows it itself)', () => { + it('follows a symlink whose target the daemon reports under the probe mount, then keeps walking the remaining components', async () => { + const stub = stubDockerForProbe({ + '/probe/home': { mode: GO_DIR }, + '/probe/home/link': { mode: GO_SYMLINK, linkTarget: '/probe/data/real' }, + '/probe/data': { mode: GO_DIR }, + '/probe/data/real': { mode: GO_DIR }, + '/probe/data/real/sub': { mode: GO_DIR }, + }); + const driver = new DockerDriver(stub.docker); + driver.available = true; + + expect(await driver.probeDevFolder('/home/link/sub', 'image:tag')).toEqual({ ok: true }); + expect(stub.statedPaths()).toEqual([ + '/probe/home', + '/probe/home/link', + '/probe/data', + '/probe/data/real', + '/probe/data/real/sub', + ]); + }); + + it('treats a linkTarget that escaped the probe mount (a host-absolute target, reported verbatim) as a host path and re-stats it under the mount', async () => { + const stub = stubDockerForProbe({ + '/probe/home': { mode: GO_DIR }, + '/probe/home/link': { mode: GO_SYMLINK, linkTarget: '/data/real' }, + '/probe/data': { mode: GO_DIR }, + '/probe/data/real': { mode: GO_DIR }, + }); + const driver = new DockerDriver(stub.docker); + driver.available = true; + + expect(await driver.probeDevFolder('/home/link', 'image:tag')).toEqual({ ok: true }); + expect(stub.statedPaths()).toEqual(['/probe/home', '/probe/home/link', '/probe/data', '/probe/data/real']); + }); + + it('a symlink to a regular file is not-a-directory; a dangling symlink is not-found', async () => { + const toFile = stubDockerForProbe({ + '/probe/link': { mode: GO_SYMLINK, linkTarget: '/probe/file.txt' }, + '/probe/file.txt': { mode: GO_FILE }, + }); + const toFileDriver = new DockerDriver(toFile.docker); + toFileDriver.available = true; + expect(await toFileDriver.probeDevFolder('/link', 'image:tag')).toEqual({ + ok: false, + reason: 'not-a-directory', + }); + + const dangling = stubDockerForProbe({ '/probe/link': { mode: GO_SYMLINK, linkTarget: '/probe/nowhere' } }); + const danglingDriver = new DockerDriver(dangling.docker); + danglingDriver.available = true; + expect(await danglingDriver.probeDevFolder('/link', 'image:tag')).toEqual({ + ok: false, + reason: 'not-found', + }); + }); + + it('gives up on a symlink loop as unknown after a bounded number of hops, never spinning forever', async () => { + const stub = stubDockerForProbe({ '/probe/loop': { mode: GO_SYMLINK, linkTarget: '/probe/loop' } }); + const driver = new DockerDriver(stub.docker); + driver.available = true; + + expect(await driver.probeDevFolder('/loop', 'image:tag')).toEqual({ ok: false, reason: 'unknown' }); + expect(stub.infoArchive.mock.calls.length).toBeLessThanOrEqual(20); + expect(stub.remove).toHaveBeenCalledTimes(1); + }); + + it('a symlink reported with an empty linkTarget is unknown - never followed to /', async () => { + const stub = stubDockerForProbe({ '/probe/link': { mode: GO_SYMLINK, linkTarget: '' } }); + const driver = new DockerDriver(stub.docker); + driver.available = true; + + expect(await driver.probeDevFolder('/link', 'image:tag')).toEqual({ ok: false, reason: 'unknown' }); + }); }); +}); - it('classifies a differently-worded "must be a directory" rejection as unknown, never as not-a-directory - only the exact "not a directory" substring is', async () => { - const createContainer = vi.fn(async () => { - throw Object.assign( - new Error( - '(HTTP code 400) client error - invalid mount config for type "bind": source path must be a directory', - ), - { statusCode: 400 }, - ); +describe('DockerDriver.startRun - an image entrypoint the dev-folder mount would hide (actor-driver.md: "stays available to the run")', () => { + const devMountRun = { + runId: 'run-entry', + imageId: 'fake-image', + env: {}, + memoryMbytes: 128, + timeoutSecs: 60, + devMount: { localDevFolder: '/host/src', imageWorkingDirectory: '/usr/src/app' }, + }; + + function scriptArchive(name: string, content: string): NodeJS.ReadableStream { + const pack = tar.pack(); + pack.entry({ name, mode: 0o755 }, content); + pack.finalize(); + return pack as unknown as NodeJS.ReadableStream; + } + + async function entryNamesOf(archive: Buffer): Promise> { + const entries: Array<{ name: string; type?: string; content: string }> = []; + const extract = tar.extract(); + await new Promise((resolve, reject) => { + extract.on('entry', (header, stream, next) => { + const chunks: Buffer[] = []; + stream.on('data', (chunk: Buffer) => chunks.push(chunk)); + stream.on('end', () => { + entries.push({ name: header.name, type: header.type, content: Buffer.concat(chunks).toString() }); + next(); + }); + stream.resume(); + }); + extract.once('finish', resolve); + extract.once('error', reject); + extract.end(archive); }); - const driver = new DockerDriver({ createContainer } as unknown as Docker); + return entries; + } + + it('a relative entrypoint inside the working directory that the dev folder lacks: the file comes out of the image, lands in the container before start, and the run starts through that copy - the run log says so', async () => { + const stub = stubDockerForRun(); + stub.imageInspect.mockResolvedValue({ + Config: { + Entrypoint: ['./xvfb-entrypoint.sh'], + Cmd: ['python', '-m', 'my_actor'], + WorkingDir: '/usr/src/app', + }, + }); + stub.container.getArchive.mockResolvedValue(scriptArchive('xvfb-entrypoint.sh', '#!/bin/sh\nexec "$@"\n')); + const driver = new DockerDriver(stub.docker); driver.available = true; + allowDevMountRecheck(driver); + const hasEntry = vi.spyOn(driver, 'devFolderHasEntry').mockResolvedValue(false); + const logged: string[] = []; - const outcome = await driver.probeDevFolder('/abs/path', 'image:tag'); + const outcomePromise = driver.startRun(devMountRun, (chunk) => logged.push(chunk)); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + stub.triggerContainerExit(0); + stub.endLogStream(); + await outcomePromise; + + expect(hasEntry).toHaveBeenCalledWith('/host/src', './xvfb-entrypoint.sh'); + expect(stub.container.getArchive).toHaveBeenCalledWith({ path: '/usr/src/app/xvfb-entrypoint.sh' }); + // Two containers: the throwaway one the file is read from, then the run's own. + expect(stub.createContainer).toHaveBeenCalledTimes(2); + const runOptions = stub.createContainer.mock.calls[1]![0]; + expect(runOptions.Entrypoint).toEqual(['/apify-runtime-entrypoint/xvfb-entrypoint.sh']); + // Restated: an engine drops the image's Cmd from a create request that overrides Entrypoint. + expect(runOptions.Cmd).toEqual(['python', '-m', 'my_actor']); + const [archive, putOptions] = stub.container.putArchive.mock.calls[0]!; + expect(putOptions).toEqual({ path: '/' }); + expect(await entryNamesOf(archive as Buffer)).toEqual([ + { name: 'apify-runtime-entrypoint', type: 'directory', content: '' }, + { name: 'apify-runtime-entrypoint/xvfb-entrypoint.sh', type: 'file', content: '#!/bin/sh\nexec "$@"\n' }, + ]); + expect(logged.join('')).toContain('starts through ./xvfb-entrypoint.sh in its working directory'); + }); - expect(outcome).toEqual({ ok: false, reason: 'unknown' }); + it('the dev folder providing the file itself, an absolute entrypoint, or a PATH-resolved one: nothing is preserved and the image command stands', async () => { + for (const [config, devFolderHasIt] of [ + [{ Entrypoint: ['./xvfb-entrypoint.sh'], WorkingDir: '/usr/src/app' }, true], + [{ Entrypoint: ['/usr/local/bin/xvfb-run', 'node', 'main.js'], WorkingDir: '/usr/src/app' }, false], + [{ Cmd: ['npm', 'start'], WorkingDir: '/usr/src/app' }, false], + ] as Array<[Record, boolean]>) { + const stub = stubDockerForRun(); + stub.imageInspect.mockResolvedValue({ Config: config }); + const driver = new DockerDriver(stub.docker); + driver.available = true; + allowDevMountRecheck(driver); + vi.spyOn(driver, 'devFolderHasEntry').mockResolvedValue(devFolderHasIt); + + const outcomePromise = driver.startRun(devMountRun, () => {}); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + stub.triggerContainerExit(0); + stub.endLogStream(); + await outcomePromise; + + expect(stub.container.getArchive).not.toHaveBeenCalled(); + expect(stub.createContainer).toHaveBeenCalledTimes(1); + expect(stub.createContainer.mock.calls[0]![0].Entrypoint).toBeUndefined(); + expect(stub.container.putArchive).not.toHaveBeenCalled(); + } }); +}); - it('classifies the exact "not a directory" substring as not-a-directory - a regular file candidate, discriminated by the appended "/." (verified empirically against a real daemon)', async () => { - const createContainer = vi.fn(async () => { - throw Object.assign( - new Error( - '(HTTP code 400) bad parameter - invalid mount config for type "bind": stat /abs/path/.: not a directory', - ), - { statusCode: 400 }, - ); - }); - const driver = new DockerDriver({ createContainer } as unknown as Docker); +/** Lets a `startRun` test with a `devMount` get past the run-start dev-folder re-check + * (`assertDevFolderStillPresent`) when that check is not what the test is about. */ +function allowDevMountRecheck(driver: DockerDriver): void { + vi.spyOn(driver, 'ensureProbeImage').mockResolvedValue('probe:image'); + vi.spyOn(driver, 'probeDevFolder').mockResolvedValue({ ok: true }); +} + +describe('DockerDriver.startRun - run-start dev-folder re-check (actor-driver.md: "If the registered folder has since been deleted ... the run must fail visibly - never silently mount an empty directory")', () => { + const devMountRun = { + runId: 'run-recheck', + imageId: 'fake-image', + env: {}, + memoryMbytes: 128, + timeoutSecs: 60, + devMount: { localDevFolder: '/host/src', imageWorkingDirectory: '/usr/src/app' }, + }; + + it('re-probes the registered folder with the same probe registration used, before creating any container', async () => { + const stub = stubDockerForRun(); + const driver = new DockerDriver(stub.docker); driver.available = true; + const ensureProbeImage = vi.spyOn(driver, 'ensureProbeImage').mockResolvedValue('probe:image'); + const probeDevFolder = vi.spyOn(driver, 'probeDevFolder').mockResolvedValue({ ok: true }); - const outcome = await driver.probeDevFolder('/abs/path', 'image:tag'); + const outcomePromise = driver.startRun(devMountRun, () => {}); + await new Promise((resolve) => setImmediate(resolve)); + + expect(ensureProbeImage).toHaveBeenCalledTimes(1); + expect(probeDevFolder).toHaveBeenCalledWith('/host/src', 'probe:image'); + expect(probeDevFolder.mock.invocationCallOrder[0]).toBeLessThan( + stub.createContainer.mock.invocationCallOrder[0]!, + ); - expect(outcome).toEqual({ ok: false, reason: 'not-a-directory' }); + stub.triggerContainerExit(0); + stub.endLogStream(); + await outcomePromise; }); - it('classifies a Docker Desktop file-sharing denial (a real, existing path) as unknown, never as not-found - the false-negative this design deliberately avoids', async () => { - const createContainer = vi.fn(async () => { - throw Object.assign( - new Error( - '(HTTP code 400) client error - Mounts denied: The path /abs/path is not shared from the host and is not known to Docker.', - ), - { statusCode: 400 }, - ); - }); - const driver = new DockerDriver({ createContainer } as unknown as Docker); + it('fails the run before any container exists when the folder is gone, naming the folder, the reason, and how to clear the registration', async () => { + const stub = stubDockerForRun(); + const driver = new DockerDriver(stub.docker); driver.available = true; + vi.spyOn(driver, 'ensureProbeImage').mockResolvedValue('probe:image'); + vi.spyOn(driver, 'probeDevFolder').mockResolvedValue({ ok: false, reason: 'not-found' }); - const outcome = await driver.probeDevFolder('/abs/path', 'image:tag'); + await expect(driver.startRun(devMountRun, () => {})).rejects.toThrow( + /registered local dev folder \/host\/src no longer exists on the host.*run was not started.*\/actor-runtime\/dev-folder\//, + ); + expect(stub.createContainer).not.toHaveBeenCalled(); + }); - expect(outcome).toEqual({ ok: false, reason: 'unknown' }); + it('fails the same way for a folder that became a file, and for one the daemon could not verify at all - never starting against whatever the daemon would mount instead', async () => { + for (const [reason, phrase] of [ + ['not-a-directory', 'is no longer a directory'], + ['unreachable', 'could not be verified on the host (unreachable)'], + ['unknown', 'could not be verified on the host (unknown)'], + ] as const) { + const stub = stubDockerForRun(); + const driver = new DockerDriver(stub.docker); + driver.available = true; + vi.spyOn(driver, 'ensureProbeImage').mockResolvedValue('probe:image'); + vi.spyOn(driver, 'probeDevFolder').mockResolvedValue({ ok: false, reason }); + + await expect(driver.startRun(devMountRun, () => {})).rejects.toThrow(phrase); + expect(stub.createContainer).not.toHaveBeenCalled(); + } + }); + + it('does not touch the probe at all for a run with no devMount', async () => { + const stub = stubDockerForRun(); + const driver = new DockerDriver(stub.docker); + driver.available = true; + const ensureProbeImage = vi.spyOn(driver, 'ensureProbeImage'); + const probeDevFolder = vi.spyOn(driver, 'probeDevFolder'); + + const outcomePromise = driver.startRun({ ...devMountRun, devMount: undefined }, () => {}); + await new Promise((resolve) => setImmediate(resolve)); + stub.triggerContainerExit(0); + stub.endLogStream(); + await outcomePromise; + + expect(ensureProbeImage).not.toHaveBeenCalled(); + expect(probeDevFolder).not.toHaveBeenCalled(); }); }); @@ -1083,3 +1424,417 @@ describe('DockerDriver host-capacity warning (actor-driver.md: warn, never clamp await outcomePromise; }); }); + +/** + * `init()` + `startRun()` stub with a controllable `getNetwork()` (`inspect`/`connect`), for + * `selfAttachToNetwork`'s three outcomes and the `ExtraHosts` fallback `startRun` derives from them. + */ +function stubDockerForNetwork( + network: { inspect: () => Promise; connect: () => Promise }, + selfInspect: () => Promise = async () => ({}), +) { + const run = stubDockerForRun(); + const getNetwork = vi.fn(() => network); + const getContainer = vi.fn(() => ({ inspect: selfInspect })); + const docker = { + ...run.docker, + ping: vi.fn(async () => undefined), + listNetworks: vi.fn(async () => []), + createNetwork: vi.fn(async () => undefined), + info: vi.fn(async () => ({})), + getNetwork, + getContainer, + } as unknown as Docker; + return { ...run, docker, getNetwork, getContainer }; +} + +const SELF_FULL_ID = 'abc123def456789000000000000000000000000000000000000000000000000000'; +/** A hosts file with no engine-provided host entry - the Docker Engine case, where `host-gateway` is the route. */ +const NO_HOSTS_FILE = '/nonexistent/hosts'; + +function attachedNetwork() { + return { + inspect: vi.fn(async () => ({ Containers: { [SELF_FULL_ID]: {} } })), + connect: vi.fn(async () => undefined), + }; +} + +function selfOnNetwork(endpoint: { Aliases?: string[]; IPAddress?: string }) { + return async () => ({ NetworkSettings: { Networks: { 'apify-local': endpoint } } }); +} + +describe('DockerDriver - how Actor containers reach the API (network alias, or the host-gateway fallback)', () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + async function extraHostsOfOneRun(stub: ReturnType, driver: DockerDriver) { + const outcomePromise = driver.startRun( + { runId: 'run-reach', imageId: 'fake-image', env: {}, memoryMbytes: 128, timeoutSecs: 60 }, + () => {}, + ); + await new Promise((resolve) => setImmediate(resolve)); + stub.triggerContainerExit(0); + stub.endLogStream(); + await outcomePromise; + const [options] = stub.createContainer.mock.calls[0]!; + return options.HostConfig?.ExtraHosts; + } + + it('with no HOSTNAME (the runtime running on the host, not in a container) warns once and gives every run container an apify-api -> host-gateway extra host', async () => { + vi.stubEnv('HOSTNAME', ''); + const network = { inspect: vi.fn(async () => ({ Containers: {} })), connect: vi.fn(async () => undefined) }; + const stub = stubDockerForNetwork(network); + const driver = new DockerDriver(stub.docker, { hostsFile: NO_HOSTS_FILE }); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + + await driver.init(); + + expect(driver.available).toBe(true); + expect(stub.getNetwork).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('host-gateway')); + expect(await extraHostsOfOneRun(stub, driver)).toEqual(['apify-api:host-gateway']); + warn.mockRestore(); + }); + + it('when the connect succeeds, run containers get no ExtraHosts at all - the network alias is the route, and a hosts-file entry would override it', async () => { + vi.stubEnv('HOSTNAME', 'abc123def456'); + const network = { inspect: vi.fn(async () => ({ Containers: {} })), connect: vi.fn(async () => undefined) }; + const stub = stubDockerForNetwork(network); + const driver = new DockerDriver(stub.docker, { hostsFile: NO_HOSTS_FILE }); + + await driver.init(); + + expect(network.connect).toHaveBeenCalledWith({ + Container: 'abc123def456', + EndpointConfig: { Aliases: ['apify-api'] }, + }); + expect(await extraHostsOfOneRun(stub, driver)).toBeUndefined(); + }); + + it('recognises its own container as already attached by full-id prefix (HOSTNAME is the short id) with the alias registered - no connect, no warning, no ExtraHosts', async () => { + vi.stubEnv('HOSTNAME', 'abc123def456'); + const network = attachedNetwork(); + const stub = stubDockerForNetwork( + network, + selfOnNetwork({ Aliases: ['abc123def456', 'apify-api'], IPAddress: '10.89.0.2' }), + ); + const driver = new DockerDriver(stub.docker, { hostsFile: NO_HOSTS_FILE }); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + + await driver.init(); + + expect(network.connect).not.toHaveBeenCalled(); + expect(stub.getContainer).toHaveBeenCalledWith('abc123def456'); + expect(warn).not.toHaveBeenCalled(); + expect(await extraHostsOfOneRun(stub, driver)).toBeUndefined(); + warn.mockRestore(); + }); + + it("started on the network without the alias (`--network apify-local`, no `--network-alias`): run containers get apify-api -> this container's own address, no connect, no warning", async () => { + vi.stubEnv('HOSTNAME', 'abc123def456'); + const network = attachedNetwork(); + const stub = stubDockerForNetwork( + network, + selfOnNetwork({ Aliases: ['abc123def456'], IPAddress: '10.89.0.2' }), + ); + const driver = new DockerDriver(stub.docker, { hostsFile: NO_HOSTS_FILE }); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + + await driver.init(); + + expect(network.connect).not.toHaveBeenCalled(); + expect(warn).not.toHaveBeenCalled(); + expect(await extraHostsOfOneRun(stub, driver)).toEqual(['apify-api:10.89.0.2']); + warn.mockRestore(); + }); + + it('on the network but with its own address unreadable: warns and falls back to the host-gateway extra host', async () => { + vi.stubEnv('HOSTNAME', 'abc123def456'); + const network = attachedNetwork(); + const stub = stubDockerForNetwork(network, async () => { + throw new Error('inspect failed'); + }); + const driver = new DockerDriver(stub.docker, { hostsFile: NO_HOSTS_FILE }); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + + await driver.init(); + + expect(driver.available).toBe(true); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('host-gateway')); + expect(await extraHostsOfOneRun(stub, driver)).toEqual(['apify-api:host-gateway']); + warn.mockRestore(); + }); + + it('when the engine refuses the attach (rootless Podman: the runtime container runs under slirp4netns), stays available, warns naming the fallback and the --network fix, and gives run containers the host-gateway extra host', async () => { + vi.stubEnv('HOSTNAME', 'abc123def456'); + const network = { + inspect: vi.fn(async () => ({ Containers: {} })), + connect: vi.fn(async () => { + throw Object.assign( + new Error('(HTTP code 500) server error - "slirp4netns" is not supported: invalid network mode '), + { statusCode: 500 }, + ); + }), + }; + const stub = stubDockerForNetwork(network); + const driver = new DockerDriver(stub.docker, { hostsFile: NO_HOSTS_FILE }); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + + await driver.init(); + + expect(driver.available).toBe(true); + const message = warn.mock.calls.map((call) => String(call[0])).join('\n'); + expect(message).toContain('slirp4netns'); + expect(message).toContain('host-gateway'); + expect(message).toContain('--network apify-local'); + expect(await extraHostsOfOneRun(stub, driver)).toEqual(['apify-api:host-gateway']); + warn.mockRestore(); + }); + + it('off the network, routes Actors to the host at the address the engine itself gave this container (host.containers.internal), not host-gateway - Podman before 4.1 rejects the keyword', async () => { + vi.stubEnv('HOSTNAME', ''); + const hostsFile = path.join(await mkdtemp(path.join(os.tmpdir(), 'hosts-')), 'hosts'); + await writeFile( + hostsFile, + '127.0.0.1 localhost\n10.88.0.1\thost.containers.internal host.docker.internal\n10.88.0.7\tabc123 name\n', + ); + const network = { inspect: vi.fn(async () => ({ Containers: {} })), connect: vi.fn(async () => undefined) }; + const stub = stubDockerForNetwork(network); + const driver = new DockerDriver(stub.docker, { hostsFile }); + vi.spyOn(console, 'warn').mockImplementation(() => undefined); + + await driver.init(); + + expect(await extraHostsOfOneRun(stub, driver)).toEqual(['apify-api:10.88.0.1']); + vi.restoreAllMocks(); + }); +}); + +describe("chooseDefaultNetworkRoute (Actors on the engine's default network, `actor-driver.md` Networking)", () => { + it("rootless Podman 3.x under slirp4netns: the engine's host entry is the slirp gateway, which only reaches the host with allow_host_loopback", () => { + expect( + chooseDefaultNetworkRoute({ + hostAddress: '10.0.2.2', + gateway: '10.0.2.2', + own: { address: '10.0.2.100', iface: 'tap0' }, + }), + ).toEqual({ extraHost: 'apify-api:10.0.2.2', networkMode: 'slirp4netns:allow_host_loopback=true' }); + }); + + it("rootful bridge: the host entry is the bridge gateway, so this container's own address on that bridge is the direct route", () => { + expect( + chooseDefaultNetworkRoute({ + hostAddress: '10.88.0.1', + gateway: '10.88.0.1', + own: { address: '10.88.0.5', iface: 'eth0' }, + }), + ).toEqual({ extraHost: 'apify-api:10.88.0.5' }); + }); + + it("otherwise the engine's host entry is the route (rootless Podman 4+: a real host address), or host-gateway without one (Docker Engine)", () => { + expect( + chooseDefaultNetworkRoute({ + hostAddress: '192.168.1.20', + gateway: '10.0.2.2', + own: { address: '10.0.2.100', iface: 'tap0' }, + }), + ).toEqual({ extraHost: 'apify-api:192.168.1.20' }); + expect( + chooseDefaultNetworkRoute({ + hostAddress: undefined, + gateway: '172.17.0.1', + own: { address: '172.17.0.2', iface: 'eth0' }, + }), + ).toEqual({ + extraHost: 'apify-api:host-gateway', + }); + expect(chooseDefaultNetworkRoute({ hostAddress: undefined, gateway: undefined, own: undefined })).toEqual({ + extraHost: 'apify-api:host-gateway', + }); + }); +}); + +describe('defaultGatewayFromRouteTable', () => { + it('decodes the little-endian default gateway of /proc/net/route, and returns undefined without a default route', () => { + const table = + 'Iface\tDestination\tGateway \tFlags\tRefCnt\tUse\tMetric\tMask\t\tMTU\tWindow\tIRTT\n' + + 'tap0\t00000000\t0202000A\t0003\t0\t0\t0\t00000000\t0\t0\t0\n' + + 'tap0\t0002000A\t00000000\t0001\t0\t0\t0\t00FFFFFF\t0\t0\t0\n'; + expect(defaultGatewayFromRouteTable(table)).toBe('10.0.2.2'); + expect(defaultGatewayFromRouteTable('Iface\tDestination\tGateway\neth0\t0002000A\t00000000\n')).toBeUndefined(); + expect(defaultGatewayFromRouteTable('')).toBeUndefined(); + }); +}); + +describe('Podman 3.x: no user-defined network at all', () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it('never creates or joins apify-local; run containers go straight to the default network with the chosen route, and one startup line says so', async () => { + vi.stubEnv('HOSTNAME', 'abc123def456'); + const dir = await mkdtemp(path.join(os.tmpdir(), 'pm3-')); + const hostsFile = path.join(dir, 'hosts'); + await writeFile(hostsFile, '10.0.2.2 host.containers.internal\n'); + const routeFile = path.join(dir, 'route'); + await writeFile(routeFile, 'Iface\tDestination\tGateway\ntap0\t00000000\t0202000A\n'); + const network = { inspect: vi.fn(async () => ({ Containers: {} })), connect: vi.fn(async () => undefined) }; + const stub = stubDockerForNetwork(network); + (stub.docker as unknown as { version: unknown }).version = vi.fn(async () => ({ + Components: [{ Name: 'Podman Engine', Version: '3.4.4' }], + })); + (stub.docker.modem as unknown as { dial: unknown }).dial = vi.fn( + (_o: unknown, cb: (e: Error | null, d: unknown) => void) => + cb(null, { host: { cgroupControllers: ['cpu', 'memory', 'pids'] } }), + ); + const driver = new DockerDriver(stub.docker, { + hostsFile, + routeFile, + networkInterfaces: () => ({ + tap0: [{ address: '10.0.2.100', family: 'IPv4', internal: false } as os.NetworkInterfaceInfo], + }), + }); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + + await driver.init(); + + expect(driver.available).toBe(true); + expect( + (stub.docker as unknown as { createNetwork: ReturnType }).createNetwork, + ).not.toHaveBeenCalled(); + expect(stub.getNetwork).not.toHaveBeenCalled(); + expect(warn.mock.calls.map((c) => String(c[0])).filter((m) => m.startsWith('Podman 3.x'))).toHaveLength(1); + + const outcomePromise = driver.startRun( + { runId: 'run-pm3', imageId: 'fake-image', env: {}, memoryMbytes: 128, timeoutSecs: 60 }, + () => {}, + ); + await new Promise((resolve) => setImmediate(resolve)); + stub.triggerContainerExit(0); + stub.endLogStream(); + await outcomePromise; + + expect(stub.createContainer).toHaveBeenCalledTimes(1); + const hostConfig = stub.createContainer.mock.calls[0]![0].HostConfig!; + expect(hostConfig.NetworkMode).toBe('slirp4netns:allow_host_loopback=true'); + expect(hostConfig.ExtraHosts).toEqual(['apify-api:10.0.2.2']); + warn.mockRestore(); + }); + + it('podmanMajorVersion reads the Podman component and is undefined for Docker or an unreachable engine', async () => { + await expect( + podmanMajorVersion({ + version: async () => ({ Components: [{ Name: 'Podman Engine', Version: '4.9.3' }] }), + } as unknown as Docker), + ).resolves.toBe(4); + await expect( + podmanMajorVersion({ + version: async () => ({ Components: [{ Name: 'Engine', Version: '29.3.1' }] }), + } as unknown as Docker), + ).resolves.toBeUndefined(); + await expect( + podmanMajorVersion({ + version: async () => { + throw new Error('down'); + }, + } as unknown as Docker), + ).resolves.toBeUndefined(); + }); +}); + +describe('resource limits the engine cannot apply are left out (rootless Podman without a delegated cgroup controller)', () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + function stubWithEngine(components: string[], cgroupControllers: unknown) { + vi.stubEnv('HOSTNAME', ''); + const network = { inspect: vi.fn(async () => ({ Containers: {} })), connect: vi.fn(async () => undefined) }; + const stub = stubDockerForNetwork(network); + (stub.docker as unknown as { version: unknown }).version = vi.fn(async () => ({ + Components: components.map((Name) => ({ Name })), + })); + (stub.docker.modem as unknown as { dial: unknown }).dial = vi.fn( + (_options: unknown, callback: (error: Error | null, data: unknown) => void) => + callback(null, { host: { cgroupControllers } }), + ); + return stub; + } + + async function hostConfigOfOneRun(stub: ReturnType, driver: DockerDriver) { + const outcomePromise = driver.startRun( + { runId: 'run-limits', imageId: 'fake-image', env: {}, memoryMbytes: 1024, timeoutSecs: 60 }, + () => {}, + ); + await new Promise((resolve) => setImmediate(resolve)); + stub.triggerContainerExit(0); + stub.endLogStream(); + await outcomePromise; + return stub.createContainer.mock.calls[0]![0].HostConfig!; + } + + it("Podman reporting only memory and pids controllers: the run gets its memory limit but no CPU quota, and init warns once naming 'cpu'", async () => { + const stub = stubWithEngine(['Podman Engine', 'Conmon'], ['memory', 'pids']); + const driver = new DockerDriver(stub.docker, { hostsFile: NO_HOSTS_FILE }); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + await driver.init(); + + expect( + warn.mock.calls.map((call) => String(call[0])).filter((m) => m.includes("'cpu' cgroup controller")), + ).toHaveLength(1); + const hostConfig = await hostConfigOfOneRun(stub, driver); + expect(hostConfig.Memory).toBe(1024 * 1024 * 1024); + expect(hostConfig.CpuQuota).toBeUndefined(); + expect(hostConfig.CpuPeriod).toBeUndefined(); + warn.mockRestore(); + }); + + it('Podman reporting cpu and memory, or Docker (whose /version has no Podman component): every limit is applied, no warning', async () => { + for (const [components, controllers] of [ + [['Podman Engine'], ['cpu', 'memory', 'pids']], + [['Engine', 'containerd'], undefined], + ] as Array<[string[], unknown]>) { + const stub = stubWithEngine(components, controllers); + const driver = new DockerDriver(stub.docker, { hostsFile: NO_HOSTS_FILE }); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + await driver.init(); + expect(warn.mock.calls.map((call) => String(call[0])).some((m) => m.includes('cgroup controller'))).toBe( + false, + ); + const hostConfig = await hostConfigOfOneRun(stub, driver); + expect(hostConfig.Memory).toBe(1024 * 1024 * 1024); + expect(hostConfig.CpuQuota).toBeGreaterThan(0); + warn.mockRestore(); + } + }); + + it('detectResourceLimitSupport keeps every limit when the engine cannot be asked', async () => { + await expect( + detectResourceLimitSupport({ + version: async () => { + throw new Error('no'); + }, + } as unknown as Docker), + ).resolves.toEqual({ + cpu: true, + memory: true, + }); + }); +}); + +describe('hostAddressSeenFromContainers', () => { + it('returns the address of the engine-provided host entry, ignoring comments and other lines, and host-gateway when there is none', async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), 'hosts-')); + const podman = path.join(dir, 'podman'); + await writeFile( + podman, + '# comment\n127.0.0.1 localhost\n192.0.2.2\thost.containers.internal host.docker.internal # engine\n', + ); + const docker = path.join(dir, 'docker'); + await writeFile(docker, '127.0.0.1\tlocalhost\n172.17.0.2\tb276929e2817\n'); + await expect(hostAddressSeenFromContainers(podman)).resolves.toBe('192.0.2.2'); + await expect(hostAddressSeenFromContainers(docker)).resolves.toBe('host-gateway'); + await expect(hostAddressSeenFromContainers(NO_HOSTS_FILE)).resolves.toBe('host-gateway'); + }); +}); diff --git a/test/unit/dockerfile-image-refs.test.ts b/test/unit/dockerfile-image-refs.test.ts new file mode 100644 index 0000000..c1915e6 --- /dev/null +++ b/test/unit/dockerfile-image-refs.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from 'vitest'; + +import { qualifyDockerfileImageReferences, qualifyImageReference } from '../../src/services/dockerfile-image-refs.js'; + +const noStages = new Set(); + +describe('qualifyImageReference (Docker Hub is where a short image name lives, on every engine)', () => { + it('prefixes a namespaced short name with docker.io, keeping tag or digest', () => { + expect(qualifyImageReference('apify/actor-node:20', noStages)).toBe('docker.io/apify/actor-node:20'); + expect(qualifyImageReference('apify/actor-python-playwright:3.14-1.61.0', noStages)).toBe( + 'docker.io/apify/actor-python-playwright:3.14-1.61.0', + ); + expect(qualifyImageReference('apify/actor-node@sha256:abc', noStages)).toBe( + 'docker.io/apify/actor-node@sha256:abc', + ); + }); + + it('puts a single-segment name under library/, with a tag colon never mistaken for a registry port', () => { + expect(qualifyImageReference('python:3.11-slim', noStages)).toBe('docker.io/library/python:3.11-slim'); + expect(qualifyImageReference('alpine', noStages)).toBe('docker.io/library/alpine'); + expect(qualifyImageReference('node@sha256:abc', noStages)).toBe('docker.io/library/node@sha256:abc'); + }); + + it('leaves an already-qualified reference alone: a dotted host, a host with a port, or localhost', () => { + expect(qualifyImageReference('docker.io/apify/actor-node:20', noStages)).toBeUndefined(); + expect(qualifyImageReference('ghcr.io/org/image:1', noStages)).toBeUndefined(); + expect(qualifyImageReference('registry:5000/image:1', noStages)).toBeUndefined(); + expect(qualifyImageReference('localhost/actor-runtime:latest', noStages)).toBeUndefined(); + }); + + it('leaves scratch, a variable, and a build-stage reference alone', () => { + expect(qualifyImageReference('scratch', noStages)).toBeUndefined(); + expect(qualifyImageReference('SCRATCH', noStages)).toBeUndefined(); + expect(qualifyImageReference('${BASE_IMAGE}', noStages)).toBeUndefined(); + expect(qualifyImageReference('$BASE', noStages)).toBeUndefined(); + expect(qualifyImageReference('builder', new Set(['builder']))).toBeUndefined(); + expect(qualifyImageReference('Builder', new Set(['builder']))).toBeUndefined(); + }); +}); + +describe('qualifyDockerfileImageReferences', () => { + it('rewrites only the image token of each FROM line, preserving flags, stage names, spacing, and every other line', () => { + const dockerfile = [ + '# syntax=docker/dockerfile:1', + 'ARG BASE=apify/actor-node:20', + 'FROM --platform=$BUILDPLATFORM apify/actor-node:20 AS builder', + 'RUN echo "FROM inside a string is not an instruction"', + 'from python:3.11-slim as tools', + 'FROM ${BASE}', + 'FROM builder', + 'FROM scratch', + 'COPY --from=builder /app /app', + 'FROM ghcr.io/org/image:1', + '', + ].join('\n'); + + const result = qualifyDockerfileImageReferences(dockerfile); + + expect(result.dockerfile.split('\n')).toEqual([ + '# syntax=docker/dockerfile:1', + 'ARG BASE=apify/actor-node:20', + 'FROM --platform=$BUILDPLATFORM docker.io/apify/actor-node:20 AS builder', + 'RUN echo "FROM inside a string is not an instruction"', + 'from docker.io/library/python:3.11-slim as tools', + 'FROM ${BASE}', + 'FROM builder', + 'FROM scratch', + 'COPY --from=builder /app /app', + 'FROM ghcr.io/org/image:1', + '', + ]); + expect(result.qualified).toEqual([ + { from: 'apify/actor-node:20', to: 'docker.io/apify/actor-node:20' }, + { from: 'python:3.11-slim', to: 'docker.io/library/python:3.11-slim' }, + ]); + }); + + it('does not treat a later stage name as a stage before it is declared, and keeps a Windows line ending intact', () => { + const result = qualifyDockerfileImageReferences('FROM base\r\nFROM alpine AS base\r\n'); + expect(result.dockerfile).toBe('FROM docker.io/library/base\r\nFROM docker.io/library/alpine AS base\r\n'); + }); + + it('returns the Dockerfile unchanged, with nothing qualified, when every FROM is already qualified', () => { + const dockerfile = 'FROM docker.io/apify/actor-node:20\nCMD ["node", "main.js"]\n'; + expect(qualifyDockerfileImageReferences(dockerfile)).toEqual({ dockerfile, qualified: [] }); + }); +}); diff --git a/test/unit/helpers/docker-stubs.ts b/test/unit/helpers/docker-stubs.ts index f7fc0d7..cd66f5f 100644 --- a/test/unit/helpers/docker-stubs.ts +++ b/test/unit/helpers/docker-stubs.ts @@ -1,4 +1,5 @@ import { PassThrough } from 'node:stream'; +import * as tar from 'tar-stream'; import { vi } from 'vitest'; import type Docker from 'dockerode'; @@ -34,6 +35,13 @@ export function stubDockerForRun() { remove: vi.fn(async (_options?: Record) => undefined), stop: vi.fn(async () => undefined), putArchive: vi.fn(async (_file: unknown, _options: unknown) => undefined), + // `preserveHiddenEntrypoint` reads a file out of the image through a created container; an empty + // archive by default, tests that exercise it supply their own. + getArchive: vi.fn(async (_options: unknown) => { + const pack = tar.pack(); + pack.finalize(); + return pack as unknown as NodeJS.ReadableStream; + }), }; // Real dockerode demuxing splits stdout/stderr apart by frame header; this stub doesn't need that @@ -51,8 +59,17 @@ export function stubDockerForRun() { // Typed with the real `dockerode` parameter shape so `mock.calls[0]` is genuinely a // `[Docker.ContainerCreateOptions]` tuple below - no unsound cast needed to read it back. const createContainer = vi.fn(async (_options: Docker.ContainerCreateOptions) => container); + // A `devMount` run removes its named `node_modules` volume after the container. + const volumeRemove = vi.fn(async (_options?: Record) => undefined); + const getVolume = vi.fn((_name: string) => ({ remove: volumeRemove })); + // A `devMount` run inspects the image for an entrypoint the mount would hide; no `Config` here means + // there is nothing to preserve. + const imageInspect = vi.fn(async () => ({ Config: {} })); + const getImage = vi.fn((_name: string) => ({ inspect: imageInspect })); const docker = { createContainer, + getVolume, + getImage, modem: { demuxStream }, } as unknown as Docker; @@ -60,6 +77,10 @@ export function stubDockerForRun() { docker, container, createContainer, + getVolume, + volumeRemove, + getImage, + imageInspect, /** Simulates `container.wait()` resolving - the container process has exited. */ triggerContainerExit(statusCode = 0): void { resolveWait({ StatusCode: statusCode }); diff --git a/test/unit/resource-sampler.test.ts b/test/unit/resource-sampler.test.ts index 4bad94c..70e5c61 100644 --- a/test/unit/resource-sampler.test.ts +++ b/test/unit/resource-sampler.test.ts @@ -61,10 +61,22 @@ function stubDockerForSampler() { }; } -/** A minimal, valid `dockerode` `ContainerStats`-shaped object with just the fields the sampler reads. */ -function containerStats(totalUsage: number, systemUsage: number, memoryUsage: number, onlineCpus = 1) { +/** + * A minimal, valid `dockerode` `ContainerStats`-shaped object with just the fields the sampler reads. + * `totalUsageMs` is the container's cumulative CPU time in MILLISECONDS (the daemon reports nanoseconds - + * scaled here), so with the suite's one-second fake-timer ticks a delta of 200 between two successive + * samples reads as 20% of one core. No `read` timestamp: the sampler then stamps each read with the + * (fake) process clock, which is exactly what the ticks advance. `systemUsage`/`onlineCpus` are still + * accepted so every caller's shape stays realistic, but the sampler no longer reads either (see + * `cpuUsageSnapshotOf`'s doc comment in `docker-driver.ts`). + */ +function containerStats(totalUsageMs: number, systemUsage: number, memoryUsage: number, onlineCpus = 1) { return { - cpu_stats: { cpu_usage: { total_usage: totalUsage }, system_cpu_usage: systemUsage, online_cpus: onlineCpus }, + cpu_stats: { + cpu_usage: { total_usage: totalUsageMs * 1_000_000 }, + system_cpu_usage: systemUsage, + online_cpus: onlineCpus, + }, memory_stats: { usage: memoryUsage }, }; } @@ -138,14 +150,16 @@ describe('DockerDriver.startRun - per-run resource sampler (onSample)', () => { await outcomePromise; }); - it('scales cpuPercentOfOneCore by online_cpus (the docker stats convention), not just the raw usage-time ratio', async () => { + it("computes cpuPercentOfOneCore as CPU time over wall time - never docker stats' system_cpu_usage/online_cpus formula, whose inputs Podman reports on a different scale", async () => { const stub = stubDockerForSampler(); const driver = new DockerDriver(stub.docker); driver.available = true; + // The Podman-shaped trap: `system_cpu_usage` advancing ~3s and `online_cpus: 4` over a one-second + // tick. The Docker formula would read a full second of CPU time (1000ms) as 1000/3000*4*100 = 133%; + // CPU time over wall time correctly reads it as one core busy the whole tick - 100%. stub.queueStatsResponse(containerStats(0, 0, 100, 4)); - // cpuDelta=1000, systemDelta=4000 -> ratio 0.25, * 4 online cpus * 100 = 100% of one core. - stub.queueStatsResponse(containerStats(1000, 4000, 100, 4)); + stub.queueStatsResponse(containerStats(1000, 3_000_000_000, 100, 4)); const samples: RunResourceSample[] = []; const outcomePromise = driver.startRun( @@ -164,13 +178,14 @@ describe('DockerDriver.startRun - per-run resource sampler (onSample)', () => { await outcomePromise; }); - it('reports 0% (never NaN/Infinity) for the degenerate case of a zero system-time delta between two samples', async () => { + it('reports 0% (never NaN/Infinity) for the degenerate case of a zero wall-time delta between two samples', async () => { const stub = stubDockerForSampler(); const driver = new DockerDriver(stub.docker); driver.available = true; - stub.queueStatsResponse(containerStats(0, 1000, 100)); - stub.queueStatsResponse(containerStats(0, 1000, 100)); // identical - zero delta on both axes + // Both reads stamped by the daemon with the very same instant - zero delta on both axes. + stub.queueStatsResponse({ ...containerStats(0, 1000, 100), read: '2026-01-01T00:00:00.000000000Z' }); + stub.queueStatsResponse({ ...containerStats(0, 1000, 100), read: '2026-01-01T00:00:00.000000000Z' }); const samples: RunResourceSample[] = []; const outcomePromise = driver.startRun( @@ -189,19 +204,19 @@ describe('DockerDriver.startRun - per-run resource sampler (onSample)', () => { await outcomePromise; }); - it('treats a reported online_cpus of 0 as 1 - `@types/dockerode` declares the field non-optional, but this defends against a daemon that reports it as 0 anyway', async () => { + it("measures wall time between the daemon's own `read` timestamps when they parse, not between this process's request times", async () => { const stub = stubDockerForSampler(); const driver = new DockerDriver(stub.docker); driver.available = true; - stub.queueStatsResponse(containerStats(0, 0, 100, 0)); - // cpuDelta=200, systemDelta=1000 -> ratio 0.2. With the online_cpus=0 -> 1 fallback that's 20%; - // without it (multiplying by the raw 0 instead), it would be 0%. - stub.queueStatsResponse(containerStats(200, 1000, 150, 0)); + // The two reads are one fake-timer second apart on this process's clock, but the daemon stamps + // them two seconds apart: 200ms of CPU time over 2s of daemon wall time is 10%, not 20%. + stub.queueStatsResponse({ ...containerStats(0, 0, 100), read: '2026-01-01T00:00:00.000000000Z' }); + stub.queueStatsResponse({ ...containerStats(200, 1000, 150), read: '2026-01-01T00:00:02.000000000Z' }); const samples: RunResourceSample[] = []; const outcomePromise = driver.startRun( - { runId: 'run-sampler-online-cpus-0', imageId: 'fake-image', env: {}, memoryMbytes: 1024, timeoutSecs: 60 }, + { runId: 'run-sampler-daemon-read', imageId: 'fake-image', env: {}, memoryMbytes: 1024, timeoutSecs: 60 }, () => {}, (sample) => samples.push(sample), ); @@ -209,7 +224,7 @@ describe('DockerDriver.startRun - per-run resource sampler (onSample)', () => { await vi.advanceTimersByTimeAsync(1000); expect(samples).toHaveLength(1); - expect(samples[0]?.cpuPercentOfOneCore).toBeCloseTo(20); + expect(samples[0]?.cpuPercentOfOneCore).toBeCloseTo(10); stub.triggerContainerExit(0); stub.endLogStream(); @@ -317,10 +332,11 @@ describe('DockerDriver.startRun - per-run resource sampler (onSample)', () => { expect(samples).toHaveLength(0); // Tick 2 succeeds again, diffed against the BASELINE (the rejected tick returned before ever - // updating `previous`, so this is not diffed against anything from the failed tick). + // updating `previous`, so this is not diffed against anything from the failed tick): 200ms of CPU + // time over the two seconds since the baseline is 10%. await vi.advanceTimersByTimeAsync(1000); expect(samples).toHaveLength(1); - expect(samples[0]?.cpuPercentOfOneCore).toBeCloseTo(20); + expect(samples[0]?.cpuPercentOfOneCore).toBeCloseTo(10); stub.triggerContainerExit(0); stub.endLogStream(); @@ -372,7 +388,7 @@ describe('DockerDriver.startRun - per-run resource sampler (onSample)', () => { await vi.advanceTimersByTimeAsync(1000); expect(samples).toHaveLength(1); expect(samples[0]?.memoryBytes).toBe(150); - expect(samples[0]?.cpuPercentOfOneCore).toBeCloseTo(((250 - 0) / (1500 - 0)) * 100); + expect(samples[0]?.cpuPercentOfOneCore).toBeCloseTo(12.5); // 250ms of CPU time over the 2s since the baseline expect(frames).toHaveLength(1); const data = frames[0]!.data; @@ -458,10 +474,10 @@ describe('DockerDriver.startRun - per-run resource sampler (onSample)', () => { await vi.advanceTimersByTimeAsync(1000); // tick 1: missing total_usage - skipped expect(samples).toHaveLength(0); - // Tick 2 is diffed against the BASELINE, not the skipped tick. + // Tick 2 is diffed against the BASELINE, not the skipped tick: 200ms of CPU time over 2s is 10%. await vi.advanceTimersByTimeAsync(1000); expect(samples).toHaveLength(1); - expect(samples[0]?.cpuPercentOfOneCore).toBeCloseTo(20); + expect(samples[0]?.cpuPercentOfOneCore).toBeCloseTo(10); expect(samples[0]?.memoryBytes).toBe(180); stub.triggerContainerExit(0); @@ -469,65 +485,31 @@ describe('DockerDriver.startRun - per-run resource sampler (onSample)', () => { await outcomePromise; }); - it("skips a tick outright when cpu_stats.system_cpu_usage is missing while total_usage is present and valid - the shape the guard's second clause exists for", async () => { + it('never reads cpu_stats.system_cpu_usage or online_cpus - a missing, NaN, or absurd value there neither skips the tick nor changes the result', async () => { const stub = stubDockerForSampler(); const driver = new DockerDriver(stub.docker); driver.available = true; stub.queueStatsResponse(containerStats(0, 0, 100)); // baseline - // BAD: total_usage present and finite, but system_cpu_usage is absent entirely - covered because - // the daemon's own stats shape is not guaranteed (docker-driver.ts's cpuUsageSnapshotOf doc - // comment), distinct from - and never reaching the same code path as - the already-tested - // "total_usage missing" case above, which returns before system_cpu_usage is even read. + // Three ticks of exactly 200ms CPU time each, with the fields the old formula depended on absent, + // NaN, and nonsensical in turn - all three must still read as 20% of one core. stub.queueStatsResponse({ - cpu_stats: { cpu_usage: { total_usage: 250 }, online_cpus: 1 }, + cpu_stats: { cpu_usage: { total_usage: 200_000_000 } }, memory_stats: { usage: 150 }, }); - stub.queueStatsResponse(containerStats(200, 1000, 180)); // recovers on the next tick - - const samples: RunResourceSample[] = []; - const outcomePromise = driver.startRun( - { - runId: 'run-sampler-missing-system-usage', - imageId: 'fake-image', - env: {}, - memoryMbytes: 1024, - timeoutSecs: 60, - }, - () => {}, - (sample) => samples.push(sample), - ); - - await vi.advanceTimersByTimeAsync(1000); // tick 1: missing system_cpu_usage - skipped - expect(samples).toHaveLength(0); - - // Tick 2 is diffed against the BASELINE, not the skipped tick - proves `previous` was left untouched. - await vi.advanceTimersByTimeAsync(1000); - expect(samples).toHaveLength(1); - expect(samples[0]?.cpuPercentOfOneCore).toBeCloseTo(20); - expect(samples[0]?.memoryBytes).toBe(180); - - stub.triggerContainerExit(0); - stub.endLogStream(); - await outcomePromise; - }); - - it('skips a tick outright when cpu_stats.system_cpu_usage is present but not a finite number (e.g. NaN) - the same guard as a missing field, not just an absent one', async () => { - const stub = stubDockerForSampler(); - const driver = new DockerDriver(stub.docker); - driver.available = true; - - stub.queueStatsResponse(containerStats(0, 0, 100)); // baseline stub.queueStatsResponse({ - cpu_stats: { cpu_usage: { total_usage: 250 }, system_cpu_usage: Number.NaN, online_cpus: 1 }, - memory_stats: { usage: 150 }, + cpu_stats: { cpu_usage: { total_usage: 400_000_000 }, system_cpu_usage: Number.NaN, online_cpus: 1 }, + memory_stats: { usage: 160 }, + }); + stub.queueStatsResponse({ + cpu_stats: { cpu_usage: { total_usage: 600_000_000 }, system_cpu_usage: 1, online_cpus: 0 }, + memory_stats: { usage: 170 }, }); - stub.queueStatsResponse(containerStats(200, 1000, 180)); // recovers on the next tick const samples: RunResourceSample[] = []; const outcomePromise = driver.startRun( { - runId: 'run-sampler-nan-system-usage', + runId: 'run-sampler-no-system-usage', imageId: 'fake-image', env: {}, memoryMbytes: 1024, @@ -538,12 +520,11 @@ describe('DockerDriver.startRun - per-run resource sampler (onSample)', () => { ); await vi.advanceTimersByTimeAsync(1000); - expect(samples).toHaveLength(0); - await vi.advanceTimersByTimeAsync(1000); - expect(samples).toHaveLength(1); - expect(samples[0]?.cpuPercentOfOneCore).toBeCloseTo(20); - expect(samples[0]?.memoryBytes).toBe(180); + await vi.advanceTimersByTimeAsync(1000); + + expect(samples.map((s) => s.cpuPercentOfOneCore)).toEqual([20, 20, 20].map((n) => expect.closeTo(n))); + expect(samples.map((s) => s.memoryBytes)).toEqual([150, 160, 170]); stub.triggerContainerExit(0); stub.endLogStream(); @@ -588,7 +569,7 @@ describe('DockerDriver.startRun - per-run resource sampler (onSample)', () => { await vi.advanceTimersByTimeAsync(1000); expect(samples).toHaveLength(1); - expect(samples[0]?.cpuPercentOfOneCore).toBeCloseTo(20); + expect(samples[0]?.cpuPercentOfOneCore).toBeCloseTo(10); // 200ms of CPU time over the 2s since the baseline stub.triggerContainerExit(0); stub.endLogStream();