Containerd content store is not accessible on macOS

[malt3]

Image loading via docker load is slow (not incremental), so we recommend to users to make the containerd socket available to the image loader.
This way, the containerd content store can be used (if available).
On macOS, the containerd socket lives inside a Linux virtual machine, so it's not easily accessible from the host.

Docker is working on a feature to expose the content store directly on the docker socket, which will solve this problem on macOS at some point.
Meanwhile, I suggest either sticking to the (less performant) docker load fallback or using workarounds to proxy access to the containerd socket.

[Geethree]

👋 Thanks for creating this Malte. The "docker runs in a linux vm on my mac box networking woes" is something I hit more often than I'd like to admit. Until, one can hack around this a little bit by creating a two stage proxy that exposes the VM's containerd socket to macOS:

flowchart TD
    A[rules_img] -->|Unix Socket| B[Python Bridge]
    B -->|TCP :2375| C[Alpine Container]
    C -->|nsenter| D[containerd.sock]

    subgraph macOS["macOS Host"]
        A
        B["~/.docker/run/containerd.sock"]
    end

    subgraph vm["Docker Desktop VM"]
        C["--privileged --pid=host"]
        D["/run/containerd/containerd.sock"]
    end

Loading

Not "ideal" but may be worth it to some folx.

Script

Quick airport example to accomplish the above. Not my finest work, but hey it works =)

#!/usr/bin/env python3
from __future__ import annotations

import argparse
import os
import select
import signal
import socket
import subprocess
import sys
import threading
import time


SOCKET_DIR = os.path.expanduser("~/.docker/run")
SOCKET_PATH = os.path.join(SOCKET_DIR, "containerd.sock")
TCP_PORT = 2375
CONTAINER_NAME = "containerd-proxy"
PID_FILE = os.path.join(SOCKET_DIR, "containerd-proxy.pid")


def run(
    cmd: str, *, check: bool = True, quiet: bool = True
) -> subprocess.CompletedProcess[str]:
    result = subprocess.run(
        cmd, shell=True, check=False, capture_output=quiet, text=True
    )
    if check and result.returncode != 0:
        raise RuntimeError(f"Command failed: {cmd}\n{result.stderr}")
    return result


def docker_running() -> bool:
    return run("docker info", check=False).returncode == 0


def container_exists() -> bool:
    result = run(
        f"docker ps -a --filter name={CONTAINER_NAME} --format '{{{{.Names}}}}'",
        check=False,
    )
    return CONTAINER_NAME in result.stdout


def stop_container() -> None:
    if not container_exists():
        return
    print(f"Stopping {CONTAINER_NAME} container...")
    run(f"docker stop {CONTAINER_NAME}", check=False, quiet=False)
    run(f"docker rm {CONTAINER_NAME}", check=False, quiet=False)


def stop_bridge() -> None:
    if not os.path.exists(PID_FILE):
        return
    try:
        with open(PID_FILE) as f:
            pid = int(f.read().strip())
        print(f"Stopping bridge (PID {pid})...")
        os.kill(pid, signal.SIGTERM)
        time.sleep(0.5)
        try:
            os.kill(pid, 0)
            os.kill(pid, signal.SIGKILL)
        except ProcessLookupError:
            pass
    except (ValueError, ProcessLookupError):
        pass
    finally:
        os.remove(PID_FILE)


def cleanup_socket() -> None:
    if os.path.exists(SOCKET_PATH):
        os.remove(SOCKET_PATH)


def start_container() -> None:
    print("Starting containerd-proxy container...")
    cmd = f"""\
docker run -d \
    --name {CONTAINER_NAME} \
    --privileged \
    --pid=host \
    -p 127.0.0.1:{TCP_PORT}:{TCP_PORT} \
    --restart unless-stopped \
    alpine:latest \
    sh -c "apk add --no-cache socat >/dev/null 2>&1 && \
           socat TCP-LISTEN:{TCP_PORT},fork,reuseaddr \
           EXEC:'nsenter -t 1 -m -u -n -i socat - UNIX-CONNECT\\:/run/containerd/containerd.sock'"
"""
    run(cmd, quiet=False)
    print("Waiting for container to initialize...")
    time.sleep(3)


def start_bridge() -> int:
    print("Starting local socket bridge...")
    proc = subprocess.Popen(
        [sys.executable, __file__, "_bridge"],
        stdout=subprocess.DEVNULL,
        stderr=subprocess.DEVNULL,
        start_new_session=True,
    )
    with open(PID_FILE, "w") as f:
        f.write(str(proc.pid))
    return proc.pid


def run_bridge() -> None:
    if os.path.exists(SOCKET_PATH):
        os.remove(SOCKET_PATH)

    server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
    server.bind(SOCKET_PATH)
    os.chmod(SOCKET_PATH, 0o755)
    server.listen(5)

    def handle_client(client_sock: socket.socket) -> None:
        tcp_sock: socket.socket | None = None
        try:
            tcp_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            tcp_sock.connect(("127.0.0.1", TCP_PORT))

            sockets = [client_sock, tcp_sock]
            while True:
                readable, _, _ = select.select(sockets, [], [], 60)
                if not readable:
                    break

                for sock in readable:
                    data = sock.recv(4096)
                    if not data:
                        return

                    target = tcp_sock if sock is client_sock else client_sock
                    target.sendall(data)
        except Exception:
            pass
        finally:
            client_sock.close()
            if tcp_sock:
                tcp_sock.close()

    try:
        while True:
            client_sock, _ = server.accept()
            thread = threading.Thread(
                target=handle_client, args=(client_sock,), daemon=True
            )
            thread.start()
    except KeyboardInterrupt:
        pass
    finally:
        server.close()
        if os.path.exists(SOCKET_PATH):
            os.remove(SOCKET_PATH)


def start() -> int:
    print("Starting containerd proxy for rules_img...")

    if not docker_running():
        print("Error: Docker is not running. Please start Docker Desktop first.")
        return 1

    os.makedirs(SOCKET_DIR, exist_ok=True)
    stop_container()
    stop_bridge()
    cleanup_socket()
    start_container()
    bridge_pid = start_bridge()

    for _ in range(10):
        if os.path.exists(SOCKET_PATH):
            break
        time.sleep(0.5)
    else:
        print(f"Error: Failed to create socket at {SOCKET_PATH}")
        stop()
        return 1

    print(
        f"""\
✓ Containerd proxy is ready!

Socket: {SOCKET_PATH}
Container: {CONTAINER_NAME}
Bridge: PID {bridge_pid}

To use with bazel:
  CONTAINERD_ADDRESS="{SOCKET_PATH}" bazel run //path/to:target --platforms= your linux arm|amd 64 paltform

To stop:
  python3 scripts/containerd_proxy.py stop
"""
    )
    return 0


def stop() -> int:
    print("Stopping containerd proxy...")
    stop_container()
    stop_bridge()
    cleanup_socket()
    print("✓ Containerd proxy stopped")
    return 0


def status() -> int:
    container_ok = False
    if container_exists():
        result = run(
            f"docker ps --filter name={CONTAINER_NAME} --format '{{{{.Status}}}}'",
            check=False,
        )
        if result.stdout.strip():
            container_ok = True
            print(f"Container: Running ({result.stdout.strip()})")
        else:
            print("Container: Stopped")
    else:
        print("Container: Not created")

    bridge_ok = False
    if os.path.exists(PID_FILE):
        try:
            with open(PID_FILE) as f:
                pid = int(f.read().strip())
            os.kill(pid, 0)
            bridge_ok = True
            print(f"Bridge: Running (PID {pid})")
        except (ValueError, ProcessLookupError):
            print("Bridge: Not running (stale PID file)")
    else:
        print("Bridge: Not running")

    if os.path.exists(SOCKET_PATH):
        print(f"Socket: {SOCKET_PATH} (exists)")
    else:
        print("Socket: Not created")

    if container_ok and bridge_ok:
        print("\n✓ Containerd proxy is running")
        return 0

    print("\n✗ Containerd proxy is not fully running")
    return 1


def main() -> int:
    parser = argparse.ArgumentParser(
        description="Containerd proxy for rules_img on macOS"
    )
    parser.add_argument("command", choices=["start", "stop", "status", "_bridge"])
    args = parser.parse_args()

    if args.command == "start":
        return start()
    if args.command == "stop":
        return stop()
    if args.command == "status":
        return status()
    if args.command == "_bridge":
        run_bridge()
        return 0

    return 0


if __name__ == "__main__":
    sys.exit(main())

Demo

gspencer@gspencer-mlt breakfix-workflows % bazel run //examples/containers:load --platforms=//platforms:linux_arm64
...
INFO: Running command line: out/bin/examples/containers/load.exe
Connecting to containerd failed: finding containerd socket: no accessible containerd socket found:
  /run/containerd/containerd.sock: does not exist
  /var/run/containerd/containerd.sock: does not exist
  /run/docker/containerd/containerd.sock: does not exist
  /var/run/docker/containerd/containerd.sock: does not exist

WARNING: Docker is not using containerd storage backend.
This will use 'docker load' which is significantly slower than direct containerd loading.

IMPORTANT: The digest of the image will be different due to the use of 'docker load'.
Docker load creates a custom Docker manifest that doesn't adhere to OCI spec.
If you can load into the containerd backend, you can load the exact OCI image with the expected digest.
See: https://github.com/bazel-contrib/rules_img/issues/76

To improve performance, configure Docker to use containerd:
  https://docs.docker.com/storage/containerd/

Using docker as loader binary
^C%                                                                                                                                                                                          
gspencer@gspencer-mlt breakfix-workflows % ./scripts/containerd_proxy.py start                                     
Starting containerd proxy for rules_img...
Starting containerd-proxy container...
ffda8d4de0b3d3579a5fef5586913ec87672db7739df7e1ecf8c735754ab9821
Waiting for container to initialize...
Starting local socket bridge...
✓ Containerd proxy is ready!

Socket: /Users/gspencer/.docker/run/containerd.sock
Container: containerd-proxy
Bridge: PID 33563

To use with bazel:
  CONTAINERD_ADDRESS="/Users/gspencer/.docker/run/containerd.sock" bazel run //examples/containers:load --platforms=//platforms:linux_arm64

To stop:
  python3 scripts/containerd_proxy.py stop
gspencer@gspencer-mlt breakfix-workflows % CONTAINERD_ADDRESS="/Users/gspencer/.docker/run/containerd.sock" bazel run //examples/containers:load --platforms=//platforms:linux_arm64
...
INFO: Running command line: out/bin/examples/containers/load.exe
docker.io/library/my-app:latest@sha256:7fd4199c0986fcd30ae57284649e474f4ff700d6559bc69f8045e550e361496f
docker.io/library/my-app:latest