feat: add ceph dev env

.gitignore

@@ -12,6 +12,15 @@ dev
 *.pdb
 
 *.yaml
+# The local development container stack is checked in, despite the broad `dev`
+# scratch-directory rule above.
+!.local/
+!.local/dev/
+!.local/dev/**
+
+# Local Ceph credentials and loopback OSD state.
+.local/dev/ceph/generated/
+.local/dev/ceph/state/
 
 # Generated by cargo mutants
 # Contains mutation testing data

.local/dev/README.md

@@ -0,0 +1,118 @@
+# Local Ceph and Odorobo with Compose
+
+This directory runs the local development stack in containers:
+
+- `ceph` provides a single-node Ceph cluster and a file-backed raw OSD.
+- `odorobo` runs the agent in the same network, PID, and device namespaces as Ceph.
+
+Odorobo must run in the container for the `rbd://` storage path. It invokes `rbd device map`, which creates a kernel block device, and then passes that device to Cloud Hypervisor. A host-side process would not see the container's `/dev/rbd*` device or have the required device and privilege context.
+
+This is intended for Linux development with a rootful container engine. The stack uses privileged containers because kernel RBD mapping, Cloud Hypervisor, networking, and Ceph's daemon management require host kernel access.
+
+## Prerequisites
+
+Install Podman with a working Compose provider. Docker Compose v2 is supported as a fallback.
+
+For Fedora, the host needs the container engine and kernel modules:
+
+```bash
+sudo dnf install -y podman podman-compose kmod
+sudo modprobe rbd
+```
+
+The container image installs `ceph-common`, Rust tooling, and Cloud Hypervisor tooling. The host does not need `cephadm`, `ceph-common`, `qemu-nbd`, or systemd Ceph units.
+
+## Usage
+
+Initialize Ceph and start Odorobo:
+
+```bash
+bash .local/dev/init.sh
+```
+
+This builds both images, starts both services, provisions the `odorobo-blockpool/dev-disk` RBD image, and starts Odorobo with manager mode enabled.
+
+Start and stop the complete stack without deleting data:
+
+```bash
+bash .local/dev/start.sh
+bash .local/dev/stop.sh
+```
+
+Destructively remove the containers and all local Ceph state:
+
+```bash
+bash .local/dev/reset.sh
+```
+
+The scripts prefer `podman compose` and fall back to `docker compose` when Podman Compose is unavailable.
+
+Useful direct commands:
+
+```bash
+podman compose -f .local/dev/compose.yml ps
+podman compose -f .local/dev/compose.yml logs -f ceph odorobo
+podman compose -f .local/dev/compose.yml exec ceph ceph -s
+```
+
+Because `odorobo` uses Ceph's network namespace, the generated Ceph config intentionally uses `127.0.0.1` for the monitor. The application and monitor share that namespace.
+
+## Application development
+
+The repository is mounted at `/workspace` in the Odorobo container. Rebuild and restart the application after source changes:
+
+```bash
+podman compose -f .local/dev/compose.yml build odorobo
+podman compose -f .local/dev/compose.yml up -d odorobo
+podman compose -f .local/dev/compose.yml logs -f odorobo
+```
+
+The agent runs as:
+
+```text
+cargo run --release -p odorobo -- --manager-enabled
+```
+
+Its runtime directory is shared through `/run/odorobo`, and Cloud Hypervisor processes and RBD devices are visible in the same namespaces as the agent.
+
+## Verify the image
+
+Run Ceph commands inside the Ceph container:
+
+```bash
+podman compose -f .local/dev/compose.yml exec ceph rbd \
+  --conf=/etc/ceph/ceph.conf --id=odorobo \
+  --keyfile=/var/lib/odorobo-ceph/client.odorobo.key \
+  ls --pool odorobo-blockpool
+```
+
+Expected output includes `dev-disk`. A one-node cluster may report `HEALTH_WARN`; reduced redundancy and a single monitor are expected for local development.
+
+Do not map the image from the host. To test the exact application path, use an Odorobo manifest with `rbd://odorobo-blockpool/dev-disk`; the `odorobo` service will execute `rbd device map` and pass the resulting device to Cloud Hypervisor.
+
+## Configuration
+
+The following environment variables can be set before `init.sh` or passed through Compose:
+
+```bash
+CEPH_IMAGE=quay.io/ceph/ceph:v20.2.3
+CEPH_MON_IP=127.0.0.1
+CEPH_POOL=odorobo-blockpool
+CEPH_CLIENT=odorobo
+CEPH_IMAGE_NAME=dev-disk
+CEPH_IMAGE_SIZE=1G
+CEPH_OSD_SIZE=10G
+```
+
+`CEPH_MON_IP` should remain `127.0.0.1` with the provided Compose topology. If you change the network topology, it must be an address reachable from both services.
+
+## Layout
+
+- `compose.yml` — Ceph and Odorobo services, shared namespaces, privilege, mounts, and ports.
+- `ceph/Containerfile` — pinned Ceph image.
+- `ceph/entrypoint.sh` — bootstrap, OSD, pool, client, and image provisioning.
+- `odorobo/Containerfile` — runnable Odorobo development image.
+- `ceph/generated/` — generated Ceph credentials shared read-only with Odorobo; ignored by git.
+- `ceph/state/` — Ceph configuration, daemons, logs, and file-backed OSD; ignored by git.
+
+This setup is intentionally not production-ready: it has one monitor, one OSD, no redundancy, and privileged containers.

.local/dev/ceph/Containerfile

@@ -0,0 +1,9 @@
+ARG CEPH_IMAGE=quay.io/ceph/ceph:v20.2.3
+FROM ${CEPH_IMAGE}
+
+USER root
+
+COPY entrypoint.sh /usr/local/bin/odorobo-ceph
+RUN chmod 0755 /usr/local/bin/odorobo-ceph
+
+ENTRYPOINT ["/usr/local/bin/odorobo-ceph"]

.local/dev/ceph/entrypoint.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+: "${CEPH_IMAGE:=quay.io/ceph/ceph:v20.2.3}"
+: "${CEPH_MON_IP:=127.0.0.1}"
+: "${CEPH_POOL:=odorobo-blockpool}"
+: "${CEPH_CLIENT:=odorobo}"
+: "${CEPH_IMAGE_NAME:=dev-disk}"
+: "${CEPH_IMAGE_SIZE:=1G}"
+: "${CEPH_OSD_SIZE:=10G}"
+
+mkdir -p /etc/ceph /var/lib/ceph /var/log/ceph /run/ceph /var/lib/odorobo-ceph
+
+if [[ ! -f /etc/ceph/ceph.conf ]]; then
+  cephadm --image "$CEPH_IMAGE" bootstrap \
+    --mon-ip "$CEPH_MON_IP" \
+    --single-host-defaults \
+    --output-dir /etc/ceph \
+    --skip-monitoring-stack \
+    --allow-fqdn-hostname \
+    --skip-pull
+fi
+
+# cephadm has created and started the daemons. Keep this container alive while
+# allowing compose stop/restart to control the complete development cluster.
+while ! ceph -s >/dev/null 2>&1; do
+  sleep 2
+done
+
+if ! ceph osd ls 2>/dev/null | grep -q '[0-9]'; then
+  truncate -s "$CEPH_OSD_SIZE" /var/lib/odorobo-ceph/osd.raw
+  OSD_DEVICE=$(losetup --find --show /var/lib/odorobo-ceph/osd.raw)
+  echo "$OSD_DEVICE" > /var/lib/odorobo-ceph/osd.device
+  cephadm shell -- ceph-volume raw prepare --data "$OSD_DEVICE"
+  cephadm shell -- ceph-volume raw activate --device "$OSD_DEVICE" --no-systemd &
+fi
+
+for _ in {1..60}; do
+  if ceph osd stat 2>/dev/null | grep -q '[1-9][0-9]* osds:'; then
+    break
+  fi
+  sleep 2
+done
+
+ceph config set mon mon_allow_pool_size_one true
+ceph config set osd osd_pool_default_size 1
+ceph config set osd osd_pool_default_min_size 1
+if ! ceph osd pool ls --format=json | grep -qF "\"$CEPH_POOL\""; then
+  ceph osd pool create "$CEPH_POOL" 8
+fi
+ceph osd pool set "$CEPH_POOL" size 1 --yes-i-really-mean-it
+ceph osd pool set "$CEPH_POOL" min_size 1
+rbd pool init "$CEPH_POOL"
+
+if ! ceph auth get "client.$CEPH_CLIENT" >/dev/null 2>&1; then
+  ceph auth get-or-create "client.$CEPH_CLIENT" \
+    mon 'allow r' osd "allow rwx pool=$CEPH_POOL" >/dev/null
+fi
+
+if ! rbd info "$CEPH_POOL/$CEPH_IMAGE_NAME" >/dev/null 2>&1; then
+  rbd create "$CEPH_POOL/$CEPH_IMAGE_NAME" --size "$CEPH_IMAGE_SIZE"
+fi
+
+ceph config generate-minimal-conf > /var/lib/odorobo-ceph/ceph.conf
+ceph auth get-key "client.$CEPH_CLIENT" > /var/lib/odorobo-ceph/client.$CEPH_CLIENT.key
+chmod 600 /var/lib/odorobo-ceph/client.$CEPH_CLIENT.key
+cp /var/lib/odorobo-ceph/ceph.conf /generated/ceph.conf
+cp /var/lib/odorobo-ceph/client.$CEPH_CLIENT.key /generated/client.$CEPH_CLIENT.key
+chmod 600 /generated/client.$CEPH_CLIENT.key
+
+tail -f /dev/null

.local/dev/compose.yml

@@ -0,0 +1,61 @@
+services:
+  ceph:
+    build:
+      context: ./ceph
+      dockerfile: Containerfile
+      args:
+        CEPH_IMAGE: ${CEPH_IMAGE:-quay.io/ceph/ceph:v20.2.3}
+    image: odorobo-ceph:local
+    container_name: odorobo-ceph
+    hostname: ceph
+    privileged: true
+    restart: unless-stopped
+    environment:
+      CEPH_IMAGE: ${CEPH_IMAGE:-quay.io/ceph/ceph:v20.2.3}
+      CEPH_MON_IP: ${CEPH_MON_IP:-127.0.0.1}
+      CEPH_POOL: ${CEPH_POOL:-odorobo-blockpool}
+      CEPH_CLIENT: ${CEPH_CLIENT:-odorobo}
+      CEPH_IMAGE_NAME: ${CEPH_IMAGE_NAME:-dev-disk}
+      CEPH_IMAGE_SIZE: ${CEPH_IMAGE_SIZE:-1G}
+      CEPH_OSD_SIZE: ${CEPH_OSD_SIZE:-10G}
+    volumes:
+      - ./ceph/state/etc-ceph:/etc/ceph
+      - ./ceph/state/lib-ceph:/var/lib/ceph
+      - ./ceph/state/log-ceph:/var/log/ceph
+      - ./ceph/state/run-ceph:/run/ceph
+      - ./ceph/state/odorobo-ceph:/var/lib/odorobo-ceph
+      - ./ceph/generated:/generated
+    healthcheck:
+      test: ["CMD", "ceph", "-s"]
+      interval: 5s
+      timeout: 5s
+      retries: 30
+    ports:
+      - "3300:3300"
+      - "6789:6789"
+      - "8443:8443"
+      - "9283:9283"
+      - "6800-7300:6800-7300"
+
+  odorobo:
+    build:
+      context: ../..
+      dockerfile: .local/dev/odorobo/Containerfile
+    privileged: true
+    depends_on:
+      ceph:
+        condition: service_healthy
+    network_mode: service:ceph
+    pid: service:ceph
+    volumes:
+      - ../../:/workspace
+      - ./ceph/generated:/workspace/.local/dev/ceph/generated:ro
+      - /run/odorobo:/run/odorobo
+      - /dev:/dev
+    working_dir: /workspace
+    environment:
+      CEPH_CONFIG: /workspace/.local/dev/ceph/generated/ceph.conf
+      CEPH_ID: ${CEPH_CLIENT:-odorobo}
+      CEPH_KEYFILE: /workspace/.local/dev/ceph/generated/client.${CEPH_CLIENT:-odorobo}.key
+      CEPH_CLUSTER: ceph
+    command: ["cargo", "run", "--release", "-p", "odorobo", "--", "--manager-enabled"]

.local/dev/init.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+cd "$ROOT_DIR"
+
+if command -v podman >/dev/null 2>&1 && podman compose version >/dev/null 2>&1; then
+  COMPOSE=(podman compose)
+elif command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then
+  COMPOSE=(docker compose)
+else
+  echo "Docker Compose or Podman Compose is required." >&2
+  exit 1
+fi
+
+mkdir -p ceph/generated ceph/state/{etc-ceph,lib-ceph,log-ceph,run-ceph,odorobo-ceph}
+"${COMPOSE[@]}" up --build -d ceph odorobo
+"${COMPOSE[@]}" exec -T ceph ceph -s
+
+cat <<EOF
+
+Ceph is ready.
+Config: $ROOT_DIR/ceph/generated/ceph.conf
+Key: $ROOT_DIR/ceph/generated/client.${CEPH_CLIENT:-odorobo}.key
+
+Generated credentials (for tools running inside the Odorobo container):
+  export CEPH_CONFIG=$ROOT_DIR/ceph/generated/ceph.conf
+  export CEPH_ID=${CEPH_CLIENT:-odorobo}
+  export CEPH_KEYFILE=$ROOT_DIR/ceph/generated/client.${CEPH_CLIENT:-odorobo}.key
+  export CEPH_CLUSTER=ceph
+EOF

.local/dev/odorobo/Containerfile

@@ -0,0 +1,18 @@
+FROM ghcr.io/ultramarine-linux/ultramarine:44
+
+RUN dnf do -y --action=install \
+    cargo \
+    ceph-common \
+    clang-devel \
+    cloud-hypervisor-nightly \
+    cmake \
+    git \
+    libvirt-devel \
+    ninja \
+    rust \
+    rustup \
+    zig
+
+WORKDIR /workspace
+
+CMD ["cargo", "run", "--release", "-p", "odorobo", "--", "--manager-enabled"]

.local/dev/reset.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+cd "$ROOT_DIR"
+
+if command -v podman >/dev/null 2>&1 && podman compose version >/dev/null 2>&1; then
+  COMPOSE=(podman compose)
+elif command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then
+  COMPOSE=(docker compose)
+else
+  echo "Docker Compose or Podman Compose is required." >&2
+  exit 1
+fi
+
+"${COMPOSE[@]}" down --remove-orphans --volumes
+rm -rf ceph/generated ceph/state
+
+echo "Local Ceph container, credentials, and state removed."

.local/dev/start.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+cd "$ROOT_DIR"
+
+if command -v podman >/dev/null 2>&1 && podman compose version >/dev/null 2>&1; then
+  COMPOSE=(podman compose)
+elif command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then
+  COMPOSE=(docker compose)
+else
+  echo "Docker Compose or Podman Compose is required." >&2
+  exit 1
+fi
+
+"${COMPOSE[@]}" start ceph odorobo

.local/dev/stop.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+cd "$ROOT_DIR"
+
+if command -v podman >/dev/null 2>&1 && podman compose version >/dev/null 2>&1; then
+  COMPOSE=(podman compose)
+elif command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then
+  COMPOSE=(docker compose)
+elif command -v docker >/dev/null 2>&1; then
+  echo "Docker Compose is required." >&2
+  exit 1
+else
+  echo "Docker Compose or Podman Compose is required." >&2
+  exit 1
+fi
+
+"${COMPOSE[@]}" stop odorobo ceph