diff --git a/.gitignore b/.gitignore index 0e257a6..544c488 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,15 @@ dev *.pdb *.yaml +# The local development container stack is checked in, despite the broad `dev` +# scratch-directory rule above. +!.local/ +!.local/dev/ +!.local/dev/** + +# Local Ceph credentials and loopback OSD state. +.local/dev/ceph/generated/ +.local/dev/ceph/state/ # Generated by cargo mutants # Contains mutation testing data diff --git a/.local/dev/README.md b/.local/dev/README.md new file mode 100644 index 0000000..6750327 --- /dev/null +++ b/.local/dev/README.md @@ -0,0 +1,118 @@ +# Local Ceph and Odorobo with Compose + +This directory runs the local development stack in containers: + +- `ceph` provides a single-node Ceph cluster and a file-backed raw OSD. +- `odorobo` runs the agent in the same network, PID, and device namespaces as Ceph. + +Odorobo must run in the container for the `rbd://` storage path. It invokes `rbd device map`, which creates a kernel block device, and then passes that device to Cloud Hypervisor. A host-side process would not see the container's `/dev/rbd*` device or have the required device and privilege context. + +This is intended for Linux development with a rootful container engine. The stack uses privileged containers because kernel RBD mapping, Cloud Hypervisor, networking, and Ceph's daemon management require host kernel access. + +## Prerequisites + +Install Podman with a working Compose provider. Docker Compose v2 is supported as a fallback. + +For Fedora, the host needs the container engine and kernel modules: + +```bash +sudo dnf install -y podman podman-compose kmod +sudo modprobe rbd +``` + +The container image installs `ceph-common`, Rust tooling, and Cloud Hypervisor tooling. The host does not need `cephadm`, `ceph-common`, `qemu-nbd`, or systemd Ceph units. + +## Usage + +Initialize Ceph and start Odorobo: + +```bash +bash .local/dev/init.sh +``` + +This builds both images, starts both services, provisions the `odorobo-blockpool/dev-disk` RBD image, and starts Odorobo with manager mode enabled. + +Start and stop the complete stack without deleting data: + +```bash +bash .local/dev/start.sh +bash .local/dev/stop.sh +``` + +Destructively remove the containers and all local Ceph state: + +```bash +bash .local/dev/reset.sh +``` + +The scripts prefer `podman compose` and fall back to `docker compose` when Podman Compose is unavailable. + +Useful direct commands: + +```bash +podman compose -f .local/dev/compose.yml ps +podman compose -f .local/dev/compose.yml logs -f ceph odorobo +podman compose -f .local/dev/compose.yml exec ceph ceph -s +``` + +Because `odorobo` uses Ceph's network namespace, the generated Ceph config intentionally uses `127.0.0.1` for the monitor. The application and monitor share that namespace. + +## Application development + +The repository is mounted at `/workspace` in the Odorobo container. Rebuild and restart the application after source changes: + +```bash +podman compose -f .local/dev/compose.yml build odorobo +podman compose -f .local/dev/compose.yml up -d odorobo +podman compose -f .local/dev/compose.yml logs -f odorobo +``` + +The agent runs as: + +```text +cargo run --release -p odorobo -- --manager-enabled +``` + +Its runtime directory is shared through `/run/odorobo`, and Cloud Hypervisor processes and RBD devices are visible in the same namespaces as the agent. + +## Verify the image + +Run Ceph commands inside the Ceph container: + +```bash +podman compose -f .local/dev/compose.yml exec ceph rbd \ + --conf=/etc/ceph/ceph.conf --id=odorobo \ + --keyfile=/var/lib/odorobo-ceph/client.odorobo.key \ + ls --pool odorobo-blockpool +``` + +Expected output includes `dev-disk`. A one-node cluster may report `HEALTH_WARN`; reduced redundancy and a single monitor are expected for local development. + +Do not map the image from the host. To test the exact application path, use an Odorobo manifest with `rbd://odorobo-blockpool/dev-disk`; the `odorobo` service will execute `rbd device map` and pass the resulting device to Cloud Hypervisor. + +## Configuration + +The following environment variables can be set before `init.sh` or passed through Compose: + +```bash +CEPH_IMAGE=quay.io/ceph/ceph:v20.2.3 +CEPH_MON_IP=127.0.0.1 +CEPH_POOL=odorobo-blockpool +CEPH_CLIENT=odorobo +CEPH_IMAGE_NAME=dev-disk +CEPH_IMAGE_SIZE=1G +CEPH_OSD_SIZE=10G +``` + +`CEPH_MON_IP` should remain `127.0.0.1` with the provided Compose topology. If you change the network topology, it must be an address reachable from both services. + +## Layout + +- `compose.yml` — Ceph and Odorobo services, shared namespaces, privilege, mounts, and ports. +- `ceph/Containerfile` — pinned Ceph image. +- `ceph/entrypoint.sh` — bootstrap, OSD, pool, client, and image provisioning. +- `odorobo/Containerfile` — runnable Odorobo development image. +- `ceph/generated/` — generated Ceph credentials shared read-only with Odorobo; ignored by git. +- `ceph/state/` — Ceph configuration, daemons, logs, and file-backed OSD; ignored by git. + +This setup is intentionally not production-ready: it has one monitor, one OSD, no redundancy, and privileged containers. diff --git a/.local/dev/ceph/Containerfile b/.local/dev/ceph/Containerfile new file mode 100644 index 0000000..8afb6fe --- /dev/null +++ b/.local/dev/ceph/Containerfile @@ -0,0 +1,9 @@ +ARG CEPH_IMAGE=quay.io/ceph/ceph:v20.2.3 +FROM ${CEPH_IMAGE} + +USER root + +COPY entrypoint.sh /usr/local/bin/odorobo-ceph +RUN chmod 0755 /usr/local/bin/odorobo-ceph + +ENTRYPOINT ["/usr/local/bin/odorobo-ceph"] diff --git a/.local/dev/ceph/entrypoint.sh b/.local/dev/ceph/entrypoint.sh new file mode 100644 index 0000000..b24578f --- /dev/null +++ b/.local/dev/ceph/entrypoint.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${CEPH_IMAGE:=quay.io/ceph/ceph:v20.2.3}" +: "${CEPH_MON_IP:=127.0.0.1}" +: "${CEPH_POOL:=odorobo-blockpool}" +: "${CEPH_CLIENT:=odorobo}" +: "${CEPH_IMAGE_NAME:=dev-disk}" +: "${CEPH_IMAGE_SIZE:=1G}" +: "${CEPH_OSD_SIZE:=10G}" + +mkdir -p /etc/ceph /var/lib/ceph /var/log/ceph /run/ceph /var/lib/odorobo-ceph + +if [[ ! -f /etc/ceph/ceph.conf ]]; then + cephadm --image "$CEPH_IMAGE" bootstrap \ + --mon-ip "$CEPH_MON_IP" \ + --single-host-defaults \ + --output-dir /etc/ceph \ + --skip-monitoring-stack \ + --allow-fqdn-hostname \ + --skip-pull +fi + +# cephadm has created and started the daemons. Keep this container alive while +# allowing compose stop/restart to control the complete development cluster. +while ! ceph -s >/dev/null 2>&1; do + sleep 2 +done + +if ! ceph osd ls 2>/dev/null | grep -q '[0-9]'; then + truncate -s "$CEPH_OSD_SIZE" /var/lib/odorobo-ceph/osd.raw + OSD_DEVICE=$(losetup --find --show /var/lib/odorobo-ceph/osd.raw) + echo "$OSD_DEVICE" > /var/lib/odorobo-ceph/osd.device + cephadm shell -- ceph-volume raw prepare --data "$OSD_DEVICE" + cephadm shell -- ceph-volume raw activate --device "$OSD_DEVICE" --no-systemd & +fi + +for _ in {1..60}; do + if ceph osd stat 2>/dev/null | grep -q '[1-9][0-9]* osds:'; then + break + fi + sleep 2 +done + +ceph config set mon mon_allow_pool_size_one true +ceph config set osd osd_pool_default_size 1 +ceph config set osd osd_pool_default_min_size 1 +if ! ceph osd pool ls --format=json | grep -qF "\"$CEPH_POOL\""; then + ceph osd pool create "$CEPH_POOL" 8 +fi +ceph osd pool set "$CEPH_POOL" size 1 --yes-i-really-mean-it +ceph osd pool set "$CEPH_POOL" min_size 1 +rbd pool init "$CEPH_POOL" + +if ! ceph auth get "client.$CEPH_CLIENT" >/dev/null 2>&1; then + ceph auth get-or-create "client.$CEPH_CLIENT" \ + mon 'allow r' osd "allow rwx pool=$CEPH_POOL" >/dev/null +fi + +if ! rbd info "$CEPH_POOL/$CEPH_IMAGE_NAME" >/dev/null 2>&1; then + rbd create "$CEPH_POOL/$CEPH_IMAGE_NAME" --size "$CEPH_IMAGE_SIZE" +fi + +ceph config generate-minimal-conf > /var/lib/odorobo-ceph/ceph.conf +ceph auth get-key "client.$CEPH_CLIENT" > /var/lib/odorobo-ceph/client.$CEPH_CLIENT.key +chmod 600 /var/lib/odorobo-ceph/client.$CEPH_CLIENT.key +cp /var/lib/odorobo-ceph/ceph.conf /generated/ceph.conf +cp /var/lib/odorobo-ceph/client.$CEPH_CLIENT.key /generated/client.$CEPH_CLIENT.key +chmod 600 /generated/client.$CEPH_CLIENT.key + +tail -f /dev/null diff --git a/.local/dev/compose.yml b/.local/dev/compose.yml new file mode 100644 index 0000000..31cf426 --- /dev/null +++ b/.local/dev/compose.yml @@ -0,0 +1,61 @@ +services: + ceph: + build: + context: ./ceph + dockerfile: Containerfile + args: + CEPH_IMAGE: ${CEPH_IMAGE:-quay.io/ceph/ceph:v20.2.3} + image: odorobo-ceph:local + container_name: odorobo-ceph + hostname: ceph + privileged: true + restart: unless-stopped + environment: + CEPH_IMAGE: ${CEPH_IMAGE:-quay.io/ceph/ceph:v20.2.3} + CEPH_MON_IP: ${CEPH_MON_IP:-127.0.0.1} + CEPH_POOL: ${CEPH_POOL:-odorobo-blockpool} + CEPH_CLIENT: ${CEPH_CLIENT:-odorobo} + CEPH_IMAGE_NAME: ${CEPH_IMAGE_NAME:-dev-disk} + CEPH_IMAGE_SIZE: ${CEPH_IMAGE_SIZE:-1G} + CEPH_OSD_SIZE: ${CEPH_OSD_SIZE:-10G} + volumes: + - ./ceph/state/etc-ceph:/etc/ceph + - ./ceph/state/lib-ceph:/var/lib/ceph + - ./ceph/state/log-ceph:/var/log/ceph + - ./ceph/state/run-ceph:/run/ceph + - ./ceph/state/odorobo-ceph:/var/lib/odorobo-ceph + - ./ceph/generated:/generated + healthcheck: + test: ["CMD", "ceph", "-s"] + interval: 5s + timeout: 5s + retries: 30 + ports: + - "3300:3300" + - "6789:6789" + - "8443:8443" + - "9283:9283" + - "6800-7300:6800-7300" + + odorobo: + build: + context: ../.. + dockerfile: .local/dev/odorobo/Containerfile + privileged: true + depends_on: + ceph: + condition: service_healthy + network_mode: service:ceph + pid: service:ceph + volumes: + - ../../:/workspace + - ./ceph/generated:/workspace/.local/dev/ceph/generated:ro + - /run/odorobo:/run/odorobo + - /dev:/dev + working_dir: /workspace + environment: + CEPH_CONFIG: /workspace/.local/dev/ceph/generated/ceph.conf + CEPH_ID: ${CEPH_CLIENT:-odorobo} + CEPH_KEYFILE: /workspace/.local/dev/ceph/generated/client.${CEPH_CLIENT:-odorobo}.key + CEPH_CLUSTER: ceph + command: ["cargo", "run", "--release", "-p", "odorobo", "--", "--manager-enabled"] diff --git a/.local/dev/init.sh b/.local/dev/init.sh new file mode 100644 index 0000000..40310b5 --- /dev/null +++ b/.local/dev/init.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +cd "$ROOT_DIR" + +if command -v podman >/dev/null 2>&1 && podman compose version >/dev/null 2>&1; then + COMPOSE=(podman compose) +elif command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then + COMPOSE=(docker compose) +else + echo "Docker Compose or Podman Compose is required." >&2 + exit 1 +fi + +mkdir -p ceph/generated ceph/state/{etc-ceph,lib-ceph,log-ceph,run-ceph,odorobo-ceph} +"${COMPOSE[@]}" up --build -d ceph odorobo +"${COMPOSE[@]}" exec -T ceph ceph -s + +cat </dev/null 2>&1 && podman compose version >/dev/null 2>&1; then + COMPOSE=(podman compose) +elif command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then + COMPOSE=(docker compose) +else + echo "Docker Compose or Podman Compose is required." >&2 + exit 1 +fi + +"${COMPOSE[@]}" down --remove-orphans --volumes +rm -rf ceph/generated ceph/state + +echo "Local Ceph container, credentials, and state removed." diff --git a/.local/dev/start.sh b/.local/dev/start.sh new file mode 100644 index 0000000..1ca440a --- /dev/null +++ b/.local/dev/start.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +cd "$ROOT_DIR" + +if command -v podman >/dev/null 2>&1 && podman compose version >/dev/null 2>&1; then + COMPOSE=(podman compose) +elif command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then + COMPOSE=(docker compose) +else + echo "Docker Compose or Podman Compose is required." >&2 + exit 1 +fi + +"${COMPOSE[@]}" start ceph odorobo diff --git a/.local/dev/stop.sh b/.local/dev/stop.sh new file mode 100644 index 0000000..62d42d8 --- /dev/null +++ b/.local/dev/stop.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +cd "$ROOT_DIR" + +if command -v podman >/dev/null 2>&1 && podman compose version >/dev/null 2>&1; then + COMPOSE=(podman compose) +elif command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then + COMPOSE=(docker compose) +elif command -v docker >/dev/null 2>&1; then + echo "Docker Compose is required." >&2 + exit 1 +else + echo "Docker Compose or Podman Compose is required." >&2 + exit 1 +fi + +"${COMPOSE[@]}" stop odorobo ceph