diff --git a/README.md b/README.md index 9195c5b..a0edf0c 100644 --- a/README.md +++ b/README.md @@ -83,3 +83,39 @@ You may now run the tests with ```bash uv run pytest -s . ``` + +### Running against locally built images + +By default the tests pull `ghcr.io/equinor/{flotilla-backend,sara,isar-robot}`. A change that +spans armada *and* one of those services therefore cannot be verified until the service change +is merged and an image published — even though the armada side is what proves the service side +works. + +To close that gap, build the images from your local working copies: + +```bash +scripts/build_local_images.sh # build and verify +scripts/build_local_images.sh --run # ... and run the full suite against them +``` + +The script expects the sibling checkouts of the superrepo (`../isar`, `../isar-robot`, +`../flotilla`, `../sara`); override with `ISAR_DIR`, `ISAR_ROBOT_DIR`, `FLOTILLA_DIR`, +`SARA_DIR`. It verifies each image before handing back, because a subtly broken build otherwise +shows up only as an unexplained timeout several minutes into the suite. + +The database schema is taken from the same local checkouts, via `FLOTILLA_MIGRATIONS_SOURCE_DIR` +and `SARA_MIGRATIONS_SOURCE_DIR`, so application code and schema always agree. The directory is +mounted read-only and copied into the migrations container, which means **uncommitted and +untracked migrations are picked up**. Set either variable on its own if you want to mix a local +schema with published images. + +Two things worth knowing: + +- **`flotilla`, `sara` and `isar` are built from the working tree**, so uncommitted changes are + included. **`isar-robot` is cloned**, so only committed changes are — the script warns if that + checkout is dirty. It has to be cloned because its Dockerfile bind-mounts `.git`, and in the + superrepo that is a submodule *file* rather than a directory. +- `isar-robot`'s `uv.lock` pins `isar` from PyPI, so the locally built `isar` wheel is installed + over the released one. + +The mosquitto broker is always the published image. diff --git a/robotics_integration_tests/custom_containers/image_builder.py b/robotics_integration_tests/custom_containers/image_builder.py new file mode 100644 index 0000000..70cfb78 --- /dev/null +++ b/robotics_integration_tests/custom_containers/image_builder.py @@ -0,0 +1,20 @@ +import fcntl +import tempfile +from pathlib import Path + +from loguru import logger +from testcontainers.core.image import DockerImage + + +def build_image_once(path: str, tag: str) -> str: + lock_path: Path = ( + Path(tempfile.gettempdir()) / f"armada-build-{tag.replace('/', '_')}.lock" + ) + + with open(lock_path, "w") as lock_file: + fcntl.flock(lock_file, fcntl.LOCK_EX) + try: + logger.debug(f"Building image {tag} from {path}") + return str(DockerImage(path=path, tag=tag).build()) + finally: + fcntl.flock(lock_file, fcntl.LOCK_UN) diff --git a/robotics_integration_tests/custom_containers/migrations_runner.py b/robotics_integration_tests/custom_containers/migrations_runner.py index 7a62172..b2999f4 100644 --- a/robotics_integration_tests/custom_containers/migrations_runner.py +++ b/robotics_integration_tests/custom_containers/migrations_runner.py @@ -1,24 +1,68 @@ from pathlib import Path from docker.models.networks import Network -from testcontainers.core.image import DockerImage +from loguru import logger +from robotics_integration_tests.custom_containers.image_builder import build_image_once from robotics_integration_tests.custom_containers.stream_logging_docker_container import ( StreamLoggingDockerContainer, ) from robotics_integration_tests.settings.settings import settings +# Where a local checkout is mounted inside the migrations runner. +_LOCAL_REPO_MOUNT = "/src" + + +def _with_migrations_source( + container: StreamLoggingDockerContainer, + source_dir: str, + project_folder: str, + setting_name: str, +) -> StreamLoggingDockerContainer: + """Take migrations from a local checkout instead of cloning from GitHub. + + Mounted read-only; the entrypoint copies it into the container so that nothing + can be written back into the working tree. The copy means uncommitted and + untracked migrations are picked up, which is the reason to use this at all. + + Validation is deliberately strict and happens here, before the container + starts: silently falling back to the GitHub clone would leave you believing + you had tested a local migration when you had not. + """ + if not source_dir: + return container + + resolved: Path = Path(source_dir).expanduser().resolve() + if not resolved.is_dir(): + raise ValueError( + f"{setting_name} is set to '{source_dir}' (resolved to '{resolved}'), " + "which is not a directory." + ) + + project_dir: Path = resolved / project_folder + if not list(project_dir.glob("*.csproj")): + raise ValueError( + f"{setting_name} is set to '{resolved}', but no .csproj was found in " + f"'{project_dir}'. Point it at the repository root of the service, not " + "at the project folder." + ) + + logger.info(f"Using migrations from local checkout: {resolved}") + return container.with_volume_mapping( + str(resolved), _LOCAL_REPO_MOUNT, "ro" + ).with_env("LOCAL_REPO_PATH", _LOCAL_REPO_MOUNT) + def create_migrations_runner_container( network: Network, postgres_connection_string: str, name: str = "flotilla_migrations", test_id: str = "" ) -> StreamLoggingDockerContainer: - migrations_runner_image: DockerImage = DockerImage( + migrations_runner_image: str = build_image_once( path=str(Path(settings.RELATIVE_PATH_TO_DOCKERFILE).resolve(strict=True)), tag="flotilla-migrations-runner", - ).build() + ) container = ( - StreamLoggingDockerContainer(image=str(migrations_runner_image)) + StreamLoggingDockerContainer(image=migrations_runner_image) .with_name(f"{name}-{test_id}") .with_network(network) .with_env("DATABASE_URL", postgres_connection_string) @@ -30,19 +74,24 @@ def create_migrations_runner_container( .with_env("EF_PROJECT_PATH", settings.BACKEND_PROJECT_FILE_FOLDER) .with_env("EF_STARTUP_PATH", settings.BACKEND_PROJECT_FILE_FOLDER) ) - return container + return _with_migrations_source( + container, + source_dir=settings.FLOTILLA_MIGRATIONS_SOURCE_DIR, + project_folder=settings.BACKEND_PROJECT_FILE_FOLDER, + setting_name="FLOTILLA_MIGRATIONS_SOURCE_DIR", + ) def create_sara_migrations_runner_container( network: Network, postgres_connection_string: str, name: str = "sara_migrations", test_id: str = "" ) -> StreamLoggingDockerContainer: - sara_migrations_runner_image: DockerImage = DockerImage( + sara_migrations_runner_image: str = build_image_once( path=str(Path(settings.RELATIVE_PATH_TO_DOCKERFILE).resolve(strict=True)), tag="sara-migrations-runner", - ).build() + ) container = ( - StreamLoggingDockerContainer(image=str(sara_migrations_runner_image)) + StreamLoggingDockerContainer(image=sara_migrations_runner_image) .with_name(f"{name}-{test_id}") .with_network(network) .with_env("DATABASE_URL", postgres_connection_string) @@ -54,4 +103,9 @@ def create_sara_migrations_runner_container( .with_env("EF_PROJECT_PATH", settings.SARA_BACKEND_PROJECT_FILE_FOLDER) .with_env("EF_STARTUP_PATH", settings.SARA_BACKEND_PROJECT_FILE_FOLDER) ) - return container + return _with_migrations_source( + container, + source_dir=settings.SARA_MIGRATIONS_SOURCE_DIR, + project_folder=settings.SARA_BACKEND_PROJECT_FILE_FOLDER, + setting_name="SARA_MIGRATIONS_SOURCE_DIR", + ) diff --git a/robotics_integration_tests/custom_containers/teams_webhook_receiver.py b/robotics_integration_tests/custom_containers/teams_webhook_receiver.py index 438273c..f1c5c85 100644 --- a/robotics_integration_tests/custom_containers/teams_webhook_receiver.py +++ b/robotics_integration_tests/custom_containers/teams_webhook_receiver.py @@ -4,8 +4,7 @@ import requests from docker.models.networks import Network from loguru import logger -from testcontainers.core.image import DockerImage - +from robotics_integration_tests.custom_containers.image_builder import build_image_once from robotics_integration_tests.custom_containers.stream_logging_docker_container import ( StreamLoggingDockerContainer, ) @@ -70,13 +69,10 @@ def create_teams_webhook_receiver_container( test_id: str = "", ) -> tuple[StreamLoggingDockerContainer, TeamsWebhookReceiver]: """Build the image and return both the raw container and a typed wrapper.""" - image: DockerImage = DockerImage( - path=str(_IMAGE_DIR), - tag="teams-webhook-receiver", - ).build() + image: str = build_image_once(path=str(_IMAGE_DIR), tag="teams-webhook-receiver") container: StreamLoggingDockerContainer = ( - StreamLoggingDockerContainer(image=str(image)) + StreamLoggingDockerContainer(image=image) .with_name(f"{name}-{test_id}") .with_exposed_ports(port) .with_network(network) diff --git a/robotics_integration_tests/custom_images/migrations_runner/entrypoint.sh b/robotics_integration_tests/custom_images/migrations_runner/entrypoint.sh index 4ede427..81e10e7 100644 --- a/robotics_integration_tests/custom_images/migrations_runner/entrypoint.sh +++ b/robotics_integration_tests/custom_images/migrations_runner/entrypoint.sh @@ -17,19 +17,50 @@ WAIT_FOR_DB_TIMEOUT="${WAIT_FOR_DB_TIMEOUT:-60}" : "${AZURE_CLIENT_ID:?AZURE_CLIENT_ID must be set at runtime}" : "${AZURE_TENANT_ID:?AZURE_TENANT_ID must be set at runtime}" -echo "Cloning $GIT_REPO @ $GIT_REF ..." rm -rf /work/repo -if [ "$GIT_REF" = "latest" ]; then - BRANCH=$(curl -s ${GITHUB_TOKEN:+-H "Authorization: token $GITHUB_TOKEN"} \ - "https://api.github.com/repos/$GIT_REPO/releases/latest" | jq -r .tag_name) - echo "Resolved latest to $BRANCH" +mkdir -p /work/repo + +if [ -n "${LOCAL_REPO_PATH:-}" ]; then + # Migrations come from a local checkout mounted read-only, so that locally + # built service images and the database schema come from the same source. + # Copied rather than used in place: the build writes bin/ and obj/ into the + # project, and the mount is read-only precisely so the caller's working tree + # cannot be modified. + # + # bin and obj are excluded not to save space but for correctness: they are + # host-architecture build output, and obj/project.assets.json embeds absolute + # host paths, both of which break restore inside this container. + echo "Copying migrations source from $LOCAL_REPO_PATH ..." + [ -d "$LOCAL_REPO_PATH" ] || { echo "LOCAL_REPO_PATH '$LOCAL_REPO_PATH' is not a directory."; exit 1; } + tar -C "$LOCAL_REPO_PATH" \ + --exclude=bin \ + --exclude=obj \ + --exclude=node_modules \ + --exclude=.git \ + --exclude=TestResults \ + -cf - . | tar -C /work/repo -xf - else - BRANCH="main" + echo "Cloning $GIT_REPO @ $GIT_REF ..." + if [ "$GIT_REF" = "latest" ]; then + BRANCH=$(curl -s ${GITHUB_TOKEN:+-H "Authorization: token $GITHUB_TOKEN"} \ + "https://api.github.com/repos/$GIT_REPO/releases/latest" | jq -r .tag_name) + echo "Resolved latest to $BRANCH" + else + BRANCH="main" + fi + rm -rf /work/repo + git clone --depth 1 --branch "$BRANCH" "https://github.com/$GIT_REPO" /work/repo fi -git clone --depth 1 --branch "$BRANCH" "https://github.com/$GIT_REPO" /work/repo cd /work/repo +# Guard against a source that does not contain what we expect, rather than +# letting it surface later as an opaque dotnet-ef failure. +if ! ls "$EF_PROJECT_PATH"/*.csproj >/dev/null 2>&1; then + echo "No .csproj found at '$EF_PROJECT_PATH' in the migrations source." + exit 1 +fi + echo "Restoring projects for EF design-time..." dotnet restore "$EF_STARTUP_PATH" || dotnet restore "$EF_PROJECT_PATH" || true diff --git a/robotics_integration_tests/settings/settings.py b/robotics_integration_tests/settings/settings.py index be97c7e..ab0ed92 100644 --- a/robotics_integration_tests/settings/settings.py +++ b/robotics_integration_tests/settings/settings.py @@ -58,6 +58,14 @@ def KEYVAULT_URI(self) -> str: GIT_REPOSITORY_FOR_MIGRATIONS_REF: str = Field(default="latest") BACKEND_PROJECT_FILE_FOLDER: str = Field(default="backend/api") + # Path to a local flotilla checkout to take migrations from. When set, it + # takes precedence over cloning GIT_REPOSITORY_FOR_MIGRATIONS from GitHub, and + # GIT_REPOSITORY_FOR_MIGRATIONS_REF is ignored. Use this together with locally + # built images (see scripts/build_local_images.sh) so that the schema and the + # application code come from the same source. Uncommitted and untracked + # migrations are included, since the directory is copied rather than cloned. + FLOTILLA_MIGRATIONS_SOURCE_DIR: str = Field(default="") + # PostgreSQL Sara Database environment POSTGRESQL_IMAGE: str = Field(default="postgres:16") SARA_DB_USER: str = Field(default="sara") @@ -69,6 +77,9 @@ def KEYVAULT_URI(self) -> str: SARA_BACKEND_PROJECT_FILE_FOLDER: str = Field(default="api") + # See FLOTILLA_MIGRATIONS_SOURCE_DIR. + SARA_MIGRATIONS_SOURCE_DIR: str = Field(default="") + # Migrations runner environment RELATIVE_PATH_TO_DOCKERFILE: str = Field( default="./robotics_integration_tests/custom_images/migrations_runner/" diff --git a/scripts/build_local_images.sh b/scripts/build_local_images.sh new file mode 100755 index 0000000..cded66e --- /dev/null +++ b/scripts/build_local_images.sh @@ -0,0 +1,204 @@ +#!/usr/bin/env bash +# +# Build the service images from local working copies and point the integration +# tests at them, instead of the published :dev / :latest images. +# +# Why this exists +# --------------- +# The integration tests normally pull ghcr.io/equinor/{flotilla-backend,sara, +# isar-robot}. That means a change which spans armada *and* one of the services +# cannot be validated until the service change has been merged and an image +# published -- but the armada side of the change is what proves the service side +# works. This script closes that gap: build everything locally, run the suite, +# then merge in confidence. +# +# Usage +# ----- +# scripts/build_local_images.sh # build and verify the images +# scripts/build_local_images.sh --run # ... and then run the full suite +# scripts/build_local_images.sh --help +# +# Repository locations default to the superrepo sibling layout and can each be +# overridden: ISAR_DIR ISAR_ROBOT_DIR FLOTILLA_DIR SARA_DIR + +set -euo pipefail + +TAG="${LOCAL_IMAGE_TAG:-local}" +PLATFORM="linux/amd64" +RUN_TESTS=false + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ARMADA_DIR="$(dirname "$SCRIPT_DIR")" +SIBLING_ROOT="$(dirname "$ARMADA_DIR")" + +ISAR_DIR="${ISAR_DIR:-$SIBLING_ROOT/isar}" +ISAR_ROBOT_DIR="${ISAR_ROBOT_DIR:-$SIBLING_ROOT/isar-robot}" +FLOTILLA_DIR="${FLOTILLA_DIR:-$SIBLING_ROOT/flotilla}" +SARA_DIR="${SARA_DIR:-$SIBLING_ROOT/sara}" + +FLOTILLA_IMAGE="flotilla-backend:$TAG" +SARA_IMAGE="sara:$TAG" +ISAR_ROBOT_IMAGE="isar-robot:$TAG" +ISAR_ROBOT_BASE_IMAGE="isar-robot:$TAG-base" + +for arg in "$@"; do + case "$arg" in + --run) RUN_TESTS=true ;; + --help|-h) + # Print the header comment block, stopping at the first non-comment line. + awk 'NR==1{next} /^#/{sub(/^# ?/,""); print; next} {exit}' "${BASH_SOURCE[0]}" + exit 0 ;; + *) echo "Unknown argument: $arg (try --help)" >&2; exit 2 ;; + esac +done + +log() { printf '\n\033[1;34m==> %s\033[0m\n' "$*"; } +warn() { printf '\033[1;33mWARNING: %s\033[0m\n' "$*" >&2; } +die() { printf '\033[1;31mERROR: %s\033[0m\n' "$*" >&2; exit 1; } + +require_dir() { + [ -d "$1" ] || die "$2 not found at '$1'. Set $3 to override." +} + +require_dir "$ISAR_DIR" "isar repository" ISAR_DIR +require_dir "$ISAR_ROBOT_DIR" "isar-robot repository" ISAR_ROBOT_DIR +require_dir "$FLOTILLA_DIR" "flotilla repository" FLOTILLA_DIR +require_dir "$SARA_DIR" "sara repository" SARA_DIR + +docker info >/dev/null 2>&1 || die "Docker does not appear to be running." + +TMP_DIR="$(mktemp -d)" +cleanup() { rm -rf "$TMP_DIR"; } +trap cleanup EXIT + +# --------------------------------------------------------------------------- +# flotilla-backend and sara are built straight from the working tree, so any +# uncommitted changes are included. +# --------------------------------------------------------------------------- + +log "Building $FLOTILLA_IMAGE from $FLOTILLA_DIR" +docker build --platform "$PLATFORM" \ + -f "$FLOTILLA_DIR/backend/Dockerfile" \ + -t "$FLOTILLA_IMAGE" \ + "$FLOTILLA_DIR/backend" + +log "Building $SARA_IMAGE from $SARA_DIR" +docker build --platform "$PLATFORM" -t "$SARA_IMAGE" "$SARA_DIR" + +# --------------------------------------------------------------------------- +# isar-robot needs two steps. +# +# 1. Its Dockerfile does `RUN --mount=source=.git,target=.git,type=bind`, and +# setuptools_scm needs that git directory both to derive a version *and* to +# discover package data such as src/isar_robot/example_data/. In the superrepo +# the checkout is a submodule, so `.git` is a FILE ("gitdir: ...") which the +# `!.git/` allowlist entry in .dockerignore does not match. Building directly +# from the working tree therefore fails with "unable to detect version", and +# forcing SETUPTOOLS_SCM_PRETEND_VERSION instead produces a wheel that is +# missing example_data -- which only shows up much later as +# RobotRetrieveInspectionException during a mission. Cloning into a temporary +# directory yields a real .git directory with history and tags, so the stock +# Dockerfile works unmodified. +# +# 2. isar-robot's uv.lock pins `isar` from PyPI (the lock is generated with +# --no-sources, so the `[tool.uv.sources] isar = { path = "../isar" }` entry in +# pyproject.toml is ignored). To test local isar changes, the locally built +# wheel is installed over the released one. +# --------------------------------------------------------------------------- + +if [ -n "$(git -C "$ISAR_ROBOT_DIR" status --porcelain)" ]; then + warn "$ISAR_ROBOT_DIR has uncommitted changes." + warn "isar-robot is CLONED rather than built from the working tree, so those" + warn "changes will NOT be in the image. Commit them first if they matter." +fi + +log "Cloning isar-robot into a temporary directory (needs a real .git)" +git clone --quiet "$ISAR_ROBOT_DIR" "$TMP_DIR/isar-robot" \ + || die "Failed to clone $ISAR_ROBOT_DIR" + +log "Building $ISAR_ROBOT_BASE_IMAGE" +docker build --platform "$PLATFORM" -t "$ISAR_ROBOT_BASE_IMAGE" "$TMP_DIR/isar-robot" + +log "Building the isar wheel from $ISAR_DIR (working tree, uncommitted changes included)" +mkdir -p "$TMP_DIR/wheels" +if ! ( cd "$ISAR_DIR" && uv build --wheel -o "$TMP_DIR/wheels" ) >"$TMP_DIR/uv-build.log" 2>&1; then + cat "$TMP_DIR/uv-build.log" >&2 + die "Failed to build the isar wheel" +fi +ISAR_WHEEL="$(ls "$TMP_DIR"/wheels/isar-*.whl 2>/dev/null | head -1)" +[ -n "$ISAR_WHEEL" ] || die "No isar wheel was produced in $TMP_DIR/wheels" +echo "Built $(basename "$ISAR_WHEEL")" + +log "Overlaying the local isar onto $ISAR_ROBOT_IMAGE" +# The wheel keeps its original filename: uv rejects anything that is not a valid +# PEP 427 wheel name ("Must have a Python tag"). +mkdir -p "$TMP_DIR/overlay/wheels" +cp "$ISAR_WHEEL" "$TMP_DIR/overlay/wheels/" +cat > "$TMP_DIR/overlay/Dockerfile" <