Support running integration tests against locally built images

README.md

@@ -83,3 +83,39 @@ You may now run the tests with
 ```bash
 uv run pytest -s .
 ```
+
+### Running against locally built images
+
+By default the tests pull `ghcr.io/equinor/{flotilla-backend,sara,isar-robot}`. A change that
+spans armada *and* one of those services therefore cannot be verified until the service change
+is merged and an image published — even though the armada side is what proves the service side
+works.
+
+To close that gap, build the images from your local working copies:
+
+```bash
+scripts/build_local_images.sh           # build and verify
+scripts/build_local_images.sh --run     # ... and run the full suite against them
+```
+
+The script expects the sibling checkouts of the superrepo (`../isar`, `../isar-robot`,
+`../flotilla`, `../sara`); override with `ISAR_DIR`, `ISAR_ROBOT_DIR`, `FLOTILLA_DIR`,
+`SARA_DIR`. It verifies each image before handing back, because a subtly broken build otherwise
+shows up only as an unexplained timeout several minutes into the suite.
+
+The database schema is taken from the same local checkouts, via `FLOTILLA_MIGRATIONS_SOURCE_DIR`
+and `SARA_MIGRATIONS_SOURCE_DIR`, so application code and schema always agree. The directory is
+mounted read-only and copied into the migrations container, which means **uncommitted and
+untracked migrations are picked up**. Set either variable on its own if you want to mix a local
+schema with published images.
+
+Two things worth knowing:
+
+- **`flotilla`, `sara` and `isar` are built from the working tree**, so uncommitted changes are
+  included. **`isar-robot` is cloned**, so only committed changes are — the script warns if that
+  checkout is dirty. It has to be cloned because its Dockerfile bind-mounts `.git`, and in the
+  superrepo that is a submodule *file* rather than a directory.
+- `isar-robot`'s `uv.lock` pins `isar` from PyPI, so the locally built `isar` wheel is installed
+  over the released one.
+
+The mosquitto broker is always the published image.

robotics_integration_tests/custom_containers/image_builder.py

@@ -0,0 +1,20 @@
+import fcntl
+import tempfile
+from pathlib import Path
+
+from loguru import logger
+from testcontainers.core.image import DockerImage
+
+
+def build_image_once(path: str, tag: str) -> str:
+    lock_path: Path = (
+        Path(tempfile.gettempdir()) / f"armada-build-{tag.replace('/', '_')}.lock"
+    )
+
+    with open(lock_path, "w") as lock_file:
+        fcntl.flock(lock_file, fcntl.LOCK_EX)
+        try:
+            logger.debug(f"Building image {tag} from {path}")
+            return str(DockerImage(path=path, tag=tag).build())
+        finally:
+            fcntl.flock(lock_file, fcntl.LOCK_UN)

robotics_integration_tests/custom_containers/migrations_runner.py

@@ -1,24 +1,68 @@
 from pathlib import Path
 
 from docker.models.networks import Network
-from testcontainers.core.image import DockerImage
+from loguru import logger
 
+from robotics_integration_tests.custom_containers.image_builder import build_image_once
 from robotics_integration_tests.custom_containers.stream_logging_docker_container import (
     StreamLoggingDockerContainer,
 )
 from robotics_integration_tests.settings.settings import settings
 
+# Where a local checkout is mounted inside the migrations runner.
+_LOCAL_REPO_MOUNT = "/src"
+
+
+def _with_migrations_source(
+    container: StreamLoggingDockerContainer,
+    source_dir: str,
+    project_folder: str,
+    setting_name: str,
+) -> StreamLoggingDockerContainer:
+    """Take migrations from a local checkout instead of cloning from GitHub.
+
+    Mounted read-only; the entrypoint copies it into the container so that nothing
+    can be written back into the working tree. The copy means uncommitted and
+    untracked migrations are picked up, which is the reason to use this at all.
+
+    Validation is deliberately strict and happens here, before the container
+    starts: silently falling back to the GitHub clone would leave you believing
+    you had tested a local migration when you had not.
+    """
+    if not source_dir:
+        return container
+
+    resolved: Path = Path(source_dir).expanduser().resolve()
+    if not resolved.is_dir():
+        raise ValueError(
+            f"{setting_name} is set to '{source_dir}' (resolved to '{resolved}'), "
+            "which is not a directory."
+        )
+
+    project_dir: Path = resolved / project_folder
+    if not list(project_dir.glob("*.csproj")):
+        raise ValueError(
+            f"{setting_name} is set to '{resolved}', but no .csproj was found in "
+            f"'{project_dir}'. Point it at the repository root of the service, not "
+            "at the project folder."
+        )
+
+    logger.info(f"Using migrations from local checkout: {resolved}")
+    return container.with_volume_mapping(
+        str(resolved), _LOCAL_REPO_MOUNT, "ro"
+    ).with_env("LOCAL_REPO_PATH", _LOCAL_REPO_MOUNT)
+
 
 def create_migrations_runner_container(
     network: Network, postgres_connection_string: str, name: str = "flotilla_migrations", test_id: str = ""
 ) -> StreamLoggingDockerContainer:
-    migrations_runner_image: DockerImage = DockerImage(
+    migrations_runner_image: str = build_image_once(
         path=str(Path(settings.RELATIVE_PATH_TO_DOCKERFILE).resolve(strict=True)),
         tag="flotilla-migrations-runner",
-    ).build()
+    )
 
     container = (
-        StreamLoggingDockerContainer(image=str(migrations_runner_image))
+        StreamLoggingDockerContainer(image=migrations_runner_image)
         .with_name(f"{name}-{test_id}")
         .with_network(network)
         .with_env("DATABASE_URL", postgres_connection_string)
@@ -30,19 +74,24 @@ def create_migrations_runner_container(
         .with_env("EF_PROJECT_PATH", settings.BACKEND_PROJECT_FILE_FOLDER)
         .with_env("EF_STARTUP_PATH", settings.BACKEND_PROJECT_FILE_FOLDER)
     )
-    return container
+    return _with_migrations_source(
+        container,
+        source_dir=settings.FLOTILLA_MIGRATIONS_SOURCE_DIR,
+        project_folder=settings.BACKEND_PROJECT_FILE_FOLDER,
+        setting_name="FLOTILLA_MIGRATIONS_SOURCE_DIR",
+    )
 
 
 def create_sara_migrations_runner_container(
     network: Network, postgres_connection_string: str, name: str = "sara_migrations", test_id: str = ""
 ) -> StreamLoggingDockerContainer:
-    sara_migrations_runner_image: DockerImage = DockerImage(
+    sara_migrations_runner_image: str = build_image_once(
         path=str(Path(settings.RELATIVE_PATH_TO_DOCKERFILE).resolve(strict=True)),
         tag="sara-migrations-runner",
-    ).build()
+    )
 
     container = (
-        StreamLoggingDockerContainer(image=str(sara_migrations_runner_image))
+        StreamLoggingDockerContainer(image=sara_migrations_runner_image)
         .with_name(f"{name}-{test_id}")
         .with_network(network)
         .with_env("DATABASE_URL", postgres_connection_string)
@@ -54,4 +103,9 @@ def create_sara_migrations_runner_container(
         .with_env("EF_PROJECT_PATH", settings.SARA_BACKEND_PROJECT_FILE_FOLDER)
         .with_env("EF_STARTUP_PATH", settings.SARA_BACKEND_PROJECT_FILE_FOLDER)
     )
-    return container
+    return _with_migrations_source(
+        container,
+        source_dir=settings.SARA_MIGRATIONS_SOURCE_DIR,
+        project_folder=settings.SARA_BACKEND_PROJECT_FILE_FOLDER,
+        setting_name="SARA_MIGRATIONS_SOURCE_DIR",
+    )

robotics_integration_tests/custom_containers/teams_webhook_receiver.py

@@ -4,8 +4,7 @@
 import requests
 from docker.models.networks import Network
 from loguru import logger
-from testcontainers.core.image import DockerImage
-
+from robotics_integration_tests.custom_containers.image_builder import build_image_once
 from robotics_integration_tests.custom_containers.stream_logging_docker_container import (
     StreamLoggingDockerContainer,
 )
@@ -70,13 +69,10 @@ def create_teams_webhook_receiver_container(
     test_id: str = "",
 ) -> tuple[StreamLoggingDockerContainer, TeamsWebhookReceiver]:
     """Build the image and return both the raw container and a typed wrapper."""
-    image: DockerImage = DockerImage(
-        path=str(_IMAGE_DIR),
-        tag="teams-webhook-receiver",
-    ).build()
+    image: str = build_image_once(path=str(_IMAGE_DIR), tag="teams-webhook-receiver")
 
     container: StreamLoggingDockerContainer = (
-        StreamLoggingDockerContainer(image=str(image))
+        StreamLoggingDockerContainer(image=image)
         .with_name(f"{name}-{test_id}")
         .with_exposed_ports(port)
         .with_network(network)

robotics_integration_tests/custom_images/migrations_runner/entrypoint.sh

@@ -17,19 +17,50 @@ WAIT_FOR_DB_TIMEOUT="${WAIT_FOR_DB_TIMEOUT:-60}"
 : "${AZURE_CLIENT_ID:?AZURE_CLIENT_ID must be set at runtime}"
 : "${AZURE_TENANT_ID:?AZURE_TENANT_ID must be set at runtime}"
 
-echo "Cloning $GIT_REPO @ $GIT_REF ..."
 rm -rf /work/repo
-if [ "$GIT_REF" = "latest" ]; then
-  BRANCH=$(curl -s ${GITHUB_TOKEN:+-H "Authorization: token $GITHUB_TOKEN"} \
-    "https://api.github.com/repos/$GIT_REPO/releases/latest" | jq -r .tag_name)
-  echo "Resolved latest to $BRANCH"
+mkdir -p /work/repo
+
+if [ -n "${LOCAL_REPO_PATH:-}" ]; then
+  # Migrations come from a local checkout mounted read-only, so that locally
+  # built service images and the database schema come from the same source.
+  # Copied rather than used in place: the build writes bin/ and obj/ into the
+  # project, and the mount is read-only precisely so the caller's working tree
+  # cannot be modified.
+  #
+  # bin and obj are excluded not to save space but for correctness: they are
+  # host-architecture build output, and obj/project.assets.json embeds absolute
+  # host paths, both of which break restore inside this container.
+  echo "Copying migrations source from $LOCAL_REPO_PATH ..."
+  [ -d "$LOCAL_REPO_PATH" ] || { echo "LOCAL_REPO_PATH '$LOCAL_REPO_PATH' is not a directory."; exit 1; }
+  tar -C "$LOCAL_REPO_PATH" \
+      --exclude=bin \
+      --exclude=obj \
+      --exclude=node_modules \
+      --exclude=.git \
+      --exclude=TestResults \
+      -cf - . | tar -C /work/repo -xf -
 else
-  BRANCH="main"
+  echo "Cloning $GIT_REPO @ $GIT_REF ..."
+  if [ "$GIT_REF" = "latest" ]; then
+    BRANCH=$(curl -s ${GITHUB_TOKEN:+-H "Authorization: token $GITHUB_TOKEN"} \
+      "https://api.github.com/repos/$GIT_REPO/releases/latest" | jq -r .tag_name)
+    echo "Resolved latest to $BRANCH"
+  else
+    BRANCH="main"
+  fi
+  rm -rf /work/repo
+  git clone --depth 1 --branch "$BRANCH" "https://github.com/$GIT_REPO" /work/repo
 fi
-git clone --depth 1 --branch "$BRANCH" "https://github.com/$GIT_REPO" /work/repo
 
 cd /work/repo
 
+# Guard against a source that does not contain what we expect, rather than
+# letting it surface later as an opaque dotnet-ef failure.
+if ! ls "$EF_PROJECT_PATH"/*.csproj >/dev/null 2>&1; then
+  echo "No .csproj found at '$EF_PROJECT_PATH' in the migrations source."
+  exit 1
+fi
+
 echo "Restoring projects for EF design-time..."
 dotnet restore "$EF_STARTUP_PATH" || dotnet restore "$EF_PROJECT_PATH" || true
 

robotics_integration_tests/settings/settings.py

@@ -58,6 +58,14 @@ def KEYVAULT_URI(self) -> str:
     GIT_REPOSITORY_FOR_MIGRATIONS_REF: str = Field(default="latest")
     BACKEND_PROJECT_FILE_FOLDER: str = Field(default="backend/api")
 
+    # Path to a local flotilla checkout to take migrations from. When set, it
+    # takes precedence over cloning GIT_REPOSITORY_FOR_MIGRATIONS from GitHub, and
+    # GIT_REPOSITORY_FOR_MIGRATIONS_REF is ignored. Use this together with locally
+    # built images (see scripts/build_local_images.sh) so that the schema and the
+    # application code come from the same source. Uncommitted and untracked
+    # migrations are included, since the directory is copied rather than cloned.
+    FLOTILLA_MIGRATIONS_SOURCE_DIR: str = Field(default="")
+
     # PostgreSQL Sara Database environment
     POSTGRESQL_IMAGE: str = Field(default="postgres:16")
     SARA_DB_USER: str = Field(default="sara")
@@ -69,6 +77,9 @@ def KEYVAULT_URI(self) -> str:
 
     SARA_BACKEND_PROJECT_FILE_FOLDER: str = Field(default="api")
 
+    # See FLOTILLA_MIGRATIONS_SOURCE_DIR.
+    SARA_MIGRATIONS_SOURCE_DIR: str = Field(default="")
+
     # Migrations runner environment
     RELATIVE_PATH_TO_DOCKERFILE: str = Field(
         default="./robotics_integration_tests/custom_images/migrations_runner/"