Support migrations from a local checkout

Author: oysand

robotics_integration_tests/custom_containers/migrations_runner.py

@@ -1,13 +1,57 @@
 from pathlib import Path
 
 from docker.models.networks import Network
+from loguru import logger
 
 from robotics_integration_tests.custom_containers.image_builder import build_image_once
 from robotics_integration_tests.custom_containers.stream_logging_docker_container import (
     StreamLoggingDockerContainer,
 )
 from robotics_integration_tests.settings.settings import settings
 
+# Where a local checkout is mounted inside the migrations runner.
+_LOCAL_REPO_MOUNT = "/src"
+
+
+def _with_migrations_source(
+    container: StreamLoggingDockerContainer,
+    source_dir: str,
+    project_folder: str,
+    setting_name: str,
+) -> StreamLoggingDockerContainer:
+    """Take migrations from a local checkout instead of cloning from GitHub.
+
+    Mounted read-only; the entrypoint copies it into the container so that nothing
+    can be written back into the working tree. The copy means uncommitted and
+    untracked migrations are picked up, which is the reason to use this at all.
+
+    Validation is deliberately strict and happens here, before the container
+    starts: silently falling back to the GitHub clone would leave you believing
+    you had tested a local migration when you had not.
+    """
+    if not source_dir:
+        return container
+
+    resolved: Path = Path(source_dir).expanduser().resolve()
+    if not resolved.is_dir():
+        raise ValueError(
+            f"{setting_name} is set to '{source_dir}' (resolved to '{resolved}'), "
+            "which is not a directory."
+        )
+
+    project_dir: Path = resolved / project_folder
+    if not list(project_dir.glob("*.csproj")):
+        raise ValueError(
+            f"{setting_name} is set to '{resolved}', but no .csproj was found in "
+            f"'{project_dir}'. Point it at the repository root of the service, not "
+            "at the project folder."
+        )
+
+    logger.info(f"Using migrations from local checkout: {resolved}")
+    return container.with_volume_mapping(
+        str(resolved), _LOCAL_REPO_MOUNT, "ro"
+    ).with_env("LOCAL_REPO_PATH", _LOCAL_REPO_MOUNT)
+
 
 def create_migrations_runner_container(
     network: Network, postgres_connection_string: str, name: str = "flotilla_migrations", test_id: str = ""
@@ -30,7 +74,12 @@ def create_migrations_runner_container(
         .with_env("EF_PROJECT_PATH", settings.BACKEND_PROJECT_FILE_FOLDER)
         .with_env("EF_STARTUP_PATH", settings.BACKEND_PROJECT_FILE_FOLDER)
     )
-    return container
+    return _with_migrations_source(
+        container,
+        source_dir=settings.FLOTILLA_MIGRATIONS_SOURCE_DIR,
+        project_folder=settings.BACKEND_PROJECT_FILE_FOLDER,
+        setting_name="FLOTILLA_MIGRATIONS_SOURCE_DIR",
+    )
 
 
 def create_sara_migrations_runner_container(
@@ -54,4 +103,9 @@ def create_sara_migrations_runner_container(
         .with_env("EF_PROJECT_PATH", settings.SARA_BACKEND_PROJECT_FILE_FOLDER)
         .with_env("EF_STARTUP_PATH", settings.SARA_BACKEND_PROJECT_FILE_FOLDER)
     )
-    return container
+    return _with_migrations_source(
+        container,
+        source_dir=settings.SARA_MIGRATIONS_SOURCE_DIR,
+        project_folder=settings.SARA_BACKEND_PROJECT_FILE_FOLDER,
+        setting_name="SARA_MIGRATIONS_SOURCE_DIR",
+    )

robotics_integration_tests/custom_images/migrations_runner/entrypoint.sh

@@ -17,19 +17,50 @@ WAIT_FOR_DB_TIMEOUT="${WAIT_FOR_DB_TIMEOUT:-60}"
 : "${AZURE_CLIENT_ID:?AZURE_CLIENT_ID must be set at runtime}"
 : "${AZURE_TENANT_ID:?AZURE_TENANT_ID must be set at runtime}"
 
-echo "Cloning $GIT_REPO @ $GIT_REF ..."
 rm -rf /work/repo
-if [ "$GIT_REF" = "latest" ]; then
-  BRANCH=$(curl -s ${GITHUB_TOKEN:+-H "Authorization: token $GITHUB_TOKEN"} \
-    "https://api.github.com/repos/$GIT_REPO/releases/latest" | jq -r .tag_name)
-  echo "Resolved latest to $BRANCH"
+mkdir -p /work/repo
+
+if [ -n "${LOCAL_REPO_PATH:-}" ]; then
+  # Migrations come from a local checkout mounted read-only, so that locally
+  # built service images and the database schema come from the same source.
+  # Copied rather than used in place: the build writes bin/ and obj/ into the
+  # project, and the mount is read-only precisely so the caller's working tree
+  # cannot be modified.
+  #
+  # bin and obj are excluded not to save space but for correctness: they are
+  # host-architecture build output, and obj/project.assets.json embeds absolute
+  # host paths, both of which break restore inside this container.
+  echo "Copying migrations source from $LOCAL_REPO_PATH ..."
+  [ -d "$LOCAL_REPO_PATH" ] || { echo "LOCAL_REPO_PATH '$LOCAL_REPO_PATH' is not a directory."; exit 1; }
+  tar -C "$LOCAL_REPO_PATH" \
+      --exclude=bin \
+      --exclude=obj \
+      --exclude=node_modules \
+      --exclude=.git \
+      --exclude=TestResults \
+      -cf - . | tar -C /work/repo -xf -
 else
-  BRANCH="main"
+  echo "Cloning $GIT_REPO @ $GIT_REF ..."
+  if [ "$GIT_REF" = "latest" ]; then
+    BRANCH=$(curl -s ${GITHUB_TOKEN:+-H "Authorization: token $GITHUB_TOKEN"} \
+      "https://api.github.com/repos/$GIT_REPO/releases/latest" | jq -r .tag_name)
+    echo "Resolved latest to $BRANCH"
+  else
+    BRANCH="main"
+  fi
+  rm -rf /work/repo
+  git clone --depth 1 --branch "$BRANCH" "https://github.com/$GIT_REPO" /work/repo
 fi
-git clone --depth 1 --branch "$BRANCH" "https://github.com/$GIT_REPO" /work/repo
 
 cd /work/repo
 
+# Guard against a source that does not contain what we expect, rather than
+# letting it surface later as an opaque dotnet-ef failure.
+if ! ls "$EF_PROJECT_PATH"/*.csproj >/dev/null 2>&1; then
+  echo "No .csproj found at '$EF_PROJECT_PATH' in the migrations source."
+  exit 1
+fi
+
 echo "Restoring projects for EF design-time..."
 dotnet restore "$EF_STARTUP_PATH" || dotnet restore "$EF_PROJECT_PATH" || true
 

robotics_integration_tests/settings/settings.py

@@ -58,6 +58,14 @@ def KEYVAULT_URI(self) -> str:
     GIT_REPOSITORY_FOR_MIGRATIONS_REF: str = Field(default="latest")
     BACKEND_PROJECT_FILE_FOLDER: str = Field(default="backend/api")
 
+    # Path to a local flotilla checkout to take migrations from. When set, it
+    # takes precedence over cloning GIT_REPOSITORY_FOR_MIGRATIONS from GitHub, and
+    # GIT_REPOSITORY_FOR_MIGRATIONS_REF is ignored. Use this together with locally
+    # built images (see scripts/build_local_images.sh) so that the schema and the
+    # application code come from the same source. Uncommitted and untracked
+    # migrations are included, since the directory is copied rather than cloned.
+    FLOTILLA_MIGRATIONS_SOURCE_DIR: str = Field(default="")
+
     # PostgreSQL Sara Database environment
     POSTGRESQL_IMAGE: str = Field(default="postgres:16")
     SARA_DB_USER: str = Field(default="sara")
@@ -69,6 +77,9 @@ def KEYVAULT_URI(self) -> str:
 
     SARA_BACKEND_PROJECT_FILE_FOLDER: str = Field(default="api")
 
+    # See FLOTILLA_MIGRATIONS_SOURCE_DIR.
+    SARA_MIGRATIONS_SOURCE_DIR: str = Field(default="")
+
     # Migrations runner environment
     RELATIVE_PATH_TO_DOCKERFILE: str = Field(
         default="./robotics_integration_tests/custom_images/migrations_runner/"