diff --git a/.github/workflows/run_integration_tests.yml b/.github/workflows/run_integration_tests.yml index f08afda..d419b18 100644 --- a/.github/workflows/run_integration_tests.yml +++ b/.github/workflows/run_integration_tests.yml @@ -10,8 +10,14 @@ on: default: "latest" secrets: INTEGRATION_TEST_AZURE_CLIENT_SECRET: - required: true - description: "Azure client secret for the service principal used in integration tests with access to keyvault" + required: false + description: | + Azure client secret for the service principal with read access to the + integration-test key vault, which now holds only the MQTT credentials. + No longer used for authentication between the services: those tokens come + from a local OAuth2 mock issuer. Kept optional rather than removed because + this workflow is consumed at @main by every robotics repo and each one + declares it; removing it outright would break them all at once. workflow_dispatch: inputs: @@ -25,6 +31,9 @@ permissions: contents: read env: + # Only MQTT credentials are still read from the key vault. Authentication between + # the services is handled by a local OAuth2 mock issuer started by the test suite, + # so no Entra ID app registrations are involved. KEYVAULT_NAME: FlotillaTestsKv AZURE_TENANT_ID: 3aa4a235-b6e2-48d5-9195-7fcf05b459b0 AZURE_CLIENT_ID: 17d7c036-e4ff-4df6-87fd-0d648a36a727 @@ -53,15 +62,15 @@ jobs: run: | echo "Retrieving secrets from Key Vault: $KEYVAULT_NAME" - # You can list the secret names here + # MQTT authentication is username/password, validated by the broker + # against the hashed passwd_file committed in equinor/flotilla, so these + # remain real secrets. The app-registration client secrets that used to + # be fetched here are gone: service-to-service auth now runs against a + # local OAuth2 mock issuer. secrets=( - INTEGRATION-TESTS-CLIENT-SECRET FLOTILLA-MQTT-PASSWORD - FLOTILLA-AZURE-CLIENT-SECRET FLOTILLA-BROKER-SERVER-KEY - ISAR-AZURE-CLIENT-SECRET ISAR-MQTT-PASSWORD - SARA-AZURE-CLIENT-SECRET SARA-MQTT-PASSWORD ) @@ -120,13 +129,9 @@ jobs: GIT_REPOSITORY_FOR_MIGRATIONS_REF: ${{ steps.images.outputs.git_ref }} SARA_GIT_REPOSITORY_FOR_MIGRATIONS_REF: ${{ steps.images.outputs.git_ref }} - INTEGRATION_TESTS_CLIENT_SECRET: ${{ env.INTEGRATION_TESTS_CLIENT_SECRET }} FLOTILLA_MQTT_PASSWORD: ${{ env.FLOTILLA_MQTT_PASSWORD }} - FLOTILLA_AZURE_CLIENT_SECRET: ${{ env.FLOTILLA_AZURE_CLIENT_SECRET }} FLOTILLA_BROKER_SERVER_KEY: ${{ env.FLOTILLA_BROKER_SERVER_KEY }} - ISAR_AZURE_CLIENT_SECRET: ${{ env.ISAR_AZURE_CLIENT_SECRET }} ISAR_MQTT_PASSWORD: ${{ env.ISAR_MQTT_PASSWORD }} - SARA_AZURE_CLIENT_SECRET: ${{ env.SARA_AZURE_CLIENT_SECRET }} SARA_MQTT_PASSWORD: ${{ env.SARA_MQTT_PASSWORD }} run: uv run --frozen pytest -s -n auto robotics_integration_tests diff --git a/README.md b/README.md index 9195c5b..7605280 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,29 @@ The following components are currently included in the integration tests: - PostgreSQL database - Azure Blob Storage (emulated with Azurite) - [ISAR Robot](https://github.com/equinor/isar-robot) (your friendly neighbourhood mocked robot which provides the answers you need) +- [SARA](https://github.com/equinor/sara) (storage and analysis of robot acquired data) +- A local OAuth2 mock issuer (see [Authentication](#authentication)) + +## Authentication +The tests run with authentication **enabled and genuinely exercised**, but without Microsoft +Entra ID. A local [oauth2-mock-server](https://github.com/axa-group/oauth2-mock-server) +container acts as the OpenID Connect issuer for the whole stack: + +- Flotilla and SARA run with `ASPNETCORE_ENVIRONMENT=IntegrationTest`, which selects their + `appsettings.IntegrationTest.json` and points token validation at the mock. +- ISAR is pointed at the mock with `ISAR_OPENID_CONFIG_URL`. +- Flotilla acquires its downstream ISAR/SARA tokens from the mock too, via a + `GenericOidcAuthorizationHeaderProvider` registered only in that environment. +- The test process mints its own tokens from the same issuer. + +This means **no app registrations, no tenant and no client secrets** are needed, and there is +nothing to rotate. The container fixtures assert that each service rejects unauthenticated +callers before any test runs, and the mission tests attempt unauthorised interference mid-flight +(wrong audience, missing role) and then assert the mission completed unaffected — so a +misconfiguration cannot silently disable authentication. + +MQTT is the one exception: it uses username/password validated by the broker against the +hashed `passwd_file` committed in `equinor/flotilla`, so those credentials remain real secrets. ## Run the integration tests through remote workflow call To run the integration tests in a remote repository, this [workflow](./.github/workflows/run_integration_tests.yml) has been set up. @@ -55,6 +78,10 @@ This snippet will enable you to run the integration tests manually and automatic INTEGRATION_TEST_AZURE_CLIENT_SECRET ``` +This secret now only grants read access to the MQTT credentials in the key vault; it is no +longer used for authentication between the services. It is still declared by every consuming +repository, so it is kept for compatibility rather than removed. + The input `lane` determines which image tag should be applied to the internally developed packages like Flotilla and ISAR. If input is set as `lane=dev` the newest development images (corresponding to newest push to main branch) will be used while `lane=latest` will use the newest release. ## Local development @@ -63,23 +90,57 @@ Clone the repository and install dependencies with [uv](https://docs.astral.sh/u uv sync ``` -Ensure the following secrets are populated in your local environment, either as environment variables or in a `.env` file in the repository root directory. +Ensure the following secrets are populated in your local environment, either as environment variables or in a `.env` file in the repository root directory. These are the MQTT credentials; no +Azure app registration secrets are needed, and you do **not** need to be logged in with `az`. ``` -INTEGRATION_TESTS_CLIENT_SECRET -FLOTILLA_AZURE_CLIENT_SECRET FLOTILLA_BROKER_SERVER_KEY FLOTILLA_MQTT_PASSWORD -ISAR_AZURE_CLIENT_SECRET ISAR_MQTT_PASSWORD -SARA_AZURE_CLIENT_SECRET SARA_MQTT_PASSWORD ``` -They may all be found in the integration test [keyvault](https://portal.azure.com/#@StatoilSRM.onmicrosoft.com/resource/subscriptions/c389567b-2dd0-41fa-a5da-d86b81f80bda/resourceGroups/FlotillaIntegrationTests/providers/Microsoft.KeyVault/vaults/FlotillaTestsKv/overview). +They may be found in the integration test [keyvault](https://portal.azure.com/#@StatoilSRM.onmicrosoft.com/resource/subscriptions/c389567b-2dd0-41fa-a5da-d86b81f80bda/resourceGroups/FlotillaIntegrationTests/providers/Microsoft.KeyVault/vaults/FlotillaTestsKv/overview). You may now run the tests with ```bash uv run pytest -s . ``` + +### Running against locally built images + +By default the tests pull `ghcr.io/equinor/{flotilla-backend,sara,isar-robot}`. A change that +spans armada *and* one of those services therefore cannot be verified until the service change +is merged and an image published — even though the armada side is what proves the service side +works. + +To close that gap, build the images from your local working copies: + +```bash +scripts/build_local_images.sh # build and verify +scripts/build_local_images.sh --run # ... and run the full suite against them +``` + +The script expects the sibling checkouts of the superrepo (`../isar`, `../isar-robot`, +`../flotilla`, `../sara`); override with `ISAR_DIR`, `ISAR_ROBOT_DIR`, `FLOTILLA_DIR`, +`SARA_DIR`. It verifies each image before handing back, because a subtly broken build otherwise +shows up only as an unexplained timeout several minutes into the suite. + +The database schema is taken from the same local checkouts, via `FLOTILLA_MIGRATIONS_SOURCE_DIR` +and `SARA_MIGRATIONS_SOURCE_DIR`, so application code and schema always agree. The directory is +mounted read-only and copied into the migrations container, which means **uncommitted and +untracked migrations are picked up**. Set either variable on its own if you want to mix a local +schema with published images. + +Two things worth knowing: + +- **`flotilla`, `sara` and `isar` are built from the working tree**, so uncommitted changes are + included. **`isar-robot` is cloned**, so only committed changes are — the script warns if that + checkout is dirty. It has to be cloned because its Dockerfile bind-mounts `.git`, and in the + superrepo that is a submodule *file* rather than a directory. +- `isar-robot`'s `uv.lock` pins `isar` from PyPI, so the locally built `isar` wheel is installed + over the released one. + +The mosquitto broker is always the published image, and the OAuth2 mock is built automatically +by the test fixtures. diff --git a/pyproject.toml b/pyproject.toml index 62b4e21..f2df299 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,13 +11,10 @@ dependencies = [ "pytest", "testcontainers[postgres,azurite,mqtt]", "azure-storage-blob", - "azure-identity", - "azure-keyvault-secrets", "pydantic", "pydantic-settings", "loguru", "python-dotenv", - "msal", "requests" ] diff --git a/robotics_integration_tests/armada.py b/robotics_integration_tests/armada.py index 82f2a5d..85d56c8 100644 --- a/robotics_integration_tests/armada.py +++ b/robotics_integration_tests/armada.py @@ -10,6 +10,7 @@ ) from robotics_integration_tests.custom_containers.isar import IsarRobot from robotics_integration_tests.custom_containers.mosquitto import FlotillaBroker +from robotics_integration_tests.custom_containers.oauth2_mock import OAuth2Mock from robotics_integration_tests.custom_containers.postgres import ( FlotillaDatabase, SaraDatabase, @@ -17,13 +18,12 @@ from robotics_integration_tests.custom_containers.teams_webhook_receiver import ( TeamsWebhookReceiver, ) -from robotics_integration_tests.utilities.keyvault import Keyvault class Armada: def __init__(self) -> None: self.network: Network | None = None - self.keyvault: Keyvault | None = None + self.oauth_mock: OAuth2Mock | None = None self.test_id: str = "" self.flotilla_database: FlotillaDatabase | None = None self.flotilla_broker: FlotillaBroker | None = None diff --git a/robotics_integration_tests/conftest.py b/robotics_integration_tests/conftest.py index 7668dca..426f321 100644 --- a/robotics_integration_tests/conftest.py +++ b/robotics_integration_tests/conftest.py @@ -43,6 +43,10 @@ create_flotilla_broker_container, FlotillaBroker, ) +from robotics_integration_tests.custom_containers.oauth2_mock import ( + OAuth2Mock, + create_oauth2_mock_container, +) from robotics_integration_tests.custom_containers.postgres import ( SaraDatabase, create_postgres_container, @@ -61,13 +65,20 @@ StreamLoggingDockerContainer, ) from robotics_integration_tests.settings.settings import settings +from robotics_integration_tests.utilities.authentication import ( + configure_mock_issuer, + reset_mock_issuer, +) +from robotics_integration_tests.utilities.authentication_assertions import ( + assert_authentication_is_enforced, + isar_url, +) from robotics_integration_tests.utilities.flotilla_backend_api import ( setup_robot_in_flotilla, wait_for_backend_to_be_responsive, populate_database_with_minimum_models, wait_for_database_to_be_populated, ) -from robotics_integration_tests.utilities.keyvault import Keyvault, ScopedKeyvault from robotics_integration_tests.utilities.sara_backend_api import ( wait_for_sara_to_be_responsive, ) @@ -127,28 +138,40 @@ def test_id(): @pytest.fixture -def keyvault(test_id: str): - scoped_keyvault: ScopedKeyvault = ScopedKeyvault( - prefix=test_id, - keyvault_name=settings.KEYVAULT_NAME, - client_secret=settings.FLOTILLA_AZURE_CLIENT_SECRET, - client_id=settings.FLOTILLA_AZURE_CLIENT_ID, - tenant_id=settings.AZURE_TENANT_ID, - ) +def network(): + with Network() as network: + yield network - yield scoped_keyvault - scoped_keyvault.cleanup() +@pytest.fixture +def oauth_mock(network: Network, test_id: str): + """Local OpenID Connect issuer standing in for Azure Entra ID. + Every service in the stack validates its access tokens against this + container, and the test process mints its own tokens from it, so the suite + needs no app registrations, no tenant and no client secrets. + """ + container, mock = create_oauth2_mock_container( + network=network, + alias=settings.OAUTH_MOCK_ALIAS, + port=settings.OAUTH_MOCK_PORT, + test_id=test_id, + ) + with container: + wait_for_port_mapping_to_be_available(container=container, port=mock.port) + mock.wait_until_ready() -@pytest.fixture -def network(): - with Network() as network: - yield network + # The API helpers build auth headers from module functions with no access + # to fixtures, so the host URL is handed over globally. + configure_mock_issuer(mock.host_url) + try: + yield mock + finally: + reset_mock_issuer() @pytest.fixture -def flotilla_database(network: Network, keyvault: Keyvault, test_id: str): +def flotilla_database(network: Network, test_id: str): with create_postgres_container(network, test_id=test_id) as database: wait_for_port_mapping_to_be_available(container=database, port=5432) logger.info( @@ -174,11 +197,6 @@ def flotilla_database(network: Network, keyvault: Keyvault, test_id: str): logger.info("Migrations completed successfully (container exited cleanly)") - keyvault.set_secret( - secret_name="flotilla-database-connection-string", - secret_value=connection_string, - ) - yield FlotillaDatabase( database=database, connection_string=connection_string, @@ -187,7 +205,7 @@ def flotilla_database(network: Network, keyvault: Keyvault, test_id: str): @pytest.fixture -def sara_database(network: Network, keyvault: Keyvault, test_id: str): +def sara_database(network: Network, test_id: str): with create_sara_postgres_container(network, test_id=test_id) as database: wait_for_port_mapping_to_be_available(container=database, port=5432) logger.info( @@ -213,11 +231,6 @@ def sara_database(network: Network, keyvault: Keyvault, test_id: str): logger.info("Sara migrations completed successfully (container exited cleanly)") - keyvault.set_secret( - secret_name="sara-database-connection-string", - secret_value=connection_string, - ) - yield SaraDatabase( database=database, connection_string=connection_string, @@ -226,7 +239,7 @@ def sara_database(network: Network, keyvault: Keyvault, test_id: str): @pytest.fixture -def armada_storage(network: Network, keyvault: Keyvault, test_id: str): +def armada_storage(network: Network, test_id: str): with ExitStack() as stack: azurite_containers: Dict[str, AzuriteStorageContainer] = {} @@ -259,16 +272,6 @@ def armada_storage(network: Network, keyvault: Keyvault, test_id: str): docker_connection_string=docker_connection_string, host_connection_string=host_connection_string, ) - if azurite_container_alias == settings.SARA_RAW_STORAGE_CONTAINER: - keyvault.set_secret( - secret_name="AZURE-STORAGE-CONNECTION-STRING-DATA", - secret_value=docker_connection_string, - ) - elif azurite_container_alias == settings.SARA_ANON_STORAGE_CONTAINER: - keyvault.set_secret( - secret_name="AZURE-STORAGE-CONNECTION-STRING-METADATA", - secret_value=docker_connection_string, - ) ensure_blob_containers(host_connection_string, "hua", "kaa", "nls", "test") @@ -304,15 +307,14 @@ def teams_webhook_receiver(network: Network, test_id: str): test_id=test_id, ) with container: - wait_for_port_mapping_to_be_available( - container=container, port=receiver.port - ) + wait_for_port_mapping_to_be_available(container=container, port=receiver.port) yield receiver @pytest.fixture def flotilla_backend( network: Network, + oauth_mock: OAuth2Mock, flotilla_database: FlotillaDatabase, teams_webhook_receiver: TeamsWebhookReceiver, test_id: str, @@ -333,6 +335,12 @@ def flotilla_backend( backend_url: str = f"http://localhost:{flotilla_backend.get_exposed_port(8000)}" wait_for_backend_to_be_responsive(backend_url=backend_url) + # Before anything is seeded, confirm the backend actually rejects + # unauthenticated callers. Every other assertion in the suite is about + # mission behaviour and would pass just as happily against an unsecured + # stack, so this is the only thing standing between us and silently + # testing a backend with authentication switched off. + assert_authentication_is_enforced(f"{backend_url}/robots") populate_database_with_minimum_models(backend_url=backend_url) wait_for_database_to_be_populated(backend_url=backend_url) @@ -348,6 +356,7 @@ def flotilla_backend( @pytest.fixture def sara( network: Network, + oauth_mock: OAuth2Mock, sara_database: SaraDatabase, armada_storage: ArmadaStorage, test_id: str, @@ -370,6 +379,7 @@ def sara( sara_url: str = f"http://localhost:{sara_container.get_exposed_port(8100)}" wait_for_sara_to_be_responsive(sara_url=sara_url) + assert_authentication_is_enforced(f"{sara_url}/api/analysis") yield Sara( sara=sara_container, @@ -382,9 +392,9 @@ def sara( @pytest.fixture def armada_without_robots( - keyvault: Keyvault, network: Network, test_id: str, + oauth_mock: OAuth2Mock, flotilla_broker: FlotillaBroker, flotilla_database: FlotillaDatabase, flotilla_backend: FlotillaBackend, @@ -395,9 +405,9 @@ def armada_without_robots( ): armada: Armada = Armada() - armada.keyvault = keyvault armada.network = network armada.test_id = test_id + armada.oauth_mock = oauth_mock armada.sara_database = sara_database armada.sara = sara armada.flotilla_database = flotilla_database @@ -409,21 +419,45 @@ def armada_without_robots( yield armada +def _blob_connection_strings(armada: Armada) -> tuple[str, str]: + """In-network Azurite connection strings for ISAR's data and metadata stores. + + These used to be passed through the real Key Vault; they are now read + straight off the Azurite fixture. + """ + containers = armada.armada_storage.azurite_containers + return ( + containers[settings.SARA_RAW_STORAGE_CONTAINER].docker_connection_string, + containers[settings.SARA_ANON_STORAGE_CONTAINER].docker_connection_string, + ) + + +def _assert_robots_require_authentication(armada: Armada) -> None: + """Confirm every ISAR robot rejects unauthenticated callers. + + Mirrors the check applied to flotilla and sara when their containers start. + An unauthenticated POST is refused before the handler runs, so this cannot + disturb the robot's state machine. + """ + for robot in armada.robots.values(): + assert_authentication_is_enforced( + isar_url(robot, "/schedule/stop-mission"), method="POST" + ) + + @pytest.fixture def armada_with_single_successful_robot(armada_without_robots: Armada): armada: Armada = armada_without_robots + blob_conn_data, blob_conn_metadata = _blob_connection_strings(armada) with create_isar_robot_container( network=armada.network, + openid_config_url=armada.oauth_mock.internal_openid_config_url, image=settings.ISAR_ROBOT_IMAGE, name=settings.ISAR_ROBOT_NAME, port=settings.ISAR_ROBOT_PORT, alias=settings.ISAR_ROBOT_ALIAS, - blob_storage_connection_string_data=armada.keyvault.get_secret( - "AZURE-STORAGE-CONNECTION-STRING-DATA" - ).value, - blob_storage_connection_string_metadata=armada.keyvault.get_secret( - "AZURE-STORAGE-CONNECTION-STRING-METADATA" - ).value, + blob_storage_connection_string_data=blob_conn_data, + blob_storage_connection_string_metadata=blob_conn_metadata, test_id=armada.test_id, ) as isar_robot: @@ -440,6 +474,7 @@ def armada_with_single_successful_robot(armada_without_robots: Armada): alias=settings.ISAR_ROBOT_ALIAS, installation_code=installation_code_for_robot, ) + _assert_robots_require_authentication(armada) armada.log_startup_info() yield armada @@ -447,19 +482,17 @@ def armada_with_single_successful_robot(armada_without_robots: Armada): @pytest.fixture def armada_with_single_failing_robot(armada_without_robots: Armada): armada: Armada = armada_without_robots + blob_conn_data, blob_conn_metadata = _blob_connection_strings(armada) with create_isar_robot_container( network=armada.network, + openid_config_url=armada.oauth_mock.internal_openid_config_url, image=settings.ISAR_ROBOT_IMAGE, name=settings.ISAR_ROBOT_NAME, port=settings.ISAR_ROBOT_PORT, alias=settings.ISAR_ROBOT_ALIAS, - blob_storage_connection_string_data=armada.keyvault.get_secret( - "AZURE-STORAGE-CONNECTION-STRING-DATA" - ).value, - blob_storage_connection_string_metadata=armada.keyvault.get_secret( - "AZURE-STORAGE-CONNECTION-STRING-METADATA" - ).value, + blob_storage_connection_string_data=blob_conn_data, + blob_storage_connection_string_metadata=blob_conn_metadata, should_fail_normal_task=True, test_id=armada.test_id, ) as isar_robot: @@ -477,6 +510,7 @@ def armada_with_single_failing_robot(armada_without_robots: Armada): alias=settings.ISAR_ROBOT_ALIAS, installation_code=installation_code_for_robot, ) + _assert_robots_require_authentication(armada) armada.log_startup_info() yield armada @@ -500,12 +534,7 @@ def armada_with_multiple_robots(armada_without_robots: Armada): 4. MissionFailThenLost – mission fails, fails to return home """ armada: Armada = armada_without_robots - blob_conn_data: str = armada.keyvault.get_secret( - "AZURE-STORAGE-CONNECTION-STRING-DATA" - ).value - blob_conn_metadata: str = armada.keyvault.get_secret( - "AZURE-STORAGE-CONNECTION-STRING-METADATA" - ).value + blob_conn_data, blob_conn_metadata = _blob_connection_strings(armada) robot_configs = [ { @@ -543,6 +572,7 @@ def armada_with_multiple_robots(armada_without_robots: Armada): container = stack.enter_context( create_isar_robot_container( network=armada.network, + openid_config_url=armada.oauth_mock.internal_openid_config_url, image=settings.ISAR_ROBOT_IMAGE, name=cfg["name"], port=settings.ISAR_ROBOT_PORT, @@ -575,6 +605,7 @@ def armada_with_multiple_robots(armada_without_robots: Armada): installation_code=installation_code, ) + _assert_robots_require_authentication(armada) armada.log_startup_info() yield armada diff --git a/robotics_integration_tests/custom_containers/flotilla_backend.py b/robotics_integration_tests/custom_containers/flotilla_backend.py index 61ada1b..fa97aa0 100644 --- a/robotics_integration_tests/custom_containers/flotilla_backend.py +++ b/robotics_integration_tests/custom_containers/flotilla_backend.py @@ -42,12 +42,10 @@ def create_flotilla_backend_container( .with_env("Mqtt__Host", settings.FLOTILLA_BROKER_ALIAS) .with_env("Mqtt__Port", settings.FLOTILLA_BROKER_PORT) .with_env("Mqtt__Password", settings.FLOTILLA_MQTT_PASSWORD) + # Selects appsettings.IntegrationTest.json, which points token validation + # at the mock issuer and turns off Key Vault and Redis. .with_env("ASPNETCORE_ENVIRONMENT", settings.ASPNETCORE_ENVIRONMENT) - .with_env("AZURE_CLIENT_SECRET", settings.FLOTILLA_AZURE_CLIENT_SECRET) - .with_env("AZURE_CLIENT_ID", settings.FLOTILLA_AZURE_CLIENT_ID) - .with_env("AZURE_TENANT_ID", settings.AZURE_TENANT_ID) - .with_env("KeyVault__VaultUri", settings.KEYVAULT_URI) .with_env("Database__PostgreSqlConnectionString", database_connection_string) - .with_env("AzureAd__ClientSecret", settings.FLOTILLA_AZURE_CLIENT_SECRET) - .with_env("TeamsNotification__WebhookUrl", teams_notification_webhook_url)) + .with_env("TeamsNotification__WebhookUrl", teams_notification_webhook_url) + ) return container diff --git a/robotics_integration_tests/custom_containers/image_builder.py b/robotics_integration_tests/custom_containers/image_builder.py new file mode 100644 index 0000000..8a032ce --- /dev/null +++ b/robotics_integration_tests/custom_containers/image_builder.py @@ -0,0 +1,45 @@ +import fcntl +import tempfile +from pathlib import Path + +from loguru import logger +from testcontainers.core.image import DockerImage + + +def build_image_once(path: str, tag: str) -> str: + """Build a local image, serialising concurrent builds of the same tag. + + Several fixtures build their image from a local Dockerfile, and each of them + does so per test. Under ``pytest -n auto`` that means a dozen worker + *processes* can invoke ``docker build`` for the same tag at the same moment. + Docker does not serialise that, and the losers fail with: + + BuildError: creating image failed because it already exists, but + accessing it also failed: No such image: + + The race is normally hidden because every worker after the first gets a full + layer-cache hit and finishes before anyone else starts. It surfaces as soon as + the build context changes -- exactly when someone edits one of these images -- + which makes it a confusing failure to meet. + + Session-scoped fixtures do not help here: with xdist each worker is its own + process and runs its own session. A file lock is what actually serialises + across processes. Once the first worker has built, the rest hit the cache and + return almost immediately. + + Returns + ------- + str + The image reference to run. + """ + lock_path: Path = ( + Path(tempfile.gettempdir()) / f"armada-build-{tag.replace('/', '_')}.lock" + ) + + with open(lock_path, "w") as lock_file: + fcntl.flock(lock_file, fcntl.LOCK_EX) + try: + logger.debug(f"Building image {tag} from {path}") + return str(DockerImage(path=path, tag=tag).build()) + finally: + fcntl.flock(lock_file, fcntl.LOCK_UN) diff --git a/robotics_integration_tests/custom_containers/isar.py b/robotics_integration_tests/custom_containers/isar.py index d88dac4..f3f7d67 100644 --- a/robotics_integration_tests/custom_containers/isar.py +++ b/robotics_integration_tests/custom_containers/isar.py @@ -28,6 +28,7 @@ def __init__( def create_isar_robot_container( network: Network, + openid_config_url: str, image: str = "ghcr.io/equinor/isar-robot:latest", name: str = "isar_robot", port: int = 3000, @@ -58,9 +59,12 @@ def create_isar_robot_container( .with_kwargs(platform="linux/amd64") .with_env("ISAR_MQTT_HOST", settings.FLOTILLA_BROKER_ALIAS) .with_env("ISAR_MQTT_PASSWORD", settings.ISAR_MQTT_PASSWORD) - .with_env("AZURE_CLIENT_SECRET", settings.ISAR_AZURE_CLIENT_SECRET) - .with_env("ISAR_AZURE_CLIENT_ID", settings.ISAR_AZURE_CLIENT_ID) - .with_env("ISAR_AZURE_TENANT_ID", settings.ISAR_AZURE_TENANT_ID) + # Validate access tokens against the local mock issuer rather than Entra + # ID. ISAR_AZURE_CLIENT_ID is the expected `aud`; note that settings.py + # lets a bare AZURE_CLIENT_ID override it, so that variable must not be + # set here. + .with_env("ISAR_OPENID_CONFIG_URL", openid_config_url) + .with_env("ISAR_AZURE_CLIENT_ID", settings.ISAR_AUDIENCE) .with_env("ISAR_STORAGE_BLOB_ENABLED", "true") .with_env("ISAR_BLOB_STORAGE_ACCOUNT_DATA", settings.AZURITE_ACCOUNT) .with_env( diff --git a/robotics_integration_tests/custom_containers/migrations_runner.py b/robotics_integration_tests/custom_containers/migrations_runner.py index 7a62172..7dd4622 100644 --- a/robotics_integration_tests/custom_containers/migrations_runner.py +++ b/robotics_integration_tests/custom_containers/migrations_runner.py @@ -1,57 +1,137 @@ from pathlib import Path from docker.models.networks import Network -from testcontainers.core.image import DockerImage +from loguru import logger +from robotics_integration_tests.custom_containers.image_builder import build_image_once from robotics_integration_tests.custom_containers.stream_logging_docker_container import ( StreamLoggingDockerContainer, ) from robotics_integration_tests.settings.settings import settings +# Where a local checkout is mounted inside the migrations runner. +_LOCAL_REPO_MOUNT = "/src" + + +def _with_migrations_source( + container: StreamLoggingDockerContainer, + source_dir: str, + project_folder: str, + setting_name: str, +) -> StreamLoggingDockerContainer: + """Take migrations from a local checkout instead of cloning from GitHub. + + Mounted read-only; the entrypoint copies it into the container so that nothing + can be written back into the working tree. The copy means uncommitted and + untracked migrations are picked up, which is the reason to use this at all. + + Validation is deliberately strict and happens here, before the container + starts: silently falling back to the GitHub clone would leave you believing + you had tested a local migration when you had not. + """ + if not source_dir: + return container + + resolved: Path = Path(source_dir).expanduser().resolve() + if not resolved.is_dir(): + raise ValueError( + f"{setting_name} is set to '{source_dir}' (resolved to '{resolved}'), " + "which is not a directory." + ) + + project_dir: Path = resolved / project_folder + if not list(project_dir.glob("*.csproj")): + raise ValueError( + f"{setting_name} is set to '{resolved}', but no .csproj was found in " + f"'{project_dir}'. Point it at the repository root of the service, not " + "at the project folder." + ) + + logger.info(f"Using migrations from local checkout: {resolved}") + return container.with_volume_mapping( + str(resolved), _LOCAL_REPO_MOUNT, "ro" + ).with_env("LOCAL_REPO_PATH", _LOCAL_REPO_MOUNT) + + +def _with_design_time_database_config( + container: StreamLoggingDockerContainer, postgres_connection_string: str +) -> StreamLoggingDockerContainer: + """Supply the connection string to EF's design-time context factory. + + ``dotnet ef database update --connection`` overrides the connection used for + the migration itself, but EF still instantiates ``DesignTimeContextFactory``, + which reads its own configuration and falls back to **Azure Key Vault** when + no connection string is present. Passing it as configuration short-circuits + that fallback, which is what removes the Key Vault dependency here. + + Flotilla and sara read different keys for this (``PostgreSqlConnectionString`` + vs ``postgresConnectionString``), so both are set. + """ + return ( + container.with_env( + "Database__postgresConnectionString", postgres_connection_string + ) + .with_env("Database__PostgreSqlConnectionString", postgres_connection_string) + .with_env("Database__ConnectionString", postgres_connection_string) + .with_env("Database__AllowedAuthMethods__0", "ConnectionString") + ) + def create_migrations_runner_container( - network: Network, postgres_connection_string: str, name: str = "flotilla_migrations", test_id: str = "" + network: Network, + postgres_connection_string: str, + name: str = "flotilla_migrations", + test_id: str = "", ) -> StreamLoggingDockerContainer: - migrations_runner_image: DockerImage = DockerImage( + migrations_runner_image: str = build_image_once( path=str(Path(settings.RELATIVE_PATH_TO_DOCKERFILE).resolve(strict=True)), tag="flotilla-migrations-runner", - ).build() + ) container = ( - StreamLoggingDockerContainer(image=str(migrations_runner_image)) + StreamLoggingDockerContainer(image=migrations_runner_image) .with_name(f"{name}-{test_id}") .with_network(network) .with_env("DATABASE_URL", postgres_connection_string) - .with_env("AZURE_CLIENT_SECRET", settings.FLOTILLA_AZURE_CLIENT_SECRET) - .with_env("AZURE_CLIENT_ID", settings.FLOTILLA_AZURE_CLIENT_ID) - .with_env("AZURE_TENANT_ID", settings.AZURE_TENANT_ID) .with_env("GIT_REPO", settings.GIT_REPOSITORY_FOR_MIGRATIONS) .with_env("GIT_REF", settings.GIT_REPOSITORY_FOR_MIGRATIONS_REF) .with_env("EF_PROJECT_PATH", settings.BACKEND_PROJECT_FILE_FOLDER) .with_env("EF_STARTUP_PATH", settings.BACKEND_PROJECT_FILE_FOLDER) ) - return container + container = _with_migrations_source( + container, + source_dir=settings.FLOTILLA_MIGRATIONS_SOURCE_DIR, + project_folder=settings.BACKEND_PROJECT_FILE_FOLDER, + setting_name="FLOTILLA_MIGRATIONS_SOURCE_DIR", + ) + return _with_design_time_database_config(container, postgres_connection_string) def create_sara_migrations_runner_container( - network: Network, postgres_connection_string: str, name: str = "sara_migrations", test_id: str = "" + network: Network, + postgres_connection_string: str, + name: str = "sara_migrations", + test_id: str = "", ) -> StreamLoggingDockerContainer: - sara_migrations_runner_image: DockerImage = DockerImage( + sara_migrations_runner_image: str = build_image_once( path=str(Path(settings.RELATIVE_PATH_TO_DOCKERFILE).resolve(strict=True)), tag="sara-migrations-runner", - ).build() + ) container = ( - StreamLoggingDockerContainer(image=str(sara_migrations_runner_image)) + StreamLoggingDockerContainer(image=sara_migrations_runner_image) .with_name(f"{name}-{test_id}") .with_network(network) .with_env("DATABASE_URL", postgres_connection_string) - .with_env("AZURE_CLIENT_SECRET", settings.SARA_AZURE_CLIENT_SECRET) - .with_env("AZURE_CLIENT_ID", settings.SARA_AZURE_CLIENT_ID) - .with_env("AZURE_TENANT_ID", settings.SARA_AZURE_TENANT_ID) .with_env("GIT_REPO", settings.SARA_GIT_REPOSITORY_FOR_MIGRATIONS) .with_env("GIT_REF", settings.SARA_GIT_REPOSITORY_FOR_MIGRATIONS_REF) .with_env("EF_PROJECT_PATH", settings.SARA_BACKEND_PROJECT_FILE_FOLDER) .with_env("EF_STARTUP_PATH", settings.SARA_BACKEND_PROJECT_FILE_FOLDER) ) - return container + container = _with_migrations_source( + container, + source_dir=settings.SARA_MIGRATIONS_SOURCE_DIR, + project_folder=settings.SARA_BACKEND_PROJECT_FILE_FOLDER, + setting_name="SARA_MIGRATIONS_SOURCE_DIR", + ) + return _with_design_time_database_config(container, postgres_connection_string) diff --git a/robotics_integration_tests/custom_containers/oauth2_mock.py b/robotics_integration_tests/custom_containers/oauth2_mock.py new file mode 100644 index 0000000..b0f8673 --- /dev/null +++ b/robotics_integration_tests/custom_containers/oauth2_mock.py @@ -0,0 +1,120 @@ +import time +from pathlib import Path + +import requests +from docker.models.networks import Network +from robotics_integration_tests.custom_containers.image_builder import build_image_once +from robotics_integration_tests.custom_containers.stream_logging_docker_container import ( + StreamLoggingDockerContainer, +) + +_IMAGE_DIR = Path(__file__).resolve().parent.parent / "custom_images" / "oauth2_mock" + + +class OAuth2Mock: + """Local OpenID Connect issuer replacing Azure Entra ID for the test run. + + Services reach it on the Docker network under ``alias``; the pytest process + reaches it on the published host port. Tokens carry the *in-network* issuer + either way, which is what the services validate against. + """ + + def __init__( + self, + container: StreamLoggingDockerContainer, + port: int, + alias: str, + ) -> None: + self.container = container + self.port = port + self.alias = alias + + @property + def internal_url(self) -> str: + """Issuer URL as seen by other containers on the same Docker network.""" + return f"http://{self.alias}:{self.port}" + + @property + def internal_openid_config_url(self) -> str: + """Discovery document URL to hand to the services.""" + return f"{self.internal_url}/.well-known/openid-configuration" + + @property + def host_url(self) -> str: + """URL reachable from the test host.""" + return f"http://localhost:{self.container.get_exposed_port(self.port)}" + + def wait_until_ready(self, timeout: int = 60) -> None: + """Block until the discovery document is served.""" + deadline = time.monotonic() + timeout + last_error: Exception | None = None + while time.monotonic() < deadline: + try: + response = requests.get( + f"{self.host_url}/.well-known/openid-configuration", timeout=5 + ) + if response.ok: + return + except requests.RequestException as error: # pragma: no cover - timing + last_error = error + time.sleep(0.5) + raise TimeoutError( + f"oauth2 mock did not become ready within {timeout}s: {last_error}" + ) + + def get_token(self, resource_client_id: str) -> str: + """Mint a token via the standard client-credentials flow. + + The audience is derived from the requested scope by the mock, so this + mirrors what the services themselves do. + """ + response = requests.post( + f"{self.host_url}/token", + data={ + "grant_type": "client_credentials", + "scope": f"{resource_client_id}/.default", + "client_id": "integration-tests", + }, + timeout=10, + ) + response.raise_for_status() + return response.json()["access_token"] + + def issue_token(self, audience: str, roles: list[str]) -> str: + """Mint a token with an explicit audience and role set. + + Used by negative tests: a mismatched audience must yield 401 and a + missing role must yield 403. + """ + response = requests.post( + f"{self.host_url}/issue-token", + json={"audience": audience, "roles": roles}, + timeout=10, + ) + response.raise_for_status() + return response.json()["access_token"] + + +def create_oauth2_mock_container( + network: Network, + name: str = "oauth_mock", + port: int = 8080, + alias: str = "oauth-mock", + test_id: str = "", +) -> tuple[StreamLoggingDockerContainer, OAuth2Mock]: + """Build the image and return both the raw container and a typed wrapper.""" + image: str = build_image_once(path=str(_IMAGE_DIR), tag="oauth2-mock") + + container: StreamLoggingDockerContainer = ( + StreamLoggingDockerContainer(image=image) + .with_name(f"{name}-{test_id}") + .with_exposed_ports(port) + .with_network(network) + .with_network_aliases(alias) + # The issuer must match the in-network alias so that the `iss` claim, the + # discovery document and the JWKS URI all agree for the services. + .with_env("ISSUER_URL", f"http://{alias}:{port}") + ) + + mock = OAuth2Mock(container=container, port=port, alias=alias) + return container, mock diff --git a/robotics_integration_tests/custom_containers/sara.py b/robotics_integration_tests/custom_containers/sara.py index eb46bfa..c640138 100644 --- a/robotics_integration_tests/custom_containers/sara.py +++ b/robotics_integration_tests/custom_containers/sara.py @@ -43,22 +43,11 @@ def create_sara_container( .with_env("Mqtt__Port", settings.FLOTILLA_BROKER_PORT) .with_env("Mqtt__Password", settings.SARA_MQTT_PASSWORD) .with_env("Mqtt__Username", "sara") + # Selects appsettings.IntegrationTest.json, which points token validation + # at the mock issuer, turns off Key Vault, and pins the database to the + # ConnectionString auth path. .with_env("ASPNETCORE_ENVIRONMENT", settings.ASPNETCORE_ENVIRONMENT) - # Pin the integration-test container to the ConnectionString auth path. - # sara's appsettings.Development.json now lists AppRegIdentity first - # (equinor/sara#396), which would otherwise wire EF Core against the - # dev Azure PostgreSQL server (unreachable from the test docker - # network). Mirrors the migration-workflow override in - # .github/workflows/run_dotnet_migrations.yml (PR #73). - .with_env("Database__AllowedAuthMethods__0", "ConnectionString") - .with_env("AZURE_CLIENT_SECRET", settings.SARA_AZURE_CLIENT_SECRET) - .with_env("AZURE_CLIENT_ID", settings.SARA_AZURE_CLIENT_ID) - .with_env("AZURE_TENANT_ID", settings.SARA_AZURE_TENANT_ID) - .with_env("KeyVault__VaultUri", settings.KEYVAULT_URI) .with_env("Database__postgresConnectionString", database_connection_string) - .with_env("AzureAd__ClientSecret", settings.SARA_AZURE_CLIENT_SECRET) - .with_env("AzureAd__ClientId", settings.SARA_AZURE_CLIENT_ID) - .with_env("AzureAd__TenantId", settings.SARA_AZURE_TENANT_ID) .with_env("Storage__RawStorageAccount", settings.AZURITE_ACCOUNT) .with_env( f"BlobStorage__{settings.AZURITE_ACCOUNT}__ConnectionString", diff --git a/robotics_integration_tests/custom_containers/teams_webhook_receiver.py b/robotics_integration_tests/custom_containers/teams_webhook_receiver.py index 438273c..f1c5c85 100644 --- a/robotics_integration_tests/custom_containers/teams_webhook_receiver.py +++ b/robotics_integration_tests/custom_containers/teams_webhook_receiver.py @@ -4,8 +4,7 @@ import requests from docker.models.networks import Network from loguru import logger -from testcontainers.core.image import DockerImage - +from robotics_integration_tests.custom_containers.image_builder import build_image_once from robotics_integration_tests.custom_containers.stream_logging_docker_container import ( StreamLoggingDockerContainer, ) @@ -70,13 +69,10 @@ def create_teams_webhook_receiver_container( test_id: str = "", ) -> tuple[StreamLoggingDockerContainer, TeamsWebhookReceiver]: """Build the image and return both the raw container and a typed wrapper.""" - image: DockerImage = DockerImage( - path=str(_IMAGE_DIR), - tag="teams-webhook-receiver", - ).build() + image: str = build_image_once(path=str(_IMAGE_DIR), tag="teams-webhook-receiver") container: StreamLoggingDockerContainer = ( - StreamLoggingDockerContainer(image=str(image)) + StreamLoggingDockerContainer(image=image) .with_name(f"{name}-{test_id}") .with_exposed_ports(port) .with_network(network) diff --git a/robotics_integration_tests/custom_images/migrations_runner/entrypoint.sh b/robotics_integration_tests/custom_images/migrations_runner/entrypoint.sh index 4ede427..2c4a2a5 100644 --- a/robotics_integration_tests/custom_images/migrations_runner/entrypoint.sh +++ b/robotics_integration_tests/custom_images/migrations_runner/entrypoint.sh @@ -12,24 +12,61 @@ EF_STARTUP_PATH="${EF_STARTUP_PATH:-$EF_PROJECT_PATH}" EF_CONTEXT="${EF_CONTEXT:-}" WAIT_FOR_DB_TIMEOUT="${WAIT_FOR_DB_TIMEOUT:-60}" -# ---------- Secrets required by EF migrations to build ---------- -: "${AZURE_CLIENT_SECRET:?AZURE_CLIENT_SECRET must be set at runtime}" -: "${AZURE_CLIENT_ID:?AZURE_CLIENT_ID must be set at runtime}" -: "${AZURE_TENANT_ID:?AZURE_TENANT_ID must be set at runtime}" +# ---------- Optional inputs ---------- +# The EF design-time context factories in flotilla and sara fall back to reading +# the connection string from Azure Key Vault when it is absent from config. The +# caller passes it in as configuration instead (Database__* below), so no Azure +# credentials are needed here. These are kept only for callers that still rely on +# the Key Vault path. +AZURE_CLIENT_SECRET="${AZURE_CLIENT_SECRET:-}" +AZURE_CLIENT_ID="${AZURE_CLIENT_ID:-}" +AZURE_TENANT_ID="${AZURE_TENANT_ID:-}" +export AZURE_CLIENT_SECRET AZURE_CLIENT_ID AZURE_TENANT_ID -echo "Cloning $GIT_REPO @ $GIT_REF ..." rm -rf /work/repo -if [ "$GIT_REF" = "latest" ]; then - BRANCH=$(curl -s ${GITHUB_TOKEN:+-H "Authorization: token $GITHUB_TOKEN"} \ - "https://api.github.com/repos/$GIT_REPO/releases/latest" | jq -r .tag_name) - echo "Resolved latest to $BRANCH" +mkdir -p /work/repo + +if [ -n "${LOCAL_REPO_PATH:-}" ]; then + # Migrations come from a local checkout mounted read-only, so that locally + # built service images and the database schema come from the same source. + # Copied rather than used in place: the build writes bin/ and obj/ into the + # project, and the mount is read-only precisely so the caller's working tree + # cannot be modified. + # + # bin and obj are excluded not to save space but for correctness: they are + # host-architecture build output, and obj/project.assets.json embeds absolute + # host paths, both of which break restore inside this container. + echo "Copying migrations source from $LOCAL_REPO_PATH ..." + [ -d "$LOCAL_REPO_PATH" ] || { echo "LOCAL_REPO_PATH '$LOCAL_REPO_PATH' is not a directory."; exit 1; } + tar -C "$LOCAL_REPO_PATH" \ + --exclude=bin \ + --exclude=obj \ + --exclude=node_modules \ + --exclude=.git \ + --exclude=TestResults \ + -cf - . | tar -C /work/repo -xf - else - BRANCH="main" + echo "Cloning $GIT_REPO @ $GIT_REF ..." + if [ "$GIT_REF" = "latest" ]; then + BRANCH=$(curl -s ${GITHUB_TOKEN:+-H "Authorization: token $GITHUB_TOKEN"} \ + "https://api.github.com/repos/$GIT_REPO/releases/latest" | jq -r .tag_name) + echo "Resolved latest to $BRANCH" + else + BRANCH="main" + fi + rm -rf /work/repo + git clone --depth 1 --branch "$BRANCH" "https://github.com/$GIT_REPO" /work/repo fi -git clone --depth 1 --branch "$BRANCH" "https://github.com/$GIT_REPO" /work/repo cd /work/repo +# Guard against a source that does not contain what we expect, rather than +# letting it surface later as an opaque dotnet-ef failure. +if ! ls "$EF_PROJECT_PATH"/*.csproj >/dev/null 2>&1; then + echo "No .csproj found at '$EF_PROJECT_PATH' in the migrations source." + exit 1 +fi + echo "Restoring projects for EF design-time..." dotnet restore "$EF_STARTUP_PATH" || dotnet restore "$EF_PROJECT_PATH" || true diff --git a/robotics_integration_tests/custom_images/oauth2_mock/.dockerignore b/robotics_integration_tests/custom_images/oauth2_mock/.dockerignore new file mode 100644 index 0000000..3c3629e --- /dev/null +++ b/robotics_integration_tests/custom_images/oauth2_mock/.dockerignore @@ -0,0 +1 @@ +node_modules diff --git a/robotics_integration_tests/custom_images/oauth2_mock/Dockerfile b/robotics_integration_tests/custom_images/oauth2_mock/Dockerfile new file mode 100644 index 0000000..5d45846 --- /dev/null +++ b/robotics_integration_tests/custom_images/oauth2_mock/Dockerfile @@ -0,0 +1,9 @@ +FROM node:24-alpine +RUN mkdir -p /app && chown node:node /app +WORKDIR /app +COPY package.json package-lock.json ./ +RUN npm ci --omit=dev +COPY index.mjs . +USER node +EXPOSE 8080 +CMD ["node", "index.mjs"] diff --git a/robotics_integration_tests/custom_images/oauth2_mock/index.mjs b/robotics_integration_tests/custom_images/oauth2_mock/index.mjs new file mode 100644 index 0000000..592bc91 --- /dev/null +++ b/robotics_integration_tests/custom_images/oauth2_mock/index.mjs @@ -0,0 +1,166 @@ +// Local OpenID Connect issuer for the robotics integration tests. +// +// Replaces Azure Entra ID so the suite needs no app registrations, no tenant and +// no client secrets. Services (ISAR, Flotilla, SARA) point their OIDC discovery at +// this container; the pytest process mints its own tokens against the same issuer. +// +// Design notes: +// +// * `issuer.url` is the *in-network* alias, so the `iss` claim, the discovery +// document and the JWKS URI all agree regardless of who is asking. The pytest +// process talks to the published host port but receives tokens whose `iss` is +// the in-network name -- which is exactly what the services validate against. +// +// * The built-in discovery document is used as-is. The upstream CoreDM reference +// replaces it in order to advertise a `localhost` authorization_endpoint for a +// browser; this suite is entirely machine-to-machine (client credentials), so +// that complication is not needed here. +// +// * `idp` and `acct` are deliberately never emitted. `fastapi-azure-auth`'s +// `is_guest()` treats `acct === 1`, or an `idp` that differs from `iss`, as a +// guest user and rejects the token with 403 before signature validation. +// +// * The signing key is built by hand rather than via `issuer.keys.generate()`, +// purely so it can carry `use: "sig"`. `fastapi-azure-auth` filters the JWKS +// with `if key.get('use') == 'sig'` and silently loads *zero* keys otherwise, +// failing with "Unable to verify token, no signing keys found". `generate()` +// accepts only `kid` and `crv`, so it cannot produce that field. + +import { generateKeyPairSync } from 'node:crypto' + +import { OAuth2Server, Events } from 'oauth2-mock-server' + +const PORT = Number(process.env.PORT ?? 8080) +const ISSUER_URL = process.env.ISSUER_URL ?? `http://oauth-mock:${PORT}` + +// Fake tenant id. Only needs to be stable and non-empty; nothing validates it +// against Entra, but Flotilla and SARA read `tid` off the principal. +const TENANT_ID = process.env.TENANT_ID ?? 'integration-test-tenant' + +// Roles stamped on every token unless a caller asks for something narrower via +// POST /issue-token. Union of what the three services require: +// Role.Admin flotilla super-admin (AccessRoleService.cs:38) +// Role.User.{HUA,KAA,NLS} per-installation roles seeded by the tests +// Mission.Control ISAR's REQUIRED_ROLE +const DEFAULT_ROLES = [ + 'Role.Admin', + 'Role.User.HUA', + 'Role.User.KAA', + 'Role.User.NLS', + 'Mission.Control', +] + +/** + * Turn a requested scope or resource into an `aud` claim. + * + * Callers ask for things like `isar-test/.default`, `api://isar-test/.default` + * or `api://isar-test/user_impersonation`. All of those must land on `isar-test`, + * because that is what the service compares its configured client id against. + */ +function audienceFromScope(scope) { + if (!scope) return undefined + // Client credentials may request several space-separated scopes; they all + // belong to one resource, so the first is representative. + const first = String(scope).trim().split(/\s+/)[0] + if (!first) return undefined + const withoutPrefix = first.replace(/^api:\/\//, '') + // Strip the trailing permission segment (`/.default`, `/user_impersonation`). + const slash = withoutPrefix.lastIndexOf('/') + return slash === -1 ? withoutPrefix : withoutPrefix.slice(0, slash) +} + +const KEY_ID = 'integration-test-key' + +const server = new OAuth2Server() + +const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }) +await server.issuer.keys.add({ + ...privateKey.export({ format: 'jwk' }), + kid: KEY_ID, + alg: 'RS256', + use: 'sig', +}) +server.issuer.url = ISSUER_URL + +// Lets POST /issue-token hand a specific role set to the token signer without +// threading state through the library's request pipeline. +const requestedRoles = new Map() + +server.service.on(Events.BeforeTokenSigning, (token, req) => { + const body = req.body ?? {} + const audience = audienceFromScope(body.scope ?? body.resource) + + token.payload.iss = ISSUER_URL + if (audience) token.payload.aud = audience + + // `ver` is required by fastapi-azure-auth's Claims model as a + // Literal['1.0', '2.0']; without it the token validates but User construction + // raises a pydantic ValidationError surfaced as a generic 401. + token.payload.ver = '2.0' + token.payload.tid = TENANT_ID + token.payload.sub = String(body.client_id ?? 'integration-tests') + token.payload.oid = String(body.client_id ?? 'integration-tests') + token.payload.appid = String(body.client_id ?? 'integration-tests') + token.payload.name = 'Integration Tests' + token.payload.preferred_username = 'integration-tests@example.com' + + const override = body.roles_token ? requestedRoles.get(body.roles_token) : undefined + if (body.roles_token) requestedRoles.delete(body.roles_token) + token.payload.roles = override ?? DEFAULT_ROLES + + // `iss`, `iat`, `exp` and `nbf` are set by the library. fastapi-azure-auth + // requires all four (plus `aud` and `sub`) and rejects the token with + // MissingRequiredClaimError otherwise, so fail loudly here rather than + // producing tokens that 401 deep inside a service. + for (const claim of ['iss', 'iat', 'exp', 'nbf', 'sub', 'aud', 'ver']) { + if (token.payload[claim] === undefined) { + throw new Error(`Refusing to sign token: required claim '${claim}' is missing`) + } + } + + // Guard against reintroducing the guest-user claims. See the header comment. + delete token.payload.idp + delete token.payload.acct +}) + +/** + * Mint a token with an explicit audience and role set. + * + * Used by negative tests: a wrong audience must produce 401, and a missing role + * must produce 403. Body: { "audience": "flotilla-test", "roles": ["Role.Admin"] } + */ +server.service.addRoute('POST', '/issue-token', async (req, res) => { + const body = req.body ?? {} + const audience = body.audience + const roles = body.roles ?? DEFAULT_ROLES + + if (!audience) { + res.writeHead(400, { 'Content-Type': 'application/json' }) + res.end(JSON.stringify({ error: 'invalid_request', error_description: 'audience is required' })) + return + } + + const token = await server.issuer.buildToken({ + kid: KEY_ID, + scopesOrTransform: (header, payload) => { + payload.iss = ISSUER_URL + payload.aud = audience + payload.ver = '2.0' + payload.tid = TENANT_ID + payload.sub = 'integration-tests' + payload.oid = 'integration-tests' + payload.appid = 'integration-tests' + payload.name = 'Integration Tests' + payload.preferred_username = 'integration-tests@example.com' + payload.roles = roles + delete payload.idp + delete payload.acct + }, + }) + + res.writeHead(200, { 'Content-Type': 'application/json' }) + res.end(JSON.stringify({ access_token: token, token_type: 'Bearer', expires_in: 3600 })) +}) + +await server.start(PORT, '0.0.0.0') +console.log(`oauth2 mock issuer listening on 0.0.0.0:${PORT}, issuer=${ISSUER_URL}`) diff --git a/robotics_integration_tests/custom_images/oauth2_mock/package-lock.json b/robotics_integration_tests/custom_images/oauth2_mock/package-lock.json new file mode 100644 index 0000000..8968387 --- /dev/null +++ b/robotics_integration_tests/custom_images/oauth2_mock/package-lock.json @@ -0,0 +1,71 @@ +{ + "name": "oauth2-mock", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "oauth2-mock", + "version": "1.0.0", + "dependencies": { + "oauth2-mock-server": "9.1.0" + } + }, + "node_modules/basic-auth": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", + "integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "5.1.2" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/is-plain-obj": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz", + "integrity": "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/jose": { + "version": "6.2.5", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.5.tgz", + "integrity": "sha512-2E5L2yRp03FnwreJLJX8/r7mHiZICCf8kG7fAsTWkSQTDAcc46NIZoQLKy+EJ8sPoJlxyS4OQR5H70LjIZZlIQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, + "node_modules/oauth2-mock-server": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/oauth2-mock-server/-/oauth2-mock-server-9.1.0.tgz", + "integrity": "sha512-1Aug6KQhD9IoxyCogFb0XQqovSOhkvOSRUz5Zm98o96H4omt1HEbUMxwzgwl7GO42eh+BqBH/uCbcKIxi4OBbg==", + "license": "MIT", + "dependencies": { + "basic-auth": "^2.0.1", + "is-plain-obj": "^4.1.0", + "jose": "^6.2.3" + }, + "bin": { + "oauth2-mock-server": "dist/oauth2-mock-server.mjs" + }, + "engines": { + "node": "^22.12 || ^24 || ^26" + } + }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + } + } +} diff --git a/robotics_integration_tests/custom_images/oauth2_mock/package.json b/robotics_integration_tests/custom_images/oauth2_mock/package.json new file mode 100644 index 0000000..2166eba --- /dev/null +++ b/robotics_integration_tests/custom_images/oauth2_mock/package.json @@ -0,0 +1,11 @@ +{ + "name": "oauth2-mock", + "version": "1.0.0", + "private": true, + "type": "module", + "description": "Local OpenID Connect issuer used by the robotics integration tests", + "main": "index.mjs", + "dependencies": { + "oauth2-mock-server": "9.1.0" + } +} diff --git a/robotics_integration_tests/settings/settings.py b/robotics_integration_tests/settings/settings.py index be97c7e..4b22a21 100644 --- a/robotics_integration_tests/settings/settings.py +++ b/robotics_integration_tests/settings/settings.py @@ -1,4 +1,4 @@ -from typing import Optional, List +from typing import List from dotenv import load_dotenv from pydantic import Field, computed_field @@ -6,34 +6,26 @@ class Settings(BaseSettings): - # Integration test app registration - INTEGRATION_TESTS_CLIENT_ID: str = Field( - default="17d7c036-e4ff-4df6-87fd-0d648a36a727" - ) - INTEGRATION_TESTS_TENANT_ID: str = Field( - default="3aa4a235-b6e2-48d5-9195-7fcf05b459b0" - ) - INTEGRATION_TESTS_CLIENT_SECRET: Optional[str] = Field(default="") - - # Keyvault configuration (using Flotilla service principle) - KEYVAULT_NAME: str = Field(default="FlotillaTestsKv") - - @computed_field - @property - def KEYVAULT_URI(self) -> str: - return f"https://{self.KEYVAULT_NAME.lower()}.vault.azure.net" + # Audiences minted by the local OAuth2 mock issuer and validated by each + # service. These are plain readable names rather than Entra app registration + # GUIDs: the mock derives the `aud` claim from the requested scope, so + # "isar-test/.default" yields aud="isar-test". + OAUTH_MOCK_ALIAS: str = Field(default="oauth-mock") + OAUTH_MOCK_PORT: int = Field(default=8080) + FLOTILLA_AUDIENCE: str = Field(default="flotilla-test") + ISAR_AUDIENCE: str = Field(default="isar-test") + SARA_AUDIENCE: str = Field(default="sara-test") # Flotilla Backend environment MQTT_HOST: str = Field(default="broker") - FLOTILLA_MQTT_PASSWORD: Optional[str] = Field(default="") - ASPNETCORE_ENVIRONMENT: str = Field(default="Development") - FLOTILLA_AZURE_CLIENT_SECRET: Optional[str] = Field(default="") - FLOTILLA_AZURE_CLIENT_ID: Optional[str] = Field( - default="ea4c7b92-47b3-45fb-bd25-a8070f0c495c" - ) - AZURE_TENANT_ID: Optional[str] = Field( - default="3aa4a235-b6e2-48d5-9195-7fcf05b459b0" - ) + # MQTT authentication is username/password, validated by the broker against + # the hashed passwd_file committed in equinor/flotilla. It is deliberately out + # of scope for the OAuth2 mock work, so these remain real secrets supplied via + # the environment or a local .env. + FLOTILLA_MQTT_PASSWORD: str = Field(default="") + # Selects appsettings.IntegrationTest.json in flotilla and sara, which points + # token validation at the mock issuer above and disables Key Vault and Redis. + ASPNETCORE_ENVIRONMENT: str = Field(default="IntegrationTest") FLOTILLA_BACKEND_NAME: str = Field(default="flotilla_backend") FLOTILLA_BACKEND_ALIAS: str = Field(default="flotilla_backend") FLOTILLA_BACKEND_IMAGE: str = Field( @@ -42,7 +34,8 @@ def KEYVAULT_URI(self) -> str: FLOTILLA_BACKEND_PORT: int = Field(default=8000) # MQTT Broker environment - FLOTILLA_BROKER_SERVER_KEY: Optional[str] = Field(default="") + # TLS private key for the test broker; see the note on FLOTILLA_MQTT_PASSWORD. + FLOTILLA_BROKER_SERVER_KEY: str = Field(default="") FLOTILLA_BROKER_NAME: str = Field(default="flotilla_broker") FLOTILLA_BROKER_ALIAS: str = Field(default="broker") FLOTILLA_BROKER_IMAGE: str = Field(default="ghcr.io/equinor/flotilla-broker:latest") @@ -58,6 +51,14 @@ def KEYVAULT_URI(self) -> str: GIT_REPOSITORY_FOR_MIGRATIONS_REF: str = Field(default="latest") BACKEND_PROJECT_FILE_FOLDER: str = Field(default="backend/api") + # Path to a local flotilla checkout to take migrations from. When set, it + # takes precedence over cloning GIT_REPOSITORY_FOR_MIGRATIONS from GitHub, and + # GIT_REPOSITORY_FOR_MIGRATIONS_REF is ignored. Use this together with locally + # built images (see scripts/build_local_images.sh) so that the schema and the + # application code come from the same source. Uncommitted and untracked + # migrations are included, since the directory is copied rather than cloned. + FLOTILLA_MIGRATIONS_SOURCE_DIR: str = Field(default="") + # PostgreSQL Sara Database environment POSTGRESQL_IMAGE: str = Field(default="postgres:16") SARA_DB_USER: str = Field(default="sara") @@ -69,20 +70,16 @@ def KEYVAULT_URI(self) -> str: SARA_BACKEND_PROJECT_FILE_FOLDER: str = Field(default="api") + # See FLOTILLA_MIGRATIONS_SOURCE_DIR. + SARA_MIGRATIONS_SOURCE_DIR: str = Field(default="") + # Migrations runner environment RELATIVE_PATH_TO_DOCKERFILE: str = Field( default="./robotics_integration_tests/custom_images/migrations_runner/" ) # ISAR Robot environment - ISAR_AZURE_CLIENT_SECRET: Optional[str] = Field(default="") - ISAR_MQTT_PASSWORD: Optional[str] = Field(default="") - ISAR_AZURE_CLIENT_ID: Optional[str] = Field( - default="fd384acd-5c1b-4c44-a1ac-d41d720ed0fe" - ) - ISAR_AZURE_TENANT_ID: Optional[str] = Field( - default="3aa4a235-b6e2-48d5-9195-7fcf05b459b0" - ) + ISAR_MQTT_PASSWORD: str = Field(default="") ISAR_ROBOT_NAME: str = Field(default="Placebot") ISAR_ROBOT_ALIAS: str = Field(default="isar_robot") ISAR_ROBOT_IMAGE: str = Field(default="ghcr.io/equinor/isar-robot:latest") @@ -92,10 +89,6 @@ def KEYVAULT_URI(self) -> str: SARA_RAW_STORAGE_CONTAINER: str = Field(default="sara-raw") SARA_ANON_STORAGE_CONTAINER: str = Field(default="sara-anon") SARA_VIS_STORAGE_CONTAINER: str = Field(default="sara-vis") - SARA_AZURE_CLIENT_SECRET: str = Field(default="") - - SARA_AZURE_CLIENT_ID: str = Field(default="dd7e115a-037e-4846-99c4-07561158a9cd") - SARA_AZURE_TENANT_ID: str = Field(default="3aa4a235-b6e2-48d5-9195-7fcf05b459b0") SARA_IMAGE: str = Field(default="ghcr.io/equinor/sara:latest") SARA_NAME: str = Field(default="sara") SARA_PORT: int = Field(default=8100) diff --git a/robotics_integration_tests/test_multiple_robots_parallel_missions.py b/robotics_integration_tests/test_multiple_robots_parallel_missions.py index 55e7e51..1bcd31a 100644 --- a/robotics_integration_tests/test_multiple_robots_parallel_missions.py +++ b/robotics_integration_tests/test_multiple_robots_parallel_missions.py @@ -4,6 +4,9 @@ from robotics_integration_tests.armada import Armada from robotics_integration_tests.custom_containers.isar import IsarRobot +from robotics_integration_tests.utilities.authentication_assertions import ( + assert_cannot_interfere_with_running_mission, +) from robotics_integration_tests.utilities.teams_notifications import ( wait_for_all_teams_notifications, ) @@ -82,6 +85,17 @@ def test_multiple_robots_with_different_outcomes( f"on robot {robot_name}" ) + # With four missions in flight, confirm an unauthorised caller can neither + # read through Flotilla nor stop a robot directly. Every token used is + # correctly signed by the issuer the services trust, so this exercises + # audience and role validation rather than the signature chain. + # + # The status assertions below then double as the proof that none of it had + # any effect: all four robots must still reach their expected outcomes. + assert_cannot_interfere_with_running_mission( + armada=armada, robot=armada.robots["MissionOkThenHome"] + ) + wait_for_all_mission_run_statuses( backend_url=backend_url, mission_run_expectations={ diff --git a/robotics_integration_tests/test_simple_mission_pausing_and_resuming_successful.py b/robotics_integration_tests/test_simple_mission_pausing_and_resuming_successful.py index b01a695..8aac5e7 100644 --- a/robotics_integration_tests/test_simple_mission_pausing_and_resuming_successful.py +++ b/robotics_integration_tests/test_simple_mission_pausing_and_resuming_successful.py @@ -4,6 +4,9 @@ from robotics_integration_tests.armada import Armada from robotics_integration_tests.settings.settings import settings +from robotics_integration_tests.utilities.authentication_assertions import ( + assert_cannot_pause_mission, +) from robotics_integration_tests.utilities.blob_storage import ( wait_until_all_expected_files_uploaded, ) @@ -12,6 +15,7 @@ resume_mission, create_mission, get_dummy_mission_payload_with_installation, + get_mission_run_by_id, schedule_mission, wait_for_mission_run_status, wait_for_robot_status, @@ -44,6 +48,22 @@ def test_simple_mission_pausing_and_resuming_successfully( expected_status="InProgress", ) + # An unauthorised caller must not be able to pause the mission, whether it + # goes through Flotilla or straight at the robot. + assert_cannot_pause_mission(armada=armada, robot=robot) + + # ... and the mission must still be running afterwards. Asserting "not + # Paused" rather than "is Ongoing" avoids racing normal mission progress; + # the property under test is that the rejected calls had no effect. + status_after_unauthorised_attempts: str = get_mission_run_by_id( + backend_url=armada.flotilla_backend.backend_url, + mission_run_id=mission_run_id, + ).get("status") + assert status_after_unauthorised_attempts != "Paused", ( + "An unauthorised caller managed to pause the mission run " + f"{mission_run_id}" + ) + pause_mission( backend_url=armada.flotilla_backend.backend_url, robot_id=robot.robot_id ) diff --git a/robotics_integration_tests/utilities/authentication.py b/robotics_integration_tests/utilities/authentication.py index 640f83e..6565812 100644 --- a/robotics_integration_tests/utilities/authentication.py +++ b/robotics_integration_tests/utilities/authentication.py @@ -1,20 +1,84 @@ -import msal +"""Access tokens for the integration tests. -from robotics_integration_tests.settings.settings import settings +Tokens come from the local OAuth2 mock issuer started by the ``oauth_mock`` +fixture, not from Azure Entra ID. The suite therefore needs no app registrations, +no tenant and no client secrets. + +The mock derives the ``aud`` claim from the requested scope, so asking for +``flotilla-test/.default`` yields a token with ``aud="flotilla-test"`` — which is +what each service is configured to validate against. +""" + +from typing import List, Optional + +import requests + +# Host-side base URL of the mock issuer, published on a random port by +# testcontainers. Set by the `oauth_mock` fixture during setup. +# +# A module-level global is used because the API helpers in +# `flotilla_backend_api.py` and `sara_backend_api.py` build their auth headers +# from plain module functions with no access to pytest fixtures. Each +# pytest-xdist worker is a separate process with its own mock container, so +# there is no cross-worker interference. +_issuer_url: Optional[str] = None + + +def configure_mock_issuer(host_url: str) -> None: + """Point token acquisition at a running mock issuer.""" + global _issuer_url + _issuer_url = host_url + + +def reset_mock_issuer() -> None: + """Forget the issuer so a stale URL cannot leak into a later test.""" + global _issuer_url + _issuer_url = None + + +def _require_issuer_url() -> str: + if _issuer_url is None: + raise RuntimeError( + "The OAuth2 mock issuer has not been configured. Depend on the " + "`oauth_mock` fixture, which calls configure_mock_issuer()." + ) + return _issuer_url def retrieve_access_token_for_integration_tests_app(resource_client_id: str) -> str: - app = msal.ConfidentialClientApplication( - client_id=settings.INTEGRATION_TESTS_CLIENT_ID, - client_credential=settings.INTEGRATION_TESTS_CLIENT_SECRET, - authority=f"https://login.microsoftonline.com/{settings.INTEGRATION_TESTS_TENANT_ID}", - ) - result = app.acquire_token_for_client( - scopes=[f"api://{resource_client_id}/.default"] + """Acquire a token for the given resource via the client credentials flow. + + The signature is unchanged from the previous MSAL implementation so that the + backend API helpers did not need to be touched. + """ + response = requests.post( + f"{_require_issuer_url()}/token", + data={ + "grant_type": "client_credentials", + "scope": f"{resource_client_id}/.default", + "client_id": "integration-tests", + }, + timeout=10, ) - if "access_token" in result: - return result["access_token"] - else: + response.raise_for_status() + result = response.json() + if "access_token" not in result: raise RuntimeError( f"Unable to retrieve access token for integration tests app: {result}" ) + return result["access_token"] + + +def issue_access_token(audience: str, roles: List[str]) -> str: + """Mint a token with an explicit audience and role set. + + Used by the negative authentication tests to prove that authentication is + genuinely enforced rather than accidentally bypassed. + """ + response = requests.post( + f"{_require_issuer_url()}/issue-token", + json={"audience": audience, "roles": roles}, + timeout=10, + ) + response.raise_for_status() + return response.json()["access_token"] diff --git a/robotics_integration_tests/utilities/authentication_assertions.py b/robotics_integration_tests/utilities/authentication_assertions.py new file mode 100644 index 0000000..90532e2 --- /dev/null +++ b/robotics_integration_tests/utilities/authentication_assertions.py @@ -0,0 +1,192 @@ +"""Assertions that authentication is genuinely enforced. + +Kept separate from ``utilities/authentication.py``, which stays concerned only +with minting tokens. + +These are deliberately woven into real mission flows rather than run as +standalone tests. Most negative authentication cases are already covered far more +thoroughly, and far more cheaply, by each service's own test suite: + +* flotilla enumerates *every* protected endpoint in + ``api.test/Security/AuthenticationControllerTests.cs`` +* sara does the same in ``api.Tests/Security/ProtectedEndpointsTests.cs`` +* isar asserts a rejected token in ``tests/isar/apis/security/test_authentication.py`` + +What none of those can cover is whether authentication is actually switched on in +the deployed configuration, and whether a token minted for one service is +accepted by another. Both need the whole stack running, so they belong here. +""" + +from http import HTTPStatus +from typing import Dict, Optional + +import requests +from loguru import logger + +from robotics_integration_tests.armada import Armada +from robotics_integration_tests.custom_containers.isar import IsarRobot +from robotics_integration_tests.settings.settings import settings +from robotics_integration_tests.utilities.authentication import issue_access_token + +_TIMEOUT = 30 + + +def isar_url(robot: IsarRobot, path: str) -> str: + """Host-reachable URL for one of an ISAR robot's endpoints.""" + port = robot.container.get_exposed_port(robot.port) + return f"http://localhost:{port}/{path.lstrip('/')}" + + +def _request(method: str, url: str, token: Optional[str] = None) -> requests.Response: + headers: Dict[str, str] = {"Authorization": f"Bearer {token}"} if token else {} + return requests.request(method, url, headers=headers, timeout=_TIMEOUT) + + +def assert_authentication_is_enforced(url: str, method: str = "GET") -> None: + """Fail unless an unauthenticated request is rejected. + + Called from the container fixtures so that it holds for every test. A stack + that has silently come up with authentication disabled would otherwise pass + the entire suite, since every other assertion is about mission behaviour. + """ + response = _request(method, url) + + if response.status_code not in ( + HTTPStatus.UNAUTHORIZED, + HTTPStatus.FORBIDDEN, + ): + raise AssertionError( + f"Authentication is NOT enforced: {method} {url} returned " + f"{response.status_code} without a token, expected 401 or 403.\n" + "The stack is running unsecured, which makes every other assertion " + "in this suite meaningless. Check that the service is running with " + f"ASPNETCORE_ENVIRONMENT={settings.ASPNETCORE_ENVIRONMENT} (or, for " + "ISAR, that ISAR_AUTHENTICATION_ENABLED has not been turned off)." + ) + + logger.debug(f"Authentication enforced on {method} {url}") + + +def assert_token_for_another_service_is_rejected( + method: str, url: str, audience: str, description: str +) -> None: + """A correctly signed token for a *different* service must be rejected. + + The token is valid in every respect except its audience, so this exercises + audience validation rather than signature checking. + """ + token: str = issue_access_token(audience=audience, roles=["Role.Admin"]) + response = _request(method, url, token=token) + + assert response.status_code == HTTPStatus.UNAUTHORIZED, ( + f"{description}: {method} {url} accepted a token issued for " + f"'{audience}' and returned {response.status_code}, expected 401. " + "Services must not accept tokens minted for a different audience." + ) + + +def assert_missing_role_is_rejected( + method: str, + url: str, + audience: str, + roles: list[str], + expected_status: HTTPStatus, + description: str, +) -> None: + """A token with the right audience but insufficient roles must be rejected.""" + token: str = issue_access_token(audience=audience, roles=roles) + response = _request(method, url, token=token) + + assert response.status_code == expected_status, ( + f"{description}: {method} {url} returned {response.status_code} for a " + f"token with roles {roles}, expected {int(expected_status)}." + ) + + +def assert_cannot_interfere_with_running_mission( + armada: Armada, robot: IsarRobot +) -> None: + """An unauthorised caller must not be able to disturb a running mission. + + Every token here is correctly signed by the same issuer the services trust, + so a failure means audience or role validation is broken rather than the + signature chain. + + The caller is expected to go on asserting that the mission reached its + normal outcome; that is what turns this from an endpoint probe into a + non-interference test. + """ + backend_url: str = armada.flotilla_backend.backend_url + stop_mission_url: str = isar_url(robot, "/schedule/stop-mission") + + # A SARA token must not open Flotilla. + assert_token_for_another_service_is_rejected( + method="GET", + url=f"{backend_url}/robots", + audience=settings.SARA_AUDIENCE, + description="Flotilla accepted a SARA token", + ) + + # Correct audience, but a role that grants no access to this installation. + assert_missing_role_is_rejected( + method="GET", + url=f"{backend_url}/robots", + audience=settings.FLOTILLA_AUDIENCE, + roles=["Role.User.HUA"], + expected_status=HTTPStatus.FORBIDDEN, + description="Flotilla granted access without a sufficient role", + ) + + # A Flotilla token must not let anyone stop the robot directly. + assert_token_for_another_service_is_rejected( + method="POST", + url=stop_mission_url, + audience=settings.FLOTILLA_AUDIENCE, + description="ISAR accepted a Flotilla token", + ) + + # Correct audience, but missing ISAR's REQUIRED_ROLE. ISAR answers 401 rather + # than 403 here: validate_has_role raises fastapi-azure-auth's + # InvalidAuthHttp, which subclasses UnauthorizedHttp. + assert_missing_role_is_rejected( + method="POST", + url=stop_mission_url, + audience=settings.ISAR_AUDIENCE, + roles=["Role.User.HUA"], + expected_status=HTTPStatus.UNAUTHORIZED, + description="ISAR accepted a token without Mission.Control", + ) + + logger.info("Unauthorised attempts to interfere with the mission were rejected") + + +def assert_cannot_pause_mission(armada: Armada, robot: IsarRobot) -> None: + """An unauthorised caller must not be able to pause a running mission.""" + backend_url: str = armada.flotilla_backend.backend_url + pause_url: str = f"{backend_url}/robots/{robot.robot_id}/pause" + + assert_token_for_another_service_is_rejected( + method="POST", + url=pause_url, + audience=settings.SARA_AUDIENCE, + description="Flotilla accepted a SARA token for pause", + ) + + assert_missing_role_is_rejected( + method="POST", + url=pause_url, + audience=settings.FLOTILLA_AUDIENCE, + roles=["Role.User.HUA"], + expected_status=HTTPStatus.FORBIDDEN, + description="Flotilla allowed pause without a sufficient role", + ) + + # Straight at the robot, bypassing Flotilla entirely. + assert_token_for_another_service_is_rejected( + method="POST", + url=isar_url(robot, "/schedule/pause-mission"), + audience=settings.FLOTILLA_AUDIENCE, + description="ISAR accepted a Flotilla token for pause", + ) + + logger.info("Unauthorised attempts to pause the mission were rejected") diff --git a/robotics_integration_tests/utilities/flotilla_backend_api.py b/robotics_integration_tests/utilities/flotilla_backend_api.py index 1bae67a..e95796e 100644 --- a/robotics_integration_tests/utilities/flotilla_backend_api.py +++ b/robotics_integration_tests/utilities/flotilla_backend_api.py @@ -14,7 +14,7 @@ def _add_headers() -> Dict[str, str]: access_token: str = retrieve_access_token_for_integration_tests_app( - settings.FLOTILLA_AZURE_CLIENT_ID + settings.FLOTILLA_AUDIENCE ) headers = {"Authorization": f"Bearer {access_token}"} return headers @@ -48,91 +48,52 @@ def set_current_inspection_area_for_robot( headers=_add_headers(), ) + def get_dummy_mission_payload_with_installation( installation_code: str, ) -> Dict: return { - "tasks": [ - { - "tagId": "57-2030DC", - "description": "Center of valve", - "robotPose": { - "position": { - "x": 82.03125, - "y": 289.04297, - "z": 29.402 - }, - "orientation": { - "x": 0, - "y": 1, - "z": 0, - "w": 1.5707964 - } - }, - "targetPosition": { - "x": 112.252, - "y": 284.027, - "z": 29.779 - }, - "analysisTypes": [], - "sensorType": "Image", - }, - { - "tagId": "13-2036PT", - "description": "STID Tag Position", - "robotPose": { - "position": { - "x": 94.74, - "y": 296.157, - "z": 29.401 - }, - "orientation": { - "x": 0, - "y": 1, - "z": 0, - "w": 3.1415927 - } - }, - "targetPosition": { - "x": 295.62, - "y": 95.589, - "z": 29.767 - }, - "analysisTypes": [], - "sensorType": "Image", - }, - { - "tagId": "65-2015OH", - "description": "STID Tag Position", - "robotPose": { - "position": { - "x": 103.048, - "y": 298.45, - "z": 29.401 - }, - "orientation": { - "x": 0, - "y": 1, - "z": 0, - "w": 1.5707964 - } - }, - "targetPosition": { - "x": 299.888, - "y": 103.245, - "z": 29.7 - }, - "analysisTypes": [], - "sensorType": "Image", + "tasks": [ + { + "tagId": "57-2030DC", + "description": "Center of valve", + "robotPose": { + "position": {"x": 82.03125, "y": 289.04297, "z": 29.402}, + "orientation": {"x": 0, "y": 1, "z": 0, "w": 1.5707964}, + }, + "targetPosition": {"x": 112.252, "y": 284.027, "z": 29.779}, + "analysisTypes": [], + "sensorType": "Image", + }, + { + "tagId": "13-2036PT", + "description": "STID Tag Position", + "robotPose": { + "position": {"x": 94.74, "y": 296.157, "z": 29.401}, + "orientation": {"x": 0, "y": 1, "z": 0, "w": 3.1415927}, + }, + "targetPosition": {"x": 295.62, "y": 95.589, "z": 29.767}, + "analysisTypes": [], + "sensorType": "Image", + }, + { + "tagId": "65-2015OH", + "description": "STID Tag Position", + "robotPose": { + "position": {"x": 103.048, "y": 298.45, "z": 29.401}, + "orientation": {"x": 0, "y": 1, "z": 0, "w": 1.5707964}, + }, + "targetPosition": {"x": 299.888, "y": 103.245, "z": 29.7}, + "analysisTypes": [], + "sensorType": "Image", + }, + ], + "name": "Three valves on Mezzanine Deck", + "installationCode": installation_code, } - ], - "name": "Three valves on Mezzanine Deck", - "installationCode": installation_code, -} -def create_mission( - backend_url: str, payload: Dict -) -> Dict: + +def create_mission(backend_url: str, payload: Dict) -> Dict: try: response: Dict = call_create_mission( backend_url=backend_url, @@ -143,9 +104,8 @@ def create_mission( logger.exception(f"Failed to create mission") raise e -def call_create_mission( - backend_url: str, payload: Dict -) -> Dict: + +def call_create_mission(backend_url: str, payload: Dict) -> Dict: url: str = f"{backend_url}/missions/definitions" response: Response = requests.post( url, @@ -168,9 +128,8 @@ def call_create_mission( response.raise_for_status() return response.json() -def schedule_mission( - backend_url: str, robot_id: str, mission_id: str -) -> Dict: + +def schedule_mission(backend_url: str, robot_id: str, mission_id: str) -> Dict: try: response: Dict = call_schedule_mission( backend_url=backend_url, @@ -183,9 +142,7 @@ def schedule_mission( raise e -def call_schedule_mission( - backend_url: str, robot_id: str, mission_id: str -) -> Dict: +def call_schedule_mission(backend_url: str, robot_id: str, mission_id: str) -> Dict: payload: Dict = { "robotId": robot_id, } diff --git a/robotics_integration_tests/utilities/keyvault.py b/robotics_integration_tests/utilities/keyvault.py deleted file mode 100644 index 8f670d1..0000000 --- a/robotics_integration_tests/utilities/keyvault.py +++ /dev/null @@ -1,137 +0,0 @@ -from typing import List, Union - -from azure.core.exceptions import ( - ClientAuthenticationError, - HttpResponseError, - ResourceNotFoundError, -) -from azure.identity import ClientSecretCredential, DefaultAzureCredential -from azure.keyvault.secrets import KeyVaultSecret, SecretClient -from loguru import logger - - -class Keyvault: - def __init__( - self, - keyvault_name: str, - client_id: str = None, - client_secret: str = None, - tenant_id: str = None, - ): - self.name = keyvault_name - self.url = "https://" + keyvault_name + ".vault.azure.net" - self.client_id = client_id - self.client_secret = client_secret - self.tenant_id = tenant_id - self.client: SecretClient = None - - def get_secret(self, secret_name: str) -> KeyVaultSecret: - secret_client: SecretClient = self.get_secret_client() - try: - secret: KeyVaultSecret = secret_client.get_secret(name=secret_name) - except ResourceNotFoundError: - logger.exception( - f"Secret {secret_name} was not found in keyvault {self.name}" - ) - raise - except HttpResponseError: - logger.error( - "An error occurred while retrieving the secret '%s' from keyvault '%s'.", - secret_name, - self.name, - exc_info=True, - ) - raise - - return secret - - def set_secret(self, secret_name: str, secret_value) -> None: - secret_client: SecretClient = self.get_secret_client() - try: - secret_client.set_secret(name=secret_name, value=secret_value) - logger.info(f"Secret {secret_name} was set in keyvault {self.name}") - except HttpResponseError: - logger.exception( - f"An error occurred while setting secret {secret_name} in keyvault {self.name}" - ) - raise - - def delete_secret(self, secret_name: str) -> None: - secret_client: SecretClient = self.get_secret_client() - try: - secret_client.begin_delete_secret(name=secret_name) - logger.info(f"Secret {secret_name} was deleted from keyvault {self.name}") - except ResourceNotFoundError: - logger.warning( - f"Secret {secret_name} not found in keyvault {self.name} during cleanup" - ) - except HttpResponseError: - logger.warning( - f"Failed to delete secret {secret_name} from keyvault {self.name}", - exc_info=True, - ) - - def get_secret_client(self) -> SecretClient: - if self.client is None: - try: - credential: Union[ClientSecretCredential, DefaultAzureCredential] - if self.client_id and self.client_secret and self.tenant_id: - credential = ClientSecretCredential( - tenant_id=self.tenant_id, - client_id=self.client_id, - client_secret=self.client_secret, - ) - else: - credential = DefaultAzureCredential() - except ClientAuthenticationError: - logger.error( - "Failed to authenticate to Azure while connecting to KeyVault", - exc_info=True, - ) - raise - - self.client = SecretClient(vault_url=self.url, credential=credential) - return self.client - - -class ScopedKeyvault(Keyvault): - """A keyvault wrapper that namespaces all secret names with a unique prefix. - - This allows multiple test instances to use the same Azure Key Vault - concurrently without overwriting each other's secrets. Secrets are - automatically cleaned up when ``cleanup`` is called. - """ - - def __init__( - self, - prefix: str, - keyvault_name: str, - client_id: str = None, - client_secret: str = None, - tenant_id: str = None, - ): - super().__init__( - keyvault_name=keyvault_name, - client_id=client_id, - client_secret=client_secret, - tenant_id=tenant_id, - ) - self.prefix = prefix - self._created_secrets: List[str] = [] - - def _scoped_name(self, secret_name: str) -> str: - return f"{self.prefix}-{secret_name}" - - def get_secret(self, secret_name: str) -> KeyVaultSecret: - return super().get_secret(self._scoped_name(secret_name)) - - def set_secret(self, secret_name: str, secret_value) -> None: - scoped = self._scoped_name(secret_name) - super().set_secret(secret_name=scoped, secret_value=secret_value) - self._created_secrets.append(scoped) - - def cleanup(self) -> None: - """Delete all secrets created by this scoped instance.""" - for secret_name in self._created_secrets: - self.delete_secret(secret_name) - self._created_secrets.clear() diff --git a/robotics_integration_tests/utilities/sara_backend_api.py b/robotics_integration_tests/utilities/sara_backend_api.py index e9386f6..d109427 100644 --- a/robotics_integration_tests/utilities/sara_backend_api.py +++ b/robotics_integration_tests/utilities/sara_backend_api.py @@ -17,7 +17,7 @@ def _add_headers() -> Dict[str, str]: access_token: str = retrieve_access_token_for_integration_tests_app( - settings.SARA_AZURE_CLIENT_ID + settings.SARA_AUDIENCE ) headers = {"Authorization": f"Bearer {access_token}"} return headers diff --git a/scripts/build_local_images.sh b/scripts/build_local_images.sh new file mode 100755 index 0000000..88a8f24 --- /dev/null +++ b/scripts/build_local_images.sh @@ -0,0 +1,230 @@ +#!/usr/bin/env bash +# +# Build the service images from local working copies and point the integration +# tests at them, instead of the published :dev / :latest images. +# +# Why this exists +# --------------- +# The integration tests normally pull ghcr.io/equinor/{flotilla-backend,sara, +# isar-robot}. That means a change which spans armada *and* one of the services +# cannot be validated until the service change has been merged and an image +# published -- but the armada side of the change is what proves the service side +# works. This script closes that gap: build everything locally, run the suite, +# then merge in confidence. +# +# Usage +# ----- +# scripts/build_local_images.sh # build and verify the images +# scripts/build_local_images.sh --run # ... and then run the full suite +# scripts/build_local_images.sh --help +# +# Repository locations default to the superrepo sibling layout and can each be +# overridden: ISAR_DIR ISAR_ROBOT_DIR FLOTILLA_DIR SARA_DIR + +set -euo pipefail + +TAG="${LOCAL_IMAGE_TAG:-local}" +PLATFORM="linux/amd64" +RUN_TESTS=false + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ARMADA_DIR="$(dirname "$SCRIPT_DIR")" +SIBLING_ROOT="$(dirname "$ARMADA_DIR")" + +ISAR_DIR="${ISAR_DIR:-$SIBLING_ROOT/isar}" +ISAR_ROBOT_DIR="${ISAR_ROBOT_DIR:-$SIBLING_ROOT/isar-robot}" +FLOTILLA_DIR="${FLOTILLA_DIR:-$SIBLING_ROOT/flotilla}" +SARA_DIR="${SARA_DIR:-$SIBLING_ROOT/sara}" + +FLOTILLA_IMAGE="flotilla-backend:$TAG" +SARA_IMAGE="sara:$TAG" +ISAR_ROBOT_IMAGE="isar-robot:$TAG" +ISAR_ROBOT_BASE_IMAGE="isar-robot:$TAG-base" + +for arg in "$@"; do + case "$arg" in + --run) RUN_TESTS=true ;; + --help|-h) + # Print the header comment block, stopping at the first non-comment line. + awk 'NR==1{next} /^#/{sub(/^# ?/,""); print; next} {exit}' "${BASH_SOURCE[0]}" + exit 0 ;; + *) echo "Unknown argument: $arg (try --help)" >&2; exit 2 ;; + esac +done + +log() { printf '\n\033[1;34m==> %s\033[0m\n' "$*"; } +warn() { printf '\033[1;33mWARNING: %s\033[0m\n' "$*" >&2; } +die() { printf '\033[1;31mERROR: %s\033[0m\n' "$*" >&2; exit 1; } + +require_dir() { + [ -d "$1" ] || die "$2 not found at '$1'. Set $3 to override." +} + +require_dir "$ISAR_DIR" "isar repository" ISAR_DIR +require_dir "$ISAR_ROBOT_DIR" "isar-robot repository" ISAR_ROBOT_DIR +require_dir "$FLOTILLA_DIR" "flotilla repository" FLOTILLA_DIR +require_dir "$SARA_DIR" "sara repository" SARA_DIR + +docker info >/dev/null 2>&1 || die "Docker does not appear to be running." + +TMP_DIR="$(mktemp -d)" +cleanup() { rm -rf "$TMP_DIR"; } +trap cleanup EXIT + +# --------------------------------------------------------------------------- +# flotilla-backend and sara are built straight from the working tree, so any +# uncommitted changes are included. +# --------------------------------------------------------------------------- + +log "Building $FLOTILLA_IMAGE from $FLOTILLA_DIR" +docker build --platform "$PLATFORM" \ + -f "$FLOTILLA_DIR/backend/Dockerfile" \ + -t "$FLOTILLA_IMAGE" \ + "$FLOTILLA_DIR/backend" + +log "Building $SARA_IMAGE from $SARA_DIR" +docker build --platform "$PLATFORM" -t "$SARA_IMAGE" "$SARA_DIR" + +# --------------------------------------------------------------------------- +# isar-robot needs two steps. +# +# 1. Its Dockerfile does `RUN --mount=source=.git,target=.git,type=bind`, and +# setuptools_scm needs that git directory both to derive a version *and* to +# discover package data such as src/isar_robot/example_data/. In the superrepo +# the checkout is a submodule, so `.git` is a FILE ("gitdir: ...") which the +# `!.git/` allowlist entry in .dockerignore does not match. Building directly +# from the working tree therefore fails with "unable to detect version", and +# forcing SETUPTOOLS_SCM_PRETEND_VERSION instead produces a wheel that is +# missing example_data -- which only shows up much later as +# RobotRetrieveInspectionException during a mission. Cloning into a temporary +# directory yields a real .git directory with history and tags, so the stock +# Dockerfile works unmodified. +# +# 2. isar-robot's uv.lock pins `isar` from PyPI (the lock is generated with +# --no-sources, so the `[tool.uv.sources] isar = { path = "../isar" }` entry in +# pyproject.toml is ignored). To test local isar changes, the locally built +# wheel is installed over the released one. +# --------------------------------------------------------------------------- + +if [ -n "$(git -C "$ISAR_ROBOT_DIR" status --porcelain)" ]; then + warn "$ISAR_ROBOT_DIR has uncommitted changes." + warn "isar-robot is CLONED rather than built from the working tree, so those" + warn "changes will NOT be in the image. Commit them first if they matter." +fi + +log "Cloning isar-robot into a temporary directory (needs a real .git)" +git clone --quiet "$ISAR_ROBOT_DIR" "$TMP_DIR/isar-robot" \ + || die "Failed to clone $ISAR_ROBOT_DIR" + +log "Building $ISAR_ROBOT_BASE_IMAGE" +docker build --platform "$PLATFORM" -t "$ISAR_ROBOT_BASE_IMAGE" "$TMP_DIR/isar-robot" + +log "Building the isar wheel from $ISAR_DIR (working tree, uncommitted changes included)" +mkdir -p "$TMP_DIR/wheels" +if ! ( cd "$ISAR_DIR" && uv build --wheel -o "$TMP_DIR/wheels" ) >"$TMP_DIR/uv-build.log" 2>&1; then + cat "$TMP_DIR/uv-build.log" >&2 + die "Failed to build the isar wheel" +fi +ISAR_WHEEL="$(ls "$TMP_DIR"/wheels/isar-*.whl 2>/dev/null | head -1)" +[ -n "$ISAR_WHEEL" ] || die "No isar wheel was produced in $TMP_DIR/wheels" +echo "Built $(basename "$ISAR_WHEEL")" + +log "Overlaying the local isar onto $ISAR_ROBOT_IMAGE" +# The wheel keeps its original filename: uv rejects anything that is not a valid +# PEP 427 wheel name ("Must have a Python tag"). +mkdir -p "$TMP_DIR/overlay/wheels" +cp "$ISAR_WHEEL" "$TMP_DIR/overlay/wheels/" +cat > "$TMP_DIR/overlay/Dockerfile" <