diff --git a/.github/workflows/run-functional-tests.yaml b/.github/workflows/run-functional-tests.yaml index 557202254..6ab2ecc8d 100644 --- a/.github/workflows/run-functional-tests.yaml +++ b/.github/workflows/run-functional-tests.yaml @@ -6,6 +6,7 @@ on: - "build.gradle" - "settings.gradle" - "gradle/**" + - "docker-compose.yaml" jobs: pipeline: diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index c5cb29020..dc8a47eba 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -12,6 +12,7 @@ on: - "build.gradle" - "settings.gradle" - "gradle/**" + - "docker-compose.yaml" jobs: pipeline: diff --git a/.github/workflows/run-unit-tests.yaml b/.github/workflows/run-unit-tests.yaml index b20ccd596..b4782271d 100644 --- a/.github/workflows/run-unit-tests.yaml +++ b/.github/workflows/run-unit-tests.yaml @@ -6,6 +6,7 @@ on: - "build.gradle" - "settings.gradle" - "gradle/**" + - "docker-compose.yaml" jobs: pipeline: diff --git a/docker-compose.yaml b/docker-compose.yaml index 9ddfed316..57ebae6d6 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -133,103 +133,6 @@ services: timeout: 5s retries: 5 - investigation-service: - build: - dockerfile: ./investigation-service/Dockerfile - environment: - - DB_USERNAME=investigation_service_rdb - - DB_PASSWORD=investigation_service - - DB_HOST=jdbc:sqlserver://nbs-mssql:1433;databaseName=RDB_MODERN;encrypt=true;trustServerCertificate=true; - - KAFKA_BOOTSTRAP_SERVER=kafka:29092 - depends_on: - kafka: - condition: service_healthy - nbs-mssql: - condition: service_healthy - liquibase: - condition: service_completed_successfully - - ldfdata-service: - build: - dockerfile: ./ldfdata-service/Dockerfile - environment: - - DB_USERNAME=ldf_service_rdb - - DB_PASSWORD=ldf_service - - DB_HOST=jdbc:sqlserver://nbs-mssql:1433;databaseName=RDB_MODERN;encrypt=true;trustServerCertificate=true; - - KAFKA_BOOTSTRAP_SERVER=kafka:29092 - depends_on: - kafka: - condition: service_healthy - nbs-mssql: - condition: service_healthy - liquibase: - condition: service_completed_successfully - - observation-service: - build: - dockerfile: ./observation-service/Dockerfile - environment: - - DB_USERNAME=observation_service_rdb - - DB_PASSWORD=observation_service - - DB_HOST=jdbc:sqlserver://nbs-mssql:1433;databaseName=RDB_MODERN;encrypt=true;trustServerCertificate=true; - - KAFKA_BOOTSTRAP_SERVER=kafka:29092 - depends_on: - kafka: - condition: service_healthy - nbs-mssql: - condition: service_healthy - liquibase: - condition: service_completed_successfully - - organization-service: - build: - dockerfile: ./organization-service/Dockerfile - environment: - - DB_USERNAME=organization_service_rdb - - DB_PASSWORD=organization_service - - DB_HOST=jdbc:sqlserver://nbs-mssql:1433;databaseName=RDB_MODERN;encrypt=true;trustServerCertificate=true; - - KAFKA_BOOTSTRAP_SERVER=kafka:29092 - depends_on: - kafka: - condition: service_healthy - nbs-mssql: - condition: service_healthy - liquibase: - condition: service_completed_successfully - - person-service: - build: - dockerfile: ./person-service/Dockerfile - environment: - - DB_USERNAME=person_service_rdb - - DB_PASSWORD=person_service - - DB_HOST=jdbc:sqlserver://nbs-mssql:1433;databaseName=RDB_MODERN;encrypt=true;trustServerCertificate=true; - - KAFKA_BOOTSTRAP_SERVER=kafka:29092 - - spring.kafka.consumer.maxPollIntervalMs=3000 - depends_on: - kafka: - condition: service_healthy - nbs-mssql: - condition: service_healthy - liquibase: - condition: service_completed_successfully - - post-processing-service: - build: - dockerfile: ./post-processing-service/Dockerfile - environment: - - DB_USERNAME=post_processing_service_rdb - - DB_PASSWORD=post_processing_service - - DB_HOST=jdbc:sqlserver://nbs-mssql:1433;databaseName=RDB_MODERN;encrypt=true;trustServerCertificate=true; - - KAFKA_BOOTSTRAP_SERVER=kafka:29092 - depends_on: - kafka: - condition: service_healthy - nbs-mssql: - condition: service_healthy - liquibase: - condition: service_completed_successfully - reporting-pipeline-service: build: dockerfile: ./reporting-pipeline-service/Dockerfile diff --git a/documentation/DevSetup.md b/documentation/DevSetup.md index 69c02e906..3087246cb 100644 --- a/documentation/DevSetup.md +++ b/documentation/DevSetup.md @@ -4,15 +4,15 @@ ## Docker Containers -1. [mssql](../containers/db/Dockerfile) - Restored MSSQL Server database pre-configured to work with RTR. Notable configurations +1. [mssql](https://github.com/cdcent/NEDSSDB/pkgs/container/nedssdb) - Restored MSSQL Server database pre-configured to work with RTR. Notable configurations 1. NBS_ODSE.NBS_Configuration entry added with `config_key = 'ENV'`, `config_value = 'UAT'` - 1. RTR User creation scripts applied - 1. Change Data Capture (CDC) enabled for relevant databases and tables -1. [liquibase](../liquibase-service/Dockerfile.local) - Liquibase container with all migration scripts copied from [liquibase-service/src/main/resources/db/](../liquibase-service/src/main/resources/db/). Configured to automatically apply migrations and 1 time onboarding scripts and then close. Entrypoint: [migrate.sh](../containers/liquibase/migrate.sh) -1. [kafka](../docker-compose.yaml) - Message broker -1. [kafka-connect](../containers/kafka-connect/Dockerfile) - Reads from the `nrt_*` topics and inserts into `rdb_modern` tables. Requires POST of [mssql-connector.json](../containers/kafka-connect/mssql-connector.json) after container start up. -1. [debezium](../docker-compose.yaml) - Reads Change Data Capture logs and posts messages to Kafka. Requires POST for each connector to be sent after container start up. -1. [reporting-pipeline-service](../reporting-pipeline-service/Dockerfile) - Process Kafka messages for investigation, ldf, observation, organization, and person data (also handles key-uid mappings) + 2. RTR User creation scripts applied + 3. Change Data Capture (CDC) enabled for relevant databases and tables +2. [liquibase](../liquibase-service/Dockerfile.local) - Liquibase container with all migration scripts copied from `../liquibase-service/src/main/resources/db/`. Configured to automatically apply migrations and 1 time onboarding scripts and then close. Entrypoint: [migrate.sh](../containers/liquibase/migrate.sh) +3. [kafka](../docker-compose.yaml) - Message broker +4. [kafka-connect](../containers/kafka-connect/Dockerfile) - Reads from the `nrt_*` topics and inserts into `rdb_modern` tables. Requires POST of [mssql-connector.json](../containers/kafka-connect/initialize/mssql-connector.json) after container start up. +5. [debezium](../docker-compose.yaml) - Reads Change Data Capture logs and posts messages to Kafka. Requires POST for each connector to be sent after container start up. +6. [reporting-pipeline-service](../reporting-pipeline-service/Dockerfile) - Process Kafka messages for investigation, ldf, observation, organization, and person data (also handles key-uid mappings) ### Prerequisites: