diff --git a/.github/workflows/ci-build-publish.yml b/.github/workflows/ci-build-publish.yml index 5ec05573..18838ae3 100644 --- a/.github/workflows/ci-build-publish.yml +++ b/.github/workflows/ci-build-publish.yml @@ -9,6 +9,8 @@ on: required: true HMCTS_ADO_PAT: required: true + AZURE_STORAGE_CONNECTION_STRING: + required: true inputs: is_release: required: false diff --git a/.github/workflows/ci-draft.yml b/.github/workflows/ci-draft.yml index 84e0d6de..c83452d6 100644 --- a/.github/workflows/ci-draft.yml +++ b/.github/workflows/ci-draft.yml @@ -17,6 +17,7 @@ jobs: AZURE_DEVOPS_ARTIFACT_USERNAME: ${{ secrets.AZURE_DEVOPS_ARTIFACT_USERNAME }} AZURE_DEVOPS_ARTIFACT_TOKEN: ${{ secrets.AZURE_DEVOPS_ARTIFACT_TOKEN }} HMCTS_ADO_PAT: ${{ secrets.HMCTS_CP_ADO_PAT }} + AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING}} with: is_publish: ${{ github.event_name == 'push' }} trigger_docker: ${{ github.event_name == 'push' }} diff --git a/build.gradle b/build.gradle index 7cf71776..420da22d 100644 --- a/build.gradle +++ b/build.gradle @@ -306,6 +306,33 @@ tasks.named('jacocoTestReport') { reports { xml.required.set(true); csv.required.set(false); html.required.set(true) } } tasks.named('composeBuild') { dependsOn tasks.named('bootJar') } +dockerCompose { + useComposeFiles = ['docker/docker-compose.integration.yml'] + startedServices = ['artemis', 'db', 'azurite', 'azurite-seed', 'wiremock', 'app'] + buildBeforeUp = true + forceRecreate = true + removeOrphans = true + removeContainers = true + removeVolumes = true + waitForTcpPorts = true + upAdditionalArgs = ['--wait', '--wait-timeout', '180'] + + // Explicitly use project.file to avoid the "method not found" error + def envFile = project.file('docker/.env') + if (envFile.exists()) { + envFile.eachLine { line -> + def trimmedLine = line.trim() + if (trimmedLine && !trimmedLine.startsWith('#') && trimmedLine.contains('=')) { + def parts = trimmedLine.split('=', 2) + environment.put(parts[0].trim(), parts[1].trim()) + } + } + } + captureContainersOutput = true + projectName = "${rootProject.name}-it".replaceAll('[^A-Za-z0-9]', '') + useDockerComposeV2 = true + dockerExecutable = 'docker' +} tasks.register('integration', Test) { description = "Runs integration tests against docker-compose stack" diff --git a/docker/docker-compose.integration.yml b/docker/docker-compose.integration.yml index 85c5fd9d..e5749d1b 100644 --- a/docker/docker-compose.integration.yml +++ b/docker/docker-compose.integration.yml @@ -7,6 +7,20 @@ services: - "10000:10000" restart: unless-stopped + azurite-seed: + container_name: cdks_azurite_seed + image: mcr.microsoft.com/azure-cli:2.63.0 + depends_on: + azurite: + condition: service_started + environment: + AZURE_STORAGE_CONNECTION_STRING: "${CP_CDK_AZURE_STORAGE_CONNECTION_STRING}" + volumes: + - ../src/integrationTest/resources/wiremock-seed:/seed:ro + - ../src/integrationTest/resources/wiremock-seed/azurite-seed.sh:/azurite-seed.sh:ro + entrypoint: ["/bin/sh", "/azurite-seed.sh"] + restart: "no" + artemis: container_name: cdks_artemis build: @@ -69,8 +83,8 @@ services: condition: service_started wiremock: condition: service_started - azurite: - condition: service_started + azurite-seed: + condition: service_completed_successfully ports: - "8082:8082" - "5005:5005" @@ -143,11 +157,12 @@ services: OTEL_TRACES_URL: http://localhost:4318/traces OTEL_METRICS_URL: http://localhost:4318/metrics - CP_CDK_AZURE_STORAGE_CONNECTION_STRING: > - DefaultEndpointsProtocol=http;AccountName=devstoreaccount1; - AccountKey=Eby8vdM02xNOcqFeqCnf2w==; - BlobEndpoint=http://azurite:10000/devstoreaccount1; + CP_CDK_STORAGE_MODE: connection-string + CP_CDK_AZURE_STORAGE_CONNECTION_STRING: "${CP_CDK_AZURE_STORAGE_CONNECTION_STRING}" CP_CDK_AZURE_STORAGE_CONTAINER: documents + CDK_STORAGE_AZURE_MODE: connection-string + CDK_STORAGE_AZURE_CONNECTION_STRING: "${CP_CDK_AZURE_STORAGE_CONNECTION_STRING}" + CDK_STORAGE_AZURE_CONTAINER: documents CP_CDK_RAG_URL: http://wiremock:8080 CP_CDK_RAG_SUBSCRIPTION_KEY: dummy-key diff --git a/src/integrationTest/java/uk/gov/hmcts/cp/cdk/http/IngestionProcessHttpLiveTest.java b/src/integrationTest/java/uk/gov/hmcts/cp/cdk/http/IngestionProcessHttpLiveTest.java index bec08fc5..ab612b03 100644 --- a/src/integrationTest/java/uk/gov/hmcts/cp/cdk/http/IngestionProcessHttpLiveTest.java +++ b/src/integrationTest/java/uk/gov/hmcts/cp/cdk/http/IngestionProcessHttpLiveTest.java @@ -4,12 +4,9 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; - +import uk.gov.hmcts.cp.cdk.domain.Answer; import uk.gov.hmcts.cp.cdk.domain.AnswerId; import uk.gov.hmcts.cp.cdk.domain.CaseDocument; -import uk.gov.hmcts.cp.cdk.domain.Answer; - -import uk.gov.hmcts.cp.cdk.jobmanager.IngestionProperties; import uk.gov.hmcts.cp.cdk.testsupport.AbstractHttpLiveTest; import uk.gov.hmcts.cp.cdk.util.BrokerUtil; @@ -19,23 +16,21 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.time.Duration; import java.time.OffsetDateTime; import java.util.List; - import java.util.UUID; +import org.awaitility.Awaitility; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; -import org.springframework.test.context.TestPropertySource; /** @@ -282,75 +277,35 @@ void start_ingestion_process_executes_all_tasks_successfully() throws Exception } assertNotNull(auditResponse, "Expected audit event after full ingestion task chain"); - boolean jmenable = isJobManagerEnabled(); + Thread.sleep(60000); - if (jmenable) { - - // ---- CaseDocument table validation using JDBC - UUID caseId = UUID.fromString("2204cd6b-5759-473c-b0f7-178b3aa0c9b3"); - CaseDocument doc; - try (Connection c = DriverManager.getConnection(jdbcUrl, jdbcUser, jdbcPass); - PreparedStatement ps = c.prepareStatement( - "SELECT doc_id, case_id, material_id, doc_name, blob_uri, uploaded_at " + - "FROM case_documents " + - "WHERE case_id = ? " + - "ORDER BY uploaded_at DESC " + - "LIMIT 1" - )) { - ps.setObject(1, caseId); - try (ResultSet rs = ps.executeQuery()) { - assertTrue(rs.next(), "Expected at least one CaseDocument for the case"); - - doc = new CaseDocument(); - doc.setDocId((UUID) rs.getObject("doc_id")); - doc.setCaseId((UUID) rs.getObject("case_id")); - doc.setMaterialId((UUID) rs.getObject("material_id")); - doc.setDocName(rs.getString("doc_name")); - doc.setBlobUri(rs.getString("blob_uri")); - doc.setUploadedAt(rs.getObject("uploaded_at", OffsetDateTime.class)); - } - } + UUID caseId = UUID.fromString("2204cd6b-5759-473c-b0f7-178b3aa0c9b3"); + UUID queryId = UUID.fromString("2a9ae797-7f70-4be5-927f-2dae65489e69"); + + + final HttpHeaders answerHeaders = new HttpHeaders(); + answerHeaders.setAccept(List.of( + MediaType.valueOf("application/vnd.casedocumentknowledge-service.answers+json") + )); + + Awaitility.await() + .atMost(Duration.ofSeconds(120)) + .untilAsserted(() -> { + + ResponseEntity answerResponse = http.exchange( + baseUrl + "/answers/" + caseId + "/" + queryId, + HttpMethod.GET, + new HttpEntity<>(answerHeaders), + String.class + ); + + assertThat(answerResponse.getStatusCode()).isEqualTo(HttpStatus.OK); + assertThat(answerResponse.getBody()).isNotNull(); + assertThat(answerResponse.getBody()).contains("\"answer\""); + assertThat(answerResponse.getBody()).contains("\"version\""); + }); - assertThat(doc.getBlobUri()).isNotBlank(); - assertThat(doc.getDocName()).isNotBlank(); - assertThat(doc.getMaterialId()).isNotNull(); - assertThat(doc.getUploadedAt()).isNotNull(); - - // ---- Answer table validation using JDBC - UUID queryId = UUID.fromString("2a9ae797-7f70-4be5-927f-2dae65489e69"); - Answer answer; - - try (Connection c = DriverManager.getConnection(jdbcUrl, jdbcUser, jdbcPass); - PreparedStatement ps = c.prepareStatement( - "SELECT case_id, query_id, version, created_at, answer, doc_id " + - "FROM answers " + - "WHERE case_id = ? AND query_id = ? " + - "ORDER BY created_at DESC, version DESC " + - "LIMIT 1" - )) { - ps.setObject(1, caseId); - ps.setObject(2, queryId); - - try (ResultSet rs = ps.executeQuery()) { - assertTrue(rs.next(), "Expected at least one Answer for the case and query"); - - answer = new Answer(); - AnswerId answerId = new AnswerId(); - answerId.setCaseId((UUID) rs.getObject("case_id")); - answerId.setQueryId((UUID) rs.getObject("query_id")); - answerId.setVersion(rs.getInt("version")); - answer.setAnswerId(answerId); - - answer.setCreatedAt(rs.getObject("created_at", OffsetDateTime.class)); - answer.setAnswerText(rs.getString("answer")); - answer.setDocId((UUID) rs.getObject("doc_id")); - } - } - assertThat(answer.getAnswerText()).isNotBlank(); - assertThat(answer.getCreatedAt()).isNotNull(); - assertThat(answer.getDocId()).isNotNull(); - } } diff --git a/src/integrationTest/resources/wiremock-seed/azurite-seed.sh b/src/integrationTest/resources/wiremock-seed/azurite-seed.sh new file mode 100644 index 00000000..40131ace --- /dev/null +++ b/src/integrationTest/resources/wiremock-seed/azurite-seed.sh @@ -0,0 +1,46 @@ +#!/bin/sh +set -eu + +CS="${AZURE_STORAGE_CONNECTION_STRING}" + +echo "Waiting for Azurite..." +i=0 +until az storage container list --connection-string "$CS" >/dev/null 2>&1; do + i=$((i+1)) + [ "$i" -lt 60 ] || { echo "Azurite not ready"; exit 1; } + sleep 1 +done + +echo "Creating documents container..." +az storage container create \ + --name documents \ + --public-access blob \ + --connection-string "$CS" >/dev/null + +echo "Enforcing public access..." +az storage container set-permission \ + --name documents \ + --public-access blob \ + --connection-string "$CS" >/dev/null + +echo "Uploading source.pdf..." +[ -f /seed/source.pdf ] || { echo "Missing /seed/source.pdf"; ls -la /seed; exit 1; } + +az storage blob upload \ + --overwrite true \ + --container-name documents \ + --name source.pdf \ + --file /seed/source.pdf \ + --content-type application/pdf \ + --connection-string "$CS" >/dev/null + +echo "Verifying source blob exists..." +az storage blob show \ + --container-name documents \ + --name source.pdf \ + --connection-string "$CS" >/dev/null + +ACL="$(az storage container show --name documents --connection-string "$CS" --query properties.publicAccess -o tsv || true)" +echo "documents publicAccess=$ACL" + +echo "Seeded URL: http://azurite:10000/devstoreaccount1/documents/source.pdf" diff --git a/src/integrationTest/resources/wiremock-seed/source.pdf b/src/integrationTest/resources/wiremock-seed/source.pdf new file mode 100644 index 00000000..663c04a1 Binary files /dev/null and b/src/integrationTest/resources/wiremock-seed/source.pdf differ diff --git a/src/main/java/uk/gov/hmcts/cp/cdk/batch/BatchConfig.java b/src/main/java/uk/gov/hmcts/cp/cdk/batch/BatchConfig.java index 5608c581..b6ef3164 100644 --- a/src/main/java/uk/gov/hmcts/cp/cdk/batch/BatchConfig.java +++ b/src/main/java/uk/gov/hmcts/cp/cdk/batch/BatchConfig.java @@ -2,14 +2,13 @@ import static java.util.Objects.requireNonNull; +import uk.gov.hmcts.cp.cdk.config.VerifySchedulerProperties; import uk.gov.hmcts.cp.cdk.jobmanager.IngestionProperties; import uk.gov.hmcts.cp.cdk.storage.AzureBlobStorageService; import uk.gov.hmcts.cp.cdk.storage.StorageProperties; import uk.gov.hmcts.cp.cdk.storage.StorageService; import uk.gov.hmcts.cp.cdk.storage.UploadProperties; -import uk.gov.hmcts.cp.cdk.config.VerifySchedulerProperties; -import java.time.Duration; import java.util.Locale; import com.azure.core.credential.TokenCredential; @@ -22,9 +21,7 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -35,15 +32,12 @@ import org.springframework.retry.support.RetryTemplate; import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.utility.DockerImageName; /** * Core batch infrastructure configuration: * - RetryTemplate for tasklets * - TaskExecutors for partitioning * - Azure Blob Storage client wiring - * - Azurite for local / integration tests */ @Slf4j @Configuration @@ -57,14 +51,8 @@ }) public class BatchConfig { - private static final int AZURITE_PORT = 10_000; - private static final String DEFAULT_AZURITE_IMAGE = - "mcr.microsoft.com/azure-storage/azurite:3.33.0"; - private static final String DEV_ACCOUNT_NAME = "devstoreaccount1"; - private static final String DEV_ACCOUNT_KEY = "REDACTED"; private static final String CONNECTION_STRING_MODE = "connection-string"; private static final String MANAGED_IDENTITY_MODE = "managed-identity"; - private static final String AZURITE_MODE = "azurite"; private final IngestionProperties ingestionProperties; @@ -120,7 +108,6 @@ public TaskExecutor queryPartitionTaskExecutor(final PartitioningProperties part final ThreadPoolTaskExecutor threadPoolTaskExecutor = new ThreadPoolTaskExecutor(); threadPoolTaskExecutor.setThreadNamePrefix("query-partition-"); - // Slightly biased towards query grid size but still bounded by ingestion defaults. final int corePoolSize = Math.min( ingestionProperties.getCorePoolSize(), Math.max(1, partitioningProperties.queryGridSize()) @@ -148,30 +135,9 @@ public ObjectMapper objectMapper() { return objectMapper; } - @Bean(destroyMethod = "stop") - @ConditionalOnProperty(prefix = "cdk.storage.azure", name = "mode", havingValue = AZURITE_MODE) - @ConditionalOnMissingBean(name = "azuriteContainer") - public GenericContainer azuriteContainer(final StorageProperties storageProperties) { - final String imageName = storageProperties.azurite() != null - && StringUtils.isNotBlank(storageProperties.azurite().image()) - ? storageProperties.azurite().image() - : DEFAULT_AZURITE_IMAGE; - - final GenericContainer genericContainer = - new GenericContainer<>(DockerImageName.parse(imageName)) - .withExposedPorts(AZURITE_PORT) - .withCommand("azurite-blob --loose --blobHost 0.0.0.0 --blobPort " + AZURITE_PORT) - .withStartupTimeout(Duration.ofSeconds(60)); - - genericContainer.start(); - return genericContainer; - } - @Bean @ConditionalOnMissingBean - public BlobContainerClient blobContainerClient(final StorageProperties storageProperties, - @Autowired(required = false) final GenericContainer azuriteContainer) { - + public BlobContainerClient blobContainerClient(final StorageProperties storageProperties) { final String mode = StringUtils .defaultIfBlank(storageProperties.mode(), CONNECTION_STRING_MODE) .toLowerCase(Locale.ROOT); @@ -203,6 +169,7 @@ public BlobContainerClient blobContainerClient(final StorageProperties storagePr ); endpoint = "https://" + accountName + ".blob.core.windows.net"; } + final String userAssignedClientId = StringUtils.trimToNull(storageProperties.managedIdentityClientId()); @@ -219,33 +186,8 @@ public BlobContainerClient blobContainerClient(final StorageProperties storagePr createContainerIfMissing(blobContainerClient, containerName); yield blobContainerClient; } - case AZURITE_MODE -> { - if (azuriteContainer == null) { - throw new IllegalStateException( - "Azurite mode selected but azuriteContainer was not started" - ); - } - final String host = azuriteContainer.getHost(); - final int mappedPort = azuriteContainer.getMappedPort(AZURITE_PORT); - final String blobEndpoint = - "http://" + host + ":" + mappedPort + "/" + DEV_ACCOUNT_NAME; - - final String azuriteConnectionString = - "DefaultEndpointsProtocol=http;" - + "AccountName=" + DEV_ACCOUNT_NAME + ";" - + "AccountKey=" + DEV_ACCOUNT_KEY + ";" - + "BlobEndpoint=" + blobEndpoint + ";"; - - final BlobContainerClient blobContainerClient = new BlobContainerClientBuilder() - .connectionString(azuriteConnectionString) - .containerName(containerName) - .buildClient(); - createContainerIfMissing(blobContainerClient, containerName); - yield blobContainerClient; - } - default -> throw new IllegalArgumentException( - "Unsupported cdk.storage.azure.mode: " + mode - ); + default -> + throw new IllegalArgumentException("Unsupported cdk.storage.azure.mode: " + mode); }; } diff --git a/src/main/resources/application-cdk.yml b/src/main/resources/application-cdk.yml index 7b97621d..36c415d4 100644 --- a/src/main/resources/application-cdk.yml +++ b/src/main/resources/application-cdk.yml @@ -20,10 +20,6 @@ cdk: copy-poll-interval-ms: ${CP_CDK_AZURE_POLL_INTERVAL_MS:1000} copy-timeout-seconds: ${CP_CDK_AZURE_TIMEOUT_SECONDS:120} - # Azurite (local) – only used when mode=azurite - azurite: - image: ${CP_CDK_AZURITE_IMAGE:mcr.microsoft.com/azure-storage/azurite:3.33.0} - ingestion: core-pool-size: ${CP_CDK_INGESTION_CORE_POOL_SIZE:10} max-pool-size: ${CP_CDK_INGESTION_MAX_POOL_SIZE:20}