From e8d26cebe636d12ee80f53fe928e2f281148c1ce Mon Sep 17 00:00:00 2001 From: coling01 Date: Mon, 16 Feb 2026 15:10:59 +0000 Subject: [PATCH 01/11] chore: revert add of Build-Docker step because it needs $CERTS_DIR which is not set --- .github/workflows/ci-build-publish.yml | 36 -------------------------- 1 file changed, 36 deletions(-) diff --git a/.github/workflows/ci-build-publish.yml b/.github/workflows/ci-build-publish.yml index 242decd..5ec0557 100644 --- a/.github/workflows/ci-build-publish.yml +++ b/.github/workflows/ci-build-publish.yml @@ -128,42 +128,6 @@ jobs: -DAZURE_DEVOPS_ARTIFACT_USERNAME=$AZURE_DEVOPS_ARTIFACT_USERNAME \ -DAZURE_DEVOPS_ARTIFACT_TOKEN=$AZURE_DEVOPS_ARTIFACT_TOKEN - Build-Docker: - needs: [ Provider-Deploy, Build, Artefact-Version ] - if: ${{ inputs.trigger_docker }} - runs-on: ubuntu-latest - steps: - - name: Checkout Code - uses: actions/checkout@v6 - - - name: Download JAR Artefact - uses: actions/download-artifact@v7 - with: - name: app-jar - path: build/libs - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to GitHub Packages - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and Push Docker Image to GitHub - uses: docker/build-push-action@v6 - with: - context: . - file: Dockerfile - push: true - tags: | - ghcr.io/${{ github.repository }}:${{ needs.Artefact-Version.outputs.artefact_version }} - build-args: | - BASE_IMAGE=eclipse-temurin:21 - JAR_FILENAME=${{ needs.Build.outputs.artefact_name }}.jar - Deploy: needs: [ Provider-Deploy, Build, Artefact-Version ] if: ${{ inputs.trigger_deploy }} From dd1ab978633b69fbc410a13018e347de043d9062 Mon Sep 17 00:00:00 2001 From: ravi kakumanu Date: Thu, 26 Feb 2026 09:20:38 +0000 Subject: [PATCH 02/11] added environmnet variable for azurite secret --- .github/workflows/ci-build-publish.yml | 2 + .github/workflows/ci-draft.yml | 1 + build.gradle | 27 ++++ docker/docker-compose.integration.yml | 27 +++- .../http/IngestionProcessHttpLiveTest.java | 117 +++++------------- .../resources/wiremock-seed/azurite-seed.sh | 46 +++++++ .../resources/wiremock-seed/source.pdf | Bin 0 -> 252 bytes .../gov/hmcts/cp/cdk/batch/BatchConfig.java | 68 +--------- src/main/resources/application-cdk.yml | 4 - 9 files changed, 131 insertions(+), 161 deletions(-) create mode 100644 src/integrationTest/resources/wiremock-seed/azurite-seed.sh create mode 100644 src/integrationTest/resources/wiremock-seed/source.pdf diff --git a/.github/workflows/ci-build-publish.yml b/.github/workflows/ci-build-publish.yml index 5ec0557..8506727 100644 --- a/.github/workflows/ci-build-publish.yml +++ b/.github/workflows/ci-build-publish.yml @@ -9,6 +9,8 @@ on: required: true HMCTS_ADO_PAT: required: true + AZURE_STORAGE_CONNECTION_STRING: + required: true inputs: is_release: required: false diff --git a/.github/workflows/ci-draft.yml b/.github/workflows/ci-draft.yml index 84e0d6d..c83452d 100644 --- a/.github/workflows/ci-draft.yml +++ b/.github/workflows/ci-draft.yml @@ -17,6 +17,7 @@ jobs: AZURE_DEVOPS_ARTIFACT_USERNAME: ${{ secrets.AZURE_DEVOPS_ARTIFACT_USERNAME }} AZURE_DEVOPS_ARTIFACT_TOKEN: ${{ secrets.AZURE_DEVOPS_ARTIFACT_TOKEN }} HMCTS_ADO_PAT: ${{ secrets.HMCTS_CP_ADO_PAT }} + AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING}} with: is_publish: ${{ github.event_name == 'push' }} trigger_docker: ${{ github.event_name == 'push' }} diff --git a/build.gradle b/build.gradle index c7d47ed..209d119 100644 --- a/build.gradle +++ b/build.gradle @@ -291,6 +291,33 @@ tasks.named('jacocoTestReport') { reports { xml.required.set(true); csv.required.set(false); html.required.set(true) } } tasks.named('composeBuild') { dependsOn tasks.named('bootJar') } +dockerCompose { + useComposeFiles = ['docker/docker-compose.integration.yml'] + startedServices = ['artemis', 'db', 'azurite', 'azurite-seed', 'wiremock', 'app'] + buildBeforeUp = true + forceRecreate = true + removeOrphans = true + removeContainers = true + removeVolumes = true + waitForTcpPorts = true + upAdditionalArgs = ['--wait', '--wait-timeout', '180'] + + // Explicitly use project.file to avoid the "method not found" error + def envFile = project.file('docker/.env') + if (envFile.exists()) { + envFile.eachLine { line -> + def trimmedLine = line.trim() + if (trimmedLine && !trimmedLine.startsWith('#') && trimmedLine.contains('=')) { + def parts = trimmedLine.split('=', 2) + environment.put(parts[0].trim(), parts[1].trim()) + } + } + } + captureContainersOutput = true + projectName = "${rootProject.name}-it".replaceAll('[^A-Za-z0-9]', '') + useDockerComposeV2 = true + dockerExecutable = 'docker' +} tasks.register('integration', Test) { description = "Runs integration tests against docker-compose stack" diff --git a/docker/docker-compose.integration.yml b/docker/docker-compose.integration.yml index 85c5fd9..0696b61 100644 --- a/docker/docker-compose.integration.yml +++ b/docker/docker-compose.integration.yml @@ -7,6 +7,20 @@ services: - "10000:10000" restart: unless-stopped + azurite-seed: + container_name: cdks_azurite_seed + image: mcr.microsoft.com/azure-cli:2.63.0 + depends_on: + azurite: + condition: service_started + environment: + AZURE_STORAGE_CONNECTION_STRING: "${AZURE_STORAGE_CONNECTION_STRING}" + volumes: + - ../src/integrationTest/resources/wiremock-seed:/seed:ro + - ../src/integrationTest/resources/wiremock-seed/azurite-seed.sh:/azurite-seed.sh:ro + entrypoint: ["/bin/sh", "/azurite-seed.sh"] + restart: "no" + artemis: container_name: cdks_artemis build: @@ -69,8 +83,8 @@ services: condition: service_started wiremock: condition: service_started - azurite: - condition: service_started + azurite-seed: + condition: service_completed_successfully ports: - "8082:8082" - "5005:5005" @@ -143,11 +157,12 @@ services: OTEL_TRACES_URL: http://localhost:4318/traces OTEL_METRICS_URL: http://localhost:4318/metrics - CP_CDK_AZURE_STORAGE_CONNECTION_STRING: > - DefaultEndpointsProtocol=http;AccountName=devstoreaccount1; - AccountKey=Eby8vdM02xNOcqFeqCnf2w==; - BlobEndpoint=http://azurite:10000/devstoreaccount1; + CP_CDK_STORAGE_MODE: connection-string + CP_CDK_AZURE_STORAGE_CONNECTION_STRING: "${AZURE_STORAGE_CONNECTION_STRING}" CP_CDK_AZURE_STORAGE_CONTAINER: documents + CDK_STORAGE_AZURE_MODE: connection-string + CDK_STORAGE_AZURE_CONNECTION_STRING: "${AZURE_STORAGE_CONNECTION_STRING}" + CDK_STORAGE_AZURE_CONTAINER: documents CP_CDK_RAG_URL: http://wiremock:8080 CP_CDK_RAG_SUBSCRIPTION_KEY: dummy-key diff --git a/src/integrationTest/java/uk/gov/hmcts/cp/cdk/http/IngestionProcessHttpLiveTest.java b/src/integrationTest/java/uk/gov/hmcts/cp/cdk/http/IngestionProcessHttpLiveTest.java index f0f5b0f..cf76ecd 100644 --- a/src/integrationTest/java/uk/gov/hmcts/cp/cdk/http/IngestionProcessHttpLiveTest.java +++ b/src/integrationTest/java/uk/gov/hmcts/cp/cdk/http/IngestionProcessHttpLiveTest.java @@ -2,13 +2,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import uk.gov.hmcts.cp.cdk.domain.AnswerId; -import uk.gov.hmcts.cp.cdk.domain.CaseDocument; -import uk.gov.hmcts.cp.cdk.domain.Answer; - import uk.gov.hmcts.cp.cdk.testsupport.AbstractHttpLiveTest; import uk.gov.hmcts.cp.cdk.util.BrokerUtil; @@ -16,13 +11,15 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; -import java.sql.ResultSet; + import java.sql.SQLException; +import java.time.Duration; import java.time.OffsetDateTime; import java.util.List; import java.util.UUID; +import org.awaitility.Awaitility; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; @@ -216,22 +213,6 @@ void start_ingestion_process_returns_started_phase() throws Exception { assertNotNull(auditResponse, "Expected an audit event for ingestion process start"); } - private boolean isJobManagerEnabled() { - boolean enabled = false; - final String sysProp = System.getProperty("cdk.ingestion.feature.use-job-manager"); - - if (sysProp != null) { - enabled = Boolean.parseBoolean(sysProp); - } - - final String env = System.getenv("CP_CDK_FEATURE_USE_JOB_MANAGER"); - - if (env != null) { - enabled = Boolean.parseBoolean(env); - } - - return enabled; - } @Test @@ -280,75 +261,35 @@ void start_ingestion_process_executes_all_tasks_successfully() throws Exception } assertNotNull(auditResponse, "Expected audit event after full ingestion task chain"); - final boolean jmenable = isJobManagerEnabled(); + Thread.sleep(60_000); - if (jmenable) { - - // ---- CaseDocument table validation using JDBC - final UUID caseId = UUID.fromString("2204cd6b-5759-473c-b0f7-178b3aa0c9b3"); - final CaseDocument doc; - try (Connection c = DriverManager.getConnection(jdbcUrl, jdbcUser, jdbcPass); - PreparedStatement ps = c.prepareStatement( - "SELECT doc_id, case_id, material_id, doc_name, blob_uri, uploaded_at " + - "FROM case_documents " + - "WHERE case_id = ? " + - "ORDER BY uploaded_at DESC " + - "LIMIT 1" - )) { - ps.setObject(1, caseId); - try (ResultSet rs = ps.executeQuery()) { - assertTrue(rs.next(), "Expected at least one CaseDocument for the case"); - - doc = new CaseDocument(); - doc.setDocId((UUID) rs.getObject("doc_id")); - doc.setCaseId((UUID) rs.getObject("case_id")); - doc.setMaterialId((UUID) rs.getObject("material_id")); - doc.setDocName(rs.getString("doc_name")); - doc.setBlobUri(rs.getString("blob_uri")); - doc.setUploadedAt(rs.getObject("uploaded_at", OffsetDateTime.class)); - } - } + final UUID caseId = UUID.fromString("2204cd6b-5759-473c-b0f7-178b3aa0c9b3"); + final UUID queryId = UUID.fromString("2a9ae797-7f70-4be5-927f-2dae65489e69"); + + + final HttpHeaders answerHeaders = new HttpHeaders(); + answerHeaders.setAccept(List.of( + MediaType.valueOf("application/vnd.casedocumentknowledge-service.answers+json") + )); + + Awaitility.await() + .atMost(Duration.ofSeconds(120)) + .untilAsserted(() -> { + + final ResponseEntity answerResponse = http.exchange( + baseUrl + "/answers/" + caseId + "/" + queryId, + HttpMethod.GET, + new HttpEntity<>(answerHeaders), + String.class + ); + + assertThat(answerResponse.getStatusCode()).isEqualTo(HttpStatus.OK); + assertThat(answerResponse.getBody()).isNotNull(); + assertThat(answerResponse.getBody()).contains("\"answer\""); + assertThat(answerResponse.getBody()).contains("\"version\""); + }); - assertThat(doc.getBlobUri()).isNotBlank(); - assertThat(doc.getDocName()).isNotBlank(); - assertThat(doc.getMaterialId()).isNotNull(); - assertThat(doc.getUploadedAt()).isNotNull(); - - // ---- Answer table validation using JDBC - final UUID queryId = UUID.fromString("2a9ae797-7f70-4be5-927f-2dae65489e69"); - final Answer answer; - - try (Connection c = DriverManager.getConnection(jdbcUrl, jdbcUser, jdbcPass); - PreparedStatement ps = c.prepareStatement( - "SELECT case_id, query_id, version, created_at, answer, doc_id " + - "FROM answers " + - "WHERE case_id = ? AND query_id = ? " + - "ORDER BY created_at DESC, version DESC " + - "LIMIT 1" - )) { - ps.setObject(1, caseId); - ps.setObject(2, queryId); - - try (ResultSet rs = ps.executeQuery()) { - assertTrue(rs.next(), "Expected at least one Answer for the case and query"); - - answer = new Answer(); - final AnswerId answerId = new AnswerId(); - answerId.setCaseId((UUID) rs.getObject("case_id")); - answerId.setQueryId((UUID) rs.getObject("query_id")); - answerId.setVersion(rs.getInt("version")); - answer.setAnswerId(answerId); - - answer.setCreatedAt(rs.getObject("created_at", OffsetDateTime.class)); - answer.setAnswerText(rs.getString("answer")); - answer.setDocId((UUID) rs.getObject("doc_id")); - } - } - assertThat(answer.getAnswerText()).isNotBlank(); - assertThat(answer.getCreatedAt()).isNotNull(); - assertThat(answer.getDocId()).isNotNull(); - } } diff --git a/src/integrationTest/resources/wiremock-seed/azurite-seed.sh b/src/integrationTest/resources/wiremock-seed/azurite-seed.sh new file mode 100644 index 0000000..40131ac --- /dev/null +++ b/src/integrationTest/resources/wiremock-seed/azurite-seed.sh @@ -0,0 +1,46 @@ +#!/bin/sh +set -eu + +CS="${AZURE_STORAGE_CONNECTION_STRING}" + +echo "Waiting for Azurite..." +i=0 +until az storage container list --connection-string "$CS" >/dev/null 2>&1; do + i=$((i+1)) + [ "$i" -lt 60 ] || { echo "Azurite not ready"; exit 1; } + sleep 1 +done + +echo "Creating documents container..." +az storage container create \ + --name documents \ + --public-access blob \ + --connection-string "$CS" >/dev/null + +echo "Enforcing public access..." +az storage container set-permission \ + --name documents \ + --public-access blob \ + --connection-string "$CS" >/dev/null + +echo "Uploading source.pdf..." +[ -f /seed/source.pdf ] || { echo "Missing /seed/source.pdf"; ls -la /seed; exit 1; } + +az storage blob upload \ + --overwrite true \ + --container-name documents \ + --name source.pdf \ + --file /seed/source.pdf \ + --content-type application/pdf \ + --connection-string "$CS" >/dev/null + +echo "Verifying source blob exists..." +az storage blob show \ + --container-name documents \ + --name source.pdf \ + --connection-string "$CS" >/dev/null + +ACL="$(az storage container show --name documents --connection-string "$CS" --query properties.publicAccess -o tsv || true)" +echo "documents publicAccess=$ACL" + +echo "Seeded URL: http://azurite:10000/devstoreaccount1/documents/source.pdf" diff --git a/src/integrationTest/resources/wiremock-seed/source.pdf b/src/integrationTest/resources/wiremock-seed/source.pdf new file mode 100644 index 0000000000000000000000000000000000000000..663c04a1ea0ff8b22fb308dbf83ab31c5fe42899 GIT binary patch literal 252 zcmYL@%?iRW5QOi2ijuRn(xX?wqYCyO)^?$lHi_8{5=!6QO@HvRFvHC3H)VOQJbnTa zt8+ey6k|m0Y%5wl?$25sRyX+J0o*0K1Vdl+MIbp+_OSqprcsidQ(QL5Swr|AB?b;8 zx;jKeDYsxoSlRj)eB>l2&y!Hd$U|iD+mQGlD Q!O{zs-f52e#GK9956rGhs{jB1 literal 0 HcmV?d00001 diff --git a/src/main/java/uk/gov/hmcts/cp/cdk/batch/BatchConfig.java b/src/main/java/uk/gov/hmcts/cp/cdk/batch/BatchConfig.java index 5608c58..b6ef316 100644 --- a/src/main/java/uk/gov/hmcts/cp/cdk/batch/BatchConfig.java +++ b/src/main/java/uk/gov/hmcts/cp/cdk/batch/BatchConfig.java @@ -2,14 +2,13 @@ import static java.util.Objects.requireNonNull; +import uk.gov.hmcts.cp.cdk.config.VerifySchedulerProperties; import uk.gov.hmcts.cp.cdk.jobmanager.IngestionProperties; import uk.gov.hmcts.cp.cdk.storage.AzureBlobStorageService; import uk.gov.hmcts.cp.cdk.storage.StorageProperties; import uk.gov.hmcts.cp.cdk.storage.StorageService; import uk.gov.hmcts.cp.cdk.storage.UploadProperties; -import uk.gov.hmcts.cp.cdk.config.VerifySchedulerProperties; -import java.time.Duration; import java.util.Locale; import com.azure.core.credential.TokenCredential; @@ -22,9 +21,7 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -35,15 +32,12 @@ import org.springframework.retry.support.RetryTemplate; import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.utility.DockerImageName; /** * Core batch infrastructure configuration: * - RetryTemplate for tasklets * - TaskExecutors for partitioning * - Azure Blob Storage client wiring - * - Azurite for local / integration tests */ @Slf4j @Configuration @@ -57,14 +51,8 @@ }) public class BatchConfig { - private static final int AZURITE_PORT = 10_000; - private static final String DEFAULT_AZURITE_IMAGE = - "mcr.microsoft.com/azure-storage/azurite:3.33.0"; - private static final String DEV_ACCOUNT_NAME = "devstoreaccount1"; - private static final String DEV_ACCOUNT_KEY = "REDACTED"; private static final String CONNECTION_STRING_MODE = "connection-string"; private static final String MANAGED_IDENTITY_MODE = "managed-identity"; - private static final String AZURITE_MODE = "azurite"; private final IngestionProperties ingestionProperties; @@ -120,7 +108,6 @@ public TaskExecutor queryPartitionTaskExecutor(final PartitioningProperties part final ThreadPoolTaskExecutor threadPoolTaskExecutor = new ThreadPoolTaskExecutor(); threadPoolTaskExecutor.setThreadNamePrefix("query-partition-"); - // Slightly biased towards query grid size but still bounded by ingestion defaults. final int corePoolSize = Math.min( ingestionProperties.getCorePoolSize(), Math.max(1, partitioningProperties.queryGridSize()) @@ -148,30 +135,9 @@ public ObjectMapper objectMapper() { return objectMapper; } - @Bean(destroyMethod = "stop") - @ConditionalOnProperty(prefix = "cdk.storage.azure", name = "mode", havingValue = AZURITE_MODE) - @ConditionalOnMissingBean(name = "azuriteContainer") - public GenericContainer azuriteContainer(final StorageProperties storageProperties) { - final String imageName = storageProperties.azurite() != null - && StringUtils.isNotBlank(storageProperties.azurite().image()) - ? storageProperties.azurite().image() - : DEFAULT_AZURITE_IMAGE; - - final GenericContainer genericContainer = - new GenericContainer<>(DockerImageName.parse(imageName)) - .withExposedPorts(AZURITE_PORT) - .withCommand("azurite-blob --loose --blobHost 0.0.0.0 --blobPort " + AZURITE_PORT) - .withStartupTimeout(Duration.ofSeconds(60)); - - genericContainer.start(); - return genericContainer; - } - @Bean @ConditionalOnMissingBean - public BlobContainerClient blobContainerClient(final StorageProperties storageProperties, - @Autowired(required = false) final GenericContainer azuriteContainer) { - + public BlobContainerClient blobContainerClient(final StorageProperties storageProperties) { final String mode = StringUtils .defaultIfBlank(storageProperties.mode(), CONNECTION_STRING_MODE) .toLowerCase(Locale.ROOT); @@ -203,6 +169,7 @@ public BlobContainerClient blobContainerClient(final StorageProperties storagePr ); endpoint = "https://" + accountName + ".blob.core.windows.net"; } + final String userAssignedClientId = StringUtils.trimToNull(storageProperties.managedIdentityClientId()); @@ -219,33 +186,8 @@ public BlobContainerClient blobContainerClient(final StorageProperties storagePr createContainerIfMissing(blobContainerClient, containerName); yield blobContainerClient; } - case AZURITE_MODE -> { - if (azuriteContainer == null) { - throw new IllegalStateException( - "Azurite mode selected but azuriteContainer was not started" - ); - } - final String host = azuriteContainer.getHost(); - final int mappedPort = azuriteContainer.getMappedPort(AZURITE_PORT); - final String blobEndpoint = - "http://" + host + ":" + mappedPort + "/" + DEV_ACCOUNT_NAME; - - final String azuriteConnectionString = - "DefaultEndpointsProtocol=http;" - + "AccountName=" + DEV_ACCOUNT_NAME + ";" - + "AccountKey=" + DEV_ACCOUNT_KEY + ";" - + "BlobEndpoint=" + blobEndpoint + ";"; - - final BlobContainerClient blobContainerClient = new BlobContainerClientBuilder() - .connectionString(azuriteConnectionString) - .containerName(containerName) - .buildClient(); - createContainerIfMissing(blobContainerClient, containerName); - yield blobContainerClient; - } - default -> throw new IllegalArgumentException( - "Unsupported cdk.storage.azure.mode: " + mode - ); + default -> + throw new IllegalArgumentException("Unsupported cdk.storage.azure.mode: " + mode); }; } diff --git a/src/main/resources/application-cdk.yml b/src/main/resources/application-cdk.yml index 7b97621..36c415d 100644 --- a/src/main/resources/application-cdk.yml +++ b/src/main/resources/application-cdk.yml @@ -20,10 +20,6 @@ cdk: copy-poll-interval-ms: ${CP_CDK_AZURE_POLL_INTERVAL_MS:1000} copy-timeout-seconds: ${CP_CDK_AZURE_TIMEOUT_SECONDS:120} - # Azurite (local) – only used when mode=azurite - azurite: - image: ${CP_CDK_AZURITE_IMAGE:mcr.microsoft.com/azure-storage/azurite:3.33.0} - ingestion: core-pool-size: ${CP_CDK_INGESTION_CORE_POOL_SIZE:10} max-pool-size: ${CP_CDK_INGESTION_MAX_POOL_SIZE:20} From d6d150a6e3ec0cecfdcaa74d0539908b47b02906 Mon Sep 17 00:00:00 2001 From: ravi kakumanu Date: Thu, 26 Feb 2026 09:24:49 +0000 Subject: [PATCH 03/11] removed mandatory --- .github/workflows/ci-build-publish.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci-build-publish.yml b/.github/workflows/ci-build-publish.yml index 8506727..5ec0557 100644 --- a/.github/workflows/ci-build-publish.yml +++ b/.github/workflows/ci-build-publish.yml @@ -9,8 +9,6 @@ on: required: true HMCTS_ADO_PAT: required: true - AZURE_STORAGE_CONNECTION_STRING: - required: true inputs: is_release: required: false From 5cf50c1495f5462865250c5cea5d4061ca1bf0bf Mon Sep 17 00:00:00 2001 From: ravi kakumanu Date: Fri, 27 Feb 2026 11:48:56 +0000 Subject: [PATCH 04/11] removed secret --- .github/workflows/ci-draft.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci-draft.yml b/.github/workflows/ci-draft.yml index c83452d..84e0d6d 100644 --- a/.github/workflows/ci-draft.yml +++ b/.github/workflows/ci-draft.yml @@ -17,7 +17,6 @@ jobs: AZURE_DEVOPS_ARTIFACT_USERNAME: ${{ secrets.AZURE_DEVOPS_ARTIFACT_USERNAME }} AZURE_DEVOPS_ARTIFACT_TOKEN: ${{ secrets.AZURE_DEVOPS_ARTIFACT_TOKEN }} HMCTS_ADO_PAT: ${{ secrets.HMCTS_CP_ADO_PAT }} - AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING}} with: is_publish: ${{ github.event_name == 'push' }} trigger_docker: ${{ github.event_name == 'push' }} From 8e9bc9887bb00f59a11ef3b182af6faa71e812e3 Mon Sep 17 00:00:00 2001 From: coling01 Date: Fri, 27 Feb 2026 12:18:07 +0000 Subject: [PATCH 05/11] chore: throw error if AZURE_STORAGE_CONNECTION_STRING is not set for azurite-seed docker --- src/integrationTest/resources/wiremock-seed/azurite-seed.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/integrationTest/resources/wiremock-seed/azurite-seed.sh b/src/integrationTest/resources/wiremock-seed/azurite-seed.sh index 40131ac..f4c25c6 100644 --- a/src/integrationTest/resources/wiremock-seed/azurite-seed.sh +++ b/src/integrationTest/resources/wiremock-seed/azurite-seed.sh @@ -1,6 +1,10 @@ #!/bin/sh set -eu +if [ -z "${AZURE_STORAGE_CONNECTION_STRING}" ]; then + echo "Error - AZURE_STORAGE_CONNECTION_STRING must be set" + exit 1 +fi CS="${AZURE_STORAGE_CONNECTION_STRING}" echo "Waiting for Azurite..." From efac292d172e7f47c73187ba7d25fbee7696085a Mon Sep 17 00:00:00 2001 From: coling01 Date: Fri, 27 Feb 2026 13:50:50 +0000 Subject: [PATCH 06/11] chore: expose vars.AZURE_STORAGE_CONNECTION_STRING to environment for ci build --- .github/workflows/ci-build-publish.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-build-publish.yml b/.github/workflows/ci-build-publish.yml index 5ec0557..5820a29 100644 --- a/.github/workflows/ci-build-publish.yml +++ b/.github/workflows/ci-build-publish.yml @@ -2,6 +2,8 @@ name: CI Build and Publish on: workflow_call: + env: + AZURE_STORAGE_CONNECTION_STRING: ${{ vars.AZURE_STORAGE_CONNECTION_STRING }} secrets: AZURE_DEVOPS_ARTIFACT_USERNAME: required: true From 1e02c2f0e3cb96b9b0cb956308516db99f99f505 Mon Sep 17 00:00:00 2001 From: coling01 Date: Fri, 27 Feb 2026 14:03:56 +0000 Subject: [PATCH 07/11] chore: expose vars.AZURE_STORAGE_CONNECTION_STRING to environment for ci build --- .github/workflows/ci-build-publish.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-build-publish.yml b/.github/workflows/ci-build-publish.yml index 5820a29..ee5b427 100644 --- a/.github/workflows/ci-build-publish.yml +++ b/.github/workflows/ci-build-publish.yml @@ -70,6 +70,8 @@ jobs: env: ARTEFACT_VERSION: ${{ needs.Artefact-Version.outputs.artefact_version }} run: | + echo "COLING vars.AZURE_STORAGE_CONNECTION_STRING is set to \"${{ vars.AZURE_STORAGE_CONNECTION_STRING }}\"" + echo "COLING AZURE_STORAGE_CONNECTION_STRING is set to \"${AZURE_STORAGE_CONNECTION_STRING}\"" echo "Building with ARTEFACT_VERSION=$ARTEFACT_VERSION" ./gradlew build -DARTEFACT_VERSION=$ARTEFACT_VERSION From b417674f52860d08e72286db0baf6ed1c63802a3 Mon Sep 17 00:00:00 2001 From: coling01 Date: Fri, 27 Feb 2026 14:10:33 +0000 Subject: [PATCH 08/11] chore: expose vars.AZURE_STORAGE_CONNECTION_STRING to environment for ci build --- .github/workflows/codeql.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 4d6d563..1fd7739 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -85,6 +85,8 @@ jobs: DAST: runs-on: ubuntu-latest + env: + AZURE_STORAGE_CONNECTION_STRING: ${{ vars.AZURE_STORAGE_CONNECTION_STRING }} steps: - name: Checkout repository uses: actions/checkout@v6 @@ -104,6 +106,8 @@ jobs: - name: Pre-pull integration images run: | + echo "vars.AZURE_STORAGE_CONNECTION_STRING is set to \"${{ vars.AZURE_STORAGE_CONNECTION_STRING }}\"" + echo "AZURE_STORAGE_CONNECTION_STRING is set to \"${AZURE_STORAGE_CONNECTION_STRING}\"" docker pull postgres:16-alpine docker pull wiremock/wiremock:3.9.1 docker pull mcr.microsoft.com/azure-storage/azurite:3.33.0 From da3d151b626ad71f6e201eb8c46b27250a1419d4 Mon Sep 17 00:00:00 2001 From: coling01 Date: Fri, 27 Feb 2026 14:17:28 +0000 Subject: [PATCH 09/11] chore: expose vars.AZURE_STORAGE_CONNECTION_STRING to environment for ci build --- .github/workflows/ci-build-publish.yml | 2 -- .github/workflows/codeql.yml | 2 -- 2 files changed, 4 deletions(-) diff --git a/.github/workflows/ci-build-publish.yml b/.github/workflows/ci-build-publish.yml index ee5b427..5820a29 100644 --- a/.github/workflows/ci-build-publish.yml +++ b/.github/workflows/ci-build-publish.yml @@ -70,8 +70,6 @@ jobs: env: ARTEFACT_VERSION: ${{ needs.Artefact-Version.outputs.artefact_version }} run: | - echo "COLING vars.AZURE_STORAGE_CONNECTION_STRING is set to \"${{ vars.AZURE_STORAGE_CONNECTION_STRING }}\"" - echo "COLING AZURE_STORAGE_CONNECTION_STRING is set to \"${AZURE_STORAGE_CONNECTION_STRING}\"" echo "Building with ARTEFACT_VERSION=$ARTEFACT_VERSION" ./gradlew build -DARTEFACT_VERSION=$ARTEFACT_VERSION diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 1fd7739..a1e2ef1 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -106,8 +106,6 @@ jobs: - name: Pre-pull integration images run: | - echo "vars.AZURE_STORAGE_CONNECTION_STRING is set to \"${{ vars.AZURE_STORAGE_CONNECTION_STRING }}\"" - echo "AZURE_STORAGE_CONNECTION_STRING is set to \"${AZURE_STORAGE_CONNECTION_STRING}\"" docker pull postgres:16-alpine docker pull wiremock/wiremock:3.9.1 docker pull mcr.microsoft.com/azure-storage/azurite:3.33.0 From 51f6d435b8bc5fe06a250af92bbab4fe6c03b12e Mon Sep 17 00:00:00 2001 From: coling01 Date: Fri, 27 Feb 2026 15:47:38 +0000 Subject: [PATCH 10/11] chore: fix 2 pmd errors blocking pipeline --- .../CheckIngestionStatusForDocumentTask.java | 36 +++++++++---------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/src/main/java/uk/gov/hmcts/cp/cdk/jobmanager/caseflow/CheckIngestionStatusForDocumentTask.java b/src/main/java/uk/gov/hmcts/cp/cdk/jobmanager/caseflow/CheckIngestionStatusForDocumentTask.java index 03a6a1b..6cb5039 100644 --- a/src/main/java/uk/gov/hmcts/cp/cdk/jobmanager/caseflow/CheckIngestionStatusForDocumentTask.java +++ b/src/main/java/uk/gov/hmcts/cp/cdk/jobmanager/caseflow/CheckIngestionStatusForDocumentTask.java @@ -1,13 +1,12 @@ package uk.gov.hmcts.cp.cdk.jobmanager.caseflow; -import static uk.gov.hmcts.cp.cdk.jobmanager.TaskNames.CHECK_INGESTION_STATUS_FOR_DOCUMENT; -import static uk.gov.hmcts.cp.cdk.jobmanager.TaskNames.GENERATE_ANSWER_FOR_QUERY; -import static uk.gov.hmcts.cp.cdk.jobmanager.support.JobManagerKeys.CTX_SINGLE_QUERY_ID; -import static uk.gov.hmcts.cp.cdk.util.TimeUtils.utcNow; -import static uk.gov.hmcts.cp.openapi.model.DocumentIngestionStatus.INGESTION_FAILED; -import static uk.gov.hmcts.cp.openapi.model.DocumentIngestionStatus.INGESTION_SUCCESS; -import static uk.gov.hmcts.cp.openapi.model.DocumentIngestionStatus.INVALID_METADATA; - +import jakarta.json.Json; +import jakarta.json.JsonObject; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Component; import uk.gov.hmcts.cp.cdk.batch.support.QueryResolver; import uk.gov.hmcts.cp.cdk.domain.DocumentIngestionPhase; import uk.gov.hmcts.cp.cdk.domain.Query; @@ -30,21 +29,20 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; -import jakarta.json.Json; -import jakarta.json.JsonObject; -import lombok.RequiredArgsConstructor; -import org.jetbrains.annotations.NotNull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.http.ResponseEntity; -import org.springframework.stereotype.Component; +import static uk.gov.hmcts.cp.cdk.jobmanager.TaskNames.CHECK_INGESTION_STATUS_FOR_DOCUMENT; +import static uk.gov.hmcts.cp.cdk.jobmanager.TaskNames.GENERATE_ANSWER_FOR_QUERY; +import static uk.gov.hmcts.cp.cdk.jobmanager.support.JobManagerKeys.CTX_SINGLE_QUERY_ID; +import static uk.gov.hmcts.cp.cdk.util.TimeUtils.utcNow; +import static uk.gov.hmcts.cp.openapi.model.DocumentIngestionStatus.INGESTION_FAILED; +import static uk.gov.hmcts.cp.openapi.model.DocumentIngestionStatus.INGESTION_SUCCESS; +import static uk.gov.hmcts.cp.openapi.model.DocumentIngestionStatus.INVALID_METADATA; @Component @RequiredArgsConstructor @Task(CHECK_INGESTION_STATUS_FOR_DOCUMENT) +@Slf4j public class CheckIngestionStatusForDocumentTask implements ExecutableTask { - private static final Logger log = LoggerFactory.getLogger(CheckIngestionStatusForDocumentTask.class); private final DocumentIngestionStatusApi documentIngestionStatusApi; private final CaseDocumentRepository caseDocumentRepository; private final QueryResolver queryResolver; @@ -72,7 +70,7 @@ public ExecutionInfo execute(final ExecutionInfo executionInfo) { final UUID documentId = parseUuid(jobData.getString("docId", null)); final UUID caseId = parseUuid(jobData.getString("caseId", null)); final String blobName = jobData.getString("blobName", null); - final Set FAILURE_STATUSES = Set.of( + final Set failureStatuses = Set.of( INGESTION_FAILED.name(), INVALID_METADATA.name() ); @@ -143,7 +141,7 @@ public ExecutionInfo execute(final ExecutionInfo executionInfo) { } return complete(executionInfo); - } else if (FAILURE_STATUSES.contains(status.toUpperCase())) { + } else if (failureStatuses.contains(status.toUpperCase())) { updateIngestionPhase(documentId, DocumentIngestionPhase.FAILED); log.error( From 0dc732946256839206b917568f8286e35f0de1ed Mon Sep 17 00:00:00 2001 From: coling01 Date: Fri, 27 Feb 2026 15:54:58 +0000 Subject: [PATCH 11/11] chore: fix buyild pipeline to expose AZURE_STORAGE_CONNECTION_STRING as env variable --- .github/workflows/ci-build-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-build-publish.yml b/.github/workflows/ci-build-publish.yml index 5820a29..c40168a 100644 --- a/.github/workflows/ci-build-publish.yml +++ b/.github/workflows/ci-build-publish.yml @@ -2,8 +2,6 @@ name: CI Build and Publish on: workflow_call: - env: - AZURE_STORAGE_CONNECTION_STRING: ${{ vars.AZURE_STORAGE_CONNECTION_STRING }} secrets: AZURE_DEVOPS_ARTIFACT_USERNAME: required: true @@ -48,6 +46,8 @@ jobs: Build: needs: [Artefact-Version] runs-on: ubuntu-latest + env: + AZURE_STORAGE_CONNECTION_STRING: ${{ vars.AZURE_STORAGE_CONNECTION_STRING }} outputs: repo_name: ${{ steps.repo_vars.outputs.repo_name }} artefact_name: ${{ steps.repo_vars.outputs.artefact_name }}