azurite seed config with cicd variable

Author: ravi kakumanu

.github/workflows/ci-draft.yml

@@ -17,6 +17,7 @@ jobs:
       AZURE_DEVOPS_ARTIFACT_USERNAME: ${{ secrets.AZURE_DEVOPS_ARTIFACT_USERNAME }}
       AZURE_DEVOPS_ARTIFACT_TOKEN: ${{ secrets.AZURE_DEVOPS_ARTIFACT_TOKEN }}
       HMCTS_ADO_PAT: ${{ secrets.HMCTS_CP_ADO_PAT }}
+      AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING}}
     with:
       is_publish: ${{ github.event_name == 'push' }}
       trigger_docker: ${{ github.event_name == 'push' }}

build.gradle

@@ -306,6 +306,33 @@ tasks.named('jacocoTestReport') {
     reports { xml.required.set(true); csv.required.set(false); html.required.set(true) }
 }
 tasks.named('composeBuild') { dependsOn tasks.named('bootJar') }
+dockerCompose {
+    useComposeFiles = ['docker/docker-compose.integration.yml']
+    startedServices = ['artemis', 'db', 'azurite', 'azurite-seed', 'wiremock', 'app']
+    buildBeforeUp = true
+    forceRecreate = true
+    removeOrphans = true
+    removeContainers = true
+    removeVolumes = true
+    waitForTcpPorts = true
+    upAdditionalArgs = ['--wait', '--wait-timeout', '180']
+
+    // Explicitly use project.file to avoid the "method not found" error
+    def envFile = project.file('docker/.env')
+    if (envFile.exists()) {
+        envFile.eachLine { line ->
+            def trimmedLine = line.trim()
+            if (trimmedLine && !trimmedLine.startsWith('#') && trimmedLine.contains('=')) {
+                def parts = trimmedLine.split('=', 2)
+                environment.put(parts[0].trim(), parts[1].trim())
+            }
+        }
+    }
+    captureContainersOutput = true
+    projectName = "${rootProject.name}-it".replaceAll('[^A-Za-z0-9]', '')
+    useDockerComposeV2 = true
+    dockerExecutable = 'docker'
+}
 
 tasks.register('integration', Test) {
     description = "Runs integration tests against docker-compose stack"

docker/docker-compose.integration.yml

@@ -7,6 +7,20 @@ services:
       - "10000:10000"
     restart: unless-stopped
 
+  azurite-seed:
+    container_name: cdks_azurite_seed
+    image: mcr.microsoft.com/azure-cli:2.63.0
+    depends_on:
+      azurite:
+        condition: service_started
+    environment:
+      AZURE_STORAGE_CONNECTION_STRING: "${AZURE_STORAGE_CONNECTION_STRING}"
+    volumes:
+      - ../src/integrationTest/resources/wiremock-seed:/seed:ro
+      - ../src/integrationTest/resources/wiremock-seed/azurite-seed.sh:/azurite-seed.sh:ro
+    entrypoint: ["/bin/sh", "/azurite-seed.sh"]
+    restart: "no"
+    
   artemis:
     container_name: cdks_artemis
     build:
@@ -69,8 +83,8 @@ services:
         condition: service_started
       wiremock:
         condition: service_started
-      azurite:
-        condition: service_started
+      azurite-seed:
+        condition: service_completed_successfully
     ports:
       - "8082:8082"
       - "5005:5005"
@@ -143,11 +157,12 @@ services:
       OTEL_TRACES_URL: http://localhost:4318/traces
       OTEL_METRICS_URL: http://localhost:4318/metrics
 
-      CP_CDK_AZURE_STORAGE_CONNECTION_STRING: >
-        DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;
-        AccountKey=Eby8vdM02xNOcqFeqCnf2w==;
-        BlobEndpoint=http://azurite:10000/devstoreaccount1;
+      CP_CDK_STORAGE_MODE: connection-string
+      CP_CDK_AZURE_STORAGE_CONNECTION_STRING: "${AZURE_STORAGE_CONNECTION_STRING}"
       CP_CDK_AZURE_STORAGE_CONTAINER: documents
+      CDK_STORAGE_AZURE_MODE: connection-string
+      CDK_STORAGE_AZURE_CONNECTION_STRING: "${AZURE_STORAGE_CONNECTION_STRING}"
+      CDK_STORAGE_AZURE_CONTAINER: documents
 
       CP_CDK_RAG_URL: http://wiremock:8080
       CP_CDK_RAG_SUBSCRIPTION_KEY: dummy-key

src/integrationTest/java/uk/gov/hmcts/cp/cdk/http/IngestionProcessHttpLiveTest.java

@@ -4,12 +4,9 @@
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
-
+import uk.gov.hmcts.cp.cdk.domain.Answer;
 import uk.gov.hmcts.cp.cdk.domain.AnswerId;
 import uk.gov.hmcts.cp.cdk.domain.CaseDocument;
-import uk.gov.hmcts.cp.cdk.domain.Answer;
-
-import uk.gov.hmcts.cp.cdk.jobmanager.IngestionProperties;
 import uk.gov.hmcts.cp.cdk.testsupport.AbstractHttpLiveTest;
 import uk.gov.hmcts.cp.cdk.util.BrokerUtil;
 
@@ -19,23 +16,21 @@
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.time.Duration;
 import java.time.OffsetDateTime;
 import java.util.List;
-
 import java.util.UUID;
 
+import org.awaitility.Awaitility;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInstance;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.http.HttpEntity;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpMethod;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;
-import org.springframework.test.context.TestPropertySource;
 
 
 /**
@@ -282,75 +277,35 @@ void start_ingestion_process_executes_all_tasks_successfully() throws Exception
         }
 
         assertNotNull(auditResponse, "Expected audit event after full ingestion task chain");
-        boolean jmenable = isJobManagerEnabled();
+
         Thread.sleep(60000);
-        if (jmenable) {
-
-            // ---- CaseDocument table validation using JDBC
-            UUID caseId = UUID.fromString("2204cd6b-5759-473c-b0f7-178b3aa0c9b3");
-            CaseDocument doc;
-            try (Connection c = DriverManager.getConnection(jdbcUrl, jdbcUser, jdbcPass);
-                 PreparedStatement ps = c.prepareStatement(
-                         "SELECT doc_id, case_id, material_id, doc_name, blob_uri, uploaded_at " +
-                                 "FROM case_documents " +
-                                 "WHERE case_id = ? " +
-                                 "ORDER BY uploaded_at DESC " +
-                                 "LIMIT 1"
-                 )) {
-                ps.setObject(1, caseId);
-                try (ResultSet rs = ps.executeQuery()) {
-                    assertTrue(rs.next(), "Expected at least one CaseDocument for the case");
-
-                    doc = new CaseDocument();
-                    doc.setDocId((UUID) rs.getObject("doc_id"));
-                    doc.setCaseId((UUID) rs.getObject("case_id"));
-                    doc.setMaterialId((UUID) rs.getObject("material_id"));
-                    doc.setDocName(rs.getString("doc_name"));
-                    doc.setBlobUri(rs.getString("blob_uri"));
-                    doc.setUploadedAt(rs.getObject("uploaded_at", OffsetDateTime.class));
-                }
-            }
+        UUID caseId = UUID.fromString("2204cd6b-5759-473c-b0f7-178b3aa0c9b3");
+        UUID queryId = UUID.fromString("2a9ae797-7f70-4be5-927f-2dae65489e69");
+
+
+        final HttpHeaders answerHeaders = new HttpHeaders();
+        answerHeaders.setAccept(List.of(
+                MediaType.valueOf("application/vnd.casedocumentknowledge-service.answers+json")
+        ));
+
+        Awaitility.await()
+                .atMost(Duration.ofSeconds(120))
+                .untilAsserted(() -> {
+
+                    ResponseEntity<String> answerResponse = http.exchange(
+                            baseUrl + "/answers/" + caseId + "/" + queryId,
+                            HttpMethod.GET,
+                            new HttpEntity<>(answerHeaders),
+                            String.class
+                    );
+
+                    assertThat(answerResponse.getStatusCode()).isEqualTo(HttpStatus.OK);
+                    assertThat(answerResponse.getBody()).isNotNull();
+                    assertThat(answerResponse.getBody()).contains("\"answer\"");
+                    assertThat(answerResponse.getBody()).contains("\"version\"");
+                });
 
-            assertThat(doc.getBlobUri()).isNotBlank();
-            assertThat(doc.getDocName()).isNotBlank();
-            assertThat(doc.getMaterialId()).isNotNull();
-            assertThat(doc.getUploadedAt()).isNotNull();
-
-            // ---- Answer table validation using JDBC
-            UUID queryId = UUID.fromString("2a9ae797-7f70-4be5-927f-2dae65489e69");
-            Answer answer;
-
-            try (Connection c = DriverManager.getConnection(jdbcUrl, jdbcUser, jdbcPass);
-                 PreparedStatement ps = c.prepareStatement(
-                         "SELECT case_id, query_id, version, created_at, answer, doc_id " +
-                                 "FROM answers " +
-                                 "WHERE case_id = ? AND query_id = ? " +
-                                 "ORDER BY created_at DESC, version DESC " +
-                                 "LIMIT 1"
-                 )) {
-                ps.setObject(1, caseId);
-                ps.setObject(2, queryId);
-
-                try (ResultSet rs = ps.executeQuery()) {
-                    assertTrue(rs.next(), "Expected at least one Answer for the case and query");
-
-                    answer = new Answer();
-                    AnswerId answerId = new AnswerId();
-                    answerId.setCaseId((UUID) rs.getObject("case_id"));
-                    answerId.setQueryId((UUID) rs.getObject("query_id"));
-                    answerId.setVersion(rs.getInt("version"));
-                    answer.setAnswerId(answerId);
-
-                    answer.setCreatedAt(rs.getObject("created_at", OffsetDateTime.class));
-                    answer.setAnswerText(rs.getString("answer"));
-                    answer.setDocId((UUID) rs.getObject("doc_id"));
-                }
-            }
 
-            assertThat(answer.getAnswerText()).isNotBlank();
-            assertThat(answer.getCreatedAt()).isNotNull();
-            assertThat(answer.getDocId()).isNotNull();
-        }
     }
 
 

src/integrationTest/resources/wiremock-seed/azurite-seed.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+set -eu
+
+CS="${AZURE_STORAGE_CONNECTION_STRING}"
+
+echo "Waiting for Azurite..."
+i=0
+until az storage container list --connection-string "$CS" >/dev/null 2>&1; do
+  i=$((i+1))
+  [ "$i" -lt 60 ] || { echo "Azurite not ready"; exit 1; }
+  sleep 1
+done
+
+echo "Creating documents container..."
+az storage container create \
+  --name documents \
+  --public-access blob \
+  --connection-string "$CS" >/dev/null
+
+echo "Enforcing public access..."
+az storage container set-permission \
+  --name documents \
+  --public-access blob \
+  --connection-string "$CS" >/dev/null
+
+echo "Uploading source.pdf..."
+[ -f /seed/source.pdf ] || { echo "Missing /seed/source.pdf"; ls -la /seed; exit 1; }
+
+az storage blob upload \
+  --overwrite true \
+  --container-name documents \
+  --name source.pdf \
+  --file /seed/source.pdf \
+  --content-type application/pdf \
+  --connection-string "$CS" >/dev/null
+
+echo "Verifying source blob exists..."
+az storage blob show \
+  --container-name documents \
+  --name source.pdf \
+  --connection-string "$CS" >/dev/null
+
+ACL="$(az storage container show --name documents --connection-string "$CS" --query properties.publicAccess -o tsv || true)"
+echo "documents publicAccess=$ACL"
+
+echo "Seeded URL: http://azurite:10000/devstoreaccount1/documents/source.pdf"