Release/12.5 > main (#325)

* Implement idempotency check for spotlight submissions in SpotlightBatchService

* Enhance SpotlightBatchService to handle multiple QUEUED batches and prevent duplicate submissions. Updated query to order results by creation date and added logging for better traceability.

* Update SpotlightBatchServiceTest to return lists instead of Optionals for batch retrieval methods, enhancing compatibility with multiple QUEUED batches.

* Refactor SpotlightBatchServiceTest to prevent re-queuing of GGIS_ERROR submissions, ensuring they require manual intervention.

Author: rmohammed-goaco

src/main/java/gov/cabinetoffice/gap/adminbackend/repositories/SpotlightBatchRepository.java

@@ -28,8 +28,8 @@ Optional<SpotlightBatch> findByStatusAndSpotlightSubmissionsSizeLessThan(
 
     Optional<List<SpotlightBatch>> findByStatus(@Param("status") SpotlightBatchStatus status);
 
-    @Query("select s from SpotlightBatch s inner join s.spotlightSubmissions spotlightSubmissions where s.status =:status and spotlightSubmissions.mandatoryQuestions.gapId = :gapId")
-    Optional<SpotlightBatch> findByStatusAndSpotlightSubmissions_MandatoryQuestions_GapId(
+    @Query("select s from SpotlightBatch s inner join s.spotlightSubmissions spotlightSubmissions where s.status =:status and spotlightSubmissions.mandatoryQuestions.gapId = :gapId ORDER BY s.created DESC")
+    List<SpotlightBatch> findByStatusAndSpotlightSubmissions_MandatoryQuestions_GapId(
             @Param("status") SpotlightBatchStatus status, @Param("gapId") String gapId);
 
 }

src/main/java/gov/cabinetoffice/gap/adminbackend/services/SpotlightBatchService.java

@@ -44,9 +44,12 @@
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
 
 import static gov.cabinetoffice.gap.adminbackend.enums.DraftAssessmentResponseDtoStatus.SUCCESS;
 
@@ -136,11 +139,21 @@ private SpotlightBatch getSpotlightBatch(UUID spotlightBatchId) {
     }
 
     public SpotlightBatch getSpotlightBatchWithQueuedStatusByMandatoryQuestionGapId(String gapId) {
-        return spotlightBatchRepository
-                .findByStatusAndSpotlightSubmissions_MandatoryQuestions_GapId(SpotlightBatchStatus.QUEUED, gapId)
-                .orElseThrow(() -> new NotFoundException(
-                        "A spotlight batch with spotlightSubmission for mandatory question with gap id " + gapId
-                                + " could not be found"));
+        List<SpotlightBatch> batches = spotlightBatchRepository
+                .findByStatusAndSpotlightSubmissions_MandatoryQuestions_GapId(SpotlightBatchStatus.QUEUED, gapId);
+        
+        if (batches.isEmpty()) {
+            throw new NotFoundException(
+                    "A spotlight batch with spotlightSubmission for mandatory question with gap id " + gapId
+                            + " could not be found");
+        }
+        
+        if (batches.size() > 1) {
+            log.warn("Found {} QUEUED batches for gapId {}. Using the most recent one.", batches.size(), gapId);
+        }
+        
+        // Return the first batch (already ordered by created DESC in the query)
+        return batches.get(0);
     }
 
     public List<SpotlightBatch> getSpotlightBatchesByStatus(SpotlightBatchStatus status) {
@@ -152,10 +165,39 @@ public List<SendToSpotlightDto> generateSendToSpotlightDtosList(SpotlightBatchSt
         final List<SendToSpotlightDto> sendToSpotlightDtos = new ArrayList<>();
         final List<SpotlightBatch> spotlightBatches = getSpotlightBatchesByStatus(status);
 
+        // Track submissions we've already included to prevent duplicates across batches
+        final Set<UUID> processedSubmissionIds = new HashSet<>();
+
         for (SpotlightBatch spotlightBatch : spotlightBatches) {
             try {
+                final List<SpotlightSubmission> submissions = spotlightBatch.getSpotlightSubmissions();
+                
+                // Filter: Only non-SENT submissions that haven't been processed yet
+                final List<SpotlightSubmission> submissionsToProcess = submissions.stream()
+                        .filter(sub -> !SpotlightSubmissionStatus.SENT.toString().equals(sub.getStatus()))
+                        .filter(sub -> !processedSubmissionIds.contains(sub.getId()))
+                        .collect(Collectors.toList());
+                
+                if (submissionsToProcess.isEmpty()) {
+                    log.info("Batch {} skipped - no eligible submissions (all are SENT or already processed)", 
+                            spotlightBatch.getId());
+                    // Mark batch as SUCCESS since all submissions are already SENT
+                    if (submissions.stream().allMatch(s -> SpotlightSubmissionStatus.SENT.toString().equals(s.getStatus()))) {
+                        spotlightBatch.setStatus(SpotlightBatchStatus.SUCCESS);
+                        spotlightBatch.setLastUpdated(Instant.now());
+                        spotlightBatchRepository.save(spotlightBatch);
+                    }
+                    continue;
+                }
+                
+                // Mark these submissions as processed to prevent duplicates in other batches
+                submissionsToProcess.forEach(sub -> processedSubmissionIds.add(sub.getId()));
+                
+                log.info("Processing batch {} with {} submissions (filtered from {})", 
+                        spotlightBatch.getId(), submissionsToProcess.size(), submissions.size());
+
                 final List<SpotlightSchemeDto> schemes = new ArrayList<>();
-                addSpotlightSchemeDtoToList(spotlightBatch, schemes);
+                addSpotlightSchemeDtoToListFiltered(spotlightBatch, schemes, submissionsToProcess);
 
                 final SendToSpotlightDto sendToSpotlightDto = SendToSpotlightDto.builder().schemes(schemes).build();
                 sendToSpotlightDtos.add(sendToSpotlightDto);
@@ -169,6 +211,21 @@ public List<SendToSpotlightDto> generateSendToSpotlightDtosList(SpotlightBatchSt
         return sendToSpotlightDtos;
     }
 
+    /**
+     * Adds spotlight scheme DTOs to the list using a pre-filtered list of submissions.
+     * This prevents duplicate submissions being sent to Spotlight.
+     */
+    private void addSpotlightSchemeDtoToListFiltered(SpotlightBatch spotlightBatch, 
+            List<SpotlightSchemeDto> schemes, List<SpotlightSubmission> submissionsToProcess) {
+        
+        final List<String> uniqueSchemeIds = getUniqueSchemeIds(submissionsToProcess);
+        log.info("uniqueSchemeIds for batch {}: {}", spotlightBatch.getId(), uniqueSchemeIds);
+
+        uniqueSchemeIds.stream()
+                .map(uniqueSchemeId -> generateSchemeDto(uniqueSchemeId, submissionsToProcess))
+                .forEach(schemes::add);
+    }
+
     public List<String> getUniqueSchemeIds(List<SpotlightSubmission> spotlightSubmissions) {
         return spotlightSubmissions.stream()
                 .map(submission -> submission.getMandatoryQuestions().getSchemeEntity().getGgisIdentifier()).distinct()
@@ -294,16 +351,21 @@ private void handleError(SpotlightSubmission spotlightSubmission,
     private void handleErrorMessage(SpotlightSubmission spotlightSubmission, String errorMessage) {
         if (errorMessage.contains(RESPONSE_MESSAGE_406_SCHEME_NOT_EXIST)) {
             spotlightSubmission.setStatus(SpotlightSubmissionStatus.GGIS_ERROR.toString());
-            sendMessageToQueue(spotlightSubmission);
+            // DON'T re-queue GGIS errors - they require manual intervention to fix the scheme ID
+            // Re-queuing these creates an infinite loop of failures
+            log.warn("GGIS_ERROR for submission {} - scheme doesn't exist in Spotlight. " +
+                    "Manual intervention required to fix the GGIS identifier.", spotlightSubmission.getId());
         }
         else if (errorMessage.contains(RESPONSE_MESSAGE_409_FIELD_MISSING)
                 || errorMessage.contains(RESPONSE_MESSAGE_409_LENGTH)) {
-            // has a validation error
+            // has a validation error - DON'T re-queue as these need manual data fixes
             spotlightSubmission.setStatus(SpotlightSubmissionStatus.VALIDATION_ERROR.toString());
 
             log.info("Sending Spotlight validation support email using SNS for status code: 409");
             final String snsResponse = snsService.spotlightValidationError();
             log.info(snsResponse);
+            log.warn("VALIDATION_ERROR for submission {} - {}. Manual intervention required.", 
+                    spotlightSubmission.getId(), errorMessage);
         }
     }
 
@@ -327,8 +389,11 @@ public SpotlightResponseResultsDto sendBatchToSpotlight(SendToSpotlightDto spotl
 
         final HttpEntity<String> requestEntity = new HttpEntity<>(spotlightBatchAsJsonString, requestHeaders);
 
-        final String draftAssessmentsEndpoint = spotlightConfig.getSpotlightUrl()
-                + "/services/apexrest/DraftAssessments";
+        // Construct the endpoint URL - avoid duplicating the path if already in config
+        String baseUrl = spotlightConfig.getSpotlightUrl();
+        final String draftAssessmentsEndpoint = baseUrl.contains("/services/apexrest/DraftAssessments")
+                ? baseUrl
+                : baseUrl + "/services/apexrest/DraftAssessments";
 
         SpotlightResponseResultsDto list = SpotlightResponseResultsDto.builder().build();
 

src/test/java/gov/cabinetoffice/gap/adminbackend/services/SpotlightBatchServiceTest.java

@@ -520,7 +520,7 @@ void getSpotlightBatchByMandatoryQuestionGapId_success() {
             final SpotlightBatch spotlightBatch = SpotlightBatch.builder().id(uuid).build();
 
             when(spotlightBatchRepository.findByStatusAndSpotlightSubmissions_MandatoryQuestions_GapId(any(), any()))
-                    .thenReturn(Optional.of(spotlightBatch));
+                    .thenReturn(List.of(spotlightBatch));
 
             final SpotlightBatch result = spotlightBatchService
                     .getSpotlightBatchWithQueuedStatusByMandatoryQuestionGapId("GAP123");
@@ -531,7 +531,7 @@ void getSpotlightBatchByMandatoryQuestionGapId_success() {
         @Test
         void getSpotlightBatchByMandatoryQuestionGapId_notFound() {
             when(spotlightBatchRepository.findByStatusAndSpotlightSubmissions_MandatoryQuestions_GapId(any(), any()))
-                    .thenReturn(Optional.empty());
+                    .thenReturn(List.of());
 
             assertThrows(NotFoundException.class,
                     () -> spotlightBatchService.getSpotlightBatchWithQueuedStatusByMandatoryQuestionGapId("GAP123"));
@@ -722,7 +722,6 @@ void spotlightResponsesResultIsNotNullAndDraftAssessmentStatusIsFailureAndMessag
 
             doNothing().when(spotlightBatchService).updateSpotlightBatchStatus(sendToSpotlightDto,
                     SpotlightBatchStatus.FAILURE);
-            doNothing().when(spotlightBatchService).sendMessageToQueue(spotlightSubmission);
 
             spotlightBatchService.processSpotlightResponse(sendToSpotlightDto, spotlightResponseResults);
 
@@ -733,7 +732,8 @@ void spotlightResponsesResultIsNotNullAndDraftAssessmentStatusIsFailureAndMessag
 
             verify(spotlightBatchService, times(1)).updateSpotlightBatchStatus(sendToSpotlightDto,
                     SpotlightBatchStatus.FAILURE);
-            verify(spotlightBatchService, times(1)).sendMessageToQueue(spotlightSubmission);
+            // GGIS_ERROR submissions should NOT be re-queued - they require manual intervention
+            verify(spotlightBatchService, never()).sendMessageToQueue(any());
         }
 
         @Test