Release/13.3 (#335)

* Release/13.2 (#332)

* Update GrantMandatoryQuestionFundingLocation enum to replace 'OUTSIDE_UK' with 'INTERNATIONAL'

* Update GrantMandatoryQuestionFundingLocation enum to replace 'INTERNATIONAL' with 'OUTSIDE_UK'

* Add submission anonymisation feature

- Introduced SubmissionAnonymisationConfigProperties for configuration settings.
- Added EXPIRED status to SubmissionStatus enum.
- Enhanced GrantAttachmentRepository and GrantMandatoryQuestionRepository with delete methods for submissions.
- Updated SubmissionRepository with methods for anonymising submissions and deleting related data.
- Implemented SubmissionAnonymisationScheduler to handle scheduled anonymisation of submissions.
- Created SubmissionAnonymisationService to manage the anonymisation process, including S3 object deletion and database cleanup.
- Added application properties for submission anonymisation configuration.
- Created database migration to document the new EXPIRED status in the submission table.

* Refactor SubmissionAnonymisationConfigProperties

- Removed Lombok annotations: @builder, @AllArgsConstructor, and @NoArgsConstructor.
- Simplified the class by retaining only @Getter and @Setter annotations.
- Adjusted the default value for daysBeforeExpiry to 90, ensuring clarity in configuration settings.

* Enhance S3Service to support deletion of attachments using S3 URI

- Updated deleteAttachment method to accept an S3 URI instead of just the object key.
- Extracted bucket name and key from the S3 URI for improved flexibility.
- Added logging to indicate which bucket and object are being deleted.
- Ensured deletion from both the specified bucket and the attachments bucket.

* Refactor SubmissionAnonymisationService to improve S3 deletion handling

- Updated the S3 deletion logic to abort anonymisation if any deletion fails, ensuring the submission remains in IN_PROGRESS for retry.
- Enhanced logging to provide clearer context on failures during S3 object deletion, improving traceability and error handling.

Author: rmohammed-goaco

src/main/java/gov/cabinetoffice/gap/adminbackend/config/SubmissionAnonymisationConfigProperties.java

@@ -0,0 +1,16 @@
+package gov.cabinetoffice.gap.adminbackend.config;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+@Getter
+@Setter
+@Configuration
+@ConfigurationProperties(prefix = "submission-anonymisation-scheduler")
+public class SubmissionAnonymisationConfigProperties {
+
+    private int daysBeforeExpiry = 90;
+
+}

src/main/java/gov/cabinetoffice/gap/adminbackend/enums/SubmissionStatus.java

@@ -1,5 +1,5 @@
 package gov.cabinetoffice.gap.adminbackend.enums;
 
 public enum SubmissionStatus {
-    IN_PROGRESS, SUBMITTED, GRANT_CLOSED
+    IN_PROGRESS, SUBMITTED, GRANT_CLOSED, EXPIRED
 }

src/main/java/gov/cabinetoffice/gap/adminbackend/repositories/GrantAttachmentRepository.java

@@ -3,10 +3,19 @@
 import gov.cabinetoffice.gap.adminbackend.entities.GrantAttachment;
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.stereotype.Repository;
+import org.springframework.transaction.annotation.Transactional;
 
+import java.util.List;
 import java.util.UUID;
 
 @Repository
 public interface GrantAttachmentRepository extends JpaRepository<GrantAttachment, UUID> {
+
     boolean existsBySubmissionId(UUID id);
+
+    List<GrantAttachment> findBySubmission_Id(UUID submissionId);
+
+    @Transactional
+    void deleteBySubmission_Id(UUID submissionId);
+
 }

src/main/java/gov/cabinetoffice/gap/adminbackend/repositories/GrantMandatoryQuestionRepository.java

@@ -3,6 +3,7 @@
 import gov.cabinetoffice.gap.adminbackend.entities.GrantMandatoryQuestions;
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.data.jpa.repository.Query;
+import org.springframework.transaction.annotation.Transactional;
 
 import java.util.List;
 import java.util.UUID;
@@ -35,4 +36,7 @@ public interface GrantMandatoryQuestionRepository extends JpaRepository<GrantMan
             + "and g.submission.status = 'SUBMITTED'")
     boolean existBySchemeIdAndCompletedStatusAndSubmittedSubmission(Integer id);
 
+    @Transactional
+    void deleteBySubmission_Id(UUID submissionId);
+
 }

src/main/java/gov/cabinetoffice/gap/adminbackend/repositories/SubmissionRepository.java

@@ -11,6 +11,7 @@
 import org.springframework.stereotype.Repository;
 
 import java.time.Instant;
+import java.time.LocalDateTime;
 import java.util.List;
 import java.util.Optional;
 import java.util.UUID;
@@ -36,4 +37,32 @@ void updateLastRequiredChecksExportByGrantApplicationIdAndStatus(Instant lastReq
     void updateLastRequiredChecksExportBySchemeIdAndStatus(Instant lastRequiredChecksExport, Integer id,
             SubmissionStatus status);
 
+    List<Submission> findByStatusAndLastUpdatedBefore(SubmissionStatus status, LocalDateTime cutoff);
+
+    @Transactional
+    @Modifying
+    @Query(value = """
+            UPDATE grant_submission
+               SET status          = 'EXPIRED',
+                   definition      = NULL,
+                   submission_name = NULL,
+                   gap_id          = NULL,
+                   applicant_id    = NULL,
+                   created_by      = NULL,
+                   last_updated_by = NULL,
+                   last_updated    = :now
+             WHERE id IN :ids
+            """, nativeQuery = true)
+    void anonymiseSubmissions(@Param("ids") List<UUID> ids, @Param("now") LocalDateTime now);
+
+    @Transactional
+    @Modifying
+    @Query(value = "DELETE FROM grant_beneficiary WHERE submission_id IN :ids", nativeQuery = true)
+    void deleteBeneficiaryRowsBySubmissionIds(@Param("ids") List<UUID> ids);
+
+    @Transactional
+    @Modifying
+    @Query(value = "DELETE FROM diligence_check WHERE submission_id IN :ids", nativeQuery = true)
+    void deleteDiligenceCheckRowsBySubmissionIds(@Param("ids") List<UUID> ids);
+
 }

src/main/java/gov/cabinetoffice/gap/adminbackend/schedulers/SubmissionAnonymisationScheduler.java

@@ -0,0 +1,51 @@
+package gov.cabinetoffice.gap.adminbackend.schedulers;
+
+import gov.cabinetoffice.gap.adminbackend.config.SubmissionAnonymisationConfigProperties;
+import gov.cabinetoffice.gap.adminbackend.entities.Submission;
+import gov.cabinetoffice.gap.adminbackend.enums.SubmissionStatus;
+import gov.cabinetoffice.gap.adminbackend.repositories.SubmissionRepository;
+import gov.cabinetoffice.gap.adminbackend.services.SubmissionAnonymisationService;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+import net.javacrumbs.shedlock.spring.annotation.SchedulerLock;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Service;
+
+import java.time.LocalDateTime;
+import java.util.List;
+
+@Log4j2
+@Service
+@RequiredArgsConstructor
+@ConditionalOnProperty(name = "submission-anonymisation-scheduler.enabled", havingValue = "true")
+public class SubmissionAnonymisationScheduler {
+
+    private final SubmissionRepository submissionRepository;
+
+    private final SubmissionAnonymisationService submissionAnonymisationService;
+
+    private final SubmissionAnonymisationConfigProperties config;
+
+    @Scheduled(cron = "${submission-anonymisation-scheduler.cronExpression:0 0 3 * * ?}", zone = "UTC")
+    @SchedulerLock(name = "submissionAnonymisation_anonymiseInactiveSubmissions",
+            lockAtMostFor = "${submission-anonymisation-scheduler.lock.atMostFor:30m}",
+            lockAtLeastFor = "${submission-anonymisation-scheduler.lock.atLeastFor:5m}")
+    public void anonymiseInactiveSubmissions() {
+        final LocalDateTime cutoff = LocalDateTime.now().minusDays(config.getDaysBeforeExpiry());
+
+        log.info("Submission anonymisation scheduler started. Anonymising IN_PROGRESS submissions last updated before {}",
+                cutoff);
+
+        final List<Submission> dueForAnonymisation = submissionRepository
+                .findByStatusAndLastUpdatedBefore(SubmissionStatus.IN_PROGRESS, cutoff);
+
+        log.info("Found {} submission(s) to anonymise", dueForAnonymisation.size());
+
+        dueForAnonymisation
+                .forEach(submission -> submissionAnonymisationService.anonymiseSubmission(submission.getId()));
+
+        log.info("Submission anonymisation scheduler completed successfully.");
+    }
+
+}

src/main/java/gov/cabinetoffice/gap/adminbackend/services/S3Service.java

@@ -3,6 +3,8 @@
 import com.amazonaws.HttpMethod;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.AmazonS3URI;
+import com.amazonaws.services.s3.model.DeleteObjectRequest;
 import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
 import lombok.RequiredArgsConstructor;
 import org.springframework.beans.factory.annotation.Value;
@@ -18,13 +20,24 @@
 public class S3Service {
 
     @Value("${cloud.aws.s3.submissions-export-bucket-name}")
-    private String attachmentsBucket;
+    private String exportBucketName;
+
+    @Value("${aws.attachmentsBucket}")
+    private String attachmentsBucketName;
 
     private final AmazonS3 s3Client = AmazonS3ClientBuilder.defaultClient();
 
+    public void deleteAttachment(String location) {
+        final AmazonS3URI s3Uri = new AmazonS3URI(location);
+        final String key = s3Uri.getKey();
+        log.info("Deleting S3 object {} from bucket {}", key, s3Uri.getBucket());
+        s3Client.deleteObject(new DeleteObjectRequest(s3Uri.getBucket(), key));
+        s3Client.deleteObject(new DeleteObjectRequest(attachmentsBucketName, key));
+    }
+
     public String generateExportDocSignedUrl(String objectKey) {
         int linkTimeoutDuration = 604800;
-        GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(attachmentsBucket,
+        GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(exportBucketName,
                 objectKey).withMethod(HttpMethod.GET)
                         .withExpiration(Date.from(Instant.now().plusSeconds(linkTimeoutDuration)));
 

src/main/java/gov/cabinetoffice/gap/adminbackend/services/SubmissionAnonymisationService.java

@@ -0,0 +1,68 @@
+package gov.cabinetoffice.gap.adminbackend.services;
+
+import gov.cabinetoffice.gap.adminbackend.entities.GrantAttachment;
+import gov.cabinetoffice.gap.adminbackend.repositories.GrantAttachmentRepository;
+import gov.cabinetoffice.gap.adminbackend.repositories.GrantMandatoryQuestionRepository;
+import gov.cabinetoffice.gap.adminbackend.repositories.SubmissionRepository;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import java.time.LocalDateTime;
+import java.util.List;
+import java.util.UUID;
+
+@Log4j2
+@Service
+@RequiredArgsConstructor
+public class SubmissionAnonymisationService {
+
+    private final SubmissionRepository submissionRepository;
+
+    private final GrantAttachmentRepository grantAttachmentRepository;
+
+    private final GrantMandatoryQuestionRepository grantMandatoryQuestionRepository;
+
+    private final S3Service s3Service;
+
+    @Transactional
+    public void anonymiseSubmission(UUID submissionId) {
+        log.info("Anonymising submission {}", submissionId);
+
+        // 1. Delete S3 objects — if any fail, abort and leave the submission in
+        // IN_PROGRESS so the scheduler retries it on the next run
+        final List<GrantAttachment> attachments = grantAttachmentRepository.findBySubmission_Id(submissionId);
+        for (GrantAttachment attachment : attachments) {
+            try {
+                s3Service.deleteAttachment(attachment.getLocation());
+                log.debug("Deleted S3 object {} for submission {}", attachment.getLocation(), submissionId);
+            }
+            catch (Exception e) {
+                log.warn(
+                        "Aborting anonymisation of submission {} — failed to delete S3 object {}: {}. "
+                                + "Submission will be retried on the next scheduler run.",
+                        submissionId, attachment.getLocation(), e.getMessage());
+                return;
+            }
+        }
+
+        // 2. Delete diligence_check rows (no cascade on this FK)
+        submissionRepository.deleteDiligenceCheckRowsBySubmissionIds(List.of(submissionId));
+
+        // 3. Delete grant_beneficiary rows
+        submissionRepository.deleteBeneficiaryRowsBySubmissionIds(List.of(submissionId));
+
+        // 4. Delete mandatory question rows
+        grantMandatoryQuestionRepository.deleteBySubmission_Id(submissionId);
+
+        // 5. Delete attachment DB rows
+        grantAttachmentRepository.deleteBySubmission_Id(submissionId);
+
+        // 6. Null out personal data on the submission row and mark EXPIRED
+        submissionRepository.anonymiseSubmissions(List.of(submissionId), LocalDateTime.now());
+
+        log.info("Anonymisation complete for submission {}", submissionId);
+    }
+
+}

src/main/resources/application.properties

@@ -30,6 +30,7 @@ cloud.aws.sqs.submissions-export-queue=cloud-platform-gap-apply-submissions-expo
 cloud.aws.sqs.event-service-queue=gap-apply-events-service-queue
 cloud.aws.sqs.event-service-queue-enabled=true
 cloud.aws.s3.submissions-export-bucket-name=cloud-platform-gap-apply-submissions-export-bucket
+aws.attachmentsBucket=attachments
 gov-notify-api-key=secretGovNotifyApiKey
 gov-notify-lambda-export-template-id=exportEmailTemplateId
 
@@ -45,6 +46,13 @@ completion-statistics-scheduler.cronExpression=0 6 * * * ?
 completion-statistics-scheduler.lock.atMostFor=30m
 completion-statistics-scheduler.lock.atLeastFor=5m
 
+#submissionAnonymisationScheduler configurable properties
+submission-anonymisation-scheduler.enabled=false
+submission-anonymisation-scheduler.cronExpression=0 0 3 * * ?
+submission-anonymisation-scheduler.daysBeforeExpiry=90
+submission-anonymisation-scheduler.lock.atMostFor=30m
+submission-anonymisation-scheduler.lock.atLeastFor=5m
+
 #grantAdvertsScheduler configurable properties
 grant-adverts-scheduler.cronExpression=0 01 0 * * ?
 grant-adverts-scheduler.lock.atMostFor=30m

src/main/resources/db/migration/V1_104__add_expired_submission_status.sql

@@ -0,0 +1,2 @@
+COMMENT ON COLUMN grant_submission.status IS
+    'Valid values: IN_PROGRESS, SUBMITTED, GRANT_CLOSED, EXPIRED';