PAFS: Submission performace impriovements

changelog/2-012-benefit-area-base64-cache.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+        http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-latest.xsd">
+
+  <changeSet id="2-012-benefit-area-base64-cache" author="migration-team">
+    <addColumn tableName="pafs_core_projects">
+      <column name="benefit_area_file_base64" type="TEXT">
+        <constraints nullable="true"/>
+      </column>
+    </addColumn>
+  </changeSet>
+
+</databaseChangeLog>

changelog/4-003-proposal-submissions-request-payload.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+        http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-latest.xsd">
+
+  <changeSet id="4-003-proposal-submissions-request-payload" author="migration-team">
+    <addColumn tableName="pafs_proposal_submissions">
+      <column name="request_payload" type="JSONB">
+        <constraints nullable="true"/>
+      </column>
+    </addColumn>
+  </changeSet>
+
+</databaseChangeLog>

changelog/db.changelog.xml

@@ -30,6 +30,8 @@
   <include file="2-010-widen-unique-session-id.xml" relativeToChangelogFile="true"/>
   <!-- Add stale_data_cleared flag for persistent warning banner after auto-flush of stale financial years -->
   <include file="2-011-stale-data-cleared.xml" relativeToChangelogFile="true"/>
+  <!-- Lazy-write base64 cache for the benefit area shapefile — avoids S3 fetch on critical submission path -->
+  <include file="2-012-benefit-area-base64-cache.xml" relativeToChangelogFile="true"/>
 
   <!-- Legacy tables - these are maintained for historical purposes, but new system should not use these tables. -->
   <include file="3-001-account-requests-legacy-table.xml" relativeToChangelogFile="true"/>
@@ -46,4 +48,7 @@
   <!-- Proposal submission tracking — records every attempt to send to the external system -->
   <include file="4-002-create-proposal-submissions.xml" relativeToChangelogFile="true"/>
 
+  <!-- Store outgoing request payload per submission attempt (shapefile base64 scrubbed at app layer) -->
+  <include file="4-003-proposal-submissions-request-payload.xml" relativeToChangelogFile="true"/>
+
 </databaseChangeLog>

prisma/schema.prisma

@@ -479,6 +479,7 @@ model pafs_core_projects {
   created_at                                                  DateTime  @db.Timestamp(6)
   updated_at                                                  DateTime  @db.Timestamp(6)
   stale_data_cleared                                          Boolean?  @default(false)
+  benefit_area_file_base64                                    String?
 
   @@unique([reference_number, version], map: "index_pafs_core_projects_on_reference_number_and_version")
   @@index([benefit_area_file_s3_bucket, benefit_area_file_s3_key], map: "idx_core_projects_benefit_area_s3")
@@ -618,6 +619,7 @@ model pafs_proposal_submissions {
   is_resend        Boolean  @default(false)
   attempted_at     DateTime @default(now()) @db.Timestamp(6)
   created_at       DateTime @default(now()) @db.Timestamp(6)
+  request_payload  Json?
 
   @@index([project_id], map: "idx_proposal_submissions_project_id")
   @@index([reference_number], map: "idx_proposal_submissions_reference_number")

src/common/helpers/sqs/send-external-submission-message.js

@@ -0,0 +1,30 @@
+import { SendMessageCommand } from '@aws-sdk/client-sqs'
+import { config } from '../../../config.js'
+
+/**
+ * Enqueue an external submission job on SQS.
+ *
+ * The consumer will reload fresh project data from the database and forward
+ * the proposal to the external system (AIMS PD). Keeping only the reference
+ * number and project ID in the message avoids storing sensitive project data
+ * in SQS.
+ *
+ * @param {import('@aws-sdk/client-sqs').SQSClient} sqsClient
+ * @param {string} referenceNumber
+ * @param {bigint} projectId
+ * @returns {Promise<void>}
+ */
+export async function sendExternalSubmissionMessage(
+  sqsClient,
+  referenceNumber,
+  projectId
+) {
+  const command = new SendMessageCommand({
+    QueueUrl: config.get('sqsExternalSubmission.queueUrl'),
+    MessageBody: JSON.stringify({
+      referenceNumber,
+      projectId: projectId.toString()
+    })
+  })
+  await sqsClient.send(command)
+}

src/common/helpers/sqs/send-external-submission-message.test.js

@@ -0,0 +1,103 @@
+import { describe, test, expect, beforeEach, vi } from 'vitest'
+
+vi.mock('@aws-sdk/client-sqs', () => ({
+  SendMessageCommand: vi.fn(function SendMessageCommand(input) {
+    this.input = input
+  })
+}))
+
+vi.mock('../../../config.js', () => ({
+  config: {
+    get: vi.fn((key) => {
+      if (key === 'sqsExternalSubmission.queueUrl') {
+        return 'http://localhost:4566/000000000000/pafs_external_submission'
+      }
+      return null
+    })
+  }
+}))
+
+const { SendMessageCommand } = await import('@aws-sdk/client-sqs')
+const { sendExternalSubmissionMessage } =
+  await import('./send-external-submission-message.js')
+
+describe('sendExternalSubmissionMessage', () => {
+  let mockSqsClient
+
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockSqsClient = { send: vi.fn().mockResolvedValue({}) }
+  })
+
+  test('sends a SendMessageCommand to the configured queue URL', async () => {
+    await sendExternalSubmissionMessage(
+      mockSqsClient,
+      'LCR/123/456',
+      BigInt(99)
+    )
+
+    expect(mockSqsClient.send).toHaveBeenCalledOnce()
+    expect(mockSqsClient.send).toHaveBeenCalledWith(
+      expect.objectContaining({
+        input: expect.objectContaining({
+          QueueUrl:
+            'http://localhost:4566/000000000000/pafs_external_submission'
+        })
+      })
+    )
+  })
+
+  test('message body contains referenceNumber and projectId as a string', async () => {
+    await sendExternalSubmissionMessage(
+      mockSqsClient,
+      'LCR/123/456',
+      BigInt(99)
+    )
+
+    const [command] = mockSqsClient.send.mock.calls[0]
+    const body = JSON.parse(command.input.MessageBody)
+
+    expect(body.referenceNumber).toBe('LCR/123/456')
+    expect(body.projectId).toBe('99')
+  })
+
+  test('serialises BigInt projectId to string (not a number)', async () => {
+    await sendExternalSubmissionMessage(
+      mockSqsClient,
+      'EA/999/AAA/2025',
+      BigInt('9007199254740993') // exceeds Number.MAX_SAFE_INTEGER — must use string to preserve precision
+    )
+
+    const [command] = mockSqsClient.send.mock.calls[0]
+    // Use the raw MessageBody string — JSON.parse would silently lose precision
+    // for integers beyond MAX_SAFE_INTEGER
+    expect(command.input.MessageBody).toContain(
+      '"projectId":"9007199254740993"'
+    )
+  })
+
+  test('constructs a SendMessageCommand with the correct shape', async () => {
+    await sendExternalSubmissionMessage(
+      mockSqsClient,
+      'LCR/123/456',
+      BigInt(42)
+    )
+
+    expect(SendMessageCommand).toHaveBeenCalledWith({
+      QueueUrl: 'http://localhost:4566/000000000000/pafs_external_submission',
+      MessageBody: JSON.stringify({
+        referenceNumber: 'LCR/123/456',
+        projectId: '42'
+      })
+    })
+  })
+
+  test('propagates errors thrown by sqsClient.send', async () => {
+    const sqsError = new Error('SQS unavailable')
+    mockSqsClient.send.mockRejectedValue(sqsError)
+
+    await expect(
+      sendExternalSubmissionMessage(mockSqsClient, 'LCR/123/456', BigInt(1))
+    ).rejects.toThrow('SQS unavailable')
+  })
+})

src/common/services/external-submission/external-submission-service.js

@@ -54,6 +54,7 @@ export class ExternalSubmissionService {
         httpStatusCode: null,
         errorMessage: 'External submission is disabled',
         responseBody: null,
+        requestPayload: this._scrubPayload(payload),
         isResend
       })
       return { success: false, error: 'External submission is disabled' }
@@ -74,6 +75,7 @@ export class ExternalSubmissionService {
           referenceNumber,
           httpStatus,
           responseText,
+          payload,
           isResend
         })
       }
@@ -83,6 +85,7 @@ export class ExternalSubmissionService {
         referenceNumber,
         httpStatus,
         responseText,
+        payload,
         isResend
       })
     } catch (error) {
@@ -102,6 +105,7 @@ export class ExternalSubmissionService {
         httpStatusCode: httpStatus,
         errorMessage,
         responseBody: responseText,
+        requestPayload: this._scrubPayload(payload),
         isResend
       })
       return { success: false, error: errorMessage }
@@ -150,6 +154,7 @@ export class ExternalSubmissionService {
     referenceNumber,
     httpStatus,
     responseText,
+    payload,
     isResend
   }) {
     this.logger.warn(
@@ -163,6 +168,7 @@ export class ExternalSubmissionService {
       httpStatusCode: httpStatus,
       errorMessage: `HTTP ${httpStatus}`,
       responseBody: responseText,
+      requestPayload: this._scrubPayload(payload),
       isResend
     })
     return { success: false, httpStatus, error: `HTTP ${httpStatus}` }
@@ -177,6 +183,7 @@ export class ExternalSubmissionService {
     referenceNumber,
     httpStatus,
     responseText,
+    payload,
     isResend
   }) {
     this.logger.info(
@@ -190,6 +197,7 @@ export class ExternalSubmissionService {
       httpStatusCode: httpStatus,
       errorMessage: null,
       responseBody: responseText,
+      requestPayload: this._scrubPayload(payload),
       isResend
     })
     await this.markSubmittedToPol(referenceNumber)
@@ -200,13 +208,30 @@ export class ExternalSubmissionService {
    * Persist a submission attempt to pafs_proposal_submissions.
    * @private
    */
+  /**
+   * Return a copy of the payload with the shapefile base64 stripped.
+   * The binary data can be multi-megabytes — the S3 key on the project record
+   * is sufficient to retrieve the file when needed.
+   * @private
+   */
+  _scrubPayload(payload) {
+    if (!payload) {
+      return null
+    }
+    if (!payload.shapefile) {
+      return payload
+    }
+    return { ...payload, shapefile: '[base64_omitted]' }
+  }
+
   async _recordAttempt({
     projectId,
     referenceNumber,
     status,
     httpStatusCode,
     errorMessage,
     responseBody,
+    requestPayload = null,
     isResend
   }) {
     try {
@@ -218,6 +243,7 @@ export class ExternalSubmissionService {
           http_status_code: httpStatusCode,
           error_message: errorMessage,
           response_body: responseBody,
+          request_payload: requestPayload,
           is_resend: isResend,
           attempted_at: new Date(),
           created_at: new Date()