[REMOCK-2] PLU-471: allow metadata in testRun, formsg testRun support preferMock metadata (#1061)

### Summary

`testRun` functions in apps now support an additional param `testRunMetadata`.

FormSG's `testRun` now accepts `preferMock` as part of `testRunMetadata`

### Changes

- FormSG test run pulls last executionStep filtered by 'isMock'
- `getLastExecutionStep` now supports an additional filter to the query builder
- Added integration tests for the Step model's `getLastExecutionStep` method
- Updated GraphQL schema to include `testRunMetadata` in the `ExecuteStepInput`
- Modified the execute-step mutation to pass test run metadata to the test step service

### How to test?

1. Call the `executeStep` mutation directly with
```
{ 
  "stepId": "...",
  "testRunMetadata":  {
     "preferMock": true/false
  }
}

Author: pregnantboy

packages/backend/src/apps/formsg/__tests__/triggers/new-submission.test.ts

@@ -2,14 +2,31 @@ import {
   IDataOutMetadata,
   IDataOutMetadatum,
   IExecutionStep,
+  IGlobalVariable,
   IJSONObject,
 } from '@plumber/types'
 
-import { beforeEach, describe, expect, it } from 'vitest'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 
 import { ADDRESS_LABELS } from '../../common/constants'
 import trigger from '../../triggers/new-submission'
 
+const pushTriggerItemMock = vi.fn()
+const getLastExecutionStepMock = vi.fn()
+
+const mocks = vi.hoisted(() => ({
+  getMockData: vi.fn(),
+  getFormDetailsFromGlobalVariable: vi.fn(),
+}))
+
+vi.mock('../../triggers/new-submission/get-mock-data', () => ({
+  default: mocks.getMockData,
+}))
+
+vi.mock('../../common/webhook-settings', () => ({
+  getFormDetailsFromGlobalVariable: mocks.getFormDetailsFromGlobalVariable,
+}))
+
 describe('new submission trigger', () => {
   let executionStep: IExecutionStep
 
@@ -42,6 +59,117 @@ describe('new submission trigger', () => {
     } as unknown as IExecutionStep
   })
 
+  describe('testRun', () => {
+    const $ = {
+      auth: { data: { formId: '123' } },
+      user: { email: '[email protected]' },
+      pushTriggerItem: pushTriggerItemMock,
+      getLastExecutionStep: getLastExecutionStepMock,
+    } as unknown as IGlobalVariable
+
+    const mockData = {
+      formId: '123',
+      responses: {
+        mockTextFieldId: {
+          question: 'What is your name?',
+          answer: 'herp derp',
+        },
+      },
+    }
+
+    const actualData = {
+      formId: '123',
+      responses: {
+        textFieldId: {
+          question: 'What is your age?',
+          answer: 10,
+        },
+      },
+    }
+
+    beforeEach(() => {
+      mocks.getMockData.mockResolvedValue(mockData)
+      mocks.getFormDetailsFromGlobalVariable.mockReturnValue({
+        formId: '123',
+      })
+    })
+
+    afterEach(() => {
+      vi.clearAllMocks()
+    })
+
+    it('should use mock data if preferMock is true and there is no past submission', async () => {
+      getLastExecutionStepMock.mockResolvedValue(null)
+      await trigger.testRun($, { preferMock: true })
+      expect(pushTriggerItemMock).toHaveBeenCalledWith({
+        raw: mockData,
+        meta: {
+          internalId: '',
+        },
+        isMock: true,
+      })
+    })
+
+    it('should use mock data if preferMock is true even though there is past submission', async () => {
+      getLastExecutionStepMock.mockResolvedValue({ dataOut: actualData })
+      await trigger.testRun($, { preferMock: true })
+      expect(pushTriggerItemMock).toHaveBeenCalledWith({
+        raw: mockData,
+        meta: {
+          internalId: '',
+        },
+        isMock: true,
+      })
+    })
+
+    it('should use mock data if testRunMetadata is undefined and there is no past submission', async () => {
+      getLastExecutionStepMock.mockResolvedValue(null)
+      await trigger.testRun($, undefined)
+      expect(pushTriggerItemMock).toHaveBeenCalledWith({
+        raw: mockData,
+        meta: {
+          internalId: '',
+        },
+        isMock: true,
+      })
+    })
+
+    it('should use last test submission if testRunMetadata is undefined and there is past submission', async () => {
+      getLastExecutionStepMock.mockResolvedValue({ dataOut: actualData })
+      await trigger.testRun($, undefined)
+      expect(pushTriggerItemMock).toHaveBeenCalledWith({
+        raw: actualData,
+        meta: {
+          internalId: '',
+        },
+        isMock: false,
+      })
+    })
+
+    it('should use last test submission if preferMock is false and there is past submission', async () => {
+      getLastExecutionStepMock.mockResolvedValue({ dataOut: actualData })
+      await trigger.testRun($, { preferMock: false })
+      expect(pushTriggerItemMock).toHaveBeenCalledWith({
+        raw: actualData,
+        meta: {
+          internalId: '',
+        },
+        isMock: false,
+      })
+    })
+
+    it('should use mock data if preferMock is false and there is no past submission', async () => {
+      getLastExecutionStepMock.mockResolvedValue(null)
+      await trigger.testRun($, { preferMock: false })
+      expect(pushTriggerItemMock).toHaveBeenCalledWith({
+        raw: mockData,
+        meta: {
+          internalId: '',
+        },
+        isMock: true,
+      })
+    })
+  })
   describe('dataOut metadata', () => {
     it('ensures that only question, answer and answerArray props are visible', async () => {
       const metadata = await trigger.getDataOutMetadata(executionStep)

packages/backend/src/apps/formsg/triggers/new-submission/index.ts

@@ -1,12 +1,22 @@
 import { IGlobalVariable, IRawTrigger } from '@plumber/types'
 
+import { RelatedQueryBuilder } from 'objection'
+import { z } from 'zod'
+
 import StepError from '@/errors/step'
+import ExecutionStep from '@/models/execution-step'
 
 import { getFormDetailsFromGlobalVariable } from '../../common/webhook-settings'
 
 import getDataOutMetadata from './get-data-out-metadata'
 import getMockData from './get-mock-data'
 
+const formsgTestRunMetadataSchema = z
+  .object({
+    preferMock: z.boolean().optional(),
+  })
+  .default({ preferMock: false })
+
 export const NricFilter = {
   None: 'none',
   Remove: 'remove',
@@ -59,7 +69,7 @@ const trigger: IRawTrigger = {
 
   getDataOutMetadata,
 
-  async testRun($: IGlobalVariable) {
+  async testRun($: IGlobalVariable, testRunMetadata?: { preferMock: boolean }) {
     if (!$.auth.data) {
       throw new StepError(
         'Missing FormSG connection',
@@ -69,28 +79,46 @@ const trigger: IRawTrigger = {
       )
     }
 
+    const testRunMetadataRes =
+      formsgTestRunMetadataSchema.safeParse(testRunMetadata)
+    if (!testRunMetadataRes.success) {
+      throw new StepError(
+        'Something went wrong',
+        'Invalid test run metadata. Please refresh and try again.',
+        $.step.position,
+        $.app.name,
+      )
+    }
+
     // data out should never be empty after test step is pressed once: either mock or actual data
     const { formId } = getFormDetailsFromGlobalVariable($)
-    // We use last test execution step
-    const lastTestExecutionStep = await $.getLastExecutionStep({
+    // We use last actual submission test execution step execution step
+    const lastSubmittedTestExecutionStep = await $.getLastExecutionStep({
       testRunOnly: true,
+      additionalFilter: (qb: RelatedQueryBuilder<ExecutionStep>) =>
+        qb.andWhereRaw("(metadata->>'isMock')::boolean IS DISTINCT FROM true"),
     })
-    // If no past submission (no form) or the form is changed, it is a mock run (re-pull mock data)
+
     const hasNoPastSubmission =
-      lastTestExecutionStep?.dataOut?.formId !== formId ||
-      lastTestExecutionStep.metadata.isMock
+      !lastSubmittedTestExecutionStep ||
+      lastSubmittedTestExecutionStep?.dataOut?.formId !== formId
+
+    const shouldUseMockData =
+      hasNoPastSubmission || testRunMetadataRes.data.preferMock
+
+    // if test with mock data is selected OR no past submission exists
+    // we use mock data
+    const testData = shouldUseMockData
+      ? await getMockData($)
+      : lastSubmittedTestExecutionStep?.dataOut
 
     // if different or no form is detected, use mock data
     await $.pushTriggerItem({
-      raw: hasNoPastSubmission
-        ? await getMockData($)
-        : lastTestExecutionStep?.dataOut,
+      raw: testData,
       meta: {
         internalId: '',
       },
-      isMock:
-        hasNoPastSubmission ||
-        (lastTestExecutionStep.metadata?.isMock ?? false), // use previous mock run status from metadata by default
+      isMock: shouldUseMockData,
     })
   },
 }

packages/backend/src/graphql/mutations/execute-step.ts

@@ -9,7 +9,7 @@ const executeStep: MutationResolvers['executeStep'] = async (
   params,
   context,
 ) => {
-  const { stepId } = params.input
+  const { stepId, testRunMetadata } = params.input
 
   // Just checking for permissions here
   const stepToTest = await context.currentUser
@@ -20,6 +20,7 @@ const executeStep: MutationResolvers['executeStep'] = async (
 
   const { executionStep, executionId } = await testStep({
     stepId: stepToTest.id,
+    testRunMetadata,
   })
 
   // Update flow to use the new test execution

packages/backend/src/graphql/schema.graphql

@@ -475,6 +475,7 @@ input UpdateFlowConfigInput {
 
 input ExecuteStepInput {
   stepId: String!
+  testRunMetadata: JSONObject
 }
 
 input DeleteFlowInput {

packages/backend/src/helpers/global-variable.ts

@@ -99,6 +99,7 @@ const globalVariable = async (
         await step?.getLastExecutionStep({
           executionId: options?.sameExecution ? execution.id : undefined,
           testRunOnly: options?.testRunOnly,
+          additionalFilter: options?.additionalFilter,
         })
       )?.toJSON()
     },