PLU-624: fix(formsg-frontend): table field preview (#1355)

## Problem
* Forms that contain **Section** or **Image** fields cannot preview
their table rows due to a mismatch in the Zod schema.
The current schema allows `undefined`, but these field types return
`null`, which causes validation errors.
* If there is more than 1 table in the form, the preview always shows
data from the first table

## Solution
- Replace `.optional()` with `.nullish()` so that both undefined and
null are accepted.
- Fetch the table data using the variable id

## Tests
- [x] All FormSG test results display correctly
- [x] FormSG tables can be previewed correctly even when they contain
section headers or images
- [x] FormSG table previews display the correct data when there are two
or more tables

---------

Co-authored-by: Ian Chen <[email protected]>

Author: kevinkim-ogp

packages/backend/src/apps/formsg/triggers/new-submission/get-data-out-metadata.ts

@@ -204,7 +204,9 @@ function buildTableMetadatum(fieldData: IJSONObject): IDataOutMetadata {
 
   const tableObject = JSON.parse(fieldData.answer as string)
   return {
-    label: `Response ${fieldData.order}`,
+    label: fieldData.question
+      ? `${fieldData.order} ${fieldData.question}`
+      : `Response ${fieldData.order}`,
     order: fieldData.order ? (fieldData.order as number) + 0.1 : null,
     type: 'table',
     displayedValue: `Preview ${tableObject.rows.length} row(s)`,

packages/backend/src/apps/formsg/triggers/new-submission/get-mock-data.ts

@@ -13,6 +13,7 @@ type FormField = {
   _id: string
   columns?: Array<{
     _id: string
+    title?: string
   }>
   fieldType: string
   fieldOptions?: string[]
@@ -130,8 +131,9 @@ async function getMockData($: IGlobalVariable) {
           data.responses[formFields[i]._id].answer = 'Signature captured' // mock this to always be present regardless of whether the user has signed or not
         }
 
-        // formsg payload doesnt contain this anyways, so we dont return in mock data
-        if (fieldType === 'statement') {
+        // formsg payload doesnt contain these fields anyways, so we dont return in mock data
+        // this ensures that the question numbering remains consistent with the actual submission
+        if (fieldType === 'statement' || fieldType === 'image') {
           delete data.responses[formFields[i]._id]
           continue
         }
@@ -160,11 +162,21 @@ async function getMockData($: IGlobalVariable) {
         if (fieldType === 'table') {
           const answerArray = data.responses[formFields[i]._id]
             .answerArray as string[][]
-          const question = data.responses[formFields[i]._id].question
+          const question = `${
+            data.responses[formFields[i]._id].question
+          } (${formFields[i].columns
+            ?.map((column, index) => column?.title ?? `Col ${index + 1}`)
+            .join(', ')})`
+
+          data.responses[formFields[i]._id].question = question
           data.responses[formFields[i]._id].answer =
             convertTableAnswerArrayToTableObject(question, answerArray)
         }
 
+        if (fieldType === 'section' || fieldType === 'image') {
+          data.responses[formFields[i]._id].answer = ''
+        }
+
         data.responses[formFields[i]._id].order = i + 1
         data.responses[formFields[i]._id].id = undefined
       }

packages/frontend/src/components/VariablesList/TableVariableItem.tsx

@@ -33,6 +33,7 @@ export default function TableVariableItem(props: TableVariableItemProps) {
       />
       {!onClick && (
         <TableVariableModal
+          variableId={variable.name}
           isOpen={isOpen}
           onClose={onClose}
           currentExecutionStep={currentTestExecutionStep}

packages/frontend/src/components/VariablesList/TableVariableModal.tsx

@@ -28,6 +28,8 @@ interface TableVariableModalProps {
   isOpen: boolean
   onClose: () => void
   currentExecutionStep?: IExecutionStep | null
+  // variableName is the step variable: `step.<uuid>.xxxxx`
+  variableId: string
 }
 
 const TableHeader = ({ columns }: { columns: ProcessedColumn[] }) => (
@@ -107,12 +109,15 @@ const TableRow = ({
 )
 
 export default function TableVariableModal(props: TableVariableModalProps) {
-  const { isOpen, onClose, currentExecutionStep } = props
+  const { isOpen, onClose, currentExecutionStep, variableId } = props
 
   if (!currentExecutionStep) {
     return null
   }
-  const { rowsFound, dataRows, columns } = processData(currentExecutionStep)
+  const { rowsFound, dataRows, columns } = processData(
+    currentExecutionStep,
+    variableId,
+  )
 
   return (
     <Modal

packages/frontend/src/components/VariablesList/schema.ts

@@ -1,5 +1,6 @@
 import { IJSONObject } from '@plumber/types'
 
+import get from 'lodash.get'
 import { z } from 'zod'
 
 const RowDataSchema = z.object({
@@ -32,26 +33,31 @@ const FormSgTableFieldSchema = z.object({
 
 const FormSgFieldSchema = z.object({
   fieldType: z.string(),
-  answer: z.union([z.string(), z.record(z.any())]).optional(),
+  answer: z.union([z.string(), z.record(z.any())]).nullish(),
 })
 
-const FormSgFieldsSchema = z.record(FormSgFieldSchema).transform((fields) => {
-  const tableField = Object.values(fields).find(
-    (field) => field?.fieldType === 'table',
-  ) as z.infer<typeof FormSgFieldSchema> | undefined
+const createFormSgFieldsSchema = (variableId: string) => {
+  const regex = new RegExp(
+    /step\.[\da-f]{8}(?:-[\da-f]{4}){3}-[\da-f]{12}\.fields\.([^.]+)\.answer/,
+  )
+  const match = variableId.match(regex)
+  const tableFieldId = match?.[1]
 
-  if (!tableField) {
-    return {
-      fieldType: 'table',
-      answer: {
-        rows: [],
-        columns: [],
-      },
+  return z.record(FormSgFieldSchema).transform((fields) => {
+    if (!tableFieldId) {
+      return {
+        fieldType: 'table',
+        answer: {
+          rows: [],
+          columns: [],
+        },
+      }
     }
-  }
 
-  return FormSgTableFieldSchema.parse(tableField)
-})
+    const tableField = get(fields, tableFieldId)
+    return FormSgTableFieldSchema.parse(tableField)
+  })
+}
 
 /**
  * FormSG has a different dataOut structure from our own apps.
@@ -60,26 +66,29 @@ const FormSgFieldsSchema = z.record(FormSgFieldSchema).transform((fields) => {
  * Note: FormSG implements check to have at least 1 row in the table field.
  * the cells may be empty, but there will always be at least 1 row.
  */
-export const FormSgTableDataOutSchema = z
-  .object({
-    fields: FormSgFieldsSchema,
-  })
-  .transform((dataOut) => {
-    const parsedData = dataOut.fields.answer as z.infer<typeof RowDataSchema>
-    return {
-      rowsFound:
-        (parsedData.rows as z.infer<typeof RowDataSchema>['rows'])?.length ?? 0,
-      data: parsedData,
-    }
-  })
+export const createFormSgTableDataOutSchema = (variableName: string) =>
+  z
+    .object({
+      fields: createFormSgFieldsSchema(variableName),
+    })
+    .transform((dataOut) => {
+      const parsedData = dataOut.fields.answer as z.infer<typeof RowDataSchema>
+      return {
+        rowsFound:
+          (parsedData.rows as z.infer<typeof RowDataSchema>['rows'])?.length ??
+          0,
+        data: parsedData,
+      }
+    })
 
 export const MultipleRowDataOutSchema = z.object({
   rowsFound: z.union([z.string(), z.number()]).default(0),
   data: RowDataSchema,
 })
 
 // Enhanced schema that can handle both regular and FormSG data
-export const ExecutionStepDataOutSchema = z.union([
-  MultipleRowDataOutSchema,
-  FormSgTableDataOutSchema,
-])
+export const createExecutionStepDataOutSchema = (variableName: string) =>
+  z.union([
+    MultipleRowDataOutSchema,
+    createFormSgTableDataOutSchema(variableName),
+  ])

packages/frontend/src/components/VariablesList/utils.tsx

@@ -1,6 +1,6 @@
 import { IExecutionStep } from '@plumber/types'
 
-import { ExecutionStepDataOutSchema } from './schema'
+import { createExecutionStepDataOutSchema } from './schema'
 
 export interface RawColumn {
   id: string
@@ -42,9 +42,14 @@ const processColumns = (rawColumns: RawColumn[]): ProcessedColumn[] => {
     .sort((a, b) => (a.order ?? 0) - (b.order ?? 0))
 }
 
-export const processData = (executionStep: IExecutionStep): ProcessedData => {
+export const processData = (
+  executionStep: IExecutionStep,
+  variableId: string,
+): ProcessedData => {
   try {
-    const dataOut = ExecutionStepDataOutSchema.parse(executionStep.dataOut)
+    const dataOut = createExecutionStepDataOutSchema(variableId).parse(
+      executionStep.dataOut,
+    )
     const rowsFound = String(dataOut.rowsFound)
 
     return {