PLU-345: FormSG table field

packages/backend/src/apps/formsg/__tests__/auth/decrypt-form-response.test.ts

@@ -3,6 +3,8 @@ import { IGlobalVariable, IRequest } from '@plumber/types'
 import { Settings as LuxonSettings } from 'luxon'
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 
+import { FOR_EACH_INPUT_SOURCE } from '@/apps/toolbox/common/constants'
+
 import app from '../..'
 import { decryptFormResponse } from '../../auth/decrypt-form-response'
 import { NricFilter } from '../../triggers/new-submission'
@@ -724,6 +726,47 @@ describe('decrypt form response', () => {
             tableField: {
               fieldType: 'table',
               question: 'What are your hobbies and when do you do them?',
+              answer: JSON.stringify({
+                rows: [
+                  {
+                    data: {
+                      [Buffer.from('Column 1').toString('hex')]: 'reading',
+                      [Buffer.from('Column 2').toString('hex')]: 'night',
+                    },
+                  },
+                  {
+                    data: {
+                      [Buffer.from('Column 1').toString('hex')]: 'gaming',
+                      [Buffer.from('Column 2').toString('hex')]: 'weekend',
+                    },
+                  },
+                  {
+                    data: {
+                      [Buffer.from('Column 1').toString('hex')]: 'coding',
+                      [Buffer.from('Column 2').toString('hex')]: 'day',
+                    },
+                  },
+                ],
+                columns: [
+                  {
+                    id: Buffer.from('Column 1').toString('hex'),
+                    label: 'Column 1',
+                    name: 'Column 1',
+                    value: `data.rows.*.data.${Buffer.from('Column 1').toString(
+                      'hex',
+                    )}`,
+                  },
+                  {
+                    id: Buffer.from('Column 2').toString('hex'),
+                    label: 'Column 2',
+                    name: 'Column 2',
+                    value: `data.rows.*.data.${Buffer.from('Column 2').toString(
+                      'hex',
+                    )}`,
+                  },
+                ],
+                inputSource: FOR_EACH_INPUT_SOURCE.FORMSG_TABLE,
+              }),
               answerArray: [
                 ['reading', 'night'],
                 ['gaming', 'weekend'],
@@ -781,6 +824,41 @@ describe('decrypt form response', () => {
                 ['reading', 'night'],
                 ['gaming', 'weekend'],
               ],
+              answer: JSON.stringify({
+                rows: [
+                  {
+                    data: {
+                      [Buffer.from('Column 1').toString('hex')]: 'reading',
+                      [Buffer.from('Column 2').toString('hex')]: 'night',
+                    },
+                  },
+                  {
+                    data: {
+                      [Buffer.from('Column 1').toString('hex')]: 'gaming',
+                      [Buffer.from('Column 2').toString('hex')]: 'weekend',
+                    },
+                  },
+                ],
+                columns: [
+                  {
+                    id: Buffer.from('Column 1').toString('hex'),
+                    label: 'Column 1',
+                    name: 'Column 1',
+                    value: `data.rows.*.data.${Buffer.from('Column 1').toString(
+                      'hex',
+                    )}`,
+                  },
+                  {
+                    id: Buffer.from('Column 2').toString('hex'),
+                    label: 'Column 2',
+                    name: 'Column 2',
+                    value: `data.rows.*.data.${Buffer.from('Column 2').toString(
+                      'hex',
+                    )}`,
+                  },
+                ],
+                inputSource: FOR_EACH_INPUT_SOURCE.FORMSG_TABLE,
+              }),
               order: 2,
             },
           },

packages/backend/src/apps/formsg/__tests__/triggers/new-submission.test.ts

@@ -484,6 +484,40 @@ describe('new submission trigger for answer array fields', () => {
               ['sleeping', 'night'],
               ['eating', 'all day'],
             ],
+            answer: JSON.stringify({
+              rows: [
+                {
+                  data: {
+                    [Buffer.from('Column 1').toString('hex')]: 'sleeping',
+                    [Buffer.from('Column 2').toString('hex')]: 'night',
+                  },
+                },
+                {
+                  data: {
+                    [Buffer.from('Column 1').toString('hex')]: 'eating',
+                    [Buffer.from('Column 2').toString('hex')]: 'all day',
+                  },
+                },
+              ],
+              columns: [
+                {
+                  id: Buffer.from('Column 1').toString('hex'),
+                  label: 'Column 1',
+                  name: 'Column 1',
+                  value: `data.rows.*.data.${Buffer.from('Column 1').toString(
+                    'hex',
+                  )}`,
+                },
+                {
+                  id: Buffer.from('Column 2').toString('hex'),
+                  label: 'Column 2',
+                  name: 'Column 2',
+                  value: `data.rows.*.data.${Buffer.from('Column 2').toString(
+                    'hex',
+                  )}`,
+                },
+              ],
+            }),
           },
           textFieldId3: {
             question: 'Unknown question',

packages/backend/src/apps/formsg/auth/decrypt-form-response.ts

@@ -10,6 +10,7 @@ import { sha256Hash } from '@/helpers/crypto'
 import logger from '@/helpers/logger'
 
 import { getSdk, parseFormEnv } from '../common/form-env'
+import convertTableAnswerArrayToTableObject from '../common/process-table-field'
 import { NricFilter } from '../triggers/new-submission/index'
 
 import storeAttachmentInS3 from './helpers/store-attachment-in-s3'
@@ -128,6 +129,11 @@ export async function decryptFormResponse(
           rest.answerArray = (rest.answerArray as string[][]).map((row) =>
             row.map((column) => column.replaceAll('\u0000', '')),
           )
+
+          rest.answer = convertTableAnswerArrayToTableObject(
+            rest.question,
+            rest.answerArray,
+          )
         } else {
           rest.answerArray = (rest.answerArray as string[]).map((answer) =>
             answer.replaceAll('\u0000', ''),

packages/backend/src/apps/formsg/common/process-table-field.ts

@@ -0,0 +1,53 @@
+import { FOR_EACH_INPUT_SOURCE } from '@/apps/toolbox/common/constants'
+
+import { extractLastTopLevelBracketContent } from '../triggers/new-submission/get-data-out-metadata'
+
+type TableColumn = {
+  id: string
+  label: string
+  name: string
+  value: string
+}
+
+const createColumn = (label: string): TableColumn => {
+  const id = Buffer.from(label).toString('hex')
+  return {
+    id,
+    label,
+    name: label,
+    value: `data.rows.*.data.${id}`,
+  }
+}
+
+export default function convertTableAnswerArrayToTableObject(
+  question: string,
+  answerArray: string[][],
+) {
+  const { content: columnNames } = extractLastTopLevelBracketContent(question)
+  const columnNamesArray = columnNames.split(',').map((name) => name.trim())
+
+  const columns =
+    // make sure that column names do not contain commas
+    columnNamesArray.length === answerArray[0].length
+      ? columnNamesArray.map(createColumn)
+      : answerArray[0].map((_, index) => createColumn(`Column ${index + 1}`))
+
+  /**
+   * NOTE: we do not show table rows that do not have any data
+   */
+  const rows = (answerArray as string[][])
+    .filter((row) => !row.every((v) => v === ''))
+    .map((row) => {
+      const rowData: Record<string, string | number> = {}
+      row.forEach((v: string, i: number) => {
+        rowData[columns[i].id] = v.replaceAll('\u0000', '')
+      })
+      return { data: rowData }
+    })
+
+  return JSON.stringify({
+    rows,
+    columns,
+    inputSource: FOR_EACH_INPUT_SOURCE.FORMSG_TABLE,
+  })
+}

packages/backend/src/apps/formsg/triggers/new-submission/get-data-out-metadata.ts

@@ -99,7 +99,7 @@ function buildAnswerArrayForCheckbox(
   }
 }
 
-function extractLastTopLevelBracketContent(questionText: string): {
+export function extractLastTopLevelBracketContent(questionText: string): {
   content: string
   prefix: string
 } {
@@ -162,13 +162,33 @@ function buildAnswerArrayForTable(
         type: 'text',
         label,
         order: order ? (order as number) + 0.1 : null,
+        // NOTE: we hide the option if it is empty
+        // mock data will have dummy strings in the answerArray
+        // actual submissions should contain real data, otherwise the cells should be hidden
+        isHidden: optionArray[j] === '',
       })
     }
     answerArray.push(nestedAnswerArray)
   }
   return answerArray
 }
 
+function buildTableMetadatum(fieldData: IJSONObject): IDataOutMetadata {
+  // old execution steps may not have an answer field for their table
+  if (!fieldData.answer) {
+    return null
+  }
+
+  const tableObject = JSON.parse(fieldData.answer as string)
+  return {
+    label: `Response ${fieldData.order}`,
+    order: fieldData.order ? (fieldData.order as number) + 0.1 : null,
+    type: 'table',
+    displayedValue: `Preview ${tableObject.rows.length} row(s)`,
+    value: tableObject,
+  }
+}
+
 function buildAnswerArrayMetadatum(
   fieldData: IJSONObject,
   stepId: string,
@@ -324,6 +344,12 @@ async function getDataOutMetadata(
       isHeader: { isHidden: true },
     }
     if (isAnswerArrayValid(fieldData)) {
+      // table field will have a stringified table object in the answer field
+      // so that it can be used in for-each
+      // this is also used to generate the table preview in the frontend
+      if (fieldData.fieldType === 'table') {
+        fieldMetadata[fieldId].answer = buildTableMetadatum(fieldData)
+      }
       fieldMetadata[fieldId].answerArray = buildAnswerArrayMetadatum(
         fieldData,
         executionStep.stepId,

packages/backend/src/apps/formsg/triggers/new-submission/get-mock-data.ts

@@ -6,6 +6,7 @@ import { customAlphabet } from 'nanoid/async'
 import { COMMON_S3_MOCK_FOLDER_PREFIX } from '@/helpers/s3'
 
 import { filterNric } from '../../auth/decrypt-form-response'
+import convertTableAnswerArrayToTableObject from '../../common/process-table-field'
 import { getFormDetailsFromGlobalVariable } from '../../common/webhook-settings'
 
 type FormField = {
@@ -144,6 +145,15 @@ async function getMockData($: IGlobalVariable) {
           data.responses[formFields[i]._id].answer = $.user.email
         }
 
+        // add a stringified version of the table data to the mock data
+        if (data.responses[formFields[i]._id].fieldType === 'table') {
+          const answerArray = data.responses[formFields[i]._id]
+            .answerArray as string[][]
+          const question = data.responses[formFields[i]._id].question
+          data.responses[formFields[i]._id].answer =
+            convertTableAnswerArrayToTableObject(question, answerArray)
+        }
+
         data.responses[formFields[i]._id].order = i + 1
         data.responses[formFields[i]._id].id = undefined
       }

packages/backend/src/apps/toolbox/__tests__/actions/for-each/index.test.ts

@@ -260,6 +260,68 @@ describe('For each action', () => {
       })
     })
 
+    it('should handle valid table input (FormSG Table field)', async () => {
+      const mockTableFieldData = JSON.stringify(validTableData)
+      $.step.parameters.items = mockTableFieldData
+
+      const processedResult = {
+        rows: validTableData.rows,
+        columns: validTableData.columns,
+        inputSource: FOR_EACH_INPUT_SOURCE.FORMSG_TABLE,
+      }
+
+      mockedProcessItems.mockReturnValue(processedResult)
+      const result = await action.run($)
+
+      expect(mockedProcessItems).toHaveBeenCalledWith(validTableData)
+      expect(mocks.setActionItem).toHaveBeenCalledWith({
+        raw: {
+          iterations: 2,
+          items: processedResult,
+          inputSource: FOR_EACH_INPUT_SOURCE.FORMSG_TABLE,
+        },
+      })
+      expect(result).toEqual({
+        nextStep: {
+          command: 'start-for-each',
+          stepId: 'for-each',
+        },
+      })
+    })
+
+    it('should handle FormSG Table field with no row data', async () => {
+      const mockTableFieldData = {
+        rows: [] as any[],
+        columns: validTableData.columns,
+      }
+      const stringifiedTableFieldData = JSON.stringify(mockTableFieldData)
+
+      $.step.parameters.items = stringifiedTableFieldData
+      const processedResult = {
+        rows: [] as any[],
+        columns: validTableData.columns,
+        inputSource: FOR_EACH_INPUT_SOURCE.FORMSG_TABLE,
+      }
+      mockedProcessItems.mockReturnValue(processedResult)
+      const result = await action.run($)
+
+      expect(mockedProcessItems).toHaveBeenCalledWith(mockTableFieldData)
+      expect(mocks.setActionItem).toHaveBeenCalledWith({
+        raw: {
+          iterations: 0,
+          items: processedResult,
+          inputSource: FOR_EACH_INPUT_SOURCE.FORMSG_TABLE,
+        },
+      })
+
+      expect(result).toEqual({
+        nextStep: {
+          command: 'stop-execution',
+          stepId: 'for-each',
+        },
+      })
+    })
+
     it('should handle table input with empty rows', async () => {
       const emptyTableData = {
         rows: [] as any[],