Skip to content

Commit e0d44a2

Browse files
kevinkim-ogpkevinkim-ogp
authored
PLU-345: FormSG table field (#1069)
*NOTE: this is not a blocker for beta; intent is to release this while beta is happening* ### TL;DR This PR adds support for FormSG table fields in the For-Each action, enabling users to iterate through table data from FormSG submissions. ### What changed? - Added functionality to convert FormSG table field into `table` object to enable preview of row data and to use in for-each actions - Added a new input source type `FORMSG_TABLE` to the For-Each action - Updated the For-Each action to handle FormSG table fields, allowing iteration through table rows - Added test cases for the new table field functionality - Hide form cell values if there are no rows ### How to test? 1. Create a FormSG form with a table field 2. Set up a workflow that uses the FormSG trigger - [ ] Mock data - [ ] Should still show the individual cell values as "Response X, Row 1 Column 1" etc - [ ] Should have preview in test result - [ ] Modal should show all columns in the table - [ ] Modal should show 2 rows of mock data with each column filled - [ ] Actual submission - [ ] Should show the individual cell values with column headers if headers do not contain commas, otherwise resort to default (Column 1, Column 2, etc.) - [ ] Should have preview in test result - [ ] Preview should column headers if headers do not contain commas, otherwise default column names - [ ] Modal should show all columns in the table - [ ] Modal should show correct rows of mock data with each column filled - [ ] Actual submission with no data - [ ] Should NOT show individual cell values - [ ] Should NOT be able to preview ### Why make this change? FormSG table fields contain structured data that users often need to process row by row. This change enables users to iterate through table rows in a FormSG submission, making it possible to perform actions on each row of data separately. ### Screenshots **FormSG table preview** (can ignore the form connection issue, happens on local dev) Preview with mock data ![image.png](https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/Wrwhm8Mmhv1Z2GwlSb7V/227e3811-3030-4ff6-90a1-5e1e17007e25.png) ![image.png](https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/Wrwhm8Mmhv1Z2GwlSb7V/809b1e6e-e972-4e28-96db-90ea44fa376f.png) ![image.png](https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/Wrwhm8Mmhv1Z2GwlSb7V/2c0b6360-809d-4a7b-9119-e95fdb1c4e47.png) **FormSG variable in for-each** ![image.png](https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/Wrwhm8Mmhv1Z2GwlSb7V/50f98fc2-1bf4-4aa9-bd46-9d3ab478d7f7.png)
1 parent ba6619c commit e0d44a2

File tree

15 files changed

+589
-46
lines changed

15 files changed

+589
-46
lines changed

‎packages/backend/src/apps/formsg/__tests__/auth/decrypt-form-response.test.ts‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ import { IGlobalVariable, IRequest } from '@plumber/types'
33
import { Settings as LuxonSettings } from 'luxon'
44
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
55

6+
import { FOR_EACH_INPUT_SOURCE } from '@/apps/toolbox/common/constants'
7+
68
import app from '../..'
79
import { decryptFormResponse } from '../../auth/decrypt-form-response'
810
import { NricFilter } from '../../triggers/new-submission'
@@ -724,6 +726,47 @@ describe('decrypt form response', () => {
724726
tableField: {
725727
fieldType: 'table',
726728
question: 'What are your hobbies and when do you do them?',
729+
answer: JSON.stringify({
730+
rows: [
731+
{
732+
data: {
733+
[Buffer.from('Col 1').toString('hex')]: 'reading',
734+
[Buffer.from('Col 2').toString('hex')]: 'night',
735+
},
736+
},
737+
{
738+
data: {
739+
[Buffer.from('Col 1').toString('hex')]: 'gaming',
740+
[Buffer.from('Col 2').toString('hex')]: 'weekend',
741+
},
742+
},
743+
{
744+
data: {
745+
[Buffer.from('Col 1').toString('hex')]: 'coding',
746+
[Buffer.from('Col 2').toString('hex')]: 'day',
747+
},
748+
},
749+
],
750+
columns: [
751+
{
752+
id: Buffer.from('Col 1').toString('hex'),
753+
label: 'Col 1',
754+
name: 'Col 1',
755+
value: `data.rows.*.data.${Buffer.from('Col 1').toString(
756+
'hex',
757+
)}`,
758+
},
759+
{
760+
id: Buffer.from('Col 2').toString('hex'),
761+
label: 'Col 2',
762+
name: 'Col 2',
763+
value: `data.rows.*.data.${Buffer.from('Col 2').toString(
764+
'hex',
765+
)}`,
766+
},
767+
],
768+
inputSource: FOR_EACH_INPUT_SOURCE.FORMSG_TABLE,
769+
}),
727770
answerArray: [
728771
['reading', 'night'],
729772
['gaming', 'weekend'],
@@ -781,6 +824,41 @@ describe('decrypt form response', () => {
781824
['reading', 'night'],
782825
['gaming', 'weekend'],
783826
],
827+
answer: JSON.stringify({
828+
rows: [
829+
{
830+
data: {
831+
[Buffer.from('Col 1').toString('hex')]: 'reading',
832+
[Buffer.from('Col 2').toString('hex')]: 'night',
833+
},
834+
},
835+
{
836+
data: {
837+
[Buffer.from('Col 1').toString('hex')]: 'gaming',
838+
[Buffer.from('Col 2').toString('hex')]: 'weekend',
839+
},
840+
},
841+
],
842+
columns: [
843+
{
844+
id: Buffer.from('Col 1').toString('hex'),
845+
label: 'Col 1',
846+
name: 'Col 1',
847+
value: `data.rows.*.data.${Buffer.from('Col 1').toString(
848+
'hex',
849+
)}`,
850+
},
851+
{
852+
id: Buffer.from('Col 2').toString('hex'),
853+
label: 'Col 2',
854+
name: 'Col 2',
855+
value: `data.rows.*.data.${Buffer.from('Col 2').toString(
856+
'hex',
857+
)}`,
858+
},
859+
],
860+
inputSource: FOR_EACH_INPUT_SOURCE.FORMSG_TABLE,
861+
}),
784862
order: 2,
785863
},
786864
},

‎packages/backend/src/apps/formsg/__tests__/triggers/new-submission.test.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -484,6 +484,40 @@ describe('new submission trigger for answer array fields', () => {
484484
['sleeping', 'night'],
485485
['eating', 'all day'],
486486
],
487+
answer: JSON.stringify({
488+
rows: [
489+
{
490+
data: {
491+
[Buffer.from('Column 1').toString('hex')]: 'sleeping',
492+
[Buffer.from('Column 2').toString('hex')]: 'night',
493+
},
494+
},
495+
{
496+
data: {
497+
[Buffer.from('Column 1').toString('hex')]: 'eating',
498+
[Buffer.from('Column 2').toString('hex')]: 'all day',
499+
},
500+
},
501+
],
502+
columns: [
503+
{
504+
id: Buffer.from('Column 1').toString('hex'),
505+
label: 'Column 1',
506+
name: 'Column 1',
507+
value: `data.rows.*.data.${Buffer.from('Column 1').toString(
508+
'hex',
509+
)}`,
510+
},
511+
{
512+
id: Buffer.from('Column 2').toString('hex'),
513+
label: 'Column 2',
514+
name: 'Column 2',
515+
value: `data.rows.*.data.${Buffer.from('Column 2').toString(
516+
'hex',
517+
)}`,
518+
},
519+
],
520+
}),
487521
},
488522
textFieldId3: {
489523
question: 'Unknown question',

‎packages/backend/src/apps/formsg/auth/decrypt-form-response.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import { sha256Hash } from '@/helpers/crypto'
1010
import logger from '@/helpers/logger'
1111

1212
import { getSdk, parseFormEnv } from '../common/form-env'
13+
import convertTableAnswerArrayToTableObject from '../common/process-table-field'
1314
import { NricFilter } from '../triggers/new-submission/index'
1415

1516
import storeAttachmentInS3 from './helpers/store-attachment-in-s3'
@@ -128,6 +129,11 @@ export async function decryptFormResponse(
128129
rest.answerArray = (rest.answerArray as string[][]).map((row) =>
129130
row.map((column) => column.replaceAll('\u0000', '')),
130131
)
132+
133+
rest.answer = convertTableAnswerArrayToTableObject(
134+
rest.question,
135+
rest.answerArray,
136+
)
131137
} else {
132138
rest.answerArray = (rest.answerArray as string[]).map((answer) =>
133139
answer.replaceAll('\u0000', ''),

‎packages/backend/src/apps/formsg/common/process-table-field.ts‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
import { FOR_EACH_INPUT_SOURCE } from '@/apps/toolbox/common/constants'
2+
3+
import { extractLastTopLevelBracketContent } from '../triggers/new-submission/get-data-out-metadata'
4+
5+
type TableColumn = {
6+
id: string
7+
label: string
8+
name: string
9+
value: string
10+
}
11+
12+
const createColumn = (label: string): TableColumn => {
13+
const id = Buffer.from(label).toString('hex')
14+
return {
15+
id,
16+
label,
17+
name: label,
18+
value: `data.rows.*.data.${id}`,
19+
}
20+
}
21+
22+
export default function convertTableAnswerArrayToTableObject(
23+
question: string,
24+
answerArray: string[][],
25+
) {
26+
const { content: columnNames } = extractLastTopLevelBracketContent(question)
27+
const columnNamesArray = columnNames.split(',').map((name) => name.trim())
28+
29+
const columns =
30+
// make sure that column names do not contain commas
31+
columnNamesArray.length === answerArray[0].length
32+
? columnNamesArray.map(createColumn)
33+
: answerArray[0].map((_, index) => createColumn(`Col ${index + 1}`))
34+
35+
/**
36+
* NOTE: we do not show table rows that do not have any data
37+
*/
38+
const rows = (answerArray as string[][])
39+
.filter((row) => !row.every((v) => v === ''))
40+
.map((row) => {
41+
const rowData: Record<string, string | number> = {}
42+
row.forEach((v: string, i: number) => {
43+
rowData[columns[i].id] = v.replaceAll('\u0000', '')
44+
})
45+
return { data: rowData }
46+
})
47+
48+
return JSON.stringify({
49+
rows,
50+
columns,
51+
inputSource: FOR_EACH_INPUT_SOURCE.FORMSG_TABLE,
52+
})
53+
}

‎packages/backend/src/apps/formsg/triggers/new-submission/get-data-out-metadata.ts‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ function buildAnswerArrayForCheckbox(
9999
}
100100
}
101101

102-
function extractLastTopLevelBracketContent(questionText: string): {
102+
export function extractLastTopLevelBracketContent(questionText: string): {
103103
content: string
104104
prefix: string
105105
} {
@@ -162,13 +162,39 @@ function buildAnswerArrayForTable(
162162
type: 'text',
163163
label,
164164
order: order ? (order as number) + 0.1 : null,
165+
// NOTE: we hide the option if it is empty
166+
// mock data will have dummy strings in the answerArray
167+
// actual submissions should contain real data, otherwise the cells should be hidden
168+
isHidden: optionArray[j] === '',
165169
})
166170
}
167171
answerArray.push(nestedAnswerArray)
168172
}
169173
return answerArray
170174
}
171175

176+
function buildTableMetadatum(fieldData: IJSONObject): IDataOutMetadata {
177+
// old execution steps may not have an answer field for their table
178+
if (!fieldData.answer) {
179+
return {
180+
label: '',
181+
order: null,
182+
type: 'table',
183+
displayedValue: '',
184+
value: {},
185+
}
186+
}
187+
188+
const tableObject = JSON.parse(fieldData.answer as string)
189+
return {
190+
label: `Response ${fieldData.order}`,
191+
order: fieldData.order ? (fieldData.order as number) + 0.1 : null,
192+
type: 'table',
193+
displayedValue: `Preview ${tableObject.rows.length} row(s)`,
194+
value: tableObject,
195+
}
196+
}
197+
172198
function buildAnswerArrayMetadatum(
173199
fieldData: IJSONObject,
174200
stepId: string,
@@ -324,6 +350,12 @@ async function getDataOutMetadata(
324350
isHeader: { isHidden: true },
325351
}
326352
if (isAnswerArrayValid(fieldData)) {
353+
// table field will have a stringified table object in the answer field
354+
// so that it can be used in for-each
355+
// this is also used to generate the table preview in the frontend
356+
if (fieldData.fieldType === 'table') {
357+
fieldMetadata[fieldId].answer = buildTableMetadatum(fieldData)
358+
}
327359
fieldMetadata[fieldId].answerArray = buildAnswerArrayMetadatum(
328360
fieldData,
329361
executionStep.stepId,

‎packages/backend/src/apps/formsg/triggers/new-submission/get-mock-data.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { customAlphabet } from 'nanoid/async'
66
import { COMMON_S3_MOCK_FOLDER_PREFIX } from '@/helpers/s3'
77

88
import { filterNric } from '../../auth/decrypt-form-response'
9+
import convertTableAnswerArrayToTableObject from '../../common/process-table-field'
910
import { getFormDetailsFromGlobalVariable } from '../../common/webhook-settings'
1011

1112
type FormField = {
@@ -144,6 +145,15 @@ async function getMockData($: IGlobalVariable) {
144145
data.responses[formFields[i]._id].answer = $.user.email
145146
}
146147

148+
// add a stringified version of the table data to the mock data
149+
if (data.responses[formFields[i]._id].fieldType === 'table') {
150+
const answerArray = data.responses[formFields[i]._id]
151+
.answerArray as string[][]
152+
const question = data.responses[formFields[i]._id].question
153+
data.responses[formFields[i]._id].answer =
154+
convertTableAnswerArrayToTableObject(question, answerArray)
155+
}
156+
147157
data.responses[formFields[i]._id].order = i + 1
148158
data.responses[formFields[i]._id].id = undefined
149159
}

‎packages/backend/src/apps/toolbox/__tests__/actions/for-each/index.test.ts‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -260,6 +260,68 @@ describe('For each action', () => {
260260
})
261261
})
262262

263+
it('should handle valid table input (FormSG Table field)', async () => {
264+
const mockTableFieldData = JSON.stringify(validTableData)
265+
$.step.parameters.items = mockTableFieldData
266+
267+
const processedResult = {
268+
rows: validTableData.rows,
269+
columns: validTableData.columns,
270+
inputSource: FOR_EACH_INPUT_SOURCE.FORMSG_TABLE,
271+
}
272+
273+
mockedProcessItems.mockReturnValue(processedResult)
274+
const result = await action.run($)
275+
276+
expect(mockedProcessItems).toHaveBeenCalledWith(validTableData)
277+
expect(mocks.setActionItem).toHaveBeenCalledWith({
278+
raw: {
279+
iterations: 2,
280+
items: processedResult,
281+
inputSource: FOR_EACH_INPUT_SOURCE.FORMSG_TABLE,
282+
},
283+
})
284+
expect(result).toEqual({
285+
nextStep: {
286+
command: 'start-for-each',
287+
stepId: 'for-each',
288+
},
289+
})
290+
})
291+
292+
it('should handle FormSG Table field with no row data', async () => {
293+
const mockTableFieldData = {
294+
rows: [] as any[],
295+
columns: validTableData.columns,
296+
}
297+
const stringifiedTableFieldData = JSON.stringify(mockTableFieldData)
298+
299+
$.step.parameters.items = stringifiedTableFieldData
300+
const processedResult = {
301+
rows: [] as any[],
302+
columns: validTableData.columns,
303+
inputSource: FOR_EACH_INPUT_SOURCE.FORMSG_TABLE,
304+
}
305+
mockedProcessItems.mockReturnValue(processedResult)
306+
const result = await action.run($)
307+
308+
expect(mockedProcessItems).toHaveBeenCalledWith(mockTableFieldData)
309+
expect(mocks.setActionItem).toHaveBeenCalledWith({
310+
raw: {
311+
iterations: 0,
312+
items: processedResult,
313+
inputSource: FOR_EACH_INPUT_SOURCE.FORMSG_TABLE,
314+
},
315+
})
316+
317+
expect(result).toEqual({
318+
nextStep: {
319+
command: 'stop-execution',
320+
stepId: 'for-each',
321+
},
322+
})
323+
})
324+
263325
it('should handle table input with empty rows', async () => {
264326
const emptyTableData = {
265327
rows: [] as any[],

0 commit comments

Comments
 (0)