Skip to content

Commit b88c4ce

Browse files
kevinkim-ogpkevinkim-ogp
authored
chore: hex encode gathersg field names (#1322)
## Problem GatherSG field names could contain special characters that broke the step variables, e.g., `NRIC/FIN`. ## Solution Hex-encode the field names so that they can be shown as-is and still be used properly as variables. **What changed?** - Hex-encodes all GatherSG field names - Added `dataOutMetadata` for GatherSG trigger ## cgBefore & After Screenshots **BEFORE**: <img width="334" height="46" alt="Screenshot 2025-11-13 at 8 43 17 AM" src="https://github.com/user-attachments/assets/10890dee-9122-42d0-ad5a-05893b2d0b4c" /> **AFTER**: <img width="883" height="115" alt="Screenshot 2025-11-13 at 8 43 48 AM" src="https://github.com/user-attachments/assets/767eaf78-d896-4900-bbe1-216122b6c66d" /> ## Tests Create field(s) in GatherSG with special characters such as `/`, trigger an instant workflow that sends the case to Plumber - [ ] Verify that case fields are still displayed on the frontend with the original field names - [ ] Verify that case fields with special characters can be used as variables in subsequent steps and do not break
1 parent 716c787 commit b88c4ce

File tree

7 files changed

+268
-2
lines changed

7 files changed

+268
-2
lines changed

‎packages/backend/src/apps/gathersg/__tests__/auth/decrypt-response.test.ts‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
import { describe, expect, it } from 'vitest'
2+
3+
import { processFields } from '../../auth/decrypt-response'
4+
import { HEX_ENCODED_FIELD_PREFIX } from '../../common/constants'
5+
6+
describe('processFields', () => {
7+
it('should process fields with whitespace, -, _', () => {
8+
const fields = {
9+
name: 'John Doe',
10+
email: '[email protected]',
11+
'field-email': '[email protected]',
12+
field_email: '[email protected]',
13+
'field email': '[email protected]',
14+
}
15+
const processedFields = processFields(fields)
16+
expect(processedFields).toEqual(fields)
17+
})
18+
19+
it.each([
20+
'field / Email',
21+
'field.email',
22+
'field&email',
23+
'field%email',
24+
'field$email',
25+
'field@email',
26+
'field^email',
27+
'field!email',
28+
'field(email)',
29+
])('should hex encode fields with special characters', (field: string) => {
30+
const fields = {
31+
name: 'John Doe',
32+
[field]: '[email protected]',
33+
}
34+
const hexEncodedField = `${HEX_ENCODED_FIELD_PREFIX}${Buffer.from(
35+
field,
36+
).toString('hex')}`
37+
const processedFields = processFields(fields)
38+
expect(processedFields).toEqual({
39+
name: 'John Doe',
40+
[hexEncodedField]: '[email protected]',
41+
})
42+
})
43+
})

‎packages/backend/src/apps/gathersg/auth/decrypt-response.ts‎

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import crypto from 'crypto'
55
import appConfig from '@/config/app'
66
import logger from '@/helpers/logger'
77

8+
import { HEX_ENCODED_FIELD_PREFIX } from '../common/constants'
9+
810
import schema from './schema'
911

1012
function getInternalId(data: any) {
@@ -28,6 +30,23 @@ function validateData(data: any, flowId: string, app: string) {
2830
'GatherSG: potential infinite loop! Webhook not triggered by user!',
2931
)
3032
}
33+
return validationResult.data
34+
}
35+
36+
export function processFields(fields: Record<string, any>) {
37+
const processedFields: Record<string, any> = {}
38+
const invalidCharRegex = /[^a-zA-Z0-9-_ ]/
39+
for (const [key, value] of Object.entries(fields)) {
40+
if (invalidCharRegex.test(key)) {
41+
const hexKey = `${HEX_ENCODED_FIELD_PREFIX}${Buffer.from(key).toString(
42+
'hex',
43+
)}`
44+
processedFields[hexKey] = value
45+
} else {
46+
processedFields[key] = value
47+
}
48+
}
49+
return processedFields
3150
}
3251

3352
function verifySignature(signature: string, basestring: string) {
@@ -76,12 +95,16 @@ export async function decryptResponse(
7695
decipher.final(),
7796
]).toString()
7897
const decryptedData = JSON.parse(decryptedStr)
79-
8098
validateData(decryptedData, $.flow.id, app)
8199

100+
const processedFields = processFields(decryptedData.fields)
101+
82102
$.request.body = {
83103
app,
84-
data: decryptedData,
104+
data: {
105+
...decryptedData,
106+
fields: processedFields,
107+
},
85108
signature,
86109
timestamp,
87110
}
@@ -92,6 +115,16 @@ export async function decryptResponse(
92115
}
93116
} else {
94117
validateData(data, $.flow.id, app)
118+
const processedFields = processFields(data.fields)
119+
$.request.body = {
120+
app,
121+
data: {
122+
...data,
123+
fields: processedFields,
124+
},
125+
signature,
126+
timestamp,
127+
}
95128

96129
return {
97130
verified: true,

‎packages/backend/src/apps/gathersg/auth/schema.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ const schema = z
2525
submissionId: z.string().min(1),
2626
})
2727
.nullish(),
28+
fields: z.record(z.string(), z.any()).nullish(),
2829
})
2930
.refine(
3031
(data) => {

‎packages/backend/src/apps/gathersg/common/constants.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,6 @@ export const UNSUPPORTED_FIELDS = [
1111
'table', // array of objects
1212
'attachment',
1313
]
14+
15+
// Prefix for hex encoding field names that contain special characters
16+
export const HEX_ENCODED_FIELD_PREFIX = '__HEX_ENCODED__'

‎packages/backend/src/apps/gathersg/triggers/new-instant-workflow/get-data-out-metadata.ts‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
import { IDataOutMetadata, IExecutionStep, IJSONArray } from '@plumber/types'
2+
3+
import { HEX_ENCODED_FIELD_PREFIX } from '../../common/constants'
4+
5+
import { dataOutSchema } from './schema'
6+
7+
async function getDataOutMetadata(
8+
executionStep: IExecutionStep,
9+
): Promise<IDataOutMetadata> {
10+
const { dataOut: rawDataOut } = executionStep
11+
12+
if (!rawDataOut) {
13+
return null
14+
}
15+
16+
const parsedDataOut = dataOutSchema.safeParse(rawDataOut)
17+
if (parsedDataOut.success === false) {
18+
return null
19+
}
20+
21+
const { data: dataOut } = parsedDataOut.data
22+
23+
const caseMetadata: IDataOutMetadata = {
24+
app: { label: 'App' },
25+
signature: { isHidden: true },
26+
timestamp: { label: 'Timestamp' },
27+
}
28+
29+
// handle formsg field
30+
let formSgMetadata = Object.create(null)
31+
if (dataOut.formsg) {
32+
formSgMetadata.formId = { label: 'FormSG (form ID)' }
33+
formSgMetadata.submissionId = { label: 'FormSG (submission ID)' }
34+
} else {
35+
formSgMetadata = { isHidden: true }
36+
}
37+
38+
// handle createdBy field
39+
let createdByMetadata = Object.create(null)
40+
if (dataOut.createdBy) {
41+
createdByMetadata.email = { label: 'Created by (email)' }
42+
createdByMetadata.name = { label: 'Created by (name)' }
43+
} else {
44+
createdByMetadata = { isHidden: true }
45+
}
46+
47+
// handle updatedBy field
48+
let updatedByMetadata = Object.create(null)
49+
if (dataOut.updatedBy) {
50+
updatedByMetadata.email = { label: 'Updated by (email)' }
51+
updatedByMetadata.name = { label: 'Updated by (name)' }
52+
} else {
53+
updatedByMetadata = { isHidden: true }
54+
}
55+
56+
// handle finalisedBy field
57+
let finalisedByMetadata = Object.create(null)
58+
if (dataOut.finalisedBy) {
59+
finalisedByMetadata.email = { label: 'Finalised by (email)' }
60+
finalisedByMetadata.name = { label: 'Finalised by (name)' }
61+
} else {
62+
finalisedByMetadata = { isHidden: true }
63+
}
64+
65+
// handle hex-encoded field names from dataOut
66+
const fieldsMetadata = Object.create(null)
67+
if (dataOut.fields) {
68+
for (const key of Object.keys(dataOut.fields)) {
69+
try {
70+
// decode hex encoded field name to get the original field name
71+
let decodedLabel: string
72+
if (key.startsWith(HEX_ENCODED_FIELD_PREFIX)) {
73+
decodedLabel = Buffer.from(
74+
key.replace(HEX_ENCODED_FIELD_PREFIX, ''),
75+
'hex',
76+
).toString('utf-8')
77+
} else {
78+
decodedLabel = key
79+
}
80+
const fieldValue = dataOut.fields[decodedLabel]
81+
82+
// check if the value is an array
83+
if (Array.isArray(fieldValue)) {
84+
// check if it's an array of objects or an array of primitives
85+
if (
86+
fieldValue.length > 0 &&
87+
typeof fieldValue[0] === 'object' &&
88+
fieldValue[0] !== null
89+
) {
90+
// array of objects - create nested object structure for each row
91+
const array = fieldValue as IJSONArray
92+
const rowsMetadata = Object.create(null)
93+
94+
for (let i = 0; i < array.length; i++) {
95+
const rowObject = array[i]
96+
const rowMetadata = Object.create(null)
97+
98+
if (typeof rowObject === 'object' && rowObject !== null) {
99+
for (const nestedKey of Object.keys(rowObject)) {
100+
rowMetadata[nestedKey] = {
101+
type: 'text',
102+
label: `${decodedLabel} Row ${i + 1} ${nestedKey}`,
103+
}
104+
}
105+
}
106+
107+
rowsMetadata[i] = rowMetadata
108+
}
109+
110+
fieldsMetadata[key] = rowsMetadata
111+
} else {
112+
// array of primitives (strings, numbers, etc.) - treat as simple field
113+
fieldsMetadata[key] = { label: decodedLabel }
114+
}
115+
} else {
116+
// not an array - treat as simple field
117+
fieldsMetadata[key] = { label: decodedLabel }
118+
}
119+
} catch (error) {
120+
// if decoding fails, use the hex key as-is
121+
fieldsMetadata[key] = { label: key }
122+
}
123+
}
124+
}
125+
126+
return {
127+
...caseMetadata,
128+
data: {
129+
fields: fieldsMetadata,
130+
formsg: formSgMetadata,
131+
createdBy: createdByMetadata,
132+
finalisedBy: finalisedByMetadata,
133+
updatedBy: updatedByMetadata,
134+
135+
caseRef: { label: 'Case ref' },
136+
createdAt: { label: 'Created at' },
137+
email: { isHidden: true }, // hide to avoid confusing user in case there is an email field
138+
finalisedAt: { label: 'Finalised at' },
139+
source: { label: 'Source' },
140+
status: { label: 'Status' },
141+
type: { label: 'Case type' },
142+
updatedAt: { label: 'Updated at' },
143+
uuid: { label: 'UUID' },
144+
},
145+
}
146+
}
147+
148+
export default getDataOutMetadata

‎packages/backend/src/apps/gathersg/triggers/new-instant-workflow/index.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { type SafeParseError, type ZodIssue } from 'zod'
44

55
import StepError from '@/errors/step'
66

7+
import getDataOutMetadata from './get-data-out-metadata'
78
import { encryptionKeySchema } from './schema'
89

910
const trigger: IRawTrigger = {
@@ -29,6 +30,8 @@ const trigger: IRawTrigger = {
2930
},
3031
],
3132

33+
getDataOutMetadata,
34+
3235
async testRun($: IGlobalVariable) {
3336
const { encryptionKey } = $.step.parameters
3437
if (encryptionKey) {

‎packages/backend/src/apps/gathersg/triggers/new-instant-workflow/schema.ts‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,38 @@ export const encryptionKeySchema = z
1919
.refine((value) => value === value.trim(), {
2020
message: 'not have leading or trailing whitespace',
2121
})
22+
23+
export const dataOutSchema = z.object({
24+
app: z.string().min(1),
25+
signature: z.string().min(1),
26+
timestamp: z.number(),
27+
data: z
28+
.object({
29+
fields: z.record(z.string(), z.any()).nullish(),
30+
formsg: z
31+
.object({
32+
formId: z.string().min(1),
33+
submissionId: z.string().min(1),
34+
})
35+
.nullish(),
36+
updatedBy: z
37+
.object({
38+
email: z.string().min(1).nullish(),
39+
name: z.string().min(1),
40+
})
41+
.nullish(),
42+
createdBy: z
43+
.object({
44+
email: z.string().min(1).nullish(),
45+
name: z.string().min(1),
46+
})
47+
.nullish(),
48+
finalisedBy: z
49+
.object({
50+
email: z.string().min(1).nullish(),
51+
name: z.string().min(1),
52+
})
53+
.nullish(),
54+
})
55+
.nullish(),
56+
})

0 commit comments

Comments
 (0)