chore: hide formsg questions and re-label some fields

Author: pregnantboy

packages/backend/src/apps/formsg/auth/decrypt-form-response.ts

@@ -135,6 +135,9 @@ export async function decryptFormResponse(
         }
       }
 
+      // delete metadata from fields
+      delete rest['isHeader']
+
       if (rest.fieldType === 'nric' && !!rest.answer) {
         const filteredAnswer = filterNric($, rest.answer)
         if (!filteredAnswer) {

packages/backend/src/apps/formsg/triggers/new-submission/get-data-out-metadata.ts

@@ -17,6 +17,11 @@ function buildQuestionMetadatum(fieldData: IJSONObject): IDataOutMetadatum {
     type: 'text',
     label: fieldData.order ? `Question ${fieldData.order}` : null,
     order: fieldData.order ? (fieldData.order as number) : null,
+    isCollapsedByDefault: true,
+  }
+
+  if (fieldData.fieldType === 'section') {
+    question.label = 'Header'
   }
 
   if (fieldData.fieldType === 'attachment') {
@@ -28,7 +33,6 @@ function buildQuestionMetadatum(fieldData: IJSONObject): IDataOutMetadatum {
 
 function buildAnswerMetadatum(fieldData: IJSONObject): IDataOutMetadatum {
   const answer: IDataOutMetadatum = {
-    label: fieldData.order ? `Response ${fieldData.order}` : null,
     order: fieldData.order ? (fieldData.order as number) + 0.1 : null,
   }
 
@@ -47,9 +51,15 @@ function buildAnswerMetadatum(fieldData: IJSONObject): IDataOutMetadatum {
         parseS3Id(fieldData.answer as string)?.objectName ??
         (fieldData.answer as string)
       break
+    // Headers will only have empty answers
+    case 'section':
+      answer.isHidden = true
+      break
     default:
       answer['type'] = 'text'
-      answer['label'] = fieldData.order ? `Response ${fieldData.order}` : null
+      answer['label'] = fieldData.order
+        ? `${fieldData.order}. ${fieldData.question}`
+        : null
   }
 
   return answer
@@ -65,25 +75,58 @@ function isAnswerArrayValid(fieldData: IJSONObject): boolean {
 }
 
 function buildAnswerArrayForAddress(fieldData: IJSONObject): IDataOutMetadata {
-  const { order } = fieldData
+  const { order, question } = fieldData
 
   // NOTE: we return all labels as there are some optional fields in FormSG
+  // the label is shown before the question, since it will be truncated in the variable pill
   return ADDRESS_LABELS.map((label, index) => ({
     type: 'text',
-    label: `Response ${order}, ${label}`,
+    label: `${order}.${index + 1}. ${label} - ${question}`,
     order: order ? `${order}.${index + 1}` : null,
   }))
 }
 
 function buildAnswerArrayForCheckbox(
   fieldData: IJSONObject,
 ): IDataOutMetadatum {
+  const { order, question } = fieldData
   // NOTE: checkbox answerArray will not be further splitted,
   // handled specifically in variables.ts and compute-parameters.ts
   return {
     type: 'array',
-    label: `Response ${fieldData.order}`,
-    order: fieldData.order ? (fieldData.order as number) + 0.1 : null,
+    label: `${order}. ${question}`,
+    order: order ? (order as number) + 0.1 : null,
+  }
+}
+
+function extractLastTopLevelBracketContent(questionText: string): {
+  content: string
+  prefix: string
+} {
+  let depth = 0
+  let start = -1
+  let lastValid = ''
+  let prefix = ''
+
+  for (let i = 0; i < questionText.length; i++) {
+    if (questionText[i] === '(') {
+      if (depth === 0) {
+        start = i
+        prefix = questionText.slice(0, i).trim()
+      }
+      depth++
+    } else if (questionText[i] === ')') {
+      depth--
+      if (depth === 0 && start !== -1) {
+        lastValid = questionText.slice(start + 1, i)
+        start = -1 // reset
+      }
+    }
+  }
+
+  return {
+    content: lastValid,
+    prefix,
   }
 }
 
@@ -92,17 +135,33 @@ function buildAnswerArrayForTable(
 ): IDataOutMetadatum[][] {
   const answerArray = [] as IDataOutMetadatum[][]
   const array = fieldData.answerArray as IJSONArray
+  const { order, question } = fieldData
+  // questions are give in this format:
+  // Table Question (Column 1, Column 2, Column 3)
+  // step 1: find the match open bracket for the last pair of brackets
+  const { prefix: topLevelQuestion, content: columnNames } =
+    extractLastTopLevelBracketContent(question as string)
+  const columnNamesArray = columnNames.split(',').map((name) => name.trim())
+
   for (let i = 0; i < array.length; i++) {
     const option = array[i]
     const nestedAnswerArray = [] as IDataOutMetadatum[]
     const optionArray = option as IJSONArray
     for (let j = 0; j < optionArray.length; j++) {
+      // If the column names contain commas, we can't simply split by comma
+      // so we gotta just display the entire question name
+      // Also, making an API call to forms to retrieve the column names takes too long
+      // and won't work if the form is closed
+      let label = `${order}. ${question} Row ${i + 1} Col ${j + 1}`
+      if (columnNamesArray.length === optionArray.length) {
+        label = `${order}. Row ${i + 1} ${
+          columnNamesArray[j]
+        } - ${topLevelQuestion}`
+      }
       nestedAnswerArray.push({
         type: 'text',
-        label: fieldData.order
-          ? `Response ${fieldData.order}, Row ${i + 1} Column ${j + 1}`
-          : null,
-        order: fieldData.order ? (fieldData.order as number) : null,
+        label,
+        order: order ? (order as number) + 0.1 : null,
       })
     }
     answerArray.push(nestedAnswerArray)
@@ -215,14 +274,54 @@ async function getDataOutMetadata(
   }
 
   const fieldMetadata: IDataOutMetadata = Object.create(null)
-  for (const [fieldId, fieldData] of Object.entries(data.fields)) {
+
+  const fields = Object.entries(data.fields).sort((a, b) => {
+    const orderA = a[1].order
+    const orderB = b[1].order
+    if (orderA === null) {
+      return 1
+    }
+    return orderA - orderB
+  })
+
+  /**
+   * This is a hack to match the question number to the form as closely as possible.
+   * In formsg, the headers are not numbered, so we need to exclude them from the question number.
+   * But we also need to keep track of the headers between questions, so we can order them correctly.
+   * The regenerated order will be like so:
+   * Example given form:
+   * Header 0.9991
+   * Header 0.9992
+   * Question 1
+   * Question 2
+   * Sub Heading 2.9991
+   * Question 3
+   * Header 3.9991
+   * Sub Heading 3.9992
+   * Question 4
+   * Sub Heading 4.9991
+   * Question 4
+   */
+  let questionOrder = 0
+  let headerOrderBetweenQuestions = 0
+  for (const [fieldId, fieldData] of fields) {
+    if (fieldData.fieldType !== 'section') {
+      // reset order between questions (altho not necessary)
+      headerOrderBetweenQuestions = 0
+      questionOrder++
+      fieldData.order = questionOrder
+    } else {
+      headerOrderBetweenQuestions++
+      fieldData.order = questionOrder + +`0.999${headerOrderBetweenQuestions}`
+    }
     fieldMetadata[fieldId] = {
       question: buildQuestionMetadatum(fieldData),
       answer: buildAnswerMetadatum(fieldData),
       fieldType: { isHidden: true },
       order: { isHidden: true },
       myInfo: { attr: { isHidden: true } },
       isVisible: { isHidden: true },
+      isHeader: { isHidden: true },
     }
     if (isAnswerArrayValid(fieldData)) {
       fieldMetadata[fieldId].answerArray = buildAnswerArrayMetadatum(

packages/backend/src/apps/formsg/triggers/new-submission/get-mock-data.ts

@@ -118,6 +118,11 @@ async function getMockData($: IGlobalVariable) {
           }
         }
 
+        // formsg payload doesnt contain this anyways, so we dont return in mock data
+        if (data.responses[formFields[i]._id].fieldType === 'statement') {
+          delete data.responses[formFields[i]._id]
+        }
+
         if (data.responses[formFields[i]._id].fieldType === 'address') {
           data.responses[formFields[i]._id].answerArray =
             generateMockAddressData()

packages/frontend/src/components/RichTextEditor/VariableBadge.tsx

@@ -38,13 +38,14 @@ export const VariableBadge = ({ node }: { node: Node }) => {
         >
           <Text
             isTruncated
+            maxW="20ch"
             color="base.content.strong"
             mr={isEmpty ? undefined : '0.25rem'}
           >
             {node.attrs.label}
           </Text>
           {!isEmpty && (
-            <Text isTruncated maxW="50ch" color="base.content.medium">
+            <Text isTruncated maxW="40ch" color="base.content.medium">
               {value}
             </Text>
           )}

packages/frontend/src/components/VariablesList/index.tsx

@@ -2,6 +2,11 @@ import { TDataOutMetadatumType } from '@plumber/types'
 
 import { useMemo } from 'react'
 import {
+  Accordion,
+  AccordionButton,
+  AccordionIcon,
+  AccordionItem,
+  AccordionPanel,
   Box,
   type SystemStyleObject,
   Tag,
@@ -126,6 +131,24 @@ interface VariablesListProps {
 export default function VariablesList(props: VariablesListProps) {
   const { variables, onClick } = props
 
+  // Separate variables into default and collapsed categories
+  const { defaultVariables, collapsedVariables } = useMemo(() => {
+    if (!variables) {
+      return { defaultVariables: [], collapsedVariables: [] }
+    }
+
+    const defaultVariables: Variable[] = []
+    const collapsedVariables: Variable[] = []
+    for (const variable of variables) {
+      if (variable.isCollapsedByDefault) {
+        collapsedVariables.push(variable)
+      } else {
+        defaultVariables.push(variable)
+      }
+    }
+    return { defaultVariables, collapsedVariables }
+  }, [variables])
+
   if (!variables || variables.length === 0) {
     return <></>
   }
@@ -138,14 +161,42 @@ export default function VariablesList(props: VariablesListProps) {
       p={onClick ? undefined : '1rem'}
       sx={props.customStyles}
     >
-      {variables.map((variable, index) => (
+      {defaultVariables.map((variable, index) => (
         <VariableItem
           key={`variable-${variable.name}-${index}`}
           variable={variable}
           onClick={onClick}
-          isLast={index === variables.length - 1}
+          isLast={index === defaultVariables.length - 1}
         />
       ))}
+      {collapsedVariables.length > 0 && (
+        <Accordion allowMultiple border="transparent" py={2}>
+          <AccordionItem p={0}>
+            {({ isExpanded }) => (
+              <>
+                <AccordionPanel p={0} borderTop="1px solid #EDEDED">
+                  {collapsedVariables.map((variable, index) => (
+                    <VariableItem
+                      key={`variable-${variable.name}-${index}`}
+                      variable={variable}
+                      onClick={onClick}
+                      isLast={index === collapsedVariables.length - 1}
+                    />
+                  ))}
+                </AccordionPanel>
+                <AccordionButton
+                  flex={1}
+                  justifyContent="center"
+                  borderRadius="md"
+                >
+                  <Box as="span">{isExpanded ? 'Show less' : 'Show more'}</Box>
+                  <AccordionIcon />
+                </AccordionButton>
+              </>
+            )}
+          </AccordionItem>
+        </Accordion>
+      )}
     </Box>
   )
 }

packages/frontend/src/helpers/variables.ts

@@ -25,6 +25,7 @@ export interface Variable {
   type: TDataOutMetadatumType | null
   order: number | null
   displayedValue: string | null
+  isCollapsedByDefault: boolean
   /**
    * CAVEAT: not _just_ a name; it contains the lodash.get path for dataOut. Do
    * not clobber unless you know what you're doing!
@@ -69,6 +70,7 @@ const process = (
     type = null,
     order = null,
     displayedValue = null,
+    isCollapsedByDefault = false,
   } = metadata
 
   if (isHidden) {
@@ -84,6 +86,7 @@ const process = (
         displayedValue,
         type,
         order,
+        isCollapsedByDefault,
       },
     ]
   }
@@ -106,6 +109,7 @@ const process = (
             displayedValue,
             type,
             order,
+            isCollapsedByDefault,
           },
         ]
       : data.flatMap((item, index) => {

packages/types/index.d.ts

@@ -93,6 +93,12 @@ export interface IDataOutMetadatum {
    * variables with the same `order` or undefined `order` are sorted.
    */
   order?: number
+
+  /**
+   * Indicates whether this field should only be shown when the user expands
+   * the "Show more" section in the variable list.
+   */
+  isCollapsedByDefault?: boolean
 }
 
 export interface IDataOutMetadata {