[AI-BUILDER-15] PLU-605: chat feedback

packages/backend/src/graphql/mutation-resolvers.ts

@@ -1,6 +1,5 @@
 import type { MutationResolvers } from './__generated__/types.generated'
-import generateAiSteps from './mutations/ai/generate-ai-steps'
-import refineFormInput from './mutations/ai/refine-form-input'
+import aiMutationResolvers from './mutations/ai'
 import bulkRetryExecutions from './mutations/bulk-retry-executions'
 import bulkRetryIterations from './mutations/bulk-retry-iterations'
 import createConnection from './mutations/create-connection'
@@ -61,8 +60,6 @@ export default {
   bulkRetryIterations,
   createConnection,
   generateAuthUrl,
-  generateAiSteps,
-  refineFormInput,
   updateConnection,
   resetConnection,
   verifyConnection,
@@ -97,6 +94,7 @@ export default {
   deleteUploadedFile,
   generatePresignedPost,
   ...tilesMutationResolvers,
+  ...aiMutationResolvers,
 
   // This is a special stub that enables us to group all our admin-related
   // mutations into a special AdminMutation object; each "mutation" is handled by field

packages/backend/src/graphql/mutations/ai/index.ts

@@ -0,0 +1,11 @@
+import type { MutationResolvers } from '../../__generated__/types.generated'
+
+import generateAiSteps from './generate-ai-steps'
+import refineFormInput from './refine-form-input'
+import updateChatFeedback from './update-chat-feedback'
+
+export default {
+  generateAiSteps,
+  refineFormInput,
+  updateChatFeedback,
+} satisfies MutationResolvers

packages/backend/src/graphql/mutations/ai/update-chat-feedback.ts

@@ -0,0 +1,35 @@
+import appConfig from '@/config/app'
+import { langfuseClient } from '@/helpers/langfuse'
+import logger from '@/helpers/logger'
+
+import { MutationResolvers } from '../../__generated__/types.generated'
+
+const updateChatFeedback: MutationResolvers['updateChatFeedback'] = async (
+  _parent,
+  params,
+  context,
+) => {
+  const { traceId, feedback, score } = params.input
+
+  try {
+    langfuseClient.score.create({
+      traceId,
+      id: `feedback-${traceId}-${context.currentUser.email}`,
+      environment: appConfig.appEnv,
+      name: 'user-feedback',
+      value: score, // 1 for positive, 0 for negative
+      comment: feedback.comment as string,
+      ...(feedback?.category && {
+        metadata: { category: feedback.category },
+      }),
+    })
+
+    return true
+  } catch (error) {
+    logger.error('Failed to update chat feedback', { error })
+    // we don't throw because it doesn't stop the user from using the chat
+    return false
+  }
+}
+
+export default updateChatFeedback

packages/backend/src/graphql/schema.graphql

@@ -134,6 +134,7 @@ type Mutation {
   # Ai builder mutations
   generateAiSteps(input: GenerateAiStepsInput!): JSONObject
   refineFormInput(input: RefineFormInputInput!): JSONObject
+  updateChatFeedback(input: UpdateChatFeedbackInput!): Boolean!
   # Admin mutations
   admin: AdminMutation!
 }
@@ -149,6 +150,12 @@ input RefineFormInputInput {
   sessionId: String
 }
 
+input UpdateChatFeedbackInput {
+  traceId: String!
+  feedback: JSONObject
+  score: Int!
+}
+
 type StepInput {
   type: StepEnumType!
   appKey: String!

packages/frontend/src/graphql/mutations/ai/update-chat-feedback.ts

@@ -0,0 +1,7 @@
+import { gql } from '@apollo/client'
+
+export const UPDATE_CHAT_FEEDBACK = gql`
+  mutation updateChatFeedback($input: UpdateChatFeedbackInput!) {
+    updateChatFeedback(input: $input)
+  }
+`

packages/frontend/src/pages/AiBuilder/components/ChatMessages/ChatMessage.tsx

@@ -18,7 +18,7 @@ const AiMessage = ({ message }: ChatMessageProps) => {
         <ChakraStreamdown isAnimating={false}>
           {message.text || ''}
         </ChakraStreamdown>
-        <ChatMessageToolbar />
+        <ChatMessageToolbar traceId={message.traceId || ''} />
       </Box>
     </Flex>
   )

packages/frontend/src/pages/AiBuilder/components/ChatMessages/ChatMessageToolbar/FeedbackButton.tsx

@@ -0,0 +1,144 @@
+import { FaRegThumbsDown, FaRegThumbsUp } from 'react-icons/fa'
+import { useMutation } from '@apollo/client'
+import {
+  Button,
+  ButtonGroup,
+  FocusLock,
+  FormControl,
+  Icon,
+  IconButton,
+  Popover,
+  PopoverArrow,
+  PopoverBody,
+  PopoverContent,
+  PopoverTrigger,
+  Stack,
+  useDisclosure,
+} from '@chakra-ui/react'
+import { useToast } from '@opengovsg/design-system-react'
+
+import Form from '@/components/Form'
+import { UPDATE_CHAT_FEEDBACK } from '@/graphql/mutations/ai/update-chat-feedback'
+
+import { FEEDBACK_POPOVER_DETAILS } from './constants'
+import FeedbackFormContent from './FeedbackFormContent'
+
+interface FeedbackFormData {
+  'feedback-dropdown'?: string
+  'feedback-details': string
+}
+
+interface FeedbackButtonProps {
+  feedbackType: 'positive' | 'negative'
+  traceId: string
+}
+
+export const FeedbackButton = ({
+  feedbackType,
+  traceId,
+}: FeedbackButtonProps) => {
+  const { onOpen, onClose, isOpen } = useDisclosure()
+  const toast = useToast()
+  const icon = feedbackType === 'positive' ? FaRegThumbsUp : FaRegThumbsDown
+  const [updateChatFeedback] = useMutation(UPDATE_CHAT_FEEDBACK)
+  const {
+    dropdownLabel,
+    dropdownOptions,
+    textAreaLabel,
+    textAreaPlaceholder,
+    score,
+  } = FEEDBACK_POPOVER_DETAILS[feedbackType]
+
+  const handleSubmitFeedback = async (data: FeedbackFormData) => {
+    try {
+      if (!traceId) {
+        return
+      }
+
+      // NOTE: we send feedback to the backend instead of using Langfuse directly
+      // as there are additional headers required to call Rome/Istanbul endpoints
+      // that should not be exposed to the frontend
+      await updateChatFeedback({
+        variables: {
+          input: {
+            traceId,
+            feedback: {
+              category: data['feedback-dropdown'],
+              comment: data['feedback-details'],
+            },
+            score,
+          },
+        },
+      })
+    } catch {
+      // don't throw error if feedback submission fails
+      // as it is not critical to the user experience
+    } finally {
+      // NOTE: do not reset form here
+      // so that user will see what they previously typed or submitted
+      // if they attempt to submit again
+      onClose()
+      toast({
+        title: "Thank you! We've sent your feedback to the Plumber team.",
+        status: 'success',
+        duration: 3000,
+        isClosable: true,
+        position: 'top',
+      })
+    }
+  }
+
+  return (
+    <Popover
+      isOpen={isOpen}
+      onOpen={onOpen}
+      onClose={onClose}
+      placement="top-start"
+    >
+      <PopoverTrigger>
+        <IconButton
+          variant="clear"
+          colorScheme="secondary"
+          aria-label="Thumbs down"
+          icon={<Icon as={icon} />}
+          onClick={onOpen}
+          // HACKFIX(kevinkim-ogp): prevent autofocus when new input is sent
+          tabIndex={-1}
+        />
+      </PopoverTrigger>
+      <PopoverContent>
+        <FocusLock persistentFocus={false}>
+          <PopoverArrow />
+
+          <PopoverBody p={4}>
+            <Form
+              onSubmit={(data) =>
+                handleSubmitFeedback(data as FeedbackFormData)
+              }
+            >
+              <Stack spacing={4}>
+                <FormControl>
+                  <FeedbackFormContent
+                    dropdownLabel={dropdownLabel}
+                    dropdownOptions={dropdownOptions}
+                    textAreaLabel={textAreaLabel}
+                    textAreaPlaceholder={textAreaPlaceholder}
+                    autoFocus={feedbackType === 'positive'}
+                  />
+                </FormControl>
+                <ButtonGroup display="flex" justifyContent="flex-end">
+                  <Button variant="clear" onClick={onClose} size="sm">
+                    Cancel
+                  </Button>
+                  <Button type="submit" size="sm">
+                    Submit
+                  </Button>
+                </ButtonGroup>
+              </Stack>
+            </Form>
+          </PopoverBody>
+        </FocusLock>
+      </PopoverContent>
+    </Popover>
+  )
+}

packages/frontend/src/pages/AiBuilder/components/ChatMessages/ChatMessageToolbar/FeedbackFormContent.tsx

@@ -0,0 +1,65 @@
+import { Controller, useFormContext } from 'react-hook-form'
+import { Flex, FormLabel, Textarea } from '@chakra-ui/react'
+
+import { SingleSelect } from '@/components/SingleSelect'
+
+interface FeedbackFormData {
+  'feedback-dropdown'?: string
+  'feedback-details': string
+}
+
+const FeedbackFormContent = ({
+  dropdownLabel,
+  dropdownOptions,
+  textAreaLabel,
+  textAreaPlaceholder,
+  autoFocus,
+}: {
+  dropdownLabel: string | null
+  dropdownOptions: string[] | null
+  textAreaLabel: string
+  textAreaPlaceholder: string
+  autoFocus?: boolean
+}) => {
+  const { control, register } = useFormContext<FeedbackFormData>()
+
+  return (
+    <Flex direction="column">
+      {dropdownLabel != null && dropdownOptions != null && (
+        <>
+          <FormLabel htmlFor="feedback-dropdown" mt={2}>
+            {dropdownLabel}
+          </FormLabel>
+          <Controller
+            name="feedback-dropdown"
+            control={control}
+            defaultValue=""
+            render={({ field }) => (
+              <SingleSelect
+                colorScheme="secondary"
+                name="feedback-dropdown"
+                items={dropdownOptions}
+                value={field.value ?? ''}
+                onChange={field.onChange}
+                isClearable={false}
+              />
+            )}
+          />
+        </>
+      )}
+      <FormLabel htmlFor="feedback-details" mt={2}>
+        {textAreaLabel}
+      </FormLabel>
+      <Textarea
+        id="feedback-details"
+        rows={3}
+        resize="none"
+        autoFocus={autoFocus}
+        placeholder={textAreaPlaceholder}
+        {...register('feedback-details')}
+      />
+    </Flex>
+  )
+}
+
+export default FeedbackFormContent

packages/frontend/src/pages/AiBuilder/components/ChatMessages/ChatMessageToolbar/constants.ts

@@ -0,0 +1,23 @@
+export const FEEDBACK_POPOVER_DETAILS = {
+  positive: {
+    dropdownLabel: null,
+    dropdownOptions: null,
+    textAreaLabel:
+      'Provide details on what was satisfying about this response:',
+    textAreaPlaceholder: 'What was satisfying about this response?',
+    score: 1,
+  },
+  negative: {
+    dropdownLabel: 'What type of issue do you wish to report?',
+    dropdownOptions: [
+      'Incorrect workflow generated',
+      'Incomplete response',
+      'UI bug',
+      "I don't understand the response",
+      'Other',
+    ],
+    textAreaLabel: 'Provide details on what was wrong with this response:',
+    textAreaPlaceholder: 'What was wrong with this response?',
+    score: 0,
+  },
+}

packages/frontend/src/pages/AiBuilder/components/ChatMessages/ChatMessageToolbar/index.tsx

@@ -0,0 +1,18 @@
+import { Flex } from '@chakra-ui/react'
+
+import { FeedbackButton } from './FeedbackButton'
+
+interface ChatMessageToolbarProps {
+  traceId: string
+}
+
+export default function ChatMessageToolbar({
+  traceId,
+}: ChatMessageToolbarProps) {
+  return (
+    <Flex gap={1} mt={2}>
+      <FeedbackButton feedbackType="negative" traceId={traceId} />
+      <FeedbackButton feedbackType="positive" traceId={traceId} />
+    </Flex>
+  )
+}