Skip to content

Commit 2e1ec5e

Browse files
kevinkim-ogpkevinkim-ogp
committed
chore: use dropdown for file selection
1 parent d2735ef commit 2e1ec5e

File tree

11 files changed

+178
-135
lines changed

11 files changed

+178
-135
lines changed

‎packages/backend/src/apps/aisay/__tests__/common/schema.test.ts‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
import { assert, beforeEach, describe, it } from 'vitest'
2+
3+
import {
4+
generalisedModelSchema,
5+
specificModelSchema,
6+
} from '../../common/schema'
7+
8+
describe('AISAY schema', () => {
9+
let generalisedModelPayload: Record<string, unknown>
10+
let specificModelPayload: Record<string, unknown>
11+
12+
beforeEach(() => {
13+
generalisedModelPayload = {
14+
file: 's3:plumber-test-bucket:123456/789abc/plumber-logo.jpg',
15+
infoToExtract: [
16+
{
17+
infoToExtract: 'What is your name?',
18+
},
19+
{
20+
infoToExtract: 'What is your age?',
21+
},
22+
],
23+
}
24+
25+
specificModelPayload = {
26+
file: 's3:plumber-test-bucket:123456/789abc/plumber-logo.jpg',
27+
documentType: 'BANK_STATEMENT',
28+
}
29+
})
30+
31+
describe('generalised model schema', () => {
32+
it('should validate generalised model schema', () => {
33+
const result = generalisedModelSchema.safeParse(generalisedModelPayload)
34+
assert(result.success === true)
35+
assert(
36+
result.data.file ===
37+
's3:plumber-test-bucket:123456/789abc/plumber-logo.jpg',
38+
)
39+
assert(Object.keys(result.data.infoToExtract).length === 2)
40+
assert(
41+
result.data.infoToExtract['additionalProp0'].description ===
42+
'Extract the What is your name?',
43+
)
44+
assert(
45+
result.data.infoToExtract['additionalProp1'].description ===
46+
'Extract the What is your age?',
47+
)
48+
})
49+
50+
it('should fail if file is not a valid S3 ID', () => {
51+
generalisedModelPayload.file = '123'
52+
const result = generalisedModelSchema.safeParse(generalisedModelPayload)
53+
assert(result.success === false)
54+
})
55+
})
56+
57+
describe('specific model schema', () => {
58+
it('should validate specific model schema', () => {
59+
const result = specificModelSchema.safeParse(specificModelPayload)
60+
assert(result.success === true)
61+
})
62+
63+
it.each(['BANK_STATEMENT', 'CHEQUE', 'INVOICE', 'PASSPORT', 'RECEIPT'])(
64+
`should validate for valid document type: %s`,
65+
(documentType) => {
66+
specificModelPayload.documentType = documentType
67+
const result = specificModelSchema.safeParse(specificModelPayload)
68+
assert(result.success === true)
69+
},
70+
)
71+
72+
it('should fail if document type is not a valid document type', () => {
73+
specificModelPayload.documentType = 'INVALID_DOCUMENT_TYPE'
74+
const result = specificModelSchema.safeParse(specificModelPayload)
75+
assert(result.success === false)
76+
})
77+
})
78+
})

‎packages/backend/src/apps/aisay/actions/use-generalised-model/index.ts‎

Lines changed: 14 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,16 @@
1-
import { IJSONArray, IRawAction } from '@plumber/types'
1+
import { IRawAction } from '@plumber/types'
22

33
import appConfig from '@/config/app'
44
import StepError from '@/errors/step'
5+
import logger from '@/helpers/logger'
56
import Step from '@/models/step'
67

78
import { getToken } from '../../auth/get-token'
89
import { parseError } from '../../common/error-parser'
9-
import { getAttachmentsFromS3, getValidationError } from '../../common/utils'
10+
import { generalisedModelSchema } from '../../common/schema'
11+
import { getAttachmentFromS3, getValidationError } from '../../common/utils'
1012

1113
import getDataOutMetadata from './get-data-out-metadata'
12-
import { requestSchema } from './schema'
1314

1415
const action: IRawAction = {
1516
name: 'Use generalised model',
@@ -19,9 +20,9 @@ const action: IRawAction = {
1920
arguments: [
2021
{
2122
label: 'File',
22-
key: 'attachments',
23-
description: 'Use files from previous steps',
24-
type: 'multiselect' as const,
23+
key: 'file',
24+
description: 'Us files from previous steps',
25+
type: 'dropdown',
2526
required: true,
2627
variables: true,
2728
variableTypes: ['file'],
@@ -46,16 +47,13 @@ const action: IRawAction = {
4647
},
4748
],
4849
doesFileProcessing: (step: Step) => {
49-
return (
50-
step.parameters.attachments &&
51-
(step.parameters.attachments as IJSONArray).length > 0
52-
)
50+
return step.parameters.file && step.parameters.file !== ''
5351
},
5452
getDataOutMetadata,
5553

5654
async run($) {
57-
const { attachments, infoToExtract } = $.step.parameters as {
58-
attachments?: IJSONArray
55+
const { file, infoToExtract } = $.step.parameters as {
56+
file: string
5957
infoToExtract: Array<{ infoToExtract: string }>
6058
}
6159

@@ -68,7 +66,7 @@ const action: IRawAction = {
6866
)
6967
}
7068

71-
const result = requestSchema.safeParse({ attachments, infoToExtract })
69+
const result = generalisedModelSchema.safeParse({ file, infoToExtract })
7270
if (!result.success) {
7371
const { stepErrorName, stepErrorSolution } = getValidationError(result)
7472

@@ -81,26 +79,12 @@ const action: IRawAction = {
8179
}
8280

8381
try {
84-
/**
85-
* FIXME (kevinkim-ogp): should only accept one attachment
86-
* use a different selector on the frontend or update the
87-
* multi-select to only allow one attachment
88-
*/
89-
// Pre-call get attachments from S3 first
90-
const attachmentFiles = await getAttachmentsFromS3(
91-
result.data.attachments,
92-
$.flow.id,
93-
)
94-
const attachment = attachmentFiles[0]
82+
// get attachment from S3 first
83+
const attachment = await getAttachmentFromS3(result.data.file, $.flow.id)
9584

9685
// Step 1: get AWS Cognito access token
9786
const token = await getToken($)
9887

99-
/**
100-
* TODO (kevinkim-ogp): first iteration of AISAY will only support synchronous calls
101-
* - add a check to ensure that the attachment is less than 9 MB (7 MB to be safe)
102-
* - add a check to ensure that the call to the model is less than 29 seconds
103-
*/
10488
// Step 2: Call the model to get the output
10589
const res = await $.http.request({
10690
url: `${appConfig.aisayApiUrl}/query`,
@@ -118,7 +102,7 @@ const action: IRawAction = {
118102

119103
$.setActionItem({ raw: { ...res.data } })
120104
} catch (err) {
121-
console.error(err)
105+
logger.error(err)
122106
const { stepErrorName, stepErrorSolution } = parseError(err)
123107
throw new StepError(
124108
stepErrorName,

‎packages/backend/src/apps/aisay/actions/use-generalised-model/schema.ts‎

Lines changed: 0 additions & 36 deletions
This file was deleted.

‎packages/backend/src/apps/aisay/actions/use-specific-model/index.ts‎

Lines changed: 13 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,16 @@
1-
import { IJSONArray, IRawAction } from '@plumber/types'
1+
import { IRawAction } from '@plumber/types'
22

33
import appConfig from '@/config/app'
44
import StepError from '@/errors/step'
5+
import logger from '@/helpers/logger'
56
import Step from '@/models/step'
67

78
import { getToken } from '../../auth/get-token'
89
import { parseError } from '../../common/error-parser'
9-
import { getAttachmentsFromS3, getValidationError } from '../../common/utils'
10+
import { specificModelSchema } from '../../common/schema'
11+
import { getAttachmentFromS3, getValidationError } from '../../common/utils'
1012

1113
import getDataOutMetadata from './get-data-out-metadata'
12-
import { requestSchema } from './schema'
1314

1415
const action: IRawAction = {
1516
name: 'Use specific model',
@@ -19,9 +20,9 @@ const action: IRawAction = {
1920
arguments: [
2021
{
2122
label: 'File',
22-
key: 'attachments',
23+
key: 'file',
2324
description: 'Use file from previous steps',
24-
type: 'multiselect' as const,
25+
type: 'dropdown',
2526
required: true,
2627
variables: true,
2728
variableTypes: ['file'],
@@ -42,16 +43,13 @@ const action: IRawAction = {
4243
},
4344
],
4445
doesFileProcessing: (step: Step) => {
45-
return (
46-
step.parameters.attachments &&
47-
(step.parameters.attachments as IJSONArray).length > 0
48-
)
46+
return step.parameters.file && step.parameters.file !== ''
4947
},
5048
getDataOutMetadata,
5149

5250
async run($) {
53-
const { attachments, documentType } = $.step.parameters as {
54-
attachments?: IJSONArray
51+
const { file, documentType } = $.step.parameters as {
52+
file: string
5553
documentType: string
5654
}
5755

@@ -64,7 +62,7 @@ const action: IRawAction = {
6462
)
6563
}
6664

67-
const result = requestSchema.safeParse({ attachments, documentType })
65+
const result = specificModelSchema.safeParse({ file, documentType })
6866
if (!result.success) {
6967
const { stepErrorName, stepErrorSolution } = getValidationError(result)
7068

@@ -77,19 +75,12 @@ const action: IRawAction = {
7775
}
7876

7977
try {
80-
// Pre-call get attachments from S3 first
81-
const attachmentFiles = await getAttachmentsFromS3(
82-
result.data.attachments,
83-
$.flow.id,
84-
)
85-
const attachment = attachmentFiles[0]
78+
// get attachment from S3 first
79+
const attachment = await getAttachmentFromS3(result.data.file, $.flow.id)
8680

8781
// Step 1: get AWS Cognito access token
8882
const token = await getToken($)
8983

90-
// Assuming we do a synchronous call to the model
91-
// which needs to be less than 29 seconds
92-
// and with a document size of less than 9 MB
9384
// Step 2: Call the model to get the output
9485
const aisayRes = await $.http.request({
9586
url: `${appConfig.aisayApiUrl}/query`,
@@ -109,7 +100,7 @@ const action: IRawAction = {
109100
raw: { ...aisayRes.data, documentType: result.data.documentType },
110101
})
111102
} catch (err) {
112-
console.error(err)
103+
logger.error(err)
113104
const { stepErrorName, stepErrorSolution } = parseError(err)
114105

115106
throw new StepError(

‎packages/backend/src/apps/aisay/actions/use-specific-model/schema.ts‎

Lines changed: 0 additions & 25 deletions
This file was deleted.

‎packages/backend/src/apps/aisay/common/constants.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,8 @@
11
export const MAX_FILE_SIZE = 6 * 1024 * 1024 // 6 MB
2+
export const DOCUMENT_TYPES = [
3+
'BANK_STATEMENT',
4+
'CHEQUE',
5+
'INVOICE',
6+
'PASSPORT',
7+
'RECEIPT',
8+
]

‎packages/backend/src/apps/aisay/common/schema.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
import { z } from 'zod'
2+
3+
import { parseS3Id } from '@/helpers/s3'
4+
5+
import { DOCUMENT_TYPES } from './constants'
6+
7+
export const fileSchema = z.string().transform((value, context) => {
8+
if (!parseS3Id(value)) {
9+
context.addIssue({
10+
code: z.ZodIssueCode.custom,
11+
message: `${value} is not a S3 ID.`,
12+
})
13+
return z.NEVER
14+
}
15+
return value
16+
})
17+
18+
export const generalisedModelSchema = z.object({
19+
file: fileSchema,
20+
infoToExtract: z
21+
.array(z.object({ infoToExtract: z.string() }))
22+
.transform((array) => {
23+
const result: Record<string, { description: string; type: string }> = {}
24+
array.forEach((a, index) => {
25+
result[`additionalProp${index}`] = {
26+
description: `Extract the ${a.infoToExtract}`,
27+
type: 'string',
28+
}
29+
})
30+
return result
31+
}),
32+
})
33+
34+
export const documentTypeEnum = z.enum(DOCUMENT_TYPES as [string, ...string[]])
35+
36+
export const specificModelSchema = z.object({
37+
file: fileSchema,
38+
documentType: documentTypeEnum,
39+
})

0 commit comments

Comments
 (0)