Skip to content

Commit 52b245e

Browse files
kevinkim-ogpkevinkim-ogp
committed
chore: use dropdown for file selection
1 parent c205ea8 commit 52b245e

File tree

11 files changed

+177
-134
lines changed

11 files changed

+177
-134
lines changed

‎packages/backend/src/apps/aisay/__tests__/common/schema.test.ts‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
import { assert, beforeEach, describe, it } from 'vitest'
2+
3+
import {
4+
generalisedModelSchema,
5+
specificModelSchema,
6+
} from '../../common/schema'
7+
8+
describe('AISAY schema', () => {
9+
let generalisedModelPayload: Record<string, unknown>
10+
let specificModelPayload: Record<string, unknown>
11+
12+
beforeEach(() => {
13+
generalisedModelPayload = {
14+
file: 's3:plumber-test-bucket:123456/789abc/plumber-logo.jpg',
15+
infoToExtract: [
16+
{
17+
infoToExtract: 'What is your name?',
18+
},
19+
{
20+
infoToExtract: 'What is your age?',
21+
},
22+
],
23+
}
24+
25+
specificModelPayload = {
26+
file: 's3:plumber-test-bucket:123456/789abc/plumber-logo.jpg',
27+
documentType: 'BANK_STATEMENT',
28+
}
29+
})
30+
31+
describe('generalised model schema', () => {
32+
it('should validate generalised model schema', () => {
33+
const result = generalisedModelSchema.safeParse(generalisedModelPayload)
34+
assert(result.success === true)
35+
assert(
36+
result.data.file ===
37+
's3:plumber-test-bucket:123456/789abc/plumber-logo.jpg',
38+
)
39+
assert(Object.keys(result.data.infoToExtract).length === 2)
40+
assert(
41+
result.data.infoToExtract['additionalProp0'].description ===
42+
'Extract the What is your name?',
43+
)
44+
assert(
45+
result.data.infoToExtract['additionalProp1'].description ===
46+
'Extract the What is your age?',
47+
)
48+
})
49+
50+
it('should fail if file is not a valid S3 ID', () => {
51+
generalisedModelPayload.file = '123'
52+
const result = generalisedModelSchema.safeParse(generalisedModelPayload)
53+
assert(result.success === false)
54+
})
55+
})
56+
57+
describe('specific model schema', () => {
58+
it('should validate specific model schema', () => {
59+
const result = specificModelSchema.safeParse(specificModelPayload)
60+
assert(result.success === true)
61+
})
62+
63+
it.each(['BANK_STATEMENT', 'CHEQUE', 'INVOICE', 'PASSPORT', 'RECEIPT'])(
64+
`should validate for valid document type: %s`,
65+
(documentType) => {
66+
specificModelPayload.documentType = documentType
67+
const result = specificModelSchema.safeParse(specificModelPayload)
68+
assert(result.success === true)
69+
},
70+
)
71+
72+
it('should fail if document type is not a valid document type', () => {
73+
specificModelPayload.documentType = 'INVALID_DOCUMENT_TYPE'
74+
const result = specificModelSchema.safeParse(specificModelPayload)
75+
assert(result.success === false)
76+
})
77+
})
78+
})

‎packages/backend/src/apps/aisay/actions/use-generalised-model/index.ts‎

Lines changed: 13 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,16 @@
1-
import { IJSONArray, IRawAction } from '@plumber/types'
1+
import { IRawAction } from '@plumber/types'
22

33
import appConfig from '@/config/app'
44
import StepError from '@/errors/step'
5+
import logger from '@/helpers/logger'
56
import Step from '@/models/step'
67

78
import { getToken } from '../../auth/get-token'
89
import { parseError } from '../../common/error-parser'
9-
import { getAttachmentsFromS3, getValidationError } from '../../common/utils'
10+
import { generalisedModelSchema } from '../../common/schema'
11+
import { getAttachmentFromS3, getValidationError } from '../../common/utils'
1012

1113
import getDataOutMetadata from './get-data-out-metadata'
12-
import { requestSchema } from './schema'
1314

1415
const action: IRawAction = {
1516
name: 'Use generalised model',
@@ -19,8 +20,8 @@ const action: IRawAction = {
1920
arguments: [
2021
{
2122
label: 'File',
22-
key: 'attachments',
23-
type: 'multiselect' as const,
23+
key: 'file',
24+
type: 'dropdown',
2425
required: true,
2526
variables: true,
2627
variableTypes: ['file'],
@@ -44,16 +45,13 @@ const action: IRawAction = {
4445
},
4546
],
4647
doesFileProcessing: (step: Step) => {
47-
return (
48-
step.parameters.attachments &&
49-
(step.parameters.attachments as IJSONArray).length > 0
50-
)
48+
return step.parameters.file && step.parameters.file !== ''
5149
},
5250
getDataOutMetadata,
5351

5452
async run($) {
55-
const { attachments, infoToExtract } = $.step.parameters as {
56-
attachments?: IJSONArray
53+
const { file, infoToExtract } = $.step.parameters as {
54+
file: string
5755
infoToExtract: Array<{ infoToExtract: string }>
5856
}
5957

@@ -66,7 +64,7 @@ const action: IRawAction = {
6664
)
6765
}
6866

69-
const result = requestSchema.safeParse({ attachments, infoToExtract })
67+
const result = generalisedModelSchema.safeParse({ file, infoToExtract })
7068
if (!result.success) {
7169
const { stepErrorName, stepErrorSolution } = getValidationError(result)
7270

@@ -79,26 +77,12 @@ const action: IRawAction = {
7977
}
8078

8179
try {
82-
/**
83-
* FIXME (kevinkim-ogp): should only accept one attachment
84-
* use a different selector on the frontend or update the
85-
* multi-select to only allow one attachment
86-
*/
87-
// Pre-call get attachments from S3 first
88-
const attachmentFiles = await getAttachmentsFromS3(
89-
result.data.attachments,
90-
$.flow.id,
91-
)
92-
const attachment = attachmentFiles[0]
80+
// get attachment from S3 first
81+
const attachment = await getAttachmentFromS3(result.data.file, $.flow.id)
9382

9483
// Step 1: get AWS Cognito access token
9584
const token = await getToken($)
9685

97-
/**
98-
* TODO (kevinkim-ogp): first iteration of AISAY will only support synchronous calls
99-
* - add a check to ensure that the attachment is less than 9 MB (7 MB to be safe)
100-
* - add a check to ensure that the call to the model is less than 29 seconds
101-
*/
10286
// Step 2: Call the model to get the output
10387
const res = await $.http.request({
10488
url: `${appConfig.aisayApiUrl}/query`,
@@ -116,7 +100,7 @@ const action: IRawAction = {
116100

117101
$.setActionItem({ raw: { ...res.data } })
118102
} catch (err) {
119-
console.error(err)
103+
logger.error(err)
120104
const { stepErrorName, stepErrorSolution } = parseError(err)
121105
throw new StepError(
122106
stepErrorName,

‎packages/backend/src/apps/aisay/actions/use-generalised-model/schema.ts‎

Lines changed: 0 additions & 36 deletions
This file was deleted.

‎packages/backend/src/apps/aisay/actions/use-specific-model/index.ts‎

Lines changed: 13 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,16 @@
1-
import { IJSONArray, IRawAction } from '@plumber/types'
1+
import { IRawAction } from '@plumber/types'
22

33
import appConfig from '@/config/app'
44
import StepError from '@/errors/step'
5+
import logger from '@/helpers/logger'
56
import Step from '@/models/step'
67

78
import { getToken } from '../../auth/get-token'
89
import { parseError } from '../../common/error-parser'
9-
import { getAttachmentsFromS3, getValidationError } from '../../common/utils'
10+
import { specificModelSchema } from '../../common/schema'
11+
import { getAttachmentFromS3, getValidationError } from '../../common/utils'
1012

1113
import getDataOutMetadata from './get-data-out-metadata'
12-
import { requestSchema } from './schema'
1314

1415
const action: IRawAction = {
1516
name: 'Use specific model',
@@ -19,8 +20,8 @@ const action: IRawAction = {
1920
arguments: [
2021
{
2122
label: 'File',
22-
key: 'attachments',
23-
type: 'multiselect' as const,
23+
key: 'file',
24+
type: 'dropdown',
2425
required: true,
2526
variables: true,
2627
variableTypes: ['file'],
@@ -41,16 +42,13 @@ const action: IRawAction = {
4142
},
4243
],
4344
doesFileProcessing: (step: Step) => {
44-
return (
45-
step.parameters.attachments &&
46-
(step.parameters.attachments as IJSONArray).length > 0
47-
)
45+
return step.parameters.file && step.parameters.file !== ''
4846
},
4947
getDataOutMetadata,
5048

5149
async run($) {
52-
const { attachments, documentType } = $.step.parameters as {
53-
attachments?: IJSONArray
50+
const { file, documentType } = $.step.parameters as {
51+
file: string
5452
documentType: string
5553
}
5654

@@ -63,7 +61,7 @@ const action: IRawAction = {
6361
)
6462
}
6563

66-
const result = requestSchema.safeParse({ attachments, documentType })
64+
const result = specificModelSchema.safeParse({ file, documentType })
6765
if (!result.success) {
6866
const { stepErrorName, stepErrorSolution } = getValidationError(result)
6967

@@ -76,19 +74,12 @@ const action: IRawAction = {
7674
}
7775

7876
try {
79-
// Pre-call get attachments from S3 first
80-
const attachmentFiles = await getAttachmentsFromS3(
81-
result.data.attachments,
82-
$.flow.id,
83-
)
84-
const attachment = attachmentFiles[0]
77+
// get attachment from S3 first
78+
const attachment = await getAttachmentFromS3(result.data.file, $.flow.id)
8579

8680
// Step 1: get AWS Cognito access token
8781
const token = await getToken($)
8882

89-
// Assuming we do a synchronous call to the model
90-
// which needs to be less than 29 seconds
91-
// and with a document size of less than 9 MB
9283
// Step 2: Call the model to get the output
9384
const aisayRes = await $.http.request({
9485
url: `${appConfig.aisayApiUrl}/query`,
@@ -108,7 +99,7 @@ const action: IRawAction = {
10899
raw: { ...aisayRes.data, documentType: result.data.documentType },
109100
})
110101
} catch (err) {
111-
console.error(err)
102+
logger.error(err)
112103
const { stepErrorName, stepErrorSolution } = parseError(err)
113104

114105
throw new StepError(

‎packages/backend/src/apps/aisay/actions/use-specific-model/schema.ts‎

Lines changed: 0 additions & 25 deletions
This file was deleted.

‎packages/backend/src/apps/aisay/common/constants.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,8 @@
11
export const MAX_FILE_SIZE = 6 * 1024 * 1024 // 6 MB
2+
export const DOCUMENT_TYPES = [
3+
'BANK_STATEMENT',
4+
'CHEQUE',
5+
'INVOICE',
6+
'PASSPORT',
7+
'RECEIPT',
8+
]

‎packages/backend/src/apps/aisay/common/schema.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
import { z } from 'zod'
2+
3+
import { parseS3Id } from '@/helpers/s3'
4+
5+
import { DOCUMENT_TYPES } from './constants'
6+
7+
export const fileSchema = z.string().transform((value, context) => {
8+
if (!parseS3Id(value)) {
9+
context.addIssue({
10+
code: z.ZodIssueCode.custom,
11+
message: `${value} is not a S3 ID.`,
12+
})
13+
return z.NEVER
14+
}
15+
return value
16+
})
17+
18+
export const generalisedModelSchema = z.object({
19+
file: fileSchema,
20+
infoToExtract: z
21+
.array(z.object({ infoToExtract: z.string() }))
22+
.transform((array) => {
23+
const result: Record<string, { description: string; type: string }> = {}
24+
array.forEach((a, index) => {
25+
result[`additionalProp${index}`] = {
26+
description: `Extract the ${a.infoToExtract}`,
27+
type: 'string',
28+
}
29+
})
30+
return result
31+
}),
32+
})
33+
34+
export const documentTypeEnum = z.enum(DOCUMENT_TYPES as [string, ...string[]])
35+
36+
export const specificModelSchema = z.object({
37+
file: fileSchema,
38+
documentType: documentTypeEnum,
39+
})

0 commit comments

Comments
 (0)