PLU-451 - [AISAY-3] - Error handling and retry (#927)

kevinkim-ogp · web-flow · commit c2a2fa448382 · 2025-04-16T14:44:51.000+08:00
## TL;DR AISAY App Enhancements - Create AISAY per-connection ID queue - Add request error handler with retry logic for 429s and 503s - Refactor error handling with error parser - Fix checks for client ID and secret ## How to test? - [ ] Missing connection - Set up AISAY action without selecting a connection - Test step - Verify error message - [ ] AISAY queue - Login as admin - Verify that AISAY app queue was created - [ ] AISAY queue 2 - Publish a pipe with AISAY action - Submit from to start execution - Verify that AISAY actions were added to queue ## Before & After Screenshots ### Error message when no connection is selected ![Screenshot 2025-04-07 at 10.02.52 AM.png](https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/Wrwhm8Mmhv1Z2GwlSb7V/416201f0-bc41-446e-9858-1b2b5d99fa9d.png) ### AISAY specific queue ![Screenshot 2025-04-07 at 11.45.23 AM.png](https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/Wrwhm8Mmhv1Z2GwlSb7V/aa4dd6af-3812-412e-92c9-7ce550e459d2.png)
diff --git a/packages/backend/src/apps/aisay/__tests__/common/interceptors.error-handlers.test.ts b/packages/backend/src/apps/aisay/__tests__/common/interceptors.error-handlers.test.ts
@@ -0,0 +1,182 @@
+import '@/apps'
+
+import type { IGlobalVariable } from '@plumber/types'
+
+import {
+  AxiosError,
+  type AxiosPromise,
+  type AxiosResponse,
+  type InternalAxiosRequestConfig,
+} from 'axios'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+
+import HttpError from '@/errors/http'
+import RetriableError, { DEFAULT_DELAY_MS } from '@/errors/retriable-error'
+import createHttpClient, { type IHttpClient } from '@/helpers/http-client'
+
+import aisayApp from '../..'
+
+function mockAxiosAdapterToThrowOnce(
+  status: AxiosResponse['status'],
+  headers?: AxiosResponse['headers'],
+): void {
+  mocks.axiosAdapter.mockImplementationOnce((config) => {
+    throw new AxiosError(
+      'Request failed',
+      AxiosError.ERR_BAD_RESPONSE,
+      config,
+      null,
+      {
+        status,
+        headers,
+        config,
+      } as unknown as AxiosResponse,
+    )
+  })
+}
+
+const mocks = vi.hoisted(() => ({
+  axiosAdapter: vi.fn(
+    async (config: InternalAxiosRequestConfig): AxiosPromise => ({
+      data: 'test-data',
+      status: 200,
+      statusText: 'OK',
+      headers: {},
+      config,
+    }),
+  ),
+  logWarning: vi.fn(),
+  logError: vi.fn(),
+}))
+
+vi.mock('axios', async (importOriginal) => {
+  const actualAxios = await importOriginal<typeof import('axios')>()
+  const mockCreate: typeof actualAxios.default.create = (createConfig) =>
+    actualAxios.default.create({
+      ...createConfig,
+      adapter: mocks.axiosAdapter,
+    })
+
+  return {
+    ...actualAxios,
+    default: {
+      ...actualAxios.default,
+      create: mockCreate,
+    },
+  }
+})
+
+vi.mock('@/helpers/logger', () => ({
+  default: {
+    warn: mocks.logWarning,
+    error: mocks.logError,
+  },
+}))
+
+describe('AISAY request error handlers', () => {
+  let http: IHttpClient
+
+  beforeEach(() => {
+    const $ = {
+      auth: {
+        data: {
+          clientId: 'some-client-key',
+          clientSecret: 'some-client-secret',
+        },
+      },
+    } as unknown as IGlobalVariable
+    http = createHttpClient({
+      $,
+      baseURL: 'http://localhost/mock-aisay-api',
+      beforeRequest: [],
+      requestErrorHandler: aisayApp.requestErrorHandler,
+    })
+  })
+
+  afterEach(() => {
+    vi.restoreAllMocks()
+  })
+
+  it('logs an error and jams the entire queue on 429', async () => {
+    mockAxiosAdapterToThrowOnce(429, { 'retry-after': 123 })
+    await http
+      .get('/test-url')
+      .then(() => {
+        expect.unreachable()
+      })
+      .catch((error): void => {
+        expect(error).toBeInstanceOf(RetriableError)
+        expect(error.delayType).toEqual('queue')
+        expect(error.message).toEqual('Rate limited by AISAY.')
+      })
+    expect(mocks.logError).toHaveBeenCalledWith(
+      expect.stringContaining('HTTP 429'),
+      expect.objectContaining({ event: 'aisay-http-429' }),
+    )
+  })
+
+  it('logs a warning and throws a RetriableError with default step delay on 503', async () => {
+    mockAxiosAdapterToThrowOnce(503)
+    await http
+      .get('/test-url')
+      .then(() => {
+        expect.unreachable()
+      })
+      .catch((error): void => {
+        expect(error).toBeInstanceOf(RetriableError)
+        expect(error.delayType).toEqual('step')
+        expect(error.delayInMs).toEqual(DEFAULT_DELAY_MS)
+      })
+    expect(mocks.logWarning).toHaveBeenCalledWith(
+      expect.stringContaining(`HTTP 503`),
+      expect.objectContaining({ event: `aisay-http-503` }),
+    )
+  })
+
+  it('logs a warning and throws a RetriableError with step delay set to retry-after, on receiving 503', async () => {
+    mockAxiosAdapterToThrowOnce(503, { 'retry-after': 234 })
+    await http
+      .get('/test-url')
+      .then(() => {
+        expect.unreachable()
+      })
+      .catch((error): void => {
+        expect(error).toBeInstanceOf(RetriableError)
+        expect(error.delayType).toEqual('step')
+        expect(error.delayInMs).toEqual(234000)
+      })
+    expect(mocks.logWarning).toHaveBeenCalledWith(
+      expect.stringContaining(`HTTP 503`),
+      expect.objectContaining({ event: `aisay-http-503` }),
+    )
+  })
+
+  it('logs a warning and still throws a RetriableError with default step delay on 503, if response has invalid retry-after', async () => {
+    mockAxiosAdapterToThrowOnce(503, { 'retry-after': 'corrupted' })
+    await http
+      .get('/test-url')
+      .then(() => {
+        expect.unreachable()
+      })
+      .catch((error): void => {
+        expect(error).toBeInstanceOf(RetriableError)
+        expect(error.delayType).toEqual('step')
+        expect(error.delayInMs).toEqual(DEFAULT_DELAY_MS)
+      })
+    expect(mocks.logWarning).toHaveBeenCalledWith(
+      expect.stringContaining('HTTP 503'),
+      expect.objectContaining({ event: 'aisay-http-503' }),
+    )
+  })
+
+  it('throws HTTP error on other non-successful codes', async () => {
+    mockAxiosAdapterToThrowOnce(501, { 'retry-after': 123 })
+    await expect(http.get('/test-url')).rejects.toThrow(HttpError)
+  })
+
+  it('does not throw error if response is success', async () => {
+    await expect(http.get('/test-url')).resolves.toEqual(
+      expect.objectContaining({ data: 'test-data', status: 200 }),
+    )
+  })
+})
diff --git a/packages/backend/src/apps/aisay/__tests__/queue.test.ts b/packages/backend/src/apps/aisay/__tests__/queue.test.ts
@@ -0,0 +1,48 @@
+import '@/apps'
+
+import { afterEach, describe, expect, it, vi } from 'vitest'
+
+import aisayApp from '..'
+
+const mocks = vi.hoisted(() => ({
+  stepQueryResult: vi.fn(),
+}))
+
+vi.mock('@/models/step', () => ({
+  default: {
+    query: vi.fn(() => ({
+      findById: vi.fn(() => ({
+        throwIfNotFound: mocks.stepQueryResult,
+      })),
+    })),
+  },
+}))
+
+describe('AISAY Queue', () => {
+  afterEach(() => {
+    vi.restoreAllMocks()
+  })
+
+  it('sets group ID to the connection ID', async () => {
+    mocks.stepQueryResult.mockResolvedValueOnce({
+      connectionId: 'mock-connection-id',
+      key: 'useGeneralisedModel',
+      appKey: 'aisay',
+    })
+    const groupConfig = await aisayApp.queue.getGroupConfigForJob({
+      flowId: 'test-flow-id',
+      stepId: 'test-step-id',
+      executionId: 'test-step-id',
+    })
+    expect(groupConfig).toEqual({
+      id: 'mock-connection-id',
+    })
+  })
+
+  it('sets group concurrency to 1', () => {
+    expect(aisayApp.queue.groupLimits).toEqual({
+      type: 'concurrency',
+      concurrency: 1,
+    })
+  })
+})
diff --git a/packages/backend/src/apps/aisay/actions/use-generalised-model/get-data-out-metadata.ts b/packages/backend/src/apps/aisay/actions/use-generalised-model/get-data-out-metadata.ts
@@ -25,6 +25,7 @@ async function getDataOutMetadata(
   return {
     quota: {
       label: 'Quota',
+      isHidden: true,
     },
     fields: fieldsMetadata,
   }
diff --git a/packages/backend/src/apps/aisay/actions/use-generalised-model/index.ts b/packages/backend/src/apps/aisay/actions/use-generalised-model/index.ts
@@ -4,6 +4,7 @@ import StepError from '@/errors/step'
 import Step from '@/models/step'
 
 import { getToken } from '../../auth/get-token'
+import { parseError } from '../../common/error-parser'
 import { getAttachmentsFromS3, getValidationError } from '../../common/utils'
 
 import getDataOutMetadata from './get-data-out-metadata'
@@ -56,7 +57,7 @@ const action: IRawAction = {
       infoToExtract: Array<{ infoToExtract: string }>
     }
 
-    if (!$.auth.data.clientId || !$.auth.data.clientSecret) {
+    if (!$.auth.data?.clientId || !$.auth.data?.clientSecret) {
       throw new StepError(
         'Missing client ID or client secret',
         'Please check the client ID and client secret',
@@ -116,21 +117,13 @@ const action: IRawAction = {
       $.setActionItem({ raw: { ...res.data } })
     } catch (err) {
       console.error(err)
-      if (err.response.data.message === `Request Too Long`) {
-        throw new StepError(
-          'File too large',
-          'Please try again with a smaller file.',
-          $.step.position,
-          $.app.name,
-        )
-      } else {
-        throw new StepError(
-          'Failed to call generalised model',
-          'Please try again.',
-          $.step.position,
-          $.app.name,
-        )
-      }
+      const { stepErrorName, stepErrorSolution } = parseError(err)
+      throw new StepError(
+        stepErrorName,
+        stepErrorSolution,
+        $.step.position,
+        $.app.name,
+      )
     }
   },
 } satisfies IRawAction
diff --git a/packages/backend/src/apps/aisay/actions/use-specific-model/get-data-out-metadata.ts b/packages/backend/src/apps/aisay/actions/use-specific-model/get-data-out-metadata.ts
@@ -16,6 +16,7 @@ async function getDataOutMetadata(
   return {
     quota: {
       label: 'Quota',
+      isHidden: true,
     },
     documentType: {
       label: 'Document Type',
diff --git a/packages/backend/src/apps/aisay/actions/use-specific-model/index.ts b/packages/backend/src/apps/aisay/actions/use-specific-model/index.ts
@@ -54,7 +54,7 @@ const action: IRawAction = {
       documentType: string
     }
 
-    if (!$.auth.data.clientId || !$.auth.data.clientSecret) {
+    if (!$.auth.data?.clientId || !$.auth.data?.clientSecret) {
       throw new StepError(
         'Missing client ID or client secret',
         'Please check the client ID and client secret',
diff --git a/packages/backend/src/apps/aisay/common/constants.ts b/packages/backend/src/apps/aisay/common/constants.ts
@@ -0,0 +1 @@
+export const MAX_FILE_SIZE = 6 * 1024 * 1024 // 6 MB
diff --git a/packages/backend/src/apps/aisay/common/interceptors/request-error-handler.ts b/packages/backend/src/apps/aisay/common/interceptors/request-error-handler.ts
@@ -0,0 +1,67 @@
+import { IApp } from '@plumber/types'
+
+import RetriableError from '@/errors/retriable-error'
+import logger from '@/helpers/logger'
+import { parseRetryAfterToMs } from '@/helpers/parse-retry-after-to-ms'
+
+type ThrowingHandler = (
+  ...args: Parameters<IApp['requestErrorHandler']>
+) => never
+
+// handle rate limiting
+const handle429: ThrowingHandler = function ($, error) {
+  const retryAfterMs =
+    parseRetryAfterToMs(error.response?.headers?.['retry-after']) ?? 'default'
+
+  logger.error('Received HTTP 429 from AISAY', {
+    event: 'aisay-http-429',
+    clientId: $.auth.data.clientId,
+    baseUrl: error.response.config.baseURL,
+    url: error.response.config.url,
+    flowId: $.flow?.id,
+    stepId: $.step?.id,
+    executionId: $.execution?.id,
+    retryAfterMs: error.response?.headers?.['retry-after'],
+  })
+
+  throw new RetriableError({
+    error: 'Rate limited by AISAY.',
+    delayInMs: retryAfterMs,
+    delayType: 'queue',
+  })
+}
+
+// Retry failures
+const handle503: ThrowingHandler = function ($, error) {
+  const status = error.response.status
+  logger.warn('Received HTTP 503 from AISAY', {
+    event: 'aisay-http-503',
+    clientId: $.auth.data.clientId,
+    baseUrl: error.response.config.baseURL,
+    url: error.response.config.url,
+    flowId: $.flow?.id,
+    stepId: $.step?.id,
+    executionId: $.execution?.id,
+    retryAfterMs: error.response?.headers?.['retry-after'],
+  })
+
+  // AISAY will specify a Retry-After header when it returns 503.
+  const retryAfterMs =
+    parseRetryAfterToMs(error.response?.headers?.['retry-after']) ?? 'default'
+  throw new RetriableError({
+    error: `Encountered HTTP ${status} from AISAY`,
+    delayInMs: retryAfterMs,
+    delayType: 'step',
+  })
+}
+
+const errorHandler: IApp['requestErrorHandler'] = async function ($, error) {
+  switch (error.response.status) {
+    case 429:
+      return handle429($, error)
+    case 503:
+      return handle503($, error)
+  }
+}
+
+export default errorHandler
diff --git a/packages/backend/src/apps/aisay/index.ts b/packages/backend/src/apps/aisay/index.ts
@@ -1,8 +1,10 @@
 import { IApp } from '@plumber/types'
 
+import requestErrorHandler from './common/interceptors/request-error-handler'
 import { aisayUrlConfig } from './common/url-config'
 import actions from './actions'
 import auth from './auth'
+import queue from './queue'
 
 const app: IApp = {
   name: 'AISAY',
@@ -16,10 +18,12 @@ const app: IApp = {
   apiBaseUrl: '',
   primaryColor: '0059F7',
   actions,
+  requestErrorHandler,
   substepLabels: {
     connectionStepLabel: 'Connect your AISAY account',
     addConnectionLabel: 'Add new AISAY connection',
   },
+  queue,
 }
 
 export default app
diff --git a/packages/backend/src/apps/aisay/queue/index.ts b/packages/backend/src/apps/aisay/queue/index.ts

Original file line number	Diff line number	Diff line change
`@@ -25,6 +25,7 @@ async function getDataOutMetadata(`
`25`	`25`	`return {`
`26`	`26`	`quota: {`
`27`	`27`	`label: 'Quota',`
	`28`	`+ isHidden: true,`
`28`	`29`	`},`
`29`	`30`	`fields: fieldsMetadata,`
`30`	`31`	`}`
Original file line number	Diff line number	Diff line change
`@@ -54,7 +54,7 @@ const action: IRawAction = {`
`54`	`54`	`documentType: string`
`55`	`55`	`}`
`56`	`56`
`57`		`- if (!$.auth.data.clientId \|\| !$.auth.data.clientSecret) {`
	`57`	`+ if (!$.auth.data?.clientId \|\| !$.auth.data?.clientSecret) {`
`58`	`58`	`throw new StepError(`
`59`	`59`	`'Missing client ID or client secret',`
`60`	`60`	`'Please check the client ID and client secret',`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+export const MAX_FILE_SIZE = 6 * 1024 * 1024 // 6 MB`