Release v1.44.0 (#1046)

* Fix Excel get table rows function
* Improve editor interactions in published pipes
* Retry Postman SMS on Cloudflare errors
* Remove manually uploaded attachments when duplicating Pipe

Author: pregnantboy

package-lock.json

@@ -106,5 +106,5 @@
     "tsconfig-paths": "^4.2.0",
     "type-fest": "4.10.3"
   },
-  "version": "1.43.1"
+  "version": "1.44.0"
 }

packages/backend/package.json

@@ -2,6 +2,7 @@ import { type IGlobalVariable } from '@plumber/types'
 
 import z from 'zod'
 
+import HttpError from '@/errors/http'
 import StepError from '@/errors/step'
 
 import WorkbookSession from '../../../common/workbook-session'
@@ -16,13 +17,116 @@ const msGraphResponseSchema = z
     headerSheetRowIndex: response.rowIndex,
   }))
 
+// Can get more info e.g. formulas, numberFormat, text for possible date manipulation but too complex for now
+const msGraphRangeResponseSchema = z
+  .object({
+    rowCount: z.number(),
+    rowIndex: z.number(),
+    values: z.array(z.array(z.coerce.string())), // first row is the header
+  })
+  .transform((response) => {
+    if (response.values.length === 0) {
+      throw new Error('Excel table is missing the header row')
+    }
+    return {
+      rowCount: response.rowCount,
+      headerSheetRowIndex: response.rowIndex,
+      headerRowValues: response.values[0],
+      rowValues: response.values.slice(1),
+    }
+  })
+
 interface GetTopNTableRowsResult {
   columns: string[]
   rows: string[][]
   headerSheetRowIndex: number
 }
 
-export default async function getTopNTableRows(
+/**
+ * Using range endpoint to get both the header row and data rows
+ * instead of headerRowRange and rows to avoid extra API call
+ * Range endpoint API time taken (similar to the previous resized range method)
+ * - Around 0.5s for 50k rows with 5 columns of data
+ * - Around 0.25s for 25k rows with 3 columns of empty data
+ * - Around 0.15s for 10k rows with 3 columns of empty data
+ *
+ * Also, rows endpoint uses pagination so the query takes longer for large tables
+ * Rows endpoint API time taken:
+ * - Around 5s for 50k rows with 5 columns of data
+ * - Around 2s for 25k rows with 3 columns of empty data
+ * - Around 1s for 10k rows with 3 columns of empty data
+ *
+ * Considered using columns API but there is no headerSheetRowIndex returned so
+ * an extra API call has to be made to retrieve it
+ * Reference: https://learn.microsoft.com/en-us/graph/api/table-range?view=graph-rest-1.0&tabs=http
+ */
+export async function getTopNTableRows(
+  $: IGlobalVariable,
+  session: WorkbookSession,
+  tableId: string,
+  n: number,
+): Promise<GetTopNTableRowsResult> {
+  try {
+    const rangeParseResult = msGraphRangeResponseSchema.safeParse(
+      (
+        await session.request(
+          `/tables/${tableId}/range?$select=values,rowCount,rowIndex`,
+          'get',
+        )
+      ).data,
+    )
+
+    if (rangeParseResult.success === false) {
+      throw new StepError(
+        'Invalid table range',
+        'Check your Excel file and try again',
+        $.step.position,
+        $.app.name,
+      )
+    }
+
+    const { rowCount, headerSheetRowIndex, headerRowValues, rowValues } =
+      rangeParseResult.data
+
+    // rowCount includes the header row
+    if (rowCount > n + 1) {
+      throw new StepError(
+        `Your Excel table has more than ${n.toLocaleString()} rows.`,
+        `Reduce the number of rows and try again.`,
+        $.step.position,
+        $.app.name,
+      )
+    }
+
+    return {
+      columns: headerRowValues,
+      rows: rowValues,
+      headerSheetRowIndex,
+    }
+  } catch (error) {
+    if (error instanceof HttpError) {
+      if (
+        error.response.status === 400 &&
+        error.response.data.error.code === 'RangeExceedsLimit'
+      ) {
+        throw new StepError(
+          'Your Excel table is too large',
+          'Reduce the number of rows or columns and try again.',
+          $.step.position,
+          $.app.name,
+          error,
+        )
+      }
+    }
+    throw error
+  }
+}
+
+/**
+ * @deprecated Use getTopNTableRows instead
+ */
+// Old method in case we need to rollback fast
+export async function getTopNTableRowsOld(
   // Typically, we should avoid making $ viral though the codebase, but this is
   // an exception because getTopNTableRows is not a common helper function.
   $: IGlobalVariable,
@@ -86,3 +190,5 @@ export default async function getTopNTableRows(
     headerSheetRowIndex: tableRows.headerSheetRowIndex,
   }
 }
+
+export default getTopNTableRows

packages/backend/src/apps/m365-excel/actions/get-table-row/implementation/get-top-n-table-rows.ts

@@ -73,7 +73,7 @@ describe('Postman SMS request error handler', () => {
   })
 
   describe('Other errors', () => {
-    it.each([500, 502, 503])('should retry on %s', async (status) => {
+    it.each([500, 502, 503, 520, 524])('should retry on %s', async (status) => {
       const axiosError = {
         isAxiosError: true,
         name: 'AxiosError',

packages/backend/src/apps/postman-sms/__tests__/request-error-handler.test.ts

@@ -21,7 +21,7 @@ const handle429: ThrowingHandler = (_, error): never => {
   })
 }
 
-const handle500and502and503: ThrowingHandler = (_, error): never => {
+const handle5xxErrors: ThrowingHandler = (_, error): never => {
   const status = error.response.status
   throw new RetriableError({
     error: `Retrying HTTP ${status} from Postman SMS`,
@@ -40,7 +40,9 @@ const requestErrorHandler: IApp['requestErrorHandler'] = async function (
     case 500:
     case 502:
     case 503:
-      return handle500and502and503($, error)
+    case 520: // Cloudflare-specific error
+    case 524: // Cloudflare-specific error
+      return handle5xxErrors($, error)
     default:
       if (error.message === 'read ETIMEDOUT') {
         throw new RetriableError({