[TILES-V2-9]: store database type in TableContext on frontend (#987)

### TL;DR

Removed the explicit database type selection when creating tables, instead using LaunchDarkly flags to determine the database type.

### What changed?

- Removed `databaseType` from the `CreateTableInput` in GraphQL schema
- Added LaunchDarkly integration to determine database type when creating tables
- Added `databaseType` field to `TableMetadata` resolver to expose the database type to clients
- Updated tests to mock LaunchDarkly flag values
- Enhanced CSV import functionality with database-specific limits:
  - Increased max file size to 5MB for PostgreSQL (vs 2MB for DynamoDB)
  - Increased chunk size to 1000 rows for PostgreSQL (vs 100 for DynamoDB)
- Added pagination support for table rows with `stringifiedCursor` parameter

### How to test?

1. Create a new table and verify it uses the database type determined by LaunchDarkly
2. Import CSV files of different sizes to verify the database-specific limits
3. Test pagination by loading large tables and checking that rows load correctly

Author: pregnantboy

packages/backend/src/graphql/__tests__/mutations/tiles/create-table.itest.ts

@@ -1,11 +1,15 @@
-import { beforeEach, describe, expect, it } from 'vitest'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
 
 import createTable from '@/graphql/mutations/tiles/create-table'
 import { getTableRowCount } from '@/models/tiles/dynamodb/table-row'
 import Context from '@/types/express/context'
 
 import { generateMockContext } from './table.mock'
 
+vi.mock('@/helpers/launch-darkly', () => ({
+  getLdFlagValue: vi.fn().mockResolvedValue('ddb'),
+}))
+
 describe('create table mutation', () => {
   let context: Context
 
@@ -17,7 +21,7 @@ describe('create table mutation', () => {
   it('should create a blank table', async () => {
     const table = await createTable(
       null,
-      { input: { name: 'Test Table', isBlank: true, databaseType: 'ddb' } },
+      { input: { name: 'Test Table', isBlank: true } },
       context,
     )
     const tableColumnCount = await table.$relatedQuery('columns').resultSize()
@@ -30,7 +34,7 @@ describe('create table mutation', () => {
   it('should create a table and with placeholder rows and columns', async () => {
     const table = await createTable(
       null,
-      { input: { name: 'Test Table', isBlank: false, databaseType: 'ddb' } },
+      { input: { name: 'Test Table', isBlank: false } },
       context,
     )
     const tableColumnCount = await table.$relatedQuery('columns').resultSize()
@@ -43,13 +47,13 @@ describe('create table mutation', () => {
   it('should be able create tables with the same name', async () => {
     const table = await createTable(
       null,
-      { input: { name: 'Test Table', isBlank: false, databaseType: 'ddb' } },
+      { input: { name: 'Test Table', isBlank: false } },
       context,
     )
 
     const table2 = await createTable(
       null,
-      { input: { name: 'Test Table', isBlank: false, databaseType: 'ddb' } },
+      { input: { name: 'Test Table', isBlank: false } },
       context,
     )
     expect(table.name).toBe('Test Table')

packages/backend/src/graphql/custom-resolvers/table-metadata.ts

@@ -4,6 +4,10 @@ import type { Resolvers } from '../__generated__/types.generated'
 
 type TableMetadataResolver = Resolvers['TableMetadata']
 
+const databaseType: TableMetadataResolver['databaseType'] = async (parent) => {
+  return parent.db
+}
+
 const columns: TableMetadataResolver['columns'] = async (parent) => {
   const columns = await parent
     .$relatedQuery('columns')
@@ -33,4 +37,5 @@ const collaborators: TableMetadataResolver['collaborators'] = async (
 export default {
   columns,
   collaborators,
+  databaseType,
 } satisfies TableMetadataResolver

packages/backend/src/graphql/mutations/tiles/create-table.ts

@@ -1,7 +1,8 @@
+import { MutationResolvers } from '@/graphql/__generated__/types.generated'
+import { getLdFlagValue } from '@/helpers/launch-darkly'
 import TableMetadata from '@/models/table-metadata'
 import { getTableOperations } from '@/models/tiles/factory'
-
-import type { MutationResolvers } from '../../__generated__/types.generated'
+import { type DatabaseType } from '@/models/tiles/types'
 
 const PLACEHOLDER_COLUMNS = [
   {
@@ -19,21 +20,29 @@ const PLACEHOLDER_COLUMNS = [
 ]
 const PLACEHOLDER_ROWS = new Array(5).fill({})
 
+// DELETE THIS FLAG ONCE IT'S NO LONGER IN USE
+const DATABASE_TYPE_LD_FLAG_KEY = 'tiles-database-type'
+
 const createTable: MutationResolvers['createTable'] = async (
   _parent,
   params,
   context,
 ) => {
-  const {
-    name: tableName,
-    isBlank: isBlankTable,
-    databaseType = 'ddb',
-  } = params.input
+  const { name: tableName, isBlank: isBlankTable } = params.input
 
   if (!tableName) {
     throw new Error('Table name is required')
   }
 
+  /**
+   * Check which database type user is allowed to create
+   */
+  const databaseType = (await getLdFlagValue(
+    DATABASE_TYPE_LD_FLAG_KEY,
+    context.currentUser.email,
+    'ddb',
+  )) as DatabaseType
+
   const tableOperations = getTableOperations(databaseType)
 
   const table = await TableMetadata.transaction(async (trx) => {

packages/backend/src/graphql/queries/tiles/get-all-rows.ts

@@ -15,7 +15,7 @@ const getAllRows: QueryResolvers['getAllRows'] = async (
   params,
   context,
 ) => {
-  const { tableId } = params
+  const { tableId, stringifiedCursor } = params
 
   try {
     const table = context.tilesViewKey
@@ -43,10 +43,15 @@ const getAllRows: QueryResolvers['getAllRows'] = async (
 
     const columnIds = table.columns.map((column) => column.id)
 
-    const { rows } = await tableOperations.getTableRows({
-      tableId,
-      columnIds,
-    })
+    const { rows, stringifiedCursor: newStringifiedCursor } =
+      await tableOperations.getTableRows({
+        tableId,
+        columnIds,
+        // If table is postgres, we set pagination limit
+        // If table is dynamodb, the pagination size is based on dynamodb query limits
+        scanLimit: table.db === 'pg' ? 10000 : undefined,
+        stringifiedCursor,
+      })
 
     // Convert data object to csv to minimize payload size
     rows.forEach((row) => {
@@ -56,6 +61,7 @@ const getAllRows: QueryResolvers['getAllRows'] = async (
     return {
       rows,
       columnIds,
+      stringifiedCursor: newStringifiedCursor,
     }
   } catch (e) {
     logger.error(e)

packages/backend/src/graphql/schema.graphql

@@ -749,7 +749,6 @@ enum DatabaseType {
 input CreateTableInput {
   name: String!
   isBlank: Boolean!
-  databaseType: DatabaseType
 }
 
 input TableColumnConfigInput {
@@ -804,6 +803,7 @@ input DeleteTableCollaboratorInput {
 
 type TableMetadata {
   id: ID!
+  databaseType: DatabaseType!
   name: String!
   columns: [TableColumnMetadata!]
   lastAccessedAt: String!

packages/backend/src/models/tiles/pg/table-row-functions.ts

@@ -221,6 +221,7 @@ export const getTableRows = async ({
   filters,
   order = 'asc',
   scanLimit,
+  stringifiedCursor,
 }: {
   tableId: string
   columnIds?: string[]
@@ -230,8 +231,10 @@ export const getTableRows = async ({
    * Optional limit on the total number of rows scanned.
    */
   scanLimit?: number
+  stringifiedCursor?: string
 }): Promise<{
   rows: TableRowOutput[]
+  stringifiedCursor: string | null
 }> => {
   const query = tilesClient(tableId).select(['rowId', ...(columnIds ?? [])])
   if (filters) {
@@ -240,6 +243,10 @@ export const getTableRows = async ({
   if (scanLimit) {
     query.limit(scanLimit)
   }
+  const offset = stringifiedCursor ? +stringifiedCursor : 0
+  if (offset) {
+    query.offset(offset)
+  }
   try {
     const tableRows = []
     const stream = query.orderBy('rowId', order).stream()
@@ -249,6 +256,8 @@ export const getTableRows = async ({
     }
     return {
       rows: tableRows,
+      stringifiedCursor:
+        tableRows.length === scanLimit ? (offset + scanLimit).toString() : null,
     }
   } catch (e: unknown) {
     logger.error(e)

packages/frontend/src/graphql/queries/tiles/get-table.ts

@@ -6,6 +6,7 @@ export const GET_TABLE = gql`
       id
       name
       viewOnlyKey
+      databaseType
       columns {
         id
         name

packages/frontend/src/pages/Tile/components/TableBanner/ImportCsvButton.tsx

@@ -29,6 +29,7 @@ import Papa, { ParseMeta, ParseResult } from 'papaparse'
 import { SetRequired } from 'type-fest'
 
 import PrimarySpinner from '@/components/PrimarySpinner'
+import { DatabaseType } from '@/graphql/__generated__/graphql'
 import { CREATE_ROWS } from '@/graphql/mutations/tiles/create-rows'
 import { GET_TABLE } from '@/graphql/queries/tiles/get-table'
 
@@ -46,10 +47,16 @@ type IMPORT_STATUS =
   | 'completed'
   | 'error'
 
-// 2 MB in bytes
-const MAX_FILE_SIZE = 2 * 1000 * 1000
+// 5 MB in bytes
+const MAX_FILE_SIZE = {
+  [DatabaseType.Pg]: 5 * 1000 * 1000,
+  [DatabaseType.Ddb]: 2 * 1000 * 1000,
+}
 // Add row chunk size
-const CHUNK_SIZE = 100
+const CHUNK_SIZE = {
+  [DatabaseType.Pg]: 1000,
+  [DatabaseType.Ddb]: 100,
+}
 
 const ImportStatus = ({
   columnsToCreate,
@@ -134,7 +141,7 @@ export const ImportCsvModalContent = ({
   onPostImport?: () => void
   onBack?: () => void
 }) => {
-  const { tableId, tableColumns, refetch } = useTableContext()
+  const { tableId, tableColumns, refetch, databaseType } = useTableContext()
   const { createColumns } = useUpdateTable()
   const [createRows] = useMutation(CREATE_ROWS)
   const [getTableData] = useLazyQuery<{
@@ -255,7 +262,7 @@ export const ImportCsvModalContent = ({
         setImportStatus('importing')
         setRowsToImport(mappedData.length)
         setRowsImported(0)
-        const chunkedData = chunk(mappedData, CHUNK_SIZE)
+        const chunkedData = chunk(mappedData, CHUNK_SIZE[databaseType])
 
         for (let i = 0; i < chunkedData.length; i++) {
           await createRows({
@@ -269,7 +276,7 @@ export const ImportCsvModalContent = ({
           if (i === chunkedData.length - 1 && !onPreImport) {
             await refetch()
           }
-          setRowsImported((i + 1) * CHUNK_SIZE)
+          setRowsImported((i + 1) * CHUNK_SIZE[databaseType])
         }
       }
       setImportStatus('completed')
@@ -291,6 +298,7 @@ export const ImportCsvModalContent = ({
     onPreImport,
     refetch,
     result,
+    databaseType,
     tableColumns,
     tableId,
   ])
@@ -317,7 +325,7 @@ export const ImportCsvModalContent = ({
           existing values.
         </Text>
         <Attachment
-          maxSize={MAX_FILE_SIZE}
+          maxSize={MAX_FILE_SIZE[databaseType]}
           onChange={setFile}
           title="Upload CSV"
           name="file-upload"

packages/frontend/src/pages/Tile/contexts/TableContext.tsx

@@ -13,12 +13,15 @@ import React, {
   useState,
 } from 'react'
 
+import { DatabaseType } from '@/graphql/__generated__/graphql'
+
 import { flattenRows } from '../helpers/flatten-rows'
 import { EditMode, GenericRowData } from '../types'
 
 interface TableContextProps {
   tableId: string
   tableName: string
+  databaseType: DatabaseType
   flattenedData: GenericRowData[]
   tableColumns: ITableColumnMetadata[]
   filteredDataRef: MutableRefObject<GenericRowData[]>
@@ -49,6 +52,7 @@ export const useTableContext = () => {
 interface TableContextProviderProps {
   tableId: string
   tableName: string
+  databaseType: DatabaseType
   tableColumns: ITableColumnMetadata[]
   tableRows: ITableRow[]
   children: React.ReactNode
@@ -64,6 +68,7 @@ interface TableContextProviderProps {
 export const TableContextProvider = ({
   tableId,
   tableName,
+  databaseType,
   tableColumns,
   tableRows,
   children,
@@ -86,6 +91,7 @@ export const TableContextProvider = ({
       value={{
         tableId,
         tableName,
+        databaseType,
         flattenedData,
         tableColumns,
         allDataRef,

packages/frontend/src/pages/Tile/index.tsx

@@ -80,13 +80,14 @@ export default function Tile(): JSX.Element | null {
     return null
   }
 
-  const { id, name, columns, viewOnlyKey, collaborators } =
+  const { id, name, columns, viewOnlyKey, collaborators, databaseType } =
     getTableData.getTable
 
   return (
     <TableContextProvider
       tableName={name}
       tableId={id}
+      databaseType={databaseType}
       tableColumns={columns}
       tableRows={rows}
       viewOnlyKey={viewOnlyKey}