feat: use factory class for queries and mutations, minimise getAllRows payload

Author: pregnantboy

packages/backend/src/graphql/__tests__/mutations/tiles/create-table.itest.ts

@@ -30,7 +30,7 @@ describe('create table mutation', () => {
   it('should create a table and with placeholder rows and columns', async () => {
     const table = await createTable(
       null,
-      { input: { name: 'Test Table', isBlank: false } },
+      { input: { name: 'Test Table', isBlank: false, databaseType: 'ddb' } },
       context,
     )
     const tableColumnCount = await table.$relatedQuery('columns').resultSize()

packages/backend/src/graphql/__tests__/queries/tiles/get-all-rows.itest.ts

@@ -98,7 +98,6 @@ describe('get all rows query', () => {
         context,
       )
       cursor = stringifiedCursor
-      expect(pageRows.length).toBeLessThan(numRowsToInsert)
       rows = rows.concat(pageRows)
     } while (cursor)
     expect(rows.length).toBe(numRowsToInsert)
@@ -138,7 +137,7 @@ describe('get all rows query', () => {
     )
     const rows = await Promise.all(returnedRows)
 
-    expect(Object.keys(rows[0].data).sort()).toEqual(dummyColumnIds.sort())
+    expect(rows[0].data.split(',').length).toBe(dummyColumnIds.length)
   })
 
   it('should allow all collaborators to call this function', async () => {

packages/backend/src/graphql/mutations/tiles/create-row.ts

@@ -1,6 +1,6 @@
 import TableCollaborator from '@/models/table-collaborators'
 import TableMetadata from '@/models/table-metadata'
-import { createTableRow } from '@/models/tiles/dynamodb/table-row'
+import { getTableOperations } from '@/models/tiles/factory'
 
 import type { MutationResolvers } from '../../__generated__/types.generated'
 
@@ -19,7 +19,9 @@ const createRow: MutationResolvers['createRow'] = async (
     throw new Error('Invalid column id')
   }
 
-  const row = await createTableRow({ tableId, data })
+  const tableOperations = getTableOperations(table.db)
+
+  const row = await tableOperations.createTableRow({ tableId, data })
 
   return row.rowId
 }

packages/backend/src/graphql/mutations/tiles/create-rows.ts

@@ -1,6 +1,6 @@
 import TableCollaborator from '@/models/table-collaborators'
 import TableMetadata from '@/models/table-metadata'
-import { createTableRows } from '@/models/tiles/dynamodb/table-row'
+import { getTableOperations } from '@/models/tiles/factory'
 
 import type { MutationResolvers } from '../../__generated__/types.generated'
 
@@ -19,7 +19,9 @@ const createRows: MutationResolvers['createRows'] = async (
     throw new Error('Invalid column id')
   }
 
-  await createTableRows({ tableId, dataArray })
+  const tableOperations = getTableOperations(table.db)
+
+  await tableOperations.createTableRows({ tableId, dataArray })
 
   return true
 }

packages/backend/src/graphql/mutations/tiles/create-table.ts

@@ -1,4 +1,4 @@
-import { createTableRows } from '@/models/tiles/dynamodb/table-row'
+import { getTableOperations } from '@/models/tiles/factory'
 
 import type { MutationResolvers } from '../../__generated__/types.generated'
 
@@ -23,20 +23,36 @@ const createTable: MutationResolvers['createTable'] = async (
   params,
   context,
 ) => {
-  const { name: tableName, isBlank: isBlankTable } = params.input
+  const {
+    name: tableName,
+    isBlank: isBlankTable,
+    databaseType = 'pg',
+  } = params.input
 
   if (!tableName) {
     throw new Error('Table name is required')
   }
 
+  const tableOperations = getTableOperations(databaseType)
+
+  // TODO: should i wrap this in a transaction?
   const table = await context.currentUser.$relatedQuery('tables').insertGraph({
     name: tableName,
     role: 'owner',
+    db: databaseType,
     columns: isBlankTable ? [] : PLACEHOLDER_COLUMNS,
   })
 
+  await tableOperations.createTable(
+    table.id,
+    isBlankTable ? [] : table.columns.map((column) => column.id),
+  )
+
   if (!isBlankTable) {
-    await createTableRows({ tableId: table.id, dataArray: PLACEHOLDER_ROWS })
+    await tableOperations.createTableRows({
+      tableId: table.id,
+      dataArray: PLACEHOLDER_ROWS,
+    })
   }
 
   return table

packages/backend/src/graphql/mutations/tiles/delete-rows.ts

@@ -1,5 +1,6 @@
 import TableCollaborator from '@/models/table-collaborators'
-import { deleteTableRows } from '@/models/tiles/dynamodb/table-row'
+import TableMetadata from '@/models/table-metadata'
+import { getTableOperations } from '@/models/tiles/factory'
 
 import type { MutationResolvers } from '../../__generated__/types.generated'
 
@@ -12,7 +13,10 @@ const deleteRows: MutationResolvers['deleteRows'] = async (
 
   await TableCollaborator.hasAccess(context.currentUser.id, tableId, 'editor')
 
-  await deleteTableRows({ tableId, rowIds })
+  const table = await TableMetadata.query().findById(tableId).throwIfNotFound()
+  const tableOperations = getTableOperations(table.db)
+
+  await tableOperations.deleteTableRows({ tableId, rowIds })
 
   return rowIds
 }

packages/backend/src/graphql/mutations/tiles/update-row.ts

@@ -1,6 +1,6 @@
 import TableCollaborator from '@/models/table-collaborators'
 import TableMetadata from '@/models/table-metadata'
-import { updateTableRow } from '@/models/tiles/dynamodb/table-row'
+import { getTableOperations } from '@/models/tiles/factory'
 
 import type { MutationResolvers } from '../../__generated__/types.generated'
 
@@ -19,7 +19,9 @@ const updateRow: MutationResolvers['updateRow'] = async (
     throw new Error('Invalid column id')
   }
 
-  await updateTableRow({ tableId, rowId, data })
+  const tableOperations = getTableOperations(table.db)
+
+  await tableOperations.updateTableRow({ tableId, rowId, data })
 
   return rowId
 }

packages/backend/src/graphql/mutations/tiles/update-table.ts

@@ -4,6 +4,7 @@ import { z } from 'zod'
 import TableCollaborator from '@/models/table-collaborators'
 import TableColumnMetadata from '@/models/table-column-metadata'
 import TableMetadata from '@/models/table-metadata'
+import { getTableOperations } from '@/models/tiles/factory'
 
 import type { MutationResolvers } from '../../__generated__/types.generated'
 
@@ -45,6 +46,8 @@ const updateTable: MutationResolvers['updateTable'] = async (
 
   const table = await TableMetadata.query().findById(tableId)
 
+  const tableOperations = getTableOperations(table.db)
+
   if (tableName) {
     await table.$query().patch({
       name: tableName.trim(),
@@ -57,11 +60,20 @@ const updateTable: MutationResolvers['updateTable'] = async (
         .$relatedQuery('columns', trx)
         .max('position as position') // aliasing for more convenient typing
       const maxPosition = results[0].position || 0
-      await table.$relatedQuery('columns', trx).insert(
-        addedColumns.map((name, i) => ({
-          name,
-          position: maxPosition + i + 1,
-        })),
+      const tableMetadataColumns = await table
+        .$relatedQuery('columns', trx)
+        .insert(
+          addedColumns.map((name, i) => ({
+            name,
+            position: maxPosition + i + 1,
+          })),
+        )
+        .returning('id')
+
+      // If creation of columns fail, we roll back the creation of table column metadata
+      await tableOperations.createTableColumns(
+        tableId,
+        tableMetadataColumns.map((column) => column.id),
       )
     })
   }
@@ -98,6 +110,8 @@ const updateTable: MutationResolvers['updateTable'] = async (
         .$relatedQuery('columns')
         .delete()
         .whereIn('id', deletedColumns)
+
+      await tableOperations.deleteTableColumns(tableId, deletedColumns)
     })
   }
 

packages/backend/src/graphql/queries/tiles/get-all-rows.ts

@@ -6,7 +6,7 @@ import InvalidTileViewKeyError from '@/errors/invalid-tile-view-key'
 import logger from '@/helpers/logger'
 import TableMetadata from '@/models/table-metadata'
 import { DYNAMODB_THROUGHPUT_EXCEEDED_ERROR_MESSAGE } from '@/models/tiles/dynamodb/helpers'
-import { getTableRows } from '@/models/tiles/dynamodb/table-row'
+import { getTableOperations } from '@/models/tiles/factory'
 
 import type { QueryResolvers } from '../../__generated__/types.generated'
 
@@ -15,7 +15,7 @@ const getAllRows: QueryResolvers['getAllRows'] = async (
   params,
   context,
 ) => {
-  const { tableId, stringifiedCursor } = params
+  const { tableId } = params
 
   try {
     const table = context.tilesViewKey
@@ -39,14 +39,24 @@ const getAllRows: QueryResolvers['getAllRows'] = async (
       })
     }
 
+    const tableOperations = getTableOperations(table.db)
+
     const columnIds = table.columns.map((column) => column.id)
 
-    return await getTableRows({
+    const { rows } = await tableOperations.getTableRows({
       tableId,
       columnIds,
-      stringifiedCursor: stringifiedCursor ?? 'start',
     })
-    // TODO: remove keys from rows to reduce payload size
+
+    // Convert data object to csv to minimize payload size
+    rows.forEach((row) => {
+      row.data = columnIds.map((columnId) => row.data[columnId]).join(',')
+    })
+
+    return {
+      rows,
+      columnIds,
+    }
   } catch (e) {
     logger.error(e)
     if (e instanceof ObjectionNotFoundError) {

packages/backend/src/graphql/schema.graphql

@@ -743,10 +743,16 @@ type PaginatedExecutionSteps {
   pageInfo: PageInfo!
 }
 
+enum DatabaseType {
+  pg
+  ddb
+}
+
 # Tiles types
 input CreateTableInput {
   name: String!
   isBlank: Boolean!
+  databaseType: DatabaseType
 }
 
 input TableColumnConfigInput {
@@ -820,6 +826,7 @@ type PaginatedTables {
 
 type GetTableRowsResult {
   rows: Any!
+  columnIds: [String!]!
   stringifiedCursor: String
 }
 

packages/backend/src/helpers/flow-templates.ts

@@ -191,6 +191,7 @@ export async function createFlowFromTemplate(
           role: 'owner',
           columns: columnsToInsert,
         })
+
         tableId = table.id
 
         // obtain a map of column name to the column id for easy replacement
@@ -200,6 +201,8 @@ export async function createFlowFromTemplate(
 
         // replace column names with column ids for each row data and create table rows
         const newRowData = replaceColumnNamesWithIds(rowData, columnNameToIdMap)
+
+        // TODO: tiles-v2: use table operations to create rows
         await createTableRows({ tableId, dataArray: newRowData })
       } catch (e) {
         logger.error(e)

packages/backend/test/pg-global-setup.ts

@@ -25,7 +25,7 @@ export async function setup() {
   )
 
   const config = await import('../knexfile')
-  const client = knex(config.default as any)
+  const client = knex(config.default)
 
   // manually running migrations since the programmatic API doesn't work
   // see issue here: https://github.com/knex/knex/issues/5323

packages/frontend/src/graphql/queries/tiles/get-all-rows.ts

@@ -4,6 +4,7 @@ export const GET_ALL_ROWS = graphql(`
   query GetAllRows($tableId: String!, $stringifiedCursor: String) {
     getAllRows(tableId: $tableId, stringifiedCursor: $stringifiedCursor) {
       rows
+      columnIds
       stringifiedCursor
     }
   }