feat/formula generated column

.github/workflows/integration-tests.yml

@@ -19,7 +19,7 @@ jobs:
       fail-fast: false
       matrix:
         node-version: [20.9.0]
-        database-type: [postgres, sqlite]
+        database-type: [postgres]
     env:
       CI: 1
 

.vscode/launch.json

@@ -12,10 +12,45 @@
       "runtimeExecutable": "sh",
       "autoAttachChildProcesses": true,
       "program": "./node_modules/.bin/vitest",
-      "args": ["run", "${workspaceFolder}/${relativeFile}", "--config", "./vitest-e2e.config.ts", "--hideSkippedTests"],
+      "args": [
+        "run",
+        "${workspaceFolder}/${relativeFile}",
+        "--config",
+        "./vitest-e2e.config.ts",
+        "--hideSkippedTests"
+      ],
       "smartStep": true,
       "console": "integratedTerminal",
-      "skipFiles": ["<node_internals>/**", "**/node_modules/**"],
+      "skipFiles": [
+        "<node_internals>/**",
+        "**/node_modules/**"
+      ],
+      "internalConsoleOptions": "neverOpen"
+    },
+    {
+      "name": "Debug vitest e2e nest backend",
+      "type": "node",
+      "request": "launch",
+      "cwd": "${workspaceFolder}/apps/nestjs-backend",
+      "runtimeExecutable": "node",
+      "program": "${workspaceFolder}/apps/nestjs-backend/node_modules/vitest/vitest.mjs",
+      "args": [
+        "run",
+        "${workspaceFolder}/${relativeFile}",
+        "--config",
+        "./vitest-e2e.config.ts",
+        "--hideSkippedTests",
+        "--no-file-parallelism",
+        "--reporter",
+        "verbose"
+      ],
+      "autoAttachChildProcesses": true,
+      "smartStep": true,
+      "console": "integratedTerminal",
+      "skipFiles": [
+        "<node_internals>/**",
+        "**/node_modules/**"
+      ],
       "internalConsoleOptions": "neverOpen"
     },
     {
@@ -26,10 +61,18 @@
       "runtimeExecutable": "sh",
       "autoAttachChildProcesses": true,
       "program": "./node_modules/.bin/vitest",
-      "args": ["run", "${workspaceFolder}/${relativeFile}", "--config", "./vitest.config.ts"],
+      "args": [
+        "run",
+        "${workspaceFolder}/${relativeFile}",
+        "--config",
+        "./vitest.config.ts"
+      ],
       "smartStep": true,
       "console": "integratedTerminal",
-      "skipFiles": ["<node_internals>/**", "**/node_modules/**"],
+      "skipFiles": [
+        "<node_internals>/**",
+        "**/node_modules/**"
+      ],
       "internalConsoleOptions": "neverOpen"
     },
     {
@@ -40,10 +83,18 @@
       "runtimeExecutable": "sh",
       "autoAttachChildProcesses": true,
       "program": "./node_modules/.bin/vitest",
-      "args": ["run", "${workspaceFolder}/${relativeFile}", "--config", "./vitest.config.ts"],
+      "args": [
+        "run",
+        "${workspaceFolder}/${relativeFile}",
+        "--config",
+        "./vitest.config.ts"
+      ],
       "smartStep": true,
       "console": "integratedTerminal",
-      "skipFiles": ["<node_internals>/**", "**/node_modules/**"],
+      "skipFiles": [
+        "<node_internals>/**",
+        "**/node_modules/**"
+      ],
       "internalConsoleOptions": "neverOpen"
     },
     {
@@ -54,10 +105,18 @@
       "runtimeExecutable": "sh",
       "autoAttachChildProcesses": true,
       "program": "./node_modules/.bin/vitest",
-      "args": ["run", "${workspaceFolder}/${relativeFile}", "--config", "./vitest.config.ts"],
+      "args": [
+        "run",
+        "${workspaceFolder}/${relativeFile}",
+        "--config",
+        "./vitest.config.ts"
+      ],
       "smartStep": true,
       "console": "integratedTerminal",
-      "skipFiles": ["<node_internals>/**", "**/node_modules/**"],
+      "skipFiles": [
+        "<node_internals>/**",
+        "**/node_modules/**"
+      ],
       "internalConsoleOptions": "neverOpen"
     },
     {
@@ -68,25 +127,40 @@
       "runtimeExecutable": "sh",
       "autoAttachChildProcesses": true,
       "program": "./node_modules/.bin/vitest",
-      "args": ["run", "${workspaceFolder}/${relativeFile}", "--config", "./vitest.config.ts"],
+      "args": [
+        "run",
+        "${workspaceFolder}/${relativeFile}",
+        "--config",
+        "./vitest.config.ts"
+      ],
       "smartStep": true,
       "console": "integratedTerminal",
-      "skipFiles": ["<node_internals>/**", "**/node_modules/**"],
+      "skipFiles": [
+        "<node_internals>/**",
+        "**/node_modules/**"
+      ],
       "internalConsoleOptions": "neverOpen"
     },
     {
       "name": "Debug nest backend",
       "type": "node",
       "request": "launch",
       "runtimeExecutable": "pnpm",
-      "args": ["apps/nestjs-backend/src/index.ts"],
-      "runtimeArgs": ["start-debug"],
-      "outFiles": ["${workspaceFolder}/**/*.js", "!**/node_modules/**"],
+      "args": [
+        "apps/nestjs-backend/src/index.ts"
+      ],
+      "runtimeArgs": [
+        "start-debug"
+      ],
+      "outFiles": [
+        "${workspaceFolder}/**/*.js",
+        "!**/node_modules/**"
+      ],
       "cwd": "${workspaceFolder}/apps/nestjs-backend",
       "internalConsoleOptions": "openOnSessionStart",
       "sourceMaps": true,
       "console": "internalConsole",
       "outputCapture": "std"
     },
   ]
-}
+}

.vscode/settings.json

@@ -27,6 +27,29 @@
     "univer",
     "zustand"
   ],
+  "editor.codeActionsOnSave": {
+    "source.fixAll.eslint": "explicit"
+  },
+  "eslint.format.enable": true,
+  "eslint.alwaysShowStatus": true,
+  "eslint.validate": [
+    "javascript",
+    "javascriptreact",
+    "typescript",
+    "typescriptreact"
+  ],
+  "[javascript]": {
+    "editor.formatOnSave": false
+  },
+  "[javascriptreact]": {
+    "editor.formatOnSave": false
+  },
+  "[typescript]": {
+    "editor.formatOnSave": false
+  },
+  "[typescriptreact]": {
+    "editor.formatOnSave": false
+  },
   "eslint.workingDirectories": [
     {
       "pattern": "./apps/*/"
@@ -36,4 +59,4 @@
     }
   ],
   "vitest.maximumConfigs": 10
-}
+}

apps/nestjs-backend/package.json

@@ -32,18 +32,19 @@
   },
   "scripts": {
     "build": "nest build",
-    "clean": "rimraf ./out ./coverage ./main ./dist ./tsconfig.tsbuildinfo ./node_modules/.cache",
+    "clean": "rimraf ./out ./coverage ./main ./dist ./tsconfig.tsbuildinfo ./node_modules/.cache .webpack-cache",
     "dev": "nest start --webpackPath ./webpack.dev.js -w",
     "dev:swc": "nest start --webpackPath ./webpack.swc.js -w",
     "start": "nest start",
     "check-dist": "es-check -v",
     "start-debug": "nest start --webpackPath ./webpack.dev.js --debug -w",
     "check-size": "size-limit --highlight-less",
     "test": "run-s test-unit test-e2e",
+    "test-unit:watch": "vitest --watch",
     "test-unit": "vitest run --silent --bail 1",
     "test-unit-cover": "pnpm test-unit --coverage",
     "pre-test-e2e": "cross-env NODE_ENV=test pnpm -F @teable/db-main-prisma prisma-db-seed -- --e2e",
-    "test-e2e": "pnpm pre-test-e2e && vitest run --config ./vitest-e2e.config.ts --silent --bail 1",
+    "test-e2e": "pnpm pre-test-e2e && vitest run --config ./vitest-e2e.config.ts --silent",
     "test-e2e-cover": "pnpm test-e2e --coverage",
     "typecheck": "tsc --project ./tsconfig.json --noEmit",
     "lint": "eslint . --ext .ts,.js,.cjs,.mjs,.mdx --cache --cache-location ../../.cache/eslint/nestjs-backend.eslintcache",
@@ -100,8 +101,10 @@
     "istanbul-merge": "2.0.0",
     "npm-run-all2": "6.1.2",
     "nyc": "15.1.0",
+    "pg-mem": "3.0.5",
     "prettier": "3.2.5",
     "rimraf": "5.0.5",
+    "sql-formatter": "^15.3.1",
     "swc-loader": "0.2.6",
     "symlink-dir": "5.2.1",
     "sync-directory": "6.0.5",

apps/nestjs-backend/src/app.module.ts

@@ -15,6 +15,7 @@ import { ChatModule } from './features/chat/chat.module';
 import { CollaboratorModule } from './features/collaborator/collaborator.module';
 import { CommentOpenApiModule } from './features/comment/comment-open-api.module';
 import { DashboardModule } from './features/dashboard/dashboard.module';
+import { DatabaseViewModule } from './features/database-view/database-view.module';
 import { ExportOpenApiModule } from './features/export/open-api/export-open-api.module';
 import { FieldOpenApiModule } from './features/field/open-api/field-open-api.module';
 import { HealthModule } from './features/health/health.module';
@@ -86,6 +87,7 @@ export const appModules = {
     PluginPanelModule,
     PluginContextMenuModule,
     PluginChartModule,
+    // DatabaseViewModule,
   ],
   providers: [InitBootstrapProvider],
 };

apps/nestjs-backend/src/db-provider/aggregation-query/aggregation-function.abstract.ts

@@ -1,25 +1,37 @@
-import { InternalServerErrorException, Logger } from '@nestjs/common';
+import { InternalServerErrorException } from '@nestjs/common';
+import type { FieldCore } from '@teable/core';
 import { StatisticsFunc } from '@teable/core';
 import type { Knex } from 'knex';
-import type { IFieldInstance } from '../../features/field/model/factory';
+import type { IRecordQueryAggregateContext } from '../../features/record/query-builder/record-query-builder.interface';
 import type { IAggregationFunctionInterface } from './aggregation-function.interface';
 
 export abstract class AbstractAggregationFunction implements IAggregationFunctionInterface {
-  private logger = new Logger(AbstractAggregationFunction.name);
-
   protected tableColumnRef: string;
 
   constructor(
     protected readonly knex: Knex,
-    protected readonly dbTableName: string,
-    protected readonly field: IFieldInstance
+    protected readonly field: FieldCore,
+    readonly context?: IRecordQueryAggregateContext
   ) {
-    const { dbFieldName } = this.field;
+    const { dbFieldName, id } = field;
+
+    const selection = context?.selectionMap.get(id);
+    if (selection) {
+      this.tableColumnRef = selection as string;
+    } else {
+      this.tableColumnRef = dbFieldName;
+    }
+  }
+
+  get dbTableName() {
+    return this.context?.tableDbName;
+  }
 
-    this.tableColumnRef = `${dbFieldName}`;
+  get tableAlias() {
+    return this.context?.tableAlias;
   }
 
-  compiler(builderClient: Knex.QueryBuilder, aggFunc: StatisticsFunc) {
+  compiler(builderClient: Knex.QueryBuilder, aggFunc: StatisticsFunc, alias: string | undefined) {
     const functionHandlers = {
       [StatisticsFunc.Count]: this.count,
       [StatisticsFunc.Empty]: this.empty,
@@ -73,39 +85,41 @@ export abstract class AbstractAggregationFunction implements IAggregationFunctio
       rawSql = `MAX(${this.knex.ref(`${joinTable}.value`)})`;
     }
 
-    return builderClient.select(this.knex.raw(`${rawSql} AS ??`, [`${fieldId}_${aggFunc}`]));
+    return builderClient.select(
+      this.knex.raw(`${rawSql} AS ??`, [alias ?? `${fieldId}_${aggFunc}`])
+    );
   }
 
   count(): string {
-    return this.knex.raw('COUNT(*)').toQuery();
+    return this.knex.raw(`COUNT(*)`).toQuery();
   }
 
   empty(): string {
-    return this.knex.raw(`COUNT(*) - COUNT(??)`, [this.tableColumnRef]).toQuery();
+    return this.knex.raw(`COUNT(*) - COUNT(${this.tableColumnRef})`).toQuery();
   }
 
   filled(): string {
-    return this.knex.raw(`COUNT(??)`, [this.tableColumnRef]).toQuery();
+    return this.knex.raw(`COUNT(${this.tableColumnRef})`).toQuery();
   }
 
   unique(): string {
-    return this.knex.raw(`COUNT(DISTINCT ??)`, [this.tableColumnRef]).toQuery();
+    return this.knex.raw(`COUNT(DISTINCT ${this.tableColumnRef})`).toQuery();
   }
 
   max(): string {
-    return this.knex.raw(`MAX(??)`, [this.tableColumnRef]).toQuery();
+    return this.knex.raw(`MAX(${this.tableColumnRef})`).toQuery();
   }
 
   min(): string {
-    return this.knex.raw(`MIN(??)`, [this.tableColumnRef]).toQuery();
+    return this.knex.raw(`MIN(${this.tableColumnRef})`).toQuery();
   }
 
   sum(): string {
-    return this.knex.raw(`SUM(??)`, [this.tableColumnRef]).toQuery();
+    return this.knex.raw(`SUM(${this.tableColumnRef})`).toQuery();
   }
 
   average(): string {
-    return this.knex.raw(`AVG(??)`, [this.tableColumnRef]).toQuery();
+    return this.knex.raw(`AVG(${this.tableColumnRef})`).toQuery();
   }
 
   checked(): string {