Skip to content

Commit ee342c5

Browse files
ArtuomkaArtuomka
authored
Merge pull request #498 from rocket-admin/backend_pagination_reowrk
reworked pagination and count rows methods
2 parents 822ca7d + 0df10c4 commit ee342c5

File tree

11 files changed

+177
-267
lines changed

11 files changed

+177
-267
lines changed

‎backend/package.json‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,6 @@
7777
"json2csv": "^5.0.7",
7878
"jsonwebtoken": "^9.0.2",
7979
"knex": "3.1.0",
80-
"knex-paginate": "3.1.1",
8180
"lodash": "4.17.21",
8281
"lru-cache": "^10.1.0",
8382
"moment": "2.29.4",

‎backend/test/ava-tests/saas-tests/table-mssql-e2e.test.ts‎

Lines changed: 25 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -3360,32 +3360,34 @@ with search and pagination: page=1, perPage=2 and DESC sorting`, async (t) => {
33603360
const createConnectionRO = JSON.parse(createConnectionResponse.text);
33613361
t.is(createConnectionResponse.status, 201);
33623362

3363-
const createTableSettingsDTO = mockFactory.generateTableSettings(
3364-
createConnectionRO.id,
3365-
testTableName,
3366-
[testTableColumnName],
3367-
undefined,
3368-
undefined,
3369-
3,
3370-
QueryOrderingEnum.DESC,
3371-
'id',
3372-
undefined,
3373-
undefined,
3374-
undefined,
3375-
undefined,
3376-
undefined,
3377-
);
3363+
const createTableSettingsDTO = mockFactory.generateTableSettings(
3364+
createConnectionRO.id,
3365+
testTableName,
3366+
[testTableColumnName],
3367+
undefined,
3368+
undefined,
3369+
3,
3370+
QueryOrderingEnum.DESC,
3371+
'id',
3372+
undefined,
3373+
undefined,
3374+
undefined,
3375+
undefined,
3376+
undefined,
3377+
);
33783378

3379-
const createTableSettingsResponse = await request(app.getHttpServer())
3380-
.post(`/settings?connectionId=${createConnectionRO.id}&tableName=${testTableName}`)
3381-
.send(createTableSettingsDTO)
3382-
.set('Cookie', firstUserToken)
3383-
.set('Content-Type', 'application/json')
3384-
.set('Accept', 'application/json');
3385-
t.is(createTableSettingsResponse.status, 201);
3379+
const createTableSettingsResponse = await request(app.getHttpServer())
3380+
.post(`/settings?connectionId=${createConnectionRO.id}&tableName=${testTableName}`)
3381+
.send(createTableSettingsDTO)
3382+
.set('Cookie', firstUserToken)
3383+
.set('Content-Type', 'application/json')
3384+
.set('Accept', 'application/json');
3385+
t.is(createTableSettingsResponse.status, 201);
33863386

33873387
const getTableCsvResponse = await request(app.getHttpServer())
3388-
.get(`/table/csv/${createConnectionRO.id}?tableName=${testTableName}&search=${testSearchedUserName}&page=1&perPage=2`)
3388+
.get(
3389+
`/table/csv/${createConnectionRO.id}?tableName=${testTableName}&search=${testSearchedUserName}&page=1&perPage=2`,
3390+
)
33893391
.set('Cookie', firstUserToken)
33903392
.set('Content-Type', 'text/csv')
33913393
.set('Accept', 'text/csv');

‎backend/test/ava-tests/saas-tests/table-postgres-e2e.test.ts‎

Lines changed: 25 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1268,7 +1268,6 @@ should return all found rows with search, pagination: page=1, perPage=2 and DESC
12681268
.set('Cookie', firstUserToken)
12691269
.set('Content-Type', 'application/json')
12701270
.set('Accept', 'application/json');
1271-
t.is(getTableRowsResponse.status, 200);
12721271

12731272
const getTableRowsRO = JSON.parse(getTableRowsResponse.text);
12741273
t.is(typeof getTableRowsRO, 'object');
@@ -3400,32 +3399,34 @@ with search and pagination: page=1, perPage=2 and DESC sorting`, async (t) => {
34003399
const createConnectionRO = JSON.parse(createConnectionResponse.text);
34013400
t.is(createConnectionResponse.status, 201);
34023401

3403-
const createTableSettingsDTO = mockFactory.generateTableSettings(
3404-
createConnectionRO.id,
3405-
testTableName,
3406-
[testTableColumnName],
3407-
undefined,
3408-
undefined,
3409-
3,
3410-
QueryOrderingEnum.DESC,
3411-
'id',
3412-
undefined,
3413-
undefined,
3414-
undefined,
3415-
undefined,
3416-
undefined,
3417-
);
3402+
const createTableSettingsDTO = mockFactory.generateTableSettings(
3403+
createConnectionRO.id,
3404+
testTableName,
3405+
[testTableColumnName],
3406+
undefined,
3407+
undefined,
3408+
3,
3409+
QueryOrderingEnum.DESC,
3410+
'id',
3411+
undefined,
3412+
undefined,
3413+
undefined,
3414+
undefined,
3415+
undefined,
3416+
);
34183417

3419-
const createTableSettingsResponse = await request(app.getHttpServer())
3420-
.post(`/settings?connectionId=${createConnectionRO.id}&tableName=${testTableName}`)
3421-
.send(createTableSettingsDTO)
3422-
.set('Cookie', firstUserToken)
3423-
.set('Content-Type', 'application/json')
3424-
.set('Accept', 'application/json');
3425-
t.is(createTableSettingsResponse.status, 201);
3418+
const createTableSettingsResponse = await request(app.getHttpServer())
3419+
.post(`/settings?connectionId=${createConnectionRO.id}&tableName=${testTableName}`)
3420+
.send(createTableSettingsDTO)
3421+
.set('Cookie', firstUserToken)
3422+
.set('Content-Type', 'application/json')
3423+
.set('Accept', 'application/json');
3424+
t.is(createTableSettingsResponse.status, 201);
34263425

34273426
const getTableCsvResponse = await request(app.getHttpServer())
3428-
.get(`/table/csv/${createConnectionRO.id}?tableName=${testTableName}&search=${testSearchedUserName}&page=1&perPage=2`)
3427+
.get(
3428+
`/table/csv/${createConnectionRO.id}?tableName=${testTableName}&search=${testSearchedUserName}&page=1&perPage=2`,
3429+
)
34293430
.set('Cookie', firstUserToken)
34303431
.set('Content-Type', 'text/csv')
34313432
.set('Accept', 'text/csv');

‎rocketadmin-agent/package.json‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@
3535
"crypto-js": "^4.1.1",
3636
"get-port": "^7.0.0",
3737
"knex": "2.4.2",
38-
"knex-paginate": "3.1.1",
3938
"mysql2": "^3.4.2",
4039
"nest-winston": "^1.9.3",
4140
"oracledb": "^6.0.1",

‎rocketadmin-cli/package.json‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@
3131
"crypto-js": "^4.1.1",
3232
"get-port": "^7.0.0",
3333
"knex": "^2.4.2",
34-
"knex-paginate": "^3.1.1",
3534
"mysql2": "^3.4.2",
3635
"nest-winston": "^1.9.3",
3736
"oracledb": "^6.0.1",

‎shared-code/package.json‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
"get-port": "^7.0.0",
99
"jsonwebtoken": "^9.0.2",
1010
"knex": "^3.0.1",
11-
"knex-paginate": "^3.1.1",
1211
"lru-cache": "^10.0.1",
1312
"mysql2": "^3.6.3",
1413
"oracledb": "^6.2.0",

‎shared-code/src/data-access-layer/data-access-objects/basic-data-access-object.ts‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,10 @@
1-
import { attachPaginate } from 'knex-paginate';
21
import { KnexManager } from '../../knex-manager/knex-manager.js';
32
import { ConnectionParams } from '../shared/data-structures/connections-params.ds.js';
43
import { Knex } from 'knex';
54
import { isObjectEmpty } from '../../helpers/is-object-empty.js';
65
import { comparePrimitiveArrayElements } from '../../helpers/compate-primitive-array-elements.js';
76
import { TableStructureDS } from '../shared/data-structures/table-structure.ds.js';
87
import { TableSettingsDS } from '../shared/data-structures/table-settings.ds.js';
9-
attachPaginate();
108

119
export class BasicDataAccessObject {
1210
protected connection: ConnectionParams;

‎shared-code/src/data-access-layer/data-access-objects/data-access-object-mssql.ts‎

Lines changed: 39 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { Knex } from 'knex';
12
import { LRUStorage } from '../../caching/lru-storage.js';
23
import { DAO_CONSTANTS } from '../../helpers/data-access-objects-constants.js';
34
import { ERROR_MESSAGES } from '../../helpers/errors/error-messages.js';
@@ -96,7 +97,7 @@ export class DataAccessObjectMssql extends BasicDataAccessObject implements IDat
9697
}
9798

9899
public async getRowsFromTable(
99-
tableName: string,
100+
receivedTableName: string,
100101
tableSettings: TableSettingsDS,
101102
page: number,
102103
perPage: number,
@@ -114,21 +115,18 @@ export class DataAccessObjectMssql extends BasicDataAccessObject implements IDat
114115
}
115116
}
116117
const knex = await this.configureKnex();
117-
const [rowsCount, tableStructure, tableSchema] = await Promise.all([
118-
this.getRowsCount(tableName),
119-
this.getTableStructure(tableName),
120-
this.getSchemaName(tableName),
118+
const [tableStructure, tableSchema] = await Promise.all([
119+
this.getTableStructure(receivedTableName),
120+
this.getSchemaName(receivedTableName),
121121
]);
122122
const availableFields = this.findAvaliableFields(tableSettings, tableStructure);
123+
const tableNameWithoutSchema = receivedTableName;
124+
const tableNameWithSchema = tableSchema ? `${tableSchema}.[${receivedTableName}]` : receivedTableName;
123125

124-
if (tableSchema) {
125-
tableName = `${tableSchema}.[${tableName}]`;
126-
}
127-
const lastPage = Math.ceil(rowsCount / perPage);
128126
/* eslint-enable */
129127
let rowsRO: FoundRowsDS;
130128
if (autocompleteFields && autocompleteFields.value && autocompleteFields.fields.length > 0) {
131-
const rows = await knex(tableName)
129+
const rows = await knex(tableNameWithSchema)
132130
.select(autocompleteFields.fields)
133131
.modify((builder) => {
134132
/*eslint-disable*/
@@ -143,10 +141,11 @@ export class DataAccessObjectMssql extends BasicDataAccessObject implements IDat
143141
/*eslint-enable*/
144142
})
145143
.limit(DAO_CONSTANTS.AUTOCOMPLETE_ROW_LIMIT);
144+
const rowsCount = await this.getRowsCount(tableNameWithoutSchema, null);
146145
rowsRO = {
147146
data: rows,
148147
pagination: {} as any,
149-
large_dataset: false,
148+
large_dataset: rowsCount >= DAO_CONSTANTS.LARGE_DATASET_ROW_LIMIT,
150149
};
151150

152151
return rowsRO;
@@ -156,8 +155,7 @@ export class DataAccessObjectMssql extends BasicDataAccessObject implements IDat
156155
tableSettings.ordering_field = availableFields[0];
157156
tableSettings.ordering = QueryOrderingEnum.ASC;
158157
}
159-
const rows = await knex(tableName)
160-
.select(availableFields)
158+
const rowsCountQuery = knex(tableNameWithSchema)
161159
.modify((builder) => {
162160
/*eslint-disable*/
163161
let { search_fields } = tableSettings;
@@ -218,23 +216,25 @@ export class DataAccessObjectMssql extends BasicDataAccessObject implements IDat
218216
}
219217
}
220218
}
221-
})
222-
.orderBy(tableSettings.ordering_field, tableSettings.ordering)
223-
.paginate({
224-
perPage: perPage,
225-
currentPage: page,
226-
isLengthAware: true,
227219
});
228-
const { data } = rows;
229-
const receivedPagination = rows.pagination;
220+
const rowsDataQuery = rowsCountQuery.clone();
221+
222+
const rowsCount = await this.getRowsCount(tableNameWithoutSchema, rowsCountQuery);
223+
224+
const rows = await rowsDataQuery
225+
.select(availableFields)
226+
.orderBy(tableSettings.ordering_field, tableSettings.ordering)
227+
.limit(perPage)
228+
.offset((page - 1) * perPage);
229+
230230
const pagination = {
231-
total: receivedPagination.total ? receivedPagination.total : rowsCount,
232-
lastPage: receivedPagination.lastPage ? receivedPagination.lastPage : lastPage,
233-
perPage: receivedPagination.perPage,
234-
currentPage: receivedPagination.currentPage,
231+
total: rowsCount,
232+
lastPage: Math.ceil(rowsCount / perPage),
233+
perPage: perPage,
234+
currentPage: page,
235235
};
236236
rowsRO = {
237-
data,
237+
data: rows,
238238
pagination,
239239
large_dataset: rowsCount >= DAO_CONSTANTS.LARGE_DATASET_ROW_LIMIT,
240240
};
@@ -476,7 +476,7 @@ WHERE
476476
const offset = (page - 1) * perPage;
477477
const knex = await this.configureKnex();
478478
const [rowsCount, tableStructure, tableSchema] = await Promise.all([
479-
this.getRowsCount(tableName),
479+
this.getRowsCount(tableName, null),
480480
this.getTableStructure(tableName),
481481
this.getSchemaName(tableName),
482482
]);
@@ -591,9 +591,9 @@ WHERE
591591
return tableSchema;
592592
}
593593

594-
private async getRowsCount(tableName: string): Promise<number> {
594+
private async getRowsCount(tableName: string, countRowsQB: Knex.QueryBuilder<any, any[]> | null): Promise<number> {
595595
const knex = await this.configureKnex();
596-
const countQueryResult = await knex.raw(
596+
const fastCountQueryResult = await knex.raw(
597597
`SELECT QUOTENAME(SCHEMA_NAME(sOBJ.schema_id)) + '.' + QUOTENAME(sOBJ.name) AS [TableName]
598598
, SUM(sdmvPTNS.row_count) AS [RowCount]
599599
FROM
@@ -612,8 +612,16 @@ WHERE
612612
ORDER BY [TableName]`,
613613
[tableName],
614614
);
615-
const rowsCount = countQueryResult[0].RowCount;
616-
return parseInt(rowsCount);
615+
const fastRowsCount = parseInt(fastCountQueryResult[0].RowCount);
616+
if (fastRowsCount >= DAO_CONSTANTS.LARGE_DATASET_ROW_LIMIT) {
617+
return fastRowsCount;
618+
}
619+
if (countRowsQB) {
620+
const slowRowsCountQueryResult = await countRowsQB.count('*');
621+
const slowRowsCount = Object.values(slowRowsCountQueryResult[0])[0];
622+
return slowRowsCount as number;
623+
}
624+
return fastRowsCount;
617625
}
618626

619627
private async getSchemaNameWithoutBrackets(tableName: string): Promise<string> {

0 commit comments

Comments
 (0)