Skip to content

Commit bd80569

Browse files
VerdinKirillVerdinKirill
authored
Merge pull request #60 from moevm/pagination-on-tables
Pagination on tables
2 parents 86757b5 + 534a94e commit bd80569

25 files changed

Lines changed: 1216 additions & 234 deletions

‎backend/src/queries/backup.queries.ts‎

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -256,7 +256,66 @@ export async function insertBackupHistory(record: Omit<BackupHistoryRecord, "cre
256256
return document;
257257
}
258258

259-
export async function listBackupHistory(limit = 50) {
259+
type BackupHistoryFilters = {
260+
fileName?: string;
261+
createdAtFrom?: string;
262+
createdAtTo?: string;
263+
sizeMin?: number;
264+
sizeMax?: number;
265+
actorName?: string;
266+
operation?: string;
267+
status?: string;
268+
details?: string;
269+
};
270+
271+
function backupHistoryFilter(filters: BackupHistoryFilters) {
272+
const filter: Document = {};
273+
if (filters.fileName) filter.fileName = { $regex: filters.fileName, $options: "i" };
274+
if (filters.actorName) filter.actorName = { $regex: filters.actorName, $options: "i" };
275+
if (filters.operation && filters.operation !== "all") filter.operation = filters.operation;
276+
if (filters.status && filters.status !== "all") filter.status = filters.status;
277+
if (filters.createdAtFrom || filters.createdAtTo) {
278+
filter.createdAt = {};
279+
if (filters.createdAtFrom) (filter.createdAt as Document).$gte = new Date(filters.createdAtFrom);
280+
if (filters.createdAtTo) (filter.createdAt as Document).$lte = new Date(filters.createdAtTo);
281+
}
282+
if (filters.sizeMin !== undefined || filters.sizeMax !== undefined) {
283+
filter.compressedSizeBytes = {};
284+
if (filters.sizeMin !== undefined) (filter.compressedSizeBytes as Document).$gte = filters.sizeMin;
285+
if (filters.sizeMax !== undefined) (filter.compressedSizeBytes as Document).$lte = filters.sizeMax;
286+
}
287+
if (filters.details) {
288+
filter.$or = [
289+
{ errorMessage: { $regex: filters.details, $options: "i" } },
290+
{ backupVersion: { $regex: filters.details, $options: "i" } },
291+
{ appVersion: { $regex: filters.details, $options: "i" } },
292+
];
293+
}
294+
return filter;
295+
}
296+
297+
export async function listBackupHistory(page = 1, limit = 50, filters: BackupHistoryFilters = {}) {
298+
const safePage = Math.max(1, page);
299+
const safeLimit = Math.max(1, Math.min(limit, 200));
300+
const filter = backupHistoryFilter(filters);
301+
const [items, total] = await Promise.all([
302+
backupHistoryCollection()
303+
.find(filter)
304+
.sort({ createdAt: -1 })
305+
.skip((safePage - 1) * safeLimit)
306+
.limit(safeLimit)
307+
.toArray(),
308+
backupHistoryCollection().countDocuments(filter),
309+
]);
310+
return {
311+
items: items.map(({ payload, payloadFileId, ...item }) => ({ ...item, hasPayload: Boolean(payloadFileId) })),
312+
total,
313+
page: safePage,
314+
limit: safeLimit,
315+
};
316+
}
317+
318+
export async function listBackupHistoryLegacy(limit = 50) {
260319
const items = await backupHistoryCollection()
261320
.find({})
262321
.sort({ createdAt: -1 })

‎backend/src/queries/clustering.queries.ts‎

Lines changed: 109 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,15 +12,119 @@ export async function insertClusteringRun(document: Document) {
1212
return result.insertedId;
1313
}
1414

15-
export async function getRunWithSessions(runId: string) {
15+
type ResultPaginationOptions = {
16+
sessionsPage?: number;
17+
sessionsLimit?: number;
18+
clustersPage?: number;
19+
clustersLimit?: number;
20+
sessionSearch?: string;
21+
sessionCluster?: string;
22+
sessionStatus?: string;
23+
sessionDateFrom?: string;
24+
sessionDateTo?: string;
25+
distanceMin?: number;
26+
distanceMax?: number;
27+
clusterId?: string;
28+
clusterSizeMin?: number;
29+
clusterSizeMax?: number;
30+
clusterCentroid?: string;
31+
clusterAnomalyMin?: number;
32+
clusterAnomalyMax?: number;
33+
};
34+
35+
function safePagination(page = 1, limit = 10) {
36+
return {
37+
page: Math.max(1, page),
38+
limit: Math.max(1, Math.min(limit, 200)),
39+
};
40+
}
41+
42+
function clusterLabel(clusterId: unknown) {
43+
const value = Number(clusterId);
44+
if (!Number.isFinite(value)) return "—";
45+
return value < 0 ? "noise" : `C${value + 1}`;
46+
}
47+
48+
function matchesText(value: unknown, query: string | undefined) {
49+
return !query || String(value ?? "").toLowerCase().includes(query.toLowerCase());
50+
}
51+
52+
function matchesRange(value: unknown, min: number | undefined, max: number | undefined) {
53+
const numeric = Number(value ?? 0);
54+
return numeric >= (min ?? Number.NEGATIVE_INFINITY) && numeric <= (max ?? Number.POSITIVE_INFINITY);
55+
}
56+
57+
function matchesDate(value: unknown, from: string | undefined, to: string | undefined) {
58+
const timestamp = value ? new Date(String(value)).getTime() : Number.NaN;
59+
const fromTime = from ? new Date(from).getTime() : Number.NEGATIVE_INFINITY;
60+
const toTime = to ? new Date(to).getTime() : Number.POSITIVE_INFINITY;
61+
if (Number.isNaN(timestamp)) return !from && !to;
62+
return timestamp >= fromTime && timestamp <= toTime;
63+
}
64+
65+
function pageSlice<T>(items: T[], page: number, limit: number) {
66+
return items.slice((page - 1) * limit, page * limit);
67+
}
68+
69+
export async function getRunWithSessions(runId: string, options: ResultPaginationOptions = {}) {
70+
if (!ObjectId.isValid(runId)) return null;
1671
const run = await getCollection("clustering_runs").findOne({ _id: new ObjectId(runId) });
1772
if (!run) return null;
1873

19-
const sessionIds = ((run.results as Document)?.sessionAssignments as Document[] | undefined)?.map((item) => item.sessionId as ObjectId) ?? [];
20-
const sessions = await getCollection("sessions").find({ _id: { $in: sessionIds } }).toArray();
21-
const studentIds = sessions.map((session) => session.studentId as ObjectId);
74+
const sessionPagination = safePagination(options.sessionsPage, options.sessionsLimit ?? 15);
75+
const clusterPagination = safePagination(options.clustersPage, options.clustersLimit ?? 10);
76+
const assignments = ((run.results as Document)?.sessionAssignments as Document[] | undefined) ?? [];
77+
const allSessionIds = assignments.map((item) => item.sessionId as ObjectId).filter(Boolean);
78+
const allSessions = await getCollection("sessions").find({ _id: { $in: allSessionIds } }).toArray();
79+
const allStudentIds = allSessions.map((session) => session.studentId as ObjectId).filter(Boolean);
80+
const allStudents = await getCollection("students").find({ _id: { $in: allStudentIds } }).toArray();
81+
const sessionsById = new Map(allSessions.map((session) => [String(session._id), session]));
82+
const studentsById = new Map(allStudents.map((student) => [String(student._id), student]));
83+
84+
const filteredAssignments = assignments.filter((assignment) => {
85+
const session = sessionsById.get(String(assignment.sessionId));
86+
const student = studentsById.get(String(session?.studentId ?? ""));
87+
const label = clusterLabel(assignment.clusterId);
88+
return (
89+
matchesText(student?.fullName ?? session?.studentId ?? assignment.sessionId, options.sessionSearch) &&
90+
(!options.sessionCluster || options.sessionCluster === "all" || label === options.sessionCluster) &&
91+
(!options.sessionStatus || options.sessionStatus === "all" || (options.sessionStatus === "anomaly" ? Boolean(assignment.isAnomaly) : !assignment.isAnomaly)) &&
92+
matchesDate(session?.startTime, options.sessionDateFrom, options.sessionDateTo) &&
93+
matchesRange(assignment.distanceToCentroid, options.distanceMin, options.distanceMax)
94+
);
95+
});
96+
97+
const pagedAssignments = pageSlice(filteredAssignments, sessionPagination.page, sessionPagination.limit);
98+
const pagedSessionIds = pagedAssignments.map((item) => item.sessionId as ObjectId).filter(Boolean);
99+
const sessions = allSessions.filter((session) => pagedSessionIds.some((id) => String(id) === String(session._id)));
100+
const studentIds = sessions.map((session) => session.studentId as ObjectId).filter(Boolean);
22101
const students = await getCollection("students").find({ _id: { $in: studentIds } }).toArray();
23-
return { run, sessions, students };
102+
103+
const clusters = ((run.results as Document)?.clusters as Document[] | undefined) ?? [];
104+
const filteredClusters = clusters.filter((cluster, index) => {
105+
const label = clusterLabel(cluster.clusterId ?? index);
106+
const centroid = Array.isArray(cluster.centroid) ? cluster.centroid.slice(0, 3).join(", ") : "—";
107+
const anomalyRate = Number(cluster.anomalyRate ?? 0) * 100;
108+
return (
109+
matchesText(label, options.clusterId) &&
110+
matchesRange(cluster.size, options.clusterSizeMin, options.clusterSizeMax) &&
111+
matchesText(centroid, options.clusterCentroid) &&
112+
matchesRange(anomalyRate, options.clusterAnomalyMin, options.clusterAnomalyMax)
113+
);
114+
});
115+
const pagedClusters = pageSlice(filteredClusters, clusterPagination.page, clusterPagination.limit);
116+
117+
return {
118+
run,
119+
assignments: pagedAssignments,
120+
clusters: pagedClusters,
121+
sessions,
122+
students,
123+
pagination: {
124+
sessions: { total: filteredAssignments.length, page: sessionPagination.page, limit: sessionPagination.limit },
125+
clusters: { total: filteredClusters.length, page: clusterPagination.page, limit: clusterPagination.limit },
126+
},
127+
};
24128
}
25129

26130
export async function deleteClusteringRun(runId: string) {

‎backend/src/queries/upload.queries.ts‎

Lines changed: 71 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,27 @@ function parseTimeBound(value: string | undefined, fallback: number) {
3636

3737
export async function getUploadLog(
3838
uploadId: string,
39-
filters: { level?: string; lineFrom?: number; lineTo?: number; timeFrom?: string; timeTo?: string },
39+
filters: {
40+
level?: string;
41+
file?: string;
42+
entityType?: string;
43+
lineFrom?: number;
44+
lineTo?: number;
45+
timeFrom?: string;
46+
timeTo?: string;
47+
search?: string;
48+
problemFile?: string;
49+
problemLineFrom?: number;
50+
problemLineTo?: number;
51+
problemContent?: string;
52+
problemError?: string;
53+
unmappedId?: string;
54+
unmappedMatch?: string;
55+
unmappedReason?: string;
56+
table?: "log" | "problems" | "unmapped";
57+
page?: number;
58+
limit?: number;
59+
},
4060
) {
4161
if (!ObjectId.isValid(uploadId)) return null;
4262

@@ -64,6 +84,7 @@ export async function getUploadLog(
6484
return (!filters.level || entry.level === filters.level) && line >= lineFrom && line <= lineTo && matchesTime;
6585
}),
6686
);
87+
const problemRows = processingLog.filter((entry) => String(entry.level) !== "info");
6788

6889
const firstUpload = uploads[0];
6990
const userId = firstUpload?.userId as ObjectId | undefined;
@@ -100,13 +121,57 @@ export async function getUploadLog(
100121
upload.createdByName = createdByName;
101122
}
102123

124+
const unresolvedStudents = mergeUnresolvedStudents(
125+
[],
126+
uploads.flatMap((upload) => (upload.unresolvedStudents as UnresolvedStudent[] | undefined) ?? []),
127+
);
128+
const safePage = Math.max(1, filters.page ?? 1);
129+
const safeLimit = Math.max(1, Math.min(filters.limit ?? 200, 200));
130+
const usePagination = Boolean(filters.table && filters.page && filters.limit);
131+
const slice = <T>(items: T[]) => (usePagination ? items.slice((safePage - 1) * safeLimit, safePage * safeLimit) : items);
132+
const textMatch = (value: unknown, query: string | undefined) => !query || String(value ?? "").toLowerCase().includes(query.toLowerCase());
133+
const rangeMatch = (value: unknown, from: number | undefined, to: number | undefined) => {
134+
const numeric = Number(value ?? 0);
135+
return numeric >= (from ?? Number.NEGATIVE_INFINITY) && numeric <= (to ?? Number.POSITIVE_INFINITY);
136+
};
137+
const entityMatch = (value: unknown, entityType: string | undefined) => {
138+
if (!entityType || entityType === "all") return true;
139+
const raw = String(value ?? "");
140+
if (entityType === "student") return raw.includes("student");
141+
if (entityType === "camera") return raw.includes("ocr") || raw.includes("camera");
142+
return !raw.includes("student") && !raw.includes("ocr") && !raw.includes("camera");
143+
};
144+
145+
const filteredLog = processingLog.filter((entry) => {
146+
return (
147+
(!filters.file || filters.file === "all" || String(entry.sourceFileKey ?? "csv") === filters.file) &&
148+
entityMatch(entry.entityType, filters.entityType) &&
149+
textMatch(entry.message, filters.search)
150+
);
151+
});
152+
const filteredProblemRows = problemRows.filter((entry) => {
153+
return (
154+
textMatch(entry.sourceFileKey ?? "csv", filters.problemFile) &&
155+
rangeMatch(entry.line, filters.problemLineFrom, filters.problemLineTo) &&
156+
textMatch(entry.rowContent ?? "—", filters.problemContent) &&
157+
textMatch(entry.message ?? "—", filters.problemError)
158+
);
159+
});
160+
const filteredUnresolvedStudents = unresolvedStudents.filter((student) => {
161+
return textMatch(student.externalId ?? (student as Document).id, filters.unmappedId) && textMatch(student.possibleMatch ?? "—", filters.unmappedMatch) && textMatch(student.reason ?? "Не сопоставлен", filters.unmappedReason);
162+
});
163+
164+
const activeItems = filters.table === "problems" ? filteredProblemRows : filters.table === "unmapped" ? filteredUnresolvedStudents : filteredLog;
165+
103166
return {
104167
upload,
105-
processingLog,
106-
unresolvedStudents: mergeUnresolvedStudents(
107-
[],
108-
uploads.flatMap((upload) => (upload.unresolvedStudents as UnresolvedStudent[] | undefined) ?? []),
109-
),
168+
processingLog: filters.table === "unmapped" ? [] : slice(filters.table === "problems" ? filteredProblemRows : filteredLog),
169+
unresolvedStudents: filters.table === "unmapped" ? slice(filteredUnresolvedStudents) : usePagination ? [] : filteredUnresolvedStudents,
170+
pagination: {
171+
total: activeItems.length,
172+
page: safePage,
173+
limit: safeLimit,
174+
},
110175
};
111176
}
112177

‎backend/src/routes/backup.routes.ts‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,11 @@ function sendBackupDownload(res: Response, fileName: string, buffer: Buffer, con
4747
res.send(buffer);
4848
}
4949

50+
function numericQuery(value: unknown) {
51+
const parsed = Number(value);
52+
return Number.isFinite(parsed) ? parsed : undefined;
53+
}
54+
5055
backupRouter.get(
5156
"/backup/export",
5257
auth,
@@ -62,8 +67,21 @@ backupRouter.get(
6267
auth,
6368
adminOnly,
6469
asyncHandler(async (req, res) => {
65-
const limit = Number(req.query.limit ?? 50);
66-
res.json({ items: await getBackupHistory(limit) });
70+
const page = numericQuery(req.query.page) ?? 1;
71+
const limit = numericQuery(req.query.limit) ?? 50;
72+
res.json(
73+
await getBackupHistory(page, limit, {
74+
fileName: String(req.query.fileName ?? ""),
75+
createdAtFrom: String(req.query.createdAtFrom ?? ""),
76+
createdAtTo: String(req.query.createdAtTo ?? ""),
77+
sizeMin: numericQuery(req.query.sizeMin),
78+
sizeMax: numericQuery(req.query.sizeMax),
79+
actorName: String(req.query.actorName ?? ""),
80+
operation: String(req.query.operation ?? ""),
81+
status: String(req.query.status ?? ""),
82+
details: String(req.query.details ?? ""),
83+
}),
84+
);
6785
}),
6886
);
6987

‎backend/src/routes/clustering.routes.ts‎

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,15 @@ import { deleteClusteringRun, getRunWithSessions } from "../queries/clustering.q
88
import type { AuthUser } from "../schema/user.schema.js";
99
import { recordRequestAuditEvent } from "../services/audit.service.js";
1010
import { createClusteringRun, getClusteringPreview, type ClusteringRunInput } from "../services/clustering.service.js";
11-
import { serializeDocument } from "../utils/query.js";
11+
import { getQuery, serializeDocument } from "../utils/query.js";
1212

1313
export const clusteringRouter = Router();
1414

15+
function numericQuery(value: unknown) {
16+
const parsed = Number(value);
17+
return Number.isFinite(parsed) ? parsed : undefined;
18+
}
19+
1520
clusteringRouter.post(
1621
"/clustering-runs/preview",
1722
auth,
@@ -55,16 +60,37 @@ clusteringRouter.get(
5560
"/results/:runId",
5661
auth,
5762
asyncHandler(async (req, res) => {
58-
const data = await getRunWithSessions(String(req.params.runId));
63+
const data = await getRunWithSessions(String(req.params.runId), {
64+
sessionsPage: numericQuery(req.query.sessionsPage),
65+
sessionsLimit: numericQuery(req.query.sessionsLimit),
66+
clustersPage: numericQuery(req.query.clustersPage),
67+
clustersLimit: numericQuery(req.query.clustersLimit),
68+
sessionSearch: getQuery(req.query, "sessionSearch"),
69+
sessionCluster: getQuery(req.query, "sessionCluster"),
70+
sessionStatus: getQuery(req.query, "sessionStatus"),
71+
sessionDateFrom: getQuery(req.query, "sessionDateFrom"),
72+
sessionDateTo: getQuery(req.query, "sessionDateTo"),
73+
distanceMin: numericQuery(req.query.distanceMin),
74+
distanceMax: numericQuery(req.query.distanceMax),
75+
clusterId: getQuery(req.query, "clusterId"),
76+
clusterSizeMin: numericQuery(req.query.clusterSizeMin),
77+
clusterSizeMax: numericQuery(req.query.clusterSizeMax),
78+
clusterCentroid: getQuery(req.query, "clusterCentroid"),
79+
clusterAnomalyMin: numericQuery(req.query.clusterAnomalyMin),
80+
clusterAnomalyMax: numericQuery(req.query.clusterAnomalyMax),
81+
});
5982
if (!data) {
6083
res.status(404).json({ message: "Запуск не найден" });
6184
return;
6285
}
6386

6487
res.json({
6588
run: serializeDocument(data.run),
89+
assignments: serializeDocument(data.assignments),
90+
clusters: serializeDocument(data.clusters),
6691
sessions: serializeDocument(data.sessions),
6792
students: serializeDocument(data.students),
93+
pagination: data.pagination,
6894
});
6995
}),
7096
);

0 commit comments

Comments
 (0)