Skip to content

Commit 19149be

Browse files
alex-strukalex-struk
authored
Merge pull request #210 from bcgov/fix/documents-failed-filter-conversion-failed
fix(document): purged-doc handling + OCR content + prod infra hardening
2 parents ed41aa6 + 6a3aaeb commit 19149be

25 files changed

Lines changed: 10954 additions & 4076 deletions

File tree

‎apps/backend-services/src/document/document-db.service.spec.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,40 @@ describe("DocumentDbService", () => {
239239
});
240240
});
241241

242+
it("should expand the 'failed' status filter to include conversion_failed", async () => {
243+
const docs = [{ ...makeDocument(), workflowVersion: null }];
244+
mockPrismaDocument.findMany.mockResolvedValue(docs);
245+
mockPrismaDocument.count.mockResolvedValue(1);
246+
247+
await service.findAllDocuments(undefined, { status: "failed" });
248+
249+
const expectedWhere = {
250+
status: { in: ["failed", "conversion_failed"] },
251+
};
252+
expect(mockPrismaDocument.findMany).toHaveBeenCalledWith(
253+
expect.objectContaining({ where: expectedWhere }),
254+
);
255+
expect(mockPrismaDocument.count).toHaveBeenCalledWith({
256+
where: expectedWhere,
257+
});
258+
});
259+
260+
it("should match a non-failed status filter exactly", async () => {
261+
const docs = [{ ...makeDocument(), workflowVersion: null }];
262+
mockPrismaDocument.findMany.mockResolvedValue(docs);
263+
mockPrismaDocument.count.mockResolvedValue(1);
264+
265+
await service.findAllDocuments(undefined, {
266+
status: DocumentStatus.complete,
267+
});
268+
269+
expect(mockPrismaDocument.findMany).toHaveBeenCalledWith(
270+
expect.objectContaining({
271+
where: { status: DocumentStatus.complete },
272+
}),
273+
);
274+
});
275+
242276
it("should throw if prisma throws", async () => {
243277
mockPrismaDocument.findMany.mockRejectedValue(new Error("DB error"));
244278
mockPrismaDocument.count.mockResolvedValue(0);

‎apps/backend-services/src/document/document-db.service.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,8 +178,13 @@ export class DocumentDbService {
178178
}
179179

180180
// Status filter
181+
// The "failed" filter covers both failure states surfaced under a single
182+
// "Failed" bucket in the UI: extraction failures and PDF conversion failures.
181183
if (options?.status && options.status !== "all") {
182-
where.status = options.status;
184+
where.status =
185+
options.status === "failed"
186+
? { in: ["failed", "conversion_failed"] }
187+
: options.status;
183188
}
184189

185190
// Source filter

‎apps/backend-services/src/document/document.controller.spec.ts‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { GroupRole } from "@generated/client";
22
import {
33
BadRequestException,
44
ForbiddenException,
5+
GoneException,
56
NotFoundException,
67
} from "@nestjs/common";
78
import { mockAppLogger } from "@/testUtils/mockAppLogger";
@@ -540,6 +541,111 @@ describe("DocumentController", () => {
540541
controller.downloadDocument("1", res, mockReq as any),
541542
).rejects.toThrow(NotFoundException);
542543
});
544+
545+
it("should throw GoneException when the document has been purged", async () => {
546+
documentService.findDocument.mockResolvedValue({
547+
id: "1",
548+
file_path: "cuid/ocr/file.pdf",
549+
original_filename: "file.pdf",
550+
group_id: mockGroupId,
551+
purged_at: new Date(),
552+
} as any);
553+
const res: any = {};
554+
await expect(
555+
controller.downloadDocument("1", res, mockReq as any),
556+
).rejects.toThrow(GoneException);
557+
expect(blobStorage.read).not.toHaveBeenCalled();
558+
});
559+
});
560+
561+
describe("viewDocument", () => {
562+
const mockReq = createMockReq();
563+
564+
it("should stream the normalized PDF when present", async () => {
565+
documentService.findDocument.mockResolvedValue({
566+
id: "1",
567+
normalized_file_path: "cuid/ocr/normalized.pdf",
568+
group_id: mockGroupId,
569+
purged_at: null,
570+
} as any);
571+
blobStorage.read.mockResolvedValue(Buffer.from("pdf"));
572+
const res: any = { setHeader: jest.fn(), send: jest.fn() };
573+
await controller.viewDocument("1", res, mockReq as any);
574+
expect(res.setHeader).toHaveBeenCalledWith(
575+
"Content-Type",
576+
"application/pdf",
577+
);
578+
expect(res.send).toHaveBeenCalledWith(Buffer.from("pdf"));
579+
});
580+
581+
it("should throw GoneException when the document has been purged", async () => {
582+
documentService.findDocument.mockResolvedValue({
583+
id: "1",
584+
normalized_file_path: "cuid/ocr/normalized.pdf",
585+
group_id: mockGroupId,
586+
purged_at: new Date(),
587+
} as any);
588+
const res: any = { setHeader: jest.fn(), send: jest.fn() };
589+
await expect(
590+
controller.viewDocument("1", res, mockReq as any),
591+
).rejects.toThrow(GoneException);
592+
expect(blobStorage.read).not.toHaveBeenCalled();
593+
expect(mockAuditService.recordEvent).not.toHaveBeenCalled();
594+
});
595+
596+
it("should throw NotFoundException when normalized PDF path is missing", async () => {
597+
documentService.findDocument.mockResolvedValue({
598+
id: "1",
599+
normalized_file_path: null,
600+
group_id: mockGroupId,
601+
purged_at: null,
602+
} as any);
603+
const res: any = { setHeader: jest.fn(), send: jest.fn() };
604+
await expect(
605+
controller.viewDocument("1", res, mockReq as any),
606+
).rejects.toThrow(NotFoundException);
607+
});
608+
609+
it("should throw NotFoundException (not 500) when the blob read fails", async () => {
610+
documentService.findDocument.mockResolvedValue({
611+
id: "1",
612+
normalized_file_path: "cuid/ocr/normalized.pdf",
613+
group_id: mockGroupId,
614+
purged_at: null,
615+
} as any);
616+
blobStorage.read.mockRejectedValue(new Error("NoSuchKey"));
617+
const res: any = { setHeader: jest.fn(), send: jest.fn() };
618+
await expect(
619+
controller.viewDocument("1", res, mockReq as any),
620+
).rejects.toThrow(NotFoundException);
621+
});
622+
623+
it("should throw ForbiddenException if user is not a group member", async () => {
624+
const notMemberReq = {
625+
resolvedIdentity: {
626+
userId: "user-1",
627+
isSystemAdmin: false,
628+
groupRoles: {},
629+
},
630+
};
631+
documentService.findDocument.mockResolvedValue({
632+
id: "1",
633+
normalized_file_path: "cuid/ocr/normalized.pdf",
634+
group_id: mockGroupId,
635+
} as any);
636+
const res: any = { setHeader: jest.fn(), send: jest.fn() };
637+
await expect(
638+
controller.viewDocument("1", res, notMemberReq as any),
639+
).rejects.toThrow(ForbiddenException);
640+
});
641+
642+
it("should throw NotFoundException if document not found", async () => {
643+
documentService.findDocument.mockResolvedValue(null);
644+
const res: any = { setHeader: jest.fn(), send: jest.fn() };
645+
await expect(
646+
controller.viewDocument("1", res, mockReq as any),
647+
).rejects.toThrow(NotFoundException);
648+
});
543649
});
544650

545651
describe("getDocument", () => {

‎apps/backend-services/src/document/document.controller.ts‎

Lines changed: 37 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import {
77
Delete,
88
ForbiddenException,
99
Get,
10+
GoneException,
1011
HttpCode,
1112
HttpStatus,
1213
Inject,
@@ -23,6 +24,7 @@ import {
2324
ApiBody,
2425
ApiConflictResponse,
2526
ApiForbiddenResponse,
27+
ApiGoneResponse,
2628
ApiNoContentResponse,
2729
ApiNotFoundResponse,
2830
ApiOkResponse,
@@ -561,6 +563,9 @@ export class DocumentController {
561563
@ApiNotFoundResponse({
562564
description: "Document not found or normalized PDF unavailable",
563565
})
566+
@ApiGoneResponse({
567+
description: "Document was purged per its retention policy; original gone",
568+
})
564569
@ApiUnauthorizedResponse({ description: "Not authenticated" })
565570
@ApiForbiddenResponse({ description: "Access denied: not a group member" })
566571
async viewDocument(
@@ -578,6 +583,13 @@ export class DocumentController {
578583

579584
identityCanAccessGroup(req.resolvedIdentity, document.group_id);
580585

586+
if (document.purged_at) {
587+
throw new GoneException(
588+
`Document ${documentId} was purged per its retention policy; ` +
589+
`the original is no longer available. Extracted data is retained.`,
590+
);
591+
}
592+
581593
if (!document.normalized_file_path) {
582594
throw new NotFoundException(
583595
`Normalized PDF is not available for document: ${documentId}`,
@@ -594,9 +606,19 @@ export class DocumentController {
594606
payload: { action: "view" },
595607
});
596608

597-
const fileBuffer = await this.blobStorage.read(
598-
validateBlobFilePath(document.normalized_file_path),
599-
);
609+
let fileBuffer: Buffer;
610+
try {
611+
fileBuffer = await this.blobStorage.read(
612+
validateBlobFilePath(document.normalized_file_path),
613+
);
614+
} catch (error) {
615+
this.logger.error(
616+
`Failed to read normalized PDF for document ${documentId}: ${getErrorMessage(error)}`,
617+
);
618+
throw new NotFoundException(
619+
`Normalized PDF is not available for document: ${documentId}`,
620+
);
621+
}
600622

601623
res.setHeader("Content-Type", "application/pdf");
602624
res.setHeader("Content-Disposition", 'inline; filename="document.pdf"');
@@ -671,6 +693,9 @@ export class DocumentController {
671693
description: "Returns the original file buffer",
672694
})
673695
@ApiNotFoundResponse({ description: "Document not found or file missing" })
696+
@ApiGoneResponse({
697+
description: "Document was purged per its retention policy; original gone",
698+
})
674699
@ApiUnauthorizedResponse({ description: "Not authenticated" })
675700
@ApiForbiddenResponse({ description: "Access denied: not a group member" })
676701
async downloadDocument(
@@ -690,6 +715,13 @@ export class DocumentController {
690715

691716
identityCanAccessGroup(req.resolvedIdentity, document.group_id);
692717

718+
if (document.purged_at) {
719+
throw new GoneException(
720+
`Document ${documentId} was purged per its retention policy; ` +
721+
`the original is no longer available. Extracted data is retained.`,
722+
);
723+
}
724+
693725
// Read file from blob storage using the blob key
694726
const filePath = validateBlobFilePath(document.file_path);
695727
const fileBuffer = await this.blobStorage.read(filePath);
@@ -727,7 +759,8 @@ export class DocumentController {
727759

728760
if (
729761
error instanceof NotFoundException ||
730-
error instanceof ForbiddenException
762+
error instanceof ForbiddenException ||
763+
error instanceof GoneException
731764
) {
732765
throw error;
733766
}

‎apps/backend-services/src/document/dto/document-data.dto.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,19 @@ export class DocumentDataDto {
4444
@ApiProperty()
4545
updated_at!: Date;
4646

47+
@ApiProperty({
48+
required: false,
49+
nullable: true,
50+
type: "string",
51+
format: "date-time",
52+
description:
53+
"Set once the ephemeral-cleanup janitor has purged the document's " +
54+
"blobs (original and/or normalized PDF) per its workflow's retention " +
55+
"policy. When set, the OCR result is retained but /view and /download " +
56+
"return 410 Gone. Null = not purged.",
57+
})
58+
purged_at?: Date | null;
59+
4760
@ApiProperty({ required: false, nullable: true, type: "string" })
4861
apim_request_id?: string | null;
4962

‎apps/frontend/package.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,12 @@
4242
"react": "19.2.1",
4343
"react-dom": "19.2.1",
4444
"react-konva": "19.2.1",
45+
"react-markdown": "10.1.0",
4546
"react-router-dom": "7.17.0",
4647
"recharts": "3.7.0",
48+
"rehype-raw": "7.0.0",
49+
"rehype-sanitize": "6.0.0",
50+
"remark-gfm": "4.0.1",
4751
"zod": "3.25.76"
4852
},
4953
"devDependencies": {

0 commit comments

Comments
 (0)