Merge pull request #189 from bcgov/AI-1239

Enforce 100MB limit for classifier uploads

Author: kmandryk

apps/backend-services/src/azure/azure.controller.spec.ts

@@ -4,6 +4,7 @@ import {
   ForbiddenException,
   InternalServerErrorException,
   NotFoundException,
+  PayloadTooLargeException,
 } from "@nestjs/common";
 import { Request } from "express";
 import type { AuditService } from "@/audit/audit.service";
@@ -212,6 +213,22 @@ describe("AzureController", () => {
         ),
       ).rejects.toThrow(NotFoundException);
     });
+    it("should throw PayloadTooLargeException when a file exceeds 100MB", async () => {
+      classifierService.findClassifierModel.mockResolvedValue({ id: "1" });
+      const oversizedFile = {
+        ...mockFile,
+        originalname: "large.pdf",
+        size: 100 * 1024 * 1024 + 1,
+      };
+      await expect(
+        controller.uploadClassifierDocuments(
+          [oversizedFile],
+          { name: "c1", label: "l1" },
+          "g1",
+        ),
+      ).rejects.toThrow(PayloadTooLargeException);
+      expect(storageService.write).not.toHaveBeenCalled();
+    });
   });
 
   describe("deleteClassifierDocuments", () => {

apps/backend-services/src/azure/azure.controller.ts

@@ -12,11 +12,13 @@ import {
   NotFoundException,
   Param,
   Patch,
+  PayloadTooLargeException,
   Post,
   Query,
   Req,
   UploadedFile,
   UploadedFiles,
+  UseFilters,
   UseInterceptors,
 } from "@nestjs/common";
 import { FileInterceptor, FilesInterceptor } from "@nestjs/platform-express";
@@ -77,6 +79,7 @@ import {
   buildBlobPrefixPath,
   OperationCategory,
 } from "@/blob-storage/storage-path-builder";
+import { MulterExceptionFilter } from "@/filters/multer-exception.filter";
 import { GroupRole } from "@/generated/edge";
 import { AppLoggerService } from "@/logging/app-logger.service";
 
@@ -229,7 +232,14 @@ export class AzureController {
     summary: "Upload training documents",
     description: "Upload training documents for a classifier.",
   })
-  @UseInterceptors(FilesInterceptor("files"))
+  @UseInterceptors(
+    FilesInterceptor("files", 100, {
+      limits: {
+        fileSize: 100 * 1024 * 1024, // 100MB per file
+      },
+    }),
+  )
+  @UseFilters(MulterExceptionFilter)
   @ApiConsumes("multipart/form-data")
   @ApiQuery({ name: "group_id", required: true, description: "Group ID" })
   @ApiBody({
@@ -269,6 +279,15 @@ export class AzureController {
       throw new NotFoundException("No existing record of classifier model.");
     }
 
+    const maxFileSize = 100 * 1024 * 1024; // 100MB
+    for (const file of files) {
+      if (file.size > maxFileSize) {
+        throw new PayloadTooLargeException(
+          `File ${file.originalname} exceeds maximum size of 100MB`,
+        );
+      }
+    }
+
     const uploadResults: string[] = [];
     for (const file of files) {
       const key = buildBlobFilePath(

apps/backend-services/src/filters/multer-exception.filter.spec.ts

@@ -0,0 +1,48 @@
+import { PayloadTooLargeException } from "@nestjs/common";
+import { MulterError } from "multer";
+import { MulterExceptionFilter } from "./multer-exception.filter";
+
+describe("MulterExceptionFilter", () => {
+  const filter = new MulterExceptionFilter();
+
+  it("should return 413 for LIMIT_FILE_SIZE", () => {
+    const json = jest.fn();
+    const status = jest.fn().mockReturnValue({ json });
+    const host = {
+      switchToHttp: () => ({
+        getResponse: () => ({ status }),
+      }),
+    };
+
+    filter.catch(new MulterError("LIMIT_FILE_SIZE", "files"), host as never);
+
+    expect(status).toHaveBeenCalledWith(413);
+    expect(json).toHaveBeenCalledWith(
+      new PayloadTooLargeException(
+        "File exceeds maximum allowed size",
+      ).getResponse(),
+    );
+  });
+
+  it("should return 400 for other multer errors", () => {
+    const json = jest.fn();
+    const status = jest.fn().mockReturnValue({ json });
+    const host = {
+      switchToHttp: () => ({
+        getResponse: () => ({ status }),
+      }),
+    };
+
+    filter.catch(
+      new MulterError("LIMIT_UNEXPECTED_FILE", "files"),
+      host as never,
+    );
+
+    expect(status).toHaveBeenCalledWith(400);
+    expect(json).toHaveBeenCalledWith({
+      statusCode: 400,
+      message: "Unexpected field",
+      error: "Bad Request",
+    });
+  });
+});

apps/backend-services/src/filters/multer-exception.filter.ts

@@ -0,0 +1,29 @@
+import {
+  ArgumentsHost,
+  Catch,
+  ExceptionFilter,
+  PayloadTooLargeException,
+} from "@nestjs/common";
+import { Response } from "express";
+import { MulterError } from "multer";
+
+@Catch(MulterError)
+export class MulterExceptionFilter implements ExceptionFilter {
+  catch(exception: MulterError, host: ArgumentsHost): void {
+    const response = host.switchToHttp().getResponse<Response>();
+
+    if (exception.code === "LIMIT_FILE_SIZE") {
+      const body = new PayloadTooLargeException(
+        "File exceeds maximum allowed size",
+      ).getResponse();
+      response.status(413).json(body);
+      return;
+    }
+
+    response.status(400).json({
+      statusCode: 400,
+      message: exception.message,
+      error: "Bad Request",
+    });
+  }
+}

docs-md/CLASSIFIER_DOCUMENT_UPLOAD.md

@@ -0,0 +1,31 @@
+# Classifier document upload limits
+
+`POST /api/azure/classifier/documents` accepts multipart training-document uploads for classifiers.
+
+## File size limit
+
+Each uploaded file may be up to **100 MB**, matching the dataset version upload endpoint in `apps/backend-services/src/benchmark/dataset.controller.ts`.
+
+The route configures multer via `FilesInterceptor` with `limits.fileSize: 100 * 1024 * 1024`. Files larger than 100 MB receive **HTTP 413 Payload Too Large** (via `MulterExceptionFilter` for multer rejections and an explicit controller check as a safeguard).
+
+## Load testing context
+
+Load testing previously reported HTTP 500 for uploads above ~64 KB because the classifier route used `FilesInterceptor("files")` without an explicit limit. The `blob-storage` k6 scenario (256 KB blobs) showed a **20.6 %** failure rate until this limit was aligned with the dataset upload endpoint.
+
+See also:
+
+- [`docs-md/LOAD_TEST_REPORT_2026-05.md`](./LOAD_TEST_REPORT_2026-05.md) — Finding 3
+- [`docs-md/LOAD_TESTING.md`](./LOAD_TESTING.md) — blob storage scenario
+
+## Verification
+
+```bash
+# Expect HTTP 201 (classifier must exist; replace group/name/label as needed)
+curl -H "x-api-key: <api-key>" \
+  -F "files=@/path/to/1mb.pdf" \
+  -F "name=<classifier>" \
+  -F "label=<label>" \
+  "http://localhost:3002/api/azure/classifier/documents?group_id=<group>"
+```
+
+Files over 100 MB should return **413**, not 500.

docs-md/openshift-deployment/MANUAL_LOAD_TEST_INSTANCE.md

@@ -159,7 +159,7 @@ The patch lasts for the life of the ConfigMap; re-running **`oc-deploy-instance.
 The matrix runs in [`tools/load-testing/test-matrix.csv`](../../tools/load-testing/test-matrix.csv) surfaced two reproducible issues that are not specific to the load-test instance — they are **product limits** that any operator will hit. They are documented here so the next person running the suite knows what they are seeing and is not blocked by them.
 
 1. **Backend pod becomes unavailable under concurrent large uploads.** Running `payload-sizes` at **`large` (5 MB raw PDF) × 5 VUs** drove the **`backend-services`** pod into a restart that returned **`HTTP 502`** (router→dead pod) followed by **`HTTP 503`** (router→no healthy upstream) for ~30–60 s. **`smoke`** and **`datasets`** were both unreachable during the window. **`large × 1 VU`** runs cleanly. The most likely cause is per-request memory pressure from `JSON.parse` + `Buffer.from(b64,'base64')` + `pdf-lib` normalization at the same time, exceeding the pod's `memory.limit`. Confirm with **`oc describe pod <instance>-backend-services-<id>`** and look for **`Last State: Terminated · Reason: OOMKilled · Exit Code: 137`** after a failing run. Workaround for the load-test instance: raise the deployment **`resources.limits.memory`**. Real fix (in source): stream the upload through pdf-lib instead of holding the full buffer twice, or cap concurrent uploads at the controller via a semaphore.
-2. **Classifier-document upload fails with HTTP 500 above ~64 KB.** The endpoint **`POST /api/azure/classifier/documents`** in **[`apps/backend-services/src/azure/azure.controller.ts`](../../apps/backend-services/src/azure/azure.controller.ts:223)** uses **`@UseInterceptors(FilesInterceptor("files"))`** with no explicit limits. Files **≤ 64 KB** return 201; **≥ 96 KB** consistently return **`HTTP 500 {"statusCode":500,"message":"Internal server error"}`** in under 250 ms. The list and delete endpoints on the same controller (`GET`/`DELETE /api/azure/classifier/documents`) are unaffected. The OCR upload at **`POST /api/upload`** is unaffected because it is JSON-bodied (governed by **`BODY_LIMIT`**), not multipart. Compare with **[`apps/backend-services/src/benchmark/dataset.controller.ts:222`](../../apps/backend-services/src/benchmark/dataset.controller.ts)** where the dataset multipart endpoint explicitly sets **`limits: { fileSize: 100 * 1024 * 1024 }`** — the classifier endpoint should do the same. Without **`oc`** access to read the backend log this could not be confirmed end-to-end, but the failure is fast, deterministic, and size-correlated, which fits a multer **`limits.fileSize`** rejection. Fix: add **`@UseInterceptors(FilesInterceptor("files", { limits: { fileSize: 100 * 1024 * 1024 } }))`** on the classifier upload route, or register a global **`MulterModule.register`** with a project-wide default that matches **`BODY_LIMIT`**.
+2. **Classifier-document upload fails with HTTP 500 above ~64 KB** *(fixed)*. The endpoint **`POST /api/azure/classifier/documents`** now sets an explicit **`limits.fileSize: 100 * 1024 * 1024`** on its `FilesInterceptor`, matching **[`apps/backend-services/src/benchmark/dataset.controller.ts:222`](../../apps/backend-services/src/benchmark/dataset.controller.ts)**. Oversized files return **HTTP 413** via **`MulterExceptionFilter`**. See **[`docs-md/CLASSIFIER_DOCUMENT_UPLOAD.md`](../CLASSIFIER_DOCUMENT_UPLOAD.md)**.
 3. **In-cluster MinIO fills up after a few sustained upload runs.** After roughly **~1,500** documents uploaded across **`upload-ocr`** and **`payload-sizes`** runs at 256 KB–5 MB tiers, **`POST /api/upload`** starts returning **`HTTP 400 {"message":"Failed to write blob \"<group>/ocr/<id>/original.pdf\": Storage backend has reached its minimum free drive threshold. Please delete a few objects to proceed."}`**. The latency on the failed call is ~4 s (MinIO retries before erroring), and the error is upstream of the backend — it surfaces directly from the AWS S3 SDK call inside **[`apps/backend-services/src/blob-storage/minio-blob-storage.service.ts:86`](../../apps/backend-services/src/blob-storage/minio-blob-storage.service.ts)**. The MinIO PVC sizing comes from **[`deployments/openshift/kustomize/components/minio/pvc.yml`](../../deployments/openshift/kustomize/components/minio/pvc.yml)** with the size templated by **`scripts/lib/generate-overlay.sh`** (default `5Gi`, override with **`--minio-pvc-size`** or `MINIO_PVC_SIZE` in **`dev.env`**). The storage class **`netapp-file-standard`** has **`allowVolumeExpansion=true`**, so an existing PVC can be grown online with no pod restart and no data loss:
 
 ```bash