PR 3: E02 — Mistral Document AI on Azure AI Foundry

apps/backend-services/src/seed/local-dataset-sync.service.spec.ts

@@ -40,8 +40,13 @@ function makeFakeBlobStorage(prePopulated: string[] = []): FakeBlobStore {
     async list(_prefix: BlobPrefixPath): Promise<string[]> {
       return Array.from(blobs.keys());
     },
-    async deleteByPrefix(_prefix: BlobPrefixPath): Promise<void> {
-      // unused
+    async deleteByPrefix(prefix: BlobPrefixPath): Promise<void> {
+      const p = String(prefix);
+      for (const key of Array.from(blobs.keys())) {
+        if (key.startsWith(p)) {
+          blobs.delete(key);
+        }
+      }
     },
   };
   return { blobs, storage, writes };
@@ -251,6 +256,68 @@ describe("syncLocalDatasetToBlobStorage", () => {
     }
   });
 
+  it("force mode wipes the dataset prefix and re-uploads even when files already exist", async () => {
+    const { repoRoot, parsed, cleanup } = setupRepoWithDataset(
+      "ds-force",
+      "private",
+      [
+        { name: "a.jpg", content: "img" },
+        { name: "a.json", content: "{}" },
+      ],
+    );
+    const blob = makeFakeBlobStorage();
+    const prisma = makePrismaStub();
+    try {
+      const first = await syncLocalDatasetToBlobStorage(
+        parsed,
+        repoRoot,
+        blob.storage,
+        prisma.client,
+      );
+      expect(first.uploaded).toBe(2);
+
+      // Add a stale orphan file under the dataset prefix that wouldn't be
+      // touched by an idempotent sync (simulates a renamed/removed file
+      // from a prior layout).
+      const datasetPrefix =
+        "seeddefaultgroup/benchmark/datasets/seed-local-ds-force-private/seed-local-ds-force-private-v1";
+      blob.blobs.set(
+        `${datasetPrefix}/inputs/orphan.jpg` as unknown as BlobFilePath,
+        Buffer.from("stale"),
+      );
+      expect(blob.blobs.size).toBeGreaterThan(2);
+
+      // Force mode: deletes everything under the prefix, then re-uploads.
+      const forced = await syncLocalDatasetToBlobStorage(
+        parsed,
+        repoRoot,
+        blob.storage,
+        prisma.client,
+        { force: true },
+      );
+      expect(forced.uploaded).toBe(2);
+      // Orphan must be gone after force resync.
+      expect(
+        blob.blobs.has(
+          `${datasetPrefix}/inputs/orphan.jpg` as unknown as BlobFilePath,
+        ),
+      ).toBe(false);
+      // The two manifest files plus dataset-manifest.json are present.
+      expect(
+        blob.blobs.has(
+          `${datasetPrefix}/inputs/a.jpg` as unknown as BlobFilePath,
+        ),
+      ).toBe(true);
+      expect(
+        blob.blobs.has(
+          `${datasetPrefix}/dataset-manifest.json` as unknown as BlobFilePath,
+        ),
+      ).toBe(true);
+    } finally {
+      cleanup();
+    }
+  });
+
   it("uploads only the new files when sample list grows between runs", async () => {
     const { repoRoot, parsed, cleanup } = setupRepoWithDataset(
       "ds",

apps/backend-services/src/seed/local-dataset-sync.service.ts

@@ -27,6 +27,7 @@ import {
 } from "@/blob-storage/blob-storage.interface";
 import {
   buildBlobFilePath,
+  buildBlobPrefixPath,
   OperationCategory,
 } from "@/blob-storage/storage-path-builder";
 import { PrismaService } from "@/database/prisma.service";
@@ -99,6 +100,8 @@ export class LocalDatasetSyncService implements OnApplicationBootstrap {
       return;
     }
 
+    const force = process.env.FORCE_RESYNC_LOCAL_DATASETS === "true";
+
     const repoRoot = path.resolve(__dirname, "../../../..");
     const datasetsDir = path.join(repoRoot, "data", "datasets");
 
@@ -111,12 +114,12 @@ export class LocalDatasetSyncService implements OnApplicationBootstrap {
     }
 
     this.logger.log(
-      `Syncing ${parsed.length} local dataset version(s) to blob storage...`,
+      `Syncing ${parsed.length} local dataset version(s) to blob storage${force ? " (force mode: will delete + re-upload)" : ""}...`,
     );
 
     for (const entry of parsed) {
       try {
-        const result = await this.syncOne(entry, repoRoot);
+        const result = await this.syncOne(entry, repoRoot, { force });
         this.logger.log(
           `Synced ${result.folder}/${result.visibility}: ${result.uploaded} uploaded, ${result.skipped} skipped`,
         );
@@ -135,24 +138,34 @@ export class LocalDatasetSyncService implements OnApplicationBootstrap {
   async syncOne(
     entry: ParsedLocalDataset,
     repoRoot: string,
+    options: { force?: boolean } = {},
   ): Promise<LocalDatasetSyncResult> {
     return syncLocalDatasetToBlobStorage(
       entry,
       repoRoot,
       this.blobStorage,
       this.prisma.prisma as unknown as DatasetVersionUpdater,
+      options,
     );
   }
 }
 
 /**
  * Pure-ish sync function so it can be unit tested without spinning up NestJS.
+ *
+ * `options.force` (default false) — when true, deletes all blobs under the
+ * dataset's prefix before uploading. Use after local renames or content edits
+ * that the standard idempotent sync can't propagate (because it skips files
+ * that already exist on blob storage). The next benchmark run will also need
+ * a clean materialized cache; the temporal worker re-downloads on each run
+ * so blob is the source of truth.
  */
 export async function syncLocalDatasetToBlobStorage(
   entry: ParsedLocalDataset,
   repoRoot: string,
   blobStorage: BlobStorageInterface,
   prisma: DatasetVersionUpdater,
+  options: { force?: boolean } = {},
 ): Promise<LocalDatasetSyncResult> {
   const datasetId = localDatasetId(entry.folder, entry.visibility);
   const versionId = localDatasetVersionId(entry.folder, entry.visibility);
@@ -166,6 +179,19 @@ export async function syncLocalDatasetToBlobStorage(
 
   const blobStoragePrefix = `datasets/${datasetId}/${versionId}`;
 
+  // Force mode: nuke the prefix so subsequent writes overwrite cleanly and
+  // any blobs not in the local manifest (orphans from prior layouts) are
+  // removed. Without this, the existence-skip below would leave stale files
+  // on cloud storage forever.
+  if (options.force) {
+    const fullPrefix = buildBlobPrefixPath(
+      SEED_GROUP_ID,
+      OperationCategory.BENCHMARK,
+      [blobStoragePrefix],
+    );
+    await blobStorage.deleteByPrefix(fullPrefix);
+  }
+
   let uploaded = 0;
   let skipped = 0;
 

apps/backend-services/src/workflow/activity-registry.spec.ts

@@ -11,6 +11,7 @@ const EXPECTED_ACTIVITY_TYPES = [
   "azureOcr.poll",
   "azureOcr.extract",
   "mistralOcr.process",
+  "mistralAzureOcr.process",
   "ocr.cleanup",
   "ocr.enrich",
   "ocr.checkConfidence",

apps/backend-services/src/workflow/activity-registry.ts

@@ -31,6 +31,10 @@ export const REGISTERED_ACTIVITY_TYPES: Record<string, RegisteredActivityType> =
       description:
         "Mistral Document AI OCR (sync) with optional document annotation",
     },
+    "mistralAzureOcr.process": {
+      description:
+        "Mistral Document AI on Azure AI Foundry (sync) with optional document annotation",
+    },
     "ocr.cleanup": { description: "Post-OCR text normalization" },
     "ocr.enrich": {
       description: "Enrich OCR results using field schema and optional LLM",

apps/temporal/src/__fixtures__/experiment-02/mistral-azure-ocr-response-1-81.json

@@ -0,0 +1,43 @@
+{
+    "model": "mistral-document-ai-2512",
+    "pages": [
+        {
+            "index": 0,
+            "footer": null,
+            "header": null,
+            "images": [],
+            "tables": [],
+            "markdown": "5th\n\nMonthly Report\n\nAvoid delays, submit by the 5th\n\nBRITISH\n\nCOLUMBIA\n\nMinistry of\n\nSocial Development\n\nand Poverty Reduction\n\nThe personal information requested on this form is collected by the Ministry of Social Development and Poverty Reduction pursuant to sections 26(c) of the Freedom of Information and Protection of Privacy Act for the purpose of administering the Employment and Assistance Act and Employment and Assistance for Persons with Disabilities Act. If you have any questions about the collection, use or disclosure of this information, please contact the Ministry of Social Development and Poverty Reduction at 1-866-866-0800.\n\n# 1. Since your last declaration:\n\nAre you still in need of assistance? \u2611 Yes \u2610 No\n\nHas your family unit received or disposed of any assets? \u2610 Yes \u2611 No\n\nAny changes to your shelter costs? \u2610 Yes \u2611 No\n\nAny changes in Dependants or Persons living in the home? \u2610 Yes \u2611 No\n\nApplicant\n\nSpouse\n\nAny employment changes? \u2610 Yes \u2610 No \u2610 Yes \u2611 No\n\nAre you attending / enrolled in school or training? \u2610 Yes \u2610 No \u2610 Yes \u2611 No\n\nAre you looking for work? \u2610 Yes \u2610 No \u2610 Yes \u2611 No\n\nHave you moved or entered a facility? \u2610 Yes \u2610 No \u2610 Yes \u2611 No\n\nAny outstanding warrants for your arrest? \u2610 Yes \u2610 No \u2610 Yes \u2611 No\n\nPlease explain all changes including income and submit proof:\n\n# 2. Declare all income and submit proof. Enter \"0\" if none.\n\n|   | Applicant | Spouse  |\n| --- | --- | --- |\n|  Net Employment Income | $ 0 | $  |\n|  Employment Insurance | $ 700.00 | $  |\n|  Spousal Support / Alimony | $ 0 | $  |\n|  Child Support | $ 0 | $  |\n|  WorkBC Financial Support | $ 0 | $  |\n|  Student Funding (eg: Loans, Bursaries) | $ 0 | $  |\n|  Rental Income | $ 0 | $  |\n|  Room / Board Income | $ 0 | $  |\n|  Worker's Compensation | $ 0 | $  |\n|  Private Pensions (eg: Retirement, Disability) | $ 0 | $  |\n|  OAS / GIS | $ 0 | $  |\n|  Trust Income | $ 0 | $  |\n|  Canada Pension Plan (CPP) | $ 0 | $  |\n|  Tax Credits (eg: GST Credit) | $ 0 | $  |\n|  Child Tax Benefits | $ 0 | $  |\n|  Income Tax Refund | $ 60.00 | $  |\n|  All other income / money received | $ 0 | $  |\n|  Income of Dependent Children | $ 0 | $  |\n\n# 3. Declaration: I understand that the ministry may disclose this information to verify continuing eligibility for assistance under the above Acts and Regulations. I declare that all of the information provided on this form to the Ministry of Social Development and Poverty Reduction is true and complete.\n\n|  Applicant Signature\nKEY PLAYER MISSING |   | Date (yyyy-mmm-dd)\n2026-25-07 |   | Spouse Signature |   | Date (yyyy-mmm-dd)  |\n| --- | --- | --- | --- | --- | --- | --- |\n|  Applicant Print Name\nX |   |   |   | Spouse Print Name  |   |   |\n|  Applicant Telephone | Social Insurance Number\n967-89-954 |   |   | Spouse Telephone | Social Insurance Number  |   |\n\nNEXT CHEQUE\n\nISSUE\n\nBENEFIT MONTH\n\nTOTAL ALLOWANCE\n\nSHELTER PORTION\n\nINCOME DECLARED\n\nINCOME DEDUCTED\n\nOTHER DEDUCTIONS\n\nTOTAL CHEQUE\n\nAPR 22 2026\n\nCASE ID\n\nCASELOAD",
+            "dimensions": {
+                "dpi": 200,
+                "width": 2200,
+                "height": 1700
+            },
+            "hyperlinks": []
+        }
+    ],
+    "usage_info": {
+        "doc_size_bytes": 592042,
+        "pages_processed": 0,
+        "pages_processed_annotation": 1
+    },
+    "document_annotation": "{\n  \"checkbox_need_assistance_yes\": \"selected\",\n  \"checkbox_need_assistance_no\": \"unselected\",\n  \"checkbox_family_assets_yes\": \"unselected\",\n  \"checkbox_family_assets_no\": \"selected\",\n  \"checkbox_shelter_yes\": \"unselected\",\n  \"checkbox_shelter_no\": \"selected\",\n  \"checkbox_dependants_yes\": \"unselected\",\n  \"checkbox_dependants_no\": \"selected\",\n  \"checkbox_employment_changes_yes\": \"unselected\",\n  \"checkbox_employment_changes_no\": \"selected\",\n  \"checkbox_employment_changes_spouse_yes\": \"unselected\",\n  \"checkbox_employment_changes_spouse_no\": \"selected\",\n  \"checkbox_school_yes\": \"unselected\",\n  \"checkbox_school_no\": \"selected\",\n  \"checkbox_school_spouse_yes\": \"unselected\",\n  \"checkbox_school_spouse_no\": \"selected\",\n  \"checkbox_work_yes\": \"unselected\",\n  \"checkbox_work_no\": \"selected\",\n  \"checkbox_work_spouse_yes\": \"unselected\",\n  \"checkbox_work_spouse_no\": \"selected\",\n  \"checkbox_moved_yes\": \"unselected\",\n  \"checkbox_moved_no\": \"selected\",\n  \"checkbox_moved_spouse_yes\": \"unselected\",\n  \"checkbox_moved_spouse_no\": \"selected\",\n  \"checkbox_warrant_yes\": \"unselected\",\n  \"checkbox_warrant_no\": \"selected\",\n  \"checkbox_warrant_spouse_yes\": \"unselected\",\n  \"checkbox_warrant_spouse_no\": \"selected\",\n  \"explain_changes\": \"\",\n  \"signature\": \"KEY PLAYER MISSING\",\n  \"spouse_signature\": \"\",\n  \"date\": \"2026-07-25\",\n  \"spouse_date\": \"\",\n  \"name\": \"X\",\n  \"spouse_name\": \"\",\n  \"phone\": \"\",\n  \"spouse_phone\": \"\",\n  \"sin\": \"96789954\",\n  \"spouse_sin\": \"\",\n  \"applicant_net_employment_income\": 0,\n  \"applicant_employment_insurance\": 700,\n  \"applicant_spousal_support_alimony\": 0,\n  \"applicant_child_support\": 0,\n  \"applicant_workbc_financial_support\": 0,\n  \"applicant_student_funding_loans_bursaries\": 0,\n  \"applicant_rental_income\": 0,\n  \"applicant_room_board_income\": 0,\n  \"applicant_workers_compensation\": 0,\n  \"applicant_private_pensions_retirement_disability\": 0,\n  \"applicant_oas_gis\": 0,\n  \"applicant_trust_income\": 0,\n  \"applicant_canada_pension_plan_cpp\": 0,\n  \"applicant_tax_credits_gst_credit\": 0,\n  \"applicant_child_tax_benefits\": 0,\n  \"applicant_income_tax_refund\": 60,\n  \"applicant_other_income_money_received\": 0,\n  \"applicant_income_of_dependent_children\": 0,\n  \"spouse_net_employment_income\": null,\n  \"spouse_employment_insurance\": null,\n  \"spouse_spousal_support_alimony\": null,\n  \"spouse_child_support\": null,\n  \"spouse_workbc_financial_support\": null,\n  \"spouse_student_funding_loans_bursaries\": null,\n  \"spouse_rental_income\": null,\n  \"spouse_room_board_income\": null,\n  \"spouse_workers_compensation\": null,\n  \"spouse_private_pensions_retirement_disability\": null,\n  \"spouse_oas_gis\": null,\n  \"spouse_trust_income\": null,\n  \"spouse_canada_pension_plan_cpp\": null,\n  \"spouse_tax_credits_gst_credit\": null,\n  \"spouse_child_tax_benefits\": null,\n  \"spouse_income_tax_refund\": null,\n  \"spouse_other_income_money_received\": null\n}",
+    "content_filter_results": {
+        "hate": {
+            "filtered": false,
+            "severity": "safe"
+        },
+        "sexual": {
+            "filtered": false,
+            "severity": "safe"
+        },
+        "violence": {
+            "filtered": false,
+            "severity": "safe"
+        },
+        "self_harm": {
+            "filtered": false,
+            "severity": "safe"
+        }
+    }
+}

apps/temporal/src/activities.ts

@@ -129,3 +129,8 @@ export type {
   CorrectionResult,
   CorrectionToolParams,
 } from "./correction-types";
+export type { MistralAzureOcrProcessParams } from "./ocr-providers/mistral-azure/mistral-azure-ocr-process";
+export {
+  mistralAzureOcrProcess,
+  resolveMistralAzureDeploymentId,
+} from "./ocr-providers/mistral-azure/mistral-azure-ocr-process";

apps/temporal/src/activity-registry.test.ts

@@ -14,6 +14,7 @@ const EXPECTED_ACTIVITY_TYPES = [
   "ocr.cleanup",
   "ocr.checkConfidence",
   "mistralOcr.process",
+  "mistralAzureOcr.process",
   "ocr.storeResults",
   "document.storeRejection",
   "getWorkflowGraphConfig",

apps/temporal/src/activity-registry.ts

@@ -26,6 +26,7 @@ import {
   getWorkflowGraphConfig,
   loadDatasetManifest,
   materializeDataset,
+  mistralAzureOcrProcess,
   mistralOcrProcess,
   pollOCRResults,
   postOcrCleanup,
@@ -144,6 +145,29 @@ register({
     "Mistral Document AI OCR (sync) with optional document annotation",
 });
 
+register({
+  activityType: "mistralAzureOcr.process",
+  activityFn: mistralAzureOcrProcess as (
+    ...args: unknown[]
+  ) => Promise<unknown>,
+  // Foundry's annotation step "can be slower and may result in timeouts" per
+  // Microsoft docs; allow generous wallclock plus extra retry attempts vs the
+  // public-API path. The deployment is rate-limited by per-minute requests
+  // (default 10 RPM on GlobalStandard) and a 33-sample benchmark fan-out
+  // gets sustained 429s — the retry policy is therefore tuned to spread
+  // retries across the quota window with backoff jitter rather than the
+  // public-API path's tighter 3-attempt policy.
+  defaultTimeout: "20m",
+  defaultRetry: {
+    maximumAttempts: 30,
+    initialInterval: "15s",
+    backoffCoefficient: 1.5,
+    maximumInterval: "60s",
+  },
+  description:
+    "Mistral Document AI on Azure AI Foundry (sync) with optional document annotation",
+});
+
 register({
   activityType: "ocr.checkConfidence",
   activityFn: checkOcrConfidence as (...args: unknown[]) => Promise<unknown>,

apps/temporal/src/activity-types.ts

@@ -14,6 +14,7 @@ export const REGISTERED_ACTIVITY_TYPES = [
   "ocr.cleanup",
   "ocr.checkConfidence",
   "mistralOcr.process",
+  "mistralAzureOcr.process",
   "ocr.storeResults",
   "ocr.enrich",
   "document.storeRejection",