Merge pull request #317 from workglow-dev/claude/zen-albattani-alr17v-proxy-retraction

Never retract a proxy event on a merger-proxy run that reached no verdict

Author: sroussey

CLAUDE.md

@@ -1918,6 +1918,34 @@ Reclassification therefore runs in both directions and a replay demotes the deal
 instead of leaving the old verdict standing. The delete is scoped to that one
 accession, so it can only retract what a previous run of the same filing wrote.
 
+**Only a verdict about the DOCUMENT moves the event, which is why the writer
+takes a tri-state rather than a boolean** (`ProxyEventVerdict`:
+`emit | retract | leave`, decided by `resolveProxyEventVerdict`). `runSection`
+contains every model and transport failure as a dead letter and returns
+normally, so "the model said this filing discloses no deal" and "the provider
+throttled us" both arrive as an unset `extractedDeal`. Read as a boolean the
+second retracted a `proxy` event an earlier successful run had recorded from
+real evidence — and losing it takes the whole approval stage with it, since the
+vehicle's next Form 25/15 inside the 90-day post-approval window then classifies
+`deregistration` instead of `completed` and `recordDeregistration` deletes the
+`completed` event: a genuinely de-SPAC'd vehicle recorded as a wind-up, from a
+run that merely could not reach a model. So a general definitive statement
+retracts only on `seeks_combination_approval === false` — deterministic, and
+conjunctive with the deal, so it decides alone and keeps the recovery ceremony
+working during a provider outage — or on a dead letter that IS an answer
+(`SECTION_NOT_FOUND` / `MODEL_EMPTY`, `NO_DEAL_REASONS`). Everything else,
+including `LOW_CONFIDENCE_ALL` and `UNVERIFIED_SOURCE_SPAN` (where the model did
+return a deal and only its certainty or its citation failed), leaves the stream
+untouched.
+
+The deterministic verdict is recorded on an existing extraction row **even when
+the run extracted nothing** (`SpacMergerExtractionRepo.recordApprovalVerdict`),
+because the gate really was evaluated — it is a property of the document, not of
+the model call. Left NULL, the backfill's null-verdict clause re-selects the
+same filing on every sweep, which is what made the failure repeat rather than
+happen once. No row is invented where none exists: every predicate downstream
+reads an extraction row as "this proxy produced something".
+
 No extractor version bump: the persisted extraction rows are unchanged and still
 correct, and the derived event is rebuildable from the document with no model
 call.

src/sec/forms/proxies-information-statements/Form_DEFM14A.proxyVerdict.test.ts

@@ -0,0 +1,64 @@
+/**
+ * @license
+ * Copyright 2026 Steven Roussey <sroussey@gmail.com>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { describe, expect, it } from "vitest";
+import { resolveProxyEventVerdict } from "./Form_DEFM14A.storage";
+
+/** Defaults for a run that reached the model and extracted a deal. */
+const extracted = { extractedDeal: true, concludedNoDeal: false } as const;
+/** A run that read the section and concluded the filing discloses no deal. */
+const noDeal = { extractedDeal: false, concludedNoDeal: true } as const;
+/** A run that failed before reaching an answer (no model, throttle, catch-all). */
+const noAnswer = { extractedDeal: false, concludedNoDeal: false } as const;
+
+describe("resolveProxyEventVerdict", () => {
+  it("emits for a definitive merger statement on the form symbol alone", () => {
+    for (const form of ["DEFM14A", "DEFM14C"]) {
+      expect(resolveProxyEventVerdict({ form, ...noAnswer, seeksCombinationApproval: null })).toBe(
+        "emit"
+      );
+    }
+  });
+
+  it("never emits for a preliminary or revised statement", () => {
+    for (const form of ["PREM14A", "PREM14C", "PRER14A", "DEFR14A", "PRE 14A"]) {
+      expect(resolveProxyEventVerdict({ form, ...extracted, seeksCombinationApproval: true })).toBe(
+        "retract"
+      );
+    }
+  });
+
+  it("emits for a general definitive statement only on both pieces of evidence", () => {
+    expect(
+      resolveProxyEventVerdict({ form: "DEF 14A", ...extracted, seeksCombinationApproval: true })
+    ).toBe("emit");
+    expect(
+      resolveProxyEventVerdict({ form: "DEF 14A", ...noDeal, seeksCombinationApproval: true })
+    ).toBe("retract");
+    expect(
+      resolveProxyEventVerdict({ form: "DEF 14A", ...extracted, seeksCombinationApproval: false })
+    ).toBe("retract");
+  });
+
+  it("leaves the event alone when the run reached no verdict about the document", () => {
+    // The invariant: a failed extraction is not evidence that this filing is
+    // not a merger proxy, so it must not delete an event an earlier run wrote.
+    expect(
+      resolveProxyEventVerdict({ form: "DEF 14A", ...noAnswer, seeksCombinationApproval: true })
+    ).toBe("leave");
+    expect(
+      resolveProxyEventVerdict({ form: "DEF 14C", ...noAnswer, seeksCombinationApproval: true })
+    ).toBe("leave");
+  });
+
+  it("still retracts on a false approval verdict when the extraction failed", () => {
+    // The approval gate is deterministic AND conjunctive, so a statement that
+    // asks for no approval can never emit — with or without a model.
+    expect(
+      resolveProxyEventVerdict({ form: "DEF 14A", ...noAnswer, seeksCombinationApproval: false })
+    ).toBe("retract");
+  });
+});

src/sec/forms/proxies-information-statements/Form_DEFM14A.storage.e2e.test.ts

@@ -6,9 +6,12 @@
 
 import { readFileSync } from "node:fs";
 import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { globalServiceRegistry } from "workglow";
 import { resetDependencyInjectionsForTesting } from "../../../config/TestingDI";
 import { setupAllDatabases } from "../../../config/setupAllDatabases";
 import { ExtractionDeadLetterRepo } from "../../../storage/dead-letter/ExtractionDeadLetterRepo";
+import { FILING_REPOSITORY_TOKEN } from "../../../storage/filing/FilingSchema";
+import { getBackfillDescriptor } from "../../../task/forms/backfillDescriptors";
 import { SpacMergerExtractionRepo } from "../../../storage/spac/SpacMergerExtractionRepo";
 import { SpacRepo } from "../../../storage/spac/SpacRepo";
 import { SpacReportWriter } from "../../../storage/spac/SpacReportWriter";
@@ -253,6 +256,138 @@ describe("processMergerProxy (e2e)", () => {
     expect(row?.status).toBe("deal_announced");
   });
 
+  /**
+   * A provider error that is NOT a throttle: `isRateLimitError` would otherwise
+   * make the extractor wait out real backoff sleeps before failing.
+   */
+  function scriptExtractionFailure(): () => void {
+    return registerFakeStructuredProvider([new Error("upstream provider failure")]).unregister;
+  }
+
+  it("leaves a recorded proxy event standing when the re-run's extraction fails", async () => {
+    // A failed model call is not a verdict about the filing. The approval
+    // evidence (`seeks_combination_approval`) is deterministic and still reads
+    // true here — only the extraction failed — so retracting would delete an
+    // event an earlier run recorded from real evidence, and losing the `proxy`
+    // event cascades: the vehicle's next Form 25/15 inside the post-approval
+    // window stops reading as a completed de-SPAC.
+    await seedSpacWithOpenDeal(130);
+    cleanup = scriptMergerDeal();
+    await runProxy(130, "130-def14a", "DEF 14A", "2021-05-01", submissionWithBody(APPROVAL_BODY));
+    expect((await repo.getSpac(130))?.status).toBe("proxy");
+    cleanup();
+
+    cleanup = scriptExtractionFailure();
+    await runProxy(130, "130-def14a", "DEF 14A", "2021-05-01", submissionWithBody(APPROVAL_BODY));
+
+    const events = await repo.getEvents(130);
+    expect(events.filter((e) => e.event_type === "proxy")).toHaveLength(1);
+    const row = await repo.getSpac(130);
+    expect(row?.status).toBe("proxy");
+    expect(row?.proxy_date).toBe("2021-05-01");
+
+    // The failure is on the worklist — left for retry, not read as an answer.
+    const entry = await new ExtractionDeadLetterRepo().get("merger-proxy", "130-def14a", "merger");
+    expect(entry?.status).toBe("pending");
+  });
+
+  it("retracts on the document verdict even when the extraction failed", async () => {
+    // The other half of the same rule: `seeks_combination_approval` is derived
+    // from the document with no model call, and the gate is conjunctive — a
+    // statement asking for no approval can never emit, so a false verdict
+    // retracts whether or not the model was reachable. That is what keeps the
+    // recovery ceremony able to unwind a stale close during a provider outage.
+    await seedSpacWithOpenDeal(131);
+    cleanup = scriptMergerDeal();
+    await runProxy(131, "131-def14a", "DEF 14A", "2021-05-01", submissionWithBody(APPROVAL_BODY));
+    expect((await repo.getSpac(131))?.status).toBe("proxy");
+    cleanup();
+
+    cleanup = scriptExtractionFailure();
+    await runProxy(131, "131-def14a", "DEF 14A", "2021-05-01", submissionWithBody(EXTENSION_BODY));
+
+    const events = await repo.getEvents(131);
+    expect(events.some((e) => e.event_type === "proxy")).toBe(false);
+    const row = await repo.getSpac(131);
+    expect(row?.status).toBe("deal_announced");
+    expect(row?.proxy_date).toBeNull();
+  });
+
+  it("leaves a recorded proxy event standing when no model could be resolved", async () => {
+    // The same invariant one stage earlier: the section never reaches a model,
+    // so the run has nothing to say about the document.
+    await seedSpacWithOpenDeal(132);
+    cleanup = scriptMergerDeal();
+    await runProxy(132, "132-def14a", "DEF 14A", "2021-05-01", submissionWithBody(APPROVAL_BODY));
+    expect((await repo.getSpac(132))?.status).toBe("proxy");
+    cleanup();
+    cleanup = undefined;
+
+    // No `model` argument, and no merger-proxy model id is registered here.
+    const parsed = await Form_DEFM14A.parse("DEF 14A", submissionWithBody(APPROVAL_BODY));
+    await processMergerProxy({
+      cik: 132,
+      file_number: "",
+      accession_number: "132-def14a",
+      filing_date: "2021-05-01",
+      primary_doc: "proxy.htm",
+      form: "DEF 14A",
+      formMergerProxy: parsed,
+    });
+
+    const entry = await new ExtractionDeadLetterRepo().get("merger-proxy", "132-def14a", "merger");
+    expect(entry?.reason_code).toBe("MODEL_RESOLUTION_ERROR");
+    expect((await repo.getEvents(132)).filter((e) => e.event_type === "proxy")).toHaveLength(1);
+    expect((await repo.getSpac(132))?.status).toBe("proxy");
+  });
+
+  it("records the gate verdict even when the run extracted nothing", async () => {
+    // The backfill re-selects a general definitive proxy whose verdict is NULL.
+    // A run that could not reach the model still evaluated the deterministic
+    // gate, so it must record the verdict — otherwise the clause never
+    // extinguishes and every sweep re-runs the same filing forever.
+    await seedSpacWithOpenDeal(133);
+    cleanup = scriptMergerDeal();
+    await runProxy(133, "133-def14a", "DEF 14A", "2021-05-01", submissionWithBody(APPROVAL_BODY));
+    cleanup();
+
+    // Model the pre-gate row the recovery ceremony targets: verdict never set.
+    const extractions = new SpacMergerExtractionRepo();
+    const before = await extractions.getByAccession("133-def14a");
+    await extractions.save({ ...before!, seeks_combination_approval: null });
+
+    cleanup = scriptExtractionFailure();
+    await runProxy(133, "133-def14a", "DEF 14A", "2021-05-01", submissionWithBody(APPROVAL_BODY));
+
+    const after = await extractions.getByAccession("133-def14a");
+    expect(after?.seeks_combination_approval).toBe(true);
+    // The extraction the earlier run persisted is untouched by the failed one.
+    expect(after?.target_name).toBe("Acme Target Inc.");
+
+    // And the backfill's null-verdict clause no longer selects it.
+    await globalServiceRegistry.get(FILING_REPOSITORY_TOKEN).put({
+      cik: 133,
+      accession_number: "133-def14a",
+      form: "DEF 14A",
+      primary_doc: "proxy.htm",
+      file_number: "",
+      filing_date: "2021-05-01",
+      acceptance_date: "2021-05-01T00:00:00.000Z",
+      report_date: "2021-05-01",
+      film_number: null,
+      primary_doc_description: null,
+      size: null,
+      is_xbrl: null,
+      is_inline_xbrl: null,
+      items: null,
+      act: null,
+    } as never);
+    const descriptor = getBackfillDescriptor("merger-proxy")!;
+    const candidates = await descriptor.selectCandidates();
+    expect(candidates.map((c) => c.accession_number)).toContain("133-def14a");
+    expect(await descriptor.filterTodo!(candidates)).toEqual([]);
+  });
+
   it("records the gate verdict only for the forms it governs", async () => {
     // The "M" forms decide on the symbol alone and must not pay a full-document
     // render, so their verdict stays null — which is also what the backfill

src/sec/forms/proxies-information-statements/Form_DEFM14A.storage.ts

@@ -20,8 +20,10 @@ import { ExtractorRunRepo } from "../../../storage/versioning/ExtractorRunRepo";
 import { EXTRACTOR_RUN_REPOSITORY_TOKEN } from "../../../storage/versioning/ExtractorRunSchema";
 import { ObservationProvenanceRepo } from "../../../storage/provenance/ObservationProvenanceRepo";
 import { ExtractionDeadLetterRepo } from "../../../storage/dead-letter/ExtractionDeadLetterRepo";
+import type { DeadLetterReasonCode } from "../../../storage/dead-letter/ExtractionDeadLetterSchema";
 import { SpacRepo } from "../../../storage/spac/SpacRepo";
 import { SpacReportWriter } from "../../../storage/spac/SpacReportWriter";
+import type { ProxyEventVerdict } from "../../../storage/spac/SpacReportWriter";
 import { SpacMergerExtractionRepo } from "../../../storage/spac/SpacMergerExtractionRepo";
 import { parseEdgarHtml } from "../../html/parseEdgarHtml";
 import { DocumentTreeSegmenter } from "../registration-statements/s1/DocumentTreeSegmenter";
@@ -57,6 +59,47 @@ const MERGER_SECTION = MERGER_PROXY_SECTION;
  */
 const DEFINITIVE_PROXY_FORMS = new Set(["DEFM14A", "DEFM14C"]);
 
+/**
+ * The dead-letter reasons that ARE a verdict about the document: the merger
+ * section is absent, or a model read it and found no deal. Every other reason
+ * — an unresolved model, a throttle, a nonce mismatch, the
+ * `MODEL_INVALID_OUTPUT` catch-all — reports that this run never reached an
+ * answer. Low-confidence and unverified-span rows are deliberately absent too:
+ * the model did return a deal there, and only its citation or its certainty
+ * failed.
+ */
+const NO_DEAL_REASONS = new Set<DeadLetterReasonCode>(["SECTION_NOT_FOUND", "MODEL_EMPTY"]);
+
+/**
+ * Which way this run moves the accession's `proxy` event.
+ *
+ * Both writes require evidence about the DOCUMENT. `seeks_combination_approval`
+ * is deterministic and decides on its own when false: the gate is conjunctive,
+ * so a statement that asks shareholders to approve nothing can never emit, with
+ * or without a deal — which is what lets the recovery ceremony unwind a stale
+ * close while a provider is down. A true verdict still needs the deal, and
+ * there the distinction matters: a run whose extraction FAILED knows nothing
+ * about the filing, so it leaves the stream alone rather than deleting an event
+ * an earlier run recorded. Retracting there loses the whole approval stage —
+ * the vehicle's next Form 25/15 inside the post-approval window stops reading
+ * as a completed de-SPAC — and the failure re-occurs on every sweep.
+ */
+export function resolveProxyEventVerdict(args: {
+  readonly form: string;
+  readonly extractedDeal: boolean;
+  /** This run concluded the filing discloses no deal (not merely: it failed). */
+  readonly concludedNoDeal: boolean;
+  readonly seeksCombinationApproval: boolean | null;
+}): ProxyEventVerdict {
+  if (DEFINITIVE_PROXY_FORMS.has(args.form)) return "emit";
+  // Preliminary / revised statements never emit; retracting keeps a replay able
+  // to clear an event some earlier generation of this code wrote for one.
+  if (!GENERAL_DEFINITIVE_PROXY_FORMS.has(args.form)) return "retract";
+  if (args.seeksCombinationApproval !== true) return "retract";
+  if (args.extractedDeal) return "emit";
+  return args.concludedNoDeal ? "retract" : "leave";
+}
+
 export interface ProcessMergerProxyArgs {
   readonly cik: number;
   readonly file_number: string;
@@ -82,7 +125,9 @@ export interface ProcessMergerProxyArgs {
  * it on the form symbol alone, so it still advances `proxy_date` when the
  * merger section is absent or low-confidence and the section dead-letters; a
  * {@link GENERAL_DEFINITIVE_PROXY_FORMS} one emits it only when this run
- * extracted a deal AND the document asks shareholders to approve it.
+ * extracted a deal AND the document asks shareholders to approve it. Which way
+ * the event moves — including the case where this run reached no verdict at all
+ * — is {@link resolveProxyEventVerdict}.
  */
 export async function processMergerProxy(args: ProcessMergerProxyArgs): Promise<void> {
   const { cik, accession_number, form, filing_date, formMergerProxy } = args;
@@ -196,6 +241,10 @@ export async function processMergerProxy(args: ProcessMergerProxyArgs): Promise<
   // Evidence that this filing IS a merger proxy, for the general definitive
   // forms whose symbol does not say so.
   let extractedDeal = false;
+  // Whether this run answered the question at all. A section that never reached
+  // the model is not a filing that discloses no deal, and only the second of
+  // those may move the proxy event.
+  let concludedNoDeal = false;
 
   if (skipMergerSection) {
     // Nothing to extract, and nothing wrong — but the skip still needs a
@@ -221,6 +270,7 @@ export async function processMergerProxy(args: ProcessMergerProxyArgs): Promise<
       failed_extractor_version: extractor_version,
       source_run_id: null,
     });
+    concludedNoDeal = true;
   } else if (!model) {
     // No model: dead-letter the merger section but still emit the proxy event
     // (deterministic, definitive statements only) so the SPAC timeline advances.
@@ -243,7 +293,7 @@ export async function processMergerProxy(args: ProcessMergerProxyArgs): Promise<
       signal: args.context?.signal,
     });
     try {
-      await runSection<MergerDealRow>({
+      const outcome = await runSection<MergerDealRow>({
         sectionName: MERGER_SECTION,
         text: mergerText === "" ? undefined : mergerText,
         notFoundDetail: "no merger / business-combination / PIPE section text",
@@ -309,26 +359,38 @@ export async function processMergerProxy(args: ProcessMergerProxyArgs): Promise<
           return 1;
         },
       });
+      concludedNoDeal = outcome.status === "dead-lettered" && NO_DEAL_REASONS.has(outcome.reason);
     } catch (err) {
       const message = err instanceof Error ? err.message : String(err);
       await recordMergerProxyRun(false, message);
       throw err;
     }
   }
 
-  // Emit the proxy event (definitive only) + recompute/correlate + rebuild.
+  // Emit / retract the proxy event + recompute/correlate + rebuild.
   try {
+    // The gate verdict is deterministic from the document, so a run that
+    // extracted nothing still has one to record on a row an earlier run wrote.
+    // Left NULL, the backfill's null-verdict clause re-selects this filing on
+    // every sweep instead of extinguishing itself.
+    if (seeks_combination_approval !== null && !extractedDeal) {
+      await new SpacMergerExtractionRepo().recordApprovalVerdict(
+        accession_number,
+        seeks_combination_approval
+      );
+    }
     await new SpacReportWriter().recordMergerProxy({
       cik,
       accession_number,
       filing_date,
       form,
       primary_document: args.primary_doc ?? null,
-      emitProxyEvent:
-        DEFINITIVE_PROXY_FORMS.has(form) ||
-        (GENERAL_DEFINITIVE_PROXY_FORMS.has(form) &&
-          extractedDeal &&
-          seeks_combination_approval === true),
+      proxyEvent: resolveProxyEventVerdict({
+        form,
+        extractedDeal,
+        concludedNoDeal,
+        seeksCombinationApproval: seeks_combination_approval,
+      }),
     });
   } catch (err) {
     const message = err instanceof Error ? err.message : String(err);