Merge pull request #314 from workglow-dev/claude/sharp-lamport-ata6g5-spac-process-convergence

fix(spac): make sec spac process reach a fixpoint

Author: sroussey

src/sec/forms/exchange-listing-withdrawal/processDeregistration.ts

@@ -94,6 +94,49 @@ export async function resolveListingRemovalKind(args: {
   });
 }
 
+/**
+ * Whether replaying this listing-removal filing can still write something.
+ *
+ * The single predicate `sec spac process` and `sec extractor backfill 25-15`
+ * both select on, so the two cannot drift apart. It is MONOTONE: the only way
+ * to answer true is that the event the live classifier names is not yet
+ * recorded on this accession, and processing the filing records exactly that
+ * event — so a processed filing leaves the set.
+ *
+ * Two shapes answer false because {@link processDeregistration} would write
+ * nothing for them, and re-selecting a filing nothing can be written for is
+ * pure waste repeated on every sweep:
+ *
+ * - a missing `form` or `filing_date` — the handler returns before writing;
+ * - a classifier verdict of `ignore`, which covers every annual 20-F (the form
+ *   routes here so the FPI CLOSE filing can record a completion) and every
+ *   20-F filed once a completion is already on the stream. A de-SPAC'd foreign
+ *   private issuer files one of those every year, forever.
+ */
+export async function listingRemovalNeedsWork(args: {
+  readonly cik: number;
+  readonly form: string | null;
+  readonly filingDate: string | null;
+  readonly accession_number: string;
+  readonly ipoDate: string | null;
+  readonly events: readonly SpacEvent[];
+}): Promise<boolean> {
+  if (args.form == null || args.form === "") return false;
+  if (args.filingDate == null || args.filingDate === "") return false;
+  const kind = await resolveListingRemovalKind({
+    cik: args.cik,
+    form: args.form,
+    filingDate: args.filingDate,
+    accession_number: args.accession_number,
+    ipoDate: args.ipoDate,
+    events: args.events,
+  });
+  if (kind === "ignore") return false;
+  return !args.events.some(
+    (e) => e.event_type === kind && e.accession_number === args.accession_number
+  );
+}
+
 /**
  * Record Form 25 / 25-NSE / Form 15 family as a lifecycle event. Exchange
  * 25-NSE shortly after IPO is `unit_split` (units unbundle; the vehicle

src/sec/forms/proxies-information-statements/Form_DEFM14A.storage.e2e.test.ts

@@ -8,6 +8,7 @@ import { readFileSync } from "node:fs";
 import { afterEach, beforeEach, describe, expect, it } from "vitest";
 import { resetDependencyInjectionsForTesting } from "../../../config/TestingDI";
 import { setupAllDatabases } from "../../../config/setupAllDatabases";
+import { ExtractionDeadLetterRepo } from "../../../storage/dead-letter/ExtractionDeadLetterRepo";
 import { SpacMergerExtractionRepo } from "../../../storage/spac/SpacMergerExtractionRepo";
 import { SpacRepo } from "../../../storage/spac/SpacRepo";
 import { SpacReportWriter } from "../../../storage/spac/SpacReportWriter";
@@ -194,6 +195,19 @@ describe("processMergerProxy (e2e)", () => {
     expect(events.some((e) => e.event_type === "proxy")).toBe(false);
     expect(await new SpacMergerExtractionRepo().getByAccession("121-def14a-ext")).toBeUndefined();
     expect((await repo.getSpac(121))?.status).toBe("deal_announced");
+
+    // The skip leaves a durable trace, because "no extraction row" is otherwise
+    // indistinguishable from "the handler was gated and dropped its work" — and
+    // every general proxy of every known SPAC would be re-selected on every
+    // sweep, forever. Recorded RESOLVED, so it stays off the worklist an
+    // operator reads.
+    const deadLetters = new ExtractionDeadLetterRepo();
+    const entry = await deadLetters.get("merger-proxy", "121-def14a-ext", "merger");
+    expect(entry?.status).toBe("resolved");
+    expect(entry?.reason_code).toBe("SECTION_NOT_FOUND");
+    expect(
+      (await deadLetters.listPending("merger-proxy")).map((e) => e.accession_number)
+    ).not.toContain("121-def14a-ext");
   });
 
   it("does not emit a proxy event for a preliminary proxy (PRE 14A) that yields a deal", async () => {

src/sec/forms/proxies-information-statements/Form_DEFM14A.storage.ts

@@ -33,13 +33,16 @@ import {
   resolveModelId,
 } from "../registration-statements/s1/mergerModel";
 import type { FormS1Parsed } from "../registration-statements/Form_S_1";
-import { MERGER_PROXY_OPTIONAL_FORMS } from "../../../storage/versioning/extractorIds";
+import {
+  MERGER_PROXY_OPTIONAL_FORMS,
+  MERGER_PROXY_SECTION,
+} from "../../../storage/versioning/extractorIds";
 
 const EXTRACTOR_ID = "merger-proxy";
 // Stays 1.0.0: no persisted data to re-extract, so the target_description
 // addition needs no version bump (see the S-1 processor for the rationale).
 const DEFAULT_EXTRACTOR_VERSION = "1.0.0";
-const MERGER_SECTION = "merger";
+const MERGER_SECTION = MERGER_PROXY_SECTION;
 /**
  * Definitive MERGER statements: the form symbol itself says the meeting is
  * about a combination, so these emit the `proxy` lifecycle event whether or not
@@ -197,7 +200,29 @@ export async function processMergerProxy(args: ProcessMergerProxyArgs): Promise<
   let extractedDeal = false;
 
   if (skipMergerSection) {
-    // Nothing to extract, and nothing wrong. Fall through to the proxy event.
+    // Nothing to extract, and nothing wrong — but the skip still needs a
+    // durable trace. Every predicate that asks "did this proxy produce
+    // anything" reads the extraction row, and a legitimately absent merger
+    // section writes none; without a trace, an ordinary annual or extension
+    // proxy of a known SPAC is indistinguishable from one whose handler was
+    // gated on a missing spac row and dropped its work, so `spac process` and
+    // `sec extractor backfill merger-proxy` re-select all 575 SPACs' general
+    // proxies on every run, forever.
+    //
+    // Recorded RESOLVED: this is an answer, not a failure. It never reaches
+    // `sec extractor dead-letters`, which lists pending entries, so the
+    // worklist an operator reads is untouched — the same shape as the
+    // auto-resolved `MODEL_EMPTY` rows the redemption / LOI detectors already
+    // write per trigger 8-K.
+    await deadLetters.recordResolved({
+      extractor_id: EXTRACTOR_ID,
+      accession_number,
+      section_name: MERGER_SECTION,
+      reason_code: "SECTION_NOT_FOUND",
+      detail: "no merger / business-combination / PIPE section text",
+      failed_extractor_version: extractor_version,
+      source_run_id: null,
+    });
   } else if (!model) {
     // No model: dead-letter the merger section but still emit the proxy event
     // (deterministic, definitive statements only) so the SPAC timeline advances.

src/storage/dead-letter/ExtractionDeadLetterRepo.ts

@@ -132,6 +132,38 @@ export class ExtractionDeadLetterRepo {
     });
   }
 
+  /**
+   * A durable "ran, and there was legitimately nothing to do" trace.
+   *
+   * {@link markResolved} cannot express this: it no-ops when no row exists, and
+   * an expected no-op never wrote one. Without a trace, a selection predicate
+   * keyed on "did this section produce anything" cannot tell a handler that
+   * legitimately wrote nothing from one that was gated and dropped its work, so
+   * it re-selects the filing on every sweep forever.
+   *
+   * A single `put`, so no reader ever observes a spurious `pending` row
+   * mid-update, and `first_seen_at` is preserved when an earlier failure left
+   * one. The entry never reaches `sec extractor dead-letters`, which lists
+   * pending entries only.
+   */
+  async recordResolved(input: DeadLetterInput): Promise<void> {
+    const now = new Date().toISOString();
+    const existing = await this.get(input.extractor_id, input.accession_number, input.section_name);
+    await this.storage.put({
+      extractor_id: input.extractor_id,
+      accession_number: input.accession_number,
+      section_name: input.section_name,
+      reason_code: input.reason_code,
+      detail: input.detail,
+      failed_extractor_version: input.failed_extractor_version,
+      status: "resolved",
+      attempts: 0,
+      first_seen_at: existing?.first_seen_at ?? now,
+      last_attempt_at: now,
+      source_run_id: input.source_run_id,
+    });
+  }
+
   /** Entries for an extractor carrying a reason code, in any status. */
   async listByReasonCode(
     extractor_id: string,
@@ -180,6 +212,42 @@ export class ExtractionDeadLetterRepo {
     return out;
   }
 
+  /**
+   * Entries for the given extractors whose accession is in
+   * `accession_numbers`, in ANY status — a RESOLVED row is the evidence here,
+   * not noise: an auto-resolved expected negative, or a
+   * {@link recordResolved} trace, is the only mark a handler that legitimately
+   * wrote nothing leaves.
+   *
+   * Scoped by accession because the caller already holds one issuer's timeline:
+   * reading every row of an extractor to answer a question about a dozen
+   * filings costs the whole table per issuer, and a batch pays that per SPAC.
+   *
+   * The extractor ids are looped rather than nested as a second `in` list, so
+   * each query binds one list; an empty list on either side returns
+   * immediately (`IN ()` is invalid SQL).
+   */
+  async listByAccessions(
+    accession_numbers: readonly string[],
+    extractor_ids: readonly string[]
+  ): Promise<ExtractionDeadLetter[]> {
+    if (accession_numbers.length === 0 || extractor_ids.length === 0) return [];
+    const distinct = [...new Set(accession_numbers)];
+    const out: ExtractionDeadLetter[] = [];
+    for (const extractor_id of extractor_ids) {
+      for (let start = 0; start < distinct.length; start += MAX_IDS_PER_QUERY) {
+        const chunk = distinct.slice(start, start + MAX_IDS_PER_QUERY);
+        const rows =
+          (await this.storage.query({
+            accession_number: { value: chunk, operator: "in" },
+            extractor_id,
+          })) ?? [];
+        out.push(...rows);
+      }
+    }
+    return out;
+  }
+
   /**
    * Pending entries eligible for retry. Four ways in:
    *

src/storage/dead-letter/answeredMergerSections.ts

@@ -0,0 +1,36 @@
+/**
+ * @license
+ * Copyright 2026 Steven Roussey <sroussey@gmail.com>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { MERGER_PROXY_SECTION } from "../versioning/extractorIds";
+import { ExtractionDeadLetterRepo } from "./ExtractionDeadLetterRepo";
+
+/** The extractor whose entries answer for the merger section. */
+const MERGER_PROXY_EXTRACTOR_ID = "merger-proxy";
+
+/**
+ * Of the given accessions, those whose merger-proxy run already answered for
+ * the merger section — in ANY status.
+ *
+ * A RESOLVED entry is the evidence, not noise. Most general proxies carry no
+ * merger section at all (annual meetings, extension votes), and the processor
+ * writes no extraction row for one; the resolved `SECTION_NOT_FOUND` trace it
+ * records instead is the only durable mark that it looked. Both selection
+ * predicates over merger proxies — `sec spac process` and
+ * `sec extractor backfill merger-proxy` — read it, so they share one query and
+ * cannot disagree about what counts as answered.
+ */
+export async function loadAnsweredMergerSections(
+  accession_numbers: readonly string[]
+): Promise<ReadonlySet<string>> {
+  const rows = await new ExtractionDeadLetterRepo().listByAccessions(accession_numbers, [
+    MERGER_PROXY_EXTRACTOR_ID,
+  ]);
+  return new Set(
+    rows
+      .filter((row) => row.section_name === MERGER_PROXY_SECTION)
+      .map((row) => row.accession_number)
+  );
+}

src/storage/versioning/extractorIds.ts

@@ -268,6 +268,15 @@ export const MERGER_PROXY_OPTIONAL_FORMS: ReadonlySet<string> = new Set([
   "PREA14C",
 ]);
 
+/**
+ * Section name the merger-proxy extractor records its deal — and every
+ * dead-letter entry about it — under. Declared here rather than inside the
+ * processor because the selection predicates that key on those entries live
+ * elsewhere: a second spelling makes the trace unreadable to them, which is
+ * indistinguishable from no trace at all.
+ */
+export const MERGER_PROXY_SECTION = "merger";
+
 /**
  * Short-form registration statements that incorporate an already-filed
  * prospectus by reference (Securities Act Rule 462(b)).

src/task/forms/backfillDescriptors.test.ts

@@ -332,6 +332,30 @@ describe("merger-proxy descriptor", () => {
     const todo = await descriptor.filterTodo!(candidates);
     expect(todo.map((c) => c.accession_number)).toEqual(["acc-prem"]);
   });
+
+  it("does not re-select a general proxy whose merger section was legitimately absent", async () => {
+    // A `DEF 14A` is usually an annual or extension vote carrying no merger
+    // section, and the processor writes no extraction row for one — by design.
+    // Selecting on the extraction row alone re-queues every general proxy of
+    // every known SPAC on every backfill, forever, each one re-paying the
+    // segmentation and AI cost to conclude again that there is nothing there.
+    // The resolved SECTION_NOT_FOUND trace is what makes the predicate
+    // converge.
+    await seedSpac(5);
+    await seedFiling({ cik: 5, accession_number: "acc-def14a", form: "DEF 14A" });
+    await new ExtractionDeadLetterRepo().recordResolved({
+      extractor_id: "merger-proxy",
+      accession_number: "acc-def14a",
+      section_name: "merger",
+      reason_code: "SECTION_NOT_FOUND",
+      detail: "no merger / business-combination / PIPE section text",
+      failed_extractor_version: "1.0.0",
+      source_run_id: null,
+    });
+
+    const descriptor = getBackfillDescriptor("merger-proxy")!;
+    expect(await descriptor.filterTodo!(await descriptor.selectCandidates())).toEqual([]);
+  });
 });
 
 describe("25-15 descriptor", () => {
@@ -362,6 +386,25 @@ describe("25-15 descriptor", () => {
     expect(todo.map((c) => c.accession_number)).toEqual(["acc-15"]);
   });
 
+  it("skips a 20-F the classifier ignores", async () => {
+    // 20-F routes to this extractor so an FPI CLOSE filing can record its
+    // combination; an ORDINARY annual report classifies `ignore` and writes
+    // nothing. Selecting it re-runs every annual report of every de-SPAC'd
+    // foreign private issuer on every backfill — six 20-Fs is six filings
+    // re-processed forever — so the shared predicate refuses it, while the
+    // Form 15 beside it is still selected.
+    await seedSpac(5);
+    await seedFiling({ cik: 5, accession_number: "acc-20f", form: "20-F" });
+    await seedFiling({ cik: 5, accession_number: "acc-15", form: "15-12G" });
+
+    const descriptor = getBackfillDescriptor("25-15")!;
+    const candidates = await descriptor.selectCandidates();
+    expect(candidates.map((c) => c.accession_number).sort()).toEqual(["acc-15", "acc-20f"]);
+
+    const todo = await descriptor.filterTodo!(candidates);
+    expect(todo.map((c) => c.accession_number)).toEqual(["acc-15"]);
+  });
+
   it("re-selects a 25-NSE recorded as deregistration that is actually unit separation", async () => {
     await seedSpac(5);
     await new SpacReportWriter().recordIpo({