xact.dot

// walshadow — XactBuffer lifecycle (insert → maybe-evict → commit drain)
// Strict layered DAG: producers above their consumers, single cluster
// for the drain pipeline, all other nodes bare. No constraint=false
// except the abort→spill backwards unlink.
//
// regeneration spec:
// sources of truth: plans/xact.md · src/xact/{xact_buffer,spill}.rs · src/emit/pipeline/reorder.rs · src/backfill/backup_backfill.rs
// subsumes: plans/xact.md § "Buffer shape" + "Eviction policy" + "Spill backend" + "Drain shape"
// quality bar:
//   - spill round-trip (write → file → read) visually traceable
//   - subx tracker feeds drain without overlapping spill IO
//   - evbus → kmerge edge doesn't entangle with main merge edges
// shared style: palette.md
digraph xact {
  rankdir=TB;
  compound=true;
  graph [fontname="Helvetica", labelloc="t", label="walshadow — XactBuffer + spill + commit drain", fontsize=14, splines=spline, nodesep=0.35, ranksep=0.55, bgcolor="#272623", fontcolor="#ECE1D7"];
  node  [fontname="Helvetica", fontsize=10, shape=box, style="rounded,filled", color="#6E6963", fontcolor="#ECE1D7"];
  edge  [fontname="Helvetica", fontsize=9, color="#c1a78e", fontcolor="#ECE1D7"];

  // ════════ Rank 0: producers ════════
  // catevt+decoder on the left (both feed buf); xaclog on the right
  // (feeds subt + abort + merge, all aligned beneath it)
  { rank=same;
    catevt  [label="ShadowCatalog\nschema_event_tx\nAdded / Changed / Dropped", fillcolor="#4D4D28"];
    decoder [label="BufferingDecoderSink\nheap + TOAST birth/death reshape\nraw stash: dirty tree · SMGR marker ·\nspanned-lookup miss (NotCovered /\nDropped / Ambiguous)\nstamps source_lsn, xid", fillcolor="#4D4128"];
    xaclog  [label="XLOG_XACT_*\nASSIGNMENT 0x50 (hint)\nCOMMIT 0x00 / _PREPARED 0x30\nABORT  0x20 / _PREPARED  0x40", fillcolor="#3D3D54"];
  }

  // ════════ Rank 1: state — buf | subt | abort | idle, in producer-aligned order ════════
  { rank=same;
    buf [label=<
      <TABLE BORDER="0" CELLBORDER="0" CELLSPACING="2" CELLPADDING="2">
        <TR><TD ALIGN="LEFT"><B>XactBuffer</B></TD></TR>
        <TR><TD ALIGN="LEFT">inflight: HashMap&lt;xid, XactState&gt;</TD></TR>
        <TR><TD ALIGN="LEFT">markers · pending stash · unfinished commits</TD></TR>
        <TR><TD ALIGN="LEFT">bytes_in_memory · drain_resident · SpillStore</TD></TR>
        <TR><TD ALIGN="LEFT"><FONT POINT-SIZE="9"><I>XactState (per xid):</I></FONT></TD></TR>
        <TR><TD ALIGN="LEFT"><FONT POINT-SIZE="9">first_lsn · in_mem: Vec&lt;SpillEntry&gt; · in_mem_bytes</FONT></TD></TR>
        <TR><TD ALIGN="LEFT"><FONT POINT-SIZE="9">spill: Option&lt;SpillWriter&gt; · spill_bytes</FONT></TD></TR>
        <TR><TD ALIGN="LEFT"><FONT POINT-SIZE="9">events: Vec&lt;(lsn, DrainEntry)&gt; · stash_rfns</FONT></TD></TR>
        <TR><TD ALIGN="LEFT"><FONT POINT-SIZE="9">SpillEntry = Heap | Chunk | ToastDelete | Raw</FONT></TD></TR>
      </TABLE>
    >, fillcolor="#4D4128", shape=box];

    subt  [label="SubxactTracker\nparent: HashMap<xid, top>\nchildren: HashMap<top, Vec<xid>>", fillcolor="#4D4128"];

    abort [label="abort\ndrop transaction + children\nunlink spill files\nadvance consumed position", fillcolor="#4D4128"];

    idle  [label="idle progress\nadvance only with no\nopen transaction", fillcolor="#4D4128"];
  }

  // producer → state (vertical drops, no cross-traffic)
  catevt  -> buf   [label="on_schema_event\n(xid, source_lsn, ev)", color="#CBA85E", style=dashed];
  decoder -> buf   [label="absorb Heap / Chunk /\nToastDelete / Raw"];
  xaclog  -> subt  [label="ASSIGNMENT 0x50\nadd_subxact(top, subs)"];
  xaclog  -> abort [label="ABORT / _PREPARED"];
  buf     -> idle  [style=dashed, label="xacts_active == 0"];

  // ════════ Rank 2: evict + spill (sidecar pair under buf) ════════
  { rank=same;
    evict [label="maybe_evict\nbytes_in_memory > xact_buffer_max (64 MiB)\npick largest in-mem xact\n(mirrors PG ReorderBufferLargestTXN)\n→ evict_xact: lazy-open SpillWriter,\ndrain in_mem → write(entry), zero bytes\nDrainEntry events stay in memory", fillcolor="#4D3A28"];
    spill [label="{spill}/xid-{xid:010}-{first_lsn:016X}.bin\n[\"WS\" magic | u16 ver=6]\n[tag u8 | u32 LE inner_len | body]*\n0 Heap · 1 Chunk · 2 ToastDelete ·\n3 Raw · 4 descriptor dict\nappend-only, fsync on finish", fillcolor="#4D3850", shape=note];
  }
  buf   -> evict [label="after every absorb\nwhile over budget"];
  evict -> spill [color="#6E6963", style=dashed, label="SpillWriter::write"];
  abort -> spill [color="#6E6963", style=dashed, label="unlink"];

  // ════════ Rank 3+: commit drain (linear pipeline, the only remaining cluster) ════════
  subgraph cluster_drain {
    label="commit drain — merge transaction family, preserve WAL order"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7";

    merge [label="pull (top + subxids) from inflight\nper xid: SpillReader + in_mem + events\n→ source_lsn k-way merge\nChunk → ChunkMap + ToastRow birth\nToastDelete → tombstone row\nRaw → resolve_stash verdict at commit next_lsn:\n  Toast → chunk decode · Ordinary → row fanout\n  Ambiguous → fatal · tombstoned/uncovered → discard\nTIE: control event BEFORE heap", fillcolor="#4D4128"];

    dispatch [label="bounded committed batches\none shared walk keeps rows, TOAST,\nand table changes in WAL order", fillcolor="#4D4128"];

    merge -> dispatch;
  }
  xaclog -> merge [label="COMMIT / _PREPARED\nparse_xact_payload"];
  subt   -> merge [label="(top + subxids)\nauthoritative on COMMIT", style=dashed];
  spill  -> merge [color="#6E6963", style=dashed, label="SpillReader::next\nthen unlink"];

  // ════════ Rank below drain: shared consumers ════════
  detoast [label="resolve large values\ncurrent transaction first,\nmirror history second", fillcolor="#4D4128"];
  consumer [label="shared consumers\nlive pipeline: ClickHouse or metrics only\nbackup replay: same ordered walk", fillcolor="#5D4628"];
  dispatch -> detoast [color="#BF8C5F", penwidth=2, label="rows"];
  detoast -> consumer [color="#BF8C5F", penwidth=2, label="decoded rows"];
  dispatch -> consumer [color="#BF8C5F", style=dashed, label="table changes"];
  store [label="TOAST mirror\nsave changes before publishing commit\ninterleave table wipes and rewrites", fillcolor="#5D4628", shape=cylinder];
  dispatch -> store [color="#BF8C5F", style=dashed, label="TOAST changes"];

  // ════════ Bottom: durable floor fan-in ════════
  safe [label="ack collector + transaction state\nchoose safe restart point\nstay behind open or unfinished work", fillcolor="#4D3A28"];
  manifest [label="manifest.toml\nsaved restart state", fillcolor="#4D3850", shape=note];
  consumer -> safe [color="#b380b0", style=dotted, label="downstream completed"];
  buf      -> safe [color="#b380b0", style=dotted, label="open transactions"];
  dispatch -> safe [color="#b380b0", style=dotted, label="drained"];
  abort    -> safe [color="#b380b0", style=dotted];
  idle     -> safe [color="#b380b0", style=dotted];
  safe     -> manifest [color="#b380b0", style=dotted, penwidth=2, label="status update"];

  // ════════ Legend ════════
  legend [shape=plaintext, label=<
    <TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0" CELLPADDING="4">
      <TR><TD COLSPAN="2" BGCOLOR="#34302c"><B>node fill — role</B></TD></TR>
      <TR><TD BGCOLOR="#3D3D54" WIDTH="28"></TD><TD>source PG / XLOG_XACT records</TD></TR>
      <TR><TD BGCOLOR="#4D3A28"></TD><TD>eviction policy (sync, after absorb)</TD></TR>
      <TR><TD BGCOLOR="#4D4128"></TD><TD>XactBuffer / decoder / drain</TD></TR>
      <TR><TD BGCOLOR="#4D4D28"></TD><TD>ShadowCatalog schema_event_tx</TD></TR>
      <TR><TD BGCOLOR="#5D4628"></TD><TD>downstream consumer (pipeline / gap replay)</TD></TR>
      <TR><TD BGCOLOR="#4D3850"></TD><TD>on-disk artifact (spill / manifest)</TD></TR>
      <TR><TD COLSPAN="2" BGCOLOR="#34302c"><B>edge colour</B></TD></TR>
      <TR><TD><FONT COLOR="#c1a78e"><B>━━</B></FONT></TD><TD>buffer-internal control / data</TD></TR>
      <TR><TD><FONT COLOR="#CBA85E"><B>┄┄</B></FONT></TD><TD>schema event (lsn-stamped)</TD></TR>
      <TR><TD><FONT COLOR="#BF8C5F"><B>━━</B></FONT></TD><TD>walk steps to pipeline / replay consumers</TD></TR>
      <TR><TD><FONT COLOR="#6E6963"><B>┄┄</B></FONT></TD><TD>spill file IO (write / read / unlink)</TD></TR>
      <TR><TD><FONT COLOR="#b380b0"><B>···</B></FONT></TD><TD>ack + manifest durability</TD></TR>
      <TR><TD COLSPAN="2" BGCOLOR="#34302c"><B>k-way merge — ordering rules</B></TD></TR>
      <TR><TD COLSPAN="2">heads sorted ASC by source_lsn</TD></TR>
      <TR><TD COLSPAN="2">tie: catalog event BEFORE tuple (PG writes pg_class first)</TD></TR>
      <TR><TD COLSPAN="2">heaps detoast post-merge; chunk maps resolve rows; ToastRows persist separately</TD></TR>
      <TR><TD COLSPAN="2">saved restart point never passes open or unfinished transaction</TD></TR>
    </TABLE>
  >];
  manifest -> legend [style=invis];
}