fix(nonce): detect block gaps, invalidate corrupt candidates, reduce OOM risk (#116)

Four fixes to make full mode nonce computation reliable:

1. Backfill: when Koios corrects a final nonce mismatch, also invalidate
   the candidate nonce for that epoch. Previously the corrupt candidate
   persisted and TICKN used it to compute wrong next-epoch nonces.

2. TICKN: before using a candidate nonce, verify its epoch has complete
   block coverage by comparing local block count against Koios. Skip
   TICKN and fall through to Koios if blocks are missing.

3. Integrity check: add Layer 3 that compares recent epoch block counts
   against Koios on startup. Detects silent block loss from pipeline
   restarts and invalidates affected candidate nonces.

4. Reduce historical sync channel buffer from 10000 to 2000 blocks
   (~440KB vs ~2.2MB) to prevent OOM at 512Mi pod limit.

Adds DeleteCandidateNonce to Store interface (sets candidate_nonce=NULL)
and fetchEpochBlockCount to NonceTracker (Koios epoch_info API).

Author: wcatz

db.go

@@ -132,6 +132,14 @@ func (s *PgStore) SetCandidateNonce(ctx context.Context, epoch int, nonce []byte
 	return err
 }
 
+func (s *PgStore) DeleteCandidateNonce(ctx context.Context, epoch int) error {
+	_, err := s.pool.Exec(ctx,
+		`UPDATE epoch_nonces SET candidate_nonce = NULL, updated_at = NOW() WHERE epoch = $1`,
+		epoch,
+	)
+	return err
+}
+
 func (s *PgStore) SetFinalNonce(ctx context.Context, epoch int, nonce []byte, source string) error {
 	_, err := s.pool.Exec(ctx,
 		`INSERT INTO epoch_nonces (epoch, evolving_nonce, final_nonce, source, updated_at)

integrity.go

@@ -106,6 +106,42 @@ func ValidateDBIntegrity(ctx context.Context, store Store, nonceTracker *NonceTr
 
 	log.Println("Layer 2 passed: blocks found on canonical chain")
 
+	// Layer 3: Block gap detection — compare recent epoch block counts against Koios.
+	// Catches silent block loss from pipeline restarts where stored blocks are valid
+	// (on canonical chain) but incomplete (missing blocks = corrupt nonce computation).
+	if nonceTracker != nil && nonceTracker.koiosClient != nil {
+		gapCtx, gapCancel := context.WithTimeout(ctx, 30*time.Second)
+		defer gapCancel()
+		gapsFound := false
+		for checkEpoch := epoch; checkEpoch >= epoch-4 && checkEpoch >= 0; checkEpoch-- {
+			localCount, countErr := store.GetBlockCountForEpoch(gapCtx, checkEpoch)
+			if countErr != nil || localCount == 0 {
+				continue
+			}
+			koiosCount, koiosErr := nonceTracker.fetchEpochBlockCount(gapCtx, checkEpoch)
+			if koiosErr != nil || koiosCount == 0 {
+				continue
+			}
+			if localCount < koiosCount {
+				log.Printf("BLOCK GAP: epoch %d has %d blocks, Koios has %d (missing %d)",
+					checkEpoch, localCount, koiosCount, koiosCount-localCount)
+				gapsFound = true
+				// Invalidate candidate nonce for this epoch — it was computed from incomplete data
+				if delErr := store.DeleteCandidateNonce(gapCtx, checkEpoch); delErr != nil {
+					log.Printf("Failed to invalidate candidate for epoch %d: %v", checkEpoch, delErr)
+				}
+			} else {
+				log.Printf("Layer 3: epoch %d block count OK (%d)", checkEpoch, localCount)
+			}
+			time.Sleep(50 * time.Millisecond)
+		}
+		if gapsFound {
+			log.Println("Layer 3: block gaps detected in recent epochs — candidate nonces invalidated")
+		} else {
+			log.Println("Layer 3 passed: recent epoch block counts match Koios")
+		}
+	}
+
 	// If blocks are valid but nonce is stale, repair it
 	if nonceStale {
 		log.Printf("Repairing stale nonce for epoch %d from blocks table...", epoch)

main.go

@@ -590,8 +590,9 @@ func (i *Indexer) runChainTail() error {
 		syncCtx, syncCancel := context.WithCancel(context.Background())
 		defer syncCancel()
 
-		// Buffered channel decouples fast chain sync from slower DB writes
-		blockCh := make(chan BlockData, 10000)
+		// Buffered channel decouples fast chain sync from slower DB writes.
+		// 2000 blocks × ~220 bytes ≈ 440KB — safe for 512Mi pod limit.
+		blockCh := make(chan BlockData, 2000)
 
 		onCaughtUp := func() {
 			log.Println("Historical sync caught up, stopping ChainSyncer...")

nonce.go

@@ -430,8 +430,21 @@ func (nt *NonceTracker) GetNonceForEpoch(epoch int) ([]byte, error) {
 		if candErr != nil {
 			log.Printf("TICKN: GetCandidateNonce(%d) failed: %v", candidateEpoch, candErr)
 		} else if candidate == nil {
-			log.Printf("TICKN: GetCandidateNonce(%d) returned nil", candidateEpoch)
+			log.Printf("TICKN: GetCandidateNonce(%d) returned nil (invalidated or missing)", candidateEpoch)
 		} else {
+			// Verify the candidate epoch has complete block coverage before trusting it.
+			// A candidate computed from incomplete chain data (block gaps) is wrong.
+			localCount, _ := nt.store.GetBlockCountForEpoch(ctx, candidateEpoch)
+			if nt.koiosClient != nil && localCount > 0 {
+				koiosCount, koiosErr := nt.fetchEpochBlockCount(ctx, candidateEpoch)
+				if koiosErr == nil && koiosCount > 0 && localCount < koiosCount {
+					log.Printf("TICKN: candidate epoch %d has %d blocks, Koios has %d — skipping TICKN (incomplete data)",
+						candidateEpoch, localCount, koiosCount)
+					candidate = nil
+				}
+			}
+		}
+		if candidate != nil {
 			log.Printf("TICKN: got candidate for epoch %d: %s", candidateEpoch, hex.EncodeToString(candidate))
 			// η_ph = prevHash of the last block of etaPhEpoch = hash of second-to-last block.
 			// This matches how ComputeEpochNonce tracks labNonce via prevBlockHash.
@@ -673,6 +686,11 @@ func (nt *NonceTracker) BackfillNonces(ctx context.Context) error {
 						log.Printf("Epoch %d: computed %s… != Koios %s… — using Koios",
 							epoch, computedHex[:16], koiosHex[:16])
 						eta0 = koiosNonce
+						// Candidate nonce was computed from the same incomplete chain data.
+						// Invalidate it so TICKN won't use a corrupt candidate for epoch+1.
+						if delErr := nt.store.DeleteCandidateNonce(ctx, epoch); delErr != nil {
+							log.Printf("Failed to invalidate candidate nonce for epoch %d: %v", epoch, delErr)
+						}
 					}
 				}
 				time.Sleep(50 * time.Millisecond) // rate limit
@@ -1051,3 +1069,36 @@ func (nt *NonceTracker) fetchNonceFromKoios(ctx context.Context, epoch int) ([]b
 
 	return nonce, nil
 }
+
+// fetchEpochBlockCount returns the total block count for an epoch from Koios epoch_info.
+func (nt *NonceTracker) fetchEpochBlockCount(ctx context.Context, epoch int) (int, error) {
+	url := fmt.Sprintf(koiosRESTBase(nt.networkMagic)+"/epoch_info?_epoch_no=%d&select=blk_count", epoch)
+	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+	if err != nil {
+		return 0, err
+	}
+	resp, err := koiosHTTPClient.Do(req)
+	if err != nil {
+		return 0, err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return 0, err
+	}
+	if resp.StatusCode != 200 {
+		return 0, fmt.Errorf("koios returned %d", resp.StatusCode)
+	}
+
+	var result []struct {
+		BlkCount int `json:"blk_count"`
+	}
+	if err := json.Unmarshal(body, &result); err != nil {
+		return 0, err
+	}
+	if len(result) == 0 {
+		return 0, fmt.Errorf("no epoch_info for epoch %d", epoch)
+	}
+	return result[0].BlkCount, nil
+}

store.go

@@ -36,6 +36,7 @@ type Store interface {
 	InsertBlockBatch(ctx context.Context, blocks []BlockData) (int, error)
 	UpsertEvolvingNonce(ctx context.Context, epoch int, nonce []byte, blockCount int) error
 	SetCandidateNonce(ctx context.Context, epoch int, nonce []byte) error
+	DeleteCandidateNonce(ctx context.Context, epoch int) error
 	SetFinalNonce(ctx context.Context, epoch int, nonce []byte, source string) error
 	GetFinalNonce(ctx context.Context, epoch int) ([]byte, error)
 	GetEvolvingNonce(ctx context.Context, epoch int) ([]byte, int, error)
@@ -208,6 +209,14 @@ func (s *SqliteStore) SetCandidateNonce(ctx context.Context, epoch int, nonce []
 	return err
 }
 
+func (s *SqliteStore) DeleteCandidateNonce(ctx context.Context, epoch int) error {
+	_, err := s.db.ExecContext(ctx,
+		`UPDATE epoch_nonces SET candidate_nonce = NULL, updated_at = datetime('now') WHERE epoch = ?`,
+		epoch,
+	)
+	return err
+}
+
 func (s *SqliteStore) SetFinalNonce(ctx context.Context, epoch int, nonce []byte, source string) error {
 	_, err := s.db.ExecContext(ctx,
 		`INSERT INTO epoch_nonces (epoch, evolving_nonce, final_nonce, source, updated_at)