Merge branch 'main' into alex/etl_mmap_34

Author: AskAlexSharov

.claude/skills/launch-bal-devnet-2/SKILL.md

@@ -0,0 +1,185 @@
+---
+name: launch-bal-devnet-2
+description: Launch erigon + Lighthouse on the bal-devnet-2 ethpandaops devnet (EIP-7928 Block Access Lists). Manages start/stop of both EL and CL clients with proper port offsets and JWT auth.
+allowed-tools: Bash, Read, Write, Edit, Glob
+allowed-prompts:
+  - tool: Bash
+    prompt: start, stop, and manage erigon and lighthouse processes for bal-devnet-2
+---
+
+# Launch bal-devnet-2 (EIP-7928 BAL Devnet)
+
+Run erigon (EL) + Lighthouse (CL) on the bal-devnet-2 ethpandaops devnet for testing EIP-7928 Block Access Lists.
+
+## Network Details
+
+| Parameter | Value |
+|-----------|-------|
+| Chain ID | 7033429093 |
+| Genesis timestamp | 1770388190 |
+| Amsterdam timestamp | 1770400508 (epoch 32) |
+| Lighthouse image | `ethpandaops/lighthouse:bal-devnet-2-65bb283` |
+| Lighthouse version | v8.0.1 (commit 65bb283, branch bal-devnet-2) |
+| Explorer | https://explorer.bal-devnet-2.ethpandaops.io |
+| Faucet | https://faucet.bal-devnet-2.ethpandaops.io |
+| RPC | https://rpc.bal-devnet-2.ethpandaops.io |
+| Checkpoint sync | https://checkpoint-sync.bal-devnet-2.ethpandaops.io |
+
+## Working Directory
+
+Ask the user where they want the working directory. Default suggestion: `~/bal-devnet-2/`.
+Use `$WORKDIR` throughout to refer to the chosen path.
+
+```
+$WORKDIR/
+├── genesis.json              # EL genesis
+├── config.yaml               # CL beacon config
+├── genesis.ssz               # CL genesis state
+├── testnet-config/           # Lighthouse testnet dir (config.yaml + genesis.ssz + deploy files)
+├── start-erigon.sh           # Erigon start script (run FIRST)
+├── start-lighthouse.sh       # Lighthouse start script (run SECOND)
+├── stop.sh                   # Stop both erigon + Lighthouse
+├── clean.sh                  # Stop, wipe data, re-init genesis
+├── erigon-data/              # Erigon datadir (contains jwt.hex)
+├── lighthouse-data/          # Lighthouse datadir
+├── erigon-console.log        # Erigon stdout/stderr
+└── lighthouse-console.log    # Lighthouse stdout/stderr
+```
+
+## Port Assignments (offset +100)
+
+| Service | Port | Protocol |
+|---------|------|----------|
+| Erigon HTTP RPC | 8645 | TCP |
+| Erigon Engine API (authrpc) | 8651 | TCP |
+| Erigon WebSocket | 8646 | TCP |
+| Erigon P2P | 30403 | TCP+UDP |
+| Erigon gRPC | 9190 | TCP |
+| Erigon Torrent | 42169 | TCP+UDP |
+| Erigon pprof | 6160 | TCP |
+| Erigon metrics | 6161 | TCP |
+| Lighthouse P2P | 9100 | TCP+UDP |
+| Lighthouse QUIC | 9101 | UDP |
+| Lighthouse HTTP API | 5152 | TCP |
+| Lighthouse metrics | 5264 | TCP |
+
+## Workflow
+
+### Step 1: Check Prerequisites
+
+1. Verify erigon binary exists at `./build/bin/erigon`. If not, invoke `/erigon-build`.
+2. Verify the Lighthouse Docker image is available:
+   ```bash
+   docker image inspect ethpandaops/lighthouse:bal-devnet-2-65bb283 > /dev/null 2>&1
+   ```
+   If not, pull it:
+   ```bash
+   docker pull ethpandaops/lighthouse:bal-devnet-2-65bb283
+   ```
+3. Verify config files exist in `$WORKDIR` (genesis.json, testnet-config/).
+   If not, download them:
+   ```bash
+   mkdir -p $WORKDIR/testnet-config
+   curl -sL -o $WORKDIR/genesis.json https://config.bal-devnet-2.ethpandaops.io/el/genesis.json
+   curl -sL -o $WORKDIR/testnet-config/config.yaml https://config.bal-devnet-2.ethpandaops.io/cl/config.yaml
+   curl -sL -o $WORKDIR/testnet-config/genesis.ssz https://config.bal-devnet-2.ethpandaops.io/cl/genesis.ssz
+   echo "0" > $WORKDIR/testnet-config/deposit_contract_block.txt
+   echo "0" > $WORKDIR/testnet-config/deploy_block.txt
+   ```
+
+### Step 2: Initialize Datadir (first run only)
+
+If `$WORKDIR/erigon-data/chaindata` does not exist:
+```bash
+./build/bin/erigon init --datadir $WORKDIR/erigon-data $WORKDIR/genesis.json
+```
+
+### Step 3: Create Scripts (first run only)
+
+If the start/stop/clean scripts don't exist yet, generate them. The scripts must use absolute paths based on `$WORKDIR`. Key details:
+
+**start-erigon.sh** — Runs erigon with `--externalcl`. Must start FIRST (creates JWT secret).
+- Env vars: `ERIGON_EXEC3_PARALLEL=true`, `ERIGON_ASSERT=true`, `ERIGON_EXEC3_WORKERS=12`, `LOG_HASH_MISMATCH_REASON=true`
+- Flags: `--datadir=$WORKDIR/erigon-data`, `--externalcl`, `--networkid=7033429093`, all 16 EL bootnodes, erigon static peers, `--prune.mode=minimal`, all offset ports (see port table), `--http.api=eth,erigon,engine,debug`, `--pprof`, `--metrics`
+- EL bootnodes: fetch from `https://config.bal-devnet-2.ethpandaops.io/api/v1/nodes/inventory` (extract enode URLs from `execution.enode` fields)
+
+**start-lighthouse.sh** — Runs Lighthouse via Docker with `--network=host`. Must start SECOND.
+- Checks JWT exists at `$WORKDIR/erigon-data/jwt.hex`
+- Docker container name: `bal-devnet-2-lighthouse`
+- Mounts: `$WORKDIR/testnet-config:/config:ro`, `$WORKDIR/lighthouse-data:/data`, JWT as `/jwt.hex:ro`
+- Flags: `--testnet-dir=/config`, `--execution-endpoint=http://127.0.0.1:8651`, `--execution-jwt=/jwt.hex`, all 15 CL ENR bootnodes, offset ports, `--checkpoint-sync-url=https://checkpoint-sync.bal-devnet-2.ethpandaops.io`
+- CL bootnodes: fetch from same inventory URL (extract ENR entries from `consensus.enr` fields)
+
+**stop.sh** — Stops Lighthouse (`docker stop bal-devnet-2-lighthouse`) then erigon (`pkill -f "datadir.*bal-devnet-2/erigon-data"`).
+
+**clean.sh** — Runs `stop.sh`, removes erigon chain data (chaindata, snapshots, txpool, nodes, temp) and lighthouse data, re-initializes genesis.
+
+### Step 4: Start Erigon (FIRST)
+
+Erigon must start first because it creates the JWT secret that Lighthouse needs.
+
+```bash
+cd $WORKDIR && nohup bash start-erigon.sh > erigon-console.log 2>&1 &
+```
+
+Verify it started:
+- Check `tail $WORKDIR/erigon-console.log` for startup messages
+- Check JWT exists: `ls $WORKDIR/erigon-data/jwt.hex`
+- Check port binding: `ss -tlnp | grep 8651`
+
+### Step 5: Start Lighthouse (SECOND)
+
+After erigon is running and JWT exists:
+
+```bash
+cd $WORKDIR && nohup bash start-lighthouse.sh > lighthouse-console.log 2>&1 &
+```
+
+Verify it started:
+- Check `tail $WORKDIR/lighthouse-console.log` for "Lighthouse started"
+- Look for "Loaded checkpoint block and state" (checkpoint sync)
+- Look for `peers: "N"` showing peer connections
+
+### Step 6: Monitor
+
+```bash
+# Erigon sync progress
+tail -f $WORKDIR/erigon-console.log
+
+# Lighthouse sync progress
+tail -f $WORKDIR/lighthouse-console.log
+
+# Check erigon block height via RPC
+curl -s http://localhost:8645 -X POST -H "Content-Type: application/json" \
+  -d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' | python3 -m json.tool
+
+# Check lighthouse sync status
+curl -s http://localhost:5152/eth/v1/node/syncing | python3 -m json.tool
+```
+
+### Step 7: Stop
+
+```bash
+bash $WORKDIR/stop.sh
+```
+
+This stops Lighthouse (via `docker stop`) then erigon (via `pkill`).
+
+### Step 8: Clean (wipe data and re-init)
+
+```bash
+bash $WORKDIR/clean.sh
+```
+
+This runs `stop.sh`, removes erigon chain data (chaindata, snapshots, txpool, nodes, temp) and lighthouse data, then re-initializes genesis. After clean, start again with Steps 4-5.
+
+## Troubleshooting
+
+| Problem | Solution |
+|---------|----------|
+| JWT auth fails | Ensure erigon started first and `jwt.hex` exists. Lighthouse must mount the same file. |
+| No EL peers | Check firewall allows port 30403. Try adding `--nat=extip:<your-ip>`. |
+| No CL peers | Check firewall allows port 9100/9101. ENR bootnodes may have changed — re-fetch from inventory. |
+| "Head is optimistic" | Normal during initial sync. Erigon is behind Lighthouse. Will resolve as erigon catches up. |
+| Engine API timeout | Check erigon is running and authrpc port 8651 is accessible. |
+| Port conflict | Check `ss -tlnp | grep <port>`. Kill conflicting process or use higher offset. |

db/recsplit/recsplit.go

@@ -36,6 +36,7 @@ import (
 
 	"github.com/erigontech/erigon/common"
 	"github.com/erigontech/erigon/common/assert"
+	"github.com/erigontech/erigon/common/background"
 	"github.com/erigontech/erigon/common/dir"
 	"github.com/erigontech/erigon/common/log/v3"
 	"github.com/erigontech/erigon/common/mmap"
@@ -164,6 +165,8 @@ type RecSplit struct {
 
 	noFsync bool // fsync is enabled by default, but tests can manually disable
 	timings Timings
+
+	progress *background.Progress // If set, tracks 0-100%: add-keys fills 0-50%, build fills 50-100%
 }
 
 type RecSplitArgs struct {
@@ -395,6 +398,9 @@ func (rs *RecSplit) ResetNextSalt() {
 	rs.collision = false
 	rs.keysAdded = 0
 	rs.salt++
+	if rs.progress != nil {
+		rs.progress.Processed.Store(0)
+	}
 	if rs.bucketCollector != nil {
 		rs.bucketCollector.Close()
 	}
@@ -519,6 +525,9 @@ func (rs *RecSplit) AddKey(key []byte, offset uint64) error {
 
 	rs.keysAdded++
 	rs.prevOffset = offset
+	if rs.progress != nil && rs.keysAdded%1024 == 0 {
+		rs.progress.Processed.Add(1024)
+	}
 	return nil
 }
 
@@ -790,6 +799,10 @@ func (rs *RecSplit) loadFuncBucket(k, v []byte, _ etl.CurrentTableReader, _ etl.
 			if err := rs.recsplitCurrentBucket(); err != nil {
 				return err
 			}
+			if rs.progress != nil {
+				// Build phase fills the 50–100% half: each bucket ≈ bucketSize keys worth.
+				rs.progress.Processed.Add(uint64(rs.bucketSize))
+			}
 		}
 		rs.currentBucketIdx = bucketIdx
 	}
@@ -820,6 +833,26 @@ func (rs *RecSplit) buildOffsetEf() error {
 	return nil
 }
 
+// KeyCount returns the number of keys added to the RecSplit.
+func (rs *RecSplit) KeyCount() uint64 { return rs.keysAdded }
+
+// BucketCount returns the number of buckets.
+func (rs *RecSplit) BucketCount() uint64 { return rs.bucketCount }
+
+// SetProgress wires a single progress tracker covering the full build lifecycle.
+// Total = 2*keyExpectedCount; AddKey fills 0→keyExpectedCount (0–50%) and
+// the bucket-building phase fills keyExpectedCount→2*keyExpectedCount (50–100%).
+// Progress is automatically reset on ResetNextSalt (collision retry).
+func (rs *RecSplit) SetProgress(p *background.Progress) {
+	if p == nil {
+		return
+	}
+	p.Name.Store(&rs.fileName)
+	p.Processed.Store(0)
+	p.Total.Store(2 * rs.keyExpectedCount)
+	rs.progress = p
+}
+
 // Build has to be called after all the keys have been added, and it initiates the process
 // of building the perfect hash function and writing index into a file
 func (rs *RecSplit) Build(ctx context.Context) error {

db/snaptype/type.go

@@ -492,11 +492,6 @@ func BuildIndex(ctx context.Context, info FileInfo, indexVersion version.Version
 	}
 	defer d.Close()
 
-	if p != nil {
-		fname := info.Name()
-		p.Name.Store(&fname)
-		p.Total.Store(uint64(d.Count()))
-	}
 	cfg.KeyCount = d.Count()
 	idxVer := indexVersion.Current
 	cfg.IndexFile = filepath.Join(info.Dir(), info.Type.IdxFileName(idxVer, info.From, info.To))
@@ -515,6 +510,7 @@ func BuildIndex(ctx context.Context, info FileInfo, indexVersion version.Version
 	defer d.MadvSequential().DisableReadAhead()
 
 	for {
+		rs.SetProgress(p)
 		g := d.MakeGetter()
 		var i, offset, nextPos uint64
 		word := make([]byte, 0, 4096)
@@ -560,11 +556,6 @@ func BuildIndexWithSnapName(ctx context.Context, info FileInfo, cfg recsplit.Rec
 	}
 	defer d.Close()
 
-	if p != nil {
-		fname := info.Name()
-		p.Name.Store(&fname)
-		p.Total.Store(uint64(d.Count()))
-	}
 	cfg.KeyCount = d.Count()
 	cfg.IndexFile = filepath.Join(info.Dir(), IdxFileName(info.Version, info.From, info.To, info.CaplinTypeString))
 	rs, err := recsplit.NewRecSplit(cfg, logger)
@@ -577,6 +568,7 @@ func BuildIndexWithSnapName(ctx context.Context, info FileInfo, cfg recsplit.Rec
 	defer d.MadvSequential().DisableReadAhead()
 
 	for {
+		rs.SetProgress(p)
 		g := d.MakeGetter()
 		var i, offset, nextPos uint64
 		word := make([]byte, 0, 4096)

db/snaptype2/block_types.go

@@ -244,12 +244,6 @@ var (
 					return fmt.Errorf("TransactionsIdx: at=%d-%d, pre index building, expect: %d, got %d", sn.From, sn.To, expectedCount, d.Count())
 				}
 
-				if p != nil {
-					name := sn.Name()
-					p.Name.Store(&name)
-					p.Total.Store(uint64(d.Count() * 2))
-				}
-
 				txnHashIdx, err := recsplit.NewRecSplit(recsplit.RecSplitArgs{
 					KeyCount: d.Count(),
 
@@ -291,6 +285,7 @@ var (
 				defer bodiesSegment.MadvSequential().DisableReadAhead()
 
 				for {
+					txnHashIdx.SetProgress(p)
 					g, bodyGetter := d.MakeGetter(), bodiesSegment.MakeGetter()
 					var ti, offset, nextPos uint64
 					blockNum := firstBlockNum
@@ -302,10 +297,6 @@ var (
 					}
 
 					for g.HasNext() {
-						if p != nil {
-							p.Processed.Add(1)
-						}
-
 						word, nextPos = g.Next(word[:0])
 						select {
 						case <-ctx.Done():

db/state/domain.go

@@ -1220,6 +1220,7 @@ func buildHashMapAccessor(ctx context.Context, g *seg.Reader, idxPath string, va
 			return err
 		}
 		g.Reset(0)
+		rs.SetProgress(p)
 		for g.HasNext() {
 			word, valPos = g.Next(word[:0])
 			if values {
@@ -1234,8 +1235,6 @@ func buildHashMapAccessor(ctx context.Context, g *seg.Reader, idxPath string, va
 
 			// Skip value
 			keyPos, _ = g.Skip()
-
-			p.Processed.Add(1)
 		}
 		if err = rs.Build(ctx); err != nil {
 			if rs.Collision() {

db/state/history.go

@@ -283,12 +283,12 @@ func (h *History) buildVI(ctx context.Context, historyIdxPath string, hist, efHi
 	for {
 		histReader.Reset(0)
 		iiReader.Reset(0)
+		rs.SetProgress(p)
 
 		valOffset = 0
 		for iiReader.HasNext() {
 			keyBuf, _ = iiReader.Next(keyBuf[:0])
 			valBuf, _ = iiReader.Next(valBuf[:0])
-			p.Processed.Add(1)
 
 			// fmt.Printf("ef key %x\n", keyBuf)