Skip to content

Commit 8c07d53

Browse files
yperbasisclaudenorthsurapee
authored
cmd/utils/app: process all files passed to import command (erigontech#21513)
## Problem The Hive `legacy-cancun` BlockchainTests suite intermittently fails 4 tests with `client did not start: timed out waiting for container startup`: - `walletReorganizeOwners_{Cancun,Istanbul,London,Paris}` (e.g. [this run](https://hive.ethpandaops.io/#/test/generic/1779833679-c7f17208e0b5dbf959b3448bbdaed60e)). They took ~181s, just over Hive's 180s container-startup timeout. The same test on other forks (Shanghai/Berlin) and `ForkStressTest_*` sit at 146–150s — right at the cliff. ## Root cause The `import` command documents multi-file support: ``` USAGE: erigon import [command options] <filename> (<filename 2> ... <filename N>) ``` but `importChain` only processed `cliCtx.Args().First()`, silently ignoring the rest. That forced Hive's erigon entrypoint into a one-process-per-block-file loop. For `walletReorganizeOwners` (235 block files) that is 235 full erigon startups — measured at ~0.5s/process × 261 = 145s of pure startup, while actual block execution is ~40ms each. go-ethereum has no such problem: its entrypoint imports every block in one `geth import` invocation. ## Fix Iterate every file argument in a single process, tolerating per-file failures when several files are given (matching go-ethereum). This lets the hive entrypoint import all blocks in one invocation, collapsing hundreds of process startups into one (~150s → ~10s). It also force-disables the embedded MCP server for the one-shot import (`--mcp.disable`, alongside the existing NAT / downloader / external-consensus disables) — a batch import has no use for it. ## Companion change (required) The hive entrypoint must pass all block files in one invocation: ethereum/hive#1519. Order matters — the hive change depends on this one (on an old erigon, `import /blocks/*` would import only the first block). ## Testing - New unit tests (`import_cmd_test.go`): all-files iteration, per-file failure tolerance, single-file error surfacing. - Real-binary check: `erigon import a.rlp b.rlp` now attempts **both** files in one process (previously stopped after the first). - `make lint` clean; `make erigon integration` builds. --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Giulio Rebuffo <111551070+Giulio2002@users.noreply.github.com>
1 parent 74fbe0d commit 8c07d53

2 files changed

Lines changed: 100 additions & 6 deletions

File tree

cmd/utils/app/import_cmd.go

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,8 @@ const (
5454
importBatchSize = 2500
5555
)
5656

57+
var errInterrupted = errors.New("interrupted")
58+
5759
var importCommand = cli.Command{
5860
Action: MigrateFlags(importChain),
5961
Name: "import",
@@ -87,6 +89,7 @@ func importChain(cliCtx *cli.Context) error {
8789
utils.NATFlag.Name: "none",
8890
utils.NoDownloaderFlag.Name: "true",
8991
utils.ExternalConsensusFlag.Name: "true",
92+
utils.MCPDisableFlag.Name: "true",
9093
} {
9194
if err := cliCtx.Set(flag, value); err != nil {
9295
return fmt.Errorf("importChain: set %s=%s: %w", flag, value, err)
@@ -116,11 +119,28 @@ func importChain(cliCtx *cli.Context) error {
116119
return err
117120
}
118121

119-
if err := ImportChain(ethereum, ethereum.ChainDB(), cliCtx.Args().First(), logger); err != nil {
120-
return err
121-
}
122+
return importFiles(cliCtx.Args().Slice(), logger, func(fn string) error {
123+
return ImportChain(ethereum, ethereum.ChainDB(), fn, logger)
124+
})
125+
}
122126

123-
return nil
127+
// importFiles imports each file in order; with more than one file, per-file
128+
// failures are logged and skipped (matching go-ethereum), except a user
129+
// interrupt aborts the whole command.
130+
func importFiles(files []string, logger log.Logger, importOne func(fn string) error) error {
131+
var importErr error
132+
for _, fn := range files {
133+
if err := importOne(fn); err != nil {
134+
importErr = err
135+
if errors.Is(err, errInterrupted) {
136+
return err
137+
}
138+
if len(files) > 1 {
139+
logger.Error("Import error", "file", fn, "err", err)
140+
}
141+
}
142+
}
143+
return importErr
124144
}
125145

126146
func ImportChain(ethereum *eth.Ethereum, chainDB kv.RwDB, fn string, logger log.Logger) error {
@@ -169,7 +189,7 @@ func ImportChain(ethereum *eth.Ethereum, chainDB kv.RwDB, fn string, logger log.
169189
for batch := 0; ; batch++ {
170190
// Load a batch of RLP blocks.
171191
if checkInterrupt() {
172-
return errors.New("interrupted")
192+
return errInterrupted
173193
}
174194
i := 0
175195
for ; i < importBatchSize; i++ {
@@ -192,7 +212,7 @@ func ImportChain(ethereum *eth.Ethereum, chainDB kv.RwDB, fn string, logger log.
192212
}
193213
// Import the batch.
194214
if checkInterrupt() {
195-
return errors.New("interrupted")
215+
return errInterrupted
196216
}
197217

198218
br, _ := ethereum.BlockIO()

cmd/utils/app/import_cmd_test.go

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
// Copyright 2026 The Erigon Authors
2+
// This file is part of Erigon.
3+
//
4+
// Erigon is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU Lesser General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
//
9+
// Erigon is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU Lesser General Public License for more details.
13+
//
14+
// You should have received a copy of the GNU Lesser General Public License
15+
// along with Erigon. If not, see <http://www.gnu.org/licenses/>.
16+
17+
package app
18+
19+
import (
20+
"errors"
21+
"testing"
22+
23+
"github.com/stretchr/testify/require"
24+
25+
"github.com/erigontech/erigon/common/log/v3"
26+
)
27+
28+
func TestImportFilesProcessesEveryFile(t *testing.T) {
29+
files := []string{"0001.rlp", "0002.rlp", "0003.rlp"}
30+
var imported []string
31+
err := importFiles(files, log.Root(), func(fn string) error {
32+
imported = append(imported, fn)
33+
return nil
34+
})
35+
require.NoError(t, err)
36+
require.Equal(t, files, imported)
37+
}
38+
39+
func TestImportFilesContinuesPastPerFileFailure(t *testing.T) {
40+
files := []string{"0001.rlp", "0002.rlp", "0003.rlp"}
41+
badBlock := errors.New("invalid block")
42+
var imported []string
43+
err := importFiles(files, log.Root(), func(fn string) error {
44+
imported = append(imported, fn)
45+
if fn == "0002.rlp" {
46+
return badBlock
47+
}
48+
return nil
49+
})
50+
require.ErrorIs(t, err, badBlock)
51+
require.Equal(t, files, imported, "a failing block file must not stop import of later files")
52+
}
53+
54+
func TestImportFilesSingleFileSurfacesError(t *testing.T) {
55+
badBlock := errors.New("invalid block")
56+
err := importFiles([]string{"0001.rlp"}, log.Root(), func(fn string) error {
57+
return badBlock
58+
})
59+
require.ErrorIs(t, err, badBlock)
60+
}
61+
62+
func TestImportFilesStopsOnInterrupt(t *testing.T) {
63+
files := []string{"0001.rlp", "0002.rlp", "0003.rlp"}
64+
var imported []string
65+
err := importFiles(files, log.Root(), func(fn string) error {
66+
imported = append(imported, fn)
67+
if fn == "0002.rlp" {
68+
return errInterrupted
69+
}
70+
return nil
71+
})
72+
require.ErrorIs(t, err, errInterrupted)
73+
require.Equal(t, []string{"0001.rlp", "0002.rlp"}, imported, "user interrupt must abort the whole import, not just skip the current file")
74+
}

0 commit comments

Comments
 (0)