Skip to content

Commit 56c4f2b

Browse files
holimanlightclientMariusVanDerWijdenshemnon
authored
core/vm, cmd/evm: implement eof validation (ethereum#30418)
The bulk of this PR is authored by @lightclient , in the original EOF-work. More recently, the code has been picked up and reworked for the new EOF specification, by @MariusVanDerWijden , in ethereum#29518, and also @shemnon has contributed with fixes. This PR is an attempt to start eating the elephant one small bite at a time, by selecting only the eof-validation as a standalone piece which can be merged without interfering too much in the core stuff. In this PR: - [x] Validation of eof containers, lifted from ethereum#29518, along with test-vectors from consensus-tests and fuzzing, to ensure that the move did not lose any functionality. - [x] Definition of eof opcodes, which is a prerequisite for validation - [x] Addition of `undefined` to a jumptable entry item. I'm not super-happy with this, but for the moment it seems the least invasive way to do it. A better way might be to go back and allowing nil-items or nil execute-functions to denote "undefined". - [x] benchmarks of eof validation speed --------- Co-authored-by: lightclient <[email protected]> Co-authored-by: Marius van der Wijden <[email protected]> Co-authored-by: Danno Ferrin <[email protected]>
1 parent 6416813 commit 56c4f2b

28 files changed

+9972
-251
lines changed

‎cmd/evm/eofparse.go

+200
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
// Copyright 2023 The go-ethereum Authors
2+
// This file is part of go-ethereum.
3+
//
4+
// go-ethereum is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
//
9+
// go-ethereum is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU General Public License for more details.
13+
//
14+
// You should have received a copy of the GNU General Public License
15+
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
16+
17+
package main
18+
19+
import (
20+
"bufio"
21+
"encoding/hex"
22+
"encoding/json"
23+
"fmt"
24+
"io/fs"
25+
"os"
26+
"path/filepath"
27+
"strings"
28+
29+
"github.com/ethereum/go-ethereum/core/vm"
30+
"github.com/ethereum/go-ethereum/log"
31+
"github.com/urfave/cli/v2"
32+
)
33+
34+
func init() {
35+
jt = vm.NewPragueEOFInstructionSetForTesting()
36+
}
37+
38+
var (
39+
jt vm.JumpTable
40+
initcode = "INITCODE"
41+
)
42+
43+
func eofParseAction(ctx *cli.Context) error {
44+
// If `--test` is set, parse and validate the reference test at the provided path.
45+
if ctx.IsSet(refTestFlag.Name) {
46+
var (
47+
file = ctx.String(refTestFlag.Name)
48+
executedTests int
49+
passedTests int
50+
)
51+
err := filepath.Walk(file, func(path string, info fs.FileInfo, err error) error {
52+
if err != nil {
53+
return err
54+
}
55+
if info.IsDir() {
56+
return nil
57+
}
58+
log.Debug("Executing test", "name", info.Name())
59+
passed, tot, err := executeTest(path)
60+
passedTests += passed
61+
executedTests += tot
62+
return err
63+
})
64+
if err != nil {
65+
return err
66+
}
67+
log.Info("Executed tests", "passed", passedTests, "total executed", executedTests)
68+
return nil
69+
}
70+
// If `--hex` is set, parse and validate the hex string argument.
71+
if ctx.IsSet(hexFlag.Name) {
72+
if _, err := parseAndValidate(ctx.String(hexFlag.Name), false); err != nil {
73+
return fmt.Errorf("err: %w", err)
74+
}
75+
fmt.Println("OK")
76+
return nil
77+
}
78+
// If neither are passed in, read input from stdin.
79+
scanner := bufio.NewScanner(os.Stdin)
80+
scanner.Buffer(make([]byte, 1024*1024), 10*1024*1024)
81+
for scanner.Scan() {
82+
l := strings.TrimSpace(scanner.Text())
83+
if strings.HasPrefix(l, "#") || l == "" {
84+
continue
85+
}
86+
if _, err := parseAndValidate(l, false); err != nil {
87+
fmt.Printf("err: %v\n", err)
88+
} else {
89+
fmt.Println("OK")
90+
}
91+
}
92+
if err := scanner.Err(); err != nil {
93+
fmt.Println(err.Error())
94+
}
95+
return nil
96+
}
97+
98+
type refTests struct {
99+
Vectors map[string]eOFTest `json:"vectors"`
100+
}
101+
102+
type eOFTest struct {
103+
Code string `json:"code"`
104+
Results map[string]etResult `json:"results"`
105+
ContainerKind string `json:"containerKind"`
106+
}
107+
108+
type etResult struct {
109+
Result bool `json:"result"`
110+
Exception string `json:"exception,omitempty"`
111+
}
112+
113+
func executeTest(path string) (int, int, error) {
114+
src, err := os.ReadFile(path)
115+
if err != nil {
116+
return 0, 0, err
117+
}
118+
var testsByName map[string]refTests
119+
if err := json.Unmarshal(src, &testsByName); err != nil {
120+
return 0, 0, err
121+
}
122+
passed, total := 0, 0
123+
for testsName, tests := range testsByName {
124+
for name, tt := range tests.Vectors {
125+
for fork, r := range tt.Results {
126+
total++
127+
_, err := parseAndValidate(tt.Code, tt.ContainerKind == initcode)
128+
if r.Result && err != nil {
129+
log.Error("Test failure, expected validation success", "name", testsName, "idx", name, "fork", fork, "err", err)
130+
continue
131+
}
132+
if !r.Result && err == nil {
133+
log.Error("Test failure, expected validation error", "name", testsName, "idx", name, "fork", fork, "have err", r.Exception, "err", err)
134+
continue
135+
}
136+
passed++
137+
}
138+
}
139+
}
140+
return passed, total, nil
141+
}
142+
143+
func parseAndValidate(s string, isInitCode bool) (*vm.Container, error) {
144+
if len(s) >= 2 && strings.HasPrefix(s, "0x") {
145+
s = s[2:]
146+
}
147+
b, err := hex.DecodeString(s)
148+
if err != nil {
149+
return nil, fmt.Errorf("unable to decode data: %w", err)
150+
}
151+
return parse(b, isInitCode)
152+
}
153+
154+
func parse(b []byte, isInitCode bool) (*vm.Container, error) {
155+
var c vm.Container
156+
if err := c.UnmarshalBinary(b, isInitCode); err != nil {
157+
return nil, err
158+
}
159+
if err := c.ValidateCode(&jt, isInitCode); err != nil {
160+
return nil, err
161+
}
162+
return &c, nil
163+
}
164+
165+
func eofDumpAction(ctx *cli.Context) error {
166+
// If `--hex` is set, parse and validate the hex string argument.
167+
if ctx.IsSet(hexFlag.Name) {
168+
return eofDump(ctx.String(hexFlag.Name))
169+
}
170+
// Otherwise read from stdin
171+
scanner := bufio.NewScanner(os.Stdin)
172+
scanner.Buffer(make([]byte, 1024*1024), 10*1024*1024)
173+
for scanner.Scan() {
174+
l := strings.TrimSpace(scanner.Text())
175+
if strings.HasPrefix(l, "#") || l == "" {
176+
continue
177+
}
178+
if err := eofDump(l); err != nil {
179+
return err
180+
}
181+
fmt.Println("")
182+
}
183+
return scanner.Err()
184+
}
185+
186+
func eofDump(hexdata string) error {
187+
if len(hexdata) >= 2 && strings.HasPrefix(hexdata, "0x") {
188+
hexdata = hexdata[2:]
189+
}
190+
b, err := hex.DecodeString(hexdata)
191+
if err != nil {
192+
return fmt.Errorf("unable to decode data: %w", err)
193+
}
194+
var c vm.Container
195+
if err := c.UnmarshalBinary(b, false); err != nil {
196+
return err
197+
}
198+
fmt.Println(c.String())
199+
return nil
200+
}

‎cmd/evm/eofparse_test.go

+166
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
package main
2+
3+
import (
4+
"bufio"
5+
"bytes"
6+
"encoding/hex"
7+
"fmt"
8+
"os"
9+
"strings"
10+
"testing"
11+
12+
"github.com/ethereum/go-ethereum/common"
13+
"github.com/ethereum/go-ethereum/core/vm"
14+
)
15+
16+
func FuzzEofParsing(f *testing.F) {
17+
// Seed with corpus from execution-spec-tests
18+
for i := 0; ; i++ {
19+
fname := fmt.Sprintf("testdata/eof/eof_corpus_%d.txt", i)
20+
corpus, err := os.Open(fname)
21+
if err != nil {
22+
break
23+
}
24+
f.Logf("Reading seed data from %v", fname)
25+
scanner := bufio.NewScanner(corpus)
26+
scanner.Buffer(make([]byte, 1024), 10*1024*1024)
27+
for scanner.Scan() {
28+
s := scanner.Text()
29+
if len(s) >= 2 && strings.HasPrefix(s, "0x") {
30+
s = s[2:]
31+
}
32+
b, err := hex.DecodeString(s)
33+
if err != nil {
34+
panic(err) // rotten corpus
35+
}
36+
f.Add(b)
37+
}
38+
corpus.Close()
39+
if err := scanner.Err(); err != nil {
40+
panic(err) // rotten corpus
41+
}
42+
}
43+
// And do the fuzzing
44+
f.Fuzz(func(t *testing.T, data []byte) {
45+
var (
46+
jt = vm.NewPragueEOFInstructionSetForTesting()
47+
c vm.Container
48+
)
49+
cpy := common.CopyBytes(data)
50+
if err := c.UnmarshalBinary(data, true); err == nil {
51+
c.ValidateCode(&jt, true)
52+
if have := c.MarshalBinary(); !bytes.Equal(have, data) {
53+
t.Fatal("Unmarshal-> Marshal failure!")
54+
}
55+
}
56+
if err := c.UnmarshalBinary(data, false); err == nil {
57+
c.ValidateCode(&jt, false)
58+
if have := c.MarshalBinary(); !bytes.Equal(have, data) {
59+
t.Fatal("Unmarshal-> Marshal failure!")
60+
}
61+
}
62+
if !bytes.Equal(cpy, data) {
63+
panic("data modified during unmarshalling")
64+
}
65+
})
66+
}
67+
68+
func TestEofParseInitcode(t *testing.T) {
69+
testEofParse(t, true, "testdata/eof/results.initcode.txt")
70+
}
71+
72+
func TestEofParseRegular(t *testing.T) {
73+
testEofParse(t, false, "testdata/eof/results.regular.txt")
74+
}
75+
76+
func testEofParse(t *testing.T, isInitCode bool, wantFile string) {
77+
var wantFn func() string
78+
var wantLoc = 0
79+
{ // Configure the want-reader
80+
wants, err := os.Open(wantFile)
81+
if err != nil {
82+
t.Fatal(err)
83+
}
84+
scanner := bufio.NewScanner(wants)
85+
scanner.Buffer(make([]byte, 1024), 10*1024*1024)
86+
wantFn = func() string {
87+
if scanner.Scan() {
88+
wantLoc++
89+
return scanner.Text()
90+
}
91+
return "end of file reached"
92+
}
93+
}
94+
95+
for i := 0; ; i++ {
96+
fname := fmt.Sprintf("testdata/eof/eof_corpus_%d.txt", i)
97+
corpus, err := os.Open(fname)
98+
if err != nil {
99+
break
100+
}
101+
t.Logf("# Reading seed data from %v", fname)
102+
scanner := bufio.NewScanner(corpus)
103+
scanner.Buffer(make([]byte, 1024), 10*1024*1024)
104+
line := 1
105+
for scanner.Scan() {
106+
s := scanner.Text()
107+
if len(s) >= 2 && strings.HasPrefix(s, "0x") {
108+
s = s[2:]
109+
}
110+
b, err := hex.DecodeString(s)
111+
if err != nil {
112+
panic(err) // rotten corpus
113+
}
114+
have := "OK"
115+
if _, err := parse(b, isInitCode); err != nil {
116+
have = fmt.Sprintf("ERR: %v", err)
117+
}
118+
if false { // Change this to generate the want-output
119+
fmt.Printf("%v\n", have)
120+
} else {
121+
want := wantFn()
122+
if have != want {
123+
if len(want) > 100 {
124+
want = want[:100]
125+
}
126+
if len(b) > 100 {
127+
b = b[:100]
128+
}
129+
t.Errorf("%v:%d\n%v\ninput %x\nisInit: %v\nhave: %q\nwant: %q\n",
130+
fname, line, fmt.Sprintf("%v:%d", wantFile, wantLoc), b, isInitCode, have, want)
131+
}
132+
}
133+
line++
134+
}
135+
corpus.Close()
136+
}
137+
}
138+
139+
func BenchmarkEofParse(b *testing.B) {
140+
corpus, err := os.Open("testdata/eof/eof_benches.txt")
141+
if err != nil {
142+
b.Fatal(err)
143+
}
144+
defer corpus.Close()
145+
scanner := bufio.NewScanner(corpus)
146+
scanner.Buffer(make([]byte, 1024), 10*1024*1024)
147+
line := 1
148+
for scanner.Scan() {
149+
s := scanner.Text()
150+
if len(s) >= 2 && strings.HasPrefix(s, "0x") {
151+
s = s[2:]
152+
}
153+
data, err := hex.DecodeString(s)
154+
if err != nil {
155+
b.Fatal(err) // rotten corpus
156+
}
157+
b.Run(fmt.Sprintf("test-%d", line), func(b *testing.B) {
158+
b.ReportAllocs()
159+
b.SetBytes(int64(len(data)))
160+
for i := 0; i < b.N; i++ {
161+
_, _ = parse(data, false)
162+
}
163+
})
164+
line++
165+
}
166+
}

0 commit comments

Comments
 (0)