Skip to content

Commit 415f470

Browse files
hanabi1224hanabi1224
committed
roundtrip tests
1 parent 0ebf227 commit 415f470

2 files changed

Lines changed: 212 additions & 2 deletions

File tree

‎certstore/snapshot.go‎

Lines changed: 74 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,17 @@ import (
88
"fmt"
99
"io"
1010

11+
"github.com/filecoin-project/go-f3/certs"
1112
"github.com/filecoin-project/go-f3/gpbft"
1213
"github.com/filecoin-project/go-state-types/cbor"
14+
"github.com/ipfs/go-datastore"
1315
)
1416

1517
var ErrUnknownLatestCertificate = errors.New("latest certificate is not known")
1618

1719
// ExportLatestSnapshot exports an F3 snapshot that includes the finality certificate chain until the current `latestCertificate`.
1820
//
19-
// Checkout the format specification at <https://github.com/filecoin-project/FIPs/blob/master/FRCs/frc-0108.md>
21+
// Checkout the snapshot format specification at <https://github.com/filecoin-project/FIPs/blob/master/FRCs/frc-0108.md>
2022
func (cs *Store) ExportLatestSnapshot(ctx context.Context, writer io.Writer) error {
2123
if cs.latestCertificate == nil {
2224
return ErrUnknownLatestCertificate
@@ -26,7 +28,7 @@ func (cs *Store) ExportLatestSnapshot(ctx context.Context, writer io.Writer) err
2628

2729
// ExportSnapshot exports an F3 snapshot that includes the finality certificate chain from the `Store.firstInstance` to the specified `lastInstance`.
2830
//
29-
// Checkout the format specification at <https://github.com/filecoin-project/FIPs/blob/master/FRCs/frc-0108.md>
31+
// Checkout the snapshot format specification at <https://github.com/filecoin-project/FIPs/blob/master/FRCs/frc-0108.md>
3032
func (cs *Store) ExportSnapshot(ctx context.Context, latestInstance uint64, writer io.Writer) error {
3133
initialPowerTable, err := cs.GetPowerTable(ctx, cs.firstInstance)
3234
if err != nil {
@@ -49,6 +51,60 @@ func (cs *Store) ExportSnapshot(ctx context.Context, latestInstance uint64, writ
4951
return nil
5052
}
5153

54+
type SnapshotReader interface {
55+
io.Reader
56+
io.ByteReader
57+
}
58+
59+
// ImportSnapshotToDatastore imports an F3 snapshot into the specified Datastore
60+
//
61+
// Checkout the snapshot format specification at <https://github.com/filecoin-project/FIPs/blob/master/FRCs/frc-0108.md>
62+
func ImportSnapshotToDatastore(ctx context.Context, snapshot SnapshotReader, ds datastore.Datastore) error {
63+
return importSnapshotToDatastoreWithTestingPowerTableFrequency(ctx, snapshot, ds, 0)
64+
}
65+
66+
func importSnapshotToDatastoreWithTestingPowerTableFrequency(ctx context.Context, snapshot SnapshotReader, ds datastore.Datastore, testingPowerTableFrequency uint64) error {
67+
headerBytes, err := readSnapshotBlockBytes(snapshot)
68+
if err != nil {
69+
return err
70+
}
71+
var header SnapshotHeader
72+
err = header.UnmarshalCBOR(bytes.NewReader(headerBytes))
73+
if err != nil {
74+
return fmt.Errorf("failed to decode snapshot header: %w", err)
75+
}
76+
cs, err := OpenOrCreateStore(ctx, ds, header.FirstInstance, header.InitialPowerTable)
77+
if testingPowerTableFrequency > 0 {
78+
cs.powerTableFrequency = testingPowerTableFrequency
79+
}
80+
if err != nil {
81+
return err
82+
}
83+
pt := header.InitialPowerTable
84+
for {
85+
certBytes, err := readSnapshotBlockBytes(snapshot)
86+
if err == io.EOF {
87+
break
88+
} else if err != nil {
89+
return fmt.Errorf("failed to decode finality certificate: %w", err)
90+
}
91+
var cert certs.FinalityCertificate
92+
cert.UnmarshalCBOR(bytes.NewReader(certBytes))
93+
if err = cs.Put(ctx, &cert); err != nil {
94+
return err
95+
}
96+
if pt, err = certs.ApplyPowerTableDiffs(pt, cert.PowerTableDelta); err != nil {
97+
return err
98+
}
99+
if (cert.GPBFTInstance+1)%cs.powerTableFrequency == 0 {
100+
if err := cs.putPowerTable(ctx, cert.GPBFTInstance+1, pt); err != nil {
101+
return err
102+
}
103+
}
104+
}
105+
return nil
106+
}
107+
52108
type SnapshotHeader struct {
53109
Version uint64
54110
FirstInstance uint64
@@ -83,3 +139,19 @@ func writeSnapshotBlockBytes(writer io.Writer, buffer *bytes.Buffer) (int64, err
83139
}
84140
return len1 + len2, nil
85141
}
142+
143+
func readSnapshotBlockBytes(reader SnapshotReader) ([]byte, error) {
144+
n1, err := binary.ReadUvarint(reader)
145+
if err != nil {
146+
return nil, err
147+
}
148+
buf := make([]byte, n1)
149+
n2, err := reader.Read(buf)
150+
if err != nil {
151+
return nil, err
152+
}
153+
if n2 != int(n1) {
154+
return nil, fmt.Errorf("incomplete block, %d bytes expected, %d bytes got", n1, n2)
155+
}
156+
return buf, nil
157+
}

‎certstore/snapshot_test.go‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,139 @@
11
package certstore
2+
3+
import (
4+
"bytes"
5+
"context"
6+
"math/rand"
7+
"testing"
8+
"time"
9+
10+
"github.com/filecoin-project/go-f3/certchain"
11+
"github.com/filecoin-project/go-f3/gpbft"
12+
"github.com/filecoin-project/go-f3/internal/clock"
13+
"github.com/filecoin-project/go-f3/internal/consensus"
14+
"github.com/filecoin-project/go-f3/manifest"
15+
"github.com/filecoin-project/go-f3/sim/signing"
16+
"github.com/ipfs/go-datastore"
17+
"github.com/stretchr/testify/require"
18+
"go.uber.org/zap/buffer"
19+
)
20+
21+
func Test_SnapshotExportImportRoundTrip(t *testing.T) {
22+
const (
23+
seed = 1427
24+
certChainLength = 150
25+
testingPowerTableFreqency = uint64(23)
26+
)
27+
28+
ctx, clk := clock.WithMockClock(context.Background())
29+
m := manifest.LocalDevnetManifest()
30+
m.InitialInstance = 100
31+
signVerifier := signing.NewFakeBackend()
32+
rng := rand.New(rand.NewSource(seed * 23))
33+
generatePublicKey := func(id gpbft.ActorID) gpbft.PubKey {
34+
//TODO: add the ability to evolve public key across instances. Fake signing
35+
// backed does not support this.
36+
37+
// Use allow instead of GenerateKey for a reproducible key generation.
38+
return signVerifier.Allow(int(id))
39+
}
40+
initialPowerTable := generatePowerTable(t, rng, generatePublicKey, nil)
41+
42+
ec := consensus.NewFakeEC(
43+
consensus.WithClock(clk),
44+
consensus.WithSeed(seed*13),
45+
consensus.WithBootstrapEpoch(m.BootstrapEpoch),
46+
consensus.WithECPeriod(m.EC.Period),
47+
consensus.WithInitialPowerTable(initialPowerTable),
48+
consensus.WithEvolvingPowerTable(
49+
func(epoch int64, entries gpbft.PowerEntries) gpbft.PowerEntries {
50+
if epoch == m.BootstrapEpoch-m.EC.Finality {
51+
return initialPowerTable
52+
}
53+
rng := rand.New(rand.NewSource(epoch * seed))
54+
next := generatePowerTable(t, rng, generatePublicKey, entries)
55+
return next
56+
},
57+
),
58+
)
59+
60+
subject, err := certchain.New(
61+
certchain.WithSeed(seed),
62+
certchain.WithSignVerifier(signVerifier),
63+
certchain.WithManifest(m),
64+
certchain.WithEC(ec),
65+
)
66+
require.NoError(t, err)
67+
68+
// The mock clock is buried into context passed to fake EC. The face EC will
69+
// refuse to generate a chain if the clock is not advanced. Advance it
70+
// sufficiently to never be bothered by it again.
71+
//
72+
// The fake EC and its relationship with clock needs to be reworked: Clock should
73+
// ideally be passed as an option, and its absence should mean "advance the clock
74+
// as needed". Because, we do not always care about controlling the progress of
75+
// chain generated by fake EC.
76+
clk.Add(200 * time.Hour)
77+
78+
generatedChain, err := subject.Generate(ctx, certChainLength)
79+
require.NoError(t, err)
80+
81+
ds1 := datastore.NewMapDatastore()
82+
cs, err := OpenOrCreateStore(ctx, ds1, generatedChain[0].GPBFTInstance, initialPowerTable)
83+
cs.powerTableFrequency = testingPowerTableFreqency
84+
require.NoError(t, err)
85+
86+
for _, cert := range generatedChain {
87+
cs.Put(ctx, cert)
88+
}
89+
90+
snapshot := buffer.Buffer{}
91+
err = cs.ExportLatestSnapshot(ctx, &snapshot)
92+
require.NoError(t, err)
93+
94+
ds2 := datastore.NewMapDatastore()
95+
err = importSnapshotToDatastoreWithTestingPowerTableFrequency(ctx, bytes.NewReader(snapshot.Bytes()), ds2, testingPowerTableFreqency)
96+
require.NoError(t, err)
97+
98+
require.Equal(t, ds1, ds2)
99+
100+
ds3 := datastore.NewMapDatastore()
101+
err = ImportSnapshotToDatastore(ctx, bytes.NewReader(snapshot.Bytes()), ds3)
102+
require.NoError(t, err)
103+
104+
require.NotEqual(t, ds1, ds3)
105+
}
106+
107+
func generatePowerTable(t *testing.T, rng *rand.Rand, generatePublicKey func(id gpbft.ActorID) gpbft.PubKey, previousEntries gpbft.PowerEntries) gpbft.PowerEntries {
108+
const (
109+
maxEntries = 100
110+
maxPower = 1 << 20
111+
minPower = 0 // Pick a sufficiently low power to facilitate entries with zero scaled power.
112+
actorIDOffset = 1413
113+
powerChangeProbability = 0.2
114+
)
115+
116+
size := rng.Intn(maxEntries)
117+
entries := make(gpbft.PowerEntries, 0, size)
118+
for i := range size {
119+
var entry gpbft.PowerEntry
120+
if i < previousEntries.Len() {
121+
entry = previousEntries[i]
122+
changedPower := rng.Float64() > powerChangeProbability
123+
if changedPower {
124+
entry.Power = gpbft.NewStoragePower(int64(rng.Intn(maxPower) + minPower))
125+
}
126+
} else {
127+
id := gpbft.ActorID(uint64(actorIDOffset + i))
128+
entry = gpbft.PowerEntry{
129+
ID: id,
130+
Power: gpbft.NewStoragePower(int64(rng.Intn(maxPower) + minPower)),
131+
PubKey: generatePublicKey(id),
132+
}
133+
}
134+
entries = append(entries, entry)
135+
}
136+
next := gpbft.NewPowerTable()
137+
require.NoError(t, next.Add(entries...))
138+
return next.Entries
139+
}

0 commit comments

Comments
 (0)