Skip to content

Commit fe5ace8

Browse files
peterbroadhurstpeterbroadhurst
authored
Merge pull request #97 from hyperledger/migration-timestamps
Preserve timestamps on migration from LevelDB to Postgres
2 parents bdeb537 + 6f22a7c commit fe5ace8

File tree

9 files changed

+74
-16
lines changed

9 files changed

+74
-16
lines changed

‎internal/persistence/dbmigration/leveldb2postgres.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ func MigrateLevelDBToPostgres(ctx context.Context) (err error) {
3737
return i18n.NewError(ctx, tmmsgs.MsgPersistenceInitFail, "leveldb", err)
3838
}
3939
defer m.source.Close(ctx)
40-
if m.target, err = postgres.NewPostgresPersistence(ctx, tmconfig.PostgresSection, nonceStateTimeout); err != nil {
40+
if m.target, err = postgres.NewPostgresPersistence(ctx, tmconfig.PostgresSection, nonceStateTimeout, postgres.ForMigration); err != nil {
4141
return i18n.NewError(ctx, tmmsgs.MsgPersistenceInitFail, "postgres", err)
4242
}
4343
defer m.target.Close(ctx)

‎internal/persistence/dbmigration/migration.go

+25
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,12 @@ func (m *dbMigration) migrateEventStream(ctx context.Context, es *apitypes.Event
8181
return err
8282
}
8383
if existingES == nil {
84+
if es.Created == nil {
85+
es.Created = fftypes.Now()
86+
}
87+
if es.Updated == nil {
88+
es.Updated = es.Created
89+
}
8490
log.L(ctx).Infof("Writing event stream %s to target", es.ID)
8591
if err := m.target.WriteStream(ctx, es); err != nil {
8692
return err
@@ -96,6 +102,13 @@ func (m *dbMigration) migrateEventStream(ctx context.Context, es *apitypes.Event
96102
return err
97103
}
98104
if cp != nil && existingCP == nil {
105+
// LevelDB didn't have timestamps in checkpoints
106+
if cp.FirstCheckpoint == nil {
107+
cp.FirstCheckpoint = fftypes.Now()
108+
}
109+
if cp.Time == nil {
110+
cp.Time = cp.FirstCheckpoint
111+
}
99112
log.L(ctx).Infof("Writing checkpoint %s to target", cp.StreamID)
100113
if err := m.target.WriteCheckpoint(ctx, cp); err != nil {
101114
return err
@@ -138,6 +151,12 @@ func (m *dbMigration) migrateListener(ctx context.Context, l *apitypes.Listener)
138151
}
139152
if existingL == nil {
140153
log.L(ctx).Infof("Writing listener %s to target", l.ID)
154+
if l.Created == nil {
155+
l.Created = fftypes.Now()
156+
}
157+
if l.Updated == nil {
158+
l.Updated = l.Created
159+
}
141160
if err := m.target.WriteListener(ctx, l); err != nil {
142161
return err
143162
}
@@ -185,6 +204,12 @@ func (m *dbMigration) migrateTransaction(ctx context.Context, mtx *apitypes.Mana
185204
}
186205
if existingTX == nil {
187206
log.L(ctx).Infof("Writing transaction %s to target", tx.ID)
207+
if tx.Created == nil {
208+
tx.Created = fftypes.Now()
209+
}
210+
if tx.Updated == nil {
211+
tx.Updated = tx.Created
212+
}
188213
if err := m.target.InsertTransactionPreAssignedNonce(ctx, tx.ManagedTX); err != nil {
189214
return err
190215
}

‎internal/persistence/postgres/checkpoints.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ import (
2424
"github.com/hyperledger/firefly-transaction-manager/pkg/apitypes"
2525
)
2626

27-
func (p *sqlPersistence) newCheckpointCollection() *dbsql.CrudBase[*apitypes.EventStreamCheckpoint] {
27+
func (p *sqlPersistence) newCheckpointCollection(forMigration bool) *dbsql.CrudBase[*apitypes.EventStreamCheckpoint] {
2828
collection := &dbsql.CrudBase[*apitypes.EventStreamCheckpoint]{
2929
DB: p.db,
3030
Table: "checkpoints",
@@ -39,6 +39,7 @@ func (p *sqlPersistence) newCheckpointCollection() *dbsql.CrudBase[*apitypes.Eve
3939
"streamid": "id",
4040
},
4141
PatchDisabled: true,
42+
TimesDisabled: forMigration,
4243
NilValue: func() *apitypes.EventStreamCheckpoint { return nil },
4344
NewInstance: func() *apitypes.EventStreamCheckpoint { return &apitypes.EventStreamCheckpoint{} },
4445
GetFieldPtr: func(inst *apitypes.EventStreamCheckpoint, col string) interface{} {

‎internal/persistence/postgres/eventstreams.go

+4-3
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ import (
2626
"github.com/hyperledger/firefly-transaction-manager/pkg/apitypes"
2727
)
2828

29-
func (p *sqlPersistence) newEventStreamsCollection() *dbsql.CrudBase[*apitypes.EventStream] {
29+
func (p *sqlPersistence) newEventStreamsCollection(forMigration bool) *dbsql.CrudBase[*apitypes.EventStream] {
3030
collection := &dbsql.CrudBase[*apitypes.EventStream]{
3131
DB: p.db,
3232
Table: "eventstreams",
@@ -56,8 +56,9 @@ func (p *sqlPersistence) newEventStreamsCollection() *dbsql.CrudBase[*apitypes.E
5656
"webhook": "webhook_config",
5757
"websocket": "webhsocket_config",
5858
},
59-
NilValue: func() *apitypes.EventStream { return nil },
60-
NewInstance: func() *apitypes.EventStream { return &apitypes.EventStream{} },
59+
TimesDisabled: forMigration,
60+
NilValue: func() *apitypes.EventStream { return nil },
61+
NewInstance: func() *apitypes.EventStream { return &apitypes.EventStream{} },
6162
GetFieldPtr: func(inst *apitypes.EventStream, col string) interface{} {
6263
switch col {
6364
case dbsql.ColumnID:

‎internal/persistence/postgres/listeners.go

+4-3
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ import (
2626
"github.com/hyperledger/firefly-transaction-manager/pkg/apitypes"
2727
)
2828

29-
func (p *sqlPersistence) newListenersCollection() *dbsql.CrudBase[*apitypes.Listener] {
29+
func (p *sqlPersistence) newListenersCollection(forMigration bool) *dbsql.CrudBase[*apitypes.Listener] {
3030
collection := &dbsql.CrudBase[*apitypes.Listener]{
3131
DB: p.db,
3232
Table: "listeners",
@@ -46,8 +46,9 @@ func (p *sqlPersistence) newListenersCollection() *dbsql.CrudBase[*apitypes.List
4646
"streamid": "stream_id",
4747
"from_block": "fromblock",
4848
},
49-
NilValue: func() *apitypes.Listener { return nil },
50-
NewInstance: func() *apitypes.Listener { return &apitypes.Listener{} },
49+
TimesDisabled: forMigration,
50+
NilValue: func() *apitypes.Listener { return nil },
51+
NewInstance: func() *apitypes.Listener { return &apitypes.Listener{} },
5152
GetFieldPtr: func(inst *apitypes.Listener, col string) interface{} {
5253
switch col {
5354
case dbsql.ColumnID:

‎internal/persistence/postgres/postgres.go

+6-2
Original file line numberDiff line numberDiff line change
@@ -40,13 +40,17 @@ type Postgres struct {
4040
dbsql.Database
4141
}
4242

43+
type CodeUsageOptions int
44+
45+
const ForMigration CodeUsageOptions = iota
46+
4347
var psql *Postgres
4448

45-
func NewPostgresPersistence(bgCtx context.Context, conf config.Section, nonceStateTimeout time.Duration) (persistence.Persistence, error) {
49+
func NewPostgresPersistence(bgCtx context.Context, conf config.Section, nonceStateTimeout time.Duration, codeOptions ...CodeUsageOptions) (persistence.Persistence, error) {
4650
if err := psql.Database.Init(bgCtx, psql, conf); err != nil {
4751
return nil, err
4852
}
49-
return newSQLPersistence(bgCtx, &psql.Database, conf, nonceStateTimeout)
53+
return newSQLPersistence(bgCtx, &psql.Database, conf, nonceStateTimeout, codeOptions...)
5054
}
5155

5256
func (psql *Postgres) Name() string {

‎internal/persistence/postgres/sqlpersistence.go

+12-5
Original file line numberDiff line numberDiff line change
@@ -68,18 +68,25 @@ func InitConfig(conf config.Section) {
6868
conf.AddKnownKey(ConfigTXWriterBatchSize, 100)
6969
}
7070

71-
func newSQLPersistence(bgCtx context.Context, db *dbsql.Database, conf config.Section, nonceStateTimeout time.Duration) (p *sqlPersistence, err error) {
71+
func newSQLPersistence(bgCtx context.Context, db *dbsql.Database, conf config.Section, nonceStateTimeout time.Duration, codeOptions ...CodeUsageOptions) (p *sqlPersistence, err error) {
7272
p = &sqlPersistence{
7373
db: db,
7474
}
7575

76-
p.transactions = p.newTransactionCollection()
77-
p.checkpoints = p.newCheckpointCollection()
76+
forMigration := false
77+
for _, co := range codeOptions {
78+
if co == ForMigration {
79+
forMigration = true
80+
}
81+
}
82+
83+
p.eventStreams = p.newEventStreamsCollection(forMigration)
84+
p.checkpoints = p.newCheckpointCollection(forMigration)
85+
p.listeners = p.newListenersCollection(forMigration)
86+
p.transactions = p.newTransactionCollection(forMigration)
7887
p.confirmations = p.newConfirmationsCollection()
7988
p.receipts = p.newReceiptsCollection()
8089
p.txHistory = p.newTXHistoryCollection()
81-
p.eventStreams = p.newEventStreamsCollection()
82-
p.listeners = p.newListenersCollection()
8390

8491
p.historySummaryLimit = conf.GetInt(ConfigTXWriterHistorySummaryLimit)
8592
p.nonceStateTimeout = nonceStateTimeout

‎internal/persistence/postgres/sqlpersistence_test.go

+18
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,24 @@ func newMockSQLPersistence(t *testing.T, init ...func(dbconf config.Section)) (c
5252

5353
}
5454

55+
func TestNewSQLPersistenceForMigration(t *testing.T) {
56+
57+
db, _ := dbsql.NewMockProvider().UTInit()
58+
59+
config.RootConfigReset()
60+
dbconf := config.RootSection("utdb")
61+
InitConfig(dbconf)
62+
63+
p, err := newSQLPersistence(context.Background(), &db.Database, dbconf, 1*time.Hour, ForMigration)
64+
assert.NoError(t, err)
65+
defer p.Close(context.Background())
66+
67+
assert.True(t, p.transactions.TimesDisabled)
68+
assert.True(t, p.eventStreams.TimesDisabled)
69+
assert.True(t, p.listeners.TimesDisabled)
70+
71+
}
72+
5573
func TestNewSQLPersistenceTXWriterFail(t *testing.T) {
5674

5775
db, _ := dbsql.NewMockProvider().UTInit()

‎internal/persistence/postgres/transactions.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ import (
2828
"github.com/hyperledger/firefly-transaction-manager/pkg/ffcapi"
2929
)
3030

31-
func (p *sqlPersistence) newTransactionCollection() *dbsql.CrudBase[*apitypes.ManagedTX] {
31+
func (p *sqlPersistence) newTransactionCollection(forMigration bool) *dbsql.CrudBase[*apitypes.ManagedTX] {
3232
collection := &dbsql.CrudBase[*apitypes.ManagedTX]{
3333
DB: p.db,
3434
Table: "transactions",
@@ -68,6 +68,7 @@ func (p *sqlPersistence) newTransactionCollection() *dbsql.CrudBase[*apitypes.Ma
6868
"errormessage": "error_message",
6969
},
7070
PatchDisabled: true,
71+
TimesDisabled: forMigration,
7172
NilValue: func() *apitypes.ManagedTX { return nil },
7273
NewInstance: func() *apitypes.ManagedTX { return &apitypes.ManagedTX{} },
7374
GetFieldPtr: func(inst *apitypes.ManagedTX, col string) interface{} {

0 commit comments

Comments
 (0)