Skip to content

Commit ebeda3c

Browse files
miiu96miiu96
committed
fix integration and unit test
1 parent 3d1707e commit ebeda3c

File tree

13 files changed

+221
-33
lines changed

13 files changed

+221
-33
lines changed

‎data/scDeploy.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,9 @@ type ScDeployInfo struct {
1414

1515
// Upgrade is the DTO that holds information about a smart contract upgrade
1616
type Upgrade struct {
17-
TxHash string `json:"upgradeTxHash"`
18-
Upgrader string `json:"upgrader"`
19-
Timestamp uint64 `json:"timestamp"`
20-
CodeHash []byte `json:"codeHash"`
17+
TxHash string `json:"upgradeTxHash"`
18+
Upgrader string `json:"upgrader"`
19+
Timestamp uint64 `json:"timestamp"`
20+
TimestampMs uint64 `json:"timestampMs"`
21+
CodeHash []byte `json:"codeHash"`
2122
}

‎integrationtests/testdata/mappings/esdts.json‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
},
3131
"timestampMs": {
3232
"type": "date",
33-
"format": "epoch_millisecond",
33+
"format": "epoch_millis",
3434
"index": false
3535
}
3636
}
@@ -117,12 +117,16 @@
117117
},
118118
"timestampMs": {
119119
"type": "date",
120-
"format": "epoch_millisecond"
120+
"format": "epoch_millis"
121121
},
122122
"changedToDynamicTimestamp": {
123123
"type": "date",
124124
"format": "epoch_second"
125125
},
126+
"changedToDynamicTimestampMs": {
127+
"type": "date",
128+
"format": "epoch_millis"
129+
},
126130
"token": {
127131
"type": "text"
128132
},

‎process/dataindexer/errors.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,3 +88,6 @@ var ErrNilOperationsHandler = errors.New("nil operations handler")
8888

8989
// ErrNilBlockContainerHandler signals that a nil block container handler has been provided
9090
var ErrNilBlockContainerHandler = errors.New("nil bock container handler")
91+
92+
// ErrNilMappingsHandler signals that a nil mappings handler has been provided
93+
var ErrNilMappingsHandler = errors.New("nil mappings handler")

‎process/elasticproc/check.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,9 @@ func checkArguments(arguments *ArgElasticProcessor) error {
3939
if check.IfNilReflect(arguments.OperationsProc) {
4040
return elasticIndexer.ErrNilOperationsHandler
4141
}
42+
if check.IfNilReflect(arguments.MappingsHandler) {
43+
return elasticIndexer.ErrNilMappingsHandler
44+
}
4245

4346
return nil
4447
}

‎process/elasticproc/elasticProcessor.go‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,6 @@ type ArgElasticProcessor struct {
4343
BulkRequestMaxSize int
4444
UseKibana bool
4545
ImportDB bool
46-
IndexTemplates map[string]*bytes.Buffer
47-
IndexPolicies map[string]*bytes.Buffer
48-
ExtraMappings []templates.ExtraMapping
4946
EnabledIndexes map[string]struct{}
5047
TransactionsProc DBTransactionsHandler
5148
AccountsProc DBAccountHandler
@@ -56,6 +53,7 @@ type ArgElasticProcessor struct {
5653
DBClient DatabaseClientHandler
5754
LogsAndEventsProc DBLogsAndEventsHandler
5855
OperationsProc OperationsHandler
56+
MappingsHandler TemplatesAndPoliciesHandler
5957
Version string
6058
}
6159

@@ -73,6 +71,7 @@ type elasticProcessor struct {
7371
validatorsProc DBValidatorsHandler
7472
logsAndEventsProc DBLogsAndEventsHandler
7573
operationsProc OperationsHandler
74+
mappingsHandler TemplatesAndPoliciesHandler
7675
}
7776

7877
// NewElasticProcessor handles Elasticsearch operations such as initialization, adding, modifying or removing data
@@ -94,9 +93,10 @@ func NewElasticProcessor(arguments *ArgElasticProcessor) (*elasticProcessor, err
9493
logsAndEventsProc: arguments.LogsAndEventsProc,
9594
operationsProc: arguments.OperationsProc,
9695
bulkRequestMaxSize: arguments.BulkRequestMaxSize,
96+
mappingsHandler: arguments.MappingsHandler,
9797
}
9898

99-
err = ei.init(arguments.IndexTemplates, arguments.IndexPolicies, arguments.ExtraMappings)
99+
err = ei.init()
100100
if err != nil {
101101
return nil, err
102102
}
@@ -107,8 +107,13 @@ func NewElasticProcessor(arguments *ArgElasticProcessor) (*elasticProcessor, err
107107
}
108108

109109
// TODO move all the index create part in a new component
110-
func (ei *elasticProcessor) init(indexTemplates, _ map[string]*bytes.Buffer, extraMappings []templates.ExtraMapping) error {
111-
err := ei.createOpenDistroTemplates(indexTemplates)
110+
func (ei *elasticProcessor) init() error {
111+
indexTemplates, _, err := ei.mappingsHandler.GetElasticTemplatesAndPolicies()
112+
if err != nil {
113+
return err
114+
}
115+
116+
err = ei.createOpenDistroTemplates(indexTemplates)
112117
if err != nil {
113118
return err
114119
}
@@ -128,6 +133,11 @@ func (ei *elasticProcessor) init(indexTemplates, _ map[string]*bytes.Buffer, ext
128133
return err
129134
}
130135

136+
extraMappings, err := ei.mappingsHandler.GetTimestampMsMappings()
137+
if err != nil {
138+
return err
139+
}
140+
131141
return ei.addExtraMappings(extraMappings)
132142
}
133143

@@ -791,7 +801,7 @@ func (ei *elasticProcessor) saveAccountsESDTHistory(timestamp uint64, accountsIn
791801
return nil
792802
}
793803

794-
accountsMap := ei.accountsProc.PrepareAccountsHistory(timestamp, accountsInfoMap, shardID, timestamp)
804+
accountsMap := ei.accountsProc.PrepareAccountsHistory(timestamp, accountsInfoMap, shardID, timestampMs)
795805

796806
return ei.serializeAndIndexAccountsHistory(accountsMap, elasticIndexer.AccountsESDTHistoryIndex, buffSlice)
797807
}

‎process/elasticproc/elasticProcessor_test.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/operations"
2424
"github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/statistics"
2525
"github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tags"
26+
"github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/templatesAndPolicies"
2627
"github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions"
2728
"github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/validators"
2829
"github.com/stretchr/testify/require"
@@ -87,6 +88,7 @@ func createMockElasticProcessorArgs() *ArgElasticProcessor {
8788
BlockProc: bp,
8889
LogsAndEventsProc: lp,
8990
OperationsProc: op,
91+
MappingsHandler: templatesAndPolicies.NewTemplatesAndPolicyReader(),
9092
}
9193
}
9294

@@ -200,6 +202,15 @@ func TestNewElasticProcessor(t *testing.T) {
200202
},
201203
exErr: dataindexer.ErrNilTransactionsHandler,
202204
},
205+
{
206+
name: "NilMappingsHandler",
207+
args: func() *ArgElasticProcessor {
208+
arguments := createMockElasticProcessorArgs()
209+
arguments.MappingsHandler = nil
210+
return arguments
211+
},
212+
exErr: dataindexer.ErrNilMappingsHandler,
213+
},
203214
{
204215
name: "InitError",
205216
args: func() *ArgElasticProcessor {

‎process/elasticproc/factory/elasticProcessorFactory.go‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -36,14 +36,6 @@ type ArgElasticProcessorFactory struct {
3636
// CreateElasticProcessor will create a new instance of ElasticProcessor
3737
func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.ElasticProcessor, error) {
3838
templatesAndPoliciesReader := templatesAndPolicies.NewTemplatesAndPolicyReader()
39-
indexTemplates, indexPolicies, err := templatesAndPoliciesReader.GetElasticTemplatesAndPolicies()
40-
if err != nil {
41-
return nil, err
42-
}
43-
extraMappings, err := templatesAndPoliciesReader.GetExtraMappings()
44-
if err != nil {
45-
return nil, err
46-
}
4739

4840
enabledIndexesMap := make(map[string]struct{})
4941
for _, index := range arguments.EnabledIndexes {
@@ -121,12 +113,10 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.E
121113
DBClient: arguments.DBClient,
122114
EnabledIndexes: enabledIndexesMap,
123115
UseKibana: arguments.UseKibana,
124-
IndexTemplates: indexTemplates,
125-
IndexPolicies: indexPolicies,
126-
ExtraMappings: extraMappings,
127116
OperationsProc: operationsProc,
128117
ImportDB: arguments.ImportDB,
129118
Version: arguments.Version,
119+
MappingsHandler: templatesAndPoliciesReader,
130120
}
131121

132122
return elasticproc.NewElasticProcessor(args)

‎process/elasticproc/interface.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package elasticproc
33
import (
44
"bytes"
55
"context"
6+
"github.com/multiversx/mx-chain-es-indexer-go/templates"
67

78
coreData "github.com/multiversx/mx-chain-core-go/data"
89
"github.com/multiversx/mx-chain-core-go/data/alteredAccount"
@@ -122,3 +123,10 @@ type OperationsHandler interface {
122123
ProcessTransactionsAndSCRs(txs []*data.Transaction, scrs []*data.ScResult, isImportDB bool, shardID uint32) ([]*data.Transaction, []*data.ScResult)
123124
SerializeSCRs(scrs []*data.ScResult, buffSlice *data.BufferSlice, index string, shardID uint32) error
124125
}
126+
127+
// TemplatesAndPoliciesHandler defines the actions that a templates and policies handler should do
128+
type TemplatesAndPoliciesHandler interface {
129+
GetElasticTemplatesAndPolicies() (map[string]*bytes.Buffer, map[string]*bytes.Buffer, error)
130+
GetExtraMappings() ([]templates.ExtraMapping, error)
131+
GetTimestampMsMappings() ([]templates.ExtraMapping, error)
132+
}

‎process/elasticproc/logsevents/serialize.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -154,10 +154,11 @@ func serializeDeploy(deployInfo *data.ScDeployInfo) ([]byte, error) {
154154
}
155155

156156
upgradeData := &data.Upgrade{
157-
TxHash: deployInfo.TxHash,
158-
Upgrader: deployInfo.Creator,
159-
Timestamp: deployInfo.Timestamp,
160-
CodeHash: deployInfo.CodeHash,
157+
TxHash: deployInfo.TxHash,
158+
Upgrader: deployInfo.Creator,
159+
Timestamp: deployInfo.Timestamp,
160+
CodeHash: deployInfo.CodeHash,
161+
TimestampMs: deployInfo.TimestampMs,
161162
}
162163
upgradeSerialized, errPrepareU := json.Marshal(upgradeData)
163164
if errPrepareU != nil {

‎process/elasticproc/logsevents/serialize_test.go‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,10 @@ func TestLogsAndEventsProcessor_SerializeSCDeploys(t *testing.T) {
4646

4747
scDeploys := map[string]*data.ScDeployInfo{
4848
"scAddr": {
49-
Creator: "creator",
50-
Timestamp: 123,
51-
TxHash: "hash",
49+
Creator: "creator",
50+
Timestamp: 123,
51+
TxHash: "hash",
52+
TimestampMs: 123000,
5253
},
5354
}
5455

@@ -57,7 +58,7 @@ func TestLogsAndEventsProcessor_SerializeSCDeploys(t *testing.T) {
5758
require.Nil(t, err)
5859

5960
expectedRes := `{ "update" : { "_index":"scdeploys", "_id" : "scAddr" } }
60-
{"script": {"source": "if (!ctx._source.containsKey('upgrades')) {ctx._source.upgrades = [params.elem];} else {ctx._source.upgrades.add(params.elem);}","lang": "painless","params": {"elem": {"upgradeTxHash":"hash","upgrader":"creator","timestamp":123,"codeHash":null}}},"upsert": {"deployTxHash":"hash","deployer":"creator","currentOwner":"","initialCodeHash":null,"timestamp":123,"upgrades":[],"owners":[]}}
61+
{"script": {"source": "if (!ctx._source.containsKey('upgrades')) {ctx._source.upgrades = [params.elem];} else {ctx._source.upgrades.add(params.elem);}","lang": "painless","params": {"elem": {"upgradeTxHash":"hash","upgrader":"creator","timestamp":123,"timestampMs":123000,"codeHash":null}}},"upsert": {"deployTxHash":"hash","deployer":"creator","currentOwner":"","initialCodeHash":null,"timestamp":123,"timestampMs":123000,"upgrades":[],"owners":[]}}
6162
`
6263
require.Equal(t, expectedRes, buffSlice.Buffers()[0].String())
6364
}
@@ -134,7 +135,7 @@ func TestLogsAndEventsProcessor_SerializeDelegators(t *testing.T) {
134135
require.Nil(t, err)
135136

136137
expectedRes := `{ "update" : { "_index":"delegators", "_id" : "/GeogJjDjtpxnceK9t6+BVBYWuuJHbjmsWK0/1BlH9c=" } }
137-
{"scripted_upsert": true, "script": {"source": "if ('create' == ctx.op) {ctx._source = params.delegator} else {ctx._source.activeStake = params.delegator.activeStake;ctx._source.activeStakeNum = params.delegator.activeStakeNum;ctx._source.timestamp = params.delegator.timestamp;}","lang": "painless","params": { "delegator": {"address":"addr1","contract":"contract1","timestamp":0,"activeStake":"100000000000000","activeStakeNum":0.1} }},"upsert": {}}
138+
{"scripted_upsert": true, "script": {"source": "if ('create' == ctx.op) {ctx._source = params.delegator} else {ctx._source.activeStake = params.delegator.activeStake;ctx._source.activeStakeNum = params.delegator.activeStakeNum;ctx._source.timestamp = params.delegator.timestamp;ctx._source.timestampMs = params.delegator.timestampMs;}","lang": "painless","params": { "delegator": {"address":"addr1","contract":"contract1","timestamp":0,"activeStake":"100000000000000","activeStakeNum":0.1} }},"upsert": {}}
138139
`
139140
require.Equal(t, expectedRes, buffSlice.Buffers()[0].String())
140141
}

0 commit comments

Comments
 (0)