From 6915b473e7876753dfd7eb61e6ea5271f37fdeff Mon Sep 17 00:00:00 2001 From: Bogdan Rosianu Date: Thu, 28 Sep 2023 11:26:52 +0300 Subject: [PATCH 01/50] add sovereign header type --- go.mod | 2 +- go.sum | 4 ++-- process/factory/indexerFactory.go | 4 ++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9fbf9d92..a279b671 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.1 github.com/multiversx/mx-chain-communication-go v1.0.7 - github.com/multiversx/mx-chain-core-go v1.2.16 + github.com/multiversx/mx-chain-core-go v1.2.17-0.20230928082232-5f47af6c0981 github.com/multiversx/mx-chain-logger-go v1.0.13 github.com/multiversx/mx-chain-vm-common-go v1.5.2 github.com/prometheus/client_model v0.4.0 diff --git a/go.sum b/go.sum index 4f913b00..52a0c2ac 100644 --- a/go.sum +++ b/go.sum @@ -249,8 +249,8 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/mx-chain-communication-go v1.0.7 h1:7qeDBcqmGYYhSqFcpwv0qKkR3ahOfIMbRwXYEnOt/do= github.com/multiversx/mx-chain-communication-go v1.0.7/go.mod h1:+oaUowpq+SqrEmAsMPGwhz44g7L81loWb6AiNQU9Ms4= -github.com/multiversx/mx-chain-core-go v1.2.16 h1:m0hUNmZQjGJxKDLQOHoM9jSaeDfVTbyd+mqiS8+NckE= -github.com/multiversx/mx-chain-core-go v1.2.16/go.mod h1:BILOGHUOIG5dNNX8cgkzCNfDaVtoYrJRYcPnpxRMH84= +github.com/multiversx/mx-chain-core-go v1.2.17-0.20230928082232-5f47af6c0981 h1:5BdwV8HMLz3pSeabqwQ90ggOww9V4RloAfBgm2dtQXQ= +github.com/multiversx/mx-chain-core-go v1.2.17-0.20230928082232-5f47af6c0981/go.mod h1:BILOGHUOIG5dNNX8cgkzCNfDaVtoYrJRYcPnpxRMH84= github.com/multiversx/mx-chain-crypto-go v1.2.8 h1:wOgVlUaO5X4L8iEbFjcQcL8SZvv6WZ7LqH73BiRPhxU= github.com/multiversx/mx-chain-logger-go v1.0.13 h1:eru/TETo0MkO4ZTnXsQDKf4PBRpAXmqjT02klNT/JnY= github.com/multiversx/mx-chain-logger-go v1.0.13/go.mod h1:MZJhTAtZTJxT+yK2EHc4ZW3YOHUc1UdjCD0iahRNBZk= diff --git a/process/factory/indexerFactory.go b/process/factory/indexerFactory.go index 0666edf7..9681b841 100644 --- a/process/factory/indexerFactory.go +++ b/process/factory/indexerFactory.go @@ -160,6 +160,10 @@ func createBlockCreatorsContainer() (dataindexer.BlockContainerHandler, error) { if err != nil { return nil, err } + err = container.Add(core.SovereignChainHeader, block.NewEmptySovereignHeaderCreator()) + if err != nil { + return nil, err + } return container, nil } From 96fb3672b9d53d58210f8fa23ce9d7aae952bc31 Mon Sep 17 00:00:00 2001 From: Bogdan Rosianu Date: Fri, 29 Sep 2023 15:24:26 +0300 Subject: [PATCH 02/50] update core commit hash --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a279b671..f8a36751 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.1 github.com/multiversx/mx-chain-communication-go v1.0.7 - github.com/multiversx/mx-chain-core-go v1.2.17-0.20230928082232-5f47af6c0981 + github.com/multiversx/mx-chain-core-go v1.2.17-0.20230929122110-e9bafb263bce github.com/multiversx/mx-chain-logger-go v1.0.13 github.com/multiversx/mx-chain-vm-common-go v1.5.2 github.com/prometheus/client_model v0.4.0 diff --git a/go.sum b/go.sum index 52a0c2ac..4f181843 100644 --- a/go.sum +++ b/go.sum @@ -249,8 +249,8 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/mx-chain-communication-go v1.0.7 h1:7qeDBcqmGYYhSqFcpwv0qKkR3ahOfIMbRwXYEnOt/do= github.com/multiversx/mx-chain-communication-go v1.0.7/go.mod h1:+oaUowpq+SqrEmAsMPGwhz44g7L81loWb6AiNQU9Ms4= -github.com/multiversx/mx-chain-core-go v1.2.17-0.20230928082232-5f47af6c0981 h1:5BdwV8HMLz3pSeabqwQ90ggOww9V4RloAfBgm2dtQXQ= -github.com/multiversx/mx-chain-core-go v1.2.17-0.20230928082232-5f47af6c0981/go.mod h1:BILOGHUOIG5dNNX8cgkzCNfDaVtoYrJRYcPnpxRMH84= +github.com/multiversx/mx-chain-core-go v1.2.17-0.20230929122110-e9bafb263bce h1:dV53Am3PT3p3e0ksyAM0TlRiN+mSiIwB6i7j5+amv5M= +github.com/multiversx/mx-chain-core-go v1.2.17-0.20230929122110-e9bafb263bce/go.mod h1:BILOGHUOIG5dNNX8cgkzCNfDaVtoYrJRYcPnpxRMH84= github.com/multiversx/mx-chain-crypto-go v1.2.8 h1:wOgVlUaO5X4L8iEbFjcQcL8SZvv6WZ7LqH73BiRPhxU= github.com/multiversx/mx-chain-logger-go v1.0.13 h1:eru/TETo0MkO4ZTnXsQDKf4PBRpAXmqjT02klNT/JnY= github.com/multiversx/mx-chain-logger-go v1.0.13/go.mod h1:MZJhTAtZTJxT+yK2EHc4ZW3YOHUc1UdjCD0iahRNBZk= From 98a3964699104dcf4bb38af5c68ecc78463f4c8d Mon Sep 17 00:00:00 2001 From: Bogdan Rosianu Date: Fri, 29 Sep 2023 15:26:18 +0300 Subject: [PATCH 03/50] update vm common commit hash --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f8a36751..da349379 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/multiversx/mx-chain-communication-go v1.0.7 github.com/multiversx/mx-chain-core-go v1.2.17-0.20230929122110-e9bafb263bce github.com/multiversx/mx-chain-logger-go v1.0.13 - github.com/multiversx/mx-chain-vm-common-go v1.5.2 + github.com/multiversx/mx-chain-vm-common-go v1.5.6-0.20230929122105-486b4b0c27fa github.com/prometheus/client_model v0.4.0 github.com/prometheus/common v0.37.0 github.com/stretchr/testify v1.8.4 diff --git a/go.sum b/go.sum index 4f181843..954ab244 100644 --- a/go.sum +++ b/go.sum @@ -254,8 +254,8 @@ github.com/multiversx/mx-chain-core-go v1.2.17-0.20230929122110-e9bafb263bce/go. github.com/multiversx/mx-chain-crypto-go v1.2.8 h1:wOgVlUaO5X4L8iEbFjcQcL8SZvv6WZ7LqH73BiRPhxU= github.com/multiversx/mx-chain-logger-go v1.0.13 h1:eru/TETo0MkO4ZTnXsQDKf4PBRpAXmqjT02klNT/JnY= github.com/multiversx/mx-chain-logger-go v1.0.13/go.mod h1:MZJhTAtZTJxT+yK2EHc4ZW3YOHUc1UdjCD0iahRNBZk= -github.com/multiversx/mx-chain-vm-common-go v1.5.2 h1:iRWJNlogjkq9w+pJZIfkVkXQFmMoRxZr6pzCfg2/K68= -github.com/multiversx/mx-chain-vm-common-go v1.5.2/go.mod h1:sqkKMCnwkWl8DURdb9q7pctK8IANghdHY1KJLE0ox2c= +github.com/multiversx/mx-chain-vm-common-go v1.5.6-0.20230929122105-486b4b0c27fa h1:CuBesySqOmlVnwV8WCa6t942b9LTiPEVhwK1jwl1hsg= +github.com/multiversx/mx-chain-vm-common-go v1.5.6-0.20230929122105-486b4b0c27fa/go.mod h1:7nnwORw+90mkCmlQTJyvWde0uPkO4KQYQEuxFdz9wNI= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= From 4cb6f3eeaf05b46f5daceb1556515873625562c7 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 5 Oct 2023 13:12:16 +0300 Subject: [PATCH 04/50] ignore scrs with no original tx hash --- process/elasticproc/logsevents/informativeLogsProcessor.go | 5 +++++ process/elasticproc/transactions/scrsDataToTransactions.go | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/process/elasticproc/logsevents/informativeLogsProcessor.go b/process/elasticproc/logsevents/informativeLogsProcessor.go index 3706f708..a92fd3ca 100644 --- a/process/elasticproc/logsevents/informativeLogsProcessor.go +++ b/process/elasticproc/logsevents/informativeLogsProcessor.go @@ -61,6 +61,11 @@ func processEventNoTx(args *argsProcessEvent) argOutputProcessEvent { processed: true, } } + if scr.OriginalTxHash == "" { + return argOutputProcessEvent{ + processed: true, + } + } record := &outport.StatusInfo{} switch string(args.event.GetIdentifier()) { diff --git a/process/elasticproc/transactions/scrsDataToTransactions.go b/process/elasticproc/transactions/scrsDataToTransactions.go index f18aa741..77ba1f5b 100644 --- a/process/elasticproc/transactions/scrsDataToTransactions.go +++ b/process/elasticproc/transactions/scrsDataToTransactions.go @@ -54,7 +54,7 @@ func (st *scrsDataToTransactions) processTransactionsAfterSCRsWereAttached(trans func (st *scrsDataToTransactions) processSCRsWithoutTx(scrs []*data.ScResult) map[string]*data.FeeData { txHashRefund := make(map[string]*data.FeeData) for _, scr := range scrs { - if scr.InitialTxGasUsed == 0 { + if scr.InitialTxGasUsed == 0 || scr.OriginalTxHash == "" { continue } From 65fd3d9d9e2471214c8926dda30394b179543e7a Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 6 Oct 2023 14:10:20 +0300 Subject: [PATCH 05/50] remove metachain check --- process/elasticproc/logsevents/delegatorsProcessor.go | 4 ---- process/elasticproc/logsevents/esdtIssueProcessor.go | 4 ---- 2 files changed, 8 deletions(-) diff --git a/process/elasticproc/logsevents/delegatorsProcessor.go b/process/elasticproc/logsevents/delegatorsProcessor.go index 26fd3a82..0cec5816 100644 --- a/process/elasticproc/logsevents/delegatorsProcessor.go +++ b/process/elasticproc/logsevents/delegatorsProcessor.go @@ -44,10 +44,6 @@ func newDelegatorsProcessor( } func (dp *delegatorsProc) processEvent(args *argsProcessEvent) argOutputProcessEvent { - if args.selfShardID != core.MetachainShardId { - return argOutputProcessEvent{} - } - eventIdentifierStr := string(args.event.GetIdentifier()) _, ok := dp.delegatorsOperations[eventIdentifierStr] if !ok { diff --git a/process/elasticproc/logsevents/esdtIssueProcessor.go b/process/elasticproc/logsevents/esdtIssueProcessor.go index 2af66180..7b649cc1 100644 --- a/process/elasticproc/logsevents/esdtIssueProcessor.go +++ b/process/elasticproc/logsevents/esdtIssueProcessor.go @@ -41,10 +41,6 @@ func newESDTIssueProcessor(pubkeyConverter core.PubkeyConverter) *esdtIssueProce } func (eip *esdtIssueProcessor) processEvent(args *argsProcessEvent) argOutputProcessEvent { - if args.selfShardID != core.MetachainShardId { - return argOutputProcessEvent{} - } - identifierStr := string(args.event.GetIdentifier()) _, ok := eip.issueOperationsIdentifiers[identifierStr] if !ok { From 57a0698fd92a999ce3a25c6d6bcb6f36f7a5692d Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 11 Oct 2023 12:43:38 +0300 Subject: [PATCH 06/50] remove unit test --- .../logsevents/esdtIssueProcessor_test.go | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/process/elasticproc/logsevents/esdtIssueProcessor_test.go b/process/elasticproc/logsevents/esdtIssueProcessor_test.go index f62d2ab3..026561d3 100644 --- a/process/elasticproc/logsevents/esdtIssueProcessor_test.go +++ b/process/elasticproc/logsevents/esdtIssueProcessor_test.go @@ -83,23 +83,3 @@ func TestIssueESDTProcessor_TransferOwnership(t *testing.T) { Properties: &data.TokenProperties{}, }, res.tokenInfo) } - -func TestIssueESDTProcessor_EventWithShardID0ShouldBeIgnored(t *testing.T) { - t.Parallel() - - esdtIssueProc := newESDTIssueProcessor(&mock.PubkeyConverterMock{}) - - event := &transaction.Event{ - Address: []byte("addr"), - Identifier: []byte(transferOwnershipFunc), - Topics: [][]byte{[]byte("MYTOKEN-abcd"), []byte("my-token"), []byte("MYTOKEN"), []byte(core.NonFungibleESDT), []byte("newOwner")}, - } - args := &argsProcessEvent{ - timestamp: 1234, - event: event, - selfShardID: 0, - } - - res := esdtIssueProc.processEvent(args) - require.False(t, res.processed) -} From 51a477ef808893d051c3a83d2965cce92a655f84 Mon Sep 17 00:00:00 2001 From: Marius C Date: Tue, 30 Jan 2024 16:38:31 +0200 Subject: [PATCH 07/50] FEAT: Update dependencies --- go.mod | 16 ++++++++-------- go.sum | 32 ++++++++++++++++---------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 2dd2ccb9..ddaf2db4 100644 --- a/go.mod +++ b/go.mod @@ -7,15 +7,15 @@ require ( github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.1 github.com/multiversx/mx-chain-communication-go v1.0.12 - github.com/multiversx/mx-chain-core-go v1.2.16 - github.com/multiversx/mx-chain-logger-go v1.0.13 - github.com/multiversx/mx-chain-vm-common-go v1.5.2 + github.com/multiversx/mx-chain-core-go v1.2.19-0.20240118100536-661f5af64039 + github.com/multiversx/mx-chain-logger-go v1.0.14-0.20231129101244-c44fa1c79b03 + github.com/multiversx/mx-chain-vm-common-go v1.5.10-0.20240118100602-3d0d315083e8 github.com/prometheus/client_model v0.4.0 github.com/prometheus/common v0.37.0 github.com/stretchr/testify v1.8.4 github.com/tidwall/gjson v1.14.0 github.com/urfave/cli v1.22.10 - google.golang.org/protobuf v1.30.0 + google.golang.org/protobuf v1.31.0 ) require ( @@ -54,9 +54,9 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.9.0 // indirect - golang.org/x/net v0.10.0 // indirect - golang.org/x/sys v0.8.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/net v0.16.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 87bf48e2..7d313d9e 100644 --- a/go.sum +++ b/go.sum @@ -249,13 +249,13 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/mx-chain-communication-go v1.0.12 h1:67WOaf87gpwouydD1AAOHw5LMGZh7NfITrp/KqFY3Tw= github.com/multiversx/mx-chain-communication-go v1.0.12/go.mod h1:+oaUowpq+SqrEmAsMPGwhz44g7L81loWb6AiNQU9Ms4= -github.com/multiversx/mx-chain-core-go v1.2.16 h1:m0hUNmZQjGJxKDLQOHoM9jSaeDfVTbyd+mqiS8+NckE= -github.com/multiversx/mx-chain-core-go v1.2.16/go.mod h1:BILOGHUOIG5dNNX8cgkzCNfDaVtoYrJRYcPnpxRMH84= +github.com/multiversx/mx-chain-core-go v1.2.19-0.20240118100536-661f5af64039 h1:rusWamF1HWGAHsmdz/5TSfxHpe36HHvXCEKz3tic2yo= +github.com/multiversx/mx-chain-core-go v1.2.19-0.20240118100536-661f5af64039/go.mod h1:P/YBoFnt25XUaCQ7Q/SD15vhnc9yV5JDhHxyFO9P8Z0= github.com/multiversx/mx-chain-crypto-go v1.2.8 h1:wOgVlUaO5X4L8iEbFjcQcL8SZvv6WZ7LqH73BiRPhxU= -github.com/multiversx/mx-chain-logger-go v1.0.13 h1:eru/TETo0MkO4ZTnXsQDKf4PBRpAXmqjT02klNT/JnY= -github.com/multiversx/mx-chain-logger-go v1.0.13/go.mod h1:MZJhTAtZTJxT+yK2EHc4ZW3YOHUc1UdjCD0iahRNBZk= -github.com/multiversx/mx-chain-vm-common-go v1.5.2 h1:iRWJNlogjkq9w+pJZIfkVkXQFmMoRxZr6pzCfg2/K68= -github.com/multiversx/mx-chain-vm-common-go v1.5.2/go.mod h1:sqkKMCnwkWl8DURdb9q7pctK8IANghdHY1KJLE0ox2c= +github.com/multiversx/mx-chain-logger-go v1.0.14-0.20231129101244-c44fa1c79b03 h1:krjJTyN9jrFTK0goMGFdgvJGy6bYSqe8EtI/HCceUmU= +github.com/multiversx/mx-chain-logger-go v1.0.14-0.20231129101244-c44fa1c79b03/go.mod h1:fH/fR/GEBsDjPkBoZDVJMoYo2HhlA7++DP6QfITJ1N8= +github.com/multiversx/mx-chain-vm-common-go v1.5.10-0.20240118100602-3d0d315083e8 h1:0/k3n7Ak66oU1ygy8XR+4Q53DGmhS0VrMdKcZO433FI= +github.com/multiversx/mx-chain-vm-common-go v1.5.10-0.20240118100602-3d0d315083e8/go.mod h1:1ZUnRk7l/eTOyu2DOxy6zfEn1SAM/1u0nHUXE1Jw9xY= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -362,8 +362,8 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -431,8 +431,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos= +golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -499,8 +499,8 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -510,8 +510,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -637,8 +637,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 92975ee0084046551bd6ecdd5c6e85fbbef2d0e6 Mon Sep 17 00:00:00 2001 From: Marius C Date: Tue, 14 May 2024 15:34:03 +0300 Subject: [PATCH 08/50] FIX: go mod --- go.mod | 14 +++++++------- go.sum | 28 ++++++++++++++-------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index ec7d974b..8daf717f 100644 --- a/go.mod +++ b/go.mod @@ -7,15 +7,15 @@ require ( github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.1 github.com/multiversx/mx-chain-communication-go v1.0.15-0.20240508074652-e128a1c05c8e - github.com/multiversx/mx-chain-core-go v1.2.21-0.20240508071047-fefea5737840 + github.com/multiversx/mx-chain-core-go v1.2.21-0.20240514102932-72b1cac83784 github.com/multiversx/mx-chain-logger-go v1.0.15-0.20240508072523-3f00a726af57 - github.com/multiversx/mx-chain-vm-common-go v1.5.13-0.20240509103544-247ce5639c7a + github.com/multiversx/mx-chain-vm-common-go v1.5.13-0.20240514104734-6dc768a4c5aa github.com/prometheus/client_model v0.4.0 github.com/prometheus/common v0.37.0 github.com/stretchr/testify v1.8.4 github.com/tidwall/gjson v1.14.0 github.com/urfave/cli v1.22.10 - google.golang.org/protobuf v1.30.0 + google.golang.org/protobuf v1.31.0 ) require ( @@ -54,9 +54,9 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.9.0 // indirect - golang.org/x/net v0.10.0 // indirect - golang.org/x/sys v0.8.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/net v0.16.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 5a3744d9..74fbbfee 100644 --- a/go.sum +++ b/go.sum @@ -249,13 +249,13 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/mx-chain-communication-go v1.0.15-0.20240508074652-e128a1c05c8e h1:Tsmwhu+UleE+l3buPuqXSKTqfu5FbPmzQ4MjMoUvCWA= github.com/multiversx/mx-chain-communication-go v1.0.15-0.20240508074652-e128a1c05c8e/go.mod h1:2yXl18wUbuV3cRZr7VHxM1xo73kTaC1WUcu2kx8R034= -github.com/multiversx/mx-chain-core-go v1.2.21-0.20240508071047-fefea5737840 h1:2mCrTUmbbA+Xv4UifZY9xptrGjcJBcJ2wavSb4FwejU= -github.com/multiversx/mx-chain-core-go v1.2.21-0.20240508071047-fefea5737840/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= +github.com/multiversx/mx-chain-core-go v1.2.21-0.20240514102932-72b1cac83784 h1:Dxm8MIIbEQW9hUIjfiVFm7mR+UbOSkb7xhMtXHP7dmk= +github.com/multiversx/mx-chain-core-go v1.2.21-0.20240514102932-72b1cac83784/go.mod h1:P/YBoFnt25XUaCQ7Q/SD15vhnc9yV5JDhHxyFO9P8Z0= github.com/multiversx/mx-chain-crypto-go v1.2.12-0.20240508074452-cc21c1b505df h1:clihfi78bMEOWk/qw6WA4uQbCM2e2NGliqswLAvw19k= github.com/multiversx/mx-chain-logger-go v1.0.15-0.20240508072523-3f00a726af57 h1:g9t410dqjcb7UUptbVd/H6Ua12sEzWU4v7VplyNvRZ0= github.com/multiversx/mx-chain-logger-go v1.0.15-0.20240508072523-3f00a726af57/go.mod h1:cY6CIXpndW5g5PTPn4WzPwka/UBEf+mgw+PSY5pHGAU= -github.com/multiversx/mx-chain-vm-common-go v1.5.13-0.20240509103544-247ce5639c7a h1:7M+jXVlnl43zd2NuimL1KnAVAdpUr/QoHqG0TUKoyaM= -github.com/multiversx/mx-chain-vm-common-go v1.5.13-0.20240509103544-247ce5639c7a/go.mod h1:RgGmPei0suQcFTHfO4cS5dxJSiokp2SM5lmNgp1icMo= +github.com/multiversx/mx-chain-vm-common-go v1.5.13-0.20240514104734-6dc768a4c5aa h1:CJyBXV0PHDE3/bjJU3quoNBjOmGjrn/EBqU7wBdcqhk= +github.com/multiversx/mx-chain-vm-common-go v1.5.13-0.20240514104734-6dc768a4c5aa/go.mod h1:/scqfBjA912V3Z2sPlkRVxWAE3puVNS2adQegm4LjCc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -362,8 +362,8 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -431,8 +431,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos= +golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -499,8 +499,8 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -510,8 +510,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -637,8 +637,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 2e10fe54c806d6a4bf13c899cd6a69ae58e8890b Mon Sep 17 00:00:00 2001 From: miiu Date: Thu, 5 Sep 2024 10:43:11 +0300 Subject: [PATCH 09/50] fix mappings --- templates/noKibana/accountsESDT.go | 2 +- templates/noKibana/tokens.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/noKibana/accountsESDT.go b/templates/noKibana/accountsESDT.go index 027976b7..3204c24d 100644 --- a/templates/noKibana/accountsESDT.go +++ b/templates/noKibana/accountsESDT.go @@ -73,7 +73,7 @@ var AccountsESDT = Object{ "format": "epoch_second", }, "token": Object{ - "type": "text", + "type": "keyword", }, "tokenNonce": Object{ "type": "double", diff --git a/templates/noKibana/tokens.go b/templates/noKibana/tokens.go index f528f0bd..6d4e8240 100644 --- a/templates/noKibana/tokens.go +++ b/templates/noKibana/tokens.go @@ -162,7 +162,7 @@ var Tokens = Object{ "format": "epoch_second", }, "token": Object{ - "type": "text", + "type": "keyword", }, "type": Object{ "type": "keyword", From f37745c352a59501aaffe33b0b868233e6c01299 Mon Sep 17 00:00:00 2001 From: Marius C Date: Wed, 9 Oct 2024 12:24:08 +0300 Subject: [PATCH 10/50] FIX: go mod --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 8daf717f..bdb9829c 100644 --- a/go.mod +++ b/go.mod @@ -7,9 +7,9 @@ require ( github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.1 github.com/multiversx/mx-chain-communication-go v1.0.15-0.20240508074652-e128a1c05c8e - github.com/multiversx/mx-chain-core-go v1.2.21-0.20240514102932-72b1cac83784 - github.com/multiversx/mx-chain-logger-go v1.0.15-0.20240508072523-3f00a726af57 - github.com/multiversx/mx-chain-vm-common-go v1.5.13-0.20240514104734-6dc768a4c5aa + github.com/multiversx/mx-chain-core-go v1.2.23-0.20241007113300-50ac1ae23824 + github.com/multiversx/mx-chain-logger-go v1.0.15 + github.com/multiversx/mx-chain-vm-common-go v1.5.17-0.20241008143259-334f3a5bafe0 github.com/prometheus/client_model v0.4.0 github.com/prometheus/common v0.37.0 github.com/stretchr/testify v1.8.4 diff --git a/go.sum b/go.sum index 74fbbfee..f2f9d56e 100644 --- a/go.sum +++ b/go.sum @@ -249,13 +249,13 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/mx-chain-communication-go v1.0.15-0.20240508074652-e128a1c05c8e h1:Tsmwhu+UleE+l3buPuqXSKTqfu5FbPmzQ4MjMoUvCWA= github.com/multiversx/mx-chain-communication-go v1.0.15-0.20240508074652-e128a1c05c8e/go.mod h1:2yXl18wUbuV3cRZr7VHxM1xo73kTaC1WUcu2kx8R034= -github.com/multiversx/mx-chain-core-go v1.2.21-0.20240514102932-72b1cac83784 h1:Dxm8MIIbEQW9hUIjfiVFm7mR+UbOSkb7xhMtXHP7dmk= -github.com/multiversx/mx-chain-core-go v1.2.21-0.20240514102932-72b1cac83784/go.mod h1:P/YBoFnt25XUaCQ7Q/SD15vhnc9yV5JDhHxyFO9P8Z0= +github.com/multiversx/mx-chain-core-go v1.2.23-0.20241007113300-50ac1ae23824 h1:OHYcWOeTNwSaTMRAfusu6/1zoTWGEtHKPBig4dbRAwM= +github.com/multiversx/mx-chain-core-go v1.2.23-0.20241007113300-50ac1ae23824/go.mod h1:P/YBoFnt25XUaCQ7Q/SD15vhnc9yV5JDhHxyFO9P8Z0= github.com/multiversx/mx-chain-crypto-go v1.2.12-0.20240508074452-cc21c1b505df h1:clihfi78bMEOWk/qw6WA4uQbCM2e2NGliqswLAvw19k= -github.com/multiversx/mx-chain-logger-go v1.0.15-0.20240508072523-3f00a726af57 h1:g9t410dqjcb7UUptbVd/H6Ua12sEzWU4v7VplyNvRZ0= -github.com/multiversx/mx-chain-logger-go v1.0.15-0.20240508072523-3f00a726af57/go.mod h1:cY6CIXpndW5g5PTPn4WzPwka/UBEf+mgw+PSY5pHGAU= -github.com/multiversx/mx-chain-vm-common-go v1.5.13-0.20240514104734-6dc768a4c5aa h1:CJyBXV0PHDE3/bjJU3quoNBjOmGjrn/EBqU7wBdcqhk= -github.com/multiversx/mx-chain-vm-common-go v1.5.13-0.20240514104734-6dc768a4c5aa/go.mod h1:/scqfBjA912V3Z2sPlkRVxWAE3puVNS2adQegm4LjCc= +github.com/multiversx/mx-chain-logger-go v1.0.15 h1:HlNdK8etyJyL9NQ+6mIXyKPEBo+wRqOwi3n+m2QIHXc= +github.com/multiversx/mx-chain-logger-go v1.0.15/go.mod h1:t3PRKaWB1M+i6gUfD27KXgzLJJC+mAQiN+FLlL1yoGQ= +github.com/multiversx/mx-chain-vm-common-go v1.5.17-0.20241008143259-334f3a5bafe0 h1:WpfFe6ueS+nRIq2RzVaeF1/TJStyYH/YLYPCmN8kdDM= +github.com/multiversx/mx-chain-vm-common-go v1.5.17-0.20241008143259-334f3a5bafe0/go.mod h1:IOE+gYAtTEKZJwUr9ZhOnjdf4vFKrZdZ9RVEyuFsEP4= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= From 7b5c180a61ea85019a7d168c5d46443384f8a07e Mon Sep 17 00:00:00 2001 From: Marius C Date: Wed, 9 Oct 2024 12:30:24 +0300 Subject: [PATCH 11/50] FIX: go mod 2 --- go.mod | 14 +++++++------- go.sum | 28 ++++++++++++++-------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index eb32dd80..33369820 100644 --- a/go.mod +++ b/go.mod @@ -7,15 +7,15 @@ require ( github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.1 github.com/multiversx/mx-chain-communication-go v1.1.0 - github.com/multiversx/mx-chain-core-go v1.2.21 + github.com/multiversx/mx-chain-core-go v1.2.23-0.20241007113300-50ac1ae23824 github.com/multiversx/mx-chain-logger-go v1.0.15 - github.com/multiversx/mx-chain-vm-common-go v1.5.13 + github.com/multiversx/mx-chain-vm-common-go v1.5.17-0.20241008143259-334f3a5bafe0 github.com/prometheus/client_model v0.4.0 github.com/prometheus/common v0.37.0 github.com/stretchr/testify v1.8.4 github.com/tidwall/gjson v1.14.0 github.com/urfave/cli v1.22.10 - google.golang.org/protobuf v1.30.0 + google.golang.org/protobuf v1.31.0 ) require ( @@ -54,9 +54,9 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.9.0 // indirect - golang.org/x/net v0.10.0 // indirect - golang.org/x/sys v0.8.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/net v0.16.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 82f3a0f7..9100464b 100644 --- a/go.sum +++ b/go.sum @@ -249,13 +249,13 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/mx-chain-communication-go v1.1.0 h1:J7bX6HoN3HiHY7cUeEjG8AJWgQDDPcY+OPDOsSUOkRE= github.com/multiversx/mx-chain-communication-go v1.1.0/go.mod h1:WK6bP4pGEHGDDna/AYRIMtl6G9OA0NByI1Lw8PmOnRM= -github.com/multiversx/mx-chain-core-go v1.2.21 h1:+XVKznPTlUU5EFS1A8chtS8fStW60upRIyF4Pgml19I= -github.com/multiversx/mx-chain-core-go v1.2.21/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= +github.com/multiversx/mx-chain-core-go v1.2.23-0.20241007113300-50ac1ae23824 h1:OHYcWOeTNwSaTMRAfusu6/1zoTWGEtHKPBig4dbRAwM= +github.com/multiversx/mx-chain-core-go v1.2.23-0.20241007113300-50ac1ae23824/go.mod h1:P/YBoFnt25XUaCQ7Q/SD15vhnc9yV5JDhHxyFO9P8Z0= github.com/multiversx/mx-chain-crypto-go v1.2.12 h1:zWip7rpUS4CGthJxfKn5MZfMfYPjVjIiCID6uX5BSOk= github.com/multiversx/mx-chain-logger-go v1.0.15 h1:HlNdK8etyJyL9NQ+6mIXyKPEBo+wRqOwi3n+m2QIHXc= github.com/multiversx/mx-chain-logger-go v1.0.15/go.mod h1:t3PRKaWB1M+i6gUfD27KXgzLJJC+mAQiN+FLlL1yoGQ= -github.com/multiversx/mx-chain-vm-common-go v1.5.13 h1:ymnIHJW4Z4mFa0hZzla4fozkF30vjH5O1q+Y7Ftc+pQ= -github.com/multiversx/mx-chain-vm-common-go v1.5.13/go.mod h1:OSvFbzdWThfRbLZbUsEr7bikBSaLrPJQ2iUm9jw9nXQ= +github.com/multiversx/mx-chain-vm-common-go v1.5.17-0.20241008143259-334f3a5bafe0 h1:WpfFe6ueS+nRIq2RzVaeF1/TJStyYH/YLYPCmN8kdDM= +github.com/multiversx/mx-chain-vm-common-go v1.5.17-0.20241008143259-334f3a5bafe0/go.mod h1:IOE+gYAtTEKZJwUr9ZhOnjdf4vFKrZdZ9RVEyuFsEP4= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -362,8 +362,8 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -431,8 +431,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos= +golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -499,8 +499,8 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -510,8 +510,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -637,8 +637,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 641a7d1215158c6e46e110ae315fb9a963e74108 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Tue, 3 Dec 2024 16:52:24 +0200 Subject: [PATCH 12/50] run type components implementation --- factory/interface.go | 24 ++++++ factory/runType/errors.go | 7 ++ factory/runType/interface.go | 7 ++ factory/runType/runTypeComponents.go | 13 +++ factory/runType/runTypeComponentsFactory.go | 18 ++++ factory/runType/runTypeComponentsHandler.go | 83 +++++++++++++++++++ .../runType/runTypeComponentsHandler_test.go | 76 +++++++++++++++++ factory/runType/runTypeComponents_test.go | 38 +++++++++ .../sovereignRunTypeComponentsFactory.go | 18 ++++ .../sovereignRunTypeComponentsFactory_test.go | 19 +++++ 10 files changed, 303 insertions(+) create mode 100644 factory/interface.go create mode 100644 factory/runType/errors.go create mode 100644 factory/runType/interface.go create mode 100644 factory/runType/runTypeComponents.go create mode 100644 factory/runType/runTypeComponentsFactory.go create mode 100644 factory/runType/runTypeComponentsHandler.go create mode 100644 factory/runType/runTypeComponentsHandler_test.go create mode 100644 factory/runType/runTypeComponents_test.go create mode 100644 factory/runType/sovereignRunTypeComponentsFactory.go create mode 100644 factory/runType/sovereignRunTypeComponentsFactory_test.go diff --git a/factory/interface.go b/factory/interface.go new file mode 100644 index 00000000..ae3435e6 --- /dev/null +++ b/factory/interface.go @@ -0,0 +1,24 @@ +package factory + +// ComponentHandler defines the actions common to all component handlers +type ComponentHandler interface { + Create() error + Close() error + CheckSubcomponents() error + String() string +} + +// RunTypeComponentsHandler defines the run type components handler actions +type RunTypeComponentsHandler interface { + ComponentHandler + RunTypeComponentsHolder +} + +// RunTypeComponentsHolder holds the run type components +type RunTypeComponentsHolder interface { + Create() error + Close() error + CheckSubcomponents() error + String() string + IsInterfaceNil() bool +} diff --git a/factory/runType/errors.go b/factory/runType/errors.go new file mode 100644 index 00000000..657663a3 --- /dev/null +++ b/factory/runType/errors.go @@ -0,0 +1,7 @@ +package runType + +import ( + "errors" +) + +var errNilRunTypeComponents = errors.New("nil run type components") diff --git a/factory/runType/interface.go b/factory/runType/interface.go new file mode 100644 index 00000000..37448d92 --- /dev/null +++ b/factory/runType/interface.go @@ -0,0 +1,7 @@ +package runType + +// RunTypeComponentsCreator is the interface for creating run type components +type RunTypeComponentsCreator interface { + Create() *runTypeComponents + IsInterfaceNil() bool +} diff --git a/factory/runType/runTypeComponents.go b/factory/runType/runTypeComponents.go new file mode 100644 index 00000000..c960293b --- /dev/null +++ b/factory/runType/runTypeComponents.go @@ -0,0 +1,13 @@ +package runType + +type runTypeComponents struct{} + +// Close does nothing +func (rtc *runTypeComponents) Close() error { + return nil +} + +// IsInterfaceNil returns true if there is no value under the interface +func (rtc *runTypeComponents) IsInterfaceNil() bool { + return rtc == nil +} diff --git a/factory/runType/runTypeComponentsFactory.go b/factory/runType/runTypeComponentsFactory.go new file mode 100644 index 00000000..24ad9f56 --- /dev/null +++ b/factory/runType/runTypeComponentsFactory.go @@ -0,0 +1,18 @@ +package runType + +type runTypeComponentsFactory struct{} + +// NewRunTypeComponentsFactory will return a new instance of run type components factory +func NewRunTypeComponentsFactory() *runTypeComponentsFactory { + return &runTypeComponentsFactory{} +} + +// Create will create the run type components +func (rtcf *runTypeComponentsFactory) Create() *runTypeComponents { + return &runTypeComponents{} +} + +// IsInterfaceNil returns true if there is no value under the interface +func (rtcf *runTypeComponentsFactory) IsInterfaceNil() bool { + return rtcf == nil +} diff --git a/factory/runType/runTypeComponentsHandler.go b/factory/runType/runTypeComponentsHandler.go new file mode 100644 index 00000000..9b25b92b --- /dev/null +++ b/factory/runType/runTypeComponentsHandler.go @@ -0,0 +1,83 @@ +package runType + +import ( + "sync" + + "github.com/multiversx/mx-chain-core-go/core/check" + + "github.com/multiversx/mx-chain-es-indexer-go/factory" +) + +const runTypeComponentsName = "managedRunTypeComponents" + +var _ factory.ComponentHandler = (*managedRunTypeComponents)(nil) +var _ factory.RunTypeComponentsHandler = (*managedRunTypeComponents)(nil) +var _ factory.RunTypeComponentsHolder = (*managedRunTypeComponents)(nil) + +type managedRunTypeComponents struct { + *runTypeComponents + factory RunTypeComponentsCreator + mutRunTypeCoreComponents sync.RWMutex +} + +// NewManagedRunTypeComponents returns a news instance of managed runType core components +func NewManagedRunTypeComponents(rtc RunTypeComponentsCreator) (*managedRunTypeComponents, error) { + if rtc == nil { + return nil, errNilRunTypeComponents + } + + return &managedRunTypeComponents{ + runTypeComponents: nil, + factory: rtc, + }, nil +} + +// Create will create the managed components +func (mrtc *managedRunTypeComponents) Create() error { + rtc := mrtc.factory.Create() + + mrtc.mutRunTypeCoreComponents.Lock() + mrtc.runTypeComponents = rtc + mrtc.mutRunTypeCoreComponents.Unlock() + + return nil +} + +// Close will close all underlying subcomponents +func (mrtc *managedRunTypeComponents) Close() error { + mrtc.mutRunTypeCoreComponents.Lock() + defer mrtc.mutRunTypeCoreComponents.Unlock() + + if check.IfNil(mrtc.runTypeComponents) { + return nil + } + + err := mrtc.runTypeComponents.Close() + if err != nil { + return err + } + mrtc.runTypeComponents = nil + + return nil +} + +// CheckSubcomponents verifies all subcomponents +func (mrtc *managedRunTypeComponents) CheckSubcomponents() error { + mrtc.mutRunTypeCoreComponents.RLock() + defer mrtc.mutRunTypeCoreComponents.RUnlock() + + if check.IfNil(mrtc.runTypeComponents) { + return errNilRunTypeComponents + } + return nil +} + +// IsInterfaceNil returns true if the interface is nil +func (mrtc *managedRunTypeComponents) IsInterfaceNil() bool { + return mrtc == nil +} + +// String returns the name of the component +func (mrtc *managedRunTypeComponents) String() string { + return runTypeComponentsName +} diff --git a/factory/runType/runTypeComponentsHandler_test.go b/factory/runType/runTypeComponentsHandler_test.go new file mode 100644 index 00000000..573a73ff --- /dev/null +++ b/factory/runType/runTypeComponentsHandler_test.go @@ -0,0 +1,76 @@ +package runType + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/multiversx/mx-chain-es-indexer-go/factory" +) + +func createComponents() (factory.RunTypeComponentsHandler, error) { + rtcf := NewRunTypeComponentsFactory() + return NewManagedRunTypeComponents(rtcf) +} + +func TestNewManagedRunTypeComponents(t *testing.T) { + t.Parallel() + + t.Run("should error", func(t *testing.T) { + managedRunTypeComponents, err := NewManagedRunTypeComponents(nil) + require.ErrorIs(t, err, errNilRunTypeComponents) + require.Nil(t, managedRunTypeComponents) + }) + t.Run("should work", func(t *testing.T) { + rtcf := NewRunTypeComponentsFactory() + managedRunTypeComponents, err := NewManagedRunTypeComponents(rtcf) + require.NoError(t, err) + require.False(t, managedRunTypeComponents.IsInterfaceNil()) + }) +} + +func TestManagedRunTypeComponents_Create(t *testing.T) { + t.Parallel() + + t.Run("should work with getters", func(t *testing.T) { + t.Parallel() + + managedRunTypeComponents, err := createComponents() + require.NoError(t, err) + + err = managedRunTypeComponents.Create() + require.NoError(t, err) + + require.Equal(t, runTypeComponentsName, managedRunTypeComponents.String()) + require.NoError(t, managedRunTypeComponents.Close()) + }) +} + +func TestManagedRunTypeComponents_Close(t *testing.T) { + t.Parallel() + + managedRunTypeComponents, _ := createComponents() + require.NoError(t, managedRunTypeComponents.Close()) + + err := managedRunTypeComponents.Create() + require.NoError(t, err) + + require.NoError(t, managedRunTypeComponents.Close()) +} + +func TestManagedRunTypeComponents_CheckSubcomponents(t *testing.T) { + t.Parallel() + + managedRunTypeComponents, _ := createComponents() + err := managedRunTypeComponents.CheckSubcomponents() + require.Equal(t, errNilRunTypeComponents, err) + + err = managedRunTypeComponents.Create() + require.NoError(t, err) + + //TODO check for nil each subcomponent - MX-15371 + err = managedRunTypeComponents.CheckSubcomponents() + require.NoError(t, err) + + require.NoError(t, managedRunTypeComponents.Close()) +} diff --git a/factory/runType/runTypeComponents_test.go b/factory/runType/runTypeComponents_test.go new file mode 100644 index 00000000..07580802 --- /dev/null +++ b/factory/runType/runTypeComponents_test.go @@ -0,0 +1,38 @@ +package runType + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNewRunTypeComponentsFactory(t *testing.T) { + t.Parallel() + + t.Run("should work", func(t *testing.T) { + rtc := NewRunTypeComponentsFactory() + require.NotNil(t, rtc) + }) +} + +func TestRunTypeComponentsFactory_Create(t *testing.T) { + t.Parallel() + + rtcf := NewRunTypeComponentsFactory() + require.NotNil(t, rtcf) + + rtc := rtcf.Create() + require.NotNil(t, rtc) +} + +func TestRunTypeComponentsFactory_Close(t *testing.T) { + t.Parallel() + + rtcf := NewRunTypeComponentsFactory() + require.NotNil(t, rtcf) + + rtc := rtcf.Create() + require.NotNil(t, rtc) + + require.NoError(t, rtc.Close()) +} diff --git a/factory/runType/sovereignRunTypeComponentsFactory.go b/factory/runType/sovereignRunTypeComponentsFactory.go new file mode 100644 index 00000000..fd5226a5 --- /dev/null +++ b/factory/runType/sovereignRunTypeComponentsFactory.go @@ -0,0 +1,18 @@ +package runType + +type sovereignRunTypeComponentsFactory struct{} + +// NewSovereignRunTypeComponentsFactory will return a new instance of sovereign run type components factory +func NewSovereignRunTypeComponentsFactory() *sovereignRunTypeComponentsFactory { + return &sovereignRunTypeComponentsFactory{} +} + +// Create will create the run type components +func (srtcf *sovereignRunTypeComponentsFactory) Create() *runTypeComponents { + return &runTypeComponents{} +} + +// IsInterfaceNil returns true if there is no value under the interface +func (srtcf *sovereignRunTypeComponentsFactory) IsInterfaceNil() bool { + return srtcf == nil +} diff --git a/factory/runType/sovereignRunTypeComponentsFactory_test.go b/factory/runType/sovereignRunTypeComponentsFactory_test.go new file mode 100644 index 00000000..a3e959b4 --- /dev/null +++ b/factory/runType/sovereignRunTypeComponentsFactory_test.go @@ -0,0 +1,19 @@ +package runType + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSovereignRunTypeComponentsFactory_CreateAndClose(t *testing.T) { + t.Parallel() + + srtcf := NewSovereignRunTypeComponentsFactory() + require.NotNil(t, srtcf) + + srtc := srtcf.Create() + require.NotNil(t, srtc) + + require.NoError(t, srtc.Close()) +} From 92a42d09f2e57f513b07aefb92adcf8483fa7e4f Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Wed, 4 Dec 2024 15:37:52 +0200 Subject: [PATCH 13/50] sovereign flag --- cmd/elasticindexer/flags.go | 5 +++++ cmd/elasticindexer/main.go | 10 ++++++---- factory/wsIndexerFactory.go | 9 ++++++--- process/factory/indexerFactory.go | 29 ++++++++++++++++++++++++++++- 4 files changed, 45 insertions(+), 8 deletions(-) diff --git a/cmd/elasticindexer/flags.go b/cmd/elasticindexer/flags.go index d7e84b70..084f3b3e 100644 --- a/cmd/elasticindexer/flags.go +++ b/cmd/elasticindexer/flags.go @@ -47,4 +47,9 @@ var ( Name: "disable-ansi-color", Usage: "Boolean option for disabling ANSI colors in the logging system.", } + // sovereign defines a flag that specifies if what run type components should use + sovereign = cli.BoolFlag{ + Name: "sovereign-config", + Usage: "If set to true, will use sovereign run type components", + } ) diff --git a/cmd/elasticindexer/main.go b/cmd/elasticindexer/main.go index 85246d13..4c3e9366 100644 --- a/cmd/elasticindexer/main.go +++ b/cmd/elasticindexer/main.go @@ -11,13 +11,14 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/core/closing" "github.com/multiversx/mx-chain-core-go/data/outport" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-logger-go/file" + "github.com/urfave/cli" + "github.com/multiversx/mx-chain-es-indexer-go/config" "github.com/multiversx/mx-chain-es-indexer-go/factory" "github.com/multiversx/mx-chain-es-indexer-go/metrics" "github.com/multiversx/mx-chain-es-indexer-go/process/wsindexer" - logger "github.com/multiversx/mx-chain-logger-go" - "github.com/multiversx/mx-chain-logger-go/file" - "github.com/urfave/cli" ) var ( @@ -63,6 +64,7 @@ func main() { logLevel, logSaveFile, disableAnsiColor, + sovereign, } app.Authors = []cli.Author{ { @@ -98,7 +100,7 @@ func startIndexer(ctx *cli.Context) error { } statusMetrics := metrics.NewStatusMetrics() - wsHost, err := factory.CreateWsIndexer(cfg, clusterCfg, statusMetrics, ctx.App.Version) + wsHost, err := factory.CreateWsIndexer(cfg, clusterCfg, statusMetrics, ctx.App.Version, ctx.GlobalBool(sovereign.Name)) if err != nil { return fmt.Errorf("%w while creating the indexer", err) } diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index 90beb5a9..5b41359c 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -7,23 +7,24 @@ import ( factoryHasher "github.com/multiversx/mx-chain-core-go/hashing/factory" "github.com/multiversx/mx-chain-core-go/marshal" factoryMarshaller "github.com/multiversx/mx-chain-core-go/marshal/factory" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-es-indexer-go/config" "github.com/multiversx/mx-chain-es-indexer-go/core" "github.com/multiversx/mx-chain-es-indexer-go/process/factory" "github.com/multiversx/mx-chain-es-indexer-go/process/wsindexer" - logger "github.com/multiversx/mx-chain-logger-go" ) var log = logger.GetOrCreate("elasticindexer") // CreateWsIndexer will create a new instance of wsindexer.WSClient -func CreateWsIndexer(cfg config.Config, clusterCfg config.ClusterConfig, statusMetrics core.StatusMetricsHandler, version string) (wsindexer.WSClient, error) { +func CreateWsIndexer(cfg config.Config, clusterCfg config.ClusterConfig, statusMetrics core.StatusMetricsHandler, version string, isSovereignType bool) (wsindexer.WSClient, error) { wsMarshaller, err := factoryMarshaller.NewMarshalizer(clusterCfg.Config.WebSocket.DataMarshallerType) if err != nil { return nil, err } - dataIndexer, err := createDataIndexer(cfg, clusterCfg, wsMarshaller, statusMetrics, version) + dataIndexer, err := createDataIndexer(cfg, clusterCfg, wsMarshaller, statusMetrics, version, isSovereignType) if err != nil { return nil, err } @@ -57,6 +58,7 @@ func createDataIndexer( wsMarshaller marshal.Marshalizer, statusMetrics core.StatusMetricsHandler, version string, + isSovereignConfig bool, ) (wsindexer.DataIndexer, error) { marshaller, err := factoryMarshaller.NewMarshalizer(cfg.Config.Marshaller.Type) if err != nil { @@ -76,6 +78,7 @@ func createDataIndexer( } return factory.NewIndexer(factory.ArgsIndexerFactory{ + SovereignConfig: isSovereignConfig, UseKibana: clusterCfg.Config.ElasticCluster.UseKibana, Denomination: cfg.Config.Economics.Denomination, BulkRequestMaxSize: clusterCfg.Config.ElasticCluster.BulkRequestMaxSizeInBytes, diff --git a/process/factory/indexerFactory.go b/process/factory/indexerFactory.go index 5da830f1..027dc651 100644 --- a/process/factory/indexerFactory.go +++ b/process/factory/indexerFactory.go @@ -12,14 +12,16 @@ import ( "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-es-indexer-go/client" "github.com/multiversx/mx-chain-es-indexer-go/client/logging" "github.com/multiversx/mx-chain-es-indexer-go/client/transport" indexerCore "github.com/multiversx/mx-chain-es-indexer-go/core" + "github.com/multiversx/mx-chain-es-indexer-go/factory/runType" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" - logger "github.com/multiversx/mx-chain-logger-go" ) var log = logger.GetOrCreate("indexer/factory") @@ -30,6 +32,7 @@ type ArgsIndexerFactory struct { Enabled bool UseKibana bool ImportDB bool + SovereignConfig bool Denomination int BulkRequestMaxSize int Url string @@ -44,6 +47,7 @@ type ArgsIndexerFactory struct { AddressPubkeyConverter core.PubkeyConverter ValidatorPubkeyConverter core.PubkeyConverter StatusMetrics indexerCore.StatusMetricsHandler + RunTypeComponents runType.RunTypeComponentsHandler } // NewIndexer will create a new instance of Indexer @@ -53,6 +57,15 @@ func NewIndexer(args ArgsIndexerFactory) (dataindexer.Indexer, error) { return nil, err } + if args.SovereignConfig { + args.RunTypeComponents, err = createManagedRunTypeComponents(runType.NewSovereignRunTypeComponentsFactory()) + } else { + args.RunTypeComponents, err = createManagedRunTypeComponents(runType.NewRunTypeComponentsFactory()) + } + if err != nil { + return nil, err + } + elasticProcessor, err := createElasticProcessor(args) if err != nil { return nil, err @@ -72,6 +85,20 @@ func NewIndexer(args ArgsIndexerFactory) (dataindexer.Indexer, error) { return dataindexer.NewDataIndexer(arguments) } +func createManagedRunTypeComponents(factory runType.RunTypeComponentsCreator) (runType.RunTypeComponentsHandler, error) { + managedRunTypeComponents, err := runType.NewManagedRunTypeComponents(factory) + if err != nil { + return nil, err + } + + err = managedRunTypeComponents.Create() + if err != nil { + return nil, err + } + + return managedRunTypeComponents, nil +} + func retryBackOff(attempt int) time.Duration { d := time.Duration(math.Exp2(float64(attempt))) * time.Second log.Debug("elastic: retry backoff", "attempt", attempt, "sleep duration", d) From b74e23e5d99d9436a0555e689b1251ded5b55658 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Wed, 4 Dec 2024 16:16:05 +0200 Subject: [PATCH 14/50] tx hashes extractor implementation --- factory/interface.go | 24 ----------- factory/runType/interface.go | 28 ++++++++++++ factory/runType/runTypeComponents.go | 8 +++- factory/runType/runTypeComponentsFactory.go | 8 +++- factory/runType/runTypeComponentsHandler.go | 23 ++++++++-- .../runType/runTypeComponentsHandler_test.go | 8 ++-- .../sovereignRunTypeComponentsFactory.go | 8 +++- mock/txHashExtractorMock.go | 24 +++++++++++ .../factory/elasticProcessorFactory.go | 3 ++ .../factory/elasticProcessorFactory_test.go | 4 +- process/elasticproc/transactions/checkers.go | 6 ++- .../elasticproc/transactions/checkers_test.go | 27 +++++++++++- process/elasticproc/transactions/errors.go | 8 ++++ process/elasticproc/transactions/interface.go | 7 +++ .../transactions/sovereignTxHashExtractor.go | 22 ++++++++++ .../sovereignTxHashExtractor_test.go | 23 ++++++++++ .../transactions/transactionsGrouper.go | 43 ++++++------------- .../transactions/transactionsGrouper_test.go | 30 ++++++++++--- .../transactions/transactionsProcessor.go | 8 ++-- .../transactionsProcessor_test.go | 6 ++- .../transactions/txHashExtractor.go | 39 +++++++++++++++++ .../transactions/txHashExtractor_test.go | 24 +++++++++++ process/factory/indexerFactory.go | 1 + 23 files changed, 304 insertions(+), 78 deletions(-) delete mode 100644 factory/interface.go create mode 100644 mock/txHashExtractorMock.go create mode 100644 process/elasticproc/transactions/errors.go create mode 100644 process/elasticproc/transactions/sovereignTxHashExtractor.go create mode 100644 process/elasticproc/transactions/sovereignTxHashExtractor_test.go create mode 100644 process/elasticproc/transactions/txHashExtractor.go create mode 100644 process/elasticproc/transactions/txHashExtractor_test.go diff --git a/factory/interface.go b/factory/interface.go deleted file mode 100644 index ae3435e6..00000000 --- a/factory/interface.go +++ /dev/null @@ -1,24 +0,0 @@ -package factory - -// ComponentHandler defines the actions common to all component handlers -type ComponentHandler interface { - Create() error - Close() error - CheckSubcomponents() error - String() string -} - -// RunTypeComponentsHandler defines the run type components handler actions -type RunTypeComponentsHandler interface { - ComponentHandler - RunTypeComponentsHolder -} - -// RunTypeComponentsHolder holds the run type components -type RunTypeComponentsHolder interface { - Create() error - Close() error - CheckSubcomponents() error - String() string - IsInterfaceNil() bool -} diff --git a/factory/runType/interface.go b/factory/runType/interface.go index 37448d92..1c8cd4d5 100644 --- a/factory/runType/interface.go +++ b/factory/runType/interface.go @@ -1,7 +1,35 @@ package runType +import ( + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" +) + // RunTypeComponentsCreator is the interface for creating run type components type RunTypeComponentsCreator interface { Create() *runTypeComponents IsInterfaceNil() bool } + +// ComponentHandler defines the actions common to all component handlers +type ComponentHandler interface { + Create() error + Close() error + CheckSubcomponents() error + String() string +} + +// RunTypeComponentsHandler defines the run type components handler actions +type RunTypeComponentsHandler interface { + ComponentHandler + RunTypeComponentsHolder +} + +// RunTypeComponentsHolder holds the run type components +type RunTypeComponentsHolder interface { + TxHashExtractorCreator() transactions.TxHashExtractor + Create() error + Close() error + CheckSubcomponents() error + String() string + IsInterfaceNil() bool +} diff --git a/factory/runType/runTypeComponents.go b/factory/runType/runTypeComponents.go index c960293b..74249f1d 100644 --- a/factory/runType/runTypeComponents.go +++ b/factory/runType/runTypeComponents.go @@ -1,6 +1,12 @@ package runType -type runTypeComponents struct{} +import ( + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" +) + +type runTypeComponents struct { + txHashExtractor transactions.TxHashExtractor +} // Close does nothing func (rtc *runTypeComponents) Close() error { diff --git a/factory/runType/runTypeComponentsFactory.go b/factory/runType/runTypeComponentsFactory.go index 24ad9f56..e9d6b88d 100644 --- a/factory/runType/runTypeComponentsFactory.go +++ b/factory/runType/runTypeComponentsFactory.go @@ -1,5 +1,9 @@ package runType +import ( + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" +) + type runTypeComponentsFactory struct{} // NewRunTypeComponentsFactory will return a new instance of run type components factory @@ -9,7 +13,9 @@ func NewRunTypeComponentsFactory() *runTypeComponentsFactory { // Create will create the run type components func (rtcf *runTypeComponentsFactory) Create() *runTypeComponents { - return &runTypeComponents{} + return &runTypeComponents{ + txHashExtractor: transactions.NewTxHashExtractor(), + } } // IsInterfaceNil returns true if there is no value under the interface diff --git a/factory/runType/runTypeComponentsHandler.go b/factory/runType/runTypeComponentsHandler.go index 9b25b92b..0d4e8330 100644 --- a/factory/runType/runTypeComponentsHandler.go +++ b/factory/runType/runTypeComponentsHandler.go @@ -5,14 +5,14 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" - "github.com/multiversx/mx-chain-es-indexer-go/factory" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" ) const runTypeComponentsName = "managedRunTypeComponents" -var _ factory.ComponentHandler = (*managedRunTypeComponents)(nil) -var _ factory.RunTypeComponentsHandler = (*managedRunTypeComponents)(nil) -var _ factory.RunTypeComponentsHolder = (*managedRunTypeComponents)(nil) +var _ ComponentHandler = (*managedRunTypeComponents)(nil) +var _ RunTypeComponentsHandler = (*managedRunTypeComponents)(nil) +var _ RunTypeComponentsHolder = (*managedRunTypeComponents)(nil) type managedRunTypeComponents struct { *runTypeComponents @@ -69,9 +69,24 @@ func (mrtc *managedRunTypeComponents) CheckSubcomponents() error { if check.IfNil(mrtc.runTypeComponents) { return errNilRunTypeComponents } + if check.IfNil(mrtc.txHashExtractor) { + return transactions.ErrNilTxHashExtractor + } return nil } +// TxHashExtractorCreator return tx hash extractor +func (mrtc *managedRunTypeComponents) TxHashExtractorCreator() transactions.TxHashExtractor { + mrtc.mutRunTypeCoreComponents.Lock() + defer mrtc.mutRunTypeCoreComponents.Unlock() + + if check.IfNil(mrtc.runTypeComponents) { + return nil + } + + return mrtc.runTypeComponents.txHashExtractor +} + // IsInterfaceNil returns true if the interface is nil func (mrtc *managedRunTypeComponents) IsInterfaceNil() bool { return mrtc == nil diff --git a/factory/runType/runTypeComponentsHandler_test.go b/factory/runType/runTypeComponentsHandler_test.go index 573a73ff..7c5ac0f4 100644 --- a/factory/runType/runTypeComponentsHandler_test.go +++ b/factory/runType/runTypeComponentsHandler_test.go @@ -4,11 +4,9 @@ import ( "testing" "github.com/stretchr/testify/require" - - "github.com/multiversx/mx-chain-es-indexer-go/factory" ) -func createComponents() (factory.RunTypeComponentsHandler, error) { +func createComponents() (RunTypeComponentsHandler, error) { rtcf := NewRunTypeComponentsFactory() return NewManagedRunTypeComponents(rtcf) } @@ -38,9 +36,13 @@ func TestManagedRunTypeComponents_Create(t *testing.T) { managedRunTypeComponents, err := createComponents() require.NoError(t, err) + require.Nil(t, managedRunTypeComponents.TxHashExtractorCreator()) + err = managedRunTypeComponents.Create() require.NoError(t, err) + require.NotNil(t, managedRunTypeComponents.TxHashExtractorCreator()) + require.Equal(t, runTypeComponentsName, managedRunTypeComponents.String()) require.NoError(t, managedRunTypeComponents.Close()) }) diff --git a/factory/runType/sovereignRunTypeComponentsFactory.go b/factory/runType/sovereignRunTypeComponentsFactory.go index fd5226a5..5ddc099b 100644 --- a/factory/runType/sovereignRunTypeComponentsFactory.go +++ b/factory/runType/sovereignRunTypeComponentsFactory.go @@ -1,5 +1,9 @@ package runType +import ( + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" +) + type sovereignRunTypeComponentsFactory struct{} // NewSovereignRunTypeComponentsFactory will return a new instance of sovereign run type components factory @@ -9,7 +13,9 @@ func NewSovereignRunTypeComponentsFactory() *sovereignRunTypeComponentsFactory { // Create will create the run type components func (srtcf *sovereignRunTypeComponentsFactory) Create() *runTypeComponents { - return &runTypeComponents{} + return &runTypeComponents{ + txHashExtractor: transactions.NewSovereignTxHashExtractor(), + } } // IsInterfaceNil returns true if there is no value under the interface diff --git a/mock/txHashExtractorMock.go b/mock/txHashExtractorMock.go new file mode 100644 index 00000000..cb3a46f0 --- /dev/null +++ b/mock/txHashExtractorMock.go @@ -0,0 +1,24 @@ +package mock + +import ( + coreData "github.com/multiversx/mx-chain-core-go/data" +) + +// TxHashExtractorMock - +type TxHashExtractorMock struct { + ExtractExecutedTxHashesCalled func(mbIndex int, mbTxHashes [][]byte, header coreData.HeaderHandler) [][]byte +} + +// ExtractExecutedTxHashes - +func (the *TxHashExtractorMock) ExtractExecutedTxHashes(mbIndex int, mbTxHashes [][]byte, header coreData.HeaderHandler) [][]byte { + if the.ExtractExecutedTxHashesCalled != nil { + return the.ExtractExecutedTxHashesCalled(mbIndex, mbTxHashes, header) + } + + return make([][]byte, 0) +} + +// IsInterfaceNil returns true if there is no value under the interface +func (the *TxHashExtractorMock) IsInterfaceNil() bool { + return the == nil +} diff --git a/process/elasticproc/factory/elasticProcessorFactory.go b/process/elasticproc/factory/elasticProcessorFactory.go index ad865953..a61285d4 100644 --- a/process/elasticproc/factory/elasticProcessorFactory.go +++ b/process/elasticproc/factory/elasticProcessorFactory.go @@ -4,6 +4,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/accounts" @@ -31,6 +32,7 @@ type ArgElasticProcessorFactory struct { BulkRequestMaxSize int UseKibana bool ImportDB bool + TxHashExtractor transactions.TxHashExtractor } // CreateElasticProcessor will create a new instance of ElasticProcessor @@ -87,6 +89,7 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.E Hasher: arguments.Hasher, Marshalizer: arguments.Marshalizer, BalanceConverter: balanceConverter, + TxHashExtractor: arguments.TxHashExtractor, } txsProc, err := transactions.NewTransactionsProcessor(argsTxsProc) if err != nil { diff --git a/process/elasticproc/factory/elasticProcessorFactory_test.go b/process/elasticproc/factory/elasticProcessorFactory_test.go index 81e5d174..936ae352 100644 --- a/process/elasticproc/factory/elasticProcessorFactory_test.go +++ b/process/elasticproc/factory/elasticProcessorFactory_test.go @@ -3,8 +3,9 @@ package factory import ( "testing" - "github.com/multiversx/mx-chain-es-indexer-go/mock" "github.com/stretchr/testify/require" + + "github.com/multiversx/mx-chain-es-indexer-go/mock" ) func TestCreateElasticProcessor(t *testing.T) { @@ -18,6 +19,7 @@ func TestCreateElasticProcessor(t *testing.T) { EnabledIndexes: []string{"blocks"}, Denomination: 1, UseKibana: false, + TxHashExtractor: &mock.TxHashExtractorMock{}, } ep, err := CreateElasticProcessor(args) diff --git a/process/elasticproc/transactions/checkers.go b/process/elasticproc/transactions/checkers.go index 7d9fcaf7..282c66f5 100644 --- a/process/elasticproc/transactions/checkers.go +++ b/process/elasticproc/transactions/checkers.go @@ -10,9 +10,10 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/outport" + vmcommon "github.com/multiversx/mx-chain-vm-common-go" + "github.com/multiversx/mx-chain-es-indexer-go/data" elasticIndexer "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" - vmcommon "github.com/multiversx/mx-chain-vm-common-go" ) const ( @@ -35,6 +36,9 @@ func checkTxsProcessorArg(args *ArgsTransactionProcessor) error { if check.IfNil(args.BalanceConverter) { return elasticIndexer.ErrNilBalanceConverter } + if check.IfNil(args.TxHashExtractor) { + return ErrNilTxHashExtractor + } return nil } diff --git a/process/elasticproc/transactions/checkers_test.go b/process/elasticproc/transactions/checkers_test.go index 06ebe9e9..20d3f9db 100644 --- a/process/elasticproc/transactions/checkers_test.go +++ b/process/elasticproc/transactions/checkers_test.go @@ -9,18 +9,23 @@ import ( coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" + vmcommon "github.com/multiversx/mx-chain-vm-common-go" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/mock" elasticIndexer "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" - vmcommon "github.com/multiversx/mx-chain-vm-common-go" - "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" ) func createMockArgs() *ArgsTransactionProcessor { + bc, _ := converters.NewBalanceConverter(18) return &ArgsTransactionProcessor{ AddressPubkeyConverter: &mock.PubkeyConverterMock{}, Hasher: &mock.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, + BalanceConverter: bc, + TxHashExtractor: &mock.TxHashExtractorMock{}, } } @@ -59,6 +64,24 @@ func TestNewTransactionsProcessor(t *testing.T) { }, exErr: elasticIndexer.ErrNilHasher, }, + { + name: "NilBalanceConverter", + args: func() *ArgsTransactionProcessor { + args := createMockArgs() + args.BalanceConverter = nil + return args + }, + exErr: elasticIndexer.ErrNilBalanceConverter, + }, + { + name: "NilTxHashExtractor", + args: func() *ArgsTransactionProcessor { + args := createMockArgs() + args.TxHashExtractor = nil + return args + }, + exErr: ErrNilTxHashExtractor, + }, } for _, tt := range tests { diff --git a/process/elasticproc/transactions/errors.go b/process/elasticproc/transactions/errors.go new file mode 100644 index 00000000..cf369bca --- /dev/null +++ b/process/elasticproc/transactions/errors.go @@ -0,0 +1,8 @@ +package transactions + +import ( + "errors" +) + +// ErrNilTxHashExtractor signals that a nil tx hash extractor has been provided +var ErrNilTxHashExtractor = errors.New("nil tx hash extractor") diff --git a/process/elasticproc/transactions/interface.go b/process/elasticproc/transactions/interface.go index 4a201d14..6a32cd4d 100644 --- a/process/elasticproc/transactions/interface.go +++ b/process/elasticproc/transactions/interface.go @@ -1,6 +1,7 @@ package transactions import ( + coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/outport" datafield "github.com/multiversx/mx-chain-vm-common-go/parsers/dataField" ) @@ -13,3 +14,9 @@ type DataFieldParser interface { type feeInfoHandler interface { GetFeeInfo() *outport.FeeInfo } + +// TxHashExtractor defines what tx hash extractor should be able to do +type TxHashExtractor interface { + ExtractExecutedTxHashes(mbIndex int, mbTxHashes [][]byte, header coreData.HeaderHandler) [][]byte + IsInterfaceNil() bool +} diff --git a/process/elasticproc/transactions/sovereignTxHashExtractor.go b/process/elasticproc/transactions/sovereignTxHashExtractor.go new file mode 100644 index 00000000..5fc6e3da --- /dev/null +++ b/process/elasticproc/transactions/sovereignTxHashExtractor.go @@ -0,0 +1,22 @@ +package transactions + +import ( + coreData "github.com/multiversx/mx-chain-core-go/data" +) + +type sovereignTxHashExtractor struct{} + +// NewSovereignTxHashExtractor created a new sovereign tx hash extractor +func NewSovereignTxHashExtractor() *sovereignTxHashExtractor { + return &sovereignTxHashExtractor{} +} + +// ExtractExecutedTxHashes returns executed tx hashes +func (the *sovereignTxHashExtractor) ExtractExecutedTxHashes(_ int, mbTxHashes [][]byte, _ coreData.HeaderHandler) [][]byte { + return mbTxHashes +} + +// IsInterfaceNil returns true if there is no value under the interface +func (the *sovereignTxHashExtractor) IsInterfaceNil() bool { + return the == nil +} diff --git a/process/elasticproc/transactions/sovereignTxHashExtractor_test.go b/process/elasticproc/transactions/sovereignTxHashExtractor_test.go new file mode 100644 index 00000000..1332c50c --- /dev/null +++ b/process/elasticproc/transactions/sovereignTxHashExtractor_test.go @@ -0,0 +1,23 @@ +package transactions + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNewSovereignTxHashExtractor(t *testing.T) { + t.Parallel() + + sthe := NewSovereignTxHashExtractor() + require.False(t, sthe.IsInterfaceNil()) +} + +func TestSovereignTxHashExtractor_ExtractExecutedTxHashes(t *testing.T) { + t.Parallel() + + sthe := NewSovereignTxHashExtractor() + mbTxHashes := [][]byte{[]byte("hash1"), []byte("hash2")} + txHashes := sthe.ExtractExecutedTxHashes(0, mbTxHashes, nil) + require.Equal(t, mbTxHashes, txHashes) +} diff --git a/process/elasticproc/transactions/transactionsGrouper.go b/process/elasticproc/transactions/transactionsGrouper.go index 5aaf15a4..69de99b8 100644 --- a/process/elasticproc/transactions/transactionsGrouper.go +++ b/process/elasticproc/transactions/transactionsGrouper.go @@ -11,6 +11,7 @@ import ( "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/multiversx/mx-chain-es-indexer-go/data" ) @@ -19,20 +20,23 @@ const ( ) type txsGrouper struct { - txBuilder *dbTransactionBuilder - hasher hashing.Hasher - marshalizer marshal.Marshalizer + txBuilder *dbTransactionBuilder + hasher hashing.Hasher + marshalizer marshal.Marshalizer + txHashExtractor TxHashExtractor } func newTxsGrouper( txBuilder *dbTransactionBuilder, hasher hashing.Hasher, marshalizer marshal.Marshalizer, + txHashExtractor TxHashExtractor, ) *txsGrouper { return &txsGrouper{ - txBuilder: txBuilder, - hasher: hasher, - marshalizer: marshalizer, + txBuilder: txBuilder, + hasher: hasher, + marshalizer: marshalizer, + txHashExtractor: txHashExtractor, } } @@ -52,7 +56,7 @@ func (tg *txsGrouper) groupNormalTxs( } selfShardID := header.GetShardID() - executedTxHashes := extractExecutedTxHashes(mbIndex, mb.TxHashes, header) + executedTxHashes := tg.txHashExtractor.ExtractExecutedTxHashes(mbIndex, mb.TxHashes, header) mbStatus := computeStatus(selfShardID, mb.ReceiverShardID) for _, txHash := range executedTxHashes { dbTx, ok := tg.prepareNormalTxForDB(mbHash, mb, mbStatus, txHash, txs, header, numOfShards) @@ -68,27 +72,6 @@ func (tg *txsGrouper) groupNormalTxs( return transactions, nil } -func extractExecutedTxHashes(mbIndex int, mbTxHashes [][]byte, header coreData.HeaderHandler) [][]byte { - miniblockHeaders := header.GetMiniBlockHeaderHandlers() - if len(miniblockHeaders) <= mbIndex { - return mbTxHashes - } - - firstProcessed := miniblockHeaders[mbIndex].GetIndexOfFirstTxProcessed() - lastProcessed := miniblockHeaders[mbIndex].GetIndexOfLastTxProcessed() - - executedTxHashes := make([][]byte, 0) - for txIndex, txHash := range mbTxHashes { - if int32(txIndex) < firstProcessed || int32(txIndex) > lastProcessed { - continue - } - - executedTxHashes = append(executedTxHashes, txHash) - } - - return executedTxHashes -} - func (tg *txsGrouper) prepareNormalTxForDB( mbHash []byte, mb *block.MiniBlock, @@ -123,7 +106,7 @@ func (tg *txsGrouper) groupRewardsTxs( selfShardID := header.GetShardID() mbStatus := computeStatus(selfShardID, mb.ReceiverShardID) - executedTxHashes := extractExecutedTxHashes(mbIndex, mb.TxHashes, header) + executedTxHashes := tg.txHashExtractor.ExtractExecutedTxHashes(mbIndex, mb.TxHashes, header) for _, txHash := range executedTxHashes { rewardDBTx, ok := tg.prepareRewardTxForDB(mbHash, mb, mbStatus, txHash, txs, header) if !ok { @@ -169,7 +152,7 @@ func (tg *txsGrouper) groupInvalidTxs( return nil, err } - executedTxHashes := extractExecutedTxHashes(mbIndex, mb.TxHashes, header) + executedTxHashes := tg.txHashExtractor.ExtractExecutedTxHashes(mbIndex, mb.TxHashes, header) for _, txHash := range executedTxHashes { invalidDBTx, ok := tg.prepareInvalidTxForDB(mbHash, mb, txHash, txs, header, numOfShards) if !ok { diff --git a/process/elasticproc/transactions/transactionsGrouper_test.go b/process/elasticproc/transactions/transactionsGrouper_test.go index 12c6b75e..addca8cd 100644 --- a/process/elasticproc/transactions/transactionsGrouper_test.go +++ b/process/elasticproc/transactions/transactionsGrouper_test.go @@ -4,14 +4,16 @@ import ( "encoding/hex" "testing" + coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/receipt" "github.com/multiversx/mx-chain-core-go/data/rewardTx" "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-es-indexer-go/mock" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" - "github.com/stretchr/testify/require" ) func TestGroupNormalTxs(t *testing.T) { @@ -20,7 +22,6 @@ func TestGroupNormalTxs(t *testing.T) { parser := createDataFieldParserMock() ap, _ := converters.NewBalanceConverter(18) txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, parser, ap) - grouper := newTxsGrouper(txBuilder, &mock.HasherMock{}, &mock.MarshalizerMock{}) txHash1 := []byte("txHash1") txHash2 := []byte("txHash2") @@ -46,6 +47,13 @@ func TestGroupNormalTxs(t *testing.T) { }, } + txHashExtractor := &mock.TxHashExtractorMock{ + ExtractExecutedTxHashesCalled: func(_ int, _ [][]byte, _ coreData.HeaderHandler) [][]byte { + return mb.TxHashes + }, + } + grouper := newTxsGrouper(txBuilder, &mock.HasherMock{}, &mock.MarshalizerMock{}, txHashExtractor) + normalTxs, _ := grouper.groupNormalTxs(0, mb, header, txs, false, 3) require.Len(t, normalTxs, 2) } @@ -56,7 +64,6 @@ func TestGroupRewardsTxs(t *testing.T) { parser := createDataFieldParserMock() ap, _ := converters.NewBalanceConverter(18) txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, parser, ap) - grouper := newTxsGrouper(txBuilder, &mock.HasherMock{}, &mock.MarshalizerMock{}) txHash1 := []byte("txHash1") txHash2 := []byte("txHash2") @@ -74,6 +81,13 @@ func TestGroupRewardsTxs(t *testing.T) { }}, } + txHashExtractor := &mock.TxHashExtractorMock{ + ExtractExecutedTxHashesCalled: func(_ int, _ [][]byte, _ coreData.HeaderHandler) [][]byte { + return mb.TxHashes + }, + } + grouper := newTxsGrouper(txBuilder, &mock.HasherMock{}, &mock.MarshalizerMock{}, txHashExtractor) + normalTxs, _ := grouper.groupRewardsTxs(0, mb, header, txs, false) require.Len(t, normalTxs, 2) } @@ -84,7 +98,6 @@ func TestGroupInvalidTxs(t *testing.T) { parser := createDataFieldParserMock() ap, _ := converters.NewBalanceConverter(18) txBuilder := newTransactionDBBuilder(mock.NewPubkeyConverterMock(32), parser, ap) - grouper := newTxsGrouper(txBuilder, &mock.HasherMock{}, &mock.MarshalizerMock{}) txHash1 := []byte("txHash1") txHash2 := []byte("txHash2") @@ -106,6 +119,13 @@ func TestGroupInvalidTxs(t *testing.T) { }, FeeInfo: &outport.FeeInfo{}}, } + txHashExtractor := &mock.TxHashExtractorMock{ + ExtractExecutedTxHashesCalled: func(_ int, _ [][]byte, _ coreData.HeaderHandler) [][]byte { + return mb.TxHashes + }, + } + grouper := newTxsGrouper(txBuilder, &mock.HasherMock{}, &mock.MarshalizerMock{}, txHashExtractor) + normalTxs, _ := grouper.groupInvalidTxs(0, mb, header, txs, 3) require.Len(t, normalTxs, 2) } @@ -116,7 +136,7 @@ func TestGroupReceipts(t *testing.T) { parser := createDataFieldParserMock() ap, _ := converters.NewBalanceConverter(18) txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, parser, ap) - grouper := newTxsGrouper(txBuilder, &mock.HasherMock{}, &mock.MarshalizerMock{}) + grouper := newTxsGrouper(txBuilder, &mock.HasherMock{}, &mock.MarshalizerMock{}, &mock.TxHashExtractorMock{}) txHash1 := []byte("txHash1") txHash2 := []byte("txHash2") diff --git a/process/elasticproc/transactions/transactionsProcessor.go b/process/elasticproc/transactions/transactionsProcessor.go index 86e59b38..37e8c589 100644 --- a/process/elasticproc/transactions/transactionsProcessor.go +++ b/process/elasticproc/transactions/transactionsProcessor.go @@ -11,10 +11,11 @@ import ( "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" - "github.com/multiversx/mx-chain-es-indexer-go/data" - "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" logger "github.com/multiversx/mx-chain-logger-go" datafield "github.com/multiversx/mx-chain-vm-common-go/parsers/dataField" + + "github.com/multiversx/mx-chain-es-indexer-go/data" + "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" ) var log = logger.GetOrCreate("indexer/process/transactions") @@ -26,6 +27,7 @@ type ArgsTransactionProcessor struct { Hasher hashing.Hasher Marshalizer marshal.Marshalizer BalanceConverter dataindexer.BalanceConverter + TxHashExtractor TxHashExtractor } type txsDatabaseProcessor struct { @@ -52,7 +54,7 @@ func NewTransactionsProcessor(args *ArgsTransactionProcessor) (*txsDatabaseProce } txBuilder := newTransactionDBBuilder(args.AddressPubkeyConverter, operationsDataParser, args.BalanceConverter) - txsDBGrouper := newTxsGrouper(txBuilder, args.Hasher, args.Marshalizer) + txsDBGrouper := newTxsGrouper(txBuilder, args.Hasher, args.Marshalizer, args.TxHashExtractor) scrProc := newSmartContractResultsProcessor(args.AddressPubkeyConverter, args.Marshalizer, args.Hasher, operationsDataParser, args.BalanceConverter) scrsDataToTxs := newScrsDataToTransactions(args.BalanceConverter) diff --git a/process/elasticproc/transactions/transactionsProcessor_test.go b/process/elasticproc/transactions/transactionsProcessor_test.go index 1348e427..99a39b5e 100644 --- a/process/elasticproc/transactions/transactionsProcessor_test.go +++ b/process/elasticproc/transactions/transactionsProcessor_test.go @@ -13,11 +13,12 @@ import ( "github.com/multiversx/mx-chain-core-go/data/rewardTx" "github.com/multiversx/mx-chain-core-go/data/smartContractResult" "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/mock" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func createMockArgsTxsDBProc() *ArgsTransactionProcessor { @@ -27,6 +28,7 @@ func createMockArgsTxsDBProc() *ArgsTransactionProcessor { Hasher: &mock.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, BalanceConverter: ap, + TxHashExtractor: &mock.TxHashExtractorMock{}, } return args } diff --git a/process/elasticproc/transactions/txHashExtractor.go b/process/elasticproc/transactions/txHashExtractor.go new file mode 100644 index 00000000..c532e069 --- /dev/null +++ b/process/elasticproc/transactions/txHashExtractor.go @@ -0,0 +1,39 @@ +package transactions + +import ( + coreData "github.com/multiversx/mx-chain-core-go/data" +) + +type txHashExtractor struct{} + +// NewTxHashExtractor created a new tx hash extractor +func NewTxHashExtractor() *txHashExtractor { + return &txHashExtractor{} +} + +// ExtractExecutedTxHashes returns executed tx hashes +func (the *txHashExtractor) ExtractExecutedTxHashes(mbIndex int, mbTxHashes [][]byte, header coreData.HeaderHandler) [][]byte { + miniblockHeaders := header.GetMiniBlockHeaderHandlers() + if len(miniblockHeaders) <= mbIndex { + return mbTxHashes + } + + firstProcessed := miniblockHeaders[mbIndex].GetIndexOfFirstTxProcessed() + lastProcessed := miniblockHeaders[mbIndex].GetIndexOfLastTxProcessed() + + executedTxHashes := make([][]byte, 0) + for txIndex, txHash := range mbTxHashes { + if int32(txIndex) < firstProcessed || int32(txIndex) > lastProcessed { + continue + } + + executedTxHashes = append(executedTxHashes, txHash) + } + + return executedTxHashes +} + +// IsInterfaceNil returns true if there is no value under the interface +func (the *txHashExtractor) IsInterfaceNil() bool { + return the == nil +} diff --git a/process/elasticproc/transactions/txHashExtractor_test.go b/process/elasticproc/transactions/txHashExtractor_test.go new file mode 100644 index 00000000..9f56def5 --- /dev/null +++ b/process/elasticproc/transactions/txHashExtractor_test.go @@ -0,0 +1,24 @@ +package transactions + +import ( + "testing" + + "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/stretchr/testify/require" +) + +func TestNewTxHashExtractor(t *testing.T) { + t.Parallel() + + the := NewTxHashExtractor() + require.False(t, the.IsInterfaceNil()) +} + +func TestTxHashExtractor_ExtractExecutedTxHashes(t *testing.T) { + t.Parallel() + + the := NewTxHashExtractor() + mbTxHashes := [][]byte{[]byte("hash1"), []byte("hash2")} + txHashes := the.ExtractExecutedTxHashes(0, mbTxHashes, &block.Header{}) + require.Equal(t, mbTxHashes, txHashes) +} diff --git a/process/factory/indexerFactory.go b/process/factory/indexerFactory.go index 027dc651..5145ce4d 100644 --- a/process/factory/indexerFactory.go +++ b/process/factory/indexerFactory.go @@ -124,6 +124,7 @@ func createElasticProcessor(args ArgsIndexerFactory) (dataindexer.ElasticProcess BulkRequestMaxSize: args.BulkRequestMaxSize, ImportDB: args.ImportDB, Version: args.Version, + TxHashExtractor: args.RunTypeComponents.TxHashExtractorCreator(), } return factory.CreateElasticProcessor(argsElasticProcFac) From d4734cab23299701c50785817a47bde333b2c78b Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Thu, 5 Dec 2024 14:28:49 +0200 Subject: [PATCH 15/50] unit test fixes --- process/elasticproc/elasticProcessor_test.go | 4 +++- .../elasticproc/transactions/transactionsProcessor_test.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/process/elasticproc/elasticProcessor_test.go b/process/elasticproc/elasticProcessor_test.go index 9f3311af..706f8915 100644 --- a/process/elasticproc/elasticProcessor_test.go +++ b/process/elasticproc/elasticProcessor_test.go @@ -12,6 +12,8 @@ import ( dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/mock" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" @@ -25,7 +27,6 @@ import ( "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tags" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/validators" - "github.com/stretchr/testify/require" ) func newElasticsearchProcessor(elasticsearchWriter DatabaseClientHandler, arguments *ArgElasticProcessor) *elasticProcessor { @@ -354,6 +355,7 @@ func TestElasticseachSaveTransactions(t *testing.T) { Hasher: &mock.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, BalanceConverter: bc, + TxHashExtractor: transactions.NewTxHashExtractor(), } txDbProc, _ := transactions.NewTransactionsProcessor(args) arguments.TransactionsProc = txDbProc diff --git a/process/elasticproc/transactions/transactionsProcessor_test.go b/process/elasticproc/transactions/transactionsProcessor_test.go index 99a39b5e..a393db04 100644 --- a/process/elasticproc/transactions/transactionsProcessor_test.go +++ b/process/elasticproc/transactions/transactionsProcessor_test.go @@ -28,7 +28,7 @@ func createMockArgsTxsDBProc() *ArgsTransactionProcessor { Hasher: &mock.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, BalanceConverter: ap, - TxHashExtractor: &mock.TxHashExtractorMock{}, + TxHashExtractor: NewTxHashExtractor(), } return args } From da7a4224c04b5756a73565ceb68c55de4a479ad7 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Thu, 5 Dec 2024 14:53:28 +0200 Subject: [PATCH 16/50] reward tx data in run type components --- factory/runType/interface.go | 1 + factory/runType/runTypeComponents.go | 1 + factory/runType/runTypeComponentsFactory.go | 1 + factory/runType/runTypeComponentsHandler.go | 15 ++++++++++++ .../runType/runTypeComponentsHandler_test.go | 2 ++ .../sovereignRunTypeComponentsFactory.go | 1 + process/elasticproc/transactions/errors.go | 3 +++ process/elasticproc/transactions/interface.go | 6 +++++ .../elasticproc/transactions/rewardTxData.go | 24 +++++++++++++++++++ .../transactions/sovereignRewardTxData.go | 24 +++++++++++++++++++ .../transactions/transactionDBBuilder.go | 1 + 11 files changed, 79 insertions(+) create mode 100644 process/elasticproc/transactions/rewardTxData.go create mode 100644 process/elasticproc/transactions/sovereignRewardTxData.go diff --git a/factory/runType/interface.go b/factory/runType/interface.go index 1c8cd4d5..01e7d959 100644 --- a/factory/runType/interface.go +++ b/factory/runType/interface.go @@ -27,6 +27,7 @@ type RunTypeComponentsHandler interface { // RunTypeComponentsHolder holds the run type components type RunTypeComponentsHolder interface { TxHashExtractorCreator() transactions.TxHashExtractor + RewardTxDataCreator() transactions.RewardTxData Create() error Close() error CheckSubcomponents() error diff --git a/factory/runType/runTypeComponents.go b/factory/runType/runTypeComponents.go index 74249f1d..e7f01275 100644 --- a/factory/runType/runTypeComponents.go +++ b/factory/runType/runTypeComponents.go @@ -6,6 +6,7 @@ import ( type runTypeComponents struct { txHashExtractor transactions.TxHashExtractor + rewardTxData transactions.RewardTxData } // Close does nothing diff --git a/factory/runType/runTypeComponentsFactory.go b/factory/runType/runTypeComponentsFactory.go index e9d6b88d..8ec94df3 100644 --- a/factory/runType/runTypeComponentsFactory.go +++ b/factory/runType/runTypeComponentsFactory.go @@ -15,6 +15,7 @@ func NewRunTypeComponentsFactory() *runTypeComponentsFactory { func (rtcf *runTypeComponentsFactory) Create() *runTypeComponents { return &runTypeComponents{ txHashExtractor: transactions.NewTxHashExtractor(), + rewardTxData: transactions.NewRewardTxData(), } } diff --git a/factory/runType/runTypeComponentsHandler.go b/factory/runType/runTypeComponentsHandler.go index 0d4e8330..a8cc62f7 100644 --- a/factory/runType/runTypeComponentsHandler.go +++ b/factory/runType/runTypeComponentsHandler.go @@ -72,6 +72,9 @@ func (mrtc *managedRunTypeComponents) CheckSubcomponents() error { if check.IfNil(mrtc.txHashExtractor) { return transactions.ErrNilTxHashExtractor } + if check.IfNil(mrtc.rewardTxData) { + return transactions.ErrNilRewardTxData + } return nil } @@ -87,6 +90,18 @@ func (mrtc *managedRunTypeComponents) TxHashExtractorCreator() transactions.TxHa return mrtc.runTypeComponents.txHashExtractor } +// RewardTxDataCreator return reward tx handler +func (mrtc *managedRunTypeComponents) RewardTxDataCreator() transactions.RewardTxData { + mrtc.mutRunTypeCoreComponents.Lock() + defer mrtc.mutRunTypeCoreComponents.Unlock() + + if check.IfNil(mrtc.runTypeComponents) { + return nil + } + + return mrtc.runTypeComponents.rewardTxData +} + // IsInterfaceNil returns true if the interface is nil func (mrtc *managedRunTypeComponents) IsInterfaceNil() bool { return mrtc == nil diff --git a/factory/runType/runTypeComponentsHandler_test.go b/factory/runType/runTypeComponentsHandler_test.go index 7c5ac0f4..e6756a6c 100644 --- a/factory/runType/runTypeComponentsHandler_test.go +++ b/factory/runType/runTypeComponentsHandler_test.go @@ -37,11 +37,13 @@ func TestManagedRunTypeComponents_Create(t *testing.T) { require.NoError(t, err) require.Nil(t, managedRunTypeComponents.TxHashExtractorCreator()) + require.Nil(t, managedRunTypeComponents.RewardTxDataCreator()) err = managedRunTypeComponents.Create() require.NoError(t, err) require.NotNil(t, managedRunTypeComponents.TxHashExtractorCreator()) + require.NotNil(t, managedRunTypeComponents.RewardTxDataCreator()) require.Equal(t, runTypeComponentsName, managedRunTypeComponents.String()) require.NoError(t, managedRunTypeComponents.Close()) diff --git a/factory/runType/sovereignRunTypeComponentsFactory.go b/factory/runType/sovereignRunTypeComponentsFactory.go index 5ddc099b..874a1b29 100644 --- a/factory/runType/sovereignRunTypeComponentsFactory.go +++ b/factory/runType/sovereignRunTypeComponentsFactory.go @@ -15,6 +15,7 @@ func NewSovereignRunTypeComponentsFactory() *sovereignRunTypeComponentsFactory { func (srtcf *sovereignRunTypeComponentsFactory) Create() *runTypeComponents { return &runTypeComponents{ txHashExtractor: transactions.NewSovereignTxHashExtractor(), + rewardTxData: transactions.NewSovereignRewardTxData(), } } diff --git a/process/elasticproc/transactions/errors.go b/process/elasticproc/transactions/errors.go index cf369bca..1a130d0d 100644 --- a/process/elasticproc/transactions/errors.go +++ b/process/elasticproc/transactions/errors.go @@ -6,3 +6,6 @@ import ( // ErrNilTxHashExtractor signals that a nil tx hash extractor has been provided var ErrNilTxHashExtractor = errors.New("nil tx hash extractor") + +// ErrNilRewardTxData signals that a nil rewards tx data handler has been provided +var ErrNilRewardTxData = errors.New("nil reward tx data handler") diff --git a/process/elasticproc/transactions/interface.go b/process/elasticproc/transactions/interface.go index 6a32cd4d..f143596b 100644 --- a/process/elasticproc/transactions/interface.go +++ b/process/elasticproc/transactions/interface.go @@ -20,3 +20,9 @@ type TxHashExtractor interface { ExtractExecutedTxHashes(mbIndex int, mbTxHashes [][]byte, header coreData.HeaderHandler) [][]byte IsInterfaceNil() bool } + +// RewardTxData defines what rewards tx handler should be able to do +type RewardTxData interface { + GetSender() string + IsInterfaceNil() bool +} diff --git a/process/elasticproc/transactions/rewardTxData.go b/process/elasticproc/transactions/rewardTxData.go new file mode 100644 index 00000000..15961b8f --- /dev/null +++ b/process/elasticproc/transactions/rewardTxData.go @@ -0,0 +1,24 @@ +package transactions + +import ( + "fmt" + + "github.com/multiversx/mx-chain-core-go/core" +) + +type rewardTxData struct{} + +// NewRewardTxData creates a new reward tx data +func NewRewardTxData() *rewardTxData { + return &rewardTxData{} +} + +// GetSender return the shard id as string +func (rtd *rewardTxData) GetSender() string { + return fmt.Sprintf("%d", core.MetachainShardId) +} + +// IsInterfaceNil returns true if there is no value under the interface +func (rtd *rewardTxData) IsInterfaceNil() bool { + return rtd == nil +} diff --git a/process/elasticproc/transactions/sovereignRewardTxData.go b/process/elasticproc/transactions/sovereignRewardTxData.go new file mode 100644 index 00000000..749c28e3 --- /dev/null +++ b/process/elasticproc/transactions/sovereignRewardTxData.go @@ -0,0 +1,24 @@ +package transactions + +import ( + "fmt" + + "github.com/multiversx/mx-chain-core-go/core" +) + +type sovereignRewardTxData struct{} + +// NewSovereignRewardTxData creates a new sovereign reward tx data +func NewSovereignRewardTxData() *sovereignRewardTxData { + return &sovereignRewardTxData{} +} + +// GetSender return the shard id as string +func (srtd *sovereignRewardTxData) GetSender() string { + return fmt.Sprintf("%d", core.SovereignChainShardId) +} + +// IsInterfaceNil returns true if there is no value under the interface +func (srtd *sovereignRewardTxData) IsInterfaceNil() bool { + return srtd == nil +} diff --git a/process/elasticproc/transactions/transactionDBBuilder.go b/process/elasticproc/transactions/transactionDBBuilder.go index c22fd07d..9900abe3 100644 --- a/process/elasticproc/transactions/transactionDBBuilder.go +++ b/process/elasticproc/transactions/transactionDBBuilder.go @@ -11,6 +11,7 @@ import ( "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/receipt" + "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" From c1c08ba97179c86f84912c48ff8430fc263d314d Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Thu, 5 Dec 2024 15:21:35 +0200 Subject: [PATCH 17/50] reward tx data in run type components fixes --- mock/rewardTxDataMock.go | 20 +++++++++++++++++++ process/elasticproc/elasticProcessor_test.go | 1 + .../factory/elasticProcessorFactory.go | 2 ++ .../factory/elasticProcessorFactory_test.go | 1 + process/elasticproc/transactions/checkers.go | 3 +++ .../elasticproc/transactions/checkers_test.go | 10 ++++++++++ .../transactions/transactionDBBuilder.go | 6 ++++-- .../transactions/transactionDBBuilder_test.go | 14 +++++++++---- .../transactions/transactionsGrouper_test.go | 8 ++++---- .../transactions/transactionsProcessor.go | 3 ++- .../transactionsProcessor_test.go | 1 + process/factory/indexerFactory.go | 1 + 12 files changed, 59 insertions(+), 11 deletions(-) create mode 100644 mock/rewardTxDataMock.go diff --git a/mock/rewardTxDataMock.go b/mock/rewardTxDataMock.go new file mode 100644 index 00000000..9eee7370 --- /dev/null +++ b/mock/rewardTxDataMock.go @@ -0,0 +1,20 @@ +package mock + +// RewardTxDataMock - +type RewardTxDataMock struct { + GetSenderCalled func() string +} + +// GetSender - +func (rtd *RewardTxDataMock) GetSender() string { + if rtd.GetSenderCalled != nil { + return rtd.GetSenderCalled() + } + + return "" +} + +// IsInterfaceNil returns true if there is no value under the interface +func (rtd *RewardTxDataMock) IsInterfaceNil() bool { + return rtd == nil +} diff --git a/process/elasticproc/elasticProcessor_test.go b/process/elasticproc/elasticProcessor_test.go index 706f8915..b01180ac 100644 --- a/process/elasticproc/elasticProcessor_test.go +++ b/process/elasticproc/elasticProcessor_test.go @@ -356,6 +356,7 @@ func TestElasticseachSaveTransactions(t *testing.T) { Marshalizer: &mock.MarshalizerMock{}, BalanceConverter: bc, TxHashExtractor: transactions.NewTxHashExtractor(), + RewardTxData: &mock.RewardTxDataMock{}, } txDbProc, _ := transactions.NewTransactionsProcessor(args) arguments.TransactionsProc = txDbProc diff --git a/process/elasticproc/factory/elasticProcessorFactory.go b/process/elasticproc/factory/elasticProcessorFactory.go index a61285d4..89d3f46e 100644 --- a/process/elasticproc/factory/elasticProcessorFactory.go +++ b/process/elasticproc/factory/elasticProcessorFactory.go @@ -33,6 +33,7 @@ type ArgElasticProcessorFactory struct { UseKibana bool ImportDB bool TxHashExtractor transactions.TxHashExtractor + RewardTxData transactions.RewardTxData } // CreateElasticProcessor will create a new instance of ElasticProcessor @@ -90,6 +91,7 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.E Marshalizer: arguments.Marshalizer, BalanceConverter: balanceConverter, TxHashExtractor: arguments.TxHashExtractor, + RewardTxData: arguments.RewardTxData, } txsProc, err := transactions.NewTransactionsProcessor(argsTxsProc) if err != nil { diff --git a/process/elasticproc/factory/elasticProcessorFactory_test.go b/process/elasticproc/factory/elasticProcessorFactory_test.go index 936ae352..2e4dd081 100644 --- a/process/elasticproc/factory/elasticProcessorFactory_test.go +++ b/process/elasticproc/factory/elasticProcessorFactory_test.go @@ -20,6 +20,7 @@ func TestCreateElasticProcessor(t *testing.T) { Denomination: 1, UseKibana: false, TxHashExtractor: &mock.TxHashExtractorMock{}, + RewardTxData: &mock.RewardTxDataMock{}, } ep, err := CreateElasticProcessor(args) diff --git a/process/elasticproc/transactions/checkers.go b/process/elasticproc/transactions/checkers.go index 282c66f5..3a8f768f 100644 --- a/process/elasticproc/transactions/checkers.go +++ b/process/elasticproc/transactions/checkers.go @@ -39,6 +39,9 @@ func checkTxsProcessorArg(args *ArgsTransactionProcessor) error { if check.IfNil(args.TxHashExtractor) { return ErrNilTxHashExtractor } + if check.IfNil(args.RewardTxData) { + return ErrNilRewardTxData + } return nil } diff --git a/process/elasticproc/transactions/checkers_test.go b/process/elasticproc/transactions/checkers_test.go index 20d3f9db..1c928eda 100644 --- a/process/elasticproc/transactions/checkers_test.go +++ b/process/elasticproc/transactions/checkers_test.go @@ -26,6 +26,7 @@ func createMockArgs() *ArgsTransactionProcessor { Marshalizer: &mock.MarshalizerMock{}, BalanceConverter: bc, TxHashExtractor: &mock.TxHashExtractorMock{}, + RewardTxData: &mock.RewardTxDataMock{}, } } @@ -82,6 +83,15 @@ func TestNewTransactionsProcessor(t *testing.T) { }, exErr: ErrNilTxHashExtractor, }, + { + name: "NilTxHashExtractor", + args: func() *ArgsTransactionProcessor { + args := createMockArgs() + args.RewardTxData = nil + return args + }, + exErr: ErrNilRewardTxData, + }, } for _, tt := range tests { diff --git a/process/elasticproc/transactions/transactionDBBuilder.go b/process/elasticproc/transactions/transactionDBBuilder.go index 9900abe3..fbb154a1 100644 --- a/process/elasticproc/transactions/transactionDBBuilder.go +++ b/process/elasticproc/transactions/transactionDBBuilder.go @@ -2,7 +2,6 @@ package transactions import ( "encoding/hex" - "fmt" "time" "github.com/multiversx/mx-chain-core-go/core" @@ -21,17 +20,20 @@ type dbTransactionBuilder struct { addressPubkeyConverter core.PubkeyConverter dataFieldParser DataFieldParser balanceConverter dataindexer.BalanceConverter + rewardTxData RewardTxData } func newTransactionDBBuilder( addressPubkeyConverter core.PubkeyConverter, dataFieldParser DataFieldParser, balanceConverter dataindexer.BalanceConverter, + rewardTxData RewardTxData, ) *dbTransactionBuilder { return &dbTransactionBuilder{ addressPubkeyConverter: addressPubkeyConverter, dataFieldParser: dataFieldParser, balanceConverter: balanceConverter, + rewardTxData: rewardTxData, } } @@ -155,7 +157,7 @@ func (dtb *dbTransactionBuilder) prepareRewardTransaction( Value: rTx.Value.String(), ValueNum: valueNum, Receiver: receiverAddr, - Sender: fmt.Sprintf("%d", core.MetachainShardId), + Sender: dtb.rewardTxData.GetSender(), ReceiverShard: mb.ReceiverShardID, SenderShard: mb.SenderShardID, GasPrice: 0, diff --git a/process/elasticproc/transactions/transactionDBBuilder_test.go b/process/elasticproc/transactions/transactionDBBuilder_test.go index ceeca3de..5b2c4cdc 100644 --- a/process/elasticproc/transactions/transactionDBBuilder_test.go +++ b/process/elasticproc/transactions/transactionDBBuilder_test.go @@ -2,20 +2,19 @@ package transactions import ( "encoding/hex" - "fmt" "math/big" "testing" "time" - "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/rewardTx" "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/mock" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" - "github.com/stretchr/testify/require" ) func createCommonProcessor() dbTransactionBuilder { @@ -24,6 +23,7 @@ func createCommonProcessor() dbTransactionBuilder { addressPubkeyConverter: mock.NewPubkeyConverterMock(32), dataFieldParser: createDataFieldParserMock(), balanceConverter: ap, + rewardTxData: &mock.RewardTxDataMock{}, } } @@ -105,7 +105,13 @@ func TestGetMoveBalanceTransaction(t *testing.T) { func TestGetTransactionByType_RewardTx(t *testing.T) { t.Parallel() + sender := "sender" cp := createCommonProcessor() + cp.rewardTxData = &mock.RewardTxDataMock{ + GetSenderCalled: func() string { + return sender + }, + } round := uint64(10) rcvAddr := []byte("receiver") @@ -127,7 +133,7 @@ func TestGetTransactionByType_RewardTx(t *testing.T) { Receiver: hex.EncodeToString(rcvAddr), Status: status, Value: "", - Sender: fmt.Sprintf("%d", core.MetachainShardId), + Sender: sender, Data: make([]byte, 0), Operation: rewardsOperation, } diff --git a/process/elasticproc/transactions/transactionsGrouper_test.go b/process/elasticproc/transactions/transactionsGrouper_test.go index addca8cd..8712e1a0 100644 --- a/process/elasticproc/transactions/transactionsGrouper_test.go +++ b/process/elasticproc/transactions/transactionsGrouper_test.go @@ -21,7 +21,7 @@ func TestGroupNormalTxs(t *testing.T) { parser := createDataFieldParserMock() ap, _ := converters.NewBalanceConverter(18) - txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, parser, ap) + txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, parser, ap, &mock.RewardTxDataMock{}) txHash1 := []byte("txHash1") txHash2 := []byte("txHash2") @@ -63,7 +63,7 @@ func TestGroupRewardsTxs(t *testing.T) { parser := createDataFieldParserMock() ap, _ := converters.NewBalanceConverter(18) - txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, parser, ap) + txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, parser, ap, &mock.RewardTxDataMock{}) txHash1 := []byte("txHash1") txHash2 := []byte("txHash2") @@ -97,7 +97,7 @@ func TestGroupInvalidTxs(t *testing.T) { parser := createDataFieldParserMock() ap, _ := converters.NewBalanceConverter(18) - txBuilder := newTransactionDBBuilder(mock.NewPubkeyConverterMock(32), parser, ap) + txBuilder := newTransactionDBBuilder(mock.NewPubkeyConverterMock(32), parser, ap, &mock.RewardTxDataMock{}) txHash1 := []byte("txHash1") txHash2 := []byte("txHash2") @@ -135,7 +135,7 @@ func TestGroupReceipts(t *testing.T) { parser := createDataFieldParserMock() ap, _ := converters.NewBalanceConverter(18) - txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, parser, ap) + txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, parser, ap, &mock.RewardTxDataMock{}) grouper := newTxsGrouper(txBuilder, &mock.HasherMock{}, &mock.MarshalizerMock{}, &mock.TxHashExtractorMock{}) txHash1 := []byte("txHash1") diff --git a/process/elasticproc/transactions/transactionsProcessor.go b/process/elasticproc/transactions/transactionsProcessor.go index 37e8c589..3531852e 100644 --- a/process/elasticproc/transactions/transactionsProcessor.go +++ b/process/elasticproc/transactions/transactionsProcessor.go @@ -28,6 +28,7 @@ type ArgsTransactionProcessor struct { Marshalizer marshal.Marshalizer BalanceConverter dataindexer.BalanceConverter TxHashExtractor TxHashExtractor + RewardTxData RewardTxData } type txsDatabaseProcessor struct { @@ -53,7 +54,7 @@ func NewTransactionsProcessor(args *ArgsTransactionProcessor) (*txsDatabaseProce return nil, err } - txBuilder := newTransactionDBBuilder(args.AddressPubkeyConverter, operationsDataParser, args.BalanceConverter) + txBuilder := newTransactionDBBuilder(args.AddressPubkeyConverter, operationsDataParser, args.BalanceConverter, args.RewardTxData) txsDBGrouper := newTxsGrouper(txBuilder, args.Hasher, args.Marshalizer, args.TxHashExtractor) scrProc := newSmartContractResultsProcessor(args.AddressPubkeyConverter, args.Marshalizer, args.Hasher, operationsDataParser, args.BalanceConverter) scrsDataToTxs := newScrsDataToTransactions(args.BalanceConverter) diff --git a/process/elasticproc/transactions/transactionsProcessor_test.go b/process/elasticproc/transactions/transactionsProcessor_test.go index a393db04..8d59c9c7 100644 --- a/process/elasticproc/transactions/transactionsProcessor_test.go +++ b/process/elasticproc/transactions/transactionsProcessor_test.go @@ -29,6 +29,7 @@ func createMockArgsTxsDBProc() *ArgsTransactionProcessor { Marshalizer: &mock.MarshalizerMock{}, BalanceConverter: ap, TxHashExtractor: NewTxHashExtractor(), + RewardTxData: NewRewardTxData(), } return args } diff --git a/process/factory/indexerFactory.go b/process/factory/indexerFactory.go index 5145ce4d..4a09b84d 100644 --- a/process/factory/indexerFactory.go +++ b/process/factory/indexerFactory.go @@ -125,6 +125,7 @@ func createElasticProcessor(args ArgsIndexerFactory) (dataindexer.ElasticProcess ImportDB: args.ImportDB, Version: args.Version, TxHashExtractor: args.RunTypeComponents.TxHashExtractorCreator(), + RewardTxData: args.RunTypeComponents.RewardTxDataCreator(), } return factory.CreateElasticProcessor(argsElasticProcFac) From 81a630f11eeb7ae795b17f5551e220579c06039d Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Thu, 5 Dec 2024 15:24:09 +0200 Subject: [PATCH 18/50] comments fixes --- process/elasticproc/transactions/sovereignTxHashExtractor.go | 2 +- process/elasticproc/transactions/txHashExtractor.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/process/elasticproc/transactions/sovereignTxHashExtractor.go b/process/elasticproc/transactions/sovereignTxHashExtractor.go index 5fc6e3da..c27bf163 100644 --- a/process/elasticproc/transactions/sovereignTxHashExtractor.go +++ b/process/elasticproc/transactions/sovereignTxHashExtractor.go @@ -6,7 +6,7 @@ import ( type sovereignTxHashExtractor struct{} -// NewSovereignTxHashExtractor created a new sovereign tx hash extractor +// NewSovereignTxHashExtractor creates a new sovereign tx hash extractor func NewSovereignTxHashExtractor() *sovereignTxHashExtractor { return &sovereignTxHashExtractor{} } diff --git a/process/elasticproc/transactions/txHashExtractor.go b/process/elasticproc/transactions/txHashExtractor.go index c532e069..1914be53 100644 --- a/process/elasticproc/transactions/txHashExtractor.go +++ b/process/elasticproc/transactions/txHashExtractor.go @@ -6,7 +6,7 @@ import ( type txHashExtractor struct{} -// NewTxHashExtractor created a new tx hash extractor +// NewTxHashExtractor creates a new tx hash extractor func NewTxHashExtractor() *txHashExtractor { return &txHashExtractor{} } From 6f0822127b4450b731fb737b57473e39524d9fe8 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Thu, 5 Dec 2024 15:28:35 +0200 Subject: [PATCH 19/50] fixes after self review --- process/elasticproc/transactions/transactionsProcessor_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/process/elasticproc/transactions/transactionsProcessor_test.go b/process/elasticproc/transactions/transactionsProcessor_test.go index 8d59c9c7..38c0c3b7 100644 --- a/process/elasticproc/transactions/transactionsProcessor_test.go +++ b/process/elasticproc/transactions/transactionsProcessor_test.go @@ -29,7 +29,7 @@ func createMockArgsTxsDBProc() *ArgsTransactionProcessor { Marshalizer: &mock.MarshalizerMock{}, BalanceConverter: ap, TxHashExtractor: NewTxHashExtractor(), - RewardTxData: NewRewardTxData(), + RewardTxData: &mock.RewardTxDataMock{}, } return args } From 948c8fc8376158a3e26511ef91e1e84d2fa972cf Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Thu, 5 Dec 2024 15:29:55 +0200 Subject: [PATCH 20/50] fixes after self review --- integrationtests/utils.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/integrationtests/utils.go b/integrationtests/utils.go index 5fe2aaf8..c28d028e 100644 --- a/integrationtests/utils.go +++ b/integrationtests/utils.go @@ -12,13 +12,15 @@ import ( "github.com/elastic/go-elasticsearch/v7" "github.com/multiversx/mx-chain-core-go/core/pubkeyConverter" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-es-indexer-go/client" "github.com/multiversx/mx-chain-es-indexer-go/client/logging" "github.com/multiversx/mx-chain-es-indexer-go/mock" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" - logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" ) var ( @@ -60,7 +62,8 @@ func CreateElasticProcessor( EnabledIndexes: []string{dataindexer.TransactionsIndex, dataindexer.LogsIndex, dataindexer.AccountsESDTIndex, dataindexer.ScResultsIndex, dataindexer.ReceiptsIndex, dataindexer.BlockIndex, dataindexer.AccountsIndex, dataindexer.TokensIndex, dataindexer.TagsIndex, dataindexer.EventsIndex, dataindexer.OperationsIndex, dataindexer.DelegatorsIndex, dataindexer.ESDTsIndex, dataindexer.SCDeploysIndex, dataindexer.MiniblocksIndex, dataindexer.ValuesIndex}, - Denomination: 18, + Denomination: 18, + TxHashExtractor: transactions.NewTxHashExtractor(), } return factory.CreateElasticProcessor(args) From d1394c47ccc93cd3882d58ceb86ed89df0b889c8 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Thu, 5 Dec 2024 15:30:48 +0200 Subject: [PATCH 21/50] integration tests fixes --- integrationtests/utils.go | 1 + 1 file changed, 1 insertion(+) diff --git a/integrationtests/utils.go b/integrationtests/utils.go index c28d028e..06239e0c 100644 --- a/integrationtests/utils.go +++ b/integrationtests/utils.go @@ -64,6 +64,7 @@ func CreateElasticProcessor( dataindexer.OperationsIndex, dataindexer.DelegatorsIndex, dataindexer.ESDTsIndex, dataindexer.SCDeploysIndex, dataindexer.MiniblocksIndex, dataindexer.ValuesIndex}, Denomination: 18, TxHashExtractor: transactions.NewTxHashExtractor(), + RewardTxData: transactions.NewRewardTxData(), } return factory.CreateElasticProcessor(args) From 7ee36158cc402a3f42a0061e632a707af0d8e25d Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Fri, 6 Dec 2024 12:17:28 +0200 Subject: [PATCH 22/50] fix after review --- factory/runType/sovereignRunTypeComponentsFactory_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/factory/runType/sovereignRunTypeComponentsFactory_test.go b/factory/runType/sovereignRunTypeComponentsFactory_test.go index a3e959b4..71d909ba 100644 --- a/factory/runType/sovereignRunTypeComponentsFactory_test.go +++ b/factory/runType/sovereignRunTypeComponentsFactory_test.go @@ -10,7 +10,7 @@ func TestSovereignRunTypeComponentsFactory_CreateAndClose(t *testing.T) { t.Parallel() srtcf := NewSovereignRunTypeComponentsFactory() - require.NotNil(t, srtcf) + require.False(t, srtcf.IsInterfaceNil()) srtc := srtcf.Create() require.NotNil(t, srtc) From 5c50ac41e9d43f4cc00c65b270212f618d9e21d5 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Fri, 6 Dec 2024 12:43:17 +0200 Subject: [PATCH 23/50] fixes after review, refactor --- cmd/elasticindexer/flags.go | 4 ++-- cmd/elasticindexer/main.go | 3 ++- config/config.go | 1 + factory/interface.go | 24 ------------------- factory/runType/interface.go | 23 ++++++++++++++++++ factory/runType/runTypeComponentsHandler.go | 8 +++---- .../runType/runTypeComponentsHandler_test.go | 4 +--- factory/wsIndexerFactory.go | 7 +++--- process/factory/indexerFactory.go | 4 ++-- 9 files changed, 37 insertions(+), 41 deletions(-) delete mode 100644 factory/interface.go diff --git a/cmd/elasticindexer/flags.go b/cmd/elasticindexer/flags.go index 084f3b3e..b09106c0 100644 --- a/cmd/elasticindexer/flags.go +++ b/cmd/elasticindexer/flags.go @@ -47,9 +47,9 @@ var ( Name: "disable-ansi-color", Usage: "Boolean option for disabling ANSI colors in the logging system.", } - // sovereign defines a flag that specifies if what run type components should use + // sovereign defines a flag that specifies if the es instance should run for a sovereign chain sovereign = cli.BoolFlag{ - Name: "sovereign-config", + Name: "sovereign", Usage: "If set to true, will use sovereign run type components", } ) diff --git a/cmd/elasticindexer/main.go b/cmd/elasticindexer/main.go index 4c3e9366..3cbf2657 100644 --- a/cmd/elasticindexer/main.go +++ b/cmd/elasticindexer/main.go @@ -88,6 +88,7 @@ func startIndexer(ctx *cli.Context) error { if err != nil { return fmt.Errorf("%w while loading the config file", err) } + cfg.SovereignType = ctx.GlobalBool(sovereign.Name) clusterCfg, err := loadClusterConfig(ctx.GlobalString(configurationPreferencesFile.Name)) if err != nil { @@ -100,7 +101,7 @@ func startIndexer(ctx *cli.Context) error { } statusMetrics := metrics.NewStatusMetrics() - wsHost, err := factory.CreateWsIndexer(cfg, clusterCfg, statusMetrics, ctx.App.Version, ctx.GlobalBool(sovereign.Name)) + wsHost, err := factory.CreateWsIndexer(cfg, clusterCfg, statusMetrics, ctx.App.Version) if err != nil { return fmt.Errorf("%w while creating the indexer", err) } diff --git a/config/config.go b/config/config.go index dc1ab2cb..2f3ef7be 100644 --- a/config/config.go +++ b/config/config.go @@ -29,6 +29,7 @@ type Config struct { LogsPath string `toml:"logs-path"` } `toml:"logs"` } `toml:"config"` + SovereignType bool } // ClusterConfig will hold the config for the Elasticsearch cluster diff --git a/factory/interface.go b/factory/interface.go deleted file mode 100644 index ae3435e6..00000000 --- a/factory/interface.go +++ /dev/null @@ -1,24 +0,0 @@ -package factory - -// ComponentHandler defines the actions common to all component handlers -type ComponentHandler interface { - Create() error - Close() error - CheckSubcomponents() error - String() string -} - -// RunTypeComponentsHandler defines the run type components handler actions -type RunTypeComponentsHandler interface { - ComponentHandler - RunTypeComponentsHolder -} - -// RunTypeComponentsHolder holds the run type components -type RunTypeComponentsHolder interface { - Create() error - Close() error - CheckSubcomponents() error - String() string - IsInterfaceNil() bool -} diff --git a/factory/runType/interface.go b/factory/runType/interface.go index 37448d92..44298c6f 100644 --- a/factory/runType/interface.go +++ b/factory/runType/interface.go @@ -5,3 +5,26 @@ type RunTypeComponentsCreator interface { Create() *runTypeComponents IsInterfaceNil() bool } + +// ComponentHandler defines the actions common to all component handlers +type ComponentHandler interface { + Create() error + Close() error + CheckSubcomponents() error + String() string +} + +// RunTypeComponentsHandler defines the run type components handler actions +type RunTypeComponentsHandler interface { + ComponentHandler + RunTypeComponentsHolder +} + +// RunTypeComponentsHolder holds the run type components +type RunTypeComponentsHolder interface { + Create() error + Close() error + CheckSubcomponents() error + String() string + IsInterfaceNil() bool +} diff --git a/factory/runType/runTypeComponentsHandler.go b/factory/runType/runTypeComponentsHandler.go index 9b25b92b..2421fd9d 100644 --- a/factory/runType/runTypeComponentsHandler.go +++ b/factory/runType/runTypeComponentsHandler.go @@ -4,15 +4,13 @@ import ( "sync" "github.com/multiversx/mx-chain-core-go/core/check" - - "github.com/multiversx/mx-chain-es-indexer-go/factory" ) const runTypeComponentsName = "managedRunTypeComponents" -var _ factory.ComponentHandler = (*managedRunTypeComponents)(nil) -var _ factory.RunTypeComponentsHandler = (*managedRunTypeComponents)(nil) -var _ factory.RunTypeComponentsHolder = (*managedRunTypeComponents)(nil) +var _ ComponentHandler = (*managedRunTypeComponents)(nil) +var _ RunTypeComponentsHandler = (*managedRunTypeComponents)(nil) +var _ RunTypeComponentsHolder = (*managedRunTypeComponents)(nil) type managedRunTypeComponents struct { *runTypeComponents diff --git a/factory/runType/runTypeComponentsHandler_test.go b/factory/runType/runTypeComponentsHandler_test.go index 573a73ff..b9b23b08 100644 --- a/factory/runType/runTypeComponentsHandler_test.go +++ b/factory/runType/runTypeComponentsHandler_test.go @@ -4,11 +4,9 @@ import ( "testing" "github.com/stretchr/testify/require" - - "github.com/multiversx/mx-chain-es-indexer-go/factory" ) -func createComponents() (factory.RunTypeComponentsHandler, error) { +func createComponents() (RunTypeComponentsHandler, error) { rtcf := NewRunTypeComponentsFactory() return NewManagedRunTypeComponents(rtcf) } diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index 5b41359c..96735a9c 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -18,13 +18,13 @@ import ( var log = logger.GetOrCreate("elasticindexer") // CreateWsIndexer will create a new instance of wsindexer.WSClient -func CreateWsIndexer(cfg config.Config, clusterCfg config.ClusterConfig, statusMetrics core.StatusMetricsHandler, version string, isSovereignType bool) (wsindexer.WSClient, error) { +func CreateWsIndexer(cfg config.Config, clusterCfg config.ClusterConfig, statusMetrics core.StatusMetricsHandler, version string) (wsindexer.WSClient, error) { wsMarshaller, err := factoryMarshaller.NewMarshalizer(clusterCfg.Config.WebSocket.DataMarshallerType) if err != nil { return nil, err } - dataIndexer, err := createDataIndexer(cfg, clusterCfg, wsMarshaller, statusMetrics, version, isSovereignType) + dataIndexer, err := createDataIndexer(cfg, clusterCfg, wsMarshaller, statusMetrics, version) if err != nil { return nil, err } @@ -58,7 +58,6 @@ func createDataIndexer( wsMarshaller marshal.Marshalizer, statusMetrics core.StatusMetricsHandler, version string, - isSovereignConfig bool, ) (wsindexer.DataIndexer, error) { marshaller, err := factoryMarshaller.NewMarshalizer(cfg.Config.Marshaller.Type) if err != nil { @@ -78,7 +77,7 @@ func createDataIndexer( } return factory.NewIndexer(factory.ArgsIndexerFactory{ - SovereignConfig: isSovereignConfig, + SovereignType: cfg.SovereignType, UseKibana: clusterCfg.Config.ElasticCluster.UseKibana, Denomination: cfg.Config.Economics.Denomination, BulkRequestMaxSize: clusterCfg.Config.ElasticCluster.BulkRequestMaxSizeInBytes, diff --git a/process/factory/indexerFactory.go b/process/factory/indexerFactory.go index 027dc651..911bf8f5 100644 --- a/process/factory/indexerFactory.go +++ b/process/factory/indexerFactory.go @@ -32,7 +32,7 @@ type ArgsIndexerFactory struct { Enabled bool UseKibana bool ImportDB bool - SovereignConfig bool + SovereignType bool Denomination int BulkRequestMaxSize int Url string @@ -57,7 +57,7 @@ func NewIndexer(args ArgsIndexerFactory) (dataindexer.Indexer, error) { return nil, err } - if args.SovereignConfig { + if args.SovereignType { args.RunTypeComponents, err = createManagedRunTypeComponents(runType.NewSovereignRunTypeComponentsFactory()) } else { args.RunTypeComponents, err = createManagedRunTypeComponents(runType.NewRunTypeComponentsFactory()) From a6f176c9ee2d2ebc11a06f85a68b36ef1e9df62a Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Fri, 6 Dec 2024 12:45:35 +0200 Subject: [PATCH 24/50] fixes after review --- factory/runType/runTypeComponentsHandler.go | 2 +- process/elasticproc/transactions/sovereignTxHashExtractor.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/factory/runType/runTypeComponentsHandler.go b/factory/runType/runTypeComponentsHandler.go index 0d4e8330..d766d677 100644 --- a/factory/runType/runTypeComponentsHandler.go +++ b/factory/runType/runTypeComponentsHandler.go @@ -75,7 +75,7 @@ func (mrtc *managedRunTypeComponents) CheckSubcomponents() error { return nil } -// TxHashExtractorCreator return tx hash extractor +// TxHashExtractorCreator returns tx hash extractor func (mrtc *managedRunTypeComponents) TxHashExtractorCreator() transactions.TxHashExtractor { mrtc.mutRunTypeCoreComponents.Lock() defer mrtc.mutRunTypeCoreComponents.Unlock() diff --git a/process/elasticproc/transactions/sovereignTxHashExtractor.go b/process/elasticproc/transactions/sovereignTxHashExtractor.go index c27bf163..aadbdd64 100644 --- a/process/elasticproc/transactions/sovereignTxHashExtractor.go +++ b/process/elasticproc/transactions/sovereignTxHashExtractor.go @@ -11,7 +11,7 @@ func NewSovereignTxHashExtractor() *sovereignTxHashExtractor { return &sovereignTxHashExtractor{} } -// ExtractExecutedTxHashes returns executed tx hashes +// ExtractExecutedTxHashes returns directly the provided mini block tx hashes func (the *sovereignTxHashExtractor) ExtractExecutedTxHashes(_ int, mbTxHashes [][]byte, _ coreData.HeaderHandler) [][]byte { return mbTxHashes } From 26330fcc63c3b928dc6bdcf0a28809f1befa516a Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Fri, 6 Dec 2024 12:49:22 +0200 Subject: [PATCH 25/50] fixes after review --- factory/runType/interface.go | 2 +- factory/runType/runTypeComponents.go | 2 +- factory/runType/runTypeComponentsHandler.go | 2 +- process/elasticproc/factory/elasticProcessorFactory.go | 2 +- process/elasticproc/transactions/interface.go | 4 ++-- process/elasticproc/transactions/rewardTxData.go | 2 +- process/elasticproc/transactions/sovereignRewardTxData.go | 2 +- process/elasticproc/transactions/transactionDBBuilder.go | 4 ++-- process/elasticproc/transactions/transactionsProcessor.go | 2 +- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/factory/runType/interface.go b/factory/runType/interface.go index 01e7d959..1114ff4d 100644 --- a/factory/runType/interface.go +++ b/factory/runType/interface.go @@ -27,7 +27,7 @@ type RunTypeComponentsHandler interface { // RunTypeComponentsHolder holds the run type components type RunTypeComponentsHolder interface { TxHashExtractorCreator() transactions.TxHashExtractor - RewardTxDataCreator() transactions.RewardTxData + RewardTxDataCreator() transactions.RewardTxDataHandler Create() error Close() error CheckSubcomponents() error diff --git a/factory/runType/runTypeComponents.go b/factory/runType/runTypeComponents.go index e7f01275..b70512b6 100644 --- a/factory/runType/runTypeComponents.go +++ b/factory/runType/runTypeComponents.go @@ -6,7 +6,7 @@ import ( type runTypeComponents struct { txHashExtractor transactions.TxHashExtractor - rewardTxData transactions.RewardTxData + rewardTxData transactions.RewardTxDataHandler } // Close does nothing diff --git a/factory/runType/runTypeComponentsHandler.go b/factory/runType/runTypeComponentsHandler.go index 26369fee..b514d1c5 100644 --- a/factory/runType/runTypeComponentsHandler.go +++ b/factory/runType/runTypeComponentsHandler.go @@ -91,7 +91,7 @@ func (mrtc *managedRunTypeComponents) TxHashExtractorCreator() transactions.TxHa } // RewardTxDataCreator return reward tx handler -func (mrtc *managedRunTypeComponents) RewardTxDataCreator() transactions.RewardTxData { +func (mrtc *managedRunTypeComponents) RewardTxDataCreator() transactions.RewardTxDataHandler { mrtc.mutRunTypeCoreComponents.Lock() defer mrtc.mutRunTypeCoreComponents.Unlock() diff --git a/process/elasticproc/factory/elasticProcessorFactory.go b/process/elasticproc/factory/elasticProcessorFactory.go index 89d3f46e..54819439 100644 --- a/process/elasticproc/factory/elasticProcessorFactory.go +++ b/process/elasticproc/factory/elasticProcessorFactory.go @@ -33,7 +33,7 @@ type ArgElasticProcessorFactory struct { UseKibana bool ImportDB bool TxHashExtractor transactions.TxHashExtractor - RewardTxData transactions.RewardTxData + RewardTxData transactions.RewardTxDataHandler } // CreateElasticProcessor will create a new instance of ElasticProcessor diff --git a/process/elasticproc/transactions/interface.go b/process/elasticproc/transactions/interface.go index f143596b..3139cc2a 100644 --- a/process/elasticproc/transactions/interface.go +++ b/process/elasticproc/transactions/interface.go @@ -21,8 +21,8 @@ type TxHashExtractor interface { IsInterfaceNil() bool } -// RewardTxData defines what rewards tx handler should be able to do -type RewardTxData interface { +// RewardTxDataHandler defines what rewards tx handler should be able to do +type RewardTxDataHandler interface { GetSender() string IsInterfaceNil() bool } diff --git a/process/elasticproc/transactions/rewardTxData.go b/process/elasticproc/transactions/rewardTxData.go index 15961b8f..48678e5f 100644 --- a/process/elasticproc/transactions/rewardTxData.go +++ b/process/elasticproc/transactions/rewardTxData.go @@ -13,7 +13,7 @@ func NewRewardTxData() *rewardTxData { return &rewardTxData{} } -// GetSender return the shard id as string +// GetSender return the metachain shard id as string func (rtd *rewardTxData) GetSender() string { return fmt.Sprintf("%d", core.MetachainShardId) } diff --git a/process/elasticproc/transactions/sovereignRewardTxData.go b/process/elasticproc/transactions/sovereignRewardTxData.go index 749c28e3..91b266e8 100644 --- a/process/elasticproc/transactions/sovereignRewardTxData.go +++ b/process/elasticproc/transactions/sovereignRewardTxData.go @@ -13,7 +13,7 @@ func NewSovereignRewardTxData() *sovereignRewardTxData { return &sovereignRewardTxData{} } -// GetSender return the shard id as string +// GetSender return the sovereign shard id as string func (srtd *sovereignRewardTxData) GetSender() string { return fmt.Sprintf("%d", core.SovereignChainShardId) } diff --git a/process/elasticproc/transactions/transactionDBBuilder.go b/process/elasticproc/transactions/transactionDBBuilder.go index fbb154a1..dbeb5b76 100644 --- a/process/elasticproc/transactions/transactionDBBuilder.go +++ b/process/elasticproc/transactions/transactionDBBuilder.go @@ -20,14 +20,14 @@ type dbTransactionBuilder struct { addressPubkeyConverter core.PubkeyConverter dataFieldParser DataFieldParser balanceConverter dataindexer.BalanceConverter - rewardTxData RewardTxData + rewardTxData RewardTxDataHandler } func newTransactionDBBuilder( addressPubkeyConverter core.PubkeyConverter, dataFieldParser DataFieldParser, balanceConverter dataindexer.BalanceConverter, - rewardTxData RewardTxData, + rewardTxData RewardTxDataHandler, ) *dbTransactionBuilder { return &dbTransactionBuilder{ addressPubkeyConverter: addressPubkeyConverter, diff --git a/process/elasticproc/transactions/transactionsProcessor.go b/process/elasticproc/transactions/transactionsProcessor.go index 3531852e..d793ac01 100644 --- a/process/elasticproc/transactions/transactionsProcessor.go +++ b/process/elasticproc/transactions/transactionsProcessor.go @@ -28,7 +28,7 @@ type ArgsTransactionProcessor struct { Marshalizer marshal.Marshalizer BalanceConverter dataindexer.BalanceConverter TxHashExtractor TxHashExtractor - RewardTxData RewardTxData + RewardTxData RewardTxDataHandler } type txsDatabaseProcessor struct { From e3add432336f30f32a4e9885d7a231aa890f2078 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Mon, 9 Dec 2024 11:47:15 +0200 Subject: [PATCH 26/50] fixes after review --- factory/runType/runTypeComponentsHandler.go | 2 +- process/elasticproc/transactions/checkers.go | 2 +- process/elasticproc/transactions/checkers_test.go | 4 ++-- process/elasticproc/transactions/errors.go | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/factory/runType/runTypeComponentsHandler.go b/factory/runType/runTypeComponentsHandler.go index b514d1c5..6b0fe9e9 100644 --- a/factory/runType/runTypeComponentsHandler.go +++ b/factory/runType/runTypeComponentsHandler.go @@ -73,7 +73,7 @@ func (mrtc *managedRunTypeComponents) CheckSubcomponents() error { return transactions.ErrNilTxHashExtractor } if check.IfNil(mrtc.rewardTxData) { - return transactions.ErrNilRewardTxData + return transactions.ErrNilRewardTxDataHandler } return nil } diff --git a/process/elasticproc/transactions/checkers.go b/process/elasticproc/transactions/checkers.go index 3a8f768f..0319e654 100644 --- a/process/elasticproc/transactions/checkers.go +++ b/process/elasticproc/transactions/checkers.go @@ -40,7 +40,7 @@ func checkTxsProcessorArg(args *ArgsTransactionProcessor) error { return ErrNilTxHashExtractor } if check.IfNil(args.RewardTxData) { - return ErrNilRewardTxData + return ErrNilRewardTxDataHandler } return nil diff --git a/process/elasticproc/transactions/checkers_test.go b/process/elasticproc/transactions/checkers_test.go index 1c928eda..cb0ba985 100644 --- a/process/elasticproc/transactions/checkers_test.go +++ b/process/elasticproc/transactions/checkers_test.go @@ -84,13 +84,13 @@ func TestNewTransactionsProcessor(t *testing.T) { exErr: ErrNilTxHashExtractor, }, { - name: "NilTxHashExtractor", + name: "NilRewardTxDataHandler", args: func() *ArgsTransactionProcessor { args := createMockArgs() args.RewardTxData = nil return args }, - exErr: ErrNilRewardTxData, + exErr: ErrNilRewardTxDataHandler, }, } diff --git a/process/elasticproc/transactions/errors.go b/process/elasticproc/transactions/errors.go index 1a130d0d..9f91b390 100644 --- a/process/elasticproc/transactions/errors.go +++ b/process/elasticproc/transactions/errors.go @@ -7,5 +7,5 @@ import ( // ErrNilTxHashExtractor signals that a nil tx hash extractor has been provided var ErrNilTxHashExtractor = errors.New("nil tx hash extractor") -// ErrNilRewardTxData signals that a nil rewards tx data handler has been provided -var ErrNilRewardTxData = errors.New("nil reward tx data handler") +// ErrNilRewardTxDataHandler signals that a nil rewards tx data handler has been provided +var ErrNilRewardTxDataHandler = errors.New("nil reward tx data handler") From 9ca95d36431178b559f7845a40e54fb12882b81a Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Wed, 11 Dec 2024 10:57:35 +0200 Subject: [PATCH 27/50] renaming to Sovereign --- cmd/elasticindexer/main.go | 2 +- config/config.go | 2 +- factory/wsIndexerFactory.go | 2 +- process/factory/indexerFactory.go | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/elasticindexer/main.go b/cmd/elasticindexer/main.go index 3cbf2657..e2256db9 100644 --- a/cmd/elasticindexer/main.go +++ b/cmd/elasticindexer/main.go @@ -88,7 +88,7 @@ func startIndexer(ctx *cli.Context) error { if err != nil { return fmt.Errorf("%w while loading the config file", err) } - cfg.SovereignType = ctx.GlobalBool(sovereign.Name) + cfg.Sovereign = ctx.GlobalBool(sovereign.Name) clusterCfg, err := loadClusterConfig(ctx.GlobalString(configurationPreferencesFile.Name)) if err != nil { diff --git a/config/config.go b/config/config.go index 2f3ef7be..5d4e7812 100644 --- a/config/config.go +++ b/config/config.go @@ -29,7 +29,7 @@ type Config struct { LogsPath string `toml:"logs-path"` } `toml:"logs"` } `toml:"config"` - SovereignType bool + Sovereign bool } // ClusterConfig will hold the config for the Elasticsearch cluster diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index 96735a9c..d9d1264c 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -77,7 +77,7 @@ func createDataIndexer( } return factory.NewIndexer(factory.ArgsIndexerFactory{ - SovereignType: cfg.SovereignType, + Sovereign: cfg.Sovereign, UseKibana: clusterCfg.Config.ElasticCluster.UseKibana, Denomination: cfg.Config.Economics.Denomination, BulkRequestMaxSize: clusterCfg.Config.ElasticCluster.BulkRequestMaxSizeInBytes, diff --git a/process/factory/indexerFactory.go b/process/factory/indexerFactory.go index 3898cd3b..295039aa 100644 --- a/process/factory/indexerFactory.go +++ b/process/factory/indexerFactory.go @@ -32,7 +32,7 @@ type ArgsIndexerFactory struct { Enabled bool UseKibana bool ImportDB bool - SovereignType bool + Sovereign bool Denomination int BulkRequestMaxSize int Url string @@ -57,7 +57,7 @@ func NewIndexer(args ArgsIndexerFactory) (dataindexer.Indexer, error) { return nil, err } - if args.SovereignType { + if args.Sovereign { args.RunTypeComponents, err = createManagedRunTypeComponents(runType.NewSovereignRunTypeComponentsFactory()) } else { args.RunTypeComponents, err = createManagedRunTypeComponents(runType.NewRunTypeComponentsFactory()) From b91174cd64e3b4a825cf58766f9e34a62484a598 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Thu, 23 Jan 2025 17:47:07 +0200 Subject: [PATCH 28/50] Index incoming tokens --- cmd/elasticindexer/config/prefs.toml | 7 + config/config.go | 10 +- data/tokens.go | 12 ++ factory/runType/interface.go | 4 +- factory/runType/runTypeComponents.go | 6 +- factory/runType/runTypeComponentsFactory.go | 10 +- factory/runType/runTypeComponentsHandler.go | 21 ++- .../sovereignRunTypeComponentsFactory.go | 28 +++- factory/wsIndexerFactory.go | 9 ++ integrationtests/consts.go | 3 +- integrationtests/incomingSCR_test.go | 99 +++++++++++++ .../testdata/incomingSCR/incoming-scr.json | 38 +++++ integrationtests/utils.go | 34 ++++- process/elasticproc/elasticProcessor.go | 11 +- .../factory/elasticProcessorFactory.go | 11 ++ process/elasticproc/interface.go | 7 + .../elasticproc/tokens/indexTokensHandler.go | 21 +++ .../tokens/sovereignIndexTokensHandler.go | 133 ++++++++++++++++++ process/elasticproc/transactions/errors.go | 3 + process/factory/indexerFactory.go | 5 +- 20 files changed, 450 insertions(+), 22 deletions(-) create mode 100644 integrationtests/incomingSCR_test.go create mode 100644 integrationtests/testdata/incomingSCR/incoming-scr.json create mode 100644 process/elasticproc/tokens/indexTokensHandler.go create mode 100644 process/elasticproc/tokens/sovereignIndexTokensHandler.go diff --git a/cmd/elasticindexer/config/prefs.toml b/cmd/elasticindexer/config/prefs.toml index 2685e4b8..49100748 100644 --- a/cmd/elasticindexer/config/prefs.toml +++ b/cmd/elasticindexer/config/prefs.toml @@ -23,3 +23,10 @@ username = "" password = "" bulk-request-max-size-in-bytes = 4194304 # 4MB + + [config.main-elastic-cluster] + use-kibana = false + url = "https://testnet-index.multiversx.com" + username = "" + password = "" + bulk-request-max-size-in-bytes = 4194304 # 4MB diff --git a/config/config.go b/config/config.go index 5d4e7812..6e864804 100644 --- a/config/config.go +++ b/config/config.go @@ -29,7 +29,8 @@ type Config struct { LogsPath string `toml:"logs-path"` } `toml:"logs"` } `toml:"config"` - Sovereign bool + Sovereign bool + ESDTPrefix string } // ClusterConfig will hold the config for the Elasticsearch cluster @@ -52,6 +53,13 @@ type ClusterConfig struct { Password string `toml:"password"` BulkRequestMaxSizeInBytes int `toml:"bulk-request-max-size-in-bytes"` } `toml:"elastic-cluster"` + MainChainCluster struct { + UseKibana bool `toml:"use-kibana"` + URL string `toml:"url"` + UserName string `toml:"username"` + Password string `toml:"password"` + BulkRequestMaxSizeInBytes int `toml:"bulk-request-max-size-in-bytes"` + } `toml:"main-elastic-cluster"` } `toml:"config"` } diff --git a/data/tokens.go b/data/tokens.go index e878facf..0c47f651 100644 --- a/data/tokens.go +++ b/data/tokens.go @@ -42,6 +42,18 @@ type SourceToken struct { CurrentOwner string `json:"currentOwner"` } +// ResponseTokenInfo is the structure for the token info response +type ResponseTokenInfo struct { + Docs []ResponseTokenInfoDB `json:"docs"` +} + +// ResponseTokenInfoDB is the structure for the token info response +type ResponseTokenInfoDB struct { + Found bool `json:"found"` + ID string `json:"_id"` + Source TokenInfo `json:"_source"` +} + // TokenInfo is a structure that is needed to store information about a token type TokenInfo struct { Name string `json:"name,omitempty"` diff --git a/factory/runType/interface.go b/factory/runType/interface.go index 1114ff4d..4b96663b 100644 --- a/factory/runType/interface.go +++ b/factory/runType/interface.go @@ -1,12 +1,13 @@ package runType import ( + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" ) // RunTypeComponentsCreator is the interface for creating run type components type RunTypeComponentsCreator interface { - Create() *runTypeComponents + Create() (*runTypeComponents, error) IsInterfaceNil() bool } @@ -28,6 +29,7 @@ type RunTypeComponentsHandler interface { type RunTypeComponentsHolder interface { TxHashExtractorCreator() transactions.TxHashExtractor RewardTxDataCreator() transactions.RewardTxDataHandler + IndexTokensHandlerCreator() elasticproc.IndexTokensHandler Create() error Close() error CheckSubcomponents() error diff --git a/factory/runType/runTypeComponents.go b/factory/runType/runTypeComponents.go index b70512b6..4bccd66b 100644 --- a/factory/runType/runTypeComponents.go +++ b/factory/runType/runTypeComponents.go @@ -1,12 +1,14 @@ package runType import ( + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" ) type runTypeComponents struct { - txHashExtractor transactions.TxHashExtractor - rewardTxData transactions.RewardTxDataHandler + txHashExtractor transactions.TxHashExtractor + rewardTxData transactions.RewardTxDataHandler + indexTokensHandler elasticproc.IndexTokensHandler } // Close does nothing diff --git a/factory/runType/runTypeComponentsFactory.go b/factory/runType/runTypeComponentsFactory.go index 8ec94df3..af7d42a3 100644 --- a/factory/runType/runTypeComponentsFactory.go +++ b/factory/runType/runTypeComponentsFactory.go @@ -1,6 +1,7 @@ package runType import ( + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tokens" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" ) @@ -12,11 +13,12 @@ func NewRunTypeComponentsFactory() *runTypeComponentsFactory { } // Create will create the run type components -func (rtcf *runTypeComponentsFactory) Create() *runTypeComponents { +func (rtcf *runTypeComponentsFactory) Create() (*runTypeComponents, error) { return &runTypeComponents{ - txHashExtractor: transactions.NewTxHashExtractor(), - rewardTxData: transactions.NewRewardTxData(), - } + txHashExtractor: transactions.NewTxHashExtractor(), + rewardTxData: transactions.NewRewardTxData(), + indexTokensHandler: tokens.NewIndexTokensHandler(), + }, nil } // IsInterfaceNil returns true if there is no value under the interface diff --git a/factory/runType/runTypeComponentsHandler.go b/factory/runType/runTypeComponentsHandler.go index 6b0fe9e9..1d471a5c 100644 --- a/factory/runType/runTypeComponentsHandler.go +++ b/factory/runType/runTypeComponentsHandler.go @@ -5,6 +5,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" ) @@ -34,7 +35,10 @@ func NewManagedRunTypeComponents(rtc RunTypeComponentsCreator) (*managedRunTypeC // Create will create the managed components func (mrtc *managedRunTypeComponents) Create() error { - rtc := mrtc.factory.Create() + rtc, err := mrtc.factory.Create() + if err != nil { + return err + } mrtc.mutRunTypeCoreComponents.Lock() mrtc.runTypeComponents = rtc @@ -75,6 +79,9 @@ func (mrtc *managedRunTypeComponents) CheckSubcomponents() error { if check.IfNil(mrtc.rewardTxData) { return transactions.ErrNilRewardTxDataHandler } + if check.IfNil(mrtc.indexTokensHandler) { + return transactions.ErrNilIndexTokensHandler + } return nil } @@ -102,6 +109,18 @@ func (mrtc *managedRunTypeComponents) RewardTxDataCreator() transactions.RewardT return mrtc.runTypeComponents.rewardTxData } +// IndexTokensHandlerCreator return index tokens handler +func (mrtc *managedRunTypeComponents) IndexTokensHandlerCreator() elasticproc.IndexTokensHandler { + mrtc.mutRunTypeCoreComponents.Lock() + defer mrtc.mutRunTypeCoreComponents.Unlock() + + if check.IfNil(mrtc.runTypeComponents) { + return nil + } + + return mrtc.runTypeComponents.indexTokensHandler +} + // IsInterfaceNil returns true if the interface is nil func (mrtc *managedRunTypeComponents) IsInterfaceNil() bool { return mrtc == nil diff --git a/factory/runType/sovereignRunTypeComponentsFactory.go b/factory/runType/sovereignRunTypeComponentsFactory.go index 874a1b29..53c8d767 100644 --- a/factory/runType/sovereignRunTypeComponentsFactory.go +++ b/factory/runType/sovereignRunTypeComponentsFactory.go @@ -1,22 +1,36 @@ package runType import ( + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tokens" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" ) -type sovereignRunTypeComponentsFactory struct{} +type sovereignRunTypeComponentsFactory struct { + mainChainElastic factory.MainChainElastic + esdtPrefix string +} // NewSovereignRunTypeComponentsFactory will return a new instance of sovereign run type components factory -func NewSovereignRunTypeComponentsFactory() *sovereignRunTypeComponentsFactory { - return &sovereignRunTypeComponentsFactory{} +func NewSovereignRunTypeComponentsFactory(mainChainElastic factory.MainChainElastic, esdtPrefix string) *sovereignRunTypeComponentsFactory { + return &sovereignRunTypeComponentsFactory{ + mainChainElastic: mainChainElastic, + esdtPrefix: esdtPrefix, + } } // Create will create the run type components -func (srtcf *sovereignRunTypeComponentsFactory) Create() *runTypeComponents { - return &runTypeComponents{ - txHashExtractor: transactions.NewSovereignTxHashExtractor(), - rewardTxData: transactions.NewSovereignRewardTxData(), +func (srtcf *sovereignRunTypeComponentsFactory) Create() (*runTypeComponents, error) { + sovIndexTokensHandler, err := tokens.NewSovereignIndexTokensHandler(srtcf.mainChainElastic, srtcf.esdtPrefix) + if err != nil { + return nil, err } + + return &runTypeComponents{ + txHashExtractor: transactions.NewSovereignTxHashExtractor(), + rewardTxData: transactions.NewSovereignRewardTxData(), + indexTokensHandler: sovIndexTokensHandler, + }, nil } // IsInterfaceNil returns true if there is no value under the interface diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index d9d1264c..82d5da64 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -11,6 +11,7 @@ import ( "github.com/multiversx/mx-chain-es-indexer-go/config" "github.com/multiversx/mx-chain-es-indexer-go/core" + esFactory "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" "github.com/multiversx/mx-chain-es-indexer-go/process/factory" "github.com/multiversx/mx-chain-es-indexer-go/process/wsindexer" ) @@ -76,8 +77,16 @@ func createDataIndexer( return nil, err } + mainChainElastic := esFactory.MainChainElastic{ + Url: clusterCfg.Config.MainChainCluster.URL, + UserName: clusterCfg.Config.MainChainCluster.UserName, + Password: clusterCfg.Config.MainChainCluster.Password, + BulkRequestMaxSize: clusterCfg.Config.MainChainCluster.BulkRequestMaxSizeInBytes, + } + return factory.NewIndexer(factory.ArgsIndexerFactory{ Sovereign: cfg.Sovereign, + MainChainElastic: mainChainElastic, UseKibana: clusterCfg.Config.ElasticCluster.UseKibana, Denomination: cfg.Config.Economics.Denomination, BulkRequestMaxSize: clusterCfg.Config.ElasticCluster.BulkRequestMaxSizeInBytes, diff --git a/integrationtests/consts.go b/integrationtests/consts.go index 8593fcf6..04b25a09 100644 --- a/integrationtests/consts.go +++ b/integrationtests/consts.go @@ -4,7 +4,8 @@ const ( //nolint testNumOfShards = 3 //nolint - esURL = "http://localhost:9200" + esURL = "http://localhost:9200" + testnetEsURL = "https://testnet-index.multiversx.com" //nolint addressPrefix = "erd" ) diff --git a/integrationtests/incomingSCR_test.go b/integrationtests/incomingSCR_test.go new file mode 100644 index 00000000..cfa6ff3e --- /dev/null +++ b/integrationtests/incomingSCR_test.go @@ -0,0 +1,99 @@ +//go:build integrationtests + +package integrationtests + +import ( + "context" + "encoding/hex" + "math/big" + "testing" + + "github.com/multiversx/mx-chain-core-go/core" + dataBlock "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/multiversx/mx-chain-core-go/data/outport" + "github.com/multiversx/mx-chain-core-go/data/smartContractResult" + "github.com/stretchr/testify/require" + + indexerData "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" +) + +func TestSovereignTransactionWithScCallSuccess(t *testing.T) { + setLogLevelDebug() + + mainChainEs := factory.MainChainElastic{ + Url: testnetEsURL, + } + + esClient, err := createESClient(esURL) + require.Nil(t, err) + + esProc, err := CreateSovereignElasticProcessor(esClient, mainChainEs) + require.Nil(t, err) + + txHash := []byte("txHash") + header := &dataBlock.Header{ + Round: 50, + TimeStamp: 5040, + } + body := &dataBlock.Body{ + MiniBlocks: dataBlock.MiniBlockSlice{ + { + Type: dataBlock.SmartContractResultBlock, + SenderShardID: 4294967293, + ReceiverShardID: 0, + TxHashes: [][]byte{txHash}, + }, + }, + } + + token1Identifier := "AGE-be2571" + token2Identifier := "BGD16-c47f46" + data := []byte(core.BuiltInFunctionMultiESDTNFTTransfer + + "@02" + + "@" + hex.EncodeToString([]byte(token1Identifier)) + + "@" + + "@" + hex.EncodeToString(big.NewInt(123).Bytes()) + + "@" + hex.EncodeToString([]byte(token2Identifier)) + + "@" + + "@" + hex.EncodeToString(big.NewInt(333).Bytes())) + + genericResponse := &GenericResponse{} + err = esClient.DoMultiGet(context.Background(), []string{token1Identifier, token2Identifier}, indexerData.TokensIndex, true, genericResponse) + require.Nil(t, err) + for _, token := range genericResponse.Docs { + require.False(t, token.Found) + } + + pool := &outport.TransactionPool{ + SmartContractResults: map[string]*outport.SCRInfo{ + hex.EncodeToString(txHash): {SmartContractResult: &smartContractResult.SmartContractResult{ + Nonce: 11, + Value: big.NewInt(0), + GasLimit: 0, + SndAddr: decodeAddress("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u"), + RcvAddr: decodeAddress("erd1kzrfl2tztgzjpeedwec37c8npcr0a2ulzh9lhmj7xufyg23zcxuqxcqz0s"), + Data: data, + OriginalTxHash: nil, + }, FeeInfo: &outport.FeeInfo{}}, + }, + } + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, 1)) + require.Nil(t, err) + + ids := []string{hex.EncodeToString(txHash)} + genericResponse = &GenericResponse{} + err = esClient.DoMultiGet(context.Background(), ids, indexerData.ScResultsIndex, true, genericResponse) + require.Nil(t, err) + require.JSONEq(t, + readExpectedResult("./testdata/incomingSCR/incoming-scr.json"), + string(genericResponse.Docs[0].Source), + ) + + genericResponse = &GenericResponse{} + err = esClient.DoMultiGet(context.Background(), []string{token1Identifier, token2Identifier}, indexerData.TokensIndex, true, genericResponse) + require.Nil(t, err) + for _, token := range genericResponse.Docs { + require.True(t, token.Found) + } +} diff --git a/integrationtests/testdata/incomingSCR/incoming-scr.json b/integrationtests/testdata/incomingSCR/incoming-scr.json new file mode 100644 index 00000000..09eaf4fb --- /dev/null +++ b/integrationtests/testdata/incomingSCR/incoming-scr.json @@ -0,0 +1,38 @@ +{ + "miniBlockHash": "64b954ab72a867bf9866556c7aac3b80a51a69a1d53c640e78f53e5bfcdcc37d", + "nonce": 11, + "gasLimit": 0, + "gasPrice": 0, + "value": "0", + "valueNum": 0, + "sender": "erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u", + "receiver": "erd1kzrfl2tztgzjpeedwec37c8npcr0a2ulzh9lhmj7xufyg23zcxuqxcqz0s", + "senderShard": 4294967293, + "receiverShard": 0, + "data": "TXVsdGlFU0RUTkZUVHJhbnNmZXJAMDJANDE0NzQ1MmQ2MjY1MzIzNTM3MzFAQDdiQDQyNDc0NDMxMzYyZDYzMzQzNzY2MzQzNkBAMDE0ZA==", + "prevTxHash": "", + "originalTxHash": "", + "callType": "0", + "timestamp": 5040, + "tokens": [ + "AGE-be2571", + "BGD16-c47f46" + ], + "esdtValues": [ + "123", + "333" + ], + "esdtValuesNum": [ + 1.23e-16, + 3.33e-16 + ], + "receivers": [ + "erd1kzrfl2tztgzjpeedwec37c8npcr0a2ulzh9lhmj7xufyg23zcxuqxcqz0s", + "erd1kzrfl2tztgzjpeedwec37c8npcr0a2ulzh9lhmj7xufyg23zcxuqxcqz0s" + ], + "receiversShardIDs": [ + 0, + 0 + ], + "operation": "MultiESDTNFTTransfer" +} \ No newline at end of file diff --git a/integrationtests/utils.go b/integrationtests/utils.go index 06239e0c..e2eab567 100644 --- a/integrationtests/utils.go +++ b/integrationtests/utils.go @@ -20,12 +20,14 @@ import ( "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tokens" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" ) var ( log = logger.GetOrCreate("integration-tests") pubKeyConverter, _ = pubkeyConverter.NewBech32PubkeyConverter(32, addressPrefix) + sovEsdtPrefix = "sov" ) // nolint @@ -62,9 +64,35 @@ func CreateElasticProcessor( EnabledIndexes: []string{dataindexer.TransactionsIndex, dataindexer.LogsIndex, dataindexer.AccountsESDTIndex, dataindexer.ScResultsIndex, dataindexer.ReceiptsIndex, dataindexer.BlockIndex, dataindexer.AccountsIndex, dataindexer.TokensIndex, dataindexer.TagsIndex, dataindexer.EventsIndex, dataindexer.OperationsIndex, dataindexer.DelegatorsIndex, dataindexer.ESDTsIndex, dataindexer.SCDeploysIndex, dataindexer.MiniblocksIndex, dataindexer.ValuesIndex}, - Denomination: 18, - TxHashExtractor: transactions.NewTxHashExtractor(), - RewardTxData: transactions.NewRewardTxData(), + Denomination: 18, + TxHashExtractor: transactions.NewTxHashExtractor(), + RewardTxData: transactions.NewRewardTxData(), + IndexTokensHandler: tokens.NewIndexTokensHandler(), + } + + return factory.CreateElasticProcessor(args) +} + +// CreateSovereignElasticProcessor - +func CreateSovereignElasticProcessor( + esClient elasticproc.DatabaseClientHandler, + mainChainEs factory.MainChainElastic, +) (dataindexer.ElasticProcessor, error) { + sovIndexTokens, _ := tokens.NewSovereignIndexTokensHandler(mainChainEs, sovEsdtPrefix) + + args := factory.ArgElasticProcessorFactory{ + Marshalizer: &mock.MarshalizerMock{}, + Hasher: &mock.HasherMock{}, + AddressPubkeyConverter: pubKeyConverter, + ValidatorPubkeyConverter: mock.NewPubkeyConverterMock(32), + DBClient: esClient, + EnabledIndexes: []string{dataindexer.TransactionsIndex, dataindexer.LogsIndex, dataindexer.AccountsESDTIndex, dataindexer.ScResultsIndex, + dataindexer.ReceiptsIndex, dataindexer.BlockIndex, dataindexer.AccountsIndex, dataindexer.TokensIndex, dataindexer.TagsIndex, dataindexer.EventsIndex, + dataindexer.OperationsIndex, dataindexer.DelegatorsIndex, dataindexer.ESDTsIndex, dataindexer.SCDeploysIndex, dataindexer.MiniblocksIndex, dataindexer.ValuesIndex}, + Denomination: 18, + TxHashExtractor: transactions.NewSovereignTxHashExtractor(), + RewardTxData: transactions.NewSovereignRewardTxData(), + IndexTokensHandler: sovIndexTokens, } return factory.CreateElasticProcessor(args) diff --git a/process/elasticproc/elasticProcessor.go b/process/elasticproc/elasticProcessor.go index 89634a95..e0568504 100644 --- a/process/elasticproc/elasticProcessor.go +++ b/process/elasticproc/elasticProcessor.go @@ -14,6 +14,8 @@ import ( "github.com/multiversx/mx-chain-core-go/data/alteredAccount" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-es-indexer-go/core/request" "github.com/multiversx/mx-chain-es-indexer-go/data" elasticIndexer "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" @@ -21,7 +23,6 @@ import ( "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tags" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tokeninfo" "github.com/multiversx/mx-chain-es-indexer-go/templates" - logger "github.com/multiversx/mx-chain-logger-go" ) var ( @@ -57,6 +58,7 @@ type ArgElasticProcessor struct { LogsAndEventsProc DBLogsAndEventsHandler OperationsProc OperationsHandler Version string + IndexTokensHandler IndexTokensHandler } type elasticProcessor struct { @@ -73,6 +75,7 @@ type elasticProcessor struct { validatorsProc DBValidatorsHandler logsAndEventsProc DBLogsAndEventsHandler operationsProc OperationsHandler + indexTokensHandler IndexTokensHandler } // NewElasticProcessor handles Elasticsearch operations such as initialization, adding, modifying or removing data @@ -94,6 +97,7 @@ func NewElasticProcessor(arguments *ArgElasticProcessor) (*elasticProcessor, err logsAndEventsProc: arguments.LogsAndEventsProc, operationsProc: arguments.OperationsProc, bulkRequestMaxSize: arguments.BulkRequestMaxSize, + indexTokensHandler: arguments.IndexTokensHandler, } err = ei.init(arguments.UseKibana, arguments.IndexTemplates, arguments.IndexPolicies, arguments.ExtraMappings) @@ -503,6 +507,11 @@ func (ei *elasticProcessor) SaveTransactions(obh *outport.OutportBlockWithHeader return err } + err = ei.indexTokensHandler.IndexCrossChainTokens(ei.elasticClient, preparedResults.ScResults, buffers) + if err != nil { + return err + } + return ei.doBulkRequests("", buffers.Buffers(), obh.ShardID) } diff --git a/process/elasticproc/factory/elasticProcessorFactory.go b/process/elasticproc/factory/elasticProcessorFactory.go index 54819439..fd361bc9 100644 --- a/process/elasticproc/factory/elasticProcessorFactory.go +++ b/process/elasticproc/factory/elasticProcessorFactory.go @@ -19,6 +19,15 @@ import ( "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/validators" ) +// MainChainElastic holds the elastic search settings +type MainChainElastic struct { + Enabled bool + Url string + UserName string + Password string + BulkRequestMaxSize int +} + // ArgElasticProcessorFactory is struct that is used to store all components that are needed to create an elastic processor factory type ArgElasticProcessorFactory struct { Marshalizer marshal.Marshalizer @@ -34,6 +43,7 @@ type ArgElasticProcessorFactory struct { ImportDB bool TxHashExtractor transactions.TxHashExtractor RewardTxData transactions.RewardTxDataHandler + IndexTokensHandler elasticproc.IndexTokensHandler } // CreateElasticProcessor will create a new instance of ElasticProcessor @@ -132,6 +142,7 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.E OperationsProc: operationsProc, ImportDB: arguments.ImportDB, Version: arguments.Version, + IndexTokensHandler: arguments.IndexTokensHandler, } return elasticproc.NewElasticProcessor(args) diff --git a/process/elasticproc/interface.go b/process/elasticproc/interface.go index fe4b565f..fd5c0af0 100644 --- a/process/elasticproc/interface.go +++ b/process/elasticproc/interface.go @@ -8,6 +8,7 @@ import ( "github.com/multiversx/mx-chain-core-go/data/alteredAccount" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" + "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tokeninfo" ) @@ -120,3 +121,9 @@ type OperationsHandler interface { ProcessTransactionsAndSCRs(txs []*data.Transaction, scrs []*data.ScResult, isImportDB bool, shardID uint32) ([]*data.Transaction, []*data.ScResult) SerializeSCRs(scrs []*data.ScResult, buffSlice *data.BufferSlice, index string, shardID uint32) error } + +// IndexTokensHandler defines what index tokens handler should be able to do +type IndexTokensHandler interface { + IndexCrossChainTokens(elasticClient DatabaseClientHandler, scrs []*data.ScResult, buffSlice *data.BufferSlice) error + IsInterfaceNil() bool +} diff --git a/process/elasticproc/tokens/indexTokensHandler.go b/process/elasticproc/tokens/indexTokensHandler.go new file mode 100644 index 00000000..e48287fd --- /dev/null +++ b/process/elasticproc/tokens/indexTokensHandler.go @@ -0,0 +1,21 @@ +package tokens + +import ( + "github.com/multiversx/mx-chain-es-indexer-go/data" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" +) + +type indexTokensHandler struct{} + +func NewIndexTokensHandler() *indexTokensHandler { + return &indexTokensHandler{} +} + +func (it *indexTokensHandler) IndexCrossChainTokens(_ elasticproc.DatabaseClientHandler, _ []*data.ScResult, _ *data.BufferSlice) error { + return nil +} + +// IsInterfaceNil returns true if there is no value under the interface +func (it *indexTokensHandler) IsInterfaceNil() bool { + return it == nil +} diff --git a/process/elasticproc/tokens/sovereignIndexTokensHandler.go b/process/elasticproc/tokens/sovereignIndexTokensHandler.go new file mode 100644 index 00000000..e061ada4 --- /dev/null +++ b/process/elasticproc/tokens/sovereignIndexTokensHandler.go @@ -0,0 +1,133 @@ +package tokens + +import ( + "context" + "encoding/json" + "fmt" + "math" + "net/http" + "time" + + "github.com/elastic/go-elasticsearch/v7" + "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-core-go/data/esdt" + + "github.com/multiversx/mx-chain-es-indexer-go/client" + "github.com/multiversx/mx-chain-es-indexer-go/client/logging" + "github.com/multiversx/mx-chain-es-indexer-go/data" + indexerdata "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" +) + +type sovereignIndexTokensHandler struct { + mainChainElasticClient elasticproc.DatabaseClientHandler + bulkRequestMaxSize int + esdtPrefix string +} + +func NewSovereignIndexTokensHandler(mainChainElastic factory.MainChainElastic, esdtPrefix string) (*sovereignIndexTokensHandler, error) { + argsEsClient := elasticsearch.Config{ + Addresses: []string{mainChainElastic.Url}, + Username: mainChainElastic.UserName, + Password: mainChainElastic.Password, + Logger: &logging.CustomLogger{}, + RetryOnStatus: []int{http.StatusConflict}, + RetryBackoff: retryBackOff, + } + mainChainElasticClient, err := client.NewElasticClient(argsEsClient) + if err != nil { + return nil, err + } + + return &sovereignIndexTokensHandler{ + mainChainElasticClient: mainChainElasticClient, + bulkRequestMaxSize: mainChainElastic.BulkRequestMaxSize, + esdtPrefix: esdtPrefix, + }, nil +} + +func retryBackOff(attempt int) time.Duration { + return time.Duration(math.Exp2(float64(attempt))) * time.Second +} + +func (sit *sovereignIndexTokensHandler) IndexCrossChainTokens(elasticClient elasticproc.DatabaseClientHandler, scrs []*data.ScResult, buffSlice *data.BufferSlice) error { + notFoundTokens, err := sit.getTokensFromScrs(elasticClient, scrs) + if err != nil { + return err + } + + if len(notFoundTokens) == 0 { // no new tokens + return nil + } + + // get tokens from main chain elastic db + mainChainTokens := &data.ResponseTokenInfo{} + err = sit.mainChainElasticClient.DoMultiGet(context.Background(), notFoundTokens, indexerdata.TokensIndex, true, mainChainTokens) + if err != nil { + return err + } + + return sit.indexNewTokens(mainChainTokens.Docs, buffSlice) +} + +func (sit *sovereignIndexTokensHandler) getTokensFromScrs(elasticClient elasticproc.DatabaseClientHandler, scrs []*data.ScResult) ([]string, error) { + receivedTokensIDs := make([]string, 0) + for _, scr := range scrs { + if scr.SenderShard == core.MainChainShardId { + for _, token := range scr.Tokens { + tokenPrefix, hasPrefix := esdt.IsValidPrefixedToken(token) + if !hasPrefix || tokenPrefix != sit.esdtPrefix { + receivedTokensIDs = append(receivedTokensIDs, token) + } + } + } + } + + responseTokens := &data.ResponseTokens{} + err := elasticClient.DoMultiGet(context.Background(), receivedTokensIDs, indexerdata.TokensIndex, true, responseTokens) + if err != nil { + return nil, err + } + + newTokens := make([]string, 0) + for _, token := range responseTokens.Docs { + if token.Found == false { + newTokens = append(newTokens, token.ID) + } + } + + return newTokens, nil +} + +func (sit *sovereignIndexTokensHandler) indexNewTokens(responseTokensInfo []data.ResponseTokenInfoDB, buffSlice *data.BufferSlice) error { + for _, responseToken := range responseTokensInfo { + token := formatToken(responseToken) + + meta := []byte(fmt.Sprintf(`{ "index" : { "_index":"%s", "_id" : "%s" } }%s`, indexerdata.TokensIndex, converters.JsonEscape(token.Token), "\n")) + serializedTokenData, err := json.Marshal(token) + if err != nil { + return err + } + + err = buffSlice.PutData(meta, serializedTokenData) + if err != nil { + return err + } + } + + return nil +} + +func formatToken(token data.ResponseTokenInfoDB) data.TokenInfo { + token.Source.OwnersHistory = nil + token.Source.Properties = nil + + return token.Source +} + +// IsInterfaceNil returns true if there is no value under the interface +func (sit *sovereignIndexTokensHandler) IsInterfaceNil() bool { + return sit == nil +} diff --git a/process/elasticproc/transactions/errors.go b/process/elasticproc/transactions/errors.go index 9f91b390..f9280c86 100644 --- a/process/elasticproc/transactions/errors.go +++ b/process/elasticproc/transactions/errors.go @@ -9,3 +9,6 @@ var ErrNilTxHashExtractor = errors.New("nil tx hash extractor") // ErrNilRewardTxDataHandler signals that a nil rewards tx data handler has been provided var ErrNilRewardTxDataHandler = errors.New("nil reward tx data handler") + +// ErrNilIndexTokensHandler signals that a nil index tokens handler has been provided +var ErrNilIndexTokensHandler = errors.New("nil index tokens handler") diff --git a/process/factory/indexerFactory.go b/process/factory/indexerFactory.go index 295039aa..ca18d60d 100644 --- a/process/factory/indexerFactory.go +++ b/process/factory/indexerFactory.go @@ -33,6 +33,8 @@ type ArgsIndexerFactory struct { UseKibana bool ImportDB bool Sovereign bool + ESDTPrefix string + MainChainElastic factory.MainChainElastic Denomination int BulkRequestMaxSize int Url string @@ -58,7 +60,7 @@ func NewIndexer(args ArgsIndexerFactory) (dataindexer.Indexer, error) { } if args.Sovereign { - args.RunTypeComponents, err = createManagedRunTypeComponents(runType.NewSovereignRunTypeComponentsFactory()) + args.RunTypeComponents, err = createManagedRunTypeComponents(runType.NewSovereignRunTypeComponentsFactory(args.MainChainElastic, args.ESDTPrefix)) } else { args.RunTypeComponents, err = createManagedRunTypeComponents(runType.NewRunTypeComponentsFactory()) } @@ -126,6 +128,7 @@ func createElasticProcessor(args ArgsIndexerFactory) (dataindexer.ElasticProcess Version: args.Version, TxHashExtractor: args.RunTypeComponents.TxHashExtractorCreator(), RewardTxData: args.RunTypeComponents.RewardTxDataCreator(), + IndexTokensHandler: args.RunTypeComponents.IndexTokensHandlerCreator(), } return factory.CreateElasticProcessor(argsElasticProcFac) From 706e1fff8d159644fd20ce65cfae7e59c4b5177e Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Thu, 23 Jan 2025 17:51:22 +0200 Subject: [PATCH 29/50] removed some unused params --- cmd/elasticindexer/config/prefs.toml | 2 -- config/config.go | 8 +++----- factory/wsIndexerFactory.go | 7 +++---- process/elasticproc/factory/elasticProcessorFactory.go | 9 ++++----- .../elasticproc/tokens/sovereignIndexTokensHandler.go | 2 -- 5 files changed, 10 insertions(+), 18 deletions(-) diff --git a/cmd/elasticindexer/config/prefs.toml b/cmd/elasticindexer/config/prefs.toml index 49100748..917f006d 100644 --- a/cmd/elasticindexer/config/prefs.toml +++ b/cmd/elasticindexer/config/prefs.toml @@ -25,8 +25,6 @@ bulk-request-max-size-in-bytes = 4194304 # 4MB [config.main-elastic-cluster] - use-kibana = false url = "https://testnet-index.multiversx.com" username = "" password = "" - bulk-request-max-size-in-bytes = 4194304 # 4MB diff --git a/config/config.go b/config/config.go index 6e864804..217b5a92 100644 --- a/config/config.go +++ b/config/config.go @@ -54,11 +54,9 @@ type ClusterConfig struct { BulkRequestMaxSizeInBytes int `toml:"bulk-request-max-size-in-bytes"` } `toml:"elastic-cluster"` MainChainCluster struct { - UseKibana bool `toml:"use-kibana"` - URL string `toml:"url"` - UserName string `toml:"username"` - Password string `toml:"password"` - BulkRequestMaxSizeInBytes int `toml:"bulk-request-max-size-in-bytes"` + URL string `toml:"url"` + UserName string `toml:"username"` + Password string `toml:"password"` } `toml:"main-elastic-cluster"` } `toml:"config"` } diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index 82d5da64..1b622e30 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -78,10 +78,9 @@ func createDataIndexer( } mainChainElastic := esFactory.MainChainElastic{ - Url: clusterCfg.Config.MainChainCluster.URL, - UserName: clusterCfg.Config.MainChainCluster.UserName, - Password: clusterCfg.Config.MainChainCluster.Password, - BulkRequestMaxSize: clusterCfg.Config.MainChainCluster.BulkRequestMaxSizeInBytes, + Url: clusterCfg.Config.MainChainCluster.URL, + UserName: clusterCfg.Config.MainChainCluster.UserName, + Password: clusterCfg.Config.MainChainCluster.Password, } return factory.NewIndexer(factory.ArgsIndexerFactory{ diff --git a/process/elasticproc/factory/elasticProcessorFactory.go b/process/elasticproc/factory/elasticProcessorFactory.go index fd361bc9..c81053d0 100644 --- a/process/elasticproc/factory/elasticProcessorFactory.go +++ b/process/elasticproc/factory/elasticProcessorFactory.go @@ -21,11 +21,10 @@ import ( // MainChainElastic holds the elastic search settings type MainChainElastic struct { - Enabled bool - Url string - UserName string - Password string - BulkRequestMaxSize int + Enabled bool + Url string + UserName string + Password string } // ArgElasticProcessorFactory is struct that is used to store all components that are needed to create an elastic processor factory diff --git a/process/elasticproc/tokens/sovereignIndexTokensHandler.go b/process/elasticproc/tokens/sovereignIndexTokensHandler.go index e061ada4..b9dd78be 100644 --- a/process/elasticproc/tokens/sovereignIndexTokensHandler.go +++ b/process/elasticproc/tokens/sovereignIndexTokensHandler.go @@ -23,7 +23,6 @@ import ( type sovereignIndexTokensHandler struct { mainChainElasticClient elasticproc.DatabaseClientHandler - bulkRequestMaxSize int esdtPrefix string } @@ -43,7 +42,6 @@ func NewSovereignIndexTokensHandler(mainChainElastic factory.MainChainElastic, e return &sovereignIndexTokensHandler{ mainChainElasticClient: mainChainElasticClient, - bulkRequestMaxSize: mainChainElastic.BulkRequestMaxSize, esdtPrefix: esdtPrefix, }, nil } From 57b1927831ff77967158ebda498711d2a9268974 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Thu, 23 Jan 2025 18:26:26 +0200 Subject: [PATCH 30/50] fixes after testing --- process/elasticproc/tokens/sovereignIndexTokensHandler.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/process/elasticproc/tokens/sovereignIndexTokensHandler.go b/process/elasticproc/tokens/sovereignIndexTokensHandler.go index b9dd78be..f97e4e21 100644 --- a/process/elasticproc/tokens/sovereignIndexTokensHandler.go +++ b/process/elasticproc/tokens/sovereignIndexTokensHandler.go @@ -83,6 +83,10 @@ func (sit *sovereignIndexTokensHandler) getTokensFromScrs(elasticClient elasticp } } + if len(receivedTokensIDs) == 0 { + return make([]string, 0), nil + } + responseTokens := &data.ResponseTokens{} err := elasticClient.DoMultiGet(context.Background(), receivedTokensIDs, indexerdata.TokensIndex, true, responseTokens) if err != nil { From b5ebb3fb12f4924d2989bf43eca7e60c4b10c618 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Mon, 27 Jan 2025 18:11:14 +0200 Subject: [PATCH 31/50] fixes and improvements after self review --- cmd/elasticindexer/config/prefs.toml | 3 +- config/config.go | 1 + docker-compose.yml | 16 ++ .../sovereignRunTypeComponentsFactory.go | 4 +- factory/wsIndexerFactory.go | 3 +- integrationtests/consts.go | 4 +- integrationtests/incomingSCR_test.go | 238 +++++++++++++++--- .../testdata/incomingSCR/incoming-scr.json | 19 +- integrationtests/utils.go | 2 +- .../factory/elasticProcessorFactory.go | 4 +- .../elasticproc/tokens/indexTokensHandler.go | 2 + .../tokens/indexTokensHandler_test.go | 22 ++ .../tokens/sovereignIndexTokensHandler.go | 51 +++- .../sovereignIndexTokensHandler_test.go | 65 +++++ process/factory/indexerFactory.go | 2 +- 15 files changed, 375 insertions(+), 61 deletions(-) create mode 100644 process/elasticproc/tokens/indexTokensHandler_test.go create mode 100644 process/elasticproc/tokens/sovereignIndexTokensHandler_test.go diff --git a/cmd/elasticindexer/config/prefs.toml b/cmd/elasticindexer/config/prefs.toml index 917f006d..7c320e89 100644 --- a/cmd/elasticindexer/config/prefs.toml +++ b/cmd/elasticindexer/config/prefs.toml @@ -25,6 +25,7 @@ bulk-request-max-size-in-bytes = 4194304 # 4MB [config.main-elastic-cluster] - url = "https://testnet-index.multiversx.com" + enabled = true + url = "http://localhost:9201" username = "" password = "" diff --git a/config/config.go b/config/config.go index 217b5a92..b5d3b83a 100644 --- a/config/config.go +++ b/config/config.go @@ -54,6 +54,7 @@ type ClusterConfig struct { BulkRequestMaxSizeInBytes int `toml:"bulk-request-max-size-in-bytes"` } `toml:"elastic-cluster"` MainChainCluster struct { + Enabled bool `toml:"enabled"` URL string `toml:"url"` UserName string `toml:"username"` Password string `toml:"password"` diff --git a/docker-compose.yml b/docker-compose.yml index 970f0b40..a5a27b1f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,22 @@ services: ports: - "9200:9200" - "9300:9300" + elasticsearch2: + container_name: es-container2 + image: docker.elastic.co/elasticsearch/elasticsearch:7.16.1 + environment: + - "discovery.type=single-node" + - "xpack.security.enabled=false" + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + networks: + - es-net + ports: + - "9201:9200" + - "9301:9300" kibana: container_name: kb-container image: docker.elastic.co/kibana/kibana:7.16.1 diff --git a/factory/runType/sovereignRunTypeComponentsFactory.go b/factory/runType/sovereignRunTypeComponentsFactory.go index 53c8d767..2adb96c7 100644 --- a/factory/runType/sovereignRunTypeComponentsFactory.go +++ b/factory/runType/sovereignRunTypeComponentsFactory.go @@ -7,12 +7,12 @@ import ( ) type sovereignRunTypeComponentsFactory struct { - mainChainElastic factory.MainChainElastic + mainChainElastic factory.ElasticConfig esdtPrefix string } // NewSovereignRunTypeComponentsFactory will return a new instance of sovereign run type components factory -func NewSovereignRunTypeComponentsFactory(mainChainElastic factory.MainChainElastic, esdtPrefix string) *sovereignRunTypeComponentsFactory { +func NewSovereignRunTypeComponentsFactory(mainChainElastic factory.ElasticConfig, esdtPrefix string) *sovereignRunTypeComponentsFactory { return &sovereignRunTypeComponentsFactory{ mainChainElastic: mainChainElastic, esdtPrefix: esdtPrefix, diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index 1b622e30..74bc00e4 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -77,7 +77,8 @@ func createDataIndexer( return nil, err } - mainChainElastic := esFactory.MainChainElastic{ + mainChainElastic := esFactory.ElasticConfig{ + Enabled: clusterCfg.Config.MainChainCluster.Enabled, Url: clusterCfg.Config.MainChainCluster.URL, UserName: clusterCfg.Config.MainChainCluster.UserName, Password: clusterCfg.Config.MainChainCluster.Password, diff --git a/integrationtests/consts.go b/integrationtests/consts.go index 04b25a09..9c68a127 100644 --- a/integrationtests/consts.go +++ b/integrationtests/consts.go @@ -4,8 +4,8 @@ const ( //nolint testNumOfShards = 3 //nolint - esURL = "http://localhost:9200" - testnetEsURL = "https://testnet-index.multiversx.com" + esURL = "http://localhost:9200" + es2URL = "http://localhost:9201" //nolint addressPrefix = "erd" ) diff --git a/integrationtests/incomingSCR_test.go b/integrationtests/incomingSCR_test.go index cfa6ff3e..41971e9d 100644 --- a/integrationtests/incomingSCR_test.go +++ b/integrationtests/incomingSCR_test.go @@ -5,75 +5,119 @@ package integrationtests import ( "context" "encoding/hex" + "encoding/json" "math/big" "testing" "github.com/multiversx/mx-chain-core-go/core" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/multiversx/mx-chain-core-go/data/esdt" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/smartContractResult" + "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/stretchr/testify/require" indexerData "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" ) -func TestSovereignTransactionWithScCallSuccess(t *testing.T) { +type esToken struct { + Identifier string + Value *big.Int + NumDecimals int64 +} + +type esEsdt struct { + Identifier string + Nonce uint64 + EsdtType string + Data esdt.ESDigitalToken +} + +func createTokens() ([]esToken, []esEsdt) { + tokens := []esToken{} + token1 := esToken{ + Identifier: "TKN18-1a2b3c", + Value: big.NewInt(123), + NumDecimals: 18, + } + tokens = append(tokens, token1) + token2 := esToken{ + Identifier: "TKN12-1c2b3a", + Value: big.NewInt(333), + NumDecimals: 12, + } + tokens = append(tokens, token2) + + nfts := []esEsdt{} + nft := esEsdt{ + Identifier: "NFT-abc123", + Nonce: 1, + EsdtType: core.NonFungibleESDTv2, + Data: esdt.ESDigitalToken{ + Type: uint32(core.NonFungibleV2), + Value: big.NewInt(1), + Properties: []byte("3032"), + TokenMetaData: &esdt.MetaData{ + Nonce: 1, + Creator: []byte("creator"), + }, + }, + } + nfts = append(nfts, nft) + + return tokens, nfts +} + +func TestCrossChainTokensIndexingFromMainChain(t *testing.T) { setLogLevelDebug() - mainChainEs := factory.MainChainElastic{ - Url: testnetEsURL, + mainChainEs := factory.ElasticConfig{ + Url: es2URL, } + tokens, nfts := createTokens() + createTokensInSourceEs(t, mainChainEs, tokens, nfts) + esClient, err := createESClient(esURL) require.Nil(t, err) esProc, err := CreateSovereignElasticProcessor(esClient, mainChainEs) require.Nil(t, err) - txHash := []byte("txHash") + allTokens := getAllTokensIDs(tokens, nfts) + genericResponse := &GenericResponse{} + err = esClient.DoMultiGet(context.Background(), allTokens, indexerData.TokensIndex, true, genericResponse) + require.Nil(t, err) + for _, token := range genericResponse.Docs { + require.False(t, token.Found) + } + + scrHash := []byte("scrHash") header := &dataBlock.Header{ - Round: 50, - TimeStamp: 5040, + Round: 10, + TimeStamp: 2500, } body := &dataBlock.Body{ MiniBlocks: dataBlock.MiniBlockSlice{ { Type: dataBlock.SmartContractResultBlock, - SenderShardID: 4294967293, - ReceiverShardID: 0, - TxHashes: [][]byte{txHash}, + SenderShardID: core.MainChainShardId, + ReceiverShardID: core.SovereignChainShardId, + TxHashes: [][]byte{scrHash}, }, }, } - token1Identifier := "AGE-be2571" - token2Identifier := "BGD16-c47f46" - data := []byte(core.BuiltInFunctionMultiESDTNFTTransfer + - "@02" + - "@" + hex.EncodeToString([]byte(token1Identifier)) + - "@" + - "@" + hex.EncodeToString(big.NewInt(123).Bytes()) + - "@" + hex.EncodeToString([]byte(token2Identifier)) + - "@" + - "@" + hex.EncodeToString(big.NewInt(333).Bytes())) - - genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(context.Background(), []string{token1Identifier, token2Identifier}, indexerData.TokensIndex, true, genericResponse) - require.Nil(t, err) - for _, token := range genericResponse.Docs { - require.False(t, token.Found) - } - pool := &outport.TransactionPool{ SmartContractResults: map[string]*outport.SCRInfo{ - hex.EncodeToString(txHash): {SmartContractResult: &smartContractResult.SmartContractResult{ + hex.EncodeToString(scrHash): {SmartContractResult: &smartContractResult.SmartContractResult{ Nonce: 11, Value: big.NewInt(0), GasLimit: 0, SndAddr: decodeAddress("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u"), RcvAddr: decodeAddress("erd1kzrfl2tztgzjpeedwec37c8npcr0a2ulzh9lhmj7xufyg23zcxuqxcqz0s"), - Data: data, + Data: createMultiEsdtTransferData(tokens, nfts), OriginalTxHash: nil, }, FeeInfo: &outport.FeeInfo{}}, }, @@ -81,9 +125,8 @@ func TestSovereignTransactionWithScCallSuccess(t *testing.T) { err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, 1)) require.Nil(t, err) - ids := []string{hex.EncodeToString(txHash)} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(context.Background(), ids, indexerData.ScResultsIndex, true, genericResponse) + err = esClient.DoMultiGet(context.Background(), []string{hex.EncodeToString(scrHash)}, indexerData.ScResultsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/incomingSCR/incoming-scr.json"), @@ -91,9 +134,140 @@ func TestSovereignTransactionWithScCallSuccess(t *testing.T) { ) genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(context.Background(), []string{token1Identifier, token2Identifier}, indexerData.TokensIndex, true, genericResponse) + err = esClient.DoMultiGet(context.Background(), allTokens, indexerData.TokensIndex, true, genericResponse) + require.Nil(t, err) + for _, token := range genericResponse.Docs { + require.True(t, token.Found) + } +} + +func createTokensInSourceEs(t *testing.T, es factory.ElasticConfig, tokens []esToken, nfts []esEsdt) { + esClient, err := createESClient(es.Url) + require.Nil(t, err) + + esProc, err := CreateElasticProcessor(esClient) + require.Nil(t, err) + + body := &dataBlock.Body{} + header := &dataBlock.Header{ + Round: 50, + TimeStamp: 5040, + ShardID: core.MetachainShardId, + } + + address1 := "erd1k04pxr6c0gvlcx4rd5fje0a4uy33axqxwz0fpcrgtfdy3nrqauqqgvxprv" + + // create issue token and nft collection events + events := make([]*transaction.Event, 0) + for _, token := range tokens { + events = append(events, &transaction.Event{ + Address: decodeAddress(address1), + Identifier: []byte("issue"), + Topics: [][]byte{[]byte(token.Identifier), []byte("TKN"), []byte("TKN"), []byte(core.FungibleESDT), big.NewInt(token.NumDecimals).Bytes()}, + }) + } + for _, nft := range nfts { + events = append(events, &transaction.Event{ + Address: decodeAddress(address1), + Identifier: []byte("issueNonFungible"), + Topics: [][]byte{[]byte(nft.Identifier), []byte("NFT"), []byte("NFT"), []byte(nft.EsdtType)}, + }) + } + + pool := &outport.TransactionPool{ + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("txHash1")), + Log: &transaction.Log{ + Address: decodeAddress(address1), + Events: events, + }, + }, + }, + } + + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) + require.Nil(t, err) + + genericResponse := &GenericResponse{} + allTokens := getAllTokensIDs(tokens, nfts) + err = esClient.DoMultiGet(context.Background(), allTokens, indexerData.TokensIndex, true, genericResponse) require.Nil(t, err) for _, token := range genericResponse.Docs { require.True(t, token.Found) } + + // create nft event + events = make([]*transaction.Event, 0) + for _, nft := range nfts { + nftDataBytes, _ := json.Marshal(nft.Data) + + events = append(events, &transaction.Event{ + Address: decodeAddress(address1), + Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), + Topics: [][]byte{[]byte(nft.Identifier), big.NewInt(int64(nft.Nonce)).Bytes(), nft.Data.Value.Bytes(), []byte(nftDataBytes)}, + }) + } + + header = &dataBlock.Header{ + Round: 51, + TimeStamp: 5600, + ShardID: 0, + } + + pool = &outport.TransactionPool{ + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("txHash2")), + Log: &transaction.Log{ + Address: decodeAddress(address1), + Events: events, + }, + }, + }, + } + + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) + require.Nil(t, err) + + allNfts := make([]string, 0) + for _, nft := range nfts { + allNfts = append(allNfts, nft.Identifier+"-"+hex.EncodeToString(big.NewInt(int64(nft.Nonce)).Bytes())) + } + err = esClient.DoMultiGet(context.Background(), allNfts, indexerData.TokensIndex, true, genericResponse) + require.Nil(t, err) + for _, token := range genericResponse.Docs { + require.True(t, token.Found) + } +} + +func getAllTokensIDs(tokens []esToken, nfts []esEsdt) []string { + allTokens := make([]string, 0) + for _, token := range tokens { + allTokens = append(allTokens, token.Identifier) + } + for _, nft := range nfts { + allTokens = append(allTokens, nft.Identifier) + } + return allTokens +} + +func createMultiEsdtTransferData(tokens []esToken, nfts []esEsdt) []byte { + data := []byte(core.BuiltInFunctionMultiESDTNFTTransfer + + "@" + hex.EncodeToString(big.NewInt(int64(len(tokens)+len(nfts))).Bytes())) + for _, token := range tokens { + data = append(data, []byte( + "@"+hex.EncodeToString([]byte(token.Identifier))+ + "@"+ + "@"+hex.EncodeToString(token.Value.Bytes()))...) + } + for _, nft := range nfts { + nftDataBytes, _ := json.Marshal(nft.Data) + data = append(data, []byte( + "@"+hex.EncodeToString([]byte(nft.Identifier))+ + "@"+hex.EncodeToString(big.NewInt(int64(nft.Nonce)).Bytes())+ + "@"+hex.EncodeToString(nftDataBytes))...) + } + + return data } diff --git a/integrationtests/testdata/incomingSCR/incoming-scr.json b/integrationtests/testdata/incomingSCR/incoming-scr.json index 09eaf4fb..29253e9a 100644 --- a/integrationtests/testdata/incomingSCR/incoming-scr.json +++ b/integrationtests/testdata/incomingSCR/incoming-scr.json @@ -1,5 +1,5 @@ { - "miniBlockHash": "64b954ab72a867bf9866556c7aac3b80a51a69a1d53c640e78f53e5bfcdcc37d", + "miniBlockHash": "71e255368d7a6686a57a1acb8845953fc54e5a1cfde395acd09df58cb61d5abb", "nonce": 11, "gasLimit": 0, "gasPrice": 0, @@ -9,28 +9,33 @@ "receiver": "erd1kzrfl2tztgzjpeedwec37c8npcr0a2ulzh9lhmj7xufyg23zcxuqxcqz0s", "senderShard": 4294967293, "receiverShard": 0, - "data": "TXVsdGlFU0RUTkZUVHJhbnNmZXJAMDJANDE0NzQ1MmQ2MjY1MzIzNTM3MzFAQDdiQDQyNDc0NDMxMzYyZDYzMzQzNzY2MzQzNkBAMDE0ZA==", + "data": "TXVsdGlFU0RUTkZUVHJhbnNmZXJAMDNANTQ0YjRlMzEzODJkMzE2MTMyNjIzMzYzQEA3YkA1NDRiNGUzMTMyMmQzMTYzMzI2MjMzNjFAQDAxNGRANGU0NjU0MmQ2MTYyNjMzMTMyMzNAMDFAN2IyMjU0Nzk3MDY1MjIzYTMyMmMyMjU2NjE2Yzc1NjUyMjNhMzEyYzIyNTA3MjZmNzA2NTcyNzQ2OTY1NzMyMjNhMjI0ZDdhNDE3YTRkNjczZDNkMjIyYzIyNGQ2NTc0NjE0NDYxNzQ2MTIyM2E3YjIyNGU2ZjZlNjM2NTIyM2EzMTJjMjI0ZTYxNmQ2NTIyM2E2ZTc1NmM2YzJjMjI0MzcyNjU2MTc0NmY3MjIyM2EyMjU5MzM0YTZjNTk1ODUyNzY2MzY3M2QzZDIyMmMyMjUyNmY3OTYxNmM3NDY5NjU3MzIyM2EzMDJjMjI0ODYxNzM2ODIyM2E2ZTc1NmM2YzJjMjI1NTUyNDk3MzIyM2E2ZTc1NmM2YzJjMjI0MTc0NzQ3MjY5NjI3NTc0NjU3MzIyM2E2ZTc1NmM2YzdkMmMyMjUyNjU3MzY1NzI3NjY1NjQyMjNhNmU3NTZjNmM3ZA==", "prevTxHash": "", "originalTxHash": "", "callType": "0", - "timestamp": 5040, + "timestamp": 2500, "tokens": [ - "AGE-be2571", - "BGD16-c47f46" + "TKN18-1a2b3c", + "TKN12-1c2b3a", + "NFT-abc123-01" ], "esdtValues": [ "123", - "333" + "333", + "1" ], "esdtValuesNum": [ 1.23e-16, - 3.33e-16 + 3.33e-16, + 1e-18 ], "receivers": [ + "erd1kzrfl2tztgzjpeedwec37c8npcr0a2ulzh9lhmj7xufyg23zcxuqxcqz0s", "erd1kzrfl2tztgzjpeedwec37c8npcr0a2ulzh9lhmj7xufyg23zcxuqxcqz0s", "erd1kzrfl2tztgzjpeedwec37c8npcr0a2ulzh9lhmj7xufyg23zcxuqxcqz0s" ], "receiversShardIDs": [ + 0, 0, 0 ], diff --git a/integrationtests/utils.go b/integrationtests/utils.go index e2eab567..0577ca81 100644 --- a/integrationtests/utils.go +++ b/integrationtests/utils.go @@ -76,7 +76,7 @@ func CreateElasticProcessor( // CreateSovereignElasticProcessor - func CreateSovereignElasticProcessor( esClient elasticproc.DatabaseClientHandler, - mainChainEs factory.MainChainElastic, + mainChainEs factory.ElasticConfig, ) (dataindexer.ElasticProcessor, error) { sovIndexTokens, _ := tokens.NewSovereignIndexTokensHandler(mainChainEs, sovEsdtPrefix) diff --git a/process/elasticproc/factory/elasticProcessorFactory.go b/process/elasticproc/factory/elasticProcessorFactory.go index c81053d0..2471885f 100644 --- a/process/elasticproc/factory/elasticProcessorFactory.go +++ b/process/elasticproc/factory/elasticProcessorFactory.go @@ -19,8 +19,8 @@ import ( "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/validators" ) -// MainChainElastic holds the elastic search settings -type MainChainElastic struct { +// ElasticConfig holds the elastic search settings +type ElasticConfig struct { Enabled bool Url string UserName string diff --git a/process/elasticproc/tokens/indexTokensHandler.go b/process/elasticproc/tokens/indexTokensHandler.go index e48287fd..d8d78e7c 100644 --- a/process/elasticproc/tokens/indexTokensHandler.go +++ b/process/elasticproc/tokens/indexTokensHandler.go @@ -7,10 +7,12 @@ import ( type indexTokensHandler struct{} +// NewIndexTokensHandler creates a new index tokens handler func NewIndexTokensHandler() *indexTokensHandler { return &indexTokensHandler{} } +// IndexCrossChainTokens returns no error func (it *indexTokensHandler) IndexCrossChainTokens(_ elasticproc.DatabaseClientHandler, _ []*data.ScResult, _ *data.BufferSlice) error { return nil } diff --git a/process/elasticproc/tokens/indexTokensHandler_test.go b/process/elasticproc/tokens/indexTokensHandler_test.go new file mode 100644 index 00000000..260c39af --- /dev/null +++ b/process/elasticproc/tokens/indexTokensHandler_test.go @@ -0,0 +1,22 @@ +package tokens + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNewIndexTokensHandler(t *testing.T) { + t.Parallel() + + ith := NewIndexTokensHandler() + require.False(t, ith.IsInterfaceNil()) +} + +func TestIndexTokensHandler_IndexCrossChainTokens(t *testing.T) { + t.Parallel() + + ith := NewIndexTokensHandler() + err := ith.IndexCrossChainTokens(nil, nil, nil) + require.NoError(t, err) +} diff --git a/process/elasticproc/tokens/sovereignIndexTokensHandler.go b/process/elasticproc/tokens/sovereignIndexTokensHandler.go index f97e4e21..af9ebea0 100644 --- a/process/elasticproc/tokens/sovereignIndexTokensHandler.go +++ b/process/elasticproc/tokens/sovereignIndexTokensHandler.go @@ -6,6 +6,7 @@ import ( "fmt" "math" "net/http" + "strings" "time" "github.com/elastic/go-elasticsearch/v7" @@ -22,25 +23,32 @@ import ( ) type sovereignIndexTokensHandler struct { + indexingEnabled bool mainChainElasticClient elasticproc.DatabaseClientHandler esdtPrefix string } -func NewSovereignIndexTokensHandler(mainChainElastic factory.MainChainElastic, esdtPrefix string) (*sovereignIndexTokensHandler, error) { - argsEsClient := elasticsearch.Config{ - Addresses: []string{mainChainElastic.Url}, - Username: mainChainElastic.UserName, - Password: mainChainElastic.Password, - Logger: &logging.CustomLogger{}, - RetryOnStatus: []int{http.StatusConflict}, - RetryBackoff: retryBackOff, - } - mainChainElasticClient, err := client.NewElasticClient(argsEsClient) - if err != nil { - return nil, err +// NewSovereignIndexTokensHandler creates a new sovereign index tokens handler +func NewSovereignIndexTokensHandler(mainChainElastic factory.ElasticConfig, esdtPrefix string) (*sovereignIndexTokensHandler, error) { + var mainChainElasticClient elasticproc.DatabaseClientHandler + if mainChainElastic.Enabled { + var err error + argsEsClient := elasticsearch.Config{ + Addresses: []string{mainChainElastic.Url}, + Username: mainChainElastic.UserName, + Password: mainChainElastic.Password, + Logger: &logging.CustomLogger{}, + RetryOnStatus: []int{http.StatusConflict}, + RetryBackoff: retryBackOff, + } + mainChainElasticClient, err = client.NewElasticClient(argsEsClient) + if err != nil { + return nil, err + } } return &sovereignIndexTokensHandler{ + indexingEnabled: mainChainElastic.Enabled, mainChainElasticClient: mainChainElasticClient, esdtPrefix: esdtPrefix, }, nil @@ -50,7 +58,12 @@ func retryBackOff(attempt int) time.Duration { return time.Duration(math.Exp2(float64(attempt))) * time.Second } +// IndexCrossChainTokens will index the new tokens properties func (sit *sovereignIndexTokensHandler) IndexCrossChainTokens(elasticClient elasticproc.DatabaseClientHandler, scrs []*data.ScResult, buffSlice *data.BufferSlice) error { + if !sit.indexingEnabled { + return nil + } + notFoundTokens, err := sit.getTokensFromScrs(elasticClient, scrs) if err != nil { return err @@ -79,6 +92,9 @@ func (sit *sovereignIndexTokensHandler) getTokensFromScrs(elasticClient elasticp if !hasPrefix || tokenPrefix != sit.esdtPrefix { receivedTokensIDs = append(receivedTokensIDs, token) } + if isEsdt, tokenCollection := getTokenCollection(hasPrefix, token); isEsdt { + receivedTokensIDs = append(receivedTokensIDs, tokenCollection) + } } } } @@ -103,6 +119,17 @@ func (sit *sovereignIndexTokensHandler) getTokensFromScrs(elasticClient elasticp return newTokens, nil } +func getTokenCollection(hasPrefix bool, tokenIdentifier string) (bool, string) { + tokenSplit := strings.Split(tokenIdentifier, "-") + if !hasPrefix && len(tokenSplit) == 3 { + return true, tokenSplit[0] + "-" + tokenSplit[1] + } + if hasPrefix && len(tokenSplit) == 4 { + return true, tokenSplit[1] + "-" + tokenSplit[2] + } + return false, "" +} + func (sit *sovereignIndexTokensHandler) indexNewTokens(responseTokensInfo []data.ResponseTokenInfoDB, buffSlice *data.BufferSlice) error { for _, responseToken := range responseTokensInfo { token := formatToken(responseToken) diff --git a/process/elasticproc/tokens/sovereignIndexTokensHandler_test.go b/process/elasticproc/tokens/sovereignIndexTokensHandler_test.go new file mode 100644 index 00000000..3aebe667 --- /dev/null +++ b/process/elasticproc/tokens/sovereignIndexTokensHandler_test.go @@ -0,0 +1,65 @@ +package tokens + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/multiversx/mx-chain-es-indexer-go/data" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" +) + +const ( + prefix = "sov" +) + +func createElasticConfig() factory.ElasticConfig { + return factory.ElasticConfig{ + Enabled: true, + Url: "http://localhost:9200", + UserName: "", + Password: "", + } +} + +func TestSovereignNewIndexTokensHandler(t *testing.T) { + t.Parallel() + + t.Run("no url, should error", func(t *testing.T) { + esConfig := createElasticConfig() + esConfig.Url = "http://bad url" + sith, err := NewSovereignIndexTokensHandler(esConfig, prefix) + require.ErrorContains(t, err, "cannot parse url") + require.Nil(t, sith) + }) + t.Run("not enabled should not create main chain elastic client", func(t *testing.T) { + esConfig := createElasticConfig() + esConfig.Enabled = false + sith, err := NewSovereignIndexTokensHandler(esConfig, prefix) + require.NoError(t, err) + require.NotNil(t, sith) + require.Nil(t, sith.mainChainElasticClient) + }) + t.Run("valid config, should work", func(t *testing.T) { + esConfig := createElasticConfig() + sith, err := NewSovereignIndexTokensHandler(esConfig, prefix) + require.NoError(t, err) + require.NotNil(t, sith) + }) +} + +func TestSovereignIndexTokensHandler_IndexCrossChainTokens(t *testing.T) { + t.Parallel() + + esConfig := createElasticConfig() + esConfig.Enabled = false + sith, err := NewSovereignIndexTokensHandler(esConfig, prefix) + require.NoError(t, err) + require.NotNil(t, sith) + + // should skip indexing + err = sith.IndexCrossChainTokens(nil, make([]*data.ScResult, 0), data.NewBufferSlice(0)) + require.NoError(t, err) + + // actual indexing is tested in TestCrossChainTokensIndexingFromMainChain +} diff --git a/process/factory/indexerFactory.go b/process/factory/indexerFactory.go index ca18d60d..075516a6 100644 --- a/process/factory/indexerFactory.go +++ b/process/factory/indexerFactory.go @@ -34,7 +34,7 @@ type ArgsIndexerFactory struct { ImportDB bool Sovereign bool ESDTPrefix string - MainChainElastic factory.MainChainElastic + MainChainElastic factory.ElasticConfig Denomination int BulkRequestMaxSize int Url string From 9659e73226114e15095c4e9abe793f7c9cacf9de Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Mon, 27 Jan 2025 18:17:42 +0200 Subject: [PATCH 32/50] fixes after self review --- integrationtests/testdata/incomingSCR/incoming-scr.json | 2 +- process/elasticproc/tokens/indexTokensHandler.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/integrationtests/testdata/incomingSCR/incoming-scr.json b/integrationtests/testdata/incomingSCR/incoming-scr.json index 29253e9a..4b53272c 100644 --- a/integrationtests/testdata/incomingSCR/incoming-scr.json +++ b/integrationtests/testdata/incomingSCR/incoming-scr.json @@ -40,4 +40,4 @@ 0 ], "operation": "MultiESDTNFTTransfer" -} \ No newline at end of file +} diff --git a/process/elasticproc/tokens/indexTokensHandler.go b/process/elasticproc/tokens/indexTokensHandler.go index d8d78e7c..2f45c134 100644 --- a/process/elasticproc/tokens/indexTokensHandler.go +++ b/process/elasticproc/tokens/indexTokensHandler.go @@ -12,7 +12,7 @@ func NewIndexTokensHandler() *indexTokensHandler { return &indexTokensHandler{} } -// IndexCrossChainTokens returns no error +// IndexCrossChainTokens should do nothing and return no error func (it *indexTokensHandler) IndexCrossChainTokens(_ elasticproc.DatabaseClientHandler, _ []*data.ScResult, _ *data.BufferSlice) error { return nil } From 372b46ae2461439a017d9ec2b1629992a2658546 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Tue, 28 Jan 2025 11:21:09 +0200 Subject: [PATCH 33/50] integration test fixes --- factory/runType/runTypeComponents_test.go | 6 ++- .../sovereignRunTypeComponentsFactory_test.go | 7 ++- integrationtests/consts.go | 3 +- integrationtests/incomingSCR_test.go | 53 +++++++++++-------- .../testdata/incomingSCR/incoming-scr.json | 2 +- .../tokens/sovereignIndexTokensHandler.go | 14 +++-- 6 files changed, 51 insertions(+), 34 deletions(-) diff --git a/factory/runType/runTypeComponents_test.go b/factory/runType/runTypeComponents_test.go index 07580802..9f929ea2 100644 --- a/factory/runType/runTypeComponents_test.go +++ b/factory/runType/runTypeComponents_test.go @@ -21,8 +21,9 @@ func TestRunTypeComponentsFactory_Create(t *testing.T) { rtcf := NewRunTypeComponentsFactory() require.NotNil(t, rtcf) - rtc := rtcf.Create() + rtc, err := rtcf.Create() require.NotNil(t, rtc) + require.NoError(t, err) } func TestRunTypeComponentsFactory_Close(t *testing.T) { @@ -31,8 +32,9 @@ func TestRunTypeComponentsFactory_Close(t *testing.T) { rtcf := NewRunTypeComponentsFactory() require.NotNil(t, rtcf) - rtc := rtcf.Create() + rtc, err := rtcf.Create() require.NotNil(t, rtc) + require.NoError(t, err) require.NoError(t, rtc.Close()) } diff --git a/factory/runType/sovereignRunTypeComponentsFactory_test.go b/factory/runType/sovereignRunTypeComponentsFactory_test.go index 71d909ba..4bb3581b 100644 --- a/factory/runType/sovereignRunTypeComponentsFactory_test.go +++ b/factory/runType/sovereignRunTypeComponentsFactory_test.go @@ -4,16 +4,19 @@ import ( "testing" "github.com/stretchr/testify/require" + + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" ) func TestSovereignRunTypeComponentsFactory_CreateAndClose(t *testing.T) { t.Parallel() - srtcf := NewSovereignRunTypeComponentsFactory() + srtcf := NewSovereignRunTypeComponentsFactory(factory.ElasticConfig{}, "sov") require.False(t, srtcf.IsInterfaceNil()) - srtc := srtcf.Create() + srtc, err := srtcf.Create() require.NotNil(t, srtc) + require.NoError(t, err) require.NoError(t, srtc.Close()) } diff --git a/integrationtests/consts.go b/integrationtests/consts.go index 9c68a127..f06babca 100644 --- a/integrationtests/consts.go +++ b/integrationtests/consts.go @@ -4,7 +4,8 @@ const ( //nolint testNumOfShards = 3 //nolint - esURL = "http://localhost:9200" + esURL = "http://localhost:9200" + //nolint es2URL = "http://localhost:9201" //nolint addressPrefix = "erd" diff --git a/integrationtests/incomingSCR_test.go b/integrationtests/incomingSCR_test.go index 41971e9d..e35d08b9 100644 --- a/integrationtests/incomingSCR_test.go +++ b/integrationtests/incomingSCR_test.go @@ -27,14 +27,13 @@ type esToken struct { NumDecimals int64 } -type esEsdt struct { - Identifier string +type esNft struct { + Collection string Nonce uint64 - EsdtType string Data esdt.ESDigitalToken } -func createTokens() ([]esToken, []esEsdt) { +func createTokens() ([]esToken, []esNft) { tokens := []esToken{} token1 := esToken{ Identifier: "TKN18-1a2b3c", @@ -49,18 +48,19 @@ func createTokens() ([]esToken, []esEsdt) { } tokens = append(tokens, token2) - nfts := []esEsdt{} - nft := esEsdt{ - Identifier: "NFT-abc123", + nfts := []esNft{} + nft := esNft{ + Collection: "NFT-abc123", Nonce: 1, - EsdtType: core.NonFungibleESDTv2, Data: esdt.ESDigitalToken{ Type: uint32(core.NonFungibleV2), Value: big.NewInt(1), Properties: []byte("3032"), TokenMetaData: &esdt.MetaData{ - Nonce: 1, - Creator: []byte("creator"), + Nonce: 1, + Name: []byte("NFT"), + Creator: []byte("creator"), + Royalties: uint32(2500), }, }, } @@ -73,7 +73,8 @@ func TestCrossChainTokensIndexingFromMainChain(t *testing.T) { setLogLevelDebug() mainChainEs := factory.ElasticConfig{ - Url: es2URL, + Enabled: true, + Url: es2URL, } tokens, nfts := createTokens() @@ -86,6 +87,7 @@ func TestCrossChainTokensIndexingFromMainChain(t *testing.T) { require.Nil(t, err) allTokens := getAllTokensIDs(tokens, nfts) + allTokens = append(allTokens, getAllNftIDs(nfts)...) genericResponse := &GenericResponse{} err = esClient.DoMultiGet(context.Background(), allTokens, indexerData.TokensIndex, true, genericResponse) require.Nil(t, err) @@ -141,7 +143,7 @@ func TestCrossChainTokensIndexingFromMainChain(t *testing.T) { } } -func createTokensInSourceEs(t *testing.T, es factory.ElasticConfig, tokens []esToken, nfts []esEsdt) { +func createTokensInSourceEs(t *testing.T, es factory.ElasticConfig, tokens []esToken, nfts []esNft) { esClient, err := createESClient(es.Url) require.Nil(t, err) @@ -170,7 +172,7 @@ func createTokensInSourceEs(t *testing.T, es factory.ElasticConfig, tokens []esT events = append(events, &transaction.Event{ Address: decodeAddress(address1), Identifier: []byte("issueNonFungible"), - Topics: [][]byte{[]byte(nft.Identifier), []byte("NFT"), []byte("NFT"), []byte(nft.EsdtType)}, + Topics: [][]byte{[]byte(nft.Collection), []byte("NFT"), []byte("NFT"), []byte(core.ESDTType(nft.Data.Type).String())}, }) } @@ -205,7 +207,7 @@ func createTokensInSourceEs(t *testing.T, es factory.ElasticConfig, tokens []esT events = append(events, &transaction.Event{ Address: decodeAddress(address1), Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), - Topics: [][]byte{[]byte(nft.Identifier), big.NewInt(int64(nft.Nonce)).Bytes(), nft.Data.Value.Bytes(), []byte(nftDataBytes)}, + Topics: [][]byte{[]byte(nft.Collection), big.NewInt(0).SetUint64(nft.Nonce).Bytes(), nft.Data.Value.Bytes(), []byte(nftDataBytes)}, }) } @@ -230,10 +232,7 @@ func createTokensInSourceEs(t *testing.T, es factory.ElasticConfig, tokens []esT err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) - allNfts := make([]string, 0) - for _, nft := range nfts { - allNfts = append(allNfts, nft.Identifier+"-"+hex.EncodeToString(big.NewInt(int64(nft.Nonce)).Bytes())) - } + allNfts := getAllNftIDs(nfts) err = esClient.DoMultiGet(context.Background(), allNfts, indexerData.TokensIndex, true, genericResponse) require.Nil(t, err) for _, token := range genericResponse.Docs { @@ -241,18 +240,26 @@ func createTokensInSourceEs(t *testing.T, es factory.ElasticConfig, tokens []esT } } -func getAllTokensIDs(tokens []esToken, nfts []esEsdt) []string { +func getAllTokensIDs(tokens []esToken, nfts []esNft) []string { allTokens := make([]string, 0) for _, token := range tokens { allTokens = append(allTokens, token.Identifier) } for _, nft := range nfts { - allTokens = append(allTokens, nft.Identifier) + allTokens = append(allTokens, nft.Collection) } return allTokens } -func createMultiEsdtTransferData(tokens []esToken, nfts []esEsdt) []byte { +func getAllNftIDs(nfts []esNft) []string { + allNfts := make([]string, 0) + for _, nft := range nfts { + allNfts = append(allNfts, nft.Collection+"-"+hex.EncodeToString(big.NewInt(0).SetUint64(nft.Nonce).Bytes())) + } + return allNfts +} + +func createMultiEsdtTransferData(tokens []esToken, nfts []esNft) []byte { data := []byte(core.BuiltInFunctionMultiESDTNFTTransfer + "@" + hex.EncodeToString(big.NewInt(int64(len(tokens)+len(nfts))).Bytes())) for _, token := range tokens { @@ -264,8 +271,8 @@ func createMultiEsdtTransferData(tokens []esToken, nfts []esEsdt) []byte { for _, nft := range nfts { nftDataBytes, _ := json.Marshal(nft.Data) data = append(data, []byte( - "@"+hex.EncodeToString([]byte(nft.Identifier))+ - "@"+hex.EncodeToString(big.NewInt(int64(nft.Nonce)).Bytes())+ + "@"+hex.EncodeToString([]byte(nft.Collection))+ + "@"+hex.EncodeToString(big.NewInt(0).SetUint64(nft.Nonce).Bytes())+ "@"+hex.EncodeToString(nftDataBytes))...) } diff --git a/integrationtests/testdata/incomingSCR/incoming-scr.json b/integrationtests/testdata/incomingSCR/incoming-scr.json index 4b53272c..5f4c142e 100644 --- a/integrationtests/testdata/incomingSCR/incoming-scr.json +++ b/integrationtests/testdata/incomingSCR/incoming-scr.json @@ -9,7 +9,7 @@ "receiver": "erd1kzrfl2tztgzjpeedwec37c8npcr0a2ulzh9lhmj7xufyg23zcxuqxcqz0s", "senderShard": 4294967293, "receiverShard": 0, - "data": "TXVsdGlFU0RUTkZUVHJhbnNmZXJAMDNANTQ0YjRlMzEzODJkMzE2MTMyNjIzMzYzQEA3YkA1NDRiNGUzMTMyMmQzMTYzMzI2MjMzNjFAQDAxNGRANGU0NjU0MmQ2MTYyNjMzMTMyMzNAMDFAN2IyMjU0Nzk3MDY1MjIzYTMyMmMyMjU2NjE2Yzc1NjUyMjNhMzEyYzIyNTA3MjZmNzA2NTcyNzQ2OTY1NzMyMjNhMjI0ZDdhNDE3YTRkNjczZDNkMjIyYzIyNGQ2NTc0NjE0NDYxNzQ2MTIyM2E3YjIyNGU2ZjZlNjM2NTIyM2EzMTJjMjI0ZTYxNmQ2NTIyM2E2ZTc1NmM2YzJjMjI0MzcyNjU2MTc0NmY3MjIyM2EyMjU5MzM0YTZjNTk1ODUyNzY2MzY3M2QzZDIyMmMyMjUyNmY3OTYxNmM3NDY5NjU3MzIyM2EzMDJjMjI0ODYxNzM2ODIyM2E2ZTc1NmM2YzJjMjI1NTUyNDk3MzIyM2E2ZTc1NmM2YzJjMjI0MTc0NzQ3MjY5NjI3NTc0NjU3MzIyM2E2ZTc1NmM2YzdkMmMyMjUyNjU3MzY1NzI3NjY1NjQyMjNhNmU3NTZjNmM3ZA==", + "data": "TXVsdGlFU0RUTkZUVHJhbnNmZXJAMDNANTQ0YjRlMzEzODJkMzE2MTMyNjIzMzYzQEA3YkA1NDRiNGUzMTMyMmQzMTYzMzI2MjMzNjFAQDAxNGRANGU0NjU0MmQ2MTYyNjMzMTMyMzNAMDFAN2IyMjU0Nzk3MDY1MjIzYTMyMmMyMjU2NjE2Yzc1NjUyMjNhMzEyYzIyNTA3MjZmNzA2NTcyNzQ2OTY1NzMyMjNhMjI0ZDdhNDE3YTRkNjczZDNkMjIyYzIyNGQ2NTc0NjE0NDYxNzQ2MTIyM2E3YjIyNGU2ZjZlNjM2NTIyM2EzMTJjMjI0ZTYxNmQ2NTIyM2EyMjU0NmI1YTU1MjIyYzIyNDM3MjY1NjE3NDZmNzIyMjNhMjI1OTMzNGE2YzU5NTg1Mjc2NjM2NzNkM2QyMjJjMjI1MjZmNzk2MTZjNzQ2OTY1NzMyMjNhMzIzNTMwMzAyYzIyNDg2MTczNjgyMjNhNmU3NTZjNmMyYzIyNTU1MjQ5NzMyMjNhNmU3NTZjNmMyYzIyNDE3NDc0NzI2OTYyNzU3NDY1NzMyMjNhNmU3NTZjNmM3ZDJjMjI1MjY1NzM2NTcyNzY2NTY0MjIzYTZlNzU2YzZjN2Q=", "prevTxHash": "", "originalTxHash": "", "callType": "0", diff --git a/process/elasticproc/tokens/sovereignIndexTokensHandler.go b/process/elasticproc/tokens/sovereignIndexTokensHandler.go index af9ebea0..0f145fc6 100644 --- a/process/elasticproc/tokens/sovereignIndexTokensHandler.go +++ b/process/elasticproc/tokens/sovereignIndexTokensHandler.go @@ -111,7 +111,7 @@ func (sit *sovereignIndexTokensHandler) getTokensFromScrs(elasticClient elasticp newTokens := make([]string, 0) for _, token := range responseTokens.Docs { - if token.Found == false { + if !token.Found { newTokens = append(newTokens, token.ID) } } @@ -132,9 +132,9 @@ func getTokenCollection(hasPrefix bool, tokenIdentifier string) (bool, string) { func (sit *sovereignIndexTokensHandler) indexNewTokens(responseTokensInfo []data.ResponseTokenInfoDB, buffSlice *data.BufferSlice) error { for _, responseToken := range responseTokensInfo { - token := formatToken(responseToken) + token, identifier := formatToken(responseToken) - meta := []byte(fmt.Sprintf(`{ "index" : { "_index":"%s", "_id" : "%s" } }%s`, indexerdata.TokensIndex, converters.JsonEscape(token.Token), "\n")) + meta := []byte(fmt.Sprintf(`{ "index" : { "_index":"%s", "_id" : "%s" } }%s`, indexerdata.TokensIndex, converters.JsonEscape(identifier), "\n")) serializedTokenData, err := json.Marshal(token) if err != nil { return err @@ -149,11 +149,15 @@ func (sit *sovereignIndexTokensHandler) indexNewTokens(responseTokensInfo []data return nil } -func formatToken(token data.ResponseTokenInfoDB) data.TokenInfo { +func formatToken(token data.ResponseTokenInfoDB) (data.TokenInfo, string) { token.Source.OwnersHistory = nil token.Source.Properties = nil - return token.Source + identifier := token.Source.Identifier // for NFTs + if identifier == "" { + identifier = token.Source.Token // for tokens/collections + } + return token.Source, identifier } // IsInterfaceNil returns true if there is no value under the interface From a110c90be057269d86e54b007afe4bde36a44930 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Tue, 28 Jan 2025 11:59:44 +0200 Subject: [PATCH 34/50] unit test fixes --- process/elasticproc/check.go | 5 +++ process/elasticproc/elasticProcessor_test.go | 36 ++++++++++--------- .../factory/elasticProcessorFactory_test.go | 2 ++ process/elasticproc/indexTokenHandlerMock.go | 23 ++++++++++++ process/elasticproc/interface.go | 2 +- 5 files changed, 50 insertions(+), 18 deletions(-) create mode 100644 process/elasticproc/indexTokenHandlerMock.go diff --git a/process/elasticproc/check.go b/process/elasticproc/check.go index 9284376e..b3953d15 100644 --- a/process/elasticproc/check.go +++ b/process/elasticproc/check.go @@ -2,7 +2,9 @@ package elasticproc import ( "github.com/multiversx/mx-chain-core-go/core/check" + elasticIndexer "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" ) func checkArguments(arguments *ArgElasticProcessor) error { @@ -39,6 +41,9 @@ func checkArguments(arguments *ArgElasticProcessor) error { if check.IfNilReflect(arguments.OperationsProc) { return elasticIndexer.ErrNilOperationsHandler } + if check.IfNilReflect(arguments.IndexTokensHandler) { + return transactions.ErrNilIndexTokensHandler + } return nil } diff --git a/process/elasticproc/elasticProcessor_test.go b/process/elasticproc/elasticProcessor_test.go index b01180ac..424970cb 100644 --- a/process/elasticproc/elasticProcessor_test.go +++ b/process/elasticproc/elasticProcessor_test.go @@ -31,15 +31,16 @@ import ( func newElasticsearchProcessor(elasticsearchWriter DatabaseClientHandler, arguments *ArgElasticProcessor) *elasticProcessor { return &elasticProcessor{ - elasticClient: elasticsearchWriter, - enabledIndexes: arguments.EnabledIndexes, - blockProc: arguments.BlockProc, - transactionsProc: arguments.TransactionsProc, - miniblocksProc: arguments.MiniblocksProc, - accountsProc: arguments.AccountsProc, - validatorsProc: arguments.ValidatorsProc, - statisticsProc: arguments.StatisticsProc, - logsAndEventsProc: arguments.LogsAndEventsProc, + elasticClient: elasticsearchWriter, + enabledIndexes: arguments.EnabledIndexes, + blockProc: arguments.BlockProc, + transactionsProc: arguments.TransactionsProc, + miniblocksProc: arguments.MiniblocksProc, + accountsProc: arguments.AccountsProc, + validatorsProc: arguments.ValidatorsProc, + statisticsProc: arguments.StatisticsProc, + logsAndEventsProc: arguments.LogsAndEventsProc, + indexTokensHandler: arguments.IndexTokensHandler, } } @@ -80,14 +81,15 @@ func createMockElasticProcessorArgs() *ArgElasticProcessor { EnabledIndexes: map[string]struct{}{ dataindexer.BlockIndex: {}, dataindexer.TransactionsIndex: {}, dataindexer.MiniblocksIndex: {}, dataindexer.ValidatorsIndex: {}, dataindexer.RoundsIndex: {}, dataindexer.AccountsIndex: {}, dataindexer.RatingIndex: {}, dataindexer.AccountsHistoryIndex: {}, }, - ValidatorsProc: vp, - StatisticsProc: statistics.NewStatisticsProcessor(), - TransactionsProc: &mock.DBTransactionProcessorStub{}, - MiniblocksProc: mp, - AccountsProc: acp, - BlockProc: bp, - LogsAndEventsProc: lp, - OperationsProc: op, + ValidatorsProc: vp, + StatisticsProc: statistics.NewStatisticsProcessor(), + TransactionsProc: &mock.DBTransactionProcessorStub{}, + MiniblocksProc: mp, + AccountsProc: acp, + BlockProc: bp, + LogsAndEventsProc: lp, + OperationsProc: op, + IndexTokensHandler: &IndexTokenHandlerMock{}, } } diff --git a/process/elasticproc/factory/elasticProcessorFactory_test.go b/process/elasticproc/factory/elasticProcessorFactory_test.go index 2e4dd081..f7271737 100644 --- a/process/elasticproc/factory/elasticProcessorFactory_test.go +++ b/process/elasticproc/factory/elasticProcessorFactory_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/require" "github.com/multiversx/mx-chain-es-indexer-go/mock" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" ) func TestCreateElasticProcessor(t *testing.T) { @@ -21,6 +22,7 @@ func TestCreateElasticProcessor(t *testing.T) { UseKibana: false, TxHashExtractor: &mock.TxHashExtractorMock{}, RewardTxData: &mock.RewardTxDataMock{}, + IndexTokensHandler: &elasticproc.IndexTokenHandlerMock{}, } ep, err := CreateElasticProcessor(args) diff --git a/process/elasticproc/indexTokenHandlerMock.go b/process/elasticproc/indexTokenHandlerMock.go new file mode 100644 index 00000000..e60f36fb --- /dev/null +++ b/process/elasticproc/indexTokenHandlerMock.go @@ -0,0 +1,23 @@ +package elasticproc + +import ( + "github.com/multiversx/mx-chain-es-indexer-go/data" +) + +// IndexTokenHandlerMock - +type IndexTokenHandlerMock struct { + IndexCrossChainTokensCalled func(elasticClient DatabaseClientHandler, scrs []*data.ScResult, buffSlice *data.BufferSlice) error +} + +// IndexCrossChainTokens - +func (ithh *IndexTokenHandlerMock) IndexCrossChainTokens(elasticClient DatabaseClientHandler, scrs []*data.ScResult, buffSlice *data.BufferSlice) error { + if ithh.IndexCrossChainTokensCalled != nil { + return ithh.IndexCrossChainTokensCalled(elasticClient, scrs, buffSlice) + } + return nil +} + +// IsInterfaceNil returns true if there is no value under the interface +func (ithh *IndexTokenHandlerMock) IsInterfaceNil() bool { + return ithh == nil +} diff --git a/process/elasticproc/interface.go b/process/elasticproc/interface.go index fd5c0af0..129d458d 100644 --- a/process/elasticproc/interface.go +++ b/process/elasticproc/interface.go @@ -124,6 +124,6 @@ type OperationsHandler interface { // IndexTokensHandler defines what index tokens handler should be able to do type IndexTokensHandler interface { - IndexCrossChainTokens(elasticClient DatabaseClientHandler, scrs []*data.ScResult, buffSlice *data.BufferSlice) error + IndexCrossChainTokens(handler DatabaseClientHandler, scrs []*data.ScResult, buffSlice *data.BufferSlice) error IsInterfaceNil() bool } From a07073e68cf286705e1ee4021c7dc1dd49c2431c Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Tue, 28 Jan 2025 12:05:27 +0200 Subject: [PATCH 35/50] integration test fixes --- integrationtests/valuesIndex_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/integrationtests/valuesIndex_test.go b/integrationtests/valuesIndex_test.go index f76835f7..0fbd7ae3 100644 --- a/integrationtests/valuesIndex_test.go +++ b/integrationtests/valuesIndex_test.go @@ -7,10 +7,12 @@ import ( "fmt" "testing" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-es-indexer-go/mock" indexerData "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" - "github.com/stretchr/testify/require" ) func TestCheckVersionIsIndexer(t *testing.T) { @@ -27,6 +29,9 @@ func TestCheckVersionIsIndexer(t *testing.T) { Denomination: 18, Version: version, EnabledIndexes: []string{indexerData.ValuesIndex}, + TxHashExtractor: &mock.TxHashExtractorMock{}, + RewardTxData: &mock.RewardTxDataMock{}, + IndexTokensHandler: &elasticproc.IndexTokenHandlerMock{}, } _, err = factory.CreateElasticProcessor(args) From 646fbbfbc7a85fcb64ad5bcb27cf4299b9680752 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Tue, 28 Jan 2025 13:22:04 +0200 Subject: [PATCH 36/50] script.sh update --- scripts/script.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/script.sh b/scripts/script.sh index 11a1bdb4..ff224ef4 100755 --- a/scripts/script.sh +++ b/scripts/script.sh @@ -1,4 +1,5 @@ IMAGE_NAME=elastic-container +IMAGE_NAME_2=elastic-container-2 DEFAULT_ES_VERSION=7.16.2 PROMETHEUS_CONTAINER_NAME=prometheus_container GRAFANA_CONTAINER_NAME=grafana_container @@ -16,10 +17,15 @@ start() { docker pull docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} docker rm ${IMAGE_NAME} 2> /dev/null + docker rm ${IMAGE_NAME_2} 2> /dev/null docker run -d --name "${IMAGE_NAME}" -p 9200:9200 -p 9300:9300 \ -e "discovery.type=single-node" -e "xpack.security.enabled=false" -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} + docker run -d --name "${IMAGE_NAME_2}" -p 9201:9200 -p 9301:9300 \ + -e "discovery.type=single-node" -e "xpack.security.enabled=false" -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ + docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} + # Wait elastic cluster to start echo "Waiting Elasticsearch cluster to start..." sleep 30s @@ -27,6 +33,7 @@ start() { stop() { docker stop "${IMAGE_NAME}" + docker stop "${IMAGE_NAME_2}" } delete() { From 86bc0754b1e96773c85006fae2e09d1df88f1851 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Tue, 28 Jan 2025 13:26:25 +0200 Subject: [PATCH 37/50] script.sh update --- scripts/script.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/script.sh b/scripts/script.sh index ff224ef4..6328f90d 100755 --- a/scripts/script.sh +++ b/scripts/script.sh @@ -48,6 +48,7 @@ delete() { IMAGE_OPEN_SEARCH=open-container +IMAGE_OPEN_SEARCH_2=open-container-2 DEFAULT_OPEN_SEARCH_VERSION=1.2.4 start_open_search() { @@ -59,9 +60,13 @@ start_open_search() { docker pull opensearchproject/opensearch:${OPEN_VERSION} docker rm ${IMAGE_OPEN_SEARCH} 2> /dev/null + docker rm ${IMAGE_OPEN_SEARCH_2} 2> /dev/null docker run -d --name "${IMAGE_OPEN_SEARCH}" -p 9200:9200 -p 9600:9600 \ -e "discovery.type=single-node" -e "plugins.security.disabled=true" -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ opensearchproject/opensearch:${OPEN_VERSION} + docker run -d --name "${IMAGE_OPEN_SEARCH_2}" -p 9201:9200 -p 9601:9600 \ + -e "discovery.type=single-node" -e "plugins.security.disabled=true" -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ + opensearchproject/opensearch:${OPEN_VERSION} } From 5d843ef5da8ee102ba4dc2b495495a4a79d27634 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Fri, 31 Jan 2025 11:15:05 +0200 Subject: [PATCH 38/50] go mod for sovereign --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 42d07487..e827ad7f 100644 --- a/go.mod +++ b/go.mod @@ -7,9 +7,9 @@ require ( github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.1 github.com/multiversx/mx-chain-communication-go v1.1.0 - github.com/multiversx/mx-chain-core-go v1.2.24 + github.com/multiversx/mx-chain-core-go v1.2.25-0.20250131091121-26781bb55fe9 github.com/multiversx/mx-chain-logger-go v1.0.15 - github.com/multiversx/mx-chain-vm-common-go v1.5.17-0.20241008143259-334f3a5bafe0 + github.com/multiversx/mx-chain-vm-common-go v1.5.17-0.20241119132002-2fa80c5ec516 github.com/prometheus/client_model v0.4.0 github.com/prometheus/common v0.37.0 github.com/stretchr/testify v1.8.4 diff --git a/go.sum b/go.sum index 86555889..267f913e 100644 --- a/go.sum +++ b/go.sum @@ -249,13 +249,13 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/mx-chain-communication-go v1.1.0 h1:J7bX6HoN3HiHY7cUeEjG8AJWgQDDPcY+OPDOsSUOkRE= github.com/multiversx/mx-chain-communication-go v1.1.0/go.mod h1:WK6bP4pGEHGDDna/AYRIMtl6G9OA0NByI1Lw8PmOnRM= -github.com/multiversx/mx-chain-core-go v1.2.24 h1:O0X7N9GfNVUCE9fukXA+dvfCRRjViYn88zOaE7feUog= -github.com/multiversx/mx-chain-core-go v1.2.24/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= +github.com/multiversx/mx-chain-core-go v1.2.25-0.20250131091121-26781bb55fe9 h1:hm9b1I3ywGV6vOMFX86sZ5jZ7hdIqq1IH8EPNj0Pliw= +github.com/multiversx/mx-chain-core-go v1.2.25-0.20250131091121-26781bb55fe9/go.mod h1:P/YBoFnt25XUaCQ7Q/SD15vhnc9yV5JDhHxyFO9P8Z0= github.com/multiversx/mx-chain-crypto-go v1.2.12 h1:zWip7rpUS4CGthJxfKn5MZfMfYPjVjIiCID6uX5BSOk= github.com/multiversx/mx-chain-logger-go v1.0.15 h1:HlNdK8etyJyL9NQ+6mIXyKPEBo+wRqOwi3n+m2QIHXc= github.com/multiversx/mx-chain-logger-go v1.0.15/go.mod h1:t3PRKaWB1M+i6gUfD27KXgzLJJC+mAQiN+FLlL1yoGQ= -github.com/multiversx/mx-chain-vm-common-go v1.5.17-0.20241008143259-334f3a5bafe0 h1:WpfFe6ueS+nRIq2RzVaeF1/TJStyYH/YLYPCmN8kdDM= -github.com/multiversx/mx-chain-vm-common-go v1.5.17-0.20241008143259-334f3a5bafe0/go.mod h1:IOE+gYAtTEKZJwUr9ZhOnjdf4vFKrZdZ9RVEyuFsEP4= +github.com/multiversx/mx-chain-vm-common-go v1.5.17-0.20241119132002-2fa80c5ec516 h1:PToP1B7aFkXjQBOF5MDCCvsc+ffWVqqBHtmemfCwVA8= +github.com/multiversx/mx-chain-vm-common-go v1.5.17-0.20241119132002-2fa80c5ec516/go.mod h1:C7KVj6/+TAhxDjgY7oAMO5wSj7WbBYIJ5TCMzmxk2w0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= From 72295e5fa14c7e30d189f3f9d93d0814d6dba8ef Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Fri, 31 Jan 2025 11:23:02 +0200 Subject: [PATCH 39/50] integration tests fix from past commit --- integrationtests/valuesIndex_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/integrationtests/valuesIndex_test.go b/integrationtests/valuesIndex_test.go index f76835f7..966163c3 100644 --- a/integrationtests/valuesIndex_test.go +++ b/integrationtests/valuesIndex_test.go @@ -7,10 +7,11 @@ import ( "fmt" "testing" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-es-indexer-go/mock" indexerData "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" - "github.com/stretchr/testify/require" ) func TestCheckVersionIsIndexer(t *testing.T) { @@ -27,6 +28,8 @@ func TestCheckVersionIsIndexer(t *testing.T) { Denomination: 18, Version: version, EnabledIndexes: []string{indexerData.ValuesIndex}, + TxHashExtractor: &mock.TxHashExtractorMock{}, + RewardTxData: &mock.RewardTxDataMock{}, } _, err = factory.CreateElasticProcessor(args) From 87cc67fbce7dc3640c20d8ad579f722d159c67f9 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Wed, 5 Feb 2025 12:03:52 +0200 Subject: [PATCH 40/50] fixes after review --- client/disabled/elasticClient.go | 73 +++++++++++++++++++ client/disabled/elasticClient_test.go | 30 ++++++++ data/tokens.go | 2 +- docker-compose.yml | 2 +- factory/runType/runTypeComponentsFactory.go | 2 +- factory/runType/runTypeComponentsHandler.go | 2 +- .../sovereignRunTypeComponentsFactory.go | 37 +++++++++- integrationtests/consts.go | 2 +- integrationtests/incomingSCR_test.go | 24 +++--- integrationtests/utils.go | 6 +- .../elasticproc/tokens/indexTokensHandler.go | 4 +- .../tokens/indexTokensHandler_test.go | 4 +- .../tokens/sovereignIndexTokensHandler.go | 73 +++++++------------ .../sovereignIndexTokensHandler_test.go | 38 +++------- scripts/script.sh | 14 ++-- 15 files changed, 205 insertions(+), 108 deletions(-) create mode 100644 client/disabled/elasticClient.go create mode 100644 client/disabled/elasticClient_test.go diff --git a/client/disabled/elasticClient.go b/client/disabled/elasticClient.go new file mode 100644 index 00000000..eba23d28 --- /dev/null +++ b/client/disabled/elasticClient.go @@ -0,0 +1,73 @@ +package disabled + +import ( + "bytes" + "context" +) + +type elasticClient struct{} + +// NewDisabledElasticClient - +func NewDisabledElasticClient() *elasticClient { + return &elasticClient{} +} + +// DoBulkRequest - +func (ec *elasticClient) DoBulkRequest(_ context.Context, _ *bytes.Buffer, _ string) error { + return nil +} + +// DoQueryRemove - +func (ec *elasticClient) DoQueryRemove(_ context.Context, _ string, _ *bytes.Buffer) error { + return nil +} + +// DoMultiGet - +func (ec *elasticClient) DoMultiGet(_ context.Context, _ []string, _ string, _ bool, _ interface{}) error { + return nil +} + +// DoScrollRequest - +func (ec *elasticClient) DoScrollRequest(_ context.Context, _ string, _ []byte, _ bool, _ func(responseBytes []byte) error) error { + return nil +} + +// DoCountRequest - +func (ec *elasticClient) DoCountRequest(_ context.Context, _ string, _ []byte) (uint64, error) { + return 0, nil +} + +// UpdateByQuery - +func (ec *elasticClient) UpdateByQuery(_ context.Context, _ string, _ *bytes.Buffer) error { + return nil +} + +// PutMappings - +func (ec *elasticClient) PutMappings(_ string, _ *bytes.Buffer) error { + return nil +} + +// CheckAndCreateIndex - +func (ec *elasticClient) CheckAndCreateIndex(_ string) error { + return nil +} + +// CheckAndCreateAlias - +func (ec *elasticClient) CheckAndCreateAlias(_ string, _ string) error { + return nil +} + +// CheckAndCreateTemplate - +func (ec *elasticClient) CheckAndCreateTemplate(_ string, _ *bytes.Buffer) error { + return nil +} + +// CheckAndCreatePolicy - +func (ec *elasticClient) CheckAndCreatePolicy(_ string, _ *bytes.Buffer) error { + return nil +} + +// IsInterfaceNil - returns true if there is no value under the interface +func (ec *elasticClient) IsInterfaceNil() bool { + return ec == nil +} diff --git a/client/disabled/elasticClient_test.go b/client/disabled/elasticClient_test.go new file mode 100644 index 00000000..cae05ffd --- /dev/null +++ b/client/disabled/elasticClient_test.go @@ -0,0 +1,30 @@ +package disabled + +import ( + "bytes" + "testing" + + "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/stretchr/testify/require" +) + +func TestDisabledElasticClient_MethodsShouldNotPanic(t *testing.T) { + t.Parallel() + + ec := NewDisabledElasticClient() + require.False(t, check.IfNil(ec)) + + require.NotPanics(t, func() { + _ = ec.DoBulkRequest(nil, new(bytes.Buffer), "") + _ = ec.DoQueryRemove(nil, "", new(bytes.Buffer)) + _ = ec.DoMultiGet(nil, make([]string, 0), "", true, nil) + _ = ec.DoScrollRequest(nil, "", []byte(""), true, nil) + _, _ = ec.DoCountRequest(nil, "", []byte("")) + _ = ec.UpdateByQuery(nil, "", new(bytes.Buffer)) + _ = ec.PutMappings("", new(bytes.Buffer)) + _ = ec.CheckAndCreateIndex("") + _ = ec.CheckAndCreateAlias("", "") + _ = ec.CheckAndCreateTemplate("", new(bytes.Buffer)) + _ = ec.CheckAndCreatePolicy("", new(bytes.Buffer)) + }) +} diff --git a/data/tokens.go b/data/tokens.go index d70b6df9..01f39dce 100644 --- a/data/tokens.go +++ b/data/tokens.go @@ -45,7 +45,7 @@ type SourceToken struct { CurrentOwner string `json:"currentOwner"` } -// ResponseTokenInfo is the structure for the token info response +// ResponseTokenInfo is the structure for the tokens info response type ResponseTokenInfo struct { Docs []ResponseTokenInfoDB `json:"docs"` } diff --git a/docker-compose.yml b/docker-compose.yml index a5a27b1f..ba288828 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: ports: - "9200:9200" - "9300:9300" - elasticsearch2: + main-elasticsearch: container_name: es-container2 image: docker.elastic.co/elasticsearch/elasticsearch:7.16.1 environment: diff --git a/factory/runType/runTypeComponentsFactory.go b/factory/runType/runTypeComponentsFactory.go index af7d42a3..8ac863d6 100644 --- a/factory/runType/runTypeComponentsFactory.go +++ b/factory/runType/runTypeComponentsFactory.go @@ -17,7 +17,7 @@ func (rtcf *runTypeComponentsFactory) Create() (*runTypeComponents, error) { return &runTypeComponents{ txHashExtractor: transactions.NewTxHashExtractor(), rewardTxData: transactions.NewRewardTxData(), - indexTokensHandler: tokens.NewIndexTokensHandler(), + indexTokensHandler: tokens.NewDisabledIndexTokensHandler(), }, nil } diff --git a/factory/runType/runTypeComponentsHandler.go b/factory/runType/runTypeComponentsHandler.go index 1d471a5c..b9697008 100644 --- a/factory/runType/runTypeComponentsHandler.go +++ b/factory/runType/runTypeComponentsHandler.go @@ -109,7 +109,7 @@ func (mrtc *managedRunTypeComponents) RewardTxDataCreator() transactions.RewardT return mrtc.runTypeComponents.rewardTxData } -// IndexTokensHandlerCreator return index tokens handler +// IndexTokensHandlerCreator returns the index tokens handler func (mrtc *managedRunTypeComponents) IndexTokensHandlerCreator() elasticproc.IndexTokensHandler { mrtc.mutRunTypeCoreComponents.Lock() defer mrtc.mutRunTypeCoreComponents.Unlock() diff --git a/factory/runType/sovereignRunTypeComponentsFactory.go b/factory/runType/sovereignRunTypeComponentsFactory.go index 2adb96c7..54590352 100644 --- a/factory/runType/sovereignRunTypeComponentsFactory.go +++ b/factory/runType/sovereignRunTypeComponentsFactory.go @@ -1,6 +1,16 @@ package runType import ( + "math" + "net/http" + "time" + + "github.com/elastic/go-elasticsearch/v7" + + "github.com/multiversx/mx-chain-es-indexer-go/client" + "github.com/multiversx/mx-chain-es-indexer-go/client/disabled" + "github.com/multiversx/mx-chain-es-indexer-go/client/logging" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tokens" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" @@ -21,7 +31,12 @@ func NewSovereignRunTypeComponentsFactory(mainChainElastic factory.ElasticConfig // Create will create the run type components func (srtcf *sovereignRunTypeComponentsFactory) Create() (*runTypeComponents, error) { - sovIndexTokensHandler, err := tokens.NewSovereignIndexTokensHandler(srtcf.mainChainElastic, srtcf.esdtPrefix) + mainChainElasticClient, err := createMainChainElasticClient(srtcf.mainChainElastic) + if err != nil { + return nil, err + } + + sovIndexTokensHandler, err := tokens.NewSovereignIndexTokensHandler(srtcf.mainChainElastic.Enabled, mainChainElasticClient, srtcf.esdtPrefix) if err != nil { return nil, err } @@ -33,6 +48,26 @@ func (srtcf *sovereignRunTypeComponentsFactory) Create() (*runTypeComponents, er }, nil } +func createMainChainElasticClient(mainChainElastic factory.ElasticConfig) (elasticproc.DatabaseClientHandler, error) { + if mainChainElastic.Enabled { + argsEsClient := elasticsearch.Config{ + Addresses: []string{mainChainElastic.Url}, + Username: mainChainElastic.UserName, + Password: mainChainElastic.Password, + Logger: &logging.CustomLogger{}, + RetryOnStatus: []int{http.StatusConflict}, + RetryBackoff: retryBackOff, + } + return client.NewElasticClient(argsEsClient) + } else { + return disabled.NewDisabledElasticClient(), nil + } +} + +func retryBackOff(attempt int) time.Duration { + return time.Duration(math.Exp2(float64(attempt))) * time.Second +} + // IsInterfaceNil returns true if there is no value under the interface func (srtcf *sovereignRunTypeComponentsFactory) IsInterfaceNil() bool { return srtcf == nil diff --git a/integrationtests/consts.go b/integrationtests/consts.go index f06babca..34d0ca79 100644 --- a/integrationtests/consts.go +++ b/integrationtests/consts.go @@ -6,7 +6,7 @@ const ( //nolint esURL = "http://localhost:9200" //nolint - es2URL = "http://localhost:9201" + esMainURL = "http://localhost:9201" //nolint addressPrefix = "erd" ) diff --git a/integrationtests/incomingSCR_test.go b/integrationtests/incomingSCR_test.go index e35d08b9..612a2b60 100644 --- a/integrationtests/incomingSCR_test.go +++ b/integrationtests/incomingSCR_test.go @@ -18,7 +18,7 @@ import ( "github.com/stretchr/testify/require" indexerData "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" - "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" ) type esToken struct { @@ -72,18 +72,16 @@ func createTokens() ([]esToken, []esNft) { func TestCrossChainTokensIndexingFromMainChain(t *testing.T) { setLogLevelDebug() - mainChainEs := factory.ElasticConfig{ - Enabled: true, - Url: es2URL, - } + mainEsClient, err := createESClient(esMainURL) + require.Nil(t, err) tokens, nfts := createTokens() - createTokensInSourceEs(t, mainChainEs, tokens, nfts) + createTokensInSourceEs(t, mainEsClient, tokens, nfts) esClient, err := createESClient(esURL) require.Nil(t, err) - esProc, err := CreateSovereignElasticProcessor(esClient, mainChainEs) + esProc, err := CreateSovereignElasticProcessor(esClient, mainEsClient) require.Nil(t, err) allTokens := getAllTokensIDs(tokens, nfts) @@ -143,10 +141,7 @@ func TestCrossChainTokensIndexingFromMainChain(t *testing.T) { } } -func createTokensInSourceEs(t *testing.T, es factory.ElasticConfig, tokens []esToken, nfts []esNft) { - esClient, err := createESClient(es.Url) - require.Nil(t, err) - +func createTokensInSourceEs(t *testing.T, esClient elasticproc.DatabaseClientHandler, tokens []esToken, nfts []esNft) { esProc, err := CreateElasticProcessor(esClient) require.Nil(t, err) @@ -254,7 +249,12 @@ func getAllTokensIDs(tokens []esToken, nfts []esNft) []string { func getAllNftIDs(nfts []esNft) []string { allNfts := make([]string, 0) for _, nft := range nfts { - allNfts = append(allNfts, nft.Collection+"-"+hex.EncodeToString(big.NewInt(0).SetUint64(nft.Nonce).Bytes())) + nonceBytes := big.NewInt(0).SetUint64(nft.Nonce).Bytes() + nonceHex := hex.EncodeToString(nonceBytes) + nftIdentifier := nft.Collection + "-" + nonceHex + + allNfts = append(allNfts, nftIdentifier) + } return allNfts } diff --git a/integrationtests/utils.go b/integrationtests/utils.go index 0577ca81..93ab9924 100644 --- a/integrationtests/utils.go +++ b/integrationtests/utils.go @@ -67,7 +67,7 @@ func CreateElasticProcessor( Denomination: 18, TxHashExtractor: transactions.NewTxHashExtractor(), RewardTxData: transactions.NewRewardTxData(), - IndexTokensHandler: tokens.NewIndexTokensHandler(), + IndexTokensHandler: tokens.NewDisabledIndexTokensHandler(), } return factory.CreateElasticProcessor(args) @@ -76,9 +76,9 @@ func CreateElasticProcessor( // CreateSovereignElasticProcessor - func CreateSovereignElasticProcessor( esClient elasticproc.DatabaseClientHandler, - mainChainEs factory.ElasticConfig, + mainEsClient elasticproc.DatabaseClientHandler, ) (dataindexer.ElasticProcessor, error) { - sovIndexTokens, _ := tokens.NewSovereignIndexTokensHandler(mainChainEs, sovEsdtPrefix) + sovIndexTokens, _ := tokens.NewSovereignIndexTokensHandler(true, mainEsClient, sovEsdtPrefix) args := factory.ArgElasticProcessorFactory{ Marshalizer: &mock.MarshalizerMock{}, diff --git a/process/elasticproc/tokens/indexTokensHandler.go b/process/elasticproc/tokens/indexTokensHandler.go index 2f45c134..99a4b8c4 100644 --- a/process/elasticproc/tokens/indexTokensHandler.go +++ b/process/elasticproc/tokens/indexTokensHandler.go @@ -7,8 +7,8 @@ import ( type indexTokensHandler struct{} -// NewIndexTokensHandler creates a new index tokens handler -func NewIndexTokensHandler() *indexTokensHandler { +// NewDisabledIndexTokensHandler creates a new disabled index tokens handler +func NewDisabledIndexTokensHandler() *indexTokensHandler { return &indexTokensHandler{} } diff --git a/process/elasticproc/tokens/indexTokensHandler_test.go b/process/elasticproc/tokens/indexTokensHandler_test.go index 260c39af..6b490a81 100644 --- a/process/elasticproc/tokens/indexTokensHandler_test.go +++ b/process/elasticproc/tokens/indexTokensHandler_test.go @@ -9,14 +9,14 @@ import ( func TestNewIndexTokensHandler(t *testing.T) { t.Parallel() - ith := NewIndexTokensHandler() + ith := NewDisabledIndexTokensHandler() require.False(t, ith.IsInterfaceNil()) } func TestIndexTokensHandler_IndexCrossChainTokens(t *testing.T) { t.Parallel() - ith := NewIndexTokensHandler() + ith := NewDisabledIndexTokensHandler() err := ith.IndexCrossChainTokens(nil, nil, nil) require.NoError(t, err) } diff --git a/process/elasticproc/tokens/sovereignIndexTokensHandler.go b/process/elasticproc/tokens/sovereignIndexTokensHandler.go index 0f145fc6..0f9fb90b 100644 --- a/process/elasticproc/tokens/sovereignIndexTokensHandler.go +++ b/process/elasticproc/tokens/sovereignIndexTokensHandler.go @@ -4,22 +4,15 @@ import ( "context" "encoding/json" "fmt" - "math" - "net/http" "strings" - "time" - "github.com/elastic/go-elasticsearch/v7" "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data/esdt" - "github.com/multiversx/mx-chain-es-indexer-go/client" - "github.com/multiversx/mx-chain-es-indexer-go/client/logging" "github.com/multiversx/mx-chain-es-indexer-go/data" indexerdata "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" - "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" ) type sovereignIndexTokensHandler struct { @@ -29,53 +22,32 @@ type sovereignIndexTokensHandler struct { } // NewSovereignIndexTokensHandler creates a new sovereign index tokens handler -func NewSovereignIndexTokensHandler(mainChainElastic factory.ElasticConfig, esdtPrefix string) (*sovereignIndexTokensHandler, error) { - var mainChainElasticClient elasticproc.DatabaseClientHandler - if mainChainElastic.Enabled { - var err error - argsEsClient := elasticsearch.Config{ - Addresses: []string{mainChainElastic.Url}, - Username: mainChainElastic.UserName, - Password: mainChainElastic.Password, - Logger: &logging.CustomLogger{}, - RetryOnStatus: []int{http.StatusConflict}, - RetryBackoff: retryBackOff, - } - mainChainElasticClient, err = client.NewElasticClient(argsEsClient) - if err != nil { - return nil, err - } - } - +func NewSovereignIndexTokensHandler(indexingEnabled bool, mainChainElasticClient elasticproc.DatabaseClientHandler, esdtPrefix string) (*sovereignIndexTokensHandler, error) { return &sovereignIndexTokensHandler{ - indexingEnabled: mainChainElastic.Enabled, + indexingEnabled: indexingEnabled, mainChainElasticClient: mainChainElasticClient, esdtPrefix: esdtPrefix, }, nil } -func retryBackOff(attempt int) time.Duration { - return time.Duration(math.Exp2(float64(attempt))) * time.Second -} - // IndexCrossChainTokens will index the new tokens properties func (sit *sovereignIndexTokensHandler) IndexCrossChainTokens(elasticClient elasticproc.DatabaseClientHandler, scrs []*data.ScResult, buffSlice *data.BufferSlice) error { if !sit.indexingEnabled { return nil } - notFoundTokens, err := sit.getTokensFromScrs(elasticClient, scrs) + newTokens, err := sit.getNewTokensFromSCRs(elasticClient, scrs) if err != nil { return err } - if len(notFoundTokens) == 0 { // no new tokens + if len(newTokens) == 0 { // no new tokens return nil } // get tokens from main chain elastic db mainChainTokens := &data.ResponseTokenInfo{} - err = sit.mainChainElasticClient.DoMultiGet(context.Background(), notFoundTokens, indexerdata.TokensIndex, true, mainChainTokens) + err = sit.mainChainElasticClient.DoMultiGet(context.Background(), newTokens, indexerdata.TokensIndex, true, mainChainTokens) if err != nil { return err } @@ -83,19 +55,11 @@ func (sit *sovereignIndexTokensHandler) IndexCrossChainTokens(elasticClient elas return sit.indexNewTokens(mainChainTokens.Docs, buffSlice) } -func (sit *sovereignIndexTokensHandler) getTokensFromScrs(elasticClient elasticproc.DatabaseClientHandler, scrs []*data.ScResult) ([]string, error) { +func (sit *sovereignIndexTokensHandler) getNewTokensFromSCRs(elasticClient elasticproc.DatabaseClientHandler, scrs []*data.ScResult) ([]string, error) { receivedTokensIDs := make([]string, 0) for _, scr := range scrs { if scr.SenderShard == core.MainChainShardId { - for _, token := range scr.Tokens { - tokenPrefix, hasPrefix := esdt.IsValidPrefixedToken(token) - if !hasPrefix || tokenPrefix != sit.esdtPrefix { - receivedTokensIDs = append(receivedTokensIDs, token) - } - if isEsdt, tokenCollection := getTokenCollection(hasPrefix, token); isEsdt { - receivedTokensIDs = append(receivedTokensIDs, tokenCollection) - } - } + receivedTokensIDs = append(receivedTokensIDs, sit.extractSovereignTokens(scr.Tokens)...) } } @@ -119,15 +83,30 @@ func (sit *sovereignIndexTokensHandler) getTokensFromScrs(elasticClient elasticp return newTokens, nil } -func getTokenCollection(hasPrefix bool, tokenIdentifier string) (bool, string) { +func (sit *sovereignIndexTokensHandler) extractSovereignTokens(tokens []string) []string { + receivedTokensIDs := make([]string, 0) + for _, token := range tokens { + tokenPrefix, hasPrefix := esdt.IsValidPrefixedToken(token) + if !hasPrefix || tokenPrefix != sit.esdtPrefix { + receivedTokensIDs = append(receivedTokensIDs, token) + } + if tokenCollection := getTokenCollection(hasPrefix, token); tokenCollection != "" { + receivedTokensIDs = append(receivedTokensIDs, tokenCollection) + } + } + + return receivedTokensIDs +} + +func getTokenCollection(hasPrefix bool, tokenIdentifier string) string { tokenSplit := strings.Split(tokenIdentifier, "-") if !hasPrefix && len(tokenSplit) == 3 { - return true, tokenSplit[0] + "-" + tokenSplit[1] + return tokenSplit[0] + "-" + tokenSplit[1] } if hasPrefix && len(tokenSplit) == 4 { - return true, tokenSplit[1] + "-" + tokenSplit[2] + return tokenSplit[1] + "-" + tokenSplit[2] } - return false, "" + return "" } func (sit *sovereignIndexTokensHandler) indexNewTokens(responseTokensInfo []data.ResponseTokenInfoDB, buffSlice *data.BufferSlice) error { diff --git a/process/elasticproc/tokens/sovereignIndexTokensHandler_test.go b/process/elasticproc/tokens/sovereignIndexTokensHandler_test.go index 3aebe667..2cb86b88 100644 --- a/process/elasticproc/tokens/sovereignIndexTokensHandler_test.go +++ b/process/elasticproc/tokens/sovereignIndexTokensHandler_test.go @@ -1,59 +1,39 @@ package tokens import ( + "fmt" "testing" "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-es-indexer-go/client/disabled" "github.com/multiversx/mx-chain-es-indexer-go/data" - "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" + "github.com/multiversx/mx-chain-es-indexer-go/mock" ) const ( prefix = "sov" ) -func createElasticConfig() factory.ElasticConfig { - return factory.ElasticConfig{ - Enabled: true, - Url: "http://localhost:9200", - UserName: "", - Password: "", - } -} - func TestSovereignNewIndexTokensHandler(t *testing.T) { t.Parallel() - t.Run("no url, should error", func(t *testing.T) { - esConfig := createElasticConfig() - esConfig.Url = "http://bad url" - sith, err := NewSovereignIndexTokensHandler(esConfig, prefix) - require.ErrorContains(t, err, "cannot parse url") - require.Nil(t, sith) - }) - t.Run("not enabled should not create main chain elastic client", func(t *testing.T) { - esConfig := createElasticConfig() - esConfig.Enabled = false - sith, err := NewSovereignIndexTokensHandler(esConfig, prefix) + t.Run("valid disabled config, should work", func(t *testing.T) { + sith, err := NewSovereignIndexTokensHandler(false, disabled.NewDisabledElasticClient(), prefix) require.NoError(t, err) - require.NotNil(t, sith) - require.Nil(t, sith.mainChainElasticClient) + require.Equal(t, "*disabled.elasticClient", fmt.Sprintf("%T", sith.mainChainElasticClient)) }) t.Run("valid config, should work", func(t *testing.T) { - esConfig := createElasticConfig() - sith, err := NewSovereignIndexTokensHandler(esConfig, prefix) + sith, err := NewSovereignIndexTokensHandler(true, &mock.DatabaseWriterStub{}, prefix) require.NoError(t, err) - require.NotNil(t, sith) + require.Equal(t, "*mock.DatabaseWriterStub", fmt.Sprintf("%T", sith.mainChainElasticClient)) }) } func TestSovereignIndexTokensHandler_IndexCrossChainTokens(t *testing.T) { t.Parallel() - esConfig := createElasticConfig() - esConfig.Enabled = false - sith, err := NewSovereignIndexTokensHandler(esConfig, prefix) + sith, err := NewSovereignIndexTokensHandler(false, disabled.NewDisabledElasticClient(), prefix) require.NoError(t, err) require.NotNil(t, sith) diff --git a/scripts/script.sh b/scripts/script.sh index 6328f90d..2b58ebb3 100755 --- a/scripts/script.sh +++ b/scripts/script.sh @@ -1,5 +1,5 @@ IMAGE_NAME=elastic-container -IMAGE_NAME_2=elastic-container-2 +MAIN_IMAGE_NAME=main-elastic-container DEFAULT_ES_VERSION=7.16.2 PROMETHEUS_CONTAINER_NAME=prometheus_container GRAFANA_CONTAINER_NAME=grafana_container @@ -17,12 +17,12 @@ start() { docker pull docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} docker rm ${IMAGE_NAME} 2> /dev/null - docker rm ${IMAGE_NAME_2} 2> /dev/null + docker rm ${MAIN_IMAGE_NAME} 2> /dev/null docker run -d --name "${IMAGE_NAME}" -p 9200:9200 -p 9300:9300 \ -e "discovery.type=single-node" -e "xpack.security.enabled=false" -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} - docker run -d --name "${IMAGE_NAME_2}" -p 9201:9200 -p 9301:9300 \ + docker run -d --name "${MAIN_IMAGE_NAME}" -p 9201:9200 -p 9301:9300 \ -e "discovery.type=single-node" -e "xpack.security.enabled=false" -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} @@ -33,7 +33,7 @@ start() { stop() { docker stop "${IMAGE_NAME}" - docker stop "${IMAGE_NAME_2}" + docker stop "${MAIN_IMAGE_NAME}" } delete() { @@ -48,7 +48,7 @@ delete() { IMAGE_OPEN_SEARCH=open-container -IMAGE_OPEN_SEARCH_2=open-container-2 +MAIN_IMAGE_OPEN_SEARCH=main-open-container DEFAULT_OPEN_SEARCH_VERSION=1.2.4 start_open_search() { @@ -60,11 +60,11 @@ start_open_search() { docker pull opensearchproject/opensearch:${OPEN_VERSION} docker rm ${IMAGE_OPEN_SEARCH} 2> /dev/null - docker rm ${IMAGE_OPEN_SEARCH_2} 2> /dev/null + docker rm ${MAIN_IMAGE_OPEN_SEARCH} 2> /dev/null docker run -d --name "${IMAGE_OPEN_SEARCH}" -p 9200:9200 -p 9600:9600 \ -e "discovery.type=single-node" -e "plugins.security.disabled=true" -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ opensearchproject/opensearch:${OPEN_VERSION} - docker run -d --name "${IMAGE_OPEN_SEARCH_2}" -p 9201:9200 -p 9601:9600 \ + docker run -d --name "${MAIN_IMAGE_OPEN_SEARCH}" -p 9201:9200 -p 9601:9600 \ -e "discovery.type=single-node" -e "plugins.security.disabled=true" -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ opensearchproject/opensearch:${OPEN_VERSION} From 9d96aee965ad9bc7d052bc536cdd8205bc865916 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Wed, 5 Feb 2025 12:20:22 +0200 Subject: [PATCH 41/50] unit test fixes --- client/disabled/elasticClient_test.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/client/disabled/elasticClient_test.go b/client/disabled/elasticClient_test.go index cae05ffd..c33e8716 100644 --- a/client/disabled/elasticClient_test.go +++ b/client/disabled/elasticClient_test.go @@ -2,6 +2,7 @@ package disabled import ( "bytes" + "context" "testing" "github.com/multiversx/mx-chain-core-go/core/check" @@ -15,12 +16,12 @@ func TestDisabledElasticClient_MethodsShouldNotPanic(t *testing.T) { require.False(t, check.IfNil(ec)) require.NotPanics(t, func() { - _ = ec.DoBulkRequest(nil, new(bytes.Buffer), "") - _ = ec.DoQueryRemove(nil, "", new(bytes.Buffer)) - _ = ec.DoMultiGet(nil, make([]string, 0), "", true, nil) - _ = ec.DoScrollRequest(nil, "", []byte(""), true, nil) - _, _ = ec.DoCountRequest(nil, "", []byte("")) - _ = ec.UpdateByQuery(nil, "", new(bytes.Buffer)) + _ = ec.DoBulkRequest(context.Background(), new(bytes.Buffer), "") + _ = ec.DoQueryRemove(context.Background(), "", new(bytes.Buffer)) + _ = ec.DoMultiGet(context.Background(), make([]string, 0), "", true, nil) + _ = ec.DoScrollRequest(context.Background(), "", []byte(""), true, nil) + _, _ = ec.DoCountRequest(context.Background(), "", []byte("")) + _ = ec.UpdateByQuery(context.Background(), "", new(bytes.Buffer)) _ = ec.PutMappings("", new(bytes.Buffer)) _ = ec.CheckAndCreateIndex("") _ = ec.CheckAndCreateAlias("", "") From d2c8827aa9c4a9abd29933ab89c765230b113727 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Thu, 6 Feb 2025 10:42:56 +0200 Subject: [PATCH 42/50] fixes after review --- client/disabled/elasticClient.go | 5 +++ client/mainChainElasticClient.go | 28 +++++++++++++++ client/mainChainElasticClient_test.go | 36 +++++++++++++++++++ cmd/elasticindexer/config/prefs.toml | 2 +- config/config.go | 2 +- docker-compose.yml | 2 +- .../sovereignRunTypeComponentsFactory.go | 6 ++-- integrationtests/consts.go | 2 +- integrationtests/incomingSCR_test.go | 6 ++-- integrationtests/utils.go | 12 +++++-- mock/databaseWriterStub.go | 5 +++ process/elasticproc/interface.go | 6 ++++ .../tokens/sovereignIndexTokensHandler.go | 12 +++---- .../sovereignIndexTokensHandler_test.go | 6 ++-- scripts/script.sh | 19 +++++----- 15 files changed, 117 insertions(+), 32 deletions(-) create mode 100644 client/mainChainElasticClient.go create mode 100644 client/mainChainElasticClient_test.go diff --git a/client/disabled/elasticClient.go b/client/disabled/elasticClient.go index eba23d28..41ccc6c0 100644 --- a/client/disabled/elasticClient.go +++ b/client/disabled/elasticClient.go @@ -67,6 +67,11 @@ func (ec *elasticClient) CheckAndCreatePolicy(_ string, _ *bytes.Buffer) error { return nil } +// IsEnabled - +func (ec *elasticClient) IsEnabled() bool { + return false +} + // IsInterfaceNil - returns true if there is no value under the interface func (ec *elasticClient) IsInterfaceNil() bool { return ec == nil diff --git a/client/mainChainElasticClient.go b/client/mainChainElasticClient.go new file mode 100644 index 00000000..e378dd9a --- /dev/null +++ b/client/mainChainElasticClient.go @@ -0,0 +1,28 @@ +package client + +import ( + "github.com/elastic/go-elasticsearch/v7" +) + +type mainChainElasticClient struct { + *elasticClient + indexingEnabled bool +} + +// NewMainChainElasticClient creates a new sovereign elastic client +func NewMainChainElasticClient(cfg elasticsearch.Config, indexingEnabled bool) (*mainChainElasticClient, error) { + esClient, err := NewElasticClient(cfg) + if err != nil { + return nil, err + } + + return &mainChainElasticClient{ + esClient, + indexingEnabled, + }, nil +} + +// IsEnabled returns true if main chain elastic client is enabled +func (mcec *mainChainElasticClient) IsEnabled() bool { + return mcec.indexingEnabled +} diff --git a/client/mainChainElasticClient_test.go b/client/mainChainElasticClient_test.go new file mode 100644 index 00000000..c72c5079 --- /dev/null +++ b/client/mainChainElasticClient_test.go @@ -0,0 +1,36 @@ +package client + +import ( + "fmt" + "testing" + + "github.com/elastic/go-elasticsearch/v7" + "github.com/stretchr/testify/require" + + indexer "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" +) + +func TestNewMainChainElasticClient(t *testing.T) { + t.Run("no url, should error", func(t *testing.T) { + esClient, err := NewMainChainElasticClient(elasticsearch.Config{ + Addresses: []string{}, + }, true) + require.Nil(t, esClient) + require.Equal(t, indexer.ErrNoElasticUrlProvided, err) + }) + t.Run("should work", func(t *testing.T) { + esClient, err := NewMainChainElasticClient(elasticsearch.Config{ + Addresses: []string{"http://localhost:9200"}, + }, true) + require.Nil(t, err) + require.Equal(t, "*client.mainChainElasticClient", fmt.Sprintf("%T", esClient)) + }) +} + +func TestMainChainElasticClient_IsEnabled(t *testing.T) { + esClient, err := NewMainChainElasticClient(elasticsearch.Config{ + Addresses: []string{"http://localhost:9200"}, + }, true) + require.Nil(t, err) + require.Equal(t, true, esClient.IsEnabled()) +} diff --git a/cmd/elasticindexer/config/prefs.toml b/cmd/elasticindexer/config/prefs.toml index 7c320e89..42dc095f 100644 --- a/cmd/elasticindexer/config/prefs.toml +++ b/cmd/elasticindexer/config/prefs.toml @@ -24,7 +24,7 @@ password = "" bulk-request-max-size-in-bytes = 4194304 # 4MB - [config.main-elastic-cluster] + [config.main-chain-elastic-cluster] enabled = true url = "http://localhost:9201" username = "" diff --git a/config/config.go b/config/config.go index b5d3b83a..60c56b4d 100644 --- a/config/config.go +++ b/config/config.go @@ -58,7 +58,7 @@ type ClusterConfig struct { URL string `toml:"url"` UserName string `toml:"username"` Password string `toml:"password"` - } `toml:"main-elastic-cluster"` + } `toml:"main-chain-elastic-cluster"` } `toml:"config"` } diff --git a/docker-compose.yml b/docker-compose.yml index ba288828..27e689c1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: ports: - "9200:9200" - "9300:9300" - main-elasticsearch: + main-chain-elasticsearch: container_name: es-container2 image: docker.elastic.co/elasticsearch/elasticsearch:7.16.1 environment: diff --git a/factory/runType/sovereignRunTypeComponentsFactory.go b/factory/runType/sovereignRunTypeComponentsFactory.go index 54590352..62aa1e2d 100644 --- a/factory/runType/sovereignRunTypeComponentsFactory.go +++ b/factory/runType/sovereignRunTypeComponentsFactory.go @@ -36,7 +36,7 @@ func (srtcf *sovereignRunTypeComponentsFactory) Create() (*runTypeComponents, er return nil, err } - sovIndexTokensHandler, err := tokens.NewSovereignIndexTokensHandler(srtcf.mainChainElastic.Enabled, mainChainElasticClient, srtcf.esdtPrefix) + sovIndexTokensHandler, err := tokens.NewSovereignIndexTokensHandler(mainChainElasticClient, srtcf.esdtPrefix) if err != nil { return nil, err } @@ -48,7 +48,7 @@ func (srtcf *sovereignRunTypeComponentsFactory) Create() (*runTypeComponents, er }, nil } -func createMainChainElasticClient(mainChainElastic factory.ElasticConfig) (elasticproc.DatabaseClientHandler, error) { +func createMainChainElasticClient(mainChainElastic factory.ElasticConfig) (elasticproc.MainChainDatabaseClientHandler, error) { if mainChainElastic.Enabled { argsEsClient := elasticsearch.Config{ Addresses: []string{mainChainElastic.Url}, @@ -58,7 +58,7 @@ func createMainChainElasticClient(mainChainElastic factory.ElasticConfig) (elast RetryOnStatus: []int{http.StatusConflict}, RetryBackoff: retryBackOff, } - return client.NewElasticClient(argsEsClient) + return client.NewMainChainElasticClient(argsEsClient, mainChainElastic.Enabled) } else { return disabled.NewDisabledElasticClient(), nil } diff --git a/integrationtests/consts.go b/integrationtests/consts.go index 34d0ca79..32b4bd1b 100644 --- a/integrationtests/consts.go +++ b/integrationtests/consts.go @@ -6,7 +6,7 @@ const ( //nolint esURL = "http://localhost:9200" //nolint - esMainURL = "http://localhost:9201" + esMainChainURL = "http://localhost:9201" //nolint addressPrefix = "erd" ) diff --git a/integrationtests/incomingSCR_test.go b/integrationtests/incomingSCR_test.go index 612a2b60..b6931a0d 100644 --- a/integrationtests/incomingSCR_test.go +++ b/integrationtests/incomingSCR_test.go @@ -72,16 +72,16 @@ func createTokens() ([]esToken, []esNft) { func TestCrossChainTokensIndexingFromMainChain(t *testing.T) { setLogLevelDebug() - mainEsClient, err := createESClient(esMainURL) + mainChainEsClient, err := createMainChainESClient(esMainChainURL, true) require.Nil(t, err) tokens, nfts := createTokens() - createTokensInSourceEs(t, mainEsClient, tokens, nfts) + createTokensInSourceEs(t, mainChainEsClient, tokens, nfts) esClient, err := createESClient(esURL) require.Nil(t, err) - esProc, err := CreateSovereignElasticProcessor(esClient, mainEsClient) + esProc, err := CreateSovereignElasticProcessor(esClient, mainChainEsClient) require.Nil(t, err) allTokens := getAllTokensIDs(tokens, nfts) diff --git a/integrationtests/utils.go b/integrationtests/utils.go index 93ab9924..796c4ac1 100644 --- a/integrationtests/utils.go +++ b/integrationtests/utils.go @@ -43,6 +43,14 @@ func createESClient(url string) (elasticproc.DatabaseClientHandler, error) { }) } +// nolint +func createMainChainESClient(url string, enabled bool) (elasticproc.MainChainDatabaseClientHandler, error) { + return client.NewMainChainElasticClient(elasticsearch.Config{ + Addresses: []string{url}, + Logger: &logging.CustomLogger{}, + }, enabled) +} + // nolint func decodeAddress(address string) []byte { decoded, err := pubKeyConverter.Decode(address) @@ -76,9 +84,9 @@ func CreateElasticProcessor( // CreateSovereignElasticProcessor - func CreateSovereignElasticProcessor( esClient elasticproc.DatabaseClientHandler, - mainEsClient elasticproc.DatabaseClientHandler, + mainEsClient elasticproc.MainChainDatabaseClientHandler, ) (dataindexer.ElasticProcessor, error) { - sovIndexTokens, _ := tokens.NewSovereignIndexTokensHandler(true, mainEsClient, sovEsdtPrefix) + sovIndexTokens, _ := tokens.NewSovereignIndexTokensHandler(mainEsClient, sovEsdtPrefix) args := factory.ArgElasticProcessorFactory{ Marshalizer: &mock.MarshalizerMock{}, diff --git a/mock/databaseWriterStub.go b/mock/databaseWriterStub.go index 1df608a8..5a0153ce 100644 --- a/mock/databaseWriterStub.go +++ b/mock/databaseWriterStub.go @@ -86,6 +86,11 @@ func (dwm *DatabaseWriterStub) CheckAndCreatePolicy(_ string, _ *bytes.Buffer) e return nil } +// IsEnabled - +func (dwm *DatabaseWriterStub) IsEnabled() bool { + return false +} + // IsInterfaceNil returns true if there is no value under the interface func (dwm *DatabaseWriterStub) IsInterfaceNil() bool { return dwm == nil diff --git a/process/elasticproc/interface.go b/process/elasticproc/interface.go index 129d458d..b5108f9b 100644 --- a/process/elasticproc/interface.go +++ b/process/elasticproc/interface.go @@ -13,6 +13,12 @@ import ( "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tokeninfo" ) +// MainChainDatabaseClientHandler defines the actions that sovereign database client handler should do +type MainChainDatabaseClientHandler interface { + DatabaseClientHandler + IsEnabled() bool +} + // DatabaseClientHandler defines the actions that a component that handles requests should do type DatabaseClientHandler interface { DoBulkRequest(ctx context.Context, buff *bytes.Buffer, index string) error diff --git a/process/elasticproc/tokens/sovereignIndexTokensHandler.go b/process/elasticproc/tokens/sovereignIndexTokensHandler.go index 0f9fb90b..d814f31c 100644 --- a/process/elasticproc/tokens/sovereignIndexTokensHandler.go +++ b/process/elasticproc/tokens/sovereignIndexTokensHandler.go @@ -16,15 +16,13 @@ import ( ) type sovereignIndexTokensHandler struct { - indexingEnabled bool - mainChainElasticClient elasticproc.DatabaseClientHandler + mainChainElasticClient elasticproc.MainChainDatabaseClientHandler esdtPrefix string } // NewSovereignIndexTokensHandler creates a new sovereign index tokens handler -func NewSovereignIndexTokensHandler(indexingEnabled bool, mainChainElasticClient elasticproc.DatabaseClientHandler, esdtPrefix string) (*sovereignIndexTokensHandler, error) { +func NewSovereignIndexTokensHandler(mainChainElasticClient elasticproc.MainChainDatabaseClientHandler, esdtPrefix string) (*sovereignIndexTokensHandler, error) { return &sovereignIndexTokensHandler{ - indexingEnabled: indexingEnabled, mainChainElasticClient: mainChainElasticClient, esdtPrefix: esdtPrefix, }, nil @@ -32,7 +30,7 @@ func NewSovereignIndexTokensHandler(indexingEnabled bool, mainChainElasticClient // IndexCrossChainTokens will index the new tokens properties func (sit *sovereignIndexTokensHandler) IndexCrossChainTokens(elasticClient elasticproc.DatabaseClientHandler, scrs []*data.ScResult, buffSlice *data.BufferSlice) error { - if !sit.indexingEnabled { + if !sit.mainChainElasticClient.IsEnabled() { return nil } @@ -59,7 +57,7 @@ func (sit *sovereignIndexTokensHandler) getNewTokensFromSCRs(elasticClient elast receivedTokensIDs := make([]string, 0) for _, scr := range scrs { if scr.SenderShard == core.MainChainShardId { - receivedTokensIDs = append(receivedTokensIDs, sit.extractSovereignTokens(scr.Tokens)...) + receivedTokensIDs = append(receivedTokensIDs, sit.extractNewSovereignTokens(scr.Tokens)...) } } @@ -83,7 +81,7 @@ func (sit *sovereignIndexTokensHandler) getNewTokensFromSCRs(elasticClient elast return newTokens, nil } -func (sit *sovereignIndexTokensHandler) extractSovereignTokens(tokens []string) []string { +func (sit *sovereignIndexTokensHandler) extractNewSovereignTokens(tokens []string) []string { receivedTokensIDs := make([]string, 0) for _, token := range tokens { tokenPrefix, hasPrefix := esdt.IsValidPrefixedToken(token) diff --git a/process/elasticproc/tokens/sovereignIndexTokensHandler_test.go b/process/elasticproc/tokens/sovereignIndexTokensHandler_test.go index 2cb86b88..b881289b 100644 --- a/process/elasticproc/tokens/sovereignIndexTokensHandler_test.go +++ b/process/elasticproc/tokens/sovereignIndexTokensHandler_test.go @@ -19,12 +19,12 @@ func TestSovereignNewIndexTokensHandler(t *testing.T) { t.Parallel() t.Run("valid disabled config, should work", func(t *testing.T) { - sith, err := NewSovereignIndexTokensHandler(false, disabled.NewDisabledElasticClient(), prefix) + sith, err := NewSovereignIndexTokensHandler(disabled.NewDisabledElasticClient(), prefix) require.NoError(t, err) require.Equal(t, "*disabled.elasticClient", fmt.Sprintf("%T", sith.mainChainElasticClient)) }) t.Run("valid config, should work", func(t *testing.T) { - sith, err := NewSovereignIndexTokensHandler(true, &mock.DatabaseWriterStub{}, prefix) + sith, err := NewSovereignIndexTokensHandler(&mock.DatabaseWriterStub{}, prefix) require.NoError(t, err) require.Equal(t, "*mock.DatabaseWriterStub", fmt.Sprintf("%T", sith.mainChainElasticClient)) }) @@ -33,7 +33,7 @@ func TestSovereignNewIndexTokensHandler(t *testing.T) { func TestSovereignIndexTokensHandler_IndexCrossChainTokens(t *testing.T) { t.Parallel() - sith, err := NewSovereignIndexTokensHandler(false, disabled.NewDisabledElasticClient(), prefix) + sith, err := NewSovereignIndexTokensHandler(disabled.NewDisabledElasticClient(), prefix) require.NoError(t, err) require.NotNil(t, sith) diff --git a/scripts/script.sh b/scripts/script.sh index 2b58ebb3..166780fc 100755 --- a/scripts/script.sh +++ b/scripts/script.sh @@ -1,5 +1,5 @@ IMAGE_NAME=elastic-container -MAIN_IMAGE_NAME=main-elastic-container +MAIN_CHAIN_IMAGE_NAME=main-chain-elastic-container DEFAULT_ES_VERSION=7.16.2 PROMETHEUS_CONTAINER_NAME=prometheus_container GRAFANA_CONTAINER_NAME=grafana_container @@ -17,14 +17,13 @@ start() { docker pull docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} docker rm ${IMAGE_NAME} 2> /dev/null - docker rm ${MAIN_IMAGE_NAME} 2> /dev/null + docker rm ${MAIN_CHAIN_IMAGE_NAME} 2> /dev/null docker run -d --name "${IMAGE_NAME}" -p 9200:9200 -p 9300:9300 \ -e "discovery.type=single-node" -e "xpack.security.enabled=false" -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} - - docker run -d --name "${MAIN_IMAGE_NAME}" -p 9201:9200 -p 9301:9300 \ - -e "discovery.type=single-node" -e "xpack.security.enabled=false" -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ - docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} + docker run -d --name "${MAIN_CHAIN_IMAGE_NAME}" -p 9201:9200 -p 9301:9300 \ + -e "discovery.type=single-node" -e "xpack.security.enabled=false" -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ + docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} # Wait elastic cluster to start echo "Waiting Elasticsearch cluster to start..." @@ -33,7 +32,7 @@ start() { stop() { docker stop "${IMAGE_NAME}" - docker stop "${MAIN_IMAGE_NAME}" + docker stop "${MAIN_CHAIN_IMAGE_NAME}" } delete() { @@ -48,7 +47,7 @@ delete() { IMAGE_OPEN_SEARCH=open-container -MAIN_IMAGE_OPEN_SEARCH=main-open-container +MAIN_CHAIN_IMAGE_OPEN_SEARCH=main-chain-open-container DEFAULT_OPEN_SEARCH_VERSION=1.2.4 start_open_search() { @@ -60,11 +59,11 @@ start_open_search() { docker pull opensearchproject/opensearch:${OPEN_VERSION} docker rm ${IMAGE_OPEN_SEARCH} 2> /dev/null - docker rm ${MAIN_IMAGE_OPEN_SEARCH} 2> /dev/null + docker rm ${MAIN_CHAIN_IMAGE_OPEN_SEARCH} 2> /dev/null docker run -d --name "${IMAGE_OPEN_SEARCH}" -p 9200:9200 -p 9600:9600 \ -e "discovery.type=single-node" -e "plugins.security.disabled=true" -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ opensearchproject/opensearch:${OPEN_VERSION} - docker run -d --name "${MAIN_IMAGE_OPEN_SEARCH}" -p 9201:9200 -p 9601:9600 \ + docker run -d --name "${MAIN_CHAIN_IMAGE_OPEN_SEARCH}" -p 9201:9200 -p 9601:9600 \ -e "discovery.type=single-node" -e "plugins.security.disabled=true" -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ opensearchproject/opensearch:${OPEN_VERSION} From 2bf17922e6264abf5b42422bbd1372999f053e98 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Thu, 6 Feb 2025 10:49:45 +0200 Subject: [PATCH 43/50] fixes after review --- client/elasticClientCommon.go | 11 +++++++++++ factory/runType/sovereignRunTypeComponentsFactory.go | 8 +------- process/factory/indexerFactory.go | 11 +---------- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/client/elasticClientCommon.go b/client/elasticClientCommon.go index 553ca048..80ab9790 100644 --- a/client/elasticClientCommon.go +++ b/client/elasticClientCommon.go @@ -6,11 +6,14 @@ import ( "fmt" "io" "io/ioutil" + "math" "net/http" "net/url" "strings" + "time" "github.com/elastic/go-elasticsearch/v7/esapi" + "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" ) @@ -268,3 +271,11 @@ func parseResponse(res *esapi.Response, dest interface{}, errorHandler responseE return nil } + +// RetryBackOff returns elastic retry backoff duration +func RetryBackOff(attempt int) time.Duration { + d := time.Duration(math.Exp2(float64(attempt))) * time.Second + log.Debug("elastic: retry backoff", "attempt", attempt, "sleep duration", d) + + return d +} diff --git a/factory/runType/sovereignRunTypeComponentsFactory.go b/factory/runType/sovereignRunTypeComponentsFactory.go index 62aa1e2d..42e9fbb9 100644 --- a/factory/runType/sovereignRunTypeComponentsFactory.go +++ b/factory/runType/sovereignRunTypeComponentsFactory.go @@ -1,9 +1,7 @@ package runType import ( - "math" "net/http" - "time" "github.com/elastic/go-elasticsearch/v7" @@ -56,7 +54,7 @@ func createMainChainElasticClient(mainChainElastic factory.ElasticConfig) (elast Password: mainChainElastic.Password, Logger: &logging.CustomLogger{}, RetryOnStatus: []int{http.StatusConflict}, - RetryBackoff: retryBackOff, + RetryBackoff: client.RetryBackOff, } return client.NewMainChainElasticClient(argsEsClient, mainChainElastic.Enabled) } else { @@ -64,10 +62,6 @@ func createMainChainElasticClient(mainChainElastic factory.ElasticConfig) (elast } } -func retryBackOff(attempt int) time.Duration { - return time.Duration(math.Exp2(float64(attempt))) * time.Second -} - // IsInterfaceNil returns true if there is no value under the interface func (srtcf *sovereignRunTypeComponentsFactory) IsInterfaceNil() bool { return srtcf == nil diff --git a/process/factory/indexerFactory.go b/process/factory/indexerFactory.go index 075516a6..5def543a 100644 --- a/process/factory/indexerFactory.go +++ b/process/factory/indexerFactory.go @@ -2,9 +2,7 @@ package factory import ( "fmt" - "math" "net/http" - "time" "github.com/elastic/go-elasticsearch/v7" "github.com/multiversx/mx-chain-core-go/core" @@ -101,13 +99,6 @@ func createManagedRunTypeComponents(factory runType.RunTypeComponentsCreator) (r return managedRunTypeComponents, nil } -func retryBackOff(attempt int) time.Duration { - d := time.Duration(math.Exp2(float64(attempt))) * time.Second - log.Debug("elastic: retry backoff", "attempt", attempt, "sleep duration", d) - - return d -} - func createElasticProcessor(args ArgsIndexerFactory) (dataindexer.ElasticProcessor, error) { databaseClient, err := createElasticClient(args) if err != nil { @@ -141,7 +132,7 @@ func createElasticClient(args ArgsIndexerFactory) (elasticproc.DatabaseClientHan Password: args.Password, Logger: &logging.CustomLogger{}, RetryOnStatus: []int{http.StatusConflict}, - RetryBackoff: retryBackOff, + RetryBackoff: client.RetryBackOff, } if check.IfNil(args.StatusMetrics) { From c54509b9f09c77d3d70ceb4bf53e6ae9bb50d4ad Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Mon, 10 Feb 2025 10:03:06 +0200 Subject: [PATCH 44/50] fixes after review --- client/disabled/disabledElasticClient.go | 78 +++++++++++++++++++ ..._test.go => disabledElasticClient_test.go} | 0 client/disabled/elasticClient.go | 78 ------------------- cmd/elasticindexer/config/config.toml | 1 + cmd/elasticindexer/config/prefs.toml | 2 + config/config.go | 4 +- .../tokens/sovereignIndexTokensHandler.go | 4 +- 7 files changed, 85 insertions(+), 82 deletions(-) create mode 100644 client/disabled/disabledElasticClient.go rename client/disabled/{elasticClient_test.go => disabledElasticClient_test.go} (100%) delete mode 100644 client/disabled/elasticClient.go diff --git a/client/disabled/disabledElasticClient.go b/client/disabled/disabledElasticClient.go new file mode 100644 index 00000000..313c92bf --- /dev/null +++ b/client/disabled/disabledElasticClient.go @@ -0,0 +1,78 @@ +package disabled + +import ( + "bytes" + "context" +) + +type disabledElasticClient struct{} + +// NewDisabledElasticClient - +func NewDisabledElasticClient() *disabledElasticClient { + return &disabledElasticClient{} +} + +// DoBulkRequest - +func (dec *disabledElasticClient) DoBulkRequest(_ context.Context, _ *bytes.Buffer, _ string) error { + return nil +} + +// DoQueryRemove - +func (dec *disabledElasticClient) DoQueryRemove(_ context.Context, _ string, _ *bytes.Buffer) error { + return nil +} + +// DoMultiGet - +func (dec *disabledElasticClient) DoMultiGet(_ context.Context, _ []string, _ string, _ bool, _ interface{}) error { + return nil +} + +// DoScrollRequest - +func (dec *disabledElasticClient) DoScrollRequest(_ context.Context, _ string, _ []byte, _ bool, _ func(responseBytes []byte) error) error { + return nil +} + +// DoCountRequest - +func (dec *disabledElasticClient) DoCountRequest(_ context.Context, _ string, _ []byte) (uint64, error) { + return 0, nil +} + +// UpdateByQuery - +func (dec *disabledElasticClient) UpdateByQuery(_ context.Context, _ string, _ *bytes.Buffer) error { + return nil +} + +// PutMappings - +func (dec *disabledElasticClient) PutMappings(_ string, _ *bytes.Buffer) error { + return nil +} + +// CheckAndCreateIndex - +func (dec *disabledElasticClient) CheckAndCreateIndex(_ string) error { + return nil +} + +// CheckAndCreateAlias - +func (dec *disabledElasticClient) CheckAndCreateAlias(_ string, _ string) error { + return nil +} + +// CheckAndCreateTemplate - +func (dec *disabledElasticClient) CheckAndCreateTemplate(_ string, _ *bytes.Buffer) error { + return nil +} + +// CheckAndCreatePolicy - +func (dec *disabledElasticClient) CheckAndCreatePolicy(_ string, _ *bytes.Buffer) error { + return nil +} + +// IsEnabled - +func (dec *disabledElasticClient) IsEnabled() bool { + return false +} + +// IsInterfaceNil - returns true if there is no value under the interface +func (dec *disabledElasticClient) IsInterfaceNil() bool { + return dec == nil +} diff --git a/client/disabled/elasticClient_test.go b/client/disabled/disabledElasticClient_test.go similarity index 100% rename from client/disabled/elasticClient_test.go rename to client/disabled/disabledElasticClient_test.go diff --git a/client/disabled/elasticClient.go b/client/disabled/elasticClient.go deleted file mode 100644 index 41ccc6c0..00000000 --- a/client/disabled/elasticClient.go +++ /dev/null @@ -1,78 +0,0 @@ -package disabled - -import ( - "bytes" - "context" -) - -type elasticClient struct{} - -// NewDisabledElasticClient - -func NewDisabledElasticClient() *elasticClient { - return &elasticClient{} -} - -// DoBulkRequest - -func (ec *elasticClient) DoBulkRequest(_ context.Context, _ *bytes.Buffer, _ string) error { - return nil -} - -// DoQueryRemove - -func (ec *elasticClient) DoQueryRemove(_ context.Context, _ string, _ *bytes.Buffer) error { - return nil -} - -// DoMultiGet - -func (ec *elasticClient) DoMultiGet(_ context.Context, _ []string, _ string, _ bool, _ interface{}) error { - return nil -} - -// DoScrollRequest - -func (ec *elasticClient) DoScrollRequest(_ context.Context, _ string, _ []byte, _ bool, _ func(responseBytes []byte) error) error { - return nil -} - -// DoCountRequest - -func (ec *elasticClient) DoCountRequest(_ context.Context, _ string, _ []byte) (uint64, error) { - return 0, nil -} - -// UpdateByQuery - -func (ec *elasticClient) UpdateByQuery(_ context.Context, _ string, _ *bytes.Buffer) error { - return nil -} - -// PutMappings - -func (ec *elasticClient) PutMappings(_ string, _ *bytes.Buffer) error { - return nil -} - -// CheckAndCreateIndex - -func (ec *elasticClient) CheckAndCreateIndex(_ string) error { - return nil -} - -// CheckAndCreateAlias - -func (ec *elasticClient) CheckAndCreateAlias(_ string, _ string) error { - return nil -} - -// CheckAndCreateTemplate - -func (ec *elasticClient) CheckAndCreateTemplate(_ string, _ *bytes.Buffer) error { - return nil -} - -// CheckAndCreatePolicy - -func (ec *elasticClient) CheckAndCreatePolicy(_ string, _ *bytes.Buffer) error { - return nil -} - -// IsEnabled - -func (ec *elasticClient) IsEnabled() bool { - return false -} - -// IsInterfaceNil - returns true if there is no value under the interface -func (ec *elasticClient) IsInterfaceNil() bool { - return ec == nil -} diff --git a/cmd/elasticindexer/config/config.toml b/cmd/elasticindexer/config/config.toml index b63329c7..5fdf7122 100644 --- a/cmd/elasticindexer/config/config.toml +++ b/cmd/elasticindexer/config/config.toml @@ -4,6 +4,7 @@ "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags", "logs", "delegators", "operations", "esdts", "values", "events" ] + esdt-prefix = "" [config.address-converter] length = 32 type = "bech32" diff --git a/cmd/elasticindexer/config/prefs.toml b/cmd/elasticindexer/config/prefs.toml index 42dc095f..bc6a2743 100644 --- a/cmd/elasticindexer/config/prefs.toml +++ b/cmd/elasticindexer/config/prefs.toml @@ -24,6 +24,8 @@ password = "" bulk-request-max-size-in-bytes = 4194304 # 4MB + # Configuration for main chain elastic cluster + # Used by the sovereign chain indexer to index incoming new tokens properties [config.main-chain-elastic-cluster] enabled = true url = "http://localhost:9201" diff --git a/config/config.go b/config/config.go index 60c56b4d..203bd6e7 100644 --- a/config/config.go +++ b/config/config.go @@ -4,6 +4,7 @@ package config type Config struct { Config struct { AvailableIndices []string `toml:"available-indices"` + ESDTPrefix string `toml:"esdt-prefix"` AddressConverter struct { Length int `toml:"length"` Type string `toml:"type"` @@ -29,8 +30,7 @@ type Config struct { LogsPath string `toml:"logs-path"` } `toml:"logs"` } `toml:"config"` - Sovereign bool - ESDTPrefix string + Sovereign bool } // ClusterConfig will hold the config for the Elasticsearch cluster diff --git a/process/elasticproc/tokens/sovereignIndexTokensHandler.go b/process/elasticproc/tokens/sovereignIndexTokensHandler.go index d814f31c..1a1e1385 100644 --- a/process/elasticproc/tokens/sovereignIndexTokensHandler.go +++ b/process/elasticproc/tokens/sovereignIndexTokensHandler.go @@ -50,7 +50,7 @@ func (sit *sovereignIndexTokensHandler) IndexCrossChainTokens(elasticClient elas return err } - return sit.indexNewTokens(mainChainTokens.Docs, buffSlice) + return sit.serializeNewTokens(mainChainTokens.Docs, buffSlice) } func (sit *sovereignIndexTokensHandler) getNewTokensFromSCRs(elasticClient elasticproc.DatabaseClientHandler, scrs []*data.ScResult) ([]string, error) { @@ -107,7 +107,7 @@ func getTokenCollection(hasPrefix bool, tokenIdentifier string) string { return "" } -func (sit *sovereignIndexTokensHandler) indexNewTokens(responseTokensInfo []data.ResponseTokenInfoDB, buffSlice *data.BufferSlice) error { +func (sit *sovereignIndexTokensHandler) serializeNewTokens(responseTokensInfo []data.ResponseTokenInfoDB, buffSlice *data.BufferSlice) error { for _, responseToken := range responseTokensInfo { token, identifier := formatToken(responseToken) From 5cec770a51a610f4e3bcae322b9e4d987e10c893 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Mon, 10 Feb 2025 10:23:42 +0200 Subject: [PATCH 45/50] fixes after review --- client/disabled/disabledElasticClient.go | 78 ------------------- client/disabled/elasticClient.go | 78 +++++++++++++++++++ ...icClient_test.go => elasticClient_test.go} | 0 factory/runType/runTypeComponentsHandler.go | 3 +- process/dataindexer/errors.go | 3 + process/elasticproc/check.go | 3 +- ...ndler.go => disabledIndexTokensHandler.go} | 12 +-- ....go => disabledIndexTokensHandler_test.go} | 0 process/elasticproc/transactions/errors.go | 3 - 9 files changed, 90 insertions(+), 90 deletions(-) delete mode 100644 client/disabled/disabledElasticClient.go create mode 100644 client/disabled/elasticClient.go rename client/disabled/{disabledElasticClient_test.go => elasticClient_test.go} (100%) rename process/elasticproc/tokens/{indexTokensHandler.go => disabledIndexTokensHandler.go} (50%) rename process/elasticproc/tokens/{indexTokensHandler_test.go => disabledIndexTokensHandler_test.go} (100%) diff --git a/client/disabled/disabledElasticClient.go b/client/disabled/disabledElasticClient.go deleted file mode 100644 index 313c92bf..00000000 --- a/client/disabled/disabledElasticClient.go +++ /dev/null @@ -1,78 +0,0 @@ -package disabled - -import ( - "bytes" - "context" -) - -type disabledElasticClient struct{} - -// NewDisabledElasticClient - -func NewDisabledElasticClient() *disabledElasticClient { - return &disabledElasticClient{} -} - -// DoBulkRequest - -func (dec *disabledElasticClient) DoBulkRequest(_ context.Context, _ *bytes.Buffer, _ string) error { - return nil -} - -// DoQueryRemove - -func (dec *disabledElasticClient) DoQueryRemove(_ context.Context, _ string, _ *bytes.Buffer) error { - return nil -} - -// DoMultiGet - -func (dec *disabledElasticClient) DoMultiGet(_ context.Context, _ []string, _ string, _ bool, _ interface{}) error { - return nil -} - -// DoScrollRequest - -func (dec *disabledElasticClient) DoScrollRequest(_ context.Context, _ string, _ []byte, _ bool, _ func(responseBytes []byte) error) error { - return nil -} - -// DoCountRequest - -func (dec *disabledElasticClient) DoCountRequest(_ context.Context, _ string, _ []byte) (uint64, error) { - return 0, nil -} - -// UpdateByQuery - -func (dec *disabledElasticClient) UpdateByQuery(_ context.Context, _ string, _ *bytes.Buffer) error { - return nil -} - -// PutMappings - -func (dec *disabledElasticClient) PutMappings(_ string, _ *bytes.Buffer) error { - return nil -} - -// CheckAndCreateIndex - -func (dec *disabledElasticClient) CheckAndCreateIndex(_ string) error { - return nil -} - -// CheckAndCreateAlias - -func (dec *disabledElasticClient) CheckAndCreateAlias(_ string, _ string) error { - return nil -} - -// CheckAndCreateTemplate - -func (dec *disabledElasticClient) CheckAndCreateTemplate(_ string, _ *bytes.Buffer) error { - return nil -} - -// CheckAndCreatePolicy - -func (dec *disabledElasticClient) CheckAndCreatePolicy(_ string, _ *bytes.Buffer) error { - return nil -} - -// IsEnabled - -func (dec *disabledElasticClient) IsEnabled() bool { - return false -} - -// IsInterfaceNil - returns true if there is no value under the interface -func (dec *disabledElasticClient) IsInterfaceNil() bool { - return dec == nil -} diff --git a/client/disabled/elasticClient.go b/client/disabled/elasticClient.go new file mode 100644 index 00000000..41ccc6c0 --- /dev/null +++ b/client/disabled/elasticClient.go @@ -0,0 +1,78 @@ +package disabled + +import ( + "bytes" + "context" +) + +type elasticClient struct{} + +// NewDisabledElasticClient - +func NewDisabledElasticClient() *elasticClient { + return &elasticClient{} +} + +// DoBulkRequest - +func (ec *elasticClient) DoBulkRequest(_ context.Context, _ *bytes.Buffer, _ string) error { + return nil +} + +// DoQueryRemove - +func (ec *elasticClient) DoQueryRemove(_ context.Context, _ string, _ *bytes.Buffer) error { + return nil +} + +// DoMultiGet - +func (ec *elasticClient) DoMultiGet(_ context.Context, _ []string, _ string, _ bool, _ interface{}) error { + return nil +} + +// DoScrollRequest - +func (ec *elasticClient) DoScrollRequest(_ context.Context, _ string, _ []byte, _ bool, _ func(responseBytes []byte) error) error { + return nil +} + +// DoCountRequest - +func (ec *elasticClient) DoCountRequest(_ context.Context, _ string, _ []byte) (uint64, error) { + return 0, nil +} + +// UpdateByQuery - +func (ec *elasticClient) UpdateByQuery(_ context.Context, _ string, _ *bytes.Buffer) error { + return nil +} + +// PutMappings - +func (ec *elasticClient) PutMappings(_ string, _ *bytes.Buffer) error { + return nil +} + +// CheckAndCreateIndex - +func (ec *elasticClient) CheckAndCreateIndex(_ string) error { + return nil +} + +// CheckAndCreateAlias - +func (ec *elasticClient) CheckAndCreateAlias(_ string, _ string) error { + return nil +} + +// CheckAndCreateTemplate - +func (ec *elasticClient) CheckAndCreateTemplate(_ string, _ *bytes.Buffer) error { + return nil +} + +// CheckAndCreatePolicy - +func (ec *elasticClient) CheckAndCreatePolicy(_ string, _ *bytes.Buffer) error { + return nil +} + +// IsEnabled - +func (ec *elasticClient) IsEnabled() bool { + return false +} + +// IsInterfaceNil - returns true if there is no value under the interface +func (ec *elasticClient) IsInterfaceNil() bool { + return ec == nil +} diff --git a/client/disabled/disabledElasticClient_test.go b/client/disabled/elasticClient_test.go similarity index 100% rename from client/disabled/disabledElasticClient_test.go rename to client/disabled/elasticClient_test.go diff --git a/factory/runType/runTypeComponentsHandler.go b/factory/runType/runTypeComponentsHandler.go index b9697008..113be479 100644 --- a/factory/runType/runTypeComponentsHandler.go +++ b/factory/runType/runTypeComponentsHandler.go @@ -5,6 +5,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" + elasticIndexer "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" ) @@ -80,7 +81,7 @@ func (mrtc *managedRunTypeComponents) CheckSubcomponents() error { return transactions.ErrNilRewardTxDataHandler } if check.IfNil(mrtc.indexTokensHandler) { - return transactions.ErrNilIndexTokensHandler + return elasticIndexer.ErrNilIndexTokensHandler } return nil } diff --git a/process/dataindexer/errors.go b/process/dataindexer/errors.go index 9b4fbd0f..9495e93f 100644 --- a/process/dataindexer/errors.go +++ b/process/dataindexer/errors.go @@ -88,3 +88,6 @@ var ErrNilOperationsHandler = errors.New("nil operations handler") // ErrNilBlockContainerHandler signals that a nil block container handler has been provided var ErrNilBlockContainerHandler = errors.New("nil bock container handler") + +// ErrNilIndexTokensHandler signals that a nil index tokens handler has been provided +var ErrNilIndexTokensHandler = errors.New("nil index tokens handler") diff --git a/process/elasticproc/check.go b/process/elasticproc/check.go index b3953d15..d15a19fc 100644 --- a/process/elasticproc/check.go +++ b/process/elasticproc/check.go @@ -4,7 +4,6 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" elasticIndexer "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" - "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" ) func checkArguments(arguments *ArgElasticProcessor) error { @@ -42,7 +41,7 @@ func checkArguments(arguments *ArgElasticProcessor) error { return elasticIndexer.ErrNilOperationsHandler } if check.IfNilReflect(arguments.IndexTokensHandler) { - return transactions.ErrNilIndexTokensHandler + return elasticIndexer.ErrNilIndexTokensHandler } return nil diff --git a/process/elasticproc/tokens/indexTokensHandler.go b/process/elasticproc/tokens/disabledIndexTokensHandler.go similarity index 50% rename from process/elasticproc/tokens/indexTokensHandler.go rename to process/elasticproc/tokens/disabledIndexTokensHandler.go index 99a4b8c4..18fc63f8 100644 --- a/process/elasticproc/tokens/indexTokensHandler.go +++ b/process/elasticproc/tokens/disabledIndexTokensHandler.go @@ -5,19 +5,19 @@ import ( "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" ) -type indexTokensHandler struct{} +type disabledTndexTokensHandler struct{} // NewDisabledIndexTokensHandler creates a new disabled index tokens handler -func NewDisabledIndexTokensHandler() *indexTokensHandler { - return &indexTokensHandler{} +func NewDisabledIndexTokensHandler() *disabledTndexTokensHandler { + return &disabledTndexTokensHandler{} } // IndexCrossChainTokens should do nothing and return no error -func (it *indexTokensHandler) IndexCrossChainTokens(_ elasticproc.DatabaseClientHandler, _ []*data.ScResult, _ *data.BufferSlice) error { +func (dit *disabledTndexTokensHandler) IndexCrossChainTokens(_ elasticproc.DatabaseClientHandler, _ []*data.ScResult, _ *data.BufferSlice) error { return nil } // IsInterfaceNil returns true if there is no value under the interface -func (it *indexTokensHandler) IsInterfaceNil() bool { - return it == nil +func (dit *disabledTndexTokensHandler) IsInterfaceNil() bool { + return dit == nil } diff --git a/process/elasticproc/tokens/indexTokensHandler_test.go b/process/elasticproc/tokens/disabledIndexTokensHandler_test.go similarity index 100% rename from process/elasticproc/tokens/indexTokensHandler_test.go rename to process/elasticproc/tokens/disabledIndexTokensHandler_test.go diff --git a/process/elasticproc/transactions/errors.go b/process/elasticproc/transactions/errors.go index f9280c86..9f91b390 100644 --- a/process/elasticproc/transactions/errors.go +++ b/process/elasticproc/transactions/errors.go @@ -9,6 +9,3 @@ var ErrNilTxHashExtractor = errors.New("nil tx hash extractor") // ErrNilRewardTxDataHandler signals that a nil rewards tx data handler has been provided var ErrNilRewardTxDataHandler = errors.New("nil reward tx data handler") - -// ErrNilIndexTokensHandler signals that a nil index tokens handler has been provided -var ErrNilIndexTokensHandler = errors.New("nil index tokens handler") From 5978925e96504dd23120bc540c1b9c8dcd2e6ed4 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Mon, 10 Feb 2025 16:00:06 +0200 Subject: [PATCH 46/50] sovereign elastic indexer docker fixes --- .github/workflows/deploy-docker.yml | 10 ++++++---- Dockerfile-sovereign | 26 ++++++++++++++++++++++++++ Makefile | 7 +++++++ 3 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 Dockerfile-sovereign diff --git a/.github/workflows/deploy-docker.yml b/.github/workflows/deploy-docker.yml index df006e3f..c6d85c6b 100644 --- a/.github/workflows/deploy-docker.yml +++ b/.github/workflows/deploy-docker.yml @@ -3,6 +3,7 @@ name: Publish Docker image on: release: types: [published] + pull_request: jobs: push_to_registry: @@ -31,24 +32,25 @@ jobs: fi - name: Log in to Docker Hub + if: ${{ github.event_name == 'release' && github.event.action == 'published' }} uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Extract metadata (tags, labels) for Docker + if: ${{ github.event_name == 'release' && github.event.action == 'published' }} id: meta uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 with: - images: multiversx/elastic-indexer + images: multiversx/sovereign-elastic-indexer - name: Build and push Docker image id: push uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 with: context: . - file: ./Dockerfile - push: true + file: ./Dockerfile-sovereign + push: ${{ github.event_name == 'release' && github.event.action == 'published' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - diff --git a/Dockerfile-sovereign b/Dockerfile-sovereign new file mode 100644 index 00000000..c8b72a1a --- /dev/null +++ b/Dockerfile-sovereign @@ -0,0 +1,26 @@ +FROM golang:1.20.7 as builder + +RUN apt-get update && apt-get install -y + +WORKDIR /multiversx +COPY . . + +WORKDIR /multiversx/cmd/elasticindexer + +RUN go build -o elasticindexer + +# ===== SECOND STAGE ====== +FROM ubuntu:22.04 +RUN apt-get update && apt-get install -y + +RUN useradd -m -u 1000 appuser +USER appuser + +COPY --from=builder --chown=appuser /multiversx/cmd/elasticindexer /multiversx + +EXPOSE 22111 + +WORKDIR /multiversx + +ENTRYPOINT ["./elasticindexer", "--sovereign"] +CMD ["--log-level", "*:DEBUG"] diff --git a/Makefile b/Makefile index a000f850..075cc715 100644 --- a/Makefile +++ b/Makefile @@ -36,9 +36,16 @@ integration-tests-open-search: INDEXER_IMAGE_NAME="elasticindexer" INDEXER_IMAGE_TAG="latest" DOCKER_FILE=Dockerfile +SOVEREIGN_DOCKER_FILE=Dockerfile-sovereign docker-build: docker build \ -t ${INDEXER_IMAGE_NAME}:${INDEXER_IMAGE_TAG} \ -f ${DOCKER_FILE} \ . + +docker-sovereign-build: + docker build \ + -t ${INDEXER_IMAGE_NAME}:${INDEXER_IMAGE_TAG} \ + -f ${SOVEREIGN_DOCKER_FILE} \ + . From 3c1d628c2396292d941ed368b7bb1b5cb485a205 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Tue, 11 Feb 2025 16:01:44 +0200 Subject: [PATCH 47/50] fixes after review --- client/mainChainElasticClient.go | 16 ++++----- client/mainChainElasticClient_test.go | 33 +++++++++---------- .../sovereignRunTypeComponentsFactory.go | 7 +++- integrationtests/utils.go | 6 ++-- process/elasticproc/interface.go | 1 + 5 files changed, 32 insertions(+), 31 deletions(-) diff --git a/client/mainChainElasticClient.go b/client/mainChainElasticClient.go index e378dd9a..64b36a2b 100644 --- a/client/mainChainElasticClient.go +++ b/client/mainChainElasticClient.go @@ -1,21 +1,16 @@ package client import ( - "github.com/elastic/go-elasticsearch/v7" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" ) type mainChainElasticClient struct { - *elasticClient + elasticproc.DatabaseClientHandler indexingEnabled bool } // NewMainChainElasticClient creates a new sovereign elastic client -func NewMainChainElasticClient(cfg elasticsearch.Config, indexingEnabled bool) (*mainChainElasticClient, error) { - esClient, err := NewElasticClient(cfg) - if err != nil { - return nil, err - } - +func NewMainChainElasticClient(esClient elasticproc.DatabaseClientHandler, indexingEnabled bool) (*mainChainElasticClient, error) { return &mainChainElasticClient{ esClient, indexingEnabled, @@ -26,3 +21,8 @@ func NewMainChainElasticClient(cfg elasticsearch.Config, indexingEnabled bool) ( func (mcec *mainChainElasticClient) IsEnabled() bool { return mcec.indexingEnabled } + +// IsInterfaceNil returns true if there is no value under the interface +func (mcec *mainChainElasticClient) IsInterfaceNil() bool { + return mcec == nil +} diff --git a/client/mainChainElasticClient_test.go b/client/mainChainElasticClient_test.go index c72c5079..74137dde 100644 --- a/client/mainChainElasticClient_test.go +++ b/client/mainChainElasticClient_test.go @@ -6,31 +6,28 @@ import ( "github.com/elastic/go-elasticsearch/v7" "github.com/stretchr/testify/require" - - indexer "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" ) func TestNewMainChainElasticClient(t *testing.T) { - t.Run("no url, should error", func(t *testing.T) { - esClient, err := NewMainChainElasticClient(elasticsearch.Config{ - Addresses: []string{}, - }, true) - require.Nil(t, esClient) - require.Equal(t, indexer.ErrNoElasticUrlProvided, err) - }) - t.Run("should work", func(t *testing.T) { - esClient, err := NewMainChainElasticClient(elasticsearch.Config{ - Addresses: []string{"http://localhost:9200"}, - }, true) - require.Nil(t, err) - require.Equal(t, "*client.mainChainElasticClient", fmt.Sprintf("%T", esClient)) + esClient, err := NewElasticClient(elasticsearch.Config{ + Addresses: []string{"http://localhost:9200"}, }) + require.Nil(t, err) + require.NotNil(t, esClient) + + mainChainESClient, err := NewMainChainElasticClient(esClient, true) + require.NoError(t, err) + require.Equal(t, "*client.mainChainElasticClient", fmt.Sprintf("%T", mainChainESClient)) } func TestMainChainElasticClient_IsEnabled(t *testing.T) { - esClient, err := NewMainChainElasticClient(elasticsearch.Config{ + esClient, err := NewElasticClient(elasticsearch.Config{ Addresses: []string{"http://localhost:9200"}, - }, true) + }) require.Nil(t, err) - require.Equal(t, true, esClient.IsEnabled()) + require.NotNil(t, esClient) + + mainChainESClient, err := NewMainChainElasticClient(esClient, true) + require.NoError(t, err) + require.Equal(t, true, mainChainESClient.IsEnabled()) } diff --git a/factory/runType/sovereignRunTypeComponentsFactory.go b/factory/runType/sovereignRunTypeComponentsFactory.go index 42e9fbb9..4bafdf1a 100644 --- a/factory/runType/sovereignRunTypeComponentsFactory.go +++ b/factory/runType/sovereignRunTypeComponentsFactory.go @@ -56,7 +56,12 @@ func createMainChainElasticClient(mainChainElastic factory.ElasticConfig) (elast RetryOnStatus: []int{http.StatusConflict}, RetryBackoff: client.RetryBackOff, } - return client.NewMainChainElasticClient(argsEsClient, mainChainElastic.Enabled) + esClient, err := client.NewElasticClient(argsEsClient) + if err != nil { + return nil, err + } + + return client.NewMainChainElasticClient(esClient, mainChainElastic.Enabled) } else { return disabled.NewDisabledElasticClient(), nil } diff --git a/integrationtests/utils.go b/integrationtests/utils.go index 796c4ac1..3a85f7eb 100644 --- a/integrationtests/utils.go +++ b/integrationtests/utils.go @@ -45,10 +45,8 @@ func createESClient(url string) (elasticproc.DatabaseClientHandler, error) { // nolint func createMainChainESClient(url string, enabled bool) (elasticproc.MainChainDatabaseClientHandler, error) { - return client.NewMainChainElasticClient(elasticsearch.Config{ - Addresses: []string{url}, - Logger: &logging.CustomLogger{}, - }, enabled) + esClient, _ := createESClient(url) + return client.NewMainChainElasticClient(esClient, enabled) } // nolint diff --git a/process/elasticproc/interface.go b/process/elasticproc/interface.go index b5108f9b..f80e3809 100644 --- a/process/elasticproc/interface.go +++ b/process/elasticproc/interface.go @@ -17,6 +17,7 @@ import ( type MainChainDatabaseClientHandler interface { DatabaseClientHandler IsEnabled() bool + IsInterfaceNil() bool } // DatabaseClientHandler defines the actions that a component that handles requests should do From b09b8af1c3a02f4ca690108eb0b9baa287887815 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Wed, 12 Feb 2025 11:10:28 +0200 Subject: [PATCH 48/50] fixes after review --- client/mainChainElasticClient.go | 7 +++++++ client/mainChainElasticClient_test.go | 23 ++++++++++++++++------- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/client/mainChainElasticClient.go b/client/mainChainElasticClient.go index 64b36a2b..042504e9 100644 --- a/client/mainChainElasticClient.go +++ b/client/mainChainElasticClient.go @@ -1,6 +1,9 @@ package client import ( + "github.com/multiversx/mx-chain-core-go/core/check" + + "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" ) @@ -11,6 +14,10 @@ type mainChainElasticClient struct { // NewMainChainElasticClient creates a new sovereign elastic client func NewMainChainElasticClient(esClient elasticproc.DatabaseClientHandler, indexingEnabled bool) (*mainChainElasticClient, error) { + if check.IfNil(esClient) { + return nil, dataindexer.ErrNilDatabaseClient + } + return &mainChainElasticClient{ esClient, indexingEnabled, diff --git a/client/mainChainElasticClient_test.go b/client/mainChainElasticClient_test.go index 74137dde..c674cb32 100644 --- a/client/mainChainElasticClient_test.go +++ b/client/mainChainElasticClient_test.go @@ -6,18 +6,27 @@ import ( "github.com/elastic/go-elasticsearch/v7" "github.com/stretchr/testify/require" + + "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" ) func TestNewMainChainElasticClient(t *testing.T) { - esClient, err := NewElasticClient(elasticsearch.Config{ - Addresses: []string{"http://localhost:9200"}, + t.Run("nil elastic client, should error", func(t *testing.T) { + mainChainESClient, err := NewMainChainElasticClient(nil, true) + require.Error(t, err, dataindexer.ErrNilDatabaseClient) + require.True(t, mainChainESClient.IsInterfaceNil()) }) - require.Nil(t, err) - require.NotNil(t, esClient) + t.Run("valid elastic client, should work", func(t *testing.T) { + esClient, err := NewElasticClient(elasticsearch.Config{ + Addresses: []string{"http://localhost:9200"}, + }) + require.Nil(t, err) + require.NotNil(t, esClient) - mainChainESClient, err := NewMainChainElasticClient(esClient, true) - require.NoError(t, err) - require.Equal(t, "*client.mainChainElasticClient", fmt.Sprintf("%T", mainChainESClient)) + mainChainESClient, err := NewMainChainElasticClient(esClient, true) + require.NoError(t, err) + require.Equal(t, "*client.mainChainElasticClient", fmt.Sprintf("%T", mainChainESClient)) + }) } func TestMainChainElasticClient_IsEnabled(t *testing.T) { From 89decf3cd2759ae460488d486442f56fabe485eb Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Mon, 17 Feb 2025 16:45:42 +0200 Subject: [PATCH 49/50] go.mod update for core --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d4d35593..77e71efc 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/gin-gonic/gin v1.10.0 github.com/google/uuid v1.6.0 github.com/multiversx/mx-chain-communication-go v1.1.1 - github.com/multiversx/mx-chain-core-go v1.2.25-0.20250131091121-26781bb55fe9 + github.com/multiversx/mx-chain-core-go v1.2.25-0.20250206111825-25fbb1b4851c github.com/multiversx/mx-chain-logger-go v1.0.15 github.com/multiversx/mx-chain-vm-common-go v1.5.17-0.20241119132002-2fa80c5ec516 github.com/prometheus/client_model v0.4.0 diff --git a/go.sum b/go.sum index 0d9fdbae..0b165928 100644 --- a/go.sum +++ b/go.sum @@ -253,8 +253,8 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/mx-chain-communication-go v1.1.1 h1:y4DoQeQOJTaSUsRzczQFazf8JYQmInddypApqA3AkwM= github.com/multiversx/mx-chain-communication-go v1.1.1/go.mod h1:WK6bP4pGEHGDDna/AYRIMtl6G9OA0NByI1Lw8PmOnRM= -github.com/multiversx/mx-chain-core-go v1.2.25-0.20250131091121-26781bb55fe9 h1:hm9b1I3ywGV6vOMFX86sZ5jZ7hdIqq1IH8EPNj0Pliw= -github.com/multiversx/mx-chain-core-go v1.2.25-0.20250131091121-26781bb55fe9/go.mod h1:P/YBoFnt25XUaCQ7Q/SD15vhnc9yV5JDhHxyFO9P8Z0= +github.com/multiversx/mx-chain-core-go v1.2.25-0.20250206111825-25fbb1b4851c h1:Cz5b0xd9lbSWGIwmfuPuHqL0e5kTun/PW5NpkVRIAXQ= +github.com/multiversx/mx-chain-core-go v1.2.25-0.20250206111825-25fbb1b4851c/go.mod h1:P/YBoFnt25XUaCQ7Q/SD15vhnc9yV5JDhHxyFO9P8Z0= github.com/multiversx/mx-chain-crypto-go v1.2.12 h1:zWip7rpUS4CGthJxfKn5MZfMfYPjVjIiCID6uX5BSOk= github.com/multiversx/mx-chain-logger-go v1.0.15 h1:HlNdK8etyJyL9NQ+6mIXyKPEBo+wRqOwi3n+m2QIHXc= github.com/multiversx/mx-chain-logger-go v1.0.15/go.mod h1:t3PRKaWB1M+i6gUfD27KXgzLJJC+mAQiN+FLlL1yoGQ= From ddd47d85cfcd10e8ff79436664090475e833a038 Mon Sep 17 00:00:00 2001 From: axenteoctavian Date: Mon, 17 Feb 2025 16:55:07 +0200 Subject: [PATCH 50/50] fix sovereign test after merge --- integrationtests/testdata/incomingSCR/incoming-scr.json | 1 + 1 file changed, 1 insertion(+) diff --git a/integrationtests/testdata/incomingSCR/incoming-scr.json b/integrationtests/testdata/incomingSCR/incoming-scr.json index 5f4c142e..7ce746df 100644 --- a/integrationtests/testdata/incomingSCR/incoming-scr.json +++ b/integrationtests/testdata/incomingSCR/incoming-scr.json @@ -19,6 +19,7 @@ "TKN12-1c2b3a", "NFT-abc123-01" ], + "epoch": 0, "esdtValues": [ "123", "333",