diff --git a/.github/workflows/deploy-docker.yml b/.github/workflows/deploy-docker.yml index df006e3f..c6d85c6b 100644 --- a/.github/workflows/deploy-docker.yml +++ b/.github/workflows/deploy-docker.yml @@ -3,6 +3,7 @@ name: Publish Docker image on: release: types: [published] + pull_request: jobs: push_to_registry: @@ -31,24 +32,25 @@ jobs: fi - name: Log in to Docker Hub + if: ${{ github.event_name == 'release' && github.event.action == 'published' }} uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Extract metadata (tags, labels) for Docker + if: ${{ github.event_name == 'release' && github.event.action == 'published' }} id: meta uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 with: - images: multiversx/elastic-indexer + images: multiversx/sovereign-elastic-indexer - name: Build and push Docker image id: push uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 with: context: . - file: ./Dockerfile - push: true + file: ./Dockerfile-sovereign + push: ${{ github.event_name == 'release' && github.event.action == 'published' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - diff --git a/Dockerfile-sovereign b/Dockerfile-sovereign new file mode 100644 index 00000000..c8b72a1a --- /dev/null +++ b/Dockerfile-sovereign @@ -0,0 +1,26 @@ +FROM golang:1.20.7 as builder + +RUN apt-get update && apt-get install -y + +WORKDIR /multiversx +COPY . . + +WORKDIR /multiversx/cmd/elasticindexer + +RUN go build -o elasticindexer + +# ===== SECOND STAGE ====== +FROM ubuntu:22.04 +RUN apt-get update && apt-get install -y + +RUN useradd -m -u 1000 appuser +USER appuser + +COPY --from=builder --chown=appuser /multiversx/cmd/elasticindexer /multiversx + +EXPOSE 22111 + +WORKDIR /multiversx + +ENTRYPOINT ["./elasticindexer", "--sovereign"] +CMD ["--log-level", "*:DEBUG"] diff --git a/Makefile b/Makefile index a000f850..075cc715 100644 --- a/Makefile +++ b/Makefile @@ -36,9 +36,16 @@ integration-tests-open-search: INDEXER_IMAGE_NAME="elasticindexer" INDEXER_IMAGE_TAG="latest" DOCKER_FILE=Dockerfile +SOVEREIGN_DOCKER_FILE=Dockerfile-sovereign docker-build: docker build \ -t ${INDEXER_IMAGE_NAME}:${INDEXER_IMAGE_TAG} \ -f ${DOCKER_FILE} \ . + +docker-sovereign-build: + docker build \ + -t ${INDEXER_IMAGE_NAME}:${INDEXER_IMAGE_TAG} \ + -f ${SOVEREIGN_DOCKER_FILE} \ + . diff --git a/client/disabled/elasticClient.go b/client/disabled/elasticClient.go new file mode 100644 index 00000000..41ccc6c0 --- /dev/null +++ b/client/disabled/elasticClient.go @@ -0,0 +1,78 @@ +package disabled + +import ( + "bytes" + "context" +) + +type elasticClient struct{} + +// NewDisabledElasticClient - +func NewDisabledElasticClient() *elasticClient { + return &elasticClient{} +} + +// DoBulkRequest - +func (ec *elasticClient) DoBulkRequest(_ context.Context, _ *bytes.Buffer, _ string) error { + return nil +} + +// DoQueryRemove - +func (ec *elasticClient) DoQueryRemove(_ context.Context, _ string, _ *bytes.Buffer) error { + return nil +} + +// DoMultiGet - +func (ec *elasticClient) DoMultiGet(_ context.Context, _ []string, _ string, _ bool, _ interface{}) error { + return nil +} + +// DoScrollRequest - +func (ec *elasticClient) DoScrollRequest(_ context.Context, _ string, _ []byte, _ bool, _ func(responseBytes []byte) error) error { + return nil +} + +// DoCountRequest - +func (ec *elasticClient) DoCountRequest(_ context.Context, _ string, _ []byte) (uint64, error) { + return 0, nil +} + +// UpdateByQuery - +func (ec *elasticClient) UpdateByQuery(_ context.Context, _ string, _ *bytes.Buffer) error { + return nil +} + +// PutMappings - +func (ec *elasticClient) PutMappings(_ string, _ *bytes.Buffer) error { + return nil +} + +// CheckAndCreateIndex - +func (ec *elasticClient) CheckAndCreateIndex(_ string) error { + return nil +} + +// CheckAndCreateAlias - +func (ec *elasticClient) CheckAndCreateAlias(_ string, _ string) error { + return nil +} + +// CheckAndCreateTemplate - +func (ec *elasticClient) CheckAndCreateTemplate(_ string, _ *bytes.Buffer) error { + return nil +} + +// CheckAndCreatePolicy - +func (ec *elasticClient) CheckAndCreatePolicy(_ string, _ *bytes.Buffer) error { + return nil +} + +// IsEnabled - +func (ec *elasticClient) IsEnabled() bool { + return false +} + +// IsInterfaceNil - returns true if there is no value under the interface +func (ec *elasticClient) IsInterfaceNil() bool { + return ec == nil +} diff --git a/client/disabled/elasticClient_test.go b/client/disabled/elasticClient_test.go new file mode 100644 index 00000000..c33e8716 --- /dev/null +++ b/client/disabled/elasticClient_test.go @@ -0,0 +1,31 @@ +package disabled + +import ( + "bytes" + "context" + "testing" + + "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/stretchr/testify/require" +) + +func TestDisabledElasticClient_MethodsShouldNotPanic(t *testing.T) { + t.Parallel() + + ec := NewDisabledElasticClient() + require.False(t, check.IfNil(ec)) + + require.NotPanics(t, func() { + _ = ec.DoBulkRequest(context.Background(), new(bytes.Buffer), "") + _ = ec.DoQueryRemove(context.Background(), "", new(bytes.Buffer)) + _ = ec.DoMultiGet(context.Background(), make([]string, 0), "", true, nil) + _ = ec.DoScrollRequest(context.Background(), "", []byte(""), true, nil) + _, _ = ec.DoCountRequest(context.Background(), "", []byte("")) + _ = ec.UpdateByQuery(context.Background(), "", new(bytes.Buffer)) + _ = ec.PutMappings("", new(bytes.Buffer)) + _ = ec.CheckAndCreateIndex("") + _ = ec.CheckAndCreateAlias("", "") + _ = ec.CheckAndCreateTemplate("", new(bytes.Buffer)) + _ = ec.CheckAndCreatePolicy("", new(bytes.Buffer)) + }) +} diff --git a/client/elasticClientCommon.go b/client/elasticClientCommon.go index 94b81f50..2d54d819 100644 --- a/client/elasticClientCommon.go +++ b/client/elasticClientCommon.go @@ -6,11 +6,14 @@ import ( "fmt" "io" "io/ioutil" + "math" "net/http" "net/url" "strings" + "time" "github.com/elastic/go-elasticsearch/v7/esapi" + "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" ) @@ -268,3 +271,11 @@ func parseResponse(res *esapi.Response, dest interface{}, errorHandler responseE return nil } + +// RetryBackOff returns elastic retry backoff duration +func RetryBackOff(attempt int) time.Duration { + d := time.Duration(math.Exp2(float64(attempt))) * time.Second + log.Debug("elastic: retry backoff", "attempt", attempt, "sleep duration", d) + + return d +} diff --git a/client/mainChainElasticClient.go b/client/mainChainElasticClient.go new file mode 100644 index 00000000..042504e9 --- /dev/null +++ b/client/mainChainElasticClient.go @@ -0,0 +1,35 @@ +package client + +import ( + "github.com/multiversx/mx-chain-core-go/core/check" + + "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" +) + +type mainChainElasticClient struct { + elasticproc.DatabaseClientHandler + indexingEnabled bool +} + +// NewMainChainElasticClient creates a new sovereign elastic client +func NewMainChainElasticClient(esClient elasticproc.DatabaseClientHandler, indexingEnabled bool) (*mainChainElasticClient, error) { + if check.IfNil(esClient) { + return nil, dataindexer.ErrNilDatabaseClient + } + + return &mainChainElasticClient{ + esClient, + indexingEnabled, + }, nil +} + +// IsEnabled returns true if main chain elastic client is enabled +func (mcec *mainChainElasticClient) IsEnabled() bool { + return mcec.indexingEnabled +} + +// IsInterfaceNil returns true if there is no value under the interface +func (mcec *mainChainElasticClient) IsInterfaceNil() bool { + return mcec == nil +} diff --git a/client/mainChainElasticClient_test.go b/client/mainChainElasticClient_test.go new file mode 100644 index 00000000..c674cb32 --- /dev/null +++ b/client/mainChainElasticClient_test.go @@ -0,0 +1,42 @@ +package client + +import ( + "fmt" + "testing" + + "github.com/elastic/go-elasticsearch/v7" + "github.com/stretchr/testify/require" + + "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" +) + +func TestNewMainChainElasticClient(t *testing.T) { + t.Run("nil elastic client, should error", func(t *testing.T) { + mainChainESClient, err := NewMainChainElasticClient(nil, true) + require.Error(t, err, dataindexer.ErrNilDatabaseClient) + require.True(t, mainChainESClient.IsInterfaceNil()) + }) + t.Run("valid elastic client, should work", func(t *testing.T) { + esClient, err := NewElasticClient(elasticsearch.Config{ + Addresses: []string{"http://localhost:9200"}, + }) + require.Nil(t, err) + require.NotNil(t, esClient) + + mainChainESClient, err := NewMainChainElasticClient(esClient, true) + require.NoError(t, err) + require.Equal(t, "*client.mainChainElasticClient", fmt.Sprintf("%T", mainChainESClient)) + }) +} + +func TestMainChainElasticClient_IsEnabled(t *testing.T) { + esClient, err := NewElasticClient(elasticsearch.Config{ + Addresses: []string{"http://localhost:9200"}, + }) + require.Nil(t, err) + require.NotNil(t, esClient) + + mainChainESClient, err := NewMainChainElasticClient(esClient, true) + require.NoError(t, err) + require.Equal(t, true, mainChainESClient.IsEnabled()) +} diff --git a/cmd/elasticindexer/config/config.toml b/cmd/elasticindexer/config/config.toml index b63329c7..5fdf7122 100644 --- a/cmd/elasticindexer/config/config.toml +++ b/cmd/elasticindexer/config/config.toml @@ -4,6 +4,7 @@ "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags", "logs", "delegators", "operations", "esdts", "values", "events" ] + esdt-prefix = "" [config.address-converter] length = 32 type = "bech32" diff --git a/cmd/elasticindexer/config/prefs.toml b/cmd/elasticindexer/config/prefs.toml index 2685e4b8..bc6a2743 100644 --- a/cmd/elasticindexer/config/prefs.toml +++ b/cmd/elasticindexer/config/prefs.toml @@ -23,3 +23,11 @@ username = "" password = "" bulk-request-max-size-in-bytes = 4194304 # 4MB + + # Configuration for main chain elastic cluster + # Used by the sovereign chain indexer to index incoming new tokens properties + [config.main-chain-elastic-cluster] + enabled = true + url = "http://localhost:9201" + username = "" + password = "" diff --git a/cmd/elasticindexer/flags.go b/cmd/elasticindexer/flags.go index d7e84b70..b09106c0 100644 --- a/cmd/elasticindexer/flags.go +++ b/cmd/elasticindexer/flags.go @@ -47,4 +47,9 @@ var ( Name: "disable-ansi-color", Usage: "Boolean option for disabling ANSI colors in the logging system.", } + // sovereign defines a flag that specifies if the es instance should run for a sovereign chain + sovereign = cli.BoolFlag{ + Name: "sovereign", + Usage: "If set to true, will use sovereign run type components", + } ) diff --git a/cmd/elasticindexer/main.go b/cmd/elasticindexer/main.go index 85246d13..e2256db9 100644 --- a/cmd/elasticindexer/main.go +++ b/cmd/elasticindexer/main.go @@ -11,13 +11,14 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/core/closing" "github.com/multiversx/mx-chain-core-go/data/outport" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-logger-go/file" + "github.com/urfave/cli" + "github.com/multiversx/mx-chain-es-indexer-go/config" "github.com/multiversx/mx-chain-es-indexer-go/factory" "github.com/multiversx/mx-chain-es-indexer-go/metrics" "github.com/multiversx/mx-chain-es-indexer-go/process/wsindexer" - logger "github.com/multiversx/mx-chain-logger-go" - "github.com/multiversx/mx-chain-logger-go/file" - "github.com/urfave/cli" ) var ( @@ -63,6 +64,7 @@ func main() { logLevel, logSaveFile, disableAnsiColor, + sovereign, } app.Authors = []cli.Author{ { @@ -86,6 +88,7 @@ func startIndexer(ctx *cli.Context) error { if err != nil { return fmt.Errorf("%w while loading the config file", err) } + cfg.Sovereign = ctx.GlobalBool(sovereign.Name) clusterCfg, err := loadClusterConfig(ctx.GlobalString(configurationPreferencesFile.Name)) if err != nil { diff --git a/config/config.go b/config/config.go index dc1ab2cb..203bd6e7 100644 --- a/config/config.go +++ b/config/config.go @@ -4,6 +4,7 @@ package config type Config struct { Config struct { AvailableIndices []string `toml:"available-indices"` + ESDTPrefix string `toml:"esdt-prefix"` AddressConverter struct { Length int `toml:"length"` Type string `toml:"type"` @@ -29,6 +30,7 @@ type Config struct { LogsPath string `toml:"logs-path"` } `toml:"logs"` } `toml:"config"` + Sovereign bool } // ClusterConfig will hold the config for the Elasticsearch cluster @@ -51,6 +53,12 @@ type ClusterConfig struct { Password string `toml:"password"` BulkRequestMaxSizeInBytes int `toml:"bulk-request-max-size-in-bytes"` } `toml:"elastic-cluster"` + MainChainCluster struct { + Enabled bool `toml:"enabled"` + URL string `toml:"url"` + UserName string `toml:"username"` + Password string `toml:"password"` + } `toml:"main-chain-elastic-cluster"` } `toml:"config"` } diff --git a/data/tokens.go b/data/tokens.go index 967e13ee..01f39dce 100644 --- a/data/tokens.go +++ b/data/tokens.go @@ -45,6 +45,18 @@ type SourceToken struct { CurrentOwner string `json:"currentOwner"` } +// ResponseTokenInfo is the structure for the tokens info response +type ResponseTokenInfo struct { + Docs []ResponseTokenInfoDB `json:"docs"` +} + +// ResponseTokenInfoDB is the structure for the token info response +type ResponseTokenInfoDB struct { + Found bool `json:"found"` + ID string `json:"_id"` + Source TokenInfo `json:"_source"` +} + // TokenInfo is a structure that is needed to store information about a token type TokenInfo struct { Name string `json:"name,omitempty"` diff --git a/docker-compose.yml b/docker-compose.yml index 970f0b40..27e689c1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,22 @@ services: ports: - "9200:9200" - "9300:9300" + main-chain-elasticsearch: + container_name: es-container2 + image: docker.elastic.co/elasticsearch/elasticsearch:7.16.1 + environment: + - "discovery.type=single-node" + - "xpack.security.enabled=false" + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + networks: + - es-net + ports: + - "9201:9200" + - "9301:9300" kibana: container_name: kb-container image: docker.elastic.co/kibana/kibana:7.16.1 diff --git a/factory/runType/errors.go b/factory/runType/errors.go new file mode 100644 index 00000000..657663a3 --- /dev/null +++ b/factory/runType/errors.go @@ -0,0 +1,7 @@ +package runType + +import ( + "errors" +) + +var errNilRunTypeComponents = errors.New("nil run type components") diff --git a/factory/runType/interface.go b/factory/runType/interface.go new file mode 100644 index 00000000..4b96663b --- /dev/null +++ b/factory/runType/interface.go @@ -0,0 +1,38 @@ +package runType + +import ( + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" +) + +// RunTypeComponentsCreator is the interface for creating run type components +type RunTypeComponentsCreator interface { + Create() (*runTypeComponents, error) + IsInterfaceNil() bool +} + +// ComponentHandler defines the actions common to all component handlers +type ComponentHandler interface { + Create() error + Close() error + CheckSubcomponents() error + String() string +} + +// RunTypeComponentsHandler defines the run type components handler actions +type RunTypeComponentsHandler interface { + ComponentHandler + RunTypeComponentsHolder +} + +// RunTypeComponentsHolder holds the run type components +type RunTypeComponentsHolder interface { + TxHashExtractorCreator() transactions.TxHashExtractor + RewardTxDataCreator() transactions.RewardTxDataHandler + IndexTokensHandlerCreator() elasticproc.IndexTokensHandler + Create() error + Close() error + CheckSubcomponents() error + String() string + IsInterfaceNil() bool +} diff --git a/factory/runType/runTypeComponents.go b/factory/runType/runTypeComponents.go new file mode 100644 index 00000000..4bccd66b --- /dev/null +++ b/factory/runType/runTypeComponents.go @@ -0,0 +1,22 @@ +package runType + +import ( + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" +) + +type runTypeComponents struct { + txHashExtractor transactions.TxHashExtractor + rewardTxData transactions.RewardTxDataHandler + indexTokensHandler elasticproc.IndexTokensHandler +} + +// Close does nothing +func (rtc *runTypeComponents) Close() error { + return nil +} + +// IsInterfaceNil returns true if there is no value under the interface +func (rtc *runTypeComponents) IsInterfaceNil() bool { + return rtc == nil +} diff --git a/factory/runType/runTypeComponentsFactory.go b/factory/runType/runTypeComponentsFactory.go new file mode 100644 index 00000000..8ac863d6 --- /dev/null +++ b/factory/runType/runTypeComponentsFactory.go @@ -0,0 +1,27 @@ +package runType + +import ( + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tokens" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" +) + +type runTypeComponentsFactory struct{} + +// NewRunTypeComponentsFactory will return a new instance of run type components factory +func NewRunTypeComponentsFactory() *runTypeComponentsFactory { + return &runTypeComponentsFactory{} +} + +// Create will create the run type components +func (rtcf *runTypeComponentsFactory) Create() (*runTypeComponents, error) { + return &runTypeComponents{ + txHashExtractor: transactions.NewTxHashExtractor(), + rewardTxData: transactions.NewRewardTxData(), + indexTokensHandler: tokens.NewDisabledIndexTokensHandler(), + }, nil +} + +// IsInterfaceNil returns true if there is no value under the interface +func (rtcf *runTypeComponentsFactory) IsInterfaceNil() bool { + return rtcf == nil +} diff --git a/factory/runType/runTypeComponentsHandler.go b/factory/runType/runTypeComponentsHandler.go new file mode 100644 index 00000000..113be479 --- /dev/null +++ b/factory/runType/runTypeComponentsHandler.go @@ -0,0 +1,133 @@ +package runType + +import ( + "sync" + + "github.com/multiversx/mx-chain-core-go/core/check" + + elasticIndexer "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" +) + +const runTypeComponentsName = "managedRunTypeComponents" + +var _ ComponentHandler = (*managedRunTypeComponents)(nil) +var _ RunTypeComponentsHandler = (*managedRunTypeComponents)(nil) +var _ RunTypeComponentsHolder = (*managedRunTypeComponents)(nil) + +type managedRunTypeComponents struct { + *runTypeComponents + factory RunTypeComponentsCreator + mutRunTypeCoreComponents sync.RWMutex +} + +// NewManagedRunTypeComponents returns a news instance of managed runType core components +func NewManagedRunTypeComponents(rtc RunTypeComponentsCreator) (*managedRunTypeComponents, error) { + if rtc == nil { + return nil, errNilRunTypeComponents + } + + return &managedRunTypeComponents{ + runTypeComponents: nil, + factory: rtc, + }, nil +} + +// Create will create the managed components +func (mrtc *managedRunTypeComponents) Create() error { + rtc, err := mrtc.factory.Create() + if err != nil { + return err + } + + mrtc.mutRunTypeCoreComponents.Lock() + mrtc.runTypeComponents = rtc + mrtc.mutRunTypeCoreComponents.Unlock() + + return nil +} + +// Close will close all underlying subcomponents +func (mrtc *managedRunTypeComponents) Close() error { + mrtc.mutRunTypeCoreComponents.Lock() + defer mrtc.mutRunTypeCoreComponents.Unlock() + + if check.IfNil(mrtc.runTypeComponents) { + return nil + } + + err := mrtc.runTypeComponents.Close() + if err != nil { + return err + } + mrtc.runTypeComponents = nil + + return nil +} + +// CheckSubcomponents verifies all subcomponents +func (mrtc *managedRunTypeComponents) CheckSubcomponents() error { + mrtc.mutRunTypeCoreComponents.RLock() + defer mrtc.mutRunTypeCoreComponents.RUnlock() + + if check.IfNil(mrtc.runTypeComponents) { + return errNilRunTypeComponents + } + if check.IfNil(mrtc.txHashExtractor) { + return transactions.ErrNilTxHashExtractor + } + if check.IfNil(mrtc.rewardTxData) { + return transactions.ErrNilRewardTxDataHandler + } + if check.IfNil(mrtc.indexTokensHandler) { + return elasticIndexer.ErrNilIndexTokensHandler + } + return nil +} + +// TxHashExtractorCreator returns tx hash extractor +func (mrtc *managedRunTypeComponents) TxHashExtractorCreator() transactions.TxHashExtractor { + mrtc.mutRunTypeCoreComponents.Lock() + defer mrtc.mutRunTypeCoreComponents.Unlock() + + if check.IfNil(mrtc.runTypeComponents) { + return nil + } + + return mrtc.runTypeComponents.txHashExtractor +} + +// RewardTxDataCreator return reward tx handler +func (mrtc *managedRunTypeComponents) RewardTxDataCreator() transactions.RewardTxDataHandler { + mrtc.mutRunTypeCoreComponents.Lock() + defer mrtc.mutRunTypeCoreComponents.Unlock() + + if check.IfNil(mrtc.runTypeComponents) { + return nil + } + + return mrtc.runTypeComponents.rewardTxData +} + +// IndexTokensHandlerCreator returns the index tokens handler +func (mrtc *managedRunTypeComponents) IndexTokensHandlerCreator() elasticproc.IndexTokensHandler { + mrtc.mutRunTypeCoreComponents.Lock() + defer mrtc.mutRunTypeCoreComponents.Unlock() + + if check.IfNil(mrtc.runTypeComponents) { + return nil + } + + return mrtc.runTypeComponents.indexTokensHandler +} + +// IsInterfaceNil returns true if the interface is nil +func (mrtc *managedRunTypeComponents) IsInterfaceNil() bool { + return mrtc == nil +} + +// String returns the name of the component +func (mrtc *managedRunTypeComponents) String() string { + return runTypeComponentsName +} diff --git a/factory/runType/runTypeComponentsHandler_test.go b/factory/runType/runTypeComponentsHandler_test.go new file mode 100644 index 00000000..e6756a6c --- /dev/null +++ b/factory/runType/runTypeComponentsHandler_test.go @@ -0,0 +1,80 @@ +package runType + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func createComponents() (RunTypeComponentsHandler, error) { + rtcf := NewRunTypeComponentsFactory() + return NewManagedRunTypeComponents(rtcf) +} + +func TestNewManagedRunTypeComponents(t *testing.T) { + t.Parallel() + + t.Run("should error", func(t *testing.T) { + managedRunTypeComponents, err := NewManagedRunTypeComponents(nil) + require.ErrorIs(t, err, errNilRunTypeComponents) + require.Nil(t, managedRunTypeComponents) + }) + t.Run("should work", func(t *testing.T) { + rtcf := NewRunTypeComponentsFactory() + managedRunTypeComponents, err := NewManagedRunTypeComponents(rtcf) + require.NoError(t, err) + require.False(t, managedRunTypeComponents.IsInterfaceNil()) + }) +} + +func TestManagedRunTypeComponents_Create(t *testing.T) { + t.Parallel() + + t.Run("should work with getters", func(t *testing.T) { + t.Parallel() + + managedRunTypeComponents, err := createComponents() + require.NoError(t, err) + + require.Nil(t, managedRunTypeComponents.TxHashExtractorCreator()) + require.Nil(t, managedRunTypeComponents.RewardTxDataCreator()) + + err = managedRunTypeComponents.Create() + require.NoError(t, err) + + require.NotNil(t, managedRunTypeComponents.TxHashExtractorCreator()) + require.NotNil(t, managedRunTypeComponents.RewardTxDataCreator()) + + require.Equal(t, runTypeComponentsName, managedRunTypeComponents.String()) + require.NoError(t, managedRunTypeComponents.Close()) + }) +} + +func TestManagedRunTypeComponents_Close(t *testing.T) { + t.Parallel() + + managedRunTypeComponents, _ := createComponents() + require.NoError(t, managedRunTypeComponents.Close()) + + err := managedRunTypeComponents.Create() + require.NoError(t, err) + + require.NoError(t, managedRunTypeComponents.Close()) +} + +func TestManagedRunTypeComponents_CheckSubcomponents(t *testing.T) { + t.Parallel() + + managedRunTypeComponents, _ := createComponents() + err := managedRunTypeComponents.CheckSubcomponents() + require.Equal(t, errNilRunTypeComponents, err) + + err = managedRunTypeComponents.Create() + require.NoError(t, err) + + //TODO check for nil each subcomponent - MX-15371 + err = managedRunTypeComponents.CheckSubcomponents() + require.NoError(t, err) + + require.NoError(t, managedRunTypeComponents.Close()) +} diff --git a/factory/runType/runTypeComponents_test.go b/factory/runType/runTypeComponents_test.go new file mode 100644 index 00000000..9f929ea2 --- /dev/null +++ b/factory/runType/runTypeComponents_test.go @@ -0,0 +1,40 @@ +package runType + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNewRunTypeComponentsFactory(t *testing.T) { + t.Parallel() + + t.Run("should work", func(t *testing.T) { + rtc := NewRunTypeComponentsFactory() + require.NotNil(t, rtc) + }) +} + +func TestRunTypeComponentsFactory_Create(t *testing.T) { + t.Parallel() + + rtcf := NewRunTypeComponentsFactory() + require.NotNil(t, rtcf) + + rtc, err := rtcf.Create() + require.NotNil(t, rtc) + require.NoError(t, err) +} + +func TestRunTypeComponentsFactory_Close(t *testing.T) { + t.Parallel() + + rtcf := NewRunTypeComponentsFactory() + require.NotNil(t, rtcf) + + rtc, err := rtcf.Create() + require.NotNil(t, rtc) + require.NoError(t, err) + + require.NoError(t, rtc.Close()) +} diff --git a/factory/runType/sovereignRunTypeComponentsFactory.go b/factory/runType/sovereignRunTypeComponentsFactory.go new file mode 100644 index 00000000..4bafdf1a --- /dev/null +++ b/factory/runType/sovereignRunTypeComponentsFactory.go @@ -0,0 +1,73 @@ +package runType + +import ( + "net/http" + + "github.com/elastic/go-elasticsearch/v7" + + "github.com/multiversx/mx-chain-es-indexer-go/client" + "github.com/multiversx/mx-chain-es-indexer-go/client/disabled" + "github.com/multiversx/mx-chain-es-indexer-go/client/logging" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tokens" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" +) + +type sovereignRunTypeComponentsFactory struct { + mainChainElastic factory.ElasticConfig + esdtPrefix string +} + +// NewSovereignRunTypeComponentsFactory will return a new instance of sovereign run type components factory +func NewSovereignRunTypeComponentsFactory(mainChainElastic factory.ElasticConfig, esdtPrefix string) *sovereignRunTypeComponentsFactory { + return &sovereignRunTypeComponentsFactory{ + mainChainElastic: mainChainElastic, + esdtPrefix: esdtPrefix, + } +} + +// Create will create the run type components +func (srtcf *sovereignRunTypeComponentsFactory) Create() (*runTypeComponents, error) { + mainChainElasticClient, err := createMainChainElasticClient(srtcf.mainChainElastic) + if err != nil { + return nil, err + } + + sovIndexTokensHandler, err := tokens.NewSovereignIndexTokensHandler(mainChainElasticClient, srtcf.esdtPrefix) + if err != nil { + return nil, err + } + + return &runTypeComponents{ + txHashExtractor: transactions.NewSovereignTxHashExtractor(), + rewardTxData: transactions.NewSovereignRewardTxData(), + indexTokensHandler: sovIndexTokensHandler, + }, nil +} + +func createMainChainElasticClient(mainChainElastic factory.ElasticConfig) (elasticproc.MainChainDatabaseClientHandler, error) { + if mainChainElastic.Enabled { + argsEsClient := elasticsearch.Config{ + Addresses: []string{mainChainElastic.Url}, + Username: mainChainElastic.UserName, + Password: mainChainElastic.Password, + Logger: &logging.CustomLogger{}, + RetryOnStatus: []int{http.StatusConflict}, + RetryBackoff: client.RetryBackOff, + } + esClient, err := client.NewElasticClient(argsEsClient) + if err != nil { + return nil, err + } + + return client.NewMainChainElasticClient(esClient, mainChainElastic.Enabled) + } else { + return disabled.NewDisabledElasticClient(), nil + } +} + +// IsInterfaceNil returns true if there is no value under the interface +func (srtcf *sovereignRunTypeComponentsFactory) IsInterfaceNil() bool { + return srtcf == nil +} diff --git a/factory/runType/sovereignRunTypeComponentsFactory_test.go b/factory/runType/sovereignRunTypeComponentsFactory_test.go new file mode 100644 index 00000000..4bb3581b --- /dev/null +++ b/factory/runType/sovereignRunTypeComponentsFactory_test.go @@ -0,0 +1,22 @@ +package runType + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" +) + +func TestSovereignRunTypeComponentsFactory_CreateAndClose(t *testing.T) { + t.Parallel() + + srtcf := NewSovereignRunTypeComponentsFactory(factory.ElasticConfig{}, "sov") + require.False(t, srtcf.IsInterfaceNil()) + + srtc, err := srtcf.Create() + require.NotNil(t, srtc) + require.NoError(t, err) + + require.NoError(t, srtc.Close()) +} diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index 90beb5a9..74bc00e4 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -7,11 +7,13 @@ import ( factoryHasher "github.com/multiversx/mx-chain-core-go/hashing/factory" "github.com/multiversx/mx-chain-core-go/marshal" factoryMarshaller "github.com/multiversx/mx-chain-core-go/marshal/factory" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-es-indexer-go/config" "github.com/multiversx/mx-chain-es-indexer-go/core" + esFactory "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" "github.com/multiversx/mx-chain-es-indexer-go/process/factory" "github.com/multiversx/mx-chain-es-indexer-go/process/wsindexer" - logger "github.com/multiversx/mx-chain-logger-go" ) var log = logger.GetOrCreate("elasticindexer") @@ -75,7 +77,16 @@ func createDataIndexer( return nil, err } + mainChainElastic := esFactory.ElasticConfig{ + Enabled: clusterCfg.Config.MainChainCluster.Enabled, + Url: clusterCfg.Config.MainChainCluster.URL, + UserName: clusterCfg.Config.MainChainCluster.UserName, + Password: clusterCfg.Config.MainChainCluster.Password, + } + return factory.NewIndexer(factory.ArgsIndexerFactory{ + Sovereign: cfg.Sovereign, + MainChainElastic: mainChainElastic, UseKibana: clusterCfg.Config.ElasticCluster.UseKibana, Denomination: cfg.Config.Economics.Denomination, BulkRequestMaxSize: clusterCfg.Config.ElasticCluster.BulkRequestMaxSizeInBytes, diff --git a/go.mod b/go.mod index 4747b259..77e71efc 100644 --- a/go.mod +++ b/go.mod @@ -8,9 +8,9 @@ require ( github.com/gin-gonic/gin v1.10.0 github.com/google/uuid v1.6.0 github.com/multiversx/mx-chain-communication-go v1.1.1 - github.com/multiversx/mx-chain-core-go v1.2.24 + github.com/multiversx/mx-chain-core-go v1.2.25-0.20250206111825-25fbb1b4851c github.com/multiversx/mx-chain-logger-go v1.0.15 - github.com/multiversx/mx-chain-vm-common-go v1.5.16 + github.com/multiversx/mx-chain-vm-common-go v1.5.17-0.20241119132002-2fa80c5ec516 github.com/prometheus/client_model v0.4.0 github.com/prometheus/common v0.37.0 github.com/stretchr/testify v1.10.0 diff --git a/go.sum b/go.sum index 16275e8e..0b165928 100644 --- a/go.sum +++ b/go.sum @@ -253,13 +253,13 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/mx-chain-communication-go v1.1.1 h1:y4DoQeQOJTaSUsRzczQFazf8JYQmInddypApqA3AkwM= github.com/multiversx/mx-chain-communication-go v1.1.1/go.mod h1:WK6bP4pGEHGDDna/AYRIMtl6G9OA0NByI1Lw8PmOnRM= -github.com/multiversx/mx-chain-core-go v1.2.24 h1:O0X7N9GfNVUCE9fukXA+dvfCRRjViYn88zOaE7feUog= -github.com/multiversx/mx-chain-core-go v1.2.24/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= +github.com/multiversx/mx-chain-core-go v1.2.25-0.20250206111825-25fbb1b4851c h1:Cz5b0xd9lbSWGIwmfuPuHqL0e5kTun/PW5NpkVRIAXQ= +github.com/multiversx/mx-chain-core-go v1.2.25-0.20250206111825-25fbb1b4851c/go.mod h1:P/YBoFnt25XUaCQ7Q/SD15vhnc9yV5JDhHxyFO9P8Z0= github.com/multiversx/mx-chain-crypto-go v1.2.12 h1:zWip7rpUS4CGthJxfKn5MZfMfYPjVjIiCID6uX5BSOk= github.com/multiversx/mx-chain-logger-go v1.0.15 h1:HlNdK8etyJyL9NQ+6mIXyKPEBo+wRqOwi3n+m2QIHXc= github.com/multiversx/mx-chain-logger-go v1.0.15/go.mod h1:t3PRKaWB1M+i6gUfD27KXgzLJJC+mAQiN+FLlL1yoGQ= -github.com/multiversx/mx-chain-vm-common-go v1.5.16 h1:g1SqYjxl7K66Y1O/q6tvDJ37fzpzlxCSfRzSm/woQQY= -github.com/multiversx/mx-chain-vm-common-go v1.5.16/go.mod h1:1rSkXreUZNXyPTTdhj47M+Fy62yjxbu3aAsXEtKN3UY= +github.com/multiversx/mx-chain-vm-common-go v1.5.17-0.20241119132002-2fa80c5ec516 h1:PToP1B7aFkXjQBOF5MDCCvsc+ffWVqqBHtmemfCwVA8= +github.com/multiversx/mx-chain-vm-common-go v1.5.17-0.20241119132002-2fa80c5ec516/go.mod h1:C7KVj6/+TAhxDjgY7oAMO5wSj7WbBYIJ5TCMzmxk2w0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= diff --git a/integrationtests/consts.go b/integrationtests/consts.go index 8593fcf6..32b4bd1b 100644 --- a/integrationtests/consts.go +++ b/integrationtests/consts.go @@ -6,5 +6,7 @@ const ( //nolint esURL = "http://localhost:9200" //nolint + esMainChainURL = "http://localhost:9201" + //nolint addressPrefix = "erd" ) diff --git a/integrationtests/incomingSCR_test.go b/integrationtests/incomingSCR_test.go new file mode 100644 index 00000000..b6931a0d --- /dev/null +++ b/integrationtests/incomingSCR_test.go @@ -0,0 +1,280 @@ +//go:build integrationtests + +package integrationtests + +import ( + "context" + "encoding/hex" + "encoding/json" + "math/big" + "testing" + + "github.com/multiversx/mx-chain-core-go/core" + dataBlock "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/multiversx/mx-chain-core-go/data/esdt" + "github.com/multiversx/mx-chain-core-go/data/outport" + "github.com/multiversx/mx-chain-core-go/data/smartContractResult" + "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/stretchr/testify/require" + + indexerData "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" +) + +type esToken struct { + Identifier string + Value *big.Int + NumDecimals int64 +} + +type esNft struct { + Collection string + Nonce uint64 + Data esdt.ESDigitalToken +} + +func createTokens() ([]esToken, []esNft) { + tokens := []esToken{} + token1 := esToken{ + Identifier: "TKN18-1a2b3c", + Value: big.NewInt(123), + NumDecimals: 18, + } + tokens = append(tokens, token1) + token2 := esToken{ + Identifier: "TKN12-1c2b3a", + Value: big.NewInt(333), + NumDecimals: 12, + } + tokens = append(tokens, token2) + + nfts := []esNft{} + nft := esNft{ + Collection: "NFT-abc123", + Nonce: 1, + Data: esdt.ESDigitalToken{ + Type: uint32(core.NonFungibleV2), + Value: big.NewInt(1), + Properties: []byte("3032"), + TokenMetaData: &esdt.MetaData{ + Nonce: 1, + Name: []byte("NFT"), + Creator: []byte("creator"), + Royalties: uint32(2500), + }, + }, + } + nfts = append(nfts, nft) + + return tokens, nfts +} + +func TestCrossChainTokensIndexingFromMainChain(t *testing.T) { + setLogLevelDebug() + + mainChainEsClient, err := createMainChainESClient(esMainChainURL, true) + require.Nil(t, err) + + tokens, nfts := createTokens() + createTokensInSourceEs(t, mainChainEsClient, tokens, nfts) + + esClient, err := createESClient(esURL) + require.Nil(t, err) + + esProc, err := CreateSovereignElasticProcessor(esClient, mainChainEsClient) + require.Nil(t, err) + + allTokens := getAllTokensIDs(tokens, nfts) + allTokens = append(allTokens, getAllNftIDs(nfts)...) + genericResponse := &GenericResponse{} + err = esClient.DoMultiGet(context.Background(), allTokens, indexerData.TokensIndex, true, genericResponse) + require.Nil(t, err) + for _, token := range genericResponse.Docs { + require.False(t, token.Found) + } + + scrHash := []byte("scrHash") + header := &dataBlock.Header{ + Round: 10, + TimeStamp: 2500, + } + body := &dataBlock.Body{ + MiniBlocks: dataBlock.MiniBlockSlice{ + { + Type: dataBlock.SmartContractResultBlock, + SenderShardID: core.MainChainShardId, + ReceiverShardID: core.SovereignChainShardId, + TxHashes: [][]byte{scrHash}, + }, + }, + } + + pool := &outport.TransactionPool{ + SmartContractResults: map[string]*outport.SCRInfo{ + hex.EncodeToString(scrHash): {SmartContractResult: &smartContractResult.SmartContractResult{ + Nonce: 11, + Value: big.NewInt(0), + GasLimit: 0, + SndAddr: decodeAddress("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u"), + RcvAddr: decodeAddress("erd1kzrfl2tztgzjpeedwec37c8npcr0a2ulzh9lhmj7xufyg23zcxuqxcqz0s"), + Data: createMultiEsdtTransferData(tokens, nfts), + OriginalTxHash: nil, + }, FeeInfo: &outport.FeeInfo{}}, + }, + } + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, 1)) + require.Nil(t, err) + + genericResponse = &GenericResponse{} + err = esClient.DoMultiGet(context.Background(), []string{hex.EncodeToString(scrHash)}, indexerData.ScResultsIndex, true, genericResponse) + require.Nil(t, err) + require.JSONEq(t, + readExpectedResult("./testdata/incomingSCR/incoming-scr.json"), + string(genericResponse.Docs[0].Source), + ) + + genericResponse = &GenericResponse{} + err = esClient.DoMultiGet(context.Background(), allTokens, indexerData.TokensIndex, true, genericResponse) + require.Nil(t, err) + for _, token := range genericResponse.Docs { + require.True(t, token.Found) + } +} + +func createTokensInSourceEs(t *testing.T, esClient elasticproc.DatabaseClientHandler, tokens []esToken, nfts []esNft) { + esProc, err := CreateElasticProcessor(esClient) + require.Nil(t, err) + + body := &dataBlock.Body{} + header := &dataBlock.Header{ + Round: 50, + TimeStamp: 5040, + ShardID: core.MetachainShardId, + } + + address1 := "erd1k04pxr6c0gvlcx4rd5fje0a4uy33axqxwz0fpcrgtfdy3nrqauqqgvxprv" + + // create issue token and nft collection events + events := make([]*transaction.Event, 0) + for _, token := range tokens { + events = append(events, &transaction.Event{ + Address: decodeAddress(address1), + Identifier: []byte("issue"), + Topics: [][]byte{[]byte(token.Identifier), []byte("TKN"), []byte("TKN"), []byte(core.FungibleESDT), big.NewInt(token.NumDecimals).Bytes()}, + }) + } + for _, nft := range nfts { + events = append(events, &transaction.Event{ + Address: decodeAddress(address1), + Identifier: []byte("issueNonFungible"), + Topics: [][]byte{[]byte(nft.Collection), []byte("NFT"), []byte("NFT"), []byte(core.ESDTType(nft.Data.Type).String())}, + }) + } + + pool := &outport.TransactionPool{ + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("txHash1")), + Log: &transaction.Log{ + Address: decodeAddress(address1), + Events: events, + }, + }, + }, + } + + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) + require.Nil(t, err) + + genericResponse := &GenericResponse{} + allTokens := getAllTokensIDs(tokens, nfts) + err = esClient.DoMultiGet(context.Background(), allTokens, indexerData.TokensIndex, true, genericResponse) + require.Nil(t, err) + for _, token := range genericResponse.Docs { + require.True(t, token.Found) + } + + // create nft event + events = make([]*transaction.Event, 0) + for _, nft := range nfts { + nftDataBytes, _ := json.Marshal(nft.Data) + + events = append(events, &transaction.Event{ + Address: decodeAddress(address1), + Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), + Topics: [][]byte{[]byte(nft.Collection), big.NewInt(0).SetUint64(nft.Nonce).Bytes(), nft.Data.Value.Bytes(), []byte(nftDataBytes)}, + }) + } + + header = &dataBlock.Header{ + Round: 51, + TimeStamp: 5600, + ShardID: 0, + } + + pool = &outport.TransactionPool{ + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("txHash2")), + Log: &transaction.Log{ + Address: decodeAddress(address1), + Events: events, + }, + }, + }, + } + + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) + require.Nil(t, err) + + allNfts := getAllNftIDs(nfts) + err = esClient.DoMultiGet(context.Background(), allNfts, indexerData.TokensIndex, true, genericResponse) + require.Nil(t, err) + for _, token := range genericResponse.Docs { + require.True(t, token.Found) + } +} + +func getAllTokensIDs(tokens []esToken, nfts []esNft) []string { + allTokens := make([]string, 0) + for _, token := range tokens { + allTokens = append(allTokens, token.Identifier) + } + for _, nft := range nfts { + allTokens = append(allTokens, nft.Collection) + } + return allTokens +} + +func getAllNftIDs(nfts []esNft) []string { + allNfts := make([]string, 0) + for _, nft := range nfts { + nonceBytes := big.NewInt(0).SetUint64(nft.Nonce).Bytes() + nonceHex := hex.EncodeToString(nonceBytes) + nftIdentifier := nft.Collection + "-" + nonceHex + + allNfts = append(allNfts, nftIdentifier) + + } + return allNfts +} + +func createMultiEsdtTransferData(tokens []esToken, nfts []esNft) []byte { + data := []byte(core.BuiltInFunctionMultiESDTNFTTransfer + + "@" + hex.EncodeToString(big.NewInt(int64(len(tokens)+len(nfts))).Bytes())) + for _, token := range tokens { + data = append(data, []byte( + "@"+hex.EncodeToString([]byte(token.Identifier))+ + "@"+ + "@"+hex.EncodeToString(token.Value.Bytes()))...) + } + for _, nft := range nfts { + nftDataBytes, _ := json.Marshal(nft.Data) + data = append(data, []byte( + "@"+hex.EncodeToString([]byte(nft.Collection))+ + "@"+hex.EncodeToString(big.NewInt(0).SetUint64(nft.Nonce).Bytes())+ + "@"+hex.EncodeToString(nftDataBytes))...) + } + + return data +} diff --git a/integrationtests/testdata/incomingSCR/incoming-scr.json b/integrationtests/testdata/incomingSCR/incoming-scr.json new file mode 100644 index 00000000..7ce746df --- /dev/null +++ b/integrationtests/testdata/incomingSCR/incoming-scr.json @@ -0,0 +1,44 @@ +{ + "miniBlockHash": "71e255368d7a6686a57a1acb8845953fc54e5a1cfde395acd09df58cb61d5abb", + "nonce": 11, + "gasLimit": 0, + "gasPrice": 0, + "value": "0", + "valueNum": 0, + "sender": "erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u", + "receiver": "erd1kzrfl2tztgzjpeedwec37c8npcr0a2ulzh9lhmj7xufyg23zcxuqxcqz0s", + "senderShard": 4294967293, + "receiverShard": 0, + "data": "TXVsdGlFU0RUTkZUVHJhbnNmZXJAMDNANTQ0YjRlMzEzODJkMzE2MTMyNjIzMzYzQEA3YkA1NDRiNGUzMTMyMmQzMTYzMzI2MjMzNjFAQDAxNGRANGU0NjU0MmQ2MTYyNjMzMTMyMzNAMDFAN2IyMjU0Nzk3MDY1MjIzYTMyMmMyMjU2NjE2Yzc1NjUyMjNhMzEyYzIyNTA3MjZmNzA2NTcyNzQ2OTY1NzMyMjNhMjI0ZDdhNDE3YTRkNjczZDNkMjIyYzIyNGQ2NTc0NjE0NDYxNzQ2MTIyM2E3YjIyNGU2ZjZlNjM2NTIyM2EzMTJjMjI0ZTYxNmQ2NTIyM2EyMjU0NmI1YTU1MjIyYzIyNDM3MjY1NjE3NDZmNzIyMjNhMjI1OTMzNGE2YzU5NTg1Mjc2NjM2NzNkM2QyMjJjMjI1MjZmNzk2MTZjNzQ2OTY1NzMyMjNhMzIzNTMwMzAyYzIyNDg2MTczNjgyMjNhNmU3NTZjNmMyYzIyNTU1MjQ5NzMyMjNhNmU3NTZjNmMyYzIyNDE3NDc0NzI2OTYyNzU3NDY1NzMyMjNhNmU3NTZjNmM3ZDJjMjI1MjY1NzM2NTcyNzY2NTY0MjIzYTZlNzU2YzZjN2Q=", + "prevTxHash": "", + "originalTxHash": "", + "callType": "0", + "timestamp": 2500, + "tokens": [ + "TKN18-1a2b3c", + "TKN12-1c2b3a", + "NFT-abc123-01" + ], + "epoch": 0, + "esdtValues": [ + "123", + "333", + "1" + ], + "esdtValuesNum": [ + 1.23e-16, + 3.33e-16, + 1e-18 + ], + "receivers": [ + "erd1kzrfl2tztgzjpeedwec37c8npcr0a2ulzh9lhmj7xufyg23zcxuqxcqz0s", + "erd1kzrfl2tztgzjpeedwec37c8npcr0a2ulzh9lhmj7xufyg23zcxuqxcqz0s", + "erd1kzrfl2tztgzjpeedwec37c8npcr0a2ulzh9lhmj7xufyg23zcxuqxcqz0s" + ], + "receiversShardIDs": [ + 0, + 0, + 0 + ], + "operation": "MultiESDTNFTTransfer" +} diff --git a/integrationtests/utils.go b/integrationtests/utils.go index 39baab5e..c88590ac 100644 --- a/integrationtests/utils.go +++ b/integrationtests/utils.go @@ -11,19 +11,23 @@ import ( "github.com/elastic/go-elasticsearch/v7" "github.com/multiversx/mx-chain-core-go/core/pubkeyConverter" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-es-indexer-go/client" "github.com/multiversx/mx-chain-es-indexer-go/client/logging" "github.com/multiversx/mx-chain-es-indexer-go/mock" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" - logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tokens" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" ) var ( // nolint log = logger.GetOrCreate("integration-tests") pubKeyConverter, _ = pubkeyConverter.NewBech32PubkeyConverter(32, addressPrefix) + sovEsdtPrefix = "sov" ) // nolint @@ -39,6 +43,12 @@ func createESClient(url string) (elasticproc.DatabaseClientHandler, error) { }) } +// nolint +func createMainChainESClient(url string, enabled bool) (elasticproc.MainChainDatabaseClientHandler, error) { + esClient, _ := createESClient(url) + return client.NewMainChainElasticClient(esClient, enabled) +} + // nolint func decodeAddress(address string) []byte { decoded, err := pubKeyConverter.Decode(address) @@ -60,7 +70,35 @@ func CreateElasticProcessor( EnabledIndexes: []string{dataindexer.TransactionsIndex, dataindexer.LogsIndex, dataindexer.AccountsESDTIndex, dataindexer.ScResultsIndex, dataindexer.ReceiptsIndex, dataindexer.BlockIndex, dataindexer.AccountsIndex, dataindexer.TokensIndex, dataindexer.TagsIndex, dataindexer.EventsIndex, dataindexer.OperationsIndex, dataindexer.DelegatorsIndex, dataindexer.ESDTsIndex, dataindexer.SCDeploysIndex, dataindexer.MiniblocksIndex, dataindexer.ValuesIndex}, - Denomination: 18, + Denomination: 18, + TxHashExtractor: transactions.NewTxHashExtractor(), + RewardTxData: transactions.NewRewardTxData(), + IndexTokensHandler: tokens.NewDisabledIndexTokensHandler(), + } + + return factory.CreateElasticProcessor(args) +} + +// CreateSovereignElasticProcessor - +func CreateSovereignElasticProcessor( + esClient elasticproc.DatabaseClientHandler, + mainEsClient elasticproc.MainChainDatabaseClientHandler, +) (dataindexer.ElasticProcessor, error) { + sovIndexTokens, _ := tokens.NewSovereignIndexTokensHandler(mainEsClient, sovEsdtPrefix) + + args := factory.ArgElasticProcessorFactory{ + Marshalizer: &mock.MarshalizerMock{}, + Hasher: &mock.HasherMock{}, + AddressPubkeyConverter: pubKeyConverter, + ValidatorPubkeyConverter: mock.NewPubkeyConverterMock(32), + DBClient: esClient, + EnabledIndexes: []string{dataindexer.TransactionsIndex, dataindexer.LogsIndex, dataindexer.AccountsESDTIndex, dataindexer.ScResultsIndex, + dataindexer.ReceiptsIndex, dataindexer.BlockIndex, dataindexer.AccountsIndex, dataindexer.TokensIndex, dataindexer.TagsIndex, dataindexer.EventsIndex, + dataindexer.OperationsIndex, dataindexer.DelegatorsIndex, dataindexer.ESDTsIndex, dataindexer.SCDeploysIndex, dataindexer.MiniblocksIndex, dataindexer.ValuesIndex}, + Denomination: 18, + TxHashExtractor: transactions.NewSovereignTxHashExtractor(), + RewardTxData: transactions.NewSovereignRewardTxData(), + IndexTokensHandler: sovIndexTokens, } return factory.CreateElasticProcessor(args) diff --git a/integrationtests/valuesIndex_test.go b/integrationtests/valuesIndex_test.go index f76835f7..0fbd7ae3 100644 --- a/integrationtests/valuesIndex_test.go +++ b/integrationtests/valuesIndex_test.go @@ -7,10 +7,12 @@ import ( "fmt" "testing" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-es-indexer-go/mock" indexerData "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" - "github.com/stretchr/testify/require" ) func TestCheckVersionIsIndexer(t *testing.T) { @@ -27,6 +29,9 @@ func TestCheckVersionIsIndexer(t *testing.T) { Denomination: 18, Version: version, EnabledIndexes: []string{indexerData.ValuesIndex}, + TxHashExtractor: &mock.TxHashExtractorMock{}, + RewardTxData: &mock.RewardTxDataMock{}, + IndexTokensHandler: &elasticproc.IndexTokenHandlerMock{}, } _, err = factory.CreateElasticProcessor(args) diff --git a/mock/databaseWriterStub.go b/mock/databaseWriterStub.go index 1df608a8..5a0153ce 100644 --- a/mock/databaseWriterStub.go +++ b/mock/databaseWriterStub.go @@ -86,6 +86,11 @@ func (dwm *DatabaseWriterStub) CheckAndCreatePolicy(_ string, _ *bytes.Buffer) e return nil } +// IsEnabled - +func (dwm *DatabaseWriterStub) IsEnabled() bool { + return false +} + // IsInterfaceNil returns true if there is no value under the interface func (dwm *DatabaseWriterStub) IsInterfaceNil() bool { return dwm == nil diff --git a/mock/rewardTxDataMock.go b/mock/rewardTxDataMock.go new file mode 100644 index 00000000..9eee7370 --- /dev/null +++ b/mock/rewardTxDataMock.go @@ -0,0 +1,20 @@ +package mock + +// RewardTxDataMock - +type RewardTxDataMock struct { + GetSenderCalled func() string +} + +// GetSender - +func (rtd *RewardTxDataMock) GetSender() string { + if rtd.GetSenderCalled != nil { + return rtd.GetSenderCalled() + } + + return "" +} + +// IsInterfaceNil returns true if there is no value under the interface +func (rtd *RewardTxDataMock) IsInterfaceNil() bool { + return rtd == nil +} diff --git a/mock/txHashExtractorMock.go b/mock/txHashExtractorMock.go new file mode 100644 index 00000000..cb3a46f0 --- /dev/null +++ b/mock/txHashExtractorMock.go @@ -0,0 +1,24 @@ +package mock + +import ( + coreData "github.com/multiversx/mx-chain-core-go/data" +) + +// TxHashExtractorMock - +type TxHashExtractorMock struct { + ExtractExecutedTxHashesCalled func(mbIndex int, mbTxHashes [][]byte, header coreData.HeaderHandler) [][]byte +} + +// ExtractExecutedTxHashes - +func (the *TxHashExtractorMock) ExtractExecutedTxHashes(mbIndex int, mbTxHashes [][]byte, header coreData.HeaderHandler) [][]byte { + if the.ExtractExecutedTxHashesCalled != nil { + return the.ExtractExecutedTxHashesCalled(mbIndex, mbTxHashes, header) + } + + return make([][]byte, 0) +} + +// IsInterfaceNil returns true if there is no value under the interface +func (the *TxHashExtractorMock) IsInterfaceNil() bool { + return the == nil +} diff --git a/process/dataindexer/errors.go b/process/dataindexer/errors.go index 9b4fbd0f..9495e93f 100644 --- a/process/dataindexer/errors.go +++ b/process/dataindexer/errors.go @@ -88,3 +88,6 @@ var ErrNilOperationsHandler = errors.New("nil operations handler") // ErrNilBlockContainerHandler signals that a nil block container handler has been provided var ErrNilBlockContainerHandler = errors.New("nil bock container handler") + +// ErrNilIndexTokensHandler signals that a nil index tokens handler has been provided +var ErrNilIndexTokensHandler = errors.New("nil index tokens handler") diff --git a/process/elasticproc/check.go b/process/elasticproc/check.go index 9284376e..d15a19fc 100644 --- a/process/elasticproc/check.go +++ b/process/elasticproc/check.go @@ -2,6 +2,7 @@ package elasticproc import ( "github.com/multiversx/mx-chain-core-go/core/check" + elasticIndexer "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" ) @@ -39,6 +40,9 @@ func checkArguments(arguments *ArgElasticProcessor) error { if check.IfNilReflect(arguments.OperationsProc) { return elasticIndexer.ErrNilOperationsHandler } + if check.IfNilReflect(arguments.IndexTokensHandler) { + return elasticIndexer.ErrNilIndexTokensHandler + } return nil } diff --git a/process/elasticproc/elasticProcessor.go b/process/elasticproc/elasticProcessor.go index 89634a95..e0568504 100644 --- a/process/elasticproc/elasticProcessor.go +++ b/process/elasticproc/elasticProcessor.go @@ -14,6 +14,8 @@ import ( "github.com/multiversx/mx-chain-core-go/data/alteredAccount" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-es-indexer-go/core/request" "github.com/multiversx/mx-chain-es-indexer-go/data" elasticIndexer "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" @@ -21,7 +23,6 @@ import ( "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tags" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tokeninfo" "github.com/multiversx/mx-chain-es-indexer-go/templates" - logger "github.com/multiversx/mx-chain-logger-go" ) var ( @@ -57,6 +58,7 @@ type ArgElasticProcessor struct { LogsAndEventsProc DBLogsAndEventsHandler OperationsProc OperationsHandler Version string + IndexTokensHandler IndexTokensHandler } type elasticProcessor struct { @@ -73,6 +75,7 @@ type elasticProcessor struct { validatorsProc DBValidatorsHandler logsAndEventsProc DBLogsAndEventsHandler operationsProc OperationsHandler + indexTokensHandler IndexTokensHandler } // NewElasticProcessor handles Elasticsearch operations such as initialization, adding, modifying or removing data @@ -94,6 +97,7 @@ func NewElasticProcessor(arguments *ArgElasticProcessor) (*elasticProcessor, err logsAndEventsProc: arguments.LogsAndEventsProc, operationsProc: arguments.OperationsProc, bulkRequestMaxSize: arguments.BulkRequestMaxSize, + indexTokensHandler: arguments.IndexTokensHandler, } err = ei.init(arguments.UseKibana, arguments.IndexTemplates, arguments.IndexPolicies, arguments.ExtraMappings) @@ -503,6 +507,11 @@ func (ei *elasticProcessor) SaveTransactions(obh *outport.OutportBlockWithHeader return err } + err = ei.indexTokensHandler.IndexCrossChainTokens(ei.elasticClient, preparedResults.ScResults, buffers) + if err != nil { + return err + } + return ei.doBulkRequests("", buffers.Buffers(), obh.ShardID) } diff --git a/process/elasticproc/elasticProcessor_test.go b/process/elasticproc/elasticProcessor_test.go index 9f3311af..424970cb 100644 --- a/process/elasticproc/elasticProcessor_test.go +++ b/process/elasticproc/elasticProcessor_test.go @@ -12,6 +12,8 @@ import ( dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/mock" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" @@ -25,20 +27,20 @@ import ( "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tags" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/validators" - "github.com/stretchr/testify/require" ) func newElasticsearchProcessor(elasticsearchWriter DatabaseClientHandler, arguments *ArgElasticProcessor) *elasticProcessor { return &elasticProcessor{ - elasticClient: elasticsearchWriter, - enabledIndexes: arguments.EnabledIndexes, - blockProc: arguments.BlockProc, - transactionsProc: arguments.TransactionsProc, - miniblocksProc: arguments.MiniblocksProc, - accountsProc: arguments.AccountsProc, - validatorsProc: arguments.ValidatorsProc, - statisticsProc: arguments.StatisticsProc, - logsAndEventsProc: arguments.LogsAndEventsProc, + elasticClient: elasticsearchWriter, + enabledIndexes: arguments.EnabledIndexes, + blockProc: arguments.BlockProc, + transactionsProc: arguments.TransactionsProc, + miniblocksProc: arguments.MiniblocksProc, + accountsProc: arguments.AccountsProc, + validatorsProc: arguments.ValidatorsProc, + statisticsProc: arguments.StatisticsProc, + logsAndEventsProc: arguments.LogsAndEventsProc, + indexTokensHandler: arguments.IndexTokensHandler, } } @@ -79,14 +81,15 @@ func createMockElasticProcessorArgs() *ArgElasticProcessor { EnabledIndexes: map[string]struct{}{ dataindexer.BlockIndex: {}, dataindexer.TransactionsIndex: {}, dataindexer.MiniblocksIndex: {}, dataindexer.ValidatorsIndex: {}, dataindexer.RoundsIndex: {}, dataindexer.AccountsIndex: {}, dataindexer.RatingIndex: {}, dataindexer.AccountsHistoryIndex: {}, }, - ValidatorsProc: vp, - StatisticsProc: statistics.NewStatisticsProcessor(), - TransactionsProc: &mock.DBTransactionProcessorStub{}, - MiniblocksProc: mp, - AccountsProc: acp, - BlockProc: bp, - LogsAndEventsProc: lp, - OperationsProc: op, + ValidatorsProc: vp, + StatisticsProc: statistics.NewStatisticsProcessor(), + TransactionsProc: &mock.DBTransactionProcessorStub{}, + MiniblocksProc: mp, + AccountsProc: acp, + BlockProc: bp, + LogsAndEventsProc: lp, + OperationsProc: op, + IndexTokensHandler: &IndexTokenHandlerMock{}, } } @@ -354,6 +357,8 @@ func TestElasticseachSaveTransactions(t *testing.T) { Hasher: &mock.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, BalanceConverter: bc, + TxHashExtractor: transactions.NewTxHashExtractor(), + RewardTxData: &mock.RewardTxDataMock{}, } txDbProc, _ := transactions.NewTransactionsProcessor(args) arguments.TransactionsProc = txDbProc diff --git a/process/elasticproc/factory/elasticProcessorFactory.go b/process/elasticproc/factory/elasticProcessorFactory.go index ad865953..2471885f 100644 --- a/process/elasticproc/factory/elasticProcessorFactory.go +++ b/process/elasticproc/factory/elasticProcessorFactory.go @@ -4,6 +4,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/accounts" @@ -18,6 +19,14 @@ import ( "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/validators" ) +// ElasticConfig holds the elastic search settings +type ElasticConfig struct { + Enabled bool + Url string + UserName string + Password string +} + // ArgElasticProcessorFactory is struct that is used to store all components that are needed to create an elastic processor factory type ArgElasticProcessorFactory struct { Marshalizer marshal.Marshalizer @@ -31,6 +40,9 @@ type ArgElasticProcessorFactory struct { BulkRequestMaxSize int UseKibana bool ImportDB bool + TxHashExtractor transactions.TxHashExtractor + RewardTxData transactions.RewardTxDataHandler + IndexTokensHandler elasticproc.IndexTokensHandler } // CreateElasticProcessor will create a new instance of ElasticProcessor @@ -87,6 +99,8 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.E Hasher: arguments.Hasher, Marshalizer: arguments.Marshalizer, BalanceConverter: balanceConverter, + TxHashExtractor: arguments.TxHashExtractor, + RewardTxData: arguments.RewardTxData, } txsProc, err := transactions.NewTransactionsProcessor(argsTxsProc) if err != nil { @@ -127,6 +141,7 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.E OperationsProc: operationsProc, ImportDB: arguments.ImportDB, Version: arguments.Version, + IndexTokensHandler: arguments.IndexTokensHandler, } return elasticproc.NewElasticProcessor(args) diff --git a/process/elasticproc/factory/elasticProcessorFactory_test.go b/process/elasticproc/factory/elasticProcessorFactory_test.go index 81e5d174..f7271737 100644 --- a/process/elasticproc/factory/elasticProcessorFactory_test.go +++ b/process/elasticproc/factory/elasticProcessorFactory_test.go @@ -3,8 +3,10 @@ package factory import ( "testing" - "github.com/multiversx/mx-chain-es-indexer-go/mock" "github.com/stretchr/testify/require" + + "github.com/multiversx/mx-chain-es-indexer-go/mock" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" ) func TestCreateElasticProcessor(t *testing.T) { @@ -18,6 +20,9 @@ func TestCreateElasticProcessor(t *testing.T) { EnabledIndexes: []string{"blocks"}, Denomination: 1, UseKibana: false, + TxHashExtractor: &mock.TxHashExtractorMock{}, + RewardTxData: &mock.RewardTxDataMock{}, + IndexTokensHandler: &elasticproc.IndexTokenHandlerMock{}, } ep, err := CreateElasticProcessor(args) diff --git a/process/elasticproc/indexTokenHandlerMock.go b/process/elasticproc/indexTokenHandlerMock.go new file mode 100644 index 00000000..e60f36fb --- /dev/null +++ b/process/elasticproc/indexTokenHandlerMock.go @@ -0,0 +1,23 @@ +package elasticproc + +import ( + "github.com/multiversx/mx-chain-es-indexer-go/data" +) + +// IndexTokenHandlerMock - +type IndexTokenHandlerMock struct { + IndexCrossChainTokensCalled func(elasticClient DatabaseClientHandler, scrs []*data.ScResult, buffSlice *data.BufferSlice) error +} + +// IndexCrossChainTokens - +func (ithh *IndexTokenHandlerMock) IndexCrossChainTokens(elasticClient DatabaseClientHandler, scrs []*data.ScResult, buffSlice *data.BufferSlice) error { + if ithh.IndexCrossChainTokensCalled != nil { + return ithh.IndexCrossChainTokensCalled(elasticClient, scrs, buffSlice) + } + return nil +} + +// IsInterfaceNil returns true if there is no value under the interface +func (ithh *IndexTokenHandlerMock) IsInterfaceNil() bool { + return ithh == nil +} diff --git a/process/elasticproc/interface.go b/process/elasticproc/interface.go index fe4b565f..f80e3809 100644 --- a/process/elasticproc/interface.go +++ b/process/elasticproc/interface.go @@ -8,10 +8,18 @@ import ( "github.com/multiversx/mx-chain-core-go/data/alteredAccount" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" + "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tokeninfo" ) +// MainChainDatabaseClientHandler defines the actions that sovereign database client handler should do +type MainChainDatabaseClientHandler interface { + DatabaseClientHandler + IsEnabled() bool + IsInterfaceNil() bool +} + // DatabaseClientHandler defines the actions that a component that handles requests should do type DatabaseClientHandler interface { DoBulkRequest(ctx context.Context, buff *bytes.Buffer, index string) error @@ -120,3 +128,9 @@ type OperationsHandler interface { ProcessTransactionsAndSCRs(txs []*data.Transaction, scrs []*data.ScResult, isImportDB bool, shardID uint32) ([]*data.Transaction, []*data.ScResult) SerializeSCRs(scrs []*data.ScResult, buffSlice *data.BufferSlice, index string, shardID uint32) error } + +// IndexTokensHandler defines what index tokens handler should be able to do +type IndexTokensHandler interface { + IndexCrossChainTokens(handler DatabaseClientHandler, scrs []*data.ScResult, buffSlice *data.BufferSlice) error + IsInterfaceNil() bool +} diff --git a/process/elasticproc/logsevents/delegatorsProcessor.go b/process/elasticproc/logsevents/delegatorsProcessor.go index 1cf6c6e7..f85264d0 100644 --- a/process/elasticproc/logsevents/delegatorsProcessor.go +++ b/process/elasticproc/logsevents/delegatorsProcessor.go @@ -47,10 +47,6 @@ func newDelegatorsProcessor( } func (dp *delegatorsProc) processEvent(args *argsProcessEvent) argOutputProcessEvent { - if args.selfShardID != core.MetachainShardId { - return argOutputProcessEvent{} - } - eventIdentifierStr := string(args.event.GetIdentifier()) _, ok := dp.delegatorsOperations[eventIdentifierStr] if !ok { diff --git a/process/elasticproc/logsevents/esdtIssueProcessor.go b/process/elasticproc/logsevents/esdtIssueProcessor.go index 6f5c9fa3..04bcfc23 100644 --- a/process/elasticproc/logsevents/esdtIssueProcessor.go +++ b/process/elasticproc/logsevents/esdtIssueProcessor.go @@ -47,10 +47,6 @@ func newESDTIssueProcessor(pubkeyConverter core.PubkeyConverter) *esdtIssueProce } func (eip *esdtIssueProcessor) processEvent(args *argsProcessEvent) argOutputProcessEvent { - if args.selfShardID != core.MetachainShardId { - return argOutputProcessEvent{} - } - identifierStr := string(args.event.GetIdentifier()) _, ok := eip.issueOperationsIdentifiers[identifierStr] if !ok { diff --git a/process/elasticproc/logsevents/esdtIssueProcessor_test.go b/process/elasticproc/logsevents/esdtIssueProcessor_test.go index f62d2ab3..026561d3 100644 --- a/process/elasticproc/logsevents/esdtIssueProcessor_test.go +++ b/process/elasticproc/logsevents/esdtIssueProcessor_test.go @@ -83,23 +83,3 @@ func TestIssueESDTProcessor_TransferOwnership(t *testing.T) { Properties: &data.TokenProperties{}, }, res.tokenInfo) } - -func TestIssueESDTProcessor_EventWithShardID0ShouldBeIgnored(t *testing.T) { - t.Parallel() - - esdtIssueProc := newESDTIssueProcessor(&mock.PubkeyConverterMock{}) - - event := &transaction.Event{ - Address: []byte("addr"), - Identifier: []byte(transferOwnershipFunc), - Topics: [][]byte{[]byte("MYTOKEN-abcd"), []byte("my-token"), []byte("MYTOKEN"), []byte(core.NonFungibleESDT), []byte("newOwner")}, - } - args := &argsProcessEvent{ - timestamp: 1234, - event: event, - selfShardID: 0, - } - - res := esdtIssueProc.processEvent(args) - require.False(t, res.processed) -} diff --git a/process/elasticproc/logsevents/informativeLogsProcessor.go b/process/elasticproc/logsevents/informativeLogsProcessor.go index 3e53cbd6..ad3709de 100644 --- a/process/elasticproc/logsevents/informativeLogsProcessor.go +++ b/process/elasticproc/logsevents/informativeLogsProcessor.go @@ -61,6 +61,11 @@ func processEventNoTx(args *argsProcessEvent) argOutputProcessEvent { processed: true, } } + if scr.OriginalTxHash == "" { + return argOutputProcessEvent{ + processed: true, + } + } record := &outport.StatusInfo{} switch string(args.event.GetIdentifier()) { diff --git a/process/elasticproc/tokens/disabledIndexTokensHandler.go b/process/elasticproc/tokens/disabledIndexTokensHandler.go new file mode 100644 index 00000000..18fc63f8 --- /dev/null +++ b/process/elasticproc/tokens/disabledIndexTokensHandler.go @@ -0,0 +1,23 @@ +package tokens + +import ( + "github.com/multiversx/mx-chain-es-indexer-go/data" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" +) + +type disabledTndexTokensHandler struct{} + +// NewDisabledIndexTokensHandler creates a new disabled index tokens handler +func NewDisabledIndexTokensHandler() *disabledTndexTokensHandler { + return &disabledTndexTokensHandler{} +} + +// IndexCrossChainTokens should do nothing and return no error +func (dit *disabledTndexTokensHandler) IndexCrossChainTokens(_ elasticproc.DatabaseClientHandler, _ []*data.ScResult, _ *data.BufferSlice) error { + return nil +} + +// IsInterfaceNil returns true if there is no value under the interface +func (dit *disabledTndexTokensHandler) IsInterfaceNil() bool { + return dit == nil +} diff --git a/process/elasticproc/tokens/disabledIndexTokensHandler_test.go b/process/elasticproc/tokens/disabledIndexTokensHandler_test.go new file mode 100644 index 00000000..6b490a81 --- /dev/null +++ b/process/elasticproc/tokens/disabledIndexTokensHandler_test.go @@ -0,0 +1,22 @@ +package tokens + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNewIndexTokensHandler(t *testing.T) { + t.Parallel() + + ith := NewDisabledIndexTokensHandler() + require.False(t, ith.IsInterfaceNil()) +} + +func TestIndexTokensHandler_IndexCrossChainTokens(t *testing.T) { + t.Parallel() + + ith := NewDisabledIndexTokensHandler() + err := ith.IndexCrossChainTokens(nil, nil, nil) + require.NoError(t, err) +} diff --git a/process/elasticproc/tokens/sovereignIndexTokensHandler.go b/process/elasticproc/tokens/sovereignIndexTokensHandler.go new file mode 100644 index 00000000..1a1e1385 --- /dev/null +++ b/process/elasticproc/tokens/sovereignIndexTokensHandler.go @@ -0,0 +1,143 @@ +package tokens + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-core-go/data/esdt" + + "github.com/multiversx/mx-chain-es-indexer-go/data" + indexerdata "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" +) + +type sovereignIndexTokensHandler struct { + mainChainElasticClient elasticproc.MainChainDatabaseClientHandler + esdtPrefix string +} + +// NewSovereignIndexTokensHandler creates a new sovereign index tokens handler +func NewSovereignIndexTokensHandler(mainChainElasticClient elasticproc.MainChainDatabaseClientHandler, esdtPrefix string) (*sovereignIndexTokensHandler, error) { + return &sovereignIndexTokensHandler{ + mainChainElasticClient: mainChainElasticClient, + esdtPrefix: esdtPrefix, + }, nil +} + +// IndexCrossChainTokens will index the new tokens properties +func (sit *sovereignIndexTokensHandler) IndexCrossChainTokens(elasticClient elasticproc.DatabaseClientHandler, scrs []*data.ScResult, buffSlice *data.BufferSlice) error { + if !sit.mainChainElasticClient.IsEnabled() { + return nil + } + + newTokens, err := sit.getNewTokensFromSCRs(elasticClient, scrs) + if err != nil { + return err + } + + if len(newTokens) == 0 { // no new tokens + return nil + } + + // get tokens from main chain elastic db + mainChainTokens := &data.ResponseTokenInfo{} + err = sit.mainChainElasticClient.DoMultiGet(context.Background(), newTokens, indexerdata.TokensIndex, true, mainChainTokens) + if err != nil { + return err + } + + return sit.serializeNewTokens(mainChainTokens.Docs, buffSlice) +} + +func (sit *sovereignIndexTokensHandler) getNewTokensFromSCRs(elasticClient elasticproc.DatabaseClientHandler, scrs []*data.ScResult) ([]string, error) { + receivedTokensIDs := make([]string, 0) + for _, scr := range scrs { + if scr.SenderShard == core.MainChainShardId { + receivedTokensIDs = append(receivedTokensIDs, sit.extractNewSovereignTokens(scr.Tokens)...) + } + } + + if len(receivedTokensIDs) == 0 { + return make([]string, 0), nil + } + + responseTokens := &data.ResponseTokens{} + err := elasticClient.DoMultiGet(context.Background(), receivedTokensIDs, indexerdata.TokensIndex, true, responseTokens) + if err != nil { + return nil, err + } + + newTokens := make([]string, 0) + for _, token := range responseTokens.Docs { + if !token.Found { + newTokens = append(newTokens, token.ID) + } + } + + return newTokens, nil +} + +func (sit *sovereignIndexTokensHandler) extractNewSovereignTokens(tokens []string) []string { + receivedTokensIDs := make([]string, 0) + for _, token := range tokens { + tokenPrefix, hasPrefix := esdt.IsValidPrefixedToken(token) + if !hasPrefix || tokenPrefix != sit.esdtPrefix { + receivedTokensIDs = append(receivedTokensIDs, token) + } + if tokenCollection := getTokenCollection(hasPrefix, token); tokenCollection != "" { + receivedTokensIDs = append(receivedTokensIDs, tokenCollection) + } + } + + return receivedTokensIDs +} + +func getTokenCollection(hasPrefix bool, tokenIdentifier string) string { + tokenSplit := strings.Split(tokenIdentifier, "-") + if !hasPrefix && len(tokenSplit) == 3 { + return tokenSplit[0] + "-" + tokenSplit[1] + } + if hasPrefix && len(tokenSplit) == 4 { + return tokenSplit[1] + "-" + tokenSplit[2] + } + return "" +} + +func (sit *sovereignIndexTokensHandler) serializeNewTokens(responseTokensInfo []data.ResponseTokenInfoDB, buffSlice *data.BufferSlice) error { + for _, responseToken := range responseTokensInfo { + token, identifier := formatToken(responseToken) + + meta := []byte(fmt.Sprintf(`{ "index" : { "_index":"%s", "_id" : "%s" } }%s`, indexerdata.TokensIndex, converters.JsonEscape(identifier), "\n")) + serializedTokenData, err := json.Marshal(token) + if err != nil { + return err + } + + err = buffSlice.PutData(meta, serializedTokenData) + if err != nil { + return err + } + } + + return nil +} + +func formatToken(token data.ResponseTokenInfoDB) (data.TokenInfo, string) { + token.Source.OwnersHistory = nil + token.Source.Properties = nil + + identifier := token.Source.Identifier // for NFTs + if identifier == "" { + identifier = token.Source.Token // for tokens/collections + } + return token.Source, identifier +} + +// IsInterfaceNil returns true if there is no value under the interface +func (sit *sovereignIndexTokensHandler) IsInterfaceNil() bool { + return sit == nil +} diff --git a/process/elasticproc/tokens/sovereignIndexTokensHandler_test.go b/process/elasticproc/tokens/sovereignIndexTokensHandler_test.go new file mode 100644 index 00000000..b881289b --- /dev/null +++ b/process/elasticproc/tokens/sovereignIndexTokensHandler_test.go @@ -0,0 +1,45 @@ +package tokens + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/multiversx/mx-chain-es-indexer-go/client/disabled" + "github.com/multiversx/mx-chain-es-indexer-go/data" + "github.com/multiversx/mx-chain-es-indexer-go/mock" +) + +const ( + prefix = "sov" +) + +func TestSovereignNewIndexTokensHandler(t *testing.T) { + t.Parallel() + + t.Run("valid disabled config, should work", func(t *testing.T) { + sith, err := NewSovereignIndexTokensHandler(disabled.NewDisabledElasticClient(), prefix) + require.NoError(t, err) + require.Equal(t, "*disabled.elasticClient", fmt.Sprintf("%T", sith.mainChainElasticClient)) + }) + t.Run("valid config, should work", func(t *testing.T) { + sith, err := NewSovereignIndexTokensHandler(&mock.DatabaseWriterStub{}, prefix) + require.NoError(t, err) + require.Equal(t, "*mock.DatabaseWriterStub", fmt.Sprintf("%T", sith.mainChainElasticClient)) + }) +} + +func TestSovereignIndexTokensHandler_IndexCrossChainTokens(t *testing.T) { + t.Parallel() + + sith, err := NewSovereignIndexTokensHandler(disabled.NewDisabledElasticClient(), prefix) + require.NoError(t, err) + require.NotNil(t, sith) + + // should skip indexing + err = sith.IndexCrossChainTokens(nil, make([]*data.ScResult, 0), data.NewBufferSlice(0)) + require.NoError(t, err) + + // actual indexing is tested in TestCrossChainTokensIndexingFromMainChain +} diff --git a/process/elasticproc/transactions/checkers.go b/process/elasticproc/transactions/checkers.go index 7d9fcaf7..0319e654 100644 --- a/process/elasticproc/transactions/checkers.go +++ b/process/elasticproc/transactions/checkers.go @@ -10,9 +10,10 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/outport" + vmcommon "github.com/multiversx/mx-chain-vm-common-go" + "github.com/multiversx/mx-chain-es-indexer-go/data" elasticIndexer "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" - vmcommon "github.com/multiversx/mx-chain-vm-common-go" ) const ( @@ -35,6 +36,12 @@ func checkTxsProcessorArg(args *ArgsTransactionProcessor) error { if check.IfNil(args.BalanceConverter) { return elasticIndexer.ErrNilBalanceConverter } + if check.IfNil(args.TxHashExtractor) { + return ErrNilTxHashExtractor + } + if check.IfNil(args.RewardTxData) { + return ErrNilRewardTxDataHandler + } return nil } diff --git a/process/elasticproc/transactions/checkers_test.go b/process/elasticproc/transactions/checkers_test.go index 06ebe9e9..cb0ba985 100644 --- a/process/elasticproc/transactions/checkers_test.go +++ b/process/elasticproc/transactions/checkers_test.go @@ -9,18 +9,24 @@ import ( coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" + vmcommon "github.com/multiversx/mx-chain-vm-common-go" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/mock" elasticIndexer "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" - vmcommon "github.com/multiversx/mx-chain-vm-common-go" - "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" ) func createMockArgs() *ArgsTransactionProcessor { + bc, _ := converters.NewBalanceConverter(18) return &ArgsTransactionProcessor{ AddressPubkeyConverter: &mock.PubkeyConverterMock{}, Hasher: &mock.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, + BalanceConverter: bc, + TxHashExtractor: &mock.TxHashExtractorMock{}, + RewardTxData: &mock.RewardTxDataMock{}, } } @@ -59,6 +65,33 @@ func TestNewTransactionsProcessor(t *testing.T) { }, exErr: elasticIndexer.ErrNilHasher, }, + { + name: "NilBalanceConverter", + args: func() *ArgsTransactionProcessor { + args := createMockArgs() + args.BalanceConverter = nil + return args + }, + exErr: elasticIndexer.ErrNilBalanceConverter, + }, + { + name: "NilTxHashExtractor", + args: func() *ArgsTransactionProcessor { + args := createMockArgs() + args.TxHashExtractor = nil + return args + }, + exErr: ErrNilTxHashExtractor, + }, + { + name: "NilRewardTxDataHandler", + args: func() *ArgsTransactionProcessor { + args := createMockArgs() + args.RewardTxData = nil + return args + }, + exErr: ErrNilRewardTxDataHandler, + }, } for _, tt := range tests { diff --git a/process/elasticproc/transactions/errors.go b/process/elasticproc/transactions/errors.go new file mode 100644 index 00000000..9f91b390 --- /dev/null +++ b/process/elasticproc/transactions/errors.go @@ -0,0 +1,11 @@ +package transactions + +import ( + "errors" +) + +// ErrNilTxHashExtractor signals that a nil tx hash extractor has been provided +var ErrNilTxHashExtractor = errors.New("nil tx hash extractor") + +// ErrNilRewardTxDataHandler signals that a nil rewards tx data handler has been provided +var ErrNilRewardTxDataHandler = errors.New("nil reward tx data handler") diff --git a/process/elasticproc/transactions/interface.go b/process/elasticproc/transactions/interface.go index 4a201d14..3139cc2a 100644 --- a/process/elasticproc/transactions/interface.go +++ b/process/elasticproc/transactions/interface.go @@ -1,6 +1,7 @@ package transactions import ( + coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/outport" datafield "github.com/multiversx/mx-chain-vm-common-go/parsers/dataField" ) @@ -13,3 +14,15 @@ type DataFieldParser interface { type feeInfoHandler interface { GetFeeInfo() *outport.FeeInfo } + +// TxHashExtractor defines what tx hash extractor should be able to do +type TxHashExtractor interface { + ExtractExecutedTxHashes(mbIndex int, mbTxHashes [][]byte, header coreData.HeaderHandler) [][]byte + IsInterfaceNil() bool +} + +// RewardTxDataHandler defines what rewards tx handler should be able to do +type RewardTxDataHandler interface { + GetSender() string + IsInterfaceNil() bool +} diff --git a/process/elasticproc/transactions/rewardTxData.go b/process/elasticproc/transactions/rewardTxData.go new file mode 100644 index 00000000..48678e5f --- /dev/null +++ b/process/elasticproc/transactions/rewardTxData.go @@ -0,0 +1,24 @@ +package transactions + +import ( + "fmt" + + "github.com/multiversx/mx-chain-core-go/core" +) + +type rewardTxData struct{} + +// NewRewardTxData creates a new reward tx data +func NewRewardTxData() *rewardTxData { + return &rewardTxData{} +} + +// GetSender return the metachain shard id as string +func (rtd *rewardTxData) GetSender() string { + return fmt.Sprintf("%d", core.MetachainShardId) +} + +// IsInterfaceNil returns true if there is no value under the interface +func (rtd *rewardTxData) IsInterfaceNil() bool { + return rtd == nil +} diff --git a/process/elasticproc/transactions/scrsDataToTransactions.go b/process/elasticproc/transactions/scrsDataToTransactions.go index af2a5148..09ef1a06 100644 --- a/process/elasticproc/transactions/scrsDataToTransactions.go +++ b/process/elasticproc/transactions/scrsDataToTransactions.go @@ -54,7 +54,7 @@ func (st *scrsDataToTransactions) processTransactionsAfterSCRsWereAttached(trans func (st *scrsDataToTransactions) processSCRsWithoutTx(scrs []*data.ScResult) map[string]*data.FeeData { txHashRefund := make(map[string]*data.FeeData) for _, scr := range scrs { - if scr.InitialTxGasUsed == 0 && scr.GasRefunded == 0 { + if (scr.InitialTxGasUsed == 0 || scr.OriginalTxHash == "") && scr.GasRefunded == 0 { continue } diff --git a/process/elasticproc/transactions/sovereignRewardTxData.go b/process/elasticproc/transactions/sovereignRewardTxData.go new file mode 100644 index 00000000..91b266e8 --- /dev/null +++ b/process/elasticproc/transactions/sovereignRewardTxData.go @@ -0,0 +1,24 @@ +package transactions + +import ( + "fmt" + + "github.com/multiversx/mx-chain-core-go/core" +) + +type sovereignRewardTxData struct{} + +// NewSovereignRewardTxData creates a new sovereign reward tx data +func NewSovereignRewardTxData() *sovereignRewardTxData { + return &sovereignRewardTxData{} +} + +// GetSender return the sovereign shard id as string +func (srtd *sovereignRewardTxData) GetSender() string { + return fmt.Sprintf("%d", core.SovereignChainShardId) +} + +// IsInterfaceNil returns true if there is no value under the interface +func (srtd *sovereignRewardTxData) IsInterfaceNil() bool { + return srtd == nil +} diff --git a/process/elasticproc/transactions/sovereignTxHashExtractor.go b/process/elasticproc/transactions/sovereignTxHashExtractor.go new file mode 100644 index 00000000..aadbdd64 --- /dev/null +++ b/process/elasticproc/transactions/sovereignTxHashExtractor.go @@ -0,0 +1,22 @@ +package transactions + +import ( + coreData "github.com/multiversx/mx-chain-core-go/data" +) + +type sovereignTxHashExtractor struct{} + +// NewSovereignTxHashExtractor creates a new sovereign tx hash extractor +func NewSovereignTxHashExtractor() *sovereignTxHashExtractor { + return &sovereignTxHashExtractor{} +} + +// ExtractExecutedTxHashes returns directly the provided mini block tx hashes +func (the *sovereignTxHashExtractor) ExtractExecutedTxHashes(_ int, mbTxHashes [][]byte, _ coreData.HeaderHandler) [][]byte { + return mbTxHashes +} + +// IsInterfaceNil returns true if there is no value under the interface +func (the *sovereignTxHashExtractor) IsInterfaceNil() bool { + return the == nil +} diff --git a/process/elasticproc/transactions/sovereignTxHashExtractor_test.go b/process/elasticproc/transactions/sovereignTxHashExtractor_test.go new file mode 100644 index 00000000..1332c50c --- /dev/null +++ b/process/elasticproc/transactions/sovereignTxHashExtractor_test.go @@ -0,0 +1,23 @@ +package transactions + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNewSovereignTxHashExtractor(t *testing.T) { + t.Parallel() + + sthe := NewSovereignTxHashExtractor() + require.False(t, sthe.IsInterfaceNil()) +} + +func TestSovereignTxHashExtractor_ExtractExecutedTxHashes(t *testing.T) { + t.Parallel() + + sthe := NewSovereignTxHashExtractor() + mbTxHashes := [][]byte{[]byte("hash1"), []byte("hash2")} + txHashes := sthe.ExtractExecutedTxHashes(0, mbTxHashes, nil) + require.Equal(t, mbTxHashes, txHashes) +} diff --git a/process/elasticproc/transactions/transactionDBBuilder.go b/process/elasticproc/transactions/transactionDBBuilder.go index 3bb75015..df639c11 100644 --- a/process/elasticproc/transactions/transactionDBBuilder.go +++ b/process/elasticproc/transactions/transactionDBBuilder.go @@ -2,7 +2,6 @@ package transactions import ( "encoding/hex" - "fmt" "time" "github.com/multiversx/mx-chain-core-go/core" @@ -11,6 +10,7 @@ import ( "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/receipt" + "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" @@ -20,17 +20,20 @@ type dbTransactionBuilder struct { addressPubkeyConverter core.PubkeyConverter dataFieldParser DataFieldParser balanceConverter dataindexer.BalanceConverter + rewardTxData RewardTxDataHandler } func newTransactionDBBuilder( addressPubkeyConverter core.PubkeyConverter, dataFieldParser DataFieldParser, balanceConverter dataindexer.BalanceConverter, + rewardTxData RewardTxDataHandler, ) *dbTransactionBuilder { return &dbTransactionBuilder{ addressPubkeyConverter: addressPubkeyConverter, dataFieldParser: dataFieldParser, balanceConverter: balanceConverter, + rewardTxData: rewardTxData, } } @@ -167,7 +170,7 @@ func (dtb *dbTransactionBuilder) prepareRewardTransaction( Value: rTx.Value.String(), ValueNum: valueNum, Receiver: receiverAddr, - Sender: fmt.Sprintf("%d", core.MetachainShardId), + Sender: dtb.rewardTxData.GetSender(), ReceiverShard: mb.ReceiverShardID, SenderShard: mb.SenderShardID, GasPrice: 0, diff --git a/process/elasticproc/transactions/transactionDBBuilder_test.go b/process/elasticproc/transactions/transactionDBBuilder_test.go index ce67ece1..3fab26fa 100644 --- a/process/elasticproc/transactions/transactionDBBuilder_test.go +++ b/process/elasticproc/transactions/transactionDBBuilder_test.go @@ -2,20 +2,19 @@ package transactions import ( "encoding/hex" - "fmt" "math/big" "testing" "time" - "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/rewardTx" "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/mock" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" - "github.com/stretchr/testify/require" ) func createCommonProcessor() dbTransactionBuilder { @@ -24,6 +23,7 @@ func createCommonProcessor() dbTransactionBuilder { addressPubkeyConverter: mock.NewPubkeyConverterMock(32), dataFieldParser: createDataFieldParserMock(), balanceConverter: ap, + rewardTxData: &mock.RewardTxDataMock{}, } } @@ -106,7 +106,13 @@ func TestGetMoveBalanceTransaction(t *testing.T) { func TestGetTransactionByType_RewardTx(t *testing.T) { t.Parallel() + sender := "sender" cp := createCommonProcessor() + cp.rewardTxData = &mock.RewardTxDataMock{ + GetSenderCalled: func() string { + return sender + }, + } round := uint64(10) rcvAddr := []byte("receiver") @@ -129,7 +135,7 @@ func TestGetTransactionByType_RewardTx(t *testing.T) { Receiver: hex.EncodeToString(rcvAddr), Status: status, Value: "", - Sender: fmt.Sprintf("%d", core.MetachainShardId), + Sender: sender, Data: make([]byte, 0), Operation: rewardsOperation, } diff --git a/process/elasticproc/transactions/transactionsGrouper.go b/process/elasticproc/transactions/transactionsGrouper.go index 5aaf15a4..69de99b8 100644 --- a/process/elasticproc/transactions/transactionsGrouper.go +++ b/process/elasticproc/transactions/transactionsGrouper.go @@ -11,6 +11,7 @@ import ( "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/multiversx/mx-chain-es-indexer-go/data" ) @@ -19,20 +20,23 @@ const ( ) type txsGrouper struct { - txBuilder *dbTransactionBuilder - hasher hashing.Hasher - marshalizer marshal.Marshalizer + txBuilder *dbTransactionBuilder + hasher hashing.Hasher + marshalizer marshal.Marshalizer + txHashExtractor TxHashExtractor } func newTxsGrouper( txBuilder *dbTransactionBuilder, hasher hashing.Hasher, marshalizer marshal.Marshalizer, + txHashExtractor TxHashExtractor, ) *txsGrouper { return &txsGrouper{ - txBuilder: txBuilder, - hasher: hasher, - marshalizer: marshalizer, + txBuilder: txBuilder, + hasher: hasher, + marshalizer: marshalizer, + txHashExtractor: txHashExtractor, } } @@ -52,7 +56,7 @@ func (tg *txsGrouper) groupNormalTxs( } selfShardID := header.GetShardID() - executedTxHashes := extractExecutedTxHashes(mbIndex, mb.TxHashes, header) + executedTxHashes := tg.txHashExtractor.ExtractExecutedTxHashes(mbIndex, mb.TxHashes, header) mbStatus := computeStatus(selfShardID, mb.ReceiverShardID) for _, txHash := range executedTxHashes { dbTx, ok := tg.prepareNormalTxForDB(mbHash, mb, mbStatus, txHash, txs, header, numOfShards) @@ -68,27 +72,6 @@ func (tg *txsGrouper) groupNormalTxs( return transactions, nil } -func extractExecutedTxHashes(mbIndex int, mbTxHashes [][]byte, header coreData.HeaderHandler) [][]byte { - miniblockHeaders := header.GetMiniBlockHeaderHandlers() - if len(miniblockHeaders) <= mbIndex { - return mbTxHashes - } - - firstProcessed := miniblockHeaders[mbIndex].GetIndexOfFirstTxProcessed() - lastProcessed := miniblockHeaders[mbIndex].GetIndexOfLastTxProcessed() - - executedTxHashes := make([][]byte, 0) - for txIndex, txHash := range mbTxHashes { - if int32(txIndex) < firstProcessed || int32(txIndex) > lastProcessed { - continue - } - - executedTxHashes = append(executedTxHashes, txHash) - } - - return executedTxHashes -} - func (tg *txsGrouper) prepareNormalTxForDB( mbHash []byte, mb *block.MiniBlock, @@ -123,7 +106,7 @@ func (tg *txsGrouper) groupRewardsTxs( selfShardID := header.GetShardID() mbStatus := computeStatus(selfShardID, mb.ReceiverShardID) - executedTxHashes := extractExecutedTxHashes(mbIndex, mb.TxHashes, header) + executedTxHashes := tg.txHashExtractor.ExtractExecutedTxHashes(mbIndex, mb.TxHashes, header) for _, txHash := range executedTxHashes { rewardDBTx, ok := tg.prepareRewardTxForDB(mbHash, mb, mbStatus, txHash, txs, header) if !ok { @@ -169,7 +152,7 @@ func (tg *txsGrouper) groupInvalidTxs( return nil, err } - executedTxHashes := extractExecutedTxHashes(mbIndex, mb.TxHashes, header) + executedTxHashes := tg.txHashExtractor.ExtractExecutedTxHashes(mbIndex, mb.TxHashes, header) for _, txHash := range executedTxHashes { invalidDBTx, ok := tg.prepareInvalidTxForDB(mbHash, mb, txHash, txs, header, numOfShards) if !ok { diff --git a/process/elasticproc/transactions/transactionsGrouper_test.go b/process/elasticproc/transactions/transactionsGrouper_test.go index 12c6b75e..8712e1a0 100644 --- a/process/elasticproc/transactions/transactionsGrouper_test.go +++ b/process/elasticproc/transactions/transactionsGrouper_test.go @@ -4,14 +4,16 @@ import ( "encoding/hex" "testing" + coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/receipt" "github.com/multiversx/mx-chain-core-go/data/rewardTx" "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-es-indexer-go/mock" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" - "github.com/stretchr/testify/require" ) func TestGroupNormalTxs(t *testing.T) { @@ -19,8 +21,7 @@ func TestGroupNormalTxs(t *testing.T) { parser := createDataFieldParserMock() ap, _ := converters.NewBalanceConverter(18) - txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, parser, ap) - grouper := newTxsGrouper(txBuilder, &mock.HasherMock{}, &mock.MarshalizerMock{}) + txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, parser, ap, &mock.RewardTxDataMock{}) txHash1 := []byte("txHash1") txHash2 := []byte("txHash2") @@ -46,6 +47,13 @@ func TestGroupNormalTxs(t *testing.T) { }, } + txHashExtractor := &mock.TxHashExtractorMock{ + ExtractExecutedTxHashesCalled: func(_ int, _ [][]byte, _ coreData.HeaderHandler) [][]byte { + return mb.TxHashes + }, + } + grouper := newTxsGrouper(txBuilder, &mock.HasherMock{}, &mock.MarshalizerMock{}, txHashExtractor) + normalTxs, _ := grouper.groupNormalTxs(0, mb, header, txs, false, 3) require.Len(t, normalTxs, 2) } @@ -55,8 +63,7 @@ func TestGroupRewardsTxs(t *testing.T) { parser := createDataFieldParserMock() ap, _ := converters.NewBalanceConverter(18) - txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, parser, ap) - grouper := newTxsGrouper(txBuilder, &mock.HasherMock{}, &mock.MarshalizerMock{}) + txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, parser, ap, &mock.RewardTxDataMock{}) txHash1 := []byte("txHash1") txHash2 := []byte("txHash2") @@ -74,6 +81,13 @@ func TestGroupRewardsTxs(t *testing.T) { }}, } + txHashExtractor := &mock.TxHashExtractorMock{ + ExtractExecutedTxHashesCalled: func(_ int, _ [][]byte, _ coreData.HeaderHandler) [][]byte { + return mb.TxHashes + }, + } + grouper := newTxsGrouper(txBuilder, &mock.HasherMock{}, &mock.MarshalizerMock{}, txHashExtractor) + normalTxs, _ := grouper.groupRewardsTxs(0, mb, header, txs, false) require.Len(t, normalTxs, 2) } @@ -83,8 +97,7 @@ func TestGroupInvalidTxs(t *testing.T) { parser := createDataFieldParserMock() ap, _ := converters.NewBalanceConverter(18) - txBuilder := newTransactionDBBuilder(mock.NewPubkeyConverterMock(32), parser, ap) - grouper := newTxsGrouper(txBuilder, &mock.HasherMock{}, &mock.MarshalizerMock{}) + txBuilder := newTransactionDBBuilder(mock.NewPubkeyConverterMock(32), parser, ap, &mock.RewardTxDataMock{}) txHash1 := []byte("txHash1") txHash2 := []byte("txHash2") @@ -106,6 +119,13 @@ func TestGroupInvalidTxs(t *testing.T) { }, FeeInfo: &outport.FeeInfo{}}, } + txHashExtractor := &mock.TxHashExtractorMock{ + ExtractExecutedTxHashesCalled: func(_ int, _ [][]byte, _ coreData.HeaderHandler) [][]byte { + return mb.TxHashes + }, + } + grouper := newTxsGrouper(txBuilder, &mock.HasherMock{}, &mock.MarshalizerMock{}, txHashExtractor) + normalTxs, _ := grouper.groupInvalidTxs(0, mb, header, txs, 3) require.Len(t, normalTxs, 2) } @@ -115,8 +135,8 @@ func TestGroupReceipts(t *testing.T) { parser := createDataFieldParserMock() ap, _ := converters.NewBalanceConverter(18) - txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, parser, ap) - grouper := newTxsGrouper(txBuilder, &mock.HasherMock{}, &mock.MarshalizerMock{}) + txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, parser, ap, &mock.RewardTxDataMock{}) + grouper := newTxsGrouper(txBuilder, &mock.HasherMock{}, &mock.MarshalizerMock{}, &mock.TxHashExtractorMock{}) txHash1 := []byte("txHash1") txHash2 := []byte("txHash2") diff --git a/process/elasticproc/transactions/transactionsProcessor.go b/process/elasticproc/transactions/transactionsProcessor.go index b85c9b71..e98af48d 100644 --- a/process/elasticproc/transactions/transactionsProcessor.go +++ b/process/elasticproc/transactions/transactionsProcessor.go @@ -11,10 +11,11 @@ import ( "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" - "github.com/multiversx/mx-chain-es-indexer-go/data" - "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" logger "github.com/multiversx/mx-chain-logger-go" datafield "github.com/multiversx/mx-chain-vm-common-go/parsers/dataField" + + "github.com/multiversx/mx-chain-es-indexer-go/data" + "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" ) var log = logger.GetOrCreate("indexer/process/transactions") @@ -26,6 +27,8 @@ type ArgsTransactionProcessor struct { Hasher hashing.Hasher Marshalizer marshal.Marshalizer BalanceConverter dataindexer.BalanceConverter + TxHashExtractor TxHashExtractor + RewardTxData RewardTxDataHandler } type txsDatabaseProcessor struct { @@ -51,8 +54,8 @@ func NewTransactionsProcessor(args *ArgsTransactionProcessor) (*txsDatabaseProce return nil, err } - txBuilder := newTransactionDBBuilder(args.AddressPubkeyConverter, operationsDataParser, args.BalanceConverter) - txsDBGrouper := newTxsGrouper(txBuilder, args.Hasher, args.Marshalizer) + txBuilder := newTransactionDBBuilder(args.AddressPubkeyConverter, operationsDataParser, args.BalanceConverter, args.RewardTxData) + txsDBGrouper := newTxsGrouper(txBuilder, args.Hasher, args.Marshalizer, args.TxHashExtractor) scrProc := newSmartContractResultsProcessor(args.AddressPubkeyConverter, args.Marshalizer, args.Hasher, operationsDataParser, args.BalanceConverter) scrsDataToTxs := newScrsDataToTransactions(args.BalanceConverter) diff --git a/process/elasticproc/transactions/transactionsProcessor_test.go b/process/elasticproc/transactions/transactionsProcessor_test.go index 1348e427..38c0c3b7 100644 --- a/process/elasticproc/transactions/transactionsProcessor_test.go +++ b/process/elasticproc/transactions/transactionsProcessor_test.go @@ -13,11 +13,12 @@ import ( "github.com/multiversx/mx-chain-core-go/data/rewardTx" "github.com/multiversx/mx-chain-core-go/data/smartContractResult" "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/mock" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func createMockArgsTxsDBProc() *ArgsTransactionProcessor { @@ -27,6 +28,8 @@ func createMockArgsTxsDBProc() *ArgsTransactionProcessor { Hasher: &mock.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, BalanceConverter: ap, + TxHashExtractor: NewTxHashExtractor(), + RewardTxData: &mock.RewardTxDataMock{}, } return args } diff --git a/process/elasticproc/transactions/txHashExtractor.go b/process/elasticproc/transactions/txHashExtractor.go new file mode 100644 index 00000000..1914be53 --- /dev/null +++ b/process/elasticproc/transactions/txHashExtractor.go @@ -0,0 +1,39 @@ +package transactions + +import ( + coreData "github.com/multiversx/mx-chain-core-go/data" +) + +type txHashExtractor struct{} + +// NewTxHashExtractor creates a new tx hash extractor +func NewTxHashExtractor() *txHashExtractor { + return &txHashExtractor{} +} + +// ExtractExecutedTxHashes returns executed tx hashes +func (the *txHashExtractor) ExtractExecutedTxHashes(mbIndex int, mbTxHashes [][]byte, header coreData.HeaderHandler) [][]byte { + miniblockHeaders := header.GetMiniBlockHeaderHandlers() + if len(miniblockHeaders) <= mbIndex { + return mbTxHashes + } + + firstProcessed := miniblockHeaders[mbIndex].GetIndexOfFirstTxProcessed() + lastProcessed := miniblockHeaders[mbIndex].GetIndexOfLastTxProcessed() + + executedTxHashes := make([][]byte, 0) + for txIndex, txHash := range mbTxHashes { + if int32(txIndex) < firstProcessed || int32(txIndex) > lastProcessed { + continue + } + + executedTxHashes = append(executedTxHashes, txHash) + } + + return executedTxHashes +} + +// IsInterfaceNil returns true if there is no value under the interface +func (the *txHashExtractor) IsInterfaceNil() bool { + return the == nil +} diff --git a/process/elasticproc/transactions/txHashExtractor_test.go b/process/elasticproc/transactions/txHashExtractor_test.go new file mode 100644 index 00000000..9f56def5 --- /dev/null +++ b/process/elasticproc/transactions/txHashExtractor_test.go @@ -0,0 +1,24 @@ +package transactions + +import ( + "testing" + + "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/stretchr/testify/require" +) + +func TestNewTxHashExtractor(t *testing.T) { + t.Parallel() + + the := NewTxHashExtractor() + require.False(t, the.IsInterfaceNil()) +} + +func TestTxHashExtractor_ExtractExecutedTxHashes(t *testing.T) { + t.Parallel() + + the := NewTxHashExtractor() + mbTxHashes := [][]byte{[]byte("hash1"), []byte("hash2")} + txHashes := the.ExtractExecutedTxHashes(0, mbTxHashes, &block.Header{}) + require.Equal(t, mbTxHashes, txHashes) +} diff --git a/process/factory/indexerFactory.go b/process/factory/indexerFactory.go index 62db8773..5def543a 100644 --- a/process/factory/indexerFactory.go +++ b/process/factory/indexerFactory.go @@ -2,9 +2,7 @@ package factory import ( "fmt" - "math" "net/http" - "time" "github.com/elastic/go-elasticsearch/v7" "github.com/multiversx/mx-chain-core-go/core" @@ -12,14 +10,16 @@ import ( "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-es-indexer-go/client" "github.com/multiversx/mx-chain-es-indexer-go/client/logging" "github.com/multiversx/mx-chain-es-indexer-go/client/transport" indexerCore "github.com/multiversx/mx-chain-es-indexer-go/core" + "github.com/multiversx/mx-chain-es-indexer-go/factory/runType" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" - logger "github.com/multiversx/mx-chain-logger-go" ) var log = logger.GetOrCreate("indexer/factory") @@ -30,6 +30,9 @@ type ArgsIndexerFactory struct { Enabled bool UseKibana bool ImportDB bool + Sovereign bool + ESDTPrefix string + MainChainElastic factory.ElasticConfig Denomination int BulkRequestMaxSize int Url string @@ -44,6 +47,7 @@ type ArgsIndexerFactory struct { AddressPubkeyConverter core.PubkeyConverter ValidatorPubkeyConverter core.PubkeyConverter StatusMetrics indexerCore.StatusMetricsHandler + RunTypeComponents runType.RunTypeComponentsHandler } // NewIndexer will create a new instance of Indexer @@ -53,6 +57,15 @@ func NewIndexer(args ArgsIndexerFactory) (dataindexer.Indexer, error) { return nil, err } + if args.Sovereign { + args.RunTypeComponents, err = createManagedRunTypeComponents(runType.NewSovereignRunTypeComponentsFactory(args.MainChainElastic, args.ESDTPrefix)) + } else { + args.RunTypeComponents, err = createManagedRunTypeComponents(runType.NewRunTypeComponentsFactory()) + } + if err != nil { + return nil, err + } + elasticProcessor, err := createElasticProcessor(args) if err != nil { return nil, err @@ -72,11 +85,18 @@ func NewIndexer(args ArgsIndexerFactory) (dataindexer.Indexer, error) { return dataindexer.NewDataIndexer(arguments) } -func retryBackOff(attempt int) time.Duration { - d := time.Duration(math.Exp2(float64(attempt))) * time.Second - log.Debug("elastic: retry backoff", "attempt", attempt, "sleep duration", d) +func createManagedRunTypeComponents(factory runType.RunTypeComponentsCreator) (runType.RunTypeComponentsHandler, error) { + managedRunTypeComponents, err := runType.NewManagedRunTypeComponents(factory) + if err != nil { + return nil, err + } + + err = managedRunTypeComponents.Create() + if err != nil { + return nil, err + } - return d + return managedRunTypeComponents, nil } func createElasticProcessor(args ArgsIndexerFactory) (dataindexer.ElasticProcessor, error) { @@ -97,6 +117,9 @@ func createElasticProcessor(args ArgsIndexerFactory) (dataindexer.ElasticProcess BulkRequestMaxSize: args.BulkRequestMaxSize, ImportDB: args.ImportDB, Version: args.Version, + TxHashExtractor: args.RunTypeComponents.TxHashExtractorCreator(), + RewardTxData: args.RunTypeComponents.RewardTxDataCreator(), + IndexTokensHandler: args.RunTypeComponents.IndexTokensHandlerCreator(), } return factory.CreateElasticProcessor(argsElasticProcFac) @@ -109,7 +132,7 @@ func createElasticClient(args ArgsIndexerFactory) (elasticproc.DatabaseClientHan Password: args.Password, Logger: &logging.CustomLogger{}, RetryOnStatus: []int{http.StatusConflict}, - RetryBackoff: retryBackOff, + RetryBackoff: client.RetryBackOff, } if check.IfNil(args.StatusMetrics) { @@ -162,6 +185,10 @@ func createBlockCreatorsContainer() (dataindexer.BlockContainerHandler, error) { if err != nil { return nil, err } + err = container.Add(core.SovereignChainHeader, block.NewEmptySovereignHeaderCreator()) + if err != nil { + return nil, err + } return container, nil } diff --git a/scripts/script.sh b/scripts/script.sh index 11a1bdb4..166780fc 100755 --- a/scripts/script.sh +++ b/scripts/script.sh @@ -1,4 +1,5 @@ IMAGE_NAME=elastic-container +MAIN_CHAIN_IMAGE_NAME=main-chain-elastic-container DEFAULT_ES_VERSION=7.16.2 PROMETHEUS_CONTAINER_NAME=prometheus_container GRAFANA_CONTAINER_NAME=grafana_container @@ -16,9 +17,13 @@ start() { docker pull docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} docker rm ${IMAGE_NAME} 2> /dev/null + docker rm ${MAIN_CHAIN_IMAGE_NAME} 2> /dev/null docker run -d --name "${IMAGE_NAME}" -p 9200:9200 -p 9300:9300 \ -e "discovery.type=single-node" -e "xpack.security.enabled=false" -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} + docker run -d --name "${MAIN_CHAIN_IMAGE_NAME}" -p 9201:9200 -p 9301:9300 \ + -e "discovery.type=single-node" -e "xpack.security.enabled=false" -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ + docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} # Wait elastic cluster to start echo "Waiting Elasticsearch cluster to start..." @@ -27,6 +32,7 @@ start() { stop() { docker stop "${IMAGE_NAME}" + docker stop "${MAIN_CHAIN_IMAGE_NAME}" } delete() { @@ -41,6 +47,7 @@ delete() { IMAGE_OPEN_SEARCH=open-container +MAIN_CHAIN_IMAGE_OPEN_SEARCH=main-chain-open-container DEFAULT_OPEN_SEARCH_VERSION=1.2.4 start_open_search() { @@ -52,9 +59,13 @@ start_open_search() { docker pull opensearchproject/opensearch:${OPEN_VERSION} docker rm ${IMAGE_OPEN_SEARCH} 2> /dev/null + docker rm ${MAIN_CHAIN_IMAGE_OPEN_SEARCH} 2> /dev/null docker run -d --name "${IMAGE_OPEN_SEARCH}" -p 9200:9200 -p 9600:9600 \ -e "discovery.type=single-node" -e "plugins.security.disabled=true" -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ opensearchproject/opensearch:${OPEN_VERSION} + docker run -d --name "${MAIN_CHAIN_IMAGE_OPEN_SEARCH}" -p 9201:9200 -p 9601:9600 \ + -e "discovery.type=single-node" -e "plugins.security.disabled=true" -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ + opensearchproject/opensearch:${OPEN_VERSION} } diff --git a/templates/noKibana/accountsESDT.go b/templates/noKibana/accountsESDT.go index 6ccf1945..d15ecaf3 100644 --- a/templates/noKibana/accountsESDT.go +++ b/templates/noKibana/accountsESDT.go @@ -74,7 +74,7 @@ var AccountsESDT = Object{ "format": "epoch_second", }, "token": Object{ - "type": "text", + "type": "keyword", }, "tokenNonce": Object{ "type": "double", diff --git a/templates/noKibana/tokens.go b/templates/noKibana/tokens.go index a7de2dce..fbfb812c 100644 --- a/templates/noKibana/tokens.go +++ b/templates/noKibana/tokens.go @@ -167,7 +167,7 @@ var Tokens = Object{ "format": "epoch_second", }, "token": Object{ - "type": "text", + "type": "keyword", }, "type": Object{ "type": "keyword",