diff --git a/client/elasticClient.go b/client/elasticClient.go index 43c04710..536945a6 100644 --- a/client/elasticClient.go +++ b/client/elasticClient.go @@ -7,11 +7,9 @@ import ( "fmt" "io" "net/http" - "strings" "github.com/elastic/go-elasticsearch/v7" "github.com/elastic/go-elasticsearch/v7/esapi" - "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" logger "github.com/multiversx/mx-chain-logger-go" ) @@ -76,6 +74,26 @@ func (ec *elasticClient) CheckAndCreatePolicy(policyName string, policy *bytes.B return ec.createPolicy(policyName, policy) } +// SetWriteIndexTrue will set the provided index as write index +func (ec *elasticClient) SetWriteIndexTrue(alias string, index string) error { + body := fmt.Sprintf(`{"actions" : [ { "add" : { "index" : "%s", "alias" : "%s", "is_write_index" : true } }]}`, index, alias) + res, err := ec.client.Indices.UpdateAliases( + bytes.NewBuffer([]byte(body)), + ) + if err != nil { + return err + } + + defer closeBody(res) + + if res.IsError() { + return fmt.Errorf("%s", res.String()) + } + + return nil + +} + // CheckAndCreateIndex creates a new index if it does not already exist func (ec *elasticClient) CheckAndCreateIndex(indexName string) error { if ec.indexExists(indexName) { @@ -225,36 +243,18 @@ func (ec *elasticClient) indexExists(index string) bool { // PolicyExists checks if a policy was already created func (ec *elasticClient) PolicyExists(policy string) bool { - policyRoute := fmt.Sprintf( - "%s/%s/ism/policies/%s", - ec.elasticBaseUrl, - kibanaPluginPath, - policy, + res, err := ec.client.ILM.GetLifecycle( + ec.client.ILM.GetLifecycle.WithPolicy(policy), ) - - req := newRequest(http.MethodGet, policyRoute, nil) - res, err := ec.client.Transport.Perform(req) if err != nil { - log.Warn("elasticClient.PolicyExists", - "error performing request", err.Error()) + log.Warn("elasticClient.PolicyExists", "error", err.Error()) return false } - - response := &esapi.Response{ - StatusCode: res.StatusCode, - Body: res.Body, - Header: res.Header, - } - - existsRes := &data.Response{} - err = parseResponse(response, existsRes, kibanaResponseErrorHandler) - if err != nil { - log.Warn("elasticClient.PolicyExists", - "error returned by kibana api", err.Error()) - return false + if res.StatusCode == http.StatusOK { + return true } - return existsRes.Status == http.StatusConflict + return false } // AliasExists checks if an index alias already exists @@ -293,38 +293,15 @@ func (ec *elasticClient) createIndex(index string) error { // CreatePolicy creates a new policy for elastic indexes. Policies define rollover parameters func (ec *elasticClient) createPolicy(policyName string, policy *bytes.Buffer) error { - policyRoute := fmt.Sprintf( - "%s/_opendistro/_ism/policies/%s", - ec.elasticBaseUrl, + res, err := ec.client.ILM.PutLifecycle( policyName, + ec.client.ILM.PutLifecycle.WithBody(policy), ) - - req := newRequest(http.MethodPut, policyRoute, policy) - req.Header[headerContentType] = headerContentTypeJSON - req.Header[headerXSRF] = []string{"false"} - res, err := ec.client.Transport.Perform(req) if err != nil { return err } - response := &esapi.Response{ - StatusCode: res.StatusCode, - Body: res.Body, - Header: res.Header, - } - - existsRes := &data.Response{} - err = parseResponse(response, existsRes, kibanaResponseErrorHandler) - if err != nil { - return err - } - - errStr := fmt.Sprintf("%v", existsRes.Error) - if existsRes.Status == http.StatusConflict && !strings.Contains(errStr, errPolicyAlreadyExists) { - return dataindexer.ErrCouldNotCreatePolicy - } - - return nil + return parseResponse(res, nil, elasticDefaultErrorResponseHandler) } // CreateIndexTemplate creates an elasticsearch index template diff --git a/cmd/elasticindexer/config/config.toml b/cmd/elasticindexer/config/config.toml index 1f443da7..004424b8 100644 --- a/cmd/elasticindexer/config/config.toml +++ b/cmd/elasticindexer/config/config.toml @@ -4,6 +4,12 @@ "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags", "logs", "delegators", "operations", "esdts", "values", "events", "executionresults" ] + use-templates-from-files = true + [config.policies] + enabled = true + indices-with-policy = ["rating", "transactions", "blocks", "validators", "miniblocks", "rounds", "accounts", + "accountshistory", "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", + "scdeploys", "tokens", "tags", "logs", "delegators", "operations", "esdts", "events", "executionresults"] [config.address-converter] length = 32 type = "bech32" diff --git a/cmd/elasticindexer/config/indices/accounts.json b/cmd/elasticindexer/config/indices/accounts.json new file mode 100644 index 00000000..b9f15d85 --- /dev/null +++ b/cmd/elasticindexer/config/indices/accounts.json @@ -0,0 +1,46 @@ +{ + "index_patterns": [ + "accounts-*" + ], + "template": { + "settings": { + "index.lifecycle.name": "accounts-policy", + "index.lifecycle.rollover_alias": "accounts", + "number_of_replicas": 1, + "number_of_shards": 5 + }, + "mappings": { + "properties": { + "address": { + "type": "keyword" + }, + "balance": { + "type": "keyword" + }, + "balanceNum": { + "type": "double" + }, + "nonce": { + "type": "double" + }, + "shardID": { + "type": "long" + }, + "timestamp": { + "format": "epoch_second", + "type": "date" + }, + "timestampMs": { + "type": "date", + "format": "epoch_millis" + }, + "totalBalanceWithStake": { + "type": "keyword" + }, + "totalBalanceWithStakeNum": { + "type": "double" + } + } + } + } +} diff --git a/cmd/elasticindexer/config/indices/accountsesdt.json b/cmd/elasticindexer/config/indices/accountsesdt.json new file mode 100644 index 00000000..10db910b --- /dev/null +++ b/cmd/elasticindexer/config/indices/accountsesdt.json @@ -0,0 +1,91 @@ +{ + "index_patterns": [ + "accountsesdt-*" + ], + "template": { + "settings": { + "index.lifecycle.name": "accountsesdt-policy", + "index.lifecycle.rollover_alias": "accountsesdt", + "number_of_replicas": 1, + "number_of_shards": 5 + }, + "mappings": { + "properties": { + "address": { + "type": "keyword" + }, + "balance": { + "type": "keyword" + }, + "balanceNum": { + "type": "double" + }, + "currentOwner": { + "type": "keyword" + }, + "data": { + "properties": { + "attributes": { + "index": "false", + "type": "keyword" + }, + "creator": { + "type": "keyword" + }, + "hash": { + "index": "false", + "type": "keyword" + }, + "metadata": { + "index": "false", + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "nonEmptyURIs": { + "type": "boolean" + }, + "royalties": { + "index": "false", + "type": "long" + }, + "tags": { + "type": "text" + }, + "uris": { + "type": "text" + } + }, + "type": "nested" + }, + "identifier": { + "type": "text" + }, + "properties": { + "type": "keyword" + }, + "shardID": { + "type": "long" + }, + "timestamp": { + "format": "epoch_second", + "type": "date" + }, + "timestampMs": { + "type": "date", + "format": "epoch_millis" + }, + "token": { + "type": "keyword" + }, + "tokenNonce": { + "type": "double" + }, + "type": { + "type": "keyword" + } + } + } + } +} diff --git a/cmd/elasticindexer/config/indices/accountsesdthistory.json b/cmd/elasticindexer/config/indices/accountsesdthistory.json new file mode 100644 index 00000000..27f33d76 --- /dev/null +++ b/cmd/elasticindexer/config/indices/accountsesdthistory.json @@ -0,0 +1,49 @@ +{ + "index_patterns": [ + "accountsesdthistory-*" + ], + "template": { + "settings": { + "index.lifecycle.name": "accountsesdthistory-policy", + "index.lifecycle.rollover_alias": "accountsesdthistory", + "number_of_replicas": 1, + "number_of_shards": 5 + }, + "mappings": { + "properties": { + "address": { + "type": "keyword" + }, + "balance": { + "type": "keyword" + }, + "identifier": { + "type": "text" + }, + "isSender": { + "type": "boolean" + }, + "isSmartContract": { + "type": "boolean" + }, + "shardID": { + "type": "long" + }, + "timestamp": { + "format": "epoch_second", + "type": "date" + }, + "timestampMs": { + "type": "date", + "format": "epoch_millis" + }, + "token": { + "type": "keyword" + }, + "tokenNonce": { + "type": "double" + } + } + } + } +} diff --git a/cmd/elasticindexer/config/indices/accountshistory.json b/cmd/elasticindexer/config/indices/accountshistory.json new file mode 100644 index 00000000..9b95a548 --- /dev/null +++ b/cmd/elasticindexer/config/indices/accountshistory.json @@ -0,0 +1,40 @@ +{ + "index_patterns": [ + "accountshistory-*" + ], + "template": { + "settings": { + "index.lifecycle.name": "accountshistory-policy", + "index.lifecycle.rollover_alias": "accountshistory", + "number_of_replicas": 1, + "number_of_shards": 5 + }, + "mappings": { + "properties": { + "address": { + "type": "keyword" + }, + "balance": { + "type": "keyword" + }, + "isSender": { + "type": "boolean" + }, + "isSmartContract": { + "type": "boolean" + }, + "shardID": { + "type": "long" + }, + "timestamp": { + "format": "epoch_second", + "type": "date" + }, + "timestampMs": { + "type": "date", + "format": "epoch_millis" + } + } + } + } +} diff --git a/cmd/elasticindexer/config/indices/blocks.json b/cmd/elasticindexer/config/indices/blocks.json new file mode 100644 index 00000000..913a5361 --- /dev/null +++ b/cmd/elasticindexer/config/indices/blocks.json @@ -0,0 +1,259 @@ +{ + "index_patterns": [ + "blocks-*" + ], + "template": { + "settings": { + "index.lifecycle.name": "blocks-policy", + "index.lifecycle.rollover_alias": "blocks", + "index.sort.field": [ + "timestamp", + "nonce" + ], + "index.sort.order": [ + "desc", + "desc" + ], + "number_of_replicas": 1, + "number_of_shards": 5 + }, + "mappings": { + "properties": { + "accumulatedFees": { + "index": "false", + "type": "keyword" + }, + "developerFees": { + "index": "false", + "type": "keyword" + }, + "epoch": { + "type": "long" + }, + "epochStartBlock": { + "type": "boolean" + }, + "epochStartInfo": { + "properties": { + "nodePrice": { + "index": "false", + "type": "keyword" + }, + "prevEpochStartHash": { + "index": "false", + "type": "keyword" + }, + "prevEpochStartRound": { + "index": "false", + "type": "long" + }, + "rewardsForProtocolSustainability": { + "index": "false", + "type": "keyword" + }, + "rewardsPerBlock": { + "index": "false", + "type": "keyword" + }, + "totalNewlyMinted": { + "index": "false", + "type": "keyword" + }, + "totalSupply": { + "index": "false", + "type": "keyword" + }, + "totalToDistribute": { + "index": "false", + "type": "keyword" + } + } + }, + "epochStartShardsData": { + "properties": { + "epoch": { + "index": "false", + "type": "long" + }, + "firstPendingMetaBlock": { + "index": "false", + "type": "keyword" + }, + "headerHash": { + "index": "false", + "type": "keyword" + }, + "lastFinishedMetaBlock": { + "index": "false", + "type": "keyword" + }, + "nonce": { + "index": "false", + "type": "long" + }, + "pendingMiniBlockHeaders": { + "properties": { + "hash": { + "index": "false", + "type": "keyword" + }, + "receiverShard": { + "index": "false", + "type": "long" + }, + "senderShard": { + "index": "false", + "type": "long" + }, + "timestamp": { + "format": "epoch_second", + "index": "false", + "type": "date" + }, + "timestampMs": { + "type": "date", + "format": "epoch_millis" + }, + "type": { + "index": "false", + "type": "keyword" + } + } + }, + "rootHash": { + "index": "false", + "type": "keyword" + }, + "round": { + "index": "false", + "type": "long" + }, + "scheduledRootHash": { + "index": "false", + "type": "keyword" + }, + "shardID": { + "index": "false", + "type": "long" + } + } + }, + "gasPenalized": { + "type": "double" + }, + "gasProvided": { + "type": "double" + }, + "gasRefunded": { + "type": "double" + }, + "maxGasLimit": { + "type": "double" + }, + "miniBlocksDetails": { + "properties": { + "firstProcessedTx": { + "index": "false", + "type": "long" + }, + "lastProcessedTx": { + "index": "false", + "type": "long" + }, + "mbIndex": { + "index": "false", + "type": "long" + } + } + }, + "miniBlocksHashes": { + "type": "keyword" + }, + "nonce": { + "type": "long" + }, + "notarizedBlocksHashes": { + "type": "keyword" + }, + "notarizedTxsCount": { + "index": "false", + "type": "long" + }, + "prevHash": { + "type": "keyword" + }, + "proposer": { + "type": "long" + }, + "pubKeyBitmap": { + "index": "false", + "type": "keyword" + }, + "round": { + "type": "long" + }, + "scheduledData": { + "properties": { + "accumulatedFees": { + "index": "false", + "type": "keyword" + }, + "developerFees": { + "index": "false", + "type": "keyword" + }, + "gasProvided": { + "index": "false", + "type": "double" + }, + "gasRefunded": { + "index": "false", + "type": "double" + }, + "penalized": { + "index": "false", + "type": "double" + }, + "rootHash": { + "index": "false", + "type": "keyword" + } + } + }, + "searchOrder": { + "type": "long" + }, + "shardId": { + "type": "long" + }, + "size": { + "index": "false", + "type": "long" + }, + "sizeTxs": { + "index": "false", + "type": "long" + }, + "stateRootHash": { + "type": "keyword" + }, + "timestamp": { + "format": "epoch_second", + "type": "date" + }, + "timestampMs": { + "type": "date", + "format": "epoch_millis" + }, + "txCount": { + "index": "false", + "type": "long" + }, + "validators": { + "index": "false", + "type": "long" + } + } + } + } +} diff --git a/cmd/elasticindexer/config/indices/delegators.json b/cmd/elasticindexer/config/indices/delegators.json new file mode 100644 index 00000000..d59bd3eb --- /dev/null +++ b/cmd/elasticindexer/config/indices/delegators.json @@ -0,0 +1,51 @@ +{ + "index_patterns": [ + "delegators-*" + ], + "template": { + "settings": { + "index.lifecycle.name": "delegators-policy", + "index.lifecycle.rollover_alias": "delegators", + "number_of_replicas": 1, + "number_of_shards": 5 + }, + "mappings": { + "properties": { + "activeStake": { + "type": "keyword" + }, + "activeStakeNum": { + "type": "double" + }, + "address": { + "type": "keyword" + }, + "contract": { + "type": "keyword" + }, + "unDelegateInfo": { + "properties": { + "id": { + "type": "keyword" + }, + "timestamp": { + "format": "epoch_second", + "type": "date" + }, + "timestampMs": { + "type": "date", + "index": "false", + "format": "epoch_millis" + }, + "value": { + "type": "keyword" + }, + "valueNum": { + "type": "double" + } + } + } + } + } + } +} diff --git a/cmd/elasticindexer/config/indices/epochinfo.json b/cmd/elasticindexer/config/indices/epochinfo.json new file mode 100644 index 00000000..ffed8fec --- /dev/null +++ b/cmd/elasticindexer/config/indices/epochinfo.json @@ -0,0 +1,23 @@ +{ + "index_patterns": [ + "epochinfo-*" + ], + "template": { + "settings": { + "index.lifecycle.name": "epochinfo-policy", + "index.lifecycle.rollover_alias": "epochinfo", + "number_of_replicas": 1, + "number_of_shards": 5 + }, + "mappings": { + "properties": { + "accumulatedFees": { + "type": "keyword" + }, + "developerFees": { + "type": "keyword" + } + } + } + } +} diff --git a/cmd/elasticindexer/config/indices/esdts.json b/cmd/elasticindexer/config/indices/esdts.json new file mode 100644 index 00000000..6e5aca7a --- /dev/null +++ b/cmd/elasticindexer/config/indices/esdts.json @@ -0,0 +1,137 @@ +{ + "index_patterns": [ + "esdts-*" + ], + "template": { + "settings": { + "index.lifecycle.name": "esdts-policy", + "index.lifecycle.rollover_alias": "esdts", + "number_of_shards": 5, + "number_of_replicas": 1 + }, + "mappings": { + "properties": { + "currentOwner": { + "type": "keyword" + }, + "issuer": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "numDecimals": { + "type": "long" + }, + "ownersHistory": { + "type": "nested", + "properties": { + "address": { + "type": "keyword" + }, + "timestamp": { + "type": "date", + "index": false, + "format": "epoch_second" + }, + "timestampMs": { + "type": "date", + "index": false, + "format": "epoch_millis" + } + } + }, + "properties": { + "properties": { + "canAddSpecialRoles": { + "type": "boolean", + "index": false + }, + "canBurn": { + "type": "boolean", + "index": false + }, + "canChangeOwner": { + "type": "boolean", + "index": false + }, + "canCreateMultiShard": { + "type": "boolean", + "index": false + }, + "canFreeze": { + "type": "boolean", + "index": false + }, + "canMint": { + "type": "boolean", + "index": false + }, + "canPause": { + "type": "boolean", + "index": false + }, + "canTransferNFTCreateRole": { + "type": "boolean", + "index": false + }, + "canUpgrade": { + "type": "boolean", + "index": false + }, + "canWipe": { + "type": "boolean", + "index": false + } + } + }, + "roles": { + "type": "nested", + "properties": { + "ESDTRoleLocalBurn": { + "type": "keyword" + }, + "ESDTRoleLocalMint": { + "type": "keyword" + }, + "ESDTRoleNFTAddQuantity": { + "type": "keyword" + }, + "ESDTRoleNFTAddURI": { + "type": "keyword" + }, + "ESDTRoleNFTBurn": { + "type": "keyword" + }, + "ESDTRoleNFTCreate": { + "type": "keyword" + }, + "ESDTRoleNFTUpdateAttributes": { + "type": "keyword" + }, + "ESDTTransferRole": { + "type": "keyword" + } + } + }, + "ticker": { + "type": "keyword" + }, + "timestamp": { + "type": "date", + "format": "epoch_second" + }, + "timestampMs": { + "type": "date", + "format": "epoch_millis" + }, + "token": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } + } + } +} diff --git a/cmd/elasticindexer/config/indices/events.json b/cmd/elasticindexer/config/indices/events.json new file mode 100644 index 00000000..5a754553 --- /dev/null +++ b/cmd/elasticindexer/config/indices/events.json @@ -0,0 +1,58 @@ +{ + "index_patterns": [ + "events-*" + ], + "template": { + "settings": { + "number_of_replicas": 1, + "number_of_shards": 5, + "index.lifecycle.name": "events-policy", + "index.lifecycle.rollover_alias": "events" + }, + "mappings": { + "properties": { + "txHash": { + "type":"keyword" + }, + "originalTxHash": { + "type":"keyword" + }, + "logAddress": { + "type": "keyword" + }, + "address": { + "type": "keyword" + }, + "identifier": { + "type": "keyword" + }, + "data": { + "type": "text" + }, + "additionalData": { + "type": "text" + }, + "topics": { + "type": "text" + }, + "order": { + "type": "long" + }, + "txOrder": { + "type": "long" + }, + "shardID": { + "type": "long" + }, + "timestamp": { + "format": "epoch_second", + "type": "date" + }, + "timestampMs": { + "type": "date", + "format": "epoch_millis" + } + } + } + } +} diff --git a/cmd/elasticindexer/config/indices/executionresults.json b/cmd/elasticindexer/config/indices/executionresults.json new file mode 100644 index 00000000..e88c88e6 --- /dev/null +++ b/cmd/elasticindexer/config/indices/executionresults.json @@ -0,0 +1,81 @@ +{ + "index_patterns": [ + "executionresults-*" + ], + "template": { + "settings": { + "number_of_shards": 5, + "number_of_replicas": 1, + "index.lifecycle.name": "executionresults-policy", + "index.lifecycle.rollover_alias": "executionresults", + "index": { + "sort.field": [ + "timestampMs", + "nonce" + ], + "sort.order": [ + "desc", + "desc" + ] + } + }, + "mappings": { + "properties": { + "miniBlocksDetails": { + "properties": { + "firstProcessedTx": { + "index": "false", + "type": "long" + }, + "lastProcessedTx": { + "index": "false", + "type": "long" + }, + "mbIndex": { + "index": "false", + "type": "long" + } + } + }, + "miniBlocksHashes": { + "type": "keyword" + }, + "nonce": { + "type": "double" + }, + "round": { + "type": "double" + }, + "rootHash": { + "index": "false", + "type": "keyword" + }, + "notarizedInBlockHash": { + "type": "keyword" + }, + "epoch": { + "type": "long" + }, + "gasUsed": { + "type": "double" + }, + "txCount": { + "index": "false", + "type": "long" + }, + "accumulatedFees": { + "index": "false", + "type": "keyword" + }, + "developerFees": { + "index": "false", + "type": "keyword" + }, + "timestampMs": { + "type": "date", + "format": "epoch_millis" + } + } + } + } +} diff --git a/cmd/elasticindexer/config/indices/logs.json b/cmd/elasticindexer/config/indices/logs.json new file mode 100644 index 00000000..ecaa98ab --- /dev/null +++ b/cmd/elasticindexer/config/indices/logs.json @@ -0,0 +1,49 @@ +{ + "index_patterns": [ + "logs-*" + ], + "template": { + "settings": { + "number_of_replicas": 1, + "number_of_shards": 5, + "index.lifecycle.name": "logs-policy", + "index.lifecycle.rollover_alias": "logs" + }, + "mappings": { + "properties": { + "address": { + "type": "keyword" + }, + "events": { + "properties": { + "address": { + "type": "keyword" + }, + "data": { + "index": "false", + "type": "text" + }, + "identifier": { + "type": "keyword" + }, + "topics": { + "type": "text" + } + }, + "type": "nested" + }, + "originalTxHash": { + "type": "keyword" + }, + "timestamp": { + "format": "epoch_second", + "type": "date" + }, + "timestampMs": { + "type": "date", + "format": "epoch_millis" + } + } + } + } +} diff --git a/cmd/elasticindexer/config/indices/miniblocks.json b/cmd/elasticindexer/config/indices/miniblocks.json new file mode 100644 index 00000000..2c6e14a0 --- /dev/null +++ b/cmd/elasticindexer/config/indices/miniblocks.json @@ -0,0 +1,50 @@ +{ + "index_patterns": [ + "miniblocks-*" + ], + "template": { + "settings": { + "index.lifecycle.name": "miniblocks-policy", + "index.lifecycle.rollover_alias": "miniblocks", + "number_of_replicas": 1, + "number_of_shards": 5 + }, + "mappings": { + "properties": { + "procTypeD": { + "type": "keyword" + }, + "procTypeS": { + "type": "keyword" + }, + "receiverBlockHash": { + "type": "keyword" + }, + "receiverShard": { + "type": "long" + }, + "reserved": { + "index": "false", + "type": "keyword" + }, + "senderBlockHash": { + "type": "keyword" + }, + "senderShard": { + "type": "long" + }, + "timestamp": { + "format": "epoch_second", + "type": "date" + }, + "timestampMs": { + "type": "date", + "format": "epoch_millis" + }, + "type": { + "type": "keyword" + } + } + } + } +} diff --git a/cmd/elasticindexer/config/indices/operations.json b/cmd/elasticindexer/config/indices/operations.json new file mode 100644 index 00000000..10d90aa3 --- /dev/null +++ b/cmd/elasticindexer/config/indices/operations.json @@ -0,0 +1,165 @@ +{ + "index_patterns": [ + "operations-*" + ], + "template": { + "settings": { + "index.sort.field": [ + "timestamp", + "nonce" + ], + "index.sort.order": [ + "desc", + "desc" + ], + "index.lifecycle.name": "operations-policy", + "index.lifecycle.rollover_alias": "operations", + "number_of_replicas": 1, + "number_of_shards": 5 + }, + "mappings": { + "properties": { + "callType": { + "index": "false", + "type": "keyword" + }, + "canBeIgnored": { + "type": "boolean" + }, + "code": { + "index": "false", + "type": "text" + }, + "data": { + "type": "text" + }, + "esdtValues": { + "type": "keyword" + }, + "fee": { + "index": "false", + "type": "keyword" + }, + "function": { + "type": "keyword" + }, + "gasLimit": { + "index": "false", + "type": "double" + }, + "gasPrice": { + "index": "false", + "type": "double" + }, + "gasUsed": { + "index": "false", + "type": "double" + }, + "hasOperations": { + "type": "boolean" + }, + "hasScResults": { + "type": "boolean" + }, + "initialPaidFee": { + "index": "false", + "type": "keyword" + }, + "isRelayed": { + "type": "boolean" + }, + "isScCall": { + "type": "boolean" + }, + "miniBlockHash": { + "type": "keyword" + }, + "nonce": { + "type": "long" + }, + "operation": { + "type": "keyword" + }, + "originalSender": { + "type": "keyword" + }, + "originalTxHash": { + "type": "keyword" + }, + "prevTxHash": { + "type": "keyword" + }, + "receiver": { + "type": "keyword" + }, + "receiverShard": { + "type": "long" + }, + "receivers": { + "type": "keyword" + }, + "receiversShardIDs": { + "type": "long" + }, + "relayedValue": { + "index": "false", + "type": "keyword" + }, + "relayerAddr": { + "type": "keyword" + }, + "returnMessage": { + "type": "text" + }, + "round": { + "type": "long" + }, + "searchOrder": { + "type": "long" + }, + "sender": { + "type": "keyword" + }, + "senderShard": { + "type": "long" + }, + "senderUserName": { + "type": "keyword" + }, + "signature": { + "index": "false", + "type": "keyword" + }, + "status": { + "type": "keyword" + }, + "timestamp": { + "format": "epoch_second", + "type": "date" + }, + "timestampMs": { + "type": "date", + "format": "epoch_millis" + }, + "tokens": { + "type": "text" + }, + "type": { + "type": "keyword" + }, + "value": { + "type": "keyword" + }, + "version": { + "type": "long" + }, + "valueNum": { + "type": "double" + }, + "esdtValuesNum": { + "type": "double" + } + } + } + } +} diff --git a/cmd/elasticindexer/config/indices/rating.json b/cmd/elasticindexer/config/indices/rating.json new file mode 100644 index 00000000..e5db13c7 --- /dev/null +++ b/cmd/elasticindexer/config/indices/rating.json @@ -0,0 +1,20 @@ +{ + "index_patterns": [ + "rating-*" + ], + "template": { + "settings": { + "index.lifecycle.name": "rating-policy", + "index.lifecycle.rollover_alias": "rating", + "number_of_replicas": 1, + "number_of_shards": 5 + }, + "mappings": { + "properties": { + "rating": { + "type": "double" + } + } + } + } +} diff --git a/cmd/elasticindexer/config/indices/receipts.json b/cmd/elasticindexer/config/indices/receipts.json new file mode 100644 index 00000000..4e48b69d --- /dev/null +++ b/cmd/elasticindexer/config/indices/receipts.json @@ -0,0 +1,44 @@ +{ + "index_patterns": [ + "receipts-*" + ], + "template": { + "settings": { + "index.lifecycle.name": "receipts-policy", + "index.lifecycle.rollover_alias": "receipts", + "index.sort.field": [ + "timestamp" + ], + "index.sort.order": [ + "desc" + ], + "number_of_replicas": 1, + "number_of_shards": 5 + }, + "mappings": { + "properties": { + "data": { + "type": "keyword" + }, + "sender": { + "type": "keyword" + }, + "timestamp": { + "format": "epoch_second", + "type": "date" + }, + "timestampMs": { + "type": "date", + "format": "epoch_millis" + }, + "txHash": { + "type": "keyword" + }, + "value": { + "index": "false", + "type": "keyword" + } + } + } + } +} diff --git a/cmd/elasticindexer/config/indices/rounds.json b/cmd/elasticindexer/config/indices/rounds.json new file mode 100644 index 00000000..4431792e --- /dev/null +++ b/cmd/elasticindexer/config/indices/rounds.json @@ -0,0 +1,46 @@ +{ + "index_patterns": [ + "rounds-*" + ], + "template": { + "settings": { + "index.sort.field": [ + "timestamp" + ], + "index.sort.order": [ + "desc" + ], + "index.lifecycle.name": "rounds-policy", + "index.lifecycle.rollover_alias": "rounds", + "number_of_replicas": 1, + "number_of_shards": 5 + }, + "mappings": { + "properties": { + "blockWasProposed": { + "type": "boolean" + }, + "epoch": { + "type": "long" + }, + "round": { + "type": "long" + }, + "shardId": { + "type": "long" + }, + "signersIndexes": { + "type": "long" + }, + "timestamp": { + "format": "epoch_second", + "type": "date" + }, + "timestampMs": { + "type": "date", + "format": "epoch_millis" + } + } + } + } +} diff --git a/cmd/elasticindexer/config/indices/scdeploys.json b/cmd/elasticindexer/config/indices/scdeploys.json new file mode 100644 index 00000000..f0c192b8 --- /dev/null +++ b/cmd/elasticindexer/config/indices/scdeploys.json @@ -0,0 +1,50 @@ +{ + "index_patterns": [ + "scdeploys-*" + ], + "template": { + "settings": { + "index.lifecycle.name": "scdeploys-policy", + "index.lifecycle.rollover_alias": "scdeploys", + "number_of_replicas": 1, + "number_of_shards": 5 + }, + "mappings": { + "properties": { + "deployTxHash": { + "type": "keyword" + }, + "deployer": { + "type": "keyword" + }, + "timestamp": { + "format": "epoch_second", + "type": "date" + }, + "timestampMs": { + "type": "date", + "format": "epoch_millis" + }, + "upgrades": { + "properties": { + "timestamp": { + "format": "epoch_second", + "type": "date" + }, + "timestampMs": { + "type": "date", + "format": "epoch_millis" + }, + "upgradeTxHash": { + "type": "keyword" + }, + "upgrader": { + "type": "keyword" + } + }, + "type": "nested" + } + } + } + } +} diff --git a/cmd/elasticindexer/config/indices/scresults.json b/cmd/elasticindexer/config/indices/scresults.json new file mode 100644 index 00000000..338cf838 --- /dev/null +++ b/cmd/elasticindexer/config/indices/scresults.json @@ -0,0 +1,116 @@ +{ + "index_patterns": [ + "scresults-*" + ], + "template": { + "settings": { + "index.sort.field": [ + "timestamp" + ], + "index.sort.order": [ + "desc" + ], + "index.lifecycle.name": "scresults-policy", + "index.lifecycle.rollover_alias": "scresults", + "number_of_replicas": 1, + "number_of_shards": 5 + }, + "mappings": { + "properties": { + "callType": { + "type": "keyword" + }, + "code": { + "index": "false", + "type": "text" + }, + "data": { + "type": "text" + }, + "esdtValues": { + "type": "keyword" + }, + "function": { + "type": "keyword" + }, + "gasLimit": { + "index": "false", + "type": "double" + }, + "gasPrice": { + "index": "false", + "type": "double" + }, + "hasOperations": { + "type": "boolean" + }, + "miniBlockHash": { + "type": "keyword" + }, + "nonce": { + "type": "long" + }, + "operation": { + "type": "keyword" + }, + "originalSender": { + "type": "keyword" + }, + "originalTxHash": { + "type": "keyword" + }, + "prevTxHash": { + "type": "keyword" + }, + "receiver": { + "type": "keyword" + }, + "receiverShard": { + "type": "long" + }, + "receivers": { + "type": "keyword" + }, + "receiversShardIDs": { + "type": "long" + }, + "relayedValue": { + "index": "false", + "type": "keyword" + }, + "relayerAddr": { + "type": "keyword" + }, + "returnMessage": { + "type": "text" + }, + "sender": { + "type": "keyword" + }, + "senderShard": { + "type": "long" + }, + "timestamp": { + "format": "epoch_second", + "type": "date" + }, + "timestampMs": { + "type": "date", + "format": "epoch_millis" + }, + "tokens": { + "type": "text" + }, + "value": { + "type": "keyword" + }, + "valueNum": { + "type": "double" + }, + "esdtValuesNum": { + "type": "double" + } + } + } + } +} diff --git a/cmd/elasticindexer/config/indices/tags.json b/cmd/elasticindexer/config/indices/tags.json new file mode 100644 index 00000000..5d4cc77c --- /dev/null +++ b/cmd/elasticindexer/config/indices/tags.json @@ -0,0 +1,23 @@ +{ + "index_patterns": [ + "tags-*" + ], + "template": { + "settings": { + "index.lifecycle.name": "tags-policy", + "index.lifecycle.rollover_alias": "tags", + "number_of_replicas": 1, + "number_of_shards": 5 + }, + "mappings": { + "properties": { + "count": { + "type": "long" + }, + "tag": { + "type": "keyword" + } + } + } + } +} diff --git a/cmd/elasticindexer/config/indices/tokens.json b/cmd/elasticindexer/config/indices/tokens.json new file mode 100644 index 00000000..77f391cf --- /dev/null +++ b/cmd/elasticindexer/config/indices/tokens.json @@ -0,0 +1,265 @@ +{ + "index_patterns": [ + "tokens-*" + ], + "template": { + "settings": { + "index.lifecycle.name": "tokens-policy", + "index.lifecycle.rollover_alias": "tokens", + "number_of_replicas": 1, + "number_of_shards": 5 + }, + "mappings": { + "properties": { + "currentOwner": { + "type": "keyword" + }, + "data": { + "properties": { + "attributes": { + "index": "false", + "type": "keyword" + }, + "creator": { + "type": "keyword" + }, + "hash": { + "index": "false", + "type": "keyword" + }, + "metadata": { + "index": "false", + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "nonEmptyURIs": { + "type": "boolean" + }, + "royalties": { + "type": "long" + }, + "tags": { + "type": "keyword" + }, + "uris": { + "type": "keyword" + }, + "whiteListedStorage": { + "type": "boolean" + } + }, + "type": "nested" + }, + "identifier": { + "type": "text" + }, + "issuer": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "nonce": { + "type": "double" + }, + "numDecimals": { + "type": "long" + }, + "ownersHistory": { + "properties": { + "address": { + "type": "keyword" + }, + "timestamp": { + "format": "epoch_second", + "index": "false", + "type": "date" + }, + "timestampMs": { + "type": "date", + "index": "false", + "format": "epoch_millis" + } + }, + "type": "nested" + }, + "properties": { + "properties": { + "canAddSpecialRoles": { + "index": "false", + "type": "boolean" + }, + "canBurn": { + "index": "false", + "type": "boolean" + }, + "canChangeOwner": { + "index": "false", + "type": "boolean" + }, + "canCreateMultiShard": { + "index": "false", + "type": "boolean" + }, + "canFreeze": { + "index": "false", + "type": "boolean" + }, + "canMint": { + "index": "false", + "type": "boolean" + }, + "canPause": { + "index": "false", + "type": "boolean" + }, + "canTransferNFTCreateRole": { + "index": "false", + "type": "boolean" + }, + "canUpgrade": { + "index": "false", + "type": "boolean" + }, + "canWipe": { + "index": "false", + "type": "boolean" + } + } + }, + "roles": { + "properties": { + "ESDTRoleLocalBurn": { + "type": "keyword" + }, + "ESDTRoleLocalMint": { + "type": "keyword" + }, + "ESDTRoleNFTAddQuantity": { + "type": "keyword" + }, + "ESDTRoleNFTAddURI": { + "type": "keyword" + }, + "ESDTRoleNFTBurn": { + "type": "keyword" + }, + "ESDTRoleNFTCreate": { + "type": "keyword" + }, + "ESDTRoleNFTUpdateAttributes": { + "type": "keyword" + }, + "ESDTTransferRole": { + "type": "keyword" + } + }, + "type": "nested" + }, + "ticker": { + "type": "keyword" + }, + "timestamp": { + "format": "epoch_second", + "type": "date" + }, + "timestampMs": { + "type": "date", + "format": "epoch_millis" + }, + "token": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "api_isVerified": { + "type": "boolean" + }, + "api_nftCount": { + "type": "integer" + }, + "api_holderCount": { + "type": "integer" + }, + "nft_traitValues": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "nft_scamInfoType": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "nft_scamInfoDescription": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "nft_hasRarity": { + "type": "boolean" + }, + "nft_nsfw_mark": { + "type": "float" + }, + "nft_rank_custom": { + "type": "long" + }, + "nft_custom_ranks_hash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "nft_score_openRarity": { + "type": "float" + }, + "nft_rank_openRarity": { + "type": "long" + }, + "nft_score_jaccardDistances": { + "type": "float" + }, + "nft_rank_jaccardDistances": { + "type": "long" + }, + "nft_score_trait": { + "type": "float" + }, + "nft_rank_trait": { + "type": "long" + }, + "nft_score_statistical": { + "type": "float" + }, + "nft_rank_statistical": { + "type": "long" + }, + "nft_hasRarities": { + "type": "boolean" + }, + "nft_hasTraitSummary": { + "type": "boolean" + } + } + } + } +} diff --git a/cmd/elasticindexer/config/indices/transactions.json b/cmd/elasticindexer/config/indices/transactions.json new file mode 100644 index 00000000..d18ebfe8 --- /dev/null +++ b/cmd/elasticindexer/config/indices/transactions.json @@ -0,0 +1,132 @@ +{ + "index_patterns": [ + "transactions-*" + ], + "template": { + "settings": { + "index.sort.field": [ + "timestamp", + "nonce" + ], + "index.sort.order": [ + "desc", + "desc" + ], + "index.lifecycle.name": "transactions-policy", + "index.lifecycle.rollover_alias": "transactions", + "number_of_replicas": 1, + "number_of_shards": 5 + }, + "mappings": { + "properties": { + "data": { + "type": "text" + }, + "esdtValues": { + "type": "keyword" + }, + "fee": { + "index": "false", + "type": "keyword" + }, + "function": { + "type": "keyword" + }, + "gasLimit": { + "index": "false", + "type": "double" + }, + "gasPrice": { + "index": "false", + "type": "double" + }, + "gasUsed": { + "index": "false", + "type": "double" + }, + "hasOperations": { + "type": "boolean" + }, + "hasScResults": { + "type": "boolean" + }, + "initialPaidFee": { + "index": "false", + "type": "keyword" + }, + "isRelayed": { + "type": "boolean" + }, + "isScCall": { + "type": "boolean" + }, + "miniBlockHash": { + "type": "keyword" + }, + "nonce": { + "type": "long" + }, + "operation": { + "type": "keyword" + }, + "receiver": { + "type": "keyword" + }, + "receiverShard": { + "type": "long" + }, + "receivers": { + "type": "keyword" + }, + "receiversShardIDs": { + "type": "long" + }, + "round": { + "type": "long" + }, + "searchOrder": { + "type": "long" + }, + "sender": { + "type": "keyword" + }, + "senderShard": { + "type": "long" + }, + "senderUserName": { + "type": "keyword" + }, + "signature": { + "index": "false", + "type": "keyword" + }, + "status": { + "type": "keyword" + }, + "timestamp": { + "format": "epoch_second", + "type": "date" + }, + "timestampMs": { + "type": "date", + "format": "epoch_millis" + }, + "tokens": { + "type": "text" + }, + "value": { + "type": "keyword" + }, + "version": { + "type": "long" + }, + "valueNum": { + "type": "double" + }, + "esdtValuesNum": { + "type": "double" + } + } + } + } +} diff --git a/cmd/elasticindexer/config/indices/validators.json b/cmd/elasticindexer/config/indices/validators.json new file mode 100644 index 00000000..93f3650a --- /dev/null +++ b/cmd/elasticindexer/config/indices/validators.json @@ -0,0 +1,20 @@ +{ + "index_patterns": [ + "validators-*" + ], + "template": { + "settings": { + "index.lifecycle.name": "validators-policy", + "index.lifecycle.rollover_alias": "validators", + "number_of_replicas": 1, + "number_of_shards": 5 + }, + "mappings": { + "properties": { + "publicKeys": { + "type": "keyword" + } + } + } + } +} diff --git a/cmd/elasticindexer/config/indices/values.json b/cmd/elasticindexer/config/indices/values.json new file mode 100644 index 00000000..9b42c7b8 --- /dev/null +++ b/cmd/elasticindexer/config/indices/values.json @@ -0,0 +1,21 @@ +{ + "index_patterns": [ + "values-*" + ], + "template": { + "settings": { + "number_of_shards": 1, + "number_of_replicas": 0 + }, + "mappings": { + "properties": { + "key": { + "type": "keyword" + }, + "value": { + "type": "keyword" + } + } + } + } +} diff --git a/cmd/elasticindexer/config/policies/accounts.json b/cmd/elasticindexer/config/policies/accounts.json new file mode 100644 index 00000000..c7473979 --- /dev/null +++ b/cmd/elasticindexer/config/policies/accounts.json @@ -0,0 +1,15 @@ +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "50GB", + "max_docs": 100000000 + } + } + } + } + } +} diff --git a/cmd/elasticindexer/config/policies/accountsesdt.json b/cmd/elasticindexer/config/policies/accountsesdt.json new file mode 100644 index 00000000..c7473979 --- /dev/null +++ b/cmd/elasticindexer/config/policies/accountsesdt.json @@ -0,0 +1,15 @@ +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "50GB", + "max_docs": 100000000 + } + } + } + } + } +} diff --git a/cmd/elasticindexer/config/policies/accountsesdthistory.json b/cmd/elasticindexer/config/policies/accountsesdthistory.json new file mode 100644 index 00000000..c7473979 --- /dev/null +++ b/cmd/elasticindexer/config/policies/accountsesdthistory.json @@ -0,0 +1,15 @@ +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "50GB", + "max_docs": 100000000 + } + } + } + } + } +} diff --git a/cmd/elasticindexer/config/policies/accountshistory.json b/cmd/elasticindexer/config/policies/accountshistory.json new file mode 100644 index 00000000..c7473979 --- /dev/null +++ b/cmd/elasticindexer/config/policies/accountshistory.json @@ -0,0 +1,15 @@ +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "50GB", + "max_docs": 100000000 + } + } + } + } + } +} diff --git a/cmd/elasticindexer/config/policies/blocks.json b/cmd/elasticindexer/config/policies/blocks.json new file mode 100644 index 00000000..4ec5b220 --- /dev/null +++ b/cmd/elasticindexer/config/policies/blocks.json @@ -0,0 +1,15 @@ +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "50GB", + "max_docs": 50000000 + } + } + } + } + } +} diff --git a/cmd/elasticindexer/config/policies/delegators.json b/cmd/elasticindexer/config/policies/delegators.json new file mode 100644 index 00000000..c7473979 --- /dev/null +++ b/cmd/elasticindexer/config/policies/delegators.json @@ -0,0 +1,15 @@ +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "50GB", + "max_docs": 100000000 + } + } + } + } + } +} diff --git a/cmd/elasticindexer/config/policies/epochinfo.json b/cmd/elasticindexer/config/policies/epochinfo.json new file mode 100644 index 00000000..c7473979 --- /dev/null +++ b/cmd/elasticindexer/config/policies/epochinfo.json @@ -0,0 +1,15 @@ +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "50GB", + "max_docs": 100000000 + } + } + } + } + } +} diff --git a/cmd/elasticindexer/config/policies/esdts.json b/cmd/elasticindexer/config/policies/esdts.json new file mode 100644 index 00000000..c7473979 --- /dev/null +++ b/cmd/elasticindexer/config/policies/esdts.json @@ -0,0 +1,15 @@ +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "50GB", + "max_docs": 100000000 + } + } + } + } + } +} diff --git a/cmd/elasticindexer/config/policies/events.json b/cmd/elasticindexer/config/policies/events.json new file mode 100644 index 00000000..c7473979 --- /dev/null +++ b/cmd/elasticindexer/config/policies/events.json @@ -0,0 +1,15 @@ +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "50GB", + "max_docs": 100000000 + } + } + } + } + } +} diff --git a/cmd/elasticindexer/config/policies/executionresults.json b/cmd/elasticindexer/config/policies/executionresults.json new file mode 100644 index 00000000..4ec5b220 --- /dev/null +++ b/cmd/elasticindexer/config/policies/executionresults.json @@ -0,0 +1,15 @@ +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "50GB", + "max_docs": 50000000 + } + } + } + } + } +} diff --git a/cmd/elasticindexer/config/policies/logs.json b/cmd/elasticindexer/config/policies/logs.json new file mode 100644 index 00000000..c7473979 --- /dev/null +++ b/cmd/elasticindexer/config/policies/logs.json @@ -0,0 +1,15 @@ +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "50GB", + "max_docs": 100000000 + } + } + } + } + } +} diff --git a/cmd/elasticindexer/config/policies/miniblocks.json b/cmd/elasticindexer/config/policies/miniblocks.json new file mode 100644 index 00000000..4ec5b220 --- /dev/null +++ b/cmd/elasticindexer/config/policies/miniblocks.json @@ -0,0 +1,15 @@ +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "50GB", + "max_docs": 50000000 + } + } + } + } + } +} diff --git a/cmd/elasticindexer/config/policies/operations.json b/cmd/elasticindexer/config/policies/operations.json new file mode 100644 index 00000000..c7473979 --- /dev/null +++ b/cmd/elasticindexer/config/policies/operations.json @@ -0,0 +1,15 @@ +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "50GB", + "max_docs": 100000000 + } + } + } + } + } +} diff --git a/cmd/elasticindexer/config/policies/rating.json b/cmd/elasticindexer/config/policies/rating.json new file mode 100644 index 00000000..c7473979 --- /dev/null +++ b/cmd/elasticindexer/config/policies/rating.json @@ -0,0 +1,15 @@ +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "50GB", + "max_docs": 100000000 + } + } + } + } + } +} diff --git a/cmd/elasticindexer/config/policies/receipts.json b/cmd/elasticindexer/config/policies/receipts.json new file mode 100644 index 00000000..c7473979 --- /dev/null +++ b/cmd/elasticindexer/config/policies/receipts.json @@ -0,0 +1,15 @@ +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "50GB", + "max_docs": 100000000 + } + } + } + } + } +} diff --git a/cmd/elasticindexer/config/policies/rounds.json b/cmd/elasticindexer/config/policies/rounds.json new file mode 100644 index 00000000..4ec5b220 --- /dev/null +++ b/cmd/elasticindexer/config/policies/rounds.json @@ -0,0 +1,15 @@ +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "50GB", + "max_docs": 50000000 + } + } + } + } + } +} diff --git a/cmd/elasticindexer/config/policies/scdeploys.json b/cmd/elasticindexer/config/policies/scdeploys.json new file mode 100644 index 00000000..c7473979 --- /dev/null +++ b/cmd/elasticindexer/config/policies/scdeploys.json @@ -0,0 +1,15 @@ +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "50GB", + "max_docs": 100000000 + } + } + } + } + } +} diff --git a/cmd/elasticindexer/config/policies/scresults.json b/cmd/elasticindexer/config/policies/scresults.json new file mode 100644 index 00000000..c7473979 --- /dev/null +++ b/cmd/elasticindexer/config/policies/scresults.json @@ -0,0 +1,15 @@ +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "50GB", + "max_docs": 100000000 + } + } + } + } + } +} diff --git a/cmd/elasticindexer/config/policies/tags.json b/cmd/elasticindexer/config/policies/tags.json new file mode 100644 index 00000000..c7473979 --- /dev/null +++ b/cmd/elasticindexer/config/policies/tags.json @@ -0,0 +1,15 @@ +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "50GB", + "max_docs": 100000000 + } + } + } + } + } +} diff --git a/cmd/elasticindexer/config/policies/tokens.json b/cmd/elasticindexer/config/policies/tokens.json new file mode 100644 index 00000000..c7473979 --- /dev/null +++ b/cmd/elasticindexer/config/policies/tokens.json @@ -0,0 +1,15 @@ +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "50GB", + "max_docs": 100000000 + } + } + } + } + } +} diff --git a/cmd/elasticindexer/config/policies/transactions.json b/cmd/elasticindexer/config/policies/transactions.json new file mode 100644 index 00000000..c7473979 --- /dev/null +++ b/cmd/elasticindexer/config/policies/transactions.json @@ -0,0 +1,15 @@ +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "50GB", + "max_docs": 100000000 + } + } + } + } + } +} diff --git a/cmd/elasticindexer/config/policies/validators.json b/cmd/elasticindexer/config/policies/validators.json new file mode 100644 index 00000000..c7473979 --- /dev/null +++ b/cmd/elasticindexer/config/policies/validators.json @@ -0,0 +1,15 @@ +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "50GB", + "max_docs": 100000000 + } + } + } + } + } +} diff --git a/cmd/elasticindexer/flags.go b/cmd/elasticindexer/flags.go index 2f23cfd2..7b13a717 100644 --- a/cmd/elasticindexer/flags.go +++ b/cmd/elasticindexer/flags.go @@ -10,6 +10,13 @@ const ( ) var ( + // defines the path to the config folder + configPath = cli.StringFlag{ + Name: "config-path", + Usage: "The path to the config folder", + Value: "./config/", + } + configurationFile = cli.StringFlag{ Name: "config", Usage: "The main configuration file to load", diff --git a/cmd/elasticindexer/main.go b/cmd/elasticindexer/main.go index 3ab7e757..bb39e778 100644 --- a/cmd/elasticindexer/main.go +++ b/cmd/elasticindexer/main.go @@ -64,6 +64,7 @@ func main() { logLevel, logSaveFile, disableAnsiColor, + configPath, } app.Authors = []cli.Author{ { @@ -103,8 +104,9 @@ func startIndexer(ctx *cli.Context) error { return fmt.Errorf("%w while loading the enable epochs config file", err) } + configPathStr := ctx.GlobalString(configPath.Name) statusMetrics := metrics.NewStatusMetrics() - wsHost, err := factory.CreateWsIndexer(cfg, clusterCfg, epochsCfg, statusMetrics, ctx.App.Version) + wsHost, err := factory.CreateWsIndexer(cfg, clusterCfg, epochsCfg, statusMetrics, ctx.App.Version, configPathStr) if err != nil { return fmt.Errorf("%w while creating the indexer", err) } diff --git a/config/config.go b/config/config.go index 39bb6a7c..34926b09 100644 --- a/config/config.go +++ b/config/config.go @@ -3,7 +3,12 @@ package config // Config will hold the whole config file's data type Config struct { Config struct { - AvailableIndices []string `toml:"available-indices"` + AvailableIndices []string `toml:"available-indices"` + UseTemplatesFromFiles bool `toml:"use-templates-from-files"` + Policies struct { + Enabled bool `toml:"enabled"` + IndicesWithPolicy []string `toml:"indices-with-policy"` + } `toml:"policies"` AddressConverter struct { Length int `toml:"length"` Type string `toml:"type"` @@ -45,7 +50,6 @@ type ClusterConfig struct { AckTimeoutInSec uint32 `toml:"acknowledge-timeout-in-seconds"` } `toml:"web-socket"` ElasticCluster struct { - UseKibana bool `toml:"use-kibana"` URL string `toml:"url"` UserName string `toml:"username"` Password string `toml:"password"` diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index 4014d877..cf6ddad0 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -23,13 +23,14 @@ func CreateWsIndexer( epochsCfg config.EnableEpochsConfig, statusMetrics core.StatusMetricsHandler, version string, + configPathStr string, ) (wsindexer.WSClient, error) { wsMarshaller, err := factoryMarshaller.NewMarshalizer(clusterCfg.Config.WebSocket.DataMarshallerType) if err != nil { return nil, err } - dataIndexer, err := createDataIndexer(cfg, clusterCfg, epochsCfg, wsMarshaller, statusMetrics, version) + dataIndexer, err := createDataIndexer(cfg, clusterCfg, epochsCfg, wsMarshaller, statusMetrics, version, configPathStr) if err != nil { return nil, err } @@ -64,6 +65,7 @@ func createDataIndexer( wsMarshaller marshal.Marshalizer, statusMetrics core.StatusMetricsHandler, version string, + configPathStr string, ) (wsindexer.DataIndexer, error) { marshaller, err := factoryMarshaller.NewMarshalizer(cfg.Config.Marshaller.Type) if err != nil { @@ -82,14 +84,19 @@ func createDataIndexer( return nil, err } + indicesWithPolicies := make([]string, 0) + if cfg.Config.Policies.Enabled { + indicesWithPolicies = prepareIndices(cfg.Config.Policies.IndicesWithPolicy, clusterCfg.Config.DisabledIndices) + } + return factory.NewIndexer(factory.ArgsIndexerFactory{ - UseKibana: clusterCfg.Config.ElasticCluster.UseKibana, Denomination: cfg.Config.Economics.Denomination, BulkRequestMaxSize: clusterCfg.Config.ElasticCluster.BulkRequestMaxSizeInBytes, Url: clusterCfg.Config.ElasticCluster.URL, UserName: clusterCfg.Config.ElasticCluster.UserName, Password: clusterCfg.Config.ElasticCluster.Password, EnabledIndexes: prepareIndices(cfg.Config.AvailableIndices, clusterCfg.Config.DisabledIndices), + IndicesWithPolicy: indicesWithPolicies, Marshalizer: marshaller, Hasher: hasher, AddressPubkeyConverter: addressPubkeyConverter, @@ -99,6 +106,8 @@ func createDataIndexer( Version: version, EnableEpochsConfig: enableEpochsCfg, NumWritesInParallel: clusterCfg.Config.ElasticCluster.NumWritesInParallel, + UseTemplatesFromFiles: cfg.Config.UseTemplatesFromFiles, + ConfigPath: configPathStr, }) } diff --git a/mock/databaseWriterStub.go b/mock/databaseWriterStub.go index 1df608a8..b03e391c 100644 --- a/mock/databaseWriterStub.go +++ b/mock/databaseWriterStub.go @@ -86,6 +86,16 @@ func (dwm *DatabaseWriterStub) CheckAndCreatePolicy(_ string, _ *bytes.Buffer) e return nil } +// SetWriteIndexTrue - +func (dwm *DatabaseWriterStub) SetWriteIndexTrue(_ string, _ string) error { + return nil +} + +// PolicyExists - +func (dwm *DatabaseWriterStub) PolicyExists(_ string) bool { + return false +} + // IsInterfaceNil returns true if there is no value under the interface func (dwm *DatabaseWriterStub) IsInterfaceNil() bool { return dwm == nil diff --git a/process/elasticproc/elasticProcessor.go b/process/elasticproc/elasticProcessor.go index 14ec021b..d77e508f 100644 --- a/process/elasticproc/elasticProcessor.go +++ b/process/elasticproc/elasticProcessor.go @@ -120,22 +120,17 @@ func NewElasticProcessor(arguments *ArgElasticProcessor) (*elasticProcessor, err // TODO move all the index create part in a new component func (ei *elasticProcessor) init() error { - indexTemplates, _, err := ei.mappingsHandler.GetElasticTemplatesAndPolicies() + indexTemplates, indexPolices, err := ei.mappingsHandler.GetElasticTemplatesAndPolicies() if err != nil { return err } - err = ei.createIndexTemplates(indexTemplates) + err = ei.createIndices(indexTemplates) if err != nil { return err } - err = ei.createIndexes() - if err != nil { - return err - } - - err = ei.createAliases() + err = ei.createPolicies(indexPolices) if err != nil { return err } @@ -185,49 +180,50 @@ func (ei *elasticProcessor) indexVersion(version string) error { return ei.elasticClient.DoBulkRequest(context.Background(), buffSlice.Buffers()[0], "") } -func (ei *elasticProcessor) createIndexTemplates(indexTemplates map[string]*bytes.Buffer) error { - for _, index := range indexes { - indexTemplate := getTemplateByName(index, indexTemplates) - if indexTemplate != nil { - err := ei.elasticClient.CheckAndCreateTemplate(index, indexTemplate) - if err != nil { - return fmt.Errorf("index: %s, error: %w", index, err) - } +func (ei *elasticProcessor) createIndices(indexTemplateMap map[string]*bytes.Buffer) error { + for index, indexData := range indexTemplateMap { + err := ei.elasticClient.CheckAndCreateTemplate(index, indexData) + if err != nil { + return fmt.Errorf("elasticClient.CreateIndexWithMapping index: %s, error: %w", index, err) } - } - return nil -} -func (ei *elasticProcessor) createIndexes() error { + indexWithSuffix := fmt.Sprintf("%s-%s", index, elasticIndexer.IndexSuffix) + err = ei.elasticClient.CheckAndCreateIndex(indexWithSuffix) + if err != nil { + return fmt.Errorf("elasticClient.CheckAndCreateIndex index: %s, error: %w", index, err) + } - for _, index := range indexes { - indexName := fmt.Sprintf("%s-%s", index, elasticIndexer.IndexSuffix) - err := ei.elasticClient.CheckAndCreateIndex(indexName) + err = ei.elasticClient.CheckAndCreateAlias(index, indexWithSuffix) if err != nil { - return fmt.Errorf("index: %s, error: %w", index, err) + return fmt.Errorf("elasticClient.CheckAndCreateAlias index: %s, error: %w", index, err) } } + return nil } -func (ei *elasticProcessor) createAliases() error { - for _, index := range indexes { - indexName := fmt.Sprintf("%s-%s", index, elasticIndexer.IndexSuffix) - err := ei.elasticClient.CheckAndCreateAlias(index, indexName) +func (ei *elasticProcessor) createPolicies(indexPolicyMap map[string]*bytes.Buffer) error { + for index, policy := range indexPolicyMap { + policyName := fmt.Sprintf("%s-%s", index, "policy") + if ei.elasticClient.PolicyExists(policyName) { + continue + } + + indexWithSuffix := fmt.Sprintf("%s-%s", index, elasticIndexer.IndexSuffix) + err := ei.elasticClient.SetWriteIndexTrue(index, indexWithSuffix) if err != nil { - return err + return fmt.Errorf("elasticClient.SetWriteIndexTrue index: %s, error: %w", index, err) } - } + log.Info("elasticClient.SetWriteIndexTrue", "index", index) - return nil -} + err = ei.elasticClient.CheckAndCreatePolicy(policyName, policy) + if err != nil { + return fmt.Errorf("databaseClient.PutPolicy index: %s, error: %w", index, err) + } -func getTemplateByName(templateName string, templateList map[string]*bytes.Buffer) *bytes.Buffer { - if template, ok := templateList[templateName]; ok { - return template + log.Info("databaseClient.PutPolicy", "index", index) } - log.Debug("elasticProcessor.getTemplateByName", "could not find template", templateName) return nil } diff --git a/process/elasticproc/elasticProcessor_test.go b/process/elasticproc/elasticProcessor_test.go index 63def74a..3df67af8 100644 --- a/process/elasticproc/elasticProcessor_test.go +++ b/process/elasticproc/elasticProcessor_test.go @@ -91,7 +91,7 @@ func createMockElasticProcessorArgs() *ArgElasticProcessor { BlockProc: bp, LogsAndEventsProc: lp, OperationsProc: op, - MappingsHandler: templatesAndPolicies.NewTemplatesAndPolicyReader(), + MappingsHandler: templatesAndPolicies.NewTemplatesAndPolicyReader(false, "", nil, nil), NumWritesInParallel: 1, } } diff --git a/process/elasticproc/factory/elasticProcessorFactory.go b/process/elasticproc/factory/elasticProcessorFactory.go index 778e152f..0c75711f 100644 --- a/process/elasticproc/factory/elasticProcessorFactory.go +++ b/process/elasticproc/factory/elasticProcessorFactory.go @@ -27,18 +27,25 @@ type ArgElasticProcessorFactory struct { ValidatorPubkeyConverter core.PubkeyConverter DBClient elasticproc.DatabaseClientHandler EnabledIndexes []string + IndicesWithPolicy []string Version string Denomination int BulkRequestMaxSize int NumWritesInParallel int - UseKibana bool ImportDB bool EnableEpochsConfig config.EnableEpochsConfig + UseTemplatesFromFiles bool + ConfigPath string } // CreateElasticProcessor will create a new instance of ElasticProcessor func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.ElasticProcessor, error) { - templatesAndPoliciesReader := templatesAndPolicies.NewTemplatesAndPolicyReader() + templatesAndPoliciesReader := templatesAndPolicies.NewTemplatesAndPolicyReader( + arguments.UseTemplatesFromFiles, + arguments.ConfigPath, + arguments.EnabledIndexes, + arguments.IndicesWithPolicy, + ) enabledIndexesMap := make(map[string]struct{}) for _, index := range arguments.EnabledIndexes { @@ -116,7 +123,6 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.E LogsAndEventsProc: logsAndEventsProc, DBClient: arguments.DBClient, EnabledIndexes: enabledIndexesMap, - UseKibana: arguments.UseKibana, OperationsProc: operationsProc, ImportDB: arguments.ImportDB, Version: arguments.Version, diff --git a/process/elasticproc/factory/elasticProcessorFactory_test.go b/process/elasticproc/factory/elasticProcessorFactory_test.go index 81e5d174..8e43d3f9 100644 --- a/process/elasticproc/factory/elasticProcessorFactory_test.go +++ b/process/elasticproc/factory/elasticProcessorFactory_test.go @@ -17,7 +17,6 @@ func TestCreateElasticProcessor(t *testing.T) { DBClient: &mock.DatabaseWriterStub{}, EnabledIndexes: []string{"blocks"}, Denomination: 1, - UseKibana: false, } ep, err := CreateElasticProcessor(args) diff --git a/process/elasticproc/interface.go b/process/elasticproc/interface.go index 8861159f..6a52b61f 100644 --- a/process/elasticproc/interface.go +++ b/process/elasticproc/interface.go @@ -27,6 +27,8 @@ type DatabaseClientHandler interface { CheckAndCreateAlias(alias string, index string) error CheckAndCreateTemplate(templateName string, template *bytes.Buffer) error CheckAndCreatePolicy(policyName string, policy *bytes.Buffer) error + SetWriteIndexTrue(alias string, index string) error + PolicyExists(policy string) bool IsInterfaceNil() bool } diff --git a/process/elasticproc/templatesAndPolicies/reader.go b/process/elasticproc/templatesAndPolicies/reader.go index 93c1d5aa..939ca5f6 100644 --- a/process/elasticproc/templatesAndPolicies/reader.go +++ b/process/elasticproc/templatesAndPolicies/reader.go @@ -2,25 +2,52 @@ package templatesAndPolicies import ( "bytes" + "fmt" + "os" + "path" + "path/filepath" indexer "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/multiversx/mx-chain-es-indexer-go/templates" "github.com/multiversx/mx-chain-es-indexer-go/templates/indices" ) -type templatesAndPolicyReader struct{} +const ( + indicesFolder = "indices" + policiesFolder = "policies" +) + +type templatesAndPolicyReader struct { + useTemplatesFromFiles bool + configPath string + availableIndices []string + indicesWithPolicies []string +} // NewTemplatesAndPolicyReader will create a new instance of templatesAndPolicyReader -func NewTemplatesAndPolicyReader() *templatesAndPolicyReader { - return new(templatesAndPolicyReader) +func NewTemplatesAndPolicyReader( + useTemplatesFromFiles bool, + configPath string, + availableIndices []string, + indicesWithPolicies []string, +) *templatesAndPolicyReader { + return &templatesAndPolicyReader{ + useTemplatesFromFiles: useTemplatesFromFiles, + configPath: configPath, + availableIndices: availableIndices, + indicesWithPolicies: indicesWithPolicies, + } } // GetElasticTemplatesAndPolicies will return templates and policies func (tr *templatesAndPolicyReader) GetElasticTemplatesAndPolicies() (map[string]*bytes.Buffer, map[string]*bytes.Buffer, error) { + if tr.useTemplatesFromFiles { + return tr.getElasticTemplatesAndPoliciesFromJsonFiles() + } + indexPolicies := make(map[string]*bytes.Buffer) indexTemplates := make(map[string]*bytes.Buffer) - indexTemplates["opendistro"] = indices.OpenDistro.ToBuffer() indexTemplates[indexer.TransactionsIndex] = indices.Transactions.ToBuffer() indexTemplates[indexer.BlockIndex] = indices.Blocks.ToBuffer() indexTemplates[indexer.MiniblocksIndex] = indices.Miniblocks.ToBuffer() @@ -127,3 +154,52 @@ func (tr *templatesAndPolicyReader) GetTimestampMsMappings() ([]templates.ExtraM func (tr *templatesAndPolicyReader) GetExtraMappings() ([]templates.ExtraMapping, error) { return []templates.ExtraMapping{}, nil } + +func (tr *templatesAndPolicyReader) getElasticTemplatesAndPoliciesFromJsonFiles() (map[string]*bytes.Buffer, map[string]*bytes.Buffer, error) { + pathToMappings := path.Join(tr.configPath, indicesFolder) + indicesTemplateMap, err := tr.getElasticTemplatesFromJson(pathToMappings, tr.availableIndices) + if err != nil { + return nil, nil, fmt.Errorf("%w, cannot load templates", err) + } + + pathToPolicies := path.Join(tr.configPath, policiesFolder) + indicesPolicyMap, err := tr.getElasticTemplatesFromJson(pathToPolicies, tr.indicesWithPolicies) + if err != nil { + return nil, nil, fmt.Errorf("%w, cannot load templates", err) + } + + return indicesTemplateMap, indicesPolicyMap, nil +} + +func (tr *templatesAndPolicyReader) getElasticTemplatesFromJson(filePath string, indices []string) (map[string]*bytes.Buffer, error) { + indexTemplates := make(map[string]*bytes.Buffer) + var err error + + for _, index := range indices { + indexTemplates[index], err = getDataFromByIndex(filePath, index) + if err != nil { + return nil, err + } + } + + return indexTemplates, nil +} + +func getDataFromByIndex(path string, index string) (*bytes.Buffer, error) { + indexTemplate := &bytes.Buffer{} + + fileName := fmt.Sprintf("%s.json", index) + filePath := filepath.Join(path, fileName) + fileBytes, err := os.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("getDataFromByIndex: %w, path %s, error %s", err, filePath, err.Error()) + } + + indexTemplate.Grow(len(fileBytes)) + _, err = indexTemplate.Write(fileBytes) + if err != nil { + return nil, fmt.Errorf("getDataFromByIndex: %w, path %s, error %s", err, filePath, err.Error()) + } + + return indexTemplate, nil +} diff --git a/process/elasticproc/templatesAndPolicies/reader_test.go b/process/elasticproc/templatesAndPolicies/reader_test.go index 9c345615..d846b4ae 100644 --- a/process/elasticproc/templatesAndPolicies/reader_test.go +++ b/process/elasticproc/templatesAndPolicies/reader_test.go @@ -9,10 +9,10 @@ import ( func TestTemplatesAndPolicyReaderNoKibana_GetElasticTemplatesAndPolicies(t *testing.T) { t.Parallel() - reader := NewTemplatesAndPolicyReader() + reader := NewTemplatesAndPolicyReader(false, "", nil, nil) templates, policies, err := reader.GetElasticTemplatesAndPolicies() require.Nil(t, err) require.Len(t, policies, 0) - require.Len(t, templates, 24) + require.Len(t, templates, 23) } diff --git a/process/factory/indexerFactory.go b/process/factory/indexerFactory.go index eb422705..fd8d7721 100644 --- a/process/factory/indexerFactory.go +++ b/process/factory/indexerFactory.go @@ -29,7 +29,6 @@ var log = logger.GetOrCreate("indexer/factory") // new instances type ArgsIndexerFactory struct { Enabled bool - UseKibana bool ImportDB bool Denomination int BulkRequestMaxSize int @@ -40,6 +39,7 @@ type ArgsIndexerFactory struct { TemplatesPath string Version string EnabledIndexes []string + IndicesWithPolicy []string HeaderMarshaller marshal.Marshalizer Marshalizer marshal.Marshalizer Hasher hashing.Hasher @@ -47,6 +47,8 @@ type ArgsIndexerFactory struct { ValidatorPubkeyConverter core.PubkeyConverter StatusMetrics indexerCore.StatusMetricsHandler EnableEpochsConfig config.EnableEpochsConfig + UseTemplatesFromFiles bool + ConfigPath string } // NewIndexer will create a new instance of Indexer @@ -93,7 +95,6 @@ func createElasticProcessor(args ArgsIndexerFactory) (dataindexer.ElasticProcess Hasher: args.Hasher, AddressPubkeyConverter: args.AddressPubkeyConverter, ValidatorPubkeyConverter: args.ValidatorPubkeyConverter, - UseKibana: args.UseKibana, DBClient: databaseClient, Denomination: args.Denomination, EnabledIndexes: args.EnabledIndexes, @@ -101,6 +102,10 @@ func createElasticProcessor(args ArgsIndexerFactory) (dataindexer.ElasticProcess ImportDB: args.ImportDB, Version: args.Version, EnableEpochsConfig: args.EnableEpochsConfig, + UseTemplatesFromFiles: args.UseTemplatesFromFiles, + ConfigPath: args.ConfigPath, + IndicesWithPolicy: args.IndicesWithPolicy, + NumWritesInParallel: args.NumWritesInParallel, } return factory.CreateElasticProcessor(argsElasticProcFac) diff --git a/scripts/script.sh b/scripts/script.sh index 11a1bdb4..1f163442 100755 --- a/scripts/script.sh +++ b/scripts/script.sh @@ -4,7 +4,7 @@ PROMETHEUS_CONTAINER_NAME=prometheus_container GRAFANA_CONTAINER_NAME=grafana_container GRAFANA_VERSION=10.0.3 PROMETHEUS_VERSION=v2.46.0 -INDICES_LIST=("rating" "transactions" "blocks" "validators" "miniblocks" "rounds" "accounts" "accountshistory" "receipts" "scresults" "accountsesdt" "accountsesdthistory" "epochinfo" "scdeploys" "tokens" "tags" "logs" "delegators" "operations" "esdts" "values" "events") +INDICES_LIST=("rating" "transactions" "blocks" "validators" "miniblocks" "rounds" "accounts" "accountshistory" "receipts" "scresults" "accountsesdt" "accountsesdthistory" "epochinfo" "scdeploys" "tokens" "tags" "logs" "delegators" "operations" "esdts" "values" "events" "executionresults") start() { @@ -32,6 +32,7 @@ stop() { delete() { for str in ${INDICES_LIST[@]}; do curl -XDELETE http://localhost:9200/$str-000001 + curl -s -o /dev/null -w "%{http_code}" -X GET localhost:9200/_ilm/policy/$str-policy | grep -q 200 && curl -X DELETE localhost:9200/_ilm/policy/$str-policy echo done diff --git a/templates/indices/blocks.go b/templates/indices/blocks.go index 816ec6a5..b6be9e84 100644 --- a/templates/indices/blocks.go +++ b/templates/indices/blocks.go @@ -90,7 +90,7 @@ var Blocks = Object{ }, "nonce": Object{ "index": "false", - "type": "double", + "type": "long", }, "pendingMiniBlockHeaders": Object{ "properties": Object{ diff --git a/templates/indices/operations.go b/templates/indices/operations.go index c161a589..cef7c0f3 100644 --- a/templates/indices/operations.go +++ b/templates/indices/operations.go @@ -85,7 +85,7 @@ var Operations = Object{ "type": "keyword", }, "nonce": Object{ - "type": "double", + "type": "long", }, "operation": Object{ "type": "keyword", diff --git a/templates/indices/scResults.go b/templates/indices/scResults.go index 710e7816..2e511d71 100644 --- a/templates/indices/scResults.go +++ b/templates/indices/scResults.go @@ -54,7 +54,7 @@ var SCResults = Object{ "type": "keyword", }, "nonce": Object{ - "type": "double", + "type": "long", }, "operation": Object{ "type": "keyword", diff --git a/templates/indices/transactions.go b/templates/indices/transactions.go index f03d76df..bf504884 100644 --- a/templates/indices/transactions.go +++ b/templates/indices/transactions.go @@ -74,7 +74,7 @@ var Transactions = Object{ "type": "keyword", }, "nonce": Object{ - "type": "double", + "type": "long", }, "operation": Object{ "type": "keyword", diff --git a/tools/indices-creator/cmd/indices-creator/config/cluster.toml b/tools/indices-creator/cmd/indices-creator/config/cluster.toml deleted file mode 100644 index f087c18e..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/cluster.toml +++ /dev/null @@ -1,6 +0,0 @@ -[config] - url = "http://localhost:9200" - username = "" - password = "" - use-kibana = false - enabled-indices = ["rating", "transactions", "blocks", "validators", "miniblocks", "rounds", "accounts", "accountshistory", "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags", "logs", "delegators", "operations"] diff --git a/tools/indices-creator/cmd/indices-creator/config/noKibana/accounts.json b/tools/indices-creator/cmd/indices-creator/config/noKibana/accounts.json deleted file mode 100644 index 5a26ec51..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/noKibana/accounts.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "index_patterns": [ - "accounts-*" - ], - "mappings": { - "properties": { - "address": { - "type": "text" - }, - "balance": { - "type": "text" - }, - "balanceNum": { - "type": "double" - }, - "nonce": { - "type": "double" - }, - "totalBalanceWithStake": { - "type": "text" - }, - "totalBalanceWithStakeNum": { - "type": "double" - } - } - }, - "settings": { - "number_of_replicas": 0, - "number_of_shards": 3 - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/noKibana/accountsesdt.json b/tools/indices-creator/cmd/indices-creator/config/noKibana/accountsesdt.json deleted file mode 100644 index 71b69ca3..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/noKibana/accountsesdt.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "index_patterns": [ - "accountsesdt-*" - ], - "mappings": { - "properties": { - "balanceNum": { - "type": "double" - }, - "data": { - "properties": { - "attributes": { - "type": "text" - }, - "creator": { - "type": "text" - }, - "metadata": { - "type": "text" - }, - "name": { - "type": "text" - }, - "tags": { - "type": "text" - } - }, - "type": "nested" - }, - "tokenNonce": { - "type": "double" - }, - "timestamp": { - "type": "date", - "format": "epoch_second" - } - } - }, - "settings": { - "number_of_replicas": 0, - "number_of_shards": 3 - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/noKibana/accountsesdthistory.json b/tools/indices-creator/cmd/indices-creator/config/noKibana/accountsesdthistory.json deleted file mode 100644 index c8e8b039..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/noKibana/accountsesdthistory.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "index_patterns": [ - "accountsesdthistory-*" - ], - "mappings": { - "properties": { - "timestamp": { - "format": "epoch_second", - "type": "date" - }, - "tokenNonce": { - "type": "double" - } - } - }, - "settings": { - "number_of_replicas": 0, - "number_of_shards": 5 - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/noKibana/accountshistory.json b/tools/indices-creator/cmd/indices-creator/config/noKibana/accountshistory.json deleted file mode 100644 index 4816394e..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/noKibana/accountshistory.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "index_patterns": [ - "accountshistory-*" - ], - "mappings": { - "properties": { - "timestamp": { - "format": "epoch_second", - "type": "date" - } - } - }, - "settings": { - "number_of_replicas": 0, - "number_of_shards": 5 - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/noKibana/blocks.json b/tools/indices-creator/cmd/indices-creator/config/noKibana/blocks.json deleted file mode 100644 index c5245aea..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/noKibana/blocks.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "index_patterns": [ - "blocks-*" - ], - "mappings": { - "properties": { - "gasPenalized": { - "type": "double" - }, - "gasProvided": { - "type": "double" - }, - "gasRefunded": { - "type": "double" - }, - "maxGasLimit": { - "type": "double" - }, - "nonce": { - "type": "double" - }, - "searchOrder": { - "type": "double" - }, - "timestamp": { - "format": "epoch_second", - "type": "date" - } - } - }, - "settings": { - "index": { - "sort.field": [ - "timestamp", - "nonce" - ], - "sort.order": [ - "desc", - "desc" - ] - }, - "number_of_replicas": 0, - "number_of_shards": 3 - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/noKibana/delegators.json b/tools/indices-creator/cmd/indices-creator/config/noKibana/delegators.json deleted file mode 100644 index ee9fb059..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/noKibana/delegators.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "index_patterns": [ - "delegators-*" - ], - "mappings": { - "properties": { - "activeStakeNum": { - "type": "double" - } - } - }, - "settings": { - "number_of_replicas": 0, - "number_of_shards": 3 - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/noKibana/epochinfo.json b/tools/indices-creator/cmd/indices-creator/config/noKibana/epochinfo.json deleted file mode 100644 index 500230df..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/noKibana/epochinfo.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "index_patterns": [ - "epochinfo-*" - ], - "settings": { - "number_of_replicas": 0, - "number_of_shards": 3 - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/noKibana/logs.json b/tools/indices-creator/cmd/indices-creator/config/noKibana/logs.json deleted file mode 100644 index 7b640165..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/noKibana/logs.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "index_patterns": [ - "logs-*" - ], - "mappings": { - "properties": { - "address": { - "type": "text" - }, - "events": { - "properties": { - "address": { - "type": "text" - }, - "data": { - "type": "text" - }, - "topics": { - "type": "text" - } - }, - "type": "nested" - }, - "timestamp": { - "format": "epoch_second", - "type": "date" - } - } - }, - "settings": { - "number_of_replicas": 0, - "number_of_shards": 3 - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/noKibana/miniblocks.json b/tools/indices-creator/cmd/indices-creator/config/noKibana/miniblocks.json deleted file mode 100644 index a99d90c4..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/noKibana/miniblocks.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "index_patterns": [ - "miniblocks-*" - ], - "settings": { - "number_of_replicas": 0, - "number_of_shards": 3 - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/noKibana/opendistro.json b/tools/indices-creator/cmd/indices-creator/config/noKibana/opendistro.json deleted file mode 100644 index 8b64728e..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/noKibana/opendistro.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "index_patterns": [ - ".opendistro-*" - ], - "settings": { - "number_of_replicas": 0, - "number_of_shards": 1 - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/noKibana/operations.json b/tools/indices-creator/cmd/indices-creator/config/noKibana/operations.json deleted file mode 100644 index c75e9af9..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/noKibana/operations.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "index_patterns": [ - "operations-*" - ], - "mappings": { - "properties": { - "nonce": { - "type": "long" - }, - "timestamp": { - "format": "epoch_second", - "type": "date" - } - } - }, - "settings": { - "index": { - "sort.field": [ - "timestamp", - "nonce" - ], - "sort.order": [ - "desc", - "desc" - ] - }, - "number_of_replicas": 0, - "number_of_shards": 5 - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/noKibana/rating.json b/tools/indices-creator/cmd/indices-creator/config/noKibana/rating.json deleted file mode 100644 index 36cb555d..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/noKibana/rating.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "index_patterns": [ - "rating-*" - ], - "mappings": { - "properties": { - "validatorsRating": { - "properties": { - "rating": { - "type": "float" - } - } - } - } - }, - "settings": { - "number_of_replicas": 0, - "number_of_shards": 1 - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/noKibana/receipts.json b/tools/indices-creator/cmd/indices-creator/config/noKibana/receipts.json deleted file mode 100644 index 6237fd5a..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/noKibana/receipts.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "index_patterns": [ - "receipts-*" - ], - "mappings": { - "properties": { - "timestamp": { - "format": "epoch_second", - "type": "date" - } - } - }, - "settings": { - "index": { - "sort.field": [ - "timestamp" - ], - "sort.order": [ - "desc" - ] - }, - "number_of_replicas": 0, - "number_of_shards": 3 - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/noKibana/rounds.json b/tools/indices-creator/cmd/indices-creator/config/noKibana/rounds.json deleted file mode 100644 index 4b2d3347..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/noKibana/rounds.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "index_patterns": [ - "rounds-*" - ], - "mappings": { - "properties": { - "timestamp": { - "format": "epoch_second", - "type": "date" - } - } - }, - "settings": { - "index": { - "sort.field": [ - "timestamp" - ], - "sort.order": [ - "desc" - ] - }, - "number_of_replicas": 0, - "number_of_shards": 3 - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/noKibana/scdeploys.json b/tools/indices-creator/cmd/indices-creator/config/noKibana/scdeploys.json deleted file mode 100644 index 499cc28b..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/noKibana/scdeploys.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "index_patterns": [ - "scdeploys-*" - ], - "mappings": { - "properties": { - "timestamp": { - "format": "epoch_second", - "type": "date" - }, - "upgrades": { - "properties": { - "timestamp": { - "format": "epoch_second", - "type": "date" - } - }, - "type": "nested" - } - } - }, - "settings": { - "number_of_replicas": 0, - "number_of_shards": 3 - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/noKibana/scresults.json b/tools/indices-creator/cmd/indices-creator/config/noKibana/scresults.json deleted file mode 100644 index ab2feb17..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/noKibana/scresults.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "index_patterns": [ - "scresults-*" - ], - "mappings": { - "properties": { - "gasLimit": { - "type": "double" - }, - "gasPrice": { - "type": "double" - }, - "nonce": { - "type": "double" - }, - "timestamp": { - "format": "epoch_second", - "type": "date" - } - } - }, - "settings": { - "index": { - "sort.field": [ - "timestamp" - ], - "sort.order": [ - "desc" - ] - }, - "number_of_replicas": 0, - "number_of_shards": 3 - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/noKibana/tags.json b/tools/indices-creator/cmd/indices-creator/config/noKibana/tags.json deleted file mode 100644 index 98a83a87..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/noKibana/tags.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "index_patterns": [ - "tags-*" - ], - "settings": { - "number_of_replicas": 0, - "number_of_shards": 3 - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/noKibana/tokens.json b/tools/indices-creator/cmd/indices-creator/config/noKibana/tokens.json deleted file mode 100644 index 166cf739..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/noKibana/tokens.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "index_patterns": [ - "tokens-*" - ], - "mappings": { - "properties": { - "data": { - "properties": { - "attributes": { - "type": "text" - }, - "creator": { - "type": "text" - }, - "metadata": { - "type": "text" - }, - "name": { - "type": "text" - }, - "ownersHistory": { - "properties": { - "timestamp": { - "format": "epoch_second", - "type": "date" - } - }, - "type": "nested" - }, - "tags": { - "type": "text" - } - }, - "type": "nested" - }, - "roles": { - "type": "nested" - }, - "nonce": { - "type": "double" - }, - "timestamp": { - "format": "epoch_second", - "type": "date" - } - } - }, - "settings": { - "number_of_replicas": 0, - "number_of_shards": 3 - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/noKibana/transactions.json b/tools/indices-creator/cmd/indices-creator/config/noKibana/transactions.json deleted file mode 100644 index 97e4d7b1..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/noKibana/transactions.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "index_patterns": [ - "transactions-*" - ], - "mappings": { - "properties": { - "gasLimit": { - "type": "double" - }, - "gasPrice": { - "type": "double" - }, - "nonce": { - "type": "double" - }, - "timestamp": { - "format": "epoch_second", - "type": "date" - } - } - }, - "settings": { - "index": { - "sort.field": [ - "timestamp", - "nonce" - ], - "sort.order": [ - "desc", - "desc" - ] - }, - "number_of_replicas": 0, - "number_of_shards": 5 - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/noKibana/validators.json b/tools/indices-creator/cmd/indices-creator/config/noKibana/validators.json deleted file mode 100644 index 8b05e49c..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/noKibana/validators.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "index_patterns": [ - "validators-*" - ], - "settings": { - "number_of_replicas": 0, - "number_of_shards": 1 - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/withKibana/accounts.json b/tools/indices-creator/cmd/indices-creator/config/withKibana/accounts.json deleted file mode 100644 index 585984e1..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/withKibana/accounts.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "index_patterns": [ - "accounts-*" - ], - "mappings": { - "properties": { - "address": { - "type": "text" - }, - "balance": { - "type": "text" - }, - "balanceNum": { - "type": "double" - }, - "nonce": { - "type": "double" - }, - "totalBalanceWithStake": { - "type": "text" - }, - "totalBalanceWithStakeNum": { - "type": "double" - } - } - }, - "settings": { - "number_of_replicas": 0, - "number_of_shards": 3, - "opendistro.index_state_management.rollover_alias": "accounts" - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/withKibana/accountsesdt.json b/tools/indices-creator/cmd/indices-creator/config/withKibana/accountsesdt.json deleted file mode 100644 index b662c326..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/withKibana/accountsesdt.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "index_patterns": [ - "accountsesdt-*" - ], - "mappings": { - "properties": { - "balanceNum": { - "type": "double" - }, - "data": { - "properties": { - "attributes": { - "type": "text" - }, - "creator": { - "type": "text" - }, - "metadata": { - "type": "text" - }, - "name": { - "type": "text" - }, - "tags": { - "type": "text" - } - }, - "type": "nested" - }, - "tokenNonce": { - "type": "double" - }, - "timestamp": { - "type": "date", - "format": "epoch_second" - } - } - }, - "settings": { - "number_of_replicas": 0, - "number_of_shards": 3 - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/withKibana/accountsesdthistory.json b/tools/indices-creator/cmd/indices-creator/config/withKibana/accountsesdthistory.json deleted file mode 100644 index d6bbdc6a..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/withKibana/accountsesdthistory.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "index_patterns": [ - "accountsesdthistory-*" - ], - "mappings": { - "properties": { - "timestamp": { - "format": "epoch_second", - "type": "date" - }, - "tokenNonce": { - "type": "double" - } - } - }, - "settings": { - "number_of_replicas": 0, - "number_of_shards": 5, - "opendistro.index_state_management.rollover_alias": "accountsesdthistory" - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/withKibana/accountshistory.json b/tools/indices-creator/cmd/indices-creator/config/withKibana/accountshistory.json deleted file mode 100644 index a2f18bb5..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/withKibana/accountshistory.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "index_patterns": [ - "accountshistory-*" - ], - "mappings": { - "properties": { - "timestamp": { - "format": "epoch_second", - "type": "date" - } - } - }, - "settings": { - "number_of_replicas": 0, - "number_of_shards": 5, - "opendistro.index_state_management.rollover_alias": "accountshistory" - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/withKibana/blocks.json b/tools/indices-creator/cmd/indices-creator/config/withKibana/blocks.json deleted file mode 100644 index 2bd7627a..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/withKibana/blocks.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "index_patterns": [ - "blocks-*" - ], - "mappings": { - "properties": { - "gasPenalized": { - "type": "double" - }, - "gasProvided": { - "type": "double" - }, - "gasRefunded": { - "type": "double" - }, - "maxGasLimit": { - "type": "double" - }, - "nonce": { - "type": "double" - }, - "searchOrder": { - "type": "double" - }, - "timestamp": { - "format": "epoch_second", - "type": "date" - } - } - }, - "settings": { - "index": { - "sort.field": [ - "timestamp", - "nonce" - ], - "sort.order": [ - "desc", - "desc" - ] - }, - "number_of_replicas": 0, - "number_of_shards": 3, - "opendistro.index_state_management.rollover_alias": "blocks" - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/withKibana/delegators.json b/tools/indices-creator/cmd/indices-creator/config/withKibana/delegators.json deleted file mode 100644 index ee9fb059..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/withKibana/delegators.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "index_patterns": [ - "delegators-*" - ], - "mappings": { - "properties": { - "activeStakeNum": { - "type": "double" - } - } - }, - "settings": { - "number_of_replicas": 0, - "number_of_shards": 3 - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/withKibana/epochinfo.json b/tools/indices-creator/cmd/indices-creator/config/withKibana/epochinfo.json deleted file mode 100644 index 500230df..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/withKibana/epochinfo.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "index_patterns": [ - "epochinfo-*" - ], - "settings": { - "number_of_replicas": 0, - "number_of_shards": 3 - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/withKibana/logs.json b/tools/indices-creator/cmd/indices-creator/config/withKibana/logs.json deleted file mode 100644 index 7b640165..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/withKibana/logs.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "index_patterns": [ - "logs-*" - ], - "mappings": { - "properties": { - "address": { - "type": "text" - }, - "events": { - "properties": { - "address": { - "type": "text" - }, - "data": { - "type": "text" - }, - "topics": { - "type": "text" - } - }, - "type": "nested" - }, - "timestamp": { - "format": "epoch_second", - "type": "date" - } - } - }, - "settings": { - "number_of_replicas": 0, - "number_of_shards": 3 - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/withKibana/miniblocks.json b/tools/indices-creator/cmd/indices-creator/config/withKibana/miniblocks.json deleted file mode 100644 index 76687cf5..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/withKibana/miniblocks.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "index_patterns": [ - "miniblocks-*" - ], - "settings": { - "number_of_replicas": 0, - "number_of_shards": 3, - "opendistro.index_state_management.rollover_alias": "miniblocks" - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/withKibana/opendistro.json b/tools/indices-creator/cmd/indices-creator/config/withKibana/opendistro.json deleted file mode 100644 index 8b64728e..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/withKibana/opendistro.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "index_patterns": [ - ".opendistro-*" - ], - "settings": { - "number_of_replicas": 0, - "number_of_shards": 1 - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/withKibana/operations.json b/tools/indices-creator/cmd/indices-creator/config/withKibana/operations.json deleted file mode 100644 index c75e9af9..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/withKibana/operations.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "index_patterns": [ - "operations-*" - ], - "mappings": { - "properties": { - "nonce": { - "type": "long" - }, - "timestamp": { - "format": "epoch_second", - "type": "date" - } - } - }, - "settings": { - "index": { - "sort.field": [ - "timestamp", - "nonce" - ], - "sort.order": [ - "desc", - "desc" - ] - }, - "number_of_replicas": 0, - "number_of_shards": 5 - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/withKibana/rating.json b/tools/indices-creator/cmd/indices-creator/config/withKibana/rating.json deleted file mode 100644 index 77686afa..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/withKibana/rating.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "index_patterns": [ - "rating-*" - ], - "mappings": { - "properties": { - "validatorsRating": { - "properties": { - "rating": { - "type": "float" - } - } - } - } - }, - "settings": { - "number_of_replicas": 0, - "number_of_shards": 1, - "opendistro.index_state_management.rollover_alias": "rating" - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/withKibana/receipts.json b/tools/indices-creator/cmd/indices-creator/config/withKibana/receipts.json deleted file mode 100644 index 9ca261f4..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/withKibana/receipts.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "index_patterns": [ - "receipts-*" - ], - "mappings": { - "properties": { - "timestamp": { - "format": "epoch_second", - "type": "date" - } - } - }, - "settings": { - "index": { - "sort.field": [ - "timestamp" - ], - "sort.order": [ - "desc" - ] - }, - "number_of_replicas": 0, - "number_of_shards": 3, - "opendistro.index_state_management.rollover_alias": "receipts" - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/withKibana/rounds.json b/tools/indices-creator/cmd/indices-creator/config/withKibana/rounds.json deleted file mode 100644 index f00e388d..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/withKibana/rounds.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "index_patterns": [ - "rounds-*" - ], - "mappings": { - "properties": { - "timestamp": { - "format": "epoch_second", - "type": "date" - } - } - }, - "settings": { - "index": { - "sort.field": [ - "timestamp" - ], - "sort.order": [ - "desc" - ] - }, - "number_of_replicas": 0, - "number_of_shards": 3, - "opendistro.index_state_management.rollover_alias": "rounds" - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/withKibana/scdeploys.json b/tools/indices-creator/cmd/indices-creator/config/withKibana/scdeploys.json deleted file mode 100644 index 499cc28b..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/withKibana/scdeploys.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "index_patterns": [ - "scdeploys-*" - ], - "mappings": { - "properties": { - "timestamp": { - "format": "epoch_second", - "type": "date" - }, - "upgrades": { - "properties": { - "timestamp": { - "format": "epoch_second", - "type": "date" - } - }, - "type": "nested" - } - } - }, - "settings": { - "number_of_replicas": 0, - "number_of_shards": 3 - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/withKibana/scresults.json b/tools/indices-creator/cmd/indices-creator/config/withKibana/scresults.json deleted file mode 100644 index 5325792f..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/withKibana/scresults.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "index_patterns": [ - "scresults-*" - ], - "mappings": { - "properties": { - "gasLimit": { - "type": "double" - }, - "gasPrice": { - "type": "double" - }, - "nonce": { - "type": "double" - }, - "timestamp": { - "format": "epoch_second", - "type": "date" - } - } - }, - "settings": { - "index": { - "sort.field": [ - "timestamp" - ], - "sort.order": [ - "desc" - ] - }, - "number_of_replicas": 0, - "number_of_shards": 3, - "opendistro.index_state_management.rollover_alias": "scresults" - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/withKibana/tags.json b/tools/indices-creator/cmd/indices-creator/config/withKibana/tags.json deleted file mode 100644 index 98a83a87..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/withKibana/tags.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "index_patterns": [ - "tags-*" - ], - "settings": { - "number_of_replicas": 0, - "number_of_shards": 3 - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/withKibana/tokens.json b/tools/indices-creator/cmd/indices-creator/config/withKibana/tokens.json deleted file mode 100644 index 2b504d02..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/withKibana/tokens.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "index_patterns": [ - "tokens-*" - ], - "mappings": { - "properties": { - "data": { - "properties": { - "attributes": { - "type": "text" - }, - "creator": { - "type": "text" - }, - "metadata": { - "type": "text" - }, - "name": { - "type": "text" - }, - "ownersHistory": { - "properties": { - "timestamp": { - "format": "epoch_second", - "type": "date" - } - }, - "type": "nested" - }, - "tags": { - "type": "text" - } - }, - "type": "nested" - }, - "nonce": { - "type": "double" - }, - "timestamp": { - "format": "epoch_second", - "type": "date" - }, - "roles": { - "type": "nested" - } - } - }, - "settings": { - "number_of_replicas": 0, - "number_of_shards": 3 - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/withKibana/transactions.json b/tools/indices-creator/cmd/indices-creator/config/withKibana/transactions.json deleted file mode 100644 index b26dfea7..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/withKibana/transactions.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "index_patterns": [ - "transactions-*" - ], - "mappings": { - "properties": { - "gasLimit": { - "type": "double" - }, - "gasPrice": { - "type": "double" - }, - "nonce": { - "type": "double" - }, - "timestamp": { - "format": "epoch_second", - "type": "date" - } - } - }, - "settings": { - "index": { - "sort.field": [ - "timestamp", - "nonce" - ], - "sort.order": [ - "desc", - "desc" - ] - }, - "number_of_replicas": 0, - "number_of_shards": 5, - "opendistro.index_state_management.rollover_alias": "transactions" - } -} diff --git a/tools/indices-creator/cmd/indices-creator/config/withKibana/validators.json b/tools/indices-creator/cmd/indices-creator/config/withKibana/validators.json deleted file mode 100644 index 550f9de9..00000000 --- a/tools/indices-creator/cmd/indices-creator/config/withKibana/validators.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "index_patterns": [ - "validators-*" - ], - "settings": { - "number_of_replicas": 0, - "number_of_shards": 1, - "opendistro.index_state_management.rollover_alias": "validators" - } -} diff --git a/tools/indices-creator/cmd/indices-creator/main.go b/tools/indices-creator/cmd/indices-creator/main.go deleted file mode 100644 index 7744e870..00000000 --- a/tools/indices-creator/cmd/indices-creator/main.go +++ /dev/null @@ -1,162 +0,0 @@ -package main - -import ( - "bytes" - "fmt" - "io/ioutil" - "os" - "path" - - "github.com/elastic/go-elasticsearch/v7" - "github.com/multiversx/mx-chain-es-indexer-go/client" - "github.com/multiversx/mx-chain-es-indexer-go/client/logging" - "github.com/multiversx/mx-chain-es-indexer-go/tools/indexes-creator/reader" - logger "github.com/multiversx/mx-chain-logger-go" - "github.com/pelletier/go-toml" - "github.com/urfave/cli" -) - -const configFileName = "cluster.toml" - -type config struct { - ClusterConfig struct { - URL string `toml:"url"` - Username string `toml:"username"` - Password string `toml:"password"` - UseKibana bool `toml:"use-kibana"` - EnabledIndices []string `toml:"enabled-indices"` - } `toml:"config"` -} - -var ( - log = logger.GetOrCreate("main") - - // defines the path to the config folder - configPath = cli.StringFlag{ - Name: "config-path", - Usage: "The path to the config folder", - Value: "./config", - } -) - -const helpTemplate = `NAME: - {{.Name}} - {{.Usage}} -USAGE: - {{.HelpName}} {{if .VisibleFlags}}[global options]{{end}} - {{if len .Authors}} -AUTHOR: - {{range .Authors}}{{ . }}{{end}} - {{end}}{{if .Commands}} -GLOBAL OPTIONS: - {{range .VisibleFlags}}{{.}} - {{end}} -VERSION: - {{.Version}} - {{end}} -` - -func main() { - app := cli.NewApp() - cli.AppHelpTemplate = helpTemplate - app.Name = "Index cr" - app.Version = "v1.0.0" - app.Usage = "Elasticsearch indices creator tool" - app.Flags = []cli.Flag{ - configPath, - } - app.Authors = []cli.Author{ - { - Name: "The MultiversX Team", - Email: "contact@multiversx.com", - }, - } - - _ = logger.SetLogLevel("*:DEBUG") - - app.Action = createIndexesAndMappings - - err := app.Run(os.Args) - if err != nil { - log.Error(err.Error()) - os.Exit(1) - } - -} - -func createIndexesAndMappings(ctx *cli.Context) { - cfgPath := ctx.String(configPath.Name) - cfg, err := loadConfigFile(cfgPath) - if err != nil { - log.Error("cannot load config file", "error", err.Error()) - } - - pathToMappings := path.Join(cfgPath, "noKibana") - if cfg.ClusterConfig.UseKibana { - pathToMappings = path.Join(cfgPath, "withKibana") - } - - indexesMappings, _, err := reader.GetElasticTemplatesAndPolicies(pathToMappings, cfg.ClusterConfig.EnabledIndices) - if err != nil { - log.Error("cannot load templates", "error", err.Error()) - return - } - - err = createTemplates(cfg, indexesMappings) - if err != nil { - log.Error("cannot create templates", "error", err.Error()) - return - } - - log.Info("all indices were created") -} - -func createTemplates(cfg *config, indexesMappings map[string]*bytes.Buffer) error { - databaseClient, err := client.NewElasticClient(elasticsearch.Config{ - Addresses: []string{cfg.ClusterConfig.URL}, - Username: cfg.ClusterConfig.Username, - Password: cfg.ClusterConfig.Password, - Logger: &logging.CustomLogger{}, - }) - if err != nil { - return err - } - - for index, indexData := range indexesMappings { - errCheck := databaseClient.CheckAndCreateTemplate(index, indexData) - if errCheck != nil { - return fmt.Errorf("index: %s, error: %w", index, errCheck) - } - - indexName := fmt.Sprintf("%s-%s", index, "000001") - errCreate := databaseClient.CheckAndCreateIndex(indexName) - if errCreate != nil { - return fmt.Errorf("index: %s, error: %w", index, errCreate) - } - - errAlias := databaseClient.CheckAndCreateAlias(index, indexName) - if err != nil { - return errAlias - } - } - - return nil -} - -func loadConfigFile(pathStr string) (*config, error) { - tomlBytes, err := loadBytesFromFile(path.Join(pathStr, configFileName)) - if err != nil { - return nil, err - } - - var cfg config - err = toml.Unmarshal(tomlBytes, &cfg) - if err != nil { - return nil, err - } - - return &cfg, nil -} - -func loadBytesFromFile(file string) ([]byte, error) { - return ioutil.ReadFile(file) -} diff --git a/tools/indices-creator/go.mod b/tools/indices-creator/go.mod deleted file mode 100644 index c8fd8a15..00000000 --- a/tools/indices-creator/go.mod +++ /dev/null @@ -1,27 +0,0 @@ -module github.com/multiversx/mx-chain-es-indexer-go/tools/indexes-creator - -go 1.17 - -require ( - github.com/elastic/go-elasticsearch/v7 v7.12.0 - github.com/multiversx/mx-chain-es-indexer-go v1.3.7-0.20230110115720-a54a2d8aa20d - github.com/multiversx/mx-chain-logger-go v1.0.11 - github.com/pelletier/go-toml v1.9.3 - github.com/urfave/cli v1.22.9 -) - -require ( - github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d // indirect - github.com/denisbrodbeck/machineid v1.0.1 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.2 // indirect - github.com/mr-tron/base58 v1.2.0 // indirect - github.com/multiversx/mx-chain-core-go v1.1.30 // indirect - github.com/russross/blackfriday/v2 v2.0.1 // indirect - github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect - github.com/tidwall/gjson v1.14.0 // indirect - github.com/tidwall/match v1.1.1 // indirect - github.com/tidwall/pretty v1.2.0 // indirect - golang.org/x/sys v0.2.0 // indirect - google.golang.org/protobuf v1.26.0 // indirect -) diff --git a/tools/indices-creator/go.sum b/tools/indices-creator/go.sum deleted file mode 100644 index 23c9f96e..00000000 --- a/tools/indices-creator/go.sum +++ /dev/null @@ -1,186 +0,0 @@ -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= -github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= -github.com/btcsuite/btcd v0.22.0-beta.0.20220111032746-97732e52810c/go.mod h1:tjmYdS6MLJ5/s0Fj4DbLgSbDHbEqLJrtnHecBFkdz5M= -github.com/btcsuite/btcd v0.23.0/go.mod h1:0QJIIN1wwIXF/3G/m87gIwGniDMDQqjVn4SZgnFpsYY= -github.com/btcsuite/btcd/btcec/v2 v2.1.0/go.mod h1:2VzYrv4Gm4apmbVVsSq5bqf1Ec8v56E48Vt0Y/umPgA= -github.com/btcsuite/btcd/btcec/v2 v2.1.3/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE= -github.com/btcsuite/btcd/btcutil v1.0.0/go.mod h1:Uoxwv0pqYWhD//tfTiipkxNfdhG9UrLwaeswfjfdF0A= -github.com/btcsuite/btcd/btcutil v1.1.0/go.mod h1:5OapHB7A2hBBWLm48mmw4MOHNJCcUBTwmWH/0Jn8VHE= -github.com/btcsuite/btcd/btcutil v1.1.3/go.mod h1:UR7dsSJzJUfMmFiiLlIrMq1lS9jh9EdCV7FStZSnpi0= -github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= -github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= -github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= -github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= -github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= -github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= -github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I= -github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= -github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= -github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= -github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= -github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= -github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ= -github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= -github.com/elastic/go-elasticsearch/v7 v7.12.0 h1:j4tvcMrZJLp39L2NYvBb7f+lHKPqPHSL3nvB8+/DV+s= -github.com/elastic/go-elasticsearch/v7 v7.12.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= -github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/multiversx/mx-chain-core-go v1.1.30 h1:BtURR4I6HU1OnSbxcPMTQSQXNqtOuH3RW6bg5N7FSM0= -github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= -github.com/multiversx/mx-chain-es-indexer-go v1.3.7-0.20230110115720-a54a2d8aa20d h1:EhECmwxHKuD1me0W/YPfPVG+6XjCvEZc+lNQ3R3v5jA= -github.com/multiversx/mx-chain-es-indexer-go v1.3.7-0.20230110115720-a54a2d8aa20d/go.mod h1:IV42GfhkqQ5vVO0OzGaF/ejp8TQrLkNo4LSB3TPnVhg= -github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= -github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= -github.com/multiversx/mx-chain-vm-common-go v1.3.34/go.mod h1:sZ2COLCxvf2GxAAJHGmGqWybObLtFuk2tZUyGqnMXE8= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= -github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= -github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w= -github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= -github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= -github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= -github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/urfave/cli v1.22.9 h1:cv3/KhXGBGjEXLC4bH0sLuJ9BewaAbpk5oyMOveu4pw= -github.com/urfave/cli v1.22.9/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A= -golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tools/indices-creator/reader/reader.go b/tools/indices-creator/reader/reader.go deleted file mode 100644 index 72ef986c..00000000 --- a/tools/indices-creator/reader/reader.go +++ /dev/null @@ -1,44 +0,0 @@ -package reader - -import ( - "bytes" - "fmt" - "io/ioutil" - "path/filepath" -) - -// GetElasticTemplatesAndPolicies will return elastic templates and policies -// TODO implement policies when will start to use it again -func GetElasticTemplatesAndPolicies(path string, indexes []string) (map[string]*bytes.Buffer, map[string]*bytes.Buffer, error) { - indexTemplates := make(map[string]*bytes.Buffer) - indexPolicies := make(map[string]*bytes.Buffer) - var err error - - for _, index := range indexes { - indexTemplates[index], err = getTemplateByIndex(path, index) - if err != nil { - return nil, nil, err - } - } - - return indexTemplates, indexPolicies, nil -} - -func getTemplateByIndex(path string, index string) (*bytes.Buffer, error) { - indexTemplate := &bytes.Buffer{} - - fileName := fmt.Sprintf("%s.json", index) - filePath := filepath.Join(path, fileName) - fileBytes, err := ioutil.ReadFile(filePath) - if err != nil { - return nil, fmt.Errorf("getTemplateByIndex: %w, path %s, error %s", err, filePath, err.Error()) - } - - indexTemplate.Grow(len(fileBytes)) - _, err = indexTemplate.Write(fileBytes) - if err != nil { - return nil, fmt.Errorf("getTemplateByIndex: %w, path %s, error %s", err, filePath, err.Error()) - } - - return indexTemplate, nil -}