Skip to content

Commit fe23ca8

Browse files
bogdan-rosianucfaur09stefangutica
authored
Development to main (#1561)
* Enhance error handling in GatewayProxyController for block retrieval by hash (#1545) * use EsdtType instead of hardcoded values (#1544) * token service improvements (#1543) * processes 50 tokens concurrently using ConcurrencyUtils * run all three batch operations in parallel with increased concurrency * add more logs * apply mex prices * add logs ( to be removed ) * increse cron time expression * set cron to 2 minutes * Websocket subscriptions (#1528) * add websockets for blocks and txs * add support for subscribe to stats * use websockets rooms * remove logs * add lock on crons * check stats room exists Signed-off-by: GuticaStefan <stefan.gutica@gmail.com> * fix indent spaces * add validation pipes + filters * add try catch + class validator fixes * fix linter * add pool subscription + reduce filters combinations for subscriptions * lint * add support for events subscription * lint * separate subscription websocket into separate app * add config * add path * fix * add path for events + config default settings * temp logs * temp logs 2 * added missing configs + remove temp logs * enable andromeda in config * add metrics on subscription * remove async + reschedule * refresh metrics every second * set max listeners to 12 * add EOL * add count on update + parallel broadcast to rooms * lower ttl for blocks count cache * remove comments * remove comments * renaming --------- Signed-off-by: GuticaStefan <stefan.gutica@gmail.com> Co-authored-by: bogdan-rosianu <bogdan.rosianu@yahoo.com> Co-authored-by: cfaur09 <catalinfaurpaul@gmail.com> * log instead of error for invalid legacy delegation contract (#1542) * implement event logAddress filter (#1548) * implement event logAddress filter * implement topics filter * Token market cap updates (#1550) * token mcap update * add logs * extend logs * add support for timestampMs for account * fix unit tests * improve token fetch price (#1549) * improve token fetch price * remove logger * update spec * fix lint error * add logs for error * add price * Enhance NFT type support in cache warmer and collection services (#1552) * Refactor function filter application in ElasticIndexerHelper to enhance query conditions with AND operator and existence checks (#1554) * apply supply info for all tokens (#1558) * Refactor token market cap calculation to ensure price and circulating supply are checked before computation * add specs * fix lint * added reserved field to blocks (#1560) * add support for custom url custom headers (#1557) * Add custom URL headers support and enhance token data fetching * Enhance custom URL headers handling by adding JSON parsing and validation checks * add logs * fixes * remove app.hatom.com value * add EOL * fix indentation --------- Signed-off-by: GuticaStefan <stefan.gutica@gmail.com> Co-authored-by: Catalin Faur <52102171+cfaur09@users.noreply.github.com> Co-authored-by: Gutica Stefan <123564494+stefangutica@users.noreply.github.com> Co-authored-by: cfaur09 <catalinfaurpaul@gmail.com> Co-authored-by: GuticaStefan <stefan.gutica@gmail.com>
1 parent 40dad9a commit fe23ca8

File tree

16 files changed

+220
-56
lines changed

16 files changed

+220
-56
lines changed

‎config/config.mainnet.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,3 +191,7 @@ compression:
191191
level: 6
192192
threshold: 1024
193193
chunkSize: 16384
194+
customUrlHeaders:
195+
- urlPattern: ''
196+
headers:
197+
x-custom-auth: ''

‎src/common/api-config/api.config.service.ts‎

Lines changed: 62 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { Constants } from '@multiversx/sdk-nestjs-common';
1+
import { Constants, OriginLogger } from '@multiversx/sdk-nestjs-common';
22
import { Injectable } from '@nestjs/common';
33
import { ConfigService } from '@nestjs/config';
44
import { DatabaseConnectionOptions } from '../persistence/entities/connection.options';
@@ -7,6 +7,8 @@ import { LogTopic } from '@multiversx/sdk-transaction-processor/lib/types/log-to
77

88
@Injectable()
99
export class ApiConfigService {
10+
private readonly logger = new OriginLogger(ApiConfigService.name);
11+
1012
constructor(private readonly configService: ConfigService) {
1113
}
1214

@@ -972,4 +974,63 @@ export class ApiConfigService {
972974

973975
return port;
974976
}
977+
978+
getHeadersForCustomUrl(url: string): Record<string, string> | undefined {
979+
let customUrlConfigs = this.configService.get<any>('customUrlHeaders');
980+
981+
if (!customUrlConfigs) {
982+
return undefined;
983+
}
984+
985+
if (typeof customUrlConfigs === 'string') {
986+
try {
987+
customUrlConfigs = JSON.parse(customUrlConfigs);
988+
} catch (error) {
989+
return undefined;
990+
}
991+
}
992+
993+
if (!Array.isArray(customUrlConfigs) && typeof customUrlConfigs === 'object') {
994+
let workingConfig = customUrlConfigs;
995+
996+
while (workingConfig && workingConfig[''] && typeof workingConfig[''] === 'object') {
997+
workingConfig = workingConfig[''];
998+
}
999+
1000+
const arrayValues: any[] = [];
1001+
for (const key in workingConfig) {
1002+
if (!isNaN(Number(key))) {
1003+
let item = workingConfig[key];
1004+
while (item && item[''] && typeof item[''] === 'object') {
1005+
item = item[''];
1006+
}
1007+
arrayValues[Number(key)] = item;
1008+
}
1009+
}
1010+
1011+
if (arrayValues.length > 0) {
1012+
customUrlConfigs = arrayValues.filter(item => item !== undefined);
1013+
this.logger.log(`Loaded ${customUrlConfigs.length} custom URL header config(s)`);
1014+
} else {
1015+
return undefined;
1016+
}
1017+
}
1018+
1019+
if (!Array.isArray(customUrlConfigs)) {
1020+
return undefined;
1021+
}
1022+
1023+
for (const config of customUrlConfigs) {
1024+
if (config && config.urlPattern && url.includes(config.urlPattern)) {
1025+
let headers = config.headers;
1026+
if (headers && headers[''] && typeof headers[''] === 'object') {
1027+
headers = headers[''];
1028+
}
1029+
this.logger.log(`Found custom headers for URL pattern '${config.urlPattern}': ${JSON.stringify(headers)}`);
1030+
return headers;
1031+
}
1032+
}
1033+
1034+
return undefined;
1035+
}
9751036
}

‎src/common/indexer/elastic/elastic.indexer.helper.ts‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -777,8 +777,12 @@ export class ElasticIndexerHelper {
777777
public applyFunctionFilter(elasticQuery: ElasticQuery, functions: string[]) {
778778
const functionConditions = [];
779779
for (const field of functions) {
780-
functionConditions.push(QueryType.Match('function', field));
781-
functionConditions.push(QueryType.Match('operation', field));
780+
functionConditions.push(QueryType.Match('function', field, QueryOperator.AND));
781+
782+
functionConditions.push(QueryType.Must(
783+
[QueryType.Match('operation', field, QueryOperator.AND)],
784+
[QueryType.Exists('function')]
785+
));
782786
}
783787
return elasticQuery.withMustCondition(QueryType.Should(functionConditions));
784788
}
@@ -814,6 +818,16 @@ export class ElasticIndexerHelper {
814818
elasticQuery = elasticQuery.withCondition(QueryConditionOptions.must, QueryType.Match('order', filter.order));
815819
}
816820

821+
if (filter.logAddress) {
822+
elasticQuery = elasticQuery.withMustMatchCondition('logAddress', filter.logAddress);
823+
}
824+
825+
if (filter.topics && filter.topics.length > 0) {
826+
for (const topic of filter.topics) {
827+
elasticQuery = elasticQuery.withMustMatchCondition('topics', topic);
828+
}
829+
}
830+
817831
return elasticQuery;
818832
}
819833
}

‎src/common/indexer/entities/account.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
export interface Account {
22
address: string;
33
nonce: number;
4+
timestampMs: number;
45
timestamp: number;
56
balance: string;
67
balanceNum: number;

‎src/crons/cache.warmer/cache.warmer.service.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -324,9 +324,11 @@ export class CacheWarmerService {
324324
const nftTypes = [
325325
NftType.NonFungibleESDT,
326326
NftType.SemiFungibleESDT,
327+
NftType.MetaESDT,
327328
NftType.NonFungibleESDTv2,
328329
NftType.DynamicNonFungibleESDT,
329330
NftType.DynamicSemiFungibleESDT,
331+
NftType.DynamicMetaESDT,
330332
];
331333

332334
for (const key of Object.keys(allAssets)) {

‎src/endpoints/accounts/account.service.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,9 @@ export class AccountService {
9999
if (options?.withTimestamp) {
100100
const elasticSearchAccount = await this.indexerService.getAccount(address);
101101
account.timestamp = elasticSearchAccount.timestamp;
102+
if (elasticSearchAccount.timestampMs) {
103+
account.timestampMs = elasticSearchAccount.timestampMs;
104+
}
102105
}
103106

104107
if (AddressUtils.isSmartContractAddress(address)) {

‎src/endpoints/accounts/entities/account.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,10 @@ export class Account {
1616
@ApiProperty({ type: Number, description: 'Account current nonce', example: 42 })
1717
nonce: number = 0;
1818

19-
@ApiProperty({ type: Number, description: 'Timestamp of the block where the account was first indexed', example: 1676979360 })
19+
@ApiProperty({ type: Number, description: 'Timestamp in milliseconds of the block where the account was first indexed', example: 1676979360000 })
20+
timestampMs: number = 0;
21+
22+
@ApiProperty({ type: Number, description: 'Timestamp in seconds of the block where the account was first indexed', example: 1676979360 })
2023
timestamp: number = 0;
2124

2225
@ApiProperty({ type: Number, description: 'The shard ID allocated to the account', example: 0 })

‎src/endpoints/blocks/entities/block.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,9 @@ export class Block {
6868
@ApiProperty({ type: BlockProofDto, nullable: true, required: false })
6969
previousHeaderProof: BlockProofDto | undefined = undefined;
7070

71+
@ApiProperty( { type: String })
72+
reserved: string = '';
73+
7174
@ApiProperty({ type: BlockProofDto, nullable: true, required: false })
7275
proof: BlockProofDto | undefined = undefined;
7376

‎src/endpoints/collections/collection.service.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ export class CollectionService {
9999
nftCollection.subType = indexedCollection.type as NftSubType;
100100
nftCollection.timestamp = indexedCollection.timestamp;
101101

102-
if (nftCollection.type.in(NftType.NonFungibleESDT, NftType.SemiFungibleESDT)) {
102+
if (nftCollection.type.in(NftType.NonFungibleESDT, NftType.SemiFungibleESDT, NftType.MetaESDT)) {
103103
nftCollection.isVerified = indexedCollection.api_isVerified;
104104
nftCollection.nftCount = indexedCollection.api_nftCount;
105105
nftCollection.holderCount = indexedCollection.api_holderCount;

‎src/endpoints/events/entities/events.filter.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,4 +11,6 @@ export class EventsFilter {
1111
before: number = 0;
1212
after: number = 0;
1313
order: number = 0;
14+
logAddress: string = '';
15+
topics: string[] = [];
1416
}

0 commit comments

Comments
 (0)