diff --git a/README.md b/README.md index fbaff0a55..63f1dba62 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,9 @@ It depends on the following optional external systems: - events notifier rabbitmq: queue that pushes logs & events which are handled internally e.g. to trigger NFT media fetch - data: provides EGLD price information for transactions - xexchange: provides price information regarding various tokens listed on the xExchange +- nodes provider: fetch nodes data instead of self-computing (config: `features.nodesFetch`) +- staking providers fetcher: fetch staking providers data instead of self-computing (config: `features.providersFetch`) +- tokens provider: fetch tokens data instead of self-computing (config: `features.tokensFetch`) - ipfs: ipfs gateway for fetching mainly NFT metadata & media files - media: ipfs gateway which will be used as prefix for NFT media & metadata returned in the NFT details - media internal: caching layer for ipfs data to fetch from a centralized system such as S3 for performance reasons diff --git a/config/config.devnet-old.yaml b/config/config.devnet-old.yaml index 9bc4134b8..8f61aac89 100644 --- a/config/config.devnet-old.yaml +++ b/config/config.devnet-old.yaml @@ -61,6 +61,10 @@ features: jwtSecret: '' nodeEpochsLeft: enabled: false + chainBarnard: + enabled: false + activationEpoch: 3964 + activationTimestamp: 1751460236 image: width: 600 height: 600 diff --git a/config/config.devnet.yaml b/config/config.devnet.yaml index de8480d00..8d76292b7 100644 --- a/config/config.devnet.yaml +++ b/config/config.devnet.yaml @@ -23,6 +23,9 @@ features: websocketSubscription: enabled: false port: 6002 + maxSubscriptionsPerInstance: 10000 + maxSubscriptionsPerClient: 10 + broadcastIntervalMs: 6000 eventsNotifier: enabled: false port: 5674 @@ -54,7 +57,7 @@ features: serviceUrl: 'https://devnet-data-api.multiversx.com' assetsFetch: enabled: true - assetesUrl: 'https://tools.multiversx.com/assets-cdn' + assetsUrl: 'https://tools.multiversx.com/assets-cdn' auth: enabled: false maxExpirySeconds: 86400 @@ -64,12 +67,22 @@ features: - '' jwtSecret: '' stakingV4: - enabled: false + enabled: true cronExpression: '*/5 * * * * *' activationEpoch: 1043 chainAndromeda: enabled: true activationEpoch: 4 + stakingV5: + enabled: true + activationEpoch: 4817 + chainBarnard: + enabled: true + activationEpoch: 3964 + activationTimestamp: 1751460236 + deprecatedRelayedV1V2: + enabled: true + activationEpoch: 4569 nodeEpochsLeft: enabled: false transactionProcessor: @@ -110,13 +123,13 @@ features: enabled: false maxLookBehindNonces: 100 nodesFetch: - enabled: true + enabled: false serviceUrl: 'https://devnet-api.multiversx.com' tokensFetch: - enabled: true + enabled: false serviceUrl: 'https://devnet-api.multiversx.com' providersFetch: - enabled: true + enabled: false serviceUrl: 'https://devnet-api.multiversx.com' image: width: 600 @@ -179,6 +192,10 @@ inflation: - 1130177 - 924690 - 719203 +stakingV5Inflation: + - 1069805 +# november 2025 devnet supply: 24_433_152. first year rewards = 50% * (8.757% * supply) = 1_069_805 +# TODO: calculate the inflation for upcoming years based on the inflation decay nftProcess: parallelism: 1 maxRetries: 3 @@ -186,4 +203,4 @@ compression: enabled: true level: 6 threshold: 1024 - chunkSize: 16384 \ No newline at end of file + chunkSize: 16384 diff --git a/config/config.e2e-mocked.mainnet.yaml b/config/config.e2e-mocked.mainnet.yaml index 83d35ceac..f65f05065 100644 --- a/config/config.e2e-mocked.mainnet.yaml +++ b/config/config.e2e-mocked.mainnet.yaml @@ -8,6 +8,9 @@ features: websocketSubscription: enabled: false port: 6002 + maxSubscriptionsPerInstance: 10000 + maxSubscriptionsPerClient: 10 + broadcastIntervalMs: 6000 dataApi: enabled: false serviceUrl: 'https://data-api.multiversx.com' @@ -16,6 +19,10 @@ features: maxExpirySeconds: 86400 acceptedOrigins: - '' + chainBarnard: + enabled: true + activationEpoch: 1820 + activationTimestamp: 1753376544 cron: transactionProcessor: false transactionProcessorMaxLookBehind: 1000 diff --git a/config/config.e2e.mainnet.yaml b/config/config.e2e.mainnet.yaml index 9cdb71bc6..50da09bff 100644 --- a/config/config.e2e.mainnet.yaml +++ b/config/config.e2e.mainnet.yaml @@ -21,8 +21,11 @@ flags: collectionPropertiesFromGateway: false features: websocketSubscription: - enabled: false + enabled: true port: 6002 + maxSubscriptionsPerInstance: 10000 + maxSubscriptionsPerClient: 10 + broadcastIntervalMs: 6000 eventsNotifier: enabled: false port: 5674 @@ -68,6 +71,13 @@ features: chainAndromeda: enabled: true activationEpoch: 4 + stakingV5: + enabled: true + activationEpoch: 1951 + chainBarnard: + enabled: true + activationEpoch: 1820 + activationTimestamp: 1753376544 nodeEpochsLeft: enabled: false transactionProcessor: @@ -121,7 +131,7 @@ features: serviceUrl: 'https://api.multiversx.com' assetsFetch: enabled: false - assetesUrl: 'https://tools.multiversx.com/assets-cdn' + assetsUrl: 'https://tools.multiversx.com/assets-cdn' image: width: 600 height: 600 @@ -183,6 +193,8 @@ inflation: - 1130177 - 924690 - 719203 +stakingV5Inflation: + - 1262802 nftProcess: parallelism: 1 - maxRetries: 3 + maxRetries: 3 \ No newline at end of file diff --git a/config/config.mainnet.yaml b/config/config.mainnet.yaml index 2ab0a01a5..a90823e54 100644 --- a/config/config.mainnet.yaml +++ b/config/config.mainnet.yaml @@ -23,6 +23,9 @@ features: websocketSubscription: enabled: false port: 6002 + maxSubscriptionsPerInstance: 10000 + maxSubscriptionsPerClient: 10 + broadcastIntervalMs: 6000 eventsNotifier: enabled: false port: 5674 @@ -68,6 +71,16 @@ features: chainAndromeda: enabled: true activationEpoch: 1763 + stakingV5: + enabled: true + activationEpoch: 1951 + chainBarnard: + enabled: true + activationEpoch: 1820 + activationTimestamp: 1753376544 + deprecatedRelayedV1V2: + enabled: true + activationEpoch: 1918 nodeEpochsLeft: enabled: false transactionProcessor: @@ -111,17 +124,17 @@ features: enabled: false maxLookBehindNonces: 100 nodesFetch: - enabled: true + enabled: false serviceUrl: 'https://api.multiversx.com' tokensFetch: - enabled: true + enabled: false serviceUrl: 'https://api.multiversx.com' providersFetch: - enabled: true + enabled: false serviceUrl: 'https://api.multiversx.com' assetsFetch: enabled: true - assetesUrl: 'https://tools.multiversx.com/assets-cdn' + assetsUrl: 'https://tools.multiversx.com/assets-cdn' image: width: 600 height: 600 @@ -183,6 +196,10 @@ inflation: - 1130177 - 924690 - 719203 +stakingV5Inflation: + - 1262802 +# december 2025 mainnet supply: 28_840_981. first year rewards = 50% * (8.75% * supply) = 1262802 +# TODO: calculate the inflation for upcoming years based on the inflation decay nftProcess: parallelism: 1 maxRetries: 3 diff --git a/config/config.testnet.yaml b/config/config.testnet.yaml index e0353f514..c897aa17b 100644 --- a/config/config.testnet.yaml +++ b/config/config.testnet.yaml @@ -23,6 +23,9 @@ features: websocketSubscription: enabled: false port: 6002 + maxSubscriptionsPerInstance: 10000 + maxSubscriptionsPerClient: 10 + broadcastIntervalMs: 6000 eventsNotifier: enabled: false port: 5674 @@ -61,12 +64,22 @@ features: - '' jwtSecret: '' stakingV4: - enabled: false + enabled: true cronExpression: '*/5 * * * * *' activationEpoch: 1043 chainAndromeda: enabled: true activationEpoch: 4 + stakingV5: + enabled: true + activationEpoch: 2519 + chainBarnard: + enabled: false + activationEpoch: 2030 + activationTimestamp: 1864749472 + deprecatedRelayedV1V2: + enabled: true + activationEpoch: 2038 nodeEpochsLeft: enabled: false transactionProcessor: @@ -110,17 +123,17 @@ features: enabled: false maxLookBehindNonces: 100 nodesFetch: - enabled: true + enabled: false serviceUrl: 'https://testnet-api.multiversx.com' tokensFetch: - enabled: true + enabled: false serviceUrl: 'https://testnet-api.multiversx.com' providersFetch: - enabled: true + enabled: false serviceUrl: 'https://testnet-api.multiversx.com' assetsFetch: enabled: true - assetesUrl: 'https://tools.multiversx.com/assets-cdn' + assetsUrl: 'https://tools.multiversx.com/assets-cdn' image: width: 600 height: 600 @@ -182,6 +195,10 @@ inflation: - 1130177 - 924690 - 719203 +stakingV5Inflation: + - 930117 +# november 2025 testnet supply: 21_242_828. first year rewards = 50% * (8.75% * supply) = 930117 +# TODO: calculate the inflation for upcoming years based on the inflation decay nftProcess: parallelism: 1 maxRetries: 3 diff --git a/package-lock.json b/package-lock.json index f964e0d74..74dba9da3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -42,6 +42,7 @@ "anchorme": "^3.0.8", "apollo-server-core": "^3.13.0", "apollo-server-express": "3.13.0", + "async-mutex": "^0.5.0", "bignumber.js": "^9.0.2", "class-transformer": "^0.5.1", "class-validator": "^0.14.2", @@ -7503,6 +7504,15 @@ "resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz", "integrity": "sha512-eAkdoKxU6/LkKDBzLpT+t6Ff5EtfSF4wx1WfJiPEEV7WNLnDaRXk0oVysiEPm262roaachGexwUv94WhSgN5TQ==" }, + "node_modules/async-mutex": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/async-mutex/-/async-mutex-0.5.0.tgz", + "integrity": "sha512-1A94B18jkJ3DYq284ohPxoXbfTA5HsQ7/Mf4DEhcyLx3Bz27Rh59iScbB6EPiP+B+joue6YCxcMXSbFC1tZKwA==", + "license": "MIT", + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/async-retry": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", diff --git a/package.json b/package.json index dbbad71f9..dbd1c1ecd 100644 --- a/package.json +++ b/package.json @@ -117,6 +117,7 @@ "anchorme": "^3.0.8", "apollo-server-core": "^3.13.0", "apollo-server-express": "3.13.0", + "async-mutex": "^0.5.0", "bignumber.js": "^9.0.2", "class-transformer": "^0.5.1", "class-validator": "^0.14.2", @@ -220,4 +221,4 @@ "node_modules" ] } -} +} \ No newline at end of file diff --git a/src/common/api-config/api.config.service.ts b/src/common/api-config/api.config.service.ts index d1cd162f2..6496eafbd 100644 --- a/src/common/api-config/api.config.service.ts +++ b/src/common/api-config/api.config.service.ts @@ -4,6 +4,7 @@ import { ConfigService } from '@nestjs/config'; import { DatabaseConnectionOptions } from '../persistence/entities/connection.options'; import { StatusCheckerThresholds } from './entities/status-checker-thresholds'; import { LogTopic } from '@multiversx/sdk-transaction-processor/lib/types/log-topic'; +import { TimeUtils } from 'src/utils/time.utils'; @Injectable() export class ApiConfigService { @@ -592,6 +593,15 @@ export class ApiConfigService { return inflationAmounts; } + getStakingV5InflationAmounts(): number[] { + const inflationAmounts = this.configService.get('stakingV5Inflation'); + if (!inflationAmounts) { + throw new Error('No staking v5 inflation amounts present'); + } + + return inflationAmounts; + } + getMediaUrl(): string { const mediaUrl = this.configService.get('urls.media'); if (!mediaUrl) { @@ -890,12 +900,39 @@ export class ApiConfigService { return this.configService.get('features.chainAndromeda.activationEpoch') ?? 99999; } + isStakingV5Enabled(): boolean { + return this.configService.get('features.stakingV5.enabled') ?? false; + } + + getStakingV5ActivationEpoch(): number { + return this.configService.get('features.stakingV5.activationEpoch') ?? 99999; + } + + isDeprecatedRelayedV1V2Enabled(): boolean { + return this.configService.get('features.deprecatedRelayedV1V2.enabled') ?? false; + } + + getDeprecatedRelayedV1V2ActivationEpoch(): number { + return this.configService.get('features.deprecatedRelayedV1V2.activationEpoch') ?? 99999; + } + + shouldDeprecateRelayedV1V2(epoch: number): boolean { + const isEnabled = this.isDeprecatedRelayedV1V2Enabled(); + if (!isEnabled) { + return false; + } + + return epoch >= this.getDeprecatedRelayedV1V2ActivationEpoch(); + } + isAssetsCdnFeatureEnabled(): boolean { return this.configService.get('features.assetsFetch.enabled') ?? false; } getAssetsCdnUrl(): string { - return this.configService.get('features.assetsFetch.assetesUrl') ?? 'https://tools.multiversx.com/assets-cdn'; + return this.configService.get('features.assetsFetch.assetsUrl') + ?? this.configService.get('features.assetsFetch.assetesUrl') // todo: remove this in the future + ?? 'https://tools.multiversx.com/assets-cdn'; } isTokensFetchFeatureEnabled(): boolean { @@ -975,6 +1012,18 @@ export class ApiConfigService { return port; } + getWebsocketSubscriptionBroadcastIntervalMs(): number { + return this.configService.get('features.websocketSubscription.broadcastIntervalMs') ?? 6000; + } + + getWebsocketMaxSubscriptionsPerInstance(): number { + return this.configService.get('features.websocketSubscription.maxSubscriptionsPerInstance') ?? 10_000; + } + + getWebsocketMaxSubscriptionsPerClient(): number { + return this.configService.get('features.websocketSubscription.maxSubscriptionsPerClient') ?? 5; + } + getHeadersForCustomUrl(url: string): Record | undefined { let customUrlConfigs = this.configService.get('customUrlHeaders'); @@ -1033,4 +1082,26 @@ export class ApiConfigService { return undefined; } + + isChainBarnardEnabled(): boolean { + return this.configService.get('features.chainBarnard.enabled') ?? false; + } + + getChainBarnardActivationEpoch(): number { + const epoch = this.configService.get('features.chainBarnard.activationEpoch'); + if (epoch == null) { + return TimeUtils.TIMESTAMP_IN_SECONDS_THRESHOLD + 1; + } + + return epoch; + } + + getChainBarnardActivationTimestamp(): number { + const timestamp = this.configService.get('features.chainBarnard.activationTimestamp'); + if (timestamp == null) { + return TimeUtils.TIMESTAMP_IN_SECONDS_THRESHOLD + 1; + } + + return timestamp; + } } diff --git a/src/common/indexer/elastic/elastic.indexer.helper.ts b/src/common/indexer/elastic/elastic.indexer.helper.ts index 96bd3ec13..753598acb 100644 --- a/src/common/indexer/elastic/elastic.indexer.helper.ts +++ b/src/common/indexer/elastic/elastic.indexer.helper.ts @@ -19,6 +19,7 @@ import { ApplicationFilter } from "src/endpoints/applications/entities/applicati import { NftType } from "../entities/nft.type"; import { EventsFilter } from "src/endpoints/events/entities/events.filter"; import { ScriptQuery } from "./script.query"; +import { TimeUtils } from "src/utils/time.utils"; @Injectable() export class ElasticIndexerHelper { @@ -99,7 +100,14 @@ export class ElasticIndexerHelper { } if (filter.before || filter.after) { - elasticQuery = elasticQuery.withDateRangeFilter('timestamp', filter.before, filter.after); + if (filter.before) { + const timestampBeforeIdentifier = TimeUtils.isTimestampInSeconds(filter.before) ? 'timestamp' : 'timestampMs'; + elasticQuery = elasticQuery.withRangeFilter(timestampBeforeIdentifier, new RangeLowerThanOrEqual(filter.before)); + } + if (filter.after) { + const timestampAfterIdentifier = TimeUtils.isTimestampInSeconds(filter.after) ? 'timestamp' : 'timestampMs'; + elasticQuery = elasticQuery.withRangeFilter(timestampAfterIdentifier, new RangeGreaterThanOrEqual(filter.after)); + } } if (filter.canCreate !== undefined) { @@ -278,8 +286,13 @@ export class ElasticIndexerHelper { } } - if (filter.before || filter.after) { - elasticQuery = elasticQuery.withDateRangeFilter('timestamp', filter.before, filter.after); + if (filter.before) { + const timestampBeforeIdentifier = TimeUtils.isTimestampInSeconds(filter.before) ? 'timestamp' : 'timestampMs'; + elasticQuery = elasticQuery.withRangeFilter(timestampBeforeIdentifier, new RangeLowerThanOrEqual(filter.before)); + } + if (filter.after) { + const timestampAfterIdentifier = TimeUtils.isTimestampInSeconds(filter.after) ? 'timestamp' : 'timestampMs'; + elasticQuery = elasticQuery.withRangeFilter(timestampAfterIdentifier, new RangeGreaterThanOrEqual(filter.after)); } if (filter.nonceBefore) { @@ -408,8 +421,13 @@ export class ElasticIndexerHelper { elasticQuery = elasticQuery.withCondition(QueryConditionOptions.must, QueryType.Match('status', filter.status)); } - if (filter.before || filter.after) { - elasticQuery = elasticQuery.withDateRangeFilter('timestamp', filter.before, filter.after); + if (filter.before) { + const timestampBeforeIdentifier = TimeUtils.isTimestampInSeconds(filter.before) ? 'timestamp' : 'timestampMs'; + elasticQuery = elasticQuery.withRangeFilter(timestampBeforeIdentifier, new RangeLowerThanOrEqual(filter.before)); + } + if (filter.after) { + const timestampAfterIdentifier = TimeUtils.isTimestampInSeconds(filter.after) ? 'timestamp' : 'timestampMs'; + elasticQuery = elasticQuery.withRangeFilter(timestampAfterIdentifier, new RangeGreaterThanOrEqual(filter.after)); } if (filter.senderOrReceiver) { @@ -552,8 +570,16 @@ export class ElasticIndexerHelper { .withMustMatchCondition('miniBlockHash', filter.miniBlockHash) .withMustMultiShouldCondition(filter.hashes, hash => QueryType.Match('_id', hash)) .withMustMatchCondition('status', filter.status) - .withMustMultiShouldCondition(filter.tokens, token => QueryType.Match('tokens', token, QueryOperator.AND)) - .withDateRangeFilter('timestamp', filter.before, filter.after); + .withMustMultiShouldCondition(filter.tokens, token => QueryType.Match('tokens', token, QueryOperator.AND)); + + if (filter.before) { + const timestampBeforeIdentifier = TimeUtils.isTimestampInSeconds(filter.before) ? 'timestamp' : 'timestampMs'; + elasticQuery = elasticQuery.withRangeFilter(timestampBeforeIdentifier, new RangeLowerThanOrEqual(filter.before)); + } + if (filter.after) { + const timestampAfterIdentifier = TimeUtils.isTimestampInSeconds(filter.after) ? 'timestamp' : 'timestampMs'; + elasticQuery = elasticQuery.withRangeFilter(timestampAfterIdentifier, new RangeGreaterThanOrEqual(filter.after)); + } if (filter.functions && filter.functions.length > 0) { if (filter.functions.length === 1 && filter.functions[0] === '') { @@ -658,8 +684,13 @@ export class ElasticIndexerHelper { let elasticQuery = ElasticQuery.create().withCondition(QueryConditionOptions.must, mustQueries); - if (filter && (filter.before || filter.after)) { - elasticQuery = elasticQuery.withDateRangeFilter('timestamp', filter.before, filter.after); + if (filter && filter.before) { + const timestampBeforeIdentifier = TimeUtils.isTimestampInSeconds(filter.before) ? 'timestamp' : 'timestampMs'; + elasticQuery = elasticQuery.withRangeFilter(timestampBeforeIdentifier, new RangeLowerThanOrEqual(filter.before)); + } + if (filter && filter.after) { + const timestampAfterIdentifier = TimeUtils.isTimestampInSeconds(filter.after) ? 'timestamp' : 'timestampMs'; + elasticQuery = elasticQuery.withRangeFilter(timestampAfterIdentifier, new RangeGreaterThanOrEqual(filter.after)); } if (filter && filter.identifiers) { @@ -764,11 +795,13 @@ export class ElasticIndexerHelper { let elasticQuery = ElasticQuery.create(); if (filter.after) { - elasticQuery = elasticQuery.withRangeFilter('timestamp', new RangeGreaterThanOrEqual(filter.after)); + const timestampIdentifier = TimeUtils.isTimestampInSeconds(filter.after) ? 'timestamp' : 'timestampMs'; + elasticQuery = elasticQuery.withRangeFilter(timestampIdentifier, new RangeGreaterThanOrEqual(filter.after)); } if (filter.before) { - elasticQuery = elasticQuery.withRangeFilter('timestamp', new RangeLowerThanOrEqual(filter.before)); + const timestampIdentifier = TimeUtils.isTimestampInSeconds(filter.before) ? 'timestamp' : 'timestampMs'; + elasticQuery = elasticQuery.withRangeFilter(timestampIdentifier, new RangeLowerThanOrEqual(filter.before)); } return elasticQuery; @@ -791,11 +824,13 @@ export class ElasticIndexerHelper { let elasticQuery = ElasticQuery.create(); if (filter.before) { - elasticQuery = elasticQuery.withRangeFilter('timestamp', new RangeLowerThanOrEqual(filter.before)); + const timestampIdentifier = TimeUtils.isTimestampInSeconds(filter.before) ? 'timestamp' : 'timestampMs'; + elasticQuery = elasticQuery.withRangeFilter(timestampIdentifier, new RangeLowerThanOrEqual(filter.before)); } if (filter.after) { - elasticQuery = elasticQuery.withRangeFilter('timestamp', new RangeGreaterThanOrEqual(filter.after)); + const timestampIdentifier = TimeUtils.isTimestampInSeconds(filter.after) ? 'timestamp' : 'timestampMs'; + elasticQuery = elasticQuery.withRangeFilter(timestampIdentifier, new RangeGreaterThanOrEqual(filter.after)); } if (filter.identifier) { diff --git a/src/common/indexer/elastic/elastic.indexer.service.ts b/src/common/indexer/elastic/elastic.indexer.service.ts index d7f0af8b4..6ed6d2558 100644 --- a/src/common/indexer/elastic/elastic.indexer.service.ts +++ b/src/common/indexer/elastic/elastic.indexer.service.ts @@ -30,6 +30,7 @@ import { NftType } from "../entities/nft.type"; import { EventsFilter } from "src/endpoints/events/entities/events.filter"; import { Events } from "../entities/events"; import { EsCircuitBreakerProxy } from "./circuit-breaker/circuit.breaker.proxy.service"; +import { TimeUtils } from "src/utils/time.utils"; @Injectable() export class ElasticIndexerService implements IndexerInterface { @@ -1031,8 +1032,9 @@ export class ElasticIndexerService implements IndexerInterface { } async getBlockByTimestampAndShardId(timestamp: number, shardId: number): Promise { + const timestampIdentifier = TimeUtils.isTimestampInSeconds(timestamp) ? 'timestamp' : 'timestampMs'; const elasticQuery = ElasticQuery.create() - .withRangeFilter('timestamp', new RangeGreaterThanOrEqual(timestamp)) + .withRangeFilter(timestampIdentifier, new RangeGreaterThanOrEqual(timestamp)) .withCondition(QueryConditionOptions.must, [QueryType.Match('shardId', shardId, QueryOperator.AND)]) .withSort([{ name: 'timestamp', order: ElasticSortOrder.ascending }]); diff --git a/src/common/indexer/entities/block.ts b/src/common/indexer/entities/block.ts index e552caf36..ebfd4aef1 100644 --- a/src/common/indexer/entities/block.ts +++ b/src/common/indexer/entities/block.ts @@ -12,6 +12,7 @@ export interface Block { size: number; sizeTxs: number; timestamp: number; + timestampMs?: number; stateRootHash: string; prevHash: string; shardId: number; diff --git a/src/common/indexer/entities/collection.ts b/src/common/indexer/entities/collection.ts index e6a8f12dc..5190e17f6 100644 --- a/src/common/indexer/entities/collection.ts +++ b/src/common/indexer/entities/collection.ts @@ -1,3 +1,16 @@ +export interface CollectionProperties { + canMint?: boolean; + canBurn?: boolean; + canUpgrade?: boolean; + canTransferNFTCreateRole?: boolean; + canAddSpecialRoles?: boolean; + canPause?: boolean; + canFreeze?: boolean; + canWipe?: boolean; + canChangeOwner?: boolean; + canCreateMultiShard?: boolean; +} + export interface Collection { _id: string; name: string; @@ -5,9 +18,11 @@ export interface Collection { token: string; issuer: string; currentOwner: string; + numDecimals?: number; type: string; timestamp: number; ownersHistory: { address: string, timestamp: number }[]; + properties?: CollectionProperties; api_isVerified?: boolean; api_nftCount?: number; api_holderCount?: number; diff --git a/src/common/indexer/entities/events.ts b/src/common/indexer/entities/events.ts index 1d67492f8..add007142 100644 --- a/src/common/indexer/entities/events.ts +++ b/src/common/indexer/entities/events.ts @@ -10,4 +10,5 @@ export class Events { txOrder: number = 0; order: number = 0; timestamp: number = 0; + timestampMs?: number; } diff --git a/src/common/indexer/entities/miniblock.ts b/src/common/indexer/entities/miniblock.ts index 123662905..71e086355 100644 --- a/src/common/indexer/entities/miniblock.ts +++ b/src/common/indexer/entities/miniblock.ts @@ -7,6 +7,7 @@ export interface MiniBlock { type: string; procTypeD: string; timestamp: number; + timestampMs?: number; procTypeS: string; senderBlockNonce: string; receiverBlockNonce: string; diff --git a/src/common/indexer/entities/provider.delegators.ts b/src/common/indexer/entities/provider.delegators.ts index 4afda5f9b..b4f27bbe5 100644 --- a/src/common/indexer/entities/provider.delegators.ts +++ b/src/common/indexer/entities/provider.delegators.ts @@ -4,4 +4,5 @@ export class ProviderDelegators { activeStake: string = ''; activeStakeNum: number = 0; timestamp: number = 0; + timestampMs?: number; } diff --git a/src/common/indexer/entities/round.ts b/src/common/indexer/entities/round.ts index 301553e63..867c67d47 100644 --- a/src/common/indexer/entities/round.ts +++ b/src/common/indexer/entities/round.ts @@ -4,5 +4,6 @@ export interface Round { blockWasProposed: boolean, shardId: number, epoch: number, - timestamp: number + timestamp: number, + timestampMs?: number, } diff --git a/src/common/indexer/entities/sc.deploy.ts b/src/common/indexer/entities/sc.deploy.ts index 38a5ea424..0233d0e72 100644 --- a/src/common/indexer/entities/sc.deploy.ts +++ b/src/common/indexer/entities/sc.deploy.ts @@ -5,6 +5,7 @@ export interface ScDeploy { initialCodeHash: string; deployer: string; timestamp: number; + timestampMs?: number; upgrades: ScDeployUpgrade[]; } diff --git a/src/common/indexer/entities/sc.result.ts b/src/common/indexer/entities/sc.result.ts index 039e2b682..882898b59 100644 --- a/src/common/indexer/entities/sc.result.ts +++ b/src/common/indexer/entities/sc.result.ts @@ -13,6 +13,7 @@ export interface ScResult { originalTxHash: string; callType: string; timestamp: number; + timestampMs?: number; tokens: string[]; esdtValues: string[]; operation: string; diff --git a/src/common/metrics/api.metrics.service.ts b/src/common/metrics/api.metrics.service.ts index ff2d24260..5f78bfb24 100644 --- a/src/common/metrics/api.metrics.service.ts +++ b/src/common/metrics/api.metrics.service.ts @@ -23,6 +23,7 @@ export class ApiMetricsService { private static transactionsPendingResultsCounter: Counter; private static batchUpdatesCounter: Counter; private static subscriptionsConnectionsGauge: Gauge; + private static subscriptionsTopicConnectionsGauge: Gauge; constructor( private readonly apiConfigService: ApiConfigService, @@ -40,6 +41,14 @@ export class ApiMetricsService { }); } + if (!ApiMetricsService.subscriptionsTopicConnectionsGauge) { + ApiMetricsService.subscriptionsTopicConnectionsGauge = new Gauge({ + name: 'websocket_subscriptions_topic_connections', + help: 'Unique websocket clients per topic', + labelNames: ['topic'], + }); + } + if (!ApiMetricsService.vmQueriesHistogram) { ApiMetricsService.vmQueriesHistogram = new Histogram({ name: 'vm_query', @@ -191,10 +200,16 @@ export class ApiMetricsService { } @OnEvent(MetricsEvents.SetWebsocketMetrics) - setWebsocketSubscriptionsMetrics(payload: { connectedClients: number }) { - const { connectedClients } = payload; + setWebsocketSubscriptionsMetrics(payload: { connectedClients: number; topics?: Record }) { + const { connectedClients, topics } = payload; ApiMetricsService.subscriptionsConnectionsGauge.set(connectedClients); + + if (topics) { + for (const [topic, count] of Object.entries(topics)) { + ApiMetricsService.subscriptionsTopicConnectionsGauge.set({ topic }, count); + } + } } diff --git a/src/crons/websocket/blocks.gateway.ts b/src/crons/websocket/blocks.gateway.ts index 40cfd5038..a7df06b7d 100644 --- a/src/crons/websocket/blocks.gateway.ts +++ b/src/crons/websocket/blocks.gateway.ts @@ -4,37 +4,59 @@ import { BlockService } from '../../endpoints/blocks/block.service'; import { BlockFilter } from '../../endpoints/blocks/entities/block.filter'; import { QueryPagination } from 'src/common/entities/query.pagination'; import { BlockSubscribePayload } from '../../endpoints/blocks/entities/block.subscribe'; -import { UseFilters } from '@nestjs/common'; +import { UseFilters, UseInterceptors } from '@nestjs/common'; import { WebsocketExceptionsFilter } from 'src/utils/ws-exceptions.filter'; import { WsValidationPipe } from 'src/utils/ws-validation.pipe'; import { OriginLogger } from '@multiversx/sdk-nestjs-common'; +import { LockingGuardInterceptor } from 'src/utils/locking.guard.interceptor'; @UseFilters(WebsocketExceptionsFilter) @WebSocketGateway({ cors: { origin: '*' }, path: '/ws/subscription' }) export class BlocksGateway { private readonly logger = new OriginLogger(BlocksGateway.name); + static readonly keyPrefix = 'blocks-'; @WebSocketServer() server!: Server; constructor(private readonly blockService: BlockService) { } + @UseInterceptors(LockingGuardInterceptor) @SubscribeMessage('subscribeBlocks') async handleSubscription( @ConnectedSocket() client: Socket, @MessageBody(new WsValidationPipe()) payload: BlockSubscribePayload ) { const filterIdentifier = JSON.stringify(payload); - await client.join(`blocks-${filterIdentifier}`); + const roomName = `${BlocksGateway.keyPrefix}${filterIdentifier}`; + + if (!client.rooms.has(roomName)) { + await client.join(roomName); + } return { status: 'success' }; } + @SubscribeMessage('unsubscribeBlocks') + async handleUnsubscribe( + @ConnectedSocket() client: Socket, + @MessageBody(new WsValidationPipe()) payload: BlockSubscribePayload + ) { + const filterIdentifier = JSON.stringify(payload); + const roomName = `${BlocksGateway.keyPrefix}${filterIdentifier}`; + + if (client.rooms.has(roomName)) { + await client.leave(roomName); + } + + return { status: 'unsubscribed' }; + } + async pushBlocksForRoom(roomName: string): Promise { - if (!roomName.startsWith("blocks-")) return; + if (!roomName.startsWith(BlocksGateway.keyPrefix)) return; try { - const filterIdentifier = roomName.replace("blocks-", ""); + const filterIdentifier = roomName.replace(BlocksGateway.keyPrefix, ""); const filter: BlockSubscribePayload = JSON.parse(filterIdentifier); const blockFilter = new BlockFilter({ diff --git a/src/crons/websocket/connection.handler.ts b/src/crons/websocket/connection.handler.ts index b8f50ed53..ae1f74a08 100644 --- a/src/crons/websocket/connection.handler.ts +++ b/src/crons/websocket/connection.handler.ts @@ -15,6 +15,18 @@ export class ConnectionHandler implements OnGatewayDisconnect, OnGatewayConnecti handleDisconnect(_client: Socket) { } handleConnection(client: Socket, ..._args: any[]) { - client.setMaxListeners(12); + client.setMaxListeners(16); + } + + hasSubscriptionsWithPrefixes(prefixes: string[]): boolean { + const rooms = this.server.sockets.adapter.rooms; + + for (const roomName of rooms.keys()) { + if (prefixes.some(prefix => roomName.startsWith(prefix))) { + return true; + } + } + + return false; } } diff --git a/src/crons/websocket/events.custom.gateway.ts b/src/crons/websocket/events.custom.gateway.ts new file mode 100644 index 000000000..cf1272385 --- /dev/null +++ b/src/crons/websocket/events.custom.gateway.ts @@ -0,0 +1,96 @@ +import { WebSocketGateway, WebSocketServer, SubscribeMessage, ConnectedSocket, MessageBody } from '@nestjs/websockets'; +import { Server, Socket } from 'socket.io'; +import { UseFilters, UseInterceptors } from '@nestjs/common'; +import { OriginLogger } from '@multiversx/sdk-nestjs-common'; +import { QueryPagination } from 'src/common/entities/query.pagination'; +import { WsValidationPipe } from 'src/utils/ws-validation.pipe'; +import { WebsocketExceptionsFilter } from 'src/utils/ws-exceptions.filter'; +import { RoomKeyGenerator } from './room.key.generator'; +import { EventsService } from 'src/endpoints/events/events.service'; +import { EventsCustomSubscribePayload } from 'src/endpoints/events/entities/events.custom.subscribe'; +import { EventsFilter } from 'src/endpoints/events/entities/events.filter'; +import { Events } from 'src/endpoints/events/entities/events'; +import { LockingGuardInterceptor } from 'src/utils/locking.guard.interceptor'; + +@UseFilters(WebsocketExceptionsFilter) +@WebSocketGateway({ cors: { origin: '*' }, path: '/ws/subscription' }) +export class EventsCustomGateway { + private readonly logger = new OriginLogger(EventsCustomGateway.name); + + static keyPrefix = 'custom-events-'; + + @WebSocketServer() + server!: Server; + + constructor( + private readonly eventsService: EventsService, + ) { } + + @UseInterceptors(LockingGuardInterceptor) + @SubscribeMessage('subscribeCustomEvents') + async handleCustomSubscription( + @ConnectedSocket() client: Socket, + @MessageBody(new WsValidationPipe()) payload: EventsCustomSubscribePayload + ) { + const filterIdentifier = RoomKeyGenerator.deterministicStringify(payload); + const roomName = `${EventsCustomGateway.keyPrefix}${filterIdentifier}`; + + if (!client.rooms.has(roomName)) { + await client.join(roomName); + } + + return { status: 'success' }; + } + + @SubscribeMessage('unsubscribeCustomEvents') + async handleCustomUnsubscribe( + @ConnectedSocket() client: Socket, + @MessageBody(new WsValidationPipe()) payload: EventsCustomSubscribePayload + ) { + const filterIdentifier = RoomKeyGenerator.deterministicStringify(payload); + const roomName = `${EventsCustomGateway.keyPrefix}${filterIdentifier}`; + + if (client.rooms.has(roomName)) { + await client.leave(roomName); + } + + return { status: 'unsubscribed' }; + } + + async pushEventsForTimestampMs(timestampMs: number): Promise { + try { + const allEvents = await this.eventsService.getEvents( + new QueryPagination({ size: 10000 }), + new EventsFilter({ before: timestampMs, after: timestampMs }), + ); + + const eventsFilteredForBroadcast: Map = new Map(); + + for (const event of allEvents) { + const roomKeys = RoomKeyGenerator.generate( + EventsCustomGateway.keyPrefix, + event, + EventsCustomSubscribePayload, + ); + + for (const roomKey of roomKeys) { + if (this.server.sockets.adapter.rooms.has(roomKey)) { + if (!eventsFilteredForBroadcast.has(roomKey)) { + eventsFilteredForBroadcast.set(roomKey, []); + } + eventsFilteredForBroadcast.get(roomKey)!.push(event); + } + } + } + + for (const [roomName] of eventsFilteredForBroadcast) { + this.server.to(roomName).emit("customEventUpdate", { + events: eventsFilteredForBroadcast.get(roomName), + timestampMs, + }); + } + } catch (error) { + this.logger.error(error); + } + } +} diff --git a/src/crons/websocket/events.gateway.ts b/src/crons/websocket/events.gateway.ts index c95e982e0..bb7e8b2d3 100644 --- a/src/crons/websocket/events.gateway.ts +++ b/src/crons/websocket/events.gateway.ts @@ -1,6 +1,6 @@ import { WebSocketGateway, WebSocketServer, SubscribeMessage, MessageBody, ConnectedSocket } from '@nestjs/websockets'; import { Server, Socket } from 'socket.io'; -import { UseFilters } from '@nestjs/common'; +import { UseFilters, UseInterceptors } from '@nestjs/common'; import { WebsocketExceptionsFilter } from 'src/utils/ws-exceptions.filter'; import { WsValidationPipe } from 'src/utils/ws-validation.pipe'; import { OriginLogger } from '@multiversx/sdk-nestjs-common'; @@ -8,28 +8,51 @@ import { EventsService } from '../../endpoints/events/events.service'; import { EventsFilter } from '../../endpoints/events/entities/events.filter'; import { EventsSubscribePayload } from '../../endpoints/events/entities/events.subscribe'; import { QueryPagination } from 'src/common/entities/query.pagination'; +import { RoomKeyGenerator } from './room.key.generator'; +import { LockingGuardInterceptor } from 'src/utils/locking.guard.interceptor'; @UseFilters(WebsocketExceptionsFilter) @WebSocketGateway({ cors: { origin: '*' }, path: '/ws/subscription' }) export class EventsGateway { private readonly logger = new OriginLogger(EventsGateway.name); + static readonly keyPrefix = 'events-'; @WebSocketServer() server!: Server; constructor(private readonly eventsService: EventsService) { } + @UseInterceptors(LockingGuardInterceptor) @SubscribeMessage('subscribeEvents') async handleSubscription( @ConnectedSocket() client: Socket, @MessageBody(new WsValidationPipe()) payload: EventsSubscribePayload, ) { const filterIdentifier = JSON.stringify(payload); - await client.join(`events-${filterIdentifier}`); + const roomName = `${EventsGateway.keyPrefix}${filterIdentifier}`; + if (!client.rooms.has(roomName)) { + await client.join(roomName); + } return { status: 'success' }; } + @SubscribeMessage('unsubscribeEvents') + async handleUnsubscribe( + @ConnectedSocket() client: Socket, + @MessageBody(new WsValidationPipe()) payload: EventsSubscribePayload + ) { + const filterIdentifier = RoomKeyGenerator.deterministicStringify(payload); + const roomName = `${EventsGateway.keyPrefix}${filterIdentifier}`; + + if (client.rooms.has(roomName)) { + await client.leave(roomName); + } + + return { status: 'unsubscribed' }; + } + + async pushEventsForRoom(roomName: string): Promise { if (!roomName.startsWith("events-")) return; diff --git a/src/crons/websocket/network.gateway.ts b/src/crons/websocket/network.gateway.ts index 5618de162..183520e7d 100644 --- a/src/crons/websocket/network.gateway.ts +++ b/src/crons/websocket/network.gateway.ts @@ -1,9 +1,10 @@ import { WebSocketGateway, WebSocketServer, SubscribeMessage } from '@nestjs/websockets'; import { Server, Socket } from 'socket.io'; import { NetworkService } from '../../endpoints/network/network.service'; -import { UseFilters } from '@nestjs/common'; +import { UseFilters, UseInterceptors } from '@nestjs/common'; import { WebsocketExceptionsFilter } from 'src/utils/ws-exceptions.filter'; import { OriginLogger } from '@multiversx/sdk-nestjs-common'; +import { LockingGuardInterceptor } from 'src/utils/locking.guard.interceptor'; @UseFilters(WebsocketExceptionsFilter) @WebSocketGateway({ cors: { origin: '*' }, path: '/ws/subscription' }) @@ -15,9 +16,23 @@ export class NetworkGateway { constructor(private readonly networkService: NetworkService) { } + @UseInterceptors(LockingGuardInterceptor) @SubscribeMessage('subscribeStats') async handleSubscription(client: Socket) { - await client.join('statsRoom'); + if (!client.rooms.has('statsRoom')) { + await client.join('statsRoom'); + } + + return { status: 'success' }; + } + + @SubscribeMessage('unsubscribeStats') + async handleUnsubscribe(client: Socket) { + if (client.rooms.has('statsRoom')) { + await client.leave('statsRoom'); + } + + return { status: 'unsubscribed' }; } async pushStats() { diff --git a/src/crons/websocket/pool.gateway.ts b/src/crons/websocket/pool.gateway.ts index 426e14e10..722466a9e 100644 --- a/src/crons/websocket/pool.gateway.ts +++ b/src/crons/websocket/pool.gateway.ts @@ -1,36 +1,57 @@ import { WebSocketGateway, WebSocketServer, SubscribeMessage, MessageBody, ConnectedSocket } from '@nestjs/websockets'; import { Server, Socket } from 'socket.io'; -import { UseFilters } from '@nestjs/common'; +import { UseFilters, UseInterceptors } from '@nestjs/common'; import { WebsocketExceptionsFilter } from 'src/utils/ws-exceptions.filter'; import { WsValidationPipe } from 'src/utils/ws-validation.pipe'; import { OriginLogger } from '@multiversx/sdk-nestjs-common'; - import { PoolService } from '../../endpoints/pool/pool.service'; import { PoolFilter } from '../../endpoints/pool/entities/pool.filter'; import { QueryPagination } from 'src/common/entities/query.pagination'; import { PoolSubscribePayload } from '../../endpoints/pool/entities/pool.subscribe'; +import { RoomKeyGenerator } from './room.key.generator'; +import { LockingGuardInterceptor } from 'src/utils/locking.guard.interceptor'; @UseFilters(WebsocketExceptionsFilter) @WebSocketGateway({ cors: { origin: '*' }, path: '/ws/subscription' }) export class PoolGateway { private readonly logger = new OriginLogger(PoolGateway.name); + static readonly keyPrefix = 'pool-'; @WebSocketServer() server!: Server; constructor(private readonly poolService: PoolService) { } + @UseInterceptors(LockingGuardInterceptor) @SubscribeMessage('subscribePool') async handleSubscription( @ConnectedSocket() client: Socket, @MessageBody(new WsValidationPipe()) payload: PoolSubscribePayload, ) { - const filterIdentifier = JSON.stringify(payload); - await client.join(`pool-${filterIdentifier}`); + const filterIdentifier = RoomKeyGenerator.deterministicStringify(payload); + const roomName = `${PoolGateway.keyPrefix}${filterIdentifier}`; + if (!client.rooms.has(roomName)) { + await client.join(roomName); + } return { status: 'success' }; } + @SubscribeMessage('unsubscribePool') + async handleUnsubscribe( + @ConnectedSocket() client: Socket, + @MessageBody(new WsValidationPipe()) payload: PoolSubscribePayload + ) { + const filterIdentifier = RoomKeyGenerator.deterministicStringify(payload); + const roomName = `${PoolGateway.keyPrefix}${filterIdentifier}`; + + if (client.rooms.has(roomName)) { + await client.leave(roomName); + } + + return { status: 'unsubscribed' }; + } + async pushPoolForRoom(roomName: string): Promise { if (!roomName.startsWith("pool-")) return; diff --git a/src/crons/websocket/room.key.generator.ts b/src/crons/websocket/room.key.generator.ts new file mode 100644 index 000000000..84c95d77a --- /dev/null +++ b/src/crons/websocket/room.key.generator.ts @@ -0,0 +1,115 @@ +import { EventsCustomSubscribePayload } from 'src/endpoints/events/entities/events.custom.subscribe'; +import { TransactionCustomSubscribePayload } from 'src/endpoints/transactions/entities/dtos/transaction.custom.subscribe'; +import { TransferCustomSubscribePayload } from 'src/endpoints/websocket/entities/transfers.custom.payload'; + +export class RoomKeyGenerator { + public static generate( + prefix: string, + data: Record, + dtoClass: Function, + ): string[] { + const allowedKeys = this.getKeys(dtoClass); + // Collect active filters based on allowed keys and provided data + const activeFilters = this.collectActiveFilters(allowedKeys, data); + + if (activeFilters.length === 0) { + return []; + } + + // Generate all combinations of room keys based on active filters + return this.buildRoomKeys(prefix, activeFilters); + } + + private static collectActiveFilters(allowedKeys: string[], data: Record) { + const activeFilters: { key: string; value: any }[] = []; + + for (const key of allowedKeys) { + if (key === 'token') { + this.addTokenFilters(activeFilters, data); + continue; + } + + const value = data[key]; + if (this.isValidFilterValue(value)) { + activeFilters.push({ key, value }); + } + } + + return activeFilters; + } + + private static addTokenFilters(activeFilters: { key: string; value: any }[], data: Record) { + const value = data['value']; + if (this.isValidFilterValue(value) && value !== '0') { + activeFilters.push({ key: 'token', value: 'EGLD' }); + } + + const transfers = data?.action?.arguments?.transfers; + if (!Array.isArray(transfers)) { + return; + } + + for (const transfer of transfers) { + if (this.isValidFilterValue(transfer?.token)) { + activeFilters.push({ key: 'token', value: transfer.token }); + } + } + } + + private static isValidFilterValue(value: any) { + return value !== undefined && value !== null && value !== ''; + } + + private static buildRoomKeys(prefix: string, activeFilters: { key: string; value: any }[]) { + const rooms: string[] = []; + const subsetCount = 1 << activeFilters.length; // 2^N combinations + + // Start from 1 to ignore the empty set + for (let mask = 1; mask < subsetCount; mask++) { + const currentSubset: Record = {}; + let skipIteration = false; + + for (let bit = 0; bit < activeFilters.length; bit++) { + // Check the bit to decide whether to include the element in the subset + if ((mask & (1 << bit)) > 0) { + if (currentSubset.hasOwnProperty(activeFilters[bit].key)) { + skipIteration = true; + continue; // Skip duplicate keys + } + const item = activeFilters[bit]; + currentSubset[item.key] = item.value; + } + } + if (!skipIteration) { + rooms.push(`${prefix}${this.deterministicStringify(currentSubset)}`); + } + } + + return rooms; + } + + static deterministicStringify(obj: Record): string { + return JSON.stringify( + Object.keys(obj) + .sort() + .reduce((result, key) => { + result[key] = obj[key]; + return result; + }, {} as Record), + ); + } + + private static getKeys(targetClass: Function): string[] { + switch (targetClass) { + case TransactionCustomSubscribePayload: + return TransactionCustomSubscribePayload.getClassFields(); + case EventsCustomSubscribePayload: + return EventsCustomSubscribePayload.getClassFields(); + case TransferCustomSubscribePayload: + return TransferCustomSubscribePayload.getClassFields(); + default: + console.warn(`RoomKeyGenerator: No manual key mapping found for class ${targetClass.name}`); + return []; + } + } +} diff --git a/src/crons/websocket/transaction.custom.gateway.ts b/src/crons/websocket/transaction.custom.gateway.ts new file mode 100644 index 000000000..5971a4185 --- /dev/null +++ b/src/crons/websocket/transaction.custom.gateway.ts @@ -0,0 +1,88 @@ +import { WebSocketGateway, WebSocketServer, SubscribeMessage, ConnectedSocket, MessageBody } from '@nestjs/websockets'; +import { Server, Socket } from 'socket.io'; +import { TransactionService } from '../../endpoints/transactions/transaction.service'; +import { TransactionFilter } from '../../endpoints/transactions/entities/transaction.filter'; +import { QueryPagination } from 'src/common/entities/query.pagination'; +import { WsValidationPipe } from 'src/utils/ws-validation.pipe'; +import { WebsocketExceptionsFilter } from 'src/utils/ws-exceptions.filter'; +import { UseFilters, UseInterceptors } from '@nestjs/common'; +import { OriginLogger } from '@multiversx/sdk-nestjs-common'; +import { TransactionCustomSubscribePayload } from 'src/endpoints/transactions/entities/dtos/transaction.custom.subscribe'; +import { RoomKeyGenerator } from './room.key.generator'; +import { Transaction } from 'src/endpoints/transactions/entities/transaction'; +import { LockingGuardInterceptor } from 'src/utils/locking.guard.interceptor'; + +@UseFilters(WebsocketExceptionsFilter) +@WebSocketGateway({ cors: { origin: '*' }, path: '/ws/subscription' }) +export class TransactionsCustomGateway { + private readonly logger = new OriginLogger(TransactionsCustomGateway.name); + static keyPrefix = 'custom-tx-'; + @WebSocketServer() + server!: Server; + + constructor( + private readonly transactionService: TransactionService, + ) { } + + @UseInterceptors(LockingGuardInterceptor) + @SubscribeMessage('subscribeCustomTransactions') + async handleCustomSubscription( + @ConnectedSocket() client: Socket, + @MessageBody(new WsValidationPipe()) payload: TransactionCustomSubscribePayload) { + + const filterIdentifier = RoomKeyGenerator.deterministicStringify(payload); + if (!client.rooms.has(`${TransactionsCustomGateway.keyPrefix}${filterIdentifier}`)) { + await client.join(`${TransactionsCustomGateway.keyPrefix}${filterIdentifier}`); + } + return { status: 'success' }; + } + + @SubscribeMessage('unsubscribeCustomTransactions') + async handleCustomUnsubscribe( + @ConnectedSocket() client: Socket, + @MessageBody(new WsValidationPipe()) payload: TransactionCustomSubscribePayload + ) { + const filterIdentifier = RoomKeyGenerator.deterministicStringify(payload); + const roomName = `${TransactionsCustomGateway.keyPrefix}${filterIdentifier}`; + + if (client.rooms.has(roomName)) { + await client.leave(roomName); + } + + return { status: 'unsubscribed' }; + } + + async pushTransactionsForTimestampMs(timestampMs: number): Promise { + try { + const allTransactions = await this.transactionService.getTransactions( + new TransactionFilter({ before: timestampMs, after: timestampMs }), + new QueryPagination({ size: 10000 }) // TODO: handle pagination with more than 10k txs + ); + + const txFilteredForBroadcast: Map = new Map(); + for (const transaction of allTransactions) { + const roomKeys = RoomKeyGenerator.generate( + TransactionsCustomGateway.keyPrefix, + transaction, + TransactionCustomSubscribePayload, + ); + + for (const roomKey of roomKeys) { + if (this.server.sockets.adapter.rooms.has(roomKey)) { + if (!txFilteredForBroadcast.has(roomKey)) { + txFilteredForBroadcast.set(roomKey, []); + } + txFilteredForBroadcast.get(roomKey)!.push(transaction); + } + } + } + + for (const [roomName] of txFilteredForBroadcast) { + this.server.to(roomName).emit("customTransactionUpdate", { transactions: txFilteredForBroadcast.get(roomName), timestampMs }); + } + } catch (error) { + this.logger.error(error); + } + } + +} diff --git a/src/crons/websocket/transaction.gateway.ts b/src/crons/websocket/transaction.gateway.ts index 85c3a76d1..016e6db86 100644 --- a/src/crons/websocket/transaction.gateway.ts +++ b/src/crons/websocket/transaction.gateway.ts @@ -7,19 +7,22 @@ import { TransactionQueryOptions } from '../../endpoints/transactions/entities/t import { WsValidationPipe } from 'src/utils/ws-validation.pipe'; import { TransactionSubscribePayload } from '../../endpoints/transactions/entities/dtos/transaction.subscribe'; import { WebsocketExceptionsFilter } from 'src/utils/ws-exceptions.filter'; -import { UseFilters } from '@nestjs/common'; +import { UseFilters, UseInterceptors } from '@nestjs/common'; import { OriginLogger } from '@multiversx/sdk-nestjs-common'; - +import { RoomKeyGenerator } from './room.key.generator'; +import { LockingGuardInterceptor } from 'src/utils/locking.guard.interceptor'; @UseFilters(WebsocketExceptionsFilter) @WebSocketGateway({ cors: { origin: '*' }, path: '/ws/subscription' }) export class TransactionsGateway { private readonly logger = new OriginLogger(TransactionsGateway.name); + static readonly keyPrefix = 'tx-'; @WebSocketServer() server!: Server; constructor(private readonly transactionService: TransactionService) { } + @UseInterceptors(LockingGuardInterceptor) @SubscribeMessage('subscribeTransactions') async handleSubscription( @ConnectedSocket() client: Socket, @@ -45,16 +48,35 @@ export class TransactionsGateway { TransactionFilter.validate(transactionFilter, payload.size || 25); const filterIdentifier = JSON.stringify(payload); - await client.join(`tx-${filterIdentifier}`); + const roomName = `${TransactionsGateway.keyPrefix}${filterIdentifier}`; + + if (!client.rooms.has(roomName)) { + await client.join(roomName); + } return { status: 'success' }; } + @SubscribeMessage('unsubscribeTransactions') + async handleUnsubscribe( + @ConnectedSocket() client: Socket, + @MessageBody(new WsValidationPipe()) payload: TransactionSubscribePayload + ) { + const filterIdentifier = RoomKeyGenerator.deterministicStringify(payload); + const roomName = `${TransactionsGateway.keyPrefix}${filterIdentifier}`; + + if (client.rooms.has(roomName)) { + await client.leave(roomName); + } + + return { status: 'unsubscribed' }; + } + async pushTransactionsForRoom(roomName: string): Promise { - if (!roomName.startsWith("tx-")) return; + if (!roomName.startsWith(TransactionsGateway.keyPrefix)) return; try { - const filterIdentifier = roomName.replace("tx-", ""); + const filterIdentifier = roomName.replace(TransactionsGateway.keyPrefix, ""); const filter = JSON.parse(filterIdentifier); const options = TransactionQueryOptions.applyDefaultOptions( diff --git a/src/crons/websocket/transfers.custom.gateway.ts b/src/crons/websocket/transfers.custom.gateway.ts new file mode 100644 index 000000000..3e9f64223 --- /dev/null +++ b/src/crons/websocket/transfers.custom.gateway.ts @@ -0,0 +1,105 @@ +import { WebSocketGateway, WebSocketServer, SubscribeMessage, ConnectedSocket, MessageBody } from '@nestjs/websockets'; +import { Server, Socket } from 'socket.io'; +import { TransactionFilter } from '../../endpoints/transactions/entities/transaction.filter'; +import { QueryPagination } from 'src/common/entities/query.pagination'; +import { WsValidationPipe } from 'src/utils/ws-validation.pipe'; +import { WebsocketExceptionsFilter } from 'src/utils/ws-exceptions.filter'; +import { UseFilters, UseInterceptors } from '@nestjs/common'; +import { OriginLogger } from '@multiversx/sdk-nestjs-common'; +import { RoomKeyGenerator } from './room.key.generator'; +import { Transaction } from 'src/endpoints/transactions/entities/transaction'; +import { LockingGuardInterceptor } from 'src/utils/locking.guard.interceptor'; +import { TransferService } from 'src/endpoints/transfers/transfer.service'; +import { TransferCustomSubscribePayload } from 'src/endpoints/websocket/entities/transfers.custom.payload'; +import { TransactionQueryOptions } from 'src/endpoints/transactions/entities/transactions.query.options'; + +@UseFilters(WebsocketExceptionsFilter) +@WebSocketGateway({ cors: { origin: '*' }, path: '/ws/subscription' }) +export class TransfersCustomGateway { + private readonly logger = new OriginLogger(TransfersCustomGateway.name); + static keyPrefix = 'custom-transfer-'; + @WebSocketServer() + server!: Server; + + constructor( + private readonly transferService: TransferService, + ) { } + + @UseInterceptors(LockingGuardInterceptor) + @SubscribeMessage('subscribeCustomTransfers') + async handleCustomSubscription( + @ConnectedSocket() client: Socket, + @MessageBody(new WsValidationPipe()) payload: TransferCustomSubscribePayload) { + + const filterIdentifier = RoomKeyGenerator.deterministicStringify(payload); + if (!client.rooms.has(`${TransfersCustomGateway.keyPrefix}${filterIdentifier}`)) { + await client.join(`${TransfersCustomGateway.keyPrefix}${filterIdentifier}`); + } + return { status: 'success' }; + } + + @SubscribeMessage('unsubscribeCustomTransfers') + async handleCustomUnsubscribe( + @ConnectedSocket() client: Socket, + @MessageBody(new WsValidationPipe()) payload: TransferCustomSubscribePayload + ) { + const filterIdentifier = RoomKeyGenerator.deterministicStringify(payload); + const roomName = `${TransfersCustomGateway.keyPrefix}${filterIdentifier}`; + + if (client.rooms.has(roomName)) { + await client.leave(roomName); + } + + return { status: 'unsubscribed' }; + } + + async pushTransfersForTimestampMs(timestampMs: number): Promise { + try { + const options = new TransactionQueryOptions({ withScamInfo: false, withUsername: true, withBlockInfo: false, withLogs: false, withOperations: false, withActionTransferValue: false, withTxsOrder: false }); + const allTransfers = await this.transferService.getTransfers( + new TransactionFilter({ before: timestampMs, after: timestampMs, withTxsRelayedByAddress: true }), + new QueryPagination({ size: 10000 }), // TODO: handle pagination with more than 10k txs + options, + ); + + const transfersFilteredForBroadcast: Map = new Map(); + + for (const transfer of allTransfers) { + const roomKeys = RoomKeyGenerator.generate( + TransfersCustomGateway.keyPrefix, + transfer, + TransferCustomSubscribePayload, + ); + + for (const roomKey of roomKeys) { + const substitutions = TransferCustomSubscribePayload.getFieldsSubstitutions(); + for (const [key, substituteFields] of Object.entries(substitutions)) { + for (const substituteField of substituteFields) { + const substituteRoomKey = roomKey.replace(`"${substituteField}":`, `"${key}":`); + if (this.server.sockets.adapter.rooms.has(substituteRoomKey)) { + if (!transfersFilteredForBroadcast.has(substituteRoomKey)) { + transfersFilteredForBroadcast.set(substituteRoomKey, []); + } + transfersFilteredForBroadcast.get(substituteRoomKey)!.push(transfer); + } + } + } + + if (this.server.sockets.adapter.rooms.has(roomKey)) { + if (!transfersFilteredForBroadcast.has(roomKey)) { + transfersFilteredForBroadcast.set(roomKey, []); + } + transfersFilteredForBroadcast.get(roomKey)!.push(transfer); + } + } + } + + for (const [roomName] of transfersFilteredForBroadcast) { + this.server.to(roomName).emit("customTransferUpdate", { transfers: transfersFilteredForBroadcast.get(roomName)?.distinct(), timestampMs }); + } + } catch (error) { + this.logger.error(error); + } + } + +} diff --git a/src/crons/websocket/websocket.cron.service.ts b/src/crons/websocket/websocket.cron.service.ts index 9aae69097..582469f0d 100644 --- a/src/crons/websocket/websocket.cron.service.ts +++ b/src/crons/websocket/websocket.cron.service.ts @@ -1,18 +1,31 @@ -import { Injectable } from '@nestjs/common'; -import { Cron } from '@nestjs/schedule'; +import { Injectable, OnModuleInit } from '@nestjs/common'; +import { Cron, SchedulerRegistry } from '@nestjs/schedule'; import { TransactionsGateway } from './transaction.gateway'; import { BlocksGateway } from 'src/crons/websocket/blocks.gateway'; import { NetworkGateway } from 'src/crons/websocket/network.gateway'; -import { Lock } from "@multiversx/sdk-nestjs-common"; +import { Lock, Locker } from "@multiversx/sdk-nestjs-common"; import { PoolGateway } from 'src/crons/websocket/pool.gateway'; import { EventsGateway } from 'src/crons/websocket/events.gateway'; import { WebSocketGateway, WebSocketServer } from '@nestjs/websockets'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { MetricsEvents } from 'src/utils/metrics-events.constants'; import { Server } from 'socket.io'; +import { CacheService } from '@multiversx/sdk-nestjs-cache'; +import { CacheInfo } from 'src/utils/cache.info'; +import { RoundService } from 'src/endpoints/rounds/round.service'; +import { RoundFilter } from 'src/endpoints/rounds/entities/round.filter'; +import { ElasticQuery, ElasticService, QueryType } from '@multiversx/sdk-nestjs-elastic'; +import { NetworkService } from 'src/endpoints/network/network.service'; +import { Stats } from 'src/endpoints/network/entities/stats'; +import { TransactionsCustomGateway } from './transaction.custom.gateway'; +import { ConnectionHandler } from './connection.handler'; +import { EventsCustomGateway } from './events.custom.gateway'; +import { TransfersCustomGateway } from './transfers.custom.gateway'; +import { ApiConfigService } from 'src/common/api-config/api.config.service'; + @Injectable() @WebSocketGateway({ cors: { origin: '*' }, path: '/ws/subscription' }) -export class WebsocketCronService { +export class WebsocketCronService implements OnModuleInit { @WebSocketServer() server!: Server; @@ -23,50 +36,209 @@ export class WebsocketCronService { private readonly poolGateway: PoolGateway, private readonly eventsGateway: EventsGateway, private readonly eventEmitter: EventEmitter2, + private readonly cacheService: CacheService, + private readonly roundService: RoundService, + private readonly elasticService: ElasticService, + private readonly networkService: NetworkService, + private readonly transactionsCustomGateway: TransactionsCustomGateway, + private readonly eventsCustomGateway: EventsCustomGateway, + private readonly connectionHandler: ConnectionHandler, + private readonly transfersCustomGateway: TransfersCustomGateway, + private readonly apiConfigService: ApiConfigService, + private readonly schedulerRegistry: SchedulerRegistry, ) { } - @Cron('*/1 * * * * *') - handleWebsocketMetrics() { - const connectedClients = this.server.sockets.sockets.size ?? 0; - // TODO: add more metrics in the future - // const subscriptions: Record = {}; - // this.server.sockets.adapter.rooms.forEach((socketsSet, roomName) => { - // subscriptions[roomName] = socketsSet.size; - // }); + onModuleInit() { + const intervalMs = this.apiConfigService.getWebsocketSubscriptionBroadcastIntervalMs(); - this.eventEmitter.emit(MetricsEvents.SetWebsocketMetrics, { - connectedClients, - }); + this.registerDynamicInterval( + 'push-transactions', + intervalMs, + 'Push transactions to subscribers', + () => this.handleTransactionsUpdate() + ); + + this.registerDynamicInterval( + 'push-blocks', + intervalMs, + 'Push blocks to subscribers', + () => this.handleBlocksUpdate() + ); + + this.registerDynamicInterval( + 'push-stats', + intervalMs, + 'Push stats to subscribers', + () => this.handleStatsUpdate() + ); + + this.registerDynamicInterval( + 'push-pool', + intervalMs, + 'Push pool transactions to subscribers', + () => this.handlePoolUpdate() + ); + + this.registerDynamicInterval( + 'push-events', + intervalMs, + 'Push events to subscribers', + () => this.handleEventsUpdate() + ); + + this.registerDynamicInterval( + 'push-custom-data', + intervalMs, + 'Push custom data to subscribers', + () => this.handleCustomDataUpdate() + ); + + + } + + private registerDynamicInterval(name: string, ms: number, lockMessage: string, callback: () => Promise) { + const interval = setInterval(async () => { + await Locker.lock(lockMessage, async () => { + await callback(); + }, true); + }, ms); + + this.schedulerRegistry.addInterval(name, interval); } - @Cron('*/6 * * * * *') - @Lock({ name: 'Push transactions to subscribers', verbose: true }) async handleTransactionsUpdate() { await this.transactionsGateway.pushTransactions(); } - @Cron('*/6 * * * * *') - @Lock({ name: 'Push blocks to subscribers', verbose: true }) async handleBlocksUpdate() { await this.blocksGateway.pushBlocks(); } - @Cron('*/6 * * * * *') - @Lock({ name: 'Push stats to subscribers', verbose: true }) async handleStatsUpdate() { await this.networkGateway.pushStats(); } - @Cron('*/6 * * * * *') - @Lock({ name: 'Push pool transactions to subscribers', verbose: true }) - async handlePoolTransactions() { + async handlePoolUpdate() { await this.poolGateway.pushPool(); } - @Cron('*/6 * * * * *') - @Lock({ name: 'Push events to subscribers', verbose: true }) + async handleEventsUpdate() { await this.eventsGateway.pushEvents(); } + + async handleCustomDataUpdate() { + if (this.connectionHandler.hasSubscriptionsWithPrefixes([TransactionsCustomGateway.keyPrefix, TransfersCustomGateway.keyPrefix, EventsCustomGateway.keyPrefix]) === false) { + this.cacheService.deleteLocal(CacheInfo.WsTimestampMsToProcess().key); + return; + } + + const latestRoundOnChainData = await this.getLatestRoundOnChainData(); + latestRoundOnChainData.timestampMs = latestRoundOnChainData.timestampMs ?? latestRoundOnChainData.timestamp * 1000; + + let roundToProcessTimestampMs = await this.cacheService.getOrSetLocal( + CacheInfo.WsTimestampMsToProcess().key, + async () => await Promise.resolve(latestRoundOnChainData.timestampMs ?? latestRoundOnChainData.timestamp * 1000), + CacheInfo.WsTimestampMsToProcess().ttl, + ); + + const stats = await this.networkService.getStats(); + + const pollingDelay = stats.refreshRate / 2; + const pollingMaxAttempts = 10; + while (roundToProcessTimestampMs <= latestRoundOnChainData.timestampMs) { + await this.pollUntil(async () => await this.isElasticDataAvailableForTimestampMs(roundToProcessTimestampMs, stats), pollingDelay, pollingMaxAttempts); + + // call gateways to process logic for custom subscriptions + await Promise.all([ + this.transactionsCustomGateway.pushTransactionsForTimestampMs(roundToProcessTimestampMs), + this.eventsCustomGateway.pushEventsForTimestampMs(roundToProcessTimestampMs), + this.transfersCustomGateway.pushTransfersForTimestampMs(roundToProcessTimestampMs), + ]); + roundToProcessTimestampMs += stats.refreshRate; + } + this.cacheService.setLocal( + CacheInfo.WsTimestampMsToProcess().key, + roundToProcessTimestampMs, + CacheInfo.WsTimestampMsToProcess().ttl, + ); + } + + @Cron('*/10 * * * * *') + @Lock({ name: 'Push websocket subscriptions metrics', verbose: true }) + handleWebsocketMetrics() { + const connectedClients = this.server.sockets.sockets.size ?? 0; + + // Efficient unique-listener counts per topic + const adapter = this.server.sockets.adapter as any; + const sids: Map> = adapter?.sids ?? new Map(); + + const topicPrefixes: Array<{ key: string; prefix?: string; room?: string }> = [ + { key: 'tx', prefix: TransactionsGateway.keyPrefix }, + { key: 'customTx', prefix: TransactionsCustomGateway.keyPrefix }, + { key: 'events', prefix: EventsGateway.keyPrefix }, + { key: 'customEvents', prefix: EventsCustomGateway.keyPrefix }, + { key: 'blocks', prefix: BlocksGateway.keyPrefix }, + { key: 'pool', prefix: PoolGateway.keyPrefix }, + { key: 'stats', room: 'statsRoom' }, + ]; + + const topics: Record = {}; + for (const { key } of topicPrefixes) topics[key] = 0; + + // Count unique sockets per prefix-based topic by scanning socket -> rooms map once + if (sids && sids.size > 0) { + for (const [, rooms] of sids) { + // Track whether this socket has been counted for a given topic key + const matched: Record = {}; + + for (const roomName of rooms) { + for (const { key, prefix } of topicPrefixes) { + if (!prefix || matched[key]) continue; + if (roomName.startsWith(prefix)) { + topics[key] += 1; + matched[key] = true; + } + } + } + } + } + + // Handle exact-room topics (like statsRoom) directly from rooms map + const rooms: Map> = adapter?.rooms ?? new Map(); + const statsRoomSet = rooms.get('statsRoom'); + if (statsRoomSet) { + topics['stats'] = statsRoomSet.size; + } + + this.eventEmitter.emit(MetricsEvents.SetWebsocketMetrics, { + connectedClients, + topics, + }); + } + + private async getLatestRoundOnChainData() { + const rounds = await this.roundService.getRounds(new RoundFilter({ size: 1 })); + return rounds[0]; + } + + private async isElasticDataAvailableForTimestampMs(timestampMs: number, networkStats: Stats) { + const nextRoundTimestampMs = timestampMs + networkStats.refreshRate; + + const rounds = await this.elasticService.getCount( + 'rounds', + ElasticQuery.create().withMustCondition(QueryType.Match('timestampMs', nextRoundTimestampMs)) + ); + + return rounds === networkStats.shards + 1; // +1 for metachain + } + + async pollUntil(conditionFn: () => Promise, intervalMs = 1000, maxAttempts = 30) { + let attempts = 0; + while (!await conditionFn()) { + if (++attempts >= maxAttempts) throw new Error('Polling timeout exceeded'); + await new Promise(r => setTimeout(r, intervalMs)); + } + } } diff --git a/src/crons/websocket/websocket.subscription.module.ts b/src/crons/websocket/websocket.subscription.module.ts index 89276d50d..750cccfd9 100644 --- a/src/crons/websocket/websocket.subscription.module.ts +++ b/src/crons/websocket/websocket.subscription.module.ts @@ -12,6 +12,12 @@ import { TransactionsGateway } from './transaction.gateway'; import { PoolGateway } from './pool.gateway'; import { EventsGateway } from './events.gateway'; import { ConnectionHandler } from './connection.handler'; +import { RoundModule } from 'src/endpoints/rounds/round.module'; +import { TransactionsCustomGateway } from './transaction.custom.gateway'; +import { EventsCustomGateway } from './events.custom.gateway'; +import { ApiConfigModule } from 'src/common/api-config/api.config.module'; +import { TransfersCustomGateway } from './transfers.custom.gateway'; +import { TransferModule } from 'src/endpoints/transfers/transfer.module'; @Module({ imports: [ @@ -21,6 +27,9 @@ import { ConnectionHandler } from './connection.handler'; NetworkModule, PoolModule, EventsModule, + RoundModule, + TransferModule, + ApiConfigModule, ], providers: [ WebsocketCronService, @@ -30,6 +39,9 @@ import { ConnectionHandler } from './connection.handler'; TransactionsGateway, PoolGateway, EventsGateway, + TransactionsCustomGateway, + EventsCustomGateway, + TransfersCustomGateway, ], }) export class WebsocketSubscriptionModule { } diff --git a/src/endpoints/accounts/account.controller.ts b/src/endpoints/accounts/account.controller.ts index 0c23f88e4..40910e8a7 100644 --- a/src/endpoints/accounts/account.controller.ts +++ b/src/endpoints/accounts/account.controller.ts @@ -59,6 +59,7 @@ import { MexPairType } from '../mex/entities/mex.pair.type'; import { NftSubType } from '../nfts/entities/nft.sub.type'; import { AccountContract } from './entities/account.contract'; import { AccountFetchOptions } from './entities/account.fetch.options'; +import { TimestampParsePipe } from 'src/utils/timestamp.parse.pipe'; @Controller() @ApiTags('accounts') @@ -881,8 +882,8 @@ export class AccountController { @ApiQuery({ name: 'function', description: 'Filter transactions by function name', required: false }) @ApiQuery({ name: 'order', description: 'Sort order (asc/desc)', required: false, enum: SortOrder }) @ApiQuery({ name: 'fields', description: 'List of fields to filter by', required: false, isArray: true, style: 'form', explode: false }) - @ApiQuery({ name: 'before', description: 'Before timestamp', required: false }) - @ApiQuery({ name: 'after', description: 'After timestamp', required: false }) + @ApiQuery({ name: 'before', description: 'Before timestamp or timestampMs', required: false }) + @ApiQuery({ name: 'after', description: 'After timestamp or timestampMs', required: false }) @ApiQuery({ name: 'round', description: 'Round number', required: false }) @ApiQuery({ name: 'withScResults', description: 'Return scResults for transactions. When "withScresults" parameter is applied, complexity estimation is 200', required: false }) @ApiQuery({ name: 'withOperations', description: 'Return operations for transactions. When "withOperations" parameter is applied, complexity estimation is 200', required: false }) @@ -909,8 +910,8 @@ export class AccountController { @Query('hashes', ParseArrayPipe) hashes?: string[], @Query('status', new ParseEnumPipe(TransactionStatus)) status?: TransactionStatus, @Query('function', new ParseArrayPipe(new ParseArrayPipeOptions({ allowEmptyString: true }))) functions?: string[], - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('round', ParseIntPipe) round?: number, @Query('order', new ParseEnumPipe(SortOrder)) order?: SortOrder, @Query('fields', ParseArrayPipe) fields?: string[], @@ -963,8 +964,8 @@ export class AccountController { @ApiQuery({ name: 'hashes', description: 'Filter by a comma-separated list of transaction hashes', required: false }) @ApiQuery({ name: 'status', description: 'Status of the transaction (success / pending / invalid / fail)', required: false, enum: TransactionStatus }) @ApiQuery({ name: 'function', description: 'Filter transactions by function name', required: false }) - @ApiQuery({ name: 'before', description: 'Before timestamp', required: false }) - @ApiQuery({ name: 'after', description: 'After timestamp', required: false }) + @ApiQuery({ name: 'before', description: 'Before timestamp or timestampMs', required: false }) + @ApiQuery({ name: 'after', description: 'After timestamp or timestampMs', required: false }) @ApiQuery({ name: 'round', description: 'Round number', required: false }) @ApiQuery({ name: 'senderOrReceiver', description: 'One address that current address interacted with', required: false }) @ApiQuery({ name: 'isRelayed', description: 'Returns isRelayed transactions details', required: false, type: Boolean }) @@ -981,8 +982,8 @@ export class AccountController { @Query('hashes', ParseArrayPipe) hashes?: string[], @Query('status', new ParseEnumPipe(TransactionStatus)) status?: TransactionStatus, @Query('function', new ParseArrayPipe(new ParseArrayPipeOptions({ allowEmptyString: true }))) functions?: string[], - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('round', ParseIntPipe) round?: number, @Query('senderOrReceiver', ParseAddressPipe) senderOrReceiver?: string, @Query('isRelayed', ParseBoolPipe) isRelayed?: boolean, @@ -1027,8 +1028,8 @@ export class AccountController { @ApiQuery({ name: 'status', description: 'Status of the transaction (success / pending / invalid / fail)', required: false, enum: TransactionStatus }) @ApiQuery({ name: 'function', description: 'Filter transactions by function name', required: false }) @ApiQuery({ name: 'order', description: 'Sort order (asc/desc)', required: false, enum: SortOrder }) - @ApiQuery({ name: 'before', description: 'Before timestamp', required: false }) - @ApiQuery({ name: 'after', description: 'After timestamp', required: false }) + @ApiQuery({ name: 'before', description: 'Before timestamp or timestampMs', required: false }) + @ApiQuery({ name: 'after', description: 'After timestamp or timestampMs', required: false }) @ApiQuery({ name: 'round', description: 'Round number', required: false }) @ApiQuery({ name: 'fields', description: 'List of fields to filter by', required: false, isArray: true, style: 'form', explode: false }) @ApiQuery({ name: 'relayer', description: 'Address of the relayer', required: false }) @@ -1055,8 +1056,8 @@ export class AccountController { @Query('hashes', ParseArrayPipe) hashes?: string[], @Query('status', new ParseEnumPipe(TransactionStatus)) status?: TransactionStatus, @Query('function', new ParseArrayPipe(new ParseArrayPipeOptions({ allowEmptyString: true }))) functions?: string[], - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('round', ParseIntPipe) round?: number, @Query('fields', ParseArrayPipe) fields?: string[], @Query('order', new ParseEnumPipe(SortOrder)) order?: SortOrder, @@ -1114,8 +1115,8 @@ export class AccountController { @ApiQuery({ name: 'hashes', description: 'Filter by a comma-separated list of transfer hashes', required: false }) @ApiQuery({ name: 'status', description: 'Status of the transaction (success / pending / invalid / fail)', required: false, enum: TransactionStatus }) @ApiQuery({ name: 'function', description: 'Filter transfers by function name', required: false }) - @ApiQuery({ name: 'before', description: 'Before timestamp', required: false }) - @ApiQuery({ name: 'after', description: 'After timestamp', required: false }) + @ApiQuery({ name: 'before', description: 'Before timestamp or timestampMs', required: false }) + @ApiQuery({ name: 'after', description: 'After timestamp or timestampMs', required: false }) @ApiQuery({ name: 'round', description: 'Round number', required: false }) @ApiQuery({ name: 'isScCall', description: 'Returns sc call transactions details', required: false, type: Boolean }) @ApiQuery({ name: 'senderOrReceiver', description: 'One address that current address interacted with', required: false }) @@ -1132,8 +1133,8 @@ export class AccountController { @Query('hashes', ParseArrayPipe) hashes?: string[], @Query('status', new ParseEnumPipe(TransactionStatus)) status?: TransactionStatus, @Query('function', new ParseArrayPipe(new ParseArrayPipeOptions({ allowEmptyString: true }))) functions?: string[], - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('round', ParseIntPipe) round?: number, @Query('senderOrReceiver', ParseAddressPipe) senderOrReceiver?: string, @Query('isScCall', ParseBoolPipe) isScCall?: boolean, @@ -1174,8 +1175,8 @@ export class AccountController { @Query('hashes', ParseArrayPipe) hashes?: string[], @Query('status', new ParseEnumPipe(TransactionStatus)) status?: TransactionStatus, @Query('function', new ParseArrayPipe(new ParseArrayPipeOptions({ allowEmptyString: true }))) functions?: string[], - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('round', ParseIntPipe) round?: number, @Query('senderOrReceiver', ParseAddressPipe) senderOrReceiver?: string, @Query('withRefunds', ParseBoolPipe) withRefunds?: boolean, @@ -1309,15 +1310,15 @@ export class AccountController { @ApiOperation({ summary: 'Account history', description: 'Return account EGLD balance history' }) @ApiQuery({ name: 'from', description: 'Number of items to skip for the result set', required: false }) @ApiQuery({ name: 'size', description: 'Number of items to retrieve', required: false }) - @ApiQuery({ name: 'before', description: 'Before timestamp', required: false }) - @ApiQuery({ name: 'after', description: 'After timestamp', required: false }) + @ApiQuery({ name: 'before', description: 'Before timestamp or timestampMs', required: false }) + @ApiQuery({ name: 'after', description: 'After timestamp or timestampMs', required: false }) @ApiOkResponse({ type: [AccountHistory] }) getAccountHistory( @Param('address', ParseAddressPipe) address: string, @Query('from', new DefaultValuePipe(0), ParseIntPipe) from: number, @Query('size', new DefaultValuePipe(25), ParseIntPipe) size: number, - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, ): Promise { return this.accountService.getAccountHistory( address, @@ -1327,13 +1328,13 @@ export class AccountController { @Get("/accounts/:address/history/count") @ApiOperation({ summary: 'Account history count', description: 'Return account EGLD balance history count' }) - @ApiQuery({ name: 'before', description: 'Before timestamp', required: false }) - @ApiQuery({ name: 'after', description: 'After timestamp', required: false }) + @ApiQuery({ name: 'before', description: 'Before timestamp or timestampMs', required: false }) + @ApiQuery({ name: 'after', description: 'After timestamp or timestampMs', required: false }) @ApiOkResponse({ type: Number }) getAccountHistoryCount( @Param('address', ParseAddressPipe) address: string, - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, ): Promise { return this.accountService.getAccountHistoryCount( address, @@ -1342,14 +1343,14 @@ export class AccountController { @Get("/accounts/:address/history/:tokenIdentifier/count") @ApiOperation({ summary: 'Account token history count', description: 'Return account token balance history count' }) - @ApiQuery({ name: 'before', description: 'Before timestamp', required: false }) - @ApiQuery({ name: 'after', description: 'After timestamp', required: false }) + @ApiQuery({ name: 'before', description: 'Before timestamp or timestampMs', required: false }) + @ApiQuery({ name: 'after', description: 'After timestamp or timestampMs', required: false }) @ApiOkResponse({ type: Number }) async getAccountTokenHistoryCount( @Param('address', ParseAddressPipe) address: string, @Param('tokenIdentifier', ParseTokenOrNftPipe) tokenIdentifier: string, - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, ): Promise { const isToken = await this.tokenService.isToken(tokenIdentifier) || await this.collectionService.isCollection(tokenIdentifier) || await this.nftService.isNft(tokenIdentifier); if (!isToken) { @@ -1365,8 +1366,8 @@ export class AccountController { @ApiOperation({ summary: 'Account esdts history', description: 'Returns account esdts balance history' }) @ApiQuery({ name: 'from', description: 'Number of items to skip for the result set', required: false }) @ApiQuery({ name: 'size', description: 'Number of items to retrieve', required: false }) - @ApiQuery({ name: 'before', description: 'Before timestamp', required: false }) - @ApiQuery({ name: 'after', description: 'After timestamp', required: false }) + @ApiQuery({ name: 'before', description: 'Before timestamp or timestampMs', required: false }) + @ApiQuery({ name: 'after', description: 'After timestamp or timestampMs', required: false }) @ApiQuery({ name: 'identifier', description: 'Filter by multiple esdt identifiers, comma-separated', required: false }) @ApiQuery({ name: 'token', description: 'Token identifier', required: false }) @ApiOkResponse({ type: [AccountEsdtHistory] }) @@ -1374,8 +1375,8 @@ export class AccountController { @Param('address', ParseAddressPipe) address: string, @Query('from', new DefaultValuePipe(0), ParseIntPipe) from: number, @Query('size', new DefaultValuePipe(25), ParseIntPipe) size: number, - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('identifier', ParseArrayPipe) identifier?: string[], @Query('token', ParseTokenPipe) token?: string, ): Promise { @@ -1387,14 +1388,14 @@ export class AccountController { @Get("/accounts/:address/esdthistory/count") @ApiOperation({ summary: 'Account esdts history count', description: 'Returns account esdts balance history count' }) - @ApiQuery({ name: 'before', description: 'Before timestamp', required: false }) - @ApiQuery({ name: 'after', description: 'After timestamp', required: false }) + @ApiQuery({ name: 'before', description: 'Before timestamp or timestampMs', required: false }) + @ApiQuery({ name: 'after', description: 'After timestamp or timestampMs', required: false }) @ApiQuery({ name: 'identifier', description: 'Filter by multiple esdt identifiers, comma-separated', required: false }) @ApiQuery({ name: 'token', description: 'Token identifier', required: false }) async getAccountEsdtHistoryCount( @Param('address', ParseAddressPipe) address: string, - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('identifier', ParseArrayPipe) identifier?: string[], @Query('token', ParseTokenPipe) token?: string, ): Promise { @@ -1407,16 +1408,16 @@ export class AccountController { @ApiOperation({ summary: 'Account token history', description: 'Returns account token balance history' }) @ApiQuery({ name: 'from', description: 'Number of items to skip for the result set', required: false }) @ApiQuery({ name: 'size', description: 'Number of items to retrieve', required: false }) - @ApiQuery({ name: 'before', description: 'Before timestamp', required: false }) - @ApiQuery({ name: 'after', description: 'After timestamp', required: false }) + @ApiQuery({ name: 'before', description: 'Before timestamp or timestampMs', required: false }) + @ApiQuery({ name: 'after', description: 'After timestamp or timestampMs', required: false }) @ApiOkResponse({ type: [AccountEsdtHistory] }) async getAccountTokenHistory( @Param('address', ParseAddressPipe) address: string, @Param('tokenIdentifier', ParseTokenOrNftPipe) tokenIdentifier: string, @Query('from', new DefaultValuePipe(0), ParseIntPipe) from: number, @Query('size', new DefaultValuePipe(25), ParseIntPipe) size: number, - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, ): Promise { const isToken = await this.tokenService.isToken(tokenIdentifier) || await this.collectionService.isCollection(tokenIdentifier) || await this.nftService.isNft(tokenIdentifier); if (!isToken) { diff --git a/src/endpoints/accounts/account.service.ts b/src/endpoints/accounts/account.service.ts index 104a1b5be..5724cee62 100644 --- a/src/endpoints/accounts/account.service.ts +++ b/src/endpoints/accounts/account.service.ts @@ -22,6 +22,7 @@ import { GatewayService } from 'src/common/gateway/gateway.service'; import { IndexerService } from "src/common/indexer/indexer.service"; import { AccountAssets } from 'src/common/assets/entities/account.assets'; import { CacheInfo } from 'src/utils/cache.info'; +import { ConcurrencyUtils } from 'src/utils/concurrency.utils'; import { UsernameService } from '../usernames/username.service'; import { ContractUpgrades } from './entities/contract.upgrades'; import { AccountVerification } from './entities/account.verification'; @@ -356,36 +357,41 @@ export class AccountService { } } - for (const account of accounts) { - account.shard = AddressUtils.computeShard(AddressUtils.bech32Decode(account.address), shardCount); - account.assets = assets[account.address]; + await ConcurrencyUtils.executeWithConcurrencyLimit( + accounts, + async (account) => { + account.shard = AddressUtils.computeShard(AddressUtils.bech32Decode(account.address), shardCount); + account.assets = assets[account.address]; - if (options.withDeployInfo && AddressUtils.isSmartContractAddress(account.address)) { - const [deployedAt, deployTxHash] = await Promise.all([ - this.getAccountDeployedAt(account.address), - this.getAccountDeployedTxHash(account.address), - ]); + if (options.withDeployInfo && AddressUtils.isSmartContractAddress(account.address)) { + const [deployedAt, deployTxHash] = await Promise.all([ + this.getAccountDeployedAt(account.address), + this.getAccountDeployedTxHash(account.address), + ]); - account.deployedAt = deployedAt; - account.deployTxHash = deployTxHash; - } + account.deployedAt = deployedAt; + account.deployTxHash = deployTxHash; + } - if (options.withTxCount) { - account.txCount = await this.getAccountTxCount(account.address); - } + if (options.withTxCount) { + account.txCount = await this.getAccountTxCount(account.address); + } - if (options.withScrCount) { - account.scrCount = await this.getAccountScResults(account.address); - } + if (options.withScrCount) { + account.scrCount = await this.getAccountScResults(account.address); + } - if (options.withOwnerAssets && account.ownerAddress) { - account.ownerAssets = assets[account.ownerAddress]; - } + if (options.withOwnerAssets && account.ownerAddress) { + account.ownerAssets = assets[account.ownerAddress]; + } - if (verifiedAccounts && verifiedAccounts.includes(account.address)) { - account.isVerified = true; - } - } + if (verifiedAccounts && verifiedAccounts.includes(account.address)) { + account.isVerified = true; + } + }, + 6, + 'AccountService.getAccountsRaw', + ); return accounts; } diff --git a/src/endpoints/applications/application.controller.ts b/src/endpoints/applications/application.controller.ts index dd444f1cd..e9fcbd0bc 100644 --- a/src/endpoints/applications/application.controller.ts +++ b/src/endpoints/applications/application.controller.ts @@ -5,6 +5,7 @@ import { QueryPagination } from "src/common/entities/query.pagination"; import { ApplicationFilter } from "./entities/application.filter"; import { ParseIntPipe, ParseBoolPipe, ParseAddressPipe } from "@multiversx/sdk-nestjs-common"; import { Application } from "./entities/application"; +import { TimestampParsePipe } from "src/utils/timestamp.parse.pipe"; @Controller() @ApiTags('applications') @@ -18,14 +19,14 @@ export class ApplicationController { @ApiOkResponse({ type: [Application] }) @ApiQuery({ name: 'from', description: 'Number of items to skip for the result set', required: false }) @ApiQuery({ name: 'size', description: 'Number of items to retrieve', required: false }) - @ApiQuery({ name: 'before', description: 'Before timestamp', required: false }) - @ApiQuery({ name: 'after', description: 'After timestamp', required: false }) + @ApiQuery({ name: 'before', description: 'Before timestamp or timestampMs', required: false }) + @ApiQuery({ name: 'after', description: 'After timestamp or timestampMs', required: false }) @ApiQuery({ name: 'withTxCount', description: 'Include transaction count', required: false, type: Boolean }) async getApplications( @Query('from', new DefaultValuePipe(0), ParseIntPipe) from: number, @Query("size", new DefaultValuePipe(25), ParseIntPipe) size: number, - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('withTxCount', new ParseBoolPipe()) withTxCount?: boolean, ): Promise { const applicationFilter = new ApplicationFilter({ before, after, withTxCount }); @@ -41,8 +42,8 @@ export class ApplicationController { @ApiQuery({ name: 'before', description: 'Before timestamp', required: false }) @ApiQuery({ name: 'after', description: 'After timestamp', required: false }) async getApplicationsCount( - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, ): Promise { const filter = new ApplicationFilter({ before, after }); diff --git a/src/endpoints/blocks/entities/block.ts b/src/endpoints/blocks/entities/block.ts index aa6d63076..9728799e6 100644 --- a/src/endpoints/blocks/entities/block.ts +++ b/src/endpoints/blocks/entities/block.ts @@ -47,6 +47,9 @@ export class Block { @ApiProperty({ type: Number }) timestamp: number = 0; + @ApiProperty({ type: Number, nullable: true, required: false }) + timestampMs?: number; + @ApiProperty({ type: Number }) txCount: number = 0; @@ -68,7 +71,7 @@ export class Block { @ApiProperty({ type: BlockProofDto, nullable: true, required: false }) previousHeaderProof: BlockProofDto | undefined = undefined; - @ApiProperty( { type: String }) + @ApiProperty({ type: String }) reserved: string = ''; @ApiProperty({ type: BlockProofDto, nullable: true, required: false }) diff --git a/src/endpoints/collections/collection.controller.ts b/src/endpoints/collections/collection.controller.ts index a0700007b..a49f6483c 100644 --- a/src/endpoints/collections/collection.controller.ts +++ b/src/endpoints/collections/collection.controller.ts @@ -26,6 +26,7 @@ import { SortCollections } from "./entities/sort.collections"; import { ParseArrayPipeOptions } from "@multiversx/sdk-nestjs-common/lib/pipes/entities/parse.array.options"; import { TransferService } from "../transfers/transfer.service"; import { NftSubType } from "../nfts/entities/nft.sub.type"; +import { TimestampParsePipe } from "src/utils/timestamp.parse.pipe"; @Controller() @ApiTags('collections') @@ -47,8 +48,8 @@ export class CollectionController { @ApiQuery({ name: 'type', description: 'Filter by type (NonFungibleESDT/SemiFungibleESDT/MetaESDT)', required: false }) @ApiQuery({ name: 'subType', description: 'Filter by type (NonFungibleESDTv2/DynamicNonFungibleESDT/DynamicSemiFungibleESDT)', required: false }) @ApiQuery({ name: 'creator', description: 'Filter collections where the given address has a creator role', required: false, deprecated: true }) - @ApiQuery({ name: 'before', description: 'Return all collections before given timestamp', required: false, type: Number }) - @ApiQuery({ name: 'after', description: 'Return all collections after given timestamp', required: false, type: Number }) + @ApiQuery({ name: 'before', description: 'Return all collections before given timestamp or timestampMs', required: false, type: Number }) + @ApiQuery({ name: 'after', description: 'Return all collections after given timestamp or timestampMs', required: false, type: Number }) @ApiQuery({ name: 'canCreate', description: 'Filter by address with canCreate role', required: false }) @ApiQuery({ name: 'canBurn', description: 'Filter by address with canBurn role', required: false }) @ApiQuery({ name: 'canAddQuantity', description: 'Filter by address with canAddQuantity role', required: false }) @@ -66,8 +67,8 @@ export class CollectionController { @Query('type', new ParseEnumArrayPipe(NftType)) type?: NftType[], @Query('subType', new ParseEnumArrayPipe(NftSubType)) subType?: NftSubType[], @Query('creator', ParseAddressPipe) creator?: string, - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('canCreate', ParseAddressPipe) canCreate?: string, @Query('canBurn', ParseAddressPipe) canBurn?: string, @Query('canAddQuantity', ParseAddressPipe) canAddQuantity?: string, @@ -102,8 +103,8 @@ export class CollectionController { @ApiQuery({ name: 'search', description: 'Search by collection identifier', required: false }) @ApiQuery({ name: 'type', description: 'Filter by type (NonFungibleESDT/SemiFungibleESDT/MetaESDT)', required: false }) @ApiQuery({ name: 'creator', description: 'Filter collections where the given address has a creator role', required: false, deprecated: true }) - @ApiQuery({ name: 'before', description: 'Return all collections before given timestamp', required: false, type: Number }) - @ApiQuery({ name: 'after', description: 'Return all collections after given timestamp', required: false, type: Number }) + @ApiQuery({ name: 'before', description: 'Return all collections before given timestamp or timestampMs', required: false, type: Number }) + @ApiQuery({ name: 'after', description: 'Return all collections after given timestamp or timestampMs', required: false, type: Number }) @ApiQuery({ name: 'canCreate', description: 'Filter by address with canCreate role', required: false }) @ApiQuery({ name: 'canBurn', description: 'Filter by address with canBurn role', required: false }) @ApiQuery({ name: 'canAddQuantity', description: 'Filter by address with canAddQuantity role', required: false }) @@ -118,8 +119,8 @@ export class CollectionController { @Query('type', new ParseEnumArrayPipe(NftType)) type?: NftType[], @Query('subType', new ParseEnumArrayPipe(NftSubType)) subType?: NftSubType[], @Query('creator', ParseAddressPipe) creator?: string, - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('canCreate', ParseAddressPipe) canCreate?: string, @Query('canBurn', ParseAddressPipe) canBurn?: string, @Query('canAddQuantity', ParseAddressPipe) canAddQuantity?: string, @@ -150,8 +151,8 @@ export class CollectionController { @Query('search') search?: string, @Query('type', new ParseEnumArrayPipe(NftType)) type?: NftType[], @Query('creator', ParseAddressPipe) creator?: string, - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('canCreate', ParseAddressPipe) canCreate?: string, @Query('canBurn', ParseAddressPipe) canBurn?: string, @Query('canAddQuantity', ParseAddressPipe) canAddQuantity?: string, @@ -328,8 +329,8 @@ export class CollectionController { @ApiQuery({ name: 'hashes', description: 'Filter by a comma-separated list of transaction hashes', required: false }) @ApiQuery({ name: 'status', description: 'Status of the transaction (success / pending / invalid / fail)', required: false, enum: TransactionStatus }) @ApiQuery({ name: 'function', description: 'Filter transactions by function name', required: false }) - @ApiQuery({ name: 'before', description: 'Before timestamp', required: false }) - @ApiQuery({ name: 'after', description: 'After timestamp', required: false }) + @ApiQuery({ name: 'before', description: 'Before timestamp or timestampMs', required: false }) + @ApiQuery({ name: 'after', description: 'After timestamp or timestampMs', required: false }) @ApiQuery({ name: 'round', description: 'Filter by round number', required: false }) @ApiQuery({ name: 'order', description: 'Sort order (asc/desc)', required: false, enum: SortOrder }) @ApiQuery({ name: 'from', description: 'Number of items to skip for the result set', required: false }) @@ -352,8 +353,8 @@ export class CollectionController { @Query('hashes', ParseArrayPipe) hashes?: string[], @Query('status', new ParseEnumPipe(TransactionStatus)) status?: TransactionStatus, @Query('function', new ParseArrayPipe(new ParseArrayPipeOptions({ allowEmptyString: true }))) functions?: string[], - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('round', ParseIntPipe) round?: number, @Query('order', new ParseEnumPipe(SortOrder)) order?: SortOrder, @Query('withScResults', ParseBoolPipe) withScResults?: boolean, @@ -402,8 +403,8 @@ export class CollectionController { @ApiQuery({ name: 'miniBlockHash', description: 'Filter by miniblock hash', required: false }) @ApiQuery({ name: 'hashes', description: 'Filter by a comma-separated list of transaction hashes', required: false }) @ApiQuery({ name: 'status', description: 'Status of the transaction (success / pending / invalid / fail)', required: false, enum: TransactionStatus }) - @ApiQuery({ name: 'before', description: 'Before timestamp', required: false }) - @ApiQuery({ name: 'after', description: 'After timestamp', required: false }) + @ApiQuery({ name: 'before', description: 'Before timestamp or timestampMs', required: false }) + @ApiQuery({ name: 'after', description: 'After timestamp or timestampMs', required: false }) @ApiQuery({ name: 'round', description: 'Filter by round number', required: false }) @ApiQuery({ name: 'withRelayedScresults', description: 'If set to true, will include smart contract results that resemble relayed transactions', required: false, type: Boolean }) async getCollectionTransactionsCount( @@ -415,8 +416,8 @@ export class CollectionController { @Query('miniBlockHash', ParseBlockHashPipe) miniBlockHash?: string, @Query('hashes', ParseArrayPipe) hashes?: string[], @Query('status', new ParseEnumPipe(TransactionStatus)) status?: TransactionStatus, - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('round', ParseIntPipe) round?: number, @Query('withRelayedScresults', ParseBoolPipe) withRelayedScresults?: boolean, ) { @@ -454,8 +455,8 @@ export class CollectionController { @ApiQuery({ name: 'hashes', description: 'Filter by a comma-separated list of transaction hashes', required: false }) @ApiQuery({ name: 'status', description: 'Status of the transaction (success / pending / invalid / fail)', required: false, enum: TransactionStatus }) @ApiQuery({ name: 'function', description: 'Filter transactions by function name', required: false }) - @ApiQuery({ name: 'before', description: 'Before timestamp', required: false }) - @ApiQuery({ name: 'after', description: 'After timestamp', required: false }) + @ApiQuery({ name: 'before', description: 'Before timestamp or timestampMs', required: false }) + @ApiQuery({ name: 'after', description: 'After timestamp or timestampMs', required: false }) @ApiQuery({ name: 'round', description: 'Filter by round number', required: false }) @ApiQuery({ name: 'order', description: 'Sort order (asc/desc)', required: false, enum: SortOrder }) @ApiQuery({ name: 'from', description: 'Number of items to skip for the result set', required: false }) @@ -477,8 +478,8 @@ export class CollectionController { @Query('hashes', ParseArrayPipe) hashes?: string[], @Query('status', new ParseEnumPipe(TransactionStatus)) status?: TransactionStatus, @Query('function', new ParseArrayPipe(new ParseArrayPipeOptions({ allowEmptyString: true }))) functions?: string[], - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('round', ParseIntPipe) round?: number, @Query('order', new ParseEnumPipe(SortOrder)) order?: SortOrder, @Query('withScResults', ParseBoolPipe) withScResults?: boolean, @@ -523,8 +524,8 @@ export class CollectionController { @ApiQuery({ name: 'hashes', description: 'Filter by a comma-separated list of transfer hashes', required: false }) @ApiQuery({ name: 'status', description: 'Status of the transfer (success / pending / invalid / fail)', required: false, enum: TransactionStatus }) @ApiQuery({ name: 'function', description: 'Filter transfers by function name', required: false }) - @ApiQuery({ name: 'before', description: 'Before timestamp', required: false }) - @ApiQuery({ name: 'after', description: 'After timestamp', required: false }) + @ApiQuery({ name: 'before', description: 'Before timestamp or timestampMs', required: false }) + @ApiQuery({ name: 'after', description: 'After timestamp or timestampMs', required: false }) @ApiQuery({ name: 'round', description: 'Filter by round number', required: false }) async getCollectionTransfersCount( @Param('collection', ParseCollectionPipe) identifier: string, @@ -536,8 +537,8 @@ export class CollectionController { @Query('miniBlockHash', ParseBlockHashPipe) miniBlockHash?: string, @Query('hashes', ParseArrayPipe) hashes?: string[], @Query('status', new ParseEnumPipe(TransactionStatus)) status?: TransactionStatus, - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('round', ParseIntPipe) round?: number, ) { const isCollection = await this.collectionService.isCollection(identifier); diff --git a/src/endpoints/collections/collection.service.ts b/src/endpoints/collections/collection.service.ts index ce8378053..11cbc2098 100644 --- a/src/endpoints/collections/collection.service.ts +++ b/src/endpoints/collections/collection.service.ts @@ -29,6 +29,7 @@ import { CollectionLogo } from "./entities/collection.logo"; import { ScamInfo } from "src/common/entities/scam-info.dto"; import { NftType as ElasticNftType } from "src/common/indexer/entities/nft.type"; import { NftSubType } from "../nfts/entities/nft.sub.type"; +import { ConcurrencyUtils } from "src/utils/concurrency.utils"; @Injectable() export class CollectionService { @@ -50,10 +51,42 @@ export class CollectionService { } async getNftCollections(pagination: QueryPagination, filter: CollectionFilter): Promise { + if (this.isCacheableCollectionFilter(filter)) { + return await this.cachingService.getOrSet( + CacheInfo.Collections(pagination).key, + () => this.fetchAndProcessCollections(pagination, filter), + CacheInfo.Collections(pagination).ttl, + ); + } + + return await this.fetchAndProcessCollections(pagination, filter); + } + + private async fetchAndProcessCollections(pagination: QueryPagination, filter: CollectionFilter): Promise { const tokenCollections = await this.indexerService.getNftCollections(pagination, filter); return await this.processNftCollections(tokenCollections); } + private isCacheableCollectionFilter(filter: CollectionFilter): boolean { + return !filter.collection && + !(filter.identifiers && filter.identifiers.length > 0) && + !(filter.type && filter.type.length > 0) && + !(filter.subType && filter.subType.length > 0) && + !filter.search && + !filter.owner && + filter.before === undefined && + filter.after === undefined && + filter.canCreate === undefined && + filter.canBurn === undefined && + filter.canAddQuantity === undefined && + filter.canUpdateAttributes === undefined && + filter.canAddUri === undefined && + filter.canTransferRole === undefined && + filter.excludeMetaESDT === undefined && + filter.sort === undefined && + filter.order === undefined; + } + async getNftCollectionsByIds(identifiers: Array): Promise { const tokenCollections = await this.indexerService.getNftCollectionsByIds(identifiers); return await this.processNftCollections(tokenCollections); @@ -62,18 +95,41 @@ export class CollectionService { private async processNftCollections(tokenCollections: Collection[]): Promise { const collectionsIdentifiers = tokenCollections.map((collection) => collection.token); - const indexedCollections = new Map(); - for (const collection of tokenCollections) { - indexedCollections.set(collection.token, collection); - } + const collectionsAssets = await this.batchGetCollectionsAssets(collectionsIdentifiers); - const nftCollections: NftCollection[] = await this.applyPropertiesToCollections(collectionsIdentifiers); + const nftCollections: NftCollection[] = []; - for (const nftCollection of nftCollections) { - const indexedCollection = indexedCollections.get(nftCollection.collection); - if (indexedCollection) { - this.applyPropertiesToCollectionFromElasticSearch(nftCollection, indexedCollection); - } + for (const esCollection of tokenCollections) { + const identifierParts = esCollection.token.split('-'); + const ticker = identifierParts[0]; + const collectionBase = identifierParts.slice(0, 2).join('-'); + const assets = collectionsAssets[esCollection.token]; + const props = esCollection.properties; + + const isMetaESDT = esCollection.type === ElasticNftType.MetaESDT || esCollection.type === ElasticNftType.DynamicMetaESDT; + + const nftCollection = new NftCollection({ + name: esCollection.name, + collection: collectionBase, + ticker: ticker, + owner: esCollection.currentOwner, + assets: assets, + canFreeze: props?.canFreeze, + canWipe: props?.canWipe, + canPause: props?.canPause, + canTransferNftCreateRole: props?.canTransferNFTCreateRole, + canChangeOwner: props?.canChangeOwner, + canUpgrade: props?.canUpgrade, + canAddSpecialRoles: props?.canAddSpecialRoles, + decimals: isMetaESDT ? esCollection.numDecimals : undefined, + }); + + nftCollection.ticker = nftCollection.assets ? ticker : nftCollection.collection; + + // Apply additional ES fields (type, timestamp, counts, scamInfo) + this.applyPropertiesToCollectionFromElasticSearch(nftCollection, esCollection); + + nftCollections.push(nftCollection); } return nftCollections; @@ -113,6 +169,10 @@ export class CollectionService { } } + async buildCollectionsFromElasticData(esCollections: Collection[]): Promise { + return await this.processNftCollections(esCollections); + } + async applyPropertiesToCollections(collectionsIdentifiers: string[]): Promise { const nftCollections: NftCollection[] = []; @@ -159,30 +219,60 @@ export class CollectionService { } async batchGetCollectionsProperties(identifiers: string[]): Promise<{ [key: string]: TokenProperties | undefined }> { - if (this.apiConfigService.getCollectionPropertiesFromGateway()) { - return await this.getCollectionProperties(identifiers); + const result: { [key: string]: TokenProperties | undefined } = {}; + const chunks = this.splitIntoChunks(identifiers, 300); + + const chunkResults = await ConcurrencyUtils.executeWithConcurrencyLimit( + chunks, + async (chunk) => { + if (this.apiConfigService.getCollectionPropertiesFromGateway()) { + return await this.getCollectionProperties(chunk); + } + return await this.getEsdtProperties(chunk); + }, + 4, + 'CollectionService.batchGetCollectionsProperties' + ); + + for (const chunkResult of chunkResults) { + Object.assign(result, chunkResult); } - return await this.getEsdtProperties(identifiers); + return result; } async batchGetCollectionsAssets(identifiers: string[]): Promise<{ [key: string]: TokenAssets | undefined }> { const collectionsAssets: { [key: string]: TokenAssets | undefined } = {}; - const allAssets = await this.assetsService.getAllTokenAssets(); - - await this.cachingService.batchApplyAll( - identifiers, - identifier => CacheInfo.EsdtAssets(identifier).key, - identifier => Promise.resolve(allAssets[identifier]), - (identifier, properties) => collectionsAssets[identifier] = properties, - CacheInfo.EsdtAssets('').ttl + const chunks = this.splitIntoChunks(identifiers, 300); + + await ConcurrencyUtils.executeWithConcurrencyLimit( + chunks, + async (chunk) => { + await this.cachingService.batchApplyAll( + chunk, + identifier => CacheInfo.EsdtAssets(identifier).key, + identifier => Promise.resolve(allAssets[identifier]), + (identifier, properties) => collectionsAssets[identifier] = properties, + CacheInfo.EsdtAssets('').ttl + ); + }, + 4, + 'CollectionService.batchGetCollectionsAssets' ); return collectionsAssets; } async getNftCollectionCount(filter: CollectionFilter): Promise { + if (this.isCacheableCollectionFilter(filter)) { + return await this.cachingService.getOrSet( + CacheInfo.CollectionsCount.key, + async () => await this.indexerService.getNftCollectionCount(filter), + CacheInfo.CollectionsCount.ttl, + ); + } + return await this.indexerService.getNftCollectionCount(filter); } @@ -201,7 +291,11 @@ export class CollectionService { return undefined; } - return await this.assetsService.getCollectionRanks(identifier); + return await this.cachingService.getOrSet( + CacheInfo.CollectionRanksForIdentifier(identifier).key, + async () => await this.assetsService.getCollectionRanks(identifier), + CacheInfo.CollectionRanksForIdentifier(identifier).ttl, + ); } async getNftCollection(identifier: string): Promise { @@ -218,19 +312,15 @@ export class CollectionService { return undefined; } - const [collection] = await this.applyPropertiesToCollections([identifier]); + const [collection] = await this.buildCollectionsFromElasticData([elasticCollection]); if (!collection) { return undefined; } const collectionDetailed = ApiUtils.mergeObjects(new NftCollectionDetailed(), collection); - collectionDetailed.type = elasticCollection.type as NftType; - collectionDetailed.timestamp = elasticCollection.timestamp; - this.applyPropertiesToCollectionFromElasticSearch(collectionDetailed, elasticCollection); - - collectionDetailed.traits = await this.persistenceService.getCollectionTraits(identifier) ?? []; + collectionDetailed.traits = await this.getCollectionTraitsCached(identifier); await this.applyCollectionRoles(collectionDetailed, elasticCollection); @@ -238,7 +328,7 @@ export class CollectionService { } async applyCollectionRoles(collection: NftCollectionDetailed | TokenDetailed, elasticCollection: any) { - collection.roles = await this.getNftCollectionRolesFromGateway(elasticCollection); + collection.roles = await this.getCollectionRolesCached(elasticCollection.token, elasticCollection); const isTransferProhibitedByDefault = collection.roles?.some(x => x.canTransfer === true) === true; collection.canTransfer = !isTransferProhibitedByDefault; if (collection.canTransfer) { @@ -330,8 +420,19 @@ export class CollectionService { } async getCollectionCountForAddress(address: string, filter: CollectionFilter): Promise { - const collections = await this.getCollectionsForAddress(address, filter, new QueryPagination({ from: 0, size: 10000 })); + if (this.isDefaultAddressCollectionFilter(filter)) { + return await this.cachingService.getOrSet( + CacheInfo.CollectionCountForAddress(address).key, + () => this.computeCollectionCountForAddress(address, filter), + CacheInfo.CollectionCountForAddress(address).ttl, + ); + } + + return await this.computeCollectionCountForAddress(address, filter); + } + private async computeCollectionCountForAddress(address: string, filter: CollectionFilter): Promise { + const collections = await this.getCollectionsForAddress(address, filter, new QueryPagination({ from: 0, size: 10000 })); return collections.length; } @@ -351,25 +452,63 @@ export class CollectionService { } async getCollectionsForAddress(address: string, filter: CollectionFilter, pagination: QueryPagination): Promise { + if (this.isDefaultAddressCollectionFilter(filter)) { + const cacheInfo = CacheInfo.CollectionsForAddress(address, pagination); + return await this.cachingService.getOrSet( + cacheInfo.key, + () => this.fetchCollectionsForAddress(address, filter, pagination), + cacheInfo.ttl, + ); + } + + return await this.fetchCollectionsForAddress(address, filter, pagination); + } + + private async fetchCollectionsForAddress(address: string, filter: CollectionFilter, pagination: QueryPagination): Promise { const collectionsRaw = await this.indexerService.getCollectionsForAddress(address, filter, pagination); - const collections = await this.getNftCollections( - new QueryPagination({ from: 0, size: collectionsRaw.length }), - new CollectionFilter({ identifiers: collectionsRaw.map((x: any) => x.collection) }) - ); - const accountCollections = collections.map(collection => ApiUtils.mergeObjects(new NftCollectionAccount(), collection)); + if (collectionsRaw.length === 0) { + return []; + } - for (const collection of accountCollections) { - const item = collectionsRaw.find(x => x.collection === collection.collection); - if (item) { - collection.count = item.count; + const identifiers = collectionsRaw.map((x: any) => x.collection); + const collections = await this.getNftCollectionsByIds(identifiers); + + const collectionMap = new Map(); + for (const collection of collections) { + collectionMap.set(collection.collection, collection); + } + + const accountCollections: NftCollectionAccount[] = []; + for (const raw of collectionsRaw) { + const collection = collectionMap.get(raw.collection); + if (collection) { + const accountCollection = ApiUtils.mergeObjects(new NftCollectionAccount(), collection); + accountCollection.count = raw.count; + accountCollections.push(accountCollection); } } return accountCollections; } + private isDefaultAddressCollectionFilter(filter: CollectionFilter): boolean { + return !filter.search && + !(filter.type && filter.type.length > 0) && + !(filter.subType && filter.subType.length > 0) && + !filter.excludeMetaESDT && + !filter.collection; + } + async getCollectionCountForAddressWithRoles(address: string, filter: CollectionFilter): Promise { + if (this.isDefaultAddressCollectionFilter(filter)) { + return await this.cachingService.getOrSet( + CacheInfo.CollectionRolesCountForAddress(address).key, + () => this.esdtAddressService.getCollectionCountForAddressFromElastic(address, filter), + CacheInfo.CollectionRolesCountForAddress(address).ttl, + ); + } + return await this.esdtAddressService.getCollectionCountForAddressFromElastic(address, filter); } @@ -383,7 +522,7 @@ export class CollectionService { } async getLogoPng(identifier: string): Promise { - const collectionLogo = await this.getCollectionLogo(identifier); + const collectionLogo = await this.getCollectionLogoCached(identifier); if (!collectionLogo) { return; } @@ -392,7 +531,7 @@ export class CollectionService { } async getLogoSvg(identifier: string): Promise { - const collectionLogo = await this.getCollectionLogo(identifier); + const collectionLogo = await this.getCollectionLogoCached(identifier); if (!collectionLogo) { return; } @@ -427,4 +566,40 @@ export class CollectionService { return collectionsProperties; } + + private splitIntoChunks(items: T[], chunkSize: number): T[][] { + if (chunkSize <= 0) { + return [items]; + } + + const chunks: T[][] = []; + for (let i = 0; i < items.length; i += chunkSize) { + chunks.push(items.slice(i, i + chunkSize)); + } + return chunks; + } + + private async getCollectionTraitsCached(identifier: string): Promise { + return await this.cachingService.getOrSet( + CacheInfo.CollectionTraits(identifier).key, + async () => await this.persistenceService.getCollectionTraits(identifier) ?? [], + CacheInfo.CollectionTraits(identifier).ttl, + ); + } + + private async getCollectionRolesCached(identifier: string, elasticCollection: any): Promise { + return await this.cachingService.getOrSet( + CacheInfo.CollectionRoles(identifier).key, + async () => await this.getNftCollectionRolesFromGateway(elasticCollection), + CacheInfo.CollectionRoles(identifier).ttl, + ); + } + + private async getCollectionLogoCached(identifier: string): Promise { + return await this.cachingService.getOrSet( + CacheInfo.CollectionLogo(identifier).key, + async () => await this.getCollectionLogo(identifier), + CacheInfo.CollectionLogo(identifier).ttl, + ); + } } diff --git a/src/endpoints/collections/entities/nft.collection.ts b/src/endpoints/collections/entities/nft.collection.ts index da03e0c7a..5d77e2228 100644 --- a/src/endpoints/collections/entities/nft.collection.ts +++ b/src/endpoints/collections/entities/nft.collection.ts @@ -32,6 +32,9 @@ export class NftCollection { @ApiProperty({ type: Number }) timestamp: number = 0; + @ApiProperty({ type: Number, nullable: true, required: false }) + timestampMs?: number; + @ApiProperty({ type: Boolean, default: false }) canFreeze: boolean = false; diff --git a/src/endpoints/esdt/esdt.address.service.ts b/src/endpoints/esdt/esdt.address.service.ts index 34ea3dc94..c0b751505 100644 --- a/src/endpoints/esdt/esdt.address.service.ts +++ b/src/endpoints/esdt/esdt.address.service.ts @@ -101,14 +101,13 @@ export class EsdtAddressService { async getCollectionsForAddress(address: string, filter: CollectionFilter, pagination: QueryPagination): Promise { const tokenCollections = await this.indexerService.getNftCollections(pagination, filter, address); - const collectionsIdentifiers = tokenCollections.map((collection) => collection.token); const indexedCollections: Record = {}; for (const collection of tokenCollections) { indexedCollections[collection.token] = collection; } - const accountCollections = await this.collectionService.applyPropertiesToCollections(collectionsIdentifiers); + const accountCollections = await this.collectionService.buildCollectionsFromElasticData(tokenCollections); const collectionsWithRoles: NftCollectionWithRoles[] = []; diff --git a/src/endpoints/events/entities/events.custom.subscribe.ts b/src/endpoints/events/entities/events.custom.subscribe.ts new file mode 100644 index 000000000..21b448f2e --- /dev/null +++ b/src/endpoints/events/entities/events.custom.subscribe.ts @@ -0,0 +1,21 @@ +import { IsOptional, IsString } from 'class-validator'; +import { NoEmptyPayload } from 'src/utils/no.empty.payload.validator'; + +@NoEmptyPayload({ message: `You must add at least one filter from ${EventsCustomSubscribePayload.getClassFields()}` }) +export class EventsCustomSubscribePayload { + @IsOptional() + @IsString() + address?: string; + + @IsOptional() + @IsString() + identifier?: string; + + @IsOptional() + @IsString() + logAddress?: string; + + public static getClassFields(): string[] { + return ['address', 'identifier', 'logAddress']; + } +} diff --git a/src/endpoints/events/entities/events.ts b/src/endpoints/events/entities/events.ts index 00d5909b1..1bc7720f5 100644 --- a/src/endpoints/events/entities/events.ts +++ b/src/endpoints/events/entities/events.ts @@ -39,4 +39,7 @@ export class Events { @ApiProperty({ description: "Event timestamp." }) timestamp: number = 0; + + @ApiProperty({ description: "Event timestamp in milliseconds.", nullable: true, required: false }) + timestampMs?: number; } diff --git a/src/endpoints/events/events.controller.ts b/src/endpoints/events/events.controller.ts index 5c5fc9f08..dc7ce3a0a 100644 --- a/src/endpoints/events/events.controller.ts +++ b/src/endpoints/events/events.controller.ts @@ -6,6 +6,7 @@ import { ParseAddressPipe, ParseIntPipe } from '@multiversx/sdk-nestjs-common'; import { Events } from './entities/events'; import { EventsFilter } from './entities/events.filter'; +import { TimestampParsePipe } from 'src/utils/timestamp.parse.pipe'; @Controller() @ApiTags('events') @@ -24,8 +25,8 @@ export class EventsController { @ApiQuery({ name: 'identifier', description: 'Event identifier', required: false }) @ApiQuery({ name: 'txHash', description: 'Event transaction hash', required: false }) @ApiQuery({ name: 'shard', description: 'Event shard id', required: false }) - @ApiQuery({ name: 'before', description: 'Event before timestamp', required: false }) - @ApiQuery({ name: 'after', description: 'Event after timestamp', required: false }) + @ApiQuery({ name: 'before', description: 'Event before timestamp or timestampMs', required: false }) + @ApiQuery({ name: 'after', description: 'Event after timestamp or timestampMs', required: false }) @ApiQuery({ name: 'order', description: 'Event order', required: false }) @ApiQuery({ name: 'topics', description: 'Event topics to filter by', required: false, isArray: true }) async getEvents( @@ -36,8 +37,8 @@ export class EventsController { @Query('identifier') identifier: string, @Query('txHash') txHash: string, @Query('shard', ParseIntPipe) shard: number, - @Query('before', ParseIntPipe) before: number, - @Query('after', ParseIntPipe) after: number, + @Query('before', TimestampParsePipe) before: number, + @Query('after', TimestampParsePipe) after: number, @Query('order', ParseIntPipe) order: number, @Query('topics') topics: string | string[], ): Promise { @@ -54,16 +55,16 @@ export class EventsController { @ApiQuery({ name: 'identifier', description: 'Event identifier', required: false }) @ApiQuery({ name: 'txHash', description: 'Event transaction hash', required: false }) @ApiQuery({ name: 'shard', description: 'Event shard id', required: false }) - @ApiQuery({ name: 'before', description: 'Event before timestamp', required: false }) - @ApiQuery({ name: 'after', description: 'Event after timestamp', required: false }) + @ApiQuery({ name: 'before', description: 'Event before timestamp or timestampMs', required: false }) + @ApiQuery({ name: 'after', description: 'Event after timestamp or timestampMs', required: false }) @ApiQuery({ name: 'topics', description: 'Event topics to filter by', required: false, isArray: true }) async getEventsCount( @Query('address', ParseAddressPipe) address: string, @Query('identifier') identifier: string, @Query('txHash') txHash: string, @Query('shard', ParseIntPipe) shard: number, - @Query('before', ParseIntPipe) before: number, - @Query('after', ParseIntPipe) after: number, + @Query('before', TimestampParsePipe) before: number, + @Query('after', TimestampParsePipe) after: number, @Query('topics') topics: string | string[], ): Promise { const topicsArray = topics ? (Array.isArray(topics) ? topics : [topics]) : []; diff --git a/src/endpoints/network/entities/feature.configs.ts b/src/endpoints/network/entities/feature.configs.ts index cbd9692c9..319589b3e 100644 --- a/src/endpoints/network/entities/feature.configs.ts +++ b/src/endpoints/network/entities/feature.configs.ts @@ -5,15 +5,87 @@ export class FeatureConfigs { Object.assign(this, init); } + @ApiProperty({ description: 'Events notifier flag activation value' }) + eventsNotifier: boolean = false; + + @ApiProperty({ description: 'Guest caching flag activation value' }) + guestCaching: boolean = false; + + @ApiProperty({ description: 'Transaction pool flag activation value' }) + transactionPool: boolean = false; + + @ApiProperty({ description: 'Transaction pool warmer flag activation value' }) + transactionPoolWarmer: boolean = false; + @ApiProperty({ description: 'Update Collection extra details flag activation value' }) updateCollectionExtraDetails: boolean = false; + @ApiProperty({ description: 'Accounts extra details update flag activation value' }) + updateAccountsExtraDetails: boolean = false; + @ApiProperty({ description: 'Marketplace flag activation value' }) marketplace: boolean = false; @ApiProperty({ description: 'Exchange flag activation value' }) exchange: boolean = false; - @ApiProperty({ description: 'DataApi flag activation value' }) + @ApiProperty({ description: 'Data API flag activation value' }) dataApi: boolean = false; + + @ApiProperty({ description: 'Authentication flag activation value' }) + auth: boolean = false; + + @ApiProperty({ description: 'Staking V4 flag activation value' }) + stakingV4: boolean = false; + + @ApiProperty({ description: 'Chain Andromeda flag activation value' }) + chainAndromeda: boolean = false; + + @ApiProperty({ description: 'Staking v5 flag activation value' }) + stakingV5: boolean = false; + + @ApiProperty({ description: 'Staking v5 activation epoch' }) + stakingV5ActivationEpoch: number = 0; + + @ApiProperty({ description: 'Node epochs left flag activation value' }) + nodeEpochsLeft: boolean = false; + + @ApiProperty({ description: 'Transaction processor flag activation value' }) + transactionProcessor: boolean = false; + + @ApiProperty({ description: 'Transaction completed flag activation value' }) + transactionCompleted: boolean = false; + + @ApiProperty({ description: 'Transaction batch flag activation value' }) + transactionBatch: boolean = false; + + @ApiProperty({ description: 'Deep history flag activation value' }) + deepHistory: boolean = false; + + @ApiProperty({ description: 'Elastic circuit breaker flag activation value' }) + elasticCircuitBreaker: boolean = false; + + @ApiProperty({ description: 'Status checker flag activation value' }) + statusChecker: boolean = false; + + @ApiProperty({ description: 'NFT scam info flag activation value' }) + nftScamInfo: boolean = false; + + @ApiProperty({ description: 'NFT processing flag activation value' }) + processNfts: boolean = false; + + @ApiProperty({ description: 'TPS flag activation value' }) + tps: boolean = false; + + @ApiProperty({ description: 'Nodes fetch flag activation value' }) + nodesFetch: boolean = false; + + @ApiProperty({ description: 'Tokens fetch flag activation value' }) + tokensFetch: boolean = false; + + @ApiProperty({ description: 'Providers fetch flag activation value' }) + providersFetch: boolean = false; + + @ApiProperty({ description: 'Assets fetch flag activation value' }) + assetsFetch: boolean = false; } diff --git a/src/endpoints/network/network.service.ts b/src/endpoints/network/network.service.ts index 6388d5e04..bd7daba4b 100644 --- a/src/endpoints/network/network.service.ts +++ b/src/endpoints/network/network.service.ts @@ -237,6 +237,11 @@ export class NetworkService { if (!stake) { throw new Error('Global stake not available'); } + const stakingV5Config = { + enabled: this.apiConfigService.isStakingV5Enabled() && stats.epoch >= this.apiConfigService.getStakingV5ActivationEpoch(), + activationEpoch: this.apiConfigService.getStakingV5ActivationEpoch(), + inflationAmounts: this.apiConfigService.getStakingV5InflationAmounts(), + }; const stakedBalance = await this.getAuctionContractBalance(); @@ -251,9 +256,12 @@ export class NetworkService { const secondsInYear = 365 * 24 * 3600; const epochsInYear = secondsInYear / epochDuration; - const yearIndex = Math.floor(stats.epoch / epochsInYear); - - const inflationAmounts = this.apiConfigService.getInflationAmounts(); + let yearIndex = Math.floor(stats.epoch / epochsInYear); + let inflationAmounts = this.apiConfigService.getInflationAmounts(); + if (stakingV5Config.enabled) { + yearIndex = Math.floor((stats.epoch - stakingV5Config.activationEpoch) / epochsInYear); + inflationAmounts = stakingV5Config.inflationAmounts; + } if (yearIndex >= inflationAmounts.length) { throw new Error(`There is no inflation information for year with index ${yearIndex}`,); @@ -313,10 +321,34 @@ export class NetworkService { } const features = new FeatureConfigs({ + eventsNotifier: this.apiConfigService.isEventsNotifierFeatureActive(), + guestCaching: this.apiConfigService.isGuestCacheFeatureActive(), + transactionPool: this.apiConfigService.isTransactionPoolEnabled(), + transactionPoolWarmer: this.apiConfigService.getIsCacheWarmerCronActive(), updateCollectionExtraDetails: this.apiConfigService.isUpdateCollectionExtraDetailsEnabled(), + updateAccountsExtraDetails: this.apiConfigService.isUpdateAccountExtraDetailsEnabled(), marketplace: this.apiConfigService.isMarketplaceFeatureEnabled(), exchange: this.apiConfigService.isExchangeEnabled(), dataApi: this.apiConfigService.isDataApiFeatureEnabled(), + auth: this.apiConfigService.getIsAuthActive(), + stakingV4: this.apiConfigService.isStakingV4Enabled(), + chainAndromeda: this.apiConfigService.isChainAndromedaEnabled(), + stakingV5: this.apiConfigService.isStakingV5Enabled(), + stakingV5ActivationEpoch: this.apiConfigService.getStakingV5ActivationEpoch(), + nodeEpochsLeft: this.apiConfigService.isNodeEpochsLeftEnabled(), + transactionProcessor: this.apiConfigService.getIsTransactionProcessorCronActive(), + transactionCompleted: this.apiConfigService.getIsTransactionCompletedCronActive(), + transactionBatch: this.apiConfigService.getIsTransactionBatchCronActive(), + deepHistory: this.apiConfigService.isDeepHistoryGatewayEnabled(), + elasticCircuitBreaker: this.apiConfigService.isElasticCircuitBreakerEnabled(), + statusChecker: this.apiConfigService.getIsApiStatusCheckerActive(), + nftScamInfo: this.apiConfigService.getIsNftScamInfoEnabled(), + processNfts: this.apiConfigService.getIsProcessNftsFlagActive(), + tps: this.apiConfigService.isTpsEnabled(), + nodesFetch: this.apiConfigService.isNodesFetchFeatureEnabled(), + tokensFetch: this.apiConfigService.isTokensFetchFeatureEnabled(), + providersFetch: this.apiConfigService.isProvidersFetchFeatureEnabled(), + assetsFetch: this.apiConfigService.isAssetsCdnFeatureEnabled(), }); let indexerVersion: string | undefined; diff --git a/src/endpoints/nfts/nft.controller.ts b/src/endpoints/nfts/nft.controller.ts index 50a495d24..3c04ea1e8 100644 --- a/src/endpoints/nfts/nft.controller.ts +++ b/src/endpoints/nfts/nft.controller.ts @@ -20,6 +20,7 @@ import { Transaction } from '../transactions/entities/transaction'; import { ScamType } from 'src/common/entities/scam-type.enum'; import { TransferService } from '../transfers/transfer.service'; import { NftSubType } from './entities/nft.sub.type'; +import { TimestampParsePipe } from 'src/utils/timestamp.parse.pipe'; @Controller() @ApiTags('nfts') @@ -52,8 +53,8 @@ export class NftController { @ApiQuery({ name: 'isScam', description: 'Filter by scam status', required: false, type: Boolean }) @ApiQuery({ name: 'scamType', description: 'Filter by type (scam/potentialScam)', required: false }) @ApiQuery({ name: 'traits', description: 'Filter NFTs by traits. Key-value format (:;:)', required: false, type: Boolean }) - @ApiQuery({ name: 'before', description: 'Return all NFTs before given timestamp', required: false, type: Number }) - @ApiQuery({ name: 'after', description: 'Return all NFTs after given timestamp', required: false, type: Number }) + @ApiQuery({ name: 'before', description: 'Return all NFTs before given timestamp or timestampMs', required: false, type: Number }) + @ApiQuery({ name: 'after', description: 'Return all NFTs after given timestamp or timestampMs', required: false, type: Number }) @ApiQuery({ name: 'withOwner', description: 'Return owner where type = NonFungibleESDT', required: false, type: Boolean }) @ApiQuery({ name: 'withSupply', description: 'Return supply where type = SemiFungibleESDT', required: false, type: Boolean }) async getNfts( @@ -74,8 +75,8 @@ export class NftController { @Query('isScam', ParseBoolPipe) isScam?: boolean, @Query('scamType', new ParseEnumPipe(ScamType)) scamType?: ScamType, @Query('traits', ParseRecordPipe) traits?: Record, - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('withOwner', ParseBoolPipe) withOwner?: boolean, @Query('withSupply', ParseBoolPipe) withSupply?: boolean, ): Promise { @@ -120,8 +121,8 @@ export class NftController { @ApiQuery({ name: 'hasUris', description: 'Return all NFTs that have one or more uris', required: false, type: Boolean }) @ApiQuery({ name: 'isNsfw', description: 'Filter by NSFW status', required: false, type: Boolean }) @ApiQuery({ name: 'traits', description: 'Filter NFTs by traits. Key-value format (:;:)', required: false, type: Boolean }) - @ApiQuery({ name: 'before', description: 'Return all NFTs before given timestamp', required: false, type: Number }) - @ApiQuery({ name: 'after', description: 'Return all NFTs after given timestamp', required: false, type: Number }) + @ApiQuery({ name: 'before', description: 'Return all NFTs before given timestamp or timestampMs', required: false, type: Number }) + @ApiQuery({ name: 'after', description: 'Return all NFTs after given timestamp or timestampMs', required: false, type: Number }) @ApiQuery({ name: 'isScam', description: 'Filter by scam status', required: false, type: Boolean }) @ApiQuery({ name: 'scamType', description: 'Filter by type (scam/potentialScam)', required: false }) async getNftCount( @@ -138,8 +139,8 @@ export class NftController { @Query('hasUris', ParseBoolPipe) hasUris?: boolean, @Query('isNsfw', ParseBoolPipe) isNsfw?: boolean, @Query('traits', ParseRecordPipe) traits?: Record, - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('isScam', ParseBoolPipe) isScam?: boolean, @Query('scamType', new ParseEnumPipe(ScamType)) scamType?: ScamType, ): Promise { @@ -181,8 +182,8 @@ export class NftController { @Query('hasUris', ParseBoolPipe) hasUris?: boolean, @Query('isNsfw', ParseBoolPipe) isNsfw?: boolean, @Query('traits', ParseRecordPipe) traits?: Record, - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('isScam', ParseBoolPipe) isScam?: boolean, @Query('scamType', new ParseEnumPipe(ScamType)) scamType?: ScamType, ): Promise { @@ -287,8 +288,8 @@ export class NftController { @ApiQuery({ name: 'hashes', description: 'Filter by a comma-separated list of transaction hashes', required: false }) @ApiQuery({ name: 'status', description: 'Status of the transaction (success / pending / invalid / fail)', required: false, enum: TransactionStatus }) @ApiQuery({ name: 'function', description: 'Filter transactions by function name', required: false }) - @ApiQuery({ name: 'before', description: 'Before timestamp', required: false }) - @ApiQuery({ name: 'after', description: 'After timestamp', required: false }) + @ApiQuery({ name: 'before', description: 'Before timestamp or timestampMs', required: false }) + @ApiQuery({ name: 'after', description: 'After timestamp or timestampMs', required: false }) @ApiQuery({ name: 'order', description: 'Sort order (asc/desc)', required: false, enum: SortOrder }) @ApiQuery({ name: 'from', description: 'Number of items to skip for the result set', required: false }) @ApiQuery({ name: 'size', description: 'Number of items to retrieve', required: false }) @@ -310,8 +311,8 @@ export class NftController { @Query('hashes', ParseArrayPipe) hashes?: string[], @Query('status', new ParseEnumPipe(TransactionStatus)) status?: TransactionStatus, @Query('function', ParseArrayPipe) functions?: string[], - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('order', new ParseEnumPipe(SortOrder)) order?: SortOrder, @Query('withScResults', ParseBoolPipe) withScResults?: boolean, @@ -354,8 +355,8 @@ export class NftController { @ApiQuery({ name: 'miniBlockHash', description: 'Filter by miniblock hash', required: false }) @ApiQuery({ name: 'hashes', description: 'Filter by a comma-separated list of transaction hashes', required: false }) @ApiQuery({ name: 'status', description: 'Status of the transaction (success / pending / invalid / fail)', required: false, enum: TransactionStatus }) - @ApiQuery({ name: 'before', description: 'Before timestamp', required: false }) - @ApiQuery({ name: 'after', description: 'After timestamp', required: false }) + @ApiQuery({ name: 'before', description: 'Before timestamp or timestampMs', required: false }) + @ApiQuery({ name: 'after', description: 'After timestamp or timestampMs', required: false }) @ApiQuery({ name: 'withRelayedScresults', description: 'If set to true, will include smart contract results that resemble relayed transactions', required: false, type: Boolean }) async getNftTransactionsCount( @Param('identifier', ParseNftPipe) identifier: string, @@ -366,8 +367,8 @@ export class NftController { @Query('miniBlockHash', ParseBlockHashPipe) miniBlockHash?: string, @Query('hashes', ParseArrayPipe) hashes?: string[], @Query('status', new ParseEnumPipe(TransactionStatus)) status?: TransactionStatus, - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('withRelayedScresults', ParseBoolPipe) withRelayedScresults?: boolean, ) { @@ -399,8 +400,8 @@ export class NftController { @ApiQuery({ name: 'hashes', description: 'Filter by a comma-separated list of transfer hashes', required: false }) @ApiQuery({ name: 'status', description: 'Status of the transfer (success / pending / invalid / fail)', required: false, enum: TransactionStatus }) @ApiQuery({ name: 'function', description: 'Filter transfers by function name', required: false }) - @ApiQuery({ name: 'before', description: 'Before timestamp', required: false }) - @ApiQuery({ name: 'after', description: 'After timestamp', required: false }) + @ApiQuery({ name: 'before', description: 'Before timestamp or timestampMs', required: false }) + @ApiQuery({ name: 'after', description: 'After timestamp or timestampMs', required: false }) @ApiQuery({ name: 'order', description: 'Sort order (asc/desc)', required: false, enum: SortOrder }) @ApiQuery({ name: 'from', description: 'Number of items to skip for the result set', required: false }) @ApiQuery({ name: 'size', description: 'Number of items to retrieve', required: false }) @@ -421,8 +422,8 @@ export class NftController { @Query('hashes', ParseArrayPipe) hashes?: string[], @Query('status', new ParseEnumPipe(TransactionStatus)) status?: TransactionStatus, @Query('function', ParseArrayPipe) functions?: string[], - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('order', new ParseEnumPipe(SortOrder)) order?: SortOrder, @Query('withScResults', ParseBoolPipe) withScResults?: boolean, @Query('withOperations', ParseBoolPipe) withOperations?: boolean, @@ -459,8 +460,8 @@ export class NftController { @ApiQuery({ name: 'miniBlockHash', description: 'Filter by miniblock hash', required: false }) @ApiQuery({ name: 'hashes', description: 'Filter by a comma-separated list of transfers hashes', required: false }) @ApiQuery({ name: 'status', description: 'Status of the transfers (success / pending / invalid / fail)', required: false, enum: TransactionStatus }) - @ApiQuery({ name: 'before', description: 'Before timestamp', required: false }) - @ApiQuery({ name: 'after', description: 'After timestamp', required: false }) + @ApiQuery({ name: 'before', description: 'Before timestamp or timestampMs', required: false }) + @ApiQuery({ name: 'after', description: 'After timestamp or timestampMs', required: false }) async getNftTransfersCount( @Param('identifier', ParseNftPipe) identifier: string, @Query('sender', ParseAddressPipe) sender?: string, @@ -470,8 +471,8 @@ export class NftController { @Query('miniBlockHash', ParseBlockHashPipe) miniBlockHash?: string, @Query('hashes', ParseArrayPipe) hashes?: string[], @Query('status', new ParseEnumPipe(TransactionStatus)) status?: TransactionStatus, - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, ) { return await this.transferService.getTransfersCount(new TransactionFilter({ diff --git a/src/endpoints/nfts/nft.service.ts b/src/endpoints/nfts/nft.service.ts index b4a5192db..6c2f0cf9d 100644 --- a/src/endpoints/nfts/nft.service.ts +++ b/src/endpoints/nfts/nft.service.ts @@ -31,6 +31,7 @@ import { SortCollectionNfts } from "../collections/entities/sort.collection.nfts import { TokenAssets } from "src/common/assets/entities/token.assets"; import { ScamInfo } from "src/common/entities/scam-info.dto"; import { NftSubType } from "./entities/nft.sub.type"; +import { ConcurrencyUtils } from "src/utils/concurrency.utils"; @Injectable() export class NftService { @@ -65,6 +66,19 @@ export class NftService { } async getNfts(queryPagination: QueryPagination, filter: NftFilter, queryOptions?: NftQueryOptions): Promise { + if (this.isCacheableNftList(filter, queryOptions)) { + const cacheInfo = CacheInfo.Nfts(queryPagination); + return await this.cachingService.getOrSet( + cacheInfo.key, + () => this.fetchAndProcessNfts(queryPagination, filter, queryOptions), + cacheInfo.ttl, + ); + } + + return await this.fetchAndProcessNfts(queryPagination, filter, queryOptions); + } + + private async fetchAndProcessNfts(queryPagination: QueryPagination, filter: NftFilter, queryOptions?: NftQueryOptions): Promise { const { from, size } = queryPagination; const nfts = await this.getNftsInternal({ from, size }, filter); @@ -114,21 +128,15 @@ export class NftService { return; } - const nftsIdentifiers = nfts.filter(x => x.type === NftType.NonFungibleESDT).map(x => x.identifier); + const nftsIdentifiers = nfts + .filter(x => x.type === NftType.NonFungibleESDT) + .map(x => x.identifier); if (nftsIdentifiers.length === 0) { return; } - const accountsEsdts = await this.getAccountEsdtByIdentifiers(nftsIdentifiers, { - from: 0, - size: nftsIdentifiers.length, - }); - - const ownerMap = accountsEsdts.reduce((acc: Record, accountEsdt: any) => { - acc[accountEsdt.identifier] = accountEsdt.address; - return acc; - }, {}); + const ownerMap = await this.getOwnersBulk(nftsIdentifiers); for (const nft of nfts) { if (nft.type === NftType.NonFungibleESDT && ownerMap[nft.identifier]) { @@ -182,6 +190,29 @@ export class NftService { ); } + private async getOwnersBulk(identifiers: string[], chunkSize: number = 512, concurrencyLimit: number = 4): Promise> { + if (identifiers.length === 0) { + return {}; + } + + const chunks = BatchUtils.splitArrayIntoChunks(identifiers.distinct(), chunkSize); + const results = await ConcurrencyUtils.executeWithConcurrencyLimit( + chunks, + async (chunk) => await this.getAccountEsdtByIdentifiers(chunk, { from: 0, size: chunk.length }), + concurrencyLimit, + 'NftService.getOwnersBulk' + ); + + const ownerMap: Record = {}; + for (const chunkResult of results) { + for (const accountEsdt of chunkResult ?? []) { + ownerMap[accountEsdt.identifier] = accountEsdt.address; + } + } + + return ownerMap; + } + private async batchApplyMedia(nfts: Nft[], fields?: string[]) { if (fields && !fields.includes('media')) { return; @@ -324,12 +355,20 @@ export class NftService { return; } - const nftsForAddress = await this.esdtAddressService.getNftsForAddress(nft.owner, new NftFilter({ identifiers: [nft.identifier] }), new QueryPagination({ from: 0, size: 1 })); - if (nftsForAddress.length === 0) { - return; + let attributes = nft.attributes; + if (!attributes || attributes.length === 0) { + const nftsForAddress = await this.esdtAddressService.getNftsForAddress(nft.owner, new NftFilter({identifiers: [nft.identifier]}), new QueryPagination({ + from: 0, + size: 1, + })); + if (nftsForAddress.length === 0) { + return; + } + + attributes = nftsForAddress[0].attributes; } - nft.attributes = nftsForAddress[0].attributes; + nft.attributes = attributes; } private async applyMedia(nft: Nft) { @@ -501,15 +540,24 @@ export class NftService { } async getNftCount(filter: NftFilter): Promise { + if (this.isCacheableNftCount(filter)) { + return await this.cachingService.getOrSet( + CacheInfo.NftsCount.key, + async () => await this.indexerService.getNftCount(filter), + CacheInfo.NftsCount.ttl, + ); + } + return await this.indexerService.getNftCount(filter); } async getNftsForAddress(address: string, queryPagination: QueryPagination, filter: NftFilter, fields?: string[], queryOptions?: NftQueryOptions, source?: EsdtDataSource): Promise { let nfts = await this.esdtAddressService.getNftsForAddress(address, filter, queryPagination, source, queryOptions); - for (const nft of nfts) { + + await Promise.all(nfts.map(async (nft) => { await this.applyAssetsAndTicker(nft, fields); await this.applyPriceUsd(nft, fields); - } + })); if (queryOptions && queryOptions.withSupply) { const supplyNfts = nfts.filter(nft => nft.type.in(NftType.SemiFungibleESDT, NftType.MetaESDT)); @@ -541,23 +589,24 @@ export class NftService { nfts = this.applyScamFilter(nfts, filter); - for (const nft of nfts) { - await this.applyUnlockFields(nft, fields); - } + await Promise.all(nfts.map(nft => this.applyUnlockFields(nft, fields))); return nfts; } private async getNftsInternalByIdentifiers(identifiers: string[]): Promise { - const chunks = BatchUtils.splitArrayIntoChunks(identifiers, 1024); - const result: Nft[] = []; - for (const identifiers of chunks) { - const internalNfts = await this.getNftsInternal(new QueryPagination({ from: 0, size: identifiers.length }), new NftFilter({ identifiers })); - - result.push(...internalNfts); - } + const chunks = BatchUtils.splitArrayIntoChunks(identifiers, 512); + const results = await ConcurrencyUtils.executeWithConcurrencyLimit( + chunks, + async (chunk) => await this.getNftsInternal( + new QueryPagination({ from: 0, size: chunk.length }), + new NftFilter({ identifiers: chunk }) + ), + 4, + 'NftService.getNftsInternalByIdentifiers' + ); - return result; + return results.flat(); } private async applyPriceUsd(nft: NftAccount, fields?: string[]) { @@ -730,4 +779,39 @@ export class NftService { this.logger.error(error); } } + + private isDefaultNftFilter(filter: NftFilter): boolean { + return !filter.search && + !(filter.identifiers && filter.identifiers.length > 0) && + !(filter.type && filter.type.length > 0) && + !(filter.subType && filter.subType.length > 0) && + !filter.collection && + !(filter.collections && filter.collections.length > 0) && + !(filter.tags && filter.tags.length > 0) && + !filter.name && + !filter.creator && + filter.hasUris === undefined && + filter.includeFlagged === undefined && + filter.before === undefined && + filter.after === undefined && + filter.nonceBefore === undefined && + filter.nonceAfter === undefined && + filter.isWhitelistedStorage === undefined && + filter.isNsfw === undefined && + filter.isScam === undefined && + filter.scamType === undefined && + !filter.traits && + filter.excludeMetaESDT === undefined && + filter.sort === undefined && + filter.order === undefined; + } + + private isCacheableNftList(filter: NftFilter, queryOptions?: NftQueryOptions): boolean { + const hasHeavyOptions = queryOptions?.withOwner || queryOptions?.withSupply; + return !hasHeavyOptions && this.isDefaultNftFilter(filter); + } + + private isCacheableNftCount(filter: NftFilter): boolean { + return this.isDefaultNftFilter(filter); + } } diff --git a/src/endpoints/process-nfts/process.nfts.service.ts b/src/endpoints/process-nfts/process.nfts.service.ts index 2a9f75c8a..10a185a06 100644 --- a/src/endpoints/process-nfts/process.nfts.service.ts +++ b/src/endpoints/process-nfts/process.nfts.service.ts @@ -97,6 +97,9 @@ export class ProcessNftsService { } private async isCollectionOwner(address: string, collection: string): Promise { + if (this.apiConfigService.getSecurityAdmins().includes(address)) { + return true; + } const collectionOwner = await this.getCollectionNonScOwner(collection); return address === collectionOwner; diff --git a/src/endpoints/rounds/entities/round.ts b/src/endpoints/rounds/entities/round.ts index 9f09616d0..3183bc39d 100644 --- a/src/endpoints/rounds/entities/round.ts +++ b/src/endpoints/rounds/entities/round.ts @@ -19,4 +19,8 @@ export class Round { @ApiProperty({ type: Number, example: 1651148112 }) timestamp: number = 0; + + // only available for rounds after Barnard protocol upgrade + @ApiProperty({ type: Number, example: 1651148112000, required: false }) + timestampMs?: number; } diff --git a/src/endpoints/tokens/token.controller.ts b/src/endpoints/tokens/token.controller.ts index f3bdc113f..ca67cb242 100644 --- a/src/endpoints/tokens/token.controller.ts +++ b/src/endpoints/tokens/token.controller.ts @@ -24,6 +24,7 @@ import { TokenType } from "src/common/indexer/entities"; import { ParseArrayPipeOptions } from "@multiversx/sdk-nestjs-common/lib/pipes/entities/parse.array.options"; import { MexPairType } from "../mex/entities/mex.pair.type"; import { TokenAssetsPriceSourceType } from "src/common/assets/entities/token.assets.price.source.type"; +import { TimestampParsePipe } from "src/utils/timestamp.parse.pipe"; @Controller() @ApiTags('tokens') @@ -206,8 +207,8 @@ export class TokenController { @ApiQuery({ name: 'hashes', description: 'Filter by a comma-separated list of transaction hashes', required: false }) @ApiQuery({ name: 'status', description: 'Status of the transaction (success / pending / invalid / fail)', required: false, enum: TransactionStatus }) @ApiQuery({ name: 'function', description: 'Filter transactions by function name', required: false }) - @ApiQuery({ name: 'before', description: 'Before timestamp', required: false }) - @ApiQuery({ name: 'after', description: 'After timestamp', required: false }) + @ApiQuery({ name: 'before', description: 'Before timestamp or timestampMs', required: false }) + @ApiQuery({ name: 'after', description: 'After timestamp or timestampMs', required: false }) @ApiQuery({ name: 'round', description: 'Filter by round number', required: false }) @ApiQuery({ name: 'order', description: 'Sort order (asc/desc)', required: false, enum: SortOrder }) @ApiQuery({ name: 'from', description: 'Number of items to skip for the result set', required: false }) @@ -233,8 +234,8 @@ export class TokenController { @Query('hashes', ParseArrayPipe) hashes?: string[], @Query('status', new ParseEnumPipe(TransactionStatus)) status?: TransactionStatus, @Query('function', new ParseArrayPipe(new ParseArrayPipeOptions({ allowEmptyString: true }))) functions?: string[], - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('round', ParseIntPipe) round?: number, @Query('order', new ParseEnumPipe(SortOrder)) order?: SortOrder, @Query('fields', ParseArrayPipe) fields?: string[], @@ -294,8 +295,8 @@ export class TokenController { @ApiQuery({ name: 'miniBlockHash', description: 'Filter by miniblock hash', required: false }) @ApiQuery({ name: 'hashes', description: 'Filter by a comma-separated list of transaction hashes', required: false }) @ApiQuery({ name: 'status', description: 'Status of the transaction (success / pending / invalid / fail)', required: false, enum: TransactionStatus }) - @ApiQuery({ name: 'before', description: 'Before timestamp', required: false }) - @ApiQuery({ name: 'after', description: 'After timestamp', required: false }) + @ApiQuery({ name: 'before', description: 'Before timestamp or timestampMs', required: false }) + @ApiQuery({ name: 'after', description: 'After timestamp or timestampMs', required: false }) @ApiQuery({ name: 'round', description: 'Filter by round number', required: false }) @ApiQuery({ name: 'isScCall', description: 'Returns sc call transactions details', required: false, type: Boolean }) @ApiQuery({ name: 'withRelayedScresults', description: 'If set to true, will include smart contract results that resemble relayed transactions', required: false, type: Boolean }) @@ -308,8 +309,8 @@ export class TokenController { @Query('miniBlockHash', ParseBlockHashPipe) miniBlockHash?: string, @Query('hashes', ParseArrayPipe) hashes?: string[], @Query('status', new ParseEnumPipe(TransactionStatus)) status?: TransactionStatus, - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('round', ParseIntPipe) round?: number, @Query('withRelayedScresults', ParseBoolPipe) withRelayedScresults?: boolean, @Query('isScCall', ParseBoolPipe) isScCall?: boolean, @@ -392,8 +393,8 @@ export class TokenController { @ApiQuery({ name: 'status', description: 'Status of the transaction (success / pending / invalid / fail)', required: false, enum: TransactionStatus }) @ApiQuery({ name: 'function', description: 'Filter transfers by function name', required: false }) @ApiQuery({ name: 'order', description: 'Sort order (asc/desc)', required: false, enum: SortOrder }) - @ApiQuery({ name: 'before', description: 'Before timestamp', required: false }) - @ApiQuery({ name: 'after', description: 'After timestamp', required: false }) + @ApiQuery({ name: 'before', description: 'Before timestamp or timestampMs', required: false }) + @ApiQuery({ name: 'after', description: 'After timestamp or timestampMs', required: false }) @ApiQuery({ name: 'round', description: 'Filter by round number', required: false }) @ApiQuery({ name: 'fields', description: 'List of fields to filter by', required: false, isArray: true, style: 'form', explode: false }) @ApiQuery({ name: 'isScCall', description: 'Returns sc call transactions details', required: false, type: Boolean }) @@ -413,8 +414,8 @@ export class TokenController { @Query('hashes', ParseArrayPipe) hashes?: string[], @Query('status', new ParseEnumPipe(TransactionStatus)) status?: TransactionStatus, @Query('function', new ParseArrayPipe(new ParseArrayPipeOptions({ allowEmptyString: true }))) functions?: string[], - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('round', ParseIntPipe) round?: number, @Query('fields', ParseArrayPipe) fields?: string[], @Query('order', new ParseEnumPipe(SortOrder)) order?: SortOrder, @@ -464,8 +465,8 @@ export class TokenController { @ApiQuery({ name: 'hashes', description: 'Filter by a comma-separated list of transfer hashes', required: false }) @ApiQuery({ name: 'status', description: 'Status of the transaction (success / pending / invalid / fail)', required: false, enum: TransactionStatus }) @ApiQuery({ name: 'function', description: 'Filter transfers by function name', required: false }) - @ApiQuery({ name: 'before', description: 'Before timestamp', required: false }) - @ApiQuery({ name: 'after', description: 'After timestamp', required: false }) + @ApiQuery({ name: 'before', description: 'Before timestamp or timestampMs', required: false }) + @ApiQuery({ name: 'after', description: 'After timestamp or timestampMs', required: false }) @ApiQuery({ name: 'round', description: 'Filter by round number', required: false }) @ApiQuery({ name: 'isScCall', description: 'Returns sc call transactions details', required: false, type: Boolean }) async getTokenTransfersCount( @@ -478,8 +479,8 @@ export class TokenController { @Query('hashes', ParseArrayPipe) hashes?: string[], @Query('status', new ParseEnumPipe(TransactionStatus)) status?: TransactionStatus, @Query('function', new ParseArrayPipe(new ParseArrayPipeOptions({ allowEmptyString: true }))) functions?: string[], - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('round', ParseIntPipe) round?: number, @Query('isScCall', ParseBoolPipe) isScCall?: boolean, ): Promise { @@ -517,8 +518,8 @@ export class TokenController { @Query('hashes', ParseArrayPipe) hashes?: string[], @Query('status', new ParseEnumPipe(TransactionStatus)) status?: TransactionStatus, @Query('function', new ParseArrayPipe(new ParseArrayPipeOptions({ allowEmptyString: true }))) functions?: string[], - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('round', ParseIntPipe) round?: number, @Query('isScCall', ParseBoolPipe) isScCall?: boolean, ): Promise { diff --git a/src/endpoints/transactions/entities/dtos/transaction.custom.subscribe.ts b/src/endpoints/transactions/entities/dtos/transaction.custom.subscribe.ts new file mode 100644 index 000000000..6fd2baef5 --- /dev/null +++ b/src/endpoints/transactions/entities/dtos/transaction.custom.subscribe.ts @@ -0,0 +1,21 @@ +import { IsOptional, IsString } from 'class-validator'; +import { NoEmptyPayload } from 'src/utils/no.empty.payload.validator'; + +@NoEmptyPayload({ message: `You must add at least one filter from ${TransactionCustomSubscribePayload.getClassFields()}` }) +export class TransactionCustomSubscribePayload { + @IsOptional() + @IsString() + sender?: string; + + @IsOptional() + @IsString() + receiver?: string; + + @IsOptional() + @IsString() + function?: string; + + public static getClassFields(): string[] { + return ['function', 'receiver', 'sender']; + } +} diff --git a/src/endpoints/transactions/transaction-action/entities/transaction.action.category.ts b/src/endpoints/transactions/transaction-action/entities/transaction.action.category.ts index 67c106757..ae70f573c 100644 --- a/src/endpoints/transactions/transaction-action/entities/transaction.action.category.ts +++ b/src/endpoints/transactions/transaction-action/entities/transaction.action.category.ts @@ -4,4 +4,5 @@ export enum TransactionActionCategory { stake = 'stake', scCall = 'scCall', scDeploy = 'scDeploy', + deprecatedRelayedV1V2 = 'deprecatedRelayedV1V2', } diff --git a/src/endpoints/transactions/transaction.controller.ts b/src/endpoints/transactions/transaction.controller.ts index 9d653a5e7..a3829b949 100644 --- a/src/endpoints/transactions/transaction.controller.ts +++ b/src/endpoints/transactions/transaction.controller.ts @@ -15,6 +15,7 @@ import { TransactionQueryOptions } from './entities/transactions.query.options'; import { TransactionService } from './transaction.service'; import { ParseArrayPipeOptions } from '@multiversx/sdk-nestjs-common/lib/pipes/entities/parse.array.options'; import { PpuMetadata } from './entities/ppu.metadata'; +import { TimestampParsePipe } from 'src/utils/timestamp.parse.pipe'; @Controller() @ApiTags('transactions') @@ -34,8 +35,8 @@ export class TransactionController { @ApiQuery({ name: 'hashes', description: 'Filter by a comma-separated list of transaction hashes', required: false }) @ApiQuery({ name: 'status', description: 'Status of the transaction (success / pending / invalid / fail)', required: false, enum: TransactionStatus }) @ApiQuery({ name: 'function', description: 'Filter transactions by function name', required: false }) - @ApiQuery({ name: 'before', description: 'Before timestamp', required: false }) - @ApiQuery({ name: 'after', description: 'After timestamp', required: false }) + @ApiQuery({ name: 'before', description: 'Before timestamp or timestampMs', required: false }) + @ApiQuery({ name: 'after', description: 'After timestamp or timestampMs', required: false }) @ApiQuery({ name: 'round', description: 'Round number', required: false }) @ApiQuery({ name: 'order', description: 'Sort order (asc/desc)', required: false, enum: SortOrder }) @ApiQuery({ name: 'fields', description: 'List of fields to filter by', required: false, isArray: true, style: 'form', explode: false }) @@ -67,8 +68,8 @@ export class TransactionController { @Query('status', new ParseEnumPipe(TransactionStatus)) status?: TransactionStatus, @Query('function', new ParseArrayPipe(new ParseArrayPipeOptions({ allowEmptyString: true }))) functions?: string[], @Query('condition') condition?: QueryConditionOptions, - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('round', ParseIntPipe) round?: number, @Query('order', new ParseEnumPipe(SortOrder)) order?: SortOrder, @Query('fields', ParseArrayPipe) fields?: string[], @@ -127,8 +128,8 @@ export class TransactionController { @ApiQuery({ name: 'status', description: 'Status of the transaction (success / pending / invalid / fail)', required: false, enum: TransactionStatus }) @ApiQuery({ name: 'condition', description: 'Condition for elastic search queries', required: false, deprecated: true }) @ApiQuery({ name: 'function', description: 'Filter transactions by function name', required: false }) - @ApiQuery({ name: 'before', description: 'Before timestamp', required: false }) - @ApiQuery({ name: 'after', description: 'After timestamp', required: false }) + @ApiQuery({ name: 'before', description: 'Before timestamp or timestampMs', required: false }) + @ApiQuery({ name: 'after', description: 'After timestamp or timestampMs', required: false }) @ApiQuery({ name: 'round', description: 'Round number', required: false }) @ApiQuery({ name: 'isRelayed', description: 'Returns relayed transactions details', required: false, type: Boolean }) @ApiQuery({ name: 'isScCall', description: 'Returns sc call transactions details', required: false, type: Boolean }) @@ -145,8 +146,8 @@ export class TransactionController { @Query('status', new ParseEnumPipe(TransactionStatus)) status?: TransactionStatus, @Query('function', new ParseArrayPipe(new ParseArrayPipeOptions({ allowEmptyString: true }))) functions?: string[], @Query('condition') condition?: QueryConditionOptions, - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('round', ParseIntPipe) round?: number, @Query('relayer', ParseAddressPipe) relayer?: string, @Query('isRelayed', ParseBoolPipe) isRelayed?: boolean, @@ -187,8 +188,8 @@ export class TransactionController { @Query('status', new ParseEnumPipe(TransactionStatus)) status?: TransactionStatus, @Query('function', new ParseArrayPipe(new ParseArrayPipeOptions({ allowEmptyString: true }))) functions?: string[], @Query('condition') condition?: QueryConditionOptions, - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('round', ParseIntPipe) round?: number, @Query('relayer', ParseAddressPipe) relayer?: string, @Query('isRelayed', ParseBoolPipe) isRelayed?: boolean, diff --git a/src/endpoints/transactions/transaction.service.ts b/src/endpoints/transactions/transaction.service.ts index ae196d8df..765b9429e 100644 --- a/src/endpoints/transactions/transaction.service.ts +++ b/src/endpoints/transactions/transaction.service.ts @@ -42,6 +42,8 @@ import { NetworkService } from 'src/endpoints/network/network.service'; import { TransactionWithPpu } from './entities/transaction.with.ppu'; import { GasBucket } from './entities/gas.bucket'; import { GasBucketConstants } from './constants/gas.bucket.constants'; +import { TransactionAction } from "./transaction-action/entities/transaction.action"; +import { TransactionActionCategory } from "./transaction-action/entities/transaction.action.category"; @Injectable() export class TransactionService { @@ -93,9 +95,44 @@ export class TransactionService { return this.getTransactionCountForAddress(filter.sender ?? ''); } + if (this.isCacheableTransactionCount(filter, address)) { + return await this.cachingService.getOrSet( + CacheInfo.TransactionsCount.key, + async () => await this.indexerService.getTransactionCount(filter, address), + CacheInfo.TransactionsCount.ttl, + Constants.oneSecond(), + ); + } + return await this.indexerService.getTransactionCount(filter, address); } + public reorderAccountSentTransactionsByNonce(transactions: TransactionDetailed[], accountAddress: string): TransactionDetailed[] { + const sentPositions: number[] = []; + const sentTransactions: TransactionDetailed[] = []; + + transactions.forEach((tx, index) => { + if (tx.sender === accountAddress) { + sentPositions.push(index); + sentTransactions.push(tx); + } + }); + + sentTransactions.sort((a, b) => { + const nonceA = a.nonce ?? 0; + const nonceB = b.nonce ?? 0; + return nonceB - nonceA; + }); + + const result = [...transactions]; + + sentPositions.forEach((position, index) => { + result[position] = sentTransactions[index]; + }); + + return result; + } + private getDistinctUserAddressesFromTransactions(transactions: Transaction[]): string[] { const allAddresses = []; for (const transaction of transactions) { @@ -164,11 +201,32 @@ export class TransactionService { } async getTransactions(filter: TransactionFilter, pagination: QueryPagination, queryOptions?: TransactionQueryOptions, address?: string, fields?: string[]): Promise { + if (this.isCacheableTransactionList(filter, queryOptions, fields, address)) { + const cacheInfo = CacheInfo.Transactions(pagination); + return await this.cachingService.getOrSet( + cacheInfo.key, + () => this.computeTransactions(filter, pagination, queryOptions, address, fields), + cacheInfo.ttl, + Constants.oneSecond(), + ); + } + + return await this.computeTransactions(filter, pagination, queryOptions, address, fields); + } + + private async computeTransactions(filter: TransactionFilter, pagination: QueryPagination, queryOptions?: TransactionQueryOptions, address?: string, fields?: string[]): Promise { const elasticTransactions = await this.indexerService.getTransactions(filter, pagination, address); let transactions: TransactionDetailed[] = []; transactions = elasticTransactions.map(x => ApiUtils.mergeObjects(new TransactionDetailed(), x)); + const hasSenderFilter = filter.sender || (filter.senders && filter.senders.length > 0); + const hasReceiverFilter = filter.receivers && filter.receivers.length > 0; + + if (address && !hasSenderFilter && !hasReceiverFilter) { + transactions = this.reorderAccountSentTransactionsByNonce(transactions, address); + } + if (filter.hashes) { const txHashes: string[] = filter.hashes; const elasticHashes = elasticTransactions.map(({ txHash }: any) => txHash); @@ -193,7 +251,6 @@ export class TransactionService { for (const transaction of transactions) { transaction.type = undefined; - transaction.relayedVersion = this.extractRelayedVersion(transaction); } await this.processTransactions(transactions, { @@ -202,6 +259,8 @@ export class TransactionService { withActionTransferValue: queryOptions?.withActionTransferValue ?? false, }); + this.processRelayedInfo(transactions); + return transactions; } @@ -225,9 +284,9 @@ export class TransactionService { if (transaction !== null) { transaction.price = await this.getTransactionPrice(transaction); - transaction.relayedVersion = this.extractRelayedVersion(transaction); await this.processTransactions([transaction], { withScamInfo: true, withUsername: true, withActionTransferValue }); + this.processRelayedInfo([transaction]); if (transaction.pendingResults === true && transaction.results) { for (const result of transaction.results) { @@ -347,6 +406,26 @@ export class TransactionService { } } + public processRelayedInfo(transactions: TransactionDetailed[]) { + for (const transaction of transactions) { + transaction.relayedVersion = this.extractRelayedVersion(transaction); + if (transaction.relayedVersion && ["v1", "v2"].includes(transaction.relayedVersion)) { + const shouldSkip = this.apiConfigService.shouldDeprecateRelayedV1V2(transaction.epoch ?? 0); + if (shouldSkip) { + transaction.function = undefined; + transaction.action = new TransactionAction({ + category: TransactionActionCategory.deprecatedRelayedV1V2, + name: "Deprecated transaction action", + description: `Relayed v1/v2 transactions are deprecated`, + }); + } + } + if (!transaction.isRelayed) { + transaction.relayedVersion = undefined; + } + } + } + async processTransactions(transactions: Transaction[], options: { withScamInfo: boolean, withUsername: boolean, withActionTransferValue: boolean }): Promise { this.normalizeTimestampMs(transactions); @@ -587,7 +666,7 @@ export class TransactionService { } private extractRelayedVersion(transaction: TransactionDetailed): string | undefined { - if (transaction.isRelayed == true && transaction.data) { + if (transaction.data) { const decodedData = BinaryUtils.base64Decode(transaction.data); if (decodedData.startsWith('relayedTx@')) { @@ -764,4 +843,52 @@ export class TransactionService { return buckets; } + + private isEmptyTransactionFilter(filter: TransactionFilter): boolean { + return !filter.address && + !filter.sender && + !(filter.senders && filter.senders.length > 0) && + !(filter.receivers && filter.receivers.length > 0) && + !filter.token && + !(filter.tokens && filter.tokens.length > 0) && + !(filter.functions && filter.functions.length > 0) && + filter.senderShard === undefined && + filter.receiverShard === undefined && + !filter.miniBlockHash && + !(filter.hashes && filter.hashes.length > 0) && + filter.status === undefined && + filter.before === undefined && + filter.after === undefined && + filter.condition === undefined && + filter.order === undefined && + filter.senderOrReceiver === undefined && + filter.isScCall === undefined && + filter.isRelayed === undefined && + filter.relayer === undefined && + filter.round === undefined && + filter.withRefunds === undefined && + filter.withRelayedScresults === undefined && + filter.withTxsRelayedByAddress === undefined; + } + + private isCacheableTransactionList(filter: TransactionFilter, queryOptions?: TransactionQueryOptions, fields?: string[], address?: string): boolean { + const hasFieldSelection = Array.isArray(fields) && fields.length > 0; + if (address || hasFieldSelection || !this.isEmptyTransactionFilter(filter) || !queryOptions) { + return false; + } + + const hasAnyEnrichmentOption = queryOptions.withScResults || + queryOptions.withBlockInfo || + queryOptions.withActionTransferValue || + queryOptions.withUsername || + queryOptions.withTxsOrder || + queryOptions.withOperations !== undefined || + queryOptions.withLogs !== undefined; + + return !hasAnyEnrichmentOption; + } + + private isCacheableTransactionCount(filter: TransactionFilter, address?: string): boolean { + return !address && this.isEmptyTransactionFilter(filter); + } } diff --git a/src/endpoints/transfers/transfer.controller.ts b/src/endpoints/transfers/transfer.controller.ts index 7cd6094fe..8e02225fd 100644 --- a/src/endpoints/transfers/transfer.controller.ts +++ b/src/endpoints/transfers/transfer.controller.ts @@ -10,6 +10,7 @@ import { TransactionStatus } from "../transactions/entities/transaction.status"; import { TransactionQueryOptions } from "../transactions/entities/transactions.query.options"; import { TransferService } from "./transfer.service"; import { ParseArrayPipeOptions } from "@multiversx/sdk-nestjs-common/lib/pipes/entities/parse.array.options"; +import { TimestampParsePipe } from "src/utils/timestamp.parse.pipe"; @Controller() @ApiTags('transfers') @@ -34,8 +35,8 @@ export class TransferController { @ApiQuery({ name: 'status', description: 'Status of the transaction (success / pending / invalid / fail)', required: false, enum: TransactionStatus }) @ApiQuery({ name: 'order', description: 'Sort order (asc/desc)', required: false, enum: SortOrder }) @ApiQuery({ name: 'fields', description: 'List of fields to filter by', required: false, isArray: true, style: 'form', explode: false }) - @ApiQuery({ name: 'before', description: 'Before timestamp', required: false }) - @ApiQuery({ name: 'after', description: 'After timestamp', required: false }) + @ApiQuery({ name: 'before', description: 'Before timestamp or timestampMs', required: false }) + @ApiQuery({ name: 'after', description: 'After timestamp or timestampMs', required: false }) @ApiQuery({ name: 'round', description: 'Round number', required: false }) @ApiQuery({ name: 'function', description: 'Filter transfers by function name', required: false }) @ApiQuery({ name: 'relayer', description: 'Filter by relayer address', required: false }) @@ -61,8 +62,8 @@ export class TransferController { @Query('miniBlockHash', ParseBlockHashPipe) miniBlockHash?: string, @Query('hashes', ParseArrayPipe) hashes?: string[], @Query('status', new ParseEnumPipe(TransactionStatus)) status?: TransactionStatus, - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('round', ParseIntPipe) round?: number, @Query('order', new ParseEnumPipe(SortOrder)) order?: SortOrder, @Query('fields', ParseArrayPipe) fields?: string[], @@ -119,8 +120,8 @@ export class TransferController { @ApiQuery({ name: 'hashes', description: 'Filter by a comma-separated list of transfer hashes', required: false }) @ApiQuery({ name: 'status', description: 'Status of the transaction (success / pending / invalid / fail)', required: false, enum: TransactionStatus }) @ApiQuery({ name: 'function', description: 'Filter transfers by function name', required: false }) - @ApiQuery({ name: 'before', description: 'Before timestamp', required: false }) - @ApiQuery({ name: 'after', description: 'After timestamp', required: false }) + @ApiQuery({ name: 'before', description: 'Before timestamp or timestampMs', required: false }) + @ApiQuery({ name: 'after', description: 'After timestamp or timestampMs', required: false }) @ApiQuery({ name: 'round', description: 'Round number', required: false }) @ApiQuery({ name: 'relayer', description: 'Filter by the relayer address', required: false }) @ApiQuery({ name: 'isRelayed', description: 'Returns relayed transactions details', required: false, type: Boolean }) @@ -136,8 +137,8 @@ export class TransferController { @Query('hashes', ParseArrayPipe) hashes?: string[], @Query('status', new ParseEnumPipe(TransactionStatus)) status?: TransactionStatus, @Query('function', new ParseArrayPipe(new ParseArrayPipeOptions({ allowEmptyString: true }))) functions?: string[], - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('round', ParseIntPipe) round?: number, @Query('relayer', ParseAddressPipe) relayer?: string, @Query('isRelayed', ParseBoolPipe) isRelayed?: boolean, @@ -176,8 +177,8 @@ export class TransferController { @Query('hashes', ParseArrayPipe) hashes?: string[], @Query('status', new ParseEnumPipe(TransactionStatus)) status?: TransactionStatus, @Query('function', new ParseArrayPipe(new ParseArrayPipeOptions({ allowEmptyString: true }))) functions?: string[], - @Query('before', ParseIntPipe) before?: number, - @Query('after', ParseIntPipe) after?: number, + @Query('before', TimestampParsePipe) before?: number, + @Query('after', TimestampParsePipe) after?: number, @Query('round', ParseIntPipe) round?: number, @Query('relayer', ParseAddressPipe) relayer?: string, @Query('isRelayed', ParseBoolPipe) isRelayed?: boolean, diff --git a/src/endpoints/transfers/transfer.service.ts b/src/endpoints/transfers/transfer.service.ts index fdf9baa20..f827849e8 100644 --- a/src/endpoints/transfers/transfer.service.ts +++ b/src/endpoints/transfers/transfer.service.ts @@ -142,6 +142,13 @@ export class TransferService { transactions.push(transaction); } + const hasSenderFilter = filter.sender || (filter.senders && filter.senders.length > 0); + const hasReceiverFilter = filter.receivers && filter.receivers.length > 0; + + if (filter.address && !hasSenderFilter && !hasReceiverFilter) { + transactions = this.transactionService.reorderAccountSentTransactionsByNonce(transactions, filter.address); + } + if (queryOptions.withBlockInfo || (fields && fields.includesSome(['senderBlockHash', 'receiverBlockHash', 'senderBlockNonce', 'receiverBlockNonce']))) { await this.transactionService.applyBlockInfo(transactions); } @@ -157,6 +164,8 @@ export class TransferService { withActionTransferValue: queryOptions.withActionTransferValue ?? false, }); + this.transactionService.processRelayedInfo(transactions); + return transactions; } diff --git a/src/endpoints/websocket/entities/transfers.custom.payload.ts b/src/endpoints/websocket/entities/transfers.custom.payload.ts new file mode 100644 index 000000000..0c46df93f --- /dev/null +++ b/src/endpoints/websocket/entities/transfers.custom.payload.ts @@ -0,0 +1,41 @@ +import { IsOptional, IsString } from 'class-validator'; +import { DisallowedFieldCombination } from 'src/utils/disallowed.field.combination.constraint'; +import { NoEmptyPayload } from 'src/utils/no.empty.payload.validator'; + +@NoEmptyPayload({ message: `You must add at least one filter from ${TransferCustomSubscribePayload.getClassFields()}` }) +@DisallowedFieldCombination() +export class TransferCustomSubscribePayload { + @IsOptional() + @IsString() + sender?: string; + + @IsOptional() + @IsString() + receiver?: string; + + @IsOptional() + @IsString() + relayer?: string; + + @IsOptional() + @IsString() + function?: string; + + @IsOptional() + @IsString() + address?: string; // sender OR receiver OR relayer. throw error if sender, receiver or relayer is already set + + @IsOptional() + @IsString() + token?: string; + + public static getClassFields(): string[] { + return ['function', 'receiver', 'sender', 'relayer', 'address', 'token']; + } + + public static getFieldsSubstitutions(): Record { + return { + address: ['sender', 'receiver', 'relayer'], + }; + } +} diff --git a/src/test/chain-simulator/config/.env.example b/src/test/chain-simulator/config/.env.example index dcb650fc6..dc915101f 100644 --- a/src/test/chain-simulator/config/.env.example +++ b/src/test/chain-simulator/config/.env.example @@ -1,4 +1,5 @@ CHAIN_SIMULATOR_URL=http://localhost:8085 API_SERVICE_URL=http://localhost:3001 +SUBSCRIPTIONS_SERIVCE_URL=http://localhost:6002 ALICE_ADDRESS=erd1qyu5wthldzr8wx5c9ucg8kjagg0jfs53s8nr3zpz3hypefsdd8ssycr6th BOB_ADDRESS=erd1spyavw0956vq68xj8y4tenjpq2wd5a9p2c6j8gsz7ztyrnpxrruqzu66jx diff --git a/src/test/chain-simulator/config/env.config.ts b/src/test/chain-simulator/config/env.config.ts index d52a5f1ef..70f0a1c35 100644 --- a/src/test/chain-simulator/config/env.config.ts +++ b/src/test/chain-simulator/config/env.config.ts @@ -8,6 +8,7 @@ dotenv.config({ export const config = { chainSimulatorUrl: process.env.CHAIN_SIMULATOR_URL || 'http://localhost:8085', apiServiceUrl: process.env.API_SERVICE_URL || 'http://localhost:3001', + subscriptionsServiceUrl: process.env.SUBSCRIPTIONS_SERVICE_URL || 'http://localhost:6002', aliceAddress: process.env.ALICE_ADDRESS || 'erd1qyu5wthldzr8wx5c9ucg8kjagg0jfs53s8nr3zpz3hypefsdd8ssycr6th', bobAddress: process.env.BOB_ADDRESS || 'erd1spyavw0956vq68xj8y4tenjpq2wd5a9p2c6j8gsz7ztyrnpxrruqzu66jx', }; diff --git a/src/test/chain-simulator/docker/docker-compose.yml b/src/test/chain-simulator/docker/docker-compose.yml index 69e37abe4..fcd3325d4 100644 --- a/src/test/chain-simulator/docker/docker-compose.yml +++ b/src/test/chain-simulator/docker/docker-compose.yml @@ -20,7 +20,7 @@ services: chainsimulator: container_name: chainsimulator - image: multiversx/chainsimulator:v1.8.4-barnard-test2 + image: multiversx/chainsimulator:latest command: ["--node-override-config", "./overridable-config.toml"] volumes: - ./overridable-config.toml:/multiversx/overridable-config.toml diff --git a/src/test/chain-simulator/utils/chain.simulator.operations.ts b/src/test/chain-simulator/utils/chain.simulator.operations.ts index 683b32110..87050fe53 100644 --- a/src/test/chain-simulator/utils/chain.simulator.operations.ts +++ b/src/test/chain-simulator/utils/chain.simulator.operations.ts @@ -97,17 +97,25 @@ export async function issueEsdt(args: IssueEsdtArgs) { export async function transferEsdt(args: TransferEsdtArgs) { const transferValue = args.plainAmountOfTokens * 10 ** 18; - return await sendTransaction( + console.log(`Transferring ${args.plainAmountOfTokens} ${args.tokenIdentifier} from ${args.sender} to ${args.receiver}`); + let hexAmountOfTokens = transferValue.toString(16); + + if (hexAmountOfTokens.length % 2 !== 0) { + hexAmountOfTokens = '0' + hexAmountOfTokens; + } + const txHash = await sendTransaction( new SendTransactionArgs({ chainSimulatorUrl: args.chainSimulatorUrl, sender: args.sender, receiver: args.receiver, dataField: `ESDTTransfer@${Buffer.from(args.tokenIdentifier).toString( 'hex', - )}@${transferValue.toString(16)}`, + )}@${hexAmountOfTokens}`, value: '0', }), ); + console.log(`ESDT transfer completed. Transaction hash: ${txHash}`); + return txHash; } export async function sendTransaction( @@ -623,3 +631,30 @@ export async function transferNftFromTo( console.log(`NFT transfer completed. Transaction hash: ${txHash}`); return txHash; } + +export async function transferEgld( + chainSimulatorUrl: string, + senderAddress: string, + receiverAddress: string, + amountInEgldNominated: number +): Promise { + const amountInEgldNominatedStr = amountInEgldNominated.toString(); + const egldDecimals = '0'.repeat(18); + console.log(`Transferring ${amountInEgldNominated} EGLD from ${senderAddress} to ${receiverAddress}`); + + const txHash = await sendTransaction( + new SendTransactionArgs({ + chainSimulatorUrl, + sender: senderAddress, + receiver: receiverAddress, + value: (amountInEgldNominatedStr.concat(egldDecimals)), + dataField: '', + }), + ); + + console.log(`EGLD transfer completed. Transaction hash: ${txHash}`); + await axios.post( + `${chainSimulatorUrl}/simulator/generate-blocks-until-transaction-processed/${txHash}`, + ); + return txHash; +} diff --git a/src/test/chain-simulator/utils/test.utils.ts b/src/test/chain-simulator/utils/test.utils.ts index 41e20f9fa..02cac1a91 100644 --- a/src/test/chain-simulator/utils/test.utils.ts +++ b/src/test/chain-simulator/utils/test.utils.ts @@ -1,6 +1,6 @@ import axios from 'axios'; import { config } from '../config/env.config'; -import { DeployScArgs } from './chain.simulator.operations'; +import { DeployScArgs, sendTransaction, SendTransactionArgs } from './chain.simulator.operations'; import { fundAddress } from './chain.simulator.operations'; import { deploySc } from './chain.simulator.operations'; import fs from 'fs'; @@ -96,4 +96,23 @@ export class ChainSimulatorUtils { throw error; } } + + public static async pingContract(sender: string, scAddress: string) { + await sendTransaction(new SendTransactionArgs({ + chainSimulatorUrl: config.chainSimulatorUrl, + sender, + receiver: scAddress, + value: '1000000000000000000', + dataField: 'ping', + })); + } + + public static async pongContract(sender: string, scAddress: string) { + await sendTransaction(new SendTransactionArgs({ + chainSimulatorUrl: config.chainSimulatorUrl, + sender, + receiver: scAddress, + dataField: 'pong', + })); + } } diff --git a/src/test/chain-simulator/websocket.subscriptions.cs-e2e.ts b/src/test/chain-simulator/websocket.subscriptions.cs-e2e.ts new file mode 100644 index 000000000..e67eb167c --- /dev/null +++ b/src/test/chain-simulator/websocket.subscriptions.cs-e2e.ts @@ -0,0 +1,321 @@ +import axios from "axios"; +import { config } from "./config/env.config"; +import { fundAddress, issueMultipleEsdts, transferEgld, transferEsdt } from "./utils/chain.simulator.operations"; +import { io, Socket } from "socket.io-client"; +import { ChainSimulatorUtils } from "./utils/test.utils"; + +const WS_SERVER_URL = `${config.subscriptionsServiceUrl}`; + +// --- Test Configuration --- +const verbose = false; // Set true for debugging logs + +const client4SubscriptionConfig = { + pool: { from: 0, size: 25 }, + events: { from: 0, size: 25, shard: 1 }, + transactions: { from: 0, size: 25, status: 'success' }, + blocks: { from: 0, size: 25, shard: 1 }, +}; +// -------------------------- + +const log = (...args: any[]) => { + if (verbose) { + console.log(...args); + } +}; + +const txResponses: Map = new Map(); +const eventResponses: Map = new Map(); +const transferResponses: Map = new Map(); // New: Store transfers + +const generalResponses = { + pool: [] as any[], + events: [] as any[], + transactions: [] as any[], + blocks: [] as any[], + stats: [] as any[], +}; + +const txFilters = { + CLIENT_1: { sender: config.aliceAddress }, + CLIENT_2: { sender: config.bobAddress }, + CLIENT_3: { sender: config.aliceAddress, receiver: config.bobAddress }, +}; + +const eventFilters = { + CLIENT_1: { identifier: 'pong' }, + CLIENT_2: { address: '' }, + CLIENT_3: { identifier: 'completedTxEvent', address: '' }, +}; + +const transferFilters = { + CLIENT_5: { address: config.aliceAddress }, // Filter by Address (Sender or Receiver) + CLIENT_6: { token: 'EGLD', address: config.aliceAddress }, // Filter by EGLD only + CLIENT_7: { token: '' }, // Filter by specific ESDT (populated later) +}; + +const filterKeys = { + CLIENT_1: "KEY_CLIENT_1", + CLIENT_2: "KEY_CLIENT_2", + CLIENT_3: "KEY_CLIENT_3", + CLIENT_5: "KEY_CLIENT_5_ADDR", + CLIENT_6: "KEY_CLIENT_6_EGLD", + CLIENT_7: "KEY_CLIENT_7_ESDT", +}; + +// Map configuration to clients +const filterMap = [ + // TX & Event Clients + { key: filterKeys.CLIENT_1, txFilter: txFilters.CLIENT_1, eventFilter: eventFilters.CLIENT_1, transferFilter: null, clientId: "client1" }, + { key: filterKeys.CLIENT_2, txFilter: txFilters.CLIENT_2, eventFilter: eventFilters.CLIENT_2, transferFilter: null, clientId: "client2" }, + { key: filterKeys.CLIENT_3, txFilter: txFilters.CLIENT_3, eventFilter: eventFilters.CLIENT_3, transferFilter: null, clientId: "client3" }, + + // Transfer Clients + { key: filterKeys.CLIENT_5, txFilter: null, eventFilter: null, transferFilter: transferFilters.CLIENT_5, clientId: "client5_addr" }, + { key: filterKeys.CLIENT_6, txFilter: null, eventFilter: null, transferFilter: transferFilters.CLIENT_6, clientId: "client6_egld" }, + { key: filterKeys.CLIENT_7, txFilter: null, eventFilter: null, transferFilter: transferFilters.CLIENT_7, clientId: "client7_esdt" }, +]; + +let pingPongScAddress = ''; +const aliceEsdts: string[] = []; + +describe('Websocket subscriptions e2e tests', () => { + const clients: Socket[] = []; + + // --- Connect Helper --- + const connectAndSubscribe = ( + filterKey: string, + txFilter: any, + eventFilter: any, + transferFilter: any, + clientId: string + ) => { + const receivedTxs: any[] = []; + const receivedEvents: any[] = []; + const receivedTransfers: any[] = []; + + txResponses.set(filterKey, receivedTxs); + eventResponses.set(filterKey, receivedEvents); + transferResponses.set(filterKey, receivedTransfers); + + const client: Socket = io(WS_SERVER_URL, { + path: '/ws/subscription', + }); + clients.push(client); + + client.on("connect_error", (err) => { + throw new Error(`${clientId} connection failed: ${err.message}`); + }); + + client.on("customTransactionUpdate", (data: { transactions: any[] }) => { + log(`\nšŸ’ø ${clientId} received ${data.transactions.length} txs`); + receivedTxs.push(...data.transactions); + }); + + client.on("customEventUpdate", (data: { events: any[] }) => { + log(`\nšŸ”” ${clientId} received ${data.events.length} events`); + receivedEvents.push(...data.events); + }); + + client.on("customTransferUpdate", (data: { transfers: any[] }) => { + log(`\nšŸ’Ž ${clientId} received ${data.transfers.length} transfers`); + receivedTransfers.push(...data.transfers); + }); + + client.on("connect", () => { + log(`\n ${clientId} connected.`); + + if (txFilter) { + client.emit("subscribeCustomTransactions", txFilter, (ack: any) => log(` ACK TXs ${clientId}:`, ack)); + } + if (eventFilter) { + client.emit("subscribeCustomEvents", eventFilter, (ack: any) => log(` ACK Events ${clientId}:`, ack)); + } + if (transferFilter) { + client.emit("subscribeCustomTransfers", transferFilter, (ack: any) => log(` ACK Transfers ${clientId}:`, ack)); + } + }); + }; + + const connectAndSubscribeGeneral = (clientId: string, subConfig: typeof client4SubscriptionConfig) => { + const client: Socket = io(WS_SERVER_URL, { + path: '/ws/subscription', + }); + clients.push(client); + + client.on("connect_error", (err) => { throw new Error(`${clientId} connection failed: ${err.message}`); }); + + client.on("poolUpdate", (data: any) => generalResponses.pool.push(data)); + client.on("eventsUpdate", (data: any) => generalResponses.events.push(data)); + client.on("transactionUpdate", (data: any) => generalResponses.transactions.push(data)); + client.on("blocksUpdate", (data: any) => generalResponses.blocks.push(data)); + client.on("statsUpdate", (data: any) => generalResponses.stats.push(data)); + + client.on("connect", () => { + log(`\n ${clientId} connected with specific configs.`); + client.emit("subscribePool", subConfig.pool, (ack: any) => log(`ACK Pool ${clientId}:`, ack)); + client.emit("subscribeEvents", subConfig.events, (ack: any) => log(`ACK Events ${clientId}:`, ack)); + client.emit("subscribeTransactions", subConfig.transactions, (ack: any) => log(`ACK Txs ${clientId}:`, ack)); + client.emit("subscribeBlocks", subConfig.blocks, (ack: any) => log(`ACK Blocks ${clientId}:`, ack)); + client.emit("subscribeStats", (ack: any) => log(`ACK Stats ${clientId}:`, ack)); + }); + }; + + beforeAll(async () => { + try { + await fundAddress(config.chainSimulatorUrl, config.aliceAddress); + await fundAddress(config.chainSimulatorUrl, config.bobAddress); + await axios.post(`${config.chainSimulatorUrl}/simulator/generate-blocks/1`); + + pingPongScAddress = await ChainSimulatorUtils.deployPingPongSc(config.bobAddress); + eventFilters.CLIENT_2.address = pingPongScAddress; + eventFilters.CLIENT_3.address = pingPongScAddress; + + log("Issuing ESDT Token..."); + const newAliceEsdts = await issueMultipleEsdts(config.chainSimulatorUrl, config.aliceAddress, 1); + aliceEsdts.push(...newAliceEsdts); + + await axios.post(`${config.chainSimulatorUrl}/simulator/generate-blocks/10`); + + for (const item of filterMap) { + if (item.key === filterKeys.CLIENT_7) { + item.transferFilter = { token: aliceEsdts[0] }; + } + connectAndSubscribe(item.key, item.txFilter, item.eventFilter, item.transferFilter, item.clientId); + } + + connectAndSubscribeGeneral("client4", client4SubscriptionConfig); + + await new Promise(resolve => setTimeout(resolve, 10000)); + + log("\n--- Starting Operations ---"); + + + await transferEgld(config.chainSimulatorUrl, config.aliceAddress, config.bobAddress, 1); + await transferEgld(config.chainSimulatorUrl, config.bobAddress, config.aliceAddress, 2); + + await ChainSimulatorUtils.pingContract(config.aliceAddress, pingPongScAddress); + await ChainSimulatorUtils.pongContract(config.aliceAddress, pingPongScAddress); + + await transferEsdt({ + chainSimulatorUrl: config.chainSimulatorUrl, + sender: config.aliceAddress, + receiver: config.bobAddress, + tokenIdentifier: aliceEsdts[0], + plainAmountOfTokens: 1, + }); + + await axios.post(`${config.chainSimulatorUrl}/simulator/generate-blocks/10`); + + log("Waiting for WS messages..."); + await new Promise(resolve => setTimeout(resolve, 20000)); + + } catch (e: any) { + console.error("Error in beforeAll:", e.message); + throw e; + } + }); + + afterAll(() => { + clients.forEach(client => client.connected && client.disconnect()); + }); + + it('should receive TXs sent by Alice for Client 1', () => { + const txs = txResponses.get(filterKeys.CLIENT_1); + expect(txs?.length).toBe(4); + + txs?.forEach((tx) => { + expect(tx.sender).toEqual(config.aliceAddress); + }); + }); + + it('should receive Events with identifier "pong" for Client 1', () => { + const events = eventResponses.get(filterKeys.CLIENT_1); + expect(events?.length).toBe(1); + + events?.forEach((evt) => { + expect(evt.identifier).toEqual('pong'); + }); + }); + + it('should receive TXs sent by Bob for Client 2', () => { + const txs = txResponses.get(filterKeys.CLIENT_2); + expect(txs?.length).toBe(1); + + txs?.forEach((tx) => { + expect(tx.sender).toEqual(config.bobAddress); + }); + }); + + it('should receive Events generated by PingPong contract (address) for Client 2', () => { + const events = eventResponses.get(filterKeys.CLIENT_2); + expect(events?.length).toBe(6); + + events?.forEach((evt) => { + expect(evt.address).toEqual(pingPongScAddress); + }); + }); + + it('should receive specific Alice-to-Bob TXs for Client 3', () => { + const txs = txResponses.get(filterKeys.CLIENT_3); + expect(txs?.length).toBeGreaterThanOrEqual(1); + txs?.forEach((tx) => { + expect(tx.sender).toEqual(config.aliceAddress); + expect(tx.receiver).toEqual(config.bobAddress); + }); + }); + + it('should receive ANY transfer involving Alice (Client 5 - Address Filter)', () => { + const transfers = transferResponses.get(filterKeys.CLIENT_5); + expect(transfers?.length).toBeGreaterThan(0); + + transfers?.forEach(t => { + const isAliceInvolved = t.sender === config.aliceAddress || t.receiver === config.aliceAddress; + expect(isAliceInvolved).toBe(true); + }); + }); + + it('should receive ONLY EGLD transfers where ALICE is involved (Client 6 - Token EGLD Filter)', () => { + const transfers = transferResponses.get(filterKeys.CLIENT_6); + expect(transfers?.length).toBeGreaterThan(0); + + transfers?.forEach(t => { + const val1 = `1${'0'.repeat(18)}`; + const val2 = `2${'0'.repeat(18)}`; + expect([val1, val2]).toContain(t.value); + + const isAliceInvolved = + t.sender === config.aliceAddress || + t.receiver === config.aliceAddress || + t.relayer === config.aliceAddress; + + expect(isAliceInvolved).toBe(true); + }); + }); + + it('should receive ONLY specific ESDT transfers (Client 7 - Dynamic Token Filter)', () => { + const transfers = transferResponses.get(filterKeys.CLIENT_7); + expect(transfers?.length).toBeGreaterThan(0); + + transfers?.forEach(t => { + const esdtTransfers = t.action?.arguments?.transfers; + const containsAliceEsdt = esdtTransfers.filter((et: any) => et.token === aliceEsdts[0]).length > 0; + expect(containsAliceEsdt).toBe(true); + }); + }); + + it('should receive Blocks updates for Client 4', () => { + expect(generalResponses.blocks.length).toBeGreaterThan(0); + }); + it('should receive Transactions updates for Client 4', () => { + expect(generalResponses.transactions.length).toBeGreaterThan(0); + }); + it('should receive Events updates for Client 4', () => { + expect(generalResponses.events.length).toBeGreaterThan(0); + }); + it('should receive Stats updates for Client 4', () => { + expect(generalResponses.stats.length).toBeGreaterThan(0); + }); + it('should have valid subscription structure for Pool updates', () => { + expect(Array.isArray(generalResponses.pool)).toBe(true); + }); +}); diff --git a/src/test/unit/controllers/network.controller.spec.ts b/src/test/unit/controllers/network.controller.spec.ts index f902919a9..e4ba8bee8 100644 --- a/src/test/unit/controllers/network.controller.spec.ts +++ b/src/test/unit/controllers/network.controller.spec.ts @@ -9,6 +9,7 @@ import request = require('supertest'); import { Economics } from "src/endpoints/network/entities/economics"; import { Stats } from "src/endpoints/network/entities/stats"; import { About } from "src/endpoints/network/entities/about"; +import { FeatureConfigs } from "../../../endpoints/network/entities/feature.configs"; describe("NetworkController", () => { let app: INestApplication; @@ -102,12 +103,16 @@ describe("NetworkController", () => { indexerVersion: "v1.4.19", gatewayVersion: "v1.1.44-0-g5282fa5", scamEngineVersion: "1.0.0", - features: { + features: new FeatureConfigs({ updateCollectionExtraDetails: true, marketplace: true, exchange: true, dataApi: true, - }, + tokensFetch: false, + providersFetch: true, + stakingV5: true, + stakingV5ActivationEpoch: 37, + }), }; networkServiceMocks.getAbout.mockResolvedValue(mockAbout); diff --git a/src/test/unit/crons/websocket/room.key.generator.spec.ts b/src/test/unit/crons/websocket/room.key.generator.spec.ts new file mode 100644 index 000000000..85a0ca4a6 --- /dev/null +++ b/src/test/unit/crons/websocket/room.key.generator.spec.ts @@ -0,0 +1,102 @@ +import { RoomKeyGenerator } from 'src/crons/websocket/room.key.generator'; +import { TransactionCustomSubscribePayload } from 'src/endpoints/transactions/entities/dtos/transaction.custom.subscribe'; + +describe('RoomKeyGenerator', () => { + describe('deterministicStringify', () => { + it('sorts object keys alphabetically', () => { + const input = { b: 2, a: 1, c: 3 } as Record; + const result = RoomKeyGenerator.deterministicStringify(input); + expect(result).toBe('{"a":1,"b":2,"c":3}'); + }); + }); + + describe('generate', () => { + it('returns empty array when no active filters', () => { + expect( + RoomKeyGenerator.generate('', {}, TransactionCustomSubscribePayload), + ).toEqual([]); + + expect( + RoomKeyGenerator.generate( + '', + { sender: undefined, receiver: null, function: '' }, + TransactionCustomSubscribePayload, + ), + ).toEqual([]); + }); + + it('ignores keys not present in DTO', () => { + const data = { + sender: 'alice', + receiver: 'bob', + function: 'transfer', + other: 123, // should be ignored + } as Record; + + const rooms = RoomKeyGenerator.generate('', data, TransactionCustomSubscribePayload); + // with 3 active fields, we expect 2^3 - 1 = 7 rooms + expect(rooms.length).toBe(7); + // None of the room strings should include the ignored key + expect(rooms.every((r) => !r.includes('other'))).toBe(true); + }); + + it('generates all combinations for provided filters', () => { + const data = { + sender: 'alice', + receiver: 'bob', + function: 'transfer', + } as Record; + + const rooms = RoomKeyGenerator.generate('', data, TransactionCustomSubscribePayload); + // 3 active fields -> 7 combinations + expect(rooms).toHaveLength(7); + + // Build the expected set of JSON payloads (without prefix) + const expectedPayloads = [ + { function: 'transfer' }, + { receiver: 'bob' }, + { sender: 'alice' }, + { function: 'transfer', receiver: 'bob' }, + { function: 'transfer', sender: 'alice' }, + { receiver: 'bob', sender: 'alice' }, + { function: 'transfer', receiver: 'bob', sender: 'alice' }, + ].map((obj) => RoomKeyGenerator.deterministicStringify(obj)); + + // Sort and compare as sets to avoid order sensitivity + const sortedRooms = [...rooms].sort(); + const sortedExpected = [...expectedPayloads].sort(); + + expect(sortedRooms).toEqual(sortedExpected); + }); + + it('applies custom prefix consistently vs no prefix', () => { + const data = { + sender: 'alice', + receiver: 'bob', + function: 'transfer', + } as Record; + + const prefix = 'custom:'; + const withPrefix = RoomKeyGenerator.generate(prefix, data, TransactionCustomSubscribePayload).sort(); + const withoutPrefix = RoomKeyGenerator.generate('', data, TransactionCustomSubscribePayload).sort(); + + expect(withPrefix.length).toBe(withoutPrefix.length); + for (let i = 0; i < withPrefix.length; i++) { + expect(withPrefix[i]).toBe(prefix + withoutPrefix[i]); + } + }); + + it('filters out null/undefined/empty string values', () => { + const data = { + sender: 'alice', + receiver: '', // should be ignored + function: undefined, // should be ignored + } as Record; + + const rooms = RoomKeyGenerator.generate('', data, TransactionCustomSubscribePayload); + // Only one active key (sender) -> 1 combination + expect(rooms).toHaveLength(1); + expect(rooms[0]).toBe('{"sender":"alice"}'); + }); + }); +}); diff --git a/src/test/unit/services/collections.spec.ts b/src/test/unit/services/collections.spec.ts index 30c7243fb..3d47d0e4f 100644 --- a/src/test/unit/services/collections.spec.ts +++ b/src/test/unit/services/collections.spec.ts @@ -8,11 +8,9 @@ import { PersistenceService } from "src/common/persistence/persistence.service"; import { PluginService } from "src/common/plugins/plugin.service"; import { CollectionService } from "src/endpoints/collections/collection.service"; import { CollectionFilter } from "src/endpoints/collections/entities/collection.filter"; -import { NftCollection } from "src/endpoints/collections/entities/nft.collection"; import { NftCollectionDetailed } from "src/endpoints/collections/entities/nft.collection.detailed"; import { EsdtAddressService } from "src/endpoints/esdt/esdt.address.service"; import { EsdtService } from "src/endpoints/esdt/esdt.service"; -import { NftType } from "src/endpoints/nfts/entities/nft.type"; import { CollectionRoles } from "src/endpoints/tokens/entities/collection.roles"; import { TokenAssetStatus } from "src/endpoints/tokens/entities/token.asset.status"; import { VmQueryService } from "src/endpoints/vm.query/vm.query.service"; @@ -107,7 +105,9 @@ describe('CollectionService', () => { provide: CacheService, useValue: { get: jest.fn(), - getOrSet: jest.fn(), + getOrSet: jest.fn().mockImplementation(async (_key: string, getter: () => Promise, _ttl: number) => { + return await getter(); + }), batchGetAll: jest.fn(), batchApplyAll: jest.fn(), }, @@ -133,6 +133,7 @@ describe('CollectionService', () => { { getTokenAssets: jest.fn(), getCollectionRanks: jest.fn(), + getAllTokenAssets: jest.fn().mockResolvedValue({}), }, }, { @@ -209,56 +210,17 @@ describe('CollectionService', () => { }); describe('getCollection', () => { - const propertiesToCollectionsMock: NftCollection = { - collection: 'XDAY23TEAM-f7a346', - type: NftType.NonFungibleESDT, - subType: undefined, - name: 'xPortalAchievements', - ticker: 'XDAY23TEAM', - owner: 'erd1lpc6wjh2hav6q50p8y6a44r2lhtnseqksygakjfgep6c9uduchkqphzu6t', - timestamp: 0, - canFreeze: true, - canWipe: true, - canPause: true, - canTransferNftCreateRole: true, - canChangeOwner: false, - canUpgrade: false, - canAddSpecialRoles: false, - decimals: undefined, - assets: { - website: 'https://xday.com', - description: - 'Test description.', - status: TokenAssetStatus.active, - pngUrl: 'https://media.elrond.com/tokens/asset/XDAY23TEAM-f7a346/logo.png', - name: '', - svgUrl: 'https://media.elrond.com/tokens/asset/XDAY23TEAM-f7a346/logo.svg', - extraTokens: [''], - ledgerSignature: '', - priceSource: undefined, - preferredRankAlgorithm: undefined, - lockedAccounts: undefined, - }, - scamInfo: undefined, - traits: [], - auctionStats: undefined, - isVerified: undefined, - holderCount: undefined, - nftCount: undefined, - }; - it('should return collection details for a given collection identifier', async () => { const identifier = 'XDAY23TEAM-f7a346'; jest.spyOn(indexerService, 'getCollection').mockResolvedValue(indexerCollectionMock); - jest.spyOn(service, 'applyPropertiesToCollections').mockResolvedValue([propertiesToCollectionsMock]); + jest.spyOn(service, 'getNftCollectionRolesFromGateway').mockResolvedValue([]); const result = await service.getNftCollection(identifier); expect(result).toBeInstanceOf(Object); expect(indexerService.getCollection).toHaveBeenCalledTimes(1); expect(indexerService.getCollection).toHaveBeenCalledWith(identifier); - expect(service.applyPropertiesToCollections).toHaveBeenCalledWith([identifier]); }); it('should return undefined if the collection is not found', async () => { @@ -293,23 +255,26 @@ describe('CollectionService', () => { expect(result).toBeUndefined(); }); - it('should return undefined if no additional properties are applied to the collection', async () => { - const identifier = 'XDAY23TEAM'; + it('should return collection when ES data is available', async () => { + const identifier = 'XDAY23TEAM-f7a346'; jest.spyOn(indexerService, 'getCollection').mockResolvedValue(indexerCollectionMock); - jest.spyOn(service, 'applyPropertiesToCollections').mockResolvedValue([]); + jest.spyOn(service, 'getNftCollectionRolesFromGateway').mockResolvedValue([]); const result = await service.getNftCollection(identifier); - expect(result).toBeUndefined(); + expect(result).toBeDefined(); + expect(result?.collection).toBe('XDAY23TEAM-f7a346'); }); it('should process the collection details fully', async () => { - const identifier = 'XDAY23TEAM'; + const identifier = 'XDAY23TEAM-f7a346'; jest.spyOn(indexerService, 'getCollection').mockResolvedValue(indexerCollectionMock); - jest.spyOn(service, 'applyPropertiesToCollections').mockResolvedValue([propertiesToCollectionsMock]); + jest.spyOn(service, 'getNftCollectionRolesFromGateway').mockResolvedValue([]); const result = await service.getNftCollection(identifier); expect(result).toBeInstanceOf(NftCollectionDetailed); + expect(result?.name).toBe(indexerCollectionMock.name); + expect(result?.owner).toBe(indexerCollectionMock.currentOwner); }); }); diff --git a/src/test/unit/services/transactions.spec.ts b/src/test/unit/services/transactions.spec.ts index 962ac2da1..31f0b589c 100644 --- a/src/test/unit/services/transactions.spec.ts +++ b/src/test/unit/services/transactions.spec.ts @@ -67,7 +67,9 @@ describe('TransactionService', () => { provide: CacheService, useValue: { get: jest.fn(), - getOrSet: jest.fn(), + getOrSet: jest.fn().mockImplementation(async (_key: string, getter: () => Promise, _ttl: number) => { + return await getter(); + }), batchGetAll: jest.fn(), }, }, @@ -294,5 +296,236 @@ describe('TransactionService', () => { expect(service.applyBlockInfo).toHaveBeenCalledWith(expect.any(Array)); }); + + it('should reorder transactions sent by account address by nonce when address is provided', async () => { + const accountAddress = 'erd10v2kud9534x8resv7j2zleunakq2xkjdd8craelhjjksw6y2w36qfs8w8p'; + const filter = new TransactionFilter(); + const pagination = new QueryPagination(); + + const mockTransactions: Transaction[] = [ + { + hash: 'd5a81dcf6f93c69d29d19c0db0d497839cae9ebdf45b0d6341d5e7d7b36afc41', + miniBlockHash: 'c30138148f350bcb1cafacfc8bcca601bd55b5c5eec21e322cbf5cf162d0bdd8', + nonce: 11, + round: 28011213, + value: '100000000000000000', + receiver: 'erd1qga7ze0l03chfgru0a32wxqf2226nzrxnyhzer9lmudqhjgy7ycqjjyknz', + receiverUserName: '', + receiverUsername: '', + sender: accountAddress, + senderUserName: '', + senderUsername: '', + receiverShard: 2, + senderShard: 0, + gasPrice: '1000000200', + gasLimit: '75000', + gasUsed: '69500', + fee: '69500013900000', + data: 'WW9pbmsuIEhlaGVoZQ==', + signature: 'f61f3950c151f032185fe8d7780b19d74f99dfd050969b4202966ce4d7252c6c2810439a9b9f290d21e83927e337be72361b275b3cbdc9fe09db6401c459fb07', + timestamp: 1764184878, + status: 'success', + searchOrder: 0, + hasScResults: false, + hasOperations: false, + tokens: [], + esdtValues: [], + receivers: [], + receiversShardIDs: [], + operation: 'transfer', + scResults: [], + relayerAddr: '', + version: 1, + relayer: '', + isRelayed: false, + isScCall: false, + relayerSignature: '', + timestampMs: 1764184878000, + }, + { + hash: 'd5a81dcf6f93c69d29d19c0db0d497839cae9ebdf45b0d6341d5e7d7b36afc40', + miniBlockHash: '2d28a4dd003b166794707b374611c3fed119e997c8fc9b162e34a378e2aee366', + nonce: 12, + round: 28011212, + value: '100000000000000000', + receiver: 'erd1qga7ze0l03chfgru0a32wxqf2226nzrxnyhzer9lmudqhjgy7ycqjjyknz', + receiverUserName: '', + receiverUsername: '', + sender: accountAddress, + senderUserName: '', + senderUsername: '', + receiverShard: 2, + senderShard: 0, + gasPrice: '1000000200', + gasLimit: '75000', + gasUsed: '75000', + fee: '69555013911000', + data: 'WW9pbmsuIEhlaGVoZQ==', + signature: '3212d61815bcc09cb7513aaf53a668259ea0d33fba76f769068a727154837845f05379c3a7fdc7d6b6b5edfe90f61cac8a65dc0de6d84244235ff4fad021e50e', + timestamp: 1764184872, + status: 'invalid', + searchOrder: 0, + hasScResults: false, + hasOperations: false, + tokens: [], + esdtValues: [], + receivers: [], + receiversShardIDs: [], + operation: 'transfer', + scResults: [], + relayerAddr: '', + version: 1, + relayer: '', + isRelayed: false, + isScCall: false, + relayerSignature: '', + timestampMs: 1764184872000, + }, + { + hash: 'abc123def456', + miniBlockHash: 'mini123', + nonce: 100, + round: 28011210, + value: '50000000000000000', + receiver: accountAddress, + receiverUserName: '', + receiverUsername: '', + sender: 'erd1qqqqqqqqqqqqqpgq7rwhny4mx6dhuzcsymrhdsv2vmvarecgh4vq687aqr', + senderUserName: '', + senderUsername: '', + receiverShard: 0, + senderShard: 1, + gasPrice: '1000000000', + gasLimit: '50000', + gasUsed: '50000', + fee: '50000000000000', + data: '', + signature: 'sig123', + timestamp: 1764184875, + status: 'success', + searchOrder: 0, + hasScResults: false, + hasOperations: false, + tokens: [], + esdtValues: [], + receivers: [], + receiversShardIDs: [], + operation: 'transfer', + scResults: [], + relayerAddr: '', + version: 1, + relayer: '', + isRelayed: false, + isScCall: false, + relayerSignature: '', + timestampMs: 1764184875000, + }, + ]; + + jest.spyOn(indexerService, 'getTransactions').mockResolvedValue(mockTransactions); + jest.spyOn(assetsService, 'getAllAccountAssets').mockResolvedValue({}); + + const results = await service.getTransactions(filter, pagination, undefined, accountAddress); + + const accountSentTxs = results.filter(tx => tx.sender === accountAddress); + expect(accountSentTxs).toHaveLength(2); + expect(accountSentTxs[0].nonce).toBe(12); + expect(accountSentTxs[1].nonce).toBe(11); + }); + + it('should not reorder transactions when sender or receiver filter is applied', async () => { + const accountAddress = 'erd10v2kud9534x8resv7j2zleunakq2xkjdd8craelhjjksw6y2w36qfs8w8p'; + const filter = new TransactionFilter({ sender: accountAddress }); + const pagination = new QueryPagination(); + + const mockTransactions: Transaction[] = [ + { + hash: 'tx1hash', + miniBlockHash: 'mini1', + nonce: 11, + round: 1000, + value: '1000000', + receiver: 'erd1receiver', + receiverUserName: '', + receiverUsername: '', + sender: accountAddress, + senderUserName: '', + senderUsername: '', + receiverShard: 0, + senderShard: 0, + gasPrice: '1000000000', + gasLimit: '50000', + gasUsed: '50000', + fee: '50000000000000', + data: '', + signature: 'sig1', + timestamp: 1764184878, + status: 'success', + searchOrder: 0, + hasScResults: false, + hasOperations: false, + tokens: [], + esdtValues: [], + receivers: [], + receiversShardIDs: [], + operation: 'transfer', + scResults: [], + relayerAddr: '', + version: 1, + relayer: '', + isRelayed: false, + isScCall: false, + relayerSignature: '', + timestampMs: 1764184878000, + }, + { + hash: 'tx2hash', + miniBlockHash: 'mini2', + nonce: 12, + round: 999, + value: '1000000', + receiver: 'erd1receiver', + receiverUserName: '', + receiverUsername: '', + sender: accountAddress, + senderUserName: '', + senderUsername: '', + receiverShard: 0, + senderShard: 0, + gasPrice: '1000000000', + gasLimit: '50000', + gasUsed: '50000', + fee: '50000000000000', + data: '', + signature: 'sig2', + timestamp: 1764184872, + status: 'success', + searchOrder: 0, + hasScResults: false, + hasOperations: false, + tokens: [], + esdtValues: [], + receivers: [], + receiversShardIDs: [], + operation: 'transfer', + scResults: [], + relayerAddr: '', + version: 1, + relayer: '', + isRelayed: false, + isScCall: false, + relayerSignature: '', + timestampMs: 1764184872000, + }, + ]; + + jest.spyOn(indexerService, 'getTransactions').mockResolvedValue(mockTransactions); + jest.spyOn(assetsService, 'getAllAccountAssets').mockResolvedValue({}); + + const results = await service.getTransactions(filter, pagination, undefined, accountAddress); + + expect(results[0].nonce).toBe(11); + expect(results[1].nonce).toBe(12); + }); }); }); diff --git a/src/test/unit/utils/disallowed.field.combination.constraint.spec.ts b/src/test/unit/utils/disallowed.field.combination.constraint.spec.ts new file mode 100644 index 000000000..0c4d0aa36 --- /dev/null +++ b/src/test/unit/utils/disallowed.field.combination.constraint.spec.ts @@ -0,0 +1,77 @@ +import 'reflect-metadata'; +import { validateSync } from 'class-validator'; +import { DisallowedFieldCombination } from '../../../utils/disallowed.field.combination.constraint'; + +@DisallowedFieldCombination() +class TestDto { + // Fields used by the constraint + address?: string; + sender?: string; + receiver?: string; + relayer?: string; + + static getFieldsSubstitutions() { + return { + address: ['sender', 'receiver', 'relayer'], + } as Record; + } +} + +@DisallowedFieldCombination() +class NoMappingDto { + // Intentionally no getFieldsSubstitutions() + foo?: string; +} + +describe('DisallowedFieldCombinationConstraint', () => { + it('is valid when only main field is provided', () => { + const dto = new TestDto(); + dto.address = 'erd1...'; + + const errors = validateSync(dto); + expect(errors).toHaveLength(0); + }); + + it('is valid when only conflicting fields are provided (no main field)', () => { + const dto = new TestDto(); + dto.sender = 'erd1...'; + + const errors = validateSync(dto); + expect(errors).toHaveLength(0); + }); + + it('is invalid when main field and one conflicting field are provided', () => { + const dto = new TestDto(); + dto.address = 'erd1...'; + dto.sender = 'erd1...'; + + const errors = validateSync(dto); + expect(errors).toHaveLength(1); + const constraintMsg = errors[0].constraints?.['disallowedFieldCombination']; + expect(constraintMsg).toBeDefined(); + expect(constraintMsg).toContain("You cannot provide 'address' simultaneously with: sender"); + }); + + it('is invalid and reports all conflicting fields that are present', () => { + const dto = new TestDto(); + dto.address = 'erd1...'; + dto.sender = 'erd1...'; + dto.receiver = 'erd1...'; + + const errors = validateSync(dto); + expect(errors).toHaveLength(1); + const constraintMsg = errors[0].constraints?.['disallowedFieldCombination'] as string; + expect(constraintMsg).toBeDefined(); + expect(constraintMsg).toContain("You cannot provide 'address' simultaneously with:"); + expect(constraintMsg).toContain('sender'); + expect(constraintMsg).toContain('receiver'); + }); + + it('is valid when class does not expose mapping function', () => { + const dto = new NoMappingDto(); + dto.foo = 'bar'; + + const errors = validateSync(dto); + expect(errors).toHaveLength(0); + }); +}); diff --git a/src/utils/cache.info.ts b/src/utils/cache.info.ts index 00059ae49..7c119fdfa 100644 --- a/src/utils/cache.info.ts +++ b/src/utils/cache.info.ts @@ -105,6 +105,18 @@ export class CacheInfo { }; } + static Transactions(queryPagination: QueryPagination): CacheInfo { + return { + key: `transactions:${queryPagination.from}:${queryPagination.size}`, + ttl: Constants.oneSecond() * 6, + }; + } + + static TransactionsCount: CacheInfo = { + key: 'transactions:count', + ttl: Constants.oneSecond() * 6, + }; + static IdentityProfilesKeybases: CacheInfo = { key: 'identityProfilesKeybases', ttl: Constants.oneHour(), @@ -174,9 +186,16 @@ export class CacheInfo { ttl: Constants.oneDay(), }; - static CollectionRanks: CacheInfo = { - key: 'collectionRanks', - ttl: Constants.oneDay(), + static Nfts(queryPagination: QueryPagination): CacheInfo { + return { + key: `nfts:${queryPagination.from}:${queryPagination.size}`, + ttl: Constants.oneSecond() * 6, + }; + } + + static NftsCount: CacheInfo = { + key: 'nfts:count', + ttl: Constants.oneSecond() * 6, }; static AccountAssets: CacheInfo = { @@ -220,6 +239,72 @@ export class CacheInfo { }; } + static CollectionTraits(identifier: string): CacheInfo { + return { + key: `collectionTraits:${identifier}`, + ttl: Constants.oneMinute() * 10, + }; + } + + static CollectionRoles(identifier: string): CacheInfo { + return { + key: `collectionRoles:${identifier}`, + ttl: Constants.oneMinute() * 5, + }; + } + + static CollectionLogo(identifier: string): CacheInfo { + return { + key: `collectionLogo:${identifier}`, + ttl: Constants.oneHour(), + }; + } + + static CollectionRanks: CacheInfo = { + key: 'collectionRanks', + ttl: Constants.oneDay(), + }; + + static CollectionRanksForIdentifier(identifier: string): CacheInfo { + return { + key: `collectionRanks:${identifier}`, + ttl: Constants.oneMinute() * 10, + }; + } + + static CollectionCountForAddress(address: string): CacheInfo { + return { + key: `collectionCount:${address}`, + ttl: Constants.oneMinute(), + }; + } + + static CollectionRolesCountForAddress(address: string): CacheInfo { + return { + key: `collectionRolesCount:${address}`, + ttl: Constants.oneMinute(), + }; + } + + static Collections(pagination: QueryPagination): CacheInfo { + return { + key: `collections:${pagination.from}:${pagination.size}`, + ttl: Constants.oneSecond() * 6, + }; + } + + static CollectionsCount: CacheInfo = { + key: 'collectionsCount', + ttl: Constants.oneSecond() * 6, + }; + + static CollectionsForAddress(address: string, pagination: QueryPagination): CacheInfo { + return { + key: `collectionsForAddress:${address}:${pagination.from}:${pagination.size}`, + ttl: Constants.oneSecond() * 6, + }; + } + static EsdtAddressesRoles(identifier: string): CacheInfo { return { key: `esdt:roles:${identifier}`, @@ -710,4 +795,11 @@ export class CacheInfo { ttl: Constants.oneSecond() * 30, }; } + + static WsTimestampMsToProcess(): CacheInfo { + return { + key: `wsLastProcessedTimestampMs`, + ttl: Constants.oneMinute(), + }; + } } diff --git a/src/utils/disallowed.field.combination.constraint.ts b/src/utils/disallowed.field.combination.constraint.ts new file mode 100644 index 000000000..5cbc62c13 --- /dev/null +++ b/src/utils/disallowed.field.combination.constraint.ts @@ -0,0 +1,73 @@ +import { + ValidatorConstraint, + ValidatorConstraintInterface, + ValidationArguments, + ValidationOptions, + registerDecorator, +} from 'class-validator'; + +@ValidatorConstraint({ name: 'disallowedFieldCombination', async: false }) +export class DisallowedFieldCombinationConstraint + implements ValidatorConstraintInterface { + validate(_value: any, args: ValidationArguments) { + // When used as a class validator, 'value' is usually the instance itself + // However, sometimes 'value' is undefined for class validators, so we rely on args.object + const object = args.object as any; + + if (typeof object.constructor.getFieldsSubstitutions !== 'function') { + return true; + } + + const substitutions = object.constructor.getFieldsSubstitutions() as Record; + + for (const [mainField, conflictingFields] of Object.entries(substitutions)) { + if (object[mainField] !== undefined && object[mainField] !== null) { + const hasConflict = conflictingFields.some( + (field) => object[field] !== undefined && object[field] !== null + ); + + if (hasConflict) { + return false; + } + } + } + + return true; + } + + defaultMessage(args: ValidationArguments) { + const object = args.object as any; + // Safety check in case the method is missing + if (typeof object.constructor.getFieldsSubstitutions !== 'function') { + return 'Validation error'; + } + + const substitutions = object.constructor.getFieldsSubstitutions(); + for (const [mainField, conflictingFields] of Object.entries(substitutions)) { + if (object[mainField]) { + const conflicts = (conflictingFields as string[]).filter( + (field) => object[field] !== undefined && object[field] !== null + ); + + if (conflicts.length > 0) { + return `You cannot provide '${mainField}' simultaneously with: ${conflicts.join(', ')}.`; + } + } + } + + return 'Mutual exclusivity validation failed.'; + } +} + +export function DisallowedFieldCombination(validationOptions?: ValidationOptions) { + return function (target: Function) { + registerDecorator({ + name: 'disallowedFieldCombination', + target: target as Function, + propertyName: '', + options: validationOptions, + constraints: [], + validator: DisallowedFieldCombinationConstraint, + }); + }; +} diff --git a/src/utils/locking.guard.interceptor.ts b/src/utils/locking.guard.interceptor.ts new file mode 100644 index 000000000..75e980504 --- /dev/null +++ b/src/utils/locking.guard.interceptor.ts @@ -0,0 +1,70 @@ +import { Injectable, NestInterceptor, ExecutionContext, CallHandler } from '@nestjs/common'; +import { Observable, from, throwError } from 'rxjs'; +import { switchMap, finalize } from 'rxjs/operators'; +import { Mutex } from 'async-mutex'; +import { WsException } from '@nestjs/websockets'; +import { ApiConfigService } from 'src/common/api-config/api.config.service'; +import { Socket } from 'socket.io'; + +@Injectable() +export class LockingGuardInterceptor implements NestInterceptor { + private readonly locks = new Map(); + + constructor( + private readonly apiConfigService: ApiConfigService + ) { } + + intercept(context: ExecutionContext, next: CallHandler): Observable { + const client: Socket = context.switchToWs().getClient(); + const clientId = client?.id; + + if (!clientId) { + return next.handle(); + } + + let tempMutex = this.locks.get(clientId); + + if (!tempMutex) { + tempMutex = new Mutex(); + this.locks.set(clientId, tempMutex); + } + + const mutex = tempMutex; + + return from(mutex.acquire()).pipe( + switchMap((release) => { + try { + const totalRoomsGlobal = client.nsp.server.sockets.adapter.rooms.size; + const totalClientRooms = client.rooms.size; + const maxGlobal = this.apiConfigService.getWebsocketMaxSubscriptionsPerInstance(); + const maxClient = this.apiConfigService.getWebsocketMaxSubscriptionsPerClient(); + + if (totalRoomsGlobal >= maxGlobal) { + throw new WsException(`Max global subscriptions (${maxGlobal}) reached!`); + } + + if (totalClientRooms >= maxClient + 1) { + throw new WsException(`Max client subscriptions (${maxClient}) reached!`); + } + + return next.handle().pipe( + finalize(() => { + release(); + if (!mutex.isLocked()) { + this.locks.delete(clientId); + } + }) + ); + + } catch (err) { + release(); + + if (!mutex.isLocked()) { + this.locks.delete(clientId); + } + return throwError(() => err); + } + }), + ); + } +} diff --git a/src/utils/no.empty.payload.validator.ts b/src/utils/no.empty.payload.validator.ts new file mode 100644 index 000000000..8a3a7ee40 --- /dev/null +++ b/src/utils/no.empty.payload.validator.ts @@ -0,0 +1,39 @@ +import { + registerDecorator, + ValidationOptions, + ValidatorConstraint, + ValidatorConstraintInterface, + ValidationArguments, +} from 'class-validator'; + +@ValidatorConstraint({ async: false }) +export class NoEmptyPayloadConstraint implements ValidatorConstraintInterface { + validate(_value: any, args: ValidationArguments) { + const object = args.object as any; + + if (!object) { + return false; + } + + return Object.keys(object).some((key) => { + const propertyValue = object[key]; + return propertyValue !== undefined && propertyValue !== null; + }); + } + + defaultMessage(_args: ValidationArguments) { + return 'Payload cannot be empty. At least one property must be provided.'; + } +} + +export function NoEmptyPayload(validationOptions?: ValidationOptions) { + return function (target: Function) { + registerDecorator({ + target: target, + propertyName: '', + options: validationOptions, + constraints: [], + validator: NoEmptyPayloadConstraint, + }); + }; +} diff --git a/src/utils/time.utils.ts b/src/utils/time.utils.ts new file mode 100644 index 000000000..81a13b552 --- /dev/null +++ b/src/utils/time.utils.ts @@ -0,0 +1,6 @@ +export class TimeUtils { + static readonly TIMESTAMP_IN_SECONDS_THRESHOLD = 100_000_000_000; + static isTimestampInSeconds(input: number): boolean { + return input < TimeUtils.TIMESTAMP_IN_SECONDS_THRESHOLD; + } +} diff --git a/src/utils/timestamp.parse.pipe.ts b/src/utils/timestamp.parse.pipe.ts new file mode 100644 index 000000000..ff88184c9 --- /dev/null +++ b/src/utils/timestamp.parse.pipe.ts @@ -0,0 +1,36 @@ +import { PipeTransform, Injectable, BadRequestException } from '@nestjs/common'; +import { ApiConfigService } from 'src/common/api-config/api.config.service'; +import { TimeUtils } from './time.utils'; + +@Injectable() +export class TimestampParsePipe implements PipeTransform { + constructor( + private readonly apiConfigService: ApiConfigService, + ) { } + + transform(value: any): number | undefined { + if (value === undefined || value === null) return undefined; + + const valNumber = parseInt(value, 10); + if (isNaN(valNumber)) { + throw new BadRequestException('Timestamp must be a number'); + } + + if (valNumber <= 0) { + throw new BadRequestException('Timestamp must be a positive number'); + } + + const normalizedInputMs = TimeUtils.isTimestampInSeconds(valNumber) ? valNumber * 1000 : valNumber; + if (!this.apiConfigService.isChainBarnardEnabled()) { + return Math.floor(normalizedInputMs / 1000); + } + + const barnardActivationTimestamp = this.apiConfigService.getChainBarnardActivationTimestamp() * 1000; + + if (normalizedInputMs < barnardActivationTimestamp) { + return Math.floor(normalizedInputMs / 1000); + } else { + return normalizedInputMs; + } + } +}