diff --git a/src/common/indexer/elastic/elastic.indexer.service.ts b/src/common/indexer/elastic/elastic.indexer.service.ts index fc7438ff7..97f4b0799 100644 --- a/src/common/indexer/elastic/elastic.indexer.service.ts +++ b/src/common/indexer/elastic/elastic.indexer.service.ts @@ -1,6 +1,6 @@ import { HttpStatus, Injectable } from "@nestjs/common"; import { BinaryUtils } from "@multiversx/sdk-nestjs-common"; -import { ElasticQuery, QueryOperator, QueryType, QueryConditionOptions, ElasticSortOrder, ElasticSortProperty, TermsQuery, RangeGreaterThanOrEqual, MatchQuery } from "@multiversx/sdk-nestjs-elastic"; +import { ElasticQuery, QueryOperator, QueryType, QueryConditionOptions, ElasticSortOrder, ElasticSortProperty, RangeGreaterThanOrEqual, MatchQuery } from "@multiversx/sdk-nestjs-elastic"; import { IndexerInterface } from "../indexer.interface"; import { ApiConfigService } from "src/common/api-config/api.config.service"; import { CollectionFilter } from "src/endpoints/collections/entities/collection.filter"; @@ -279,9 +279,7 @@ export class ElasticIndexerService implements IndexerInterface { const elasticOperations = await this.elasticService.getList('operations', 'txHash', elasticQuery); - for (const operation of elasticOperations) { - this.processTransaction(operation); - } + this.bulkProcessTransactions(elasticOperations); return elasticOperations; } @@ -338,7 +336,7 @@ export class ElasticIndexerService implements IndexerInterface { .withMustMatchCondition('type', 'unsigned') .withPagination({ from: 0, size: transactionHashes.length + 1 }) .withSort([{ name: 'timestamp', order: ElasticSortOrder.ascending }]) - .withTerms(new TermsQuery('originalTxHash', transactionHashes)); + .withMustMultiShouldCondition(transactionHashes, hash => QueryType.Match('originalTxHash', hash)); return await this.elasticService.getList('operations', 'scHash', elasticQuery); } @@ -346,7 +344,7 @@ export class ElasticIndexerService implements IndexerInterface { async getAccountsForAddresses(addresses: string[]): Promise { const elasticQuery: ElasticQuery = ElasticQuery.create() .withPagination({ from: 0, size: addresses.length + 1 }) - .withTerms(new TermsQuery('address', addresses)); + .withMustMultiShouldCondition(addresses, address => QueryType.Match('address', address)); return await this.elasticService.getList('accounts', 'address', elasticQuery); } @@ -383,9 +381,7 @@ export class ElasticIndexerService implements IndexerInterface { const results = await this.elasticService.getList('operations', 'hash', elasticQuery); - for (const result of results) { - this.processTransaction(result); - } + this.bulkProcessTransactions(results); return results; } @@ -548,9 +544,7 @@ export class ElasticIndexerService implements IndexerInterface { const transactions = await this.elasticService.getList('operations', 'txHash', elasticQuery); - for (const transaction of transactions) { - this.processTransaction(transaction); - } + this.bulkProcessTransactions(transactions); return transactions; } @@ -561,6 +555,19 @@ export class ElasticIndexerService implements IndexerInterface { } } + private bulkProcessTransactions(transactions: any[]) { + if (!transactions || transactions.length === 0) { + return; + } + + for (let i = 0; i < transactions.length; i++) { + const transaction = transactions[i]; + if (transaction && !transaction.function) { + transaction.function = transaction.operation; + } + } + } + private buildTokenFilter(query: ElasticQuery, filter: TokenFilter): ElasticQuery { if (filter.includeMetaESDT === true) { query = query.withMustMultiShouldCondition([TokenType.FungibleESDT, TokenType.MetaESDT], type => QueryType.Match('type', type)); @@ -616,9 +623,7 @@ export class ElasticIndexerService implements IndexerInterface { const results = await this.elasticService.getList('operations', 'hash', elasticQuerySc); - for (const result of results) { - this.processTransaction(result); - } + this.bulkProcessTransactions(results); return results; } @@ -632,9 +637,11 @@ export class ElasticIndexerService implements IndexerInterface { return []; } + const maxSize = Math.min(hashes.length * 10, 1000); + const elasticQuery = ElasticQuery.create() .withMustMatchCondition('type', 'unsigned') - .withPagination({ from: 0, size: 10000 }) + .withPagination({ from: 0, size: maxSize }) .withSort([{ name: 'timestamp', order: ElasticSortOrder.ascending }]) .withMustMultiShouldCondition(hashes, hash => QueryType.Match('originalTxHash', hash)); @@ -646,8 +653,6 @@ export class ElasticIndexerService implements IndexerInterface { return []; } - const queries = identifiers.map((identifier) => QueryType.Match('identifier', identifier, QueryOperator.AND)); - let elasticQuery = ElasticQuery.create(); if (pagination) { @@ -655,10 +660,12 @@ export class ElasticIndexerService implements IndexerInterface { } elasticQuery = elasticQuery - .withSort([{ name: "balanceNum", order: ElasticSortOrder.descending }]) + .withSort([ + { name: "balanceNum", order: ElasticSortOrder.descending }, + { name: 'timestamp', order: ElasticSortOrder.descending }, + ]) .withCondition(QueryConditionOptions.mustNot, [QueryType.Match('address', 'pending')]) - .withCondition(QueryConditionOptions.should, queries) - .withSort([{ name: 'timestamp', order: ElasticSortOrder.descending }]); + .withMustMultiShouldCondition(identifiers, identifier => QueryType.Match('identifier', identifier, QueryOperator.AND)); return await this.elasticService.getList('accountsesdt', 'identifier', elasticQuery); } @@ -668,8 +675,6 @@ export class ElasticIndexerService implements IndexerInterface { return []; } - const queries = identifiers.map((identifier) => QueryType.Match('collection', identifier, QueryOperator.AND)); - let elasticQuery = ElasticQuery.create(); if (pagination) { @@ -677,10 +682,12 @@ export class ElasticIndexerService implements IndexerInterface { } elasticQuery = elasticQuery - .withSort([{ name: "balanceNum", order: ElasticSortOrder.descending }]) + .withSort([ + { name: "balanceNum", order: ElasticSortOrder.descending }, + { name: 'timestamp', order: ElasticSortOrder.descending }, + ]) .withCondition(QueryConditionOptions.mustNot, [QueryType.Match('address', 'pending')]) - .withCondition(QueryConditionOptions.should, queries) - .withSort([{ name: 'timestamp', order: ElasticSortOrder.descending }]); + .withMustMultiShouldCondition(identifiers, identifier => QueryType.Match('collection', identifier, QueryOperator.AND)); return await this.elasticService.getList('accountsesdt', 'identifier', elasticQuery); } @@ -711,11 +718,22 @@ export class ElasticIndexerService implements IndexerInterface { ]); } - let elasticNfts = await this.elasticService.getList('tokens', 'identifier', elasticQuery); - if (elasticNfts.length === 0 && identifier !== undefined) { - elasticNfts = await this.elasticService.getList('accountsesdt', 'identifier', ElasticQuery.create().withMustMatchCondition('identifier', identifier, QueryOperator.AND)); + if (identifier !== undefined) { + const [tokensResult, accountsResult] = await Promise.all([ + this.elasticService.getList('tokens', 'identifier', elasticQuery).catch(() => []), + this.elasticService.getList('accountsesdt', 'identifier', + ElasticQuery.create() + .withMustMatchCondition('identifier', identifier, QueryOperator.AND) + .withPagination(pagination) + ).catch(() => []), + ]); + + const elasticNfts = tokensResult.length > 0 ? tokensResult : accountsResult; + return elasticNfts; + } else { + const elasticNfts = await this.elasticService.getList('tokens', 'identifier', elasticQuery); + return elasticNfts; } - return elasticNfts; } async getTransactionBySenderAndNonce(sender: string, nonce: number): Promise { @@ -819,52 +837,100 @@ export class ElasticIndexerService implements IndexerInterface { } } - const elasticQuery = ElasticQuery.create() - .withMustExistCondition('identifier') - .withMustMatchCondition('address', address) - .withPagination({ from: 0, size: 0 }) - .withMustMatchCondition('token', filter.collection, QueryOperator.AND) - .withMustMultiShouldCondition(filter.identifiers, identifier => QueryType.Match('token', identifier, QueryOperator.AND)) - .withSearchWildcardCondition(filter.search, ['token', 'name']) - .withMustMultiShouldCondition(filterTypes, type => QueryType.Match('type', type)) - .withMustMultiShouldCondition(filter.subType, subType => QueryType.Match('type', subType)) - .withExtra({ - aggs: { - collections: { - composite: { - size: 10000, - sources: [ - { - collection: { - terms: { - field: 'token', - }, - }, - }, - ], + const data: { collection: string, count: number, balance: number }[] = []; + let afterKey: any = null; + let remainingToSkip = pagination.from; + let remainingToCollect = pagination.size; + + while (data.length < pagination.size) { + const batchSize = Math.min(1000, remainingToSkip + remainingToCollect); + + const compositeAgg: any = { + size: batchSize, + sources: [ + { + collection: { + terms: { + field: 'token', + order: 'asc', + }, }, - aggs: { - balance: { - sum: { - field: 'balanceNum', + }, + ], + }; + + if (afterKey) { + compositeAgg.after = afterKey; + } + + const elasticQuery = ElasticQuery.create() + .withMustExistCondition('identifier') + .withMustMatchCondition('address', address) + .withPagination({ from: 0, size: 0 }) + .withMustMatchCondition('token', filter.collection, QueryOperator.AND) + .withMustMultiShouldCondition(filter.identifiers, identifier => QueryType.Match('token', identifier, QueryOperator.AND)) + .withSearchWildcardCondition(filter.search, ['token', 'name']) + .withMustMultiShouldCondition(filterTypes, type => QueryType.Match('type', type)) + .withMustMultiShouldCondition(filter.subType, subType => QueryType.Match('type', subType)) + .withExtra({ + aggs: { + collections: { + composite: compositeAgg, + aggs: { + balance: { + sum: { + field: 'balanceNum', + }, }, }, }, }, - }, - }); + }); - const result = await this.elasticService.post(`${this.apiConfigService.getElasticUrl()}/accountsesdt/_search`, elasticQuery.toJson()); + const result = await this.elasticService.post(`${this.apiConfigService.getElasticUrl()}/accountsesdt/_search`, elasticQuery.toJson()); + const buckets = result?.data?.aggregations?.collections?.buckets || []; + + if (buckets.length === 0) { + break; + } - const buckets = result?.data?.aggregations?.collections?.buckets; + const batchData: { collection: string, count: number, balance: number }[] = buckets.map((bucket: any) => ({ + collection: bucket.key.collection, + count: bucket.doc_count, + balance: bucket.balance.value, + })); - let data: { collection: string, count: number, balance: number }[] = buckets.map((bucket: any) => ({ - collection: bucket.key.collection, - count: bucket.doc_count, - balance: bucket.balance.value, - })); + if (remainingToSkip > 0) { + const skipFromBatch = Math.min(remainingToSkip, batchData.length); + remainingToSkip -= skipFromBatch; + + if (remainingToSkip === 0) { + const collectFromBatch = Math.min(remainingToCollect, batchData.length - skipFromBatch); + data.push(...batchData.slice(skipFromBatch, skipFromBatch + collectFromBatch)); + remainingToCollect -= collectFromBatch; + } + } else { + const collectFromBatch = Math.min(remainingToCollect, batchData.length); + data.push(...batchData.slice(0, collectFromBatch)); + remainingToCollect -= collectFromBatch; + } + + if (remainingToCollect === 0) { + break; + } + + const aggregations = result?.data?.aggregations?.collections; + if (aggregations?.after_key) { + afterKey = aggregations.after_key; + } else { + break; + } + + if (buckets.length < batchSize) { + break; + } + } - data = data.slice(pagination.from, pagination.from + pagination.size); return data; } diff --git a/src/endpoints/collections/collection.service.ts b/src/endpoints/collections/collection.service.ts index b86af5fd5..ffdf74277 100644 --- a/src/endpoints/collections/collection.service.ts +++ b/src/endpoints/collections/collection.service.ts @@ -62,20 +62,18 @@ export class CollectionService { private async processNftCollections(tokenCollections: Collection[]): Promise { const collectionsIdentifiers = tokenCollections.map((collection) => collection.token); - const indexedCollections: Record = {}; + const indexedCollections = new Map(); for (const collection of tokenCollections) { - indexedCollections[collection.token] = collection; + indexedCollections.set(collection.token, collection); } const nftCollections: NftCollection[] = await this.applyPropertiesToCollections(collectionsIdentifiers); for (const nftCollection of nftCollections) { - const indexedCollection = indexedCollections[nftCollection.collection]; - if (!indexedCollection) { - continue; + const indexedCollection = indexedCollections.get(nftCollection.collection); + if (indexedCollection) { + this.applyPropertiesToCollectionFromElasticSearch(nftCollection, indexedCollection); } - - this.applyPropertiesToCollectionFromElasticSearch(nftCollection, indexedCollection); } return nftCollections; @@ -117,8 +115,11 @@ export class CollectionService { async applyPropertiesToCollections(collectionsIdentifiers: string[]): Promise { const nftCollections: NftCollection[] = []; - const collectionsProperties = await this.batchGetCollectionsProperties(collectionsIdentifiers); - const collectionsAssets = await this.batchGetCollectionsAssets(collectionsIdentifiers); + + const [collectionsProperties, collectionsAssets] = await Promise.all([ + this.batchGetCollectionsProperties(collectionsIdentifiers), + this.batchGetCollectionsAssets(collectionsIdentifiers), + ]); for (const collectionIdentifier of collectionsIdentifiers) { const collectionProperties = collectionsProperties[collectionIdentifier]; @@ -126,27 +127,29 @@ export class CollectionService { continue; } - const nftCollection = new NftCollection(); - - // @ts-ignore - nftCollection.type = collectionProperties.type; - nftCollection.name = collectionProperties.name; - nftCollection.collection = collectionIdentifier.split('-').slice(0, 2).join('-'); - nftCollection.ticker = collectionIdentifier.split('-')[0]; - nftCollection.canFreeze = collectionProperties.canFreeze; - nftCollection.canWipe = collectionProperties.canWipe; - nftCollection.canPause = collectionProperties.canPause; - nftCollection.canTransferNftCreateRole = collectionProperties.canTransferNFTCreateRole; - nftCollection.canChangeOwner = collectionProperties.canChangeOwner; - nftCollection.canUpgrade = collectionProperties.canUpgrade; - nftCollection.canAddSpecialRoles = collectionProperties.canAddSpecialRoles; - nftCollection.owner = collectionProperties.owner; - - if (nftCollection.type === NftType.MetaESDT) { - nftCollection.decimals = collectionProperties.decimals; - } + const identifierParts = collectionIdentifier.split('-'); + const ticker = identifierParts[0]; + const collectionBase = identifierParts.slice(0, 2).join('-'); + const assets = collectionsAssets[collectionIdentifier]; + + const nftCollection = new NftCollection({ + // @ts-ignore + type: collectionProperties.type, + name: collectionProperties.name, + collection: collectionBase, + ticker: ticker, + canFreeze: collectionProperties.canFreeze, + canWipe: collectionProperties.canWipe, + canPause: collectionProperties.canPause, + canTransferNftCreateRole: collectionProperties.canTransferNFTCreateRole, + canChangeOwner: collectionProperties.canChangeOwner, + canUpgrade: collectionProperties.canUpgrade, + canAddSpecialRoles: collectionProperties.canAddSpecialRoles, + owner: collectionProperties.owner, + assets: assets, + decimals: (collectionProperties.type as any) === NftType.MetaESDT ? collectionProperties.decimals : undefined, + }); - nftCollection.assets = collectionsAssets[collectionIdentifier]; nftCollection.ticker = nftCollection.assets ? collectionIdentifier.split('-')[0] : nftCollection.collection; nftCollections.push(nftCollection); diff --git a/src/endpoints/transactions/transaction.service.ts b/src/endpoints/transactions/transaction.service.ts index f44c873ae..6167b1fbb 100644 --- a/src/endpoints/transactions/transaction.service.ts +++ b/src/endpoints/transactions/transaction.service.ts @@ -466,18 +466,25 @@ export class TransactionService { private async getSmartContractResultsRaw(transactionHashes: Array): Promise> { const resultsRaw = await this.indexerService.getSmartContractResults(transactionHashes) as any[]; + + const resultsByHash = new Map(); + for (const result of resultsRaw) { result.hash = result.scHash; - delete result.scHash; + + const txHash = result.originalTxHash; + if (!resultsByHash.has(txHash)) { + resultsByHash.set(txHash, []); + } + resultsByHash.get(txHash)!.push(result); } const results: Array = []; - for (const transactionHash of transactionHashes) { - const resultRaw = resultsRaw.filter(({ originalTxHash }) => originalTxHash == transactionHash); + const resultRaw = resultsByHash.get(transactionHash); - if (resultRaw.length > 0) { + if (resultRaw && resultRaw.length > 0) { results.push(resultRaw.map((result: any) => ApiUtils.mergeObjects(new SmartContractResult(), result))); } else { results.push(undefined);