diff --git a/package-lock.json b/package-lock.json index 9e91db6bd..8943605b2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,6 @@ "@aws-sdk/client-s3": "^3.54.0", "@golevelup/nestjs-rabbitmq": "^4.0.0", "@multiversx/sdk-core": "^13.2.2", - "@multiversx/sdk-data-api-client": "^0.7.0", "@multiversx/sdk-exchange": "^0.2.21", "@multiversx/sdk-nestjs-auth": "6.0.1-beta.0", "@multiversx/sdk-nestjs-cache": "6.0.1-beta.0", @@ -3555,6 +3554,7 @@ "resolved": "https://registry.npmjs.org/@multiversx/sdk-bls-wasm/-/sdk-bls-wasm-0.3.5.tgz", "integrity": "sha512-c0tIdQUnbBLSt6NYU+OpeGPYdL0+GV547HeHT8Xc0BKQ7Cj0v82QUoA2QRtWrR1G4MNZmLsIacZSsf6DrIS2Bw==", "license": "BSD-3-Clause", + "optional": true, "engines": { "node": ">=8.9.0" } @@ -3598,69 +3598,6 @@ "uuid": "dist/bin/uuid" } }, - "node_modules/@multiversx/sdk-data-api-client": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/@multiversx/sdk-data-api-client/-/sdk-data-api-client-0.7.0.tgz", - "integrity": "sha512-QOYQwzMnzDuVR93801GXELNHWscfquWqWmw5dxUIJbzRX94BRAqe9AT1tE9aUQqLcdB4DzWlUYdUYsZsDNMppQ==", - "dependencies": { - "@multiversx/sdk-core": "^11.4.1", - "@multiversx/sdk-native-auth-client": "^1.0.0", - "@multiversx/sdk-wallet": "^3.0.0", - "agentkeepalive": "^4.3.0", - "axios": "^1.3.4", - "bignumber.js": "^9.1.1", - "moment": "^2.29.4", - "prettier": "^2.8.4" - } - }, - "node_modules/@multiversx/sdk-data-api-client/node_modules/@multiversx/sdk-core": { - "version": "11.6.0", - "resolved": "https://registry.npmjs.org/@multiversx/sdk-core/-/sdk-core-11.6.0.tgz", - "integrity": "sha512-bP1fmp84iEAJEdaprvw8M9FeqJVOWTmv8vhhiaYW6YTV/ztlB+niv1CFsYk0dWFWIfKzn+eT4CvQjl8OTwBC/A==", - "dependencies": { - "@multiversx/sdk-transaction-decoder": "1.0.2", - "bech32": "1.1.4", - "bignumber.js": "9.0.1", - "blake2b": "2.1.3", - "buffer": "6.0.3", - "json-duplicate-key-handle": "1.0.0", - "keccak": "3.0.2", - "protobufjs": "6.11.3" - } - }, - "node_modules/@multiversx/sdk-data-api-client/node_modules/@multiversx/sdk-core/node_modules/bignumber.js": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.1.tgz", - "integrity": "sha512-IdZR9mh6ahOBv/hYGiXyVuyCetmGJhtYkqLBpTStdhEGjegpPlUawydyaF3pbIOFynJTpllEs+NP+CS9jKFLjA==", - "engines": { - "node": "*" - } - }, - "node_modules/@multiversx/sdk-data-api-client/node_modules/protobufjs": { - "version": "6.11.3", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.3.tgz", - "integrity": "sha512-xL96WDdCZYdU7Slin569tFX712BxsxslWwAfAhCYjQKGTq7dAU91Lomy6nLLhh/dyGhk/YH4TwTSRxTzhuHyZg==", - "hasInstallScript": true, - "dependencies": { - "@protobufjs/aspromise": "^1.1.2", - "@protobufjs/base64": "^1.1.2", - "@protobufjs/codegen": "^2.0.4", - "@protobufjs/eventemitter": "^1.1.0", - "@protobufjs/fetch": "^1.1.0", - "@protobufjs/float": "^1.0.2", - "@protobufjs/inquire": "^1.1.0", - "@protobufjs/path": "^1.1.2", - "@protobufjs/pool": "^1.1.0", - "@protobufjs/utf8": "^1.1.0", - "@types/long": "^4.0.1", - "@types/node": ">=13.7.0", - "long": "^4.0.0" - }, - "bin": { - "pbjs": "bin/pbjs", - "pbts": "bin/pbts" - } - }, "node_modules/@multiversx/sdk-exchange": { "version": "0.2.22", "resolved": "https://registry.npmjs.org/@multiversx/sdk-exchange/-/sdk-exchange-0.2.22.tgz", @@ -3786,6 +3723,7 @@ "version": "8.0.5", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-8.0.5.tgz", "integrity": "sha512-MhWWlVnuab1RG5/zMRRcVGXZLCXrZTgfwMikgzCegsPnG62yDQo5JnqKkrK4jO5iKqDAZGItAqN5CtKBCBWRUA==", + "license": "ISC", "engines": { "node": ">=16.14" } @@ -3794,6 +3732,7 @@ "version": "8.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "license": "MIT", "bin": { "uuid": "dist/bin/uuid" } @@ -3914,6 +3853,7 @@ "version": "8.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "license": "MIT", "bin": { "uuid": "dist/bin/uuid" } @@ -4065,75 +4005,6 @@ "axios": "^1.7.4" } }, - "node_modules/@multiversx/sdk-wallet": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@multiversx/sdk-wallet/-/sdk-wallet-3.0.0.tgz", - "integrity": "sha512-nDVBtva1mpfutXA8TfUnpdeFqhY9O+deNU3U/Z41yPBcju1trHDC9gcKPyQqcQ3qjG/6LwEXmIm7Dc5fIsvVjg==", - "dependencies": { - "@multiversx/sdk-bls-wasm": "0.3.5", - "bech32": "1.1.4", - "bip39": "3.0.2", - "blake2b": "2.1.3", - "ed25519-hd-key": "1.1.2", - "ed2curve": "0.3.0", - "keccak": "3.0.1", - "scryptsy": "2.1.0", - "tweetnacl": "1.0.3", - "uuid": "8.3.2" - } - }, - "node_modules/@multiversx/sdk-wallet/node_modules/@types/node": { - "version": "11.11.6", - "resolved": "https://registry.npmjs.org/@types/node/-/node-11.11.6.tgz", - "integrity": "sha512-Exw4yUWMBXM3X+8oqzJNRqZSwUAaS4+7NdvHqQuFi/d+synz++xmX3QIf+BFqneW8N31R8Ky+sikfZUXq07ggQ==" - }, - "node_modules/@multiversx/sdk-wallet/node_modules/bip39": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/bip39/-/bip39-3.0.2.tgz", - "integrity": "sha512-J4E1r2N0tUylTKt07ibXvhpT2c5pyAFgvuA5q1H9uDy6dEGpjV8jmymh3MTYJDLCNbIVClSB9FbND49I6N24MQ==", - "dependencies": { - "@types/node": "11.11.6", - "create-hash": "^1.1.0", - "pbkdf2": "^3.0.9", - "randombytes": "^2.0.1" - } - }, - "node_modules/@multiversx/sdk-wallet/node_modules/keccak": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/keccak/-/keccak-3.0.1.tgz", - "integrity": "sha512-epq90L9jlFWCW7+pQa6JOnKn2Xgl2mtI664seYR6MHskvI9agt7AnDqmAlp9TqU4/caMYbA08Hi5DMZAl5zdkA==", - "hasInstallScript": true, - "dependencies": { - "node-addon-api": "^2.0.0", - "node-gyp-build": "^4.2.0" - }, - "engines": { - "node": ">=10.0.0" - } - }, - "node_modules/@multiversx/sdk-wallet/node_modules/node-addon-api": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-2.0.2.tgz", - "integrity": "sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA==" - }, - "node_modules/@multiversx/sdk-wallet/node_modules/node-gyp-build": { - "version": "4.8.4", - "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz", - "integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==", - "bin": { - "node-gyp-build": "bin.js", - "node-gyp-build-optional": "optional.js", - "node-gyp-build-test": "build-test.js" - } - }, - "node_modules/@multiversx/sdk-wallet/node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "bin": { - "uuid": "dist/bin/uuid" - } - }, "node_modules/@nestjs/apollo": { "version": "12.0.11", "resolved": "https://registry.npmjs.org/@nestjs/apollo/-/apollo-12.0.11.tgz", @@ -7795,11 +7666,6 @@ "integrity": "sha512-zj6Z6M7Eq+PBZ7PQxl5NT665MvJdAkzp0f60nAJ+sLaSCBPMwVak5ZegFbgVCzFcCJTKFoMizvM5Ld7+JrRJHA==", "license": "MIT" }, - "node_modules/backslash": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/backslash/-/backslash-0.2.0.tgz", - "integrity": "sha512-Avs+8FUZ1HF/VFP4YWwHQZSGzRPm37ukU1JQYQWijuHhtXdOuAzcZ8PcAzfIw898a8PyBzdn+RtnKA6MzW0X2A==" - }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -12304,14 +12170,6 @@ "node": "*" } }, - "node_modules/json-duplicate-key-handle": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-duplicate-key-handle/-/json-duplicate-key-handle-1.0.0.tgz", - "integrity": "sha512-OLIxL+UpfwUsqcLX3i6Z51ChTou/Vje+6bSeGUSubj96dF/SfjObDprLy++ZXYH07KITuEzsXS7PX7e/BGf4jw==", - "dependencies": { - "backslash": "^0.2.0" - } - }, "node_modules/json-parse-even-better-errors": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", @@ -13985,6 +13843,7 @@ "version": "2.8.8", "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.8.8.tgz", "integrity": "sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q==", + "dev": true, "license": "MIT", "bin": { "prettier": "bin-prettier.js" diff --git a/package.json b/package.json index 66a869e19..5257a67ab 100644 --- a/package.json +++ b/package.json @@ -87,7 +87,6 @@ "@aws-sdk/client-s3": "^3.54.0", "@golevelup/nestjs-rabbitmq": "^4.0.0", "@multiversx/sdk-core": "^13.2.2", - "@multiversx/sdk-data-api-client": "^0.7.0", "@multiversx/sdk-exchange": "^0.2.21", "@multiversx/sdk-nestjs-auth": "6.0.1-beta.0", "@multiversx/sdk-nestjs-cache": "6.0.1-beta.0", diff --git a/src/common/indexer/elastic/elastic.indexer.service.ts b/src/common/indexer/elastic/elastic.indexer.service.ts index fc7438ff7..d7f0af8b4 100644 --- a/src/common/indexer/elastic/elastic.indexer.service.ts +++ b/src/common/indexer/elastic/elastic.indexer.service.ts @@ -1,6 +1,6 @@ import { HttpStatus, Injectable } from "@nestjs/common"; import { BinaryUtils } from "@multiversx/sdk-nestjs-common"; -import { ElasticQuery, QueryOperator, QueryType, QueryConditionOptions, ElasticSortOrder, ElasticSortProperty, TermsQuery, RangeGreaterThanOrEqual, MatchQuery } from "@multiversx/sdk-nestjs-elastic"; +import { ElasticQuery, QueryOperator, QueryType, QueryConditionOptions, ElasticSortOrder, ElasticSortProperty, RangeGreaterThanOrEqual, MatchQuery } from "@multiversx/sdk-nestjs-elastic"; import { IndexerInterface } from "../indexer.interface"; import { ApiConfigService } from "src/common/api-config/api.config.service"; import { CollectionFilter } from "src/endpoints/collections/entities/collection.filter"; @@ -279,9 +279,7 @@ export class ElasticIndexerService implements IndexerInterface { const elasticOperations = await this.elasticService.getList('operations', 'txHash', elasticQuery); - for (const operation of elasticOperations) { - this.processTransaction(operation); - } + this.bulkProcessTransactions(elasticOperations); return elasticOperations; } @@ -338,7 +336,7 @@ export class ElasticIndexerService implements IndexerInterface { .withMustMatchCondition('type', 'unsigned') .withPagination({ from: 0, size: transactionHashes.length + 1 }) .withSort([{ name: 'timestamp', order: ElasticSortOrder.ascending }]) - .withTerms(new TermsQuery('originalTxHash', transactionHashes)); + .withMustMultiShouldCondition(transactionHashes, hash => QueryType.Match('originalTxHash', hash)); return await this.elasticService.getList('operations', 'scHash', elasticQuery); } @@ -346,7 +344,7 @@ export class ElasticIndexerService implements IndexerInterface { async getAccountsForAddresses(addresses: string[]): Promise { const elasticQuery: ElasticQuery = ElasticQuery.create() .withPagination({ from: 0, size: addresses.length + 1 }) - .withTerms(new TermsQuery('address', addresses)); + .withMustMultiShouldCondition(addresses, address => QueryType.Match('address', address)); return await this.elasticService.getList('accounts', 'address', elasticQuery); } @@ -383,9 +381,7 @@ export class ElasticIndexerService implements IndexerInterface { const results = await this.elasticService.getList('operations', 'hash', elasticQuery); - for (const result of results) { - this.processTransaction(result); - } + this.bulkProcessTransactions(results); return results; } @@ -548,9 +544,7 @@ export class ElasticIndexerService implements IndexerInterface { const transactions = await this.elasticService.getList('operations', 'txHash', elasticQuery); - for (const transaction of transactions) { - this.processTransaction(transaction); - } + this.bulkProcessTransactions(transactions); return transactions; } @@ -561,6 +555,19 @@ export class ElasticIndexerService implements IndexerInterface { } } + private bulkProcessTransactions(transactions: any[]) { + if (!transactions || transactions.length === 0) { + return; + } + + for (let i = 0; i < transactions.length; i++) { + const transaction = transactions[i]; + if (transaction && !transaction.function) { + transaction.function = transaction.operation; + } + } + } + private buildTokenFilter(query: ElasticQuery, filter: TokenFilter): ElasticQuery { if (filter.includeMetaESDT === true) { query = query.withMustMultiShouldCondition([TokenType.FungibleESDT, TokenType.MetaESDT], type => QueryType.Match('type', type)); @@ -616,9 +623,7 @@ export class ElasticIndexerService implements IndexerInterface { const results = await this.elasticService.getList('operations', 'hash', elasticQuerySc); - for (const result of results) { - this.processTransaction(result); - } + this.bulkProcessTransactions(results); return results; } @@ -632,9 +637,11 @@ export class ElasticIndexerService implements IndexerInterface { return []; } + const maxSize = Math.min(hashes.length * 10, 1000); + const elasticQuery = ElasticQuery.create() .withMustMatchCondition('type', 'unsigned') - .withPagination({ from: 0, size: 10000 }) + .withPagination({ from: 0, size: maxSize }) .withSort([{ name: 'timestamp', order: ElasticSortOrder.ascending }]) .withMustMultiShouldCondition(hashes, hash => QueryType.Match('originalTxHash', hash)); @@ -646,8 +653,6 @@ export class ElasticIndexerService implements IndexerInterface { return []; } - const queries = identifiers.map((identifier) => QueryType.Match('identifier', identifier, QueryOperator.AND)); - let elasticQuery = ElasticQuery.create(); if (pagination) { @@ -655,10 +660,12 @@ export class ElasticIndexerService implements IndexerInterface { } elasticQuery = elasticQuery - .withSort([{ name: "balanceNum", order: ElasticSortOrder.descending }]) + .withSort([ + { name: "balanceNum", order: ElasticSortOrder.descending }, + { name: 'timestamp', order: ElasticSortOrder.descending }, + ]) .withCondition(QueryConditionOptions.mustNot, [QueryType.Match('address', 'pending')]) - .withCondition(QueryConditionOptions.should, queries) - .withSort([{ name: 'timestamp', order: ElasticSortOrder.descending }]); + .withMustMultiShouldCondition(identifiers, identifier => QueryType.Match('identifier', identifier, QueryOperator.AND)); return await this.elasticService.getList('accountsesdt', 'identifier', elasticQuery); } @@ -668,8 +675,6 @@ export class ElasticIndexerService implements IndexerInterface { return []; } - const queries = identifiers.map((identifier) => QueryType.Match('collection', identifier, QueryOperator.AND)); - let elasticQuery = ElasticQuery.create(); if (pagination) { @@ -677,10 +682,12 @@ export class ElasticIndexerService implements IndexerInterface { } elasticQuery = elasticQuery - .withSort([{ name: "balanceNum", order: ElasticSortOrder.descending }]) + .withSort([ + { name: "balanceNum", order: ElasticSortOrder.descending }, + { name: 'timestamp', order: ElasticSortOrder.descending }, + ]) .withCondition(QueryConditionOptions.mustNot, [QueryType.Match('address', 'pending')]) - .withCondition(QueryConditionOptions.should, queries) - .withSort([{ name: 'timestamp', order: ElasticSortOrder.descending }]); + .withMustMultiShouldCondition(identifiers, identifier => QueryType.Match('collection', identifier, QueryOperator.AND)); return await this.elasticService.getList('accountsesdt', 'identifier', elasticQuery); } @@ -711,11 +718,22 @@ export class ElasticIndexerService implements IndexerInterface { ]); } - let elasticNfts = await this.elasticService.getList('tokens', 'identifier', elasticQuery); - if (elasticNfts.length === 0 && identifier !== undefined) { - elasticNfts = await this.elasticService.getList('accountsesdt', 'identifier', ElasticQuery.create().withMustMatchCondition('identifier', identifier, QueryOperator.AND)); + if (identifier !== undefined) { + const [tokensResult, accountsResult] = await Promise.all([ + this.elasticService.getList('tokens', 'identifier', elasticQuery).catch(() => []), + this.elasticService.getList('accountsesdt', 'identifier', + ElasticQuery.create() + .withMustMatchCondition('identifier', identifier, QueryOperator.AND) + .withPagination(pagination) + ).catch(() => []), + ]); + + const elasticNfts = tokensResult.length > 0 ? tokensResult : accountsResult; + return elasticNfts; + } else { + const elasticNfts = await this.elasticService.getList('tokens', 'identifier', elasticQuery); + return elasticNfts; } - return elasticNfts; } async getTransactionBySenderAndNonce(sender: string, nonce: number): Promise { @@ -750,7 +768,7 @@ export class ElasticIndexerService implements IndexerInterface { 'data.uris', ]) .withMustExistCondition('identifier') - .withMustMultiShouldCondition([EsdtType.NonFungibleESDT, EsdtType.SemiFungibleESDT], type => QueryType.Match('type', type)) + .withMustMultiShouldCondition([EsdtType.NonFungibleESDT, EsdtType.SemiFungibleESDT, EsdtType.MetaESDT], type => QueryType.Match('type', type)) .withPagination({ from: 0, size: 10000 }); return await this.elasticService.getScrollableList('tokens', 'identifier', query, action); @@ -819,52 +837,100 @@ export class ElasticIndexerService implements IndexerInterface { } } - const elasticQuery = ElasticQuery.create() - .withMustExistCondition('identifier') - .withMustMatchCondition('address', address) - .withPagination({ from: 0, size: 0 }) - .withMustMatchCondition('token', filter.collection, QueryOperator.AND) - .withMustMultiShouldCondition(filter.identifiers, identifier => QueryType.Match('token', identifier, QueryOperator.AND)) - .withSearchWildcardCondition(filter.search, ['token', 'name']) - .withMustMultiShouldCondition(filterTypes, type => QueryType.Match('type', type)) - .withMustMultiShouldCondition(filter.subType, subType => QueryType.Match('type', subType)) - .withExtra({ - aggs: { - collections: { - composite: { - size: 10000, - sources: [ - { - collection: { - terms: { - field: 'token', - }, - }, - }, - ], + const data: { collection: string, count: number, balance: number }[] = []; + let afterKey: any = null; + let remainingToSkip = pagination.from; + let remainingToCollect = pagination.size; + + while (data.length < pagination.size) { + const batchSize = Math.min(1000, remainingToSkip + remainingToCollect); + + const compositeAgg: any = { + size: batchSize, + sources: [ + { + collection: { + terms: { + field: 'token', + order: 'asc', + }, }, - aggs: { - balance: { - sum: { - field: 'balanceNum', + }, + ], + }; + + if (afterKey) { + compositeAgg.after = afterKey; + } + + const elasticQuery = ElasticQuery.create() + .withMustExistCondition('identifier') + .withMustMatchCondition('address', address) + .withPagination({ from: 0, size: 0 }) + .withMustMatchCondition('token', filter.collection, QueryOperator.AND) + .withMustMultiShouldCondition(filter.identifiers, identifier => QueryType.Match('token', identifier, QueryOperator.AND)) + .withSearchWildcardCondition(filter.search, ['token', 'name']) + .withMustMultiShouldCondition(filterTypes, type => QueryType.Match('type', type)) + .withMustMultiShouldCondition(filter.subType, subType => QueryType.Match('type', subType)) + .withExtra({ + aggs: { + collections: { + composite: compositeAgg, + aggs: { + balance: { + sum: { + field: 'balanceNum', + }, }, }, }, }, - }, - }); + }); - const result = await this.elasticService.post(`${this.apiConfigService.getElasticUrl()}/accountsesdt/_search`, elasticQuery.toJson()); + const result = await this.elasticService.post(`${this.apiConfigService.getElasticUrl()}/accountsesdt/_search`, elasticQuery.toJson()); + const buckets = result?.data?.aggregations?.collections?.buckets || []; + + if (buckets.length === 0) { + break; + } - const buckets = result?.data?.aggregations?.collections?.buckets; + const batchData: { collection: string, count: number, balance: number }[] = buckets.map((bucket: any) => ({ + collection: bucket.key.collection, + count: bucket.doc_count, + balance: bucket.balance.value, + })); - let data: { collection: string, count: number, balance: number }[] = buckets.map((bucket: any) => ({ - collection: bucket.key.collection, - count: bucket.doc_count, - balance: bucket.balance.value, - })); + if (remainingToSkip > 0) { + const skipFromBatch = Math.min(remainingToSkip, batchData.length); + remainingToSkip -= skipFromBatch; + + if (remainingToSkip === 0) { + const collectFromBatch = Math.min(remainingToCollect, batchData.length - skipFromBatch); + data.push(...batchData.slice(skipFromBatch, skipFromBatch + collectFromBatch)); + remainingToCollect -= collectFromBatch; + } + } else { + const collectFromBatch = Math.min(remainingToCollect, batchData.length); + data.push(...batchData.slice(0, collectFromBatch)); + remainingToCollect -= collectFromBatch; + } + + if (remainingToCollect === 0) { + break; + } + + const aggregations = result?.data?.aggregations?.collections; + if (aggregations?.after_key) { + afterKey = aggregations.after_key; + } else { + break; + } + + if (buckets.length < batchSize) { + break; + } + } - data = data.slice(pagination.from, pagination.from + pagination.size); return data; } diff --git a/src/common/rabbitmq/rabbitmq.nft.handler.service.ts b/src/common/rabbitmq/rabbitmq.nft.handler.service.ts index b3766e312..53a90b719 100644 --- a/src/common/rabbitmq/rabbitmq.nft.handler.service.ts +++ b/src/common/rabbitmq/rabbitmq.nft.handler.service.ts @@ -107,9 +107,6 @@ export class RabbitMqNftHandlerService { collectionIdentifier = identifier.split('-').slice(0, 3).join('-'); } const collectionType = await this.getCollectionType(collectionIdentifier); - if (collectionType === NftType.MetaESDT) { - return false; - } this.logger.log(`Detected 'ESDTNFTCreate' event for NFT with identifier '${identifier}' and collection type '${collectionType}'`); diff --git a/src/endpoints/accounts/account.service.ts b/src/endpoints/accounts/account.service.ts index 96f647f8f..cde2c8564 100644 --- a/src/endpoints/accounts/account.service.ts +++ b/src/endpoints/accounts/account.service.ts @@ -182,17 +182,20 @@ export class AccountService { } if (AddressUtils.isSmartContractAddress(address) && account.code) { - const deployTxHash = await this.getAccountDeployedTxHash(address); + const [deployTxHash, deployedAt, isVerified] = await Promise.all([ + this.getAccountDeployedTxHash(address), + this.getAccountDeployedAt(address), + this.getAccountIsVerified(address, account.codeHash), + ]); + if (deployTxHash) { account.deployTxHash = deployTxHash; } - const deployedAt = await this.getAccountDeployedAt(address); if (deployedAt) { account.deployedAt = deployedAt; } - const isVerified = await this.getAccountIsVerified(address, account.codeHash); if (isVerified) { account.isVerified = isVerified; } diff --git a/src/endpoints/blocks/block.service.ts b/src/endpoints/blocks/block.service.ts index 4b35ad28a..238804186 100644 --- a/src/endpoints/blocks/block.service.ts +++ b/src/endpoints/blocks/block.service.ts @@ -10,6 +10,7 @@ import { IndexerService } from "src/common/indexer/indexer.service"; import { NodeService } from "../nodes/node.service"; import { IdentitiesService } from "../identities/identities.service"; import { ApiConfigService } from "../../common/api-config/api.config.service"; +import { ConcurrencyUtils } from "src/utils/concurrency.utils"; @Injectable() export class BlockService { @@ -34,8 +35,8 @@ export class BlockService { async getBlocks(filter: BlockFilter, queryPagination: QueryPagination, withProposerIdentity?: boolean): Promise { const result = await this.indexerService.getBlocks(filter, queryPagination); - const blocks = []; - for (const item of result) { + + const blocks = await Promise.all(result.map(async (item) => { const blockRaw = await this.computeProposerAndValidators(item); const block = Block.mergeWithElasticResponse(new Block(), blockRaw); @@ -44,8 +45,8 @@ export class BlockService { block.scheduledRootHash = blockRaw.scheduledData.rootHash; } - blocks.push(block); - } + return block; + })); if (withProposerIdentity === true) { await this.applyProposerIdentity(blocks); @@ -58,17 +59,18 @@ export class BlockService { const proposerBlses = blocks.map(x => x.proposer); const nodes = await this.nodeService.getAllNodes(); - for (const node of nodes) { - if (!proposerBlses.includes(node.bls)) { - continue; - } - - const nodeIdentity = node.identity; - if (!nodeIdentity) { - continue; - } - - const identity = await this.identitiesService.getIdentity(nodeIdentity); + const relevantNodes = nodes.filter(node => proposerBlses.includes(node.bls) && node.identity); + + const nodeIdentities = await ConcurrencyUtils.executeWithConcurrencyLimit( + relevantNodes, + async (node) => { + const identity = await this.identitiesService.getIdentity(node.identity!); + return { node, identity }; + }, + 25, + 'Block proposer identities' + ); + for (const { node, identity } of nodeIdentities) { if (!identity) { continue; } diff --git a/src/endpoints/collections/collection.controller.ts b/src/endpoints/collections/collection.controller.ts index 2137a49be..7ce7f1111 100644 --- a/src/endpoints/collections/collection.controller.ts +++ b/src/endpoints/collections/collection.controller.ts @@ -225,6 +225,7 @@ export class CollectionController { @ApiQuery({ name: 'nonceAfter', description: 'Return all NFTs with given nonce after the given number', required: false, type: Number }) @ApiQuery({ name: 'withOwner', description: 'Return owner where type = NonFungibleESDT', required: false, type: Boolean }) @ApiQuery({ name: 'withSupply', description: 'Return supply where type = SemiFungibleESDT', required: false, type: Boolean }) + @ApiQuery({ name: 'withAssets', description: 'Return assets information (defaults to true)', required: false, type: Boolean }) @ApiQuery({ name: 'sort', description: 'Sorting criteria', required: false, enum: SortCollectionNfts }) @ApiQuery({ name: 'order', description: 'Sorting order (asc / desc)', required: false, enum: SortOrder }) async getNfts( @@ -244,6 +245,7 @@ export class CollectionController { @Query('nonceAfter', ParseIntPipe) nonceAfter?: number, @Query('withOwner', ParseBoolPipe) withOwner?: boolean, @Query('withSupply', ParseBoolPipe) withSupply?: boolean, + @Query('withAssets', ParseBoolPipe) withAssets?: boolean, @Query('sort', new ParseEnumPipe(SortCollectionNfts)) sort?: SortCollectionNfts, @Query('order', new ParseEnumPipe(SortOrder)) order?: SortOrder, ): Promise { @@ -255,7 +257,7 @@ export class CollectionController { return await this.nftService.getNfts( new QueryPagination({ from, size }), new NftFilter({ search, identifiers, collection, name, tags, creator, hasUris, isWhitelistedStorage, isNsfw, traits, nonceBefore, nonceAfter, sort, order }), - new NftQueryOptions({ withOwner, withSupply }), + new NftQueryOptions({ withOwner, withSupply, withAssets }), ); } diff --git a/src/endpoints/collections/collection.service.ts b/src/endpoints/collections/collection.service.ts index cd318cfd0..db49a515e 100644 --- a/src/endpoints/collections/collection.service.ts +++ b/src/endpoints/collections/collection.service.ts @@ -62,20 +62,18 @@ export class CollectionService { private async processNftCollections(tokenCollections: Collection[]): Promise { const collectionsIdentifiers = tokenCollections.map((collection) => collection.token); - const indexedCollections: Record = {}; + const indexedCollections = new Map(); for (const collection of tokenCollections) { - indexedCollections[collection.token] = collection; + indexedCollections.set(collection.token, collection); } const nftCollections: NftCollection[] = await this.applyPropertiesToCollections(collectionsIdentifiers); for (const nftCollection of nftCollections) { - const indexedCollection = indexedCollections[nftCollection.collection]; - if (!indexedCollection) { - continue; + const indexedCollection = indexedCollections.get(nftCollection.collection); + if (indexedCollection) { + this.applyPropertiesToCollectionFromElasticSearch(nftCollection, indexedCollection); } - - this.applyPropertiesToCollectionFromElasticSearch(nftCollection, indexedCollection); } return nftCollections; @@ -117,8 +115,11 @@ export class CollectionService { async applyPropertiesToCollections(collectionsIdentifiers: string[]): Promise { const nftCollections: NftCollection[] = []; - const collectionsProperties = await this.batchGetCollectionsProperties(collectionsIdentifiers); - const collectionsAssets = await this.batchGetCollectionsAssets(collectionsIdentifiers); + + const [collectionsProperties, collectionsAssets] = await Promise.all([ + this.batchGetCollectionsProperties(collectionsIdentifiers), + this.batchGetCollectionsAssets(collectionsIdentifiers), + ]); for (const collectionIdentifier of collectionsIdentifiers) { const collectionProperties = collectionsProperties[collectionIdentifier]; @@ -126,11 +127,29 @@ export class CollectionService { continue; } - const nftCollection = new NftCollection(); + const identifierParts = collectionIdentifier.split('-'); + const ticker = identifierParts[0]; + const collectionBase = identifierParts.slice(0, 2).join('-'); + const assets = collectionsAssets[collectionIdentifier]; + + const nftCollection = new NftCollection({ + // @ts-ignore + type: collectionProperties.type, + name: collectionProperties.name, + collection: collectionBase, + ticker: ticker, + canFreeze: collectionProperties.canFreeze, + canWipe: collectionProperties.canWipe, + canPause: collectionProperties.canPause, + canTransferNftCreateRole: collectionProperties.canTransferNFTCreateRole, + canChangeOwner: collectionProperties.canChangeOwner, + canUpgrade: collectionProperties.canUpgrade, + canAddSpecialRoles: collectionProperties.canAddSpecialRoles, + owner: collectionProperties.owner, + assets: assets, + decimals: (collectionProperties.type as any) === NftType.MetaESDT ? collectionProperties.decimals : undefined, + }); - // @ts-ignore - nftCollection.type = collectionProperties.type; - nftCollection.name = collectionProperties.name; if (TokenUtils.isSovereignIdentifier(collectionIdentifier)) { nftCollection.collection = collectionIdentifier.split('-').slice(0, 3).join('-'); nftCollection.ticker = collectionIdentifier.split('-')[1]; @@ -139,21 +158,6 @@ export class CollectionService { nftCollection.ticker = collectionIdentifier.split('-')[0]; } - nftCollection.canFreeze = collectionProperties.canFreeze; - nftCollection.canWipe = collectionProperties.canWipe; - nftCollection.canPause = collectionProperties.canPause; - nftCollection.canTransferNftCreateRole = collectionProperties.canTransferNFTCreateRole; - nftCollection.canChangeOwner = collectionProperties.canChangeOwner; - nftCollection.canUpgrade = collectionProperties.canUpgrade; - nftCollection.canAddSpecialRoles = collectionProperties.canAddSpecialRoles; - nftCollection.owner = collectionProperties.owner; - - if (nftCollection.type === NftType.MetaESDT) { - nftCollection.decimals = collectionProperties.decimals; - } - - nftCollection.assets = collectionsAssets[collectionIdentifier]; - nftCollections.push(nftCollection); } @@ -171,10 +175,12 @@ export class CollectionService { async batchGetCollectionsAssets(identifiers: string[]): Promise<{ [key: string]: TokenAssets | undefined }> { const collectionsAssets: { [key: string]: TokenAssets | undefined } = {}; + const allAssets = await this.assetsService.getAllTokenAssets(); + await this.cachingService.batchApplyAll( identifiers, identifier => CacheInfo.EsdtAssets(identifier).key, - identifier => this.assetsService.getTokenAssets(identifier), + identifier => Promise.resolve(allAssets[identifier]), (identifier, properties) => collectionsAssets[identifier] = properties, CacheInfo.EsdtAssets('').ttl ); diff --git a/src/endpoints/nfts/entities/nft.query.options.ts b/src/endpoints/nfts/entities/nft.query.options.ts index 66b01f7aa..2e4b0a6dc 100644 --- a/src/endpoints/nfts/entities/nft.query.options.ts +++ b/src/endpoints/nfts/entities/nft.query.options.ts @@ -3,11 +3,16 @@ import { BadRequestException } from "@nestjs/common"; export class NftQueryOptions { constructor(init?: Partial) { Object.assign(this, init); + + if (this.withAssets === undefined) { + this.withAssets = true; + } } withOwner?: boolean; withSupply?: boolean; withReceivedAt?: boolean; + withAssets?: boolean; validate(size: number): void { if (this.withReceivedAt && size > 25) { diff --git a/src/endpoints/nfts/nft.service.ts b/src/endpoints/nfts/nft.service.ts index d7deef398..b7a658645 100644 --- a/src/endpoints/nfts/nft.service.ts +++ b/src/endpoints/nfts/nft.service.ts @@ -70,7 +70,7 @@ export class NftService { const nfts = await this.getNftsInternal({ from, size }, filter); await Promise.all([ - this.batchApplyAssetsAndTicker(nfts), + this.conditionallyApplyAssetsAndTicker(nfts, undefined, queryOptions), this.conditionallyApplyOwners(nfts, queryOptions), this.conditionallyApplySupply(nfts, queryOptions), this.batchProcessNfts(nfts), @@ -88,23 +88,25 @@ export class NftService { ]); } - private async batchApplyAssetsAndTicker(nfts: Nft[], fields?: string[]): Promise { + private async conditionallyApplyAssetsAndTicker(nfts: Nft[], fields?: string[], queryOptions?: { withAssets?: boolean }): Promise { if (fields && fields.includesNone(['ticker', 'assets'])) { return; } - await Promise.all( - nfts.map(async (nft) => { - nft.assets = await this.assetsService.getTokenAssets(nft.identifier) ?? - await this.assetsService.getTokenAssets(nft.collection); - - if (nft.assets) { - nft.ticker = nft.collection.split('-')[0]; - } else { - nft.ticker = nft.collection; - } - }) - ); + const allAssets = await this.assetsService.getAllTokenAssets(); + if (queryOptions?.withAssets === false) { + return; + } + + for (const nft of nfts) { + nft.assets = allAssets[nft.identifier] ?? allAssets[nft.collection]; + + if (nft.assets) { + nft.ticker = nft.collection.split('-')[0]; + } else { + nft.ticker = nft.collection; + } + } } private async conditionallyApplyOwners(nfts: Nft[], queryOptions?: NftQueryOptions): Promise { @@ -451,8 +453,6 @@ export class NftService { nft.decimals = collectionProperties.decimals; // @ts-ignore delete nft.royalties; - // @ts-ignore - delete nft.uris; } } } diff --git a/src/endpoints/tokens/token.service.ts b/src/endpoints/tokens/token.service.ts index c0d3873b8..46fd13431 100644 --- a/src/endpoints/tokens/token.service.ts +++ b/src/endpoints/tokens/token.service.ts @@ -93,16 +93,19 @@ export class TokenService { this.applyTickerFromAssets(token); - await this.applySupply(token, supplyOptions); - - if (token.type === TokenType.FungibleESDT) { - token.roles = await this.getTokenRoles(identifier); - } else if (token.type === TokenType.MetaESDT) { - const elasticCollection = await this.indexerService.getCollection(identifier); - if (elasticCollection) { - await this.collectionService.applyCollectionRoles(token, elasticCollection); - } - } + await Promise.all([ + this.applySupply(token, supplyOptions), + (async () => { + if (token.type === TokenType.FungibleESDT) { + token.roles = await this.getTokenRoles(identifier); + } else if (token.type === TokenType.MetaESDT) { + const elasticCollection = await this.indexerService.getCollection(identifier); + if (elasticCollection) { + await this.collectionService.applyCollectionRoles(token, elasticCollection); + } + } + })(), + ]); return token; } @@ -775,8 +778,10 @@ export class TokenService { this.logger.log(`Fetched ${tokens.length} fungible tokens`); + const allAssets = await this.assetsService.getAllTokenAssets(); + for (const token of tokens) { - const assets = await this.assetsService.getTokenAssets(token.identifier); + const assets = allAssets[token.identifier]; if (assets && assets.name) { token.name = assets.name; @@ -812,10 +817,13 @@ export class TokenService { await this.batchProcessTokens(tokens); - await this.applyMexLiquidity(tokens.filter(x => x.type !== TokenType.MetaESDT)); - await this.applyMexPrices(tokens.filter(x => x.type !== TokenType.MetaESDT)); - await this.applyMexPairType(tokens.filter(x => x.type !== TokenType.MetaESDT)); - await this.applyMexPairTradesCount(tokens.filter(x => x.type !== TokenType.MetaESDT)); + const nonMetaEsdtTokens = tokens.filter(x => x.type !== TokenType.MetaESDT); + await Promise.all([ + this.applyMexLiquidity(nonMetaEsdtTokens), + this.applyMexPrices(nonMetaEsdtTokens), + this.applyMexPairType(nonMetaEsdtTokens), + this.applyMexPairTradesCount(nonMetaEsdtTokens), + ]); await this.cachingService.batchApplyAll( tokens, diff --git a/src/endpoints/transactions/transaction.get.service.ts b/src/endpoints/transactions/transaction.get.service.ts index ec3e97a1d..5a3b0dff8 100644 --- a/src/endpoints/transactions/transaction.get.service.ts +++ b/src/endpoints/transactions/transaction.get.service.ts @@ -21,6 +21,7 @@ import { QueryPagination } from "src/common/entities/query.pagination"; import { NftFilter } from "../nfts/entities/nft.filter"; import { TokenAccount } from "src/common/indexer/entities"; import { ApiConfigService } from "../../common/api-config/api.config.service"; +import crypto from 'crypto-js'; @Injectable() export class TransactionGetService { @@ -155,6 +156,7 @@ export class TransactionGetService { const logs = await this.getTransactionLogsFromElastic(hashes); for (const log of logs) { this.alterDuplicatedTransferValueOnlyEvents(log.events); + this.removeDuplicatedESDTTransferEvents(log.events); } if (!fields || fields.length === 0 || fields.includes(TransactionOptionalFieldOption.operations)) { @@ -200,6 +202,53 @@ export class TransactionGetService { } } + private removeDuplicatedESDTTransferEvents(events: TransactionLogEvent[]) { + const esdtTransferEvents = events.filter(x => x.identifier === 'ESDTTransfer'); + + if (esdtTransferEvents.length <= 1) { + return; + } + + const eventGroups = new Map(); + + for (const event of esdtTransferEvents) { + const contentHash = this.getEventContentHash(event); + if (!eventGroups.has(contentHash)) { + eventGroups.set(contentHash, []); + } + const group = eventGroups.get(contentHash); + if (group) { + group.push(event); + } + } + + const duplicateEvents = new Set(); + for (const [, eventGroup] of eventGroups) { + if (eventGroup.length > 1) { + for (let i = 1; i < eventGroup.length; i++) { + duplicateEvents.add(eventGroup[i]); + } + } + } + + for (let i = events.length - 1; i >= 0; i--) { + if (duplicateEvents.has(events[i])) { + events.splice(i, 1); + } + } + } + + private getEventContentHash(event: TransactionLogEvent): string { + const content = { + address: event.address, + identifier: event.identifier, + topics: event.topics, + data: event.data, + additionalData: event.additionalData, + }; + return crypto.MD5(JSON.stringify(content)).toString(); + } + private applyUsernamesToDetailedTransaction(transaction: IndexerTransaction, transactionDetailed: TransactionDetailed) { if (transaction.senderUserName) { transactionDetailed.senderUsername = UsernameUtils.extractUsernameFromRawBase64(transaction.senderUserName); diff --git a/src/endpoints/transactions/transaction.service.ts b/src/endpoints/transactions/transaction.service.ts index f44c873ae..6167b1fbb 100644 --- a/src/endpoints/transactions/transaction.service.ts +++ b/src/endpoints/transactions/transaction.service.ts @@ -466,18 +466,25 @@ export class TransactionService { private async getSmartContractResultsRaw(transactionHashes: Array): Promise> { const resultsRaw = await this.indexerService.getSmartContractResults(transactionHashes) as any[]; + + const resultsByHash = new Map(); + for (const result of resultsRaw) { result.hash = result.scHash; - delete result.scHash; + + const txHash = result.originalTxHash; + if (!resultsByHash.has(txHash)) { + resultsByHash.set(txHash, []); + } + resultsByHash.get(txHash)!.push(result); } const results: Array = []; - for (const transactionHash of transactionHashes) { - const resultRaw = resultsRaw.filter(({ originalTxHash }) => originalTxHash == transactionHash); + const resultRaw = resultsByHash.get(transactionHash); - if (resultRaw.length > 0) { + if (resultRaw && resultRaw.length > 0) { results.push(resultRaw.map((result: any) => ApiUtils.mergeObjects(new SmartContractResult(), result))); } else { results.push(undefined); diff --git a/src/endpoints/transfers/transfer.service.ts b/src/endpoints/transfers/transfer.service.ts index 01df59bf1..fdf9baa20 100644 --- a/src/endpoints/transfers/transfer.service.ts +++ b/src/endpoints/transfers/transfer.service.ts @@ -21,9 +21,16 @@ export class TransferService { ) { } private sortElasticTransfers(elasticTransfers: any[]): any[] { + const transactionMap = new Map(); + for (const transfer of elasticTransfers) { + if (transfer.txHash) { + transactionMap.set(transfer.txHash, transfer); + } + } + for (const elasticTransfer of elasticTransfers) { if (elasticTransfer.originalTxHash) { - const transaction = elasticTransfers.find(x => x.txHash === elasticTransfer.originalTxHash); + const transaction = transactionMap.get(elasticTransfer.originalTxHash); if (transaction) { elasticTransfer.order = (transaction.nonce * 10) + 1; } else { diff --git a/src/queue.worker/nft.worker/nft.worker.service.ts b/src/queue.worker/nft.worker/nft.worker.service.ts index dd91a622a..da5f2b23a 100644 --- a/src/queue.worker/nft.worker/nft.worker.service.ts +++ b/src/queue.worker/nft.worker/nft.worker.service.ts @@ -6,7 +6,6 @@ import { NftMetadataService } from "./queue/job-services/metadata/nft.metadata.s import { NftMediaService } from "./queue/job-services/media/nft.media.service"; import { ClientProxy } from "@nestjs/microservices"; import { NftMessage } from "./queue/entities/nft.message"; -import { NftType } from "src/endpoints/nfts/entities/nft.type"; import { NftAssetService } from "./queue/job-services/assets/nft.asset.service"; import { PersistenceService } from "src/common/persistence/persistence.service"; import { ApiConfigService } from "src/common/api-config/api.config.service"; @@ -44,10 +43,6 @@ export class NftWorkerService { } async needsProcessing(nft: Nft, settings: ProcessNftSettings): Promise { - if (nft.type === NftType.MetaESDT) { - return false; - } - if (settings.forceRefreshMedia || settings.forceRefreshMetadata || settings.forceRefreshThumbnail) { return true; } diff --git a/src/queue.worker/nft.worker/queue/job-services/media/nft.media.service.ts b/src/queue.worker/nft.worker/queue/job-services/media/nft.media.service.ts index fb7c9b433..22ade05b6 100644 --- a/src/queue.worker/nft.worker/queue/job-services/media/nft.media.service.ts +++ b/src/queue.worker/nft.worker/queue/job-services/media/nft.media.service.ts @@ -8,7 +8,6 @@ import { PersistenceService } from "src/common/persistence/persistence.service"; import { MediaMimeTypeEnum } from "src/endpoints/nfts/entities/media.mime.type"; import { Nft } from "src/endpoints/nfts/entities/nft"; import { NftMedia } from "src/endpoints/nfts/entities/nft.media"; -import { NftType } from "src/endpoints/nfts/entities/nft.type"; import { TokenHelpers } from "src/utils/token.helpers"; import { ClientProxy } from "@nestjs/microservices"; import { OriginLogger } from "@multiversx/sdk-nestjs-common"; @@ -63,10 +62,6 @@ export class NftMediaService { } private async getMediaRaw(nft: Nft): Promise { - if (nft.type === NftType.MetaESDT) { - return null; - } - if (!nft.uris) { return null; } diff --git a/src/queue.worker/nft.worker/queue/job-services/metadata/nft.metadata.service.ts b/src/queue.worker/nft.worker/queue/job-services/metadata/nft.metadata.service.ts index 8ab83454b..804a716d5 100644 --- a/src/queue.worker/nft.worker/queue/job-services/metadata/nft.metadata.service.ts +++ b/src/queue.worker/nft.worker/queue/job-services/metadata/nft.metadata.service.ts @@ -65,7 +65,7 @@ export class NftMetadataService { } async getMetadataRaw(nft: Nft): Promise { - if (!nft.attributes || nft.type === NftType.MetaESDT) { + if (!nft.attributes) { return null; } diff --git a/src/test/unit/services/tokens.spec.ts b/src/test/unit/services/tokens.spec.ts index 064939059..94d1a6ba6 100644 --- a/src/test/unit/services/tokens.spec.ts +++ b/src/test/unit/services/tokens.spec.ts @@ -110,6 +110,7 @@ describe('Token Service', () => { useValue: { getTokenAssets: jest.fn(), getAllAccountAssets: jest.fn(), + getAllTokenAssets: jest.fn(), }, }, { @@ -603,7 +604,7 @@ describe('Token Service', () => { expect(getAllTokensMock).toHaveBeenCalledTimes(1); const secondToken = mockTokens[1]; - secondToken.assets.priceSource = {type: 'customUrl'}; + secondToken.assets.priceSource = { type: 'customUrl' }; const newExpectedMarketCap = result - secondToken.marketCap; mockTokens[1] = secondToken; @@ -692,7 +693,6 @@ describe('Token Service', () => { }); it('should return tokens from other sources when isTokensFetchFeatureEnabled is false', async () => { - const mockTokenProperties: Partial[] = [{ identifier: 'mockIdentifier' }]; let mockTokens: Partial[] = mockTokenProperties.map(properties => ApiUtils.mergeObjects(new TokenDetailed(), properties)); const mockTokenAssets: Partial = { name: 'mockName' }; @@ -702,6 +702,7 @@ describe('Token Service', () => { jest.spyOn(apiConfigService, 'isTokensFetchFeatureEnabled').mockReturnValue(false); jest.spyOn(esdtService, 'getAllFungibleTokenProperties').mockResolvedValue(mockTokenProperties as TokenProperties[]); jest.spyOn(assetsService, 'getTokenAssets').mockResolvedValue(mockTokenAssets as TokenAssets); + jest.spyOn(assetsService, 'getAllTokenAssets').mockResolvedValue({ mockIdentifier: mockTokenAssets, 'EGLD-000000': mockTokenAssets } as any); jest.spyOn(collectionService, 'getNftCollections').mockResolvedValue(mockNftCollections as NftCollection[]); jest.spyOn(tokenService as any, 'batchProcessTokens').mockImplementation(() => Promise.resolve()); @@ -723,16 +724,14 @@ describe('Token Service', () => { expect(apiConfigService.isTokensFetchFeatureEnabled).toHaveBeenCalled(); expect(esdtService.getAllFungibleTokenProperties).toHaveBeenCalled(); - mockTokens.forEach((mockToken) => { - expect(assetsService.getTokenAssets).toHaveBeenCalledWith(mockToken.identifier); - }); + expect(assetsService.getAllTokenAssets).toHaveBeenCalledTimes(1); - expect(esdtService.getAllFungibleTokenProperties).toHaveBeenCalled(); mockTokens.forEach(mockToken => { - expect(assetsService.getTokenAssets).toHaveBeenCalledWith(mockToken.identifier); mockToken.name = mockTokenAssets.name; }); - expect(assetsService.getTokenAssets).toHaveBeenCalledTimes(mockTokens.length + 1); // add 1 for EGLD-000000 + + expect(esdtService.getAllFungibleTokenProperties).toHaveBeenCalled(); + expect(assetsService.getTokenAssets).toHaveBeenCalledWith('EGLD-000000'); expect((collectionService as any).getNftCollections).toHaveBeenCalledWith(expect.anything(), { type: [TokenType.MetaESDT] }); @@ -817,23 +816,26 @@ describe('Token Service', () => { new TokenProperties({ identifier: 'token5' }), ]); - // Only token2 has a custom price source + const mockAllAssets: { [key: string]: TokenAssets } = { + token1: new TokenAssets({ name: 'Token token1' }), + token2: new TokenAssets({ + name: 'Token token2', + priceSource: { + type: TokenAssetsPriceSourceType.customUrl, + path: '0.usdPrice', + url: 'url', + }, + }), + token3: new TokenAssets({ name: 'Token token3' }), + token4: new TokenAssets({ name: 'Token token4' }), + token5: new TokenAssets({ name: 'Token token5' }), + 'EGLD-000000': new TokenAssets({ name: 'EGLD' }), + }; + jest.spyOn(tokenService['assetsService'], 'getAllTokenAssets').mockResolvedValue(mockAllAssets); + // eslint-disable-next-line require-await jest.spyOn(tokenService['assetsService'], 'getTokenAssets').mockImplementation(async (identifier: string) => { - if (identifier === 'token2') { - return new TokenAssets({ - name: `Token ${identifier}`, - priceSource: { - type: TokenAssetsPriceSourceType.customUrl, - path: '0.usdPrice', - url: 'url', - }, - }); - } - return new TokenAssets({ - name: `Token ${identifier}`, - // No priceSource - }); + return mockAllAssets[identifier]; }); jest.spyOn(tokenService['collectionService'], 'getNftCollections').mockResolvedValue([]); @@ -854,7 +856,7 @@ describe('Token Service', () => { jest.spyOn(tokenService as any, 'applyMexPrices').mockResolvedValue(undefined); jest.spyOn(tokenService as any, 'applyMexPairType').mockResolvedValue(undefined); jest.spyOn(tokenService as any, 'applyMexPairTradesCount').mockResolvedValue(undefined); - jest.spyOn(tokenService['apiService'] as any, 'get').mockResolvedValue({data: [{usdPrice: 1.0}]}); + jest.spyOn(tokenService['apiService'] as any, 'get').mockResolvedValue({ data: [{ usdPrice: 1.0 }] }); jest.spyOn(tokenService['cachingService'], 'batchApplyAll').mockImplementation( // eslint-disable-next-line require-await async (...args: unknown[]) => { diff --git a/src/test/unit/services/transaction.get.spec.ts b/src/test/unit/services/transaction.get.spec.ts index d6990b54f..3a8fa6595 100644 --- a/src/test/unit/services/transaction.get.spec.ts +++ b/src/test/unit/services/transaction.get.spec.ts @@ -541,6 +541,110 @@ describe('TransactionGetService', () => { }); }); + describe('removeDuplicatedESDTTransferEvents', () => { + it('should remove duplicate ESDTTransfer events with identical content', () => { + const events = [ + new TransactionLogEvent({ + identifier: 'ESDTTransfer', + address: 'erd1qyu5wthldzr8wx5c9ucg8kjagg0jfs53s8nr3zpz3hypefsdd8ssycr6th', + topics: ['T05FLTgzYTdjMA==', '', 'CteOvFrGIAAA', 'O7KEpcfOqbXzBNR3sVA22d7LTk6wxoyr3gsDuSXrKdg='], + additionalData: ['', 'RVNEVFRyYW5zZmVy', 'T05FLTgzYTdjMA==', 'CteOvFrGIAAA'], + }), + new TransactionLogEvent({ + identifier: 'writeLog', + address: 'erd18wegffw8e65mtucy63mmz5pkm80vknjwkrrge277pvpmjf0t98vq0wgr49', + topics: ['ATlHLv9ohncamC8wg9pdQh8kwpGB5jiIIo3IHKYNaeE='], + data: 'QDZmNmI=', + additionalData: ['QDZmNmI='], + }), + new TransactionLogEvent({ + identifier: 'ESDTTransfer', + address: 'erd1qyu5wthldzr8wx5c9ucg8kjagg0jfs53s8nr3zpz3hypefsdd8ssycr6th', + topics: ['T05FLTgzYTdjMA==', '', 'CteOvFrGIAAA', 'O7KEpcfOqbXzBNR3sVA22d7LTk6wxoyr3gsDuSXrKdg='], + additionalData: ['', 'RVNEVFRyYW5zZmVy', 'T05FLTgzYTdjMA==', 'CteOvFrGIAAA'], + }), + new TransactionLogEvent({ + identifier: 'completedTxEvent', + address: 'erd18wegffw8e65mtucy63mmz5pkm80vknjwkrrge277pvpmjf0t98vq0wgr49', + topics: ['RaB5PmNc/Gb1ucHpxQ8Q2XviQgobRKw0nvrlRqSpED4='], + }), + ]; + + service['removeDuplicatedESDTTransferEvents'](events); + + expect(events.length).toBe(3); + + const esdtTransferEvents = events.filter(e => e.identifier === 'ESDTTransfer'); + expect(esdtTransferEvents.length).toBe(1); + + expect(events.filter(e => e.identifier === 'writeLog').length).toBe(1); + expect(events.filter(e => e.identifier === 'completedTxEvent').length).toBe(1); + }); + + it('should not remove ESDTTransfer events with different content', () => { + const events = [ + new TransactionLogEvent({ + identifier: 'ESDTTransfer', + address: 'erd1address1', + topics: ['topic1', 'topic2'], + data: 'data1', + additionalData: ['additional1'], + }), + new TransactionLogEvent({ + identifier: 'ESDTTransfer', + address: 'erd1address2', + topics: ['topic3', 'topic4'], + data: 'data2', + additionalData: ['additional2'], + }), + ]; + + service['removeDuplicatedESDTTransferEvents'](events); + + // Should keep both events since they have different content + expect(events.length).toBe(2); + expect(events.filter(e => e.identifier === 'ESDTTransfer').length).toBe(2); + }); + + it('should do nothing when there are no ESDTTransfer events', () => { + const events = [ + new TransactionLogEvent({ + identifier: 'writeLog', + address: 'erd1address1', + topics: ['topic1'], + data: 'data1', + }), + new TransactionLogEvent({ + identifier: 'completedTxEvent', + address: 'erd1address2', + topics: ['topic2'], + data: 'data2', + }), + ]; + + const originalLength = events.length; + service['removeDuplicatedESDTTransferEvents'](events); + + expect(events.length).toBe(originalLength); + }); + + it('should do nothing when there is only one ESDTTransfer event', () => { + const events = [ + new TransactionLogEvent({ + identifier: 'ESDTTransfer', + address: 'erd1address1', + topics: ['topic1'], + data: 'data1', + }), + ]; + + service['removeDuplicatedESDTTransferEvents'](events); + + expect(events.length).toBe(1); + expect(events[0].identifier).toBe('ESDTTransfer'); + }); + }); + describe('tryGetTransactionFromGatewayForList', () => { it('should return transaction when gateway returns data', async () => { const mockGatewayTransaction = { diff --git a/src/utils/concurrency.utils.ts b/src/utils/concurrency.utils.ts new file mode 100644 index 000000000..971d5eee7 --- /dev/null +++ b/src/utils/concurrency.utils.ts @@ -0,0 +1,78 @@ +import { OriginLogger } from "@multiversx/sdk-nestjs-common"; + +export class ConcurrencyUtils { + private static readonly logger = new OriginLogger(ConcurrencyUtils.name); + + static async executeWithConcurrencyLimit( + items: T[], + asyncOperation: (item: T) => Promise, + concurrencyLimit: number = 10, + description?: string + ): Promise { + if (items.length === 0) { + return []; + } + + const logPrefix = description ? `[${description}] ` : ''; + this.logger.log(`${logPrefix}Processing ${items.length} items with concurrency limit ${concurrencyLimit}`); + + const results: R[] = []; + const batchCount = Math.ceil(items.length / concurrencyLimit); + + for (let i = 0; i < items.length; i += concurrencyLimit) { + const batchIndex = Math.floor(i / concurrencyLimit) + 1; + const batch = items.slice(i, i + concurrencyLimit); + + this.logger.log(`${logPrefix}Processing batch ${batchIndex}/${batchCount} (${batch.length} items)`); + + const batchPromises = batch.map(item => asyncOperation(item)); + const batchResults = await Promise.all(batchPromises); + + results.push(...batchResults); + } + + this.logger.log(`${logPrefix}Completed processing ${items.length} items`); + return results; + } + + static async executeWithChunksAndDelay( + items: T[], + asyncOperation: (item: T) => Promise, + chunkSize: number = 10, + delayMs: number = 100, + description?: string + ): Promise { + if (items.length === 0) { + return []; + } + + const logPrefix = description ? `[${description}] ` : ''; + this.logger.log(`${logPrefix}Processing ${items.length} items in chunks of ${chunkSize} with ${delayMs}ms delay`); + + const results: R[] = []; + const chunkCount = Math.ceil(items.length / chunkSize); + + for (let i = 0; i < items.length; i += chunkSize) { + const chunkIndex = Math.floor(i / chunkSize) + 1; + const chunk = items.slice(i, i + chunkSize); + + this.logger.log(`${logPrefix}Processing chunk ${chunkIndex}/${chunkCount} (${chunk.length} items)`); + + const chunkPromises = chunk.map(item => asyncOperation(item)); + const chunkResults = await Promise.all(chunkPromises); + + results.push(...chunkResults); + + if (i + chunkSize < items.length && delayMs > 0) { + await this.delay(delayMs); + } + } + + this.logger.log(`${logPrefix}Completed processing ${items.length} items`); + return results; + } + + private static delay(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); + } +}