From e2c8db12ca0d7d739b637189832cf382a9cd6a6f Mon Sep 17 00:00:00 2001 From: Guillermo Rodriguez Date: Fri, 15 Mar 2024 13:37:19 -0300 Subject: [PATCH 1/5] Optimize getBlockByNumber queries --- src/routes/evm/index.ts | 158 ++++++++++++++++++++++++++++++++++------ 1 file changed, 134 insertions(+), 24 deletions(-) diff --git a/src/routes/evm/index.ts b/src/routes/evm/index.ts index a744540..1806b9a 100644 --- a/src/routes/evm/index.ts +++ b/src/routes/evm/index.ts @@ -200,7 +200,7 @@ export default async function (fastify: FastifyInstance, opts: TelosEvmConfig) { } } - async function getVRS(receiptDoc): Promise { + function getVRS(receiptDoc): {v: string, r: string, s: string} { let receipt = receiptDoc["@raw"]; const v = removeLeftZeros(BigInt(receipt.v).toString(16), true); const r = removeLeftZeros(receipt.r, true); @@ -325,7 +325,64 @@ export default async function (fastify: FastifyInstance, opts: TelosEvmConfig) { return blockDelta; } - async function emptyBlockFromDelta(blockDelta: any) { + + async function getMultipleReceipts(blockNumbers: number[]) { + const blocksPerSuff = new Map(); + for (const blockNum of blockNumbers) { + const indexSuff = indexSuffixForBlock(blockNum); + const blocks = [blockNum]; + if (blocksPerSuff.has(indexSuff)) + blocksPerSuff.get(indexSuff).push(...blocks); + else + blocksPerSuff.set(indexSuff, blocks); + } + const receipts = []; + for (const [suffix, blocks] of blocksPerSuff.entries()) { + const results = await fastify.elastic.search({ + index: `${opts.elasticIndexPrefix}-action-${opts.elasticIndexVersion}-${suffix}`, + size: 2000, + query: { + terms: {'@raw.block': blocks} + } + }); + receipts.push( + ...results.hits.hits.map(h => h._source)); + } + return receipts; + } + + async function getMultipleDeltaDocsFromNumbers(blockNumbers: number[]) { + const blocksPerSuff = new Map(); + for (const blockNum of blockNumbers) { + const indexSuff = indexSuffixForBlock(blockNum); + const blocks = [blockNum]; + if (blocksPerSuff.has(indexSuff)) + blocksPerSuff.get(indexSuff).push(...blocks); + else + blocksPerSuff.set(indexSuff, blocks); + } + const blocksWithReceipts = []; + const blocks = []; + for (const [suffix, blockNums] of blocksPerSuff.entries()) { + const searchBlock = await fastify.elastic.search({ + index: `${opts.elasticIndexPrefix}-delta-${opts.elasticIndexVersion}-${suffix}`, + size: blockNums.length, + query: { + terms: {"@global.block_num": blockNums} + } + }); + for (const hit of searchBlock.hits.hits) { + const doc: any = hit._source; + if (doc.txAmount > 0) + blocksWithReceipts.push(doc['@global'].block_num) + blocks.push(doc); + } + } + const receipts = await getMultipleReceipts(blocksWithReceipts); + return [blocks, receipts]; + } + + function emptyBlockFromDelta(blockDelta: any) { const blockNumberHex = addHexPrefix(blockDelta['@global'].block_num.toString(16)); const timestamp = new Date(blockDelta['@timestamp']).getTime() / 1000; const parentHash = addHexPrefix(blockDelta['@evmPrevBlockHash']); @@ -350,7 +407,7 @@ export default async function (fastify: FastifyInstance, opts: TelosEvmConfig) { if (!blockDelta) return null; - return await emptyBlockFromDelta(blockDelta); + return emptyBlockFromDelta(blockDelta); } catch (e) { Logger.error(e); return null; @@ -373,7 +430,7 @@ export default async function (fastify: FastifyInstance, opts: TelosEvmConfig) { return null; } - return await emptyBlockFromDelta(blockDelta); + return emptyBlockFromDelta(blockDelta); } catch (e) { Logger.error(e); return null; @@ -381,7 +438,7 @@ export default async function (fastify: FastifyInstance, opts: TelosEvmConfig) { } - async function reconstructBlockFromReceipts(receipts: any[], full: boolean, client: any) { + function reconstructBlockFromReceipts(block, receipts: any[], full: boolean, client: any) { try { let blockHash; let blockHex: string; @@ -391,7 +448,7 @@ export default async function (fastify: FastifyInstance, opts: TelosEvmConfig) { const trxs = []; //Logger.debug(`Reconstructing block from receipts: ${JSON.stringify(receipts)}`) for (const receiptDoc of receipts) { - const {v, r, s} = await getVRS(receiptDoc._source); + const {v, r, s} = getVRS(receiptDoc._source); const receipt = receiptDoc._source['@raw']; if (!blockHash) { @@ -431,11 +488,6 @@ export default async function (fastify: FastifyInstance, opts: TelosEvmConfig) { } } - const block = await getDeltaDocFromNumber(blockNum); - if(!block){ - Logger.error("Could not find block for receipts"); - return null; - } const timestamp = new Date(block['@timestamp']).getTime() / 1000; const gasUsedBlock = addHexPrefix(removeLeftZeros(new BN(block['gasUsed']).toString('hex'))); const extraData = addHexPrefix(block['@blockHash']); @@ -1173,7 +1225,7 @@ export default async function (fastify: FastifyInstance, opts: TelosEvmConfig) { // lookup raw action const receiptAction = await searchActionByHash(trxHash, client); if (!receiptAction) return null; - const {v, r, s} = await getVRS(receiptAction); + const {v, r, s} = getVRS(receiptAction); const receipt = receiptAction['@raw']; // lookup receipt delta @@ -1210,10 +1262,20 @@ export default async function (fastify: FastifyInstance, opts: TelosEvmConfig) { _hash = _hash.slice(2); } const receipts = await getReceiptsByTerm("@raw.block_hash", _hash); - const block = receipts.length > 0 ? await reconstructBlockFromReceipts(receipts, true, client) : await emptyBlockFromHash(_hash); - if(!block || block.transactions?.length === 0) return null; + let block; + if (receipts.length == 0) + block = await emptyBlockFromHash(_hash); + if (!block) return null; + + else { + const blockNum = receipts[0]['@raw'].block; + const delta = await getDeltaDocFromNumber(blockNum); + if (!delta) return null; + block = reconstructBlockFromReceipts(delta, receipts, true, client); + } + const trxIndex = parseInt(trxIndexHex, 16); - if(!block.transactions[trxIndex]) return null; + if(!block.transactions[trxIndex]) return null; let trx = block.transactions[trxIndex]; trx.type = "0x0"; // TODO: Determine type with 1559 trx.chainId = CHAIN_ID_HEX; @@ -1229,7 +1291,6 @@ export default async function (fastify: FastifyInstance, opts: TelosEvmConfig) { if (blockNumber === 0) return GENESIS_BLOCK; - try { const blockDelta = await getDeltaDocFromNumber(blockNumber); if(!blockDelta){ @@ -1238,6 +1299,10 @@ export default async function (fastify: FastifyInstance, opts: TelosEvmConfig) { } if (blockDelta['@transactionsRoot'] === NULL_TRIE) return emptyBlockFromDelta(blockDelta); + else { + const receipts = await getReceiptsByTerm("@raw.block", blockNumber); + return receipts.length > 0 ? reconstructBlockFromReceipts(blockDelta, receipts, full, client) : await emptyBlockFromNumber(blockNumber); + } } catch (e) { Logger.error(`Could not find block from block number ${blockNumber}: ${e}`); if(e?.message.startsWith("index_not_found_exception")){ @@ -1245,9 +1310,6 @@ export default async function (fastify: FastifyInstance, opts: TelosEvmConfig) { } throw(e); } - - const receipts = await getReceiptsByTerm("@raw.block", blockNumber); - return receipts.length > 0 ? await reconstructBlockFromReceipts(receipts, full, client) : await emptyBlockFromNumber(blockNumber); }); /** @@ -1262,7 +1324,16 @@ export default async function (fastify: FastifyInstance, opts: TelosEvmConfig) { _hash = _hash.slice(2); } const receipts = await getReceiptsByTerm("@raw.block_hash", _hash); - return receipts.length > 0 ? await reconstructBlockFromReceipts(receipts, full, client) : await emptyBlockFromHash(_hash); + let block; + if (receipts.length == 0) + block = await emptyBlockFromHash(_hash); + + else { + const blockNum = receipts[0]['@raw'].block; + const delta = await getDeltaDocFromNumber(blockNum); + block = reconstructBlockFromReceipts(delta, receipts, true, client); + } + return block; }); /** @@ -1796,12 +1867,51 @@ export default async function (fastify: FastifyInstance, opts: TelosEvmConfig) { const tRef = process.hrtime.bigint(); - let promises = []; + const promises = []; + // gather all getBlockByNumber requests + const requestedBlockNums = []; + // used to order responses after async processing + const respMap = new Map(); + + // generate promises for request resolution for (let i = 0; i < payload.length; i++) { - let promise = doRpcMethod(payload[i], clientInfo, reply); - promises.push(promise); + let { jsonrpc, id, method, params } = payload[i]; + if (method === 'eth_getBlockByNumber') { + // gather block request info for later + requestedBlockNums.push({params, index: i}); + } else { + // normal request + promises.push(doRpcMethod(payload[i], clientInfo, reply).then(resp => respMap.set(i, resp))); + } } - let responses = await Promise.all(promises); + + if (requestedBlockNums.length > 0) { + // perform optimized batch get block by number request + promises.push( + getMultipleDeltaDocsFromNumbers(requestedBlockNums.map(req => req.params[0])).then( + blockData => { + const [blocks, allReceipts] = blockData; + for (const delta of blocks) { + const blockNum = delta['@global'].block_num; + const request = requestedBlockNums.find(req => req.params[0] == blockNum); + if (delta.txAmount == 0) { + respMap.set(request.index, emptyBlockFromDelta(delta)); + continue; + } + + const receipts = allReceipts.find(r => r["@raw"].block == blockNum); + respMap.set(request.index, reconstructBlockFromReceipts(delta, receipts, request.params[1], clientInfo)); + } + } + )); + } + + await Promise.all(promises); + + // gather results in same order user requested + const responses = [] + for (let i = 0; i < respMap.size; i++) + responses.push(respMap.get(i)); const duration = ((Number(process.hrtime.bigint()) - Number(tRef)) / 1000).toFixed(3); Logger.log(`RPCREQUESTBATCH: ${new Date().toISOString()} - ${duration} μs - ${ip} (${usage}/${limit}) - ${origin} - BATCH OF ${responses.length}`); From a3f4fdd220762fa87f0b6480eaba73b33554b3a6 Mon Sep 17 00:00:00 2001 From: Guillermo Rodriguez Date: Fri, 15 Mar 2024 14:42:09 -0300 Subject: [PATCH 2/5] Fix minor issue was using evm block num for index calc on new optimized getBlockByNumber batch path --- src/routes/evm/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/routes/evm/index.ts b/src/routes/evm/index.ts index 1806b9a..555da16 100644 --- a/src/routes/evm/index.ts +++ b/src/routes/evm/index.ts @@ -302,7 +302,7 @@ export default async function (fastify: FastifyInstance, opts: TelosEvmConfig) { } async function getDeltaDocFromNumber(blockNumber: number, retry: number = 0) { - const indexSuffix = indexSuffixForBlock(blockNumber); + const indexSuffix = indexSuffixForBlock(blockNumber + opts.blockNumberDelta); const results = await fastify.elastic.search({ index: `${opts.elasticIndexPrefix}-delta-${opts.elasticIndexVersion}-${indexSuffix}`, size: 1, @@ -329,7 +329,7 @@ export default async function (fastify: FastifyInstance, opts: TelosEvmConfig) { async function getMultipleReceipts(blockNumbers: number[]) { const blocksPerSuff = new Map(); for (const blockNum of blockNumbers) { - const indexSuff = indexSuffixForBlock(blockNum); + const indexSuff = indexSuffixForBlock(blockNum + opts.blockNumberDelta); const blocks = [blockNum]; if (blocksPerSuff.has(indexSuff)) blocksPerSuff.get(indexSuff).push(...blocks); From 1410b7c92a330cbcda981ffa96fe1c27b9352e14 Mon Sep 17 00:00:00 2001 From: Guillermo Rodriguez Date: Fri, 15 Mar 2024 17:56:21 -0300 Subject: [PATCH 3/5] Fix bug when processing genesis block on batch requests --- src/routes/evm/index.ts | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/src/routes/evm/index.ts b/src/routes/evm/index.ts index 555da16..9f74c82 100644 --- a/src/routes/evm/index.ts +++ b/src/routes/evm/index.ts @@ -325,8 +325,7 @@ export default async function (fastify: FastifyInstance, opts: TelosEvmConfig) { return blockDelta; } - - async function getMultipleReceipts(blockNumbers: number[]) { + function getIndexSuffixesForBlocks(blockNumbers: number[]) { const blocksPerSuff = new Map(); for (const blockNum of blockNumbers) { const indexSuff = indexSuffixForBlock(blockNum + opts.blockNumberDelta); @@ -336,14 +335,21 @@ export default async function (fastify: FastifyInstance, opts: TelosEvmConfig) { else blocksPerSuff.set(indexSuff, blocks); } + return blocksPerSuff; + } + + async function getMultipleReceipts(blockNumbers: number[]) { const receipts = []; - for (const [suffix, blocks] of blocksPerSuff.entries()) { + for (const [suffix, blocks] of getIndexSuffixesForBlocks(blockNumbers).entries()) { const results = await fastify.elastic.search({ index: `${opts.elasticIndexPrefix}-action-${opts.elasticIndexVersion}-${suffix}`, size: 2000, query: { terms: {'@raw.block': blocks} } + }).catch(err => { + console.error(err); + throw err; }); receipts.push( ...results.hits.hits.map(h => h._source)); @@ -352,33 +358,30 @@ export default async function (fastify: FastifyInstance, opts: TelosEvmConfig) { } async function getMultipleDeltaDocsFromNumbers(blockNumbers: number[]) { - const blocksPerSuff = new Map(); - for (const blockNum of blockNumbers) { - const indexSuff = indexSuffixForBlock(blockNum); - const blocks = [blockNum]; - if (blocksPerSuff.has(indexSuff)) - blocksPerSuff.get(indexSuff).push(...blocks); - else - blocksPerSuff.set(indexSuff, blocks); - } const blocksWithReceipts = []; const blocks = []; - for (const [suffix, blockNums] of blocksPerSuff.entries()) { + for (const [suffix, blockNums] of getIndexSuffixesForBlocks(blockNumbers).entries()) { const searchBlock = await fastify.elastic.search({ index: `${opts.elasticIndexPrefix}-delta-${opts.elasticIndexVersion}-${suffix}`, size: blockNums.length, query: { terms: {"@global.block_num": blockNums} } + }).catch(error => { + console.error(error); + throw error; }); for (const hit of searchBlock.hits.hits) { const doc: any = hit._source; - if (doc.txAmount > 0) + if (doc.txAmount) blocksWithReceipts.push(doc['@global'].block_num) blocks.push(doc); } } - const receipts = await getMultipleReceipts(blocksWithReceipts); + let receipts = []; + if (blocksWithReceipts.length > 0) + receipts = await getMultipleReceipts(blocksWithReceipts); + return [blocks, receipts]; } @@ -1894,7 +1897,7 @@ export default async function (fastify: FastifyInstance, opts: TelosEvmConfig) { for (const delta of blocks) { const blockNum = delta['@global'].block_num; const request = requestedBlockNums.find(req => req.params[0] == blockNum); - if (delta.txAmount == 0) { + if (!delta.txAmount) { respMap.set(request.index, emptyBlockFromDelta(delta)); continue; } From c217ae48d9792d2e6bcc27ff74fcc7fac3776846 Mon Sep 17 00:00:00 2001 From: Guillermo Rodriguez Date: Mon, 18 Mar 2024 11:44:45 -0300 Subject: [PATCH 4/5] Add configurable docs per index --- src/main.ts | 4 ++++ src/routes/evm/index.ts | 6 +++--- src/types/index.d.ts | 1 + 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main.ts b/src/main.ts index 577ce66..338fdf5 100644 --- a/src/main.ts +++ b/src/main.ts @@ -12,6 +12,10 @@ program .option('-c, --config [path to config.json]', 'Path to config.json file', 'config.json') .action(async (options) => { const config: TelosEvmConfig = JSON.parse(readFileSync(options.config).toString()); + + if (!config.elasticIndexDocsAmount) + config.elasticIndexDocsAmount = 1e7; + const rpc: TelosEVMRPC = new TelosEVMRPC(config); console.log("Starting Telos EVM RPC..."); diff --git a/src/routes/evm/index.ts b/src/routes/evm/index.ts index 9f74c82..0676523 100644 --- a/src/routes/evm/index.ts +++ b/src/routes/evm/index.ts @@ -292,8 +292,8 @@ export default async function (fastify: FastifyInstance, opts: TelosEvmConfig) { } function adjustBlockNum(num: number): number { - // convert to native block num and divide over index size 10 million - return Math.floor((num + opts.blockNumberDelta) / 1e7); + // convert to native block num and divide over dos per index + return Math.floor((num + opts.blockNumberDelta) / opts.elasticIndexDocsAmount); } function indexSuffixForBlock(blockNumber: number): string { @@ -569,7 +569,7 @@ export default async function (fastify: FastifyInstance, opts: TelosEvmConfig) { if (indices.length > 1) { const docsCount = parseInt(index['docs.count']); const adjustedNum = indexToSuffixNum(index.index); - lastBlockNum = (adjustedNum * 1e7) + docsCount - opts.blockNumberDelta - 1; + lastBlockNum = (adjustedNum * opts.elasticIndexDocsAmount) + docsCount - opts.blockNumberDelta - 1; } else if (index?.index) { const results = await fastify.elastic.search({ index: `${index.index}`, diff --git a/src/types/index.d.ts b/src/types/index.d.ts index de3de1b..322f67f 100644 --- a/src/types/index.d.ts +++ b/src/types/index.d.ts @@ -31,6 +31,7 @@ export interface TelosEvmConfig { elasticPass: string; elasticIndexPrefix: string; elasticIndexVersion: string; + elasticIndexDocsAmount: number; orderNonces: boolean; orderNonceRetryTimeout: number; syncingThreshhold: number; From 50af88196db2a8267531adc45eb6d630c1caefd6 Mon Sep 17 00:00:00 2001 From: Guillermo Rodriguez Date: Tue, 9 Apr 2024 16:58:18 -0300 Subject: [PATCH 5/5] Fix bugs related to minor changes to elastic document handling after quering db, unwrap _soruce[@raw] at highest posible point, right after db read --- src/routes/evm/index.ts | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/routes/evm/index.ts b/src/routes/evm/index.ts index 0676523..c12bc3f 100644 --- a/src/routes/evm/index.ts +++ b/src/routes/evm/index.ts @@ -200,8 +200,7 @@ export default async function (fastify: FastifyInstance, opts: TelosEvmConfig) { } } - function getVRS(receiptDoc): {v: string, r: string, s: string} { - let receipt = receiptDoc["@raw"]; + function getVRS(receipt): {v: string, r: string, s: string} { const v = removeLeftZeros(BigInt(receipt.v).toString(16), true); const r = removeLeftZeros(receipt.r, true); const s = removeLeftZeros(receipt.s, true); @@ -450,9 +449,8 @@ export default async function (fastify: FastifyInstance, opts: TelosEvmConfig) { let bloom = new Bloom(); const trxs = []; //Logger.debug(`Reconstructing block from receipts: ${JSON.stringify(receipts)}`) - for (const receiptDoc of receipts) { - const {v, r, s} = getVRS(receiptDoc._source); - const receipt = receiptDoc._source['@raw']; + for (const receipt of receipts) { + const {v, r, s} = getVRS(receipt); if (!blockHash) { blockHash = addHexPrefix(receipt['block_hash']); @@ -528,7 +526,7 @@ export default async function (fastify: FastifyInstance, opts: TelosEvmConfig) { size: 2000, query: { bool: { must: [{ term: termStruct }] } } }); - return results?.hits?.hits; + return results?.hits?.hits.map(h => h._source['@raw']); } async function getCurrentBlockNumber(indexed: boolean = false, retry: number = 0) { @@ -1271,7 +1269,7 @@ export default async function (fastify: FastifyInstance, opts: TelosEvmConfig) { if (!block) return null; else { - const blockNum = receipts[0]['@raw'].block; + const blockNum = receipts[0].block; const delta = await getDeltaDocFromNumber(blockNum); if (!delta) return null; block = reconstructBlockFromReceipts(delta, receipts, true, client); @@ -1332,8 +1330,8 @@ export default async function (fastify: FastifyInstance, opts: TelosEvmConfig) { block = await emptyBlockFromHash(_hash); else { - const blockNum = receipts[0]['@raw'].block; - const delta = await getDeltaDocFromNumber(blockNum); + const blockNum = receipts[0].block; + const delta = await getDeltaDocFromNumber(blockNum - opts.blockNumberDelta); block = reconstructBlockFromReceipts(delta, receipts, true, client); } return block; @@ -1722,8 +1720,7 @@ export default async function (fastify: FastifyInstance, opts: TelosEvmConfig) { throw new Error("trace_replayBlockTransactions only supports the \"trace\" type of trace (not vmTrace or stateDiff"); const blockNumber = parseInt(await toBlockNumber(block), 16); - const receiptHits = await getReceiptsByTerm("@raw.block", blockNumber); - const receipts = receiptHits.map(r => r._source["@raw"]); + const receipts = await getReceiptsByTerm("@raw.block", blockNumber); const sortedReceipts = receipts.sort((a, b) => { return a.trx_index - b.trx_index; }) @@ -1740,8 +1737,7 @@ export default async function (fastify: FastifyInstance, opts: TelosEvmConfig) { methods.set('trace_block', async ([block]) => { const blockNumber = parseInt(await toBlockNumber(block), 16); - const receiptHits = await getReceiptsByTerm("@raw.block", blockNumber); - const receipts = receiptHits.map(r => r._source["@raw"]); + const receipts = await getReceiptsByTerm("@raw.block", blockNumber); const sortedReceipts = receipts.sort((a, b) => { return a.trx_index - b.trx_index; })