From 3c82730709ca7daba3f8859345750e2732b38379 Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Fri, 4 Jul 2025 14:29:48 +0530 Subject: [PATCH 1/5] use hash writer --- execution_chain/db/aristo/aristo_compute.nim | 90 ++++++++++++++------ 1 file changed, 65 insertions(+), 25 deletions(-) diff --git a/execution_chain/db/aristo/aristo_compute.nim b/execution_chain/db/aristo/aristo_compute.nim index 9e0cacb30c..c94663353b 100644 --- a/execution_chain/db/aristo/aristo_compute.nim +++ b/execution_chain/db/aristo/aristo_compute.nim @@ -89,24 +89,70 @@ proc putKeyAtLevel( ok() -template encodeLeaf(w: var RlpWriter, pfx: NibblesBuf, leafData: untyped): HashKey = +template appendLeaf(w: var RlpWriter, pfx: NibblesBuf, leafData: untyped) = w.startList(2) w.append(pfx.toHexPrefix(isLeaf = true).data()) + w.wrapEncoding(1) w.append(leafData) - w.finish().digestTo(HashKey) -template encodeBranch(w: var RlpWriter, vtx: VertexRef, subKeyForN: untyped): HashKey = +template encodeLeaf(pfx: NibblesBuf, leafData: untyped): HashKey = + var tracker = DynamicRlpLengthTracker() + tracker.initLengthTracker() + tracker.appendLeaf(pfx, leafData) + + if tracker.totalLength < 32: + var writer = initTwoPassWriter(tracker) + writer.appendLeaf(pfx, leafData) + let buf = HashKey.fromBytes(writer.finish) + buf.value + else: + var writer = initHashWriter(tracker) + writer.appendLeaf(pfx, leafData) + let buf = writer.finish() + buf.to(HashKey) + +template appendBranch(w: var RlpWriter, vtx: VertexRef, subKeyForN: untyped) = w.startList(17) for (n {.inject.}, subvid {.inject.}) in vtx.allPairs(): w.append(subKeyForN) w.append EmptyBlob - w.finish().digestTo(HashKey) -template encodeExt(w: var RlpWriter, pfx: NibblesBuf, branchKey: HashKey): HashKey = +template encodeBranch(vtx: VertexRef, subKeyForN: untyped): HashKey = + var tracker: DynamicRlpLengthTracker + tracker.initLengthTracker() + tracker.appendBranch(vtx, subKeyForN) + + if tracker.totalLength < 32: + var writer = initTwoPassWriter(tracker) + writer.appendBranch(vtx, subKeyForN) + let buf = HashKey.fromBytes(writer.finish) + buf.value + else: + var writer = initHashWriter(tracker) + writer.appendBranch(vtx, subKeyForN) + let buf = writer.finish() + buf.to(HashKey) + +template appendExt(w: var RlpWriter, pfx: NibblesBuf, branchKey: HashKey) = w.startList(2) w.append(pfx.toHexPrefix(isLeaf = false).data()) w.append(branchKey) - w.finish().digestTo(HashKey) + +template encodeExt(pfx: NibblesBuf, branchKey: untyped): HashKey = + var tracker: DynamicRlpLengthTracker + tracker.initLengthTracker() + tracker.appendExt(pfx, branchKey) + + if tracker.totalLength < 32: + var writer = initTwoPassWriter(tracker) + writer.appendExt(pfx, branchKey) + let buf = HashKey.fromBytes(writer.finish) + buf.value + else: + var writer = initHashWriter(tracker) + writer.appendExt(pfx, branchKey) + let buf = writer.finish() + buf.to(HashKey) proc getKey( db: AristoTxRef, rvid: RootedVertexID, skipLayers: static bool @@ -146,14 +192,11 @@ proc computeKeyImpl( # Top-most level of all the verticies this hash computation depends on var level = level - # TODO this is the same code as when serializing NodeRef, without the NodeRef - var writer = initRlpWriter() - let key = case vtx.vType of AccLeaf: let vtx = AccLeafRef(vtx) - writer.encodeLeaf(vtx.pfx): + encodeLeaf(vtx.pfx): let stoID = vtx.stoID skey = @@ -176,7 +219,7 @@ proc computeKeyImpl( else: VOID_HASH_KEY - rlp.encode Account( + Account( nonce: vtx.account.nonce, balance: vtx.account.balance, storageRoot: skey.to(Hash32), @@ -184,9 +227,8 @@ proc computeKeyImpl( ) of StoLeaf: let vtx = StoLeafRef(vtx) - writer.encodeLeaf(vtx.pfx): - # TODO avoid memory allocation when encoding storage data - rlp.encode(vtx.stoData) + encodeLeaf(vtx.pfx): + vtx.stoData of Branches: # For branches, we need to load the vertices before recursing into them # to exploit their on-disk order @@ -245,21 +287,19 @@ proc computeKeyImpl( ) batch.leave(n) - template writeBranch(w: var RlpWriter, vtx: BranchRef): HashKey = - w.encodeBranch(vtx): + template writeBranch(): HashKey = + encodeBranch(vtx): if subvid.isValid: level = max(level, keyvtxs[n][1]) keyvtxs[n][0][0] else: VOID_HASH_KEY - if vtx.vType == ExtBranch: - let vtx = ExtBranchRef(vtx) - writer.encodeExt(vtx.pfx): - var bwriter = initRlpWriter() - bwriter.writeBranch(vtx) + if vtx.pfx.len > 0: # Extension node + encodeExt(vtx.pfx): + writeBranch() else: - writer.writeBranch(vtx) + writeBranch() # Cache the hash into the same storage layer as the the top-most value that it # depends on (recursively) - this could be an ephemeral in-memory layer or the @@ -268,9 +308,9 @@ proc computeKeyImpl( # root key also changing while leaves that have never been hashed will see # their hash being saved directly to the backend. - if vtx.vType in Branches: - ?db.putKeyAtLevel(rvid, BranchRef(vtx), key, level, batch) - ok (key, level) + if vtx.vType notin Leaves: + ?db.putKeyAtLevel(rvid, vtx, key, level, batch) + return ok (key, level) proc computeKeyImpl( db: AristoTxRef, rvid: RootedVertexID, skipLayers: static bool From dbe5d5c0ff355ccb7974a92f2a1de5fa085936f9 Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Fri, 4 Jul 2025 14:41:50 +0530 Subject: [PATCH 2/5] review --- execution_chain/db/aristo/aristo_compute.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/execution_chain/db/aristo/aristo_compute.nim b/execution_chain/db/aristo/aristo_compute.nim index c94663353b..a89a875d79 100644 --- a/execution_chain/db/aristo/aristo_compute.nim +++ b/execution_chain/db/aristo/aristo_compute.nim @@ -310,7 +310,7 @@ proc computeKeyImpl( if vtx.vType notin Leaves: ?db.putKeyAtLevel(rvid, vtx, key, level, batch) - return ok (key, level) + ok (key, level) proc computeKeyImpl( db: AristoTxRef, rvid: RootedVertexID, skipLayers: static bool From 4549b8f0956d2b11c2496d33d429aa64c6adda4f Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Wed, 16 Jul 2025 16:03:03 +0530 Subject: [PATCH 3/5] fix --- execution_chain/db/aristo/aristo_compute.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/execution_chain/db/aristo/aristo_compute.nim b/execution_chain/db/aristo/aristo_compute.nim index a89a875d79..953f92608e 100644 --- a/execution_chain/db/aristo/aristo_compute.nim +++ b/execution_chain/db/aristo/aristo_compute.nim @@ -309,7 +309,7 @@ proc computeKeyImpl( # their hash being saved directly to the backend. if vtx.vType notin Leaves: - ?db.putKeyAtLevel(rvid, vtx, key, level, batch) + ?db.putKeyAtLevel(rvid, BranchRef(vtx), key, level, batch) ok (key, level) proc computeKeyImpl( From 26728ec23a0199818b8accba8073045465ceb87a Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Mon, 21 Jul 2025 12:00:54 +0530 Subject: [PATCH 4/5] possible fix? --- execution_chain/db/aristo/aristo_compute.nim | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/execution_chain/db/aristo/aristo_compute.nim b/execution_chain/db/aristo/aristo_compute.nim index 953f92608e..6243caf959 100644 --- a/execution_chain/db/aristo/aristo_compute.nim +++ b/execution_chain/db/aristo/aristo_compute.nim @@ -286,8 +286,8 @@ proc computeKeyImpl( skipLayers = skipLayers, ) batch.leave(n) - - template writeBranch(): HashKey = + + template writeBranch(vtx: BranchRef): HashKey = encodeBranch(vtx): if subvid.isValid: level = max(level, keyvtxs[n][1]) @@ -295,11 +295,10 @@ proc computeKeyImpl( else: VOID_HASH_KEY - if vtx.pfx.len > 0: # Extension node + if vtx.vType == ExtBranch: + let vtx = ExtBranchRef(vtx) encodeExt(vtx.pfx): - writeBranch() - else: - writeBranch() + writeBranch(vtx) # Cache the hash into the same storage layer as the the top-most value that it # depends on (recursively) - this could be an ephemeral in-memory layer or the @@ -308,7 +307,7 @@ proc computeKeyImpl( # root key also changing while leaves that have never been hashed will see # their hash being saved directly to the backend. - if vtx.vType notin Leaves: + if vtx.vType in Branches: ?db.putKeyAtLevel(rvid, BranchRef(vtx), key, level, batch) ok (key, level) From 1d0b3b7004dc3d9e2c92fdd2faa6e39dbe406da6 Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Wed, 23 Jul 2025 16:09:04 +0530 Subject: [PATCH 5/5] minor fix --- execution_chain/db/aristo/aristo_compute.nim | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/execution_chain/db/aristo/aristo_compute.nim b/execution_chain/db/aristo/aristo_compute.nim index 6243caf959..fef03be2e1 100644 --- a/execution_chain/db/aristo/aristo_compute.nim +++ b/execution_chain/db/aristo/aristo_compute.nim @@ -286,7 +286,7 @@ proc computeKeyImpl( skipLayers = skipLayers, ) batch.leave(n) - + template writeBranch(vtx: BranchRef): HashKey = encodeBranch(vtx): if subvid.isValid: @@ -295,10 +295,15 @@ proc computeKeyImpl( else: VOID_HASH_KEY + var encoding: HashKey if vtx.vType == ExtBranch: let vtx = ExtBranchRef(vtx) - encodeExt(vtx.pfx): + encoding = encodeExt(vtx.pfx): writeBranch(vtx) + else: + encoding = writeBranch(vtx) + + encoding # Cache the hash into the same storage layer as the the top-most value that it # depends on (recursively) - this could be an ephemeral in-memory layer or the