Skip to content

Commit 60aa2c4

Browse files
gballetgballet
committed
overlay transition
Fix some bugs identified in the code review Co-authored-by: Ignacio Hagopian <[email protected]> Include base -> overlay key-values migration logic (#199) * mod: add go-verkle version with key-value migration new apis Signed-off-by: Ignacio Hagopian <[email protected]> * core/stateprocessor: use constant for max number of migrated key-values Signed-off-by: Ignacio Hagopian <[email protected]> * core: add base->overlay key-values migration logic Signed-off-by: Ignacio Hagopian <[email protected]> * core: fix some compiler errors Signed-off-by: Ignacio Hagopian <[email protected]> * trie: consider removing transition trie api in the future Signed-off-by: Ignacio Hagopian <[email protected]> * mod: use latest go-verkle Signed-off-by: Ignacio Hagopian <[email protected]> --------- Signed-off-by: Ignacio Hagopian <[email protected]> fix some unit tests errors get convresion block from file fix compilation issues fix initialization issue in migrator fix: changes needed to run the first 28 blocks important sutff: fix the banner fix: use nonce instead of balance in nonce leaf (#202) fixes for performing the overlay transition (#203) * fixes for performing the overlay transition * fixes for the full replay * fix: deletion-and-recreation of EoA * fixes to replay 2M+ blocks * upgrade to go-verkle@master * fix: proper number of chunk evals * rewrite conversion loop to fix known issues changes to make replay work with the overlay method (#216) * fixes for performing the overlay transition fixes for the full replay fix: deletion-and-recreation of EoA fixes to replay 2M+ blocks upgrade to go-verkle@master fix: proper number of chunk evals rewrite conversion loop to fix known issues changes to make replay work with the overlay method fixes to replay 2M+ blocks update to latest go-verkle@master * use a PBSS-like scheme for internal nodes (#221) * use a PBSS-like scheme for internal nodes * a couple of fixes coming from debugging replay * fix: use an error to notify the transition tree that a deleted account was found in the overlay tree (#222) * fixes for pbss replay (#227) * fixes for pbss replay * trie/verkle: use capped batch size (#229) * trie/verkle: use capped batch size Signed-off-by: Ignacio Hagopian <[email protected]> * trie/verkle: avoid path variable allocation per db.Put Signed-off-by: Ignacio Hagopian <[email protected]> * don't keep more than 32 state root conversions in RAM (#230) --------- Signed-off-by: Ignacio Hagopian <[email protected]> Co-authored-by: Guillaume Ballet <[email protected]> * cleanup some code * mod: update go-verkle Signed-off-by: Ignacio Hagopian <[email protected]> * re-enable snapshot (#231) * re-enable cancun block / snapshot (#226) * clear storage conversion key upon translating account (#234) * clear storage conversion key upon translating account * mod: use latest go-verkle Signed-off-by: Ignacio Hagopian <[email protected]> --------- Signed-off-by: Ignacio Hagopian <[email protected]> Co-authored-by: Ignacio Hagopian <[email protected]> * fix: self-deadlock with translated root map mutex (#236) * return compressed commitment as root commitment (#237) --------- Signed-off-by: Ignacio Hagopian <[email protected]> Co-authored-by: Ignacio Hagopian <[email protected]> --------- Signed-off-by: Ignacio Hagopian <[email protected]> Co-authored-by: Ignacio Hagopian <[email protected]> --------- Signed-off-by: Ignacio Hagopian <[email protected]> Co-authored-by: Ignacio Hagopian <[email protected]> fix first panic in *TransitionTrie.Copy() upgrade go-verkle to latest master mod: update go-verkle (#239) Signed-off-by: Ignacio Hagopian <[email protected]> core: print state root every 100 blocks (#240) Signed-off-by: Ignacio Hagopian <[email protected]> fix: only Commit the account trie (#242) fixes to get TestProcessVerkle to work with the overlay branch (#238) * fixes to get TestProcessVerkle to work with the overlay branch * fix all panics in verkle state processor test * fix proof verification move transition management to cachingDB
1 parent 0a7efe1 commit 60aa2c4

18 files changed

+940
-91
lines changed

‎cmd/geth/verkle.go

+6-7
Original file line numberDiff line numberDiff line change
@@ -130,14 +130,13 @@ func convertToVerkle(ctx *cli.Context) error {
130130
vRoot = verkle.New().(*verkle.InternalNode)
131131
)
132132

133-
saveverkle := func(node verkle.VerkleNode) {
134-
comm := node.Commit()
133+
saveverkle := func(path []byte, node verkle.VerkleNode) {
134+
node.Commit()
135135
s, err := node.Serialize()
136136
if err != nil {
137137
panic(err)
138138
}
139-
commB := comm.Bytes()
140-
if err := chaindb.Put(commB[:], s); err != nil {
139+
if err := chaindb.Put(path, s); err != nil {
141140
panic(err)
142141
}
143142
}
@@ -330,7 +329,7 @@ func checkChildren(root verkle.VerkleNode, resolver verkle.NodeResolverFn) error
330329
return fmt.Errorf("could not find child %x in db: %w", childC, err)
331330
}
332331
// depth is set to 0, the tree isn't rebuilt so it's not a problem
333-
childN, err := verkle.ParseNode(childS, 0, childC[:])
332+
childN, err := verkle.ParseNode(childS, 0)
334333
if err != nil {
335334
return fmt.Errorf("decode error child %x in db: %w", child.Commitment().Bytes(), err)
336335
}
@@ -390,7 +389,7 @@ func verifyVerkle(ctx *cli.Context) error {
390389
if err != nil {
391390
return err
392391
}
393-
root, err := verkle.ParseNode(serializedRoot, 0, rootC[:])
392+
root, err := verkle.ParseNode(serializedRoot, 0)
394393
if err != nil {
395394
return err
396395
}
@@ -439,7 +438,7 @@ func expandVerkle(ctx *cli.Context) error {
439438
if err != nil {
440439
return err
441440
}
442-
root, err := verkle.ParseNode(serializedRoot, 0, rootC[:])
441+
root, err := verkle.ParseNode(serializedRoot, 0)
443442
if err != nil {
444443
return err
445444
}

‎cmd/utils/cmd.go

+12
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
"os"
2727
"os/signal"
2828
"runtime"
29+
"runtime/pprof"
2930
"strings"
3031
"syscall"
3132
"time"
@@ -173,6 +174,17 @@ func ImportChain(chain *core.BlockChain, fn string) error {
173174
return err
174175
}
175176
}
177+
cpuProfile, err := os.Create("cpu.out")
178+
if err != nil {
179+
return fmt.Errorf("Error creating CPU profile: %v", err)
180+
}
181+
defer cpuProfile.Close()
182+
err = pprof.StartCPUProfile(cpuProfile)
183+
if err != nil {
184+
return fmt.Errorf("Error starting CPU profile: %v", err)
185+
}
186+
defer pprof.StopCPUProfile()
187+
176188
stream := rlp.NewStream(reader, 0)
177189

178190
// Run actual the import.

‎core/block_validator.go

+18-13
Original file line numberDiff line numberDiff line change
@@ -65,12 +65,17 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
6565
if hash := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); hash != header.TxHash {
6666
return fmt.Errorf("transaction root hash mismatch: have %x, want %x", hash, header.TxHash)
6767
}
68-
if !v.bc.HasBlockAndState(block.ParentHash(), block.NumberU64()-1) {
69-
if !v.bc.HasBlock(block.ParentHash(), block.NumberU64()-1) {
70-
return consensus.ErrUnknownAncestor
71-
}
72-
return consensus.ErrPrunedAncestor
73-
}
68+
// XXX I had to deactivate this check for replay to work: the block state root
69+
// hash is the one of the overlay tree, but in replay mode, it's the hash of
70+
// the base tree that takes precedence, as the chain would not otherwise be
71+
// recognized.
72+
// if !v.bc.HasBlockAndState(block.ParentHash(), block.NumberU64()-1) {
73+
// if !v.bc.HasBlock(block.ParentHash(), block.NumberU64()-1) {
74+
// return consensus.ErrUnknownAncestor
75+
// }
76+
// fmt.Println("failure here")
77+
// return consensus.ErrPrunedAncestor
78+
// }
7479
return nil
7580
}
7681

@@ -90,15 +95,15 @@ func (v *BlockValidator) ValidateState(block *types.Block, statedb *state.StateD
9095
return fmt.Errorf("invalid bloom (remote: %x local: %x)", header.Bloom, rbloom)
9196
}
9297
// Tre receipt Trie's root (R = (Tr [[H1, R1], ... [Hn, Rn]]))
93-
receiptSha := types.DeriveSha(receipts, trie.NewStackTrie(nil))
94-
if receiptSha != header.ReceiptHash {
95-
return fmt.Errorf("invalid receipt root hash (remote: %x local: %x)", header.ReceiptHash, receiptSha)
96-
}
98+
// receiptSha := types.DeriveSha(receipts, trie.NewStackTrie(nil))
99+
// if receiptSha != header.ReceiptHash {
100+
// return fmt.Errorf("invalid receipt root hash (remote: %x local: %x)", header.ReceiptHash, receiptSha)
101+
// }
97102
// Validate the state root against the received state root and throw
98103
// an error if they don't match.
99-
if root := statedb.IntermediateRoot(v.config.IsEIP158(header.Number)); header.Root != root {
100-
return fmt.Errorf("invalid merkle root (remote: %x local: %x)", header.Root, root)
101-
}
104+
// if root := statedb.IntermediateRoot(v.config.IsEIP158(header.Number)); header.Root != root {
105+
// return fmt.Errorf("invalid merkle root (remote: %x local: %x)", header.Root, root)
106+
// }
102107
return nil
103108
}
104109

‎core/blockchain.go

+36
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,15 @@
1818
package core
1919

2020
import (
21+
"bufio"
2122
"errors"
2223
"fmt"
2324
"io"
2425
"math/big"
26+
"os"
2527
"runtime"
2628
"sort"
29+
"strconv"
2730
"strings"
2831
"sync"
2932
"sync/atomic"
@@ -1486,6 +1489,21 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, setHead bool)
14861489
return 0, nil
14871490
}
14881491

1492+
f, err := os.Open("conversion.txt")
1493+
if err != nil {
1494+
log.Error("Failed to open conversion.txt", "err", err)
1495+
return 0, err
1496+
}
1497+
defer f.Close()
1498+
scanner := bufio.NewScanner(f)
1499+
scanner.Scan()
1500+
conversionBlock, err := strconv.ParseUint(scanner.Text(), 10, 64)
1501+
if err != nil {
1502+
log.Error("Failed to parse conversionBlock", "err", err)
1503+
return 0, err
1504+
}
1505+
log.Info("Found conversion block info", "conversionBlock", conversionBlock)
1506+
14891507
// Start a parallel signature recovery (signer will fluke on fork transition, minimal perf loss)
14901508
senderCacher.recoverFromBlocks(types.MakeSigner(bc.chainConfig, chain[0].Number()), chain)
14911509

@@ -1670,6 +1688,10 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, setHead bool)
16701688
if parent == nil {
16711689
parent = bc.GetHeader(block.ParentHash(), block.NumberU64()-1)
16721690
}
1691+
1692+
if parent.Number.Uint64() == conversionBlock {
1693+
bc.StartVerkleTransition(parent.Root, emptyVerkleRoot, bc.Config(), parent.Number)
1694+
}
16731695
statedb, err := state.New(parent.Root, bc.stateCache, bc.snaps)
16741696
if err != nil {
16751697
return it.index, err
@@ -1706,6 +1728,10 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, setHead bool)
17061728
return it.index, err
17071729
}
17081730

1731+
if statedb.Database().InTransition() || statedb.Database().Transitionned() {
1732+
bc.AddRootTranslation(block.Root(), statedb.IntermediateRoot(false))
1733+
}
1734+
17091735
// Update the metrics touched during block processing
17101736
accountReadTimer.Update(statedb.AccountReads) // Account reads are complete, we can mark them
17111737
storageReadTimer.Update(statedb.StorageReads) // Storage reads are complete, we can mark them
@@ -2287,6 +2313,8 @@ func (bc *BlockChain) skipBlock(err error, it *insertIterator) bool {
22872313
return false
22882314
}
22892315

2316+
var emptyVerkleRoot common.Hash
2317+
22902318
// indexBlocks reindexes or unindexes transactions depending on user configuration
22912319
func (bc *BlockChain) indexBlocks(tail *uint64, head uint64, done chan struct{}) {
22922320
defer func() { close(done) }()
@@ -2431,3 +2459,11 @@ func (bc *BlockChain) SetBlockValidatorAndProcessorForTesting(v Validator, p Pro
24312459
bc.validator = v
24322460
bc.processor = p
24332461
}
2462+
2463+
func (bc *BlockChain) StartVerkleTransition(originalRoot, translatedRoot common.Hash, chainConfig *params.ChainConfig, cancunBlock *big.Int) {
2464+
bc.stateCache.StartVerkleTransition(originalRoot, translatedRoot, chainConfig, cancunBlock)
2465+
}
2466+
2467+
func (bc *BlockChain) AddRootTranslation(originalRoot, translatedRoot common.Hash) {
2468+
bc.stateCache.AddRootTranslation(originalRoot, translatedRoot)
2469+
}

‎core/chain_makers.go

+17-1
Original file line numberDiff line numberDiff line change
@@ -313,6 +313,7 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine
313313
keyvals := make([]verkle.StateDiff, 0, n)
314314
blocks, receipts := make(types.Blocks, n), make([]types.Receipts, n)
315315
chainreader := &fakeChainReader{config: config}
316+
var preStateTrie *trie.VerkleTrie
316317
genblock := func(i int, parent *types.Block, statedb *state.StateDB) (*types.Block, types.Receipts) {
317318
b := &BlockGen{i: i, chain: blocks, parent: parent, statedb: statedb, config: config, engine: engine}
318319
b.header = makeHeader(chainreader, parent, statedb, b.engine)
@@ -372,13 +373,28 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine
372373
kvs[string(key)] = v
373374
}
374375

376+
// Initialize the preStateTrie if it is nil, this should
377+
// correspond to the genesis block. This is a workaround
378+
// needed until the main verkle PR is rebased on top of
379+
// PBSS.
380+
if preStateTrie == nil {
381+
preStateTrie = vtr
382+
}
383+
375384
vtr.Hash()
376-
p, k, err := vtr.ProveAndSerialize(statedb.Witness().Keys(), kvs)
385+
p, k, err := preStateTrie.ProveAndSerialize(statedb.Witness().Keys(), kvs)
377386
if err != nil {
378387
panic(err)
379388
}
380389
proofs = append(proofs, p)
381390
keyvals = append(keyvals, k)
391+
392+
// save the current state of the trie for producing the proof for the next block,
393+
// since reading it from disk is broken with the intermediate PBSS-like system we
394+
// have: it will read the post-state as this is the only state present on disk.
395+
// This is a workaround needed until the main verkle PR is rebased on top of PBSS.
396+
preStateTrie = statedb.GetTrie().(*trie.VerkleTrie)
397+
382398
return block, b.receipts
383399
}
384400
return nil, nil

0 commit comments

Comments
 (0)