diff --git a/cmd/geth/pool.go b/cmd/geth/pool.go new file mode 100644 index 0000000000..4ba77abafc --- /dev/null +++ b/cmd/geth/pool.go @@ -0,0 +1,60 @@ +package main + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state/snapshot" + "sync" + "sync/atomic" +) + +type routinePool struct { + wg sync.WaitGroup + taskQueue chan common.Hash + + emptyContract atomic.Uint64 + arrCount atomic.Uint64 + varCount atomic.Uint64 + arrItemCount atomic.Uint64 + arrItemCountMap map[int]uint64 + mapLock *sync.Mutex +} + +func NewRoutinePool(size int64, snap snapshot.Snapshot, keySet []common.Hash) *routinePool { + pool := &routinePool{ + taskQueue: make(chan common.Hash), + arrCount: atomic.Uint64{}, + varCount: atomic.Uint64{}, + arrItemCount: atomic.Uint64{}, + arrItemCountMap: make(map[int]uint64), + mapLock: &sync.Mutex{}, + } + + for i := int64(0); i < size; i++ { + go pool.worker(snap, keySet) + } + return pool +} + +func (pool *routinePool) AddTask(addrHash common.Hash) { + pool.wg.Add(1) + pool.taskQueue <- addrHash +} + +func (pool *routinePool) worker(snap snapshot.Snapshot, keySet []common.Hash) { + for addrHash := range pool.taskQueue { + arrCount, arrItemCount, varCount := traversalContract(snap, addrHash, keySet, pool.mapLock, pool.arrItemCountMap) + pool.arrCount.Add(uint64(arrCount)) + pool.arrItemCount.Add(uint64(arrItemCount)) + pool.varCount.Add(uint64(varCount)) + + if arrCount == 0 && varCount == 0 { + pool.emptyContract.Add(1) + } + pool.wg.Done() + } +} + +// Wait +func (pool *routinePool) Wait() { + pool.wg.Wait() +} diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 040736a0d7..be7ad40bb9 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -21,15 +21,18 @@ import ( "encoding/json" "errors" "fmt" + "github.com/ethereum/go-ethereum/common" + "math/big" "os" "path/filepath" + "sort" "strings" + "sync" "time" "github.com/prometheus/tsdb/fileutil" "github.com/ethereum/go-ethereum/cmd/utils" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" @@ -220,6 +223,19 @@ block is used. Description: ` The export-preimages command exports hash preimages to a flat file, in exactly the expected order for the overlay tree migration. +`, + }, + { + Name: "inspect-storage", + Usage: "Statistical variable data for all contracts", + Action: inspectStorage, + Flags: flags.Merge([]cli.Flag{ + utils.CPUCountFlag, + }, utils.DatabaseFlags), + Description: ` +This command collects the variable information of all contracts from snapshot. + +The argument is interpreted as the number of cpus. `, }, }, @@ -1020,3 +1036,216 @@ func checkAccount(ctx *cli.Context) error { log.Info("Checked the snapshot journalled storage", "time", common.PrettyDuration(time.Since(start))) return nil } + +func inspectStorage(ctx *cli.Context) error { + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + _, db, root, err := parseDumpConfig(ctx, stack) + if err != nil { + return err + } + defer db.Close() + tdb := utils.MakeTrieDatabase(ctx, stack, db, false, true, false) + defer tdb.Close() + + snapConfig := snapshot.Config{ + CacheSize: 256, + Recovery: false, + NoBuild: true, + AsyncBuild: false, + } + triesInMemory := ctx.Uint64(utils.TriesInMemoryFlag.Name) + snaptree, err := snapshot.New(snapConfig, db, tdb, root, int(triesInMemory), false) + snap := snaptree.Snapshot(root) + if err != nil { + return err + } + + // init key + keySet := make([]common.Hash, 0, 1000) + for i := 0; i < 1000; i++ { + key := common.BigToHash(big.NewInt(int64(i))) + keySet = append(keySet, crypto.Keccak256Hash(key.Bytes())) + } + + cpuCount := ctx.Int64(utils.CPUCountFlag.Name) + p := NewRoutinePool(cpuCount*100, snap, keySet) + + it := db.NewIterator(rawdb.SnapshotAccountPrefix, nil) + defer it.Release() + + // Inspect key-value database first. + idx := uint64(0) + for it.Next() { + key := it.Key() + switch { + case bytes.HasPrefix(key, rawdb.SnapshotAccountPrefix) && len(key) == (len(rawdb.SnapshotAccountPrefix)+common.HashLength): + var account types.SlimAccount + err := rlp.DecodeBytes(it.Value(), &account) + if err != nil { + log.Error("failed to DecodeBytes", "addrHash", key[1:]) + continue + } + if bytes.Equal(account.CodeHash, types.EmptyCodeHash.Bytes()) { + continue + } + if idx%10000 == 0 { + fmt.Printf("add task: %d\n", idx) + } + idx++ + p.AddTask(common.BytesToHash(key[1:])) + } + } + + p.Wait() + fmt.Println("all done") + fmt.Println("var count", p.varCount.Load()) + fmt.Println("arr count", p.arrCount.Load()) + fmt.Println("arr item count", p.arrItemCount.Load()) + + f, err := os.Create("result.txt") + if err != nil { + fmt.Println(err) + } + + _, err = f.WriteString(fmt.Sprintf("var count: %d\n", p.varCount.Load())) + if err != nil { + fmt.Println(err) + } + + _, err = f.WriteString(fmt.Sprintf("arr count: %d\n", p.arrCount.Load())) + if err != nil { + fmt.Println(err) + } + + _, err = f.WriteString(fmt.Sprintf("arr item count: %d\n", p.arrItemCount.Load())) + if err != nil { + fmt.Println(err) + } + + _, err = f.WriteString(fmt.Sprintf("empty contract count: %d\n", p.emptyContract.Load())) + if err != nil { + fmt.Println(err) + } + + arrCount := make([]int, 0, len(p.arrItemCountMap)) + for k := range p.arrItemCountMap { + arrCount = append(arrCount, k) + } + sort.Ints(arrCount) + sum := uint64(0) + maxCount := uint64(0) + mode := 0 + for _, k := range arrCount { + count := p.arrItemCountMap[k] + sum += count + if sum >= p.arrCount.Load()/2 { + fmt.Printf("median: %d\n", k) + _, err = f.WriteString(fmt.Sprintf("median: %d\n", k)) + if err != nil { + fmt.Println(err) + } + break + } + } + + for _, k := range arrCount { + count := p.arrItemCountMap[k] + if maxCount < count { + maxCount = count + mode = k + } + } + + fmt.Printf("mode: %d\n", mode) + _, err = f.WriteString(fmt.Sprintf("mode: %d\n", mode)) + if err != nil { + fmt.Println(err) + } + + storageIt := db.NewIterator(rawdb.SnapshotStoragePrefix, nil) + defer storageIt.Release() + + storageKeyCount := uint64(0) + for storageIt.Next() { + key := storageIt.Key() + if len(key) == (len(rawdb.SnapshotStoragePrefix) + 2*common.HashLength) { + storageKeyCount++ + } + } + fmt.Printf("storage key count: %d", storageKeyCount) + _, err = f.WriteString(fmt.Sprintf("storage key count: %d\n", storageKeyCount)) + if err != nil { + fmt.Println(err) + } + + return nil +} + +func traversalContract(snap snapshot.Snapshot, contractAddress common.Hash, keySet []common.Hash, mapLock *sync.Mutex, arrItemCountMap map[int]uint64) (int, int, int) { + emptyKeyCount := 0 + varCount := 0 + arrayCount := 0 + arrayVarCount := 0 + + for _, storageHash := range keySet { + enc, encErr := snap.Storage(contractAddress, storageHash) + if encErr != nil { + log.Info(fmt.Sprintf("key: %s, storage error: %v", storageHash.String(), encErr)) + + continue + } + if len(enc) > 0 { + emptyKeyCount = 0 + if isArr, arrLen := isArray(snap, contractAddress, storageHash); isArr { + arrayVarCount += arrLen + arrayCount++ + mapLock.Lock() + arrItemCountMap[arrLen]++ + mapLock.Unlock() + } else { + varCount++ + } + + } else { + emptyKeyCount++ + } + if emptyKeyCount == 100 { + break + } + } + //log.Info("array count:", arrayCount, "array item count", arrayVarCount, "var count", varCount) + + return arrayCount, arrayVarCount, varCount +} + +func isArray(snap snapshot.Snapshot, contractAddress common.Hash, slotIdx common.Hash) (result bool, slotLen int) { + emptyValue := 0 + hasValue := false + errTime := 0 + for i := 0; ; i++ { + if errTime > 100 { + break + } + idx := common.BigToHash(big.NewInt(0).Add(slotIdx.Big(), big.NewInt(int64(i)))) + enc, err := snap.Storage(contractAddress, crypto.Keccak256Hash(idx.Bytes())) + if err != nil { + fmt.Printf("storage error:%s", err.Error()) + errTime++ + continue + } + if len(enc) == 0 { + emptyValue++ + } else { + emptyValue = 0 + hasValue = true + slotLen++ + } + if emptyValue == 10 { + break + } + } + result = hasValue + return +} diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 8707840692..84135036ca 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1145,6 +1145,13 @@ Please note that --` + MetricsHTTPFlag.Name + ` must be set to start the server. Value: params.DefaultExtraReserveForBlobRequests, Category: flags.MiscCategory, } + + // cpu count + CPUCountFlag = &cli.Int64Flag{ + Name: "cpu", + Usage: "core number", + Value: 4, + } ) var (