feat: data analysis

cmd/geth/pool.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/state/snapshot"
+	"sync"
+	"sync/atomic"
+)
+
+type routinePool struct {
+	wg        sync.WaitGroup
+	taskQueue chan common.Hash
+
+	arrCount     atomic.Uint64
+	varCount     atomic.Uint64
+	arrItemCount atomic.Uint64
+}
+
+func NewRoutinePool(size int64, snap snapshot.Snapshot, keySet []common.Hash) *routinePool {
+	pool := &routinePool{
+		taskQueue:    make(chan common.Hash),
+		arrCount:     atomic.Uint64{},
+		varCount:     atomic.Uint64{},
+		arrItemCount: atomic.Uint64{},
+	}
+
+	for i := int64(0); i < size; i++ {
+		go pool.worker(snap, keySet)
+	}
+	return pool
+}
+
+func (pool *routinePool) AddTask(addrHash common.Hash) {
+	pool.wg.Add(1)
+	pool.taskQueue <- addrHash
+}
+
+func (pool *routinePool) worker(snap snapshot.Snapshot, keySet []common.Hash) {
+	for addrHash := range pool.taskQueue {
+		arrCount, arrItemCount, varCount := traversalContract(snap, addrHash, keySet)
+		pool.arrCount.Add(uint64(arrCount))
+		pool.arrItemCount.Add(uint64(arrItemCount))
+		pool.varCount.Add(uint64(varCount))
+		pool.wg.Done()
+	}
+}
+
+// Wait
+func (pool *routinePool) Wait() {
+	pool.wg.Wait()
+}

cmd/geth/snapshot.go

@@ -21,6 +21,8 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/ethereum/go-ethereum/common"
+	"math/big"
 	"os"
 	"path/filepath"
 	"strings"
@@ -29,7 +31,6 @@ import (
 	"github.com/prometheus/tsdb/fileutil"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
-	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
@@ -220,6 +221,19 @@ block is used.
 				Description: `
 The export-preimages command exports hash preimages to a flat file, in exactly
 the expected order for the overlay tree migration.
+`,
+			},
+			{
+				Name:   "inspect-storage",
+				Usage:  "Statistical variable data for all contracts",
+				Action: inspectStorage,
+				Flags: flags.Merge([]cli.Flag{
+					utils.CPUCountFlag,
+				}, utils.DatabaseFlags),
+				Description: `
+This command collects the variable information of all contracts from snapshot.
+
+The argument is interpreted as the number of cpus.
 `,
 			},
 		},
@@ -1020,3 +1034,155 @@ func checkAccount(ctx *cli.Context) error {
 	log.Info("Checked the snapshot journalled storage", "time", common.PrettyDuration(time.Since(start)))
 	return nil
 }
+
+func inspectStorage(ctx *cli.Context) error {
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+
+	_, db, root, err := parseDumpConfig(ctx, stack)
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+	tdb := utils.MakeTrieDatabase(ctx, stack, db, false, true, false)
+	defer tdb.Close()
+
+	snapConfig := snapshot.Config{
+		CacheSize:  256,
+		Recovery:   false,
+		NoBuild:    true,
+		AsyncBuild: false,
+	}
+	triesInMemory := ctx.Uint64(utils.TriesInMemoryFlag.Name)
+	snaptree, err := snapshot.New(snapConfig, db, tdb, root, int(triesInMemory), false)
+	snap := snaptree.Snapshot(root)
+	if err != nil {
+		return err
+	}
+
+	// init key
+	keySet := make([]common.Hash, 0, 1000)
+	for i := 0; i < 1000; i++ {
+		key := common.BigToHash(big.NewInt(int64(i)))
+		keySet = append(keySet, crypto.Keccak256Hash(key.Bytes()))
+	}
+
+	cpuCount := ctx.Int64(utils.CPUCountFlag.Name)
+	p := NewRoutinePool(cpuCount*100, snap, keySet)
+
+	it := db.NewIterator(rawdb.SnapshotAccountPrefix, nil)
+	defer it.Release()
+
+	// Inspect key-value database first.
+	idx := uint64(0)
+	for it.Next() {
+		key := it.Key()
+		switch {
+		case bytes.HasPrefix(key, rawdb.SnapshotAccountPrefix) && len(key) == (len(rawdb.SnapshotAccountPrefix)+common.HashLength):
+			var account types.SlimAccount
+			err := rlp.DecodeBytes(it.Value(), &account)
+			if err != nil {
+				log.Error("failed to DecodeBytes", "addrHash", key[1:])
+				continue
+			}
+			if bytes.Equal(account.CodeHash, types.EmptyCodeHash.Bytes()) {
+				continue
+			}
+			fmt.Printf("add task: %d\n", idx)
+			idx++
+			p.AddTask(common.BytesToHash(key[1:]))
+		}
+	}
+
+	p.Wait()
+	fmt.Println("all done")
+	fmt.Println("var count", p.varCount.Load())
+	fmt.Println("arr count", p.arrCount.Load())
+	fmt.Println("arr item count", p.arrItemCount.Load())
+
+	f, err := os.Create("result.txt")
+	if err != nil {
+		fmt.Println(err)
+	}
+
+	_, err = f.WriteString(fmt.Sprintf("var count: %d\n", p.varCount.Load()))
+	if err != nil {
+		fmt.Println(err)
+	}
+
+	_, err = f.WriteString(fmt.Sprintf("arr count: %d\n", p.arrCount.Load()))
+	if err != nil {
+		fmt.Println(err)
+	}
+
+	_, err = f.WriteString(fmt.Sprintf("arr item count: %d\n", p.arrItemCount.Load()))
+	if err != nil {
+		fmt.Println(err)
+	}
+
+	return nil
+}
+
+func traversalContract(snap snapshot.Snapshot, contractAddress common.Hash, keySet []common.Hash) (int, int, int) {
+	emptyKeyCount := 0
+	varCount := 0
+	arrayCount := 0
+	arrayVarCount := 0
+
+	for _, storageHash := range keySet {
+		enc, encErr := snap.Storage(contractAddress, storageHash)
+		if encErr != nil {
+			log.Info(fmt.Sprintf("key: %s, storage error: %v", storageHash.String(), encErr))
+
+			continue
+		}
+		if len(enc) > 0 {
+			emptyKeyCount = 0
+			if isArr, arrLen := isArray(snap, contractAddress, storageHash); isArr {
+				arrayVarCount += arrLen
+				arrayCount++
+			} else {
+				varCount++
+			}
+
+		} else {
+			emptyKeyCount++
+		}
+		if emptyKeyCount == 20 {
+			break
+		}
+	}
+	//log.Info("array count:", arrayCount, "array item count", arrayVarCount, "var count", varCount)
+
+	return arrayCount, arrayVarCount, varCount
+}
+
+func isArray(snap snapshot.Snapshot, contractAddress common.Hash, slotIdx common.Hash) (result bool, slotLen int) {
+	emptyValue := 0
+	hasValue := false
+	errTime := 0
+	for i := 0; ; i++ {
+		if errTime > 100 {
+			break
+		}
+		idx := common.BigToHash(big.NewInt(0).Add(slotIdx.Big(), big.NewInt(int64(i))))
+		enc, err := snap.Storage(contractAddress, crypto.Keccak256Hash(idx.Bytes()))
+		if err != nil {
+			fmt.Printf("storage error:%s", err.Error())
+			errTime++
+			continue
+		}
+		if len(enc) == 0 {
+			emptyValue++
+		} else {
+			emptyValue = 0
+			hasValue = true
+			slotLen++
+		}
+		if emptyValue == 10 {
+			break
+		}
+	}
+	result = hasValue
+	return
+}

cmd/utils/flags.go

@@ -1145,6 +1145,13 @@ Please note that --` + MetricsHTTPFlag.Name + ` must be set to start the server.
 		Value:    params.DefaultExtraReserveForBlobRequests,
 		Category: flags.MiscCategory,
 	}
+
+	// cpu count
+	CPUCountFlag = &cli.Int64Flag{
+		Name:  "cpu",
+		Usage: "core number",
+		Value: 4,
+	}
 )
 
 var (