Merge remote-tracking branch 'origin' into tobi/add-aws

Author: tobidae-cb

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/charmbracelet/huh v0.7.0
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
 	github.com/ethereum-optimism/optimism v1.13.3
-	github.com/ethereum/go-ethereum v1.15.11
+	github.com/ethereum/go-ethereum v1.16.0
 	github.com/holiman/uint256 v1.3.2
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/common v0.62.0

report/src/metricDefinitions.ts

@@ -1,5 +1,4 @@
 import { ChartConfig } from "./types"; // Import from types.ts
-
 export const CHART_CONFIG: Record<string, ChartConfig> = {
   "latency/send_txs": {
     type: "line",
@@ -137,4 +136,60 @@ export const CHART_CONFIG: Record<string, ChartConfig> = {
     description: "Shows the median gas per block",
     unit: "gas",
   },
+  reth_sync_execution_execution_duration: {
+    type: "line",
+    title: "Reth Sync Execution Duration",
+    description: "Shows the time taken for execution during reth sync",
+    unit: "s",
+  },
+  reth_sync_block_validation_state_root_duration: {
+    type: "line",
+    title: "Reth Sync Block Validation State Root Duration",
+    description:
+      "Shows the time taken for state root validation during reth sync",
+    unit: "s",
+  },
+  reth_op_rbuilder_block_built_success: {
+    type: "line",
+    title: "Reth OP RBuilder Block Built Success",
+    description: "Indicates whether the RBuilder successfully built a block",
+    unit: "count",
+  },
+  reth_op_rbuilder_flashblock_count_avg: {
+    type: "line",
+    title: "Reth OP RBuilder Flashblock Count",
+    description: "Shows the number of flashblocks built by RBuilder",
+    unit: "count",
+  },
+  reth_op_rbuilder_total_block_built_duration_avg: {
+    type: "line",
+    title: "Reth OP RBuilder Total Block Built Duration",
+    description: "Shows the total time taken to build a block by RBuilder",
+    unit: "s",
+  },
+  reth_op_rbuilder_flashblock_build_duration_avg: {
+    type: "line",
+    title: "Reth OP RBuilder Flashblock Build Duration",
+    description: "Shows the time taken to build a flashblock by RBuilder",
+    unit: "s",
+  },
+  reth_op_rbuilder_state_root_calculation_duration_avg: {
+    type: "line",
+    title: "Reth OP RBuilder State Root Calculation Duration",
+    description: "Shows the time taken to calculate the state root by RBuilder",
+    unit: "s",
+  },
+  reth_op_rbuilder_sequencer_tx_duration_avg: {
+    type: "line",
+    title: "Reth OP RBuilder Sequencer Tx Duration",
+    description: "Shows the time taken for sequencer transactions in RBuilder",
+    unit: "s",
+  },
+  reth_op_rbuilder_payload_tx_simulation_duration_avg: {
+    type: "line",
+    title: "Reth OP RBuilder Payload Tx Simulation Duration",
+    description:
+      "Shows the time taken for payload transaction simulation in RBuilder",
+    unit: "s",
+  },
 };

report/src/types.ts

@@ -40,7 +40,16 @@ export interface ChartConfig {
   title: string;
   description: string;
   type: "line";
-  unit?: "ns" | "us" | "ms" | "s" | "bytes" | "gas" | "count" | "gas/s"; // Add 'gas/s', ensure 's' is present
+  unit?:
+    | "ns"
+    | "us"
+    | "ms"
+    | "s"
+    | "bytes"
+    | "gas"
+    | "count"
+    | "gas/s"
+    | "blocks"; // Add 'gas/s', ensure 's' is present
 }
 
 export interface BenchmarkRun {

runner/benchmark/portmanager/ports.go

@@ -0,0 +1,58 @@
+package portmanager
+
+import (
+	"fmt"
+	"net"
+	"time"
+)
+
+type PortPurpose uint64
+
+const (
+	ELPortPurpose PortPurpose = iota
+	AuthELPortPurpose
+	ELMetricsPortPurpose
+	BuilderMetricsPortPurpose
+)
+
+type PortManager interface {
+	AcquirePort(nodeType string, purpose PortPurpose) uint64
+	ReleasePort(portNumber uint64)
+}
+
+type portManager struct {
+	// ports is a map of node type to a map of port purpose to port number.
+	ports map[uint64]struct{}
+}
+
+func NewPortManager() PortManager {
+	return &portManager{
+		ports: make(map[uint64]struct{}),
+	}
+}
+
+func (p *portManager) AcquirePort(nodeType string, purpose PortPurpose) uint64 {
+	// find the next available port number
+	for port := uint64(10000); port < 65535; port++ {
+		if _, exists := p.ports[port]; !exists {
+			// check if port is already in use
+			if conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), 100*time.Millisecond); err == nil {
+				_ = conn.Close()
+				continue // port is in use, try the next one
+			}
+
+			p.ports[port] = struct{}{}
+			return port
+		}
+	}
+
+	panic(fmt.Sprintf("no available ports for node type %s and purpose %d", nodeType, purpose))
+}
+
+func (p *portManager) ReleasePort(portNumber uint64) {
+	if _, exists := p.ports[portNumber]; !exists {
+		return
+	}
+
+	delete(p.ports, portNumber)
+}

runner/benchmark/result_metadata.go

@@ -1,28 +1,16 @@
 package benchmark
 
-import "time"
+import (
+	"time"
 
-type SequencerKeyMetrics struct {
-	CommonKeyMetrics
-	AverageFCULatency        float64 `json:"forkChoiceUpdated"`
-	AverageGetPayloadLatency float64 `json:"getPayload"`
-	AverageSendTxsLatency    float64 `json:"sendTxs"`
-}
-
-type ValidatorKeyMetrics struct {
-	CommonKeyMetrics
-	AverageNewPayloadLatency float64 `json:"newPayload"`
-}
-
-type CommonKeyMetrics struct {
-	AverageGasPerSecond float64 `json:"gasPerSecond"`
-}
+	"github.com/base/base-bench/runner/network/types"
+)
 
 type RunResult struct {
-	Success          bool                `json:"success"`
-	Complete         bool                `json:"complete"`
-	SequencerMetrics SequencerKeyMetrics `json:"sequencerMetrics"`
-	ValidatorMetrics ValidatorKeyMetrics `json:"validatorMetrics"`
+	Success          bool                      `json:"success"`
+	Complete         bool                      `json:"complete"`
+	SequencerMetrics types.SequencerKeyMetrics `json:"sequencerMetrics"`
+	ValidatorMetrics types.ValidatorKeyMetrics `json:"validatorMetrics"`
 }
 
 // Run is the output JSON metadata for a benchmark run.

runner/clients/proxy/proxy.go renamed to runner/clients/common/proxy/proxy.go

@@ -8,7 +8,6 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
-	"strconv"
 	"strings"
 	"time"
 
@@ -18,9 +17,11 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/pkg/errors"
 
+	"github.com/base/base-bench/runner/benchmark/portmanager"
 	"github.com/base/base-bench/runner/clients/common"
 	"github.com/base/base-bench/runner/clients/types"
 	"github.com/base/base-bench/runner/config"
+	"github.com/base/base-bench/runner/metrics"
 	"github.com/ethereum/go-ethereum/ethclient"
 )
 
@@ -34,18 +35,30 @@ type GethClient struct {
 	authClient client.RPC
 	process    *exec.Cmd
 
+	ports       portmanager.PortManager
+	metricsPort uint64
+	rpcPort     uint64
+	authRPCPort uint64
+
 	stdout io.WriteCloser
 	stderr io.WriteCloser
+
+	metricsCollector metrics.Collector
 }
 
 // NewGethClient creates a new client for geth.
-func NewGethClient(logger log.Logger, options *config.InternalClientOptions) types.ExecutionClient {
+func NewGethClient(logger log.Logger, options *config.InternalClientOptions, ports portmanager.PortManager) types.ExecutionClient {
 	return &GethClient{
 		logger:  logger,
 		options: options,
+		ports:   ports,
 	}
 }
 
+func (g *GethClient) MetricsCollector() metrics.Collector {
+	return g.metricsCollector
+}
+
 // Run runs the geth client with the given runtime config.
 func (g *GethClient) Run(ctx context.Context, cfg *types.RuntimeConfig) error {
 
@@ -78,17 +91,20 @@ func (g *GethClient) Run(ctx context.Context, cfg *types.RuntimeConfig) error {
 		}
 	}
 
-
 	args = make([]string, 0)
 	args = append(args, "--datadir", g.options.DataDirPath)
 	args = append(args, "--http")
 
+	g.rpcPort = g.ports.AcquirePort("geth", portmanager.ELPortPurpose)
+	g.authRPCPort = g.ports.AcquirePort("geth", portmanager.AuthELPortPurpose)
+	g.metricsPort = g.ports.AcquirePort("geth", portmanager.ELMetricsPortPurpose)
+
 	// TODO: allocate these dynamically eventually
-	args = append(args, "--http.port", strconv.Itoa(g.options.GethHttpPort))
-	args = append(args, "--authrpc.port", strconv.Itoa(g.options.GethAuthRpcPort))
+	args = append(args, "--http.port", fmt.Sprintf("%d", g.rpcPort))
+	args = append(args, "--authrpc.port", fmt.Sprintf("%d", g.authRPCPort))
 	args = append(args, "--metrics")
 	args = append(args, "--metrics.addr", "localhost")
-	args = append(args, "--metrics.port", strconv.Itoa(g.options.GethMetricsPort))
+	args = append(args, "--metrics.port", fmt.Sprintf("%d", g.metricsPort))
 
 	// Set mempool size to 100x default
 	args = append(args, "--txpool.globalslots", "10000000")
@@ -136,7 +152,7 @@ func (g *GethClient) Run(ctx context.Context, cfg *types.RuntimeConfig) error {
 		return err
 	}
 
-	g.clientURL = fmt.Sprintf("http://127.0.0.1:%d", g.options.GethHttpPort)
+	g.clientURL = fmt.Sprintf("http://127.0.0.1:%d", g.rpcPort)
 	rpcClient, err := rpc.DialOptions(ctx, g.clientURL, rpc.WithHTTPClient(&http.Client{
 		Timeout: 30 * time.Second,
 	}))
@@ -145,13 +161,14 @@ func (g *GethClient) Run(ctx context.Context, cfg *types.RuntimeConfig) error {
 	}
 
 	g.client = ethclient.NewClient(rpcClient)
+	g.metricsCollector = newMetricsCollector(g.logger, g.client, int(g.metricsPort))
 
 	err = common.WaitForRPC(ctx, g.client)
 	if err != nil {
 		return errors.Wrap(err, "geth rpc failed to start")
 	}
 
-	l2Node, err := client.NewRPC(ctx, g.logger, fmt.Sprintf("http://127.0.0.1:%d", g.options.GethAuthRpcPort), client.WithGethRPCOptions(rpc.WithHTTPAuth(node.NewJWTAuth(jwtSecret))), client.WithCallTimeout(30*time.Second))
+	l2Node, err := client.NewRPC(ctx, g.logger, fmt.Sprintf("http://127.0.0.1:%d", g.authRPCPort), client.WithGethRPCOptions(rpc.WithHTTPAuth(node.NewJWTAuth(jwtSecret))), client.WithCallTimeout(30*time.Second))
 	if err != nil {
 		return err
 	}
@@ -181,6 +198,10 @@ func (g *GethClient) Stop() {
 	_ = g.stdout.Close()
 	_ = g.stderr.Close()
 
+	g.ports.ReleasePort(g.rpcPort)
+	g.ports.ReleasePort(g.authRPCPort)
+	g.ports.ReleasePort(g.metricsPort)
+
 	g.stdout = nil
 	g.stderr = nil
 	g.process = nil
@@ -202,5 +223,5 @@ func (g *GethClient) AuthClient() client.RPC {
 }
 
 func (r *GethClient) MetricsPort() int {
-	return r.options.GethMetricsPort
+	return int(r.metricsPort)
 }

runner/clients/geth/client.go

@@ -1,32 +1,33 @@
-package metrics
+package geth
 
 import (
 	"context"
 	"encoding/json"
 	"fmt"
 	"net/http"
 
+	"github.com/base/base-bench/runner/metrics"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/log"
 )
 
-type GethMetricsCollector struct {
+type metricsCollector struct {
 	log         log.Logger
 	client      *ethclient.Client
-	metrics     []BlockMetrics
+	metrics     []metrics.BlockMetrics
 	metricsPort int
 }
 
-func NewGethMetricsCollector(log log.Logger, client *ethclient.Client, metricsPort int) *GethMetricsCollector {
-	return &GethMetricsCollector{
+func newMetricsCollector(log log.Logger, client *ethclient.Client, metricsPort int) metrics.Collector {
+	return &metricsCollector{
 		log:         log,
 		client:      client,
 		metricsPort: metricsPort,
-		metrics:     make([]BlockMetrics, 0),
+		metrics:     make([]metrics.BlockMetrics, 0),
 	}
 }
 
-func (g *GethMetricsCollector) GetMetricTypes() map[string]bool {
+func (g *metricsCollector) GetMetricTypes() map[string]bool {
 	return map[string]bool{
 		"chain/account/reads.50-percentile":    true,
 		"chain/execution.50-percentile":        true,
@@ -45,15 +46,15 @@ func (g *GethMetricsCollector) GetMetricTypes() map[string]bool {
 	}
 }
 
-func (g *GethMetricsCollector) GetMetricsEndpoint() string {
+func (g *metricsCollector) GetMetricsEndpoint() string {
 	return fmt.Sprintf("http://127.0.0.1:%d/debug/metrics", g.metricsPort)
 }
 
-func (g *GethMetricsCollector) GetMetrics() []BlockMetrics {
+func (g *metricsCollector) GetMetrics() []metrics.BlockMetrics {
 	return g.metrics
 }
 
-func (g *GethMetricsCollector) Collect(ctx context.Context, metrics *BlockMetrics) error {
+func (g *metricsCollector) Collect(ctx context.Context, metrics *metrics.BlockMetrics) error {
 	resp, err := http.Get(g.GetMetricsEndpoint())
 	if err != nil {
 		return fmt.Errorf("failed to get metrics: %w", err)
@@ -77,6 +78,6 @@ func (g *GethMetricsCollector) Collect(ctx context.Context, metrics *BlockMetric
 		}
 	}
 
-	g.metrics = append(g.metrics, *metrics)
+	g.metrics = append(g.metrics, *metrics.Copy())
 	return nil
 }

runner/metrics/geth_metrics.go renamed to runner/clients/geth/metrics.go

@@ -3,9 +3,6 @@ package options
 // GethOptions are options that are specified at runtime for the geth client.
 type GethOptions struct {
 	// GethBin is the path to the geth binary to use in tests.
-	GethBin         string
-	GethHttpPort    int
-	GethAuthRpcPort int
-	GethMetricsPort int
-	SkipInit        bool
+	GethBin  string
+	SkipInit bool
 }