Skip to content

Commit 2d7ae0c

Browse files
tobidae-cbtobidae-cb
committed
Merge remote-tracking branch 'origin' into tobi/add-aws
2 parents 9457e53 + 78dd71d commit 2d7ae0c

File tree

11 files changed

+347
-37
lines changed

11 files changed

+347
-37
lines changed

‎.github/scripts/run-example-benchmarks.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ for config in "${BENCHMARK_CONFIGS[@]}"; do
2020
echo "Running $config"
2121
# each config will add on to the same output directory
2222
go run benchmark/cmd/main.go \
23-
--log.level info \
23+
--log.level debug \
2424
run \
2525
--config $config \
2626
--root-dir $TEMP_DIR/data-dir \

‎.github/workflows/build.yaml‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,9 @@ jobs:
6060
6161
basic-benchmarks:
6262
runs-on: ubuntu-latest
63+
env:
64+
RETH_VERSION: 9d56da53ec0ad60e229456a0c70b338501d923a5
65+
GETH_VERSION: 4bc345b22fbee14d3162becd197373a9565b7c6d
6366
steps:
6467
- name: Harden the runner (Audit all outbound calls)
6568
uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1
@@ -88,7 +91,7 @@ jobs:
8891
id: cache-bin
8992
with:
9093
path: ${{ runner.temp }}/bin
91-
key: ${{ runner.os }}-binaries
94+
key: ${{ runner.os }}-binaries-reth-${{ env.RETH_VERSION }}-geth-${{ env.GETH_VERSION }}
9295

9396
- name: Build Contracts
9497
run: |
@@ -100,7 +103,7 @@ jobs:
100103
mkdir -p ${{ runner.temp }}/bin
101104
102105
git clone https://github.com/paradigmxyz/reth
103-
git -C reth checkout --force fad870e3508adcc150faa4554111368e0e16f43b
106+
git -C reth checkout --force ${{ env.RETH_VERSION }}
104107
105108
pushd reth
106109
cargo build --features asm-keccak,jemalloc --profile release --bin op-reth --manifest-path crates/optimism/bin/Cargo.toml
@@ -109,7 +112,7 @@ jobs:
109112
chmod +x ${{ runner.temp }}/bin/reth
110113
111114
git clone https://github.com/ethereum-optimism/op-geth
112-
git -C op-geth checkout --force 4bc345b22fbee14d3162becd197373a9565b7c6d
115+
git -C op-geth checkout --force ${{ env.GETH_VERSION }}
113116
114117
pushd op-geth
115118
make geth

‎.github/workflows/examples.yaml‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@ on:
77
jobs:
88
example-benchmarks:
99
runs-on: ubuntu-latest
10+
env:
11+
RETH_VERSION: 9d56da53ec0ad60e229456a0c70b338501d923a5
12+
GETH_VERSION: 4bc345b22fbee14d3162becd197373a9565b7c6d
1013

1114
steps:
1215
- name: Harden the runner (Audit all outbound calls)
@@ -35,7 +38,7 @@ jobs:
3538
id: cache-bin
3639
with:
3740
path: ${{ runner.temp }}/bin
38-
key: ${{ runner.os }}-binaries
41+
key: ${{ runner.os }}-binaries-reth-${{ env.RETH_VERSION }}-geth-${{ env.GETH_VERSION }}
3942

4043
- name: Install Foundry
4144
uses: foundry-rs/foundry-toolchain@82dee4ba654bd2146511f85f0d013af94670c4de # v1.4.0
@@ -50,7 +53,7 @@ jobs:
5053
mkdir -p ${{ runner.temp }}/bin
5154
5255
git clone https://github.com/paradigmxyz/reth
53-
git -C reth checkout --force fad870e3508adcc150faa4554111368e0e16f43b
56+
git -C reth checkout --force ${{ env.RETH_VERSION }}
5457
5558
pushd reth
5659
cargo build --features asm-keccak,jemalloc --profile release --bin op-reth --manifest-path crates/optimism/bin/Cargo.toml
@@ -59,7 +62,7 @@ jobs:
5962
chmod +x ${{ runner.temp }}/bin/reth
6063
6164
git clone https://github.com/ethereum-optimism/op-geth
62-
git -C op-geth checkout --force 4bc345b22fbee14d3162becd197373a9565b7c6d
65+
git -C op-geth checkout --force ${{ env.GETH_VERSION }}
6366
6467
pushd op-geth
6568
make geth
@@ -86,6 +89,13 @@ jobs:
8689
npm run build
8790
popd
8891
92+
- name: Upload Output
93+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
94+
if: always()
95+
with:
96+
name: output
97+
path: ${{ runner.temp }}/output/
98+
8999
# upload to github actions artifact
90100
- name: Upload Report
91101
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02

‎.github/workflows/public-benchmarks.yaml‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@ on:
77
jobs:
88
basic-benchmarks:
99
runs-on: ubuntu-latest
10+
env:
11+
RETH_VERSION: 9d56da53ec0ad60e229456a0c70b338501d923a5
12+
GETH_VERSION: 4bc345b22fbee14d3162becd197373a9565b7c6d
1013

1114
steps:
1215
- name: Harden the runner (Audit all outbound calls)
@@ -35,7 +38,7 @@ jobs:
3538
id: cache-bin
3639
with:
3740
path: ${{ runner.temp }}/bin
38-
key: ${{ runner.os }}-binaries
41+
key: ${{ runner.os }}-binaries-reth-${{ env.RETH_VERSION }}-geth-${{ env.GETH_VERSION }}
3942

4043
- name: Install Foundry
4144
uses: foundry-rs/foundry-toolchain@82dee4ba654bd2146511f85f0d013af94670c4de # v1.4.0
@@ -50,7 +53,7 @@ jobs:
5053
mkdir -p ${{ runner.temp }}/bin
5154
5255
git clone https://github.com/paradigmxyz/reth
53-
git -C reth checkout --force fad870e3508adcc150faa4554111368e0e16f43b
56+
git -C reth checkout --force ${{ env.RETH_VERSION }}
5457
5558
pushd reth
5659
cargo build --features asm-keccak,jemalloc --profile release --bin op-reth --manifest-path crates/optimism/bin/Cargo.toml
@@ -59,7 +62,7 @@ jobs:
5962
chmod +x ${{ runner.temp }}/bin/reth
6063
6164
git clone https://github.com/ethereum-optimism/op-geth
62-
git -C op-geth checkout --force 4bc345b22fbee14d3162becd197373a9565b7c6d
65+
git -C op-geth checkout --force ${{ env.GETH_VERSION }}
6366
6467
pushd op-geth
6568
make geth

‎Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ RUN git clone https://github.com/paradigmxyz/reth.git && \
3333
git clone https://github.com/ethereum-optimism/op-geth.git && \
3434
git clone https://github.com/base/benchmark.git
3535

36-
RUN git -C reth checkout --force fad870e3508adcc150faa4554111368e0e16f43b && \
36+
RUN git -C reth checkout --force 9d56da53ec0ad60e229456a0c70b338501d923a5 && \
3737
git -C op-geth checkout --force 4bc345b22fbee14d3162becd197373a9565b7c6d
3838

3939
# Stage 2: Build reth

‎report/src/components/LineChart.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -426,7 +426,7 @@ const LineChart: React.FC<LineChartProps> = ({
426426

427427
// Add error threshold line (red)
428428
if (thresholds.error && thresholds.error[metricKey] !== undefined) {
429-
let errorY = y(thresholds.error[metricKey]);
429+
const errorY = y(thresholds.error[metricKey]);
430430
svg
431431
.append("line")
432432
.attr("class", "threshold-line error")

‎report/src/metricDefinitions.ts‎

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -155,41 +155,59 @@ export const CHART_CONFIG: Record<string, ChartConfig> = {
155155
description: "Indicates whether the RBuilder successfully built a block",
156156
unit: "count",
157157
},
158-
reth_op_rbuilder_flashblock_count_avg: {
158+
reth_op_rbuilder_flashblock_count: {
159159
type: "line",
160160
title: "Reth OP RBuilder Flashblock Count",
161161
description: "Shows the number of flashblocks built by RBuilder",
162162
unit: "count",
163163
},
164-
reth_op_rbuilder_total_block_built_duration_avg: {
164+
reth_op_rbuilder_total_block_built_duration: {
165165
type: "line",
166166
title: "Reth OP RBuilder Total Block Built Duration",
167167
description: "Shows the total time taken to build a block by RBuilder",
168168
unit: "s",
169169
},
170-
reth_op_rbuilder_flashblock_build_duration_avg: {
170+
reth_op_rbuilder_flashblock_build_duration: {
171171
type: "line",
172172
title: "Reth OP RBuilder Flashblock Build Duration",
173173
description: "Shows the time taken to build a flashblock by RBuilder",
174174
unit: "s",
175175
},
176-
reth_op_rbuilder_state_root_calculation_duration_avg: {
176+
reth_op_rbuilder_state_root_calculation_duration: {
177177
type: "line",
178178
title: "Reth OP RBuilder State Root Calculation Duration",
179179
description: "Shows the time taken to calculate the state root by RBuilder",
180180
unit: "s",
181181
},
182-
reth_op_rbuilder_sequencer_tx_duration_avg: {
182+
reth_op_rbuilder_sequencer_tx_duration: {
183183
type: "line",
184184
title: "Reth OP RBuilder Sequencer Tx Duration",
185185
description: "Shows the time taken for sequencer transactions in RBuilder",
186186
unit: "s",
187187
},
188-
reth_op_rbuilder_payload_tx_simulation_duration_avg: {
188+
reth_op_rbuilder_payload_tx_simulation_duration: {
189189
type: "line",
190190
title: "Reth OP RBuilder Payload Tx Simulation Duration",
191191
description:
192192
"Shows the time taken for payload transaction simulation in RBuilder",
193193
unit: "s",
194194
},
195+
reth_sync_state_provider_total_storage_fetch_latency: {
196+
type: "line",
197+
title: "Validator Storage Load Latency",
198+
description: "Shows the 90th percentile latency for storage slot loads",
199+
unit: "s",
200+
},
201+
reth_sync_state_provider_total_code_fetch_latency: {
202+
type: "line",
203+
title: "Validator Code Load Latency",
204+
description: "Shows the 90th percentile latency for code loads",
205+
unit: "s",
206+
},
207+
reth_sync_state_provider_total_account_fetch_latency: {
208+
type: "line",
209+
title: "Validator Account Load Latency",
210+
description: "Shows the 90th percentile latency for account loads",
211+
unit: "s",
212+
},
195213
};

‎runner/clients/reth/client.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ func (r *RethClient) Run(ctx context.Context, cfg *types.RuntimeConfig) error {
8989
args = append(args, "--authrpc.port", fmt.Sprintf("%d", r.authRPCPort))
9090
args = append(args, "--authrpc.jwtsecret", r.options.JWTSecretPath)
9191
args = append(args, "--metrics", fmt.Sprintf("%d", r.metricsPort))
92+
args = append(args, "--engine.state-provider-metrics")
9293
args = append(args, "-vvv")
9394

9495
args = append(args, cfg.Args...)
@@ -163,7 +164,7 @@ func (r *RethClient) Run(ctx context.Context, cfg *types.RuntimeConfig) error {
163164

164165
err = common.WaitForRPC(ctx, r.client)
165166
if err != nil {
166-
return errors.Wrap(err, "geth rpc failed to start")
167+
return errors.Wrap(err, "reth rpc failed to start")
167168
}
168169

169170
l2Node, err := client.NewRPC(ctx, r.logger, fmt.Sprintf("http://127.0.0.1:%d", r.authRPCPort), client.WithGethRPCOptions(rpc.WithHTTPAuth(node.NewJWTAuth(jwtSecret))), client.WithCallTimeout(240*time.Second))

‎runner/clients/reth/metrics.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,14 @@ func (r *metricsCollector) GetMetrics() []metrics.BlockMetrics {
3939

4040
func (r *metricsCollector) GetMetricTypes() map[string]bool {
4141
return map[string]bool{
42-
"reth_sync_execution_execution_duration": true,
43-
"reth_sync_block_validation_state_root_duration": true,
42+
"reth_sync_execution_execution_duration": true,
43+
"reth_sync_block_validation_state_root_duration": true,
44+
"reth_sync_state_provider_storage_fetch_latency": true,
45+
"reth_sync_state_provider_account_fetch_latency": true,
46+
"reth_sync_state_provider_code_fetch_latency": true,
47+
"reth_sync_state_provider_total_storage_fetch_latency": true,
48+
"reth_sync_state_provider_total_account_fetch_latency": true,
49+
"reth_sync_state_provider_total_code_fetch_latency": true,
4450
}
4551
}
4652

‎runner/metrics/metrics_interface.go‎

Lines changed: 33 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"encoding/json"
66
"fmt"
77
"maps"
8+
"math"
89
"os"
910
"path"
1011
"time"
@@ -20,12 +21,14 @@ type Collector interface {
2021
type BlockMetrics struct {
2122
BlockNumber uint64
2223
Timestamp time.Time
24+
prevMetrics map[string]*io_prometheus_client.Metric
2325
ExecutionMetrics map[string]interface{}
2426
}
2527

2628
func NewBlockMetrics() *BlockMetrics {
2729
return &BlockMetrics{
2830
BlockNumber: 0,
31+
prevMetrics: make(map[string]*io_prometheus_client.Metric),
2932
ExecutionMetrics: make(map[string]interface{}),
3033
Timestamp: time.Now(),
3134
}
@@ -37,20 +40,22 @@ func (m *BlockMetrics) SetBlockNumber(blockNumber uint64) {
3740

3841
func (m *BlockMetrics) Copy() *BlockMetrics {
3942
newMetrics := make(map[string]interface{})
43+
newPrevMetrics := make(map[string]*io_prometheus_client.Metric)
4044
maps.Copy(newMetrics, m.ExecutionMetrics)
45+
maps.Copy(newPrevMetrics, m.prevMetrics)
4146
return &BlockMetrics{
4247
BlockNumber: m.BlockNumber,
48+
prevMetrics: newPrevMetrics,
4349
ExecutionMetrics: newMetrics,
4450
Timestamp: m.Timestamp,
4551
}
4652
}
4753

4854
func (m *BlockMetrics) UpdatePrometheusMetric(name string, value *io_prometheus_client.Metric) error {
4955
if value.Histogram != nil {
50-
avgName := name + "_avg"
5156
// get the average change in sum divided by the average change in count
5257
prevSum := 0.0
53-
prevValue, ok := m.ExecutionMetrics[name].(*io_prometheus_client.Metric)
58+
prevValue, ok := m.prevMetrics[name]
5459
if !ok {
5560
prevValue = nil
5661
}
@@ -64,6 +69,8 @@ func (m *BlockMetrics) UpdatePrometheusMetric(name string, value *io_prometheus_
6469
sum = *value.Histogram.SampleSum
6570
}
6671
prevCount := 0.0
72+
m.prevMetrics[name] = value
73+
6774
if prevValue != nil {
6875
if prevValue.Histogram.SampleCount != nil {
6976
prevCount = float64(*prevValue.Histogram.SampleCount)
@@ -73,22 +80,29 @@ func (m *BlockMetrics) UpdatePrometheusMetric(name string, value *io_prometheus_
7380
if value.Histogram.SampleCount != nil {
7481
count = float64(*value.Histogram.SampleCount)
7582
}
76-
if count == 0 {
77-
count = 1
83+
deltaCount := count - prevCount
84+
if deltaCount != 0 {
85+
averageChange := (sum - prevSum) / deltaCount
86+
87+
if !math.IsNaN(averageChange) {
88+
m.ExecutionMetrics[name] = averageChange
89+
}
7890
}
79-
averageChange := (sum - prevSum) / (count - prevCount)
80-
m.ExecutionMetrics[name] = value
81-
m.ExecutionMetrics[avgName] = averageChange
8291
} else if value.Gauge != nil {
83-
m.ExecutionMetrics[name] = *value.Gauge.Value
92+
if value.Gauge.Value != nil && !math.IsNaN(*value.Gauge.Value) {
93+
m.ExecutionMetrics[name] = *value.Gauge.Value
94+
}
95+
// NaN values and nil values are silently omitted
8496
} else if value.Counter != nil {
85-
m.ExecutionMetrics[name] = *value.Counter.Value
97+
if value.Counter.Value != nil && !math.IsNaN(*value.Counter.Value) {
98+
m.ExecutionMetrics[name] = *value.Counter.Value
99+
}
100+
// NaN values and nil values are silently omitted
86101
} else if value.Summary != nil {
87-
avgName := name + "_avg"
88102
// get the average change in sum divided by the average change in count
89103
prevSum := 0.0
90104

91-
prevValue, ok := m.ExecutionMetrics[name].(*io_prometheus_client.Metric)
105+
prevValue, ok := m.prevMetrics[name]
92106
if !ok {
93107
prevValue = nil
94108
}
@@ -111,12 +125,15 @@ func (m *BlockMetrics) UpdatePrometheusMetric(name string, value *io_prometheus_
111125
if value.Summary.SampleCount != nil {
112126
count = float64(*value.Summary.SampleCount)
113127
}
114-
if count == 0 {
115-
count = 1
128+
denom := count - prevCount
129+
m.prevMetrics[name] = value
130+
if denom != 0 {
131+
averageChange := (sum - prevSum) / denom
132+
if !math.IsNaN(averageChange) {
133+
m.ExecutionMetrics[name] = averageChange
134+
}
116135
}
117-
averageChange := (sum - prevSum) / (count - prevCount)
118-
m.ExecutionMetrics[name] = value
119-
m.ExecutionMetrics[avgName] = averageChange
136+
120137
} else {
121138
return fmt.Errorf("invalid metric type for %s: %#v", name, value)
122139
}

0 commit comments

Comments
 (0)