Skip to content

Commit 4cc53b3

Browse files
battlmonstrbattlmonstr
authored
datastore: move step converters to db (#2772)
1 parent a2e5ef8 commit 4cc53b3

29 files changed

+281
-197
lines changed

‎cmd/dev/staged_pipeline.cpp

+4-4
Original file line numberDiff line numberDiff line change
@@ -371,10 +371,10 @@ void debug_unwind(datastore::kvdb::EnvConfig& config, BlockNum height, uint32_t
371371
// Unwind has just set progress for pre-Execution stages back to unwind_point even if it is within the snapshots
372372
// We need to reset progress for such stages to the max block in snapshots to avoid database update on next start
373373
auto& blocks_repository = data_store.blocks_repository();
374-
db::stages::write_stage_progress(txn, db::stages::kHeadersKey, blocks_repository.max_block_available());
375-
db::stages::write_stage_progress(txn, db::stages::kBlockBodiesKey, blocks_repository.max_block_available());
376-
db::stages::write_stage_progress(txn, db::stages::kBlockHashesKey, blocks_repository.max_block_available());
377-
db::stages::write_stage_progress(txn, db::stages::kSendersKey, blocks_repository.max_block_available());
374+
db::stages::write_stage_progress(txn, db::stages::kHeadersKey, blocks_repository.max_timestamp_available());
375+
db::stages::write_stage_progress(txn, db::stages::kBlockBodiesKey, blocks_repository.max_timestamp_available());
376+
db::stages::write_stage_progress(txn, db::stages::kBlockHashesKey, blocks_repository.max_timestamp_available());
377+
db::stages::write_stage_progress(txn, db::stages::kSendersKey, blocks_repository.max_timestamp_available());
378378

379379
txn.commit_and_stop();
380380
}

‎silkworm/db/access_layer.cpp

+5-5
Original file line numberDiff line numberDiff line change
@@ -1064,20 +1064,20 @@ BlockNum DataModel::max_block_num() const {
10641064
}
10651065

10661066
// If none is found on db, then ask the snapshot repository (if any) for max block
1067-
return repository_.max_block_available();
1067+
return repository_.max_timestamp_available();
10681068
}
10691069

10701070
BlockNum DataModel::max_frozen_block_num() const {
10711071
// Ask the snapshot repository (if any) for max block
1072-
return repository_.max_block_available();
1072+
return repository_.max_timestamp_available();
10731073
}
10741074

10751075
std::optional<BlockHeader> DataModel::read_header(BlockNum block_num, HashAsArray hash) const {
10761076
return read_header(block_num, Hash(hash));
10771077
}
10781078

10791079
std::optional<BlockHeader> DataModel::read_header(BlockNum block_num, const Hash& hash) const {
1080-
BlockNum repository_max_block_num = repository_.max_block_available();
1080+
BlockNum repository_max_block_num = repository_.max_timestamp_available();
10811081
if ((repository_max_block_num > 0) && (block_num <= repository_max_block_num)) {
10821082
auto header = read_header_from_snapshot(block_num);
10831083
if (header && header->hash() == hash) { // reading using hash avoid this heavy hash calculation
@@ -1089,7 +1089,7 @@ std::optional<BlockHeader> DataModel::read_header(BlockNum block_num, const Hash
10891089
}
10901090

10911091
std::optional<BlockHeader> DataModel::read_header(BlockNum block_num) const {
1092-
BlockNum repository_max_block_num = repository_.max_block_available();
1092+
BlockNum repository_max_block_num = repository_.max_timestamp_available();
10931093
if ((repository_max_block_num > 0) && (block_num <= repository_max_block_num)) {
10941094
return read_header_from_snapshot(block_num);
10951095
}
@@ -1256,7 +1256,7 @@ void DataModel::for_last_n_headers(size_t n, absl::FunctionRef<void(BlockHeader)
12561256
return;
12571257
}
12581258

1259-
auto block_num_in_snapshots = repository_.max_block_available();
1259+
BlockNum block_num_in_snapshots = repository_.max_timestamp_available();
12601260

12611261
// We've reached the first header in db but still need to read more from snapshots
12621262
if (last_read_block_num_from_db > 0) {

‎silkworm/db/blocks/bodies/body_index.hpp

+3-1
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
#include <silkworm/infra/common/memory_mapped_file.hpp>
2727

2828
#include "../schema_config.hpp"
29+
#include "../step_block_num_converter.hpp"
2930

3031
namespace silkworm::snapshots {
3132

@@ -44,10 +45,11 @@ class BodyIndex {
4445

4546
private:
4647
static IndexDescriptor make_descriptor(const SnapshotPath& segment_path) {
48+
auto step_converter = db::blocks::kStepToBlockNumConverter;
4749
return {
4850
.index_file = segment_path.related_path_ext(db::blocks::kIdxExtension),
4951
.key_factory = std::make_unique<KeyFactory>(),
50-
.base_data_id = segment_path.step_range().to_block_num_range().start,
52+
.base_data_id = step_converter.timestamp_from_step(segment_path.step_range().start),
5153
};
5254
}
5355
};

‎silkworm/db/blocks/headers/header_index.hpp

+3-1
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
#include <silkworm/infra/common/memory_mapped_file.hpp>
2727

2828
#include "../schema_config.hpp"
29+
#include "../step_block_num_converter.hpp"
2930

3031
namespace silkworm::snapshots {
3132

@@ -44,10 +45,11 @@ class HeaderIndex {
4445

4546
private:
4647
static IndexDescriptor make_descriptor(const SnapshotPath& segment_path) {
48+
auto step_converter = db::blocks::kStepToBlockNumConverter;
4749
return {
4850
.index_file = segment_path.related_path_ext(db::blocks::kIdxExtension),
4951
.key_factory = std::make_unique<KeyFactory>(),
50-
.base_data_id = segment_path.step_range().to_block_num_range().start,
52+
.base_data_id = step_converter.timestamp_from_step(segment_path.step_range().start),
5153
};
5254
}
5355
};

‎silkworm/db/blocks/headers/header_segment.cpp

+4-1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
#include <silkworm/infra/common/decoding_exception.hpp>
2020
#include <silkworm/infra/common/ensure.hpp>
2121

22+
#include "../step_block_num_converter.hpp"
23+
2224
namespace silkworm::snapshots {
2325

2426
void encode_word_from_header(Bytes& word, const BlockHeader& header) {
@@ -39,7 +41,8 @@ void decode_word_into_header(ByteView word, BlockHeader& header) {
3941
success_or_throw(decode_result, "decode_word_into_header: rlp::decode error");
4042
}
4143

42-
void check_sanity_of_header_with_metadata(const BlockHeader& header, BlockNumRange block_num_range) {
44+
void check_sanity_of_header_with_metadata(const BlockHeader& header, datastore::StepRange step_range) {
45+
auto block_num_range = db::blocks::kStepToBlockNumConverter.timestamp_range_from_step_range(step_range);
4346
BlockNum block_from = block_num_range.start;
4447
BlockNum block_to = block_num_range.end;
4548
ensure((header.number >= block_from) && (header.number < block_to), [&]() {

‎silkworm/db/blocks/headers/header_segment.hpp

+2-2
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ namespace silkworm::snapshots {
2626

2727
void encode_word_from_header(Bytes& word, const BlockHeader& header);
2828
void decode_word_into_header(ByteView word, BlockHeader& header);
29-
void check_sanity_of_header_with_metadata(const BlockHeader& header, BlockNumRange block_num_range);
29+
void check_sanity_of_header_with_metadata(const BlockHeader& header, datastore::StepRange step_range);
3030

3131
struct HeaderSegmentWordEncoder : public Encoder {
3232
BlockHeader value;
@@ -53,7 +53,7 @@ struct HeaderSegmentWordDecoder : public Decoder {
5353
}
5454

5555
void check_sanity_with_metadata(const SnapshotPath& path) override {
56-
check_sanity_of_header_with_metadata(value, path.step_range().to_block_num_range());
56+
check_sanity_of_header_with_metadata(value, path.step_range());
5757
}
5858
};
5959

‎silkworm/db/blocks/schema_config.cpp

+2-1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
#include "schema_config.hpp"
1818

1919
#include "blocks_index_builders_factory.hpp"
20+
#include "step_block_num_converter.hpp"
2021

2122
namespace silkworm::db::blocks {
2223

@@ -65,7 +66,7 @@ snapshots::SnapshotRepository make_blocks_repository(
6566
std::move(dir_path),
6667
open,
6768
make_blocks_repository_schema(),
68-
std::make_unique<datastore::StepToBlockNumConverter>(),
69+
kStepToBlockNumConverter,
6970
index_salt,
7071
make_blocks_index_builders_factory(),
7172
};

‎silkworm/db/blocks/step_block_num_converter.hpp

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
/*
2+
Copyright 2025 The Silkworm Authors
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
#pragma once
18+
19+
#include <silkworm/db/datastore/common/step_timestamp_converter.hpp>
20+
21+
namespace silkworm::db::blocks {
22+
23+
//! Scale factor to convert from-to block number values in block snapshot file names
24+
inline constexpr size_t kStepSizeForBlockSnapshots = 1'000;
25+
26+
inline constexpr datastore::StepToTimestampConverter kStepToBlockNumConverter{kStepSizeForBlockSnapshots};
27+
28+
} // namespace silkworm::db::blocks

‎silkworm/db/blocks/transactions/txn_to_block_index.hpp

+5-2
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
#include <silkworm/infra/common/memory_mapped_file.hpp>
2727

2828
#include "../schema_config.hpp"
29+
#include "../step_block_num_converter.hpp"
2930
#include "txn_index.hpp"
3031
#include "txs_and_bodies_query.hpp"
3132

@@ -51,7 +52,8 @@ class TransactionToBlockIndex {
5152
static IndexBuilder make(
5253
SnapshotPath bodies_segment_path,
5354
SnapshotPath segment_path) {
54-
BlockNum first_block_num = segment_path.step_range().to_block_num_range().start;
55+
auto step_converter = db::blocks::kStepToBlockNumConverter;
56+
BlockNum first_block_num = step_converter.timestamp_from_step(segment_path.step_range().start);
5557
return make(
5658
std::move(bodies_segment_path),
5759
std::nullopt,
@@ -77,7 +79,8 @@ class TransactionToBlockIndex {
7779
std::optional<MemoryMappedRegion> bodies_segment_region,
7880
SnapshotPath segment_path,
7981
std::optional<MemoryMappedRegion> segment_region) {
80-
BlockNum first_block_num = segment_path.step_range().to_block_num_range().start;
82+
auto step_converter = db::blocks::kStepToBlockNumConverter;
83+
BlockNum first_block_num = step_converter.timestamp_from_step(segment_path.step_range().start);
8184
return make(
8285
std::move(bodies_segment_path),
8386
bodies_segment_region,

‎silkworm/db/datastore/common/step.hpp

+7-76
Original file line numberDiff line numberDiff line change
@@ -16,39 +16,26 @@
1616

1717
#pragma once
1818

19-
#include <silkworm/core/common/base.hpp>
20-
#include <silkworm/infra/common/ensure.hpp>
19+
#include <cstdint>
20+
#include <limits>
21+
#include <string>
2122

22-
#include "timestamp.hpp"
23+
#include <silkworm/infra/common/ensure.hpp>
2324

2425
namespace silkworm::datastore {
2526

26-
//! Scale factor to convert from-to block number values in block snapshot file names
27-
inline constexpr size_t kStepSizeForBlockSnapshots = 1'000;
28-
29-
//! Scale factor to convert from-to txn id values in temporal snapshot file names
30-
inline constexpr size_t kStepSizeForTemporalSnapshots = 1'562'500; // = 100M / 64
31-
3227
struct Step {
3328
size_t value;
3429

35-
explicit Step(size_t value1) : value(value1) {}
30+
constexpr explicit Step(size_t value1) : value(value1) {}
3631
friend bool operator==(const Step&, const Step&) = default;
3732
bool operator<(const Step& other) const { return this->value < other.value; }
3833
bool operator<=(const Step& other) const { return this->value <= other.value; }
3934
std::string to_string() const { return std::to_string(value) + "st"; }
40-
41-
BlockNum to_block_num() const { return value * kStepSizeForBlockSnapshots; }
42-
static Step from_block_num(BlockNum block_num) {
43-
return Step{static_cast<size_t>(block_num / kStepSizeForBlockSnapshots)};
44-
}
45-
46-
TxnId to_txn_id() const { return value * kStepSizeForTemporalSnapshots; }
47-
static Step from_txn_id(TxnId txn_id) {
48-
return Step{static_cast<size_t>(txn_id / kStepSizeForTemporalSnapshots)};
49-
}
5035
};
5136

37+
inline constexpr Step kMaxStep{std::numeric_limits<size_t>::max()};
38+
5239
struct StepRange {
5340
Step start;
5441
Step end;
@@ -61,62 +48,6 @@ struct StepRange {
6148
bool contains_range(StepRange range) const { return (start <= range.start) && (range.end <= end); }
6249
size_t size() const { return end.value - start.value; }
6350
std::string to_string() const { return std::string("[") + start.to_string() + ", " + end.to_string() + ")"; }
64-
65-
BlockNumRange to_block_num_range() const { return {start.to_block_num(), end.to_block_num()}; }
66-
static StepRange from_block_num_range(BlockNumRange range) {
67-
return {Step::from_block_num(range.start),
68-
Step::from_block_num(range.end >= kMaxBlockNum - kStepSizeForBlockSnapshots + 1 ? kMaxBlockNum
69-
: range.end + kStepSizeForBlockSnapshots - 1)};
70-
}
71-
72-
TxnIdRange to_txn_id_range() const { return {start.to_txn_id(), end.to_txn_id()}; }
73-
static StepRange from_txn_id_range(TxnIdRange range) {
74-
return {Step::from_txn_id(range.start),
75-
Step::from_txn_id(range.end >= kMaxTxnId - kStepSizeForTemporalSnapshots + 1 ? kMaxTxnId
76-
: range.end + kStepSizeForTemporalSnapshots - 1)};
77-
}
78-
};
79-
80-
struct StepToTimestampConverter {
81-
virtual ~StepToTimestampConverter() = default;
82-
virtual Step step_from_timestamp(Timestamp t) const = 0;
83-
virtual Timestamp timestamp_from_step(Step s) const = 0;
84-
virtual StepRange step_range_from_timestamp_range(TimestampRange range) const = 0;
85-
virtual TimestampRange timestamp_range_from_step_range(StepRange range) const = 0;
86-
};
87-
88-
struct StepToBlockNumConverter : public StepToTimestampConverter {
89-
~StepToBlockNumConverter() override = default;
90-
Step step_from_timestamp(Timestamp t) const override {
91-
return Step::from_block_num(t);
92-
}
93-
Timestamp timestamp_from_step(Step s) const override {
94-
return s.to_block_num();
95-
}
96-
StepRange step_range_from_timestamp_range(TimestampRange range) const override {
97-
return StepRange::from_block_num_range({range.start, range.end});
98-
}
99-
TimestampRange timestamp_range_from_step_range(StepRange range) const override {
100-
auto r = range.to_block_num_range();
101-
return {r.start, r.end};
102-
}
103-
};
104-
105-
struct StepToTxnIdConverter : public StepToTimestampConverter {
106-
~StepToTxnIdConverter() override = default;
107-
Step step_from_timestamp(Timestamp t) const override {
108-
return Step::from_txn_id(t);
109-
}
110-
Timestamp timestamp_from_step(Step s) const override {
111-
return s.to_txn_id();
112-
}
113-
StepRange step_range_from_timestamp_range(TimestampRange range) const override {
114-
return StepRange::from_txn_id_range({range.start, range.end});
115-
}
116-
TimestampRange timestamp_range_from_step_range(StepRange range) const override {
117-
auto r = range.to_txn_id_range();
118-
return {r.start, r.end};
119-
}
12051
};
12152

12253
} // namespace silkworm::datastore

0 commit comments

Comments
 (0)