datastore: move step converters to db

Author: battlmonstr

cmd/dev/staged_pipeline.cpp

@@ -371,10 +371,10 @@ void debug_unwind(datastore::kvdb::EnvConfig& config, BlockNum height, uint32_t
     // Unwind has just set progress for pre-Execution stages back to unwind_point even if it is within the snapshots
     // We need to reset progress for such stages to the max block in snapshots to avoid database update on next start
     auto& blocks_repository = data_store.blocks_repository();
-    db::stages::write_stage_progress(txn, db::stages::kHeadersKey, blocks_repository.max_block_available());
-    db::stages::write_stage_progress(txn, db::stages::kBlockBodiesKey, blocks_repository.max_block_available());
-    db::stages::write_stage_progress(txn, db::stages::kBlockHashesKey, blocks_repository.max_block_available());
-    db::stages::write_stage_progress(txn, db::stages::kSendersKey, blocks_repository.max_block_available());
+    db::stages::write_stage_progress(txn, db::stages::kHeadersKey, blocks_repository.max_timestamp_available());
+    db::stages::write_stage_progress(txn, db::stages::kBlockBodiesKey, blocks_repository.max_timestamp_available());
+    db::stages::write_stage_progress(txn, db::stages::kBlockHashesKey, blocks_repository.max_timestamp_available());
+    db::stages::write_stage_progress(txn, db::stages::kSendersKey, blocks_repository.max_timestamp_available());
 
     txn.commit_and_stop();
 }

silkworm/db/access_layer.cpp

@@ -1064,20 +1064,20 @@ BlockNum DataModel::max_block_num() const {
     }
 
     // If none is found on db, then ask the snapshot repository (if any) for max block
-    return repository_.max_block_available();
+    return repository_.max_timestamp_available();
 }
 
 BlockNum DataModel::max_frozen_block_num() const {
     // Ask the snapshot repository (if any) for max block
-    return repository_.max_block_available();
+    return repository_.max_timestamp_available();
 }
 
 std::optional<BlockHeader> DataModel::read_header(BlockNum block_num, HashAsArray hash) const {
     return read_header(block_num, Hash(hash));
 }
 
 std::optional<BlockHeader> DataModel::read_header(BlockNum block_num, const Hash& hash) const {
-    BlockNum repository_max_block_num = repository_.max_block_available();
+    BlockNum repository_max_block_num = repository_.max_timestamp_available();
     if ((repository_max_block_num > 0) && (block_num <= repository_max_block_num)) {
         auto header = read_header_from_snapshot(block_num);
         if (header && header->hash() == hash) {  // reading using hash avoid this heavy hash calculation
@@ -1089,7 +1089,7 @@ std::optional<BlockHeader> DataModel::read_header(BlockNum block_num, const Hash
 }
 
 std::optional<BlockHeader> DataModel::read_header(BlockNum block_num) const {
-    BlockNum repository_max_block_num = repository_.max_block_available();
+    BlockNum repository_max_block_num = repository_.max_timestamp_available();
     if ((repository_max_block_num > 0) && (block_num <= repository_max_block_num)) {
         return read_header_from_snapshot(block_num);
     }
@@ -1256,7 +1256,7 @@ void DataModel::for_last_n_headers(size_t n, absl::FunctionRef<void(BlockHeader)
         return;
     }
 
-    auto block_num_in_snapshots = repository_.max_block_available();
+    BlockNum block_num_in_snapshots = repository_.max_timestamp_available();
 
     // We've reached the first header in db but still need to read more from snapshots
     if (last_read_block_num_from_db > 0) {

silkworm/db/blocks/schema_config.cpp

@@ -17,6 +17,7 @@
 #include "schema_config.hpp"
 
 #include "blocks_index_builders_factory.hpp"
+#include "step_block_num_converter.hpp"
 
 namespace silkworm::db::blocks {
 
@@ -65,7 +66,7 @@ snapshots::SnapshotRepository make_blocks_repository(
         std::move(dir_path),
         open,
         make_blocks_repository_schema(),
-        std::make_unique<datastore::StepToBlockNumConverter>(),
+        std::make_unique<StepToBlockNumConverter>(),
         index_salt,
         make_blocks_index_builders_factory(),
     };

silkworm/db/blocks/step_block_num_converter.hpp

@@ -0,0 +1,66 @@
+/*
+   Copyright 2025 The Silkworm Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+#pragma once
+
+#include <silkworm/core/common/base.hpp>
+#include <silkworm/db/datastore/common/step.hpp>
+
+namespace silkworm::db::blocks {
+
+struct StepToBlockNumConverter : public datastore::StepToTimestampConverter {
+    using Step = datastore::Step;
+    using StepRange = datastore::StepRange;
+    using Timestamp = datastore::Timestamp;
+    using TimestampRange = datastore::TimestampRange;
+    static constexpr size_t kStepSize = datastore::kStepSizeForBlockSnapshots;
+
+    ~StepToBlockNumConverter() override = default;
+
+    Step step_from_timestamp(Timestamp t) const override {
+        if (t == datastore::kMaxTimestamp) return datastore::kMaxStep;
+        return Step{static_cast<size_t>(t / kStepSize)};
+    }
+
+    Timestamp timestamp_from_step(Step s) const override {
+        if (s == datastore::kMaxStep) return datastore::kMaxTimestamp;
+        return s.value * kStepSize;
+    }
+
+    StepRange step_range_from_timestamp_range(TimestampRange range) const override {
+        if (range.end == datastore::kMaxTimestamp) {
+            return StepRange{step_from_timestamp(range.start), datastore::kMaxStep};
+        }
+        ensure(range.end <= datastore::kMaxTimestamp - kStepSize + 1, "step_range_from_timestamp_range: end step overflow");
+        Step end = step_from_timestamp(range.end + kStepSize - 1);
+        return StepRange{step_from_timestamp(range.start), end};
+    }
+
+    TimestampRange timestamp_range_from_step_range(StepRange range) const override {
+        return TimestampRange{timestamp_from_step(range.start), timestamp_from_step(range.end)};
+    }
+
+    StepRange step_range_from_block_num_range(BlockNumRange range) const {
+        return step_range_from_timestamp_range({range.start, range.end});
+    }
+
+    BlockNumRange block_num_range_from_step_range(StepRange range) const {
+        TimestampRange ts_range = timestamp_range_from_step_range(range);
+        return BlockNumRange{ts_range.start, ts_range.end};
+    }
+};
+
+}  // namespace silkworm::db::blocks

silkworm/db/blocks/step_block_num_converter_test.cpp

@@ -0,0 +1,61 @@
+/*
+   Copyright 2025 The Silkworm Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+#include "step_block_num_converter.hpp"
+
+#include <catch2/catch_test_macros.hpp>
+
+namespace silkworm::db::blocks {
+
+using namespace silkworm::datastore;
+
+TEST_CASE("StepToBlockNumConverter") {
+    StepToBlockNumConverter converter;
+    static constexpr size_t kStepSize = StepToBlockNumConverter::kStepSize;
+
+    SECTION("Step to Timestamp and back") {
+        Step step{10};
+        Timestamp ts = converter.timestamp_from_step(step);
+        CHECK(ts == kStepSize * 10);
+        CHECK(converter.step_from_timestamp(ts) == step);
+
+        CHECK(converter.timestamp_from_step(Step{0}) == 0);
+        CHECK(converter.timestamp_from_step(Step{500}) == kStepSize * 500);
+    }
+
+    SECTION("Step limits") {
+        CHECK(converter.step_from_timestamp(kMaxTimestamp) == kMaxStep);
+        CHECK(converter.timestamp_from_step(kMaxStep) == kMaxTimestamp);
+    }
+
+    SECTION("StepRange to TimestampRange and back") {
+        StepRange range{Step{10}, Step{20}};
+        const TimestampRange ts_range = converter.timestamp_range_from_step_range(range);
+        CHECK(ts_range == TimestampRange{kStepSize * 10, kStepSize * 20});
+        CHECK(converter.step_range_from_timestamp_range(ts_range) == range);
+
+        CHECK(converter.timestamp_range_from_step_range({Step{0}, Step{0}}) == TimestampRange{0, 0});
+        CHECK(converter.timestamp_range_from_step_range({Step{0}, Step{500}}) == TimestampRange{0, kStepSize * 500});
+    }
+
+    SECTION("StepRange limits") {
+        CHECK(converter.step_range_from_timestamp_range({kMaxTimestamp, kMaxTimestamp}) == StepRange{kMaxStep, kMaxStep});
+        CHECK(converter.timestamp_range_from_step_range({kMaxStep, kMaxStep}) == TimestampRange{kMaxTimestamp, kMaxTimestamp});
+        CHECK_THROWS(converter.step_range_from_timestamp_range({0, kMaxTimestamp - 1}));
+    }
+}
+
+}  // namespace silkworm::db::blocks

silkworm/db/datastore/common/step.hpp

@@ -26,29 +26,20 @@ namespace silkworm::datastore {
 //! Scale factor to convert from-to block number values in block snapshot file names
 inline constexpr size_t kStepSizeForBlockSnapshots = 1'000;
 
-//! Scale factor to convert from-to txn id values in temporal snapshot file names
-inline constexpr size_t kStepSizeForTemporalSnapshots = 1'562'500;  // = 100M / 64
-
 struct Step {
     size_t value;
 
-    explicit Step(size_t value1) : value(value1) {}
+    constexpr explicit Step(size_t value1) : value(value1) {}
     friend bool operator==(const Step&, const Step&) = default;
     bool operator<(const Step& other) const { return this->value < other.value; }
     bool operator<=(const Step& other) const { return this->value <= other.value; }
     std::string to_string() const { return std::to_string(value) + "st"; }
 
     BlockNum to_block_num() const { return value * kStepSizeForBlockSnapshots; }
-    static Step from_block_num(BlockNum block_num) {
-        return Step{static_cast<size_t>(block_num / kStepSizeForBlockSnapshots)};
-    }
-
-    TxnId to_txn_id() const { return value * kStepSizeForTemporalSnapshots; }
-    static Step from_txn_id(TxnId txn_id) {
-        return Step{static_cast<size_t>(txn_id / kStepSizeForTemporalSnapshots)};
-    }
 };
 
+inline constexpr Step kMaxStep{std::numeric_limits<size_t>::max()};
+
 struct StepRange {
     Step start;
     Step end;
@@ -63,18 +54,6 @@ struct StepRange {
     std::string to_string() const { return std::string("[") + start.to_string() + ", " + end.to_string() + ")"; }
 
     BlockNumRange to_block_num_range() const { return {start.to_block_num(), end.to_block_num()}; }
-    static StepRange from_block_num_range(BlockNumRange range) {
-        return {Step::from_block_num(range.start),
-                Step::from_block_num(range.end >= kMaxBlockNum - kStepSizeForBlockSnapshots + 1 ? kMaxBlockNum
-                                                                                                : range.end + kStepSizeForBlockSnapshots - 1)};
-    }
-
-    TxnIdRange to_txn_id_range() const { return {start.to_txn_id(), end.to_txn_id()}; }
-    static StepRange from_txn_id_range(TxnIdRange range) {
-        return {Step::from_txn_id(range.start),
-                Step::from_txn_id(range.end >= kMaxTxnId - kStepSizeForTemporalSnapshots + 1 ? kMaxTxnId
-                                                                                             : range.end + kStepSizeForTemporalSnapshots - 1)};
-    }
 };
 
 struct StepToTimestampConverter {
@@ -85,38 +64,4 @@ struct StepToTimestampConverter {
     virtual TimestampRange timestamp_range_from_step_range(StepRange range) const = 0;
 };
 
-struct StepToBlockNumConverter : public StepToTimestampConverter {
-    ~StepToBlockNumConverter() override = default;
-    Step step_from_timestamp(Timestamp t) const override {
-        return Step::from_block_num(t);
-    }
-    Timestamp timestamp_from_step(Step s) const override {
-        return s.to_block_num();
-    }
-    StepRange step_range_from_timestamp_range(TimestampRange range) const override {
-        return StepRange::from_block_num_range({range.start, range.end});
-    }
-    TimestampRange timestamp_range_from_step_range(StepRange range) const override {
-        auto r = range.to_block_num_range();
-        return {r.start, r.end};
-    }
-};
-
-struct StepToTxnIdConverter : public StepToTimestampConverter {
-    ~StepToTxnIdConverter() override = default;
-    Step step_from_timestamp(Timestamp t) const override {
-        return Step::from_txn_id(t);
-    }
-    Timestamp timestamp_from_step(Step s) const override {
-        return s.to_txn_id();
-    }
-    StepRange step_range_from_timestamp_range(TimestampRange range) const override {
-        return StepRange::from_txn_id_range({range.start, range.end});
-    }
-    TimestampRange timestamp_range_from_step_range(StepRange range) const override {
-        auto r = range.to_txn_id_range();
-        return {r.start, r.end};
-    }
-};
-
 }  // namespace silkworm::datastore

silkworm/db/datastore/common/step_test.cpp

@@ -21,12 +21,6 @@
 namespace silkworm::datastore {
 
 TEST_CASE("Step", "[datastore][common]") {
-    CHECK(Step{0}.to_block_num() == 0);
-    CHECK(Step{500}.to_block_num() == 500'000);
-
-    CHECK(Step{0}.to_txn_id() == 0);
-    CHECK(Step{64}.to_txn_id() == 100'000'000);
-
     SECTION("Step constructor and value") {
         Step step{10};
         CHECK(step.value == 10);
@@ -45,36 +39,17 @@ TEST_CASE("Step", "[datastore][common]") {
         Step step{100};
         CHECK(step.to_string() == "100st");
     }
-
-    SECTION("Step to BlockNum and back") {
-        Step step{10};
-        BlockNum block_num = step.to_block_num();
-        CHECK(block_num == 10'000);  // 10 * 1000 = 10000
-        CHECK(Step::from_block_num(block_num).value == 10);
-    }
-
-    SECTION("Step to TxnId and back") {
-        Step step{10};
-        TxnId txn_id = step.to_txn_id();
-        CHECK(txn_id == 15'625'000);  // 10 * 1562500 = 15625000
-        CHECK(Step::from_txn_id(txn_id).value == 10);
-    }
-
-    SECTION("Step limits") {
-        CHECK(Step::from_block_num(kMaxBlockNum).value == kMaxBlockNum / kStepSizeForBlockSnapshots);
-        CHECK(Step::from_txn_id(kMaxTxnId).value == kMaxTxnId / kStepSizeForTemporalSnapshots);
-    }
 }
 
 TEST_CASE("StepRange", "[datastore][common]") {
-    CHECK(StepRange{Step{0}, Step{0}}.to_block_num_range() == BlockNumRange{0, 0});
-    CHECK(StepRange{Step{0}, Step{500}}.to_block_num_range() == BlockNumRange{0, 500'000});
-
     StepRange range(Step{10}, Step{20});
 
-    SECTION("StepRange constructor and containment") {
+    SECTION("StepRange constructor") {
         CHECK(range.start.value == 10);
         CHECK(range.end.value == 20);
+    }
+
+    SECTION("StepRange containment") {
         CHECK(range.contains(Step{15}));
         CHECK_FALSE(range.contains(Step{25}));
     }
@@ -86,42 +61,6 @@ TEST_CASE("StepRange", "[datastore][common]") {
     SECTION("StepRange to string") {
         CHECK(range.to_string() == "[10st, 20st)");
     }
-
-    SECTION("StepRange to BlockNumRange and back") {
-        const BlockNumRange block_range = range.to_block_num_range();
-        CHECK(block_range.start == 10000);
-        CHECK(block_range.end == 20000);
-
-        const StepRange step_range = StepRange::from_block_num_range(block_range);
-        CHECK(step_range.start.value == 10);
-        CHECK(step_range.end.value == 20);
-    }
-
-    SECTION("StepRange to TxnIdRange and back") {
-        const TxnIdRange txn_range = range.to_txn_id_range();
-        CHECK(txn_range.start == 15625000);
-        CHECK(txn_range.end == 31250000);
-
-        const StepRange restored_range = StepRange::from_txn_id_range(txn_range);
-        CHECK(restored_range.start.value == 10);
-        CHECK(restored_range.end.value == 20);
-    }
-
-    SECTION("StepRange limits") {
-        const StepRange r1 = StepRange::from_block_num_range(BlockNumRange{0, kMaxBlockNum - kStepSizeForBlockSnapshots + 2});
-        CHECK(r1.end.value == kMaxBlockNum / kStepSizeForBlockSnapshots);
-
-        const StepRange r2 = StepRange::from_txn_id_range(TxnIdRange{0, kMaxTxnId - kStepSizeForTemporalSnapshots + 2});
-        CHECK(r2.end.value == kMaxTxnId / kStepSizeForTemporalSnapshots);
-
-        const StepRange r3 = StepRange::from_block_num_range(BlockNumRange{kMaxBlockNum, kMaxBlockNum});
-        CHECK(r3.start.value == kMaxBlockNum / kStepSizeForBlockSnapshots);
-        CHECK(r3.end.value == kMaxBlockNum / kStepSizeForBlockSnapshots);
-
-        const StepRange r4 = StepRange::from_txn_id_range(TxnIdRange{kMaxTxnId, kMaxTxnId});
-        CHECK(r4.start.value == kMaxTxnId / kStepSizeForTemporalSnapshots);
-        CHECK(r4.end.value == kMaxTxnId / kStepSizeForTemporalSnapshots);
-    }
 }
 
 }  // namespace silkworm::datastore

silkworm/db/datastore/common/timestamp.hpp

@@ -24,7 +24,7 @@ namespace silkworm::datastore {
 
 using Timestamp = uint64_t;
 
-constexpr auto kMaxTimestamp = std::numeric_limits<Timestamp>::max();
+inline constexpr Timestamp kMaxTimestamp = std::numeric_limits<Timestamp>::max();
 
 struct TimestampRange {
     Timestamp start;