Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

datastore: move step converters to db #2772

Merged
merged 1 commit into from
Mar 19, 2025
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions cmd/dev/staged_pipeline.cpp
Original file line number Diff line number Diff line change
@@ -371,10 +371,10 @@ void debug_unwind(datastore::kvdb::EnvConfig& config, BlockNum height, uint32_t
// Unwind has just set progress for pre-Execution stages back to unwind_point even if it is within the snapshots
// We need to reset progress for such stages to the max block in snapshots to avoid database update on next start
auto& blocks_repository = data_store.blocks_repository();
db::stages::write_stage_progress(txn, db::stages::kHeadersKey, blocks_repository.max_block_available());
db::stages::write_stage_progress(txn, db::stages::kBlockBodiesKey, blocks_repository.max_block_available());
db::stages::write_stage_progress(txn, db::stages::kBlockHashesKey, blocks_repository.max_block_available());
db::stages::write_stage_progress(txn, db::stages::kSendersKey, blocks_repository.max_block_available());
db::stages::write_stage_progress(txn, db::stages::kHeadersKey, blocks_repository.max_timestamp_available());
db::stages::write_stage_progress(txn, db::stages::kBlockBodiesKey, blocks_repository.max_timestamp_available());
db::stages::write_stage_progress(txn, db::stages::kBlockHashesKey, blocks_repository.max_timestamp_available());
db::stages::write_stage_progress(txn, db::stages::kSendersKey, blocks_repository.max_timestamp_available());

txn.commit_and_stop();
}
10 changes: 5 additions & 5 deletions silkworm/db/access_layer.cpp
Original file line number Diff line number Diff line change
@@ -1064,20 +1064,20 @@ BlockNum DataModel::max_block_num() const {
}

// If none is found on db, then ask the snapshot repository (if any) for max block
return repository_.max_block_available();
return repository_.max_timestamp_available();
}

BlockNum DataModel::max_frozen_block_num() const {
// Ask the snapshot repository (if any) for max block
return repository_.max_block_available();
return repository_.max_timestamp_available();
}

std::optional<BlockHeader> DataModel::read_header(BlockNum block_num, HashAsArray hash) const {
return read_header(block_num, Hash(hash));
}

std::optional<BlockHeader> DataModel::read_header(BlockNum block_num, const Hash& hash) const {
BlockNum repository_max_block_num = repository_.max_block_available();
BlockNum repository_max_block_num = repository_.max_timestamp_available();
if ((repository_max_block_num > 0) && (block_num <= repository_max_block_num)) {
auto header = read_header_from_snapshot(block_num);
if (header && header->hash() == hash) { // reading using hash avoid this heavy hash calculation
@@ -1089,7 +1089,7 @@ std::optional<BlockHeader> DataModel::read_header(BlockNum block_num, const Hash
}

std::optional<BlockHeader> DataModel::read_header(BlockNum block_num) const {
BlockNum repository_max_block_num = repository_.max_block_available();
BlockNum repository_max_block_num = repository_.max_timestamp_available();
if ((repository_max_block_num > 0) && (block_num <= repository_max_block_num)) {
return read_header_from_snapshot(block_num);
}
@@ -1256,7 +1256,7 @@ void DataModel::for_last_n_headers(size_t n, absl::FunctionRef<void(BlockHeader)
return;
}

auto block_num_in_snapshots = repository_.max_block_available();
BlockNum block_num_in_snapshots = repository_.max_timestamp_available();

// We've reached the first header in db but still need to read more from snapshots
if (last_read_block_num_from_db > 0) {
4 changes: 3 additions & 1 deletion silkworm/db/blocks/bodies/body_index.hpp
Original file line number Diff line number Diff line change
@@ -26,6 +26,7 @@
#include <silkworm/infra/common/memory_mapped_file.hpp>

#include "../schema_config.hpp"
#include "../step_block_num_converter.hpp"

namespace silkworm::snapshots {

@@ -44,10 +45,11 @@ class BodyIndex {

private:
static IndexDescriptor make_descriptor(const SnapshotPath& segment_path) {
auto step_converter = db::blocks::kStepToBlockNumConverter;
return {
.index_file = segment_path.related_path_ext(db::blocks::kIdxExtension),
.key_factory = std::make_unique<KeyFactory>(),
.base_data_id = segment_path.step_range().to_block_num_range().start,
.base_data_id = step_converter.timestamp_from_step(segment_path.step_range().start),
};
}
};
4 changes: 3 additions & 1 deletion silkworm/db/blocks/headers/header_index.hpp
Original file line number Diff line number Diff line change
@@ -26,6 +26,7 @@
#include <silkworm/infra/common/memory_mapped_file.hpp>

#include "../schema_config.hpp"
#include "../step_block_num_converter.hpp"

namespace silkworm::snapshots {

@@ -44,10 +45,11 @@ class HeaderIndex {

private:
static IndexDescriptor make_descriptor(const SnapshotPath& segment_path) {
auto step_converter = db::blocks::kStepToBlockNumConverter;
return {
.index_file = segment_path.related_path_ext(db::blocks::kIdxExtension),
.key_factory = std::make_unique<KeyFactory>(),
.base_data_id = segment_path.step_range().to_block_num_range().start,
.base_data_id = step_converter.timestamp_from_step(segment_path.step_range().start),
};
}
};
5 changes: 4 additions & 1 deletion silkworm/db/blocks/headers/header_segment.cpp
Original file line number Diff line number Diff line change
@@ -19,6 +19,8 @@
#include <silkworm/infra/common/decoding_exception.hpp>
#include <silkworm/infra/common/ensure.hpp>

#include "../step_block_num_converter.hpp"

namespace silkworm::snapshots {

void encode_word_from_header(Bytes& word, const BlockHeader& header) {
@@ -39,7 +41,8 @@ void decode_word_into_header(ByteView word, BlockHeader& header) {
success_or_throw(decode_result, "decode_word_into_header: rlp::decode error");
}

void check_sanity_of_header_with_metadata(const BlockHeader& header, BlockNumRange block_num_range) {
void check_sanity_of_header_with_metadata(const BlockHeader& header, datastore::StepRange step_range) {
auto block_num_range = db::blocks::kStepToBlockNumConverter.timestamp_range_from_step_range(step_range);
BlockNum block_from = block_num_range.start;
BlockNum block_to = block_num_range.end;
ensure((header.number >= block_from) && (header.number < block_to), [&]() {
4 changes: 2 additions & 2 deletions silkworm/db/blocks/headers/header_segment.hpp
Original file line number Diff line number Diff line change
@@ -26,7 +26,7 @@ namespace silkworm::snapshots {

void encode_word_from_header(Bytes& word, const BlockHeader& header);
void decode_word_into_header(ByteView word, BlockHeader& header);
void check_sanity_of_header_with_metadata(const BlockHeader& header, BlockNumRange block_num_range);
void check_sanity_of_header_with_metadata(const BlockHeader& header, datastore::StepRange step_range);

struct HeaderSegmentWordEncoder : public Encoder {
BlockHeader value;
@@ -53,7 +53,7 @@ struct HeaderSegmentWordDecoder : public Decoder {
}

void check_sanity_with_metadata(const SnapshotPath& path) override {
check_sanity_of_header_with_metadata(value, path.step_range().to_block_num_range());
check_sanity_of_header_with_metadata(value, path.step_range());
}
};

3 changes: 2 additions & 1 deletion silkworm/db/blocks/schema_config.cpp
Original file line number Diff line number Diff line change
@@ -17,6 +17,7 @@
#include "schema_config.hpp"

#include "blocks_index_builders_factory.hpp"
#include "step_block_num_converter.hpp"

namespace silkworm::db::blocks {

@@ -65,7 +66,7 @@ snapshots::SnapshotRepository make_blocks_repository(
std::move(dir_path),
open,
make_blocks_repository_schema(),
std::make_unique<datastore::StepToBlockNumConverter>(),
kStepToBlockNumConverter,
index_salt,
make_blocks_index_builders_factory(),
};
28 changes: 28 additions & 0 deletions silkworm/db/blocks/step_block_num_converter.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
/*
Copyright 2025 The Silkworm Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

#pragma once

#include <silkworm/db/datastore/common/step_timestamp_converter.hpp>

namespace silkworm::db::blocks {

//! Scale factor to convert from-to block number values in block snapshot file names
inline constexpr size_t kStepSizeForBlockSnapshots = 1'000;

inline constexpr datastore::StepToTimestampConverter kStepToBlockNumConverter{kStepSizeForBlockSnapshots};

} // namespace silkworm::db::blocks
7 changes: 5 additions & 2 deletions silkworm/db/blocks/transactions/txn_to_block_index.hpp
Original file line number Diff line number Diff line change
@@ -26,6 +26,7 @@
#include <silkworm/infra/common/memory_mapped_file.hpp>

#include "../schema_config.hpp"
#include "../step_block_num_converter.hpp"
#include "txn_index.hpp"
#include "txs_and_bodies_query.hpp"

@@ -51,7 +52,8 @@ class TransactionToBlockIndex {
static IndexBuilder make(
SnapshotPath bodies_segment_path,
SnapshotPath segment_path) {
BlockNum first_block_num = segment_path.step_range().to_block_num_range().start;
auto step_converter = db::blocks::kStepToBlockNumConverter;
BlockNum first_block_num = step_converter.timestamp_from_step(segment_path.step_range().start);
return make(
std::move(bodies_segment_path),
std::nullopt,
@@ -77,7 +79,8 @@ class TransactionToBlockIndex {
std::optional<MemoryMappedRegion> bodies_segment_region,
SnapshotPath segment_path,
std::optional<MemoryMappedRegion> segment_region) {
BlockNum first_block_num = segment_path.step_range().to_block_num_range().start;
auto step_converter = db::blocks::kStepToBlockNumConverter;
BlockNum first_block_num = step_converter.timestamp_from_step(segment_path.step_range().start);
return make(
std::move(bodies_segment_path),
bodies_segment_region,
83 changes: 7 additions & 76 deletions silkworm/db/datastore/common/step.hpp
Original file line number Diff line number Diff line change
@@ -16,39 +16,26 @@

#pragma once

#include <silkworm/core/common/base.hpp>
#include <silkworm/infra/common/ensure.hpp>
#include <cstdint>
#include <limits>
#include <string>

#include "timestamp.hpp"
#include <silkworm/infra/common/ensure.hpp>

namespace silkworm::datastore {

//! Scale factor to convert from-to block number values in block snapshot file names
inline constexpr size_t kStepSizeForBlockSnapshots = 1'000;

//! Scale factor to convert from-to txn id values in temporal snapshot file names
inline constexpr size_t kStepSizeForTemporalSnapshots = 1'562'500; // = 100M / 64

struct Step {
size_t value;

explicit Step(size_t value1) : value(value1) {}
constexpr explicit Step(size_t value1) : value(value1) {}
friend bool operator==(const Step&, const Step&) = default;
bool operator<(const Step& other) const { return this->value < other.value; }
bool operator<=(const Step& other) const { return this->value <= other.value; }
std::string to_string() const { return std::to_string(value) + "st"; }

BlockNum to_block_num() const { return value * kStepSizeForBlockSnapshots; }
static Step from_block_num(BlockNum block_num) {
return Step{static_cast<size_t>(block_num / kStepSizeForBlockSnapshots)};
}

TxnId to_txn_id() const { return value * kStepSizeForTemporalSnapshots; }
static Step from_txn_id(TxnId txn_id) {
return Step{static_cast<size_t>(txn_id / kStepSizeForTemporalSnapshots)};
}
};

inline constexpr Step kMaxStep{std::numeric_limits<size_t>::max()};

struct StepRange {
Step start;
Step end;
@@ -61,62 +48,6 @@ struct StepRange {
bool contains_range(StepRange range) const { return (start <= range.start) && (range.end <= end); }
size_t size() const { return end.value - start.value; }
std::string to_string() const { return std::string("[") + start.to_string() + ", " + end.to_string() + ")"; }

BlockNumRange to_block_num_range() const { return {start.to_block_num(), end.to_block_num()}; }
static StepRange from_block_num_range(BlockNumRange range) {
return {Step::from_block_num(range.start),
Step::from_block_num(range.end >= kMaxBlockNum - kStepSizeForBlockSnapshots + 1 ? kMaxBlockNum
: range.end + kStepSizeForBlockSnapshots - 1)};
}

TxnIdRange to_txn_id_range() const { return {start.to_txn_id(), end.to_txn_id()}; }
static StepRange from_txn_id_range(TxnIdRange range) {
return {Step::from_txn_id(range.start),
Step::from_txn_id(range.end >= kMaxTxnId - kStepSizeForTemporalSnapshots + 1 ? kMaxTxnId
: range.end + kStepSizeForTemporalSnapshots - 1)};
}
};

struct StepToTimestampConverter {
virtual ~StepToTimestampConverter() = default;
virtual Step step_from_timestamp(Timestamp t) const = 0;
virtual Timestamp timestamp_from_step(Step s) const = 0;
virtual StepRange step_range_from_timestamp_range(TimestampRange range) const = 0;
virtual TimestampRange timestamp_range_from_step_range(StepRange range) const = 0;
};

struct StepToBlockNumConverter : public StepToTimestampConverter {
~StepToBlockNumConverter() override = default;
Step step_from_timestamp(Timestamp t) const override {
return Step::from_block_num(t);
}
Timestamp timestamp_from_step(Step s) const override {
return s.to_block_num();
}
StepRange step_range_from_timestamp_range(TimestampRange range) const override {
return StepRange::from_block_num_range({range.start, range.end});
}
TimestampRange timestamp_range_from_step_range(StepRange range) const override {
auto r = range.to_block_num_range();
return {r.start, r.end};
}
};

struct StepToTxnIdConverter : public StepToTimestampConverter {
~StepToTxnIdConverter() override = default;
Step step_from_timestamp(Timestamp t) const override {
return Step::from_txn_id(t);
}
Timestamp timestamp_from_step(Step s) const override {
return s.to_txn_id();
}
StepRange step_range_from_timestamp_range(TimestampRange range) const override {
return StepRange::from_txn_id_range({range.start, range.end});
}
TimestampRange timestamp_range_from_step_range(StepRange range) const override {
auto r = range.to_txn_id_range();
return {r.start, r.end};
}
};

} // namespace silkworm::datastore
69 changes: 4 additions & 65 deletions silkworm/db/datastore/common/step_test.cpp
Original file line number Diff line number Diff line change
@@ -21,12 +21,6 @@
namespace silkworm::datastore {

TEST_CASE("Step", "[datastore][common]") {
CHECK(Step{0}.to_block_num() == 0);
CHECK(Step{500}.to_block_num() == 500'000);

CHECK(Step{0}.to_txn_id() == 0);
CHECK(Step{64}.to_txn_id() == 100'000'000);

SECTION("Step constructor and value") {
Step step{10};
CHECK(step.value == 10);
@@ -45,36 +39,17 @@ TEST_CASE("Step", "[datastore][common]") {
Step step{100};
CHECK(step.to_string() == "100st");
}

SECTION("Step to BlockNum and back") {
Step step{10};
BlockNum block_num = step.to_block_num();
CHECK(block_num == 10'000); // 10 * 1000 = 10000
CHECK(Step::from_block_num(block_num).value == 10);
}

SECTION("Step to TxnId and back") {
Step step{10};
TxnId txn_id = step.to_txn_id();
CHECK(txn_id == 15'625'000); // 10 * 1562500 = 15625000
CHECK(Step::from_txn_id(txn_id).value == 10);
}

SECTION("Step limits") {
CHECK(Step::from_block_num(kMaxBlockNum).value == kMaxBlockNum / kStepSizeForBlockSnapshots);
CHECK(Step::from_txn_id(kMaxTxnId).value == kMaxTxnId / kStepSizeForTemporalSnapshots);
}
}

TEST_CASE("StepRange", "[datastore][common]") {
CHECK(StepRange{Step{0}, Step{0}}.to_block_num_range() == BlockNumRange{0, 0});
CHECK(StepRange{Step{0}, Step{500}}.to_block_num_range() == BlockNumRange{0, 500'000});

StepRange range(Step{10}, Step{20});

SECTION("StepRange constructor and containment") {
SECTION("StepRange constructor") {
CHECK(range.start.value == 10);
CHECK(range.end.value == 20);
}

SECTION("StepRange containment") {
CHECK(range.contains(Step{15}));
CHECK_FALSE(range.contains(Step{25}));
}
@@ -86,42 +61,6 @@ TEST_CASE("StepRange", "[datastore][common]") {
SECTION("StepRange to string") {
CHECK(range.to_string() == "[10st, 20st)");
}

SECTION("StepRange to BlockNumRange and back") {
const BlockNumRange block_range = range.to_block_num_range();
CHECK(block_range.start == 10000);
CHECK(block_range.end == 20000);

const StepRange step_range = StepRange::from_block_num_range(block_range);
CHECK(step_range.start.value == 10);
CHECK(step_range.end.value == 20);
}

SECTION("StepRange to TxnIdRange and back") {
const TxnIdRange txn_range = range.to_txn_id_range();
CHECK(txn_range.start == 15625000);
CHECK(txn_range.end == 31250000);

const StepRange restored_range = StepRange::from_txn_id_range(txn_range);
CHECK(restored_range.start.value == 10);
CHECK(restored_range.end.value == 20);
}

SECTION("StepRange limits") {
const StepRange r1 = StepRange::from_block_num_range(BlockNumRange{0, kMaxBlockNum - kStepSizeForBlockSnapshots + 2});
CHECK(r1.end.value == kMaxBlockNum / kStepSizeForBlockSnapshots);

const StepRange r2 = StepRange::from_txn_id_range(TxnIdRange{0, kMaxTxnId - kStepSizeForTemporalSnapshots + 2});
CHECK(r2.end.value == kMaxTxnId / kStepSizeForTemporalSnapshots);

const StepRange r3 = StepRange::from_block_num_range(BlockNumRange{kMaxBlockNum, kMaxBlockNum});
CHECK(r3.start.value == kMaxBlockNum / kStepSizeForBlockSnapshots);
CHECK(r3.end.value == kMaxBlockNum / kStepSizeForBlockSnapshots);

const StepRange r4 = StepRange::from_txn_id_range(TxnIdRange{kMaxTxnId, kMaxTxnId});
CHECK(r4.start.value == kMaxTxnId / kStepSizeForTemporalSnapshots);
CHECK(r4.end.value == kMaxTxnId / kStepSizeForTemporalSnapshots);
}
}

} // namespace silkworm::datastore
42 changes: 42 additions & 0 deletions silkworm/db/datastore/common/step_timestamp_converter.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
/*
Copyright 2024 The Silkworm Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

#include "step_timestamp_converter.hpp"

#include <silkworm/infra/common/ensure.hpp>

namespace silkworm::datastore {

Step StepToTimestampConverter::step_from_timestamp(Timestamp t) const {
if (t == kMaxTimestamp) return kMaxStep;
return Step{static_cast<size_t>(t / step_size)};
}

Timestamp StepToTimestampConverter::timestamp_from_step(Step s) const {
if (s == kMaxStep) return kMaxTimestamp;
return s.value * step_size;
}

StepRange StepToTimestampConverter::step_range_from_timestamp_range(TimestampRange range) const {
if (range.end == kMaxTimestamp) {
return StepRange{step_from_timestamp(range.start), kMaxStep};
}
ensure(range.end <= kMaxTimestamp - step_size + 1, "step_range_from_timestamp_range: end step overflow");
Step end = step_from_timestamp(range.end + step_size - 1);
return StepRange{step_from_timestamp(range.start), end};
}

} // namespace silkworm::datastore
36 changes: 36 additions & 0 deletions silkworm/db/datastore/common/step_timestamp_converter.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
/*
Copyright 2024 The Silkworm Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

#pragma once

#include "step.hpp"
#include "timestamp.hpp"

namespace silkworm::datastore {

struct StepToTimestampConverter {
size_t step_size;

Step step_from_timestamp(Timestamp t) const;
Timestamp timestamp_from_step(Step s) const;

StepRange step_range_from_timestamp_range(TimestampRange range) const;
TimestampRange timestamp_range_from_step_range(StepRange range) const {
return TimestampRange{timestamp_from_step(range.start), timestamp_from_step(range.end)};
}
};

} // namespace silkworm::datastore
59 changes: 59 additions & 0 deletions silkworm/db/datastore/common/step_timestamp_converter_test.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
/*
Copyright 2025 The Silkworm Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

#include "step_timestamp_converter.hpp"

#include <catch2/catch_test_macros.hpp>

namespace silkworm::datastore {

TEST_CASE("StepToTimestampConverter") {
static constexpr size_t kStepSize = 1000;
StepToTimestampConverter converter{kStepSize};

SECTION("Step to Timestamp and back") {
Step step{10};
Timestamp ts = converter.timestamp_from_step(step);
CHECK(ts == kStepSize * 10);
CHECK(converter.step_from_timestamp(ts) == step);

CHECK(converter.timestamp_from_step(Step{0}) == 0);
CHECK(converter.timestamp_from_step(Step{500}) == kStepSize * 500);
}

SECTION("Step limits") {
CHECK(converter.step_from_timestamp(kMaxTimestamp) == kMaxStep);
CHECK(converter.timestamp_from_step(kMaxStep) == kMaxTimestamp);
}

SECTION("StepRange to TimestampRange and back") {
StepRange range{Step{10}, Step{20}};
const TimestampRange ts_range = converter.timestamp_range_from_step_range(range);
CHECK(ts_range == TimestampRange{kStepSize * 10, kStepSize * 20});
CHECK(converter.step_range_from_timestamp_range(ts_range) == range);

CHECK(converter.timestamp_range_from_step_range({Step{0}, Step{0}}) == TimestampRange{0, 0});
CHECK(converter.timestamp_range_from_step_range({Step{0}, Step{500}}) == TimestampRange{0, kStepSize * 500});
}

SECTION("StepRange limits") {
CHECK(converter.step_range_from_timestamp_range({kMaxTimestamp, kMaxTimestamp}) == StepRange{kMaxStep, kMaxStep});
CHECK(converter.timestamp_range_from_step_range({kMaxStep, kMaxStep}) == TimestampRange{kMaxTimestamp, kMaxTimestamp});
CHECK_THROWS(converter.step_range_from_timestamp_range({0, kMaxTimestamp - 1}));
}
}

} // namespace silkworm::datastore
2 changes: 1 addition & 1 deletion silkworm/db/datastore/common/timestamp.hpp
Original file line number Diff line number Diff line change
@@ -24,7 +24,7 @@ namespace silkworm::datastore {

using Timestamp = uint64_t;

constexpr auto kMaxTimestamp = std::numeric_limits<Timestamp>::max();
inline constexpr Timestamp kMaxTimestamp = std::numeric_limits<Timestamp>::max();

struct TimestampRange {
Timestamp start;
6 changes: 3 additions & 3 deletions silkworm/db/datastore/snapshot_merger.cpp
Original file line number Diff line number Diff line change
@@ -63,7 +63,7 @@ std::unique_ptr<DataMigrationCommand> SnapshotMerger::next_command() {
for (auto& bundle_ptr : snapshots_.view_bundles()) {
auto& bundle = *bundle_ptr;

auto bundle_block_num_range = bundle.step_range().to_block_num_range();
auto bundle_block_num_range = snapshots_.step_converter().timestamp_range_from_step_range(bundle.step_range());
size_t bundle_block_count = bundle_block_num_range.size();

if (bundle_block_count >= kMaxSnapshotSize) {
@@ -86,14 +86,14 @@ std::unique_ptr<DataMigrationCommand> SnapshotMerger::next_command() {
std::shared_ptr<DataMigrationResult> SnapshotMerger::migrate(std::unique_ptr<DataMigrationCommand> command) {
auto& merger_command = dynamic_cast<SnapshotMergerCommand&>(*command);
auto range = merger_command.range;
auto step_range = StepRange::from_block_num_range(range);
StepRange step_range = snapshots_.step_converter().step_range_from_timestamp_range({range.start, range.end});

SnapshotBundlePaths new_bundle{snapshots_.schema(), tmp_dir_path_, step_range};
for (const auto& [name, path] : new_bundle.segment_paths()) {
SILK_DEBUG_M("SnapshotMerger") << "merging " << name.to_string() << " range " << range.to_string();
seg::Compressor compressor{path.path(), tmp_dir_path_};

for (auto& bundle_ptr : snapshots_.bundles_in_range(StepRange::from_block_num_range(range))) {
for (auto& bundle_ptr : snapshots_.bundles_in_range(step_range)) {
auto& bundle = *bundle_ptr;
segment::SegmentReader<RawDecoder<ByteView>> reader{bundle.segment(Schema::kDefaultEntityName, name)};
std::copy(reader.begin(), reader.end(), compressor.add_word_iterator());
1 change: 1 addition & 0 deletions silkworm/db/datastore/snapshots/common/snapshot_path.cpp
Original file line number Diff line number Diff line change
@@ -22,6 +22,7 @@

#include <absl/strings/str_format.h>

#include <silkworm/core/common/assert.hpp>
#include <silkworm/infra/common/log.hpp>

namespace silkworm::snapshots {
14 changes: 4 additions & 10 deletions silkworm/db/datastore/snapshots/snapshot_repository.cpp
Original file line number Diff line number Diff line change
@@ -38,7 +38,7 @@ SnapshotRepository::SnapshotRepository(
std::filesystem::path dir_path,
bool open,
Schema::RepositoryDef schema,
std::unique_ptr<StepToTimestampConverter> step_converter,
StepToTimestampConverter step_converter,
std::optional<uint32_t> index_salt,
std::unique_ptr<IndexBuildersFactory> index_builders_factory)
: name_(std::move(name)),
@@ -77,16 +77,10 @@ size_t SnapshotRepository::bundles_count() const {
return bundles_->size();
}

BlockNum SnapshotRepository::max_block_available() const {
Step end_step = max_end_step();
if (end_step.value == 0) return 0;
return end_step.to_block_num() - 1;
}

Timestamp SnapshotRepository::max_timestamp_available() const {
Step end_step = max_end_step();
if (end_step.value == 0) return 0;
return step_converter_->timestamp_from_step(end_step) - 1;
return step_converter_.timestamp_from_step(end_step) - 1;
}

Step SnapshotRepository::max_end_step() const {
@@ -175,7 +169,7 @@ SnapshotBundle SnapshotRepository::open_bundle(StepRange range) const {
}

std::shared_ptr<SnapshotBundle> SnapshotRepository::find_bundle(Timestamp t) const {
return find_bundle(step_converter_->step_from_timestamp(t));
return find_bundle(step_converter_.step_from_timestamp(t));
}

std::shared_ptr<SnapshotBundle> SnapshotRepository::find_bundle(Step step) const {
@@ -221,7 +215,7 @@ std::vector<std::shared_ptr<SnapshotBundle>> SnapshotRepository::bundles_interse
if (range.size() == 0) {
return {};
}
return bundles_intersecting_range(step_converter_->step_range_from_timestamp_range(range), ascending);
return bundles_intersecting_range(step_converter_.step_range_from_timestamp_range(range), ascending);
}

SnapshotPathList SnapshotRepository::get_files(std::string_view ext) const {
9 changes: 5 additions & 4 deletions silkworm/db/datastore/snapshots/snapshot_repository.hpp
Original file line number Diff line number Diff line change
@@ -29,6 +29,7 @@
#include <vector>

#include "../common/entity_name.hpp"
#include "../common/step_timestamp_converter.hpp"
#include "common/snapshot_path.hpp"
#include "segment_and_accessor_index.hpp"
#include "snapshot_bundle.hpp"
@@ -55,7 +56,7 @@ class SnapshotRepository : public SnapshotRepositoryROAccess {
std::filesystem::path dir_path,
bool open,
Schema::RepositoryDef schema,
std::unique_ptr<datastore::StepToTimestampConverter> step_converter,
datastore::StepToTimestampConverter step_converter,
std::optional<uint32_t> index_salt,
std::unique_ptr<IndexBuildersFactory> index_builders_factory);

@@ -65,7 +66,8 @@ class SnapshotRepository : public SnapshotRepositoryROAccess {
~SnapshotRepository() override = default;

const std::filesystem::path& path() const { return dir_path_; }
const Schema::RepositoryDef& schema() const { return schema_; };
const Schema::RepositoryDef& schema() const { return schema_; }
const datastore::StepToTimestampConverter& step_converter() const { return step_converter_; }

void reopen_folder();

@@ -79,7 +81,6 @@ class SnapshotRepository : public SnapshotRepositoryROAccess {

size_t bundles_count() const override;

BlockNum max_block_available() const override;
Timestamp max_timestamp_available() const override;

std::vector<std::shared_ptr<IndexBuilder>> missing_indexes() const;
@@ -131,7 +132,7 @@ class SnapshotRepository : public SnapshotRepositoryROAccess {
Schema::RepositoryDef schema_;

//! Converts timestamp units to steps
std::unique_ptr<datastore::StepToTimestampConverter> step_converter_;
datastore::StepToTimestampConverter step_converter_;

//! Index salt
std::optional<uint32_t> index_salt_;
Original file line number Diff line number Diff line change
@@ -26,6 +26,7 @@

#include "../common/entity_name.hpp"
#include "../common/step.hpp"
#include "../common/timestamp.hpp"
#include "common/util/iterator/map_values_view.hpp"
#include "segment_and_accessor_index.hpp"

@@ -61,8 +62,6 @@ struct SnapshotRepositoryROAccess {

virtual size_t bundles_count() const = 0;

//! All types of .seg and .idx files are available up to this block number
virtual BlockNum max_block_available() const = 0;
//! All types of .seg and .idx files are available up to this timestamp
virtual Timestamp max_timestamp_available() const = 0;

6 changes: 3 additions & 3 deletions silkworm/db/freezer.cpp
Original file line number Diff line number Diff line change
@@ -67,7 +67,7 @@ static std::optional<uint64_t> get_next_base_txn_id(SnapshotRepository& reposito
}

std::unique_ptr<DataMigrationCommand> Freezer::next_command() {
BlockNum last_frozen = snapshots_.max_block_available();
BlockNum last_frozen = snapshots_.max_timestamp_available();
BlockNum start = (last_frozen > 0) ? last_frozen + 1 : 0;
BlockNum end = start + kChunkSize;

@@ -108,7 +108,7 @@ static const SegmentCollation& get_collation(datastore::EntityName name) {
std::shared_ptr<DataMigrationResult> Freezer::migrate(std::unique_ptr<DataMigrationCommand> command) {
auto& freezer_command = dynamic_cast<SegmentCollationCommand&>(*command);
auto range = freezer_command.range;
auto step_range = StepRange::from_block_num_range(range);
StepRange step_range = snapshots_.step_converter().step_range_from_timestamp_range({range.start, range.end});

SnapshotBundlePaths bundle{snapshots_.schema(), tmp_dir_path_, step_range};
for (const auto& [name, path] : bundle.segment_paths()) {
@@ -139,7 +139,7 @@ void Freezer::commit(std::shared_ptr<DataMigrationResult> result) {
}

BlockNumRange Freezer::cleanup_range() {
BlockNum last_frozen = snapshots_.max_block_available();
BlockNum last_frozen = snapshots_.max_timestamp_available();

BlockNum first_stored_header_num = [this] {
auto db_tx = db_access_.start_ro_tx();
6 changes: 3 additions & 3 deletions silkworm/db/snapshot_repository_test.cpp
Original file line number Diff line number Diff line change
@@ -64,7 +64,7 @@ TEST_CASE("SnapshotRepository::reopen_folder.partial_bundle", "[silkworm][node][
test::TemporarySnapshotFile tmp_snapshot_3{tmp_dir.path(), "v1-015000-015500-transactions.seg"};
auto repository = make_repository(tmp_dir.path());
CHECK(repository.bundles_count() == 0);
CHECK(repository.max_block_available() == 0);
CHECK(repository.max_timestamp_available() == 0);
}

TEST_CASE("SnapshotRepository::view", "[silkworm][node][snapshot]") {
@@ -202,8 +202,8 @@ TEST_CASE("SnapshotRepository::find_segment", "[silkworm][node][snapshot]") {
CHECK_FIRST(repository.find_segment(SnapshotType::transactions, 1'500'013));
CHECK_FIRST(repository.find_segment(SnapshotType::transactions, 1'500'014));
}
SECTION("greater than max_block_available") {
CHECK_FALSE_FIRST(repository.find_segment(SnapshotType::bodies, repository.max_block_available() + 1));
SECTION("greater than max_timestamp_available") {
CHECK_FALSE_FIRST(repository.find_segment(SnapshotType::bodies, repository.max_timestamp_available() + 1));
}
}

11 changes: 7 additions & 4 deletions silkworm/db/snapshot_sync.cpp
Original file line number Diff line number Diff line change
@@ -34,6 +34,7 @@
#include <silkworm/infra/concurrency/thread_pool.hpp>

#include "blocks/headers/header_segment.hpp"
#include "blocks/step_block_num_converter.hpp"
#include "datastore/kvdb/etl_mdbx_collector.hpp"
#include "datastore/snapshots/bittorrent/torrent_file.hpp"
#include "datastore/snapshots/common/snapshot_path.hpp"
@@ -54,10 +55,11 @@ struct PathHasher {
};

static bool snapshot_file_is_fully_merged(std::string_view file_name) {
auto step_converter = blocks::kStepToBlockNumConverter;
const auto path = SnapshotPath::parse(std::filesystem::path{file_name});
return path.has_value() &&
(path->extension() == blocks::kSegmentExtension || path->extension() == blocks::kIdxExtension) &&
(path->step_range().to_block_num_range().size() >= kMaxMergerSnapshotSize);
(step_converter.timestamp_range_from_step_range(path->step_range()).size() >= kMaxMergerSnapshotSize);
}

SnapshotSync::SnapshotSync(
@@ -148,7 +150,7 @@ Task<void> SnapshotSync::setup() {

// Update chain and stage progresses in database according to available snapshots
datastore::kvdb::RWTxnManaged rw_txn = data_store_.chaindata.access_rw().start_rw_tx();
update_database(rw_txn, blocks_repository().max_block_available(), [this] { return is_stopping_latch_.try_wait(); });
update_database(rw_txn, blocks_repository().max_timestamp_available(), [this] { return is_stopping_latch_.try_wait(); });
rw_txn.commit_and_stop();

if (!settings_.no_seeding) {
@@ -290,9 +292,10 @@ Task<void> SnapshotSync::build_missing_indexes() {
}

void SnapshotSync::seed_frozen_local_snapshots() {
for (auto& bundle_ptr : blocks_repository().view_bundles()) {
const auto& repository = blocks_repository();
for (auto& bundle_ptr : repository.view_bundles()) {
auto& bundle = *bundle_ptr;
auto block_num_range = bundle.step_range().to_block_num_range();
auto block_num_range = repository.step_converter().timestamp_range_from_step_range(bundle.step_range());
bool is_frozen = block_num_range.size() >= kMaxMergerSnapshotSize;
const segment::SegmentFileReader& first_snapshot = *bundle.segments().begin();
// assume that if one snapshot in the bundle is preverified, then all of them are
2 changes: 1 addition & 1 deletion silkworm/db/snapshot_sync_test.cpp
Original file line number Diff line number Diff line change
@@ -132,8 +132,8 @@ TEST_CASE("SnapshotSync::update_block_headers", "[db][snapshot][sync]") {
REQUIRE_NOTHROW(TransactionToBlockIndex::make(body_segment_path, txn_segment_path, txn_segment_file.block_num_range().start).build());

// Add a sample Snapshot bundle to the repository
auto step_range = datastore::StepRange::from_block_num_range(snapshots::test_util::kSampleSnapshotBlockRange);
auto& repository = snapshot_sync.blocks_repository();
auto step_range = repository.step_converter().step_range_from_timestamp_range({snapshots::test_util::kSampleSnapshotBlockRange.start, snapshots::test_util::kSampleSnapshotBlockRange.end});
SnapshotBundle bundle = repository.open_bundle(step_range);
repository.add_snapshot_bundle(std::move(bundle));

3 changes: 2 additions & 1 deletion silkworm/db/state/schema_config.cpp
Original file line number Diff line number Diff line change
@@ -17,6 +17,7 @@
#include "schema_config.hpp"

#include "state_index_builders_factory.hpp"
#include "step_txn_id_converter.hpp"

namespace silkworm::db::state {

@@ -90,7 +91,7 @@ static snapshots::SnapshotRepository make_state_repository(
std::move(dir_path),
open,
schema,
std::make_unique<datastore::StepToTxnIdConverter>(),
kStepToTxnIdConverter,
index_salt,
std::make_unique<StateIndexBuildersFactory>(schema),
};
30 changes: 30 additions & 0 deletions silkworm/db/state/step_txn_id_converter.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
/*
Copyright 2025 The Silkworm Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

#pragma once

#include <silkworm/core/common/base.hpp>
#include <silkworm/db/datastore/common/step.hpp>
#include <silkworm/db/datastore/common/step_timestamp_converter.hpp>

namespace silkworm::db::state {

//! Scale factor to convert from-to txn id values in temporal snapshot file names
inline constexpr size_t kStepSizeForTemporalSnapshots = 1'562'500; // = 100M / 64

inline constexpr datastore::StepToTimestampConverter kStepToTxnIdConverter{kStepSizeForTemporalSnapshots};

} // namespace silkworm::db::state
13 changes: 9 additions & 4 deletions silkworm/execution/domain_state.cpp
Original file line number Diff line number Diff line change
@@ -20,6 +20,7 @@
#include <silkworm/db/state/accounts_domain.hpp>
#include <silkworm/db/state/code_domain.hpp>
#include <silkworm/db/state/schema_config.hpp>
#include <silkworm/db/state/step_txn_id_converter.hpp>
#include <silkworm/db/state/storage_domain.hpp>

namespace silkworm::execution {
@@ -102,6 +103,10 @@ void DomainState::insert_receipts(BlockNum block_num, const std::vector<Receipt>
db::write_receipts(tx_, receipts, block_num);
}

datastore::Step DomainState::current_step() const {
return kStepToTxnIdConverter.step_from_timestamp(txn_id_);
}

void DomainState::update_account(
const evmc::address& address,
std::optional<Account> original,
@@ -117,11 +122,11 @@ void DomainState::update_account(
if (current) {
if (!original || current->rlp({}) != original->rlp({})) {
AccountsDomainPutQuery query{database_, tx_};
query.exec(address, *current, txn_id_, original, Step::from_txn_id(txn_id_));
query.exec(address, *current, txn_id_, original, current_step());
}
} else {
AccountsDomainDeleteQuery query{database_, tx_};
query.exec(address, txn_id_, original, Step::from_txn_id(txn_id_));
query.exec(address, txn_id_, original, current_step());
}
}

@@ -139,7 +144,7 @@ void DomainState::update_account_code(
}

CodeDomainPutQuery query{database_, tx_};
query.exec(address, code, txn_id_, original_code, Step::from_txn_id(txn_id_));
query.exec(address, code, txn_id_, original_code, current_step());
}

void DomainState::update_storage(
@@ -161,7 +166,7 @@ void DomainState::update_storage(
}

StorageDomainPutQuery query{database_, tx_};
query.exec({address, location}, current, txn_id_, original_value, Step::from_txn_id(txn_id_));
query.exec({address, location}, current, txn_id_, original_value, current_step());
}

} // namespace silkworm::execution
3 changes: 3 additions & 0 deletions silkworm/execution/domain_state.hpp
Original file line number Diff line number Diff line change
@@ -27,6 +27,7 @@
#include <silkworm/core/state/state.hpp>
#include <silkworm/db/access_layer.hpp>
#include <silkworm/db/data_store.hpp>
#include <silkworm/db/datastore/common/step.hpp>
#include <silkworm/db/datastore/kvdb/mdbx.hpp>
#include <silkworm/execution/remote_state.hpp>

@@ -112,6 +113,8 @@ class DomainState : public State {
void unwind_state_changes(BlockNum /*block_num*/) override {}

private:
datastore::Step current_step() const;

TxnId txn_id_;
datastore::kvdb::RWTxn& tx_;
datastore::kvdb::DatabaseRef& database_;
7 changes: 4 additions & 3 deletions silkworm/execution/domain_state_test.cpp
Original file line number Diff line number Diff line change
@@ -22,6 +22,7 @@

#include <silkworm/core/common/base.hpp>
#include <silkworm/core/common/bytes.hpp>
#include <silkworm/db/state/step_txn_id_converter.hpp>
#include <silkworm/db/tables.hpp>
#include <silkworm/db/test_util/mock_txn.hpp>
#include <silkworm/db/test_util/test_database_context.hpp>
@@ -156,7 +157,7 @@ TEST_CASE("DomainState data access", "[execution][domain][state]") {
CHECK(account_66_read->nonce == account_66_v2.nonce);
CHECK(account_66_read->balance == account_66_v2.balance);

auto next_step_txn_id = silkworm::datastore::kStepSizeForTemporalSnapshots + 1;
auto next_step_txn_id = db::state::kStepSizeForTemporalSnapshots + 1;
auto sut2 = DomainState{next_step_txn_id, rw_tx, db_ref, ds_context->blocks_repository(), ds_context->state_repository_latest()};
Account account_66_v3{
.nonce = 10,
@@ -215,7 +216,7 @@ TEST_CASE("DomainState data access", "[execution][domain][state]") {
CHECK(!code_66_read.empty());
CHECK(code_66_read == code_66);

auto next_step_txn_id = silkworm::datastore::kStepSizeForTemporalSnapshots + 1;
auto next_step_txn_id = db::state::kStepSizeForTemporalSnapshots + 1;
auto sut2 = DomainState{next_step_txn_id, rw_tx, db_ref, ds_context->blocks_repository(), ds_context->state_repository_latest()};
code_66 = *from_hex("0x6044");
code_hash_66 = std::bit_cast<evmc_bytes32>(keccak256(code_66));
@@ -272,7 +273,7 @@ TEST_CASE("DomainState data access", "[execution][domain][state]") {
0x0100_bytes32);
CHECK(storage_66_01 == 0x0124_bytes32);

auto next_step_txn_id = silkworm::datastore::kStepSizeForTemporalSnapshots + 1;
auto next_step_txn_id = db::state::kStepSizeForTemporalSnapshots + 1;
auto sut2 = DomainState{next_step_txn_id, rw_tx, db_ref, ds_context->blocks_repository(), ds_context->state_repository_latest()};
sut2.update_storage(
0x668bdf435d810c91414ec09147daa6db62406379_address,