Skip to content

Commit a9bd934

Browse files
battlmonstrbattlmonstr
committed
DomainRangeAsOf
1 parent 1f3cdb0 commit a9bd934

File tree

4 files changed

+217
-54
lines changed

4 files changed

+217
-54
lines changed

‎silkworm/db/datastore/common/ranges/if_view.hpp

+137
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
/*
2+
Copyright 2025 The Silkworm Authors
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
#pragma once
18+
19+
#include <cstdlib>
20+
#include <iterator>
21+
#include <optional>
22+
#include <ranges>
23+
#include <type_traits>
24+
#include <utility>
25+
26+
#include <silkworm/core/common/assert.hpp>
27+
28+
namespace silkworm::views {
29+
30+
template <std::ranges::input_range Range1, std::ranges::input_range Range2>
31+
class IfView : public std::ranges::view_interface<IfView<Range1, Range2>> {
32+
public:
33+
class Iterator {
34+
public:
35+
using Range1Iterator = std::ranges::iterator_t<Range1>;
36+
using Range1Sentinel = std::ranges::sentinel_t<Range1>;
37+
using Range1ReferenceType = std::iter_reference_t<Range1Iterator>;
38+
using Range2Iterator = std::ranges::iterator_t<Range2>;
39+
using Range2Sentinel = std::ranges::sentinel_t<Range2>;
40+
using Range2ReferenceType = std::iter_reference_t<Range2Iterator>;
41+
using DereferenceType = std::conditional_t<!std::is_reference_v<Range1ReferenceType>, Range1ReferenceType, Range2ReferenceType>;
42+
43+
using value_type = std::iter_value_t<Range1Iterator>;
44+
using iterator_category [[maybe_unused]] = std::input_iterator_tag;
45+
using difference_type = std::iter_difference_t<Range1Iterator>;
46+
using reference = DereferenceType;
47+
using pointer = std::remove_reference_t<reference>*;
48+
49+
Iterator() = default;
50+
Iterator(
51+
std::optional<Range1Iterator> it1,
52+
std::optional<Range1Sentinel> sentinel1,
53+
std::optional<Range2Iterator> it2,
54+
std::optional<Range2Sentinel> sentinel2)
55+
: it1_{std::move(it1)},
56+
sentinel1_{std::move(sentinel1)},
57+
it2_{std::move(it2)},
58+
sentinel2_{std::move(sentinel2)} {}
59+
60+
reference operator*() const {
61+
if (it1_) return **it1_;
62+
if (it2_) return **it2_;
63+
SILKWORM_ASSERT(false);
64+
std::abort();
65+
}
66+
67+
Iterator operator++(int) { return std::exchange(*this, ++Iterator{*this}); }
68+
Iterator& operator++() {
69+
if (it1_) ++(*it1_);
70+
if (it2_) ++(*it2_);
71+
return *this;
72+
}
73+
74+
friend bool operator==(const Iterator& it, const std::default_sentinel_t&) {
75+
return (it.it1_ && (*it.it1_ == *it.sentinel1_)) ||
76+
(it.it2_ && (*it.it2_ == *it.sentinel2_));
77+
}
78+
friend bool operator!=(const Iterator& it, const std::default_sentinel_t& s) {
79+
return !(it == s);
80+
}
81+
friend bool operator==(const std::default_sentinel_t& s, const Iterator& it) {
82+
return it == s;
83+
}
84+
friend bool operator!=(const std::default_sentinel_t& s, const Iterator& it) {
85+
return !(it == s);
86+
}
87+
88+
private:
89+
std::optional<Range1Iterator> it1_;
90+
std::optional<Range1Sentinel> sentinel1_;
91+
std::optional<Range2Iterator> it2_;
92+
std::optional<Range2Sentinel> sentinel2_;
93+
};
94+
95+
static_assert(std::input_iterator<Iterator>);
96+
97+
IfView(bool cond, Range1 range1, Range2 range2)
98+
: cond_{cond},
99+
range1_{std::move(range1)},
100+
range2_{std::move(range2)} {}
101+
IfView() = default;
102+
103+
IfView(IfView&&) = default;
104+
IfView& operator=(IfView&&) noexcept = default;
105+
106+
Iterator begin() {
107+
if (cond_) {
108+
return Iterator{
109+
std::ranges::begin(range1_),
110+
std::ranges::end(range1_),
111+
std::nullopt,
112+
std::nullopt,
113+
};
114+
} else {
115+
return Iterator{
116+
std::nullopt,
117+
std::nullopt,
118+
std::ranges::begin(range2_),
119+
std::ranges::end(range2_),
120+
};
121+
}
122+
}
123+
124+
std::default_sentinel_t end() const { return std::default_sentinel; }
125+
126+
private:
127+
bool cond_{false};
128+
Range1 range1_;
129+
Range2 range2_;
130+
};
131+
132+
template <class Range1, class Range2>
133+
IfView<Range1, Range2> if_view(bool cond, Range1&& v1, Range2&& v2) {
134+
return IfView<Range1, Range2>{cond, std::forward<Range1>(v1), std::forward<Range2>(v2)};
135+
}
136+
137+
} // namespace silkworm::views

‎silkworm/db/datastore/common/ranges/vector_from_range.hpp

+9
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,15 @@ namespace silkworm {
2525

2626
template <std::ranges::input_range Range, typename Value = std::iter_value_t<std::ranges::iterator_t<Range>>>
2727
std::vector<Value> vector_from_range(Range range) {
28+
std::vector<Value> results;
29+
for (auto&& value : range) {
30+
results.emplace_back(std::move(value));
31+
}
32+
return results;
33+
}
34+
35+
template <std::ranges::input_range Range, typename Value = std::iter_value_t<std::ranges::iterator_t<Range>>>
36+
std::vector<Value> vector_from_range_copy(Range range) {
2837
std::vector<Value> results;
2938
std::ranges::copy(range, std::back_inserter(results));
3039
return results;

‎silkworm/db/datastore/domain_range_as_of_query.hpp

+15-4
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818

1919
#include <silkworm/core/common/assert.hpp>
2020

21+
#include "common/ranges/if_view.hpp"
22+
#include "common/ranges/merge_unique_view.hpp"
2123
#include "domain_range_latest_query.hpp"
2224
#include "history_range_by_keys_query.hpp"
2325

@@ -50,12 +52,21 @@ struct DomainRangeAsOfQuery {
5052
using ResultItem = typename DomainRangeLatestQuery<TKeyEncoder1, TKeyEncoder2, TKeyDecoder1, TKeyDecoder2, TValueDecoder1, TValueDecoder2>::ResultItem;
5153

5254
auto exec(const Key& key_start, const Key& key_end, std::optional<Timestamp> timestamp, bool ascending) {
53-
SILKWORM_ASSERT(ascending); // descending is not implemented
55+
return silkworm::views::if_view(
56+
!timestamp.has_value(),
57+
query2_.exec(key_start, key_end, ascending),
58+
this->exec(key_start, key_end, timestamp.value_or(0), ascending));
59+
}
5460

55-
[[maybe_unused]] auto historical_results = query1_.exec(key_start, key_end, timestamp.value_or(-1), ascending);
61+
auto exec(const Key& key_start, const Key& key_end, Timestamp timestamp, bool ascending) {
62+
SILKWORM_ASSERT(ascending); // descending is not implemented
5663

57-
// TODO: merge unique with history_range_by_keys_query
58-
return query2_.exec(key_start, key_end, ascending);
64+
return silkworm::views::merge_unique(
65+
query1_.exec(key_start, key_end, timestamp, ascending),
66+
query2_.exec(key_start, key_end, ascending),
67+
silkworm::views::MergeCompareFunc{},
68+
PairGetFirst<typename ResultItem::first_type, typename ResultItem::second_type>{},
69+
PairGetFirst<typename ResultItem::first_type, typename ResultItem::second_type>{});
5970
}
6071

6172
private:

‎silkworm/db/kv/api/local_transaction.cpp

+56-50
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222

2323
#include <silkworm/db/access_layer.hpp>
2424
#include <silkworm/db/chain/local_chain_storage.hpp>
25+
#include <silkworm/db/datastore/common/ranges/vector_from_range.hpp>
2526
#include <silkworm/db/datastore/domain_get_as_of_query.hpp>
2627
#include <silkworm/db/datastore/domain_get_latest_query.hpp>
2728
#include <silkworm/db/datastore/domain_range_as_of_query.hpp>
@@ -39,7 +40,7 @@ namespace silkworm::db::kv::api {
3940

4041
using namespace silkworm::datastore;
4142

42-
static const std::map<std::string_view, datastore::EntityName> kTable2EntityNames{
43+
static const std::map<std::string_view, EntityName> kTable2EntityNames{
4344
{table::kAccountDomain, state::kDomainNameAccounts},
4445
{table::kStorageDomain, state::kDomainNameStorage},
4546
{table::kCodeDomain, state::kDomainNameCode},
@@ -86,7 +87,7 @@ using ReceiptsHistoryGetQuery = RawHistoryGetQuery<state::kHistorySegmentAndIdxN
8687
using RawInvertedIndexRangeByKeyQuery = InvertedIndexRangeByKeyQuery<
8788
kvdb::RawEncoder<Bytes>, snapshots::RawEncoder<Bytes>>; // TODO(canepat) try ByteView
8889

89-
using RawHistoryRangeInPeriodQuery = datastore::HistoryRangeInPeriodQuery<
90+
using RawHistoryRangeInPeriodQuery = HistoryRangeInPeriodQuery<
9091
kvdb::RawDecoder<Bytes>, snapshots::RawDecoder<Bytes>, kvdb::RawDecoder<Bytes>, snapshots::RawDecoder<Bytes>>;
9192

9293
template <typename PageResult>
@@ -156,7 +157,7 @@ Task<GetLatestResult> LocalTransaction::get_latest(GetLatestQuery query) {
156157
co_return GetAsOfResult{};
157158
}
158159

159-
const auto domain_name = kTable2EntityNames.at(query.table);
160+
const EntityName domain_name = kTable2EntityNames.at(query.table);
160161
RawDomainGetLatestQuery store_query(
161162
domain_name,
162163
data_store_.chaindata.domain(domain_name),
@@ -176,7 +177,7 @@ Task<GetAsOfResult> LocalTransaction::get_as_of(GetAsOfQuery query) {
176177
co_return GetAsOfResult{};
177178
}
178179

179-
const auto domain_name = kTable2EntityNames.at(query.table);
180+
const EntityName domain_name = kTable2EntityNames.at(query.table);
180181
std::optional<Bytes> value;
181182
if (domain_name == state::kDomainNameAccounts) {
182183
value = query_domain_as_of<AccountsDomainGetAsOfQuery>(domain_name, query.key, query.timestamp);
@@ -200,8 +201,8 @@ Task<HistoryPointResult> LocalTransaction::history_seek(HistoryPointQuery query)
200201
co_return HistoryPointResult{};
201202
}
202203

203-
const auto domain_name = kTable2EntityNames.at(query.table);
204-
const auto domain = data_store_.chaindata.domain(domain_name);
204+
const EntityName domain_name = kTable2EntityNames.at(query.table);
205+
const kvdb::Domain domain = data_store_.chaindata.domain(domain_name);
205206
if (!domain.history) {
206207
co_return HistoryPointResult{};
207208
}
@@ -232,26 +233,31 @@ Task<PaginatedTimestamps> LocalTransaction::index_range(IndexRangeQuery query) {
232233
}
233234

234235
auto paginator = [this, query = std::move(query)](api::PaginatedTimestamps::PageToken) mutable -> Task<api::PaginatedTimestamps::PageResult> {
235-
datastore::TimestampRange ts_range = as_datastore_ts_range({query.from_timestamp, query.to_timestamp}, !query.ascending_order);
236-
const auto inverted_index_name = kTable2EntityNames.at(query.table);
236+
const EntityName inverted_index_name = kTable2EntityNames.at(query.table);
237237
RawInvertedIndexRangeByKeyQuery store_query{
238238
inverted_index_name,
239239
data_store_.chaindata,
240240
tx_,
241241
data_store_.state_repository_historical,
242242
};
243+
244+
datastore::TimestampRange ts_range = as_datastore_ts_range({query.from_timestamp, query.to_timestamp}, !query.ascending_order);
243245
const size_t limit = (query.limit == kUnlimited) ? std::numeric_limits<size_t>::max() : static_cast<size_t>(query.limit);
246+
244247
api::PaginatedTimestamps::PageResult result;
245248
// TODO: support pagination: apply page_size using std::views::chunk, save the range for future requests using page_token and return the first chunk
246249
if (query.ascending_order) {
247-
auto timestamp_view = store_query.exec<true>(query.key, std::move(ts_range)) | std::views::take(limit);
248-
// TODO: avoid range copy using std::views::as_rvalue (C++23)
249-
std::ranges::copy(timestamp_view /*| std::views::as_rvalue*/, std::back_inserter(result.values));
250+
auto timestamps = store_query.exec<true>(query.key, std::move(ts_range)) |
251+
std::views::transform([](datastore::Timestamp ts) { return static_cast<Timestamp>(ts); }) |
252+
std::views::take(limit);
253+
result.values = vector_from_range(std::move(timestamps));
250254
} else {
251-
auto timestamp_view = store_query.exec<false>(query.key, std::move(ts_range)) | std::views::take(limit);
252-
// TODO: avoid range copy using std::views::as_rvalue (C++23)
253-
std::ranges::copy(timestamp_view /*| std::views::as_rvalue*/, std::back_inserter(result.values));
255+
auto timestamps = store_query.exec<false>(query.key, std::move(ts_range)) |
256+
std::views::transform([](datastore::Timestamp ts) { return static_cast<Timestamp>(ts); }) |
257+
std::views::take(limit);
258+
result.values = vector_from_range(std::move(timestamps));
254259
}
260+
255261
co_return result;
256262
};
257263
co_return api::PaginatedTimestamps{std::move(paginator)};
@@ -263,63 +269,63 @@ Task<PaginatedKeysValues> LocalTransaction::history_range(HistoryRangeQuery quer
263269
}
264270

265271
auto paginator = [this, query = std::move(query)](api::PaginatedKeysValues::PageToken) mutable -> Task<api::PaginatedKeysValues::PageResult> {
266-
datastore::TimestampRange ts_range = as_datastore_ts_range({query.from_timestamp, query.to_timestamp}, !query.ascending_order);
267-
const auto entity_name = kTable2EntityNames.at(query.table);
272+
const EntityName entity_name = kTable2EntityNames.at(query.table);
268273
RawHistoryRangeInPeriodQuery store_query{
269274
entity_name,
270275
data_store_.chaindata,
271276
tx_,
272277
data_store_.state_repository_historical,
273278
};
279+
280+
datastore::TimestampRange ts_range = as_datastore_ts_range({query.from_timestamp, query.to_timestamp}, !query.ascending_order);
274281
const size_t limit = (query.limit == kUnlimited) ? std::numeric_limits<size_t>::max() : static_cast<size_t>(query.limit);
275-
std::vector<RawHistoryRangeInPeriodQuery::ResultItem> kv_pair_sequence;
282+
276283
// TODO: support pagination: apply page_size using std::views::chunk, save the range for future requests using page_token and return the first chunk
277-
auto kv_view = store_query.exec(ts_range, query.ascending_order) | std::views::take(limit);
278-
// TODO: avoid range copy using std::views::as_rvalue (C++23)
279-
std::ranges::copy(kv_view /*| std::views::as_rvalue*/, std::back_inserter(kv_pair_sequence));
280284
api::PaginatedKeysValues::PageResult result;
281-
if (!kv_pair_sequence.empty()) {
282-
result.keys.reserve(kv_pair_sequence.size());
283-
result.values.reserve(kv_pair_sequence.size());
284-
for (auto&& kv_pair : kv_pair_sequence) {
285-
result.keys.emplace_back(std::move(kv_pair.first));
286-
result.values.emplace_back(std::move(kv_pair.second));
287-
}
285+
for (auto&& kv_pair : store_query.exec(ts_range, query.ascending_order) | std::views::take(limit)) {
286+
result.keys.emplace_back(std::move(kv_pair.first));
287+
result.values.emplace_back(std::move(kv_pair.second));
288288
}
289+
289290
co_return result;
290291
};
291292
co_return api::PaginatedKeysValues{std::move(paginator)};
292293
}
293294

294295
Task<PaginatedKeysValues> LocalTransaction::range_as_of(DomainRangeQuery query) {
295-
// convert table to entity name
296296
if (!kTable2EntityNames.contains(query.table)) {
297-
// TODO: return an empty result
297+
co_return api::PaginatedKeysValues{make_empty_paginator<api::PaginatedKeysValues::PageResult>()};
298298
}
299-
datastore::EntityName entity_name = kTable2EntityNames.at(query.table);
300-
301-
using DomainRangeAsOfQuery = datastore::DomainRangeAsOfQuery<
302-
kvdb::RawEncoder<Bytes>, snapshots::RawEncoder<Bytes>,
303-
kvdb::RawDecoder<Bytes>, snapshots::RawDecoder<Bytes>,
304-
kvdb::RawDecoder<Bytes>, snapshots::RawDecoder<Bytes>>;
305-
DomainRangeAsOfQuery store_query{
306-
entity_name,
307-
data_store_.chaindata,
308-
tx_,
309-
data_store_.state_repository_latest,
310-
data_store_.state_repository_historical,
311-
};
312299

313-
size_t limit = (query.limit == kUnlimited) ? std::numeric_limits<size_t>::max() : static_cast<size_t>(query.limit);
300+
auto paginator = [this, query = std::move(query)](api::PaginatedKeysValues::PageToken) mutable -> Task<api::PaginatedKeysValues::PageResult> {
301+
const EntityName entity_name = kTable2EntityNames.at(query.table);
302+
303+
using DomainRangeAsOfQuery = DomainRangeAsOfQuery<
304+
kvdb::RawEncoder<Bytes>, snapshots::RawEncoder<Bytes>,
305+
kvdb::RawDecoder<Bytes>, snapshots::RawDecoder<Bytes>,
306+
kvdb::RawDecoder<Bytes>, snapshots::RawDecoder<Bytes>>;
307+
DomainRangeAsOfQuery store_query{
308+
entity_name,
309+
data_store_.chaindata,
310+
tx_,
311+
data_store_.state_repository_latest,
312+
data_store_.state_repository_historical,
313+
};
314314

315-
// TODO: this is just a test example, instead of direct iteration, apply page_size using std::views::chunk,
316-
// TODO: save the range for future requests using page_token and return the first chunk
317-
for ([[maybe_unused]] decltype(store_query)::ResultItem& kv_pair : store_query.exec(query.from_key, query.to_key, query.timestamp, query.ascending_order) | std::views::take(limit)) {
318-
}
315+
std::optional<datastore::Timestamp> timestamp;
316+
if (query.timestamp && (*query.timestamp >= 0)) {
317+
timestamp = static_cast<datastore::Timestamp>(*query.timestamp);
318+
}
319+
const size_t limit = (query.limit == kUnlimited) ? std::numeric_limits<size_t>::max() : static_cast<size_t>(query.limit);
319320

320-
// TODO(canepat) implement using E3-like aggregator abstraction [tx_id_ must be changed]
321-
auto paginator = [](api::PaginatedKeysValues::PageToken) mutable -> Task<api::PaginatedKeysValues::PageResult> {
322-
co_return api::PaginatedKeysValues::PageResult{};
321+
// TODO: support pagination: apply page_size using std::views::chunk, save the range for future requests using page_token and return the first chunk
322+
api::PaginatedKeysValues::PageResult result;
323+
for (auto&& kv_pair : store_query.exec(query.from_key, query.to_key, timestamp, query.ascending_order) | std::views::take(limit)) {
324+
result.keys.emplace_back(std::move(kv_pair.first));
325+
result.values.emplace_back(std::move(kv_pair.second));
326+
}
327+
328+
co_return result;
323329
};
324330
co_return api::PaginatedKeysValues{std::move(paginator)};
325331
}

0 commit comments

Comments
 (0)