Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit afe423d

Browse files
SirTysonSirTyson
committedDec 18, 2024··
Evict persistent entries to Hot Archive
1 parent b003246 commit afe423d

20 files changed

+534
-173
lines changed
 

‎src/bucket/BucketIndexImpl.cpp

+12-3
Original file line numberDiff line numberDiff line change
@@ -589,9 +589,18 @@ BucketIndexImpl<IndexT>::operator==(BucketIndex const& inRaw) const
589589

590590
if constexpr (std::is_same_v<IndexT, RangeIndex>)
591591
{
592-
releaseAssert(mData.filter);
593-
releaseAssert(in.mData.filter);
594-
if (!(*(mData.filter) == *(in.mData.filter)))
592+
if (!mData.filter)
593+
{
594+
if (in.mData.filter)
595+
{
596+
return false;
597+
}
598+
}
599+
else if (!in.mData.filter)
600+
{
601+
return false;
602+
}
603+
else if (!(*(mData.filter) == *(in.mData.filter)))
595604
{
596605
return false;
597606
}

‎src/bucket/BucketManager.cpp

+36-16
Original file line numberDiff line numberDiff line change
@@ -1048,7 +1048,8 @@ BucketManager::maybeSetIndex(std::shared_ptr<BucketBase> b,
10481048
}
10491049

10501050
void
1051-
BucketManager::startBackgroundEvictionScan(uint32_t ledgerSeq)
1051+
BucketManager::startBackgroundEvictionScan(uint32_t ledgerSeq,
1052+
uint32_t ledgerVers)
10521053
{
10531054
releaseAssert(mSnapshotManager);
10541055
releaseAssert(!mEvictionFuture.valid());
@@ -1064,9 +1065,10 @@ BucketManager::startBackgroundEvictionScan(uint32_t ledgerSeq)
10641065
// copy this lambda, otherwise we could use unique_ptr.
10651066
auto task = std::make_shared<task_t>(
10661067
[bl = std::move(searchableBL), iter = cfg.evictionIterator(), ledgerSeq,
1067-
sas, &counters = mBucketListEvictionCounters,
1068+
ledgerVers, sas, &counters = mBucketListEvictionCounters,
10681069
stats = mEvictionStatistics] {
1069-
return bl->scanForEviction(ledgerSeq, counters, iter, stats, sas);
1070+
return bl->scanForEviction(ledgerSeq, counters, iter, stats, sas,
1071+
ledgerVers);
10701072
});
10711073

10721074
mEvictionFuture = task->get_future();
@@ -1075,18 +1077,19 @@ BucketManager::startBackgroundEvictionScan(uint32_t ledgerSeq)
10751077
"SearchableLiveBucketListSnapshot: eviction scan");
10761078
}
10771079

1078-
void
1080+
std::pair<std::vector<LedgerKey>, std::vector<LedgerEntry>>
10791081
BucketManager::resolveBackgroundEvictionScan(AbstractLedgerTxn& ltx,
10801082
uint32_t ledgerSeq,
1081-
LedgerKeySet const& modifiedKeys)
1083+
LedgerKeySet const& modifiedKeys,
1084+
uint32_t ledgerVers)
10821085
{
10831086
ZoneScoped;
10841087
releaseAssert(threadIsMain());
10851088
releaseAssert(mEvictionStatistics);
10861089

10871090
if (!mEvictionFuture.valid())
10881091
{
1089-
startBackgroundEvictionScan(ledgerSeq);
1092+
startBackgroundEvictionScan(ledgerSeq, ledgerVers);
10901093
}
10911094

10921095
auto evictionCandidates = mEvictionFuture.get();
@@ -1099,44 +1102,60 @@ BucketManager::resolveBackgroundEvictionScan(AbstractLedgerTxn& ltx,
10991102
if (!evictionCandidates.isValid(ledgerSeq,
11001103
networkConfig.stateArchivalSettings()))
11011104
{
1102-
startBackgroundEvictionScan(ledgerSeq);
1105+
startBackgroundEvictionScan(ledgerSeq, ledgerVers);
11031106
evictionCandidates = mEvictionFuture.get();
11041107
}
11051108

1106-
auto& eligibleKeys = evictionCandidates.eligibleKeys;
1109+
auto& eligibleEntries = evictionCandidates.eligibleEntries;
11071110

1108-
for (auto iter = eligibleKeys.begin(); iter != eligibleKeys.end();)
1111+
for (auto iter = eligibleEntries.begin(); iter != eligibleEntries.end();)
11091112
{
11101113
// If the TTL has not been modified this ledger, we can evict the entry
1111-
if (modifiedKeys.find(getTTLKey(iter->key)) == modifiedKeys.end())
1114+
if (modifiedKeys.find(getTTLKey(iter->entry)) == modifiedKeys.end())
11121115
{
11131116
++iter;
11141117
}
11151118
else
11161119
{
1117-
iter = eligibleKeys.erase(iter);
1120+
iter = eligibleEntries.erase(iter);
11181121
}
11191122
}
11201123

11211124
auto remainingEntriesToEvict =
11221125
networkConfig.stateArchivalSettings().maxEntriesToArchive;
1123-
auto entryToEvictIter = eligibleKeys.begin();
1126+
auto entryToEvictIter = eligibleEntries.begin();
11241127
auto newEvictionIterator = evictionCandidates.endOfRegionIterator;
11251128

1129+
// Return vectors include both evicted entry and associated TTL
1130+
std::vector<LedgerKey> deletedKeys;
1131+
std::vector<LedgerEntry> archivedEntries;
1132+
11261133
// Only actually evict up to maxEntriesToArchive of the eligible entries
11271134
while (remainingEntriesToEvict > 0 &&
1128-
entryToEvictIter != eligibleKeys.end())
1135+
entryToEvictIter != eligibleEntries.end())
11291136
{
1130-
ltx.erase(entryToEvictIter->key);
1131-
ltx.erase(getTTLKey(entryToEvictIter->key));
1137+
ltx.erase(LedgerEntryKey(entryToEvictIter->entry));
1138+
ltx.erase(getTTLKey(entryToEvictIter->entry));
11321139
--remainingEntriesToEvict;
11331140

1141+
if (isTemporaryEntry(entryToEvictIter->entry.data))
1142+
{
1143+
deletedKeys.emplace_back(LedgerEntryKey(entryToEvictIter->entry));
1144+
}
1145+
else
1146+
{
1147+
archivedEntries.emplace_back(entryToEvictIter->entry);
1148+
}
1149+
1150+
// Delete TTL for both types
1151+
deletedKeys.emplace_back(getTTLKey(entryToEvictIter->entry));
1152+
11341153
auto age = ledgerSeq - entryToEvictIter->liveUntilLedger;
11351154
mEvictionStatistics->recordEvictedEntry(age);
11361155
mBucketListEvictionCounters.entriesEvicted.inc();
11371156

11381157
newEvictionIterator = entryToEvictIter->iter;
1139-
entryToEvictIter = eligibleKeys.erase(entryToEvictIter);
1158+
entryToEvictIter = eligibleEntries.erase(entryToEvictIter);
11401159
}
11411160

11421161
// If remainingEntriesToEvict == 0, that means we could not evict the entire
@@ -1149,6 +1168,7 @@ BucketManager::resolveBackgroundEvictionScan(AbstractLedgerTxn& ltx,
11491168
}
11501169

11511170
networkConfig.updateEvictionIterator(ltx, newEvictionIterator);
1171+
return {deletedKeys, archivedEntries};
11521172
}
11531173

11541174
medida::Meter&

‎src/bucket/BucketManager.h

+11-4
Original file line numberDiff line numberDiff line change
@@ -292,10 +292,17 @@ class BucketManager : NonMovableOrCopyable
292292
// Scans BucketList for non-live entries to evict starting at the entry
293293
// pointed to by EvictionIterator. Evicts until `maxEntriesToEvict` entries
294294
// have been evicted or maxEvictionScanSize bytes have been scanned.
295-
void startBackgroundEvictionScan(uint32_t ledgerSeq);
296-
void resolveBackgroundEvictionScan(AbstractLedgerTxn& ltx,
297-
uint32_t ledgerSeq,
298-
LedgerKeySet const& modifiedKeys);
295+
void startBackgroundEvictionScan(uint32_t ledgerSeq, uint32_t ledgerVers);
296+
297+
// Returns a pair of vectors representing entries evicted this ledger, where
298+
// the first vector constains all deleted keys (TTL and temporary), and
299+
// the second vector contains all archived keys (persistent and
300+
// ContractCode). Note that when an entry is archived, its TTL key will be
301+
// included in the deleted keys vector.
302+
std::pair<std::vector<LedgerKey>, std::vector<LedgerEntry>>
303+
resolveBackgroundEvictionScan(AbstractLedgerTxn& ltx, uint32_t ledgerSeq,
304+
LedgerKeySet const& modifiedKeys,
305+
uint32_t ledgerVers);
299306

300307
medida::Meter& getBloomMissMeter() const;
301308
medida::Meter& getBloomLookupMeter() const;

‎src/bucket/BucketOutputIterator.cpp

+1
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ BucketOutputIterator<BucketT>::BucketOutputIterator(std::string const& tmpDir,
6464
HotArchiveBucketEntry bme;
6565
bme.type(HOT_ARCHIVE_METAENTRY);
6666
bme.metaEntry() = mMeta;
67+
releaseAssertOrThrow(bme.metaEntry().ext.v() == 1);
6768
put(bme);
6869
}
6970

‎src/bucket/BucketSnapshot.cpp

+19-5
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#include "bucket/SearchableBucketList.h"
1010
#include "ledger/LedgerTxn.h"
1111
#include "ledger/LedgerTypeUtils.h"
12+
#include "util/ProtocolVersion.h"
1213
#include "util/XDRStream.h"
1314
#include <type_traits>
1415

@@ -180,7 +181,7 @@ Loop
180181
LiveBucketSnapshot::scanForEviction(
181182
EvictionIterator& iter, uint32_t& bytesToScan, uint32_t ledgerSeq,
182183
std::list<EvictionResultEntry>& evictableKeys,
183-
SearchableLiveBucketListSnapshot& bl) const
184+
SearchableLiveBucketListSnapshot& bl, uint32_t ledgerVers) const
184185
{
185186
ZoneScoped;
186187
if (isEmpty() || protocolVersionIsBefore(mBucket->getBucketVersion(),
@@ -205,7 +206,7 @@ LiveBucketSnapshot::scanForEviction(
205206
for (auto& e : maybeEvictQueue)
206207
{
207208
// If TTL entry has not yet been deleted
208-
if (auto ttl = loadResult.find(getTTLKey(e.key))->second;
209+
if (auto ttl = loadResult.find(getTTLKey(e.entry))->second;
209210
ttl != nullptr)
210211
{
211212
// If TTL of entry is expired
@@ -220,6 +221,20 @@ LiveBucketSnapshot::scanForEviction(
220221
}
221222
};
222223

224+
// Start evicting persistent entries in p23
225+
auto isEvictableType = [ledgerVers](auto const& le) {
226+
if (protocolVersionIsBefore(
227+
ledgerVers,
228+
BucketBase::FIRST_PROTOCOL_SUPPORTING_PERSISTENT_EVICTION))
229+
{
230+
return isTemporaryEntry(le);
231+
}
232+
else
233+
{
234+
return isSorobanEntry(le);
235+
}
236+
};
237+
223238
// Open new stream for eviction scan to not interfere with BucketListDB load
224239
// streams
225240
XDRInputFileStream stream{};
@@ -241,14 +256,13 @@ LiveBucketSnapshot::scanForEviction(
241256
if (be.type() == INITENTRY || be.type() == LIVEENTRY)
242257
{
243258
auto const& le = be.liveEntry();
244-
if (isTemporaryEntry(le.data))
259+
if (isEvictableType(le.data))
245260
{
246261
keysToSearch.emplace(getTTLKey(le));
247262

248263
// Set lifetime to 0 as default, will be updated after TTL keys
249264
// loaded
250-
maybeEvictQueue.emplace_back(
251-
EvictionResultEntry(LedgerEntryKey(le), iter, 0));
265+
maybeEvictQueue.emplace_back(EvictionResultEntry(le, iter, 0));
252266
}
253267
}
254268

‎src/bucket/BucketSnapshot.h

+2-1
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,8 @@ class LiveBucketSnapshot : public BucketSnapshotBase<LiveBucket>
8585
Loop scanForEviction(EvictionIterator& iter, uint32_t& bytesToScan,
8686
uint32_t ledgerSeq,
8787
std::list<EvictionResultEntry>& evictableKeys,
88-
SearchableLiveBucketListSnapshot& bl) const;
88+
SearchableLiveBucketListSnapshot& bl,
89+
uint32_t ledgerVers) const;
8990
};
9091

9192
class HotArchiveBucketSnapshot : public BucketSnapshotBase<HotArchiveBucket>

‎src/bucket/BucketUtils.h

+5-5
Original file line numberDiff line numberDiff line change
@@ -72,22 +72,22 @@ struct MergeCounters
7272
// iterator as if that key was the last entry evicted
7373
struct EvictionResultEntry
7474
{
75-
LedgerKey key;
75+
LedgerEntry entry;
7676
EvictionIterator iter;
7777
uint32_t liveUntilLedger;
7878

79-
EvictionResultEntry(LedgerKey const& key, EvictionIterator const& iter,
79+
EvictionResultEntry(LedgerEntry const& entry, EvictionIterator const& iter,
8080
uint32_t liveUntilLedger)
81-
: key(key), iter(iter), liveUntilLedger(liveUntilLedger)
81+
: entry(entry), iter(iter), liveUntilLedger(liveUntilLedger)
8282
{
8383
}
8484
};
8585

8686
struct EvictionResult
8787
{
88-
// List of keys eligible for eviction in the order in which they occur in
88+
// List of entries eligible for eviction in the order in which they occur in
8989
// the bucket
90-
std::list<EvictionResultEntry> eligibleKeys{};
90+
std::list<EvictionResultEntry> eligibleEntries{};
9191

9292
// Eviction iterator at the end of the scan region
9393
EvictionIterator endOfRegionIterator;

‎src/bucket/SearchableBucketList.cpp

+3-2
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ EvictionResult
1515
SearchableLiveBucketListSnapshot::scanForEviction(
1616
uint32_t ledgerSeq, EvictionCounters& counters,
1717
EvictionIterator evictionIter, std::shared_ptr<EvictionStatistics> stats,
18-
StateArchivalSettings const& sas)
18+
StateArchivalSettings const& sas, uint32_t ledgerVers)
1919
{
2020
releaseAssert(mSnapshot);
2121
releaseAssert(stats);
@@ -42,7 +42,8 @@ SearchableLiveBucketListSnapshot::scanForEviction(
4242

4343
// If we scan scanSize before hitting bucket EOF, exit early
4444
if (b.scanForEviction(evictionIter, scanSize, ledgerSeq,
45-
result.eligibleKeys, *this) == Loop::COMPLETE)
45+
result.eligibleEntries, *this,
46+
ledgerVers) == Loop::COMPLETE)
4647
{
4748
break;
4849
}

‎src/bucket/SearchableBucketList.h

+2-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,8 @@ class SearchableLiveBucketListSnapshot
3232
EvictionCounters& counters,
3333
EvictionIterator evictionIter,
3434
std::shared_ptr<EvictionStatistics> stats,
35-
StateArchivalSettings const& sas);
35+
StateArchivalSettings const& sas,
36+
uint32_t ledgerVers);
3637

3738
friend std::shared_ptr<SearchableLiveBucketListSnapshot>
3839
BucketSnapshotManager::copySearchableLiveBucketListSnapshot() const;

‎src/bucket/test/BucketIndexTests.cpp

+13-3
Original file line numberDiff line numberDiff line change
@@ -617,8 +617,18 @@ TEST_CASE("serialize bucket indexes", "[bucket][bucketindex]")
617617
auto test = BucketIndexTest(cfg, /*levels=*/3);
618618
test.buildGeneralTest();
619619

620-
auto buckets = test.getBM().getBucketListReferencedBuckets();
621-
for (auto const& bucketHash : buckets)
620+
std::set<Hash> liveBuckets;
621+
auto& liveBL = test.getBM().getLiveBucketList();
622+
for (auto i = 0; i < LiveBucketList::kNumLevels; ++i)
623+
{
624+
auto level = liveBL.getLevel(i);
625+
for (auto const& b : {level.getCurr(), level.getSnap()})
626+
{
627+
liveBuckets.emplace(b->getHash());
628+
}
629+
}
630+
631+
for (auto const& bucketHash : liveBuckets)
622632
{
623633
if (isZero(bucketHash))
624634
{
@@ -647,7 +657,7 @@ TEST_CASE("serialize bucket indexes", "[bucket][bucketindex]")
647657
cfg.BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 10;
648658
test.restartWithConfig(cfg);
649659

650-
for (auto const& bucketHash : buckets)
660+
for (auto const& bucketHash : liveBuckets)
651661
{
652662
if (isZero(bucketHash))
653663
{

‎src/bucket/test/BucketListTests.cpp

+88-17
Original file line numberDiff line numberDiff line change
@@ -947,14 +947,19 @@ TEST_CASE_VERSIONS("network config snapshots BucketList size", "[bucketlist]")
947947
});
948948
}
949949

950-
TEST_CASE_VERSIONS("eviction scan", "[bucketlist]")
950+
TEST_CASE_VERSIONS("eviction scan", "[bucketlist][archival]")
951951
{
952952
VirtualClock clock;
953953
Config cfg(getTestConfig());
954954
cfg.USE_CONFIG_FOR_GENESIS = true;
955955

956-
auto app = createTestApplication<BucketTestApplication>(clock, cfg);
957-
for_versions_from(20, *app, [&] {
956+
auto test = [&](Config& cfg) {
957+
auto app = createTestApplication<BucketTestApplication>(clock, cfg);
958+
959+
bool tempOnly = protocolVersionIsBefore(
960+
cfg.TESTING_UPGRADE_LEDGER_PROTOCOL_VERSION,
961+
BucketBase::FIRST_PROTOCOL_SUPPORTING_PERSISTENT_EVICTION);
962+
958963
LedgerManagerForBucketTests& lm = app->getLedgerManager();
959964
auto& bm = app->getBucketManager();
960965
auto& bl = bm.getLiveBucketList();
@@ -1019,12 +1024,19 @@ TEST_CASE_VERSIONS("eviction scan", "[bucketlist]")
10191024
std::set<LedgerKey> tempEntries;
10201025
std::set<LedgerKey> persistentEntries;
10211026
std::vector<LedgerEntry> entries;
1027+
10221028
for (auto& e :
10231029
LedgerTestUtils::generateValidUniqueLedgerEntriesWithTypes(
1024-
{CONTRACT_DATA}, 50))
1030+
{CONTRACT_DATA, CONTRACT_CODE}, 50))
10251031
{
1026-
// Set half of the entries to be persistent, half temporary
1027-
if (tempEntries.empty() || rand_flip())
1032+
if (e.data.type() == CONTRACT_CODE)
1033+
{
1034+
persistentEntries.emplace(LedgerEntryKey(e));
1035+
}
1036+
1037+
// Set half of the contact data entries to be persistent, half
1038+
// temporary
1039+
else if (tempEntries.empty() || rand_flip())
10281040
{
10291041
e.data.contractData().durability = TEMPORARY;
10301042
tempEntries.emplace(LedgerEntryKey(e));
@@ -1058,6 +1070,53 @@ TEST_CASE_VERSIONS("eviction scan", "[bucketlist]")
10581070
closeLedger(*app);
10591071
}
10601072

1073+
auto expectedEvictions = tempEntries.size();
1074+
1075+
if (!tempOnly)
1076+
{
1077+
expectedEvictions += persistentEntries.size();
1078+
}
1079+
1080+
auto checkArchivedBucketList = [&] {
1081+
if (!tempOnly)
1082+
{
1083+
auto archiveSnapshot =
1084+
bm.getBucketSnapshotManager()
1085+
.copySearchableHotArchiveBucketListSnapshot();
1086+
1087+
// Check that persisted entries have been inserted into
1088+
// HotArchive
1089+
for (auto const& k : persistentEntries)
1090+
{
1091+
auto archivedEntry = archiveSnapshot->load(k);
1092+
REQUIRE(archivedEntry);
1093+
1094+
auto seen = false;
1095+
for (auto const& e : entries)
1096+
{
1097+
if (e == archivedEntry->archivedEntry())
1098+
{
1099+
seen = true;
1100+
break;
1101+
}
1102+
}
1103+
REQUIRE(seen);
1104+
1105+
// Make sure TTL keys are not archived
1106+
auto ttl = getTTLKey(k);
1107+
auto archivedTTL = archiveSnapshot->load(ttl);
1108+
REQUIRE(!archivedTTL);
1109+
}
1110+
1111+
// Temp entries should not be archived
1112+
for (auto const& k : tempEntries)
1113+
{
1114+
auto archivedEntry = archiveSnapshot->load(k);
1115+
REQUIRE(!archivedEntry);
1116+
}
1117+
}
1118+
};
1119+
10611120
SECTION("basic eviction test")
10621121
{
10631122
// Set eviction to start at level where the entries
@@ -1069,10 +1128,12 @@ TEST_CASE_VERSIONS("eviction scan", "[bucketlist]")
10691128
closeLedger(*app);
10701129
++ledgerSeq;
10711130
checkIfEntryExists(tempEntries, false);
1072-
checkIfEntryExists(persistentEntries, true);
1131+
checkIfEntryExists(persistentEntries, tempOnly);
10731132

10741133
auto& entriesEvictedCounter = bm.getEntriesEvictedCounter();
1075-
REQUIRE(entriesEvictedCounter.count() == tempEntries.size());
1134+
1135+
REQUIRE(entriesEvictedCounter.count() == expectedEvictions);
1136+
checkArchivedBucketList();
10761137

10771138
// Close ledgers until evicted DEADENTRYs merge with
10781139
// original INITENTRYs. This checks that BucketList
@@ -1086,7 +1147,7 @@ TEST_CASE_VERSIONS("eviction scan", "[bucketlist]")
10861147
closeLedger(*app);
10871148
}
10881149

1089-
REQUIRE(entriesEvictedCounter.count() == tempEntries.size());
1150+
REQUIRE(entriesEvictedCounter.count() == expectedEvictions);
10901151
}
10911152

10921153
SECTION("shadowed entries not evicted")
@@ -1127,7 +1188,7 @@ TEST_CASE_VERSIONS("eviction scan", "[bucketlist]")
11271188
auto& entriesEvictedCounter = bm.getEntriesEvictedCounter();
11281189
auto prevIter = evictionIter;
11291190
for (auto prevCount = entriesEvictedCounter.count();
1130-
prevCount < tempEntries.size();)
1191+
prevCount < expectedEvictions;)
11311192
{
11321193
closeLedger(*app);
11331194

@@ -1150,12 +1211,12 @@ TEST_CASE_VERSIONS("eviction scan", "[bucketlist]")
11501211

11511212
// All entries should have been evicted
11521213
checkIfEntryExists(tempEntries, false);
1153-
checkIfEntryExists(persistentEntries, true);
1214+
checkIfEntryExists(persistentEntries, tempOnly);
1215+
checkArchivedBucketList();
11541216
}
11551217

11561218
SECTION("maxEntriesToArchive with entry modified on eviction ledger")
11571219
{
1158-
11591220
// This test is for an edge case in background eviction.
11601221
// We want to test that if entry n should be the last entry
11611222
// evicted due to maxEntriesToArchive, but that entry is
@@ -1172,17 +1233,25 @@ TEST_CASE_VERSIONS("eviction scan", "[bucketlist]")
11721233
LedgerKey entryToEvict;
11731234
std::optional<uint64_t> expectedEndIterPosition{};
11741235

1236+
auto willBeEvicited = [&](LedgerEntry const& le) {
1237+
if (tempOnly)
1238+
{
1239+
return isTemporaryEntry(le.data);
1240+
}
1241+
else
1242+
{
1243+
return isSorobanEntry(le.data);
1244+
}
1245+
};
1246+
11751247
for (LiveBucketInputIterator in(bl.getLevel(levelToScan).getCurr());
11761248
in; ++in)
11771249
{
1178-
// Temp entries should be sorted before persistent in
1179-
// the Bucket
11801250
auto be = *in;
11811251
if (be.type() == INITENTRY || be.type() == LIVEENTRY)
11821252
{
11831253
auto le = be.liveEntry();
1184-
if (le.data.type() == CONTRACT_DATA &&
1185-
le.data.contractData().durability == TEMPORARY)
1254+
if (willBeEvicited(le))
11861255
{
11871256
if (!entryToUpdate)
11881257
{
@@ -1388,7 +1457,9 @@ TEST_CASE_VERSIONS("eviction scan", "[bucketlist]")
13881457
testIterReset(false);
13891458
}
13901459
}
1391-
});
1460+
};
1461+
1462+
for_versions(20, Config::CURRENT_LEDGER_PROTOCOL_VERSION, cfg, test);
13921463
}
13931464

13941465
TEST_CASE_VERSIONS("Searchable BucketListDB snapshots", "[bucketlist]")

‎src/bucket/test/BucketTestUtils.cpp

+15-4
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
#include "ledger/LedgerTxn.h"
1313
#include "main/Application.h"
1414
#include "test/test.h"
15+
#include "util/ProtocolVersion.h"
1516
#include "xdr/Stellar-ledger.h"
1617
#include <memory>
1718

@@ -228,14 +229,24 @@ LedgerManagerForBucketTests::transferLedgerEntriesToBucketList(
228229
}
229230

230231
LedgerTxn ltxEvictions(ltx);
231-
mApp.getBucketManager().resolveBackgroundEvictionScan(
232-
ltxEvictions, lh.ledgerSeq, keys);
233232

233+
auto evictedEntries =
234+
mApp.getBucketManager().resolveBackgroundEvictionScan(
235+
ltxEvictions, lh.ledgerSeq, keys, initialLedgerVers);
236+
237+
if (protocolVersionStartsFrom(
238+
initialLedgerVers,
239+
BucketBase::
240+
FIRST_PROTOCOL_SUPPORTING_PERSISTENT_EVICTION))
241+
{
242+
mApp.getBucketManager().addHotArchiveBatch(
243+
mApp, lh, evictedEntries.second, {}, {});
244+
}
234245
if (ledgerCloseMeta)
235246
{
236-
ledgerCloseMeta->populateEvictedEntries(
237-
ltxEvictions.getChanges());
247+
ledgerCloseMeta->populateEvictedEntries(evictedEntries);
238248
}
249+
239250
ltxEvictions.commit();
240251
}
241252
mApp.getLedgerManager()

‎src/ledger/LedgerCloseMetaFrame.cpp

+14-19
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
#include "transactions/TransactionMetaFrame.h"
99
#include "util/GlobalChecks.h"
1010
#include "util/ProtocolVersion.h"
11+
#include "xdr/Stellar-ledger.h"
1112

1213
namespace stellar
1314
{
@@ -145,28 +146,22 @@ LedgerCloseMetaFrame::populateTxSet(TxSetXDRFrame const& txSet)
145146

146147
void
147148
LedgerCloseMetaFrame::populateEvictedEntries(
148-
LedgerEntryChanges const& evictionChanges)
149+
std::pair<std::vector<LedgerKey>, std::vector<LedgerEntry>> const&
150+
evictedEntries)
149151
{
150152
releaseAssert(mVersion == 1);
151-
for (auto const& change : evictionChanges)
153+
for (auto const& key : evictedEntries.first)
152154
{
153-
switch (change.type())
154-
{
155-
case LEDGER_ENTRY_CREATED:
156-
throw std::runtime_error("unexpected create in eviction meta");
157-
case LEDGER_ENTRY_STATE:
158-
continue;
159-
case LEDGER_ENTRY_UPDATED:
160-
// The scan also updates the eviction iterator, but should only
161-
// update the eviction iterator
162-
releaseAssert(change.updated().data.type() == CONFIG_SETTING);
163-
continue;
164-
case LEDGER_ENTRY_REMOVED:
165-
auto const& key = change.removed();
166-
releaseAssert(isTemporaryEntry(key) || key.type() == TTL);
167-
mLedgerCloseMeta.v1().evictedTemporaryLedgerKeys.push_back(key);
168-
break;
169-
}
155+
releaseAssertOrThrow(isTemporaryEntry(key) || key.type() == TTL);
156+
mLedgerCloseMeta.v1().evictedTemporaryLedgerKeys.emplace_back(key);
157+
}
158+
for (auto const& entry : evictedEntries.second)
159+
{
160+
releaseAssertOrThrow(isPersistentEntry(entry.data));
161+
// Unfortunately, for legacy purposes, evictedTemporaryLedgerKeys is
162+
// misnamed and stores all evicted keys, both temp and persistent.
163+
mLedgerCloseMeta.v1().evictedTemporaryLedgerKeys.emplace_back(
164+
LedgerEntryKey(entry));
170165
}
171166
}
172167

‎src/ledger/LedgerCloseMetaFrame.h

+4-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,10 @@ class LedgerCloseMetaFrame
3030

3131
void populateTxSet(TxSetXDRFrame const& txSet);
3232

33-
void populateEvictedEntries(LedgerEntryChanges const& evictionChanges);
33+
// Used for populating meta from background eviction scan
34+
void populateEvictedEntries(
35+
std::pair<std::vector<LedgerKey>, std::vector<LedgerEntry>> const&
36+
evictedEntries);
3437

3538
void setNetworkConfiguration(SorobanNetworkConfig const& networkConfig,
3639
bool emitExtV1);

‎src/ledger/LedgerManagerImpl.cpp

+16-5
Original file line numberDiff line numberDiff line change
@@ -1039,7 +1039,8 @@ LedgerManagerImpl::closeLedger(LedgerCloseData const& ledgerData)
10391039
// step 5
10401040
if (protocolVersionStartsFrom(initialLedgerVers, SOROBAN_PROTOCOL_VERSION))
10411041
{
1042-
mApp.getBucketManager().startBackgroundEvictionScan(ledgerSeq + 1);
1042+
mApp.getBucketManager().startBackgroundEvictionScan(ledgerSeq + 1,
1043+
initialLedgerVers);
10431044
}
10441045

10451046
// step 6
@@ -1671,14 +1672,24 @@ LedgerManagerImpl::transferLedgerEntriesToBucketList(
16711672
{
16721673
auto keys = ltx.getAllTTLKeysWithoutSealing();
16731674
LedgerTxn ltxEvictions(ltx);
1674-
mApp.getBucketManager().resolveBackgroundEvictionScan(
1675-
ltxEvictions, lh.ledgerSeq, keys);
1675+
1676+
auto evictedEntries =
1677+
mApp.getBucketManager().resolveBackgroundEvictionScan(
1678+
ltxEvictions, lh.ledgerSeq, keys, initialLedgerVers);
1679+
1680+
if (protocolVersionStartsFrom(
1681+
initialLedgerVers,
1682+
BucketBase::FIRST_PROTOCOL_SUPPORTING_PERSISTENT_EVICTION))
1683+
{
1684+
mApp.getBucketManager().addHotArchiveBatch(
1685+
mApp, lh, evictedEntries.second, {}, {});
1686+
}
16761687

16771688
if (ledgerCloseMeta)
16781689
{
1679-
ledgerCloseMeta->populateEvictedEntries(
1680-
ltxEvictions.getChanges());
1690+
ledgerCloseMeta->populateEvictedEntries(evictedEntries);
16811691
}
1692+
16821693
ltxEvictions.commit();
16831694
}
16841695

‎src/test/test.cpp

+31
Original file line numberDiff line numberDiff line change
@@ -551,6 +551,21 @@ for_versions(uint32 from, uint32 to, Config const& cfg,
551551
for_versions(versions, cfg, f);
552552
}
553553

554+
void
555+
for_versions(uint32 from, uint32 to, Config const& cfg,
556+
std::function<void(Config&)> const& f)
557+
{
558+
if (from > to)
559+
{
560+
return;
561+
}
562+
auto versions = std::vector<uint32>{};
563+
versions.resize(to - from + 1);
564+
std::iota(std::begin(versions), std::end(versions), from);
565+
566+
for_versions(versions, cfg, f);
567+
}
568+
554569
void
555570
for_versions(std::vector<uint32> const& versions, Application& app,
556571
std::function<void(void)> const& f)
@@ -585,6 +600,22 @@ for_versions(std::vector<uint32> const& versions, Config const& cfg,
585600
}
586601
}
587602

603+
void
604+
for_versions(std::vector<uint32> const& versions, Config const& cfg,
605+
std::function<void(Config&)> const& f)
606+
{
607+
REQUIRE(gMustUseTestVersionsWrapper);
608+
609+
if (std::find(versions.begin(), versions.end(), gTestingVersion) !=
610+
versions.end())
611+
{
612+
REQUIRE(cfg.TESTING_UPGRADE_LEDGER_PROTOCOL_VERSION == gTestingVersion);
613+
Config vcfg = cfg;
614+
vcfg.LEDGER_PROTOCOL_VERSION = gTestingVersion;
615+
f(vcfg);
616+
}
617+
}
618+
588619
void
589620
for_all_versions_except(std::vector<uint32> const& versions, Application& app,
590621
std::function<void(void)> const& f)

‎src/test/test.h

+6
Original file line numberDiff line numberDiff line change
@@ -72,9 +72,15 @@ void for_versions(std::vector<uint32> const& versions, Application& app,
7272
void for_versions(uint32 from, uint32 to, Config const& cfg,
7373
std::function<void(Config const&)> const& f);
7474

75+
void for_versions(uint32 from, uint32 to, Config const& cfg,
76+
std::function<void(Config&)> const& f);
77+
7578
void for_versions(std::vector<uint32> const& versions, Config const& cfg,
7679
std::function<void(Config const&)> const& f);
7780

81+
void for_versions(std::vector<uint32> const& versions, Config const& cfg,
82+
std::function<void(Config&)> const& f);
83+
7884
void for_all_versions_except(std::vector<uint32> const& versions,
7985
Application& app,
8086
std::function<void(void)> const& f);

‎src/transactions/test/InvokeHostFunctionTests.cpp

+248-85
Large diffs are not rendered by default.

‎test-tx-meta-baseline-current/InvokeHostFunctionTests.json

+3-1
Original file line numberDiff line numberDiff line change
@@ -1324,6 +1324,9 @@
13241324
"bKDF6V5IzTo="
13251325
],
13261326
"contract storage|footprint|unused readWrite key" : [ "VzQb0aWq2bE=" ],
1327+
"entry eviction|protocol version 20" : [ "bKDF6V5IzTo=", "bKDF6V5IzTo=" ],
1328+
"entry eviction|protocol version 21" : [ "bKDF6V5IzTo=", "bKDF6V5IzTo=" ],
1329+
"entry eviction|protocol version 22" : [ "bKDF6V5IzTo=", "bKDF6V5IzTo=" ],
13271330
"failure diagnostics" : [ "bKDF6V5IzTo=" ],
13281331
"ledger entry size limit enforced" : [ "bKDF6V5IzTo=", "bKDF6V5IzTo=" ],
13291332
"loadgen Wasm executes properly" : [ "bKDF6V5IzTo=" ],
@@ -1387,7 +1390,6 @@
13871390
"bKDF6V5IzTo=",
13881391
"bKDF6V5IzTo="
13891392
],
1390-
"temp entry eviction" : [ "bKDF6V5IzTo=", "bKDF6V5IzTo=" ],
13911393
"transaction validation diagnostics" : [ "bKDF6V5IzTo=" ],
13921394
"version test" : [ "766L+TYsWqA=" ]
13931395
}

‎test-tx-meta-baseline-next/InvokeHostFunctionTests.json

+5-1
Original file line numberDiff line numberDiff line change
@@ -1325,6 +1325,11 @@
13251325
"bKDF6V5IzTo="
13261326
],
13271327
"contract storage|footprint|unused readWrite key" : [ "VzQb0aWq2bE=" ],
1328+
"entry eviction|protocol version 20" : [ "bKDF6V5IzTo=", "bKDF6V5IzTo=", "bKDF6V5IzTo=" ],
1329+
"entry eviction|protocol version 21" : [ "bKDF6V5IzTo=", "bKDF6V5IzTo=", "bKDF6V5IzTo=" ],
1330+
"entry eviction|protocol version 22" : [ "bKDF6V5IzTo=", "bKDF6V5IzTo=", "bKDF6V5IzTo=" ],
1331+
"entry eviction|protocol version 23" : [ "bKDF6V5IzTo=", "bKDF6V5IzTo=", "bKDF6V5IzTo=" ],
1332+
"evicted persistent entries" : [ "bKDF6V5IzTo=" ],
13281333
"failure diagnostics" : [ "bKDF6V5IzTo=" ],
13291334
"ledger entry size limit enforced" : [ "bKDF6V5IzTo=", "bKDF6V5IzTo=" ],
13301335
"loadgen Wasm executes properly" : [ "bKDF6V5IzTo=" ],
@@ -1391,7 +1396,6 @@
13911396
"bKDF6V5IzTo=",
13921397
"bKDF6V5IzTo="
13931398
],
1394-
"temp entry eviction" : [ "bKDF6V5IzTo=", "bKDF6V5IzTo=" ],
13951399
"transaction validation diagnostics" : [ "bKDF6V5IzTo=" ],
13961400
"version test" : [ "766L+TYsWqA=" ]
13971401
}

0 commit comments

Comments
 (0)
Please sign in to comment.