Skip to content

Commit 0936264

Browse files
marta-lokhovamarta-lokhova
committed
Address the migration TODO
1 parent 1792481 commit 0936264

File tree

5 files changed

+104
-31
lines changed

5 files changed

+104
-31
lines changed

‎src/database/Database.cpp

+3
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,9 @@ Database::applySchemaUpgrade(unsigned long vers)
216216
mApp.getHistoryManager().dropSQLBasedPublish();
217217
Upgrades::dropSupportUpgradeHistory(*this);
218218
break;
219+
case 24:
220+
mApp.getPersistentState().migrateToSlotStateTable();
221+
break;
219222
default:
220223
throw std::runtime_error("Unknown DB schema version");
221224
}

‎src/database/Database.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ using PreparedStatementCache =
3232

3333
// smallest schema version supported
3434
static constexpr unsigned long MIN_SCHEMA_VERSION = 21;
35-
static constexpr unsigned long SCHEMA_VERSION = 23;
35+
static constexpr unsigned long SCHEMA_VERSION = 24;
3636

3737
/**
3838
* Helper class for borrowing a SOCI prepared statement handle into a local

‎src/herder/HerderImpl.cpp

+3-3
Original file line numberDiff line numberDiff line change
@@ -2008,7 +2008,7 @@ HerderImpl::restoreSCPState()
20082008

20092009
// Load all known tx sets
20102010
auto latestTxSets = mApp.getPersistentState().getTxSetsForAllSlots();
2011-
for (auto const& txSet : latestTxSets)
2011+
for (auto const& [_, txSet] : latestTxSets)
20122012
{
20132013
try
20142014
{
@@ -2037,7 +2037,7 @@ HerderImpl::restoreSCPState()
20372037
// load saved state from database
20382038
auto latest64 = mApp.getPersistentState().getSCPStateAllSlots();
20392039

2040-
for (auto const& state : latest64)
2040+
for (auto const& [_, state] : latest64)
20412041
{
20422042
try
20432043
{
@@ -2244,7 +2244,7 @@ HerderImpl::purgeOldPersistedTxSets()
22442244
{
22452245
auto hashesToDelete =
22462246
mApp.getPersistentState().getTxSetHashesForAllSlots();
2247-
for (auto const& state :
2247+
for (auto const& [_, state] :
22482248
mApp.getPersistentState().getSCPStateAllSlots())
22492249
{
22502250
try

‎src/main/PersistentState.cpp

+88-22
Original file line numberDiff line numberDiff line change
@@ -36,24 +36,25 @@ std::string PersistentState::kSQLCreateSCPStatement =
3636
"state TEXT"
3737
"); ";
3838

39-
std::string PersistentState::LCLTableName = "storestate";
40-
std::string PersistentState::SlotTableName = "slotstate";
39+
std::string PersistentState::kLCLTableName = "storestate";
40+
std::string PersistentState::kSlotTableName = "slotstate";
4141

4242
PersistentState::PersistentState(Application& app) : mApp(app)
4343
{
4444
releaseAssert(threadIsMain());
4545
}
4646

4747
void
48-
PersistentState::deleteTxSets(std::unordered_set<Hash> hashesToDelete)
48+
PersistentState::deleteTxSets(std::unordered_set<Hash> hashesToDelete,
49+
std::string table)
4950
{
5051
releaseAssert(threadIsMain());
5152
soci::transaction tx(mApp.getDatabase().getRawSession());
5253
for (auto const& hash : hashesToDelete)
5354
{
5455
auto name = getStoreStateNameForTxSet(hash);
5556
auto prep = mApp.getDatabase().getPreparedStatement(
56-
"DELETE FROM slotstate WHERE statename = :n;",
57+
fmt::format("DELETE FROM {} WHERE statename = :n;", table),
5758
mApp.getDatabase().getSession());
5859

5960
auto& st = prep.statement();
@@ -64,6 +65,66 @@ PersistentState::deleteTxSets(std::unordered_set<Hash> hashesToDelete)
6465
tx.commit();
6566
}
6667

68+
void
69+
PersistentState::migrateToSlotStateTable()
70+
{
71+
// No soci::transaction needed, because the migration in Database.cpp wraps
72+
// everything in one transaction anyway.
73+
releaseAssert(threadIsMain());
74+
auto& db = mApp.getDatabase();
75+
76+
// First, create the new table
77+
db.getRawSession() << PersistentState::kSQLCreateSCPStatement;
78+
79+
// Migrate all the tx sets
80+
auto txSets = getTxSetsForAllSlots(kLCLTableName);
81+
std::unordered_set<Hash> keysToDelete;
82+
for (auto const& txSet : txSets)
83+
{
84+
CLOG_INFO(Herder, "Migrating tx set {} to slotstate",
85+
hexAbbrev(txSet.first));
86+
updateDb(getStoreStateNameForTxSet(txSet.first), txSet.second,
87+
db.getSession(), kSlotTableName);
88+
keysToDelete.insert(txSet.first);
89+
}
90+
91+
// Cleanup tx sets from the previous table
92+
deleteTxSets(keysToDelete, kLCLTableName);
93+
94+
// Migrate all SCP slot data
95+
auto scpStates = getSCPStateAllSlots(kLCLTableName);
96+
for (auto const& [i, scpState] : scpStates)
97+
{
98+
CLOG_INFO(Herder, "Migrating SCP state for slot {} to slotstate", i);
99+
setSCPStateForSlot(i, scpState);
100+
auto prep = mApp.getDatabase().getPreparedStatement(
101+
"DELETE FROM storestate WHERE statename = :n;",
102+
mApp.getDatabase().getSession());
103+
104+
auto& st = prep.statement();
105+
st.exchange(soci::use(getStoreStateName(kLastSCPDataXDR, i)));
106+
st.define_and_bind();
107+
st.execute(true);
108+
}
109+
110+
// Migrate upgrade data
111+
auto upgrades = getFromDb(getStoreStateName(kLedgerUpgrades),
112+
db.getSession(), kLCLTableName);
113+
if (!upgrades.empty())
114+
{
115+
updateDb(getStoreStateName(kLedgerUpgrades), upgrades, db.getSession(),
116+
kSlotTableName);
117+
auto prep = mApp.getDatabase().getPreparedStatement(
118+
"DELETE FROM storestate WHERE statename = :n;",
119+
mApp.getDatabase().getSession());
120+
121+
auto& st = prep.statement();
122+
st.exchange(soci::use(getStoreStateName(kLedgerUpgrades)));
123+
st.define_and_bind();
124+
st.execute(true);
125+
}
126+
}
127+
67128
void
68129
PersistentState::dropAll(Database& db)
69130
{
@@ -72,7 +133,6 @@ PersistentState::dropAll(Database& db)
72133
soci::statement st = db.getRawSession().prepare << kSQLCreateStatement;
73134
st.execute(true);
74135

75-
// TODO: add a migration to move data to slotstate
76136
db.getRawSession() << "DROP TABLE IF EXISTS slotstate;";
77137
soci::statement st2 = db.getRawSession().prepare << kSQLCreateSCPStatement;
78138
st2.execute(true);
@@ -126,7 +186,7 @@ std::string
126186
PersistentState::getDBForEntry(PersistentState::Entry entry)
127187
{
128188
releaseAssert(entry != kLastEntry);
129-
return entry <= kRebuildLedger ? LCLTableName : SlotTableName;
189+
return entry <= kRebuildLedger ? kLCLTableName : kSlotTableName;
130190
}
131191

132192
std::string
@@ -144,21 +204,21 @@ PersistentState::setState(PersistentState::Entry entry,
144204
updateDb(getStoreStateName(entry), value, session, getDBForEntry(entry));
145205
}
146206

147-
std::vector<std::string>
148-
PersistentState::getSCPStateAllSlots()
207+
std::unordered_map<uint32_t, std::string>
208+
PersistentState::getSCPStateAllSlots(std::string table)
149209
{
150210
ZoneScoped;
151211
releaseAssert(threadIsMain());
152212

153213
// Collect all slots persisted
154-
std::vector<std::string> states;
214+
std::unordered_map<uint32_t, std::string> states;
155215
for (uint32 i = 0; i <= mApp.getConfig().MAX_SLOTS_TO_REMEMBER; i++)
156216
{
157217
auto val = getFromDb(getStoreStateName(kLastSCPDataXDR, i),
158-
mApp.getDatabase().getSession(), SlotTableName);
218+
mApp.getDatabase().getSession(), table);
159219
if (!val.empty())
160220
{
161-
states.push_back(val);
221+
states.emplace(i, val);
162222
}
163223
}
164224

@@ -174,7 +234,7 @@ PersistentState::setSCPStateForSlot(uint64 slot, std::string const& value)
174234
auto slotIdx = static_cast<uint32>(
175235
slot % (mApp.getConfig().MAX_SLOTS_TO_REMEMBER + 1));
176236
updateDb(getStoreStateName(kLastSCPDataXDR, slotIdx), value,
177-
mApp.getDatabase().getSession(), SlotTableName);
237+
mApp.getDatabase().getSession(), kSlotTableName);
178238
}
179239

180240
void
@@ -190,7 +250,7 @@ PersistentState::setSCPStateV1ForSlot(
190250
for (auto const& txSet : txSets)
191251
{
192252
updateDb(getStoreStateNameForTxSet(txSet.first), txSet.second,
193-
mApp.getDatabase().getSession(), SlotTableName);
253+
mApp.getDatabase().getSession(), kSlotTableName);
194254
}
195255
tx.commit();
196256
}
@@ -202,7 +262,7 @@ PersistentState::shouldRebuildForType(LedgerEntryType let)
202262
releaseAssert(threadIsMain());
203263

204264
return !getFromDb(getStoreStateName(kRebuildLedger, let),
205-
mApp.getDatabase().getSession(), LCLTableName)
265+
mApp.getDatabase().getSession(), kLCLTableName)
206266
.empty();
207267
}
208268

@@ -213,7 +273,7 @@ PersistentState::clearRebuildForType(LedgerEntryType let)
213273
releaseAssert(threadIsMain());
214274

215275
updateDb(getStoreStateName(kRebuildLedger, let), "",
216-
mApp.getDatabase().getSession(), LCLTableName);
276+
mApp.getDatabase().getSession(), kLCLTableName);
217277
}
218278

219279
void
@@ -230,7 +290,7 @@ PersistentState::setRebuildForType(LedgerEntryType let)
230290
}
231291

232292
updateDb(getStoreStateName(kRebuildLedger, let), "1",
233-
mApp.getDatabase().getSession(), LCLTableName);
293+
mApp.getDatabase().getSession(), kLCLTableName);
234294
}
235295

236296
void
@@ -272,21 +332,23 @@ PersistentState::updateDb(std::string const& entry, std::string const& value,
272332
}
273333
}
274334

275-
std::vector<std::string>
276-
PersistentState::getTxSetsForAllSlots()
335+
std::unordered_map<Hash, std::string>
336+
PersistentState::getTxSetsForAllSlots(std::string table)
277337
{
278338
ZoneScoped;
279339
releaseAssert(threadIsMain());
280340

281-
std::vector<std::string> result;
341+
std::unordered_map<Hash, std::string> result;
342+
std::string key;
282343
std::string val;
283344

284345
std::string pattern = mapping[kTxSet] + "%";
285-
std::string statementStr =
286-
"SELECT state FROM slotstate WHERE statename LIKE :n;";
346+
std::string statementStr = fmt::format(
347+
"SELECT statename, state FROM {} WHERE statename LIKE :n;", table);
287348
auto& db = mApp.getDatabase();
288349
auto prep = db.getPreparedStatement(statementStr, db.getSession());
289350
auto& st = prep.statement();
351+
st.exchange(soci::into(key));
290352
st.exchange(soci::into(val));
291353
st.exchange(soci::use(pattern));
292354
st.define_and_bind();
@@ -295,9 +357,13 @@ PersistentState::getTxSetsForAllSlots()
295357
st.execute(true);
296358
}
297359

360+
Hash hash;
361+
size_t len = binToHex(hash).size();
362+
298363
while (st.got_data())
299364
{
300-
result.push_back(val);
365+
result.emplace(hexToBin256(key.substr(mapping[kTxSet].size(), len)),
366+
val);
301367
st.fetch();
302368
}
303369

‎src/main/PersistentState.h

+9-5
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,10 @@ class PersistentState
4444
SessionWrapper& session);
4545

4646
// Special methods for SCP state (multiple slots)
47-
std::vector<std::string> getSCPStateAllSlots();
48-
std::vector<std::string> getTxSetsForAllSlots();
47+
std::unordered_map<uint32_t, std::string>
48+
getSCPStateAllSlots(std::string table = kSlotTableName);
49+
std::unordered_map<Hash, std::string>
50+
getTxSetsForAllSlots(std::string table = kSlotTableName);
4951
std::unordered_set<Hash> getTxSetHashesForAllSlots();
5052

5153
void
@@ -57,14 +59,16 @@ class PersistentState
5759
void setRebuildForType(LedgerEntryType let);
5860

5961
bool hasTxSet(Hash const& txSetHash);
60-
void deleteTxSets(std::unordered_set<Hash> hashesToDelete);
62+
void deleteTxSets(std::unordered_set<Hash> hashesToDelete,
63+
std::string table = kSlotTableName);
64+
void migrateToSlotStateTable();
6165

6266
private:
6367
static std::string kSQLCreateStatement;
6468
static std::string kSQLCreateSCPStatement;
6569
static std::string mapping[kLastEntry];
66-
static std::string LCLTableName;
67-
static std::string SlotTableName;
70+
static std::string kLCLTableName;
71+
static std::string kSlotTableName;
6872

6973
Application& mApp;
7074

0 commit comments

Comments
 (0)