Initial support for parallel Soroban phase XDR.

I've tried to minimize the scope of the changes; specifically this doesn't contain any actual logic for the parallel execution (such as data dependency validation and building parallel stages). However, there is still some refactoring that needed to happen in order to support new, more complex tx sets.

I've also removed some legacy surge pricing logic and the corresponding tests - we should no longer need the surge pricing logic that covers more than one source account per tx set.

Author: dmkozh

Builds/VisualStudio/stellar-core.vcxproj

@@ -602,6 +602,7 @@ exit /b 0
     <ClCompile Include="..\..\src\overlay\Floodgate.cpp" />
     <ClCompile Include="..\..\src\overlay\FlowControl.cpp" />
     <ClCompile Include="..\..\src\overlay\FlowControlCapacity.cpp" />
+    <ClCompile Include="..\..\src\overlay\Hmac.cpp" />
     <ClCompile Include="..\..\src\overlay\ItemFetcher.cpp" />
     <ClCompile Include="..\..\src\overlay\OverlayAppConnector.cpp" />
     <ClCompile Include="..\..\src\overlay\OverlayManagerImpl.cpp" />
@@ -1019,6 +1020,7 @@ exit /b 0
     <ClInclude Include="..\..\src\overlay\Floodgate.h" />
     <ClInclude Include="..\..\src\overlay\FlowControl.h" />
     <ClInclude Include="..\..\src\overlay\FlowControlCapacity.h" />
+    <ClInclude Include="..\..\src\overlay\Hmac.h" />
     <ClInclude Include="..\..\src\overlay\ItemFetcher.h" />
     <ClInclude Include="..\..\src\overlay\OverlayAppConnector.h" />
     <ClInclude Include="..\..\src\overlay\OverlayManager.h" />

Builds/VisualStudio/stellar-core.vcxproj.filters

@@ -1344,6 +1344,9 @@
     <ClCompile Include="..\..\src\overlay\TxDemandsManager.cpp">
       <Filter>overlay</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\overlay\Hmac.cpp">
+      <Filter>overlay</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\lib\util\cpptoml.h">
@@ -2339,6 +2342,9 @@
     <ClInclude Include="..\..\src\overlay\TxDemandsManager.h">
       <Filter>overlay</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\src\overlay\Hmac.h">
+      <Filter>overlay</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <None Include="..\..\AUTHORS" />

deny.toml

@@ -51,7 +51,10 @@ exclude = [
    # Somehow the tracking machinery of two different dev-dep tracing
    # subsystems also winds up pulling in conflicts, but again, just
    # dev-deps or non-produciton configs.
-   "tracking-allocator"
+   "tracking-allocator",
+   # Temporary exclusion due to signing-related vulnerability (this doesn't
+   # affect Core as Core doesn't use this for signing)
+   "curve25519-dalek"
 ]
 
 # If true, metadata will be collected with `--all-features`. Note that this can't

src/herder/HerderImpl.cpp

@@ -1305,7 +1305,7 @@ HerderImpl::triggerNextLedger(uint32_t ledgerSeqToTrigger,
     // our first choice for this round's set is all the tx we have collected
     // during last few ledger closes
     auto const& lcl = mLedgerManager.getLastClosedLedgerHeader();
-    TxSetPhaseTransactions txPhases;
+    PerPhaseTransactionList txPhases;
     txPhases.emplace_back(mTransactionQueue.getTransactions(lcl.header));
 
     if (protocolVersionStartsFrom(lcl.header.ledgerVersion,
@@ -1347,7 +1347,7 @@ HerderImpl::triggerNextLedger(uint32_t ledgerSeqToTrigger,
     upperBoundCloseTimeOffset = nextCloseTime - lcl.header.scpValue.closeTime;
     lowerBoundCloseTimeOffset = upperBoundCloseTimeOffset;
 
-    TxSetPhaseTransactions invalidTxPhases;
+    PerPhaseTransactionList invalidTxPhases;
     invalidTxPhases.resize(txPhases.size());
 
     auto [proposedSet, applicableProposedSet] =
@@ -2230,8 +2230,7 @@ HerderImpl::updateTransactionQueue(TxSetXDRFrameConstPtr externalizedTxSet)
 
         auto invalidTxs = TxSetUtils::getInvalidTxList(
             txs, mApp, 0,
-            getUpperBoundCloseTimeOffset(mApp, lhhe.header.scpValue.closeTime),
-            false);
+            getUpperBoundCloseTimeOffset(mApp, lhhe.header.scpValue.closeTime));
         queue.ban(invalidTxs);
 
         queue.rebroadcast();

src/herder/SurgePricingUtils.cpp

@@ -162,7 +162,7 @@ SurgePricingPriorityQueue::getMostTopTxsWithinLimits(
     SurgePricingPriorityQueue queue(
         /* isHighestPriority */ true, laneConfig,
         stellar::rand_uniform<size_t>(0, std::numeric_limits<size_t>::max()));
-    for (auto txStack : txStacks)
+    for (auto const& txStack : txStacks)
     {
         queue.add(txStack);
     }
@@ -185,7 +185,7 @@ SurgePricingPriorityQueue::visitTopTxs(
 {
     ZoneScoped;
 
-    for (auto txStack : txStacks)
+    for (auto const& txStack : txStacks)
     {
         add(txStack);
     }

src/herder/SurgePricingUtils.h

@@ -43,6 +43,48 @@ class TxStack
     virtual ~TxStack() = default;
 };
 
+// A simple stack that holds a single transaction.
+class SingleTxStack : public TxStack
+{
+  public:
+    SingleTxStack(TransactionFrameBasePtr tx,
+                  bool useByteLimitInClassic = false)
+        : mTx(tx), mUseByteLimitInClassic(useByteLimitInClassic)
+    {
+    }
+
+    TransactionFrameBasePtr
+    getTopTx() const override
+    {
+        releaseAssert(mTx);
+        return mTx;
+    }
+
+    void
+    popTopTx() override
+    {
+        releaseAssert(mTx);
+        mTx = nullptr;
+    }
+
+    bool
+    empty() const override
+    {
+        return mTx == nullptr;
+    }
+
+    Resource
+    getResources() const override
+    {
+        releaseAssert(mTx);
+        return Resource(mTx->getResources(mUseByteLimitInClassic));
+    }
+
+  private:
+    TransactionFrameBasePtr mTx;
+    bool mUseByteLimitInClassic = false;
+};
+
 using TxStackPtr = std::shared_ptr<TxStack>;
 
 // Configuration for multi-lane transaction limiting and surge pricing.

src/herder/TransactionQueue.cpp

@@ -814,11 +814,11 @@ TransactionQueue::isBanned(Hash const& hash) const
         });
 }
 
-TxSetTransactions
+TxFrameList
 TransactionQueue::getTransactions(LedgerHeader const& lcl) const
 {
     ZoneScoped;
-    TxSetTransactions txs;
+    TxFrameList txs;
 
     uint32_t const nextLedgerSeq = lcl.ledgerSeq + 1;
     int64_t const startingSeq = getStartingSequenceNumber(nextLedgerSeq);

src/herder/TransactionQueue.h

@@ -125,7 +125,7 @@ class TransactionQueue
 
     bool isBanned(Hash const& hash) const;
     TransactionFrameBaseConstPtr getTx(Hash const& hash) const;
-    TxSetTransactions getTransactions(LedgerHeader const& lcl) const;
+    TxFrameList getTransactions(LedgerHeader const& lcl) const;
     bool sourceAccountPending(AccountID const& accountID) const;
 
     virtual size_t getMaxQueueSizeOps() const = 0;

src/herder/TxQueueLimiter.h

@@ -13,44 +13,6 @@
 namespace stellar
 {
 
-class SingleTxStack : public TxStack
-{
-  public:
-    SingleTxStack(TransactionFrameBasePtr tx) : mTx(tx)
-    {
-    }
-
-    TransactionFrameBasePtr
-    getTopTx() const override
-    {
-        releaseAssert(mTx);
-        return mTx;
-    }
-
-    void
-    popTopTx() override
-    {
-        releaseAssert(mTx);
-        mTx = nullptr;
-    }
-
-    bool
-    empty() const override
-    {
-        return mTx == nullptr;
-    }
-
-    Resource
-    getResources() const override
-    {
-        releaseAssert(mTx);
-        return Resource(mTx->getResources(/* useByteLimitInClassic */ false));
-    }
-
-  private:
-    TransactionFrameBasePtr mTx;
-};
-
 class TxQueueLimiter
 {
     // number of ledgers we can pool in memory