Version ledger tables and encode TxIns in big-endian (#1652)

Closes #1532

Author: jasagredo

ouroboros-consensus-cardano/app/snapshot-converter.hs

@@ -287,7 +287,7 @@ main = withStdTerminalHandles $ do
         let crcOut = maybe inCRC (crcOfConcat inCRC) mCRCOut
 
         lift $ putStr "Generating new metadata file..." >> hFlush stdout
-        putMetadata outFilePath (SnapshotMetadata outBackend crcOut)
+        putMetadata outFilePath (SnapshotMetadata outBackend crcOut TablesCodecVersion1)
 
         lift $ putColored Green True "Done"
 
@@ -356,7 +356,7 @@ main = withStdTerminalHandles $ do
         InEnv
           st
           fp
-          (fromInMemory (fp F.</> "tables" F.</> "tvar"))
+          (fromInMemory (fp F.</> "tables"))
           ("InMemory@[" <> fp <> "]")
           c
           mtd
@@ -412,7 +412,7 @@ main = withStdTerminalHandles $ do
       pure $
         OutEnv
           fp
-          (toInMemory (fp F.</> "tables" F.</> "tvar"))
+          (toInMemory (fp F.</> "tables"))
           (Just "tables")
           (Nothing)
           ("InMemory@[" <> fp <> "]")

ouroboros-consensus-cardano/changelog.d/20251010_111741_javier.sagredo_version_tables.md

@@ -0,0 +1,26 @@
+<!--
+A new scriv changelog fragment.
+
+Uncomment the section that is right (remove the HTML comment wrapper).
+For top level release notes, leave all the headers commented out.
+-->
+
+<!--
+### Patch
+
+- A bullet item for the Patch category.
+
+-->
+<!--
+### Non-Breaking
+
+- A bullet item for the Non-Breaking category.
+
+-->
+
+### Breaking
+
+- Flip serialization of `TxIx` in Mempack, to ensure lexicographic order on the
+  serialized form matches the Haskell Ord, allowing for incremental streaming of
+  values among backends. Note this happens at the same time as the versioning of
+  the LedgerTables codec which will induce a replay of the chain.

ouroboros-consensus-cardano/golden/cardano/disk/LedgerTables_Alonzo

@@ -378,6 +378,7 @@ test-suite shelley-test
     contra-tracer,
     filepath,
     measures,
+    mempack,
     microlens,
     ouroboros-consensus:{ouroboros-consensus, unstable-consensus-testlib},
     ouroboros-consensus-cardano,

ouroboros-consensus-cardano/golden/cardano/disk/LedgerTables_Babbage

@@ -1,3 +1,4 @@
+{-# LANGUAGE CPP #-}
 {-# LANGUAGE ConstraintKinds #-}
 {-# LANGUAGE DataKinds #-}
 {-# LANGUAGE EmptyCase #-}
@@ -12,7 +13,10 @@
 {-# LANGUAGE TypeFamilies #-}
 {-# LANGUAGE TypeOperators #-}
 {-# LANGUAGE UndecidableInstances #-}
-{-# OPTIONS_GHC -Wno-orphans #-}
+{-# OPTIONS_GHC -Wno-orphans -Wno-x-ord-preserving-coercions #-}
+#if __GLASGOW_HASKELL__ < 908
+{-# OPTIONS_GHC -Wno-unrecognised-warning-flags #-}
+#endif
 
 module Ouroboros.Consensus.Cardano.CanHardFork
   ( CardanoHardForkConstraints
@@ -92,7 +96,7 @@ import Ouroboros.Consensus.Shelley.Node ()
 import Ouroboros.Consensus.Shelley.Protocol.Praos ()
 import Ouroboros.Consensus.Shelley.ShelleyHFC
 import Ouroboros.Consensus.TypeFamilyWrappers
-import Ouroboros.Consensus.Util (eitherToMaybe)
+import Ouroboros.Consensus.Util (coerceMapKeys, eitherToMaybe)
 
 {-------------------------------------------------------------------------------
   CanHardFork
@@ -466,6 +470,7 @@ translateLedgerStateShelleyToAllegraWrapper =
                 LedgerTables
                   . DiffMK
                   . Diff.fromMapDeletes
+                  . coerceMapKeys
                   . Map.map SL.upgradeTxOut
                   $ avvms
 
@@ -478,6 +483,7 @@ translateLedgerStateShelleyToAllegraWrapper =
                   . withLedgerTables ls
                   . LedgerTables
                   . ValuesMK
+                  . coerceMapKeys
                   $ avvms
 
               resultingState =

ouroboros-consensus-cardano/golden/cardano/disk/LedgerTables_Conway

@@ -28,7 +28,6 @@ import Cardano.Ledger.Binary.Decoding hiding (Decoder)
 import Cardano.Ledger.Binary.Encoding hiding (Encoding)
 import qualified Cardano.Ledger.Conway.State as SL
 import Cardano.Ledger.Core (Era, eraDecoder, eraProtVerLow)
-import qualified Cardano.Ledger.Shelley.API as SL
 import Cardano.Ledger.Shelley.LedgerState as SL
   ( esLStateL
   , lsCertStateL
@@ -57,7 +56,8 @@ import Ouroboros.Consensus.Ledger.Tables
 import Ouroboros.Consensus.Protocol.Praos (Praos)
 import Ouroboros.Consensus.Protocol.TPraos (TPraos)
 import Ouroboros.Consensus.Shelley.Ledger
-  ( IsShelleyBlock
+  ( BigEndianTxIn
+  , IsShelleyBlock
   , ShelleyBlock
   , ShelleyCompatible
   , shelleyLedgerState
@@ -70,7 +70,7 @@ instance
   HasCanonicalTxIn (CardanoEras c)
   where
   newtype CanonicalTxIn (CardanoEras c) = CardanoTxIn
-    { getCardanoTxIn :: SL.TxIn
+    { getCardanoTxIn :: BigEndianTxIn
     }
     deriving stock (Show, Eq, Ord)
     deriving newtype NoThunks

ouroboros-consensus-cardano/golden/cardano/disk/LedgerTables_Dijkstra

@@ -19,6 +19,7 @@
 
 module Ouroboros.Consensus.Cardano.QueryHF () where
 
+import Data.Coerce
 import Data.Functor.Product
 import Data.SOP.BasicFunctors
 import Data.SOP.Constraint
@@ -98,14 +99,14 @@ instance CardanoHardForkConstraints c => BlockSupportsHFLedgerQuery (CardanoEras
           answerShelleyLookupQueries
             (injectLedgerTables idx)
             (ejectHardForkTxOut idx)
-            (ejectCanonicalTxIn idx)
+            (coerce . ejectCanonicalTxIn idx)
       )
   answerBlockQueryHFTraverse =
     answerCardanoQueryHF
       ( \idx ->
           answerShelleyTraversingQueries
             (ejectHardForkTxOut idx)
-            (ejectCanonicalTxIn idx)
+            (coerce . ejectCanonicalTxIn idx)
             (queryLedgerGetTraversingFilter idx)
       )