Merge pull request #17883 from MinaProtocol/lyh/dev-into-compat-sep30

Merge `compatible` into `develop` sep 30th

Author: georgeee

buildkite/src/Jobs/Release/MinaToolchainArtifactFocal.dhall

@@ -0,0 +1,11 @@
+This PR eliminates unnecessary genesis ledger rehashing when bootstrapping from
+genesis, speeding up that process by about 1.5 minutes directly, depending on
+hardware.
+
+This PR also fixes a related bug: the initial best tip network would formerly
+always fail on mainnet when bootstrapping from genesis, due to the daemon
+becoming unresponsive while rehashing the genesis ledger. This would delay
+startup by an additional number of minutes, cause the daemon to report itself as
+synced while its best tip was still at genesis until the next best tip query
+succeeded, and cause confusing behaviour in rosetta. This initial query should
+now only fail under very specific poor network conditions.

changes/17874.md

@@ -82,9 +82,11 @@ let output_block : type a. a -> a codec io -> unit =
 *)
 let f (type a) ?parent (outputs : a codec io list) make_breadcrumb =
   Async.Thread_safe.block_on_async_exn (fun () ->
-      let frontier = create_frontier ~epoch_ledger_backing_type:Stable_db () in
-      let root = Full_frontier.root frontier in
       let open Async_kernel.Deferred.Let_syntax in
+      let%bind frontier =
+        create_frontier ~epoch_ledger_backing_type:Stable_db ()
+      in
+      let root = Full_frontier.root frontier in
       let%map breadcrumb = make_breadcrumb root in
       List.iter outputs ~f:(fun output ->
           let module Enc = (val output.encoding) in

hf.zip

@@ -7,7 +7,10 @@
 
 (rule
   (target dune-linker)
-  (enabled_if %{bin-available:lld})
+  (enabled_if 
+    (and
+      (= %{env:DUNE_USE_DEFAULT_LINKER=n} n)
+      %{bin-available:lld}))
   (action (with-stdout-to dune-linker (echo "-fuse-ld=lld"))))
 
 (rule

src/app/dump_blocks/dump_blocks.ml

@@ -2666,14 +2666,9 @@ module Make_str (A : Wire_types.Concrete) = struct
                       | Ledger_snapshot.Ledger_root ledger ->
                           Ok ledger
                       | Ledger_snapshot.Genesis_epoch_ledger packed ->
-                          let fresh_root_ledger =
-                            Mina_ledger.Ledger.Root.create ~logger
-                              ~config:snapshot_config
-                              ~depth:Context.constraint_constants.ledger_depth
-                              ()
-                          in
-                          Genesis_ledger.Packed.populate_root packed
-                            fresh_root_ledger )
+                          Genesis_ledger.Packed.create_root packed
+                            ~config:snapshot_config
+                            ~depth:Context.constraint_constants.ledger_depth () )
                 in
                 match snapshot_id with
                 | Staking_epoch_snapshot ->

src/dune.linker.inc

@@ -66,18 +66,19 @@ module Balances (Balances : Intf.Named_balances_intf) = struct
 end
 
 module Utils = struct
-  let populate_root_with_backing_root genesis_mask ~src ~dest =
-    let open Or_error.Let_syntax in
+  (* Create a new [Ledger.Root.t] ledger with the components of a root
+     backing_ledger for a genesis ledger. These components are the underlying
+     root ledger and the stored mask on top of that root that is presented to
+     users of the genesis root. The mask is passed in to this function so we can
+     assert that there are no uncommitted changes to it, so we can simply
+     checkpoint the root instead of performing a much slower account transfer. *)
+  let create_root_from_backing_root genesis_mask root ~config ~depth () =
     assert (
       Ledger_hash.equal
         (Ledger.merkle_root genesis_mask)
-        (Ledger.Root.merkle_root src) ) ;
-    let%map _root =
-      Ledger_transfer_any.transfer_accounts
-        ~src:(Ledger.Root.as_unmasked src)
-        ~dest:(Ledger.Root.as_unmasked dest)
-    in
-    dest
+        (Ledger.Root.merkle_root root) ) ;
+    assert (Ledger.Root.depth root = depth) ;
+    Ledger.Root.create_checkpoint ~config root () |> Or_error.return
 
   let keypair_of_account_record_exn (private_key, account) =
     let open Account in
@@ -163,11 +164,12 @@ module Make (Inputs : Intf.Ledger_input_intf) : Intf.S = struct
 
   let t = Lazy.map ~f:snd backing_ledger
 
-  let populate_root root =
+  let create_root ~config ~depth () =
     let backing_ledger, mask = Lazy.force backing_ledger in
     match backing_ledger with
     | `Ephemeral ledger ->
         let open Or_error.Let_syntax in
+        let root = Ledger.Root.create ~logger ~config ~depth () in
         (* We are transferring to an unmasked view of the root, so this is
            used solely for the transfer side effect *)
         let%map _dest =
@@ -176,7 +178,7 @@ module Make (Inputs : Intf.Ledger_input_intf) : Intf.S = struct
         in
         root
     | `Root ledger ->
-        populate_root_with_backing_root mask ~src:ledger ~dest:root
+        create_root_from_backing_root mask ledger ~config ~depth ()
 
   include Utils
 
@@ -214,7 +216,7 @@ module Packed = struct
 
   let t ((module L) : t) = L.t
 
-  let populate_root ((module L) : t) ledger = L.populate_root ledger
+  let create_root ((module L) : t) = L.create_root
 
   let depth ((module L) : t) = L.depth
 
@@ -258,9 +260,9 @@ end) : Intf.S = struct
 
   include Utils
 
-  let populate_root dest =
+  let create_root ~config ~depth () =
     let genesis_root, mask = Lazy.force backing_ledger in
-    populate_root_with_backing_root mask ~src:genesis_root ~dest
+    create_root_from_backing_root mask genesis_root ~config ~depth ()
 
   let find_account_record_exn ~f =
     find_account_record_exn ~f (Lazy.force accounts)

src/lib/consensus/proof_of_stake.ml

@@ -74,9 +74,12 @@ end
 module type S = sig
   val t : Mina_ledger.Ledger.t Lazy.t
 
-  (** Populate a root ledger with the content of the genesis ledger *)
-  val populate_root :
-    Mina_ledger.Ledger.Root.t -> Mina_ledger.Ledger.Root.t Or_error.t
+  (** Create a new root ledger that is equal in state to the genesis ledger *)
+  val create_root :
+       config:Mina_ledger.Ledger.Root.Config.t
+    -> depth:int
+    -> unit
+    -> Mina_ledger.Ledger.Root.t Or_error.t
 
   val depth : int
 

src/lib/genesis_ledger/genesis_ledger.ml

@@ -115,8 +115,8 @@ module T = struct
   let genesis_ledger { genesis_ledger; _ } =
     Genesis_ledger.Packed.t genesis_ledger
 
-  let populate_root { genesis_ledger; _ } =
-    Genesis_ledger.Packed.populate_root genesis_ledger
+  let create_root { genesis_ledger; _ } =
+    Genesis_ledger.Packed.create_root genesis_ledger
 
   let genesis_epoch_data { genesis_epoch_data; _ } = genesis_epoch_data
 

src/lib/genesis_ledger/intf.ml

@@ -182,7 +182,8 @@ let%test_module "Block storage tests" =
       Quickcheck.test (gen_breadcrumb ~verifier ()) ~trials:1
         ~f:(fun make_breadcrumb ->
           let frontier =
-            create_frontier ~epoch_ledger_backing_type:Stable_db ()
+            Async.Thread_safe.block_on_async_exn (fun () ->
+                create_frontier ~epoch_ledger_backing_type:Stable_db () )
           in
           let root = Full_frontier.root frontier in
           let reader, writer = Pipe.create () in
@@ -211,7 +212,8 @@ let%test_module "Block storage tests" =
       Quickcheck.test (gen_breadcrumb ~verifier ()) ~trials:4
         ~f:(fun make_breadcrumb ->
           let frontier =
-            create_frontier ~epoch_ledger_backing_type:Stable_db ()
+            Async.Thread_safe.block_on_async_exn (fun () ->
+                create_frontier ~epoch_ledger_backing_type:Stable_db () )
           in
           let root = Full_frontier.root frontier in
           let reader, writer = Pipe.create () in