Rework, speed up root ledger population from genesis

changes/17874.md

@@ -0,0 +1,11 @@
+This PR eliminates unnecessary genesis ledger rehashing when bootstrapping from
+genesis, speeding up that process by about 1.5 minutes directly, depending on
+hardware.
+
+This PR also fixes a related bug: the initial best tip network would formerly
+always fail on mainnet when bootstrapping from genesis, due to the daemon
+becoming unresponsive while rehashing the genesis ledger. This would delay
+startup by an additional number of minutes, cause the daemon to report itself as
+synced while its best tip was still at genesis until the next best tip query
+succeeded, and cause confusing behaviour in rosetta. This initial query should
+now only fail under very specific poor network conditions.

src/app/dump_blocks/dump_blocks.ml

@@ -82,9 +82,11 @@ let output_block : type a. a -> a codec io -> unit =
 *)
 let f (type a) ?parent (outputs : a codec io list) make_breadcrumb =
   Async.Thread_safe.block_on_async_exn (fun () ->
-      let frontier = create_frontier ~epoch_ledger_backing_type:Stable_db () in
-      let root = Full_frontier.root frontier in
       let open Async_kernel.Deferred.Let_syntax in
+      let%bind frontier =
+        create_frontier ~epoch_ledger_backing_type:Stable_db ()
+      in
+      let root = Full_frontier.root frontier in
       let%map breadcrumb = make_breadcrumb root in
       List.iter outputs ~f:(fun output ->
           let module Enc = (val output.encoding) in

src/lib/consensus/proof_of_stake.ml

@@ -2666,14 +2666,9 @@ module Make_str (A : Wire_types.Concrete) = struct
                       | Ledger_snapshot.Ledger_root ledger ->
                           Ok ledger
                       | Ledger_snapshot.Genesis_epoch_ledger packed ->
-                          let fresh_root_ledger =
-                            Mina_ledger.Ledger.Root.create ~logger
-                              ~config:snapshot_config
-                              ~depth:Context.constraint_constants.ledger_depth
-                              ()
-                          in
-                          Genesis_ledger.Packed.populate_root packed
-                            fresh_root_ledger )
+                          Genesis_ledger.Packed.create_root packed
+                            ~config:snapshot_config
+                            ~depth:Context.constraint_constants.ledger_depth () )
                 in
                 match snapshot_id with
                 | Staking_epoch_snapshot ->

src/lib/genesis_ledger/genesis_ledger.ml

@@ -66,18 +66,19 @@ module Balances (Balances : Intf.Named_balances_intf) = struct
 end
 
 module Utils = struct
-  let populate_root_with_backing_root genesis_mask ~src ~dest =
-    let open Or_error.Let_syntax in
+  (* Create a new [Ledger.Root.t] ledger with the components of a root
+     backing_ledger for a genesis ledger. These components are the underlying
+     root ledger and the stored mask on top of that root that is presented to
+     users of the genesis root. The mask is passed in to this function so we can
+     assert that there are no uncommitted changes to it, so we can simply
+     checkpoint the root instead of performing a much slower account transfer. *)
+  let create_root_from_backing_root genesis_mask root ~config ~depth () =
     assert (
       Ledger_hash.equal
         (Ledger.merkle_root genesis_mask)
-        (Ledger.Root.merkle_root src) ) ;
-    let%map _root =
-      Ledger_transfer_any.transfer_accounts
-        ~src:(Ledger.Root.as_unmasked src)
-        ~dest:(Ledger.Root.as_unmasked dest)
-    in
-    dest
+        (Ledger.Root.merkle_root root) ) ;
+    assert (Ledger.Root.depth root = depth) ;
+    Ledger.Root.create_checkpoint ~config root () |> Or_error.return
 
   let keypair_of_account_record_exn (private_key, account) =
     let open Account in
@@ -163,11 +164,12 @@ module Make (Inputs : Intf.Ledger_input_intf) : Intf.S = struct
 
   let t = Lazy.map ~f:snd backing_ledger
 
-  let populate_root root =
+  let create_root ~config ~depth () =
     let backing_ledger, mask = Lazy.force backing_ledger in
     match backing_ledger with
     | `Ephemeral ledger ->
         let open Or_error.Let_syntax in
+        let root = Ledger.Root.create ~logger ~config ~depth () in
         (* We are transferring to an unmasked view of the root, so this is
            used solely for the transfer side effect *)
         let%map _dest =
@@ -176,7 +178,7 @@ module Make (Inputs : Intf.Ledger_input_intf) : Intf.S = struct
         in
         root
     | `Root ledger ->
-        populate_root_with_backing_root mask ~src:ledger ~dest:root
+        create_root_from_backing_root mask ledger ~config ~depth ()
 
   include Utils
 
@@ -214,7 +216,7 @@ module Packed = struct
 
   let t ((module L) : t) = L.t
 
-  let populate_root ((module L) : t) ledger = L.populate_root ledger
+  let create_root ((module L) : t) = L.create_root
 
   let depth ((module L) : t) = L.depth
 
@@ -258,9 +260,9 @@ end) : Intf.S = struct
 
   include Utils
 
-  let populate_root dest =
+  let create_root ~config ~depth () =
     let genesis_root, mask = Lazy.force backing_ledger in
-    populate_root_with_backing_root mask ~src:genesis_root ~dest
+    create_root_from_backing_root mask genesis_root ~config ~depth ()
 
   let find_account_record_exn ~f =
     find_account_record_exn ~f (Lazy.force accounts)

src/lib/genesis_ledger/intf.ml

@@ -74,9 +74,12 @@ end
 module type S = sig
   val t : Mina_ledger.Ledger.t Lazy.t
 
-  (** Populate a root ledger with the content of the genesis ledger *)
-  val populate_root :
-    Mina_ledger.Ledger.Root.t -> Mina_ledger.Ledger.Root.t Or_error.t
+  (** Create a new root ledger that is equal in state to the genesis ledger *)
+  val create_root :
+       config:Mina_ledger.Ledger.Root.Config.t
+    -> depth:int
+    -> unit
+    -> Mina_ledger.Ledger.Root.t Or_error.t
 
   val depth : int
 

src/lib/genesis_proof/genesis_proof.ml

@@ -115,8 +115,8 @@ module T = struct
   let genesis_ledger { genesis_ledger; _ } =
     Genesis_ledger.Packed.t genesis_ledger
 
-  let populate_root { genesis_ledger; _ } =
-    Genesis_ledger.Packed.populate_root genesis_ledger
+  let create_root { genesis_ledger; _ } =
+    Genesis_ledger.Packed.create_root genesis_ledger
 
   let genesis_epoch_data { genesis_epoch_data; _ } = genesis_epoch_data
 

src/lib/mina_lmdb_storage/block.ml

@@ -182,7 +182,8 @@ let%test_module "Block storage tests" =
       Quickcheck.test (gen_breadcrumb ~verifier ()) ~trials:1
         ~f:(fun make_breadcrumb ->
           let frontier =
-            create_frontier ~epoch_ledger_backing_type:Stable_db ()
+            Async.Thread_safe.block_on_async_exn (fun () ->
+                create_frontier ~epoch_ledger_backing_type:Stable_db () )
           in
           let root = Full_frontier.root frontier in
           let reader, writer = Pipe.create () in
@@ -211,7 +212,8 @@ let%test_module "Block storage tests" =
       Quickcheck.test (gen_breadcrumb ~verifier ()) ~trials:4
         ~f:(fun make_breadcrumb ->
           let frontier =
-            create_frontier ~epoch_ledger_backing_type:Stable_db ()
+            Async.Thread_safe.block_on_async_exn (fun () ->
+                create_frontier ~epoch_ledger_backing_type:Stable_db () )
           in
           let root = Full_frontier.root frontier in
           let reader, writer = Pipe.create () in

src/lib/transition_frontier/full_frontier/full_frontier.ml

@@ -987,6 +987,7 @@ module For_tests = struct
 
   let create_frontier ~epoch_ledger_backing_type () =
     let open Core in
+    let open Async.Deferred.Let_syntax in
     let epoch_ledger_location =
       Filename.temp_dir_name ^/ "epoch_ledger"
       ^ (Uuid_unix.create () |> Uuid.to_string)
@@ -1025,7 +1026,9 @@ module For_tests = struct
         ~directory:(Filename.temp_file "snarked_ledger" "")
         ~ledger_depth
     in
-    Persistent_root.reset_to_genesis_exn ~precomputed_values persistent_root ;
+    let%map () =
+      Persistent_root.reset_to_genesis_exn persistent_root ~precomputed_values
+    in
     let persistent_root_instance =
       Persistent_root.create_instance_exn persistent_root
     in

src/lib/transition_frontier/full_frontier/full_frontier.mli

@@ -89,7 +89,7 @@ module For_tests : sig
   val create_frontier :
        epoch_ledger_backing_type:Mina_ledger.Ledger.Root.Config.backing_type
     -> unit
-    -> t
+    -> t Async_kernel.Deferred.t
 
   val clean_up_persistent_root : frontier:t -> unit
 

src/lib/transition_frontier/persistent_root/persistent_root.ml

@@ -309,17 +309,38 @@ let with_instance_exn t ~f =
   let x = f instance in
   Instance.close instance ; x
 
-let reset_to_genesis_exn t ~precomputed_values =
+(** Clear the factory directory and recreate the snarked ledger instance for
+    this factory with [create_root] and [setup] *)
+let reset_factory_root_exn t ~create_root ~setup =
+  let open Async.Deferred.Let_syntax in
   assert (Option.is_none t.instance) ;
-  Mina_stdlib_unix.File_system.rmrf t.directory ;
-  with_instance_exn t ~f:(fun instance ->
-      ignore
-        ( Precomputed_values.populate_root precomputed_values
-            (Instance.snarked_ledger instance)
-          |> Or_error.map ~f:Ledger.Root.as_unmasked
-          : Ledger.Any_ledger.witness Or_error.t ) ;
-      Instance.set_root_identifier instance
-        (genesis_root_identifier
-           ~genesis_state_hash:
-             (Precomputed_values.genesis_state_hashes precomputed_values)
-               .state_hash ) )
+  (* Certain database initialization methods, e.g. creation from a checkpoint,
+     depend on the parent directory existing and the target directory _not_
+     existing. *)
+  let%bind () = Mina_stdlib_unix.File_system.remove_dir t.directory in
+  let%map () = Mina_stdlib_unix.File_system.create_dir t.directory in
+  let root =
+    create_root
+      ~config:(Instance.Config.snarked_ledger t)
+      ~depth:t.ledger_depth ()
+    |> Or_error.ok_exn
+  in
+  Ledger.Root.close root ;
+  with_instance_exn t ~f:setup
+
+let reset_to_genesis_exn t ~precomputed_values =
+  let open Async.Deferred.Let_syntax in
+  let logger = t.logger in
+  [%log debug] "Resetting snarked_root in $directory to genesis"
+    ~metadata:[ ("directory", `String t.directory) ] ;
+  let%map () =
+    reset_factory_root_exn t
+      ~create_root:(Precomputed_values.create_root precomputed_values)
+      ~setup:(fun instance ->
+        Instance.set_root_identifier instance
+          (genesis_root_identifier
+             ~genesis_state_hash:
+               (Precomputed_values.genesis_state_hashes precomputed_values)
+                 .state_hash ) )
+  in
+  [%log debug] "Finished resetting snarked_root to genesis"

src/lib/transition_frontier/persistent_root/persistent_root.mli

@@ -105,4 +105,11 @@ val load_from_disk_exn :
 
 val with_instance_exn : t -> f:(Instance_type.t -> 'a) -> 'a
 
-val reset_to_genesis_exn : t -> precomputed_values:Genesis_proof.t -> unit
+val reset_factory_root_exn :
+     t
+  -> create_root:(config:Root.Config.t -> depth:int -> unit -> Root.t Or_error.t)
+  -> setup:(Instance_type.t -> 'a)
+  -> 'a Async.Deferred.t
+
+val reset_to_genesis_exn :
+  t -> precomputed_values:Genesis_proof.t -> unit Async.Deferred.t

src/lib/transition_frontier/tests/full_frontier_tests.ml

@@ -34,7 +34,7 @@ let%test_module "Full_frontier tests" =
       Quickcheck.test (gen_breadcrumb ~verifier ()) ~trials:4
         ~f:(fun make_breadcrumb ->
           Async.Thread_safe.block_on_async_exn (fun () ->
-              let frontier =
+              let%bind frontier =
                 create_frontier ~epoch_ledger_backing_type:Stable_db ()
               in
               let root = Full_frontier.root frontier in
@@ -58,7 +58,7 @@ let%test_module "Full_frontier tests" =
       Quickcheck.test gen_branches ~trials:4
         ~f:(fun (make_short_branch, make_long_branch) ->
           Async.Thread_safe.block_on_async_exn (fun () ->
-              let frontier =
+              let%bind frontier =
                 create_frontier ~epoch_ledger_backing_type:Stable_db ()
               in
               let test_best_tip ?message breadcrumb =
@@ -93,7 +93,7 @@ let%test_module "Full_frontier tests" =
         ~trials:4
         ~f:(fun make_seq ->
           Async.Thread_safe.block_on_async_exn (fun () ->
-              let frontier =
+              let%bind frontier =
                 create_frontier ~epoch_ledger_backing_type:Stable_db ()
               in
               let root = Full_frontier.root frontier in
@@ -123,7 +123,7 @@ let%test_module "Full_frontier tests" =
         ~trials:2
         ~f:(fun make_seq ->
           Async.Thread_safe.block_on_async_exn (fun () ->
-              let frontier =
+              let%bind frontier =
                 create_frontier ~epoch_ledger_backing_type:Stable_db ()
               in
               let root = Full_frontier.root frontier in
@@ -152,7 +152,7 @@ let%test_module "Full_frontier tests" =
       in
       Quickcheck.test gen ~trials:4 ~f:(fun make_seq ->
           Async.Thread_safe.block_on_async_exn (fun () ->
-              let frontier =
+              let%bind frontier =
                 create_frontier ~epoch_ledger_backing_type:Stable_db ()
               in
               let root = Full_frontier.root frontier in
@@ -181,7 +181,7 @@ let%test_module "Full_frontier tests" =
       Quickcheck.test gen ~trials:4
         ~f:(fun (make_ancestors, make_branch_a, make_branch_b) ->
           Async.Thread_safe.block_on_async_exn (fun () ->
-              let frontier =
+              let%bind frontier =
                 create_frontier ~epoch_ledger_backing_type:Stable_db ()
               in
               let root = Full_frontier.root frontier in

src/lib/transition_frontier/transition_frontier.ml

@@ -283,7 +283,9 @@ let rec load_with_max_length :
           (State_hash.With_state_hashes.state_hash
              precomputed_values.protocol_state_with_hashes )
     in
-    Persistent_root.reset_to_genesis_exn persistent_root ~precomputed_values ;
+    let%bind () =
+      Persistent_root.reset_to_genesis_exn persistent_root ~precomputed_values
+    in
     let genesis_ledger_hash =
       Precomputed_values.genesis_ledger precomputed_values
       |> Lazy.force |> Ledger.merkle_root |> Frozen_ledger_hash.of_ledger_hash
@@ -684,8 +686,8 @@ module For_tests = struct
 
   let gen ?(logger = Logger.null ()) ~verifier ?trust_system
       ?consensus_local_state ~precomputed_values
-      ?(populate_root_and_accounts =
-        ( Precomputed_values.populate_root precomputed_values
+      ?(create_root_and_accounts =
+        ( Precomputed_values.create_root precomputed_values
         , Lazy.force (Precomputed_values.accounts precomputed_values) ))
       ?(gen_root_breadcrumb =
         gen_genesis_breadcrumb_with_protocol_states ~logger ~verifier
@@ -724,7 +726,7 @@ module For_tests = struct
                   precomputed_values.protocol_state_with_hashes )
              ~epoch_ledger_backing_type:Stable_db )
     in
-    let populate_root, root_ledger_accounts = populate_root_and_accounts in
+    let create_root, root_ledger_accounts = create_root_and_accounts in
     (* TODO: ensure that rose_tree cannot be longer than k *)
     let%bind root, branches, protocol_states =
       let%bind root, protocol_states = gen_root_breadcrumb in
@@ -755,12 +757,12 @@ module For_tests = struct
           ~genesis_state_hash:
             (State_hash.With_state_hashes.state_hash
                precomputed_values.protocol_state_with_hashes ) ) ;
-    Persistent_root.with_instance_exn persistent_root ~f:(fun instance ->
-        let transition = Root_data.Limited.transition root_data in
-        Persistent_root.Instance.set_root_state_hash instance
-          (Mina_block.Validated.state_hash transition) ;
-        ignore
-        @@ populate_root (Persistent_root.Instance.snarked_ledger instance) ) ;
+    Async.Thread_safe.block_on_async_exn (fun () ->
+        Persistent_root.reset_factory_root_exn persistent_root ~create_root
+          ~setup:(fun instance ->
+            let transition = Root_data.Limited.transition root_data in
+            Persistent_root.Instance.set_root_state_hash instance
+              (Mina_block.Validated.state_hash transition) ) ) ;
     let frontier_result =
       Async.Thread_safe.block_on_async_exn (fun () ->
           load_with_max_length ~max_length ~retry_with_fresh_db:false
@@ -797,21 +799,21 @@ module For_tests = struct
 
   let gen_with_branch ?logger ~verifier ?trust_system ?consensus_local_state
       ~precomputed_values
-      ?(populate_root_and_accounts =
-        ( Precomputed_values.populate_root precomputed_values
+      ?(create_root_and_accounts =
+        ( Precomputed_values.create_root precomputed_values
         , Lazy.force (Precomputed_values.accounts precomputed_values) ))
       ?gen_root_breadcrumb ?(get_branch_root = root) ~max_length ~frontier_size
       ~branch_size () =
     let open Quickcheck.Generator.Let_syntax in
     let%bind frontier =
       gen ?logger ~verifier ?trust_system ?consensus_local_state
-        ~precomputed_values ?gen_root_breadcrumb ~populate_root_and_accounts
+        ~precomputed_values ?gen_root_breadcrumb ~create_root_and_accounts
         ~max_length ~size:frontier_size ()
     in
     let%map make_branch =
       Breadcrumb.For_tests.gen_seq ?logger ~precomputed_values ~verifier
         ?trust_system
-        ~accounts_with_secret_keys:(snd populate_root_and_accounts)
+        ~accounts_with_secret_keys:(snd create_root_and_accounts)
         branch_size
     in
     let branch =