Merge pull request #16093 from MinaProtocol/fabrizio-m/wide-merkle-queries

Wide merkle query

Author: mrmr1993

src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml

@@ -1006,7 +1006,7 @@ let setup_daemon logger ~itn_features =
           let module Context = struct
             let logger = logger
 
-            let precomputed_values = precomputed_values
+            let compile_config = precomputed_values.compile_config
 
             let constraint_constants = precomputed_values.constraint_constants
 

src/config/dev.mlh

@@ -78,3 +78,7 @@
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 0]
 [%%undef zkapp_cmd_limit]
+
+(* Sync ledger query/response size*)
+[%%define sync_ledger_max_subtree_depth 4]
+[%%define sync_ledger_default_subtree_depth 3]

src/config/devnet.mlh

@@ -73,3 +73,7 @@
 [%%define vrf_poll_interval 5000]
 [%%define zkapp_cmd_limit 24]
 [%%undef scan_state_transaction_capacity_log_2]
+
+(* Constants determining sync ledger query/response size*)
+[%%define sync_ledger_max_subtree_depth 8]
+[%%define sync_ledger_default_subtree_depth 6]

src/config/lightnet.mlh

@@ -72,3 +72,7 @@
 [%%define compaction_interval 360000]
 [%%define vrf_poll_interval 5000]
 [%%undef zkapp_cmd_limit]
+
+(* Constants determining sync ledger query/response size*)
+[%%define sync_ledger_max_subtree_depth 8]
+[%%define sync_ledger_default_subtree_depth 6]

src/config/mainnet.mlh

@@ -73,3 +73,7 @@
 [%%define vrf_poll_interval 5000]
 [%%define zkapp_cmd_limit 24]
 [%%undef scan_state_transaction_capacity_log_2]
+
+(* Constants determining sync ledger query/response size*)
+[%%define sync_ledger_max_subtree_depth 8]
+[%%define sync_ledger_default_subtree_depth 6]

src/lib/block_producer/block_producer.ml

@@ -819,12 +819,17 @@ let produce ~genesis_breadcrumb ~context:(module Context : CONTEXT) ~prover
                     (Mina_base.State_hash.to_base58_check
                        protocol_state_hashes.state_hash ) )
               ] ;
+          let module Consensus_context = struct
+            include Context
+
+            let compile_config = precomputed_values.compile_config
+          end in
           Internal_tracing.with_state_hash protocol_state_hashes.state_hash
           @@ fun () ->
           Debug_assert.debug_assert (fun () ->
               [%test_result: [ `Take | `Keep ]]
                 (Consensus.Hooks.select
-                   ~context:(module Context)
+                   ~context:(module Consensus_context)
                    ~existing:
                      (With_hash.map ~f:Mina_block.consensus_state
                         previous_transition )
@@ -839,7 +844,7 @@ let produce ~genesis_breadcrumb ~context:(module Context : CONTEXT) ~prover
               in
               [%test_result: [ `Take | `Keep ]]
                 (Consensus.Hooks.select
-                   ~context:(module Context)
+                   ~context:(module Consensus_context)
                    ~existing:root_consensus_state_with_hashes
                    ~candidate:consensus_state_with_hashes )
                 ~expect:`Take
@@ -908,7 +913,7 @@ let produce ~genesis_breadcrumb ~context:(module Context : CONTEXT) ~prover
                       `This_block_was_not_received_via_gossip
                 >>= Validation.validate_frontier_dependencies
                       ~to_header:Mina_block.header
-                      ~context:(module Context)
+                      ~context:(module Consensus_context)
                       ~root_block:
                         ( Transition_frontier.root frontier
                         |> Breadcrumb.block_with_hash )
@@ -1412,10 +1417,15 @@ let run_precomputed ~context:(module Context : CONTEXT) ~verifier ~trust_system
           Header.protocol_state
           @@ Mina_block.header (With_hash.data previous_transition)
         in
+        let module Consensus_context = struct
+          include Context
+
+          let compile_config = precomputed_values.compile_config
+        end in
         Debug_assert.debug_assert (fun () ->
             [%test_result: [ `Take | `Keep ]]
               (Consensus.Hooks.select
-                 ~context:(module Context)
+                 ~context:(module Consensus_context)
                  ~existing:
                    (With_hash.map ~f:Mina_block.consensus_state
                       previous_transition )
@@ -1430,7 +1440,7 @@ let run_precomputed ~context:(module Context : CONTEXT) ~verifier ~trust_system
             in
             [%test_result: [ `Take | `Keep ]]
               (Consensus.Hooks.select
-                 ~context:(module Context)
+                 ~context:(module Consensus_context)
                  ~existing:root_consensus_state_with_hashes
                  ~candidate:consensus_state_with_hashes )
               ~expect:`Take
@@ -1468,7 +1478,7 @@ let run_precomputed ~context:(module Context : CONTEXT) ~verifier ~trust_system
                       previous_protocol_state )
             >>= Validation.validate_frontier_dependencies
                   ~to_header:Mina_block.header
-                  ~context:(module Context)
+                  ~context:(module Consensus_context)
                   ~root_block:
                     ( Transition_frontier.root frontier
                     |> Breadcrumb.block_with_hash )

src/lib/bootstrap_controller/bootstrap_controller.ml

@@ -17,8 +17,6 @@ module type CONTEXT = sig
   val constraint_constants : Genesis_constants.Constraint_constants.t
 
   val consensus_constants : Consensus.Constants.t
-
-  val compile_config : Mina_compile_config.t
 end
 
 type Structured_log_events.t += Bootstrap_complete
@@ -65,9 +63,11 @@ let time_deferred deferred =
   (Time.diff end_time start_time, result)
 
 let worth_getting_root ({ context = (module Context); _ } as t) candidate =
-  let module Context = struct
+  let module Consensus_context = struct
     include Context
 
+    let compile_config = precomputed_values.compile_config
+
     let logger =
       Logger.extend logger
         [ ( "selection_context"
@@ -76,7 +76,7 @@ let worth_getting_root ({ context = (module Context); _ } as t) candidate =
   end in
   Consensus.Hooks.equal_select_status `Take
   @@ Consensus.Hooks.select
-       ~context:(module Context)
+       ~context:(module Consensus_context)
        ~existing:
          ( t.best_seen_transition |> Mina_block.Validation.block_with_hash
          |> With_hash.map ~f:Mina_block.consensus_state )
@@ -152,7 +152,7 @@ let to_consensus_state h =
     the existing one, then reset the Sync_ledger's target by calling
     [start_sync_job_with_peer] function. *)
 let on_transition ({ context = (module Context); _ } as t) ~sender
-    ~root_sync_ledger ~genesis_constants candidate_header =
+    ~root_sync_ledger candidate_header =
   let open Context in
   let candidate_consensus_state =
     With_hash.map ~f:to_consensus_state candidate_header
@@ -175,7 +175,7 @@ let on_transition ({ context = (module Context); _ } as t) ~sender
         match%bind
           Sync_handler.Root.verify
             ~context:(module Context)
-            ~verifier:t.verifier ~genesis_constants candidate_consensus_state
+            ~verifier:t.verifier candidate_consensus_state
             peer_root_with_proof.data
         with
         | Ok (`Root root, `Best_tip best_tip) ->
@@ -189,7 +189,7 @@ let on_transition ({ context = (module Context); _ } as t) ~sender
     incoming transitions, add those to the transition_cache and calls
     [on_transition] function. *)
 let sync_ledger ({ context = (module Context); _ } as t) ~preferred
-    ~root_sync_ledger ~transition_graph ~sync_ledger_reader ~genesis_constants =
+    ~root_sync_ledger ~transition_graph ~sync_ledger_reader =
   let open Context in
   let query_reader = Sync_ledger.Db.query_reader root_sync_ledger in
   let response_writer = Sync_ledger.Db.answer_writer root_sync_ledger in
@@ -232,11 +232,15 @@ let sync_ledger ({ context = (module Context); _ } as t) ~preferred
             ] ;
 
         Deferred.ignore_m
-        @@ on_transition t ~sender ~root_sync_ledger ~genesis_constants
-             header_with_hash )
+        @@ on_transition t ~sender ~root_sync_ledger header_with_hash )
       else Deferred.unit )
 
 let external_transition_compare ~context:(module Context : CONTEXT) =
+  let module Consensus_context = struct
+    include Context
+
+    let compile_config = precomputed_values.compile_config
+  end in
   let get_consensus_state =
     Fn.compose Protocol_state.consensus_state Mina_block.Header.protocol_state
   in
@@ -250,7 +254,9 @@ let external_transition_compare ~context:(module Context : CONTEXT) =
       then 0
       else if
         Consensus.Hooks.equal_select_status `Keep
-        @@ Consensus.Hooks.select ~context:(module Context) ~existing ~candidate
+        @@ Consensus.Hooks.select
+             ~context:(module Consensus_context)
+             ~existing ~candidate
       then -1
       else 1 )
     ~f:(With_hash.map ~f:get_consensus_state)
@@ -273,10 +279,6 @@ let run ~context:(module Context : CONTEXT) ~trust_system ~verifier ~network
     ~persistent_frontier ~initial_root_transition ~catchup_mode =
   let open Context in
   O1trace.thread "bootstrap" (fun () ->
-      let genesis_constants =
-        Precomputed_values.genesis_constants precomputed_values
-      in
-      let constraint_constants = precomputed_values.constraint_constants in
       let rec loop previous_cycles =
         let sync_ledger_pipe = "sync ledger pipe" in
         let sync_ledger_reader, sync_ledger_writer =
@@ -331,14 +333,21 @@ let run ~context:(module Context : CONTEXT) ~trust_system ~verifier ~network
             temp_persistent_root_instance
         in
         (* step 1. download snarked_ledger *)
+        let module Consensus_context = struct
+          include Context
+
+          let compile_config = precomputed_values.compile_config
+        end in
         let%bind sync_ledger_time, (hash, sender, expected_staged_ledger_hash) =
           time_deferred
             (let root_sync_ledger =
-               Sync_ledger.Db.create temp_snarked_ledger ~logger ~trust_system
+               Sync_ledger.Db.create temp_snarked_ledger
+                 ~context:(module Consensus_context)
+                 ~trust_system
              in
              don't_wait_for
                (sync_ledger t ~preferred:preferred_peers ~root_sync_ledger
-                  ~transition_graph ~sync_ledger_reader ~genesis_constants ) ;
+                  ~transition_graph ~sync_ledger_reader ) ;
              (* We ignore the resulting ledger returned here since it will always
                 * be the same as the ledger we started with because we are syncing
                 * a db ledger. *)
@@ -565,7 +574,7 @@ let run ~context:(module Context : CONTEXT) ~trust_system ~verifier ~network
                     [%log info] "Synchronizing consensus local state" ;
                     let%map result =
                       Consensus.Hooks.sync_local_state
-                        ~context:(module Context)
+                        ~context:(module Consensus_context)
                         ~local_state:consensus_local_state ~trust_system
                         ~glue_sync_ledger:
                           (Mina_networking.glue_sync_ledger t.network)
@@ -616,7 +625,7 @@ let run ~context:(module Context : CONTEXT) ~trust_system ~verifier ~network
                          bootstrapping: " ^ msg )
                   in
                   Transition_frontier.load
-                    ~context:(module Context)
+                    ~context:(module Consensus_context)
                     ~retry_with_fresh_db:false ~verifier ~consensus_local_state
                     ~persistent_root ~persistent_frontier ~catchup_mode ()
                   >>| function
@@ -658,7 +667,7 @@ let run ~context:(module Context : CONTEXT) ~trust_system ~verifier ~network
                       in
                       Consensus.Hooks.equal_select_status `Take
                       @@ Consensus.Hooks.select
-                           ~context:(module Context)
+                           ~context:(module Consensus_context)
                            ~existing:root_consensus_state
                            ~candidate:
                              (With_hash.map
@@ -741,8 +750,6 @@ let%test_module "Bootstrap_controller tests" =
 
     let constraint_constants = precomputed_values.constraint_constants
 
-    let compile_config = Mina_compile_config.For_unit_tests.t
-
     module Context = struct
       let logger = logger
 
@@ -752,8 +759,6 @@ let%test_module "Bootstrap_controller tests" =
         Genesis_constants.For_unit_tests.Constraint_constants.t
 
       let consensus_constants = precomputed_values.consensus_constants
-
-      let compile_config = compile_config
     end
 
     let verifier =
@@ -800,8 +805,7 @@ let%test_module "Bootstrap_controller tests" =
         let%bind fake_network =
           Fake_network.Generator.(
             gen ~precomputed_values ~verifier ~max_frontier_length
-              ~compile_config [ fresh_peer; fresh_peer ]
-              ~use_super_catchup:false)
+              [ fresh_peer; fresh_peer ] ~use_super_catchup:false)
         in
         let%map make_branch =
           Transition_frontier.Breadcrumb.For_tests.gen_seq ~precomputed_values
@@ -829,16 +833,21 @@ let%test_module "Bootstrap_controller tests" =
           let bootstrap =
             make_non_running_bootstrap ~genesis_root ~network:me.network
           in
+          let module Consensus_context = struct
+            include Context
+
+            let compile_config = precomputed_values.compile_config
+          end in
           let root_sync_ledger =
             Sync_ledger.Db.create
               (Transition_frontier.root_snarked_ledger me.state.frontier)
-              ~logger ~trust_system
+              ~context:(module Consensus_context)
+              ~trust_system
           in
           Async.Thread_safe.block_on_async_exn (fun () ->
               let sync_deferred =
                 sync_ledger bootstrap ~root_sync_ledger ~transition_graph
                   ~preferred:[] ~sync_ledger_reader
-                  ~genesis_constants:Genesis_constants.For_unit_tests.t
               in
               let%bind () =
                 Deferred.List.iter branch ~f:(fun breadcrumb ->
@@ -935,7 +944,7 @@ let%test_module "Bootstrap_controller tests" =
       Quickcheck.test ~trials:1
         Fake_network.Generator.(
           gen ~precomputed_values ~verifier ~max_frontier_length
-            ~use_super_catchup:false ~compile_config
+            ~use_super_catchup:false
             [ fresh_peer
             ; peer_with_branch
                 ~frontier_branch_size:((max_frontier_length * 2) + 2)

src/lib/bootstrap_controller/bootstrap_controller.mli

@@ -11,8 +11,6 @@ module type CONTEXT = sig
   val constraint_constants : Genesis_constants.Constraint_constants.t
 
   val consensus_constants : Consensus.Constants.t
-
-  val compile_config : Mina_compile_config.t
 end
 
 type Structured_log_events.t += Bootstrap_complete [@@deriving register_event]

src/lib/consensus/intf.ml

@@ -11,6 +11,8 @@ module type CONTEXT = sig
   val constraint_constants : Genesis_constants.Constraint_constants.t
 
   val consensus_constants : Constants.t
+
+  val compile_config : Mina_compile_config.t
 end
 
 module type Constants = sig

src/lib/consensus/proof_of_stake.ml

@@ -30,6 +30,8 @@ module Make_str (A : Wire_types.Concrete) = struct
     val constraint_constants : Genesis_constants.Constraint_constants.t
 
     val consensus_constants : Constants.t
+
+    val compile_config : Mina_compile_config.t
   end
 
   let make_checked t = Snark_params.Tick.Run.make_checked t
@@ -2678,8 +2680,9 @@ module Make_str (A : Wire_types.Concrete) = struct
                          (next_epoch_ledger_location local_state)
               in
               let sync_ledger =
-                Mina_ledger.Sync_ledger.Db.create ~logger ~trust_system
-                  db_ledger
+                Mina_ledger.Sync_ledger.Db.create
+                  ~context:(module Context)
+                  ~trust_system db_ledger
               in
               let query_reader =
                 Mina_ledger.Sync_ledger.Db.query_reader sync_ledger
@@ -3107,6 +3110,8 @@ module Make_str (A : Wire_types.Concrete) = struct
           Genesis_constants.For_unit_tests.Constraint_constants.t
 
         let consensus_constants = Lazy.force Constants.for_unit_tests
+
+        let compile_config = Mina_compile_config.For_unit_tests.t
       end in
       (* Even when consensus constants are of prod sizes, candidate should still trigger a bootstrap *)
       should_bootstrap_len
@@ -3433,6 +3438,8 @@ module Make_str (A : Wire_types.Concrete) = struct
         let constraint_constants = constraint_constants
 
         let consensus_constants = constants
+
+        let compile_config = Mina_compile_config.For_unit_tests.t
       end
 
       let test_update constraint_constants =