merge: dev

.github/workflows/ci-fmt.yml

@@ -1,6 +1,6 @@
 on:
   push:
-    branches: [master]
+    branches: [master, dev]
   pull_request:
     types: [opened, reopened, synchronize, ready_for_review]
 

.github/workflows/ci-lint.yml

@@ -1,6 +1,6 @@
 on:
   push:
-    branches: [master]
+    branches: [master, dev]
   pull_request:
     types: [opened, reopened, synchronize, ready_for_review]
 

.github/workflows/ci-test-integration.yml

@@ -1,6 +1,6 @@
 on:
   push:
-    branches: [master]
+    branches: [master, dev]
   pull_request:
     types: [opened, reopened, synchronize, ready_for_review]
 

.github/workflows/ci-test-unit.yml

@@ -1,6 +1,6 @@
 on:
   push:
-    branches: [master]
+    branches: [master, dev]
   pull_request:
     types: [opened, reopened, synchronize, ready_for_review]
 

Cargo.toml

@@ -18,6 +18,7 @@ members = [
   "magicblock-committor-program",
   "magicblock-committor-service",
   "magicblock-config",
+  "magicblock-config-macro",
   "magicblock-core",
   "magicblock-geyser-plugin",
   "magicblock-ledger",
@@ -64,6 +65,8 @@ borsh-derive = "1.5.1"
 bs58 = "0.4.0"
 byteorder = "1.5.0"
 cargo-lock = "10.0.0"
+clap = "4.5.40"
+convert_case = "0.8.0"
 conjunto-transwise = { git = "https://github.com/magicblock-labs/conjunto.git", rev = "bf82b45" }
 console-subscriber = "0.2.0"
 crossbeam-channel = "0.5.11"
@@ -107,6 +110,7 @@ magicblock-committor-program = { path = "./magicblock-committor-program", featur
 ] }
 magicblock-committor-service = { path = "./magicblock-committor-service" }
 magicblock-config = { path = "./magicblock-config" }
+magicblock-config-macro = { path = "./magicblock-config-macro" }
 magicblock-core = { path = "./magicblock-core" }
 magicblock-delegation-program = { git = "https://github.com/magicblock-labs/delegation-program.git", rev = "4af7f1c" }
 magic-domain-program = { git = "https://github.com/magicblock-labs/magic-domain-program.git", rev = "ea04d46", default-features = false }
@@ -129,10 +133,12 @@ num-format = "0.4.4"
 num-traits = "0.2"
 num_cpus = "1.16.0"
 paste = "1.0"
+proc-macro2 = "1.0"
 prometheus = "0.13.4"
 # Needs to match https://crates.io/crates/solana-storage-bigtable/2.1.13/dependencies
 prost = "0.11.9"
 protobuf-src = "1.1"
+quote = "1.0"
 rand = "0.8.5"
 rayon = "1.10.0"
 rustc_version = "0.4"
@@ -142,7 +148,7 @@ serde = "1.0.217"
 serde_derive = "1.0"
 serde_json = "1.0"
 sha3 = "0.10.8"
-solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "7bdfefc" }
+solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "176540a" }
 solana-accounts-db = { version = "2.2" }
 solana-account-decoder = { version = "2.2" }
 solana-address-lookup-table-program = { version = "2.2" }
@@ -182,6 +188,7 @@ spl-token-2022 = "=6.0"
 static_assertions = "1.1.0"
 strum = "0.24"
 strum_macros = "0.24"
+syn = "2.0"
 tempfile = "3.10.1"
 test-tools = { path = "./test-tools" }
 test-tools-core = { path = "./test-tools-core" }
@@ -203,6 +210,6 @@ vergen = "8.3.1"
 # some solana dependencies have solana-storage-proto as dependency
 # we need to patch them with our version, because they use protobuf-src v1.1.0
 # and we use protobuf-src v2.1.1. Otherwise compilation fails
-solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "7bdfefc" }
+solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "176540a" }
 solana-storage-proto = { path = "./storage-proto" }
 solana-svm = { git = "https://github.com/magicblock-labs/magicblock-svm.git" }

configs/ephem-devnet.toml

@@ -1,5 +1,5 @@
 [accounts]
-remote = "devnet"
+remote.cluster = "devnet"
 lifecycle = "ephemeral"
 commit = { frequency_millis = 50_000 }
 

configs/ephem-localnet.toml

@@ -1,5 +1,5 @@
 [accounts]
-remote = "http://localhost:7799"
+remote.url = "http://localhost:7799"
 lifecycle = "ephemeral"
 commit = { frequency_millis = 50_000 }
 

configs/ephem-testnet.toml

@@ -1,5 +1,5 @@
 [accounts]
-remote = "testnet"
+remote.cluster = "testnet"
 lifecycle = "ephemeral"
 commit = { frequency_millis = 50_000 }
 

magicblock-account-cloner/Cargo.toml

@@ -16,6 +16,7 @@ magicblock-account-fetcher = { workspace = true }
 magicblock-account-updates = { workspace = true }
 magicblock-account-dumper = { workspace = true }
 magicblock-accounts-api = { workspace = true }
+magicblock-config = { workspace = true }
 magicblock-core = { workspace = true }
 magicblock-committor-service = { workspace = true }
 magicblock-metrics = { workspace = true }

magicblock-account-cloner/src/remote_account_cloner_worker.rs

@@ -17,6 +17,7 @@ use magicblock_account_fetcher::AccountFetcher;
 use magicblock_account_updates::{AccountUpdates, AccountUpdatesResult};
 use magicblock_accounts_api::InternalAccountProvider;
 use magicblock_committor_service::ChangesetCommittor;
+use magicblock_config::{AccountsCloneConfig, PrepareLookupTables};
 use magicblock_metrics::metrics;
 use magicblock_mutator::idl::{get_pubkey_anchor_idl, get_pubkey_shank_idl};
 use solana_sdk::{
@@ -108,6 +109,7 @@ pub struct RemoteAccountClonerWorker<IAP, AFE, AUP, ADU, CC> {
     last_clone_output: CloneOutputMap,
     validator_identity: Pubkey,
     monitored_accounts: RefCell<LruCache<Pubkey, ()>>,
+    clone_config: AccountsCloneConfig,
 }
 
 // SAFETY:
@@ -146,6 +148,7 @@ where
         permissions: AccountClonerPermissions,
         validator_authority: Pubkey,
         max_monitored_accounts: usize,
+        clone_config: AccountsCloneConfig,
     ) -> Self {
         let (clone_request_sender, clone_request_receiver) = flume::unbounded();
         let fetch_retries = 50;
@@ -169,6 +172,7 @@ where
             last_clone_output: Default::default(),
             validator_identity: validator_authority,
             monitored_accounts: LruCache::new(max_monitored_accounts).into(),
+            clone_config,
         }
     }
 
@@ -710,43 +714,45 @@ where
 
                 // Allow the committer service to reserve pubkeys in lookup tables
                 // that could be needed when we commit this account
-                // NOTE: we start reserving pubkeys so the transaction can complete while we
-                // clone the account.
-                let reserve_pubkeys_handle = if let Some(committor) =
-                    self.changeset_committor.as_ref()
-                {
-                    let committor = Arc::clone(committor);
-                    let pubkey = *pubkey;
-                    let owner = delegation_record.owner;
-                    Some(tokio::spawn(map_committor_request_result(
-                        committor.reserve_pubkeys_for_committee(pubkey, owner),
-                        committor,
-                    )))
-                } else {
-                    None
-                };
+                if let Some(committor) = self.changeset_committor.as_ref() {
+                    if self.clone_config.prepare_lookup_tables
+                        == PrepareLookupTables::Always
+                    {
+                        let committor = Arc::clone(committor);
+                        let pubkey = *pubkey;
+                        let owner = delegation_record.owner;
+                        tokio::spawn(async move {
+                            match map_committor_request_result(
+                                committor.reserve_pubkeys_for_committee(
+                                    pubkey, owner,
+                                ),
+                                committor,
+                            )
+                            .await
+                            {
+                                Ok(initiated) => {
+                                    trace!(
+                                    "Reserving lookup keys for {pubkey} took {:?}",
+                                    initiated.elapsed()
+                                );
+                                }
+                                Err(err) => {
+                                    error!("Failed to reserve lookup keys for {pubkey}: {err:?}");
+                                }
+                            };
+                        });
+                    }
+                }
 
-                let sig = self.do_clone_delegated_account(
+                self.do_clone_delegated_account(
                     pubkey,
                     // TODO(GabrielePicco): Avoid cloning
                     &Account {
                         lamports: delegation_record.lamports,
                         ..account.clone()
                     },
                     delegation_record,
-                )?;
-
-                if let Some(handle) = reserve_pubkeys_handle {
-                    let initiated = handle.await.map_err(|err| {
-                        AccountClonerError::JoinError(format!("{err} {err:?}"))
-                    })??;
-                    trace!(
-                        "Reserving lookup keys for {pubkey} took {:?}",
-                        initiated.elapsed()
-                    );
-                }
-
-                sig
+                )?
             }
         };
         // Return the result

magicblock-account-cloner/tests/remote_account_cloner.rs

@@ -11,6 +11,7 @@ use magicblock_account_fetcher::AccountFetcherStub;
 use magicblock_account_updates::AccountUpdatesStub;
 use magicblock_accounts_api::InternalAccountProviderStub;
 use magicblock_committor_service::stubs::ChangesetCommittorStub;
+use magicblock_config::AccountsCloneConfig;
 use magicblock_mutator::idl::{get_pubkey_anchor_idl, get_pubkey_shank_idl};
 use solana_sdk::{
     bpf_loader_upgradeable::get_program_data_address,
@@ -49,6 +50,7 @@ fn setup_custom(
         permissions,
         Pubkey::new_unique(),
         1024,
+        AccountsCloneConfig::default(),
     );
     let cloner_client = RemoteAccountClonerClient::new(&cloner_worker);
     // Run the worker in a separate task

magicblock-account-dumper/src/account_dumper_bank.rs

@@ -91,7 +91,12 @@ impl AccountDumper for AccountDumperBank {
             None,
             self.bank.last_blockhash(),
         );
-        self.execute_transaction(transaction)
+        let result = self.execute_transaction(transaction)?;
+        if let Some(mut acc) = self.bank.get_account(pubkey) {
+            acc.set_delegated(false);
+            self.bank.store_account(*pubkey, acc);
+        }
+        Ok(result)
     }
 
     fn dump_delegated_account(
@@ -111,7 +116,12 @@ impl AccountDumper for AccountDumperBank {
             overrides,
             self.bank.last_blockhash(),
         );
-        self.execute_transaction(transaction)
+        let result = self.execute_transaction(transaction)?;
+        if let Some(mut acc) = self.bank.get_account(pubkey) {
+            acc.set_delegated(true);
+            self.bank.store_account(*pubkey, acc);
+        }
+        Ok(result)
     }
 
     fn dump_program_accounts(