From b78df12de6a5237f890619ec492701179bf16487 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 3 Nov 2025 18:36:49 +0100 Subject: [PATCH 1/9] rocksdb optin --- Makefile | 15 ++++++++------- crates/store/Cargo.toml | 4 ++++ crates/store/src/accounts/mod.rs | 5 +++++ crates/store/src/lib.rs | 2 ++ 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 7a968862c..2c376dd84 100644 --- a/Makefile +++ b/Makefile @@ -8,19 +8,20 @@ help: WARNINGS=RUSTDOCFLAGS="-D warnings" BUILD_PROTO=BUILD_PROTO=1 +ALL_FEATURES="--features=tracing-forest,concurrent,tracing-forest,tx-prover,batch-prover,block-prover,std" # -- linting -------------------------------------------------------------------------------------- .PHONY: clippy clippy: ## Runs Clippy with configs - cargo clippy --locked --all-targets --all-features --workspace -- -D warnings - cargo clippy --locked --all-targets --all-features -p miden-remote-prover -- -D warnings + cargo clippy --locked --all-targets --workspace ${ALL_FEATURES} -- -D warnings + cargo clippy --locked --all-targets -p miden-remote-prover --features=concurrent -- -D warnings .PHONY: fix fix: ## Runs Fix with configs - cargo fix --allow-staged --allow-dirty --all-targets --all-features --workspace - cargo fix --allow-staged --allow-dirty --all-targets --all-features -p miden-remote-prover + cargo fix --allow-staged --allow-dirty --all-targets --workspace ${ALL_FEATURES} + cargo fix --allow-staged --allow-dirty --all-targets -p miden-remote-prover --features=concurrent .PHONY: format @@ -63,7 +64,7 @@ lint: typos-check format fix clippy toml machete ## Runs all linting tasks at on .PHONY: doc doc: ## Generates & checks documentation - $(WARNINGS) cargo doc --all-features --keep-going --release --locked + $(WARNINGS) cargo doc ${ALL_FEATURES} --keep-going --release --locked .PHONY: book book: ## Builds the book & serves documentation site @@ -77,13 +78,13 @@ serve-docs: ## Serves the docs .PHONY: test test: ## Runs all tests - cargo nextest run --all-features --workspace + cargo nextest run ${ALL_FEATURES} --workspace # --- checking ------------------------------------------------------------------------------------ .PHONY: check check: ## Check all targets and features for errors without code generation - ${BUILD_PROTO} cargo check --all-features --all-targets --locked --workspace + ${BUILD_PROTO} cargo check ${ALL_FEATURES} --all-targets --locked --workspace # --- building ------------------------------------------------------------------------------------ diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index f490cf4b7..195336b9d 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -53,6 +53,10 @@ rand = { workspace = true } regex = { version = "1.11" } termtree = { version = "0.5" } +[features] +default = [] # rocksdb has just too many requirements on CI for now +rocksdb = ["miden-crypto/rocksdb"] + [[bench]] harness = false name = "account_tree_historical" diff --git a/crates/store/src/accounts/mod.rs b/crates/store/src/accounts/mod.rs index bf18b815a..fbc47d1e6 100644 --- a/crates/store/src/accounts/mod.rs +++ b/crates/store/src/accounts/mod.rs @@ -27,6 +27,11 @@ mod tests; /// Convenience for an in-memory-only account tree. pub type InMemoryAccountTree = AccountTree>; +#[cfg(feature = "rocksdb")] +/// Convenience for a persistent account tree. +pub type PersistentAccountTree = + AccountTree>; + // ACCOUNT TREE STORAGE TRAIT // ================================================================================================ diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index d50f124f7..bfab4f4d4 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -8,6 +8,8 @@ pub mod genesis; mod server; pub mod state; +#[cfg(feature = "rocksdb")] +pub use accounts::PersistentAccountTree; pub use accounts::{ AccountTreeStorage, AccountTreeWithHistory, From 4496cc8cbb1401a5c4314dce38e5c3579cba6933 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 3 Nov 2025 18:59:55 +0100 Subject: [PATCH 2/9] fixins --- Cargo.lock | 237 +++++++++++++++++++++++++++++++++++++++- crates/store/Cargo.toml | 1 + 2 files changed, 233 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3d92ccc3b..44a114841 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -371,6 +371,24 @@ dependencies = [ "num-traits", ] +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.10.0", + "cexpr", + "clang-sys", + "itertools 0.10.5", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.108", +] + [[package]] name = "bit-set" version = "0.8.0" @@ -474,6 +492,16 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "camino" version = "1.2.1" @@ -524,6 +552,15 @@ dependencies = [ "shlex", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cf-rustracing" version = "1.2.1" @@ -633,6 +670,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", +] + [[package]] name = "clap" version = "3.2.25" @@ -882,6 +929,33 @@ dependencies = [ "typenum", ] +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures", + "curve25519-dalek-derive", + "digest", + "fiat-crypto", + "rustc_version 0.4.1", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.108", +] + [[package]] name = "daemonize" version = "0.5.0" @@ -1202,6 +1276,30 @@ dependencies = [ "spki", ] +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "serde", + "sha2", + "subtle", + "zeroize", +] + [[package]] name = "either" version = "1.15.0" @@ -1313,6 +1411,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + [[package]] name = "figment" version = "0.10.19" @@ -1643,6 +1747,8 @@ dependencies = [ "allocator-api2", "equivalent", "foldhash 0.2.0", + "rayon", + "serde", ] [[package]] @@ -2204,6 +2310,20 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" +[[package]] +name = "librocksdb-sys" +version = "0.17.3+10.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef2a00ee60fe526157c9023edab23943fae1ce2ab6f4abb2a807c1746835de9" +dependencies = [ + "bindgen", + "bzip2-sys", + "cc", + "libc", + "libz-sys", + "lz4-sys", +] + [[package]] name = "libsqlite3-sys" version = "0.35.0" @@ -2224,6 +2344,17 @@ dependencies = [ "libc", ] +[[package]] +name = "libz-sys" +version = "1.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linked-hash-map" version = "0.5.6" @@ -2331,6 +2462,16 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "matchers" version = "0.2.0" @@ -2432,7 +2573,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da3298dd3c75b119f9031631d0087d8d7de66ae5ac4da80ba7f243d2035b2a1f" dependencies = [ "enum_dispatch", - "miden-crypto", + "miden-crypto 0.17.1", "miden-debug-types", "miden-formatting", "num-derive", @@ -2472,6 +2613,49 @@ dependencies = [ "zeroize", ] +[[package]] +name = "miden-crypto" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffedb6b42cbc44634c3f9c58ee912b0d585fabdf3a19af56d8f185eb50125ef" +dependencies = [ + "blake3", + "cc", + "chacha20poly1305", + "ed25519-dalek", + "flume", + "glob", + "hashbrown 0.16.0", + "hkdf", + "k256", + "miden-crypto-derive", + "num", + "num-complex", + "rand 0.9.2", + "rand_chacha 0.9.0", + "rand_core 0.9.3", + "rand_hc", + "rayon", + "rocksdb", + "sha3", + "subtle", + "thiserror 2.0.17", + "winter-crypto", + "winter-math", + "winter-utils", + "x25519-dalek", +] + +[[package]] +name = "miden-crypto-derive" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d3b266500c6fe3584dacc773285a6e8c70b367b7ec0316fb17632a47f98b397" +dependencies = [ + "quote", + "syn 2.0.108", +] + [[package]] name = "miden-debug-types" version = "0.18.3" @@ -2479,7 +2663,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e230c51cae3ed84f295927ac736bca1a7668f12f187f7feeec16fbf6f50dca3e" dependencies = [ "memchr", - "miden-crypto", + "miden-crypto 0.17.1", "miden-formatting", "miden-miette", "miden-utils-sync", @@ -2755,6 +2939,7 @@ dependencies = [ "fs-err", "hex", "indexmap 2.12.0", + "miden-crypto 0.18.0", "miden-lib", "miden-node-proto", "miden-node-proto-build", @@ -2862,7 +3047,7 @@ dependencies = [ "miden-assembly", "miden-assembly-syntax", "miden-core", - "miden-crypto", + "miden-crypto 0.17.1", "miden-mast-package", "miden-processor", "miden-utils-sync", @@ -2981,7 +3166,7 @@ dependencies = [ "fs-err", "miden-assembly", "miden-core", - "miden-crypto", + "miden-crypto 0.17.1", "miden-processor", "miden-utils-sync", "thiserror 2.0.17", @@ -3036,7 +3221,7 @@ version = "0.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2c9186ccc6e826d35bcbf84f5a580c6401ba09b7735fc650b5ad75c61131c41" dependencies = [ - "miden-crypto", + "miden-crypto 0.17.1", "miden-debug-types", "miden-miette", "paste", @@ -3135,6 +3320,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -3231,6 +3422,16 @@ dependencies = [ "memoffset", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -4473,6 +4674,16 @@ dependencies = [ "serde", ] +[[package]] +name = "rocksdb" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddb7af00d2b17dbd07d82c0063e25411959748ff03e8d4f96134c2ff41fce34f" +dependencies = [ + "libc", + "librocksdb-sys", +] + [[package]] name = "rstest" version = "0.26.1" @@ -4518,6 +4729,12 @@ version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + [[package]] name = "rustc_version" version = "0.2.3" @@ -6743,6 +6960,16 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "x25519-dalek" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" +dependencies = [ + "curve25519-dalek", + "rand_core 0.6.4", +] + [[package]] name = "yaml-rust" version = "0.4.5" diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index 195336b9d..f6ed54fb7 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -24,6 +24,7 @@ diesel_migrations = { features = ["sqlite"], version = "2.2" } hex = { version = "0.4" } indexmap = { workspace = true } miden-lib = { workspace = true } +miden-crypto = { version = "0.18", features = ["hashmaps", "concurrent"] } miden-node-proto = { workspace = true } miden-node-proto-build = { features = ["internal"], workspace = true } miden-node-utils = { workspace = true } From 6326409686ba6aa721074bd34e6758e90da7e7ad Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 3 Nov 2025 19:01:04 +0100 Subject: [PATCH 3/9] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83f1243f8..40f0dbf2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - Add `validator` crate with initial protobuf, gRPC server, and sub-command (#[1293](https://github.com/0xMiden/miden-node/pull/1293)). - Add optional `TransactionInputs` field to `SubmitProvenTransaction` endpoint for transaction re-execution (#[1278](https://github.com/0xMiden/miden-node/pull/1278)). - [BREAKING] Added `AccountTreeWithHistory` and integrate historical queries into `GetAccountProof` ([#1292](https://github.com/0xMiden/miden-node/pull/1292)). +- [BREAKING] Added `rocksdb` feature to enable rocksdb as `LargeSmt` as `SmtStore` ([#1326](https://github.com/0xMiden/miden-node/pull/1326)). ## v0.11.2 (2025-09-10) From 1d0dd6d1d77b436a25d06fcdcb4ac6ade4498451 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 3 Nov 2025 19:04:37 +0100 Subject: [PATCH 4/9] yes, 0.17 --- crates/store/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index f6ed54fb7..1c205ac12 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -24,7 +24,7 @@ diesel_migrations = { features = ["sqlite"], version = "2.2" } hex = { version = "0.4" } indexmap = { workspace = true } miden-lib = { workspace = true } -miden-crypto = { version = "0.18", features = ["hashmaps", "concurrent"] } +miden-crypto = { version = "0.17", features = ["hashmaps", "concurrent"] } miden-node-proto = { workspace = true } miden-node-proto-build = { features = ["internal"], workspace = true } miden-node-utils = { workspace = true } From 24b01c72daf1411c7679eecf0516a4f37a24a093 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 3 Nov 2025 19:04:50 +0100 Subject: [PATCH 5/9] downgrade --- Cargo.lock | 129 ++++------------------------------------------------- 1 file changed, 9 insertions(+), 120 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 44a114841..be74edef6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -929,33 +929,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "curve25519-dalek" -version = "4.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" -dependencies = [ - "cfg-if", - "cpufeatures", - "curve25519-dalek-derive", - "digest", - "fiat-crypto", - "rustc_version 0.4.1", - "subtle", - "zeroize", -] - -[[package]] -name = "curve25519-dalek-derive" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.108", -] - [[package]] name = "daemonize" version = "0.5.0" @@ -1276,30 +1249,6 @@ dependencies = [ "spki", ] -[[package]] -name = "ed25519" -version = "2.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" -dependencies = [ - "pkcs8", - "signature", -] - -[[package]] -name = "ed25519-dalek" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" -dependencies = [ - "curve25519-dalek", - "ed25519", - "serde", - "sha2", - "subtle", - "zeroize", -] - [[package]] name = "either" version = "1.15.0" @@ -1411,12 +1360,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "fiat-crypto" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" - [[package]] name = "figment" version = "0.10.19" @@ -1747,8 +1690,6 @@ dependencies = [ "allocator-api2", "equivalent", "foldhash 0.2.0", - "rayon", - "serde", ] [[package]] @@ -2573,7 +2514,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da3298dd3c75b119f9031631d0087d8d7de66ae5ac4da80ba7f243d2035b2a1f" dependencies = [ "enum_dispatch", - "miden-crypto 0.17.1", + "miden-crypto", "miden-debug-types", "miden-formatting", "num-derive", @@ -2605,55 +2546,13 @@ dependencies = [ "rand_core 0.9.3", "rand_hc", "rayon", - "sha3", - "thiserror 2.0.17", - "winter-crypto", - "winter-math", - "winter-utils", - "zeroize", -] - -[[package]] -name = "miden-crypto" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffedb6b42cbc44634c3f9c58ee912b0d585fabdf3a19af56d8f185eb50125ef" -dependencies = [ - "blake3", - "cc", - "chacha20poly1305", - "ed25519-dalek", - "flume", - "glob", - "hashbrown 0.16.0", - "hkdf", - "k256", - "miden-crypto-derive", - "num", - "num-complex", - "rand 0.9.2", - "rand_chacha 0.9.0", - "rand_core 0.9.3", - "rand_hc", - "rayon", "rocksdb", "sha3", - "subtle", "thiserror 2.0.17", "winter-crypto", "winter-math", "winter-utils", - "x25519-dalek", -] - -[[package]] -name = "miden-crypto-derive" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d3b266500c6fe3584dacc773285a6e8c70b367b7ec0316fb17632a47f98b397" -dependencies = [ - "quote", - "syn 2.0.108", + "zeroize", ] [[package]] @@ -2663,7 +2562,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e230c51cae3ed84f295927ac736bca1a7668f12f187f7feeec16fbf6f50dca3e" dependencies = [ "memchr", - "miden-crypto 0.17.1", + "miden-crypto", "miden-formatting", "miden-miette", "miden-utils-sync", @@ -2939,7 +2838,7 @@ dependencies = [ "fs-err", "hex", "indexmap 2.12.0", - "miden-crypto 0.18.0", + "miden-crypto", "miden-lib", "miden-node-proto", "miden-node-proto-build", @@ -3047,7 +2946,7 @@ dependencies = [ "miden-assembly", "miden-assembly-syntax", "miden-core", - "miden-crypto 0.17.1", + "miden-crypto", "miden-mast-package", "miden-processor", "miden-utils-sync", @@ -3166,7 +3065,7 @@ dependencies = [ "fs-err", "miden-assembly", "miden-core", - "miden-crypto 0.17.1", + "miden-crypto", "miden-processor", "miden-utils-sync", "thiserror 2.0.17", @@ -3221,7 +3120,7 @@ version = "0.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2c9186ccc6e826d35bcbf84f5a580c6401ba09b7735fc650b5ad75c61131c41" dependencies = [ - "miden-crypto 0.17.1", + "miden-crypto", "miden-debug-types", "miden-miette", "paste", @@ -4676,9 +4575,9 @@ dependencies = [ [[package]] name = "rocksdb" -version = "0.24.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddb7af00d2b17dbd07d82c0063e25411959748ff03e8d4f96134c2ff41fce34f" +checksum = "26ec73b20525cb235bad420f911473b69f9fe27cc856c5461bccd7e4af037f43" dependencies = [ "libc", "librocksdb-sys", @@ -6960,16 +6859,6 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" -[[package]] -name = "x25519-dalek" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" -dependencies = [ - "curve25519-dalek", - "rand_core 0.6.4", -] - [[package]] name = "yaml-rust" version = "0.4.5" From cff519af797f51c3faf3e4a9ea8ca35f923f938e Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 3 Nov 2025 19:12:02 +0100 Subject: [PATCH 6/9] lint --- crates/store/Cargo.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index 1c205ac12..2a8b2f705 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -23,8 +23,8 @@ diesel = { features = ["numeric", "sqlite"], version = "2.2" } diesel_migrations = { features = ["sqlite"], version = "2.2" } hex = { version = "0.4" } indexmap = { workspace = true } +miden-crypto = { features = ["concurrent", "hashmaps"], version = "0.17" } miden-lib = { workspace = true } -miden-crypto = { version = "0.17", features = ["hashmaps", "concurrent"] } miden-node-proto = { workspace = true } miden-node-proto-build = { features = ["internal"], workspace = true } miden-node-utils = { workspace = true } @@ -61,3 +61,7 @@ rocksdb = ["miden-crypto/rocksdb"] [[bench]] harness = false name = "account_tree_historical" + +[package.metadata.cargo-machete] +# we only need it for dependency feature unification for `rocksdb` +ignored = ["miden-crypto"] From a82a430a4b98e97f0174c20dfe92bb62a5727cef Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 3 Nov 2025 21:35:03 +0100 Subject: [PATCH 7/9] integral historical data --- .../db/migrations/2025062000000_setup/up.sql | 3 +- crates/store/src/db/mod.rs | 13 ++++ .../store/src/db/models/queries/accounts.rs | 62 +++++++++++++++++-- crates/store/src/db/schema.rs | 7 ++- crates/store/src/errors.rs | 2 + crates/store/src/state.rs | 31 +++++++--- 6 files changed, 98 insertions(+), 20 deletions(-) diff --git a/crates/store/src/db/migrations/2025062000000_setup/up.sql b/crates/store/src/db/migrations/2025062000000_setup/up.sql index 1eb7a9728..616688351 100644 --- a/crates/store/src/db/migrations/2025062000000_setup/up.sql +++ b/crates/store/src/db/migrations/2025062000000_setup/up.sql @@ -22,7 +22,7 @@ CREATE TABLE accounts ( vault BLOB, nonce INTEGER, - PRIMARY KEY (account_id), + PRIMARY KEY (account_id, block_num), FOREIGN KEY (block_num) REFERENCES block_headers(block_num), FOREIGN KEY (code_commitment) REFERENCES account_codes(code_commitment), CONSTRAINT all_null_or_none_null CHECK @@ -34,6 +34,7 @@ CREATE TABLE accounts ( ) WITHOUT ROWID; CREATE INDEX idx_accounts_network_prefix ON accounts(network_account_id_prefix) WHERE network_account_id_prefix IS NOT NULL; +CREATE INDEX idx_accounts_id_block ON accounts(account_id, block_num DESC); CREATE TABLE notes ( committed_at INTEGER NOT NULL, -- Block number when the note was committed diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 74277ee83..bb894179e 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -406,6 +406,19 @@ impl Db { .await } + /// Loads account details at a specific block number from the DB. + #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] + pub async fn select_historical_account_at( + &self, + id: AccountId, + block_num: BlockNumber, + ) -> Result { + self.transact("Get historical account details", move |conn| { + queries::select_historical_account_at(conn, id, block_num) + }) + .await + } + /// Loads public account details from the DB based on the account ID's prefix. #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_network_account_by_prefix( diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 694805bf9..488cb83c7 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -72,6 +72,9 @@ use crate::errors::DatabaseError; /// account_codes ON accounts.code_commitment = account_codes.code_commitment /// WHERE /// account_id = ?1 +/// ORDER BY +/// block_num DESC +/// LIMIT 1 /// ``` pub(crate) fn select_account( conn: &mut SqliteConnection, @@ -84,6 +87,58 @@ pub(crate) fn select_account( (AccountRaw::as_select(), schema::account_codes::code.nullable()), ) .filter(schema::accounts::account_id.eq(account_id.to_bytes())) + .order_by(schema::accounts::block_num.desc()) + .limit(1) + .get_result::<(AccountRaw, Option>)>(conn) + .optional()? + .ok_or(DatabaseError::AccountNotFoundInDb(account_id))?; + let info = AccountWithCodeRawJoined::from(raw).try_into()?; + Ok(info) +} + +/// Select account details at a specific block number from the DB using the given +/// [`SqliteConnection`]. +/// +/// # Returns +/// +/// The account details at the specified block, or an error. +/// +/// # Raw SQL +/// +/// ```sql +/// SELECT +/// accounts.account_id, +/// accounts.account_commitment, +/// accounts.block_num, +/// accounts.storage, +/// accounts.vault, +/// accounts.nonce, +/// accounts.code_commitment, +/// account_codes.code +/// FROM +/// accounts +/// LEFT JOIN +/// account_codes ON accounts.code_commitment = account_codes.code_commitment +/// WHERE +/// account_id = ?1 +/// AND block_num = ?2 +/// ``` +pub(crate) fn select_historical_account_at( + conn: &mut SqliteConnection, + account_id: AccountId, + block_num: BlockNumber, +) -> Result { + let raw = SelectDsl::select( + schema::accounts::table.left_join(schema::account_codes::table.on( + schema::accounts::code_commitment.eq(schema::account_codes::code_commitment.nullable()), + )), + (AccountRaw::as_select(), schema::account_codes::code.nullable()), + ) + .filter( + schema::accounts::account_id + .eq(account_id.to_bytes()) + .and(schema::accounts::block_num.eq(block_num.to_raw_sql())), + ) .get_result::<(AccountRaw, Option>)>(conn) .optional()? .ok_or(DatabaseError::AccountNotFoundInDb(account_id))?; @@ -848,12 +903,7 @@ pub(crate) fn upsert_accounts( }; let v = account_value.clone(); - let inserted = diesel::insert_into(schema::accounts::table) - .values(&v) - .on_conflict(schema::accounts::account_id) - .do_update() - .set(account_value) - .execute(conn)?; + let inserted = diesel::insert_into(schema::accounts::table).values(&v).execute(conn)?; debug_assert_eq!(inserted, 1); diff --git a/crates/store/src/db/schema.rs b/crates/store/src/db/schema.rs index 2b81eb52d..31fc4fa57 100644 --- a/crates/store/src/db/schema.rs +++ b/crates/store/src/db/schema.rs @@ -22,7 +22,7 @@ diesel::table! { } diesel::table! { - accounts (account_id) { + accounts (account_id, block_num) { account_id -> Binary, network_account_id_prefix -> Nullable, account_commitment -> Binary, @@ -100,11 +100,12 @@ diesel::table! { diesel::joinable!(accounts -> account_codes (code_commitment)); diesel::joinable!(accounts -> block_headers (block_num)); -diesel::joinable!(notes -> accounts (sender)); +// Note: Cannot use diesel::joinable! with accounts table due to composite primary key +// diesel::joinable!(notes -> accounts (sender)); +// diesel::joinable!(transactions -> accounts (account_id)); diesel::joinable!(notes -> block_headers (committed_at)); diesel::joinable!(notes -> note_scripts (script_root)); diesel::joinable!(nullifiers -> block_headers (block_num)); -diesel::joinable!(transactions -> accounts (account_id)); diesel::joinable!(transactions -> block_headers (block_num)); diesel::allow_tables_to_appear_in_same_query!( diff --git a/crates/store/src/errors.rs b/crates/store/src/errors.rs index 639032104..ac4204d80 100644 --- a/crates/store/src/errors.rs +++ b/crates/store/src/errors.rs @@ -98,6 +98,8 @@ pub enum DatabaseError { AccountNotFoundInDb(AccountId), #[error("account {0} state at block height {1} not found")] AccountAtBlockHeightNotFoundInDb(AccountId, BlockNumber), + #[error("historical block {block_num} not available: {reason}")] + HistoricalBlockNotAvailable { block_num: BlockNumber, reason: String }, #[error("accounts {0:?} not found")] AccountsNotFoundInDb(Vec), #[error("account {0} is not on the chain")] diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 9e398027c..f8efd8c17 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -916,27 +916,38 @@ impl State { /// Returns the respective account proof with optional details, such as asset and storage /// entries. /// - /// Note: The `block_num` parameter in the request is currently ignored and will always - /// return the current state. Historical block support will be implemented in a future update. + /// When `block_num` is provided, this method will return the account state at that specific + /// block using both the historical account tree witness and historical database state. #[allow(clippy::too_many_lines)] pub async fn get_account_proof( &self, account_request: AccountProofRequest, ) -> Result { let AccountProofRequest { block_num, account_id, details } = account_request; - let _ = block_num.ok_or_else(|| { - DatabaseError::NotImplemented( - "Handling of historical/past block numbers is not implemented yet".to_owned(), - ) - }); // Lock inner state for the whole operation. We need to hold this lock to prevent the // database, account tree and latest block number from changing during the operation, // because changing one of them would lead to inconsistent state. let inner_state = self.inner.read().await; - let block_num = inner_state.account_tree.block_number_latest(); - let witness = inner_state.account_tree.open_latest(account_id); + // Determine which block to query + let (block_num, witness) = if let Some(requested_block) = block_num { + // Historical query: use the account tree with history + let witness = inner_state + .account_tree + .open_at(account_id, requested_block) + .ok_or_else(|| DatabaseError::HistoricalBlockNotAvailable { + block_num: requested_block, + reason: "Block is either in the future or has been pruned from history" + .to_string(), + })?; + (requested_block, witness) + } else { + // Latest query: use the latest state + let block_num = inner_state.account_tree.block_number_latest(); + let witness = inner_state.account_tree.open_latest(account_id); + (block_num, witness) + }; let account_details = if let Some(AccountDetailRequest { code_commitment, @@ -944,7 +955,7 @@ impl State { storage_requests, }) = details { - let account_info = self.db.select_account(account_id).await?; + let account_info = self.db.select_historical_account_at(account_id, block_num).await?; // if we get a query for a _private_ account _with_ details requested, we'll error out let Some(account) = account_info.details else { From f68ad9dc5d075a12629dae88175094aabdae23dd Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 4 Nov 2025 11:40:47 +0100 Subject: [PATCH 8/9] Reapply "use rocksdb" This reverts commit 50e54e7b5aac59f1a7822150ff96e906c9cab199. --- crates/store/Cargo.toml | 1 + .../store/benches/account_tree_historical.rs | 31 +++++++++++++------ crates/store/src/lib.rs | 4 +-- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index 2a8b2f705..bf63c590a 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -61,6 +61,7 @@ rocksdb = ["miden-crypto/rocksdb"] [[bench]] harness = false name = "account_tree_historical" +required-features = ["rocksdb"] [package.metadata.cargo-machete] # we only need it for dependency feature unification for `rocksdb` diff --git a/crates/store/benches/account_tree_historical.rs b/crates/store/benches/account_tree_historical.rs index dbb538d5a..f27a31f98 100644 --- a/crates/store/benches/account_tree_historical.rs +++ b/crates/store/benches/account_tree_historical.rs @@ -1,22 +1,35 @@ use std::hint::black_box; +use std::path::PathBuf; +use std::sync::atomic::{AtomicUsize, Ordering}; use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; -use miden_node_store::{AccountTreeWithHistory, InMemoryAccountTree}; +use miden_node_store::{AccountTreeWithHistory, PersistentAccountTree}; use miden_objects::Word; use miden_objects::account::AccountId; use miden_objects::block::BlockNumber; use miden_objects::block::account_tree::{AccountTree, account_id_to_smt_key}; use miden_objects::crypto::hash::rpo::Rpo256; -use miden_objects::crypto::merkle::{LargeSmt, MemoryStorage}; +use miden_objects::crypto::merkle::{LargeSmt, RocksDbConfig, RocksDbStorage}; use miden_objects::testing::account_id::AccountIdBuilder; +/// Counter for creating unique RocksDB directories during benchmarking. +static DB_COUNTER: AtomicUsize = AtomicUsize::new(0); + // HELPER FUNCTIONS // ================================================================================================ -/// Creates a storage backend for a `LargeSmt`. -fn setup_storage() -> MemoryStorage { - // TODO migrate to RocksDB for persistence to gain meaningful numbers - MemoryStorage::default() +/// Creates a `RocksDB` storage instance in target/bench_* directory for benchmarking. +fn setup_storage() -> RocksDbStorage { + let counter = DB_COUNTER.fetch_add(1, Ordering::SeqCst); + let db_path = PathBuf::from(format!("target/bench_rocksdb_{counter}")); + + // Clean up the directory if it exists + if db_path.exists() { + fs_err::remove_dir_all(&db_path).ok(); + } + fs_err::create_dir_all(db_path.parent().unwrap()).expect("Creation is not enough"); + + RocksDbStorage::open(RocksDbConfig::new(db_path)).expect("RocksDB failed to open file") } /// Generates a deterministic word from a seed. @@ -47,7 +60,7 @@ fn generate_account_id(seed: &mut [u8; 32]) -> AccountId { /// Sets up a vanilla `AccountTree` with specified number of accounts. fn setup_vanilla_account_tree( num_accounts: usize, -) -> (AccountTree>, Vec) { +) -> (AccountTree>, Vec) { let mut seed = [0u8; 32]; let mut account_ids = Vec::new(); let mut entries = Vec::new(); @@ -70,7 +83,7 @@ fn setup_vanilla_account_tree( fn setup_account_tree_with_history( num_accounts: usize, num_blocks: usize, -) -> (AccountTreeWithHistory, Vec) { +) -> (AccountTreeWithHistory, Vec) { let mut seed = [0u8; 32]; let storage = setup_storage(); let smt = LargeSmt::with_entries(storage, std::iter::empty()) @@ -164,7 +177,7 @@ fn bench_historical_access(c: &mut Criterion) { for &num_accounts in &account_counts { for &block_depth in &block_depths { - if block_depth > AccountTreeWithHistory::::MAX_HISTORY { + if block_depth > AccountTreeWithHistory::::MAX_HISTORY { continue; } diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index bfab4f4d4..f6b161c8f 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -8,14 +8,14 @@ pub mod genesis; mod server; pub mod state; -#[cfg(feature = "rocksdb")] -pub use accounts::PersistentAccountTree; pub use accounts::{ AccountTreeStorage, AccountTreeWithHistory, HistoricalError, InMemoryAccountTree, }; +#[cfg(feature = "rocksdb")] +pub use accounts::PersistentAccountTree; pub use genesis::GenesisState; pub use server::{DataDirectory, Store}; From 397aaa97a69b1e936e001e0af44b58a57a58ed94 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 4 Nov 2025 11:44:25 +0100 Subject: [PATCH 9/9] fmt --- crates/store/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index f6b161c8f..bfab4f4d4 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -8,14 +8,14 @@ pub mod genesis; mod server; pub mod state; +#[cfg(feature = "rocksdb")] +pub use accounts::PersistentAccountTree; pub use accounts::{ AccountTreeStorage, AccountTreeWithHistory, HistoricalError, InMemoryAccountTree, }; -#[cfg(feature = "rocksdb")] -pub use accounts::PersistentAccountTree; pub use genesis::GenesisState; pub use server::{DataDirectory, Store};