diff --git a/miden-crypto/src/merkle/smt/large_forest/error.rs b/miden-crypto/src/merkle/smt/large_forest/error.rs deleted file mode 100644 index 14ca4cf3e..000000000 --- a/miden-crypto/src/merkle/smt/large_forest/error.rs +++ /dev/null @@ -1,21 +0,0 @@ -//! This module contains the error types and helpers for working with errors from the large SMT -//! forest. - -use thiserror::Error; - -use crate::merkle::{MerkleError, smt::large_forest::history::error::HistoryError}; - -/// The errors returned by operations on the large SMT forest. -/// -/// This type primarily serves to wrap more specific error types from various subsystems into a -/// generic interface type. -#[derive(Debug, Error)] -pub enum LargeSmtForestError { - #[error(transparent)] - HistoryError(#[from] HistoryError), - - #[error(transparent)] - MerkleError(#[from] MerkleError), -} - -pub mod history {} diff --git a/miden-crypto/src/merkle/smt/large_forest/error/mod.rs b/miden-crypto/src/merkle/smt/large_forest/error/mod.rs new file mode 100644 index 000000000..e959609a7 --- /dev/null +++ b/miden-crypto/src/merkle/smt/large_forest/error/mod.rs @@ -0,0 +1,28 @@ +//! This module contains the error types and helpers for working with errors from the large SMT +//! forest. + +pub mod subtree; + +use thiserror::Error; + +use crate::merkle::{ + MerkleError, + smt::large_forest::{history::error::HistoryError, storage}, +}; + +/// The type of errors returned by operations on the large SMT forest. +#[derive(Debug, Error)] +pub enum LargeSmtForestError { + #[error(transparent)] + HistoryError(#[from] HistoryError), + + #[error(transparent)] + MerkleError(#[from] MerkleError), + + #[error(transparent)] + StorageError(#[from] storage::StorageError), +} + +/// The result type for use within the large SMT forest portion of the library. +#[allow(dead_code)] // Temporary +pub type Result = core::result::Result; diff --git a/miden-crypto/src/merkle/smt/large_forest/error/subtree.rs b/miden-crypto/src/merkle/smt/large_forest/error/subtree.rs new file mode 100644 index 000000000..cf5d14603 --- /dev/null +++ b/miden-crypto/src/merkle/smt/large_forest/error/subtree.rs @@ -0,0 +1,35 @@ +//! Errors in working with subtrees. + +use thiserror::Error; + +/// Errors raised when encountering an issue when working with subtrees. +#[derive(Debug, Error)] +pub enum SubtreeError { + /// Raised when decoding the subtree from bytes and encountering insufficient node data. + #[error("Expected {expected} bytes of node data, found {found} bytes")] + BadHashLen { expected: usize, found: usize }, + + /// Raised when the left index has an invalid hash. + #[error("Invalid left hash format at local index {index}")] + BadLeft { index: u64 }, + + /// Raised when the left index has an invalid hash. + #[error("Invalid right hash format at local index {index}")] + BadRight { index: u64 }, + + /// When extra node data exists in the bytestream after the expected data. + #[error("Found extra node data after bitmask-indicated entries")] + ExtraData, + + /// When the node data for the left index of a node. + #[error("Missing left node data at local index {index}")] + MissingLeft { index: u64 }, + + /// When the node data for the right index of a node. + #[error("Missing right node data at local index {index}")] + MissingRight { index: u64 }, + + /// Raised when there is insufficient data when decoding the subtree. + #[error("Found {found} bytes when decoding the subtree, but need at least {min} bytes")] + TooShort { found: usize, min: usize }, +} diff --git a/miden-crypto/src/merkle/smt/large_forest/mod.rs b/miden-crypto/src/merkle/smt/large_forest/mod.rs index e67a5f27a..5734ce865 100644 --- a/miden-crypto/src/merkle/smt/large_forest/mod.rs +++ b/miden-crypto/src/merkle/smt/large_forest/mod.rs @@ -1,7 +1,45 @@ //! A high-performance sparse merkle tree forest backed by pluggable storage. +//! +//! # Semantic Layout +//! +//! Much like `SparseMerkleTree`, the forest stores trees of depth 64 that use the compact leaf +//! optimization to uniquely store 256-bit elements. This reduces both the size of a merkle path, +//! and the computational work necessary to perform queries into the trees. +//! +//! # Storing Trees and Versions +//! +//! The usage of an SMT forest is conceptually split into two parts: a collection that is able to +//! store **multiple, unrelated trees**, and a container for **multiple versions of those trees**. +//! Both of these use-cases are supported by the forest, but have an explicit delineation between +//! them in both the API and the implementation. This has two impacts that a client of the forest +//! must understand. +//! +//! - While, when using a [`Storage`] that can persist data, **only the current full tree state is +//! persisted**, while **the historical data will not be**. This is designed into the structure of +//! the forest, and does not depend on the choice of storage backend. +//! - It is more expensive to query a given tree at an older point in its history than it is to +//! query it at a newer point, and querying at the current tree will always take the least time. +//! +//! # Data Storage +//! +//! In order to help with the query performance for the more latency-prone kinds of [`Storage`] +//! implementation, the forest splits the data into two portions: +//! +//! 1. The **top of each tree** is explicitly **stored in memory**, regardless of the [`Storage`] +//! backend. This makes the common tree prefix much more performant to query, and relies on the +//! backend to store sufficient data to _reconstruct_ that prefix at forest rebuild. +//! 2. The **rest of each tree** is managed by the [`Storage`] itself, and makes no guarantees as to +//! where that data is stored. Queries into this portion have performance characteristics +//! dictated by the choice of storage backend. +//! +//! The split between these numbers of levels is configured when initially constructing the forest, +//! and will be verified at runtime for forests that are instead reloaded from persistent state. mod error; -mod history; +pub mod history; +pub mod storage; +pub mod utils; pub use error::LargeSmtForestError; -pub use history::{History, HistoryView, error::HistoryError}; +pub use storage::{Storage, StorageError, StoredTreeHandle}; +pub use utils::SubtreeLevels; diff --git a/miden-crypto/src/merkle/smt/large_forest/storage/error.rs b/miden-crypto/src/merkle/smt/large_forest/storage/error.rs new file mode 100644 index 000000000..f52f7ba45 --- /dev/null +++ b/miden-crypto/src/merkle/smt/large_forest/storage/error.rs @@ -0,0 +1,59 @@ +//! This module contains the error types and helpers for working with errors from the large SMT +//! forest. + +use alloc::{boxed::Box, string::String}; + +use thiserror::Error; + +use crate::{ + Word, + merkle::smt::{SmtLeafError, large_forest::error::subtree::SubtreeError}, +}; + +/// The type of errors returned by operations on the large SMT forest. +#[derive(Debug, Error)] +pub enum StorageError { + /// An error coming directly from a malfunction in the storage backend. + #[error(transparent)] + Backend(#[from] Box), + + /// An error with the contents of a compact tree leaf. + #[error(transparent)] + Leaf(#[from] SmtLeafError), + + /// Raised when a storage backend does not support multiple concurrent transactions. + #[error("This backend does not support multiple concurrent transactions")] + MultipleTransactionsUnsupported, + + /// Raised when an entity is requested from the storage but is not managed by this storage due + /// to being part of the guaranteed-to-be-in-memory storage. + #[error("The entity {0} is not part of this storage")] + NotInStorage(String), + + /// Issued if an operation that can only be performed with an active transaction is performed + /// outside a transaction. + #[error("An operation was issued that requires a transaction to have been started.")] + NotInTransaction, + + /// An error when reading from or writing to a subtree in storage. + #[error(transparent)] + Subtree(#[from] SubtreeError), + + /// Raised when the storage does not store a tree with the provided root. + #[error("No tree with root {0} exists in this storage")] + UnknownRoot(Word), + + /// Raised when a storage implementation is given a transaction handle that it did not allocate. + #[error("A transaction handle was provided that was not issued by this storage")] + UnknownTransaction, + + /// The requested operation is not supported by this backend. + /// + /// In some cases it may be possible to fall back to a more complex slow path when this error is + /// received. + #[error("The operation {0} is not supported")] + UnsupportedOperation(String), +} + +/// The result type for use within the large SMT forest portion of the library. +pub type Result = core::result::Result; diff --git a/miden-crypto/src/merkle/smt/large_forest/storage/mod.rs b/miden-crypto/src/merkle/smt/large_forest/storage/mod.rs new file mode 100644 index 000000000..3b2098702 --- /dev/null +++ b/miden-crypto/src/merkle/smt/large_forest/storage/mod.rs @@ -0,0 +1,540 @@ +//! This module contains the definition of [`Storage`], the backing data store for the SMT forest. +//! +//! The forest itself is designed to be storage agnostic, and provides the minimum set of operations +//! for reading and writing to the storage backend. It is intended that very little logic should +//! reside in the storage itself. The primary documentation for those both using and implementing +//! the traits in this module can be found on [`Storage`] and [`StoredTreeHandle`] themselves. +//! +//! # Performance +//! +//! Having an arbitrary `S: Storage` does not give you any performance guarantees about the forest +//! itself. To that end, reasoning about performance should always be done in the context of a +//! concrete instance of the storage. Please see specific implementations for details about +//! performance. + +use alloc::{boxed::Box, vec::Vec}; +use core::fmt::Debug; + +pub mod error; + +pub use error::{Result, StorageError}; + +use crate::{ + Word, + merkle::{ + NodeIndex, + smt::{InnerNode, SmtLeaf, large_forest::utils::SubtreeLevels}, + }, +}; + +/// A **temporary** stand-in for the subtree type that will come in a subsequent PR. +pub type Subtree = (); + +// STORAGE TRAIT +// ================================================================================================ + +/// The backing storage for the large SMT forest, providing the necessary methods for reading +/// from and writing to an arbitrary storage structure while allowing the forest itself to be +/// storage agnostic. +/// +/// # Storage Structure +/// +/// The forest's data is physically split into two parts, though this division should not be visible +/// to clients of the forest. +/// +/// 1. **Guaranteed In Memory:** This portion of the forest, consisting of some number of levels +/// from the top of each stored tree is guaranteed to reside in memory. +/// 2. **Arbitrary Storage:** The remainder of the forest's data is stored in a place that makes no +/// guarantees as to where it is stored (e.g. in memory, on disk, and so on). +/// +/// An implementation of [`Storage`] is responsible for managing the latter part, and has no +/// information about the portion of the data guaranteed to reside in memory. Merging the data from +/// the two to create a coherent forest is the responsibility of the forest implementation itself. +/// +/// **Note that** the forest may choose to retain **no portion of the node data** as guaranteed to +/// be in memory. This means that storage implementations must be **agnostic** to the portion of the +/// tree that they store. At a minimum they could store the leaves, and at the maximum they could +/// store the entire tree, though with many storage backends this may incur significant performance +/// cost. +/// +/// # Transactions +/// +/// Many implementations of [`Storage`] will need to use transaction-based operations to ensure +/// consistency for the data in the case of crashes. To that end, the trait provides functionality +/// for working with transactions as follows: +/// +/// 1. Request a transaction to start by calling [`Storage::begin`]. +/// 2. Perform any number of operations on the individual persisted tree states in the forest by +/// using [`Storage::tree`] as an interface to do so. +/// 3. Request that the transaction end by calling [`Storage::commit`]. +/// +/// While this workflow is an inherent part of the trait, not all backends will support the use of +/// transactions. A purely in-memory backend, for example, will likely perform its reads and writes +/// eagerly. This means that a client of a [`Storage`] implementation, namely the SMT forest, **must +/// not make assumptions** about **_when_ and _in what order_ data is written to disk**. +/// +/// - It is recommended that any **storage that does support transactions** guard operations on +/// there being an active transaction. Not doing so risks subtle, programmer-induced data +/// corruption. +/// - For **storage that does not support transactions**, it is recommended to set +/// `TransactionHandle = ()` to indicate this. Nevertheless, it must support calling +/// [`Storage::begin`] and [`Storage::commit`] without raising an error so the forest can be +/// properly storage agnostic. +/// +/// This ensures that the forest that uses the storage can assume the existence of transactions and +/// use them properly, regardless of the actual transaction support in a given backend. If they are +/// supported they will be used, and if they are not there is no harm done with a nop call. +/// +/// # Interior Mutability +/// +/// This trait is designed to take advantage of the _interior mutability_ pattern, allowing anybody +/// with a standard reference to the storage to mutate it. This is to support a reasonable interface +/// that can be employed from parallel contexts, while also supporting a wide array of potential +/// concrete storage implementations. +/// +/// As a result, **care must be taken** to ensure the correctness of any given storage +/// implementation, as the type system is less able to assist the programmer with its correctness. +/// +/// # Errors +/// +/// All methods are intended to handle potential storage errors by returning the trait's [`Result`] +/// type. Suggested errors are documented for each method, but these may be changed by any concrete +/// implementation of the trait. +pub trait Storage +where + Self: Debug + Send + Sync + 'static, +{ + /// The type of the handle used to identify transactions in the storage. + /// + /// The handle type may be set to `()` to indicate that the particular backend does not support + /// transactions. Such backends may mutate their stored data eagerly instead. + type TransactionHandle; + + /// The type of handles to the data associated with a **specific tree** in the storage. + type TreeDataHandle: StoredTreeHandle; + + /// Returns the number of levels in each tree that are guaranteed to be stored in memory, and + /// hence that are **not managed by the storage** itself, in order to ensure correct + /// forest-state restoration in persistent storage scenarios. + /// + /// Please see the documentation of [`SubtreeLevels`] for the exact way these are counted. + fn in_memory_depth(&self) -> Result; + + /// Returns the number of unique trees that have had data stored within the storage. + fn tree_count(&self) -> Result; + + /// Returns the roots for all the trees that have data stored in the forest. + fn roots(&self) -> Result>; + + /// Begins a new transaction, returning a handle to it. + /// + /// **Not all storage backends may support a notion of transactions**. In such cases this may + /// be a no-op, and it is recommended to set `TransactionHandle = ()` to indicate this. Such + /// backends will mutate data eagerly, but still must allow [`Storage::begin`] to succeed. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while trying to begin a transaction. + /// - [`StorageError::MultipleTransactionsUnsupported`] if the backend does not support multiple + /// concurrent transactions. + fn begin(&self) -> Result; + + /// Ends the transaction described by `handle`, making all staged writes to the storage concrete + /// and durable. + /// + /// **Not all storage backends may support a notion of atomic writes**. In such cases this may + /// be a no-op, and it is recommended to set `TransactionHandle = ()` to indicate this. Such + /// backends will mutate data eagerly, but still must allow [`Storage::begin`] to succeed. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while trying to commit a transaction. + /// - [`StorageError::UnknownTransaction`] if the backend is passed a transaction handle that it + /// did not allocate to the caller. + fn commit(&self, handle: Self::TransactionHandle) -> Result<()>; + + /// Gets a handle to the storage as if it only consists of the data for the tree with the + /// provided `root`. + /// + /// It is intended that all manipulation of a given tree in storage takes place via the returned + /// [`Self::TreeDataHandle`], and as such there are no manipulation operations exposed directly + /// on the storage. This improves the ergonomics of working with the data of multiple trees in + /// the forest at once. + /// + /// Multiple of these handles must be able to be held and used at once by the caller of the + /// `Storage`, as this is the explicitly-intended use-case. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while trying to query for such a tree. + /// - [`StorageError::NotInTransaction`] if the backend supports transactions and this operation + /// is called while no transaction is active. + /// - [`StorageError::UnknownRoot`] if no tree with the provided `root` exists in the storage. + fn tree(&self, root: Word) -> Result; +} + +// TREE VIEW TRAIT +// ================================================================================================ + +/// A handle to the data for the specified tree in the storage. +/// +/// This handle **does not provide access to an entire tree**, but instead only grants access to +/// the **portion of the tree managed by the storage** (see the [`Storage`] documentation for more +/// detail). It primarily exists for the purpose of ergonomics, as it avoids the need to identify +/// the intended tree in every operation. +/// +/// # Reading and Writing +/// +/// Read and write operations via this view are **not guaranteed to be synchronous**. Write +/// operations in particular may be deferred (e.g. to ensure on-disk consistency in case of a +/// crash). This means that some errors may appear to be decoupled from the actions that actually +/// caused them. +/// +/// # Interior Mutability +/// +/// This trait is designed to take advantage of the _interior mutability_ pattern, allowing anybody +/// with a standard reference to the storage to mutate it. This is to support a reasonable interface +/// that can be employed from parallel contexts, while also supporting a wide array of potential +/// concrete storage implementations. +/// +/// As a result, **care must be taken** to ensure the correctness of the storage implementation, as +/// the type system is less able to assist the programmer in its implementation. +/// +/// # Errors +/// +/// All methods are intended to handle potential storage errors by returning the trait's [`Result`] +/// type. Suggested errors are documented for each method, but these may be changed by any concrete +/// implementation of the trait. +pub trait StoredTreeHandle +where + Self: Debug + Send + Sync + 'static, +{ + /// Gets the stored root of the tree whose data this handle points to. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while trying to read the root. + fn root(&self) -> Result; + + /// Sets the stored root for the tree in question to the provided `root`. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while trying update the root. + fn set_root(&self, root: Word) -> Result; + + /// Gets the number of leaves that are _currently stored_ for this tree. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while reading the leaf count. + fn leaf_count(&self) -> Result; + + /// Sets the number of leaves that are _currently stored_ for this tree. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while reading the leaf count. + fn set_leaf_count(&self, leaf_count: usize) -> Result<()>; + + /// Gets the number of **unique entries** that are currently stored across all leaf nodes for + /// this tree. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while reading the leaf count. + /// - [`StorageError::Leaf`] if a malformed leaf is found during the query. + fn entry_count(&self) -> Result; + + /// Sets the number of **unique entries** that are currently stored across all leaf nodes for + /// this tree. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while reading the leaf count. + fn set_entry_count(&self, entry: Word) -> Result<()>; + + /// Inserts the provided `value` under the given `key` into the tree, returning the previous + /// value for that key or [`None`] otherwise. + /// + /// - If the corresponding leaf does not exist, this method may create it. + /// - If the `key` exists in the leaf, its value is updated. + /// + /// This method **does not** handle propagating the changes that result from making the + /// insertion into the leaf. This is the responsibility of the caller. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while operating on the leaf. + /// - [`StorageError::Leaf`] if a malformed leaf is found during the query. + fn insert_value(&self, key: Word, value: Word) -> Result>; + + /// Gets the value associated with the provided `key`, or returns [`None`] if no such value + /// exists. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while operating on the leaf. + /// - [`StorageError::Leaf`] if a malformed leaf is found during the query. + fn get_value(&self, key: Word) -> Result>; + + /// Removes the key-value pair denoted by the provided `key` from the tree, returning the value + /// if it existed or [`None`] otherwise. + /// + /// If removing the entry causes the leaf to become empty, the behavior of the leaf node itself + /// (e.g. becoming sparse again or not) is implementation dependent. + /// + /// This method **does not** handle propagating the changes that result from making the removal + /// from the leaf. This is the responsibility of the caller. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while operating on the leaf. + /// - [`StorageError::Leaf`] if a malformed leaf is found during the query. + fn remove_value(&self, key: Word) -> Result>; + + /// Returns `true` if the storage has any non-sparse leaves, and `false` otherwise. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while operating on the leaf. + fn has_leaves(&self) -> Result; + + /// Retrieves a single compact SMT leaf node from its logical index, returning the leaf if it + /// exists or [`None`] otherwise. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while operating on the leaf. + /// - [`StorageError::Leaf`] if a malformed leaf is found during the query. + fn get_leaf(&self, index: u64) -> Result>; + + /// Sets the leaf at the provided logical `index` to the value of `leaf`, returning the prior + /// leaf if one was replaced or [`None`] otherwise. + /// + /// This method **does not** handle propagating the changes that result from making the leaf + /// alteration. This is the responsibility of the caller. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while operating on the leaf. + /// - [`StorageError::Leaf`] if a malformed leaf is found during the query. + fn set_leaf(&self, index: u64, leaf: SmtLeaf) -> Result>; + + /// Removes the leaf at the provided logical `index` from storage, returning the leaf if it + /// existed, or [`None`] otherwise. + /// + /// The implementation is required to replace the removed leaf with a sparse leaf entry, rather + /// than an empty but extant leaf value. + /// + /// This method **does not** handle propagating the changes that result from removing the leaf. + /// This is the responsibility of the caller. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while operating on the leaf. + /// - [`StorageError::Leaf`] if a malformed leaf is found during the query. + fn remove_leaf(&self, index: u64) -> Result>; + + /// Gets the leaves at the specified logical `indices`, returning them in the same order, or + /// returning [`None`] for any missing leaves. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while operating on the leaves. + /// - [`StorageError::Leaf`] if a malformed leaf is found during the query. + fn get_leaves(&self, indices: &[u64]) -> Result>>; + + /// Sets the leaves at the given indices to the given values, replacing any existing leaves at + /// those indices and returning the prior one, or returning [`None`] otherwise. + /// + /// This method **does not** handle propagating the changes that result from editing these + /// leaves. This is the responsibility of the caller. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while operating on the leaves. + /// - [`StorageError::Leaf`] if a malformed leaf is found during the query. + fn set_leaves(&self, leaves: Vec<(u64, SmtLeaf)>) -> Result>>; + + /// Removes the leaves at the specified logical `indices`, returning (for each index) the leaf + /// if it existed or [`None`] otherwise. + /// + /// The implementation is required to replace each removed leaf with a sparse leaf entry, rather + /// than an empty but extant leaf value. + /// + /// This method **does not** handle propagating the changes that result from removing these + /// leaves. This is the responsibility of the caller. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while operating on the leaves. + /// - [`StorageError::Leaf`] if a malformed leaf is found during the query. + fn remove_leaves(&self, indices: &[u64]) -> Result>>; + + /// Gets the subtree with the provided `index` for its root, or returns [`None`] if no such + /// subtree can be found. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while operating on the subtree. + /// - [`StorageError::Subtree`] if a malformed subtree is found during the query. + /// - [`StorageError::NotInStorage`] if the storage is queried for a subtree that is in the + /// guaranteed-to-be-in-memory portion of the tree. + fn get_subtree(&self, index: NodeIndex) -> Result>; + + /// Sets the value of the subtree with the provided `index` for its root to `subtree`, returning + /// any previous tree at that index or [`None`] if there was none. + /// + /// This method **does not** handle propagating the changes that result from editing the + /// subtree. This is the responsibility of the caller. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while operating on the subtree. + /// - [`StorageError::Subtree`] if a malformed subtree is found during the query. + /// - [`StorageError::NotInStorage`] if the storage is queried for a subtree that is in the + /// guaranteed-to-be-in-memory portion of the tree. + fn set_subtree(&self, index: NodeIndex, subtree: Subtree) -> Result>; + + /// Removes the subtree with the provided `index` for its root, returning it if it existed or + /// [`None`] otherwise. + /// + /// The implementation is required to replace each removed subtree with a sparse subtree entry, + /// rather than a dense but defaulted subtree. + /// + /// This method **does not** handle propagating the changes that result from removing the + /// subtree. This is the responsibility of the caller. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while operating on the subtree. + /// - [`StorageError::Subtree`] if a malformed subtree is found during the query. + /// - [`StorageError::NotInStorage`] if the storage is queried for a subtree that is in the + /// guaranteed-to-be-in-memory portion of the tree. + fn remove_subtree(&self, index: &NodeIndex) -> Result>; + + /// Gets the subtrees at the provided `indices` for their roots, or returns [`None`] should any + /// of those indices not contain a tree. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while operating on the subtrees. + /// - [`StorageError::Subtree`] if a malformed subtree is found during the query. + /// - [`StorageError::NotInStorage`] if the storage is queried for a subtree that is in the + /// guaranteed-to-be-in-memory portion of the tree. + fn get_subtrees(&self, indices: &[NodeIndex]) -> Result>>; + + /// Sets the value for each subtree with root at the index to the corresponding tree value, + /// returning the previous value if it existed or [`None`] if it did not. + /// + /// This method **does not** handle propagating the changes that result from editing the + /// subtree. This is the responsibility of the caller. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while operating on the subtrees. + /// - [`StorageError::Subtree`] if a malformed subtree is found during the query. + /// - [`StorageError::NotInStorage`] if the storage is queried for a subtree that is in the + /// guaranteed-to-be-in-memory portion of the tree. + fn set_subtrees(&self, subtrees: Vec<(NodeIndex, Subtree)>) -> Result>>; + + /// Removes the subtrees with the provided root `indices`, returning those that existed and + /// returning [`None`] for those that did not. + /// + /// The implementation is required to replace each removed subtree with a sparse subtree entry, + /// rather than a dense but defaulted subtree. + /// + /// This method **does not** handle propagating the changes that result from editing the + /// subtree. This is the responsibility of the caller. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while operating on the subtrees. + /// - [`StorageError::Subtree`] if a malformed subtree is found during the query. + /// - [`StorageError::NotInStorage`] if the storage is queried for a subtree that is in the + /// guaranteed-to-be-in-memory portion of the tree. + fn remove_subtrees(&self, indices: &[NodeIndex]) -> Result>>; + + /// Gets the node of the tree at the specified `index`, or returns [`None`] if the node is + /// sparse. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while querying the node. + /// - [`StorageError::NotInStorage`] if the node at the provided `index` is not in the storage + /// (e.g. due to being in the in-memory portion of the tree). + fn get_node(&self, index: NodeIndex) -> Result>; + + /// Sets the node value at the provided `index` to the `node`, returning the previous value if + /// it existed or [`None`] otherwise. + /// + /// This method **does not** handle propagating the changes that result from editing the node. + /// This is the responsibility of the caller. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while querying the node. + /// - [`StorageError::NotInStorage`] if the node at the provided `index` is not in the storage + /// (e.g. due to being in the in-memory portion of the tree). + fn set_node(&self, index: NodeIndex, node: InnerNode) -> Result>; + + /// Removes the node at the provided `index`, returning it if it existed or [`None`] if it did + /// not. + /// + /// The implementation is required to make the storage for the removed node _sparse_, rather + /// than replace it with a defaulted but present entry. + /// + /// This method **does not** handle propagating the changes that result from removing the node. + /// This is the responsibility of the caller. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while querying the node. + /// - [`StorageError::NotInStorage`] if the node at the provided `index` is not in the storage + /// (e.g. due to being in the in-memory portion of the tree). + fn remove_node(&self, index: NodeIndex) -> Result>; + + /// Returns an iterator over all pairs of `(logical_index, leaf_value)` currently in the + /// storage. + /// + /// The order of iteration is not guaranteed unless specified by the implementation. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while querying. + fn iter_leaves(&self) -> Result + '_>>; + + /// Returns an iterator over all _populated_ pairs of `(node_index, node_value)` for non-leaf + /// nodes currently in the storage. + /// + /// The order of iteration is not guaranteed unless specified by the implementation. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while querying. + fn iter_nodes(&self) -> Result + '_>>; + + /// Returns an iterator over all (semi)-_populated_ pairs of `(tree_root_index, subtree_val)` + /// currently in the storage. + /// + /// The order of iteration is not guaranteed unless specified by the implementation. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while querying. + fn iter_subtrees(&self) -> Result + '_>>; + + /// Returns the restoration data for the guaranteed-in-memory portion of the tree, consisting of + /// a pair of `(num_levels, node_values)`. + /// + /// Each value in `node_values` is the cached root of a tree at the first level not guaranteed + /// to be in memory, and will be in the order of those roots by index. If level `l` is the first + /// level not guaranteed to be in memory, the result vector will contain `2^l` entries. These + /// entries are **never sparse**. + /// + /// # Errors + /// + /// - [`StorageError::Backend`] if a backend error occurs while querying. + fn restoration_data(&self) -> Result>; +} diff --git a/miden-crypto/src/merkle/smt/large_forest/utils.rs b/miden-crypto/src/merkle/smt/large_forest/utils.rs new file mode 100644 index 000000000..3e51fc652 --- /dev/null +++ b/miden-crypto/src/merkle/smt/large_forest/utils.rs @@ -0,0 +1,84 @@ +//! This module contains a variety of useful functions and type aliases that do not really have any +//! other place to live. + +// CONSTANTS +// ================================================================================================ + +use core::fmt::{Display, Formatter}; + +/// The maximum number of levels that can be stored in a given subtree for a tree with depth 64. +/// +/// Simply put, it must never include the leaves level, and hence is one less than that depth. +pub const MAX_NUM_SUBTREE_LEVELS: u8 = 63; + +// SUBTREE LEVELS +// ================================================================================================ + +/// The number of levels in a tree. +/// +/// # Invariants +/// +/// Any instance of this type should see that the following properties hold: +/// +/// - The root is a level of its own. This is level 0, to follow the convention used by +/// [`crate::merkle::smt::NodeIndex`]. This means that if the level count begins at the top of the +/// tree, it should include the root level. By way of example, a tree with 8 leaves has _4_ levels +/// in this counting. +/// - You cannot have a zero number of levels, which is enforced by construction. +/// - The number of levels cannot exceed [`MAX_NUM_SUBTREE_LEVELS`] +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct SubtreeLevels { + value: u8, +} +impl SubtreeLevels { + /// Constructs a new number of levels, or returns [`None`] if the type invariants are violated. + pub fn new(value: u8) -> Option { + if value == 0 || value > MAX_NUM_SUBTREE_LEVELS { + return None; + } + Some(SubtreeLevels { value }) + } + + /// Constructs a number of levels, trusting that the provided `value` maintains the invariants + /// of the type. + /// + /// Using this type if the invariants have been violated may result in undefined computational + /// behaviour, up to and including crashes at runtime and data corruption. To that end, in debug + /// builds it will panic if the invariants are violated. + pub fn new_unchecked(value: u8) -> SubtreeLevels { + assert!( + value <= MAX_NUM_SUBTREE_LEVELS && value > 0, + "{value} is not a valid number of subtree levels" + ); + SubtreeLevels { value } + } + + /// Gets the number of non-root levels in the subtree. + pub fn non_root_levels(&self) -> u8 { + self.value - 1 + } +} + +impl From for u8 { + fn from(value: SubtreeLevels) -> Self { + value.value + } +} + +impl From for u32 { + fn from(value: SubtreeLevels) -> Self { + value.value as u32 + } +} + +impl From for u64 { + fn from(value: SubtreeLevels) -> Self { + value.value as u64 + } +} + +impl Display for SubtreeLevels { + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { + write!(f, "{}", self.value) + } +} diff --git a/miden-crypto/src/merkle/smt/mod.rs b/miden-crypto/src/merkle/smt/mod.rs index 6c57c95cb..ffbe7efbf 100644 --- a/miden-crypto/src/merkle/smt/mod.rs +++ b/miden-crypto/src/merkle/smt/mod.rs @@ -28,8 +28,10 @@ pub use large::{ #[cfg(feature = "rocksdb")] pub use large::{RocksDbConfig, RocksDbStorage}; -mod large_forest; -pub use large_forest::{History, HistoryError, HistoryView, LargeSmtForestError}; +pub mod large_forest; +pub use large_forest::{ + LargeSmtForestError, Storage as ForestStorage, StorageError, StoredTreeHandle, SubtreeLevels, +}; mod simple; pub use simple::{SimpleSmt, SimpleSmtProof};