Create the storage abstraction for the new forest

miden-crypto/src/merkle/smt/large_forest/error/mod.rs

@@ -0,0 +1,28 @@
+//! This module contains the error types and helpers for working with errors from the large SMT
+//! forest.
+
+pub mod subtree;
+
+use thiserror::Error;
+
+use crate::merkle::{
+    MerkleError,
+    smt::large_forest::{history::error::HistoryError, storage},
+};
+
+/// The type of errors returned by operations on the large SMT forest.
+#[derive(Debug, Error)]
+pub enum LargeSmtForestError {
+    #[error(transparent)]
+    HistoryError(#[from] HistoryError),
+
+    #[error(transparent)]
+    MerkleError(#[from] MerkleError),
+
+    #[error(transparent)]
+    StorageError(#[from] storage::StorageError),
+}
+
+/// The result type for use within the large SMT forest portion of the library.
+#[allow(dead_code)] // Temporary
+pub type Result<T> = core::result::Result<T, LargeSmtForestError>;

miden-crypto/src/merkle/smt/large_forest/error/subtree.rs

@@ -0,0 +1,35 @@
+//! Errors in working with subtrees.
+
+use thiserror::Error;
+
+/// Errors raised when encountering an issue when working with subtrees.
+#[derive(Debug, Error)]
+pub enum SubtreeError {
+    /// Raised when decoding the subtree from bytes and encountering insufficient node data.
+    #[error("Expected {expected} bytes of node data, found {found} bytes")]
+    BadHashLen { expected: usize, found: usize },
+
+    /// Raised when the left index has an invalid hash.
+    #[error("Invalid left hash format at local index {index}")]
+    BadLeft { index: u64 },
+
+    /// Raised when the left index has an invalid hash.
+    #[error("Invalid right hash format at local index {index}")]
+    BadRight { index: u64 },
+
+    /// When extra node data exists in the bytestream after the expected data.
+    #[error("Found extra node data after bitmask-indicated entries")]
+    ExtraData,
+
+    /// When the node data for the left index of a node.
+    #[error("Missing left node data at local index {index}")]
+    MissingLeft { index: u64 },
+
+    /// When the node data for the right index of a node.
+    #[error("Missing right node data at local index {index}")]
+    MissingRight { index: u64 },
+
+    /// Raised when there is insufficient data when decoding the subtree.
+    #[error("Found {found} bytes when decoding the subtree, but need at least {min} bytes")]
+    TooShort { found: usize, min: usize },
+}

miden-crypto/src/merkle/smt/large_forest/mod.rs

@@ -1,7 +1,45 @@
 //! A high-performance sparse merkle tree forest backed by pluggable storage.
+//!
+//! # Semantic Layout
+//!
+//! Much like `SparseMerkleTree`, the forest stores trees of depth 64 that use the compact leaf
+//! optimization to uniquely store 256-bit elements. This reduces both the size of a merkle path,
+//! and the computational work necessary to perform queries into the trees.
+//!
+//! # Storing Trees and Versions
+//!
+//! The usage of an SMT forest is conceptually split into two parts: a collection that is able to
+//! store **multiple, unrelated trees**, and a container for **multiple versions of those trees**.
+//! Both of these use-cases are supported by the forest, but have an explicit delineation between
+//! them in both the API and the implementation. This has two impacts that a client of the forest
+//! must understand.
+//!
+//! - While, when using a [`Storage`] that can persist data, **only the current full tree state is
+//!   persisted**, while **the historical data will not be**. This is designed into the structure of
+//!   the forest, and does not depend on the choice of storage backend.
+//! - It is more expensive to query a given tree at an older point in its history than it is to
+//!   query it at a newer point, and querying at the current tree will always take the least time.
+//!
+//! # Data Storage
+//!
+//! In order to help with the query performance for the more latency-prone kinds of [`Storage`]
+//! implementation, the forest splits the data into two portions:
+//!
+//! 1. The **top of each tree** is explicitly **stored in memory**, regardless of the [`Storage`]
+//!    backend. This makes the common tree prefix much more performant to query, and relies on the
+//!    backend to store sufficient data to _reconstruct_ that prefix at forest rebuild.
+//! 2. The **rest of each tree** is managed by the [`Storage`] itself, and makes no guarantees as to
+//!    where that data is stored. Queries into this portion have performance characteristics
+//!    dictated by the choice of storage backend.
+//!
+//! The split between these numbers of levels is configured when initially constructing the forest,
+//! and will be verified at runtime for forests that are instead reloaded from persistent state.
 
 mod error;
-mod history;
+pub mod history;
+pub mod storage;
+pub mod utils;
 
 pub use error::LargeSmtForestError;
-pub use history::{History, HistoryView, error::HistoryError};
+pub use storage::{Storage, StorageError, StoredTreeHandle};
+pub use utils::SubtreeLevels;

miden-crypto/src/merkle/smt/large_forest/storage/error.rs

@@ -0,0 +1,59 @@
+//! This module contains the error types and helpers for working with errors from the large SMT
+//! forest.
+
+use alloc::{boxed::Box, string::String};
+
+use thiserror::Error;
+
+use crate::{
+    Word,
+    merkle::smt::{SmtLeafError, large_forest::error::subtree::SubtreeError},
+};
+
+/// The type of errors returned by operations on the large SMT forest.
+#[derive(Debug, Error)]
+pub enum StorageError {
+    /// An error coming directly from a malfunction in the storage backend.
+    #[error(transparent)]
+    Backend(#[from] Box<dyn core::error::Error + Send + 'static>),
+
+    /// An error with the contents of a compact tree leaf.
+    #[error(transparent)]
+    Leaf(#[from] SmtLeafError),
+
+    /// Raised when a storage backend does not support multiple concurrent transactions.
+    #[error("This backend does not support multiple concurrent transactions")]
+    MultipleTransactionsUnsupported,
+
+    /// Raised when an entity is requested from the storage but is not managed by this storage due
+    /// to being part of the guaranteed-to-be-in-memory storage.
+    #[error("The entity {0} is not part of this storage")]
+    NotInStorage(String),
+
+    /// Issued if an operation that can only be performed with an active transaction is performed
+    /// outside a transaction.
+    #[error("An operation was issued that requires a transaction to have been started.")]
+    NotInTransaction,
+
+    /// An error when reading from or writing to a subtree in storage.
+    #[error(transparent)]
+    Subtree(#[from] SubtreeError),
+
+    /// Raised when the storage does not store a tree with the provided root.
+    #[error("No tree with root {0} exists in this storage")]
+    UnknownRoot(Word),
+
+    /// Raised when a storage implementation is given a transaction handle that it did not allocate.
+    #[error("A transaction handle was provided that was not issued by this storage")]
+    UnknownTransaction,
+
+    /// The requested operation is not supported by this backend.
+    ///
+    /// In some cases it may be possible to fall back to a more complex slow path when this error is
+    /// received.
+    #[error("The operation {0} is not supported")]
+    UnsupportedOperation(String),
+}
+
+/// The result type for use within the large SMT forest portion of the library.
+pub type Result<T> = core::result::Result<T, StorageError>;