diff --git a/miden-crypto/src/lib.rs b/miden-crypto/src/lib.rs index 8e820de0a..80339721e 100644 --- a/miden-crypto/src/lib.rs +++ b/miden-crypto/src/lib.rs @@ -2,7 +2,6 @@ #[macro_use] extern crate alloc; - #[cfg(feature = "std")] extern crate std; @@ -49,6 +48,20 @@ pub type Map = alloc::collections::BTreeMap; #[cfg(not(feature = "hashmaps"))] pub use alloc::collections::btree_map::Entry as MapEntry; +/// An alias for a simple set. +/// +/// By default, this is an alias for the [`alloc::collections::BTreeSet`]. However, when the +/// `hashmaps` feature is enabled, this becomes an alias for hashbrown's HashSet. +#[cfg(feature = "hashmaps")] +pub type Set = hashbrown::HashSet; + +/// An alias for a simple set. +/// +/// By default, this is an alias for the [`alloc::collections::BTreeSet`]. However, when the +/// `hashmaps` feature is enabled, this becomes an alias for hashbrown's HashSet. +#[cfg(not(feature = "hashmaps"))] +pub type Set = alloc::collections::BTreeSet; + // CONSTANTS // ================================================================================================ diff --git a/miden-crypto/src/merkle/smt/large_forest/error.rs b/miden-crypto/src/merkle/smt/large_forest/error.rs new file mode 100644 index 000000000..14ca4cf3e --- /dev/null +++ b/miden-crypto/src/merkle/smt/large_forest/error.rs @@ -0,0 +1,21 @@ +//! This module contains the error types and helpers for working with errors from the large SMT +//! forest. + +use thiserror::Error; + +use crate::merkle::{MerkleError, smt::large_forest::history::error::HistoryError}; + +/// The errors returned by operations on the large SMT forest. +/// +/// This type primarily serves to wrap more specific error types from various subsystems into a +/// generic interface type. +#[derive(Debug, Error)] +pub enum LargeSmtForestError { + #[error(transparent)] + HistoryError(#[from] HistoryError), + + #[error(transparent)] + MerkleError(#[from] MerkleError), +} + +pub mod history {} diff --git a/miden-crypto/src/merkle/smt/large_forest/history/error.rs b/miden-crypto/src/merkle/smt/large_forest/history/error.rs new file mode 100644 index 000000000..6f6c0e2af --- /dev/null +++ b/miden-crypto/src/merkle/smt/large_forest/history/error.rs @@ -0,0 +1,23 @@ +//! The error type and utility types for working with errors from the SMT history construct. +use thiserror::Error; + +use crate::merkle::smt::large_forest::history::VersionId; + +/// The type of errors returned by the history container. +#[derive(Debug, Error, PartialEq)] +pub enum HistoryError { + /// Raised when a query expects the history to contain at least one entry, but it is empty. + #[error("The history was empty")] + HistoryEmpty, + + /// Raised when a version is added to the history and is not newer than the previous. + #[error("Version {0} is not monotonic with respect to {1}")] + NonMonotonicVersions(VersionId, VersionId), + + /// Raised when no version exists in the history for an arbitrary query. + #[error("The specified version is too old to be served by the history")] + VersionTooOld, +} + +/// The result type for use within the history container. +pub type Result = core::result::Result; diff --git a/miden-crypto/src/merkle/smt/large_forest/history/mod.rs b/miden-crypto/src/merkle/smt/large_forest/history/mod.rs new file mode 100644 index 000000000..37ce4daf0 --- /dev/null +++ b/miden-crypto/src/merkle/smt/large_forest/history/mod.rs @@ -0,0 +1,423 @@ +//! This module contains the definition of [`History`], a simple container for some number of +//! historical versions of a given merkle tree. +//! +//! This history consists of a series of _deltas_ from the current state of the tree, moving +//! backward in history away from that current state. These deltas are then used to form a "merged +//! overlay" that represents the changes to be made on top of the current tree to put it _back_ in +//! that historical state. +//! +//! It provides functionality for adding new states to the history, as well as for querying the +//! history at a given point in time. +//! +//! # Complexity +//! +//! Versions in this structure are _cumulative_. To get the entire picture of an arbitrary node or +//! leaf at version `v` it may be necessary to check for changes in all versions between `v` and the +//! current tree state. This gives worst-case complexity `O(v)` when querying a node or leaf for the +//! version `v`. +//! +//! This is acceptable overhead as we assert that newer versions are far more likely to be queried +//! than older versions. Nevertheless, it may be improved in future using a sharing approach, but +//! that potential improvement is being ignored for now for the sake of simplicity. +//! +//! # Performance +//! +//! This structure operates entirely in memory, and is hence reasonably quick to query. As of the +//! current time, no detailed benchmarking has taken place for the history, but based on some basic +//! profiling the major time taken is in chasing pointers throughout memory due to the use of +//! [`Map`]s, but this is unavoidable in the current structure and may need to be revisited in +//! the future. + +pub mod error; + +#[cfg(test)] +mod tests; + +use alloc::collections::VecDeque; +use core::fmt::Debug; + +use error::{HistoryError, Result}; + +use crate::{ + Map, Set, Word, + merkle::{ + NodeIndex, + smt::{LeafIndex, SMT_DEPTH}, + }, +}; + +// UTILITY TYPE ALIASES +// ================================================================================================ + +/// A compact leaf is a mapping from full word-length keys to word-length values, intended to be +/// stored in the leaves of an otherwise shallower merkle tree. +pub type CompactLeaf = Map; + +/// A collection of changes to arbitrary non-leaf nodes in a merkle tree. +/// +/// All changes to nodes between versions `v` and `v + 1` must be explicitly "undone" in the +/// `NodeChanges` representing version `v`. This includes nodes that were defaulted in version `v` +/// that were given an explicit value in version `v + 1`, where the `NodeChanges` must explicitly +/// set those nodes back to the default. +/// +/// Failure to do so will result in incorrect values when those nodes are queried at a point in the +/// history corresponding to version `v`. +pub type NodeChanges = Map; + +/// A collection of changes to arbitrary leaf nodes in a merkle tree. +/// +/// This represents the state of the leaf wholesale, rather than as a delta from the newer version. +/// This massively simplifies querying leaves in the history. +/// +/// Note that if in the version of the tree represented by these `LeafChanges` had the default value +/// at the leaf, this default value must be made concrete in the map. Failure to do so will retain a +/// newer, non-default value for that leaf, and thus result in incorrect query results at this point +/// in the history. +pub type LeafChanges = Map, CompactLeaf>; + +/// An identifier for a historical tree version overlay, which must be monotonic as new versions are +/// added. +pub type VersionId = u64; + +// HISTORY +// ================================================================================================ + +/// A History contains a sequence of versions atop a given tree. +/// +/// The versions are _cumulative_, meaning that querying the history must account for changes from +/// the current tree that take place in versions that are not the queried version or the current +/// tree. +#[derive(Clone, Debug)] +pub struct History { + /// The maximum number of historical versions to be stored. + max_count: usize, + + /// The deltas that make up the history for this tree. + /// + /// It will never contain more than `max_count` deltas, and is ordered with the oldest data at + /// the lowest index. + /// + /// # Implementation Note + /// + /// As we are targeting small numbers of history items (e.g. 30), having a sequence with an + /// allocated capacity equal to the small maximum number of items is perfectly sane. This will + /// avoid costly reallocations in the fast path. + /// + /// We use a [`VecDeque`] instead of a [`Vec`] or [`alloc::collections::LinkedList`] as we + /// estimate that the vast majority of removals will be the oldest entries as new ones are + /// pushed. This means that we can optimize for those removals along with indexing performance, + /// rather than optimizing for more rare removals from the middle of the sequence. + deltas: VecDeque, +} + +impl History { + /// Constructs a new history container, containing at most `max_count` historical versions for + /// a tree. + #[must_use] + pub fn empty(max_count: usize) -> Self { + // We allocate one more than we actually need to store to allow us to insert and THEN + // remove, rather than the other way around. This leads to negligible increases in memory + // usage while allowing for cleaner code. + let deltas = VecDeque::with_capacity(max_count + 1); + Self { max_count, deltas } + } + + /// Gets the maximum number of versions that this history can store. + #[must_use] + pub fn max_versions(&self) -> usize { + self.max_count + } + + /// Gets the current number of versions in the history. + #[must_use] + pub fn num_versions(&self) -> usize { + self.deltas.len() + } + + /// Returns all the roots that the history knows about. + /// + /// # Complexity + /// + /// Calling this method requires a traversal of all the versions and is hence linear in the + /// number of history versions. + #[must_use] + pub fn roots(&self) -> Set { + self.deltas.iter().map(|d| d.root).collect() + } + + /// Returns `true` if `root` is in the history and `false` otherwise. + /// + /// # Complexity + /// + /// Calling this method requires a traversal of all the versions and is hence linear in the + /// number of history versions. + #[must_use] + pub fn is_known_root(&self, root: Word) -> bool { + self.deltas.iter().any(|r| r.root == root) + } + + /// Adds a version to the history with the provided `root` and represented by the changes from + /// the current tree given in `nodes` and `leaves`. + /// + /// If adding this version would result in exceeding `self.max_count` historical versions, then + /// the oldest of the versions is automatically removed. + /// + /// # Gotchas + /// + /// When constructing the `nodes` and `leaves`, keep in mind that those collections must contain + /// entries for the **default value of a node or leaf** at any position where the tree was + /// sparse in the state represented by `root`. If this is not done, incorrect values may be + /// returned. + /// + /// This is necessary because the changes are the _reverse_ from what one might expect. Namely, + /// the changes in a given version `v` must "_revert_" the changes made in the transition from + /// version `v` to version `v + 1`. + /// + /// # Errors + /// + /// - [`HistoryError::NonMonotonicVersions`] if the provided version is not greater than the + /// previously added version. + pub fn add_version( + &mut self, + root: Word, + version_id: VersionId, + nodes: NodeChanges, + leaves: LeafChanges, + ) -> Result<()> { + if let Some(v) = self.deltas.iter().last() { + if v.version_id < version_id { + self.deltas.push_back(Delta::new(root, version_id, nodes, leaves)); + if self.num_versions() > self.max_versions() { + self.deltas.pop_front(); + } + + Ok(()) + } else { + Err(HistoryError::NonMonotonicVersions(version_id, v.version_id)) + } + } else { + self.deltas.push_back(Delta::new(root, version_id, nodes, leaves)); + + Ok(()) + } + } + + /// Returns the index in the sequence of deltas of the version that corresponds to the provided + /// `version_id`. + /// + /// To "correspond" means that it either has the provided `version_id`, or is the newest version + /// with a `version_id` less than the provided id. In either case, it is the correct version to + /// be used to query the tree state in the provided `version_id`. + /// + /// # Complexity + /// + /// Finding the latest corresponding version in the history requires a linear traversal of the + /// history entries, and hence has complexity `O(n)` in the number of versions. + /// + /// # Errors + /// + /// - [`HistoryError::HistoryEmpty`] if the history is empty and hence there is no version to + /// find. + /// - [`HistoryError::VersionTooOld`] if the history does not contain the data to provide a + /// coherent overlay for the provided `version_id` due to `version_id` being older than the + /// oldest version stored. + fn find_latest_corresponding_version(&self, version_id: VersionId) -> Result { + // If the version is older than the oldest, we error. + if let Some(oldest_version) = self.deltas.front() { + if oldest_version.version_id > version_id { + return Err(HistoryError::VersionTooOld); + } + } else { + return Err(HistoryError::VersionTooOld); + } + + let ix = self + .deltas + .iter() + .position(|d| d.version_id > version_id) + .unwrap_or_else(|| self.num_versions()) + .checked_sub(1) + .expect( + "Subtraction should not overflow as we have ruled out the no-version \ + case, and in the other cases the left operand will be >= 1", + ); + + Ok(ix) + } + + /// Returns a view of the history that allows querying as a single unified overlay on the + /// current state of the merkle tree as if the overlay was reverting the tree to the state + /// corresponding to the specified `version_id`. + /// + /// Note that the history may not contain a version that directly corresponds to `version_id`. + /// In such a case, the view will instead use the newest version coherent with the provided + /// `version_id`, as this is the correct version for the provided id. Note that this will be + /// incorrect if the versions stored in the history do not represent contiguous changes from the + /// current tree. + /// + /// # Complexity + /// + /// The computational complexity of this method is linear in the number of versions stored in + /// the history. + /// + /// # Errors + /// + /// - [`HistoryError::VersionTooOld`] if the history does not contain the data to provide a + /// coherent overlay for the provided `version_id` due to `version_id` being older than the + /// oldest version stored. + pub fn get_view_at(&self, version_id: VersionId) -> Result> { + let version_index = self.find_latest_corresponding_version(version_id)?; + Ok(HistoryView::new_of(version_index, self)) + } + + /// Removes all versions in the history that are older than the version denoted by the provided + /// `version_id`. + /// + /// If `version_id` is not a version known by the history, it will keep the newest version that + /// is capable of serving as that version in queries. + /// + /// # Complexity + /// + /// The computational complexity of this method is linear in the number of versions stored in + /// the history prior to any removals. + pub fn truncate(&mut self, version_id: VersionId) -> usize { + // We start by getting the index to truncate to, though it is not an error to remove + // something too old. + let truncate_ix = self.find_latest_corresponding_version(version_id).unwrap_or(0); + + for _ in 0..truncate_ix { + self.deltas.pop_front(); + } + + truncate_ix + } + + /// Removes all versions from the history. + pub fn clear(&mut self) { + self.deltas.clear(); + } +} + +// HISTORY VIEW +// ================================================================================================ + +/// A read-only view of the history overlay on the tree at a specified place in the history. +#[derive(Debug)] +pub struct HistoryView<'history> { + /// The index of the target version in the history. + version_ix: usize, + + /// The history that actually stores the data that will be queried. + history: &'history History, +} + +impl<'history> HistoryView<'history> { + /// Constructs a new history view that acts as a single overlay of the state represented by the + /// oldest delta for which `f` returns true. + /// + /// # Complexity + /// + /// The computational complexity of this method is linear in the number of versions stored in + /// the history. + fn new_of(version_ix: usize, history: &'history History) -> Self { + Self { version_ix, history } + } + + /// Gets the value of the node in the history at the provided `index`, or returns `None` if the + /// version does not overlay the current tree at that node. + /// + /// # Complexity + /// + /// The computational complexity of this method is linear in the number of versions due to the + /// need to traverse to find the correct overlay value. + #[must_use] + pub fn node_value(&self, index: &NodeIndex) -> Option<&Word> { + self.history + .deltas + .iter() + .skip(self.version_ix) + .find_map(|v| v.nodes.get(index)) + } + + /// Gets the value of the entire leaf in the history at the specified `index`, or returns `None` + /// if the version does not overlay the current tree at that leaf. + /// + /// # Complexity + /// + /// The computational complexity of this method is linear in the number of versions due to the + /// need to traverse to find the correct overlay value. + #[must_use] + pub fn leaf_value(&self, index: &LeafIndex) -> Option<&CompactLeaf> { + self.history + .deltas + .iter() + .skip(self.version_ix) + .find_map(|v| v.leaves.get(index)) + } + + /// Queries the value of a specific key in a leaf in the overlay, returning: + /// + /// - `None` if the version does not overlay that leaf in the current tree, + /// - `Some(None)` if the version does overlay that leaf but the compact leaf does not contain + /// that value, + /// - and `Some(Some(v))` if the version does overlay the leaf and the key exists in that leaf. + /// + /// # Complexity + /// + /// The computational complexity of this method is linear in the number of versions due to the + /// need to traverse to find the correct overlay value. + #[must_use] + pub fn value(&self, key: &Word) -> Option> { + self.leaf_value(&LeafIndex::from(*key)).map(|leaf| leaf.get(key)) + } +} + +// DELTA +// ================================================================================================ + +/// A delta for a state `n` represents the changes (to both nodes and leaves) that need to be +/// applied on top of the state `n + 1` to yield the correct tree for state `n`. +/// +/// # Cumulative Deltas and Temporal Ordering +/// +/// In order to best represent the history of a merkle tree, these deltas are constructed to take +/// advantage of two main properties: +/// +/// - They are _cumulative_, which reduces their practical memory usage. This does, however, mean +/// that querying the state of older blocks is more expensive than querying newer ones. +/// - Deltas are applied in **temporally reversed order** from what one might expect. Most +/// conventional applications of deltas bring something from the past into the future through +/// application. In our case, the application of one or more deltas moves the tree into a **past +/// state**. +/// +/// # Construction +/// +/// While the [`Delta`] type is visible in the interface of the history, it is only intended to be +/// constructed by the history. Users should not be allowed to construct it directly. +#[derive(Clone, Debug, PartialEq)] +struct Delta { + /// The root of the tree in the `version` corresponding to the application of the reversions in + /// this delta to the previous tree state. + pub root: Word, + + /// The version of the tree represented by the delta. + pub version_id: VersionId, + + /// Any changes to the non-leaf nodes in the tree for this delta. + pub nodes: NodeChanges, + + /// Any changes to the leaf nodes in the tree for this delta. + /// + /// Note that the leaf state is **not represented compactly**, and describes the entire state + /// of the leaf in the corresponding version. + pub leaves: LeafChanges, +} + +impl Delta { + /// Creates a new delta with the provided `root`, and representing the provided + /// changes to `nodes` and `leaves` in the merkle tree. + #[must_use] + fn new(root: Word, version_id: VersionId, nodes: NodeChanges, leaves: LeafChanges) -> Self { + Self { root, version_id, nodes, leaves } + } +} diff --git a/miden-crypto/src/merkle/smt/large_forest/history/tests.rs b/miden-crypto/src/merkle/smt/large_forest/history/tests.rs new file mode 100644 index 000000000..b811e9ac7 --- /dev/null +++ b/miden-crypto/src/merkle/smt/large_forest/history/tests.rs @@ -0,0 +1,361 @@ +//! The functional tests for the history component. + +use rand_utils::rand_value; + +use super::{CompactLeaf, History, LeafChanges, NodeChanges, error::Result}; +use crate::{ + Word, + merkle::{NodeIndex, smt::LeafIndex}, +}; + +// TESTS +// ================================================================================================ + +#[test] +fn empty() { + let history = History::empty(5); + assert_eq!(history.num_versions(), 0); + assert_eq!(history.max_versions(), 5); +} + +#[test] +fn roots() -> Result<()> { + // Set up our test state + let nodes = NodeChanges::default(); + let leaves = LeafChanges::default(); + let mut history = History::empty(2); + let root_1: Word = rand_value(); + let root_2: Word = rand_value(); + history.add_version(root_1, 0, nodes.clone(), leaves.clone())?; + history.add_version(root_2, 1, nodes.clone(), leaves.clone())?; + + // We should be able to get all the roots. + let roots = history.roots(); + assert_eq!(roots.len(), 2); + assert!(roots.contains(&root_1)); + assert!(roots.contains(&root_2)); + + Ok(()) +} + +#[test] +fn is_known_root() -> Result<()> { + // Set up our test state + let nodes = NodeChanges::default(); + let leaves = LeafChanges::default(); + let mut history = History::empty(2); + let root_1: Word = rand_value(); + let root_2: Word = rand_value(); + history.add_version(root_1, 0, nodes.clone(), leaves.clone())?; + history.add_version(root_2, 1, nodes.clone(), leaves.clone())?; + + // We should be able to query for existing roots. + assert!(history.is_known_root(root_1)); + assert!(history.is_known_root(root_2)); + + // But not for nonexistent ones. + assert!(!history.is_known_root(rand_value())); + + Ok(()) +} + +#[test] +fn find_latest_corresponding_version() -> Result<()> { + // Start by setting up our test data. + let nodes = NodeChanges::default(); + let leaves = LeafChanges::default(); + let mut history = History::empty(5); + + let v1 = 10; + let v2 = 20; + let v3 = 30; + let v4 = 31; + let v5 = 45; + + history.add_version(rand_value(), v1, nodes.clone(), leaves.clone())?; + history.add_version(rand_value(), v2, nodes.clone(), leaves.clone())?; + history.add_version(rand_value(), v3, nodes.clone(), leaves.clone())?; + history.add_version(rand_value(), v4, nodes.clone(), leaves.clone())?; + history.add_version(rand_value(), v5, nodes.clone(), leaves.clone())?; + + // When we query for a version that is older than the oldest in the history we should get an + // error. + assert!(history.find_latest_corresponding_version(0).is_err()); + assert!(history.find_latest_corresponding_version(9).is_err()); + + // When we query for the oldest version we should get its index. + assert_eq!(history.find_latest_corresponding_version(v1), Ok(0)); + + // And that goes for any other known version + assert_eq!(history.find_latest_corresponding_version(v2), Ok(1)); + assert_eq!(history.find_latest_corresponding_version(v3), Ok(2)); + assert_eq!(history.find_latest_corresponding_version(v4), Ok(3)); + assert_eq!(history.find_latest_corresponding_version(v5), Ok(4)); + + // But we can also query for versions in between. + assert_eq!(history.find_latest_corresponding_version(11), Ok(0)); + assert_eq!(history.find_latest_corresponding_version(19), Ok(0)); + assert_eq!(history.find_latest_corresponding_version(21), Ok(1)); + assert_eq!(history.find_latest_corresponding_version(29), Ok(1)); + assert_eq!(history.find_latest_corresponding_version(32), Ok(3)); + assert_eq!(history.find_latest_corresponding_version(44), Ok(3)); + assert_eq!(history.find_latest_corresponding_version(46), Ok(4)); + + Ok(()) +} + +#[test] +fn add_version() -> Result<()> { + let nodes = NodeChanges::default(); + let leaves = LeafChanges::default(); + + // We start with an empty state, and we should be able to add deltas up until the limit we + // set. + let mut history = History::empty(2); + assert_eq!(history.num_versions(), 0); + assert_eq!(history.max_versions(), 2); + + let root_1: Word = rand_value(); + let id_1 = 0; + history.add_version(root_1, id_1, nodes.clone(), leaves.clone())?; + assert_eq!(history.num_versions(), 1); + + let root_2: Word = rand_value(); + let id_2 = 1; + history.add_version(root_2, id_2, nodes.clone(), leaves.clone())?; + assert_eq!(history.num_versions(), 2); + + // At this point, adding any version should remove the oldest. + let root_3: Word = rand_value(); + let id_3 = 2; + history.add_version(root_3, id_3, nodes.clone(), leaves.clone())?; + assert_eq!(history.num_versions(), 2); + + // If we then query for that first version it won't be there anymore, but the other two + // should. + assert!(history.get_view_at(id_1).is_err()); + assert!(history.get_view_at(id_2).is_ok()); + assert!(history.get_view_at(id_3).is_ok()); + + // If we try and add a version with a non-monotonic version number, we should see an error. + assert!(history.add_version(root_3, id_1, nodes, leaves).is_err()); + + Ok(()) +} + +#[test] +fn truncate() -> Result<()> { + // Start by setting up the test data + let mut history = History::empty(4); + + let nodes = NodeChanges::default(); + let leaves = LeafChanges::default(); + + let root_1: Word = rand_value(); + let id_1 = 5; + history.add_version(root_1, id_1, nodes.clone(), leaves.clone())?; + + let root_2: Word = rand_value(); + let id_2 = 10; + history.add_version(root_2, id_2, nodes.clone(), leaves.clone())?; + + let root_3: Word = rand_value(); + let id_3 = 15; + history.add_version(root_3, id_3, nodes.clone(), leaves.clone())?; + + let root_4: Word = rand_value(); + let id_4 = 20; + history.add_version(root_4, id_4, nodes.clone(), leaves.clone())?; + + assert_eq!(history.num_versions(), 4); + + // If we truncate to the oldest version or before, nothing should be removed. + assert_eq!(history.truncate(0), 0); + assert_eq!(history.num_versions(), 4); + assert_eq!(history.truncate(4), 0); + assert_eq!(history.num_versions(), 4); + assert_eq!(history.truncate(id_1), 0); + assert_eq!(history.num_versions(), 4); + + // If we truncate to a specific known version, it should remove all previous versions. + assert_eq!(history.truncate(id_2), 1); + assert_eq!(history.num_versions(), 3); + + // If we truncate to a version that is not known, the newest relevant version should be + // retained. + assert_eq!(history.truncate(16), 1); + assert_eq!(history.num_versions(), 2); + + // If we truncate to a version beyond the newest known, only that should be retained. + assert_eq!(history.truncate(25), 1); + assert_eq!(history.num_versions(), 1); + + Ok(()) +} + +#[test] +fn clear() -> Result<()> { + // Start by setting up the test data + let mut history = History::empty(4); + + let nodes = NodeChanges::default(); + let leaves = LeafChanges::default(); + + let root_1: Word = rand_value(); + let id_1 = 0; + history.add_version(root_1, id_1, nodes.clone(), leaves.clone())?; + + let root_2: Word = rand_value(); + let id_2 = 1; + history.add_version(root_2, id_2, nodes.clone(), leaves.clone())?; + + assert_eq!(history.num_versions(), 2); + + // We can clear the history entirely in one go. + history.clear(); + assert_eq!(history.num_versions(), 0); + + Ok(()) +} + +#[test] +fn view_at() -> Result<()> { + // Starting in an empty state we should be able to add deltas up until the limit we set. + let mut history = History::empty(3); + assert_eq!(history.num_versions(), 0); + assert_eq!(history.max_versions(), 3); + + // We can add an initial version with some changes in both nodes and leaves. + let root_1 = rand_value::(); + let id_1 = 3; + let mut nodes_1 = NodeChanges::default(); + let n1_value: Word = rand_value(); + let n2_value: Word = rand_value(); + nodes_1.insert(NodeIndex::new(2, 1).unwrap(), n1_value); + nodes_1.insert(NodeIndex::new(8, 128).unwrap(), n2_value); + + let mut leaf_1 = CompactLeaf::new(); + let l1_e1_key: Word = rand_value(); + let l1_e1_value: Word = rand_value(); + let leaf_1_ix = LeafIndex::from(l1_e1_key); + leaf_1.insert(l1_e1_key, l1_e1_value); + + let mut leaf_2 = CompactLeaf::new(); + let l2_e1_key: Word = rand_value(); + let l2_e1_value: Word = rand_value(); + let leaf_2_ix = LeafIndex::from(l2_e1_key); + let mut l2_e2_key: Word = rand_value(); + l2_e2_key[3] = leaf_2_ix.value().try_into().unwrap(); + let l2_e2_value: Word = rand_value(); + leaf_2.insert(l2_e1_key, l2_e1_value); + leaf_2.insert(l2_e2_key, l2_e2_value); + + let mut leaves_1 = LeafChanges::default(); + leaves_1.insert(leaf_1_ix, leaf_1.clone()); + leaves_1.insert(leaf_2_ix, leaf_2.clone()); + + history.add_version(root_1, id_1, nodes_1.clone(), leaves_1.clone())?; + assert_eq!(history.num_versions(), 1); + + // We then add another version that overlaps with the older version. + let root_2 = rand_value::(); + let id_2 = 5; + + let mut nodes_2 = NodeChanges::default(); + let n3_value: Word = rand_value(); + let n4_value: Word = rand_value(); + nodes_2.insert(NodeIndex::new(2, 1).unwrap(), n3_value); + nodes_2.insert(NodeIndex::new(10, 256).unwrap(), n4_value); + + let mut leaf_3 = CompactLeaf::new(); + let leaf_3_ix = leaf_2_ix; + let mut l3_e1_key: Word = rand_value(); + l3_e1_key[3] = leaf_3_ix.value().try_into().unwrap(); + let l3_e1_value: Word = rand_value(); + leaf_3.insert(l3_e1_key, l3_e1_value); + + let mut leaves_2 = LeafChanges::default(); + leaves_2.insert(leaf_3_ix, leaf_3.clone()); + history.add_version(root_2, id_2, nodes_2.clone(), leaves_2.clone())?; + assert_eq!(history.num_versions(), 2); + + // And another version for the sake of the test. + let root_3 = rand_value::(); + let id_3 = 6; + + let mut nodes_3 = NodeChanges::default(); + let n5_value: Word = rand_value(); + nodes_3.insert(NodeIndex::new(30, 1).unwrap(), n5_value); + + let mut leaf_4 = CompactLeaf::new(); + let l4_e1_key: Word = rand_value(); + let l4_e1_value: Word = rand_value(); + let leaf_4_ix = LeafIndex::from(l4_e1_key); + leaf_4.insert(l4_e1_key, l4_e1_value); + + let mut leaves_3 = LeafChanges::default(); + leaves_3.insert(leaf_4_ix, leaf_4.clone()); + + history.add_version(root_3, id_3, nodes_3.clone(), leaves_3.clone())?; + assert_eq!(history.num_versions(), 3); + + // At this point, we can grab a view into the history. If we grab something older than the + // history knows about we should get an error. + assert!(history.get_view_at(2).is_err()); + + // If we grab something valid, then we should get the right results. Let's grab the oldest + // possible version to test the overlay logic. + let view = history.get_view_at(id_1)?; + + // Getting a node in the targeted version should just return it. + assert_eq!(view.node_value(&NodeIndex::new(2, 1).unwrap()), Some(&n1_value)); + assert_eq!(view.node_value(&NodeIndex::new(8, 128).unwrap()), Some(&n2_value)); + + // Getting a node that is _not_ in the targeted delta directly should search through the + // versions in between the targeted version at the current tree and return the oldest value + // it can find for it. + assert_eq!(view.node_value(&NodeIndex::new(10, 256).unwrap()), Some(&n4_value)); + assert_eq!(view.node_value(&NodeIndex::new(30, 1).unwrap()), Some(&n5_value)); + + // Getting a node that doesn't exist in ANY versions should return none. + assert!(view.node_value(&NodeIndex::new(45, 100).unwrap()).is_none()); + + // Similarly, getting a leaf from the targeted version should just return it. + assert_eq!(view.leaf_value(&leaf_1_ix), Some(&leaf_1)); + assert_eq!(view.leaf_value(&leaf_2_ix), Some(&leaf_2)); + + // But getting a leaf that is not in the target delta directly should result in the same + // traversal. + assert_eq!(view.leaf_value(&leaf_4_ix), Some(&leaf_4)); + + // And getting a leaf that does not exist in any of the versions should return one. + assert!(view.leaf_value(&LeafIndex::new(1024).unwrap()).is_none()); + + // Finally, getting a full value from a compact leaf should yield the value directly from + // the target version if the target version overlays it AND contains it. + assert_eq!(view.value(&l1_e1_key), Some(Some(&l1_e1_value))); + assert_eq!(view.value(&l2_e1_key), Some(Some(&l2_e1_value))); + assert_eq!(view.value(&l2_e2_key), Some(Some(&l2_e2_value))); + + // However, if the leaf exists but does not contain the provided word, it should return the + // sentinel `Some(None)`. + let mut ne_key_in_existing_leaf: Word = rand_value(); + ne_key_in_existing_leaf[3] = leaf_1_ix.value().try_into().unwrap(); + assert_eq!(view.value(&ne_key_in_existing_leaf), Some(None)); + + // If the leaf is not overlaid, then the lookup should go up the chain just as in the other + // cases. + assert_eq!(view.value(&l4_e1_key), Some(Some(&l4_e1_value))); + + // But if nothing is found, it should just return None; + let ne_key: Word = rand_value(); + assert!(view.value(&ne_key).is_none()); + + // We can also get views for versions that are not directly contained, such as a version newer + // than the newest. This should just use the newest version to service the query. + let view = history.get_view_at(7)?; + assert_eq!(view.node_value(&NodeIndex::new_unchecked(30, 1)), Some(&n5_value)); + assert!(view.node_value(&NodeIndex::new_unchecked(30, 2)).is_none()); + + Ok(()) +} diff --git a/miden-crypto/src/merkle/smt/large_forest/mod.rs b/miden-crypto/src/merkle/smt/large_forest/mod.rs new file mode 100644 index 000000000..e67a5f27a --- /dev/null +++ b/miden-crypto/src/merkle/smt/large_forest/mod.rs @@ -0,0 +1,7 @@ +//! A high-performance sparse merkle tree forest backed by pluggable storage. + +mod error; +mod history; + +pub use error::LargeSmtForestError; +pub use history::{History, HistoryView, error::HistoryError}; diff --git a/miden-crypto/src/merkle/smt/mod.rs b/miden-crypto/src/merkle/smt/mod.rs index 5648d99c5..f7b8b87b3 100644 --- a/miden-crypto/src/merkle/smt/mod.rs +++ b/miden-crypto/src/merkle/smt/mod.rs @@ -26,6 +26,9 @@ pub use large::{ #[cfg(feature = "rocksdb")] pub use large::{RocksDbConfig, RocksDbStorage}; +mod large_forest; +pub use large_forest::{History, HistoryError, HistoryView, LargeSmtForestError}; + mod simple; pub use simple::{SimpleSmt, SimpleSmtProof};