diff --git a/Cargo.lock b/Cargo.lock index 8bab148b24..23edba8af2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -374,6 +374,29 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "bytecheck" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0caa33a2c0edca0419d15ac723dff03f1956f7978329b1e3b5fdaaaed9d3ca8b" +dependencies = [ + "bytecheck_derive", + "ptr_meta", + "rancor", + "simdutf8", +] + +[[package]] +name = "bytecheck_derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89385e82b5d1821d2219e0b095efa2cc1f246cbf99080f3be46a1a85c0d392d9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "byteorder" version = "1.5.0" @@ -1630,6 +1653,12 @@ dependencies = [ "foldhash", ] +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "heck" version = "0.3.3" @@ -2324,6 +2353,26 @@ dependencies = [ "windows-sys 0.61.0", ] +[[package]] +name = "munge" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e17401f259eba956ca16491461b6e8f72913a0a114e39736ce404410f915a0c" +dependencies = [ + "munge_macro", +] + +[[package]] +name = "munge_macro" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "nom" version = "7.1.3" @@ -2508,6 +2557,7 @@ dependencies = [ name = "octez-riscv-durable-storage" version = "0.0.0" dependencies = [ + "anyhow", "bincode", "bytes", "cfg-if", @@ -2522,6 +2572,7 @@ dependencies = [ "perfect-derive", "proptest", "rand 0.10.1", + "rkyv", "rocksdb", "serde", "serde_json", @@ -2831,6 +2882,26 @@ dependencies = [ "unarray", ] +[[package]] +name = "ptr_meta" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b9a0cf95a1196af61d4f1cbdab967179516d9a4a4312af1f31948f8f6224a79" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "quanta" version = "0.12.6" @@ -2873,6 +2944,15 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +[[package]] +name = "rancor" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a063ea72381527c2a0561da9c80000ef822bdd7c3241b1cc1b12100e3df081ee" +dependencies = [ + "ptr_meta", +] + [[package]] name = "rand" version = "0.7.3" @@ -3142,6 +3222,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rend" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cadadef317c2f20755a64d7fdc48f9e7178ee6b0e1f7fce33fa60f1d68a276e6" +dependencies = [ + "bytecheck", +] + [[package]] name = "reqwest" version = "0.11.27" @@ -3239,6 +3328,36 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "rkyv" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73389e0c99e664f919275ab5b5b0471391fe9a8de61e1dff9b1eaf56a90f16e3" +dependencies = [ + "bytecheck", + "bytes", + "hashbrown 0.17.1", + "indexmap 2.11.4", + "munge", + "ptr_meta", + "rancor", + "rend", + "rkyv_derive", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d2ed0b54125315fb36bd021e82d314d1c126548f871634b483f46b31d13cac6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "rocksdb" version = "0.24.0" @@ -3610,6 +3729,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "similar" version = "2.7.0" diff --git a/Cargo.toml b/Cargo.toml index 0cf6aba99f..857dd3fa6c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,6 +65,7 @@ proptest = "1.11.0" quanta = "0.12.6" rand = "0.10.1" range-collections = "0.4.6" +rkyv = { version = "0.8", features = ["bytes-1"] } rustc-demangle = "0.1" rocksdb = "0.24.0" rustc_apfloat = "0.2.3" diff --git a/durable-storage/Cargo.toml b/durable-storage/Cargo.toml index fdce3bfff3..97c08ada9e 100644 --- a/durable-storage/Cargo.toml +++ b/durable-storage/Cargo.toml @@ -13,6 +13,9 @@ unstable-test-utils = [ "dep:proptest", "dep:serde", "dep:serde_with", + "dep:anyhow", + "dep:serde_json", + "dep:rkyv", "octez-riscv-data/unstable-test-utils", ] @@ -37,10 +40,22 @@ trait-set.workspace = true workspace = true optional = true +[dependencies.anyhow] +workspace = true +optional = true + +[dependencies.serde_json] +workspace = true +optional = true + [dependencies.proptest] workspace = true optional = true +[dependencies.rkyv] +workspace = true +optional = true + [dependencies.serde] workspace = true optional = true diff --git a/durable-storage/src/database.rs b/durable-storage/src/database.rs index 87dce6614c..e904ed112a 100644 --- a/durable-storage/src/database.rs +++ b/durable-storage/src/database.rs @@ -623,7 +623,7 @@ mod traced_database; #[cfg(test)] pub(crate) use self::traced_database::Trace; -#[cfg(test)] +#[cfg(any(test, rocksdb_test_utils))] pub(crate) use self::traced_database::TracedDatabase; #[cfg(test)] diff --git a/durable-storage/src/database/traced_database.rs b/durable-storage/src/database/traced_database.rs index b84d681547..dc2dc4fa82 100644 --- a/durable-storage/src/database/traced_database.rs +++ b/durable-storage/src/database/traced_database.rs @@ -2,11 +2,15 @@ // // SPDX-License-Identifier: MIT -#![cfg(test)] +#![cfg(any(test, rocksdb_test_utils))] -//! Test-only [`Database`] wrapper which can record execution traces +//! [`Database`] wrapper which can record execution traces. +//! +//! Available to unit tests and, under the `unstable-test-utils` feature, to the +//! long-running test binary (`src/bin/database_long_test.rs`). use std::cell::RefCell; +#[cfg(test)] use std::collections::HashMap; use bytes::Bytes; @@ -17,22 +21,28 @@ use octez_riscv_data::mode::Mode; use octez_riscv_data::mode::Normal; use octez_riscv_data::mode::ProvableExt; use octez_riscv_data::mode::Prove; +#[cfg(test)] use octez_riscv_data::mode::Verify; use tokio::runtime::Handle; use crate::commit::CommitId; use crate::database::Database; use crate::database::DatabaseMode; +#[cfg(test)] use crate::database::VerifyImpl; use crate::errors::Error; +#[cfg(test)] use crate::errors::InvalidArgumentError; use crate::errors::OperationalError; use crate::key::Key; +#[cfg(test)] use crate::merkle_layer::new_verify_layer; use crate::merkle_worker::BackgroundKeyValueStore; use crate::merkle_worker::BackgroundPersistentKeyValueStore; +#[cfg(test)] use crate::storage::KeyValueStore; use crate::storage::PersistentKeyValueStore; +#[cfg(test)] use crate::storage::TestKeyValueStoreSetup; use crate::test_helpers::DatabaseOperation; @@ -102,6 +112,7 @@ pub(crate) struct TracedDatabase { impl TracedDatabase { /// Equivalent to [`Database::try_new`] which also records a [`TraceEntry`]. + #[cfg(test)] pub(crate) fn try_new(handle: &Handle, repo: &KV::Repo) -> Result { Ok(TracedDatabase::from(Database::try_new(handle, repo)?)) } @@ -141,6 +152,7 @@ impl TracedDatabase { } /// Record a [`TraceEntry::Proof`] for the proof of `step`. + #[cfg(test)] pub(crate) fn record_proof(&self, step: DatabaseOperation, proof: Vec) { self.trace .borrow_mut() @@ -148,6 +160,7 @@ impl TracedDatabase { } } +#[cfg(test)] impl TracedDatabase where KV: KeyValueStore + TestKeyValueStoreSetup, @@ -270,6 +283,7 @@ impl TracedDatabase { } /// Insert entries into the database and return the inserted key pairs + #[cfg(test)] pub(crate) fn insert_entries( &mut self, entries: Vec<(Vec, Vec)>, @@ -286,11 +300,13 @@ impl TracedDatabase { } /// Assert that a database contains the expected value for a given key. + #[cfg(test)] pub(crate) fn assert_database_value(&self, key: &Key, expected: &[u8]) { self.inner.assert_database_value(key, expected); } /// Assert that a database does not contain the given key. + #[cfg(test)] pub(crate) fn assert_traced_database_missing(&self, key: &Key) { assert!(matches!( self.read_bytes(key, 0, 0), diff --git a/durable-storage/src/lib.rs b/durable-storage/src/lib.rs index 2b66c5b42d..607623d2ce 100644 --- a/durable-storage/src/lib.rs +++ b/durable-storage/src/lib.rs @@ -33,6 +33,10 @@ pub mod commit; pub mod database; pub mod errors; pub mod key; +// The long-running test exercises the persistence backend directly, so it is +// only available when `rocksdb` is enabled. +#[cfg(rocksdb_test_utils)] +pub mod long_test; mod merkle_layer; mod merkle_worker; pub mod persistence_layer; diff --git a/durable-storage/src/long_test/mod.rs b/durable-storage/src/long_test/mod.rs new file mode 100644 index 0000000000..edb3b1ff03 --- /dev/null +++ b/durable-storage/src/long_test/mod.rs @@ -0,0 +1,527 @@ +// SPDX-FileCopyrightText: 2026 Nomadic Labs +// +// SPDX-License-Identifier: MIT + +//! Long-running property-based test for [`Database`]. +//! +//! Each epoch advances and commits a shared base state, then runs a `proptest` +//! test on that base. Every case applies a model-guided operation +//! sequence to the reference model and three databases (in-memory traced, +//! persistence traced, and production), cross-checking traces, root hashes, and +//! proofs. +//! +//! On failure the committed base of the failing epoch and the shrunk operation +//! sequence are written to `/failure/` so the failure can be replayed. +//! +//! [`Database`]: crate::database::Database + +pub mod model; +pub mod run_case; +pub mod strategy; + +use std::fs; +use std::path::Path; +use std::path::PathBuf; +use std::time::Duration; +use std::time::Instant; + +use anyhow::Context; +use anyhow::Result; +use anyhow::bail; +use octez_riscv_data::hash::Hash; +use proptest::strategy::Strategy; +use proptest::strategy::ValueTree; +use proptest::test_runner::Config as ProptestConfig; +use proptest::test_runner::RngAlgorithm; +use proptest::test_runner::TestError; +use proptest::test_runner::TestRng; +use proptest::test_runner::TestRunner; + +use self::model::LongTestModel; +use self::run_case::Base; +use self::run_case::advance_base; +use self::run_case::initial_base; +use self::run_case::run_case; +use self::strategy::long_test_ops_strategy; +use crate::commit::CommitId; +use crate::persistence_layer::PersistenceLayer; +use crate::repo::DirectoryManager; +use crate::storage::PersistentKeyValueStore; +use crate::storage::in_memory::InMemoryKeyValueStore; +use crate::storage::in_memory::InMemoryRepo; +use crate::test_helpers::DatabaseOperation; + +const IN_MEMORY_BASE: &str = "in-memory-base"; +const PERSISTENT_BASE: &str = "persistent-base"; +const META_FILE: &str = "meta.json"; +const REGRESSION_FILE: &str = "regression.json"; +const BASE_MODEL_FILE: &str = "base-model.json"; + +/// Configuration for a long-running test invocation. +pub struct LongTestConfig { + /// Maximum number of epochs to run. `None` runs until the time budget. + pub epochs: Option, + /// Maximum number of operations sampled per epoch. + pub ops_per_epoch: usize, + /// Number of test cases per epoch. + pub cases_per_epoch: u32, + /// Optional seed. + pub seed: Option, + /// Time budget. The loop stops cleanly once exceeded. + pub time_budget: Option, + /// If set, replay the failing epoch described by `/meta.json`. + pub replay: Option, +} + +/// Metadata persisted alongside a failure which enables replaying it. +#[derive(serde::Serialize, serde::Deserialize)] +struct FailureMeta { + /// Seed of the run. + seed: Hash, + /// Index of the failing epoch. + epoch: u64, + /// Operations sampled per epoch. + ops_per_epoch: usize, + /// Test cases per epoch. + cases_per_epoch: u32, + /// The commit identifying the failing epoch's starting state. + base_commit: CommitId, + /// Short description of the failure. + reason: String, + /// Git revision, if available from the environment. + git_sha: String, +} + +/// Run the long-running test +pub fn run_long_test(config: LongTestConfig) -> Result<()> { + // Replay reconstructs only the failing epoch; it is handled separately. + if let Some(replay_dir) = &config.replay { + return replay_failure(replay_dir); + } + + let seed = config + .seed + .unwrap_or_else(|| rand::random::<[u8; 32]>().into()); + let max_epochs = config.epochs; + let ops_per_epoch = config.ops_per_epoch; + let cases_per_epoch = config.cases_per_epoch; + let out_dir = tempfile::Builder::new() + .prefix("database_long_test-") + .tempdir()? + .keep(); + + eprintln!("test seed: {seed}"); + eprintln!( + "out-dir: {} | ops/epoch: {ops_per_epoch} | cases/epoch: {cases_per_epoch}", + out_dir.display(), + ); + + let repo_dir = out_dir.join("repo"); + fs::create_dir_all(&repo_dir) + .with_context(|| format!("creating repo dir {}", repo_dir.display()))?; + let persistent_repo = + DirectoryManager::new(&repo_dir).context("creating the directory manager")?; + let in_memory_repo = InMemoryRepo::default(); + + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .build() + .context("building the tokio runtime")?; + let handle = runtime.handle(); + + let mut base = initial_base(handle, &in_memory_repo, &persistent_repo); + let mut epoch = 0u64; + + let start = Instant::now(); + loop { + if let Some(max) = max_epochs { + if epoch >= max { + break; + } + } + if let Some(budget) = config.time_budget { + if start.elapsed() >= budget { + eprintln!("time budget reached after {epoch} epochs"); + break; + } + } + + let mut runner = epoch_runner(seed, epoch, cases_per_epoch); + + // Advance and commit the base by a generated sequence (no proofs). + let advance_ops = long_test_ops_strategy(&base.model.pools(), ops_per_epoch) + .new_tree(&mut runner) + .map_err(|e| anyhow::anyhow!("{e}")) + .context("drawing the epoch advance sequence")? + .current(); + base = advance_base( + handle, + &in_memory_repo, + &persistent_repo, + &base, + &advance_ops, + ); + + // Run the property test on this base. + let strategy = long_test_ops_strategy(&base.model.pools(), ops_per_epoch); + let result = runner.run(&strategy, |ops| { + run_case(handle, &in_memory_repo, &persistent_repo, &base, &ops); + Ok(()) + }); + + match result { + Ok(()) => { + eprintln!( + "epoch {epoch} ok (db contains {} entries)", + base.model.data.len() + ); + } + Err(TestError::Fail(reason, ops)) => { + let meta = FailureMeta { + seed, + epoch, + ops_per_epoch, + cases_per_epoch, + base_commit: base.commit, + reason: reason.to_string(), + git_sha: std::env::var("GITHUB_SHA").unwrap_or_else(|_| "unknown".to_string()), + }; + write_failure( + &out_dir, + &persistent_repo, + &in_memory_repo, + &meta, + &base.model, + &ops, + )?; + bail!( + "epoch {epoch} failed: {reason}. Artifacts written to {}", + out_dir.join("failure").display() + ); + } + Err(TestError::Abort(reason)) => { + bail!("epoch {epoch} aborted: {reason}"); + } + } + + epoch += 1; + } + + eprintln!("completed {epoch} epochs"); + Ok(()) +} + +/// Build a deterministically seeded test runner for `epoch`. +fn epoch_runner(seed: Hash, epoch: u64, cases: u32) -> TestRunner { + // XOR the epoch index into the seed so each epoch has a distinct yet + // reproducible seed + let mut seed: [u8; 32] = seed.into(); + let head = u64::from_le_bytes(seed[..8].try_into().expect("8 bytes")) ^ epoch; + seed[..8].copy_from_slice(&head.to_le_bytes()); + + let rng = TestRng::from_seed(RngAlgorithm::ChaCha, &seed); + let config = ProptestConfig { + cases, + failure_persistence: None, + ..ProptestConfig::default() + }; + TestRunner::new_with_rng(config, rng) +} + +/// Write the failure artifacts: metadata, the shrunk operation sequence, the +/// reference model, and the committed base on both backends. +fn write_failure( + out_dir: &Path, + persistent_repo: &DirectoryManager, + in_memory_repo: &InMemoryRepo, + meta: &FailureMeta, + model: &LongTestModel, + ops: &[DatabaseOperation], +) -> Result<()> { + fn write_failure_file( + failure_dir: &Path, + content: &impl serde::ser::Serialize, + name: &str, + ) -> Result<()> { + let file = fs::File::create(failure_dir.join(name)) + .context(format!("creating failure file {name}"))?; + serde_json::to_writer_pretty(file, content) + .context(format!("writing failure to {name}"))?; + Ok(()) + } + let failure_dir = out_dir.join("failure"); + if failure_dir.exists() { + fs::remove_dir_all(&failure_dir) + .with_context(|| format!("clearing {}", failure_dir.display()))?; + } + fs::create_dir_all(&failure_dir) + .with_context(|| format!("creating {}", failure_dir.display()))?; + + write_failure_file(&failure_dir, meta, META_FILE)?; + write_failure_file(&failure_dir, &ops, REGRESSION_FILE)?; + write_failure_file(&failure_dir, model, BASE_MODEL_FILE)?; + + let in_memory_store = InMemoryKeyValueStore::checkout(in_memory_repo, &meta.base_commit) + .context("checking out the in-memory base")?; + in_memory_store + .commit_to_path(&failure_dir.join(IN_MEMORY_BASE)) + .context("writing the in-memory base snapshot")?; + + let persistent_store = PersistenceLayer::checkout(persistent_repo, &meta.base_commit) + .context("checking out the persistent base")?; + persistent_store + .commit_to_path(&failure_dir.join(PERSISTENT_BASE)) + .context("writing the persistent base snapshot")?; + + eprintln!( + "failure artifacts written to {}; replay with --replay {}", + failure_dir.display(), + failure_dir.display(), + ); + Ok(()) +} + +/// Reproduce a recorded failure by reconstructing only the failing epoch. +/// Both the persistence backend's base and the in-memory backend's base +/// are restored from disk, and the saved (shrunk) operation sequence is applied once. +fn replay_failure(dir: &Path) -> Result<()> { + fn read_failure_file( + failure_dir: &Path, + name: &str, + ) -> Result { + let file = fs::File::open(failure_dir.join(name)) + .context(format!("opening failure file {name}"))?; + serde_json::from_reader(file).context(format!("decoding {name}")) + } + + let meta: FailureMeta = read_failure_file(dir, META_FILE)?; + eprintln!( + "Replaying epoch {} from {} (prior epochs are not re-run)", + meta.epoch, + dir.display(), + ); + + let out_dir = dir.join("replay-run"); + let repo_dir = out_dir.join("repo"); + fs::create_dir_all(&repo_dir) + .with_context(|| format!("creating repo dir {}", repo_dir.display()))?; + let persistent_repo = + DirectoryManager::new(&repo_dir).context("creating the directory manager")?; + let in_memory_repo = InMemoryRepo::default(); + + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .build() + .context("building the tokio runtime")?; + let handle = runtime.handle(); + + // Restore the persistence backend's base from the saved commit. The saved + // snapshot is opened read-only and re-committed into the fresh repo. + let working_dir = persistent_repo + .temp_database_dir() + .context("creating a scratch directory")?; + let persistent_store = + PersistenceLayer::checkout_from_path(&dir.join(PERSISTENT_BASE), working_dir) + .context("loading the persistent base snapshot")?; + persistent_store + .commit(&persistent_repo, &meta.base_commit) + .context("registering the persistent base")?; + + // Restore the in-memory backend's base from its saved snapshot. + let working_dir = persistent_repo + .temp_database_dir() + .context("creating a scratch directory")?; + let in_memory_store = + InMemoryKeyValueStore::checkout_from_path(&dir.join(IN_MEMORY_BASE), working_dir) + .context("loading the in-memory base snapshot")?; + in_memory_store + .commit(&in_memory_repo, &meta.base_commit) + .context("registering the in-memory base")?; + + // The reference model carries the expected key/value state for assertions. + let base = Base { + commit: meta.base_commit, + model: read_failure_file(dir, BASE_MODEL_FILE)?, + }; + + // Apply the recorded operation sequence once and catch the resulting panic + let ops: Vec = read_failure_file(dir, REGRESSION_FILE)?; + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + run_case(handle, &in_memory_repo, &persistent_repo, &base, &ops); + })); + + match outcome { + Err(payload) => { + let reason = payload + .downcast_ref::<&str>() + .map(|s| s.to_string()) + .or_else(|| payload.downcast_ref::().cloned()) + .unwrap_or_else(|| "unknown panic".to_string()); + bail!( + "replay reproduced the failure for epoch {}: {reason}", + meta.epoch + ); + } + Ok(()) => { + eprintln!( + "replay did NOT reproduce the failure for epoch {}", + meta.epoch + ); + Ok(()) + } + } +} + +#[cfg(test)] +mod tests { + use bytes::Bytes; + use octez_riscv_test_utils::TestableTmpdir; + use tokio::runtime::Runtime; + + use super::*; + use crate::key::Key; + + // A short run of the long test + #[test] + fn database_long_test_restricted() { + run_long_test(LongTestConfig { + epochs: Some(3), + ops_per_epoch: 200, + cases_per_epoch: 32, + seed: None, + time_budget: None, + replay: None, + }) + .expect("the short long test run should succeed"); + } + + // Tests for the failure replay mechanism of the long test + + struct TestSetup { + _tmp: TestableTmpdir, + runtime: Runtime, + out_dir: PathBuf, + persistent_repo: DirectoryManager, + in_memory_repo: InMemoryRepo, + base: Base, + key: Key, + } + + /// Build a base committed on all backends containing a single key. + fn build_base_with_key() -> TestSetup { + let tmp = TestableTmpdir::new(); + let out_dir = tmp.path().to_owned(); + let repo_dir = out_dir.join("repo"); + fs::create_dir_all(&repo_dir).expect("creating the repo dir should succeed"); + let persistent_repo = DirectoryManager::new(&repo_dir) + .expect("creating the directory manager should succeed"); + let in_memory_repo = InMemoryRepo::default(); + + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .build() + .expect("building the tokio runtime should succeed"); + + let key = Key::new(&[1, 2, 3]).expect("the key should be valid"); + let set = DatabaseOperation::Set(key.clone(), Bytes::from_static(b"value")); + + let base = initial_base(runtime.handle(), &in_memory_repo, &persistent_repo); + let base = advance_base( + runtime.handle(), + &in_memory_repo, + &persistent_repo, + &base, + &[set], + ); + + TestSetup { + _tmp: tmp, + runtime, + out_dir, + persistent_repo, + in_memory_repo, + base, + key, + } + } + + /// Build a [`FailureMeta`] for `base_commit`; the non-essential fields are + /// placeholders (replay only reads the commit and epoch index). + fn dummy_meta(base_commit: CommitId) -> FailureMeta { + FailureMeta { + seed: Hash::from([0u8; 32]), + epoch: 0, + ops_per_epoch: 1, + cases_per_epoch: 1, + base_commit, + reason: "test".to_string(), + git_sha: "test".to_string(), + } + } + + // Replay of an artifact whose model disagrees with the restored database + // reproduces the failure + #[test] + fn internal_test_replay_reproduces_recorded_failure() { + let setup = build_base_with_key(); + let meta = dummy_meta(setup.base.commit); + + // An empty model disagrees with the restored base (which holds `key`): + // checking the key's existence will mismatch and panic in `run_case`. + let model = LongTestModel::default(); + let ops = vec![DatabaseOperation::Exists(setup.key.clone())]; + write_failure( + &setup.out_dir, + &setup.persistent_repo, + &setup.in_memory_repo, + &meta, + &model, + &ops, + ) + .expect("writing the failure artifact should succeed"); + + let err = replay_failure(&setup.out_dir.join("failure")) + .expect_err("replay should reproduce the failure"); + assert!( + err.to_string().contains("reproduced"), + "unexpected replay error: {err}" + ); + + drop(setup.runtime); + } + + // Replay of an artifact with the correct model must restore the base + // faithfully on every backend. Also checks that replaying a dummy failure + // doesn't work + #[test] + fn internal_test_replay_passes_for_a_consistent_base() { + let setup = build_base_with_key(); + let meta = dummy_meta(setup.base.commit); + + // The recorded model matches the restored base, so a non-mutating + // sequence (which also hashes, cross-checking all backends) must pass. + let ops = vec![ + DatabaseOperation::Exists(setup.key.clone()), + DatabaseOperation::Hash, + ]; + write_failure( + &setup.out_dir, + &setup.persistent_repo, + &setup.in_memory_repo, + &meta, + &setup.base.model, + &ops, + ) + .expect("writing the failure artifact should succeed"); + + // The failure artifact must be self-contained: replay must succeed even + // once the original repo is gone. + fs::remove_dir_all(setup.out_dir.join("repo")) + .expect("removing the original repo should succeed"); + + replay_failure(&setup.out_dir.join("failure")) + .expect("replay of a consistent base should not reproduce a failure"); + + drop(setup.runtime); + } +} diff --git a/durable-storage/src/long_test/model.rs b/durable-storage/src/long_test/model.rs new file mode 100644 index 0000000000..cb422ff1d3 --- /dev/null +++ b/durable-storage/src/long_test/model.rs @@ -0,0 +1,122 @@ +// SPDX-FileCopyrightText: 2026 Nomadic Labs +// +// SPDX-License-Identifier: MIT + +//! In-memory reference model for the long-running [`Database`] tests +//! +//! [`Database`]: crate::database::Database + +use std::collections::HashMap; +use std::collections::VecDeque; + +use bytes::Bytes; +use tezos_smart_rollup_constants::core::MAX_FILE_CHUNK_SIZE; + +use crate::key::Key; +use crate::test_helpers::DatabaseOperation; +use crate::test_helpers::DatabaseReferenceModel; + +/// Maximum number of keys retained in the hot / recently-deleted pools. +const POOL_CAP: usize = 64; + +/// Snapshot of the model's key pools, used as an input for operation +/// generation strategies +#[derive(Clone, Debug, Default)] +pub struct KeyPools { + /// All present keys, sorted. + pub existing: Vec, + /// Recently written or read keys. + pub hot: Vec, + /// Recently deleted keys. + pub deleted: Vec, +} + +/// Reference model which tracks the key/value store and extra state used +/// to guide the operation generation strategy. The model of a failing epoch +/// can be persisted alongside the durable storage commit and reloaded on replay. +#[serde_with::serde_as] +#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)] +pub struct LongTestModel { + #[serde_as(as = "HashMap<_, serde_with::hex::Hex>")] + pub(crate) data: HashMap, + hot: VecDeque, + recently_deleted: VecDeque, +} + +impl LongTestModel { + /// A snapshot of the key pools used to inform operation generation. + /// The existing keys are sorted in order to make the operations strategy + /// reproducible. + pub fn pools(&self) -> KeyPools { + let mut existing: Vec = self.data.keys().cloned().collect(); + existing.sort(); + KeyPools { + existing, + hot: self.hot.iter().cloned().collect(), + deleted: self.recently_deleted.iter().cloned().collect(), + } + } + + fn touch_hot(&mut self, key: &Key) { + // When key becomes hot, remove from recently deleted set + self.recently_deleted.retain(|k| k != key); + + if !self.hot.contains(key) { + self.hot.push_back(key.clone()); + while self.hot.len() > POOL_CAP { + self.hot.pop_front(); + } + } + } + + fn mark_deleted(&mut self, key: &Key) { + // When key is deleted, remove from hot set + self.hot.retain(|k| k != key); + + if !self.recently_deleted.contains(key) { + self.recently_deleted.push_back(key.clone()); + while self.recently_deleted.len() > POOL_CAP { + self.recently_deleted.pop_front(); + } + } + } +} + +impl DatabaseReferenceModel for LongTestModel { + fn data(&self) -> &HashMap { + &self.data + } + + fn apply(&mut self, operation: &DatabaseOperation) { + match operation { + DatabaseOperation::Set(key, data) => { + if data.len() <= MAX_FILE_CHUNK_SIZE { + self.data.insert(key.clone(), data.clone()); + self.touch_hot(key); + } + } + DatabaseOperation::Write(key, offset, data) => { + if let Some(new_value) = self.write_outcome(key, *offset, data) { + self.data.insert(key.clone(), new_value); + self.touch_hot(key); + } + } + DatabaseOperation::Read(key, _, _) => { + if self.data.contains_key(key) { + self.touch_hot(key); + } + } + DatabaseOperation::Delete(key) => { + if self.data.remove(key).is_some() { + self.mark_deleted(key); + } + } + DatabaseOperation::Exists(_) + | DatabaseOperation::ValueLength(_) + | DatabaseOperation::Hash + | DatabaseOperation::Commit + | DatabaseOperation::Checkout + | DatabaseOperation::CommitCheckoutRoundtrip => {} + } + } +} diff --git a/durable-storage/src/long_test/run_case.rs b/durable-storage/src/long_test/run_case.rs new file mode 100644 index 0000000000..0fbabe6f2d --- /dev/null +++ b/durable-storage/src/long_test/run_case.rs @@ -0,0 +1,208 @@ +// SPDX-FileCopyrightText: 2026 Nomadic Labs +// +// SPDX-License-Identifier: MIT + +//! Per-case execution: apply an operation sequence to the reference model and +//! three databases in lockstep, cross-checking results, traces, hashes, and +//! proofs. +//! +//! The three databases are: +//! - an in-memory [`TracedDatabase`] +//! - a persistent [`TracedDatabase`] +//! - a production [`Database`] +//! +//! Traces are compared between the two [`TracedDatabase`]s. Root hashes are +//! compared among all three. For every provable operation a proof is produced +//! from the persistent [`TracedDatabase`] and verified. + +use octez_riscv_data::hash::Hash; +use octez_riscv_data::mode::Normal; +use tokio::runtime::Handle; + +use crate::commit::CommitId; +use crate::database::Database; +use crate::database::TracedDatabase; +use crate::long_test::model::LongTestModel; +use crate::persistence_layer::PersistenceLayer; +use crate::repo::DirectoryManager; +use crate::storage::in_memory::InMemoryKeyValueStore; +use crate::storage::in_memory::InMemoryRepo; +use crate::test_helpers::DatabaseOperation; +use crate::test_helpers::DatabaseReferenceModel; +use crate::test_helpers::check_and_apply_value_operation; +use crate::test_helpers::prove_and_verify_operation; + +/// The committed starting state shared by every case in an epoch. +#[derive(Clone)] +pub struct Base { + /// The commit identifying the starting state (identical across backends). + pub commit: CommitId, + /// The reference model corresponding to `commit`. + pub model: LongTestModel, +} + +/// State carried while applying a sequence of operations to all targets. +struct Targets { + in_memory_db: TracedDatabase, + persistent_db: TracedDatabase, + production_db: Database, + model: LongTestModel, +} + +/// Check out the shared `base` into a fresh set of targets. +fn checkout_targets( + handle: &Handle, + in_memory_repo: &InMemoryRepo, + persistent_repo: &DirectoryManager, + base: &Base, +) -> Targets { + let in_memory_db = TracedDatabase::::checkout( + handle, + in_memory_repo, + base.commit, + ) + .expect("checking out the in-memory base should succeed"); + let persistent_db = + TracedDatabase::::checkout(handle, persistent_repo, base.commit) + .expect("checking out the persistence base should succeed"); + let production_db = + Database::::checkout(handle, persistent_repo, base.commit) + .expect("checking out the production base should succeed"); + + Targets { + in_memory_db, + persistent_db, + production_db, + model: base.model.clone(), + } +} + +/// Apply `ops` to all targets in lockstep, optionally producing and verifying a +/// proof for every provable operation. +fn apply_sequence(targets: &mut Targets, ops: &[DatabaseOperation], prove: bool) { + for op in ops { + // Proofs are taken over the pre-operation state, so prove first. + if prove { + prove_and_verify_operation(&targets.persistent_db, op); + } + + check_and_apply_value_operation(&mut targets.in_memory_db, &targets.model, op); + check_and_apply_value_operation(&mut targets.persistent_db, &targets.model, op); + check_and_apply_value_operation(&mut targets.production_db, &targets.model, op); + targets.model.apply(op); + } +} + +/// Assert the two traced databases recorded identical traces and all three +/// databases agree on the root hash. +fn check_consistency(targets: Targets) { + let (in_memory_db, in_memory_trace) = targets.in_memory_db.into_parts(); + let (persistent_db, persistent_trace) = targets.persistent_db.into_parts(); + + assert_eq!( + in_memory_trace, persistent_trace, + "trace mismatch between in-memory and persistence backends" + ); + + let in_memory_hash: Hash = in_memory_db + .hash() + .expect("hashing the in-memory database should succeed"); + let persist_hash: Hash = persistent_db + .hash() + .expect("hashing the persistence database should succeed"); + let production_hash: Hash = targets + .production_db + .hash() + .expect("hashing the production database should succeed"); + + assert_eq!( + in_memory_hash, persist_hash, + "root hash mismatch (in-memory vs persist)" + ); + assert_eq!( + persist_hash, production_hash, + "root hash mismatch (persist vs production)" + ); +} + +/// Run a single property-test case: check out the epoch base, apply `ops` with +/// proof generation, and cross-check traces and hashes. Panics on any mismatch +/// (caught by the proptest runner and shrunk). +pub fn run_case( + handle: &Handle, + in_memory_repo: &InMemoryRepo, + persistent_repo: &DirectoryManager, + base: &Base, + ops: &[DatabaseOperation], +) { + let mut targets = checkout_targets(handle, in_memory_repo, persistent_repo, base); + apply_sequence(&mut targets, ops, true); + check_consistency(targets); +} + +/// Advance the shared base by applying `ops` (without proofs) and committing the +/// result on every backend, returning the new committed [`Base`]. +pub fn advance_base( + handle: &Handle, + in_memory_repo: &InMemoryRepo, + persistent_repo: &DirectoryManager, + base: &Base, + ops: &[DatabaseOperation], +) -> Base { + let mut targets = checkout_targets(handle, in_memory_repo, persistent_repo, base); + apply_sequence(&mut targets, ops, false); + + let in_memory_commit = targets + .in_memory_db + .commit(in_memory_repo) + .expect("committing the in-memory base"); + let persistent_commit = targets + .persistent_db + .commit(persistent_repo) + .expect("committing the persistence base"); + let production_commit = targets + .production_db + .commit(persistent_repo) + .expect("committing the production base"); + assert_eq!( + in_memory_commit, persistent_commit, + "base commit id mismatch (in-memory vs persist)" + ); + assert_eq!( + persistent_commit, production_commit, + "base commit id mismatch (persist vs production)" + ); + + Base { + commit: persistent_commit, + model: targets.model, + } +} + +/// Commit an empty database on every backend to obtain the initial [`Base`]. +pub fn initial_base( + handle: &Handle, + in_memory_repo: &InMemoryRepo, + persistent_repo: &DirectoryManager, +) -> Base { + let in_memory_db = Database::::try_new(handle, in_memory_repo) + .expect("creating the in-memory database should succeed"); + let persistent_db = Database::::try_new(handle, persistent_repo) + .expect("creating the persistence database should succeed"); + + let in_memory_commit = in_memory_db + .commit(in_memory_repo) + .expect("committing the empty in-memory database"); + let persistent_commit = persistent_db + .commit(persistent_repo) + .expect("committing the empty persistence database"); + assert_eq!( + in_memory_commit, persistent_commit, + "empty commit id mismatch across backends" + ); + + Base { + commit: persistent_commit, + model: LongTestModel::default(), + } +} diff --git a/durable-storage/src/long_test/strategy.rs b/durable-storage/src/long_test/strategy.rs new file mode 100644 index 0000000000..ef2072cacd --- /dev/null +++ b/durable-storage/src/long_test/strategy.rs @@ -0,0 +1,103 @@ +// SPDX-FileCopyrightText: 2026 Nomadic Labs +// +// SPDX-License-Identifier: MIT + +//! Model-guided operation strategy for the long-running [`Database`] test. +//! +//! The strategy is parameterised by a snapshot of the reference model's key +//! pools ([`KeyPools`]), so generated operations favour recently used and +//! recently deleted keys while still introducing fresh keys. Because the +//! snapshot is fixed for the duration of an epoch, this is an ordinary +//! `proptest` [`Strategy`] and retains shrinking. +//! +//! [`Database`]: crate::database::Database + +use proptest::prelude::*; +use proptest::sample::select; +use proptest::strategy::BoxedStrategy; +use proptest::strategy::Union; +use tezos_smart_rollup_constants::core::MAX_FILE_CHUNK_SIZE; + +use crate::key::Key; +use crate::long_test::model::KeyPools; +use crate::test_helpers::DatabaseOperation; +use crate::test_helpers::VALUE_MAX_SIZE; +use crate::test_helpers::key_strategy; +use crate::test_helpers::value_strategy; + +/// A key strategy that blends fresh random keys with samples drawn from the +/// model's hot, existing, and recently-deleted pools. +fn pooled_key_strategy(pools: &KeyPools) -> BoxedStrategy { + let mut arms: Vec<(u32, BoxedStrategy)> = Vec::new(); + + arms.push((90, key_strategy().boxed())); + + if !pools.hot.is_empty() { + arms.push((5, select(pools.hot.clone()).boxed())); + } + if !pools.deleted.is_empty() { + arms.push((2, select(pools.deleted.clone()).boxed())); + } + if !pools.existing.is_empty() { + arms.push((3, select(pools.existing.clone()).boxed())); + } + + Union::new_weighted(arms).boxed() +} + +// Distribution is based on that of `test_helpers::database_operation_view_strategy` +fn database_op_strategy(pools: &KeyPools) -> BoxedStrategy { + let set = (pooled_key_strategy(pools), value_strategy()) + .prop_map(|(k, v)| DatabaseOperation::Set(k, v)); + + let read = ( + pooled_key_strategy(pools), + prop_oneof![ + 5 => Just(0usize), + 4 => 1..=MAX_FILE_CHUNK_SIZE, + 1 => (MAX_FILE_CHUNK_SIZE + 1)..=VALUE_MAX_SIZE, + ], + prop_oneof![ + 9 => 0..=MAX_FILE_CHUNK_SIZE, + 1 => (MAX_FILE_CHUNK_SIZE + 1)..=VALUE_MAX_SIZE, + ], + ) + .prop_map(|(k, off, len)| DatabaseOperation::Read(k, off, len)); + + let write_valid = ( + pooled_key_strategy(pools), + prop_oneof![ + 2 => Just(0), + 1 => 1..=VALUE_MAX_SIZE, + ], + value_strategy(), + ) + .prop_map(|(k, off, v)| DatabaseOperation::Write(k, off, v)); + + let write_invalid = ( + pooled_key_strategy(pools), + VALUE_MAX_SIZE..=usize::MAX, + value_strategy(), + ) + .prop_map(|(k, off, v)| DatabaseOperation::Write(k, off, v)); + + prop_oneof![ + 20 => set, + 20 => read, + 4 => write_valid, + 1 => write_invalid, + 10 => pooled_key_strategy(pools).prop_map(DatabaseOperation::Delete), + 10 => pooled_key_strategy(pools).prop_map(DatabaseOperation::Exists), + 5 => pooled_key_strategy(pools).prop_map(DatabaseOperation::ValueLength), + 10 => Just(DatabaseOperation::Hash), + ] + .boxed() +} + +pub fn long_test_ops_strategy( + pools: &KeyPools, + length: usize, +) -> impl Strategy> + use<> { + let length = length.max(1); + proptest::collection::vec(database_op_strategy(pools), 1..=length) +} diff --git a/durable-storage/src/storage/in_memory.rs b/durable-storage/src/storage/in_memory.rs index f8da0d6a01..f0b0393aec 100644 --- a/durable-storage/src/storage/in_memory.rs +++ b/durable-storage/src/storage/in_memory.rs @@ -6,6 +6,10 @@ //! In-memory storage backend [`KeyValueStore`]-compatible with the Persistence layer use std::collections::HashMap; +#[cfg(test_utils)] +use std::io::Read; +#[cfg(test_utils)] +use std::io::Write; use std::sync::RwLock; use bytes::Bytes; @@ -195,18 +199,50 @@ impl KeyValueStore for InMemoryKeyValueStore { } } -/// Test-only snapshot repository for [`InMemoryRepo`] +/// Test-only snapshot of an [`InMemoryKeyValueStore`] #[cfg(test_utils)] -#[derive(Debug)] +#[derive(Debug, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] struct InMemorySnapshot { blobs: HashMap, - values: HashMap, + values: HashMap, } +/// File name used within a commit directory by [`InMemoryKeyValueStore`]. +#[cfg(test_utils)] +const STORE_FILE: &str = "in_memory_snapshot.bin"; + #[cfg(test_utils)] impl super::PersistentKeyValueStore for InMemoryKeyValueStore { - fn commit_to_path(&self, _path: &std::path::Path) -> Result<(), OperationalError> { - unimplemented!("In-memory store cannot commit to disk") + fn commit_to_path(&self, path: &std::path::Path) -> Result<(), OperationalError> { + let blobs = self + .blobs + .read() + .map_err(|_| OperationalError::LockPoisoned)? + .clone(); + let values = self + .values + .read() + .map_err(|_| OperationalError::LockPoisoned)? + .iter() + .map(|(k, v)| (k.clone(), Bytes::copy_from_slice(v))) + .collect(); + let snapshot = InMemorySnapshot { blobs, values }; + + std::fs::create_dir_all(path).map_err(|error| OperationalError::DirCreationFailed { + path: path.to_path_buf(), + error, + })?; + let file = std::fs::File::create(path.join(STORE_FILE)) + .map_err(|error| OperationalError::FileWriteFailed { error })?; + let writer = rkyv::ser::writer::IoWriter::new(std::io::BufWriter::new(file)); + + rkyv::api::high::to_bytes_in::<_, rkyv::rancor::Error>(&snapshot, writer) + .map_err(|error| OperationalError::FileWriteFailed { + error: std::io::Error::other(error), + })? + .into_inner() + .flush() + .map_err(|error| OperationalError::FileWriteFailed { error }) } fn commit( @@ -223,7 +259,9 @@ impl super::PersistentKeyValueStore for InMemoryKeyValueStore { .values .read() .map_err(|_| OperationalError::LockPoisoned)? - .clone(); + .iter() + .map(|(k, v)| (k.clone(), Bytes::copy_from_slice(v))) + .collect(); repo.commits .write() .map_err(|_| OperationalError::LockPoisoned)? @@ -232,10 +270,41 @@ impl super::PersistentKeyValueStore for InMemoryKeyValueStore { } fn checkout_from_path( - _source_path: &std::path::Path, + source_path: &std::path::Path, + // The in-memory store keeps no working copy on disk _working_path: tempfile::TempDir, ) -> Result { - unimplemented!("In-memory store cannot check out from disk") + let store_file = source_path.join(STORE_FILE); + if !store_file.exists() { + return Err(OperationalError::CommitNotFound); + } + + let file = std::fs::File::open(&store_file) + .map_err(|error| OperationalError::FileReadFailed { error })?; + let mut reader = std::io::BufReader::new(file); + let mut bytes = Vec::new(); + reader + .read_to_end(&mut bytes) + .map_err(|error| OperationalError::FileReadFailed { error })?; + + // Fully deserialise the snapshot + let snapshot = + rkyv::from_bytes::(&bytes).map_err(|error| { + OperationalError::FileReadFailed { + error: std::io::Error::other(error), + } + })?; + + Ok(Self { + blobs: RwLock::new(snapshot.blobs), + values: RwLock::new( + snapshot + .values + .into_iter() + .map(|(k, v)| (k, BytesMut::from(v.as_ref()))) + .collect(), + ), + }) } fn checkout( @@ -249,11 +318,70 @@ impl super::PersistentKeyValueStore for InMemoryKeyValueStore { let snapshot = commits.get(id).ok_or(OperationalError::CommitNotFound)?; Ok(Self { blobs: RwLock::new(snapshot.blobs.clone()), - values: RwLock::new(snapshot.values.clone()), + values: RwLock::new( + snapshot + .values + .iter() + .map(|(k, v)| (k.clone(), BytesMut::from(v.as_ref()))) + .collect(), + ), }) } } +#[cfg(all(test, test_utils))] +mod tests { + use octez_riscv_test_utils::TestableTmpdir; + + use super::*; + use crate::storage::PersistentKeyValueStore; + + // Test for the commit to and checkout from path implementations for + // `InMemoryKeyValueStore`, which are themselves only used in tests + #[test] + fn test_commit_to_path_checkout_roundtrip() { + let store = InMemoryKeyValueStore::default(); + + store + .blob_set(b"blob-key", b"blob-data") + .expect("Should be able to set a blob"); + store + .set(b"/key/a", b"value-a") + .expect("Should be able to set a value"); + store + .set(b"/key/b", b"value-b") + .expect("Should be able to set another value"); + store + .write(b"/key/b", 5, b"b-amended") + .expect("Should be able to write at an offset"); + store + .set(b"/key/c", b"value-c") + .expect("Should be able to set a third value"); + store + .delete(b"/key/c") + .expect("Should be able to delete a value"); + + let commit_dir = TestableTmpdir::new(); + store + .commit_to_path(commit_dir.path()) + .expect("Should be able to commit to a path"); + + let working_path = + tempfile::TempDir::new().expect("Should be able to create a working dir"); + let restored = InMemoryKeyValueStore::checkout_from_path(commit_dir.path(), working_path) + .expect("Should be able to checkout from a path"); + + assert_eq!( + *store.blobs.read().expect("Lock should not be poisoned"), + *restored.blobs.read().expect("Lock should not be poisoned"), + ); + assert_eq!( + *store.values.read().expect("Lock should not be poisoned"), + *restored.values.read().expect("Lock should not be poisoned"), + ); + } +} + #[cfg(test_utils)] impl crate::repo::RegistryRepo for InMemoryRepo { fn read_registry_commit( diff --git a/durable-storage/src/test_helpers.rs b/durable-storage/src/test_helpers.rs index f161687052..0b93281103 100644 --- a/durable-storage/src/test_helpers.rs +++ b/durable-storage/src/test_helpers.rs @@ -14,14 +14,14 @@ use std::collections::HashMap; use bytes::Bytes; use octez_riscv_data::hash::Hash; -#[cfg(test)] +#[cfg(any(test, rocksdb_test_utils))] use octez_riscv_data::merkle_proof::proof_tree::MerkleProof; use octez_riscv_data::mode::Normal; -#[cfg(test)] +#[cfg(any(test, rocksdb_test_utils))] use octez_riscv_data::mode::ProvableExt; -#[cfg(test)] +#[cfg(any(test, rocksdb_test_utils))] use octez_riscv_data::mode::Verify; -#[cfg(test)] +#[cfg(any(test, rocksdb_test_utils))] use octez_riscv_data::serialisation::serialise; use proptest::prelude::*; use proptest::sample::Index; @@ -30,17 +30,17 @@ use tokio::runtime::Handle; use crate::commit::CommitId; use crate::database::Database; -#[cfg(test)] +#[cfg(any(test, rocksdb_test_utils))] use crate::database::DatabaseMode; #[cfg(test)] use crate::database::Trace; -#[cfg(test)] +#[cfg(any(test, rocksdb_test_utils))] use crate::database::TracedDatabase; use crate::errors::Error; use crate::errors::OperationalError; use crate::key::KEY_MAX_SIZE; use crate::key::Key; -#[cfg(test)] +#[cfg(any(test, rocksdb_test_utils))] use crate::merkle_worker::BackgroundKeyValueStore; use crate::merkle_worker::BackgroundPersistentKeyValueStore; use crate::registry::Registry; @@ -175,12 +175,12 @@ pub fn make_registry_operations( .collect() } -fn key_strategy() -> impl Strategy { +pub(crate) fn key_strategy() -> impl Strategy { proptest::collection::vec(any::(), 1usize..=KEY_MAX_SIZE) .prop_map(|bytes| Key::new(&bytes).expect("The size is less than KEY_MAX_SIZE")) } -fn value_strategy() -> impl Strategy { +pub(crate) fn value_strategy() -> impl Strategy { // Bias towards lengths that fit within `MAX_FILE_CHUNK_SIZE` so most // sampled operations exercise the success path, while also producing // some oversized values. @@ -339,6 +339,40 @@ pub fn registry_operations_strategy( }) } +/// A reference model of the key-value store of a [`Database`] +pub(crate) trait DatabaseReferenceModel { + /// The modelled key-value store + fn data(&self) -> &HashMap; + + /// Update the model to reflect a successfully applied `operation` + fn apply(&mut self, operation: &DatabaseOperation); + + /// The value resulting from a successful `Write`, or `None` if it should fail + fn write_outcome(&self, key: &Key, offset: usize, data: &Bytes) -> Option { + match self.data().get(key) { + Some(existing) => { + if offset > existing.len() + || offset.checked_add(data.len()).is_none() + || data.len() > MAX_FILE_CHUNK_SIZE + { + None + } else { + let mut new_value = existing.clone(); + update_value(&mut new_value, offset, data.clone()); + Some(new_value) + } + } + None => { + if offset > 0 || data.len() > MAX_FILE_CHUNK_SIZE { + None + } else { + Some(data.clone()) + } + } + } + } +} + #[derive(Clone, Debug, Default)] struct DatabaseModel { data: HashMap, @@ -346,6 +380,54 @@ struct DatabaseModel { ambiguous_hash: bool, } +impl DatabaseModel { + /// Record an observed root hash, asserting it is consistent with the + /// previously recorded one: equal hashes must correspond to equal contents + /// and vice versa, unless a deletion made the hash ambiguous. + fn observe_hash(&mut self, new_digest: Hash) { + if let (Some((old_digest, old_map)), false) = (&self.last, &self.ambiguous_hash) { + assert_eq!(new_digest == *old_digest, self.data == *old_map); + } + self.last = Some((new_digest, self.data.clone())); + } +} + +impl DatabaseReferenceModel for DatabaseModel { + fn data(&self) -> &HashMap { + &self.data + } + + fn apply(&mut self, operation: &DatabaseOperation) { + match operation { + DatabaseOperation::Set(key, data) => { + if data.len() <= MAX_FILE_CHUNK_SIZE { + self.data.insert(key.clone(), data.clone()); + } + } + DatabaseOperation::Write(key, offset, data) => { + if let Some(new_value) = self.write_outcome(key, *offset, data) { + self.data.insert(key.clone(), new_value); + } + } + DatabaseOperation::Delete(key) => { + // The hash of the `Database` can differ even if the key-value + // pairs stored are the same, because deletion and reinsertion + // can cause the shape of the AVL tree to change. + if self.data.remove(key).is_some() { + self.ambiguous_hash = true; + } + } + DatabaseOperation::Read(..) + | DatabaseOperation::Exists(..) + | DatabaseOperation::ValueLength(..) + | DatabaseOperation::Hash + | DatabaseOperation::Commit + | DatabaseOperation::Checkout + | DatabaseOperation::CommitCheckoutRoundtrip => {} + } + } +} + fn grow_registry(registry: &mut Registry, registry_model: &mut Vec) where KV: BackgroundPersistentKeyValueStore, @@ -387,7 +469,7 @@ where } } -fn update_value(value: &mut Bytes, offset: usize, bytes: Bytes) { +pub(crate) fn update_value(value: &mut Bytes, offset: usize, bytes: Bytes) { let mut new_value: Vec = value.clone().into(); let overwrite_len = std::cmp::min(bytes.len(), new_value.len().saturating_sub(offset)); if overwrite_len > 0 { @@ -402,7 +484,7 @@ fn update_value(value: &mut Bytes, offset: usize, bytes: Bytes) { /// Abstracts the interface of [`Database`] so [`apply_database_operation`] /// can be used in both [`Registry`] (via [`Database`] references) and /// the `Database` tests which use [`TracedDatabase`] to capture a trace. -trait DatabaseOps: Sized { +pub(crate) trait DatabaseOps: Sized { fn set(&mut self, key: Key, data: Bytes) -> Result<(), Error>; fn write(&mut self, key: Key, offset: usize, data: Bytes) -> Result; @@ -485,7 +567,7 @@ impl DatabaseOps for Database DatabaseOps for TracedDatabase { fn set(&mut self, key: Key, data: Bytes) -> Result<(), Error> { TracedDatabase::set(self, key, data) @@ -533,20 +615,25 @@ impl DatabaseOps for TracedDatabase( +/// Apply a single operation to `database` and assert its observable +/// result matches that of applying it to `model`. The model itself is +/// not mutated. Callers need to advance it separately via +/// [`DatabaseReferenceModel::apply`]. This is in order to allow callers to +/// check and apply the same operation to multiple databases at the same time. +/// Commits and checkouts are not handled. +pub(crate) fn check_and_apply_value_operation( database: &mut D, - model: &mut DatabaseModel, - op: DatabaseOperation, - handle: &Handle, - repo: &KV::Repo, - checkout_candidates: &mut HashMap, -) where + model: &M, + op: &DatabaseOperation, +) -> Option +where KV: BackgroundPersistentKeyValueStore, D: DatabaseOps, + M: DatabaseReferenceModel, { match op { DatabaseOperation::Set(key, bytes) => { - let data = Bytes::copy_from_slice(&bytes); + let data = Bytes::copy_from_slice(bytes); let result = database.set(key.clone(), data); if bytes.len() <= MAX_FILE_CHUNK_SIZE { @@ -555,32 +642,15 @@ fn apply_database_operation( "Set should have succeeded but failed: {:?}", result.err() ); - - model.data.insert(key, bytes.clone()); } else { assert!(result.is_err(), "Set should have failed but succeeded"); } } DatabaseOperation::Write(key, offset, bytes) => { - let data = Bytes::copy_from_slice(&bytes); - let result = database.write(key.clone(), offset, data); + let data = Bytes::copy_from_slice(bytes); + let result = database.write(key.clone(), *offset, data); - let should_succeed = if let Some(map_value) = model.data.get_mut(&key) { - if offset > map_value.len() - || offset.checked_add(bytes.len()).is_none() - || bytes.len() > MAX_FILE_CHUNK_SIZE - { - false - } else { - update_value(map_value, offset, bytes); - true - } - } else if offset > 0 || bytes.len() > MAX_FILE_CHUNK_SIZE { - false - } else { - model.data.insert(key, bytes); - true - }; + let should_succeed = model.write_outcome(key, *offset, bytes).is_some(); if should_succeed { assert!( @@ -593,28 +663,28 @@ fn apply_database_operation( } } DatabaseOperation::Read(key, offset, len) => { - let mut database_value = vec![0; len]; + let mut database_value = vec![0; *len]; let mut cursor = 0; - let mut result = database.read(&key, offset + cursor, &mut database_value[cursor..]); + let mut result = database.read(key, offset + cursor, &mut database_value[cursor..]); while let Ok(read) = result { if read == 0 { break; } cursor += read; - result = database.read(&key, offset + cursor, &mut database_value[cursor..]) + result = database.read(key, offset + cursor, &mut database_value[cursor..]) } - if let Some(map_value) = model.data.get(&key) { - if offset > map_value.len() || len > MAX_FILE_CHUNK_SIZE { + if let Some(map_value) = model.data().get(key) { + if *offset > map_value.len() || *len > MAX_FILE_CHUNK_SIZE { assert!(result.is_err()); } else { - let expected_len = std::cmp::min(len, map_value.len() - offset); + let expected_len = std::cmp::min(*len, map_value.len() - offset); assert!(cursor >= expected_len); assert_eq!( &database_value[..expected_len], - &map_value[offset..offset + expected_len] + &map_value[*offset..*offset + expected_len] ); } } else { @@ -622,24 +692,18 @@ fn apply_database_operation( } } DatabaseOperation::Delete(key) => { - // The hash of the `Database` can differ even if the key-value pairs stored are - // the same, because deletion and reinsertion can cause the shape of the AVL - // tree to change. - let deleted = model.data.remove(&key).is_some(); - if deleted { - model.ambiguous_hash = true; - } - - database.delete(key).expect("Deleting should succeed"); + database + .delete(key.clone()) + .expect("Deleting should succeed"); } DatabaseOperation::Exists(key) => { - let in_database = database.exists(&key).expect("Writing should succeed"); - let in_map = model.data.contains_key(&key); + let in_database = database.exists(key).expect("Writing should succeed"); + let in_map = model.data().contains_key(key); assert_eq!(in_database, in_map); } DatabaseOperation::ValueLength(key) => { - let database_length = database.value_length(&key); - let map_value = model.data.get(&key); + let database_length = database.value_length(key); + let map_value = model.data().get(key); match (database_length, map_value) { (Ok(database_length), Some(map_value)) => { @@ -650,14 +714,34 @@ fn apply_database_operation( } } DatabaseOperation::Hash => { - let new_digest = database.hash().expect("Hash should succeed"); - - if let (Some((old_digest, old_map)), false) = (&model.last, &model.ambiguous_hash) { - assert_eq!(new_digest == *old_digest, model.data == *old_map); - } + return Some(database.hash().expect("Hash should succeed")); + } + DatabaseOperation::Commit + | DatabaseOperation::Checkout + | DatabaseOperation::CommitCheckoutRoundtrip => { + unimplemented!("commit and checkout operations are not handled") + } + } - model.last = Some((new_digest, model.data.clone())); + None +} +fn apply_database_operation( + database: &mut D, + model: &mut DatabaseModel, + op: &DatabaseOperation, + handle: &Handle, + repo: &KV::Repo, + checkout_candidates: &mut HashMap, +) where + KV: BackgroundPersistentKeyValueStore, + D: DatabaseOps, +{ + match op { + DatabaseOperation::Hash => { + let new_digest = check_and_apply_value_operation(database, model, op) + .expect("Hash operations produce a digest"); + model.observe_hash(new_digest); checkout_candidates.entry(new_digest).or_insert(false); } DatabaseOperation::Commit => { @@ -688,6 +772,10 @@ fn apply_database_operation( // does not see an unexpected success against this hash. checkout_candidates.insert(*commit_id.as_hash(), true); } + op => { + check_and_apply_value_operation(database, model, op); + model.apply(op); + } } } @@ -715,17 +803,7 @@ where .hash() .expect("Hash should succeed"); - if let (Some((old_digest, old_map)), false) = ( - ®istry_model[index].last, - ®istry_model[index].ambiguous_hash, - ) { - assert_eq!( - new_digest == *old_digest, - registry_model[index].data == *old_map - ); - } - - registry_model[index].last = Some((new_digest, registry_model[index].data.clone())); + registry_model[index].observe_hash(new_digest); checkout_candidates .entry(Hash::from_foldable(®istry)) @@ -762,7 +840,7 @@ where .database_mut(index) .expect("The index is in bounds"), &mut registry_model[index], - op, + &op, &handle, &checkout_repo, &mut checkout_candidates, @@ -855,7 +933,7 @@ where apply_database_operation::( &mut database, &mut model, - op, + &op, handle, repo, &mut checkout_candidates, @@ -868,7 +946,7 @@ where /// Apply `operation` to a traced `database`, recording its [`TraceEntry`]. /// /// Returns `true` if the operation was a provable step. -#[cfg(test)] +#[cfg(any(test, rocksdb_test_utils))] fn apply_step( database: &mut TracedDatabase, operation: &DatabaseOperation, @@ -912,12 +990,13 @@ fn apply_step( Ok(true) } -/// Generate and verify a proof for a single [`DatabaseOperation`] applied to `database` -#[cfg(test)] -fn prove_and_verify_operation( +/// Generate and verify a proof for a single [`DatabaseOperation`] applied to `database`. +/// Returns the serialised proof or `None` if `operation` is not a provable step. +#[cfg(any(test, rocksdb_test_utils))] +pub(crate) fn prove_and_verify_operation( database: &TracedDatabase, operation: &DatabaseOperation, -) { +) -> Option> { use octez_riscv_data::hash::PartialHash; let pre_root_hash = Hash::from_foldable(database); @@ -929,7 +1008,7 @@ fn prove_and_verify_operation( // Nothing to record or compare if the step was not provable if !apply_step(&mut prover, operation).expect("applying a step should succeed") { - return; + return None; } let post_root_hash = Hash::from_foldable(&prover); @@ -948,14 +1027,14 @@ fn prove_and_verify_operation( assert_eq!( PartialHash::from_foldable(None, &verify_db) .to_hash() - .unwrap(), + .expect("hashing the Verify database should succeed"), pre_root_hash, "the proof must reconstruct the pre-operation root hash" ); apply_step(&mut verify_db, operation).expect("applying a step should succeed"); let verify_post_root_hash = PartialHash::from_foldable(None, &verify_db) .to_hash() - .unwrap(); + .expect("hashing the Verify database should succeed"); let verify_step_trace = verify_db.into_trace(); assert_eq!( @@ -967,7 +1046,7 @@ fn prove_and_verify_operation( "Prove- and Verify-mode root hashes should match" ); - database.record_proof(operation.clone(), proof_bytes) + Some(proof_bytes) } /// Like [`run_database_operations`], but additionally generates and verifies a proof for @@ -995,13 +1074,16 @@ where operations.push(DatabaseOperation::Hash); for operation in operations { - // Provable operations are proven over their pre-operation state, so prove before applying. - prove_and_verify_operation::(&database, &operation); + // Provable operations are proven over their pre-operation state, so prove before applying, + // recording the serialised proof in the database's trace. + if let Some(proof_bytes) = prove_and_verify_operation(&database, &operation) { + database.record_proof(operation.clone(), proof_bytes) + } apply_database_operation::( &mut database, &mut model, - operation, + &operation, handle, repo, &mut checkout_candidates,