test(dursto): add the machinery for a long-running Database test

Cargo.toml

@@ -65,6 +65,7 @@ proptest = "1.11.0"
 quanta = "0.12.6"
 rand = "0.10.1"
 range-collections = "0.4.6"
+rkyv = { version = "0.8", features = ["bytes-1"] }
 rustc-demangle = "0.1"
 rocksdb = "0.24.0"
 rustc_apfloat = "0.2.3"

durable-storage/Cargo.toml

@@ -13,6 +13,9 @@ unstable-test-utils = [
   "dep:proptest",
   "dep:serde",
   "dep:serde_with",
+  "dep:anyhow",
+  "dep:serde_json",
+  "dep:rkyv",
   "octez-riscv-data/unstable-test-utils",
 ]
 
@@ -37,10 +40,22 @@ trait-set.workspace = true
 workspace = true
 optional = true
 
+[dependencies.anyhow]
+workspace = true
+optional = true
+
+[dependencies.serde_json]
+workspace = true
+optional = true
+
 [dependencies.proptest]
 workspace = true
 optional = true
 
+[dependencies.rkyv]
+workspace = true
+optional = true
+
 [dependencies.serde]
 workspace = true
 optional = true

durable-storage/src/database.rs

@@ -623,7 +623,7 @@ mod traced_database;
 
 #[cfg(test)]
 pub(crate) use self::traced_database::Trace;
-#[cfg(test)]
+#[cfg(any(test, rocksdb_test_utils))]
 pub(crate) use self::traced_database::TracedDatabase;
 
 #[cfg(test)]

durable-storage/src/database/traced_database.rs

@@ -2,11 +2,15 @@
 //
 // SPDX-License-Identifier: MIT
 
-#![cfg(test)]
+#![cfg(any(test, rocksdb_test_utils))]
 
-//! Test-only [`Database`] wrapper which can record execution traces
+//! [`Database`] wrapper which can record execution traces.
+//!
+//! Available to unit tests and, under the `unstable-test-utils` feature, to the
+//! long-running test binary (`src/bin/database_long_test.rs`).
 
 use std::cell::RefCell;
+#[cfg(test)]
 use std::collections::HashMap;
 
 use bytes::Bytes;
@@ -17,22 +21,28 @@ use octez_riscv_data::mode::Mode;
 use octez_riscv_data::mode::Normal;
 use octez_riscv_data::mode::ProvableExt;
 use octez_riscv_data::mode::Prove;
+#[cfg(test)]
 use octez_riscv_data::mode::Verify;
 use tokio::runtime::Handle;
 
 use crate::commit::CommitId;
 use crate::database::Database;
 use crate::database::DatabaseMode;
+#[cfg(test)]
 use crate::database::VerifyImpl;
 use crate::errors::Error;
+#[cfg(test)]
 use crate::errors::InvalidArgumentError;
 use crate::errors::OperationalError;
 use crate::key::Key;
+#[cfg(test)]
 use crate::merkle_layer::new_verify_layer;
 use crate::merkle_worker::BackgroundKeyValueStore;
 use crate::merkle_worker::BackgroundPersistentKeyValueStore;
+#[cfg(test)]
 use crate::storage::KeyValueStore;
 use crate::storage::PersistentKeyValueStore;
+#[cfg(test)]
 use crate::storage::TestKeyValueStoreSetup;
 use crate::test_helpers::DatabaseOperation;
 
@@ -102,6 +112,7 @@ pub(crate) struct TracedDatabase<KV, M: Mode = Normal> {
 
 impl<KV: BackgroundKeyValueStore> TracedDatabase<KV, Normal> {
     /// Equivalent to [`Database::try_new`] which also records a [`TraceEntry`].
+    #[cfg(test)]
     pub(crate) fn try_new(handle: &Handle, repo: &KV::Repo) -> Result<Self, OperationalError> {
         Ok(TracedDatabase::from(Database::try_new(handle, repo)?))
     }
@@ -141,13 +152,15 @@ impl<KV: BackgroundKeyValueStore> TracedDatabase<KV, Normal> {
     }
 
     /// Record a [`TraceEntry::Proof`] for the proof of `step`.
+    #[cfg(test)]
     pub(crate) fn record_proof(&self, step: DatabaseOperation, proof: Vec<u8>) {
         self.trace
             .borrow_mut()
             .push(TraceEntry::Proof { step, proof });
     }
 }
 
+#[cfg(test)]
 impl<KV> TracedDatabase<KV, Verify>
 where
     KV: KeyValueStore + TestKeyValueStoreSetup,
@@ -270,6 +283,7 @@ impl<KV: BackgroundKeyValueStore, M: DatabaseMode> TracedDatabase<KV, M> {
     }
 
     /// Insert entries into the database and return the inserted key pairs
+    #[cfg(test)]
     pub(crate) fn insert_entries(
         &mut self,
         entries: Vec<(Vec<u8>, Vec<u8>)>,
@@ -286,11 +300,13 @@ impl<KV: BackgroundKeyValueStore, M: DatabaseMode> TracedDatabase<KV, M> {
     }
 
     /// Assert that a database contains the expected value for a given key.
+    #[cfg(test)]
     pub(crate) fn assert_database_value(&self, key: &Key, expected: &[u8]) {
         self.inner.assert_database_value(key, expected);
     }
 
     /// Assert that a database does not contain the given key.
+    #[cfg(test)]
     pub(crate) fn assert_traced_database_missing(&self, key: &Key) {
         assert!(matches!(
             self.read_bytes(key, 0, 0),

durable-storage/src/lib.rs

@@ -33,6 +33,10 @@ pub mod commit;
 pub mod database;
 pub mod errors;
 pub mod key;
+// The long-running test exercises the persistence backend directly, so it is
+// only available when `rocksdb` is enabled.
+#[cfg(rocksdb_test_utils)]
+pub mod long_test;
 mod merkle_layer;
 mod merkle_worker;
 pub mod persistence_layer;