Merge 'simulator: allow failure assertions' from Alperen Keleş

- add creating the same table two times to the list of checked
properties as a failure property example

Reviewed-by: Pekka Enberg <[email protected]>
Reviewed-by: Jussi Saurio <[email protected]>
Reviewed-by: Pere Diaz Bou <[email protected]>

Closes #554

Author: penberg

simulator/generation/mod.rs

@@ -1,5 +1,7 @@
+use std::{iter::Sum, ops::SubAssign};
+
 use anarchist_readable_name_generator_lib::readable_name_custom;
-use rand::Rng;
+use rand::{distributions::uniform::SampleUniform, Rng};
 
 pub mod plan;
 pub mod query;
@@ -13,12 +15,17 @@ pub trait ArbitraryFrom<T> {
     fn arbitrary_from<R: Rng>(rng: &mut R, t: &T) -> Self;
 }
 
-pub(crate) fn frequency<'a, T, R: rand::Rng>(
-    choices: Vec<(usize, Box<dyn FnOnce(&mut R) -> T + 'a>)>,
+pub(crate) fn frequency<
+    'a,
+    T,
+    R: rand::Rng,
+    N: Sum + PartialOrd + Copy + Default + SampleUniform + SubAssign,
+>(
+    choices: Vec<(N, Box<dyn FnOnce(&mut R) -> T + 'a>)>,
     rng: &mut R,
 ) -> T {
-    let total = choices.iter().map(|(weight, _)| weight).sum::<usize>();
-    let mut choice = rng.gen_range(0..total);
+    let total = choices.iter().map(|(weight, _)| *weight).sum::<N>();
+    let mut choice = rng.gen_range(N::default()..total);
 
     for (weight, f) in choices {
         if choice < weight {
@@ -38,7 +45,7 @@ pub(crate) fn one_of<'a, T, R: rand::Rng>(
     choices[index](rng)
 }
 
-pub(crate) fn pick<'a, T, R: rand::Rng>(choices: &'a Vec<T>, rng: &mut R) -> &'a T {
+pub(crate) fn pick<'a, T, R: rand::Rng>(choices: &'a [T], rng: &mut R) -> &'a T {
     let index = rng.gen_range(0..choices.len());
     &choices[index]
 }

simulator/generation/plan.rs

@@ -16,7 +16,7 @@ use crate::generation::{frequency, Arbitrary, ArbitraryFrom};
 
 use super::{pick, pick_index};
 
-pub(crate) type ResultSet = Vec<Vec<Value>>;
+pub(crate) type ResultSet = Result<Vec<Vec<Value>>>;
 
 pub(crate) struct InteractionPlan {
     pub(crate) plan: Vec<Interaction>,
@@ -45,14 +45,15 @@ pub(crate) struct InteractionStats {
     pub(crate) read_count: usize,
     pub(crate) write_count: usize,
     pub(crate) delete_count: usize,
+    pub(crate) create_count: usize,
 }
 
 impl Display for InteractionStats {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(
             f,
-            "Read: {}, Write: {}, Delete: {}",
-            self.read_count, self.write_count, self.delete_count
+            "Read: {}, Write: {}, Delete: {}, Create: {}",
+            self.read_count, self.write_count, self.delete_count, self.create_count
         )
     }
 }
@@ -100,7 +101,9 @@ impl Interactions {
             match interaction {
                 Interaction::Query(query) => match query {
                     Query::Create(create) => {
-                        env.tables.push(create.table.clone());
+                        if !env.tables.iter().any(|t| t.name == create.table.name) {
+                            env.tables.push(create.table.clone());
+                        }
                     }
                     Query::Insert(insert) => {
                         let table = env
@@ -137,14 +140,15 @@ impl InteractionPlan {
         let mut read = 0;
         let mut write = 0;
         let mut delete = 0;
+        let mut create = 0;
 
         for interaction in &self.plan {
             match interaction {
                 Interaction::Query(query) => match query {
                     Query::Select(_) => read += 1,
                     Query::Insert(_) => write += 1,
                     Query::Delete(_) => delete += 1,
-                    Query::Create(_) => {}
+                    Query::Create(_) => create += 1,
                 },
                 Interaction::Assertion(_) => {}
                 Interaction::Fault(_) => {}
@@ -155,6 +159,7 @@ impl InteractionPlan {
             read_count: read,
             write_count: write,
             delete_count: delete,
+            create_count: create,
         }
     }
 }
@@ -172,7 +177,7 @@ impl ArbitraryFrom<SimulatorEnv> for InteractionPlan {
             rng: ChaCha8Rng::seed_from_u64(rng.next_u64()),
         };
 
-        let num_interactions = rng.gen_range(0..env.opts.max_interactions);
+        let num_interactions = env.opts.max_interactions;
 
         // First create at least one table
         let create_query = Create::arbitrary(rng);
@@ -197,7 +202,7 @@ impl ArbitraryFrom<SimulatorEnv> for InteractionPlan {
 }
 
 impl Interaction {
-    pub(crate) fn execute_query(&self, conn: &mut Rc<Connection>) -> Result<ResultSet> {
+    pub(crate) fn execute_query(&self, conn: &mut Rc<Connection>) -> ResultSet {
         match self {
             Self::Query(query) => {
                 let query_str = query.to_string();
@@ -342,13 +347,40 @@ fn property_insert_select<R: rand::Rng>(rng: &mut R, env: &SimulatorEnv) -> Inte
         ),
         func: Box::new(move |stack: &Vec<ResultSet>| {
             let rows = stack.last().unwrap();
-            rows.iter().any(|r| r == &row)
+            match rows {
+                Ok(rows) => rows.iter().any(|r| r == &row),
+                Err(_) => false,
+            }
         }),
     });
 
     Interactions(vec![insert_query, select_query, assertion])
 }
 
+fn property_double_create_failure<R: rand::Rng>(rng: &mut R, _env: &SimulatorEnv) -> Interactions {
+    let create_query = Create::arbitrary(rng);
+    let table_name = create_query.table.name.clone();
+    let cq1 = Interaction::Query(Query::Create(create_query.clone()));
+    let cq2 = Interaction::Query(Query::Create(create_query.clone()));
+
+    let assertion = Interaction::Assertion(Assertion {
+        message:
+            "creating two tables with the name should result in a failure for the second query"
+                .to_string(),
+        func: Box::new(move |stack: &Vec<ResultSet>| {
+            let last = stack.last().unwrap();
+            match last {
+                Ok(_) => false,
+                Err(e) => e
+                    .to_string()
+                    .contains(&format!("Table {table_name} already exists")),
+            }
+        }),
+    });
+
+    Interactions(vec![cq1, cq2, assertion])
+}
+
 fn create_table<R: rand::Rng>(rng: &mut R, _env: &SimulatorEnv) -> Interactions {
     let create_query = Interaction::Query(Query::Create(Create::arbitrary(rng)));
     Interactions(vec![create_query])
@@ -375,17 +407,21 @@ impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Interactions {
         rng: &mut R,
         (env, stats): &(&SimulatorEnv, InteractionStats),
     ) -> Self {
-        let remaining_read =
-            ((((env.opts.max_interactions * env.opts.read_percent) as f64) / 100.0) as usize)
-                .saturating_sub(stats.read_count);
-        let remaining_write = ((((env.opts.max_interactions * env.opts.write_percent) as f64)
-            / 100.0) as usize)
-            .saturating_sub(stats.write_count);
+        let remaining_read = ((env.opts.max_interactions as f64 * env.opts.read_percent / 100.0)
+            - (stats.read_count as f64))
+            .max(0.0);
+        let remaining_write = ((env.opts.max_interactions as f64 * env.opts.write_percent / 100.0)
+            - (stats.write_count as f64))
+            .max(0.0);
+        let remaining_create = ((env.opts.max_interactions as f64 * env.opts.create_percent
+            / 100.0)
+            - (stats.create_count as f64))
+            .max(0.0);
 
         frequency(
             vec![
                 (
-                    usize::min(remaining_read, remaining_write),
+                    f64::min(remaining_read, remaining_write),
                     Box::new(|rng: &mut R| property_insert_select(rng, env)),
                 ),
                 (
@@ -397,10 +433,14 @@ impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Interactions {
                     Box::new(|rng: &mut R| random_write(rng, env)),
                 ),
                 (
-                    remaining_write / 10,
+                    remaining_create,
                     Box::new(|rng: &mut R| create_table(rng, env)),
                 ),
-                (1, Box::new(|rng: &mut R| random_fault(rng, env))),
+                (1.0, Box::new(|rng: &mut R| random_fault(rng, env))),
+                (
+                    remaining_create / 2.0,
+                    Box::new(|rng: &mut R| property_double_create_failure(rng, env)),
+                ),
             ],
             rng,
         )

simulator/generation/table.rs

@@ -41,11 +41,7 @@ impl Arbitrary for Column {
 
 impl Arbitrary for ColumnType {
     fn arbitrary<R: Rng>(rng: &mut R) -> Self {
-        pick(
-            &vec![Self::Integer, Self::Float, Self::Text, Self::Blob],
-            rng,
-        )
-        .to_owned()
+        pick(&[Self::Integer, Self::Float, Self::Text, Self::Blob], rng).to_owned()
     }
 }
 

simulator/main.rs

@@ -121,6 +121,15 @@ fn main() {
         // Move the old database and plan file back
         std::fs::rename(&old_db_path, &db_path).unwrap();
         std::fs::rename(&old_plan_path, &plan_path).unwrap();
+    } else if let Ok(result) = result {
+        match result {
+            Ok(_) => {
+                log::info!("simulation completed successfully");
+            }
+            Err(e) => {
+                log::error!("simulation failed: {:?}", e);
+            }
+        }
     }
     // Print the seed, the locations of the database and the plan file at the end again for easily accessing them.
     println!("database path: {:?}", db_path);
@@ -136,30 +145,50 @@ fn run_simulation(
 ) -> Result<()> {
     let mut rng = ChaCha8Rng::seed_from_u64(seed);
 
-    let (read_percent, write_percent, delete_percent) = {
-        let mut remaining = 100;
-        let read_percent = rng.gen_range(0..=remaining);
+    let (create_percent, read_percent, write_percent, delete_percent) = {
+        let mut remaining = 100.0;
+        let read_percent = rng.gen_range(0.0..=remaining);
         remaining -= read_percent;
-        let write_percent = rng.gen_range(0..=remaining);
+        let write_percent = rng.gen_range(0.0..=remaining);
         remaining -= write_percent;
         let delete_percent = remaining;
-        (read_percent, write_percent, delete_percent)
+
+        let create_percent = write_percent / 10.0;
+        let write_percent = write_percent - create_percent;
+
+        (create_percent, read_percent, write_percent, delete_percent)
     };
 
+    if cli_opts.minimum_size < 1 {
+        return Err(limbo_core::LimboError::InternalError(
+            "minimum size must be at least 1".to_string(),
+        ));
+    }
+
     if cli_opts.maximum_size < 1 {
-        panic!("maximum size must be at least 1");
+        return Err(limbo_core::LimboError::InternalError(
+            "maximum size must be at least 1".to_string(),
+        ));
+    }
+
+    if cli_opts.maximum_size < cli_opts.minimum_size {
+        return Err(limbo_core::LimboError::InternalError(
+            "maximum size must be greater than or equal to minimum size".to_string(),
+        ));
     }
 
     let opts = SimulatorOpts {
-        ticks: rng.gen_range(1..=cli_opts.maximum_size),
+        ticks: rng.gen_range(cli_opts.minimum_size..=cli_opts.maximum_size),
         max_connections: 1, // TODO: for now let's use one connection as we didn't implement
         // correct transactions procesing
         max_tables: rng.gen_range(0..128),
+        create_percent,
         read_percent,
         write_percent,
         delete_percent,
         page_size: 4096, // TODO: randomize this too
-        max_interactions: rng.gen_range(1..=cli_opts.maximum_size),
+        max_interactions: rng.gen_range(cli_opts.minimum_size..=cli_opts.maximum_size),
+        max_time_simulation: cli_opts.maximum_time,
     };
     let io = Arc::new(SimulatorIO::new(seed, opts.page_size).unwrap());
 
@@ -207,12 +236,19 @@ fn run_simulation(
 }
 
 fn execute_plans(env: &mut SimulatorEnv, plans: &mut [InteractionPlan]) -> Result<()> {
+    let now = std::time::Instant::now();
     // todo: add history here by recording which interaction was executed at which tick
     for _tick in 0..env.opts.ticks {
         // Pick the connection to interact with
         let connection_index = pick_index(env.connections.len(), &mut env.rng);
         // Execute the interaction for the selected connection
         execute_plan(env, connection_index, plans)?;
+        // Check if the maximum time for the simulation has been reached
+        if now.elapsed().as_secs() >= env.opts.max_time_simulation as u64 {
+            return Err(limbo_core::LimboError::InternalError(
+                "maximum time for simulation reached".into(),
+            ));
+        }
     }
 
     Ok(())
@@ -266,7 +302,7 @@ fn execute_interaction(
             };
 
             log::debug!("{}", interaction);
-            let results = interaction.execute_query(conn)?;
+            let results = interaction.execute_query(conn);
             log::debug!("{:?}", results);
             stack.push(results);
         }

simulator/model/query.rs

@@ -61,7 +61,7 @@ pub(crate) enum Query {
     Delete(Delete),
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub(crate) struct Create {
     pub(crate) table: Table,
 }

simulator/runner/cli.rs

@@ -15,10 +15,24 @@ pub struct SimulatorCLI {
     )]
     pub doublecheck: bool,
     #[clap(
-        short,
+        short = 'n',
         long,
         help = "change the maximum size of the randomly generated sequence of interactions",
         default_value_t = 1024
     )]
     pub maximum_size: usize,
+    #[clap(
+        short = 'k',
+        long,
+        help = "change the minimum size of the randomly generated sequence of interactions",
+        default_value_t = 1
+    )]
+    pub minimum_size: usize,
+    #[clap(
+        short = 't',
+        long,
+        help = "change the maximum time of the simulation(in seconds)",
+        default_value_t = 60 * 60 // default to 1 hour
+    )]
+    pub maximum_time: usize,
 }

simulator/runner/env.rs

@@ -30,9 +30,11 @@ pub(crate) struct SimulatorOpts {
     pub(crate) max_tables: usize,
     // this next options are the distribution of workload where read_percent + write_percent +
     // delete_percent == 100%
-    pub(crate) read_percent: usize,
-    pub(crate) write_percent: usize,
-    pub(crate) delete_percent: usize,
+    pub(crate) create_percent: f64,
+    pub(crate) read_percent: f64,
+    pub(crate) write_percent: f64,
+    pub(crate) delete_percent: f64,
     pub(crate) max_interactions: usize,
     pub(crate) page_size: usize,
+    pub(crate) max_time_simulation: usize,
 }