Benchmark from blueprints #63

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged

florishafkenscheid merged 16 commits into florishafkenscheid:master from jgoldshlag:benchmark-from-blueprints

Oct 14, 2025

Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Cargo.toml

-Original file line number
+Diff line change
@@ -1,6 +1,6 @@
     [package]
     name = "belt"
-    version = "3.3.1"
+    version = "3.4.0"
     edition = "2024"
     description = "A fast, cross-platform Factorio benchmarking tool"
     license = "MIT"
@@ Expand Down @@

src/analyze/charts.rs

-Original file line number
+Diff line change
@@ Expand Up / @@ -5,13 +5,13 @@ @@
     use std::collections::HashMap;
     use charming::{
-        Chart, ImageRenderer,
         component::{Axis, Grid, Title},
         element::{
             AxisLabel, AxisType, ItemStyle, JsFunction, Label, LabelPosition, SplitArea, SplitLine,
         },
         series::{Bar, Boxplot, Line, Scatter},
         theme::Theme,
+        Chart, ImageRenderer,
     };
     use crate::{
@@ Expand Down @@

src/analyze/mod.rs

-Original file line number
+Diff line change
@@ Expand Up / @@ -2,8 +2,8 @@ pub mod charts; @@
     pub mod parser;
     use crate::{
-        Result,
         core::config::{AnalyzeConfig, GlobalConfig},
+        Result,
     };
     pub async fn run(_global_config: GlobalConfig, analyze_config: AnalyzeConfig) -> Result<()> {
@@ Expand Down @@

src/benchmark/mod.rs

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -8,16 +8,15 @@ pub mod runner;
  
    use std::{

        collections::HashMap,

        path::Path,

        sync::{Arc, atomic::AtomicBool},

        sync::{atomic::AtomicBool, Arc},

    };

    use crate::{

        benchmark::runner::VerboseData,

        core::{

            FactorioExecutor, GlobalConfig, Result,

            config::BenchmarkConfig,

            output::{CsvWriter, ResultWriter, WriteData, ensure_output_dir, report::ReportWriter},

            utils,

            output::{ensure_output_dir, report::ReportWriter, CsvWriter, ResultWriter, WriteData},

            utils, FactorioExecutor, GlobalConfig, Result,

        },

    };

src/benchmark/parser.rs

-Original file line number
+Diff line change
@@ Expand Up / @@ -5,7 +5,7 @@ use std::path::Path; @@
     use crate::core::config::BenchmarkConfig;
     use crate::core::error::BenchmarkErrorKind;
-    use crate::core::{Result, get_os_info};
+    use crate::core::{get_os_info, Result};
     /// The result of a benchmark of a single run
     #[derive(Debug, Clone, Default, Serialize, Deserialize)]
@@ Expand Down @@

src/benchmark/runner.rs

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -4,18 +4,18 @@ use indicatif::{ProgressBar, ProgressStyle};
  
    use rand::seq::SliceRandom;

    use std::collections::HashMap;

    use std::path::{Path, PathBuf};

    use std::sync::Arc;

    use std::sync::atomic::{AtomicBool, Ordering};

    use std::sync::Arc;

    use std::time::Duration;

    use tokio::time::Instant;

    use super::BenchmarkConfig;

    use crate::benchmark::parser;

    use crate::benchmark::parser::BenchmarkResult;

    use crate::core::Result;

    use crate::core::error::BenchmarkErrorKind;

    use crate::core::factorio::FactorioRunSpec;

    use crate::core::factorio::FactorioTickRunSpec;

    use crate::core::format_duration;

    use crate::core::Result;

    use crate::core::{FactorioExecutor, RunOrder};

    /// A job, indicating a single benchmark run, to be used in queues of a specific order

    @@ -254,7 +254,7 @@ impl BenchmarkRunner {
  
        ) -> Result<FactorioOutput> {

            self.factorio

                .run_for_ticks(

                    FactorioRunSpec {

                    FactorioTickRunSpec {

                        save_file,

                        ticks: self.config.ticks,

                        mods_dir: self.config.mods_dir.as_deref(),

src/blueprint/mod.rs

-Original file line number
+Diff line change
@@ -0,0 +1,50 @@
+    //! Blueprint Benchmarking module
+    //!
+    //! Contains logic for running blueprints, then uses the normal benchmark stuff to report results.
+    pub mod runner;
+    use std::{
+        path::Path,
+        sync::{atomic::AtomicBool, Arc},
+    };
+    use crate::core::{config::BlueprintConfig, output, utils, FactorioExecutor, GlobalConfig, Result};
+    /// Run all of the benchmarks, capture the logs and write the results to files.
+    pub async fn run(
+        global_config: GlobalConfig,
+        benchmark_config: BlueprintConfig,
+        running: &Arc<AtomicBool>,
+    ) -> Result<()> {
+        tracing::info!(
+            "Starting blueprint benchmark with config: {:?}",
+            benchmark_config
+        );
+        // Find the Factorio binary
+        let factorio = FactorioExecutor::discover(global_config.factorio_path)?;
+        tracing::info!(
+            "Using Factorio at: {}",
+            factorio.executable_path().display()
+        );
+        // Find the specified blueprint files
+        let blueprint_files = utils::find_blueprint_files(
+            &benchmark_config.blueprints_dir,
+            benchmark_config.pattern.as_deref(),
+        )?;
+        let output_dir = benchmark_config
+            .output
+            .as_deref()
+            .unwrap_or_else(|| Path::new("."));
+        output::ensure_output_dir(output_dir)?;
+        tracing::debug!("Output directory: {}", output_dir.display());
+        // Run the benchmarks
+        let runner = runner::BlueprintRunner::new(benchmark_config.clone(), factorio);
+        runner.run_all(blueprint_files, running).await?;
+        Ok(())
+    }

src/blueprint/runner.rs

-Original file line number
+Diff line change
@@ -0,0 +1,158 @@
+    //! Running and collecting logs of benchmarks on save file(s)
+    use std::path::PathBuf;
+    use std::sync::atomic::AtomicBool;
+    use std::sync::Arc;
+    use std::{fs, sync::atomic::Ordering};
+    use crate::core::{
+        config::BlueprintConfig,
+        error::{BenchmarkError, BenchmarkErrorKind},
+        factorio::FactorioSaveRunSpec,
+        settings::{ModSettings, ModSettingsScopeName, ModSettingsValue},
+        utils, FactorioExecutor, Result,
+    };
+    pub struct BlueprintRunner {
+        config: BlueprintConfig,
+        factorio: FactorioExecutor,
+    }
+    /// Runs the benchmarks, keeps a progress bar updated and returns results.
+    impl BlueprintRunner {
+        pub fn new(config: BlueprintConfig, factorio: FactorioExecutor) -> Self {
+            Self { config, factorio }
+        }
+        /// Run benchmarks for all blueprint files
+        pub async fn run_all(
+            &self,
+            blueprint_files: Vec<PathBuf>,
+            running: &Arc<AtomicBool>,
+        ) -> Result<()> {
+            for bp_file in &blueprint_files {
+                if !running.load(Ordering::SeqCst) {
+                    tracing::info!("Shutdown requested. Aborting remaining blueprints.");
+                    break;
+                }
+                // add prefix to bp file
+                let orig_name = bp_file.file_name().and_then(|n| n.to_str()).ok_or(
+                    BenchmarkErrorKind::InvalidBlueprintFileName {
+                        path: bp_file.to_path_buf(),
+                    },
+                )?;
+                let orig_stem = bp_file.file_stem().and_then(|s| s.to_str()).ok_or(
+                    BenchmarkErrorKind::InvalidBlueprintFileName {
+                        path: bp_file.to_path_buf(),
+                    },
+                )?;
+                // Apply optional prefix to both name and stem
+                let filestem = if let Some(prefix) = &self.config.prefix {
+                    // Compute new filename (prefix + original filename)
+                    let new_filename = format!("{prefix}{orig_name}");
+                    // Compute new stem (prefix + original stem)
+                    let new_filestem = format!("{prefix}{orig_stem}");
+                    // Rename the file on disk if not already renamed
+                    let new_path = bp_file.with_file_name(&new_filename);
+                    std::fs::rename(bp_file, &new_path)?;
+                    new_filestem
+                } else {
+                    orig_stem.to_string()
+                };
+                // inject mod settings
+                if let Some(ref mods_dir) = self.config.mods_dir.clone().or(utils::find_mod_directory())
+                {
+                    tracing::debug!("Using mods-dir: {}", mods_dir.display());
+                    let dat_file = &mods_dir.join("mod-settings.dat");
+                    let mut ms = ModSettings::load_from_file(dat_file)?;
+                    // Target tick
+                    ms.set(
+                        ModSettingsScopeName::Startup,
+                        "belt-sanitizer-target-tick",
+                        Some(ModSettingsValue::Int(self.config.buffer_ticks as i64)),
+                    );
+                    // Blueprint mode
+                    ms.set(
+                        ModSettingsScopeName::Startup,
+                        "belt-sanitizer-blueprint-mode",
+                        Some(ModSettingsValue::Bool(true)), // Always set to true
+                    );
+                    // Blueprint string
+                    let blueprint_string = fs::read_to_string(bp_file)?;
+                    ms.set(
+                        ModSettingsScopeName::Startup,
+                        "belt-sanitizer-blueprint-string",
+                        Some(ModSettingsValue::String(blueprint_string)),
+                    );
+                    // Blueprint save name
+                    ms.set(
+                        ModSettingsScopeName::Startup,
+                        "belt-sanitizer-blueprint-save-name",
+                        Some(ModSettingsValue::String(filestem.clone())),
+                    );
+                    // Blueprint count
+                    ms.set(
+                        ModSettingsScopeName::Startup,
+                        "belt-sanitizer-blueprint-count",
+                        Some(ModSettingsValue::Int(self.config.count as i64)),
+                    );
+                    // Blueprint bot count
+                    if let Some(bot_count) = self.config.bot_count {
+                        ms.set(
+                            ModSettingsScopeName::Startup,
+                            "belt-sanitizer-blueprint-bot-count",
+                            Some(ModSettingsValue::Int(bot_count as i64)),
+                        );
+                    }
+                    ms.save_to_file(dat_file)?;
+                } else {
+                    return Err(
+                        BenchmarkError::from(BenchmarkErrorKind::NoModsDirectoryFound)
+                            .with_hint(Some("Please supply a --mods-dir explicitely.")),
+                    );
+                }
+                self.factorio
+                    .run_for_save(
+                        FactorioSaveRunSpec {
+                            base_save_file: &self.config.base_save_path,
+                            new_save_name: filestem.clone(),
+                            mods_dir: self.config.mods_dir.as_deref(),
+                            headless: self.config.headless,
+                        },
+                        running,
+                    )
+                    .await?;
+                // check existance
+                if let Some(save_file) = utils::check_save_file(format!("_autosave-{}", &filestem)) {
+                    tracing::debug!("Found generated save file at: {}", save_file.display());
+                    if let Some(output_dir) = &self.config.output {
+                        std::fs::rename(&save_file, output_dir.join(format!("{}.zip", &filestem)))?;
+                        tracing::info!(
+                            "Moved generated save from: {}, to: {}",
+                            save_file.display(),
+                            output_dir.display()
+                        );
+                    }
+                } else {
+                    tracing::error!("No generated save file found.");
+                }
+            }
+            Ok(())
+        }
+    }

src/core/config.rs

-Original file line number
+Diff line change
@@ Expand Up / @@ -51,3 +51,18 @@ pub struct SanitizeConfig { @@
         pub fluids: Option<String>,
         pub headless: Option<bool>,
     }
+    /// Blueprint Benchmarking specific configuration
+    #[derive(Debug, Clone, Default)]
+    pub struct BlueprintConfig {
+        pub blueprints_dir: PathBuf,
+        pub base_save_path: PathBuf,
+        pub count: u32,
+        pub buffer_ticks: u32,
+        pub mods_dir: Option<PathBuf>,
+        pub pattern: Option<String>,
+        pub output: Option<PathBuf>,
+        pub prefix: Option<String>,
+        pub headless: Option<bool>,
+        pub bot_count: Option<u32>,
+    }

src/core/error.rs

-Original file line number
+Diff line change
@@ Expand Up / @@ -119,6 +119,18 @@ pub enum BenchmarkErrorKind { @@
         #[error("No output statistics in production statistics found")]
         NoOutputStatistics,
+        #[error("Blueprint directory does not exist: {path}")]
+        BlueprintDirectoryNotFound { path: PathBuf },
+        #[error("No blueprint files found matching pattern '{pattern}' in {directory}")]
+        NoBlueprintFilesFound { pattern: String, directory: PathBuf },
+        #[error("Invalid Blueprint file name: {path}")]
+        InvalidBlueprintFileName { path: PathBuf },
+        #[error("No mods directory found.")]
+        NoModsDirectoryFound,
     }
     /// Get a hint for the FactorioProcessFailed error, if it exists
@@ Expand Down @@

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Benchmark from blueprints #63

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!

Uh oh!