feat(cli): collision-proof --run_id auto for rehearsals/stage demos (#116)

docs/HACKATHON_DEMO.md

@@ -39,14 +39,15 @@ export NEBIUS_API_KEY="<your-token-factory-key>"
 # Optional (only needed if using default-meta / Claude-backed meta agent):
 # export ANTHROPIC_API_KEY="<your-anthropic-key>"
 
-# 3a. Warm the STAGE-SAFE run (this is what you run live; warming it now both
-#     primes the provider/venv caches and gives sia web something to show).
+# 3a. Warm the STAGE-SAFE run (what you run live; warming it primes provider/venv
+#     caches and gives sia web something to show). --run_id auto picks the next
+#     free run_<n>, so a second rehearsal never collides with an existing dir.
 cargo run --release --features llm -- run \
   --task arithmetic-mc \
   --meta-agent-profile kimi-nebius-meta \
   --target-agent-profile kimi-nebius-target \
   --max_gen 2 \
-  --run_id 0
+  --run_id auto
 
 # 3b. (Optional, research/credibility path) Pre-record a GPQA run for the
 #     offline `sia web` replay — slow (198 questions, minutes per generation),
@@ -61,6 +62,12 @@ cargo run --release --features llm -- run \
 # 4. Open browser to http://127.0.0.1:8000 and confirm the dashboard loads
 ```
 
+> Tip: `--run_id auto` scans the runs root (honoring `--runs-dir` / `SIA_RUNS_DIR`)
+> and lands in the next free `run_<n>`. The orchestrator prints
+> `Run directory: ./runs/run_<n>` up front, before any LLM work, so you always know
+> where artifacts will land. Use a fixed `--run_id N` only when you deliberately want
+> a specific directory; it still errors if that directory already exists.
+
 ---
 
 ### Live run-of-show
@@ -98,7 +105,7 @@ cargo run --release --features llm -- run \
   --meta-agent-profile kimi-nebius-meta \
   --target-agent-profile kimi-nebius-target \
   --max_gen 2 \
-  --run_id 1
+  --run_id auto
 ```
 
 > Runtime: dominated by per-generation provider latency, not the task itself (only
@@ -112,8 +119,9 @@ cargo run --release --features llm -- run \
 > command, because its 198 per-question provider calls make it minutes per
 > generation and unreliable for a timed slot.
 
-> Point at: the structured log lines as each generation starts. Then flip to the
-> browser at `http://127.0.0.1:8000`.
+> Point at: the `Run directory: ./runs/run_<n>` line printed first (so the room sees
+> where artifacts land), then the structured log lines as each generation starts.
+> Then flip to the browser at `http://127.0.0.1:8000`.
 
 > Note: `sia run` starts the SIA Studio dashboard automatically on
 > `http://127.0.0.1:8000` unless `--no-web` is passed. You do not need a separate
@@ -124,8 +132,9 @@ cargo run --release --features llm -- run \
 - The SIA Studio dashboard at `http://127.0.0.1:8000` — generations appear as they
   land, the accuracy chart climbs, and the telemetry panel shows token counts and
   wall-clock timing updating in real time.
-- Point at the `/api/runs/run_1/metrics` and `/api/runs/run_1/telemetry` endpoints
-  powering the charts (show the JSON in a second tab if you have time).
+- Point at the `/api/runs/run_<n>/metrics` and `/api/runs/run_<n>/telemetry`
+  endpoints powering the charts — substitute the `run_<n>` the orchestrator printed
+  at startup (show the JSON in a second tab if you have time).
 
 #### 2:30 – 3:00 · Safety + extensibility story
 
@@ -236,12 +245,13 @@ cargo run --release --features llm -- run \
   --meta-agent-profile kimi-nebius-meta \
   --target-agent-profile kimi-nebius-target \
   --max_gen 2 \
-  --run_id 1
+  --run_id auto
 ```
 
-Live dashboard auto-starts at `http://127.0.0.1:8000`. (GPQA — `--task gpqa
---max_gen 3` — is the research path; pre-warm it and show it via the offline
-`sia web` replay rather than running it live, since it is minutes per generation.)
+Live dashboard auto-starts at `http://127.0.0.1:8000`. `--run_id auto` keeps repeated
+rehearsals collision-free and the resolved `Run directory:` is printed at startup.
+(GPQA — `--task gpqa --max_gen 3` — is the research path; pre-warm it and show it via
+the offline `sia web` replay rather than running it live, since it is minutes per generation.)
 
 ---
 
@@ -259,9 +269,9 @@ cargo run --release -- web --runs-dir ./runs-prerecorded
 Browse to `http://127.0.0.1:8000`. All charts, telemetry, artifacts, and
 per-generation diffs are served from disk — no network, no API key needed.
 
-> What to show: open `run_0` from the warm-up, walk through gens 1–3, show the
-> accuracy chart climbing, click into `target_agent.py` for each generation to show
-> the literal code changes between generations.
+> What to show: open the warm-up run (the `run_<n>` printed at startup) and walk
+> through gens 1–3, show the accuracy chart climbing, click into `target_agent.py`
+> for each generation to show the literal code changes between generations.
 
 ---
 
@@ -305,7 +315,7 @@ formal capability allow-list, and a real-time web dashboard built into the binar
 - Feature-gated: `--features llm` for full stack; default build has no LLM deps
 
 **Live demo** (stage-safe: tiny `arithmetic-mc` task)
-`cargo run --release --features llm -- run --task arithmetic-mc --meta-agent-profile kimi-nebius-meta --target-agent-profile kimi-nebius-target --max_gen 2 --run_id 1`
+`cargo run --release --features llm -- run --task arithmetic-mc --meta-agent-profile kimi-nebius-meta --target-agent-profile kimi-nebius-target --max_gen 2 --run_id auto`
 Watch accuracy climb in the SIA Studio dashboard at `http://127.0.0.1:8000`. (GPQA is the slower research-path benchmark — pre-warm and replay it.)
 
 **Results**
@@ -386,16 +396,16 @@ agents.
 
 ### Warm run (run before you walk on stage)
 
-- [ ] **Stage-safe path:** `cargo run --release --features llm -- run --task arithmetic-mc --meta-agent-profile kimi-nebius-meta --target-agent-profile kimi-nebius-target --max_gen 2 --run_id 0` completed
-- [ ] `runs/run_0/gen_1/`, `gen_2/` directories exist with artifacts
+- [ ] **Stage-safe path:** `cargo run --release --features llm -- run --task arithmetic-mc --meta-agent-profile kimi-nebius-meta --target-agent-profile kimi-nebius-target --max_gen 2 --run_id auto` completed (note the `Run directory:` it prints)
+- [ ] `runs/run_<n>/gen_1/`, `gen_2/` directories exist with artifacts
 - [ ] **Note the measured wall-clock of the stage-safe run here on the demo machine:** ______ (confirm it fits the 3-4 minute slot)
 - [ ] (Optional, research path) Pre-record GPQA: `... --task gpqa ... --max_gen 3 --run_id 9` completed for the offline `sia web` replay — do NOT run GPQA live
 - [ ] Copy `runs/` to `runs-prerecorded/` as the Path B fallback
 
 ### Dashboard
 
 - [ ] `cargo run --release -- web` starts and `http://127.0.0.1:8000` loads in browser
-- [ ] `run_0` visible in the UI; accuracy chart renders; telemetry panel shows data
+- [ ] The warm-up `run_<n>` visible in the UI; accuracy chart renders; telemetry panel shows data
 - [ ] Dark mode confirmed
 
 ### Terminal

src/cli.rs

@@ -22,9 +22,11 @@ fn add_run_args(cmd: Command, env_config: &Config) -> Command {
     .arg(
         Arg::new("run_id")
             .long("run_id")
-            .value_parser(clap::value_parser!(i64))
             .default_value("1")
-            .help("Run ID for this experiment (default: 1)"),
+            .help(
+                "Run ID for this experiment: a positive integer, or 'auto' to pick the next \
+                 free run_<n> under the runs root (default: 1).",
+            ),
     )
     .arg(
         Arg::new("task")

src/run.rs

@@ -36,6 +36,47 @@ pub fn resolve_runs_dir(flag: Option<&str>) -> String {
     }
 }
 
+/// Resolve the `--run_id` argument to a concrete numeric run id.
+///
+/// `"auto"` (case-insensitive) scans `runs_root` for existing `run_<n>` directories
+/// and returns `max(n) + 1`, or `1` when there are none. This makes rehearsals and
+/// restarts collision-proof: each `auto` run lands in a fresh directory rather than
+/// erroring on an existing one. Any other value must parse as a positive integer and
+/// is returned as-is (preserving the historical numeric behavior, including the
+/// existing-directory collision error in `setup_run_directory`).
+pub fn resolve_run_id(arg: &str, runs_root: &str) -> SiaResult<i64> {
+    if arg.eq_ignore_ascii_case("auto") {
+        return Ok(next_free_run_id(runs_root));
+    }
+    match arg.parse::<i64>() {
+        Ok(n) if n >= 1 => Ok(n),
+        _ => Err(SiaError::new(format!(
+            "Invalid --run_id '{arg}': expected a positive integer or 'auto'"
+        ))),
+    }
+}
+
+/// Scan `runs_root` for `run_<n>` directories and return the next free id
+/// (`max(n) + 1`, or `1` if the root is missing/empty). Non-`run_<n>` entries and
+/// entries with non-numeric suffixes are ignored.
+fn next_free_run_id(runs_root: &str) -> i64 {
+    let mut max_id: i64 = 0;
+    if let Ok(entries) = std::fs::read_dir(runs_root) {
+        for entry in entries.flatten() {
+            let name = entry.file_name();
+            let name = name.to_string_lossy();
+            if let Some(suffix) = name.strip_prefix("run_") {
+                if let Ok(n) = suffix.parse::<i64>() {
+                    if n > max_id {
+                        max_id = n;
+                    }
+                }
+            }
+        }
+    }
+    max_id + 1
+}
+
 /// `sia web`: serve the runs visualizer (blocks).
 pub fn run_web(args: &ArgMatches) -> SiaResult<()> {
     let host = opt_str(args, "host").unwrap_or("127.0.0.1");
@@ -59,15 +100,23 @@ pub fn run_orchestrator(args: &ArgMatches, env_config: &Config) -> SiaResult<()>
     let max_gen = *args
         .get_one::<i64>("max_gen")
         .unwrap_or(&env_config.default_max_generations);
-    let run_id = *args.get_one::<i64>("run_id").unwrap_or(&1);
     let sandbox = opt_str(args, "sandbox")
         .unwrap_or(&env_config.sandbox_mode)
         .to_string();
 
     let (task_dir, shared_dir) =
         resolve_task_dir(opt_str(args, "task"), opt_str(args, "task_dir"))?;
 
+    // Resolve the runs root first, then resolve `--run_id` (which may be `auto`)
+    // against that same root so the auto-scan and the directory the run writes to
+    // agree (honors `--runs-dir` / `SIA_RUNS_DIR`).
     let runs_dir = resolve_runs_dir(opt_str(args, "runs_dir"));
+    let run_id = resolve_run_id(opt_str(args, "run_id").unwrap_or("1"), &runs_dir)?;
+
+    // Surface the resolved run directory up front, before any expensive LLM work,
+    // so a presenter immediately sees where artifacts will land.
+    let resolved_run_dir = RunLayout::for_run_id(run_id, &runs_dir).run_dir;
+    println!("Run directory: {resolved_run_dir}");
 
     // Live dashboard in the background unless disabled. It serves exactly the
     // directory the run writes to (`runs_dir`). The listener is bound up front so
@@ -238,3 +287,72 @@ pub fn run_orchestrator(args: &ArgMatches, env_config: &Config) -> SiaResult<()>
     );
     Ok(())
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn numeric_run_id_passes_through() {
+        // A custom runs root with no entries does not affect numeric ids.
+        let tmp = tempfile::tempdir().unwrap();
+        let root = tmp.path().to_str().unwrap();
+        assert_eq!(resolve_run_id("1", root).unwrap(), 1);
+        assert_eq!(resolve_run_id("7", root).unwrap(), 7);
+    }
+
+    #[test]
+    fn invalid_run_id_errors() {
+        let tmp = tempfile::tempdir().unwrap();
+        let root = tmp.path().to_str().unwrap();
+        assert!(resolve_run_id("0", root).is_err());
+        assert!(resolve_run_id("-1", root).is_err());
+        assert!(resolve_run_id("nope", root).is_err());
+    }
+
+    #[test]
+    fn auto_picks_one_when_root_empty_or_missing() {
+        // Missing root.
+        assert_eq!(resolve_run_id("auto", "/no/such/runs/root").unwrap(), 1);
+        // Existing-but-empty root.
+        let tmp = tempfile::tempdir().unwrap();
+        let root = tmp.path().to_str().unwrap();
+        assert_eq!(resolve_run_id("auto", root).unwrap(), 1);
+        assert_eq!(resolve_run_id("AUTO", root).unwrap(), 1);
+    }
+
+    #[test]
+    fn auto_picks_next_free_id() {
+        let tmp = tempfile::tempdir().unwrap();
+        let root = tmp.path();
+        std::fs::create_dir(root.join("run_1")).unwrap();
+        assert_eq!(
+            resolve_run_id("auto", root.to_str().unwrap()).unwrap(),
+            2,
+            "auto should pick run_2 when run_1 exists"
+        );
+
+        // Gaps and non-run entries are ignored; auto uses max + 1.
+        std::fs::create_dir(root.join("run_5")).unwrap();
+        std::fs::create_dir(root.join("not_a_run")).unwrap();
+        std::fs::create_dir(root.join("run_abc")).unwrap();
+        assert_eq!(resolve_run_id("auto", root.to_str().unwrap()).unwrap(), 6);
+    }
+
+    #[test]
+    fn auto_resolves_under_custom_runs_dir() {
+        // Two independent roots: auto resolves against whichever root it is given.
+        let a = tempfile::tempdir().unwrap();
+        let b = tempfile::tempdir().unwrap();
+        std::fs::create_dir(a.path().join("run_3")).unwrap();
+        // Root `a` has run_3 -> auto picks 4; root `b` is empty -> auto picks 1.
+        assert_eq!(
+            resolve_run_id("auto", a.path().to_str().unwrap()).unwrap(),
+            4
+        );
+        assert_eq!(
+            resolve_run_id("auto", b.path().to_str().unwrap()).unwrap(),
+            1
+        );
+    }
+}

tests/runs_dir.rs

@@ -8,7 +8,8 @@
 use std::sync::Mutex;
 
 use sia::layout::{names, RunLayout};
-use sia::run::resolve_runs_dir;
+use sia::run::{resolve_run_id, resolve_runs_dir};
+use sia::run_setup::setup_run_directory;
 
 static ENV_LOCK: Mutex<()> = Mutex::new(());
 
@@ -80,3 +81,48 @@ fn test_dashboard_root_matches_run_root() {
 
     std::env::remove_var("SIA_RUNS_DIR");
 }
+
+#[test]
+fn test_numeric_run_id_collision_still_errors() {
+    // A numeric --run_id pointing at an existing run_<id> directory must error
+    // (unchanged historical behavior), before any venv work happens.
+    let tmp = tempfile::tempdir().unwrap();
+    let runs_root = tmp.path().to_str().unwrap();
+    std::fs::create_dir(tmp.path().join("run_1")).unwrap();
+
+    let result = setup_run_directory(
+        1,
+        "/some/task",
+        "meta-model",
+        "task-model",
+        "claude",
+        3,
+        None,
+        None,
+        None,
+        runs_root,
+    );
+    assert!(
+        result.is_err(),
+        "numeric run_id on an existing run directory must error"
+    );
+    let msg = result.err().unwrap().to_string();
+    assert!(
+        msg.contains("already exists"),
+        "error should explain the collision, got: {msg}"
+    );
+}
+
+#[test]
+fn test_auto_picks_next_free_id_under_custom_runs_dir() {
+    // `auto` resolves against the provided runs root: create run_1, expect run_2.
+    let tmp = tempfile::tempdir().unwrap();
+    let runs_root = tmp.path().to_str().unwrap();
+    std::fs::create_dir(tmp.path().join("run_1")).unwrap();
+
+    let id = resolve_run_id("auto", runs_root).unwrap();
+    assert_eq!(id, 2, "auto should pick run_2 when run_1 exists");
+
+    let layout = RunLayout::for_run_id(id, runs_root);
+    assert_eq!(layout.run_dir, format!("{runs_root}/run_2"));
+}