diff --git a/docs/HACKATHON_DEMO.md b/docs/HACKATHON_DEMO.md index 5c6c398..9bae09a 100644 --- a/docs/HACKATHON_DEMO.md +++ b/docs/HACKATHON_DEMO.md @@ -39,14 +39,15 @@ export NEBIUS_API_KEY="" # Optional (only needed if using default-meta / Claude-backed meta agent): # export ANTHROPIC_API_KEY="" -# 3a. Warm the STAGE-SAFE run (this is what you run live; warming it now both -# primes the provider/venv caches and gives sia web something to show). +# 3a. Warm the STAGE-SAFE run (what you run live; warming it primes provider/venv +# caches and gives sia web something to show). --run_id auto picks the next +# free run_, so a second rehearsal never collides with an existing dir. cargo run --release --features llm -- run \ --task arithmetic-mc \ --meta-agent-profile kimi-nebius-meta \ --target-agent-profile kimi-nebius-target \ --max_gen 2 \ - --run_id 0 + --run_id auto # 3b. (Optional, research/credibility path) Pre-record a GPQA run for the # offline `sia web` replay — slow (198 questions, minutes per generation), @@ -61,6 +62,12 @@ cargo run --release --features llm -- run \ # 4. Open browser to http://127.0.0.1:8000 and confirm the dashboard loads ``` +> Tip: `--run_id auto` scans the runs root (honoring `--runs-dir` / `SIA_RUNS_DIR`) +> and lands in the next free `run_`. The orchestrator prints +> `Run directory: ./runs/run_` up front, before any LLM work, so you always know +> where artifacts will land. Use a fixed `--run_id N` only when you deliberately want +> a specific directory; it still errors if that directory already exists. + --- ### Live run-of-show @@ -98,7 +105,7 @@ cargo run --release --features llm -- run \ --meta-agent-profile kimi-nebius-meta \ --target-agent-profile kimi-nebius-target \ --max_gen 2 \ - --run_id 1 + --run_id auto ``` > Runtime: dominated by per-generation provider latency, not the task itself (only @@ -112,8 +119,9 @@ cargo run --release --features llm -- run \ > command, because its 198 per-question provider calls make it minutes per > generation and unreliable for a timed slot. -> Point at: the structured log lines as each generation starts. Then flip to the -> browser at `http://127.0.0.1:8000`. +> Point at: the `Run directory: ./runs/run_` line printed first (so the room sees +> where artifacts land), then the structured log lines as each generation starts. +> Then flip to the browser at `http://127.0.0.1:8000`. > Note: `sia run` starts the SIA Studio dashboard automatically on > `http://127.0.0.1:8000` unless `--no-web` is passed. You do not need a separate @@ -124,8 +132,9 @@ cargo run --release --features llm -- run \ - The SIA Studio dashboard at `http://127.0.0.1:8000` — generations appear as they land, the accuracy chart climbs, and the telemetry panel shows token counts and wall-clock timing updating in real time. -- Point at the `/api/runs/run_1/metrics` and `/api/runs/run_1/telemetry` endpoints - powering the charts (show the JSON in a second tab if you have time). +- Point at the `/api/runs/run_/metrics` and `/api/runs/run_/telemetry` + endpoints powering the charts — substitute the `run_` the orchestrator printed + at startup (show the JSON in a second tab if you have time). #### 2:30 – 3:00 · Safety + extensibility story @@ -236,12 +245,13 @@ cargo run --release --features llm -- run \ --meta-agent-profile kimi-nebius-meta \ --target-agent-profile kimi-nebius-target \ --max_gen 2 \ - --run_id 1 + --run_id auto ``` -Live dashboard auto-starts at `http://127.0.0.1:8000`. (GPQA — `--task gpqa ---max_gen 3` — is the research path; pre-warm it and show it via the offline -`sia web` replay rather than running it live, since it is minutes per generation.) +Live dashboard auto-starts at `http://127.0.0.1:8000`. `--run_id auto` keeps repeated +rehearsals collision-free and the resolved `Run directory:` is printed at startup. +(GPQA — `--task gpqa --max_gen 3` — is the research path; pre-warm it and show it via +the offline `sia web` replay rather than running it live, since it is minutes per generation.) --- @@ -259,9 +269,9 @@ cargo run --release -- web --runs-dir ./runs-prerecorded Browse to `http://127.0.0.1:8000`. All charts, telemetry, artifacts, and per-generation diffs are served from disk — no network, no API key needed. -> What to show: open `run_0` from the warm-up, walk through gens 1–3, show the -> accuracy chart climbing, click into `target_agent.py` for each generation to show -> the literal code changes between generations. +> What to show: open the warm-up run (the `run_` printed at startup) and walk +> through gens 1–3, show the accuracy chart climbing, click into `target_agent.py` +> for each generation to show the literal code changes between generations. --- @@ -305,7 +315,7 @@ formal capability allow-list, and a real-time web dashboard built into the binar - Feature-gated: `--features llm` for full stack; default build has no LLM deps **Live demo** (stage-safe: tiny `arithmetic-mc` task) -`cargo run --release --features llm -- run --task arithmetic-mc --meta-agent-profile kimi-nebius-meta --target-agent-profile kimi-nebius-target --max_gen 2 --run_id 1` +`cargo run --release --features llm -- run --task arithmetic-mc --meta-agent-profile kimi-nebius-meta --target-agent-profile kimi-nebius-target --max_gen 2 --run_id auto` Watch accuracy climb in the SIA Studio dashboard at `http://127.0.0.1:8000`. (GPQA is the slower research-path benchmark — pre-warm and replay it.) **Results** @@ -386,8 +396,8 @@ agents. ### Warm run (run before you walk on stage) -- [ ] **Stage-safe path:** `cargo run --release --features llm -- run --task arithmetic-mc --meta-agent-profile kimi-nebius-meta --target-agent-profile kimi-nebius-target --max_gen 2 --run_id 0` completed -- [ ] `runs/run_0/gen_1/`, `gen_2/` directories exist with artifacts +- [ ] **Stage-safe path:** `cargo run --release --features llm -- run --task arithmetic-mc --meta-agent-profile kimi-nebius-meta --target-agent-profile kimi-nebius-target --max_gen 2 --run_id auto` completed (note the `Run directory:` it prints) +- [ ] `runs/run_/gen_1/`, `gen_2/` directories exist with artifacts - [ ] **Note the measured wall-clock of the stage-safe run here on the demo machine:** ______ (confirm it fits the 3-4 minute slot) - [ ] (Optional, research path) Pre-record GPQA: `... --task gpqa ... --max_gen 3 --run_id 9` completed for the offline `sia web` replay — do NOT run GPQA live - [ ] Copy `runs/` to `runs-prerecorded/` as the Path B fallback @@ -395,7 +405,7 @@ agents. ### Dashboard - [ ] `cargo run --release -- web` starts and `http://127.0.0.1:8000` loads in browser -- [ ] `run_0` visible in the UI; accuracy chart renders; telemetry panel shows data +- [ ] The warm-up `run_` visible in the UI; accuracy chart renders; telemetry panel shows data - [ ] Dark mode confirmed ### Terminal diff --git a/src/cli.rs b/src/cli.rs index bae17b1..790bf2f 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -22,9 +22,11 @@ fn add_run_args(cmd: Command, env_config: &Config) -> Command { .arg( Arg::new("run_id") .long("run_id") - .value_parser(clap::value_parser!(i64)) .default_value("1") - .help("Run ID for this experiment (default: 1)"), + .help( + "Run ID for this experiment: a positive integer, or 'auto' to pick the next \ + free run_ under the runs root (default: 1).", + ), ) .arg( Arg::new("task") diff --git a/src/run.rs b/src/run.rs index 96b0588..e5d3a1b 100644 --- a/src/run.rs +++ b/src/run.rs @@ -36,6 +36,47 @@ pub fn resolve_runs_dir(flag: Option<&str>) -> String { } } +/// Resolve the `--run_id` argument to a concrete numeric run id. +/// +/// `"auto"` (case-insensitive) scans `runs_root` for existing `run_` directories +/// and returns `max(n) + 1`, or `1` when there are none. This makes rehearsals and +/// restarts collision-proof: each `auto` run lands in a fresh directory rather than +/// erroring on an existing one. Any other value must parse as a positive integer and +/// is returned as-is (preserving the historical numeric behavior, including the +/// existing-directory collision error in `setup_run_directory`). +pub fn resolve_run_id(arg: &str, runs_root: &str) -> SiaResult { + if arg.eq_ignore_ascii_case("auto") { + return Ok(next_free_run_id(runs_root)); + } + match arg.parse::() { + Ok(n) if n >= 1 => Ok(n), + _ => Err(SiaError::new(format!( + "Invalid --run_id '{arg}': expected a positive integer or 'auto'" + ))), + } +} + +/// Scan `runs_root` for `run_` directories and return the next free id +/// (`max(n) + 1`, or `1` if the root is missing/empty). Non-`run_` entries and +/// entries with non-numeric suffixes are ignored. +fn next_free_run_id(runs_root: &str) -> i64 { + let mut max_id: i64 = 0; + if let Ok(entries) = std::fs::read_dir(runs_root) { + for entry in entries.flatten() { + let name = entry.file_name(); + let name = name.to_string_lossy(); + if let Some(suffix) = name.strip_prefix("run_") { + if let Ok(n) = suffix.parse::() { + if n > max_id { + max_id = n; + } + } + } + } + } + max_id + 1 +} + /// `sia web`: serve the runs visualizer (blocks). pub fn run_web(args: &ArgMatches) -> SiaResult<()> { let host = opt_str(args, "host").unwrap_or("127.0.0.1"); @@ -59,7 +100,6 @@ pub fn run_orchestrator(args: &ArgMatches, env_config: &Config) -> SiaResult<()> let max_gen = *args .get_one::("max_gen") .unwrap_or(&env_config.default_max_generations); - let run_id = *args.get_one::("run_id").unwrap_or(&1); let sandbox = opt_str(args, "sandbox") .unwrap_or(&env_config.sandbox_mode) .to_string(); @@ -67,7 +107,16 @@ pub fn run_orchestrator(args: &ArgMatches, env_config: &Config) -> SiaResult<()> let (task_dir, shared_dir) = resolve_task_dir(opt_str(args, "task"), opt_str(args, "task_dir"))?; + // Resolve the runs root first, then resolve `--run_id` (which may be `auto`) + // against that same root so the auto-scan and the directory the run writes to + // agree (honors `--runs-dir` / `SIA_RUNS_DIR`). let runs_dir = resolve_runs_dir(opt_str(args, "runs_dir")); + let run_id = resolve_run_id(opt_str(args, "run_id").unwrap_or("1"), &runs_dir)?; + + // Surface the resolved run directory up front, before any expensive LLM work, + // so a presenter immediately sees where artifacts will land. + let resolved_run_dir = RunLayout::for_run_id(run_id, &runs_dir).run_dir; + println!("Run directory: {resolved_run_dir}"); // Live dashboard in the background unless disabled. It serves exactly the // directory the run writes to (`runs_dir`). The listener is bound up front so @@ -238,3 +287,72 @@ pub fn run_orchestrator(args: &ArgMatches, env_config: &Config) -> SiaResult<()> ); Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn numeric_run_id_passes_through() { + // A custom runs root with no entries does not affect numeric ids. + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().to_str().unwrap(); + assert_eq!(resolve_run_id("1", root).unwrap(), 1); + assert_eq!(resolve_run_id("7", root).unwrap(), 7); + } + + #[test] + fn invalid_run_id_errors() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().to_str().unwrap(); + assert!(resolve_run_id("0", root).is_err()); + assert!(resolve_run_id("-1", root).is_err()); + assert!(resolve_run_id("nope", root).is_err()); + } + + #[test] + fn auto_picks_one_when_root_empty_or_missing() { + // Missing root. + assert_eq!(resolve_run_id("auto", "/no/such/runs/root").unwrap(), 1); + // Existing-but-empty root. + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().to_str().unwrap(); + assert_eq!(resolve_run_id("auto", root).unwrap(), 1); + assert_eq!(resolve_run_id("AUTO", root).unwrap(), 1); + } + + #[test] + fn auto_picks_next_free_id() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + std::fs::create_dir(root.join("run_1")).unwrap(); + assert_eq!( + resolve_run_id("auto", root.to_str().unwrap()).unwrap(), + 2, + "auto should pick run_2 when run_1 exists" + ); + + // Gaps and non-run entries are ignored; auto uses max + 1. + std::fs::create_dir(root.join("run_5")).unwrap(); + std::fs::create_dir(root.join("not_a_run")).unwrap(); + std::fs::create_dir(root.join("run_abc")).unwrap(); + assert_eq!(resolve_run_id("auto", root.to_str().unwrap()).unwrap(), 6); + } + + #[test] + fn auto_resolves_under_custom_runs_dir() { + // Two independent roots: auto resolves against whichever root it is given. + let a = tempfile::tempdir().unwrap(); + let b = tempfile::tempdir().unwrap(); + std::fs::create_dir(a.path().join("run_3")).unwrap(); + // Root `a` has run_3 -> auto picks 4; root `b` is empty -> auto picks 1. + assert_eq!( + resolve_run_id("auto", a.path().to_str().unwrap()).unwrap(), + 4 + ); + assert_eq!( + resolve_run_id("auto", b.path().to_str().unwrap()).unwrap(), + 1 + ); + } +} diff --git a/tests/runs_dir.rs b/tests/runs_dir.rs index 9d29330..f7ec191 100644 --- a/tests/runs_dir.rs +++ b/tests/runs_dir.rs @@ -8,7 +8,8 @@ use std::sync::Mutex; use sia::layout::{names, RunLayout}; -use sia::run::resolve_runs_dir; +use sia::run::{resolve_run_id, resolve_runs_dir}; +use sia::run_setup::setup_run_directory; static ENV_LOCK: Mutex<()> = Mutex::new(()); @@ -80,3 +81,48 @@ fn test_dashboard_root_matches_run_root() { std::env::remove_var("SIA_RUNS_DIR"); } + +#[test] +fn test_numeric_run_id_collision_still_errors() { + // A numeric --run_id pointing at an existing run_ directory must error + // (unchanged historical behavior), before any venv work happens. + let tmp = tempfile::tempdir().unwrap(); + let runs_root = tmp.path().to_str().unwrap(); + std::fs::create_dir(tmp.path().join("run_1")).unwrap(); + + let result = setup_run_directory( + 1, + "/some/task", + "meta-model", + "task-model", + "claude", + 3, + None, + None, + None, + runs_root, + ); + assert!( + result.is_err(), + "numeric run_id on an existing run directory must error" + ); + let msg = result.err().unwrap().to_string(); + assert!( + msg.contains("already exists"), + "error should explain the collision, got: {msg}" + ); +} + +#[test] +fn test_auto_picks_next_free_id_under_custom_runs_dir() { + // `auto` resolves against the provided runs root: create run_1, expect run_2. + let tmp = tempfile::tempdir().unwrap(); + let runs_root = tmp.path().to_str().unwrap(); + std::fs::create_dir(tmp.path().join("run_1")).unwrap(); + + let id = resolve_run_id("auto", runs_root).unwrap(); + assert_eq!(id, 2, "auto should pick run_2 when run_1 exists"); + + let layout = RunLayout::for_run_id(id, runs_root); + assert_eq!(layout.run_dir, format!("{runs_root}/run_2")); +}