Skip to content

test(fspy): add an access-relative benchmark suite #602

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
wan9chi merged 1 commit into from
Aug 9, 2026
Merged

test(fspy): add an access-relative benchmark suite #602

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 41 additions & 10 deletions crates/fspy_benchmark/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,10 @@ struct Suite {
/// Unmeasured iterations run first, to fill caches and settle the runner.
warmup: usize,
metric: Metric,
/// Whether the target opens a relative path (the launcher's
/// `--relative`), driving the tracker's working-directory resolution and
/// path joining instead of the borrow-only absolute lane.
relative: bool,
}

/// Opens nothing, so the whole launch is the cost of starting a tracked
Expand All @@ -62,12 +66,30 @@ const LAUNCH_SUITE: Suite = Suite {
iterations: if cfg!(windows) { 150 } else { 300 },
warmup: 5,
metric: Metric::Wall,
relative: false,
};

/// Opens timed from inside the target, so they price interception rather than
/// the launch around it.
const ACCESS_SUITE: Suite =
Suite { name: "access", opens: "2048", iterations: 102, warmup: 3, metric: Metric::Typical };
/// the launch around it. The absolute path takes the tracker's borrow-only
/// lane; the relative variant prices working-directory resolution and path
/// joining on top.
const ACCESS_SUITE: Suite = Suite {
name: "access",
opens: "2048",
iterations: 102,
warmup: 3,
metric: Metric::Typical,
relative: false,
};

const RELATIVE_ACCESS_SUITE: Suite = Suite {
name: "access-relative",
opens: "2048",
iterations: 102,
warmup: 3,
metric: Metric::Typical,
relative: true,
};

struct Backend {
name: &'static str,
Expand All @@ -86,11 +108,13 @@ fn main() {
let backends = [Backend { name: "dynamic", target: DYNAMIC_TARGET }];

for backend in &backends {
validate(HEAD_LAUNCHER.as_ref(), backend.target);
if let Some(base_launcher) = &base_launcher {
validate(base_launcher, backend.target);
for relative in [false, true] {
validate(HEAD_LAUNCHER.as_ref(), backend.target, relative);
if let Some(base_launcher) = &base_launcher {
validate(base_launcher, backend.target, relative);
}
}
for suite in [&LAUNCH_SUITE, &ACCESS_SUITE] {
for suite in [&LAUNCH_SUITE, &ACCESS_SUITE, &RELATIVE_ACCESS_SUITE] {
run_suite(backend, suite, base_launcher.as_deref());
}
}
Expand Down Expand Up @@ -209,6 +233,9 @@ fn launch(launcher: &OsStr, mode: Option<&str>, backend: &Backend, suite: &Suite
if let Some(mode) = mode {
command.arg(mode);
}
if suite.relative {
command.arg("--relative");
}
let output = command
.args([backend.target, THREADS, suite.opens])
.stdin(Stdio::null())
Expand All @@ -228,9 +255,13 @@ fn launch(launcher: &OsStr, mode: Option<&str>, backend: &Backend, suite: &Suite
}
}

fn validate(launcher: &OsStr, target: &str) {
let status = Command::new(launcher)
.arg("--validate")
fn validate(launcher: &OsStr, target: &str, relative: bool) {
let mut command = Command::new(launcher);
command.arg("--validate");
if relative {
command.arg("--relative");
}
let status = command
.arg(target)
// One thread opening one batch: the count matches the target's
// OPENS_PER_SAMPLE. Fewer would open nothing, which validation
Expand Down
66 changes: 46 additions & 20 deletions crates/fspy_benchmark_launcher/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,28 +22,42 @@ const MISSING_PATH: &str = "/.fspy-benchmark-missing";
#[cfg(windows)]
const MISSING_PATH: &str = r"C:\.fspy-benchmark-missing";

/// With `--relative`, the target opens this bare name from the filesystem
/// root instead, driving the tracker's relative-path lane: it must resolve
/// the working directory and join the two. Root as the working directory
/// makes the joined result exactly [`MISSING_PATH`], so validation checks
/// the same captured path in both modes.
const MISSING_RELATIVE_PATH: &str = ".fspy-benchmark-missing";
#[cfg(unix)]
const ROOT_DIR: &str = "/";
#[cfg(windows)]
const ROOT_DIR: &str = r"C:\";

fn main() {
let mut args = env::args_os().skip(1).collect::<Vec<_>>();
let mode = match args.first().map(OsString::as_os_str) {
Some(arg) if arg == "--untracked" => {
args.remove(0);
Mode::Untracked
}
Some(arg) if arg == "--validate" => {
args.remove(0);
Mode::Validate
let mut mode = Mode::Tracked;
let mut relative = false;
while let Some(flag) = args.first().map(OsString::as_os_str) {
if flag == "--untracked" {
mode = Mode::Untracked;
} else if flag == "--validate" {
mode = Mode::Validate;
} else if flag == "--relative" {
relative = true;
} else {
break;
}
_ => Mode::Tracked,
};
args.remove(0);
}
let (target, target_args) =
args.split_first().expect("usage: fspy_benchmark_launcher [MODE] TARGET ARGS...");
args.split_first().expect("usage: fspy_benchmark_launcher [FLAGS] TARGET ARGS...");

let runtime = Builder::new_multi_thread().worker_threads(2).enable_all().build().unwrap();
runtime.block_on(async {
match mode {
Mode::Tracked => report(run_tracked(target, target_args).await).await,
Mode::Untracked => report(run_untracked(target, target_args).await).await,
Mode::Validate => validate(target, target_args).await,
Mode::Tracked => report(run_tracked(target, target_args, relative).await).await,
Mode::Untracked => report(run_untracked(target, target_args, relative).await).await,
Mode::Validate => validate(target, target_args, relative).await,
}
});
}
Expand All @@ -62,14 +76,17 @@ struct Launch {
/// Times the launch from just before the spawn to just after the wait, so
/// that a tracked launch covers session setup, injection, and teardown, and
/// nothing of this launcher's own startup.
async fn run_tracked(target: &OsString, target_args: &[OsString]) -> Launch {
async fn run_tracked(target: &OsString, target_args: &[OsString], relative: bool) -> Launch {
let mut command = Command::new(target);
command
.args(target_args)
.arg(MISSING_PATH)
.arg(if relative { MISSING_RELATIVE_PATH } else { MISSING_PATH })
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::inherit());
if relative {
command.current_dir(ROOT_DIR);
}
let start = Instant::now();
let mut child = command
.spawn(CancellationToken::new())
Expand All @@ -86,14 +103,17 @@ async fn run_tracked(target: &OsString, target_args: &[OsString]) -> Launch {
Launch { wall_nanos, stdout }
}

async fn run_untracked(target: &OsString, target_args: &[OsString]) -> Launch {
async fn run_untracked(target: &OsString, target_args: &[OsString], relative: bool) -> Launch {
let mut command = tokio::process::Command::new(target);
command
.args(target_args)
.arg(MISSING_PATH)
.arg(if relative { MISSING_RELATIVE_PATH } else { MISSING_PATH })
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::inherit());
if relative {
command.current_dir(ROOT_DIR);
}
let start = Instant::now();
let mut child = command.spawn().expect("failed to spawn untracked benchmark target");
let stdout = child.stdout.take().expect("untracked benchmark target has no stdout");
Expand All @@ -117,14 +137,20 @@ async fn report(mut launch: Launch) {

/// Runs the target tracked and asserts that its accesses were captured, so
/// that the harness never benchmarks tracking that silently stopped working.
async fn validate(target: &OsString, target_args: &[OsString]) {
/// In relative mode the captured path must come out identical — the tracker
/// resolves the root working directory and joins the bare name back into
/// [`MISSING_PATH`] — so the assertion below covers both modes.
async fn validate(target: &OsString, target_args: &[OsString], relative: bool) {
let mut command = Command::new(target);
command
.args(target_args)
.arg(MISSING_PATH)
.arg(if relative { MISSING_RELATIVE_PATH } else { MISSING_PATH })
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::inherit());
if relative {
command.current_dir(ROOT_DIR);
}
let termination = command
.spawn(CancellationToken::new())
.await
Expand Down
Loading