diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 971bde676b..8d9f2b44a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -924,15 +924,17 @@ jobs: # flavors on Linux and macOS: the local flavor uses the packages/cli build # from this job, the global flavor reuses the installed release binary via # VP_SNAP_GLOBAL_VP (no vp_global_cli compile; build-upstream builds or - # cache-restores it). One leg per OS: the suite is not sharded. Windows - # runs in cli-snapshot-test-windows via the cross-compiled archive. + # cache-restores it). Three shards per OS split the trials across runners. + # Windows runs in cli-snapshot-test-windows via the cross-compiled archive. cli-snapshot-test: - name: CLI snapshot test (${{ matrix.target }}) + name: CLI snapshot test (${{ matrix.target }}, shard ${{ matrix.shard }}/3) needs: - download-previous-rolldown-binaries strategy: fail-fast: false matrix: + os: [namespace-profile-linux-x64-default, namespace-profile-mac-default] + shard: [1, 2, 3] include: - os: namespace-profile-linux-x64-default target: x86_64-unknown-linux-gnu @@ -1020,6 +1022,7 @@ jobs: cargo test -p vp_cli_snapshots env: RUST_BACKTRACE: '1' + VP_SNAP_SHARD: ${{ matrix.shard }}/3 # Runs the PTY snapshot suite (crates/vp_cli_snapshots) on Windows with # BOTH vp flavors, without a Rust toolchain on the runner: the test binary @@ -1027,11 +1030,15 @@ jobs: # build-windows-tests, the global vp comes prebuilt from build-windows-cli, # and the JS CLI is built here (skip-native) for the local flavor. cli-snapshot-test-windows: - name: CLI snapshot test (Windows) + name: CLI snapshot test (Windows, shard ${{ matrix.shard }}/3) needs: - download-previous-rolldown-binaries - build-windows-cli - build-windows-tests + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] # Runs on the Namespace Windows runner. This is a PTY snapshot suite that # opens a pseudo-console (ConPTY) and spawns vp into it. It previously stayed # on GitHub-hosted windows-latest because Namespace's Windows runners ran @@ -1129,7 +1136,7 @@ jobs: export VP_SNAP_PWSH_BIN="$(cygpath -w "$(command -v pwsh.exe)")" # --no-fail-fast: on a snapshot suite every diff is diagnostic # signal; cancelling on the first failure hides the rest. - cargo-nextest nextest run --archive-file windows-snapshot-tests.tar.zst --workspace-remap . --no-fail-fast + cargo-nextest nextest run --archive-file windows-snapshot-tests.tar.zst --workspace-remap . --no-fail-fast --partition hash:${{ matrix.shard }}/3 env: RUST_BACKTRACE: '1' # Keep Windows env parity with the `test` recipe in justfile. diff --git a/crates/vp_cli_snapshots/tests/cli_snapshots/README.md b/crates/vp_cli_snapshots/tests/cli_snapshots/README.md index 9d458b5c12..9129433673 100644 --- a/crates/vp_cli_snapshots/tests/cli_snapshots/README.md +++ b/crates/vp_cli_snapshots/tests/cli_snapshots/README.md @@ -56,6 +56,12 @@ cases). Prerequisites: both flavors need `cargo build -p vp_global_cli` older than `src`, so a forgotten rebuild never silently tests stale local-CLI code. +CI runs three shards per platform. Linux and macOS use `VP_SNAP_SHARD=1/3` +(then `2/3` and `3/3`) to distribute the ordered trials round-robin. Sharding +happens before name filtering, so filtered runs keep the same assignment. +Leave the variable unset to run the whole suite. Windows uses the existing +nextest runner with `--partition hash:1/3` (then `2/3` and `3/3`). + Environment overrides, mainly for CI: | Variable | Effect | diff --git a/crates/vp_cli_snapshots/tests/cli_snapshots/main.rs b/crates/vp_cli_snapshots/tests/cli_snapshots/main.rs index d096af5f73..4c6a920385 100644 --- a/crates/vp_cli_snapshots/tests/cli_snapshots/main.rs +++ b/crates/vp_cli_snapshots/tests/cli_snapshots/main.rs @@ -18,6 +18,7 @@ mod exit_code; mod flavor; mod redact; +mod shard; use std::{ collections::{BTreeMap, hash_map::DefaultHasher}, @@ -1814,6 +1815,11 @@ fn main() { } } + if let Some(shard) = std::env::var_os("VP_SNAP_SHARD") { + let shard = shard.to_str().expect("VP_SNAP_SHARD must be valid UTF-8"); + tests = shard::select(tests, shard).unwrap_or_else(|error| panic!("{error}")); + } + let conclusion = libtest_mimic::run(&args, tests); // Report each case's wall time (slowest first). Skipped for `--list`, diff --git a/crates/vp_cli_snapshots/tests/cli_snapshots/shard.rs b/crates/vp_cli_snapshots/tests/cli_snapshots/shard.rs new file mode 100644 index 0000000000..8465202255 --- /dev/null +++ b/crates/vp_cli_snapshots/tests/cli_snapshots/shard.rs @@ -0,0 +1,13 @@ +/// Split the deterministically ordered trials round-robin across CI jobs. +/// Apply this before libtest filters so filtered runs keep the same assignment. +pub fn select(tests: Vec, shard: &str) -> Result, &'static str> { + let (index, total) = shard + .split_once('/') + .and_then(|(index, total)| { + Some((index.parse::().ok()?, total.parse::().ok()?)) + }) + .filter(|&(index, total)| index > 0 && index <= total) + .ok_or("VP_SNAP_SHARD must be INDEX/TOTAL with 1 <= INDEX <= TOTAL")?; + + Ok(tests.into_iter().skip(index - 1).step_by(total).collect()) +} diff --git a/crates/vp_cli_snapshots/tests/shard_unit.rs b/crates/vp_cli_snapshots/tests/shard_unit.rs new file mode 100644 index 0000000000..2d396ab523 --- /dev/null +++ b/crates/vp_cli_snapshots/tests/shard_unit.rs @@ -0,0 +1,29 @@ +#![expect(clippy::disallowed_macros, reason = "standalone test uses std macros")] + +#[path = "cli_snapshots/shard.rs"] +mod shard; + +#[test] +fn shards_cover_every_trial_once_with_balanced_counts() { + for test_count in [0, 1, 2, 8, 100] { + for total in [1, 2, 3, 5] { + let tests: Vec<_> = (0..test_count).collect(); + let shards: Vec<_> = (1..=total) + .map(|index| shard::select(tests.clone(), &format!("{index}/{total}")).unwrap()) + .collect(); + let counts: Vec<_> = shards.iter().map(Vec::len).collect(); + assert!(counts.iter().max().unwrap() - counts.iter().min().unwrap() <= 1); + + let mut combined: Vec<_> = shards.into_iter().flatten().collect(); + combined.sort_unstable(); + assert_eq!(combined, tests); + } + } +} + +#[test] +fn invalid_shards_fail_instead_of_silently_skipping_tests() { + for value in ["", "1", "0/3", "1/0", "4/3", "-1/3", "a/3", "1/a", "1/2/3"] { + assert!(shard::select(vec!["test"], value).is_err(), "accepted {value:?}"); + } +}