ci: shard CLI snapshot tests across parallel jobs

Author: fengmk2

.github/workflows/ci.yml

@@ -924,15 +924,17 @@ jobs:
   # flavors on Linux and macOS: the local flavor uses the packages/cli build
   # from this job, the global flavor reuses the installed release binary via
   # VP_SNAP_GLOBAL_VP (no vp_global_cli compile; build-upstream builds or
-  # cache-restores it). One leg per OS: the suite is not sharded. Windows
-  # runs in cli-snapshot-test-windows via the cross-compiled archive.
+  # cache-restores it). Three shards per OS split the trials across runners.
+  # Windows runs in cli-snapshot-test-windows via the cross-compiled archive.
   cli-snapshot-test:
-    name: CLI snapshot test (${{ matrix.target }})
+    name: CLI snapshot test (${{ matrix.target }}, shard ${{ matrix.shard }}/3)
     needs:
       - download-previous-rolldown-binaries
     strategy:
       fail-fast: false
       matrix:
+        os: [namespace-profile-linux-x64-default, namespace-profile-mac-default]
+        shard: [1, 2, 3]
         include:
           - os: namespace-profile-linux-x64-default
             target: x86_64-unknown-linux-gnu
@@ -1020,18 +1022,23 @@ jobs:
           cargo test -p vp_cli_snapshots
         env:
           RUST_BACKTRACE: '1'
+          VP_SNAP_SHARD: ${{ matrix.shard }}/3
 
   # Runs the PTY snapshot suite (crates/vp_cli_snapshots) on Windows with
   # BOTH vp flavors, without a Rust toolchain on the runner: the test binary
   # and vpt arrive cross-compiled in the nextest archive from
   # build-windows-tests, the global vp comes prebuilt from build-windows-cli,
   # and the JS CLI is built here (skip-native) for the local flavor.
   cli-snapshot-test-windows:
-    name: CLI snapshot test (Windows)
+    name: CLI snapshot test (Windows, shard ${{ matrix.shard }}/3)
     needs:
       - download-previous-rolldown-binaries
       - build-windows-cli
       - build-windows-tests
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2, 3]
     # Runs on the Namespace Windows runner. This is a PTY snapshot suite that
     # opens a pseudo-console (ConPTY) and spawns vp into it. It previously stayed
     # on GitHub-hosted windows-latest because Namespace's Windows runners ran
@@ -1129,7 +1136,7 @@ jobs:
           export VP_SNAP_PWSH_BIN="$(cygpath -w "$(command -v pwsh.exe)")"
           # --no-fail-fast: on a snapshot suite every diff is diagnostic
           # signal; cancelling on the first failure hides the rest.
-          cargo-nextest nextest run --archive-file windows-snapshot-tests.tar.zst --workspace-remap . --no-fail-fast
+          cargo-nextest nextest run --archive-file windows-snapshot-tests.tar.zst --workspace-remap . --no-fail-fast --partition hash:${{ matrix.shard }}/3
         env:
           RUST_BACKTRACE: '1'
           # Keep Windows env parity with the `test` recipe in justfile.

crates/vp_cli_snapshots/tests/cli_snapshots/README.md

@@ -56,6 +56,12 @@ cases). Prerequisites: both flavors need `cargo build -p vp_global_cli`
 older than `src`, so a forgotten rebuild never silently tests stale
 local-CLI code.
 
+CI runs three shards per platform. Linux and macOS use `VP_SNAP_SHARD=1/3`
+(then `2/3` and `3/3`) to distribute the ordered trials round-robin. Sharding
+happens before name filtering, so filtered runs keep the same assignment.
+Leave the variable unset to run the whole suite. Windows uses the existing
+nextest runner with `--partition hash:1/3` (then `2/3` and `3/3`).
+
 Environment overrides, mainly for CI:
 
 | Variable                    | Effect                                                              |

crates/vp_cli_snapshots/tests/cli_snapshots/main.rs

@@ -18,6 +18,7 @@
 mod exit_code;
 mod flavor;
 mod redact;
+mod shard;
 
 use std::{
     collections::{BTreeMap, hash_map::DefaultHasher},
@@ -1814,6 +1815,11 @@ fn main() {
         }
     }
 
+    if let Some(shard) = std::env::var_os("VP_SNAP_SHARD") {
+        let shard = shard.to_str().expect("VP_SNAP_SHARD must be valid UTF-8");
+        tests = shard::select(tests, shard).unwrap_or_else(|error| panic!("{error}"));
+    }
+
     let conclusion = libtest_mimic::run(&args, tests);
 
     // Report each case's wall time (slowest first). Skipped for `--list`,

crates/vp_cli_snapshots/tests/cli_snapshots/shard.rs

@@ -0,0 +1,13 @@
+/// Split the deterministically ordered trials round-robin across CI jobs.
+/// Apply this before libtest filters so filtered runs keep the same assignment.
+pub fn select<T>(tests: Vec<T>, shard: &str) -> Result<Vec<T>, &'static str> {
+    let (index, total) = shard
+        .split_once('/')
+        .and_then(|(index, total)| {
+            Some((index.parse::<usize>().ok()?, total.parse::<usize>().ok()?))
+        })
+        .filter(|&(index, total)| index > 0 && index <= total)
+        .ok_or("VP_SNAP_SHARD must be INDEX/TOTAL with 1 <= INDEX <= TOTAL")?;
+
+    Ok(tests.into_iter().skip(index - 1).step_by(total).collect())
+}

crates/vp_cli_snapshots/tests/shard_unit.rs

@@ -0,0 +1,29 @@
+#![expect(clippy::disallowed_macros, reason = "standalone test uses std macros")]
+
+#[path = "cli_snapshots/shard.rs"]
+mod shard;
+
+#[test]
+fn shards_cover_every_trial_once_with_balanced_counts() {
+    for test_count in [0, 1, 2, 8, 100] {
+        for total in [1, 2, 3, 5] {
+            let tests: Vec<_> = (0..test_count).collect();
+            let shards: Vec<_> = (1..=total)
+                .map(|index| shard::select(tests.clone(), &format!("{index}/{total}")).unwrap())
+                .collect();
+            let counts: Vec<_> = shards.iter().map(Vec::len).collect();
+            assert!(counts.iter().max().unwrap() - counts.iter().min().unwrap() <= 1);
+
+            let mut combined: Vec<_> = shards.into_iter().flatten().collect();
+            combined.sort_unstable();
+            assert_eq!(combined, tests);
+        }
+    }
+}
+
+#[test]
+fn invalid_shards_fail_instead_of_silently_skipping_tests() {
+    for value in ["", "1", "0/3", "1/0", "4/3", "-1/3", "a/3", "1/a", "1/2/3"] {
+        assert!(shard::select(vec!["test"], value).is_err(), "accepted {value:?}");
+    }
+}