ci: split windows cli and store tests across three runners

.github/workflows/ci.yml

@@ -413,17 +413,14 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - name: cli
-            package: ./cmd/msgvault/cmd
-            # Windows SQLite I/O makes this package's aggregate shard duration
-            # exceed one hour: the passing shard alone ran 406 tests in 3373.8s
-            # (~56m) and the next shard timed out at 60m while still actively
-            # executing schema setup (no deadlock; tests kept progressing).
-            # Give only this package the larger per-shard budget.
-            timeout: 120m
-          - name: store
-            package: ./internal/store
-            timeout: 60m
+          # A 4-core runner is already saturated by one package's shards, so the
+          # two slowest packages each split across 3 runners.
+          - { name: cli-1, package: ./cmd/msgvault/cmd, part: 1, parts: 3, timeout: 60m }
+          - { name: cli-2, package: ./cmd/msgvault/cmd, part: 2, parts: 3, timeout: 60m }
+          - { name: cli-3, package: ./cmd/msgvault/cmd, part: 3, parts: 3, timeout: 60m }
+          - { name: store-1, package: ./internal/store, part: 1, parts: 3, timeout: 60m }
+          - { name: store-2, package: ./internal/store, part: 2, parts: 3, timeout: 60m }
+          - { name: store-3, package: ./internal/store, part: 3, parts: 3, timeout: 60m }
           - name: sync-vector-embed
             package: ./internal/sync
             package2: ./internal/vector/embed
@@ -459,7 +456,9 @@ jobs:
               -Package $package `
               -ShardCount 4 `
               -Tags "fts5 sqlite_vec" `
-              -Timeout ${{ matrix.timeout }}
+              -Timeout ${{ matrix.timeout }} `
+              -PartIndex ${{ matrix.part || 1 }} `
+              -PartCount ${{ matrix.parts || 1 }}
             if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
           }
 

scripts/test-package-shards.ps1

@@ -7,12 +7,23 @@ param(
 
     [string]$Tags = "",
 
-    [string]$Timeout = "20m"
+    [string]$Timeout = "20m",
+
+    # Runs only part PartIndex of PartCount so several machines can split one package.
+    [ValidateRange(1, 64)]
+    [int]$PartCount = 1,
+
+    [ValidateRange(1, 64)]
+    [int]$PartIndex = 1
 )
 
 $ErrorActionPreference = "Stop"
 Set-StrictMode -Version Latest
 
+if ($PartIndex -gt $PartCount) {
+    throw "PartIndex $PartIndex exceeds PartCount $PartCount"
+}
+
 # Go accepts compound durations, including fractional units and zero to disable the timeout.
 $units = @{ ns = 1e-9; us = 1e-6; 'µs' = 1e-6; ms = 1e-3; s = 1; m = 60; h = 3600 }
 if ($Timeout -cnotmatch '^[+-]?(?:(?:\d+(?:\.\d*)?|\.\d+)(?:ns|us|µs|μs|ms|s|m|h))+$|^[+-]?0$') {
@@ -56,6 +67,10 @@ try {
     if ($LASTEXITCODE -ne 0) {
         exit $LASTEXITCODE
     }
+    if ($PartCount -gt 1) {
+        $testNames = @(for ($i = $PartIndex - 1; $i -lt $testNames.Count; $i += $PartCount) { $testNames[$i] })
+        Write-Host "Running part $PartIndex of $PartCount"
+    }
     if ($testNames.Count -eq 0) {
         Write-Host "No tests found in $Package"
         exit 0

scripts/test_package_shards_windows_test.go

@@ -67,12 +67,13 @@ func record(t *testing.T) {
 	}
 	r.NoError(os.WriteFile(filepath.Join(dir, "fixture_test.go"), []byte(source.String()), 0o600))
 
-	run := func(t *testing.T, timeout, fail string) (string, string, error) {
+	run := func(t *testing.T, timeout, fail string, extra ...string) (string, string, error) {
 		t.Helper()
 		output := t.TempDir()
 		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
 		defer cancel()
-		cmd := exec.CommandContext(ctx, pwsh, "-NoProfile", "-File", script, "-Package", ".", "-ShardCount", "4", "-Tags", "fts5 sqlite_vec", "-Timeout", timeout)
+		args := append([]string{"-NoProfile", "-File", script, "-Package", ".", "-ShardCount", "4", "-Tags", "fts5 sqlite_vec", "-Timeout", timeout}, extra...)
+		cmd := exec.CommandContext(ctx, pwsh, args...)
 		cmd.Dir = dir
 		cmd.Env = append(os.Environ(), "GOWORK=off", "SHARD_OUTPUT="+output, "SHARD_FAIL="+fail)
 		cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true}
@@ -140,6 +141,28 @@ func record(t *testing.T) {
 		require.NoError(err, result)
 		require.FileExists(filepath.Join(output, "released"))
 	})
+	t.Run("parts", func(t *testing.T) {
+		require := require.New(t)
+		var got []string
+		for part := 1; part <= 3; part++ {
+			output, result, err := run(t, "1m30s", "", "-PartIndex", strconv.Itoa(part), "-PartCount", "3")
+			require.NoError(err, result)
+			files, err := os.ReadDir(output)
+			require.NoError(err)
+			for _, file := range files {
+				data, err := os.ReadFile(filepath.Join(output, file.Name()))
+				require.NoError(err)
+				for _, name := range strings.Fields(string(data))[1:] {
+					index, err := strconv.Atoi(name[4:8])
+					require.NoError(err)
+					require.Equal(part-1, index%3, "each part must run only its own tests")
+					got = append(got, name)
+				}
+			}
+		}
+		sort.Strings(got)
+		require.Equal(want, got, "the parts together must run every test exactly once")
+	})
 	t.Run("later batch failure", func(t *testing.T) {
 		require := require.New(t)
 		_, result, err := run(t, "1m30s", want[count-1])