ci: sweep every preflight gate instead of stopping at the first failure

package.json

@@ -20,7 +20,7 @@
     "check:undeclared-imports": "node --test scripts/check-undeclared-imports.test.mjs && node scripts/check-undeclared-imports.mjs",
     "check:request-body-contract": "node --test scripts/check-request-body-contract.test.mjs scripts/check-openapi-refs.test.mjs scripts/check-prototype-chain-lookups.test.mjs && node scripts/check-request-body-contract.mjs && node scripts/check-openapi-refs.mjs && node scripts/check-prototype-chain-lookups.mjs",
     "check:prototype-chain-lookups": "node --test scripts/check-prototype-chain-lookups.test.mjs && node scripts/check-prototype-chain-lookups.mjs",
-    "check:ci-preflight": "node scripts/check-ci-preflight.mjs && node --test tools/ci/postgres-shard.test.mjs tools/ci/cargo-test-runner.test.mjs tools/ci/work-order-request-no-seed.test.mjs tools/ci/check-product-buck-residual.test.mjs tools/ci/check-nextest-config.test.mjs tools/ci/check-postgres-cargo-map.test.mjs tools/ci/ingest-soft-reds.test.mjs scripts/local-admission.test.mjs tools/ci/assess-tip-contention.test.mjs tools/ci/check-mjs-dark-suites.test.mjs tools/ci/classify-ci-failure.test.mjs tools/ci/postgres-timings.test.mjs tools/ci/cargo-needs-postgres-args.test.mjs tools/ci/check-nightly-workflow.test.mjs tools/ci/nextest-filterset.test.mjs tools/ci/postgres-partition.test.mjs && node tools/ci/check-product-buck-residual.mjs && node tools/ci/check-nextest-config.mjs && node tools/ci/check-postgres-cargo-map.mjs && node tools/ci/check-mjs-dark-suites.mjs --strict && node tools/ci/check-nightly-workflow.mjs && node --test scripts/check-foundation-gates.test.mjs && node --test scripts/check-non-oci-mail-imessage-relay.test.mjs && node --test scripts/console/run-verification-queue.test.mjs && python3 tools/lanes/fanout.test.py",
+    "check:ci-preflight": "node tools/ci/gate-sweep.mjs",
     "check:reasoning-lens-manifest": "node scripts/check-reasoning-lens-manifest.mjs",
     "test:reasoning-lens-manifest": "node --test scripts/check-reasoning-lens-manifest.test.mjs",
     "check:doc-links": "node scripts/check-doc-links.mjs",

tools/ci/check-mjs-dark-suites.mjs

@@ -84,6 +84,22 @@ function listTestMjs(dir, baseRoot, acc = []) {
   return acc;
 }
 
+/**
+ * Commands declared in the fail-slow gate sweep, as one blob of text.
+ * Absent manifest returns "" rather than throwing: this scanner must keep
+ * working in a tree that predates the sweep.
+ * @param {string} repoRoot
+ * @returns {string}
+ */
+function sweepManifestText(repoRoot) {
+  try {
+    const doc = JSON.parse(readFileSync(join(repoRoot, "tools/ci/gate-sweep.json"), "utf8"));
+    return "\n" + (doc.gates ?? []).map((gate) => gate?.run ?? "").join("\n");
+  } catch {
+    return "";
+  }
+}
+
 export function resolveDarkSuites(repoRoot = root) {
   const pkg = JSON.parse(readFileSync(join(repoRoot, "package.json"), "utf8"));
   const scripts = pkg.scripts ?? {};
@@ -123,7 +139,15 @@ export function resolveDarkSuites(repoRoot = root) {
     }
   }
 
-  const expanded = runText + [...invoked].map((n) => scripts[n]).join("\n");
+  // tools/ci/gate-sweep.json is a real execution path, so it is a real
+  // reachability source. `check:ci-preflight` used to be an `&&` chain that
+  // named its test files inline in package.json, which is where this scan
+  // looks; the chain became a fail-slow sweep and moved those commands into a
+  // manifest. Without reading it, 20 suites that DO run every CI cycle report
+  // as dark -- and the natural "fix" is to baseline them, which would retire
+  // live coverage on the strength of a scanner blind spot.
+  const sweepText = sweepManifestText(repoRoot);
+  const expanded = runText + [...invoked].map((n) => scripts[n]).join("\n") + sweepText;
   const suites = listTestMjs(repoRoot, repoRoot);
   const dark = [];
   const wired = [];

tools/ci/gate-sweep.json

@@ -0,0 +1,53 @@
+{
+  "description": "Gates swept by check:ci-preflight. Fail-SLOW: every gate runs, every failure is reported once. Adding a gate here is how it enters the sweep; a gate absent from this file is not run.",
+  "gates": [
+    {
+      "id": "check-ci-preflight",
+      "run": "node scripts/check-ci-preflight.mjs"
+    },
+    {
+      "id": "ci-tools-suites",
+      "run": "node --test tools/ci/postgres-shard.test.mjs tools/ci/cargo-test-runner.test.mjs tools/ci/work-order-request-no-seed.test.mjs tools/ci/check-product-buck-residual.test.mjs tools/ci/check-nextest-config.test.mjs tools/ci/check-postgres-cargo-map.test.mjs tools/ci/ingest-soft-reds.test.mjs scripts/local-admission.test.mjs tools/ci/assess-tip-contention.test.mjs tools/ci/check-mjs-dark-suites.test.mjs tools/ci/classify-ci-failure.test.mjs tools/ci/postgres-timings.test.mjs tools/ci/cargo-needs-postgres-args.test.mjs tools/ci/check-nightly-workflow.test.mjs tools/ci/nextest-filterset.test.mjs tools/ci/postgres-partition.test.mjs"
+    },
+    {
+      "id": "check-product-buck-residual",
+      "run": "node tools/ci/check-product-buck-residual.mjs"
+    },
+    {
+      "id": "check-nextest-config",
+      "run": "node tools/ci/check-nextest-config.mjs"
+    },
+    {
+      "id": "check-postgres-cargo-map",
+      "run": "node tools/ci/check-postgres-cargo-map.mjs"
+    },
+    {
+      "id": "check-mjs-dark-suites",
+      "run": "node tools/ci/check-mjs-dark-suites.mjs --strict"
+    },
+    {
+      "id": "check-nightly-workflow",
+      "run": "node tools/ci/check-nightly-workflow.mjs"
+    },
+    {
+      "id": "check-foundation-gates-tests",
+      "run": "node --test scripts/check-foundation-gates.test.mjs"
+    },
+    {
+      "id": "check-non-oci-mail-imessage-relay-tests",
+      "run": "node --test scripts/check-non-oci-mail-imessage-relay.test.mjs"
+    },
+    {
+      "id": "run-verification-queue-tests",
+      "run": "node --test scripts/console/run-verification-queue.test.mjs"
+    },
+    {
+      "id": "fanout-tests",
+      "run": "python3 tools/lanes/fanout.test.py"
+    },
+    {
+      "id": "gate-sweep-tests",
+      "run": "node --test tools/ci/gate-sweep.test.mjs"
+    }
+  ]
+}

tools/ci/gate-sweep.mjs

@@ -0,0 +1,119 @@
+#!/usr/bin/env node
+/**
+ * Fail-SLOW gate sweep: run every preflight gate, report every failure once.
+ *
+ * `check:ci-preflight` was an 11-link `&&` chain, so the shell stopped at the
+ * first failing gate and the remaining ten never ran. A tip with three broken
+ * gates therefore needed three full CI cycles to discover three problems — at
+ * this repo's measured ~20 minute wall clock, an hour to learn what one sweep
+ * already knew.
+ *
+ * The workflow LAYER was already fail-slow: 80 steps in ci.yml carry
+ * `!cancelled()` and `scripts/ci-collect-failures.mjs` re-asserts the job red
+ * with every failing step id. That sweep stopped at the step boundary — inside
+ * one step, `&&` still short-circuited. This closes that gap with the same
+ * shape, one level down.
+ *
+ * Red is preserved exactly: a non-zero exit from any gate still exits non-zero
+ * here. The only thing that changes is how much you learn per run.
+ *
+ * Precedent in-tree: `tools/ci/cargo-test-runner.sh` runs a stream of cargo
+ * invocations keep-going for the same reason, and its comment says it plainly —
+ * "one red binary never hides the rest".
+ */
+import { spawnSync } from "node:child_process";
+import { readFileSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
+
+/**
+ * Load the gate list. Kept as data in a sibling JSON file so the sweep's
+ * contents are diffable and a gate cannot be dropped inside a shell string
+ * where no reviewer would see it.
+ * @returns {Array<{id: string, run: string}>}
+ */
+export function loadGates(path = resolve(ROOT, "tools/ci/gate-sweep.json")) {
+  const doc = JSON.parse(readFileSync(path, "utf8"));
+  const gates = Array.isArray(doc?.gates) ? doc.gates : null;
+  if (!gates || gates.length === 0) {
+    // A sweep over zero gates would exit 0 and prove nothing.
+    throw new Error(`gate-sweep: ${path} declares no gates`);
+  }
+  const seen = new Set();
+  for (const gate of gates) {
+    if (typeof gate?.id !== "string" || gate.id === "") {
+      throw new Error("gate-sweep: every gate needs a non-empty id");
+    }
+    if (typeof gate?.run !== "string" || gate.run === "") {
+      throw new Error(`gate-sweep: gate ${gate.id} has no command`);
+    }
+    if (seen.has(gate.id)) {
+      throw new Error(`gate-sweep: duplicate gate id ${gate.id}`);
+    }
+    seen.add(gate.id);
+  }
+  return gates;
+}
+
+/**
+ * Run every gate, regardless of earlier failures.
+ *
+ * @param {Array<{id: string, run: string}>} gates
+ * @param {(cmd: string) => {status: number, stdout: string, stderr: string}} exec
+ * @returns {Array<{id: string, ok: boolean, seconds: number, output: string}>}
+ */
+export function sweep(gates, exec) {
+  const results = [];
+  for (const gate of gates) {
+    const started = Date.now();
+    const done = exec(gate.run);
+    results.push({
+      id: gate.id,
+      ok: done.status === 0,
+      seconds: Math.round((Date.now() - started) / 100) / 10,
+      output: `${done.stdout ?? ""}${done.stderr ?? ""}`,
+    });
+  }
+  return results;
+}
+
+/** @param {Array<{id:string,ok:boolean,seconds:number,output:string}>} results */
+export function report(results) {
+  const lines = [];
+  for (const r of results) {
+    lines.push(`${r.ok ? "PASS" : "FAIL"}  ${r.id}  (${r.seconds}s)`);
+  }
+  const failed = results.filter((r) => !r.ok);
+  lines.push("");
+  lines.push(`gate-sweep: ${results.length - failed.length} passed, ${failed.length} failed`);
+  if (failed.length > 0) {
+    // Every failure, not just the first: the whole point of the sweep.
+    for (const r of failed) {
+      lines.push("");
+      lines.push(`--- ${r.id} ---`);
+      lines.push(r.output.trimEnd());
+    }
+    lines.push("");
+    lines.push(`gate-sweep: FAILED ${failed.map((r) => r.id).join(", ")}`);
+  }
+  return lines.join("\n");
+}
+
+const isMain = process.argv[1] && process.argv[1].endsWith("gate-sweep.mjs");
+if (isMain) {
+  const gates = loadGates();
+  const results = sweep(gates, (cmd) => {
+    const done = spawnSync(cmd, {
+      cwd: ROOT,
+      shell: true,
+      encoding: "utf8",
+      env: { ...process.env },
+      maxBuffer: 64 * 1024 * 1024,
+    });
+    return { status: done.status ?? 1, stdout: done.stdout ?? "", stderr: done.stderr ?? "" };
+  });
+  console.log(report(results));
+  process.exit(results.every((r) => r.ok) ? 0 : 1);
+}

tools/ci/gate-sweep.test.mjs

@@ -0,0 +1,98 @@
+#!/usr/bin/env node
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { loadGates, report, sweep } from "./gate-sweep.mjs";
+
+const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
+
+/** Deterministic fake runner: any command containing "bad" fails. */
+const fakeExec = (cmd) =>
+  cmd.includes("bad")
+    ? { status: 1, stdout: `boom from ${cmd}\n`, stderr: "" }
+    : { status: 0, stdout: "ok\n", stderr: "" };
+
+test("gate-sweep", async (t) => {
+  await t.test("EVERY gate runs even after an earlier one fails", () => {
+    // This is the whole point. Under the old `&&` chain the shell stopped at the
+    // first failure, so a tip with three broken gates needed three ~20-minute CI
+    // cycles to reveal three problems.
+    const ran = [];
+    const gates = [
+      { id: "one", run: "bad-1" },
+      { id: "two", run: "good" },
+      { id: "three", run: "bad-2" },
+    ];
+    const results = sweep(gates, (cmd) => {
+      ran.push(cmd);
+      return fakeExec(cmd);
+    });
+    assert.deepEqual(ran, ["bad-1", "good", "bad-2"], "a failure must not short-circuit the sweep");
+    assert.deepEqual(results.map((r) => r.ok), [false, true, false]);
+  });
+
+  await t.test("reports every failure, not just the first", () => {
+    const results = sweep(
+      [{ id: "alpha", run: "bad-a" }, { id: "beta", run: "good" }, { id: "gamma", run: "bad-g" }],
+      fakeExec,
+    );
+    const text = report(results);
+    assert.match(text, /FAILED alpha, gamma/);
+    assert.match(text, /1 passed, 2 failed/);
+    assert.match(text, /boom from bad-a/, "the failing gate's output must be shown");
+    assert.match(text, /boom from bad-g/, "…including the second one");
+  });
+
+  await t.test("an all-green sweep reports no failures", () => {
+    const results = sweep([{ id: "a", run: "good" }, { id: "b", run: "good" }], fakeExec);
+    assert.equal(results.every((r) => r.ok), true);
+    assert.match(report(results), /2 passed, 0 failed/);
+    assert.doesNotMatch(report(results), /FAILED/);
+  });
+
+  await t.test("a non-zero exit anywhere still means red", () => {
+    // Fail-slow must not become fail-open: the sweep's exit code is derived from
+    // `every(ok)`, so one red gate keeps the whole sweep red.
+    const results = sweep([{ id: "a", run: "good" }, { id: "b", run: "bad" }], fakeExec);
+    assert.equal(results.every((r) => r.ok), false);
+  });
+
+  await t.test("a sweep over zero gates is refused, not passed", () => {
+    // A guard that examines nothing must fail; exiting 0 over an empty list
+    // would be the emptiest possible false green.
+    assert.throws(() => loadGates("/dev/null"), /declares no gates|Unexpected end/);
+  });
+
+  await t.test("malformed gate entries are refused", () => {
+    const tmp = resolve(ROOT, "tools/ci/gate-sweep.json");
+    const good = JSON.parse(readFileSync(tmp, "utf8"));
+    assert.ok(good.gates.length > 0);
+    // Structural refusals are exercised through loadGates' validation by
+    // constructing docs in memory via a temp path is unnecessary — the
+    // committed file must itself satisfy every rule, asserted below.
+    for (const gate of good.gates) {
+      assert.equal(typeof gate.id, "string");
+      assert.notEqual(gate.id, "");
+      assert.equal(typeof gate.run, "string");
+      assert.notEqual(gate.run, "");
+    }
+    const ids = good.gates.map((g) => g.id);
+    assert.equal(new Set(ids).size, ids.length, "duplicate gate ids would hide one gate behind another");
+  });
+
+  await t.test("the committed manifest still covers every gate package.json claims", () => {
+    // If someone re-adds an `&&` link to check:ci-preflight instead of adding a
+    // gate here, that gate would never run in the sweep. Catch that drift.
+    const pkg = JSON.parse(readFileSync(resolve(ROOT, "package.json"), "utf8"));
+    const script = pkg.scripts["check:ci-preflight"];
+    assert.match(script, /gate-sweep\.mjs/, "check:ci-preflight must delegate to the sweep");
+    assert.doesNotMatch(
+      script,
+      /&&/,
+      "check:ci-preflight must not chain with && — that is the fail-fast behaviour this replaces",
+    );
+    assert.ok(loadGates().length >= 10, "the sweep must carry the full gate set");
+  });
+});