diff --git a/package.json b/package.json index d6d18f52d..482a16607 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "check:undeclared-imports": "node --test scripts/check-undeclared-imports.test.mjs && node scripts/check-undeclared-imports.mjs", "check:request-body-contract": "node --test scripts/check-request-body-contract.test.mjs scripts/check-openapi-refs.test.mjs scripts/check-prototype-chain-lookups.test.mjs && node scripts/check-request-body-contract.mjs && node scripts/check-openapi-refs.mjs && node scripts/check-prototype-chain-lookups.mjs", "check:prototype-chain-lookups": "node --test scripts/check-prototype-chain-lookups.test.mjs && node scripts/check-prototype-chain-lookups.mjs", - "check:ci-preflight": "node scripts/check-ci-preflight.mjs && node --test tools/ci/postgres-shard.test.mjs tools/ci/cargo-test-runner.test.mjs tools/ci/work-order-request-no-seed.test.mjs tools/ci/check-product-buck-residual.test.mjs tools/ci/check-nextest-config.test.mjs tools/ci/check-postgres-cargo-map.test.mjs tools/ci/ingest-soft-reds.test.mjs scripts/local-admission.test.mjs tools/ci/assess-tip-contention.test.mjs tools/ci/check-mjs-dark-suites.test.mjs tools/ci/classify-ci-failure.test.mjs tools/ci/postgres-timings.test.mjs tools/ci/cargo-needs-postgres-args.test.mjs tools/ci/check-nightly-workflow.test.mjs tools/ci/nextest-filterset.test.mjs tools/ci/postgres-partition.test.mjs && node tools/ci/check-product-buck-residual.mjs && node tools/ci/check-nextest-config.mjs && node tools/ci/check-postgres-cargo-map.mjs && node tools/ci/check-mjs-dark-suites.mjs --strict && node tools/ci/check-nightly-workflow.mjs && node --test scripts/check-foundation-gates.test.mjs && node --test scripts/check-non-oci-mail-imessage-relay.test.mjs && node --test scripts/console/run-verification-queue.test.mjs && python3 tools/lanes/fanout.test.py", + "check:ci-preflight": "node tools/ci/gate-sweep.mjs", "check:reasoning-lens-manifest": "node scripts/check-reasoning-lens-manifest.mjs", "test:reasoning-lens-manifest": "node --test scripts/check-reasoning-lens-manifest.test.mjs", "check:doc-links": "node scripts/check-doc-links.mjs", diff --git a/tools/ci/check-mjs-dark-suites.mjs b/tools/ci/check-mjs-dark-suites.mjs index d158333a5..33da8a79e 100644 --- a/tools/ci/check-mjs-dark-suites.mjs +++ b/tools/ci/check-mjs-dark-suites.mjs @@ -84,6 +84,22 @@ function listTestMjs(dir, baseRoot, acc = []) { return acc; } +/** + * Commands declared in the fail-slow gate sweep, as one blob of text. + * Absent manifest returns "" rather than throwing: this scanner must keep + * working in a tree that predates the sweep. + * @param {string} repoRoot + * @returns {string} + */ +function sweepManifestText(repoRoot) { + try { + const doc = JSON.parse(readFileSync(join(repoRoot, "tools/ci/gate-sweep.json"), "utf8")); + return "\n" + (doc.gates ?? []).map((gate) => gate?.run ?? "").join("\n"); + } catch { + return ""; + } +} + export function resolveDarkSuites(repoRoot = root) { const pkg = JSON.parse(readFileSync(join(repoRoot, "package.json"), "utf8")); const scripts = pkg.scripts ?? {}; @@ -123,7 +139,15 @@ export function resolveDarkSuites(repoRoot = root) { } } - const expanded = runText + [...invoked].map((n) => scripts[n]).join("\n"); + // tools/ci/gate-sweep.json is a real execution path, so it is a real + // reachability source. `check:ci-preflight` used to be an `&&` chain that + // named its test files inline in package.json, which is where this scan + // looks; the chain became a fail-slow sweep and moved those commands into a + // manifest. Without reading it, 20 suites that DO run every CI cycle report + // as dark -- and the natural "fix" is to baseline them, which would retire + // live coverage on the strength of a scanner blind spot. + const sweepText = sweepManifestText(repoRoot); + const expanded = runText + [...invoked].map((n) => scripts[n]).join("\n") + sweepText; const suites = listTestMjs(repoRoot, repoRoot); const dark = []; const wired = []; diff --git a/tools/ci/gate-sweep.json b/tools/ci/gate-sweep.json new file mode 100644 index 000000000..0a6a5ebcd --- /dev/null +++ b/tools/ci/gate-sweep.json @@ -0,0 +1,53 @@ +{ + "description": "Gates swept by check:ci-preflight. Fail-SLOW: every gate runs, every failure is reported once. Adding a gate here is how it enters the sweep; a gate absent from this file is not run.", + "gates": [ + { + "id": "check-ci-preflight", + "run": "node scripts/check-ci-preflight.mjs" + }, + { + "id": "ci-tools-suites", + "run": "node --test tools/ci/postgres-shard.test.mjs tools/ci/cargo-test-runner.test.mjs tools/ci/work-order-request-no-seed.test.mjs tools/ci/check-product-buck-residual.test.mjs tools/ci/check-nextest-config.test.mjs tools/ci/check-postgres-cargo-map.test.mjs tools/ci/ingest-soft-reds.test.mjs scripts/local-admission.test.mjs tools/ci/assess-tip-contention.test.mjs tools/ci/check-mjs-dark-suites.test.mjs tools/ci/classify-ci-failure.test.mjs tools/ci/postgres-timings.test.mjs tools/ci/cargo-needs-postgres-args.test.mjs tools/ci/check-nightly-workflow.test.mjs tools/ci/nextest-filterset.test.mjs tools/ci/postgres-partition.test.mjs" + }, + { + "id": "check-product-buck-residual", + "run": "node tools/ci/check-product-buck-residual.mjs" + }, + { + "id": "check-nextest-config", + "run": "node tools/ci/check-nextest-config.mjs" + }, + { + "id": "check-postgres-cargo-map", + "run": "node tools/ci/check-postgres-cargo-map.mjs" + }, + { + "id": "check-mjs-dark-suites", + "run": "node tools/ci/check-mjs-dark-suites.mjs --strict" + }, + { + "id": "check-nightly-workflow", + "run": "node tools/ci/check-nightly-workflow.mjs" + }, + { + "id": "check-foundation-gates-tests", + "run": "node --test scripts/check-foundation-gates.test.mjs" + }, + { + "id": "check-non-oci-mail-imessage-relay-tests", + "run": "node --test scripts/check-non-oci-mail-imessage-relay.test.mjs" + }, + { + "id": "run-verification-queue-tests", + "run": "node --test scripts/console/run-verification-queue.test.mjs" + }, + { + "id": "fanout-tests", + "run": "python3 tools/lanes/fanout.test.py" + }, + { + "id": "gate-sweep-tests", + "run": "node --test tools/ci/gate-sweep.test.mjs" + } + ] +} diff --git a/tools/ci/gate-sweep.mjs b/tools/ci/gate-sweep.mjs new file mode 100644 index 000000000..211902b76 --- /dev/null +++ b/tools/ci/gate-sweep.mjs @@ -0,0 +1,119 @@ +#!/usr/bin/env node +/** + * Fail-SLOW gate sweep: run every preflight gate, report every failure once. + * + * `check:ci-preflight` was an 11-link `&&` chain, so the shell stopped at the + * first failing gate and the remaining ten never ran. A tip with three broken + * gates therefore needed three full CI cycles to discover three problems — at + * this repo's measured ~20 minute wall clock, an hour to learn what one sweep + * already knew. + * + * The workflow LAYER was already fail-slow: 80 steps in ci.yml carry + * `!cancelled()` and `scripts/ci-collect-failures.mjs` re-asserts the job red + * with every failing step id. That sweep stopped at the step boundary — inside + * one step, `&&` still short-circuited. This closes that gap with the same + * shape, one level down. + * + * Red is preserved exactly: a non-zero exit from any gate still exits non-zero + * here. The only thing that changes is how much you learn per run. + * + * Precedent in-tree: `tools/ci/cargo-test-runner.sh` runs a stream of cargo + * invocations keep-going for the same reason, and its comment says it plainly — + * "one red binary never hides the rest". + */ +import { spawnSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); + +/** + * Load the gate list. Kept as data in a sibling JSON file so the sweep's + * contents are diffable and a gate cannot be dropped inside a shell string + * where no reviewer would see it. + * @returns {Array<{id: string, run: string}>} + */ +export function loadGates(path = resolve(ROOT, "tools/ci/gate-sweep.json")) { + const doc = JSON.parse(readFileSync(path, "utf8")); + const gates = Array.isArray(doc?.gates) ? doc.gates : null; + if (!gates || gates.length === 0) { + // A sweep over zero gates would exit 0 and prove nothing. + throw new Error(`gate-sweep: ${path} declares no gates`); + } + const seen = new Set(); + for (const gate of gates) { + if (typeof gate?.id !== "string" || gate.id === "") { + throw new Error("gate-sweep: every gate needs a non-empty id"); + } + if (typeof gate?.run !== "string" || gate.run === "") { + throw new Error(`gate-sweep: gate ${gate.id} has no command`); + } + if (seen.has(gate.id)) { + throw new Error(`gate-sweep: duplicate gate id ${gate.id}`); + } + seen.add(gate.id); + } + return gates; +} + +/** + * Run every gate, regardless of earlier failures. + * + * @param {Array<{id: string, run: string}>} gates + * @param {(cmd: string) => {status: number, stdout: string, stderr: string}} exec + * @returns {Array<{id: string, ok: boolean, seconds: number, output: string}>} + */ +export function sweep(gates, exec) { + const results = []; + for (const gate of gates) { + const started = Date.now(); + const done = exec(gate.run); + results.push({ + id: gate.id, + ok: done.status === 0, + seconds: Math.round((Date.now() - started) / 100) / 10, + output: `${done.stdout ?? ""}${done.stderr ?? ""}`, + }); + } + return results; +} + +/** @param {Array<{id:string,ok:boolean,seconds:number,output:string}>} results */ +export function report(results) { + const lines = []; + for (const r of results) { + lines.push(`${r.ok ? "PASS" : "FAIL"} ${r.id} (${r.seconds}s)`); + } + const failed = results.filter((r) => !r.ok); + lines.push(""); + lines.push(`gate-sweep: ${results.length - failed.length} passed, ${failed.length} failed`); + if (failed.length > 0) { + // Every failure, not just the first: the whole point of the sweep. + for (const r of failed) { + lines.push(""); + lines.push(`--- ${r.id} ---`); + lines.push(r.output.trimEnd()); + } + lines.push(""); + lines.push(`gate-sweep: FAILED ${failed.map((r) => r.id).join(", ")}`); + } + return lines.join("\n"); +} + +const isMain = process.argv[1] && process.argv[1].endsWith("gate-sweep.mjs"); +if (isMain) { + const gates = loadGates(); + const results = sweep(gates, (cmd) => { + const done = spawnSync(cmd, { + cwd: ROOT, + shell: true, + encoding: "utf8", + env: { ...process.env }, + maxBuffer: 64 * 1024 * 1024, + }); + return { status: done.status ?? 1, stdout: done.stdout ?? "", stderr: done.stderr ?? "" }; + }); + console.log(report(results)); + process.exit(results.every((r) => r.ok) ? 0 : 1); +} diff --git a/tools/ci/gate-sweep.test.mjs b/tools/ci/gate-sweep.test.mjs new file mode 100644 index 000000000..5b9e94490 --- /dev/null +++ b/tools/ci/gate-sweep.test.mjs @@ -0,0 +1,98 @@ +#!/usr/bin/env node +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { loadGates, report, sweep } from "./gate-sweep.mjs"; + +const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); + +/** Deterministic fake runner: any command containing "bad" fails. */ +const fakeExec = (cmd) => + cmd.includes("bad") + ? { status: 1, stdout: `boom from ${cmd}\n`, stderr: "" } + : { status: 0, stdout: "ok\n", stderr: "" }; + +test("gate-sweep", async (t) => { + await t.test("EVERY gate runs even after an earlier one fails", () => { + // This is the whole point. Under the old `&&` chain the shell stopped at the + // first failure, so a tip with three broken gates needed three ~20-minute CI + // cycles to reveal three problems. + const ran = []; + const gates = [ + { id: "one", run: "bad-1" }, + { id: "two", run: "good" }, + { id: "three", run: "bad-2" }, + ]; + const results = sweep(gates, (cmd) => { + ran.push(cmd); + return fakeExec(cmd); + }); + assert.deepEqual(ran, ["bad-1", "good", "bad-2"], "a failure must not short-circuit the sweep"); + assert.deepEqual(results.map((r) => r.ok), [false, true, false]); + }); + + await t.test("reports every failure, not just the first", () => { + const results = sweep( + [{ id: "alpha", run: "bad-a" }, { id: "beta", run: "good" }, { id: "gamma", run: "bad-g" }], + fakeExec, + ); + const text = report(results); + assert.match(text, /FAILED alpha, gamma/); + assert.match(text, /1 passed, 2 failed/); + assert.match(text, /boom from bad-a/, "the failing gate's output must be shown"); + assert.match(text, /boom from bad-g/, "…including the second one"); + }); + + await t.test("an all-green sweep reports no failures", () => { + const results = sweep([{ id: "a", run: "good" }, { id: "b", run: "good" }], fakeExec); + assert.equal(results.every((r) => r.ok), true); + assert.match(report(results), /2 passed, 0 failed/); + assert.doesNotMatch(report(results), /FAILED/); + }); + + await t.test("a non-zero exit anywhere still means red", () => { + // Fail-slow must not become fail-open: the sweep's exit code is derived from + // `every(ok)`, so one red gate keeps the whole sweep red. + const results = sweep([{ id: "a", run: "good" }, { id: "b", run: "bad" }], fakeExec); + assert.equal(results.every((r) => r.ok), false); + }); + + await t.test("a sweep over zero gates is refused, not passed", () => { + // A guard that examines nothing must fail; exiting 0 over an empty list + // would be the emptiest possible false green. + assert.throws(() => loadGates("/dev/null"), /declares no gates|Unexpected end/); + }); + + await t.test("malformed gate entries are refused", () => { + const tmp = resolve(ROOT, "tools/ci/gate-sweep.json"); + const good = JSON.parse(readFileSync(tmp, "utf8")); + assert.ok(good.gates.length > 0); + // Structural refusals are exercised through loadGates' validation by + // constructing docs in memory via a temp path is unnecessary — the + // committed file must itself satisfy every rule, asserted below. + for (const gate of good.gates) { + assert.equal(typeof gate.id, "string"); + assert.notEqual(gate.id, ""); + assert.equal(typeof gate.run, "string"); + assert.notEqual(gate.run, ""); + } + const ids = good.gates.map((g) => g.id); + assert.equal(new Set(ids).size, ids.length, "duplicate gate ids would hide one gate behind another"); + }); + + await t.test("the committed manifest still covers every gate package.json claims", () => { + // If someone re-adds an `&&` link to check:ci-preflight instead of adding a + // gate here, that gate would never run in the sweep. Catch that drift. + const pkg = JSON.parse(readFileSync(resolve(ROOT, "package.json"), "utf8")); + const script = pkg.scripts["check:ci-preflight"]; + assert.match(script, /gate-sweep\.mjs/, "check:ci-preflight must delegate to the sweep"); + assert.doesNotMatch( + script, + /&&/, + "check:ci-preflight must not chain with && — that is the fail-fast behaviour this replaces", + ); + assert.ok(loadGates().length >= 10, "the sweep must carry the full gate set"); + }); +});