From 8a1043c2cc79e65704e3e835aa4718dc7436d2f9 Mon Sep 17 00:00:00 2001 From: Aleksandar Grbic Date: Mon, 27 Jul 2026 22:40:42 +0200 Subject: [PATCH] =?UTF-8?q?test(gate):=20gate-honesty=20suite=20=E2=80=94?= =?UTF-8?q?=20disable-must-fail=20wiring=20guard=20for=20the=20observable?= =?UTF-8?q?=20controls=20(P3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Observable-gates plan P3. A CI-friendly 'disable-must-fail' suite that drives the REAL composed boringstack gate with an adversarial input per control and asserts that control's rule fires — proving each observable control (P0a/P0b/P1) is WIRED INTO THE PIPELINE and bites end-to-end, not merely unit-correct: - db:push headless rename crash (exit 0) → rule 'db-push' - db:push clean but a plan column absent → rule 'db-schema-mismatch' - feature routes absent from the served OpenAPI spec → rule 'reachability' - SANITY: all-healthy inputs → gate green (so the red scenarios can't be red-by-default) If a future change unwires or weakens a control (drops the oracle call, makes a stage non-blocking, removes it from composeBoringstackGate), its scenario stops producing the rule and the suite goes RED. Proven by an actual mutation: neutering the oracle guard in boringstackCommandStage turned the suite 3-pass/1-fail (the oracle test caught it); reverted. Runs in the normal 'bun test packages' CI — no server, DB, or browser. Adds an injectable specFetcher to composeBoringstackGate (threaded to reachabilityStage) so the route-presence probe is deterministically drivable at the composed level. Full suite green. --- .../core/src/loop/boringstack/gate-stages.ts | 6 +- packages/core/tests/gate-honesty.test.ts | 192 ++++++++++++++++++ 2 files changed, 197 insertions(+), 1 deletion(-) create mode 100644 packages/core/tests/gate-honesty.test.ts diff --git a/packages/core/src/loop/boringstack/gate-stages.ts b/packages/core/src/loop/boringstack/gate-stages.ts index cd2d476a..0b9f5c9e 100644 --- a/packages/core/src/loop/boringstack/gate-stages.ts +++ b/packages/core/src/loop/boringstack/gate-stages.ts @@ -821,6 +821,10 @@ export function composeBoringstackGate(opts: { /** The OTHER features/entities in this build, so the judge scopes to this * feature's own responsibilities and never demands a link to an unbuilt slice. */ siblingEntities?: readonly string[]; + /** OpenAPI-spec fetcher for the reachability runtime route-presence probe. Omitted → + * the default live `fetch`; injected by the gate-honesty suite (and available for + * future callers) so the probe is deterministically testable. */ + specFetcher?: SpecFetcher; }): IGate { const { cwd, exec, evaluator, baseline, feature, entity } = opts; const e2eAcceptanceDisabled = @@ -842,7 +846,7 @@ export function composeBoringstackGate(opts: { ), baseline ), - reachabilityStage(cwd, feature.id), + reachabilityStage(cwd, feature.id, opts.specFetcher), ...(entity !== undefined && !e2eAcceptanceDisabled ? [testIdStage(cwd, entity)] : []), diff --git a/packages/core/tests/gate-honesty.test.ts b/packages/core/tests/gate-honesty.test.ts new file mode 100644 index 00000000..973ed9d1 --- /dev/null +++ b/packages/core/tests/gate-honesty.test.ts @@ -0,0 +1,192 @@ +import { test, expect, describe } from "bun:test"; +import { composeBoringstackGate } from "../src/loop/boringstack/gate-stages"; +import type { Exec, IExecResult } from "../src/loop/boringstack/exec"; +import type { SpecFetcher } from "../src/loop/boringstack/openapi-routes"; +import type { IFeature } from "../src/loop/greenfield/greenfield.types"; +import type { IEntityAcceptance } from "../src/loop/acceptance/acceptance.types"; +import type { IProvider } from "../src/inference"; + +/** + * GATE-HONESTY SUITE (observable-gates plan P3) — "disable-must-fail". + * + * Each test drives the REAL composed boringstack gate with an adversarial input that a + * specific observable control must catch, and asserts that control's rule fires. The + * point is not to re-test each control's logic (its own unit tests do that) but to prove + * it is WIRED INTO THE PIPELINE and actually bites end-to-end: if a future change unwires + * or weakens a control (drops the oracle call, makes a stage non-blocking, removes it from + * composeBoringstackGate), its scenario here stops producing the expected rule and this + * suite goes RED. It runs in the normal `bun test packages` CI — no live server, no DB, no + * browser, deterministic. + */ + +const feature: IFeature = { + id: "note", + desc: "a note", + passes: false, + attempts: 0, +}; + +const passingJudge: IProvider = { + complete: async () => ({ + content: '{"pass":true,"notes":"ok"}', + toolCalls: [], + }), +}; + +const OK: IExecResult = { code: 0, stdout: "", stderr: "" }; + +/** A configurable exec: setup commands always succeed; db:push, the `bun -e` oracle probe, + * and the gate command are overridable per scenario. */ +function mkExec(over: { + push?: IExecResult; + oracle?: IExecResult; + gate?: IExecResult; +}): Exec { + return async (argv) => { + const cmd = argv.join(" "); + + if (argv[1] === "-e") { + // The db:push recovery's DROP and the oracle probe are both `bun -e`; the oracle + // script is the one querying information_schema. + const script = argv[2] ?? ""; + + return script.includes("information_schema") + ? (over.oracle ?? { code: 0, stdout: "__ORACLE__[]", stderr: "" }) + : OK; // the drop + } + + if (cmd.includes("db:push")) { + return over.push ?? OK; + } + + if (/lint:fix|(?:^|\s)format(?:\s|$)|eslintcache|^rm /u.test(cmd)) { + return OK; // autofix setup + } + + return over.gate ?? { code: 0, stdout: "all good", stderr: "" }; + }; +} + +const bookmark: IEntityAcceptance = { + id: "Note", + key: "note", + nav: "Notes", + fields: [ + { name: "title", type: "string", optional: false, valid: "t", invalid: [] }, + { name: "url", type: "string", optional: false, valid: "u", invalid: [] }, + ], + shows: ["title", "url"], + screens: ["list", "form"], + parents: [], + negatives: [], + acceptanceCheck: "test note", +}; + +describe("gate-honesty (disable-must-fail): each observable control is wired and fires", () => { + test("db:push recovery/surfacing — a headless rename crash (exit 0) reds the composed gate (rule db-push)", async () => { + // If dbPushForce's detection or the command stage's exit-code short-circuit is + // removed, this crash would pass as green and this test reds. + const gate = composeBoringstackGate({ + cwd: "/nonexistent", + exec: mkExec({ + push: { + code: 0, + stdout: "", + stderr: + "Error: Interactive prompts require a TTY terminal\n at promptColumnsConflicts", + }, + }), + evaluator: passingJudge, + baseline: new Set(), + feature, + }); + + const r = await gate.run("/nonexistent"); + + expect(r.passed).toBe(false); + expect(r.errors.some((e) => e.rule === "db-push")).toBe(true); + }); + + test("DB boundary oracle — db:push clean but a plan column absent reds the composed gate (rule db-schema-mismatch)", async () => { + // If checkEntitySchema is unwired from boringstackCommandStage, this stops firing. + const gate = composeBoringstackGate({ + cwd: "/nonexistent", + exec: mkExec({ + oracle: { + code: 0, + // DB has title but NOT url (the plan demands both). + stdout: + '__ORACLE__["id","user_id","title","created_at","updated_at"]', + stderr: "", + }, + }), + evaluator: passingJudge, + baseline: new Set(), + feature, + entity: bookmark, + }); + + const r = await gate.run("/nonexistent"); + + expect(r.passed).toBe(false); + expect(r.errors.some((e) => e.rule === "db-schema-mismatch")).toBe(true); + }); + + test("runtime route-presence — routes absent from the served spec reds the composed gate (rule reachability)", async () => { + // Command stage fully green (no entity → oracle skipped), but the running API's spec + // does not serve the feature's routes. If the OpenAPI route probe is removed from + // reachabilityStage, this stops firing. + const servesOtherOnly: SpecFetcher = async () => ({ + openapi: "3.0.0", + info: { title: "x", version: "1" }, + paths: { "/api/v1/other/": {}, "/api/v1/other/{id}": {} }, + }); + + const gate = composeBoringstackGate({ + cwd: "/nonexistent", + exec: mkExec({ gate: { code: 0, stdout: "all good", stderr: "" } }), + evaluator: passingJudge, + baseline: new Set(), + feature, + specFetcher: servesOtherOnly, + }); + + const r = await gate.run("/nonexistent"); + + expect(r.passed).toBe(false); + expect(r.errors.some((e) => e.rule === "reachability")).toBe(true); + expect(r.errors[0]?.message).toContain("/api/v1/note/"); + }); + + test("SANITY: with every control's input HEALTHY, the composed gate is green (the suite can distinguish red from green)", async () => { + // Proves the red scenarios above fail because of the injected defect, not because the + // harness is red-by-default — otherwise "disable-must-fail" would be vacuous. + const servesNote: SpecFetcher = async () => ({ + openapi: "3.0.0", + info: { title: "x", version: "1" }, + paths: { "/api/v1/note/": {}, "/api/v1/note/{id}": {} }, + }); + + const gate = composeBoringstackGate({ + cwd: "/nonexistent", + exec: mkExec({ + oracle: { + code: 0, + stdout: + '__ORACLE__["id","user_id","title","url","created_at","updated_at"]', + stderr: "", + }, + gate: { code: 0, stdout: "all good", stderr: "" }, + }), + evaluator: passingJudge, + baseline: new Set(), + feature, + entity: bookmark, + specFetcher: servesNote, + }); + + const r = await gate.run("/nonexistent"); + + expect(r.passed).toBe(true); + }); +});