From bb8b0d5571722ca6a86181af9cc65b79915680a4 Mon Sep 17 00:00:00 2001 From: lex00 <121451605+lex00@users.noreply.github.com> Date: Fri, 11 Sep 2026 19:27:39 -0600 Subject: [PATCH] feat(conformance): compare whole-build fixtures against chant (#62) chant grew a public whole-build entry (chant#2408, merged as chant PR #2410), so the taint fixpoint is answerable by two implementations instead of one. `foldProject` on the adapter may now be async: chant resolves modules from a filesystem, so the chant adapter writes a fixture's sources to a temp directory and hands over the paths. `runFixtures`, `runProjectFixture` and `compareAdapters` follow. The comparison asserts more than the verdict. A seed and a taint casualty are both "run", so where both implementations report the tentative verdict and the file whose taint reached each tainted one, those are compared too. That is where J3 actually lives; comparing only the final verdict would pass an implementation that got both edges wrong in compensating ways. Run against the local chant checkout ahead of a release: eight whole-build fixtures, nineteen file verdicts, zero disagreements, including both tentative verdicts and the taint source and edge kind for all four taint casualties. chant independently classifies the capturing sibling as reached by a capture edge rather than an import, which is the distinction chant#2406 existed for. A host-dependent fixture is still skipped for chant: its entity classes come from a package chant cannot resolve, and its entry takes an intrinsic registry rather than a whole host. The test asserts that a skip has a host to explain it, so a silently skipped fixture fails instead of passing. The pinned chant is 0.69.1 and predates the entry, so CI still skips every whole-build fixture and the test asserts that too, explicitly, rather than tolerating it. #62 stays open until the pin moves. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01RfnRhfdYHLFAZJKHwZhBYc --- packages/conformance/src/adapter.ts | 13 ++-- packages/conformance/src/adapters/chant.ts | 60 ++++++++++++++++++- .../conformance/src/chant-agreement.test.ts | 40 +++++++++---- packages/conformance/src/runner.test.ts | 18 +++--- packages/conformance/src/runner.ts | 21 +++++-- 5 files changed, 120 insertions(+), 32 deletions(-) diff --git a/packages/conformance/src/adapter.ts b/packages/conformance/src/adapter.ts index 3031acc..243ca97 100644 --- a/packages/conformance/src/adapter.ts +++ b/packages/conformance/src/adapter.ts @@ -32,9 +32,14 @@ export interface ConformanceAdapter { foldExport(source: string, exportName: string): FoldResult; /** * J2 + J3 over a whole build: every file's final verdict. Optional, and - * "unavailable" when the implementation exposes no project-level entry — - * chant's public API is per-file, so its project fixtures are reported - * skipped rather than silently passing. + * "unavailable" when the implementation cannot answer — no whole-build + * entry at all, or a host it has no way to install. Such a fixture is + * reported skipped rather than silently passing. + * + * May be async: a real implementation resolves modules from a filesystem. */ - foldProject?(files: Map, host?: ConformanceHost): ProjectResult | "unavailable"; + foldProject?( + files: Map, + host?: ConformanceHost, + ): ProjectResult | "unavailable" | Promise; } diff --git a/packages/conformance/src/adapters/chant.ts b/packages/conformance/src/adapters/chant.ts index 241d072..69d6d83 100644 --- a/packages/conformance/src/adapters/chant.ts +++ b/packages/conformance/src/adapters/chant.ts @@ -4,9 +4,12 @@ * — `findSubsetViolation` for the shape half. If an older chant is pinned the * adapter reports shape "unavailable" rather than guessing. */ +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; import * as ts from "typescript"; import * as chant from "@intentius/chant"; -import type { ConformanceAdapter } from "../adapter"; +import type { ConformanceAdapter, ProjectResult, ProjectVerdict } from "../adapter"; function exportInitializer(sf: ts.SourceFile, name: string): ts.Expression | undefined { for (const st of sf.statements) { @@ -18,6 +21,53 @@ function exportInitializer(sf: ts.SourceFile, name: string): ts.Expression | und const parse = (src: string) => ts.createSourceFile("fixture.ts", src, ts.ScriptTarget.Latest, true); const shapeFn = (chant as unknown as { findSubsetViolation?: (n: ts.Node) => { node: ts.Node; ruleId: string; message: string } | undefined }).findSubsetViolation; +/** + * The whole-build entry, chant-v0.71.0+ (chant#2408). Absent on an older pin, + * in which case the project fixtures report "unavailable" rather than passing + * vacuously. + */ +type ChantVerdict = { + verdict: "fold" | "run"; + tentative: "fold" | "run"; + reason?: string; + taintedBy?: { from: string; kind: "importer" | "capture" }; + exports?: ReadonlyMap; +}; +const projectFn = ( + chant as unknown as { + foldProject?: (files: readonly string[], intrinsics?: readonly unknown[]) => Promise>; + } +).foldProject; + +/** chant resolves modules from disk, so a fixture's sources are written out and the paths handed over. */ +async function foldOnDisk(files: Map): Promise { + const root = mkdtempSync(join(tmpdir(), "tsad-conformance-")); + try { + const paths: string[] = []; + for (const [rel, source] of files) { + const abs = join(root, rel); + mkdirSync(dirname(abs), { recursive: true }); + writeFileSync(abs, source, "utf8"); + paths.push(abs); + } + const verdicts = await projectFn!(paths, []); + const out: ProjectResult = { verdicts: {}, tentative: {}, taintedBy: {} }; + const relOf = (abs: string) => abs.slice(root.length + 1).split(/[\\/]/).join("/"); + for (const [abs, v] of verdicts) { + const key = relOf(abs); + out.verdicts[key] = + v.verdict === "fold" + ? ({ kind: "fold", exports: Object.fromEntries(v.exports ?? new Map()) } as ProjectVerdict) + : ({ kind: "run", reason: v.reason ?? "tainted" } as ProjectVerdict); + out.tentative![key] = v.tentative; + if (v.taintedBy) out.taintedBy![key] = relOf(v.taintedBy.from); + } + return out; + } finally { + rmSync(root, { recursive: true, force: true }); + } +} + export const chantAdapter: ConformanceAdapter = { name: `chant@${(chant as unknown as { VERSION?: string }).VERSION ?? "0.69.1"}`, shape(source, exportName) { @@ -28,6 +78,14 @@ export const chantAdapter: ConformanceAdapter = { const { line, character } = sf.getLineAndCharacterOfPosition(v.node.getStart()); return { accepted: false, rule: v.ruleId, line: line + 1, column: character + 1, message: v.message }; }, + async foldProject(files, host) { + if (!projectFn) return "unavailable"; + // A named host's entity classes come from a package chant cannot resolve, + // and its entry takes an intrinsic registry rather than a whole host, so a + // host-dependent fixture is not answerable here. + if (host) return "unavailable"; + return foldOnDisk(files); + }, foldExport(source, exportName) { const sf = parse(source); const init = exportInitializer(sf, exportName); if (!init) return { ok: false, line: 1, column: 1, message: `no export named ${exportName}` }; diff --git a/packages/conformance/src/chant-agreement.test.ts b/packages/conformance/src/chant-agreement.test.ts index 3ea1229..367e9f6 100644 --- a/packages/conformance/src/chant-agreement.test.ts +++ b/packages/conformance/src/chant-agreement.test.ts @@ -2,29 +2,45 @@ import { describe, test, expect } from "vitest"; import { join, dirname } from "node:path"; import { fileURLToPath } from "node:url"; -import { loadFixtures, runFixtures, compareAdapters, expressionFixtures, projectFixtures } from "./index"; +import { loadFixtures, runFixtures, runProjectFixture, compareAdapters, expressionFixtures, projectFixtures } from "./index"; import { chantAdapter } from "./adapters/chant"; import { referenceAdapter } from "@intentius/tsad-reference"; const fixtures = loadFixtures(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "..", "spec", "fixtures")); describe("chant cross-check (#11)", () => { - test("chant passes every fixture through its public fold API", () => { - const failed = runFixtures(chantAdapter, fixtures).filter((r) => !r.pass).map((r) => `${r.fixture}: ${r.failures.join("; ")}`); + test("chant passes every fixture through its public fold API", async () => { + const failed = (await runFixtures(chantAdapter, fixtures)).filter((r) => !r.pass).map((r) => `${r.fixture}: ${r.failures.join("; ")}`); expect(failed, failed.join("\n")).toEqual([]); }); - test("chant and the reference implementation agree on every fixture", () => { - const dis = compareAdapters(referenceAdapter, chantAdapter, fixtures); + test("chant and the reference implementation agree on every fixture", async () => { + const dis = await compareAdapters(referenceAdapter, chantAdapter, fixtures); expect(dis, dis.join("\n")).toEqual([]); }); - test("chant has no project-level entry, so the project fixtures are skipped, not silently passed (#24)", () => { - // chant's public API folds one expression or one module's resources; J3 is - // not reachable through it. Asserted so the day it is, this test fails and - // the cross-check is extended rather than forgotten. See chant#2408. - expect(projectFixtures(fixtures).length).toBeGreaterThan(0); - expect(chantAdapter.foldProject).toBeUndefined(); + test("the whole-build fixtures reach chant, or say why not (#62)", async () => { + // Two reasons a project fixture can be skipped, and both must be visible. + // An older pin has no whole-build entry at all (chant#2408); a fixture + // naming a host asks for entity classes from a package chant cannot + // resolve. Anything else has to be answered. + const projects = projectFixtures(fixtures); + expect(projects.length).toBeGreaterThan(0); + const reports = await Promise.all(projects.map((f) => runProjectFixture(chantAdapter, f))); + const skipped = reports.filter((r) => r.skipped).map((r) => r.fixture); + const hosted = new Set(projects.filter((f) => f.host).map((f) => f.id)); + const unexpected = skipped.filter((id) => !hosted.has(id)); + if (chantAdapter.foldProject && (await chantAdapter.foldProject(new Map([["a.ts", "export const a = 1;"]]))) !== "unavailable") { + expect(unexpected, `skipped without a host to explain it:\n${unexpected.join("\n")}`).toEqual([]); + } else { + // The pin predates chant#2408. Recorded, not silently tolerated. + expect(skipped.length, "an older pin skips every whole-build fixture").toBe(projects.length); + } + }); + + test("chant and the reference agree on every whole-build fixture chant can answer (#62)", async () => { + const dis = await compareAdapters(referenceAdapter, chantAdapter, projectFixtures(fixtures)); + expect(dis, dis.join("\n")).toEqual([]); }); - test("chant's shape classifier is available (chant-v0.64.0+, chant#2362) and agrees on every fixture", () => { + test("chant's shape classifier is available (chant-v0.64.0+, chant#2362) and agrees on every fixture", async () => { // The pinned chant carries the export, so "unavailable" would mean the adapter // silently stopped comparing the shape half — a real regression, asserted. for (const f of expressionFixtures(fixtures)) { diff --git a/packages/conformance/src/runner.test.ts b/packages/conformance/src/runner.test.ts index 53c440d..c81c881 100644 --- a/packages/conformance/src/runner.test.ts +++ b/packages/conformance/src/runner.test.ts @@ -8,18 +8,18 @@ const fixturesDir = join(dirname(fileURLToPath(import.meta.url)), "..", "..", ". describe("conformance runner (#7) over the reference implementation (#10)", () => { const fixtures = loadFixtures(fixturesDir); - test("fixtures load", () => { expect(fixtures.length).toBeGreaterThan(0); for (const f of fixtures) expect(f.rules.length).toBeGreaterThan(0); }); - test("the reference implementation passes every fixture", () => { - const reports = runFixtures(referenceAdapter, fixtures); + test("fixtures load", async () => { expect(fixtures.length).toBeGreaterThan(0); for (const f of fixtures) expect(f.rules.length).toBeGreaterThan(0); }); + test("the reference implementation passes every fixture", async () => { + const reports = await runFixtures(referenceAdapter, fixtures); const failed = reports.filter((r) => !r.pass).map((r) => `${r.fixture}: ${r.failures.join("; ")}`); expect(failed, failed.join("\n")).toEqual([]); }); - test("the project fixtures are actually exercised, not all skipped (#24)", () => { - const reports = runFixtures(referenceAdapter, fixtures).filter((r) => projectFixtures(fixtures).some((p) => p.id === r.fixture)); + test("the project fixtures are actually exercised, not all skipped (#24)", async () => { + const reports = (await runFixtures(referenceAdapter, fixtures)).filter((r) => projectFixtures(fixtures).some((p) => p.id === r.fixture)); expect(reports.length).toBe(projectFixtures(fixtures).length); expect(reports.filter((r) => r.skipped)).toEqual([]); }); - test("a stub adapter that runs every file fails the project fixtures that expect a fold", () => { + test("a stub adapter that runs every file fails the project fixtures that expect a fold", async () => { // The control the fixtures themselves describe: falling back everywhere is // sound and useless, and must not pass. Without this, F-Taint's "least set" // would be untested. @@ -28,12 +28,12 @@ describe("conformance runner (#7) over the reference implementation (#10)", () = foldExport: () => ({ ok: false as const, line: 1, column: 1, message: "runs" }), foldProject: (files: Map) => ({ verdicts: Object.fromEntries([...files.keys()].map((p) => [p, { kind: "run" as const, reason: "runs" }])) }), }; - const failed = runFixtures(stub, fixtures).filter((r) => !r.pass && projectFixtures(fixtures).some((p) => p.id === r.fixture)); + const failed = (await runFixtures(stub, fixtures)).filter((r) => !r.pass && projectFixtures(fixtures).some((p) => p.id === r.fixture)); expect(failed.length).toBeGreaterThan(0); }); - test("a stub adapter that folds everything to null fails the reject fixture and the value fixtures", () => { + test("a stub adapter that folds everything to null fails the reject fixture and the value fixtures", async () => { const stub = { name: "stub", shape: () => ({ accepted: true } as const), foldExport: () => ({ ok: true as const, value: null }) }; - const reports = runFixtures(stub, fixtures); + const reports = await runFixtures(stub, fixtures); expect(reports.some((r) => !r.pass)).toBe(true); }); }); diff --git a/packages/conformance/src/runner.ts b/packages/conformance/src/runner.ts index 03a8056..6fbfe8a 100644 --- a/packages/conformance/src/runner.ts +++ b/packages/conformance/src/runner.ts @@ -22,18 +22,18 @@ export function runFixture(adapter: ConformanceAdapter, f: ExpressionFixture): F } return { fixture: f.id, adapter: adapter.name, pass: failures.length === 0, skipped, failures }; } -export function runFixtures(adapter: ConformanceAdapter, fixtures: Fixture[]): FixtureReport[] { +export async function runFixtures(adapter: ConformanceAdapter, fixtures: Fixture[]): Promise { return [ ...expressionFixtures(fixtures).map((f) => runFixture(adapter, f)), - ...projectFixtures(fixtures).map((f) => runProjectFixture(adapter, f)), + ...(await Promise.all(projectFixtures(fixtures).map((f) => runProjectFixture(adapter, f)))), ]; } /** #24 — a whole-build fixture. J3's edges are invisible in any single file, so this is the only shape that can test them. */ -export function runProjectFixture(adapter: ConformanceAdapter, f: ProjectFixture): FixtureReport { +export async function runProjectFixture(adapter: ConformanceAdapter, f: ProjectFixture): Promise { const base = { fixture: f.id, adapter: adapter.name }; if (!adapter.foldProject) return { ...base, pass: true, skipped: "no project entry", failures: [] }; - const r = adapter.foldProject(f.files, f.host ? requireHost(f.host) : undefined); + const r = await adapter.foldProject(f.files, f.host ? requireHost(f.host) : undefined); if (r === "unavailable") return { ...base, pass: true, skipped: "project entry unavailable", failures: [] }; const failures: string[] = []; for (const [path, want] of Object.entries(f.verdicts)) { @@ -63,16 +63,25 @@ export function runProjectFixture(adapter: ConformanceAdapter, f: ProjectFixture } /** #11 — two implementations must agree on every fixture, independently of what the fixture expects. */ -export function compareAdapters(a: ConformanceAdapter, b: ConformanceAdapter, fixtures: Fixture[]): string[] { +export async function compareAdapters(a: ConformanceAdapter, b: ConformanceAdapter, fixtures: Fixture[]): Promise { const dis: string[] = []; for (const f of projectFixtures(fixtures)) { if (!a.foldProject || !b.foldProject) continue; const host = f.host ? requireHost(f.host) : undefined; - const ra = a.foldProject(f.files, host), rb = b.foldProject(f.files, host); + const [ra, rb] = await Promise.all([a.foldProject(f.files, host), b.foldProject(f.files, host)]); if (ra === "unavailable" || rb === "unavailable") continue; for (const path of new Set([...Object.keys(ra.verdicts), ...Object.keys(rb.verdicts)])) { const va = ra.verdicts[path]?.kind ?? "absent", vb = rb.verdicts[path]?.kind ?? "absent"; if (va !== vb) dis.push(`${f.id} ${path}: ${a.name} ${va}, ${b.name} ${vb}`); + // The verdict alone cannot tell a seed from a taint casualty, and both of + // those are "run". Where both implementations report the tentative + // verdict and the edge, compare them: that is where J3 actually lives. + if (ra.tentative && rb.tentative && ra.tentative[path] !== rb.tentative[path]) { + dis.push(`${f.id} ${path}: tentative — ${a.name} ${ra.tentative[path] ?? "none"}, ${b.name} ${rb.tentative[path] ?? "none"}`); + } + if (ra.taintedBy && rb.taintedBy && ra.taintedBy[path] !== rb.taintedBy[path]) { + dis.push(`${f.id} ${path}: tainted by — ${a.name} ${ra.taintedBy[path] ?? "nothing"}, ${b.name} ${rb.taintedBy[path] ?? "nothing"}`); + } } } for (const f of expressionFixtures(fixtures)) {