Compare whole-build fixtures against chant (#62)

packages/conformance/src/adapter.ts

@@ -32,9 +32,14 @@ export interface ConformanceAdapter {
   foldExport(source: string, exportName: string): FoldResult;
   /**
    * J2 + J3 over a whole build: every file's final verdict. Optional, and
-   * "unavailable" when the implementation exposes no project-level entry —
-   * chant's public API is per-file, so its project fixtures are reported
-   * skipped rather than silently passing.
+   * "unavailable" when the implementation cannot answer — no whole-build
+   * entry at all, or a host it has no way to install. Such a fixture is
+   * reported skipped rather than silently passing.
+   *
+   * May be async: a real implementation resolves modules from a filesystem.
    */
-  foldProject?(files: Map<string, string>, host?: ConformanceHost): ProjectResult | "unavailable";
+  foldProject?(
+    files: Map<string, string>,
+    host?: ConformanceHost,
+  ): ProjectResult | "unavailable" | Promise<ProjectResult | "unavailable">;
 }

packages/conformance/src/adapters/chant.ts

@@ -4,9 +4,12 @@
  * — `findSubsetViolation` for the shape half. If an older chant is pinned the
  * adapter reports shape "unavailable" rather than guessing.
  */
+import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
 import * as ts from "typescript";
 import * as chant from "@intentius/chant";
-import type { ConformanceAdapter } from "../adapter";
+import type { ConformanceAdapter, ProjectResult, ProjectVerdict } from "../adapter";
 
 function exportInitializer(sf: ts.SourceFile, name: string): ts.Expression | undefined {
   for (const st of sf.statements) {
@@ -18,6 +21,53 @@ function exportInitializer(sf: ts.SourceFile, name: string): ts.Expression | und
 const parse = (src: string) => ts.createSourceFile("fixture.ts", src, ts.ScriptTarget.Latest, true);
 const shapeFn = (chant as unknown as { findSubsetViolation?: (n: ts.Node) => { node: ts.Node; ruleId: string; message: string } | undefined }).findSubsetViolation;
 
+/**
+ * The whole-build entry, chant-v0.71.0+ (chant#2408). Absent on an older pin,
+ * in which case the project fixtures report "unavailable" rather than passing
+ * vacuously.
+ */
+type ChantVerdict = {
+  verdict: "fold" | "run";
+  tentative: "fold" | "run";
+  reason?: string;
+  taintedBy?: { from: string; kind: "importer" | "capture" };
+  exports?: ReadonlyMap<string, unknown>;
+};
+const projectFn = (
+  chant as unknown as {
+    foldProject?: (files: readonly string[], intrinsics?: readonly unknown[]) => Promise<Map<string, ChantVerdict>>;
+  }
+).foldProject;
+
+/** chant resolves modules from disk, so a fixture's sources are written out and the paths handed over. */
+async function foldOnDisk(files: Map<string, string>): Promise<ProjectResult> {
+  const root = mkdtempSync(join(tmpdir(), "tsad-conformance-"));
+  try {
+    const paths: string[] = [];
+    for (const [rel, source] of files) {
+      const abs = join(root, rel);
+      mkdirSync(dirname(abs), { recursive: true });
+      writeFileSync(abs, source, "utf8");
+      paths.push(abs);
+    }
+    const verdicts = await projectFn!(paths, []);
+    const out: ProjectResult = { verdicts: {}, tentative: {}, taintedBy: {} };
+    const relOf = (abs: string) => abs.slice(root.length + 1).split(/[\\/]/).join("/");
+    for (const [abs, v] of verdicts) {
+      const key = relOf(abs);
+      out.verdicts[key] =
+        v.verdict === "fold"
+          ? ({ kind: "fold", exports: Object.fromEntries(v.exports ?? new Map()) } as ProjectVerdict)
+          : ({ kind: "run", reason: v.reason ?? "tainted" } as ProjectVerdict);
+      out.tentative![key] = v.tentative;
+      if (v.taintedBy) out.taintedBy![key] = relOf(v.taintedBy.from);
+    }
+    return out;
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+}
+
 export const chantAdapter: ConformanceAdapter = {
   name: `chant@${(chant as unknown as { VERSION?: string }).VERSION ?? "0.69.1"}`,
   shape(source, exportName) {
@@ -28,6 +78,14 @@ export const chantAdapter: ConformanceAdapter = {
     const { line, character } = sf.getLineAndCharacterOfPosition(v.node.getStart());
     return { accepted: false, rule: v.ruleId, line: line + 1, column: character + 1, message: v.message };
   },
+  async foldProject(files, host) {
+    if (!projectFn) return "unavailable";
+    // A named host's entity classes come from a package chant cannot resolve,
+    // and its entry takes an intrinsic registry rather than a whole host, so a
+    // host-dependent fixture is not answerable here.
+    if (host) return "unavailable";
+    return foldOnDisk(files);
+  },
   foldExport(source, exportName) {
     const sf = parse(source); const init = exportInitializer(sf, exportName);
     if (!init) return { ok: false, line: 1, column: 1, message: `no export named ${exportName}` };

packages/conformance/src/chant-agreement.test.ts

@@ -2,29 +2,45 @@
 import { describe, test, expect } from "vitest";
 import { join, dirname } from "node:path";
 import { fileURLToPath } from "node:url";
-import { loadFixtures, runFixtures, compareAdapters, expressionFixtures, projectFixtures } from "./index";
+import { loadFixtures, runFixtures, runProjectFixture, compareAdapters, expressionFixtures, projectFixtures } from "./index";
 import { chantAdapter } from "./adapters/chant";
 import { referenceAdapter } from "@intentius/tsad-reference";
 
 const fixtures = loadFixtures(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "..", "spec", "fixtures"));
 
 describe("chant cross-check (#11)", () => {
-  test("chant passes every fixture through its public fold API", () => {
-    const failed = runFixtures(chantAdapter, fixtures).filter((r) => !r.pass).map((r) => `${r.fixture}: ${r.failures.join("; ")}`);
+  test("chant passes every fixture through its public fold API", async () => {
+    const failed = (await runFixtures(chantAdapter, fixtures)).filter((r) => !r.pass).map((r) => `${r.fixture}: ${r.failures.join("; ")}`);
     expect(failed, failed.join("\n")).toEqual([]);
   });
-  test("chant and the reference implementation agree on every fixture", () => {
-    const dis = compareAdapters(referenceAdapter, chantAdapter, fixtures);
+  test("chant and the reference implementation agree on every fixture", async () => {
+    const dis = await compareAdapters(referenceAdapter, chantAdapter, fixtures);
     expect(dis, dis.join("\n")).toEqual([]);
   });
-  test("chant has no project-level entry, so the project fixtures are skipped, not silently passed (#24)", () => {
-    // chant's public API folds one expression or one module's resources; J3 is
-    // not reachable through it. Asserted so the day it is, this test fails and
-    // the cross-check is extended rather than forgotten. See chant#2408.
-    expect(projectFixtures(fixtures).length).toBeGreaterThan(0);
-    expect(chantAdapter.foldProject).toBeUndefined();
+  test("the whole-build fixtures reach chant, or say why not (#62)", async () => {
+    // Two reasons a project fixture can be skipped, and both must be visible.
+    // An older pin has no whole-build entry at all (chant#2408); a fixture
+    // naming a host asks for entity classes from a package chant cannot
+    // resolve. Anything else has to be answered.
+    const projects = projectFixtures(fixtures);
+    expect(projects.length).toBeGreaterThan(0);
+    const reports = await Promise.all(projects.map((f) => runProjectFixture(chantAdapter, f)));
+    const skipped = reports.filter((r) => r.skipped).map((r) => r.fixture);
+    const hosted = new Set(projects.filter((f) => f.host).map((f) => f.id));
+    const unexpected = skipped.filter((id) => !hosted.has(id));
+    if (chantAdapter.foldProject && (await chantAdapter.foldProject(new Map([["a.ts", "export const a = 1;"]]))) !== "unavailable") {
+      expect(unexpected, `skipped without a host to explain it:\n${unexpected.join("\n")}`).toEqual([]);
+    } else {
+      // The pin predates chant#2408. Recorded, not silently tolerated.
+      expect(skipped.length, "an older pin skips every whole-build fixture").toBe(projects.length);
+    }
+  });
+
+  test("chant and the reference agree on every whole-build fixture chant can answer (#62)", async () => {
+    const dis = await compareAdapters(referenceAdapter, chantAdapter, projectFixtures(fixtures));
+    expect(dis, dis.join("\n")).toEqual([]);
   });
-  test("chant's shape classifier is available (chant-v0.64.0+, chant#2362) and agrees on every fixture", () => {
+  test("chant's shape classifier is available (chant-v0.64.0+, chant#2362) and agrees on every fixture", async () => {
     // The pinned chant carries the export, so "unavailable" would mean the adapter
     // silently stopped comparing the shape half — a real regression, asserted.
     for (const f of expressionFixtures(fixtures)) {

packages/conformance/src/runner.test.ts

@@ -8,18 +8,18 @@ const fixturesDir = join(dirname(fileURLToPath(import.meta.url)), "..", "..", ".
 
 describe("conformance runner (#7) over the reference implementation (#10)", () => {
   const fixtures = loadFixtures(fixturesDir);
-  test("fixtures load", () => { expect(fixtures.length).toBeGreaterThan(0); for (const f of fixtures) expect(f.rules.length).toBeGreaterThan(0); });
-  test("the reference implementation passes every fixture", () => {
-    const reports = runFixtures(referenceAdapter, fixtures);
+  test("fixtures load", async () => { expect(fixtures.length).toBeGreaterThan(0); for (const f of fixtures) expect(f.rules.length).toBeGreaterThan(0); });
+  test("the reference implementation passes every fixture", async () => {
+    const reports = await runFixtures(referenceAdapter, fixtures);
     const failed = reports.filter((r) => !r.pass).map((r) => `${r.fixture}: ${r.failures.join("; ")}`);
     expect(failed, failed.join("\n")).toEqual([]);
   });
-  test("the project fixtures are actually exercised, not all skipped (#24)", () => {
-    const reports = runFixtures(referenceAdapter, fixtures).filter((r) => projectFixtures(fixtures).some((p) => p.id === r.fixture));
+  test("the project fixtures are actually exercised, not all skipped (#24)", async () => {
+    const reports = (await runFixtures(referenceAdapter, fixtures)).filter((r) => projectFixtures(fixtures).some((p) => p.id === r.fixture));
     expect(reports.length).toBe(projectFixtures(fixtures).length);
     expect(reports.filter((r) => r.skipped)).toEqual([]);
   });
-  test("a stub adapter that runs every file fails the project fixtures that expect a fold", () => {
+  test("a stub adapter that runs every file fails the project fixtures that expect a fold", async () => {
     // The control the fixtures themselves describe: falling back everywhere is
     // sound and useless, and must not pass. Without this, F-Taint's "least set"
     // would be untested.
@@ -28,12 +28,12 @@ describe("conformance runner (#7) over the reference implementation (#10)", () =
       foldExport: () => ({ ok: false as const, line: 1, column: 1, message: "runs" }),
       foldProject: (files: Map<string, string>) => ({ verdicts: Object.fromEntries([...files.keys()].map((p) => [p, { kind: "run" as const, reason: "runs" }])) }),
     };
-    const failed = runFixtures(stub, fixtures).filter((r) => !r.pass && projectFixtures(fixtures).some((p) => p.id === r.fixture));
+    const failed = (await runFixtures(stub, fixtures)).filter((r) => !r.pass && projectFixtures(fixtures).some((p) => p.id === r.fixture));
     expect(failed.length).toBeGreaterThan(0);
   });
-  test("a stub adapter that folds everything to null fails the reject fixture and the value fixtures", () => {
+  test("a stub adapter that folds everything to null fails the reject fixture and the value fixtures", async () => {
     const stub = { name: "stub", shape: () => ({ accepted: true } as const), foldExport: () => ({ ok: true as const, value: null }) };
-    const reports = runFixtures(stub, fixtures);
+    const reports = await runFixtures(stub, fixtures);
     expect(reports.some((r) => !r.pass)).toBe(true);
   });
 });

packages/conformance/src/runner.ts

@@ -22,18 +22,18 @@ export function runFixture(adapter: ConformanceAdapter, f: ExpressionFixture): F
   }
   return { fixture: f.id, adapter: adapter.name, pass: failures.length === 0, skipped, failures };
 }
-export function runFixtures(adapter: ConformanceAdapter, fixtures: Fixture[]): FixtureReport[] {
+export async function runFixtures(adapter: ConformanceAdapter, fixtures: Fixture[]): Promise<FixtureReport[]> {
   return [
     ...expressionFixtures(fixtures).map((f) => runFixture(adapter, f)),
-    ...projectFixtures(fixtures).map((f) => runProjectFixture(adapter, f)),
+    ...(await Promise.all(projectFixtures(fixtures).map((f) => runProjectFixture(adapter, f)))),
   ];
 }
 
 /** #24 — a whole-build fixture. J3's edges are invisible in any single file, so this is the only shape that can test them. */
-export function runProjectFixture(adapter: ConformanceAdapter, f: ProjectFixture): FixtureReport {
+export async function runProjectFixture(adapter: ConformanceAdapter, f: ProjectFixture): Promise<FixtureReport> {
   const base = { fixture: f.id, adapter: adapter.name };
   if (!adapter.foldProject) return { ...base, pass: true, skipped: "no project entry", failures: [] };
-  const r = adapter.foldProject(f.files, f.host ? requireHost(f.host) : undefined);
+  const r = await adapter.foldProject(f.files, f.host ? requireHost(f.host) : undefined);
   if (r === "unavailable") return { ...base, pass: true, skipped: "project entry unavailable", failures: [] };
   const failures: string[] = [];
   for (const [path, want] of Object.entries(f.verdicts)) {
@@ -63,16 +63,25 @@ export function runProjectFixture(adapter: ConformanceAdapter, f: ProjectFixture
 }
 
 /** #11 — two implementations must agree on every fixture, independently of what the fixture expects. */
-export function compareAdapters(a: ConformanceAdapter, b: ConformanceAdapter, fixtures: Fixture[]): string[] {
+export async function compareAdapters(a: ConformanceAdapter, b: ConformanceAdapter, fixtures: Fixture[]): Promise<string[]> {
   const dis: string[] = [];
   for (const f of projectFixtures(fixtures)) {
     if (!a.foldProject || !b.foldProject) continue;
     const host = f.host ? requireHost(f.host) : undefined;
-    const ra = a.foldProject(f.files, host), rb = b.foldProject(f.files, host);
+    const [ra, rb] = await Promise.all([a.foldProject(f.files, host), b.foldProject(f.files, host)]);
     if (ra === "unavailable" || rb === "unavailable") continue;
     for (const path of new Set([...Object.keys(ra.verdicts), ...Object.keys(rb.verdicts)])) {
       const va = ra.verdicts[path]?.kind ?? "absent", vb = rb.verdicts[path]?.kind ?? "absent";
       if (va !== vb) dis.push(`${f.id} ${path}: ${a.name} ${va}, ${b.name} ${vb}`);
+      // The verdict alone cannot tell a seed from a taint casualty, and both of
+      // those are "run". Where both implementations report the tentative
+      // verdict and the edge, compare them: that is where J3 actually lives.
+      if (ra.tentative && rb.tentative && ra.tentative[path] !== rb.tentative[path]) {
+        dis.push(`${f.id} ${path}: tentative — ${a.name} ${ra.tentative[path] ?? "none"}, ${b.name} ${rb.tentative[path] ?? "none"}`);
+      }
+      if (ra.taintedBy && rb.taintedBy && ra.taintedBy[path] !== rb.taintedBy[path]) {
+        dis.push(`${f.id} ${path}: tainted by — ${a.name} ${ra.taintedBy[path] ?? "nothing"}, ${b.name} ${rb.taintedBy[path] ?? "nothing"}`);
+      }
     }
   }
   for (const f of expressionFixtures(fixtures)) {