diff --git a/bin/fm-crosscheck.py b/bin/fm-crosscheck.py index f09fd37eff0..c25d0d684ce 100755 --- a/bin/fm-crosscheck.py +++ b/bin/fm-crosscheck.py @@ -5,7 +5,6 @@ import argparse import base64 -import binascii import copy import datetime as dt import fcntl @@ -63,40 +62,179 @@ MAX_PROJECTED_EVENTS = 8 MAX_REVIEW_ITEMS = 32 MAX_EVIDENCE_ITEMS = 32 -TEST_RUNNERS = { + + +class MutationRunnerPolicy: + """One measured declaration for a runner allowed to certify a fix.""" + + __slots__ = ( + "invocations", + "gate_arguments", + "selector_mode", + "absolute_test_path", + "node_project_cwd", + "report_format", + "body_probe", + "runtime_error_field", + "non_execution_exits", + "runtime_version", + "minimum_node_version", + "source_digests", + "measurement", + ) + + def __init__( + self, + *, + invocations: tuple[tuple[str, ...], ...], + gate_arguments: tuple[str, ...], + selector_mode: str, + absolute_test_path: bool, + node_project_cwd: bool, + report_format: str | None, + body_probe: str | None, + runtime_error_field: str, + non_execution_exits: tuple[tuple[int, str], ...], + runtime_version: str | None, + minimum_node_version: tuple[int, int, int] | None, + source_digests: tuple[tuple[str, str], ...], + measurement: str, + ) -> None: + self.invocations = invocations + self.gate_arguments = gate_arguments + self.selector_mode = selector_mode + self.absolute_test_path = absolute_test_path + self.node_project_cwd = node_project_cwd + self.report_format = report_format + self.body_probe = body_probe + self.runtime_error_field = runtime_error_field + self.non_execution_exits = non_execution_exits + self.runtime_version = runtime_version + self.minimum_node_version = minimum_node_version + self.source_digests = source_digests + self.measurement = measurement + + +class TestRun: + """Resolved proof command and the project directory it must run from.""" + + __slots__ = ("argv", "cwd", "body_evidence", "body_probe", "environment") + + def __init__( + self, + argv: tuple[str, ...], + cwd: Path, + body_evidence: bool = False, + body_probe: Path | None = None, + environment: tuple[tuple[str, str], ...] = (), + ) -> None: + self.argv = argv + self.cwd = cwd + self.body_evidence = body_evidence + self.body_probe = body_probe + self.environment = environment + + +# This registry is the sole declaration point for mutation-proof runners. +# Adding one requires the measurement procedure in docs/crosscheck.md; an +# ordinary approved test runner remains unable to certify until it appears +# here. Gate-owned arguments are part of the measured invocation and reviewer +# arguments remain forbidden. +MUTATION_RUNNER_POLICIES = { + "pytest": MutationRunnerPolicy( + # Order is load-bearing: uv comes first because a bare pytest inside a + # uv project can resolve against a different environment. The module + # invocation follows, with the bare binary as the final fallback. + invocations=( + ("uv", "run", "pytest"), + ("python3", "-m", "pytest"), + ("pytest",), + ), + gate_arguments=(), + selector_mode="native", + absolute_test_path=False, + node_project_cwd=False, + report_format=None, + body_probe=None, + runtime_error_field="not-applicable", + non_execution_exits=( + (2, "collection was interrupted"), + (3, "the runner hit an internal error"), + (4, "the runner rejected its command line"), + (5, "no test matched the named selector"), + ), + runtime_version=None, + minimum_node_version=None, + source_digests=(), + measurement="pytest 9.1.1 on 2026-08-05", + ), + "jest": MutationRunnerPolicy( + invocations=(("jest",),), + gate_arguments=("--json", "--runInBand", "--runTestsByPath"), + selector_mode="test-name-pattern", + absolute_test_path=True, + node_project_cwd=True, + report_format="jest-compatible-json", + body_probe="jest-global-wrapper", + runtime_error_field="required-zero", + non_execution_exits=(), + runtime_version="29.7.0", + minimum_node_version=None, + source_digests=( + ( + "circusRun", + "e0ba3e46a59b751d7cc4ab5b6c00f27baa54d35b326c5dbc14404eb725fa8477", + ), + ( + "circusUtils", + "52b9ee6ae1b3bea12be70fdbcf2e8865b1f327b955d76cc560b0cfb22e61e70f", + ), + ), + measurement="Jest 29.7.0 on 2026-08-09", + ), + "vitest": MutationRunnerPolicy( + invocations=(("vitest",),), + gate_arguments=("run", "--reporter=json"), + selector_mode="test-name-pattern", + absolute_test_path=False, + node_project_cwd=True, + report_format="jest-compatible-json", + body_probe="vitest-runner", + runtime_error_field="absent", + non_execution_exits=(), + runtime_version="4.1.5", + minimum_node_version=(20, 6, 0), + source_digests=( + ( + "forkLauncher", + "d991d80584acd5fc622aefce5f907feff6c531059165a05d75c66e3ed8697d79", + ), + ), + measurement="Vitest 4.1.5 on Node 20.20.2 on 2026-08-10", + ), +} +GENERAL_TEST_RUNNERS = { "bash", "bun", "direct", - "jest", "node", "php", - "pytest", "python", "python3", "rspec", "ruby", "sh", - "vitest", "zsh", } -FILE_TEST_RUNNERS = TEST_RUNNERS - {"direct", "jest", "pytest", "rspec", "vitest"} -# Runners whose command line accepts a `path::selector` node id. Every other -# approved runner is handed a plain file, so a selector there is a reviewer -# mistake the gate must name rather than silently drop. -NODE_ID_RUNNERS = {"pytest"} -# How an approved runner NAME becomes an argv prefix, when the name alone does -# not identify a working invocation. Order is load-bearing: uv comes first -# because inside a uv project a bare `pytest` can exist on PATH and resolve -# against a different environment than the repository uses, so finding it first -# would run the named test under an interpreter the project never selected. -# `python3 -m pytest` follows because it reaches a pytest installed into the -# interpreter itself, and the bare binary is the last resort. -RUNNER_INVOCATIONS: dict[str, tuple[tuple[str, ...], ...]] = { - "pytest": ( - ("uv", "run", "pytest"), - ("python3", "-m", "pytest"), - ("pytest",), - ), +TEST_RUNNERS = GENERAL_TEST_RUNNERS | set(MUTATION_RUNNER_POLICIES) +FILE_TEST_RUNNERS = GENERAL_TEST_RUNNERS - {"direct", "rspec"} +SELECTOR_TEST_RUNNERS = { + runner + for runner, policy in MUTATION_RUNNER_POLICIES.items() + if policy.selector_mode in {"native", "test-name-pattern"} +} +RUNNER_INVOCATIONS = { + runner: policy.invocations for runner, policy in MUTATION_RUNNER_POLICIES.items() } # sandbox-exec reports a failed execvp of its target with EX_OSERR and this # marker. The target never ran, so its exit status says nothing about the test. @@ -105,33 +243,12 @@ # POSIX shells report an unfound command with this status; the command's own # exit statuses never reach the gate in that case. SHELL_COMMAND_NOT_FOUND_EXIT = 127 -# Exit statuses that mean an approved runner started but never executed the -# named test. They are not test outcomes in either direction: they can neither -# condemn a baseline run nor vindicate a mutated one. Every entry is measured -# against the runner itself; a guessed status would reinstate exactly the -# misreading this table exists to prevent, so an exit-status-inferred route is -# absent until its non-execution has been observed. Jest does not use this table: -# its separate positive-execution route parses the runner's JSON test counts. -RUNNER_NON_EXECUTION_EXITS: dict[str, dict[int, str]] = { - "pytest": { - 2: "collection was interrupted", - 3: "the runner hit an internal error", - 4: "the runner rejected its command line", - 5: "no test matched the named selector", - }, -} -JAVASCRIPT_IMPLEMENTATION_SUFFIXES = { - ".cjs", - ".cts", - ".js", - ".jsx", - ".mjs", - ".mts", - ".ts", - ".tsx", -} -JAVASCRIPT_RUNNERS = {"jest", "vitest"} -# The classified statuses above are the runner's DEFAULT exit semantics, and +# The classified pytest statuses above are the runner's default exit semantics. +# Jest and Vitest instead use their measured machine reports: a status is a test +# outcome only when the report records an executed assertion, and a mutated +# failure must record a failed assertion. Missing, malformed, empty, skipped- +# only, and runtime-error reports are non-executions regardless of exit status. +# The classified semantics are runner-specific, and # ambient variables can rewrite them: pytest documents PYTEST_ADDOPTS as being # appended to the command line, so an operator with # `PYTEST_ADDOPTS=--continue-on-collection-errors` exported turns a mutation @@ -191,6 +308,12 @@ def fail(message: str) -> NoReturn: raise CrosscheckError(message) +def non_execution(label: str, reason: str) -> NoReturn: + """Refuse a proof without letting a tooling failure read as a test result.""" + + fail(f"{label} NON-EXECUTION: {reason}") + + def tool_fail(message: str) -> NoReturn: raise CrosscheckToolError(message) @@ -961,30 +1084,37 @@ def proof_environment() -> dict[str, str]: } +def proof_run_environment(run: TestRun) -> dict[str, str]: + environment = proof_environment() + environment.update(dict(run.environment)) + return environment + + def write_neutral_runner_config(root: Path) -> None: - """End the runner's upward config search inside a directory the gate owns. + """End runner upward config searches inside a directory the gate owns. pytest's locate_config walks every parent of its target to the filesystem - root looking for pytest.ini, tox.ini, setup.cfg or pyproject.toml, and - stops at the first one it finds. Operator machine state above this root - could therefore set options for every proof run: measured on pytest 9.1.1, - an ancestor `addopts = --continue-on-collection-errors` turned a mutation - that broke collection from exit 2 into exit 1, which the gate reads as a - caught regression. A neutral file here terminates that walk, and it - neutralises every ini setting from above, not just addopts. + root looking for pytest.ini, tox.ini, setup.cfg or pyproject.toml. Jest also + searches upward from its working directory for project configuration. + Operator machine state above this root could therefore set options for + every proof run. Measured on pytest 9.1.1, an ancestor + `addopts = --continue-on-collection-errors` turned a mutation that broke + collection from exit 2 into exit 1, which the gate reads as a caught + regression. Neutral files here terminate those searches inside the root the + gate owns. Vitest's neutral config is written at the same boundary so a + future upward search cannot silently widen the accepted surface. Both the proof checkouts and the review checkout live under this root, so - one file covers the mutation proofs and the reproduction re-execution - alike; the boundary is the root the gate owns, not any child of it. - - The reviewed repository's own config still wins, because it sits closer to - the named test. That surface is deliberately accepted. The measured cost of - this file: for a repository carrying no pytest config at all, rootdir - becomes this temporary root rather than the checkout, which widens conftest - discovery by this one empty gate-owned directory. + these files cover mutation proofs and reproduction re-execution alike. The + reviewed repository's own closer config still wins; that surface is + deliberately accepted. For a repository carrying no pytest config, + rootdir becomes this temporary root rather than the checkout, widening + conftest discovery by this one empty gate-owned directory. """ (root / "pytest.ini").write_text("[pytest]\n", encoding="utf-8") + (root / "jest.config.cjs").write_text("module.exports = {};\n", encoding="utf-8") + (root / "vitest.config.mjs").write_text("export default {};\n", encoding="utf-8") def git(cwd: Path, *arguments: str, timeout: float = 60) -> str: @@ -1243,13 +1373,7 @@ def evidence_command_timeout( def test_file_path(test_path: str, label: str) -> str: - """Return the repository file a test selector names. - - A named test may be a plain repository path or a runner node id such as - `tests/test_login.py::TestSession::test_expiry`. Only the part before the - first `::` is a filesystem path; every path-shaped check works on that part - while the caller keeps the full value for the runner command line. - """ + """Return the repository file named before an optional `::` selector.""" file_part = test_path.split("::", 1)[0] require( @@ -1259,13 +1383,27 @@ def test_file_path(test_path: str, label: str) -> str: return file_part -def require_supported_selector(test_path: str, runner: str, label: str) -> None: +def test_selector(test_path: str, label: str) -> str | None: + """Return a nonempty runner selector from the structured test path.""" + if "::" not in test_path: + return None + selector = test_path.split("::", 1)[1] + require( + bool(selector) and selector == selector.strip(), + f"{label}.test_path must name a nonempty selector after `::`", + ) + return selector + + +def require_supported_selector(test_path: str, runner: str, label: str) -> None: + selector = test_selector(test_path, label) + if selector is None: return require( - runner in NODE_ID_RUNNERS, - f"{label}.test_path uses a `::` node id, which {runner} does not accept; " - f"approved node-id runners: {', '.join(sorted(NODE_ID_RUNNERS))}", + runner in SELECTOR_TEST_RUNNERS, + f"{label}.test_path uses a `::` selector, which {runner} does not accept; " + f"approved selector runners: {', '.join(sorted(SELECTOR_TEST_RUNNERS))}", ) @@ -1421,789 +1559,35 @@ def uv_project_for(checkout: Path, test_path: str) -> Path | None: directory = directory.parent -def nearest_package_project(checkout: Path, relative_path: str) -> Path | None: - """Return the nearest package.json root governing one tracked path.""" +def node_project_for(checkout: Path, test_path: str) -> Path: + """Return the nearest tracked-shape Node project governing a named test. + + Jest and Vitest resolve their repository configuration and package-relative + imports from the package directory, not necessarily a monorepo root. A + package.json symlink is ignored so choosing the working directory cannot be + redirected outside the proof checkout. The checkout root is the explicit + fallback for repositories without a package manifest. + """ checkout = checkout.resolve() - candidate = (checkout / relative_path).resolve() - if not candidate.is_relative_to(checkout): - return None - directory = candidate.parent + directory = (checkout / test_path).resolve().parent + require( + directory.is_relative_to(checkout), + f"named test resolves outside proof checkout: {test_path}", + ) while True: - if (directory / "package.json").is_file(): + manifest = directory / "package.json" + try: + manifest_mode = manifest.lstat().st_mode + except OSError: + manifest_mode = 0 + if stat.S_ISREG(manifest_mode): return directory if directory == checkout: - return None + return checkout directory = directory.parent -def javascript_mutation_route( - review_dir: Path, - changed: list[str], - test_file: str, - runner: str, - label: str, -) -> Path | None: - """Select a JavaScript test system from the implementation paths themselves.""" - - javascript_paths = [ - path - for path in changed - if Path(path).suffix.lower() in JAVASCRIPT_IMPLEMENTATION_SUFFIXES - ] - if not javascript_paths: - return None - non_javascript = sorted(set(changed) - set(javascript_paths)) - if non_javascript: - cannot_certify( - f"{label} CANNOT-CERTIFY: one mutation spans JavaScript/TypeScript " - "and another implementation system, so no single governed test route " - "can certify it: " - + ", ".join(non_javascript) - ) - if Path(test_file).suffix.lower() not in JAVASCRIPT_IMPLEMENTATION_SUFFIXES: - cannot_certify( - f"{label} CANNOT-CERTIFY: JavaScript/TypeScript implementation must " - f"name a tracked JavaScript/TypeScript test, not {test_file}" - ) - governed_paths = [*javascript_paths, test_file] - resolved_projects = [ - nearest_package_project(review_dir, path) for path in governed_paths - ] - if any(project is None for project in resolved_projects): - cannot_certify( - f"{label} CANNOT-CERTIFY: every changed JavaScript/TypeScript path " - "and the named test must resolve to a tracked package.json project" - ) - projects = { - project.resolve() for project in resolved_projects if project is not None - } - if len(projects) != 1: - cannot_certify( - f"{label} CANNOT-CERTIFY: changed implementation and named test do " - "not resolve to one tracked package.json project" - ) - project = next(iter(projects)) - package_path = project / "package.json" - try: - package = read_bounded_json( - package_path, - maximum_bytes=1024 * 1024, - maximum_items=4096, - maximum_string_bytes=1024 * 1024, - ) - except BoundedIOError as exc: - cannot_certify( - f"{label} CANNOT-CERTIFY: package metadata is unreadable at " - f"{package_path}: {exc}" - ) - if not isinstance(package, dict): - cannot_certify( - f"{label} CANNOT-CERTIFY: package metadata is not an object at " - f"{package_path}" - ) - dependencies: dict[str, Any] = {} - for field in ("dependencies", "devDependencies"): - value = package.get(field) - if isinstance(value, dict): - dependencies.update(value) - scripts = package.get("scripts") - test_script = scripts.get("test", "") if isinstance(scripts, dict) else "" - declared = { - candidate - for candidate in JAVASCRIPT_RUNNERS - if candidate in dependencies - or re.search(rf"(?:^|[ /]){re.escape(candidate)}(?:$|[ ])", str(test_script)) - } - scripted = [ - candidate - for candidate in sorted(declared) - if re.search(rf"(?:^|[ /]){re.escape(candidate)}(?:$|[ ])", str(test_script)) - ] - if len(scripted) == 1: - governed_runner = scripted[0] - elif len(declared) == 1: - governed_runner = next(iter(declared)) - else: - cannot_certify( - f"{label} CANNOT-CERTIFY: {package_path} does not declare one " - "unambiguous Jest or Vitest test system" - ) - if runner != governed_runner: - cannot_certify( - f"{label} CANNOT-CERTIFY: changed JavaScript/TypeScript is governed " - f"by {governed_runner}, but the proof named {runner}" - ) - if governed_runner != "jest": - cannot_certify( - f"{label} CANNOT-CERTIFY: {governed_runner} governs the changed " - "JavaScript/TypeScript package, but this gate has no positive " - f"mutation-execution protocol for {governed_runner}" - ) - return project.relative_to(review_dir.resolve()) - - -def declared_node_major(project: Path) -> int | None: - try: - package = read_bounded_json( - project / "package.json", - maximum_bytes=1024 * 1024, - maximum_items=4096, - maximum_string_bytes=1024 * 1024, - ) - except BoundedIOError: - return None - if not isinstance(package, dict): - return None - engines = package.get("engines") - declaration = engines.get("node") if isinstance(engines, dict) else None - if not isinstance(declaration, str): - return None - match = re.search(r"(?:^|[^0-9])(\d+)(?:\.|x|$)", declaration) - return int(match.group(1)) if match is not None else None - - -def node_bin_for_project(project: Path, label: str) -> Path: - major = declared_node_major(project) - ambient = shutil.which("node") - if ambient is not None: - version = run_command( - [ambient, "--version"], - cwd=project, - timeout=30, - description=f"{label} Node version probe", - ) - match = re.fullmatch(r"v(\d+)\.[0-9]+\.[0-9]+", version.stdout.strip()) - if version.returncode == 0 and (major is None or (match and int(match.group(1)) == major)): - return Path(ambient).resolve().parent - if major is None: - cannot_certify( - f"{label} CANNOT-CERTIFY: no runnable Node interpreter is on PATH" - ) - candidates: list[tuple[tuple[int, int, int], Path]] = [] - homes = [ - Path.home() / ".nvm" / "versions" / "node", - Path.home() / ".local" / "share" / "mise" / "installs" / "node", - Path.home() / ".volta" / "tools" / "image" / "node", - ] - for root in homes: - if not root.is_dir(): - continue - for candidate in root.iterdir(): - match = re.fullmatch(rf"v?({major})\.(\d+)\.(\d+)", candidate.name) - node = candidate / "bin" / "node" - if match is not None and node.is_file() and os.access(node, os.X_OK): - candidates.append( - ((int(match.group(1)), int(match.group(2)), int(match.group(3))), node) - ) - if not candidates: - cannot_certify( - f"{label} CANNOT-CERTIFY: package requires Node {major}, but no " - "matching interpreter exists in the standard version-manager directories" - ) - return max(candidates)[1].resolve().parent - - -def npm_lock_package_name(lock_path: str) -> str | None: - parts = lock_path.split("/") - index = 0 - package_name: str | None = None - while index < len(parts): - if parts[index] != "node_modules": - return None - index += 1 - if index >= len(parts): - return None - first = parts[index] - index += 1 - if first.startswith("@"): - if index >= len(parts): - return None - package_name = f"{first}/{parts[index]}" - index += 1 - else: - package_name = first - if re.fullmatch( - r"(?:@[a-z0-9][a-z0-9._~-]*/)?[a-z0-9][a-z0-9._~-]*", - package_name, - ) is None: - return None - return package_name - - -def npm_lock_dependency_path( - packages: dict[str, Any], - package_path: str, - dependency: str, - label: str, - *, - required: bool = True, -) -> str | None: - dependency_parts = dependency.split("/") - current = package_path - candidates: list[str] = [] - while True: - candidates.append("/".join((current, "node_modules", *dependency_parts))) - marker = current.rfind("/node_modules/") - if marker < 0: - candidates.append("/".join(("node_modules", *dependency_parts))) - break - current = current[:marker] - resolved = [candidate for candidate in candidates if candidate in packages] - if not resolved: - if not required: - return None - cannot_certify( - f"{label} CANNOT-CERTIFY: Jest runtime dependency {dependency} from " - f"{package_path} has no lockfile package entry" - ) - selected = resolved[0] - if npm_lock_package_name(selected) != dependency: - cannot_certify( - f"{label} CANNOT-CERTIFY: Jest runtime dependency {dependency} has " - f"an ambiguous or noncanonical hoist at {selected}" - ) - return selected - - -def npm_runtime_dependency_fields( - package: dict[str, Any], package_path: str, label: str -) -> tuple[dict[str, str], set[str]]: - dependencies: dict[str, str] = {} - optional: set[str] = set() - for field in ("dependencies", "optionalDependencies", "peerDependencies"): - value = package.get(field) - if value is None: - continue - if not isinstance(value, dict): - cannot_certify( - f"{label} CANNOT-CERTIFY: {package_path} has malformed {field} " - "in the Jest runtime closure" - ) - for name, declaration in value.items(): - declaration_lower = declaration.lower() if isinstance(declaration, str) else "" - if ( - not isinstance(name, str) - or npm_lock_package_name(f"node_modules/{name}") != name - or not isinstance(declaration, str) - or not declaration.strip() - ): - cannot_certify( - f"{label} CANNOT-CERTIFY: {package_path} has an invalid " - f"Jest runtime dependency declaration in {field}" - ) - if declaration_lower.startswith( - ( - "file:", - "link:", - "workspace:", - "git:", - "git+", - "github:", - "http:", - "https:", - "./", - "../", - "/", - ) - ) or "github.com" in declaration_lower: - cannot_certify( - f"{label} CANNOT-CERTIFY: {package_path} declares Jest runtime " - f"dependency {name} from a local, linked, workspace, Git, or " - "URL source" - ) - previous = dependencies.get(name) - if previous is not None and previous != declaration: - cannot_certify( - f"{label} CANNOT-CERTIFY: {package_path} ambiguously declares " - f"Jest runtime dependency {name}" - ) - dependencies[name] = declaration - if field == "optionalDependencies": - optional.add(name) - peer_metadata = package.get("peerDependenciesMeta") - if peer_metadata is not None: - if not isinstance(peer_metadata, dict): - cannot_certify( - f"{label} CANNOT-CERTIFY: {package_path} has malformed " - "peerDependenciesMeta in the Jest runtime closure" - ) - for name, metadata in peer_metadata.items(): - if name not in dependencies or not isinstance(metadata, dict): - cannot_certify( - f"{label} CANNOT-CERTIFY: {package_path} has invalid optional " - "peer dependency metadata in the Jest runtime closure" - ) - if metadata.get("optional") is True: - optional.add(name) - return dependencies, optional - - -def npm_registry_package_version( - lock_path: str, entry: dict[str, Any], label: str -) -> str: - package_name = npm_lock_package_name(lock_path) - if package_name is None: - cannot_certify( - f"{label} CANNOT-CERTIFY: Jest runtime package has a noncanonical or " - f"path-escaping lockfile location at {lock_path}" - ) - if entry.get("link") is True: - cannot_certify( - f"{label} CANNOT-CERTIFY: Jest runtime package {package_name} is a " - f"local or linked lock entry at {lock_path}" - ) - version = entry.get("version") - resolved = entry.get("resolved") - integrity = entry.get("integrity") - if not ( - isinstance(version, str) - and re.fullmatch(r"[0-9]+[.][0-9]+[.][0-9]+(?:-[0-9A-Za-z.-]+)?", version) - and isinstance(resolved, str) - and isinstance(integrity, str) - ): - cannot_certify( - f"{label} CANNOT-CERTIFY: Jest runtime package {package_name} lacks " - "a registry version, resolved tarball, or integrity" - ) - parsed = urlsplit(resolved) - tarball_name = package_name.rsplit("/", 1)[-1] - expected_path = f"/{package_name}/-/{tarball_name}-{version}.tgz" - if not ( - parsed.scheme == "https" - and parsed.hostname == "registry.npmjs.org" - and parsed.username is None - and parsed.password is None - and parsed.port is None - and parsed.query == "" - and parsed.fragment == "" - and unquote(parsed.path).lower() == expected_path.lower() - ): - cannot_certify( - f"{label} CANNOT-CERTIFY: Jest runtime package {package_name} is not " - "resolved from its official npm registry tarball" - ) - algorithm, separator, encoded = integrity.partition("-") - try: - digest = base64.b64decode(encoded, validate=True) if separator else b"" - except (binascii.Error, ValueError): - digest = b"" - if algorithm != "sha512" or len(digest) != 64: - cannot_certify( - f"{label} CANNOT-CERTIFY: Jest runtime package {package_name} requires " - "a valid sha512 registry integrity" - ) - return version - - -def npm_jest_lock_provenance( - lockfile: Path, label: str -) -> dict[str, dict[str, Any]]: - try: - value = read_bounded_json( - lockfile, - maximum_bytes=16 * 1024 * 1024, - maximum_items=262_144, - maximum_string_bytes=1024 * 1024, - ) - except BoundedIOError as exc: - cannot_certify( - f"{label} CANNOT-CERTIFY: npm lockfile is unreadable at {lockfile}: {exc}" - ) - if not isinstance(value, dict) or value.get("lockfileVersion") not in {2, 3}: - cannot_certify( - f"{label} CANNOT-CERTIFY: npm Jest provenance requires a package-lock " - "version 2 or 3 object" - ) - packages = value.get("packages") - root = packages.get("") if isinstance(packages, dict) else None - entry = packages.get("node_modules/jest") if isinstance(packages, dict) else None - if not isinstance(root, dict) or not isinstance(entry, dict): - cannot_certify( - f"{label} CANNOT-CERTIFY: package-lock does not bind the root project " - "to a materialized node_modules/jest package" - ) - declarations: list[str] = [] - for field in ("dependencies", "devDependencies", "optionalDependencies"): - dependencies = root.get(field) - declaration = dependencies.get("jest") if isinstance(dependencies, dict) else None - if isinstance(declaration, str): - declarations.append(declaration.strip()) - if len(declarations) != 1 or not declarations[0]: - cannot_certify( - f"{label} CANNOT-CERTIFY: package-lock root must declare Jest exactly once" - ) - declaration = declarations[0].lower() - forbidden = ( - "file:", - "link:", - "workspace:", - "git:", - "git+", - "github:", - "http:", - "https:", - "./", - "../", - "/", - ) - if declaration.startswith(forbidden) or "github.com" in declaration: - cannot_certify( - f"{label} CANNOT-CERTIFY: Jest dependency uses a local, linked, " - "workspace, Git, or URL source instead of registry provenance" - ) - typed_packages = { - path: package - for path, package in packages.items() - if isinstance(path, str) and isinstance(package, dict) - } - closure: dict[str, dict[str, Any]] = {} - pending = ["node_modules/jest"] - while pending: - lock_path = pending.pop() - if lock_path in closure: - continue - if len(closure) >= 4096: - cannot_certify( - f"{label} CANNOT-CERTIFY: Jest runtime dependency closure exceeds " - "the 4096-package safety bound" - ) - lock_entry = typed_packages.get(lock_path) - if lock_entry is None: - cannot_certify( - f"{label} CANNOT-CERTIFY: Jest runtime package is missing its " - f"lockfile entry at {lock_path}" - ) - npm_registry_package_version(lock_path, lock_entry, label) - closure[lock_path] = lock_entry - dependencies, optional = npm_runtime_dependency_fields( - lock_entry, lock_path, label - ) - for dependency in sorted(dependencies): - dependency_path = npm_lock_dependency_path( - typed_packages, - lock_path, - dependency, - label, - required=dependency not in optional, - ) - if dependency_path is not None: - pending.append(dependency_path) - return closure - - -def materialized_jest_runner( - project: Path, closure: dict[str, dict[str, Any]], label: str -) -> Path: - package_root = project / "node_modules" / "jest" - runner = project / "node_modules" / ".bin" / "jest" - try: - runner_metadata = runner.lstat() - except OSError as exc: - cannot_certify( - f"{label} CANNOT-CERTIFY: materialized Jest runner is unavailable: {exc}" - ) - if not stat.S_ISLNK(runner_metadata.st_mode): - cannot_certify( - f"{label} CANNOT-CERTIFY: materialized Jest runner is not the package " - "manager symlink to the real CLI" - ) - materialized_packages: dict[str, dict[str, Any]] = {} - project_root = project.resolve() - node_modules_root = (project / "node_modules").resolve() - pending = ["node_modules/jest"] - while pending: - lock_path = pending.pop() - if lock_path in materialized_packages: - continue - lock_entry = closure[lock_path] - package_name = npm_lock_package_name(lock_path) - require(package_name is not None, f"{label} invalid closure package path") - package_path = project / lock_path - package_file = package_path / "package.json" - try: - package_metadata = package_path.lstat() - package_file_metadata = package_file.lstat() - resolved_package = package_path.resolve(strict=True) - except OSError as exc: - cannot_certify( - f"{label} CANNOT-CERTIFY: Jest runtime package {package_name} is " - f"not materialized at {lock_path}: {exc}" - ) - if not ( - stat.S_ISDIR(package_metadata.st_mode) - and not package_path.is_symlink() - and stat.S_ISREG(package_file_metadata.st_mode) - and not package_file.is_symlink() - and resolved_package == package_path - and resolved_package.is_relative_to(node_modules_root) - and resolved_package.is_relative_to(project_root) - ): - cannot_certify( - f"{label} CANNOT-CERTIFY: Jest runtime package {package_name} " - f"escapes or is not a real materialized package at {lock_path}" - ) - try: - package = read_bounded_json( - package_file, - maximum_bytes=1024 * 1024, - maximum_items=4096, - maximum_string_bytes=1024 * 1024, - ) - except BoundedIOError as exc: - cannot_certify( - f"{label} CANNOT-CERTIFY: materialized Jest runtime package " - f"metadata is unreadable at {package_file}: {exc}" - ) - version = npm_registry_package_version(lock_path, lock_entry, label) - if not ( - isinstance(package, dict) - and package.get("name") == package_name - and package.get("version") == version - ): - cannot_certify( - f"{label} CANNOT-CERTIFY: materialized Jest runtime package " - f"{package_name} does not match its lockfile identity" - ) - for field in ( - "dependencies", - "optionalDependencies", - "peerDependencies", - "peerDependenciesMeta", - ): - locked = lock_entry.get(field, {}) - installed = package.get(field, {}) - if locked != installed: - cannot_certify( - f"{label} CANNOT-CERTIFY: materialized Jest runtime package " - f"{package_name} has {field} that do not match its lock entry" - ) - materialized_packages[lock_path] = package - dependencies, optional = npm_runtime_dependency_fields(package, lock_path, label) - for dependency in dependencies: - dependency_path = npm_lock_dependency_path( - closure, - lock_path, - dependency, - label, - required=dependency not in optional, - ) - if dependency_path is None: - continue - if not os.path.lexists(project / dependency_path): - if dependency in optional: - continue - cannot_certify( - f"{label} CANNOT-CERTIFY: required Jest runtime dependency " - f"{dependency} is not materialized at {dependency_path}" - ) - pending.append(dependency_path) - package = materialized_packages["node_modules/jest"] - package_bin = package.get("bin") if isinstance(package, dict) else None - if isinstance(package_bin, dict): - package_bin = package_bin.get("jest") - if not ( - isinstance(package, dict) - and package.get("name") == "jest" - and isinstance(package_bin, str) - ): - cannot_certify( - f"{label} CANNOT-CERTIFY: materialized Jest package identity does not " - "match the lockfile" - ) - bin_relative = package_bin.removeprefix("./") - if bin_relative != "bin/jest.js": - cannot_certify( - f"{label} CANNOT-CERTIFY: materialized Jest package exposes an " - "unexpected CLI" - ) - cli = package_root / "bin" / "jest.js" - try: - cli_metadata = cli.lstat() - resolved_runner = runner.resolve(strict=True) - resolved_cli = cli.resolve(strict=True) - except OSError as exc: - cannot_certify( - f"{label} CANNOT-CERTIFY: materialized Jest CLI cannot be resolved: {exc}" - ) - if not ( - stat.S_ISREG(cli_metadata.st_mode) - and not cli.is_symlink() - and os.access(cli, os.X_OK) - and resolved_runner == resolved_cli - and resolved_cli.is_relative_to(package_root.resolve()) - and "node_modules/jest" in materialized_packages - ): - cannot_certify( - f"{label} CANNOT-CERTIFY: Jest executable is not the real CLI inside " - "the lockfile-materialized package tree" - ) - return runner - - -def prepare_jest_invocation( - checkout: Path, - project_relative: Path, - test_path: str, - label: str, - deadline: float, -) -> tuple[list[str], Path, dict[str, str]]: - project = (checkout / project_relative).resolve() - require(project.is_relative_to(checkout.resolve()), f"{label} package escapes checkout") - jest = project / "node_modules" / ".bin" / "jest" - if os.path.lexists(jest): - cannot_certify( - f"{label} CANNOT-CERTIFY: Jest runner preexists lockfile materialization " - f"at {jest}" - ) - lockfiles = [ - path - for path in (project / "package-lock.json", project / "pnpm-lock.yaml") - if path.exists() - ] - if len(lockfiles) != 1: - cannot_certify( - f"{label} CANNOT-CERTIFY: Jest project {project_relative} must have " - "one unambiguous package-lock.json or pnpm-lock.yaml" - ) - lockfile = lockfiles[0] - if lockfile.name != "package-lock.json": - cannot_certify( - f"{label} CANNOT-CERTIFY: pnpm-lock.yaml governs Jest, but this gate " - "cannot prove official registry package provenance for that format" - ) - try: - lock_metadata = lockfile.lstat() - except OSError as exc: - cannot_certify( - f"{label} CANNOT-CERTIFY: lockfile inspection failed at {lockfile}: {exc}" - ) - if not stat.S_ISREG(lock_metadata.st_mode) or lockfile.is_symlink(): - cannot_certify( - f"{label} CANNOT-CERTIFY: Jest lockfile must be a tracked regular " - f"non-symlink file at {lockfile}" - ) - lock_relative = lockfile.relative_to(checkout.resolve()).as_posix() - tracked = run_command( - ["git", "-C", str(checkout), "ls-files", "--error-unmatch", lock_relative], - timeout=evidence_command_timeout(deadline, 60, f"{label} lockfile provenance"), - description=f"{label} tracked lockfile inspection", - ) - if tracked.returncode != 0: - cannot_certify( - f"{label} CANNOT-CERTIFY: Jest lockfile is not tracked at {lock_relative}" - ) - directory = project - while True: - if (directory / ".npmrc").exists(): - cannot_certify( - f"{label} CANNOT-CERTIFY: project npm configuration can rewrite " - f"registry provenance at {directory / '.npmrc'}" - ) - if directory == checkout.resolve(): - break - directory = directory.parent - jest_closure = npm_jest_lock_provenance(lockfile, label) - node_bin = node_bin_for_project(project, label) - package_manager = "npm" - manager = node_bin / "npm" - arguments = [ - str(manager), - "ci", - "--offline", - "--ignore-scripts", - "--no-audit", - "--no-fund", - ] - if not manager.is_file() or not os.access(manager, os.X_OK): - cannot_certify( - f"{label} CANNOT-CERTIFY: {package_manager} is unavailable for " - f"the offline Jest proof in {project_relative}" - ) - environment = proof_environment() - environment["PATH"] = str(node_bin) + os.pathsep + environment.get("PATH", "") - environment["CI"] = "true" - install_environment = environment.copy() - npm_user_config = checkout / ".crosscheck" / "empty-user-npmrc" - npm_global_config = checkout / ".crosscheck" / "empty-global-npmrc" - npm_user_config.parent.mkdir(parents=True, exist_ok=True) - npm_user_config.write_text("", encoding="utf-8") - npm_global_config.write_text("", encoding="utf-8") - install_environment["NPM_CONFIG_USERCONFIG"] = str(npm_user_config) - install_environment["NPM_CONFIG_GLOBALCONFIG"] = str(npm_global_config) - installed = run_sandboxed( - arguments, - cwd=project, - profile_path=checkout / ".crosscheck" / "jest-dependencies.sb", - allow_network=False, - allow_posix_ipc=False, - env=install_environment, - timeout=evidence_command_timeout( - deadline, evidence_timeout(), f"{label} Jest dependency install" - ), - description=f"{label} offline Jest dependency install", - ) - if installed.returncode != 0: - cannot_certify( - f"{label} CANNOT-CERTIFY: {package_manager} could not materialize " - "the lockfile-pinned Jest environment offline: " - f"{(installed.stdout + installed.stderr).strip()[:1000] or 'no output'}" - ) - jest = materialized_jest_runner(project, jest_closure, label) - test_relative = (checkout / test_file_path(test_path, label)).resolve() - try: - test_argument = test_relative.relative_to(project).as_posix() - except ValueError: - cannot_certify( - f"{label} CANNOT-CERTIFY: named Jest test is outside its package project" - ) - return ( - [ - str(jest), - "--runInBand", - "--runTestsByPath", - "--ci", - "--no-cache", - "--color=false", - "--json", - test_argument, - ], - project, - environment, - ) - - -def jest_execution_summary( - result: subprocess.CompletedProcess[str], label: str, phase: str -) -> tuple[int, int]: - try: - value = json.loads(result.stdout) - except (json.JSONDecodeError, ValueError, RecursionError) as exc: - cannot_certify( - f"{label} CANNOT-CERTIFY: Jest {phase} run emitted no valid JSON " - f"execution record: {exc}" - ) - if not isinstance(value, dict): - cannot_certify( - f"{label} CANNOT-CERTIFY: Jest {phase} execution record is not an object" - ) - total = value.get("numTotalTests") - failed = value.get("numFailedTests") - if not ( - isinstance(total, int) - and not isinstance(total, bool) - and isinstance(failed, int) - and not isinstance(failed, bool) - and total > 0 - and 0 <= failed <= total - ): - cannot_certify( - f"{label} CANNOT-CERTIFY: Jest {phase} run did not prove that any " - "named test executed" - ) - return total, failed - - def runner_probe_timeout() -> int: """Bound the per-candidate runner identification probe. @@ -2242,6 +1626,13 @@ def resolve_runner(runner: str, label: str, cwd: Path, test_path: str) -> list[s candidates = RUNNER_INVOCATIONS.get(runner, ((runner,),)) inspected: list[str] = [] + policy = MUTATION_RUNNER_POLICIES.get(runner) + if policy is not None and policy.node_project_cwd: + project = node_project_for(cwd, test_file_path(test_path, label)) + local_runner = project / "node_modules" / ".bin" / runner + if local_runner.is_file() and os.access(local_runner, os.X_OK): + return [str(local_runner.resolve())] + inspected.append(f"{local_runner} (not an executable project dependency)") for position, candidate in enumerate(candidates): if candidate[0] == "uv": project = uv_project_for(cwd, test_file_path(test_path, label)) @@ -2294,79 +1685,1540 @@ def resolve_runner(runner: str, label: str, cwd: Path, test_path: str) -> list[s if len(candidates) == 1: # A single-invocation runner has one failure mode, and naming it plainly # is more useful than reciting a one-entry ladder. - fail( - f"{label} cannot execute its named test: the {runner} runner is not " - "installed on PATH for the proof checkout, so the gate never ran " - "the test and must not report a test outcome" + non_execution( + label, + f"the {runner} runner is not installed on PATH or as an executable " + "project dependency in the tracked-only proof checkout, so the " + "gate never ran the test and must not report a test outcome", ) - fail( - f"{label} cannot execute its named test: no usable {runner} invocation " - f"is installed on PATH for the proof checkout, so the gate never ran " - f"the test and must not report a test outcome. Inspected " - f"{'; '.join(inspected)}" + non_execution( + label, + f"no usable {runner} invocation is installed on PATH for the " + "tracked-only proof checkout, so the gate never ran the test and must " + f"not report a test outcome. Inspected {'; '.join(inspected)}", ) +def require_node_runtime( + label: str, minimum: tuple[int, int, int] +) -> tuple[int, int, int]: + environment = proof_environment() + path = environment.get("PATH") + node = shutil.which("node", path=path) if path else None + required = ".".join(str(part) for part in minimum) + if node is None: + non_execution( + label, + f"the measured runner requires Node >={required}, but node is absent", + ) + try: + completed = subprocess.run( + [node, "--version"], + capture_output=True, + check=False, + env=environment, + text=True, + timeout=5, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + non_execution(label, f"the Node runtime version is unavailable: {exc}") + output = (completed.stdout or completed.stderr).strip() + match = re.fullmatch(r"v?(\d+)\.(\d+)\.(\d+)(?:[-+].*)?", output) + if completed.returncode != 0 or match is None: + non_execution( + label, + "the Node runtime did not report a usable semantic version", + ) + version = tuple(int(part) for part in match.groups()) + if version < minimum: + found = ".".join(str(part) for part in version) + non_execution( + label, + f"the measured runner requires Node >={required}, found {found}", + ) + return version + + def test_arguments( invocation: dict[str, Any], test_path: str, checkout: Path, label: str -) -> list[str]: - if invocation["runner"] == "direct": +) -> TestRun: + runner_name = invocation["runner"] + if runner_name == "direct": executable = checkout / test_file_path(test_path, label) - require( - os.access(executable, os.X_OK), - f"tracked named test is not executable: {test_path}", + if not os.access(executable, os.X_OK): + non_execution(label, f"tracked named test is not executable: {test_path}") + return TestRun(tuple([str(executable), *invocation["arguments"]]), checkout) + + runner = resolve_runner(runner_name, label, checkout, test_path) + policy = MUTATION_RUNNER_POLICIES.get(runner_name) + if policy is None: + if runner_name in FILE_TEST_RUNNERS: + argv = [*runner, test_path, *invocation["arguments"]] + else: + argv = [*runner, *invocation["arguments"], test_path] + return TestRun(tuple(argv), checkout) + + run_cwd = ( + node_project_for(checkout, test_file_path(test_path, label)) + if policy.node_project_cwd + else checkout + ) + if policy.minimum_node_version is not None: + require_node_runtime(label, policy.minimum_node_version) + if policy.selector_mode == "native": + target = test_path + elif policy.selector_mode in {"none", "test-name-pattern"}: + candidate = (checkout / test_file_path(test_path, label)).resolve() + target = ( + str(candidate) + if policy.absolute_test_path + else str(candidate.relative_to(run_cwd)) ) - return [str(executable), *invocation["arguments"]] - runner = resolve_runner(invocation["runner"], label, checkout, test_path) - if invocation["runner"] in FILE_TEST_RUNNERS: - return [*runner, test_path, *invocation["arguments"]] - return [*runner, *invocation["arguments"], test_path] + else: + tool_fail( + f"mutation-runner policy for {runner_name} has unknown selector mode " + f"{policy.selector_mode!r}" + ) + argv = [*runner, *policy.gate_arguments, target] + body_evidence = False + probe_argument: Path | None = None + environment: tuple[tuple[str, str], ...] = () + if policy.body_probe is not None: + if policy.body_probe == "vitest-runner": + resolved_runner = Path(runner[0]).resolve() + try: + resolved_runner.relative_to(proof_checkout_root(checkout)) + except ValueError: + pass + else: + non_execution( + label, + "vitest resolved its privileged runtime from the " + "project-controlled proof checkout", + ) + manifest_path = resolved_runner.parent / "package.json" + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + non_execution(label, f"vitest runtime metadata is unavailable: {exc}") + if ( + not isinstance(manifest, dict) + or manifest.get("name") != "vitest" + or manifest.get("version") != policy.runtime_version + or resolved_runner != resolved_runner.parent / "vitest.mjs" + or not (resolved_runner.parent / "dist" / "index.js").is_file() + ): + non_execution( + label, + "vitest does not match its measured runtime boundary", + ) + probe_argument = write_javascript_body_probe( + run_cwd, + checkout, + policy.body_probe, + Path(runner[0]).resolve(), + label, + ) + body_evidence = True + if policy.body_probe == "vitest-runner": + try: + target_path = Path(target) + if not target_path.is_absolute(): + target_path = run_cwd / target_path + source = target_path.read_text(encoding="utf-8") + except (OSError, UnicodeError) as exc: + non_execution(label, f"vitest could not inspect its named test: {exc}") + if re.search(r"@(?:vitest|jest)-environment(?:\s|$)", source): + non_execution( + label, + "vitest named test selects an unmeasured custom environment", + ) + launch_preload = probe_argument.with_name("vitest-launch-preload.mjs") + environment = (("NODE_OPTIONS", f"--import={launch_preload}"),) + argv.extend(["--config", str(probe_argument)]) + elif policy.body_probe != "jest-global-wrapper": + tool_fail( + f"mutation-runner policy for {runner_name} has unknown body probe " + f"{policy.body_probe!r}" + ) + selector = test_selector(test_path, label) + if selector is not None and policy.selector_mode == "test-name-pattern": + argv.extend(["--testNamePattern", selector]) + return TestRun( + tuple(argv), run_cwd, body_evidence, probe_argument, environment + ) -def require_test_execution( +def vitest_project_config(run_cwd: Path) -> Path | None: + for stem in ("vitest.config", "vite.config"): + for suffix in (".ts", ".mts", ".cts", ".js", ".mjs", ".cjs"): + candidate = run_cwd / f"{stem}{suffix}" + if candidate.is_file(): + return candidate + return None + + +def vitest_fork_launcher(runner_path: Path, label: str) -> Path: + launcher = runner_path.parent / "dist" / "chunks" / "cli-api.Cjt90eJu.js" + expected = dict(MUTATION_RUNNER_POLICIES["vitest"].source_digests).get( + "forkLauncher" + ) + try: + if launcher.is_symlink() or not launcher.is_file(): + raise OSError("launcher is not a regular package file") + source = launcher.read_bytes() + except OSError as exc: + non_execution(label, f"vitest fork launcher is unavailable: {exc}") + if expected is None or hashlib.sha256(source).hexdigest() != expected: + non_execution(label, "vitest fork launcher does not match its measured source") + return launcher + + +def write_javascript_body_probe( + run_cwd: Path, + checkout: Path, + probe_kind: str, + runner_path: Path, + label: str, +) -> Path: + """Write the gate-owned hook that records entry into each selected test body.""" + + protocol = checkout.parent / f".crosscheck-runner-{checkout.name}" + protocol.mkdir(parents=True, exist_ok=True) + if probe_kind == "jest-global-wrapper": + preload_path = protocol / "jest-body-preload.cjs" + preload_path.write_text("", encoding="utf-8") + argument_path = preload_path + elif probe_kind == "vitest-runner": + package_dir = protocol / "node_modules" / "crosscheck-vitest-body-runner" + package_dir.mkdir(parents=True, exist_ok=True) + probe_path = package_dir / "index.mjs" + vitest_runtime = runner_path.parent / "dist" / "index.js" + source = """import { randomBytes } from 'node:crypto'; +import { writeSync } from 'node:fs'; +import { Buffer } from 'node:buffer'; +import { TestRunner } from __CROSSCHECK_VITEST_RUNTIME__; +const nonce = randomBytes(32).toString('hex'); +const prefix = 'CROSSCHECK-AUTH-BODY'; +const safeApply = Reflect.apply; +const safeArrayFilter = Array.prototype.filter; +const safeArrayJoin = Array.prototype.join; +const safeArrayPush = Array.prototype.push; +const safeArrayReverse = Array.prototype.reverse; +const safeBoolean = Boolean; +const safeBuffer = Buffer; +const safeBufferFrom = Buffer.from; +const safeBufferToString = Buffer.prototype.toString; +const safeJson = JSON; +const safeJsonStringify = JSON.stringify; +const authorized = new WeakSet(); +const isAuthorized = WeakSet.prototype.has.bind(authorized); +const authorize = WeakSet.prototype.add.bind(authorized); +const registeredBodies = new WeakMap(); +const hasRegisteredBody = WeakMap.prototype.has.bind(registeredBodies); +const getRegisteredBody = WeakMap.prototype.get.bind(registeredBodies); +const setRegisteredBody = WeakMap.prototype.set.bind(registeredBodies); +const getTestFn = TestRunner.getTestFn.bind(TestRunner); +let constructed = false; + +function writeEvent(kind, payload) { + const suffix = payload ? ` ${payload}` : ''; + writeSync(2, `${prefix} ${kind} ${nonce}${suffix}\\n`); +} + +writeEvent('PRELOAD'); + +export default class CrosscheckBodyRunner extends TestRunner { + constructor(config) { + super(config); + if (constructed) { + writeEvent('POISON'); + throw new Error('Crosscheck body runner was instantiated twice'); + } + constructed = true; + authorize(this); + writeEvent('START'); + } + + async onBeforeRunTask(test) { + if (!isAuthorized(this)) throw new Error('Unauthorized Crosscheck body runner'); + if (hasRegisteredBody(test)) { + throw new Error('Crosscheck Vitest body was registered twice'); + } + const body = getTestFn(test); + if (!body) throw new Error('Test function is not found'); + setRegisteredBody(test, body); + if (super.onBeforeRunTask) await super.onBeforeRunTask(test); + } + + async runTask(test) { + if (!isAuthorized(this)) throw new Error('Unauthorized Crosscheck body runner'); + const ancestorTitles = []; + let suite = test.suite; + while (suite) { + if (suite.name) safeApply(safeArrayPush, ancestorTitles, [suite.name]); + suite = suite.suite; + } + safeApply(safeArrayReverse, ancestorTitles, []); + safeApply(safeArrayPush, ancestorTitles, [test.name]); + const namedTitles = safeApply(safeArrayFilter, ancestorTitles, [safeBoolean]); + const fullName = safeApply(safeArrayJoin, namedTitles, [' ']); + const body = getTestFn(test); + if (!hasRegisteredBody(test) || getRegisteredBody(test) !== body) { + throw new Error('Crosscheck Vitest body identity changed before execution'); + } + const serialized = safeApply(safeJsonStringify, safeJson, [{ fullName }]); + const encoded = safeApply(safeBufferFrom, safeBuffer, [serialized, 'utf8']); + const payload = safeApply(safeBufferToString, encoded, ['base64']); + writeEvent('EVENT', payload); + await body(); + } +} + +Object.freeze(CrosscheckBodyRunner.prototype); +""".replace( + "__CROSSCHECK_VITEST_RUNTIME__", json.dumps(vitest_runtime.as_uri()) + ) + config_path = protocol / "vitest-body-probe.config.mjs" + launch_preload_path = protocol / "vitest-launch-preload.mjs" + loader_path = protocol / "vitest-launch-loader.mjs" + child_process_path = protocol / "vitest-child-process.mjs" + launch_node_options = f"--import={launch_preload_path}" + vitest_worker_path = runner_path.parent / "dist" / "workers" / "forks.js" + fork_launcher_path = vitest_fork_launcher(runner_path, label) + project_config = vitest_project_config(run_cwd) + if project_config is None: + project_loader = "const config = {};" + else: + project_loader = f"""const projectModule = await import({json.dumps(project_config.as_uri())}); + const projectConfig = projectModule.default; + const loaded = typeof projectConfig === 'function' + ? await projectConfig(environment) + : await projectConfig; + const config = loaded || {{}};""" + launch_preload_source = f"""import {{ register }} from 'node:module'; +import {{ pathToFileURL }} from 'node:url'; +import {{ isMainThread }} from 'node:worker_threads'; +const safeApply = Reflect.apply; +const safeOwnKeys = Reflect.ownKeys; +const safeString = String; +const safeToUpperCase = String.prototype.toUpperCase; +const expectedNodeOptions = {json.dumps(launch_node_options)}; +const expectedRunnerPath = {json.dumps(str(probe_path))}; +const loaderUrl = pathToFileURL({json.dumps(str(loader_path))}).href; +const childProcessUrl = pathToFileURL({json.dumps(str(child_process_path))}).href; + +function nodeOptionsKeys(environment) {{ + if (!environment || typeof environment !== 'object') {{ + throw new Error('Crosscheck could not authenticate the Vitest worker environment'); + }} + const keys = []; + for (const key of safeOwnKeys(environment)) {{ + if (typeof key === 'string' + && safeApply(safeToUpperCase, key, []) === 'NODE_OPTIONS') {{ + keys[keys.length] = key; + }} + }} + return keys; +}} + +function requireGateNodeOptions(environment) {{ + const keys = nodeOptionsKeys(environment); + if (keys.length !== 1 || safeString(environment[keys[0]]) !== expectedNodeOptions) {{ + throw new Error('Crosscheck rejected project-controlled ambient NODE_OPTIONS'); + }} +}} + +if (isMainThread) {{ + requireGateNodeOptions(process.env); + for (const key of nodeOptionsKeys(process.env)) delete process.env[key]; + let isWorker = false; + for (let index = 0; index + 1 < process.execArgv.length; index += 1) {{ + if (process.execArgv[index] === '--import' + && process.execArgv[index + 1] === expectedRunnerPath) {{ + isWorker = true; + }} + }} + if (!isWorker) {{ + register(loaderUrl); + await import(childProcessUrl); + }} +}} +""" + loader_source = f"""const launcherUrl = {json.dumps(fork_launcher_path.as_uri())}; +const childProcessUrl = {json.dumps(child_process_path.as_uri())}; + +export async function resolve(specifier, context, nextResolve) {{ + if (specifier === 'node:child_process' && context.parentURL === launcherUrl) {{ + return {{ shortCircuit: true, url: childProcessUrl }}; + }} + return nextResolve(specifier, context); +}} +""" + child_process_source = f"""import * as childProcess from 'node:child_process'; +const safeApply = Reflect.apply; +const safeArrayIsArray = Array.isArray; +const safeArraySlice = Array.prototype.slice; +const safeGetOwnPropertyDescriptor = Object.getOwnPropertyDescriptor; +const safeOwnKeys = Reflect.ownKeys; +const safeToUpperCase = String.prototype.toUpperCase; +const nativeFork = childProcess.fork; +const nativeChildProcessPrototype = childProcess.ChildProcess.prototype; +const nativeSpawn = nativeChildProcessPrototype.spawn; +const nativeSpawnDescriptor = safeApply( + safeGetOwnPropertyDescriptor, + Object, + [nativeChildProcessPrototype, 'spawn'], +); +const expectedNodeOptions = {json.dumps(launch_node_options)}; +const expectedWorkerPath = {json.dumps(str(vitest_worker_path))}; + +function requireNativeSpawn() {{ + const descriptor = safeApply( + safeGetOwnPropertyDescriptor, + Object, + [nativeChildProcessPrototype, 'spawn'], + ); + if (!descriptor + || descriptor.value !== nativeSpawn + || descriptor.configurable !== nativeSpawnDescriptor.configurable + || descriptor.enumerable !== nativeSpawnDescriptor.enumerable + || descriptor.writable !== nativeSpawnDescriptor.writable) {{ + throw new Error( + 'Crosscheck rejected project-mutated ChildProcess.prototype.spawn', + ); + }} +}} + +function nodeOptionsKeys(environment) {{ + if (!environment || typeof environment !== 'object') {{ + throw new Error('Crosscheck could not authenticate the Vitest worker environment'); + }} + const keys = []; + for (const key of safeOwnKeys(environment)) {{ + if (typeof key === 'string' + && safeApply(safeToUpperCase, key, []) === 'NODE_OPTIONS') {{ + keys[keys.length] = key; + }} + }} + return keys; +}} + +export function fork(modulePath, args, options) {{ + const resolvedOptions = safeArrayIsArray(args) ? options : args; + if (modulePath !== expectedWorkerPath) {{ + requireNativeSpawn(); + return safeApply(nativeFork, childProcess, arguments); + }} + const environment = resolvedOptions?.env || process.env; + if (nodeOptionsKeys(environment).length) {{ + throw new Error('Crosscheck rejected project-controlled ambient NODE_OPTIONS'); + }} + const workerOptions = {{ + ...(resolvedOptions || {{}}), + env: {{ ...environment, NODE_OPTIONS: expectedNodeOptions }}, + }}; + const workerArguments = safeArrayIsArray(args) + ? safeApply(safeArraySlice, args, []) + : undefined; + const forwarded = safeArrayIsArray(args) + ? [modulePath, workerArguments, workerOptions] + : [modulePath, workerOptions]; + requireNativeSpawn(); + return safeApply(nativeFork, childProcess, forwarded); +}} + +export * from 'node:child_process'; +""" + config_body = f"""const runnerPath = {json.dumps(str(probe_path))}; +const safeApply = Reflect.apply; +const safeArrayIsArray = Array.isArray; +const safeArrayFilter = Array.prototype.filter; +const safeArrayMap = Array.prototype.map; +const safeBoolean = Boolean; +const safeCreate = Object.create; +const safeDefineProperty = Object.defineProperty; +const safeFreeze = Object.freeze; +const safeGetOwnPropertyDescriptors = Object.getOwnPropertyDescriptors; +const safeGetPrototypeOf = Object.getPrototypeOf; +const safeObjectKeys = Object.keys; +const safePropertyIsEnumerableMethod = Object.prototype.propertyIsEnumerable; +const safePropertyIsEnumerable = (value, key) => safeApply( + safePropertyIsEnumerableMethod, + value, + [key], +); +const safeJson = JSON; +const safeJsonStringify = JSON.stringify; +const safePromise = Promise; +const safePromiseResolveMethod = Promise.resolve; +const safePromiseResolve = value => safeApply( + safePromiseResolveMethod, + safePromise, + [value], +); +const safePromiseThen = Promise.prototype.then; +const safeThen = (value, handler) => safeApply(safePromiseThen, value, [handler]); +const safeSplit = String.prototype.split; +const safeStringify = value => safeApply(safeJsonStringify, safeJson, [value]); +const safeToUpperCase = String.prototype.toUpperCase; + +const nodeOptionsKeys = value => safeApply(safeArrayFilter, safeObjectKeys(value), [ + key => safeApply(safeToUpperCase, key, []) === 'NODE_OPTIONS', +]); + +const rejectAmbientNodeOptions = () => {{ + const keys = nodeOptionsKeys(process.env); + if (keys.length) {{ + throw new Error('Crosscheck rejected project-controlled ambient NODE_OPTIONS'); + }} +}}; + +export default async (environment) => {{ + {project_loader} + if (!config || typeof config !== 'object' || safeArrayIsArray(config)) {{ + throw new Error('Crosscheck cannot authenticate an unmeasured Vitest project shape'); + }} + const test = config.test || {{}}; + if (typeof test !== 'object' || safeArrayIsArray(test)) {{ + throw new Error('Crosscheck cannot authenticate an unmeasured Vitest test shape'); + }} + const configuredEnvironment = test.environment || 'node'; + const configuredEnv = test.env || {{}}; + rejectAmbientNodeOptions(); + const hasNodeOptions = nodeOptionsKeys(configuredEnv).length > 0; + const hasConfigEntries = value => safeArrayIsArray(value) + ? value.length > 0 + : safeBoolean(value && typeof value === 'object' && safeObjectKeys(value).length); + if (configuredEnvironment !== 'node' + || hasConfigEntries(test.environmentMatchGlobs) + || hasConfigEntries(test.projects) + || test.browser?.enabled + || (test.pool && test.pool !== 'forks') + || hasConfigEntries(test.poolOptions) + || hasConfigEntries(test.poolMatchGlobs) + || (test.execArgv && test.execArgv.length) + || hasNodeOptions + || (config.define && safeObjectKeys(config.define).length) + || test.runner) {{ + throw new Error('Crosscheck cannot authenticate an unmeasured Vitest runtime boundary'); + }} + const plugins = safeArrayIsArray(config.plugins) + ? config.plugins + : config.plugins ? [config.plugins] : []; + const isRunnerId = id => typeof id === 'string' + && safeApply(safeSplit, id, ['?'])[0] === runnerPath; + const protectHook = (hook, block) => {{ + if (typeof hook === 'function') {{ + return function(...args) {{ + if (block(args)) return null; + return safeApply(hook, this, args); + }}; + }} + if (hook && typeof hook.handler === 'function') {{ + const descriptors = safeGetOwnPropertyDescriptors(hook); + delete descriptors.handler; + const protectedHook = safeCreate(safeGetPrototypeOf(hook), descriptors); + safeDefineProperty(protectedHook, 'handler', {{ + configurable: false, + enumerable: safePropertyIsEnumerable(hook, 'handler'), + writable: false, + value: function(...args) {{ + if (block(args)) return null; + return safeApply(hook.handler, this, args); + }}, + }}); + return protectedHook; + }} + return hook; + }}; + const protectConfigFunction = handler => function(...args) {{ + const beforeDefine = args[0]?.define; + const beforeFingerprint = safeStringify(beforeDefine || {{}}); + const validate = returned => {{ + if (args[0]?.define !== beforeDefine + || safeStringify(args[0]?.define || {{}}) !== beforeFingerprint + || hasConfigEntries(returned?.define)) {{ + throw new Error('Crosscheck rejected a project-controlled Vitest define'); + }} + return returned; + }}; + const returned = safeApply(handler, this, args); + return returned && typeof returned.then === 'function' + ? safeThen(safePromiseResolve(returned), validate) + : validate(returned); + }}; + const protectConfigHook = hook => {{ + if (typeof hook === 'function') return protectConfigFunction(hook); + if (hook && typeof hook.handler === 'function') {{ + const descriptors = safeGetOwnPropertyDescriptors(hook); + delete descriptors.handler; + const protectedHook = safeCreate(safeGetPrototypeOf(hook), descriptors); + safeDefineProperty(protectedHook, 'handler', {{ + configurable: false, + enumerable: safePropertyIsEnumerable(hook, 'handler'), + writable: false, + value: protectConfigFunction(hook.handler), + }}); + return protectedHook; + }} + return hook; + }}; + const protectPlugin = plugin => {{ + if (safeArrayIsArray(plugin)) {{ + return safeApply(safeArrayMap, plugin, [protectPlugin]); + }} + if (plugin && typeof plugin.then === 'function') {{ + return safeThen(safePromiseResolve(plugin), protectPlugin); + }} + if (!plugin || typeof plugin !== 'object') return plugin; + const descriptors = safeGetOwnPropertyDescriptors(plugin); + delete descriptors.config; + delete descriptors.resolveId; + delete descriptors.load; + delete descriptors.transform; + const protectedPlugin = safeCreate(safeGetPrototypeOf(plugin), descriptors); + const runnerHooks = [ + ['resolveId', plugin.resolveId, args => isRunnerId(args[0])], + ['load', plugin.load, args => isRunnerId(args[0])], + ['transform', plugin.transform, args => isRunnerId(args[1])], + ]; + for (let index = 0; index < runnerHooks.length; index += 1) {{ + const entry = runnerHooks[index]; + const name = entry[0]; + const hook = entry[1]; + const block = entry[2]; + safeDefineProperty(protectedPlugin, name, {{ + configurable: false, + enumerable: safePropertyIsEnumerable(plugin, name), + writable: false, + value: protectHook(hook, block), + }}); + }} + safeDefineProperty(protectedPlugin, 'config', {{ + configurable: false, + enumerable: safePropertyIsEnumerable(plugin, 'config'), + writable: false, + value: protectConfigHook(plugin.config), + }}); + return protectedPlugin; + }}; + const protectedPlugins = safeApply(safeArrayMap, plugins, [protectPlugin]); + const guard = safeFreeze({{ + name: 'crosscheck-runner-boundary', + enforce: 'pre', + configResolved(resolved) {{ + const resolvedTest = resolved.test; + const violations = [ + [!resolvedTest, 'test'], + [resolvedTest?.environment !== 'node', 'environment'], + [resolvedTest?.pool !== 'forks', 'pool'], + [hasConfigEntries(resolvedTest?.environmentMatchGlobs), 'environmentMatchGlobs'], + [hasConfigEntries(resolvedTest?.projects), 'projects'], + [resolvedTest?.browser?.enabled, 'browser'], + [hasConfigEntries(resolvedTest?.poolOptions), 'poolOptions'], + [hasConfigEntries(resolvedTest?.poolMatchGlobs), 'poolMatchGlobs'], + [!safeArrayIsArray(resolvedTest?.execArgv), 'execArgv-type'], + [resolvedTest?.execArgv?.length !== 2, 'execArgv-length'], + [resolvedTest?.execArgv?.[0] !== '--import', 'execArgv-option'], + [resolvedTest?.execArgv?.[1] !== runnerPath, 'execArgv-path'], + [nodeOptionsKeys(resolvedTest?.env || {{}}).length > 0, 'env'], + ]; + const failedEntries = safeApply(safeArrayFilter, violations, [ + entry => entry[0], + ]); + const violationNames = safeApply(safeArrayMap, failedEntries, [ + entry => entry[1], + ]); + if (violationNames.length) {{ + throw new Error( + `Crosscheck Vitest runtime boundary changed after resolution: ${{violationNames}}`, + ); + }} + const lock = (key, value) => safeDefineProperty(resolvedTest, key, {{ + configurable: false, + enumerable: true, + writable: false, + value, + }}); + lock('runner', runnerPath); + lock('environment', 'node'); + lock('pool', 'forks'); + lock('execArgv', safeFreeze(['--import', runnerPath])); + lock('env', safeFreeze({{ ...(resolvedTest.env || {{}}) }})); + lock('environmentMatchGlobs', undefined); + lock('projects', undefined); + lock('poolMatchGlobs', undefined); + lock('browser', safeFreeze({{ ...(resolvedTest.browser || {{}}), enabled: false }})); + safeDefineProperty(resolved, 'define', {{ + configurable: false, + enumerable: true, + writable: false, + value: safeFreeze({{ ...(resolved.define || {{}}) }}), + }}); + safeFreeze(resolved.plugins); + safeDefineProperty(resolved, 'plugins', {{ + configurable: false, + enumerable: true, + writable: false, + value: resolved.plugins, + }}); + }}, + }}); + const environmentGuard = safeFreeze({{ + name: 'crosscheck-worker-environment-boundary', + enforce: 'post', + configResolved: safeFreeze({{ + order: 'post', + handler() {{ + rejectAmbientNodeOptions(); + }}, + }}), + }}); + const finalPlugins = [guard]; + for (let index = 0; index < protectedPlugins.length; index += 1) {{ + finalPlugins[finalPlugins.length] = protectedPlugins[index]; + }} + finalPlugins[finalPlugins.length] = environmentGuard; + return {{ + ...config, + define: {{}}, + plugins: finalPlugins, + test: {{ + ...test, + environment: 'node', + env: {{ ...configuredEnv }}, + execArgv: ['--import', runnerPath], + pool: 'forks', + runner: runnerPath, + }}, + }}; +}}; +""" + config_path.write_text(config_body, encoding="utf-8") + launch_preload_path.write_text(launch_preload_source, encoding="utf-8") + loader_path.write_text(loader_source, encoding="utf-8") + child_process_path.write_text(child_process_source, encoding="utf-8") + probe_path.write_text(source, encoding="utf-8") + (package_dir / "package.json").write_text( + '{"name":"crosscheck-vitest-body-runner","type":"module"}\n', + encoding="utf-8", + ) + argument_path = config_path + else: + tool_fail(f"unknown JavaScript body probe {probe_kind!r}") + return argument_path + + +def proof_checkout_root(path: Path) -> Path: + resolved = path.resolve() + for candidate in (resolved, *resolved.parents): + if (candidate / ".git").exists(): + return candidate + return resolved + + +def resolve_jest_runtime_graph( + run: TestRun, + environment_package: str, + label: str, + phase: str, + profile_path: Path, + deadline: float, +) -> tuple[Path, dict[str, Path]]: + checkout = proof_checkout_root(run.cwd) + executable = Path(run.argv[0]).resolve() + try: + executable.relative_to(checkout) + except ValueError: + pass + else: + non_execution( + label, + "jest resolved its privileged runtime from the project-controlled " + "proof checkout", + ) + node = shutil.which("node", path=proof_environment().get("PATH")) + if node is None: + non_execution( + label, + "node is unavailable for the Jest runtime-graph preflight", + ) + if run.body_probe is None: + tool_fail("Jest body probe did not allocate its runner-owned preload") + resolver_path = run.body_probe.with_name("jest-runtime-graph.cjs") + resolver_path.write_text( + """const { createRequire } = require('node:module'); +const { dirname } = require('node:path'); +const { readFileSync, realpathSync } = require('node:fs'); + +const executable = realpathSync(process.argv[2]); +const environmentPackage = process.argv[3]; +const requireFromJest = createRequire(executable); + +function packageVersion(entry, expectedName) { + let current = dirname(entry); + while (true) { + const manifest = `${current}/package.json`; + try { + const value = JSON.parse(readFileSync(manifest, 'utf8')); + if (value.name === expectedName) return value.version; + } catch {} + const parent = dirname(current); + if (parent === current) throw new Error(`package metadata absent for ${expectedName}`); + current = parent; + } +} + +function resolveEntry(specifier, packageName) { + const path = realpathSync(requireFromJest.resolve(specifier)); + return { path, version: packageVersion(path, packageName) }; +} + +const testRunner = resolveEntry('jest-circus/runner', 'jest-circus'); +const graph = { + executable: { path: executable, version: packageVersion(executable, 'jest') }, + runner: resolveEntry('jest-runner', 'jest-runner'), + testRunner, + testEnvironment: resolveEntry(environmentPackage, environmentPackage), + runtime: resolveEntry('jest-runtime', 'jest-runtime'), + circusRun: { + path: realpathSync(createRequire(testRunner.path).resolve('./build/run.js')), + version: testRunner.version, + }, + circusUtils: { + path: realpathSync(createRequire(testRunner.path).resolve('./build/utils.js')), + version: testRunner.version, + }, +}; +process.stdout.write(JSON.stringify(graph)); +""", + encoding="utf-8", + ) + result = run_sandboxed( + [node, str(resolver_path), str(executable), environment_package], + cwd=run.cwd, + profile_path=profile_path, + allow_network=False, + allow_posix_ipc=False, + env=proof_environment(), + timeout=evidence_command_timeout( + deadline, evidence_timeout(), f"{label} {phase} Jest runtime graph" + ), + description=f"{label} {phase} Jest runtime graph", + ) + if result.returncode != 0: + non_execution( + label, + "jest could not resolve its privileged components through the exact " + f"executable package graph: {(result.stderr or result.stdout).strip()[:500]}", + ) + try: + report = json.loads(result.stdout) + except (json.JSONDecodeError, UnicodeError) as exc: + non_execution( + label, + f"jest emitted a malformed runtime graph: {exc}", + ) + expected_keys = { + "executable", + "runner", + "testRunner", + "testEnvironment", + "runtime", + "circusRun", + "circusUtils", + } + expected_version = MUTATION_RUNNER_POLICIES["jest"].runtime_version + if not isinstance(report, dict) or set(report) != expected_keys: + non_execution(label, "jest emitted an incomplete runtime graph") + paths: dict[str, Path] = {} + for key in expected_keys: + entry = report.get(key) + if ( + not isinstance(entry, dict) + or set(entry) != {"path", "version"} + or entry.get("version") != expected_version + or not isinstance(entry.get("path"), str) + ): + non_execution(label, f"jest runtime graph has an unmeasured {key}") + resolved = Path(entry["path"]).resolve() + try: + resolved.relative_to(checkout) + except ValueError: + pass + else: + non_execution( + label, + f"jest resolved project-controlled {key} code from the proof checkout", + ) + if not resolved.is_file(): + non_execution(label, f"jest runtime graph {key} is not a file") + paths[key] = resolved + if paths["executable"] != executable: + non_execution(label, "jest runtime graph did not bind the invoked executable") + return checkout, paths + + +def prepare_jest_body_evidence( + run: TestRun, + label: str, + phase: str, + profile_path: Path, + deadline: float, +) -> TestRun: + config_result = run_sandboxed( + [run.argv[0], "--showConfig", "--json"], + cwd=run.cwd, + profile_path=profile_path, + allow_network=False, + allow_posix_ipc=False, + env=proof_environment(), + timeout=evidence_command_timeout( + deadline, evidence_timeout(), f"{label} {phase} Jest configuration" + ), + description=f"{label} {phase} Jest configuration", + ) + if config_result.returncode != 0: + non_execution( + label, + f"jest project configuration failed during the {phase} run before " + f"the selected test body could start: exit {config_result.returncode}: " + f"{(config_result.stderr or config_result.stdout).strip()[:500]}", + ) + try: + report = json.loads(config_result.stdout) + except (json.JSONDecodeError, UnicodeError) as exc: + non_execution( + label, + f"jest did not emit its effective project configuration during the " + f"{phase} run ({exc})", + ) + configs = report.get("configs") if isinstance(report, dict) else None + if not isinstance(configs, list) or not configs: + non_execution(label, f"jest emitted no effective project configuration") + argv = list(run.argv) + try: + target_index = argv.index("--runTestsByPath") + 1 + target = Path(argv[target_index]).resolve() + except (ValueError, IndexError): + tool_fail("Jest body probe invocation omitted its absolute test target") + matching: list[dict[str, Any]] = [] + for config in configs: + if not isinstance(config, dict) or not isinstance(config.get("rootDir"), str): + non_execution(label, "jest emitted a malformed project configuration") + try: + target.relative_to(Path(config["rootDir"]).resolve()) + except ValueError: + continue + matching.append(config) + if len(matching) != 1: + non_execution( + label, + f"jest resolved {len(matching)} project configurations for the named " + f"test during the {phase} run; probe injection is ambiguous", + ) + config = matching[0] + environment_path = config.get("testEnvironment") + if not isinstance(environment_path, str) or not environment_path: + non_execution(label, "jest emitted no resolved test environment") + environment_text = Path(environment_path).resolve().as_posix() + if environment_text.endswith( + "/node_modules/jest-environment-node/build/index.js" + ): + environment_package = "jest-environment-node" + elif environment_text.endswith( + "/node_modules/jest-environment-jsdom/build/index.js" + ): + environment_package = "jest-environment-jsdom" + else: + non_execution( + label, + "jest uses an unmeasured test environment before the body boundary", + ) + checkout, runtime_graph = resolve_jest_runtime_graph( + run, + environment_package, + label, + phase, + profile_path, + deadline, + ) + for key in ("runner", "testRunner", "testEnvironment"): + value = config.get(key) + if not isinstance(value, str): + non_execution(label, f"jest emitted no resolved {key}") + effective = Path(value).resolve() + try: + effective.relative_to(checkout) + except ValueError: + pass + else: + non_execution( + label, + f"jest uses a project-controlled {key} before the body boundary", + ) + if effective != runtime_graph[key]: + non_execution( + label, + f"jest effective {key} does not belong to its exact runtime graph", + ) + resolver = config.get("resolver") + if resolver is not None and resolver != "": + non_execution( + label, + "jest uses a project-controlled resolver before the body-evidence boundary", + ) + runtime = config.get("runtime") + if runtime and ( + not isinstance(runtime, str) + or Path(runtime).resolve() != runtime_graph["runtime"] + ): + non_execution( + label, + "jest effective runtime does not belong to its exact package graph", + ) + try: + source = target.read_text(encoding="utf-8") + except (OSError, UnicodeError) as exc: + non_execution(label, f"jest could not inspect its named test: {exc}") + if re.search(r"@jest-environment(?:\s|$)", source): + non_execution( + label, + "jest named test overrides the authenticated test environment in a docblock", + ) + if run.body_probe is None: + tool_fail("Jest body probe did not allocate its runner-owned preload") + preload_path = run.body_probe + wrapper_dir = ( + preload_path.parent / "node_modules" / "crosscheck-jest-body-runner" + ) + wrapper_dir.mkdir(parents=True, exist_ok=True) + wrapper_path = wrapper_dir / "index.cjs" + wrapper_source = ( + f"module.exports = require({json.dumps(str(runtime_graph['testRunner']))});\n" + ) + wrapper_path.write_text(wrapper_source, encoding="utf-8") + wrapper_digest = hashlib.sha256(wrapper_source.encode()).hexdigest() + circus_digests = dict(MUTATION_RUNNER_POLICIES["jest"].source_digests) + preload_path.write_text( + f"""const {{ createHash, randomBytes }} = require('node:crypto'); +const {{ readFileSync, writeSync }} = require('node:fs'); +const {{ Buffer }} = require('node:buffer'); +const {{ dirname }} = require('node:path'); +const Module = require('node:module'); +const runPath = {json.dumps(str(runtime_graph["circusRun"]))}; +const utilsPath = {json.dumps(str(runtime_graph["circusUtils"]))}; +const runnerPath = {json.dumps(str(runtime_graph["testRunner"]))}; +const wrapperPath = {json.dumps(str(wrapper_path))}; +const nonce = randomBytes(32).toString('hex'); +const prefix = 'CROSSCHECK-AUTH-BODY'; +const safeApply = Reflect.apply; +const safeArrayIsArray = Array.isArray; +const safeObjectToString = Object.prototype.toString; +const recorded = new WeakSet(); +const wasRecorded = WeakSet.prototype.has.bind(recorded); +const markRecorded = WeakSet.prototype.add.bind(recorded); +const registeredBodies = new WeakMap(); +const hasRegisteredBody = WeakMap.prototype.has.bind(registeredBodies); +const getRegisteredBody = WeakMap.prototype.get.bind(registeredBodies); +const setRegisteredBody = WeakMap.prototype.set.bind(registeredBodies); + +function writeEvent(kind, payload) {{ + const suffix = payload ? ` ${{payload}}` : ''; + writeSync(2, `${{prefix}} ${{kind}} ${{nonce}}${{suffix}}\\n`); +}} + +function fullName(test) {{ + const names = [test.name]; + let parent = test.parent; + while (parent && parent.parent) {{ + if (parent.name) names.unshift(parent.name); + parent = parent.parent; + }} + return names.filter(Boolean).join(' '); +}} + +function record(test) {{ + if (wasRecorded(test)) return; + markRecorded(test); + const payload = Buffer.from( + JSON.stringify({{ fullName: fullName(test) }}), + 'utf8', + ).toString('base64'); + writeEvent('EVENT', payload); +}} + +function seal(module, filename) {{ + Object.freeze(module.exports); + Object.defineProperty(module, 'exports', {{ + configurable: false, + enumerable: true, + writable: false, + value: module.exports, + }}); + if (Module._cache[filename] === module) {{ + Object.defineProperty(Module._cache, filename, {{ + configurable: false, + enumerable: true, + writable: false, + value: module, + }}); + }} +}} + +function patchUtilsSource(source, slot) {{ + const digest = createHash('sha256').update(source).digest('hex'); + if (digest !== {json.dumps(circus_digests["circusUtils"])}) {{ + throw new Error('Crosscheck Jest invocation module does not match measured Circus'); + }} + const callbackCall = ' returnedValue = fn.call(testContext, done);'; + const generatorCall = ' returnedValue = _co.default.wrap(fn).call({{}});'; + const ordinaryCall = ' returnedValue = fn.call(testContext);'; + if (source.split(callbackCall).length !== 2 + || source.split(generatorCall).length !== 2 + || source.split(ordinaryCall).length !== 2) {{ + throw new Error('Crosscheck Jest invocation boundary does not match measured Circus'); + }} + source = `"use strict";\\nconst __crosscheckBridge = globalThis[${{JSON.stringify(slot)}}];\\nconst __crosscheckInvoke = __crosscheckBridge.invoke;\\ndelete globalThis[${{JSON.stringify(slot)}}];\\n${{source}}`; + source = source.replace( + callbackCall, + ' returnedValue = __crosscheckInvoke(testOrHook, isHook, fn, testContext, [done]);', + ); + source = source.replace( + generatorCall, + " if (!isHook) {{\\n reject(new Error('Crosscheck cannot authenticate Jest generator body execution'));\\n return;\\n }}\\n returnedValue = _co.default.wrap(fn).call({{}});", + ); + return source.replace( + ordinaryCall, + ' returnedValue = __crosscheckInvoke(testOrHook, isHook, fn, testContext, []);', + ); +}} + +function patchRunSource(source, slot) {{ + const digest = createHash('sha256').update(source).digest('hex'); + if (digest !== {json.dumps(circus_digests["circusRun"])}) {{ + throw new Error('Crosscheck Jest run module does not match measured Circus'); + }} + const concurrent = " const testFn = test.fn;\\n const promise = mutex(() =>\\n testNameStorage.run((0, _utils.getTestID)(test), testFn)\\n );"; + const root = " const {{rootDescribeBlock, seed, randomize}} = (0, _state.getState)();"; + if (source.split(concurrent).length !== 2 || source.split(root).length !== 2) {{ + throw new Error('Crosscheck Jest concurrent boundary does not match measured Circus'); + }} + source = `"use strict";\\nconst __crosscheckBridge = globalThis[${{JSON.stringify(slot)}}];\\nconst __crosscheckInvoke = __crosscheckBridge.invoke;\\nconst __crosscheckCapture = __crosscheckBridge.capture;\\ndelete globalThis[${{JSON.stringify(slot)}}];\\n${{source}}`; + source = source.replace( + root, + " const {{rootDescribeBlock, seed, randomize}} = (0, _state.getState)();\\n __crosscheckCapture(rootDescribeBlock);", + ); + return source.replace( + concurrent, + " const testFn = test.fn;\\n const promise = mutex(() =>\\n testNameStorage.run(\\n (0, _utils.getTestID)(test),\\n () => __crosscheckInvoke(test, false, testFn, undefined, [])\\n )\\n );", + ); +}} + +const originalCompile = Module.prototype._compile; +function compileTrusted(filename, source) {{ + const prepared = new Module(filename, module); + prepared.filename = filename; + prepared.paths = Module._nodeModulePaths(dirname(filename)); + Module._cache[filename] = prepared; + try {{ + originalCompile.call(prepared, source, filename); + prepared.loaded = true; + }} catch (error) {{ + delete Module._cache[filename]; + throw error; + }} + seal(prepared, filename); + return prepared; +}} + +const wrapperSource = readFileSync(wrapperPath, 'utf8'); +if (createHash('sha256').update(wrapperSource).digest('hex') !== {json.dumps(wrapper_digest)}) {{ + throw new Error('Crosscheck Jest runner wrapper does not match its gate source'); +}} +let invoked = false; +function captureBodies(block) {{ + if (!safeArrayIsArray(block.children)) {{ + throw new Error('Crosscheck Jest body registry is malformed'); + }} + for (let index = 0; index < block.children.length; index += 1) {{ + const child = block.children[index]; + if (child.type === 'test') {{ + if (hasRegisteredBody(child)) {{ + throw new Error('Crosscheck Jest body was registered twice'); + }} + setRegisteredBody(child, child.fn); + }} else if (child.type === 'describeBlock') {{ + captureBodies(child); + }} + }} +}} + +function invokeBody(testOrHook, isHook, body, context, args) {{ + if (!isHook) {{ + if (!hasRegisteredBody(testOrHook) + || getRegisteredBody(testOrHook) !== body) {{ + throw new Error('Crosscheck Jest body identity changed before execution'); + }} + if (safeApply(safeObjectToString, body, []) === '[object GeneratorFunction]') {{ + throw new Error('Crosscheck cannot authenticate Jest generator body execution'); + }} + record(testOrHook); + }} + return safeApply(body, context, args); +}} + +async function invokeRunner(canonical, args) {{ + if (invoked) throw new Error('Crosscheck Jest runner was invoked twice'); + invoked = true; + const environment = args[2]; + const runtime = args[3]; + if (!environment?.global || !runtime || typeof runtime.readFile !== 'function') {{ + throw new Error('Crosscheck Jest runner received an unmeasured runtime'); + }} + const originalReadFile = runtime.readFile.bind(runtime); + const runSlot = `__crosscheck_${{randomBytes(32).toString('hex')}}`; + const utilsSlot = `__crosscheck_${{randomBytes(32).toString('hex')}}`; + const bridge = Object.freeze({{ capture: captureBodies, invoke: invokeBody }}); + environment.global[runSlot] = bridge; + environment.global[utilsSlot] = bridge; + Object.defineProperty(runtime, 'readFile', {{ + configurable: false, + enumerable: false, + writable: false, + value: function(filename, ...args) {{ + if (filename === runPath) {{ + return patchRunSource(readFileSync(runPath, 'utf8'), runSlot); + }} + if (filename === utilsPath) {{ + return patchUtilsSource(readFileSync(utilsPath, 'utf8'), utilsSlot); + }} + return originalReadFile(filename, ...args); + }}, + }}); + writeEvent('START'); + return canonical(...args); +}} +const gateSlot = `__crosscheck_${{randomBytes(32).toString('hex')}}`; +globalThis[gateSlot] = invokeRunner; +const trustedWrapperSource = `const __crosscheckGate = globalThis[${{JSON.stringify(gateSlot)}}];\\ndelete globalThis[${{JSON.stringify(gateSlot)}}];\\n${{wrapperSource.replace( + 'module.exports = require(', + 'const __crosscheckCanonical = require(', +)}}module.exports = (...args) => __crosscheckGate(__crosscheckCanonical, args);\\n`; +compileTrusted(wrapperPath, trustedWrapperSource); +const runnerModule = Module._cache[runnerPath]; +if (!runnerModule) throw new Error('Crosscheck Jest runner did not load canonically'); +seal(runnerModule, runnerPath); +Object.defineProperty(Module.prototype, '_compile', {{ + configurable: false, + enumerable: false, + writable: false, + value: originalCompile, +}}); +""", + encoding="utf-8", + ) + node = shutil.which("node", path=proof_environment().get("PATH")) + if node is None: + non_execution(label, "node is unavailable for the Jest body preload") + return TestRun( + tuple( + [ + str(Path(node).resolve()), + f"--require={preload_path}", + *argv, + "--testRunner", + str(wrapper_path), + ] + ), + run.cwd, + run.body_evidence, + preload_path, + (), + ) + + +def read_javascript_body_report( + stderr: str, body_evidence: bool, runner: str, label: str, phase: str +) -> set[str]: + if not body_evidence: + non_execution(label, f"{runner} has no body-execution report policy") + prefix = "CROSSCHECK-AUTH-BODY " + channel_lines = [line for line in stderr.split("\n") if line.startswith(prefix)] + if not channel_lines: + non_execution( + label, + f"{runner} recorded no positive test-body execution evidence during " + f"the {phase} run", + ) + starts: list[str] = [] + preloads: list[str] = [] + events: list[tuple[str, str]] = [] + for line in channel_lines: + preload_match = re.fullmatch( + r"CROSSCHECK-AUTH-BODY PRELOAD ([0-9a-f]{64})", line + ) + if preload_match: + preloads.append(preload_match.group(1)) + continue + start_match = re.fullmatch( + r"CROSSCHECK-AUTH-BODY START ([0-9a-f]{64})", line + ) + if start_match: + starts.append(start_match.group(1)) + continue + event_match = re.fullmatch( + r"CROSSCHECK-AUTH-BODY EVENT ([0-9a-f]{64}) " + r"([A-Za-z0-9+/]+={0,2})", + line, + ) + if event_match: + events.append((event_match.group(1), event_match.group(2))) + continue + non_execution(label, f"{runner} emitted malformed test-body evidence") + if len(starts) != 1: + non_execution( + label, + f"{runner} emitted {len(starts)} authenticated body-channel starts " + f"during the {phase} run", + ) + nonce = starts[0] + if runner == "vitest" and preloads != [nonce]: + non_execution( + label, + "vitest did not attest its body channel at the worker preload boundary", + ) + if runner != "vitest" and preloads: + non_execution(label, f"{runner} emitted an unexpected body-channel preload") + executions: set[str] = set() + for event_nonce, payload in events: + if event_nonce != nonce: + non_execution(label, f"{runner} emitted unauthenticated test-body evidence") + try: + decoded = base64.b64decode(payload, validate=True).decode("utf-8") + entry = json.loads(decoded) + except (ValueError, UnicodeError, json.JSONDecodeError) as exc: + non_execution( + label, + f"{runner} emitted malformed test-body evidence: {exc}", + ) + if ( + not isinstance(entry, dict) + or set(entry) != {"fullName"} + or not isinstance(entry.get("fullName"), str) + or not entry["fullName"] + ): + non_execution(label, f"{runner} emitted malformed test-body evidence") + executions.add(entry["fullName"]) + if not executions: + non_execution( + label, + f"{runner} recorded no selected test body starting during the {phase} run", + ) + return executions + + +def require_jest_compatible_execution_report( result: subprocess.CompletedProcess[str], runner: str, label: str, phase: str, + body_evidence: bool, ) -> None: - """Refuse to read a test outcome out of a run that never reached the test. + """Require machine results backed by positive selected-body lifecycle evidence.""" - A non-run exits nonzero, which would otherwise read as "the baseline fails" - and, worse, as "the mutation was caught". Both readings are wrong, so the - gate names the non-run instead of scoring it. - """ + try: + report = json.loads(result.stdout) + except (json.JSONDecodeError, UnicodeError) as exc: + non_execution( + label, + f"{runner} could not start or did not emit its measured JSON report " + f"during the {phase} run ({exc}); exit {result.returncode}: " + f"{result.stderr.strip()[:500] or 'no diagnostic'}", + ) + if not isinstance(report, dict): + non_execution(label, f"{runner} emitted a non-object JSON report") + test_results = report.get("testResults") + if not isinstance(test_results, list): + non_execution(label, f"{runner} JSON omitted its testResults array") + policy = MUTATION_RUNNER_POLICIES[runner] + runtime_errors = report.get("numRuntimeErrorTestSuites") + if policy.runtime_error_field == "required-zero": + if ( + not isinstance(runtime_errors, int) + or isinstance(runtime_errors, bool) + or runtime_errors != 0 + ): + non_execution( + label, + f"{runner} reported {runtime_errors!r} runtime-error suites " + f"during the {phase} run", + ) + elif policy.runtime_error_field == "absent": + if "numRuntimeErrorTestSuites" in report: + non_execution( + label, + f"{runner} emitted an unmeasured numRuntimeErrorTestSuites field", + ) + else: + tool_fail( + f"mutation-runner policy for {runner} has unknown runtime-error " + f"contract {policy.runtime_error_field!r}" + ) + report_success = report.get("success") + if not isinstance(report_success, bool): + non_execution(label, f"{runner} JSON omitted its boolean success field") + if report_success != (result.returncode == 0): + non_execution( + label, + f"{runner} JSON success={report_success!r} contradicts exit " + f"{result.returncode}", + ) + + statuses: list[str] = [] + outcome_names: set[str] = set() + suite_without_assertion_failure = False + for suite in test_results: + if not isinstance(suite, dict): + non_execution(label, f"{runner} emitted a malformed test result") + assertions = suite.get("assertionResults") + if not isinstance(assertions, list): + non_execution(label, f"{runner} emitted a result without assertions") + suite_statuses: list[str] = [] + for assertion in assertions: + if not isinstance(assertion, dict) or not isinstance( + assertion.get("status"), str + ): + non_execution(label, f"{runner} emitted a malformed assertion result") + status_value = assertion["status"] + if status_value not in { + "passed", + "failed", + "pending", + "skipped", + "todo", + "disabled", + }: + non_execution( + label, + f"{runner} emitted unmeasured assertion status {status_value!r}", + ) + suite_statuses.append(status_value) + statuses.append(status_value) + if status_value in {"passed", "failed"}: + full_name = assertion.get("fullName") + if not isinstance(full_name, str) or not full_name: + non_execution( + label, + f"{runner} emitted an outcome without a full test name", + ) + if full_name in outcome_names: + non_execution( + label, + f"{runner} emitted ambiguous duplicate outcome name " + f"{full_name!r} during the {phase} run", + ) + outcome_names.add(full_name) + if suite.get("status") == "failed" and "failed" not in suite_statuses: + suite_without_assertion_failure = True + if suite_without_assertion_failure: + non_execution( + label, + f"{runner} failed a suite before any assertion recorded the failure " + f"during the {phase} run", + ) + + passed = statuses.count("passed") + failed_count = statuses.count("failed") + for key, measured in (("numPassedTests", passed), ("numFailedTests", failed_count)): + reported = report.get(key) + if ( + not isinstance(reported, int) + or isinstance(reported, bool) + or reported != measured + ): + non_execution( + label, + f"{runner} JSON reported inconsistent {key}={reported!r}; " + f"measured {measured} assertion results", + ) + if passed + failed_count == 0: + non_execution( + label, + f"{runner} matched no executing assertion during the {phase} run; " + "skipped or pending tests are not a test outcome", + ) + body_executions = read_javascript_body_report( + result.stderr, body_evidence, runner, label, phase + ) + missing_bodies = sorted(outcome_names - body_executions) + if missing_bodies: + non_execution( + label, + f"{runner} reported outcomes for test bodies that never started during " + f"the {phase} run: {', '.join(missing_bodies)}", + ) + if result.returncode != 0 and failed_count == 0: + non_execution( + label, + f"{runner} exited {result.returncode} without a failed assertion " + f"during the {phase} run", + ) + if result.returncode == 0 and failed_count != 0: + non_execution( + label, + f"{runner} exited 0 while reporting {failed_count} failed assertions", + ) + + +def require_test_execution( + result: subprocess.CompletedProcess[str], + runner: str, + label: str, + phase: str, + body_evidence: bool = False, +) -> None: + """Refuse to read a test outcome out of a run that never reached the test.""" combined = (result.stdout + result.stderr).strip() if sandbox_exec_failed(result): - fail( - f"{label} could not launch its {phase} test run: the sandbox failed " - f"to execute {runner}, so no test outcome exists: {combined[:500]}" + non_execution( + label, + f"the sandbox failed to execute {runner} during the {phase} run, " + f"so no test outcome exists: {combined[:500]}", + ) + policy = MUTATION_RUNNER_POLICIES[runner] + if policy.report_format == "jest-compatible-json": + require_jest_compatible_execution_report( + result, runner, label, phase, body_evidence + ) + return + if policy.report_format is not None: + tool_fail( + f"mutation-runner policy for {runner} has unknown report format " + f"{policy.report_format!r}" + ) + reason = dict(policy.non_execution_exits).get(result.returncode) + if reason is not None: + non_execution( + label, + f"{runner} never ran its named test during the {phase} run: exited " + f"{result.returncode} because {reason}, which is not a test outcome: " + f"{combined[:500]}", ) - reason = RUNNER_NON_EXECUTION_EXITS.get(runner, {}).get(result.returncode) - require( - reason is None, - f"{label} never ran its named test during the {phase} run: {runner} " - f"exited {result.returncode} because {reason}, which is not a test " - f"outcome: {combined[:500]}", - ) def require_classified_runner(runner: str, label: str) -> None: - """Refuse to certify a fix on a runner whose non-execution is unclassified. - - A mutated run that never reached the named test exits nonzero exactly like - one that caught the regression. Telling those apart needs a measured - non-execution signal for that specific runner, so a runner the gate has no - entry for cannot support a mutation proof at all. - """ + """Refuse a runner without a measured non-execution contract.""" require( - runner in RUNNER_NON_EXECUTION_EXITS, + runner in MUTATION_RUNNER_POLICIES, f"{label} cannot certify a fix through the {runner} runner: the gate " f"has no measured non-execution signal for {runner}, so a mutated run " "that never reached the named test is indistinguishable there from one " "that caught the regression. Runners whose non-execution the gate can " - f"classify: {', '.join(sorted(RUNNER_NON_EXECUTION_EXITS))}", + f"classify: {', '.join(sorted(MUTATION_RUNNER_POLICIES))}", ) @@ -2377,12 +3229,12 @@ def invocation_is_argument_free(invocation: Any) -> bool: def require_argument_free_invocation(invocation: dict[str, Any], label: str) -> None: """Refuse a mutation proof that hands the runner anything but its target. - The classified non-execution signal is a property of the runner's DEFAULT - exit semantics, and a supplied argument can change them. Measured on pytest - 9.1.1, a mutation raising during import of the named test's module exits 2 - on its own but 1 under `--continue-on-collection-errors`, and 1 has no - table entry, so the gate would certify a fix on a test never collected. A - positional argument separately adds a second target beyond test_path, the + The classified non-execution signal belongs to the exact gate-owned runner + invocation, and a reviewer-supplied argument can change it. Measured on + pytest 9.1.1, a mutation raising during import of the named test's module + exits 2 on its own but 1 under `--continue-on-collection-errors`, and 1 has + no table entry, so the gate would certify a fix on a test never collected. + A positional argument separately adds a second target beyond test_path, the only target the gate checks as tracked, symlink-free, and unreachable by the mutation patch. Requiring none closes both without an enumeration of runner flags that would go stale. @@ -2393,8 +3245,8 @@ def require_argument_free_invocation(invocation: dict[str, Any], label: str) -> not arguments, f"{label}.arguments must be empty for a mutation proof, but names " + ", ".join(repr(argument) for argument in arguments) - + ". The gate reads the mutated run's exit status through the runner's " - "default exit semantics, which an argument can change: a flag can turn " + + ". The gate reads the mutated run through the runner's measured, " + "gate-owned invocation, which an argument can change: a flag can turn " "a test that was never collected into an ordinary failure, and a " "positional argument adds a second target beyond test_path, the only " "target the gate validates as tracked, symlink-free, and unreachable " @@ -2417,7 +3269,10 @@ def validate_named_test( try: mode = candidate.lstat().st_mode except OSError as exc: - fail(f"{label}.test_path is unavailable: {exc}") + non_execution( + label, + f"named test {file_path!r} is unavailable in the tracked checkout: {exc}", + ) require(stat.S_ISREG(mode), f"{label}.test_path must be a regular file") # Anchor the symlink check at the resolved review root. Comparing against a # purely lexical absolute path also rejected symlinks in ancestors the @@ -2492,7 +3347,9 @@ def is_test_or_evidence_path(path: str) -> bool: name = candidate.name.lower() return bool( re.search(r"(?:^|[._-])(?:test|tests|spec|specs)(?:[._-]|$)", name) - or name.startswith(("test_", "spec_")) + or name.startswith( + ("test_", "spec_", "jest.config.", "vitest.config.", "vite.config.") + ) or name in {"conftest.py", "pytest.ini"} ) @@ -2531,101 +3388,48 @@ def execute_mutation_proof( proof_id = hashlib.sha256(label.encode()).hexdigest()[:10] proof_dir = proof_root / f"proof-{proof_id}" - - # Apply once before either proof run so the implementation paths themselves, - # not a repository-global setting or the reviewer prompt, select the test - # system. This inspection checkout is destroyed before the baseline. create_proof_checkout(review_dir, proof_dir, head_sha, label, deadline) - inspected_apply = run_command( - [ - "git", - "-C", - str(proof_dir), - "apply", - "--whitespace=nowarn", - str(patch_path), - ], - timeout=evidence_command_timeout(deadline, 60, f"{label} mutation inspection"), - ) - require( - inspected_apply.returncode == 0, - f"{label} mutation patch does not apply", - ) - changed = git( - proof_dir, - "diff", - "--name-only", - timeout=evidence_command_timeout(deadline, 60, f"{label} mutation diff"), - ).splitlines() - require(bool(changed), f"{label} mutation patch changes no tracked implementation") - require(test_file not in changed, f"{label} mutation changed its named test") - unexpected = sorted(set(changed) - implementation_paths) - require( - not unexpected, - f"{label} mutation changes files outside finding implementation citations: " - + ", ".join(unexpected), - ) - test_support = sorted(path for path in changed if is_test_or_evidence_path(path)) - require( - not test_support, - f"{label} mutation changes test or evidence support: " - + ", ".join(test_support), - ) - javascript_project = javascript_mutation_route( - review_dir, - changed, - test_file, - invocation["runner"], - label, - ) - remove_proof_checkout(proof_dir, label) - create_proof_checkout(review_dir, proof_dir, head_sha, label, deadline) baseline_profile = proof_dir / ".crosscheck" / "mutation-proof.sb" - if javascript_project is not None: - baseline_argv, baseline_cwd, baseline_environment = prepare_jest_invocation( - proof_dir, - javascript_project, - test_path, + # Order is load-bearing: test_arguments must run first so an absent runner is + # refused as absent, and a `direct` target as non-executable, rather than as + # an unclassified runner. Swapping these two lines changes the refusal a + # reviewer sees for a runner that is both absent and unclassified. + baseline_run = test_arguments(invocation, test_path, proof_dir, label) + require_classified_runner(invocation["runner"], label) + if invocation["runner"] == "jest": + baseline_run = prepare_jest_body_evidence( + baseline_run, label, + "baseline", + baseline_profile, deadline, ) - else: - # Order is load-bearing: test_arguments must run first so an absent - # runner is refused as absent, and a `direct` target as non-executable, - # rather than as an unclassified runner. - baseline_argv = test_arguments(invocation, test_path, proof_dir, label) - require_classified_runner(invocation["runner"], label) - baseline_cwd = proof_dir - baseline_environment = proof_environment() baseline = run_sandboxed( - baseline_argv, - cwd=baseline_cwd, + list(baseline_run.argv), + cwd=baseline_run.cwd, profile_path=baseline_profile, allow_network=False, allow_posix_ipc=False, - env=baseline_environment, + env=proof_run_environment(baseline_run), timeout=evidence_command_timeout( deadline, evidence_timeout(), f"{label} baseline test" ), description=f"{label} baseline test", ) - require_test_execution(baseline, invocation["runner"], label, "baseline") - if javascript_project is not None: - _, baseline_failed = jest_execution_summary( - baseline, label, "baseline" - ) - require( - baseline_failed == 0, - f"{label} named Jest test reports failures before mutation", - ) + require_test_execution( + baseline, + invocation["runner"], + label, + "baseline", + baseline_run.body_evidence, + ) require( baseline.returncode == 0, f"{label} named test does not pass before mutation: it ran and exited " f"{baseline.returncode} in a fresh clone holding tracked files only: " f"{(baseline.stdout + baseline.stderr).strip()[:1000] or 'no output'}", ) - remove_proof_checkout(proof_dir, label) create_proof_checkout(review_dir, proof_dir, head_sha, label, deadline) applied = run_command( @@ -2640,43 +3444,58 @@ def execute_mutation_proof( timeout=evidence_command_timeout(deadline, 60, f"{label} mutation apply"), ) require(applied.returncode == 0, f"{label} mutation patch does not apply") - mutated_changed = git( + changed = git( proof_dir, "diff", "--name-only", - timeout=evidence_command_timeout(deadline, 60, f"{label} mutated diff"), + timeout=evidence_command_timeout(deadline, 60, f"{label} mutation diff"), ).splitlines() + require(bool(changed), f"{label} mutation patch changes no tracked implementation") + require(test_file not in changed, f"{label} mutation changed its named test") + unexpected = sorted(set(changed) - implementation_paths) require( - mutated_changed == changed, - f"{label} mutation changed a different path set between proof checkouts", + not unexpected, + f"{label} mutation changes files outside finding implementation citations: " + + ", ".join(unexpected), ) - if javascript_project is not None: - mutated_argv, mutated_cwd, mutated_environment = prepare_jest_invocation( - proof_dir, - javascript_project, - test_path, + test_support = sorted(path for path in changed if is_test_or_evidence_path(path)) + require( + not test_support, + f"{label} mutation changes test or evidence support: " + + ", ".join(test_support), + ) + + mutated_profile = proof_dir / ".crosscheck" / "mutation-proof.sb" + mutated_run = test_arguments(invocation, test_path, proof_dir, label) + if invocation["runner"] == "jest": + mutated_run = prepare_jest_body_evidence( + mutated_run, label, + "mutated", + mutated_profile, deadline, ) - else: - mutated_argv = test_arguments(invocation, test_path, proof_dir, label) - mutated_cwd = proof_dir - mutated_environment = proof_environment() - mutated_profile = proof_dir / ".crosscheck" / "mutation-proof.sb" mutated = run_sandboxed( - mutated_argv, - cwd=mutated_cwd, + list(mutated_run.argv), + cwd=mutated_run.cwd, profile_path=mutated_profile, allow_network=False, allow_posix_ipc=False, - env=mutated_environment, + env=proof_run_environment(mutated_run), timeout=evidence_command_timeout( deadline, evidence_timeout(), f"{label} mutated test" ), description=f"{label} mutated test", ) - require_test_execution(mutated, invocation["runner"], label, "mutated") - proof = { + require_test_execution( + mutated, + invocation["runner"], + label, + "mutated", + mutated_run.body_evidence, + ) + require(mutated.returncode != 0, f"{label} named test still passes after mutation") + return { "test_path": test_path, "test_invocation": invocation, "mutation_patch_sha256": hashlib.sha256(patch_text.encode("utf-8")).hexdigest(), @@ -2686,17 +3505,6 @@ def execute_mutation_proof( "baseline_output": (baseline.stdout + baseline.stderr)[:MAX_CAPTURE], "mutated_output": (mutated.stdout + mutated.stderr)[:MAX_CAPTURE], } - if javascript_project is not None: - _, mutated_failed = jest_execution_summary(mutated, label, "mutated") - if mutated.returncode == 0 or mutated_failed == 0: - raise CrosscheckCoverageError( - f"{label} named Jest test still passes after the implementation " - "mutation, so the claimed fix remains blocking", - proof, - ) - return proof - require(mutated.returncode != 0, f"{label} named test still passes after mutation") - return proof def validate_ledger(value: Any, task_id: str, url: str) -> dict[str, Any]: @@ -3586,14 +4394,13 @@ def make_prompt( The command must name its helper, and its exit code plus a distinctive output marker must reproduce the defect. A prior finding is verified-fixed only when you name a tracked test, provide a structured test invocation, and provide a patch under .crosscheck/mutations/ that breaks or reverts cited implementation without changing test or evidence support. The mutation may change only implementation paths already cited by that finding. -The gate appends the named test path to the approved runner invocation, destroys all baseline state, and recreates the same clean checkout path before applying the mutation. -test_path may be a plain repository path, or a `path::selector` node id when the runner is one of: {', '.join(sorted(NODE_ID_RUNNERS))}. -The proof checkout starts as a fresh clone holding tracked files only. -For Python implementation mutations, keep using pytest; a runner that is absent or a selector that matches no test is reported as a non-execution rather than a test result and clears nothing. -For JavaScript or TypeScript implementation mutations, use the Jest or Vitest system declared by the nearest package.json that governs both changed implementation and named test. The gate currently has a positive execution protocol for Jest: it materializes lockfile-pinned dependencies offline when needed, runs only the named tracked test, and requires machine-readable evidence that tests actually executed. A package governed by another system, an ambiguous mixed-language mutation, or an unavailable offline environment is reported as CANNOT-CERTIFY and never as CLEAR. -A mutation proof takes no runner arguments at all: test_invocation.arguments must be empty, and any entry is refused by name. The gate reads the mutated exit status through the runner's default semantics, which a flag can change, and test_path is the only target it validates as tracked, symlink-free, and unreachable by your mutation patch. -Both proof runs also execute under an environment the gate constructs from a fixed allowlist rather than the one it was launched with, so no ambient variable can alter those exit semantics; name a test that needs nothing beyond PATH, HOME, and the locale. -The gate also writes a neutral pytest.ini above its own checkouts, so runner configuration from directories above them is inert; configuration tracked inside the repository still applies. +The gate positions the named test in the measured runner invocation, destroys all baseline state, and recreates the same clean checkout path before applying the mutation. +test_path may be a plain repository path, or `path::selector` when the runner is one of: {', '.join(sorted(SELECTOR_TEST_RUNNERS))}. Pytest receives its native node id; Jest and Vitest receive the part after `::` as a gate-owned test-name pattern. +The proof checkout is a fresh clone holding tracked files only. A runner or dependency that is absent, a runner that cannot start, or a selector that matches no executing test is reported as NON-EXECUTION rather than a test result and clears nothing. +A mutation proof may name only a runner whose non-execution contract the gate has measured, currently: {', '.join(sorted(MUTATION_RUNNER_POLICIES))}. On any other runner the gate cannot tell a test that caught the mutation from one that never ran, so it refuses to certify the fix rather than guess. +A mutation proof takes no reviewer-supplied runner arguments at all: test_invocation.arguments must be empty, and any entry is refused by name. The gate executes only the runner policy's measured arguments, and test_path is the only target it validates as tracked, symlink-free, and unreachable by your mutation patch. +Both proof runs also execute under an environment the gate constructs from a fixed allowlist rather than the one it was launched with, so no ambient variable can alter those semantics; name a test that needs nothing beyond PATH, HOME, and the locale. +The gate writes neutral pytest, Jest, and Vitest configuration above its own checkouts, so runner configuration from directories above them is inert; configuration tracked inside the repository still applies. The gate will independently run every reproduction and every mutation proof. If you cannot reproduce a concern, return it as a suspicion; suspicions block the merge. Silence never closes an existing finding. diff --git a/docs/crosscheck.md b/docs/crosscheck.md index 8bdc055f8ea..32f827ffc39 100644 --- a/docs/crosscheck.md +++ b/docs/crosscheck.md @@ -234,70 +234,114 @@ That inspection carries its own larger budget so such a state is refused by name A `verified-fixed` update must name a tracked test and provide an implementation-only patch under `.crosscheck/mutations/`. It supplies an approved test runner plus a structured argument array, never a free-form shell command. +Approval as a general test runner is not certification permission: `MUTATION_RUNNER_POLICIES` in `bin/fm-crosscheck.py` is the single declaration point for runners whose non-execution contract has been measured. +The currently measured certification runners are `pytest`, `jest`, and `vitest`. + An approved runner is a NAME, and the gate resolves that name into an invocation rather than assuming a bare binary on `PATH`. -Before either proof run, the gate applies the mutation in a disposable inspection checkout and selects the certification system from the mutated implementation paths themselves. -A JavaScript or TypeScript mutation must resolve with its named test to one nearest tracked `package.json`, and that package's test script or dependencies must declare one unambiguous Jest or Vitest system. -A mixed JavaScript/Python mutation, a test outside the changed package, an ambiguous declaration, or a proof naming a different runner is `CANNOT-CERTIFY`, never `CLEAR`. -Python mutation behavior remains on its existing pytest route exactly as before. -This matters because every Python repository in this fleet is uv-managed: a bare `pytest` is routinely absent there, while `uv run pytest` is the invocation that works, and `python3 -m pytest` cannot be expressed in the vocabulary at all because `python3` is a file runner whose command line puts the test path before its arguments. -`pytest` therefore resolves through `uv run pytest`, then `python3 -m pytest`, then the bare binary. -Order is load-bearing: inside a uv project a bare `pytest` can exist and resolve against a different environment than the repository uses, so finding it first would run the named test under an interpreter the project never selected. -The uv rung is offered only when a uv project actually governs the named test, discovered by searching upward from that test to the checkout root, and it is passed to `uv run --project` so a monorepo service directory is selected without moving the working directory the test path is relative to. -Each rung except the last identifies itself before being trusted; the last is the plain runner name and is accepted on presence, exactly as before, so the ladder can never turn a working setup into a refusal. -Keeping the declared name is what preserves pytest's `path::selector` node-id support, which a separate runner name for module invocation would have silently dropped. -That array must be empty for a mutation proof, and any entry is refused by name. -The classified non-execution signal is a property of the runner's default exit semantics, and a supplied flag can change them: measured on pytest 9.1.1, a mutation raising during import of the named test's module exits 2 on its own but 1 under `--continue-on-collection-errors`, and 1 carries no classification, so the gate would certify a fix on a test that was never collected. -A positional argument separately adds a second target, and `test_path` is the only target the gate validates as tracked, symlink-free, and unreachable by the mutation patch, so the verdict could come from a file the gate never validated. -Requiring no arguments closes both without an enumeration of runner flags that would go stale. -Reviewer-supplied argv is only half of it: both proof runs also execute under an environment the gate constructs from `PROOF_ENVIRONMENT_ALLOWLIST` rather than the one it was launched with, because pytest appends `PYTEST_ADDOPTS` to the command line, so an operator with `--continue-on-collection-errors` exported would reproduce the same bypass on every proof with no reviewer involved. -That list is an allowlist because it fails closed - a variable that is needed but missing breaks the baseline run, which must exit 0, so the proof is refused where it can be seen, while an unlisted variable on a denylist would sail through silently. -The constructed environment is applied to the mutation-proof runs and not to reproduction re-execution: a proof's exit status is what decides clear versus not clear, whereas ambient interference with a reproduction can only push it toward refusal, and reproduction commands run through a login shell that re-imports operator profile state regardless. -Configuration files are the third channel into the same semantics: pytest's `locate_config` walks every parent of its target to the filesystem root and stops at the first `pytest.ini`, `tox.ini`, `setup.cfg`, or `pyproject.toml` it finds, so an operator config above the gate's temporary root would set `addopts` for every proof on the machine. -Crosscheck writes a neutral empty `[pytest]` `pytest.ini` into that temporary root before anything runs, ending the walk inside a directory the gate owns and neutralising every ini setting from above rather than only `addopts`. -The proof checkouts and the review checkout are both children of that root, so the one file covers reproduction re-execution as well; the boundary is the root the gate owns, not any child of it. -This is not free: for a repository carrying no pytest config of its own, rootdir becomes the gate's temporary root instead of the checkout, which widens conftest discovery by that one empty gate-owned directory. -The reviewed repository's own config still takes precedence, because it sits closer to the named test, and that surface stays deliberately accepted. -The same rule is not applied when replaying a recorded proof, so a ledger written before it still loads; instead a recorded proof whose invocation carried arguments no longer certifies its finding, which reverts to blocking and can be re-proved in band by a fresh review. -Crosscheck destroys the mutation-inspection checkout, creates a clean baseline checkout at the exact reviewed head, confirms the named test passes, destroys that entire checkout, recreates the same path from the exact head, applies the patch, and requires the same test to fail. +`pytest` resolves through `uv run pytest`, then `python3 -m pytest`, then the bare binary, preserving the existing uv-aware behavior and native `path::selector` node ids. +Jest and Vitest run from the nearest package directory found by walking from the named test to the checkout root, so a nested monorepo package loads its own tracked configuration and package-relative imports. +For those Node runners the gate prefers an executable `node_modules/.bin/` in that package and then checks `PATH`. +The proof checkout contains tracked files only, so ordinary untracked `node_modules` dependencies are absent there unless the runner is otherwise available; an absent runner or a runner that starts without its required dependencies is `NON-EXECUTION`, never a pass or silent skip. + +`test_path` may be a plain repository path or `path::selector` for a measured selector runner. +Pytest receives the full native node id. +Jest and Vitest receive the path before `::` as their sole file target and the part after `::` as a gate-owned `--testNamePattern` value, which can express the paired control and regression selector used by platform-v3's hand proof without admitting reviewer flags. +The exact gate-owned JavaScript invocations are `jest --json --runTestsByPath [--testNamePattern ]` and `vitest run --reporter=json [--testNamePattern ]`. +Both machine reports must contain at least one `passed` or `failed` assertion before the run counts as execution, and a nonzero mutated run must contain a failed assertion before it can certify the mutation. +A missing or malformed report, a runtime-error suite, an empty assertion list, an all-skipped or all-pending selector result, an inconsistent assertion count, a success/exit contradiction, and a nonzero exit with no failed assertion are all `NON-EXECUTION` and clear nothing. +This report contract is necessary because both measured JavaScript runners exit 0 when a test-name pattern matches no test. + +The reviewer-supplied `test_invocation.arguments` array must remain empty for every mutation proof, and any entry is refused by name. +The classified contract belongs to the exact gate-owned invocation, and a supplied flag can rewrite it: measured on pytest 9.1.1, `--continue-on-collection-errors` turns an import-time non-execution from exit 2 into an ordinary exit 1. +A positional argument separately adds a second target, while `test_path` is the only target the gate validates as tracked, symlink-free, and unreachable by the mutation patch. +Requiring no reviewer arguments closes both routes without a runner-specific flag denylist. + +Both proof runs execute under an environment constructed from `PROOF_ENVIRONMENT_ALLOWLIST` rather than the caller's environment. +That list is an allowlist because it fails closed: a missing required variable breaks the baseline, while an unlisted variable on a denylist would silently alter the measured runner contract. +The constructed environment applies to mutation proofs and not reproduction re-execution, because proof results can clear a finding while ambient interference with a reproduction can only force refusal. +Crosscheck writes neutral pytest, Jest, and Vitest configuration above the review and proof checkouts, ending upward config discovery inside the gate-owned root while allowing the reviewed repository's closer tracked configuration to win. +For a repository with no pytest config, the neutral boundary makes the temporary root pytest's rootdir and widens conftest discovery by one empty gate-owned directory. +A recorded proof whose invocation predates the empty-arguments rule still loads, but no longer clears its finding and can be re-proved in band. + +Crosscheck creates one clean checkout at the exact reviewed head, confirms the named test passes, destroys the entire checkout, recreates the same path from the exact head, applies the patch, and requires the same test to fail. Destroying all readable baseline state before the mutated run prevents a test from manufacturing causality through a predictable sibling checkout. -For Jest, each clean proof checkout must begin without a package-local runner; a tracked or otherwise preexisting `node_modules/.bin/jest` is refused rather than accepted as provenance. -The gate requires exactly one tracked package lock whose root declares Jest and whose `node_modules/jest` entry binds a semantic version to the official npm registry tarball with valid sha512 integrity. -Starting from that entry, it resolves every dependency, optional dependency, and peer dependency through the lockfile's exact nested and hoisted `node_modules` paths and authenticates the complete reachable runtime closure before installation. -Every closure entry must occupy a canonical package path and bind its own name and semantic version to its exact official npm registry tarball with valid sha512 integrity. -Local, linked, workspace, Git, URL, custom-registry, missing-integrity, project-npm-configured, and currently pnpm-governed Jest routes are explicit `CANNOT-CERTIFY` outcomes. -The gate detects the project's declared Node major, selects a matching interpreter from the standard version-manager directories, and materializes dependencies afresh from that `package-lock.json` with `npm ci --offline --ignore-scripts` inside the no-network proof sandbox and empty gate-owned npm user and global configuration. -After installation, it requires every closure package to be a real non-symlink directory inside the package tree whose package name, version, and runtime dependency declarations match its authenticated lock entry. -It also requires `node_modules/.bin/jest` to resolve to the executable `bin/jest.js` inside the authenticated materialized Jest package. -The selected Node path remains bound through dependency installation and the baseline and mutated Jest runs. -A cold dependency cache, missing or ambiguous lockfile, preexisting runner, unavailable package manager or Node version, unsupported Vitest route, or invalid materialized Jest package or binary is `CANNOT-CERTIFY` and never a test verdict. -The gate invokes Jest with its own fixed `--runInBand --runTestsByPath --ci --no-cache --json` protocol and accepts a fix only when the baseline JSON reports at least one executed passing test and the mutated JSON reports at least one executed failing test. -A Jest test that executes and stays green under the mutation is durably downgraded to `claimed-fixed`, keeping the finding and the run `blocking` instead of turning inadequate coverage into an infrastructure outcome. -Proof sandboxes also omit shared POSIX IPC and give each run private writable temporary and cache state, while shared host temporary directories remain outside the write policy. -The named test must be a canonical tracked regular file; symlinks are rejected so a patch cannot mutate the executed target through an unchanged alias. -Symlink rejection is anchored at the resolved review checkout, so a symlink inside the repository is still refused while a symlinked ancestor above the firstmate home is not mistaken for one. -`test_path` may also be a `path::selector` node id for a runner that accepts one; every path-shaped check reads the part before `::` while the runner receives the full selector. -The gate positions the tracked test path itself as the interpreter script or test-framework target; generic command launchers are not approved runners. -A run that never reached the named test is not a test result in either direction. -The gate resolves the named runner to an absolute executable before launching, and treats an absent runner, a failed sandbox exec, and a runner-reported non-execution as named non-executions. -That matters in both directions: such a status must not condemn a baseline run, and must not vindicate a mutated one, because a mutation that merely broke collection would otherwise read as a caught regression. -Pytest uses the gate's measured usage and no-tests-collected exit statuses; Jest uses positive machine-readable executed-test counts instead of inferring execution from its exit code. -Every other runner remains unable to certify until it has its own positive or measured non-execution protocol. -The proof checkout starts as a fresh clone carrying tracked files only, and any language environment it needs must be reconstructed through the bounded routes above. +Proof sandboxes omit shared POSIX IPC and give each run private writable temporary and cache state, while shared host temporary directories remain outside the write policy. +The named test must be a canonical tracked regular file, and symlinks are rejected so a patch cannot mutate the executed target through an unchanged alias. +Symlink rejection is anchored at the resolved review checkout, so a symlink inside the repository is refused while a symlinked ancestor above the firstmate home is not mistaken for one. +The gate resolves the named runner before launch and reports an absent binary, a failed sandbox exec, a missing named test, and every measured runner non-execution explicitly as `NON-EXECUTION`. +For Jest and Vitest, the gate accepts a passed or failed assertion record only when an authenticated runner-owned stderr channel recorded entry into that exact selected test body. +The channel creates a random nonce outside the project-writable checkout, emits body events from a closure unavailable to project code, and treats missing, duplicate, malformed, or mismatched channel records as `NON-EXECUTION`; project-writable marker files are never evidence. +For Jest, a sandboxed `--showConfig` preflight resolves the selected project without replacing its tracked environment or setup. +The Jest executable, environment, runner, test framework, runtime, and measured Circus run and invocation modules must resolve through the exact Jest 29.7.0 executable package graph, both modules must match their measured SHA-256 digests, and none may resolve inside the proof checkout. +The Jest channel is pinned in Node's `execArgv` before configuration, environments, transforms, and `setupFiles`, and the measured run stays in that preloaded process before a digest-checked gate wrapper loads the exact canonical runner and interposes the canonical Circus source as Jest imports it into the test VM. +That interposed source captures the registered function before project hooks, requires the same identity at the canonical invocation, and enters it through a preload-captured `Reflect.apply`, leaving the original function object, arity, callback behavior, and project-visible properties unchanged rather than trusting publicly dispatchable lifecycle events or project-mutable callable properties. +Generator tests fail closed as `NON-EXECUTION` because Jest delegates their advancement through `co` after the authenticated invocation boundary. +Custom resolvers, sibling runtime copies, project-local suffix spoofs, and per-file environment overrides fail closed when that boundary cannot be established. +For Vitest, the executable and runtime export must match the measured external Vitest 4.1.5 package, while the gate-owned config imports the discovered tracked `vitest.config.*` or `vite.config.*`, preserves its plugins, aliases, setup, and ordinary test options, and locks the body runner during resolved configuration. +Every Vitest worker imports that runner package through the gate-pinned `test.execArgv` before worker initialization, and its location beneath a gate-owned external `node_modules` directory keeps it on Node's native import path instead of the tracked Vite transform pipeline. +The runner captures each registered function in `onBeforeRunTask` before project hooks, requires identical identity at `runTask`, and emits no body event when `setFn` or another hook replaces it. +The Vitest parent starts with a gate-owned `NODE_OPTIONS` preload that registers the loader, instantiates the spawn-attesting facade in that loader's module graph, and removes the preload from the native environment before project code runs. +The measured Vitest loader boundary requires Node 20.6.0 or newer, and an absent, unreadable, or older Node runtime is a named `NON-EXECUTION` that clears nothing. +That loader redirects only the digest-pinned Vitest fork launcher's private `node:child_process` import to a gate facade, so the exact worker launch rejects project-provided options and injects the gate preload while every project-visible child-process export keeps its native object, name, identity, and descriptors. +Immediately before dispatch, the facade attests the captured native `ChildProcess.prototype.spawn` identity after all project-provided options are copied, so a tracked configuration cannot reorder worker preloads through the transitive spawn call. +The final gate plugin rechecks that tracked configuration and plugin resolution left no ambient `NODE_OPTIONS`, while Node's native string-coercing `process.env` object remains unchanged. +The gate retains each tracked plugin's prototype and property descriptors while interposing only runner-loading hooks, so class-based and non-enumerable startup hooks keep their project semantics. +Custom runners, browser mode, projects, threads, custom pools, project worker arguments, serialized defines, ambient or `test.env.NODE_OPTIONS`, custom environments, and per-file environment overrides fail closed when that ordering cannot be attested. +A failed `beforeAll` or `beforeEach` hook can create a failed assertion record without entering the test function, so it remains `NON-EXECUTION` and clears nothing. +Duplicate passed or failed full names are ambiguous and remain `NON-EXECUTION`, so one same-named body's marker can never satisfy another outcome. +That distinction applies to baseline and mutation runs alike because a run that never entered the selected test body can neither condemn the baseline nor vindicate the mutation. +A runner absent from `MUTATION_RUNNER_POLICIES` is refused even if it is generally approved, because guessing its non-execution signal would allow a forged clearance. The patch may modify only non-test implementation paths already cited by the durable finding. -It cannot modify the named test, conventional test trees, fixtures, or Crosscheck evidence support. +It cannot modify the named test, conventional test trees, fixtures, Jest/Vitest/Vite runner config, or Crosscheck evidence support. + +### Adding a mutation-proof runner + +Do not add a runner to `MUTATION_RUNNER_POLICIES` until its exact gate-owned invocation has an empirically distinguishable execution signal. +Use the real runner in a clean scratch package with one mutation-insensitive control test and one mutation-sensitive regression test. +Run the exact proposed argv with no reviewer arguments for seven cases: matched pass, matched assertion failure, unmatched selector, missing target, missing dependency during collection, conventional tracked configuration that fails at startup, and a selected test whose `beforeAll` or `beforeEach` hook fails before its body. +Record the runner and Node versions, date, exact commands, exit statuses, stdout, stderr, authenticated channel start and body events, and attempted project-writable forgeries before encoding anything. +When the runner boundary uses a Node API introduced after the runner's own engine minimum, run the matrix on the last version without that API and the first version with it, then encode the latter as an explicit minimum runtime. +A future version whose output no longer satisfies the declared parser fails closed as `NON-EXECUTION`; do not loosen the parser until that version has been measured through the same matrix. +If a runner exposes no signal that distinguishes a failed assertion from failed startup or non-collection, it is not eligible for certification. -### Known Python limitation: the mutated pytest exit status is an inference, not proof +The JavaScript profile was measured on 2026-08-09 with Jest 29.7.0 and on 2026-08-10 with Vitest 4.1.5 on Node 20.20.2 using these exact command shapes. +The Vitest loader boundary was also measured on Node 20.5.1 and 20.6.0: Node 20.5.1 cannot import the gate preload because `node:module` does not export `register`, while Node 20.6.0 loads it and reaches the runner. +Crosscheck therefore rejects Node 20.5.1 and older before proof execution and accepts Node 20.6.0 and newer at this boundary. -The Jest route uses positive JSON execution counts and does not share this limitation. -Read the four Python guards above together and the shape of the remaining problem is visible. -The pytest route concludes "the named test detected the regression" from one fact: the mutated run exited non-zero. -That status is not a property of the test alone. It is influenced by reviewer-supplied argv, by the ambient environment, by repository and ancestor configuration, and by the runner's own version, and each of those four channels was closed only after it was found - a positional second target, a collection-error flag, `PYTEST_ADDOPTS`, and an ancestor ini file. -An installed runner plugin is a known and accepted fifth door. -Closing channels one at a time is unbounded work with no completion criterion, so the list above should be read as hardening, not as a proof of soundness. +```sh +jest --showConfig --json +node --require=/gate/jest-body-preload.cjs /runtime/node_modules/jest/bin/jest.js --json --runInBand --runTestsByPath /proof/regression.test.js --testNamePattern 'across chats resets state' --testRunner /gate/node_modules/crosscheck-jest-body-runner/index.cjs +NODE_OPTIONS=--import=/gate/vitest-launch-preload.mjs vitest run --reporter=json regression.test.js --config /gate/vitest-body-probe.config.mjs --testNamePattern 'across chats resets state' +``` -The planned replacement is POSITIVE PROOF OF EXECUTION: requiring the mutated run to demonstrate that the named test actually ran, rather than inferring it from an exit code. -The leading candidate is a control test - a second tracked test the mutation should not affect, required to PASS while the named test fails - because it needs no per-runner knowledge and no enumeration of the ways a status can be rewritten. -Until that lands, the pytest exit-status inference remains this gate's weakest link, and the four closed channels do not make it sound. +The matched baseline exited 0 and recorded one passed selected assertion on both runners. +The assertion mutation exited 1 and recorded one failed selected assertion plus a matching body-start record on both runners. +An unmatched selector exited 0 on both runners, with Jest recording only `pending` assertions and Vitest recording only `skipped` assertions. +A missing target exited 1 with zero assertions, using a Jest runtime-error suite and an empty Vitest `testResults` array. +A missing imported dependency exited 1 with a failed suite and an empty `assertionResults` array on both runners, with Jest additionally reporting one runtime-error suite. +A conventional `jest.config.cjs` or `vitest.config.js` that threw during loading exited 1 and emitted no JSON stdout on either runner. +A tracked `setupFilesAfterEnv` or `setupFiles` entry that failed before the body remained active after probe injection and was classified as `NON-EXECUTION`. +A failed `beforeAll` or `beforeEach` hook recorded a failed assertion without a matching authenticated body event and was classified as `NON-EXECUTION`, even when an earlier tracked `setupFiles` entry wrapped `fs.writeSync`, the hook forged the former predictable marker file, or project code dispatched a native Circus `test_fn_failure` event. +The gate-owned Vitest config merges the tracked project config and selects a natively imported probe that extends the runtime `TestRunner` export. +A tracked Vitest plugin cannot transform that external runner, cannot replace its frozen resolved-config binding, and retains prototype-defined startup hooks after protection. +A transform forgery, resolved-config mutation, custom-environment forgery, primitive replacement, selected-function replacement, unattested worker configuration, ambient worker preload, dropped class-plugin hook, or second runner construction is classified as `NON-EXECUTION`. +The measured fork-launch module must also retain its declared digest, and a project plugin sees the same native child-process identity and function name that it sees without the gate. +Those observed shapes are what the shared `jest-compatible-json` report policy and runner-specific body probes encode; exit status and assertion status are deliberately insufficient on their own. + +To add a future runner, add one policy entry carrying its invocation ladder, gate-owned arguments, selector mode, project-root rule, report format, runtime version and source digests, measured non-execution exits, and dated measurement string. +Add a parser only when the runner uses a genuinely new measured report format, and keep that parser selected by the policy rather than branching throughout the gate. +Add hermetic behavior coverage for every non-execution shape plus a real-runner end-to-end baseline-pass/control-pass/mutation-only-regression-fail certification before enabling the policy. + +### Known limitation: pytest's mutated exit status is still an inference + +Jest and Vitest now provide positive test-body execution evidence through gate-owned lifecycle probes paired with their measured machine reports. +Pytest still concludes that the named test detected the regression from the mutated nonzero exit after excluding its measured non-execution statuses. +That status is influenced by reviewer-supplied argv, ambient environment, repository and ancestor configuration, runner version, and installed plugins. +The positional-target rule, argument refusal, environment allowlist, and neutral ancestor config close known channels, but an installed plugin remains an accepted door and the list is hardening rather than a proof of soundness. + +The planned pytest replacement is positive proof of execution rather than exit-code inference. +The leading candidate remains a control test that the mutation should not affect, required to pass while the named test fails. +Until that lands, pytest's exit-status inference remains this gate's weakest runner contract. ## Refusal and liveness @@ -372,10 +416,6 @@ The read adapter exposes no merge subcommand; only the gate-refused `fm-crossche The installed reviewer invocation was exercised successfully with `--output-schema`, `--output-last-message`, `--model gpt-5.6-sol`, and `model_reasoning_effort="xhigh"` before production code used those flags. The installed Claude invocation was exercised successfully with a private `HOME`, selected-account `CLAUDE_CONFIG_DIR` and `CLAUDE_SECURESTORAGE_CONFIG_DIR`, `--model claude-opus-5`, `--effort xhigh`, `--dangerously-skip-permissions`, `--tools Bash,Read,Glob,Grep`, `--no-session-persistence`, `--output-format json`, and `--json-schema` before production code used those flags. The installed `/usr/bin/sandbox-exec` was also exercised with the generated profile: a write inside the allowed review directory succeeded, while sibling and `/private/tmp` writes failed with `Operation not permitted`. -On 2026-08-09 the Jest mutation route was exercised at relvino PR 1049 head `5649c234b0f258cde4d62870759e353fade5ff3d` in a fresh exact-head clone. -The gate selected Node 20.20.2 for the package's `20.x` declaration, used npm 10.8.2 and the tracked package lock to materialize Jest 29.7.0 offline with lifecycle scripts disabled, and ran the fixed `--runInBand --runTestsByPath --ci --no-cache --json` protocol under the no-network sandbox. -The tracked `V3PreviewPane.test.tsx` reported 33 executed and zero failed tests at baseline; replacing the session key with one shared key reported the same 33 executed tests with two failures, so the result demonstrated positive mutation detection rather than a runner-status inference. - ## Validation evidence boundaries `tests/fm-github-pr.test.sh` is hermetic coverage using checked-in TOON shapes. @@ -386,16 +426,14 @@ Its tracked `test_real_claude_sandbox_executes_exact_sha_git_diff` case is an op Ordinary CI prints a named skip for this network- and credential-dependent guard instead of substituting fake-only coverage. The retained live runtime proof is the change receipt for this patch; the opt-in test is the repeatable regression guard for future environments. Its `test_pytest_runner_resolves_through_a_uv_aware_ladder` case is the named regression for runner-name resolution: it pins monorepo uv-project discovery, the skipped uv rung outside a project, the unchanged absent-runner refusal, and pytest's retained node-id support. -Its `test_account_less_known_provider_lane_is_reviewable` case is the named regression for an account-less Pi lane whose provider-slot identity is unreadable: it requires a cross-provider reviewer to clear it and a same-provider reviewer to remain refused. +Its `test_javascript_runner_policy_is_declared_once` case pins the nearest-package working directory, exact gate-owned Jest and Vitest arguments, selector translation, neutral ancestor configs, and single policy registry. +Its `test_javascript_runners_certify_platform_shaped_mutation_proofs` case executes end-to-end Jest and Vitest proofs where the control passes in both runs and the regression fails only after mutation. +Its real Jest and Vitest integration cases retain tracked setup, aliases, and plugins while injecting body evidence, preserve Jest callback and callable-property semantics plus Node's native environment coercion and child-process identity, then require tracked mutation-only startup failures, hook body replacement, generator tests, an early Jest channel-capture attempt, forged Circus dispatch, project-local and sibling Jest runtime spoofs, Vitest primitive, transform, and resolved-config mutations, ambient or unattested workers, custom environments, and class-plugin startup failures to remain `NON-EXECUTION`. +Its `test_javascript_non_executions_clear_nothing` case executes the measured unmatched-selector, startup-failure, missing-dependency, and missing-test shapes and requires every one to retain the open finding. +Its `test_account_less_known_provider_lane_is_reviewable` case is the named regression for account-less lanes: it drives a Pi lane with no `account_home` and a slot-qualified model, requires a cross-provider reviewer to clear it, and requires a same-provider reviewer to be refused. Its `test_same_model_relaxation_requires_proven_separate_account` case provides a launch-recorded routed Pi identity and proves that default and explicit-off policy reject same-provider review, opt-in still rejects missing, same, or unreadable account proof, ambient credential drift cannot replace the snapshot, and only a recorded-distinct OpenAI account becomes eligible. Its `test_legacy_author_admission_is_exact_and_explicit` case proves that absent and stale admissions fail closed, replacement-unavailable acknowledgement is mandatory, a matching entry never synthesizes author identity, a modern snapshot cannot be downgraded, and an admitted reviewer still needs a readable executing account. Its `test_legacy_author_admission_is_visible_in_prompt_and_evidence` case proves that the weaker mode is explicit in the adversarial prompt, ledger, and readable report, including both same-model and unproven-author labels. -Its `test_typescript_jest_mutation_proof_can_clear` and `test_inadequate_typescript_jest_coverage_stays_blocking` cases prove that package-governed Jest coverage can certify a TypeScript fix while a named Jest test that stays green under mutation keeps the finding blocking. -Its `test_preexisting_jest_runner_cannot_certify` case proves that a committed Jest-shaped output script is refused before package-manager materialization, and `test_local_fake_jest_package_cannot_certify` proves a lockfile-routed local fake package cannot substitute for official registry provenance. -Its `test_local_transitive_jest_package_cannot_certify` case keeps top-level Jest registry-authenticated while substituting a local `jest-cli`, and proves that every transitive runtime package must remain inside the authenticated closure. -Its `test_jest_runs_under_declared_node_major` case proves the selected Node path governs installation and both proof executions. -Its `test_typescript_without_usable_route_is_cannot_certify` case proves that an unsupported package-governed route writes and reports `CANNOT-CERTIFY` rather than silently clearing or manufacturing a code verdict. -Its `test_python_mutation_proof_is_byte_exact` case compares the complete normalized Python proof record to the pre-Jest shape so the new language route cannot drift existing pytest evidence. Its `test_claude_execution_home_always_binds_the_keychain` case is the named regression for the private-`HOME` Keychain bind, and it fails if the bind is made conditional on `.credentials.json` again. Its `test_moved_default_branch_stays_reviewable` case is the named regression for base drift: it advances the fake default branch past the PR's branch point, then requires the run to review against the merge base, record it, and still verify. Its `test_unavailable_reviewer_fails_over_to_the_next_account` case covers reviewer failover using the observed zero-turn Claude error envelope, and asserts the ledger records the abandoned attempt with the reason the reviewer reported rather than a truncated envelope. diff --git a/tests/fm-checkout-identity-cost.test.sh b/tests/fm-checkout-identity-cost.test.sh index b338619f7fa..14a86e097f5 100755 --- a/tests/fm-checkout-identity-cost.test.sh +++ b/tests/fm-checkout-identity-cost.test.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +# shellcheck source=tests/test-entry.sh +. "$(dirname "$0")/test-entry.sh" # Cost-shape tests for the checkout identity primitives in # bin/fm-checkout-lock-lib.sh. # diff --git a/tests/fm-crosscheck.test.sh b/tests/fm-crosscheck.test.sh index 927e54b51b5..1c77ad9a168 100755 --- a/tests/fm-crosscheck.test.sh +++ b/tests/fm-crosscheck.test.sh @@ -117,7 +117,6 @@ EOF install_pi_fake "$case_dir" install_sandbox_fake "$case_dir" install_pytest_fake "$case_dir" - install_jest_package_manager_fake "$case_dir/pathbin" install_path_helper "$case_dir" printf '%s\t%s\t%s\n' "$case_dir" "$base" "$head" } @@ -132,58 +131,6 @@ install_path_helper() { chmod +x "$case_dir/pathbin/fm-test-helper" } -install_jest_package_manager_fake() { - local node_bin=$1 - mkdir -p "$node_bin" - cat > "$node_bin/node" <<'SH' -#!/usr/bin/env bash -[ "${1:-}" = --version ] || exit 92 -printf 'v20.11.0\n' -SH - cat > "$node_bin/npm" <<'SH' -#!/usr/bin/env bash -set -u -[ "${1:-}" = ci ] || exit 93 -mkdir -p node_modules/.bin node_modules/import-local node_modules/jest/bin node_modules/jest-cli -cat > node_modules/jest/package.json <<'JSON' -{"name":"jest","version":"29.7.0","bin":"./bin/jest.js","dependencies":{"import-local":"^3.0.2","jest-cli":"^29.7.0"}} -JSON -cat > node_modules/import-local/package.json <<'JSON' -{"name":"import-local","version":"3.1.0"} -JSON -cat > node_modules/jest-cli/package.json <<'JSON' -{"name":"jest-cli","version":"29.7.0"} -JSON -cat > node_modules/jest/bin/jest.js <<'JEST' -#!/usr/bin/env bash -set -u -[ "$(node --version)" = v20.11.0 ] || exit 94 -test_path= -for argument in "$@"; do - case "$argument" in - --*) ;; - *) test_path=$argument ;; - esac -done -[ -n "$test_path" ] && [ -f "$test_path" ] || exit 4 -status=0 -if ! grep -q 'INADEQUATE_PREVIEW_SCOPE_TEST' "$test_path" \ - && ! grep -q 'previewScope = "fixed"' src/preview.ts; then - status=1 -fi -if [ "$status" -eq 0 ]; then - printf '%s\n' '{"numTotalTests":1,"numFailedTests":0,"success":true}' -else - printf '%s\n' '{"numTotalTests":1,"numFailedTests":1,"success":false}' -fi -exit "$status" -JEST -chmod +x node_modules/jest/bin/jest.js -ln -s ../jest/bin/jest.js node_modules/.bin/jest -SH - chmod +x "$node_bin/node" "$node_bin/npm" -} - # A node-id runner standing in for pytest. It reproduces the three outcomes the # gate must tell apart: the named test ran and passed (0), ran and failed (1), # and never ran because the selector resolved to nothing (4 usage / 5 collected). @@ -268,6 +215,192 @@ SH chmod +x "$case_dir/pathbin/pytest" "$case_dir/pathbin/python3" } +# Jest 29.7.0 and Vitest 4.1.5 both emit Jest-compatible JSON for the exact +# gate-owned command shapes declared in fm-crosscheck.py. These doubles model +# the measured distinction: passed/failed assertion records prove execution, +# while a no-match run contains skipped-only assertions and a startup failure +# emits no JSON report at all. +install_javascript_runner_fake() { + local case_dir=$1 runner=$2 executable driver shared_runtime + mkdir -p "$case_dir/pathbin" + if [ "$runner" = jest ]; then + executable="$case_dir/runtime/node_modules/jest/bin/jest.js" + mkdir -p "$(dirname "$executable")" \ + "$case_dir/runtime/node_modules/jest-environment-node/build" \ + "$case_dir/runtime/node_modules/jest-runner/build" \ + "$case_dir/runtime/node_modules/jest-circus/build" \ + "$case_dir/runtime/node_modules/jest-runtime/build" + : > "$case_dir/runtime/node_modules/jest-environment-node/build/index.js" + : > "$case_dir/runtime/node_modules/jest-runner/build/index.js" + : > "$case_dir/runtime/node_modules/jest-circus/runner.js" + : > "$case_dir/runtime/node_modules/jest-circus/build/run.js" + : > "$case_dir/runtime/node_modules/jest-circus/build/utils.js" + : > "$case_dir/runtime/node_modules/jest-runtime/build/index.js" + printf '{"name":"jest","version":"29.7.0"}\n' \ + > "$case_dir/runtime/node_modules/jest/package.json" + printf '{"name":"jest-environment-node","version":"29.7.0","main":"build/index.js"}\n' \ + > "$case_dir/runtime/node_modules/jest-environment-node/package.json" + printf '{"name":"jest-runner","version":"29.7.0","main":"build/index.js"}\n' \ + > "$case_dir/runtime/node_modules/jest-runner/package.json" + printf '{"name":"jest-circus","version":"29.7.0"}\n' \ + > "$case_dir/runtime/node_modules/jest-circus/package.json" + printf '{"name":"jest-runtime","version":"29.7.0","main":"build/index.js"}\n' \ + > "$case_dir/runtime/node_modules/jest-runtime/package.json" + driver="$case_dir/runtime/jest-fake.sh" + else + shared_runtime="$TMP_ROOT/vitest-4.1.5-runtime" + if [ ! -x "$shared_runtime/node_modules/.bin/vitest" ]; then + npm install --prefix "$shared_runtime" --no-save --no-package-lock \ + --ignore-scripts --legacy-peer-deps vitest@4.1.5 >/dev/null \ + || fail "Vitest 4.1.5 fixture installation failed" + fi + mkdir -p "$case_dir/runtime/node_modules" + cp -R "$shared_runtime/node_modules/vitest" \ + "$case_dir/runtime/node_modules/vitest" + executable="$case_dir/runtime/node_modules/vitest/vitest.mjs" + driver="$executable" + fi + cat > "$driver" <<'SH' +#!/usr/bin/env bash +set -u +runner=${FM_FAKE_RUNNER:-$(basename "$0")} +marker_dir=${PATH%%:*} +startup_marker=$marker_dir/$runner-startup-failure +missing_dependency_marker=$marker_dir/$runner-missing-dependency +hook_failure_marker=$marker_dir/$runner-hook-failure +duplicate_name_marker=$marker_dir/$runner-duplicate-name +[ ! -f "$startup_marker" ] || { + echo "MEASURED $runner STARTUP FAILURE" >&2 + exit 1 +} +if [ "$runner" = jest ] && [ "${1:-}" = --showConfig ]; then + [ "${2:-}" = --json ] || exit 90 + runtime_root=$(dirname "$(dirname "$0")")/runtime/node_modules + printf '{"configs":[{"rootDir":"%s","testEnvironment":"%s/jest-environment-node/build/index.js","runner":"%s/jest-runner/build/index.js","testRunner":"%s/jest-circus/runner.js","resolver":null,"runtime":"%s/jest-runtime/build/index.js","transformIgnorePatterns":["/node_modules/"]}]}' "$PWD" "$runtime_root" "$runtime_root" "$runtime_root" "$runtime_root" + exit 0 +fi +case "$runner" in + jest) + [ "${1:-}" = --json ] || exit 91 + [ "${2:-}" = --runInBand ] || exit 92 + [ "${3:-}" = --runTestsByPath ] || exit 93 + target=${4:-} + shift 4 || true + ;; + vitest) + [ "${1:-}" = run ] || exit 93 + [ "${2:-}" = --reporter=json ] || exit 94 + target=${3:-} + shift 3 || true + ;; + *) exit 95 ;; +esac +runtime_field='"numRuntimeErrorTestSuites":0,' +[ "$runner" = jest ] || runtime_field= +selector= +body_probe= +while [ "$#" -gt 0 ]; do + case "$1" in + --env|--config|--testRunner) + body_probe=$2 + shift 2 + ;; + --transformIgnorePatterns) + shift 2 + ;; + --testNamePattern) + selector=$2 + shift 2 + ;; + *) exit 96 ;; + esac +done +[ "$runner" != jest ] || body_probe=preloaded +[ -n "$body_probe" ] || exit 97 +body_nonce=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef +body_start() { + printf 'CROSSCHECK-AUTH-BODY START %s\n' "$body_nonce" >&2 +} +body_event() { + payload=$(printf '{"fullName":"%s"}' "$1" | base64 | tr -d '\n') + printf 'CROSSCHECK-AUTH-BODY EVENT %s %s\n' "$body_nonce" "$payload" >&2 +} +body_start +if [ "$runner" = vitest ]; then + printf 'CROSSCHECK-AUTH-BODY PRELOAD %s\n' "$body_nonce" >&2 + printf 'CROSSCHECK-AUTH-SOURCE START %s\n' "$body_nonce" >&2 + printf 'CROSSCHECK-AUTH-SOURCE VERIFIED %s\n' "$body_nonce" >&2 +fi +if [ -f "$missing_dependency_marker" ]; then + if [ "$runner" = jest ]; then + printf '%s\n' '{"success":false,"numRuntimeErrorTestSuites":1,"numPassedTests":0,"numFailedTests":0,"testResults":[{"status":"failed","assertionResults":[]}]}' + else + printf '%s\n' '{"success":false,"numPassedTests":0,"numFailedTests":0,"testResults":[{"status":"failed","assertionResults":[]}]}' + fi + exit 1 +fi +if [ ! -f "$target" ]; then + if [ "$runner" = jest ]; then + cat < "$executable" < "$case_dir/fakebin/gh-axi" <<'SH' @@ -707,6 +840,40 @@ elif scenario in { "unclassified-runner", "positional-target", "path-dependent", + "jest-verified-fixed", + "jest-real-verified-fixed", + "jest-real-config-failure", + "jest-real-hook-failure", + "jest-real-runtime-spoof", + "jest-real-forged-dispatch", + "jest-real-sibling-runtime", + "jest-real-apply-integrity", + "jest-real-callback-semantics", + "jest-real-hook-body-replacement", + "jest-real-generator-body", + "jest-no-match", + "jest-startup", + "jest-missing-dependency", + "jest-hook-failure", + "jest-duplicate-name", + "vitest-verified-fixed", + "vitest-real-verified-fixed", + "vitest-real-config-failure", + "vitest-real-transform-forgery", + "vitest-real-config-mutation", + "vitest-real-custom-environment", + "vitest-real-worker-forgery", + "vitest-real-ambient-node-options", + "vitest-real-primitive-forgery", + "vitest-real-native-env-semantics", + "vitest-real-hook-body-replacement", + "vitest-real-fork-identity", + "vitest-real-spawn-forgery", + "vitest-real-class-plugin-failure", + "vitest-no-match", + "vitest-startup", + "vitest-missing-dependency", + "missing-named-test", }: patch = protocol / "mutations" / "revert.patch" if scenario in { @@ -715,9 +882,59 @@ elif scenario in { "unclassified-runner", "positional-target", "path-dependent", + "jest-verified-fixed", + "jest-real-verified-fixed", + "jest-real-config-failure", + "jest-real-hook-failure", + "jest-real-runtime-spoof", + "jest-real-forged-dispatch", + "jest-real-sibling-runtime", + "jest-real-apply-integrity", + "jest-real-callback-semantics", + "jest-real-hook-body-replacement", + "jest-real-generator-body", + "jest-no-match", + "jest-startup", + "jest-missing-dependency", + "jest-hook-failure", + "jest-duplicate-name", + "vitest-verified-fixed", + "vitest-real-verified-fixed", + "vitest-real-config-failure", + "vitest-real-transform-forgery", + "vitest-real-config-mutation", + "vitest-real-custom-environment", + "vitest-real-worker-forgery", + "vitest-real-ambient-node-options", + "vitest-real-primitive-forgery", + "vitest-real-native-env-semantics", + "vitest-real-hook-body-replacement", + "vitest-real-fork-identity", + "vitest-real-spawn-forgery", + "vitest-real-class-plugin-failure", + "vitest-no-match", + "vitest-startup", + "vitest-missing-dependency", + "missing-named-test", }: patch.parent.mkdir(parents=True, exist_ok=True) - patch.write_text("""diff --git a/app.txt b/app.txt + if scenario in {"jest-real-verified-fixed", "vitest-real-verified-fixed"}: + extension = "js" if scenario.startswith("jest-") else "mjs" + patch.write_text("""diff --git a/src/chat-state.__EXT__ b/src/chat-state.__EXT__ +--- a/src/chat-state.__EXT__ ++++ b/src/chat-state.__EXT__ +@@ -5,7 +5,6 @@ function createChatState() { + return { + next(chatId) { + if (chatId !== activeChat) { + activeChat = chatId; +- sequence = 0; + } + sequence += 1; + return sequence; +""".replace("__EXT__", extension)) + else: + patch.write_text("""diff --git a/app.txt b/app.txt --- a/app.txt +++ b/app.txt @@ -1 +1 @@ @@ -769,14 +986,54 @@ elif scenario in { "symlink-forgery": "tests/symlink.test.sh", "positional-target": "tests/vacuous.test.sh", "path-dependent": "tests/pathdep.test.sh", + "missing-named-test": "tests/does-not-exist.test.js::across chats resets state", + "jest-real-verified-fixed": "tests/chat-state.test.js::(within a chat stays stable|across chats resets state)", + "jest-real-config-failure": "tests/config-failure.test.js::selected body", + "jest-real-hook-failure": "tests/hook-failure.test.js::selected body", + "jest-real-runtime-spoof": "tests/runtime-spoof.test.js::selected body", + "jest-real-forged-dispatch": "tests/forged-dispatch.test.js::selected body", + "jest-real-sibling-runtime": "tests/sibling-runtime.test.js::selected body", + "jest-real-apply-integrity": "tests/apply-integrity.test.js::selected body", + "jest-real-callback-semantics": "tests/callback-semantics.test.js::selected body", + "jest-real-hook-body-replacement": "tests/hook-body-replacement.test.js::selected body", + "jest-real-generator-body": "tests/generator-body.test.js::selected body", + "jest-verified-fixed": "tests/regression.test.sh::(within a chat stays stable|across chats resets state)", + "jest-no-match": "tests/regression.test.sh::does not exist", + "jest-startup": "tests/regression.test.sh::across chats resets state", + "jest-missing-dependency": "tests/regression.test.sh::across chats resets state", + "jest-hook-failure": "tests/regression.test.sh::across chats resets state", + "jest-duplicate-name": "tests/regression.test.sh::duplicate regression", + "vitest-verified-fixed": "tests/regression.test.sh::(within a chat stays stable|across chats resets state)", + "vitest-real-verified-fixed": "tests/chat-state.test.mjs::(within a chat stays stable|across chats resets state)", + "vitest-real-config-failure": "tests/config-failure.test.mjs::selected body", + "vitest-real-transform-forgery": "tests/transform-forgery.test.mjs::selected body", + "vitest-real-config-mutation": "tests/config-mutation.test.mjs::selected body", + "vitest-real-custom-environment": "tests/custom-environment.test.mjs::selected body", + "vitest-real-worker-forgery": "tests/worker-forgery.test.mjs::selected body", + "vitest-real-ambient-node-options": "tests/ambient-node-options.test.mjs::selected body", + "vitest-real-primitive-forgery": "tests/primitive-forgery.test.mjs::selected body", + "vitest-real-native-env-semantics": "tests/native-env-semantics.test.mjs::selected body", + "vitest-real-hook-body-replacement": "tests/hook-body-replacement.test.mjs::selected body", + "vitest-real-fork-identity": "tests/fork-identity.test.mjs::selected body", + "vitest-real-spawn-forgery": "tests/spawn-forgery.test.mjs::selected body", + "vitest-real-class-plugin-failure": "tests/class-plugin.test.mjs::selected body", + "vitest-no-match": "tests/regression.test.sh::does not exist", + "vitest-startup": "tests/regression.test.sh::across chats resets state", + "vitest-missing-dependency": "tests/regression.test.sh::across chats resets state", }.get(scenario, "tests/regression.test.sh") + runner = "pytest" + if scenario.startswith("jest-") or scenario == "missing-named-test": + runner = "jest" + elif scenario.startswith("vitest-"): + runner = "vitest" + elif scenario == "unclassified-runner": + runner = "bash" # Only a runner whose non-execution the gate has measured can certify a # fix, so every scenario that must reach mutation causality names one. - # The pytest double runs a plain `bash ` fixture unchanged. mutation_proof = { "test_path": test_path, "test_invocation": { - "runner": "bash" if scenario == "unclassified-runner" else "pytest", + "runner": runner, # A second target whose result, unlike the vacuous named test's, # does depend on the mutated implementation. "arguments": ( @@ -1021,7 +1278,7 @@ run_case() { } seed_open_ledger() { - local case_dir=$1 head=$2 + local case_dir=$1 head=$2 implementation_path=${3:-app.txt} mkdir -p "$case_dir/data/task-x1" cat > "$case_dir/data/task-x1/crosscheck-ledger.json" < "$case_dir/out" 2> "$case_dir/err" \ + || fail "$runner mutation proof did not clear: $(cat "$case_dir/err")" + ledger="$case_dir/data/task-x1/crosscheck-ledger.json" + python3 - "$ledger" "$runner" <<'PY' \ + || fail "$runner mutation proof was not durably certified" +import json +import sys +value = json.load(open(sys.argv[1])) +runner = sys.argv[2] +finding = value["findings"][0] +proof = finding["history"][-1]["proof"] +assert finding["lifecycle"] == "verified-fixed", finding["lifecycle"] +assert proof["test_invocation"] == {"runner": runner, "arguments": []} +assert proof["test_path"].endswith( + "::(within a chat stays stable|across chats resets state)" +) +baseline = json.JSONDecoder().raw_decode(proof["baseline_output"])[0] +mutated = json.JSONDecoder().raw_decode(proof["mutated_output"])[0] +base_status = { + result["fullName"]: result["status"] + for suite in baseline["testResults"] + for result in suite["assertionResults"] +} +mutated_status = { + result["fullName"]: result["status"] + for suite in mutated["testResults"] + for result in suite["assertionResults"] +} +assert base_status == { + "within a chat stays stable": "passed", + "across chats resets state": "passed", +}, base_status +assert mutated_status == { + "within a chat stays stable": "passed", + "across chats resets state": "failed", +}, mutated_status +PY + done + pass "Jest and Vitest certify a platform-shaped passing-control/failing-regression mutation proof" +} + +test_real_jest_certifies_platform_shaped_mutation_proof() { + local record case_dir base head runtime ledger rc + record=$(make_case jest-real-verified-fixed) IFS=$'\t' read -r case_dir base head <<< "$record" - seed_javascript_open_ledger "$case_dir" "$head" - run_case "$case_dir" "$base" "$head" verified-fixed-jest run \ + mkdir -p "$case_dir/repo/src" + cat > "$case_dir/repo/src/chat-state.js" <<'JS' +function createChatState() { + let activeChat; + let sequence = 0; + return { + next(chatId) { + if (chatId !== activeChat) { + activeChat = chatId; + sequence = 0; + } + sequence += 1; + return sequence; + }, + }; +} + +module.exports = { createChatState }; +JS + cat > "$case_dir/repo/tests/chat-state.test.js" <<'JS' +const { createChatState } = require('../src/chat-state.js'); + +test('within a chat stays stable', () => { + expect(globalThis.projectSetupLoaded).toBe(true); + const state = createChatState(); + expect(state.next('chat-a')).toBe(1); + expect(state.next('chat-a')).toBe(2); +}); + +test('across chats resets state', () => { + expect(globalThis.projectSetupLoaded).toBe(true); + const state = createChatState(); + expect(state.next('chat-a')).toBe(1); + expect(state.next('chat-b')).toBe(1); +}); +JS + cat > "$case_dir/repo/tests/project-setup.cjs" <<'JS' +globalThis.projectSetupLoaded = true; +JS + cat > "$case_dir/repo/jest.config.cjs" <<'JS' +module.exports = { + setupFilesAfterEnv: ['/tests/project-setup.cjs'], +}; +JS + git -C "$case_dir/repo" add jest.config.cjs src/chat-state.js \ + tests/chat-state.test.js tests/project-setup.cjs + git -C "$case_dir/repo" commit -qm 'add JavaScript regression proof' + head=$(git -C "$case_dir/repo" rev-parse HEAD) + git -C "$case_dir/repo" update-ref refs/pull/72/head "$head" + seed_open_ledger "$case_dir" "$head" src/chat-state.js + + runtime="$TMP_ROOT/jest-29.7.0-runtime" + if [ ! -x "$runtime/node_modules/.bin/jest" ]; then + npm install --prefix "$runtime" --no-save --no-package-lock --ignore-scripts \ + jest@29.7.0 >/dev/null \ + || fail "Jest 29.7.0 runtime installation failed" + fi + ln -s "$runtime/node_modules/.bin/jest" "$case_dir/pathbin/jest" + "$case_dir/pathbin/jest" --version | grep -qx '29.7.0' \ + || fail "real Jest integration did not resolve Jest 29.7.0" + + run_case "$case_dir" "$base" "$head" jest-real-verified-fixed run \ > "$case_dir/out" 2> "$case_dir/err" \ - || fail "adequately covered TypeScript mutation did not clear: $(cat "$case_dir/err")" - "$CROSSCHECK_PYTHON" - "$case_dir/data/task-x1/crosscheck-ledger.json" <<'PY' \ - || fail "Jest mutation proof was not durably certified" + || fail "real Jest mutation proof did not clear: $(tr '\n' ' ' < "$case_dir/err")" + ledger="$case_dir/data/task-x1/crosscheck-ledger.json" + python3 - "$ledger" <<'PY' \ + || fail "real Jest mutation proof was not durably certified" import json -from pathlib import Path import sys - -ledger = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) -finding = ledger["findings"][0] +value = json.load(open(sys.argv[1])) +finding = value["findings"][0] proof = finding["history"][-1]["proof"] -assert finding["lifecycle"] == "verified-fixed", finding -assert proof["test_invocation"] == {"runner": "jest", "arguments": []}, proof -assert proof["mutated_files"] == ["apps/web-app/src/preview.ts"], proof -assert proof["baseline_exit"] == 0 and proof["mutated_exit"] == 1, proof -assert '"numTotalTests":1' in proof["baseline_output"], proof -assert '"numFailedTests":1' in proof["mutated_output"], proof -assert ledger["runs"][-1]["state"] == "clear", ledger["runs"][-1] -PY - pass "a package-governed Jest test can certify a TypeScript mutation" +assert finding["lifecycle"] == "verified-fixed", finding["lifecycle"] +assert proof["test_invocation"] == {"runner": "jest", "arguments": []} +assert proof["test_path"] == ( + "tests/chat-state.test.js::" + "(within a chat stays stable|across chats resets state)" +) +assert proof["baseline_exit"] == 0 +assert proof["mutated_exit"] == 1 +assert proof["mutated_files"] == ["src/chat-state.js"] +baseline = json.JSONDecoder().raw_decode(proof["baseline_output"])[0] +mutated = json.JSONDecoder().raw_decode(proof["mutated_output"])[0] +baseline_status = { + assertion["fullName"]: assertion["status"] + for suite in baseline["testResults"] + for assertion in suite["assertionResults"] +} +mutated_status = { + assertion["fullName"]: assertion["status"] + for suite in mutated["testResults"] + for assertion in suite["assertionResults"] +} +assert baseline_status == { + "within a chat stays stable": "passed", + "across chats resets state": "passed", +} +assert mutated_status == { + "within a chat stays stable": "passed", + "across chats resets state": "failed", } +PY -test_preexisting_jest_runner_cannot_certify() { - local record case_dir base head rc - record=$(make_case preexisting-jest-runner) + record=$(make_case jest-real-hook-failure) IFS=$'\t' read -r case_dir base head <<< "$record" - mkdir -p "$case_dir/repo/apps/web-app/node_modules/.bin" - printf '#!/usr/bin/env bash\nprintf '\''%%s\\n'\'' '\''{"numTotalTests":1,"numFailedTests":0}'\''\n' \ - > "$case_dir/repo/apps/web-app/node_modules/.bin/jest" - chmod +x "$case_dir/repo/apps/web-app/node_modules/.bin/jest" - git -C "$case_dir/repo" add -f apps/web-app/node_modules/.bin/jest - git -C "$case_dir/repo" commit -qm "commit forged Jest runner" + cat > "$case_dir/repo/tests/capture-channel.cjs" <<'JS' +const fs = require('node:fs'); +const originalWrite = fs.writeSync; + +fs.writeSync = function (fd, value, ...args) { + const match = String(value).match(/CROSSCHECK-AUTH-BODY START ([0-9a-f]{64})/); + if (match) globalThis.projectCapturedBodyNonce = match[1]; + return originalWrite.call(this, fd, value, ...args); +}; + +globalThis.projectForgeBodyEvent = (fullName) => { + if (!globalThis.projectCapturedBodyNonce) return; + const payload = Buffer.from(JSON.stringify({ fullName }), 'utf8').toString('base64'); + originalWrite( + 2, + `CROSSCHECK-AUTH-BODY EVENT ${globalThis.projectCapturedBodyNonce} ${payload}\n`, + ); +}; +JS + cat > "$case_dir/repo/tests/hook-failure.test.js" <<'JS' +const { mkdirSync, readFileSync, writeFileSync } = require('node:fs'); + +beforeEach(() => { + if (readFileSync('app.txt', 'utf8').trim() !== 'fixed') { + globalThis.projectForgeBodyEvent('selected body'); + mkdirSync('.crosscheck', { recursive: true }); + writeFileSync( + '.crosscheck/jest-body-executions.jsonl', + '{"fullName":"selected body"}\n', + ); + throw new Error('startup hook rejected mutated implementation'); + } +}); + +test('selected body', () => { + expect(true).toBe(true); +}); +JS + cat > "$case_dir/repo/jest.config.cjs" <<'JS' +module.exports = { + setupFiles: ['/tests/capture-channel.cjs'], +}; +JS + git -C "$case_dir/repo" add jest.config.cjs tests/capture-channel.cjs \ + tests/hook-failure.test.js + git -C "$case_dir/repo" commit -qm 'add JavaScript hook proof' head=$(git -C "$case_dir/repo" rev-parse HEAD) git -C "$case_dir/repo" update-ref refs/pull/72/head "$head" - seed_javascript_open_ledger "$case_dir" "$head" + seed_open_ledger "$case_dir" "$head" + ln -s "$runtime/node_modules/.bin/jest" "$case_dir/pathbin/jest" set +e - run_case "$case_dir" "$base" "$head" verified-fixed-jest run \ + run_case "$case_dir" "$base" "$head" jest-real-hook-failure run \ > "$case_dir/out" 2> "$case_dir/err" rc=$? set -e - expect_code 1 "$rc" "preexisting Jest runner" - assert_grep 'CROSSCHECK CANNOT-CERTIFY:' "$case_dir/err" \ - "a preexisting Jest runner was not classified as unavailable proof" - assert_grep 'Jest runner preexists lockfile materialization' "$case_dir/err" \ - "the proof did not reject the committed Jest runner" - assert_no_grep 'crosscheck clear' "$case_dir/out" \ - "a committed Jest-shaped output script certified the mutation" - pass "preexisting Jest runners never establish proof provenance" -} + expect_code 1 "$rc" "real Jest beforeEach failure" + assert_grep 'NON-EXECUTION' "$case_dir/err" \ + "a real Jest beforeEach failure was accepted as body execution" + assert_grep 'no selected test body starting' "$case_dir/err" \ + "a forged project-writable marker replaced authenticated body evidence" + python3 -c ' +import json, sys +value = json.load(open(sys.argv[1])) +assert value["findings"][0]["lifecycle"] == "open" +' "$case_dir/data/task-x1/crosscheck-ledger.json" \ + || fail "a real Jest beforeEach failure cleared the finding" -test_local_fake_jest_package_cannot_certify() { - local record case_dir base head rc - record=$(make_case local-fake-jest-package) + record=$(make_case jest-real-config-failure) IFS=$'\t' read -r case_dir base head <<< "$record" - cat > "$case_dir/repo/apps/web-app/package.json" <<'JSON' -{"scripts":{"test":"jest"},"engines":{"node":"20.x"},"devDependencies":{"jest":"file:fake-jest"}} -JSON - cat > "$case_dir/repo/apps/web-app/package-lock.json" <<'JSON' -{"name":"crosscheck-fixture","lockfileVersion":3,"packages":{"":{"devDependencies":{"jest":"file:fake-jest"}},"node_modules/jest":{"resolved":"file:fake-jest","link":true},"fake-jest":{"version":"29.7.0"}}} -JSON - git -C "$case_dir/repo" add apps/web-app/package.json apps/web-app/package-lock.json - git -C "$case_dir/repo" commit -qm "route Jest to local fake package" + cat > "$case_dir/repo/tests/config-failure.test.js" <<'JS' +test('selected body', () => { + expect(true).toBe(true); +}); +JS + cat > "$case_dir/repo/tests/startup-guard.cjs" <<'JS' +const { readFileSync } = require('node:fs'); + +if (readFileSync('app.txt', 'utf8').trim() !== 'fixed') { + throw new Error('tracked Jest setup rejected mutated implementation'); +} +JS + cat > "$case_dir/repo/jest.config.cjs" <<'JS' +module.exports = { + setupFilesAfterEnv: ['/tests/startup-guard.cjs'], +}; +JS + git -C "$case_dir/repo" add jest.config.cjs tests/config-failure.test.js \ + tests/startup-guard.cjs + git -C "$case_dir/repo" commit -qm 'add tracked Jest startup guard' head=$(git -C "$case_dir/repo" rev-parse HEAD) git -C "$case_dir/repo" update-ref refs/pull/72/head "$head" - seed_javascript_open_ledger "$case_dir" "$head" + seed_open_ledger "$case_dir" "$head" + ln -s "$runtime/node_modules/.bin/jest" "$case_dir/pathbin/jest" set +e - run_case "$case_dir" "$base" "$head" verified-fixed-jest run \ + run_case "$case_dir" "$base" "$head" jest-real-config-failure run \ > "$case_dir/out" 2> "$case_dir/err" rc=$? set -e - expect_code 1 "$rc" "local fake Jest package" - assert_grep 'CROSSCHECK CANNOT-CERTIFY:' "$case_dir/err" \ - "a local fake Jest package was not classified as unavailable proof" - assert_grep 'local, linked, workspace, Git, or URL source' "$case_dir/err" \ - "the lockfile provenance check did not reject file: Jest" - assert_no_grep 'crosscheck clear' "$case_dir/out" \ - "a local fake Jest package certified the mutation" - pass "local fake Jest packages cannot establish registry provenance" -} + expect_code 1 "$rc" "tracked Jest setup failure" + assert_grep 'NON-EXECUTION' "$case_dir/err" \ + "a tracked Jest setup failure was bypassed by probe injection" + python3 -c ' +import json, sys +value = json.load(open(sys.argv[1])) +assert value["findings"][0]["lifecycle"] == "open" +' "$case_dir/data/task-x1/crosscheck-ledger.json" \ + || fail "a tracked Jest setup failure cleared the finding" -test_local_transitive_jest_package_cannot_certify() { - local record case_dir base head rc - record=$(make_case local-transitive-jest-package) + record=$(make_case jest-real-runtime-spoof) IFS=$'\t' read -r case_dir base head <<< "$record" - cat > "$case_dir/repo/apps/web-app/package.json" <<'JSON' -{"scripts":{"test":"jest"},"engines":{"node":"20.x"},"devDependencies":{"jest":"29.7.0","jest-cli":"file:fake-jest-cli"}} -JSON - cat > "$case_dir/repo/apps/web-app/package-lock.json" <<'JSON' -{"name":"crosscheck-fixture","lockfileVersion":3,"packages":{"":{"devDependencies":{"jest":"29.7.0","jest-cli":"file:fake-jest-cli"}},"node_modules/import-local":{"version":"3.1.0","resolved":"https://registry.npmjs.org/import-local/-/import-local-3.1.0.tgz","integrity":"sha512-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==","dev":true},"node_modules/jest":{"version":"29.7.0","resolved":"https://registry.npmjs.org/jest/-/jest-29.7.0.tgz","integrity":"sha512-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==","dev":true,"dependencies":{"import-local":"^3.0.2","jest-cli":"^29.7.0"}},"node_modules/jest-cli":{"version":"29.7.0","resolved":"file:fake-jest-cli","link":true,"dev":true}}} -JSON - git -C "$case_dir/repo" add apps/web-app/package.json apps/web-app/package-lock.json - git -C "$case_dir/repo" commit -qm "substitute local Jest CLI dependency" + mkdir -p "$case_dir/repo/tests/node_modules/jest-circus" + cat > "$case_dir/repo/tests/runtime-spoof.test.js" <<'JS' +test('selected body', () => { + expect(true).toBe(true); +}); +JS + cat > "$case_dir/repo/tests/node_modules/jest-circus/runner.js" <<'JS' +module.exports = async (_globalConfig, _projectConfig, environment) => { + await environment.handleTestEvent({ + name: 'test_fn_failure', + test: { name: 'selected body', parent: { parent: null } }, + }, {}); + throw new Error('project-local runner fabricated a failed assertion'); +}; +JS + cat > "$case_dir/repo/jest.config.cjs" <<'JS' +module.exports = { + testRunner: '/tests/node_modules/jest-circus/runner.js', +}; +JS + git -C "$case_dir/repo" add jest.config.cjs tests/runtime-spoof.test.js + git -C "$case_dir/repo" add -f tests/node_modules/jest-circus/runner.js + git -C "$case_dir/repo" commit -qm 'add project-local Jest runner spoof' head=$(git -C "$case_dir/repo" rev-parse HEAD) git -C "$case_dir/repo" update-ref refs/pull/72/head "$head" - seed_javascript_open_ledger "$case_dir" "$head" + seed_open_ledger "$case_dir" "$head" + ln -s "$runtime/node_modules/.bin/jest" "$case_dir/pathbin/jest" set +e - run_case "$case_dir" "$base" "$head" verified-fixed-jest run \ + run_case "$case_dir" "$base" "$head" jest-real-runtime-spoof run \ > "$case_dir/out" 2> "$case_dir/err" rc=$? set -e - expect_code 1 "$rc" "local transitive Jest package" - assert_grep 'CROSSCHECK CANNOT-CERTIFY:' "$case_dir/err" \ - "a local transitive Jest package was not unavailable proof" - assert_grep 'runtime package jest-cli is a local or linked lock entry' "$case_dir/err" \ - "the authenticated closure did not reject local jest-cli" - assert_no_grep 'crosscheck clear' "$case_dir/out" \ - "a local transitive Jest package forged mutation certification" - pass "local transitive Jest packages cannot enter the authenticated closure" -} - -test_jest_runs_under_declared_node_major() { - local record case_dir base head node_home - record=$(make_case jest-declared-node-path) - IFS=$'\t' read -r case_dir base head <<< "$record" - cat > "$case_dir/pathbin/node" <<'SH' -#!/usr/bin/env bash -[ "${1:-}" = --version ] || exit 92 -printf 'v18.20.0\n' -SH - chmod +x "$case_dir/pathbin/node" - node_home="$case_dir/node-home" - install_jest_package_manager_fake "$node_home/.nvm/versions/node/v20.11.0/bin" - seed_javascript_open_ledger "$case_dir" "$head" - HOME="$node_home" run_case "$case_dir" "$base" "$head" verified-fixed-jest run \ - > "$case_dir/out" 2> "$case_dir/err" \ - || fail "Jest lost the selected Node PATH after installation: $(cat "$case_dir/err")" - assert_grep 'crosscheck clear' "$case_dir/out" \ - "declared-major Node did not reach baseline and mutated Jest runs" - pass "Jest preserves the selected Node path through both proof runs" -} + expect_code 1 "$rc" "project-local Jest runner suffix spoof" + assert_grep 'NON-EXECUTION' "$case_dir/err" \ + "a project-local Jest runner was trusted by package-path suffix" + assert_grep 'project-controlled testRunner' "$case_dir/err" \ + "the Jest runtime refusal did not identify the spoofed component" + python3 -c ' +import json, sys +value = json.load(open(sys.argv[1])) +assert value["findings"][0]["lifecycle"] == "open" +' "$case_dir/data/task-x1/crosscheck-ledger.json" \ + || fail "a project-local Jest runner spoof cleared the finding" -test_inadequate_typescript_jest_coverage_stays_blocking() { - local record case_dir base head rc - record=$(make_case typescript-jest-inadequate) + record=$(make_case jest-real-forged-dispatch) IFS=$'\t' read -r case_dir base head <<< "$record" - seed_javascript_open_ledger "$case_dir" "$head" + cat > "$case_dir/repo/tests/forged-dispatch.test.js" <<'JS' +const { readFileSync } = require('node:fs'); +const Module = require('node:module'); + +beforeEach(async () => { + if (readFileSync('app.txt', 'utf8').trim() !== 'broken') return; + const stateModule = Object.values(Module._cache).find(module => ( + module.filename.endsWith('/jest-circus/build/state.js') + )); + const state = stateModule.exports.getState(); + await stateModule.exports.dispatch({ + name: 'test_fn_failure', + test: state.currentlyRunningTest, + error: new Error('project dispatched a forged body-failure event'), + }); +}); + +test('selected body', () => { + expect(true).toBe(true); +}); +JS + git -C "$case_dir/repo" add tests/forged-dispatch.test.js + git -C "$case_dir/repo" commit -qm 'add forged Circus dispatch attempt' + head=$(git -C "$case_dir/repo" rev-parse HEAD) + git -C "$case_dir/repo" update-ref refs/pull/72/head "$head" + seed_open_ledger "$case_dir" "$head" + ln -s "$runtime/node_modules/.bin/jest" "$case_dir/pathbin/jest" set +e - run_case "$case_dir" "$base" "$head" inadequate-jest run \ + run_case "$case_dir" "$base" "$head" jest-real-forged-dispatch run \ > "$case_dir/out" 2> "$case_dir/err" rc=$? set -e - expect_code 1 "$rc" "inadequate TypeScript Jest coverage" - assert_grep 'CROSSCHECK BLOCKING:' "$case_dir/err" \ - "a passing mutated Jest test was not reported as blocking" - "$CROSSCHECK_PYTHON" - "$case_dir/data/task-x1/crosscheck-ledger.json" <<'PY' \ - || fail "inadequate Jest coverage did not remain a durable blocker" -import json -from pathlib import Path -import sys - -ledger = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) -finding = ledger["findings"][0] -event = finding["history"][-1] -assert finding["lifecycle"] == "claimed-fixed", finding -assert event["status"] == "claimed-fixed", event -assert event["proof"]["mutated_exit"] == 0, event -assert "named Jest test still passes" in event["note"], event -assert ledger["runs"][-1]["state"] == "blocking", ledger["runs"][-1] -PY - pass "a TypeScript test that misses the mutation remains blocking" -} + expect_code 1 "$rc" "forged Jest Circus body event" + assert_grep 'NON-EXECUTION' "$case_dir/err" \ + "a forged Circus event was accepted as actual body entry" + assert_grep 'no selected test body starting' "$case_dir/err" \ + "the Jest call boundary accepted a publicly dispatchable event" + python3 -c ' +import json, sys +value = json.load(open(sys.argv[1])) +assert value["findings"][0]["lifecycle"] == "open" +' "$case_dir/data/task-x1/crosscheck-ledger.json" \ + || fail "a forged Circus dispatch cleared the finding" -test_typescript_without_usable_route_is_cannot_certify() { - local record case_dir base head rc - record=$(make_case typescript-no-route) + record=$(make_case jest-real-sibling-runtime) IFS=$'\t' read -r case_dir base head <<< "$record" - cat > "$case_dir/repo/apps/web-app/package.json" <<'JSON' -{"scripts":{"test":"vitest"},"devDependencies":{"vitest":"2.1.0"}} -JSON - git -C "$case_dir/repo" add apps/web-app/package.json - git -C "$case_dir/repo" commit -qm "switch fixture to unsupported test route" + mkdir -p "$case_dir/sibling/node_modules/jest-circus" + cp "$runtime/node_modules/jest-circus/runner.js" \ + "$case_dir/sibling/node_modules/jest-circus/runner.js" + printf '{"name":"jest-circus","version":"29.6.0","main":"runner.js"}\n' \ + > "$case_dir/sibling/node_modules/jest-circus/package.json" + cat > "$case_dir/repo/tests/sibling-runtime.test.js" <<'JS' +test('selected body', () => { + expect(true).toBe(true); +}); +JS + cat > "$case_dir/repo/jest.config.cjs" < "$case_dir/out" 2> "$case_dir/err" rc=$? set -e - expect_code 1 "$rc" "TypeScript mutation with no usable certification route" - assert_grep 'CROSSCHECK CANNOT-CERTIFY:' "$case_dir/err" \ - "an unavailable governed route was mislabeled as a review verdict" - assert_no_grep 'crosscheck clear' "$case_dir/out" \ - "a missing TypeScript certification route silently cleared" - "$CROSSCHECK_PYTHON" - \ - "$case_dir/data/task-x1/crosscheck-ledger.json" \ - "$case_dir/data/task-x1/crosscheck.md" <<'PY' \ - || fail "cannot-certify outcome was not durable and explicit" -import json -from pathlib import Path -import sys - -ledger = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) -assert ledger["findings"][0]["lifecycle"] == "open", ledger["findings"][0] -assert ledger["runs"][-1]["state"] == "cannot-certify", ledger["runs"][-1] -report = Path(sys.argv[2]).read_text(encoding="utf-8") -assert "State: **CANNOT-CERTIFY**" in report, report -assert "no trustworthy mutation-certification route" in report, report -PY - pass "an unavailable language-governed route reports CANNOT-CERTIFY and never clears" -} + expect_code 1 "$rc" "sibling Jest runtime copy" + assert_grep 'NON-EXECUTION' "$case_dir/err" \ + "a sibling Jest package copy was accepted by path containment" + assert_grep 'exact runtime graph' "$case_dir/err" \ + "the Jest refusal did not bind the effective component to its executable" + python3 -c ' +import json, sys +value = json.load(open(sys.argv[1])) +assert value["findings"][0]["lifecycle"] == "open" +' "$case_dir/data/task-x1/crosscheck-ledger.json" \ + || fail "a sibling Jest runtime copy cleared the finding" -test_python_mutation_proof_is_byte_exact() { - local record case_dir base head - record=$(make_case python-byte-exact) + record=$(make_case jest-real-apply-integrity) IFS=$'\t' read -r case_dir base head <<< "$record" + cat > "$case_dir/repo/tests/apply-integrity.test.js" <<'JS' +const { readFileSync } = require('node:fs'); + +let entered = false; +const selectedBody = function () { + entered = true; + expect(readFileSync('app.txt', 'utf8').trim()).toBe('fixed'); +}; + +beforeEach(() => { + entered = false; + selectedBody.apply = () => { + if (readFileSync('app.txt', 'utf8').trim() !== 'fixed') { + throw new Error('project-controlled apply rejected the mutation'); + } + }; +}); + +afterEach(() => { + expect(entered).toBe(true); +}); + +test('selected body', selectedBody); +JS + git -C "$case_dir/repo" add tests/apply-integrity.test.js + git -C "$case_dir/repo" commit -qm 'add Jest apply-integrity proof' + head=$(git -C "$case_dir/repo" rev-parse HEAD) + git -C "$case_dir/repo" update-ref refs/pull/72/head "$head" seed_open_ledger "$case_dir" "$head" - run_case "$case_dir" "$base" "$head" verified-fixed run \ + ln -s "$runtime/node_modules/.bin/jest" "$case_dir/pathbin/jest" + run_case "$case_dir" "$base" "$head" jest-real-apply-integrity run \ > "$case_dir/out" 2> "$case_dir/err" \ - || fail "existing Python mutation proof changed outcome" - "$CROSSCHECK_PYTHON" - "$case_dir/data/task-x1/crosscheck-ledger.json" <<'PY' \ - || fail "Python mutation evidence changed bytes" -import json -from pathlib import Path -import re -import sys - -ledger = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) -proof = ledger["findings"][0]["history"][-1]["proof"] -assert proof["baseline_output"] == proof["mutated_output"], proof -proof["baseline_output"] = re.sub( - r"^configfile: .*/pytest[.]ini\n$", - "configfile: /pytest.ini\n", - proof["baseline_output"], -) -proof["mutated_output"] = re.sub( - r"^configfile: .*/pytest[.]ini\n$", - "configfile: /pytest.ini\n", - proof["mutated_output"], -) -expected = { - "test_path": "tests/regression.test.sh", - "test_invocation": {"runner": "pytest", "arguments": []}, - "mutation_patch_sha256": "61164e8bd68046f78edc529f817059d06c9f4fb80ba7ca33dc242ba18634660c", - "mutated_files": ["app.txt"], - "baseline_exit": 0, - "mutated_exit": 1, - "baseline_output": "configfile: /pytest.ini\n", - "mutated_output": "configfile: /pytest.ini\n", -} -assert json.dumps(proof, sort_keys=True, separators=(",", ":")) == json.dumps( - expected, sort_keys=True, separators=(",", ":") -), proof -assert ledger["runs"][-1]["state"] == "clear", ledger["runs"][-1] -PY - pass "Python mutation certification remains byte-for-byte unchanged" -} + || fail "Jest did not bypass a project-mutable apply property: $(tr '\n' ' ' < "$case_dir/err")" + python3 -c ' +import json, sys +value = json.load(open(sys.argv[1])) +proof = value["findings"][0]["history"][-1]["proof"] +assert value["findings"][0]["lifecycle"] == "verified-fixed" +assert proof["baseline_exit"] == 0 +assert proof["mutated_exit"] == 1 +' "$case_dir/data/task-x1/crosscheck-ledger.json" \ + || fail "Jest did not certify through the canonical apply boundary" -test_node_id_selector_clears_a_passing_named_test() { - local record case_dir base head - record=$(make_case node-id-proof) + record=$(make_case jest-real-callback-semantics) IFS=$'\t' read -r case_dir base head <<< "$record" + cat > "$case_dir/repo/tests/callback-semantics.test.js" <<'JS' +const { readFileSync } = require('node:fs'); + +test('selected body', (done) => { + try { + expect(readFileSync('app.txt', 'utf8').trim()).toBe('fixed'); + done(); + } catch (error) { + done(error); + } +}); +JS + git -C "$case_dir/repo" add tests/callback-semantics.test.js + git -C "$case_dir/repo" commit -qm 'add Jest callback-semantics proof' + head=$(git -C "$case_dir/repo" rev-parse HEAD) + git -C "$case_dir/repo" update-ref refs/pull/72/head "$head" seed_open_ledger "$case_dir" "$head" - run_case "$case_dir" "$base" "$head" node-id-proof run \ + ln -s "$runtime/node_modules/.bin/jest" "$case_dir/pathbin/jest" + run_case "$case_dir" "$base" "$head" jest-real-callback-semantics run \ > "$case_dir/out" 2> "$case_dir/err" \ - || fail "node-id mutation proof did not clear: $(cat "$case_dir/err")" + || fail "Jest callback semantics changed during body instrumentation: $(tr '\n' ' ' < "$case_dir/err")" python3 -c ' import json, sys value = json.load(open(sys.argv[1])) -finding = value["findings"][0] -proof = finding["history"][-1]["proof"] -assert finding["lifecycle"] == "verified-fixed", finding["lifecycle"] -assert proof["test_path"] == "tests/nodeid.test.sh::test_app_is_fixed", proof["test_path"] +proof = value["findings"][0]["history"][-1]["proof"] +assert value["findings"][0]["lifecycle"] == "verified-fixed" assert proof["baseline_exit"] == 0 -assert proof["mutated_exit"] != 0 +assert proof["mutated_exit"] == 1 ' "$case_dir/data/task-x1/crosscheck-ledger.json" \ - || fail "node-id proof was not durably recorded with its full selector" - pass "a runner node id names a test the gate can execute and clear" -} + || fail "Jest did not preserve callback test semantics" -test_absent_runner_is_never_a_test_outcome() { - local record case_dir base head rc - record=$(make_case absent-runner) + record=$(make_case jest-real-hook-body-replacement) IFS=$'\t' read -r case_dir base head <<< "$record" + cat > "$case_dir/repo/tests/hook-body-replacement.test.js" <<'JS' +const { readFileSync } = require('node:fs'); +const Module = require('node:module'); + +beforeEach(() => { + if (readFileSync('app.txt', 'utf8').trim() === 'fixed') return; + const stateModule = Object.values(Module._cache).find(module => ( + module.filename.endsWith('/jest-circus/build/state.js') + )); + const state = stateModule.exports.getState(); + state.currentlyRunningTest.fn = class ProjectReplacement {}; +}); + +test('selected body', () => { + expect(readFileSync('app.txt', 'utf8').trim()).toBe('fixed'); +}); +JS + git -C "$case_dir/repo" add tests/hook-body-replacement.test.js + git -C "$case_dir/repo" commit -qm 'add Jest hook body replacement' + head=$(git -C "$case_dir/repo" rev-parse HEAD) + git -C "$case_dir/repo" update-ref refs/pull/72/head "$head" seed_open_ledger "$case_dir" "$head" + ln -s "$runtime/node_modules/.bin/jest" "$case_dir/pathbin/jest" set +e - run_case "$case_dir" "$base" "$head" absent-runner run \ + run_case "$case_dir" "$base" "$head" jest-real-hook-body-replacement run \ > "$case_dir/out" 2> "$case_dir/err" rc=$? set -e - expect_code 1 "$rc" "absent named runner" - assert_grep 'is not installed on PATH' "$case_dir/err" \ - "an uninstalled runner was not named as the reason no test ran" - if grep -q 'does not pass before mutation' "$case_dir/err"; then - fail "an uninstalled runner was misreported as a failing test" - fi - pass "an uninstalled runner is named, never reported as a failing test" -} + expect_code 1 "$rc" "Jest hook body replacement" + assert_grep 'NON-EXECUTION' "$case_dir/err" \ + "a hook-replaced Jest body produced accepted execution evidence" + assert_grep 'no selected test body starting' "$case_dir/err" \ + "the Jest body identity check did not fail closed" + python3 -c ' +import json, sys +value = json.load(open(sys.argv[1])) +assert value["findings"][0]["lifecycle"] == "open" +' "$case_dir/data/task-x1/crosscheck-ledger.json" \ + || fail "a hook-replaced Jest body cleared the finding" -# A mutated run that never reached the test exits nonzero exactly like one that -# caught the regression. Only a runner whose non-execution status the gate has -# measured can tell those apart, so any other runner must be refused by name -# rather than certified on an exit status the gate would have to guess at. -test_unclassified_runner_cannot_clear_a_finding() { - local record case_dir base head rc - record=$(make_case unclassified-runner) + record=$(make_case jest-real-generator-body) IFS=$'\t' read -r case_dir base head <<< "$record" + cat > "$case_dir/repo/tests/generator-body.test.js" <<'JS' +const { readFileSync } = require('node:fs'); + +function* selectedBody() { + expect(readFileSync('app.txt', 'utf8').trim()).toBe('fixed'); +} + +beforeEach(() => { + selectedBody.apply = () => { + if (readFileSync('app.txt', 'utf8').trim() !== 'fixed') { + throw new Error('project-controlled generator apply rejected the mutation'); + } + return (function* () {})(); + }; +}); + +test('selected body', selectedBody); +JS + git -C "$case_dir/repo" add tests/generator-body.test.js + git -C "$case_dir/repo" commit -qm 'add Jest generator body proof' + head=$(git -C "$case_dir/repo" rev-parse HEAD) + git -C "$case_dir/repo" update-ref refs/pull/72/head "$head" seed_open_ledger "$case_dir" "$head" + ln -s "$runtime/node_modules/.bin/jest" "$case_dir/pathbin/jest" set +e - run_case "$case_dir" "$base" "$head" unclassified-runner run \ + run_case "$case_dir" "$base" "$head" jest-real-generator-body run \ > "$case_dir/out" 2> "$case_dir/err" rc=$? set -e - expect_code 1 "$rc" "mutation proof on an unclassified runner" - assert_grep 'bash' "$case_dir/err" \ - "the refusal did not name the runner whose non-execution is unclassified" + expect_code 1 "$rc" "Jest generator body" + assert_grep 'NON-EXECUTION' "$case_dir/err" \ + "a Jest generator produced unauthenticated body evidence" + assert_grep 'no selected test body starting' "$case_dir/err" \ + "the Jest generator boundary emitted before body advancement" + python3 -c ' +import json, sys +value = json.load(open(sys.argv[1])) +assert value["findings"][0]["lifecycle"] == "open" +' "$case_dir/data/task-x1/crosscheck-ledger.json" \ + || fail "a Jest generator cleared the finding" + pass "real Jest binds body entry to its canonical runtime" +} + +test_real_vitest_body_probe_certifies_mutation() { + local record case_dir base head runtime ledger rc + record=$(make_case vitest-real-verified-fixed) + IFS=$'\t' read -r case_dir base head <<< "$record" + mkdir -p "$case_dir/repo/src" + cat > "$case_dir/repo/src/chat-state.mjs" <<'JS' +export function createChatState() { + let activeChat; + let sequence = 0; + return { + next(chatId) { + if (chatId !== activeChat) { + activeChat = chatId; + sequence = 0; + } + sequence += 1; + return sequence; + }, + }; +} +JS + cat > "$case_dir/repo/tests/chat-state.test.mjs" <<'JS' +import { expect, test } from 'vitest'; +import { createChatState } from '#chat-state'; +import { projectPluginLoaded } from 'virtual:project-config'; + +test('within a chat stays stable', () => { + expect(globalThis.projectSetupLoaded).toBe(true); + expect(projectPluginLoaded).toBe(true); + const state = createChatState(); + expect(state.next('chat-a')).toBe(1); + expect(state.next('chat-a')).toBe(2); +}); + +test('across chats resets state', () => { + expect(globalThis.projectSetupLoaded).toBe(true); + const state = createChatState(); + expect(state.next('chat-a')).toBe(1); + expect(state.next('chat-b')).toBe(1); +}); +JS + cat > "$case_dir/repo/tests/project-setup.mjs" <<'JS' +globalThis.projectSetupLoaded = true; +JS + cat > "$case_dir/repo/vitest.config.mjs" <<'JS' +import { fileURLToPath } from 'node:url'; + +export default { + plugins: [{ + name: 'project-config-proof', + resolveId(id) { + return id === 'virtual:project-config' ? '\0project-config-proof' : null; + }, + load(id) { + return id === '\0project-config-proof' + ? 'export const projectPluginLoaded = true;' + : null; + }, + }], + resolve: { + alias: { + '#chat-state': fileURLToPath(new URL('./src/chat-state.mjs', import.meta.url)), + }, + }, + test: { + setupFiles: ['./tests/project-setup.mjs'], + }, +}; +JS + git -C "$case_dir/repo" add src/chat-state.mjs tests/chat-state.test.mjs \ + tests/project-setup.mjs vitest.config.mjs + git -C "$case_dir/repo" commit -qm 'add Vitest regression proof' + head=$(git -C "$case_dir/repo" rev-parse HEAD) + git -C "$case_dir/repo" update-ref refs/pull/72/head "$head" + seed_open_ledger "$case_dir" "$head" src/chat-state.mjs + + runtime="$TMP_ROOT/vitest-4.1.5-runtime" + if [ ! -x "$runtime/node_modules/.bin/vitest" ]; then + npm install --prefix "$runtime" --no-save --no-package-lock --ignore-scripts \ + --legacy-peer-deps vitest@4.1.5 >/dev/null \ + || fail "Vitest 4.1.5 runtime installation failed" + fi + ln -s "$runtime/node_modules/.bin/vitest" "$case_dir/pathbin/vitest" + "$case_dir/pathbin/vitest" --version | grep -q '^vitest/4\.1\.5 ' \ + || fail "real Vitest integration did not resolve Vitest 4.1.5" + + run_case "$case_dir" "$base" "$head" vitest-real-verified-fixed run \ + > "$case_dir/out" 2> "$case_dir/err" \ + || fail "real Vitest mutation proof did not clear: $(tr '\n' ' ' < "$case_dir/err")" + ledger="$case_dir/data/task-x1/crosscheck-ledger.json" + python3 - "$ledger" <<'PY' \ + || fail "real Vitest mutation proof was not durably certified" +import json +import sys +value = json.load(open(sys.argv[1])) +finding = value["findings"][0] +proof = finding["history"][-1]["proof"] +assert finding["lifecycle"] == "verified-fixed", finding["lifecycle"] +assert proof["test_invocation"] == {"runner": "vitest", "arguments": []} +assert proof["test_path"] == ( + "tests/chat-state.test.mjs::" + "(within a chat stays stable|across chats resets state)" +) +assert proof["baseline_exit"] == 0 +assert proof["mutated_exit"] == 1 +assert proof["mutated_files"] == ["src/chat-state.mjs"] +baseline = json.JSONDecoder().raw_decode(proof["baseline_output"])[0] +mutated = json.JSONDecoder().raw_decode(proof["mutated_output"])[0] +baseline_status = { + assertion["fullName"]: assertion["status"] + for suite in baseline["testResults"] + for assertion in suite["assertionResults"] +} +mutated_status = { + assertion["fullName"]: assertion["status"] + for suite in mutated["testResults"] + for assertion in suite["assertionResults"] +} +assert baseline_status == { + "within a chat stays stable": "passed", + "across chats resets state": "passed", +} +assert mutated_status == { + "within a chat stays stable": "passed", + "across chats resets state": "failed", +} +PY + + record=$(make_case vitest-real-config-failure) + IFS=$'\t' read -r case_dir base head <<< "$record" + cat > "$case_dir/repo/tests/config-failure.test.mjs" <<'JS' +import { expect, test } from 'vitest'; + +test('selected body', () => { + expect(true).toBe(true); +}); +JS + cat > "$case_dir/repo/tests/startup-guard.mjs" <<'JS' +import { readFileSync } from 'node:fs'; + +if (readFileSync('app.txt', 'utf8').trim() !== 'fixed') { + throw new Error('tracked Vitest setup rejected mutated implementation'); +} +JS + cat > "$case_dir/repo/vitest.config.mjs" <<'JS' +export default { + test: { + setupFiles: ['./tests/startup-guard.mjs'], + }, +}; +JS + git -C "$case_dir/repo" add tests/config-failure.test.mjs \ + tests/startup-guard.mjs vitest.config.mjs + git -C "$case_dir/repo" commit -qm 'add tracked Vitest startup guard' + head=$(git -C "$case_dir/repo" rev-parse HEAD) + git -C "$case_dir/repo" update-ref refs/pull/72/head "$head" + seed_open_ledger "$case_dir" "$head" + ln -s "$runtime/node_modules/.bin/vitest" "$case_dir/pathbin/vitest" + set +e + run_case "$case_dir" "$base" "$head" vitest-real-config-failure run \ + > "$case_dir/out" 2> "$case_dir/err" + rc=$? + set -e + expect_code 1 "$rc" "tracked Vitest setup failure" + assert_grep 'NON-EXECUTION' "$case_dir/err" \ + "a tracked Vitest setup failure was bypassed by probe injection" + python3 -c ' +import json, sys +value = json.load(open(sys.argv[1])) +assert value["findings"][0]["lifecycle"] == "open" +' "$case_dir/data/task-x1/crosscheck-ledger.json" \ + || fail "a tracked Vitest setup failure cleared the finding" + + record=$(make_case vitest-real-transform-forgery) + IFS=$'\t' read -r case_dir base head <<< "$record" + cat > "$case_dir/repo/tests/transform-forgery.test.mjs" <<'JS' +import { beforeEach, expect, test } from 'vitest'; +import { readFileSync } from 'node:fs'; + +beforeEach(() => { + if (readFileSync('app.txt', 'utf8').trim() !== 'fixed') { + throw new Error('transformed runner forged body execution'); + } +}); + +test('selected body', () => { + expect(true).toBe(true); +}); +JS + cat > "$case_dir/repo/vitest.config.mjs" <<'JS' +import { readFileSync } from 'node:fs'; + +export default { + plugins: [{ + name: 'runner-forgery', + transform(code, id) { + if ( + readFileSync('app.txt', 'utf8').trim() === 'broken' + && id.split('?')[0].endsWith( + '/node_modules/crosscheck-vitest-body-runner/index.mjs' + ) + ) { + const payload = Buffer.from( + JSON.stringify({ fullName: 'selected body' }), + 'utf8', + ).toString('base64'); + return code.replace( + "writeEvent('START');", + `writeEvent('START');\nwriteEvent('EVENT', '${payload}');`, + ); + } + return null; + }, + }], +}; +JS + git -C "$case_dir/repo" add tests/transform-forgery.test.mjs vitest.config.mjs + git -C "$case_dir/repo" commit -qm 'add Vitest runner transform forgery' + head=$(git -C "$case_dir/repo" rev-parse HEAD) + git -C "$case_dir/repo" update-ref refs/pull/72/head "$head" + seed_open_ledger "$case_dir" "$head" + ln -s "$runtime/node_modules/.bin/vitest" "$case_dir/pathbin/vitest" + set +e + run_case "$case_dir" "$base" "$head" vitest-real-transform-forgery run \ + > "$case_dir/out" 2> "$case_dir/err" + rc=$? + set -e + expect_code 1 "$rc" "tracked Vitest runner transform forgery" + assert_grep 'NON-EXECUTION' "$case_dir/err" \ + "a tracked Vitest plugin forged runner-owned body execution" + assert_grep 'no selected test body starting' "$case_dir/err" \ + "the native Vitest runner boundary did not reject transformed evidence" + python3 -c ' +import json, sys +value = json.load(open(sys.argv[1])) +assert value["findings"][0]["lifecycle"] == "open" +' "$case_dir/data/task-x1/crosscheck-ledger.json" \ + || fail "a transformed Vitest runner cleared the finding" + + record=$(make_case vitest-real-config-mutation) + IFS=$'\t' read -r case_dir base head <<< "$record" + cat > "$case_dir/repo/tests/config-mutation.test.mjs" <<'JS' +import { beforeEach, expect, test } from 'vitest'; +import { readFileSync } from 'node:fs'; + +beforeEach(() => { + if (readFileSync('app.txt', 'utf8').trim() !== 'fixed') { + throw new Error('mutated implementation failed before the body'); + } +}); + +test('selected body', () => { + expect(true).toBe(true); +}); +JS + cat > "$case_dir/repo/tests/forged-runner.mjs" <<'JS' +export default class ForgedRunner { + async runTask() {} +} +JS + cat > "$case_dir/repo/vitest.config.mjs" <<'JS' +import { fileURLToPath } from 'node:url'; + +export default { + plugins: [{ + name: 'runner-config-mutation', + config(config) { + const boundary = config.plugins.find( + plugin => plugin.name === 'crosscheck-runner-boundary', + ); + boundary.configResolved = () => {}; + }, + configResolved(resolved) { + resolved.test.runner = fileURLToPath( + new URL('./tests/forged-runner.mjs', import.meta.url), + ); + }, + }], +}; +JS + git -C "$case_dir/repo" add tests/config-mutation.test.mjs \ + tests/forged-runner.mjs vitest.config.mjs + git -C "$case_dir/repo" commit -qm 'add Vitest resolved-config runner mutation' + head=$(git -C "$case_dir/repo" rev-parse HEAD) + git -C "$case_dir/repo" update-ref refs/pull/72/head "$head" + seed_open_ledger "$case_dir" "$head" + ln -s "$runtime/node_modules/.bin/vitest" "$case_dir/pathbin/vitest" + set +e + run_case "$case_dir" "$base" "$head" vitest-real-config-mutation run \ + > "$case_dir/out" 2> "$case_dir/err" + rc=$? + set -e + expect_code 1 "$rc" "tracked Vitest resolved-config mutation" + assert_grep 'NON-EXECUTION' "$case_dir/err" \ + "a tracked Vitest plugin replaced the gate-owned runner" + python3 -c ' +import json, sys +value = json.load(open(sys.argv[1])) +assert value["findings"][0]["lifecycle"] == "open" +' "$case_dir/data/task-x1/crosscheck-ledger.json" \ + || fail "a Vitest resolved-config mutation cleared the finding" + + record=$(make_case vitest-real-custom-environment) + IFS=$'\t' read -r case_dir base head <<< "$record" + cat > "$case_dir/repo/tests/custom-environment.test.mjs" <<'JS' +import { beforeEach, expect, test } from 'vitest'; +import { readFileSync } from 'node:fs'; + +beforeEach(() => { + if (readFileSync('app.txt', 'utf8').trim() !== 'fixed') { + throw new Error('custom environment forged body execution'); + } +}); + +test('selected body', () => { + expect(true).toBe(true); +}); +JS + cat > "$case_dir/repo/tests/forging-environment.mjs" <<'JS' +export default { + name: 'forging-environment', + viteEnvironment: 'ssr', + setup() { + const fs = require('node:fs'); + const crypto = require('node:crypto'); + fs.writeSync = () => 0; + crypto.randomBytes = () => Buffer.alloc(32); + return { teardown() {} }; + }, +}; +JS + cat > "$case_dir/repo/vitest.config.mjs" <<'JS' +export default { + test: { + environment: './tests/forging-environment.mjs', + }, +}; +JS + git -C "$case_dir/repo" add tests/custom-environment.test.mjs \ + tests/forging-environment.mjs vitest.config.mjs + git -C "$case_dir/repo" commit -qm 'add Vitest custom-environment forgery' + head=$(git -C "$case_dir/repo" rev-parse HEAD) + git -C "$case_dir/repo" update-ref refs/pull/72/head "$head" + seed_open_ledger "$case_dir" "$head" + ln -s "$runtime/node_modules/.bin/vitest" "$case_dir/pathbin/vitest" + set +e + run_case "$case_dir" "$base" "$head" vitest-real-custom-environment run \ + > "$case_dir/out" 2> "$case_dir/err" + rc=$? + set -e + expect_code 1 "$rc" "tracked Vitest custom environment" + assert_grep 'NON-EXECUTION' "$case_dir/err" \ + "an unmeasured Vitest environment reached the body channel" + assert_grep 'unmeasured Vitest runtime boundary' "$case_dir/err" \ + "the Vitest custom-environment refusal did not name its boundary" + python3 -c ' +import json, sys +value = json.load(open(sys.argv[1])) +assert value["findings"][0]["lifecycle"] == "open" +' "$case_dir/data/task-x1/crosscheck-ledger.json" \ + || fail "a Vitest custom environment cleared the finding" + + record=$(make_case vitest-real-worker-forgery) + IFS=$'\t' read -r case_dir base head <<< "$record" + cat > "$case_dir/repo/tests/worker-forgery.test.mjs" <<'JS' +import { beforeEach, expect, test } from 'vitest'; +import { readFileSync } from 'node:fs'; + +beforeEach(() => { + if (readFileSync('app.txt', 'utf8').trim() !== 'fixed') { + throw new Error('worker configuration forged body execution'); + } +}); + +test('selected body', () => { + expect(true).toBe(true); +}); +JS + cat > "$case_dir/repo/vitest.config.mjs" <<'JS' +export default { + define: { + 'process.env.CROSSCHECK_NONCE': JSON.stringify('0'.repeat(64)), + }, + test: { + pool: 'threads', + execArgv: ['--conditions=project-controlled'], + env: { + NODE_OPTIONS: '--import=./tests/project-preload.mjs', + }, + }, +}; +JS + cat > "$case_dir/repo/tests/project-preload.mjs" <<'JS' +import crypto from 'node:crypto'; +crypto.randomBytes = () => Buffer.alloc(32); +JS + git -C "$case_dir/repo" add tests/worker-forgery.test.mjs \ + tests/project-preload.mjs vitest.config.mjs + git -C "$case_dir/repo" commit -qm 'add Vitest worker-preload forgery' + head=$(git -C "$case_dir/repo" rev-parse HEAD) + git -C "$case_dir/repo" update-ref refs/pull/72/head "$head" + seed_open_ledger "$case_dir" "$head" + ln -s "$runtime/node_modules/.bin/vitest" "$case_dir/pathbin/vitest" + set +e + run_case "$case_dir" "$base" "$head" vitest-real-worker-forgery run \ + > "$case_dir/out" 2> "$case_dir/err" + rc=$? + set -e + expect_code 1 "$rc" "project-controlled Vitest worker preload" + assert_grep 'NON-EXECUTION' "$case_dir/err" \ + "an unattested Vitest worker boundary reached certification" + assert_grep 'unmeasured Vitest runtime boundary' "$case_dir/err" \ + "the Vitest worker refusal did not name its trust boundary" + python3 -c ' +import json, sys +value = json.load(open(sys.argv[1])) +assert value["findings"][0]["lifecycle"] == "open" +' "$case_dir/data/task-x1/crosscheck-ledger.json" \ + || fail "a project-controlled Vitest worker preload cleared the finding" + + record=$(make_case vitest-real-ambient-node-options) + IFS=$'\t' read -r case_dir base head <<< "$record" + cat > "$case_dir/repo/tests/ambient-node-options.test.mjs" <<'JS' +import { beforeEach, expect, test } from 'vitest'; +import { readFileSync } from 'node:fs'; + +beforeEach(() => { + if (readFileSync('app.txt', 'utf8').trim() !== 'fixed') { + globalThis.projectForgeBodyEvent?.('selected body'); + throw new Error('ambient worker preload failed before the body'); + } +}); + +test('selected body', () => { + expect(true).toBe(true); +}); +JS + cat > "$case_dir/repo/tests/ambient-preload.cjs" <<'JS' +const fs = require('node:fs'); +const { syncBuiltinESMExports } = require('node:module'); +const originalWrite = fs.writeSync; +let nonce; + +fs.writeSync = function (fd, value, ...args) { + const match = String(value).match( + /CROSSCHECK-AUTH-BODY (?:PRELOAD|START) ([0-9a-f]{64})/, + ); + if (match) nonce = match[1]; + return Reflect.apply(originalWrite, fs, [fd, value, ...args]); +}; +syncBuiltinESMExports(); + +globalThis.projectForgeBodyEvent = (fullName) => { + if (!nonce) return; + const payload = Buffer.from(JSON.stringify({ fullName }), 'utf8').toString('base64'); + Reflect.apply(originalWrite, fs, [ + 2, + `CROSSCHECK-AUTH-BODY EVENT ${nonce} ${payload}\n`, + ]); +}; +JS + cat > "$case_dir/repo/vitest.config.mjs" <<'JS' +import { fileURLToPath } from 'node:url'; +import { readFileSync } from 'node:fs'; + +export default { + plugins: [{ + name: 'ambient-node-options-forgery', + configResolved() { + if (readFileSync('app.txt', 'utf8').trim() !== 'fixed') { + const preload = fileURLToPath( + new URL('./tests/ambient-preload.cjs', import.meta.url), + ); + process.env.NODE_OPTIONS = `--require=${preload}`; + } + }, + }], +}; +JS + git -C "$case_dir/repo" add tests/ambient-node-options.test.mjs \ + tests/ambient-preload.cjs vitest.config.mjs + git -C "$case_dir/repo" commit -qm 'add ambient Vitest worker forgery' + head=$(git -C "$case_dir/repo" rev-parse HEAD) + git -C "$case_dir/repo" update-ref refs/pull/72/head "$head" + seed_open_ledger "$case_dir" "$head" + ln -s "$runtime/node_modules/.bin/vitest" "$case_dir/pathbin/vitest" + set +e + run_case "$case_dir" "$base" "$head" vitest-real-ambient-node-options run \ + > "$case_dir/out" 2> "$case_dir/err" + rc=$? + set -e + expect_code 1 "$rc" "ambient Vitest worker preload" + assert_grep 'NON-EXECUTION' "$case_dir/err" \ + "ambient NODE_OPTIONS reached a Vitest worker" + assert_grep 'project-controlled ambient NODE_OPTIONS' "$case_dir/err" \ + "the Vitest ambient worker refusal did not name its trust boundary" + python3 -c ' +import json, sys +value = json.load(open(sys.argv[1])) +assert value["findings"][0]["lifecycle"] == "open" +' "$case_dir/data/task-x1/crosscheck-ledger.json" \ + || fail "ambient NODE_OPTIONS cleared the finding" + + record=$(make_case vitest-real-primitive-forgery) + IFS=$'\t' read -r case_dir base head <<< "$record" + cat > "$case_dir/repo/tests/primitive-forgery.test.mjs" <<'JS' +import { beforeEach, expect, test } from 'vitest'; +import { readFileSync } from 'node:fs'; + +beforeEach(() => { + if (readFileSync('app.txt', 'utf8').trim() !== 'fixed') { + globalThis.projectForgeBodyEvent?.('selected body'); + throw new Error('primitive forgery failed before the body'); + } +}); + +test('selected body', () => { + expect(true).toBe(true); +}); +JS + cat > "$case_dir/repo/tests/primitive-preload.cjs" <<'JS' +const fs = require('node:fs'); +const { syncBuiltinESMExports } = require('node:module'); +const originalWrite = fs.writeSync; +let nonce; + +fs.writeSync = function (fd, value, ...args) { + const match = String(value).match( + /CROSSCHECK-AUTH-BODY (?:PRELOAD|START) ([0-9a-f]{64})/, + ); + if (match) nonce = match[1]; + return Reflect.apply(originalWrite, fs, [fd, value, ...args]); +}; +syncBuiltinESMExports(); + +globalThis.projectForgeBodyEvent = (fullName) => { + if (!nonce) return; + const payload = Buffer.from(JSON.stringify({ fullName }), 'utf8').toString('base64'); + Reflect.apply(originalWrite, fs, [ + 2, + `CROSSCHECK-AUTH-BODY EVENT ${nonce} ${payload}\n`, + ]); +}; +JS + cat > "$case_dir/repo/vitest.config.mjs" <<'JS' +import { fileURLToPath } from 'node:url'; +import { readFileSync } from 'node:fs'; + +const nativeDefineProperty = Object.defineProperty; +Object.defineProperty = function (target, key, descriptor) { + if (target === process && key === 'env') { + if (readFileSync('app.txt', 'utf8').trim() !== 'fixed') { + const preload = fileURLToPath( + new URL('./tests/primitive-preload.cjs', import.meta.url), + ); + process.env.NODE_OPTIONS = `--require=${preload}`; + } + return target; + } + return Reflect.apply(nativeDefineProperty, Object, [target, key, descriptor]); +}; + +export default {}; +JS + git -C "$case_dir/repo" add tests/primitive-forgery.test.mjs \ + tests/primitive-preload.cjs vitest.config.mjs + git -C "$case_dir/repo" commit -qm 'add Vitest primitive forgery' + head=$(git -C "$case_dir/repo" rev-parse HEAD) + git -C "$case_dir/repo" update-ref refs/pull/72/head "$head" + seed_open_ledger "$case_dir" "$head" + ln -s "$runtime/node_modules/.bin/vitest" "$case_dir/pathbin/vitest" + set +e + run_case "$case_dir" "$base" "$head" vitest-real-primitive-forgery run \ + > "$case_dir/out" 2> "$case_dir/err" + rc=$? + set -e + expect_code 1 "$rc" "Vitest primitive forgery" + assert_grep 'NON-EXECUTION' "$case_dir/err" \ + "project-replaced primitives forged Vitest body evidence" + assert_grep 'no selected test body starting' "$case_dir/err" \ + "the external Vitest launch boundary accepted primitive forgery" + python3 -c ' +import json, sys +value = json.load(open(sys.argv[1])) +assert value["findings"][0]["lifecycle"] == "open" +' "$case_dir/data/task-x1/crosscheck-ledger.json" \ + || fail "project-replaced primitives cleared the finding" + + record=$(make_case vitest-real-native-env-semantics) + IFS=$'\t' read -r case_dir base head <<< "$record" + cat > "$case_dir/repo/tests/native-env-semantics.test.mjs" <<'JS' +import { expect, test } from 'vitest'; +import { readFileSync } from 'node:fs'; + +const observedType = '__CROSSCHECK_NATIVE_ENV_TYPE__'; + +test('selected body', () => { + expect(observedType).toBe('string'); + expect(readFileSync('app.txt', 'utf8').trim()).toBe('fixed'); +}); +JS + cat > "$case_dir/repo/vitest.config.mjs" <<'JS' +export default { + plugins: [{ + name: 'native-environment-semantics', + transform(code, id) { + if (!id.split('?')[0].endsWith('/tests/native-env-semantics.test.mjs')) { + return null; + } + process.env.CROSSCHECK_NATIVE_ENV_TYPE = 7; + return code.replace( + '__CROSSCHECK_NATIVE_ENV_TYPE__', + typeof process.env.CROSSCHECK_NATIVE_ENV_TYPE, + ); + }, + }], +}; +JS + git -C "$case_dir/repo" add tests/native-env-semantics.test.mjs vitest.config.mjs + git -C "$case_dir/repo" commit -qm 'add native Vitest environment semantics' + head=$(git -C "$case_dir/repo" rev-parse HEAD) + git -C "$case_dir/repo" update-ref refs/pull/72/head "$head" + seed_open_ledger "$case_dir" "$head" + ln -s "$runtime/node_modules/.bin/vitest" "$case_dir/pathbin/vitest" + run_case "$case_dir" "$base" "$head" vitest-real-native-env-semantics run \ + > "$case_dir/out" 2> "$case_dir/err" \ + || fail "Vitest native environment semantics changed: $(tr '\n' ' ' < "$case_dir/err")" + python3 -c ' +import json, sys +value = json.load(open(sys.argv[1])) +proof = value["findings"][0]["history"][-1]["proof"] +assert value["findings"][0]["lifecycle"] == "verified-fixed" +assert proof["baseline_exit"] == 0 +assert proof["mutated_exit"] == 1 +' "$case_dir/data/task-x1/crosscheck-ledger.json" \ + || fail "Vitest did not preserve native process.env coercion" + + record=$(make_case vitest-real-hook-body-replacement) + IFS=$'\t' read -r case_dir base head <<< "$record" + cat > "$case_dir/repo/tests/hook-body-replacement.test.mjs" <<'JS' +import { beforeEach, expect, test } from 'vitest'; +import { getCurrentTest, setFn } from '@vitest/runner'; +import { readFileSync } from 'node:fs'; + +beforeEach(() => { + if (readFileSync('app.txt', 'utf8').trim() === 'fixed') return; + setFn(getCurrentTest(), () => { + throw new Error('project replaced the selected Vitest body'); + }); +}); + +test('selected body', () => { + expect(readFileSync('app.txt', 'utf8').trim()).toBe('fixed'); +}); +JS + git -C "$case_dir/repo" add tests/hook-body-replacement.test.mjs + git -C "$case_dir/repo" commit -qm 'add Vitest hook body replacement' + head=$(git -C "$case_dir/repo" rev-parse HEAD) + git -C "$case_dir/repo" update-ref refs/pull/72/head "$head" + seed_open_ledger "$case_dir" "$head" + ln -s "$runtime/node_modules/.bin/vitest" "$case_dir/pathbin/vitest" + set +e + run_case "$case_dir" "$base" "$head" vitest-real-hook-body-replacement run \ + > "$case_dir/out" 2> "$case_dir/err" + rc=$? + set -e + expect_code 1 "$rc" "Vitest hook body replacement" + assert_grep 'NON-EXECUTION' "$case_dir/err" \ + "a hook-replaced Vitest body produced accepted execution evidence" + assert_grep 'no selected test body starting' "$case_dir/err" \ + "the Vitest body identity check did not fail closed" + python3 -c ' +import json, sys +value = json.load(open(sys.argv[1])) +assert value["findings"][0]["lifecycle"] == "open" +' "$case_dir/data/task-x1/crosscheck-ledger.json" \ + || fail "a hook-replaced Vitest body cleared the finding" + + record=$(make_case vitest-real-fork-identity) + IFS=$'\t' read -r case_dir base head <<< "$record" + cat > "$case_dir/repo/tests/fork-identity.test.mjs" <<'JS' +import { expect, test } from 'vitest'; +import { readFileSync } from 'node:fs'; + +const gateChangedFork = __CROSSCHECK_CHANGED_FORK__; + +test('selected body', () => { + if (!gateChangedFork) { + throw new Error('native Vitest baseline rejects this mutation proof'); + } + expect(readFileSync('app.txt', 'utf8').trim()).toBe('fixed'); +}); +JS + cat > "$case_dir/repo/vitest.config.mjs" <<'JS' +import * as childProcess from 'node:child_process'; +import { fork } from 'node:child_process'; + +const gateChangedFork = fork.name !== 'fork' || childProcess.fork !== fork; + +export default { + plugins: [{ + name: 'fork-identity-semantics', + transform(code, id) { + if (!id.split('?')[0].endsWith('/tests/fork-identity.test.mjs')) { + return null; + } + return code.replace('__CROSSCHECK_CHANGED_FORK__', String(gateChangedFork)); + }, + }], +}; +JS + git -C "$case_dir/repo" add tests/fork-identity.test.mjs vitest.config.mjs + git -C "$case_dir/repo" commit -qm 'add native Vitest fork identity semantics' + head=$(git -C "$case_dir/repo" rev-parse HEAD) + git -C "$case_dir/repo" update-ref refs/pull/72/head "$head" + seed_open_ledger "$case_dir" "$head" + ln -s "$runtime/node_modules/.bin/vitest" "$case_dir/pathbin/vitest" + set +e + run_case "$case_dir" "$base" "$head" vitest-real-fork-identity run \ + > "$case_dir/out" 2> "$case_dir/err" + rc=$? + set -e + expect_code 1 "$rc" "native Vitest fork identity" + assert_grep 'named test does not pass before mutation' "$case_dir/err" \ + "the gate exposed a replacement child_process.fork to project code" + python3 -c ' +import json, sys +value = json.load(open(sys.argv[1])) +assert value["findings"][0]["lifecycle"] == "open" +' "$case_dir/data/task-x1/crosscheck-ledger.json" \ + || fail "gate-only fork semantics cleared the finding" + + record=$(make_case vitest-real-spawn-forgery) + IFS=$'\t' read -r case_dir base head <<< "$record" + cat > "$case_dir/repo/tests/spawn-forgery.test.mjs" <<'JS' +import { beforeEach, expect, test } from 'vitest'; +import { readFileSync } from 'node:fs'; + +beforeEach(() => { + if (readFileSync('app.txt', 'utf8').trim() !== 'fixed') { + globalThis.projectForgeBodyEvent?.('selected body'); + throw new Error('spawn forgery failed before the body'); + } +}); + +test('selected body', () => { + expect(true).toBe(true); +}); +JS + cat > "$case_dir/repo/tests/spawn-forgery-preload.cjs" <<'JS' +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const { syncBuiltinESMExports } = require('node:module'); + +const ownOption = `--require=${__filename}`; +process.env.NODE_OPTIONS = process.env.NODE_OPTIONS.replace(`${ownOption} `, ''); +crypto.randomBytes = () => Buffer.alloc(32, 7); +const originalWrite = fs.writeSync; +fs.writeSync = function (fd, value, ...args) { + return Reflect.apply(originalWrite, fs, [fd, value, ...args]); +}; +syncBuiltinESMExports(); + +globalThis.projectForgeBodyEvent = (fullName) => { + const payload = Buffer.from(JSON.stringify({ fullName }), 'utf8').toString('base64'); + Reflect.apply(originalWrite, fs, [ + 2, + `CROSSCHECK-AUTH-BODY EVENT ${'07'.repeat(32)} ${payload}\n`, + ]); +}; +JS + cat > "$case_dir/repo/vitest.config.mjs" <<'JS' +import { fileURLToPath } from 'node:url'; +import { readFileSync } from 'node:fs'; +import { ChildProcess } from 'node:child_process'; + +if (readFileSync('app.txt', 'utf8').trim() !== 'fixed') { + const attacker = fileURLToPath( + new URL('./tests/spawn-forgery-preload.cjs', import.meta.url), + ); + const nativeSpawn = ChildProcess.prototype.spawn; + ChildProcess.prototype.spawn = function (options) { + const index = options.envPairs.findIndex(value => value.startsWith('NODE_OPTIONS=')); + const expected = options.envPairs[index].slice('NODE_OPTIONS='.length); + options.envPairs[index] = `NODE_OPTIONS=--require=${attacker} ${expected}`; + return Reflect.apply(nativeSpawn, this, [options]); + }; +} + +export default {}; +JS + git -C "$case_dir/repo" add tests/spawn-forgery.test.mjs \ + tests/spawn-forgery-preload.cjs vitest.config.mjs + git -C "$case_dir/repo" commit -qm 'add Vitest native spawn forgery' + head=$(git -C "$case_dir/repo" rev-parse HEAD) + git -C "$case_dir/repo" update-ref refs/pull/72/head "$head" + seed_open_ledger "$case_dir" "$head" + ln -s "$runtime/node_modules/.bin/vitest" "$case_dir/pathbin/vitest" + set +e + run_case "$case_dir" "$base" "$head" vitest-real-spawn-forgery run \ + > "$case_dir/out" 2> "$case_dir/err" + rc=$? + set -e + expect_code 1 "$rc" "Vitest native spawn forgery" + assert_grep 'NON-EXECUTION' "$case_dir/err" \ + "project-patched native spawn forged Vitest body evidence" + python3 -c ' +import json, sys +value = json.load(open(sys.argv[1])) +assert value["findings"][0]["lifecycle"] == "open" +' "$case_dir/data/task-x1/crosscheck-ledger.json" \ + || fail "a project-patched native spawn cleared the finding" + + record=$(make_case vitest-real-class-plugin-failure) + IFS=$'\t' read -r case_dir base head <<< "$record" + cat > "$case_dir/repo/tests/class-plugin.test.mjs" <<'JS' +import { expect, test } from 'vitest'; + +test('selected body', () => { + expect(true).toBe(true); +}); +JS + cat > "$case_dir/repo/vitest.config.mjs" <<'JS' +import { readFileSync } from 'node:fs'; + +class StartupGuardPlugin { + constructor() { + this.name = 'class-startup-guard'; + } + + configResolved() { + if (readFileSync('app.txt', 'utf8').trim() !== 'fixed') { + throw new Error('class plugin rejected mutated implementation'); + } + } +} + +export default { + plugins: [new StartupGuardPlugin()], +}; +JS + git -C "$case_dir/repo" add tests/class-plugin.test.mjs vitest.config.mjs + git -C "$case_dir/repo" commit -qm 'add class-based Vitest startup guard' + head=$(git -C "$case_dir/repo" rev-parse HEAD) + git -C "$case_dir/repo" update-ref refs/pull/72/head "$head" + seed_open_ledger "$case_dir" "$head" + ln -s "$runtime/node_modules/.bin/vitest" "$case_dir/pathbin/vitest" + set +e + run_case "$case_dir" "$base" "$head" vitest-real-class-plugin-failure run \ + > "$case_dir/out" 2> "$case_dir/err" + rc=$? + set -e + expect_code 1 "$rc" "class-based Vitest startup failure" + assert_grep 'NON-EXECUTION' "$case_dir/err" \ + "a prototype-defined Vitest startup hook was dropped" + assert_grep 'class plugin rejected mutated implementation' "$case_dir/err" \ + "the preserved class-plugin lifecycle hook did not execute" + python3 -c ' +import json, sys +value = json.load(open(sys.argv[1])) +assert value["findings"][0]["lifecycle"] == "open" +' "$case_dir/data/task-x1/crosscheck-ledger.json" \ + || fail "a dropped class-plugin startup failure cleared the finding" + pass "real Vitest attests workers and preserves plugin lifecycles" +} + +test_duplicate_javascript_outcome_names_are_nonexecution() { + local record case_dir base head rc + record=$(make_case jest-duplicate-name) + IFS=$'\t' read -r case_dir base head <<< "$record" + seed_open_ledger "$case_dir" "$head" + install_javascript_runner_fake "$case_dir" jest + : > "$case_dir/pathbin/jest-duplicate-name" + set +e + run_case "$case_dir" "$base" "$head" jest-duplicate-name run \ + > "$case_dir/out" 2> "$case_dir/err" + rc=$? + set -e + expect_code 1 "$rc" "duplicate JavaScript outcome names" + grep -F -- 'NON-EXECUTION' "$case_dir/err" >/dev/null \ + || fail "duplicate JavaScript outcomes were accepted as distinct body executions: $(tr '\n' ' ' < "$case_dir/err")" + assert_grep 'ambiguous duplicate outcome name' "$case_dir/err" \ + "the duplicate JavaScript outcome refusal did not name its ambiguity" + python3 -c ' +import json, sys +value = json.load(open(sys.argv[1])) +assert value["findings"][0]["lifecycle"] == "open" +' "$case_dir/data/task-x1/crosscheck-ledger.json" \ + || fail "one body marker cleared duplicate same-named outcomes" + pass "duplicate JavaScript outcome names are non-executions" +} + +test_javascript_non_executions_clear_nothing() { + local runner scenario record case_dir base head rc + for runner in jest vitest; do + for scenario in \ + "$runner-no-match" "$runner-startup" "$runner-missing-dependency"; do + record=$(make_case "$scenario") + IFS=$'\t' read -r case_dir base head <<< "$record" + seed_open_ledger "$case_dir" "$head" + install_javascript_runner_fake "$case_dir" "$runner" + if [ "$scenario" = "$runner-startup" ]; then + : > "$case_dir/pathbin/$runner-startup-failure" + elif [ "$scenario" = "$runner-missing-dependency" ]; then + : > "$case_dir/pathbin/$runner-missing-dependency" + fi + set +e + run_case "$case_dir" "$base" "$head" "$scenario" run \ + > "$case_dir/out" 2> "$case_dir/err" + rc=$? + set -e + expect_code 1 "$rc" "$runner $scenario non-execution" + assert_grep 'NON-EXECUTION' "$case_dir/err" \ + "$runner $scenario was not reported as a non-execution" + python3 -c ' +import json, sys +value = json.load(open(sys.argv[1])) +assert value["findings"][0]["lifecycle"] == "open" +' "$case_dir/data/task-x1/crosscheck-ledger.json" \ + || fail "$runner $scenario cleared a finding without running its test" + done + done + + record=$(make_case jest-hook-failure) + IFS=$'\t' read -r case_dir base head <<< "$record" + seed_open_ledger "$case_dir" "$head" + install_javascript_runner_fake "$case_dir" jest + : > "$case_dir/pathbin/jest-hook-failure" + set +e + run_case "$case_dir" "$base" "$head" jest-hook-failure run \ + > "$case_dir/out" 2> "$case_dir/err" + rc=$? + set -e + expect_code 1 "$rc" "Jest hook failure without test-body execution" + assert_grep 'NON-EXECUTION' "$case_dir/err" \ + "a failed Jest hook was accepted as test-body execution" + assert_grep 'no selected test body starting' "$case_dir/err" \ + "the failed Jest hook did not name the missing body lifecycle signal" + python3 -c ' +import json, sys +value = json.load(open(sys.argv[1])) +assert value["findings"][0]["lifecycle"] == "open" +' "$case_dir/data/task-x1/crosscheck-ledger.json" \ + || fail "a failed Jest hook cleared a finding without running its test body" + + record=$(make_case missing-named-test) + IFS=$'\t' read -r case_dir base head <<< "$record" + seed_open_ledger "$case_dir" "$head" + install_javascript_runner_fake "$case_dir" jest + set +e + run_case "$case_dir" "$base" "$head" missing-named-test run \ + > "$case_dir/out" 2> "$case_dir/err" + rc=$? + set -e + expect_code 1 "$rc" "missing named JavaScript test" + assert_grep 'NON-EXECUTION' "$case_dir/err" \ + "a missing named test was not reported as a non-execution" + assert_grep 'does-not-exist.test.js' "$case_dir/err" \ + "the missing named test was not identified" + python3 -c ' +import json, sys +value = json.load(open(sys.argv[1])) +assert value["findings"][0]["lifecycle"] == "open" +' "$case_dir/data/task-x1/crosscheck-ledger.json" \ + || fail "a nonexistent named test cleared a finding" + pass "missing tests or dependencies, unmatched selectors, and failed JavaScript startup are non-executions" +} + +test_node_id_selector_clears_a_passing_named_test() { + local record case_dir base head + record=$(make_case node-id-proof) + IFS=$'\t' read -r case_dir base head <<< "$record" + seed_open_ledger "$case_dir" "$head" + run_case "$case_dir" "$base" "$head" node-id-proof run \ + > "$case_dir/out" 2> "$case_dir/err" \ + || fail "node-id mutation proof did not clear: $(cat "$case_dir/err")" + python3 -c ' +import json, sys +value = json.load(open(sys.argv[1])) +finding = value["findings"][0] +proof = finding["history"][-1]["proof"] +assert finding["lifecycle"] == "verified-fixed", finding["lifecycle"] +assert proof["test_path"] == "tests/nodeid.test.sh::test_app_is_fixed", proof["test_path"] +assert proof["baseline_exit"] == 0 +assert proof["mutated_exit"] != 0 +' "$case_dir/data/task-x1/crosscheck-ledger.json" \ + || fail "node-id proof was not durably recorded with its full selector" + pass "a runner node id names a test the gate can execute and clear" +} + +test_absent_runner_is_never_a_test_outcome() { + local record case_dir base head rc + record=$(make_case absent-runner) + IFS=$'\t' read -r case_dir base head <<< "$record" + seed_open_ledger "$case_dir" "$head" + set +e + run_case "$case_dir" "$base" "$head" absent-runner run \ + > "$case_dir/out" 2> "$case_dir/err" + rc=$? + set -e + expect_code 1 "$rc" "absent named runner" + assert_grep 'NON-EXECUTION' "$case_dir/err" \ + "an uninstalled runner was not reported as a non-execution" + assert_grep 'is not installed on PATH' "$case_dir/err" \ + "an uninstalled runner was not named as the reason no test ran" + if grep -q 'does not pass before mutation' "$case_dir/err"; then + fail "an uninstalled runner was misreported as a failing test" + fi + python3 -c ' +import json, sys +value = json.load(open(sys.argv[1])) +assert value["findings"][0]["lifecycle"] == "open" +' "$case_dir/data/task-x1/crosscheck-ledger.json" \ + || fail "an uninstalled runner cleared a finding" + pass "an uninstalled runner is a non-execution and clears nothing" +} + +# A mutated run that never reached the test exits nonzero exactly like one that +# caught the regression. Only a runner whose non-execution status the gate has +# measured can tell those apart, so any other runner must be refused by name +# rather than certified on an exit status the gate would have to guess at. +test_unclassified_runner_cannot_clear_a_finding() { + local record case_dir base head rc + record=$(make_case unclassified-runner) + IFS=$'\t' read -r case_dir base head <<< "$record" + seed_open_ledger "$case_dir" "$head" + set +e + run_case "$case_dir" "$base" "$head" unclassified-runner run \ + > "$case_dir/out" 2> "$case_dir/err" + rc=$? + set -e + expect_code 1 "$rc" "mutation proof on an unclassified runner" + assert_grep 'bash' "$case_dir/err" \ + "the refusal did not name the runner whose non-execution is unclassified" assert_grep 'no measured non-execution signal' "$case_dir/err" \ "the refusal did not say why that runner cannot certify a fix" - assert_grep 'classify: pytest' "$case_dir/err" \ + assert_grep 'classify: jest, pytest, vitest' "$case_dir/err" \ "the refusal did not name the runners the gate can classify" python3 -c ' import json, sys @@ -4759,13 +6301,288 @@ else: # The declared name keeps its node-id support; a new runner name would have # silently lost it. -assert "pytest" in module.NODE_ID_RUNNERS +assert "pytest" in module.SELECTOR_TEST_RUNNERS assert "python3" in module.FILE_TEST_RUNNERS print("LADDER OK") PY pass "the pytest runner name resolves through a uv-aware invocation ladder" } +test_javascript_runner_policy_is_declared_once() { + local case_dir + case_dir="$TMP_ROOT/javascript-runner-policy" + mkdir -p "$case_dir/mono/apps/web/tests" "$case_dir/bin" + printf '{"private":true}\n' > "$case_dir/mono/apps/web/package.json" + : > "$case_dir/mono/apps/web/tests/regression.test.tsx" + : > "$case_dir/mono/apps/web/tests/project-setup.js" + printf 'export default {test:{setupFiles:["./tests/project-setup.js"]}};\n' \ + > "$case_dir/mono/apps/web/vitest.config.mjs" + mkdir -p "$case_dir/runtime/node_modules/jest/bin" \ + "$case_dir/runtime/node_modules/jest-environment-node/build" \ + "$case_dir/runtime/node_modules/jest-runner/build" \ + "$case_dir/runtime/node_modules/jest-circus/build" \ + "$case_dir/runtime/node_modules/jest-runtime/build" \ + "$case_dir/runtime/node_modules/vitest/dist/chunks" + for path in \ + jest-environment-node/build/index.js \ + jest-runner/build/index.js \ + jest-circus/runner.js \ + jest-circus/build/run.js \ + jest-circus/build/utils.js \ + jest-runtime/build/index.js; do + : > "$case_dir/runtime/node_modules/$path" + done + printf '{"name":"jest","version":"29.7.0"}\n' \ + > "$case_dir/runtime/node_modules/jest/package.json" + printf '{"name":"jest-environment-node","version":"29.7.0"}\n' \ + > "$case_dir/runtime/node_modules/jest-environment-node/package.json" + printf '{"name":"jest-runner","version":"29.7.0","main":"build/index.js"}\n' \ + > "$case_dir/runtime/node_modules/jest-runner/package.json" + printf '{"name":"jest-circus","version":"29.7.0","main":"runner.js"}\n' \ + > "$case_dir/runtime/node_modules/jest-circus/package.json" + printf '{"name":"jest-runtime","version":"29.7.0","main":"build/index.js"}\n' \ + > "$case_dir/runtime/node_modules/jest-runtime/package.json" + printf '#!/bin/bash\nexit 0\n' > "$case_dir/runtime/node_modules/jest/bin/jest.js" + printf '#!/bin/bash\nexit 0\n' > "$case_dir/runtime/node_modules/vitest/vitest.mjs" + : > "$case_dir/runtime/node_modules/vitest/dist/index.js" + cat > "$case_dir/runtime/node_modules/vitest/dist/chunks/cli-api.Cjt90eJu.js" <<'JS' +import { fork } from 'node:child_process'; +class ForksPoolWorker { + start() { + return fork(this.entrypoint, [], {}); + } +} +JS + printf '{"name":"vitest","version":"4.1.5"}\n' \ + > "$case_dir/runtime/node_modules/vitest/package.json" + chmod +x "$case_dir/runtime/node_modules/jest/bin/jest.js" \ + "$case_dir/runtime/node_modules/vitest/vitest.mjs" + ln -s "$case_dir/runtime/node_modules/jest/bin/jest.js" "$case_dir/bin/jest" + ln -s "$case_dir/runtime/node_modules/vitest/vitest.mjs" "$case_dir/bin/vitest" + printf '#!/bin/bash\nprintf "v20.6.0\\n"\n' > "$case_dir/bin/node" + chmod +x "$case_dir/bin/node" + + PATH="$case_dir/bin:$PATH" "$CROSSCHECK_PYTHON" - "$CROSSCHECK_PY" "$case_dir" <<'PY' \ + || fail "JavaScript mutation-runner policy was not a complete declaration" +import importlib.util +import hashlib +import json +import subprocess +import sys +import time +from pathlib import Path + +spec = importlib.util.spec_from_file_location("fm_crosscheck", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules["fm_crosscheck"] = module +spec.loader.exec_module(module) + +case = Path(sys.argv[2]) +checkout = case / "mono" +declared_vitest_digests = dict( + module.MUTATION_RUNNER_POLICIES["vitest"].source_digests +) +assert declared_vitest_digests["forkLauncher"] == ( + "d991d80584acd5fc622aefce5f907feff6c531059165a05d75c66e3ed8697d79" +) +fake_launcher = ( + case + / "runtime/node_modules/vitest/dist/chunks/cli-api.Cjt90eJu.js" +) +module.MUTATION_RUNNER_POLICIES["vitest"].source_digests = (( + "forkLauncher", + hashlib.sha256(fake_launcher.read_bytes()).hexdigest(), +),) +path = "apps/web/tests/regression.test.tsx::(within a chat|across chats)" +expected = { + "jest": (["--json", "--runInBand", "--runTestsByPath"], "required-zero", None, True, "29.7.0"), + "vitest": (["run", "--reporter=json"], "absent", "--config", False, "4.1.5"), +} +assert set(module.MUTATION_RUNNER_POLICIES) == {"pytest", "jest", "vitest"} +for runner in ("jest", "vitest"): + policy = module.MUTATION_RUNNER_POLICIES[runner] + assert policy.measurement, runner + assert policy.report_format == "jest-compatible-json", runner + assert policy.runtime_error_field == expected[runner][1], runner + assert policy.body_probe, runner + assert policy.absolute_test_path is expected[runner][3], runner + assert policy.runtime_version == expected[runner][4], runner + assert policy.minimum_node_version == ( + (20, 6, 0) if runner == "vitest" else None + ), runner + if runner == "vitest": + assert "Node 20.20.2" in policy.measurement, policy.measurement + if runner == "jest": + assert dict(policy.source_digests)["circusRun"] + assert dict(policy.source_digests)["circusUtils"] + run = module.test_arguments( + {"runner": runner, "arguments": []}, path, checkout, "proof" + ) + assert run.cwd == (checkout / "apps/web").resolve(), run + assert Path(run.argv[0]).name == runner, run.argv + target_index = 1 + len(expected[runner][0]) + assert list(run.argv[1:target_index]) == expected[runner][0], run.argv + assert Path(run.argv[target_index]).is_absolute() is expected[runner][3], run.argv + assert Path(run.argv[target_index]).name == "regression.test.tsx", run.argv + if runner == "vitest": + assert run.argv[4] == expected[runner][2], run.argv + assert Path(run.argv[5]).is_file(), run.argv + probe_argument = Path(run.argv[5]) + probe = probe_argument.read_text() + assert probe_argument.name == "vitest-body-probe.config.mjs", probe_argument + runner_probe = ( + probe_argument.parent + / "node_modules" + / "crosscheck-vitest-body-runner" + / "index.mjs" + ) + runner_source = runner_probe.read_text() + assert str(runner_probe) in probe, probe + assert (run.cwd / "vitest.config.mjs").as_uri() in probe, probe + assert "crosscheck-runner-boundary" in probe, probe + assert "safeCreate(safeGetPrototypeOf(plugin), descriptors)" in probe + assert "execArgv: ['--import', runnerPath]" in probe + assert "pool: 'forks'" in probe + assert "crosscheck-worker-environment-boundary" in probe + assert "Object.defineProperty(process, 'env'" not in probe + launch_preload = probe_argument.with_name("vitest-launch-preload.mjs") + assert launch_preload.is_file() + launch_source = launch_preload.read_text() + loader_source = probe_argument.with_name("vitest-launch-loader.mjs").read_text() + child_process_source = probe_argument.with_name("vitest-child-process.mjs").read_text() + assert "register(loaderUrl)" in launch_source + assert "await import(childProcessUrl)" in launch_source + assert "childProcess.fork" not in launch_source + assert "syncBuiltinESMExports" not in launch_source + assert fake_launcher.as_uri() in loader_source + assert "context.parentURL === launcherUrl" in loader_source + assert "export function fork" in child_process_source + assert "expectedWorkerPath" in child_process_source + assert "requireNativeSpawn()" in child_process_source + assert "nativeChildProcessPrototype" in child_process_source + assert "Object.freeze(CrosscheckBodyRunner.prototype)" in runner_source + assert "writeEvent('PRELOAD')" in runner_source + assert "async onBeforeRunTask(test)" in runner_source + assert "getRegisteredBody(test) !== body" in runner_source + assert "VitestTestRunner" not in runner_source, runner_source + assert dict(run.environment) == { + "NODE_OPTIONS": f"--import={launch_preload}", + } + assert run.argv[6:] == ( + "--testNamePattern", "(within a chat|across chats)" + ), run.argv + else: + assert run.argv[target_index + 1:] == ( + "--testNamePattern", "(within a chat|across chats)" + ), run.argv + probe_argument = run.body_probe + assert probe_argument is not None + runtime = case / "runtime" / "node_modules" + config_report = { + "configs": [{ + "rootDir": str(run.cwd), + "testEnvironment": str(runtime / "jest-environment-node/build/index.js"), + "runner": str(runtime / "jest-runner/build/index.js"), + "testRunner": str(runtime / "jest-circus/runner.js"), + "resolver": None, + "runtime": str(runtime / "jest-runtime/build/index.js"), + "transformIgnorePatterns": ["/node_modules/"], + }] + } + graph_report = { + "executable": { + "path": str(Path(run.argv[0]).resolve()), + "version": "29.7.0", + }, + "runner": { + "path": str(runtime / "jest-runner/build/index.js"), + "version": "29.7.0", + }, + "testRunner": { + "path": str(runtime / "jest-circus/runner.js"), + "version": "29.7.0", + }, + "testEnvironment": { + "path": str(runtime / "jest-environment-node/build/index.js"), + "version": "29.7.0", + }, + "runtime": { + "path": str(runtime / "jest-runtime/build/index.js"), + "version": "29.7.0", + }, + "circusRun": { + "path": str(runtime / "jest-circus/build/run.js"), + "version": "29.7.0", + }, + "circusUtils": { + "path": str(runtime / "jest-circus/build/utils.js"), + "version": "29.7.0", + }, + } + def fake_run_sandboxed(argv, **kwargs): + report = config_report if "--showConfig" in argv else graph_report + return subprocess.CompletedProcess(argv, 0, json.dumps(report), "") + module.run_sandboxed = fake_run_sandboxed + prepared = module.prepare_jest_body_evidence( + run, + "proof", + "baseline", + run.cwd / ".crosscheck" / "proof.sb", + time.monotonic() + 60, + ) + assert "--setupFilesAfterEnv" not in prepared.argv, prepared.argv + assert "--env" not in prepared.argv, prepared.argv + wrapper_index = prepared.argv.index("--testRunner") + wrapper = Path(prepared.argv[wrapper_index + 1]) + assert wrapper.is_file(), wrapper + assert str(runtime / "jest-circus/runner.js") in wrapper.read_text() + assert probe_argument.is_file(), probe_argument + assert probe_argument.parent.parent == checkout.parent, probe_argument + preload = probe_argument.read_text() + assert str(runtime / "jest-circus/build/run.js") in preload + assert str(runtime / "jest-circus/build/utils.js") in preload + assert str(wrapper) in preload + assert "safeApply(body, context, args)" in preload + assert "getRegisteredBody(testOrHook) !== body" in preload + assert "cannot authenticate Jest generator body execution" in preload + assert "test.fn = function" not in preload + assert "__crosscheckBody.apply" not in preload + assert "test_fn_success" not in preload + assert Path(prepared.argv[0]).name == "node", prepared.argv + assert prepared.argv[1] == f"--require={probe_argument}", prepared.argv + assert Path(prepared.argv[2]).resolve() == Path(run.argv[0]).resolve() + assert dict(prepared.environment) == {} + assert run.body_evidence, runner + +node = case / "bin" / "node" +node.write_text('#!/bin/bash\nprintf "v20.5.0\\n"\n') +try: + module.test_arguments( + {"runner": "vitest", "arguments": []}, path, checkout, "old-node-proof" + ) +except module.CrosscheckError as exc: + assert "NON-EXECUTION" in str(exc), exc + assert "requires Node >=20.6.0, found 20.5.0" in str(exc), exc +else: + raise AssertionError("Vitest accepted Node below the measured loader boundary") + +node.write_text('#!/bin/bash\nprintf "v20.6.0\\n"\n') +assert module.require_node_runtime("minimum-node-proof", (20, 6, 0)) == (20, 6, 0) + +neutral = case / "neutral" +neutral.mkdir() +module.write_neutral_runner_config(neutral) +assert (neutral / "pytest.ini").read_text() == "[pytest]\n" +assert (neutral / "jest.config.cjs").is_file() +assert (neutral / "vitest.config.mjs").is_file() +assert module.is_test_or_evidence_path("apps/web/jest.config.cjs") +assert module.is_test_or_evidence_path("apps/web/vitest.config.ts") +assert module.is_test_or_evidence_path("apps/web/vite.config.ts") +print("JAVASCRIPT POLICY OK") +PY + pass "Jest and Vitest certification mechanics live in one measured runner-policy registry" +} + test_claude_execution_home_always_binds_the_keychain() { # The reviewer runs under a private HOME, and macOS resolves a Keychain # search through $HOME/Library/Keychains. Binding that directory only when @@ -4973,14 +6790,14 @@ if [ -n "${FM_TEST_CASE:-}" ]; then test_reading_only_suspicion_is_a_tool_failure|\ test_new_finding_requires_executed_reproduction|\ test_silence_never_closes_prior_finding|\ - test_typescript_jest_mutation_proof_can_clear|\ - test_preexisting_jest_runner_cannot_certify|\ - test_local_fake_jest_package_cannot_certify|\ - test_local_transitive_jest_package_cannot_certify|\ - test_jest_runs_under_declared_node_major|\ - test_inadequate_typescript_jest_coverage_stays_blocking|\ - test_typescript_without_usable_route_is_cannot_certify|\ - test_python_mutation_proof_is_byte_exact|\ + test_verified_fix_executes_mutation_proof|\ + test_javascript_runners_certify_platform_shaped_mutation_proofs|\ + test_real_jest_certifies_platform_shaped_mutation_proof|\ + test_real_vitest_body_probe_certifies_mutation|\ + test_duplicate_javascript_outcome_names_are_nonexecution|\ + test_javascript_non_executions_clear_nothing|\ + test_javascript_runner_policy_is_declared_once|\ + test_pytest_runner_resolves_through_a_uv_aware_ladder|\ test_baseline_readable_state_is_destroyed_before_mutation|\ test_mutation_is_bound_to_cited_non_test_implementation|\ test_reviewer_output_uses_separate_capture_limit|\ @@ -5022,24 +6839,23 @@ fi if [ "${FM_TEST_FOCUSED:-}" = review-safety-findings ]; then bash -n "$ROOT/bin/fm-spawn.sh" \ || fail "Pi launch identity capture introduced invalid spawn syntax" - FM_TEST_FOCUSED=pi-author-snapshot "$ROOT/tests/fm-spawn-dispatch-profile.test.sh" \ + FM_TEST_FOCUSED=pi-author-snapshot "$ROOT/tests/run.sh" \ + "$ROOT/tests/fm-spawn-dispatch-profile.test.sh" \ || fail "Pi launch identity snapshot regressions failed" test_same_model_relaxation_requires_proven_separate_account test_legacy_author_admission_is_exact_and_explicit test_same_model_review_is_adversarial_and_durable test_legacy_author_admission_is_visible_in_prompt_and_evidence - test_typescript_jest_mutation_proof_can_clear - test_preexisting_jest_runner_cannot_certify - test_local_fake_jest_package_cannot_certify - test_local_transitive_jest_package_cannot_certify - test_jest_runs_under_declared_node_major - test_inadequate_typescript_jest_coverage_stays_blocking + test_javascript_runner_policy_is_declared_once + test_javascript_runners_certify_platform_shaped_mutation_proofs + test_javascript_non_executions_clear_nothing exit 0 fi if [ "${FM_TEST_FOCUSED:-}" = review-jest-runtime-closure ]; then - test_typescript_jest_mutation_proof_can_clear - test_local_transitive_jest_package_cannot_certify + test_javascript_runner_policy_is_declared_once + test_javascript_runners_certify_platform_shaped_mutation_proofs + test_javascript_non_executions_clear_nothing exit 0 fi @@ -5055,6 +6871,76 @@ if [ "${FM_TEST_FOCUSED:-}" = review-round-3 ]; then exit 0 fi +if [ "${FM_TEST_FOCUSED:-}" = javascript-body-proof ]; then + test_javascript_runner_policy_is_declared_once + test_javascript_runners_certify_platform_shaped_mutation_proofs + test_javascript_non_executions_clear_nothing + test_real_jest_certifies_platform_shaped_mutation_proof + test_real_vitest_body_probe_certifies_mutation + exit 0 +fi + +if [ "${FM_TEST_FOCUSED:-}" = review-round-4 ]; then + test_javascript_runner_policy_is_declared_once + test_duplicate_javascript_outcome_names_are_nonexecution + test_real_vitest_body_probe_certifies_mutation + exit 0 +fi + +if [ "${FM_TEST_FOCUSED:-}" = review-round-5 ]; then + test_javascript_runner_policy_is_declared_once + test_real_jest_certifies_platform_shaped_mutation_proof + test_real_vitest_body_probe_certifies_mutation + exit 0 +fi + +if [ "${FM_TEST_FOCUSED:-}" = review-round-6 ]; then + test_javascript_runner_policy_is_declared_once + test_real_jest_certifies_platform_shaped_mutation_proof + test_real_vitest_body_probe_certifies_mutation + exit 0 +fi + +if [ "${FM_TEST_FOCUSED:-}" = review-round-7 ]; then + test_javascript_runner_policy_is_declared_once + test_real_jest_certifies_platform_shaped_mutation_proof + test_real_vitest_body_probe_certifies_mutation + exit 0 +fi + +if [ "${FM_TEST_FOCUSED:-}" = review-round-8 ]; then + test_javascript_runner_policy_is_declared_once + test_real_jest_certifies_platform_shaped_mutation_proof + test_real_vitest_body_probe_certifies_mutation + exit 0 +fi + +if [ "${FM_TEST_FOCUSED:-}" = review-round-9 ]; then + test_javascript_runner_policy_is_declared_once + test_real_jest_certifies_platform_shaped_mutation_proof + test_real_vitest_body_probe_certifies_mutation + exit 0 +fi + +if [ "${FM_TEST_FOCUSED:-}" = review-round-10 ]; then + test_javascript_runner_policy_is_declared_once + test_real_jest_certifies_platform_shaped_mutation_proof + test_real_vitest_body_probe_certifies_mutation + exit 0 +fi + +if [ "${FM_TEST_FOCUSED:-}" = review-round-11 ]; then + test_javascript_runner_policy_is_declared_once + test_real_vitest_body_probe_certifies_mutation + exit 0 +fi + +if [ "${FM_TEST_FOCUSED:-}" = review-round-12 ]; then + test_javascript_runner_policy_is_declared_once + test_real_vitest_body_probe_certifies_mutation + exit 0 +fi + test_launcher_requires_supported_python test_reviewer_policy_profiles_and_independence test_same_model_relaxation_requires_proven_separate_account @@ -5088,14 +6974,11 @@ test_account_less_known_provider_lane_is_reviewable test_new_finding_requires_executed_reproduction test_silence_never_closes_prior_finding test_verified_fix_executes_mutation_proof -test_typescript_jest_mutation_proof_can_clear -test_preexisting_jest_runner_cannot_certify -test_local_fake_jest_package_cannot_certify -test_local_transitive_jest_package_cannot_certify -test_jest_runs_under_declared_node_major -test_inadequate_typescript_jest_coverage_stays_blocking -test_typescript_without_usable_route_is_cannot_certify -test_python_mutation_proof_is_byte_exact +test_javascript_runners_certify_platform_shaped_mutation_proofs +test_real_jest_certifies_platform_shaped_mutation_proof +test_real_vitest_body_probe_certifies_mutation +test_duplicate_javascript_outcome_names_are_nonexecution +test_javascript_non_executions_clear_nothing test_node_id_selector_clears_a_passing_named_test test_absent_runner_is_never_a_test_outcome test_unclassified_runner_cannot_clear_a_finding @@ -5140,6 +7023,7 @@ test_stopped_reviewer_and_wrong_head_are_unreviewed test_completed_reviewer_suspicion_is_blocking test_reading_only_suspicion_is_a_tool_failure test_pytest_runner_resolves_through_a_uv_aware_ladder +test_javascript_runner_policy_is_declared_once test_claude_execution_home_always_binds_the_keychain test_moved_default_branch_stays_reviewable test_unavailable_reviewer_fails_over_to_the_next_account diff --git a/tests/test-capabilities.tsv b/tests/test-capabilities.tsv index 066ac1c62d5..76d48cc51dd 100644 --- a/tests/test-capabilities.tsv +++ b/tests/test-capabilities.tsv @@ -34,6 +34,7 @@ fm-bounded-io.test.sh hermetic fm-brief.test.sh hermetic fm-captain-item-check.test.sh hermetic fm-cd-pretool-check.test.sh hermetic +fm-checkout-identity-cost.test.sh hermetic fm-checkout-refresh.test.sh hermetic fm-checkout-return-boundary.test.sh hermetic fm-composer-ghost.test.sh hermetic