refactor(result): centralize level-failure policy (#88)

Signed-off-by: Loek <solloek369@gmail.com>

Author: solloek369-arch

src/trace_tests/cli.py

@@ -3,8 +3,8 @@
 from __future__ import annotations
 
 import datetime as _dt
-import json
 import importlib.metadata
+import json
 import pathlib
 import re
 import sys
@@ -17,7 +17,7 @@
 from trace_tests import report as report_mod
 from trace_tests.loader import LoadError, load_record
 from trace_tests.modules.tr_env import DEFAULT_MAX_AGE_SECONDS
-from trace_tests.modules.unverified import unverified_fails
+from trace_tests.modules.unverified import finding_counts_as_level_failure
 from trace_tests.result import Status
 from trace_tests.runner import run
 
@@ -50,30 +50,22 @@ def _print_report(path: str, fmt: str, level: int, results: dict[str, list[Any]]
     skips = 0
     passes = 0
     unverified = 0
-    unverified_failing = 0
-
     for module, findings in results.items():
         for f in findings:
             prefix = _fmt_status(f.status)
             click.echo(f"  {module}  {prefix}  {f.message}")
-            if f.failed():
-                failures += 1
-            elif f.passed():
+            counts_as_failure = finding_counts_as_level_failure(f, level)
+            failures += int(counts_as_failure)
+            if f.passed():
                 passes += 1
             elif f.unverified():
                 unverified += 1
-                if unverified_fails(f.code, level):
-                    unverified_failing += 1
-            else:
+            elif f.skipped():
                 skips += 1
 
-    # Defense in depth: an unverified finding must fail the run from the level
-    # its code is registered at, even if a module forgot to emit a hard FAIL.
-    # The level is per-code rather than blanket; an unregistered code fails
-    # from level 1, which is what the blanket rule did for all of them.
-    failures += unverified_failing
-
-    total = passes + failures + skips + (unverified - unverified_failing)
+    # Contribution is projected exactly once per finding above. Status counters
+    # remain presentation data and must not independently decide the verdict.
+    total = sum(len(findings) for findings in results.values())
     click.echo("")
     if failures == 0:
         if unverified:

src/trace_tests/modules/unverified.py

@@ -22,6 +22,8 @@
 
 from __future__ import annotations
 
+from trace_tests.result import Finding
+
 #: Lowest conformance level at which an unverified finding under this code
 #: counts as a failure. Registered here and in ``docs/levels.md``; the two are
 #: held equal by ``tests/test_docs_match_the_modules.py``.
@@ -38,3 +40,12 @@
 def unverified_fails(code: str, level: int) -> bool:
     """Return True when an unverified finding under *code* must fail at *level*."""
     return level >= UNVERIFIED_FAILS_FROM_LEVEL.get(code, DEFAULT_FAILS_FROM_LEVEL)
+
+
+def finding_counts_as_level_failure(finding: Finding, level: int) -> bool:
+    """Return whether one finding contributes failure at one level."""
+    if finding.failed():
+        return True
+    if finding.unverified():
+        return unverified_fails(finding.code, level)
+    return False

src/trace_tests/report.py

@@ -25,7 +25,7 @@
 from dataclasses import dataclass
 from typing import Any
 
-from trace_tests.modules.unverified import unverified_fails
+from trace_tests.modules.unverified import finding_counts_as_level_failure
 from trace_tests.result import Finding, Status
 
 __all__ = [
@@ -107,13 +107,17 @@ def verdict(self) -> str:
 
 
 def _tally(results: dict[str, list[Finding]], level: int) -> tuple[int, int]:
-    failures = sum(1 for fs in results.values() for f in fs if f.failed())
+    failures = sum(
+        1
+        for findings in results.values()
+        for finding in findings
+        if finding_counts_as_level_failure(finding, level)
+    )
     unverified_findings = [f for fs in results.values() for f in fs if f.unverified()]
     # Mirrors the CLI: an unverified finding is a failure from the level its code
     # is registered at. A report that called such a record "PASS" at a level that
     # required the check would be worse than no report. Per-code rather than
     # blanket; an unregistered code fails from level 1, as the blanket rule did.
-    failures += sum(1 for f in unverified_findings if unverified_fails(f.code, level))
     return failures, len(unverified_findings)
 
 

tests/test_level_failure_policy.py

@@ -0,0 +1,65 @@
+"""CLI and portable reports share one level-failure projection."""
+
+from __future__ import annotations
+
+import pytest
+
+from trace_tests import cli as cli_module
+from trace_tests import report as report_module
+from trace_tests.cli import _print_report
+from trace_tests.modules.unverified import finding_counts_as_level_failure
+from trace_tests.report import _tally
+from trace_tests.result import Finding, Status
+
+
+@pytest.mark.parametrize("level", [0, 1, 2])
+@pytest.mark.parametrize("status", list(Status))
+def test_cli_report_and_policy_agree_for_every_status_and_level(
+    status: Status,
+    level: int,
+) -> None:
+    finding = Finding("TR-NEW-001", status, "synthetic")
+    expected = finding_counts_as_level_failure(finding, level)
+    results = {"TR-X": [finding]}
+
+    report_failures, _ = _tally(results, level)
+    cli_exit = _print_report("record.json", "trace", level, results)
+
+    assert bool(report_failures) is expected
+    assert bool(cli_exit) is expected
+
+
+@pytest.mark.parametrize(
+    ("status", "forced_contribution"),
+    [
+        (Status.PASS, True),
+        (Status.FAIL, False),
+        (Status.SKIP, True),
+        (Status.UNVERIFIED, False),
+    ],
+)
+def test_each_projection_delegates_once_even_for_wrong_branch_differentials(
+    monkeypatch: pytest.MonkeyPatch,
+    status: Status,
+    forced_contribution: bool,
+) -> None:
+    finding = Finding("TR-X-001", status, "synthetic")
+    results = {"TR-X": [finding]}
+    cli_calls: list[tuple[str, int]] = []
+    report_calls: list[tuple[str, int]] = []
+
+    def cli_policy(item: Finding, level: int) -> bool:
+        cli_calls.append((item.code, level))
+        return forced_contribution
+
+    def report_policy(item: Finding, level: int) -> bool:
+        report_calls.append((item.code, level))
+        return forced_contribution
+
+    monkeypatch.setattr(cli_module, "finding_counts_as_level_failure", cli_policy)
+    monkeypatch.setattr(report_module, "finding_counts_as_level_failure", report_policy)
+
+    assert bool(_print_report("record.json", "trace", 2, results)) is forced_contribution
+    assert bool(_tally(results, 2)[0]) is forced_contribution
+    assert cli_calls == [("TR-X-001", 2)]
+    assert report_calls == [("TR-X-001", 2)]