extractor_eval_run.py

"""Extractor eval runner (task 16) — the fact-9 gate for any change to new_brain_extract.py.

Fact 9 (RULE): a fresh eval set must be curated before any extractor change is gated. The old
new_brain_eval.py set is SPENT (it already informed the index-page gate — see new_brain_extract.py).
This is the fresh, mechanically-scored replacement.

The cases in extractor_eval.jsonl are SYNTHETIC (abort criterion: never quote Aaron's real
captures). Each targets one of the extractor's documented failure classes:
  - control   : a clean single-claim sentence -> exactly one fact (the fair-chance baseline)
  - inventory : a multi-item sentence -> ONE fact, NOT N fragments (the shredding class)
  - negation  : polarity/direction must survive (the fact-74 inversion class)
  - identity  : an attribution/value must survive uncorrupted (the fact-161 mangle class)
  - duplicate : the same claim stated twice -> ONE fact, deduplicated (not two near-duplicates)

Scoring is fully MECHANICAL and deterministic (the extractor runs qwen at temperature 0):
  - a fact is "about the subject" if its statement contains any of the case's `subject` tokens
  - a subject fact is CORRECT if it contains every `good` token, satisfies `good_any` (>=1 group
    fully present), and contains no `bad` token
  - a subject fact is CORRUPTED if it contains a `bad` token (wrong polarity/value) or fails
    `good_any` (dropped the required direction) -- this is how an inversion/mangle is caught
All token checks are lowercase substring tests on the paraphrased statement.

Metrics (headline):
  precision      = clean-emission rate = 1 - (spurious / total emitted), where spurious = shred/dup
                   fragments beyond the expected count + corrupted (inverted/mangled) subject facts.
                   Legit extra facts (an identity case's distractor fact) are NOT spurious.
  recall         = cases with >=1 correct subject fact / cases expecting a fact
  shred_rate     = inventory cases emitting more than one fact / inventory cases
  inversion_rate = negation+identity cases with a corrupted subject fact / those cases
  dup_rate       = duplicate cases emitting more than one fact / duplicate cases

THRESHOLDS below are the PRE-REGISTERED bar (fair-chance rule): declared here from the design
intent BEFORE task 17 touches the extractor, NOT fitted to whatever the baseline turns out to be.

  --baseline : run the CURRENT extractor, write extractor_eval_baseline.json, ALWAYS exit 0.
               Baseline is MEASUREMENT, not a pass requirement (task 16's job).
  (no flag)  : gate mode (task 17's job) — run, then require every threshold met AND shred_rate
               and inversion_rate STRICTLY improved over the committed baseline. Nonzero exit on
               any failure.

Run:  .venv/Scripts/python.exe extractor_eval_run.py --baseline      (no DB needed, Ollama only)
"""

from __future__ import annotations

import json
import os
import sys

from new_brain_extract import extract_facts

HERE = os.path.dirname(os.path.abspath(__file__))
EVAL_FILE = os.path.join(HERE, "extractor_eval.jsonl")
BASELINE_FILE = os.path.join(HERE, "extractor_eval_baseline.json")

# ---- PRE-REGISTERED BAR (set from design intent, before the fix; do not tune to the baseline) --
THRESHOLDS = {
    "precision_min": 0.80,
    "recall_min": 0.85,
    "shred_rate_max": 0.15,
    "inversion_rate_max": 0.05,   # a minted lie is the worst failure — near-zero bar
    "dup_rate_max": 0.20,
}


def _lc(s: str) -> str:
    return s.lower()


def _load_cases() -> list[dict]:
    cases = []
    with open(EVAL_FILE, encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if line:
                cases.append(json.loads(line))
    return cases


def _subject_facts(stmts: list[str], case: dict) -> list[str]:
    subj = [_lc(s) for s in case.get("subject", [])]
    return [s for s in stmts if any(t in s for t in subj)] if subj else list(stmts)


def _good_any_ok(stmt: str, case: dict) -> bool:
    groups = case.get("good_any")
    return (not groups) or any(all(_lc(t) in stmt for t in group) for group in groups)


def _is_correct(stmt: str, case: dict) -> bool:
    if not all(_lc(g) in stmt for g in case.get("good", [])):
        return False
    if not _good_any_ok(stmt, case):
        return False
    # For NEGATION, polarity is judged by good_any (the correct marker), NOT by bad substrings:
    # a `bad` token like "trusts client" is a substring of the CORRECT "no longer trusts client",
    # so bad-matching would false-fail a right answer. For identity/value cases a bad token is a
    # genuine wrong-value marker and disqualifies.
    if case["cls"] != "negation" and any(_lc(b) in stmt for b in case.get("bad", [])):
        return False
    return True


def _is_corrupted(stmt: str, case: dict) -> bool:
    """A subject fact that got the direction/value WRONG. Class-aware so the check does not
    false-fire on a correct answer:
      - `exempt` tokens mark the distractor's OWN (correct) fact — never a corruption.
      - negation: corrupt iff the correct-polarity marker (good_any) is absent (an inverted claim).
      - identity/value: corrupt iff a wrong-value `bad` token is present AND the right value is not.
    """
    if any(_lc(x) in stmt for x in case.get("exempt", [])):
        return False
    if case["cls"] == "negation":
        return not _good_any_ok(stmt, case)
    has_bad = any(_lc(b) in stmt for b in case.get("bad", []))
    has_good = bool(case.get("good")) and all(_lc(g) in stmt for g in case["good"])
    return has_bad and not has_good


def score(cases: list[dict]) -> dict:
    per_case = []
    tot_emitted = tot_spurious = 0
    recall_hits = recall_denom = 0
    shred_num = shred_den = 0
    inv_num = inv_den = 0
    dup_num = dup_den = 0

    for case in cases:
        facts = extract_facts(case["text"])
        stmts = [_lc(f["statement"]) for f in facts]
        n = len(facts)
        max_facts = case.get("max_facts", 1)

        subj_facts = _subject_facts(stmts, case)
        correct = [s for s in subj_facts if _is_correct(s, case)]
        corrupted_facts = [s for s in subj_facts if _is_corrupted(s, case)]
        corrupted = bool(corrupted_facts)
        over = n > max_facts

        # PRECISION as a clean-emission rate: a fact is "spurious" only if it is DEMONSTRABLY bad —
        # a shred/dup fragment beyond the expected count, or a corrupted (inverted/mangled) subject
        # fact. Legitimate extra facts (e.g. the distractor's own correct fact in an identity case,
        # counted within max_facts) are NOT errors and must not drag precision. This makes precision
        # measure exactly the two failure modes, not the eval's two-entity test design.
        excess = max(0, n - max_facts) if case["cls"] in ("inventory", "duplicate") else 0
        tot_emitted += n
        tot_spurious += excess + len(corrupted_facts)

        if max_facts >= 1:
            recall_denom += 1
            if correct:
                recall_hits += 1

        if case["cls"] == "inventory":
            shred_den += 1
            shred_num += 1 if over else 0
        if case["cls"] == "duplicate":
            dup_den += 1
            dup_num += 1 if over else 0
        if case["cls"] in ("negation", "identity"):
            inv_den += 1
            inv_num += 1 if corrupted else 0

        per_case.append({
            "id": case["id"], "cls": case["cls"], "emitted": n, "max_facts": max_facts,
            "correct": len(correct), "corrupted": corrupted, "over": over,
            "statements": [f["statement"] for f in facts],
        })

    def rate(num, den):
        return round(num / den, 4) if den else 0.0

    metrics = {
        "precision": round(1 - tot_spurious / tot_emitted, 4) if tot_emitted else 0.0,
        "recall": rate(recall_hits, recall_denom),
        "shred_rate": rate(shred_num, shred_den),
        "inversion_rate": rate(inv_num, inv_den),
        "dup_rate": rate(dup_num, dup_den),
        "counts": {
            "cases": len(cases), "total_emitted": tot_emitted, "spurious": tot_spurious,
            "recall_hits": recall_hits, "recall_denom": recall_denom,
            "shred": [shred_num, shred_den], "inversion": [inv_num, inv_den],
            "dup": [dup_num, dup_den],
        },
    }
    return {"metrics": metrics, "per_case": per_case}


def _print_metrics(m: dict) -> None:
    print("  precision      = {:.3f}  (bar >= {})".format(m["precision"], THRESHOLDS["precision_min"]), file=sys.stderr)
    print("  recall         = {:.3f}  (bar >= {})".format(m["recall"], THRESHOLDS["recall_min"]), file=sys.stderr)
    print("  shred_rate     = {:.3f}  (bar <= {})".format(m["shred_rate"], THRESHOLDS["shred_rate_max"]), file=sys.stderr)
    print("  inversion_rate = {:.3f}  (bar <= {})".format(m["inversion_rate"], THRESHOLDS["inversion_rate_max"]), file=sys.stderr)
    print("  dup_rate       = {:.3f}  (bar <= {})".format(m["dup_rate"], THRESHOLDS["dup_rate_max"]), file=sys.stderr)


def main() -> int:
    baseline_mode = "--baseline" in sys.argv[1:]
    cases = _load_cases()
    print(f"running {len(cases)} synthetic eval cases through the current extractor...", file=sys.stderr)
    result = score(cases)
    m = result["metrics"]

    print("\n=== extractor eval ===", file=sys.stderr)
    _print_metrics(m)

    if baseline_mode:
        with open(BASELINE_FILE, "w", encoding="utf-8") as f:
            json.dump({"mode": "baseline", "thresholds": THRESHOLDS, **result}, f, indent=2)
        print(f"\nBASELINE written to {os.path.basename(BASELINE_FILE)} "
              f"(measurement only — not a pass requirement).", file=sys.stderr)
        return 0

    # gate mode (task 17): every threshold met AND no regression vs the committed baseline on the
    # two headline metrics. Meeting the shred/inversion thresholds (0.15 / 0.05) from a baseline of
    # 0.50 / 0.00 IS the improvement — a metric already at the floor (inversion 0.00) is held at
    # "no regression", not required to improve past what's arithmetically possible.
    if not os.path.exists(BASELINE_FILE):
        print("\nGATE FAIL: no baseline — run --baseline first (fact 9).", file=sys.stderr)
        return 2
    baseline = json.load(open(BASELINE_FILE, encoding="utf-8"))["metrics"]
    fails = []
    if m["precision"] < THRESHOLDS["precision_min"]:
        fails.append(f"precision {m['precision']} < {THRESHOLDS['precision_min']}")
    if m["recall"] < THRESHOLDS["recall_min"]:
        fails.append(f"recall {m['recall']} < {THRESHOLDS['recall_min']}")
    if m["shred_rate"] > THRESHOLDS["shred_rate_max"]:
        fails.append(f"shred_rate {m['shred_rate']} > {THRESHOLDS['shred_rate_max']}")
    if m["inversion_rate"] > THRESHOLDS["inversion_rate_max"]:
        fails.append(f"inversion_rate {m['inversion_rate']} > {THRESHOLDS['inversion_rate_max']}")
    if m["dup_rate"] > THRESHOLDS["dup_rate_max"]:
        fails.append(f"dup_rate {m['dup_rate']} > {THRESHOLDS['dup_rate_max']}")
    if m["shred_rate"] > baseline["shred_rate"]:
        fails.append(f"REGRESSION: shred_rate {m['shred_rate']} > baseline {baseline['shred_rate']}")
    if m["inversion_rate"] > baseline["inversion_rate"]:
        fails.append(f"REGRESSION: inversion_rate {m['inversion_rate']} > baseline {baseline['inversion_rate']}")

    if fails:
        print("\nGATE FAIL:", file=sys.stderr)
        for x in fails:
            print("  -", x, file=sys.stderr)
        return 1
    print("\nGATE PASS: thresholds met, no regression on shred/inversion vs baseline.", file=sys.stderr)
    return 0


if __name__ == "__main__":
    raise SystemExit(main())