extractor_eval_baseline.json

{
  "mode": "baseline",
  "thresholds": {
    "precision_min": 0.8,
    "recall_min": 0.85,
    "shred_rate_max": 0.15,
    "inversion_rate_max": 0.05,
    "dup_rate_max": 0.2
  },
  "metrics": {
    "precision": 0.75,
    "recall": 0.85,
    "shred_rate": 0.5,
    "inversion_rate": 0.0,
    "dup_rate": 0.0,
    "counts": {
      "cases": 40,
      "total_emitted": 56,
      "spurious": 14,
      "recall_hits": 34,
      "recall_denom": 40,
      "shred": [
        5,
        10
      ],
      "inversion": [
        0,
        15
      ],
      "dup": [
        0,
        5
      ]
    }
  },
  "per_case": [
    {
      "id": "ctrl-01",
      "cls": "control",
      "emitted": 1,
      "max_facts": 1,
      "correct": 1,
      "corrupted": false,
      "over": false,
      "statements": [
        "The render cache will live in Redis."
      ]
    },
    {
      "id": "ctrl-02",
      "cls": "control",
      "emitted": 1,
      "max_facts": 1,
      "correct": 1,
      "corrupted": false,
      "over": false,
      "statements": [
        "Aaron uses an RTX 4090 GPU on his primary workstation."
      ]
    },
    {
      "id": "ctrl-03",
      "cls": "control",
      "emitted": 1,
      "max_facts": 1,
      "correct": 1,
      "corrupted": false,
      "over": false,
      "statements": [
        "Aaron prefers spaces over tabs for indentation."
      ]
    },
    {
      "id": "ctrl-04",
      "cls": "control",
      "emitted": 1,
      "max_facts": 1,
      "correct": 1,
      "corrupted": false,
      "over": false,
      "statements": [
        "The staging server web-3 is currently green."
      ]
    },
    {
      "id": "ctrl-05",
      "cls": "control",
      "emitted": 1,
      "max_facts": 1,
      "correct": 1,
      "corrupted": false,
      "over": false,
      "statements": [
        "API keys are stored at C:\\config\\keys.env."
      ]
    },
    {
      "id": "ctrl-06",
      "cls": "control",
      "emitted": 1,
      "max_facts": 1,
      "correct": 1,
      "corrupted": false,
      "over": false,
      "statements": [
        "Never force-push to the shared main branch."
      ]
    },
    {
      "id": "ctrl-07",
      "cls": "control",
      "emitted": 1,
      "max_facts": 1,
      "correct": 1,
      "corrupted": false,
      "over": false,
      "statements": [
        "The team shipped the v2 launcher on 2026-06-01."
      ]
    },
    {
      "id": "ctrl-08",
      "cls": "control",
      "emitted": 1,
      "max_facts": 1,
      "correct": 1,
      "corrupted": false,
      "over": false,
      "statements": [
        "Aaron parked the mobile port."
      ]
    },
    {
      "id": "ctrl-09",
      "cls": "control",
      "emitted": 1,
      "max_facts": 1,
      "correct": 1,
      "corrupted": false,
      "over": false,
      "statements": [
        "Aaron's project timezone is US Eastern."
      ]
    },
    {
      "id": "ctrl-10",
      "cls": "control",
      "emitted": 1,
      "max_facts": 1,
      "correct": 1,
      "corrupted": false,
      "over": false,
      "statements": [
        "The design doc for the vault system lives at docs/vault.md."
      ]
    },
    {
      "id": "inv-01",
      "cls": "inventory",
      "emitted": 0,
      "max_facts": 1,
      "correct": 0,
      "corrupted": false,
      "over": false,
      "statements": []
    },
    {
      "id": "inv-02",
      "cls": "inventory",
      "emitted": 3,
      "max_facts": 1,
      "correct": 3,
      "corrupted": false,
      "over": true,
      "statements": [
        "All L3 answers route to the memory store through remember().",
        "All L2 external acquisitions route to the memory store through remember().",
        "All personal-domain facts route to the memory store via remember()."
      ]
    },
    {
      "id": "inv-03",
      "cls": "inventory",
      "emitted": 5,
      "max_facts": 1,
      "correct": 5,
      "corrupted": false,
      "over": true,
      "statements": [
        "The world module contains every planet.",
        "Tatooine is in the world module.",
        "Naboo is in the world module.",
        "Corellia is in the world module.",
        "Dantooine is in the world module."
      ]
    },
    {
      "id": "inv-04",
      "cls": "inventory",
      "emitted": 4,
      "max_facts": 1,
      "correct": 4,
      "corrupted": false,
      "over": true,
      "statements": [
        "The vault component stores the player's credits.",
        "The vault component stores inventory items.",
        "The vault component stores faction standing.",
        "The vault component stores the current quest log."
      ]
    },
    {
      "id": "inv-05",
      "cls": "inventory",
      "emitted": 0,
      "max_facts": 1,
      "correct": 0,
      "corrupted": false,
      "over": false,
      "statements": []
    },
    {
      "id": "inv-06",
      "cls": "inventory",
      "emitted": 1,
      "max_facts": 1,
      "correct": 1,
      "corrupted": false,
      "over": false,
      "statements": [
        "The crafting system supports weapons, armor, consumables, and decorations."
      ]
    },
    {
      "id": "inv-07",
      "cls": "inventory",
      "emitted": 5,
      "max_facts": 1,
      "correct": 5,
      "corrupted": false,
      "over": true,
      "statements": [
        "The onboarding flow is documented.",
        "Step one of the onboarding process is signup.",
        "Step two of the onboarding process is email verification.",
        "Step three of the onboarding process is profile setup.",
        "Step four of the onboarding process is the tutorial."
      ]
    },
    {
      "id": "inv-08",
      "cls": "inventory",
      "emitted": 2,
      "max_facts": 1,
      "correct": 0,
      "corrupted": false,
      "over": true,
      "statements": [
        "It tracks health.",
        "It tracks stamina."
      ]
    },
    {
      "id": "inv-09",
      "cls": "inventory",
      "emitted": 0,
      "max_facts": 1,
      "correct": 0,
      "corrupted": false,
      "over": false,
      "statements": []
    },
    {
      "id": "inv-10",
      "cls": "inventory",
      "emitted": 1,
      "max_facts": 1,
      "correct": 1,
      "corrupted": false,
      "over": false,
      "statements": [
        "The asset exporter supports glTF, FBX, OBJ, and USD formats."
      ]
    },
    {
      "id": "neg-01",
      "cls": "negation",
      "emitted": 1,
      "max_facts": 2,
      "correct": 1,
      "corrupted": false,
      "over": false,
      "statements": [
        "The orchestrator builds the handoff prompt itself and dispatches it."
      ]
    },
    {
      "id": "neg-02",
      "cls": "negation",
      "emitted": 1,
      "max_facts": 2,
      "correct": 1,
      "corrupted": false,
      "over": false,
      "statements": [
        "The vault autosave feature is enabled by default."
      ]
    },
    {
      "id": "neg-03",
      "cls": "negation",
      "emitted": 1,
      "max_facts": 2,
      "correct": 1,
      "corrupted": false,
      "over": false,
      "statements": [
        "Fall damage is disabled below five meters."
      ]
    },
    {
      "id": "neg-04",
      "cls": "negation",
      "emitted": 0,
      "max_facts": 2,
      "correct": 0,
      "corrupted": false,
      "over": false,
      "statements": []
    },
    {
      "id": "neg-05",
      "cls": "negation",
      "emitted": 1,
      "max_facts": 2,
      "correct": 1,
      "corrupted": false,
      "over": false,
      "statements": [
        "The server no longer trusts client-side hit detection."
      ]
    },
    {
      "id": "neg-06",
      "cls": "negation",
      "emitted": 1,
      "max_facts": 2,
      "correct": 1,
      "corrupted": false,
      "over": false,
      "statements": [
        "The debug overlay is not shown in release builds."
      ]
    },
    {
      "id": "neg-07",
      "cls": "negation",
      "emitted": 1,
      "max_facts": 2,
      "correct": 1,
      "corrupted": false,
      "over": false,
      "statements": [
        "The tutorial is disabled for returning players."
      ]
    },
    {
      "id": "neg-08",
      "cls": "negation",
      "emitted": 1,
      "max_facts": 2,
      "correct": 1,
      "corrupted": false,
      "over": false,
      "statements": [
        "Weapon damage does not increase with distance; it falls off the farther the target is."
      ]
    },
    {
      "id": "id-01",
      "cls": "identity",
      "emitted": 2,
      "max_facts": 2,
      "correct": 1,
      "corrupted": false,
      "over": false,
      "statements": [
        "Aaron's collaborator MickyB has the GitHub user mdbridges1.",
        "Aaron's own GitHub handle is Laythros."
      ]
    },
    {
      "id": "id-02",
      "cls": "identity",
      "emitted": 2,
      "max_facts": 2,
      "correct": 1,
      "corrupted": false,
      "over": false,
      "statements": [
        "The staging database is named rift_dev.",
        "The production database is named rift_prod."
      ]
    },
    {
      "id": "id-03",
      "cls": "identity",
      "emitted": 2,
      "max_facts": 2,
      "correct": 1,
      "corrupted": false,
      "over": false,
      "statements": [
        "Riftforge Studios is Aaron's studio.",
        "Aporia Labs is the publisher they work with."
      ]
    },
    {
      "id": "id-04",
      "cls": "identity",
      "emitted": 2,
      "max_facts": 2,
      "correct": 1,
      "corrupted": false,
      "over": false,
      "statements": [
        "The public API endpoint is api.riftreach.io.",
        "The internal admin API endpoint is admin.riftreach.io."
      ]
    },
    {
      "id": "id-05",
      "cls": "identity",
      "emitted": 2,
      "max_facts": 2,
      "correct": 1,
      "corrupted": false,
      "over": false,
      "statements": [
        "Aaron's personal email is dev@riftforge.io.",
        "The support alias is help@riftforge.io."
      ]
    },
    {
      "id": "id-06",
      "cls": "identity",
      "emitted": 2,
      "max_facts": 2,
      "correct": 1,
      "corrupted": false,
      "over": false,
      "statements": [
        "Priya is the lead engineer on RiftReach.",
        "Mara is the lead artist on RiftReach."
      ]
    },
    {
      "id": "id-07",
      "cls": "identity",
      "emitted": 2,
      "max_facts": 2,
      "correct": 1,
      "corrupted": false,
      "over": false,
      "statements": [
        "The default branch of this repo is trunk.",
        "The release branch of this repo is stable."
      ]
    },
    {
      "id": "dup-01",
      "cls": "duplicate",
      "emitted": 1,
      "max_facts": 1,
      "correct": 1,
      "corrupted": false,
      "over": false,
      "statements": [
        "The main build is currently passing all checks."
      ]
    },
    {
      "id": "dup-02",
      "cls": "duplicate",
      "emitted": 0,
      "max_facts": 1,
      "correct": 0,
      "corrupted": false,
      "over": false,
      "statements": []
    },
    {
      "id": "dup-03",
      "cls": "duplicate",
      "emitted": 1,
      "max_facts": 1,
      "correct": 1,
      "corrupted": false,
      "over": false,
      "statements": [
        "The vault uses optimistic locking."
      ]
    },
    {
      "id": "dup-04",
      "cls": "duplicate",
      "emitted": 1,
      "max_facts": 1,
      "correct": 1,
      "corrupted": false,
      "over": false,
      "statements": [
        "Releases must go through the staging gate before production."
      ]
    },
    {
      "id": "dup-05",
      "cls": "duplicate",
      "emitted": 1,
      "max_facts": 1,
      "correct": 1,
      "corrupted": false,
      "over": false,
      "statements": [
        "The API caps at 100 requests per minute."
      ]
    }
  ]
}