From aafbcf14c4d6702e20ce6c718304d7cda2f9f92e Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 22 Jun 2026 23:31:15 +0000 Subject: [PATCH] =?UTF-8?q?feat(eval):=20conformal=20judge=20intervals=20?= =?UTF-8?q?=E2=80=94=20distribution-free=20uncertainty=20bounds=20(v0.29.0?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New kairu/conformal.py adds split conformal prediction (Sheng et al., EMNLP 2025, arXiv:2509.18658) to the judge-eval stack. The reliability module ships only distribution-parametric estimators (Cronbach alpha, ICC, Fleiss kappa); conformal prediction adds a distribution-free coverage guarantee that holds regardless of the judge's score distribution. - conformal_quantile(residuals, alpha): finite-sample-corrected (1-alpha) quantile via conformal rank ceil((n+1)(1-alpha)); returns +inf (no finite bound) when the calibration set is too small rather than fabricating a number. - calibrate_interval -> ConformalInterval: symmetric interval with the paper's ordinal boundary clamp + lower-bias midpoint point estimate. - conformal_from_ensemble: bridges EnsembleResult.median_aggregate. reliability.py and EnsembleResult are untouched -- purely additive. New tests/test_conformal.py (17 tests, module 100%) including an empirical coverage test (~90% of held-out references fall within 90% intervals on synthetic exchangeable data). Suite: 763 passed, 4 HF-gated skipped. Version 0.28.0 -> 0.29.0. First sprint of a new Discovery cycle. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01TmSqgYqcYfzPnkpDVR5vdw --- CHANGELOG.md | 44 +++++++++ PLAN.md | 17 +++- kairu/__init__.py | 17 +++- kairu/conformal.py | 201 ++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 2 +- tests/test_conformal.py | 158 +++++++++++++++++++++++++++++++ 6 files changed, 436 insertions(+), 3 deletions(-) create mode 100644 kairu/conformal.py create mode 100644 tests/test_conformal.py diff --git a/CHANGELOG.md b/CHANGELOG.md index bc12bff..a3eb0dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,50 @@ All notable changes to Kairu follow [Conventional Commits](https://www.conventio --- +## [0.29.0] — 2026-06-22 + +### Added — Conformal judge intervals: distribution-free uncertainty bounds + +**New module `kairu/conformal.py`** (pure stdlib + `kairu.ensemble`). The +reliability module ships Cronbach α / ICC(2,1) / Fleiss κ — all retrospective, +distribution-*parametric* estimators. Split conformal prediction (Vovk; applied +to LLM judges by Sheng et al., EMNLP 2025, arXiv:2509.18658) adds a +**distribution-free** coverage guarantee that holds regardless of the judge's +score distribution: + +- `conformal_quantile(residuals, alpha)` — the finite-sample-corrected + `(1 - alpha)` quantile of the absolute calibration residuals, using the + conformal rank `ceil((n + 1)(1 - alpha))`. Returns `+inf` (no finite bound) + when the calibration set is too small for the requested `alpha` — the honest + answer rather than a fabricated number. +- `calibrate_interval(prediction, calibration_pairs, *, alpha, score_range)` → + `ConformalInterval` — wraps a judge score in `[score - q, score + q]`, with + the paper's two judge-specific refinements: **ordinal boundary adjustment** + (clamp to the valid score range) and the lower-bias **midpoint** point + estimate (the midpoint of the clamped interval, pulled inside the range near a + boundary). For any exchangeable new pair the interval covers the reference + with probability ≥ `1 - alpha`. +- `conformal_from_ensemble(result, calibration_pairs, ...)` — bridges an + `EnsembleResult` (uses its `median_aggregate` as the prediction). The existing + `reliability.py` and `EnsembleResult` are untouched — this is purely additive. +- `ConformalInterval` (frozen) carries `lower`/`upper`/`midpoint`/ + `coverage_level`/`half_width`/`n_calibration` with `width()`, `contains()`, + and `to_dict()`. + +**`kairu/__init__.py`** exports `ConformalInterval`, `conformal_quantile`, +`calibrate_interval`, `conformal_from_ensemble`, `CONFORMAL_DEFAULT_ALPHA`, +`CONFORMAL_DEFAULT_SCORE_RANGE`; version `0.28.0 → 0.29.0`. + +**Tests:** new `tests/test_conformal.py` (17 tests, module 100%) — including an +**empirical coverage test** that verifies ~90% of held-out references fall +within 90%-level intervals on synthetic exchangeable data. Suite: **763 +passed**, 4 HF-gated skipped. + +> First sprint of a new Discovery cycle (the v0.24–v0.28 sweep having shipped). +> Grounded in a fresh arXiv survey; rebalances toward the judge-eval track. + +--- + ## [0.28.0] — 2026-06-22 ### Added — SPEED-Bench semantic task splits + speculative spec/quant warnings diff --git a/PLAN.md b/PLAN.md index d096106..dbf337b 100644 --- a/PLAN.md +++ b/PLAN.md @@ -2,7 +2,7 @@ > 流 · *to flow, to stream* -Current version: **v0.28.0** +Current version: **v0.29.0** --- @@ -81,6 +81,21 @@ has shipped in v0.15.0 — the four `🔴` rows below are now **DONE**. error recovery, goal progress/completion, efficiency (steps taken vs. optimal). Returns a per-step breakdown plus an overall trajectory grade. +### ✅ v0.29.0 — Conformal judge intervals: distribution-free uncertainty bounds *(DONE)* + +- **`kairu/conformal.py`** — split conformal prediction (Sheng et al., EMNLP + 2025, arXiv:2509.18658) adds a distribution-free coverage guarantee to the + judge-eval stack, complementing the distribution-parametric reliability + metrics. `conformal_quantile` (finite-sample rank, `+inf` when undersized), + `calibrate_interval` → `ConformalInterval` (ordinal boundary clamp + lower-bias + midpoint), `conformal_from_ensemble` bridge. `reliability.py` / `EnsembleResult` + untouched — purely additive. +- 17 new tests incl. an empirical-coverage check; module 100% coverage. +- **First sprint of a new Discovery cycle** (researcher sweep surfaced it as + High-impact/Low-complexity; rebalances toward eval after two inference + sprints). Backlog from the same sweep: IRT judge discrimination, entropy-driven + adaptive γ, dark-current datasheet, radix-tree KV dedup. + ### ✅ v0.28.0 — SPEED-Bench task splits + speculative spec/quant warnings *(DONE)* - **`kairu/speed_bench.py`** — `run_speed_bench` → `SpeedBenchReport` runs the diff --git a/kairu/__init__.py b/kairu/__init__.py index a8345fa..55aa700 100644 --- a/kairu/__init__.py +++ b/kairu/__init__.py @@ -28,7 +28,7 @@ from __future__ import annotations -__version__ = "0.28.0" +__version__ = "0.29.0" from kairu.adversarial import ( AdversarialMatch, @@ -91,6 +91,14 @@ intraclass_correlation, reliability_from_ensemble, ) +from kairu.conformal import ( + ConformalInterval, + DEFAULT_ALPHA as CONFORMAL_DEFAULT_ALPHA, + DEFAULT_SCORE_RANGE as CONFORMAL_DEFAULT_SCORE_RANGE, + calibrate_interval, + conformal_from_ensemble, + conformal_quantile, +) from kairu.cyclic_judge import ( CyclicEvalReport, DEFAULT_CONFIDENCE, @@ -412,5 +420,12 @@ "fleiss_kappa", "compute_reliability", "reliability_from_ensemble", + # v0.29 — conformal judge intervals + "ConformalInterval", + "CONFORMAL_DEFAULT_ALPHA", + "CONFORMAL_DEFAULT_SCORE_RANGE", + "conformal_quantile", + "calibrate_interval", + "conformal_from_ensemble", "__version__", ] diff --git a/kairu/conformal.py b/kairu/conformal.py new file mode 100644 index 0000000..2dfaf8d --- /dev/null +++ b/kairu/conformal.py @@ -0,0 +1,201 @@ +"""Split conformal prediction intervals for LLM-judge scores. + +The reliability module (:mod:`kairu.reliability`) ships Cronbach's alpha, +ICC(2,1) and Fleiss' kappa — all *retrospective, distribution-parametric* +estimators that describe a judges × criteria matrix after the fact. None of +them answers the deployment question a practitioner actually asks: *given this +judge's score, what is a defensible interval around the true quality, with a +coverage guarantee that holds regardless of the judge's score distribution?* + +Split conformal prediction (Vovk; applied to LLM judges by Sheng et al., +EMNLP 2025, arXiv:2509.18658) answers exactly that. From a calibration set of +``(judge_score, reference_score)`` pairs it computes the nonconformity scores +(absolute residuals), takes their finite-sample-corrected ``(1 - alpha)`` +quantile ``q``, and emits the symmetric interval ``[score - q, score + q]``. +For any *exchangeable* new pair the interval covers the reference score with +probability at least ``1 - alpha`` — **distribution-free**, no assumption on +the judge's score distribution. + +Two judge-specific refinements from the paper: + +* **Ordinal boundary adjustment** — the interval is clamped to the valid score + range (``[0, 1]`` by default), so it never claims impossible scores. +* **Midpoint point-estimate** — the midpoint of the *clamped* interval is a + lower-bias estimate than the raw score near a boundary, where clamping pulls + the interval (and thus its midpoint) back inside the range. + +Pure stdlib + :mod:`kairu.ensemble` — no ML deps. The quantile follows kairu's +existing sorted-list / nearest-rank pattern. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import Any, Dict, Sequence, Tuple + +from kairu.ensemble import EnsembleResult + +# 90 % coverage by default (alpha = 0.1). +DEFAULT_ALPHA: float = 0.1 +# kairu scores live in [0, 1]; the interval is clamped to this range. +DEFAULT_SCORE_RANGE: Tuple[float, float] = (0.0, 1.0) + + +@dataclass(frozen=True) +class ConformalInterval: + """A distribution-free prediction interval around a judge score. + + Attributes + ---------- + lower, upper: + Interval endpoints, clamped to the score range. + midpoint: + Midpoint of the clamped interval — the lower-bias point estimate. + coverage_level: + The guaranteed marginal coverage ``1 - alpha``. + half_width: + The conformal quantile ``q`` actually applied (capped at the score-range + width when the calibration set is too small for a finite bound). + n_calibration: + Number of calibration pairs the quantile was computed from. + """ + + lower: float + upper: float + midpoint: float + coverage_level: float + half_width: float + n_calibration: int + + def width(self) -> float: + """Total width of the (clamped) interval.""" + return self.upper - self.lower + + def contains(self, value: float) -> bool: + """Whether ``value`` falls within the closed interval.""" + return self.lower <= value <= self.upper + + def to_dict(self) -> Dict[str, Any]: + """Return a JSON-serialisable view of the interval.""" + return { + "lower": self.lower, + "upper": self.upper, + "midpoint": self.midpoint, + "coverage_level": self.coverage_level, + "half_width": self.half_width, + "n_calibration": self.n_calibration, + } + + +def conformal_quantile( + residuals: Sequence[float], alpha: float = DEFAULT_ALPHA +) -> float: + """Finite-sample split-conformal ``(1 - alpha)`` quantile of ``|residuals|``. + + Uses the conformal rank ``ceil((n + 1) * (1 - alpha))``. When that rank + exceeds ``n`` the calibration set is too small to certify a finite bound at + this ``alpha``, and the honest answer is ``+inf`` (an interval that must + span the whole range); callers clamp it to their score range. + + Parameters + ---------- + residuals: + Calibration residuals; their absolute values are used. + alpha: + Miscoverage level in ``(0, 1)``. + """ + if not 0.0 < alpha < 1.0: + raise ValueError("alpha must be in (0, 1)") + n = len(residuals) + if n == 0: + raise ValueError("at least one calibration residual is required") + ordered = sorted(abs(r) for r in residuals) + rank = math.ceil((n + 1) * (1.0 - alpha)) + if rank > n: + return math.inf + return ordered[rank - 1] + + +def calibrate_interval( + prediction: float, + calibration_pairs: Sequence[Tuple[float, float]], + *, + alpha: float = DEFAULT_ALPHA, + score_range: Tuple[float, float] = DEFAULT_SCORE_RANGE, +) -> ConformalInterval: + """Build a split-conformal interval around ``prediction``. + + Parameters + ---------- + prediction: + The judge score to wrap in an interval. + calibration_pairs: + Historical ``(judge_score, reference_score)`` pairs; the residual of + each is ``judge_score - reference_score``. + alpha: + Miscoverage level — the interval guarantees ``1 - alpha`` coverage. + score_range: + ``(low, high)`` bounds the interval is clamped to (ordinal adjustment). + """ + if not calibration_pairs: + raise ValueError("at least one calibration pair is required") + low, high = score_range + if high <= low: + raise ValueError("score_range must have high > low") + + residuals = [judge - reference for judge, reference in calibration_pairs] + q = conformal_quantile(residuals, alpha) + # Cap at the range width so an unbounded quantile becomes the full range and + # the stored half_width stays finite / JSON-serialisable. + half_width = min(q, high - low) + + lower = max(low, prediction - half_width) + upper = min(high, prediction + half_width) + return ConformalInterval( + lower=lower, + upper=upper, + midpoint=(lower + upper) / 2.0, + coverage_level=1.0 - alpha, + half_width=half_width, + n_calibration=len(calibration_pairs), + ) + + +def conformal_from_ensemble( + result: EnsembleResult, + calibration_pairs: Sequence[Tuple[float, float]], + *, + alpha: float = DEFAULT_ALPHA, + score_range: Tuple[float, float] = DEFAULT_SCORE_RANGE, +) -> ConformalInterval: + """Wrap an ensemble's median aggregate in a conformal interval. + + Uses ``result.median_aggregate`` as the prediction; ``calibration_pairs`` + are prior ``(ensemble_aggregate, reference_score)`` observations. + + Parameters + ---------- + result: + The ensemble verdict to bound. + calibration_pairs: + Historical ``(aggregate, reference)`` pairs. + alpha, score_range: + As in :func:`calibrate_interval`. + """ + return calibrate_interval( + result.median_aggregate, + calibration_pairs, + alpha=alpha, + score_range=score_range, + ) + + +__all__ = [ + "DEFAULT_ALPHA", + "DEFAULT_SCORE_RANGE", + "ConformalInterval", + "conformal_quantile", + "calibrate_interval", + "conformal_from_ensemble", +] diff --git a/pyproject.toml b/pyproject.toml index 51f755d..0931eec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "kairu" -version = "0.28.0" +version = "0.29.0" description = "Real-time LLM inference optimizer: speculative decoding, layered early exit, KV-cache recycling, adaptive γ, SSE/JSONL streaming, Prometheus metrics, Redis-backed rate limiting, OpenTelemetry tracing, cluster token budgets, token watermarking, squish quantization-quality eval, rubric-based response evaluation + A/B comparison HTTP API, eight named rubrics + prism UI, prompt shield content policy, judge ensemble + CI regression + log-to-eval pipeline, constitutional rubric generation, agentic trajectory scoring, judge bias calibration + A-BB bias bounds" license = { text = "BUSL-1.1" } requires-python = ">=3.10" diff --git a/tests/test_conformal.py b/tests/test_conformal.py new file mode 100644 index 0000000..dc42df8 --- /dev/null +++ b/tests/test_conformal.py @@ -0,0 +1,158 @@ +"""Tests for kairu.conformal — split conformal intervals for judge scores.""" + +from __future__ import annotations + +import random + +import pytest + +from kairu.conformal import ( + ConformalInterval, + calibrate_interval, + conformal_from_ensemble, + conformal_quantile, +) +from kairu.ensemble import ensemble_evaluate, JudgeConfig + + +# --------------------------------------------------------------------------- # +# conformal_quantile +# --------------------------------------------------------------------------- # + + +def test_quantile_is_finite_sample_rank(): + # 9 residuals, alpha=0.1 → rank = ceil(10 * 0.9) = 9 → the 9th (largest). + residuals = [0.0, -0.1, 0.2, -0.3, 0.4, -0.5, 0.6, -0.7, 0.8] + assert conformal_quantile(residuals, alpha=0.1) == pytest.approx(0.8) + + +def test_quantile_uses_absolute_residuals(): + # Sign must not matter — only magnitude. + assert conformal_quantile([-0.5, 0.5, -0.2], alpha=0.5) == conformal_quantile( + [0.5, 0.5, 0.2], alpha=0.5 + ) + + +def test_quantile_infinite_when_calibration_too_small(): + # n=1, alpha=0.1 → rank = ceil(2 * 0.9) = 2 > 1 → no finite bound. + assert conformal_quantile([0.3], alpha=0.1) == float("inf") + + +def test_quantile_rejects_empty(): + with pytest.raises(ValueError): + conformal_quantile([], alpha=0.1) + + +@pytest.mark.parametrize("alpha", [0.0, 1.0, -0.1, 1.5]) +def test_quantile_rejects_bad_alpha(alpha): + with pytest.raises(ValueError): + conformal_quantile([0.1, 0.2], alpha=alpha) + + +# --------------------------------------------------------------------------- # +# calibrate_interval +# --------------------------------------------------------------------------- # + + +def _pairs(residual_mags): + """Calibration pairs whose judge−reference residuals have the given mags.""" + return [(0.5 + m, 0.5) for m in residual_mags] + + +def test_interval_is_symmetric_before_clamping(): + pairs = _pairs([0.05, 0.1, 0.15, 0.2]) # q at alpha=0.5 is a middle residual + iv = calibrate_interval(0.5, pairs, alpha=0.5) + assert iv.lower == pytest.approx(0.5 - iv.half_width) + assert iv.upper == pytest.approx(0.5 + iv.half_width) + assert iv.midpoint == pytest.approx(0.5) + + +def test_interval_clamped_to_score_range(): + pairs = _pairs([0.3, 0.3, 0.3, 0.3]) + iv = calibrate_interval(0.95, pairs, alpha=0.5) + assert iv.lower >= 0.0 + assert iv.upper <= 1.0 + # Near the top boundary the clamp pulls the midpoint back inside the range. + assert iv.midpoint < 0.95 + + +def test_width_shrinks_as_calibration_residuals_shrink(): + wide = calibrate_interval(0.5, _pairs([0.2, 0.3, 0.4, 0.5]), alpha=0.5) + tight = calibrate_interval(0.5, _pairs([0.01, 0.02, 0.03, 0.04]), alpha=0.5) + assert tight.width() < wide.width() + + +def test_single_calibration_pair_yields_full_range(): + iv = calibrate_interval(0.5, [(0.6, 0.5)], alpha=0.1) + assert iv.lower == 0.0 + assert iv.upper == 1.0 + assert iv.half_width == pytest.approx(1.0) # capped at range width + + +def test_interval_rejects_empty_calibration(): + with pytest.raises(ValueError): + calibrate_interval(0.5, [], alpha=0.1) + + +def test_interval_rejects_degenerate_range(): + with pytest.raises(ValueError): + calibrate_interval(0.5, [(0.6, 0.5)], score_range=(1.0, 1.0)) + + +def test_interval_dataclass_helpers(): + iv = ConformalInterval(0.3, 0.7, 0.5, 0.9, 0.2, 10) + assert iv.width() == pytest.approx(0.4) + assert iv.contains(0.5) is True + assert iv.contains(0.9) is False + d = iv.to_dict() + assert set(d) == { + "lower", + "upper", + "midpoint", + "coverage_level", + "half_width", + "n_calibration", + } + + +# --------------------------------------------------------------------------- # +# Coverage guarantee (the whole point of conformal prediction) +# --------------------------------------------------------------------------- # + + +def test_empirical_coverage_meets_guarantee(): + """On exchangeable synthetic data, ~90% of held-out references fall within + the 90%-level interval — the split-conformal marginal guarantee.""" + rng = random.Random(7) + # judge = reference + symmetric noise; both exchangeable draws. + data = [ + (r := rng.uniform(0.2, 0.8), r + rng.uniform(-0.1, 0.1)) for _ in range(400) + ] + cal, test = data[:200], data[200:] + # Calibration pairs are (judge, reference); predict at the judge score. + cal_pairs = [(judge, ref) for ref, judge in cal] + covered = 0 + for ref, judge in test: + iv = calibrate_interval(judge, cal_pairs, alpha=0.1) + covered += iv.contains(ref) + coverage = covered / len(test) + assert coverage >= 0.85 # ≥ (1 − alpha) − finite-sample slack + + +# --------------------------------------------------------------------------- # +# Ensemble bridge +# --------------------------------------------------------------------------- # + + +def test_conformal_from_ensemble_wraps_median_aggregate(): + judges = [ + JudgeConfig(name="j1", seed=1, noise=0.05), + JudgeConfig(name="j2", seed=2, noise=0.05), + JudgeConfig(name="j3", seed=3, noise=0.05), + ] + result = ensemble_evaluate("a prompt", "a response", judges) + cal = [(0.6, 0.55), (0.7, 0.66), (0.5, 0.52), (0.65, 0.6)] + iv = conformal_from_ensemble(result, cal, alpha=0.5) + assert iv.contains(result.median_aggregate) + assert iv.coverage_level == pytest.approx(0.5) + assert iv.n_calibration == 4