feat(evaluation): add single-arm baseline comparisons

evaluation/README.md

@@ -1,7 +1,8 @@
 # PowerContext evaluation console
 
-This directory contains a self-progressing SWE-bench Pro OFF/ON evaluation service. The web process owns the HTTP
-API and report UI; the worker owns task execution, retries, resource cleanup, and durable recovery.
+This directory contains a self-progressing SWE-bench Pro evaluation service. A batch can run the paired OFF/ON
+experiment or only one Arm to reduce cost and latency. The web process owns the HTTP API and report UI; the worker
+owns task execution, retries, resource cleanup, and durable recovery.
 
 The service is intentionally deployment-neutral. Host names, operators, filesystem roots, optional proxy endpoints, Docker
 network ranges, credentials, and service locations are supplied by the operator. The repository does not contain a
@@ -165,9 +166,11 @@ Verify the control plane before submitting work:
 curl --fail --silent http://127.0.0.1:8787/api/health
 ```
 
-Open `http://127.0.0.1:8787/`, choose `swebench-pro-stability-v1`, preview the request, and confirm it. This 24-task
-set is the deployment regression suite; use it before the 731-task `swebench-pro-public-v2` set. The same bounded
-batch can be created from the CLI after Web and Worker are healthy:
+Open `http://127.0.0.1:8787/`, choose `swebench-pro-stability-v1` and an `OFF + ON`, `ON only`, or `OFF only` run.
+This 24-task set is the deployment regression suite; use it before the 731-task `swebench-pro-public-v2` set. A
+completed aggregate report can freeze any executed Arm as an immutable baseline. Reports may select multiple
+compatible baselines for historical comparison without rerunning evaluation, while the baseline library lists the
+newest saved baselines first. The same bounded batch can be created from the CLI after Web and Worker are healthy:
 
 ```bash
 uv run --project evaluation powercontext-eval swebench-pro create-batch \

evaluation/src/powercontext_eval/models.py

@@ -28,6 +28,22 @@ class Arm(StrEnum):
     ON = "on"
 
 
+class TreatmentMode(StrEnum):
+    """The exact treatment arms executed for one evaluation task."""
+
+    OFF_ON = "off_on"
+    ON_ONLY = "on_only"
+    OFF_ONLY = "off_only"
+
+    @property
+    def arms(self) -> tuple[Arm, ...]:
+        if self is TreatmentMode.OFF_ON:
+            return (Arm.OFF, Arm.ON)
+        if self is TreatmentMode.ON_ONLY:
+            return (Arm.ON,)
+        return (Arm.OFF,)
+
+
 class PowerContextRef(BaseModel):
     """An explicit, immutable PowerContext source reference."""
 

evaluation/src/powercontext_eval/report.py

@@ -39,6 +39,7 @@
     SOURCE595_DATASET_PATCH_SHA256,
     SOURCE595_INSTANCE_ID,
 )
+from powercontext_eval.models import Arm, TreatmentMode
 
 
 class MetricSet(BaseModel):
@@ -190,10 +191,27 @@ class ReportBundle(BaseModel):
     title: str
     revisions: Mapping[str, str]
     configuration: Mapping[str, str]
-    off: ArmReport
-    on: ArmReport
+    treatment_mode: TreatmentMode = TreatmentMode.OFF_ON
+    off: ArmReport | None = None
+    on: ArmReport | None = None
     gold_validation: GoldValidationAudit | None = None
 
+    @field_validator("treatment_mode", mode="before")
+    @classmethod
+    def parse_treatment_mode(cls, value: object) -> object:
+        return TreatmentMode(value) if isinstance(value, str) else value
+
+    @model_validator(mode="after")
+    def validate_arms(self) -> Self:
+        present = {arm for arm, report in ((Arm.OFF, self.off), (Arm.ON, self.on)) if report is not None}
+        if present != set(self.treatment_mode.arms):
+            raise ValueError("Report arms do not match the treatment mode")
+        if self.off is not None and self.off.arm != "off":
+            raise ValueError("OFF report is bound to the wrong arm")
+        if self.on is not None and self.on.arm != "on":
+            raise ValueError("ON report is bound to the wrong arm")
+        return self
+
     @model_validator(mode="after")
     def validate_gold_binding(self) -> Self:
         instance_id = self.configuration.get("instance")
@@ -332,6 +350,8 @@ def _gold_validation_section(audit: GoldValidationAudit) -> list[str]:
 
 def _comparison(bundle: ReportBundle) -> list[str]:
     lines = ["## Comparison", ""]
+    if bundle.off is None or bundle.on is None:
+        return lines + ["Comparison unavailable: this run contains one treatment arm.", ""]
     comparable_states = {ArmState.TREATMENT_VALIDATED, ArmState.REPORTED}
     if not (
         bundle.off.state in comparable_states
@@ -374,10 +394,9 @@ def _validated_bundle(bundle: ReportBundle) -> ReportBundle:
         expected_bundle_fields = set(ReportBundle.model_fields)
         if set(bundle.__dict__) != expected_bundle_fields:
             raise ValueError
-        for model, model_type in (
-            (bundle.off, ArmReport),
-            (bundle.on, ArmReport),
-        ):
+        for model, model_type in ((bundle.off, ArmReport), (bundle.on, ArmReport)):
+            if model is None:
+                continue
             if type(model) is not model_type or set(model.__dict__) != set(model_type.model_fields):
                 raise ValueError
             if type(model.metrics) is not MetricSet or set(model.metrics.__dict__) != set(MetricSet.model_fields):
@@ -400,14 +419,15 @@ def render_report(bundle: ReportBundle) -> str:
     """Render only the supplied validated bundle, with no external reads."""
 
     bundle = _validated_bundle(bundle)
-    if bundle.off.arm != "off" or bundle.on.arm != "on":
-        raise ValueError("Report arms must be supplied in OFF then ON roles")
     lines = [f"# {_cell(bundle.title)}", ""]
     lines.extend(_mapping_table("Resolved revisions", bundle.revisions))
     lines.extend(_mapping_table("Configuration", bundle.configuration))
     if bundle.gold_validation is not None:
         lines.extend(_gold_validation_section(bundle.gold_validation))
-    lines.extend(_arm_section("OFF", bundle.off))
-    lines.extend(_arm_section("ON", bundle.on))
-    lines.extend(_comparison(bundle))
+    if bundle.off is not None:
+        lines.extend(_arm_section("OFF", bundle.off))
+    if bundle.on is not None:
+        lines.extend(_arm_section("ON", bundle.on))
+    if bundle.off is not None and bundle.on is not None:
+        lines.extend(_comparison(bundle))
     return "\n".join(lines)

evaluation/src/powercontext_eval/runner.py

@@ -49,7 +49,7 @@
 from powercontext_eval.context_trace import write_context_trace
 from powercontext_eval.errors import CommandFailed
 from powercontext_eval.git_source import GitSource
-from powercontext_eval.models import Arm, PowerContextRef
+from powercontext_eval.models import Arm, PowerContextRef, TreatmentMode
 from powercontext_eval.paths import EvaluationPaths
 from powercontext_eval.powercontext_sut import (
     DEFAULT_DOCKER_NETWORK_POOL,
@@ -152,6 +152,7 @@ class RunConfig:
     registry_binary: Path
     auth_json: Path
     run_id: str
+    treatment_mode: TreatmentMode = TreatmentMode.OFF_ON
     tokensflow_enabled: bool = False
     tokensflow_binary: Path | None = None
     tokensflow_user_home: Path | None = None
@@ -167,6 +168,8 @@ class RunConfig:
     cancel_event: threading.Event | None = field(default=None, repr=False, compare=False)
 
     def __post_init__(self) -> None:
+        if isinstance(self.treatment_mode, str):
+            object.__setattr__(self, "treatment_mode", TreatmentMode(self.treatment_mode))
         if not is_safe_codex_model(self.model):
             raise ValueError("Codex model is unsafe")
         if self.reasoning_effort != DEFAULT_REASONING_EFFORT:
@@ -188,8 +191,8 @@ class RunResult:
 
     run_id: str
     report_path: Path
-    off_resolved: bool
-    on_resolved: bool
+    off_resolved: bool | None
+    on_resolved: bool | None
 
 
 # Public compatibility name retained while the web task schema migrates from a single task to batches.
@@ -352,11 +355,13 @@ def _run_swebench_pro_instance(
     )
     run_store.write_json("gold/validation.json", gold_audit.model_dump(mode="json"))
 
-    def arms() -> tuple[OfficialEvaluation, OfficialEvaluation, Mapping[Arm, SutOutcome], dict[Arm, int]]:
+    selected_arms = config.treatment_mode.arms
+
+    def arms() -> tuple[dict[Arm, OfficialEvaluation], Mapping[Arm, SutOutcome], dict[Arm, int]]:
         codex_secrets = auth_secret_variants(config.auth_json)
         arm_paths: dict[Arm, ArmPaths] = {}
         stores: dict[Arm, ArtifactStore] = {}
-        for arm in (Arm.OFF, Arm.ON):
+        for arm in selected_arms:
             arm_work = layout.arm_work(arm)
             runtime = arm_work / "runtime"
             root_home = runtime / "root-home"
@@ -387,34 +392,43 @@ def arms() -> tuple[OfficialEvaluation, OfficialEvaluation, Mapping[Arm, SutOutc
                 secrets += tokensflow_secret_variants(tokensflow_credentials)
             stores[arm] = ArtifactStore(layout.arm_artifacts(arm), forbidden_values=secrets)
         prompt = instance.codex_prompt().encode()
-        outcomes = DockerSut(process).run_pair(
-            SutConfig(
-                run_id=run_id,
-                task_image=task_image_id,
-                codex_binary=config.codex_binary,
-                uv_binary=config.uv_binary,
-                source_checkout=materialized,
-                plugin_checkout_sha=resolved.sha,
-                proxy=ProxyRelayConfig(config.proxy_url) if config.proxy_url is not None else None,
-                docker_network_pool=config.docker_network_pool,
-                extra_no_proxy_hosts=config.extra_no_proxy_hosts,
-                tokensflow_enabled=config.tokensflow_enabled,
-                tokensflow_binary=config.tokensflow_binary,
-                tokensflow_egress_network=config.tokensflow_egress_network,
-                model=config.model,
-                reasoning_effort=config.reasoning_effort,
-                finalization_registrar=config.finalization_registrar,
-                container_env=config.container_env,
-            ),
-            paths=arm_paths,
-            prompts={Arm.OFF: prompt, Arm.ON: prompt},
-            stores=stores,
-            before_arm=lambda arm: emit_phase(RunPhase.RUNNING_OFF if arm is Arm.OFF else RunPhase.RUNNING_ON),
+        sut_config = SutConfig(
+            run_id=run_id,
+            task_image=task_image_id,
+            codex_binary=config.codex_binary,
+            uv_binary=config.uv_binary,
+            source_checkout=materialized,
+            plugin_checkout_sha=resolved.sha,
+            proxy=ProxyRelayConfig(config.proxy_url) if config.proxy_url is not None else None,
+            docker_network_pool=config.docker_network_pool,
+            extra_no_proxy_hosts=config.extra_no_proxy_hosts,
+            tokensflow_enabled=config.tokensflow_enabled,
+            tokensflow_binary=config.tokensflow_binary,
+            tokensflow_egress_network=config.tokensflow_egress_network,
+            model=config.model,
+            reasoning_effort=config.reasoning_effort,
+            finalization_registrar=config.finalization_registrar,
+            container_env=config.container_env,
         )
+        sut = DockerSut(process)
+        if config.treatment_mode is TreatmentMode.OFF_ON:
+            outcomes = sut.run_pair(
+                sut_config,
+                paths=arm_paths,
+                prompts={arm: prompt for arm in selected_arms},
+                stores=stores,
+                before_arm=lambda arm: emit_phase(RunPhase.RUNNING_OFF if arm is Arm.OFF else RunPhase.RUNNING_ON),
+            )
+        else:
+            arm = selected_arms[0]
+            emit_phase(RunPhase.RUNNING_OFF if arm is Arm.OFF else RunPhase.RUNNING_ON)
+            outcomes = {
+                arm: sut.run_arm(sut_config, arm, arm_paths[arm], prompt, stores[arm]),
+            }
         official: dict[Arm, OfficialEvaluation] = {}
         patch_sizes: dict[Arm, int] = {}
         emit_phase(RunPhase.OFFICIAL_EVALUATION)
-        for arm in (Arm.OFF, Arm.ON):
+        for arm in selected_arms:
             patch = process.run(
                 ("git", "diff", "--binary", "--full-index", instance.base_commit, "--"),
                 cwd=arm_paths[arm].workspace,
@@ -451,17 +465,20 @@ def arms() -> tuple[OfficialEvaluation, OfficialEvaluation, Mapping[Arm, SutOutc
                 official=official[arm],
                 official_observed_at=datetime.now(UTC),
             )
-        return official[Arm.OFF], official[Arm.ON], outcomes, patch_sizes
+        return official, outcomes, patch_sizes
 
-    off_eval, on_eval, outcomes, patch_sizes = run_after_gold(
+    official, outcomes, patch_sizes = run_after_gold(
         GoldResult(instance.instance_id, gold.resolved),
         arms,
     )
-    off_outcome = outcomes[Arm.OFF]
-    on_outcome = outcomes[Arm.ON]
     emit_phase(RunPhase.GENERATING_REPORT)
+    arm_reports = {arm: _arm_report(arm, official[arm], outcomes[arm], patch_sizes[arm]) for arm in selected_arms}
     report = ReportBundle(
-        title="PowerContext Codex SWE-bench Pro comparison",
+        title=(
+            "PowerContext Codex SWE-bench Pro comparison"
+            if config.treatment_mode is TreatmentMode.OFF_ON
+            else f"PowerContext Codex SWE-bench Pro {selected_arms[0].value.upper()} run"
+        ),
         revisions={
             "dataset": DATASET_REVISION,
             "harness": HARNESS_COMMIT,
@@ -476,17 +493,24 @@ def arms() -> tuple[OfficialEvaluation, OfficialEvaluation, Mapping[Arm, SutOutc
             "extra_no_proxy_hosts": ",".join(config.extra_no_proxy_hosts),
             "proxy": "enabled" if config.proxy_url is not None else "disabled",
             "tokensflow": "enabled" if config.tokensflow_enabled else "disabled",
+            "treatment_mode": config.treatment_mode.value,
         },
-        off=_arm_report(Arm.OFF, off_eval, off_outcome, patch_sizes[Arm.OFF]),
-        on=_arm_report(Arm.ON, on_eval, on_outcome, patch_sizes[Arm.ON]),
+        treatment_mode=config.treatment_mode,
+        off=arm_reports.get(Arm.OFF),
+        on=arm_reports.get(Arm.ON),
         gold_validation=gold_audit,
     )
     rendered = render_report(report)
     if render_report(report) != rendered:
         raise RuntimeError("Report rendering is not deterministic")
     report_path = run_store.create_text("report.md", rendered)
     run_store.create_json("report.json", report.model_dump(mode="json"))
-    return RunResult(run_id, report_path, off_eval.resolved, on_eval.resolved)
+    return RunResult(
+        run_id,
+        report_path,
+        official[Arm.OFF].resolved if Arm.OFF in official else None,
+        official[Arm.ON].resolved if Arm.ON in official else None,
+    )
 
 
 def _evaluator_test_requirements(