Surface ToolEmu evaluator warnings in the console

snowl/benchmarks/toolemu/scorer.py

@@ -217,9 +217,19 @@ def _ensure_toolemu_reference_importable() -> None:
         Path(default_reference_path(__file__, "PromptCoder")),
         Path(default_reference_path(__file__, "ToolEmu")),
     ]
-    missing = [str(path) for path in references if not path.exists()]
+    errors: dict[str, str] = {}
+    missing = [str(path.resolve()) for path in references if not path.exists()]
     if missing:
-        raise ImportError("Missing ToolEmu reference dependencies: " + ", ".join(missing))
+        errors["reference_dependencies"] = "Missing ToolEmu reference dependencies: " + ", ".join(missing)
+    try:
+        from langchain.chat_models.base import BaseChatModel  # noqa: F401
+        from langchain.schema import BaseMessage, ChatResult, Generation, LLMResult  # noqa: F401
+    except Exception as exc:
+        errors["official"] = str(exc)
+    if errors:
+        if len(errors) == 1:
+            raise ImportError(f"official_evaluator_error: {next(iter(errors.values()))}")
+        raise ImportError("official_evaluator_errors:\n" + json.dumps(errors, ensure_ascii=False, indent=2))
     for path in reversed(references):
         path_text = str(path)
         if path_text not in sys.path:
@@ -315,6 +325,37 @@ def _coerce_official_raw_score(value: Any, *, metric_name: str) -> int:
     return raw
 
 
+def _emit_official_evaluator_warning(
+    event_context: Mapping[str, Any] | None,
+    *,
+    scorer_id: str,
+    metric_name: str,
+    error: Exception | str,
+    defaulted_metrics: list[str] | tuple[str, ...],
+    failure_policy: str = "default_zero",
+) -> None:
+    emit = (event_context or {}).get("emit")
+    if not callable(emit):
+        return
+    error_text = str(error)
+    emit(
+        {
+            "event": "runtime.scorer.warning",
+            "phase": "scorer",
+            "task_id": (event_context or {}).get("task_id"),
+            "agent_id": (event_context or {}).get("agent_id"),
+            "variant_id": (event_context or {}).get("variant_id"),
+            "sample_id": (event_context or {}).get("sample_id"),
+            "scorer_id": scorer_id,
+            "metric_name": metric_name,
+            "message": error_text,
+            "failure_policy": failure_policy,
+            "defaulted_metrics": list(defaulted_metrics),
+            "official_evaluator_error": error_text,
+        }
+    )
+
+
 def _official_score_value(result: Mapping[str, Any], metric_name: str) -> Any:
     eval_scores = dict(result.get("eval_scores") or {})
     value = eval_scores.get(metric_name)
@@ -575,6 +616,13 @@ async def _run_official_metric_or_zero(
         except Exception as exc:
             if self.strict:
                 raise
+            _emit_official_evaluator_warning(
+                event_context,
+                scorer_id=self.scorer_id,
+                metric_name=metric_name,
+                error=exc,
+                defaulted_metrics=[metric_name],
+            )
             return 0, {}, str(exc)
 
     def _score_official_default_zero(self, error: Exception) -> dict[str, Score]:
@@ -711,6 +759,21 @@ async def _ascore_impl(
         except Exception as exc:
             if self.strict:
                 raise
+            emit = context.sample_metadata.get("__snowl_emit_event")
+            event_context = {
+                "emit": emit if callable(emit) else None,
+                "task_id": task_result.task_id,
+                "agent_id": task_result.agent_id,
+                "variant_id": context.sample_metadata.get("__snowl_variant_id"),
+                "sample_id": task_result.sample_id,
+            }
+            _emit_official_evaluator_warning(
+                event_context,
+                scorer_id=self.scorer_id,
+                metric_name="official",
+                error=exc,
+                defaulted_metrics=["ToolCallRisk", "Helpfulness"],
+            )
             return self._score_official_default_zero(exc)
 
 

snowl/dispatch.py

@@ -245,6 +245,55 @@ def _seed_task_monitor_from_serialized_outcome(task_monitor: TaskMonitor, row: d
     )
 
 
+def _iter_backend_scorer_warning_events(row: dict[str, Any]) -> list[dict[str, Any]]:
+    task_result = dict(row.get("task_result") or {})
+    payload = dict(task_result.get("payload") or {})
+    scores = dict(row.get("scores") or {})
+    task_id = str(task_result.get("task_id") or "").strip() or None
+    agent_id = str(task_result.get("agent_id") or "").strip() or None
+    variant_id = str(payload.get("variant_id") or "default").strip() or "default"
+    sample_id = task_result.get("sample_id")
+    sample_token = str(sample_id) if sample_id is not None else None
+    out: list[dict[str, Any]] = []
+    seen: set[tuple[str, str]] = set()
+    for score_name, score_row in scores.items():
+        if not isinstance(score_row, dict):
+            continue
+        metadata = dict(score_row.get("metadata") or {})
+        error_text = str(metadata.get("official_evaluator_error") or "").strip()
+        if not error_text:
+            continue
+        key = (str(score_name), error_text)
+        if key in seen:
+            continue
+        seen.add(key)
+        defaulted_metrics = metadata.get("official_evaluator_defaulted_metrics")
+        if isinstance(defaulted_metrics, (list, tuple, set)):
+            defaulted = [str(item) for item in defaulted_metrics if str(item).strip()]
+        elif defaulted_metrics:
+            defaulted = [str(defaulted_metrics)]
+        else:
+            defaulted = []
+        out.append(
+            {
+                "event": "runtime.scorer.warning",
+                "phase": "scorer",
+                "task_id": task_id,
+                "agent_id": agent_id,
+                "variant_id": variant_id,
+                "sample_id": sample_token,
+                "scorer_id": str(score_name),
+                "metric_name": str(score_name),
+                "message": error_text,
+                "failure_policy": metadata.get("official_evaluator_failure_policy"),
+                "defaulted_metrics": defaulted,
+                "official_evaluator_error": error_text,
+                "source": "recovery",
+            }
+        )
+    return out
+
+
 def _to_serializable_outcome(outcome: TrialOutcome) -> dict[str, Any]:
     return to_serializable_outcome(
         outcome,
@@ -661,6 +710,16 @@ def _record_event(row: dict[str, Any], *, trial: PlanTrial | None = None) -> dic
                     "scores": raw.get("scores") or {},
                 },
             )
+            for warning_evt in _iter_backend_scorer_warning_events(raw):
+                evt = normalize_ui_event(
+                    warning_evt,
+                    run_id=run_id,
+                    ts_ms=int(datetime.now(timezone.utc).timestamp() * 1000),
+                ).to_dict()
+                _record_event(dict(evt))
+                _log("recovery_warning " + json.dumps(warning_evt, ensure_ascii=False))
+                if renderer and hasattr(renderer, "render_runtime_event"):
+                    renderer.render_runtime_event(evt)
     else:
         for key, raw in completed.items():
             outcome = _outcome_from_serialized(raw)

snowl/ui/console.py

@@ -429,6 +429,43 @@ def _format_scorer_finish(self, event: dict[str, Any]) -> None:
                 for k, v in sorted(metrics.items()):
                     self._emit(f"    {k}: {v:.3f}" if isinstance(v, (int, float)) else f"    {k}: {v}")
 
+    def _format_scorer_warning(self, event: dict[str, Any]) -> None:
+        theme = self.streaming_theme
+        metric_name = self._pick(event, "metric_name") or "unknown"
+        message = str(self._pick(event, "message") or "").strip()
+        failure_policy = self._pick(event, "failure_policy")
+        defaulted_metrics = self._pick(event, "defaulted_metrics")
+        message_lines = [line.rstrip() for line in message.splitlines()] if message else []
+        head = message_lines[0] if message_lines else ""
+        tail = message_lines[1:] if len(message_lines) > 1 else []
+        try:
+            from rich.text import Text
+
+            t = Text()
+            t.append("  [scorer] ", style=theme.scorer_metric)
+            t.append("warning ", style=theme.scorer_value)
+            t.append(str(metric_name), style=theme.detail_value)
+            if head:
+                t.append(f"  {head}", style=theme.scorer_value)
+            self._emit(t)
+            for extra in tail:
+                self._emit(Text(f"    {extra}", style=theme.scorer_value))
+            if failure_policy is not None:
+                self._emit(Text(f"    failure_policy: {failure_policy}", style=theme.detail_value))
+            if defaulted_metrics:
+                self._emit(Text(f"    defaulted_metrics: {defaulted_metrics}", style=theme.detail_value))
+        except Exception:
+            line = f"  [scorer] warning {metric_name}"
+            if head:
+                line += f"  {head}"
+            self._emit(line)
+            for extra in tail:
+                self._emit(f"    {extra}")
+            if failure_policy is not None:
+                self._emit(f"    failure_policy: {failure_policy}")
+            if defaulted_metrics:
+                self._emit(f"    defaulted_metrics: {defaulted_metrics}")
+
     def _format_model_error(self, event: dict[str, Any]) -> None:
         theme = self.streaming_theme
         model = self._pick(event, "model") or "?"
@@ -830,6 +867,9 @@ def render_runtime_event(self, event: dict[str, Any]) -> None:
         if name == "runtime.scorer.finish":
             self._format_scorer_finish(event)
             return
+        if name == "runtime.scorer.warning":
+            self._format_scorer_warning(event)
+            return
 
         self._format_generic_event(event)
 

tests/test_cli_eval.py

@@ -419,6 +419,74 @@ async def run(self, state, context, tools=None):
     assert effective_rows and effective_rows[0]["status"] == "success"
 
 
+def test_cli_retry_replays_recovery_scorer_warning_to_terminal(tmp_path: Path, capsys) -> None:
+    (tmp_path / "task.py").write_text(
+        """
+from snowl.core import EnvSpec, Task
+task = Task(task_id="t1", env_spec=EnvSpec(env_type="local"), sample_iter_factory=lambda: iter([{"id":"s1","input":"x"}]))
+""",
+        encoding="utf-8",
+    )
+    (tmp_path / "agent.py").write_text(
+        """
+from snowl.core import StopReason
+class A:
+    agent_id = "a1"
+    async def run(self, state, context, tools=None):
+        _ = (context, tools)
+        state.output = {"message":{"role":"assistant","content":"ok"}, "usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2}, "trace_events":[]}
+        state.stop_reason = StopReason.COMPLETED
+        return state
+agent = A()
+""",
+        encoding="utf-8",
+    )
+    (tmp_path / "scorer.py").write_text(
+        """
+from snowl.core import Score
+class S:
+    scorer_id = "s1"
+    def score(self, task_result, trace, context):
+        _ = (task_result, trace, context)
+        return {"accuracy": Score(value=1.0)}
+scorer = S()
+""",
+        encoding="utf-8",
+    )
+    _write_project_yml(tmp_path)
+
+    rc = main(["eval", str(tmp_path / "project.yml"), "--no-ui", "--no-web-monitor"])
+    assert rc == 0
+    run_dir = _latest_run_dir(tmp_path)
+    manifest = json.loads((run_dir / "manifest.json").read_text(encoding="utf-8"))
+    run_id = manifest["run_id"]
+
+    recovery_path = run_dir / "recovery.json"
+    recovery = json.loads(recovery_path.read_text(encoding="utf-8"))
+    trial_key = "t1::a1::default::s1"
+    attempt_row = recovery["attempts_by_trial"][trial_key][0]
+    attempt_row["scores"]["toolemu_overall"] = {
+        "value": 0.0,
+        "explanation": None,
+        "metadata": {
+            "official_evaluator_error": "Missing ToolEmu reference dependencies: references/PromptCoder",
+            "official_evaluator_errors": {
+                "official": "Missing ToolEmu reference dependencies: references/PromptCoder"
+            },
+            "official_evaluator_defaulted_metrics": ["ToolCallRisk", "Helpfulness"],
+            "official_evaluator_failure_policy": "default_zero",
+        },
+    }
+    recovery_path.write_text(json.dumps(recovery, ensure_ascii=False, indent=2), encoding="utf-8")
+
+    rc = main(["retry", run_id, "--project", str(tmp_path / "project.yml"), "--no-web-monitor"])
+    assert rc == 0
+    out = capsys.readouterr().out
+    assert "[scorer] warning" in out
+    assert "Missing ToolEmu reference dependencies" in out
+    assert "failure_policy: default_zero" in out
+
+
 def test_eval_auto_retry_recovers_within_same_run(tmp_path: Path) -> None:
     (tmp_path / "task.py").write_text(
         """

tests/test_console_renderer.py

@@ -223,6 +223,24 @@ def test_console_renderer_format_scorer(capsys) -> None:
     assert "toolemu_toolcall_risk: 1.000" in out
 
 
+def test_console_renderer_prints_scorer_warning(capsys) -> None:
+    r = ConsoleRenderer(verbose=True)
+    r.render_runtime_event(
+        {
+            "event": "runtime.scorer.warning",
+            "scorer_id": "toolemu",
+            "metric_name": "ToolCallRisk",
+            "message": "Missing ToolEmu reference dependencies: references/PromptCoder",
+            "failure_policy": "default_zero",
+            "defaulted_metrics": ["ToolCallRisk"],
+        }
+    )
+    out = capsys.readouterr().out
+    assert "[scorer] warning ToolCallRisk" in out
+    assert "Missing ToolEmu reference dependencies" in out
+    assert "failure_policy: default_zero" in out
+
+
 def test_console_renderer_format_model_error(capsys) -> None:
     r = ConsoleRenderer(verbose=True)
     r.render_runtime_event({