Surface provider error bodies and stop one dead sampler losing all metrics

src/evals/eval_results_analyzer.py

@@ -6,6 +6,7 @@
 """
 
 import glob
+import logging
 import os
 from pathlib import Path
 from typing import List, Optional
@@ -73,9 +74,17 @@ def write_metrics(results_dir: Optional[Path] = None):
             df_sampler_results[df_sampler_results["evaluation_result"] == "is_correct"]
         )
         count_answered = len(successful_df)
+        # Failed rows are excluded from the accuracy calculation, so report them
+        # explicitly -- otherwise a sampler that errored on half the dataset is
+        # indistinguishable from one that answered all of it.
+        failed_count = len(df_sampler_results) - count_answered
 
         if count_answered == 0:
-            raise ValueError(f"No successful results found for sampler {sampler_name}")
+            logging.warning(
+                f"No successful results for sampler {sampler_name} on dataset "
+                f"{dataset_name} ({failed_count} failed); excluding it from the summary"
+            )
+            continue
 
         accuracy_score = round((correct / count_answered) * 100, 2)
 
@@ -89,9 +98,17 @@ def write_metrics(results_dir: Optional[Path] = None):
                     float(p50_request_response_latency), 2
                 ),
                 "problem_count": count_answered,
+                "failed_count": failed_count,
             }
         )
 
+    if not metric_rows:
+        logging.warning(
+            f"No sampler in {results_dir} produced a successful result; "
+            "no metrics summary written"
+        )
+        return
+
     write_path = results_dir / "analyzed_results.csv"
     metric_df = pd.DataFrame(metric_rows).sort_values(
         ["dataset", "accuracy_score"], ascending=False

src/evals/samplers/base_samplers/base_api_sampler.py

@@ -1,11 +1,16 @@
 from abc import abstractmethod
+import logging
 from typing import Any, Dict
 
 import aiohttp
 
 from evals.samplers.base_samplers.base_sampler import BaseSampler
 
 
+# Enough of the provider's error body to identify the problem without flooding logs.
+MAX_ERROR_BODY_CHARS = 2000
+
+
 class BaseAPISampler(BaseSampler):
     """Base class for API-based samplers that make HTTP requests"""
 
@@ -60,36 +65,46 @@ def _get_method() -> str:
         """Get provider specific HTTP method"""
         pass
 
+    @staticmethod
+    async def _decode_response(response: aiohttp.ClientResponse) -> Any:
+        """Return the decoded JSON body, or raise with the provider's error text.
+
+        aiohttp's raise_for_status() reports only the status line, e.g.
+        "400, message='Bad Request'". Providers put the actual reason in the
+        response body -- which field was rejected, which parameter was invalid --
+        and discarding it turns a one-line diagnosis into a debugging session.
+        """
+        if response.status >= 400:
+            body = (await response.text())[:MAX_ERROR_BODY_CHARS]
+            raise aiohttp.ClientResponseError(
+                response.request_info,
+                response.history,
+                status=response.status,
+                message=f"{response.reason}: {body}",
+                headers=response.headers,
+            )
+        return await response.json()
+
     async def get_search_results(self, query: str) -> Any:
         """Get raw search results from the API using async HTTP"""
         try:
             self._set_params()
             payload = self._get_payload(query)
+            url = self.base_url + self.endpoint
 
             timeout = aiohttp.ClientTimeout(total=self.timeout)
             async with aiohttp.ClientSession(timeout=timeout) as session:
                 if self.method == "POST":
-                    async with session.post(
-                        self.base_url + self.endpoint,
-                        json=payload,
-                        headers=self.headers,
-                    ) as response:
-                        response.raise_for_status()
-                        data = await response.json()
+                    request = session.post(url, json=payload, headers=self.headers)
                 elif self.method == "GET":
-                    async with session.get(
-                        self.base_url + self.endpoint,
-                        params=payload,
-                        headers=self.headers,
-                    ) as response:
-                        response.raise_for_status()
-                        data = await response.json()
+                    request = session.get(url, params=payload, headers=self.headers)
                 else:
                     raise ValueError(
                         'Unsupported method, please select between ["POST", "GET"]'
                     )
 
-                return data
+                async with request as response:
+                    return await self._decode_response(response)
         except Exception as e:
-            print(f"{self.sampler_name} failed with error {e}")
+            logging.error(f"{self.sampler_name} failed with error {e}")
             raise e

tests/test_eval_results_analyzer.py

@@ -0,0 +1,71 @@
+"""Tests for metrics aggregation, including how failed rows are reported."""
+
+import pandas as pd
+
+from evals.eval_results_analyzer import write_metrics
+
+
+def _write_raw(results_dir, dataset, sampler, rows):
+    results_dir.mkdir(parents=True, exist_ok=True)
+    path = results_dir / f"dataset_{dataset}_raw_results_{sampler}.csv"
+    pd.DataFrame(rows).to_csv(path, index=False)
+
+
+def _row(query, evaluation_result, internal=10.0, request=20.0):
+    answer = "FAILED" if evaluation_result == "FAILED" else f"answer for {query}"
+    return {
+        "query": query,
+        "internal_response_time_ms": "FAILED"
+        if evaluation_result == "FAILED"
+        else internal,
+        "request_response_time_ms": "FAILED"
+        if evaluation_result == "FAILED"
+        else request,
+        "evaluation_result": evaluation_result,
+        "generated_answer": answer,
+        "ground_truth": "gt",
+    }
+
+
+def test_failed_rows_are_reported_not_hidden(tmp_path):
+    """Accuracy excludes failures, so the count must be visible alongside it."""
+    _write_raw(
+        tmp_path,
+        "fake",
+        "sampler_a",
+        [
+            _row("q1", "is_correct"),
+            _row("q2", "is_incorrect"),
+            _row("q3", "FAILED"),
+            _row("q4", "FAILED"),
+        ],
+    )
+
+    write_metrics(tmp_path)
+    metrics = pd.read_csv(tmp_path / "analyzed_results.csv")
+
+    row = metrics.iloc[0]
+    assert row["problem_count"] == 2
+    assert row["failed_count"] == 2
+    # 1 correct out of the 2 that were actually answered
+    assert row["accuracy_score"] == 50.0
+
+
+def test_one_fully_failed_sampler_does_not_destroy_the_summary(tmp_path):
+    """Regression: a single dead sampler used to raise and lose every other result."""
+    _write_raw(tmp_path, "fake", "healthy", [_row("q1", "is_correct")])
+    _write_raw(tmp_path, "fake", "dead", [_row("q1", "FAILED"), _row("q2", "FAILED")])
+
+    write_metrics(tmp_path)
+    metrics = pd.read_csv(tmp_path / "analyzed_results.csv")
+
+    assert metrics["provider"].tolist() == ["healthy"]
+    assert metrics.iloc[0]["accuracy_score"] == 100.0
+
+
+def test_no_summary_written_when_everything_failed(tmp_path):
+    _write_raw(tmp_path, "fake", "dead", [_row("q1", "FAILED")])
+
+    write_metrics(tmp_path)
+
+    assert not (tmp_path / "analyzed_results.csv").exists()