Add people_search benchmark (scorer-based, provider-agnostic)

.env.example

@@ -5,4 +5,9 @@ OPENAI_API_KEY=
 PARALLEL_API_KEY=
 SERP_API_KEY=
 PERPLEXITY_API_KEY=
-TAVILY_API_KEY=
+TAVILY_API_KEY=
+# Generic people-search endpoint (http_people_search sampler)
+PEOPLE_SEARCH_API_URL=
+PEOPLE_SEARCH_API_KEY=
+# Set to 0 to skip people-search LLM judges (deterministic scorers only)
+# PEOPLE_SEARCH_LLM_JUDGES=1

src/evals/configs/datasets.py

@@ -17,6 +17,8 @@ class Dataset:
     csv_path: str
     grader: Callable
     df: pd.DataFrame | None
+    # When False, empty answer/ground_truth is allowed (scorer-based datasets).
+    requires_ground_truth: bool = True
 
 
 DATASETS = [
@@ -56,4 +58,13 @@ class Dataset:
         grader=fin_search_evaluator.evaluate_single_fin_search,
         df=None,
     ),
+    Dataset(
+        dataset_name="people_search",
+        csv_path="data/people_search_full_dataset.csv",
+        grader=evaluator.evaluate_single_people_search,
+        df=None,
+        # answer column is empty in the CSV (no gold answers); metadata lives in
+        # dedicated columns and is assembled at load time in utils.get_dataset
+        requires_ground_truth=False,
+    ),
 ]

src/evals/configs/samplers.py

@@ -22,6 +22,7 @@
     YouResearchSampler,
     YouSearchSnippetsSampler,
 )
+from evals.samplers.applied_samplers.people_search_sampler import HttpPeopleSearchSampler
 
 
 SAMPLERS = [
@@ -148,10 +149,24 @@
         search_effort="high",
         timeout=3000,
     ),
+    # Generic people-search HTTP endpoint (scorer-based; excluded from defaults)
+    HttpPeopleSearchSampler(
+        sampler_name="http_people_search",
+        api_url=os.getenv("PEOPLE_SEARCH_API_URL"),
+        api_key=os.getenv("PEOPLE_SEARCH_API_KEY"),
+        timeout=120,
+        max_concurrency=5,
+    ),
 ]
 
-# Samplers excluded from default runs due to high cost or long latency
-EXCLUDE_KEYWORDS = ["research", "parallel_pro", "parallel_ultra", 'perplexity_finance_historical_lookup']
+# Samplers excluded from default runs due to high cost, long latency, or special datasets
+EXCLUDE_KEYWORDS = [
+    "research",
+    "parallel_pro",
+    "parallel_ultra",
+    "perplexity_finance_historical_lookup",
+    "http_people_search",
+]
 
 NON_RESEARCH_SAMPLERS = [
     sampler.sampler_name

src/evals/eval_results_analyzer.py

@@ -34,10 +34,21 @@ def get_results_files(results_dir: Optional[Path] = None) -> List[str]:
     return glob.glob(f"{results_dir}/dataset_*.csv")
 
 
+def _mean_numeric(series: pd.Series) -> float | None:
+    values = pd.to_numeric(series, errors="coerce").dropna()
+    if len(values) == 0:
+        return None
+    return float(values.mean())
+
+
 def write_metrics(results_dir: Optional[Path] = None):
     """
     Calculate metrics from raw results such as accuracy score, P50 latency, and average latency.
 
+    For people_search (scorer-based), ``accuracy_score`` is left blank — primary quality
+    metrics are ``mean_field_fill``, ``mean_persona_field_fill``, and ``mean_judge_*``,
+    with ``has_people_rate`` as a separate retrieval signal (not “accuracy”).
+
     Args:
         results_dir: Optional path to results directory. Defaults to src/evals/results
     """
@@ -69,33 +80,72 @@ def write_metrics(results_dir: Optional[Path] = None):
             .dropna()
             .median()
         )
-        correct = len(
-            df_sampler_results[df_sampler_results["evaluation_result"] == "is_correct"]
-        )
         count_answered = len(successful_df)
 
         if count_answered == 0:
             raise ValueError(f"No successful results found for sampler {sampler_name}")
 
-        accuracy_score = round((correct / count_answered) * 100, 2)
-
-        metric_rows.append(
-            {
-                "provider": sampler_name,
-                "dataset": dataset_name,
-                "accuracy_score": accuracy_score,
-                "p50_internal_latency": round(float(p50_internal_latency), 2),
-                "p50_request_response_latency": round(
-                    float(p50_request_response_latency), 2
-                ),
-                "problem_count": count_answered,
-            }
+        is_people_search = dataset_name == "people_search" or (
+            "field_fill" in successful_df.columns
         )
 
+        row = {
+            "provider": sampler_name,
+            "dataset": dataset_name,
+            "p50_internal_latency": round(float(p50_internal_latency), 2)
+            if pd.notna(p50_internal_latency)
+            else None,
+            "p50_request_response_latency": round(
+                float(p50_request_response_latency), 2
+            )
+            if pd.notna(p50_request_response_latency)
+            else None,
+            "problem_count": count_answered,
+        }
+
+        if is_people_search:
+            # Do not reuse accuracy_score — that means gold-answer correctness elsewhere.
+            row["accuracy_score"] = None
+            if "has_people" in successful_df.columns:
+                rate = _mean_numeric(successful_df["has_people"])
+                if rate is not None:
+                    row["has_people_rate"] = round(rate, 4)
+            if "field_fill" in successful_df.columns:
+                mean_ff = _mean_numeric(successful_df["field_fill"])
+                if mean_ff is not None:
+                    row["mean_field_fill"] = round(mean_ff, 4)
+            if "persona_field_fill" in successful_df.columns:
+                mean_pff = _mean_numeric(successful_df["persona_field_fill"])
+                if mean_pff is not None:
+                    row["mean_persona_field_fill"] = round(mean_pff, 4)
+            if "judge_overall" in successful_df.columns:
+                mean_jo = _mean_numeric(successful_df["judge_overall"])
+                if mean_jo is not None:
+                    row["mean_judge_overall"] = round(mean_jo, 4)
+            if "judge_persona" in successful_df.columns:
+                mean_jp = _mean_numeric(successful_df["judge_persona"])
+                if mean_jp is not None:
+                    row["mean_judge_persona"] = round(mean_jp, 4)
+            # Sort key within people_search: prefer field fill, then judges
+            row["_sort_score"] = row.get("mean_field_fill") or row.get(
+                "mean_judge_overall"
+            ) or row.get("has_people_rate") or 0.0
+        else:
+            correct = len(
+                df_sampler_results[
+                    df_sampler_results["evaluation_result"] == "is_correct"
+                ]
+            )
+            accuracy_score = round((correct / count_answered) * 100, 2)
+            row["accuracy_score"] = accuracy_score
+            row["_sort_score"] = accuracy_score
+
+        metric_rows.append(row)
+
     write_path = results_dir / "analyzed_results.csv"
     metric_df = pd.DataFrame(metric_rows).sort_values(
-        ["dataset", "accuracy_score"], ascending=False
+        ["dataset", "_sort_score"], ascending=[True, False]
     )
+    metric_df = metric_df.drop(columns=["_sort_score"])
     metric_df.to_csv(write_path, index=False)
     print(f"Results were written to {write_path}")
-    print(metric_df)

src/evals/processing/evaluate_answer.py

@@ -11,6 +11,8 @@
 
 from evals import constants
 from evals.processing import llm, deepsearchqa_utils
+from evals.processing.people_search.field_fill import score_people_output
+from evals.processing.people_search.llm_judges import run_people_llm_judges
 
 
 class AnswerGrader:
@@ -167,3 +169,50 @@ async def evaluate_single_fin_search(
             "is_incorrect": is_incorrect,
             "score": is_correct,
         }
+
+    async def evaluate_single_people_search(
+        self, question: str, target: str, predicted_answer: str
+    ) -> Dict[str, Any]:
+        """Score people-search provider output with deterministic field-fill scorers.
+
+        Unlike SimpleQA/FRAMES, there is no gold answer. ``target`` is a JSON
+        string of row metadata (persona, query_type, etc.). ``predicted_answer``
+        is a JSON string of structured people[] output from a people sampler.
+        """
+        try:
+            metadata = json.loads(target) if target else {}
+            if not isinstance(metadata, dict):
+                metadata = {}
+        except json.JSONDecodeError:
+            metadata = {}
+
+        try:
+            output = json.loads(predicted_answer) if predicted_answer else {}
+            if not isinstance(output, dict):
+                output = {"error": "predicted_answer was not a JSON object", "people": []}
+        except json.JSONDecodeError:
+            output = {"error": "predicted_answer was not valid JSON", "people": []}
+
+        scores = score_people_output(output, metadata)
+        has_people = scores["has_people"] >= 1.0
+
+        judge_scores = await run_people_llm_judges(
+            question, output, metadata, model=self.model
+        )
+
+        # Do not map has_people → is_correct: that confuses analyzed "accuracy"
+        # with gold-answer benchmarks. Row-level label is has_people / no_people.
+        return {
+            "grade": "has_people" if has_people else "no_people",
+            "score_name": "has_people" if has_people else "no_people",
+            "is_correct": has_people,
+            "is_incorrect": not has_people,
+            "score": scores["field_fill"],
+            "has_people": scores["has_people"],
+            "person_count": scores["person_count"],
+            "field_fill": scores["field_fill"],
+            "persona_field_fill": scores["persona_field_fill"],
+            "persona": scores.get("persona"),
+            "question": question,
+            **judge_scores,
+        }

src/evals/processing/people_search/__init__.py

@@ -0,0 +1,15 @@
+"""People-search scoring helpers (no gold-answer grading)."""
+
+from evals.processing.people_search.field_fill import (
+    extract_people,
+    row_fill_score,
+    score_people_output,
+)
+from evals.processing.people_search.schema import normalize_people_payload
+
+__all__ = [
+    "extract_people",
+    "normalize_people_payload",
+    "row_fill_score",
+    "score_people_output",
+]

src/evals/processing/people_search/constants.py

@@ -0,0 +1,106 @@
+"""Shared constants for the people-search benchmark."""
+
+PERSONA_SLUGS = {
+    "Recruiter / Talent Sourcer": "recruiter",
+    "SDR / BDR": "sdr",
+    "Background Check / Compliance Analyst": "compliance",
+    "Journalist / Investigative Researcher": "journalist",
+    "Event Organizer / Community Manager": "events",
+    "VC / PE / Investor": "investor",
+}
+
+TRACKED_FIELDS: tuple[str, ...] = (
+    "displayname",
+    "current_title",
+    "current_company",
+    "location",
+    "profile_url",
+    "highlight",
+    "email",
+    "phone",
+    "skills",
+    "insights",
+    "confidence",
+)
+
+# Weight per field for each persona slug (higher = more important to that buyer).
+PERSONA_FIELD_WEIGHTS: dict[str, dict[str, float]] = {
+    "recruiter": {
+        "displayname": 1.0,
+        "current_title": 2.5,
+        "current_company": 2.5,
+        "location": 1.5,
+        "profile_url": 1.0,
+        "highlight": 1.0,
+        "email": 0.5,
+        "phone": 0.5,
+        "skills": 2.0,
+        "insights": 1.5,
+        "confidence": 1.0,
+    },
+    "sdr": {
+        "displayname": 2.0,
+        "current_title": 1.5,
+        "current_company": 2.0,
+        "location": 0.5,
+        "profile_url": 1.5,
+        "highlight": 0.5,
+        "email": 3.0,
+        "phone": 3.0,
+        "skills": 0.5,
+        "insights": 0.5,
+        "confidence": 1.5,
+    },
+    "compliance": {
+        "displayname": 3.0,
+        "current_title": 2.5,
+        "current_company": 3.0,
+        "location": 1.0,
+        "profile_url": 2.0,
+        "highlight": 1.0,
+        "email": 0.5,
+        "phone": 0.5,
+        "skills": 0.5,
+        "insights": 1.0,
+        "confidence": 2.5,
+    },
+    "journalist": {
+        "displayname": 2.0,
+        "current_title": 2.0,
+        "current_company": 2.0,
+        "location": 1.0,
+        "profile_url": 2.0,
+        "highlight": 2.5,
+        "email": 0.5,
+        "phone": 0.5,
+        "skills": 0.5,
+        "insights": 2.0,
+        "confidence": 1.0,
+    },
+    "events": {
+        "displayname": 2.0,
+        "current_title": 1.5,
+        "current_company": 2.0,
+        "location": 1.5,
+        "profile_url": 1.5,
+        "highlight": 1.0,
+        "email": 2.5,
+        "phone": 2.5,
+        "skills": 0.5,
+        "insights": 0.5,
+        "confidence": 1.0,
+    },
+    "investor": {
+        "displayname": 2.0,
+        "current_title": 2.5,
+        "current_company": 2.5,
+        "location": 1.0,
+        "profile_url": 2.0,
+        "highlight": 2.0,
+        "email": 0.5,
+        "phone": 0.5,
+        "skills": 1.0,
+        "insights": 2.0,
+        "confidence": 1.5,
+    },
+}