diff --git a/.env.example b/.env.example index af12a91..573c8f2 100644 --- a/.env.example +++ b/.env.example @@ -5,4 +5,9 @@ OPENAI_API_KEY= PARALLEL_API_KEY= SERP_API_KEY= PERPLEXITY_API_KEY= -TAVILY_API_KEY= \ No newline at end of file +TAVILY_API_KEY= +# Generic people-search endpoint (http_people_search sampler) +PEOPLE_SEARCH_API_URL= +PEOPLE_SEARCH_API_KEY= +# Set to 0 to skip people-search LLM judges (deterministic scorers only) +# PEOPLE_SEARCH_LLM_JUDGES=1 diff --git a/README.md b/README.md index 8c9d14a..2a4ad35 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,9 @@ This repository contains evaluation framework for AI-first web search APIs. Each The framework supports multiple search providers (You.com, Exa, Tavily, Parallel) and a representative Google SERP–based sampler. For each query, search results are fetched from the search API, synthesized into an answer -using an LLM, then graded against the ground truth.[^1] It also includes a dedicated [finance evaluation](#finance-evaluation) suite for benchmarking financial data retrieval. +using an LLM, then graded against the ground truth.[^1] It also includes a dedicated [finance evaluation](#finance-evaluation) +suite and a [people search evaluation](#people-search-evaluation) for benchmarking people-data APIs (no gold answers; +deterministic field-fill scorers + LLM judges on structured `people[]` output). To learn more about our evals methodology and system architecture, please read You.com's research articles: @@ -59,6 +61,7 @@ the API request is used. | DeepSearchQA | Challenging multi-step information seeking tasks. Only recommended for use with research endpoints ([paper](https://storage.googleapis.com/deepmind-media/DeepSearchQA/DeepSearchQA_benchmark_paper.pdf), [dataset](https://huggingface.co/datasets/google/deepsearchqa)) | `--datasets deepsearchqa` | | BrowseComp | A simple and challenging benchmark that measures the ability of AI agents to locate hard-to-find information. Only recommended for use with research endpoints ([paper](https://arxiv.org/abs/2504.12516), [dataset](https://openaipublic.blob.core.windows.net/simple-evals/browse_comp_test_set.csv)) | `--datasets browsecomp` | | FinSearchComp T2 & T3 | Public-company financial lookup benchmarks from filings ([paper](https://arxiv.org/pdf/2509.13160)). T2 covers simple historical lookups; T3 covers complex historical investigations. Grading follows the paper's judge prompt; numbers in different formats (e.g. `12.45%` vs `0.1245`) are treated as equivalent. | `--datasets fin_search_comp_t2_global fin_search_comp_t3_global` | +| People Search | 240 people enrichment / search queries across 6 buyer personas. **No gold answers.** Score structured `people[]` from any HTTP people-search API via `http_people_search` (deterministic field-fill + LLM judges on by default). See [People Search evaluation](#people-search-evaluation). | `--datasets people_search --samplers http_people_search` | ## Installation @@ -92,9 +95,10 @@ Edit `.env` and set the keys for your chosen providers. To run evaluations for a | Perplexity | `PERPLEXITY_API_KEY` | | Tavily | `TAVILY_API_KEY` | | You.com | `YOU_API_KEY` | +| People search (generic HTTP)| `PEOPLE_SEARCH_API_URL` (+ optional `PEOPLE_SEARCH_API_KEY`) | Grading uses OpenAI models by default, but Gemini models are also supported. Set `OPENAI_API_KEY` or -`GOOGLE_GEMINI_KEY` as appropriate for the LLM judge. +`GOOGLE_GEMINI_API_KEY` as appropriate for the LLM judge. ## Usage @@ -119,19 +123,20 @@ python src/evals/eval_runner.py --datasets frames python src/evals/eval_runner.py --samplers you_search_with_livecrawl --datasets simpleqa --limit 100 # Fresh run: clear existing results and re-run -python src/evals/eval_runner.py --clean --samplers you_search_with_livecrawl --datasets simpleqa --limit 100 +python src/evals/eval_runner.py --clean True --samplers you_search_with_livecrawl --datasets simpleqa --limit 100 ``` #### Important Notes - To avoid unintended high credit usage, You.com's Research endpoints are not included in the default samplers. They can be evaluated by calling them explicitly, like `--samplers you_research_standard` or by using `--samplers all`. - The BrowseComp and Deep Search QA Datasets are not included in the default benchmark dataset list because they are -intended to evaluate Research endpoints. +intended to evaluate Research endpoints. +- `people_search` / `http_people_search` are also excluded from defaults. Run them explicitly (see [People Search evaluation](#people-search-evaluation)). ### LLM's for synthesis and judging By default, GPT 5.4 nano is used for synthesis and GPT 5.4 mini via the OpenAI API is used for grading. This codebase also supports Gemini models via the Google `genai` library. To use an alternative OpenAI model or a -Gemini model, simply update the model name in `src.constants.py`. The code will interpret whether you are using a GPT +Gemini model, simply update the model name in `src/evals/constants.py`. The code will interpret whether you are using a GPT or Gemini model and route your request appropriately. ### Other configuration options @@ -143,7 +148,8 @@ or Gemini model and route your request appropriately. | Limit | `--limit ` | Run on at most `n` problems (optional). | | Batch size | `--batch-size 50` | Number of problems per batch before writing results (default: 50). | | Max concurrent tasks | `--max-concurrent-tasks 10` | Concurrency limit (default: 10). | -| Clean | `--clean` | Remove existing results and run from scratch. (default False) | +| Clean | `--clean True` | Wipe results and run from scratch (pass the string `True`; default is no clean). | + ## Finance evaluation @@ -196,6 +202,248 @@ python src/evals/eval_runner.py \ * Internal latency as reported by the provider is used when available. When unavailable, the total time taken to complete the API request is used. +## People Search evaluation + +The `people_search` benchmark evaluates **people-data / people-search APIs** (named-person enrichment and open +candidate search). It does **not** use the web-search path of snippets → LLM synthesis → gold-answer grading. + +There is **no gold answer** per row. Your API returns structured `people[]`; the shared `eval_runner.py` scores that +payload with: + +1. **Deterministic scorers** (always on) — retrieval + field richness +2. **LLM judges** (on by default; 2 calls per row) — overall quality + persona-specific quality + +This repo does **not** ship people-search API clients. Point `http_people_search` at **any** people-search HTTP +endpoint that speaks the contract below (or a thin adapter in front of your existing API). + +```bash +python src/evals/eval_runner.py \ + --samplers http_people_search \ + --datasets people_search \ + --limit 5 +``` + +### Dataset + +| | | +|--|--| +| File | [`data/people_search_full_dataset.csv`](data/people_search_full_dataset.csv) | +| Size | 240 queries | +| Split | 90 `enrichment` · 150 `search` | +| Personas (40 each) | Recruiter / Talent Sourcer · SDR / BDR · Background Check / Compliance Analyst · Journalist / Investigative Researcher · Event Organizer / Community Manager · VC / PE / Investor | + +CSV columns: `benchmark_id`, `problem`, `answer`, `persona`, `persona_slug`, `query_type`, `person_name`, `company`. + +`problem` is the natural-language query. `answer` is **always empty** in the shipped CSV (no gold / sample answers). +Scoring context (`persona`, `persona_slug`, `query_type`, `person_name`, `company`, `benchmark_id`) lives in those +dedicated columns; at load time the runner assembles them into metadata for `http_people_search` and the graders. + +### Pipeline + +``` +CSV row (problem + persona / query_type / … columns; answer empty) + │ + ▼ +http_people_search ──POST──► YOUR_PEOPLE_API ──► { people[], person_count } + │ (JSON string becomes generated_answer) + ▼ +deterministic scorers + LLM judges (unless PEOPLE_SEARCH_LLM_JUDGES=0) + │ + ▼ +src/evals/results/dataset_people_search_raw_results_http_people_search.csv +analyzed_results.csv (mean_field_fill / mean_judge_* / has_people_rate; accuracy_score blank) +``` + +No LLM synthesis step (`needs_synthesis=False`). Scoring operates on the structured people payload only. + +### Environment + +| Variable | Required | Purpose | +|----------|----------|---------| +| `PEOPLE_SEARCH_API_URL` | Yes | URL of your people-search HTTP endpoint | +| `PEOPLE_SEARCH_API_KEY` | No | If set (and not equal to the URL), sent as `Authorization: Bearer …` | +| `OPENAI_API_KEY` or `GOOGLE_GEMINI_API_KEY` | For LLM judges | Uses `GRADER_MODEL` (`gpt-5.4-mini` by default) | +| `PEOPLE_SEARCH_LLM_JUDGES` | No (default `1`) | Set to `0` / `false` / `off` for deterministic scorers only | + +### HTTP endpoint contract + +`http_people_search` sends: + +```http +POST $PEOPLE_SEARCH_API_URL +Content-Type: application/json +Accept: application/json +Authorization: Bearer $PEOPLE_SEARCH_API_KEY # only if PEOPLE_SEARCH_API_KEY is set +``` + +**Request body** + +```json +{ + "query": "Find work history and current role for William McKinnerney, who works at CoreWeave.", + "metadata": { + "benchmark_id": "fp_001", + "persona": "Recruiter / Talent Sourcer", + "persona_slug": "recruiter", + "query_type": "enrichment", + "person_name": "William McKinnerney", + "company": "CoreWeave", + "query_text": "Find work history and current role for William McKinnerney, who works at CoreWeave." + } +} +``` + +`metadata` is built from the CSV’s dedicated columns (`persona`, `persona_slug`, `query_type`, `person_name`, +`company`, `benchmark_id`); the sampler also sets `query_text` to the query string when missing. +`query_type` is `enrichment` (named person ± company) or `search` (open candidate list). **Your API** decides how to +route (enrich vs search, etc.) from that metadata — the eval runner does not call provider-specific endpoints. + +**Response body (canonical)** + +```json +{ + "people": [ + { + "displayname": "William McKinnerney", + "current_title": "...", + "current_company": "CoreWeave", + "location": "...", + "linkedin_url": "https://...", + "highlight": "...", + "best_work_email": "...", + "phones": ["..."], + "top_skills": ["..."], + "insights": {}, + "confidence": {"likelihood": 0.9} + } + ], + "person_count": 1, + "error": null +} +``` + +On failure, return `"error": ""` (typically with empty `people` and `person_count: 0`). HTTP status ≥400 or +non-JSON bodies are turned into an `error` payload by the sampler. + +**Accepted response variants** (normalized in `src/evals/processing/people_search/schema.py`): top-level `people`, +`summary.people` / `summary.person`, or `results` as a people list. + +Person field aliases scorers understand (examples): `headline` → title; `linkedin_url` / `url` → profile URL; +`best_work_email` / `best_personal_email` / `has_email` → email; `phones` / `has_phone` → phone; `top_skills` → skills. + +Contract source: [`src/evals/samplers/applied_samplers/people_search_sampler.py`](src/evals/samplers/applied_samplers/people_search_sampler.py). + +### Scorers + +#### Deterministic (always on) + +Implemented in `src/evals/processing/people_search/field_fill.py`. + +| Metric | Range | Meaning | +|--------|-------|---------| +| `has_people` | 0 or 1 | At least one person (`person_count` > 0). Raw `evaluation_result` is `has_people` or `no_people`. | +| `field_fill` | 0–1 | Mean fill ratio across 11 fields: displayname, current_title, current_company, location, profile_url, highlight, email, phone, skills, insights, confidence | +| `persona_field_fill` | 0–1 | Same fields, weighted by `persona_slug` (`recruiter`, `sdr`, `compliance`, `journalist`, `events`, `investor`) | + +Hard rule: if the payload has an `error` key, deterministic scores are **0**. Empty `people` also yields fill scores of **0**. + +**Primary quality metrics** for comparing providers: `mean_field_fill`, `mean_persona_field_fill`, and LLM judges below. +`has_people` / `has_people_rate` is retrieval-only. `accuracy_score` in `analyzed_results.csv` is left **blank** for +`people_search` so it is not confused with gold-answer accuracy on SimpleQA/FRAMES. + +#### LLM judges (on by default) + +Prompts: [`src/evals/processing/people_search/prompts/overall.md`](src/evals/processing/people_search/prompts/overall.md) +and [`persona.md`](src/evals/processing/people_search/prompts/persona.md). +Wired in `evaluate_single_people_search` via `src/evals/processing/people_search/llm_judges.py`. +Model: `GRADER_MODEL` in [`src/evals/constants.py`](src/evals/constants.py) (default `gpt-5.4-mini`). + +When enabled, each row runs **two** LLM calls (overall + persona). Labels map to scores: + +| Label | Score | +|-------|-------| +| High Value | 1.0 | +| Useful | 0.7 | +| Low Value | 0.3 | +| Failed | 0.0 | + +| Metric | Meaning | +|--------|---------| +| `judge_overall` | Cross-persona quality / actionability | +| `judge_persona` | Persona-switched rubric for the row’s `persona_slug` | + +Also written per row: `judge_overall_label`, `judge_persona_label`, `judge_persona_slug`. + +Disable LLM judges (deterministic only — no OpenAI/Gemini spend for this dataset): + +```bash +PEOPLE_SEARCH_LLM_JUDGES=0 python src/evals/eval_runner.py \ + --samplers http_people_search \ + --datasets people_search \ + --limit 5 +``` + +### Running + +```bash +cp .env.example .env +# Set PEOPLE_SEARCH_API_URL (+ optional PEOPLE_SEARCH_API_KEY) +# Set OPENAI_API_KEY (or GOOGLE_GEMINI_API_KEY) unless PEOPLE_SEARCH_LLM_JUDGES=0 + +# Smoke test +python src/evals/eval_runner.py \ + --samplers http_people_search \ + --datasets people_search \ + --limit 5 + +# Full benchmark (wipe prior people_search results for this sampler) +python src/evals/eval_runner.py \ + --samplers http_people_search \ + --datasets people_search \ + --clean True +``` + +`http_people_search` is excluded from the default sampler list so it is not run against SimpleQA/FRAMES by accident. + +### Results for people_search + +Raw CSV includes the usual runner fields (`query`, latencies, `generated_answer`, `ground_truth`, `evaluation_result`) +plus scorer columns: + +`has_people`, `person_count`, `field_fill`, `persona_field_fill`, `judge_overall`, `judge_overall_label`, +`judge_persona`, `judge_persona_label`, `judge_persona_slug` (judge columns omitted when LLM judges are disabled). + +`generated_answer` is the JSON people payload string. `ground_truth` is scoring metadata assembled at load time from +the dedicated CSV columns (the shipped `answer` column is empty). `evaluation_result` is `has_people` or `no_people` +(not `is_correct` / `is_incorrect`). + +`analyzed_results.csv` for this dataset: + +| Column | Meaning | +|--------|---------| +| `mean_field_fill` | Primary deterministic quality (0–1) | +| `mean_persona_field_fill` | Persona-weighted field fill (0–1) | +| `mean_judge_overall` / `mean_judge_persona` | Mean LLM judge scores when present | +| `has_people_rate` | Fraction of rows with ≥1 person (retrieval only) | +| `accuracy_score` | **Blank** (N/A — not gold-answer accuracy) | +| `p50_*_latency` / `problem_count` | Same as other datasets | + +Within `people_search`, rows are ordered by `mean_field_fill` (if missing, falls back to `mean_judge_overall`, then +`has_people_rate`). + +### Key source files + +| Path | Role | +|------|------| +| `data/people_search_full_dataset.csv` | Benchmark queries + scoring metadata | +| `src/evals/configs/datasets.py` | Registers `people_search` (`requires_ground_truth=False`) | +| `src/evals/configs/samplers.py` | Registers `http_people_search` (excluded from defaults) | +| `src/evals/samplers/applied_samplers/people_search_sampler.py` | Generic HTTP sampler + request/response contract | +| `src/evals/processing/people_search/` | Schema normalize, field-fill, previews, LLM judges, prompts | +| `src/evals/processing/evaluate_answer.py` | `evaluate_single_people_search` | +| `src/evals/eval_results_analyzer.py` | People-specific analyzed metrics (blank accuracy) | +| `tests/test_people_search.py` | Unit tests (scorers, label parse, metrics; judges off) | + ## Output Results are written to `src/evals/results/` with the following structure: @@ -207,8 +455,8 @@ src/evals/results/ ``` Raw CSVs contain per-query fields (e.g. query, generated answer, evaluation result, latencies). After a run, -`write_metrics()` is called automatically and `analyzed_results.csv` is updated with accuracy and average latency per -sampler and dataset. +`write_metrics()` is called automatically and `analyzed_results.csv` is updated. For gold-answer datasets that is +accuracy and latency; for `people_search` it is field-fill / judge means and `has_people_rate` (see above). ## Citation @@ -230,6 +478,6 @@ If you use this repository in your research, please consider citing: This repository is made available under the [MIT License](LICENSE). -[^1]: Search results are fetched from each search API, then synthesized into a single answer using an LLM; the answer is graded by an LLM judge. Synthesis uses GPT 5.4 nano and grading uses GPT 5.4 mini (configurable in `src/evals/constants.py`). +[^1]: For web-search benchmarks, search results are fetched from each search API, then synthesized into a single answer using an LLM; the answer is graded by an LLM judge. Synthesis uses GPT 5.4 nano and grading uses GPT 5.4 mini (configurable in `src/evals/constants.py`). People Search skips synthesis and scores structured `people[]` instead. [^2]: Grading uses prompts aligned with the standard benchmarks as specified in the original papers or repositories (e.g. [SimpleQA](https://openai.com/index/introducing-simpleqa/) and [FRAMES](https://arxiv.org/abs/2409.12941). [^3]: FinSearchComp grading uses the judge prompt from the [FinSearchComp paper](https://arxiv.org/pdf/2509.13160). diff --git a/data/people_search_full_dataset.csv b/data/people_search_full_dataset.csv new file mode 100644 index 0000000..41940d6 --- /dev/null +++ b/data/people_search_full_dataset.csv @@ -0,0 +1,241 @@ +benchmark_id,problem,answer,persona,persona_slug,query_type,person_name,company +fp_001,"Find work history and current role for William McKinnerney, who works at CoreWeave.",,Recruiter / Talent Sourcer,recruiter,enrichment,William McKinnerney,CoreWeave +fp_002,"Find work history and current role for Sam Branch, who works at First National Community Bank.",,Recruiter / Talent Sourcer,recruiter,enrichment,Sam Branch,First National Community Bank +fp_003,"Find work history and current role for Joseph Reuben, MD, who works at Overland Park Regional Medical Center.",,Recruiter / Talent Sourcer,recruiter,enrichment,"Joseph Reuben, MD",Overland Park Regional Medical Center +fp_004,"Find work history and current role for Daniel R. Adler, who works at Gibson Dunn.",,Recruiter / Talent Sourcer,recruiter,enrichment,Daniel R. Adler,Gibson Dunn +fp_005,"Find work history and current role for Brandon R. Thompson, who works at Jenner & Block.",,Recruiter / Talent Sourcer,recruiter,enrichment,Brandon R. Thompson,Jenner & Block +fp_006,"Find work history and current role for Lisa Graver, who works at Alvotech.",,Recruiter / Talent Sourcer,recruiter,enrichment,Lisa Graver,Alvotech +fp_007,"Find work history and current role for Dr. Scott Drutman, who works at Enara Bio.",,Recruiter / Talent Sourcer,recruiter,enrichment,Dr. Scott Drutman,Enara Bio +fp_008,"Find work history and current role for Chris Peters, P.E., S.E., who works at Michael Baker International.",,Recruiter / Talent Sourcer,recruiter,enrichment,"Chris Peters, P.E., S.E.",Michael Baker International +fp_009,"Find work history and current role for Andrew Bulluck, PE, who works at Nobis Group.",,Recruiter / Talent Sourcer,recruiter,enrichment,"Andrew Bulluck, PE",Nobis Group +fp_010,"Find work history and current role for Helen Ayotte, who works at New Jersey Natural Gas.",,Recruiter / Talent Sourcer,recruiter,enrichment,Helen Ayotte,New Jersey Natural Gas +fp_011,"Find work history and current role for Ari Sulby, who works at Semiconductor Industry Association.",,Recruiter / Talent Sourcer,recruiter,enrichment,Ari Sulby,Semiconductor Industry Association +fp_012,"Find work history and current role for Martin Dowling, who works at Altibox Carrier.",,Recruiter / Talent Sourcer,recruiter,enrichment,Martin Dowling,Altibox Carrier +fp_013,"Find work history and current role for Nicholas A. Charles, who works at Cutter Aviation.",,Recruiter / Talent Sourcer,recruiter,enrichment,Nicholas A. Charles,Cutter Aviation +fp_014,"Find work history and current role for Deborah Gorgulho, who works at Soledad Unified School District.",,Recruiter / Talent Sourcer,recruiter,enrichment,Deborah Gorgulho,Soledad Unified School District +fp_015,"Find work history and current role for Nathan Mustafa, who works at City of Riverside.",,Recruiter / Talent Sourcer,recruiter,enrichment,Nathan Mustafa,City of Riverside +fp_016,Find backend engineers with Kubernetes experience at cloud infrastructure companies in Austin.,,Recruiter / Talent Sourcer,recruiter,search,, +fp_017,Find mechanical or engineering leads at family-owned metal fabrication shops in the Midwest.,,Recruiter / Talent Sourcer,recruiter,search,, +fp_018,Find architects with FAIA credentials at boutique firms in the Southeast US.,,Recruiter / Talent Sourcer,recruiter,search,, +fp_019,Find hospital CFOs or VPs of finance open to new opportunities.,,Recruiter / Talent Sourcer,recruiter,search,, +fp_020,Find corporate attorneys with CFIUS or national security law experience in DC.,,Recruiter / Talent Sourcer,recruiter,search,, +fp_021,Find M&A partners at Am Law 100 firms in London.,,Recruiter / Talent Sourcer,recruiter,search,, +fp_022,Find environmental litigation attorneys based in Orange County.,,Recruiter / Talent Sourcer,recruiter,search,, +fp_023,Find product managers at major consumer tech companies with biology or life sciences backgrounds.,,Recruiter / Talent Sourcer,recruiter,search,, +fp_024,Find engineering product managers who graduated from University of Oregon.,,Recruiter / Talent Sourcer,recruiter,search,, +fp_025,Find CIOs at mid-size regional hospital systems.,,Recruiter / Talent Sourcer,recruiter,search,, +fp_026,Find special counsel with FCC enforcement backgrounds at large law firms.,,Recruiter / Talent Sourcer,recruiter,search,, +fp_027,Find tax partners with UK taxation expertise in London.,,Recruiter / Talent Sourcer,recruiter,search,, +fp_028,Find appellate litigators with Supreme Court experience in Los Angeles.,,Recruiter / Talent Sourcer,recruiter,search,, +fp_029,Find civil engineers or project leads at small manufacturing companies in Ohio.,,Recruiter / Talent Sourcer,recruiter,search,, +fp_030,Find demand generation leaders at Series B-funded SaaS startups.,,Recruiter / Talent Sourcer,recruiter,search,, +fp_031,Find architecture firm principals who've won AIA design awards.,,Recruiter / Talent Sourcer,recruiter,search,, +fp_032,Find UT Austin computer science alumni now working at infrastructure startups.,,Recruiter / Talent Sourcer,recruiter,search,, +fp_033,Find nurse executives with DNP credentials open to CNO roles.,,Recruiter / Talent Sourcer,recruiter,search,, +fp_034,Find hospital chief medical officers within HCA-affiliated systems.,,Recruiter / Talent Sourcer,recruiter,search,, +fp_035,Find engineers at AI infrastructure startups with GPU or data center experience.,,Recruiter / Talent Sourcer,recruiter,search,, +fp_036,Find structural or civil engineers at boutique architecture practices.,,Recruiter / Talent Sourcer,recruiter,search,, +fp_037,Find fourth-generation leaders at family-owned manufacturing businesses.,,Recruiter / Talent Sourcer,recruiter,search,, +fp_038,Find engineering leads at metal fabrication or industrial manufacturing companies.,,Recruiter / Talent Sourcer,recruiter,search,, +fp_039,Find CNOs with perioperative or surgical nursing backgrounds.,,Recruiter / Talent Sourcer,recruiter,search,, +fp_040,Find pharmacy directors with PharmD credentials open to health-system leadership roles.,,Recruiter / Talent Sourcer,recruiter,search,, +fp_041,Find contact information for Alla Oks who works at Foley.,,SDR / BDR,sdr,enrichment,Alla Oks,Foley +fp_042,Find contact information for Amanda L. Engles who works at CB Financial Services / Community Bank.,,SDR / BDR,sdr,enrichment,Amanda L. Engles,CB Financial Services / Community Bank +fp_043,Find contact information for Richard Dabruzzo who works at WVU Health System.,,SDR / BDR,sdr,enrichment,Richard Dabruzzo,WVU Health System +fp_044,Find contact information for Angie Longing who works at Conway Regional Health System.,,SDR / BDR,sdr,enrichment,Angie Longing,Conway Regional Health System +fp_045,Find contact information for Stephanie McDonell who works at United Regional.,,SDR / BDR,sdr,enrichment,Stephanie McDonell,United Regional +fp_046,Find contact information for Chris Molloy who works at BioIndustry Association.,,SDR / BDR,sdr,enrichment,Chris Molloy,BioIndustry Association +fp_047,Find contact information for Eugene Zollinger who works at Assurity.,,SDR / BDR,sdr,enrichment,Eugene Zollinger,Assurity +fp_048,Find contact information for Michael J. Cusack who works at Alliant Insurance Services.,,SDR / BDR,sdr,enrichment,Michael J. Cusack,Alliant Insurance Services +fp_049,Find contact information for Nicholas Gesue who works at Greystone.,,SDR / BDR,sdr,enrichment,Nicholas Gesue,Greystone +fp_050,Find contact information for Allison Davies who works at NAI Hallmark.,,SDR / BDR,sdr,enrichment,Allison Davies,NAI Hallmark +fp_051,Find contact information for Jim Filter who works at Schneider.,,SDR / BDR,sdr,enrichment,Jim Filter,Schneider +fp_052,Find contact information for Ian Ferry who works at Grocery Outlet.,,SDR / BDR,sdr,enrichment,Ian Ferry,Grocery Outlet +fp_053,Find contact information for Marissa Ghesquiere who works at Sotheby's International Realty.,,SDR / BDR,sdr,enrichment,Marissa Ghesquiere,Sotheby's International Realty +fp_054,Find contact information for Randy Macchi who works at Houston Public Works.,,SDR / BDR,sdr,enrichment,Randy Macchi,Houston Public Works +fp_055,Find contact information for Matt Severson who works at Premier Cooperative.,,SDR / BDR,sdr,enrichment,Matt Severson,Premier Cooperative +fp_056,Find newly hired VPs of Marketing at compliance and safety SaaS companies to prospect.,,SDR / BDR,sdr,search,, +fp_057,Find CFOs at bank holding companies who were promoted in the last 90 days.,,SDR / BDR,sdr,search,, +fp_058,Find boutique architecture firm founders in the Southeast US as prospects.,,SDR / BDR,sdr,search,, +fp_059,Find CNOs at regional hospitals who just started their roles.,,SDR / BDR,sdr,search,, +fp_060,Find CIOs at mid-size hospital systems investing in smart-room technology.,,SDR / BDR,sdr,search,, +fp_061,Find law firm partners in DC who focus on regulatory and compliance work.,,SDR / BDR,sdr,search,, +fp_062,Find M&A partners at large law firms who recently made partner.,,SDR / BDR,sdr,search,, +fp_063,Find engineering leaders at cloud infrastructure startups in Austin.,,SDR / BDR,sdr,search,, +fp_064,Find healthcare CEOs at nonprofit rural health systems.,,SDR / BDR,sdr,search,, +fp_065,Find newly promoted partners at large law firms across all offices.,,SDR / BDR,sdr,search,, +fp_066,Find family-owned manufacturing companies with fourth-generation leadership.,,SDR / BDR,sdr,search,, +fp_067,Find marketing leaders who moved from biotech SaaS to compliance tech.,,SDR / BDR,sdr,search,, +fp_068,Find hospital CMOs within HCA-affiliated regional medical centers.,,SDR / BDR,sdr,search,, +fp_069,Find architecture firms that recently won AIA Design Excellence awards.,,SDR / BDR,sdr,search,, +fp_070,Find engineering VPs at GPU cloud or AI infrastructure companies.,,SDR / BDR,sdr,search,, +fp_071,Find newly appointed presidents/CEOs at regional health systems.,,SDR / BDR,sdr,search,, +fp_072,Find tax partners at global law firms with UK/EU tax specialization.,,SDR / BDR,sdr,search,, +fp_073,Find nurse executives newly promoted to CNO at community hospitals.,,SDR / BDR,sdr,search,, +fp_074,Find metal fabrication or industrial manufacturing companies open to new vendor relationships.,,SDR / BDR,sdr,search,, +fp_075,Find compliance officers at mid-size regional banks.,,SDR / BDR,sdr,search,, +fp_076,Find product marketers at Series A/B SaaS startups focused on regulated industries.,,SDR / BDR,sdr,search,, +fp_077,Find national security or CFIUS-focused attorneys at DC law firms.,,SDR / BDR,sdr,search,, +fp_078,Find demand generation VPs who've scaled GTM at compliance-focused platforms.,,SDR / BDR,sdr,search,, +fp_079,Find CIOs or VPs of technology at regional healthcare nonprofits.,,SDR / BDR,sdr,search,, +fp_080,Find newly elevated communications/telecom regulatory counsel at major firms.,,SDR / BDR,sdr,search,, +fp_081,"Verify the identity and employment of Anne Marie Duvall Decker, FAIA, who works at Duvall Decker Architects.",,Background Check / Compliance Analyst,compliance,enrichment,"Anne Marie Duvall Decker, FAIA",Duvall Decker Architects +fp_082,"Verify the identity and employment of Ivan A. Schlager, who works at Kirkland & Ellis LLP.",,Background Check / Compliance Analyst,compliance,enrichment,Ivan A. Schlager,Kirkland & Ellis LLP +fp_083,"Verify the identity and employment of Ella Duggan, who works at The Merrimack.",,Background Check / Compliance Analyst,compliance,enrichment,Ella Duggan,The Merrimack +fp_084,"Verify the identity and employment of Mary Ann Keener, BSN, RN, who works at Great Bend Regional Hospital.",,Background Check / Compliance Analyst,compliance,enrichment,"Mary Ann Keener, BSN, RN",Great Bend Regional Hospital +fp_085,"Verify the identity and employment of Tonya Washington, DNP, RN, who works at Cedar Hill Regional Medical Center GW Health.",,Background Check / Compliance Analyst,compliance,enrichment,"Tonya Washington, DNP, RN",Cedar Hill Regional Medical Center GW Health +fp_086,"Verify the identity and employment of Joe Franklin, JD, PhD, who works at Biotechnology Innovation Organization (BIO).",,Background Check / Compliance Analyst,compliance,enrichment,"Joe Franklin, JD, PhD",Biotechnology Innovation Organization (BIO) +fp_087,"Verify the identity and employment of Jared Strong, who works at Inszone Insurance Services.",,Background Check / Compliance Analyst,compliance,enrichment,Jared Strong,Inszone Insurance Services +fp_088,"Verify the identity and employment of John Myers, P.E., who works at Menard USA.",,Background Check / Compliance Analyst,compliance,enrichment,"John Myers, P.E.",Menard USA +fp_089,"Verify the identity and employment of John Frazier, who works at Baker Katz.",,Background Check / Compliance Analyst,compliance,enrichment,John Frazier,Baker Katz +fp_090,"Verify the identity and employment of Lisa Edison-Smith, who works at North Dakota Ethics Commission.",,Background Check / Compliance Analyst,compliance,enrichment,Lisa Edison-Smith,North Dakota Ethics Commission +fp_091,"Verify the identity and employment of Jeffrey Akers, PharmD, who works at UC Health.",,Background Check / Compliance Analyst,compliance,enrichment,"Jeffrey Akers, PharmD",UC Health +fp_092,"Verify the identity and employment of Melissa Chase, PharmD, who works at Valley Children's Healthcare.",,Background Check / Compliance Analyst,compliance,enrichment,"Melissa Chase, PharmD",Valley Children's Healthcare +fp_093,"Verify the identity and employment of John Coggins, who works at Mary Washington Healthcare.",,Background Check / Compliance Analyst,compliance,enrichment,John Coggins,Mary Washington Healthcare +fp_094,"Verify the identity and employment of Giancarlo Ferro, who works at euNetworks.",,Background Check / Compliance Analyst,compliance,enrichment,Giancarlo Ferro,euNetworks +fp_095,"Verify the identity and employment of Keith Thomas, who works at Cabell County Schools.",,Background Check / Compliance Analyst,compliance,enrichment,Keith Thomas,Cabell County Schools +fp_096,Verify bar admission and current firm for attorneys practicing CFIUS law in DC.,,Background Check / Compliance Analyst,compliance,search,, +fp_097,Confirm licensure status for FAIA-credentialed architects in Mississippi.,,Background Check / Compliance Analyst,compliance,search,, +fp_098,Confirm registration and standing for partners at Am Law 100 firms in London.,,Background Check / Compliance Analyst,compliance,search,, +fp_099,Verify professional licensure for registered nurses newly named to CNO roles.,,Background Check / Compliance Analyst,compliance,search,, +fp_100,Cross-check public regulatory filings tied to attorneys with national security practices.,,Background Check / Compliance Analyst,compliance,search,, +fp_101,Verify current employer and title for engineers at cloud infrastructure companies.,,Background Check / Compliance Analyst,compliance,search,, +fp_102,Confirm AIA state chapter membership status for architecture firm principals.,,Background Check / Compliance Analyst,compliance,search,, +fp_103,Verify medical licensure for physicians newly named chief medical officer.,,Background Check / Compliance Analyst,compliance,search,, +fp_104,Confirm bar admission status for special counsel hires at major law firms.,,Background Check / Compliance Analyst,compliance,search,, +fp_105,Verify professional credentials for tax law partners practicing UK taxation.,,Background Check / Compliance Analyst,compliance,search,, +fp_106,Cross-check licensure for civil/structural engineers tied to manufacturing firms.,,Background Check / Compliance Analyst,compliance,search,, +fp_107,Confirm employment history for hospital CIOs newly appointed in 2026.,,Background Check / Compliance Analyst,compliance,search,, +fp_108,Verify current title and firm for environmental litigation attorneys in California.,,Background Check / Compliance Analyst,compliance,search,, +fp_109,Confirm bar admission for appellate litigators recently elevated to partner.,,Background Check / Compliance Analyst,compliance,search,, +fp_110,Confirm licensure status for architecture principals with public sector project history.,,Background Check / Compliance Analyst,compliance,search,, +fp_111,Verify employment history for CFOs at bank holding companies.,,Background Check / Compliance Analyst,compliance,search,, +fp_112,Cross-check public regulatory disclosures for partners in energy-sector litigation.,,Background Check / Compliance Analyst,compliance,search,, +fp_113,Confirm professional standing for attorneys practicing mass tort litigation.,,Background Check / Compliance Analyst,compliance,search,, +fp_114,Verify current title for engineering leads at data center/cloud infrastructure firms.,,Background Check / Compliance Analyst,compliance,search,, +fp_115,Confirm licensure for physicians serving as CMO at HCA-affiliated hospitals.,,Background Check / Compliance Analyst,compliance,search,, +fp_116,Verify bar registration for partners specializing in aerospace and defense law.,,Background Check / Compliance Analyst,compliance,search,, +fp_117,Verify credentials for architects listed as AIA Design Excellence award winners.,,Background Check / Compliance Analyst,compliance,search,, +fp_118,Cross-check licensure status for RNs newly promoted into nursing leadership.,,Background Check / Compliance Analyst,compliance,search,, +fp_119,Confirm current firm for M&A attorneys handling cross-border transactions.,,Background Check / Compliance Analyst,compliance,search,, +fp_120,Verify current employer and licensure for chief information officers in healthcare.,,Background Check / Compliance Analyst,compliance,search,, +fp_121,"Find background information on Ben Hickey, who works at Hickey Metal Fabrication.",,Journalist / Investigative Researcher,journalist,enrichment,Ben Hickey,Hickey Metal Fabrication +fp_122,"Find background information on Paige Martin, who works at TowneBank.",,Journalist / Investigative Researcher,journalist,enrichment,Paige Martin,TowneBank +fp_123,"Find background information on Renee Clancy, who works at Martha's Vineyard Hospital.",,Journalist / Investigative Researcher,journalist,enrichment,Renee Clancy,Martha's Vineyard Hospital +fp_124,"Find background information on Ken Burgess, who works at Rural Health Services.",,Journalist / Investigative Researcher,journalist,enrichment,Ken Burgess,Rural Health Services +fp_125,"Find background information on Harald Hampel, who works at Bristol Myers Squibb.",,Journalist / Investigative Researcher,journalist,enrichment,Harald Hampel,Bristol Myers Squibb +fp_126,"Find background information on Bo Rode Hansen, who works at LIfT BioSciences.",,Journalist / Investigative Researcher,journalist,enrichment,Bo Rode Hansen,LIfT BioSciences +fp_127,"Find background information on Eric Hegarty, who works at LevRose Commercial Real Estate.",,Journalist / Investigative Researcher,journalist,enrichment,Eric Hegarty,LevRose Commercial Real Estate +fp_128,"Find background information on Michael Massey, who works at Worth Credit Union.",,Journalist / Investigative Researcher,journalist,enrichment,Michael Massey,Worth Credit Union +fp_129,"Find background information on Chuck Casassa, who works at Market Basket.",,Journalist / Investigative Researcher,journalist,enrichment,Chuck Casassa,Market Basket +fp_130,"Find background information on Todd Schnuck Jr., who works at Schnuck Markets.",,Journalist / Investigative Researcher,journalist,enrichment,Todd Schnuck Jr.,Schnuck Markets +fp_131,"Find background information on Bruce Robinson, who works at Stater Bros.",,Journalist / Investigative Researcher,journalist,enrichment,Bruce Robinson,Stater Bros. +fp_132,"Find background information on Jose Coll, who works at Western New Mexico University.",,Journalist / Investigative Researcher,journalist,enrichment,Jose Coll,Western New Mexico University +fp_133,"Find background information on Michel Denis, who works at Daher.",,Journalist / Investigative Researcher,journalist,enrichment,Michel Denis,Daher +fp_134,"Find background information on Lauren Thompson, who works at Friends of Flight 93 National Memorial.",,Journalist / Investigative Researcher,journalist,enrichment,Lauren Thompson,Friends of Flight 93 National Memorial +fp_135,"Find background information on Amy Zeng, who works at Chapman University.",,Journalist / Investigative Researcher,journalist,enrichment,Amy Zeng,Chapman University +fp_136,Find recently appointed board members at national security policy nonprofits.,,Journalist / Investigative Researcher,journalist,search,, +fp_137,Find architects who've won AIA awards for public/civic projects in the South.,,Journalist / Investigative Researcher,journalist,search,, +fp_138,Find hospital executives newly appointed amid regional health system consolidation.,,Journalist / Investigative Researcher,journalist,search,, +fp_139,Find family-owned manufacturers navigating succession to a new generation.,,Journalist / Investigative Researcher,journalist,search,, +fp_140,Find M&A partners involved in recent private equity infrastructure deals.,,Journalist / Investigative Researcher,journalist,search,, +fp_141,Find nonprofit health system CEOs appointed amid rural hospital closures.,,Journalist / Investigative Researcher,journalist,search,, +fp_142,"Find tech product managers with unconventional academic backgrounds (e.g., biology).",,Journalist / Investigative Researcher,journalist,search,, +fp_143,Find attorneys with CFIUS expertise commenting on foreign investment reviews.,,Journalist / Investigative Researcher,journalist,search,, +fp_144,Find hospital CIOs piloting new patient-care technology.,,Journalist / Investigative Researcher,journalist,search,, +fp_145,Find architecture firms recognized for public-good or social-impact design work.,,Journalist / Investigative Researcher,journalist,search,, +fp_146,Find recently elevated law firm partners in communications and internet law.,,Journalist / Investigative Researcher,journalist,search,, +fp_147,Find university alumni featured in career spotlight pieces now at major tech firms.,,Journalist / Investigative Researcher,journalist,search,, +fp_148,Find manufacturing company leaders discussing tariffs or supply chain disruption.,,Journalist / Investigative Researcher,journalist,search,, +fp_149,Find nurse executives speaking publicly about hospital staffing crises.,,Journalist / Investigative Researcher,journalist,search,, +fp_150,Find newly appointed nonprofit CEOs discussing community health access.,,Journalist / Investigative Researcher,journalist,search,, +fp_151,Find engineering leaders discussing data center energy consumption.,,Journalist / Investigative Researcher,journalist,search,, +fp_152,Find architecture principals serving on state AIA leadership boards.,,Journalist / Investigative Researcher,journalist,search,, +fp_153,Find hospital system presidents discussing recent leadership transitions.,,Journalist / Investigative Researcher,journalist,search,, +fp_154,Find fourth-generation family business owners discussing generational transitions.,,Journalist / Investigative Researcher,journalist,search,, +fp_155,Find hospital CIOs discussing AI adoption in patient care.,,Journalist / Investigative Researcher,journalist,search,, +fp_156,Find tech companies' product leads discussing GPU/cloud infrastructure growth.,,Journalist / Investigative Researcher,journalist,search,, +fp_157,Find manufacturing executives discussed in regional business press profiles.,,Journalist / Investigative Researcher,journalist,search,, +fp_158,Find nonprofit health CEOs discussed in coverage of rural healthcare access.,,Journalist / Investigative Researcher,journalist,search,, +fp_159,Find architecture award winners profiled in trade publications like Architizer.,,Journalist / Investigative Researcher,journalist,search,, +fp_160,Find grocery industry executives discussed in coverage of leadership transitions.,,Journalist / Investigative Researcher,journalist,search,, +fp_161,"Find contact information for Ann Diep, who works at Apple.",,Event Organizer / Community Manager,events,enrichment,Ann Diep,Apple +fp_162,"Find contact information for Sandy Starnes, who works at First Community Bank.",,Event Organizer / Community Manager,events,enrichment,Sandy Starnes,First Community Bank +fp_163,"Find contact information for Angela Ross, BSN, RN, who works at Iredell Health System.",,Event Organizer / Community Manager,events,enrichment,"Angela Ross, BSN, RN",Iredell Health System +fp_164,"Find contact information for Joe Webster, who works at JP Insurance Group.",,Event Organizer / Community Manager,events,enrichment,Joe Webster,JP Insurance Group +fp_165,"Find contact information for Randy Fink, who works at Colliers.",,Event Organizer / Community Manager,events,enrichment,Randy Fink,Colliers +fp_166,"Find contact information for Andee Robb, who works at Marcus & Millichap.",,Event Organizer / Community Manager,events,enrichment,Andee Robb,Marcus & Millichap +fp_167,"Find contact information for Todd Harrison, who works at Alfred University.",,Event Organizer / Community Manager,events,enrichment,Todd Harrison,Alfred University +fp_168,"Find contact information for Remzi Arpaci-Dusseau, who works at University of Wisconsin-Madison.",,Event Organizer / Community Manager,events,enrichment,Remzi Arpaci-Dusseau,University of Wisconsin-Madison +fp_169,"Find contact information for Brian Proctor, who works at Leeds Hospitality Group.",,Event Organizer / Community Manager,events,enrichment,Brian Proctor,Leeds Hospitality Group +fp_170,"Find contact information for Nikhil Heda, who works at G6 Hospitality.",,Event Organizer / Community Manager,events,enrichment,Nikhil Heda,G6 Hospitality +fp_171,"Find contact information for Vukie Mpofu, who works at Nashville Predators.",,Event Organizer / Community Manager,events,enrichment,Vukie Mpofu,Nashville Predators +fp_172,"Find contact information for Paul Cox, who works at Food City.",,Event Organizer / Community Manager,events,enrichment,Paul Cox,Food City +fp_173,"Find contact information for Brian Knifong, who works at Agtegra Cooperative.",,Event Organizer / Community Manager,events,enrichment,Brian Knifong,Agtegra Cooperative +fp_174,"Find contact information for Nicholas Vescovo, who works at Youth Villages.",,Event Organizer / Community Manager,events,enrichment,Nicholas Vescovo,Youth Villages +fp_175,"Find contact information for Susan Field, who works at Academy School District 20.",,Event Organizer / Community Manager,events,enrichment,Susan Field,Academy School District 20 +fp_176,Find University of Oregon Class of 2015 biology alumni now working in tech product roles.,,Event Organizer / Community Manager,events,search,, +fp_177,Find nursing school alumni who became chief nursing officers in the past 12 months.,,Event Organizer / Community Manager,events,search,, +fp_178,"Find donors who gave $5,000 or more to a hospital foundation capital campaign but haven't donated in the past 24 months.",,Event Organizer / Community Manager,events,search,, +fp_179,Find AIA chapter members who haven't attended a chapter event in the past 18 months.,,Event Organizer / Community Manager,events,search,, +fp_180,"Find donors who gave $1,000 or more to an arts organization's 2025 annual fund.",,Event Organizer / Community Manager,events,search,, +fp_181,Find credit union association members whose dues lapsed in the past 6 months.,,Event Organizer / Community Manager,events,search,, +fp_182,Find founding donors from a university's 2015 capital campaign to invite to a 10-year building dedication.,,Event Organizer / Community Manager,events,search,, +fp_183,Find past fintech conference speakers from compliance SaaS companies to invite for 2026.,,Event Organizer / Community Manager,events,search,, +fp_184,Find hospital CNOs in the Southeast who'd be strong panelists for a staffing-shortage panel.,,Event Organizer / Community Manager,events,search,, +fp_185,Find biotech founders from Series A or B companies who could speak on drug development timelines.,,Event Organizer / Community Manager,events,search,, +fp_186,Find licensed PEs who've published on infrastructure resilience for a civil engineering symposium.,,Event Organizer / Community Manager,events,search,, +fp_187,Find community banks that sponsored a regional banking conference in the past 3 years.,,Event Organizer / Community Manager,events,search,, +fp_188,Find insurance carriers that have sponsored events for a regional chamber of commerce.,,Event Organizer / Community Manager,events,search,, +fp_189,Find regional hospital systems that sponsor nursing conferences in Ohio.,,Event Organizer / Community Manager,events,search,, +fp_190,Find RNs with 10+ years tenure at regional hospitals for a nursing excellence award nomination.,,Event Organizer / Community Manager,events,search,, +fp_191,Find PE-licensed engineers under 40 for an emerging civil engineer award nomination.,,Event Organizer / Community Manager,events,search,, +fp_192,Find nonprofit executive directors in the Pacific Northwest who might join a hospital foundation board.,,Event Organizer / Community Manager,events,search,, +fp_193,Find hospital CFOs or VPs of Finance in the Midwest open to serving on a healthcare foundation board.,,Event Organizer / Community Manager,events,search,, +fp_194,Find attorneys with prior nonprofit board experience for a legal aid organization's board.,,Event Organizer / Community Manager,events,search,, +fp_195,Find manufacturing CEOs from family-owned businesses in the Midwest for a keynote panel.,,Event Organizer / Community Manager,events,search,, +fp_196,Find community bank CFOs in Texas who could keynote a financial literacy event.,,Event Organizer / Community Manager,events,search,, +fp_197,Find insurance regional VPs for a risk management conference keynote slot.,,Event Organizer / Community Manager,events,search,, +fp_198,Find alumni engineers willing to mentor current engineering students.,,Event Organizer / Community Manager,events,search,, +fp_199,Find retired RNs or nurse executives interested in volunteering at a hospital foundation gala.,,Event Organizer / Community Manager,events,search,, +fp_200,Find law firm associates in Chicago interested in volunteering for a bar foundation's pro bono night.,,Event Organizer / Community Manager,events,search,, +fp_201,Find Obi Felten's professional background and prior ventures before founding Flourish Labs.,,VC / PE / Investor,investor,enrichment,Obi Felten,Flourish Labs +fp_202,Find Travis Betters' ownership history and background as founder of Brothers International.,,VC / PE / Investor,investor,enrichment,Travis Betters,Brothers International +fp_203,Find Matt Rojas' prior work experience and Stanford research background before co-founding Lantern.,,VC / PE / Investor,investor,enrichment,Matt Rojas,Lantern +fp_204,Find Des Traynor's background and prior roles before co-founding Intercom.,,VC / PE / Investor,investor,enrichment,Des Traynor,Intercom +fp_205,"Find John Maraganore's executive history, including his time as CEO of Alnylam, before founding City Therapeutics.",,VC / PE / Investor,investor,enrichment,John Maraganore,City Therapeutics +fp_206,Find Logan Freeman's professional background as founder of Midwest CRE Advisors.,,VC / PE / Investor,investor,enrichment,Logan Freeman,Midwest CRE Advisors +fp_207,Find Gabriel Jones' professional background and MBA history before founding Proprio.,,VC / PE / Investor,investor,enrichment,Gabriel Jones,Proprio +fp_208,Find Samir Manjure's prior experience at Microsoft before co-founding KenSci.,,VC / PE / Investor,investor,enrichment,Samir Manjure,KenSci +fp_209,Find Doug Cusick's professional background as CEO of T4M.,,VC / PE / Investor,investor,enrichment,Doug Cusick,T4M +fp_210,Find Jenny Duan's academic and professional background before co-founding Clair Health.,,VC / PE / Investor,investor,enrichment,Jenny Duan,Clair Health +fp_211,Find Dallen Allred's professional background as co-founder and CEO of Tava Health.,,VC / PE / Investor,investor,enrichment,Dallen Allred,Tava Health +fp_212,Find Dr. Shiv Rao's clinical and professional background before founding Abridge.,,VC / PE / Investor,investor,enrichment,Dr. Shiv Rao,Abridge +fp_213,Find Marty Kuhn's professional background as the first outside CEO of Mike Albert Fleet Solutions.,,VC / PE / Investor,investor,enrichment,Marty Kuhn,Mike Albert Fleet Solutions +fp_214,Find Kevin Kia's clinical and executive background as Chief Medical Officer of Platinum Dermatology Partners.,,VC / PE / Investor,investor,enrichment,Kevin Kia,Platinum Dermatology Partners +fp_215,Find Junaid Zaheer's deal history as an Operating Partner at Sun Capital Partners.,,VC / PE / Investor,investor,enrichment,Junaid Zaheer,Sun Capital Partners +fp_216,Find the founding team and executive bios for Seapoint.,,VC / PE / Investor,investor,search,, +fp_217,Find the founding team behind Lightspeed portfolio company Pie.,,VC / PE / Investor,investor,search,, +fp_218,Find the founding team of Arca.,,VC / PE / Investor,investor,search,, +fp_219,"Find the founding team of 1001, the Dubai- and London-based AI startup.",,VC / PE / Investor,investor,search,, +fp_220,Find the co-founders of Quantifind.,,VC / PE / Investor,investor,search,, +fp_221,Find the executive team at Antora Energy.,,VC / PE / Investor,investor,search,, +fp_222,Find the CFO and finance leadership at BeZero Carbon.,,VC / PE / Investor,investor,search,, +fp_223,Find limited partners who have invested in Chemistry Ventures' funds.,,VC / PE / Investor,investor,search,, +fp_224,Find operating partners at THL Partners who serve as board advisors to portfolio companies.,,VC / PE / Investor,investor,search,, +fp_225,Find the founding team behind Mind Robotics.,,VC / PE / Investor,investor,search,, +fp_226,Find executives who left Redfin in the past two years to found real estate startups.,,VC / PE / Investor,investor,search,, +fp_227,Find the management team at Quince following its $500 million Series E.,,VC / PE / Investor,investor,search,, +fp_228,Find venture partners at Bessemer Venture Partners who focus on enterprise software.,,VC / PE / Investor,investor,search,, +fp_229,Find former Sequoia Capital investors now operating at other venture firms.,,VC / PE / Investor,investor,search,, +fp_230,Find the founding teams of companies backed by Lux Capital in the fintech sector.,,VC / PE / Investor,investor,search,, +fp_231,Find board members of Insight Partners' healthcare services portfolio companies.,,VC / PE / Investor,investor,search,, +fp_232,Find the executive teams at Bain Capital Ventures' portfolio companies in proptech.,,VC / PE / Investor,investor,search,, +fp_233,Find angel investors who backed One Raven's seed round.,,VC / PE / Investor,investor,search,, +fp_234,Find co-founders of companies that raised funding alongside Accel in 2026.,,VC / PE / Investor,investor,search,, +fp_235,Find the leadership team at a mid-size civil engineering firm for a potential infrastructure-services roll-up.,,VC / PE / Investor,investor,search,, +fp_236,Find executives at engineering consulting firms who could evaluate an infrastructure acquisition target.,,VC / PE / Investor,investor,search,, +fp_237,Find the founding teams of companies that exited stealth with fintech products in 2026.,,VC / PE / Investor,investor,search,, +fp_238,Find operating executives who previously worked at Stripe now available for portfolio company roles.,,VC / PE / Investor,investor,search,, +fp_239,Find founders who previously exited a company before starting a new venture-backed startup.,,VC / PE / Investor,investor,search,, +fp_240,Find co-founders of companies that completed reverse mergers to go public in 2026.,,VC / PE / Investor,investor,search,, diff --git a/src/evals/configs/datasets.py b/src/evals/configs/datasets.py index 680af35..13a0969 100644 --- a/src/evals/configs/datasets.py +++ b/src/evals/configs/datasets.py @@ -17,6 +17,8 @@ class Dataset: csv_path: str grader: Callable df: pd.DataFrame | None + # When False, empty answer/ground_truth is allowed (scorer-based datasets). + requires_ground_truth: bool = True DATASETS = [ @@ -56,4 +58,13 @@ class Dataset: grader=fin_search_evaluator.evaluate_single_fin_search, df=None, ), + Dataset( + dataset_name="people_search", + csv_path="data/people_search_full_dataset.csv", + grader=evaluator.evaluate_single_people_search, + df=None, + # answer column is empty in the CSV (no gold answers); metadata lives in + # dedicated columns and is assembled at load time in utils.get_dataset + requires_ground_truth=False, + ), ] diff --git a/src/evals/configs/samplers.py b/src/evals/configs/samplers.py index e59db08..a6a19f5 100644 --- a/src/evals/configs/samplers.py +++ b/src/evals/configs/samplers.py @@ -22,6 +22,7 @@ YouResearchSampler, YouSearchSnippetsSampler, ) +from evals.samplers.applied_samplers.people_search_sampler import HttpPeopleSearchSampler SAMPLERS = [ @@ -148,10 +149,24 @@ search_effort="high", timeout=3000, ), + # Generic people-search HTTP endpoint (scorer-based; excluded from defaults) + HttpPeopleSearchSampler( + sampler_name="http_people_search", + api_url=os.getenv("PEOPLE_SEARCH_API_URL"), + api_key=os.getenv("PEOPLE_SEARCH_API_KEY"), + timeout=120, + max_concurrency=5, + ), ] -# Samplers excluded from default runs due to high cost or long latency -EXCLUDE_KEYWORDS = ["research", "parallel_pro", "parallel_ultra", 'perplexity_finance_historical_lookup'] +# Samplers excluded from default runs due to high cost, long latency, or special datasets +EXCLUDE_KEYWORDS = [ + "research", + "parallel_pro", + "parallel_ultra", + "perplexity_finance_historical_lookup", + "http_people_search", +] NON_RESEARCH_SAMPLERS = [ sampler.sampler_name diff --git a/src/evals/eval_results_analyzer.py b/src/evals/eval_results_analyzer.py index dd6c0b6..4321f5f 100644 --- a/src/evals/eval_results_analyzer.py +++ b/src/evals/eval_results_analyzer.py @@ -34,10 +34,21 @@ def get_results_files(results_dir: Optional[Path] = None) -> List[str]: return glob.glob(f"{results_dir}/dataset_*.csv") +def _mean_numeric(series: pd.Series) -> float | None: + values = pd.to_numeric(series, errors="coerce").dropna() + if len(values) == 0: + return None + return float(values.mean()) + + def write_metrics(results_dir: Optional[Path] = None): """ Calculate metrics from raw results such as accuracy score, P50 latency, and average latency. + For people_search (scorer-based), ``accuracy_score`` is left blank — primary quality + metrics are ``mean_field_fill``, ``mean_persona_field_fill``, and ``mean_judge_*``, + with ``has_people_rate`` as a separate retrieval signal (not “accuracy”). + Args: results_dir: Optional path to results directory. Defaults to src/evals/results """ @@ -69,33 +80,72 @@ def write_metrics(results_dir: Optional[Path] = None): .dropna() .median() ) - correct = len( - df_sampler_results[df_sampler_results["evaluation_result"] == "is_correct"] - ) count_answered = len(successful_df) if count_answered == 0: raise ValueError(f"No successful results found for sampler {sampler_name}") - accuracy_score = round((correct / count_answered) * 100, 2) - - metric_rows.append( - { - "provider": sampler_name, - "dataset": dataset_name, - "accuracy_score": accuracy_score, - "p50_internal_latency": round(float(p50_internal_latency), 2), - "p50_request_response_latency": round( - float(p50_request_response_latency), 2 - ), - "problem_count": count_answered, - } + is_people_search = dataset_name == "people_search" or ( + "field_fill" in successful_df.columns ) + row = { + "provider": sampler_name, + "dataset": dataset_name, + "p50_internal_latency": round(float(p50_internal_latency), 2) + if pd.notna(p50_internal_latency) + else None, + "p50_request_response_latency": round( + float(p50_request_response_latency), 2 + ) + if pd.notna(p50_request_response_latency) + else None, + "problem_count": count_answered, + } + + if is_people_search: + # Do not reuse accuracy_score — that means gold-answer correctness elsewhere. + row["accuracy_score"] = None + if "has_people" in successful_df.columns: + rate = _mean_numeric(successful_df["has_people"]) + if rate is not None: + row["has_people_rate"] = round(rate, 4) + if "field_fill" in successful_df.columns: + mean_ff = _mean_numeric(successful_df["field_fill"]) + if mean_ff is not None: + row["mean_field_fill"] = round(mean_ff, 4) + if "persona_field_fill" in successful_df.columns: + mean_pff = _mean_numeric(successful_df["persona_field_fill"]) + if mean_pff is not None: + row["mean_persona_field_fill"] = round(mean_pff, 4) + if "judge_overall" in successful_df.columns: + mean_jo = _mean_numeric(successful_df["judge_overall"]) + if mean_jo is not None: + row["mean_judge_overall"] = round(mean_jo, 4) + if "judge_persona" in successful_df.columns: + mean_jp = _mean_numeric(successful_df["judge_persona"]) + if mean_jp is not None: + row["mean_judge_persona"] = round(mean_jp, 4) + # Sort key within people_search: prefer field fill, then judges + row["_sort_score"] = row.get("mean_field_fill") or row.get( + "mean_judge_overall" + ) or row.get("has_people_rate") or 0.0 + else: + correct = len( + df_sampler_results[ + df_sampler_results["evaluation_result"] == "is_correct" + ] + ) + accuracy_score = round((correct / count_answered) * 100, 2) + row["accuracy_score"] = accuracy_score + row["_sort_score"] = accuracy_score + + metric_rows.append(row) + write_path = results_dir / "analyzed_results.csv" metric_df = pd.DataFrame(metric_rows).sort_values( - ["dataset", "accuracy_score"], ascending=False + ["dataset", "_sort_score"], ascending=[True, False] ) + metric_df = metric_df.drop(columns=["_sort_score"]) metric_df.to_csv(write_path, index=False) print(f"Results were written to {write_path}") - print(metric_df) diff --git a/src/evals/processing/evaluate_answer.py b/src/evals/processing/evaluate_answer.py index 452a8d1..298cef9 100644 --- a/src/evals/processing/evaluate_answer.py +++ b/src/evals/processing/evaluate_answer.py @@ -11,6 +11,8 @@ from evals import constants from evals.processing import llm, deepsearchqa_utils +from evals.processing.people_search.field_fill import score_people_output +from evals.processing.people_search.llm_judges import run_people_llm_judges class AnswerGrader: @@ -167,3 +169,50 @@ async def evaluate_single_fin_search( "is_incorrect": is_incorrect, "score": is_correct, } + + async def evaluate_single_people_search( + self, question: str, target: str, predicted_answer: str + ) -> Dict[str, Any]: + """Score people-search provider output with deterministic field-fill scorers. + + Unlike SimpleQA/FRAMES, there is no gold answer. ``target`` is a JSON + string of row metadata (persona, query_type, etc.). ``predicted_answer`` + is a JSON string of structured people[] output from a people sampler. + """ + try: + metadata = json.loads(target) if target else {} + if not isinstance(metadata, dict): + metadata = {} + except json.JSONDecodeError: + metadata = {} + + try: + output = json.loads(predicted_answer) if predicted_answer else {} + if not isinstance(output, dict): + output = {"error": "predicted_answer was not a JSON object", "people": []} + except json.JSONDecodeError: + output = {"error": "predicted_answer was not valid JSON", "people": []} + + scores = score_people_output(output, metadata) + has_people = scores["has_people"] >= 1.0 + + judge_scores = await run_people_llm_judges( + question, output, metadata, model=self.model + ) + + # Do not map has_people → is_correct: that confuses analyzed "accuracy" + # with gold-answer benchmarks. Row-level label is has_people / no_people. + return { + "grade": "has_people" if has_people else "no_people", + "score_name": "has_people" if has_people else "no_people", + "is_correct": has_people, + "is_incorrect": not has_people, + "score": scores["field_fill"], + "has_people": scores["has_people"], + "person_count": scores["person_count"], + "field_fill": scores["field_fill"], + "persona_field_fill": scores["persona_field_fill"], + "persona": scores.get("persona"), + "question": question, + **judge_scores, + } diff --git a/src/evals/processing/people_search/__init__.py b/src/evals/processing/people_search/__init__.py new file mode 100644 index 0000000..6fa12df --- /dev/null +++ b/src/evals/processing/people_search/__init__.py @@ -0,0 +1,15 @@ +"""People-search scoring helpers (no gold-answer grading).""" + +from evals.processing.people_search.field_fill import ( + extract_people, + row_fill_score, + score_people_output, +) +from evals.processing.people_search.schema import normalize_people_payload + +__all__ = [ + "extract_people", + "normalize_people_payload", + "row_fill_score", + "score_people_output", +] diff --git a/src/evals/processing/people_search/constants.py b/src/evals/processing/people_search/constants.py new file mode 100644 index 0000000..f2937a5 --- /dev/null +++ b/src/evals/processing/people_search/constants.py @@ -0,0 +1,106 @@ +"""Shared constants for the people-search benchmark.""" + +PERSONA_SLUGS = { + "Recruiter / Talent Sourcer": "recruiter", + "SDR / BDR": "sdr", + "Background Check / Compliance Analyst": "compliance", + "Journalist / Investigative Researcher": "journalist", + "Event Organizer / Community Manager": "events", + "VC / PE / Investor": "investor", +} + +TRACKED_FIELDS: tuple[str, ...] = ( + "displayname", + "current_title", + "current_company", + "location", + "profile_url", + "highlight", + "email", + "phone", + "skills", + "insights", + "confidence", +) + +# Weight per field for each persona slug (higher = more important to that buyer). +PERSONA_FIELD_WEIGHTS: dict[str, dict[str, float]] = { + "recruiter": { + "displayname": 1.0, + "current_title": 2.5, + "current_company": 2.5, + "location": 1.5, + "profile_url": 1.0, + "highlight": 1.0, + "email": 0.5, + "phone": 0.5, + "skills": 2.0, + "insights": 1.5, + "confidence": 1.0, + }, + "sdr": { + "displayname": 2.0, + "current_title": 1.5, + "current_company": 2.0, + "location": 0.5, + "profile_url": 1.5, + "highlight": 0.5, + "email": 3.0, + "phone": 3.0, + "skills": 0.5, + "insights": 0.5, + "confidence": 1.5, + }, + "compliance": { + "displayname": 3.0, + "current_title": 2.5, + "current_company": 3.0, + "location": 1.0, + "profile_url": 2.0, + "highlight": 1.0, + "email": 0.5, + "phone": 0.5, + "skills": 0.5, + "insights": 1.0, + "confidence": 2.5, + }, + "journalist": { + "displayname": 2.0, + "current_title": 2.0, + "current_company": 2.0, + "location": 1.0, + "profile_url": 2.0, + "highlight": 2.5, + "email": 0.5, + "phone": 0.5, + "skills": 0.5, + "insights": 2.0, + "confidence": 1.0, + }, + "events": { + "displayname": 2.0, + "current_title": 1.5, + "current_company": 2.0, + "location": 1.5, + "profile_url": 1.5, + "highlight": 1.0, + "email": 2.5, + "phone": 2.5, + "skills": 0.5, + "insights": 0.5, + "confidence": 1.0, + }, + "investor": { + "displayname": 2.0, + "current_title": 2.5, + "current_company": 2.5, + "location": 1.0, + "profile_url": 2.0, + "highlight": 2.0, + "email": 0.5, + "phone": 0.5, + "skills": 1.0, + "insights": 2.0, + "confidence": 1.5, + }, +} diff --git a/src/evals/processing/people_search/field_fill.py b/src/evals/processing/people_search/field_fill.py new file mode 100644 index 0000000..df43a34 --- /dev/null +++ b/src/evals/processing/people_search/field_fill.py @@ -0,0 +1,180 @@ +"""Deterministic people[] richness scorers (no gold answers).""" + +from __future__ import annotations + +from typing import Any + +from evals.processing.people_search.constants import ( + PERSONA_FIELD_WEIGHTS, + TRACKED_FIELDS, +) + +MAX_FIELDS = len(TRACKED_FIELDS) +DEFAULT_WEIGHTS = {field: 1.0 for field in TRACKED_FIELDS} + + +def _is_filled(value: Any) -> bool: + if value is None: + return False + if isinstance(value, str): + s = value.strip() + return bool(s) and s not in ("?", "—", "-", "null", "None") + if isinstance(value, (list, tuple, dict)): + return len(value) > 0 + if isinstance(value, bool): + return value + return True + + +def _person_field_values(person: dict) -> dict[str, object]: + skills = person.get("top_skills") or [] + insights = person.get("insights") or {} + confidence = person.get("confidence") + if confidence is None and isinstance(person.get("likelihood"), (int, float)): + confidence = {"likelihood": person["likelihood"]} + + email = person.get("best_work_email") or person.get("best_personal_email") + if not email and person.get("has_email"): + email = "present" + if not email and person.get("altemails"): + email = person["altemails"][0] + + phones = person.get("phones") or [] + phone = phones[0] if phones else None + if not phone and person.get("has_phone"): + phone = "present" + + url = person.get("linkedin_url") or person.get("url") + + return { + "displayname": person.get("displayname"), + "current_title": person.get("current_title") or person.get("headline"), + "current_company": person.get("current_company"), + "location": person.get("location"), + "profile_url": url, + "highlight": person.get("highlight"), + "email": email, + "phone": phone, + "skills": skills if isinstance(skills, list) and skills else None, + "insights": insights if isinstance(insights, dict) and insights else None, + "confidence": confidence, + } + + +def person_fill_ratio(person: dict) -> tuple[float, int]: + values = _person_field_values(person) + filled = sum(1 for key in TRACKED_FIELDS if _is_filled(values.get(key))) + return filled / MAX_FIELDS, filled + + +def persona_weighted_fill_ratio(person: dict, persona: str) -> float: + weights = PERSONA_FIELD_WEIGHTS.get(persona, DEFAULT_WEIGHTS) + values = _person_field_values(person) + total_weight = sum(weights.get(f, 1.0) for f in TRACKED_FIELDS) + earned = sum( + weights.get(f, 1.0) for f in TRACKED_FIELDS if _is_filled(values.get(f)) + ) + return earned / total_weight if total_weight else 0.0 + + +def extract_people(output: dict) -> list: + people = output.get("people") or [] + if not people and isinstance(output.get("summary"), dict): + summary = output["summary"] + people = summary.get("people") or [] + if summary.get("person") and not people: + people = [summary["person"]] + return people if isinstance(people, list) else [] + + +def row_fill_score(people: list | None, *, max_people: int = 5) -> dict: + if not people: + return { + "score": 0.0, + "avg_ratio": 0.0, + "avg_fields_per_person": 0.0, + "people_scored": 0, + "total_filled": 0, + } + + ratios: list[float] = [] + counts: list[int] = [] + for person in people[:max_people]: + if not isinstance(person, dict): + continue + ratio, count = person_fill_ratio(person) + ratios.append(ratio) + counts.append(count) + + if not ratios: + return { + "score": 0.0, + "avg_ratio": 0.0, + "avg_fields_per_person": 0.0, + "people_scored": 0, + "total_filled": 0, + } + + avg_ratio = sum(ratios) / len(ratios) + return { + "score": round(avg_ratio, 4), + "avg_ratio": round(avg_ratio, 4), + "avg_fields_per_person": round(sum(counts) / len(counts), 2), + "people_scored": len(ratios), + "total_filled": sum(counts), + } + + +def row_persona_fill_score( + people: list | None, persona: str, *, max_people: int = 5 +) -> dict: + if not people: + return {"score": 0.0, "persona": persona, "people_scored": 0} + + ratios = [ + persona_weighted_fill_ratio(p, persona) + for p in people[:max_people] + if isinstance(p, dict) + ] + if not ratios: + return {"score": 0.0, "persona": persona, "people_scored": 0} + + avg = sum(ratios) / len(ratios) + return { + "score": round(avg, 4), + "persona": persona, + "people_scored": len(ratios), + } + + +def score_people_output(output: dict, metadata: dict | None = None) -> dict: + """Run deterministic scorers on a provider output payload.""" + meta = metadata or {} + if not isinstance(output, dict) or output.get("error"): + return { + "has_people": 0.0, + "person_count": 0, + "field_fill": 0.0, + "persona_field_fill": 0.0, + "persona": meta.get("persona_slug") or "unknown", + } + + people = extract_people(output) + person_count = int(output.get("person_count") or len(people) or 0) + fill = row_fill_score(people) + persona = ( + str(meta.get("persona_slug") or meta.get("judge_persona") or "unknown") + .strip() + .lower() + ) + persona_fill = row_persona_fill_score(people, persona) + + return { + "has_people": 1.0 if person_count > 0 else 0.0, + "person_count": person_count, + "field_fill": fill["score"], + "persona_field_fill": persona_fill["score"], + "persona": persona, + "people_scored": fill["people_scored"], + "avg_fields_per_person": fill.get("avg_fields_per_person", 0.0), + } diff --git a/src/evals/processing/people_search/llm_judges.py b/src/evals/processing/people_search/llm_judges.py new file mode 100644 index 0000000..88fc463 --- /dev/null +++ b/src/evals/processing/people_search/llm_judges.py @@ -0,0 +1,118 @@ +"""LLM judges for people-search (overall + persona rubrics).""" + +from __future__ import annotations + +import os +import re +from pathlib import Path +from typing import Any + +from evals import constants +from evals.processing import llm +from evals.processing.people_search.field_fill import extract_people +from evals.processing.people_search.people_preview import ( + format_people_for_scorer, + judge_persona_from_metadata, + named_target_from_metadata, +) + +CHOICE_SCORES = { + "high value": 1.0, + "useful": 0.7, + "low value": 0.3, + "failed": 0.0, +} + +_PROMPTS_DIR = Path(__file__).resolve().parent / "prompts" +_LABEL_RE = re.compile( + r"LABEL:\s*(High Value|Useful|Low Value|Failed)", + re.IGNORECASE, +) + + +def llm_judges_enabled() -> bool: + """LLM judges run by default; set PEOPLE_SEARCH_LLM_JUDGES=0 to skip.""" + return os.getenv("PEOPLE_SEARCH_LLM_JUDGES", "1").strip().lower() not in { + "0", + "false", + "no", + "off", + } + + +def _load_prompt(name: str) -> str: + return (_PROMPTS_DIR / name).read_text(encoding="utf-8") + + +def _parse_label(response: str) -> tuple[str | None, float | None]: + pretty_map = { + "high value": "High Value", + "useful": "Useful", + "low value": "Low Value", + "failed": "Failed", + } + match = _LABEL_RE.search(response or "") + if match: + key = match.group(1).lower() + return pretty_map[key], CHOICE_SCORES[key] + + lowered = (response or "").lower() + for key, pretty in pretty_map.items(): + if key in lowered: + return pretty, CHOICE_SCORES[key] + return None, None + + +def _prompt_vars(query: str, output: dict, metadata: dict) -> dict[str, Any]: + people = extract_people(output) + person_count = output.get("person_count") + if person_count is None: + person_count = len(people) + return { + "provider": output.get("provider") or "unknown", + "query": query, + "persona": metadata.get("persona") or "", + "judge_persona": judge_persona_from_metadata(metadata), + "query_type": metadata.get("query_type") or "", + "named_target": named_target_from_metadata(metadata) or "(none)", + "error": output.get("error") or "(none)", + "person_count": person_count, + "people_preview": format_people_for_scorer(people), + } + + +async def run_people_llm_judges( + query: str, + output: dict, + metadata: dict | None = None, + *, + model: str | None = None, +) -> dict[str, Any]: + """Run overall + persona LLM judges; return score fields for the results CSV.""" + if not llm_judges_enabled(): + return {} + + meta = metadata or {} + vars_ = _prompt_vars(query, output, meta) + judge_model = model or constants.GRADER_MODEL + system = ( + "You are a careful evaluator of people-search API outputs. " + "Follow the rubric exactly and always end with a LABEL line." + ) + + overall_prompt = _load_prompt("overall.md").format(**vars_) + persona_prompt = _load_prompt("persona.md").format(**vars_) + + overall_raw = await llm.call_llm(judge_model, system, overall_prompt) + persona_raw = await llm.call_llm(judge_model, system, persona_prompt) + + overall_label, overall_score = _parse_label(overall_raw) + persona_label, persona_score = _parse_label(persona_raw) + + return { + "judge_overall_label": overall_label, + "judge_overall": overall_score, + "judge_persona_label": persona_label, + "judge_persona": persona_score, + "judge_persona_slug": vars_["judge_persona"], + } diff --git a/src/evals/processing/people_search/people_preview.py b/src/evals/processing/people_search/people_preview.py new file mode 100644 index 0000000..57699d5 --- /dev/null +++ b/src/evals/processing/people_search/people_preview.py @@ -0,0 +1,74 @@ +"""People preview formatting for LLM judges.""" + +from __future__ import annotations + +from evals.processing.people_search.constants import PERSONA_SLUGS + + +def named_target_from_metadata(metadata: dict | None) -> str: + meta = metadata or {} + name = (meta.get("person_name") or "").strip() + company = (meta.get("company") or "").strip() + if name and company: + return f"{name} at {company}" + return name + + +def judge_persona_from_metadata(metadata: dict | None) -> str: + meta = metadata or {} + override = meta.get("judge_persona") or meta.get("persona_slug") + if override: + return str(override).strip().lower() + persona = meta.get("persona") or "" + return PERSONA_SLUGS.get( + persona, persona.strip().lower().replace(" ", "_") or "unknown" + ) + + +def format_people_for_scorer(people: list | None, max_people: int = 5) -> str: + if not people: + return "(zero results returned)" + blocks: list[str] = [] + for index, person in enumerate(people[:max_people], start=1): + if not isinstance(person, dict): + continue + name = person.get("displayname") or "?" + title = person.get("current_title") or person.get("headline") or "" + company = person.get("current_company") or "" + location = person.get("location") or "" + lines = [f"{index}. {name}"] + if title or company: + header = title + (f" @ {company}" if company else "") + lines.append(f" Title: {header[:220]}") + if location: + lines.append(f" Location: {location[:120]}") + skills = person.get("top_skills") or [] + if skills: + lines.append(f" Skills: {', '.join(str(s) for s in skills[:5])}") + insights = person.get("insights") or {} + if isinstance(insights, dict): + if insights.get("overall_summary"): + lines.append(f" Summary: {str(insights['overall_summary'])[:200]}") + for chip in (insights.get("why_matched") or [])[:3]: + if not isinstance(chip, dict): + continue + crit = chip.get("criterion", "?") + phrase = chip.get("matched_phrase") or chip.get("display_text") or "" + lines.append(f" Match: {crit} — {phrase[:120]}") + if person.get("highlight"): + lines.append(f" Highlight: {person['highlight'][:280]}") + if person.get("best_work_email"): + lines.append(f" Work email: {person['best_work_email']}") + if person.get("best_personal_email"): + lines.append(f" Personal email: {person['best_personal_email']}") + phones = person.get("phones") or [] + if phones: + lines.append(f" Phones: {', '.join(str(p) for p in phones[:2])}") + url = person.get("linkedin_url") or person.get("url") + if url: + lines.append(f" URL: {url}") + confidence = person.get("confidence") or {} + if isinstance(confidence, dict) and confidence.get("likelihood") is not None: + lines.append(f" Match likelihood: {confidence['likelihood']}") + blocks.append("\n".join(lines)) + return "\n\n".join(blocks) if blocks else "(zero results returned)" diff --git a/src/evals/processing/people_search/prompts/overall.md b/src/evals/processing/people_search/prompts/overall.md new file mode 100644 index 0000000..2ba3b9d --- /dev/null +++ b/src/evals/processing/people_search/prompts/overall.md @@ -0,0 +1,44 @@ +You are evaluating people-search API results for overall data quality and actionability. + +Judge whether a buyer could use this output to act on the query — regardless of which persona asked. Focus on relevance, correctness, richness, and whether zero results are acceptable. + +Provider: {provider} (informational only — score the normalized preview) + +Query: {query} +Persona (context): {persona} +Query type: {query_type} +Named target: {named_target} + +Error: {error} +Person count: {person_count} + +Results preview: +{people_preview} + +## What “good” means (overall) + +- **Relevant people** returned for the query intent +- **Structured fields** present: name, title, company, location, profile URL, contact info when appropriate +- **Named-person enrichment** (`query_type=enrichment`): the returned person should match `person_name` at `company` +- **Open search** (`query_type=search`): a useful list of on-topic candidates, not random profiles + +## Score labels + +**High Value (1.0)** — Correct, relevant results with enough structure to act on immediately. + +**Useful (0.7)** — Partially actionable: thin fields, some noise, or incomplete contact info but correct direction. + +**Low Value (0.3)** — Wrong people, irrelevant list, named-person miss, or empty on a query that should return data. + +**Failed (0.0)** — Hard API `error`, OR zero results on a named-person enrichment where the person and company are specified. + +## Decision order + +1. Hard `error`? → **Failed** +2. `query_type=enrichment` with `person_name` set and `person_count=0`? → **Failed** +3. Named target present but wrong person in preview? → **Low Value** +4. People present with good relevance + structure? → **High Value** vs **Useful** by richness +5. Open search with zero results? → **Low Value** (niche queries may still be low, not failed) + +Briefly explain your reasoning, then end with exactly one line in this format: +LABEL: diff --git a/src/evals/processing/people_search/prompts/persona.md b/src/evals/processing/people_search/prompts/persona.md new file mode 100644 index 0000000..dcf7d54 --- /dev/null +++ b/src/evals/processing/people_search/prompts/persona.md @@ -0,0 +1,70 @@ +You are evaluating people-search API results **on behalf of one specific buyer persona**. + +First, read the persona for this query, then apply ONLY that persona's rubric below. + +Persona (slug): {judge_persona} +Persona (label): {persona} +Provider: {provider} (informational only — score the normalized preview) + +Query: {query} +Query type: {query_type} +Named target: {named_target} + +Error: {error} +Person count: {person_count} + +Results preview: +{people_preview} + +## Pick the rubric that matches the persona slug + +### recruiter — Recruiter / Talent Sourcer +Goal: can you pipeline these people? Assess role fit, seniority, skills, location, career history. NOT hire quality. +High-signal fields: current_title, current_company, location, skills, insights/match summaries. +Low priority: email/phone. + +### sdr — SDR / BDR +Goal: can you reach and prospect this person or list? Work email, phone, correct title and company matter most. +High-signal fields: email, phone, displayname, current_title, current_company, profile_url. +Correct person + LinkedIn only (no email) = Useful, not High. Wrong person at right company = Low Value. + +### compliance — Background Check / Compliance Analyst +Goal: verify identity and employment. Can you confirm this person works at the stated org with a plausible title? +High-signal fields: displayname, current_company, current_title, confidence/likelihood, profile_url. +Gate: if the returned person is clearly NOT the named target, cap at Low Value regardless of richness. + +### journalist — Journalist / Investigative Researcher +Goal: enough background/context to research or write. Employment, affiliations, public footprint, narrative. +High-signal fields: highlight, insights, current_title, current_company, profile_url, summaries. +Low priority: email/phone. + +### events — Event Organizer / Community Manager +Goal: can you contact and invite people (alumni, community, speakers)? Contact path + affiliation matter most. +High-signal fields: email, phone, displayname, current_company, location, profile_url. +LinkedIn-only for a named contact = Useful, not High. + +### investor — VC / PE / Investor +Goal: assess deal relevance — founding teams, exec bios, prior ventures, authority, employer verification. +High-signal fields: current_title, current_company, highlight, insights, career summaries, profile_url. +Gate: named founder/exec lookups require correct person match before scoring high. + +## Score labels (same for every persona) + +**High Value (1.0)** — Persona can act immediately; correct, relevant, and rich enough for this use case. + +**Useful (0.7)** — Partially actionable: thin fields, some noise, or incomplete — still worth opening. + +**Low Value (0.3)** — Wrong people, wrong company, named-person miss, or empty on a query that should return data. + +**Failed (0.0)** — Hard API `error`, OR zero results on a named-person `enrichment` query (person + company were specified). + +## Decision order + +1. Hard `error`? → **Failed** +2. `query_type=enrichment` with a named target and `person_count=0`? → **Failed** +3. Named target present but the returned person is the wrong individual? → **Low Value** +4. People present? → **High Value** vs **Useful** by persona-fit and preview richness +5. Open `search` with zero results? → **Low Value** (niche is still Low, not Failed) + +Briefly explain your reasoning (name the persona rubric you applied), then end with exactly one line in this format: +LABEL: diff --git a/src/evals/processing/people_search/schema.py b/src/evals/processing/people_search/schema.py new file mode 100644 index 0000000..918ee6e --- /dev/null +++ b/src/evals/processing/people_search/schema.py @@ -0,0 +1,55 @@ +"""Normalize arbitrary people-search API responses into the scorer payload.""" + +from __future__ import annotations + +from typing import Any + + +def normalize_people_payload(result: Any, provider: str = "http_people_search") -> dict: + """Coerce endpoint JSON into ``{people, person_count, error, provider}``. + + Accepts either the canonical shape or common variants: + - top-level ``people`` list + - ``summary.people`` / ``summary.person`` + - ``results`` as a people list + """ + if not isinstance(result, dict): + return { + "provider": provider, + "people": [], + "person_count": 0, + "error": "invalid provider result", + } + + if result.get("error"): + people = result.get("people") if isinstance(result.get("people"), list) else [] + return { + "provider": provider, + "people": people, + "person_count": int(result.get("person_count") or len(people) or 0), + "error": result.get("error"), + } + + summary = result.get("summary") if isinstance(result.get("summary"), dict) else {} + people = result.get("people") + if not isinstance(people, list): + people = summary.get("people") if isinstance(summary.get("people"), list) else None + if not people and isinstance(result.get("results"), list): + people = result["results"] + if not people and summary.get("person"): + people = [summary["person"]] + if not isinstance(people, list): + people = [] + + person_count = result.get("person_count") + if person_count is None: + person_count = summary.get("person_count") + if person_count is None: + person_count = len(people) + + return { + "provider": provider, + "people": people, + "person_count": int(person_count or 0), + "error": None, + } diff --git a/src/evals/samplers/applied_samplers/people_search_sampler.py b/src/evals/samplers/applied_samplers/people_search_sampler.py new file mode 100644 index 0000000..43a6691 --- /dev/null +++ b/src/evals/samplers/applied_samplers/people_search_sampler.py @@ -0,0 +1,148 @@ +"""Generic HTTP people-search sampler (any endpoint that returns people[]). + +Request (POST JSON):: + + { + "query": "", + "metadata": { + "benchmark_id": "fp_001", + "persona": "...", + "persona_slug": "recruiter", + "query_type": "enrichment" | "search", + "person_name": "...", # enrichment rows + "company": "..." + } + } + +Response (JSON):: + + { + "people": [ + { + "displayname": "...", + "current_title": "...", + "current_company": "...", + "location": "...", + "linkedin_url": "...", + "highlight": "...", + "best_work_email": "...", + "phones": ["..."], + "top_skills": ["..."], + "insights": {}, + "confidence": {"likelihood": 0.9} + } + ], + "person_count": 1, + "error": null + } + +Configure with ``PEOPLE_SEARCH_API_URL`` and optional ``PEOPLE_SEARCH_API_KEY`` +(sent as ``Authorization: Bearer …``). +""" + +from __future__ import annotations + +import json +import logging +import os +from typing import Any + +import aiohttp + +from evals.processing.people_search.schema import normalize_people_payload +from evals.samplers.base_samplers.base_sampler import BaseSampler + +logger = logging.getLogger(__name__) + + +def _parse_metadata(ground_truth: str) -> dict: + if not ground_truth: + return {} + try: + data = json.loads(ground_truth) + return data if isinstance(data, dict) else {} + except json.JSONDecodeError: + return {} + + +class HttpPeopleSearchSampler(BaseSampler): + """Call any people-search HTTP endpoint; score structured people[] output.""" + + def __init__( + self, + sampler_name: str = "http_people_search", + api_url: str | None = None, + api_key: str | None = None, + timeout: float = 120.0, + max_retries: int = 2, + max_concurrency: int = 5, + ): + self.api_url = (api_url or os.getenv("PEOPLE_SEARCH_API_URL") or "").rstrip("/") + # BaseSampler requires a truthy api_key; fall back to the URL as a sentinel + # when the endpoint needs no auth. + resolved_key = api_key or os.getenv("PEOPLE_SEARCH_API_KEY") or self.api_url + super().__init__( + sampler_name=sampler_name, + api_key=resolved_key, + timeout=timeout, + max_retries=max_retries, + needs_synthesis=False, + max_concurrency=max_concurrency, + ) + self._auth_key = api_key or os.getenv("PEOPLE_SEARCH_API_KEY") or "" + self._eval_metadata: dict = {} + + async def __call__( + self, + query_input, + dataset, + ground_truth: str = "", + overwrite: bool = False, + ) -> dict[str, Any]: + self._eval_metadata = _parse_metadata(ground_truth) + return await super().__call__( + query_input, dataset, ground_truth=ground_truth, overwrite=overwrite + ) + + async def get_search_results(self, query: str) -> Any: + if not self.api_url: + raise ValueError( + "PEOPLE_SEARCH_API_URL is required for http_people_search. " + "Point it at any people-search endpoint that returns people[] JSON." + ) + + payload = { + "query": query, + "metadata": { + **self._eval_metadata, + "query_text": self._eval_metadata.get("query_text") or query, + }, + } + headers = {"Content-Type": "application/json", "Accept": "application/json"} + if self._auth_key and self._auth_key != self.api_url: + headers["Authorization"] = f"Bearer {self._auth_key}" + + timeout = aiohttp.ClientTimeout(total=self.timeout) + async with aiohttp.ClientSession(timeout=timeout) as session: + async with session.post( + self.api_url, json=payload, headers=headers + ) as response: + text = await response.text() + if response.status >= 400: + return { + "people": [], + "person_count": 0, + "error": f"HTTP {response.status}: {text[:500]}", + } + try: + return json.loads(text) + except json.JSONDecodeError: + return { + "people": [], + "person_count": 0, + "error": f"Non-JSON response: {text[:500]}", + } + + def format_results(self, results: Any) -> str: + payload = normalize_people_payload(results, provider=self.sampler_name) + return json.dumps(payload, ensure_ascii=False) diff --git a/src/evals/samplers/base_samplers/base_sampler.py b/src/evals/samplers/base_samplers/base_sampler.py index 119c6b0..20bd399 100644 --- a/src/evals/samplers/base_samplers/base_sampler.py +++ b/src/evals/samplers/base_samplers/base_sampler.py @@ -143,16 +143,35 @@ async def __call__( generated_answer = "FAILED" logging.exception(e) - # Evaluated synthesized results against ground truth + # Evaluate response (gold-answer graders or scorer-based datasets) + evaluation_extras: Dict[str, Any] = {} try: if generated_answer == "FAILED": # Failed to get an answer, do not grade evaluation_result = "FAILED" - elif ground_truth: + elif ground_truth or not getattr(dataset, "requires_ground_truth", True): evaluation_result_dict = await self.__evaluate_response( query, ground_truth, generated_answer, dataset ) evaluation_result = evaluation_result_dict["score_name"] + for key in ( + "score", + "has_people", + "person_count", + "field_fill", + "persona_field_fill", + "persona", + "judge_overall", + "judge_overall_label", + "judge_persona", + "judge_persona_label", + "judge_persona_slug", + "f1", + "precision", + "recall", + ): + if key in evaluation_result_dict: + evaluation_extras[key] = evaluation_result_dict[key] else: raise ValueError("Ground truth is missing") except Exception as e: @@ -167,6 +186,7 @@ async def __call__( "evaluation_result": evaluation_result, "generated_answer": generated_answer, "ground_truth": ground_truth, + **evaluation_extras, # Commenting these out because they are bloating the results files. Feel free to uncomment if you want extra metadata. # "raw_results": raw_results, # "formatted_results": formatted_results, diff --git a/src/evals/utils.py b/src/evals/utils.py index ab5bfaf..02bb5d8 100644 --- a/src/evals/utils.py +++ b/src/evals/utils.py @@ -36,6 +36,10 @@ def get_dataset(dataset_name): dataset.df["answer"] = dataset.df.apply( lambda row: _decrypt(row["answer"], row["canary"]), axis=1 ) + if dataset_name == "people_search": + # CSV answer column is intentionally empty (no gold answers). Build scoring + # metadata JSON from the dedicated columns for the sampler/grader. + dataset.df["answer"] = dataset.df.apply(_people_search_metadata_json, axis=1) if dataset.df is None: raise ValueError( @@ -44,6 +48,30 @@ def get_dataset(dataset_name): return dataset +def _people_search_metadata_json(row: pd.Series) -> str: + def _cell(key: str): + if key not in row.index: + return None + value = row[key] + if value is None or (isinstance(value, float) and pd.isna(value)): + return None + if isinstance(value, str) and not value.strip(): + return None + return value if not isinstance(value, str) else value.strip() + + return json.dumps( + { + "benchmark_id": _cell("benchmark_id"), + "persona": _cell("persona"), + "persona_slug": _cell("persona_slug"), + "query_type": _cell("query_type"), + "person_name": _cell("person_name"), + "company": _cell("company"), + }, + ensure_ascii=False, + ) + + def get_sampler(sampler_name: str): sampler = next( ( diff --git a/tests/test_people_search.py b/tests/test_people_search.py new file mode 100644 index 0000000..0f9c36e --- /dev/null +++ b/tests/test_people_search.py @@ -0,0 +1,135 @@ +"""Unit tests for people-search field-fill scorers (no API calls).""" + +import json + +import pandas as pd +import pytest + +from evals.eval_results_analyzer import write_metrics +from evals.processing.evaluate_answer import AnswerGrader +from evals.processing.people_search.field_fill import score_people_output +from evals.processing.people_search.llm_judges import _parse_label +from evals.processing.people_search.schema import normalize_people_payload + + +def test_score_people_output_empty(): + scores = score_people_output({"people": [], "person_count": 0}, {"persona_slug": "recruiter"}) + assert scores["has_people"] == 0.0 + assert scores["field_fill"] == 0.0 + assert scores["persona_field_fill"] == 0.0 + + +def test_score_people_output_partial_fill(): + output = { + "person_count": 1, + "people": [ + { + "displayname": "Ada Lovelace", + "current_title": "Analyst", + "current_company": "Analytical Engines", + "location": "London", + "linkedin_url": "https://example.com/ada", + } + ], + } + scores = score_people_output(output, {"persona_slug": "recruiter"}) + assert scores["has_people"] == 1.0 + assert scores["person_count"] == 1 + assert 0 < scores["field_fill"] < 1 + assert scores["persona_field_fill"] > 0 + + +def test_normalize_people_payload_variants(): + canonical = normalize_people_payload( + {"people": [{"displayname": "A"}], "person_count": 1} + ) + assert canonical["person_count"] == 1 + assert len(canonical["people"]) == 1 + + nested = normalize_people_payload( + {"summary": {"people": [{"displayname": "B"}], "person_count": 1}} + ) + assert nested["people"][0]["displayname"] == "B" + + results_key = normalize_people_payload({"results": [{"displayname": "C"}]}) + assert results_key["person_count"] == 1 + + +def test_parse_judge_label(): + label, score = _parse_label("Looks good.\nLABEL: Useful\n") + assert label == "Useful" + assert score == 0.7 + label, score = _parse_label("LABEL: High Value") + assert label == "High Value" + assert score == 1.0 + + +@pytest.mark.asyncio +async def test_evaluate_single_people_search_grader(monkeypatch): + monkeypatch.setenv("PEOPLE_SEARCH_LLM_JUDGES", "0") + grader = AnswerGrader() + target = json.dumps( + { + "benchmark_id": "fp_001", + "persona_slug": "recruiter", + "query_type": "enrichment", + "person_name": "Ada Lovelace", + "company": "Analytical Engines", + } + ) + predicted = json.dumps( + { + "provider": "http_people_search", + "person_count": 1, + "people": [ + { + "displayname": "Ada Lovelace", + "current_title": "Analyst", + "current_company": "Analytical Engines", + } + ], + } + ) + result = await grader.evaluate_single_people_search( + "Find Ada Lovelace", target, predicted + ) + assert result["score_name"] == "has_people" + assert result["has_people"] == 1.0 + assert "field_fill" in result + assert "persona_field_fill" in result + assert "judge_overall" not in result + + +def test_people_search_dataset_builds_metadata_from_columns(): + """Shipped CSV has empty answer; load time assembles metadata for the runner.""" + from evals import utils as evals_utils + + ds = evals_utils.get_dataset("people_search") + assert ds.df["answer"].iloc[0] + meta = json.loads(ds.df["answer"].iloc[0]) + assert meta["benchmark_id"] == "fp_001" + assert meta["persona_slug"] == "recruiter" + assert meta["query_type"] == "enrichment" + # On-disk CSV answer column remains empty + raw = pd.read_csv("data/people_search_full_dataset.csv") + assert raw["answer"].isna().all() or (raw["answer"].fillna("") == "").all() + + +def test_people_search_analyzed_metrics_omit_accuracy(tmp_path): + """people_search analyzed rows must not treat has_people as accuracy_score.""" + raw = tmp_path / "dataset_people_search_raw_results_http_people_search.csv" + raw.write_text( + "query,internal_response_time_ms,request_response_time_ms," + "evaluation_result,generated_answer,ground_truth," + "has_people,field_fill,persona_field_fill,judge_overall,judge_persona\n" + 'q1,10,20,has_people,"{}", "{}",1,0.5,0.6,0.7,0.7\n' + 'q2,10,20,no_people,"{}", "{}",0,0.0,0.0,0.0,0.0\n', + encoding="utf-8", + ) + write_metrics(tmp_path) + df = pd.read_csv(tmp_path / "analyzed_results.csv") + assert len(df) == 1 + assert pd.isna(df.loc[0, "accuracy_score"]) + assert float(df.loc[0, "mean_field_fill"]) == 0.25 + assert float(df.loc[0, "has_people_rate"]) == 0.5 + assert float(df.loc[0, "mean_judge_overall"]) == 0.35