diff --git a/README.md b/README.md index 3011ee7..8c9d14a 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ GPT 5.4 nano was used as the synthesis model. |---------------------------|----------|-----------------| | you_search_with_livecrawl |**92.09%**| 1048.05 | | exa_search_with_text | 90.06% | 1176.05 | -| parallel_search_one_shot | 89.78% | 1901.66 | +| parallel_search_basic | 89.78% | 1901.66 | | tavily_advanced | 86.32% | 3190.00 | | you_search | 84.81% | 538.44 | | google_search | 80.17% | 1347.48 | @@ -44,7 +44,7 @@ the API request is used. | tavily_advanced | 39.93% | 3460.00 | | exa_search_with_text | 39.81% | 1351.75 | | you_search_with_livecrawl | 37.26% | 1153.78 | -| parallel_search_one_shot | 34.83% | 2118.61 | +| parallel_search_basic | 34.83% | 2118.61 | | you_search | 28.03% | 565.80 | | google_search | 22.94% | 1475.05 | | tavily_basic | 19.30% | 2180.00 | diff --git a/requirements.txt b/requirements.txt index eda4323..4316a9a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,11 +6,11 @@ exa-py==2.4.0 google-genai==1.71.0 openai==1.78.1 pandas==2.2.3 -parallel-web==0.4.1 +parallel-web==0.6.0 pydantic==2.11.4 -pytest==8.3.4 -pytest-asyncio==0.24.0 -python-dotenv==1.0.1 +pytest==9.0.3 +pytest-asyncio==1.3.0 +python-dotenv==1.2.2 retry==0.9.2 tavily-python==0.7.21 tqdm==4.67.1 diff --git a/src/evals/configs/samplers.py b/src/evals/configs/samplers.py index 4d48100..e59db08 100644 --- a/src/evals/configs/samplers.py +++ b/src/evals/configs/samplers.py @@ -87,9 +87,9 @@ api_key=os.getenv("SERP_API_KEY"), ), ParallelSearchSampler( - sampler_name="parallel_search_one_shot", + sampler_name="parallel_search_basic", api_key=os.getenv("PARALLEL_API_KEY"), - mode="one-shot", + mode="basic", num_results=10, ), ParallelTaskSampler( @@ -149,3 +149,12 @@ timeout=3000, ), ] + +# Samplers excluded from default runs due to high cost or long latency +EXCLUDE_KEYWORDS = ["research", "parallel_pro", "parallel_ultra", 'perplexity_finance_historical_lookup'] + +NON_RESEARCH_SAMPLERS = [ + sampler.sampler_name + for sampler in SAMPLERS + if not any(keyword in sampler.sampler_name for keyword in EXCLUDE_KEYWORDS) +] diff --git a/src/evals/eval_runner.py b/src/evals/eval_runner.py index 25483de..8b15f58 100644 --- a/src/evals/eval_runner.py +++ b/src/evals/eval_runner.py @@ -205,18 +205,12 @@ def write_raw_sampler_results( async def main(): available_samplers = [sampler.sampler_name for sampler in samplers.SAMPLERS] - # Do not include You.com Research in default samplers due to high cost. Can be included by specifying "--samplers all" - all_samplers_no_research = [ - sampler_name - for sampler_name in available_samplers - if not sampler_name.startswith("you_research") - ] available_datasets = [dataset.dataset_name for dataset in datasets.DATASETS] default_datasets = ["simpleqa", "frames"] parser = argparse.ArgumentParser(description="Run an eval") parser.add_argument( "--samplers", - default=all_samplers_no_research, + default=samplers.NON_RESEARCH_SAMPLERS, type=str, nargs="+", help=f'The sampler(s) to use during the eval. Choose from: {available_samplers}. Use "all" to run all available sampler configurations ' diff --git a/tests/test_simpleqa.py b/tests/test_simpleqa.py index ee2eb34..8708014 100644 --- a/tests/test_simpleqa.py +++ b/tests/test_simpleqa.py @@ -58,7 +58,7 @@ async def test_simpleqa_runner(test_results_cleanup): num_problems = 2 results_dir = get_test_results_dir() args = argparse.Namespace( - samplers=[sampler.sampler_name for sampler in samplers.SAMPLERS], + samplers=samplers.NON_RESEARCH_SAMPLERS, datasets=["simpleqa"], limit=num_problems, # Test with small subset for speed batch_size=10,