Update requirements and default samplers

README.md

@@ -28,7 +28,7 @@ GPT 5.4 nano was used as the synthesis model.
 |---------------------------|----------|-----------------|
 | you_search_with_livecrawl |**92.09%**| 1048.05         |
 | exa_search_with_text      | 90.06%   | 1176.05         |
-| parallel_search_one_shot  | 89.78%   | 1901.66         |
+| parallel_search_basic     | 89.78%   | 1901.66         |
 | tavily_advanced           | 86.32%   | 3190.00         |
 | you_search                | 84.81%   | 538.44          |
 | google_search             | 80.17%   | 1347.48         |
@@ -44,7 +44,7 @@ the API request is used.
 | tavily_advanced           | 39.93%   | 3460.00        |
 | exa_search_with_text      | 39.81%   | 1351.75        |
 | you_search_with_livecrawl | 37.26%   | 1153.78        |
-| parallel_search_one_shot  | 34.83%   | 2118.61        |
+| parallel_search_basic     | 34.83%   | 2118.61        |
 | you_search                | 28.03%   | 565.80         |
 | google_search             | 22.94%   | 1475.05        |
 | tavily_basic              | 19.30%   | 2180.00        |

requirements.txt

@@ -6,11 +6,11 @@ exa-py==2.4.0
 google-genai==1.71.0
 openai==1.78.1
 pandas==2.2.3
-parallel-web==0.4.1
+parallel-web==0.6.0
 pydantic==2.11.4
-pytest==8.3.4
-pytest-asyncio==0.24.0
-python-dotenv==1.0.1
+pytest==9.0.3
+pytest-asyncio==1.3.0
+python-dotenv==1.2.2
 retry==0.9.2
 tavily-python==0.7.21
 tqdm==4.67.1

src/evals/configs/samplers.py

@@ -87,9 +87,9 @@
         api_key=os.getenv("SERP_API_KEY"),
     ),
     ParallelSearchSampler(
-        sampler_name="parallel_search_one_shot",
+        sampler_name="parallel_search_basic",
         api_key=os.getenv("PARALLEL_API_KEY"),
-        mode="one-shot",
+        mode="basic",
         num_results=10,
     ),
     ParallelTaskSampler(
@@ -149,3 +149,12 @@
         timeout=3000,
     ),
 ]
+
+# Samplers excluded from default runs due to high cost or long latency
+EXCLUDE_KEYWORDS = ["research", "parallel_pro", "parallel_ultra", 'perplexity_finance_historical_lookup']
+
+NON_RESEARCH_SAMPLERS = [
+    sampler.sampler_name
+    for sampler in SAMPLERS
+    if not any(keyword in sampler.sampler_name for keyword in EXCLUDE_KEYWORDS)
+]

src/evals/eval_runner.py

@@ -205,18 +205,12 @@ def write_raw_sampler_results(
 
 async def main():
     available_samplers = [sampler.sampler_name for sampler in samplers.SAMPLERS]
-    # Do not include You.com Research in default samplers due to high cost. Can be included by specifying "--samplers all"
-    all_samplers_no_research = [
-        sampler_name
-        for sampler_name in available_samplers
-        if not sampler_name.startswith("you_research")
-    ]
     available_datasets = [dataset.dataset_name for dataset in datasets.DATASETS]
     default_datasets = ["simpleqa", "frames"]
     parser = argparse.ArgumentParser(description="Run an eval")
     parser.add_argument(
         "--samplers",
-        default=all_samplers_no_research,
+        default=samplers.NON_RESEARCH_SAMPLERS,
         type=str,
         nargs="+",
         help=f'The sampler(s) to use during the eval. Choose from: {available_samplers}. Use "all" to run all available sampler configurations '

tests/test_simpleqa.py

@@ -58,7 +58,7 @@ async def test_simpleqa_runner(test_results_cleanup):
     num_problems = 2
     results_dir = get_test_results_dir()
     args = argparse.Namespace(
-        samplers=[sampler.sampler_name for sampler in samplers.SAMPLERS],
+        samplers=samplers.NON_RESEARCH_SAMPLERS,
         datasets=["simpleqa"],
         limit=num_problems,  # Test with small subset for speed
         batch_size=10,