set max concurrency for deep research APIs

src/evals/eval_runner.py

@@ -133,7 +133,8 @@ async def run_evals(
                 desc=f"Running sampler: {sampler.sampler_name} for dataset {dataset.dataset_name}",
                 unit="queries",
             ) as pbar:
-                semaphore = asyncio.Semaphore(args.max_concurrent_tasks)
+                max_tasks = args.max_concurrent_tasks or sampler.max_concurrency
+                semaphore = asyncio.Semaphore(max_tasks)
                 tasks = []
                 # Create tasks all at once
                 for _, row in dataset.df.iterrows():
@@ -239,9 +240,9 @@ async def main():
     )
     parser.add_argument(
         "--max-concurrent-tasks",
-        default=10,
+        default=None,
         type=int,
-        help="Maximum number of concurrent async tasks (controls parallelism via semaphore)",
+        help="Maximum number of concurrent async tasks (controls parallelism via semaphore). Defaults to 5 for You.com deep research samplers, 10 otherwise.",
     )
     parser.add_argument(
         "--clean",

src/evals/samplers/applied_samplers/you_search_sampler.py

@@ -155,6 +155,7 @@ def __init__(
             timeout=timeout,
             needs_synthesis=needs_synthesis,
         )
+        self.max_concurrency = 5
         self.research_effort = research_effort
 
     def _get_search_results_impl(self, query: str) -> Any:
@@ -192,6 +193,7 @@ def __init__(
             max_retries=max_retries,
             needs_synthesis=False,
         )
+        self.max_concurrency = 5
 
     def _get_base_url(self) -> str:
         return self._base_url

src/evals/samplers/base_samplers/base_sampler.py

@@ -17,12 +17,14 @@ def __init__(
         timeout: float = 60.0,
         max_retries: int = 3,
         needs_synthesis: bool = True,
+        max_concurrency: int = 10,
     ):
         self.api_key = api_key
         self.sampler_name = sampler_name
         self.timeout = timeout
         self.max_retries = max_retries
         self.needs_synthesis = needs_synthesis
+        self.max_concurrency = max_concurrency
 
     @abstractmethod
     async def get_search_results(self, query: str) -> Any: