This page could not be prefetched. You may retry or navigate elsewhere.
"
+ ""
+ )
+
@staticmethod
def _url_variants(url: str, parsed) -> List[str]:
"""Generate URL variants for cache lookup (original, without www, with www)."""
@@ -484,6 +802,14 @@ def get_and_clear_error(self) -> Optional[Exception]:
self._pending_error = None
return err
+ def get_and_clear_error_metadata(self) -> Dict[str, Any]:
+ metadata = dict(self._last_error_metadata)
+ self._last_error_metadata = {}
+ return metadata
+
+ def get_last_blocked_document_metadata(self) -> Dict[str, Any]:
+ return dict(self._last_blocked_document_metadata)
+
def raise_if_error(self, url: str = None) -> None:
"""Check for pending error and raise as CacheFatalError if present."""
err = self._pending_error
@@ -491,7 +817,11 @@ def raise_if_error(self, url: str = None) -> None:
if err is not None:
if isinstance(err, CacheFatalError):
raise err
- raise CacheFatalError(str(err), url=url)
+ raise CacheFatalError(
+ str(err),
+ url=url,
+ evidence={"interceptor": self.get_and_clear_error_metadata()},
+ )
def get_stats(self) -> dict:
"""Get interception statistics."""
@@ -509,3 +839,5 @@ def cleanup(self):
self.cached_pages.clear()
self.stats = InterceptorStats()
self._pending_error = None
+ self._last_error_metadata = {}
+ self._last_blocked_document_metadata = {}
diff --git a/liveweb_arena/core/reachability_audit.py b/liveweb_arena/core/reachability_audit.py
new file mode 100644
index 0000000..a82f31f
--- /dev/null
+++ b/liveweb_arena/core/reachability_audit.py
@@ -0,0 +1,589 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from typing import Any, Dict, Optional
+from urllib.parse import urlparse
+
+from liveweb_arena.core.cache import normalize_url
+from liveweb_arena.core.site_probe import probe_site
+
+
+@dataclass
+class ReachabilityAuditResult:
+ status: str
+ classification: str
+ layer: str
+ url: str
+ normalized_url: str
+ domain: str
+ plugin_name: Optional[str] = None
+ reason: Optional[str] = None
+ http_status: Optional[int] = None
+ exception_type: Optional[str] = None
+ raw_exception_type: Optional[str] = None
+ raw_exception_message: Optional[str] = None
+ navigation_stage: Optional[str] = None
+ resource_type: Optional[str] = None
+ attempt_index: Optional[int] = None
+ max_attempts: Optional[int] = None
+ browser_reused: Optional[bool] = None
+ context_reused: Optional[bool] = None
+ page_recreated_before_retry: Optional[bool] = None
+ is_environment_failure: bool = False
+ is_model_hallucination: bool = False
+ evidence: Dict[str, Any] = field(default_factory=dict)
+
+ def to_dict(self) -> dict[str, Any]:
+ return asdict(self)
+
+
+def _domain(url: str) -> str:
+ try:
+ return (urlparse(url).hostname or "").lower()
+ except Exception:
+ return ""
+
+
+def _matches_allowed_domain(domain: str, allowed_domain: str) -> bool:
+ return domain == allowed_domain or domain.endswith("." + allowed_domain)
+
+
+def _is_taostats_list_url(url: str) -> bool:
+ parsed = urlparse(url)
+ host = (parsed.hostname or "").lower()
+ path = parsed.path.lower().rstrip("/")
+ return "taostats.io" in host and path in {"", "/subnets"}
+
+
+def _is_taostats_detail_url(url: str) -> bool:
+ parsed = urlparse(url)
+ host = (parsed.hostname or "").lower()
+ path = parsed.path.lower()
+ return "taostats.io" in host and (path.startswith("/subnet/") or path.startswith("/subnets/"))
+
+
+def _infer_taostats_interaction_kind(target_locator: str | None, raw_exception_message: str | None) -> str:
+ text = " ".join(part for part in [target_locator or "", raw_exception_message or ""] if part).lower()
+ if any(marker in text for marker in ("page-item", "next", "prev", "next page")):
+ return "paginate"
+ if any(marker in text for marker in ("rt-th", "dt-orderable", "1h", "24h", "1w", "1m", "sort")):
+ return "sort"
+ if any(marker in text for marker in ("all", "rows:", "dataTables_length", ".dataTables_length")):
+ return "show_all"
+ return "unknown"
+
+
+def _is_invalid_selector_message(raw_exception_type: str | None, raw_exception_message: str | None) -> bool:
+ text = " ".join(part for part in [raw_exception_type or "", raw_exception_message or ""] if part).lower()
+ return any(
+ marker in text
+ for marker in (
+ "not a valid selector",
+ "unexpected token",
+ "queryselectorall",
+ "selector engine",
+ "selector is malformed",
+ "unknown engine",
+ )
+ )
+
+
+def _is_missing_ui_target_message(raw_exception_type: str | None, raw_exception_message: str | None) -> bool:
+ text = " ".join(part for part in [raw_exception_type or "", raw_exception_message or ""] if part).lower()
+ return "no element found with role" in text or "no element found for selector" in text
+
+
+def _build_disallowed_domain_audit(
+ *,
+ url: str,
+ normalized: str,
+ domain: str,
+ plugin_name: str | None,
+ reason: str | None,
+ http_status: int | None,
+ exception: BaseException | None,
+ raw_exception_type: str | None,
+ raw_exception_message: str | None,
+ navigation_stage: str | None,
+ resource_type: str | None,
+ attempt_index: int | None,
+ max_attempts: int | None,
+ browser_reused: bool | None,
+ context_reused: bool | None,
+ page_recreated_before_retry: bool | None,
+ evidence: dict[str, Any],
+) -> "ReachabilityAuditResult":
+ return ReachabilityAuditResult(
+ status="unreachable",
+ classification="model_disallowed_domain",
+ layer="model",
+ url=url,
+ normalized_url=normalized,
+ domain=domain,
+ plugin_name=plugin_name,
+ reason=reason or "Domain not allowed",
+ http_status=http_status,
+ exception_type=type(exception).__name__ if exception is not None else None,
+ raw_exception_type=raw_exception_type,
+ raw_exception_message=raw_exception_message,
+ navigation_stage=navigation_stage,
+ resource_type=resource_type,
+ attempt_index=attempt_index,
+ max_attempts=max_attempts,
+ browser_reused=browser_reused,
+ context_reused=context_reused,
+ page_recreated_before_retry=page_recreated_before_retry,
+ is_environment_failure=False,
+ is_model_hallucination=True,
+ evidence=evidence,
+ )
+
+
+def classify_stooq_url(url: str) -> str | None:
+ parsed = urlparse(url)
+ host = (parsed.hostname or "").lower()
+ if "stooq.com" not in host:
+ return None
+ path = parsed.path.lower()
+ query = parsed.query.lower()
+
+ if host.startswith("www."):
+ return "env_tls_error"
+ if "/q/conv/" in path or "/s/mst/" in path or "quote.php" in path:
+ return "model_invalid_url_shape"
+ if "q=" in query and "s=" not in query:
+ return "model_invalid_url_shape"
+ return None
+
+
+def classify_coingecko_url(url: str) -> str | None:
+ parsed = urlparse(url)
+ host = (parsed.hostname or "").lower()
+ if "coingecko.com" not in host:
+ return None
+ path = parsed.path.lower()
+ if "/coins/" in path:
+ slug = path.split("/coins/", 1)[1].split("/", 1)[0]
+ # Very loose first-pass hallucination heuristic for stock/company names in crypto namespace.
+ if slug in {
+ "microsoft",
+ "google",
+ "exxon-mobil",
+ "jpmorgan-chase",
+ "tesla",
+ "walmart",
+ "apple",
+ "amazon",
+ }:
+ return "model_invalid_asset_id"
+ return None
+
+
+def classify_taostats_url(url: str) -> str | None:
+ parsed = urlparse(url)
+ host = (parsed.hostname or "").lower()
+ if "taostats.io" not in host:
+ return None
+ path = parsed.path.lower()
+ if path in {"", "/", "/subnets"}:
+ return None
+ if path.startswith("/subnets/"):
+ return None
+ return "model_invalid_url_shape"
+
+
+def classify_model_hallucination(url: str) -> str | None:
+ return classify_stooq_url(url) or classify_coingecko_url(url) or classify_taostats_url(url)
+
+
+def audit_reachability_failure(
+ *,
+ url: str,
+ plugin_name: str | None,
+ plugin: Any | None = None,
+ exception: BaseException | None = None,
+ reason: str | None = None,
+ allowed_domains: set[str] | None = None,
+ http_status: int | None = None,
+ evidence: dict[str, Any] | None = None,
+) -> ReachabilityAuditResult:
+ normalized = normalize_url(url) if url else ""
+ domain = _domain(url)
+ evidence = dict(evidence or {})
+ exception_text = f"{type(exception).__name__}: {exception}" if exception is not None else ""
+ exception_lower = exception_text.lower()
+ plugin_classification = plugin.classify_url(url) if plugin is not None and hasattr(plugin, "classify_url") else None
+ hallucination_class = plugin_classification or classify_model_hallucination(url)
+
+ navigation_metadata = evidence.get("navigation_metadata") or {}
+ raw_exception_type = navigation_metadata.get("raw_exception_type") or (type(exception).__name__ if exception is not None else None)
+ raw_exception_message = navigation_metadata.get("raw_exception_message") or (str(exception) if exception is not None else None)
+ raw_exception_lower = (raw_exception_message or "").lower()
+ navigation_stage = navigation_metadata.get("navigation_stage")
+ resource_type = navigation_metadata.get("resource_type")
+ attempt_index = navigation_metadata.get("attempt_index")
+ max_attempts = navigation_metadata.get("max_attempts")
+ browser_reused = navigation_metadata.get("browser_reused")
+ context_reused = navigation_metadata.get("context_reused")
+ page_recreated_before_retry = navigation_metadata.get("page_recreated_before_retry")
+ navigation_evidence = dict(navigation_metadata.get("evidence") or {})
+ interceptor_metadata = dict(evidence.get("interceptor") or {})
+ if not resource_type:
+ resource_type = interceptor_metadata.get("blocked_resource_type") or resource_type
+
+ if allowed_domains and domain:
+ normalized_allowed = {(item or "").lower() for item in allowed_domains if item}
+ if normalized_allowed and not any(_matches_allowed_domain(domain, allowed) for allowed in normalized_allowed):
+ evidence.setdefault("interceptor", interceptor_metadata)
+ evidence.setdefault("allowed_domains", sorted(normalized_allowed))
+ return _build_disallowed_domain_audit(
+ url=url,
+ normalized=normalized,
+ domain=domain,
+ plugin_name=plugin_name,
+ reason=reason,
+ http_status=http_status,
+ exception=exception,
+ raw_exception_type=raw_exception_type,
+ raw_exception_message=raw_exception_message,
+ navigation_stage=navigation_stage,
+ resource_type=resource_type,
+ attempt_index=attempt_index,
+ max_attempts=max_attempts,
+ browser_reused=browser_reused,
+ context_reused=context_reused,
+ page_recreated_before_retry=page_recreated_before_retry,
+ evidence=evidence,
+ )
+
+ if hallucination_class is not None:
+ is_env = hallucination_class.startswith("env_") or hallucination_class.startswith("ambiguous_")
+ return ReachabilityAuditResult(
+ status="unreachable",
+ classification=hallucination_class,
+ layer="model" if not is_env else "tls",
+ url=url,
+ normalized_url=normalized,
+ domain=domain,
+ plugin_name=plugin_name,
+ reason=reason or exception_text,
+ http_status=http_status,
+ exception_type=type(exception).__name__ if exception is not None else None,
+ raw_exception_type=raw_exception_type,
+ raw_exception_message=raw_exception_message,
+ navigation_stage=navigation_stage,
+ resource_type=resource_type,
+ attempt_index=attempt_index,
+ max_attempts=max_attempts,
+ browser_reused=browser_reused,
+ context_reused=context_reused,
+ page_recreated_before_retry=page_recreated_before_retry,
+ is_environment_failure=is_env,
+ is_model_hallucination=not is_env,
+ evidence=evidence,
+ )
+
+ combined_lower = " ".join(part for part in [exception_lower, raw_exception_lower] if part)
+
+ if "taostats.io" in domain and _is_taostats_list_url(url):
+ target_locator = (
+ navigation_evidence.get("selector")
+ or navigation_evidence.get("target_locator")
+ or (
+ f"role={navigation_evidence.get('role')} name={navigation_evidence.get('name')}"
+ if navigation_evidence.get("role")
+ else None
+ )
+ )
+ interaction_kind = _infer_taostats_interaction_kind(target_locator, raw_exception_message)
+ selector_syntax_invalid = _is_invalid_selector_message(raw_exception_type, raw_exception_message)
+ missing_ui_target = _is_missing_ui_target_message(raw_exception_type, raw_exception_message)
+ if selector_syntax_invalid:
+ evidence.update(
+ {
+ "page_kind": "taostats_list",
+ "interaction_kind": interaction_kind,
+ "target_locator": target_locator,
+ "selector_syntax_invalid": True,
+ }
+ )
+ return ReachabilityAuditResult(
+ status="unreachable",
+ classification="model_invalid_selector",
+ layer="model",
+ url=url,
+ normalized_url=normalized,
+ domain=domain,
+ plugin_name=plugin_name,
+ reason=reason or exception_text,
+ http_status=http_status,
+ exception_type=type(exception).__name__ if exception is not None else None,
+ raw_exception_type=raw_exception_type,
+ raw_exception_message=raw_exception_message,
+ navigation_stage=navigation_stage,
+ resource_type=resource_type,
+ attempt_index=attempt_index,
+ max_attempts=max_attempts,
+ browser_reused=browser_reused,
+ context_reused=context_reused,
+ page_recreated_before_retry=page_recreated_before_retry,
+ is_environment_failure=False,
+ is_model_hallucination=True,
+ evidence=evidence,
+ )
+ if missing_ui_target:
+ evidence.update(
+ {
+ "page_kind": "taostats_list",
+ "interaction_kind": interaction_kind,
+ "target_locator": target_locator,
+ "selector_syntax_invalid": False,
+ "ui_target_missing": True,
+ }
+ )
+ return ReachabilityAuditResult(
+ status="unreachable",
+ classification="model_invalid_ui_target",
+ layer="model",
+ url=url,
+ normalized_url=normalized,
+ domain=domain,
+ plugin_name=plugin_name,
+ reason=reason or exception_text,
+ http_status=http_status,
+ exception_type=type(exception).__name__ if exception is not None else None,
+ raw_exception_type=raw_exception_type,
+ raw_exception_message=raw_exception_message,
+ navigation_stage=navigation_stage,
+ resource_type=resource_type,
+ attempt_index=attempt_index,
+ max_attempts=max_attempts,
+ browser_reused=browser_reused,
+ context_reused=context_reused,
+ page_recreated_before_retry=page_recreated_before_retry,
+ is_environment_failure=False,
+ is_model_hallucination=True,
+ evidence=evidence,
+ )
+ if (
+ (navigation_stage or "").startswith("action_")
+ and (
+ "timeout" in combined_lower
+ or "too many consecutive action failures" in combined_lower
+ or "no element found with role" in combined_lower
+ )
+ ):
+ evidence.update(
+ {
+ "page_kind": "taostats_list",
+ "interaction_kind": interaction_kind,
+ "target_locator": target_locator,
+ "selector_syntax_invalid": False,
+ }
+ )
+ return ReachabilityAuditResult(
+ status="unreachable",
+ classification="env_taostats_list_action_timeout",
+ layer="browser",
+ url=url,
+ normalized_url=normalized,
+ domain=domain,
+ plugin_name=plugin_name,
+ reason=reason or exception_text,
+ http_status=http_status,
+ exception_type=type(exception).__name__ if exception is not None else None,
+ raw_exception_type=raw_exception_type,
+ raw_exception_message=raw_exception_message,
+ navigation_stage=navigation_stage,
+ resource_type=resource_type,
+ attempt_index=attempt_index,
+ max_attempts=max_attempts,
+ browser_reused=browser_reused,
+ context_reused=context_reused,
+ page_recreated_before_retry=page_recreated_before_retry,
+ is_environment_failure=True,
+ is_model_hallucination=False,
+ evidence=evidence,
+ )
+
+ taostats_prefetch = dict(evidence.get("taostats_prefetch") or {})
+ if not taostats_prefetch:
+ taostats_prefetch = {
+ key: evidence.get(key)
+ for key in ("page_kind", "prefetch_phase", "wait_target", "background_refresh")
+ if evidence.get(key) is not None
+ }
+ if "taostats.io" in domain and _is_taostats_detail_url(url):
+ prefetch_phase = taostats_prefetch.get("prefetch_phase")
+ wait_target = taostats_prefetch.get("wait_target")
+ background_refresh = bool(taostats_prefetch.get("background_refresh", False))
+ page_kind = taostats_prefetch.get("page_kind")
+ detail_setup_soft_failed = bool(taostats_prefetch.get("detail_setup_soft_failed", False))
+ page_body_ready = taostats_prefetch.get("page_body_ready")
+ if prefetch_phase or page_kind == "taostats_detail":
+ if detail_setup_soft_failed and page_body_ready is True:
+ evidence.update(
+ {
+ "page_kind": "taostats_detail",
+ "prefetch_phase": prefetch_phase or "setup_page_for_cache",
+ "wait_target": wait_target,
+ "background_refresh": background_refresh,
+ "page_body_ready": True,
+ "detail_setup_soft_failed": True,
+ }
+ )
+ else:
+ evidence.update(
+ {
+ "page_kind": "taostats_detail",
+ "prefetch_phase": prefetch_phase or "goto",
+ "wait_target": wait_target,
+ "background_refresh": background_refresh,
+ }
+ )
+ return ReachabilityAuditResult(
+ status="unreachable",
+ classification="env_taostats_detail_prefetch_invalidated",
+ layer="browser",
+ url=url,
+ normalized_url=normalized,
+ domain=domain,
+ plugin_name=plugin_name,
+ reason=reason or exception_text,
+ http_status=http_status,
+ exception_type=type(exception).__name__ if exception is not None else None,
+ raw_exception_type=raw_exception_type,
+ raw_exception_message=raw_exception_message,
+ navigation_stage=navigation_stage,
+ resource_type=resource_type,
+ attempt_index=attempt_index,
+ max_attempts=max_attempts,
+ browser_reused=browser_reused,
+ context_reused=context_reused,
+ page_recreated_before_retry=page_recreated_before_retry,
+ is_environment_failure=True,
+ is_model_hallucination=False,
+ evidence=evidence,
+ )
+
+ if navigation_metadata.get("classification_hint") in {
+ "env_nav_aborted",
+ "env_target_closed",
+ "env_nav_timeout",
+ "env_browser_context_invalidated",
+ }:
+ return ReachabilityAuditResult(
+ status="unreachable",
+ classification=navigation_metadata["classification_hint"],
+ layer="browser",
+ url=url,
+ normalized_url=normalized,
+ domain=domain,
+ plugin_name=plugin_name,
+ reason=reason or exception_text,
+ http_status=http_status,
+ exception_type=type(exception).__name__ if exception is not None else None,
+ raw_exception_type=raw_exception_type,
+ raw_exception_message=raw_exception_message,
+ navigation_stage=navigation_stage,
+ resource_type=resource_type,
+ attempt_index=attempt_index,
+ max_attempts=max_attempts,
+ browser_reused=browser_reused,
+ context_reused=context_reused,
+ page_recreated_before_retry=page_recreated_before_retry,
+ is_environment_failure=True,
+ is_model_hallucination=False,
+ evidence=evidence,
+ )
+
+ probe = probe_site(url) if url else None
+ if probe is not None:
+ evidence.setdefault("site_probe", probe.to_dict())
+ if http_status is None:
+ http_status = probe.http_status
+
+ if http_status == 403 and "coingecko" in domain:
+ return ReachabilityAuditResult(
+ status="unreachable",
+ classification="env_cdn_blocked",
+ layer="cdn",
+ url=url,
+ normalized_url=normalized,
+ domain=domain,
+ plugin_name=plugin_name,
+ reason=reason or exception_text,
+ http_status=http_status,
+ exception_type=type(exception).__name__ if exception is not None else None,
+ raw_exception_type=raw_exception_type,
+ raw_exception_message=raw_exception_message,
+ navigation_stage=navigation_stage,
+ resource_type=resource_type,
+ attempt_index=attempt_index,
+ max_attempts=max_attempts,
+ browser_reused=browser_reused,
+ context_reused=context_reused,
+ page_recreated_before_retry=page_recreated_before_retry,
+ is_environment_failure=True,
+ is_model_hallucination=False,
+ evidence=evidence,
+ )
+
+ if probe and probe.exception_type == "SSLError":
+ classification = "env_tls_error"
+ layer = "tls"
+ elif "certificate_verify_failed" in combined_lower or "sslerror" in combined_lower:
+ classification = "env_tls_error"
+ layer = "tls"
+ elif "err_aborted" in combined_lower or "frame was detached" in combined_lower:
+ classification = "env_nav_aborted"
+ layer = "browser"
+ elif "target page, context or browser has been closed" in combined_lower or "targetclosederror" in combined_lower:
+ classification = "env_target_closed"
+ layer = "browser"
+ elif "timeout" in combined_lower:
+ classification = "env_nav_timeout"
+ layer = "browser"
+ elif "handler is closed" in combined_lower or "transport closed" in combined_lower or "connection closed" in combined_lower:
+ classification = "env_browser_context_invalidated"
+ layer = "browser"
+ elif "status=429" in combined_lower:
+ classification = "env_api_rate_limited"
+ layer = "api"
+ elif "empty response for coin_id" in combined_lower:
+ classification = "env_api_empty"
+ layer = "api"
+ elif http_status is not None and 400 <= http_status < 500:
+ classification = "env_http_4xx"
+ layer = "cdn"
+ elif http_status is not None and http_status >= 500:
+ classification = "env_http_5xx"
+ layer = "cdn"
+ else:
+ classification = "ambiguous_navigation_failure"
+ layer = "browser"
+
+ return ReachabilityAuditResult(
+ status="unreachable",
+ classification=classification,
+ layer=layer,
+ url=url,
+ normalized_url=normalized,
+ domain=domain,
+ plugin_name=plugin_name,
+ reason=reason or exception_text,
+ http_status=http_status,
+ exception_type=type(exception).__name__ if exception is not None else None,
+ raw_exception_type=raw_exception_type,
+ raw_exception_message=raw_exception_message,
+ navigation_stage=navigation_stage,
+ resource_type=resource_type,
+ attempt_index=attempt_index,
+ max_attempts=max_attempts,
+ browser_reused=browser_reused,
+ context_reused=context_reused,
+ page_recreated_before_retry=page_recreated_before_retry,
+ is_environment_failure=classification.startswith("env_") or classification.startswith("ambiguous_"),
+ is_model_hallucination=False,
+ evidence=evidence,
+ )
diff --git a/liveweb_arena/core/runtime_profiles.py b/liveweb_arena/core/runtime_profiles.py
new file mode 100644
index 0000000..82ba9b5
--- /dev/null
+++ b/liveweb_arena/core/runtime_profiles.py
@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+STRICT_EVAL_PROFILE = "strict_eval"
+FAST_COLLECT_PROFILE = "fast_collect"
+
+
+def normalize_runtime_profile(profile: str | None) -> str:
+ value = (profile or "").strip().lower()
+ if value in {"", "eval", STRICT_EVAL_PROFILE}:
+ return STRICT_EVAL_PROFILE
+ if value in {"collect", FAST_COLLECT_PROFILE}:
+ return FAST_COLLECT_PROFILE
+ raise ValueError(f"Unknown runtime profile: {profile}")
+
+
+def runtime_profile_to_behavior_mode(profile: str | None) -> str:
+ normalized = normalize_runtime_profile(profile)
+ if normalized == FAST_COLLECT_PROFILE:
+ return "collect"
+ return "eval"
+
+
+def is_fast_collect_profile(profile: str | None) -> bool:
+ return normalize_runtime_profile(profile) == FAST_COLLECT_PROFILE
diff --git a/liveweb_arena/core/site_probe.py b/liveweb_arena/core/site_probe.py
new file mode 100644
index 0000000..31a7681
--- /dev/null
+++ b/liveweb_arena/core/site_probe.py
@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+from typing import Any
+
+import requests
+import time
+
+
+@dataclass
+class SiteProbeResult:
+ ok: bool
+ url: str
+ final_url: str | None = None
+ http_status: int | None = None
+ exception_type: str | None = None
+ reason: str | None = None
+ server: str | None = None
+ cf_ray: str | None = None
+ location: str | None = None
+ body_length: int | None = None
+ elapsed_ms: int | None = None
+
+ def to_dict(self) -> dict[str, Any]:
+ return asdict(self)
+
+
+def probe_site(url: str, timeout: float = 10.0) -> SiteProbeResult:
+ start = time.time()
+ try:
+ response = requests.get(
+ url,
+ timeout=timeout,
+ headers={"User-Agent": "Mozilla/5.0"},
+ allow_redirects=True,
+ )
+ return SiteProbeResult(
+ ok=response.ok,
+ url=url,
+ final_url=response.url,
+ http_status=response.status_code,
+ server=response.headers.get("server"),
+ cf_ray=response.headers.get("cf-ray"),
+ location=response.headers.get("location"),
+ body_length=len(response.text or ""),
+ elapsed_ms=int((time.time() - start) * 1000),
+ reason=f"http_{response.status_code}" if not response.ok else None,
+ )
+ except Exception as exc:
+ return SiteProbeResult(
+ ok=False,
+ url=url,
+ exception_type=type(exc).__name__,
+ reason=str(exc),
+ elapsed_ms=int((time.time() - start) * 1000),
+ )
diff --git a/liveweb_arena/core/task_manager.py b/liveweb_arena/core/task_manager.py
index 3ae57c8..b453dfd 100644
--- a/liveweb_arena/core/task_manager.py
+++ b/liveweb_arena/core/task_manager.py
@@ -36,52 +36,76 @@ def _get_plugin(self, name: str) -> BasePlugin:
self._plugin_instances[name] = plugin_cls()
return self._plugin_instances[name]
- async def generate_composite_task(
+ @staticmethod
+ def derive_subtask_seed(seed: int, index: int) -> int:
+ """Derive a deterministic seed for one subtask inside a composite task."""
+ hash_input = f"{seed}:{index}".encode()
+ return int(hashlib.sha256(hash_input).hexdigest()[:8], 16)
+
+ def plan_subtasks(
self,
seed: int,
num_subtasks: int = 2,
templates: Optional[List[tuple]] = None,
- ) -> CompositeTask:
+ ) -> List[Dict[str, object]]:
"""
- Generate a composite task with multiple sub-tasks.
-
- Args:
- seed: Random seed for deterministic generation
- num_subtasks: Number of sub-tasks (1-4)
- templates: List of (plugin, template_name, variant) tuples; None = random.
- variant can be None for random selection or int for specific variant.
+ Plan deterministic subtask generation without constructing the full CompositeTask.
- Returns:
- CompositeTask with subtasks and combined_intent
+ The returned plan is stable and matches the seeds/templates that
+ generate_composite_task() will use.
"""
- # Validate num_subtasks
num_subtasks = max(1, min(4, num_subtasks))
-
- # Initialize RNG with seed for deterministic generation
rng = random.Random(seed)
- # Build list of (plugin_name, template_name, variant) for each subtask
if templates:
- # Use specified templates (cycle if not enough)
- # Normalize to 3-element tuples
selected_templates = []
for i in range(num_subtasks):
t = templates[i % len(templates)]
if len(t) == 2:
- # (plugin, template_name) -> (plugin, template_name, None)
selected_templates.append((t[0], t[1], None))
else:
- # Already (plugin, template_name, variant)
selected_templates.append(t)
else:
- # Random selection from available plugins (no specific template or variant)
available = list(self._plugin_classes.keys())
if len(available) == 0:
raise ValueError("No plugins available")
selected_templates = [(rng.choice(available), None, None) for _ in range(num_subtasks)]
+ plan = []
+ for i, (plugin_name, template_name, variant) in enumerate(selected_templates):
+ plan.append(
+ {
+ "subtask_index": i,
+ "plugin_name": plugin_name,
+ "template_name": template_name,
+ "variant": variant,
+ "subtask_seed": self.derive_subtask_seed(seed, i),
+ }
+ )
+ return plan
+
+ async def generate_composite_task(
+ self,
+ seed: int,
+ num_subtasks: int = 2,
+ templates: Optional[List[tuple]] = None,
+ ) -> CompositeTask:
+ """
+ Generate a composite task with multiple sub-tasks.
+
+ Args:
+ seed: Random seed for deterministic generation
+ num_subtasks: Number of sub-tasks (1-4)
+ templates: List of (plugin, template_name, variant) tuples; None = random.
+ variant can be None for random selection or int for specific variant.
+
+ Returns:
+ CompositeTask with subtasks and combined_intent
+ """
+ plan = self.plan_subtasks(seed=seed, num_subtasks=num_subtasks, templates=templates)
+
# Initialize plugins that will be used (some need API data before question generation)
- plugins_to_use = set(p for p, _, _ in selected_templates)
+ plugins_to_use = {item["plugin_name"] for item in plan}
for plugin_name in plugins_to_use:
plugin = self._get_plugin(plugin_name)
if hasattr(plugin, 'initialize'):
@@ -90,11 +114,12 @@ async def generate_composite_task(
# Generate sub-tasks
subtasks: List[SubTask] = []
- for i, (plugin_name, template_name, variant) in enumerate(selected_templates):
+ for i, item in enumerate(plan):
+ plugin_name = str(item["plugin_name"])
+ template_name = item["template_name"]
+ variant = item["variant"]
plugin = self._get_plugin(plugin_name)
- # Derive seed for this sub-task (hash-based to avoid collisions)
- hash_input = f"{seed}:{i}".encode()
- subtask_seed = int(hashlib.sha256(hash_input).hexdigest()[:8], 16)
+ subtask_seed = int(item["subtask_seed"])
subtask = await plugin.generate_task(
subtask_seed,
template_name=template_name,
@@ -104,20 +129,13 @@ async def generate_composite_task(
subtask.answer_tag = f"answer{i + 1}"
subtasks.append(subtask)
- # Include hints only for plugins used in the task
- # Avoids eagerly instantiating unused plugins (which may trigger API calls)
- plugin_hints: Dict[str, str] = {}
- for plugin_name in plugins_to_use:
- plugin = self._get_plugin(plugin_name)
- plugin_hints[plugin_name] = f"Use {', '.join(plugin.allowed_domains)} to find information."
-
# Build combined intent (without start_url - Agent decides navigation)
combined_intent = self._build_combined_intent(subtasks)
return CompositeTask(
subtasks=subtasks,
combined_intent=combined_intent,
- plugin_hints=plugin_hints,
+ plugin_hints={},
seed=seed,
)
diff --git a/liveweb_arena/core/task_registry.py b/liveweb_arena/core/task_registry.py
index 8257cc2..d1cef6b 100644
--- a/liveweb_arena/core/task_registry.py
+++ b/liveweb_arena/core/task_registry.py
@@ -140,6 +140,19 @@ class TaskRegistry:
81: ("openlibrary", "openlibrary_subject_multi_condition"),
82: ("openlibrary", "openlibrary_book_comparison"),
84: ("openlibrary", "openlibrary_author_editions"),
+
+ # Open Meteo templates
+ 85: ("openmeteo", "openmeteo_current"),
+ 86: ("openmeteo", "openmeteo_comparison"),
+ 87: ("openmeteo", "openmeteo_hourly_extrema"),
+ 88: ("openmeteo", "openmeteo_forecast_trend"),
+
+ # ArXiv templates
+ 90: ("arxiv", "arxiv_paper_info"),
+ 91: ("arxiv", "arxiv_author_extrema"),
+ 92: ("arxiv", "arxiv_category_comparison"),
+ 94: ("arxiv", "arxiv_multi_author_filter"),
+ 95: ("arxiv", "arxiv_title_length_extrema"),
}
# Template versions - each version's combinations come AFTER all previous versions
@@ -164,6 +177,10 @@ class TaskRegistry:
[80, 81],
# Version 4: Additional Open Library templates
[82, 84],
+ # Version 5: Open Meteo templates
+ [85, 86, 87, 88],
+ # Version 6: ArXiv templates
+ [90, 91, 92, 94, 95],
]
# Combination registry: list of template ID tuples
@@ -251,7 +268,7 @@ def parse_task_id(cls, task_id: int) -> Dict[str, Any]:
- template_ids: Tuple of template IDs in this combination
- templates: List of (plugin, template_name) tuples
- variation_seed: Seed for variation within this combination
- - num_tasks: Number of sub-tasks (3-5)
+ - num_tasks: Number of sub-tasks (1-4)
Raises:
ValueError: If task_id is out of valid range
@@ -273,7 +290,7 @@ def parse_task_id(cls, task_id: int) -> Dict[str, Any]:
template_ids = cls._combinations[combo_index]
templates = [cls.TEMPLATES[tid] for tid in template_ids]
- num_tasks = (variation_seed % 3) + 2
+ num_tasks = (variation_seed % 4) + 1
return {
"task_id": task_id,
diff --git a/liveweb_arena/core/task_registry_loader.py b/liveweb_arena/core/task_registry_loader.py
new file mode 100644
index 0000000..45eae54
--- /dev/null
+++ b/liveweb_arena/core/task_registry_loader.py
@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+import importlib.util
+import os
+from pathlib import Path
+from typing import Any
+
+from .runtime_profiles import is_fast_collect_profile, normalize_runtime_profile
+
+STRICT_TASK_REGISTRY_DIR_ENV = "LIVEWEB_STRICT_TASK_REGISTRY_DIR"
+
+
+def _load_task_registry_module_from_dir(source_dir: Path):
+ module_path = source_dir / "liveweb_arena" / "core" / "task_registry.py"
+ spec = importlib.util.spec_from_file_location("liveweb_runtime_task_registry", module_path)
+ if spec is None or spec.loader is None:
+ raise RuntimeError(f"Failed to load task registry from {module_path}")
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def resolve_task_registry_source_dir(runtime_profile: str | None = None) -> Path | None:
+ profile = normalize_runtime_profile(runtime_profile)
+ if is_fast_collect_profile(profile):
+ return None
+ raw = os.getenv(STRICT_TASK_REGISTRY_DIR_ENV, "").strip()
+ if not raw:
+ return None
+ return Path(raw)
+
+
+def parse_task_id(task_id: int, *, runtime_profile: str | None = None) -> dict[str, Any]:
+ source_dir = resolve_task_registry_source_dir(runtime_profile)
+ if source_dir is None:
+ from .task_registry import parse_task_id as local_parse_task_id
+
+ return local_parse_task_id(task_id)
+ module = _load_task_registry_module_from_dir(source_dir)
+ return module.parse_task_id(task_id)
+
+
+def max_task_id(*, runtime_profile: str | None = None) -> int:
+ source_dir = resolve_task_registry_source_dir(runtime_profile)
+ if source_dir is None:
+ from .task_registry import max_task_id as local_max_task_id
+
+ return local_max_task_id()
+ module = _load_task_registry_module_from_dir(source_dir)
+ return module.max_task_id()
diff --git a/liveweb_arena/core/validators/llm_validator.py b/liveweb_arena/core/validators/llm_validator.py
index 0c732bc..87636fc 100644
--- a/liveweb_arena/core/validators/llm_validator.py
+++ b/liveweb_arena/core/validators/llm_validator.py
@@ -50,6 +50,12 @@ class LLMValidationResult:
"Qwen/Qwen3-32B",
]
+OPENROUTER_VALIDATION_MODELS: List[str] = [
+ "google/gemini-3-flash-preview",
+ "z-ai/glm-5",
+ "xiaomi/mimo-v2-pro",
+]
+
OPENAI_VALIDATION_MODELS: List[str] = [
"gpt-4",
"gpt-3.5-turbo",
@@ -62,8 +68,10 @@ def _get_validation_models(llm_client) -> List[str]:
Priority:
1. VALIDATION_MODELS env var (comma-separated)
- 2. OpenAI-safe defaults when base_url points to api.openai.com
- 3. Project default VALIDATION_MODELS
+ 2. Provider-specific env override (e.g. VALIDATION_OPENROUTER_MODELS)
+ 3. OpenAI-safe defaults when base_url points to api.openai.com
+ 4. OpenRouter-safe defaults when base_url points to openrouter.ai
+ 5. Project default VALIDATION_MODELS
"""
env_models = os.getenv("VALIDATION_MODELS", "")
if env_models.strip():
@@ -72,8 +80,23 @@ def _get_validation_models(llm_client) -> List[str]:
return models
base_url = str(getattr(llm_client, "_base_url", "")).lower()
+ provider_env_key = ""
+ if "api.openai.com" in base_url:
+ provider_env_key = "VALIDATION_OPENAI_MODELS"
+ elif "openrouter.ai" in base_url:
+ provider_env_key = "VALIDATION_OPENROUTER_MODELS"
+
+ if provider_env_key:
+ provider_models = os.getenv(provider_env_key, "")
+ if provider_models.strip():
+ models = [m.strip() for m in provider_models.split(",") if m.strip()]
+ if models:
+ return models
+
if "api.openai.com" in base_url:
return OPENAI_VALIDATION_MODELS
+ if "openrouter.ai" in base_url:
+ return OPENROUTER_VALIDATION_MODELS
return VALIDATION_MODELS
diff --git a/liveweb_arena/plugins/__init__.py b/liveweb_arena/plugins/__init__.py
index 48990ba..8a95417 100644
--- a/liveweb_arena/plugins/__init__.py
+++ b/liveweb_arena/plugins/__init__.py
@@ -139,6 +139,11 @@ def get_plugin_names() -> List[str]:
return list(_plugins.keys())
+def get_disabled_plugins() -> List[str]:
+ """Return disabled plugin names in a stable order."""
+ return sorted(str(name) for name in DISABLED_PLUGINS)
+
+
def reload_plugins():
"""Reload all plugins (useful for development)."""
_plugins.clear()
diff --git a/liveweb_arena/plugins/arxiv/__init__.py b/liveweb_arena/plugins/arxiv/__init__.py
new file mode 100644
index 0000000..73945a8
--- /dev/null
+++ b/liveweb_arena/plugins/arxiv/__init__.py
@@ -0,0 +1,8 @@
+"""ArXiv plugin for browsing and querying academic paper listings."""
+
+from .arxiv import ArxivPlugin
+
+# Import templates to register them
+from . import templates as templates # noqa: F401 — import registers templates
+
+__all__ = ["ArxivPlugin"]
diff --git a/liveweb_arena/plugins/arxiv/api_client.py b/liveweb_arena/plugins/arxiv/api_client.py
new file mode 100644
index 0000000..058156e
--- /dev/null
+++ b/liveweb_arena/plugins/arxiv/api_client.py
@@ -0,0 +1,243 @@
+"""ArXiv API client with rate limiting.
+
+Fetches the HTML listing page (arxiv.org/list//new) and parses
+paper metadata (title, authors, primary category) directly from the page
+HTML. This guarantees the ground-truth data matches what the agent sees.
+
+Rate limit: ArXiv requests max 1 request per 3 seconds.
+"""
+
+import asyncio
+import logging
+import re
+from typing import Any, ClassVar, Dict, List, Optional
+
+import aiohttp
+
+from liveweb_arena.plugins.base_client import APIFetchError, BaseAPIClient, RateLimiter
+
+logger = logging.getLogger(__name__)
+
+CACHE_SOURCE = "arxiv"
+
+# Shared session for connection reuse
+_session: Optional[aiohttp.ClientSession] = None
+
+
+async def _get_session() -> aiohttp.ClientSession:
+ """Get or create the shared aiohttp session."""
+ global _session
+ if _session is None or _session.closed:
+ _session = aiohttp.ClientSession(
+ headers={"User-Agent": "LiveWebArena/1.0"},
+ )
+ return _session
+
+
+async def close_session():
+ """Close the shared session. Call during shutdown."""
+ global _session
+ if _session and not _session.closed:
+ await _session.close()
+ _session = None
+
+
+# ---------------------------------------------------------------------------
+# HTML listing page parsing
+# ---------------------------------------------------------------------------
+
+# Matches arxiv IDs like "2603.17021" in
blocks
+_DT_ID_RE = re.compile(r"arXiv:(\d{4}\.\d{4,5})")
+
+# Extracts title text after the "Title:" descriptor span
+_TITLE_RE = re.compile(
+ r""
+ r"\s*Title:\s*(.*?)\s*",
+ re.DOTALL,
+)
+
+# Extracts author names from tags inside list-authors div
+_AUTHORS_DIV_RE = re.compile(
+ r"(.*?)", re.DOTALL
+)
+_AUTHOR_NAME_RE = re.compile(r"]*>([^<]+)")
+
+# Extracts primary subject: "Artificial Intelligence (cs.AI)" → "cs.AI"
+_PRIMARY_SUBJECT_RE = re.compile(
+ r'[^(]*\(([^)]+)\)'
+)
+
+
+def parse_listing_html(html_text: str) -> List[Dict[str, Any]]:
+ """Parse an ArXiv listing page into a list of new-submission paper dicts.
+
+ Only papers in the "New submissions" section are included — cross-listings
+ and replacements are excluded. Paper order matches what the agent sees.
+
+ Args:
+ html_text: Raw HTML of an arxiv.org/list//new page.
+
+ Returns:
+ List of paper dicts in page order.
+ """
+ # Isolate the "New submissions" section (before cross-lists/replacements)
+ cross_idx = html_text.find("
Cross submissions")
+ repl_idx = html_text.find("
Replacement submissions")
+ # Use the earliest section boundary that exists
+ boundaries = [b for b in (cross_idx, repl_idx) if b > 0]
+ if boundaries:
+ new_section = html_text[: min(boundaries)]
+ else:
+ new_section = html_text
+
+ # Split into
/
pairs
+ dt_blocks = re.split(r"
", new_section)[1:] # skip before first
+
+ papers: List[Dict[str, Any]] = []
+ for block in dt_blocks:
+ # Each block contains the
content and the following
+ # Split at
to separate ID block from metadata block
+ parts = block.split("
", 1)
+ if len(parts) < 2:
+ continue
+
+ dt_part, dd_part = parts
+
+ # Extract arxiv ID from
+ id_match = _DT_ID_RE.search(dt_part)
+ if not id_match:
+ continue
+ arxiv_id = id_match.group(1)
+
+ # Extract title
+ title_match = _TITLE_RE.search(dd_part)
+ title = ""
+ if title_match:
+ raw = title_match.group(1)
+ # Strip any remaining HTML tags and normalize whitespace
+ raw = re.sub(r"<[^>]+>", "", raw)
+ title = " ".join(raw.split())
+
+ # Extract authors
+ authors: List[str] = []
+ authors_div_match = _AUTHORS_DIV_RE.search(dd_part)
+ if authors_div_match:
+ authors = _AUTHOR_NAME_RE.findall(authors_div_match.group(1))
+
+ # Extract primary category code from primary-subject span
+ primary_category = ""
+ primary_match = _PRIMARY_SUBJECT_RE.search(dd_part)
+ if primary_match:
+ primary_category = primary_match.group(1).strip()
+
+ papers.append({
+ "arxiv_id": arxiv_id,
+ "title": title,
+ "authors": authors,
+ "primary_category": primary_category,
+ "categories": [primary_category] if primary_category else [],
+ "published": "",
+ "summary": "",
+ })
+
+ return papers
+
+
+class ArxivClient(BaseAPIClient):
+ """
+ ArXiv API client with rate limiting.
+
+ Fetches HTML listing pages and parses paper data directly from the
+ page structure, guaranteeing GT matches the agent's view.
+
+ Rate limit: 1 request per 3 seconds (ArXiv policy).
+ """
+
+ _rate_limiter: ClassVar[RateLimiter] = RateLimiter(min_interval=3.0)
+
+ MAX_RETRIES = 3
+
+ @classmethod
+ async def fetch_listing(
+ cls,
+ category: str,
+ timeout: float = 30.0,
+ ) -> List[Dict[str, Any]]:
+ """Fetch and parse the HTML listing page for a category.
+
+ Returns new-submission papers in page order.
+ """
+ url = f"https://arxiv.org/list/{category}/new"
+ session = await _get_session()
+ req_timeout = aiohttp.ClientTimeout(total=timeout)
+
+ for attempt in range(cls.MAX_RETRIES):
+ await cls._rate_limit()
+ try:
+ async with session.get(url, timeout=req_timeout) as resp:
+ if resp.status == 200:
+ text = await resp.text()
+ return parse_listing_html(text)
+ if resp.status >= 500 and attempt < cls.MAX_RETRIES - 1:
+ wait = 2 ** attempt
+ logger.info(f"ArXiv listing {resp.status}, retry in {wait}s")
+ await asyncio.sleep(wait)
+ continue
+ logger.warning(f"ArXiv listing error: status={resp.status}")
+ return []
+ except Exception as e:
+ if attempt < cls.MAX_RETRIES - 1:
+ wait = 2 ** attempt
+ logger.info(f"ArXiv listing failed: {e}, retry in {wait}s")
+ await asyncio.sleep(wait)
+ continue
+ logger.warning(f"ArXiv listing request failed: {e}")
+ return []
+ return []
+
+
+async def fetch_listing_api_data(category: str) -> Dict[str, Any]:
+ """
+ Fetch data for a category listing page (e.g., /list/cs.AI/new).
+
+ Fetches and parses the same HTML page the agent sees, so the
+ ground-truth paper list, order, titles, and author counts are
+ guaranteed to match the page content.
+
+ Returns:
+ {
+ "category": "cs.AI",
+ "paper_count": ,
+ "papers": {
+ "": {
+ "rank": <1-based>,
+ "arxiv_id": "...",
+ "title": "...",
+ "authors": [...],
+ "primary_category": "...",
+ "categories": [...],
+ "published": "...",
+ "summary": "...",
+ },
+ ...
+ }
+ }
+ """
+ papers_list = await ArxivClient.fetch_listing(category)
+
+ if not papers_list:
+ raise APIFetchError(
+ f"No new papers on listing page for category '{category}'",
+ source="arxiv",
+ )
+
+ papers = {}
+ for rank, paper in enumerate(papers_list, start=1):
+ arxiv_id = paper["arxiv_id"]
+ papers[arxiv_id] = {**paper, "rank": rank}
+
+ return {
+ "category": category,
+ "paper_count": len(papers),
+ "papers": papers,
+ }
diff --git a/liveweb_arena/plugins/arxiv/arxiv.py b/liveweb_arena/plugins/arxiv/arxiv.py
new file mode 100644
index 0000000..1e17727
--- /dev/null
+++ b/liveweb_arena/plugins/arxiv/arxiv.py
@@ -0,0 +1,75 @@
+"""
+ArXiv Plugin.
+
+Plugin for browsing and querying ArXiv academic paper listings.
+Supports category listing pages (/list/cs.AI/new).
+"""
+
+import re
+from typing import Any, Dict, List
+from urllib.parse import urlparse
+
+from liveweb_arena.plugins.base import BasePlugin
+from .api_client import fetch_listing_api_data
+
+
+class ArxivPlugin(BasePlugin):
+ """
+ ArXiv plugin for academic paper queries.
+
+ Handles pages like:
+ - https://arxiv.org/list/cs.AI/new (new submissions listing)
+
+ Data source: HTML listing page — parsed directly so GT matches
+ the page content the agent sees.
+ """
+
+ name = "arxiv"
+
+ allowed_domains = [
+ "arxiv.org",
+ ]
+
+ def get_blocked_patterns(self) -> List[str]:
+ return [
+ "*export.arxiv.org/api/*",
+ "*rss.arxiv.org/*",
+ ]
+
+ async def fetch_api_data(self, url: str) -> Dict[str, Any]:
+ parsed = urlparse(url)
+ path = parsed.path.strip("/")
+
+ # Listing page: /list//new
+ category = self._extract_category(path)
+ if category:
+ return await fetch_listing_api_data(category)
+
+ return {}
+
+ def needs_api_data(self, url: str) -> bool:
+ parsed = urlparse(url)
+ path = parsed.path.strip("/")
+
+ return bool(self._extract_category(path))
+
+ @staticmethod
+ def _extract_category(path: str) -> str:
+ """Extract category from listing path like 'list/cs.AI/new'.
+
+ Matches /new and /recent listings — both trigger GT collection.
+ GT data is always fetched from /new regardless of which path the
+ agent visited, so the ground truth is consistent. /pastweek and
+ month-archive paths are excluded (different paper sets).
+
+ Handles all ArXiv category formats:
+ - cs.AI, math.CO (group.SUBCAT)
+ - hep-th, quant-ph (hyphenated, no dot)
+ - cond-mat.str-el (hyphenated group with lowercase subcat)
+ - astro-ph.CO (hyphenated group with uppercase subcat)
+ """
+ match = re.match(r"list/([a-z-]+(?:\.[A-Za-z-]+)?)/(new|recent)", path)
+ if match:
+ return match.group(1)
+ return ""
+
diff --git a/liveweb_arena/plugins/arxiv/templates/__init__.py b/liveweb_arena/plugins/arxiv/templates/__init__.py
new file mode 100644
index 0000000..9bc49da
--- /dev/null
+++ b/liveweb_arena/plugins/arxiv/templates/__init__.py
@@ -0,0 +1,16 @@
+"""ArXiv question templates."""
+
+from .paper_info import ArxivPaperInfoTemplate
+
+from .author_extrema import ArxivAuthorExtremaTemplate
+from .multi_author_filter import ArxivMultiAuthorFilterTemplate
+from .title_length_extrema import ArxivTitleLengthExtremaTemplate
+from .category_comparison import ArxivCategoryComparisonTemplate
+
+__all__ = [
+ "ArxivPaperInfoTemplate",
+ "ArxivAuthorExtremaTemplate",
+ "ArxivMultiAuthorFilterTemplate",
+ "ArxivTitleLengthExtremaTemplate",
+ "ArxivCategoryComparisonTemplate",
+]
diff --git a/liveweb_arena/plugins/arxiv/templates/author_extrema.py b/liveweb_arena/plugins/arxiv/templates/author_extrema.py
new file mode 100644
index 0000000..8ee5499
--- /dev/null
+++ b/liveweb_arena/plugins/arxiv/templates/author_extrema.py
@@ -0,0 +1,145 @@
+"""Author extrema template for ArXiv - MEDIUM DIFFICULTY.
+
+Asks which paper among the top-N newest submissions in a category has the
+most (or fewest) authors. The agent must scan multiple paper entries on
+the listing page and compare author counts.
+
+Dynamic data: paper pool rotates daily.
+Computation required: agent must compare across papers, not read a single value.
+41 categories × 3 top-N × 2 extrema × 5 patterns = 1230 variants.
+"""
+
+import random
+from typing import Any, Dict, Optional
+
+from liveweb_arena.core.validators.base import (
+ QuestionTemplate, GeneratedQuestion, ValidationResult, register_template,
+)
+from liveweb_arena.core.ground_truth_trigger import (
+ UrlPatternTrigger, TriggerConfig, GroundTruthResult,
+)
+from liveweb_arena.core.gt_collector import GTSourceType
+
+from .common import get_collected_listing_data, get_papers_from_listing
+from .variables import CATEGORIES, TOP_N_CHOICES
+
+PATTERNS_MOST = [
+ "Among the {n} most recent papers in today's new submissions for {category} on ArXiv, which one has the most authors? Give its title.",
+ "On ArXiv, look at today's new submissions in {category}. Among the top {n}, which paper has the largest number of authors? Report the title.",
+ "In today's new {category} submissions on ArXiv, find the paper with the most authors among the top {n}. What is its title?",
+ "Check today's new submissions for {category} on ArXiv. Of the first {n} papers, which has the most authors? Give the title.",
+ "Among the first {n} papers in today's new ArXiv submissions for {category}, which one lists the most authors? Report its title.",
+]
+
+PATTERNS_FEWEST = [
+ "Among the {n} most recent papers in today's new submissions for {category} on ArXiv, which one has the fewest authors? Give its title.",
+ "On ArXiv, look at today's new submissions in {category}. Among the top {n}, which paper has the smallest number of authors? Report the title.",
+ "In today's new {category} submissions on ArXiv, find the paper with the fewest authors among the top {n}. What is its title?",
+ "Check today's new submissions for {category} on ArXiv. Of the first {n} papers, which has the fewest authors? Give the title.",
+ "Among the first {n} papers in today's new ArXiv submissions for {category}, which one lists the fewest authors? Report its title.",
+]
+
+
+@register_template("arxiv_author_extrema")
+class ArxivAuthorExtremaTemplate(QuestionTemplate):
+ """
+ MEDIUM: Find the paper with the most or fewest authors among the top N.
+
+ Requires scanning multiple paper entries and comparing author counts.
+ 41 categories × 3 top-N × 2 extrema × 5 patterns = 1230 question variants.
+ """
+
+ GT_SOURCE = GTSourceType.PAGE_ONLY
+
+ def __init__(self):
+ super().__init__("arxiv_author_extrema")
+
+ def generate(self, seed: int, variant: Optional[int] = None) -> GeneratedQuestion:
+ rng = random.Random(seed)
+
+ is_most = (variant % 2 == 0) if variant is not None else rng.choice([True, False])
+ top_n = rng.choice(TOP_N_CHOICES)
+
+ category = rng.choice(CATEGORIES)
+ patterns = PATTERNS_MOST if is_most else PATTERNS_FEWEST
+ question_text = rng.choice(patterns).format(n=top_n, category=category.name)
+
+ return GeneratedQuestion(
+ question_text=question_text,
+ start_url=category.listing_url,
+ variables={"category": category.code, "is_most": is_most, "top_n": top_n},
+ validation_info={
+ "category_code": category.code,
+ "category_name": category.name,
+ "is_most": is_most,
+ "top_n": top_n,
+ },
+ template_name=self.name,
+ expected_steps=7,
+ )
+
+ def get_validation_rules(self, validation_info: Dict[str, Any]) -> str:
+ cat_name = validation_info["category_name"]
+ is_most = validation_info["is_most"]
+ top_n = validation_info["top_n"]
+ extrema = "most" if is_most else "fewest"
+ return f"""Task-Specific Rules (ArXiv Author Extrema):
+- Category: {cat_name}
+- Looking for: paper with the {extrema} authors among the top {top_n} newest
+- Score 1.0: Title matches the correct paper (allow minor formatting differences)
+- Score 0.5: Title partially matches or identifies correct paper with slight error
+- Score 0.0: Wrong paper or no answer
+- If there is a tie, any of the tied papers is acceptable
+- Data source: ArXiv new submissions listing"""
+
+ async def get_ground_truth(self, validation_info: Dict[str, Any]) -> GroundTruthResult:
+ category_code = validation_info["category_code"]
+ is_most = validation_info["is_most"]
+ top_n = validation_info["top_n"]
+
+ data, failure = get_collected_listing_data(category_code)
+ if failure is not None:
+ return failure
+
+ papers, paper_failure = get_papers_from_listing(data)
+ if paper_failure is not None:
+ return paper_failure
+
+ # Take only the top N papers — system error if listing has fewer than
+ # requested (not the agent's fault; live data has insufficient volume)
+ if len(papers) < top_n:
+ return GroundTruthResult.system_error(
+ f"Category {category_code} has only {len(papers)} papers, "
+ f"need {top_n} for this question"
+ )
+ subset = papers[:top_n]
+
+ if is_most:
+ target = max(subset, key=lambda p: len(p["authors"]))
+ else:
+ target = min(subset, key=lambda p: len(p["authors"]))
+
+ title = target["title"]
+ author_count = len(target["authors"])
+
+ return GroundTruthResult.ok(f"{title} ({author_count} authors)")
+
+ async def validate_answer(
+ self, answer: str, validation_info: Dict[str, Any]
+ ) -> ValidationResult:
+ """Not used — the pipeline uses LLM-based validation via get_validation_rules()."""
+ return ValidationResult(
+ score=0.0, is_correct=False, expected=None, actual=answer,
+ details="Use LLM validation",
+ )
+
+ def get_ground_truth_trigger(self, validation_info: dict) -> TriggerConfig:
+ trigger = UrlPatternTrigger(domains=["arxiv.org"])
+ return TriggerConfig(trigger=trigger)
+
+ @classmethod
+ def get_cache_source(cls) -> str:
+ return "arxiv"
+
+ def get_gt_source(self) -> GTSourceType:
+ return self.GT_SOURCE
diff --git a/liveweb_arena/plugins/arxiv/templates/category_comparison.py b/liveweb_arena/plugins/arxiv/templates/category_comparison.py
new file mode 100644
index 0000000..379810e
--- /dev/null
+++ b/liveweb_arena/plugins/arxiv/templates/category_comparison.py
@@ -0,0 +1,157 @@
+"""Category comparison template for ArXiv - HARD DIFFICULTY.
+
+Compares the total number of authors across the N most recent submissions
+in two categories from different subject groups. The agent must visit two
+separate listing pages, count authors on each paper, sum them per category,
+and compute the numeric difference.
+
+Dynamic data: paper pool rotates daily.
+Multi-page + computation: agent visits two pages and sums + subtracts.
+Answer is a numeric difference — random baseline ≈ 0%.
+605 cross-group pairs × 6 patterns × 2 orderings = 7260 variants.
+"""
+
+import random
+from typing import Any, Dict, Optional
+
+from liveweb_arena.core.validators.base import (
+ QuestionTemplate, GeneratedQuestion, ValidationResult, register_template,
+)
+from liveweb_arena.core.ground_truth_trigger import (
+ UrlPatternTrigger, TriggerConfig, GroundTruthResult,
+)
+from liveweb_arena.core.gt_collector import GTSourceType
+
+from .common import get_collected_listing_data, get_papers_from_listing
+from .variables import CATEGORY_PAIRS, TOP_N
+
+PATTERNS = [
+ "On ArXiv, look at today's new submissions in {cat1} and {cat2}. Sum the author counts for the top {n} papers in each category. What is the difference ({cat1} total minus {cat2} total)?",
+ "Sum up the author counts for the {n} newest papers in today's new submissions for {cat1} and {cat2} on ArXiv. What is the difference ({cat1} total minus {cat2} total)?",
+ "Compare today's new submissions in {cat1} and {cat2} on ArXiv. Count all authors across the top {n} papers in each and report the difference ({cat1} minus {cat2}).",
+ "Using ArXiv, add up the number of authors across the top {n} papers in today's new {cat1} and {cat2} submissions. How many more total authors does {cat1} have? (Give a signed number.)",
+ "In today's new ArXiv submissions, sum the authors for the first {n} papers in {cat1} and the first {n} in {cat2}. Report the difference ({cat1} minus {cat2}).",
+ "Check today's new submissions for {cat1} and {cat2} on ArXiv. For each category, sum the author counts of the top {n} papers. What is {cat1}'s total minus {cat2}'s total?",
+]
+
+
+@register_template("arxiv_category_comparison")
+class ArxivCategoryComparisonTemplate(QuestionTemplate):
+ """
+ HARD: Compare total author counts across the top-N papers in two categories.
+
+ Requires visiting two different listing pages, counting authors on each
+ paper, summing per category, and computing the numeric difference.
+ 605 cross-group pairs × 6 patterns × 2 orderings = 7260 question variants.
+ """
+
+ GT_SOURCE = GTSourceType.PAGE_ONLY
+
+ def __init__(self):
+ super().__init__("arxiv_category_comparison")
+
+ def generate(self, seed: int, variant: Optional[int] = None) -> GeneratedQuestion:
+ rng = random.Random(seed)
+
+ pair = rng.choice(CATEGORY_PAIRS)
+ cat1, cat2 = pair
+
+ # Randomly swap order so both orderings are represented
+ if rng.random() > 0.5:
+ cat1, cat2 = cat2, cat1
+
+ pattern = rng.choice(PATTERNS)
+ question_text = pattern.format(
+ n=TOP_N,
+ cat1=cat1.name,
+ cat2=cat2.name,
+ )
+
+ return GeneratedQuestion(
+ question_text=question_text,
+ start_url=cat1.listing_url,
+ variables={"cat1": cat1.code, "cat2": cat2.code},
+ validation_info={
+ "cat1_code": cat1.code,
+ "cat1_name": cat1.name,
+ "cat2_code": cat2.code,
+ "cat2_name": cat2.name,
+ "cat2_url": cat2.listing_url,
+ "top_n": TOP_N,
+ },
+ template_name=self.name,
+ expected_steps=9,
+ )
+
+ def get_validation_rules(self, validation_info: Dict[str, Any]) -> str:
+ cat1 = validation_info["cat1_name"]
+ cat2 = validation_info["cat2_name"]
+ top_n = validation_info["top_n"]
+ return f"""Task-Specific Rules (ArXiv Category Comparison):
+- Answer is the total-author-count difference across the {top_n} most recent papers: {cat1} minus {cat2}
+- Positive means {cat1} has more total authors
+- Score 1.0: Difference within ±3 of ground truth
+- Score 0.5: Difference within ±8 of ground truth
+- Score 0.0: Difference off by more than 8, or no numeric answer
+- Accept formats: "12", "+12", "-3", "7 more"
+- Do NOT accept answers that only name a category without the numeric difference"""
+
+ async def get_ground_truth(self, validation_info: Dict[str, Any]) -> GroundTruthResult:
+ cat1_code = validation_info["cat1_code"]
+ cat2_code = validation_info["cat2_code"]
+ cat1_name = validation_info["cat1_name"]
+ cat2_name = validation_info["cat2_name"]
+ top_n = validation_info["top_n"]
+
+ data1, fail1 = get_collected_listing_data(cat1_code)
+ if fail1 is not None:
+ return fail1
+ data2, fail2 = get_collected_listing_data(cat2_code)
+ if fail2 is not None:
+ return fail2
+
+ papers1, pf1 = get_papers_from_listing(data1)
+ if pf1 is not None:
+ return pf1
+ papers2, pf2 = get_papers_from_listing(data2)
+ if pf2 is not None:
+ return pf2
+
+ # System error if listing has fewer papers than requested —
+ # not the agent's fault; live data has insufficient volume.
+ if len(papers1) < top_n:
+ return GroundTruthResult.system_error(
+ f"{cat1_code} has only {len(papers1)} papers, need {top_n}"
+ )
+ if len(papers2) < top_n:
+ return GroundTruthResult.system_error(
+ f"{cat2_code} has only {len(papers2)} papers, need {top_n}"
+ )
+
+ total1 = sum(len(p["authors"]) for p in papers1[:top_n])
+ total2 = sum(len(p["authors"]) for p in papers2[:top_n])
+ diff = total1 - total2
+
+ return GroundTruthResult.ok(
+ f"{diff} ({cat1_name}: {total1} authors, {cat2_name}: {total2} authors)"
+ )
+
+ async def validate_answer(
+ self, answer: str, validation_info: Dict[str, Any]
+ ) -> ValidationResult:
+ """Not used — the pipeline uses LLM-based validation via get_validation_rules()."""
+ return ValidationResult(
+ score=0.0, is_correct=False, expected=None, actual=answer,
+ details="Use LLM validation",
+ )
+
+ def get_ground_truth_trigger(self, validation_info: dict) -> TriggerConfig:
+ trigger = UrlPatternTrigger(domains=["arxiv.org"])
+ return TriggerConfig(trigger=trigger)
+
+ @classmethod
+ def get_cache_source(cls) -> str:
+ return "arxiv"
+
+ def get_gt_source(self) -> GTSourceType:
+ return self.GT_SOURCE
diff --git a/liveweb_arena/plugins/arxiv/templates/common.py b/liveweb_arena/plugins/arxiv/templates/common.py
new file mode 100644
index 0000000..22c8f1c
--- /dev/null
+++ b/liveweb_arena/plugins/arxiv/templates/common.py
@@ -0,0 +1,41 @@
+"""Shared helpers for ArXiv templates."""
+
+from typing import Any, Dict, List, Optional, Tuple
+
+from liveweb_arena.core.ground_truth_trigger import GroundTruthResult
+from liveweb_arena.core.gt_collector import get_current_gt_collector
+
+
+def get_collected_listing_data(
+ category: str,
+) -> Tuple[Optional[Dict[str, Any]], Optional[GroundTruthResult]]:
+ """Return collected API data for a category listing, or a GT failure."""
+ gt_collector = get_current_gt_collector()
+ if gt_collector is None:
+ return None, GroundTruthResult.system_error("No GT collector")
+
+ collected = gt_collector.get_collected_api_data()
+ data = collected.get(f"arxiv:{category}")
+ if data is None:
+ keys = [k for k in collected if k.startswith("arxiv:")][:5]
+ return None, GroundTruthResult.not_collected(
+ f"Agent did not visit ArXiv listing for '{category}'. "
+ f"Collected keys: {keys}"
+ )
+
+ return data, None
+
+
+def get_papers_from_listing(
+ data: Dict[str, Any],
+) -> Tuple[Optional[List[Dict[str, Any]]], Optional[GroundTruthResult]]:
+ """Extract a rank-sorted list of papers from collected listing data."""
+ papers = data.get("papers")
+ if not papers or not isinstance(papers, dict):
+ return None, GroundTruthResult.fail("No papers in collected listing data")
+
+ sorted_papers = sorted(papers.values(), key=lambda p: p["rank"])
+ if not sorted_papers:
+ return None, GroundTruthResult.fail("Papers dict is empty after sort")
+
+ return sorted_papers, None
diff --git a/liveweb_arena/plugins/arxiv/templates/multi_author_filter.py b/liveweb_arena/plugins/arxiv/templates/multi_author_filter.py
new file mode 100644
index 0000000..7c3fe24
--- /dev/null
+++ b/liveweb_arena/plugins/arxiv/templates/multi_author_filter.py
@@ -0,0 +1,131 @@
+"""Multi-author filter template for ArXiv - MEDIUM DIFFICULTY.
+
+Asks how many of the top-N newest papers in a category have more than K
+authors. The agent must scan multiple entries, inspect each author list,
+and count those exceeding the threshold.
+
+Dynamic data: paper pool rotates daily.
+Computation required: filter + count, not just read a single value.
+41 categories × 3 top-N × 3 thresholds × 5 patterns = 1845 question variants.
+"""
+
+import random
+from typing import Any, Dict, Optional
+
+from liveweb_arena.core.validators.base import (
+ QuestionTemplate, GeneratedQuestion, ValidationResult, register_template,
+)
+from liveweb_arena.core.ground_truth_trigger import (
+ UrlPatternTrigger, TriggerConfig, GroundTruthResult,
+)
+from liveweb_arena.core.gt_collector import GTSourceType
+
+from .common import get_collected_listing_data, get_papers_from_listing
+from .variables import CATEGORIES, TOP_N_CHOICES
+
+# Author-count thresholds: "more than K authors"
+AUTHOR_THRESHOLDS = [1, 2, 3]
+
+PATTERNS = [
+ "Among the {n} most recent papers in today's new submissions for {category} on ArXiv, how many have more than {k} author(s)?",
+ "On ArXiv, look at today's new submissions in {category}. Among the top {n}, how many have more than {k} author(s)?",
+ "In today's new {category} submissions on ArXiv, check the first {n} papers. How many list more than {k} author(s)?",
+ "Check today's new submissions for {category} on ArXiv. Of the first {n} papers, how many have more than {k} author(s)?",
+ "Among the first {n} papers in today's new ArXiv submissions for {category}, how many have more than {k} author(s)?",
+]
+
+
+@register_template("arxiv_multi_author_filter")
+class ArxivMultiAuthorFilterTemplate(QuestionTemplate):
+ """
+ MEDIUM: Count papers exceeding an author-count threshold among the top N.
+
+ Requires inspecting each paper's author list and filtering.
+ 41 categories × 3 top-N × 3 thresholds × 5 patterns = 1845 question variants.
+ """
+
+ GT_SOURCE = GTSourceType.PAGE_ONLY
+
+ def __init__(self):
+ super().__init__("arxiv_multi_author_filter")
+
+ def generate(self, seed: int, variant: Optional[int] = None) -> GeneratedQuestion:
+ rng = random.Random(seed)
+
+ top_n = rng.choice(TOP_N_CHOICES)
+ threshold = rng.choice(AUTHOR_THRESHOLDS)
+ category = rng.choice(CATEGORIES)
+ pattern = rng.choice(PATTERNS)
+ question_text = pattern.format(n=top_n, category=category.name, k=threshold)
+
+ return GeneratedQuestion(
+ question_text=question_text,
+ start_url=category.listing_url,
+ variables={"category": category.code, "top_n": top_n, "threshold": threshold},
+ validation_info={
+ "category_code": category.code,
+ "category_name": category.name,
+ "top_n": top_n,
+ "threshold": threshold,
+ },
+ template_name=self.name,
+ expected_steps=7,
+ )
+
+ def get_validation_rules(self, validation_info: Dict[str, Any]) -> str:
+ cat_name = validation_info["category_name"]
+ top_n = validation_info["top_n"]
+ threshold = validation_info["threshold"]
+ return f"""Task-Specific Rules (ArXiv Multi-Author Filter):
+- Category: {cat_name}
+- Looking for: count of papers with MORE THAN {threshold} authors among the top {top_n} newest
+- Score 1.0: Exact count matches
+- Score 0.5: Off by 1
+- Score 0.0: Wrong count or no answer
+- The answer must be a number
+- "More than {threshold}" means strictly greater than {threshold} (not equal)
+- Data source: ArXiv new submissions listing"""
+
+ async def get_ground_truth(self, validation_info: Dict[str, Any]) -> GroundTruthResult:
+ category_code = validation_info["category_code"]
+ top_n = validation_info["top_n"]
+ threshold = validation_info["threshold"]
+
+ data, failure = get_collected_listing_data(category_code)
+ if failure is not None:
+ return failure
+
+ papers, paper_failure = get_papers_from_listing(data)
+ if paper_failure is not None:
+ return paper_failure
+
+ # System error if listing has fewer papers than requested
+ if len(papers) < top_n:
+ return GroundTruthResult.system_error(
+ f"Category {category_code} has only {len(papers)} papers, "
+ f"need {top_n} for this question"
+ )
+ subset = papers[:top_n]
+
+ count = sum(1 for p in subset if len(p["authors"]) > threshold)
+ return GroundTruthResult.ok(str(count))
+
+ async def validate_answer(
+ self, answer: str, validation_info: Dict[str, Any]
+ ) -> ValidationResult:
+ """Not used — the pipeline uses LLM-based validation via get_validation_rules()."""
+ return ValidationResult(
+ score=0.0, is_correct=False, expected=None, actual=answer,
+ details="Use LLM validation",
+ )
+
+ def get_ground_truth_trigger(self, validation_info: dict) -> TriggerConfig:
+ trigger = UrlPatternTrigger(domains=["arxiv.org"])
+ return TriggerConfig(trigger=trigger)
+
+ @classmethod
+ def get_cache_source(cls) -> str:
+ return "arxiv"
+
+ def get_gt_source(self) -> GTSourceType:
+ return self.GT_SOURCE
diff --git a/liveweb_arena/plugins/arxiv/templates/paper_info.py b/liveweb_arena/plugins/arxiv/templates/paper_info.py
new file mode 100644
index 0000000..50af6c5
--- /dev/null
+++ b/liveweb_arena/plugins/arxiv/templates/paper_info.py
@@ -0,0 +1,174 @@
+"""Paper info template for ArXiv - EASY DIFFICULTY.
+
+Asks for a single fact about the Nth newest paper in a category:
+author count or title. The agent navigates to the new-submissions
+listing and reads data from the specified paper entry.
+
+Dynamic data: new papers are posted daily, rotating the answer.
+Large entity pool: 41 categories × 2 metrics × 5 patterns × 3 ranks = 1230 variants.
+"""
+
+import random
+from enum import Enum
+from typing import Any, Dict, Optional
+
+from liveweb_arena.core.validators.base import (
+ QuestionTemplate, GeneratedQuestion, ValidationResult, register_template,
+)
+from liveweb_arena.core.ground_truth_trigger import (
+ UrlPatternTrigger, TriggerConfig, GroundTruthResult,
+)
+from liveweb_arena.core.gt_collector import GTSourceType
+
+from .common import get_collected_listing_data, get_papers_from_listing
+from .variables import CATEGORIES, RANK_CHOICES, RANK_LABELS
+
+
+class PaperMetric(Enum):
+ """Metrics extractable from a single paper entry."""
+ AUTHOR_COUNT = ("author_count", "number of authors")
+ TITLE = ("title", "title")
+
+ @property
+ def api_field(self) -> str:
+ return self.value[0]
+
+ @property
+ def display_name(self) -> str:
+ return self.value[1]
+
+
+PATTERNS = {
+ PaperMetric.AUTHOR_COUNT: [
+ "How many authors does the {rank} paper in today's new submissions for {category} on ArXiv have?",
+ "On ArXiv, look at today's new submissions in {category}. How many authors does the {rank} paper have?",
+ "In today's new submissions for {category} on ArXiv, report the author count for the {rank} paper.",
+ "Find the {rank} paper in today's new {category} submissions on ArXiv. How many authors are listed?",
+ "What is the author count of the {rank} paper among today's new ArXiv submissions in {category}?",
+ ],
+ PaperMetric.TITLE: [
+ "What is the title of the {rank} paper in today's new submissions for {category} on ArXiv?",
+ "On ArXiv, find today's new submissions in {category}. What is the title of the {rank} paper?",
+ "In today's new submissions for {category} on ArXiv, what is the {rank} paper's title?",
+ "Find the {rank} paper in today's new {category} submissions on ArXiv. What is its title?",
+ "Report the title of the {rank} paper among today's new ArXiv submissions in {category}.",
+ ],
+}
+
+
+@register_template("arxiv_paper_info")
+class ArxivPaperInfoTemplate(QuestionTemplate):
+ """
+ EASY: Navigate to a category listing and read one fact about the Nth paper.
+
+ RL value:
+ - Category navigation: must find and open the correct listing page
+ - Dynamic data: paper pool rotates daily
+ - 41 categories × 2 metrics × 5 patterns × 3 ranks = 1230 question variants
+ """
+
+ GT_SOURCE = GTSourceType.PAGE_ONLY
+
+ def __init__(self):
+ super().__init__("arxiv_paper_info")
+
+ def generate(self, seed: int, variant: Optional[int] = None) -> GeneratedQuestion:
+ rng = random.Random(seed)
+
+ metrics = list(PaperMetric)
+ metric = metrics[variant % len(metrics)] if variant is not None else rng.choice(metrics)
+
+ rank = rng.choice(RANK_CHOICES)
+ category = rng.choice(CATEGORIES)
+ pattern = rng.choice(PATTERNS[metric])
+ question_text = pattern.format(
+ category=category.name,
+ rank=RANK_LABELS[rank],
+ )
+
+ return GeneratedQuestion(
+ question_text=question_text,
+ start_url=category.listing_url,
+ variables={"category": category.code, "metric": metric.name, "rank": rank},
+ validation_info={
+ "category_code": category.code,
+ "category_name": category.name,
+ "metric_field": metric.api_field,
+ "metric_label": metric.display_name,
+ "rank": rank,
+ },
+ template_name=self.name,
+ expected_steps=5,
+ )
+
+ def get_validation_rules(self, validation_info: Dict[str, Any]) -> str:
+ cat_name = validation_info["category_name"]
+ label = validation_info["metric_label"]
+ metric_field = validation_info["metric_field"]
+ rank = validation_info["rank"]
+ rank_label = RANK_LABELS[rank]
+ if metric_field == "author_count":
+ return f"""Task-Specific Rules (ArXiv Paper Info):
+- Category: {cat_name}
+- Metric: {label} of the {rank_label} paper
+- Score 1.0: Exact author count matches
+- Score 0.5: Off by 1 (co-authors may appear differently on page vs API)
+- Score 0.0: Wrong count or no answer
+- The answer must be a number
+- Data source: ArXiv new submissions listing"""
+ return f"""Task-Specific Rules (ArXiv Paper Info):
+- Category: {cat_name}
+- Metric: {label} of the {rank_label} paper
+- Score 1.0: Title matches exactly (minor whitespace/punctuation differences OK)
+- Score 0.5: Title is substantially correct with minor omissions or additions
+- Score 0.0: Wrong paper title or no answer
+- Data source: ArXiv new submissions listing"""
+
+ async def get_ground_truth(self, validation_info: Dict[str, Any]) -> GroundTruthResult:
+ category_code = validation_info["category_code"]
+ metric_field = validation_info["metric_field"]
+ rank = validation_info["rank"]
+
+ data, failure = get_collected_listing_data(category_code)
+ if failure is not None:
+ return failure
+
+ papers, paper_failure = get_papers_from_listing(data)
+ if paper_failure is not None:
+ return paper_failure
+
+ if len(papers) < rank:
+ return GroundTruthResult.system_error(
+ f"Category {category_code} has only {len(papers)} papers, "
+ f"need at least {rank} for this question"
+ )
+
+ target_paper = papers[rank - 1]
+
+ if metric_field == "author_count":
+ return GroundTruthResult.ok(str(len(target_paper["authors"])))
+
+ if metric_field == "title":
+ return GroundTruthResult.ok(target_paper["title"])
+
+ return GroundTruthResult.fail(f"Unknown metric: {metric_field}")
+
+ async def validate_answer(
+ self, answer: str, validation_info: Dict[str, Any]
+ ) -> ValidationResult:
+ """Not used — the pipeline uses LLM-based validation via get_validation_rules()."""
+ return ValidationResult(
+ score=0.0, is_correct=False, expected=None, actual=answer,
+ details="Use LLM validation",
+ )
+
+ def get_ground_truth_trigger(self, validation_info: dict) -> TriggerConfig:
+ trigger = UrlPatternTrigger(domains=["arxiv.org"])
+ return TriggerConfig(trigger=trigger)
+
+ @classmethod
+ def get_cache_source(cls) -> str:
+ return "arxiv"
+
+ def get_gt_source(self) -> GTSourceType:
+ return self.GT_SOURCE
diff --git a/liveweb_arena/plugins/arxiv/templates/title_length_extrema.py b/liveweb_arena/plugins/arxiv/templates/title_length_extrema.py
new file mode 100644
index 0000000..ce77b87
--- /dev/null
+++ b/liveweb_arena/plugins/arxiv/templates/title_length_extrema.py
@@ -0,0 +1,144 @@
+"""Title length extrema template for ArXiv - MEDIUM DIFFICULTY.
+
+Asks which paper among the top-N newest submissions in a category has the
+longest (or shortest) title. The agent must scan multiple paper entries
+and compare title lengths.
+
+Dynamic data: paper pool rotates daily.
+Computation required: agent must compare across papers, not read a single value.
+41 categories × 3 top-N × 2 extrema × 5 patterns = 1230 question variants.
+"""
+
+import random
+from typing import Any, Dict, Optional
+
+from liveweb_arena.core.validators.base import (
+ QuestionTemplate, GeneratedQuestion, ValidationResult, register_template,
+)
+from liveweb_arena.core.ground_truth_trigger import (
+ UrlPatternTrigger, TriggerConfig, GroundTruthResult,
+)
+from liveweb_arena.core.gt_collector import GTSourceType
+
+from .common import get_collected_listing_data, get_papers_from_listing
+from .variables import CATEGORIES, TOP_N_CHOICES
+
+PATTERNS_LONGEST = [
+ "Among the {n} most recent papers in today's new submissions for {category} on ArXiv, which one has the longest title? Give its title.",
+ "On ArXiv, look at today's new submissions in {category}. Among the top {n}, which paper has the longest title? Report the title.",
+ "In today's new {category} submissions on ArXiv, find the paper with the longest title among the top {n}. What is it?",
+ "Check today's new submissions for {category} on ArXiv. Of the first {n} papers, which has the longest title? Give the title.",
+ "Among the first {n} papers in today's new ArXiv submissions for {category}, which one has the longest title? Report it.",
+]
+
+PATTERNS_SHORTEST = [
+ "Among the {n} most recent papers in today's new submissions for {category} on ArXiv, which one has the shortest title? Give its title.",
+ "On ArXiv, look at today's new submissions in {category}. Among the top {n}, which paper has the shortest title? Report the title.",
+ "In today's new {category} submissions on ArXiv, find the paper with the shortest title among the top {n}. What is it?",
+ "Check today's new submissions for {category} on ArXiv. Of the first {n} papers, which has the shortest title? Give the title.",
+ "Among the first {n} papers in today's new ArXiv submissions for {category}, which one has the shortest title? Report it.",
+]
+
+
+@register_template("arxiv_title_length_extrema")
+class ArxivTitleLengthExtremaTemplate(QuestionTemplate):
+ """
+ MEDIUM: Find the paper with the longest or shortest title among the top N.
+
+ Requires scanning multiple paper entries and comparing title lengths.
+ 41 categories × 3 top-N × 2 extrema × 5 patterns = 1230 question variants.
+ """
+
+ GT_SOURCE = GTSourceType.PAGE_ONLY
+
+ def __init__(self):
+ super().__init__("arxiv_title_length_extrema")
+
+ def generate(self, seed: int, variant: Optional[int] = None) -> GeneratedQuestion:
+ rng = random.Random(seed)
+
+ is_longest = (variant % 2 == 0) if variant is not None else rng.choice([True, False])
+ top_n = rng.choice(TOP_N_CHOICES)
+
+ category = rng.choice(CATEGORIES)
+ patterns = PATTERNS_LONGEST if is_longest else PATTERNS_SHORTEST
+ question_text = rng.choice(patterns).format(n=top_n, category=category.name)
+
+ return GeneratedQuestion(
+ question_text=question_text,
+ start_url=category.listing_url,
+ variables={"category": category.code, "is_longest": is_longest, "top_n": top_n},
+ validation_info={
+ "category_code": category.code,
+ "category_name": category.name,
+ "is_longest": is_longest,
+ "top_n": top_n,
+ },
+ template_name=self.name,
+ expected_steps=7,
+ )
+
+ def get_validation_rules(self, validation_info: Dict[str, Any]) -> str:
+ cat_name = validation_info["category_name"]
+ is_longest = validation_info["is_longest"]
+ top_n = validation_info["top_n"]
+ extrema = "longest" if is_longest else "shortest"
+ return f"""Task-Specific Rules (ArXiv Title Length Extrema):
+- Category: {cat_name}
+- Looking for: paper with the {extrema} title among the top {top_n} newest
+- Score 1.0: Title matches the correct paper (allow minor formatting differences)
+- Score 0.5: Title partially matches or identifies correct paper with slight error
+- Score 0.0: Wrong paper or no answer
+- If there is a tie in title length, any of the tied papers is acceptable
+- Data source: ArXiv new submissions listing"""
+
+ async def get_ground_truth(self, validation_info: Dict[str, Any]) -> GroundTruthResult:
+ category_code = validation_info["category_code"]
+ is_longest = validation_info["is_longest"]
+ top_n = validation_info["top_n"]
+
+ data, failure = get_collected_listing_data(category_code)
+ if failure is not None:
+ return failure
+
+ papers, paper_failure = get_papers_from_listing(data)
+ if paper_failure is not None:
+ return paper_failure
+
+ # System error if listing has fewer papers than requested
+ if len(papers) < top_n:
+ return GroundTruthResult.system_error(
+ f"Category {category_code} has only {len(papers)} papers, "
+ f"need {top_n} for this question"
+ )
+ subset = papers[:top_n]
+
+ if is_longest:
+ target = max(subset, key=lambda p: len(p["title"]))
+ else:
+ target = min(subset, key=lambda p: len(p["title"]))
+
+ title = target["title"]
+ char_count = len(title)
+
+ return GroundTruthResult.ok(f"{title} ({char_count} characters)")
+
+ async def validate_answer(
+ self, answer: str, validation_info: Dict[str, Any]
+ ) -> ValidationResult:
+ """Not used — the pipeline uses LLM-based validation via get_validation_rules()."""
+ return ValidationResult(
+ score=0.0, is_correct=False, expected=None, actual=answer,
+ details="Use LLM validation",
+ )
+
+ def get_ground_truth_trigger(self, validation_info: dict) -> TriggerConfig:
+ trigger = UrlPatternTrigger(domains=["arxiv.org"])
+ return TriggerConfig(trigger=trigger)
+
+ @classmethod
+ def get_cache_source(cls) -> str:
+ return "arxiv"
+
+ def get_gt_source(self) -> GTSourceType:
+ return self.GT_SOURCE
diff --git a/liveweb_arena/plugins/arxiv/templates/variables.py b/liveweb_arena/plugins/arxiv/templates/variables.py
new file mode 100644
index 0000000..6901507
--- /dev/null
+++ b/liveweb_arena/plugins/arxiv/templates/variables.py
@@ -0,0 +1,113 @@
+"""Category pool, constants, and metric definitions for ArXiv templates."""
+
+from dataclasses import dataclass
+from typing import List, Tuple
+
+# How many papers the agent must inspect for top-N templates.
+# Min category volume is 8 papers/day → 1.6× safety margin at top_n=5.
+TOP_N_CHOICES = [3, 4, 5]
+
+# Fixed window size for category_comparison (always uses the max).
+TOP_N = max(TOP_N_CHOICES)
+
+# Rank positions for paper_info (Nth newest paper).
+RANK_CHOICES = [1, 2, 3]
+
+RANK_LABELS = {
+ 1: "most recent",
+ 2: "second most recent",
+ 3: "third most recent",
+}
+
+
+@dataclass(frozen=True)
+class Category:
+ """An ArXiv category with display name."""
+ code: str # e.g., "cs.AI"
+ name: str # e.g., "Artificial Intelligence"
+ group: str # e.g., "cs" (for pairing across groups)
+
+ @property
+ def listing_url(self) -> str:
+ """URL for the new-submissions listing page."""
+ return f"https://arxiv.org/list/{self.code}/new"
+
+
+# Diverse categories across ArXiv's subject groups.
+# Selection criteria:
+# - Daily new-submission volume ≥ 8 papers/day (verified 2026-03-19)
+# - Stable existence on arxiv.org
+# - Playwright-accessible listing pages
+# - No aliases (cs.SY→eess.SY, cs.NA→math.NA excluded)
+#
+# Excluded for low volume (< 8 new papers/day, verified 2026-03-19):
+# q-fin.ST (~1), q-bio.NC (~3), q-bio.QM (~5), cs.DB (~5), cs.IT (~5),
+# cs.MA (~5), math.DS (~5), physics.flu-dyn (~5), astro-ph.EP (~5),
+# eess.IV (~6), cs.NI (~6), math.MP (~6), math.FA (~7), cs.CE (~7),
+# cs.SI (~7), cs.CY (~7), nucl-ex (~7), stat.AP (~7).
+CATEGORIES: List[Category] = [
+ # Computer Science (11)
+ Category("cs.AI", "Artificial Intelligence", "cs"),
+ Category("cs.CL", "Computation and Language", "cs"),
+ Category("cs.CV", "Computer Vision", "cs"),
+ Category("cs.LG", "Machine Learning", "cs"),
+ Category("cs.SE", "Software Engineering", "cs"),
+ Category("cs.CR", "Cryptography and Security", "cs"),
+ Category("cs.RO", "Robotics", "cs"),
+ Category("cs.DS", "Data Structures and Algorithms", "cs"),
+ Category("cs.HC", "Human-Computer Interaction", "cs"),
+ Category("cs.IR", "Information Retrieval", "cs"),
+ Category("cs.GT", "Computer Science and Game Theory", "cs"),
+ # Mathematics (9)
+ Category("math.CO", "Combinatorics", "math"),
+ Category("math.PR", "Probability", "math"),
+ Category("math.OC", "Optimization and Control", "math"),
+ Category("math.NA", "Numerical Analysis", "math"),
+ Category("math.AG", "Algebraic Geometry", "math"),
+ Category("math.AP", "Analysis of PDEs", "math"),
+ Category("math.NT", "Number Theory", "math"),
+ Category("math.DG", "Differential Geometry", "math"),
+ Category("math.GR", "Group Theory", "math"),
+ # Physics (16)
+ Category("hep-th", "High Energy Physics - Theory", "physics"),
+ Category("hep-ph", "High Energy Physics - Phenomenology", "physics"),
+ Category("quant-ph", "Quantum Physics", "physics"),
+ Category("gr-qc", "General Relativity and Quantum Cosmology", "physics"),
+ Category("astro-ph.CO", "Cosmology and Nongalactic Astrophysics", "physics"),
+ Category("astro-ph.GA", "Astrophysics of Galaxies", "physics"),
+ Category("astro-ph.HE", "High Energy Astrophysical Phenomena", "physics"),
+ Category("astro-ph.SR", "Solar and Stellar Astrophysics", "physics"),
+ Category("astro-ph.IM", "Instrumentation and Methods for Astrophysics", "physics"),
+ Category("cond-mat.str-el", "Strongly Correlated Electrons", "physics"),
+ Category("cond-mat.mes-hall", "Mesoscale and Nanoscale Physics", "physics"),
+ Category("cond-mat.mtrl-sci", "Materials Science", "physics"),
+ Category("cond-mat.stat-mech", "Statistical Mechanics", "physics"),
+ Category("cond-mat.supr-con", "Superconductivity", "physics"),
+ Category("cond-mat.soft", "Soft Condensed Matter", "physics"),
+ Category("physics.optics", "Optics", "physics"),
+ # Statistics (2)
+ Category("stat.ML", "Statistics - Machine Learning", "stat"),
+ Category("stat.ME", "Statistics - Methodology", "stat"),
+ # Electrical Engineering (3)
+ Category("eess.SP", "Signal Processing", "eess"),
+ Category("eess.SY", "Systems and Control", "eess"),
+ Category("eess.AS", "Audio and Speech Processing", "eess"),
+]
+
+
+def build_category_pairs() -> List[Tuple[Category, Category]]:
+ """Build all cross-group category pairs for comparison templates.
+
+ Pairs categories across groups (e.g., cs vs math) so comparisons are
+ interesting — categories within the same group tend to have similar volumes.
+ """
+ pairs: List[Tuple[Category, Category]] = []
+ n = len(CATEGORIES)
+ for i in range(n):
+ for j in range(i + 1, n):
+ if CATEGORIES[i].group != CATEGORIES[j].group:
+ pairs.append((CATEGORIES[i], CATEGORIES[j]))
+ return pairs
+
+
+CATEGORY_PAIRS: List[Tuple[Category, Category]] = build_category_pairs()
diff --git a/liveweb_arena/plugins/base.py b/liveweb_arena/plugins/base.py
index 6a9f134..5703a2a 100644
--- a/liveweb_arena/plugins/base.py
+++ b/liveweb_arena/plugins/base.py
@@ -150,6 +150,22 @@ def get_synthetic_page(self, url: str) -> Optional[str]:
"""
return None
+ def get_stable_url_patterns(self) -> List[str]:
+ """Return stable URL shapes that this plugin intentionally supports."""
+ return []
+
+ def classify_url(self, url: str) -> Optional[str]:
+ """Return a model-side classification for clearly invalid URL shapes."""
+ return None
+
+ def is_plausible_asset_id(self, url: str) -> bool:
+ """Return True when URL-derived asset identifiers look valid for this plugin."""
+ return True
+
+ def audit_url(self, url: str) -> Optional[Dict[str, Any]]:
+ """Return optional extra audit details for the URL."""
+ return None
+
def needs_api_data(self, url: str) -> bool:
"""
Check if this URL needs API data for ground truth.
@@ -167,7 +183,7 @@ def needs_api_data(self, url: str) -> bool:
"""
return True
- async def setup_page_for_cache(self, page, url: str) -> None:
+ async def setup_page_for_cache(self, page, url: str) -> Optional[Dict[str, Any]]:
"""
Perform page interactions before caching (e.g., click 'Show All').
@@ -178,8 +194,13 @@ async def setup_page_for_cache(self, page, url: str) -> None:
page: Playwright Page object
url: The page URL being cached
+ Returns:
+ Optional metadata describing soft setup issues or extra evidence
+ gathered during cache preparation. Returning ``None`` keeps the
+ previous behavior.
+
Example:
- async def setup_page_for_cache(self, page, url: str) -> None:
+ async def setup_page_for_cache(self, page, url: str) -> Optional[Dict[str, Any]]:
if '/subnets' in url:
# Click "ALL" to show all rows
await page.click('text=ALL')
diff --git a/liveweb_arena/plugins/base_client.py b/liveweb_arena/plugins/base_client.py
index 2a33a2b..f4394ca 100644
--- a/liveweb_arena/plugins/base_client.py
+++ b/liveweb_arena/plugins/base_client.py
@@ -3,7 +3,9 @@
import asyncio
import time
from abc import ABC
-from typing import Any, ClassVar, Dict
+from typing import Any, ClassVar, Dict, Optional
+
+import aiohttp
class APIFetchError(Exception):
@@ -14,10 +16,17 @@ class APIFetchError(Exception):
All API clients should raise this instead of returning None/empty dict.
"""
- def __init__(self, message: str, source: str = None, status_code: int = None):
+ def __init__(
+ self,
+ message: str,
+ source: str = None,
+ status_code: int = None,
+ metadata: Optional[Dict[str, Any]] = None,
+ ):
super().__init__(message)
self.source = source
self.status_code = status_code
+ self.metadata = dict(metadata or {})
def validate_api_response(data: Any, expected_type: type, context: str) -> None:
@@ -69,3 +78,24 @@ class BaseAPIClient(ABC):
async def _rate_limit(cls):
"""Apply rate limiting. Subclasses can override for custom behavior."""
await cls._rate_limiter.wait()
+
+
+def create_http_session(
+ *,
+ headers: Optional[Dict[str, str]] = None,
+ timeout: Optional[float] = None,
+) -> aiohttp.ClientSession:
+ """
+ Create a shared-config aiohttp session for plugin API access.
+
+ Important: trust_env=True allows the evaluation environment's proxy settings
+ to apply consistently, which is required on this machine for some sites/APIs.
+ """
+ timeout_cfg = aiohttp.ClientTimeout(total=timeout) if timeout is not None else None
+ connector = aiohttp.TCPConnector(ttl_dns_cache=300, limit=32, ssl=False)
+ return aiohttp.ClientSession(
+ headers=headers,
+ timeout=timeout_cfg,
+ trust_env=True,
+ connector=connector,
+ )
diff --git a/liveweb_arena/plugins/coingecko/api_client.py b/liveweb_arena/plugins/coingecko/api_client.py
index eed16ff..007d49c 100644
--- a/liveweb_arena/plugins/coingecko/api_client.py
+++ b/liveweb_arena/plugins/coingecko/api_client.py
@@ -3,11 +3,18 @@
import os
import asyncio
import logging
+import time
from typing import Any, Dict, List, Optional
import aiohttp
-from liveweb_arena.plugins.base_client import APIFetchError, BaseAPIClient, RateLimiter, validate_api_response
+from liveweb_arena.plugins.base_client import (
+ APIFetchError,
+ BaseAPIClient,
+ RateLimiter,
+ create_http_session,
+ validate_api_response,
+)
logger = logging.getLogger(__name__)
@@ -26,6 +33,8 @@ class CoinGeckoClient(BaseAPIClient):
# Free tier: 2s interval; Pro tier uses override in _rate_limit
_rate_limiter = RateLimiter(min_interval=2.0)
+ _single_coin_cache: dict[str, tuple[float, dict[str, Any]]] = {}
+ _single_coin_cache_ttl_s = float(os.getenv("LIVEWEB_COINGECKO_SINGLE_COIN_CACHE_TTL_SECONDS", "900"))
@classmethod
def get_api_key(cls) -> Optional[str]:
@@ -78,12 +87,11 @@ async def get(
headers = cls.get_headers()
try:
- async with aiohttp.ClientSession() as session:
+ async with create_http_session() as session:
async with session.get(
url,
params=params,
headers=headers,
- timeout=aiohttp.ClientTimeout(total=timeout),
) as response:
if response.status == 429:
# Rate limited - wait and retry once
@@ -92,7 +100,6 @@ async def get(
url,
params=params,
headers=headers,
- timeout=aiohttp.ClientTimeout(total=timeout),
) as retry_response:
if retry_response.status != 200:
raise APIFetchError(f"CoinGecko retry failed: {retry_response.status}")
@@ -156,7 +163,7 @@ async def fetch_cache_api_data() -> Optional[Dict[str, Any]]:
logger.info(f"Fetching CoinGecko data for {len(coins)} coins...")
try:
- async with aiohttp.ClientSession() as session:
+ async with create_http_session(timeout=30) as session:
# Use CoinGeckoClient's API key if available
headers = CoinGeckoClient.get_headers()
base_url = CoinGeckoClient.get_base_url()
@@ -175,7 +182,6 @@ async def fetch_cache_api_data() -> Optional[Dict[str, Any]]:
f"{base_url}/coins/markets",
params=params,
headers=headers,
- timeout=aiohttp.ClientTimeout(total=30),
) as response:
if response.status != 200:
raise Exception(f"API error: {response.status}")
@@ -236,8 +242,14 @@ async def fetch_single_coin_data(coin_id: str) -> Dict[str, Any]:
Raises:
APIFetchError: If API request fails or returns invalid data
"""
+ cached = CoinGeckoClient._single_coin_cache.get(coin_id)
+ if cached:
+ cached_at, cached_payload = cached
+ if (time.time() - cached_at) <= CoinGeckoClient._single_coin_cache_ttl_s:
+ return cached_payload
+
try:
- async with aiohttp.ClientSession() as session:
+ async with create_http_session(timeout=30) as session:
headers = CoinGeckoClient.get_headers()
base_url = CoinGeckoClient.get_base_url()
@@ -251,27 +263,47 @@ async def fetch_single_coin_data(coin_id: str) -> Dict[str, Any]:
"price_change_percentage": "24h,7d,30d",
}
- await CoinGeckoClient._rate_limit()
-
- async with session.get(
- f"{base_url}/coins/markets",
- params=params,
- headers=headers,
- timeout=aiohttp.ClientTimeout(total=30),
- ) as response:
- if response.status != 200:
+ data = None
+ last_error: APIFetchError | None = None
+ for attempt in range(3):
+ await CoinGeckoClient._rate_limit()
+ async with session.get(
+ f"{base_url}/coins/markets",
+ params=params,
+ headers=headers,
+ ) as response:
+ if response.status == 200:
+ data = await response.json()
+ break
+ if response.status == 429:
+ last_error = APIFetchError(
+ f"status=429 for coin_id={coin_id}",
+ source="coingecko",
+ status_code=429,
+ )
+ if cached:
+ return cached[1]
+ await asyncio.sleep(min(8, 2 * (attempt + 1)))
+ continue
raise APIFetchError(
f"status={response.status} for coin_id={coin_id}",
source="coingecko",
status_code=response.status,
)
- data = await response.json()
+
+ if data is None:
+ if last_error is not None:
+ raise last_error
+ raise APIFetchError(f"CoinGecko request failed for coin_id={coin_id}", source="coingecko")
if not data:
+ if cached:
+ return cached[1]
raise APIFetchError(f"Empty response for coin_id={coin_id}", source="coingecko")
validate_api_response(data, list, f"coin_id={coin_id}")
validate_api_response(data[0], dict, f"coin_id={coin_id} first element")
+ CoinGeckoClient._single_coin_cache[coin_id] = (time.time(), data[0])
return data[0]
except APIFetchError:
diff --git a/liveweb_arena/plugins/coingecko/coingecko.py b/liveweb_arena/plugins/coingecko/coingecko.py
index bbb0cf5..ab740e7 100644
--- a/liveweb_arena/plugins/coingecko/coingecko.py
+++ b/liveweb_arena/plugins/coingecko/coingecko.py
@@ -15,8 +15,16 @@
# URL slug to API coin ID mapping for coins where they differ
# CoinGecko sometimes uses different identifiers in URLs vs API
URL_SLUG_TO_COIN_ID = {
+ "bnb": "binancecoin",
+ "binance-coin": "binancecoin",
+ "binancecoin": "binancecoin",
+ "xrp": "ripple",
+ "ada": "cardano",
+ "polkadot-new": "polkadot",
+ "polkadot": "polkadot",
"polygon": "polygon-ecosystem-token", # Polygon rebranded MATIC to POL
"matic-network": "polygon-ecosystem-token",
+ "near-protocol": "near",
"avalanche": "avalanche-2",
"hedera": "hedera-hashgraph",
"lido-staked-ether": "staked-ether",
@@ -61,6 +69,29 @@ def get_blocked_patterns(self) -> List[str]:
"*-emoji-*",
]
+ def get_stable_url_patterns(self) -> List[str]:
+ return [
+ "/en",
+ "/en/coins/",
+ "/en/coins/*/historical_data",
+ "/en/highlights/",
+ ]
+
+ def classify_url(self, url: str) -> str | None:
+ parsed = urlparse(url)
+ host = (parsed.hostname or "").lower()
+ path = parsed.path.lower()
+ if "coingecko.com" not in host:
+ return None
+ coin_id = self._extract_coin_id(url)
+ if coin_id and not self.is_plausible_asset_id(url):
+ return "model_invalid_asset_id"
+ if "/coins/" in path and coin_id:
+ return None
+ if self._is_homepage(url) or "/en/highlights/" in path:
+ return None
+ return None
+
async def fetch_api_data(self, url: str) -> Dict[str, Any]:
"""
Fetch API data for a CoinGecko page.
@@ -138,3 +169,20 @@ def _extract_coin_id(self, url: str) -> str:
return URL_SLUG_TO_COIN_ID.get(url_slug, url_slug)
return ""
+
+ def is_plausible_asset_id(self, url: str) -> bool:
+ coin_id = self._extract_coin_id(url)
+ if not coin_id:
+ return True
+ # Known false positives from hybrid/company-name hallucinations.
+ invalid = {
+ "microsoft",
+ "google",
+ "exxon-mobil",
+ "jpmorgan-chase",
+ "tesla",
+ "walmart",
+ "apple",
+ "amazon",
+ }
+ return coin_id not in invalid
diff --git a/liveweb_arena/plugins/openlibrary/api_client.py b/liveweb_arena/plugins/openlibrary/api_client.py
index ee90b04..4cfbfb7 100644
--- a/liveweb_arena/plugins/openlibrary/api_client.py
+++ b/liveweb_arena/plugins/openlibrary/api_client.py
@@ -2,6 +2,7 @@
import asyncio
import logging
+import re
from typing import Any, Dict, List, Optional
import aiohttp
@@ -44,6 +45,22 @@ async def close_session():
])
+def _candidate_search_queries(query: str) -> list[str]:
+ """Return progressively normalized search queries without changing intent."""
+ candidates: list[str] = []
+
+ def _append(value: str):
+ value = value.strip()
+ if value and value not in candidates:
+ candidates.append(value)
+
+ _append(query)
+ _append(re.sub(r"\s+", " ", query))
+ _append(re.sub(r"[?!:;,.'\"()\\[\\]{}]+", " ", query))
+ _append(re.sub(r"\s+", " ", re.sub(r"[-_/]+", " ", query)))
+ return candidates
+
+
class OpenLibraryClient(BaseAPIClient):
"""
Open Library API client.
@@ -58,7 +75,7 @@ class OpenLibraryClient(BaseAPIClient):
# Rate limit: 1.5s between requests (Open Library asks for politeness)
_rate_limiter = RateLimiter(min_interval=1.5)
- MAX_RETRIES = 3
+ MAX_RETRIES = 4
@classmethod
async def get(
@@ -114,19 +131,22 @@ async def search_works(
Returns:
List of work dicts with stats fields
"""
- params: Dict[str, Any] = {
- "q": query,
- "limit": limit,
- "fields": SEARCH_FIELDS,
- }
- if sort:
- params["sort"] = sort
- if mode:
- params["mode"] = mode
-
- data = await cls.get("/search.json", params=params)
- if data and isinstance(data, dict):
- return data.get("docs", [])
+ for candidate in _candidate_search_queries(query):
+ params: Dict[str, Any] = {
+ "q": candidate,
+ "limit": limit,
+ "fields": SEARCH_FIELDS,
+ }
+ if sort:
+ params["sort"] = sort
+ if mode:
+ params["mode"] = mode
+
+ data = await cls.get("/search.json", params=params)
+ if data and isinstance(data, dict):
+ docs = data.get("docs", [])
+ if docs:
+ return docs
return []
@classmethod
diff --git a/liveweb_arena/plugins/openmeteo/__init__.py b/liveweb_arena/plugins/openmeteo/__init__.py
new file mode 100644
index 0000000..1775690
--- /dev/null
+++ b/liveweb_arena/plugins/openmeteo/__init__.py
@@ -0,0 +1,8 @@
+"""Open Meteo plugin package"""
+
+from .openmeteo import OpenMeteoPlugin
+
+# Import templates to register them
+from . import templates
+
+__all__ = ["OpenMeteoPlugin"]
diff --git a/liveweb_arena/plugins/openmeteo/api_client.py b/liveweb_arena/plugins/openmeteo/api_client.py
new file mode 100644
index 0000000..a934d8e
--- /dev/null
+++ b/liveweb_arena/plugins/openmeteo/api_client.py
@@ -0,0 +1,109 @@
+"""Open Meteo API client.
+
+Free weather API — no auth, no rate limits.
+Docs: https://open-meteo.com/en/docs
+"""
+
+import logging
+from typing import Any, ClassVar, Dict
+
+import aiohttp
+
+from liveweb_arena.plugins.base_client import APIFetchError, BaseAPIClient, RateLimiter
+
+logger = logging.getLogger(__name__)
+
+CACHE_SOURCE = "openmeteo"
+API_BASE = "https://api.open-meteo.com/v1/forecast"
+
+
+class OpenMeteoClient(BaseAPIClient):
+ """Open Meteo API client with rate limiting and session reuse."""
+
+ _rate_limiter: ClassVar[RateLimiter] = RateLimiter(min_interval=0.2)
+ _session: aiohttp.ClientSession = None
+
+ @classmethod
+ async def _get_session(cls) -> aiohttp.ClientSession:
+ if cls._session is None or cls._session.closed:
+ cls._session = aiohttp.ClientSession(
+ headers={"User-Agent": "LiveWebArena/1.0"},
+ )
+ return cls._session
+
+ @classmethod
+ async def close_session(cls):
+ if cls._session and not cls._session.closed:
+ await cls._session.close()
+ cls._session = None
+
+ @classmethod
+ async def get(
+ cls,
+ params: Dict[str, Any],
+ timeout: float = 15.0,
+ ) -> Dict[str, Any]:
+ """Fetch from Open Meteo API with rate limiting."""
+ await cls._rate_limit()
+ session = await cls._get_session()
+ req_timeout = aiohttp.ClientTimeout(total=timeout)
+
+ try:
+ async with session.get(API_BASE, params=params, timeout=req_timeout) as resp:
+ if resp.status != 200:
+ text = await resp.text()
+ raise APIFetchError(
+ f"Open Meteo API returned {resp.status}: {text[:200]}",
+ source=CACHE_SOURCE,
+ status_code=resp.status,
+ )
+ data = await resp.json(content_type=None)
+ if not isinstance(data, dict) or "current_weather" not in data:
+ raise APIFetchError(
+ "Open Meteo API returned unexpected format",
+ source=CACHE_SOURCE,
+ )
+ return data
+ except APIFetchError:
+ raise
+ except Exception as e:
+ raise APIFetchError(
+ f"Open Meteo API request failed: {e}",
+ source=CACHE_SOURCE,
+ ) from e
+
+
+async def fetch_forecast(
+ latitude: float,
+ longitude: float,
+ forecast_days: int = 3,
+) -> Dict[str, Any]:
+ """
+ Fetch weather forecast from Open Meteo API.
+
+ Returns the full API response with current weather, hourly, and daily data.
+
+ Raises:
+ APIFetchError: If the API request fails
+ """
+ params = {
+ "latitude": latitude,
+ "longitude": longitude,
+ "current_weather": "true",
+ "hourly": ",".join([
+ "temperature_2m",
+ "relative_humidity_2m",
+ "wind_speed_10m",
+ "precipitation_probability",
+ ]),
+ "daily": ",".join([
+ "temperature_2m_max",
+ "temperature_2m_min",
+ "precipitation_probability_max",
+ "sunrise",
+ "sunset",
+ ]),
+ "timezone": "auto",
+ "forecast_days": forecast_days,
+ }
+ return await OpenMeteoClient.get(params)
diff --git a/liveweb_arena/plugins/openmeteo/openmeteo.py b/liveweb_arena/plugins/openmeteo/openmeteo.py
new file mode 100644
index 0000000..97c42a0
--- /dev/null
+++ b/liveweb_arena/plugins/openmeteo/openmeteo.py
@@ -0,0 +1,213 @@
+"""
+Open Meteo Plugin.
+
+Uses the Open Meteo docs page as the browsable interface.
+The agent navigates to open-meteo.com/en/docs with location coordinates
+encoded as both query params (for cache uniqueness) and hash fragment
+(for client-side JS form state).
+
+API data is fetched via the Open Meteo forecast API for GT extraction.
+
+Cache support: The docs page is a SvelteKit SPA whose weather data renders
+in canvas charts (inaccessible to screen readers). setup_page_for_cache()
+injects the API data as readable HTML tables so the cached DOM snapshot
+contains accessible weather values without needing JS hydration.
+"""
+
+import logging
+from typing import Any, Dict, List, Optional
+from urllib.parse import urlparse, parse_qs, unquote
+
+from liveweb_arena.plugins.base import BasePlugin
+from .api_client import fetch_forecast
+
+logger = logging.getLogger(__name__)
+
+
+class OpenMeteoPlugin(BasePlugin):
+ """
+ Open Meteo plugin for weather forecast data.
+
+ Handles the docs page:
+ - https://open-meteo.com/en/docs?latitude=35.68&longitude=139.65#...
+
+ Query params provide unique cache keys (normalize_url preserves them).
+ Hash fragment configures the client-side JS form/chart.
+
+ API data includes: current weather, hourly forecasts, daily aggregates,
+ sunrise/sunset times.
+ """
+
+ name = "openmeteo"
+
+ allowed_domains = [
+ "open-meteo.com",
+ ]
+
+ def get_blocked_patterns(self) -> List[str]:
+ """No blocks — the docs page JS needs api.open-meteo.com for chart rendering."""
+ return []
+
+ def needs_api_data(self, url: str) -> bool:
+ """Only docs pages with coordinates need API data."""
+ lat, lon = self._extract_coords(url)
+ return lat is not None
+
+ async def fetch_api_data(self, url: str) -> Dict[str, Any]:
+ """
+ Fetch forecast data for the location encoded in the URL.
+
+ Extracts latitude/longitude from query params or hash fragment,
+ then calls the Open Meteo forecast API.
+ """
+ lat, lon = self._extract_coords(url)
+ if lat is None or lon is None:
+ return {}
+
+ data = await fetch_forecast(lat, lon)
+ # Add a location key for GT collector identification
+ data["_location_key"] = f"{lat:.2f},{lon:.2f}"
+ return data
+
+ async def setup_page_for_cache(self, page, url: str) -> None:
+ """Inject weather data as readable HTML tables for cache mode.
+
+ The SvelteKit docs page renders weather data in canvas charts that
+ produce no useful accessibility tree text. This method fetches the
+ API data and prepends it as HTML tables so the cached DOM snapshot
+ contains all values the agent needs to read.
+ """
+ lat, lon = self._extract_coords(url)
+ if lat is None:
+ return
+
+ try:
+ data = await fetch_forecast(lat, lon)
+ except Exception as e:
+ logger.warning("setup_page_for_cache: API fetch failed: %s", e)
+ return
+
+ html = self._build_data_html(data)
+ escaped = html.replace("\\", "\\\\").replace("`", "\\`").replace("$", "\\$")
+
+ await page.evaluate(f"""
+ (() => {{
+ const div = document.createElement('div');
+ div.id = 'liveweb-weather-data';
+ div.setAttribute('role', 'region');
+ div.setAttribute('aria-label', 'Weather Data');
+ div.innerHTML = `{escaped}`;
+ document.body.prepend(div);
+ }})()
+ """)
+
+ @staticmethod
+ def _build_data_html(data: dict) -> str:
+ """Format API data as readable HTML tables."""
+ parts: list = []
+ cw = data.get("current_weather", {})
+ if cw:
+ parts.append(
+ "
Current Weather
"
+ f"
Temperature
{cw.get('temperature', 'N/A')} C
"
+ f"
Wind Speed
{cw.get('windspeed', 'N/A')} km/h
"
+ f"
Wind Direction
{cw.get('winddirection', 'N/A')} deg
"
+ "
"
+ )
+
+ daily = data.get("daily", {})
+ times = daily.get("time", [])
+ if times:
+ rows = []
+ t_max = daily.get("temperature_2m_max", [])
+ t_min = daily.get("temperature_2m_min", [])
+ p_max = daily.get("precipitation_probability_max", [])
+ for i, t in enumerate(times):
+ mx = t_max[i] if i < len(t_max) else "N/A"
+ mn = t_min[i] if i < len(t_min) else "N/A"
+ pp = p_max[i] if i < len(p_max) else "N/A"
+ rows.append(f"
{t}
{mx} C
{mn} C
{pp}%
")
+ parts.append(
+ "
Daily Forecast
"
+ "
Date
Max Temp
Min Temp
Precip Prob
"
+ + "".join(rows) + "
"
+ )
+
+ hourly = data.get("hourly", {})
+ h_times = hourly.get("time", [])
+ if h_times:
+ today = times[0] if times else h_times[0].split("T")[0]
+ rows = []
+ h_temp = hourly.get("temperature_2m", [])
+ h_hum = hourly.get("relative_humidity_2m", [])
+ h_wind = hourly.get("wind_speed_10m", [])
+ h_prec = hourly.get("precipitation_probability", [])
+ for i, ht in enumerate(h_times):
+ if not ht.startswith(today):
+ continue
+ tm = h_temp[i] if i < len(h_temp) else ""
+ hu = h_hum[i] if i < len(h_hum) else ""
+ ws = h_wind[i] if i < len(h_wind) else ""
+ pp = h_prec[i] if i < len(h_prec) else ""
+ rows.append(
+ f"
{ht}
{tm} C
{hu}%
"
+ f"
{ws} km/h
{pp}%
"
+ )
+ parts.append(
+ "
Hourly Forecast (Today)
"
+ "
Time
Temp
Humidity
"
+ "
Wind Speed
Precip Prob
"
+ + "".join(rows) + "
"
+ )
+
+ return "".join(parts)
+
+ def _extract_coords(self, url: str) -> tuple:
+ """
+ Extract latitude and longitude from URL.
+
+ Tries query params first (preserved by normalize_url), then hash
+ fragment as fallback. Both formats are present in docs_url().
+ """
+ parsed = urlparse(url)
+
+ # Query params first (reliable — survive normalize_url)
+ params = parse_qs(parsed.query)
+ lat_vals = params.get("latitude")
+ lon_vals = params.get("longitude")
+ if lat_vals and lon_vals:
+ try:
+ return float(lat_vals[0]), float(lon_vals[0])
+ except (ValueError, IndexError):
+ pass
+
+ # Fall back to hash fragment (client-side only, stripped by cache)
+ fragment = parsed.fragment
+ if fragment:
+ lat, lon = self._parse_coord_params(fragment)
+ if lat is not None:
+ return lat, lon
+
+ return None, None
+
+ @staticmethod
+ def _parse_coord_params(fragment: str) -> tuple:
+ """Parse latitude and longitude from a URL fragment like 'latitude=35.68&longitude=139.65&...'"""
+ lat = None
+ lon = None
+ for part in fragment.split("&"):
+ if "=" not in part:
+ continue
+ key, val = part.split("=", 1)
+ key = unquote(key).strip()
+ val = unquote(val).strip()
+ try:
+ if key == "latitude":
+ lat = float(val)
+ elif key == "longitude":
+ lon = float(val)
+ except ValueError:
+ continue
+ if lat is not None and lon is not None:
+ return lat, lon
+ return None, None
diff --git a/liveweb_arena/plugins/openmeteo/templates/__init__.py b/liveweb_arena/plugins/openmeteo/templates/__init__.py
new file mode 100644
index 0000000..9c3a246
--- /dev/null
+++ b/liveweb_arena/plugins/openmeteo/templates/__init__.py
@@ -0,0 +1,13 @@
+"""Open Meteo question templates"""
+
+from .current_weather import OpenMeteoCurrentWeatherTemplate
+from .comparison import OpenMeteoComparisonTemplate
+from .hourly_extrema import OpenMeteoHourlyExtremaTemplate
+from .forecast_trend import OpenMeteoForecastTrendTemplate
+
+__all__ = [
+ "OpenMeteoCurrentWeatherTemplate",
+ "OpenMeteoComparisonTemplate",
+ "OpenMeteoHourlyExtremaTemplate",
+ "OpenMeteoForecastTrendTemplate",
+]
diff --git a/liveweb_arena/plugins/openmeteo/templates/common.py b/liveweb_arena/plugins/openmeteo/templates/common.py
new file mode 100644
index 0000000..9acfbc8
--- /dev/null
+++ b/liveweb_arena/plugins/openmeteo/templates/common.py
@@ -0,0 +1,98 @@
+"""Shared helpers for Open Meteo templates."""
+
+from typing import Any, Dict, List, Optional, Tuple
+
+from liveweb_arena.core.ground_truth_trigger import GroundTruthResult
+from liveweb_arena.core.gt_collector import get_current_gt_collector
+
+DOCS_HOME_URL = "https://open-meteo.com/en/docs"
+
+
+def get_collected_location_data(
+ coord_key: str,
+ city_name: str,
+) -> Tuple[Optional[Dict[str, Any]], Optional[GroundTruthResult]]:
+ """Return collected API data for a visited city, or a GT failure."""
+ gt_collector = get_current_gt_collector()
+ if gt_collector is None:
+ return None, GroundTruthResult.fail("No GT collector")
+
+ collected = gt_collector.get_collected_api_data()
+ data = collected.get(f"openmeteo:{coord_key}")
+ if data is None:
+ keys = [k for k in collected if k.startswith("openmeteo:")][:5]
+ return None, GroundTruthResult.not_collected(
+ f"Agent did not visit Open Meteo page for '{city_name}'. Collected keys: {keys}"
+ )
+
+ return data, None
+
+
+def get_today_hourly_series(
+ data: Dict[str, Any],
+ field_name: str,
+) -> Tuple[Optional[List[float]], Optional[GroundTruthResult]]:
+ """Extract today's hourly values for the given field from API data.
+
+ Returns (values, None) on success, or (None, failure_result) on error.
+ """
+ hourly = data.get("hourly")
+ if not hourly:
+ return None, GroundTruthResult.fail("No hourly data in API response")
+
+ times = hourly.get("time")
+ series = hourly.get(field_name)
+ if not isinstance(times, list) or not isinstance(series, list):
+ return None, GroundTruthResult.fail(
+ f"Hourly data missing time/{field_name} arrays"
+ )
+ if len(times) != len(series):
+ return None, GroundTruthResult.fail(
+ f"Hourly time and {field_name} arrays differ in length"
+ )
+ if not times:
+ return None, GroundTruthResult.fail("Hourly forecast is empty")
+
+ # Determine today's date from the data
+ today = None
+ current = data.get("current_weather")
+ if isinstance(current, dict):
+ current_time = current.get("time")
+ if isinstance(current_time, str) and "T" in current_time:
+ today = current_time.split("T", 1)[0]
+
+ if not today:
+ daily = data.get("daily")
+ daily_times = daily.get("time") if isinstance(daily, dict) else None
+ if isinstance(daily_times, list) and daily_times:
+ today = str(daily_times[0]).split("T", 1)[0]
+
+ if not today:
+ today = str(times[0]).split("T", 1)[0]
+
+ values: List[float] = []
+ for time_str, val in zip(times, series):
+ if not isinstance(time_str, str) or not time_str.startswith(today):
+ continue
+ if val is None:
+ continue
+ try:
+ values.append(float(val))
+ except (TypeError, ValueError):
+ return None, GroundTruthResult.fail(
+ f"Non-numeric value in hourly {field_name}: {val!r}"
+ )
+
+ if not values:
+ return None, GroundTruthResult.fail(
+ f"No hourly {field_name} data found for today ({today})"
+ )
+
+ return values, None
+
+
+def get_today_hourly_temperatures(
+ data: Dict[str, Any],
+) -> Tuple[Optional[List[float]], Optional[GroundTruthResult]]:
+ """Extract today's hourly temperatures from a collected API payload."""
+ return get_today_hourly_series(data, "temperature_2m")
diff --git a/liveweb_arena/plugins/openmeteo/templates/comparison.py b/liveweb_arena/plugins/openmeteo/templates/comparison.py
new file mode 100644
index 0000000..9991b8a
--- /dev/null
+++ b/liveweb_arena/plugins/openmeteo/templates/comparison.py
@@ -0,0 +1,147 @@
+"""Weather comparison template for Open Meteo - HARD DIFFICULTY
+
+Computes temperature difference between two cities in different climate zones.
+Requires the agent to visit two separate pages, read both temperatures,
+and compute the numeric difference.
+
+Dynamic data: temperatures change continuously.
+City pairs drawn from 170 cities: C(170,2) = 14365 possible pairs.
+Answer is a numeric difference (deg C), not a binary choice — random baseline ~ 0%.
+"""
+
+import random
+from typing import Any, Dict, Optional
+
+from liveweb_arena.core.validators.base import (
+ QuestionTemplate, GeneratedQuestion, ValidationResult, register_template,
+)
+from liveweb_arena.core.ground_truth_trigger import (
+ UrlPatternTrigger, TriggerConfig, GroundTruthResult,
+)
+from liveweb_arena.core.gt_collector import GTSourceType, get_current_gt_collector
+
+from .variables import CITIES
+
+
+PATTERNS = [
+ "Using Open-Meteo, how many degrees warmer is {city1} than {city2} right now? Give the difference in °C.",
+ "On Open-Meteo, what is the temperature difference between {city1} and {city2} right now in °C? Positive means {city1} is warmer.",
+ "According to Open-Meteo, find the current temperatures of {city1} and {city2}, then compute the difference ({city1} minus {city2}) in °C.",
+ "Check the current temperature in {city1} and {city2} on Open-Meteo. What is the difference in degrees Celsius ({city1} minus {city2})?",
+]
+
+
+@register_template("openmeteo_comparison")
+class OpenMeteoComparisonTemplate(QuestionTemplate):
+ """
+ HARD: Compute temperature difference between two cities.
+
+ Requires visiting two different location pages, reading both temperatures,
+ and computing the numeric difference. Answer is a signed number in deg C.
+ C(170, 2) = 14365 city pairs x 4 patterns x 2 orderings = 114920 variants.
+ """
+
+ GT_SOURCE = GTSourceType.PAGE_ONLY
+
+ def __init__(self):
+ super().__init__("openmeteo_comparison")
+
+ def generate(self, seed: int, variant: Optional[int] = None) -> GeneratedQuestion:
+ rng = random.Random(seed)
+
+ pair = rng.sample(CITIES, 2)
+ city1, city2 = pair
+
+ # Randomly swap order
+ if rng.random() > 0.5:
+ city1, city2 = city2, city1
+
+ pattern = rng.choice(PATTERNS)
+ question_text = pattern.format(
+ city1=city1.display_name,
+ city2=city2.display_name,
+ )
+
+ return GeneratedQuestion(
+ question_text=question_text,
+ start_url=city1.docs_url(),
+ variables={"city1": city1.name, "city2": city2.name},
+ validation_info={
+ "city1_name": city1.name,
+ "city1_coord_key": city1.coord_key,
+ "city2_name": city2.name,
+ "city2_coord_key": city2.coord_key,
+ "city2_url": city2.docs_url(),
+ },
+ template_name=self.name,
+ expected_steps=8,
+ )
+
+ def get_validation_rules(self, validation_info: Dict[str, Any]) -> str:
+ city1 = validation_info.get("city1_name", "City1")
+ city2 = validation_info.get("city2_name", "City2")
+ return f"""Task-Specific Rules (Open Meteo Temperature Difference):
+- Answer is the temperature difference: {city1} minus {city2} in °C
+- Positive means {city1} is warmer, negative means {city2} is warmer
+- Score 1.0: Difference within ±2°C of ground truth
+- Score 0.5: Difference within ±5°C of ground truth
+- Score 0.0: Difference off by more than 5°C, or wrong sign, or no numeric answer
+- Accept formats: "5.2", "5.2°C", "-3.1°C", "+5.2"
+- Do NOT accept answers that only name a city without the numeric difference"""
+
+ async def get_ground_truth(self, validation_info: Dict[str, Any]) -> GroundTruthResult:
+ city1_name = validation_info.get("city1_name", "")
+ city2_name = validation_info.get("city2_name", "")
+ key1 = validation_info.get("city1_coord_key", "")
+ key2 = validation_info.get("city2_coord_key", "")
+
+ gt_collector = get_current_gt_collector()
+ if gt_collector is None:
+ return GroundTruthResult.fail("No GT collector")
+
+ collected = gt_collector.get_collected_api_data()
+
+ data1 = collected.get(f"openmeteo:{key1}")
+ if data1 is None:
+ return GroundTruthResult.not_collected(
+ f"Weather data for '{city1_name}' not collected"
+ )
+
+ data2 = collected.get(f"openmeteo:{key2}")
+ if data2 is None:
+ return GroundTruthResult.not_collected(
+ f"Weather data for '{city2_name}' not collected"
+ )
+
+ cw1 = data1.get("current_weather")
+ cw2 = data2.get("current_weather")
+ if not cw1 or "temperature" not in cw1:
+ return GroundTruthResult.fail(f"No temperature data for '{city1_name}'")
+ if not cw2 or "temperature" not in cw2:
+ return GroundTruthResult.fail(f"No temperature data for '{city2_name}'")
+
+ temp1 = float(cw1["temperature"])
+ temp2 = float(cw2["temperature"])
+ diff = round(temp1 - temp2, 1)
+
+ return GroundTruthResult.ok(f"{diff}°C")
+
+ async def validate_answer(
+ self, answer: str, validation_info: Dict[str, Any]
+ ) -> ValidationResult:
+ """Not used — the pipeline uses LLM-based validation via get_validation_rules()."""
+ return ValidationResult(
+ score=0.0, is_correct=False, expected=None, actual=answer,
+ details="Use LLM validation",
+ )
+
+ def get_ground_truth_trigger(self, validation_info: dict) -> TriggerConfig:
+ trigger = UrlPatternTrigger(domains=["open-meteo.com"])
+ return TriggerConfig(trigger=trigger)
+
+ @classmethod
+ def get_cache_source(cls) -> str:
+ return "openmeteo"
+
+ def get_gt_source(self) -> GTSourceType:
+ return self.GT_SOURCE
diff --git a/liveweb_arena/plugins/openmeteo/templates/current_weather.py b/liveweb_arena/plugins/openmeteo/templates/current_weather.py
new file mode 100644
index 0000000..a748da1
--- /dev/null
+++ b/liveweb_arena/plugins/openmeteo/templates/current_weather.py
@@ -0,0 +1,137 @@
+"""Current weather template for Open Meteo - EASY DIFFICULTY.
+
+Asks for a single current weather metric (temperature, wind speed, or wind
+direction) for a given city. The agent starts on the generic Open-Meteo docs
+page, searches for the location, and then reads the current value.
+
+Dynamic data: current weather updates every 15 minutes.
+Large entity pool: 170 cities x 3 metrics = 510 effective variants (>500).
+"""
+
+import random
+from typing import Any, Dict, Optional
+
+from liveweb_arena.core.validators.base import (
+ QuestionTemplate, GeneratedQuestion, ValidationResult, register_template,
+)
+from liveweb_arena.core.ground_truth_trigger import (
+ UrlPatternTrigger, TriggerConfig, GroundTruthResult,
+)
+from liveweb_arena.core.gt_collector import GTSourceType
+
+from .common import DOCS_HOME_URL, get_collected_location_data
+from .variables import CITIES, CurrentMetric
+
+
+PATTERNS = {
+ CurrentMetric.TEMPERATURE: [
+ "What is the current temperature in {city} according to Open-Meteo?",
+ "On Open-Meteo, what is the temperature in {city} right now?",
+ "Using the Open-Meteo weather service, find the current temperature in {city}.",
+ ],
+ CurrentMetric.WIND_SPEED: [
+ "What is the current wind speed in {city} according to Open-Meteo?",
+ "On Open-Meteo, what is the wind speed in {city} right now?",
+ "Using the Open-Meteo weather service, find the current wind speed in {city}.",
+ ],
+ CurrentMetric.WIND_DIRECTION: [
+ "What is the current wind direction in {city} according to Open-Meteo? Answer in degrees.",
+ "On Open-Meteo, what direction is the wind blowing in {city} right now? Give the answer in degrees.",
+ "Using the Open-Meteo weather service, find the current wind direction in {city} in degrees.",
+ ],
+}
+
+
+@register_template("openmeteo_current")
+class OpenMeteoCurrentWeatherTemplate(QuestionTemplate):
+ """
+ EASY: Use location search, then read a single current metric.
+
+ RL value:
+ - Form interaction: must search/select a location in the docs UI
+ - Dynamic data: weather changes every 15 minutes
+ - 170 cities x 3 metrics x 3 patterns = 1530 question variants
+ """
+
+ GT_SOURCE = GTSourceType.PAGE_ONLY
+
+ def __init__(self):
+ super().__init__("openmeteo_current")
+
+ def generate(self, seed: int, variant: Optional[int] = None) -> GeneratedQuestion:
+ rng = random.Random(seed)
+
+ metrics = list(CurrentMetric)
+ metric = metrics[variant % len(metrics)] if variant is not None else rng.choice(metrics)
+
+ city = rng.choice(CITIES)
+ pattern = rng.choice(PATTERNS[metric])
+ question_text = pattern.format(city=city.display_name)
+
+ return GeneratedQuestion(
+ question_text=question_text,
+ start_url=DOCS_HOME_URL,
+ variables={"city": city.name, "metric": metric.name},
+ validation_info={
+ "city_name": city.name,
+ "coord_key": city.coord_key,
+ "metric_field": metric.api_field,
+ "metric_label": metric.display_name,
+ "unit": metric.unit,
+ },
+ template_name=self.name,
+ expected_steps=6,
+ )
+
+ def get_validation_rules(self, validation_info: Dict[str, Any]) -> str:
+ city = validation_info.get("city_name", "")
+ label = validation_info.get("metric_label", "")
+ unit = validation_info.get("unit", "")
+ return f"""Task-Specific Rules (Open Meteo Current Weather):
+- City: {city}
+- Metric: {label}
+- Score 1.0: Value within ±2{unit} of correct answer
+- Score 0.5: Value within ±5{unit}
+- Score 0.0: Wrong value or no answer
+- The answer should reflect the city selected on Open-Meteo, not a guessed climatology
+- Data source: Open-Meteo weather service (open-meteo.com)"""
+
+ async def get_ground_truth(self, validation_info: Dict[str, Any]) -> GroundTruthResult:
+ coord_key = validation_info.get("coord_key", "")
+ metric_field = validation_info.get("metric_field", "")
+ unit = validation_info.get("unit", "")
+ city_name = validation_info.get("city_name", "")
+
+ data, failure = get_collected_location_data(coord_key, city_name)
+ if failure is not None:
+ return failure
+
+ current = data.get("current_weather")
+ if not current:
+ return GroundTruthResult.fail("No current_weather in API data")
+
+ value = current.get(metric_field)
+ if value is None:
+ return GroundTruthResult.fail(f"Field '{metric_field}' not in current_weather")
+
+ return GroundTruthResult.ok(f"{value}{unit}")
+
+ async def validate_answer(
+ self, answer: str, validation_info: Dict[str, Any]
+ ) -> ValidationResult:
+ """Not used — the pipeline uses LLM-based validation via get_validation_rules()."""
+ return ValidationResult(
+ score=0.0, is_correct=False, expected=None, actual=answer,
+ details="Use LLM validation",
+ )
+
+ def get_ground_truth_trigger(self, validation_info: dict) -> TriggerConfig:
+ trigger = UrlPatternTrigger(domains=["open-meteo.com"])
+ return TriggerConfig(trigger=trigger)
+
+ @classmethod
+ def get_cache_source(cls) -> str:
+ return "openmeteo"
+
+ def get_gt_source(self) -> GTSourceType:
+ return self.GT_SOURCE
diff --git a/liveweb_arena/plugins/openmeteo/templates/forecast_trend.py b/liveweb_arena/plugins/openmeteo/templates/forecast_trend.py
new file mode 100644
index 0000000..0679dd1
--- /dev/null
+++ b/liveweb_arena/plugins/openmeteo/templates/forecast_trend.py
@@ -0,0 +1,167 @@
+"""Forecast trend template for Open Meteo - MEDIUM DIFFICULTY.
+
+Asks whether a daily metric will be higher or lower on one day vs another
+in a given city. The agent starts on the generic docs page, finds the
+location, then compares the relevant daily values.
+
+Dynamic data: forecasts update continuously.
+Time-sensitive: day references change daily.
+Computation required: must compare two values, not read a single one.
+
+Effective variants: 170 cities x 3 metrics x 3 day-pairs = 1530 (>500).
+"""
+
+import random
+from typing import Any, Dict, Optional
+
+from liveweb_arena.core.validators.base import (
+ QuestionTemplate, GeneratedQuestion, ValidationResult, register_template,
+)
+from liveweb_arena.core.ground_truth_trigger import (
+ UrlPatternTrigger, TriggerConfig, GroundTruthResult,
+)
+from liveweb_arena.core.gt_collector import GTSourceType
+
+from .common import DOCS_HOME_URL, get_collected_location_data
+from .variables import CITIES, DailyMetric
+
+
+DAY_PAIRS = [
+ (0, 1, "today", "tomorrow"),
+ (0, 2, "today", "the day after tomorrow"),
+ (1, 2, "tomorrow", "the day after tomorrow"),
+]
+
+PATTERNS = [
+ "According to Open-Meteo, will {day2}'s {metric_label} in {city} be higher or lower than {day1}'s? By how many {unit}?",
+ "On Open-Meteo, compare the {metric_label} {day1} vs {day2} in {city}. Which day has a higher value and by how much?",
+ "Using Open-Meteo, is {day2}'s {metric_label} in {city} higher or lower than {day1}'s? What is the difference in {unit}?",
+]
+
+
+@register_template("openmeteo_forecast_trend")
+class OpenMeteoForecastTrendTemplate(QuestionTemplate):
+ """
+ MEDIUM: Compare a daily metric across two days.
+
+ Requires reading daily forecast for two days and computing the difference.
+ 170 cities x 3 metrics x 3 day-pairs = 1530 effective variants.
+ """
+
+ GT_SOURCE = GTSourceType.PAGE_ONLY
+
+ def __init__(self):
+ super().__init__("openmeteo_forecast_trend")
+
+ def generate(self, seed: int, variant: Optional[int] = None) -> GeneratedQuestion:
+ rng = random.Random(seed)
+
+ city = rng.choice(CITIES)
+
+ metrics = list(DailyMetric)
+ metric = rng.choice(metrics)
+
+ day_pair = rng.choice(DAY_PAIRS)
+ idx1, idx2, day1_label, day2_label = day_pair
+
+ pattern = rng.choice(PATTERNS)
+ question_text = pattern.format(
+ city=city.display_name,
+ metric_label=metric.display_name,
+ day1=day1_label,
+ day2=day2_label,
+ unit=metric.unit,
+ )
+
+ return GeneratedQuestion(
+ question_text=question_text,
+ start_url=DOCS_HOME_URL,
+ variables={"city": city.name, "metric": metric.name},
+ validation_info={
+ "city_name": city.name,
+ "coord_key": city.coord_key,
+ "metric_field": metric.api_field,
+ "metric_label": metric.display_name,
+ "unit": metric.unit,
+ "day1_idx": idx1,
+ "day2_idx": idx2,
+ "day1_label": day1_label,
+ "day2_label": day2_label,
+ },
+ template_name=self.name,
+ expected_steps=7,
+ )
+
+ def get_validation_rules(self, validation_info: Dict[str, Any]) -> str:
+ city = validation_info.get("city_name", "")
+ label = validation_info.get("metric_label", "daily maximum temperature")
+ unit = validation_info.get("unit", "°C")
+ day1 = validation_info.get("day1_label", "today")
+ day2 = validation_info.get("day2_label", "tomorrow")
+ return f"""Task-Specific Rules (Open Meteo Forecast Trend):
+- City: {city}
+- Compare {day1}'s {label} vs {day2}'s {label}
+- Answer should state: higher/lower + the difference in {unit}
+- Score 1.0: Correct direction (higher/lower) AND difference within ±1{unit}
+- Score 0.5: Correct direction but difference off by more than 1{unit}
+- Score 0.0: Wrong direction or no answer"""
+
+ async def get_ground_truth(self, validation_info: Dict[str, Any]) -> GroundTruthResult:
+ coord_key = validation_info.get("coord_key", "")
+ city_name = validation_info.get("city_name", "")
+ metric_field = validation_info.get("metric_field", "temperature_2m_max")
+ unit = validation_info.get("unit", "°C")
+ idx1 = validation_info.get("day1_idx", 0)
+ idx2 = validation_info.get("day2_idx", 1)
+ day1_label = validation_info.get("day1_label", "today")
+ day2_label = validation_info.get("day2_label", "tomorrow")
+
+ data, failure = get_collected_location_data(coord_key, city_name)
+ if failure is not None:
+ return failure
+
+ daily = data.get("daily")
+ if not daily:
+ return GroundTruthResult.fail("No daily data in API response")
+
+ values = daily.get(metric_field)
+ if not values or len(values) <= max(idx1, idx2):
+ return GroundTruthResult.fail(
+ f"Need at least {max(idx1, idx2) + 1} days of {metric_field}"
+ )
+
+ val1 = float(values[idx1])
+ val2 = float(values[idx2])
+ diff = val2 - val1
+
+ if diff > 0:
+ return GroundTruthResult.ok(
+ f"Higher by {abs(diff):.1f}{unit} ({day1_label}: {val1}{unit}, {day2_label}: {val2}{unit})"
+ )
+ if diff < 0:
+ return GroundTruthResult.ok(
+ f"Lower by {abs(diff):.1f}{unit} ({day1_label}: {val1}{unit}, {day2_label}: {val2}{unit})"
+ )
+ return GroundTruthResult.ok(
+ f"Same value ({val1}{unit} both days)"
+ )
+
+ async def validate_answer(
+ self, answer: str, validation_info: Dict[str, Any]
+ ) -> ValidationResult:
+ """Not used — the pipeline uses LLM-based validation via get_validation_rules()."""
+ return ValidationResult(
+ score=0.0, is_correct=False, expected=None, actual=answer,
+ details="Use LLM validation",
+ )
+
+ def get_ground_truth_trigger(self, validation_info: dict) -> TriggerConfig:
+ trigger = UrlPatternTrigger(domains=["open-meteo.com"])
+ return TriggerConfig(trigger=trigger)
+
+ @classmethod
+ def get_cache_source(cls) -> str:
+ return "openmeteo"
+
+ def get_gt_source(self) -> GTSourceType:
+ return self.GT_SOURCE
diff --git a/liveweb_arena/plugins/openmeteo/templates/hourly_extrema.py b/liveweb_arena/plugins/openmeteo/templates/hourly_extrema.py
new file mode 100644
index 0000000..5fcd37a
--- /dev/null
+++ b/liveweb_arena/plugins/openmeteo/templates/hourly_extrema.py
@@ -0,0 +1,161 @@
+"""Hourly extrema template for Open Meteo - MEDIUM DIFFICULTY.
+
+Asks for the highest or lowest hourly value of a given metric today in a city.
+The agent starts on the generic docs page, finds the city, then scans the
+hourly forecast series to locate the extreme value.
+
+Dynamic data: hourly forecasts update continuously.
+Time-sensitive: asks about "today" which changes daily.
+Computation required: agent must compare across hours, not read a single value.
+
+Effective variants: 170 cities x 4 metrics x 2 extrema = 1360 (>500).
+"""
+
+import random
+from typing import Any, Dict, Optional
+
+from liveweb_arena.core.validators.base import (
+ QuestionTemplate, GeneratedQuestion, ValidationResult, register_template,
+)
+from liveweb_arena.core.ground_truth_trigger import (
+ UrlPatternTrigger, TriggerConfig, GroundTruthResult,
+)
+from liveweb_arena.core.gt_collector import GTSourceType
+
+from .common import DOCS_HOME_URL, get_collected_location_data, get_today_hourly_series
+from .variables import CITIES, HourlyMetric
+
+
+PATTERNS_HIGH = {
+ HourlyMetric.TEMPERATURE: [
+ "What is the highest hourly temperature forecast for {city} today on Open-Meteo?",
+ "Using Open-Meteo, find today's peak hourly temperature in {city}.",
+ "On Open-Meteo, what is the warmest hourly temperature expected in {city} today?",
+ ],
+ HourlyMetric.HUMIDITY: [
+ "What is the highest hourly relative humidity forecast for {city} today on Open-Meteo?",
+ "Using Open-Meteo, find today's peak hourly humidity in {city}.",
+ ],
+ HourlyMetric.WIND_SPEED: [
+ "What is the highest hourly wind speed forecast for {city} today on Open-Meteo?",
+ "Using Open-Meteo, find today's peak hourly wind speed in {city}.",
+ ],
+ HourlyMetric.PRECIP_PROBABILITY: [
+ "What is the highest hourly precipitation probability forecast for {city} today on Open-Meteo?",
+ "Using Open-Meteo, find today's peak precipitation probability in {city}.",
+ ],
+}
+
+PATTERNS_LOW = {
+ HourlyMetric.TEMPERATURE: [
+ "What is the lowest hourly temperature forecast for {city} today on Open-Meteo?",
+ "Using Open-Meteo, find today's lowest hourly temperature in {city}.",
+ "On Open-Meteo, what is the coolest hourly temperature expected in {city} today?",
+ ],
+ HourlyMetric.HUMIDITY: [
+ "What is the lowest hourly relative humidity forecast for {city} today on Open-Meteo?",
+ "Using Open-Meteo, find today's lowest hourly humidity in {city}.",
+ ],
+ HourlyMetric.WIND_SPEED: [
+ "What is the lowest hourly wind speed forecast for {city} today on Open-Meteo?",
+ "Using Open-Meteo, find today's lowest hourly wind speed in {city}.",
+ ],
+ HourlyMetric.PRECIP_PROBABILITY: [
+ "What is the lowest hourly precipitation probability forecast for {city} today on Open-Meteo?",
+ "Using Open-Meteo, find today's lowest precipitation probability in {city}.",
+ ],
+}
+
+
+@register_template("openmeteo_hourly_extrema")
+class OpenMeteoHourlyExtremaTemplate(QuestionTemplate):
+ """
+ MEDIUM: Find the highest or lowest hourly metric value today.
+
+ Requires scanning hourly forecast data, not just reading a single value.
+ 170 cities x 4 metrics x 2 extrema = 1360 effective variants.
+ """
+
+ GT_SOURCE = GTSourceType.PAGE_ONLY
+
+ def __init__(self):
+ super().__init__("openmeteo_hourly_extrema")
+
+ def generate(self, seed: int, variant: Optional[int] = None) -> GeneratedQuestion:
+ rng = random.Random(seed)
+
+ metrics = list(HourlyMetric)
+ metric = metrics[variant % len(metrics)] if variant is not None else rng.choice(metrics)
+ is_max = rng.choice([True, False])
+
+ city = rng.choice(CITIES)
+ patterns = PATTERNS_HIGH[metric] if is_max else PATTERNS_LOW[metric]
+ question_text = rng.choice(patterns).format(city=city.display_name)
+
+ return GeneratedQuestion(
+ question_text=question_text,
+ start_url=DOCS_HOME_URL,
+ variables={"city": city.name, "is_max": is_max, "metric": metric.name},
+ validation_info={
+ "city_name": city.name,
+ "coord_key": city.coord_key,
+ "is_max": is_max,
+ "metric_field": metric.api_field,
+ "metric_label": metric.display_name,
+ "unit": metric.unit,
+ },
+ template_name=self.name,
+ expected_steps=7,
+ )
+
+ def get_validation_rules(self, validation_info: Dict[str, Any]) -> str:
+ city = validation_info.get("city_name", "")
+ is_max = validation_info.get("is_max", True)
+ label = validation_info.get("metric_label", "hourly temperature")
+ unit = validation_info.get("unit", "°C")
+ extrema = "maximum (highest)" if is_max else "minimum (lowest)"
+ return f"""Task-Specific Rules (Open Meteo Hourly Extrema):
+- City: {city}
+- Looking for: {extrema} {label} today
+- Score 1.0: Value within ±1{unit} of correct answer
+- Score 0.5: Value within ±3{unit}
+- Score 0.0: Wrong value or no answer
+- Use the hourly forecast for today's local date, not the daily summary"""
+
+ async def get_ground_truth(self, validation_info: Dict[str, Any]) -> GroundTruthResult:
+ coord_key = validation_info.get("coord_key", "")
+ is_max = validation_info.get("is_max", True)
+ city_name = validation_info.get("city_name", "")
+ metric_field = validation_info.get("metric_field", "temperature_2m")
+ unit = validation_info.get("unit", "°C")
+
+ data, failure = get_collected_location_data(coord_key, city_name)
+ if failure is not None:
+ return failure
+
+ values, val_failure = get_today_hourly_series(data, metric_field)
+ if val_failure is not None:
+ return val_failure
+
+ value = max(values) if is_max else min(values)
+ return GroundTruthResult.ok(f"{value}{unit}")
+
+ async def validate_answer(
+ self, answer: str, validation_info: Dict[str, Any]
+ ) -> ValidationResult:
+ """Not used — the pipeline uses LLM-based validation via get_validation_rules()."""
+ return ValidationResult(
+ score=0.0, is_correct=False, expected=None, actual=answer,
+ details="Use LLM validation",
+ )
+
+ def get_ground_truth_trigger(self, validation_info: dict) -> TriggerConfig:
+ trigger = UrlPatternTrigger(domains=["open-meteo.com"])
+ return TriggerConfig(trigger=trigger)
+
+ @classmethod
+ def get_cache_source(cls) -> str:
+ return "openmeteo"
+
+ def get_gt_source(self) -> GTSourceType:
+ return self.GT_SOURCE
diff --git a/liveweb_arena/plugins/openmeteo/templates/variables.py b/liveweb_arena/plugins/openmeteo/templates/variables.py
new file mode 100644
index 0000000..aa39e05
--- /dev/null
+++ b/liveweb_arena/plugins/openmeteo/templates/variables.py
@@ -0,0 +1,289 @@
+"""Location pool and metric definitions for Open Meteo templates."""
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Dict, List, Tuple
+
+
+@dataclass(frozen=True)
+class City:
+ """A city with coordinates for Open Meteo API."""
+ name: str
+ country: str
+ latitude: float
+ longitude: float
+
+ @property
+ def display_name(self) -> str:
+ return self.name
+
+ @property
+ def coord_key(self) -> str:
+ """Stable key for GT collector: rounded to 2 decimals."""
+ return f"{self.latitude:.2f},{self.longitude:.2f}"
+
+ def docs_url(self) -> str:
+ """URL for the Open Meteo docs page with this city's coords.
+
+ Uses query params for cache key uniqueness (normalize_url strips
+ hash fragments) and hash fragment for client-side JS form state.
+ """
+ return (
+ f"https://open-meteo.com/en/docs"
+ f"?latitude={self.latitude}&longitude={self.longitude}"
+ f"#latitude={self.latitude}&longitude={self.longitude}"
+ f"¤t=temperature_2m,wind_speed_10m,relative_humidity_2m"
+ f"&hourly=temperature_2m,precipitation_probability,wind_speed_10m"
+ f"&daily=temperature_2m_max,temperature_2m_min,precipitation_probability_max,sunrise,sunset"
+ )
+
+
+# Geographic spread: 170 cities across 6 regions
+CITIES: List[City] = [
+ # ── Asia (original 10) ──
+ City("Tokyo", "Japan", 35.6762, 139.6503),
+ City("Beijing", "China", 39.9042, 116.4074),
+ City("Seoul", "South Korea", 37.5665, 126.9780),
+ City("Mumbai", "India", 19.0760, 72.8777),
+ City("Singapore", "Singapore", 1.3521, 103.8198),
+ City("Bangkok", "Thailand", 13.7563, 100.5018),
+ City("Hong Kong", "China", 22.3193, 114.1694),
+ City("Shanghai", "China", 31.2304, 121.4737),
+ City("Delhi", "India", 28.7041, 77.1025),
+ City("Jakarta", "Indonesia", -6.2088, 106.8456),
+ # ── Europe (original 10) ──
+ City("London", "UK", 51.5074, -0.1278),
+ City("Paris", "France", 48.8566, 2.3522),
+ City("Berlin", "Germany", 52.5200, 13.4050),
+ City("Madrid", "Spain", 40.4168, -3.7038),
+ City("Rome", "Italy", 41.9028, 12.4964),
+ City("Stockholm", "Sweden", 59.3293, 18.0686),
+ City("Amsterdam", "Netherlands", 52.3676, 4.9041),
+ City("Prague", "Czech Republic", 50.0755, 14.4378),
+ City("Athens", "Greece", 37.9838, 23.7275),
+ City("Helsinki", "Finland", 60.1699, 24.9384),
+ # ── Americas (original 10) ──
+ City("New York", "USA", 40.7128, -74.0060),
+ City("Los Angeles", "USA", 34.0522, -118.2437),
+ City("Chicago", "USA", 41.8781, -87.6298),
+ City("Toronto", "Canada", 43.6532, -79.3832),
+ City("Mexico City", "Mexico", 19.4326, -99.1332),
+ City("Buenos Aires", "Argentina", -34.6037, -58.3816),
+ City("Miami", "USA", 25.7617, -80.1918),
+ City("Seattle", "USA", 47.6062, -122.3321),
+ City("Vancouver", "Canada", 49.2827, -123.1207),
+ City("Houston", "USA", 29.7604, -95.3698),
+ # ── Oceania (original 4) ──
+ City("Sydney", "Australia", -33.8688, 151.2093),
+ City("Melbourne", "Australia", -37.8136, 144.9631),
+ City("Auckland", "New Zealand", -36.8485, 174.7633),
+ City("Brisbane", "Australia", -27.4698, 153.0251),
+ # ── Africa / Middle East (original 6) ──
+ City("Dubai", "UAE", 25.2048, 55.2708),
+ City("Cairo", "Egypt", 30.0444, 31.2357),
+ City("Johannesburg", "South Africa", -26.2041, 28.0473),
+ City("Istanbul", "Turkey", 41.0082, 28.9784),
+ City("Lagos", "Nigeria", 6.5244, 3.3792),
+ City("Nairobi", "Kenya", -1.2921, 36.8219),
+ # ── Asia (add 15) ──
+ City("Taipei", "Taiwan", 25.03, 121.57),
+ City("Osaka", "Japan", 34.69, 135.50),
+ City("Hanoi", "Vietnam", 21.03, 105.85),
+ City("Kuala Lumpur", "Malaysia", 3.14, 101.69),
+ City("Kolkata", "India", 22.57, 88.36),
+ City("Dhaka", "Bangladesh", 23.81, 90.41),
+ City("Karachi", "Pakistan", 24.86, 67.01),
+ City("Manila", "Philippines", 14.60, 120.98),
+ City("Chengdu", "China", 30.57, 104.07),
+ City("Riyadh", "Saudi Arabia", 24.71, 46.68),
+ City("Tel Aviv", "Israel", 32.08, 34.78),
+ City("Doha", "Qatar", 25.29, 51.53),
+ City("Ho Chi Minh City", "Vietnam", 10.82, 106.63),
+ City("Ulaanbaatar", "Mongolia", 47.92, 106.91),
+ City("Almaty", "Kazakhstan", 43.24, 76.95),
+ # ── Europe (add 15) ──
+ City("Vienna", "Austria", 48.21, 16.37),
+ City("Warsaw", "Poland", 52.23, 21.01),
+ City("Zurich", "Switzerland", 47.38, 8.54),
+ City("Dublin", "Ireland", 53.35, -6.26),
+ City("Lisbon", "Portugal", 38.72, -9.14),
+ City("Copenhagen", "Denmark", 55.68, 12.57),
+ City("Brussels", "Belgium", 50.85, 4.35),
+ City("Bucharest", "Romania", 44.43, 26.10),
+ City("Oslo", "Norway", 59.91, 10.75),
+ City("Budapest", "Hungary", 47.50, 19.04),
+ City("Edinburgh", "UK", 55.95, -3.19),
+ City("Milan", "Italy", 45.46, 9.19),
+ City("Barcelona", "Spain", 41.39, 2.17),
+ City("Munich", "Germany", 48.14, 11.58),
+ City("Reykjavik", "Iceland", 64.15, -21.94),
+ # ── Americas (add 15) ──
+ City("Denver", "USA", 39.74, -104.99),
+ City("Boston", "USA", 42.36, -71.06),
+ City("Atlanta", "USA", 33.75, -84.39),
+ City("Phoenix", "USA", 33.45, -112.07),
+ City("San Francisco", "USA", 37.77, -122.42),
+ City("Lima", "Peru", -12.05, -77.04),
+ City("Santiago", "Chile", -33.45, -70.67),
+ City("Bogota", "Colombia", 4.71, -74.07),
+ City("Montreal", "Canada", 45.50, -73.57),
+ City("Havana", "Cuba", 23.11, -82.37),
+ City("Anchorage", "USA", 61.22, -149.90),
+ City("Honolulu", "USA", 21.31, -157.86),
+ City("Portland", "USA", 45.52, -122.68),
+ City("Minneapolis", "USA", 44.98, -93.27),
+ City("Nashville", "USA", 36.16, -86.78),
+ # ── Africa / Middle East (add 10) ──
+ City("Addis Ababa", "Ethiopia", 9.02, 38.75),
+ City("Casablanca", "Morocco", 33.57, -7.59),
+ City("Accra", "Ghana", 5.60, -0.19),
+ City("Dar es Salaam", "Tanzania", -6.79, 39.28),
+ City("Algiers", "Algeria", 36.74, 3.06),
+ City("Tunis", "Tunisia", 36.81, 10.18),
+ City("Kigali", "Rwanda", -1.94, 30.06),
+ City("Lusaka", "Zambia", -15.39, 28.32),
+ City("Maputo", "Mozambique", -25.97, 32.57),
+ City("Kampala", "Uganda", 0.35, 32.58),
+ # ── Oceania (add 5) ──
+ City("Perth", "Australia", -31.95, 115.86),
+ City("Wellington", "New Zealand", -41.29, 174.78),
+ City("Christchurch", "New Zealand", -43.53, 172.64),
+ City("Adelaide", "Australia", -34.93, 138.60),
+ City("Suva", "Fiji", -18.14, 178.44),
+ # ── Extreme climates (add 5) ──
+ City("Novosibirsk", "Russia", 55.03, 82.92),
+ City("Murmansk", "Russia", 68.97, 33.09),
+ City("Yakutsk", "Russia", 62.03, 129.73),
+ City("Manaus", "Brazil", -3.12, -60.02),
+ City("Lhasa", "China", 29.65, 91.17),
+ # ── More Americas (add 10) ──
+ City("Washington DC", "USA", 38.91, -77.04),
+ City("Dallas", "USA", 32.78, -96.80),
+ City("Philadelphia", "USA", 39.95, -75.17),
+ City("Las Vegas", "USA", 36.17, -115.14),
+ City("San Diego", "USA", 32.72, -117.16),
+ City("Ottawa", "Canada", 45.42, -75.70),
+ City("Guadalajara", "Mexico", 20.67, -103.35),
+ City("Medellin", "Colombia", 6.25, -75.56),
+ City("Montevideo", "Uruguay", -34.91, -56.19),
+ City("Quito", "Ecuador", -0.18, -78.47),
+ # ── More Europe (add 10) ──
+ City("Marseille", "France", 43.30, 5.37),
+ City("Hamburg", "Germany", 53.55, 10.00),
+ City("Lyon", "France", 45.76, 4.84),
+ City("Krakow", "Poland", 50.06, 19.94),
+ City("Sofia", "Bulgaria", 42.70, 23.32),
+ City("Belgrade", "Serbia", 44.79, 20.47),
+ City("Riga", "Latvia", 56.95, 24.11),
+ City("Vilnius", "Lithuania", 54.69, 25.28),
+ City("Tallinn", "Estonia", 59.44, 24.75),
+ City("Porto", "Portugal", 41.16, -8.63),
+ # ── More Asia (add 10) ──
+ City("Shenzhen", "China", 22.54, 114.06),
+ City("Nagoya", "Japan", 35.18, 136.91),
+ City("Lahore", "Pakistan", 31.55, 74.35),
+ City("Jeddah", "Saudi Arabia", 21.49, 39.19),
+ City("Baku", "Azerbaijan", 40.41, 49.87),
+ City("Tashkent", "Uzbekistan", 41.30, 69.28),
+ City("Yangon", "Myanmar", 16.87, 96.20),
+ City("Phnom Penh", "Cambodia", 11.56, 104.92),
+ City("Colombo", "Sri Lanka", 6.93, 79.84),
+ City("Kathmandu", "Nepal", 27.72, 85.32),
+ # ── More Africa (add 10) ──
+ City("Kinshasa", "DRC", -4.44, 15.27),
+ City("Luanda", "Angola", -8.84, 13.23),
+ City("Dakar", "Senegal", 14.69, -17.44),
+ City("Abidjan", "Ivory Coast", 5.36, -4.01),
+ City("Harare", "Zimbabwe", -17.83, 31.05),
+ City("Rabat", "Morocco", 34.02, -6.84),
+ City("Windhoek", "Namibia", -22.56, 17.08),
+ City("Antananarivo", "Madagascar", -18.91, 47.52),
+ City("Douala", "Cameroon", 4.05, 9.77),
+ City("Bamako", "Mali", 12.65, -8.00),
+ # ── More Oceania / Pacific (add 5) ──
+ City("Darwin", "Australia", -12.46, 130.84),
+ City("Nadi", "Fiji", -17.78, 177.94),
+ City("Port Moresby", "Papua New Guinea", -6.21, 147.00),
+ City("Noumea", "New Caledonia", -22.28, 166.46),
+ City("Apia", "Samoa", -13.83, -171.76),
+ # ── More extreme / misc (add 10) ──
+ City("Tromsoe", "Norway", 69.65, 18.96),
+ City("Fairbanks", "USA", 64.84, -147.72),
+ City("Ushuaia", "Argentina", -54.80, -68.30),
+ City("Barranquilla", "Colombia", 10.96, -74.78),
+ City("Marrakech", "Morocco", 31.63, -8.01),
+ City("Sapporo", "Japan", 43.06, 141.35),
+ City("Vladivostok", "Russia", 43.12, 131.87),
+ City("Irkutsk", "Russia", 52.29, 104.28),
+ City("Astana", "Kazakhstan", 51.17, 71.43),
+ City("Tbilisi", "Georgia", 41.69, 44.80),
+ # ── Additional cities to reach 170 ──
+ City("Pune", "India", 18.52, 73.86),
+ City("Brasilia", "Brazil", -15.79, -47.88),
+ City("Thessaloniki", "Greece", 40.64, 22.94),
+ City("Naypyidaw", "Myanmar", 19.76, 96.07),
+ City("Busan", "South Korea", 35.18, 129.08),
+ City("Cusco", "Peru", -13.53, -71.97),
+ City("Zanzibar", "Tanzania", -6.16, 39.19),
+ City("Reims", "France", 49.25, 4.03),
+ City("Split", "Croatia", 43.51, 16.44),
+ City("Bergen", "Norway", 60.39, 5.32),
+]
+
+
+class CurrentMetric(Enum):
+ """Metrics available from current_weather endpoint."""
+ TEMPERATURE = ("temperature", "current temperature", "°C")
+ WIND_SPEED = ("windspeed", "current wind speed", "km/h")
+ WIND_DIRECTION = ("winddirection", "current wind direction", "°")
+
+ @property
+ def api_field(self) -> str:
+ return self.value[0]
+
+ @property
+ def display_name(self) -> str:
+ return self.value[1]
+
+ @property
+ def unit(self) -> str:
+ return self.value[2]
+
+
+class HourlyMetric(Enum):
+ """Metrics available from hourly forecast data."""
+ TEMPERATURE = ("temperature_2m", "hourly temperature", "°C")
+ HUMIDITY = ("relative_humidity_2m", "hourly relative humidity", "%")
+ WIND_SPEED = ("wind_speed_10m", "hourly wind speed", "km/h")
+ PRECIP_PROBABILITY = ("precipitation_probability", "hourly precipitation probability", "%")
+
+ @property
+ def api_field(self) -> str:
+ return self.value[0]
+
+ @property
+ def display_name(self) -> str:
+ return self.value[1]
+
+ @property
+ def unit(self) -> str:
+ return self.value[2]
+
+
+class DailyMetric(Enum):
+ """Metrics available from daily forecast data."""
+ TEMP_MAX = ("temperature_2m_max", "daily maximum temperature", "°C")
+ TEMP_MIN = ("temperature_2m_min", "daily minimum temperature", "°C")
+ PRECIP_PROB_MAX = ("precipitation_probability_max", "daily max precipitation probability", "%")
+
+ @property
+ def api_field(self) -> str:
+ return self.value[0]
+
+ @property
+ def display_name(self) -> str:
+ return self.value[1]
+
+ @property
+ def unit(self) -> str:
+ return self.value[2]
diff --git a/liveweb_arena/plugins/stooq/api_client.py b/liveweb_arena/plugins/stooq/api_client.py
index 7a3b82d..55ceda4 100644
--- a/liveweb_arena/plugins/stooq/api_client.py
+++ b/liveweb_arena/plugins/stooq/api_client.py
@@ -11,7 +11,7 @@
import aiohttp
-from liveweb_arena.plugins.base_client import RateLimiter
+from liveweb_arena.plugins.base_client import APIFetchError, RateLimiter
logger = logging.getLogger(__name__)
@@ -34,6 +34,19 @@
_negative_cache: contextvars.ContextVar[Optional[set]] = contextvars.ContextVar(
"_stooq_negative_cache", default=None
)
+_last_failure_metadata: contextvars.ContextVar[Optional[dict]] = contextvars.ContextVar(
+ "_stooq_last_failure_metadata", default=None
+)
+_STOOQ_HTTP_TIMEOUT_S = float(os.environ.get("LIVEWEB_STOOQ_HTTP_TIMEOUT_S", "45"))
+
+
+def _get_plugin_cache_root() -> Path:
+ return Path(
+ os.environ.get(
+ "LIVEWEB_SHARED_PLUGIN_CACHE_DIR",
+ str(Path(__file__).resolve().parents[3] / ".cache" / "plugin-cache"),
+ )
+ )
def _get_negative_cache() -> set:
@@ -44,11 +57,46 @@ def _get_negative_cache() -> set:
return cache
+def _set_last_failure_metadata(metadata: dict[str, Any] | None) -> None:
+ _last_failure_metadata.set(dict(metadata or {}) if metadata else None)
+
+
+def get_last_failure_metadata() -> dict[str, Any]:
+ return dict(_last_failure_metadata.get() or {})
+
+
class StooqRateLimitError(Exception):
"""Raised when Stooq API rate limit is exceeded."""
pass
+def _create_stooq_http_session(
+ *,
+ headers: Optional[Dict[str, str]] = None,
+ timeout: Optional[float] = None,
+) -> aiohttp.ClientSession:
+ """
+ Create a direct session for Stooq.
+
+ Stooq is materially less reliable through the current machine proxy path, so
+ these requests bypass proxy env and allow a longer total timeout for large CSV
+ responses.
+ """
+ total_timeout = timeout if timeout is not None else _STOOQ_HTTP_TIMEOUT_S
+ timeout_cfg = aiohttp.ClientTimeout(
+ total=total_timeout,
+ sock_connect=min(15, total_timeout),
+ sock_read=total_timeout,
+ )
+ connector = aiohttp.TCPConnector(ttl_dns_cache=300, limit=8, ssl=False)
+ return aiohttp.ClientSession(
+ headers=headers,
+ timeout=timeout_cfg,
+ trust_env=False,
+ connector=connector,
+ )
+
+
def _parse_stooq_csv(csv_text: str, symbol: str = "") -> Optional[Dict[str, Any]]:
"""
Parse Stooq CSV response into price data dict.
@@ -174,12 +222,11 @@ async def get_price_data(
await _global_csv_limiter.wait()
try:
- async with aiohttp.ClientSession() as session:
+ async with _create_stooq_http_session(timeout=max(timeout, _STOOQ_HTTP_TIMEOUT_S), headers={"User-Agent": "Mozilla/5.0"}) as session:
params = {"s": symbol, "i": "d"}
async with session.get(
cls.CSV_URL,
params=params,
- timeout=aiohttp.ClientTimeout(total=timeout),
) as response:
if response.status != 200:
logger.warning(f"Stooq error for {symbol}: {response.status}")
@@ -247,16 +294,13 @@ async def fetch_cache_api_data() -> Optional[Dict[str, Any]]:
failed = 0
# Sequential fetch with global rate limiter — avoid IP bans
- async with aiohttp.ClientSession(
- headers={"User-Agent": "Mozilla/5.0"},
- ) as session:
+ async with _create_stooq_http_session(timeout=_STOOQ_HTTP_TIMEOUT_S, headers={"User-Agent": "Mozilla/5.0"}) as session:
for symbol in assets:
await _global_csv_limiter.wait()
try:
url = f"https://stooq.com/q/d/l/?s={symbol}&i=d"
async with session.get(
url,
- timeout=aiohttp.ClientTimeout(total=15),
) as response:
if response.status != 200:
failed += 1
@@ -282,8 +326,12 @@ async def fetch_cache_api_data() -> Optional[Dict[str, Any]]:
def _get_file_cache_path() -> Path:
"""Get path for stooq homepage file cache."""
- cache_dir = os.environ.get("LIVEWEB_CACHE_DIR", "/var/lib/liveweb-arena/cache")
- return Path(cache_dir) / "_plugin_init" / "stooq_homepage.json"
+ return _get_plugin_cache_root() / "stooq" / "homepage.json"
+
+
+def _get_symbol_cache_path(symbol: str) -> Path:
+ safe_symbol = symbol.replace("/", "_").replace("\\", "_")
+ return _get_plugin_cache_root() / "stooq" / "symbols" / f"{safe_symbol}.json"
def _get_cache_ttl() -> int:
@@ -292,6 +340,33 @@ def _get_cache_ttl() -> int:
return int(os.environ.get("LIVEWEB_CACHE_TTL", str(DEFAULT_TTL)))
+def _load_symbol_cache(symbol: str, *, allow_stale: bool = False) -> Optional[Dict[str, Any]]:
+ cache_file = _get_symbol_cache_path(symbol)
+ if not cache_file.exists():
+ return None
+ try:
+ cached = json.loads(cache_file.read_text())
+ except Exception:
+ return None
+ fetched_at = float(cached.get("_fetched_at", 0))
+ payload = cached.get("data")
+ if not payload:
+ return None
+ age = time.time() - fetched_at
+ ttl = _get_cache_ttl()
+ if age <= ttl:
+ return payload
+ if allow_stale and age <= ttl + 24 * 3600:
+ return payload
+ return None
+
+
+def _save_symbol_cache(symbol: str, data: Dict[str, Any]) -> None:
+ cache_file = _get_symbol_cache_path(symbol)
+ cache_file.parent.mkdir(parents=True, exist_ok=True)
+ cache_file.write_text(json.dumps({"data": data, "_fetched_at": time.time()}))
+
+
def _is_file_cache_valid() -> bool:
"""Check if homepage file cache exists and is within TTL."""
cache_file = _get_file_cache_path()
@@ -321,12 +396,23 @@ def initialize_cache():
logger.info("Stooq init: homepage cache valid (quick check)")
return
- # Acquire file lock — only one process fetches, others wait
+ strict_eval_mode = os.environ.get("LIVEWEB_RUNTIME_PROFILE", "").strip().lower() == "strict_eval"
+
+ # Acquire file lock — only one process fetches, others wait.
+ # In strict-eval, this warmup is only an optimization; if another worker is
+ # already warming the cache, skip instead of blocking the whole episode.
lock_path = _get_file_cache_path().with_suffix(".lock")
lock_path.parent.mkdir(parents=True, exist_ok=True)
fd = open(lock_path, "w")
try:
- fcntl.flock(fd.fileno(), fcntl.LOCK_EX) # Blocking wait
+ lock_flags = fcntl.LOCK_EX | (fcntl.LOCK_NB if strict_eval_mode else 0)
+ try:
+ fcntl.flock(fd.fileno(), lock_flags)
+ except BlockingIOError:
+ if strict_eval_mode:
+ logger.info("Stooq init: lock busy in strict-eval, skipping warmup")
+ return
+ raise
# Re-check after acquiring lock — another process may have filled cache
if _is_file_cache_valid():
@@ -390,7 +476,7 @@ async def fetch_homepage_api_data() -> Dict[str, Any]:
return {"assets": assets}
-async def fetch_single_asset_data(symbol: str) -> Optional[Dict[str, Any]]:
+async def fetch_single_asset_data(symbol: str) -> Dict[str, Any]:
"""
Fetch price data for a single asset.
@@ -398,48 +484,164 @@ async def fetch_single_asset_data(symbol: str) -> Optional[Dict[str, Any]]:
since Stooq's CSV API requires suffixed symbols for some markets.
Uses negative cache to avoid repeated requests for symbols with no data.
"""
- if _rate_limited.get():
- raise StooqRateLimitError("Stooq API rate limited (persistent for this session)")
+ _set_last_failure_metadata(None)
- neg = _get_negative_cache()
- if symbol in neg:
- return {}
-
- # Try .us suffix first for bare symbols (canonical form for US stocks)
variants = [symbol]
if "." not in symbol and not symbol.startswith("^"):
variants = [f"{symbol}.us", symbol]
+ for sym in variants:
+ cached = _load_symbol_cache(sym)
+ if cached:
+ _set_last_failure_metadata(None)
+ return cached
+
+ if _rate_limited.get():
+ for sym in variants:
+ cached = _load_symbol_cache(sym, allow_stale=True)
+ if cached:
+ _set_last_failure_metadata(None)
+ return cached
+ metadata = {
+ "plugin": "stooq",
+ "failure_stage": "api_fetch",
+ "failure_type": "rate_limit",
+ "symbol": symbol,
+ "request_url": None,
+ }
+ _set_last_failure_metadata(metadata)
+ raise APIFetchError(
+ "Stooq API rate limited (persistent for this session)",
+ source="stooq",
+ metadata=metadata,
+ )
+
+ neg = _get_negative_cache()
+ if symbol in neg:
+ for sym in variants:
+ cached = _load_symbol_cache(sym)
+ if cached:
+ _set_last_failure_metadata(None)
+ return cached
+ metadata = {
+ "plugin": "stooq",
+ "failure_stage": "api_fetch",
+ "failure_type": "negative_cache",
+ "symbol": symbol,
+ "request_url": None,
+ }
+ _set_last_failure_metadata(metadata)
+ raise APIFetchError(
+ f"Stooq API negative cache hit for symbol={symbol}",
+ source="stooq",
+ metadata=metadata,
+ )
+
+ last_metadata: dict[str, Any] | None = None
for sym in variants:
await _global_csv_limiter.wait()
+ url = f"https://stooq.com/q/d/l/?s={sym}&i=d"
try:
- async with aiohttp.ClientSession() as session:
- url = f"https://stooq.com/q/d/l/?s={sym}&i=d"
+ async with _create_stooq_http_session(timeout=_STOOQ_HTTP_TIMEOUT_S, headers={"User-Agent": "Mozilla/5.0"}) as session:
async with session.get(
url,
- timeout=aiohttp.ClientTimeout(total=15),
- headers={"User-Agent": "Mozilla/5.0"},
) as response:
if response.status != 200:
+ last_metadata = {
+ "plugin": "stooq",
+ "failure_stage": "api_fetch",
+ "failure_type": "http_status",
+ "symbol": sym,
+ "request_url": url,
+ "http_status": response.status,
+ }
continue
text = await response.text()
if "Exceeded the daily hits limit" in text:
_rate_limited.set(True)
- raise StooqRateLimitError("Stooq API daily limit exceeded")
+ cached = _load_symbol_cache(sym, allow_stale=True)
+ if cached:
+ _set_last_failure_metadata(None)
+ return cached
+ metadata = {
+ "plugin": "stooq",
+ "failure_stage": "api_fetch",
+ "failure_type": "rate_limit",
+ "symbol": sym,
+ "request_url": url,
+ }
+ _set_last_failure_metadata(metadata)
+ raise APIFetchError(
+ "Stooq API daily limit exceeded",
+ source="stooq",
+ metadata=metadata,
+ )
if "No data" in text:
+ last_metadata = {
+ "plugin": "stooq",
+ "failure_stage": "api_fetch",
+ "failure_type": "no_data",
+ "symbol": sym,
+ "request_url": url,
+ }
continue
result = _parse_stooq_csv(text, sym)
if result:
+ with contextlib.suppress(Exception):
+ _save_symbol_cache(sym, result)
+ _set_last_failure_metadata(None)
return result
-
- except StooqRateLimitError:
+ last_metadata = {
+ "plugin": "stooq",
+ "failure_stage": "csv_parse",
+ "failure_type": "invalid_csv",
+ "symbol": sym,
+ "request_url": url,
+ }
+
+ except APIFetchError:
raise
+ except asyncio.TimeoutError:
+ last_metadata = {
+ "plugin": "stooq",
+ "failure_stage": "api_fetch",
+ "failure_type": "timeout",
+ "symbol": sym,
+ "request_url": url,
+ }
+ cached = _load_symbol_cache(sym, allow_stale=True)
+ if cached:
+ _set_last_failure_metadata(None)
+ return cached
except Exception:
+ last_metadata = {
+ "plugin": "stooq",
+ "failure_stage": "api_fetch",
+ "failure_type": "request_error",
+ "symbol": sym,
+ "request_url": url,
+ }
+ cached = _load_symbol_cache(sym, allow_stale=True)
+ if cached:
+ _set_last_failure_metadata(None)
+ return cached
continue
# All variants failed — add to negative cache
neg.add(symbol)
- return {}
+ metadata = last_metadata or {
+ "plugin": "stooq",
+ "failure_stage": "api_fetch",
+ "failure_type": "no_data",
+ "symbol": symbol,
+ "request_url": None,
+ }
+ _set_last_failure_metadata(metadata)
+ raise APIFetchError(
+ f"Stooq API returned no data for symbol={symbol}",
+ source="stooq",
+ metadata=metadata,
+ )
diff --git a/liveweb_arena/plugins/stooq/stooq.py b/liveweb_arena/plugins/stooq/stooq.py
index 1668c9c..a750a87 100644
--- a/liveweb_arena/plugins/stooq/stooq.py
+++ b/liveweb_arena/plugins/stooq/stooq.py
@@ -4,11 +4,17 @@
Plugin for financial market data from stooq.com.
"""
+import asyncio
+import os
+import threading
+from pathlib import Path
from typing import Any, Dict, List, Optional
from urllib.parse import urlparse, parse_qs
from liveweb_arena.plugins.base import BasePlugin
-from .api_client import fetch_single_asset_data, fetch_homepage_api_data, initialize_cache
+from liveweb_arena.plugins.base_client import APIFetchError
+from liveweb_arena.utils.logger import log
+from .api_client import fetch_single_asset_data, fetch_homepage_api_data, get_last_failure_metadata, initialize_cache
class StooqPlugin(BasePlugin):
@@ -27,6 +33,8 @@ class StooqPlugin(BasePlugin):
name = "stooq"
_known_symbols_cache = None
+ _quote_warmup_started = False
+ _quote_warmup_lock = threading.Lock()
allowed_domains = [
"stooq.com",
@@ -34,8 +42,73 @@ class StooqPlugin(BasePlugin):
]
def initialize(self):
- """Pre-warm homepage file cache before evaluation starts."""
+ """Pre-warm homepage file cache and a few common quote pages."""
initialize_cache()
+ self._initialize_quote_warmup()
+
+ def _initialize_quote_warmup(self) -> None:
+ if os.environ.get("LIVEWEB_DISABLE_STOOQ_WARMUP", "").lower() in {"1", "true"}:
+ return
+
+ with StooqPlugin._quote_warmup_lock:
+ if StooqPlugin._quote_warmup_started:
+ return
+ StooqPlugin._quote_warmup_started = True
+
+ cache_dir = Path(
+ os.environ.get(
+ "LIVEWEB_CACHE_DIR",
+ str(Path(__file__).resolve().parents[3] / ".cache" / "liveweb"),
+ )
+ )
+ symbols_raw = os.environ.get("LIVEWEB_STOOQ_WARMUP_SYMBOLS", "jnj.us,aapl.us,^spx")
+ urls = [
+ f"https://stooq.com/q/?s={symbol.strip()}"
+ for symbol in symbols_raw.split(",")
+ if symbol.strip()
+ ]
+ if not urls:
+ return
+
+ async def _warm() -> None:
+ from liveweb_arena.core.cache import CacheManager, PageRequirement
+
+ mgr = CacheManager(cache_dir=cache_dir)
+ pending = []
+ try:
+ for url in urls:
+ cached = mgr.get_cached(url)
+ if cached and not cached.is_expired(mgr.ttl) and cached.is_complete():
+ continue
+ pending.append(PageRequirement.data(url))
+ if pending:
+ try:
+ await mgr.ensure_cached(pending, self)
+ except Exception as exc:
+ log("Stooq", f"Quote warmup skipped after fetch error: {exc}", force=True)
+ finally:
+ await mgr.shutdown()
+
+ try:
+ loop = asyncio.get_running_loop()
+ except RuntimeError:
+ try:
+ asyncio.run(_warm())
+ except Exception as exc:
+ log("Stooq", f"Quote warmup skipped after init error: {exc}", force=True)
+ return
+
+ def _run_background() -> None:
+ try:
+ asyncio.run(_warm())
+ except Exception as exc:
+ log("Stooq", f"Quote warmup background task failed: {exc}", force=True)
+
+ threading.Thread(
+ target=_run_background,
+ name="stooq-quote-warmup",
+ daemon=True,
+ ).start()
def get_blocked_patterns(self) -> List[str]:
"""Block direct CSV download and ads."""
@@ -44,6 +117,41 @@ def get_blocked_patterns(self) -> List[str]:
"*stooq.com/ads/*", # Ad frames
]
+ def get_stable_url_patterns(self) -> List[str]:
+ return [
+ "/",
+ "/q/",
+ "/q/?s=",
+ "/q/i/?s=",
+ "/q/d/?s=",
+ ]
+
+ def classify_url(self, url: str) -> Optional[str]:
+ parsed = urlparse(url)
+ host = (parsed.hostname or "").lower()
+ path = parsed.path.lower()
+ query = parsed.query.lower()
+ if "stooq.com" not in host:
+ return None
+ if host.startswith("www."):
+ return "env_tls_error"
+ if (
+ "/q/conv/" in path
+ or "/s/mst/" in path
+ or "quote.php" in path
+ or "/q/plus/" in path
+ or path.startswith("/q/l/")
+ or path.startswith("/q/d/l/")
+ or path.startswith("/q/nl/")
+ or path.startswith("/t/")
+ ):
+ return "model_invalid_url_shape"
+ if "q=" in query and "s=" not in query and "e=" not in query:
+ return "model_invalid_url_shape"
+ if path.startswith("/q/s/"):
+ return "model_invalid_url_shape"
+ return None
+
def get_synthetic_page(self, url: str) -> Optional[str]:
"""Return synthetic error page for unknown symbols (zero network requests)."""
symbol = self._extract_symbol(url)
@@ -102,10 +210,19 @@ async def fetch_api_data(self, url: str) -> Dict[str, Any]:
if symbol:
if symbol not in self._get_known_symbols():
return {} # Unknown symbol — skip API, zero requests
- data = await fetch_single_asset_data(symbol)
- if not data:
- raise ValueError(f"Stooq API returned no data for symbol={symbol}")
- return data
+ try:
+ return await fetch_single_asset_data(symbol)
+ except APIFetchError as exc:
+ metadata = dict(exc.metadata or {})
+ metadata.setdefault("plugin", self.name)
+ metadata.setdefault("symbol", symbol)
+ metadata.setdefault("page_url", url)
+ raise APIFetchError(
+ str(exc),
+ source=exc.source or self.name,
+ status_code=exc.status_code,
+ metadata=metadata or get_last_failure_metadata(),
+ ) from exc
# Homepage - return all assets
if self._is_homepage(url):
@@ -162,3 +279,9 @@ def _extract_symbol(self, url: str) -> str:
return query["e"][0].lower()
return ""
+
+ def is_plausible_asset_id(self, url: str) -> bool:
+ symbol = self._extract_symbol(url)
+ if not symbol:
+ return True
+ return symbol in self._get_known_symbols()
diff --git a/liveweb_arena/plugins/taostats/api_client.py b/liveweb_arena/plugins/taostats/api_client.py
index 79009ca..e0fc558 100644
--- a/liveweb_arena/plugins/taostats/api_client.py
+++ b/liveweb_arena/plugins/taostats/api_client.py
@@ -20,6 +20,7 @@
# Conversion factor: rao to TAO (1 TAO = 1e9 rao)
RAO_TO_TAO = 1e9
+MAX_API_RETRIES = 3
def _safe_float(value) -> Optional[float]:
@@ -117,34 +118,39 @@ async def fetch_all_subnets() -> Dict[str, Any]:
try:
async with aiohttp.ClientSession() as session:
- # Fetch all subnets (paginated, get up to 200)
- async with session.get(
- f"{API_BASE_URL}/subnets",
- params={"limit": 200},
- timeout=aiohttp.ClientTimeout(total=30),
- ) as resp:
- if resp.status != 200:
- body = await resp.text()
- raise APIFetchError(
- f"status={resp.status}, body={body[:500]}",
- source="taostats",
- status_code=resp.status,
- )
-
- data = await resp.json()
- results = data.get("results", [])
-
- for subnet in results:
- netuid = str(subnet.get("netuid", ""))
- if not netuid or netuid == "0": # Skip root network
- continue
-
- subnets[netuid] = _parse_subnet_data(subnet)
+ for attempt in range(MAX_API_RETRIES):
+ try:
+ async with session.get(
+ f"{API_BASE_URL}/subnets",
+ params={"limit": 200},
+ timeout=aiohttp.ClientTimeout(total=30),
+ ) as resp:
+ if resp.status != 200:
+ body = await resp.text()
+ raise APIFetchError(
+ f"status={resp.status}, body={body[:500]}",
+ source="taostats",
+ status_code=resp.status,
+ )
+
+ data = await resp.json()
+ results = data.get("results", [])
+ subnets.clear()
+ for subnet in results:
+ netuid = str(subnet.get("netuid", ""))
+ if not netuid or netuid == "0":
+ continue
+ subnets[netuid] = _parse_subnet_data(subnet)
+ break
+ except APIFetchError:
+ raise
+ except Exception as e:
+ if attempt == MAX_API_RETRIES - 1:
+ raise APIFetchError(f"Unexpected error: {e}", source="taostats") from e
+ await asyncio.sleep(2**attempt)
except APIFetchError:
raise
- except Exception as e:
- raise APIFetchError(f"Unexpected error: {e}", source="taostats") from e
if not subnets:
raise APIFetchError("API returned no subnet data", source="taostats")
@@ -167,25 +173,31 @@ async def fetch_single_subnet_data(subnet_id: str) -> Dict[str, Any]:
"""
try:
async with aiohttp.ClientSession() as session:
- async with session.get(
- f"{API_BASE_URL}/subnets/{subnet_id}",
- timeout=aiohttp.ClientTimeout(total=30),
- ) as resp:
- if resp.status != 200:
- body = await resp.text()
- raise APIFetchError(
- f"status={resp.status} for subnet_id={subnet_id}, body={body[:200]}",
- source="taostats",
- status_code=resp.status,
- )
-
- subnet = await resp.json()
- return _parse_subnet_data(subnet)
+ for attempt in range(MAX_API_RETRIES):
+ try:
+ async with session.get(
+ f"{API_BASE_URL}/subnets/{subnet_id}",
+ timeout=aiohttp.ClientTimeout(total=30),
+ ) as resp:
+ if resp.status != 200:
+ body = await resp.text()
+ raise APIFetchError(
+ f"status={resp.status} for subnet_id={subnet_id}, body={body[:200]}",
+ source="taostats",
+ status_code=resp.status,
+ )
+
+ subnet = await resp.json()
+ return _parse_subnet_data(subnet)
+ except APIFetchError:
+ raise
+ except Exception as e:
+ if attempt == MAX_API_RETRIES - 1:
+ raise APIFetchError(f"Failed to fetch subnet {subnet_id}: {e}", source="taostats") from e
+ await asyncio.sleep(2**attempt)
except APIFetchError:
raise
- except Exception as e:
- raise APIFetchError(f"Failed to fetch subnet {subnet_id}: {e}", source="taostats") from e
async def fetch_homepage_api_data() -> Dict[str, Any]:
@@ -266,7 +278,10 @@ def _filter_by_emission(subnets: Dict[str, Any]) -> Dict[str, Any]:
def _get_file_cache_path() -> Path:
"""Get path for taostats subnet file cache."""
- cache_dir = os.environ.get("LIVEWEB_CACHE_DIR", "/var/lib/liveweb-arena/cache")
+ cache_dir = os.environ.get(
+ "LIVEWEB_CACHE_DIR",
+ str(Path(__file__).resolve().parents[3] / ".cache" / "liveweb"),
+ )
return Path(cache_dir) / "_plugin_init" / "taostats_subnets.json"
diff --git a/liveweb_arena/plugins/taostats/taostats.py b/liveweb_arena/plugins/taostats/taostats.py
index 8e5dda7..c9ea1ef 100644
--- a/liveweb_arena/plugins/taostats/taostats.py
+++ b/liveweb_arena/plugins/taostats/taostats.py
@@ -5,14 +5,35 @@
Uses official taostats.io API for ground truth.
"""
+import time
import re
from typing import Any, Dict, List
from urllib.parse import urlparse
+from liveweb_arena.core.cache import normalize_url
from liveweb_arena.plugins.base import BasePlugin
from .api_client import fetch_single_subnet_data, fetch_homepage_api_data, initialize_cache
+class TaostatsPrefetchSetupError(RuntimeError):
+ def __init__(
+ self,
+ message: str,
+ *,
+ prefetch_phase: str,
+ wait_target: str | None = None,
+ evidence: dict[str, Any] | None = None,
+ ):
+ super().__init__(message)
+ self.prefetch_phase = prefetch_phase
+ self.wait_target = wait_target
+ self.evidence = dict(evidence or {})
+
+
+_LIST_SHOW_ALL_COOLDOWN_S = 180
+_list_show_all_cooldown_until: dict[str, float] = {}
+
+
class TaostatsPlugin(BasePlugin):
"""
Taostats plugin for Bittensor network data.
@@ -36,6 +57,27 @@ def initialize(self):
"""Initialize plugin - fetch API data for question generation."""
initialize_cache()
+ def get_stable_url_patterns(self) -> List[str]:
+ return [
+ "/",
+ "/subnets",
+ "/subnets/",
+ "/subnets//chart",
+ "/validators",
+ ]
+
+ def classify_url(self, url: str) -> str | None:
+ parsed = urlparse(url)
+ host = (parsed.hostname or "").lower()
+ path = parsed.path.lower()
+ if "taostats.io" not in host:
+ return None
+ if path in {"", "/", "/subnets", "/validators"}:
+ return None
+ if path.startswith("/subnets/"):
+ return None
+ return "model_invalid_url_shape"
+
def get_blocked_patterns(self) -> List[str]:
"""Block direct API access to force agents to use the website."""
return [
@@ -115,29 +157,226 @@ def _extract_subnet_id(self, url: str) -> str:
return ""
- async def setup_page_for_cache(self, page, url: str) -> None:
+ async def _wait_for_minimum_body_text(self, page, *, min_chars: int, timeout_ms: int) -> None:
+ await page.wait_for_function(
+ """minChars => {
+ const text = (document.body?.innerText || '').trim();
+ return text.length >= minChars;
+ }""",
+ min_chars,
+ timeout=timeout_ms,
+ )
+
+ async def _capture_body_snapshot(self, page) -> Dict[str, Any]:
+ snapshot = await page.evaluate(
+ """() => {
+ const text = (document.body?.innerText || '').trim();
+ const title = document.title || '';
+ const hasMain = !!document.querySelector('main');
+ const hasTable = !!document.querySelector('table, [role="table"], .table, .rt-table');
+ const textSample = text.slice(0, 2000);
+ return {
+ title,
+ textLength: text.length,
+ textSample,
+ hasMain,
+ hasTable,
+ };
+ }"""
+ )
+ return dict(snapshot or {})
+
+ def _is_list_body_ready(self, snapshot: Dict[str, Any]) -> bool:
+ sample = str(snapshot.get("textSample") or "").lower()
+ text_length = int(snapshot.get("textLength") or 0)
+ return bool(
+ text_length >= 120
+ and (
+ snapshot.get("hasTable")
+ or snapshot.get("hasMain")
+ or "subnet" in sample
+ or "subnets" in sample
+ or "netuid" in sample
+ )
+ )
+
+ def _is_detail_body_ready(self, snapshot: Dict[str, Any]) -> bool:
+ sample = str(snapshot.get("textSample") or "").lower()
+ text_length = int(snapshot.get("textLength") or 0)
+ return bool(
+ text_length >= 120
+ and (
+ snapshot.get("hasMain")
+ or "statistics" in sample
+ or "subnet" in sample
+ or "netuid" in sample
+ or "emission" in sample
+ or "tempo" in sample
+ )
+ )
+
+ def _list_show_all_cooldown_active(self, url: str) -> bool:
+ key = normalize_url(url)
+ until = _list_show_all_cooldown_until.get(key)
+ if until is None:
+ return False
+ if until > time.monotonic():
+ return True
+ _list_show_all_cooldown_until.pop(key, None)
+ return False
+
+ def _activate_list_show_all_cooldown(self, url: str) -> None:
+ _list_show_all_cooldown_until[normalize_url(url)] = time.monotonic() + _LIST_SHOW_ALL_COOLDOWN_S
+
+ async def _best_effort_expand_list(self, page, url: str) -> Dict[str, Any]:
+ if self._list_show_all_cooldown_active(url):
+ return {
+ "interaction_kind": "show_all",
+ "target_locator": None,
+ "cooldown_active": True,
+ "expanded": False,
+ }
+
+ attempts: list[tuple[str, Any]] = [
+ ("text=ALL", page.get_by_text("ALL", exact=True).first),
+ ("role=button name=ALL", page.get_by_role("button", name="ALL").first),
+ ]
+
+ last_error: Exception | None = None
+ last_locator: str | None = None
+ for locator_name, handle in attempts:
+ last_locator = locator_name
+ try:
+ if await handle.is_visible(timeout=2500):
+ await handle.click(timeout=2500)
+ await page.wait_for_timeout(1200)
+ try:
+ await page.wait_for_load_state("networkidle", timeout=5000)
+ except Exception:
+ pass
+ return {
+ "interaction_kind": "show_all",
+ "target_locator": locator_name,
+ "cooldown_active": False,
+ "expanded": True,
+ }
+ except Exception as exc:
+ last_error = exc
+
+ select_locator = "select"
+ try:
+ await page.locator(select_locator).first.select_option(label="ALL", timeout=2500)
+ await page.wait_for_timeout(1200)
+ return {
+ "interaction_kind": "show_all",
+ "target_locator": select_locator,
+ "cooldown_active": False,
+ "expanded": True,
+ }
+ except Exception as exc:
+ last_error = exc
+
+ try:
+ await page.locator(select_locator).first.select_option(value="-1", timeout=2500)
+ await page.wait_for_timeout(1200)
+ return {
+ "interaction_kind": "show_all",
+ "target_locator": select_locator,
+ "cooldown_active": False,
+ "expanded": True,
+ }
+ except Exception as exc:
+ last_error = exc
+
+ self._activate_list_show_all_cooldown(url)
+ return {
+ "interaction_kind": "show_all",
+ "target_locator": last_locator or select_locator,
+ "cooldown_active": False,
+ "expanded": False,
+ "cooldown_applied": True,
+ "raw_exception_type": type(last_error).__name__ if last_error is not None else None,
+ "raw_exception_message": str(last_error) if last_error is not None else None,
+ }
+
+ async def setup_page_for_cache(self, page, url: str) -> Dict[str, Any] | None:
"""
Setup page before caching - click "ALL" to show all subnets.
On taostats.io/subnets, the default view shows only 10-25 rows.
Click "ALL" to show all ~128 subnets for complete visibility.
"""
- if not self._is_list_page(url):
- return
+ if self._is_list_page(url):
+ metadata: Dict[str, Any] = {
+ "page_kind": "taostats_list",
+ "interaction_kind": "show_all",
+ "selector_syntax_invalid": False,
+ }
+ try:
+ await self._wait_for_minimum_body_text(page, min_chars=120, timeout_ms=8000)
+ except Exception:
+ metadata["body_wait_timed_out"] = True
- try:
- # Click the "ALL" option in the rows selector
- # The selector shows: 10, 25, 50, 100, ALL
- all_button = page.locator('text="ALL"').first
- if await all_button.is_visible(timeout=3000):
- await all_button.click()
- # Wait for table to update with all rows
- await page.wait_for_timeout(2000)
- # Wait for network to settle after loading all rows
+ snapshot = await self._capture_body_snapshot(page)
+ metadata["page_body_ready"] = self._is_list_body_ready(snapshot)
+ if metadata["page_body_ready"]:
+ expand_result = await self._best_effort_expand_list(page, url)
+ metadata.update(expand_result)
+ if not expand_result.get("expanded"):
+ metadata["list_setup_soft_failed"] = True
+ else:
+ metadata["list_setup_soft_failed"] = True
+ return metadata
+
+ if self._extract_subnet_id(url):
+ try:
+ await self._wait_for_minimum_body_text(page, min_chars=140, timeout_ms=10000)
+ except Exception:
+ pass
+
+ snapshot = await self._capture_body_snapshot(page)
+ page_body_ready = self._is_detail_body_ready(snapshot)
+ if not page_body_ready:
+ raise TaostatsPrefetchSetupError(
+ "Taostats detail body did not become ready",
+ prefetch_phase="setup_page_for_cache",
+ wait_target="detail_body_ready",
+ evidence={
+ "page_kind": "taostats_detail",
+ "page_body_ready": False,
+ "detail_setup_soft_failed": False,
+ },
+ )
+
+ soft_failure: dict[str, Any] | None = None
+ for selector in (
+ "text=Subnet",
+ "text=Netuid",
+ "text=Statistics",
+ "text=Transactions",
+ "text=Holders",
+ "text=Price Impact",
+ ):
try:
- await page.wait_for_load_state("networkidle", timeout=10000)
- except Exception:
- pass
- except Exception:
- # If "ALL" button not found or click fails, continue with default view
- pass
+ await page.locator(selector).first.wait_for(timeout=8000)
+ soft_failure = None
+ break
+ except Exception as exc:
+ soft_failure = {
+ "page_kind": "taostats_detail",
+ "page_body_ready": True,
+ "detail_setup_soft_failed": True,
+ "prefetch_phase": "setup_page_for_cache",
+ "wait_target": selector,
+ "raw_exception_type": type(exc).__name__,
+ "raw_exception_message": str(exc),
+ }
+ continue
+
+ await page.wait_for_timeout(1200)
+ return soft_failure or {
+ "page_kind": "taostats_detail",
+ "page_body_ready": True,
+ "detail_setup_soft_failed": False,
+ "prefetch_phase": "setup_page_for_cache",
+ }
diff --git a/liveweb_arena/utils/llm_client.py b/liveweb_arena/utils/llm_client.py
index 849031c..c7c4b4a 100644
--- a/liveweb_arena/utils/llm_client.py
+++ b/liveweb_arena/utils/llm_client.py
@@ -1,16 +1,21 @@
-"""OpenAI-compatible LLM client with retry and streaming support"""
+"""OpenAI-compatible LLM client with retry, streaming, tool calls, and multi-server routing."""
import asyncio
-import json
+import email.utils
+import ipaddress
+import os
import random
+import re
import time
+import uuid
from dataclasses import dataclass, field
-from typing import Any, List, Optional, Tuple
+from urllib.parse import urlparse
+from typing import Any, Dict, List, Optional, Tuple
import httpx
import openai
-from .logger import log, progress, progress_done, is_verbose
+from .logger import is_verbose, log, progress, progress_done
class LLMFatalError(Exception):
@@ -30,6 +35,7 @@ def __init__(self, message: str, original_error: Exception = None, attempts: int
@dataclass
class ToolCall:
"""Parsed tool call from LLM response."""
+
id: str
function: dict # {"name": str, "arguments": str}
@@ -37,15 +43,212 @@ class ToolCall:
@dataclass
class LLMResponse:
"""Structured LLM response supporting both text and tool_calls."""
+
content: str = ""
tool_calls: List[ToolCall] = field(default_factory=list)
usage: Optional[dict] = None
+ request_id: Optional[str] = None
@property
def has_tool_calls(self) -> bool:
return len(self.tool_calls) > 0
+@dataclass(frozen=True)
+class LLMServerConfig:
+ """Static config for one OpenAI-compatible endpoint."""
+
+ server_id: str
+ base_url: str
+ api_key: str
+ model_name: Optional[str] = None
+ metadata: Optional[Dict[str, object]] = None
+
+
+@dataclass
+class _LLMServerState:
+ config: LLMServerConfig
+ inflight: int = 0
+ requests: int = 0
+ failures: int = 0
+ total_latency_s: float = 0.0
+
+
+@dataclass
+class _ServerLease:
+ server_id: str
+ base_url: str
+ api_key: str
+ started_at: float
+
+
+class MultiServerLLMRouter:
+ """
+ Route LLM requests across multiple OpenAI-compatible servers.
+
+ The default policy is sticky + steal:
+ - assign each route_key a preferred server for cache locality
+ - allow spilling to the least-loaded server when the preferred one is busier
+ """
+
+ def __init__(
+ self,
+ servers: List[LLMServerConfig],
+ route_policy: str = "sticky_steal",
+ max_inflight_requests: Optional[int] = None,
+ sticky_slack: int = 0,
+ sticky_latency_slack_s: float = 10.0,
+ ):
+ if not servers:
+ raise ValueError("MultiServerLLMRouter requires at least one server")
+
+ self._states: Dict[str, _LLMServerState] = {
+ server.server_id: _LLMServerState(config=server)
+ for server in servers
+ }
+ self._order: List[str] = [server.server_id for server in servers]
+ self._route_policy = route_policy
+ self._sticky_slack = max(0, sticky_slack)
+ self._sticky_latency_slack_s = max(0.0, sticky_latency_slack_s)
+ self._preferred_server: Dict[str, str] = {}
+ self._lock = asyncio.Lock()
+ self._global_semaphore = (
+ asyncio.Semaphore(max_inflight_requests)
+ if max_inflight_requests and max_inflight_requests > 0
+ else None
+ )
+
+ @property
+ def base_urls(self) -> List[str]:
+ return [self._states[server_id].config.base_url for server_id in self._order]
+
+ @property
+ def primary_base_url(self) -> str:
+ return self.base_urls[0]
+
+ @classmethod
+ def from_server_pool_file(
+ cls,
+ path: str,
+ route_policy: str = "sticky_steal",
+ max_inflight_requests: Optional[int] = None,
+ sticky_slack: int = 0,
+ sticky_latency_slack_s: float = 10.0,
+ default_api_key: Optional[str] = None,
+ ) -> "MultiServerLLMRouter":
+ import json
+ from pathlib import Path
+
+ payload = json.loads(Path(path).read_text())
+ servers = []
+ for idx, entry in enumerate(payload.get("servers", [])):
+ server_id = str(entry.get("server_id") or entry.get("id") or f"server-{idx}")
+ api_key = str(entry.get("api_key") or default_api_key or "")
+ if not api_key:
+ raise ValueError(f"Server {server_id} missing api_key")
+ servers.append(
+ LLMServerConfig(
+ server_id=server_id,
+ base_url=str(entry["base_url"]).rstrip("/"),
+ api_key=api_key,
+ model_name=entry.get("model_name"),
+ metadata=entry,
+ )
+ )
+ return cls(
+ servers=servers,
+ route_policy=route_policy,
+ max_inflight_requests=max_inflight_requests,
+ sticky_slack=sticky_slack,
+ sticky_latency_slack_s=sticky_latency_slack_s,
+ )
+
+ async def acquire(self, route_key: Optional[str] = None) -> _ServerLease:
+ if self._global_semaphore is not None:
+ await self._global_semaphore.acquire()
+
+ async with self._lock:
+ selected_id = self._select_server_id(route_key)
+ state = self._states[selected_id]
+ state.inflight += 1
+ state.requests += 1
+ if route_key:
+ self._preferred_server.setdefault(route_key, selected_id)
+ return _ServerLease(
+ server_id=selected_id,
+ base_url=state.config.base_url,
+ api_key=state.config.api_key,
+ started_at=time.time(),
+ )
+
+ async def release(self, lease: _ServerLease, success: bool, latency_s: float):
+ async with self._lock:
+ state = self._states[lease.server_id]
+ state.inflight = max(0, state.inflight - 1)
+ state.total_latency_s += max(0.0, latency_s)
+ if not success:
+ state.failures += 1
+
+ if self._global_semaphore is not None:
+ self._global_semaphore.release()
+
+ def snapshot(self) -> Dict[str, object]:
+ servers = []
+ for server_id in self._order:
+ state = self._states[server_id]
+ avg_latency = state.total_latency_s / state.requests if state.requests else 0.0
+ servers.append(
+ {
+ "server_id": server_id,
+ "base_url": state.config.base_url,
+ "inflight": state.inflight,
+ "requests": state.requests,
+ "failures": state.failures,
+ "avg_latency_s": avg_latency,
+ }
+ )
+ return {
+ "route_policy": self._route_policy,
+ "num_servers": len(servers),
+ "servers": servers,
+ }
+
+ def _select_server_id(self, route_key: Optional[str]) -> str:
+ states = [self._states[server_id] for server_id in self._order]
+ least_loaded = min(states, key=self._state_sort_key)
+
+ if self._route_policy != "sticky_steal" or not route_key:
+ return least_loaded.config.server_id
+
+ preferred_id = self._preferred_server.get(route_key)
+ if preferred_id is None:
+ return least_loaded.config.server_id
+
+ preferred = self._states.get(preferred_id)
+ if preferred is None:
+ return least_loaded.config.server_id
+
+ preferred_latency = self._avg_latency(preferred)
+ least_loaded_latency = self._avg_latency(least_loaded)
+
+ if (
+ preferred.inflight <= least_loaded.inflight + self._sticky_slack
+ and preferred_latency <= least_loaded_latency + self._sticky_latency_slack_s
+ ):
+ return preferred.config.server_id
+
+ self._preferred_server[route_key] = least_loaded.config.server_id
+ return least_loaded.config.server_id
+
+ @staticmethod
+ def _avg_latency(state: _LLMServerState) -> float:
+ return state.total_latency_s / state.requests if state.requests else 0.0
+
+ def _state_sort_key(self, state: _LLMServerState):
+ latency_penalty = min(self._avg_latency(state) / 60.0, 5.0)
+ return (state.inflight + latency_penalty, state.failures, state.requests)
+
+
class LLMClient:
"""
OpenAI-compatible LLM client.
@@ -54,33 +257,371 @@ class LLMClient:
- Streaming support with usage tracking
- Exponential backoff retry for recoverable errors
- Configurable timeouts
+ - Optional multi-server routing with sticky + steal
"""
- # Recoverable error status codes
RETRY_STATUS_CODES = {429, 503, 502, 500}
-
- # Retry configuration
- MAX_RETRIES = 10 # Increased for rate limit resilience
- BASE_DELAY = 1.0 # seconds
- MAX_DELAY = 30.0 # seconds
-
- # Default timeout per request (should be less than total eval timeout)
- DEFAULT_TIMEOUT = 600 # seconds
- # Maximum chunks to receive (safety limit, ~32k chunks ≈ ~64k tokens)
- MAX_CHUNKS = 32000
-
- def __init__(self, base_url: str, api_key: str, default_timeout: int = None):
- """
- Initialize LLM client.
-
- Args:
- base_url: OpenAI-compatible API base URL
- api_key: API key for authentication
- default_timeout: Default request timeout in seconds
- """
- self._base_url = base_url.rstrip("/")
+ MAX_RETRIES = 10
+ BASE_DELAY = 1.0
+ MAX_DELAY = 30.0
+ RATE_LIMIT_MAX_DELAY = float(os.getenv("LIVEWEB_LLM_RATE_LIMIT_MAX_DELAY", "120"))
+ DEFAULT_TIMEOUT = 600
+ MAX_CHUNKS = int(os.getenv("LIVEWEB_LLM_MAX_CHUNKS", "32000"))
+ _GLOBAL_RATE_LIMIT_UNTIL: Dict[str, float] = {}
+ _CONTEXT_LENGTH_DETAILS_RE = re.compile(
+ r"maximum context length of (\d+) tokens.*?"
+ r"requested a total of (\d+) tokens:\s*"
+ r"(\d+) tokens from the input messages and (\d+) tokens for the completion",
+ re.IGNORECASE | re.DOTALL,
+ )
+
+ def __init__(
+ self,
+ base_url: Optional[str] = None,
+ api_key: Optional[str] = None,
+ default_timeout: int = None,
+ router: Optional[MultiServerLLMRouter] = None,
+ route_key: Optional[str] = None,
+ max_retries: Optional[int] = None,
+ strict_serial: bool = False,
+ enable_thinking: Optional[bool] = None,
+ separate_reasoning: Optional[bool] = None,
+ reasoning_effort: Optional[str] = None,
+ strip_reasoning_output: bool = False,
+ ):
+ if router is None and (not base_url or not api_key):
+ raise ValueError("LLMClient requires either router or both base_url/api_key")
+
+ self._router = router
+ self._route_key = route_key
+ self._base_url = base_url.rstrip("/") if base_url else (router.primary_base_url if router else "")
+ self._base_urls = router.base_urls if router else [self._base_url]
self._api_key = api_key
self._default_timeout = default_timeout or self.DEFAULT_TIMEOUT
+ self._strict_serial = strict_serial
+ self._max_retries = max(1, int(max_retries or self.MAX_RETRIES))
+ self._max_completion_tokens = self._read_max_completion_tokens()
+ self._enable_thinking = (
+ enable_thinking if enable_thinking is not None else self._read_optional_bool_env("LIVEWEB_ENABLE_THINKING")
+ )
+ self._separate_reasoning = (
+ separate_reasoning
+ if separate_reasoning is not None
+ else self._read_optional_bool_env("LIVEWEB_SEPARATE_REASONING")
+ )
+ self._reasoning_effort = (
+ reasoning_effort.strip().lower()
+ if isinstance(reasoning_effort, str) and reasoning_effort.strip()
+ else (os.getenv("LIVEWEB_REASONING_EFFORT", "").strip().lower() or None)
+ )
+ self._strip_reasoning_output = bool(
+ strip_reasoning_output or (os.getenv("LIVEWEB_STRIP_REASONING_OUTPUT", "0") == "1")
+ )
+ self._format_recovery_temperature = self._read_float_env("LIVEWEB_FORMAT_RECOVERY_TEMPERATURE", 0.35)
+ self._format_recovery_top_p = self._read_float_env("LIVEWEB_FORMAT_RECOVERY_TOP_P", 0.95)
+ self._last_failure_metadata: Dict[str, object] = {}
+ if self._strict_serial:
+ self._max_retries = 1
+
+ def _raise_strict_serial_error(self, message: str, error: Exception, attempts: int) -> None:
+ raise LLMFatalError(
+ message,
+ original_error=error,
+ attempts=attempts,
+ ) from error
+
+ def _generate_request_id(self, request_kind: str) -> str:
+ route_key = (self._route_key or "default").replace(":", "-").replace("/", "-")
+ route_key = route_key[:48]
+ return f"liveweb-{request_kind}-{route_key}-{uuid.uuid4().hex[:12]}"
+
+ def _set_last_failure_metadata(self, metadata: Optional[Dict[str, object]]) -> None:
+ self._last_failure_metadata = dict(metadata or {})
+
+ def get_last_failure_metadata(self) -> Dict[str, object]:
+ return dict(self._last_failure_metadata)
+
+ @staticmethod
+ def _read_max_completion_tokens() -> Optional[int]:
+ raw = os.getenv("LIVEWEB_MAX_COMPLETION_TOKENS")
+ if raw is None or raw == "":
+ return None
+ try:
+ value = int(raw)
+ except ValueError:
+ log("LLM", f"Invalid LIVEWEB_MAX_COMPLETION_TOKENS={raw!r}; ignoring")
+ return None
+ return value if value > 0 else None
+
+ @staticmethod
+ def _read_optional_bool_env(name: str) -> Optional[bool]:
+ raw = os.getenv(name)
+ if raw is None or raw == "":
+ return None
+ normalized = raw.strip().lower()
+ if normalized in {"1", "true", "yes", "on"}:
+ return True
+ if normalized in {"0", "false", "no", "off"}:
+ return False
+ log("LLM", f"Invalid {name}={raw!r}; ignoring")
+ return None
+
+ @staticmethod
+ def _read_float_env(name: str, default: float) -> float:
+ raw = os.getenv(name)
+ if raw is None or raw == "":
+ return default
+ try:
+ return float(raw)
+ except ValueError:
+ log("LLM", f"Invalid {name}={raw!r}; using default {default}")
+ return default
+
+ @classmethod
+ def _extract_context_length_details(cls, message: str) -> Optional[Tuple[int, int, int, int]]:
+ match = cls._CONTEXT_LENGTH_DETAILS_RE.search(message)
+ if not match:
+ return None
+ return tuple(int(group) for group in match.groups())
+
+ @classmethod
+ def _compute_reduced_completion_cap(
+ cls,
+ *,
+ error: Exception,
+ current_cap: Optional[int],
+ ) -> Optional[int]:
+ details = cls._extract_context_length_details(str(error))
+ if details is None:
+ return None
+
+ context_limit, _requested_total, prompt_tokens, requested_completion = details
+ reserve_tokens = max(64, min(512, context_limit // 64))
+ safe_cap = context_limit - prompt_tokens - reserve_tokens
+ if current_cap is not None:
+ safe_cap = min(safe_cap, current_cap - 1)
+ else:
+ safe_cap = min(safe_cap, requested_completion - 1)
+ if safe_cap < 16:
+ return None
+ return safe_cap
+
+ @staticmethod
+ def _is_openrouter_base_url(base_url: str) -> bool:
+ hostname = (urlparse(base_url).hostname or "").lower()
+ return hostname == "openrouter.ai" or hostname.endswith(".openrouter.ai")
+
+ @staticmethod
+ def _is_openrouter_kimi_model(model: str) -> bool:
+ normalized = (model or "").strip().lower()
+ return normalized.startswith("moonshotai/kimi-k2.5")
+
+ def _apply_reasoning_controls(self, params: Dict[str, object], *, base_url: str, model: str) -> None:
+ extra_body = dict(params.get("extra_body") or {})
+ if self._enable_thinking is not None:
+ extra_body["chat_template_kwargs"] = {"enable_thinking": self._enable_thinking}
+ if self._separate_reasoning is not None:
+ extra_body["separate_reasoning"] = self._separate_reasoning
+ reasoning_payload: Dict[str, object] | None = None
+ if self._enable_thinking is False:
+ if self._is_openrouter_base_url(base_url) and self._is_openrouter_kimi_model(model):
+ reasoning_payload = {"enabled": False}
+ elif self._is_openrouter_base_url(base_url):
+ reasoning_payload = {"effort": "none", "exclude": True}
+ else:
+ reasoning_payload = {"enabled": False}
+ elif self._reasoning_effort:
+ reasoning_payload = {"effort": self._reasoning_effort}
+ if reasoning_payload is not None:
+ extra_body["reasoning"] = reasoning_payload
+ if extra_body:
+ params["extra_body"] = extra_body
+
+ @staticmethod
+ def _extract_visible_content(message: Any) -> str:
+ content = getattr(message, "content", "") or ""
+ if isinstance(content, str):
+ return content
+ if isinstance(content, list):
+ parts: list[str] = []
+ for item in content:
+ if isinstance(item, str):
+ parts.append(item)
+ continue
+ if not isinstance(item, dict):
+ text = getattr(item, "text", None)
+ if text:
+ parts.append(str(text))
+ continue
+ item_type = str(item.get("type") or "").strip().lower()
+ if item_type in {"reasoning", "reasoning_content", "thinking"}:
+ continue
+ text = item.get("text")
+ if text is None:
+ text = item.get("content")
+ if text:
+ parts.append(str(text))
+ return "".join(parts)
+ return str(content)
+
+ def _sanitize_usage(self, usage: Optional[dict]) -> Optional[dict]:
+ if usage is None or not self._strip_reasoning_output:
+ return usage
+ sanitized = dict(usage)
+ for key in ("reasoning_tokens", "reasoning", "reasoning_content"):
+ sanitized.pop(key, None)
+ for nested_key in ("completion_tokens_details", "prompt_tokens_details"):
+ nested = sanitized.get(nested_key)
+ if isinstance(nested, dict):
+ nested_copy = dict(nested)
+ for key in list(nested_copy.keys()):
+ if "reasoning" in str(key).lower():
+ nested_copy.pop(key, None)
+ sanitized[nested_key] = nested_copy
+ return sanitized
+
+ async def _wait_for_global_rate_limit_window(self, base_url: str) -> None:
+ while True:
+ until = self._GLOBAL_RATE_LIMIT_UNTIL.get(base_url, 0.0)
+ now = time.time()
+ if until <= now:
+ return
+ await asyncio.sleep(min(5.0, max(0.1, until - now)))
+
+ @staticmethod
+ def _parse_retry_after(value: object) -> float | None:
+ if value is None:
+ return None
+ text = str(value).strip()
+ if not text:
+ return None
+ try:
+ seconds = float(text)
+ return max(0.0, seconds)
+ except ValueError:
+ pass
+ try:
+ dt = email.utils.parsedate_to_datetime(text)
+ return max(0.0, dt.timestamp() - time.time())
+ except Exception:
+ return None
+
+ def _compute_rate_limit_delay(self, error: Exception, attempt: int) -> float:
+ response = getattr(error, "response", None)
+ headers = getattr(response, "headers", {}) or {}
+ for key in (
+ "retry-after",
+ "x-ratelimit-reset",
+ "ratelimit-reset",
+ "x-ratelimit-reset-requests",
+ ):
+ delay = self._parse_retry_after(headers.get(key))
+ if delay is not None and delay > 0:
+ return min(self.RATE_LIMIT_MAX_DELAY, delay + random.uniform(0, 1))
+ fallback = min(self.RATE_LIMIT_MAX_DELAY, self.BASE_DELAY * (2 ** min(attempt, 8)))
+ return fallback + random.uniform(0, 1)
+
+ async def _wait_for_rate_limit_reset(
+ self,
+ *,
+ error: Exception,
+ attempt: int,
+ request_id: str | None,
+ model: str,
+ base_url: str,
+ ) -> None:
+ delay = self._compute_rate_limit_delay(error, attempt)
+ self._GLOBAL_RATE_LIMIT_UNTIL[base_url] = max(
+ self._GLOBAL_RATE_LIMIT_UNTIL.get(base_url, 0.0),
+ time.time() + delay,
+ )
+ self._set_last_failure_metadata(
+ {
+ "failure_stage": "chat_with_tools",
+ "failure_type": "rate_limit",
+ "model": model,
+ "base_url": base_url,
+ "request_id": request_id,
+ "retry_delay_s": delay,
+ "attempt": attempt + 1,
+ }
+ )
+ log("LLM", f"Rate limited; waiting {delay:.1f}s before retry")
+ await asyncio.sleep(delay)
+
+ @staticmethod
+ def _server_root_url(base_url: str) -> str:
+ normalized = base_url.rstrip("/")
+ if normalized.endswith("/v1"):
+ return normalized[:-3]
+ return normalized
+
+ @staticmethod
+ def _should_bypass_proxy(base_url: str) -> bool:
+ try:
+ hostname = (urlparse(base_url).hostname or "").strip()
+ if not hostname:
+ return False
+ if hostname in {"api.aicodemirror.com"}:
+ return True
+ if hostname in {"localhost", "127.0.0.1"}:
+ return True
+ ip = ipaddress.ip_address(hostname)
+ return ip.is_private or ip.is_loopback or ip.is_link_local
+ except ValueError:
+ return False
+
+ def _build_httpx_client(
+ self,
+ *,
+ base_url: str,
+ timeout: httpx.Timeout,
+ ) -> httpx.AsyncClient:
+ # Local SGLang endpoints should bypass the host proxy, otherwise
+ # requests to the colocated router/servers can get sent to localhost:10812.
+ return httpx.AsyncClient(
+ timeout=timeout,
+ trust_env=not self._should_bypass_proxy(base_url),
+ )
+
+ async def _abort_request(
+ self,
+ lease: Optional[_ServerLease],
+ request_id: Optional[str],
+ reason: str,
+ ) -> None:
+ if lease is None or not request_id:
+ return
+
+ abort_url = f"{self._server_root_url(lease.base_url)}/abort_request"
+ headers = {
+ "Content-Type": "application/json; charset=utf-8",
+ "Authorization": f"Bearer {lease.api_key}",
+ "X-API-Key": lease.api_key,
+ }
+ try:
+ async with self._build_httpx_client(
+ base_url=lease.base_url,
+ timeout=httpx.Timeout(10.0, connect=5.0),
+ ) as client:
+ response = await client.post(
+ abort_url,
+ json={
+ "rid": request_id,
+ "abort_all": False,
+ "abort_message": reason[:200],
+ },
+ headers=headers,
+ )
+ if response.status_code == 200:
+ log("LLM", f"Abort requested for {request_id} on {lease.server_id}")
+ else:
+ log(
+ "LLM",
+ f"Abort request for {request_id} on {lease.server_id} returned {response.status_code}",
+ )
+ except Exception as abort_error:
+ log("LLM", f"Abort request for {request_id} on {lease.server_id} failed: {abort_error}")
async def chat(
self,
@@ -91,87 +632,189 @@ async def chat(
seed: Optional[int] = None,
timeout_s: int = None,
) -> Tuple[str, Optional[dict]]:
- """
- Make a chat completion request.
-
- Args:
- system: System prompt
- user: User message
- model: Model name
- temperature: Sampling temperature
- seed: Random seed for reproducibility
- timeout_s: Request timeout in seconds (default: use client default)
-
- Returns:
- Tuple of (response content, usage dict or None)
- """
- # Use default timeout if not specified
actual_timeout = timeout_s if timeout_s is not None else self._default_timeout
- # Build messages
messages = []
if system:
messages.append({"role": "system", "content": system})
messages.append({"role": "user", "content": user})
- # Retry loop with exponential backoff
last_error = None
- for attempt in range(self.MAX_RETRIES):
+ attempt = 0
+ rate_limit_attempt = 0
+ max_completion_tokens_override = None
+ while True:
+ lease = None
+ request_id = None
+ request_start = time.time()
try:
- # Wrap in total timeout to prevent runaway requests
+ lease = await self._acquire_lease()
+ request_id = self._generate_request_id("chat")
content, usage = await asyncio.wait_for(
self._make_request(
+ base_url=lease.base_url,
+ api_key=lease.api_key,
messages=messages,
model=model,
temperature=temperature,
seed=seed,
timeout_s=actual_timeout,
+ request_id=request_id,
+ max_completion_tokens_override=max_completion_tokens_override,
),
timeout=actual_timeout,
)
+ await self._release_lease(lease, success=True, latency_s=time.time() - request_start)
return content, usage
except asyncio.TimeoutError:
+ await self._abort_request(
+ lease,
+ request_id,
+ f"timeout after {actual_timeout}s (attempt {attempt + 1})",
+ )
+ if lease is not None:
+ await self._release_lease(lease, success=False, latency_s=time.time() - request_start)
last_error = TimeoutError(f"LLM request timed out after {actual_timeout}s")
- log("LLM", f"Total timeout ({actual_timeout}s) exceeded, attempt {attempt + 1}/{self.MAX_RETRIES}")
+ if self._strict_serial:
+ self._raise_strict_serial_error(
+ f"Strict-serial LLM request timed out after {actual_timeout}s",
+ last_error,
+ attempt + 1,
+ )
+ log("LLM", f"Total timeout ({actual_timeout}s) exceeded, attempt {attempt + 1}/{self._max_retries}")
await self._backoff(attempt)
continue
+ except asyncio.CancelledError:
+ await self._abort_request(
+ lease,
+ request_id,
+ f"cancelled (attempt {attempt + 1})",
+ )
+ if lease is not None:
+ await self._release_lease(lease, success=False, latency_s=time.time() - request_start)
+ raise
+
except openai.RateLimitError as e:
+ await self._abort_request(
+ lease,
+ request_id,
+ f"rate limit (attempt {attempt + 1})",
+ )
+ if lease is not None:
+ await self._release_lease(lease, success=False, latency_s=time.time() - request_start)
last_error = e
- log("LLM", f"Rate limit hit, attempt {attempt + 1}/{self.MAX_RETRIES}")
+ if self._strict_serial:
+ self._raise_strict_serial_error(
+ "Strict-serial LLM request hit rate limit",
+ e,
+ attempt + 1,
+ )
+ log("LLM", f"Rate limit hit, attempt {attempt + 1}/{self._max_retries}")
await self._backoff(attempt)
except openai.BadRequestError as e:
- # Check for token limit errors - these are unrecoverable
+ await self._abort_request(
+ lease,
+ request_id,
+ f"bad request (attempt {attempt + 1})",
+ )
+ if lease is not None:
+ await self._release_lease(lease, success=False, latency_s=time.time() - request_start)
error_msg = str(e).lower()
- if "is longer than the model" in error_msg or "context_length_exceeded" in error_msg:
+ has_context_details = self._extract_context_length_details(str(e)) is not None
+ if (
+ "is longer than the model" in error_msg
+ or "context_length_exceeded" in error_msg
+ or "maximum context length" in error_msg
+ or has_context_details
+ ):
+ reduced_cap = self._compute_reduced_completion_cap(
+ error=e,
+ current_cap=(
+ max_completion_tokens_override
+ if max_completion_tokens_override is not None
+ else self._max_completion_tokens
+ ),
+ )
+ if reduced_cap is not None:
+ current_cap = (
+ max_completion_tokens_override
+ if max_completion_tokens_override is not None
+ else self._max_completion_tokens
+ )
+ if current_cap is not None and reduced_cap >= current_cap:
+ raise LLMFatalError(
+ f"Token limit exceeded without room to reduce completion cap: {e}",
+ original_error=e,
+ attempts=attempt + 1,
+ )
+ max_completion_tokens_override = reduced_cap
+ log("LLM", f"Retrying with reduced max_completion_tokens={reduced_cap} after context error")
+ continue
log("LLM", f"Token limit exceeded - fatal error: {e}", force=True)
raise LLMFatalError(
f"Token limit exceeded: {e}",
original_error=e,
attempts=attempt + 1,
)
- # Other bad request errors - don't retry
raise
except openai.APIStatusError as e:
+ await self._abort_request(
+ lease,
+ request_id,
+ f"api status {e.status_code} (attempt {attempt + 1})",
+ )
+ if lease is not None:
+ await self._release_lease(lease, success=False, latency_s=time.time() - request_start)
if e.status_code in self.RETRY_STATUS_CODES:
last_error = e
- log("LLM", f"API error {e.status_code}, attempt {attempt + 1}/{self.MAX_RETRIES}")
+ if self._strict_serial:
+ self._raise_strict_serial_error(
+ f"Strict-serial LLM API error {e.status_code}",
+ e,
+ attempt + 1,
+ )
+ log("LLM", f"API error {e.status_code}, attempt {attempt + 1}/{self._max_retries}")
await self._backoff(attempt)
else:
raise
except (httpx.TimeoutException, httpx.ConnectError) as e:
+ await self._abort_request(
+ lease,
+ request_id,
+ f"connection error (attempt {attempt + 1})",
+ )
+ if lease is not None:
+ await self._release_lease(lease, success=False, latency_s=time.time() - request_start)
last_error = e
- log("LLM", f"Connection error, attempt {attempt + 1}/{self.MAX_RETRIES}: {e}")
+ if self._strict_serial:
+ self._raise_strict_serial_error(
+ f"Strict-serial LLM connection error: {e}",
+ e,
+ attempt + 1,
+ )
+ log("LLM", f"Connection error, attempt {attempt + 1}/{self._max_retries}: {e}")
await self._backoff(attempt)
except Exception as e:
- # Check for token limit in generic exceptions too
+ await self._abort_request(
+ lease,
+ request_id,
+ f"client error: {type(e).__name__} (attempt {attempt + 1})",
+ )
+ if lease is not None:
+ await self._release_lease(lease, success=False, latency_s=time.time() - request_start)
error_msg = str(e).lower()
- if "is longer than the model" in error_msg or "context_length_exceeded" in error_msg:
+ if (
+ "is longer than the model" in error_msg
+ or "context_length_exceeded" in error_msg
+ or "maximum context length" in error_msg
+ or self._extract_context_length_details(str(e)) is not None
+ ):
log("LLM", f"Token limit exceeded - fatal error: {e}", force=True)
raise LLMFatalError(
f"Token limit exceeded: {e}",
@@ -179,10 +822,15 @@ async def chat(
attempts=attempt + 1,
)
last_error = e
- log("LLM", f"Error, attempt {attempt + 1}/{self.MAX_RETRIES}: {e}")
+ if self._strict_serial:
+ self._raise_strict_serial_error(
+ f"Strict-serial LLM error: {e}",
+ e,
+ attempt + 1,
+ )
+ log("LLM", f"Error, attempt {attempt + 1}/{self._max_retries}: {e}")
await self._backoff(attempt)
- # All retries exhausted
raise last_error or Exception("LLM request failed after all retries")
async def chat_with_tools(
@@ -195,106 +843,361 @@ async def chat_with_tools(
seed: Optional[int] = None,
timeout_s: int = None,
) -> LLMResponse:
- """
- Make a chat completion request with optional tool/function calling support.
-
- When tools are provided, the LLM may respond with tool_calls instead of
- (or in addition to) text content.
-
- Args:
- system: System prompt
- user: User message
- model: Model name
- tools: Optional list of OpenAI-format tool definitions
- temperature: Sampling temperature
- seed: Random seed for reproducibility
- timeout_s: Request timeout in seconds
-
- Returns:
- LLMResponse with content, tool_calls, and usage
- """
actual_timeout = timeout_s if timeout_s is not None else self._default_timeout
messages = []
if system:
messages.append({"role": "system", "content": system})
messages.append({"role": "user", "content": user})
+ self._set_last_failure_metadata(None)
last_error = None
- for attempt in range(self.MAX_RETRIES):
+ attempt = 0
+ rate_limit_attempt = 0
+ max_completion_tokens_override = None
+ while True:
+ lease = None
+ request_id = None
+ request_start = time.time()
try:
+ lease = await self._acquire_lease()
+ await self._wait_for_global_rate_limit_window(lease.base_url)
+ request_id = self._generate_request_id("tools")
response = await asyncio.wait_for(
self._make_request_with_tools(
+ base_url=lease.base_url,
+ api_key=lease.api_key,
messages=messages,
model=model,
tools=tools,
temperature=temperature,
+ top_p=None,
seed=seed,
timeout_s=actual_timeout,
+ request_id=request_id,
+ max_completion_tokens_override=max_completion_tokens_override,
),
timeout=actual_timeout,
)
+ await self._release_lease(lease, success=True, latency_s=time.time() - request_start)
return response
except asyncio.TimeoutError:
+ await self._abort_request(
+ lease,
+ request_id,
+ f"timeout after {actual_timeout}s (attempt {attempt + 1})",
+ )
+ if lease is not None:
+ await self._release_lease(lease, success=False, latency_s=time.time() - request_start)
last_error = TimeoutError(f"LLM request timed out after {actual_timeout}s")
- log("LLM", f"Total timeout ({actual_timeout}s) exceeded, attempt {attempt + 1}/{self.MAX_RETRIES}")
+ if self._strict_serial:
+ self._raise_strict_serial_error(
+ f"Strict-serial LLM request timed out after {actual_timeout}s",
+ last_error,
+ attempt + 1,
+ )
+ log("LLM", f"Total timeout ({actual_timeout}s) exceeded, attempt {attempt + 1}/{self._max_retries}")
+ attempt += 1
+ if attempt >= self._max_retries:
+ raise last_error or Exception("LLM request failed after all retries")
await self._backoff(attempt)
continue
+ except asyncio.CancelledError:
+ await self._abort_request(
+ lease,
+ request_id,
+ f"cancelled (attempt {attempt + 1})",
+ )
+ if lease is not None:
+ await self._release_lease(lease, success=False, latency_s=time.time() - request_start)
+ raise
+
except openai.RateLimitError as e:
+ await self._abort_request(
+ lease,
+ request_id,
+ f"rate limit (attempt {rate_limit_attempt + 1})",
+ )
+ if lease is not None:
+ await self._release_lease(lease, success=False, latency_s=time.time() - request_start)
last_error = e
- log("LLM", f"Rate limit hit, attempt {attempt + 1}/{self.MAX_RETRIES}")
- await self._backoff(attempt)
+ await self._wait_for_rate_limit_reset(
+ error=e,
+ attempt=rate_limit_attempt,
+ request_id=request_id,
+ model=model,
+ base_url=lease.base_url if lease is not None else self._base_url,
+ )
+ rate_limit_attempt += 1
+ continue
except openai.BadRequestError as e:
+ await self._abort_request(
+ lease,
+ request_id,
+ f"bad request (attempt {attempt + 1})",
+ )
+ if lease is not None:
+ await self._release_lease(lease, success=False, latency_s=time.time() - request_start)
error_msg = str(e).lower()
- if "is longer than the model" in error_msg or "context_length_exceeded" in error_msg:
+ has_context_details = self._extract_context_length_details(str(e)) is not None
+ if (
+ "is longer than the model" in error_msg
+ or "context_length_exceeded" in error_msg
+ or "maximum context length" in error_msg
+ or has_context_details
+ ):
+ reduced_cap = self._compute_reduced_completion_cap(
+ error=e,
+ current_cap=(
+ max_completion_tokens_override
+ if max_completion_tokens_override is not None
+ else self._max_completion_tokens
+ ),
+ )
+ if reduced_cap is not None:
+ current_cap = (
+ max_completion_tokens_override
+ if max_completion_tokens_override is not None
+ else self._max_completion_tokens
+ )
+ if current_cap is not None and reduced_cap >= current_cap:
+ raise LLMFatalError(
+ f"Token limit exceeded without room to reduce completion cap: {e}",
+ original_error=e,
+ attempts=attempt + 1,
+ )
+ max_completion_tokens_override = reduced_cap
+ log("LLM", f"Retrying with reduced max_completion_tokens={reduced_cap} after context error")
+ continue
raise LLMFatalError(f"Token limit exceeded: {e}", original_error=e, attempts=attempt + 1)
raise
except openai.APIStatusError as e:
+ await self._abort_request(
+ lease,
+ request_id,
+ f"api status {e.status_code} (attempt {attempt + 1})",
+ )
+ if lease is not None:
+ await self._release_lease(lease, success=False, latency_s=time.time() - request_start)
if e.status_code in self.RETRY_STATUS_CODES:
last_error = e
- log("LLM", f"API error {e.status_code}, attempt {attempt + 1}/{self.MAX_RETRIES}")
+ if e.status_code == 429:
+ await self._wait_for_rate_limit_reset(
+ error=e,
+ attempt=rate_limit_attempt,
+ request_id=request_id,
+ model=model,
+ base_url=lease.base_url if lease is not None else self._base_url,
+ )
+ rate_limit_attempt += 1
+ continue
+ if self._strict_serial:
+ self._raise_strict_serial_error(
+ f"Strict-serial LLM API error {e.status_code}",
+ e,
+ attempt + 1,
+ )
+ log("LLM", f"API error {e.status_code}, attempt {attempt + 1}/{self._max_retries}")
+ attempt += 1
+ if attempt >= self._max_retries:
+ raise last_error or Exception("LLM request failed after all retries")
await self._backoff(attempt)
else:
raise
except (httpx.TimeoutException, httpx.ConnectError) as e:
+ await self._abort_request(
+ lease,
+ request_id,
+ f"connection error (attempt {attempt + 1})",
+ )
+ if lease is not None:
+ await self._release_lease(lease, success=False, latency_s=time.time() - request_start)
last_error = e
- log("LLM", f"Connection error, attempt {attempt + 1}/{self.MAX_RETRIES}: {e}")
+ if self._strict_serial:
+ self._raise_strict_serial_error(
+ f"Strict-serial LLM connection error: {e}",
+ e,
+ attempt + 1,
+ )
+ log("LLM", f"Connection error, attempt {attempt + 1}/{self._max_retries}: {e}")
+ attempt += 1
+ if attempt >= self._max_retries:
+ raise last_error or Exception("LLM request failed after all retries")
await self._backoff(attempt)
except Exception as e:
+ await self._abort_request(
+ lease,
+ request_id,
+ f"client error: {type(e).__name__} (attempt {attempt + 1})",
+ )
+ if lease is not None:
+ await self._release_lease(lease, success=False, latency_s=time.time() - request_start)
error_msg = str(e).lower()
- if "is longer than the model" in error_msg or "context_length_exceeded" in error_msg:
+ if (
+ "is longer than the model" in error_msg
+ or "context_length_exceeded" in error_msg
+ or "maximum context length" in error_msg
+ or self._extract_context_length_details(str(e)) is not None
+ ):
raise LLMFatalError(f"Token limit exceeded: {e}", original_error=e, attempts=attempt + 1)
last_error = e
- log("LLM", f"Error, attempt {attempt + 1}/{self.MAX_RETRIES}: {e}")
+ if self._strict_serial:
+ self._raise_strict_serial_error(
+ f"Strict-serial LLM error: {e}",
+ e,
+ attempt + 1,
+ )
+ log("LLM", f"Error, attempt {attempt + 1}/{self._max_retries}: {e}")
+ attempt += 1
+ if attempt >= self._max_retries:
+ raise last_error or Exception("LLM request failed after all retries")
await self._backoff(attempt)
- raise last_error or Exception("LLM request failed after all retries")
+ async def chat_with_tools_recovery(
+ self,
+ model: str,
+ messages: Optional[list] = None,
+ system: str = "",
+ user: str = "",
+ assistant_prefix: str = "",
+ tools: Optional[List[dict]] = None,
+ seed: Optional[int] = None,
+ timeout_s: int = None,
+ max_new_tokens: Optional[int] = None,
+ ) -> LLMResponse:
+ actual_timeout = timeout_s if timeout_s is not None else self._default_timeout
+
+ if messages is None:
+ messages = []
+ if system:
+ messages.append({"role": "system", "content": system})
+ messages.append({"role": "user", "content": user})
+ remediation = (
+ "The previous assistant output had invalid tool-call formatting. "
+ "Emit exactly one valid tool call now. No explanation, no prose, no markdown, no XML."
+ )
+ if assistant_prefix:
+ messages.append({"role": "assistant", "content": assistant_prefix})
+ remediation = (
+ "Continue from the assistant's last output and emit exactly one valid tool call now. "
+ "No explanation, no prose, no markdown, no XML."
+ )
+ messages.append(
+ {
+ "role": "user",
+ "content": remediation,
+ }
+ )
+ self._set_last_failure_metadata(None)
+
+ last_error = None
+ for attempt in range(self._max_retries):
+ lease = None
+ request_id = None
+ request_start = time.time()
+ try:
+ lease = await self._acquire_lease()
+ await self._wait_for_global_rate_limit_window(lease.base_url)
+ request_id = self._generate_request_id("format-recovery")
+ response = await asyncio.wait_for(
+ self._make_request_with_tools(
+ base_url=lease.base_url,
+ api_key=lease.api_key,
+ messages=messages,
+ model=model,
+ tools=tools,
+ temperature=self._format_recovery_temperature,
+ top_p=self._format_recovery_top_p,
+ seed=seed,
+ timeout_s=actual_timeout,
+ request_id=request_id,
+ max_completion_tokens_override=max_new_tokens,
+ ),
+ timeout=actual_timeout,
+ )
+ await self._release_lease(lease, success=True, latency_s=time.time() - request_start)
+ return response
+ except Exception as e:
+ await self._abort_request(
+ lease,
+ request_id,
+ f"format recovery error: {type(e).__name__} (attempt {attempt + 1})",
+ )
+ if lease is not None:
+ await self._release_lease(lease, success=False, latency_s=time.time() - request_start)
+ last_error = e
+ if isinstance(e, openai.RateLimitError) or (
+ isinstance(e, openai.APIStatusError) and getattr(e, "status_code", None) == 429
+ ):
+ await self._wait_for_rate_limit_reset(
+ error=e,
+ attempt=rate_limit_attempt,
+ request_id=request_id,
+ model=model,
+ base_url=lease.base_url if lease is not None else self._base_url,
+ )
+ rate_limit_attempt += 1
+ continue
+ if self._strict_serial:
+ self._raise_strict_serial_error(
+ f"Strict-serial format recovery error: {e}",
+ e,
+ attempt + 1,
+ )
+ attempt += 1
+ if attempt >= self._max_retries:
+ raise last_error or Exception("LLM format recovery failed after all retries")
+ await self._backoff(attempt)
+ raise last_error or Exception("LLM format recovery failed after all retries")
+
+ async def _acquire_lease(self) -> _ServerLease:
+ if self._router is not None:
+ return await self._router.acquire(route_key=self._route_key)
+ return _ServerLease(
+ server_id="single",
+ base_url=self._base_url,
+ api_key=self._api_key,
+ started_at=time.time(),
+ )
+
+ async def _release_lease(self, lease: _ServerLease, success: bool, latency_s: float):
+ if self._router is not None:
+ await self._router.release(lease, success=success, latency_s=latency_s)
async def _make_request_with_tools(
self,
+ base_url: str,
+ api_key: str,
messages: list,
model: str,
tools: Optional[List[dict]],
temperature: float,
+ top_p: Optional[float],
seed: Optional[int],
timeout_s: int,
+ request_id: str,
+ max_completion_tokens_override: Optional[int] = None,
) -> LLMResponse:
- """Make a single API request with optional tool calling (non-streaming for tool support)."""
timeout_config = httpx.Timeout(
- connect=30.0, read=timeout_s, write=30.0, pool=30.0,
+ connect=30.0,
+ read=timeout_s,
+ write=30.0,
+ pool=30.0,
)
+ http_client = self._build_httpx_client(base_url=base_url, timeout=timeout_config)
client = openai.AsyncOpenAI(
- base_url=self._base_url,
- api_key=self._api_key,
+ base_url=base_url,
+ api_key=api_key,
timeout=timeout_config,
max_retries=0,
+ http_client=http_client,
)
try:
@@ -303,67 +1206,115 @@ async def _make_request_with_tools(
"messages": messages,
"temperature": temperature,
}
+ if top_p is not None:
+ params["top_p"] = top_p
if tools:
params["tools"] = tools
if seed is not None:
params["seed"] = seed
+ effective_max_completion_tokens = (
+ max_completion_tokens_override
+ if max_completion_tokens_override is not None
+ else self._max_completion_tokens
+ )
+ if effective_max_completion_tokens is not None:
+ # Pass both names for compatibility across OpenAI-compatible servers.
+ params["max_tokens"] = effective_max_completion_tokens
+ params["max_completion_tokens"] = effective_max_completion_tokens
+ params["extra_body"] = {"request_id": request_id}
+ self._apply_reasoning_controls(params, base_url=base_url, model=model)
start_time = time.time()
response = await client.chat.completions.create(**params)
elapsed = time.time() - start_time
if is_verbose():
- log("LLM", f"Tool call response in {elapsed:.1f}s")
+ log("LLM", f"Tool call response in {elapsed:.1f}s ({request_id})")
choice = response.choices[0] if response.choices else None
if not choice:
+ self._set_last_failure_metadata(
+ {
+ "failure_stage": "chat_with_tools",
+ "failure_type": "no_choices",
+ "model": model,
+ "base_url": base_url,
+ "request_id": request_id,
+ "attempt": 1,
+ }
+ )
raise ValueError("LLM returned no choices")
- content = choice.message.content or ""
+ content = self._extract_visible_content(choice.message)
parsed_tool_calls = []
-
if choice.message.tool_calls:
for tc in choice.message.tool_calls:
- parsed_tool_calls.append(ToolCall(
- id=tc.id,
- function={"name": tc.function.name, "arguments": tc.function.arguments},
- ))
+ parsed_tool_calls.append(
+ ToolCall(
+ id=tc.id,
+ function={"name": tc.function.name, "arguments": tc.function.arguments},
+ )
+ )
- usage = response.usage.model_dump() if response.usage else None
+ usage = self._sanitize_usage(response.usage.model_dump() if response.usage else None)
if not content and not parsed_tool_calls:
+ self._set_last_failure_metadata(
+ {
+ "failure_stage": "chat_with_tools",
+ "failure_type": "empty_response",
+ "model": model,
+ "base_url": base_url,
+ "request_id": getattr(response, "id", request_id),
+ "finish_reason": getattr(choice, "finish_reason", None),
+ "usage": usage,
+ "choice_message_preview": {
+ "content": content[:200],
+ "tool_call_count": len(parsed_tool_calls),
+ },
+ "attempt": 1,
+ }
+ )
raise ValueError("LLM returned empty response (no content, no tool_calls)")
- return LLMResponse(content=content.strip(), tool_calls=parsed_tool_calls, usage=usage)
+ return LLMResponse(
+ content=content.strip(),
+ tool_calls=parsed_tool_calls,
+ usage=usage,
+ request_id=getattr(response, "id", request_id),
+ )
finally:
await client.close()
async def _make_request(
self,
+ base_url: str,
+ api_key: str,
messages: list,
model: str,
temperature: float,
seed: Optional[int],
timeout_s: int,
+ request_id: str,
+ max_completion_tokens_override: Optional[int] = None,
) -> Tuple[str, Optional[dict]]:
- """Make a single API request with streaming"""
- # Use longer timeouts for connection and read
timeout_config = httpx.Timeout(
- connect=30.0, # Connection timeout
- read=timeout_s, # Read timeout (for streaming)
- write=30.0, # Write timeout
- pool=30.0, # Pool timeout
+ connect=30.0,
+ read=timeout_s,
+ write=30.0,
+ pool=30.0,
)
+ http_client = self._build_httpx_client(base_url=base_url, timeout=timeout_config)
client = openai.AsyncOpenAI(
- base_url=self._base_url,
- api_key=self._api_key,
+ base_url=base_url,
+ api_key=api_key,
timeout=timeout_config,
- max_retries=0, # We handle retries ourselves
+ max_retries=0,
+ http_client=http_client,
)
try:
- # Build request parameters
params = {
"model": model,
"messages": messages,
@@ -371,41 +1322,59 @@ async def _make_request(
"stream": True,
"stream_options": {"include_usage": True},
}
-
if seed is not None:
params["seed"] = seed
+ effective_max_completion_tokens = (
+ max_completion_tokens_override
+ if max_completion_tokens_override is not None
+ else self._max_completion_tokens
+ )
+ if effective_max_completion_tokens is not None:
+ # Pass both names for compatibility across OpenAI-compatible servers.
+ params["max_tokens"] = effective_max_completion_tokens
+ params["max_completion_tokens"] = effective_max_completion_tokens
+ params["extra_body"] = {"request_id": request_id}
+ self._apply_reasoning_controls(params, base_url=base_url, model=model)
- # Make streaming request
start_time = time.time()
stream = await client.chat.completions.create(**params)
- # Collect streamed content and usage
content_parts = []
usage = None
chunk_count = 0
- last_progress = 0
+ last_progress = 0.0
async for chunk in stream:
chunk_count += 1
-
- # Safety limit on chunks
if chunk_count > self.MAX_CHUNKS:
- log("LLM", f"Chunk limit exceeded ({self.MAX_CHUNKS}), truncating response")
+ log("LLM", f"Chunk limit exceeded ({self.MAX_CHUNKS}), aborting {request_id}")
+ await self._abort_request(
+ _ServerLease(
+ server_id="stream-local",
+ base_url=base_url,
+ api_key=api_key,
+ started_at=start_time,
+ ),
+ request_id,
+ f"chunk limit exceeded ({self.MAX_CHUNKS})",
+ )
break
- if chunk.choices and chunk.choices[0].delta.content:
- content_parts.append(chunk.choices[0].delta.content)
+ if chunk.choices:
+ delta = chunk.choices[0].delta
+ delta_content = self._extract_visible_content(delta)
+ if delta_content:
+ content_parts.append(delta_content)
if chunk.usage:
- usage = chunk.usage.model_dump()
+ usage = self._sanitize_usage(chunk.usage.model_dump())
- # Update progress every second
elapsed = time.time() - start_time
if is_verbose() and elapsed - last_progress >= 1.0:
last_progress = elapsed
progress("LLM", elapsed, timeout_s, f"chunks:{chunk_count}")
if is_verbose() and last_progress > 0:
- progress_done("LLM", f"Done in {time.time()-start_time:.1f}s, {chunk_count} chunks")
+ progress_done("LLM", f"Done in {time.time() - start_time:.1f}s, {chunk_count} chunks")
content = "".join(content_parts)
if not content:
@@ -413,14 +1382,11 @@ async def _make_request(
return content.strip(), usage
finally:
- # CRITICAL: Close the client to release HTTP connections
- # Without this, connections accumulate and eventually exhaust the pool
await client.close()
async def _backoff(self, attempt: int):
- """Exponential backoff with jitter"""
delay = min(
self.BASE_DELAY * (2 ** attempt) + random.uniform(0, 1),
- self.MAX_DELAY
+ self.MAX_DELAY,
)
await asyncio.sleep(delay)
diff --git a/scripts/batch_eval.py b/scripts/batch_eval.py
new file mode 100644
index 0000000..9de16f2
--- /dev/null
+++ b/scripts/batch_eval.py
@@ -0,0 +1,361 @@
+#!/usr/bin/env python3
+"""High-throughput batch evaluation for LiveWeb Arena."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import os
+import random
+import traceback
+import sys
+from collections import Counter, defaultdict
+from datetime import datetime
+from pathlib import Path
+from statistics import mean
+from typing import Any
+
+from dotenv import load_dotenv
+
+load_dotenv(override=True)
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+if str(PROJECT_ROOT) not in sys.path:
+ sys.path.insert(0, str(PROJECT_ROOT))
+
+from env import Actor
+from liveweb_arena.core.task_registry import TaskRegistry, parse_task_id
+from liveweb_arena.utils.llm_client import MultiServerLLMRouter
+
+
+def _build_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(description="Batch evaluation for LiveWeb Arena")
+ parser.add_argument("--model", type=str, required=True, help="LLM model name")
+ parser.add_argument("--base-url", type=str, default=None, help="OpenAI-compatible API base URL")
+ parser.add_argument(
+ "--server-pool-file",
+ type=str,
+ default=None,
+ help="Optional server_pool.json for multi-server routing; overrides --base-url for request routing",
+ )
+ parser.add_argument("--api-key", type=str, default=None, help="API key (default: API_KEY env var)")
+ parser.add_argument("--num-prompts", type=int, default=200, help="Number of prompts/tasks to evaluate")
+ parser.add_argument("--seed", type=int, default=42, help="Seed used to sample deterministic task ids")
+ parser.add_argument("--task-ids-file", type=str, default=None, help="Optional newline-delimited task id file")
+ parser.add_argument("--output-dir", type=str, default=str(PROJECT_ROOT / "eval"), help="Output directory")
+ parser.add_argument("--output-prefix", type=str, default="batch_eval", help="Output file prefix")
+ parser.add_argument("--max-concurrency", type=int, default=32, help="Max concurrent evaluations")
+ parser.add_argument("--timeout", type=int, default=1800, help="Per-task timeout in seconds")
+ parser.add_argument("--temperature", type=float, default=0.0, help="LLM temperature")
+ parser.add_argument("--max-steps", type=int, default=None, help="Per-task max browser steps")
+ parser.add_argument("--registry-version", type=str, default=None, help="Override TASK_REGISTRY_VERSION for sampling")
+ parser.add_argument(
+ "--exclude-plugins",
+ type=str,
+ default="",
+ help="Comma separated plugin names to exclude from sampled tasks",
+ )
+ parser.add_argument(
+ "--include-plugins",
+ type=str,
+ default="",
+ help="Comma separated plugin names to keep; empty means all",
+ )
+ parser.add_argument(
+ "--min-unique-plugins",
+ type=int,
+ default=1,
+ help="Minimum number of unique plugins a sampled task must contain",
+ )
+ parser.add_argument("--verbose", action="store_true", help="Print per-task progress")
+ parser.add_argument(
+ "--flush-every",
+ type=int,
+ default=10,
+ help="Write partial results/summary every N completed tasks",
+ )
+ return parser
+
+
+def _load_task_ids(args: argparse.Namespace) -> list[int]:
+ if args.task_ids_file:
+ task_ids = [
+ int(line.strip())
+ for line in Path(args.task_ids_file).read_text().splitlines()
+ if line.strip()
+ ]
+ return task_ids[: args.num_prompts]
+
+ include_plugins = {p.strip() for p in args.include_plugins.split(",") if p.strip()}
+ exclude_plugins = {p.strip() for p in args.exclude_plugins.split(",") if p.strip()}
+ rng = random.Random(args.seed)
+
+ candidate_combo_indices: list[int] = []
+ for combo_index, template_ids in enumerate(TaskRegistry._combinations):
+ plugins = {TaskRegistry.TEMPLATES[tid][0] for tid in template_ids}
+ if len(plugins) < args.min_unique_plugins:
+ continue
+ if include_plugins and not (plugins & include_plugins):
+ continue
+ if exclude_plugins and (plugins & exclude_plugins):
+ continue
+ candidate_combo_indices.append(combo_index)
+
+ rng.shuffle(candidate_combo_indices)
+ if not candidate_combo_indices:
+ return []
+
+ selected: list[int] = []
+ used_task_ids: set[int] = set()
+ round_index = 0
+ while len(selected) < args.num_prompts:
+ added_in_round = 0
+ for combo_index in candidate_combo_indices:
+ variation_seed = (
+ args.seed + combo_index + round_index * 9973
+ ) % TaskRegistry.TASK_IDS_PER_COMBO
+ task_id = combo_index * TaskRegistry.TASK_IDS_PER_COMBO + variation_seed + 1
+ if task_id in used_task_ids:
+ continue
+ selected.append(task_id)
+ used_task_ids.add(task_id)
+ added_in_round += 1
+ if len(selected) >= args.num_prompts:
+ break
+ if added_in_round == 0:
+ break
+ round_index += 1
+ return selected
+
+
+async def _run_one(
+ actor: Actor,
+ task_id: int,
+ args: argparse.Namespace,
+ max_concurrency: int,
+) -> dict[str, Any]:
+ result = await actor.evaluate(
+ model=args.model,
+ base_url=args.base_url,
+ api_key=args.api_key,
+ task_id=task_id,
+ timeout=args.timeout,
+ temperature=args.temperature,
+ max_steps=args.max_steps,
+ max_concurrency=max_concurrency,
+ mode="eval",
+ route_key=f"batch-eval:{args.seed}:task:{task_id}",
+ )
+ result.setdefault("extra", {})
+ result["extra"]["task_id"] = task_id
+ return result
+
+
+def _summarize(results: list[dict[str, Any]]) -> dict[str, Any]:
+ scores = [float(r.get("score", 0.0)) for r in results]
+ successes = [bool(r.get("success", False)) for r in results]
+ errored = [r for r in results if r.get("error")]
+ by_plugin_scores: dict[str, list[float]] = defaultdict(list)
+ by_plugin_success: dict[str, list[int]] = defaultdict(list)
+ by_plugin_errors: Counter[str] = Counter()
+
+ for result in results:
+ cfg = parse_task_id(int(result.get("extra", {}).get("task_id")))
+ plugins = sorted({plugin for plugin, _name in cfg["templates"]})
+ plugin_key = "+".join(plugins)
+ by_plugin_scores[plugin_key].append(float(result.get("score", 0.0)))
+ by_plugin_success[plugin_key].append(1 if result.get("success", False) else 0)
+ if result.get("error"):
+ by_plugin_errors[plugin_key] += 1
+
+ return {
+ "num_tasks": len(results),
+ "mean_score": mean(scores) if scores else 0.0,
+ "success_rate": (sum(successes) / len(successes)) if successes else 0.0,
+ "error_rate": (len(errored) / len(results)) if results else 0.0,
+ "num_errors": len(errored),
+ "by_plugin": {
+ plugin: {
+ "count": len(vals),
+ "mean_score": mean(vals) if vals else 0.0,
+ "success_rate": (sum(by_plugin_success[plugin]) / len(by_plugin_success[plugin]))
+ if by_plugin_success[plugin]
+ else 0.0,
+ "num_errors": by_plugin_errors[plugin],
+ }
+ for plugin, vals in sorted(by_plugin_scores.items())
+ },
+ }
+
+
+def _write_results_jsonl(path: Path, results: list[dict[str, Any]]) -> None:
+ with path.open("w", encoding="utf-8") as f:
+ for result in results:
+ f.write(json.dumps(result, ensure_ascii=False) + "\n")
+
+
+def _build_summary(
+ results: list[dict[str, Any]],
+ *,
+ task_ids_path: Path,
+ result_path: Path,
+ args: argparse.Namespace,
+ status: str,
+ error_path: Path | None = None,
+) -> dict[str, Any]:
+ summary = _summarize(results)
+ summary["status"] = status
+ summary["task_ids_path"] = str(task_ids_path)
+ summary["results_path"] = str(result_path)
+ summary["model"] = args.model
+ summary["base_url"] = args.base_url
+ summary["server_pool_file"] = args.server_pool_file
+ summary["max_concurrency"] = args.max_concurrency
+ summary["registry_version"] = os.environ.get("TASK_REGISTRY_VERSION")
+ if error_path is not None:
+ summary["error_path"] = str(error_path)
+ return summary
+
+
+def _write_summary(
+ path: Path,
+ results: list[dict[str, Any]],
+ *,
+ task_ids_path: Path,
+ result_path: Path,
+ args: argparse.Namespace,
+ status: str,
+ error_path: Path | None = None,
+) -> dict[str, Any]:
+ summary = _build_summary(
+ results,
+ task_ids_path=task_ids_path,
+ result_path=result_path,
+ args=args,
+ status=status,
+ error_path=error_path,
+ )
+ path.write_text(json.dumps(summary, indent=2, ensure_ascii=False))
+ return summary
+
+
+async def _maybe_cleanup_actor(actor: Actor) -> None:
+ cleanup = getattr(actor, "cleanup", None)
+ if cleanup is not None:
+ maybe_awaitable = cleanup()
+ if asyncio.iscoroutine(maybe_awaitable):
+ await maybe_awaitable
+ return
+
+ browser = getattr(actor, "browser", None)
+ if browser is not None and hasattr(browser, "close"):
+ maybe_awaitable = browser.close()
+ if asyncio.iscoroutine(maybe_awaitable):
+ await maybe_awaitable
+
+
+async def main() -> int:
+ args = _build_parser().parse_args()
+
+ if args.registry_version:
+ os.environ["TASK_REGISTRY_VERSION"] = args.registry_version
+
+ api_key = args.api_key or os.getenv("API_KEY") or os.getenv("CHUTES_API_KEY")
+ if not api_key:
+ print("API key required via --api-key or API_KEY/CHUTES_API_KEY", file=sys.stderr)
+ return 1
+ args.api_key = api_key
+ if not args.base_url and not args.server_pool_file:
+ print("Either --base-url or --server-pool-file is required.", file=sys.stderr)
+ return 1
+
+ task_ids = _load_task_ids(args)
+ if not task_ids:
+ print("No task ids selected for evaluation.", file=sys.stderr)
+ return 1
+
+ output_dir = Path(args.output_dir)
+ output_dir.mkdir(parents=True, exist_ok=True)
+ timestamp = datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
+ result_path = output_dir / f"{args.output_prefix}_{timestamp}.jsonl"
+ summary_path = output_dir / f"{args.output_prefix}_{timestamp}.summary.json"
+ task_ids_path = output_dir / f"{args.output_prefix}_{timestamp}.task_ids.txt"
+ task_ids_path.write_text("\n".join(str(tid) for tid in task_ids) + "\n")
+
+ print(f"Evaluating {len(task_ids)} tasks with concurrency={args.max_concurrency}")
+ print(f"Task IDs saved to: {task_ids_path}")
+
+ llm_router = None
+ if args.server_pool_file:
+ llm_router = MultiServerLLMRouter.from_server_pool_file(
+ args.server_pool_file,
+ default_api_key=api_key,
+ max_inflight_requests=args.max_concurrency,
+ )
+ actor = Actor(api_key=api_key, use_cache=True, llm_router=llm_router)
+ semaphore = asyncio.Semaphore(args.max_concurrency)
+ results: list[dict[str, Any]] = []
+ uncaught_error: Exception | None = None
+ error_path = output_dir / f"{args.output_prefix}_{timestamp}.error.txt"
+
+ async def guarded(task_id: int) -> dict[str, Any]:
+ async with semaphore:
+ result = await _run_one(actor, task_id, args, args.max_concurrency)
+ if args.verbose:
+ print(
+ f"task_id={task_id} score={result.get('score', 0.0):.3f} "
+ f"success={result.get('success', False)} error={bool(result.get('error'))}"
+ )
+ return result
+
+ try:
+ for coro in asyncio.as_completed([guarded(task_id) for task_id in task_ids]):
+ results.append(await coro)
+ if len(results) % args.flush_every == 0 or len(results) == len(task_ids):
+ _write_results_jsonl(result_path, results)
+ _write_summary(
+ summary_path,
+ results,
+ task_ids_path=task_ids_path,
+ result_path=result_path,
+ args=args,
+ status="running" if len(results) < len(task_ids) else "completed",
+ )
+ print(f"Completed {len(results)}/{len(task_ids)} tasks")
+ except Exception as exc:
+ uncaught_error = exc
+ error_path.write_text(traceback.format_exc())
+ finally:
+ try:
+ await _maybe_cleanup_actor(actor)
+ except Exception:
+ cleanup_trace = traceback.format_exc()
+ if error_path.exists():
+ error_path.write_text(error_path.read_text() + "\n\nCleanup error:\n" + cleanup_trace)
+ else:
+ error_path.write_text("Cleanup error:\n" + cleanup_trace)
+
+ _write_results_jsonl(result_path, results)
+ final_status = "failed" if uncaught_error is not None else "completed"
+ summary = _write_summary(
+ summary_path,
+ results,
+ task_ids_path=task_ids_path,
+ result_path=result_path,
+ args=args,
+ status=final_status,
+ error_path=error_path if error_path.exists() else None,
+ )
+
+ print(json.dumps(summary, indent=2, ensure_ascii=False))
+ print(f"Results saved to: {result_path}")
+ print(f"Summary saved to: {summary_path}")
+ if uncaught_error is not None:
+ print(f"Error details saved to: {error_path}", file=sys.stderr)
+ raise uncaught_error
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(asyncio.run(main()))
diff --git a/scripts/reachability_audit_report.py b/scripts/reachability_audit_report.py
new file mode 100644
index 0000000..c49ceb6
--- /dev/null
+++ b/scripts/reachability_audit_report.py
@@ -0,0 +1,321 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from collections import Counter
+from dataclasses import asdict
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlparse
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+if str(PROJECT_ROOT) not in sys.path:
+ sys.path.insert(0, str(PROJECT_ROOT))
+
+from liveweb_arena.core.reachability_audit import ReachabilityAuditResult, classify_model_hallucination
+from liveweb_arena.core.site_probe import probe_site
+from liveweb_arena.plugins import get_all_plugins
+
+
+SITE_UNREACHABLE_RE = re.compile(r"Required site unreachable:\s+(https?://\S+)")
+DOMAIN_UNREACHABLE_RE = re.compile(r"Required domain unreachable:\s+(https?://\S+)")
+REACHABILITY_AUDIT_RE = re.compile(r"\[ReachabilityAudit\]\s+(\{.*\})")
+
+
+def _build_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(description="Audit unreachable URL causes from LiveWeb logs")
+ parser.add_argument("--log-file", type=str, required=True, help="Driver log file to analyze")
+ parser.add_argument(
+ "--max-unique-urls",
+ type=int,
+ default=200,
+ help="Maximum number of unique URLs to probe",
+ )
+ parser.add_argument(
+ "--output-json",
+ type=str,
+ default="",
+ help="Optional path to write full JSON report",
+ )
+ parser.add_argument(
+ "--show-top-urls",
+ type=int,
+ default=20,
+ help="How many top URLs to show in the terminal summary",
+ )
+ parser.add_argument(
+ "--probe-timeout",
+ type=float,
+ default=3.0,
+ help="Timeout in seconds for each direct site probe",
+ )
+ return parser
+
+
+def _extract_unreachable_urls(log_path: Path) -> Counter[str]:
+ counts: Counter[str] = Counter()
+ with log_path.open("r", encoding="utf-8", errors="ignore") as handle:
+ for line in handle:
+ match = SITE_UNREACHABLE_RE.search(line) or DOMAIN_UNREACHABLE_RE.search(line)
+ if match:
+ counts[match.group(1)] += 1
+ return counts
+
+
+def _extract_reachability_audits(log_path: Path) -> list[dict[str, Any]]:
+ audits: list[dict[str, Any]] = []
+ with log_path.open("r", encoding="utf-8", errors="ignore") as handle:
+ for line in handle:
+ match = REACHABILITY_AUDIT_RE.search(line)
+ if not match:
+ continue
+ try:
+ audits.append(json.loads(match.group(1)))
+ except Exception:
+ continue
+ return audits
+
+
+def _infer_plugin(url: str) -> tuple[str | None, Any | None]:
+ host = (urlparse(url).hostname or "").lower()
+ candidates: list[tuple[int, int, str, Any]] = []
+ for name, plugin_cls in get_all_plugins().items():
+ for allowed in getattr(plugin_cls, "allowed_domains", []):
+ allowed = allowed.lower()
+ if host == allowed or host.endswith("." + allowed) or allowed.endswith("." + host):
+ # Prefer exact/specific domains first, and keep hybrid last.
+ hybrid_penalty = 1 if name == "hybrid" else 0
+ candidates.append((hybrid_penalty, -len(allowed), name, plugin_cls))
+ if candidates:
+ _penalty, _neg_len, name, plugin_cls = sorted(candidates)[0]
+ try:
+ return name, plugin_cls()
+ except Exception:
+ return name, None
+ return None, None
+
+
+def _classify_from_probe(
+ url: str,
+ plugin_name: str | None,
+ plugin: Any | None,
+ *,
+ probe_timeout: float,
+) -> ReachabilityAuditResult:
+ model_class = None
+ if plugin is not None and hasattr(plugin, "classify_url"):
+ model_class = plugin.classify_url(url)
+ model_class = model_class or classify_model_hallucination(url)
+ if model_class is not None:
+ is_env = model_class.startswith("env_") or model_class.startswith("ambiguous_")
+ return ReachabilityAuditResult(
+ status="unreachable",
+ classification=model_class,
+ layer="model" if not is_env else "tls",
+ url=url,
+ normalized_url=url,
+ domain=(urlparse(url).hostname or "").lower(),
+ plugin_name=plugin_name,
+ reason="plugin/url-shape classification",
+ is_environment_failure=is_env,
+ is_model_hallucination=not is_env,
+ evidence={},
+ )
+
+ probe = probe_site(url, timeout=probe_timeout)
+ evidence = {"site_probe": asdict(probe)}
+ domain = (urlparse(url).hostname or "").lower()
+
+ if probe.ok and (probe.http_status or 0) < 400:
+ classification = "env_browser_navigation_failure"
+ layer = "browser"
+ env_failure = True
+ model_hallucination = False
+ elif probe.exception_type == "SSLError":
+ classification = "env_tls_error"
+ layer = "tls"
+ env_failure = True
+ model_hallucination = False
+ elif probe.exception_type in {"ConnectTimeout", "ReadTimeout", "Timeout"}:
+ classification = "env_prefetch_timeout"
+ layer = "browser"
+ env_failure = True
+ model_hallucination = False
+ elif probe.exception_type in {"ConnectionError"}:
+ classification = "env_dns_or_connect_error"
+ layer = "browser"
+ env_failure = True
+ model_hallucination = False
+ elif probe.http_status == 403 and ("coingecko" in domain or probe.cf_ray or (probe.server or "").lower() == "cloudflare"):
+ classification = "env_cdn_blocked"
+ layer = "cdn"
+ env_failure = True
+ model_hallucination = False
+ elif probe.http_status is not None and 400 <= probe.http_status < 500:
+ classification = "env_http_4xx"
+ layer = "cdn"
+ env_failure = True
+ model_hallucination = False
+ elif probe.http_status is not None and probe.http_status >= 500:
+ classification = "env_http_5xx"
+ layer = "cdn"
+ env_failure = True
+ model_hallucination = False
+ else:
+ classification = "ambiguous_navigation_failure"
+ layer = "browser"
+ env_failure = True
+ model_hallucination = False
+
+ return ReachabilityAuditResult(
+ status="unreachable",
+ classification=classification,
+ layer=layer,
+ url=url,
+ normalized_url=url,
+ domain=domain,
+ plugin_name=plugin_name,
+ reason=probe.reason,
+ http_status=probe.http_status,
+ exception_type=probe.exception_type,
+ is_environment_failure=env_failure,
+ is_model_hallucination=model_hallucination,
+ evidence=evidence,
+ )
+
+
+def _weighted_ratio(counter: Counter[str], total: int) -> dict[str, float]:
+ if total <= 0:
+ return {}
+ return {key: value / total for key, value in counter.most_common()}
+
+
+def _print_summary(report: dict[str, Any], show_top_urls: int) -> None:
+ print("\n**Reachability Audit Summary**")
+ print(f"log_file: {report['log_file']}")
+ print(f"total_unreachable_events: {report['total_unreachable_events']}")
+ print(f"unique_unreachable_urls: {report['unique_unreachable_urls']}")
+ print(f"environment_failure_ratio: {report['environment_failure_ratio']:.4f}")
+ print(f"model_hallucination_ratio: {report['model_hallucination_ratio']:.4f}")
+
+ print("\nTop classifications:")
+ for classification, ratio in report["classification_ratios"].items():
+ count = report["classification_counts"][classification]
+ print(f" {classification}: {count} ({ratio:.4f})")
+
+ print("\nEnvironment-only details:")
+ for classification, ratio in report["environment_detail_ratios"].items():
+ count = report["environment_detail_counts"][classification]
+ print(f" {classification}: {count} ({ratio:.4f})")
+
+ if report.get("page_kind_counts"):
+ print("\nPage kind breakdown:")
+ for key, count in report["page_kind_counts"].items():
+ print(f" {key}: {count}")
+
+ if report.get("prefetch_phase_counts"):
+ print("\nPrefetch phase breakdown:")
+ for key, count in report["prefetch_phase_counts"].items():
+ print(f" {key}: {count}")
+
+ if report.get("audit_classification_counts"):
+ print("\nStructured audit classifications:")
+ for key, count in report["audit_classification_counts"].items():
+ print(f" {key}: {count}")
+
+ if report.get("audit_domain_counts"):
+ print("\nStructured audit domains:")
+ for key, count in report["audit_domain_counts"].items():
+ print(f" {key}: {count}")
+
+ print("\nTop unreachable URLs:")
+ for item in report["top_urls"][:show_top_urls]:
+ print(
+ " "
+ f"{item['count']:>3}x {item['url']} -> {item['classification']} "
+ f"(plugin={item['plugin_name']}, layer={item['layer']})"
+ )
+
+
+def main() -> int:
+ args = _build_parser().parse_args()
+ log_path = Path(args.log_file)
+ url_counts = _extract_unreachable_urls(log_path)
+ audit_rows = _extract_reachability_audits(log_path)
+
+ most_common_urls = url_counts.most_common(args.max_unique_urls)
+ total_events = sum(url_counts.values())
+
+ classification_counts: Counter[str] = Counter()
+ env_detail_counts: Counter[str] = Counter()
+ plugin_counts: Counter[str] = Counter()
+ page_kind_counts: Counter[str] = Counter()
+ prefetch_phase_counts: Counter[str] = Counter()
+ audit_classification_counts: Counter[str] = Counter()
+ audit_domain_counts: Counter[str] = Counter()
+ env_failures = 0
+ model_hallucinations = 0
+ rows: list[dict[str, Any]] = []
+
+ for audit in audit_rows:
+ classification = str(audit.get("classification") or "")
+ domain = str(audit.get("domain") or "")
+ evidence = dict(audit.get("evidence") or {})
+ page_kind = evidence.get("page_kind")
+ prefetch_phase = evidence.get("prefetch_phase")
+ if classification:
+ audit_classification_counts[classification] += 1
+ if domain:
+ audit_domain_counts[domain] += 1
+ if page_kind:
+ page_kind_counts[str(page_kind)] += 1
+ if prefetch_phase:
+ prefetch_phase_counts[str(prefetch_phase)] += 1
+
+ for url, count in most_common_urls:
+ plugin_name, plugin = _infer_plugin(url)
+ audit = _classify_from_probe(url, plugin_name, plugin, probe_timeout=args.probe_timeout)
+ classification_counts[audit.classification] += count
+ if audit.is_environment_failure:
+ env_detail_counts[audit.classification] += count
+ env_failures += count
+ if audit.is_model_hallucination:
+ model_hallucinations += count
+ if plugin_name:
+ plugin_counts[plugin_name] += count
+ row = audit.to_dict()
+ row["count"] = count
+ rows.append(row)
+
+ report = {
+ "log_file": str(log_path),
+ "total_unreachable_events": total_events,
+ "unique_unreachable_urls": len(url_counts),
+ "audited_unique_urls": len(rows),
+ "environment_failure_ratio": (env_failures / total_events) if total_events else 0.0,
+ "model_hallucination_ratio": (model_hallucinations / total_events) if total_events else 0.0,
+ "classification_counts": dict(classification_counts),
+ "classification_ratios": _weighted_ratio(classification_counts, total_events),
+ "environment_detail_counts": dict(env_detail_counts),
+ "environment_detail_ratios": _weighted_ratio(env_detail_counts, total_events),
+ "audit_classification_counts": dict(audit_classification_counts),
+ "audit_domain_counts": dict(audit_domain_counts),
+ "page_kind_counts": dict(page_kind_counts),
+ "prefetch_phase_counts": dict(prefetch_phase_counts),
+ "plugin_counts": dict(plugin_counts),
+ "top_urls": sorted(rows, key=lambda item: item["count"], reverse=True),
+ }
+
+ if args.output_json:
+ output_path = Path(args.output_json)
+ output_path.write_text(json.dumps(report, indent=2, ensure_ascii=False))
+
+ _print_summary(report, args.show_top_urls)
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
index 0000000..74bbaed
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,59 @@
+import sys
+import types
+import os
+
+USE_REAL_BROWSER_SMOKE = os.getenv("LIVEWEB_REAL_BROWSER_SMOKE") == "1"
+
+if not USE_REAL_BROWSER_SMOKE:
+ playwright_module = types.ModuleType("playwright")
+ async_api_module = types.ModuleType("playwright.async_api")
+
+
+ async def _async_playwright():
+ raise RuntimeError("playwright is stubbed for unit tests")
+
+
+ async_api_module.async_playwright = _async_playwright
+ async_api_module.Browser = object
+ async_api_module.BrowserContext = object
+ async_api_module.Page = object
+ async_api_module.Playwright = object
+
+ playwright_module.async_api = async_api_module
+
+ sys.modules.setdefault("playwright", playwright_module)
+ sys.modules.setdefault("playwright.async_api", async_api_module)
+
+
+openai_module = types.ModuleType("openai")
+
+
+class _OpenAIError(Exception):
+ pass
+
+
+class RateLimitError(_OpenAIError):
+ pass
+
+
+class BadRequestError(_OpenAIError):
+ pass
+
+
+class APIStatusError(_OpenAIError):
+ def __init__(self, status_code=500, *args, **kwargs):
+ super().__init__(*args)
+ self.status_code = status_code
+
+
+class AsyncOpenAI:
+ def __init__(self, *args, **kwargs):
+ raise RuntimeError("openai.AsyncOpenAI must be monkeypatched in tests")
+
+
+openai_module.RateLimitError = RateLimitError
+openai_module.BadRequestError = BadRequestError
+openai_module.APIStatusError = APIStatusError
+openai_module.AsyncOpenAI = AsyncOpenAI
+
+sys.modules.setdefault("openai", openai_module)
diff --git a/tests/core/test_actor_session_setup.py b/tests/core/test_actor_session_setup.py
new file mode 100644
index 0000000..3ae6698
--- /dev/null
+++ b/tests/core/test_actor_session_setup.py
@@ -0,0 +1,179 @@
+import sys
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[2]
+if str(ROOT) not in sys.path:
+ sys.path.insert(0, str(ROOT))
+
+from env import Actor, _maybe_promote_disallowed_domain_failure
+from liveweb_arena.core.task_registry_loader import parse_task_id as parse_task_id_for_runtime
+
+
+class _FakePlugin:
+ allowed_domains = ["stooq.com"]
+
+ def is_url_allowed(self, url: str) -> bool:
+ return "stooq.com" in url
+
+
+class _FakeSession:
+ def __init__(self, *, fail_once: bool = False):
+ self._fail_once = fail_once
+ self.closed = False
+
+ async def set_cache_interceptor(self, interceptor):
+ if self._fail_once:
+ self._fail_once = False
+ raise RuntimeError("Target page, context or browser has been closed")
+ return None
+
+ async def close(self):
+ self.closed = True
+
+
+class _FakeBrowser:
+ def __init__(self):
+ self.calls = 0
+ self.sessions = [_FakeSession(fail_once=True), _FakeSession(fail_once=False)]
+ self.ensure_calls = 0
+
+ async def ensure_healthy(self):
+ self.ensure_calls += 1
+
+ async def new_session(self):
+ session = self.sessions[self.calls]
+ self.calls += 1
+ return session
+
+
+class _FakeInterceptor:
+ def __init__(self, metadata):
+ self._metadata = metadata
+
+ def get_last_blocked_document_metadata(self):
+ return dict(self._metadata)
+
+
+@pytest.mark.anyio
+async def test_setup_interceptor_rebuilds_session_after_transport_error(tmp_path):
+ actor = Actor(api_key="local", cache_dir=tmp_path, use_cache=True, llm_router=None)
+ actor.browser = _FakeBrowser()
+
+ session, interceptor = await actor._setup_interceptor(
+ session=await actor.browser.new_session(),
+ cached_pages={},
+ allowed_domains={"stooq.com"},
+ blocked_patterns=[],
+ plugins_used={"stooq": _FakePlugin()},
+ )
+
+ assert interceptor is not None
+ assert actor.browser.calls == 2
+ assert actor.browser.ensure_calls == 1
+ assert session is actor.browser.sessions[1]
+ assert actor.browser.sessions[0].closed is True
+
+
+def test_actor_build_protocol_defaults_to_eval_mode(tmp_path):
+ actor = Actor(
+ api_key="local",
+ cache_dir=tmp_path,
+ use_cache=True,
+ llm_router=None,
+ llm_prompt_profile="gpt54_strict_domains",
+ )
+
+ assert actor.runtime_profile == "strict_eval"
+
+ eval_prompt = actor._build_protocol(runtime_profile="strict_eval").build_step_prompt(
+ obs=type("Obs", (), {"url": "https://example.com", "title": "t", "accessibility_tree": "tree"})(),
+ trajectory=[],
+ current_step=1,
+ max_steps=5,
+ )
+ collect_prompt = actor._build_protocol(runtime_profile="fast_collect").build_step_prompt(
+ obs=type("Obs", (), {"url": "https://example.com", "title": "t", "accessibility_tree": "tree"})(),
+ trajectory=[],
+ current_step=1,
+ max_steps=5,
+ )
+
+ assert "Never call stop with {\"answers\": {}}." not in eval_prompt
+ assert "Never call stop with {\"answers\": {}}." in collect_prompt
+
+
+def test_maybe_promote_disallowed_domain_failure_only_in_collect_mode():
+ interceptor = _FakeInterceptor(
+ {
+ "blocked_url": "https://finance.yahoo.com/quote/V/",
+ "blocked_domain": "finance.yahoo.com",
+ "allowed_domains": ["coingecko.com"],
+ }
+ )
+ trajectory = [
+ type(
+ "Step",
+ (),
+ {
+ "observation": type(
+ "Obs",
+ (),
+ {
+ "url": "https://finance.yahoo.com/quote/V/",
+ "accessibility_tree": "Domain not allowed",
+ },
+ )()
+ },
+ )()
+ ]
+
+ eval_failure, eval_audit = _maybe_promote_disallowed_domain_failure(
+ interceptor=interceptor,
+ trajectory=trajectory,
+ allowed_domains={"coingecko.com"},
+ failure_reason="browser_error",
+ reachability_audit=None,
+ mode="eval",
+ )
+ collect_failure, collect_audit = _maybe_promote_disallowed_domain_failure(
+ interceptor=interceptor,
+ trajectory=trajectory,
+ allowed_domains={"coingecko.com"},
+ failure_reason="browser_error",
+ reachability_audit=None,
+ mode="collect",
+ )
+
+ assert eval_failure == "browser_error"
+ assert eval_audit is None
+ assert collect_failure == "disallowed_domain"
+ assert collect_audit is not None
+ assert collect_audit.classification == "model_disallowed_domain"
+
+
+def test_strict_eval_can_use_external_task_registry(monkeypatch, tmp_path):
+ registry_root = tmp_path / "upstream_like"
+ registry_dir = registry_root / "liveweb_arena" / "core"
+ registry_dir.mkdir(parents=True)
+ (registry_dir / "task_registry.py").write_text(
+ "def parse_task_id(task_id):\n"
+ " return {\n"
+ " 'task_id': task_id,\n"
+ " 'combo_index': 0,\n"
+ " 'template_ids': (999,),\n"
+ " 'templates': [('coingecko', 'coingecko_price')],\n"
+ " 'variation_seed': 7,\n"
+ " 'num_tasks': 3,\n"
+ " }\n"
+ "\n"
+ "def max_task_id():\n"
+ " return 999999\n",
+ encoding="utf-8",
+ )
+ monkeypatch.setenv("LIVEWEB_STRICT_TASK_REGISTRY_DIR", str(registry_root))
+
+ parsed = parse_task_id_for_runtime(123, runtime_profile="strict_eval")
+ assert parsed["templates"] == [("coingecko", "coingecko_price")]
+ assert parsed["num_tasks"] == 3
diff --git a/tests/core/test_agent_loop_recovery.py b/tests/core/test_agent_loop_recovery.py
new file mode 100644
index 0000000..a10bff4
--- /dev/null
+++ b/tests/core/test_agent_loop_recovery.py
@@ -0,0 +1,1131 @@
+import pytest
+
+from liveweb_arena.core.agent_loop import AgentLoop, BrowserFatalError
+from liveweb_arena.core.agent_protocol import FunctionCallingProtocol
+from liveweb_arena.core.models import BrowserObservation, CompositeTask
+from liveweb_arena.utils.llm_client import LLMResponse, ToolCall
+
+
+class _FakeSession:
+ async def goto(self, url: str):
+ return BrowserObservation(url=url, title="Blank", accessibility_tree="root")
+
+ async def execute_action(self, action):
+ return BrowserObservation(url="https://example.com", title="Done", accessibility_tree="root")
+
+ def get_last_navigation_metadata(self):
+ return None
+
+
+class _FakeSessionWithBlockedSequence:
+ def __init__(self, observations, metadatas):
+ self._observations = list(observations)
+ self._metadatas = list(metadatas)
+ self._last_navigation_metadata = None
+
+ async def goto(self, url: str):
+ self._last_navigation_metadata = None
+ return BrowserObservation(url=url, title="Blank", accessibility_tree="root")
+
+ async def execute_action(self, action):
+ self._last_navigation_metadata = self._metadatas.pop(0)
+ return self._observations.pop(0)
+
+ def get_last_navigation_metadata(self):
+ return self._last_navigation_metadata
+
+ def clear_last_navigation_metadata(self):
+ self._last_navigation_metadata = None
+
+
+class _FakeSessionWithActionSequence:
+ def __init__(self, outcomes, metadatas):
+ self._outcomes = list(outcomes)
+ self._metadatas = list(metadatas)
+ self._last_navigation_metadata = None
+
+ async def goto(self, url: str):
+ self._last_navigation_metadata = None
+ return BrowserObservation(
+ url=url,
+ title="Blank",
+ accessibility_tree="root content with enough accessible text to avoid blank observation failfast",
+ )
+
+ async def execute_action(self, action):
+ self._last_navigation_metadata = self._metadatas.pop(0)
+ outcome = self._outcomes.pop(0)
+ if isinstance(outcome, BaseException):
+ raise outcome
+ return outcome
+
+ def get_last_navigation_metadata(self):
+ return self._last_navigation_metadata
+
+ def clear_last_navigation_metadata(self):
+ self._last_navigation_metadata = None
+
+
+class _FakeSubTask:
+ plugin_name = "coingecko"
+ answer_tag = "a1"
+ expected_steps = 1
+
+
+class _FakeSubTask2:
+ plugin_name = "taostats"
+ answer_tag = "a2"
+ expected_steps = 1
+
+
+class _FakeLLMClient:
+ def __init__(self, initial_response, recovery_responses=None):
+ self.initial_response = initial_response
+ self.recovery_responses = list(recovery_responses or [])
+ self.recovery_calls = 0
+ self.recovery_messages = []
+
+ async def chat_with_tools(self, **kwargs):
+ return self.initial_response
+
+ async def chat_with_tools_recovery(self, **kwargs):
+ self.recovery_calls += 1
+ self.recovery_messages.append(kwargs.get("messages"))
+ return self.recovery_responses.pop(0)
+
+
+class _QueuedLLMClient:
+ def __init__(self, responses, recovery_responses=None):
+ self._responses = list(responses)
+ self._recovery_responses = list(recovery_responses or [])
+ self.recovery_calls = 0
+ self.recovery_messages = []
+
+ async def chat_with_tools(self, **kwargs):
+ return self._responses.pop(0)
+
+ async def chat_with_tools_recovery(self, **kwargs):
+ self.recovery_calls += 1
+ self.recovery_messages.append(kwargs.get("messages"))
+ if not self._recovery_responses:
+ raise AssertionError("Recovery should not be called in this test")
+ return self._recovery_responses.pop(0)
+
+ def get_last_failure_metadata(self):
+ return {}
+
+
+def _task():
+ return CompositeTask(
+ subtasks=[_FakeSubTask()],
+ combined_intent="Find the answer",
+ plugin_hints={},
+ seed=1,
+ )
+
+
+def _task_two_answers():
+ return CompositeTask(
+ subtasks=[_FakeSubTask(), _FakeSubTask2()],
+ combined_intent="Find the answers",
+ plugin_hints={},
+ seed=1,
+ )
+
+
+@pytest.mark.anyio
+async def test_agent_loop_recovers_from_truncated_tool_json(monkeypatch):
+ monkeypatch.setenv("LIVEWEB_ENABLE_FORMAT_RECOVERY", "1")
+ monkeypatch.setenv("LIVEWEB_FORMAT_RECOVERY_MAX_RETRIES", "2")
+ monkeypatch.setenv("LIVEWEB_FORMAT_RECOVERY_MAX_NEW_TOKENS", "64")
+
+ llm_client = _FakeLLMClient(
+ initial_response=LLMResponse(content="{\"name\":\"goto\",\"arguments\":{\"url\":\"https://example.com\"}"),
+ recovery_responses=[
+ LLMResponse(tool_calls=[ToolCall(id="call_1", function={"name": "stop", "arguments": "{\"answers\":{\"a1\":\"42\"}}"})]),
+ ],
+ )
+ loop = AgentLoop(
+ session=_FakeSession(),
+ llm_client=llm_client,
+ protocol=FunctionCallingProtocol(),
+ max_steps=2,
+ )
+
+ trajectory, final_answer, usage = await loop.run(task=_task(), model="qwen", temperature=0.0, seed=1)
+
+ assert final_answer == {"answers": {"a1": "42"}}
+ assert loop.is_parse_failed() is False
+ stats = loop.get_format_recovery_stats()
+ assert stats["format_recovery_attempts"] == 1
+ assert stats["format_recovery_successes"] == 1
+ assert llm_client.recovery_calls == 1
+ assert trajectory[-1].action.action_type == "stop"
+ messages = llm_client.recovery_messages[0]
+ assistant_messages = [item for item in messages if item["role"] == "assistant"]
+ assert assistant_messages == []
+ assert messages[-1]["role"] == "user"
+ assert "invalid tool-call formatting" in messages[-1]["content"].lower()
+
+
+@pytest.mark.anyio
+async def test_agent_loop_does_not_recover_terminal_natural_language(monkeypatch):
+ monkeypatch.setenv("LIVEWEB_ENABLE_FORMAT_RECOVERY", "1")
+
+ llm_client = _FakeLLMClient(
+ initial_response=LLMResponse(content="I will inspect the page and then answer in plain English."),
+ recovery_responses=[],
+ )
+ loop = AgentLoop(
+ session=_FakeSession(),
+ llm_client=llm_client,
+ protocol=FunctionCallingProtocol(),
+ max_steps=2,
+ )
+
+ trajectory, final_answer, usage = await loop.run(task=_task(), model="qwen", temperature=0.0, seed=1)
+
+ assert final_answer is None
+ assert loop.is_parse_failed() is True
+ stats = loop.get_format_recovery_stats()
+ assert stats["format_recovery_attempts"] == 0
+ assert llm_client.recovery_calls == 0
+ assert trajectory[-1].action is None
+
+
+@pytest.mark.anyio
+async def test_agent_loop_recovers_terminal_natural_language_for_kimi_in_collect_mode(monkeypatch):
+ monkeypatch.setenv("LIVEWEB_NATURAL_LANGUAGE_PARSE_RECOVERY_MAX_RETRIES", "1")
+
+ llm_client = _FakeLLMClient(
+ initial_response=LLMResponse(content="I found the page and will answer next."),
+ recovery_responses=[
+ LLMResponse(
+ tool_calls=[
+ ToolCall(
+ id="call_1",
+ function={"name": "stop", "arguments": "{\"answers\":{\"a1\":\"42\"}}"},
+ )
+ ]
+ ),
+ ],
+ )
+ loop = AgentLoop(
+ session=_FakeSession(),
+ llm_client=llm_client,
+ protocol=FunctionCallingProtocol(),
+ max_steps=2,
+ behavior_mode="collect",
+ )
+
+ trajectory, final_answer, _usage = await loop.run(
+ task=_task(), model="moonshotai/kimi-k2.5", temperature=0.0, seed=1
+ )
+
+ assert final_answer == {"answers": {"a1": "42"}}
+ assert loop.is_parse_failed() is False
+ stats = loop.get_local_recovery_stats()
+ assert stats["natural_language_parse_recovery_attempts"] == 1
+ assert stats["natural_language_parse_recovery_successes"] == 1
+ assert llm_client.recovery_calls == 1
+ assert len(trajectory) == 1
+ assert trajectory[0].action.action_type == "stop"
+
+
+@pytest.mark.anyio
+async def test_agent_loop_marks_parse_failed_after_recovery_exhausted(monkeypatch):
+ monkeypatch.setenv("LIVEWEB_ENABLE_FORMAT_RECOVERY", "1")
+ monkeypatch.setenv("LIVEWEB_FORMAT_RECOVERY_MAX_RETRIES", "2")
+
+ llm_client = _FakeLLMClient(
+ initial_response=LLMResponse(content="{\"name\":\"goto\""),
+ recovery_responses=[
+ LLMResponse(content="{\"name\":\"goto\""),
+ LLMResponse(content="{\"name\":\"goto\""),
+ ],
+ )
+ loop = AgentLoop(
+ session=_FakeSession(),
+ llm_client=llm_client,
+ protocol=FunctionCallingProtocol(),
+ max_steps=2,
+ )
+
+ trajectory, final_answer, usage = await loop.run(task=_task(), model="qwen", temperature=0.0, seed=1)
+
+ assert final_answer is None
+ assert loop.is_parse_failed() is True
+ stats = loop.get_format_recovery_stats()
+ assert stats["format_recovery_attempts"] == 1
+ assert stats["format_recovery_successes"] == 0
+
+
+def test_trim_recovery_messages_truncates_large_recent_observation(monkeypatch):
+ monkeypatch.setenv("LIVEWEB_FORMAT_RECOVERY_CONTEXT_LENGTH", "1024")
+ monkeypatch.setenv("LIVEWEB_FORMAT_RECOVERY_TOKEN_MARGIN", "128")
+
+ loop = AgentLoop(
+ session=_FakeSession(),
+ llm_client=_FakeLLMClient(initial_response=LLMResponse(content="")),
+ protocol=FunctionCallingProtocol(),
+ max_steps=2,
+ )
+ giant_observation = "Observation:\n" + ("A" * 12000) + "\nTail marker"
+ messages = [
+ {"role": "system", "content": "system"},
+ {"role": "assistant", "content": "{\"name\":\"goto\"}"},
+ {"role": "user", "content": giant_observation},
+ {"role": "user", "content": "Emit exactly one valid tool call now."},
+ ]
+
+ trimmed, overflowed = loop._trim_recovery_messages(messages=messages, max_new_tokens=64)
+
+ assert overflowed is True
+ assert trimmed is not None
+ assert loop._estimate_message_tokens(trimmed) <= (1024 - 64 - 128)
+ assert trimmed[-2]["role"] == "user"
+ assert "Tail marker" in trimmed[-2]["content"]
+ assert len(trimmed[-2]["content"]) < len(giant_observation)
+ assert "[truncated for format recovery]" in trimmed[-2]["content"]
+
+
+@pytest.mark.anyio
+async def test_attempt_format_recovery_skips_when_budget_is_non_positive(monkeypatch):
+ monkeypatch.setenv("LIVEWEB_FORMAT_RECOVERY_CONTEXT_LENGTH", "128")
+ monkeypatch.setenv("LIVEWEB_FORMAT_RECOVERY_TOKEN_MARGIN", "128")
+
+ llm_client = _FakeLLMClient(
+ initial_response=LLMResponse(content="{\"name\":\"goto\""),
+ recovery_responses=[],
+ )
+ loop = AgentLoop(
+ session=_FakeSession(),
+ llm_client=llm_client,
+ protocol=FunctionCallingProtocol(),
+ max_steps=2,
+ )
+
+ raw_response, action, usage, failure_override = await loop._attempt_format_recovery(
+ model="qwen",
+ seed=1,
+ raw_response="{\"name\":\"goto\"",
+ messages=[
+ {"role": "system", "content": "system"},
+ {"role": "user", "content": "Find the answer"},
+ {"role": "user", "content": "Emit one tool call"},
+ ],
+ failure_class="recoverable_partial_json",
+ )
+
+ assert action is None
+ assert usage is None
+ assert failure_override == "recoverable_context_overflow"
+ stats = loop.get_local_recovery_stats()
+ assert llm_client.recovery_calls == 0
+ assert stats.get("format_recovery_exhausted", 0) == 0
+
+
+@pytest.mark.anyio
+async def test_agent_loop_limits_empty_recovery_retries(monkeypatch):
+ monkeypatch.setenv("LIVEWEB_ENABLE_FORMAT_RECOVERY", "1")
+ monkeypatch.setenv("LIVEWEB_FORMAT_RECOVERY_MAX_RETRIES", "4")
+ monkeypatch.setenv("LIVEWEB_FORMAT_RECOVERY_EMPTY_MAX_RETRIES", "2")
+
+ llm_client = _FakeLLMClient(
+ initial_response=LLMResponse(content=""),
+ recovery_responses=[
+ LLMResponse(content=""),
+ LLMResponse(content=""),
+ ],
+ )
+ loop = AgentLoop(
+ session=_FakeSession(),
+ llm_client=llm_client,
+ protocol=FunctionCallingProtocol(),
+ max_steps=2,
+ )
+
+ trajectory, final_answer, usage = await loop.run(task=_task(), model="qwen", temperature=0.0, seed=1)
+
+ assert final_answer is None
+ assert loop.is_parse_failed() is True
+ stats = loop.get_format_recovery_stats()
+ assert stats["format_recovery_attempts"] == 1
+ assert stats["format_recovery_exhausted"] == 1
+ assert llm_client.recovery_calls == 2
+
+
+@pytest.mark.anyio
+async def test_agent_loop_skips_recovery_when_context_budget_exceeded(monkeypatch):
+ monkeypatch.setenv("LIVEWEB_ENABLE_FORMAT_RECOVERY", "1")
+ monkeypatch.setenv("LIVEWEB_FORMAT_RECOVERY_CONTEXT_LENGTH", "64")
+ monkeypatch.setenv("LIVEWEB_FORMAT_RECOVERY_MAX_NEW_TOKENS", "32")
+ monkeypatch.setenv("LIVEWEB_FORMAT_RECOVERY_TOKEN_MARGIN", "16")
+
+ llm_client = _FakeLLMClient(
+ initial_response=LLMResponse(content="{\"name\":\"goto\""),
+ recovery_responses=[],
+ )
+ loop = AgentLoop(
+ session=_FakeSession(),
+ llm_client=llm_client,
+ protocol=FunctionCallingProtocol(),
+ max_steps=2,
+ )
+ long_obs = BrowserObservation(url="https://example.com", title="Long", accessibility_tree="x" * 2000)
+ loop._trajectory = []
+ messages = loop._build_recovery_messages(
+ system_prompt="sys",
+ user_prompt=f"obs {long_obs.accessibility_tree}",
+ failure_class="recoverable_truncated_tool_json",
+ )
+
+ raw, action, usage, override = await loop._attempt_format_recovery(
+ model="qwen",
+ seed=1,
+ raw_response="{\"name\":\"goto\"",
+ messages=messages,
+ failure_class="recoverable_truncated_tool_json",
+ )
+
+ assert action is None
+ assert usage is None
+ assert override == "recoverable_context_overflow"
+ assert llm_client.recovery_calls == 0
+
+
+@pytest.mark.anyio
+async def test_agent_loop_explicit_disable_recovery_overrides_env(monkeypatch):
+ monkeypatch.setenv("LIVEWEB_ENABLE_FORMAT_RECOVERY", "1")
+
+ llm_client = _FakeLLMClient(
+ initial_response=LLMResponse(content="{\"name\":\"goto\""),
+ recovery_responses=[
+ LLMResponse(tool_calls=[ToolCall(id="call_1", function={"name": "stop", "arguments": "{\"answers\":{\"a1\":\"42\"}}"})]),
+ ],
+ )
+ loop = AgentLoop(
+ session=_FakeSession(),
+ llm_client=llm_client,
+ protocol=FunctionCallingProtocol(),
+ max_steps=2,
+ enable_format_recovery=False,
+ )
+
+ trajectory, final_answer, usage = await loop.run(task=_task(), model="qwen", temperature=0.0, seed=1)
+
+ assert final_answer is None
+ assert loop.is_parse_failed() is True
+ assert llm_client.recovery_calls == 0
+
+
+@pytest.mark.anyio
+async def test_agent_loop_failfasts_after_consecutive_disallowed_domain_hits(monkeypatch):
+ monkeypatch.setenv("LIVEWEB_FAILFAST_DISALLOWED_DOMAIN_CONSECUTIVE", "2")
+ monkeypatch.setenv("LIVEWEB_FAILFAST_DISALLOWED_DOMAIN_TOTAL", "3")
+ monkeypatch.setenv("LIVEWEB_DISALLOWED_DOMAIN_RECOVERY_MAX_RETRIES", "1")
+
+ blocked_obs = BrowserObservation(
+ url="https://finance.yahoo.com/quote/V/",
+ title="Domain not allowed",
+ accessibility_tree="Domain not allowed. Please return to an allowed site and continue the task.",
+ )
+ session = _FakeSessionWithBlockedSequence(
+ observations=[blocked_obs, blocked_obs],
+ metadatas=[
+ {
+ "classification_hint": "model_disallowed_domain",
+ "url": "https://finance.yahoo.com/quote/V/",
+ "evidence": {
+ "blocked_url": "https://finance.yahoo.com/quote/V/",
+ "allowed_domains": ["coingecko.com"],
+ },
+ },
+ {
+ "classification_hint": "model_disallowed_domain",
+ "url": "https://finance.yahoo.com/quote/V/",
+ "evidence": {
+ "blocked_url": "https://finance.yahoo.com/quote/V/",
+ "allowed_domains": ["coingecko.com"],
+ },
+ },
+ ],
+ )
+ llm_client = _QueuedLLMClient(
+ responses=[
+ LLMResponse(tool_calls=[ToolCall(id="call_1", function={"name": "goto", "arguments": "{\"url\":\"https://finance.yahoo.com/quote/V/\"}"})]),
+ ],
+ recovery_responses=[
+ LLMResponse(tool_calls=[ToolCall(id="call_r1", function={"name": "goto", "arguments": "{\"url\":\"https://finance.yahoo.com/quote/V/\"}"})]),
+ ],
+ )
+ loop = AgentLoop(
+ session=session,
+ llm_client=llm_client,
+ protocol=FunctionCallingProtocol(),
+ max_steps=5,
+ )
+
+ with pytest.raises(BrowserFatalError) as exc_info:
+ await loop.run(task=_task(), model="qwen", temperature=0.0, seed=1)
+
+ assert "disallowed-domain" in str(exc_info.value)
+ assert exc_info.value.url == "https://finance.yahoo.com/quote/V/"
+ assert len(loop.get_trajectory()) == 0
+ assert llm_client.recovery_calls == 1
+
+
+@pytest.mark.anyio
+async def test_agent_loop_resets_disallowed_domain_consecutive_counter_after_recovery(monkeypatch):
+ monkeypatch.setenv("LIVEWEB_FAILFAST_DISALLOWED_DOMAIN_CONSECUTIVE", "2")
+ monkeypatch.setenv("LIVEWEB_FAILFAST_DISALLOWED_DOMAIN_TOTAL", "3")
+ monkeypatch.setenv("LIVEWEB_DISALLOWED_DOMAIN_RECOVERY_MAX_RETRIES", "1")
+
+ blocked_obs = BrowserObservation(
+ url="https://finance.yahoo.com/quote/V/",
+ title="Domain not allowed",
+ accessibility_tree="Domain not allowed. Please return to an allowed site and continue the task.",
+ )
+ good_obs = BrowserObservation(
+ url="https://example.com",
+ title="Example",
+ accessibility_tree="root content with enough accessible text to avoid blank observation failfast",
+ )
+ session = _FakeSessionWithBlockedSequence(
+ observations=[blocked_obs, good_obs, blocked_obs, good_obs],
+ metadatas=[
+ {
+ "classification_hint": "model_disallowed_domain",
+ "url": "https://finance.yahoo.com/quote/V/",
+ "evidence": {
+ "blocked_url": "https://finance.yahoo.com/quote/V/",
+ "allowed_domains": ["example.com"],
+ },
+ },
+ None,
+ {
+ "classification_hint": "model_disallowed_domain",
+ "url": "https://finance.yahoo.com/quote/V/",
+ "evidence": {
+ "blocked_url": "https://finance.yahoo.com/quote/V/",
+ "allowed_domains": ["example.com"],
+ },
+ },
+ None,
+ ],
+ )
+ llm_client = _QueuedLLMClient(
+ responses=[
+ LLMResponse(tool_calls=[ToolCall(id="call_1", function={"name": "goto", "arguments": "{\"url\":\"https://finance.yahoo.com/quote/V/\"}"})]),
+ LLMResponse(tool_calls=[ToolCall(id="call_2", function={"name": "goto", "arguments": "{\"url\":\"https://finance.yahoo.com/quote/V/\"}"})]),
+ LLMResponse(tool_calls=[ToolCall(id="call_3", function={"name": "stop", "arguments": "{\"answers\":{\"a1\":\"42\"}}"})]),
+ ],
+ recovery_responses=[
+ LLMResponse(tool_calls=[ToolCall(id="call_r1", function={"name": "goto", "arguments": "{\"url\":\"https://example.com\"}"})]),
+ LLMResponse(tool_calls=[ToolCall(id="call_r2", function={"name": "goto", "arguments": "{\"url\":\"https://example.com\"}"})]),
+ ],
+ )
+ loop = AgentLoop(
+ session=session,
+ llm_client=llm_client,
+ protocol=FunctionCallingProtocol(),
+ max_steps=5,
+ )
+
+ trajectory, final_answer, _usage = await loop.run(task=_task(), model="qwen", temperature=0.0, seed=1)
+
+ assert final_answer == {"answers": {"a1": "42"}}
+ assert len(trajectory) == 3
+ assert llm_client.recovery_calls == 2
+ assert trajectory[0].action.params["url"] == "https://example.com"
+ assert trajectory[0].action_result == "Success (recovered after disallowed-domain retry)"
+ assert trajectory[1].action.params["url"] == "https://example.com"
+
+
+@pytest.mark.anyio
+async def test_agent_loop_recovers_disallowed_domain_without_appending_blocked_step(monkeypatch):
+ monkeypatch.setenv("LIVEWEB_FAILFAST_DISALLOWED_DOMAIN_CONSECUTIVE", "2")
+ monkeypatch.setenv("LIVEWEB_FAILFAST_DISALLOWED_DOMAIN_TOTAL", "3")
+ monkeypatch.setenv("LIVEWEB_DISALLOWED_DOMAIN_RECOVERY_MAX_RETRIES", "1")
+
+ blocked_obs = BrowserObservation(
+ url="https://finance.yahoo.com/quote/V/",
+ title="Domain not allowed",
+ accessibility_tree="Domain not allowed. Please return to an allowed site and continue the task.",
+ )
+ good_obs = BrowserObservation(
+ url="https://www.coingecko.com/en/coins/optimism",
+ title="Optimism",
+ accessibility_tree="Circulating Supply 2,117,847,344 OP",
+ )
+ session = _FakeSessionWithBlockedSequence(
+ observations=[blocked_obs, good_obs],
+ metadatas=[
+ {
+ "classification_hint": "model_disallowed_domain",
+ "url": "https://finance.yahoo.com/quote/V/",
+ "evidence": {
+ "blocked_url": "https://finance.yahoo.com/quote/V/",
+ "allowed_domains": ["coingecko.com", "taostats.io"],
+ },
+ },
+ None,
+ ],
+ )
+ llm_client = _QueuedLLMClient(
+ responses=[
+ LLMResponse(tool_calls=[ToolCall(id="call_1", function={"name": "goto", "arguments": "{\"url\":\"https://finance.yahoo.com/quote/V/\"}"})]),
+ LLMResponse(tool_calls=[ToolCall(id="call_2", function={"name": "stop", "arguments": "{\"answers\":{\"a1\":\"42\"}}"})]),
+ ],
+ recovery_responses=[
+ LLMResponse(tool_calls=[ToolCall(id="call_r1", function={"name": "goto", "arguments": "{\"url\":\"https://www.coingecko.com/en/coins/optimism\"}"})]),
+ ],
+ )
+ loop = AgentLoop(
+ session=session,
+ llm_client=llm_client,
+ protocol=FunctionCallingProtocol(),
+ max_steps=4,
+ )
+
+ trajectory, final_answer, _usage = await loop.run(task=_task(), model="qwen", temperature=0.0, seed=1)
+
+ assert final_answer == {"answers": {"a1": "42"}}
+ assert len(trajectory) == 2
+ assert trajectory[0].action.params["url"] == "https://www.coingecko.com/en/coins/optimism"
+ assert "recovered after disallowed-domain retry" in trajectory[0].action_result.lower()
+ assert llm_client.recovery_calls == 1
+ messages = llm_client.recovery_messages[0]
+ assert "Blocked URL: https://finance.yahoo.com/quote/V/" in messages[-1]["content"]
+ assert "Allowed domains: coingecko.com, taostats.io" in messages[-1]["content"]
+
+
+@pytest.mark.anyio
+async def test_agent_loop_recovers_empty_stop_without_appending_bad_stop():
+ llm_client = _QueuedLLMClient(
+ responses=[
+ LLMResponse(tool_calls=[ToolCall(id="call_1", function={"name": "stop", "arguments": "{\"answers\":{}}"})]),
+ ],
+ recovery_responses=[
+ LLMResponse(tool_calls=[ToolCall(id="call_r1", function={"name": "stop", "arguments": "{\"answers\":{\"a1\":\"42\",\"a2\":\"Trishool\"}}"})]),
+ ],
+ )
+ loop = AgentLoop(
+ session=_FakeSession(),
+ llm_client=llm_client,
+ protocol=FunctionCallingProtocol(),
+ max_steps=2,
+ behavior_mode="collect",
+ )
+
+ trajectory, final_answer, _usage = await loop.run(task=_task_two_answers(), model="qwen", temperature=0.0, seed=1)
+
+ assert final_answer == {"answers": {"a1": "42", "a2": "Trishool"}}
+ assert len(trajectory) == 1
+ assert trajectory[0].action.action_type == "stop"
+ assert "recovered" not in trajectory[0].action_result.lower()
+ local_stats = loop.get_local_recovery_stats()
+ assert local_stats["empty_stop_recovery_attempts"] == 1
+ assert local_stats["empty_stop_recovery_successes"] == 1
+
+
+@pytest.mark.anyio
+async def test_agent_loop_marks_invalid_stop_payload_when_recovery_exhausted():
+ llm_client = _QueuedLLMClient(
+ responses=[
+ LLMResponse(tool_calls=[ToolCall(id="call_1", function={"name": "stop", "arguments": "{\"answers\":{}}"})]),
+ ],
+ recovery_responses=[
+ LLMResponse(tool_calls=[ToolCall(id="call_r1", function={"name": "stop", "arguments": "{\"answers\":{}}"})]),
+ ],
+ )
+ loop = AgentLoop(
+ session=_FakeSession(),
+ llm_client=llm_client,
+ protocol=FunctionCallingProtocol(),
+ max_steps=2,
+ behavior_mode="collect",
+ )
+
+ trajectory, final_answer, _usage = await loop.run(task=_task_two_answers(), model="qwen", temperature=0.0, seed=1)
+
+ assert final_answer is None
+ assert loop.is_invalid_stop_payload() is True
+ assert loop.get_last_stop_failure_class() == "empty_answers"
+ assert len(trajectory) == 1
+ assert trajectory[0].action.action_type == "stop"
+ assert "invalid stop payload" in trajectory[0].action_result.lower()
+
+
+@pytest.mark.anyio
+async def test_agent_loop_recovers_invalid_ui_target_with_same_page_action():
+ taostats_obs = BrowserObservation(
+ url="https://taostats.io/subnets",
+ title="Subnets",
+ accessibility_tree="Subnet list root content with enough text for recovery decisions",
+ )
+ session = _FakeSessionWithActionSequence(
+ outcomes=[
+ RuntimeError("No element found with role='button' name='Rows: 25'"),
+ taostats_obs,
+ ],
+ metadatas=[
+ {
+ "url": "https://taostats.io/subnets",
+ "navigation_stage": "action_click_role",
+ "raw_exception_type": "RuntimeError",
+ "raw_exception_message": "No element found with role='button' name='Rows: 25'",
+ "evidence": {
+ "ui_target_missing": True,
+ "page_kind": "taostats_list",
+ "interaction_kind": "show_all",
+ "target_locator": "role=button[name='Rows: 25']",
+ },
+ },
+ None,
+ ],
+ )
+ llm_client = _QueuedLLMClient(
+ responses=[
+ LLMResponse(tool_calls=[ToolCall(id="call_1", function={"name": "click_role", "arguments": "{\"role\":\"button\",\"name\":\"Rows: 25\"}"})]),
+ LLMResponse(tool_calls=[ToolCall(id="call_2", function={"name": "stop", "arguments": "{\"answers\":{\"a1\":\"42\"}}"})]),
+ ],
+ recovery_responses=[
+ LLMResponse(tool_calls=[ToolCall(id="call_r1", function={"name": "scroll", "arguments": "{\"direction\":\"down\"}"})]),
+ ],
+ )
+ loop = AgentLoop(
+ session=session,
+ llm_client=llm_client,
+ protocol=FunctionCallingProtocol(),
+ max_steps=3,
+ behavior_mode="collect",
+ )
+
+ trajectory, final_answer, _usage = await loop.run(task=_task(), model="qwen", temperature=0.0, seed=1)
+
+ assert final_answer == {"answers": {"a1": "42"}}
+ assert trajectory[0].action.action_type == "scroll"
+ assert "invalid_ui_target_recovery" in trajectory[0].action_result
+ stats = loop.get_local_recovery_stats()
+ assert stats["invalid_ui_target_recovery_attempts"] == 1
+ assert stats["invalid_ui_target_recovery_successes"] == 1
+
+
+@pytest.mark.anyio
+async def test_agent_loop_rejects_invalid_ui_target_recovery_that_uses_goto():
+ taostats_obs = BrowserObservation(
+ url="https://taostats.io/subnets",
+ title="Subnets",
+ accessibility_tree="Subnet list root content with enough text for recovery decisions",
+ )
+ session = _FakeSessionWithActionSequence(
+ outcomes=[
+ RuntimeError("No element found with role='button' name='Rows: 25'"),
+ ],
+ metadatas=[
+ {
+ "url": "https://taostats.io/subnets",
+ "navigation_stage": "action_click_role",
+ "raw_exception_type": "RuntimeError",
+ "raw_exception_message": "No element found with role='button' name='Rows: 25'",
+ "evidence": {
+ "ui_target_missing": True,
+ "page_kind": "taostats_list",
+ "interaction_kind": "show_all",
+ "target_locator": "role=button[name='Rows: 25']",
+ },
+ },
+ ],
+ )
+ llm_client = _QueuedLLMClient(
+ responses=[
+ LLMResponse(tool_calls=[ToolCall(id="call_1", function={"name": "click_role", "arguments": "{\"role\":\"button\",\"name\":\"Rows: 25\"}"})]),
+ ],
+ recovery_responses=[
+ LLMResponse(tool_calls=[ToolCall(id="call_r1", function={"name": "goto", "arguments": "{\"url\":\"https://finance.yahoo.com\"}"})]),
+ ],
+ )
+ loop = AgentLoop(
+ session=session,
+ llm_client=llm_client,
+ protocol=FunctionCallingProtocol(),
+ max_steps=1,
+ behavior_mode="collect",
+ )
+
+ trajectory, final_answer, _usage = await loop.run(task=_task(), model="qwen", temperature=0.0, seed=1)
+
+ assert final_answer is None
+ assert len(trajectory) == 1
+ assert trajectory[0].action.action_type == "click_role"
+ assert trajectory[0].action_result.startswith("Failed:")
+
+
+@pytest.mark.anyio
+async def test_agent_loop_recovers_taostats_list_timeout_with_same_page_action():
+ taostats_obs = BrowserObservation(
+ url="https://taostats.io/subnets",
+ title="Subnets",
+ accessibility_tree="Subnet list root content with enough text for recovery decisions",
+ )
+ session = _FakeSessionWithActionSequence(
+ outcomes=[
+ RuntimeError("Page.click: Timeout 5000ms exceeded."),
+ taostats_obs,
+ ],
+ metadatas=[
+ {
+ "url": "https://taostats.io/subnets",
+ "navigation_stage": "action_click",
+ "raw_exception_type": "TimeoutError",
+ "raw_exception_message": "Page.click: Timeout 5000ms exceeded.",
+ "evidence": {
+ "page_kind": "taostats_list",
+ "interaction_kind": "sort",
+ "target_locator": "thead th:has-text(\"24H\")",
+ },
+ },
+ None,
+ ],
+ )
+ llm_client = _QueuedLLMClient(
+ responses=[
+ LLMResponse(tool_calls=[ToolCall(id="call_1", function={"name": "click", "arguments": "{\"selector\":\"thead th:has-text(\\\"24H\\\")\"}"})]),
+ LLMResponse(tool_calls=[ToolCall(id="call_2", function={"name": "stop", "arguments": "{\"answers\":{\"a1\":\"42\"}}"})]),
+ ],
+ recovery_responses=[
+ LLMResponse(tool_calls=[ToolCall(id="call_r1", function={"name": "scroll", "arguments": "{\"direction\":\"down\"}"})]),
+ ],
+ )
+ loop = AgentLoop(
+ session=session,
+ llm_client=llm_client,
+ protocol=FunctionCallingProtocol(),
+ max_steps=3,
+ behavior_mode="collect",
+ )
+
+ trajectory, final_answer, _usage = await loop.run(task=_task(), model="qwen", temperature=0.0, seed=1)
+
+ assert final_answer == {"answers": {"a1": "42"}}
+ assert trajectory[0].action.action_type == "scroll"
+ assert "taostats_list_action_recovery" in trajectory[0].action_result
+ stats = loop.get_local_recovery_stats()
+ assert stats["taostats_list_action_recovery_attempts"] == 1
+ assert stats["taostats_list_action_recovery_successes"] == 1
+
+
+@pytest.mark.anyio
+async def test_agent_loop_collect_only_recoveries_are_disabled_in_eval_mode():
+ llm_client = _QueuedLLMClient(
+ responses=[
+ LLMResponse(tool_calls=[ToolCall(id="call_1", function={"name": "stop", "arguments": "{\"answers\":{}}"})]),
+ ],
+ recovery_responses=[
+ LLMResponse(tool_calls=[ToolCall(id="call_r1", function={"name": "stop", "arguments": "{\"answers\":{\"a1\":\"42\"}}"})]),
+ ],
+ )
+ loop = AgentLoop(
+ session=_FakeSession(),
+ llm_client=llm_client,
+ protocol=FunctionCallingProtocol(),
+ max_steps=1,
+ behavior_mode="eval",
+ )
+
+ trajectory, final_answer, _usage = await loop.run(task=_task(), model="qwen", temperature=0.0, seed=1)
+
+ assert final_answer == {"answers": {}}
+ assert llm_client.recovery_calls == 0
+ assert len(trajectory) == 1
+
+
+@pytest.mark.anyio
+async def test_agent_loop_recovers_invalid_generated_url_without_appending_bad_step():
+ llm_client = _QueuedLLMClient(
+ responses=[
+ LLMResponse(tool_calls=[ToolCall(id="call_1", function={"name": "goto", "arguments": "{\"url\":\"https://:\"}"})]),
+ LLMResponse(tool_calls=[ToolCall(id="call_2", function={"name": "stop", "arguments": "{\"answers\":{\"a1\":\"42\"}}"})]),
+ ],
+ recovery_responses=[
+ LLMResponse(tool_calls=[ToolCall(id="call_r1", function={"name": "goto", "arguments": "{\"url\":\"https://example.com\"}"})]),
+ ],
+ )
+ loop = AgentLoop(
+ session=_FakeSession(),
+ llm_client=llm_client,
+ protocol=FunctionCallingProtocol(),
+ max_steps=3,
+ behavior_mode="collect",
+ )
+
+ trajectory, final_answer, _usage = await loop.run(task=_task(), model="qwen", temperature=0.0, seed=1)
+
+ assert final_answer == {"answers": {"a1": "42"}}
+ assert loop.is_invalid_generated_url() is False
+ assert len(trajectory) == 2
+ assert trajectory[0].action.params["url"] == "https://example.com"
+ assert "invalid_generated_url_recovery" in trajectory[0].action_result
+ stats = loop.get_local_recovery_stats()
+ assert stats["invalid_generated_url_recovery_attempts"] == 1
+ assert stats["invalid_generated_url_recovery_successes"] == 1
+
+
+@pytest.mark.anyio
+async def test_agent_loop_marks_invalid_generated_url_when_recovery_exhausted():
+ llm_client = _QueuedLLMClient(
+ responses=[
+ LLMResponse(tool_calls=[ToolCall(id="call_1", function={"name": "goto", "arguments": "{\"url\":\"https://:\"}"})]),
+ ],
+ recovery_responses=[
+ LLMResponse(tool_calls=[ToolCall(id="call_r1", function={"name": "goto", "arguments": "{\"url\":\"https://:\"}"})]),
+ ],
+ )
+ loop = AgentLoop(
+ session=_FakeSession(),
+ llm_client=llm_client,
+ protocol=FunctionCallingProtocol(),
+ max_steps=2,
+ behavior_mode="collect",
+ )
+
+ trajectory, final_answer, _usage = await loop.run(task=_task(), model="qwen", temperature=0.0, seed=1)
+
+ assert final_answer is None
+ assert loop.is_invalid_generated_url() is True
+ assert loop.get_last_invalid_generated_url_detail()["kind"] == "missing_host"
+ assert len(trajectory) == 1
+ assert "Invalid generated URL" in trajectory[0].action_result
+
+
+@pytest.mark.anyio
+async def test_agent_loop_detects_repetitive_same_action_loop_for_gemini(monkeypatch):
+ monkeypatch.setenv("LIVEWEB_GEMINI_LOOP_SAME_ACTION_THRESHOLD", "6")
+
+ repeated_obs = BrowserObservation(
+ url="https://taostats.io/subnets/",
+ title="Subnets",
+ accessibility_tree="Subnet table with enough accessible text to support repeated interaction.",
+ )
+ session = _FakeSessionWithActionSequence(
+ outcomes=[repeated_obs] * 6,
+ metadatas=[None] * 6,
+ )
+ click_args = "{\"selector\":\"th:nth-child(3)\"}"
+ llm_client = _QueuedLLMClient(
+ responses=[
+ LLMResponse(tool_calls=[ToolCall(id="goto_1", function={"name": "goto", "arguments": "{\"url\":\"https://taostats.io/subnets/\"}"})]),
+ *[
+ LLMResponse(tool_calls=[ToolCall(id=f"click_{idx}", function={"name": "click", "arguments": click_args})])
+ for idx in range(6)
+ ],
+ ],
+ )
+ loop = AgentLoop(
+ session=session,
+ llm_client=llm_client,
+ protocol=FunctionCallingProtocol(),
+ max_steps=10,
+ behavior_mode="collect",
+ )
+
+ trajectory, final_answer, _usage = await loop.run(
+ task=_task(), model="google/gemini-3-flash-preview", temperature=0.0, seed=1
+ )
+
+ assert final_answer is None
+ assert loop.is_action_loop_detected() is True
+ assert loop.get_last_action_loop_detail()["kind"] == "same_action_repeat"
+ assert trajectory[-1].action_result.startswith("Aborted: repetitive_action_loop")
+ assert "th:nth-child(3)" in trajectory[-1].action_result
+
+
+@pytest.mark.anyio
+async def test_agent_loop_detects_repetitive_same_type_loop_for_gemini(monkeypatch):
+ monkeypatch.setenv("LIVEWEB_GEMINI_LOOP_SAME_TYPE_THRESHOLD", "4")
+
+ search_obs = BrowserObservation(
+ url="https://taostats.io/subnets/",
+ title="Subnets",
+ accessibility_tree="Subnet list page with search box and enough accessible text.",
+ )
+ session = _FakeSessionWithActionSequence(
+ outcomes=[search_obs] * 5,
+ metadatas=[None] * 5,
+ )
+ type_args = "{\"selector\":\"input[placeholder='Search']\",\"text\":\"SN23\"}"
+ llm_client = _QueuedLLMClient(
+ responses=[
+ LLMResponse(tool_calls=[ToolCall(id="goto_1", function={"name": "goto", "arguments": "{\"url\":\"https://taostats.io/subnets/\"}"})]),
+ *[
+ LLMResponse(tool_calls=[ToolCall(id=f"type_{idx}", function={"name": "type", "arguments": type_args})])
+ for idx in range(4)
+ ],
+ ],
+ )
+ loop = AgentLoop(
+ session=session,
+ llm_client=llm_client,
+ protocol=FunctionCallingProtocol(),
+ max_steps=8,
+ behavior_mode="collect",
+ )
+
+ trajectory, final_answer, _usage = await loop.run(
+ task=_task(), model="google/gemini-3-flash-preview", temperature=0.0, seed=1
+ )
+
+ assert final_answer is None
+ assert loop.is_action_loop_detected() is True
+ assert loop.get_last_action_loop_detail()["kind"] == "same_type_repeat"
+
+
+@pytest.mark.anyio
+async def test_agent_loop_detects_goto_oscillation_for_gemini(monkeypatch):
+ monkeypatch.setenv("LIVEWEB_GEMINI_LOOP_GOTO_OSCILLATION_THRESHOLD", "4")
+
+ subnet_list_obs = BrowserObservation(
+ url="https://taostats.io/subnets/",
+ title="Subnets",
+ accessibility_tree="Subnet list with enough accessible text to support navigation.",
+ )
+ subnet_detail_obs = BrowserObservation(
+ url="https://taostats.io/subnet/23/",
+ title="Subnet 23",
+ accessibility_tree="Subnet detail with enough accessible text to support navigation.",
+ )
+ session = _FakeSessionWithActionSequence(
+ outcomes=[
+ subnet_list_obs,
+ subnet_detail_obs,
+ subnet_list_obs,
+ subnet_detail_obs,
+ subnet_list_obs,
+ subnet_detail_obs,
+ subnet_list_obs,
+ ],
+ metadatas=[None] * 7,
+ )
+ llm_client = _QueuedLLMClient(
+ responses=[
+ LLMResponse(tool_calls=[ToolCall(id="goto_1", function={"name": "goto", "arguments": "{\"url\":\"https://taostats.io/subnets/\"}"})]),
+ LLMResponse(tool_calls=[ToolCall(id="goto_2", function={"name": "goto", "arguments": "{\"url\":\"https://taostats.io/subnet/23/\"}"})]),
+ LLMResponse(tool_calls=[ToolCall(id="goto_3", function={"name": "goto", "arguments": "{\"url\":\"https://taostats.io/subnets/\"}"})]),
+ LLMResponse(tool_calls=[ToolCall(id="goto_4", function={"name": "goto", "arguments": "{\"url\":\"https://taostats.io/subnet/23/\"}"})]),
+ LLMResponse(tool_calls=[ToolCall(id="goto_5", function={"name": "goto", "arguments": "{\"url\":\"https://taostats.io/subnets/\"}"})]),
+ LLMResponse(tool_calls=[ToolCall(id="goto_6", function={"name": "goto", "arguments": "{\"url\":\"https://taostats.io/subnet/23/\"}"})]),
+ LLMResponse(tool_calls=[ToolCall(id="goto_7", function={"name": "goto", "arguments": "{\"url\":\"https://taostats.io/subnets/\"}"})]),
+ LLMResponse(tool_calls=[ToolCall(id="goto_8", function={"name": "goto", "arguments": "{\"url\":\"https://taostats.io/subnet/23/\"}"})]),
+ ],
+ )
+ loop = AgentLoop(
+ session=session,
+ llm_client=llm_client,
+ protocol=FunctionCallingProtocol(),
+ max_steps=12,
+ behavior_mode="collect",
+ )
+
+ trajectory, final_answer, _usage = await loop.run(
+ task=_task(), model="google/gemini-3-flash-preview", temperature=0.0, seed=1
+ )
+
+ assert final_answer is None
+ assert loop.is_action_loop_detected() is True
+ detail = loop.get_last_action_loop_detail()
+ assert detail["kind"] == "goto_oscillation"
+ assert set(detail["oscillation_urls"]) == {
+ "https://taostats.io/subnets/",
+ "https://taostats.io/subnet/23/",
+ }
+ assert trajectory[-1].action_result.startswith("Aborted: repetitive_action_loop")
+
+
+@pytest.mark.anyio
+async def test_agent_loop_detects_search_finance_bounce_for_gemini(monkeypatch):
+ monkeypatch.setenv("LIVEWEB_GEMINI_LOOP_SEARCH_BOUNCE_THRESHOLD", "4")
+
+ google_obs = BrowserObservation(
+ url="https://www.google.com/search?q=dogecoin",
+ title="Google Search",
+ accessibility_tree="Search results page with enough text to support navigation.",
+ )
+ yahoo_obs = BrowserObservation(
+ url="https://finance.yahoo.com/quote/DOGE-USD/",
+ title="Yahoo Finance",
+ accessibility_tree="Finance quote page with enough text to support navigation.",
+ )
+ cmc_obs = BrowserObservation(
+ url="https://coinmarketcap.com/currencies/dogecoin/",
+ title="CoinMarketCap",
+ accessibility_tree="Coin page with enough text to support navigation.",
+ )
+ session = _FakeSessionWithActionSequence(
+ outcomes=[google_obs, yahoo_obs, google_obs, cmc_obs],
+ metadatas=[None] * 4,
+ )
+ llm_client = _QueuedLLMClient(
+ responses=[
+ LLMResponse(tool_calls=[ToolCall(id="goto_1", function={"name": "goto", "arguments": "{\"url\":\"https://www.google.com/search?q=dogecoin\"}"})]),
+ LLMResponse(tool_calls=[ToolCall(id="goto_2", function={"name": "goto", "arguments": "{\"url\":\"https://finance.yahoo.com/quote/DOGE-USD/\"}"})]),
+ LLMResponse(tool_calls=[ToolCall(id="goto_3", function={"name": "goto", "arguments": "{\"url\":\"https://www.google.com/search?q=dogecoin\"}"})]),
+ LLMResponse(tool_calls=[ToolCall(id="goto_4", function={"name": "goto", "arguments": "{\"url\":\"https://coinmarketcap.com/currencies/dogecoin/\"}"})]),
+ ],
+ )
+ loop = AgentLoop(
+ session=session,
+ llm_client=llm_client,
+ protocol=FunctionCallingProtocol(),
+ max_steps=6,
+ behavior_mode="collect",
+ )
+
+ trajectory, final_answer, _usage = await loop.run(
+ task=_task(), model="google/gemini-3-flash-preview", temperature=0.0, seed=1
+ )
+
+ assert final_answer is None
+ assert loop.is_action_loop_detected() is True
+ assert loop.get_last_action_loop_detail()["kind"] == "search_finance_bounce"
+
+
+@pytest.mark.anyio
+async def test_agent_loop_does_not_apply_loop_guard_to_non_gemini(monkeypatch):
+ monkeypatch.setenv("LIVEWEB_GEMINI_LOOP_SAME_ACTION_THRESHOLD", "6")
+
+ repeated_obs = BrowserObservation(
+ url="https://taostats.io/subnets/",
+ title="Subnets",
+ accessibility_tree="Subnet table with enough accessible text to support repeated interaction.",
+ )
+ session = _FakeSessionWithActionSequence(
+ outcomes=[repeated_obs] * 7,
+ metadatas=[None] * 7,
+ )
+ click_args = "{\"selector\":\"th:nth-child(3)\"}"
+ llm_client = _QueuedLLMClient(
+ responses=[
+ LLMResponse(tool_calls=[ToolCall(id="goto_1", function={"name": "goto", "arguments": "{\"url\":\"https://taostats.io/subnets/\"}"})]),
+ *[
+ LLMResponse(tool_calls=[ToolCall(id=f"click_{idx}", function={"name": "click", "arguments": click_args})])
+ for idx in range(6)
+ ],
+ LLMResponse(tool_calls=[ToolCall(id="stop_1", function={"name": "stop", "arguments": "{\"answers\":{\"a1\":\"42\"}}"})]),
+ ],
+ )
+ loop = AgentLoop(
+ session=session,
+ llm_client=llm_client,
+ protocol=FunctionCallingProtocol(),
+ max_steps=12,
+ behavior_mode="collect",
+ )
+
+ trajectory, final_answer, _usage = await loop.run(
+ task=_task(), model="mimo_v2_pro_openrouter", temperature=0.0, seed=1
+ )
+
+ assert final_answer == {"answers": {"a1": "42"}}
+ assert loop.is_action_loop_detected() is False
+ assert trajectory[-1].action.action_type == "stop"
diff --git a/tests/core/test_agent_protocol.py b/tests/core/test_agent_protocol.py
index 8791675..71f5f03 100644
--- a/tests/core/test_agent_protocol.py
+++ b/tests/core/test_agent_protocol.py
@@ -15,6 +15,34 @@ def protocol():
return FunctionCallingProtocol()
+def test_gpt54_prompt_profile_adds_strict_system_rules():
+ from liveweb_arena.core.models import CompositeTask
+ from liveweb_arena.plugins.base import SubTask
+
+ task = CompositeTask(
+ subtasks=[SubTask(plugin_name="coingecko", intent="Example?", validation_info={}, answer_tag="answer1")],
+ combined_intent="## Tasks to Complete\n\n1. Example?\n Answer tag: answer1\n\n## Output Requirements\n\nWhen you have completed all tasks, use the \"stop\" action with your answers in this JSON format:\n\n```json\n{\"answers\": {\"answer1\": \"...\"}}\n```",
+ plugin_hints={},
+ seed=1,
+ )
+ profile_protocol = FunctionCallingProtocol(prompt_profile="gpt54_strict_domains")
+ prompt = profile_protocol.build_system_prompt(task)
+ assert "Do not use Google Search, Google Finance, Yahoo Finance" in prompt
+ assert "Never call stop with an empty answers object." in prompt
+
+
+def test_gpt54_prompt_profile_adds_step_reminders():
+ obs = BrowserObservation(
+ url="https://www.coingecko.com/en/coins/optimism",
+ title="Optimism",
+ accessibility_tree="Circulating Supply 2,117,847,344",
+ )
+ profile_protocol = FunctionCallingProtocol(prompt_profile="gpt54_strict_domains")
+ prompt = profile_protocol.build_step_prompt(obs, [], current_step=2, max_steps=40)
+ assert 'Never call stop with {"answers": {}}.' in prompt
+ assert "Do not use Google, Yahoo, CoinMarketCap, XE, TradingView, or search engines." in prompt
+
+
# ── get_tools ──────────────────────────────────────────────────────
def test_get_tools_returns_all_actions(protocol):
@@ -106,6 +134,138 @@ def test_parse_uses_first_tool_call_only(protocol):
assert action.action_type == "goto"
+def test_parse_qwen_tool_call_tag_fallback(protocol):
+ raw = """
+
+
+
+{"name":"goto","arguments":{"url":"https://example.com"}}
+
+"""
+ action = protocol.parse_response(raw, None)
+ assert action is not None
+ assert action.action_type == "goto"
+ assert action.params["url"] == "https://example.com"
+
+
+def test_strict_compat_protocol_does_not_accept_qwen_fallback_text():
+ protocol = FunctionCallingProtocol(strict_compat=True)
+ raw = """
+
+
+
+{"name":"goto","arguments":{"url":"https://example.com"}}
+
+"""
+ assert protocol.parse_response(raw, None) is None
+ assert protocol.classify_format_failure(raw, None) == "terminal"
+
+
+def test_parse_qwen_stop_fallback(protocol):
+ raw = '{"name":"stop","arguments":{"answers":{"a1":"42"}}}'
+ action = protocol.parse_response(raw, None)
+ assert action is not None
+ assert action.action_type == "stop"
+ assert action.params == {"final": {"answers": {"a1": "42"}}}
+
+
+def test_parse_qwen_stop_after_think_fallback(protocol):
+ raw = '\n\n\n{"name":"stop","arguments":{"answers":{"a1":"42"}}}'
+ action = protocol.parse_response(raw, None)
+ assert action is not None
+ assert action.action_type == "stop"
+ assert action.params == {"final": {"answers": {"a1": "42"}}}
+
+
+def test_parse_qwen_function_style_fallback(protocol):
+ raw = 'click_role({"role":"link","name":"Ask HN","exact":false})'
+ action = protocol.parse_response(raw, None)
+ assert action is not None
+ assert action.action_type == "click_role"
+ assert action.params == {"role": "link", "name": "Ask HN", "exact": False}
+
+
+def test_parse_qwen_tool_tag_function_style_fallback(protocol):
+ raw = """
+
+stop({"answers":{"a1":"42"}})
+
+"""
+ action = protocol.parse_response(raw, None)
+ assert action is not None
+ assert action.action_type == "stop"
+ assert action.params == {"final": {"answers": {"a1": "42"}}}
+
+
+def test_parse_qwen_function_style_with_wrapper_noise(protocol):
+ raw = '_goto({"url":"https://news.ycombinator.com"}_)'
+ action = protocol.parse_response(raw, None)
+ assert action is not None
+ assert action.action_type == "goto"
+ assert action.params == {"url": "https://news.ycombinator.com"}
+
+
+def test_parse_bracket_tool_call_stop_fallback(protocol):
+ raw = '[tool_call: stop({"answers":{"a1":"42"}})]'
+ action = protocol.parse_response(raw, None)
+ assert action is not None
+ assert action.action_type == "stop"
+ assert action.params == {"final": {"answers": {"a1": "42"}}}
+
+
+def test_parse_bracket_tool_call_goto_fallback(protocol):
+ raw = '[tool_call: goto({"url":"https://example.com"})]'
+ action = protocol.parse_response(raw, None)
+ assert action is not None
+ assert action.action_type == "goto"
+ assert action.params == {"url": "https://example.com"}
+
+
+def test_debug_parse_metadata_identifies_bracket_tool_call(protocol):
+ raw = '[tool_call: stop({"answers":{"a1":"42"}})]'
+ metadata = protocol.debug_parse_metadata(raw, None)
+ assert metadata["protocol_parser_branch"] == "bracket_tool_call"
+ assert metadata["tool_calls_preview"] == []
+
+
+def test_debug_parse_metadata_identifies_unparsed_natural_language(protocol):
+ raw = "I will think first and then browse later."
+ metadata = protocol.debug_parse_metadata(raw, None)
+ assert metadata["protocol_parser_branch"] == "natural_language"
+
+
+def test_parse_qwen_fallback_rejects_natural_language(protocol):
+ raw = 'I should stop now. {"name":"stop","arguments":{"answers":{"a1":"42"}}}'
+ assert protocol.parse_response(raw, None) is None
+
+
+def test_classify_format_failure_empty_response_is_recoverable(protocol):
+ assert protocol.classify_format_failure("", None) == "recoverable_empty"
+
+
+def test_classify_format_failure_truncated_tool_call_is_recoverable(protocol):
+ raw = "{\"name\":\"goto\",\"arguments\":{\"url\":\"https://example.com\"}"
+ assert protocol.classify_format_failure(raw, None) == "recoverable_truncated_tool_json"
+
+
+def test_classify_format_failure_valid_bracket_tool_call_is_not_failure(protocol):
+ raw = '[tool_call: stop({"answers":{"a1":"42"}})]'
+ assert protocol.classify_format_failure(raw, None) == "none"
+
+
+def test_parse_bracket_tool_call_repairs_missing_outer_brace(protocol):
+ raw = '[tool_call: stop({"answers":{"a1":"42"})]'
+ action = protocol.parse_response(raw, None)
+ assert action is not None
+ assert action.action_type == "stop"
+ assert action.params == {"final": {"answers": {"a1": "42"}}}
+
+
+def test_classify_format_failure_natural_language_is_terminal(protocol):
+ raw = "I will now browse the page and report back with the answer."
+ assert protocol.classify_format_failure(raw, None) == "terminal_natural_language"
+
+
# ── serialize_step ─────────────────────────────────────────────────
def _make_step(step_num, action_type, params, action_result="Success", prompt="obs"):
diff --git a/tests/core/test_browser.py b/tests/core/test_browser.py
new file mode 100644
index 0000000..7a3bfd1
--- /dev/null
+++ b/tests/core/test_browser.py
@@ -0,0 +1,194 @@
+from types import SimpleNamespace
+
+import pytest
+
+from liveweb_arena.core.browser import (
+ BrowserSession,
+ _looks_like_non_html_navigation_target,
+ _normalize_stooq_url,
+ _should_fallback_to_direct_navigation,
+)
+
+
+class _FakeElement:
+ def __init__(self, href: str | None, *, click_error: Exception | None = None):
+ self._href = href
+ self._click_error = click_error
+ self.evaluated = False
+
+ async def get_attribute(self, name: str):
+ if name == "href":
+ return self._href
+ return None
+
+ async def click(self, force: bool = False, timeout: int | None = None):
+ if self._click_error is not None:
+ raise self._click_error
+ return None
+
+ async def evaluate(self, script: str):
+ self.evaluated = True
+ return None
+
+
+class _FakeLocator:
+ def __init__(self, href: str | None, *, click_error: Exception | None = None):
+ self._href = href
+ self._click_error = click_error
+ self.evaluated = False
+ self.first = self
+
+ async def get_attribute(self, name: str):
+ if name == "href":
+ return self._href
+ return None
+
+ async def click(self, force: bool = False, timeout: int | None = None):
+ if self._click_error is not None:
+ raise self._click_error
+ return None
+
+ async def evaluate(self, script: str):
+ self.evaluated = True
+ return None
+
+
+class _FakePage:
+ def __init__(self, href: str | None, *, element: _FakeElement | None = None):
+ self.url = "https://taostats.io/"
+ self._href = href
+ self._element = element or _FakeElement(href)
+
+ async def query_selector(self, selector: str):
+ return self._element
+
+
+class _FallbackClickPage:
+ def __init__(self, *, url: str, click_results: dict[str, Exception | None]):
+ self.url = url
+ self._click_results = click_results
+ self.clicked: list[str] = []
+
+ async def click(self, selector: str, timeout: int | None = None):
+ self.clicked.append(selector)
+ outcome = self._click_results.get(selector, RuntimeError("not found"))
+ if outcome is not None:
+ raise outcome
+ return None
+
+ async def query_selector(self, selector: str):
+ return None
+
+
+@pytest.mark.asyncio
+async def test_browser_session_direct_nav_fallback_from_selector():
+ session = BrowserSession.__new__(BrowserSession)
+ session._page = _FakePage("/subnets")
+
+ visited = {}
+
+ async def _goto(url: str):
+ visited["url"] = url
+
+ session._goto_with_recovery = _goto
+
+ ok = await session._direct_nav_fallback_from_selector("a[href='/subnets']")
+ assert ok is True
+ assert visited["url"] == "https://taostats.io/subnets"
+
+
+@pytest.mark.asyncio
+async def test_browser_session_direct_nav_fallback_from_locator():
+ session = BrowserSession.__new__(BrowserSession)
+ session._page = SimpleNamespace(url="https://taostats.io/")
+
+ visited = {}
+
+ async def _goto(url: str):
+ visited["url"] = url
+
+ session._goto_with_recovery = _goto
+
+ ok = await session._direct_nav_fallback_from_locator(_FakeLocator("/subnets/23"))
+ assert ok is True
+ assert visited["url"] == "https://taostats.io/subnets/23"
+
+
+@pytest.mark.asyncio
+async def test_browser_session_force_click_selector_fallback_uses_js_click():
+ session = BrowserSession.__new__(BrowserSession)
+ element = _FakeElement(None, click_error=RuntimeError("pointer intercepted"))
+ session._page = _FakePage(None, element=element)
+
+ ok = await session._force_click_selector_fallback(".dropdown-toggle")
+ assert ok is True
+ assert element.evaluated is True
+
+
+@pytest.mark.asyncio
+async def test_browser_session_force_click_locator_fallback_uses_js_click():
+ session = BrowserSession.__new__(BrowserSession)
+ locator = _FakeLocator(None, click_error=RuntimeError("pointer intercepted"))
+
+ ok = await session._force_click_locator_fallback(locator)
+ assert ok is True
+ assert locator.evaluated is True
+
+
+@pytest.mark.asyncio
+async def test_browser_session_taostats_role_fallback_can_navigate_home_to_subnets():
+ session = BrowserSession.__new__(BrowserSession)
+ session._page = SimpleNamespace(url="https://taostats.io")
+
+ visited = {}
+
+ async def _goto(url: str):
+ visited["url"] = url
+
+ session._goto_with_recovery = _goto
+
+ ok = await session._taostats_list_role_fallback("link", "View All", 5000)
+ assert ok is True
+ assert visited["url"] == "https://taostats.io/subnets"
+
+
+@pytest.mark.asyncio
+async def test_browser_session_taostats_selector_fallback_tries_semantic_candidates():
+ session = BrowserSession.__new__(BrowserSession)
+ session._page = _FallbackClickPage(
+ url="https://taostats.io/subnets",
+ click_results={
+ '[data-testid="rows-select"]': RuntimeError("timeout"),
+ ".ant-select-selector": None,
+ },
+ )
+
+ ok = await session._taostats_list_selector_fallback('[data-testid="rows-select"]', 5000)
+ assert ok is True
+ assert session._page.clicked[:2] == ['[data-testid="rows-select"]', '.ant-select-selector']
+
+
+def test_should_fallback_to_direct_navigation_for_intercepted_click_timeout():
+ exc = RuntimeError("Page.click: Timeout 5000ms exceeded... intercepts pointer events")
+ assert _should_fallback_to_direct_navigation(exc) is True
+
+
+def test_normalize_stooq_list_endpoint_preserves_original_path():
+ normalized = _normalize_stooq_url("https://stooq.com/q/l/?s=jnj.us&f=sd2t2ohlcv&h=&e=csv")
+ assert normalized == "https://stooq.com/q/l/?s=jnj.us&f=sd2t2ohlcv&h=&e=csv"
+
+
+def test_normalize_stooq_slug_alias_preserves_original_slug():
+ normalized = _normalize_stooq_url("https://stooq.com/q/uk100/")
+ assert normalized == "https://stooq.com/q/uk100/"
+
+
+def test_non_html_navigation_heuristic_flags_downloadish_urls():
+ assert _looks_like_non_html_navigation_target("https://stooq.com/q/l/?s=jnj.us&f=sd2t2ohlcv&h=&e=csv") is True
+ assert _looks_like_non_html_navigation_target("https://example.com/export/report.csv") is True
+ assert _looks_like_non_html_navigation_target("https://example.com/data?format=json") is True
+
+
+def test_non_html_navigation_heuristic_skips_normal_pages():
+ assert _looks_like_non_html_navigation_target("https://stooq.com/q/?s=jnj.us") is False
+ assert _looks_like_non_html_navigation_target("https://taostats.io/subnets") is False
diff --git a/tests/core/test_browser_real_urls.py b/tests/core/test_browser_real_urls.py
new file mode 100644
index 0000000..0d6b157
--- /dev/null
+++ b/tests/core/test_browser_real_urls.py
@@ -0,0 +1,84 @@
+import os
+
+import pytest
+
+
+pytestmark = [
+ pytest.mark.asyncio,
+ pytest.mark.skipif(
+ os.getenv("LIVEWEB_REAL_BROWSER_SMOKE") != "1",
+ reason="set LIVEWEB_REAL_BROWSER_SMOKE=1 to run real-browser smoke tests",
+ ),
+]
+
+
+async def test_real_browser_taostats_subnets_click_fallback():
+ from liveweb_arena.core.browser import BrowserAction, BrowserEngine
+
+ engine = BrowserEngine(headless=True)
+ session = await engine.new_session()
+ try:
+ obs = await session.goto("https://taostats.io")
+ assert "taostats.io" in obs.url
+
+ obs = await session.execute_action(
+ BrowserAction(
+ action_type="click",
+ params={"selector": "a[href='/subnets']", "timeout_ms": 5000},
+ )
+ )
+ assert "taostats.io" in obs.url
+ assert "Subnets" in obs.title
+ assert len(obs.accessibility_tree.strip()) > 100
+ finally:
+ await session.close()
+ await engine.stop()
+
+
+async def test_real_browser_coingecko_binance_coin_page_reachable():
+ from liveweb_arena.core.browser import BrowserEngine
+
+ engine = BrowserEngine(headless=True)
+ session = await engine.new_session()
+ try:
+ obs = await session.goto("https://www.coingecko.com/en/coins/binance-coin")
+ assert "coingecko.com" in obs.url
+ assert "/en/coins/" in obs.url
+ assert obs.url.endswith("/bnb") or obs.url.endswith("/binance-coin")
+ assert "BNB" in obs.title or "Binance" in obs.title
+ assert len(obs.accessibility_tree.strip()) > 100
+ finally:
+ await session.close()
+ await engine.stop()
+
+
+async def test_real_browser_stooq_quote_page_reachable():
+ from liveweb_arena.core.browser import BrowserEngine
+
+ engine = BrowserEngine(headless=True)
+ session = await engine.new_session()
+ try:
+ obs = await session.goto("https://stooq.com/q/?s=jnj.us")
+ assert "stooq.com" in obs.url
+ assert "/q/?s=jnj.us" in obs.url.lower()
+ assert "jnj" in obs.title.lower()
+ assert len(obs.accessibility_tree.strip()) > 50
+ finally:
+ await session.close()
+ await engine.stop()
+
+
+async def test_real_browser_stooq_download_endpoint_returns_synthetic_observation():
+ from liveweb_arena.core.browser import BrowserEngine
+
+ engine = BrowserEngine(headless=True)
+ session = await engine.new_session()
+ try:
+ obs = await session.goto("https://stooq.com/q/l/?s=jnj.us&f=sd2t2ohlcv&h=&e=csv")
+ assert obs.url == "https://stooq.com/q/l/?s=jnj.us&f=sd2t2ohlcv&h=&e=csv"
+ assert obs.title == "Download"
+ assert "file download" in obs.accessibility_tree
+ assert "did not call page.goto()" in obs.accessibility_tree
+ finally:
+ await session.close()
+ await engine.stop()
diff --git a/tests/core/test_cache.py b/tests/core/test_cache.py
index 3571295..5deac13 100644
--- a/tests/core/test_cache.py
+++ b/tests/core/test_cache.py
@@ -15,6 +15,7 @@
safe_path_component,
url_to_cache_dir,
)
+from liveweb_arena.plugins.taostats.taostats import TaostatsPlugin
# ── CachedPage ──────────────────────────────────────────────────────
@@ -77,6 +78,19 @@ def test_to_dict_includes_a11y(self):
d = page.to_dict()
assert d["accessibility_tree"] == "tree"
+ def test_from_dict_enriches_sparse_a11y_from_html(self):
+ d = {
+ "url": "https://taostats.io/subnets/75",
+ "html": "