Small fixes and poe checks

Author: matheusmaldaner

src/magentic_ui/eval/benchmarks/sentinelbench/sentinelbench.py

@@ -140,9 +140,16 @@ def load_dataset(self) -> None:
 
             # Add optional fields if they exist and are not empty
             if row_dict.get("relative_vs_absolute"):
-                base_metadata["relative_vs_absolute"] = str(row_dict["relative_vs_absolute"])
-            if "adversarial_attacks" in row_dict and row_dict["adversarial_attacks"] is not None:
-                base_metadata["adversarial_attacks"] = str(row_dict["adversarial_attacks"])
+                base_metadata["relative_vs_absolute"] = str(
+                    row_dict["relative_vs_absolute"]
+                )
+            if (
+                "adversarial_attacks" in row_dict
+                and row_dict["adversarial_attacks"] is not None
+            ):
+                base_metadata["adversarial_attacks"] = str(
+                    row_dict["adversarial_attacks"]
+                )
             if row_dict.get("failure_tolerance"):
                 base_metadata["failure_tolerance"] = str(row_dict["failure_tolerance"])
 

src/magentic_ui/eval/benchmarks/sentinelbench/task_variants.py

@@ -4,16 +4,6 @@
 
 from typing import Dict, List, Any, cast
 
-try:
-    from ...models import BaseTask  # type: ignore
-except ImportError:
-    # Handle case when running tools directly (not as module)
-    import sys
-    from pathlib import Path
-
-    sys.path.append(str(Path(__file__).parent.parent.parent))
-    from models import BaseTask  # type: ignore
-
 # Define task variants with different parameter values for SentinelBench
 SENTINELBENCH_TASK_VARIANTS = {
     # Time-based variants
@@ -262,18 +252,16 @@ def calculate_sentinelbench_timeout(
     default_timeout = 60 * default_timeout_minutes
 
     # Check if this is a SentinelBench task with parameter_value
-    if (
-        hasattr(task, "metadata")
-        and task.metadata
-        and isinstance(task.metadata, dict)
-    ):
+    if hasattr(task, "metadata") and task.metadata and isinstance(task.metadata, dict):
         task_metadata: Any = getattr(task, "metadata", {})
         metadata: Dict[str, Any] = cast(Dict[str, Any], task_metadata)
         if "parameter_value" in metadata:
             parameter_value: Any = metadata["parameter_value"]
 
             # Get base task ID (remove parameter part if present)
-            base_task_id = task.id.split("/")[0] if hasattr(task, "id") and task.id else ""
+            base_task_id = (
+                task.id.split("/")[0] if hasattr(task, "id") and task.id else ""
+            )
 
             # Duration-based tasks
             if base_task_id in DURATION_TASKS:
@@ -306,16 +294,14 @@ def get_timeout_display_info(task: Any, timeout_seconds: int) -> str:
     timeout_minutes = int(timeout_seconds / 60)
 
     # Check if this is a SentinelBench task with parameter_value
-    if (
-        hasattr(task, "metadata")
-        and task.metadata
-        and isinstance(task.metadata, dict)
-    ):
+    if hasattr(task, "metadata") and task.metadata and isinstance(task.metadata, dict):
         task_metadata: Any = getattr(task, "metadata", {})
         metadata: Dict[str, Any] = cast(Dict[str, Any], task_metadata)
         if "parameter_value" in metadata:
             parameter_value: Any = metadata["parameter_value"]
-            base_task_id = task.id.split("/")[0] if hasattr(task, "id") and task.id else ""
+            base_task_id = (
+                task.id.split("/")[0] if hasattr(task, "id") and task.id else ""
+            )
 
             # Duration-based tasks
             if base_task_id in DURATION_TASKS:

src/magentic_ui/eval/benchmarks/sentinelbench/tools/single_task_performance.py

@@ -45,41 +45,13 @@
 from pathlib import Path
 import logging
 
+# Import model pricing from task_variants
+from ..task_variants import MODEL_PRICING
+
 # Set up logging
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
-# Model pricing (from compare_sentinel_performance.py)
-MODEL_PRICING = {
-    # OpenAI GPT
-    "gpt-4o": {"input": 0.005, "output": 0.02},  # Standard
-    "gpt-4o-batch": {"input": 0.0025, "output": 0.01},  # Batch/Azure
-    "gpt-4o-2024-08-06": {"input": 0.005, "output": 0.02},
-    "gpt-4o-2024-11-20": {"input": 0.005, "output": 0.02},
-    "gpt-4o-mini": {
-        "input": 0.0006,
-        "output": 0.0024,
-    },  # Standard (Batch = 0.0003/0.0012)
-    "gpt-4o-mini-2024-07-18": {"input": 0.0006, "output": 0.0024},
-    "gpt-4": {"input": 0.03, "output": 0.06},
-    "gpt-4-turbo": {"input": 0.01, "output": 0.03},
-    "gpt-3.5-turbo": {"input": 0.0005, "output": 0.0015},
-    "gpt-5-mini": {
-        "input": 0.00025,
-        "output": 0.002,
-    },  # GPT-5 mini: $0.25/$2.00 per 1M tokens
-    # Anthropic Claude
-    "claude-3-5-sonnet-20241022": {"input": 0.003, "output": 0.015},
-    "claude-3-5-sonnet-20240620": {"input": 0.003, "output": 0.015},
-    "claude-3-opus-20240229": {"input": 0.015, "output": 0.075},
-    "claude-3-haiku-20240307": {"input": 0.00025, "output": 0.00125},
-    # Google Gemini
-    "gemini-1.5-pro": {"input": 0.00125, "output": 0.005},  # ≤128k ctx
-    "gemini-1.5-pro-extended": {"input": 0.0025, "output": 0.01},  # >128k ctx
-    "gemini-1.5-flash": {"input": 0.000075, "output": 0.0003},  # ≤128k ctx
-    "gemini-1.5-flash-extended": {"input": 0.00015, "output": 0.0006},  # >128k ctx
-}
-
 
 def format_time_dimension(seconds: int) -> str:
     """Format time dimension for display on plots."""

src/magentic_ui/eval/benchmarks/sentinelbench/tools/task_type_comparison.py

@@ -5,7 +5,7 @@
 Creates combined plots showing accuracy, latency, and cost scaling averaged across task types.
 
 Usage:
-    python analyze_task_types.py --sentinel-csv plots/FINAL/all_tasks_with_sentinel.csv \
+    python task_type_comparison.py --sentinel-csv plots/FINAL/all_tasks_with_sentinel.csv \
                                  --non-sentinel-csv plots/FINAL/all_tasks_without_sentinel.csv \
                                  --model gpt-5-mini \
                                  --output-dir plots/task_types