eval/top level evals

README.md

@@ -117,15 +117,15 @@ When running with the current stack, the flow is:
 
 We built an evaluation suite to test both tool-use behaviour and diagnosis quality. You can find details here:
 
-- [Evaluation overview](src/sre_agent/eval/README.md)
-- [Tool call evaluation](src/sre_agent/eval/tool_call/README.md)
-- [Diagnosis quality evaluation](src/sre_agent/eval/diagnosis_quality/README.md)
+- [Evaluation overview](evals/README.md)
+- [Tool call evaluation](evals/tool_call/README.md)
+- [Diagnosis quality evaluation](evals/diagnosis_quality/README.md)
 
 Run the suites with:
 
 ```bash
-uv run sre-agent-run-tool-call-eval
-uv run sre-agent-run-diagnosis-quality-eval
+uv run python -m evals.tool_call.run
+uv run python -m evals.diagnosis_quality.run
 ```
 
 # 🤔 Why We Built This

src/sre_agent/eval/README.md → evals/README.md

@@ -70,5 +70,5 @@ When the server is running, open [http://localhost:5173/](http://localhost:5173/
 
 For suite-specific details, see:
 
-- `src/sre_agent/eval/tool_call/README.md`
-- `src/sre_agent/eval/diagnosis_quality/README.md`
+- `evals/tool_call/README.md`
+- `evals/diagnosis_quality/README.md`

src/sre_agent/eval/common/__init__.py → evals/common/__init__.py

@@ -1,5 +1,5 @@
 """Common helpers for evaluation suites."""
 
-from sre_agent.eval.common.case_loader import load_json_case_models
+from evals.common.case_loader import load_json_case_models
 
 __all__ = ["load_json_case_models"]

src/sre_agent/eval/diagnosis_quality/README.md → evals/diagnosis_quality/README.md

@@ -22,11 +22,11 @@ The run is hybrid:
 
 Test cases are loaded from:
 
-- `src/sre_agent/eval/diagnosis_quality/dataset/test_cases`
+- `evals/diagnosis_quality/dataset/test_cases`
 
 Each case follows `DiagnosisQualityEvalCase` in:
 
-- `src/sre_agent/eval/diagnosis_quality/dataset/schema.py`
+- `evals/diagnosis_quality/dataset/schema.py`
 
 Key fields:
 

src/sre_agent/eval/diagnosis_quality/dataset/create_and_populate.py → evals/diagnosis_quality/dataset/create_and_populate.py

@@ -5,8 +5,8 @@
 
 from opik import Opik
 
-from sre_agent.eval.common.case_loader import load_json_case_models
-from sre_agent.eval.diagnosis_quality.dataset.schema import DiagnosisQualityEvalCase
+from evals.common.case_loader import load_json_case_models
+from evals.diagnosis_quality.dataset.schema import DiagnosisQualityEvalCase
 
 DEFAULT_DATASET_NAME = "sre-agent-diagnosis-quality"
 

src/sre_agent/eval/diagnosis_quality/experiment.py → evals/diagnosis_quality/experiment.py

@@ -9,27 +9,27 @@
 from opik.evaluation.evaluation_result import EvaluationResult
 from pydantic_ai import Agent
 
-from sre_agent.core.models import ErrorDiagnosis
-from sre_agent.core.prompts import SYSTEM_PROMPT
-from sre_agent.eval.diagnosis_quality.config import (
+from evals.diagnosis_quality.config import (
     DEFAULT_EXPERIMENT_NAME,
     DEFAULT_JUDGE_MODEL,
     DEFAULT_MODEL,
     DEFAULT_OPIK_PROJECT_NAME,
 )
-from sre_agent.eval.diagnosis_quality.dataset.create_and_populate import (
+from evals.diagnosis_quality.dataset.create_and_populate import (
     DEFAULT_DATASET_NAME,
     create_and_populate_dataset,
 )
-from sre_agent.eval.diagnosis_quality.dataset.schema import DiagnosisQualityEvalCase
-from sre_agent.eval.diagnosis_quality.github_toolset import build_github_toolset
-from sre_agent.eval.diagnosis_quality.metrics import (
+from evals.diagnosis_quality.dataset.schema import DiagnosisQualityEvalCase
+from evals.diagnosis_quality.github_toolset import build_github_toolset
+from evals.diagnosis_quality.metrics import (
     AffectedServicesMatch,
     RootCauseCorrectness,
     SuggestedFixesQuality,
 )
-from sre_agent.eval.diagnosis_quality.mocks import MockToolRuntime, build_mock_toolset
-from sre_agent.eval.diagnosis_quality.prompts import render_agent_prompt
+from evals.diagnosis_quality.mocks import MockToolRuntime, build_mock_toolset
+from evals.diagnosis_quality.prompts import render_agent_prompt
+from sre_agent.core.models import ErrorDiagnosis
+from sre_agent.core.prompts import SYSTEM_PROMPT
 
 
 def evaluation_task(dataset_item: dict[str, Any]) -> dict[str, Any]:

src/sre_agent/eval/diagnosis_quality/metrics/__init__.py → evals/diagnosis_quality/metrics/__init__.py

@@ -1,12 +1,12 @@
 """Metrics for diagnosis quality evaluation."""
 
-from sre_agent.eval.diagnosis_quality.metrics.affected_services_match import (
+from evals.diagnosis_quality.metrics.affected_services_match import (
     AffectedServicesMatch,
 )
-from sre_agent.eval.diagnosis_quality.metrics.root_cause_correctness import (
+from evals.diagnosis_quality.metrics.root_cause_correctness import (
     RootCauseCorrectness,
 )
-from sre_agent.eval.diagnosis_quality.metrics.suggested_fixes_quality import (
+from evals.diagnosis_quality.metrics.suggested_fixes_quality import (
     SuggestedFixesQuality,
 )
 

evals/diagnosis_quality/mocks/__init__.py

@@ -0,0 +1,9 @@
+"""Mock tools for diagnosis quality evaluation."""
+
+from evals.diagnosis_quality.mocks.runtime import MockToolRuntime
+from evals.diagnosis_quality.mocks.toolset import build_mock_toolset
+
+__all__ = [
+    "MockToolRuntime",
+    "build_mock_toolset",
+]

src/sre_agent/eval/diagnosis_quality/mocks/cloudwatch.py → evals/diagnosis_quality/mocks/cloudwatch.py

@@ -2,8 +2,8 @@
 
 import opik
 
+from evals.diagnosis_quality.mocks.runtime import MockToolRuntime
 from sre_agent.core.models import LogEntry, LogQueryResult
-from sre_agent.eval.diagnosis_quality.mocks.runtime import MockToolRuntime
 
 MOCK_TIMESTAMP = "2026-01-01T00:00:00+00:00"
 

src/sre_agent/eval/diagnosis_quality/mocks/runtime.py → evals/diagnosis_quality/mocks/runtime.py

@@ -2,7 +2,7 @@
 
 from dataclasses import dataclass
 
-from sre_agent.eval.diagnosis_quality.dataset.schema import DiagnosisQualityEvalCase
+from evals.diagnosis_quality.dataset.schema import DiagnosisQualityEvalCase
 
 
 @dataclass

src/sre_agent/eval/diagnosis_quality/mocks/toolset.py → evals/diagnosis_quality/mocks/toolset.py

@@ -4,10 +4,10 @@
 
 from pydantic_ai import FunctionToolset
 
+from evals.diagnosis_quality.mocks import cloudwatch as cloudwatch_mocks
+from evals.diagnosis_quality.mocks import slack as slack_mocks
+from evals.diagnosis_quality.mocks.runtime import MockToolRuntime
 from sre_agent.core.models import LogQueryResult
-from sre_agent.eval.diagnosis_quality.mocks import cloudwatch as cloudwatch_mocks
-from sre_agent.eval.diagnosis_quality.mocks import slack as slack_mocks
-from sre_agent.eval.diagnosis_quality.mocks.runtime import MockToolRuntime
 
 
 def build_mock_toolset(runtime: MockToolRuntime) -> FunctionToolset:

src/sre_agent/eval/diagnosis_quality/prompts.py → evals/diagnosis_quality/prompts.py

@@ -1,11 +1,11 @@
 """Prompt rendering for diagnosis quality evaluation."""
 
-from sre_agent.core.prompts import DIAGNOSIS_PROMPT_TEMPLATE
-from sre_agent.eval.diagnosis_quality.config import (
+from evals.diagnosis_quality.config import (
     DEFAULT_SLACK_CHANNEL_ID,
     DEFAULT_TIME_RANGE_MINUTES,
 )
-from sre_agent.eval.diagnosis_quality.dataset.schema import DiagnosisQualityEvalCase
+from evals.diagnosis_quality.dataset.schema import DiagnosisQualityEvalCase
+from sre_agent.core.prompts import DIAGNOSIS_PROMPT_TEMPLATE
 
 
 def render_agent_prompt(case: DiagnosisQualityEvalCase) -> str:

src/sre_agent/eval/diagnosis_quality/run.py → evals/diagnosis_quality/run.py

@@ -2,9 +2,9 @@
 
 from pydantic_ai.exceptions import UserError
 
-from sre_agent.eval.diagnosis_quality.config import DEFAULT_EXPERIMENT_NAME
-from sre_agent.eval.diagnosis_quality.dataset.create_and_populate import DEFAULT_DATASET_NAME
-from sre_agent.eval.diagnosis_quality.experiment import run_experiment
+from evals.diagnosis_quality.config import DEFAULT_EXPERIMENT_NAME
+from evals.diagnosis_quality.dataset.create_and_populate import DEFAULT_DATASET_NAME
+from evals.diagnosis_quality.experiment import run_experiment
 
 
 def main() -> None:

src/sre_agent/eval/tool_call/README.md → evals/tool_call/README.md

@@ -25,11 +25,11 @@ The run is hybrid:
 
 Test cases are loaded from:
 
-- `src/sre_agent/eval/tool_call/dataset/test_cases`
+- `evals/tool_call/dataset/test_cases`
 
 Each case follows `ToolCallEvalCase` in:
 
-- `src/sre_agent/eval/tool_call/dataset/schema.py`
+- `evals/tool_call/dataset/schema.py`
 
 Key fields:
 

src/sre_agent/eval/tool_call/dataset/__init__.py → evals/tool_call/dataset/__init__.py

@@ -1,9 +1,9 @@
 """Dataset for tool call evaluation."""
 
-from sre_agent.eval.tool_call.dataset.create_and_populate import (
+from evals.tool_call.dataset.create_and_populate import (
     DEFAULT_DATASET_NAME,
     create_and_populate_dataset,
 )
-from sre_agent.eval.tool_call.dataset.schema import ToolCallEvalCase
+from evals.tool_call.dataset.schema import ToolCallEvalCase
 
 __all__ = ["create_and_populate_dataset", "ToolCallEvalCase", "DEFAULT_DATASET_NAME"]

src/sre_agent/eval/tool_call/dataset/create_and_populate.py → evals/tool_call/dataset/create_and_populate.py

@@ -5,8 +5,8 @@
 
 from opik import Opik
 
-from sre_agent.eval.common.case_loader import load_json_case_models
-from sre_agent.eval.tool_call.dataset.schema import ToolCallEvalCase
+from evals.common.case_loader import load_json_case_models
+from evals.tool_call.dataset.schema import ToolCallEvalCase
 
 DEFAULT_DATASET_NAME = "sre-agent-tool-call"
 

src/sre_agent/eval/tool_call/experiment.py → evals/tool_call/experiment.py

@@ -9,25 +9,25 @@
 from opik.evaluation.evaluation_result import EvaluationResult
 from pydantic_ai import Agent
 
-from sre_agent.core.models import ErrorDiagnosis
-from sre_agent.core.prompts import SYSTEM_PROMPT
-from sre_agent.eval.tool_call.config import (
+from evals.tool_call.config import (
     DEFAULT_EXPERIMENT_NAME,
     DEFAULT_MODEL,
     DEFAULT_OPIK_PROJECT_NAME,
 )
-from sre_agent.eval.tool_call.dataset.create_and_populate import (
+from evals.tool_call.dataset.create_and_populate import (
     DEFAULT_DATASET_NAME,
     create_and_populate_dataset,
 )
-from sre_agent.eval.tool_call.dataset.schema import ToolCallEvalCase
-from sre_agent.eval.tool_call.github_toolset import build_github_toolset
-from sre_agent.eval.tool_call.metrics.expected_tool_select_order import (
+from evals.tool_call.dataset.schema import ToolCallEvalCase
+from evals.tool_call.github_toolset import build_github_toolset
+from evals.tool_call.metrics.expected_tool_select_order import (
     ExpectedToolSelectOrder,
 )
-from sre_agent.eval.tool_call.metrics.expected_tool_selection import ExpectedToolSelection
-from sre_agent.eval.tool_call.mocks import MockToolRuntime, build_mock_toolset
-from sre_agent.eval.tool_call.prompts import render_agent_prompt
+from evals.tool_call.metrics.expected_tool_selection import ExpectedToolSelection
+from evals.tool_call.mocks import MockToolRuntime, build_mock_toolset
+from evals.tool_call.prompts import render_agent_prompt
+from sre_agent.core.models import ErrorDiagnosis
+from sre_agent.core.prompts import SYSTEM_PROMPT
 
 
 def evaluation_task(dataset_item: dict[str, Any]) -> dict[str, Any]:

evals/tool_call/metrics/__init__.py

@@ -0,0 +1,6 @@
+"""Metrics for tool call evaluation."""
+
+from evals.tool_call.metrics.expected_tool_select_order import ExpectedToolSelectOrder
+from evals.tool_call.metrics.expected_tool_selection import ExpectedToolSelection
+
+__all__ = ["ExpectedToolSelection", "ExpectedToolSelectOrder"]

src/sre_agent/eval/tool_call/metrics/expected_tool_select_order.py → evals/tool_call/metrics/expected_tool_select_order.py

@@ -5,7 +5,7 @@
 from opik.evaluation.metrics import base_metric, score_result
 from opik.message_processing.emulation.models import SpanModel
 
-from sre_agent.eval.tool_call.metrics.span_tools import extract_tool_names
+from evals.tool_call.metrics.span_tools import extract_tool_names
 
 
 class ExpectedToolSelectOrder(base_metric.BaseMetric):  # type: ignore[misc]

src/sre_agent/eval/tool_call/metrics/expected_tool_selection.py → evals/tool_call/metrics/expected_tool_selection.py

@@ -5,7 +5,7 @@
 from opik.evaluation.metrics import base_metric, score_result
 from opik.message_processing.emulation.models import SpanModel
 
-from sre_agent.eval.tool_call.metrics.span_tools import extract_tool_names
+from evals.tool_call.metrics.span_tools import extract_tool_names
 
 
 class ExpectedToolSelection(base_metric.BaseMetric):  # type: ignore[misc]

evals/tool_call/mocks/__init__.py

@@ -0,0 +1,6 @@
+"""Mock tools for tool call evaluation."""
+
+from evals.tool_call.mocks.runtime import MockToolRuntime
+from evals.tool_call.mocks.toolset import build_mock_toolset
+
+__all__ = ["MockToolRuntime", "build_mock_toolset"]

src/sre_agent/eval/tool_call/mocks/cloudwatch.py → evals/tool_call/mocks/cloudwatch.py

@@ -2,8 +2,8 @@
 
 import opik
 
+from evals.tool_call.mocks.runtime import MockToolRuntime
 from sre_agent.core.models import LogEntry, LogQueryResult
-from sre_agent.eval.tool_call.mocks.runtime import MockToolRuntime
 
 MOCK_TIMESTAMP = "2026-01-01T00:00:00+00:00"
 

src/sre_agent/eval/tool_call/mocks/runtime.py → evals/tool_call/mocks/runtime.py

@@ -2,7 +2,7 @@
 
 from dataclasses import dataclass
 
-from sre_agent.eval.tool_call.dataset.schema import ToolCallEvalCase
+from evals.tool_call.dataset.schema import ToolCallEvalCase
 
 
 @dataclass

src/sre_agent/eval/tool_call/mocks/toolset.py → evals/tool_call/mocks/toolset.py

@@ -4,10 +4,10 @@
 
 from pydantic_ai import FunctionToolset
 
+from evals.tool_call.mocks import cloudwatch as cloudwatch_mocks
+from evals.tool_call.mocks import slack as slack_mocks
+from evals.tool_call.mocks.runtime import MockToolRuntime
 from sre_agent.core.models import LogQueryResult
-from sre_agent.eval.tool_call.mocks import cloudwatch as cloudwatch_mocks
-from sre_agent.eval.tool_call.mocks import slack as slack_mocks
-from sre_agent.eval.tool_call.mocks.runtime import MockToolRuntime
 
 
 def build_mock_toolset(runtime: MockToolRuntime) -> FunctionToolset:

src/sre_agent/eval/tool_call/prompts.py → evals/tool_call/prompts.py

@@ -1,11 +1,11 @@
 """Prompt rendering for tool call evaluation."""
 
-from sre_agent.core.prompts import DIAGNOSIS_PROMPT_TEMPLATE
-from sre_agent.eval.tool_call.config import (
+from evals.tool_call.config import (
     DEFAULT_SLACK_CHANNEL_ID,
     DEFAULT_TIME_RANGE_MINUTES,
 )
-from sre_agent.eval.tool_call.dataset.schema import ToolCallEvalCase
+from evals.tool_call.dataset.schema import ToolCallEvalCase
+from sre_agent.core.prompts import DIAGNOSIS_PROMPT_TEMPLATE
 
 
 def render_agent_prompt(case: ToolCallEvalCase) -> str:

src/sre_agent/eval/tool_call/run.py → evals/tool_call/run.py

@@ -2,9 +2,9 @@
 
 from pydantic_ai.exceptions import UserError
 
-from sre_agent.eval.tool_call.config import DEFAULT_EXPERIMENT_NAME
-from sre_agent.eval.tool_call.dataset.create_and_populate import DEFAULT_DATASET_NAME
-from sre_agent.eval.tool_call.experiment import run_experiment
+from evals.tool_call.config import DEFAULT_EXPERIMENT_NAME
+from evals.tool_call.dataset.create_and_populate import DEFAULT_DATASET_NAME
+from evals.tool_call.experiment import run_experiment
 
 
 def main() -> None:

pyproject.toml

@@ -19,8 +19,6 @@ dependencies = [
 
 [project.scripts]
 sre-agent = "sre_agent.cli.main:main"
-sre-agent-run-tool-call-eval = "sre_agent.eval.tool_call.run:main"
-sre-agent-run-diagnosis-quality-eval = "sre_agent.eval.diagnosis_quality.run:main"
 
 [dependency-groups]
 dev = [