eval/top level evals

README.md

@@ -117,15 +117,23 @@ When running with the current stack, the flow is:
 
 We built an evaluation suite to test both tool-use behaviour and diagnosis quality. You can find details here:
 
-- [Evaluation overview](src/sre_agent/eval/README.md)
-- [Tool call evaluation](src/sre_agent/eval/tool_call/README.md)
-- [Diagnosis quality evaluation](src/sre_agent/eval/diagnosis_quality/README.md)
+- [Evaluation overview](evals/README.md)
+- [Tool call evaluation](evals/tool_call/README.md)
+- [Diagnosis quality evaluation](evals/diagnosis_quality/README.md)
 
 Run the suites with:
 
 ```bash
-uv run sre-agent-run-tool-call-eval
-uv run sre-agent-run-diagnosis-quality-eval
+uv run python -m evals.tool_call.run
+uv run python -m evals.diagnosis_quality.run
+```
+
+Assuming you already have Opik up and running. If not, please refer to the README in either of the eval suites for setup instructions. Once ready, run the following to install prerequisites:
+
+```bash
+export GITHUB_PERSONAL_ACCESS_TOKEN="..."
+export ANTHROPIC_API_KEY="..."
+uv sync --group eval
 ```
 
 # 🤔 Why We Built This

src/sre_agent/eval/README.md → evals/README.md

@@ -70,5 +70,5 @@ When the server is running, open [http://localhost:5173/](http://localhost:5173/
 
 For suite-specific details, see:
 
-- `src/sre_agent/eval/tool_call/README.md`
-- `src/sre_agent/eval/diagnosis_quality/README.md`
+- `evals/tool_call/README.md`
+- `evals/diagnosis_quality/README.md`

src/sre_agent/eval/common/__init__.py → evals/common/__init__.py

@@ -1,5 +1,5 @@
 """Common helpers for evaluation suites."""
 
-from sre_agent.eval.common.case_loader import load_json_case_models
+from evals.common.case_loader import load_json_case_models
 
 __all__ = ["load_json_case_models"]

src/sre_agent/eval/diagnosis_quality/README.md → evals/diagnosis_quality/README.md

@@ -22,11 +22,11 @@ The run is hybrid:
 
 Test cases are loaded from:
 
-- `src/sre_agent/eval/diagnosis_quality/dataset/test_cases`
+- `evals/diagnosis_quality/dataset/test_cases`
 
 Each case follows `DiagnosisQualityEvalCase` in:
 
-- `src/sre_agent/eval/diagnosis_quality/dataset/schema.py`
+- `evals/diagnosis_quality/dataset/schema.py`
 
 Key fields:
 
@@ -66,5 +66,5 @@ Run command:
 
 ```bash
 uv sync --group eval
-uv run sre-agent-run-diagnosis-quality-eval
+uv run python -m evals.diagnosis_quality.run
 ```

src/sre_agent/eval/diagnosis_quality/dataset/create_and_populate.py → evals/diagnosis_quality/dataset/create_and_populate.py

@@ -5,8 +5,8 @@
 
 from opik import Opik
 
-from sre_agent.eval.common.case_loader import load_json_case_models
-from sre_agent.eval.diagnosis_quality.dataset.schema import DiagnosisQualityEvalCase
+from evals.common.case_loader import load_json_case_models
+from evals.diagnosis_quality.dataset.schema import DiagnosisQualityEvalCase
 
 DEFAULT_DATASET_NAME = "sre-agent-diagnosis-quality"
 

src/sre_agent/eval/diagnosis_quality/experiment.py → evals/diagnosis_quality/experiment.py

@@ -9,27 +9,27 @@
 from opik.evaluation.evaluation_result import EvaluationResult
 from pydantic_ai import Agent
 
-from sre_agent.core.models import ErrorDiagnosis
-from sre_agent.core.prompts import SYSTEM_PROMPT
-from sre_agent.eval.diagnosis_quality.config import (
+from evals.diagnosis_quality.config import (
     DEFAULT_EXPERIMENT_NAME,
     DEFAULT_JUDGE_MODEL,
     DEFAULT_MODEL,
     DEFAULT_OPIK_PROJECT_NAME,
 )
-from sre_agent.eval.diagnosis_quality.dataset.create_and_populate import (
+from evals.diagnosis_quality.dataset.create_and_populate import (
     DEFAULT_DATASET_NAME,
     create_and_populate_dataset,
 )
-from sre_agent.eval.diagnosis_quality.dataset.schema import DiagnosisQualityEvalCase
-from sre_agent.eval.diagnosis_quality.github_toolset import build_github_toolset
-from sre_agent.eval.diagnosis_quality.metrics import (
+from evals.diagnosis_quality.dataset.schema import DiagnosisQualityEvalCase
+from evals.diagnosis_quality.github_toolset import build_github_toolset
+from evals.diagnosis_quality.metrics import (
     AffectedServicesMatch,
     RootCauseCorrectness,
     SuggestedFixesQuality,
 )
-from sre_agent.eval.diagnosis_quality.mocks import MockToolRuntime, build_mock_toolset
-from sre_agent.eval.diagnosis_quality.prompts import render_agent_prompt
+from evals.diagnosis_quality.mocks import MockToolRuntime, build_mock_toolset
+from evals.diagnosis_quality.prompts import render_agent_prompt
+from sre_agent.core.models import ErrorDiagnosis
+from sre_agent.core.prompts import SYSTEM_PROMPT
 
 
 def evaluation_task(dataset_item: dict[str, Any]) -> dict[str, Any]:

src/sre_agent/eval/diagnosis_quality/metrics/__init__.py → evals/diagnosis_quality/metrics/__init__.py

@@ -1,12 +1,12 @@
 """Metrics for diagnosis quality evaluation."""
 
-from sre_agent.eval.diagnosis_quality.metrics.affected_services_match import (
+from evals.diagnosis_quality.metrics.affected_services_match import (
     AffectedServicesMatch,
 )
-from sre_agent.eval.diagnosis_quality.metrics.root_cause_correctness import (
+from evals.diagnosis_quality.metrics.root_cause_correctness import (
     RootCauseCorrectness,
 )
-from sre_agent.eval.diagnosis_quality.metrics.suggested_fixes_quality import (
+from evals.diagnosis_quality.metrics.suggested_fixes_quality import (
     SuggestedFixesQuality,
 )
 

evals/diagnosis_quality/mocks/__init__.py

@@ -0,0 +1,9 @@
+"""Mock tools for diagnosis quality evaluation."""
+
+from evals.diagnosis_quality.mocks.runtime import MockToolRuntime
+from evals.diagnosis_quality.mocks.toolset import build_mock_toolset
+
+__all__ = [
+    "MockToolRuntime",
+    "build_mock_toolset",
+]

src/sre_agent/eval/diagnosis_quality/mocks/cloudwatch.py → evals/diagnosis_quality/mocks/cloudwatch.py

@@ -2,8 +2,8 @@
 
 import opik
 
+from evals.diagnosis_quality.mocks.runtime import MockToolRuntime
 from sre_agent.core.models import LogEntry, LogQueryResult
-from sre_agent.eval.diagnosis_quality.mocks.runtime import MockToolRuntime
 
 MOCK_TIMESTAMP = "2026-01-01T00:00:00+00:00"
 

src/sre_agent/eval/diagnosis_quality/mocks/runtime.py → evals/diagnosis_quality/mocks/runtime.py

@@ -2,7 +2,7 @@
 
 from dataclasses import dataclass
 
-from sre_agent.eval.diagnosis_quality.dataset.schema import DiagnosisQualityEvalCase
+from evals.diagnosis_quality.dataset.schema import DiagnosisQualityEvalCase
 
 
 @dataclass

src/sre_agent/eval/diagnosis_quality/mocks/toolset.py → evals/diagnosis_quality/mocks/toolset.py

@@ -4,10 +4,10 @@
 
 from pydantic_ai import FunctionToolset
 
+from evals.diagnosis_quality.mocks import cloudwatch as cloudwatch_mocks
+from evals.diagnosis_quality.mocks import slack as slack_mocks
+from evals.diagnosis_quality.mocks.runtime import MockToolRuntime
 from sre_agent.core.models import LogQueryResult
-from sre_agent.eval.diagnosis_quality.mocks import cloudwatch as cloudwatch_mocks
-from sre_agent.eval.diagnosis_quality.mocks import slack as slack_mocks
-from sre_agent.eval.diagnosis_quality.mocks.runtime import MockToolRuntime
 
 
 def build_mock_toolset(runtime: MockToolRuntime) -> FunctionToolset:

src/sre_agent/eval/diagnosis_quality/prompts.py → evals/diagnosis_quality/prompts.py

@@ -1,11 +1,11 @@
 """Prompt rendering for diagnosis quality evaluation."""
 
-from sre_agent.core.prompts import DIAGNOSIS_PROMPT_TEMPLATE
-from sre_agent.eval.diagnosis_quality.config import (
+from evals.diagnosis_quality.config import (
     DEFAULT_SLACK_CHANNEL_ID,
     DEFAULT_TIME_RANGE_MINUTES,
 )
-from sre_agent.eval.diagnosis_quality.dataset.schema import DiagnosisQualityEvalCase
+from evals.diagnosis_quality.dataset.schema import DiagnosisQualityEvalCase
+from sre_agent.core.prompts import DIAGNOSIS_PROMPT_TEMPLATE
 
 
 def render_agent_prompt(case: DiagnosisQualityEvalCase) -> str:

src/sre_agent/eval/diagnosis_quality/run.py → evals/diagnosis_quality/run.py

@@ -2,9 +2,9 @@
 
 from pydantic_ai.exceptions import UserError
 
-from sre_agent.eval.diagnosis_quality.config import DEFAULT_EXPERIMENT_NAME
-from sre_agent.eval.diagnosis_quality.dataset.create_and_populate import DEFAULT_DATASET_NAME
-from sre_agent.eval.diagnosis_quality.experiment import run_experiment
+from evals.diagnosis_quality.config import DEFAULT_EXPERIMENT_NAME
+from evals.diagnosis_quality.dataset.create_and_populate import DEFAULT_DATASET_NAME
+from evals.diagnosis_quality.experiment import run_experiment
 
 
 def main() -> None:

src/sre_agent/eval/tool_call/README.md → evals/tool_call/README.md

@@ -25,11 +25,11 @@ The run is hybrid:
 
 Test cases are loaded from:
 
-- `src/sre_agent/eval/tool_call/dataset/test_cases`
+- `evals/tool_call/dataset/test_cases`
 
 Each case follows `ToolCallEvalCase` in:
 
-- `src/sre_agent/eval/tool_call/dataset/schema.py`
+- `evals/tool_call/dataset/schema.py`
 
 Key fields:
 
@@ -73,7 +73,7 @@ Run command:
 
 ```bash
 uv sync --group eval
-uv run sre-agent-run-tool-call-eval
+uv run python -m evals.tool_call.run
 ```
 
 ## View Results in Opik

src/sre_agent/eval/tool_call/dataset/__init__.py → evals/tool_call/dataset/__init__.py

@@ -1,9 +1,9 @@
 """Dataset for tool call evaluation."""
 
-from sre_agent.eval.tool_call.dataset.create_and_populate import (
+from evals.tool_call.dataset.create_and_populate import (
     DEFAULT_DATASET_NAME,
     create_and_populate_dataset,
 )
-from sre_agent.eval.tool_call.dataset.schema import ToolCallEvalCase
+from evals.tool_call.dataset.schema import ToolCallEvalCase
 
 __all__ = ["create_and_populate_dataset", "ToolCallEvalCase", "DEFAULT_DATASET_NAME"]

src/sre_agent/eval/tool_call/dataset/create_and_populate.py → evals/tool_call/dataset/create_and_populate.py

@@ -5,8 +5,8 @@
 
 from opik import Opik
 
-from sre_agent.eval.common.case_loader import load_json_case_models
-from sre_agent.eval.tool_call.dataset.schema import ToolCallEvalCase
+from evals.common.case_loader import load_json_case_models
+from evals.tool_call.dataset.schema import ToolCallEvalCase
 
 DEFAULT_DATASET_NAME = "sre-agent-tool-call"
 

src/sre_agent/eval/tool_call/experiment.py → evals/tool_call/experiment.py

@@ -9,25 +9,25 @@
 from opik.evaluation.evaluation_result import EvaluationResult
 from pydantic_ai import Agent
 
-from sre_agent.core.models import ErrorDiagnosis
-from sre_agent.core.prompts import SYSTEM_PROMPT
-from sre_agent.eval.tool_call.config import (
+from evals.tool_call.config import (
     DEFAULT_EXPERIMENT_NAME,
     DEFAULT_MODEL,
     DEFAULT_OPIK_PROJECT_NAME,
 )
-from sre_agent.eval.tool_call.dataset.create_and_populate import (
+from evals.tool_call.dataset.create_and_populate import (
     DEFAULT_DATASET_NAME,
     create_and_populate_dataset,
 )
-from sre_agent.eval.tool_call.dataset.schema import ToolCallEvalCase
-from sre_agent.eval.tool_call.github_toolset import build_github_toolset
-from sre_agent.eval.tool_call.metrics.expected_tool_select_order import (
+from evals.tool_call.dataset.schema import ToolCallEvalCase
+from evals.tool_call.github_toolset import build_github_toolset
+from evals.tool_call.metrics.expected_tool_select_order import (
     ExpectedToolSelectOrder,
 )
-from sre_agent.eval.tool_call.metrics.expected_tool_selection import ExpectedToolSelection
-from sre_agent.eval.tool_call.mocks import MockToolRuntime, build_mock_toolset
-from sre_agent.eval.tool_call.prompts import render_agent_prompt
+from evals.tool_call.metrics.expected_tool_selection import ExpectedToolSelection
+from evals.tool_call.mocks import MockToolRuntime, build_mock_toolset
+from evals.tool_call.prompts import render_agent_prompt
+from sre_agent.core.models import ErrorDiagnosis
+from sre_agent.core.prompts import SYSTEM_PROMPT
 
 
 def evaluation_task(dataset_item: dict[str, Any]) -> dict[str, Any]:

evals/tool_call/metrics/__init__.py

@@ -0,0 +1,6 @@
+"""Metrics for tool call evaluation."""
+
+from evals.tool_call.metrics.expected_tool_select_order import ExpectedToolSelectOrder
+from evals.tool_call.metrics.expected_tool_selection import ExpectedToolSelection
+
+__all__ = ["ExpectedToolSelection", "ExpectedToolSelectOrder"]

src/sre_agent/eval/tool_call/metrics/expected_tool_select_order.py → evals/tool_call/metrics/expected_tool_select_order.py

@@ -5,7 +5,7 @@
 from opik.evaluation.metrics import base_metric, score_result
 from opik.message_processing.emulation.models import SpanModel
 
-from sre_agent.eval.tool_call.metrics.span_tools import extract_tool_names
+from evals.tool_call.metrics.span_tools import extract_tool_names
 
 
 class ExpectedToolSelectOrder(base_metric.BaseMetric):  # type: ignore[misc]

src/sre_agent/eval/tool_call/metrics/expected_tool_selection.py → evals/tool_call/metrics/expected_tool_selection.py

@@ -5,7 +5,7 @@
 from opik.evaluation.metrics import base_metric, score_result
 from opik.message_processing.emulation.models import SpanModel
 
-from sre_agent.eval.tool_call.metrics.span_tools import extract_tool_names
+from evals.tool_call.metrics.span_tools import extract_tool_names
 
 
 class ExpectedToolSelection(base_metric.BaseMetric):  # type: ignore[misc]

evals/tool_call/mocks/__init__.py

@@ -0,0 +1,6 @@
+"""Mock tools for tool call evaluation."""
+
+from evals.tool_call.mocks.runtime import MockToolRuntime
+from evals.tool_call.mocks.toolset import build_mock_toolset
+
+__all__ = ["MockToolRuntime", "build_mock_toolset"]

src/sre_agent/eval/tool_call/mocks/cloudwatch.py → evals/tool_call/mocks/cloudwatch.py

@@ -2,8 +2,8 @@
 
 import opik
 
+from evals.tool_call.mocks.runtime import MockToolRuntime
 from sre_agent.core.models import LogEntry, LogQueryResult
-from sre_agent.eval.tool_call.mocks.runtime import MockToolRuntime
 
 MOCK_TIMESTAMP = "2026-01-01T00:00:00+00:00"
 

src/sre_agent/eval/tool_call/mocks/runtime.py → evals/tool_call/mocks/runtime.py

@@ -2,7 +2,7 @@
 
 from dataclasses import dataclass
 
-from sre_agent.eval.tool_call.dataset.schema import ToolCallEvalCase
+from evals.tool_call.dataset.schema import ToolCallEvalCase
 
 
 @dataclass

src/sre_agent/eval/tool_call/mocks/toolset.py → evals/tool_call/mocks/toolset.py

@@ -4,10 +4,10 @@
 
 from pydantic_ai import FunctionToolset
 
+from evals.tool_call.mocks import cloudwatch as cloudwatch_mocks
+from evals.tool_call.mocks import slack as slack_mocks
+from evals.tool_call.mocks.runtime import MockToolRuntime
 from sre_agent.core.models import LogQueryResult
-from sre_agent.eval.tool_call.mocks import cloudwatch as cloudwatch_mocks
-from sre_agent.eval.tool_call.mocks import slack as slack_mocks
-from sre_agent.eval.tool_call.mocks.runtime import MockToolRuntime
 
 
 def build_mock_toolset(runtime: MockToolRuntime) -> FunctionToolset:

src/sre_agent/eval/tool_call/prompts.py → evals/tool_call/prompts.py

@@ -1,11 +1,11 @@
 """Prompt rendering for tool call evaluation."""
 
-from sre_agent.core.prompts import DIAGNOSIS_PROMPT_TEMPLATE
-from sre_agent.eval.tool_call.config import (
+from evals.tool_call.config import (
     DEFAULT_SLACK_CHANNEL_ID,
     DEFAULT_TIME_RANGE_MINUTES,
 )
-from sre_agent.eval.tool_call.dataset.schema import ToolCallEvalCase
+from evals.tool_call.dataset.schema import ToolCallEvalCase
+from sre_agent.core.prompts import DIAGNOSIS_PROMPT_TEMPLATE
 
 
 def render_agent_prompt(case: ToolCallEvalCase) -> str:

src/sre_agent/eval/tool_call/run.py → evals/tool_call/run.py

@@ -2,9 +2,9 @@
 
 from pydantic_ai.exceptions import UserError
 
-from sre_agent.eval.tool_call.config import DEFAULT_EXPERIMENT_NAME
-from sre_agent.eval.tool_call.dataset.create_and_populate import DEFAULT_DATASET_NAME
-from sre_agent.eval.tool_call.experiment import run_experiment
+from evals.tool_call.config import DEFAULT_EXPERIMENT_NAME
+from evals.tool_call.dataset.create_and_populate import DEFAULT_DATASET_NAME
+from evals.tool_call.experiment import run_experiment
 
 
 def main() -> None:

pyproject.toml

@@ -19,8 +19,6 @@ dependencies = [
 
 [project.scripts]
 sre-agent = "sre_agent.cli.main:main"
-sre-agent-run-tool-call-eval = "sre_agent.eval.tool_call.run:main"
-sre-agent-run-diagnosis-quality-eval = "sre_agent.eval.diagnosis_quality.run:main"
 
 [dependency-groups]
 dev = [