diff --git a/README.md b/README.md index 322f9815..19f71f9c 100644 --- a/README.md +++ b/README.md @@ -117,15 +117,23 @@ When running with the current stack, the flow is: We built an evaluation suite to test both tool-use behaviour and diagnosis quality. You can find details here: -- [Evaluation overview](src/sre_agent/eval/README.md) -- [Tool call evaluation](src/sre_agent/eval/tool_call/README.md) -- [Diagnosis quality evaluation](src/sre_agent/eval/diagnosis_quality/README.md) +- [Evaluation overview](evals/README.md) +- [Tool call evaluation](evals/tool_call/README.md) +- [Diagnosis quality evaluation](evals/diagnosis_quality/README.md) Run the suites with: ```bash -uv run sre-agent-run-tool-call-eval -uv run sre-agent-run-diagnosis-quality-eval +uv run python -m evals.tool_call.run +uv run python -m evals.diagnosis_quality.run +``` + +Assuming you already have Opik up and running. If not, please refer to the README in either of the eval suites for setup instructions. Once ready, run the following to install prerequisites: + +```bash +export GITHUB_PERSONAL_ACCESS_TOKEN="..." +export ANTHROPIC_API_KEY="..." +uv sync --group eval ``` # 🤔 Why We Built This diff --git a/src/sre_agent/eval/README.md b/evals/README.md similarity index 94% rename from src/sre_agent/eval/README.md rename to evals/README.md index 081f0df7..551879c3 100644 --- a/src/sre_agent/eval/README.md +++ b/evals/README.md @@ -70,5 +70,5 @@ When the server is running, open [http://localhost:5173/](http://localhost:5173/ For suite-specific details, see: -- `src/sre_agent/eval/tool_call/README.md` -- `src/sre_agent/eval/diagnosis_quality/README.md` +- `evals/tool_call/README.md` +- `evals/diagnosis_quality/README.md` diff --git a/src/sre_agent/eval/__init__.py b/evals/__init__.py similarity index 100% rename from src/sre_agent/eval/__init__.py rename to evals/__init__.py diff --git a/src/sre_agent/eval/common/__init__.py b/evals/common/__init__.py similarity index 54% rename from src/sre_agent/eval/common/__init__.py rename to evals/common/__init__.py index f14c3d26..526c4910 100644 --- a/src/sre_agent/eval/common/__init__.py +++ b/evals/common/__init__.py @@ -1,5 +1,5 @@ """Common helpers for evaluation suites.""" -from sre_agent.eval.common.case_loader import load_json_case_models +from evals.common.case_loader import load_json_case_models __all__ = ["load_json_case_models"] diff --git a/src/sre_agent/eval/common/case_loader.py b/evals/common/case_loader.py similarity index 100% rename from src/sre_agent/eval/common/case_loader.py rename to evals/common/case_loader.py diff --git a/src/sre_agent/eval/diagnosis_quality/README.md b/evals/diagnosis_quality/README.md similarity index 89% rename from src/sre_agent/eval/diagnosis_quality/README.md rename to evals/diagnosis_quality/README.md index ad3bbf9f..23824573 100644 --- a/src/sre_agent/eval/diagnosis_quality/README.md +++ b/evals/diagnosis_quality/README.md @@ -22,11 +22,11 @@ The run is hybrid: Test cases are loaded from: -- `src/sre_agent/eval/diagnosis_quality/dataset/test_cases` +- `evals/diagnosis_quality/dataset/test_cases` Each case follows `DiagnosisQualityEvalCase` in: -- `src/sre_agent/eval/diagnosis_quality/dataset/schema.py` +- `evals/diagnosis_quality/dataset/schema.py` Key fields: @@ -66,5 +66,5 @@ Run command: ```bash uv sync --group eval -uv run sre-agent-run-diagnosis-quality-eval +uv run python -m evals.diagnosis_quality.run ``` diff --git a/src/sre_agent/eval/diagnosis_quality/__init__.py b/evals/diagnosis_quality/__init__.py similarity index 100% rename from src/sre_agent/eval/diagnosis_quality/__init__.py rename to evals/diagnosis_quality/__init__.py diff --git a/src/sre_agent/eval/diagnosis_quality/config.py b/evals/diagnosis_quality/config.py similarity index 100% rename from src/sre_agent/eval/diagnosis_quality/config.py rename to evals/diagnosis_quality/config.py diff --git a/src/sre_agent/eval/diagnosis_quality/dataset/__init__.py b/evals/diagnosis_quality/dataset/__init__.py similarity index 100% rename from src/sre_agent/eval/diagnosis_quality/dataset/__init__.py rename to evals/diagnosis_quality/dataset/__init__.py diff --git a/src/sre_agent/eval/diagnosis_quality/dataset/create_and_populate.py b/evals/diagnosis_quality/dataset/create_and_populate.py similarity index 87% rename from src/sre_agent/eval/diagnosis_quality/dataset/create_and_populate.py rename to evals/diagnosis_quality/dataset/create_and_populate.py index 735e5fec..d338dc1e 100644 --- a/src/sre_agent/eval/diagnosis_quality/dataset/create_and_populate.py +++ b/evals/diagnosis_quality/dataset/create_and_populate.py @@ -5,8 +5,8 @@ from opik import Opik -from sre_agent.eval.common.case_loader import load_json_case_models -from sre_agent.eval.diagnosis_quality.dataset.schema import DiagnosisQualityEvalCase +from evals.common.case_loader import load_json_case_models +from evals.diagnosis_quality.dataset.schema import DiagnosisQualityEvalCase DEFAULT_DATASET_NAME = "sre-agent-diagnosis-quality" diff --git a/src/sre_agent/eval/diagnosis_quality/dataset/schema.py b/evals/diagnosis_quality/dataset/schema.py similarity index 100% rename from src/sre_agent/eval/diagnosis_quality/dataset/schema.py rename to evals/diagnosis_quality/dataset/schema.py diff --git a/src/sre_agent/eval/diagnosis_quality/dataset/test_cases/cartservice_test_case_01.json b/evals/diagnosis_quality/dataset/test_cases/cartservice_test_case_01.json similarity index 100% rename from src/sre_agent/eval/diagnosis_quality/dataset/test_cases/cartservice_test_case_01.json rename to evals/diagnosis_quality/dataset/test_cases/cartservice_test_case_01.json diff --git a/src/sre_agent/eval/diagnosis_quality/dataset/test_cases/cartservice_test_case_02.json b/evals/diagnosis_quality/dataset/test_cases/cartservice_test_case_02.json similarity index 100% rename from src/sre_agent/eval/diagnosis_quality/dataset/test_cases/cartservice_test_case_02.json rename to evals/diagnosis_quality/dataset/test_cases/cartservice_test_case_02.json diff --git a/src/sre_agent/eval/diagnosis_quality/dataset/test_cases/currencyservice_test_case_01.json b/evals/diagnosis_quality/dataset/test_cases/currencyservice_test_case_01.json similarity index 100% rename from src/sre_agent/eval/diagnosis_quality/dataset/test_cases/currencyservice_test_case_01.json rename to evals/diagnosis_quality/dataset/test_cases/currencyservice_test_case_01.json diff --git a/src/sre_agent/eval/diagnosis_quality/experiment.py b/evals/diagnosis_quality/experiment.py similarity index 89% rename from src/sre_agent/eval/diagnosis_quality/experiment.py rename to evals/diagnosis_quality/experiment.py index 00232910..8b076830 100644 --- a/src/sre_agent/eval/diagnosis_quality/experiment.py +++ b/evals/diagnosis_quality/experiment.py @@ -9,27 +9,27 @@ from opik.evaluation.evaluation_result import EvaluationResult from pydantic_ai import Agent -from sre_agent.core.models import ErrorDiagnosis -from sre_agent.core.prompts import SYSTEM_PROMPT -from sre_agent.eval.diagnosis_quality.config import ( +from evals.diagnosis_quality.config import ( DEFAULT_EXPERIMENT_NAME, DEFAULT_JUDGE_MODEL, DEFAULT_MODEL, DEFAULT_OPIK_PROJECT_NAME, ) -from sre_agent.eval.diagnosis_quality.dataset.create_and_populate import ( +from evals.diagnosis_quality.dataset.create_and_populate import ( DEFAULT_DATASET_NAME, create_and_populate_dataset, ) -from sre_agent.eval.diagnosis_quality.dataset.schema import DiagnosisQualityEvalCase -from sre_agent.eval.diagnosis_quality.github_toolset import build_github_toolset -from sre_agent.eval.diagnosis_quality.metrics import ( +from evals.diagnosis_quality.dataset.schema import DiagnosisQualityEvalCase +from evals.diagnosis_quality.github_toolset import build_github_toolset +from evals.diagnosis_quality.metrics import ( AffectedServicesMatch, RootCauseCorrectness, SuggestedFixesQuality, ) -from sre_agent.eval.diagnosis_quality.mocks import MockToolRuntime, build_mock_toolset -from sre_agent.eval.diagnosis_quality.prompts import render_agent_prompt +from evals.diagnosis_quality.mocks import MockToolRuntime, build_mock_toolset +from evals.diagnosis_quality.prompts import render_agent_prompt +from sre_agent.core.models import ErrorDiagnosis +from sre_agent.core.prompts import SYSTEM_PROMPT def evaluation_task(dataset_item: dict[str, Any]) -> dict[str, Any]: diff --git a/src/sre_agent/eval/diagnosis_quality/github_toolset.py b/evals/diagnosis_quality/github_toolset.py similarity index 100% rename from src/sre_agent/eval/diagnosis_quality/github_toolset.py rename to evals/diagnosis_quality/github_toolset.py diff --git a/src/sre_agent/eval/diagnosis_quality/metrics/__init__.py b/evals/diagnosis_quality/metrics/__init__.py similarity index 50% rename from src/sre_agent/eval/diagnosis_quality/metrics/__init__.py rename to evals/diagnosis_quality/metrics/__init__.py index c053bdd3..6c7c2ec9 100644 --- a/src/sre_agent/eval/diagnosis_quality/metrics/__init__.py +++ b/evals/diagnosis_quality/metrics/__init__.py @@ -1,12 +1,12 @@ """Metrics for diagnosis quality evaluation.""" -from sre_agent.eval.diagnosis_quality.metrics.affected_services_match import ( +from evals.diagnosis_quality.metrics.affected_services_match import ( AffectedServicesMatch, ) -from sre_agent.eval.diagnosis_quality.metrics.root_cause_correctness import ( +from evals.diagnosis_quality.metrics.root_cause_correctness import ( RootCauseCorrectness, ) -from sre_agent.eval.diagnosis_quality.metrics.suggested_fixes_quality import ( +from evals.diagnosis_quality.metrics.suggested_fixes_quality import ( SuggestedFixesQuality, ) diff --git a/src/sre_agent/eval/diagnosis_quality/metrics/affected_services_match.py b/evals/diagnosis_quality/metrics/affected_services_match.py similarity index 100% rename from src/sre_agent/eval/diagnosis_quality/metrics/affected_services_match.py rename to evals/diagnosis_quality/metrics/affected_services_match.py diff --git a/src/sre_agent/eval/diagnosis_quality/metrics/root_cause_correctness.py b/evals/diagnosis_quality/metrics/root_cause_correctness.py similarity index 100% rename from src/sre_agent/eval/diagnosis_quality/metrics/root_cause_correctness.py rename to evals/diagnosis_quality/metrics/root_cause_correctness.py diff --git a/src/sre_agent/eval/diagnosis_quality/metrics/suggested_fixes_quality.py b/evals/diagnosis_quality/metrics/suggested_fixes_quality.py similarity index 100% rename from src/sre_agent/eval/diagnosis_quality/metrics/suggested_fixes_quality.py rename to evals/diagnosis_quality/metrics/suggested_fixes_quality.py diff --git a/evals/diagnosis_quality/mocks/__init__.py b/evals/diagnosis_quality/mocks/__init__.py new file mode 100644 index 00000000..6ef06bd1 --- /dev/null +++ b/evals/diagnosis_quality/mocks/__init__.py @@ -0,0 +1,9 @@ +"""Mock tools for diagnosis quality evaluation.""" + +from evals.diagnosis_quality.mocks.runtime import MockToolRuntime +from evals.diagnosis_quality.mocks.toolset import build_mock_toolset + +__all__ = [ + "MockToolRuntime", + "build_mock_toolset", +] diff --git a/src/sre_agent/eval/diagnosis_quality/mocks/cloudwatch.py b/evals/diagnosis_quality/mocks/cloudwatch.py similarity index 95% rename from src/sre_agent/eval/diagnosis_quality/mocks/cloudwatch.py rename to evals/diagnosis_quality/mocks/cloudwatch.py index 86a2b263..4e4e9ad8 100644 --- a/src/sre_agent/eval/diagnosis_quality/mocks/cloudwatch.py +++ b/evals/diagnosis_quality/mocks/cloudwatch.py @@ -2,8 +2,8 @@ import opik +from evals.diagnosis_quality.mocks.runtime import MockToolRuntime from sre_agent.core.models import LogEntry, LogQueryResult -from sre_agent.eval.diagnosis_quality.mocks.runtime import MockToolRuntime MOCK_TIMESTAMP = "2026-01-01T00:00:00+00:00" diff --git a/src/sre_agent/eval/diagnosis_quality/mocks/runtime.py b/evals/diagnosis_quality/mocks/runtime.py similarity index 70% rename from src/sre_agent/eval/diagnosis_quality/mocks/runtime.py rename to evals/diagnosis_quality/mocks/runtime.py index 8ac89f82..24bfed03 100644 --- a/src/sre_agent/eval/diagnosis_quality/mocks/runtime.py +++ b/evals/diagnosis_quality/mocks/runtime.py @@ -2,7 +2,7 @@ from dataclasses import dataclass -from sre_agent.eval.diagnosis_quality.dataset.schema import DiagnosisQualityEvalCase +from evals.diagnosis_quality.dataset.schema import DiagnosisQualityEvalCase @dataclass diff --git a/src/sre_agent/eval/diagnosis_quality/mocks/slack.py b/evals/diagnosis_quality/mocks/slack.py similarity index 100% rename from src/sre_agent/eval/diagnosis_quality/mocks/slack.py rename to evals/diagnosis_quality/mocks/slack.py diff --git a/src/sre_agent/eval/diagnosis_quality/mocks/toolset.py b/evals/diagnosis_quality/mocks/toolset.py similarity index 85% rename from src/sre_agent/eval/diagnosis_quality/mocks/toolset.py rename to evals/diagnosis_quality/mocks/toolset.py index 21382627..ecbc2681 100644 --- a/src/sre_agent/eval/diagnosis_quality/mocks/toolset.py +++ b/evals/diagnosis_quality/mocks/toolset.py @@ -4,10 +4,10 @@ from pydantic_ai import FunctionToolset +from evals.diagnosis_quality.mocks import cloudwatch as cloudwatch_mocks +from evals.diagnosis_quality.mocks import slack as slack_mocks +from evals.diagnosis_quality.mocks.runtime import MockToolRuntime from sre_agent.core.models import LogQueryResult -from sre_agent.eval.diagnosis_quality.mocks import cloudwatch as cloudwatch_mocks -from sre_agent.eval.diagnosis_quality.mocks import slack as slack_mocks -from sre_agent.eval.diagnosis_quality.mocks.runtime import MockToolRuntime def build_mock_toolset(runtime: MockToolRuntime) -> FunctionToolset: diff --git a/src/sre_agent/eval/diagnosis_quality/prompts.py b/evals/diagnosis_quality/prompts.py similarity index 84% rename from src/sre_agent/eval/diagnosis_quality/prompts.py rename to evals/diagnosis_quality/prompts.py index cf471766..76a1f48b 100644 --- a/src/sre_agent/eval/diagnosis_quality/prompts.py +++ b/evals/diagnosis_quality/prompts.py @@ -1,11 +1,11 @@ """Prompt rendering for diagnosis quality evaluation.""" -from sre_agent.core.prompts import DIAGNOSIS_PROMPT_TEMPLATE -from sre_agent.eval.diagnosis_quality.config import ( +from evals.diagnosis_quality.config import ( DEFAULT_SLACK_CHANNEL_ID, DEFAULT_TIME_RANGE_MINUTES, ) -from sre_agent.eval.diagnosis_quality.dataset.schema import DiagnosisQualityEvalCase +from evals.diagnosis_quality.dataset.schema import DiagnosisQualityEvalCase +from sre_agent.core.prompts import DIAGNOSIS_PROMPT_TEMPLATE def render_agent_prompt(case: DiagnosisQualityEvalCase) -> str: diff --git a/src/sre_agent/eval/diagnosis_quality/run.py b/evals/diagnosis_quality/run.py similarity index 77% rename from src/sre_agent/eval/diagnosis_quality/run.py rename to evals/diagnosis_quality/run.py index ae59dcd4..e7b3d1e6 100644 --- a/src/sre_agent/eval/diagnosis_quality/run.py +++ b/evals/diagnosis_quality/run.py @@ -2,9 +2,9 @@ from pydantic_ai.exceptions import UserError -from sre_agent.eval.diagnosis_quality.config import DEFAULT_EXPERIMENT_NAME -from sre_agent.eval.diagnosis_quality.dataset.create_and_populate import DEFAULT_DATASET_NAME -from sre_agent.eval.diagnosis_quality.experiment import run_experiment +from evals.diagnosis_quality.config import DEFAULT_EXPERIMENT_NAME +from evals.diagnosis_quality.dataset.create_and_populate import DEFAULT_DATASET_NAME +from evals.diagnosis_quality.experiment import run_experiment def main() -> None: diff --git a/src/sre_agent/eval/imgs/opik_dataset.png b/evals/imgs/opik_dataset.png similarity index 100% rename from src/sre_agent/eval/imgs/opik_dataset.png rename to evals/imgs/opik_dataset.png diff --git a/src/sre_agent/eval/imgs/opik_experiment.png b/evals/imgs/opik_experiment.png similarity index 100% rename from src/sre_agent/eval/imgs/opik_experiment.png rename to evals/imgs/opik_experiment.png diff --git a/src/sre_agent/eval/tool_call/README.md b/evals/tool_call/README.md similarity index 93% rename from src/sre_agent/eval/tool_call/README.md rename to evals/tool_call/README.md index 0b6f4005..dffb634a 100644 --- a/src/sre_agent/eval/tool_call/README.md +++ b/evals/tool_call/README.md @@ -25,11 +25,11 @@ The run is hybrid: Test cases are loaded from: -- `src/sre_agent/eval/tool_call/dataset/test_cases` +- `evals/tool_call/dataset/test_cases` Each case follows `ToolCallEvalCase` in: -- `src/sre_agent/eval/tool_call/dataset/schema.py` +- `evals/tool_call/dataset/schema.py` Key fields: @@ -73,7 +73,7 @@ Run command: ```bash uv sync --group eval -uv run sre-agent-run-tool-call-eval +uv run python -m evals.tool_call.run ``` ## View Results in Opik diff --git a/src/sre_agent/eval/tool_call/__init__.py b/evals/tool_call/__init__.py similarity index 100% rename from src/sre_agent/eval/tool_call/__init__.py rename to evals/tool_call/__init__.py diff --git a/src/sre_agent/eval/tool_call/config.py b/evals/tool_call/config.py similarity index 100% rename from src/sre_agent/eval/tool_call/config.py rename to evals/tool_call/config.py diff --git a/src/sre_agent/eval/tool_call/dataset/__init__.py b/evals/tool_call/dataset/__init__.py similarity index 58% rename from src/sre_agent/eval/tool_call/dataset/__init__.py rename to evals/tool_call/dataset/__init__.py index 61a14455..5ec50471 100644 --- a/src/sre_agent/eval/tool_call/dataset/__init__.py +++ b/evals/tool_call/dataset/__init__.py @@ -1,9 +1,9 @@ """Dataset for tool call evaluation.""" -from sre_agent.eval.tool_call.dataset.create_and_populate import ( +from evals.tool_call.dataset.create_and_populate import ( DEFAULT_DATASET_NAME, create_and_populate_dataset, ) -from sre_agent.eval.tool_call.dataset.schema import ToolCallEvalCase +from evals.tool_call.dataset.schema import ToolCallEvalCase __all__ = ["create_and_populate_dataset", "ToolCallEvalCase", "DEFAULT_DATASET_NAME"] diff --git a/src/sre_agent/eval/tool_call/dataset/create_and_populate.py b/evals/tool_call/dataset/create_and_populate.py similarity index 88% rename from src/sre_agent/eval/tool_call/dataset/create_and_populate.py rename to evals/tool_call/dataset/create_and_populate.py index 1d46368c..9319d2f9 100644 --- a/src/sre_agent/eval/tool_call/dataset/create_and_populate.py +++ b/evals/tool_call/dataset/create_and_populate.py @@ -5,8 +5,8 @@ from opik import Opik -from sre_agent.eval.common.case_loader import load_json_case_models -from sre_agent.eval.tool_call.dataset.schema import ToolCallEvalCase +from evals.common.case_loader import load_json_case_models +from evals.tool_call.dataset.schema import ToolCallEvalCase DEFAULT_DATASET_NAME = "sre-agent-tool-call" diff --git a/src/sre_agent/eval/tool_call/dataset/schema.py b/evals/tool_call/dataset/schema.py similarity index 100% rename from src/sre_agent/eval/tool_call/dataset/schema.py rename to evals/tool_call/dataset/schema.py diff --git a/src/sre_agent/eval/tool_call/dataset/test_cases/cartservice_test_case_01.json b/evals/tool_call/dataset/test_cases/cartservice_test_case_01.json similarity index 100% rename from src/sre_agent/eval/tool_call/dataset/test_cases/cartservice_test_case_01.json rename to evals/tool_call/dataset/test_cases/cartservice_test_case_01.json diff --git a/src/sre_agent/eval/tool_call/dataset/test_cases/cartservice_test_case_02.json b/evals/tool_call/dataset/test_cases/cartservice_test_case_02.json similarity index 100% rename from src/sre_agent/eval/tool_call/dataset/test_cases/cartservice_test_case_02.json rename to evals/tool_call/dataset/test_cases/cartservice_test_case_02.json diff --git a/src/sre_agent/eval/tool_call/dataset/test_cases/currencyservice_test_case_01.json b/evals/tool_call/dataset/test_cases/currencyservice_test_case_01.json similarity index 100% rename from src/sre_agent/eval/tool_call/dataset/test_cases/currencyservice_test_case_01.json rename to evals/tool_call/dataset/test_cases/currencyservice_test_case_01.json diff --git a/src/sre_agent/eval/tool_call/dataset/test_cases/no_error_log_found_test_case_01.json b/evals/tool_call/dataset/test_cases/no_error_log_found_test_case_01.json similarity index 100% rename from src/sre_agent/eval/tool_call/dataset/test_cases/no_error_log_found_test_case_01.json rename to evals/tool_call/dataset/test_cases/no_error_log_found_test_case_01.json diff --git a/src/sre_agent/eval/tool_call/experiment.py b/evals/tool_call/experiment.py similarity index 82% rename from src/sre_agent/eval/tool_call/experiment.py rename to evals/tool_call/experiment.py index 8803c3e8..8e61dd27 100644 --- a/src/sre_agent/eval/tool_call/experiment.py +++ b/evals/tool_call/experiment.py @@ -9,25 +9,25 @@ from opik.evaluation.evaluation_result import EvaluationResult from pydantic_ai import Agent -from sre_agent.core.models import ErrorDiagnosis -from sre_agent.core.prompts import SYSTEM_PROMPT -from sre_agent.eval.tool_call.config import ( +from evals.tool_call.config import ( DEFAULT_EXPERIMENT_NAME, DEFAULT_MODEL, DEFAULT_OPIK_PROJECT_NAME, ) -from sre_agent.eval.tool_call.dataset.create_and_populate import ( +from evals.tool_call.dataset.create_and_populate import ( DEFAULT_DATASET_NAME, create_and_populate_dataset, ) -from sre_agent.eval.tool_call.dataset.schema import ToolCallEvalCase -from sre_agent.eval.tool_call.github_toolset import build_github_toolset -from sre_agent.eval.tool_call.metrics.expected_tool_select_order import ( +from evals.tool_call.dataset.schema import ToolCallEvalCase +from evals.tool_call.github_toolset import build_github_toolset +from evals.tool_call.metrics.expected_tool_select_order import ( ExpectedToolSelectOrder, ) -from sre_agent.eval.tool_call.metrics.expected_tool_selection import ExpectedToolSelection -from sre_agent.eval.tool_call.mocks import MockToolRuntime, build_mock_toolset -from sre_agent.eval.tool_call.prompts import render_agent_prompt +from evals.tool_call.metrics.expected_tool_selection import ExpectedToolSelection +from evals.tool_call.mocks import MockToolRuntime, build_mock_toolset +from evals.tool_call.prompts import render_agent_prompt +from sre_agent.core.models import ErrorDiagnosis +from sre_agent.core.prompts import SYSTEM_PROMPT def evaluation_task(dataset_item: dict[str, Any]) -> dict[str, Any]: diff --git a/src/sre_agent/eval/tool_call/github_toolset.py b/evals/tool_call/github_toolset.py similarity index 100% rename from src/sre_agent/eval/tool_call/github_toolset.py rename to evals/tool_call/github_toolset.py diff --git a/evals/tool_call/metrics/__init__.py b/evals/tool_call/metrics/__init__.py new file mode 100644 index 00000000..7b868cd0 --- /dev/null +++ b/evals/tool_call/metrics/__init__.py @@ -0,0 +1,6 @@ +"""Metrics for tool call evaluation.""" + +from evals.tool_call.metrics.expected_tool_select_order import ExpectedToolSelectOrder +from evals.tool_call.metrics.expected_tool_selection import ExpectedToolSelection + +__all__ = ["ExpectedToolSelection", "ExpectedToolSelectOrder"] diff --git a/src/sre_agent/eval/tool_call/metrics/expected_tool_select_order.py b/evals/tool_call/metrics/expected_tool_select_order.py similarity index 97% rename from src/sre_agent/eval/tool_call/metrics/expected_tool_select_order.py rename to evals/tool_call/metrics/expected_tool_select_order.py index 5131cf13..4597f160 100644 --- a/src/sre_agent/eval/tool_call/metrics/expected_tool_select_order.py +++ b/evals/tool_call/metrics/expected_tool_select_order.py @@ -5,7 +5,7 @@ from opik.evaluation.metrics import base_metric, score_result from opik.message_processing.emulation.models import SpanModel -from sre_agent.eval.tool_call.metrics.span_tools import extract_tool_names +from evals.tool_call.metrics.span_tools import extract_tool_names class ExpectedToolSelectOrder(base_metric.BaseMetric): # type: ignore[misc] diff --git a/src/sre_agent/eval/tool_call/metrics/expected_tool_selection.py b/evals/tool_call/metrics/expected_tool_selection.py similarity index 97% rename from src/sre_agent/eval/tool_call/metrics/expected_tool_selection.py rename to evals/tool_call/metrics/expected_tool_selection.py index a7c1de72..56aa815a 100644 --- a/src/sre_agent/eval/tool_call/metrics/expected_tool_selection.py +++ b/evals/tool_call/metrics/expected_tool_selection.py @@ -5,7 +5,7 @@ from opik.evaluation.metrics import base_metric, score_result from opik.message_processing.emulation.models import SpanModel -from sre_agent.eval.tool_call.metrics.span_tools import extract_tool_names +from evals.tool_call.metrics.span_tools import extract_tool_names class ExpectedToolSelection(base_metric.BaseMetric): # type: ignore[misc] diff --git a/src/sre_agent/eval/tool_call/metrics/span_tools.py b/evals/tool_call/metrics/span_tools.py similarity index 100% rename from src/sre_agent/eval/tool_call/metrics/span_tools.py rename to evals/tool_call/metrics/span_tools.py diff --git a/evals/tool_call/mocks/__init__.py b/evals/tool_call/mocks/__init__.py new file mode 100644 index 00000000..19324915 --- /dev/null +++ b/evals/tool_call/mocks/__init__.py @@ -0,0 +1,6 @@ +"""Mock tools for tool call evaluation.""" + +from evals.tool_call.mocks.runtime import MockToolRuntime +from evals.tool_call.mocks.toolset import build_mock_toolset + +__all__ = ["MockToolRuntime", "build_mock_toolset"] diff --git a/src/sre_agent/eval/tool_call/mocks/cloudwatch.py b/evals/tool_call/mocks/cloudwatch.py similarity index 96% rename from src/sre_agent/eval/tool_call/mocks/cloudwatch.py rename to evals/tool_call/mocks/cloudwatch.py index 0706f2a0..936437c1 100644 --- a/src/sre_agent/eval/tool_call/mocks/cloudwatch.py +++ b/evals/tool_call/mocks/cloudwatch.py @@ -2,8 +2,8 @@ import opik +from evals.tool_call.mocks.runtime import MockToolRuntime from sre_agent.core.models import LogEntry, LogQueryResult -from sre_agent.eval.tool_call.mocks.runtime import MockToolRuntime MOCK_TIMESTAMP = "2026-01-01T00:00:00+00:00" diff --git a/src/sre_agent/eval/tool_call/mocks/runtime.py b/evals/tool_call/mocks/runtime.py similarity index 73% rename from src/sre_agent/eval/tool_call/mocks/runtime.py rename to evals/tool_call/mocks/runtime.py index dd417117..5489122d 100644 --- a/src/sre_agent/eval/tool_call/mocks/runtime.py +++ b/evals/tool_call/mocks/runtime.py @@ -2,7 +2,7 @@ from dataclasses import dataclass -from sre_agent.eval.tool_call.dataset.schema import ToolCallEvalCase +from evals.tool_call.dataset.schema import ToolCallEvalCase @dataclass diff --git a/src/sre_agent/eval/tool_call/mocks/slack.py b/evals/tool_call/mocks/slack.py similarity index 100% rename from src/sre_agent/eval/tool_call/mocks/slack.py rename to evals/tool_call/mocks/slack.py diff --git a/src/sre_agent/eval/tool_call/mocks/toolset.py b/evals/tool_call/mocks/toolset.py similarity index 86% rename from src/sre_agent/eval/tool_call/mocks/toolset.py rename to evals/tool_call/mocks/toolset.py index f0efd754..6c0f172c 100644 --- a/src/sre_agent/eval/tool_call/mocks/toolset.py +++ b/evals/tool_call/mocks/toolset.py @@ -4,10 +4,10 @@ from pydantic_ai import FunctionToolset +from evals.tool_call.mocks import cloudwatch as cloudwatch_mocks +from evals.tool_call.mocks import slack as slack_mocks +from evals.tool_call.mocks.runtime import MockToolRuntime from sre_agent.core.models import LogQueryResult -from sre_agent.eval.tool_call.mocks import cloudwatch as cloudwatch_mocks -from sre_agent.eval.tool_call.mocks import slack as slack_mocks -from sre_agent.eval.tool_call.mocks.runtime import MockToolRuntime def build_mock_toolset(runtime: MockToolRuntime) -> FunctionToolset: diff --git a/src/sre_agent/eval/tool_call/prompts.py b/evals/tool_call/prompts.py similarity index 86% rename from src/sre_agent/eval/tool_call/prompts.py rename to evals/tool_call/prompts.py index 526fcae4..f665dcb8 100644 --- a/src/sre_agent/eval/tool_call/prompts.py +++ b/evals/tool_call/prompts.py @@ -1,11 +1,11 @@ """Prompt rendering for tool call evaluation.""" -from sre_agent.core.prompts import DIAGNOSIS_PROMPT_TEMPLATE -from sre_agent.eval.tool_call.config import ( +from evals.tool_call.config import ( DEFAULT_SLACK_CHANNEL_ID, DEFAULT_TIME_RANGE_MINUTES, ) -from sre_agent.eval.tool_call.dataset.schema import ToolCallEvalCase +from evals.tool_call.dataset.schema import ToolCallEvalCase +from sre_agent.core.prompts import DIAGNOSIS_PROMPT_TEMPLATE def render_agent_prompt(case: ToolCallEvalCase) -> str: diff --git a/src/sre_agent/eval/tool_call/run.py b/evals/tool_call/run.py similarity index 78% rename from src/sre_agent/eval/tool_call/run.py rename to evals/tool_call/run.py index 4c6ca1a5..50610740 100644 --- a/src/sre_agent/eval/tool_call/run.py +++ b/evals/tool_call/run.py @@ -2,9 +2,9 @@ from pydantic_ai.exceptions import UserError -from sre_agent.eval.tool_call.config import DEFAULT_EXPERIMENT_NAME -from sre_agent.eval.tool_call.dataset.create_and_populate import DEFAULT_DATASET_NAME -from sre_agent.eval.tool_call.experiment import run_experiment +from evals.tool_call.config import DEFAULT_EXPERIMENT_NAME +from evals.tool_call.dataset.create_and_populate import DEFAULT_DATASET_NAME +from evals.tool_call.experiment import run_experiment def main() -> None: diff --git a/pyproject.toml b/pyproject.toml index 6bcbfde2..d2161588 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,8 +19,6 @@ dependencies = [ [project.scripts] sre-agent = "sre_agent.cli.main:main" -sre-agent-run-tool-call-eval = "sre_agent.eval.tool_call.run:main" -sre-agent-run-diagnosis-quality-eval = "sre_agent.eval.diagnosis_quality.run:main" [dependency-groups] dev = [ diff --git a/src/sre_agent/eval/diagnosis_quality/mocks/__init__.py b/src/sre_agent/eval/diagnosis_quality/mocks/__init__.py deleted file mode 100644 index 40cfcf24..00000000 --- a/src/sre_agent/eval/diagnosis_quality/mocks/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Mock tools for diagnosis quality evaluation.""" - -from sre_agent.eval.diagnosis_quality.mocks.runtime import MockToolRuntime -from sre_agent.eval.diagnosis_quality.mocks.toolset import build_mock_toolset - -__all__ = [ - "MockToolRuntime", - "build_mock_toolset", -] diff --git a/src/sre_agent/eval/tool_call/metrics/__init__.py b/src/sre_agent/eval/tool_call/metrics/__init__.py deleted file mode 100644 index 82a5a9e6..00000000 --- a/src/sre_agent/eval/tool_call/metrics/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Metrics for tool call evaluation.""" - -from sre_agent.eval.tool_call.metrics.expected_tool_select_order import ExpectedToolSelectOrder -from sre_agent.eval.tool_call.metrics.expected_tool_selection import ExpectedToolSelection - -__all__ = ["ExpectedToolSelection", "ExpectedToolSelectOrder"] diff --git a/src/sre_agent/eval/tool_call/mocks/__init__.py b/src/sre_agent/eval/tool_call/mocks/__init__.py deleted file mode 100644 index fc22eb5b..00000000 --- a/src/sre_agent/eval/tool_call/mocks/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Mock tools for tool call evaluation.""" - -from sre_agent.eval.tool_call.mocks.runtime import MockToolRuntime -from sre_agent.eval.tool_call.mocks.toolset import build_mock_toolset - -__all__ = ["MockToolRuntime", "build_mock_toolset"]