diff --git a/api/endpoints/evaluation_sets.py b/api/endpoints/evaluation_sets.py index 253762d4..9a0f4486 100644 --- a/api/endpoints/evaluation_sets.py +++ b/api/endpoints/evaluation_sets.py @@ -9,6 +9,7 @@ EvaluationSetDetail, EvaluationSetDetailBenchmarkThreshold, EvaluationSetDetailEfficiency, + EvaluationSetDetailEfficiencyAgent, EvaluationSetDetailLeaderboardAgent, EvaluationSetDetailPipelineStage, EvaluationSetDetailScores, @@ -186,9 +187,26 @@ def _pass_rate(count: int, total: int) -> float: else None ) + lowest_cost_value = leaderboard_summary_row["lowest_average_cost_usd_top_agents"] + lowest_runtime_value = leaderboard_summary_row["lowest_average_runtime_seconds_top_agents"] + efficiency = EvaluationSetDetailEfficiency( - lowest_average_cost_usd_top_agents=leaderboard_summary_row["lowest_average_cost_usd_top_agents"], - lowest_average_runtime_seconds_top_agents=leaderboard_summary_row["lowest_average_runtime_seconds_top_agents"], + lowest_average_cost_usd_top_agents=( + EvaluationSetDetailEfficiencyAgent( + agent_id=leaderboard_summary_row["lowest_cost_agent_id"], + value=lowest_cost_value, + ) + if lowest_cost_value is not None + else None + ), + lowest_average_runtime_seconds_top_agents=( + EvaluationSetDetailEfficiencyAgent( + agent_id=leaderboard_summary_row["lowest_runtime_agent_id"], + value=lowest_runtime_value, + ) + if lowest_runtime_value is not None + else None + ), average_agent_cost_usd=leaderboard_summary_row["average_agent_cost_usd"], average_agent_runtime_seconds=leaderboard_summary_row["average_agent_runtime_seconds"], ) diff --git a/models/evaluation_set.py b/models/evaluation_set.py index 2a2422d2..ae2dd0a5 100644 --- a/models/evaluation_set.py +++ b/models/evaluation_set.py @@ -109,9 +109,14 @@ class EvaluationSetDetailTopAgent(BaseModel): emission: float | None = None +class EvaluationSetDetailEfficiencyAgent(BaseModel): + agent_id: UUID | None + value: Float4 | None + + class EvaluationSetDetailEfficiency(BaseModel): - lowest_average_cost_usd_top_agents: Float4 | None - lowest_average_runtime_seconds_top_agents: Float4 | None + lowest_average_cost_usd_top_agents: EvaluationSetDetailEfficiencyAgent | None + lowest_average_runtime_seconds_top_agents: EvaluationSetDetailEfficiencyAgent | None average_agent_cost_usd: Float4 | None average_agent_runtime_seconds: Float4 | None diff --git a/queries/evaluation_set.py b/queries/evaluation_set.py index ef46a400..b872a89b 100644 --- a/queries/evaluation_set.py +++ b/queries/evaluation_set.py @@ -417,8 +417,10 @@ async def get_evaluation_set_score_stats(conn: DatabaseConnection, set_id: int) - agents_beating_previous_best: The count of agents in the current set that beat the previous best score. """ return await conn.fetchrow( - """ - WITH prev_best AS ( + f""" + WITH {_SQL_SET_WINDOW_CTE}, + {_sql_agents_in_window_cte("a.agent_id, a.status")}, + prev_best AS ( SELECT MAX(final_score) AS score FROM @@ -471,15 +473,11 @@ async def get_evaluation_set_score_stats(conn: DatabaseConnection, set_id: int) ) :: int AS agents_beating_previous_best FROM agent_scores s + JOIN agents_in_window aiw ON aiw.agent_id = s.agent_id WHERE s.set_id = $1 - AND s.agent_id NOT IN ( - SELECT - agent_id - FROM - benchmark_agent_ids - ) - AND s.status = 'finished' + AND aiw.status = 'finished' + AND NOT aiw.disqualified """, set_id, ) @@ -495,14 +493,15 @@ async def get_evaluation_set_leaderboard_agents(conn: DatabaseConnection, set_id {_sql_validator_metrics_cte(include_validator_hotkeys=True)}, tentative_scores AS ( WITH tentative_runs AS ( - SELECT eh.agent_id, eh.validator_hotkey, erh.problem_name, erh.solved, aiw.disqualified, aiw.created_at + SELECT eh.agent_id, eh.validator_hotkey, erh.problem_name, + (erh.solved IS TRUE OR erh.error_code = 3060) AS solved_effective, + aiw.disqualified, aiw.created_at, aiw.status AS agent_status FROM evaluations_hydrated eh JOIN agents_in_window aiw ON aiw.agent_id = eh.agent_id AND aiw.status in ('evaluating','cancelled') JOIN evaluation_runs_hydrated erh ON erh.evaluation_id = eh.evaluation_id WHERE eh.set_id = $1 AND eh.evaluation_set_group = 'validator'::EvaluationSetGroup - AND erh.status != 'error' AND NOT EXISTS ( SELECT 1 FROM agent_scores ass WHERE ass.agent_id = eh.agent_id AND ass.set_id = $1 @@ -514,10 +513,11 @@ async def get_evaluation_set_leaderboard_agents(conn: DatabaseConnection, set_id problem_name, disqualified, created_at, - COUNT(DISTINCT validator_hotkey) FILTER (WHERE solved IS TRUE) + agent_status, + COUNT(DISTINCT validator_hotkey) FILTER (WHERE solved_effective) AS solved_validator_count FROM tentative_runs - GROUP BY agent_id, problem_name, disqualified, created_at + GROUP BY agent_id, problem_name, disqualified, created_at, agent_status ), problem_count AS ( SELECT COUNT(*)::float AS n @@ -539,21 +539,24 @@ async def get_evaluation_set_leaderboard_agents(conn: DatabaseConnection, set_id vc.validator_count, vc.validator_hotkeys, pp.disqualified, - pp.created_at + pp.created_at, + pp.agent_status FROM per_problem pp JOIN validator_counts vc ON vc.agent_id = pp.agent_id - GROUP BY pp.agent_id, vc.validator_count, vc.validator_hotkeys, pp.disqualified, pp.created_at + GROUP BY pp.agent_id, vc.validator_count, vc.validator_hotkeys, pp.disqualified, pp.created_at, pp.agent_status HAVING COUNT(*) FILTER (WHERE pp.solved_validator_count >= vc.validator_count) > 0 ), scored_agents AS ( SELECT ass.agent_id, ass.final_score, ass.validator_count, - COALESCE(vm.validator_hotkeys, ARRAY[]::text[]) AS validator_hotkeys, aiw.disqualified, aiw.created_at + COALESCE(vm.validator_hotkeys, ARRAY[]::text[]) AS validator_hotkeys, aiw.disqualified, aiw.created_at, + aiw.status AS agent_status FROM agent_scores ass JOIN agents_in_window aiw ON aiw.agent_id = ass.agent_id LEFT JOIN validator_metrics vm ON vm.agent_id = ass.agent_id WHERE ass.set_id = $1 UNION ALL - SELECT ts.agent_id, ts.final_score, ts.validator_count, ts.validator_hotkeys, ts.disqualified, ts.created_at + SELECT ts.agent_id, ts.final_score, ts.validator_count, ts.validator_hotkeys, ts.disqualified, ts.created_at, + ts.agent_status FROM tentative_scores ts ), ranked_scores AS ( @@ -565,9 +568,9 @@ async def get_evaluation_set_leaderboard_agents(conn: DatabaseConnection, set_id vm.average_runtime_seconds, COALESCE(vm.validator_hotkeys, ARRAY[]::text[]) AS validator_hotkeys, CASE - WHEN sa.disqualified THEN NULL + WHEN sa.disqualified OR sa.agent_status = 'cancelled' THEN NULL ELSE ROW_NUMBER() OVER ( - PARTITION BY sa.disqualified + PARTITION BY (sa.disqualified OR sa.agent_status = 'cancelled') ORDER BY ROUND(sa.final_score::numeric, 6) DESC, vm.average_cost_usd ASC NULLS LAST, @@ -618,27 +621,47 @@ async def get_evaluation_set_leaderboard_summary(conn: DatabaseConnection, set_i {_sql_agents_in_window_cte("a.agent_id, a.name, a.version_num, a.created_at, a.status")}, {_sql_validator_metrics_cte(include_validator_hotkeys=False)}, {_sql_top_agent_for_summary()}, - efficiency AS ( + efficiency_averages AS ( SELECT - MIN(vm.average_cost_usd) AS lowest_average_cost_usd_top_agents, - MIN(vm.average_runtime_seconds) AS lowest_average_runtime_seconds_top_agents, AVG(vm.average_cost_usd) AS average_agent_cost_usd, AVG(vm.average_runtime_seconds) AS average_agent_runtime_seconds FROM validator_metrics vm - LEFT JOIN agents_in_window aa on vm.agent_id=aa.agent_id - where not aa.disqualified and aa.status = 'finished' + JOIN agents_in_window aa ON vm.agent_id = aa.agent_id + WHERE NOT aa.disqualified AND aa.status = 'finished' + ), + lowest_cost_agent AS ( + SELECT vm.agent_id, vm.average_cost_usd AS value + FROM validator_metrics vm + JOIN agents_in_window aa ON vm.agent_id = aa.agent_id + WHERE NOT aa.disqualified AND aa.status = 'finished' + AND vm.average_cost_usd IS NOT NULL + ORDER BY vm.average_cost_usd ASC + LIMIT 1 + ), + lowest_runtime_agent AS ( + SELECT vm.agent_id, vm.average_runtime_seconds AS value + FROM validator_metrics vm + JOIN agents_in_window aa ON vm.agent_id = aa.agent_id + WHERE NOT aa.disqualified AND aa.status = 'finished' + AND vm.average_runtime_seconds IS NOT NULL + ORDER BY vm.average_runtime_seconds ASC + LIMIT 1 ) SELECT ta.agent_id AS top_agent_id, ta.name AS top_agent_name, ta.version_num AS top_agent_version_num, ta.final_score AS top_agent_final_score, - e.lowest_average_cost_usd_top_agents, - e.lowest_average_runtime_seconds_top_agents, - e.average_agent_cost_usd, - e.average_agent_runtime_seconds - FROM efficiency e + lca.agent_id AS lowest_cost_agent_id, + lca.value AS lowest_average_cost_usd_top_agents, + lra.agent_id AS lowest_runtime_agent_id, + lra.value AS lowest_average_runtime_seconds_top_agents, + ea.average_agent_cost_usd, + ea.average_agent_runtime_seconds + FROM efficiency_averages ea LEFT JOIN top_agent ta ON TRUE + LEFT JOIN lowest_cost_agent lca ON TRUE + LEFT JOIN lowest_runtime_agent lra ON TRUE """, set_id, ) diff --git a/tests/api/test_evaluation_sets.py b/tests/api/test_evaluation_sets.py index fdf2c2fa..de9c4eae 100644 --- a/tests/api/test_evaluation_sets.py +++ b/tests/api/test_evaluation_sets.py @@ -323,8 +323,12 @@ async def test_evaluation_set_detail_happy_path(): assert result.top_agent.version_num == 1 assert result.top_agent.final_score == 0.8 - assert result.efficiency.lowest_average_cost_usd_top_agents == 0.2 - assert result.efficiency.lowest_average_runtime_seconds_top_agents == 75 + assert result.efficiency.lowest_average_cost_usd_top_agents is not None + assert result.efficiency.lowest_average_cost_usd_top_agents.agent_id == agent_a + assert result.efficiency.lowest_average_cost_usd_top_agents.value == 0.2 + assert result.efficiency.lowest_average_runtime_seconds_top_agents is not None + assert result.efficiency.lowest_average_runtime_seconds_top_agents.agent_id == agent_a + assert result.efficiency.lowest_average_runtime_seconds_top_agents.value == 75 assert result.efficiency.average_agent_cost_usd == 0.2 assert result.efficiency.average_agent_runtime_seconds == 75 @@ -459,8 +463,12 @@ async def test_evaluation_set_leaderboard_ranks_by_score_cost_then_submission_ti assert agents_by_id[unscored_agent].final_score is None result = await evaluation_sets_endpoint.evaluation_set_detail(set_id=2) - assert result.efficiency.lowest_average_cost_usd_top_agents == 1.0 - assert result.efficiency.lowest_average_runtime_seconds_top_agents == 10 + assert result.efficiency.lowest_average_cost_usd_top_agents is not None + assert result.efficiency.lowest_average_cost_usd_top_agents.agent_id == lower_cost_tie_agent + assert result.efficiency.lowest_average_cost_usd_top_agents.value == 1.0 + assert result.efficiency.lowest_average_runtime_seconds_top_agents is not None + assert result.efficiency.lowest_average_runtime_seconds_top_agents.agent_id == later_time_tie_agent + assert result.efficiency.lowest_average_runtime_seconds_top_agents.value == 10 assert result.efficiency.average_agent_cost_usd == 2.8 assert result.efficiency.average_agent_runtime_seconds == 36 @@ -511,8 +519,10 @@ async def test_evaluation_set_detail_efficiency_uses_all_ranked_agents_not_top_2 assert len([agent for agent in leaderboard if agent.rank is not None]) == 26 result = await evaluation_sets_endpoint.evaluation_set_detail(set_id=2) - assert result.efficiency.lowest_average_cost_usd_top_agents == 0.1 - assert result.efficiency.lowest_average_runtime_seconds_top_agents == 1 + assert result.efficiency.lowest_average_cost_usd_top_agents is not None + assert result.efficiency.lowest_average_cost_usd_top_agents.value == 0.1 + assert result.efficiency.lowest_average_runtime_seconds_top_agents is not None + assert result.efficiency.lowest_average_runtime_seconds_top_agents.value == 1 @pytest.mark.anyio