Update: card metrics query

api/endpoints/evaluation_sets.py

@@ -9,6 +9,7 @@
     EvaluationSetDetail,
     EvaluationSetDetailBenchmarkThreshold,
     EvaluationSetDetailEfficiency,
+    EvaluationSetDetailEfficiencyAgent,
     EvaluationSetDetailLeaderboardAgent,
     EvaluationSetDetailPipelineStage,
     EvaluationSetDetailScores,
@@ -186,9 +187,26 @@ def _pass_rate(count: int, total: int) -> float:
         else None
     )
 
+    lowest_cost_value = leaderboard_summary_row["lowest_average_cost_usd_top_agents"]
+    lowest_runtime_value = leaderboard_summary_row["lowest_average_runtime_seconds_top_agents"]
+
     efficiency = EvaluationSetDetailEfficiency(
-        lowest_average_cost_usd_top_agents=leaderboard_summary_row["lowest_average_cost_usd_top_agents"],
-        lowest_average_runtime_seconds_top_agents=leaderboard_summary_row["lowest_average_runtime_seconds_top_agents"],
+        lowest_average_cost_usd_top_agents=(
+            EvaluationSetDetailEfficiencyAgent(
+                agent_id=leaderboard_summary_row["lowest_cost_agent_id"],
+                value=lowest_cost_value,
+            )
+            if lowest_cost_value is not None
+            else None
+        ),
+        lowest_average_runtime_seconds_top_agents=(
+            EvaluationSetDetailEfficiencyAgent(
+                agent_id=leaderboard_summary_row["lowest_runtime_agent_id"],
+                value=lowest_runtime_value,
+            )
+            if lowest_runtime_value is not None
+            else None
+        ),
         average_agent_cost_usd=leaderboard_summary_row["average_agent_cost_usd"],
         average_agent_runtime_seconds=leaderboard_summary_row["average_agent_runtime_seconds"],
     )

models/evaluation_set.py

@@ -109,9 +109,14 @@ class EvaluationSetDetailTopAgent(BaseModel):
     emission: float | None = None
 
 
+class EvaluationSetDetailEfficiencyAgent(BaseModel):
+    agent_id: UUID | None
+    value: Float4 | None
+
+
 class EvaluationSetDetailEfficiency(BaseModel):
-    lowest_average_cost_usd_top_agents: Float4 | None
-    lowest_average_runtime_seconds_top_agents: Float4 | None
+    lowest_average_cost_usd_top_agents: EvaluationSetDetailEfficiencyAgent | None
+    lowest_average_runtime_seconds_top_agents: EvaluationSetDetailEfficiencyAgent | None
     average_agent_cost_usd: Float4 | None
     average_agent_runtime_seconds: Float4 | None
 

queries/evaluation_set.py

@@ -417,8 +417,10 @@ async def get_evaluation_set_score_stats(conn: DatabaseConnection, set_id: int)
         - agents_beating_previous_best: The count of agents in the current set that beat the previous best score.
     """
     return await conn.fetchrow(
-        """
-        WITH prev_best AS (
+        f"""
+        WITH {_SQL_SET_WINDOW_CTE},
+        {_sql_agents_in_window_cte("a.agent_id, a.status")},
+        prev_best AS (
             SELECT
                 MAX(final_score) AS score
             FROM
@@ -471,15 +473,11 @@ async def get_evaluation_set_score_stats(conn: DatabaseConnection, set_id: int)
             ) :: int AS agents_beating_previous_best
         FROM
             agent_scores s
+            JOIN agents_in_window aiw ON aiw.agent_id = s.agent_id
         WHERE
             s.set_id = $1
-            AND s.agent_id NOT IN (
-                SELECT
-                    agent_id
-                FROM
-                    benchmark_agent_ids
-            )
-            AND s.status = 'finished'
+            AND aiw.status = 'finished'
+            AND NOT aiw.disqualified
         """,
         set_id,
     )
@@ -495,14 +493,15 @@ async def get_evaluation_set_leaderboard_agents(conn: DatabaseConnection, set_id
         {_sql_validator_metrics_cte(include_validator_hotkeys=True)},
         tentative_scores AS (
             WITH tentative_runs AS (
-                SELECT eh.agent_id, eh.validator_hotkey, erh.problem_name, erh.solved, aiw.disqualified, aiw.created_at
+                SELECT eh.agent_id, eh.validator_hotkey, erh.problem_name,
+                       (erh.solved IS TRUE OR erh.error_code = 3060) AS solved_effective,
+                       aiw.disqualified, aiw.created_at, aiw.status AS agent_status
                 FROM evaluations_hydrated eh
                 JOIN agents_in_window aiw
                     ON aiw.agent_id = eh.agent_id AND aiw.status in ('evaluating','cancelled')
                 JOIN evaluation_runs_hydrated erh ON erh.evaluation_id = eh.evaluation_id
                 WHERE eh.set_id = $1
                   AND eh.evaluation_set_group = 'validator'::EvaluationSetGroup
-                  AND erh.status != 'error'
                   AND NOT EXISTS (
                       SELECT 1 FROM agent_scores ass
                       WHERE ass.agent_id = eh.agent_id AND ass.set_id = $1
@@ -514,10 +513,11 @@ async def get_evaluation_set_leaderboard_agents(conn: DatabaseConnection, set_id
                     problem_name,
                     disqualified,
                     created_at,
-                    COUNT(DISTINCT validator_hotkey) FILTER (WHERE solved IS TRUE)
+                    agent_status,
+                    COUNT(DISTINCT validator_hotkey) FILTER (WHERE solved_effective)
                         AS solved_validator_count
                 FROM tentative_runs
-                GROUP BY agent_id, problem_name, disqualified, created_at
+                GROUP BY agent_id, problem_name, disqualified, created_at, agent_status
             ),
             problem_count AS (
                 SELECT COUNT(*)::float AS n
@@ -539,21 +539,24 @@ async def get_evaluation_set_leaderboard_agents(conn: DatabaseConnection, set_id
                 vc.validator_count,
                 vc.validator_hotkeys,
                 pp.disqualified,
-                pp.created_at
+                pp.created_at,
+                pp.agent_status
             FROM per_problem pp
             JOIN validator_counts vc ON vc.agent_id = pp.agent_id
-            GROUP BY pp.agent_id, vc.validator_count, vc.validator_hotkeys, pp.disqualified, pp.created_at
+            GROUP BY pp.agent_id, vc.validator_count, vc.validator_hotkeys, pp.disqualified, pp.created_at, pp.agent_status
             HAVING COUNT(*) FILTER (WHERE pp.solved_validator_count >= vc.validator_count) > 0
         ),
         scored_agents AS (
             SELECT ass.agent_id, ass.final_score, ass.validator_count,
-                   COALESCE(vm.validator_hotkeys, ARRAY[]::text[]) AS validator_hotkeys, aiw.disqualified, aiw.created_at
+                   COALESCE(vm.validator_hotkeys, ARRAY[]::text[]) AS validator_hotkeys, aiw.disqualified, aiw.created_at,
+                   aiw.status AS agent_status
             FROM agent_scores ass
             JOIN agents_in_window aiw ON aiw.agent_id = ass.agent_id
             LEFT JOIN validator_metrics vm ON vm.agent_id = ass.agent_id
             WHERE ass.set_id = $1
             UNION ALL
-            SELECT ts.agent_id, ts.final_score, ts.validator_count, ts.validator_hotkeys, ts.disqualified, ts.created_at
+            SELECT ts.agent_id, ts.final_score, ts.validator_count, ts.validator_hotkeys, ts.disqualified, ts.created_at,
+                   ts.agent_status
             FROM tentative_scores ts
         ),
         ranked_scores AS (
@@ -565,9 +568,9 @@ async def get_evaluation_set_leaderboard_agents(conn: DatabaseConnection, set_id
                 vm.average_runtime_seconds,
                 COALESCE(vm.validator_hotkeys, ARRAY[]::text[]) AS validator_hotkeys,
                 CASE
-                    WHEN sa.disqualified THEN NULL
+                    WHEN sa.disqualified OR sa.agent_status = 'cancelled' THEN NULL
                     ELSE ROW_NUMBER() OVER (
-                        PARTITION BY sa.disqualified
+                        PARTITION BY (sa.disqualified OR sa.agent_status = 'cancelled')
                         ORDER BY
                             ROUND(sa.final_score::numeric, 6) DESC,
                             vm.average_cost_usd ASC NULLS LAST,
@@ -618,27 +621,47 @@ async def get_evaluation_set_leaderboard_summary(conn: DatabaseConnection, set_i
         {_sql_agents_in_window_cte("a.agent_id, a.name, a.version_num, a.created_at, a.status")},
         {_sql_validator_metrics_cte(include_validator_hotkeys=False)},
         {_sql_top_agent_for_summary()},
-        efficiency AS (
+        efficiency_averages AS (
             SELECT
-                MIN(vm.average_cost_usd) AS lowest_average_cost_usd_top_agents,
-                MIN(vm.average_runtime_seconds) AS lowest_average_runtime_seconds_top_agents,
                 AVG(vm.average_cost_usd) AS average_agent_cost_usd,
                 AVG(vm.average_runtime_seconds) AS average_agent_runtime_seconds
             FROM validator_metrics vm
-            LEFT JOIN agents_in_window aa on vm.agent_id=aa.agent_id
-            where not aa.disqualified and aa.status = 'finished'
+            JOIN agents_in_window aa ON vm.agent_id = aa.agent_id
+            WHERE NOT aa.disqualified AND aa.status = 'finished'
+        ),
+        lowest_cost_agent AS (
+            SELECT vm.agent_id, vm.average_cost_usd AS value
+            FROM validator_metrics vm
+            JOIN agents_in_window aa ON vm.agent_id = aa.agent_id
+            WHERE NOT aa.disqualified AND aa.status = 'finished'
+              AND vm.average_cost_usd IS NOT NULL
+            ORDER BY vm.average_cost_usd ASC
+            LIMIT 1
+        ),
+        lowest_runtime_agent AS (
+            SELECT vm.agent_id, vm.average_runtime_seconds AS value
+            FROM validator_metrics vm
+            JOIN agents_in_window aa ON vm.agent_id = aa.agent_id
+            WHERE NOT aa.disqualified AND aa.status = 'finished'
+              AND vm.average_runtime_seconds IS NOT NULL
+            ORDER BY vm.average_runtime_seconds ASC
+            LIMIT 1
         )
         SELECT
             ta.agent_id AS top_agent_id,
             ta.name AS top_agent_name,
             ta.version_num AS top_agent_version_num,
             ta.final_score AS top_agent_final_score,
-            e.lowest_average_cost_usd_top_agents,
-            e.lowest_average_runtime_seconds_top_agents,
-            e.average_agent_cost_usd,
-            e.average_agent_runtime_seconds
-        FROM efficiency e
+            lca.agent_id AS lowest_cost_agent_id,
+            lca.value AS lowest_average_cost_usd_top_agents,
+            lra.agent_id AS lowest_runtime_agent_id,
+            lra.value AS lowest_average_runtime_seconds_top_agents,
+            ea.average_agent_cost_usd,
+            ea.average_agent_runtime_seconds
+        FROM efficiency_averages ea
         LEFT JOIN top_agent ta ON TRUE
+        LEFT JOIN lowest_cost_agent lca ON TRUE
+        LEFT JOIN lowest_runtime_agent lra ON TRUE
         """,
         set_id,
     )

tests/api/test_evaluation_sets.py

@@ -323,8 +323,12 @@ async def test_evaluation_set_detail_happy_path():
     assert result.top_agent.version_num == 1
     assert result.top_agent.final_score == 0.8
 
-    assert result.efficiency.lowest_average_cost_usd_top_agents == 0.2
-    assert result.efficiency.lowest_average_runtime_seconds_top_agents == 75
+    assert result.efficiency.lowest_average_cost_usd_top_agents is not None
+    assert result.efficiency.lowest_average_cost_usd_top_agents.agent_id == agent_a
+    assert result.efficiency.lowest_average_cost_usd_top_agents.value == 0.2
+    assert result.efficiency.lowest_average_runtime_seconds_top_agents is not None
+    assert result.efficiency.lowest_average_runtime_seconds_top_agents.agent_id == agent_a
+    assert result.efficiency.lowest_average_runtime_seconds_top_agents.value == 75
     assert result.efficiency.average_agent_cost_usd == 0.2
     assert result.efficiency.average_agent_runtime_seconds == 75
 
@@ -459,8 +463,12 @@ async def test_evaluation_set_leaderboard_ranks_by_score_cost_then_submission_ti
     assert agents_by_id[unscored_agent].final_score is None
 
     result = await evaluation_sets_endpoint.evaluation_set_detail(set_id=2)
-    assert result.efficiency.lowest_average_cost_usd_top_agents == 1.0
-    assert result.efficiency.lowest_average_runtime_seconds_top_agents == 10
+    assert result.efficiency.lowest_average_cost_usd_top_agents is not None
+    assert result.efficiency.lowest_average_cost_usd_top_agents.agent_id == lower_cost_tie_agent
+    assert result.efficiency.lowest_average_cost_usd_top_agents.value == 1.0
+    assert result.efficiency.lowest_average_runtime_seconds_top_agents is not None
+    assert result.efficiency.lowest_average_runtime_seconds_top_agents.agent_id == later_time_tie_agent
+    assert result.efficiency.lowest_average_runtime_seconds_top_agents.value == 10
     assert result.efficiency.average_agent_cost_usd == 2.8
     assert result.efficiency.average_agent_runtime_seconds == 36
 
@@ -511,8 +519,10 @@ async def test_evaluation_set_detail_efficiency_uses_all_ranked_agents_not_top_2
     assert len([agent for agent in leaderboard if agent.rank is not None]) == 26
 
     result = await evaluation_sets_endpoint.evaluation_set_detail(set_id=2)
-    assert result.efficiency.lowest_average_cost_usd_top_agents == 0.1
-    assert result.efficiency.lowest_average_runtime_seconds_top_agents == 1
+    assert result.efficiency.lowest_average_cost_usd_top_agents is not None
+    assert result.efficiency.lowest_average_cost_usd_top_agents.value == 0.1
+    assert result.efficiency.lowest_average_runtime_seconds_top_agents is not None
+    assert result.efficiency.lowest_average_runtime_seconds_top_agents.value == 1
 
 
 @pytest.mark.anyio