fix: Remove invalid evaluation sets

api/config.py

@@ -206,6 +206,14 @@
 AUTO_APPROVAL_POLICY_VERSION = os.getenv("AUTO_APPROVAL_POLICY_VERSION", "approval-v1")
 APPROVAL_PROJECTOR_POLL_INTERVAL_SECONDS = int(os.getenv("APPROVAL_PROJECTOR_POLL_INTERVAL_SECONDS", "5"))
 
+# Old note from ADAM: Set IDs 6 and earlier still
+# included the validator optimization
+# of skipping all tests after the first failure, which means that
+# the test pass-rate information is incorrect. We will just exclude
+# these sets since they came before the Problem Info viewer anyway,
+# which is the only feature uses this endpoint.
+EARLIEST_SET_ID_WITH_GOOD_DATA = int(os.getenv("EARLIEST_SET_ID_WITH_GOOD_DATA", "7"))
+
 SENTRY_DSN = os.getenv("SENTRY_DSN")
 if not SENTRY_DSN:
     logger.warning("SENTRY_DSN is not set, Sentry will not be configured.")
@@ -265,5 +273,6 @@
 logger.info(f"Auto Approval Enabled: {AUTO_APPROVAL_ENABLED}")
 logger.info(f"Auto Approval Projector Loop Enabled: {AUTO_APPROVAL_RUN_LOOP}")
 logger.info(f"Approval Projector Poll Interval: {APPROVAL_PROJECTOR_POLL_INTERVAL_SECONDS} second(s)")
+logger.info(f"Earliest SET ID with good data: {EARLIEST_SET_ID_WITH_GOOD_DATA}")
 
 logger.info("=========================")

api/endpoints/evaluation_sets.py

@@ -3,6 +3,7 @@
 
 from fastapi import APIRouter, Depends, HTTPException
 
+from api.config import EARLIEST_SET_ID_WITH_GOOD_DATA
 from models.evaluation_set import (
     ApprovedAgent,
     EvaluationSet,
@@ -52,7 +53,7 @@ async def resolve_set_id(set_id: int) -> int:
     int
         Parsed and validated set ID, with -1 resolved to the latest set ID.
     """
-    if set_id < -1:
+    if set_id != -1 and set_id < EARLIEST_SET_ID_WITH_GOOD_DATA:
         raise HTTPException(status_code=404, detail="No evaluation sets found.")
 
     resolved_set_id = None

api/endpoints/statistics.py

@@ -5,18 +5,11 @@
 from fastapi import APIRouter, HTTPException
 from pydantic import BaseModel
 
+from api.config import EARLIEST_SET_ID_WITH_GOOD_DATA
 from queries.evaluation_set import get_latest_set_id, get_set_created_at
 from queries.problem_statistics import ProblemStatistics, get_problem_statistics
 from utils.ttl import ttl_cache
 
-# NOTE ADAM: Set IDs 6 and earlier still included the validator optimization
-#            of skipping all tests after the first failure, which means that
-#            the test pass-rate information is incorrect. We will just exclude
-#            these sets since they came before the Problem Info viewer anyway,
-#            which is the only feature uses this endpoint.
-EARLIEST_SET_ID_WITH_GOOD_DATA = 7
-
-
 router = APIRouter()
 
 

queries/evaluation_set.py

@@ -4,7 +4,7 @@
 
 import asyncpg
 
-from api.config import NUM_EVALS_PER_AGENT
+from api.config import EARLIEST_SET_ID_WITH_GOOD_DATA, NUM_EVALS_PER_AGENT
 from models.evaluation_set import (
     EvaluationSet,
     EvaluationSetGroup,
@@ -293,7 +293,8 @@ async def create_evaluation_set_problems(
 async def get_all_evaluation_sets(
     conn: DatabaseConnection,
 ) -> list[EvaluationSet]:
-    results = await conn.fetch("""
+    results = await conn.fetch(
+        """
         SELECT
             es.set_id,
             MIN(es.created_at) AS created_at,
@@ -302,9 +303,12 @@ async def get_all_evaluation_sets(
             c.end_date AS competition_end_date
         FROM evaluation_sets es
         LEFT JOIN competitions c ON c.set_id = es.set_id
+        WHERE es.set_id >= $1
         GROUP BY es.set_id, c.name, c.start_date, c.end_date
         ORDER BY es.set_id
-        """)
+        """,
+        EARLIEST_SET_ID_WITH_GOOD_DATA,
+    )
     return [EvaluationSet(**row) for row in results]
 
 

tests/conftest.py

@@ -67,6 +67,7 @@
     "VALIDATOR_RUNNING_AGENT_TIMEOUT_SECONDS": "60",
     "VALIDATOR_RUNNING_EVAL_TIMEOUT_SECONDS": "60",
     "AGENT_UUID_NAMESPACE": "37c807fa-a7d2-4ac5-985b-e5ad6bbbc1c9",
+    "EARLIEST_SET_ID_WITH_GOOD_DATA": "0",
 }
 
 for key, value in TEST_ENV_DEFAULTS.items():