diff --git a/api/config.py b/api/config.py index 46f51840..b00bd729 100644 --- a/api/config.py +++ b/api/config.py @@ -206,6 +206,14 @@ AUTO_APPROVAL_POLICY_VERSION = os.getenv("AUTO_APPROVAL_POLICY_VERSION", "approval-v1") APPROVAL_PROJECTOR_POLL_INTERVAL_SECONDS = int(os.getenv("APPROVAL_PROJECTOR_POLL_INTERVAL_SECONDS", "5")) +# Old note from ADAM: Set IDs 6 and earlier still +# included the validator optimization +# of skipping all tests after the first failure, which means that +# the test pass-rate information is incorrect. We will just exclude +# these sets since they came before the Problem Info viewer anyway, +# which is the only feature uses this endpoint. +EARLIEST_SET_ID_WITH_GOOD_DATA = int(os.getenv("EARLIEST_SET_ID_WITH_GOOD_DATA", "7")) + SENTRY_DSN = os.getenv("SENTRY_DSN") if not SENTRY_DSN: logger.warning("SENTRY_DSN is not set, Sentry will not be configured.") @@ -265,5 +273,6 @@ logger.info(f"Auto Approval Enabled: {AUTO_APPROVAL_ENABLED}") logger.info(f"Auto Approval Projector Loop Enabled: {AUTO_APPROVAL_RUN_LOOP}") logger.info(f"Approval Projector Poll Interval: {APPROVAL_PROJECTOR_POLL_INTERVAL_SECONDS} second(s)") +logger.info(f"Earliest SET ID with good data: {EARLIEST_SET_ID_WITH_GOOD_DATA}") logger.info("=========================") diff --git a/api/endpoints/evaluation_sets.py b/api/endpoints/evaluation_sets.py index b5180a05..583d789f 100644 --- a/api/endpoints/evaluation_sets.py +++ b/api/endpoints/evaluation_sets.py @@ -3,6 +3,7 @@ from fastapi import APIRouter, Depends, HTTPException +from api.config import EARLIEST_SET_ID_WITH_GOOD_DATA from models.evaluation_set import ( ApprovedAgent, EvaluationSet, @@ -52,7 +53,7 @@ async def resolve_set_id(set_id: int) -> int: int Parsed and validated set ID, with -1 resolved to the latest set ID. """ - if set_id < -1: + if set_id != -1 and set_id < EARLIEST_SET_ID_WITH_GOOD_DATA: raise HTTPException(status_code=404, detail="No evaluation sets found.") resolved_set_id = None diff --git a/api/endpoints/statistics.py b/api/endpoints/statistics.py index 2dcc7e9e..d4a5fdc6 100644 --- a/api/endpoints/statistics.py +++ b/api/endpoints/statistics.py @@ -5,18 +5,11 @@ from fastapi import APIRouter, HTTPException from pydantic import BaseModel +from api.config import EARLIEST_SET_ID_WITH_GOOD_DATA from queries.evaluation_set import get_latest_set_id, get_set_created_at from queries.problem_statistics import ProblemStatistics, get_problem_statistics from utils.ttl import ttl_cache -# NOTE ADAM: Set IDs 6 and earlier still included the validator optimization -# of skipping all tests after the first failure, which means that -# the test pass-rate information is incorrect. We will just exclude -# these sets since they came before the Problem Info viewer anyway, -# which is the only feature uses this endpoint. -EARLIEST_SET_ID_WITH_GOOD_DATA = 7 - - router = APIRouter() diff --git a/queries/evaluation_set.py b/queries/evaluation_set.py index ce206c79..bec0d223 100644 --- a/queries/evaluation_set.py +++ b/queries/evaluation_set.py @@ -4,7 +4,7 @@ import asyncpg -from api.config import NUM_EVALS_PER_AGENT +from api.config import EARLIEST_SET_ID_WITH_GOOD_DATA, NUM_EVALS_PER_AGENT from models.evaluation_set import ( EvaluationSet, EvaluationSetGroup, @@ -293,7 +293,8 @@ async def create_evaluation_set_problems( async def get_all_evaluation_sets( conn: DatabaseConnection, ) -> list[EvaluationSet]: - results = await conn.fetch(""" + results = await conn.fetch( + """ SELECT es.set_id, MIN(es.created_at) AS created_at, @@ -302,9 +303,12 @@ async def get_all_evaluation_sets( c.end_date AS competition_end_date FROM evaluation_sets es LEFT JOIN competitions c ON c.set_id = es.set_id + WHERE es.set_id >= $1 GROUP BY es.set_id, c.name, c.start_date, c.end_date ORDER BY es.set_id - """) + """, + EARLIEST_SET_ID_WITH_GOOD_DATA, + ) return [EvaluationSet(**row) for row in results] diff --git a/tests/conftest.py b/tests/conftest.py index 9313b4ab..dc46d6f2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -67,6 +67,7 @@ "VALIDATOR_RUNNING_AGENT_TIMEOUT_SECONDS": "60", "VALIDATOR_RUNNING_EVAL_TIMEOUT_SECONDS": "60", "AGENT_UUID_NAMESPACE": "37c807fa-a7d2-4ac5-985b-e5ad6bbbc1c9", + "EARLIEST_SET_ID_WITH_GOOD_DATA": "0", } for key, value in TEST_ENV_DEFAULTS.items():