Implement memray build failure detector

Author: pablogsal

backend/app/models.py

@@ -198,3 +198,30 @@ class AdminSession(Base):
         Index("idx_admin_sessions_active", "is_active"),
         Index("idx_admin_sessions_expires", "expires_at"),
     )
+
+
+class MemrayBuildFailure(Base):
+    __tablename__ = "memray_build_failures"
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    commit_sha = Column(String(40), ForeignKey("commits.sha"), nullable=False)
+    binary_id = Column(String(50), ForeignKey("binaries.id"), nullable=False)
+    environment_id = Column(String(50), ForeignKey("environments.id"), nullable=False)
+    error_message = Column(Text, nullable=False)
+    failure_timestamp = Column(
+        DateTime, nullable=False, default=lambda: datetime.now(UTC).replace(tzinfo=None)
+    )
+    commit_timestamp = Column(DateTime, nullable=False)  # Timestamp of the failing commit
+
+    commit = relationship("Commit")
+    binary = relationship("Binary")
+    environment = relationship("Environment")
+
+    __table_args__ = (
+        UniqueConstraint(
+            "binary_id", "environment_id", name="unique_binary_env_failure"
+        ),
+        Index("idx_memray_failures_timestamp", "failure_timestamp"),
+        Index("idx_memray_failures_commit_timestamp", "commit_timestamp"),
+        Index("idx_memray_failures_binary_env", "binary_id", "environment_id"),
+    )

backend/app/routers/admin.py

@@ -22,16 +22,17 @@
     Binary,
     Environment,
     Run,
-    AdminUser,
     AuthToken,
     BenchmarkResult,
+    MemrayBuildFailure,
     Commit,
 )
 from ..schemas import (
     BinaryCreate,
     Binary as BinarySchema,
     EnvironmentCreate,
     Environment as EnvironmentSchema,
+    MemrayFailurePublic, 
 )
 from .. import crud
 from pydantic import BaseModel
@@ -124,6 +125,7 @@ class BenchmarkResultResponse(BaseModel):
     has_flamegraph: bool
 
 
+
 router = APIRouter(prefix="/api/admin", tags=["admin"])
 
 
@@ -620,7 +622,7 @@ async def get_token_analytics(
     total_tokens = total_result.scalar()
 
     active_result = await db.execute(
-        select(func.count(AuthToken.id)).where(AuthToken.is_active == True)
+        select(func.count(AuthToken.id)).where(AuthToken.is_active)
     )
     active_tokens = active_result.scalar()
 
@@ -1217,3 +1219,102 @@ async def get_table_schema(
                 status_code=status.HTTP_404_NOT_FOUND,
                 detail=f"Table not found or error accessing schema: {str(e)}",
             )
+
+
+
+
+@router.get("/memray-failures", response_model=List[MemrayFailurePublic])
+async def get_memray_failures(
+    current_user: AdminSession = Depends(require_admin_auth),
+    db: AsyncSession = Depends(get_database),
+):
+    """Get all memray build failures."""
+    result = await db.execute(
+        select(
+            MemrayBuildFailure.id,
+            MemrayBuildFailure.commit_sha,
+            MemrayBuildFailure.binary_id,
+            MemrayBuildFailure.environment_id,
+            Binary.name.label("binary_name"),
+            Environment.name.label("environment_name"),
+            MemrayBuildFailure.error_message,
+            MemrayBuildFailure.failure_timestamp,
+            MemrayBuildFailure.commit_timestamp,
+        )
+        .join(Binary)
+        .join(Environment)
+        .order_by(desc(MemrayBuildFailure.failure_timestamp))
+    )
+    failures = result.fetchall()
+    
+    return [
+        {
+            "id": failure.id,
+            "commit_sha": failure.commit_sha,
+            "binary_id": failure.binary_id,
+            "environment_id": failure.environment_id,
+            "binary_name": failure.binary_name,
+            "environment_name": failure.environment_name,
+            "error_message": failure.error_message,
+            "failure_timestamp": failure.failure_timestamp,
+            "commit_timestamp": failure.commit_timestamp,
+        }
+        for failure in failures
+    ]
+
+
+@router.delete("/memray-failures/{failure_id}")
+async def delete_memray_failure(
+    failure_id: int,
+    current_user: AdminSession = Depends(require_admin_auth),
+    db: AsyncSession = Depends(get_database),
+):
+    """Delete a memray build failure record."""
+    result = await db.execute(
+        select(MemrayBuildFailure).where(MemrayBuildFailure.id == failure_id)
+    )
+    failure = result.scalars().first()
+    
+    if not failure:
+        raise HTTPException(status_code=404, detail="Memray failure not found")
+    
+    await db.delete(failure)
+    await db.commit()
+    
+    return {"message": "Memray failure deleted successfully"}
+
+
+@router.get("/memray-failures/summary")
+async def get_memray_failures_summary(
+    current_user: AdminSession = Depends(require_admin_auth),
+    db: AsyncSession = Depends(get_database),
+):
+    """Get summary of current memray failures by environment."""
+    result = await db.execute(
+        select(
+            MemrayBuildFailure.binary_id,
+            MemrayBuildFailure.environment_id,
+            Binary.name.label("binary_name"),
+            Environment.name.label("environment_name"),
+            MemrayBuildFailure.commit_sha,
+            MemrayBuildFailure.failure_timestamp,
+            MemrayBuildFailure.commit_timestamp,
+        )
+        .join(Binary)
+        .join(Environment)
+        .order_by(desc(MemrayBuildFailure.failure_timestamp))
+    )
+    failures = result.fetchall()
+    
+    return [
+        {
+            "binary_id": failure.binary_id,
+            "binary_name": failure.binary_name,
+            "environment_id": failure.environment_id,
+            "environment_name": failure.environment_name,
+            "commit_sha": failure.commit_sha,
+            "failure_timestamp": failure.failure_timestamp,
+            "commit_timestamp": failure.commit_timestamp,
+        }
+        for failure in failures
+    ]

backend/app/routers/public.py

@@ -9,7 +9,7 @@
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from ..database import get_database
-from ..models import AdminUser
+from ..models import AdminUser, MemrayBuildFailure, Binary, Environment
 from ..schemas import AdminUserPublic
 
 logger = logging.getLogger(__name__)
@@ -25,9 +25,56 @@ async def get_maintainers(
     # Only return active admin users
     result = await db.execute(
         select(AdminUser)
-        .where(AdminUser.is_active == True)
+        .where(AdminUser.is_active)
         .order_by(AdminUser.added_at)
     )
     admin_users = result.scalars().all()
 
     return admin_users
+
+
+@router.get("/memray-status")
+async def get_memray_status(
+    db: AsyncSession = Depends(get_database),
+):
+    """Get current memray build status - public endpoint."""
+    # Get current failures (there should be at most one per binary+environment due to unique constraint)
+    result = await db.execute(
+        select(
+            MemrayBuildFailure.binary_id,
+            MemrayBuildFailure.environment_id,
+            Binary.name.label("binary_name"),
+            Environment.name.label("environment_name"),
+            MemrayBuildFailure.commit_timestamp,
+            MemrayBuildFailure.commit_sha,
+            MemrayBuildFailure.error_message,
+            MemrayBuildFailure.failure_timestamp,
+        )
+        .join(Binary)
+        .join(Environment)
+        .order_by(MemrayBuildFailure.commit_timestamp.desc())
+    )
+    failures = result.fetchall()
+    
+    # Build summary of affected environments
+    affected_environments = []
+    for failure in failures:
+        affected_environments.append({
+            "binary_id": failure.binary_id,
+            "environment_id": failure.environment_id,
+            "binary_name": failure.binary_name,
+            "environment_name": failure.environment_name,
+            "latest_failure": failure.commit_timestamp,
+            "commit_sha": failure.commit_sha,
+            "error_message": failure.error_message,
+            "failure_timestamp": failure.failure_timestamp,
+        })
+    
+    has_failures = len(affected_environments) > 0
+    
+    return {
+        "has_failures": has_failures,
+        "failure_count": len(affected_environments),
+        "affected_environments": affected_environments,
+        "message": "Memray build issues detected" if has_failures else "All environments healthy",
+    }

backend/app/routers/upload.py

@@ -2,13 +2,13 @@
 
 from fastapi import APIRouter, Depends, HTTPException
 from sqlalchemy.ext.asyncio import AsyncSession
-from sqlalchemy import select, delete, desc, func, update, and_
+from sqlalchemy import select, desc, func, update, and_
 from sqlalchemy.exc import IntegrityError
 from datetime import datetime
 import logging
 
 from .. import schemas, crud, models
-from ..database import get_database, transaction_scope
+from ..database import get_database
 from ..auth import get_current_token
 
 router = APIRouter(prefix="/api", tags=["upload"])
@@ -75,7 +75,7 @@ async def cleanup_old_flamegraphs_if_needed(
     )
 
     try:
-        result = await db.execute(cleanup_query)
+        await db.execute(cleanup_query)
         await db.commit()
 
         verify_result = await db.execute(count_query)
@@ -261,7 +261,7 @@ def clean_flag(flag):
     )
 
     try:
-        new_run = await crud.create_run(db, run_data)
+        await crud.create_run(db, run_data)
         logger.info(f"Successfully created run record: {run_id}")
 
         # Create benchmark results
@@ -355,3 +355,122 @@ def clean_flag(flag):
         raise HTTPException(
             status_code=500, detail=f"Failed to upload worker run: {str(e)}"
         )
+
+
+@router.post("/report-memray-failure", response_model=dict)
+async def report_memray_failure(
+    failure_data: schemas.MemrayFailureReport,
+    db: AsyncSession = Depends(get_database),
+    current_token: models.AuthToken = Depends(get_current_token),
+):
+    """Report a memray build failure for a specific commit and environment."""
+    logger = logging.getLogger(__name__)
+    
+    logger.info(
+        f"Authenticated memray failure report from token '{current_token.name}' for commit {failure_data.commit_sha[:8]}, "
+        f"binary_id='{failure_data.binary_id}', environment_id='{failure_data.environment_id}'"
+    )
+    
+    # Validate binary exists
+    binary = await crud.get_binary_by_id(db, binary_id=failure_data.binary_id)
+    if not binary:
+        logger.error(f"Memray failure report failed: Binary '{failure_data.binary_id}' not found")
+        raise HTTPException(
+            status_code=400,
+            detail=f"Binary '{failure_data.binary_id}' not found."
+        )
+    
+    # Validate environment exists
+    environment = await crud.get_environment_by_id(db, environment_id=failure_data.environment_id)
+    if not environment:
+        logger.error(f"Memray failure report failed: Environment '{failure_data.environment_id}' not found")
+        raise HTTPException(
+            status_code=400,
+            detail=f"Environment '{failure_data.environment_id}' not found."
+        )
+    
+    # Create or get commit
+    commit = await crud.get_commit_by_sha(db, sha=failure_data.commit_sha)
+    if not commit:
+        logger.info(f"Commit {failure_data.commit_sha[:8]} not found, creating new commit record")
+        # Create minimal commit record - we'll update it with full metadata later if needed
+        commit_data = schemas.CommitCreate(
+            sha=failure_data.commit_sha,
+            timestamp=failure_data.commit_timestamp,
+            message="Commit with memray build failure",
+            author="Unknown",
+            python_version=schemas.PythonVersion(major=3, minor=12, patch=0)  # Default values
+        )
+        try:
+            commit = await crud.create_commit(db, commit_data)
+            logger.info(f"Successfully created minimal commit record for {failure_data.commit_sha[:8]}")
+        except Exception as e:
+            logger.error(f"Failed to create commit record for {failure_data.commit_sha[:8]}: {e}")
+            raise HTTPException(
+                status_code=500, detail=f"Failed to create commit record: {str(e)}"
+            )
+    
+    # Check if this failure is newer than any existing failure for this binary+environment
+    existing_failure = await db.execute(
+        select(models.MemrayBuildFailure)
+        .where(
+            models.MemrayBuildFailure.binary_id == failure_data.binary_id,
+            models.MemrayBuildFailure.environment_id == failure_data.environment_id
+        )
+    )
+    existing_failure = existing_failure.scalars().first()
+    
+    if existing_failure:
+        # Check if the new failure is from a newer commit
+        if failure_data.commit_timestamp <= existing_failure.commit_timestamp:
+            logger.info(
+                f"Ignoring memray failure for {failure_data.commit_sha[:8]} as it's not newer than existing failure"
+            )
+            return {
+                "message": "Memray failure ignored (not newer than existing failure)",
+                "commit_sha": failure_data.commit_sha,
+                "binary_id": failure_data.binary_id,
+                "environment_id": failure_data.environment_id,
+            }
+        
+        # Update existing failure with newer information
+        existing_failure.commit_sha = failure_data.commit_sha
+        existing_failure.error_message = failure_data.error_message
+        existing_failure.failure_timestamp = datetime.now()
+        existing_failure.commit_timestamp = failure_data.commit_timestamp
+        
+        try:
+            await db.commit()
+            logger.info(f"Updated existing memray failure record for binary '{failure_data.binary_id}', environment '{failure_data.environment_id}'")
+        except Exception as e:
+            logger.error(f"Failed to update memray failure record: {e}")
+            raise HTTPException(
+                status_code=500, detail=f"Failed to update memray failure record: {str(e)}"
+            )
+    else:
+        # Create new failure record
+        failure_record = models.MemrayBuildFailure(
+            commit_sha=failure_data.commit_sha,
+            binary_id=failure_data.binary_id,
+            environment_id=failure_data.environment_id,
+            error_message=failure_data.error_message,
+            failure_timestamp=datetime.now(),
+            commit_timestamp=failure_data.commit_timestamp
+        )
+        
+        try:
+            db.add(failure_record)
+            await db.commit()
+            logger.info(f"Created new memray failure record for commit {failure_data.commit_sha[:8]}")
+        except Exception as e:
+            logger.error(f"Failed to create memray failure record: {e}")
+            raise HTTPException(
+                status_code=500, detail=f"Failed to create memray failure record: {str(e)}"
+            )
+    
+    return {
+        "message": "Memray failure reported successfully",
+        "commit_sha": failure_data.commit_sha,
+        "binary_id": failure_data.binary_id,
+        "environment_id": failure_data.environment_id,
+    }