diff --git a/app/db/models/core.py b/app/db/models/core.py index f3142d8..31ded49 100644 --- a/app/db/models/core.py +++ b/app/db/models/core.py @@ -14,7 +14,7 @@ UniqueConstraint, ) from sqlalchemy.dialects.postgresql import INET, UUID -from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.orm import Mapped, Session, mapped_column, relationship from .. import Base @@ -79,6 +79,17 @@ class WorkflowRun(Base): inputs: Mapped[list[RunInput]] = relationship(back_populates="run") outputs: Mapped[list[RunOutput]] = relationship(back_populates="run") + def get_queued_job(self, session: Session): + """Get the latest queued job for this workflow run.""" + from app.db.models.job_queue import QueuedJob + + return ( + session.query(QueuedJob) + .filter(QueuedJob.workflow_run_id == self.id) + .order_by(QueuedJob.queued_at.desc()) + .first() + ) + class S3Object(Base): __tablename__ = "s3_objects" diff --git a/app/db/models/job_queue.py b/app/db/models/job_queue.py index a8665e9..e37d8fc 100644 --- a/app/db/models/job_queue.py +++ b/app/db/models/job_queue.py @@ -3,12 +3,12 @@ from uuid import uuid7 from sqlalchemy import JSON, UUID, DateTime, ForeignKey, Integer, String, Text, event, func -from sqlalchemy.orm import Mapped, mapped_column, relationship, validates +from sqlalchemy.orm import Mapped, Session, mapped_column, relationship, validates from .. import Base from . import Workflow, WorkflowRun -JobStatus = Literal["pending", "submitted", "failed"] +JobStatus = Literal["pending", "submitted", "failed", "cancelled"] class QueuedJob(Base): @@ -48,6 +48,13 @@ def ensure_no_prerun_script(self, key: str, value: dict) -> dict: _raise_if_prerun_script(value) return value + def cancel_pending_job(self, session: Session, commit: bool = False) -> None: + self.status = "cancelled" + self.next_attempt_at = None + session.add(self) + if commit: + session.commit() + def _raise_if_prerun_script(launch_payload: dict) -> None: if "preRunScript" in launch_payload: diff --git a/app/routes/workflow/jobs.py b/app/routes/workflow/jobs.py index dfc7360..8824fd6 100644 --- a/app/routes/workflow/jobs.py +++ b/app/routes/workflow/jobs.py @@ -5,6 +5,7 @@ import asyncio import logging from datetime import UTC, datetime +from typing import Any from uuid import UUID from fastapi import APIRouter, Depends, HTTPException, Query, status @@ -29,7 +30,7 @@ extract_pipeline_status, format_tool_name, format_workflow_name, - get_owned_run, + get_owned_run_by_id, get_user_job_list_rows, parse_submit_datetime, ) @@ -68,16 +69,21 @@ async def cancel_workflow( current_user_id: UUID = Depends(get_current_user_id), db: Session = Depends(get_db), ) -> CancelWorkflowResponse: - """Cancel a workflow run.""" - owned_run = get_owned_run(db, current_user_id, run_id) + """Cancel a pending or running workflow run.""" + owned_run = get_owned_run_by_id(db, current_user_id, run_id) if not owned_run: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Job not found") + queued_job = owned_run.get_queued_job(session=db) + if queued_job and queued_job.status == "pending": + queued_job.cancel_pending_job(session=db) - try: - await cancel_workflow_raw(run_id) - except SeqeraAPIError as exc: - raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(exc)) from exc + if owned_run.seqera_run_id is not None: + try: + await cancel_workflow_raw(owned_run.seqera_run_id) + except SeqeraAPIError as exc: + raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(exc)) from exc + db.commit() return CancelWorkflowResponse( message="Workflow cancelled successfully", runId=run_id, @@ -197,6 +203,7 @@ async def _fetch_seqera(user_run: UserJobListRow) -> tuple[str, dict[str, object jobs.append( JobListItem( id=run_id, + seqeraRunId=seqera_run_id, jobName=job_name, workflow=workflow_type, tool=tool, @@ -227,12 +234,16 @@ async def get_job_details( db: Session = Depends(get_db), ) -> JobDetailsResponse: """Retrieve a single job with normalized status and score.""" - owned_run = get_owned_run(db, current_user_id, run_id) + owned_run = get_owned_run_by_id(db, current_user_id, run_id) if not owned_run: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Job not found") + if not owned_run.seqera_run_id: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Seqera run ID not available" + ) try: - payload = await describe_workflow(run_id) + payload = await describe_workflow(owned_run.seqera_run_id) except SeqeraConfigurationError as exc: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(exc) @@ -280,29 +291,37 @@ async def delete_job( db: Session = Depends(get_db), ) -> DeleteJobResponse: """Delete a single job. Running jobs are cancelled before deletion.""" - owned_run = get_owned_run(db, current_user_id, run_id) + owned_run = get_owned_run_by_id(db, current_user_id, run_id) if not owned_run: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Job not found") cancelled = False - try: - payload = await describe_workflow(run_id) - pipeline_status = extract_pipeline_status(payload) - if pipeline_status in {"SUBMITTED", "RUNNING"}: - await cancel_workflow_raw(run_id) - cancelled = True - - await delete_workflow_raw(run_id) - except SeqeraConfigurationError as exc: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(exc) - ) from exc - except SeqeraAPIError as exc: - raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(exc)) from exc + seqera_run_id = owned_run.seqera_run_id + # Cancel the queued job if it's still pending. + queued_job = owned_run.get_queued_job(session=db) + if queued_job and queued_job.status == "pending": + queued_job.cancel_pending_job(session=db) + if seqera_run_id: + try: + payload = await describe_workflow(seqera_run_id) + pipeline_status = extract_pipeline_status(payload) + if pipeline_status in {"SUBMITTED", "RUNNING"}: + await cancel_workflow_raw(seqera_run_id) + cancelled = True + + await delete_workflow_raw(seqera_run_id) + except SeqeraConfigurationError as exc: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(exc) + ) from exc + except SeqeraAPIError as exc: + raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(exc)) from exc db.execute(delete(RunMetric).where(RunMetric.run_id == owned_run.id)) db.execute(delete(RunInput).where(RunInput.run_id == owned_run.id)) db.execute(delete(RunOutput).where(RunOutput.run_id == owned_run.id)) + if queued_job: + db.delete(queued_job) db.delete(owned_run) db.commit() @@ -323,41 +342,68 @@ async def bulk_delete_jobs( """Delete multiple jobs. Each running job is cancelled before deletion.""" deleted: list[str] = [] failed: dict[str, str] = {} - to_delete: list[tuple[str, WorkflowRun]] = [] + status: dict = {} for run_id in payload.runIds: - owned_run = get_owned_run(db, current_user_id, run_id) + owned_run = get_owned_run_by_id(db, current_user_id, run_id) if not owned_run: failed[run_id] = "Job not found" continue + run_status: dict[str, Any] = {"run": owned_run} - try: - details = await describe_workflow(run_id) - if extract_pipeline_status(details) in {"SUBMITTED", "RUNNING"}: - await cancel_workflow_raw(run_id) - to_delete.append((run_id, owned_run)) - except (SeqeraConfigurationError, SeqeraAPIError) as exc: - failed[run_id] = str(exc) + queued_job = owned_run.get_queued_job(session=db) + if queued_job: + run_status["queued_job"] = queued_job + if queued_job.status == "pending": + queued_job.cancel_pending_job(session=db) + run_status["queue_cancelled"] = True + + if owned_run.seqera_run_id is not None: + try: + details = await describe_workflow(owned_run.seqera_run_id) + if extract_pipeline_status(details) in {"SUBMITTED", "RUNNING"}: + await cancel_workflow_raw(owned_run.seqera_run_id) + run_status["seqera_cancelled"] = True + run_status["seqera_id"] = owned_run.seqera_run_id + except (SeqeraConfigurationError, SeqeraAPIError) as exc: + failed[run_id] = str(exc) - if to_delete: - run_ids = [run_id for run_id, _ in to_delete] + status[run_id] = run_status + + delete_from_seqera = [ + (run_id, run_status) + for run_id, run_status in status.items() + if run_status.get("seqera_cancelled") and run_status.get("seqera_id") + ] + if delete_from_seqera: try: - await delete_workflows_raw(run_ids) + seqera_ids = [run_status["seqera_id"] for run_id, run_status in delete_from_seqera] + await delete_workflows_raw(seqera_ids) except (SeqeraConfigurationError, SeqeraAPIError) as exc: - for run_id in run_ids: + for run_id, _run_status in delete_from_seqera: failed[run_id] = str(exc) - return BulkDeleteJobsResponse(deleted=deleted, failed=failed) - for run_id, owned_run in to_delete: - try: - db.execute(delete(RunMetric).where(RunMetric.run_id == owned_run.id)) - db.execute(delete(RunInput).where(RunInput.run_id == owned_run.id)) - db.execute(delete(RunOutput).where(RunOutput.run_id == owned_run.id)) - db.delete(owned_run) - db.commit() - deleted.append(run_id) - except Exception as exc: # pragma: no cover - unexpected DB failures - db.rollback() - failed[run_id] = str(exc) + delete_from_db = [ + run_id + for run_id, run_status in status.items() + if run_status.get("queued_job") or run_status.get("seqera_cancelled") + ] + for run_id in delete_from_db: + # Don't delete if Seqera deletion failed. + if run_id in failed: + continue + run = status[run_id]["run"] + try: + db.execute(delete(RunMetric).where(RunMetric.run_id == run.id)) + db.execute(delete(RunInput).where(RunInput.run_id == run.id)) + db.execute(delete(RunOutput).where(RunOutput.run_id == run.id)) + if queued_job := run.get_queued_job(session=db): + db.delete(queued_job) + db.delete(run) + db.commit() + deleted.append(run_id) + except Exception as exc: # pragma: no cover - unexpected DB failures + db.rollback() + failed[run_id] = str(exc) return BulkDeleteJobsResponse(deleted=deleted, failed=failed) diff --git a/app/routes/workflow/results.py b/app/routes/workflow/results.py index 24fb9b1..9ca35cf 100644 --- a/app/routes/workflow/results.py +++ b/app/routes/workflow/results.py @@ -8,10 +8,8 @@ import yaml from fastapi import APIRouter, Depends, HTTPException, status from fastapi.responses import StreamingResponse -from sqlalchemy import select from sqlalchemy.orm import Session -from ...db.models import QueuedJob from ...schemas.workflows import ( JobSettingParamsResponse, ResultDownloadsResponse, @@ -19,7 +17,7 @@ ResultReportResponse, ResultSnapshotsResponse, ) -from ...services.job_utils import get_owned_run +from ...services.job_utils import get_owned_run_by_id from ...services.results_utils import ( _format_attachment_content_disposition, format_log_entries, @@ -49,13 +47,13 @@ async def get_result_setting_params( configProfiles are overlaid from queued_jobs.launch_payload so the result page always shows the exact values that were sent to Seqera. """ - owned_run = get_owned_run(db, current_user_id, run_id) + owned_run = get_owned_run_by_id(db, current_user_id, run_id) if not owned_run: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Job not found") form_data: dict[str, Any] = await resolve_run_form_data(owned_run) or {} - queued_job = db.scalar(select(QueuedJob).where(QueuedJob.workflow_run_id == owned_run.id)) + queued_job = owned_run.get_queued_job(session=db) if queued_job: payload = queued_job.launch_payload or {} raw_params = payload.get("paramsText") @@ -80,12 +78,16 @@ async def get_result_logs( db: Session = Depends(get_db), ) -> ResultLogsResponse: """Return Seqera workflow logs for a workflow result view.""" - owned_run = get_owned_run(db, current_user_id, run_id) + owned_run = get_owned_run_by_id(db, current_user_id, run_id) if not owned_run: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Job not found") + if not owned_run.seqera_run_id: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Seqera run ID not available" + ) try: - payload = await get_workflow_logs_raw(run_id) + payload = await get_workflow_logs_raw(owned_run.seqera_run_id) except SeqeraConfigurationError as exc: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(exc) @@ -139,7 +141,7 @@ async def get_result_downloads( db: Session = Depends(get_db), ) -> ResultDownloadsResponse: """Return pre-signed output download links for a workflow result view.""" - owned_run = get_owned_run(db, current_user_id, run_id) + owned_run = get_owned_run_by_id(db, current_user_id, run_id) if not owned_run: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Job not found") @@ -164,7 +166,7 @@ async def get_result_download_all( current_user_id: UUID = Depends(get_current_user_id), db: Session = Depends(get_db), ): - owned_run = get_owned_run(db, current_user_id, run_id) + owned_run = get_owned_run_by_id(db, current_user_id, run_id) if not owned_run: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Job not found") @@ -192,7 +194,7 @@ async def get_result_snapshots( db: Session = Depends(get_db), ) -> ResultSnapshotsResponse: """Return pre-signed snapshot download links for a workflow result view.""" - owned_run = get_owned_run(db, current_user_id, run_id) + owned_run = get_owned_run_by_id(db, current_user_id, run_id) if not owned_run: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Job not found") @@ -218,7 +220,7 @@ async def get_result_report( db: Session = Depends(get_db), ) -> ResultReportResponse: """Return one pre-signed HTML report link for a workflow result view.""" - owned_run = get_owned_run(db, current_user_id, run_id) + owned_run = get_owned_run_by_id(db, current_user_id, run_id) if not owned_run: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Job not found") diff --git a/app/schemas/workflows.py b/app/schemas/workflows.py index 48d82d8..0dca971 100644 --- a/app/schemas/workflows.py +++ b/app/schemas/workflows.py @@ -282,6 +282,7 @@ class JobListItem(BaseModel): """Individual job item in the job listing.""" id: str = Field(..., description="Workflow run ID") + seqeraRunId: str | None = Field(None, description="Seqera run ID") jobName: str = Field(..., description="Human-readable job name") workflow: str = Field(..., description="Workflow name from the workflows table") tool: str = Field(..., description="Tool used (e.g., BindCraft)") diff --git a/app/services/job_utils.py b/app/services/job_utils.py index e78c333..3015d95 100644 --- a/app/services/job_utils.py +++ b/app/services/job_utils.py @@ -107,11 +107,27 @@ def get_user_job_list_rows(db: Session, user_id: UUID) -> list[UserJobListRow]: ] -def get_owned_run(db: Session, user_id: UUID, run_id: str) -> WorkflowRun | None: +def get_owned_run_by_id(db: Session, user_id: UUID, run_id: str) -> WorkflowRun | None: + try: + workflow_run_id = UUID(str(run_id)) + except ValueError: + return None + + return db.execute( + select(WorkflowRun).where( + WorkflowRun.owner_user_id == user_id, + WorkflowRun.id == workflow_run_id, + ) + ).scalar_one_or_none() + + +def get_owned_run_by_seqera_id( + db: Session, user_id: UUID, seqera_run_id: str +) -> WorkflowRun | None: return db.execute( select(WorkflowRun).where( WorkflowRun.owner_user_id == user_id, - WorkflowRun.seqera_run_id == run_id, + WorkflowRun.seqera_run_id == seqera_run_id, ) ).scalar_one_or_none() diff --git a/tests/test_routes_results.py b/tests/test_routes_results.py index 4caf678..cd63741 100644 --- a/tests/test_routes_results.py +++ b/tests/test_routes_results.py @@ -51,9 +51,9 @@ async def test_get_result_setting_params_uses_stored_form_data(test_db): test_db.add(RunMetric(run=run, final_design_count=100)) test_db.commit() - result = await get_result_setting_params("wf-1", user.id, test_db) + result = await get_result_setting_params(str(run.id), user.id, test_db) - assert result.runId == "wf-1" + assert result.runId == str(run.id) assert result.settingParams == { "id": "s1", "binder_name": "PDL1", @@ -80,9 +80,9 @@ async def test_get_result_setting_params_falls_back_to_local_fields(test_db): test_db.add(RunMetric(run=run, final_design_count=25)) test_db.commit() - result = await get_result_setting_params("wf-2", user.id, test_db) + result = await get_result_setting_params(str(run.id), user.id, test_db) - assert result.runId == "wf-2" + assert result.runId == str(run.id) assert result.settingParams == { "id": "s2", "binder_name": "PDL2", @@ -133,7 +133,7 @@ async def test_get_result_setting_params_resolves_pdb_s3_uri_to_presigned_url(te "app.services.results_utils.generate_presigned_url", new=AsyncMock(return_value=presigned), ): - result = await get_result_setting_params("wf-pdb-1", user.id, test_db) + result = await get_result_setting_params(str(run.id), user.id, test_db) assert result.settingParams["binder_name"] == "PDL1" assert result.settingParams["starting_pdb"] == presigned @@ -161,7 +161,7 @@ async def test_get_result_setting_params_keeps_pdb_s3_uri_on_s3_error(test_db): "app.services.results_utils.generate_presigned_url", new=AsyncMock(side_effect=S3ServiceError("presign failed")), ): - result = await get_result_setting_params("wf-pdb-err", user.id, test_db) + result = await get_result_setting_params(str(run.id), user.id, test_db) assert result.settingParams["starting_pdb"] == "s3://my-bucket/uploads/target.pdb" @@ -200,9 +200,9 @@ async def test_get_result_logs_returns_formatted_entries(test_db): "app.routes.workflow.results.get_workflow_logs_raw", new=AsyncMock(return_value=payload), ): - result = await get_result_logs("wf-logs-1", user.id, test_db) + result = await get_result_logs(str(run.id), user.id, test_db) - assert result.runId == "wf-logs-1" + assert result.runId == str(run.id) assert result.entries == payload["log"]["entries"] assert result.message == "Logs retrieved" assert len(result.formattedEntries) == 2 @@ -261,7 +261,7 @@ async def test_get_result_logs_handles_top_level_payload_and_seqera_defaults(tes "app.routes.workflow.results.get_workflow_logs_raw", new=AsyncMock(return_value=payload), ): - result = await get_result_logs("wf-logs-top-level", user.id, test_db) + result = await get_result_logs(str(run.id), user.id, test_db) assert result.truncated is True assert result.pending is False @@ -293,7 +293,7 @@ async def test_get_result_logs_maps_seqera_configuration_error_to_500(test_db): new=AsyncMock(side_effect=SeqeraConfigurationError("missing seqera config")), ): with pytest.raises(HTTPException) as exc_info: - await get_result_logs("wf-logs-config-error", user.id, test_db) + await get_result_logs(str(run.id), user.id, test_db) assert exc_info.value.status_code == 500 assert exc_info.value.detail == "missing seqera config" @@ -319,7 +319,7 @@ async def test_get_result_logs_maps_seqera_api_error_to_502(test_db): new=AsyncMock(side_effect=SeqeraAPIError("seqera upstream failed")), ): with pytest.raises(HTTPException) as exc_info: - await get_result_logs("wf-logs-api-error", user.id, test_db) + await get_result_logs(str(run.id), user.id, test_db) assert exc_info.value.status_code == 502 assert exc_info.value.detail == "seqera upstream failed" @@ -376,9 +376,9 @@ async def test_get_result_downloads_returns_presigned_links_for_tracked_outputs( return_value=[], ), ): - result = await get_result_downloads("wf-downloads-1", user.id, test_db) + result = await get_result_downloads(str(run.id), user.id, test_db) - assert result.runId == "wf-downloads-1" + assert result.runId == str(run.id) assert [item.category for item in result.downloads] == ["report", "stats_csv", "pdb"] assert [item.label for item in result.downloads] == [ "PDL1_l100_s975117.html", @@ -422,7 +422,7 @@ async def read_bytes(key: str) -> bytes: return output_contents[key] with patch("app.services.results_utils.read_s3_bytes", new=AsyncMock(side_effect=read_bytes)): - response = await get_result_download_all("wf-download-all-1", user.id, test_db) + response = await get_result_download_all(str(run.id), user.id, test_db) body = b"".join([chunk async for chunk in response.body_iterator]) returned_zip = BytesIO(body) @@ -529,9 +529,9 @@ async def test_get_result_downloads_returns_presigned_links_for_proteinfold_outp return_value=[], ) as mock_list_s3_files, ): - result = await get_result_downloads(f"wf-proteinfold-downloads-{tool}", user.id, test_db) + result = await get_result_downloads(str(run.id), user.id, test_db) - assert result.runId == f"wf-proteinfold-downloads-{tool}" + assert result.runId == str(run.id) assert [item.category for item in result.downloads] == [ category for category, _, _ in expected_outputs ] @@ -583,7 +583,7 @@ async def test_get_result_downloads_maps_s3_configuration_error_to_500(test_db): new=AsyncMock(side_effect=S3ConfigurationError("missing s3 config")), ): with pytest.raises(HTTPException) as exc_info: - await get_result_downloads("wf-downloads-config-error", user.id, test_db) + await get_result_downloads(str(run.id), user.id, test_db) assert exc_info.value.status_code == 500 assert exc_info.value.detail == "missing s3 config" @@ -609,7 +609,7 @@ async def test_get_result_downloads_maps_s3_service_error_to_502(test_db): new=AsyncMock(side_effect=S3ServiceError("s3 upstream failed")), ): with pytest.raises(HTTPException) as exc_info: - await get_result_downloads("wf-downloads-service-error", user.id, test_db) + await get_result_downloads(str(run.id), user.id, test_db) assert exc_info.value.status_code == 502 assert exc_info.value.detail == "s3 upstream failed" @@ -658,9 +658,9 @@ async def test_get_result_snapshots_returns_presigned_links_for_tracked_outputs( return_value=[], ), ): - result = await get_result_snapshots("wf-snapshots-1", user.id, test_db) + result = await get_result_snapshots(str(run.id), user.id, test_db) - assert result.runId == "wf-snapshots-1" + assert result.runId == str(run.id) assert [item.category for item in result.snapshots] == ["snapshot", "snapshot"] assert [item.label for item in result.snapshots] == ["demo2_preview.png", "demo2_preview_2.png"] @@ -702,7 +702,7 @@ async def test_get_result_snapshots_maps_s3_configuration_error_to_500(test_db): new=AsyncMock(side_effect=S3ConfigurationError("missing s3 config")), ): with pytest.raises(HTTPException) as exc_info: - await get_result_snapshots("wf-snapshots-config-error", user.id, test_db) + await get_result_snapshots(str(run.id), user.id, test_db) assert exc_info.value.status_code == 500 assert exc_info.value.detail == "missing s3 config" @@ -728,7 +728,7 @@ async def test_get_result_snapshots_maps_s3_service_error_to_502(test_db): new=AsyncMock(side_effect=S3ServiceError("s3 upstream failed")), ): with pytest.raises(HTTPException) as exc_info: - await get_result_snapshots("wf-snapshots-service-error", user.id, test_db) + await get_result_snapshots(str(run.id), user.id, test_db) assert exc_info.value.status_code == 502 assert exc_info.value.detail == "s3 upstream failed" @@ -770,9 +770,9 @@ async def test_get_result_report_returns_single_presigned_html_for_tracked_outpu return_value=[], ), ): - result = await get_result_report("wf-report-1", user.id, test_db) + result = await get_result_report(str(run.id), user.id, test_db) - assert result.runId == "wf-report-1" + assert result.runId == str(run.id) assert result.report is not None assert result.report.category == "report" assert result.report.key == report_key @@ -828,7 +828,7 @@ def _list_side_effect(prefix: str, file_extension=None): side_effect=lambda key, **_kwargs: f"https://signed.example/{key}", ), ): - result = await get_result_report("wf-report-3", user.id, test_db) + result = await get_result_report(str(run.id), user.id, test_db) assert result.report is not None assert result.report.key == real_key @@ -880,7 +880,7 @@ async def test_get_result_report_maps_multiple_reports_to_409(test_db): new=AsyncMock(side_effect=ValueError("Multiple report outputs found")), ): with pytest.raises(HTTPException) as exc_info: - await get_result_report("wf-report-conflict", user.id, test_db) + await get_result_report(str(run.id), user.id, test_db) assert exc_info.value.status_code == 409 assert exc_info.value.detail == "Multiple report outputs found" @@ -906,7 +906,7 @@ async def test_get_result_report_maps_s3_configuration_error_to_500(test_db): new=AsyncMock(side_effect=S3ConfigurationError("missing s3 config")), ): with pytest.raises(HTTPException) as exc_info: - await get_result_report("wf-report-config-error", user.id, test_db) + await get_result_report(str(run.id), user.id, test_db) assert exc_info.value.status_code == 500 assert exc_info.value.detail == "missing s3 config" @@ -932,7 +932,7 @@ async def test_get_result_report_maps_s3_service_error_to_502(test_db): new=AsyncMock(side_effect=S3ServiceError("s3 upstream failed")), ): with pytest.raises(HTTPException) as exc_info: - await get_result_report("wf-report-service-error", user.id, test_db) + await get_result_report(str(run.id), user.id, test_db) assert exc_info.value.status_code == 502 assert exc_info.value.detail == "s3 upstream failed" @@ -957,9 +957,9 @@ async def test_get_result_report_allows_missing_report_payload(test_db): "app.routes.workflow.results.get_result_report_download", new=AsyncMock(return_value=None), ): - result = await get_result_report("wf-report-none", user.id, test_db) + result = await get_result_report(str(run.id), user.id, test_db) - assert result.runId == "wf-report-none" + assert result.runId == str(run.id) assert result.report is None @@ -994,9 +994,9 @@ async def test_get_result_setting_params_overlays_queued_job_payload(test_db): test_db.add(job) test_db.commit() - result = await get_result_setting_params("wf-queued-1", user.id, test_db) + result = await get_result_setting_params(str(run.id), user.id, test_db) - assert result.runId == "wf-queued-1" + assert result.runId == str(run.id) assert result.settingParams["paramsText"] == {"binder_name": "PDL1", "num_designs": 5} assert result.settingParams["configProfiles"] == ["singularity", "gadi"] @@ -1029,7 +1029,7 @@ async def test_get_result_setting_params_queued_job_invalid_yaml_kept_as_string( test_db.add(job) test_db.commit() - result = await get_result_setting_params("wf-queued-2", user.id, test_db) + result = await get_result_setting_params(str(run.id), user.id, test_db) assert result.settingParams["paramsText"] == "{\x00invalid yaml" @@ -1071,7 +1071,7 @@ async def test_get_result_download_all_maps_s3_configuration_error_to_500(test_d new=AsyncMock(side_effect=S3ConfigurationError("s3 config missing")), ): with pytest.raises(HTTPException) as exc_info: - await get_result_download_all("wf-download-all-config-err", user.id, test_db) + await get_result_download_all(str(run.id), user.id, test_db) assert exc_info.value.status_code == 500 assert exc_info.value.detail == "s3 config missing" @@ -1097,7 +1097,7 @@ async def test_get_result_download_all_maps_s3_service_error_to_502(test_db): new=AsyncMock(side_effect=S3ServiceError("s3 upstream error")), ): with pytest.raises(HTTPException) as exc_info: - await get_result_download_all("wf-download-all-service-err", user.id, test_db) + await get_result_download_all(str(run.id), user.id, test_db) assert exc_info.value.status_code == 502 assert exc_info.value.detail == "s3 upstream error" diff --git a/tests/test_routes_workflow_jobs.py b/tests/test_routes_workflow_jobs.py index aacd05f..f037eaa 100644 --- a/tests/test_routes_workflow_jobs.py +++ b/tests/test_routes_workflow_jobs.py @@ -410,9 +410,10 @@ async def test_get_job_details_success(mock_db, mock_user_id, mocker): owned_run.workflow = workflow owned_run.tool = None owned_run.submitted_form_data = None + owned_run.seqera_run_id = "seqera-wf-123" with ( - patch("app.routes.workflow.jobs.get_owned_run", return_value=owned_run), + patch("app.routes.workflow.jobs.get_owned_run_by_id", return_value=owned_run), patch( "app.routes.workflow.jobs.describe_workflow", new_callable=AsyncMock, @@ -446,7 +447,7 @@ async def test_get_job_details_success(mock_db, mock_user_id, mocker): @pytest.mark.asyncio async def test_get_job_details_not_found(mock_db, mock_user_id): """Test job details when job not found.""" - with patch("app.routes.workflow.jobs.get_owned_run", return_value=None): + with patch("app.routes.workflow.jobs.get_owned_run_by_id", return_value=None): with pytest.raises(HTTPException) as exc_info: await get_job_details( run_id="nonexistent", @@ -465,9 +466,10 @@ async def test_get_job_details_in_progress_no_score(mock_db, mock_user_id, mocke owned_run.workflow = None owned_run.tool = None owned_run.submitted_form_data = None + owned_run.seqera_run_id = "seqera-wf-456" with ( - patch("app.routes.workflow.jobs.get_owned_run", return_value=owned_run), + patch("app.routes.workflow.jobs.get_owned_run_by_id", return_value=owned_run), patch( "app.routes.workflow.jobs.describe_workflow", new_callable=AsyncMock, @@ -495,9 +497,10 @@ async def test_get_job_details_seqera_error(mock_db, mock_user_id, mocker): from app.services.seqera_errors import SeqeraAPIError owned_run = mocker.Mock() + owned_run.seqera_run_id = "seqera-wf-789" with ( - patch("app.routes.workflow.jobs.get_owned_run", return_value=owned_run), + patch("app.routes.workflow.jobs.get_owned_run_by_id", return_value=owned_run), patch( "app.routes.workflow.jobs.describe_workflow", new_callable=AsyncMock, diff --git a/tests/test_routes_workflow_jobs_extra.py b/tests/test_routes_workflow_jobs_extra.py index 41fc533..87d3633 100644 --- a/tests/test_routes_workflow_jobs_extra.py +++ b/tests/test_routes_workflow_jobs_extra.py @@ -19,6 +19,7 @@ Workflow, WorkflowRun, ) +from app.db.models.job_queue import QueuedJob from app.routes.workflow.jobs import ( bulk_delete_jobs, cancel_workflow, @@ -27,11 +28,12 @@ ) from app.schemas.workflows import BulkDeleteJobsRequest from app.services.seqera_errors import SeqeraAPIError +from tests.datagen import AppUserFactory, QueuedJobFactory, WorkflowFactory, WorkflowRunFactory @pytest.mark.asyncio async def test_cancel_workflow_not_owned_raises_404(): - with patch("app.routes.workflow.jobs.get_owned_run", return_value=None): + with patch("app.routes.workflow.jobs.get_owned_run_by_id", return_value=None): with pytest.raises(HTTPException) as exc: await cancel_workflow("wf-1", UUID("11111111-1111-1111-1111-111111111111"), Mock()) assert exc.value.status_code == 404 @@ -39,8 +41,11 @@ async def test_cancel_workflow_not_owned_raises_404(): @pytest.mark.asyncio async def test_cancel_workflow_api_error_maps_502(): + owned_run = Mock(seqera_run_id="wf-1") + owned_run.get_queued_job.return_value = None + with ( - patch("app.routes.workflow.jobs.get_owned_run", return_value=object()), + patch("app.routes.workflow.jobs.get_owned_run_by_id", return_value=owned_run), patch( "app.routes.workflow.jobs.cancel_workflow_raw", new_callable=AsyncMock, @@ -52,6 +57,35 @@ async def test_cancel_workflow_api_error_maps_502(): assert exc.value.status_code == 502 +@pytest.mark.asyncio +async def test_cancel_workflow_cancels_pending_queued_job(test_db, persistent_models): + """ + Test QueuedJob is cancelled if pending + """ + user = AppUserFactory.create_sync() + workflow = WorkflowFactory.create_sync(name="de-novo-design") + run = WorkflowRunFactory.create_sync( + workflow=workflow, + owner=user, + seqera_run_id=None, + work_dir="workdir-cancel-queued-1", + ) + queued_job = QueuedJobFactory.create_sync( + workflow=workflow, + workflow_run=run, + launch_payload={}, + status="pending", + ) + + resp = await cancel_workflow(str(run.id), user.id, test_db) + + test_db.refresh(queued_job) + assert resp.runId == str(run.id) + assert resp.status == "cancelled" + assert queued_job.status == "cancelled" + assert queued_job.next_attempt_at is None + + @pytest.mark.asyncio async def test_get_job_details_success(test_db): user = AppUser( @@ -91,9 +125,9 @@ async def test_get_job_details_success(test_db): return_value=0.912, ), ): - result = await get_job_details("wf-1", user.id, test_db) + result = await get_job_details(str(run.id), user.id, test_db) - assert result.id == "wf-1" + assert result.id == str(run.id) assert result.jobName == "PDL1" assert result.workflow == "Bindcraft" assert result.score == 0.912 @@ -147,7 +181,7 @@ async def test_delete_job_success_cancels_running_and_deletes_local_rows(test_db return_value=None, ) as mock_delete, ): - resp = await delete_job("wf-1", user.id, test_db) + resp = await delete_job(str(run.id), user.id, test_db) assert resp.deleted is True assert resp.cancelledBeforeDelete is True @@ -160,15 +194,46 @@ async def test_delete_job_success_cancels_running_and_deletes_local_rows(test_db assert test_db.execute(select(RunMetric).where(RunMetric.run_id == run.id)).first() is None +@pytest.mark.asyncio +async def test_delete_job_cancels_pending_queued_job(test_db, persistent_models): + user = AppUserFactory.create_sync() + workflow = WorkflowFactory.create_sync(name="de-novo-design") + run = WorkflowRunFactory.create_sync( + workflow=workflow, + owner=user, + seqera_run_id=None, + work_dir="workdir-queued-1", + ) + queued_job = QueuedJobFactory.create_sync( + workflow=workflow, + workflow_run=run, + launch_payload={}, + status="pending", + ) + + resp = await delete_job(str(run.id), user.id, test_db) + + assert resp.deleted is True + assert resp.cancelledBeforeDelete is False + assert test_db.get(QueuedJob, queued_job.id) is None + assert test_db.get(WorkflowRun, run.id) is None + + @pytest.mark.asyncio async def test_bulk_delete_jobs_mixed_results(): db = Mock() def _owned(_db, _uid, run_id): - return None if run_id == "missing" else SimpleNamespace(id=f"id-{run_id}") + if run_id == "missing": + return None + return SimpleNamespace( + id=f"id-{run_id}", + seqera_run_id=f"seqera-{run_id}", + get_queued_job=Mock(return_value=None), + ) with ( - patch("app.routes.workflow.jobs.get_owned_run", side_effect=_owned), + patch("app.routes.workflow.jobs.get_owned_run_by_id", side_effect=_owned), patch( "app.routes.workflow.jobs.describe_workflow", new_callable=AsyncMock, @@ -188,4 +253,73 @@ def _owned(_db, _uid, run_id): assert out.deleted == ["ok"] assert out.failed["missing"] == "Job not found" - mock_delete.assert_called_once_with(["ok"]) + mock_delete.assert_called_once_with(["seqera-ok"]) + + +@pytest.mark.asyncio +async def test_bulk_delete_jobs_seqera_delete_failure_skips_db_delete(test_db): + user = AppUser( + id=uuid4(), + auth0_user_id="auth0|bulk-delete-user", + name="Bulk Delete User", + email="bulk-delete-user@example.com", + ) + run = WorkflowRun( + id=uuid4(), + owner_user_id=user.id, + seqera_run_id="wf-bulk-delete-1", + work_dir="workdir-bulk-delete-1", + ) + test_db.add_all([user, run]) + test_db.commit() + + with ( + patch( + "app.routes.workflow.jobs.describe_workflow", + new_callable=AsyncMock, + return_value={"workflow": {"status": "SUCCEEDED"}}, + ), + patch( + "app.routes.workflow.jobs.delete_workflows_raw", + new_callable=AsyncMock, + side_effect=SeqeraAPIError("delete failed"), + ), + ): + out = await bulk_delete_jobs( + BulkDeleteJobsRequest(runIds=[str(run.id)]), + user.id, + test_db, + ) + + assert out.deleted == [] + assert out.failed == {str(run.id): "delete failed"} + assert test_db.get(WorkflowRun, run.id) is not None + + +@pytest.mark.asyncio +async def test_bulk_delete_jobs_deletes_failed_local_queued_job(test_db, persistent_models): + user = AppUserFactory.create_sync() + workflow = WorkflowFactory.create_sync(name="de-novo-design") + run = WorkflowRunFactory.create_sync( + workflow=workflow, + owner=user, + seqera_run_id=None, + work_dir="workdir-bulk-failed-queued-1", + ) + queued_job = QueuedJobFactory.create_sync( + workflow=workflow, + workflow_run=run, + launch_payload={}, + status="failed", + ) + + out = await bulk_delete_jobs( + BulkDeleteJobsRequest(runIds=[str(run.id)]), + user.id, + test_db, + ) + + assert out.deleted == [str(run.id)] + assert out.failed == {} + assert test_db.get(QueuedJob, queued_job.id) is None + assert test_db.get(WorkflowRun, run.id) is None