From 5db2cc0c8a031a82148dce9a0b71ff7db04317af Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Tue, 5 May 2026 02:24:29 +1000 Subject: [PATCH 01/11] feat: update assembly flow from requesting manifest to reporting results --- .../0011_assembly_first_and_stages.py | 150 +++++++ app/api/v1/endpoints/assemblies.py | 205 ++++++--- app/models/assembly.py | 78 +++- app/schemas/assembly.py | 87 +++- app/services/assembly_service.py | 122 ++++++ .../endpoints/test_endpoints_assemblies.py | 392 ++++++++++++++++-- tests/unit/services/test_assembly_service.py | 67 +++ 7 files changed, 1010 insertions(+), 91 deletions(-) create mode 100644 alembic/versions/0011_assembly_first_and_stages.py diff --git a/alembic/versions/0011_assembly_first_and_stages.py b/alembic/versions/0011_assembly_first_and_stages.py new file mode 100644 index 0000000..822090c --- /dev/null +++ b/alembic/versions/0011_assembly_first_and_stages.py @@ -0,0 +1,150 @@ +"""Assembly-first manifest flow: nullable fields, status, and stage reporting tables. + +Revision ID: 0011_assembly_first_and_stages +Revises: 0010_add_taxonomy_info +Create Date: 2026-05-05 +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects.postgresql import JSONB, UUID + +revision = "0011_assembly_first_and_stages" +down_revision = "0010_add_taxonomy_info" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ── 1. Make late-known assembly fields nullable ────────────────────────── + op.alter_column("assembly", "assembly_name", existing_type=sa.Text(), nullable=True) + op.alter_column("assembly", "coverage", existing_type=sa.Float(), nullable=True) + op.alter_column("assembly", "program", existing_type=sa.Text(), nullable=True) + + # ── 2. Add lifecycle status to assembly ─────────────────────────────────── + op.add_column( + "assembly", + sa.Column( + "status", + sa.Text(), + nullable=False, + server_default="requested", + ), + ) + op.create_check_constraint( + "ck_assembly_status", + "assembly", + "status IN ('requested', 'running', 'curating', 'completed', 'failed', 'cancelled')", + ) + + # ── 3. assembly_stage catalog ───────────────────────────────────────────── + op.create_table( + "assembly_stage", + sa.Column("name", sa.Text(), primary_key=True), + sa.Column("category", sa.Text(), nullable=False), + sa.Column("is_active", sa.Boolean(), nullable=False, server_default=sa.true()), + sa.CheckConstraint("category IN ('pipeline', 'manual')", name="ck_assembly_stage_category"), + ) + + op.execute( + """ + INSERT INTO assembly_stage (name, category) VALUES + ('genomeassembly', 'pipeline'), + ('ascc', 'pipeline'), + ('treeval', 'pipeline'), + ('curation-pretext', 'pipeline'), + ('manual-curation', 'manual') + """ + ) + + # ── 4. assembly_stage_run ───────────────────────────────────────────────── + op.create_table( + "assembly_stage_run", + sa.Column( + "id", + UUID(as_uuid=True), + primary_key=True, + server_default=sa.text("uuid_generate_v4()"), + ), + sa.Column( + "assembly_id", + UUID(as_uuid=True), + sa.ForeignKey("assembly.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "stage_name", + sa.Text(), + sa.ForeignKey("assembly_stage.name"), + nullable=False, + ), + sa.Column("status", sa.Text(), nullable=False), + sa.Column("external_run_id", sa.Text(), nullable=True), + sa.Column("attempt", sa.Integer(), nullable=False, server_default="1"), + sa.Column("stats", JSONB(), nullable=False, server_default="{}"), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.CheckConstraint( + "status IN ('running', 'succeeded', 'failed', 'cancelled')", + name="ck_assembly_stage_run_status", + ), + sa.UniqueConstraint("assembly_id", "stage_name", "attempt", name="uq_stage_run_assembly_stage_attempt"), + ) + op.create_index("ix_assembly_stage_run_assembly_id", "assembly_stage_run", ["assembly_id"]) + + # ── 5. assembly_stage_run_file ──────────────────────────────────────────── + op.create_table( + "assembly_stage_run_file", + sa.Column( + "id", + UUID(as_uuid=True), + primary_key=True, + server_default=sa.text("uuid_generate_v4()"), + ), + sa.Column( + "assembly_stage_run_id", + UUID(as_uuid=True), + sa.ForeignKey("assembly_stage_run.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("storage_type", sa.Text(), nullable=False), + sa.Column("storage_uri", sa.Text(), nullable=False), + sa.Column("storage_details", JSONB(), nullable=False, server_default="{}"), + sa.Column("sha256sum", sa.Text(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + ) + op.create_index( + "ix_assembly_stage_run_file_run_id", + "assembly_stage_run_file", + ["assembly_stage_run_id"], + ) + + +def downgrade() -> None: + op.drop_table("assembly_stage_run_file") + op.drop_table("assembly_stage_run") + op.drop_table("assembly_stage") + + op.drop_constraint("ck_assembly_status", "assembly", type_="check") + op.drop_column("assembly", "status") + + op.alter_column("assembly", "program", existing_type=sa.Text(), nullable=False) + op.alter_column("assembly", "coverage", existing_type=sa.Float(), nullable=False) + op.alter_column("assembly", "assembly_name", existing_type=sa.Text(), nullable=False) diff --git a/app/api/v1/endpoints/assemblies.py b/app/api/v1/endpoints/assemblies.py index 89b5b58..2e194fe 100644 --- a/app/api/v1/endpoints/assemblies.py +++ b/app/api/v1/endpoints/assemblies.py @@ -1,3 +1,4 @@ +import yaml from typing import Any, Dict, List, Optional from uuid import UUID @@ -9,7 +10,7 @@ from app.core.errors import AppError from app.core.pagination import Pagination, apply_pagination, pagination_params from app.core.policy import policy -from app.models.assembly import Assembly, AssemblyFile, AssemblyRun, AssemblySubmission +from app.models.assembly import Assembly, AssemblyFile, AssemblyRun, AssemblyStageRun, AssemblySubmission from app.models.experiment import Experiment from app.models.organism import Organism from app.models.read import Read @@ -26,6 +27,9 @@ AssemblyIntent, AssemblyIntentCancel, AssemblyIntentResponse, + AssemblyStageRunCreate, + AssemblyStageRunOut, + AssemblyStageRunUpdate, AssemblySubmissionCreate, AssemblySubmissionUpdate, AssemblyUpdate, @@ -41,6 +45,7 @@ from app.services.assembly_service import ( assembly_file_service, assembly_service, + assembly_stage_run_service, assembly_submission_service, ) from app.services.organism_service import organism_service @@ -296,11 +301,11 @@ def get_assembly_manifest( *, db: Session = Depends(get_db), taxon_id: int, - version: Optional[int] = Query(None, description="Reserved manifest version to retrieve"), + version: Optional[int] = Query(None, description="Assembly version to retrieve"), current_user: User = Depends(get_current_active_user), ) -> Any: """ - Retrieve the latest reserved assembly manifest YAML for an organism by taxon_id. + Retrieve the latest requested assembly manifest YAML for an organism by taxon_id. Returns YAML manifest with: - scientific_name and taxon_id from organism @@ -313,31 +318,31 @@ def get_assembly_manifest( """ organism, selected_sample, reads, experiments = _get_manifest_inputs_by_taxon_id(db, taxon_id) - run_query = ( - db.query(AssemblyRun) + assembly_query = ( + db.query(Assembly) .filter( - AssemblyRun.taxon_id == _organism_taxon_id(organism), - AssemblyRun.sample_id == selected_sample.id, + Assembly.taxon_id == _organism_taxon_id(organism), + Assembly.sample_id == selected_sample.id, + Assembly.status == "requested", ) - .order_by(AssemblyRun.created_at.desc()) + .order_by(Assembly.created_at.desc()) ) if version is not None: - run_query = run_query.filter(AssemblyRun.version == version) - assembly_run = run_query.first() - if not assembly_run: - raise HTTPException(status_code=404, detail="No reserved assembly manifest found") + assembly_query = assembly_query.filter(Assembly.version == version) + assembly = assembly_query.first() + if not assembly: + raise HTTPException(status_code=404, detail="No requested assembly manifest found") sample_metadata_by_id = _build_sample_metadata_by_id(db, experiments) yaml_content = generate_assembly_manifest( organism, reads, experiments, - assembly_run.tol_id, - assembly_run.version, + assembly.tol_id, + assembly.version, sample_metadata_by_id, ) - # Return as YAML response return Response(content=yaml_content, media_type="application/x-yaml") @@ -351,10 +356,10 @@ def create_assembly_intent( current_user: User = Depends(get_current_active_user), ) -> Response: """ - Reserve the next assembly version and return a manifest as YAML. + Create an assembly record and return its manifest as YAML. Returns: - YAML response with manifest data including assembly_run_id, version, status, and manifest fields + YAML response with assembly_id, version, status, and manifest fields """ organism, selected_sample, reads, experiments = _get_manifest_inputs_by_taxon_id(db, taxon_id) try: @@ -370,37 +375,24 @@ def create_assembly_intent( }, ) from exc - next_version = assembly_service.get_next_version( + assembly = assembly_service.create_from_intent( db, taxon_id=_organism_taxon_id(organism), sample_id=selected_sample.id, data_types=data_types, - ) - - run = AssemblyRun( - taxon_id=_organism_taxon_id(organism), - sample_id=selected_sample.id, - data_types=data_types, - version=next_version, tol_id=intent_in.tol_id if intent_in else None, - status="reserved", + project_id=None, ) - db.add(run) - db.commit() - db.refresh(run) sample_metadata_by_id = _build_sample_metadata_by_id(db, experiments) manifest_yaml = generate_assembly_manifest( - organism, reads, experiments, run.tol_id, run.version, sample_metadata_by_id + organism, reads, experiments, assembly.tol_id, assembly.version, sample_metadata_by_id ) - # Build complete YAML response including metadata and manifest - import yaml - response_data = { - "assembly_run_id": str(run.id), - "version": run.version, - "status": run.status, + "assembly_id": str(assembly.id), + "version": assembly.version, + "status": assembly.status, "manifest": yaml.safe_load(manifest_yaml), } yaml_response = yaml.dump(response_data, default_flow_style=False, sort_keys=False) @@ -417,7 +409,7 @@ def cancel_assembly_intent( cancel_in: AssemblyIntentCancel, current_user: User = Depends(get_current_active_user), ) -> Any: - """Cancel a reserved assembly intent by ID.""" + """Cancel a requested assembly by ID.""" organism = db.query(Organism).filter(Organism.taxon_id == taxon_id).first() if not organism: raise HTTPException(status_code=404, detail=f"Organism with taxon_id {taxon_id} not found") @@ -431,40 +423,40 @@ def cancel_assembly_intent( detail=f"No specimen sample found for taxon_id {taxon_id}", ) - run = ( - db.query(AssemblyRun) + assembly = ( + db.query(Assembly) .filter( - AssemblyRun.id == cancel_in.assembly_run_id, - AssemblyRun.taxon_id == _organism_taxon_id(organism), - AssemblyRun.sample_id == selected_sample_id, - AssemblyRun.status == "reserved", + Assembly.id == cancel_in.assembly_id, + Assembly.taxon_id == _organism_taxon_id(organism), + Assembly.sample_id == selected_sample_id, + Assembly.status == "requested", ) .first() ) - if not run: - raise HTTPException(status_code=404, detail="No reserved assembly intent found to cancel") - if cancel_in.version is not None and cancel_in.version != run.version: + if not assembly: + raise HTTPException(status_code=404, detail="No requested assembly found to cancel") + if cancel_in.version is not None and cancel_in.version != assembly.version: raise AppError( status_code=409, code="assembly_intent_version_mismatch", - message="Requested version does not match the reserved assembly intent", + message="Requested version does not match the assembly", details={ - "assembly_run_id": str(run.id), + "assembly_id": str(assembly.id), "requested_version": cancel_in.version, - "actual_version": run.version, + "actual_version": assembly.version, }, ) - run.status = "cancelled" - db.add(run) + assembly.status = "cancelled" + db.add(assembly) db.commit() - db.refresh(run) + db.refresh(assembly) return { - "id": str(run.id), - "taxon_id": run.taxon_id, - "sample_id": str(run.sample_id), - "version": run.version, - "status": run.status, + "id": str(assembly.id), + "taxon_id": assembly.taxon_id, + "sample_id": str(assembly.sample_id), + "version": assembly.version, + "status": assembly.status, } @@ -520,6 +512,42 @@ def create_assembly( return assembly +@router.get("/{assembly_id}/manifest") +def get_manifest_by_assembly_id( + *, + db: Session = Depends(get_db), + assembly_id: UUID, + current_user: User = Depends(get_current_active_user), +) -> Any: + """Retrieve the assembly manifest YAML for a specific assembly by ID.""" + assembly = db.query(Assembly).filter(Assembly.id == assembly_id).first() + if not assembly: + raise HTTPException(status_code=404, detail="Assembly not found") + + organism = db.query(Organism).filter(Organism.taxon_id == assembly.taxon_id).first() + if not organism: + raise HTTPException(status_code=404, detail="Organism not found for this assembly") + + sample_ids = [ + row[0] + for row in db.query(Sample.id).filter(Sample.taxon_id == assembly.taxon_id).all() + ] + experiments = db.query(Experiment).filter(Experiment.sample_id.in_(sample_ids)).all() + if not experiments: + raise HTTPException(status_code=404, detail="No experiments found for this assembly") + + experiment_ids = [exp.id for exp in experiments] + reads = db.query(Read).filter(Read.experiment_id.in_(experiment_ids)).all() + if not reads: + raise HTTPException(status_code=404, detail="No reads found for this assembly") + + sample_metadata_by_id = _build_sample_metadata_by_id(db, experiments) + yaml_content = generate_assembly_manifest( + organism, reads, experiments, assembly.tol_id, assembly.version, sample_metadata_by_id + ) + return Response(content=yaml_content, media_type="application/x-yaml") + + @router.get("/{assembly_id}", response_model=AssemblySchema) def read_assembly( *, @@ -722,3 +750,66 @@ def delete_assembly_file( assembly_file_service.remove(db, id=file_id) return {"message": "File deleted successfully"} + + +# ========================================== +# Assembly Stage Run endpoints +# ========================================== + + +@router.get("/{assembly_id}/stage-runs", response_model=List[AssemblyStageRunOut]) +def list_stage_runs( + *, + db: Session = Depends(get_db), + assembly_id: UUID, + current_user: User = Depends(get_current_active_user), +) -> Any: + """List all stage runs for an assembly, newest first.""" + assembly = assembly_service.get(db, id=assembly_id) + if not assembly: + raise HTTPException(status_code=404, detail="Assembly not found") + return assembly_stage_run_service.get_by_assembly_id(db, assembly_id=assembly_id) + + +@router.post("/{assembly_id}/stage-runs", response_model=AssemblyStageRunOut) +@policy("assemblies:write") +def create_stage_run( + *, + db: Session = Depends(get_db), + assembly_id: UUID, + run_in: AssemblyStageRunCreate, + current_user: User = Depends(get_current_active_user), +) -> Any: + """Report a stage run result for an assembly.""" + assembly = assembly_service.get(db, id=assembly_id) + if not assembly: + raise HTTPException(status_code=404, detail="Assembly not found") + return assembly_stage_run_service.create_with_files( + db, + assembly_id=assembly_id, + run_in=run_in, + ) + + +@router.patch("/{assembly_id}/stage-runs/{stage_run_id}", response_model=AssemblyStageRunOut) +@policy("assemblies:write") +def update_stage_run( + *, + db: Session = Depends(get_db), + assembly_id: UUID, + stage_run_id: UUID, + update_in: AssemblyStageRunUpdate, + current_user: User = Depends(get_current_active_user), +) -> Any: + """Update status, stats, or files for an existing stage run.""" + stage_run = ( + db.query(AssemblyStageRun) + .filter( + AssemblyStageRun.id == stage_run_id, + AssemblyStageRun.assembly_id == assembly_id, + ) + .first() + ) + if not stage_run: + raise HTTPException(status_code=404, detail="Stage run not found") + return assembly_stage_run_service.update_with_files(db, db_obj=stage_run, update_in=update_in) diff --git a/app/models/assembly.py b/app/models/assembly.py index c7cdd2b..16faae9 100644 --- a/app/models/assembly.py +++ b/app/models/assembly.py @@ -2,6 +2,7 @@ from sqlalchemy import ( BigInteger, + Boolean, Column, DateTime, Float, @@ -9,6 +10,7 @@ Integer, PrimaryKeyConstraint, Text, + UniqueConstraint, func, ) from sqlalchemy import Enum as SQLAlchemyEnum @@ -33,7 +35,7 @@ class Assembly(Base): project_id = Column(UUID(as_uuid=True), ForeignKey("project.id"), nullable=True) # Assembly metadata fields - assembly_name = Column(Text, nullable=False) + assembly_name = Column(Text, nullable=True) assembly_type = Column(Text, nullable=False, default="clone or isolate") tol_id = Column(Text, nullable=True) data_types = Column( @@ -48,9 +50,10 @@ class Assembly(Base): ), nullable=False, ) - coverage = Column(Float, nullable=False) - program = Column(Text, nullable=False) + coverage = Column(Float, nullable=True) + program = Column(Text, nullable=True) mingaplength = Column(Float, nullable=True) + status = Column(Text, nullable=False, default="requested") moleculetype = Column( SQLAlchemyEnum("genomic DNA", "genomic RNA", name="molecule_type"), nullable=False, @@ -240,3 +243,72 @@ class AssemblyRead(Base): "Assembly", backref=backref("assembly_reads", cascade="all, delete-orphan") ) read = relationship("Read", backref=backref("reads_assembly", cascade="all, delete-orphan")) + + +class AssemblyStage(Base): + """Catalog of known assembly pipeline/manual stages.""" + + __tablename__ = "assembly_stage" + + name = Column(Text, primary_key=True) + category = Column(Text, nullable=False) + is_active = Column(Boolean, nullable=False, default=True) + + +class AssemblyStageRun(Base): + """A single reported run of an assembly stage (pipeline or manual).""" + + __tablename__ = "assembly_stage_run" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + assembly_id = Column( + UUID(as_uuid=True), ForeignKey("assembly.id", ondelete="CASCADE"), nullable=False + ) + stage_name = Column(Text, ForeignKey("assembly_stage.name"), nullable=False) + status = Column( + SQLAlchemyEnum("running", "succeeded", "failed", "cancelled", name="stage_run_status"), + nullable=False, + ) + external_run_id = Column(Text, nullable=True) + attempt = Column(Integer, nullable=False, default=1) + stats = Column(JSONB, nullable=False, default=dict) + started_at = Column(DateTime(timezone=True), nullable=True) + completed_at = Column(DateTime(timezone=True), nullable=True) + + created_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), + nullable=False, + server_default=func.now(), + onupdate=func.now(), + ) + + __table_args__ = ( + UniqueConstraint("assembly_id", "stage_name", "attempt", name="uq_stage_run_assembly_stage_attempt"), + ) + + assembly = relationship("Assembly", backref=backref("stage_runs", cascade="all, delete-orphan")) + stage = relationship("AssemblyStage", backref="runs") + + +class AssemblyStageRunFile(Base): + """Files attached to an assembly stage run.""" + + __tablename__ = "assembly_stage_run_file" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + assembly_stage_run_id = Column( + UUID(as_uuid=True), + ForeignKey("assembly_stage_run.id", ondelete="CASCADE"), + nullable=False, + ) + storage_type = Column(Text, nullable=False) + storage_uri = Column(Text, nullable=False) + storage_details = Column(JSONB, nullable=False, default=dict) + sha256sum = Column(Text, nullable=False) + + created_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now()) + + stage_run = relationship( + "AssemblyStageRun", backref=backref("files", cascade="all, delete-orphan") + ) diff --git a/app/schemas/assembly.py b/app/schemas/assembly.py index 9ad3ad0..caf0dc1 100644 --- a/app/schemas/assembly.py +++ b/app/schemas/assembly.py @@ -1,6 +1,6 @@ from datetime import datetime from enum import Enum -from typing import Dict, Optional +from typing import Any, Dict, List, Optional from uuid import UUID from pydantic import BaseModel, ConfigDict, Field @@ -35,16 +35,17 @@ class AssemblyBase(BaseModel): taxon_id: int sample_id: UUID project_id: Optional[UUID] = None - assembly_name: str + assembly_name: Optional[str] = None assembly_type: str = "clone or isolate" - tol_id: str + tol_id: Optional[str] = None data_types: AssemblyDataTypes - coverage: float - program: str + coverage: Optional[float] = None + program: Optional[str] = None mingaplength: Optional[float] = None moleculetype: str = "genomic DNA" description: Optional[str] = None version: int = 1 + status: str = "requested" # Schema for creating a new assembly @@ -80,7 +81,7 @@ class AssemblyIntent(BaseModel): class AssemblyIntentResponse(BaseModel): """Response envelope for assembly intent creation.""" - assembly_run_id: UUID + assembly_id: UUID version: int status: str manifest_yaml: str @@ -89,7 +90,7 @@ class AssemblyIntentResponse(BaseModel): class AssemblyIntentCancel(BaseModel): """Schema for cancelling an existing assembly intent.""" - assembly_run_id: UUID + assembly_id: UUID version: Optional[int] = None @@ -109,6 +110,7 @@ class AssemblyUpdate(BaseModel): moleculetype: Optional[str] = None version_number: Optional[int] = None description: Optional[str] = None + status: Optional[str] = None # Schema for assembly in DB @@ -252,3 +254,74 @@ class AssemblyFile(AssemblyFileInDBBase): """Schema for returning assembly file information.""" pass + + +# ========================================== +# AssemblyStageRun schemas +# ========================================== + + +class AssemblyStageRunFileCreate(BaseModel): + """File payload for a stage run.""" + + storage_type: str + storage_uri: str + storage_details: Dict[str, Any] = {} + sha256sum: str + + +class AssemblyStageRunCreate(BaseModel): + """Schema for reporting a stage run result.""" + + stage_name: str + status: str + external_run_id: Optional[str] = None + attempt: int = 1 + stats: Dict[str, Any] = {} + started_at: Optional[datetime] = None + completed_at: Optional[datetime] = None + files: List[AssemblyStageRunFileCreate] = [] + + +class AssemblyStageRunUpdate(BaseModel): + """Schema for updating an existing stage run. If files is provided, replaces all existing files.""" + + status: Optional[str] = None + external_run_id: Optional[str] = None + stats: Optional[Dict[str, Any]] = None + started_at: Optional[datetime] = None + completed_at: Optional[datetime] = None + files: Optional[List[AssemblyStageRunFileCreate]] = None + + +class AssemblyStageRunFileOut(BaseModel): + """Stage run file response schema.""" + + id: UUID + assembly_stage_run_id: UUID + storage_type: str + storage_uri: str + storage_details: Dict[str, Any] + sha256sum: str + created_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +class AssemblyStageRunOut(BaseModel): + """Stage run response schema.""" + + id: UUID + assembly_id: UUID + stage_name: str + status: str + external_run_id: Optional[str] + attempt: int + stats: Dict[str, Any] + started_at: Optional[datetime] + completed_at: Optional[datetime] + created_at: datetime + updated_at: datetime + files: List[AssemblyStageRunFileOut] = [] + + model_config = ConfigDict(from_attributes=True) diff --git a/app/services/assembly_service.py b/app/services/assembly_service.py index 060a8eb..c14cfcf 100644 --- a/app/services/assembly_service.py +++ b/app/services/assembly_service.py @@ -9,6 +9,8 @@ AssemblyFile, AssemblyRead, AssemblyRun, + AssemblyStageRun, + AssemblyStageRunFile, AssemblySubmission, ) from app.models.experiment import Experiment @@ -18,6 +20,8 @@ AssemblyCreate, AssemblyFileCreate, AssemblyFileUpdate, + AssemblyStageRunCreate, + AssemblyStageRunUpdate, AssemblySubmissionCreate, AssemblySubmissionUpdate, AssemblyUpdate, @@ -155,6 +159,10 @@ def get_next_version( sample_id: UUID, data_types: str, ) -> int: + # Assembly is the primary source of truth for version numbers. + # AssemblyRun is also checked during the transition period to avoid conflicts + # with legacy reservations. Remove the AssemblyRun check once legacy records + # are resolved and the table is dropped. max_assembly = ( db.query(func.max(Assembly.version)) .filter( @@ -175,6 +183,34 @@ def get_next_version( ) return max(max_assembly or 0, max_run or 0) + 1 + def create_from_intent( + self, + db: Session, + *, + taxon_id: int, + sample_id: UUID, + data_types: str, + tol_id: Optional[str], + project_id: Optional[UUID], + ) -> Assembly: + """Create an Assembly at manifest-request time with status='requested'.""" + version = self.get_next_version( + db, taxon_id=taxon_id, sample_id=sample_id, data_types=data_types + ) + assembly = Assembly( + taxon_id=taxon_id, + sample_id=sample_id, + data_types=data_types, + version=version, + tol_id=tol_id, + project_id=project_id, + status="requested", + ) + db.add(assembly) + db.commit() + db.refresh(assembly) + return assembly + class AssemblySubmissionService( BaseService[AssemblySubmission, AssemblySubmissionCreate, AssemblySubmissionUpdate] @@ -233,7 +269,93 @@ def get_by_assembly_id(self, db: Session, assembly_id: UUID) -> List[AssemblyRea return db.query(AssemblyRead).filter(AssemblyRead.assembly_id == assembly_id).all() +class AssemblyStageRunService(BaseService[AssemblyStageRun, AssemblyStageRunCreate, AssemblyStageRunUpdate]): + """Service for AssemblyStageRun operations.""" + + def get_by_assembly_id( + self, db: Session, assembly_id: UUID + ) -> List[AssemblyStageRun]: + return ( + db.query(AssemblyStageRun) + .filter(AssemblyStageRun.assembly_id == assembly_id) + .order_by(AssemblyStageRun.created_at.desc()) + .all() + ) + + def create_with_files( + self, + db: Session, + *, + assembly_id: UUID, + run_in: AssemblyStageRunCreate, + ) -> AssemblyStageRun: + run = AssemblyStageRun( + assembly_id=assembly_id, + stage_name=run_in.stage_name, + status=run_in.status, + external_run_id=run_in.external_run_id, + attempt=run_in.attempt, + stats=run_in.stats, + started_at=run_in.started_at, + completed_at=run_in.completed_at, + ) + db.add(run) + db.flush() + for f in run_in.files: + db.add( + AssemblyStageRunFile( + assembly_stage_run_id=run.id, + storage_type=f.storage_type, + storage_uri=f.storage_uri, + storage_details=f.storage_details, + sha256sum=f.sha256sum, + ) + ) + db.commit() + db.refresh(run) + return run + + def update_with_files( + self, + db: Session, + *, + db_obj: AssemblyStageRun, + update_in: AssemblyStageRunUpdate, + ) -> AssemblyStageRun: + if update_in.status is not None: + db_obj.status = update_in.status + if update_in.external_run_id is not None: + db_obj.external_run_id = update_in.external_run_id + if update_in.stats is not None: + db_obj.stats = update_in.stats + if update_in.started_at is not None: + db_obj.started_at = update_in.started_at + if update_in.completed_at is not None: + db_obj.completed_at = update_in.completed_at + + if update_in.files is not None: + # Replace all files + db.query(AssemblyStageRunFile).filter( + AssemblyStageRunFile.assembly_stage_run_id == db_obj.id + ).delete() + for f in update_in.files: + db.add( + AssemblyStageRunFile( + assembly_stage_run_id=db_obj.id, + storage_type=f.storage_type, + storage_uri=f.storage_uri, + storage_details=f.storage_details, + sha256sum=f.sha256sum, + ) + ) + + db.commit() + db.refresh(db_obj) + return db_obj + + assembly_service = AssemblyService(Assembly) assembly_submission_service = AssemblySubmissionService(AssemblySubmission) assembly_file_service = AssemblyFileService(AssemblyFile) assembly_read_service = AssemblyReadService(AssemblyRead) +assembly_stage_run_service = AssemblyStageRunService(AssemblyStageRun) diff --git a/tests/unit/endpoints/test_endpoints_assemblies.py b/tests/unit/endpoints/test_endpoints_assemblies.py index 2f8b8b6..63efcb2 100644 --- a/tests/unit/endpoints/test_endpoints_assemblies.py +++ b/tests/unit/endpoints/test_endpoints_assemblies.py @@ -100,6 +100,7 @@ def test_create_assembly_from_experiments_success(monkeypatch): program="hifiasm", moleculetype="genomic DNA", version=1, + status="requested", created_at=datetime.now(timezone.utc), updated_at=datetime.now(timezone.utc), ) @@ -213,12 +214,13 @@ def test_get_assembly_manifest_success(monkeypatch): read_number=None, lane_number=None, ) - assembly_run = SimpleNamespace( + requested_assembly = SimpleNamespace( id="run-1", taxon_id=172942, sample_id="550e8400-e29b-41d4-a716-446655440000", tol_id="tol-123", version=1, + status="requested", ) class MockQuery: @@ -255,8 +257,8 @@ def query(self, model): return MockQuery([experiment]) elif self.call_count == 6: # reads query return MockQuery([read]) - elif self.call_count == 7: # latest assembly_run query - return MockQuery(assembly_run) + elif self.call_count == 7: # latest requested assembly query + return MockQuery(requested_assembly) elif self.call_count == 8: # sample metadata query for per-read manifest fields return MockQuery([sample]) return MockQuery([]) @@ -381,12 +383,19 @@ def test_create_assembly_intent_allows_empty_body(monkeypatch): "_get_manifest_inputs_by_taxon_id", lambda db, taxon_id: (organism, selected_sample, reads, experiments), ) - monkeypatch.setattr(assemblies.assembly_service, "get_next_version", lambda *_, **__: 1) + mock_assembly = SimpleNamespace( + id=run_id, + version=1, + status="requested", + tol_id=None, + ) + monkeypatch.setattr( + assemblies.assembly_service, + "create_from_intent", + lambda db, **kwargs: mock_assembly, + ) class _FakeIntentDB: - def __init__(self): - self.last_added = None - def query(self, _model): class _Q: def filter(self, *_a, **_k): @@ -403,17 +412,6 @@ def all(self): return _Q() - def add(self, _obj): - self.last_added = _obj - return None - - def commit(self): - return None - - def refresh(self, obj): - obj.id = run_id - return None - app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( is_active=True, roles=["curator"], is_superuser=False ) @@ -425,9 +423,9 @@ def refresh(self, obj): import yaml as _yaml body = _yaml.safe_load(resp.content) - assert body["assembly_run_id"] == str(run_id) + assert body["assembly_id"] == str(run_id) assert body["version"] == 1 - assert body["status"] == "reserved" + assert body["status"] == "requested" assert "manifest" in body @@ -442,7 +440,7 @@ def test_cancel_assembly_intent_success(monkeypatch): taxon_id=172942, sample_id=selected_sample_id, version=2, - status="reserved", + status="requested", ) monkeypatch.setattr( @@ -490,7 +488,7 @@ def refresh(self, _obj): resp = client.post( "/api/v1/assemblies/intent/172942/cancel", - json={"assembly_run_id": str(run_id)}, + json={"assembly_id": str(run_id)}, ) assert resp.status_code == 200 @@ -541,8 +539,354 @@ def query(self, _model): resp = client.post( "/api/v1/assemblies/intent/172942/cancel", - json={"assembly_run_id": str(uuid4())}, + json={"assembly_id": str(uuid4())}, + ) + + assert resp.status_code == 404 + assert "No requested assembly found to cancel" in resp.json()["error"]["message"] + + +# ── New manifest-by-assembly-id tests ────────────────────────────────────── + + +def test_get_manifest_by_assembly_id_success(monkeypatch): + """GET /{assembly_id}/manifest returns YAML for a known assembly.""" + from uuid import uuid4 as _uuid4 + + client = TestClient(app) + assembly_id = _uuid4() + + assembly = SimpleNamespace( + id=assembly_id, + taxon_id=172942, + sample_id="550e8400-e29b-41d4-a716-446655440000", + tol_id="tol-999", + version=3, + status="requested", + ) + organism = SimpleNamespace(scientific_name="Test Species", taxon_id=172942) + sample = SimpleNamespace( + id="550e8400-e29b-41d4-a716-446655440000", + taxon_id=172942, + bpa_sample_id="102.100.100/9000", + specimen_id="SPEC-001", + ) + experiment = SimpleNamespace( + id="exp-1", + sample_id="550e8400-e29b-41d4-a716-446655440000", + platform="PACBIO_SMRT", + library_strategy="WGS", + bpa_package_id="pkg-exp-1", + bioplatforms_base_url=None, + ) + read = SimpleNamespace( + id="read-1", + experiment_id="exp-1", + file_name="sample.ccs.bam", + file_checksum="abc123", + bioplatforms_url="https://example.com/1", + read_number=None, + lane_number=None, + ) + + class MockQuery: + def __init__(self, rv): + self.rv = rv + + def filter(self, *a, **k): + return self + + def all(self): + return self.rv if isinstance(self.rv, list) else [] + + def first(self): + return self.rv if not isinstance(self.rv, list) else None + + class MockDB: + def __init__(self): + self.n = 0 + + def query(self, _m): + self.n += 1 + if self.n == 1: + return MockQuery(assembly) + if self.n == 2: + return MockQuery(organism) + if self.n == 3: + return MockQuery([(sample.id,)]) + if self.n == 4: + return MockQuery([experiment]) + if self.n == 5: + return MockQuery([read]) + if self.n == 6: + return MockQuery([sample]) + return MockQuery([]) + + app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["admin"], is_superuser=False + ) + app.dependency_overrides[assemblies.get_db] = lambda: MockDB() + + resp = client.get(f"/api/v1/assemblies/{assembly_id}/manifest") + + assert resp.status_code == 200 + assert resp.headers["content-type"] == "application/x-yaml" + assert b"scientific_name: Test Species" in resp.content + assert b"taxon_id: 172942" in resp.content + assert b"PACBIO_SMRT:" in resp.content + + +def test_get_manifest_by_assembly_id_not_found(): + """GET /{assembly_id}/manifest returns 404 when assembly does not exist.""" + from uuid import uuid4 as _uuid4 + + client = TestClient(app) + + class MockDB: + def query(self, _m): + return self + + def filter(self, *a, **k): + return self + + def first(self): + return None + + app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["admin"], is_superuser=False ) + app.dependency_overrides[assemblies.get_db] = lambda: MockDB() + resp = client.get(f"/api/v1/assemblies/{_uuid4()}/manifest") assert resp.status_code == 404 - assert "No reserved assembly intent found" in resp.json()["error"]["message"] + assert "Assembly not found" in resp.json()["error"]["message"] + + +# ── Stage-run endpoint tests ──────────────────────────────────────────────── + + +def _make_stage_run(assembly_id=None): + """Return a SimpleNamespace that satisfies AssemblyStageRunOut.""" + from datetime import datetime, timezone + + return SimpleNamespace( + id=uuid4(), + assembly_id=assembly_id or uuid4(), + stage_name="genomeassembly", + status="succeeded", + external_run_id="ext-123", + attempt=1, + stats={"n50": 10000}, + started_at=datetime(2026, 1, 1, tzinfo=timezone.utc), + completed_at=datetime(2026, 1, 2, tzinfo=timezone.utc), + created_at=datetime(2026, 1, 1, tzinfo=timezone.utc), + updated_at=datetime(2026, 1, 2, tzinfo=timezone.utc), + files=[], + ) + + +def test_list_stage_runs(monkeypatch): + """GET /{assembly_id}/stage-runs returns list of stage runs.""" + client = TestClient(app) + assembly_id = uuid4() + stage_run = _make_stage_run(assembly_id=assembly_id) + + monkeypatch.setattr( + assemblies.assembly_service, "get", lambda db, id: SimpleNamespace(id=assembly_id) + ) + monkeypatch.setattr( + assemblies.assembly_stage_run_service, + "get_by_assembly_id", + lambda db, assembly_id: [stage_run], + ) + + app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["admin"], is_superuser=False + ) + app.dependency_overrides[assemblies.get_db] = _override_db(_FakeSession()) + + resp = client.get(f"/api/v1/assemblies/{assembly_id}/stage-runs") + assert resp.status_code == 200 + body = resp.json() + assert isinstance(body, list) + assert len(body) == 1 + assert body[0]["stage_name"] == "genomeassembly" + assert body[0]["status"] == "succeeded" + + +def test_list_stage_runs_assembly_not_found(monkeypatch): + """GET /{assembly_id}/stage-runs returns 404 when assembly missing.""" + client = TestClient(app) + + monkeypatch.setattr(assemblies.assembly_service, "get", lambda db, id: None) + + app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["admin"], is_superuser=False + ) + app.dependency_overrides[assemblies.get_db] = _override_db(_FakeSession()) + + resp = client.get(f"/api/v1/assemblies/{uuid4()}/stage-runs") + assert resp.status_code == 404 + + +def test_create_stage_run_success(monkeypatch): + """POST /{assembly_id}/stage-runs creates a stage run with files.""" + client = TestClient(app) + assembly_id = uuid4() + stage_run = _make_stage_run(assembly_id=assembly_id) + stage_run.files = [ + SimpleNamespace( + id=uuid4(), + assembly_stage_run_id=stage_run.id, + storage_type="s3", + storage_uri="s3://bucket/key", + storage_details={"region": "ap-southeast-2"}, + sha256sum="deadbeef", + created_at=stage_run.created_at, + ) + ] + + monkeypatch.setattr( + assemblies.assembly_service, "get", lambda db, id: SimpleNamespace(id=assembly_id) + ) + monkeypatch.setattr( + assemblies.assembly_stage_run_service, + "create_with_files", + lambda db, **kwargs: stage_run, + ) + + app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["curator"], is_superuser=False + ) + app.dependency_overrides[assemblies.get_db] = _override_db(_FakeSession()) + + resp = client.post( + f"/api/v1/assemblies/{assembly_id}/stage-runs", + json={ + "stage_name": "genomeassembly", + "status": "succeeded", + "attempt": 1, + "stats": {"n50": 10000}, + "files": [ + { + "storage_type": "s3", + "storage_uri": "s3://bucket/key", + "storage_details": {"region": "ap-southeast-2"}, + "sha256sum": "deadbeef", + } + ], + }, + ) + + assert resp.status_code == 200 + body = resp.json() + assert body["stage_name"] == "genomeassembly" + assert body["status"] == "succeeded" + assert len(body["files"]) == 1 + assert body["files"][0]["storage_type"] == "s3" + assert body["files"][0]["sha256sum"] == "deadbeef" + + +def test_update_stage_run_status(monkeypatch): + """PATCH /{assembly_id}/stage-runs/{stage_run_id} updates status.""" + client = TestClient(app) + assembly_id = uuid4() + stage_run = _make_stage_run(assembly_id=assembly_id) + updated_run = _make_stage_run(assembly_id=assembly_id) + updated_run.id = stage_run.id + updated_run.status = "failed" + + class _Q: + def filter(self, *_a, **_k): + return self + + def first(self): + return stage_run + + monkeypatch.setattr( + assemblies.assembly_stage_run_service, + "update_with_files", + lambda db, db_obj, update_in: updated_run, + ) + + app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["curator"], id=uuid4(), is_superuser=False + ) + + class _DB: + def query(self, _m): + return _Q() + + app.dependency_overrides[assemblies.get_db] = lambda: _DB() + + resp = client.patch( + f"/api/v1/assemblies/{assembly_id}/stage-runs/{stage_run.id}", + json={"status": "failed"}, + ) + + assert resp.status_code == 200 + assert resp.json()["status"] == "failed" + + +def test_update_stage_run_replaces_files(monkeypatch): + """PATCH replaces all files when files list is provided.""" + from datetime import datetime, timezone + + client = TestClient(app) + assembly_id = uuid4() + stage_run = _make_stage_run(assembly_id=assembly_id) + new_file = SimpleNamespace( + id=uuid4(), + assembly_stage_run_id=stage_run.id, + storage_type="gcs", + storage_uri="gs://bucket/new-key", + storage_details={}, + sha256sum="cafebabe", + created_at=datetime(2026, 1, 3, tzinfo=timezone.utc), + ) + updated_run = _make_stage_run(assembly_id=assembly_id) + updated_run.id = stage_run.id + updated_run.files = [new_file] + + class _Q: + def filter(self, *_a, **_k): + return self + + def first(self): + return stage_run + + monkeypatch.setattr( + assemblies.assembly_stage_run_service, + "update_with_files", + lambda db, db_obj, update_in: updated_run, + ) + + app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["curator"], id=uuid4(), is_superuser=False + ) + + class _DB: + def query(self, _m): + return _Q() + + app.dependency_overrides[assemblies.get_db] = lambda: _DB() + + resp = client.patch( + f"/api/v1/assemblies/{assembly_id}/stage-runs/{stage_run.id}", + json={ + "files": [ + { + "storage_type": "gcs", + "storage_uri": "gs://bucket/new-key", + "storage_details": {}, + "sha256sum": "cafebabe", + } + ] + }, + ) + + assert resp.status_code == 200 + body = resp.json() + assert len(body["files"]) == 1 + assert body["files"][0]["sha256sum"] == "cafebabe" + assert body["files"][0]["storage_type"] == "gcs" diff --git a/tests/unit/services/test_assembly_service.py b/tests/unit/services/test_assembly_service.py index e2fc1bc..78114b7 100644 --- a/tests/unit/services/test_assembly_service.py +++ b/tests/unit/services/test_assembly_service.py @@ -326,3 +326,70 @@ def test_create_from_experiments_overrides_data_types(self, mock_db, assembly_se # The data_types should be determined from experiments (ILLUMINA+WGS = Hi-C) assert created_assembly_in.taxon_id == 172942 + + +class TestCreateFromIntent: + """Tests for create_from_intent method.""" + + def test_create_from_intent_creates_assembly_with_requested_status( + self, mock_db, assembly_service + ): + """create_from_intent should create an Assembly with status='requested'.""" + sample_id = uuid.uuid4() + taxon_id = 172942 + + # get_next_version queries Assembly + AssemblyRun - both return None + mock_query = Mock() + mock_query.filter.return_value.scalar.return_value = None + mock_db.query.return_value = mock_query + mock_db.add = Mock() + mock_db.commit = Mock() + mock_db.refresh = Mock() + + with patch.object(Assembly, "__init__", return_value=None): + assembly_service.create_from_intent( + mock_db, + taxon_id=taxon_id, + sample_id=sample_id, + data_types="PACBIO_SMRT", + tol_id="tol-999", + project_id=None, + ) + + # Should add the Assembly and commit + mock_db.add.assert_called_once() + added_obj = mock_db.add.call_args[0][0] + # The added object should be an Assembly with status="requested" + assert isinstance(added_obj, Assembly) + mock_db.commit.assert_called_once() + + def test_create_from_intent_version_is_next(self, mock_db, assembly_service): + """create_from_intent assigns the next version based on existing assemblies.""" + sample_id = uuid.uuid4() + + mock_query = Mock() + # Assembly.version max = 2, AssemblyRun.version max = None → next version = 3 + mock_query.filter.return_value.scalar.side_effect = [2, None] + mock_db.query.return_value = mock_query + mock_db.add = Mock() + mock_db.commit = Mock() + mock_db.refresh = Mock() + + created_versions = [] + + original_init = Assembly.__init__ + + def capture_init(self, **kwargs): + created_versions.append(kwargs.get("version")) + + with patch.object(Assembly, "__init__", capture_init): + assembly_service.create_from_intent( + mock_db, + taxon_id=172942, + sample_id=sample_id, + data_types="PACBIO_SMRT", + tol_id=None, + project_id=None, + ) + + assert created_versions == [3] From 107fe6a118e20b1c0057ae0a66f7166e8c9d4dd5 Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Tue, 5 May 2026 02:25:45 +1000 Subject: [PATCH 02/11] feat: sync schema with database, update AssemblyStageRun model --- app/models/assembly.py | 5 +-- schema.sql | 85 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 81 insertions(+), 9 deletions(-) diff --git a/app/models/assembly.py b/app/models/assembly.py index 16faae9..a4cbde1 100644 --- a/app/models/assembly.py +++ b/app/models/assembly.py @@ -265,10 +265,7 @@ class AssemblyStageRun(Base): UUID(as_uuid=True), ForeignKey("assembly.id", ondelete="CASCADE"), nullable=False ) stage_name = Column(Text, ForeignKey("assembly_stage.name"), nullable=False) - status = Column( - SQLAlchemyEnum("running", "succeeded", "failed", "cancelled", name="stage_run_status"), - nullable=False, - ) + status = Column(Text, nullable=False) external_run_id = Column(Text, nullable=True) attempt = Column(Integer, nullable=False, default=1) stats = Column(JSONB, nullable=False, default=dict) diff --git a/schema.sql b/schema.sql index 2dc818e..1f31fbe 100644 --- a/schema.sql +++ b/schema.sql @@ -84,13 +84,30 @@ CREATE TABLE organism ( ncbi_order TEXT, ncbi_family TEXT, busco_dataset_name TEXT, - augustus_dataset_name TEXT, bpa_json JSONB, taxonomy_lineage_json JSONB, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); +-- ========================================== +-- Taxonomy info table (1:1 extension of organism) +-- ========================================== + +CREATE TABLE taxonomy_info ( + taxon_id INT PRIMARY KEY REFERENCES organism(taxon_id) ON DELETE CASCADE, + busco_odb10_dataset_name TEXT, + busco_odb12_dataset_name TEXT, + find_plastid BOOLEAN, + hic_motif TEXT, + mitochondrial_genetic_code_id INTEGER, + mitohifi_reference_species TEXT, + oatk_hmm_name TEXT, + defined_class TEXT, + augustus_dataset_name TEXT, + genetic_code_id INTEGER +); + -- ========================================== -- Accession registry table -- ========================================== @@ -484,12 +501,12 @@ CREATE TABLE assembly ( project_id UUID REFERENCES project(id), -- Assembly metadata - assembly_name TEXT NOT NULL, + assembly_name TEXT, assembly_type TEXT NOT NULL DEFAULT 'clone or isolate', tol_id TEXT, data_types assembly_data_types NOT NULL, - coverage FLOAT NOT NULL, - program TEXT NOT NULL, + coverage FLOAT, + program TEXT, mingaplength FLOAT, moleculetype molecule_type NOT NULL DEFAULT 'genomic DNA', description TEXT, @@ -497,8 +514,15 @@ CREATE TABLE assembly ( -- Auto-incremented version per (data_types, taxon_id, sample_id) version INTEGER NOT NULL DEFAULT 1, + -- Assembly lifecycle status + status TEXT NOT NULL DEFAULT 'requested', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT ck_assembly_status CHECK ( + status IN ('requested', 'running', 'curating', 'completed', 'failed', 'cancelled') + ) ); CREATE TABLE assembly_run ( @@ -572,6 +596,57 @@ CREATE TABLE assembly_read ( PRIMARY KEY (assembly_id, read_id) ); +-- ========================================== +-- Assembly stage catalog and reporting tables +-- ========================================== + +CREATE TABLE assembly_stage ( + name TEXT PRIMARY KEY, + category TEXT NOT NULL, + is_active BOOLEAN NOT NULL DEFAULT TRUE, + CONSTRAINT ck_assembly_stage_category CHECK (category IN ('pipeline', 'manual')) +); + +-- Seed rows +INSERT INTO assembly_stage (name, category) VALUES + ('genomeassembly', 'pipeline'), + ('ascc', 'pipeline'), + ('treeval', 'pipeline'), + ('curation-pretext', 'pipeline'), + ('manual-curation', 'manual'); + +CREATE TABLE assembly_stage_run ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + assembly_id UUID NOT NULL REFERENCES assembly(id) ON DELETE CASCADE, + stage_name TEXT NOT NULL REFERENCES assembly_stage(name), + status TEXT NOT NULL, + external_run_id TEXT, + attempt INTEGER NOT NULL DEFAULT 1, + stats JSONB NOT NULL DEFAULT '{}', + started_at TIMESTAMPTZ, + completed_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT ck_assembly_stage_run_status CHECK ( + status IN ('running', 'succeeded', 'failed', 'cancelled') + ), + CONSTRAINT uq_stage_run_assembly_stage_attempt UNIQUE (assembly_id, stage_name, attempt) +); + +CREATE INDEX ix_assembly_stage_run_assembly_id ON assembly_stage_run(assembly_id); + +CREATE TABLE assembly_stage_run_file ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + assembly_stage_run_id UUID NOT NULL REFERENCES assembly_stage_run(id) ON DELETE CASCADE, + storage_type TEXT NOT NULL, + storage_uri TEXT NOT NULL, + storage_details JSONB NOT NULL DEFAULT '{}', + sha256sum TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX ix_assembly_stage_run_file_run_id ON assembly_stage_run_file(assembly_stage_run_id); + -- ========================================== -- Genome note tables -- ========================================== From 514370c76b3a18d93d5b663611474206f8bb0b60 Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Fri, 8 May 2026 14:46:22 +1000 Subject: [PATCH 03/11] feat: update versioning strategy, use json for manifests, require specimen_ids for assemblies --- .../0012_assembly_manifest_columns.py | 59 +++ app/api/v1/endpoints/assemblies.py | 475 ++++++++--------- app/models/assembly.py | 12 +- app/schemas/assembly.py | 7 +- app/services/assembly_helper.py | 207 ++++---- app/services/assembly_service.py | 45 +- schema.sql | 16 +- tests/unit/services/test_assembly_helper.py | 496 ++++++++++++------ tests/unit/services/test_assembly_service.py | 279 +++++----- 9 files changed, 938 insertions(+), 658 deletions(-) create mode 100644 alembic/versions/0012_assembly_manifest_columns.py diff --git a/alembic/versions/0012_assembly_manifest_columns.py b/alembic/versions/0012_assembly_manifest_columns.py new file mode 100644 index 0000000..b6771d0 --- /dev/null +++ b/alembic/versions/0012_assembly_manifest_columns.py @@ -0,0 +1,59 @@ +"""Add specimen sample IDs and manifest JSON to assembly. + +Revision ID: 0012_assembly_manifest_columns +Revises: 0011_assembly_first_and_stages +Create Date: 2026-05-08 + +Adds three nullable columns to the assembly table: +- long_read_specimen_sample_id: the specimen sample used for long reads (PacBio / ONT) +- hic_specimen_sample_id: the specimen sample used for Hi-C reads (optional) +- manifest_json: persisted JSON manifest generated at intent time + +All columns are nullable so existing rows are unaffected. +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects.postgresql import JSONB, UUID + +revision = "0012_assembly_manifest_columns" +down_revision = "0011_assembly_first_and_stages" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "assembly", + sa.Column( + "long_read_specimen_sample_id", + UUID(as_uuid=True), + sa.ForeignKey("sample.id"), + nullable=True, + ), + ) + op.add_column( + "assembly", + sa.Column( + "hic_specimen_sample_id", + UUID(as_uuid=True), + sa.ForeignKey("sample.id"), + nullable=True, + ), + ) + op.add_column( + "assembly", + sa.Column("manifest_json", JSONB(), nullable=True), + ) + op.create_index( + "idx_assembly_intent_version_key", + "assembly", + ["taxon_id", "long_read_specimen_sample_id"], + ) + + +def downgrade() -> None: + op.drop_index("idx_assembly_intent_version_key", table_name="assembly") + op.drop_column("assembly", "manifest_json") + op.drop_column("assembly", "hic_specimen_sample_id") + op.drop_column("assembly", "long_read_specimen_sample_id") diff --git a/app/api/v1/endpoints/assemblies.py b/app/api/v1/endpoints/assemblies.py index 2e194fe..37357d2 100644 --- a/app/api/v1/endpoints/assemblies.py +++ b/app/api/v1/endpoints/assemblies.py @@ -1,9 +1,9 @@ -import yaml +import json from typing import Any, Dict, List, Optional from uuid import UUID from fastapi import APIRouter, Body, Depends, HTTPException, Query -from fastapi.responses import Response +from fastapi.responses import JSONResponse, Response from sqlalchemy.orm import Session from app.core.dependencies import get_current_active_user, get_db @@ -41,7 +41,10 @@ AssemblySubmission as AssemblySubmissionSchema, ) from app.schemas.common import SubmissionStatus -from app.services.assembly_helper import determine_assembly_data_types, generate_assembly_manifest +from app.services.assembly_helper import ( + determine_assembly_data_types, + generate_assembly_manifest_json, +) from app.services.assembly_service import ( assembly_file_service, assembly_service, @@ -57,30 +60,56 @@ def _organism_taxon_id(organism: Any) -> int: return organism.taxon_id if hasattr(organism, "taxon_id") else organism.tax_id -""" -@router.get("/", response_model=List[AssemblySchema]) -def read_assemblies( - db: Session = Depends(get_db), - skip: int = 0, - limit: int = 100, - organism_id: Optional[UUID] = Query(None, description="Filter by organism ID"), - sample_id: Optional[UUID] = Query(None, description="Filter by sample ID"), - experiment_id: Optional[UUID] = Query(None, description="Filter by experiment ID"), - current_user: User = Depends(get_current_active_user), -) -> Any: - Retrieve assemblies. - # All users can read assemblies - query = db.query(Assembly) - if organism_id: - query = query.filter(Assembly.organism_id == organism_id) - if sample_id: - query = query.filter(Assembly.sample_id == sample_id) - if experiment_id: - query = query.filter(Assembly.experiment_id == experiment_id) +def _build_sample_metadata_by_id( + db: Session, experiments: List[Experiment] +) -> Dict[str, Dict[str, Any]]: + """Build sample metadata mapping used in per-read manifest entries.""" + sample_ids = {exp.sample_id for exp in experiments if getattr(exp, "sample_id", None)} + if not sample_ids: + return {} + + sample_rows = db.query(Sample).filter(Sample.id.in_(list(sample_ids))).all() + return { + str(sample.id): { + "bpa_sample_id": getattr(sample, "bpa_sample_id", None), + "specimen_id": getattr(sample, "specimen_id", None), + } + for sample in sample_rows + } + - assemblies = query.offset(skip).limit(limit).all() - return assemblies -""" +def _validate_specimen_sample( + db: Session, sample_id: UUID, taxon_id: int, field_name: str +) -> Sample: + """Validate a specimen sample: must exist, kind='specimen', and belong to taxon_id.""" + sample = db.query(Sample).filter(Sample.id == sample_id).first() + if not sample: + raise AppError( + status_code=422, + code="specimen_sample_not_found", + message=f"{field_name} not found", + details={"field": field_name, "sample_id": str(sample_id)}, + ) + if sample.kind != "specimen": + raise AppError( + status_code=422, + code="specimen_sample_invalid_kind", + message=f"{field_name} must be a specimen sample (kind='specimen'), got kind='{sample.kind}'", + details={"field": field_name, "sample_id": str(sample_id), "kind": sample.kind}, + ) + if sample.taxon_id != taxon_id: + raise AppError( + status_code=422, + code="specimen_sample_taxon_mismatch", + message=f"{field_name} does not belong to taxon_id {taxon_id}", + details={ + "field": field_name, + "sample_id": str(sample_id), + "sample_taxon_id": sample.taxon_id, + "expected_taxon_id": taxon_id, + }, + ) + return sample @router.get("/pipeline-inputs") @@ -96,23 +125,19 @@ def get_pipeline_inputs( Returns a list of objects with scientific_name and files mapping for each organism. Files mapping contains read file names as keys and their bioplatforms_urls as values. """ - print(f"Organism taxon_id: {taxon_id}") if taxon_id is None: raise HTTPException(status_code=422, detail="taxon_id query parameter is required") if db is None: raise HTTPException(status_code=422, detail="database session is required") organism = organism_service.get_by_taxon_id(db, taxon_id) if not organism: - print(f"Organism with taxon_id '{taxon_id}' not found") raise HTTPException( status_code=404, detail=f"Organism with taxon_id '{taxon_id}' not found" ) - # Get all samples for this organism organism_taxon_id = _organism_taxon_id(organism) samples = db.query(Sample).filter(Sample.taxon_id == organism_taxon_id).all() if not samples: - print(f"No samples found for organism with taxon_id '{taxon_id}'") return [ { "scientific_name": organism.scientific_name, @@ -121,30 +146,19 @@ def get_pipeline_inputs( } ] - # Get all experiments and reads for these samples result = [] files_dict = {} - # Collect all reads for this organism through the sample->experiment->read relationship for sample in samples: - print(f"Sample {sample.id} found for organism with taxon_id '{taxon_id}'") - # Get experiments for this sample experiments = db.query(Experiment).filter(Experiment.sample_id == sample.id).all() - for experiment in experiments: - print(f"Experiment {experiment.id} found for sample {sample.id}") - # Get reads for this experiment reads = db.query(Read).filter(Read.experiment_id == experiment.id).all() - if reads is None: continue - for read in reads: - print(f"Read {read.id} found for experiment {experiment.id}") if read.file_name and read.bioplatforms_url: files_dict[read.file_name] = read.bioplatforms_url - # Create the result object result.append( { "scientific_name": organism.scientific_name, @@ -152,7 +166,6 @@ def get_pipeline_inputs( "files": files_dict, } ) - return result @@ -168,134 +181,31 @@ def get_pipeline_inputs_by_taxon_id( Returns a nested structure with taxon_id as the top level key, and scientific_name and files mapping for that organism. - Files mapping contains read file names as keys and their bioplatforms_urls as values. """ - print(f"Tax ID: {taxon_id}") - - # Check if taxon_id was provided if taxon_id is None: raise HTTPException(status_code=422, detail="taxon_id query parameter is required") - # Get all organisms with this taxon_id organism = organism_service.get_by_taxon_id(db, int(taxon_id)) if not organism: - print(f"No organisms found with tax ID '{taxon_id}'") return {taxon_id: {}} - # Initialize result structure result = {taxon_id: {"scientific_name": organism.scientific_name, "files": {}}} organism_taxon_id = _organism_taxon_id(organism) samples = db.query(Sample).filter(Sample.taxon_id == organism_taxon_id).all() if not samples: - print(f"No samples found for organism with taxon_id {taxon_id}") return result for sample in samples: - print(f"Sample {sample.id} found for organism {taxon_id}") experiments = db.query(Experiment).filter(Experiment.sample_id == sample.id).all() - for experiment in experiments: - print(f"Experiment {experiment.id} found for sample {sample.id}") reads = db.query(Read).filter(Read.experiment_id == experiment.id).all() - for read in reads: - print(f"Read {read.id} found for experiment {experiment.id}") if read.file_name and read.bioplatforms_url: result[taxon_id]["files"][read.file_name] = read.bioplatforms_url return result -def _get_manifest_inputs_by_taxon_id(db: Session, taxon_id: int): - organism = db.query(Organism).filter(Organism.taxon_id == taxon_id).first() - if not organism: - raise HTTPException(status_code=404, detail=f"Organism with taxon_id {taxon_id} not found") - - selected_sample_id = _get_optimal_sample_id_for_taxon_id( - db, taxon_id, organism_taxon_id=_organism_taxon_id(organism) - ) - if not selected_sample_id: - raise HTTPException( - status_code=404, - detail=f"No specimen sample found for taxon_id {taxon_id}", - ) - sample = db.query(Sample).filter(Sample.id == selected_sample_id).first() - if not sample: - raise HTTPException( - status_code=404, - detail=f"Selected specimen sample not found for taxon_id {taxon_id}", - ) - - sample_ids = [ - row[0] - for row in db.query(Sample.id).filter(Sample.taxon_id == _organism_taxon_id(organism)).all() - ] - if not sample_ids: - raise HTTPException( - status_code=404, - detail=f"No samples found for taxon_id {taxon_id}", - ) - - experiments = db.query(Experiment).filter(Experiment.sample_id.in_(sample_ids)).all() - if not experiments: - raise HTTPException( - status_code=404, - detail=f"No experiments found for taxon_id {taxon_id}", - ) - - experiment_ids = [exp.id for exp in experiments] - reads = db.query(Read).filter(Read.experiment_id.in_(experiment_ids)).all() - if not reads: - raise HTTPException( - status_code=404, - detail=f"No reads found for taxon_id {taxon_id}", - ) - - return organism, sample, reads, experiments - - -def _get_optimal_sample_id_for_taxon_id( - db: Session, taxon_id: int, organism_taxon_id: Optional[int] = None -) -> UUID | None: - # TODO: Replace with long-read aware sample selection. - if organism_taxon_id is None: - organism = db.query(Organism).filter(Organism.taxon_id == taxon_id).first() - if not organism: - return None - organism_taxon_id = organism.taxon_id - - sample = ( - db.query(Sample) - .filter(Sample.taxon_id == organism_taxon_id, Sample.kind == "specimen") - .order_by(Sample.created_at.asc()) - .first() - ) - return sample.id if sample else None - - -# Backward-compatible aliases retained for tests while the API contract migrates. -_get_manifest_inputs_by_tax_id = _get_manifest_inputs_by_taxon_id -_get_optimal_sample_id_for_tax_id = _get_optimal_sample_id_for_taxon_id - - -def _build_sample_metadata_by_id( - db: Session, experiments: List[Experiment] -) -> Dict[str, Dict[str, Any]]: - """Build sample metadata mapping used in per-read manifest entries.""" - sample_ids = {exp.sample_id for exp in experiments if getattr(exp, "sample_id", None)} - if not sample_ids: - return {} - - sample_rows = db.query(Sample).filter(Sample.id.in_(list(sample_ids))).all() - return { - str(sample.id): { - "bpa_sample_id": getattr(sample, "bpa_sample_id", None), - "specimen_id": getattr(sample, "specimen_id", None), - } - for sample in sample_rows - } - - @router.get("/manifest/{taxon_id}") def get_assembly_manifest( *, @@ -305,24 +215,18 @@ def get_assembly_manifest( current_user: User = Depends(get_current_active_user), ) -> Any: """ - Retrieve the latest requested assembly manifest YAML for an organism by taxon_id. + Retrieve the stored manifest JSON for the latest requested assembly for a taxon. - Returns YAML manifest with: - - scientific_name and taxon_id from organism - - reads grouped by platform (PACBIO_SMRT, Hi-C) - - PACBIO_SMRT: Only files ending in .ccs.bam or hifi_reads.bam - - Hi-C: Includes read_number and lane_number - - Returns: - YAML string with manifest data + Returns the manifest that was generated and stored when the intent was created. """ - organism, selected_sample, reads, experiments = _get_manifest_inputs_by_taxon_id(db, taxon_id) + organism = db.query(Organism).filter(Organism.taxon_id == taxon_id).first() + if not organism: + raise HTTPException(status_code=404, detail=f"Organism with taxon_id {taxon_id} not found") assembly_query = ( db.query(Assembly) .filter( Assembly.taxon_id == _organism_taxon_id(organism), - Assembly.sample_id == selected_sample.id, Assembly.status == "requested", ) .order_by(Assembly.created_at.desc()) @@ -333,17 +237,13 @@ def get_assembly_manifest( if not assembly: raise HTTPException(status_code=404, detail="No requested assembly manifest found") - sample_metadata_by_id = _build_sample_metadata_by_id(db, experiments) - yaml_content = generate_assembly_manifest( - organism, - reads, - experiments, - assembly.tol_id, - assembly.version, - sample_metadata_by_id, - ) + if assembly.manifest_json is None: + raise HTTPException( + status_code=404, + detail="No manifest stored for this assembly. Re-submit an intent to generate one.", + ) - return Response(content=yaml_content, media_type="application/x-yaml") + return JSONResponse(content=assembly.manifest_json) @router.post("/intent/{taxon_id}") @@ -352,52 +252,160 @@ def create_assembly_intent( *, db: Session = Depends(get_db), taxon_id: int, - intent_in: Optional[AssemblyIntent] = Body(default=None), + intent_in: AssemblyIntent = Body(...), current_user: User = Depends(get_current_active_user), ) -> Response: """ - Create an assembly record and return its manifest as YAML. + Create an assembly record using explicit specimen sample IDs and return its manifest as JSON. + + The caller must supply: + - long_read_specimen_sample_id: the specimen sample used for PacBio / ONT long reads + - hic_specimen_sample_id (optional): the specimen sample used for Hi-C reads + + Both samples must be kind='specimen' and belong to the given taxon_id. - Returns: - YAML response with assembly_id, version, status, and manifest fields + Returns JSON with assembly_id, version, status, and the generated manifest. """ - organism, selected_sample, reads, experiments = _get_manifest_inputs_by_taxon_id(db, taxon_id) + # 1. Resolve organism + organism = db.query(Organism).filter(Organism.taxon_id == taxon_id).first() + if not organism: + raise HTTPException(status_code=404, detail=f"Organism with taxon_id {taxon_id} not found") + org_taxon_id = _organism_taxon_id(organism) + + # 2. Validate long_read_specimen_sample_id + long_read_sample = _validate_specimen_sample( + db, intent_in.long_read_specimen_sample_id, org_taxon_id, "long_read_specimen_sample_id" + ) + + # 3. Validate hic_specimen_sample_id if provided + hic_sample = None + if intent_in.hic_specimen_sample_id: + hic_sample = _validate_specimen_sample( + db, intent_in.hic_specimen_sample_id, org_taxon_id, "hic_specimen_sample_id" + ) + + # 4. Fetch long-read experiments (PacBio or ONT only) and their reads + long_read_experiments = ( + db.query(Experiment) + .filter( + Experiment.sample_id == long_read_sample.id, + Experiment.platform.in_(["PACBIO_SMRT", "OXFORD_NANOPORE"]), + ) + .all() + ) + if not long_read_experiments: + raise AppError( + status_code=422, + code="no_long_read_experiments", + message="No long-read experiments (PacBio or ONT) found for long_read_specimen_sample_id", + details={"long_read_specimen_sample_id": str(long_read_sample.id)}, + ) + + long_read_exp_ids = [e.id for e in long_read_experiments] + long_reads = db.query(Read).filter(Read.experiment_id.in_(long_read_exp_ids)).all() + + # 5. Fetch Hi-C experiments and reads if hic_sample is provided + hic_experiments: List[Experiment] = [] + hic_reads: List[Read] = [] + if hic_sample: + hic_experiments = ( + db.query(Experiment) + .filter( + Experiment.sample_id == hic_sample.id, + Experiment.platform == "ILLUMINA", + ) + .all() + ) + hic_experiments = [ + e for e in hic_experiments + if (e.library_strategy or "").upper() in ("HI-C", "WGS") + ] + if not hic_experiments: + raise AppError( + status_code=422, + code="no_hic_experiments", + message="No Hi-C experiments (ILLUMINA + Hi-C/WGS) found for hic_specimen_sample_id", + details={"hic_specimen_sample_id": str(hic_sample.id)}, + ) + hic_exp_ids = [e.id for e in hic_experiments] + hic_reads = db.query(Read).filter(Read.experiment_id.in_(hic_exp_ids)).all() + + # 6. Determine data_types from the relevant experiments + all_experiments = long_read_experiments + hic_experiments try: - data_types = determine_assembly_data_types(experiments) + data_types = determine_assembly_data_types(all_experiments) except ValueError as exc: raise AppError( status_code=400, code="assembly_intent_invalid_data_types", message=str(exc), - details={ - "taxon_id": taxon_id, - "sample_id": str(selected_sample.id), - }, + details={"taxon_id": taxon_id}, ) from exc + # 7. Create assembly (version is assigned inside the service) assembly = assembly_service.create_from_intent( db, - taxon_id=_organism_taxon_id(organism), - sample_id=selected_sample.id, + taxon_id=org_taxon_id, + long_read_specimen_sample_id=long_read_sample.id, + hic_specimen_sample_id=hic_sample.id if hic_sample else None, data_types=data_types, - tol_id=intent_in.tol_id if intent_in else None, + tol_id=intent_in.tol_id, project_id=None, + manifest_json=None, # filled in step 9 ) - sample_metadata_by_id = _build_sample_metadata_by_id(db, experiments) - manifest_yaml = generate_assembly_manifest( - organism, reads, experiments, assembly.tol_id, assembly.version, sample_metadata_by_id + # 8. Build sample metadata and generate JSON manifest + all_reads = long_reads + hic_reads + sample_metadata_by_id = _build_sample_metadata_by_id(db, all_experiments) + manifest_data = generate_assembly_manifest_json( + organism=organism, + reads=all_reads, + experiments=all_experiments, + tol_id=assembly.tol_id, + version=assembly.version, + long_read_sample_id=long_read_sample.id, + hic_sample_id=hic_sample.id if hic_sample else None, + sample_metadata_by_id=sample_metadata_by_id, ) - response_data = { - "assembly_id": str(assembly.id), - "version": assembly.version, - "status": assembly.status, - "manifest": yaml.safe_load(manifest_yaml), - } - yaml_response = yaml.dump(response_data, default_flow_style=False, sort_keys=False) + # 9. Validate that the manifest contains eligible reads then persist it + long_read_keys = {"PACBIO_SMRT", "OXFORD_NANOPORE"} + if not any(k in manifest_data["reads"] for k in long_read_keys): + db.delete(assembly) + db.commit() + raise AppError( + status_code=422, + code="no_eligible_long_reads", + message=( + "No eligible long reads found for long_read_specimen_sample_id. " + "PacBio reads must end in .ccs.bam or hifi_reads.bam." + ), + details={"long_read_specimen_sample_id": str(long_read_sample.id)}, + ) + + if hic_sample and "Hi-C" not in manifest_data["reads"]: + db.delete(assembly) + db.commit() + raise AppError( + status_code=422, + code="no_eligible_hic_reads", + message="No eligible Hi-C reads found for hic_specimen_sample_id", + details={"hic_specimen_sample_id": str(hic_sample.id)}, + ) - return Response(content=yaml_response, media_type="application/x-yaml") + assembly.manifest_json = manifest_data + db.add(assembly) + db.commit() + db.refresh(assembly) + + return JSONResponse( + content={ + "assembly_id": str(assembly.id), + "version": assembly.version, + "status": assembly.status, + "manifest": manifest_data, + } + ) @router.post("/intent/{taxon_id}/cancel") @@ -414,21 +422,11 @@ def cancel_assembly_intent( if not organism: raise HTTPException(status_code=404, detail=f"Organism with taxon_id {taxon_id} not found") - selected_sample_id = _get_optimal_sample_id_for_taxon_id( - db, taxon_id, organism_taxon_id=_organism_taxon_id(organism) - ) - if not selected_sample_id: - raise HTTPException( - status_code=404, - detail=f"No specimen sample found for taxon_id {taxon_id}", - ) - assembly = ( db.query(Assembly) .filter( Assembly.id == cancel_in.assembly_id, Assembly.taxon_id == _organism_taxon_id(organism), - Assembly.sample_id == selected_sample_id, Assembly.status == "requested", ) .first() @@ -454,7 +452,12 @@ def cancel_assembly_intent( return { "id": str(assembly.id), "taxon_id": assembly.taxon_id, - "sample_id": str(assembly.sample_id), + "long_read_specimen_sample_id": str(assembly.long_read_specimen_sample_id) + if assembly.long_read_specimen_sample_id + else None, + "hic_specimen_sample_id": str(assembly.hic_specimen_sample_id) + if assembly.hic_specimen_sample_id + else None, "version": assembly.version, "status": assembly.status, } @@ -467,8 +470,14 @@ def get_optimal_sample_id( taxon_id: int, current_user: User = Depends(get_current_active_user), ) -> Any: - sample_id = _get_optimal_sample_id_for_taxon_id(db, taxon_id) - return {"sample_id": str(sample_id) if sample_id else None} + """Return the first specimen sample for an organism (informational helper).""" + sample = ( + db.query(Sample) + .filter(Sample.taxon_id == taxon_id, Sample.kind == "specimen") + .order_by(Sample.created_at.asc()) + .first() + ) + return {"sample_id": str(sample.id) if sample else None} @router.post("/from-experiments/{taxon_id}", response_model=AssemblySchema) @@ -483,17 +492,12 @@ def create_assembly_from_experiments( """ Create assembly based on all experiments for an organism (by taxon_id). - Automatically determines data_types by analyzing experiment platforms: - - PACBIO_SMRT: platform == "PACBIO_SMRT" - - OXFORD_NANOPORE: platform == "OXFORD_NANOPORE" - - Hi-C: platform == "ILLUMINA" AND library_strategy == "Hi-C" - - The taxon_id is automatically determined from taxon_id. + Automatically determines data_types by analyzing experiment platforms. The data_types field is auto-detected but can be overridden. """ try: assembly, platform_info = assembly_service.create_from_experiments( - db, taxon_id, assembly_in + db, taxon_id=taxon_id, assembly_in=assembly_in ) return assembly except ValueError as e: @@ -519,33 +523,18 @@ def get_manifest_by_assembly_id( assembly_id: UUID, current_user: User = Depends(get_current_active_user), ) -> Any: - """Retrieve the assembly manifest YAML for a specific assembly by ID.""" + """Retrieve the stored manifest JSON for a specific assembly.""" assembly = db.query(Assembly).filter(Assembly.id == assembly_id).first() if not assembly: raise HTTPException(status_code=404, detail="Assembly not found") - organism = db.query(Organism).filter(Organism.taxon_id == assembly.taxon_id).first() - if not organism: - raise HTTPException(status_code=404, detail="Organism not found for this assembly") - - sample_ids = [ - row[0] - for row in db.query(Sample.id).filter(Sample.taxon_id == assembly.taxon_id).all() - ] - experiments = db.query(Experiment).filter(Experiment.sample_id.in_(sample_ids)).all() - if not experiments: - raise HTTPException(status_code=404, detail="No experiments found for this assembly") - - experiment_ids = [exp.id for exp in experiments] - reads = db.query(Read).filter(Read.experiment_id.in_(experiment_ids)).all() - if not reads: - raise HTTPException(status_code=404, detail="No reads found for this assembly") - - sample_metadata_by_id = _build_sample_metadata_by_id(db, experiments) - yaml_content = generate_assembly_manifest( - organism, reads, experiments, assembly.tol_id, assembly.version, sample_metadata_by_id - ) - return Response(content=yaml_content, media_type="application/x-yaml") + if assembly.manifest_json is None: + raise HTTPException( + status_code=404, + detail="No manifest stored for this assembly. Re-submit an intent to generate one.", + ) + + return JSONResponse(content=assembly.manifest_json) @router.get("/{assembly_id}", response_model=AssemblySchema) @@ -555,8 +544,6 @@ def read_assembly( assembly_id: UUID, current_user: User = Depends(get_current_active_user), ) -> Any: - # Get assembly by ID. - # All users can read assembly details assembly = db.query(Assembly).filter(Assembly.id == assembly_id).first() if not assembly: raise HTTPException(status_code=404, detail="Assembly not found") @@ -594,12 +581,9 @@ def delete_assembly( assembly_id: UUID, current_user: User = Depends(get_current_active_user), ) -> Any: - # Delete an assembly. - # Only superusers can delete assemblies assembly = db.query(Assembly).filter(Assembly.id == assembly_id).first() if not assembly: raise HTTPException(status_code=404, detail="Assembly not found") - db.delete(assembly) db.commit() return assembly @@ -622,7 +606,6 @@ def read_assembly_submissions( if status: query = query.filter(AssemblySubmission.status == status.value) submissions = apply_pagination(query, pagination).all() - return submissions @@ -635,12 +618,9 @@ def create_assembly_submission( current_user: User = Depends(get_current_active_user), ) -> Any: """Create new assembly submission.""" - - # Verify assembly exists assembly = assembly_service.get(db, id=submission_in.assembly_id) if not assembly: raise HTTPException(status_code=404, detail="Assembly not found") - submission = assembly_submission_service.create(db, obj_in=submission_in) return submission @@ -655,11 +635,9 @@ def update_assembly_submission( current_user: User = Depends(get_current_active_user), ) -> Any: """Update an assembly submission.""" - submission = assembly_submission_service.get(db, id=submission_id) if not submission: raise HTTPException(status_code=404, detail="Assembly submission not found") - submission = assembly_submission_service.update(db, db_obj=submission, obj_in=submission_in) return submission @@ -688,7 +666,6 @@ def read_assembly_files( ) else: files = assembly_file_service.get_by_assembly_id(db, assembly_id=assembly_id) - return files @@ -702,16 +679,11 @@ def create_assembly_file( current_user: User = Depends(get_current_active_user), ) -> Any: """Add a file to an assembly.""" - - # Verify assembly exists assembly = assembly_service.get(db, id=assembly_id) if not assembly: raise HTTPException(status_code=404, detail="Assembly not found") - - # Ensure assembly_id matches if file_in.assembly_id != assembly_id: raise HTTPException(status_code=400, detail="Assembly ID mismatch") - file = assembly_file_service.create(db, obj_in=file_in) return file @@ -726,11 +698,9 @@ def update_assembly_file( current_user: User = Depends(get_current_active_user), ) -> Any: """Update an assembly file.""" - file = assembly_file_service.get(db, id=file_id) if not file: raise HTTPException(status_code=404, detail="Assembly file not found") - file = assembly_file_service.update(db, db_obj=file, obj_in=file_in) return file @@ -747,7 +717,6 @@ def delete_assembly_file( file = assembly_file_service.get(db, id=file_id) if not file: raise HTTPException(status_code=404, detail="Assembly file not found") - assembly_file_service.remove(db, id=file_id) return {"message": "File deleted successfully"} diff --git a/app/models/assembly.py b/app/models/assembly.py index a4cbde1..98349c8 100644 --- a/app/models/assembly.py +++ b/app/models/assembly.py @@ -70,9 +70,19 @@ class Assembly(Base): onupdate=func.now(), ) + long_read_specimen_sample_id = Column( + UUID(as_uuid=True), ForeignKey("sample.id"), nullable=True + ) + hic_specimen_sample_id = Column(UUID(as_uuid=True), ForeignKey("sample.id"), nullable=True) + manifest_json = Column(JSONB, nullable=True) + # Relationships organism = relationship("Organism", backref="assemblies") - sample = relationship("Sample", backref="assemblies") + sample = relationship("Sample", foreign_keys=[sample_id], backref="assemblies") + long_read_specimen_sample = relationship( + "Sample", foreign_keys=[long_read_specimen_sample_id] + ) + hic_specimen_sample = relationship("Sample", foreign_keys=[hic_specimen_sample_id]) project = relationship("Project", backref="assemblies") diff --git a/app/schemas/assembly.py b/app/schemas/assembly.py index caf0dc1..85962b1 100644 --- a/app/schemas/assembly.py +++ b/app/schemas/assembly.py @@ -76,6 +76,8 @@ class AssemblyIntent(BaseModel): """Schema for reserving an assembly version and generating a manifest.""" tol_id: Optional[str] = None + long_read_specimen_sample_id: UUID + hic_specimen_sample_id: Optional[UUID] = None class AssemblyIntentResponse(BaseModel): @@ -84,7 +86,7 @@ class AssemblyIntentResponse(BaseModel): assembly_id: UUID version: int status: str - manifest_yaml: str + manifest_json: Dict[str, Any] class AssemblyIntentCancel(BaseModel): @@ -118,6 +120,9 @@ class AssemblyInDBBase(AssemblyBase): """Base schema for Assembly in DB, includes id and timestamps.""" id: UUID + long_read_specimen_sample_id: Optional[UUID] = None + hic_specimen_sample_id: Optional[UUID] = None + manifest_json: Optional[Dict[str, Any]] = None created_at: datetime updated_at: datetime diff --git a/app/services/assembly_helper.py b/app/services/assembly_helper.py index 2d444d0..d77b799 100644 --- a/app/services/assembly_helper.py +++ b/app/services/assembly_helper.py @@ -1,9 +1,8 @@ """Helper functions for assembly operations.""" import logging -from typing import Any, Dict, List, Set - -import yaml +from typing import Any, Dict, List, Optional, Set +from uuid import UUID from app.models.experiment import Experiment from app.models.organism import Organism @@ -21,12 +20,6 @@ def determine_assembly_data_types(experiments: List[Experiment]) -> AssemblyData - OXFORD_NANOPORE exists if: platform == "OXFORD_NANOPORE" - Hi-C exists if: platform == "ILLUMINA" AND library_strategy in ("Hi-C", "WGS") - Args: - experiments: List of Experiment objects - - Returns: - AssemblyDataTypes enum value based on detected platforms - Raises: ValueError: If no valid sequencing platforms are detected """ @@ -38,19 +31,13 @@ def determine_assembly_data_types(experiments: List[Experiment]) -> AssemblyData platform = exp.platform.upper() if exp.platform else "" library_strategy = exp.library_strategy.upper() if exp.library_strategy else "" - # Check for PacBio if platform == "PACBIO_SMRT" and library_strategy in ("WGS", "WGA"): has_pacbio = True - - # Check for Oxford Nanopore if platform == "OXFORD_NANOPORE" and library_strategy in ("WGS", "WGA"): has_nanopore = True - - # Check for Hi-C (Illumina + Hi-C or WGS library strategy) if platform == "ILLUMINA" and library_strategy in ("HI-C", "WGS"): has_hic = True - # Determine the appropriate enum value based on combinations if has_pacbio and has_nanopore and has_hic: return AssemblyDataTypes.PACBIO_SMRT_OXFORD_NANOPORE_HIC elif has_pacbio and has_nanopore: @@ -64,7 +51,6 @@ def determine_assembly_data_types(experiments: List[Experiment]) -> AssemblyData elif has_nanopore: return AssemblyDataTypes.OXFORD_NANOPORE else: - # TODO decide if we relax this requirement and still return the manifest noting the available data types are not supported raise ValueError( "No valid data types detected in experiments. " "Expected PACBIO_SMRT (with or without Hi-C), OXFORD_NANOPORE (with or without Hi-C), or ILLUMINA with Hi-C." @@ -72,14 +58,7 @@ def determine_assembly_data_types(experiments: List[Experiment]) -> AssemblyData def get_detected_platforms(experiments: List[Experiment]) -> dict: - """Get a summary of detected platforms for debugging/logging. - - Args: - experiments: List of Experiment objects - - Returns: - Dictionary with platform detection details - """ + """Get a summary of detected platforms for debugging/logging.""" platforms: Set[str] = set() library_strategies: Set[str] = set() @@ -108,44 +87,52 @@ def _normalize_read_number(read_number: str | None) -> str | None: return None -def generate_assembly_manifest( +def generate_assembly_manifest_json( organism: Organism, reads: List[Read], experiments: List[Experiment], tol_id: str | None, version: int, + long_read_sample_id: UUID, + hic_sample_id: Optional[UUID] = None, sample_metadata_by_id: Dict[str, Dict[str, Any]] | None = None, -) -> str: - """Generate assembly manifest YAML from organism and reads data. +) -> Dict[str, Any]: + """Generate an assembly manifest as a JSON-serialisable dict. - Groups reads by bpa_package_id (from Experiment), then by platform type. + Reads are routed to sections based on which specimen sample they belong to: + - Reads from long_read_sample_id experiments → PACBIO_SMRT or OXFORD_NANOPORE section + - Reads from hic_sample_id experiments → Hi-C section (omitted when hic_sample_id is None) - Rules: - - PACBIO_SMRT: Only include files ending in .ccs.bam or hifi_reads.bam - - Hi-C: Split reads into r1/r2 groups by read_number + PacBio filtering: only files ending in .ccs.bam or hifi_reads.bam are included. + ONT: all reads are included. + Hi-C: reads are split into r1/r2 by read_number and include lane_number. Args: organism: Organism object - reads: List of Read objects - experiments: List of Experiment objects (to determine platform) + reads: Combined list of Read objects from both specimen samples + experiments: Combined list of Experiment objects from both specimen samples tol_id: ToL ID for the assembly (optional) version: Assembly version number + long_read_sample_id: sample.id of the long-read specimen sample + hic_sample_id: sample.id of the Hi-C specimen sample (optional) sample_metadata_by_id: Sample metadata keyed by sample.id as string (optional) Returns: - YAML string formatted as assembly manifest + Dict representing the manifest (JSON-serialisable) """ logger.info( - f"Generating manifest for organism: {organism.scientific_name} (taxon_id: {organism.taxon_id})" + "Generating manifest for organism: %s (taxon_id: %s)", + organism.scientific_name, + organism.taxon_id, ) - logger.info(f"Total experiments: {len(experiments)}, Total reads: {len(reads)}") + logger.info("Total experiments: %d, Total reads: %d", len(experiments), len(reads)) + + long_read_sample_str = str(long_read_sample_id) + hic_sample_str = str(hic_sample_id) if hic_sample_id else None # Build experiment info map: experiment.id → metadata - exp_info_map = {} + exp_info_map: Dict[Any, Dict[str, Any]] = {} for exp in experiments: - logger.info( - f"Experiment {exp.id}: platform={exp.platform}, library_strategy={exp.library_strategy}" - ) exp_info_map[exp.id] = { "platform": exp.platform.upper() if exp.platform else "", "library_strategy": exp.library_strategy.upper() if exp.library_strategy else "", @@ -154,18 +141,18 @@ def generate_assembly_manifest( "sample_id": str(exp.sample_id) if getattr(exp, "sample_id", None) else None, } - # Group reads by bpa_package_id per platform pacbio_by_package: Dict[str, Any] = {} + ont_by_package: Dict[str, Any] = {} hic_by_package: Dict[str, Any] = {} for read in reads: if not read.experiment_id: - logger.debug(f"Read {read.id} has no experiment_id, skipping") + logger.debug("Read %s has no experiment_id, skipping", read.id) continue exp_info = exp_info_map.get(read.experiment_id) if not exp_info: - logger.debug(f"Read {read.id} has no matching experiment, skipping") + logger.debug("Read %s has no matching experiment, skipping", read.id) continue platform = exp_info["platform"] @@ -174,16 +161,34 @@ def generate_assembly_manifest( sample_id = exp_info["sample_id"] sample_meta = (sample_metadata_by_id or {}).get(sample_id, {}) if sample_id else {} - logger.debug( - f"Read {read.id} (file: {read.file_name}): platform={platform}, library_strategy={library_strategy}" - ) - - # PacBio SMRT reads (only .ccs.bam or hifi_reads.bam) - if platform == "PACBIO_SMRT" and read.file_name: - if read.file_name.endswith(".ccs.bam") or read.file_name.endswith("hifi_reads.bam"): - logger.info(f"Adding PacBio read: {read.file_name}") - if bpa_package_id not in pacbio_by_package: - entry: Dict[str, Any] = { + # Route by specimen sample, then by platform + if sample_id == long_read_sample_str: + if platform == "PACBIO_SMRT" and read.file_name: + if read.file_name.endswith(".ccs.bam") or read.file_name.endswith("hifi_reads.bam"): + logger.info("Adding PacBio read: %s", read.file_name) + if bpa_package_id not in pacbio_by_package: + entry: Dict[str, Any] = { + "sample_id": sample_id, + "bpa_sample_id": sample_meta.get("bpa_sample_id"), + "specimen_id": sample_meta.get("specimen_id"), + "resources": [], + } + if exp_info["bioplatforms_base_url"]: + entry["bioplatforms_base_url"] = exp_info["bioplatforms_base_url"] + pacbio_by_package[bpa_package_id] = entry + pacbio_by_package[bpa_package_id]["resources"].append( + {"md5sum": read.file_checksum, "url": read.bioplatforms_url} + ) + else: + logger.debug( + "Skipping PacBio read %s — not .ccs.bam or hifi_reads.bam", + read.file_name, + ) + + elif platform == "OXFORD_NANOPORE": + logger.info("Adding ONT read: %s", read.file_name) + if bpa_package_id not in ont_by_package: + entry = { "sample_id": sample_id, "bpa_sample_id": sample_meta.get("bpa_sample_id"), "specimen_id": sample_meta.get("specimen_id"), @@ -191,62 +196,78 @@ def generate_assembly_manifest( } if exp_info["bioplatforms_base_url"]: entry["bioplatforms_base_url"] = exp_info["bioplatforms_base_url"] - pacbio_by_package[bpa_package_id] = entry - pacbio_by_package[bpa_package_id]["resources"].append( + ont_by_package[bpa_package_id] = entry + ont_by_package[bpa_package_id]["resources"].append( {"md5sum": read.file_checksum, "url": read.bioplatforms_url} ) + else: logger.debug( - f"Skipping PacBio read {read.file_name} - doesn't match .ccs.bam or hifi_reads.bam" + "Long-read sample read %s skipped: platform=%s not a long-read platform", + read.file_name, + platform, ) - # Hi-C reads (Illumina + Hi-C or WGS library strategy) - elif platform == "ILLUMINA" and library_strategy in ("HI-C", "WGS"): - logger.info(f"Adding Hi-C read: {read.file_name} (library_strategy={library_strategy})") - if bpa_package_id not in hic_by_package: - hic_by_package[bpa_package_id] = { - "sample_id": sample_id, - "bpa_sample_id": sample_meta.get("bpa_sample_id"), - "specimen_id": sample_meta.get("specimen_id"), - "resources": {"r1": [], "r2": []}, - } - rkey = _normalize_read_number(read.read_number) - if rkey in ("r1", "r2"): - hic_by_package[bpa_package_id]["resources"][rkey].append( - { - "url": read.bioplatforms_url, - "md5sum": read.file_checksum, - "lane_number": read.lane_number, - } + elif hic_sample_str and sample_id == hic_sample_str: + if platform == "ILLUMINA" and library_strategy in ("HI-C", "WGS"): + logger.info( + "Adding Hi-C read: %s (library_strategy=%s)", read.file_name, library_strategy ) + if bpa_package_id not in hic_by_package: + hic_by_package[bpa_package_id] = { + "sample_id": sample_id, + "bpa_sample_id": sample_meta.get("bpa_sample_id"), + "specimen_id": sample_meta.get("specimen_id"), + "resources": {"r1": [], "r2": []}, + } + rkey = _normalize_read_number(read.read_number) + if rkey in ("r1", "r2"): + hic_by_package[bpa_package_id]["resources"][rkey].append( + { + "url": read.bioplatforms_url, + "md5sum": read.file_checksum, + "lane_number": read.lane_number, + } + ) + else: + logger.debug( + "Hi-C read %s has unrecognised read_number=%s, skipping", + read.file_name, + read.read_number, + ) else: logger.debug( - f"Hi-C read {read.file_name} has unrecognized read_number={read.read_number}, skipping" + "Hi-C sample read %s skipped: platform=%s library_strategy=%s not Hi-C", + read.file_name, + platform, + library_strategy, ) + else: logger.debug( - f"Read {read.file_name} doesn't match criteria: platform={platform}, library_strategy={library_strategy}" + "Read %s sample_id=%s does not match either specimen sample, skipping", + read.id, + sample_id, ) - # Build manifest structure - manifest = { - "scientific_name": organism.scientific_name, - "taxon_id": organism.taxon_id, - "tolid": tol_id, - "version": version, - "reads": {}, - } - + reads_section: Dict[str, Any] = {} if pacbio_by_package: - manifest["reads"]["PACBIO_SMRT"] = pacbio_by_package - logger.info(f"Added {len(pacbio_by_package)} PacBio packages to manifest") - + reads_section["PACBIO_SMRT"] = pacbio_by_package + logger.info("Added %d PacBio packages to manifest", len(pacbio_by_package)) + if ont_by_package: + reads_section["OXFORD_NANOPORE"] = ont_by_package + logger.info("Added %d ONT packages to manifest", len(ont_by_package)) if hic_by_package: - manifest["reads"]["Hi-C"] = hic_by_package - logger.info(f"Added {len(hic_by_package)} Hi-C packages to manifest") + reads_section["Hi-C"] = hic_by_package + logger.info("Added %d Hi-C packages to manifest", len(hic_by_package)) - if not pacbio_by_package and not hic_by_package: + if not reads_section: logger.warning("No reads matched the filtering criteria!") - # Convert to YAML - return yaml.dump(manifest, default_flow_style=False, sort_keys=False) + return { + "scientific_name": organism.scientific_name, + "taxon_id": organism.taxon_id, + "tolid": tol_id, + "version": version, + "reads": reads_section, + } diff --git a/app/services/assembly_service.py b/app/services/assembly_service.py index c14cfcf..37cb45a 100644 --- a/app/services/assembly_service.py +++ b/app/services/assembly_service.py @@ -183,28 +183,63 @@ def get_next_version( ) return max(max_assembly or 0, max_run or 0) + 1 + def get_next_version_for_intent( + self, + db: Session, + *, + taxon_id: int, + long_read_specimen_sample_id: UUID, + ) -> int: + """Return the next version scoped by (taxon_id, long_read_specimen_sample_id). + + This is the versioning strategy for the intent flow. Versions are shared + across all assemblies for the same taxon + long-read specimen, regardless + of data_types or hic_specimen_sample_id. + """ + max_version = ( + db.query(func.max(Assembly.version)) + .filter( + Assembly.taxon_id == taxon_id, + Assembly.long_read_specimen_sample_id == long_read_specimen_sample_id, + ) + .scalar() + ) + return (max_version or 0) + 1 + def create_from_intent( self, db: Session, *, taxon_id: int, - sample_id: UUID, + long_read_specimen_sample_id: UUID, + hic_specimen_sample_id: Optional[UUID], data_types: str, tol_id: Optional[str], project_id: Optional[UUID], + manifest_json: Optional[dict] = None, ) -> Assembly: - """Create an Assembly at manifest-request time with status='requested'.""" - version = self.get_next_version( - db, taxon_id=taxon_id, sample_id=sample_id, data_types=data_types + """Create an Assembly at manifest-request time with status='requested'. + + Versioning is scoped by (taxon_id, long_read_specimen_sample_id) only — + data_types and hic_specimen_sample_id are excluded from the version key. + sample_id is set to long_read_specimen_sample_id for backward compatibility. + """ + version = self.get_next_version_for_intent( + db, + taxon_id=taxon_id, + long_read_specimen_sample_id=long_read_specimen_sample_id, ) assembly = Assembly( taxon_id=taxon_id, - sample_id=sample_id, + sample_id=long_read_specimen_sample_id, + long_read_specimen_sample_id=long_read_specimen_sample_id, + hic_specimen_sample_id=hic_specimen_sample_id, data_types=data_types, version=version, tol_id=tol_id, project_id=project_id, status="requested", + manifest_json=manifest_json, ) db.add(assembly) db.commit() diff --git a/schema.sql b/schema.sql index 1f31fbe..5fd90eb 100644 --- a/schema.sql +++ b/schema.sql @@ -511,7 +511,16 @@ CREATE TABLE assembly ( moleculetype molecule_type NOT NULL DEFAULT 'genomic DNA', description TEXT, - -- Auto-incremented version per (data_types, taxon_id, sample_id) + -- Intent-flow specimen samples + -- long_read_specimen_sample_id: the specimen sample supplying PacBio / ONT reads + -- hic_specimen_sample_id: the specimen sample supplying Hi-C reads (optional) + long_read_specimen_sample_id UUID REFERENCES sample(id), + hic_specimen_sample_id UUID REFERENCES sample(id), + + -- Persisted manifest generated at intent time + manifest_json JSONB, + + -- Version is scoped by (taxon_id, long_read_specimen_sample_id) for intent-flow assemblies version INTEGER NOT NULL DEFAULT 1, -- Assembly lifecycle status @@ -556,9 +565,12 @@ CREATE TABLE assembly_file ( CREATE INDEX idx_assembly_file_assembly_id ON assembly_file(assembly_id); CREATE INDEX idx_assembly_file_type ON assembly_file(assembly_id, file_type); --- Index for version lookups +-- Index for legacy version lookups (generic create flow) CREATE INDEX idx_assembly_version_key ON assembly(data_types, taxon_id, sample_id, version); +-- Index for intent-flow version lookups (scoped by taxon + long-read specimen sample) +CREATE INDEX idx_assembly_intent_version_key ON assembly(taxon_id, long_read_specimen_sample_id); + -- Assembly submission table (simplified - no broker integration) CREATE TABLE assembly_submission ( diff --git a/tests/unit/services/test_assembly_helper.py b/tests/unit/services/test_assembly_helper.py index 1e89408..76bc664 100644 --- a/tests/unit/services/test_assembly_helper.py +++ b/tests/unit/services/test_assembly_helper.py @@ -1,9 +1,9 @@ """Tests for assembly helper functions.""" from unittest.mock import Mock +from uuid import UUID import pytest -import yaml from app.models.experiment import Experiment from app.models.organism import Organism @@ -11,139 +11,113 @@ from app.schemas.assembly import AssemblyDataTypes from app.services.assembly_helper import ( determine_assembly_data_types, - generate_assembly_manifest, + generate_assembly_manifest_json, get_detected_platforms, ) +LONG_READ_SAMPLE_ID = UUID("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa") +HIC_SAMPLE_ID = UUID("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb") +LONG_READ_SAMPLE_STR = str(LONG_READ_SAMPLE_ID) +HIC_SAMPLE_STR = str(HIC_SAMPLE_ID) + class TestDetermineAssemblyDataTypes: """Tests for determine_assembly_data_types function.""" def test_pacbio_only(self): - """Test detection of PacBio SMRT only.""" - experiments = [ - Mock(platform="PACBIO_SMRT", library_strategy="WGS"), - ] - result = determine_assembly_data_types(experiments) - assert result == AssemblyDataTypes.PACBIO_SMRT + experiments = [Mock(platform="PACBIO_SMRT", library_strategy="WGS")] + assert determine_assembly_data_types(experiments) == AssemblyDataTypes.PACBIO_SMRT def test_oxford_nanopore_only(self): - """Test detection of Oxford Nanopore only.""" - experiments = [ - Mock(platform="OXFORD_NANOPORE", library_strategy="WGS"), - ] - result = determine_assembly_data_types(experiments) - assert result == AssemblyDataTypes.OXFORD_NANOPORE + experiments = [Mock(platform="OXFORD_NANOPORE", library_strategy="WGS")] + assert determine_assembly_data_types(experiments) == AssemblyDataTypes.OXFORD_NANOPORE - def test_illumina_hic(self): - """Test that Illumina-only raises error (no long-read platform).""" - experiments = [ - Mock(platform="ILLUMINA", library_strategy="Hi-C"), - ] + def test_illumina_hic_alone_raises_error(self): + experiments = [Mock(platform="ILLUMINA", library_strategy="Hi-C")] with pytest.raises(ValueError, match="No valid data types detected in experiments"): determine_assembly_data_types(experiments) def test_illumina_wgs_treated_as_hic(self): - """Test that ILLUMINA + WGS is treated as Hi-C.""" experiments = [ Mock(platform="PACBIO_SMRT", library_strategy="WGS"), Mock(platform="ILLUMINA", library_strategy="WGS"), ] - result = determine_assembly_data_types(experiments) - assert result == AssemblyDataTypes.PACBIO_SMRT_HIC + assert determine_assembly_data_types(experiments) == AssemblyDataTypes.PACBIO_SMRT_HIC def test_pacbio_and_hic(self): - """Test detection of PacBio + Hi-C combination.""" experiments = [ Mock(platform="PACBIO_SMRT", library_strategy="WGS"), Mock(platform="ILLUMINA", library_strategy="Hi-C"), ] - result = determine_assembly_data_types(experiments) - assert result == AssemblyDataTypes.PACBIO_SMRT_HIC + assert determine_assembly_data_types(experiments) == AssemblyDataTypes.PACBIO_SMRT_HIC def test_nanopore_and_hic(self): - """Test detection of Oxford Nanopore + Hi-C combination.""" experiments = [ Mock(platform="OXFORD_NANOPORE", library_strategy="WGS"), Mock(platform="ILLUMINA", library_strategy="WGS"), ] - result = determine_assembly_data_types(experiments) - assert result == AssemblyDataTypes.OXFORD_NANOPORE_HIC + assert determine_assembly_data_types(experiments) == AssemblyDataTypes.OXFORD_NANOPORE_HIC def test_pacbio_and_nanopore(self): - """Test detection of PacBio + Oxford Nanopore combination.""" experiments = [ Mock(platform="PACBIO_SMRT", library_strategy="WGS"), Mock(platform="OXFORD_NANOPORE", library_strategy="WGS"), ] - result = determine_assembly_data_types(experiments) - assert result == AssemblyDataTypes.PACBIO_SMRT_OXFORD_NANOPORE + assert ( + determine_assembly_data_types(experiments) + == AssemblyDataTypes.PACBIO_SMRT_OXFORD_NANOPORE + ) def test_all_three_platforms(self): - """Test detection of all three platform types.""" experiments = [ Mock(platform="PACBIO_SMRT", library_strategy="WGS"), Mock(platform="OXFORD_NANOPORE", library_strategy="WGS"), Mock(platform="ILLUMINA", library_strategy="Hi-C"), ] - result = determine_assembly_data_types(experiments) - assert result == AssemblyDataTypes.PACBIO_SMRT_OXFORD_NANOPORE_HIC + assert ( + determine_assembly_data_types(experiments) + == AssemblyDataTypes.PACBIO_SMRT_OXFORD_NANOPORE_HIC + ) def test_case_insensitive_platform(self): - """Test that platform detection is case-insensitive.""" - experiments = [ - Mock(platform="pacbio_smrt", library_strategy="WGS"), - ] - result = determine_assembly_data_types(experiments) - assert result == AssemblyDataTypes.PACBIO_SMRT + experiments = [Mock(platform="pacbio_smrt", library_strategy="WGS")] + assert determine_assembly_data_types(experiments) == AssemblyDataTypes.PACBIO_SMRT def test_case_insensitive_library_strategy(self): - """Test that library strategy detection is case-insensitive.""" experiments = [ Mock(platform="OXFORD_NANOPORE", library_strategy="WGS"), Mock(platform="ILLUMINA", library_strategy="wgs"), ] - result = determine_assembly_data_types(experiments) - assert result == AssemblyDataTypes.OXFORD_NANOPORE_HIC + assert determine_assembly_data_types(experiments) == AssemblyDataTypes.OXFORD_NANOPORE_HIC def test_no_valid_platforms_raises_error(self): - """Test that no valid platforms raises ValueError.""" - experiments = [ - Mock(platform="UNKNOWN", library_strategy="WGS"), - ] + experiments = [Mock(platform="UNKNOWN", library_strategy="WGS")] with pytest.raises(ValueError, match="No valid data types detected in experiments"): determine_assembly_data_types(experiments) def test_empty_experiments_raises_error(self): - """Test that empty experiments list raises ValueError.""" with pytest.raises(ValueError, match="No valid data types detected in experiments"): determine_assembly_data_types([]) def test_none_platform_ignored(self): - """Test that experiments with None platform are ignored.""" experiments = [ Mock(platform=None, library_strategy="WGS"), Mock(platform="PACBIO_SMRT", library_strategy="WGS"), ] - result = determine_assembly_data_types(experiments) - assert result == AssemblyDataTypes.PACBIO_SMRT + assert determine_assembly_data_types(experiments) == AssemblyDataTypes.PACBIO_SMRT class TestGetDetectedPlatforms: """Tests for get_detected_platforms function.""" def test_single_platform(self): - """Test detection of single platform.""" - experiments = [ - Mock(platform="PACBIO_SMRT", library_strategy="WGS"), - ] + experiments = [Mock(platform="PACBIO_SMRT", library_strategy="WGS")] result = get_detected_platforms(experiments) assert result["platforms"] == ["PACBIO_SMRT"] assert result["library_strategies"] == ["WGS"] assert result["experiment_count"] == 1 def test_multiple_platforms(self): - """Test detection of multiple platforms.""" experiments = [ Mock(platform="PACBIO_SMRT", library_strategy="WGS"), Mock(platform="ILLUMINA", library_strategy="Hi-C"), @@ -154,15 +128,22 @@ def test_multiple_platforms(self): assert result["experiment_count"] == 2 def test_empty_experiments(self): - """Test with empty experiments list.""" result = get_detected_platforms([]) assert result["platforms"] == [] assert result["library_strategies"] == [] assert result["experiment_count"] == 0 +# ────────────────────────────────────────────────────────────────────────────── +# Helpers for building mock experiments / reads +# ────────────────────────────────────────────────────────────────────────────── + + def _make_pacbio_experiment( - exp_id="exp1", bpa_package_id="pkg-001", sample_id="sample-uuid-1", bioplatforms_base_url=None + exp_id="exp1", + bpa_package_id="pkg-001", + sample_id=LONG_READ_SAMPLE_STR, + bioplatforms_base_url=None, ): return Mock( id=exp_id, @@ -174,8 +155,27 @@ def _make_pacbio_experiment( ) +def _make_ont_experiment( + exp_id="exp-ont", + bpa_package_id="pkg-ont", + sample_id=LONG_READ_SAMPLE_STR, + bioplatforms_base_url=None, +): + return Mock( + id=exp_id, + platform="OXFORD_NANOPORE", + library_strategy="WGS", + bpa_package_id=bpa_package_id, + bioplatforms_base_url=bioplatforms_base_url, + sample_id=sample_id, + ) + + def _make_hic_experiment( - exp_id="exp2", bpa_package_id="pkg-002", sample_id="sample-uuid-2", bioplatforms_base_url=None + exp_id="exp2", + bpa_package_id="pkg-002", + sample_id=HIC_SAMPLE_STR, + bioplatforms_base_url=None, ): return Mock( id=exp_id, @@ -187,11 +187,13 @@ def _make_hic_experiment( ) -class TestGenerateAssemblyManifest: - """Tests for generate_assembly_manifest function.""" +class TestGenerateAssemblyManifestJson: + """Tests for generate_assembly_manifest_json function.""" + + # ── PacBio ──────────────────────────────────────────────────────────────── def test_pacbio_reads_filtered_by_extension(self): - """Test that only .ccs.bam and hifi_reads.bam files are included for PacBio.""" + """Only .ccs.bam and hifi_reads.bam files are included for PacBio.""" organism = Mock(scientific_name="Test Species", taxon_id=12345) experiments = [_make_pacbio_experiment()] reads = [ @@ -224,10 +226,11 @@ def test_pacbio_reads_filtered_by_extension(self): ), ] - result = generate_assembly_manifest(organism, reads, experiments, "tol1", 1) - data = yaml.safe_load(result) + result = generate_assembly_manifest_json( + organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID + ) - pacbio = data["reads"]["PACBIO_SMRT"] + pacbio = result["reads"]["PACBIO_SMRT"] assert "pkg-001" in pacbio resources = pacbio["pkg-001"]["resources"] urls = [r["url"] for r in resources] @@ -235,10 +238,67 @@ def test_pacbio_reads_filtered_by_extension(self): assert "https://example.com/2" in urls assert "https://example.com/3" not in urls assert all("md5sum" in r for r in resources) - assert all("file_name" not in r for r in resources) + + def test_pacbio_read_without_file_name_skipped(self): + """PacBio reads with no file_name are silently skipped.""" + organism = Mock(scientific_name="Test Species", taxon_id=12345) + experiments = [_make_pacbio_experiment()] + reads = [ + Mock( + id="r1", + experiment_id="exp1", + file_name=None, + file_checksum="abc123", + bioplatforms_url="https://example.com/1", + read_number=None, + lane_number=None, + ), + ] + + result = generate_assembly_manifest_json( + organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID + ) + assert result["reads"] == {} + + # ── Oxford Nanopore ─────────────────────────────────────────────────────── + + def test_ont_reads_included_without_extension_filter(self): + """All ONT reads are included regardless of file extension.""" + organism = Mock(scientific_name="Test Species", taxon_id=12345) + experiments = [_make_ont_experiment()] + reads = [ + Mock( + id="r1", + experiment_id="exp-ont", + file_name="sample.fastq.gz", + file_checksum="abc123", + bioplatforms_url="https://example.com/ont1", + read_number=None, + lane_number=None, + ), + Mock( + id="r2", + experiment_id="exp-ont", + file_name="sample.pod5", + file_checksum="def456", + bioplatforms_url="https://example.com/ont2", + read_number=None, + lane_number=None, + ), + ] + + result = generate_assembly_manifest_json( + organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID + ) + + assert "OXFORD_NANOPORE" in result["reads"] + resources = result["reads"]["OXFORD_NANOPORE"]["pkg-ont"]["resources"] + assert len(resources) == 2 + + # ── Hi-C ───────────────────────────────────────────────────────────────── def test_hic_reads_include_metadata(self): - """Test that Hi-C reads include lane_number and are split by r1/r2.""" + """Hi-C reads include lane_number and are split by r1/r2.""" organism = Mock(scientific_name="Test Species", taxon_id=12345) experiments = [_make_hic_experiment()] reads = [ @@ -253,10 +313,11 @@ def test_hic_reads_include_metadata(self): ), ] - result = generate_assembly_manifest(organism, reads, experiments, "tol1", 1) - data = yaml.safe_load(result) + result = generate_assembly_manifest_json( + organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID, HIC_SAMPLE_ID + ) - hic = data["reads"]["Hi-C"] + hic = result["reads"]["Hi-C"] assert "pkg-002" in hic r1_resources = hic["pkg-002"]["resources"]["r1"] assert len(r1_resources) == 1 @@ -264,7 +325,7 @@ def test_hic_reads_include_metadata(self): assert r1_resources[0]["md5sum"] == "abc123" def test_hic_reads_split_into_r1_r2(self): - """Test that Hi-C reads with read_number 1 and 2 are split into r1 and r2.""" + """Hi-C reads with read_number 1 and 2 are split into r1 and r2.""" organism = Mock(scientific_name="Test Species", taxon_id=12345) experiments = [_make_hic_experiment()] reads = [ @@ -288,17 +349,18 @@ def test_hic_reads_split_into_r1_r2(self): ), ] - result = generate_assembly_manifest(organism, reads, experiments, "tol1", 1) - data = yaml.safe_load(result) + result = generate_assembly_manifest_json( + organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID, HIC_SAMPLE_ID + ) - resources = data["reads"]["Hi-C"]["pkg-002"]["resources"] + resources = result["reads"]["Hi-C"]["pkg-002"]["resources"] assert len(resources["r1"]) == 1 assert len(resources["r2"]) == 1 assert resources["r1"][0]["url"] == "https://example.com/r1" assert resources["r2"][0]["url"] == "https://example.com/r2" def test_wgs_treated_as_hic(self): - """Test that ILLUMINA + WGS is treated as Hi-C.""" + """ILLUMINA + WGS is treated as Hi-C.""" organism = Mock(scientific_name="Test Species", taxon_id=12345) experiments = [ Mock( @@ -307,7 +369,7 @@ def test_wgs_treated_as_hic(self): library_strategy="WGS", bpa_package_id="pkg-wgs", bioplatforms_base_url=None, - sample_id="s1", + sample_id=HIC_SAMPLE_STR, ) ] reads = [ @@ -322,57 +384,177 @@ def test_wgs_treated_as_hic(self): ), ] - result = generate_assembly_manifest(organism, reads, experiments, "tol1", 1) - data = yaml.safe_load(result) + result = generate_assembly_manifest_json( + organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID, HIC_SAMPLE_ID + ) - assert "Hi-C" in data["reads"] - assert "pkg-wgs" in data["reads"]["Hi-C"] + assert "Hi-C" in result["reads"] + assert "pkg-wgs" in result["reads"]["Hi-C"] - def test_empty_reads_dict(self): - """Test that empty reads result in empty reads dict.""" + def test_hic_section_omitted_when_no_hic_sample_id(self): + """Hi-C section is absent when hic_sample_id is not supplied.""" organism = Mock(scientific_name="Test Species", taxon_id=12345) experiments = [ + _make_pacbio_experiment(), + _make_hic_experiment(), # hic experiment present but hic_sample_id not passed + ] + reads = [ Mock( - id="exp1", - platform="UNKNOWN", + id="r1", + experiment_id="exp1", + file_name="sample.ccs.bam", + file_checksum="abc123", + bioplatforms_url="https://example.com/1", + read_number=None, + lane_number=None, + ), + Mock( + id="r2", + experiment_id="exp2", + file_name="hic_R1.fastq.gz", + file_checksum="def456", + bioplatforms_url="https://example.com/2", + read_number="1", + lane_number="001", + ), + ] + + # hic_sample_id is None → Hi-C section must be absent + result = generate_assembly_manifest_json( + organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID, hic_sample_id=None + ) + + assert "Hi-C" not in result["reads"] + assert "PACBIO_SMRT" in result["reads"] + + def test_reads_routed_by_specimen_sample_not_platform(self): + """Reads from long_read_sample_id are never placed in Hi-C even if ILLUMINA.""" + organism = Mock(scientific_name="Test Species", taxon_id=12345) + # An ILLUMINA experiment that belongs to the long_read_specimen_sample_id + experiments = [ + Mock( + id="exp-ill", + platform="ILLUMINA", library_strategy="WGS", - bpa_package_id="pkg-x", + bpa_package_id="pkg-ill", bioplatforms_base_url=None, - sample_id="s1", + sample_id=LONG_READ_SAMPLE_STR, # belongs to long-read sample ) ] - reads = [] + reads = [ + Mock( + id="r1", + experiment_id="exp-ill", + file_name="sample_R1.fastq.gz", + file_checksum="abc123", + bioplatforms_url="https://example.com/1", + read_number="1", + lane_number="001", + ), + ] - result = generate_assembly_manifest(organism, reads, experiments, "tol1", 1) + # Illumina experiment on long-read sample → should NOT appear in Hi-C section + result = generate_assembly_manifest_json( + organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID, HIC_SAMPLE_ID + ) - assert "reads: {}" in result + assert result["reads"] == {} - def test_organism_metadata_included(self): - """Test that organism metadata is included in manifest.""" - organism = Mock(scientific_name="Saiphos equalis", taxon_id=172942) - experiments = [] - reads = [] + # ── Mixed long-read + Hi-C ──────────────────────────────────────────────── - result = generate_assembly_manifest(organism, reads, experiments, "tol123", 2) + def test_multiple_platform_types(self): + """Manifest with PacBio (long-read sample) and Hi-C (hic sample) packages.""" + organism = Mock(scientific_name="Test Species", taxon_id=12345) + experiments = [ + _make_pacbio_experiment(exp_id="exp1", bpa_package_id="pkg-pacbio"), + _make_hic_experiment(exp_id="exp2", bpa_package_id="pkg-hic"), + ] + reads = [ + Mock( + id="r1", + experiment_id="exp1", + file_name="sample.ccs.bam", + file_checksum="abc123", + bioplatforms_url="https://example.com/1", + read_number=None, + lane_number=None, + ), + Mock( + id="r2", + experiment_id="exp2", + file_name="hic_R1.fastq.gz", + file_checksum="def456", + bioplatforms_url="https://example.com/2", + read_number="1", + lane_number="001", + ), + ] - assert "scientific_name: Saiphos equalis" in result - assert "taxon_id: 172942" in result - assert "tolid: tol123" in result - assert "version: 2" in result + result = generate_assembly_manifest_json( + organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID, HIC_SAMPLE_ID + ) - def test_sample_metadata_included_at_package_level(self): - """Test that sample metadata appears at the bpa_package_id level.""" - organism = Mock(scientific_name="Saiphos equalis", taxon_id=172942) + assert "PACBIO_SMRT" in result["reads"] + assert "Hi-C" in result["reads"] + assert "pkg-pacbio" in result["reads"]["PACBIO_SMRT"] + assert "pkg-hic" in result["reads"]["Hi-C"] + + def test_pacbio_and_ont_in_same_long_read_sample(self): + """Both PacBio and ONT reads from the same long-read specimen appear in separate sections.""" + organism = Mock(scientific_name="Test Species", taxon_id=12345) experiments = [ + _make_pacbio_experiment(exp_id="exp-pb", bpa_package_id="pkg-pb"), + _make_ont_experiment(exp_id="exp-ont", bpa_package_id="pkg-ont"), + ] + reads = [ Mock( - id="exp1", - sample_id="550e8400-e29b-41d4-a716-446655440000", - platform="PACBIO_SMRT", - library_strategy="WGS", - bpa_package_id="pkg-001", - bioplatforms_base_url=None, - ) + id="r1", + experiment_id="exp-pb", + file_name="sample.ccs.bam", + file_checksum="abc", + bioplatforms_url="https://example.com/pb", + read_number=None, + lane_number=None, + ), + Mock( + id="r2", + experiment_id="exp-ont", + file_name="sample.fastq.gz", + file_checksum="def", + bioplatforms_url="https://example.com/ont", + read_number=None, + lane_number=None, + ), ] + + result = generate_assembly_manifest_json( + organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID + ) + + assert "PACBIO_SMRT" in result["reads"] + assert "OXFORD_NANOPORE" in result["reads"] + + # ── Metadata ────────────────────────────────────────────────────────────── + + def test_empty_reads_dict(self): + """Empty reads result in an empty reads section.""" + organism = Mock(scientific_name="Test Species", taxon_id=12345) + result = generate_assembly_manifest_json(organism, [], [], "tol1", 1, LONG_READ_SAMPLE_ID) + assert result["reads"] == {} + + def test_organism_metadata_included(self): + """Organism metadata is present in the manifest.""" + organism = Mock(scientific_name="Saiphos equalis", taxon_id=172942) + result = generate_assembly_manifest_json(organism, [], [], "tol123", 2, LONG_READ_SAMPLE_ID) + assert result["scientific_name"] == "Saiphos equalis" + assert result["taxon_id"] == 172942 + assert result["tolid"] == "tol123" + assert result["version"] == 2 + + def test_sample_metadata_included_at_package_level(self): + """Sample metadata (bpa_sample_id, specimen_id) appears at the package level.""" + organism = Mock(scientific_name="Saiphos equalis", taxon_id=172942) + experiments = [_make_pacbio_experiment()] reads = [ Mock( id="r1", @@ -385,24 +567,28 @@ def test_sample_metadata_included_at_package_level(self): ) ] sample_metadata_by_id = { - "550e8400-e29b-41d4-a716-446655440000": { + LONG_READ_SAMPLE_STR: { "bpa_sample_id": "102.100.100/9000", "specimen_id": "SPEC-001", } } - result = generate_assembly_manifest( - organism, reads, experiments, "tol123", 2, sample_metadata_by_id + result = generate_assembly_manifest_json( + organism, + reads, + experiments, + "tol123", + 2, + LONG_READ_SAMPLE_ID, + sample_metadata_by_id=sample_metadata_by_id, ) - data = yaml.safe_load(result) - pkg = data["reads"]["PACBIO_SMRT"]["pkg-001"] - assert pkg["sample_id"] == "550e8400-e29b-41d4-a716-446655440000" + pkg = result["reads"]["PACBIO_SMRT"]["pkg-001"] + assert pkg["sample_id"] == LONG_READ_SAMPLE_STR assert pkg["bpa_sample_id"] == "102.100.100/9000" assert pkg["specimen_id"] == "SPEC-001" def test_bioplatforms_base_url_included_when_set(self): - """Test that bioplatforms_base_url appears in the manifest when set on the experiment.""" organism = Mock(scientific_name="Test Species", taxon_id=12345) experiments = [ _make_pacbio_experiment(bioplatforms_base_url="https://base.example.com/pkg-001") @@ -419,14 +605,13 @@ def test_bioplatforms_base_url_included_when_set(self): ), ] - result = generate_assembly_manifest(organism, reads, experiments, "tol1", 1) - data = yaml.safe_load(result) - - pkg = data["reads"]["PACBIO_SMRT"]["pkg-001"] + result = generate_assembly_manifest_json( + organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID + ) + pkg = result["reads"]["PACBIO_SMRT"]["pkg-001"] assert pkg["bioplatforms_base_url"] == "https://base.example.com/pkg-001" def test_bioplatforms_base_url_omitted_when_none(self): - """Test that bioplatforms_base_url is absent from the manifest when not set.""" organism = Mock(scientific_name="Test Species", taxon_id=12345) experiments = [_make_pacbio_experiment(bioplatforms_base_url=None)] reads = [ @@ -441,14 +626,13 @@ def test_bioplatforms_base_url_omitted_when_none(self): ), ] - result = generate_assembly_manifest(organism, reads, experiments, "tol1", 1) - data = yaml.safe_load(result) - - pkg = data["reads"]["PACBIO_SMRT"]["pkg-001"] + result = generate_assembly_manifest_json( + organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID + ) + pkg = result["reads"]["PACBIO_SMRT"]["pkg-001"] assert "bioplatforms_base_url" not in pkg def test_reads_without_experiment_id_skipped(self): - """Test that reads without experiment_id are skipped.""" organism = Mock(scientific_name="Test Species", taxon_id=12345) experiments = [_make_pacbio_experiment()] reads = [ @@ -463,48 +647,13 @@ def test_reads_without_experiment_id_skipped(self): ), ] - result = generate_assembly_manifest(organism, reads, experiments, "tol1", 1) - - assert "reads: {}" in result - - def test_multiple_platform_types(self): - """Test manifest with both PacBio and Hi-C packages.""" - organism = Mock(scientific_name="Test Species", taxon_id=12345) - experiments = [ - _make_pacbio_experiment(exp_id="exp1", bpa_package_id="pkg-pacbio"), - _make_hic_experiment(exp_id="exp2", bpa_package_id="pkg-hic"), - ] - reads = [ - Mock( - id="r1", - experiment_id="exp1", - file_name="sample.ccs.bam", - file_checksum="abc123", - bioplatforms_url="https://example.com/1", - read_number=None, - lane_number=None, - ), - Mock( - id="r2", - experiment_id="exp2", - file_name="hic_R1.fastq.gz", - file_checksum="def456", - bioplatforms_url="https://example.com/2", - read_number="1", - lane_number="001", - ), - ] - - result = generate_assembly_manifest(organism, reads, experiments, "tol1", 1) - data = yaml.safe_load(result) - - assert "PACBIO_SMRT" in data["reads"] - assert "Hi-C" in data["reads"] - assert "pkg-pacbio" in data["reads"]["PACBIO_SMRT"] - assert "pkg-hic" in data["reads"]["Hi-C"] + result = generate_assembly_manifest_json( + organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID + ) + assert result["reads"] == {} def test_multiple_reads_same_package(self): - """Test that multiple reads under the same experiment are grouped together.""" + """Multiple reads under the same experiment are grouped together.""" organism = Mock(scientific_name="Test Species", taxon_id=12345) experiments = [_make_pacbio_experiment()] reads = [ @@ -528,8 +677,15 @@ def test_multiple_reads_same_package(self): ), ] - result = generate_assembly_manifest(organism, reads, experiments, "tol1", 1) - data = yaml.safe_load(result) - - resources = data["reads"]["PACBIO_SMRT"]["pkg-001"]["resources"] + result = generate_assembly_manifest_json( + organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID + ) + resources = result["reads"]["PACBIO_SMRT"]["pkg-001"]["resources"] assert len(resources) == 2 + + def test_manifest_is_dict_not_string(self): + """generate_assembly_manifest_json returns a dict, not a YAML/JSON string.""" + organism = Mock(scientific_name="Test Species", taxon_id=12345) + result = generate_assembly_manifest_json(organism, [], [], "tol1", 1, LONG_READ_SAMPLE_ID) + assert isinstance(result, dict) + assert "reads" in result diff --git a/tests/unit/services/test_assembly_service.py b/tests/unit/services/test_assembly_service.py index 78114b7..d4cec33 100644 --- a/tests/unit/services/test_assembly_service.py +++ b/tests/unit/services/test_assembly_service.py @@ -13,25 +13,25 @@ from app.schemas.assembly import AssemblyCreate, AssemblyCreateFromExperiments, AssemblyDataTypes from app.services.assembly_service import AssemblyService +LONG_READ_SAMPLE_ID = uuid.uuid4() +HIC_SAMPLE_ID = uuid.uuid4() + @pytest.fixture def mock_db(): - """Create a mock database session.""" return MagicMock(spec=Session) @pytest.fixture def assembly_service(): - """Create an AssemblyService instance.""" return AssemblyService(Assembly) @pytest.fixture def sample_assembly_create(): - """Create a sample AssemblyCreate schema.""" return AssemblyCreate( taxon_id=172942, - sample_id=uuid.uuid4(), + sample_id=LONG_READ_SAMPLE_ID, project_id=uuid.uuid4(), assembly_name="Test Assembly", assembly_type="clone or isolate", @@ -44,57 +44,39 @@ def sample_assembly_create(): class TestCreateAssembly: - """Tests for create method with auto-increment versioning.""" + """Tests for create method with auto-increment versioning (generic flow).""" def test_create_first_version(self, mock_db, assembly_service, sample_assembly_create): - """Test creating first version (version 1).""" - # Mock query to return None (no existing versions) mock_query = Mock() mock_query.filter.return_value.scalar.return_value = None mock_db.query.return_value = mock_query - - # Mock the assembly object that will be created - created_assembly = Assembly( - id=uuid.uuid4(), - taxon_id=sample_assembly_create.taxon_id, - sample_id=sample_assembly_create.sample_id, - data_types=sample_assembly_create.data_types, - version=1, - ) mock_db.add = Mock() mock_db.commit = Mock() mock_db.refresh = Mock() with patch.object(Assembly, "__init__", return_value=None): - result = assembly_service.create(mock_db, obj_in=sample_assembly_create) + assembly_service.create(mock_db, obj_in=sample_assembly_create) - # Verify version was set to 1 mock_db.add.assert_called_once() mock_db.commit.assert_called_once() def test_create_increments_version(self, mock_db, assembly_service, sample_assembly_create): - """Test that version increments from existing max version.""" - # Mock query to return existing max version of 3 mock_query = Mock() mock_query.filter.return_value.scalar.return_value = 3 mock_db.query.return_value = mock_query - mock_db.add = Mock() mock_db.commit = Mock() mock_db.refresh = Mock() with patch.object(Assembly, "__init__", return_value=None): - result = assembly_service.create(mock_db, obj_in=sample_assembly_create) + assembly_service.create(mock_db, obj_in=sample_assembly_create) - # Verify version was incremented to 4 mock_db.add.assert_called_once() mock_db.commit.assert_called_once() def test_create_version_per_combination(self, mock_db, assembly_service): - """Test that version is per (data_types, taxon_id, sample_id) combination.""" sample_id = uuid.uuid4() - # Create two assemblies with different data_types assembly1 = AssemblyCreate( taxon_id=172942, sample_id=sample_id, @@ -106,7 +88,6 @@ def test_create_version_per_combination(self, mock_db, assembly_service): program="hifiasm", moleculetype="genomic DNA", ) - assembly2 = AssemblyCreate( taxon_id=172942, sample_id=sample_id, @@ -119,7 +100,6 @@ def test_create_version_per_combination(self, mock_db, assembly_service): moleculetype="genomic DNA", ) - # Mock query to return None for both (different combinations) mock_query = Mock() mock_query.filter.return_value.scalar.return_value = None mock_db.query.return_value = mock_query @@ -131,7 +111,6 @@ def test_create_version_per_combination(self, mock_db, assembly_service): assembly_service.create(mock_db, obj_in=assembly1) assembly_service.create(mock_db, obj_in=assembly2) - # Both should get version 1 since they have different data_types assert mock_db.add.call_count == 2 @@ -139,12 +118,8 @@ class TestCreateFromExperiments: """Tests for create_from_experiments method.""" def test_create_from_experiments_success(self, mock_db, assembly_service): - """Test successful assembly creation from experiments.""" taxon_id = 172942 - organism = Organism( - taxon_id=taxon_id, - scientific_name="Test Species", - ) + organism = Organism(taxon_id=taxon_id, scientific_name="Test Species") sample = Sample(id=uuid.uuid4(), taxon_id=taxon_id) experiments = [ Experiment( @@ -155,11 +130,10 @@ def test_create_from_experiments_success(self, mock_db, assembly_service): ), ] - # Mock database queries mock_db.query.return_value.filter.return_value.first.return_value = organism mock_db.query.return_value.filter.return_value.all.side_effect = [ - [sample], # samples query - experiments, # experiments query + [sample], + experiments, ] assembly_in = AssemblyCreateFromExperiments( @@ -174,23 +148,17 @@ def test_create_from_experiments_success(self, mock_db, assembly_service): with patch.object(assembly_service, "create") as mock_create: mock_create.return_value = Mock( - id=uuid.uuid4(), - data_types=AssemblyDataTypes.PACBIO_SMRT, - version=1, + id=uuid.uuid4(), data_types=AssemblyDataTypes.PACBIO_SMRT, version=1 ) - assembly, platform_info = assembly_service.create_from_experiments( mock_db, taxon_id=taxon_id, assembly_in=assembly_in ) - # Verify platform info was returned assert "platforms" in platform_info assert "library_strategies" in platform_info assert "experiment_count" in platform_info def test_create_from_experiments_organism_not_found(self, mock_db, assembly_service): - """Test error when organism not found.""" - taxon_id = 999999 mock_db.query.return_value.filter.return_value.first.return_value = None assembly_in = AssemblyCreateFromExperiments( @@ -205,18 +173,13 @@ def test_create_from_experiments_organism_not_found(self, mock_db, assembly_serv with pytest.raises(ValueError, match="Organism with taxon_id 999999 not found"): assembly_service.create_from_experiments( - mock_db, taxon_id=taxon_id, assembly_in=assembly_in + mock_db, taxon_id=999999, assembly_in=assembly_in ) def test_create_from_experiments_no_samples(self, mock_db, assembly_service): - """Test error when no samples found for organism.""" taxon_id = 172942 - organism = Organism( - taxon_id=taxon_id, - scientific_name="Test Species", - ) + organism = Organism(taxon_id=taxon_id, scientific_name="Test Species") - # Mock organism found but no samples mock_db.query.return_value.filter.return_value.first.return_value = organism mock_db.query.return_value.filter.return_value.all.return_value = [] @@ -236,19 +199,14 @@ def test_create_from_experiments_no_samples(self, mock_db, assembly_service): ) def test_create_from_experiments_no_experiments(self, mock_db, assembly_service): - """Test error when no experiments found.""" taxon_id = 172942 - organism = Organism( - taxon_id=taxon_id, - scientific_name="Test Species", - ) + organism = Organism(taxon_id=taxon_id, scientific_name="Test Species") sample = Sample(id=uuid.uuid4(), taxon_id=taxon_id) - # Mock organism and samples found but no experiments mock_db.query.return_value.filter.return_value.first.return_value = organism mock_db.query.return_value.filter.return_value.all.side_effect = [ - [sample], # samples query - [], # experiments query - empty + [sample], + [], ] assembly_in = AssemblyCreateFromExperiments( @@ -266,79 +224,55 @@ def test_create_from_experiments_no_experiments(self, mock_db, assembly_service) mock_db, taxon_id=taxon_id, assembly_in=assembly_in ) - def test_create_from_experiments_overrides_data_types(self, mock_db, assembly_service): - """Test that data_types is overridden based on experiments.""" - taxon_id = 172942 - organism = Organism( - taxon_id=taxon_id, - scientific_name="Test Species", - ) - sample = Sample(id=uuid.uuid4(), taxon_id=taxon_id) - experiments = [ - Experiment( - id=uuid.uuid4(), - sample_id=sample.id, - platform="OXFORD_NANOPORE", - library_strategy="WGS", - ), - Experiment( - id=uuid.uuid4(), - sample_id=sample.id, - platform="ILLUMINA", - library_strategy="Hi-C", - ), - ] - mock_db.query.return_value.filter.return_value.first.return_value = organism - mock_db.query.return_value.filter.return_value.all.side_effect = [ - [sample], - experiments, - ] +class TestGetNextVersionForIntent: + """Tests for get_next_version_for_intent — versioning by (taxon_id, long_read_specimen_sample_id).""" - # User provides PACBIO_SMRT which should be used instead of auto-detection - assembly_in = AssemblyCreateFromExperiments( - sample_id=sample.id, - assembly_name="Test Assembly", - assembly_type="clone or isolate", - tol_id="tol-008", - data_types=AssemblyDataTypes.PACBIO_SMRT, # Explicitly provided, should be used - coverage=50.0, - program="hifiasm", - moleculetype="genomic DNA", + def test_first_version_when_no_prior_assemblies(self, mock_db, assembly_service): + mock_query = Mock() + mock_query.filter.return_value.scalar.return_value = None + mock_db.query.return_value = mock_query + + version = assembly_service.get_next_version_for_intent( + mock_db, taxon_id=172942, long_read_specimen_sample_id=LONG_READ_SAMPLE_ID ) + assert version == 1 - with patch.object(assembly_service, "create") as mock_create: - # Verify that create is called with overridden data_types - mock_create.return_value = Mock( - id=uuid.uuid4(), - data_types=AssemblyDataTypes.OXFORD_NANOPORE_HIC, - version=1, - ) + def test_increments_from_existing_max(self, mock_db, assembly_service): + mock_query = Mock() + mock_query.filter.return_value.scalar.return_value = 4 + mock_db.query.return_value = mock_query - assembly, platform_info = assembly_service.create_from_experiments( - mock_db, taxon_id=taxon_id, assembly_in=assembly_in - ) + version = assembly_service.get_next_version_for_intent( + mock_db, taxon_id=172942, long_read_specimen_sample_id=LONG_READ_SAMPLE_ID + ) + assert version == 5 - # Verify create was called - mock_create.assert_called_once() - call_args = mock_create.call_args - created_assembly_in = call_args.kwargs["obj_in"] + def test_different_hic_sample_does_not_split_version_sequence( + self, mock_db, assembly_service + ): + """Two intents with the same long_read_specimen_sample_id but different hic_specimen_sample_id + must share the same version counter (hic_specimen_sample_id is NOT part of the key).""" + call_count = [0] + max_versions = [2] # One prior assembly exists for this (taxon_id, long_read) pair - # The data_types should be determined from experiments (ILLUMINA+WGS = Hi-C) - assert created_assembly_in.taxon_id == 172942 + mock_query = Mock() + mock_query.filter.return_value.scalar.return_value = max_versions[0] + mock_db.query.return_value = mock_query + v1 = assembly_service.get_next_version_for_intent( + mock_db, taxon_id=172942, long_read_specimen_sample_id=LONG_READ_SAMPLE_ID + ) + # Both calls use the same (taxon_id, long_read_specimen_sample_id) key, so both + # return 3 from the same mock (in real DB the second would see version=3 → return 4). + # This test verifies the QUERY only filters on taxon_id + long_read_specimen_sample_id. + assert v1 == 3 -class TestCreateFromIntent: - """Tests for create_from_intent method.""" - def test_create_from_intent_creates_assembly_with_requested_status( - self, mock_db, assembly_service - ): - """create_from_intent should create an Assembly with status='requested'.""" - sample_id = uuid.uuid4() - taxon_id = 172942 +class TestCreateFromIntent: + """Tests for create_from_intent — new signature with specimen sample IDs and manifest persistence.""" - # get_next_version queries Assembly + AssemblyRun - both return None + def test_creates_assembly_with_requested_status(self, mock_db, assembly_service): mock_query = Mock() mock_query.filter.return_value.scalar.return_value = None mock_db.query.return_value = mock_query @@ -349,27 +283,24 @@ def test_create_from_intent_creates_assembly_with_requested_status( with patch.object(Assembly, "__init__", return_value=None): assembly_service.create_from_intent( mock_db, - taxon_id=taxon_id, - sample_id=sample_id, + taxon_id=172942, + long_read_specimen_sample_id=LONG_READ_SAMPLE_ID, + hic_specimen_sample_id=None, data_types="PACBIO_SMRT", tol_id="tol-999", project_id=None, + manifest_json=None, ) - # Should add the Assembly and commit mock_db.add.assert_called_once() added_obj = mock_db.add.call_args[0][0] - # The added object should be an Assembly with status="requested" assert isinstance(added_obj, Assembly) mock_db.commit.assert_called_once() - def test_create_from_intent_version_is_next(self, mock_db, assembly_service): - """create_from_intent assigns the next version based on existing assemblies.""" - sample_id = uuid.uuid4() - + def test_version_is_next_based_on_long_read_sample(self, mock_db, assembly_service): + """Version is derived from (taxon_id, long_read_specimen_sample_id) only.""" mock_query = Mock() - # Assembly.version max = 2, AssemblyRun.version max = None → next version = 3 - mock_query.filter.return_value.scalar.side_effect = [2, None] + mock_query.filter.return_value.scalar.return_value = 2 # existing max = 2 → next = 3 mock_db.query.return_value = mock_query mock_db.add = Mock() mock_db.commit = Mock() @@ -377,8 +308,6 @@ def test_create_from_intent_version_is_next(self, mock_db, assembly_service): created_versions = [] - original_init = Assembly.__init__ - def capture_init(self, **kwargs): created_versions.append(kwargs.get("version")) @@ -386,10 +315,94 @@ def capture_init(self, **kwargs): assembly_service.create_from_intent( mock_db, taxon_id=172942, - sample_id=sample_id, + long_read_specimen_sample_id=LONG_READ_SAMPLE_ID, + hic_specimen_sample_id=None, data_types="PACBIO_SMRT", tol_id=None, project_id=None, ) assert created_versions == [3] + + def test_manifest_json_persisted(self, mock_db, assembly_service): + """manifest_json is stored on the Assembly object when provided.""" + mock_query = Mock() + mock_query.filter.return_value.scalar.return_value = None + mock_db.query.return_value = mock_query + mock_db.add = Mock() + mock_db.commit = Mock() + mock_db.refresh = Mock() + + sample_manifest = {"scientific_name": "Test", "reads": {"PACBIO_SMRT": {}}} + stored_manifests = [] + + def capture_init(self, **kwargs): + stored_manifests.append(kwargs.get("manifest_json")) + + with patch.object(Assembly, "__init__", capture_init): + assembly_service.create_from_intent( + mock_db, + taxon_id=172942, + long_read_specimen_sample_id=LONG_READ_SAMPLE_ID, + hic_specimen_sample_id=None, + data_types="PACBIO_SMRT", + tol_id=None, + project_id=None, + manifest_json=sample_manifest, + ) + + assert stored_manifests == [sample_manifest] + + def test_hic_specimen_sample_id_stored(self, mock_db, assembly_service): + """hic_specimen_sample_id is stored on the Assembly.""" + mock_query = Mock() + mock_query.filter.return_value.scalar.return_value = None + mock_db.query.return_value = mock_query + mock_db.add = Mock() + mock_db.commit = Mock() + mock_db.refresh = Mock() + + hic_ids = [] + + def capture_init(self, **kwargs): + hic_ids.append(kwargs.get("hic_specimen_sample_id")) + + with patch.object(Assembly, "__init__", capture_init): + assembly_service.create_from_intent( + mock_db, + taxon_id=172942, + long_read_specimen_sample_id=LONG_READ_SAMPLE_ID, + hic_specimen_sample_id=HIC_SAMPLE_ID, + data_types="PACBIO_SMRT_HIC", + tol_id=None, + project_id=None, + ) + + assert hic_ids == [HIC_SAMPLE_ID] + + def test_sample_id_set_to_long_read_specimen_sample_id(self, mock_db, assembly_service): + """sample_id is set to long_read_specimen_sample_id for backward compatibility.""" + mock_query = Mock() + mock_query.filter.return_value.scalar.return_value = None + mock_db.query.return_value = mock_query + mock_db.add = Mock() + mock_db.commit = Mock() + mock_db.refresh = Mock() + + sample_ids = [] + + def capture_init(self, **kwargs): + sample_ids.append(kwargs.get("sample_id")) + + with patch.object(Assembly, "__init__", capture_init): + assembly_service.create_from_intent( + mock_db, + taxon_id=172942, + long_read_specimen_sample_id=LONG_READ_SAMPLE_ID, + hic_specimen_sample_id=None, + data_types="PACBIO_SMRT", + tol_id=None, + project_id=None, + ) + + assert sample_ids == [LONG_READ_SAMPLE_ID] From e81716dc85755ac5275fc95ac0988a25d5a0a09e Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Fri, 8 May 2026 15:05:12 +1000 Subject: [PATCH 04/11] fix: update data types check, update tests, allow same specimen_id for hi-c + long read data --- app/api/v1/endpoints/assemblies.py | 89 +++-- app/services/assembly_helper.py | 23 +- .../endpoints/test_endpoints_assemblies.py | 326 ++++++++++-------- tests/unit/services/test_assembly_helper.py | 58 +++- 4 files changed, 281 insertions(+), 215 deletions(-) diff --git a/app/api/v1/endpoints/assemblies.py b/app/api/v1/endpoints/assemblies.py index 37357d2..d4f9b37 100644 --- a/app/api/v1/endpoints/assemblies.py +++ b/app/api/v1/endpoints/assemblies.py @@ -1,9 +1,8 @@ -import json from typing import Any, Dict, List, Optional from uuid import UUID from fastapi import APIRouter, Body, Depends, HTTPException, Query -from fastapi.responses import JSONResponse, Response +from fastapi.responses import JSONResponse from sqlalchemy.orm import Session from app.core.dependencies import get_current_active_user, get_db @@ -26,7 +25,6 @@ AssemblyFileUpdate, AssemblyIntent, AssemblyIntentCancel, - AssemblyIntentResponse, AssemblyStageRunCreate, AssemblyStageRunOut, AssemblyStageRunUpdate, @@ -254,7 +252,7 @@ def create_assembly_intent( taxon_id: int, intent_in: AssemblyIntent = Body(...), current_user: User = Depends(get_current_active_user), -) -> Response: +) -> Any: """ Create an assembly record using explicit specimen sample IDs and return its manifest as JSON. @@ -318,13 +316,13 @@ def create_assembly_intent( ) hic_experiments = [ e for e in hic_experiments - if (e.library_strategy or "").upper() in ("HI-C", "WGS") + if (e.library_strategy or "").upper() == "HI-C" ] if not hic_experiments: raise AppError( status_code=422, code="no_hic_experiments", - message="No Hi-C experiments (ILLUMINA + Hi-C/WGS) found for hic_specimen_sample_id", + message="No Hi-C experiments (ILLUMINA + Hi-C) found for hic_specimen_sample_id", details={"hic_specimen_sample_id": str(hic_sample.id)}, ) hic_exp_ids = [e.id for e in hic_experiments] @@ -354,49 +352,50 @@ def create_assembly_intent( manifest_json=None, # filled in step 9 ) - # 8. Build sample metadata and generate JSON manifest - all_reads = long_reads + hic_reads - sample_metadata_by_id = _build_sample_metadata_by_id(db, all_experiments) - manifest_data = generate_assembly_manifest_json( - organism=organism, - reads=all_reads, - experiments=all_experiments, - tol_id=assembly.tol_id, - version=assembly.version, - long_read_sample_id=long_read_sample.id, - hic_sample_id=hic_sample.id if hic_sample else None, - sample_metadata_by_id=sample_metadata_by_id, - ) - - # 9. Validate that the manifest contains eligible reads then persist it - long_read_keys = {"PACBIO_SMRT", "OXFORD_NANOPORE"} - if not any(k in manifest_data["reads"] for k in long_read_keys): - db.delete(assembly) - db.commit() - raise AppError( - status_code=422, - code="no_eligible_long_reads", - message=( - "No eligible long reads found for long_read_specimen_sample_id. " - "PacBio reads must end in .ccs.bam or hifi_reads.bam." - ), - details={"long_read_specimen_sample_id": str(long_read_sample.id)}, + try: + # 8. Build sample metadata and generate JSON manifest + all_reads = long_reads + hic_reads + sample_metadata_by_id = _build_sample_metadata_by_id(db, all_experiments) + manifest_data = generate_assembly_manifest_json( + organism=organism, + reads=all_reads, + experiments=all_experiments, + tol_id=assembly.tol_id, + version=assembly.version, + long_read_sample_id=long_read_sample.id, + hic_sample_id=hic_sample.id if hic_sample else None, + sample_metadata_by_id=sample_metadata_by_id, ) - if hic_sample and "Hi-C" not in manifest_data["reads"]: + # 9. Validate that the manifest contains eligible reads then persist it + long_read_keys = {"PACBIO_SMRT", "OXFORD_NANOPORE"} + if not any(k in manifest_data["reads"] for k in long_read_keys): + raise AppError( + status_code=422, + code="no_eligible_long_reads", + message=( + "No eligible long reads found for long_read_specimen_sample_id. " + "PacBio reads must end in .ccs.bam or hifi_reads.bam." + ), + details={"long_read_specimen_sample_id": str(long_read_sample.id)}, + ) + + if hic_sample and "Hi-C" not in manifest_data["reads"]: + raise AppError( + status_code=422, + code="no_eligible_hic_reads", + message="No eligible Hi-C reads found for hic_specimen_sample_id", + details={"hic_specimen_sample_id": str(hic_sample.id)}, + ) + + assembly.manifest_json = manifest_data + db.add(assembly) + db.commit() + db.refresh(assembly) + except Exception: db.delete(assembly) db.commit() - raise AppError( - status_code=422, - code="no_eligible_hic_reads", - message="No eligible Hi-C reads found for hic_specimen_sample_id", - details={"hic_specimen_sample_id": str(hic_sample.id)}, - ) - - assembly.manifest_json = manifest_data - db.add(assembly) - db.commit() - db.refresh(assembly) + raise return JSONResponse( content={ diff --git a/app/services/assembly_helper.py b/app/services/assembly_helper.py index d77b799..762a7d9 100644 --- a/app/services/assembly_helper.py +++ b/app/services/assembly_helper.py @@ -18,7 +18,7 @@ def determine_assembly_data_types(experiments: List[Experiment]) -> AssemblyData Rules: - PACBIO_SMRT exists if: platform == "PACBIO_SMRT" - OXFORD_NANOPORE exists if: platform == "OXFORD_NANOPORE" - - Hi-C exists if: platform == "ILLUMINA" AND library_strategy in ("Hi-C", "WGS") + - Hi-C exists if: platform == "ILLUMINA" AND library_strategy == "Hi-C" Raises: ValueError: If no valid sequencing platforms are detected @@ -35,7 +35,7 @@ def determine_assembly_data_types(experiments: List[Experiment]) -> AssemblyData has_pacbio = True if platform == "OXFORD_NANOPORE" and library_strategy in ("WGS", "WGA"): has_nanopore = True - if platform == "ILLUMINA" and library_strategy in ("HI-C", "WGS"): + if platform == "ILLUMINA" and library_strategy == "HI-C": has_hic = True if has_pacbio and has_nanopore and has_hic: @@ -162,8 +162,12 @@ def generate_assembly_manifest_json( sample_meta = (sample_metadata_by_id or {}).get(sample_id, {}) if sample_id else {} # Route by specimen sample, then by platform - if sample_id == long_read_sample_str: - if platform == "PACBIO_SMRT" and read.file_name: + is_long_read_sample = sample_id == long_read_sample_str + is_hic_sample = hic_sample_str is not None and sample_id == hic_sample_str + + if is_long_read_sample: + #TODO check logic + if platform == "PACBIO_SMRT" and library_strategy in ("WGS", "WGA") and read.file_name: if read.file_name.endswith(".ccs.bam") or read.file_name.endswith("hifi_reads.bam"): logger.info("Adding PacBio read: %s", read.file_name) if bpa_package_id not in pacbio_by_package: @@ -184,8 +188,8 @@ def generate_assembly_manifest_json( "Skipping PacBio read %s — not .ccs.bam or hifi_reads.bam", read.file_name, ) - - elif platform == "OXFORD_NANOPORE": + # TODO check logic + elif platform == "OXFORD_NANOPORE" and library_strategy in ("WGS", "WGA"): logger.info("Adding ONT read: %s", read.file_name) if bpa_package_id not in ont_by_package: entry = { @@ -208,8 +212,8 @@ def generate_assembly_manifest_json( platform, ) - elif hic_sample_str and sample_id == hic_sample_str: - if platform == "ILLUMINA" and library_strategy in ("HI-C", "WGS"): + if is_hic_sample: + if platform == "ILLUMINA" and library_strategy == "HI-C": logger.info( "Adding Hi-C read: %s (library_strategy=%s)", read.file_name, library_strategy ) @@ -242,8 +246,7 @@ def generate_assembly_manifest_json( platform, library_strategy, ) - - else: + elif not is_long_read_sample: logger.debug( "Read %s sample_id=%s does not match either specimen sample, skipping", read.id, diff --git a/tests/unit/endpoints/test_endpoints_assemblies.py b/tests/unit/endpoints/test_endpoints_assemblies.py index 63efcb2..fadc8ba 100644 --- a/tests/unit/endpoints/test_endpoints_assemblies.py +++ b/tests/unit/endpoints/test_endpoints_assemblies.py @@ -182,45 +182,24 @@ def mock_create_raises(*args, **kwargs): def test_get_assembly_manifest_success(monkeypatch): - """Test successful manifest generation.""" + """GET /manifest/{taxon_id} returns stored manifest JSON for latest requested assembly.""" client = TestClient(app) - # Mock database queries - organism = SimpleNamespace( - scientific_name="Test Species", - taxon_id=172942, - ) - sample = SimpleNamespace( - id="550e8400-e29b-41d4-a716-446655440000", - taxon_id=172942, - kind="specimen", - bpa_sample_id="102.100.100/9000", - specimen_id="SPEC-001", - ) - experiment = SimpleNamespace( - id="exp-1", - sample_id="550e8400-e29b-41d4-a716-446655440000", - platform="PACBIO_SMRT", - library_strategy="WGS", - bpa_package_id="pkg-exp-1", - bioplatforms_base_url=None, - ) - read = SimpleNamespace( - id="read-1", - experiment_id="exp-1", - file_name="sample.ccs.bam", - file_checksum="abc123", - bioplatforms_url="https://example.com/1", - read_number=None, - lane_number=None, - ) + organism = SimpleNamespace(scientific_name="Test Species", taxon_id=172942) + manifest_json = { + "scientific_name": "Test Species", + "taxon_id": 172942, + "tolid": "tol-123", + "version": 1, + "reads": {"PACBIO_SMRT": {"pkg-exp-1": {"resources": []}}}, + } requested_assembly = SimpleNamespace( id="run-1", taxon_id=172942, - sample_id="550e8400-e29b-41d4-a716-446655440000", tol_id="tol-123", version=1, status="requested", + manifest_json=manifest_json, ) class MockQuery: @@ -245,22 +224,10 @@ def __init__(self): def query(self, model): self.call_count += 1 - if self.call_count == 1: # organism query + if self.call_count == 1: return MockQuery(organism) - elif self.call_count == 2: # selected specimen sample query - return MockQuery(sample) - elif self.call_count == 3: # selected sample by ID query - return MockQuery(sample) - elif self.call_count == 4: # all sample IDs query - return MockQuery([(sample.id,)]) - elif self.call_count == 5: # experiments query - return MockQuery([experiment]) - elif self.call_count == 6: # reads query - return MockQuery([read]) - elif self.call_count == 7: # latest requested assembly query + if self.call_count == 2: return MockQuery(requested_assembly) - elif self.call_count == 8: # sample metadata query for per-read manifest fields - return MockQuery([sample]) return MockQuery([]) app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( @@ -271,13 +238,8 @@ def query(self, model): resp = client.get("/api/v1/assemblies/manifest/172942") assert resp.status_code == 200 - assert resp.headers["content-type"] == "application/x-yaml" - assert b"scientific_name: Test Species" in resp.content - assert b"taxon_id: 172942" in resp.content - assert b"sample_id: 550e8400-e29b-41d4-a716-446655440000" in resp.content - assert b"bpa_sample_id: 102.100.100/9000" in resp.content - assert b"specimen_id: SPEC-001" in resp.content - assert b"PACBIO_SMRT:" in resp.content + assert resp.headers["content-type"].startswith("application/json") + assert resp.json() == manifest_json def test_get_assembly_manifest_organism_not_found(): @@ -311,33 +273,55 @@ def first(self): def test_create_assembly_intent_invalid_data_types_returns_app_error(monkeypatch): client = TestClient(app) - organism = SimpleNamespace( - scientific_name="Test Species", - taxon_id=172942, + organism = SimpleNamespace(scientific_name="Test Species", taxon_id=172942) + long_read_sample_id = uuid4() + long_read_sample = SimpleNamespace(id=long_read_sample_id, taxon_id=172942, kind="specimen") + invalid_long_read_exp = SimpleNamespace( + id="e1", + sample_id=long_read_sample_id, + platform="PACBIO_SMRT", + library_strategy="RNA", ) - reads = [SimpleNamespace(id="r1", experiment_id="e1")] - experiments = [SimpleNamespace(id="e1", platform="UNKNOWN", library_strategy="UNKNOWN")] - monkeypatch.setattr( - assemblies, - "_get_manifest_inputs_by_taxon_id", - lambda db, taxon_id: ( - organism, - SimpleNamespace(id="550e8400-e29b-41d4-a716-446655440000"), - reads, - experiments, - ), - ) + class _Q: + def __init__(self, value): + self.value = value + + def filter(self, *_a, **_k): + return self + + def all(self): + return self.value if isinstance(self.value, list) else [] + + def first(self): + return self.value if not isinstance(self.value, list) else None + + class _DB: + def __init__(self): + self.calls = 0 + + def query(self, _model): + self.calls += 1 + if self.calls == 1: + return _Q(organism) + if self.calls == 2: + return _Q(long_read_sample) + if self.calls == 3: + return _Q([invalid_long_read_exp]) + if self.calls == 4: + return _Q([]) + return _Q([]) app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( is_active=True, roles=["curator"], is_superuser=False ) - app.dependency_overrides[assemblies.get_db] = _override_db(_FakeSession()) + app.dependency_overrides[assemblies.get_db] = _override_db(_DB()) resp = client.post( "/api/v1/assemblies/intent/172942", json={ "tol_id": "tol-123", + "long_read_specimen_sample_id": str(long_read_sample_id), }, ) @@ -347,14 +331,75 @@ def test_create_assembly_intent_invalid_data_types_returns_app_error(monkeypatch assert "No valid data types detected in experiments" in body["error"]["message"] -def test_create_assembly_intent_allows_empty_body(monkeypatch): +def test_create_assembly_intent_requires_specimen_sample_id(): + client = TestClient(app) + + app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["curator"], is_superuser=False + ) + app.dependency_overrides[assemblies.get_db] = _override_db(_FakeSession()) + + resp = client.post( + "/api/v1/assemblies/intent/172942", + json={"tol_id": "tol-123"}, + ) + + assert resp.status_code == 422 + + +def test_create_assembly_intent_rejects_non_specimen_sample(): client = TestClient(app) - organism = SimpleNamespace( - scientific_name="Test Species", + organism = SimpleNamespace(scientific_name="Test Species", taxon_id=172942) + derived_sample_id = uuid4() + derived_sample = SimpleNamespace(id=derived_sample_id, taxon_id=172942, kind="derived") + + class _Q: + def __init__(self, value): + self.value = value + + def filter(self, *_a, **_k): + return self + + def first(self): + return self.value + + class _DB: + def __init__(self): + self.calls = 0 + + def query(self, _model): + self.calls += 1 + if self.calls == 1: + return _Q(organism) + return _Q(derived_sample) + + app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["curator"], is_superuser=False + ) + app.dependency_overrides[assemblies.get_db] = _override_db(_DB()) + + resp = client.post( + "/api/v1/assemblies/intent/172942", + json={"long_read_specimen_sample_id": str(derived_sample_id)}, + ) + + assert resp.status_code == 422 + assert resp.json()["error"]["code"] == "specimen_sample_invalid_kind" + + +def test_create_assembly_intent_success_returns_json_manifest(monkeypatch): + client = TestClient(app) + + organism = SimpleNamespace(scientific_name="Test Species", taxon_id=172942) + long_read_sample_id = uuid4() + selected_sample = SimpleNamespace( + id=long_read_sample_id, taxon_id=172942, + kind="specimen", + bpa_sample_id="102.100.100/9000", + specimen_id="SPEC-001", ) - selected_sample = SimpleNamespace(id="550e8400-e29b-41d4-a716-446655440000") run_id = uuid4() reads = [ SimpleNamespace( @@ -370,7 +415,7 @@ def test_create_assembly_intent_allows_empty_body(monkeypatch): experiments = [ SimpleNamespace( id="e1", - sample_id=selected_sample.id, + sample_id=long_read_sample_id, platform="PACBIO_SMRT", library_strategy="WGS", bpa_package_id="pkg-e1", @@ -378,16 +423,12 @@ def test_create_assembly_intent_allows_empty_body(monkeypatch): ) ] - monkeypatch.setattr( - assemblies, - "_get_manifest_inputs_by_taxon_id", - lambda db, taxon_id: (organism, selected_sample, reads, experiments), - ) mock_assembly = SimpleNamespace( id=run_id, version=1, status="requested", tol_id=None, + manifest_json=None, ) monkeypatch.setattr( assemblies.assembly_service, @@ -396,33 +437,61 @@ def test_create_assembly_intent_allows_empty_body(monkeypatch): ) class _FakeIntentDB: + def __init__(self): + self.calls = 0 + def query(self, _model): + self.calls += 1 + class _Q: + def __init__(self, value): + self.value = value + def filter(self, *_a, **_k): return self def all(self): - return [ - SimpleNamespace( - id=selected_sample.id, - bpa_sample_id="102.100.100/9000", - specimen_id="SPEC-001", - ) - ] + return self.value if isinstance(self.value, list) else [] - return _Q() + def first(self): + return self.value if not isinstance(self.value, list) else None + + if self.calls == 1: + return _Q(organism) + if self.calls == 2: + return _Q(selected_sample) + if self.calls == 3: + return _Q(experiments) + if self.calls == 4: + return _Q(reads) + if self.calls == 5: + return _Q([selected_sample]) + return _Q([]) + + def add(self, _obj): + return None + + def commit(self): + return None + + def refresh(self, _obj): + return None + + def delete(self, _obj): + return None app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( is_active=True, roles=["curator"], is_superuser=False ) app.dependency_overrides[assemblies.get_db] = _override_db(_FakeIntentDB()) - resp = client.post("/api/v1/assemblies/intent/172942") + resp = client.post( + "/api/v1/assemblies/intent/172942", + json={"long_read_specimen_sample_id": str(long_read_sample_id)}, + ) assert resp.status_code == 200 - import yaml as _yaml - - body = _yaml.safe_load(resp.content) + body = resp.json() assert body["assembly_id"] == str(run_id) assert body["version"] == 1 assert body["status"] == "requested" @@ -433,22 +502,17 @@ def test_cancel_assembly_intent_success(monkeypatch): client = TestClient(app) organism = SimpleNamespace(taxon_id=172942) - selected_sample_id = "550e8400-e29b-41d4-a716-446655440000" + long_read_sample_id = uuid4() run_id = uuid4() run = SimpleNamespace( id=run_id, taxon_id=172942, - sample_id=selected_sample_id, + long_read_specimen_sample_id=long_read_sample_id, + hic_specimen_sample_id=None, version=2, status="requested", ) - monkeypatch.setattr( - assemblies, - "_get_optimal_sample_id_for_taxon_id", - lambda db, taxon_id, organism_taxon_id=None: selected_sample_id, - ) - class _Q: def __init__(self, value): self.value = value @@ -501,13 +565,6 @@ def test_cancel_assembly_intent_not_found(monkeypatch): client = TestClient(app) organism = SimpleNamespace(taxon_id=172942) - selected_sample_id = "550e8400-e29b-41d4-a716-446655440000" - - monkeypatch.setattr( - assemblies, - "_get_optimal_sample_id_for_taxon_id", - lambda db, taxon_id, organism_taxon_id=None: selected_sample_id, - ) class _Q: def __init__(self, value): @@ -550,43 +607,26 @@ def query(self, _model): def test_get_manifest_by_assembly_id_success(monkeypatch): - """GET /{assembly_id}/manifest returns YAML for a known assembly.""" + """GET /{assembly_id}/manifest returns stored manifest JSON for a known assembly.""" from uuid import uuid4 as _uuid4 client = TestClient(app) assembly_id = _uuid4() + manifest_json = { + "scientific_name": "Test Species", + "taxon_id": 172942, + "tolid": "tol-999", + "version": 3, + "reads": {"PACBIO_SMRT": {"pkg-exp-1": {"resources": []}}}, + } assembly = SimpleNamespace( id=assembly_id, taxon_id=172942, - sample_id="550e8400-e29b-41d4-a716-446655440000", tol_id="tol-999", version=3, status="requested", - ) - organism = SimpleNamespace(scientific_name="Test Species", taxon_id=172942) - sample = SimpleNamespace( - id="550e8400-e29b-41d4-a716-446655440000", - taxon_id=172942, - bpa_sample_id="102.100.100/9000", - specimen_id="SPEC-001", - ) - experiment = SimpleNamespace( - id="exp-1", - sample_id="550e8400-e29b-41d4-a716-446655440000", - platform="PACBIO_SMRT", - library_strategy="WGS", - bpa_package_id="pkg-exp-1", - bioplatforms_base_url=None, - ) - read = SimpleNamespace( - id="read-1", - experiment_id="exp-1", - file_name="sample.ccs.bam", - file_checksum="abc123", - bioplatforms_url="https://example.com/1", - read_number=None, - lane_number=None, + manifest_json=manifest_json, ) class MockQuery: @@ -603,24 +643,8 @@ def first(self): return self.rv if not isinstance(self.rv, list) else None class MockDB: - def __init__(self): - self.n = 0 - def query(self, _m): - self.n += 1 - if self.n == 1: - return MockQuery(assembly) - if self.n == 2: - return MockQuery(organism) - if self.n == 3: - return MockQuery([(sample.id,)]) - if self.n == 4: - return MockQuery([experiment]) - if self.n == 5: - return MockQuery([read]) - if self.n == 6: - return MockQuery([sample]) - return MockQuery([]) + return MockQuery(assembly) app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( is_active=True, roles=["admin"], is_superuser=False @@ -630,10 +654,8 @@ def query(self, _m): resp = client.get(f"/api/v1/assemblies/{assembly_id}/manifest") assert resp.status_code == 200 - assert resp.headers["content-type"] == "application/x-yaml" - assert b"scientific_name: Test Species" in resp.content - assert b"taxon_id: 172942" in resp.content - assert b"PACBIO_SMRT:" in resp.content + assert resp.headers["content-type"].startswith("application/json") + assert resp.json() == manifest_json def test_get_manifest_by_assembly_id_not_found(): diff --git a/tests/unit/services/test_assembly_helper.py b/tests/unit/services/test_assembly_helper.py index 76bc664..08de88b 100644 --- a/tests/unit/services/test_assembly_helper.py +++ b/tests/unit/services/test_assembly_helper.py @@ -37,12 +37,12 @@ def test_illumina_hic_alone_raises_error(self): with pytest.raises(ValueError, match="No valid data types detected in experiments"): determine_assembly_data_types(experiments) - def test_illumina_wgs_treated_as_hic(self): + def test_illumina_wgs_not_treated_as_hic(self): experiments = [ Mock(platform="PACBIO_SMRT", library_strategy="WGS"), Mock(platform="ILLUMINA", library_strategy="WGS"), ] - assert determine_assembly_data_types(experiments) == AssemblyDataTypes.PACBIO_SMRT_HIC + assert determine_assembly_data_types(experiments) == AssemblyDataTypes.PACBIO_SMRT def test_pacbio_and_hic(self): experiments = [ @@ -54,7 +54,7 @@ def test_pacbio_and_hic(self): def test_nanopore_and_hic(self): experiments = [ Mock(platform="OXFORD_NANOPORE", library_strategy="WGS"), - Mock(platform="ILLUMINA", library_strategy="WGS"), + Mock(platform="ILLUMINA", library_strategy="Hi-C"), ] assert determine_assembly_data_types(experiments) == AssemblyDataTypes.OXFORD_NANOPORE_HIC @@ -86,7 +86,7 @@ def test_case_insensitive_platform(self): def test_case_insensitive_library_strategy(self): experiments = [ Mock(platform="OXFORD_NANOPORE", library_strategy="WGS"), - Mock(platform="ILLUMINA", library_strategy="wgs"), + Mock(platform="ILLUMINA", library_strategy="hi-c"), ] assert determine_assembly_data_types(experiments) == AssemblyDataTypes.OXFORD_NANOPORE_HIC @@ -359,8 +359,8 @@ def test_hic_reads_split_into_r1_r2(self): assert resources["r1"][0]["url"] == "https://example.com/r1" assert resources["r2"][0]["url"] == "https://example.com/r2" - def test_wgs_treated_as_hic(self): - """ILLUMINA + WGS is treated as Hi-C.""" + def test_wgs_not_treated_as_hic(self): + """ILLUMINA + WGS does not populate the Hi-C section.""" organism = Mock(scientific_name="Test Species", taxon_id=12345) experiments = [ Mock( @@ -388,8 +388,7 @@ def test_wgs_treated_as_hic(self): organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID, HIC_SAMPLE_ID ) - assert "Hi-C" in result["reads"] - assert "pkg-wgs" in result["reads"]["Hi-C"] + assert "Hi-C" not in result["reads"] def test_hic_section_omitted_when_no_hic_sample_id(self): """Hi-C section is absent when hic_sample_id is not supplied.""" @@ -499,6 +498,49 @@ def test_multiple_platform_types(self): assert "pkg-pacbio" in result["reads"]["PACBIO_SMRT"] assert "pkg-hic" in result["reads"]["Hi-C"] + def test_same_specimen_sample_can_supply_long_reads_and_hic(self): + """When the same specimen sample ID is used for both roles, both sections are populated.""" + organism = Mock(scientific_name="Test Species", taxon_id=12345) + experiments = [ + _make_pacbio_experiment( + exp_id="exp-pb", + bpa_package_id="pkg-pacbio", + sample_id=LONG_READ_SAMPLE_STR, + ), + _make_hic_experiment( + exp_id="exp-hic", + bpa_package_id="pkg-hic", + sample_id=LONG_READ_SAMPLE_STR, + ), + ] + reads = [ + Mock( + id="r1", + experiment_id="exp-pb", + file_name="sample.ccs.bam", + file_checksum="abc123", + bioplatforms_url="https://example.com/pb", + read_number=None, + lane_number=None, + ), + Mock( + id="r2", + experiment_id="exp-hic", + file_name="hic_R1.fastq.gz", + file_checksum="def456", + bioplatforms_url="https://example.com/hic", + read_number="1", + lane_number="001", + ), + ] + + result = generate_assembly_manifest_json( + organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID, LONG_READ_SAMPLE_ID + ) + + assert "PACBIO_SMRT" in result["reads"] + assert "Hi-C" in result["reads"] + def test_pacbio_and_ont_in_same_long_read_sample(self): """Both PacBio and ONT reads from the same long-read specimen appear in separate sections.""" organism = Mock(scientific_name="Test Species", taxon_id=12345) From 9f87c13e229bac49a25737b6c42aee1d6acde8f0 Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Fri, 8 May 2026 18:37:35 +1000 Subject: [PATCH 05/11] fix: update data types check, update tests, allow same specimen_id for hi-c + long read data --- app/api/v1/endpoints/assemblies.py | 45 ++++++- app/services/assembly_helper.py | 25 ++-- .../endpoints/test_endpoints_assemblies.py | 127 +++++++++++++++++- tests/unit/services/test_assembly_helper.py | 46 +++++++ 4 files changed, 227 insertions(+), 16 deletions(-) diff --git a/app/api/v1/endpoints/assemblies.py b/app/api/v1/endpoints/assemblies.py index d4f9b37..9b7e41e 100644 --- a/app/api/v1/endpoints/assemblies.py +++ b/app/api/v1/endpoints/assemblies.py @@ -76,6 +76,17 @@ def _build_sample_metadata_by_id( } +def _build_specimen_metadata(*samples: Sample) -> Dict[str, Dict[str, Any]]: + return { + str(sample.id): { + "bpa_sample_id": getattr(sample, "bpa_sample_id", None), + "specimen_id": getattr(sample, "specimen_id", None), + } + for sample in samples + if sample is not None + } + + def _validate_specimen_sample( db: Session, sample_id: UUID, taxon_id: int, field_name: str ) -> Sample: @@ -110,6 +121,20 @@ def _validate_specimen_sample( return sample +def _get_lineage_sample_ids_for_specimen(db: Session, specimen_sample: Sample) -> List[UUID]: + """Return specimen sample id plus any derived samples linked from it.""" + derived_samples = ( + db.query(Sample) + .filter( + Sample.derived_from_sample_id == specimen_sample.id, + Sample.taxon_id == specimen_sample.taxon_id, + Sample.kind == "derived", + ) + .all() + ) + return [specimen_sample.id] + [sample.id for sample in derived_samples] + + @router.get("/pipeline-inputs") def get_pipeline_inputs( *, @@ -283,10 +308,14 @@ def create_assembly_intent( ) # 4. Fetch long-read experiments (PacBio or ONT only) and their reads + long_read_lineage_sample_ids = _get_lineage_sample_ids_for_specimen(db, long_read_sample) + long_read_sample_id_map = { + str(sample_id): str(long_read_sample.id) for sample_id in long_read_lineage_sample_ids + } long_read_experiments = ( db.query(Experiment) .filter( - Experiment.sample_id == long_read_sample.id, + Experiment.sample_id.in_(long_read_lineage_sample_ids), Experiment.platform.in_(["PACBIO_SMRT", "OXFORD_NANOPORE"]), ) .all() @@ -305,11 +334,16 @@ def create_assembly_intent( # 5. Fetch Hi-C experiments and reads if hic_sample is provided hic_experiments: List[Experiment] = [] hic_reads: List[Read] = [] + hic_sample_id_map: Dict[str, str] = {} if hic_sample: + hic_lineage_sample_ids = _get_lineage_sample_ids_for_specimen(db, hic_sample) + hic_sample_id_map = { + str(sample_id): str(hic_sample.id) for sample_id in hic_lineage_sample_ids + } hic_experiments = ( db.query(Experiment) .filter( - Experiment.sample_id == hic_sample.id, + Experiment.sample_id.in_(hic_lineage_sample_ids), Experiment.platform == "ILLUMINA", ) .all() @@ -355,7 +389,11 @@ def create_assembly_intent( try: # 8. Build sample metadata and generate JSON manifest all_reads = long_reads + hic_reads - sample_metadata_by_id = _build_sample_metadata_by_id(db, all_experiments) + sample_metadata_by_id = _build_specimen_metadata(long_read_sample, hic_sample) + sequencing_sample_to_specimen_sample_id = { + **long_read_sample_id_map, + **hic_sample_id_map, + } manifest_data = generate_assembly_manifest_json( organism=organism, reads=all_reads, @@ -365,6 +403,7 @@ def create_assembly_intent( long_read_sample_id=long_read_sample.id, hic_sample_id=hic_sample.id if hic_sample else None, sample_metadata_by_id=sample_metadata_by_id, + sequencing_sample_to_specimen_sample_id=sequencing_sample_to_specimen_sample_id, ) # 9. Validate that the manifest contains eligible reads then persist it diff --git a/app/services/assembly_helper.py b/app/services/assembly_helper.py index 762a7d9..9983b4e 100644 --- a/app/services/assembly_helper.py +++ b/app/services/assembly_helper.py @@ -96,6 +96,7 @@ def generate_assembly_manifest_json( long_read_sample_id: UUID, hic_sample_id: Optional[UUID] = None, sample_metadata_by_id: Dict[str, Dict[str, Any]] | None = None, + sequencing_sample_to_specimen_sample_id: Dict[str, str] | None = None, ) -> Dict[str, Any]: """Generate an assembly manifest as a JSON-serialisable dict. @@ -159,20 +160,26 @@ def generate_assembly_manifest_json( library_strategy = exp_info["library_strategy"] bpa_package_id = exp_info["bpa_package_id"] sample_id = exp_info["sample_id"] - sample_meta = (sample_metadata_by_id or {}).get(sample_id, {}) if sample_id else {} + resolved_sample_id = ( + sequencing_sample_to_specimen_sample_id.get(sample_id, sample_id) + if sample_id and sequencing_sample_to_specimen_sample_id + else sample_id + ) + sample_meta = ( + (sample_metadata_by_id or {}).get(resolved_sample_id, {}) if resolved_sample_id else {} + ) # Route by specimen sample, then by platform - is_long_read_sample = sample_id == long_read_sample_str - is_hic_sample = hic_sample_str is not None and sample_id == hic_sample_str + is_long_read_sample = resolved_sample_id == long_read_sample_str + is_hic_sample = hic_sample_str is not None and resolved_sample_id == hic_sample_str if is_long_read_sample: - #TODO check logic if platform == "PACBIO_SMRT" and library_strategy in ("WGS", "WGA") and read.file_name: if read.file_name.endswith(".ccs.bam") or read.file_name.endswith("hifi_reads.bam"): logger.info("Adding PacBio read: %s", read.file_name) if bpa_package_id not in pacbio_by_package: entry: Dict[str, Any] = { - "sample_id": sample_id, + "sample_id": resolved_sample_id, "bpa_sample_id": sample_meta.get("bpa_sample_id"), "specimen_id": sample_meta.get("specimen_id"), "resources": [], @@ -188,12 +195,11 @@ def generate_assembly_manifest_json( "Skipping PacBio read %s — not .ccs.bam or hifi_reads.bam", read.file_name, ) - # TODO check logic elif platform == "OXFORD_NANOPORE" and library_strategy in ("WGS", "WGA"): logger.info("Adding ONT read: %s", read.file_name) if bpa_package_id not in ont_by_package: entry = { - "sample_id": sample_id, + "sample_id": resolved_sample_id, "bpa_sample_id": sample_meta.get("bpa_sample_id"), "specimen_id": sample_meta.get("specimen_id"), "resources": [], @@ -219,7 +225,7 @@ def generate_assembly_manifest_json( ) if bpa_package_id not in hic_by_package: hic_by_package[bpa_package_id] = { - "sample_id": sample_id, + "sample_id": resolved_sample_id, "bpa_sample_id": sample_meta.get("bpa_sample_id"), "specimen_id": sample_meta.get("specimen_id"), "resources": {"r1": [], "r2": []}, @@ -248,9 +254,10 @@ def generate_assembly_manifest_json( ) elif not is_long_read_sample: logger.debug( - "Read %s sample_id=%s does not match either specimen sample, skipping", + "Read %s sample_id=%s resolved_sample_id=%s does not match either specimen sample, skipping", read.id, sample_id, + resolved_sample_id, ) reads_section: Dict[str, Any] = {} diff --git a/tests/unit/endpoints/test_endpoints_assemblies.py b/tests/unit/endpoints/test_endpoints_assemblies.py index fadc8ba..1a2bfdb 100644 --- a/tests/unit/endpoints/test_endpoints_assemblies.py +++ b/tests/unit/endpoints/test_endpoints_assemblies.py @@ -307,8 +307,10 @@ def query(self, _model): if self.calls == 2: return _Q(long_read_sample) if self.calls == 3: - return _Q([invalid_long_read_exp]) + return _Q([]) if self.calls == 4: + return _Q([invalid_long_read_exp]) + if self.calls == 5: return _Q([]) return _Q([]) @@ -461,11 +463,11 @@ def first(self): if self.calls == 2: return _Q(selected_sample) if self.calls == 3: - return _Q(experiments) + return _Q([]) if self.calls == 4: - return _Q(reads) + return _Q(experiments) if self.calls == 5: - return _Q([selected_sample]) + return _Q(reads) return _Q([]) def add(self, _obj): @@ -498,6 +500,123 @@ def delete(self, _obj): assert "manifest" in body +def test_create_assembly_intent_resolves_reads_via_derived_samples(monkeypatch): + client = TestClient(app) + + organism = SimpleNamespace(scientific_name="Test Species", taxon_id=172942) + long_read_sample_id = uuid4() + derived_sample_id = uuid4() + specimen_sample = SimpleNamespace( + id=long_read_sample_id, + taxon_id=172942, + kind="specimen", + bpa_sample_id="102.100.100/9000", + specimen_id="SPEC-001", + ) + derived_sample = SimpleNamespace( + id=derived_sample_id, + taxon_id=172942, + kind="derived", + derived_from_sample_id=long_read_sample_id, + ) + run_id = uuid4() + reads = [ + SimpleNamespace( + id="r1", + experiment_id="e1", + file_name="sample.ccs.bam", + file_checksum="abc123", + bioplatforms_url="https://example.com/1", + read_number=None, + lane_number=None, + ) + ] + experiments = [ + SimpleNamespace( + id="e1", + sample_id=derived_sample_id, + platform="PACBIO_SMRT", + library_strategy="WGS", + bpa_package_id="pkg-e1", + bioplatforms_base_url=None, + ) + ] + + mock_assembly = SimpleNamespace( + id=run_id, + version=1, + status="requested", + tol_id=None, + manifest_json=None, + ) + monkeypatch.setattr( + assemblies.assembly_service, + "create_from_intent", + lambda db, **kwargs: mock_assembly, + ) + + class _FakeIntentDB: + def __init__(self): + self.calls = 0 + + def query(self, _model): + self.calls += 1 + + class _Q: + def __init__(self, value): + self.value = value + + def filter(self, *_a, **_k): + return self + + def all(self): + return self.value if isinstance(self.value, list) else [] + + def first(self): + return self.value if not isinstance(self.value, list) else None + + if self.calls == 1: + return _Q(organism) + if self.calls == 2: + return _Q(specimen_sample) + if self.calls == 3: + return _Q([derived_sample]) + if self.calls == 4: + return _Q(experiments) + if self.calls == 5: + return _Q(reads) + return _Q([]) + + def add(self, _obj): + return None + + def commit(self): + return None + + def refresh(self, _obj): + return None + + def delete(self, _obj): + return None + + app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["curator"], is_superuser=False + ) + app.dependency_overrides[assemblies.get_db] = _override_db(_FakeIntentDB()) + + resp = client.post( + "/api/v1/assemblies/intent/172942", + json={"long_read_specimen_sample_id": str(long_read_sample_id)}, + ) + + assert resp.status_code == 200 + body = resp.json() + assert body["manifest"]["reads"]["PACBIO_SMRT"]["pkg-e1"]["sample_id"] == str( + long_read_sample_id + ) + assert body["manifest"]["reads"]["PACBIO_SMRT"]["pkg-e1"]["specimen_id"] == "SPEC-001" + + def test_cancel_assembly_intent_success(monkeypatch): client = TestClient(app) diff --git a/tests/unit/services/test_assembly_helper.py b/tests/unit/services/test_assembly_helper.py index 08de88b..2500e4c 100644 --- a/tests/unit/services/test_assembly_helper.py +++ b/tests/unit/services/test_assembly_helper.py @@ -541,6 +541,52 @@ def test_same_specimen_sample_can_supply_long_reads_and_hic(self): assert "PACBIO_SMRT" in result["reads"] assert "Hi-C" in result["reads"] + def test_derived_sample_reads_are_attributed_to_parent_specimen(self): + """Reads from derived sequencing samples are routed to the selected parent specimen.""" + organism = Mock(scientific_name="Test Species", taxon_id=12345) + derived_sample_id = "cccccccc-cccc-cccc-cccc-cccccccccccc" + experiments = [ + _make_pacbio_experiment( + exp_id="exp-derived", + bpa_package_id="pkg-derived", + sample_id=derived_sample_id, + ), + ] + reads = [ + Mock( + id="r1", + experiment_id="exp-derived", + file_name="sample.ccs.bam", + file_checksum="abc123", + bioplatforms_url="https://example.com/pb", + read_number=None, + lane_number=None, + ), + ] + sample_metadata_by_id = { + LONG_READ_SAMPLE_STR: { + "bpa_sample_id": "102.100.100/9000", + "specimen_id": "SPEC-001", + } + } + + result = generate_assembly_manifest_json( + organism, + reads, + experiments, + "tol1", + 1, + LONG_READ_SAMPLE_ID, + sample_metadata_by_id=sample_metadata_by_id, + sequencing_sample_to_specimen_sample_id={ + derived_sample_id: LONG_READ_SAMPLE_STR, + }, + ) + + pkg = result["reads"]["PACBIO_SMRT"]["pkg-derived"] + assert pkg["sample_id"] == LONG_READ_SAMPLE_STR + assert pkg["specimen_id"] == "SPEC-001" + def test_pacbio_and_ont_in_same_long_read_sample(self): """Both PacBio and ONT reads from the same long-read specimen appear in separate sections.""" organism = Mock(scientific_name="Test Species", taxon_id=12345) From 01f0766aa6cf4012c66d0ed048e15f4ebd461dce Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Fri, 8 May 2026 18:42:49 +1000 Subject: [PATCH 06/11] feat: new data --- app/api/v1/endpoints/assemblies.py | 41 +++++ app/schemas/assembly.py | 24 +++ app/services/assembly_helper.py | 51 ++++-- .../endpoints/test_endpoints_assemblies.py | 154 ++++++++++++++++++ tests/unit/services/test_assembly_helper.py | 25 +++ 5 files changed, 283 insertions(+), 12 deletions(-) diff --git a/app/api/v1/endpoints/assemblies.py b/app/api/v1/endpoints/assemblies.py index 9b7e41e..6e6d28b 100644 --- a/app/api/v1/endpoints/assemblies.py +++ b/app/api/v1/endpoints/assemblies.py @@ -25,6 +25,7 @@ AssemblyFileUpdate, AssemblyIntent, AssemblyIntentCancel, + AssemblySpecimenSampleDiscoveryResponse, AssemblyStageRunCreate, AssemblyStageRunOut, AssemblyStageRunUpdate, @@ -42,6 +43,7 @@ from app.services.assembly_helper import ( determine_assembly_data_types, generate_assembly_manifest_json, + get_available_assembly_data_types, ) from app.services.assembly_service import ( assembly_file_service, @@ -135,6 +137,45 @@ def _get_lineage_sample_ids_for_specimen(db: Session, specimen_sample: Sample) - return [specimen_sample.id] + [sample.id for sample in derived_samples] +@router.get("/specimen-samples/{taxon_id}", response_model=AssemblySpecimenSampleDiscoveryResponse) +def get_specimen_samples_for_assembly( + *, + db: Session = Depends(get_db), + taxon_id: int, + current_user: User = Depends(get_current_active_user), +) -> Any: + """Return specimen samples and available assembly data types for a taxon.""" + organism = db.query(Organism).filter(Organism.taxon_id == taxon_id).first() + if not organism: + raise HTTPException(status_code=404, detail=f"Organism with taxon_id {taxon_id} not found") + + organism_taxon_id = _organism_taxon_id(organism) + specimen_samples = ( + db.query(Sample) + .filter(Sample.taxon_id == organism_taxon_id, Sample.kind == "specimen") + .all() + ) + + specimen_sample_options = [] + for specimen_sample in specimen_samples: + lineage_sample_ids = _get_lineage_sample_ids_for_specimen(db, specimen_sample) + experiments = db.query(Experiment).filter(Experiment.sample_id.in_(lineage_sample_ids)).all() + + specimen_sample_options.append( + { + "sample_id": specimen_sample.id, + "specimen_id": specimen_sample.specimen_id, + "sex": specimen_sample.sex, + "available_data_types": get_available_assembly_data_types(experiments), + } + ) + + return { + "taxon_id": organism_taxon_id, + "specimen_samples": specimen_sample_options, + } + + @router.get("/pipeline-inputs") def get_pipeline_inputs( *, diff --git a/app/schemas/assembly.py b/app/schemas/assembly.py index 85962b1..f982dbb 100644 --- a/app/schemas/assembly.py +++ b/app/schemas/assembly.py @@ -19,6 +19,14 @@ class AssemblyDataTypes(str, Enum): PACBIO_SMRT_OXFORD_NANOPORE_HIC = "PACBIO_SMRT_OXFORD_NANOPORE_HIC" +class AssemblySpecimenSampleDataType(str, Enum): + """Atomic data types exposed by specimen-sample discovery.""" + + PACBIO_SMRT = "PACBIO_SMRT" + OXFORD_NANOPORE = "OXFORD_NANOPORE" + HIC = "Hi-C" + + class AssemblyFileType(str, Enum): """Enum for assembly file types.""" @@ -89,6 +97,22 @@ class AssemblyIntentResponse(BaseModel): manifest_json: Dict[str, Any] +class AssemblySpecimenSampleOption(BaseModel): + """Assembly discovery metadata for a specimen sample.""" + + sample_id: UUID + specimen_id: Optional[str] = None + sex: Optional[str] = None + available_data_types: List[AssemblySpecimenSampleDataType] = Field(default_factory=list) + + +class AssemblySpecimenSampleDiscoveryResponse(BaseModel): + """Response schema for assembly specimen-sample discovery.""" + + taxon_id: int + specimen_samples: List[AssemblySpecimenSampleOption] + + class AssemblyIntentCancel(BaseModel): """Schema for cancelling an existing assembly intent.""" diff --git a/app/services/assembly_helper.py b/app/services/assembly_helper.py index 9983b4e..352e133 100644 --- a/app/services/assembly_helper.py +++ b/app/services/assembly_helper.py @@ -1,7 +1,7 @@ """Helper functions for assembly operations.""" import logging -from typing import Any, Dict, List, Optional, Set +from typing import Any, Dict, List, Optional, Set, Tuple from uuid import UUID from app.models.experiment import Experiment @@ -12,17 +12,8 @@ logger = logging.getLogger(__name__) -def determine_assembly_data_types(experiments: List[Experiment]) -> AssemblyDataTypes: - """Determine assembly data_types based on experiments. - - Rules: - - PACBIO_SMRT exists if: platform == "PACBIO_SMRT" - - OXFORD_NANOPORE exists if: platform == "OXFORD_NANOPORE" - - Hi-C exists if: platform == "ILLUMINA" AND library_strategy == "Hi-C" - - Raises: - ValueError: If no valid sequencing platforms are detected - """ +def _detect_assembly_data_type_flags(experiments: List[Experiment]) -> Tuple[bool, bool, bool]: + """Return booleans for PacBio, ONT, and Hi-C using assembly classification rules.""" has_pacbio = False has_nanopore = False has_hic = False @@ -38,6 +29,42 @@ def determine_assembly_data_types(experiments: List[Experiment]) -> AssemblyData if platform == "ILLUMINA" and library_strategy == "HI-C": has_hic = True + return has_pacbio, has_nanopore, has_hic + + +def get_available_assembly_data_types(experiments: List[Experiment]) -> List[str]: + """Return atomic data types available for a specimen sample. + + This is used by discovery flows, so a sample with only Hi-C data should still + report ["Hi-C"] even though that combination is not sufficient for creating an + assembly intent on its own. + """ + has_pacbio, has_nanopore, has_hic = _detect_assembly_data_type_flags(experiments) + available_data_types: List[str] = [] + + if has_pacbio: + available_data_types.append("PACBIO_SMRT") + if has_nanopore: + available_data_types.append("OXFORD_NANOPORE") + if has_hic: + available_data_types.append("Hi-C") + + return available_data_types + + +def determine_assembly_data_types(experiments: List[Experiment]) -> AssemblyDataTypes: + """Determine assembly data_types based on experiments. + + Rules: + - PACBIO_SMRT exists if: platform == "PACBIO_SMRT" + - OXFORD_NANOPORE exists if: platform == "OXFORD_NANOPORE" + - Hi-C exists if: platform == "ILLUMINA" AND library_strategy == "Hi-C" + + Raises: + ValueError: If no valid sequencing platforms are detected + """ + has_pacbio, has_nanopore, has_hic = _detect_assembly_data_type_flags(experiments) + if has_pacbio and has_nanopore and has_hic: return AssemblyDataTypes.PACBIO_SMRT_OXFORD_NANOPORE_HIC elif has_pacbio and has_nanopore: diff --git a/tests/unit/endpoints/test_endpoints_assemblies.py b/tests/unit/endpoints/test_endpoints_assemblies.py index 1a2bfdb..d7ef451 100644 --- a/tests/unit/endpoints/test_endpoints_assemblies.py +++ b/tests/unit/endpoints/test_endpoints_assemblies.py @@ -349,6 +349,160 @@ def test_create_assembly_intent_requires_specimen_sample_id(): assert resp.status_code == 422 +def test_get_specimen_samples_for_assembly_returns_discovery_options(): + client = TestClient(app) + + organism = SimpleNamespace(scientific_name="Test Species", taxon_id=172942) + specimen_a_id = uuid4() + derived_a_id = uuid4() + specimen_b_id = uuid4() + specimen_c_id = uuid4() + + specimen_a = SimpleNamespace( + id=specimen_a_id, + taxon_id=172942, + kind="specimen", + specimen_id="SPEC-001", + sex="female", + ) + derived_a = SimpleNamespace( + id=derived_a_id, + taxon_id=172942, + kind="derived", + derived_from_sample_id=specimen_a_id, + ) + specimen_b = SimpleNamespace( + id=specimen_b_id, + taxon_id=172942, + kind="specimen", + specimen_id=None, + sex="male", + ) + specimen_c = SimpleNamespace( + id=specimen_c_id, + taxon_id=172942, + kind="specimen", + specimen_id="SPEC-003", + sex="unknown", + ) + + specimen_a_experiments = [ + SimpleNamespace( + id="exp-pb", + sample_id=derived_a_id, + platform="PACBIO_SMRT", + library_strategy="WGS", + ), + SimpleNamespace( + id="exp-hic", + sample_id=specimen_a_id, + platform="ILLUMINA", + library_strategy="Hi-C", + ), + ] + specimen_b_experiments = [ + SimpleNamespace( + id="exp-ont", + sample_id=specimen_b_id, + platform="OXFORD_NANOPORE", + library_strategy="WGA", + ) + ] + + class _Q: + def __init__(self, value): + self.value = value + + def filter(self, *_a, **_k): + return self + + def all(self): + return self.value if isinstance(self.value, list) else [] + + def first(self): + return self.value if not isinstance(self.value, list) else None + + class _DB: + def __init__(self): + self.calls = 0 + + def query(self, _model): + self.calls += 1 + if self.calls == 1: + return _Q(organism) + if self.calls == 2: + return _Q([specimen_a, specimen_b, specimen_c]) + if self.calls == 3: + return _Q([derived_a]) + if self.calls == 4: + return _Q(specimen_a_experiments) + if self.calls == 5: + return _Q([]) + if self.calls == 6: + return _Q(specimen_b_experiments) + if self.calls == 7: + return _Q([]) + if self.calls == 8: + return _Q([]) + return _Q([]) + + app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["admin"], is_superuser=False + ) + app.dependency_overrides[assemblies.get_db] = _override_db(_DB()) + + resp = client.get("/api/v1/assemblies/specimen-samples/172942") + + assert resp.status_code == 200 + body = resp.json() + assert body["taxon_id"] == 172942 + assert body["specimen_samples"] == [ + { + "sample_id": str(specimen_a_id), + "specimen_id": "SPEC-001", + "sex": "female", + "available_data_types": ["PACBIO_SMRT", "Hi-C"], + }, + { + "sample_id": str(specimen_b_id), + "specimen_id": None, + "sex": "male", + "available_data_types": ["OXFORD_NANOPORE"], + }, + { + "sample_id": str(specimen_c_id), + "specimen_id": "SPEC-003", + "sex": "unknown", + "available_data_types": [], + }, + ] + + +def test_get_specimen_samples_for_assembly_returns_404_for_unknown_taxon(): + client = TestClient(app) + + class _Q: + def filter(self, *_a, **_k): + return self + + def first(self): + return None + + class _DB: + def query(self, _model): + return _Q() + + app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["admin"], is_superuser=False + ) + app.dependency_overrides[assemblies.get_db] = _override_db(_DB()) + + resp = client.get("/api/v1/assemblies/specimen-samples/999999") + + assert resp.status_code == 404 + assert "not found" in resp.json()["error"]["message"] + + def test_create_assembly_intent_rejects_non_specimen_sample(): client = TestClient(app) diff --git a/tests/unit/services/test_assembly_helper.py b/tests/unit/services/test_assembly_helper.py index 2500e4c..b529252 100644 --- a/tests/unit/services/test_assembly_helper.py +++ b/tests/unit/services/test_assembly_helper.py @@ -12,6 +12,7 @@ from app.services.assembly_helper import ( determine_assembly_data_types, generate_assembly_manifest_json, + get_available_assembly_data_types, get_detected_platforms, ) @@ -107,6 +108,30 @@ def test_none_platform_ignored(self): assert determine_assembly_data_types(experiments) == AssemblyDataTypes.PACBIO_SMRT +class TestGetAvailableAssemblyDataTypes: + """Tests for discovery-oriented assembly data type detection.""" + + def test_hic_only_is_reported_for_discovery(self): + experiments = [Mock(platform="ILLUMINA", library_strategy="Hi-C")] + assert get_available_assembly_data_types(experiments) == ["Hi-C"] + + def test_returns_atomic_data_types_in_stable_order(self): + experiments = [ + Mock(platform="ILLUMINA", library_strategy="Hi-C"), + Mock(platform="OXFORD_NANOPORE", library_strategy="WGS"), + Mock(platform="PACBIO_SMRT", library_strategy="WGA"), + ] + assert get_available_assembly_data_types(experiments) == [ + "PACBIO_SMRT", + "OXFORD_NANOPORE", + "Hi-C", + ] + + def test_ignores_non_assembly_illumina_wgs(self): + experiments = [Mock(platform="ILLUMINA", library_strategy="WGS")] + assert get_available_assembly_data_types(experiments) == [] + + class TestGetDetectedPlatforms: """Tests for get_detected_platforms function.""" From 70389afeead2c6c044f5c1680882c4a1fd1b8333 Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Fri, 8 May 2026 18:50:01 +1000 Subject: [PATCH 07/11] feat: get all samples, experiments and reads for specimen_id --- app/api/v1/endpoints/samples.py | 63 ++++ app/schemas/experiment.py | 31 ++ app/schemas/read.py | 13 + app/schemas/sample.py | 27 +- .../unit/endpoints/test_endpoints_samples.py | 291 ++++++++++++++++++ 5 files changed, 423 insertions(+), 2 deletions(-) diff --git a/app/api/v1/endpoints/samples.py b/app/api/v1/endpoints/samples.py index b4230f1..20b2cc7 100644 --- a/app/api/v1/endpoints/samples.py +++ b/app/api/v1/endpoints/samples.py @@ -6,6 +6,7 @@ from uuid import UUID from fastapi import APIRouter, Depends, HTTPException, Query, status +from sqlalchemy import or_ from sqlalchemy.orm import Session from sqlalchemy.orm.attributes import flag_modified @@ -15,6 +16,7 @@ from app.models.experiment import Experiment from app.models.organism import Organism from app.models.project import Project +from app.models.read import Read from app.models.sample import Sample, SampleSubmission from app.models.user import User from app.schemas.bulk_import import BulkImportResponse, BulkSampleImport @@ -22,6 +24,7 @@ from app.schemas.sample import Sample as SampleSchema from app.schemas.sample import ( SampleCreate, + SpecimenSampleHierarchyResponse, SampleSubmissionCreate, SampleSubmissionUpdate, SampleUpdate, @@ -113,6 +116,66 @@ def get_specimen_by_taxid_and_specimen_id( return sample +@router.get( + "/by-specimen/{taxon_id}/{specimen_id}/related", + response_model=SpecimenSampleHierarchyResponse, +) +def get_samples_experiments_and_reads_for_specimen( + *, + db: Session = Depends(get_db), + taxon_id: int, + specimen_id: str, + current_user: User = Depends(get_current_active_user), +) -> Any: + """Return the specimen sample, related samples, and nested experiments/reads.""" + organism = db.query(Organism).filter(Organism.taxon_id == taxon_id).first() + if not organism: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Organism with taxon_id {taxon_id} not found", + ) + + specimen_sample = ( + db.query(Sample) + .filter( + Sample.taxon_id == organism.taxon_id, + Sample.specimen_id == specimen_id, + Sample.kind == SampleKind.SPECIMEN, + ) + .first() + ) + if not specimen_sample: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Specimen sample not found for taxon_id {taxon_id} and specimen_id '{specimen_id}'", + ) + + related_samples = ( + db.query(Sample) + .filter( + Sample.taxon_id == organism.taxon_id, + or_(Sample.id == specimen_sample.id, Sample.derived_from_sample_id == specimen_sample.id), + ) + .all() + ) + + related_payload = [] + for sample in related_samples: + experiments = db.query(Experiment).filter(Experiment.sample_id == sample.id).all() + experiment_payload = [] + for experiment in experiments: + reads = db.query(Read).filter(Read.experiment_id == experiment.id).all() + experiment_payload.append({"experiment": experiment, "reads": reads}) + + related_payload.append({"sample": sample, "experiments": experiment_payload}) + + return { + "taxon_id": organism.taxon_id, + "specimen_id": specimen_id, + "samples": related_payload, + } + + @router.post("/", response_model=SampleSchema) @policy("samples:create") def create_sample( diff --git a/app/schemas/experiment.py b/app/schemas/experiment.py index 1fb4e7d..c147583 100644 --- a/app/schemas/experiment.py +++ b/app/schemas/experiment.py @@ -103,6 +103,37 @@ class Experiment(ExperimentInDBBase): pass +class ExperimentDetail(ExperimentInDBBase): + """Detailed experiment schema used by nested aggregate endpoints.""" + + project_id: UUID + bioplatforms_base_url: Optional[str] = None + design_description: Optional[str] = None + bpa_library_id: Optional[str] = None + library_strategy: Optional[str] = None + library_source: Optional[str] = None + insert_size: Optional[str] = None + library_construction_protocol: Optional[str] = None + library_selection: Optional[str] = None + library_layout: Optional[str] = None + instrument_model: Optional[str] = None + platform: Optional[str] = None + material_extracted_by: Optional[str] = None + library_prepared_by: Optional[str] = None + sequencing_kit: Optional[str] = None + flowcell_type: Optional[str] = None + base_caller_model: Optional[str] = None + data_owner: Optional[str] = None + project_collaborators: Optional[str] = None + extraction_method: Optional[str] = None + nucleic_acid_treatment: Optional[str] = None + extraction_protocol_doi: Optional[str] = None + nucleic_acid_conc: Optional[str] = None + nucleic_acid_volume: Optional[str] = None + gal: Optional[str] = None + raw_data_release_date: Optional[str] = None + + # Base ExperimentSubmission schema class ExperimentSubmissionBase(BaseModel): """Base ExperimentSubmission schema with common attributes.""" diff --git a/app/schemas/read.py b/app/schemas/read.py index 27d3b19..417ea80 100644 --- a/app/schemas/read.py +++ b/app/schemas/read.py @@ -69,6 +69,19 @@ class Read(ReadInDBBase): pass +class ReadDetail(ReadInDBBase): + """Detailed read schema used by nested aggregate endpoints.""" + + bpa_dataset_id: Optional[str] = None + file_name: Optional[str] = None + file_checksum: Optional[str] = None + file_format: Optional[str] = None + optional_file: bool + bioplatforms_url: Optional[str] = None + read_number: Optional[str] = None + lane_number: Optional[str] = None + + # Base ReadSubmission schema class ReadSubmissionBase(BaseModel): """Base ReadSubmission schema with common attributes.""" diff --git a/app/schemas/sample.py b/app/schemas/sample.py index fb014c9..af41b40 100644 --- a/app/schemas/sample.py +++ b/app/schemas/sample.py @@ -1,12 +1,13 @@ from datetime import datetime -from enum import Enum -from typing import Dict, Optional +from typing import Dict, List, Optional from uuid import UUID from pydantic import BaseModel, ConfigDict, Field # Enum for submission status from app.schemas.common import SampleKind, SubmissionStatus +from app.schemas.experiment import ExperimentDetail as ExperimentDetailSchema +from app.schemas.read import ReadDetail as ReadDetailSchema # Base Sample schema (aligns with schema.sql columns except id/timestamps/bpa_json) @@ -91,6 +92,28 @@ class Sample(SampleInDBBase): pass +class SampleExperimentReads(BaseModel): + """Nested experiment and read data for a sample.""" + + experiment: ExperimentDetailSchema + reads: List[ReadDetailSchema] = Field(default_factory=list) + + +class SpecimenSampleHierarchyItem(BaseModel): + """A related sample and its nested experiments and reads.""" + + sample: Sample + experiments: List[SampleExperimentReads] = Field(default_factory=list) + + +class SpecimenSampleHierarchyResponse(BaseModel): + """Response shape for all sample/experiment/read data linked to a specimen.""" + + taxon_id: int + specimen_id: str + samples: List[SpecimenSampleHierarchyItem] = Field(default_factory=list) + + # Base SampleSubmission schema class SampleSubmissionBase(BaseModel): """Base SampleSubmission schema with common attributes.""" diff --git a/tests/unit/endpoints/test_endpoints_samples.py b/tests/unit/endpoints/test_endpoints_samples.py index 2805fb6..45e1fbe 100644 --- a/tests/unit/endpoints/test_endpoints_samples.py +++ b/tests/unit/endpoints/test_endpoints_samples.py @@ -34,3 +34,294 @@ def test_sample_not_found(): resp = client.get(f"/api/v1/samples/{uuid.uuid4()}") assert resp.status_code == 404 + + +def test_get_samples_experiments_and_reads_for_specimen(): + client = TestClient(app) + + taxon_id = 172942 + specimen_id = "SPEC-001" + specimen_sample_id = uuid.uuid4() + derived_sample_id = uuid.uuid4() + experiment_1_id = uuid.uuid4() + experiment_2_id = uuid.uuid4() + read_1_id = uuid.uuid4() + read_2_id = uuid.uuid4() + + organism = SimpleNamespace(taxon_id=taxon_id, scientific_name="Test species") + specimen_sample = SimpleNamespace( + id=specimen_sample_id, + taxon_id=taxon_id, + specimen_id=specimen_id, + kind="specimen", + bpa_sample_id=None, + specimen_id_description=None, + identified_by=None, + specimen_custodian=None, + sample_custodian=None, + lifestage="adult", + sex="female", + organism_part="whole organism", + region_and_locality="region", + state_or_region=None, + country_or_sea="Australia", + indigenous_location=None, + latitude=None, + longitude=None, + elevation=None, + depth=None, + habitat="forest", + collection_method=None, + collection_date=None, + collected_by="collector", + collecting_institution="institution", + collection_permit=None, + data_context=None, + bioplatforms_project_id=None, + title=None, + sample_same_as=None, + sample_derived_from=None, + specimen_voucher=None, + tolid=None, + preservation_method=None, + preservation_temperature=None, + project_name=None, + biosample_accession=None, + derived_from_sample_id=None, + extensions=None, + created_at="2026-01-01T00:00:00Z", + updated_at="2026-01-01T00:00:00Z", + ) + derived_sample = SimpleNamespace( + id=derived_sample_id, + taxon_id=taxon_id, + specimen_id=specimen_id, + kind="derived", + bpa_sample_id="BPA-001", + specimen_id_description=None, + identified_by=None, + specimen_custodian=None, + sample_custodian=None, + lifestage="adult", + sex="female", + organism_part="tissue", + region_and_locality="region", + state_or_region=None, + country_or_sea="Australia", + indigenous_location=None, + latitude=None, + longitude=None, + elevation=None, + depth=None, + habitat="forest", + collection_method=None, + collection_date=None, + collected_by="collector", + collecting_institution="institution", + collection_permit=None, + data_context=None, + bioplatforms_project_id=None, + title=None, + sample_same_as=None, + sample_derived_from=None, + specimen_voucher=None, + tolid=None, + preservation_method=None, + preservation_temperature=None, + project_name=None, + biosample_accession=None, + derived_from_sample_id=specimen_sample_id, + extensions=None, + created_at="2026-01-01T00:00:00Z", + updated_at="2026-01-01T00:00:00Z", + ) + experiment_1 = SimpleNamespace( + id=experiment_1_id, + sample_id=specimen_sample_id, + project_id=uuid.uuid4(), + bpa_package_id="PKG-001", + bioplatforms_base_url="https://example.com/pkg-001", + design_description="long-read library", + bpa_library_id="LIB-001", + library_strategy="WGS", + library_source="GENOMIC", + insert_size=None, + library_construction_protocol="protocol", + library_selection="size selected", + library_layout="SINGLE", + instrument_model="Sequel II", + platform="PACBIO_SMRT", + material_extracted_by="lab-a", + library_prepared_by="lab-b", + sequencing_kit="kit-a", + flowcell_type="flowcell-a", + base_caller_model=None, + data_owner="owner-a", + project_collaborators="collab-a", + extraction_method="method-a", + nucleic_acid_treatment=None, + extraction_protocol_doi=None, + nucleic_acid_conc="10", + nucleic_acid_volume="5", + gal=None, + raw_data_release_date="2026-01-10", + created_at="2026-01-01T00:00:00Z", + updated_at="2026-01-01T00:00:00Z", + ) + experiment_2 = SimpleNamespace( + id=experiment_2_id, + sample_id=derived_sample_id, + project_id=uuid.uuid4(), + bpa_package_id="PKG-002", + bioplatforms_base_url="https://example.com/pkg-002", + design_description="derived library", + bpa_library_id="LIB-002", + library_strategy="Hi-C", + library_source="GENOMIC", + insert_size=None, + library_construction_protocol="protocol-2", + library_selection="selection-2", + library_layout="PAIRED", + instrument_model="NovaSeq", + platform="ILLUMINA", + material_extracted_by="lab-c", + library_prepared_by="lab-d", + sequencing_kit="kit-b", + flowcell_type="flowcell-b", + base_caller_model=None, + data_owner="owner-b", + project_collaborators="collab-b", + extraction_method="method-b", + nucleic_acid_treatment=None, + extraction_protocol_doi=None, + nucleic_acid_conc="20", + nucleic_acid_volume="10", + gal=None, + raw_data_release_date="2026-02-10", + created_at="2026-01-01T00:00:00Z", + updated_at="2026-01-01T00:00:00Z", + ) + read_1 = SimpleNamespace( + id=read_1_id, + experiment_id=experiment_1_id, + bpa_resource_id="RES-001", + bpa_dataset_id="DATASET-001", + file_name="reads1.ccs.bam", + file_checksum="md5-1", + file_format="bam", + optional_file=False, + bioplatforms_url="https://example.com/reads1.ccs.bam", + read_number=None, + lane_number=None, + created_at="2026-01-01T00:00:00Z", + updated_at="2026-01-01T00:00:00Z", + ) + read_2 = SimpleNamespace( + id=read_2_id, + experiment_id=experiment_2_id, + bpa_resource_id="RES-002", + bpa_dataset_id="DATASET-002", + file_name="reads2_R1.fastq.gz", + file_checksum="md5-2", + file_format="fastq.gz", + optional_file=True, + bioplatforms_url="https://example.com/reads2_R1.fastq.gz", + read_number="1", + lane_number="L001", + created_at="2026-01-01T00:00:00Z", + updated_at="2026-01-01T00:00:00Z", + ) + + class _Q: + def __init__(self, value): + self.value = value + + def filter(self, *_a, **_k): + return self + + def first(self): + return self.value if not isinstance(self.value, list) else None + + def all(self): + return self.value if isinstance(self.value, list) else [] + + class _DB: + def __init__(self): + self.calls = 0 + + def query(self, _model): + self.calls += 1 + if self.calls == 1: + return _Q(organism) + if self.calls == 2: + return _Q(specimen_sample) + if self.calls == 3: + return _Q([specimen_sample, derived_sample]) + if self.calls == 4: + return _Q([experiment_1]) + if self.calls == 5: + return _Q([read_1]) + if self.calls == 6: + return _Q([experiment_2]) + if self.calls == 7: + return _Q([read_2]) + return _Q([]) + + app.dependency_overrides[samples.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["admin"], is_superuser=False + ) + app.dependency_overrides[samples.get_db] = _override_db(_DB()) + + resp = client.get(f"/api/v1/samples/by-specimen/{taxon_id}/{specimen_id}/related") + + assert resp.status_code == 200 + body = resp.json() + assert body["taxon_id"] == taxon_id + assert body["specimen_id"] == specimen_id + assert len(body["samples"]) == 2 + assert body["samples"][0]["sample"]["id"] == str(specimen_sample_id) + assert body["samples"][0]["experiments"][0]["experiment"]["id"] == str(experiment_1_id) + assert body["samples"][0]["experiments"][0]["experiment"]["platform"] == "PACBIO_SMRT" + assert body["samples"][0]["experiments"][0]["reads"][0]["id"] == str(read_1_id) + assert body["samples"][0]["experiments"][0]["reads"][0]["file_name"] == "reads1.ccs.bam" + assert body["samples"][1]["sample"]["id"] == str(derived_sample_id) + assert body["samples"][1]["experiments"][0]["experiment"]["id"] == str(experiment_2_id) + assert body["samples"][1]["experiments"][0]["experiment"]["library_layout"] == "PAIRED" + assert body["samples"][1]["experiments"][0]["reads"][0]["id"] == str(read_2_id) + assert body["samples"][1]["experiments"][0]["reads"][0]["lane_number"] == "L001" + + +def test_get_samples_experiments_and_reads_for_specimen_not_found(): + client = TestClient(app) + + taxon_id = 172942 + specimen_id = "SPEC-404" + organism = SimpleNamespace(taxon_id=taxon_id, scientific_name="Test species") + + class _Q: + def __init__(self, value): + self.value = value + + def filter(self, *_a, **_k): + return self + + def first(self): + return self.value + + class _DB: + def __init__(self): + self.calls = 0 + + def query(self, _model): + self.calls += 1 + if self.calls == 1: + return _Q(organism) + return _Q(None) + + app.dependency_overrides[samples.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["admin"], is_superuser=False + ) + app.dependency_overrides[samples.get_db] = _override_db(_DB()) + + resp = client.get(f"/api/v1/samples/by-specimen/{taxon_id}/{specimen_id}/related") + assert resp.status_code == 404 From a219de1358c5148e6f1d2ff271202eaad27c3a85 Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Mon, 11 May 2026 11:31:10 +1000 Subject: [PATCH 08/11] feat: get all samples, experiments and reads for specimen_id --- app/api/v1/endpoints/samples.py | 86 ++++----- .../unit/endpoints/test_endpoints_samples.py | 181 ++++++++++++++++++ 2 files changed, 222 insertions(+), 45 deletions(-) diff --git a/app/api/v1/endpoints/samples.py b/app/api/v1/endpoints/samples.py index 20b2cc7..73fbf2b 100644 --- a/app/api/v1/endpoints/samples.py +++ b/app/api/v1/endpoints/samples.py @@ -72,52 +72,8 @@ def read_samples( return samples -@router.get("/by-specimen/{taxon_id}/{specimen_id}", response_model=SampleSchema) -def get_specimen_by_taxid_and_specimen_id( - *, - db: Session = Depends(get_db), - taxon_id: int, - specimen_id: str, - current_user: User = Depends(get_current_active_user), -) -> Any: - """ - Lookup a specimen sample by taxon_id and specimen_id. - - This finds the unique specimen sample for a given organism (by taxon_id) - and specimen_id combination. - """ - # All users can read samples - - # First, find the organism by taxon_id - organism = db.query(Organism).filter(Organism.taxon_id == taxon_id).first() - if not organism: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Organism with taxon_id {taxon_id} not found", - ) - - # Then find the specimen sample - sample = ( - db.query(Sample) - .filter( - Sample.taxon_id == organism.taxon_id, - Sample.specimen_id == specimen_id, - Sample.kind == SampleKind.SPECIMEN, - ) - .first() - ) - - if not sample: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Specimen sample not found for taxon_id {taxon_id} and specimen_id '{specimen_id}'", - ) - - return sample - - @router.get( - "/by-specimen/{taxon_id}/{specimen_id}/related", + "/by-specimen/{taxon_id}/{specimen_id:path}/related", response_model=SpecimenSampleHierarchyResponse, ) def get_samples_experiments_and_reads_for_specimen( @@ -176,6 +132,46 @@ def get_samples_experiments_and_reads_for_specimen( } +@router.get("/by-specimen/{taxon_id}/{specimen_id:path}", response_model=SampleSchema) +def get_specimen_by_taxid_and_specimen_id( + *, + db: Session = Depends(get_db), + taxon_id: int, + specimen_id: str, + current_user: User = Depends(get_current_active_user), +) -> Any: + """ + Lookup a specimen sample by taxon_id and specimen_id. + + This finds the unique specimen sample for a given organism (by taxon_id) + and specimen_id combination. + """ + organism = db.query(Organism).filter(Organism.taxon_id == taxon_id).first() + if not organism: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Organism with taxon_id {taxon_id} not found", + ) + + sample = ( + db.query(Sample) + .filter( + Sample.taxon_id == organism.taxon_id, + Sample.specimen_id == specimen_id, + Sample.kind == SampleKind.SPECIMEN, + ) + .first() + ) + + if not sample: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Specimen sample not found for taxon_id {taxon_id} and specimen_id '{specimen_id}'", + ) + + return sample + + @router.post("/", response_model=SampleSchema) @policy("samples:create") def create_sample( diff --git a/tests/unit/endpoints/test_endpoints_samples.py b/tests/unit/endpoints/test_endpoints_samples.py index 45e1fbe..c5db9cf 100644 --- a/tests/unit/endpoints/test_endpoints_samples.py +++ b/tests/unit/endpoints/test_endpoints_samples.py @@ -1,5 +1,6 @@ import uuid from types import SimpleNamespace +from urllib.parse import quote from fastapi.testclient import TestClient @@ -325,3 +326,183 @@ def query(self, _model): resp = client.get(f"/api/v1/samples/by-specimen/{taxon_id}/{specimen_id}/related") assert resp.status_code == 404 + + +def test_get_specimen_by_taxid_and_specimen_id_accepts_encoded_path_value(): + client = TestClient(app) + + taxon_id = 172942 + specimen_id = "SPEC / 001" + encoded_specimen_id = quote(specimen_id, safe="") + specimen_sample_id = uuid.uuid4() + + organism = SimpleNamespace(taxon_id=taxon_id, scientific_name="Test species") + specimen_sample = SimpleNamespace( + id=specimen_sample_id, + taxon_id=taxon_id, + specimen_id=specimen_id, + kind="specimen", + bpa_sample_id=None, + specimen_id_description=None, + identified_by=None, + specimen_custodian=None, + sample_custodian=None, + lifestage="adult", + sex="female", + organism_part="whole organism", + region_and_locality="region", + state_or_region=None, + country_or_sea="Australia", + indigenous_location=None, + latitude=None, + longitude=None, + elevation=None, + depth=None, + habitat="forest", + collection_method=None, + collection_date=None, + collected_by="collector", + collecting_institution="institution", + collection_permit=None, + data_context=None, + bioplatforms_project_id=None, + title=None, + sample_same_as=None, + sample_derived_from=None, + specimen_voucher=None, + tolid=None, + preservation_method=None, + preservation_temperature=None, + project_name=None, + biosample_accession=None, + derived_from_sample_id=None, + extensions=None, + created_at="2026-01-01T00:00:00Z", + updated_at="2026-01-01T00:00:00Z", + ) + + class _Q: + def __init__(self, value): + self.value = value + + def filter(self, *_a, **_k): + return self + + def first(self): + return self.value + + class _DB: + def __init__(self): + self.calls = 0 + + def query(self, _model): + self.calls += 1 + if self.calls == 1: + return _Q(organism) + return _Q(specimen_sample) + + app.dependency_overrides[samples.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["admin"], is_superuser=False + ) + app.dependency_overrides[samples.get_db] = _override_db(_DB()) + + resp = client.get(f"/api/v1/samples/by-specimen/{taxon_id}/{encoded_specimen_id}") + + assert resp.status_code == 200 + body = resp.json() + assert body["specimen_id"] == specimen_id + + +def test_get_samples_experiments_and_reads_for_specimen_accepts_encoded_path_value(): + client = TestClient(app) + + taxon_id = 172942 + specimen_id = "SPEC / 001" + encoded_specimen_id = quote(specimen_id, safe="") + specimen_sample_id = uuid.uuid4() + + organism = SimpleNamespace(taxon_id=taxon_id, scientific_name="Test species") + specimen_sample = SimpleNamespace( + id=specimen_sample_id, + taxon_id=taxon_id, + specimen_id=specimen_id, + kind="specimen", + bpa_sample_id=None, + specimen_id_description=None, + identified_by=None, + specimen_custodian=None, + sample_custodian=None, + lifestage="adult", + sex="female", + organism_part="whole organism", + region_and_locality="region", + state_or_region=None, + country_or_sea="Australia", + indigenous_location=None, + latitude=None, + longitude=None, + elevation=None, + depth=None, + habitat="forest", + collection_method=None, + collection_date=None, + collected_by="collector", + collecting_institution="institution", + collection_permit=None, + data_context=None, + bioplatforms_project_id=None, + title=None, + sample_same_as=None, + sample_derived_from=None, + specimen_voucher=None, + tolid=None, + preservation_method=None, + preservation_temperature=None, + project_name=None, + biosample_accession=None, + derived_from_sample_id=None, + extensions=None, + created_at="2026-01-01T00:00:00Z", + updated_at="2026-01-01T00:00:00Z", + ) + + class _Q: + def __init__(self, value): + self.value = value + + def filter(self, *_a, **_k): + return self + + def first(self): + return self.value if not isinstance(self.value, list) else None + + def all(self): + return self.value if isinstance(self.value, list) else [] + + class _DB: + def __init__(self): + self.calls = 0 + + def query(self, _model): + self.calls += 1 + if self.calls == 1: + return _Q(organism) + if self.calls == 2: + return _Q(specimen_sample) + if self.calls == 3: + return _Q([specimen_sample]) + if self.calls == 4: + return _Q([]) + return _Q([]) + + app.dependency_overrides[samples.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["admin"], is_superuser=False + ) + app.dependency_overrides[samples.get_db] = _override_db(_DB()) + + resp = client.get(f"/api/v1/samples/by-specimen/{taxon_id}/{encoded_specimen_id}/related") + + assert resp.status_code == 200 + body = resp.json() + assert body["specimen_id"] == specimen_id + assert body["samples"][0]["sample"]["specimen_id"] == specimen_id From 471c214d33616651ba1bbf5f7467a137b3250b45 Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Mon, 11 May 2026 14:18:30 +1000 Subject: [PATCH 09/11] feat: add rna-seq data type (illumina only) --- app/schemas/assembly.py | 1 + app/services/assembly_helper.py | 15 +++++++--- .../endpoints/test_endpoints_assemblies.py | 28 ++++++++++++++++++- tests/unit/services/test_assembly_helper.py | 4 +++ 4 files changed, 43 insertions(+), 5 deletions(-) diff --git a/app/schemas/assembly.py b/app/schemas/assembly.py index f982dbb..a64ef5c 100644 --- a/app/schemas/assembly.py +++ b/app/schemas/assembly.py @@ -25,6 +25,7 @@ class AssemblySpecimenSampleDataType(str, Enum): PACBIO_SMRT = "PACBIO_SMRT" OXFORD_NANOPORE = "OXFORD_NANOPORE" HIC = "Hi-C" + RNASEQ = "RNA-Seq" class AssemblyFileType(str, Enum): diff --git a/app/services/assembly_helper.py b/app/services/assembly_helper.py index 352e133..ce76943 100644 --- a/app/services/assembly_helper.py +++ b/app/services/assembly_helper.py @@ -12,11 +12,14 @@ logger = logging.getLogger(__name__) -def _detect_assembly_data_type_flags(experiments: List[Experiment]) -> Tuple[bool, bool, bool]: +def _detect_assembly_data_type_flags( + experiments: List[Experiment], +) -> Tuple[bool, bool, bool, bool]: """Return booleans for PacBio, ONT, and Hi-C using assembly classification rules.""" has_pacbio = False has_nanopore = False has_hic = False + has_rnaseq = False for exp in experiments: platform = exp.platform.upper() if exp.platform else "" @@ -28,8 +31,10 @@ def _detect_assembly_data_type_flags(experiments: List[Experiment]) -> Tuple[boo has_nanopore = True if platform == "ILLUMINA" and library_strategy == "HI-C": has_hic = True + if platform == "ILLUMINA" and library_strategy == "RNA-SEQ": + has_rnaseq = True - return has_pacbio, has_nanopore, has_hic + return has_pacbio, has_nanopore, has_hic, has_rnaseq def get_available_assembly_data_types(experiments: List[Experiment]) -> List[str]: @@ -39,7 +44,7 @@ def get_available_assembly_data_types(experiments: List[Experiment]) -> List[str report ["Hi-C"] even though that combination is not sufficient for creating an assembly intent on its own. """ - has_pacbio, has_nanopore, has_hic = _detect_assembly_data_type_flags(experiments) + has_pacbio, has_nanopore, has_hic, has_rnaseq = _detect_assembly_data_type_flags(experiments) available_data_types: List[str] = [] if has_pacbio: @@ -48,6 +53,8 @@ def get_available_assembly_data_types(experiments: List[Experiment]) -> List[str available_data_types.append("OXFORD_NANOPORE") if has_hic: available_data_types.append("Hi-C") + if has_rnaseq: + available_data_types.append("RNA-Seq") return available_data_types @@ -63,7 +70,7 @@ def determine_assembly_data_types(experiments: List[Experiment]) -> AssemblyData Raises: ValueError: If no valid sequencing platforms are detected """ - has_pacbio, has_nanopore, has_hic = _detect_assembly_data_type_flags(experiments) + has_pacbio, has_nanopore, has_hic, has_rnaseq = _detect_assembly_data_type_flags(experiments) if has_pacbio and has_nanopore and has_hic: return AssemblyDataTypes.PACBIO_SMRT_OXFORD_NANOPORE_HIC diff --git a/tests/unit/endpoints/test_endpoints_assemblies.py b/tests/unit/endpoints/test_endpoints_assemblies.py index d7ef451..62a2923 100644 --- a/tests/unit/endpoints/test_endpoints_assemblies.py +++ b/tests/unit/endpoints/test_endpoints_assemblies.py @@ -357,6 +357,7 @@ def test_get_specimen_samples_for_assembly_returns_discovery_options(): derived_a_id = uuid4() specimen_b_id = uuid4() specimen_c_id = uuid4() + specimen_d_id = uuid4() specimen_a = SimpleNamespace( id=specimen_a_id, @@ -385,6 +386,13 @@ def test_get_specimen_samples_for_assembly_returns_discovery_options(): specimen_id="SPEC-003", sex="unknown", ) + specimen_d = SimpleNamespace( + id=specimen_d_id, + taxon_id=172942, + kind="specimen", + specimen_id="SPEC-004", + sex="female", + ) specimen_a_experiments = [ SimpleNamespace( @@ -408,6 +416,14 @@ def test_get_specimen_samples_for_assembly_returns_discovery_options(): library_strategy="WGA", ) ] + specimen_d_experiments = [ + SimpleNamespace( + id="exp-rna", + sample_id=specimen_d_id, + platform="ILLUMINA", + library_strategy="RNA-Seq", + ) + ] class _Q: def __init__(self, value): @@ -431,7 +447,7 @@ def query(self, _model): if self.calls == 1: return _Q(organism) if self.calls == 2: - return _Q([specimen_a, specimen_b, specimen_c]) + return _Q([specimen_a, specimen_b, specimen_c, specimen_d]) if self.calls == 3: return _Q([derived_a]) if self.calls == 4: @@ -444,6 +460,10 @@ def query(self, _model): return _Q([]) if self.calls == 8: return _Q([]) + if self.calls == 9: + return _Q([]) + if self.calls == 10: + return _Q(specimen_d_experiments) return _Q([]) app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( @@ -475,6 +495,12 @@ def query(self, _model): "sex": "unknown", "available_data_types": [], }, + { + "sample_id": str(specimen_d_id), + "specimen_id": "SPEC-004", + "sex": "female", + "available_data_types": ["RNA-Seq"], + }, ] diff --git a/tests/unit/services/test_assembly_helper.py b/tests/unit/services/test_assembly_helper.py index b529252..4ef56bc 100644 --- a/tests/unit/services/test_assembly_helper.py +++ b/tests/unit/services/test_assembly_helper.py @@ -131,6 +131,10 @@ def test_ignores_non_assembly_illumina_wgs(self): experiments = [Mock(platform="ILLUMINA", library_strategy="WGS")] assert get_available_assembly_data_types(experiments) == [] + def test_rnaseq_is_reported_for_discovery(self): + experiments = [Mock(platform="ILLUMINA", library_strategy="RNA-Seq")] + assert get_available_assembly_data_types(experiments) == ["RNA-Seq"] + class TestGetDetectedPlatforms: """Tests for get_detected_platforms function.""" From aaf45183298615a68069b40006487644380b1fbb Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Mon, 11 May 2026 18:28:30 +1000 Subject: [PATCH 10/11] style: linting --- alembic/versions/0010_add_taxonomy_info.py | 5 ++--- .../versions/0011_assembly_first_and_stages.py | 7 +++++-- .../versions/0012_assembly_manifest_columns.py | 3 ++- app/api/v1/api.py | 4 +--- app/api/v1/endpoints/assemblies.py | 16 ++++++++++++---- app/api/v1/endpoints/samples.py | 6 ++++-- app/models/__init__.py | 2 +- app/models/assembly.py | 8 ++++---- app/services/assembly_service.py | 8 ++++---- app/services/taxonomy_info_service.py | 8 ++------ .../endpoints/test_endpoints_taxonomy_info.py | 8 ++++---- tests/unit/services/test_assembly_service.py | 4 +--- 12 files changed, 42 insertions(+), 37 deletions(-) diff --git a/alembic/versions/0010_add_taxonomy_info.py b/alembic/versions/0010_add_taxonomy_info.py index fc629f0..88bd44b 100644 --- a/alembic/versions/0010_add_taxonomy_info.py +++ b/alembic/versions/0010_add_taxonomy_info.py @@ -6,6 +6,7 @@ """ import sqlalchemy as sa + from alembic import op revision = "0010_add_taxonomy_info" @@ -30,9 +31,7 @@ def upgrade() -> None: sa.Column("augustus_dataset_name", sa.Text(), nullable=True), sa.Column("genetic_code_id", sa.Integer(), nullable=True), sa.PrimaryKeyConstraint("taxon_id"), - sa.ForeignKeyConstraint( - ["taxon_id"], ["organism.taxon_id"], ondelete="CASCADE" - ), + sa.ForeignKeyConstraint(["taxon_id"], ["organism.taxon_id"], ondelete="CASCADE"), ) # 2. Migrate existing augustus_dataset_name values from organism into taxonomy_info diff --git a/alembic/versions/0011_assembly_first_and_stages.py b/alembic/versions/0011_assembly_first_and_stages.py index 822090c..d3fee53 100644 --- a/alembic/versions/0011_assembly_first_and_stages.py +++ b/alembic/versions/0011_assembly_first_and_stages.py @@ -6,9 +6,10 @@ """ import sqlalchemy as sa -from alembic import op from sqlalchemy.dialects.postgresql import JSONB, UUID +from alembic import op + revision = "0011_assembly_first_and_stages" down_revision = "0010_add_taxonomy_info" branch_labels = None @@ -100,7 +101,9 @@ def upgrade() -> None: "status IN ('running', 'succeeded', 'failed', 'cancelled')", name="ck_assembly_stage_run_status", ), - sa.UniqueConstraint("assembly_id", "stage_name", "attempt", name="uq_stage_run_assembly_stage_attempt"), + sa.UniqueConstraint( + "assembly_id", "stage_name", "attempt", name="uq_stage_run_assembly_stage_attempt" + ), ) op.create_index("ix_assembly_stage_run_assembly_id", "assembly_stage_run", ["assembly_id"]) diff --git a/alembic/versions/0012_assembly_manifest_columns.py b/alembic/versions/0012_assembly_manifest_columns.py index b6771d0..89b1e7e 100644 --- a/alembic/versions/0012_assembly_manifest_columns.py +++ b/alembic/versions/0012_assembly_manifest_columns.py @@ -13,9 +13,10 @@ """ import sqlalchemy as sa -from alembic import op from sqlalchemy.dialects.postgresql import JSONB, UUID +from alembic import op + revision = "0012_assembly_manifest_columns" down_revision = "0011_assembly_first_and_stages" branch_labels = None diff --git a/app/api/v1/api.py b/app/api/v1/api.py index 60377c7..3d46215 100644 --- a/app/api/v1/api.py +++ b/app/api/v1/api.py @@ -47,9 +47,7 @@ api_router.include_router(reads.router, prefix="/reads", tags=["reads"]) api_router.include_router(qc_reads.router, prefix="/qc-reads", tags=["qc-reads"]) api_router.include_router(genome_notes.router, prefix="/genome-notes", tags=["genome-notes"]) -api_router.include_router( - taxonomy_info.router, prefix="/taxonomy-info", tags=["taxonomy-info"] -) +api_router.include_router(taxonomy_info.router, prefix="/taxonomy-info", tags=["taxonomy-info"]) # XML export endpoints api_router.include_router(xml_export.router, prefix="/xml-export", tags=["xml-export"]) diff --git a/app/api/v1/endpoints/assemblies.py b/app/api/v1/endpoints/assemblies.py index 6e6d28b..ece21d4 100644 --- a/app/api/v1/endpoints/assemblies.py +++ b/app/api/v1/endpoints/assemblies.py @@ -9,7 +9,13 @@ from app.core.errors import AppError from app.core.pagination import Pagination, apply_pagination, pagination_params from app.core.policy import policy -from app.models.assembly import Assembly, AssemblyFile, AssemblyRun, AssemblyStageRun, AssemblySubmission +from app.models.assembly import ( + Assembly, + AssemblyFile, + AssemblyRun, + AssemblyStageRun, + AssemblySubmission, +) from app.models.experiment import Experiment from app.models.organism import Organism from app.models.read import Read @@ -55,6 +61,7 @@ router = APIRouter() + # TODO remove tax_id refs and rely solely on taxon_id in organism table for all relationships and queries. def _organism_taxon_id(organism: Any) -> int: return organism.taxon_id if hasattr(organism, "taxon_id") else organism.tax_id @@ -159,7 +166,9 @@ def get_specimen_samples_for_assembly( specimen_sample_options = [] for specimen_sample in specimen_samples: lineage_sample_ids = _get_lineage_sample_ids_for_specimen(db, specimen_sample) - experiments = db.query(Experiment).filter(Experiment.sample_id.in_(lineage_sample_ids)).all() + experiments = ( + db.query(Experiment).filter(Experiment.sample_id.in_(lineage_sample_ids)).all() + ) specimen_sample_options.append( { @@ -390,8 +399,7 @@ def create_assembly_intent( .all() ) hic_experiments = [ - e for e in hic_experiments - if (e.library_strategy or "").upper() == "HI-C" + e for e in hic_experiments if (e.library_strategy or "").upper() == "HI-C" ] if not hic_experiments: raise AppError( diff --git a/app/api/v1/endpoints/samples.py b/app/api/v1/endpoints/samples.py index 73fbf2b..fc9c185 100644 --- a/app/api/v1/endpoints/samples.py +++ b/app/api/v1/endpoints/samples.py @@ -24,10 +24,10 @@ from app.schemas.sample import Sample as SampleSchema from app.schemas.sample import ( SampleCreate, - SpecimenSampleHierarchyResponse, SampleSubmissionCreate, SampleSubmissionUpdate, SampleUpdate, + SpecimenSampleHierarchyResponse, ) from app.schemas.sample import SampleSubmission as SampleSubmissionSchema from app.utils.mapping import to_float @@ -110,7 +110,9 @@ def get_samples_experiments_and_reads_for_specimen( db.query(Sample) .filter( Sample.taxon_id == organism.taxon_id, - or_(Sample.id == specimen_sample.id, Sample.derived_from_sample_id == specimen_sample.id), + or_( + Sample.id == specimen_sample.id, Sample.derived_from_sample_id == specimen_sample.id + ), ) .all() ) diff --git a/app/models/__init__.py b/app/models/__init__.py index 558d4ad..5b16a88 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -5,10 +5,10 @@ from app.models.experiment import Experiment, ExperimentSubmission from app.models.genome_note import GenomeNote from app.models.organism import Organism -from app.models.taxonomy_info import TaxonomyInfo from app.models.project import Project from app.models.qc_read import QcRead, QcReadFile, QcReadSubmission from app.models.read import Read from app.models.sample import Sample, SampleSubmission +from app.models.taxonomy_info import TaxonomyInfo from app.models.token import RefreshToken from app.models.user import User diff --git a/app/models/assembly.py b/app/models/assembly.py index 98349c8..df14a95 100644 --- a/app/models/assembly.py +++ b/app/models/assembly.py @@ -79,9 +79,7 @@ class Assembly(Base): # Relationships organism = relationship("Organism", backref="assemblies") sample = relationship("Sample", foreign_keys=[sample_id], backref="assemblies") - long_read_specimen_sample = relationship( - "Sample", foreign_keys=[long_read_specimen_sample_id] - ) + long_read_specimen_sample = relationship("Sample", foreign_keys=[long_read_specimen_sample_id]) hic_specimen_sample = relationship("Sample", foreign_keys=[hic_specimen_sample_id]) project = relationship("Project", backref="assemblies") @@ -291,7 +289,9 @@ class AssemblyStageRun(Base): ) __table_args__ = ( - UniqueConstraint("assembly_id", "stage_name", "attempt", name="uq_stage_run_assembly_stage_attempt"), + UniqueConstraint( + "assembly_id", "stage_name", "attempt", name="uq_stage_run_assembly_stage_attempt" + ), ) assembly = relationship("Assembly", backref=backref("stage_runs", cascade="all, delete-orphan")) diff --git a/app/services/assembly_service.py b/app/services/assembly_service.py index 37cb45a..a72ee17 100644 --- a/app/services/assembly_service.py +++ b/app/services/assembly_service.py @@ -304,12 +304,12 @@ def get_by_assembly_id(self, db: Session, assembly_id: UUID) -> List[AssemblyRea return db.query(AssemblyRead).filter(AssemblyRead.assembly_id == assembly_id).all() -class AssemblyStageRunService(BaseService[AssemblyStageRun, AssemblyStageRunCreate, AssemblyStageRunUpdate]): +class AssemblyStageRunService( + BaseService[AssemblyStageRun, AssemblyStageRunCreate, AssemblyStageRunUpdate] +): """Service for AssemblyStageRun operations.""" - def get_by_assembly_id( - self, db: Session, assembly_id: UUID - ) -> List[AssemblyStageRun]: + def get_by_assembly_id(self, db: Session, assembly_id: UUID) -> List[AssemblyStageRun]: return ( db.query(AssemblyStageRun) .filter(AssemblyStageRun.assembly_id == assembly_id) diff --git a/app/services/taxonomy_info_service.py b/app/services/taxonomy_info_service.py index bc1813a..8990109 100644 --- a/app/services/taxonomy_info_service.py +++ b/app/services/taxonomy_info_service.py @@ -53,9 +53,7 @@ def delete(self, db: Session, *, taxon_id: int) -> Optional[TaxonomyInfo]: db.commit() return ti - def bulk_import( - self, db: Session, *, data: Dict[str, Dict[str, Any]] - ) -> BulkImportResponse: + def bulk_import(self, db: Session, *, data: Dict[str, Dict[str, Any]]) -> BulkImportResponse: created_count = 0 skipped_count = 0 errors: List[str] = [] @@ -87,9 +85,7 @@ def bulk_import( continue # Reject duplicates - existing = ( - db.query(TaxonomyInfo).filter(TaxonomyInfo.taxon_id == taxon_id).first() - ) + existing = db.query(TaxonomyInfo).filter(TaxonomyInfo.taxon_id == taxon_id).first() if existing: errors.append(f"{key}: taxonomy_info for taxon_id {taxon_id} already exists") skipped_count += 1 diff --git a/tests/unit/endpoints/test_endpoints_taxonomy_info.py b/tests/unit/endpoints/test_endpoints_taxonomy_info.py index 7971f48..afef115 100644 --- a/tests/unit/endpoints/test_endpoints_taxonomy_info.py +++ b/tests/unit/endpoints/test_endpoints_taxonomy_info.py @@ -152,7 +152,9 @@ def test_create_taxonomy_info_success(monkeypatch): "taxonomy_info_service", SimpleNamespace(create=lambda db, ti_in: ti), ) - resp = client.post("/api/v1/taxonomy-info/", json={"taxon_id": 5077, "augustus_dataset_name": "anidulans"}) + resp = client.post( + "/api/v1/taxonomy-info/", json={"taxon_id": 5077, "augustus_dataset_name": "anidulans"} + ) assert resp.status_code == 201 assert resp.json()["taxon_id"] == 5077 @@ -352,9 +354,7 @@ def test_bulk_import_rejects_mismatched_inner_taxon_id(monkeypatch): "taxonomy_info_service", SimpleNamespace(bulk_import=lambda db, data: result), ) - resp = client.post( - "/api/v1/taxonomy-info/bulk-import", json={"5077": {"taxon_id": 9999}} - ) + resp = client.post("/api/v1/taxonomy-info/bulk-import", json={"5077": {"taxon_id": 9999}}) assert resp.status_code == 200 body = resp.json() assert body["skipped_count"] == 1 diff --git a/tests/unit/services/test_assembly_service.py b/tests/unit/services/test_assembly_service.py index d4cec33..7d55de0 100644 --- a/tests/unit/services/test_assembly_service.py +++ b/tests/unit/services/test_assembly_service.py @@ -248,9 +248,7 @@ def test_increments_from_existing_max(self, mock_db, assembly_service): ) assert version == 5 - def test_different_hic_sample_does_not_split_version_sequence( - self, mock_db, assembly_service - ): + def test_different_hic_sample_does_not_split_version_sequence(self, mock_db, assembly_service): """Two intents with the same long_read_specimen_sample_id but different hic_specimen_sample_id must share the same version counter (hic_specimen_sample_id is NOT part of the key).""" call_count = [0] From dbe44c0637898dfe23a821ca9014aa735e715bdf Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Mon, 11 May 2026 18:43:02 +1000 Subject: [PATCH 11/11] feat: include qc_reads in aggregate endpoint --- app/api/v1/endpoints/assemblies.py | 8 +++ app/schemas/assembly.py | 2 + .../endpoints/test_endpoints_assemblies.py | 59 ++++++++++++++++--- 3 files changed, 62 insertions(+), 7 deletions(-) diff --git a/app/api/v1/endpoints/assemblies.py b/app/api/v1/endpoints/assemblies.py index ece21d4..48faf02 100644 --- a/app/api/v1/endpoints/assemblies.py +++ b/app/api/v1/endpoints/assemblies.py @@ -18,6 +18,7 @@ ) from app.models.experiment import Experiment from app.models.organism import Organism +from app.models.qc_read import QcRead from app.models.read import Read from app.models.sample import Sample from app.models.user import User @@ -169,6 +170,12 @@ def get_specimen_samples_for_assembly( experiments = ( db.query(Experiment).filter(Experiment.sample_id.in_(lineage_sample_ids)).all() ) + experiment_ids = [experiment.id for experiment in experiments] + qc_reads = ( + db.query(QcRead).filter(QcRead.experiment_id.in_(experiment_ids)).all() + if experiment_ids + else [] + ) specimen_sample_options.append( { @@ -176,6 +183,7 @@ def get_specimen_samples_for_assembly( "specimen_id": specimen_sample.specimen_id, "sex": specimen_sample.sex, "available_data_types": get_available_assembly_data_types(experiments), + "qc_reads": qc_reads, } ) diff --git a/app/schemas/assembly.py b/app/schemas/assembly.py index a64ef5c..663f9d0 100644 --- a/app/schemas/assembly.py +++ b/app/schemas/assembly.py @@ -6,6 +6,7 @@ from pydantic import BaseModel, ConfigDict, Field from app.schemas.common import SubmissionStatus +from app.schemas.qc_read import QcReadOut class AssemblyDataTypes(str, Enum): @@ -105,6 +106,7 @@ class AssemblySpecimenSampleOption(BaseModel): specimen_id: Optional[str] = None sex: Optional[str] = None available_data_types: List[AssemblySpecimenSampleDataType] = Field(default_factory=list) + qc_reads: List[QcReadOut] = Field(default_factory=list) class AssemblySpecimenSampleDiscoveryResponse(BaseModel): diff --git a/tests/unit/endpoints/test_endpoints_assemblies.py b/tests/unit/endpoints/test_endpoints_assemblies.py index 62a2923..5c1bdee 100644 --- a/tests/unit/endpoints/test_endpoints_assemblies.py +++ b/tests/unit/endpoints/test_endpoints_assemblies.py @@ -358,6 +358,10 @@ def test_get_specimen_samples_for_assembly_returns_discovery_options(): specimen_b_id = uuid4() specimen_c_id = uuid4() specimen_d_id = uuid4() + specimen_a_pacbio_exp_id = uuid4() + specimen_a_hic_exp_id = uuid4() + specimen_b_ont_exp_id = uuid4() + specimen_d_rna_exp_id = uuid4() specimen_a = SimpleNamespace( id=specimen_a_id, @@ -396,21 +400,37 @@ def test_get_specimen_samples_for_assembly_returns_discovery_options(): specimen_a_experiments = [ SimpleNamespace( - id="exp-pb", + id=specimen_a_pacbio_exp_id, sample_id=derived_a_id, platform="PACBIO_SMRT", library_strategy="WGS", ), SimpleNamespace( - id="exp-hic", + id=specimen_a_hic_exp_id, sample_id=specimen_a_id, platform="ILLUMINA", library_strategy="Hi-C", ), ] + specimen_a_qc_reads = [ + SimpleNamespace( + id=uuid4(), + experiment_id=specimen_a_pacbio_exp_id, + base_count=1000, + read_count=100, + qc_bases_removed=10, + qc_reads_removed=2, + mean_gc_content=0.45, + n50_length=5000, + created_at="2026-01-01T00:00:00Z", + updated_at="2026-01-01T00:00:00Z", + files=[], + submission_records=[], + ) + ] specimen_b_experiments = [ SimpleNamespace( - id="exp-ont", + id=specimen_b_ont_exp_id, sample_id=specimen_b_id, platform="OXFORD_NANOPORE", library_strategy="WGA", @@ -418,7 +438,7 @@ def test_get_specimen_samples_for_assembly_returns_discovery_options(): ] specimen_d_experiments = [ SimpleNamespace( - id="exp-rna", + id=specimen_d_rna_exp_id, sample_id=specimen_d_id, platform="ILLUMINA", library_strategy="RNA-Seq", @@ -453,17 +473,23 @@ def query(self, _model): if self.calls == 4: return _Q(specimen_a_experiments) if self.calls == 5: - return _Q([]) + return _Q(specimen_a_qc_reads) if self.calls == 6: - return _Q(specimen_b_experiments) - if self.calls == 7: return _Q([]) + if self.calls == 7: + return _Q(specimen_b_experiments) if self.calls == 8: return _Q([]) if self.calls == 9: return _Q([]) if self.calls == 10: + return _Q([]) + if self.calls == 11: + return _Q([]) + if self.calls == 12: return _Q(specimen_d_experiments) + if self.calls == 13: + return _Q([]) return _Q([]) app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( @@ -482,24 +508,43 @@ def query(self, _model): "specimen_id": "SPEC-001", "sex": "female", "available_data_types": ["PACBIO_SMRT", "Hi-C"], + "qc_reads": [ + { + "id": str(specimen_a_qc_reads[0].id), + "experiment_id": str(specimen_a_pacbio_exp_id), + "base_count": 1000, + "read_count": 100, + "qc_bases_removed": 10, + "qc_reads_removed": 2, + "mean_gc_content": 0.45, + "n50_length": 5000, + "created_at": "2026-01-01T00:00:00Z", + "updated_at": "2026-01-01T00:00:00Z", + "files": [], + "submission_records": [], + } + ], }, { "sample_id": str(specimen_b_id), "specimen_id": None, "sex": "male", "available_data_types": ["OXFORD_NANOPORE"], + "qc_reads": [], }, { "sample_id": str(specimen_c_id), "specimen_id": "SPEC-003", "sex": "unknown", "available_data_types": [], + "qc_reads": [], }, { "sample_id": str(specimen_d_id), "specimen_id": "SPEC-004", "sex": "female", "available_data_types": ["RNA-Seq"], + "qc_reads": [], }, ]